projects / reactos.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
history | raw | HEAD
Sync with trunk head (r48786)
[reactos.git] / ntoskrnl / mm / ARM3 / pool.c
1 /*
2 * PROJECT: ReactOS Kernel
3 * LICENSE: BSD - See COPYING.ARM in the top level directory
4 * FILE: ntoskrnl/mm/ARM3/pool.c
5 * PURPOSE: ARM Memory Manager Pool Allocator
6 * PROGRAMMERS: ReactOS Portable Systems Group
7 */
8
9 /* INCLUDES *******************************************************************/
10
11 #include <ntoskrnl.h>
12 #define NDEBUG
13 #include <debug.h>
14
15 #line 15 "ARM³::POOL"
16 #define MODULE_INVOLVED_IN_ARM3
17 #include "../ARM3/miarm.h"
18
19 /* GLOBALS ********************************************************************/
20
21 LIST_ENTRY MmNonPagedPoolFreeListHead[MI_MAX_FREE_PAGE_LISTS];
22 PFN_NUMBER MmNumberOfFreeNonPagedPool, MiExpansionPoolPagesInitialCharge;
23 PVOID MmNonPagedPoolEnd0;
24 PFN_NUMBER MiStartOfInitialPoolFrame, MiEndOfInitialPoolFrame;
25 KGUARDED_MUTEX MmPagedPoolMutex;
26 MM_PAGED_POOL_INFO MmPagedPoolInfo;
27 SIZE_T MmAllocatedNonPagedPool;
28 ULONG MmSpecialPoolTag;
29 ULONG MmConsumedPoolPercentage;
30 BOOLEAN MmProtectFreedNonPagedPool;
31
32 /* PRIVATE FUNCTIONS **********************************************************/
33
34 VOID
35 NTAPI
36 MiProtectFreeNonPagedPool(IN PVOID VirtualAddress,
37 IN ULONG PageCount)
38 {
39 PMMPTE PointerPte, LastPte;
40 MMPTE TempPte;
41
42 /* If pool is physical, can't protect PTEs */
43 if (MI_IS_PHYSICAL_ADDRESS(VirtualAddress)) return;
44
45 /* Get PTE pointers and loop */
46 PointerPte = MiAddressToPte(VirtualAddress);
47 LastPte = PointerPte + PageCount;
48 do
49 {
50 /* Capture the PTE for safety */
51 TempPte = *PointerPte;
52
53 /* Mark it as an invalid PTE, set proto bit to recognize it as pool */
54 TempPte.u.Hard.Valid = 0;
55 TempPte.u.Soft.Prototype = 1;
56 MI_WRITE_INVALID_PTE(PointerPte, TempPte);
57 } while (++PointerPte < LastPte);
58
59 /* Flush the TLB */
60 KeFlushEntireTb(TRUE, TRUE);
61 }
62
63 BOOLEAN
64 NTAPI
65 MiUnProtectFreeNonPagedPool(IN PVOID VirtualAddress,
66 IN ULONG PageCount)
67 {
68 PMMPTE PointerPte;
69 MMPTE TempPte;
70 PFN_NUMBER UnprotectedPages = 0;
71
72 /* If pool is physical, can't protect PTEs */
73 if (MI_IS_PHYSICAL_ADDRESS(VirtualAddress)) return FALSE;
74
75 /* Get, and capture the PTE */
76 PointerPte = MiAddressToPte(VirtualAddress);
77 TempPte = *PointerPte;
78
79 /* Loop protected PTEs */
80 while ((TempPte.u.Hard.Valid == 0) && (TempPte.u.Soft.Prototype == 1))
81 {
82 /* Unprotect the PTE */
83 TempPte.u.Hard.Valid = 1;
84 TempPte.u.Soft.Prototype = 0;
85 MI_WRITE_VALID_PTE(PointerPte, TempPte);
86
87 /* One more page */
88 if (++UnprotectedPages == PageCount) break;
89
90 /* Capture next PTE */
91 TempPte = *(++PointerPte);
92 }
93
94 /* Return if any pages were unprotected */
95 return UnprotectedPages ? TRUE : FALSE;
96 }
97
98 VOID
99 FORCEINLINE
100 MiProtectedPoolUnProtectLinks(IN PLIST_ENTRY Links,
101 OUT PVOID* PoolFlink,
102 OUT PVOID* PoolBlink)
103 {
104 BOOLEAN Safe;
105 PVOID PoolVa;
106
107 /* Initialize variables */
108 *PoolFlink = *PoolBlink = NULL;
109
110 /* Check if the list has entries */
111 if (IsListEmpty(Links) == FALSE)
112 {
113 /* We are going to need to forward link to do an insert */
114 PoolVa = Links->Flink;
115
116 /* So make it safe to access */
117 Safe = MiUnProtectFreeNonPagedPool(PoolVa, 1);
118 if (Safe) PoolFlink = PoolVa;
119 }
120
121 /* Are we going to need a backward link too? */
122 if (Links != Links->Blink)
123 {
124 /* Get the head's backward link for the insert */
125 PoolVa = Links->Blink;
126
127 /* Make it safe to access */
128 Safe = MiUnProtectFreeNonPagedPool(PoolVa, 1);
129 if (Safe) PoolBlink = PoolVa;
130 }
131 }
132
133 VOID
134 FORCEINLINE
135 MiProtectedPoolProtectLinks(IN PVOID PoolFlink,
136 IN PVOID PoolBlink)
137 {
138 /* Reprotect the pages, if they got unprotected earlier */
139 if (PoolFlink) MiProtectFreeNonPagedPool(PoolFlink, 1);
140 if (PoolBlink) MiProtectFreeNonPagedPool(PoolBlink, 1);
141 }
142
143 VOID
144 NTAPI
145 MiProtectedPoolInsertList(IN PLIST_ENTRY ListHead,
146 IN PLIST_ENTRY Entry,
147 IN BOOLEAN Critical)
148 {
149 PVOID PoolFlink, PoolBlink;
150
151 /* Make the list accessible */
152 MiProtectedPoolUnProtectLinks(ListHead, &PoolFlink, &PoolBlink);
153
154 /* Now insert in the right position */
155 Critical ? InsertHeadList(ListHead, Entry) : InsertTailList(ListHead, Entry);
156
157 /* And reprotect the pages containing the free links */
158 MiProtectedPoolProtectLinks(PoolFlink, PoolBlink);
159 }
160
161 VOID
162 NTAPI
163 MiProtectedPoolRemoveEntryList(IN PLIST_ENTRY Entry)
164 {
165 PVOID PoolFlink, PoolBlink;
166
167 /* Make the list accessible */
168 MiProtectedPoolUnProtectLinks(Entry, &PoolFlink, &PoolBlink);
169
170 /* Now remove */
171 RemoveEntryList(Entry);
172
173 /* And reprotect the pages containing the free links */
174 if (PoolFlink) MiProtectFreeNonPagedPool(PoolFlink, 1);
175 if (PoolBlink) MiProtectFreeNonPagedPool(PoolBlink, 1);
176 }
177
178 VOID
179 NTAPI
180 MiInitializeNonPagedPoolThresholds(VOID)
181 {
182 PFN_NUMBER Size = MmMaximumNonPagedPoolInPages;
183
184 /* Default low threshold of 8MB or one third of nonpaged pool */
185 MiLowNonPagedPoolThreshold = (8 * _1MB) >> PAGE_SHIFT;
186 MiLowNonPagedPoolThreshold = min(MiLowNonPagedPoolThreshold, Size / 3);
187
188 /* Default high threshold of 20MB or 50% */
189 MiHighNonPagedPoolThreshold = (20 * _1MB) >> PAGE_SHIFT;
190 MiHighNonPagedPoolThreshold = min(MiHighNonPagedPoolThreshold, Size / 2);
191 ASSERT(MiLowNonPagedPoolThreshold < MiHighNonPagedPoolThreshold);
192 }
193
194 VOID
195 NTAPI
196 MiInitializePoolEvents(VOID)
197 {
198 KIRQL OldIrql;
199 PFN_NUMBER FreePoolInPages;
200
201 /* Lock paged pool */
202 KeAcquireGuardedMutex(&MmPagedPoolMutex);
203
204 /* Total size of the paged pool minus the allocated size, is free */
205 FreePoolInPages = MmSizeOfPagedPoolInPages - MmPagedPoolInfo.AllocatedPagedPool;
206
207 /* Check the initial state high state */
208 if (FreePoolInPages >= MiHighPagedPoolThreshold)
209 {
210 /* We have plenty of pool */
211 KeSetEvent(MiHighPagedPoolEvent, 0, FALSE);
212 }
213 else
214 {
215 /* We don't */
216 KeClearEvent(MiHighPagedPoolEvent);
217 }
218
219 /* Check the initial low state */
220 if (FreePoolInPages <= MiLowPagedPoolThreshold)
221 {
222 /* We're very low in free pool memory */
223 KeSetEvent(MiLowPagedPoolEvent, 0, FALSE);
224 }
225 else
226 {
227 /* We're not */
228 KeClearEvent(MiLowPagedPoolEvent);
229 }
230
231 /* Release the paged pool lock */
232 KeReleaseGuardedMutex(&MmPagedPoolMutex);
233
234 /* Now it's time for the nonpaged pool lock */
235 OldIrql = KeAcquireQueuedSpinLock(LockQueueMmNonPagedPoolLock);
236
237 /* Free pages are the maximum minus what's been allocated */
238 FreePoolInPages = MmMaximumNonPagedPoolInPages - MmAllocatedNonPagedPool;
239
240 /* Check if we have plenty */
241 if (FreePoolInPages >= MiHighNonPagedPoolThreshold)
242 {
243 /* We do, set the event */
244 KeSetEvent(MiHighNonPagedPoolEvent, 0, FALSE);
245 }
246 else
247 {
248 /* We don't, clear the event */
249 KeClearEvent(MiHighNonPagedPoolEvent);
250 }
251
252 /* Check if we have very little */
253 if (FreePoolInPages <= MiLowNonPagedPoolThreshold)
254 {
255 /* We do, set the event */
256 KeSetEvent(MiLowNonPagedPoolEvent, 0, FALSE);
257 }
258 else
259 {
260 /* We don't, clear it */
261 KeClearEvent(MiLowNonPagedPoolEvent);
262 }
263
264 /* We're done, release the nonpaged pool lock */
265 KeReleaseQueuedSpinLock(LockQueueMmNonPagedPoolLock, OldIrql);
266 }
267
268 VOID
269 NTAPI
270 MiInitializeNonPagedPool(VOID)
271 {
272 ULONG i;
273 PFN_NUMBER PoolPages;
274 PMMFREE_POOL_ENTRY FreeEntry, FirstEntry;
275 PMMPTE PointerPte;
276 PAGED_CODE();
277
278 //
279 // We keep 4 lists of free pages (4 lists help avoid contention)
280 //
281 for (i = 0; i < MI_MAX_FREE_PAGE_LISTS; i++)
282 {
283 //
284 // Initialize each of them
285 //
286 InitializeListHead(&MmNonPagedPoolFreeListHead[i]);
287 }
288
289 //
290 // Calculate how many pages the initial nonpaged pool has
291 //
292 PoolPages = BYTES_TO_PAGES(MmSizeOfNonPagedPoolInBytes);
293 MmNumberOfFreeNonPagedPool = PoolPages;
294
295 //
296 // Initialize the first free entry
297 //
298 FreeEntry = MmNonPagedPoolStart;
299 FirstEntry = FreeEntry;
300 FreeEntry->Size = PoolPages;
301 FreeEntry->Signature = MM_FREE_POOL_SIGNATURE;
302 FreeEntry->Owner = FirstEntry;
303
304 //
305 // Insert it into the last list
306 //
307 InsertHeadList(&MmNonPagedPoolFreeListHead[MI_MAX_FREE_PAGE_LISTS - 1],
308 &FreeEntry->List);
309
310 //
311 // Now create free entries for every single other page
312 //
313 while (PoolPages-- > 1)
314 {
315 //
316 // Link them all back to the original entry
317 //
318 FreeEntry = (PMMFREE_POOL_ENTRY)((ULONG_PTR)FreeEntry + PAGE_SIZE);
319 FreeEntry->Owner = FirstEntry;
320 FreeEntry->Signature = MM_FREE_POOL_SIGNATURE;
321 }
322
323 //
324 // Validate and remember first allocated pool page
325 //
326 PointerPte = MiAddressToPte(MmNonPagedPoolStart);
327 ASSERT(PointerPte->u.Hard.Valid == 1);
328 MiStartOfInitialPoolFrame = PFN_FROM_PTE(PointerPte);
329
330 //
331 // Keep track of where initial nonpaged pool ends
332 //
333 MmNonPagedPoolEnd0 = (PVOID)((ULONG_PTR)MmNonPagedPoolStart +
334 MmSizeOfNonPagedPoolInBytes);
335
336 //
337 // Validate and remember last allocated pool page
338 //
339 PointerPte = MiAddressToPte((PVOID)((ULONG_PTR)MmNonPagedPoolEnd0 - 1));
340 ASSERT(PointerPte->u.Hard.Valid == 1);
341 MiEndOfInitialPoolFrame = PFN_FROM_PTE(PointerPte);
342
343 //
344 // Validate the first nonpaged pool expansion page (which is a guard page)
345 //
346 PointerPte = MiAddressToPte(MmNonPagedPoolExpansionStart);
347 ASSERT(PointerPte->u.Hard.Valid == 0);
348
349 //
350 // Calculate the size of the expansion region alone
351 //
352 MiExpansionPoolPagesInitialCharge =
353 BYTES_TO_PAGES(MmMaximumNonPagedPoolInBytes - MmSizeOfNonPagedPoolInBytes);
354
355 //
356 // Remove 2 pages, since there's a guard page on top and on the bottom
357 //
358 MiExpansionPoolPagesInitialCharge -= 2;
359
360 //
361 // Now initialize the nonpaged pool expansion PTE space. Remember there's a
362 // guard page on top so make sure to skip it. The bottom guard page will be
363 // guaranteed by the fact our size is off by one.
364 //
365 MiInitializeSystemPtes(PointerPte + 1,
366 MiExpansionPoolPagesInitialCharge,
367 NonPagedPoolExpansion);
368 }
369
370 PVOID
371 NTAPI
372 MiAllocatePoolPages(IN POOL_TYPE PoolType,
373 IN SIZE_T SizeInBytes)
374 {
375 PFN_NUMBER SizeInPages, PageFrameNumber;
376 ULONG i;
377 KIRQL OldIrql;
378 PLIST_ENTRY NextEntry, NextHead, LastHead;
379 PMMPTE PointerPte, StartPte;
380 MMPTE TempPte;
381 PMMPFN Pfn1;
382 PVOID BaseVa, BaseVaStart;
383 PMMFREE_POOL_ENTRY FreeEntry;
384 PKSPIN_LOCK_QUEUE LockQueue;
385
386 //
387 // Figure out how big the allocation is in pages
388 //
389 SizeInPages = BYTES_TO_PAGES(SizeInBytes);
390
391 //
392 // Handle paged pool
393 //
394 if ((PoolType & BASE_POOL_TYPE_MASK) == PagedPool)
395 {
396 //
397 // Lock the paged pool mutex
398 //
399 KeAcquireGuardedMutex(&MmPagedPoolMutex);
400
401 //
402 // Find some empty allocation space
403 //
404 i = RtlFindClearBitsAndSet(MmPagedPoolInfo.PagedPoolAllocationMap,
405 SizeInPages,
406 MmPagedPoolInfo.PagedPoolHint);
407 if (i == 0xFFFFFFFF)
408 {
409 //
410 // Get the page bit count
411 //
412 i = ((SizeInPages - 1) / 1024) + 1;
413 DPRINT1("Paged pool expansion: %d %x\n", i, SizeInPages);
414
415 //
416 // Check if there is enougn paged pool expansion space left
417 //
418 if (MmPagedPoolInfo.NextPdeForPagedPoolExpansion >
419 MiAddressToPte(MmPagedPoolInfo.LastPteForPagedPool))
420 {
421 //
422 // Out of memory!
423 //
424 DPRINT1("OUT OF PAGED POOL!!!\n");
425 KeReleaseGuardedMutex(&MmPagedPoolMutex);
426 return NULL;
427 }
428
429 //
430 // Check if we'll have to expand past the last PTE we have available
431 //
432 if (((i - 1) + MmPagedPoolInfo.NextPdeForPagedPoolExpansion) >
433 MiAddressToPte(MmPagedPoolInfo.LastPteForPagedPool))
434 {
435 //
436 // We can only support this much then
437 //
438 SizeInPages = MiAddressToPte(MmPagedPoolInfo.LastPteForPagedPool) -
439 MmPagedPoolInfo.NextPdeForPagedPoolExpansion +
440 1;
441 ASSERT(SizeInPages < i);
442 i = SizeInPages;
443 }
444 else
445 {
446 //
447 // Otherwise, there is plenty of space left for this expansion
448 //
449 SizeInPages = i;
450 }
451
452 //
453 // Get the template PTE we'll use to expand
454 //
455 TempPte = ValidKernelPte;
456
457 //
458 // Get the first PTE in expansion space
459 //
460 PointerPte = MmPagedPoolInfo.NextPdeForPagedPoolExpansion;
461 BaseVa = MiPteToAddress(PointerPte);
462 BaseVaStart = BaseVa;
463
464 //
465 // Lock the PFN database and loop pages
466 //
467 OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock);
468 do
469 {
470 //
471 // It should not already be valid
472 //
473 ASSERT(PointerPte->u.Hard.Valid == 0);
474
475 /* Request a page */
476 PageFrameNumber = MiRemoveAnyPage(0);
477 TempPte.u.Hard.PageFrameNumber = PageFrameNumber;
478
479 #if (_MI_PAGING_LEVELS >= 3)
480 /* On PAE/x64 systems, there's no double-buffering */
481 ASSERT(FALSE);
482 #else
483 //
484 // Save it into our double-buffered system page directory
485 //
486 /* This seems to be making the assumption that one PDE is one page long */
487 C_ASSERT(PAGE_SIZE == (PD_COUNT * (sizeof(MMPTE) * PDE_COUNT)));
488 MmSystemPagePtes[(ULONG_PTR)PointerPte & (PAGE_SIZE - 1) /
489 sizeof(MMPTE)] = TempPte;
490
491 /* Initialize the PFN */
492 MiInitializePfnForOtherProcess(PageFrameNumber,
493 PointerPte,
494 MmSystemPageDirectory[(PointerPte - (PMMPTE)PDE_BASE) / PDE_COUNT]);
495
496 /* Write the actual PTE now */
497 MI_WRITE_VALID_PTE(PointerPte++, TempPte);
498 #endif
499 //
500 // Move on to the next expansion address
501 //
502 BaseVa = (PVOID)((ULONG_PTR)BaseVa + PAGE_SIZE);
503 } while (--i > 0);
504
505 //
506 // Release the PFN database lock
507 //
508 KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql);
509
510 //
511 // These pages are now available, clear their availablity bits
512 //
513 RtlClearBits(MmPagedPoolInfo.PagedPoolAllocationMap,
514 (MmPagedPoolInfo.NextPdeForPagedPoolExpansion -
515 MiAddressToPte(MmPagedPoolInfo.FirstPteForPagedPool)) *
516 1024,
517 SizeInPages * 1024);
518
519 //
520 // Update the next expansion location
521 //
522 MmPagedPoolInfo.NextPdeForPagedPoolExpansion += SizeInPages;
523
524 //
525 // Zero out the newly available memory
526 //
527 RtlZeroMemory(BaseVaStart, SizeInPages * PAGE_SIZE);
528
529 //
530 // Now try consuming the pages again
531 //
532 SizeInPages = BYTES_TO_PAGES(SizeInBytes);
533 i = RtlFindClearBitsAndSet(MmPagedPoolInfo.PagedPoolAllocationMap,
534 SizeInPages,
535 0);
536 if (i == 0xFFFFFFFF)
537 {
538 //
539 // Out of memory!
540 //
541 DPRINT1("OUT OF PAGED POOL!!!\n");
542 KeReleaseGuardedMutex(&MmPagedPoolMutex);
543 return NULL;
544 }
545 }
546
547 //
548 // Update the pool hint if the request was just one page
549 //
550 if (SizeInPages == 1) MmPagedPoolInfo.PagedPoolHint = i + 1;
551
552 //
553 // Update the end bitmap so we know the bounds of this allocation when
554 // the time comes to free it
555 //
556 RtlSetBit(MmPagedPoolInfo.EndOfPagedPoolBitmap, i + SizeInPages - 1);
557
558 //
559 // Now we can release the lock (it mainly protects the bitmap)
560 //
561 KeReleaseGuardedMutex(&MmPagedPoolMutex);
562
563 //
564 // Now figure out where this allocation starts
565 //
566 BaseVa = (PVOID)((ULONG_PTR)MmPagedPoolStart + (i << PAGE_SHIFT));
567
568 //
569 // Flush the TLB
570 //
571 KeFlushEntireTb(TRUE, TRUE);
572
573 /* Setup a demand-zero writable PTE */
574 MI_MAKE_SOFTWARE_PTE(&TempPte, MM_READWRITE);
575
576 //
577 // Find the first and last PTE, then loop them all
578 //
579 PointerPte = MiAddressToPte(BaseVa);
580 StartPte = PointerPte + SizeInPages;
581 do
582 {
583 //
584 // Write the demand zero PTE and keep going
585 //
586 ASSERT(PointerPte->u.Hard.Valid == 0);
587 *PointerPte++ = TempPte;
588 } while (PointerPte < StartPte);
589
590 //
591 // Return the allocation address to the caller
592 //
593 return BaseVa;
594 }
595
596 //
597 // Allocations of less than 4 pages go into their individual buckets
598 //
599 i = SizeInPages - 1;
600 if (i >= MI_MAX_FREE_PAGE_LISTS) i = MI_MAX_FREE_PAGE_LISTS - 1;
601
602 //
603 // Loop through all the free page lists based on the page index
604 //
605 NextHead = &MmNonPagedPoolFreeListHead[i];
606 LastHead = &MmNonPagedPoolFreeListHead[MI_MAX_FREE_PAGE_LISTS];
607
608 //
609 // Acquire the nonpaged pool lock
610 //
611 OldIrql = KeAcquireQueuedSpinLock(LockQueueMmNonPagedPoolLock);
612 do
613 {
614 //
615 // Now loop through all the free page entries in this given list
616 //
617 NextEntry = NextHead->Flink;
618 while (NextEntry != NextHead)
619 {
620 /* Is freed non paged pool enabled */
621 if (MmProtectFreedNonPagedPool)
622 {
623 /* We need to be able to touch this page, unprotect it */
624 MiUnProtectFreeNonPagedPool(NextEntry, 0);
625 }
626
627 //
628 // Grab the entry and see if it can handle our allocation
629 //
630 FreeEntry = CONTAINING_RECORD(NextEntry, MMFREE_POOL_ENTRY, List);
631 ASSERT(FreeEntry->Signature == MM_FREE_POOL_SIGNATURE);
632 if (FreeEntry->Size >= SizeInPages)
633 {
634 //
635 // It does, so consume the pages from here
636 //
637 FreeEntry->Size -= SizeInPages;
638
639 //
640 // The allocation will begin in this free page area
641 //
642 BaseVa = (PVOID)((ULONG_PTR)FreeEntry +
643 (FreeEntry->Size << PAGE_SHIFT));
644
645 /* Remove the item from the list, depending if pool is protected */
646 MmProtectFreedNonPagedPool ?
647 MiProtectedPoolRemoveEntryList(&FreeEntry->List) :
648 RemoveEntryList(&FreeEntry->List);
649
650 //
651 // However, check if its' still got space left
652 //
653 if (FreeEntry->Size != 0)
654 {
655 /* Check which list to insert this entry into */
656 i = FreeEntry->Size - 1;
657 if (i >= MI_MAX_FREE_PAGE_LISTS) i = MI_MAX_FREE_PAGE_LISTS - 1;
658
659 /* Insert the entry into the free list head, check for prot. pool */
660 MmProtectFreedNonPagedPool ?
661 MiProtectedPoolInsertList(&MmNonPagedPoolFreeListHead[i], &FreeEntry->List, TRUE) :
662 InsertTailList(&MmNonPagedPoolFreeListHead[i], &FreeEntry->List);
663
664 /* Is freed non paged pool protected? */
665 if (MmProtectFreedNonPagedPool)
666 {
667 /* Protect the freed pool! */
668 MiProtectFreeNonPagedPool(FreeEntry, FreeEntry->Size);
669 }
670 }
671
672 //
673 // Grab the PTE for this allocation
674 //
675 PointerPte = MiAddressToPte(BaseVa);
676 ASSERT(PointerPte->u.Hard.Valid == 1);
677
678 //
679 // Grab the PFN NextEntry and index
680 //
681 Pfn1 = MiGetPfnEntry(PFN_FROM_PTE(PointerPte));
682
683 //
684 // Now mark it as the beginning of an allocation
685 //
686 ASSERT(Pfn1->u3.e1.StartOfAllocation == 0);
687 Pfn1->u3.e1.StartOfAllocation = 1;
688
689 /* Mark it as special pool if needed */
690 ASSERT(Pfn1->u4.VerifierAllocation == 0);
691 if (PoolType & 64) Pfn1->u4.VerifierAllocation = 1;
692
693 //
694 // Check if the allocation is larger than one page
695 //
696 if (SizeInPages != 1)
697 {
698 //
699 // Navigate to the last PFN entry and PTE
700 //
701 PointerPte += SizeInPages - 1;
702 ASSERT(PointerPte->u.Hard.Valid == 1);
703 Pfn1 = MiGetPfnEntry(PointerPte->u.Hard.PageFrameNumber);
704 }
705
706 //
707 // Mark this PFN as the last (might be the same as the first)
708 //
709 ASSERT(Pfn1->u3.e1.EndOfAllocation == 0);
710 Pfn1->u3.e1.EndOfAllocation = 1;
711
712 //
713 // Release the nonpaged pool lock, and return the allocation
714 //
715 KeReleaseQueuedSpinLock(LockQueueMmNonPagedPoolLock, OldIrql);
716 return BaseVa;
717 }
718
719 //
720 // Try the next free page entry
721 //
722 NextEntry = FreeEntry->List.Flink;
723
724 /* Is freed non paged pool protected? */
725 if (MmProtectFreedNonPagedPool)
726 {
727 /* Protect the freed pool! */
728 MiProtectFreeNonPagedPool(FreeEntry, FreeEntry->Size);
729 }
730 }
731 } while (++NextHead < LastHead);
732
733 //
734 // If we got here, we're out of space.
735 // Start by releasing the lock
736 //
737 KeReleaseQueuedSpinLock(LockQueueMmNonPagedPoolLock, OldIrql);
738
739 //
740 // Allocate some system PTEs
741 //
742 StartPte = MiReserveSystemPtes(SizeInPages, NonPagedPoolExpansion);
743 PointerPte = StartPte;
744 if (StartPte == NULL)
745 {
746 //
747 // Ran out of memory
748 //
749 DPRINT1("Out of NP Expansion Pool\n");
750 return NULL;
751 }
752
753 //
754 // Acquire the pool lock now
755 //
756 OldIrql = KeAcquireQueuedSpinLock(LockQueueMmNonPagedPoolLock);
757
758 //
759 // Lock the PFN database too
760 //
761 LockQueue = &KeGetCurrentPrcb()->LockQueue[LockQueuePfnLock];
762 KeAcquireQueuedSpinLockAtDpcLevel(LockQueue);
763
764 //
765 // Loop the pages
766 //
767 TempPte = ValidKernelPte;
768 do
769 {
770 /* Allocate a page */
771 PageFrameNumber = MiRemoveAnyPage(0);
772
773 /* Get the PFN entry for it and fill it out */
774 Pfn1 = MiGetPfnEntry(PageFrameNumber);
775 Pfn1->u3.e2.ReferenceCount = 1;
776 Pfn1->u2.ShareCount = 1;
777 Pfn1->PteAddress = PointerPte;
778 Pfn1->u3.e1.PageLocation = ActiveAndValid;
779 Pfn1->u4.VerifierAllocation = 0;
780
781 /* Write the PTE for it */
782 TempPte.u.Hard.PageFrameNumber = PageFrameNumber;
783 MI_WRITE_VALID_PTE(PointerPte++, TempPte);
784 } while (--SizeInPages > 0);
785
786 //
787 // This is the last page
788 //
789 Pfn1->u3.e1.EndOfAllocation = 1;
790
791 //
792 // Get the first page and mark it as such
793 //
794 Pfn1 = MiGetPfnEntry(StartPte->u.Hard.PageFrameNumber);
795 Pfn1->u3.e1.StartOfAllocation = 1;
796
797 /* Mark it as a verifier allocation if needed */
798 ASSERT(Pfn1->u4.VerifierAllocation == 0);
799 if (PoolType & 64) Pfn1->u4.VerifierAllocation = 1;
800
801 //
802 // Release the PFN and nonpaged pool lock
803 //
804 KeReleaseQueuedSpinLockFromDpcLevel(LockQueue);
805 KeReleaseQueuedSpinLock(LockQueueMmNonPagedPoolLock, OldIrql);
806
807 //
808 // Return the address
809 //
810 return MiPteToAddress(StartPte);
811 }
812
813 ULONG
814 NTAPI
815 MiFreePoolPages(IN PVOID StartingVa)
816 {
817 PMMPTE PointerPte, StartPte;
818 PMMPFN Pfn1, StartPfn;
819 PFN_NUMBER FreePages, NumberOfPages;
820 KIRQL OldIrql;
821 PMMFREE_POOL_ENTRY FreeEntry, NextEntry, LastEntry;
822 ULONG i, End;
823
824 //
825 // Handle paged pool
826 //
827 if ((StartingVa >= MmPagedPoolStart) && (StartingVa <= MmPagedPoolEnd))
828 {
829 //
830 // Calculate the offset from the beginning of paged pool, and convert it
831 // into pages
832 //
833 i = ((ULONG_PTR)StartingVa - (ULONG_PTR)MmPagedPoolStart) >> PAGE_SHIFT;
834 End = i;
835
836 //
837 // Now use the end bitmap to scan until we find a set bit, meaning that
838 // this allocation finishes here
839 //
840 while (!RtlTestBit(MmPagedPoolInfo.EndOfPagedPoolBitmap, End)) End++;
841
842 //
843 // Now calculate the total number of pages this allocation spans
844 //
845 NumberOfPages = End - i + 1;
846
847 /* Delete the actual pages */
848 PointerPte = MmPagedPoolInfo.FirstPteForPagedPool + i;
849 FreePages = MiDeleteSystemPageableVm(PointerPte, NumberOfPages, 0, NULL);
850 ASSERT(FreePages == NumberOfPages);
851
852 //
853 // Acquire the paged pool lock
854 //
855 KeAcquireGuardedMutex(&MmPagedPoolMutex);
856
857 //
858 // Clear the allocation and free bits
859 //
860 RtlClearBit(MmPagedPoolInfo.EndOfPagedPoolBitmap, i);
861 RtlClearBits(MmPagedPoolInfo.PagedPoolAllocationMap, i, NumberOfPages);
862
863 //
864 // Update the hint if we need to
865 //
866 if (i < MmPagedPoolInfo.PagedPoolHint) MmPagedPoolInfo.PagedPoolHint = i;
867
868 //
869 // Release the lock protecting the bitmaps
870 //
871 KeReleaseGuardedMutex(&MmPagedPoolMutex);
872
873 //
874 // And finally return the number of pages freed
875 //
876 return NumberOfPages;
877 }
878
879 //
880 // Get the first PTE and its corresponding PFN entry
881 //
882 StartPte = PointerPte = MiAddressToPte(StartingVa);
883 StartPfn = Pfn1 = MiGetPfnEntry(PointerPte->u.Hard.PageFrameNumber);
884
885 //
886 // Loop until we find the last PTE
887 //
888 while (Pfn1->u3.e1.EndOfAllocation == 0)
889 {
890 //
891 // Keep going
892 //
893 PointerPte++;
894 Pfn1 = MiGetPfnEntry(PointerPte->u.Hard.PageFrameNumber);
895 }
896
897 //
898 // Now we know how many pages we have
899 //
900 NumberOfPages = PointerPte - StartPte + 1;
901
902 //
903 // Acquire the nonpaged pool lock
904 //
905 OldIrql = KeAcquireQueuedSpinLock(LockQueueMmNonPagedPoolLock);
906
907 //
908 // Mark the first and last PTEs as not part of an allocation anymore
909 //
910 StartPfn->u3.e1.StartOfAllocation = 0;
911 Pfn1->u3.e1.EndOfAllocation = 0;
912
913 //
914 // Assume we will free as many pages as the allocation was
915 //
916 FreePages = NumberOfPages;
917
918 //
919 // Peek one page past the end of the allocation
920 //
921 PointerPte++;
922
923 //
924 // Guard against going past initial nonpaged pool
925 //
926 if (MiGetPfnEntryIndex(Pfn1) == MiEndOfInitialPoolFrame)
927 {
928 //
929 // This page is on the outskirts of initial nonpaged pool, so ignore it
930 //
931 Pfn1 = NULL;
932 }
933 else
934 {
935 /* Sanity check */
936 ASSERT((ULONG_PTR)StartingVa + NumberOfPages <= (ULONG_PTR)MmNonPagedPoolEnd);
937
938 /* Check if protected pool is enabled */
939 if (MmProtectFreedNonPagedPool)
940 {
941 /* The freed block will be merged, it must be made accessible */
942 MiUnProtectFreeNonPagedPool(MiPteToAddress(PointerPte), 0);
943 }
944
945 //
946 // Otherwise, our entire allocation must've fit within the initial non
947 // paged pool, or the expansion nonpaged pool, so get the PFN entry of
948 // the next allocation
949 //
950 if (PointerPte->u.Hard.Valid == 1)
951 {
952 //
953 // It's either expansion or initial: get the PFN entry
954 //
955 Pfn1 = MiGetPfnEntry(PointerPte->u.Hard.PageFrameNumber);
956 }
957 else
958 {
959 //
960 // This means we've reached the guard page that protects the end of
961 // the expansion nonpaged pool
962 //
963 Pfn1 = NULL;
964 }
965
966 }
967
968 //
969 // Check if this allocation actually exists
970 //
971 if ((Pfn1) && (Pfn1->u3.e1.StartOfAllocation == 0))
972 {
973 //
974 // It doesn't, so we should actually locate a free entry descriptor
975 //
976 FreeEntry = (PMMFREE_POOL_ENTRY)((ULONG_PTR)StartingVa +
977 (NumberOfPages << PAGE_SHIFT));
978 ASSERT(FreeEntry->Signature == MM_FREE_POOL_SIGNATURE);
979 ASSERT(FreeEntry->Owner == FreeEntry);
980
981 /* Consume this entry's pages */
982 FreePages += FreeEntry->Size;
983
984 /* Remove the item from the list, depending if pool is protected */
985 MmProtectFreedNonPagedPool ?
986 MiProtectedPoolRemoveEntryList(&FreeEntry->List) :
987 RemoveEntryList(&FreeEntry->List);
988 }
989
990 //
991 // Now get the official free entry we'll create for the caller's allocation
992 //
993 FreeEntry = StartingVa;
994
995 //
996 // Check if the our allocation is the very first page
997 //
998 if (MiGetPfnEntryIndex(StartPfn) == MiStartOfInitialPoolFrame)
999 {
1000 //
1001 // Then we can't do anything or we'll risk underflowing
1002 //
1003 Pfn1 = NULL;
1004 }
1005 else
1006 {
1007 //
1008 // Otherwise, get the PTE for the page right before our allocation
1009 //
1010 PointerPte -= NumberOfPages + 1;
1011
1012 /* Check if protected pool is enabled */
1013 if (MmProtectFreedNonPagedPool)
1014 {
1015 /* The freed block will be merged, it must be made accessible */
1016 MiUnProtectFreeNonPagedPool(MiPteToAddress(PointerPte), 0);
1017 }
1018
1019 /* Check if this is valid pool, or a guard page */
1020 if (PointerPte->u.Hard.Valid == 1)
1021 {
1022 //
1023 // It's either expansion or initial nonpaged pool, get the PFN entry
1024 //
1025 Pfn1 = MiGetPfnEntry(PointerPte->u.Hard.PageFrameNumber);
1026 }
1027 else
1028 {
1029 //
1030 // We must've reached the guard page, so don't risk touching it
1031 //
1032 Pfn1 = NULL;
1033 }
1034 }
1035
1036 //
1037 // Check if there is a valid PFN entry for the page before the allocation
1038 // and then check if this page was actually the end of an allocation.
1039 // If it wasn't, then we know for sure it's a free page
1040 //
1041 if ((Pfn1) && (Pfn1->u3.e1.EndOfAllocation == 0))
1042 {
1043 //
1044 // Get the free entry descriptor for that given page range
1045 //
1046 FreeEntry = (PMMFREE_POOL_ENTRY)((ULONG_PTR)StartingVa - PAGE_SIZE);
1047 ASSERT(FreeEntry->Signature == MM_FREE_POOL_SIGNATURE);
1048 FreeEntry = FreeEntry->Owner;
1049
1050 /* Check if protected pool is enabled */
1051 if (MmProtectFreedNonPagedPool)
1052 {
1053 /* The freed block will be merged, it must be made accessible */
1054 MiUnProtectFreeNonPagedPool(FreeEntry, 0);
1055 }
1056
1057 //
1058 // Check if the entry is small enough to be indexed on a free list
1059 // If it is, we'll want to re-insert it, since we're about to
1060 // collapse our pages on top of it, which will change its count
1061 //
1062 if (FreeEntry->Size < (MI_MAX_FREE_PAGE_LISTS - 1))
1063 {
1064 /* Remove the item from the list, depending if pool is protected */
1065 MmProtectFreedNonPagedPool ?
1066 MiProtectedPoolRemoveEntryList(&FreeEntry->List) :
1067 RemoveEntryList(&FreeEntry->List);
1068
1069 //
1070 // Update its size
1071 //
1072 FreeEntry->Size += FreePages;
1073
1074 //
1075 // And now find the new appropriate list to place it in
1076 //
1077 i = (ULONG)(FreeEntry->Size - 1);
1078 if (i >= MI_MAX_FREE_PAGE_LISTS) i = MI_MAX_FREE_PAGE_LISTS - 1;
1079
1080 /* Insert the entry into the free list head, check for prot. pool */
1081 MmProtectFreedNonPagedPool ?
1082 MiProtectedPoolInsertList(&MmNonPagedPoolFreeListHead[i], &FreeEntry->List, TRUE) :
1083 InsertTailList(&MmNonPagedPoolFreeListHead[i], &FreeEntry->List);
1084 }
1085 else
1086 {
1087 //
1088 // Otherwise, just combine our free pages into this entry
1089 //
1090 FreeEntry->Size += FreePages;
1091 }
1092 }
1093
1094 //
1095 // Check if we were unable to do any compaction, and we'll stick with this
1096 //
1097 if (FreeEntry == StartingVa)
1098 {
1099 //
1100 // Well, now we are a free entry. At worse we just have our newly freed
1101 // pages, at best we have our pages plus whatever entry came after us
1102 //
1103 FreeEntry->Size = FreePages;
1104
1105 //
1106 // Find the appropriate list we should be on
1107 //
1108 i = FreeEntry->Size - 1;
1109 if (i >= MI_MAX_FREE_PAGE_LISTS) i = MI_MAX_FREE_PAGE_LISTS - 1;
1110
1111 /* Insert the entry into the free list head, check for prot. pool */
1112 MmProtectFreedNonPagedPool ?
1113 MiProtectedPoolInsertList(&MmNonPagedPoolFreeListHead[i], &FreeEntry->List, TRUE) :
1114 InsertTailList(&MmNonPagedPoolFreeListHead[i], &FreeEntry->List);
1115 }
1116
1117 //
1118 // Just a sanity check
1119 //
1120 ASSERT(FreePages != 0);
1121
1122 //
1123 // Get all the pages between our allocation and its end. These will all now
1124 // become free page chunks.
1125 //
1126 NextEntry = StartingVa;
1127 LastEntry = (PMMFREE_POOL_ENTRY)((ULONG_PTR)NextEntry + (FreePages << PAGE_SHIFT));
1128 do
1129 {
1130 //
1131 // Link back to the parent free entry, and keep going
1132 //
1133 NextEntry->Owner = FreeEntry;
1134 NextEntry->Signature = MM_FREE_POOL_SIGNATURE;
1135 NextEntry = (PMMFREE_POOL_ENTRY)((ULONG_PTR)NextEntry + PAGE_SIZE);
1136 } while (NextEntry != LastEntry);
1137
1138 /* Is freed non paged pool protected? */
1139 if (MmProtectFreedNonPagedPool)
1140 {
1141 /* Protect the freed pool! */
1142 MiProtectFreeNonPagedPool(FreeEntry, FreeEntry->Size);
1143 }
1144
1145 //
1146 // We're done, release the lock and let the caller know how much we freed
1147 //
1148 KeReleaseQueuedSpinLock(LockQueueMmNonPagedPoolLock, OldIrql);
1149 return NumberOfPages;
1150 }
1151
1152
1153 BOOLEAN
1154 NTAPI
1155 MiRaisePoolQuota(IN POOL_TYPE PoolType,
1156 IN ULONG CurrentMaxQuota,
1157 OUT PULONG NewMaxQuota)
1158 {
1159 //
1160 // Not implemented
1161 //
1162 UNIMPLEMENTED;
1163 *NewMaxQuota = CurrentMaxQuota + 65536;
1164 return TRUE;
1165 }
1166
1167 /* PUBLIC FUNCTIONS ***********************************************************/
1168
1169 /*
1170 * @unimplemented
1171 */
1172 PVOID
1173 NTAPI
1174 MmAllocateMappingAddress(IN SIZE_T NumberOfBytes,
1175 IN ULONG PoolTag)
1176 {
1177 UNIMPLEMENTED;
1178 return NULL;
1179 }
1180
1181 /*
1182 * @unimplemented
1183 */
1184 VOID
1185 NTAPI
1186 MmFreeMappingAddress(IN PVOID BaseAddress,
1187 IN ULONG PoolTag)
1188 {
1189 UNIMPLEMENTED;
1190 }
1191
1192 /* EOF */
A free Windows-compatible Operating System