2 * PROJECT: ReactOS Kernel
3 * LICENSE: BSD - See COPYING.ARM in the top level directory
4 * FILE: ntoskrnl/mm/ARM3/expool.c
5 * PURPOSE: ARM Memory Manager Executive Pool Manager
6 * PROGRAMMERS: ReactOS Portable Systems Group
9 /* INCLUDES *******************************************************************/
15 #define MODULE_INVOLVED_IN_ARM3
16 #include "../ARM3/miarm.h"
18 #undef ExAllocatePoolWithQuota
19 #undef ExAllocatePoolWithQuotaTag
21 /* GLOBALS ********************************************************************/
23 #define POOL_BIG_TABLE_ENTRY_FREE 0x1
25 typedef struct _POOL_DPC_CONTEXT
27 PPOOL_TRACKER_TABLE PoolTrackTable
;
28 SIZE_T PoolTrackTableSize
;
29 PPOOL_TRACKER_TABLE PoolTrackTableExpansion
;
30 SIZE_T PoolTrackTableSizeExpansion
;
31 } POOL_DPC_CONTEXT
, *PPOOL_DPC_CONTEXT
;
33 ULONG ExpNumberOfPagedPools
;
34 POOL_DESCRIPTOR NonPagedPoolDescriptor
;
35 PPOOL_DESCRIPTOR ExpPagedPoolDescriptor
[16 + 1];
36 PPOOL_DESCRIPTOR PoolVector
[2];
37 PKGUARDED_MUTEX ExpPagedPoolMutex
;
38 SIZE_T PoolTrackTableSize
, PoolTrackTableMask
;
39 SIZE_T PoolBigPageTableSize
, PoolBigPageTableHash
;
40 PPOOL_TRACKER_TABLE PoolTrackTable
;
41 PPOOL_TRACKER_BIG_PAGES PoolBigPageTable
;
42 KSPIN_LOCK ExpTaggedPoolLock
;
44 BOOLEAN ExStopBadTags
;
45 KSPIN_LOCK ExpLargePoolTableLock
;
46 LONG ExpPoolBigEntriesInUse
;
50 /* Pool block/header/list access macros */
51 #define POOL_ENTRY(x) (PPOOL_HEADER)((ULONG_PTR)(x) - sizeof(POOL_HEADER))
52 #define POOL_FREE_BLOCK(x) (PLIST_ENTRY)((ULONG_PTR)(x) + sizeof(POOL_HEADER))
53 #define POOL_BLOCK(x, i) (PPOOL_HEADER)((ULONG_PTR)(x) + ((i) * POOL_BLOCK_SIZE))
54 #define POOL_NEXT_BLOCK(x) POOL_BLOCK((x), (x)->BlockSize)
55 #define POOL_PREV_BLOCK(x) POOL_BLOCK((x), -((x)->PreviousSize))
58 * Pool list access debug macros, similar to Arthur's pfnlist.c work.
59 * Microsoft actually implements similar checks in the Windows Server 2003 SP1
60 * pool code, but only for checked builds.
62 * As of Vista, however, an MSDN Blog entry by a Security Team Manager indicates
63 * that these checks are done even on retail builds, due to the increasing
64 * number of kernel-mode attacks which depend on dangling list pointers and other
65 * kinds of list-based attacks.
67 * For now, I will leave these checks on all the time, but later they are likely
68 * to be DBG-only, at least until there are enough kernel-mode security attacks
69 * against ReactOS to warrant the performance hit.
71 * For now, these are not made inline, so we can get good stack traces.
75 ExpDecodePoolLink(IN PLIST_ENTRY Link
)
77 return (PLIST_ENTRY
)((ULONG_PTR
)Link
& ~1);
82 ExpEncodePoolLink(IN PLIST_ENTRY Link
)
84 return (PLIST_ENTRY
)((ULONG_PTR
)Link
| 1);
89 ExpCheckPoolLinks(IN PLIST_ENTRY ListHead
)
91 if ((ExpDecodePoolLink(ExpDecodePoolLink(ListHead
->Flink
)->Blink
) != ListHead
) ||
92 (ExpDecodePoolLink(ExpDecodePoolLink(ListHead
->Blink
)->Flink
) != ListHead
))
94 KeBugCheckEx(BAD_POOL_HEADER
,
97 (ULONG_PTR
)ExpDecodePoolLink(ExpDecodePoolLink(ListHead
->Flink
)->Blink
),
98 (ULONG_PTR
)ExpDecodePoolLink(ExpDecodePoolLink(ListHead
->Blink
)->Flink
));
104 ExpInitializePoolListHead(IN PLIST_ENTRY ListHead
)
106 ListHead
->Flink
= ListHead
->Blink
= ExpEncodePoolLink(ListHead
);
111 ExpIsPoolListEmpty(IN PLIST_ENTRY ListHead
)
113 return (ExpDecodePoolLink(ListHead
->Flink
) == ListHead
);
118 ExpRemovePoolEntryList(IN PLIST_ENTRY Entry
)
120 PLIST_ENTRY Blink
, Flink
;
121 Flink
= ExpDecodePoolLink(Entry
->Flink
);
122 Blink
= ExpDecodePoolLink(Entry
->Blink
);
123 Flink
->Blink
= ExpEncodePoolLink(Blink
);
124 Blink
->Flink
= ExpEncodePoolLink(Flink
);
129 ExpRemovePoolHeadList(IN PLIST_ENTRY ListHead
)
131 PLIST_ENTRY Entry
, Flink
;
132 Entry
= ExpDecodePoolLink(ListHead
->Flink
);
133 Flink
= ExpDecodePoolLink(Entry
->Flink
);
134 ListHead
->Flink
= ExpEncodePoolLink(Flink
);
135 Flink
->Blink
= ExpEncodePoolLink(ListHead
);
141 ExpRemovePoolTailList(IN PLIST_ENTRY ListHead
)
143 PLIST_ENTRY Entry
, Blink
;
144 Entry
= ExpDecodePoolLink(ListHead
->Blink
);
145 Blink
= ExpDecodePoolLink(Entry
->Blink
);
146 ListHead
->Blink
= ExpEncodePoolLink(Blink
);
147 Blink
->Flink
= ExpEncodePoolLink(ListHead
);
153 ExpInsertPoolTailList(IN PLIST_ENTRY ListHead
,
154 IN PLIST_ENTRY Entry
)
157 ExpCheckPoolLinks(ListHead
);
158 Blink
= ExpDecodePoolLink(ListHead
->Blink
);
159 Entry
->Flink
= ExpEncodePoolLink(ListHead
);
160 Entry
->Blink
= ExpEncodePoolLink(Blink
);
161 Blink
->Flink
= ExpEncodePoolLink(Entry
);
162 ListHead
->Blink
= ExpEncodePoolLink(Entry
);
163 ExpCheckPoolLinks(ListHead
);
168 ExpInsertPoolHeadList(IN PLIST_ENTRY ListHead
,
169 IN PLIST_ENTRY Entry
)
172 ExpCheckPoolLinks(ListHead
);
173 Flink
= ExpDecodePoolLink(ListHead
->Flink
);
174 Entry
->Flink
= ExpEncodePoolLink(Flink
);
175 Entry
->Blink
= ExpEncodePoolLink(ListHead
);
176 Flink
->Blink
= ExpEncodePoolLink(Entry
);
177 ListHead
->Flink
= ExpEncodePoolLink(Entry
);
178 ExpCheckPoolLinks(ListHead
);
183 ExpCheckPoolHeader(IN PPOOL_HEADER Entry
)
185 PPOOL_HEADER PreviousEntry
, NextEntry
;
187 /* Is there a block before this one? */
188 if (Entry
->PreviousSize
)
191 PreviousEntry
= POOL_PREV_BLOCK(Entry
);
193 /* The two blocks must be on the same page! */
194 if (PAGE_ALIGN(Entry
) != PAGE_ALIGN(PreviousEntry
))
196 /* Something is awry */
197 KeBugCheckEx(BAD_POOL_HEADER
,
199 (ULONG_PTR
)PreviousEntry
,
204 /* This block should also indicate that it's as large as we think it is */
205 if (PreviousEntry
->BlockSize
!= Entry
->PreviousSize
)
207 /* Otherwise, someone corrupted one of the sizes */
208 KeBugCheckEx(BAD_POOL_HEADER
,
210 (ULONG_PTR
)PreviousEntry
,
215 else if (PAGE_ALIGN(Entry
) != Entry
)
217 /* If there's no block before us, we are the first block, so we should be on a page boundary */
218 KeBugCheckEx(BAD_POOL_HEADER
,
225 /* This block must have a size */
226 if (!Entry
->BlockSize
)
228 /* Someone must've corrupted this field */
229 KeBugCheckEx(BAD_POOL_HEADER
,
236 /* Okay, now get the next block */
237 NextEntry
= POOL_NEXT_BLOCK(Entry
);
239 /* If this is the last block, then we'll be page-aligned, otherwise, check this block */
240 if (PAGE_ALIGN(NextEntry
) != NextEntry
)
242 /* The two blocks must be on the same page! */
243 if (PAGE_ALIGN(Entry
) != PAGE_ALIGN(NextEntry
))
245 /* Something is messed up */
246 KeBugCheckEx(BAD_POOL_HEADER
,
248 (ULONG_PTR
)NextEntry
,
253 /* And this block should think we are as large as we truly are */
254 if (NextEntry
->PreviousSize
!= Entry
->BlockSize
)
256 /* Otherwise, someone corrupted the field */
257 KeBugCheckEx(BAD_POOL_HEADER
,
259 (ULONG_PTR
)NextEntry
,
268 ExpCheckPoolBlocks(IN PVOID Block
)
270 BOOLEAN FoundBlock
= FALSE
;
274 /* Get the first entry for this page, make sure it really is the first */
275 Entry
= PAGE_ALIGN(Block
);
276 ASSERT(Entry
->PreviousSize
== 0);
278 /* Now scan each entry */
281 /* When we actually found our block, remember this */
282 if (Entry
== Block
) FoundBlock
= TRUE
;
284 /* Now validate this block header */
285 ExpCheckPoolHeader(Entry
);
287 /* And go to the next one, keeping track of our size */
288 Size
+= Entry
->BlockSize
;
289 Entry
= POOL_NEXT_BLOCK(Entry
);
291 /* If we hit the last block, stop */
292 if (Size
>= (PAGE_SIZE
/ POOL_BLOCK_SIZE
)) break;
294 /* If we hit the end of the page, stop */
295 if (PAGE_ALIGN(Entry
) == Entry
) break;
298 /* We must've found our block, and we must have hit the end of the page */
299 if ((PAGE_ALIGN(Entry
) != Entry
) || !(FoundBlock
))
301 /* Otherwise, the blocks are messed up */
302 KeBugCheckEx(BAD_POOL_HEADER
, 10, (ULONG_PTR
)Block
, __LINE__
, (ULONG_PTR
)Entry
);
308 ExpCheckPoolIrqlLevel(IN POOL_TYPE PoolType
,
309 IN SIZE_T NumberOfBytes
,
313 // Validate IRQL: It must be APC_LEVEL or lower for Paged Pool, and it must
314 // be DISPATCH_LEVEL or lower for Non Paged Pool
316 if (((PoolType
& BASE_POOL_TYPE_MASK
) == PagedPool
) ?
317 (KeGetCurrentIrql() > APC_LEVEL
) :
318 (KeGetCurrentIrql() > DISPATCH_LEVEL
))
321 // Take the system down
323 KeBugCheckEx(BAD_POOL_CALLER
,
324 !Entry
? POOL_ALLOC_IRQL_INVALID
: POOL_FREE_IRQL_INVALID
,
327 !Entry
? NumberOfBytes
: (ULONG_PTR
)Entry
);
333 ExpComputeHashForTag(IN ULONG Tag
,
334 IN SIZE_T BucketMask
)
337 // Compute the hash by multiplying with a large prime number and then XORing
338 // with the HIDWORD of the result.
340 // Finally, AND with the bucket mask to generate a valid index/bucket into
343 ULONGLONG Result
= 40543 * Tag
;
344 return (ULONG
)BucketMask
& ((ULONG
)Result
^ (Result
>> 32));
349 ExpComputePartialHashForAddress(IN PVOID BaseAddress
)
353 // Compute the hash by converting the address into a page number, and then
354 // XORing each nibble with the next one.
356 // We do *NOT* AND with the bucket mask at this point because big table expansion
357 // might happen. Therefore, the final step of the hash must be performed
358 // while holding the expansion pushlock, and this is why we call this a
359 // "partial" hash only.
361 Result
= (ULONG
)((ULONG_PTR
)BaseAddress
>> PAGE_SHIFT
);
362 return (Result
>> 24) ^ (Result
>> 16) ^ (Result
>> 8) ^ Result
;
365 /* PRIVATE FUNCTIONS **********************************************************/
372 ULONG i
, Key
, Hash
, Index
;
373 PPOOL_TRACKER_TABLE TrackTable
= PoolTrackTable
;
443 // Loop all 64 hot tags
445 ASSERT((sizeof(TagList
) / sizeof(ULONG
)) == 64);
446 for (i
= 0; i
< sizeof(TagList
) / sizeof(ULONG
); i
++)
449 // Get the current tag, and compute its hash in the tracker table
452 Hash
= ExpComputeHashForTag(Key
, PoolTrackTableMask
);
455 // Loop all the hashes in this index/bucket
461 // Find an empty entry, and make sure this isn't the last hash that
464 // On checked builds, also make sure this is the first time we are
467 ASSERT(TrackTable
[Hash
].Key
!= Key
);
468 if (!(TrackTable
[Hash
].Key
) && (Hash
!= PoolTrackTableSize
- 1))
471 // It has been seeded, move on to the next tag
473 TrackTable
[Hash
].Key
= Key
;
478 // This entry was already taken, compute the next possible hash while
479 // making sure we're not back at our initial index.
481 ASSERT(TrackTable
[Hash
].Key
!= Key
);
482 Hash
= (Hash
+ 1) & PoolTrackTableMask
;
483 if (Hash
== Index
) break;
490 ExpRemovePoolTracker(IN ULONG Key
,
491 IN SIZE_T NumberOfBytes
,
492 IN POOL_TYPE PoolType
)
495 PPOOL_TRACKER_TABLE Table
, TableEntry
;
496 SIZE_T TableMask
, TableSize
;
499 // Remove the PROTECTED_POOL flag which is not part of the tag
501 Key
&= ~PROTECTED_POOL
;
504 // With WinDBG you can set a tag you want to break on when an allocation is
507 if (Key
== PoolHitTag
) DbgBreakPoint();
510 // Why the double indirection? Because normally this function is also used
511 // when doing session pool allocations, which has another set of tables,
512 // sizes, and masks that live in session pool. Now we don't support session
513 // pool so we only ever use the regular tables, but I'm keeping the code this
514 // way so that the day we DO support session pool, it won't require that
517 Table
= PoolTrackTable
;
518 TableMask
= PoolTrackTableMask
;
519 TableSize
= PoolTrackTableSize
;
522 // Compute the hash for this key, and loop all the possible buckets
524 Hash
= ExpComputeHashForTag(Key
, TableMask
);
529 // Have we found the entry for this tag? */
531 TableEntry
= &Table
[Hash
];
532 if (TableEntry
->Key
== Key
)
535 // Decrement the counters depending on if this was paged or nonpaged
538 if ((PoolType
& BASE_POOL_TYPE_MASK
) == NonPagedPool
)
540 InterlockedIncrement(&TableEntry
->NonPagedFrees
);
541 InterlockedExchangeAddSizeT(&TableEntry
->NonPagedBytes
,
542 -(SSIZE_T
)NumberOfBytes
);
545 InterlockedIncrement(&TableEntry
->PagedFrees
);
546 InterlockedExchangeAddSizeT(&TableEntry
->PagedBytes
,
547 -(SSIZE_T
)NumberOfBytes
);
552 // We should have only ended up with an empty entry if we've reached
555 if (!TableEntry
->Key
) ASSERT(Hash
== TableMask
);
558 // This path is hit when we don't have an entry, and the current bucket
559 // is full, so we simply try the next one
561 Hash
= (Hash
+ 1) & TableMask
;
562 if (Hash
== Index
) break;
566 // And finally this path is hit when all the buckets are full, and we need
567 // some expansion. This path is not yet supported in ReactOS and so we'll
570 DPRINT1("Out of pool tag space, ignoring...\n");
575 ExpInsertPoolTracker(IN ULONG Key
,
576 IN SIZE_T NumberOfBytes
,
577 IN POOL_TYPE PoolType
)
581 PPOOL_TRACKER_TABLE Table
, TableEntry
;
582 SIZE_T TableMask
, TableSize
;
585 // Remove the PROTECTED_POOL flag which is not part of the tag
587 Key
&= ~PROTECTED_POOL
;
590 // With WinDBG you can set a tag you want to break on when an allocation is
593 if (Key
== PoolHitTag
) DbgBreakPoint();
596 // There is also an internal flag you can set to break on malformed tags
598 if (ExStopBadTags
) ASSERT(Key
& 0xFFFFFF00);
601 // ASSERT on ReactOS features not yet supported
603 ASSERT(!(PoolType
& SESSION_POOL_MASK
));
604 ASSERT(KeGetCurrentProcessorNumber() == 0);
607 // Why the double indirection? Because normally this function is also used
608 // when doing session pool allocations, which has another set of tables,
609 // sizes, and masks that live in session pool. Now we don't support session
610 // pool so we only ever use the regular tables, but I'm keeping the code this
611 // way so that the day we DO support session pool, it won't require that
614 Table
= PoolTrackTable
;
615 TableMask
= PoolTrackTableMask
;
616 TableSize
= PoolTrackTableSize
;
619 // Compute the hash for this key, and loop all the possible buckets
621 Hash
= ExpComputeHashForTag(Key
, TableMask
);
626 // Do we already have an entry for this tag? */
628 TableEntry
= &Table
[Hash
];
629 if (TableEntry
->Key
== Key
)
632 // Increment the counters depending on if this was paged or nonpaged
635 if ((PoolType
& BASE_POOL_TYPE_MASK
) == NonPagedPool
)
637 InterlockedIncrement(&TableEntry
->NonPagedAllocs
);
638 InterlockedExchangeAddSizeT(&TableEntry
->NonPagedBytes
, NumberOfBytes
);
641 InterlockedIncrement(&TableEntry
->PagedAllocs
);
642 InterlockedExchangeAddSizeT(&TableEntry
->PagedBytes
, NumberOfBytes
);
647 // We don't have an entry yet, but we've found a free bucket for it
649 if (!(TableEntry
->Key
) && (Hash
!= PoolTrackTableSize
- 1))
652 // We need to hold the lock while creating a new entry, since other
653 // processors might be in this code path as well
655 ExAcquireSpinLock(&ExpTaggedPoolLock
, &OldIrql
);
656 if (!PoolTrackTable
[Hash
].Key
)
659 // We've won the race, so now create this entry in the bucket
661 ASSERT(Table
[Hash
].Key
== 0);
662 PoolTrackTable
[Hash
].Key
= Key
;
663 TableEntry
->Key
= Key
;
665 ExReleaseSpinLock(&ExpTaggedPoolLock
, OldIrql
);
668 // Now we force the loop to run again, and we should now end up in
669 // the code path above which does the interlocked increments...
675 // This path is hit when we don't have an entry, and the current bucket
676 // is full, so we simply try the next one
678 Hash
= (Hash
+ 1) & TableMask
;
679 if (Hash
== Index
) break;
683 // And finally this path is hit when all the buckets are full, and we need
684 // some expansion. This path is not yet supported in ReactOS and so we'll
687 DPRINT1("Out of pool tag space, ignoring...\n");
693 ExInitializePoolDescriptor(IN PPOOL_DESCRIPTOR PoolDescriptor
,
694 IN POOL_TYPE PoolType
,
699 PLIST_ENTRY NextEntry
, LastEntry
;
702 // Setup the descriptor based on the caller's request
704 PoolDescriptor
->PoolType
= PoolType
;
705 PoolDescriptor
->PoolIndex
= PoolIndex
;
706 PoolDescriptor
->Threshold
= Threshold
;
707 PoolDescriptor
->LockAddress
= PoolLock
;
710 // Initialize accounting data
712 PoolDescriptor
->RunningAllocs
= 0;
713 PoolDescriptor
->RunningDeAllocs
= 0;
714 PoolDescriptor
->TotalPages
= 0;
715 PoolDescriptor
->TotalBytes
= 0;
716 PoolDescriptor
->TotalBigPages
= 0;
719 // Nothing pending for now
721 PoolDescriptor
->PendingFrees
= NULL
;
722 PoolDescriptor
->PendingFreeDepth
= 0;
725 // Loop all the descriptor's allocation lists and initialize them
727 NextEntry
= PoolDescriptor
->ListHeads
;
728 LastEntry
= NextEntry
+ POOL_LISTS_PER_PAGE
;
729 while (NextEntry
< LastEntry
)
731 ExpInitializePoolListHead(NextEntry
);
736 // Note that ReactOS does not support Session Pool Yet
738 ASSERT(PoolType
!= PagedPoolSession
);
744 InitializePool(IN POOL_TYPE PoolType
,
747 PPOOL_DESCRIPTOR Descriptor
;
752 // Check what kind of pool this is
754 if (PoolType
== NonPagedPool
)
757 // Compute the track table size and convert it from a power of two to an
760 // NOTE: On checked builds, we'll assert if the registry table size was
761 // invalid, while on retail builds we'll just break out of the loop at
764 TableSize
= min(PoolTrackTableSize
, MmSizeOfNonPagedPoolInBytes
>> 8);
765 for (i
= 0; i
< 32; i
++)
769 ASSERT((TableSize
& ~1) == 0);
770 if (!(TableSize
& ~1)) break;
776 // If we hit bit 32, than no size was defined in the registry, so
777 // we'll use the default size of 2048 entries.
779 // Otherwise, use the size from the registry, as long as it's not
780 // smaller than 64 entries.
784 PoolTrackTableSize
= 2048;
788 PoolTrackTableSize
= max(1 << i
, 64);
792 // Loop trying with the biggest specified size first, and cut it down
793 // by a power of two each iteration in case not enough memory exist
798 // Do not allow overflow
800 if ((PoolTrackTableSize
+ 1) > (MAXULONG_PTR
/ sizeof(POOL_TRACKER_TABLE
)))
802 PoolTrackTableSize
>>= 1;
807 // Allocate the tracker table and exit the loop if this worked
809 PoolTrackTable
= MiAllocatePoolPages(NonPagedPool
,
810 (PoolTrackTableSize
+ 1) *
811 sizeof(POOL_TRACKER_TABLE
));
812 if (PoolTrackTable
) break;
815 // Otherwise, as long as we're not down to the last bit, keep
818 if (PoolTrackTableSize
== 1)
820 KeBugCheckEx(MUST_SUCCEED_POOL_EMPTY
,
826 PoolTrackTableSize
>>= 1;
830 // Finally, add one entry, compute the hash, and zero the table
832 PoolTrackTableSize
++;
833 PoolTrackTableMask
= PoolTrackTableSize
- 2;
835 RtlZeroMemory(PoolTrackTable
,
836 PoolTrackTableSize
* sizeof(POOL_TRACKER_TABLE
));
839 // We now do the exact same thing with the tracker table for big pages
841 TableSize
= min(PoolBigPageTableSize
, MmSizeOfNonPagedPoolInBytes
>> 8);
842 for (i
= 0; i
< 32; i
++)
846 ASSERT((TableSize
& ~1) == 0);
847 if (!(TableSize
& ~1)) break;
853 // For big pages, the default tracker table is 4096 entries, while the
854 // minimum is still 64
858 PoolBigPageTableSize
= 4096;
862 PoolBigPageTableSize
= max(1 << i
, 64);
866 // Again, run the exact same loop we ran earlier, but this time for the
867 // big pool tracker instead
871 if ((PoolBigPageTableSize
+ 1) > (MAXULONG_PTR
/ sizeof(POOL_TRACKER_BIG_PAGES
)))
873 PoolBigPageTableSize
>>= 1;
877 PoolBigPageTable
= MiAllocatePoolPages(NonPagedPool
,
878 PoolBigPageTableSize
*
879 sizeof(POOL_TRACKER_BIG_PAGES
));
880 if (PoolBigPageTable
) break;
882 if (PoolBigPageTableSize
== 1)
884 KeBugCheckEx(MUST_SUCCEED_POOL_EMPTY
,
891 PoolBigPageTableSize
>>= 1;
895 // An extra entry is not needed for for the big pool tracker, so just
896 // compute the hash and zero it
898 PoolBigPageTableHash
= PoolBigPageTableSize
- 1;
899 RtlZeroMemory(PoolBigPageTable
,
900 PoolBigPageTableSize
* sizeof(POOL_TRACKER_BIG_PAGES
));
901 for (i
= 0; i
< PoolBigPageTableSize
; i
++) PoolBigPageTable
[i
].Va
= (PVOID
)1;
904 // During development, print this out so we can see what's happening
906 DPRINT1("EXPOOL: Pool Tracker Table at: 0x%p with 0x%lx bytes\n",
907 PoolTrackTable
, PoolTrackTableSize
* sizeof(POOL_TRACKER_TABLE
));
908 DPRINT1("EXPOOL: Big Pool Tracker Table at: 0x%p with 0x%lx bytes\n",
909 PoolBigPageTable
, PoolBigPageTableSize
* sizeof(POOL_TRACKER_BIG_PAGES
));
912 // Insert the generic tracker for all of big pool
914 ExpInsertPoolTracker('looP',
915 ROUND_TO_PAGES(PoolBigPageTableSize
*
916 sizeof(POOL_TRACKER_BIG_PAGES
)),
920 // No support for NUMA systems at this time
922 ASSERT(KeNumberNodes
== 1);
925 // Initialize the tag spinlock
927 KeInitializeSpinLock(&ExpTaggedPoolLock
);
930 // Initialize the nonpaged pool descriptor
932 PoolVector
[NonPagedPool
] = &NonPagedPoolDescriptor
;
933 ExInitializePoolDescriptor(PoolVector
[NonPagedPool
],
942 // No support for NUMA systems at this time
944 ASSERT(KeNumberNodes
== 1);
947 // Allocate the pool descriptor
949 Descriptor
= ExAllocatePoolWithTag(NonPagedPool
,
950 sizeof(KGUARDED_MUTEX
) +
951 sizeof(POOL_DESCRIPTOR
),
956 // This is really bad...
958 KeBugCheckEx(MUST_SUCCEED_POOL_EMPTY
,
966 // Setup the vector and guarded mutex for paged pool
968 PoolVector
[PagedPool
] = Descriptor
;
969 ExpPagedPoolMutex
= (PKGUARDED_MUTEX
)(Descriptor
+ 1);
970 ExpPagedPoolDescriptor
[0] = Descriptor
;
971 KeInitializeGuardedMutex(ExpPagedPoolMutex
);
972 ExInitializePoolDescriptor(Descriptor
,
979 // Insert the generic tracker for all of nonpaged pool
981 ExpInsertPoolTracker('looP',
982 ROUND_TO_PAGES(PoolTrackTableSize
* sizeof(POOL_TRACKER_TABLE
)),
989 ExLockPool(IN PPOOL_DESCRIPTOR Descriptor
)
992 // Check if this is nonpaged pool
994 if ((Descriptor
->PoolType
& BASE_POOL_TYPE_MASK
) == NonPagedPool
)
997 // Use the queued spin lock
999 return KeAcquireQueuedSpinLock(LockQueueNonPagedPoolLock
);
1004 // Use the guarded mutex
1006 KeAcquireGuardedMutex(Descriptor
->LockAddress
);
1013 ExUnlockPool(IN PPOOL_DESCRIPTOR Descriptor
,
1017 // Check if this is nonpaged pool
1019 if ((Descriptor
->PoolType
& BASE_POOL_TYPE_MASK
) == NonPagedPool
)
1022 // Use the queued spin lock
1024 KeReleaseQueuedSpinLock(LockQueueNonPagedPoolLock
, OldIrql
);
1029 // Use the guarded mutex
1031 KeReleaseGuardedMutex(Descriptor
->LockAddress
);
1037 ExpGetPoolTagInfoTarget(IN PKDPC Dpc
,
1038 IN PVOID DeferredContext
,
1039 IN PVOID SystemArgument1
,
1040 IN PVOID SystemArgument2
)
1042 PPOOL_DPC_CONTEXT Context
= DeferredContext
;
1043 UNREFERENCED_PARAMETER(Dpc
);
1044 ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL
);
1047 // Make sure we win the race, and if we did, copy the data atomically
1049 if (KeSignalCallDpcSynchronize(SystemArgument2
))
1051 RtlCopyMemory(Context
->PoolTrackTable
,
1053 Context
->PoolTrackTableSize
* sizeof(POOL_TRACKER_TABLE
));
1056 // This is here because ReactOS does not yet support expansion
1058 ASSERT(Context
->PoolTrackTableSizeExpansion
== 0);
1062 // Regardless of whether we won or not, we must now synchronize and then
1063 // decrement the barrier since this is one more processor that has completed
1066 KeSignalCallDpcSynchronize(SystemArgument2
);
1067 KeSignalCallDpcDone(SystemArgument1
);
1072 ExGetPoolTagInfo(IN PSYSTEM_POOLTAG_INFORMATION SystemInformation
,
1073 IN ULONG SystemInformationLength
,
1074 IN OUT PULONG ReturnLength OPTIONAL
)
1076 ULONG TableSize
, CurrentLength
;
1078 NTSTATUS Status
= STATUS_SUCCESS
;
1079 PSYSTEM_POOLTAG TagEntry
;
1080 PPOOL_TRACKER_TABLE Buffer
, TrackerEntry
;
1081 POOL_DPC_CONTEXT Context
;
1082 ASSERT(KeGetCurrentIrql() == PASSIVE_LEVEL
);
1085 // Keep track of how much data the caller's buffer must hold
1087 CurrentLength
= FIELD_OFFSET(SYSTEM_POOLTAG_INFORMATION
, TagInfo
);
1090 // Initialize the caller's buffer
1092 TagEntry
= &SystemInformation
->TagInfo
[0];
1093 SystemInformation
->Count
= 0;
1096 // Capture the number of entries, and the total size needed to make a copy
1099 EntryCount
= (ULONG
)PoolTrackTableSize
;
1100 TableSize
= EntryCount
* sizeof(POOL_TRACKER_TABLE
);
1103 // Allocate the "Generic DPC" temporary buffer
1105 Buffer
= ExAllocatePoolWithTag(NonPagedPool
, TableSize
, 'ofnI');
1106 if (!Buffer
) return STATUS_INSUFFICIENT_RESOURCES
;
1109 // Do a "Generic DPC" to atomically retrieve the tag and allocation data
1111 Context
.PoolTrackTable
= Buffer
;
1112 Context
.PoolTrackTableSize
= PoolTrackTableSize
;
1113 Context
.PoolTrackTableExpansion
= NULL
;
1114 Context
.PoolTrackTableSizeExpansion
= 0;
1115 KeGenericCallDpc(ExpGetPoolTagInfoTarget
, &Context
);
1118 // Now parse the results
1120 for (TrackerEntry
= Buffer
; TrackerEntry
< (Buffer
+ EntryCount
); TrackerEntry
++)
1123 // If the entry is empty, skip it
1125 if (!TrackerEntry
->Key
) continue;
1128 // Otherwise, add one more entry to the caller's buffer, and ensure that
1129 // enough space has been allocated in it
1131 SystemInformation
->Count
++;
1132 CurrentLength
+= sizeof(*TagEntry
);
1133 if (SystemInformationLength
< CurrentLength
)
1136 // The caller's buffer is too small, so set a failure code. The
1137 // caller will know the count, as well as how much space is needed.
1139 // We do NOT break out of the loop, because we want to keep incrementing
1140 // the Count as well as CurrentLength so that the caller can know the
1143 Status
= STATUS_INFO_LENGTH_MISMATCH
;
1148 // Small sanity check that our accounting is working correctly
1150 ASSERT(TrackerEntry
->PagedAllocs
>= TrackerEntry
->PagedFrees
);
1151 ASSERT(TrackerEntry
->NonPagedAllocs
>= TrackerEntry
->NonPagedFrees
);
1154 // Return the data into the caller's buffer
1156 TagEntry
->TagUlong
= TrackerEntry
->Key
;
1157 TagEntry
->PagedAllocs
= TrackerEntry
->PagedAllocs
;
1158 TagEntry
->PagedFrees
= TrackerEntry
->PagedFrees
;
1159 TagEntry
->PagedUsed
= TrackerEntry
->PagedBytes
;
1160 TagEntry
->NonPagedAllocs
= TrackerEntry
->NonPagedAllocs
;
1161 TagEntry
->NonPagedFrees
= TrackerEntry
->NonPagedFrees
;
1162 TagEntry
->NonPagedUsed
= TrackerEntry
->NonPagedBytes
;
1168 // Free the "Generic DPC" temporary buffer, return the buffer length and status
1171 if (ReturnLength
) *ReturnLength
= CurrentLength
;
1177 ExpAddTagForBigPages(IN PVOID Va
,
1179 IN ULONG NumberOfPages
,
1180 IN POOL_TYPE PoolType
)
1186 PPOOL_TRACKER_BIG_PAGES Entry
, EntryEnd
, EntryStart
;
1187 ASSERT(((ULONG_PTR
)Va
& POOL_BIG_TABLE_ENTRY_FREE
) == 0);
1188 ASSERT(!(PoolType
& SESSION_POOL_MASK
));
1191 // As the table is expandable, these values must only be read after acquiring
1192 // the lock to avoid a teared access during an expansion
1194 Hash
= ExpComputePartialHashForAddress(Va
);
1195 KeAcquireSpinLock(&ExpLargePoolTableLock
, &OldIrql
);
1196 Hash
&= PoolBigPageTableHash
;
1197 TableSize
= PoolBigPageTableSize
;
1200 // We loop from the current hash bucket to the end of the table, and then
1201 // rollover to hash bucket 0 and keep going from there. If we return back
1202 // to the beginning, then we attempt expansion at the bottom of the loop
1204 EntryStart
= Entry
= &PoolBigPageTable
[Hash
];
1205 EntryEnd
= &PoolBigPageTable
[TableSize
];
1209 // Make sure that this is a free entry and attempt to atomically make the
1213 if (((ULONG_PTR
)OldVa
& POOL_BIG_TABLE_ENTRY_FREE
) &&
1214 (InterlockedCompareExchangePointer(&Entry
->Va
, Va
, OldVa
) == OldVa
))
1217 // We now own this entry, write down the size and the pool tag
1220 Entry
->NumberOfPages
= NumberOfPages
;
1223 // Add one more entry to the count, and see if we're getting within
1224 // 25% of the table size, at which point we'll do an expansion now
1225 // to avoid blocking too hard later on.
1227 // Note that we only do this if it's also been the 16th time that we
1228 // keep losing the race or that we are not finding a free entry anymore,
1229 // which implies a massive number of concurrent big pool allocations.
1231 InterlockedIncrement(&ExpPoolBigEntriesInUse
);
1232 if ((i
>= 16) && (ExpPoolBigEntriesInUse
> (TableSize
/ 4)))
1234 DPRINT1("Should attempt expansion since we now have %d entries\n",
1235 ExpPoolBigEntriesInUse
);
1239 // We have our entry, return
1241 KeReleaseSpinLock(&ExpLargePoolTableLock
, OldIrql
);
1246 // We don't have our entry yet, so keep trying, making the entry list
1247 // circular if we reach the last entry. We'll eventually break out of
1248 // the loop once we've rolled over and returned back to our original
1252 if (++Entry
>= EntryEnd
) Entry
= &PoolBigPageTable
[0];
1253 } while (Entry
!= EntryStart
);
1256 // This means there's no free hash buckets whatsoever, so we would now have
1257 // to attempt expanding the table
1259 DPRINT1("Big pool expansion needed, not implemented!\n");
1260 KeReleaseSpinLock(&ExpLargePoolTableLock
, OldIrql
);
1266 ExpFindAndRemoveTagBigPages(IN PVOID Va
,
1267 OUT PULONG_PTR BigPages
,
1268 IN POOL_TYPE PoolType
)
1270 BOOLEAN FirstTry
= TRUE
;
1273 ULONG PoolTag
, Hash
;
1274 PPOOL_TRACKER_BIG_PAGES Entry
;
1275 ASSERT(((ULONG_PTR
)Va
& POOL_BIG_TABLE_ENTRY_FREE
) == 0);
1276 ASSERT(!(PoolType
& SESSION_POOL_MASK
));
1279 // As the table is expandable, these values must only be read after acquiring
1280 // the lock to avoid a teared access during an expansion
1282 Hash
= ExpComputePartialHashForAddress(Va
);
1283 KeAcquireSpinLock(&ExpLargePoolTableLock
, &OldIrql
);
1284 Hash
&= PoolBigPageTableHash
;
1285 TableSize
= PoolBigPageTableSize
;
1288 // Loop while trying to find this big page allocation
1290 while (PoolBigPageTable
[Hash
].Va
!= Va
)
1293 // Increment the size until we go past the end of the table
1295 if (++Hash
>= TableSize
)
1298 // Is this the second time we've tried?
1303 // This means it was never inserted into the pool table and it
1304 // received the special "BIG" tag -- return that and return 0
1305 // so that the code can ask Mm for the page count instead
1307 KeReleaseSpinLock(&ExpLargePoolTableLock
, OldIrql
);
1313 // The first time this happens, reset the hash index and try again
1321 // Now capture all the information we need from the entry, since after we
1322 // release the lock, the data can change
1324 Entry
= &PoolBigPageTable
[Hash
];
1325 *BigPages
= Entry
->NumberOfPages
;
1326 PoolTag
= Entry
->Key
;
1329 // Set the free bit, and decrement the number of allocations. Finally, release
1330 // the lock and return the tag that was located
1332 InterlockedIncrement((PLONG
)&Entry
->Va
);
1333 InterlockedDecrement(&ExpPoolBigEntriesInUse
);
1334 KeReleaseSpinLock(&ExpLargePoolTableLock
, OldIrql
);
1340 ExQueryPoolUsage(OUT PULONG PagedPoolPages
,
1341 OUT PULONG NonPagedPoolPages
,
1342 OUT PULONG PagedPoolAllocs
,
1343 OUT PULONG PagedPoolFrees
,
1344 OUT PULONG PagedPoolLookasideHits
,
1345 OUT PULONG NonPagedPoolAllocs
,
1346 OUT PULONG NonPagedPoolFrees
,
1347 OUT PULONG NonPagedPoolLookasideHits
)
1350 PPOOL_DESCRIPTOR PoolDesc
;
1353 // Assume all failures
1355 *PagedPoolPages
= 0;
1356 *PagedPoolAllocs
= 0;
1357 *PagedPoolFrees
= 0;
1360 // Tally up the totals for all the apged pool
1362 for (i
= 0; i
< ExpNumberOfPagedPools
+ 1; i
++)
1364 PoolDesc
= ExpPagedPoolDescriptor
[i
];
1365 *PagedPoolPages
+= PoolDesc
->TotalPages
+ PoolDesc
->TotalBigPages
;
1366 *PagedPoolAllocs
+= PoolDesc
->RunningAllocs
;
1367 *PagedPoolFrees
+= PoolDesc
->RunningDeAllocs
;
1371 // The first non-paged pool has a hardcoded well-known descriptor name
1373 PoolDesc
= &NonPagedPoolDescriptor
;
1374 *NonPagedPoolPages
= PoolDesc
->TotalPages
+ PoolDesc
->TotalBigPages
;
1375 *NonPagedPoolAllocs
= PoolDesc
->RunningAllocs
;
1376 *NonPagedPoolFrees
= PoolDesc
->RunningDeAllocs
;
1379 // If the system has more than one non-paged pool, copy the other descriptor
1383 if (ExpNumberOfNonPagedPools
> 1)
1385 for (i
= 0; i
< ExpNumberOfNonPagedPools
; i
++)
1387 PoolDesc
= ExpNonPagedPoolDescriptor
[i
];
1388 *NonPagedPoolPages
+= PoolDesc
->TotalPages
+ PoolDesc
->TotalBigPages
;
1389 *NonPagedPoolAllocs
+= PoolDesc
->RunningAllocs
;
1390 *NonPagedPoolFrees
+= PoolDesc
->RunningDeAllocs
;
1396 // FIXME: Not yet supported
1398 *NonPagedPoolLookasideHits
+= 0;
1399 *PagedPoolLookasideHits
+= 0;
1402 /* PUBLIC FUNCTIONS ***********************************************************/
1409 ExAllocatePoolWithTag(IN POOL_TYPE PoolType
,
1410 IN SIZE_T NumberOfBytes
,
1413 PPOOL_DESCRIPTOR PoolDesc
;
1414 PLIST_ENTRY ListHead
;
1415 PPOOL_HEADER Entry
, NextEntry
, FragmentEntry
;
1417 USHORT BlockSize
, i
;
1419 PKPRCB Prcb
= KeGetCurrentPrcb();
1420 PGENERAL_LOOKASIDE LookasideList
;
1423 // Some sanity checks
1426 ASSERT(Tag
!= ' GIB');
1427 ASSERT(NumberOfBytes
!= 0);
1428 ExpCheckPoolIrqlLevel(PoolType
, NumberOfBytes
, NULL
);
1431 // Not supported in ReactOS
1433 ASSERT(!(PoolType
& SESSION_POOL_MASK
));
1436 // Check if verifier or special pool is enabled
1438 if (ExpPoolFlags
& (POOL_FLAG_VERIFIER
| POOL_FLAG_SPECIAL_POOL
))
1441 // For verifier, we should call the verification routine
1443 if (ExpPoolFlags
& POOL_FLAG_VERIFIER
)
1445 DPRINT1("Driver Verifier is not yet supported\n");
1449 // For special pool, we check if this is a suitable allocation and do
1450 // the special allocation if needed
1452 if (ExpPoolFlags
& POOL_FLAG_SPECIAL_POOL
)
1455 // Check if this is a special pool allocation
1457 if (MmUseSpecialPool(NumberOfBytes
, Tag
))
1460 // Try to allocate using special pool
1462 Entry
= MmAllocateSpecialPool(NumberOfBytes
, Tag
, PoolType
, 2);
1463 if (Entry
) return Entry
;
1469 // Get the pool type and its corresponding vector for this request
1471 OriginalType
= PoolType
;
1472 PoolType
= PoolType
& BASE_POOL_TYPE_MASK
;
1473 PoolDesc
= PoolVector
[PoolType
];
1474 ASSERT(PoolDesc
!= NULL
);
1477 // Check if this is a big page allocation
1479 if (NumberOfBytes
> POOL_MAX_ALLOC
)
1482 // Allocate pages for it
1484 Entry
= MiAllocatePoolPages(OriginalType
, NumberOfBytes
);
1488 // Must succeed pool is deprecated, but still supported. These allocation
1489 // failures must cause an immediate bugcheck
1491 if (OriginalType
& MUST_SUCCEED_POOL_MASK
)
1493 KeBugCheckEx(MUST_SUCCEED_POOL_EMPTY
,
1495 NonPagedPoolDescriptor
.TotalPages
,
1496 NonPagedPoolDescriptor
.TotalBigPages
,
1501 // Internal debugging
1506 // This flag requests printing failures, and can also further specify
1507 // breaking on failures
1509 if (ExpPoolFlags
& POOL_FLAG_DBGPRINT_ON_FAILURE
)
1511 DPRINT1("EX: ExAllocatePool (%p, 0x%x) returning NULL\n",
1514 if (ExpPoolFlags
& POOL_FLAG_CRASH_ON_FAILURE
) DbgBreakPoint();
1518 // Finally, this flag requests an exception, which we are more than
1521 if (OriginalType
& POOL_RAISE_IF_ALLOCATION_FAILURE
)
1523 ExRaiseStatus(STATUS_INSUFFICIENT_RESOURCES
);
1528 // Increment required counters
1530 InterlockedExchangeAdd((PLONG
)&PoolDesc
->TotalBigPages
,
1531 (LONG
)BYTES_TO_PAGES(NumberOfBytes
));
1532 InterlockedExchangeAddSizeT(&PoolDesc
->TotalBytes
, NumberOfBytes
);
1533 InterlockedIncrement((PLONG
)&PoolDesc
->RunningAllocs
);
1536 // Add a tag for the big page allocation and switch to the generic "BIG"
1537 // tag if we failed to do so, then insert a tracker for this alloation.
1539 if (!ExpAddTagForBigPages(Entry
,
1541 (ULONG
)BYTES_TO_PAGES(NumberOfBytes
),
1546 ExpInsertPoolTracker(Tag
, ROUND_TO_PAGES(NumberOfBytes
), OriginalType
);
1551 // Should never request 0 bytes from the pool, but since so many drivers do
1552 // it, we'll just assume they want 1 byte, based on NT's similar behavior
1554 if (!NumberOfBytes
) NumberOfBytes
= 1;
1557 // A pool allocation is defined by its data, a linked list to connect it to
1558 // the free list (if necessary), and a pool header to store accounting info.
1559 // Calculate this size, then convert it into a block size (units of pool
1562 // Note that i cannot overflow (past POOL_LISTS_PER_PAGE) because any such
1563 // request would've been treated as a POOL_MAX_ALLOC earlier and resulted in
1564 // the direct allocation of pages.
1566 i
= (USHORT
)((NumberOfBytes
+ sizeof(POOL_HEADER
) + (POOL_BLOCK_SIZE
- 1))
1570 // Handle lookaside list optimization for both paged and nonpaged pool
1572 if (i
<= MAXIMUM_PROCESSORS
)
1575 // Try popping it from the per-CPU lookaside list
1577 LookasideList
= (PoolType
== PagedPool
) ?
1578 Prcb
->PPPagedLookasideList
[i
- 1].P
:
1579 Prcb
->PPNPagedLookasideList
[i
- 1].P
;
1580 LookasideList
->TotalAllocates
++;
1581 Entry
= (PPOOL_HEADER
)InterlockedPopEntrySList(&LookasideList
->ListHead
);
1585 // We failed, try popping it from the global list
1587 LookasideList
= (PoolType
== PagedPool
) ?
1588 Prcb
->PPPagedLookasideList
[i
- 1].L
:
1589 Prcb
->PPNPagedLookasideList
[i
- 1].L
;
1590 LookasideList
->TotalAllocates
++;
1591 Entry
= (PPOOL_HEADER
)InterlockedPopEntrySList(&LookasideList
->ListHead
);
1595 // If we were able to pop it, update the accounting and return the block
1599 LookasideList
->AllocateHits
++;
1602 // Get the real entry, write down its pool type, and track it
1605 Entry
->PoolType
= PoolType
+ 1;
1606 ExpInsertPoolTracker(Tag
,
1607 Entry
->BlockSize
* POOL_BLOCK_SIZE
,
1611 // Return the pool allocation
1613 Entry
->PoolTag
= Tag
;
1614 (POOL_FREE_BLOCK(Entry
))->Flink
= NULL
;
1615 (POOL_FREE_BLOCK(Entry
))->Blink
= NULL
;
1616 return POOL_FREE_BLOCK(Entry
);
1621 // Loop in the free lists looking for a block if this size. Start with the
1622 // list optimized for this kind of size lookup
1624 ListHead
= &PoolDesc
->ListHeads
[i
];
1628 // Are there any free entries available on this list?
1630 if (!ExpIsPoolListEmpty(ListHead
))
1633 // Acquire the pool lock now
1635 OldIrql
= ExLockPool(PoolDesc
);
1638 // And make sure the list still has entries
1640 if (ExpIsPoolListEmpty(ListHead
))
1643 // Someone raced us (and won) before we had a chance to acquire
1648 ExUnlockPool(PoolDesc
, OldIrql
);
1654 // Remove a free entry from the list
1655 // Note that due to the way we insert free blocks into multiple lists
1656 // there is a guarantee that any block on this list will either be
1657 // of the correct size, or perhaps larger.
1659 ExpCheckPoolLinks(ListHead
);
1660 Entry
= POOL_ENTRY(ExpRemovePoolHeadList(ListHead
));
1661 ExpCheckPoolLinks(ListHead
);
1662 ExpCheckPoolBlocks(Entry
);
1663 ASSERT(Entry
->BlockSize
>= i
);
1664 ASSERT(Entry
->PoolType
== 0);
1667 // Check if this block is larger that what we need. The block could
1668 // not possibly be smaller, due to the reason explained above (and
1669 // we would've asserted on a checked build if this was the case).
1671 if (Entry
->BlockSize
!= i
)
1674 // Is there an entry before this one?
1676 if (Entry
->PreviousSize
== 0)
1679 // There isn't anyone before us, so take the next block and
1680 // turn it into a fragment that contains the leftover data
1681 // that we don't need to satisfy the caller's request
1683 FragmentEntry
= POOL_BLOCK(Entry
, i
);
1684 FragmentEntry
->BlockSize
= Entry
->BlockSize
- i
;
1687 // And make it point back to us
1689 FragmentEntry
->PreviousSize
= i
;
1692 // Now get the block that follows the new fragment and check
1693 // if it's still on the same page as us (and not at the end)
1695 NextEntry
= POOL_NEXT_BLOCK(FragmentEntry
);
1696 if (PAGE_ALIGN(NextEntry
) != NextEntry
)
1699 // Adjust this next block to point to our newly created
1702 NextEntry
->PreviousSize
= FragmentEntry
->BlockSize
;
1708 // There is a free entry before us, which we know is smaller
1709 // so we'll make this entry the fragment instead
1711 FragmentEntry
= Entry
;
1714 // And then we'll remove from it the actual size required.
1715 // Now the entry is a leftover free fragment
1717 Entry
->BlockSize
-= i
;
1720 // Now let's go to the next entry after the fragment (which
1721 // used to point to our original free entry) and make it
1722 // reference the new fragment entry instead.
1724 // This is the entry that will actually end up holding the
1727 Entry
= POOL_NEXT_BLOCK(Entry
);
1728 Entry
->PreviousSize
= FragmentEntry
->BlockSize
;
1731 // And now let's go to the entry after that one and check if
1732 // it's still on the same page, and not at the end
1734 NextEntry
= POOL_BLOCK(Entry
, i
);
1735 if (PAGE_ALIGN(NextEntry
) != NextEntry
)
1738 // Make it reference the allocation entry
1740 NextEntry
->PreviousSize
= i
;
1745 // Now our (allocation) entry is the right size
1747 Entry
->BlockSize
= i
;
1750 // And the next entry is now the free fragment which contains
1751 // the remaining difference between how big the original entry
1752 // was, and the actual size the caller needs/requested.
1754 FragmentEntry
->PoolType
= 0;
1755 BlockSize
= FragmentEntry
->BlockSize
;
1758 // Now check if enough free bytes remained for us to have a
1759 // "full" entry, which contains enough bytes for a linked list
1760 // and thus can be used for allocations (up to 8 bytes...)
1762 ExpCheckPoolLinks(&PoolDesc
->ListHeads
[BlockSize
- 1]);
1766 // Insert the free entry into the free list for this size
1768 ExpInsertPoolTailList(&PoolDesc
->ListHeads
[BlockSize
- 1],
1769 POOL_FREE_BLOCK(FragmentEntry
));
1770 ExpCheckPoolLinks(POOL_FREE_BLOCK(FragmentEntry
));
1775 // We have found an entry for this allocation, so set the pool type
1776 // and release the lock since we're done
1778 Entry
->PoolType
= PoolType
+ 1;
1779 ExpCheckPoolBlocks(Entry
);
1780 ExUnlockPool(PoolDesc
, OldIrql
);
1783 // Increment required counters
1785 InterlockedExchangeAddSizeT(&PoolDesc
->TotalBytes
, Entry
->BlockSize
* POOL_BLOCK_SIZE
);
1786 InterlockedIncrement((PLONG
)&PoolDesc
->RunningAllocs
);
1789 // Track this allocation
1791 ExpInsertPoolTracker(Tag
,
1792 Entry
->BlockSize
* POOL_BLOCK_SIZE
,
1796 // Return the pool allocation
1798 Entry
->PoolTag
= Tag
;
1799 (POOL_FREE_BLOCK(Entry
))->Flink
= NULL
;
1800 (POOL_FREE_BLOCK(Entry
))->Blink
= NULL
;
1801 return POOL_FREE_BLOCK(Entry
);
1803 } while (++ListHead
!= &PoolDesc
->ListHeads
[POOL_LISTS_PER_PAGE
]);
1806 // There were no free entries left, so we have to allocate a new fresh page
1808 Entry
= MiAllocatePoolPages(PoolType
, PAGE_SIZE
);
1812 // Must succeed pool is deprecated, but still supported. These allocation
1813 // failures must cause an immediate bugcheck
1815 if (OriginalType
& MUST_SUCCEED_POOL_MASK
)
1817 KeBugCheckEx(MUST_SUCCEED_POOL_EMPTY
,
1819 NonPagedPoolDescriptor
.TotalPages
,
1820 NonPagedPoolDescriptor
.TotalBigPages
,
1825 // Internal debugging
1830 // This flag requests printing failures, and can also further specify
1831 // breaking on failures
1833 if (ExpPoolFlags
& POOL_FLAG_DBGPRINT_ON_FAILURE
)
1835 DPRINT1("EX: ExAllocatePool (%p, 0x%x) returning NULL\n",
1838 if (ExpPoolFlags
& POOL_FLAG_CRASH_ON_FAILURE
) DbgBreakPoint();
1842 // Finally, this flag requests an exception, which we are more than
1845 if (OriginalType
& POOL_RAISE_IF_ALLOCATION_FAILURE
)
1847 ExRaiseStatus(STATUS_INSUFFICIENT_RESOURCES
);
1851 // Return NULL to the caller in all other cases
1857 // Setup the entry data
1860 Entry
->BlockSize
= i
;
1861 Entry
->PoolType
= PoolType
+ 1;
1864 // This page will have two entries -- one for the allocation (which we just
1865 // created above), and one for the remaining free bytes, which we're about
1866 // to create now. The free bytes are the whole page minus what was allocated
1867 // and then converted into units of block headers.
1869 BlockSize
= (PAGE_SIZE
/ POOL_BLOCK_SIZE
) - i
;
1870 FragmentEntry
= POOL_BLOCK(Entry
, i
);
1871 FragmentEntry
->Ulong1
= 0;
1872 FragmentEntry
->BlockSize
= BlockSize
;
1873 FragmentEntry
->PreviousSize
= i
;
1876 // Increment required counters
1878 InterlockedIncrement((PLONG
)&PoolDesc
->TotalPages
);
1879 InterlockedExchangeAddSizeT(&PoolDesc
->TotalBytes
, Entry
->BlockSize
* POOL_BLOCK_SIZE
);
1882 // Now check if enough free bytes remained for us to have a "full" entry,
1883 // which contains enough bytes for a linked list and thus can be used for
1884 // allocations (up to 8 bytes...)
1886 if (FragmentEntry
->BlockSize
!= 1)
1889 // Excellent -- acquire the pool lock
1891 OldIrql
= ExLockPool(PoolDesc
);
1894 // And insert the free entry into the free list for this block size
1896 ExpCheckPoolLinks(&PoolDesc
->ListHeads
[BlockSize
- 1]);
1897 ExpInsertPoolTailList(&PoolDesc
->ListHeads
[BlockSize
- 1],
1898 POOL_FREE_BLOCK(FragmentEntry
));
1899 ExpCheckPoolLinks(POOL_FREE_BLOCK(FragmentEntry
));
1902 // Release the pool lock
1904 ExpCheckPoolBlocks(Entry
);
1905 ExUnlockPool(PoolDesc
, OldIrql
);
1910 // Simply do a sanity check
1912 ExpCheckPoolBlocks(Entry
);
1916 // Increment performance counters and track this allocation
1918 InterlockedIncrement((PLONG
)&PoolDesc
->RunningAllocs
);
1919 ExpInsertPoolTracker(Tag
,
1920 Entry
->BlockSize
* POOL_BLOCK_SIZE
,
1924 // And return the pool allocation
1926 ExpCheckPoolBlocks(Entry
);
1927 Entry
->PoolTag
= Tag
;
1928 return POOL_FREE_BLOCK(Entry
);
1936 ExAllocatePool(POOL_TYPE PoolType
,
1937 SIZE_T NumberOfBytes
)
1940 // Use a default tag of "None"
1942 return ExAllocatePoolWithTag(PoolType
, NumberOfBytes
, TAG_NONE
);
1950 ExFreePoolWithTag(IN PVOID P
,
1953 PPOOL_HEADER Entry
, NextEntry
;
1957 PPOOL_DESCRIPTOR PoolDesc
;
1959 BOOLEAN Combined
= FALSE
;
1960 PFN_NUMBER PageCount
, RealPageCount
;
1961 PKPRCB Prcb
= KeGetCurrentPrcb();
1962 PGENERAL_LOOKASIDE LookasideList
;
1965 // Check if any of the debug flags are enabled
1967 if (ExpPoolFlags
& (POOL_FLAG_CHECK_TIMERS
|
1968 POOL_FLAG_CHECK_WORKERS
|
1969 POOL_FLAG_CHECK_RESOURCES
|
1970 POOL_FLAG_VERIFIER
|
1971 POOL_FLAG_CHECK_DEADLOCK
|
1972 POOL_FLAG_SPECIAL_POOL
))
1975 // Check if special pool is enabled
1977 if (ExpPoolFlags
& POOL_FLAG_SPECIAL_POOL
)
1980 // Check if it was allocated from a special pool
1982 if (MmIsSpecialPoolAddress(P
))
1985 // Was deadlock verification also enabled? We can do some extra
1986 // checks at this point
1988 if (ExpPoolFlags
& POOL_FLAG_CHECK_DEADLOCK
)
1990 DPRINT1("Verifier not yet supported\n");
1994 // It is, so handle it via special pool free routine
1996 MmFreeSpecialPool(P
);
2002 // For non-big page allocations, we'll do a bunch of checks in here
2004 if (PAGE_ALIGN(P
) != P
)
2007 // Get the entry for this pool allocation
2008 // The pointer math here may look wrong or confusing, but it is quite right
2014 // Get the pool type
2016 PoolType
= (Entry
->PoolType
- 1) & BASE_POOL_TYPE_MASK
;
2019 // FIXME: Many other debugging checks go here
2021 ExpCheckPoolIrqlLevel(PoolType
, 0, P
);
2026 // Check if this is a big page allocation
2028 if (PAGE_ALIGN(P
) == P
)
2031 // We need to find the tag for it, so first we need to find out what
2032 // kind of allocation this was (paged or nonpaged), then we can go
2033 // ahead and try finding the tag for it. Remember to get rid of the
2034 // PROTECTED_POOL tag if it's found.
2036 // Note that if at insertion time, we failed to add the tag for a big
2037 // pool allocation, we used a special tag called 'BIG' to identify the
2038 // allocation, and we may get this tag back. In this scenario, we must
2039 // manually get the size of the allocation by actually counting through
2040 // the PFN database.
2042 PoolType
= MmDeterminePoolType(P
);
2043 ExpCheckPoolIrqlLevel(PoolType
, 0, P
);
2044 Tag
= ExpFindAndRemoveTagBigPages(P
, &PageCount
, PoolType
);
2047 DPRINT1("We do not know the size of this allocation. This is not yet supported\n");
2048 ASSERT(Tag
== ' GIB');
2049 PageCount
= 1; // We are going to lie! This might screw up accounting?
2051 else if (Tag
& PROTECTED_POOL
)
2053 Tag
&= ~PROTECTED_POOL
;
2057 // We have our tag and our page count, so we can go ahead and remove this
2060 ExpRemovePoolTracker(Tag
, PageCount
<< PAGE_SHIFT
, PoolType
);
2063 // Check if any of the debug flags are enabled
2065 if (ExpPoolFlags
& (POOL_FLAG_CHECK_TIMERS
|
2066 POOL_FLAG_CHECK_WORKERS
|
2067 POOL_FLAG_CHECK_RESOURCES
|
2068 POOL_FLAG_CHECK_DEADLOCK
))
2071 // Was deadlock verification also enabled? We can do some extra
2072 // checks at this point
2074 if (ExpPoolFlags
& POOL_FLAG_CHECK_DEADLOCK
)
2076 DPRINT1("Verifier not yet supported\n");
2080 // FIXME: Many debugging checks go here
2087 PoolDesc
= PoolVector
[PoolType
];
2088 InterlockedIncrement((PLONG
)&PoolDesc
->RunningDeAllocs
);
2089 InterlockedExchangeAddSizeT(&PoolDesc
->TotalBytes
,
2090 -(LONG_PTR
)(PageCount
<< PAGE_SHIFT
));
2093 // Do the real free now and update the last counter with the big page count
2095 RealPageCount
= MiFreePoolPages(P
);
2096 ASSERT(RealPageCount
== PageCount
);
2097 InterlockedExchangeAdd((PLONG
)&PoolDesc
->TotalBigPages
,
2098 -(LONG
)RealPageCount
);
2103 // Get the entry for this pool allocation
2104 // The pointer math here may look wrong or confusing, but it is quite right
2110 // Get the size of the entry, and it's pool type, then load the descriptor
2111 // for this pool type
2113 BlockSize
= Entry
->BlockSize
;
2114 PoolType
= (Entry
->PoolType
- 1) & BASE_POOL_TYPE_MASK
;
2115 PoolDesc
= PoolVector
[PoolType
];
2118 // Make sure that the IRQL makes sense
2120 ExpCheckPoolIrqlLevel(PoolType
, 0, P
);
2123 // Get the pool tag and get rid of the PROTECTED_POOL flag
2125 Tag
= Entry
->PoolTag
;
2126 if (Tag
& PROTECTED_POOL
) Tag
&= ~PROTECTED_POOL
;
2129 // Stop tracking this allocation
2131 ExpRemovePoolTracker(Tag
,
2132 BlockSize
* POOL_BLOCK_SIZE
,
2133 Entry
->PoolType
- 1);
2136 // Is this allocation small enough to have come from a lookaside list?
2138 if (BlockSize
<= MAXIMUM_PROCESSORS
)
2141 // Try pushing it into the per-CPU lookaside list
2143 LookasideList
= (PoolType
== PagedPool
) ?
2144 Prcb
->PPPagedLookasideList
[BlockSize
- 1].P
:
2145 Prcb
->PPNPagedLookasideList
[BlockSize
- 1].P
;
2146 LookasideList
->TotalFrees
++;
2147 if (ExQueryDepthSList(&LookasideList
->ListHead
) < LookasideList
->Depth
)
2149 LookasideList
->FreeHits
++;
2150 InterlockedPushEntrySList(&LookasideList
->ListHead
, P
);
2155 // We failed, try to push it into the global lookaside list
2157 LookasideList
= (PoolType
== PagedPool
) ?
2158 Prcb
->PPPagedLookasideList
[BlockSize
- 1].L
:
2159 Prcb
->PPNPagedLookasideList
[BlockSize
- 1].L
;
2160 LookasideList
->TotalFrees
++;
2161 if (ExQueryDepthSList(&LookasideList
->ListHead
) < LookasideList
->Depth
)
2163 LookasideList
->FreeHits
++;
2164 InterlockedPushEntrySList(&LookasideList
->ListHead
, P
);
2170 // Get the pointer to the next entry
2172 NextEntry
= POOL_BLOCK(Entry
, BlockSize
);
2175 // Update performance counters
2177 InterlockedIncrement((PLONG
)&PoolDesc
->RunningDeAllocs
);
2178 InterlockedExchangeAddSizeT(&PoolDesc
->TotalBytes
, -BlockSize
* POOL_BLOCK_SIZE
);
2181 // Acquire the pool lock
2183 OldIrql
= ExLockPool(PoolDesc
);
2188 if (TagToFree
&& TagToFree
!= Entry
->PoolTag
)
2190 DPRINT1("Freeing pool - invalid tag specified: %.4s != %.4s\n", (char*)&TagToFree
, (char*)&Entry
->PoolTag
);
2191 KeBugCheckEx(BAD_POOL_CALLER
, 0x0A, (ULONG_PTR
)P
, Entry
->PoolTag
, TagToFree
);
2195 // Check if the next allocation is at the end of the page
2197 ExpCheckPoolBlocks(Entry
);
2198 if (PAGE_ALIGN(NextEntry
) != NextEntry
)
2201 // We may be able to combine the block if it's free
2203 if (NextEntry
->PoolType
== 0)
2206 // The next block is free, so we'll do a combine
2211 // Make sure there's actual data in the block -- anything smaller
2212 // than this means we only have the header, so there's no linked list
2215 if ((NextEntry
->BlockSize
!= 1))
2218 // The block is at least big enough to have a linked list, so go
2219 // ahead and remove it
2221 ExpCheckPoolLinks(POOL_FREE_BLOCK(NextEntry
));
2222 ExpRemovePoolEntryList(POOL_FREE_BLOCK(NextEntry
));
2223 ExpCheckPoolLinks(ExpDecodePoolLink((POOL_FREE_BLOCK(NextEntry
))->Flink
));
2224 ExpCheckPoolLinks(ExpDecodePoolLink((POOL_FREE_BLOCK(NextEntry
))->Blink
));
2228 // Our entry is now combined with the next entry
2230 Entry
->BlockSize
= Entry
->BlockSize
+ NextEntry
->BlockSize
;
2235 // Now check if there was a previous entry on the same page as us
2237 if (Entry
->PreviousSize
)
2240 // Great, grab that entry and check if it's free
2242 NextEntry
= POOL_PREV_BLOCK(Entry
);
2243 if (NextEntry
->PoolType
== 0)
2246 // It is, so we can do a combine
2251 // Make sure there's actual data in the block -- anything smaller
2252 // than this means we only have the header so there's no linked list
2255 if ((NextEntry
->BlockSize
!= 1))
2258 // The block is at least big enough to have a linked list, so go
2259 // ahead and remove it
2261 ExpCheckPoolLinks(POOL_FREE_BLOCK(NextEntry
));
2262 ExpRemovePoolEntryList(POOL_FREE_BLOCK(NextEntry
));
2263 ExpCheckPoolLinks(ExpDecodePoolLink((POOL_FREE_BLOCK(NextEntry
))->Flink
));
2264 ExpCheckPoolLinks(ExpDecodePoolLink((POOL_FREE_BLOCK(NextEntry
))->Blink
));
2268 // Combine our original block (which might've already been combined
2269 // with the next block), into the previous block
2271 NextEntry
->BlockSize
= NextEntry
->BlockSize
+ Entry
->BlockSize
;
2274 // And now we'll work with the previous block instead
2281 // By now, it may have been possible for our combined blocks to actually
2282 // have made up a full page (if there were only 2-3 allocations on the
2283 // page, they could've all been combined).
2285 if ((PAGE_ALIGN(Entry
) == Entry
) &&
2286 (PAGE_ALIGN(POOL_NEXT_BLOCK(Entry
)) == POOL_NEXT_BLOCK(Entry
)))
2289 // In this case, release the pool lock, update the performance counter,
2290 // and free the page
2292 ExUnlockPool(PoolDesc
, OldIrql
);
2293 InterlockedExchangeAdd((PLONG
)&PoolDesc
->TotalPages
, -1);
2294 MiFreePoolPages(Entry
);
2299 // Otherwise, we now have a free block (or a combination of 2 or 3)
2301 Entry
->PoolType
= 0;
2302 BlockSize
= Entry
->BlockSize
;
2303 ASSERT(BlockSize
!= 1);
2306 // Check if we actually did combine it with anyone
2311 // Get the first combined block (either our original to begin with, or
2312 // the one after the original, depending if we combined with the previous)
2314 NextEntry
= POOL_NEXT_BLOCK(Entry
);
2317 // As long as the next block isn't on a page boundary, have it point
2320 if (PAGE_ALIGN(NextEntry
) != NextEntry
) NextEntry
->PreviousSize
= BlockSize
;
2324 // Insert this new free block, and release the pool lock
2326 ExpInsertPoolHeadList(&PoolDesc
->ListHeads
[BlockSize
- 1], POOL_FREE_BLOCK(Entry
));
2327 ExpCheckPoolLinks(POOL_FREE_BLOCK(Entry
));
2328 ExUnlockPool(PoolDesc
, OldIrql
);
2339 // Just free without checking for the tag
2341 ExFreePoolWithTag(P
, 0);
2349 ExQueryPoolBlockSize(IN PVOID PoolBlock
,
2350 OUT PBOOLEAN QuotaCharged
)
2365 ExAllocatePoolWithQuota(IN POOL_TYPE PoolType
,
2366 IN SIZE_T NumberOfBytes
)
2369 // Allocate the pool
2371 return ExAllocatePoolWithQuotaTag(PoolType
, NumberOfBytes
, 'enoN');
2379 ExAllocatePoolWithTagPriority(IN POOL_TYPE PoolType
,
2380 IN SIZE_T NumberOfBytes
,
2382 IN EX_POOL_PRIORITY Priority
)
2385 // Allocate the pool
2388 return ExAllocatePoolWithTag(PoolType
, NumberOfBytes
, Tag
);
2396 ExAllocatePoolWithQuotaTag(IN POOL_TYPE PoolType
,
2397 IN SIZE_T NumberOfBytes
,
2401 // Allocate the pool
2404 return ExAllocatePoolWithTag(PoolType
, NumberOfBytes
, Tag
);