1 /* COPYRIGHT: See COPYING in the top level directory
2 * PROJECT: ReactOS system libraries
3 * FILE: lib/rtl/heappage.c
4 * PURPOSE: RTL Page Heap implementation
5 * PROGRAMMERS: Copyright 2011 Aleksey Bragin
9 http://msdn.microsoft.com/en-us/library/ms220938(VS.80).aspx
12 /* INCLUDES *****************************************************************/
20 /* TYPES **********************************************************************/
22 typedef struct _DPH_BLOCK_INFORMATION
31 SINGLE_LIST_ENTRY FreePushList
;
36 } DPH_BLOCK_INFORMATION
, *PDPH_BLOCK_INFORMATION
;
38 typedef struct _DPH_HEAP_BLOCK
42 struct _DPH_HEAP_BLOCK
*pNextAlloc
;
43 LIST_ENTRY AvailableEntry
;
44 RTL_BALANCED_LINKS TableLinks
;
46 PUCHAR pUserAllocation
;
48 ULONG nVirtualBlockSize
;
49 ULONG nVirtualAccessSize
;
50 ULONG nUserRequestedSize
;
51 ULONG nUserActualSize
;
54 PRTL_TRACE_BLOCK StackTrace
;
55 LIST_ENTRY AdjacencyEntry
;
56 PUCHAR pVirtualRegion
;
57 } DPH_HEAP_BLOCK
, *PDPH_HEAP_BLOCK
;
59 typedef struct _DPH_HEAP_ROOT
63 PRTL_CRITICAL_SECTION HeapCritSect
;
64 ULONG nRemoteLockAcquired
;
66 PDPH_HEAP_BLOCK pVirtualStorageListHead
;
67 PDPH_HEAP_BLOCK pVirtualStorageListTail
;
68 ULONG nVirtualStorageRanges
;
69 ULONG nVirtualStorageBytes
;
71 RTL_AVL_TABLE BusyNodesTable
;
72 PDPH_HEAP_BLOCK NodeToAllocate
;
73 ULONG nBusyAllocations
;
74 ULONG nBusyAllocationBytesCommitted
;
76 PDPH_HEAP_BLOCK pFreeAllocationListHead
;
77 PDPH_HEAP_BLOCK pFreeAllocationListTail
;
78 ULONG nFreeAllocations
;
79 ULONG nFreeAllocationBytesCommitted
;
81 LIST_ENTRY AvailableAllocationHead
;
82 ULONG nAvailableAllocations
;
83 ULONG nAvailableAllocationBytesCommitted
;
85 PDPH_HEAP_BLOCK pUnusedNodeListHead
;
86 PDPH_HEAP_BLOCK pUnusedNodeListTail
;
88 ULONG nBusyAllocationBytesAccessible
;
89 PDPH_HEAP_BLOCK pNodePoolListHead
;
90 PDPH_HEAP_BLOCK pNodePoolListTail
;
98 PRTL_TRACE_BLOCK CreateStackTrace
;
100 } DPH_HEAP_ROOT
, *PDPH_HEAP_ROOT
;
102 /* GLOBALS ********************************************************************/
104 BOOLEAN RtlpPageHeapEnabled
= FALSE
;
105 ULONG RtlpDphGlobalFlags
;
106 ULONG RtlpPageHeapSizeRangeStart
, RtlpPageHeapSizeRangeEnd
;
107 ULONG RtlpPageHeapDllRangeStart
, RtlpPageHeapDllRangeEnd
;
108 WCHAR RtlpDphTargetDlls
[512];
110 ULONG RtlpDphBreakOptions
;
111 ULONG RtlpDphDebugOptions
;
113 LIST_ENTRY RtlpDphPageHeapList
;
114 BOOLEAN RtlpDphPageHeapListInitialized
;
115 RTL_CRITICAL_SECTION RtlpDphPageHeapListLock
;
116 ULONG RtlpDphPageHeapListLength
;
117 UNICODE_STRING RtlpDphTargetDllsUnicode
;
121 LONG RtlpDphAllocFails
;
122 LONG RtlpDphReleaseFails
;
123 LONG RtlpDphFreeFails
;
124 LONG RtlpDphProtectFails
;
126 #define DPH_RESERVE_SIZE 0x100000
127 #define DPH_POOL_SIZE 0x4000
129 /* RtlpDphBreakOptions */
130 #define DPH_BREAK_ON_RESERVE_FAIL 0x01
131 #define DPH_BREAK_ON_COMMIT_FAIL 0x02
132 #define DPH_BREAK_ON_RELEASE_FAIL 0x04
133 #define DPH_BREAK_ON_FREE_FAIL 0x08
134 #define DPH_BREAK_ON_PROTECT_FAIL 0x10
136 /* RtlpDphDebugOptions */
137 #define DPH_DEBUG_INTERNAL_VALIDATE 0x01
138 #define DPH_DEBUG_VERBOSE 0x04
141 #define DPH_FILL 0xEEEEEEEE
144 #define DPH_SIGNATURE 0xFFEEDDCC
146 /* FUNCTIONS ******************************************************************/
149 RtlpSecMemFreeVirtualMemory(HANDLE Process
, PVOID
*Base
, PSIZE_T Size
, ULONG Type
)
152 //PVOID *SavedBase = Base;
153 //PSIZE_T SavedSize = Size;
155 /* Free the memory */
156 Status
= ZwFreeVirtualMemory(Process
, Base
, Size
, Type
);
158 /* Flush secure memory cache if needed and retry freeing */
160 if (Status
== STATUS_INVALID_PAGE_PROTECTION
&&
161 Process
== NtCurrentProcess() &&
162 RtlFlushSecureMemoryCache(*SavedBase
, *SavedSize
))
164 Status
= ZwFreeVirtualMemory(NtCurrentProcess(), SavedBase
, SavedSize
, Type
);
172 RtlpDphAllocateVm(PVOID
*Base
, SIZE_T Size
, ULONG Type
, ULONG Protection
)
175 Status
= ZwAllocateVirtualMemory(NtCurrentProcess(),
182 /* Check for failures */
183 if (!NT_SUCCESS(Status
))
185 if (Type
== MEM_RESERVE
)
187 _InterlockedIncrement(&RtlpDphCounter
);
188 if (RtlpDphBreakOptions
& DPH_BREAK_ON_RESERVE_FAIL
)
190 DPRINT1("Page heap: AllocVm (%p, %p, %x) failed with %x \n", Base
, Size
, Type
, Status
);
197 _InterlockedIncrement(&RtlpDphAllocFails
);
198 if (RtlpDphBreakOptions
& DPH_BREAK_ON_COMMIT_FAIL
)
200 DPRINT1("Page heap: AllocVm (%p, %p, %x) failed with %x \n", Base
, Size
, Type
, Status
);
211 RtlpDphFreeVm(PVOID Base
, SIZE_T Size
, ULONG Type
, ULONG Protection
)
215 /* Free the memory */
216 Status
= RtlpSecMemFreeVirtualMemory(NtCurrentProcess(), &Base
, &Size
, Type
);
218 /* Log/report failures */
219 if (!NT_SUCCESS(Status
))
221 if (Type
== MEM_RELEASE
)
223 _InterlockedIncrement(&RtlpDphReleaseFails
);
224 if (RtlpDphBreakOptions
& DPH_BREAK_ON_RELEASE_FAIL
)
226 DPRINT1("Page heap: FreeVm (%p, %p, %x) failed with %x \n", Base
, Size
, Type
, Status
);
233 _InterlockedIncrement(&RtlpDphFreeFails
);
234 if (RtlpDphBreakOptions
& DPH_BREAK_ON_FREE_FAIL
)
236 DPRINT1("Page heap: FreeVm (%p, %p, %x) failed with %x \n", Base
, Size
, Type
, Status
);
247 RtlpDphProtectVm(PVOID Base
, SIZE_T Size
, ULONG Protection
)
252 /* Change protection */
253 Status
= ZwProtectVirtualMemory(NtCurrentProcess(), &Base
, &Size
, Protection
, &OldProtection
);
255 /* Log/report failures */
256 if (!NT_SUCCESS(Status
))
258 _InterlockedIncrement(&RtlpDphProtectFails
);
259 if (RtlpDphBreakOptions
& DPH_BREAK_ON_PROTECT_FAIL
)
261 DPRINT1("Page heap: ProtectVm (%p, %p, %x) failed with %x \n", Base
, Size
, Protection
, Status
);
271 RtlpDphAddNewPool(PDPH_HEAP_ROOT DphRoot
, PDPH_HEAP_BLOCK NodeBlock
, PVOID Virtual
, SIZE_T Size
, BOOLEAN Add
)
273 PDPH_HEAP_BLOCK DphNode
, DphStartNode
;
276 NodeCount
= (Size
>> 6) - 1;
277 DphStartNode
= Virtual
;
279 /* Set pNextAlloc for all blocks */
280 for (DphNode
= Virtual
; NodeCount
> 0; DphNode
++, NodeCount
--)
281 DphNode
->pNextAlloc
= DphNode
+ 1;
283 /* and the last one */
284 DphNode
->pNextAlloc
= NULL
;
286 /* Add it to the tail of unused node list */
287 if (DphRoot
->pUnusedNodeListTail
)
288 DphRoot
->pUnusedNodeListTail
->pNextAlloc
= DphStartNode
;
290 DphRoot
->pUnusedNodeListHead
= DphStartNode
;
292 DphRoot
->pUnusedNodeListTail
= DphNode
;
294 /* Increase counters */
295 DphRoot
->nUnusedNodes
+= NodeCount
;
303 PDPH_HEAP_BLOCK NTAPI
304 RtlpDphFindAvailableMemory(PDPH_HEAP_ROOT DphRoot
,
306 PDPH_HEAP_BLOCK
*Prev
)
313 PDPH_HEAP_BLOCK NTAPI
314 RtlpDphTakeNodeFromUnusedList(PDPH_HEAP_ROOT DphRoot
)
321 RtlpDphReturnNodeToUnusedList(PDPH_HEAP_ROOT DphRoot
,
322 PDPH_HEAP_BLOCK Node
)
327 RtlpDphRemoveFromAvailableList(PDPH_HEAP_ROOT DphRoot
,
328 PDPH_HEAP_BLOCK Node
)
334 RtlpDphCoalesceNodeIntoAvailable(PDPH_HEAP_ROOT DphRoot
,
335 PDPH_HEAP_BLOCK Node
)
340 PDPH_HEAP_BLOCK NTAPI
341 RtlpDphAllocateNode(PDPH_HEAP_ROOT DphRoot
)
343 PDPH_HEAP_BLOCK Node
;
345 ULONG Size
= DPH_POOL_SIZE
;
348 /* Check for the easy case */
349 if (DphRoot
->pUnusedNodeListHead
)
351 /* Just take a node from this list */
352 Node
= RtlpDphTakeNodeFromUnusedList(DphRoot
);
357 /* There is a need to make free space */
358 Node
= RtlpDphFindAvailableMemory(DphRoot
, DPH_POOL_SIZE
, NULL
);
360 if (!DphRoot
->pUnusedNodeListHead
&& !Node
)
362 /* Retry with a smaller request */
364 Node
= RtlpDphFindAvailableMemory(DphRoot
, PAGE_SIZE
, NULL
);
367 if (!DphRoot
->pUnusedNodeListHead
)
371 RtlpDphRemoveFromAvailableList(DphRoot
, Node
);
372 Ptr
= Node
->pVirtualBlock
;
376 /* No free space, need to alloc a new VM block */
377 Size
= DPH_POOL_SIZE
;
378 Status
= RtlpDphAllocateVm(&Ptr
, DPH_RESERVE_SIZE
, MEM_COMMIT
, PAGE_NOACCESS
);
380 if (!NT_SUCCESS(Status
))
382 /* Retry with a smaller size */
383 Status
= RtlpDphAllocateVm(&Ptr
, 0x10000, MEM_COMMIT
, PAGE_NOACCESS
);
384 if (!NT_SUCCESS(Status
)) return NULL
;
388 /* VM is allocated at this point, set protection */
389 Status
= RtlpDphProtectVm(Ptr
, Size
, PAGE_READWRITE
);
390 if (!NT_SUCCESS(Status
))
396 /* Zero the memory */
397 if (Node
) RtlZeroMemory(Ptr
, Size
);
399 /* Add a new pool based on this VM */
400 RtlpDphAddNewPool(DphRoot
, Node
, Ptr
, Size
, TRUE
);
408 if (Node
->nVirtualBlockSize
> Size
)
410 Node
->pVirtualBlock
+= Size
;
411 Node
->nVirtualBlockSize
-= Size
;
413 RtlpDphCoalesceNodeIntoAvailable(DphRoot
, Node
);
417 RtlpDphReturnNodeToUnusedList(DphRoot
, Node
);
422 return RtlpDphTakeNodeFromUnusedList(DphRoot
);
426 RtlpDphPlaceOnPoolList(PDPH_HEAP_ROOT DphRoot
, PDPH_HEAP_BLOCK DphNode
)
432 RtlpDphPlaceOnVirtualList(PDPH_HEAP_ROOT DphRoot
, PDPH_HEAP_BLOCK DphNode
)
437 RTL_GENERIC_COMPARE_RESULTS
439 RtlpDphCompareNodeForTable(IN PRTL_AVL_TABLE Table
,
440 IN PVOID FirstStruct
,
441 IN PVOID SecondStruct
)
450 RtlpDphAllocateNodeForTable(IN PRTL_AVL_TABLE Table
,
460 RtlpDphFreeNodeForTable(IN PRTL_AVL_TABLE Table
,
468 RtlpDphInitializeDelayedFreeQueue()
471 return STATUS_SUCCESS
;
475 RtlpDphTargetDllsLogicInitialize()
478 return STATUS_SUCCESS
;
482 RtlpDphInternalValidatePageHeap(PDPH_HEAP_ROOT DphRoot
, PVOID Address
, ULONG Value
)
488 RtlpDphProcessStartupInitialization()
491 PTEB Teb
= NtCurrentTeb();
493 /* Initialize the DPH heap list and its critical section */
494 InitializeListHead(&RtlpDphPageHeapList
);
495 Status
= RtlInitializeCriticalSection(&RtlpDphPageHeapListLock
);
496 if (!NT_SUCCESS(Status
))
502 /* Initialize delayed-free queue */
503 Status
= RtlpDphInitializeDelayedFreeQueue();
504 if (!NT_SUCCESS(Status
)) return Status
;
506 /* Initialize the target dlls string */
507 RtlInitUnicodeString(&RtlpDphTargetDllsUnicode
, RtlpDphTargetDlls
);
508 Status
= RtlpDphTargetDllsLogicInitialize();
510 /* Per-process DPH init is done */
511 RtlpDphPageHeapListInitialized
= TRUE
;
513 DPRINT1("Page heap: pid 0x%X: page heap enabled with flags 0x%X.\n", Teb
->ClientId
.UniqueProcess
, RtlpDphGlobalFlags
);
519 RtlpPageHeapCreate(ULONG Flags
,
524 PRTL_HEAP_PARAMETERS Parameters
)
528 PDPH_HEAP_ROOT DphRoot
;
529 PDPH_HEAP_BLOCK DphNode
;
532 LARGE_INTEGER PerfCounter
;
534 /* Check for a DPH bypass flag */
535 if ((ULONG_PTR
)Parameters
== -1) return NULL
;
537 /* Make sure no user-allocated stuff was provided */
538 if (Addr
|| Lock
) return NULL
;
540 /* Allocate minimum amount of virtual memory */
541 MemSize
= DPH_RESERVE_SIZE
;
542 Status
= RtlpDphAllocateVm(&Base
, MemSize
, MEM_COMMIT
, PAGE_NOACCESS
);
543 if (!NT_SUCCESS(Status
))
550 Status
= RtlpDphProtectVm(Base
, 2*PAGE_SIZE
+ DPH_POOL_SIZE
, PAGE_READWRITE
);
551 if (!NT_SUCCESS(Status
))
553 //RtlpDphFreeVm(Base, 0, 0, 0);
558 /* Start preparing the 1st page. Fill it with the default filler */
559 RtlFillMemory(Base
, PAGE_SIZE
, DPH_FILL
);
561 /* Set flags in the "HEAP" structure */
562 HeapPtr
= (PHEAP
)Base
;
563 HeapPtr
->Flags
= Flags
| HEAP_FLAG_PAGE_ALLOCS
;
564 HeapPtr
->ForceFlags
= Flags
| HEAP_FLAG_PAGE_ALLOCS
;
566 /* Set 1st page to read only now */
567 Status
= RtlpDphProtectVm(Base
, PAGE_SIZE
, PAGE_READONLY
);
568 if (!NT_SUCCESS(Status
))
574 /* 2nd page is the real DPH root block */
575 DphRoot
= (PDPH_HEAP_ROOT
)((PCHAR
)Base
+ PAGE_SIZE
);
577 /* Initialize the DPH root */
578 DphRoot
->Signature
= DPH_SIGNATURE
;
579 DphRoot
->HeapFlags
= Flags
;
580 DphRoot
->HeapCritSect
= (PRTL_CRITICAL_SECTION
)((PCHAR
)DphRoot
+ DPH_POOL_SIZE
);
581 DphRoot
->ExtraFlags
= RtlpDphGlobalFlags
;
583 ZwQueryPerformanceCounter(&PerfCounter
, NULL
);
584 DphRoot
->Seed
= PerfCounter
.LowPart
;
586 RtlInitializeCriticalSection(DphRoot
->HeapCritSect
);
588 /* Create a normal heap for this paged heap */
589 DphRoot
->NormalHeap
= RtlCreateHeap(Flags
, NULL
, TotalSize
, CommitSize
, NULL
, (PRTL_HEAP_PARAMETERS
)-1);
590 if (!DphRoot
->NormalHeap
)
596 /* 3rd page: a pool for DPH allocations */
597 RtlpDphAddNewPool(DphRoot
, NULL
, DphRoot
+ 1, DPH_POOL_SIZE
- sizeof(DPH_HEAP_ROOT
), FALSE
);
599 /* Allocate internal heap blocks. For the root */
600 DphNode
= RtlpDphAllocateNode(DphRoot
);
601 ASSERT(DphNode
!= NULL
);
602 DphNode
->pVirtualBlock
= (PUCHAR
)DphRoot
;
603 DphNode
->nVirtualBlockSize
= DPH_POOL_SIZE
;
604 RtlpDphPlaceOnPoolList(DphRoot
, DphNode
);
606 /* For the memory we allocated as a whole */
607 DphNode
= RtlpDphAllocateNode(DphRoot
);
608 ASSERT(DphNode
!= NULL
);
609 DphNode
->pVirtualBlock
= Base
;
610 DphNode
->nVirtualBlockSize
= MemSize
;
611 RtlpDphPlaceOnVirtualList(DphRoot
, DphNode
);
613 /* For the remaining part */
614 DphNode
= RtlpDphAllocateNode(DphRoot
);
615 ASSERT(DphNode
!= NULL
);
616 DphNode
->pVirtualBlock
= (PUCHAR
)Base
+ 2*PAGE_SIZE
+ DPH_POOL_SIZE
;
617 DphNode
->nVirtualBlockSize
= MemSize
- (2*PAGE_SIZE
+ DPH_POOL_SIZE
);
618 RtlpDphCoalesceNodeIntoAvailable(DphRoot
, DphNode
);
620 //DphRoot->CreateStackTrace = RtlpDphLogStackTrace(1);
622 /* Initialize AVL-based busy nodes table */
623 RtlInitializeGenericTableAvl(&DphRoot
->BusyNodesTable
,
624 RtlpDphCompareNodeForTable
,
625 RtlpDphAllocateNodeForTable
,
626 RtlpDphFreeNodeForTable
,
629 /* Initialize per-process startup info */
630 if (!RtlpDphPageHeapListInitialized
) RtlpDphProcessStartupInitialization();
632 /* Acquire the heap list lock */
633 RtlEnterCriticalSection(&RtlpDphPageHeapListLock
);
635 /* Insert this heap to the tail of the global list */
636 InsertTailList(&RtlpDphPageHeapList
, &DphRoot
->NextHeap
);
638 /* Note we increased the size of the list */
639 RtlpDphPageHeapListLength
++;
641 /* Release the heap list lock */
642 RtlLeaveCriticalSection(&RtlpDphPageHeapListLock
);
644 if (RtlpDphDebugOptions
& DPH_DEBUG_VERBOSE
)
646 DPRINT1("Page heap: process 0x%X created heap @ %p (%p, flags 0x%X)\n",
647 NtCurrentTeb()->ClientId
.UniqueProcess
, (PUCHAR
)DphRoot
- PAGE_SIZE
, DphRoot
->NormalHeap
, DphRoot
->ExtraFlags
);
650 /* Perform internal validation if required */
651 if (RtlpDphDebugOptions
& DPH_DEBUG_INTERNAL_VALIDATE
)
652 RtlpDphInternalValidatePageHeap(DphRoot
, NULL
, 0);
654 return (PUCHAR
)DphRoot
- PAGE_SIZE
;
658 RtlpPageHeapDestroy(HANDLE HeapPtr
)
664 RtlpPageHeapAllocate(IN PVOID HeapPtr
,
672 RtlpPageHeapFree(HANDLE HeapPtr
,
680 RtlpPageHeapReAllocate(HANDLE HeapPtr
,
689 RtlpPageHeapGetUserInfo(PVOID HeapHandle
,
699 RtlpPageHeapSetUserValue(PVOID HeapHandle
,
709 RtlpPageHeapSetUserFlags(PVOID HeapHandle
,
712 ULONG UserFlagsReset
,
719 RtlpPageHeapSize(HANDLE HeapPtr
,