3 * Copyright (C) 2011 Timo Kreuzer (timo.kreuzer@reactos.org)
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License as published by
7 * the Free Software Foundation; either version 2 of the License, or
8 * (at your option) any later version.
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
15 * You should have received a copy of the GNU General Public License along
16 * with this program; if not, write to the Free Software Foundation, Inc.,
17 * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
23 DBG_DEFAULT_CHANNEL(HEAP
);
25 #define FREELDR_HEAP_VERIFIER
27 #define REDZONE_MARK 0xCCCCCCCCCCCCCCCCULL
28 #define REDZONE_ALLOCATION 24
29 #define REDZONE_LOW_OFFSET 16
30 #define REDZONE_SIZE(Block) ((ULONG64*)Block->Data)
31 #define REDZONE_LOW(Block) ((ULONG64*)Block->Data + 1)
32 #define REDZONE_HI(Block) ((ULONG64*)((PUCHAR)Block->Data + 16 + *REDZONE_SIZE(Block)))
34 PVOID FrLdrDefaultHeap
;
37 typedef struct _BLOCK_DATA
41 } BLOCK_DATA
, *PBLOCK_DATA
;
43 typedef struct _HEAP_BLOCK
49 } HEAP_BLOCK
, *PHEAP_BLOCK
;
54 SIZE_T CurrentAllocBytes
;
58 SIZE_T LargestAllocation
;
59 ULONGLONG AllocationTime
;
61 ULONG_PTR TerminatingBlock
;
68 TYPE_OF_MEMORY MemoryType
)
74 TRACE("HeapCreate(MemoryType=%ld)\n", MemoryType
);
76 /* Allocate some memory for the heap */
77 MaximumSize
= ALIGN_UP_BY(MaximumSize
, MM_PAGE_SIZE
);
78 Heap
= MmAllocateMemoryWithType(MaximumSize
, MemoryType
);
81 ERR("HEAP: Failed to allocate heap of size 0x%lx, Type\n",
82 MaximumSize
, MemoryType
);
86 /* Initialize the heap header */
87 Heap
->MaximumSize
= MaximumSize
;
88 Heap
->CurrentAllocBytes
= 0;
89 Heap
->MaxAllocBytes
= 0;
92 Heap
->LargestAllocation
= 0;
94 /* Calculate what's left to process */
95 Remaining
= (MaximumSize
- sizeof(HEAP
)) / sizeof(HEAP_BLOCK
);
96 TRACE("Remaining = %ld\n", Remaining
);
98 /* Substract 2 for the terminating entry (header + free entry) */
101 Block
= &Heap
->Blocks
;
104 /* Create free blocks */
105 while (Remaining
> 1)
107 /* Initialize this free block */
108 Block
->Size
= (USHORT
)min(MAXUSHORT
, Remaining
- 1);
109 Block
->PreviousSize
= PreviousSize
;
111 Block
->Data
[0].Flink
= (Block
- &Heap
->Blocks
) + Block
->Size
+ 1;
112 Block
->Data
[0].Blink
= (Block
- &Heap
->Blocks
) - 1 - PreviousSize
;
114 /* Substract current block size from remainder */
115 Remaining
-= (Block
->Size
+ 1);
117 /* Go to next block */
118 PreviousSize
= Block
->Size
;
119 Block
= Block
+ Block
->Size
+ 1;
121 TRACE("Remaining = %ld\n", Remaining
);
124 /* Now finish with a terminating block */
125 Heap
->TerminatingBlock
= Block
- &Heap
->Blocks
;
127 Block
->PreviousSize
= PreviousSize
;
129 Block
->Data
[0].Flink
= 0;
130 Block
->Data
[0].Blink
= (Block
- &Heap
->Blocks
) - 1 - PreviousSize
;
131 Heap
->Blocks
.Data
[0].Blink
= Heap
->TerminatingBlock
;
140 PHEAP Heap
= HeapHandle
;
142 /* Mark all pages as firmware temporary, so they are free for the kernel */
143 MmMarkPagesInLookupTable(PageLookupTableAddress
,
144 (ULONG_PTR
)Heap
/ MM_PAGE_SIZE
,
145 (PFN_COUNT
)(Heap
->MaximumSize
/ MM_PAGE_SIZE
),
146 LoaderFirmwareTemporary
);
149 /* Make sure everything is dead */
150 RtlFillMemory(Heap
, Heap
->MaximumSize
, 0xCCCCCCCC);
154 #ifdef FREELDR_HEAP_VERIFIER
159 PHEAP Heap
= HeapHandle
;
162 /* Loop all heap chunks */
163 for (Block
= &Heap
->Blocks
;
165 Block
= Block
+ 1 + Block
->Size
)
167 /* Continue, if its not free */
170 /* Verify size and redzones */
171 ASSERT(*REDZONE_SIZE(Block
) <= Block
->Size
* sizeof(HEAP_BLOCK
));
172 ASSERT(*REDZONE_LOW(Block
) == REDZONE_MARK
);
173 ASSERT(*REDZONE_HI(Block
) == REDZONE_MARK
);
178 #endif /* FREELDR_HEAP_VERIFIER */
184 PHEAP Heap
= HeapHandle
;
186 PUCHAR StartAddress
, EndAddress
;
187 PFN_COUNT FreePages
, AllFreePages
= 0;
189 TRACE("HeapRelease(%p)\n", HeapHandle
);
191 /* Loop all heap chunks */
192 for (Block
= &Heap
->Blocks
;
194 Block
= Block
+ 1 + Block
->Size
)
196 /* Continue, if its not free */
199 #ifdef FREELDR_HEAP_VERIFIER
200 /* Verify size and redzones */
201 ASSERT(*REDZONE_SIZE(Block
) <= Block
->Size
* sizeof(HEAP_BLOCK
));
202 ASSERT(*REDZONE_LOW(Block
) == REDZONE_MARK
);
203 ASSERT(*REDZONE_HI(Block
) == REDZONE_MARK
);
208 /* Calculate page aligned start address of the free region */
209 StartAddress
= ALIGN_UP_POINTER_BY(Block
->Data
, PAGE_SIZE
);
211 /* Walk over adjacent free blocks */
212 while (Block
->Tag
== 0) Block
= Block
+ Block
->Size
+ 1;
214 /* Check if this was the last block */
215 if (Block
->Size
== 0)
217 /* Align the end address up to cover the end of the heap */
218 EndAddress
= ALIGN_UP_POINTER_BY(Block
->Data
, PAGE_SIZE
);
222 /* Align the end address down to not cover any allocations */
223 EndAddress
= ALIGN_DOWN_POINTER_BY(Block
->Data
, PAGE_SIZE
);
226 /* Check if we have free pages */
227 if (EndAddress
> StartAddress
)
229 /* Calculate the size of the free region in pages */
230 FreePages
= (PFN_COUNT
)((EndAddress
- StartAddress
) / MM_PAGE_SIZE
);
231 AllFreePages
+= FreePages
;
233 /* Now mark the pages free */
234 MmMarkPagesInLookupTable(PageLookupTableAddress
,
235 (ULONG_PTR
)StartAddress
/ MM_PAGE_SIZE
,
240 /* bail out, if it was the last block */
241 if (Block
->Size
== 0) break;
244 TRACE("HeapRelease() done, freed %lu of %lu pages\n", AllFreePages
, Heap
->MaximumSize
/ MM_PAGE_SIZE
);
248 FrLdrHeapCleanupAll(VOID
)
253 Heap
= FrLdrDefaultHeap
;
254 TRACE("Heap statistics for default heap:\n"
255 "CurrentAlloc=0x%lx, MaxAlloc=0x%lx, LargestAllocation=0x%lx\n"
256 "NumAllocs=%ld, NumFrees=%ld\n",
257 Heap
->CurrentAllocBytes
, Heap
->MaxAllocBytes
, Heap
->LargestAllocation
,
258 Heap
->NumAllocs
, Heap
->NumFrees
);
259 TRACE("AllocTime = %I64d, FreeTime = %I64d, sum = %I64d\n",
260 Heap
->AllocationTime
, Heap
->FreeTime
, Heap
->AllocationTime
+ Heap
->FreeTime
);
263 /* Release free pages from the default heap */
264 FrLdrHeapRelease(FrLdrDefaultHeap
);
267 Heap
= FrLdrTempHeap
;
268 TRACE("Heap statistics for temp heap:\n"
269 "CurrentAlloc=0x%lx, MaxAlloc=0x%lx, LargestAllocation=0x%lx\n"
270 "NumAllocs=%ld, NumFrees=%ld\n",
271 Heap
->CurrentAllocBytes
, Heap
->MaxAllocBytes
, Heap
->LargestAllocation
,
272 Heap
->NumAllocs
, Heap
->NumFrees
);
275 /* Destroy the temp heap */
276 FrLdrHeapDestroy(FrLdrTempHeap
);
280 FrLdrHeapRemoveFreeList(
284 PHEAP_BLOCK Previous
, Next
;
286 Next
= &Heap
->Blocks
+ Block
->Data
[0].Flink
;
287 Previous
= &Heap
->Blocks
+ Block
->Data
[0].Blink
;
288 ASSERT((Next
->Tag
== 0) || (Next
->Tag
== 'dnE#'));
289 ASSERT(Next
->Data
[0].Blink
== Block
- &Heap
->Blocks
);
290 ASSERT((Previous
->Tag
== 0) || (Previous
->Tag
== 'dnE#'));
291 ASSERT(Previous
->Data
[0].Flink
== Block
- &Heap
->Blocks
);
293 Next
->Data
[0].Blink
= Previous
- &Heap
->Blocks
;
294 Previous
->Data
[0].Flink
= Next
- &Heap
->Blocks
;
298 FrLdrHeapInsertFreeList(
300 PHEAP_BLOCK FreeBlock
)
302 PHEAP_BLOCK ListHead
, NextBlock
;
303 ASSERT(FreeBlock
->Tag
== 0);
305 /* Terminating block serves as free list head */
306 ListHead
= &Heap
->Blocks
+ Heap
->TerminatingBlock
;
308 for (NextBlock
= &Heap
->Blocks
+ ListHead
->Data
[0].Flink
;
309 NextBlock
< FreeBlock
;
310 NextBlock
= &Heap
->Blocks
+ NextBlock
->Data
[0].Flink
);
312 FreeBlock
->Data
[0].Flink
= NextBlock
- &Heap
->Blocks
;
313 FreeBlock
->Data
[0].Blink
= NextBlock
->Data
[0].Blink
;
314 NextBlock
->Data
[0].Blink
= FreeBlock
- &Heap
->Blocks
;
315 NextBlock
= &Heap
->Blocks
+ FreeBlock
->Data
[0].Blink
;
316 NextBlock
->Data
[0].Flink
= FreeBlock
- &Heap
->Blocks
;
325 PHEAP Heap
= HeapHandle
;
326 PHEAP_BLOCK Block
, NextBlock
;
327 USHORT BlockSize
, Remaining
;
328 #if DBG && !defined(_M_ARM)
329 ULONGLONG Time
= __rdtsc();
332 #ifdef FREELDR_HEAP_VERIFIER
333 /* Verify the heap */
334 FrLdrHeapVerify(HeapHandle
);
336 /* Add space for a size field and 2 redzones */
337 ByteSize
+= REDZONE_ALLOCATION
;
340 /* Check if the allocation is too large */
341 if ((ByteSize
+ sizeof(HEAP_BLOCK
)) > MAXUSHORT
* sizeof(HEAP_BLOCK
))
343 ERR("HEAP: Allocation of 0x%lx bytes too large\n", ByteSize
);
347 /* We need a proper tag */
348 if (Tag
== 0) Tag
= 'enoN';
350 /* Calculate alloc size */
351 BlockSize
= (USHORT
)((ByteSize
+ sizeof(HEAP_BLOCK
) - 1) / sizeof(HEAP_BLOCK
));
353 /* Walk the free block list */
354 Block
= &Heap
->Blocks
+ Heap
->TerminatingBlock
;
355 for (Block
= &Heap
->Blocks
+ Block
->Data
[0].Flink
;
357 Block
= &Heap
->Blocks
+ Block
->Data
[0].Flink
)
359 ASSERT(Block
->Tag
== 0);
361 /* Continue, if its too small */
362 if (Block
->Size
< BlockSize
) continue;
364 /* This block is just fine, use it */
367 /* Remove this entry from the free list */
368 FrLdrHeapRemoveFreeList(Heap
, Block
);
370 /* Calculate the remaining size */
371 Remaining
= Block
->Size
- BlockSize
;
373 /* Check if the remaining space is large enough for a new block */
376 /* Make the allocated block as large as necessary */
377 Block
->Size
= BlockSize
;
379 /* Get pointer to the new block */
380 NextBlock
= Block
+ 1 + BlockSize
;
382 /* Make it a free block */
384 NextBlock
->Size
= Remaining
- 1;
385 NextBlock
->PreviousSize
= BlockSize
;
386 BlockSize
= NextBlock
->Size
;
387 FrLdrHeapInsertFreeList(Heap
, NextBlock
);
389 /* Advance to the next block */
390 NextBlock
= NextBlock
+ 1 + BlockSize
;
394 /* Not enough left, use the full block */
395 BlockSize
= Block
->Size
;
397 /* Get the next block */
398 NextBlock
= Block
+ 1 + BlockSize
;
401 /* Update the next blocks back link */
402 NextBlock
->PreviousSize
= BlockSize
;
404 /* Update heap usage */
406 Heap
->CurrentAllocBytes
+= Block
->Size
* sizeof(HEAP_BLOCK
);
407 Heap
->MaxAllocBytes
= max(Heap
->MaxAllocBytes
, Heap
->CurrentAllocBytes
);
408 Heap
->LargestAllocation
= max(Heap
->LargestAllocation
,
409 Block
->Size
* sizeof(HEAP_BLOCK
));
410 #if DBG && !defined(_M_ARM)
411 Heap
->AllocationTime
+= (__rdtsc() - Time
);
413 TRACE("HeapAllocate(%p, %ld, %.4s) -> return %p\n",
414 HeapHandle
, ByteSize
, &Tag
, Block
->Data
);
416 /* HACK: zero out the allocation */
417 RtlZeroMemory(Block
->Data
, Block
->Size
* sizeof(HEAP_BLOCK
));
419 #ifdef FREELDR_HEAP_VERIFIER
420 /* Write size and redzones */
421 *REDZONE_SIZE(Block
) = ByteSize
- REDZONE_ALLOCATION
;
422 *REDZONE_LOW(Block
) = REDZONE_MARK
;
423 *REDZONE_HI(Block
) = REDZONE_MARK
;
425 /* Allocation starts after size field and redzone */
426 return (PUCHAR
)Block
->Data
+ REDZONE_LOW_OFFSET
;
428 /* Return pointer to the data */
432 /* We found nothing */
433 WARN("HEAP: nothing suitable found for 0x%lx bytes\n", ByteSize
);
443 PHEAP Heap
= HeapHandle
;
444 PHEAP_BLOCK Block
, PrevBlock
, NextBlock
;
445 #if DBG && !defined(_M_ARM)
446 ULONGLONG Time
= __rdtsc();
448 TRACE("HeapFree(%p, %p)\n", HeapHandle
, Pointer
);
449 ASSERT(Tag
!= 'dnE#');
451 #ifdef FREELDR_HEAP_VERIFIER
452 /* Verify the heap */
453 FrLdrHeapVerify(HeapHandle
);
456 /* Check if the block is really inside this heap */
457 if ((Pointer
< (PVOID
)(Heap
+ 1)) ||
458 (Pointer
> (PVOID
)((PUCHAR
)Heap
+ Heap
->MaximumSize
)))
460 ERR("HEAP: trying to free %p outside of heap %p\n", Pointer
, Heap
);
464 Block
= ((PHEAP_BLOCK
)Pointer
) - 1;
465 #ifdef FREELDR_HEAP_VERIFIER
466 Block
= (PHEAP_BLOCK
)((PUCHAR
)Block
- REDZONE_LOW_OFFSET
);
468 /* Verify size and redzones */
469 ASSERT(*REDZONE_SIZE(Block
) <= Block
->Size
* sizeof(HEAP_BLOCK
));
470 ASSERT(*REDZONE_LOW(Block
) == REDZONE_MARK
);
471 ASSERT(*REDZONE_HI(Block
) == REDZONE_MARK
);
474 /* Check if the tag matches */
475 if ((Tag
&& (Block
->Tag
!= Tag
)) || (Block
->Tag
== 0))
477 ERR("HEAP: Bad tag! Pointer=%p: block tag '%.4s', requested '%.4s', size=0x%lx\n",
478 Pointer
, &Block
->Tag
, &Tag
, Block
->Size
);
487 RtlFillMemory(Block
->Data
, Block
->Size
* sizeof(HEAP_BLOCK
), 0xCCCCCCCC);
490 /* Update heap usage */
492 Heap
->CurrentAllocBytes
-= Block
->Size
* sizeof(HEAP_BLOCK
);
494 /* Get pointers to the next and previous block */
495 PrevBlock
= Block
- Block
->PreviousSize
- 1;
496 NextBlock
= Block
+ Block
->Size
+ 1;
498 /* Check if next block is free */
499 if ((NextBlock
->Tag
== 0) &&
500 ((Block
->Size
+ NextBlock
->Size
+ 1) <= MAXUSHORT
))
502 /* Merge next block into current */
503 Block
->Size
+= NextBlock
->Size
+ 1;
504 FrLdrHeapRemoveFreeList(Heap
, NextBlock
);
506 NextBlock
= Block
+ Block
->Size
+ 1;
509 /* Check if there is a block before and it's free */
510 if ((Block
->PreviousSize
!= 0) && (PrevBlock
->Tag
== 0) &&
511 ((PrevBlock
->Size
+ Block
->Size
+ 1) <= MAXUSHORT
))
513 /* Merge current block into previous */
514 PrevBlock
->Size
+= Block
->Size
+ 1;
519 /* Insert the entry into the free list */
520 FrLdrHeapInsertFreeList(Heap
, Block
);
523 /* Update the next block's back link */
524 NextBlock
->PreviousSize
= Block
->Size
;
525 #if DBG && !defined(_M_ARM)
526 Heap
->FreeTime
+= (__rdtsc() - Time
);
531 /* Wrapper functions *********************************************************/
534 MmInitializeHeap(PVOID PageLookupTable
)
536 TRACE("MmInitializeHeap()\n");
538 /* Create the default heap */
539 FrLdrDefaultHeap
= FrLdrHeapCreate(DEFAULT_HEAP_SIZE
, LoaderOsloaderHeap
);
540 ASSERT(FrLdrDefaultHeap
);
542 /* Create a temporary heap */
543 FrLdrTempHeap
= FrLdrHeapCreate(TEMP_HEAP_SIZE
, LoaderFirmwareTemporary
);
544 ASSERT(FrLdrTempHeap
);
546 TRACE("MmInitializeHeap() done, default heap %p, temp heap %p\n",
547 FrLdrDefaultHeap
, FrLdrTempHeap
);
552 ExAllocatePoolWithTag(
553 IN POOL_TYPE PoolType
,
554 IN SIZE_T NumberOfBytes
,
557 return FrLdrHeapAllocateEx(FrLdrDefaultHeap
, NumberOfBytes
, Tag
);
563 IN POOL_TYPE PoolType
,
564 IN SIZE_T NumberOfBytes
)
566 return FrLdrHeapAllocateEx(FrLdrDefaultHeap
, NumberOfBytes
, 0);
574 FrLdrHeapFreeEx(FrLdrDefaultHeap
, P
, 0);
583 FrLdrHeapFreeEx(FrLdrDefaultHeap
, P
, Tag
);
595 ptr
= FrLdrHeapAllocateEx(FrLdrDefaultHeap
, Size
, ' ltR');
596 if (ptr
&& (Flags
& HEAP_ZERO_MEMORY
))
598 RtlZeroMemory(ptr
, Size
);
611 FrLdrHeapFreeEx(FrLdrDefaultHeap
, HeapBase
, ' ltR');