3 * Copyright (C) 2011 Timo Kreuzer (timo.kreuzer@reactos.org)
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License as published by
7 * the Free Software Foundation; either version 2 of the License, or
8 * (at your option) any later version.
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
15 * You should have received a copy of the GNU General Public License along
16 * with this program; if not, write to the Free Software Foundation, Inc.,
17 * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
23 DBG_DEFAULT_CHANNEL(HEAP
);
25 #define FREELDR_HEAP_VERIFIER
27 #define REDZONE_MARK 0xCCCCCCCCCCCCCCCCULL
28 #define REDZONE_ALLOCATION 24
29 #define REDZONE_LOW_OFFSET 16
30 #define REDZONE_SIZE(Block) ((ULONG64*)Block->Data)
31 #define REDZONE_LOW(Block) ((ULONG64*)Block->Data + 1)
32 #define REDZONE_HI(Block) ((ULONG64*)((PUCHAR)Block->Data + 16 + *REDZONE_SIZE(Block)))
34 PVOID FrLdrDefaultHeap
;
37 typedef struct _BLOCK_DATA
41 } BLOCK_DATA
, *PBLOCK_DATA
;
43 typedef struct _HEAP_BLOCK
49 } HEAP_BLOCK
, *PHEAP_BLOCK
;
54 SIZE_T CurrentAllocBytes
;
58 SIZE_T LargestAllocation
;
59 ULONGLONG AllocationTime
;
61 ULONG_PTR TerminatingBlock
;
68 TYPE_OF_MEMORY MemoryType
)
75 TRACE("HeapCreate(MemoryType=%ld)\n", MemoryType
);
77 /* Allocate some memory for the heap */
78 MaximumSize
= ALIGN_UP_BY(MaximumSize
, MM_PAGE_SIZE
);
79 Heap
= MmAllocateMemoryWithType(MaximumSize
, MemoryType
);
82 ERR("HEAP: Failed to allocate heap of size 0x%lx, Type %lu\n",
83 MaximumSize
, MemoryType
);
87 /* Initialize the heap header */
88 Heap
->MaximumSize
= MaximumSize
;
89 Heap
->CurrentAllocBytes
= 0;
90 Heap
->MaxAllocBytes
= 0;
93 Heap
->LargestAllocation
= 0;
95 /* Calculate what's left to process */
96 Remaining
= (MaximumSize
- sizeof(HEAP
)) / sizeof(HEAP_BLOCK
);
97 TRACE("Remaining = %ld\n", Remaining
);
99 /* Substract 2 for the terminating entry (header + free entry) */
102 Block
= &Heap
->Blocks
;
105 /* Create free blocks */
106 while (Remaining
> 1)
108 /* Initialize this free block */
109 Block
->Size
= (USHORT
)min(MAXUSHORT
, Remaining
- 1);
110 Block
->PreviousSize
= PreviousSize
;
112 Block
->Data
[0].Flink
= (Block
- &Heap
->Blocks
) + Block
->Size
+ 1;
113 Block
->Data
[0].Blink
= (Block
- &Heap
->Blocks
) - 1 - PreviousSize
;
115 /* Substract current block size from remainder */
116 Remaining
-= (Block
->Size
+ 1);
118 /* Go to next block */
119 PreviousSize
= Block
->Size
;
120 Block
= Block
+ Block
->Size
+ 1;
122 TRACE("Remaining = %ld\n", Remaining
);
125 /* Now finish with a terminating block */
126 Heap
->TerminatingBlock
= Block
- &Heap
->Blocks
;
128 Block
->PreviousSize
= PreviousSize
;
130 Block
->Data
[0].Flink
= 0;
131 Block
->Data
[0].Blink
= (Block
- &Heap
->Blocks
) - 1 - PreviousSize
;
132 Heap
->Blocks
.Data
[0].Blink
= Heap
->TerminatingBlock
;
141 PHEAP Heap
= HeapHandle
;
143 /* Mark all pages as firmware temporary, so they are free for the kernel */
144 MmMarkPagesInLookupTable(PageLookupTableAddress
,
145 (ULONG_PTR
)Heap
/ MM_PAGE_SIZE
,
146 (PFN_COUNT
)(Heap
->MaximumSize
/ MM_PAGE_SIZE
),
147 LoaderFirmwareTemporary
);
150 /* Make sure everything is dead */
151 RtlFillMemory(Heap
, Heap
->MaximumSize
, 0xCCCCCCCC);
155 #ifdef FREELDR_HEAP_VERIFIER
160 PHEAP Heap
= HeapHandle
;
163 /* Loop all heap chunks */
164 for (Block
= &Heap
->Blocks
;
166 Block
= Block
+ 1 + Block
->Size
)
168 /* Continue, if its not free */
171 /* Verify size and redzones */
172 ASSERT(*REDZONE_SIZE(Block
) <= Block
->Size
* sizeof(HEAP_BLOCK
));
173 ASSERT(*REDZONE_LOW(Block
) == REDZONE_MARK
);
174 ASSERT(*REDZONE_HI(Block
) == REDZONE_MARK
);
179 #endif /* FREELDR_HEAP_VERIFIER */
185 PHEAP Heap
= HeapHandle
;
187 PUCHAR StartAddress
, EndAddress
;
188 PFN_COUNT FreePages
, AllFreePages
= 0;
190 TRACE("HeapRelease(%p)\n", HeapHandle
);
192 /* Loop all heap chunks */
193 for (Block
= &Heap
->Blocks
;
195 Block
= Block
+ 1 + Block
->Size
)
197 /* Continue, if its not free */
200 #ifdef FREELDR_HEAP_VERIFIER
201 /* Verify size and redzones */
202 ASSERT(*REDZONE_SIZE(Block
) <= Block
->Size
* sizeof(HEAP_BLOCK
));
203 ASSERT(*REDZONE_LOW(Block
) == REDZONE_MARK
);
204 ASSERT(*REDZONE_HI(Block
) == REDZONE_MARK
);
209 /* Calculate page aligned start address of the free region */
210 StartAddress
= ALIGN_UP_POINTER_BY(Block
->Data
, PAGE_SIZE
);
212 /* Walk over adjacent free blocks */
213 while (Block
->Tag
== 0) Block
= Block
+ Block
->Size
+ 1;
215 /* Check if this was the last block */
216 if (Block
->Size
== 0)
218 /* Align the end address up to cover the end of the heap */
219 EndAddress
= ALIGN_UP_POINTER_BY(Block
->Data
, PAGE_SIZE
);
223 /* Align the end address down to not cover any allocations */
224 EndAddress
= ALIGN_DOWN_POINTER_BY(Block
->Data
, PAGE_SIZE
);
227 /* Check if we have free pages */
228 if (EndAddress
> StartAddress
)
230 /* Calculate the size of the free region in pages */
231 FreePages
= (PFN_COUNT
)((EndAddress
- StartAddress
) / MM_PAGE_SIZE
);
232 AllFreePages
+= FreePages
;
234 /* Now mark the pages free */
235 MmMarkPagesInLookupTable(PageLookupTableAddress
,
236 (ULONG_PTR
)StartAddress
/ MM_PAGE_SIZE
,
241 /* bail out, if it was the last block */
242 if (Block
->Size
== 0) break;
245 TRACE("HeapRelease() done, freed %lu of %lu pages\n", AllFreePages
, Heap
->MaximumSize
/ MM_PAGE_SIZE
);
249 FrLdrHeapCleanupAll(VOID
)
254 Heap
= FrLdrDefaultHeap
;
255 TRACE("Heap statistics for default heap:\n"
256 "CurrentAlloc=0x%lx, MaxAlloc=0x%lx, LargestAllocation=0x%lx\n"
257 "NumAllocs=%ld, NumFrees=%ld\n",
258 Heap
->CurrentAllocBytes
, Heap
->MaxAllocBytes
, Heap
->LargestAllocation
,
259 Heap
->NumAllocs
, Heap
->NumFrees
);
260 TRACE("AllocTime = %I64d, FreeTime = %I64d, sum = %I64d\n",
261 Heap
->AllocationTime
, Heap
->FreeTime
, Heap
->AllocationTime
+ Heap
->FreeTime
);
264 /* Release free pages from the default heap */
265 FrLdrHeapRelease(FrLdrDefaultHeap
);
268 Heap
= FrLdrTempHeap
;
269 TRACE("Heap statistics for temp heap:\n"
270 "CurrentAlloc=0x%lx, MaxAlloc=0x%lx, LargestAllocation=0x%lx\n"
271 "NumAllocs=%ld, NumFrees=%ld\n",
272 Heap
->CurrentAllocBytes
, Heap
->MaxAllocBytes
, Heap
->LargestAllocation
,
273 Heap
->NumAllocs
, Heap
->NumFrees
);
276 /* Destroy the temp heap */
277 FrLdrHeapDestroy(FrLdrTempHeap
);
281 FrLdrHeapRemoveFreeList(
285 PHEAP_BLOCK Previous
, Next
;
287 Next
= &Heap
->Blocks
+ Block
->Data
[0].Flink
;
288 Previous
= &Heap
->Blocks
+ Block
->Data
[0].Blink
;
289 ASSERT((Next
->Tag
== 0) || (Next
->Tag
== 'dnE#'));
290 ASSERT(Next
->Data
[0].Blink
== Block
- &Heap
->Blocks
);
291 ASSERT((Previous
->Tag
== 0) || (Previous
->Tag
== 'dnE#'));
292 ASSERT(Previous
->Data
[0].Flink
== Block
- &Heap
->Blocks
);
294 Next
->Data
[0].Blink
= Previous
- &Heap
->Blocks
;
295 Previous
->Data
[0].Flink
= Next
- &Heap
->Blocks
;
299 FrLdrHeapInsertFreeList(
301 PHEAP_BLOCK FreeBlock
)
303 PHEAP_BLOCK ListHead
, NextBlock
;
304 ASSERT(FreeBlock
->Tag
== 0);
306 /* Terminating block serves as free list head */
307 ListHead
= &Heap
->Blocks
+ Heap
->TerminatingBlock
;
309 for (NextBlock
= &Heap
->Blocks
+ ListHead
->Data
[0].Flink
;
310 NextBlock
< FreeBlock
;
311 NextBlock
= &Heap
->Blocks
+ NextBlock
->Data
[0].Flink
);
313 FreeBlock
->Data
[0].Flink
= NextBlock
- &Heap
->Blocks
;
314 FreeBlock
->Data
[0].Blink
= NextBlock
->Data
[0].Blink
;
315 NextBlock
->Data
[0].Blink
= FreeBlock
- &Heap
->Blocks
;
316 NextBlock
= &Heap
->Blocks
+ FreeBlock
->Data
[0].Blink
;
317 NextBlock
->Data
[0].Flink
= FreeBlock
- &Heap
->Blocks
;
326 PHEAP Heap
= HeapHandle
;
327 PHEAP_BLOCK Block
, NextBlock
;
328 USHORT BlockSize
, Remaining
;
329 #if DBG && !defined(_M_ARM)
330 ULONGLONG Time
= __rdtsc();
333 #ifdef FREELDR_HEAP_VERIFIER
334 /* Verify the heap */
335 FrLdrHeapVerify(HeapHandle
);
337 /* Add space for a size field and 2 redzones */
338 ByteSize
+= REDZONE_ALLOCATION
;
341 /* Check if the allocation is too large */
342 if ((ByteSize
+ sizeof(HEAP_BLOCK
)) > MAXUSHORT
* sizeof(HEAP_BLOCK
))
344 ERR("HEAP: Allocation of 0x%lx bytes too large\n", ByteSize
);
348 /* We need a proper tag */
349 if (Tag
== 0) Tag
= 'enoN';
351 /* Calculate alloc size */
352 BlockSize
= (USHORT
)((ByteSize
+ sizeof(HEAP_BLOCK
) - 1) / sizeof(HEAP_BLOCK
));
354 /* Walk the free block list */
355 Block
= &Heap
->Blocks
+ Heap
->TerminatingBlock
;
356 for (Block
= &Heap
->Blocks
+ Block
->Data
[0].Flink
;
358 Block
= &Heap
->Blocks
+ Block
->Data
[0].Flink
)
360 ASSERT(Block
->Tag
== 0);
362 /* Continue, if its too small */
363 if (Block
->Size
< BlockSize
) continue;
365 /* This block is just fine, use it */
368 /* Remove this entry from the free list */
369 FrLdrHeapRemoveFreeList(Heap
, Block
);
371 /* Calculate the remaining size */
372 Remaining
= Block
->Size
- BlockSize
;
374 /* Check if the remaining space is large enough for a new block */
377 /* Make the allocated block as large as necessary */
378 Block
->Size
= BlockSize
;
380 /* Get pointer to the new block */
381 NextBlock
= Block
+ 1 + BlockSize
;
383 /* Make it a free block */
385 NextBlock
->Size
= Remaining
- 1;
386 NextBlock
->PreviousSize
= BlockSize
;
387 BlockSize
= NextBlock
->Size
;
388 FrLdrHeapInsertFreeList(Heap
, NextBlock
);
390 /* Advance to the next block */
391 NextBlock
= NextBlock
+ 1 + BlockSize
;
395 /* Not enough left, use the full block */
396 BlockSize
= Block
->Size
;
398 /* Get the next block */
399 NextBlock
= Block
+ 1 + BlockSize
;
402 /* Update the next blocks back link */
403 NextBlock
->PreviousSize
= BlockSize
;
405 /* Update heap usage */
407 Heap
->CurrentAllocBytes
+= Block
->Size
* sizeof(HEAP_BLOCK
);
408 Heap
->MaxAllocBytes
= max(Heap
->MaxAllocBytes
, Heap
->CurrentAllocBytes
);
409 Heap
->LargestAllocation
= max(Heap
->LargestAllocation
,
410 Block
->Size
* sizeof(HEAP_BLOCK
));
411 #if DBG && !defined(_M_ARM)
412 Heap
->AllocationTime
+= (__rdtsc() - Time
);
414 TRACE("HeapAllocate(%p, %ld, %.4s) -> return %p\n",
415 HeapHandle
, ByteSize
, &Tag
, Block
->Data
);
417 /* HACK: zero out the allocation */
418 RtlZeroMemory(Block
->Data
, Block
->Size
* sizeof(HEAP_BLOCK
));
420 #ifdef FREELDR_HEAP_VERIFIER
421 /* Write size and redzones */
422 *REDZONE_SIZE(Block
) = ByteSize
- REDZONE_ALLOCATION
;
423 *REDZONE_LOW(Block
) = REDZONE_MARK
;
424 *REDZONE_HI(Block
) = REDZONE_MARK
;
426 /* Allocation starts after size field and redzone */
427 return (PUCHAR
)Block
->Data
+ REDZONE_LOW_OFFSET
;
429 /* Return pointer to the data */
433 /* We found nothing */
434 WARN("HEAP: nothing suitable found for 0x%lx bytes\n", ByteSize
);
444 PHEAP Heap
= HeapHandle
;
445 PHEAP_BLOCK Block
, PrevBlock
, NextBlock
;
446 #if DBG && !defined(_M_ARM)
447 ULONGLONG Time
= __rdtsc();
450 TRACE("HeapFree(%p, %p)\n", HeapHandle
, Pointer
);
451 ASSERT(Tag
!= 'dnE#');
453 #ifdef FREELDR_HEAP_VERIFIER
454 /* Verify the heap */
455 FrLdrHeapVerify(HeapHandle
);
458 /* Check if the block is really inside this heap */
459 if ((Pointer
< (PVOID
)(Heap
+ 1)) ||
460 (Pointer
> (PVOID
)((PUCHAR
)Heap
+ Heap
->MaximumSize
)))
462 ERR("HEAP: trying to free %p outside of heap %p\n", Pointer
, Heap
);
466 Block
= ((PHEAP_BLOCK
)Pointer
) - 1;
467 #ifdef FREELDR_HEAP_VERIFIER
468 Block
= (PHEAP_BLOCK
)((PUCHAR
)Block
- REDZONE_LOW_OFFSET
);
470 /* Verify size and redzones */
471 ASSERT(*REDZONE_SIZE(Block
) <= Block
->Size
* sizeof(HEAP_BLOCK
));
472 ASSERT(*REDZONE_LOW(Block
) == REDZONE_MARK
);
473 ASSERT(*REDZONE_HI(Block
) == REDZONE_MARK
);
476 /* Check if the tag matches */
477 if ((Tag
&& (Block
->Tag
!= Tag
)) || (Block
->Tag
== 0))
479 ERR("HEAP: Bad tag! Pointer=%p: block tag '%.4s', requested '%.4s', size=0x%lx\n",
480 Pointer
, &Block
->Tag
, &Tag
, Block
->Size
);
489 RtlFillMemory(Block
->Data
, Block
->Size
* sizeof(HEAP_BLOCK
), 0xCCCCCCCC);
492 /* Update heap usage */
494 Heap
->CurrentAllocBytes
-= Block
->Size
* sizeof(HEAP_BLOCK
);
496 /* Get pointers to the next and previous block */
497 PrevBlock
= Block
- Block
->PreviousSize
- 1;
498 NextBlock
= Block
+ Block
->Size
+ 1;
500 /* Check if next block is free */
501 if ((NextBlock
->Tag
== 0) &&
502 ((Block
->Size
+ NextBlock
->Size
+ 1) <= MAXUSHORT
))
504 /* Merge next block into current */
505 Block
->Size
+= NextBlock
->Size
+ 1;
506 FrLdrHeapRemoveFreeList(Heap
, NextBlock
);
508 NextBlock
= Block
+ Block
->Size
+ 1;
511 /* Check if there is a block before and it's free */
512 if ((Block
->PreviousSize
!= 0) && (PrevBlock
->Tag
== 0) &&
513 ((PrevBlock
->Size
+ Block
->Size
+ 1) <= MAXUSHORT
))
515 /* Merge current block into previous */
516 PrevBlock
->Size
+= Block
->Size
+ 1;
521 /* Insert the entry into the free list */
522 FrLdrHeapInsertFreeList(Heap
, Block
);
525 /* Update the next block's back link */
526 NextBlock
->PreviousSize
= Block
->Size
;
527 #if DBG && !defined(_M_ARM)
528 Heap
->FreeTime
+= (__rdtsc() - Time
);
533 /* Wrapper functions *********************************************************/
536 MmInitializeHeap(PVOID PageLookupTable
)
538 TRACE("MmInitializeHeap()\n");
540 /* Create the default heap */
541 FrLdrDefaultHeap
= FrLdrHeapCreate(DEFAULT_HEAP_SIZE
, LoaderOsloaderHeap
);
542 ASSERT(FrLdrDefaultHeap
);
544 /* Create a temporary heap */
545 FrLdrTempHeap
= FrLdrHeapCreate(TEMP_HEAP_SIZE
, LoaderFirmwareTemporary
);
546 ASSERT(FrLdrTempHeap
);
548 TRACE("MmInitializeHeap() done, default heap %p, temp heap %p\n",
549 FrLdrDefaultHeap
, FrLdrTempHeap
);
554 ExAllocatePoolWithTag(
555 IN POOL_TYPE PoolType
,
556 IN SIZE_T NumberOfBytes
,
559 return FrLdrHeapAllocateEx(FrLdrDefaultHeap
, NumberOfBytes
, Tag
);
565 IN POOL_TYPE PoolType
,
566 IN SIZE_T NumberOfBytes
)
568 return FrLdrHeapAllocateEx(FrLdrDefaultHeap
, NumberOfBytes
, 0);
576 FrLdrHeapFreeEx(FrLdrDefaultHeap
, P
, 0);
585 FrLdrHeapFreeEx(FrLdrDefaultHeap
, P
, Tag
);
597 ptr
= FrLdrHeapAllocateEx(FrLdrDefaultHeap
, Size
, ' ltR');
598 if (ptr
&& (Flags
& HEAP_ZERO_MEMORY
))
600 RtlZeroMemory(ptr
, Size
);
613 FrLdrHeapFreeEx(FrLdrDefaultHeap
, HeapBase
, ' ltR');