[SHELL32_APITEST] Follow-up to #6796 (25e2f5f)
[reactos.git] / boot / freeldr / freeldr / lib / mm / heap.c
1 /*
2 * FreeLoader
3 * Copyright (C) 2011 Timo Kreuzer (timo.kreuzer@reactos.org)
4 *
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License as published by
7 * the Free Software Foundation; either version 2 of the License, or
8 * (at your option) any later version.
9 *
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
14 *
15 * You should have received a copy of the GNU General Public License along
16 * with this program; if not, write to the Free Software Foundation, Inc.,
17 * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
18 */
19
20 #include <freeldr.h>
21
22 #include <debug.h>
23 DBG_DEFAULT_CHANNEL(HEAP);
24
25 #define FREELDR_HEAP_VERIFIER
26
27 #define REDZONE_MARK 0xCCCCCCCCCCCCCCCCULL
28 #define REDZONE_ALLOCATION 24
29 #define REDZONE_LOW_OFFSET 16
30 #define REDZONE_SIZE(Block) ((ULONG64*)Block->Data)
31 #define REDZONE_LOW(Block) ((ULONG64*)Block->Data + 1)
32 #define REDZONE_HI(Block) ((ULONG64*)((PUCHAR)Block->Data + 16 + *REDZONE_SIZE(Block)))
33
34 PVOID FrLdrDefaultHeap;
35 PVOID FrLdrTempHeap;
36
37 typedef struct _BLOCK_DATA
38 {
39 ULONG_PTR Flink:32;
40 ULONG_PTR Blink:32;
41 } BLOCK_DATA, *PBLOCK_DATA;
42
43 typedef struct _HEAP_BLOCK
44 {
45 USHORT Size;
46 USHORT PreviousSize;
47 ULONG Tag;
48 BLOCK_DATA Data[];
49 } HEAP_BLOCK, *PHEAP_BLOCK;
50
51 typedef struct _HEAP
52 {
53 SIZE_T MaximumSize;
54 SIZE_T CurrentAllocBytes;
55 SIZE_T MaxAllocBytes;
56 ULONG NumAllocs;
57 ULONG NumFrees;
58 SIZE_T LargestAllocation;
59 ULONGLONG AllocationTime;
60 ULONGLONG FreeTime;
61 ULONG_PTR TerminatingBlock;
62 HEAP_BLOCK Blocks;
63 } HEAP, *PHEAP;
64
65 PVOID
66 FrLdrHeapCreate(
67 SIZE_T MaximumSize,
68 TYPE_OF_MEMORY MemoryType)
69 {
70 PHEAP Heap;
71 PHEAP_BLOCK Block;
72 SIZE_T Remaining;
73 USHORT PreviousSize;
74
75 TRACE("HeapCreate(MemoryType=%ld)\n", MemoryType);
76
77 /* Allocate some memory for the heap */
78 MaximumSize = ALIGN_UP_BY(MaximumSize, MM_PAGE_SIZE);
79 Heap = MmAllocateMemoryWithType(MaximumSize, MemoryType);
80 if (!Heap)
81 {
82 ERR("HEAP: Failed to allocate heap of size 0x%lx, Type %lu\n",
83 MaximumSize, MemoryType);
84 return NULL;
85 }
86
87 /* Initialize the heap header */
88 Heap->MaximumSize = MaximumSize;
89 Heap->CurrentAllocBytes = 0;
90 Heap->MaxAllocBytes = 0;
91 Heap->NumAllocs = 0;
92 Heap->NumFrees = 0;
93 Heap->LargestAllocation = 0;
94
95 /* Calculate what's left to process */
96 Remaining = (MaximumSize - sizeof(HEAP)) / sizeof(HEAP_BLOCK);
97 TRACE("Remaining = %ld\n", Remaining);
98
99 /* Substract 2 for the terminating entry (header + free entry) */
100 Remaining -= 2;
101
102 Block = &Heap->Blocks;
103 PreviousSize = 0;
104
105 /* Create free blocks */
106 while (Remaining > 1)
107 {
108 /* Initialize this free block */
109 Block->Size = (USHORT)min(MAXUSHORT, Remaining - 1);
110 Block->PreviousSize = PreviousSize;
111 Block->Tag = 0;
112 Block->Data[0].Flink = (Block - &Heap->Blocks) + Block->Size + 1;
113 Block->Data[0].Blink = (Block - &Heap->Blocks) - 1 - PreviousSize;
114
115 /* Substract current block size from remainder */
116 Remaining -= (Block->Size + 1);
117
118 /* Go to next block */
119 PreviousSize = Block->Size;
120 Block = Block + Block->Size + 1;
121
122 TRACE("Remaining = %ld\n", Remaining);
123 }
124
125 /* Now finish with a terminating block */
126 Heap->TerminatingBlock = Block - &Heap->Blocks;
127 Block->Size = 0;
128 Block->PreviousSize = PreviousSize;
129 Block->Tag = 'dnE#';
130 Block->Data[0].Flink = 0;
131 Block->Data[0].Blink = (Block - &Heap->Blocks) - 1 - PreviousSize;
132 Heap->Blocks.Data[0].Blink = Heap->TerminatingBlock;
133
134 return Heap;
135 }
136
137 VOID
138 FrLdrHeapDestroy(
139 PVOID HeapHandle)
140 {
141 PHEAP Heap = HeapHandle;
142
143 /* Mark all pages as firmware temporary, so they are free for the kernel */
144 MmMarkPagesInLookupTable(PageLookupTableAddress,
145 (ULONG_PTR)Heap / MM_PAGE_SIZE,
146 (PFN_COUNT)(Heap->MaximumSize / MM_PAGE_SIZE),
147 LoaderFirmwareTemporary);
148
149 #if DBG
150 /* Make sure everything is dead */
151 RtlFillMemory(Heap, Heap->MaximumSize, 0xCCCCCCCC);
152 #endif
153 }
154
155 #ifdef FREELDR_HEAP_VERIFIER
156 VOID
157 FrLdrHeapVerify(
158 PVOID HeapHandle)
159 {
160 PHEAP Heap = HeapHandle;
161 PHEAP_BLOCK Block;
162
163 /* Loop all heap chunks */
164 for (Block = &Heap->Blocks;
165 Block->Size != 0;
166 Block = Block + 1 + Block->Size)
167 {
168 /* Continue, if its not free */
169 if (Block->Tag != 0)
170 {
171 /* Verify size and redzones */
172 ASSERT(*REDZONE_SIZE(Block) <= Block->Size * sizeof(HEAP_BLOCK));
173 ASSERT(*REDZONE_LOW(Block) == REDZONE_MARK);
174 ASSERT(*REDZONE_HI(Block) == REDZONE_MARK);
175 continue;
176 }
177 }
178 }
179 #endif /* FREELDR_HEAP_VERIFIER */
180
181 VOID
182 FrLdrHeapRelease(
183 PVOID HeapHandle)
184 {
185 PHEAP Heap = HeapHandle;
186 PHEAP_BLOCK Block;
187 PUCHAR StartAddress, EndAddress;
188 PFN_COUNT FreePages, AllFreePages = 0;
189
190 TRACE("HeapRelease(%p)\n", HeapHandle);
191
192 /* Loop all heap chunks */
193 for (Block = &Heap->Blocks;
194 Block->Size != 0;
195 Block = Block + 1 + Block->Size)
196 {
197 /* Continue, if its not free */
198 if (Block->Tag != 0)
199 {
200 #ifdef FREELDR_HEAP_VERIFIER
201 /* Verify size and redzones */
202 ASSERT(*REDZONE_SIZE(Block) <= Block->Size * sizeof(HEAP_BLOCK));
203 ASSERT(*REDZONE_LOW(Block) == REDZONE_MARK);
204 ASSERT(*REDZONE_HI(Block) == REDZONE_MARK);
205 #endif
206 continue;
207 }
208
209 /* Calculate page aligned start address of the free region */
210 StartAddress = ALIGN_UP_POINTER_BY(Block->Data, PAGE_SIZE);
211
212 /* Walk over adjacent free blocks */
213 while (Block->Tag == 0) Block = Block + Block->Size + 1;
214
215 /* Check if this was the last block */
216 if (Block->Size == 0)
217 {
218 /* Align the end address up to cover the end of the heap */
219 EndAddress = ALIGN_UP_POINTER_BY(Block->Data, PAGE_SIZE);
220 }
221 else
222 {
223 /* Align the end address down to not cover any allocations */
224 EndAddress = ALIGN_DOWN_POINTER_BY(Block->Data, PAGE_SIZE);
225 }
226
227 /* Check if we have free pages */
228 if (EndAddress > StartAddress)
229 {
230 /* Calculate the size of the free region in pages */
231 FreePages = (PFN_COUNT)((EndAddress - StartAddress) / MM_PAGE_SIZE);
232 AllFreePages += FreePages;
233
234 /* Now mark the pages free */
235 MmMarkPagesInLookupTable(PageLookupTableAddress,
236 (ULONG_PTR)StartAddress / MM_PAGE_SIZE,
237 FreePages,
238 LoaderFree);
239 }
240
241 /* bail out, if it was the last block */
242 if (Block->Size == 0) break;
243 }
244
245 TRACE("HeapRelease() done, freed %lu of %lu pages\n", AllFreePages, Heap->MaximumSize / MM_PAGE_SIZE);
246 }
247
248 VOID
249 FrLdrHeapCleanupAll(VOID)
250 {
251 #if DBG
252 PHEAP Heap;
253
254 Heap = FrLdrDefaultHeap;
255 TRACE("Heap statistics for default heap:\n"
256 "CurrentAlloc=0x%lx, MaxAlloc=0x%lx, LargestAllocation=0x%lx\n"
257 "NumAllocs=%ld, NumFrees=%ld\n",
258 Heap->CurrentAllocBytes, Heap->MaxAllocBytes, Heap->LargestAllocation,
259 Heap->NumAllocs, Heap->NumFrees);
260 TRACE("AllocTime = %I64d, FreeTime = %I64d, sum = %I64d\n",
261 Heap->AllocationTime, Heap->FreeTime, Heap->AllocationTime + Heap->FreeTime);
262 #endif
263
264 /* Release free pages from the default heap */
265 FrLdrHeapRelease(FrLdrDefaultHeap);
266
267 #if DBG
268 Heap = FrLdrTempHeap;
269 TRACE("Heap statistics for temp heap:\n"
270 "CurrentAlloc=0x%lx, MaxAlloc=0x%lx, LargestAllocation=0x%lx\n"
271 "NumAllocs=%ld, NumFrees=%ld\n",
272 Heap->CurrentAllocBytes, Heap->MaxAllocBytes, Heap->LargestAllocation,
273 Heap->NumAllocs, Heap->NumFrees);
274 #endif
275
276 /* Destroy the temp heap */
277 FrLdrHeapDestroy(FrLdrTempHeap);
278 }
279
280 static VOID
281 FrLdrHeapRemoveFreeList(
282 PHEAP Heap,
283 PHEAP_BLOCK Block)
284 {
285 PHEAP_BLOCK Previous, Next;
286
287 Next = &Heap->Blocks + Block->Data[0].Flink;
288 Previous = &Heap->Blocks + Block->Data[0].Blink;
289 ASSERT((Next->Tag == 0) || (Next->Tag == 'dnE#'));
290 ASSERT(Next->Data[0].Blink == Block - &Heap->Blocks);
291 ASSERT((Previous->Tag == 0) || (Previous->Tag == 'dnE#'));
292 ASSERT(Previous->Data[0].Flink == Block - &Heap->Blocks);
293
294 Next->Data[0].Blink = Previous - &Heap->Blocks;
295 Previous->Data[0].Flink = Next - &Heap->Blocks;
296 }
297
298 static VOID
299 FrLdrHeapInsertFreeList(
300 PHEAP Heap,
301 PHEAP_BLOCK FreeBlock)
302 {
303 PHEAP_BLOCK ListHead, NextBlock;
304 ASSERT(FreeBlock->Tag == 0);
305
306 /* Terminating block serves as free list head */
307 ListHead = &Heap->Blocks + Heap->TerminatingBlock;
308
309 for (NextBlock = &Heap->Blocks + ListHead->Data[0].Flink;
310 NextBlock < FreeBlock;
311 NextBlock = &Heap->Blocks + NextBlock->Data[0].Flink);
312
313 FreeBlock->Data[0].Flink = NextBlock - &Heap->Blocks;
314 FreeBlock->Data[0].Blink = NextBlock->Data[0].Blink;
315 NextBlock->Data[0].Blink = FreeBlock - &Heap->Blocks;
316 NextBlock = &Heap->Blocks + FreeBlock->Data[0].Blink;
317 NextBlock->Data[0].Flink = FreeBlock - &Heap->Blocks;
318 }
319
320 PVOID
321 FrLdrHeapAllocateEx(
322 PVOID HeapHandle,
323 SIZE_T ByteSize,
324 ULONG Tag)
325 {
326 PHEAP Heap = HeapHandle;
327 PHEAP_BLOCK Block, NextBlock;
328 USHORT BlockSize, Remaining;
329 #if DBG && !defined(_M_ARM)
330 ULONGLONG Time = __rdtsc();
331 #endif
332
333 #ifdef FREELDR_HEAP_VERIFIER
334 /* Verify the heap */
335 FrLdrHeapVerify(HeapHandle);
336
337 /* Add space for a size field and 2 redzones */
338 ByteSize += REDZONE_ALLOCATION;
339 #endif
340
341 /* Check if the allocation is too large */
342 if ((ByteSize + sizeof(HEAP_BLOCK)) > MAXUSHORT * sizeof(HEAP_BLOCK))
343 {
344 ERR("HEAP: Allocation of 0x%lx bytes too large\n", ByteSize);
345 return NULL;
346 }
347
348 /* We need a proper tag */
349 if (Tag == 0) Tag = 'enoN';
350
351 /* Calculate alloc size */
352 BlockSize = (USHORT)((ByteSize + sizeof(HEAP_BLOCK) - 1) / sizeof(HEAP_BLOCK));
353
354 /* Walk the free block list */
355 Block = &Heap->Blocks + Heap->TerminatingBlock;
356 for (Block = &Heap->Blocks + Block->Data[0].Flink;
357 Block->Size != 0;
358 Block = &Heap->Blocks + Block->Data[0].Flink)
359 {
360 ASSERT(Block->Tag == 0);
361
362 /* Continue, if its too small */
363 if (Block->Size < BlockSize) continue;
364
365 /* This block is just fine, use it */
366 Block->Tag = Tag;
367
368 /* Remove this entry from the free list */
369 FrLdrHeapRemoveFreeList(Heap, Block);
370
371 /* Calculate the remaining size */
372 Remaining = Block->Size - BlockSize;
373
374 /* Check if the remaining space is large enough for a new block */
375 if (Remaining > 1)
376 {
377 /* Make the allocated block as large as necessary */
378 Block->Size = BlockSize;
379
380 /* Get pointer to the new block */
381 NextBlock = Block + 1 + BlockSize;
382
383 /* Make it a free block */
384 NextBlock->Tag = 0;
385 NextBlock->Size = Remaining - 1;
386 NextBlock->PreviousSize = BlockSize;
387 BlockSize = NextBlock->Size;
388 FrLdrHeapInsertFreeList(Heap, NextBlock);
389
390 /* Advance to the next block */
391 NextBlock = NextBlock + 1 + BlockSize;
392 }
393 else
394 {
395 /* Not enough left, use the full block */
396 BlockSize = Block->Size;
397
398 /* Get the next block */
399 NextBlock = Block + 1 + BlockSize;
400 }
401
402 /* Update the next blocks back link */
403 NextBlock->PreviousSize = BlockSize;
404
405 /* Update heap usage */
406 Heap->NumAllocs++;
407 Heap->CurrentAllocBytes += Block->Size * sizeof(HEAP_BLOCK);
408 Heap->MaxAllocBytes = max(Heap->MaxAllocBytes, Heap->CurrentAllocBytes);
409 Heap->LargestAllocation = max(Heap->LargestAllocation,
410 Block->Size * sizeof(HEAP_BLOCK));
411 #if DBG && !defined(_M_ARM)
412 Heap->AllocationTime += (__rdtsc() - Time);
413 #endif
414 TRACE("HeapAllocate(%p, %ld, %.4s) -> return %p\n",
415 HeapHandle, ByteSize, &Tag, Block->Data);
416
417 /* HACK: zero out the allocation */
418 RtlZeroMemory(Block->Data, Block->Size * sizeof(HEAP_BLOCK));
419
420 #ifdef FREELDR_HEAP_VERIFIER
421 /* Write size and redzones */
422 *REDZONE_SIZE(Block) = ByteSize - REDZONE_ALLOCATION;
423 *REDZONE_LOW(Block) = REDZONE_MARK;
424 *REDZONE_HI(Block) = REDZONE_MARK;
425
426 /* Allocation starts after size field and redzone */
427 return (PUCHAR)Block->Data + REDZONE_LOW_OFFSET;
428 #endif
429 /* Return pointer to the data */
430 return Block->Data;
431 }
432
433 /* We found nothing */
434 WARN("HEAP: nothing suitable found for 0x%lx bytes\n", ByteSize);
435 return NULL;
436 }
437
438 VOID
439 FrLdrHeapFreeEx(
440 PVOID HeapHandle,
441 PVOID Pointer,
442 ULONG Tag)
443 {
444 PHEAP Heap = HeapHandle;
445 PHEAP_BLOCK Block, PrevBlock, NextBlock;
446 #if DBG && !defined(_M_ARM)
447 ULONGLONG Time = __rdtsc();
448 #endif
449
450 TRACE("HeapFree(%p, %p)\n", HeapHandle, Pointer);
451 ASSERT(Tag != 'dnE#');
452
453 #ifdef FREELDR_HEAP_VERIFIER
454 /* Verify the heap */
455 FrLdrHeapVerify(HeapHandle);
456 #endif
457
458 /* Check if the block is really inside this heap */
459 if ((Pointer < (PVOID)(Heap + 1)) ||
460 (Pointer > (PVOID)((PUCHAR)Heap + Heap->MaximumSize)))
461 {
462 ERR("HEAP: trying to free %p outside of heap %p\n", Pointer, Heap);
463 ASSERT(FALSE);
464 }
465
466 Block = ((PHEAP_BLOCK)Pointer) - 1;
467 #ifdef FREELDR_HEAP_VERIFIER
468 Block = (PHEAP_BLOCK)((PUCHAR)Block - REDZONE_LOW_OFFSET);
469
470 /* Verify size and redzones */
471 ASSERT(*REDZONE_SIZE(Block) <= Block->Size * sizeof(HEAP_BLOCK));
472 ASSERT(*REDZONE_LOW(Block) == REDZONE_MARK);
473 ASSERT(*REDZONE_HI(Block) == REDZONE_MARK);
474 #endif
475
476 /* Check if the tag matches */
477 if ((Tag && (Block->Tag != Tag)) || (Block->Tag == 0))
478 {
479 ERR("HEAP: Bad tag! Pointer=%p: block tag '%.4s', requested '%.4s', size=0x%lx\n",
480 Pointer, &Block->Tag, &Tag, Block->Size);
481 ASSERT(FALSE);
482 }
483
484 /* Mark as free */
485 Block->Tag = 0;
486
487 #if DBG
488 /* Erase contents */
489 RtlFillMemory(Block->Data, Block->Size * sizeof(HEAP_BLOCK), 0xCCCCCCCC);
490 #endif
491
492 /* Update heap usage */
493 Heap->NumFrees++;
494 Heap->CurrentAllocBytes -= Block->Size * sizeof(HEAP_BLOCK);
495
496 /* Get pointers to the next and previous block */
497 PrevBlock = Block - Block->PreviousSize - 1;
498 NextBlock = Block + Block->Size + 1;
499
500 /* Check if next block is free */
501 if ((NextBlock->Tag == 0) &&
502 ((Block->Size + NextBlock->Size + 1) <= MAXUSHORT))
503 {
504 /* Merge next block into current */
505 Block->Size += NextBlock->Size + 1;
506 FrLdrHeapRemoveFreeList(Heap, NextBlock);
507
508 NextBlock = Block + Block->Size + 1;
509 }
510
511 /* Check if there is a block before and it's free */
512 if ((Block->PreviousSize != 0) && (PrevBlock->Tag == 0) &&
513 ((PrevBlock->Size + Block->Size + 1) <= MAXUSHORT))
514 {
515 /* Merge current block into previous */
516 PrevBlock->Size += Block->Size + 1;
517 Block = PrevBlock;
518 }
519 else
520 {
521 /* Insert the entry into the free list */
522 FrLdrHeapInsertFreeList(Heap, Block);
523 }
524
525 /* Update the next block's back link */
526 NextBlock->PreviousSize = Block->Size;
527 #if DBG && !defined(_M_ARM)
528 Heap->FreeTime += (__rdtsc() - Time);
529 #endif
530 }
531
532
533 /* Wrapper functions *********************************************************/
534
535 VOID
536 MmInitializeHeap(PVOID PageLookupTable)
537 {
538 TRACE("MmInitializeHeap()\n");
539
540 /* Create the default heap */
541 FrLdrDefaultHeap = FrLdrHeapCreate(DEFAULT_HEAP_SIZE, LoaderOsloaderHeap);
542 ASSERT(FrLdrDefaultHeap);
543
544 /* Create a temporary heap */
545 FrLdrTempHeap = FrLdrHeapCreate(TEMP_HEAP_SIZE, LoaderFirmwareTemporary);
546 ASSERT(FrLdrTempHeap);
547
548 TRACE("MmInitializeHeap() done, default heap %p, temp heap %p\n",
549 FrLdrDefaultHeap, FrLdrTempHeap);
550 }
551
552 PVOID
553 NTAPI
554 ExAllocatePoolWithTag(
555 IN POOL_TYPE PoolType,
556 IN SIZE_T NumberOfBytes,
557 IN ULONG Tag)
558 {
559 return FrLdrHeapAllocateEx(FrLdrDefaultHeap, NumberOfBytes, Tag);
560 }
561
562 PVOID
563 NTAPI
564 ExAllocatePool(
565 IN POOL_TYPE PoolType,
566 IN SIZE_T NumberOfBytes)
567 {
568 return FrLdrHeapAllocateEx(FrLdrDefaultHeap, NumberOfBytes, 0);
569 }
570
571 VOID
572 NTAPI
573 ExFreePool(
574 IN PVOID P)
575 {
576 FrLdrHeapFreeEx(FrLdrDefaultHeap, P, 0);
577 }
578
579 VOID
580 NTAPI
581 ExFreePoolWithTag(
582 IN PVOID P,
583 IN ULONG Tag)
584 {
585 FrLdrHeapFreeEx(FrLdrDefaultHeap, P, Tag);
586 }
587
588 PVOID
589 NTAPI
590 RtlAllocateHeap(
591 IN PVOID HeapHandle,
592 IN ULONG Flags,
593 IN SIZE_T Size)
594 {
595 PVOID ptr;
596
597 ptr = FrLdrHeapAllocateEx(FrLdrDefaultHeap, Size, ' ltR');
598 if (ptr && (Flags & HEAP_ZERO_MEMORY))
599 {
600 RtlZeroMemory(ptr, Size);
601 }
602
603 return ptr;
604 }
605
606 BOOLEAN
607 NTAPI
608 RtlFreeHeap(
609 IN PVOID HeapHandle,
610 IN ULONG Flags,
611 IN PVOID HeapBase)
612 {
613 FrLdrHeapFreeEx(FrLdrDefaultHeap, HeapBase, ' ltR');
614 return TRUE;
615 }
616