[BOOT] Spelling fixes by Josh Soref. CORE-12286
[reactos.git] / reactos / boot / freeldr / freeldr / lib / mm / heap.c
1 /*
2 * FreeLoader
3 * Copyright (C) 2011 Timo Kreuzer (timo.kreuzer@reactos.org)
4 *
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License as published by
7 * the Free Software Foundation; either version 2 of the License, or
8 * (at your option) any later version.
9 *
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
14 *
15 * You should have received a copy of the GNU General Public License along
16 * with this program; if not, write to the Free Software Foundation, Inc.,
17 * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
18 */
19
20 #include <freeldr.h>
21 #include <debug.h>
22
23 #define FREELDR_HEAP_VERIFIER
24
25 DBG_DEFAULT_CHANNEL(HEAP);
26
27 #define REDZONE_MARK 0xCCCCCCCCCCCCCCCCULL
28 #define REDZONE_ALLOCATION 24
29 #define REDZONE_LOW_OFFSET 16
30 #define REDZONE_SIZE(Block) ((ULONG64*)Block->Data)
31 #define REDZONE_LOW(Block) ((ULONG64*)Block->Data + 1)
32 #define REDZONE_HI(Block) ((ULONG64*)((PUCHAR)Block->Data + 16 + *REDZONE_SIZE(Block)))
33
34 PVOID FrLdrDefaultHeap;
35 PVOID FrLdrTempHeap;
36
37 typedef struct _BLOCK_DATA
38 {
39 ULONG_PTR Flink:32;
40 ULONG_PTR Blink:32;
41 } BLOCK_DATA, *PBLOCK_DATA;
42
43 typedef struct _HEAP_BLOCK
44 {
45 USHORT Size;
46 USHORT PreviousSize;
47 ULONG Tag;
48 BLOCK_DATA Data[];
49 } HEAP_BLOCK, *PHEAP_BLOCK;
50
51 typedef struct _HEAP
52 {
53 SIZE_T MaximumSize;
54 SIZE_T CurrentAllocBytes;
55 SIZE_T MaxAllocBytes;
56 ULONG NumAllocs;
57 ULONG NumFrees;
58 SIZE_T LargestAllocation;
59 ULONGLONG AllocationTime;
60 ULONGLONG FreeTime;
61 ULONG_PTR TerminatingBlock;
62 HEAP_BLOCK Blocks;
63 } HEAP, *PHEAP;
64
65 PVOID
66 FrLdrHeapCreate(
67 SIZE_T MaximumSize,
68 TYPE_OF_MEMORY MemoryType)
69 {
70 PHEAP Heap;
71 PHEAP_BLOCK Block;
72 SIZE_T Remaining;
73 USHORT PreviousSize;
74 TRACE("HeapCreate(MemoryType=%ld)\n", MemoryType);
75
76 /* Allocate some memory for the heap */
77 MaximumSize = ALIGN_UP_BY(MaximumSize, MM_PAGE_SIZE);
78 Heap = MmAllocateMemoryWithType(MaximumSize, MemoryType);
79 if (!Heap)
80 {
81 ERR("HEAP: Failed to allocate heap of size 0x%lx, Type\n",
82 MaximumSize, MemoryType);
83 return NULL;
84 }
85
86 /* Initialize the heap header */
87 Heap->MaximumSize = MaximumSize;
88 Heap->CurrentAllocBytes = 0;
89 Heap->MaxAllocBytes = 0;
90 Heap->NumAllocs = 0;
91 Heap->NumFrees = 0;
92 Heap->LargestAllocation = 0;
93
94 /* Calculate what's left to process */
95 Remaining = (MaximumSize - sizeof(HEAP)) / sizeof(HEAP_BLOCK);
96 TRACE("Remaining = %ld\n", Remaining);
97
98 /* Substract 2 for the terminating entry (header + free entry) */
99 Remaining -= 2;
100
101 Block = &Heap->Blocks;
102 PreviousSize = 0;
103
104 /* Create free blocks */
105 while (Remaining > 1)
106 {
107 /* Initialize this free block */
108 Block->Size = (USHORT)min(MAXUSHORT, Remaining - 1);
109 Block->PreviousSize = PreviousSize;
110 Block->Tag = 0;
111 Block->Data[0].Flink = (Block - &Heap->Blocks) + Block->Size + 1;
112 Block->Data[0].Blink = (Block - &Heap->Blocks) - 1 - PreviousSize;
113
114 /* Substract current block size from remainder */
115 Remaining -= (Block->Size + 1);
116
117 /* Go to next block */
118 PreviousSize = Block->Size;
119 Block = Block + Block->Size + 1;
120
121 TRACE("Remaining = %ld\n", Remaining);
122 }
123
124 /* Now finish with a terminating block */
125 Heap->TerminatingBlock = Block - &Heap->Blocks;
126 Block->Size = 0;
127 Block->PreviousSize = PreviousSize;
128 Block->Tag = 'dnE#';
129 Block->Data[0].Flink = 0;
130 Block->Data[0].Blink = (Block - &Heap->Blocks) - 1 - PreviousSize;
131 Heap->Blocks.Data[0].Blink = Heap->TerminatingBlock;
132
133 return Heap;
134 }
135
136 VOID
137 FrLdrHeapDestroy(
138 PVOID HeapHandle)
139 {
140 PHEAP Heap = HeapHandle;
141
142 /* Mark all pages as firmware temporary, so they are free for the kernel */
143 MmMarkPagesInLookupTable(PageLookupTableAddress,
144 (ULONG_PTR)Heap / MM_PAGE_SIZE,
145 (PFN_COUNT)(Heap->MaximumSize / MM_PAGE_SIZE),
146 LoaderFirmwareTemporary);
147
148 #if DBG
149 /* Make sure everything is dead */
150 RtlFillMemory(Heap, Heap->MaximumSize, 0xCCCCCCCC);
151 #endif
152 }
153
154 #ifdef FREELDR_HEAP_VERIFIER
155 VOID
156 FrLdrHeapVerify(
157 PVOID HeapHandle)
158 {
159 PHEAP Heap = HeapHandle;
160 PHEAP_BLOCK Block;
161
162 /* Loop all heap chunks */
163 for (Block = &Heap->Blocks;
164 Block->Size != 0;
165 Block = Block + 1 + Block->Size)
166 {
167 /* Continue, if its not free */
168 if (Block->Tag != 0)
169 {
170 /* Verify size and redzones */
171 ASSERT(*REDZONE_SIZE(Block) <= Block->Size * sizeof(HEAP_BLOCK));
172 ASSERT(*REDZONE_LOW(Block) == REDZONE_MARK);
173 ASSERT(*REDZONE_HI(Block) == REDZONE_MARK);
174 continue;
175 }
176 }
177 }
178 #endif /* FREELDR_HEAP_VERIFIER */
179
180 VOID
181 FrLdrHeapRelease(
182 PVOID HeapHandle)
183 {
184 PHEAP Heap = HeapHandle;
185 PHEAP_BLOCK Block;
186 PUCHAR StartAddress, EndAddress;
187 PFN_COUNT FreePages, AllPages, AllFreePages = 0;
188 TRACE("HeapRelease(%p)\n", HeapHandle);
189
190 /* Loop all heap chunks */
191 for (Block = &Heap->Blocks;
192 Block->Size != 0;
193 Block = Block + 1 + Block->Size)
194 {
195 /* Continue, if its not free */
196 if (Block->Tag != 0)
197 {
198 #ifdef FREELDR_HEAP_VERIFIER
199 /* Verify size and redzones */
200 ASSERT(*REDZONE_SIZE(Block) <= Block->Size * sizeof(HEAP_BLOCK));
201 ASSERT(*REDZONE_LOW(Block) == REDZONE_MARK);
202 ASSERT(*REDZONE_HI(Block) == REDZONE_MARK);
203 #endif
204 continue;
205 }
206
207 /* Calculate page aligned start address of the free region */
208 StartAddress = ALIGN_UP_POINTER_BY(Block->Data, PAGE_SIZE);
209
210 /* Walk over adjacent free blocks */
211 while (Block->Tag == 0) Block = Block + Block->Size + 1;
212
213 /* Check if this was the last block */
214 if (Block->Size == 0)
215 {
216 /* Align the end address up to cover the end of the heap */
217 EndAddress = ALIGN_UP_POINTER_BY(Block->Data, PAGE_SIZE);
218 }
219 else
220 {
221 /* Align the end address down to not cover any allocations */
222 EndAddress = ALIGN_DOWN_POINTER_BY(Block->Data, PAGE_SIZE);
223 }
224
225 /* Check if we have free pages */
226 if (EndAddress > StartAddress)
227 {
228 /* Calculate the size of the free region in pages */
229 FreePages = (PFN_COUNT)((EndAddress - StartAddress) / MM_PAGE_SIZE);
230 AllFreePages += FreePages;
231
232 /* Now mark the pages free */
233 MmMarkPagesInLookupTable(PageLookupTableAddress,
234 (ULONG_PTR)StartAddress / MM_PAGE_SIZE,
235 FreePages,
236 LoaderFree);
237 }
238
239 /* bail out, if it was the last block */
240 if (Block->Size == 0) break;
241 }
242
243 AllPages = Heap->MaximumSize / MM_PAGE_SIZE;
244 TRACE("HeapRelease() done, freed %lu of %lu pages\n", AllFreePages, AllPages);
245 }
246
247 VOID
248 FrLdrHeapCleanupAll(VOID)
249 {
250 PHEAP Heap;
251
252 Heap = FrLdrDefaultHeap;
253 TRACE("Heap statistics for default heap:\n"
254 "CurrentAlloc=0x%lx, MaxAlloc=0x%lx, LargestAllocation=0x%lx\n"
255 "NumAllocs=%ld, NumFrees=%ld\n",
256 Heap->CurrentAllocBytes, Heap->MaxAllocBytes, Heap->LargestAllocation,
257 Heap->NumAllocs, Heap->NumFrees);
258 TRACE("AllocTime = %I64d, FreeTime = %I64d, sum = %I64d\n",
259 Heap->AllocationTime, Heap->FreeTime, Heap->AllocationTime + Heap->FreeTime);
260
261
262 /* Release free pages from the default heap */
263 FrLdrHeapRelease(FrLdrDefaultHeap);
264
265 Heap = FrLdrTempHeap;
266 TRACE("Heap statistics for temp heap:\n"
267 "CurrentAlloc=0x%lx, MaxAlloc=0x%lx, LargestAllocation=0x%lx\n"
268 "NumAllocs=%ld, NumFrees=%ld\n",
269 Heap->CurrentAllocBytes, Heap->MaxAllocBytes, Heap->LargestAllocation,
270 Heap->NumAllocs, Heap->NumFrees);
271
272 /* Destroy the temp heap */
273 FrLdrHeapDestroy(FrLdrTempHeap);
274 }
275
276 static VOID
277 FrLdrHeapRemoveFreeList(
278 PHEAP Heap,
279 PHEAP_BLOCK Block)
280 {
281 PHEAP_BLOCK Previous, Next;
282
283 Next = &Heap->Blocks + Block->Data[0].Flink;
284 Previous = &Heap->Blocks + Block->Data[0].Blink;
285 ASSERT((Next->Tag == 0) || (Next->Tag == 'dnE#'));
286 ASSERT(Next->Data[0].Blink == Block - &Heap->Blocks);
287 ASSERT((Previous->Tag == 0) || (Previous->Tag == 'dnE#'));
288 ASSERT(Previous->Data[0].Flink == Block - &Heap->Blocks);
289
290 Next->Data[0].Blink = Previous - &Heap->Blocks;
291 Previous->Data[0].Flink = Next - &Heap->Blocks;
292 }
293
294 static VOID
295 FrLdrHeapInsertFreeList(
296 PHEAP Heap,
297 PHEAP_BLOCK FreeBlock)
298 {
299 PHEAP_BLOCK ListHead, NextBlock;
300 ASSERT(FreeBlock->Tag == 0);
301
302 /* Terminating block serves as free list head */
303 ListHead = &Heap->Blocks + Heap->TerminatingBlock;
304
305 for (NextBlock = &Heap->Blocks + ListHead->Data[0].Flink;
306 NextBlock < FreeBlock;
307 NextBlock = &Heap->Blocks + NextBlock->Data[0].Flink);
308
309 FreeBlock->Data[0].Flink = NextBlock - &Heap->Blocks;
310 FreeBlock->Data[0].Blink = NextBlock->Data[0].Blink;
311 NextBlock->Data[0].Blink = FreeBlock - &Heap->Blocks;
312 NextBlock = &Heap->Blocks + FreeBlock->Data[0].Blink;
313 NextBlock->Data[0].Flink = FreeBlock - &Heap->Blocks;
314 }
315
316 PVOID
317 FrLdrHeapAllocateEx(
318 PVOID HeapHandle,
319 SIZE_T ByteSize,
320 ULONG Tag)
321 {
322 PHEAP Heap = HeapHandle;
323 PHEAP_BLOCK Block, NextBlock;
324 USHORT BlockSize, Remaining;
325 #if DBG && !defined(_M_ARM)
326 ULONGLONG Time = __rdtsc();
327 #endif
328
329 #ifdef FREELDR_HEAP_VERIFIER
330 /* Verify the heap */
331 FrLdrHeapVerify(HeapHandle);
332
333 /* Add space for a size field and 2 redzones */
334 ByteSize += REDZONE_ALLOCATION;
335 #endif
336
337 /* Check if the allocation is too large */
338 if ((ByteSize + sizeof(HEAP_BLOCK)) > MAXUSHORT * sizeof(HEAP_BLOCK))
339 {
340 ERR("HEAP: Allocation of 0x%lx bytes too large\n", ByteSize);
341 return NULL;
342 }
343
344 /* We need a proper tag */
345 if (Tag == 0) Tag = 'enoN';
346
347 /* Calculate alloc size */
348 BlockSize = (USHORT)((ByteSize + sizeof(HEAP_BLOCK) - 1) / sizeof(HEAP_BLOCK));
349
350 /* Walk the free block list */
351 Block = &Heap->Blocks + Heap->TerminatingBlock;
352 for (Block = &Heap->Blocks + Block->Data[0].Flink;
353 Block->Size != 0;
354 Block = &Heap->Blocks + Block->Data[0].Flink)
355 {
356 ASSERT(Block->Tag == 0);
357
358 /* Continue, if its too small */
359 if (Block->Size < BlockSize) continue;
360
361 /* This block is just fine, use it */
362 Block->Tag = Tag;
363
364 /* Remove this entry from the free list */
365 FrLdrHeapRemoveFreeList(Heap, Block);
366
367 /* Calculate the remaining size */
368 Remaining = Block->Size - BlockSize;
369
370 /* Check if the remaining space is large enough for a new block */
371 if (Remaining > 1)
372 {
373 /* Make the allocated block as large as necessary */
374 Block->Size = BlockSize;
375
376 /* Get pointer to the new block */
377 NextBlock = Block + 1 + BlockSize;
378
379 /* Make it a free block */
380 NextBlock->Tag = 0;
381 NextBlock->Size = Remaining - 1;
382 NextBlock->PreviousSize = BlockSize;
383 BlockSize = NextBlock->Size;
384 FrLdrHeapInsertFreeList(Heap, NextBlock);
385
386 /* Advance to the next block */
387 NextBlock = NextBlock + 1 + BlockSize;
388 }
389 else
390 {
391 /* Not enough left, use the full block */
392 BlockSize = Block->Size;
393
394 /* Get the next block */
395 NextBlock = Block + 1 + BlockSize;
396 }
397
398 /* Update the next blocks back link */
399 NextBlock->PreviousSize = BlockSize;
400
401 /* Update heap usage */
402 Heap->NumAllocs++;
403 Heap->CurrentAllocBytes += Block->Size * sizeof(HEAP_BLOCK);
404 Heap->MaxAllocBytes = max(Heap->MaxAllocBytes, Heap->CurrentAllocBytes);
405 Heap->LargestAllocation = max(Heap->LargestAllocation,
406 Block->Size * sizeof(HEAP_BLOCK));
407 #if DBG && !defined(_M_ARM)
408 Heap->AllocationTime += (__rdtsc() - Time);
409 #endif
410 TRACE("HeapAllocate(%p, %ld, %.4s) -> return %p\n",
411 HeapHandle, ByteSize, &Tag, Block->Data);
412
413 /* HACK: zero out the allocation */
414 RtlZeroMemory(Block->Data, Block->Size * sizeof(HEAP_BLOCK));
415
416 #ifdef FREELDR_HEAP_VERIFIER
417 /* Write size and redzones */
418 *REDZONE_SIZE(Block) = ByteSize - REDZONE_ALLOCATION;
419 *REDZONE_LOW(Block) = REDZONE_MARK;
420 *REDZONE_HI(Block) = REDZONE_MARK;
421
422 /* Allocation starts after size field and redzone */
423 return (PUCHAR)Block->Data + REDZONE_LOW_OFFSET;
424 #endif
425 /* Return pointer to the data */
426 return Block->Data;
427 }
428
429 /* We found nothing */
430 WARN("HEAP: nothing suitable found for 0x%lx bytes\n", ByteSize);
431 return NULL;
432 }
433
434 VOID
435 FrLdrHeapFreeEx(
436 PVOID HeapHandle,
437 PVOID Pointer,
438 ULONG Tag)
439 {
440 PHEAP Heap = HeapHandle;
441 PHEAP_BLOCK Block, PrevBlock, NextBlock;
442 #if DBG && !defined(_M_ARM)
443 ULONGLONG Time = __rdtsc();
444 #endif
445 TRACE("HeapFree(%p, %p)\n", HeapHandle, Pointer);
446 ASSERT(Tag != 'dnE#');
447
448 #ifdef FREELDR_HEAP_VERIFIER
449 /* Verify the heap */
450 FrLdrHeapVerify(HeapHandle);
451 #endif
452
453 /* Check if the block is really inside this heap */
454 if ((Pointer < (PVOID)(Heap + 1)) ||
455 (Pointer > (PVOID)((PUCHAR)Heap + Heap->MaximumSize)))
456 {
457 ERR("HEAP: trying to free %p outside of heap %p\n", Pointer, Heap);
458 ASSERT(FALSE);
459 }
460
461 Block = ((PHEAP_BLOCK)Pointer) - 1;
462 #ifdef FREELDR_HEAP_VERIFIER
463 Block = (PHEAP_BLOCK)((PUCHAR)Block - REDZONE_LOW_OFFSET);
464
465 /* Verify size and redzones */
466 ASSERT(*REDZONE_SIZE(Block) <= Block->Size * sizeof(HEAP_BLOCK));
467 ASSERT(*REDZONE_LOW(Block) == REDZONE_MARK);
468 ASSERT(*REDZONE_HI(Block) == REDZONE_MARK);
469 #endif
470
471 /* Check if the tag matches */
472 if ((Tag && (Block->Tag != Tag)) || (Block->Tag == 0))
473 {
474 ERR("HEAP: Bad tag! Pointer=%p: block tag '%.4s', requested '%.4s', size=0x%lx\n",
475 Pointer, &Block->Tag, &Tag, Block->Size);
476 ASSERT(FALSE);
477 }
478
479 /* Mark as free */
480 Block->Tag = 0;
481
482 #if DBG
483 /* Erase contents */
484 RtlFillMemory(Block->Data, Block->Size * sizeof(HEAP_BLOCK), 0xCCCCCCCC);
485 #endif
486
487 /* Update heap usage */
488 Heap->NumFrees++;
489 Heap->CurrentAllocBytes -= Block->Size * sizeof(HEAP_BLOCK);
490
491 /* Get pointers to the next and previous block */
492 PrevBlock = Block - Block->PreviousSize - 1;
493 NextBlock = Block + Block->Size + 1;
494
495 /* Check if next block is free */
496 if ((NextBlock->Tag == 0) &&
497 ((Block->Size + NextBlock->Size + 1) <= MAXUSHORT))
498 {
499 /* Merge next block into current */
500 Block->Size += NextBlock->Size + 1;
501 FrLdrHeapRemoveFreeList(Heap, NextBlock);
502
503 NextBlock = Block + Block->Size + 1;
504 }
505
506 /* Check if there is a block before and it's free */
507 if ((Block->PreviousSize != 0) && (PrevBlock->Tag == 0) &&
508 ((PrevBlock->Size + Block->Size + 1) <= MAXUSHORT))
509 {
510 /* Merge current block into previous */
511 PrevBlock->Size += Block->Size + 1;
512 Block = PrevBlock;
513 }
514 else
515 {
516 /* Insert the entry into the free list */
517 FrLdrHeapInsertFreeList(Heap, Block);
518 }
519
520 /* Update the next block's back link */
521 NextBlock->PreviousSize = Block->Size;
522 #if DBG && !defined(_M_ARM)
523 Heap->FreeTime += (__rdtsc() - Time);
524 #endif
525 }
526
527
528 /* Wrapper functions *********************************************************/
529
530 VOID
531 MmInitializeHeap(PVOID PageLookupTable)
532 {
533 TRACE("MmInitializeHeap()\n");
534
535 /* Create the default heap */
536 FrLdrDefaultHeap = FrLdrHeapCreate(DEFAULT_HEAP_SIZE, LoaderOsloaderHeap);
537 ASSERT(FrLdrDefaultHeap);
538
539 /* Create a temporary heap */
540 FrLdrTempHeap = FrLdrHeapCreate(TEMP_HEAP_SIZE, LoaderFirmwareTemporary);
541 ASSERT(FrLdrTempHeap);
542
543 TRACE("MmInitializeHeap() done, default heap %p, temp heap %p\n",
544 FrLdrDefaultHeap, FrLdrTempHeap);
545 }
546
547 PVOID
548 NTAPI
549 ExAllocatePoolWithTag(
550 IN POOL_TYPE PoolType,
551 IN SIZE_T NumberOfBytes,
552 IN ULONG Tag)
553 {
554 return FrLdrHeapAllocateEx(FrLdrDefaultHeap, NumberOfBytes, Tag);
555 }
556
557 PVOID
558 NTAPI
559 ExAllocatePool(
560 IN POOL_TYPE PoolType,
561 IN SIZE_T NumberOfBytes)
562 {
563 return FrLdrHeapAllocateEx(FrLdrDefaultHeap, NumberOfBytes, 0);
564 }
565
566 VOID
567 NTAPI
568 ExFreePool(
569 IN PVOID P)
570 {
571 FrLdrHeapFreeEx(FrLdrDefaultHeap, P, 0);
572 }
573
574 VOID
575 NTAPI
576 ExFreePoolWithTag(
577 IN PVOID P,
578 IN ULONG Tag)
579 {
580 FrLdrHeapFreeEx(FrLdrDefaultHeap, P, Tag);
581 }
582
583 PVOID
584 NTAPI
585 RtlAllocateHeap(
586 IN PVOID HeapHandle,
587 IN ULONG Flags,
588 IN SIZE_T Size)
589 {
590 PVOID ptr;
591
592 ptr = FrLdrHeapAllocateEx(FrLdrDefaultHeap, Size, ' ltR');
593 if (ptr && (Flags & HEAP_ZERO_MEMORY))
594 {
595 RtlZeroMemory(ptr, Size);
596 }
597
598 return ptr;
599 }
600
601 BOOLEAN
602 NTAPI
603 RtlFreeHeap(
604 IN PVOID HeapHandle,
605 IN ULONG Flags,
606 IN PVOID HeapBase)
607 {
608 FrLdrHeapFreeEx(FrLdrDefaultHeap, HeapBase, ' ltR');
609 return TRUE;
610 }
611