2 * COPYRIGHT: See COPYING in the top level directory
3 * PROJECT: ReactOS kernel
4 * FILE: ntoskrnl/mm/balance.c
5 * PURPOSE: kernel memory managment functions
7 * PROGRAMMERS: David Welch (welch@cwcom.net)
8 * Cameron Gutman (cameron.gutman@reactos.org)
11 /* INCLUDES *****************************************************************/
17 #include "ARM3/miarm.h"
19 #if defined (ALLOC_PRAGMA)
20 #pragma alloc_text(INIT, MmInitializeBalancer)
21 #pragma alloc_text(INIT, MmInitializeMemoryConsumer)
22 #pragma alloc_text(INIT, MiInitBalancerThread)
26 /* TYPES ********************************************************************/
27 typedef struct _MM_ALLOCATION_REQUEST
33 MM_ALLOCATION_REQUEST
, *PMM_ALLOCATION_REQUEST
;
34 /* GLOBALS ******************************************************************/
36 MM_MEMORY_CONSUMER MiMemoryConsumers
[MC_MAXIMUM
];
37 static ULONG MiMinimumAvailablePages
;
38 static ULONG MiNrTotalPages
;
39 static LIST_ENTRY AllocationListHead
;
40 static KSPIN_LOCK AllocationListLock
;
41 static ULONG MiMinimumPagesPerRun
;
43 static CLIENT_ID MiBalancerThreadId
;
44 static HANDLE MiBalancerThreadHandle
= NULL
;
45 static KEVENT MiBalancerEvent
;
46 static KTIMER MiBalancerTimer
;
48 /* FUNCTIONS ****************************************************************/
53 MmInitializeBalancer(ULONG NrAvailablePages
, ULONG NrSystemPages
)
55 memset(MiMemoryConsumers
, 0, sizeof(MiMemoryConsumers
));
56 InitializeListHead(&AllocationListHead
);
57 KeInitializeSpinLock(&AllocationListLock
);
59 MiNrTotalPages
= NrAvailablePages
;
62 MiMinimumAvailablePages
= 128;
63 MiMinimumPagesPerRun
= 256;
64 if ((NrAvailablePages
+ NrSystemPages
) >= 8192)
66 MiMemoryConsumers
[MC_CACHE
].PagesTarget
= NrAvailablePages
/ 4 * 3;
68 else if ((NrAvailablePages
+ NrSystemPages
) >= 4096)
70 MiMemoryConsumers
[MC_CACHE
].PagesTarget
= NrAvailablePages
/ 3 * 2;
74 MiMemoryConsumers
[MC_CACHE
].PagesTarget
= NrAvailablePages
/ 8;
76 MiMemoryConsumers
[MC_USER
].PagesTarget
= NrAvailablePages
- MiMinimumAvailablePages
;
82 MmInitializeMemoryConsumer(ULONG Consumer
,
83 NTSTATUS (*Trim
)(ULONG Target
, ULONG Priority
,
86 MiMemoryConsumers
[Consumer
].Trim
= Trim
;
92 IN PFN_NUMBER PageFrameIndex
97 MmReleasePageMemoryConsumer(ULONG Consumer
, PFN_NUMBER Page
)
99 PMM_ALLOCATION_REQUEST Request
;
105 DPRINT1("Tried to release page zero.\n");
106 KeBugCheck(MEMORY_MANAGEMENT
);
109 if (MmGetReferenceCountPage(Page
) == 1)
111 if(Consumer
== MC_USER
) MmRemoveLRUUserPage(Page
);
112 (void)InterlockedDecrementUL(&MiMemoryConsumers
[Consumer
].PagesUsed
);
113 if ((Entry
= ExInterlockedRemoveHeadList(&AllocationListHead
, &AllocationListLock
)) == NULL
)
115 OldIrql
= KeAcquireQueuedSpinLock(LockQueuePfnLock
);
116 MmDereferencePage(Page
);
117 KeReleaseQueuedSpinLock(LockQueuePfnLock
, OldIrql
);
121 Request
= CONTAINING_RECORD(Entry
, MM_ALLOCATION_REQUEST
, ListEntry
);
122 MiZeroPhysicalPage(Page
);
123 Request
->Page
= Page
;
124 KeSetEvent(&Request
->Event
, IO_NO_INCREMENT
, FALSE
);
129 OldIrql
= KeAcquireQueuedSpinLock(LockQueuePfnLock
);
130 MmDereferencePage(Page
);
131 KeReleaseQueuedSpinLock(LockQueuePfnLock
, OldIrql
);
134 return(STATUS_SUCCESS
);
139 MiTrimMemoryConsumer(ULONG Consumer
, ULONG InitialTarget
)
141 ULONG Target
= InitialTarget
;
142 ULONG NrFreedPages
= 0;
145 /* Make sure we can trim this consumer */
146 if (!MiMemoryConsumers
[Consumer
].Trim
)
148 /* Return the unmodified initial target */
149 return InitialTarget
;
152 if (MiMemoryConsumers
[Consumer
].PagesUsed
> MiMemoryConsumers
[Consumer
].PagesTarget
)
154 /* Consumer page limit exceeded */
155 Target
= max(Target
, MiMemoryConsumers
[Consumer
].PagesUsed
- MiMemoryConsumers
[Consumer
].PagesTarget
);
157 if (MmAvailablePages
< MiMinimumAvailablePages
)
159 /* Global page limit exceeded */
160 Target
= (ULONG
)max(Target
, MiMinimumAvailablePages
- MmAvailablePages
);
167 /* If there was no initial target,
168 * swap at least MiMinimumPagesPerRun */
169 Target
= max(Target
, MiMinimumPagesPerRun
);
172 /* Now swap the pages out */
173 Status
= MiMemoryConsumers
[Consumer
].Trim(Target
, 0, &NrFreedPages
);
175 DPRINT("Trimming consumer %lu: Freed %lu pages with a target of %lu pages\n", Consumer
, NrFreedPages
, Target
);
177 if (!NT_SUCCESS(Status
))
179 KeBugCheck(MEMORY_MANAGEMENT
);
182 /* Update the target */
183 if (NrFreedPages
< Target
)
184 Target
-= NrFreedPages
;
188 /* Return the remaining pages needed to meet the target */
193 /* Initial target is zero and we don't have anything else to add */
199 MmTrimUserMemory(ULONG Target
, ULONG Priority
, PULONG NrFreedPages
)
201 PFN_NUMBER CurrentPage
;
207 CurrentPage
= MmGetLRUFirstUserPage();
208 while (CurrentPage
!= 0 && Target
> 0)
210 Status
= MmPageOutPhysicalAddress(CurrentPage
);
211 if (NT_SUCCESS(Status
))
213 DPRINT("Succeeded\n");
218 NextPage
= MmGetLRUNextUserPage(CurrentPage
);
219 if (NextPage
<= CurrentPage
)
221 /* We wrapped around, so we're done */
224 CurrentPage
= NextPage
;
227 return STATUS_SUCCESS
;
231 MiIsBalancerThread(VOID
)
233 return (MiBalancerThreadHandle
!= NULL
) &&
234 (PsGetCurrentThreadId() == MiBalancerThreadId
.UniqueThread
);
239 MiDeletePte(IN PMMPTE PointerPte
,
240 IN PVOID VirtualAddress
,
241 IN PEPROCESS CurrentProcess
,
242 IN PMMPTE PrototypePte
);
246 MmRebalanceMemoryConsumers(VOID
)
248 if (MiBalancerThreadHandle
!= NULL
&&
249 !MiIsBalancerThread())
251 KeSetEvent(&MiBalancerEvent
, IO_NO_INCREMENT
, FALSE
);
257 MmRequestPageMemoryConsumer(ULONG Consumer
, BOOLEAN CanWait
,
258 PPFN_NUMBER AllocatedPage
)
265 * Make sure we don't exceed our individual target.
267 PagesUsed
= InterlockedIncrementUL(&MiMemoryConsumers
[Consumer
].PagesUsed
);
268 if (PagesUsed
> MiMemoryConsumers
[Consumer
].PagesTarget
&&
269 !MiIsBalancerThread())
271 MmRebalanceMemoryConsumers();
275 * Allocate always memory for the non paged pool and for the pager thread.
277 if ((Consumer
== MC_SYSTEM
) || MiIsBalancerThread())
279 OldIrql
= KeAcquireQueuedSpinLock(LockQueuePfnLock
);
280 Page
= MmAllocPage(Consumer
);
281 KeReleaseQueuedSpinLock(LockQueuePfnLock
, OldIrql
);
284 KeBugCheck(NO_PAGES_AVAILABLE
);
286 if (Consumer
== MC_USER
) MmInsertLRULastUserPage(Page
);
287 *AllocatedPage
= Page
;
288 if (MmAvailablePages
< MiMinimumAvailablePages
)
289 MmRebalanceMemoryConsumers();
290 return(STATUS_SUCCESS
);
294 * Make sure we don't exceed global targets.
296 if (MmAvailablePages
< MiMinimumAvailablePages
)
298 MM_ALLOCATION_REQUEST Request
;
302 (void)InterlockedDecrementUL(&MiMemoryConsumers
[Consumer
].PagesUsed
);
303 MmRebalanceMemoryConsumers();
304 return(STATUS_NO_MEMORY
);
307 /* Insert an allocation request. */
309 KeInitializeEvent(&Request
.Event
, NotificationEvent
, FALSE
);
311 ExInterlockedInsertTailList(&AllocationListHead
, &Request
.ListEntry
, &AllocationListLock
);
312 MmRebalanceMemoryConsumers();
314 KeWaitForSingleObject(&Request
.Event
,
323 KeBugCheck(NO_PAGES_AVAILABLE
);
326 if(Consumer
== MC_USER
) MmInsertLRULastUserPage(Page
);
327 *AllocatedPage
= Page
;
329 if (MmAvailablePages
< MiMinimumAvailablePages
)
331 MmRebalanceMemoryConsumers();
334 return(STATUS_SUCCESS
);
338 * Actually allocate the page.
340 OldIrql
= KeAcquireQueuedSpinLock(LockQueuePfnLock
);
341 Page
= MmAllocPage(Consumer
);
342 KeReleaseQueuedSpinLock(LockQueuePfnLock
, OldIrql
);
345 KeBugCheck(NO_PAGES_AVAILABLE
);
347 if(Consumer
== MC_USER
) MmInsertLRULastUserPage(Page
);
348 *AllocatedPage
= Page
;
350 if (MmAvailablePages
< MiMinimumAvailablePages
)
352 MmRebalanceMemoryConsumers();
355 return(STATUS_SUCCESS
);
359 extern MMPFNLIST MmModifiedPageListByColor
[];
362 MiBalancerThread(PVOID Unused
)
364 PVOID WaitObjects
[2];
368 WaitObjects
[0] = &MiBalancerEvent
;
369 WaitObjects
[1] = &MiBalancerTimer
;
373 Status
= KeWaitForMultipleObjects(2,
382 if (Status
== STATUS_WAIT_0
|| Status
== STATUS_WAIT_1
)
384 ULONG InitialTarget
= 0;
386 #if (_MI_PAGING_LEVELS == 2)
387 if (!MiIsBalancerThread())
389 /* Clean up the unused PDEs */
391 PEPROCESS Process
= PsGetCurrentProcess();
393 /* Acquire PFN lock */
394 KIRQL OldIrql
= KeAcquireQueuedSpinLock(LockQueuePfnLock
);
396 for (Address
= (ULONG_PTR
)MI_LOWEST_VAD_ADDRESS
;
397 Address
< (ULONG_PTR
)MM_HIGHEST_VAD_ADDRESS
;
398 Address
+= (PAGE_SIZE
* PTE_COUNT
))
400 if (MiQueryPageTableReferences((PVOID
)Address
) == 0)
402 pointerPde
= MiAddressToPde(Address
);
403 if (pointerPde
->u
.Hard
.Valid
)
404 MiDeletePte(pointerPde
, MiPdeToPte(pointerPde
), Process
, NULL
);
405 ASSERT(pointerPde
->u
.Hard
.Valid
== 0);
409 KeReleaseQueuedSpinLock(LockQueuePfnLock
, OldIrql
);
414 ULONG OldTarget
= InitialTarget
;
416 /* Trim each consumer */
417 for (i
= 0; i
< MC_MAXIMUM
; i
++)
419 InitialTarget
= MiTrimMemoryConsumer(i
, InitialTarget
);
422 /* No pages left to swap! */
423 if (InitialTarget
!= 0 &&
424 InitialTarget
== OldTarget
)
427 KeBugCheck(NO_PAGES_AVAILABLE
);
429 } while (InitialTarget
!= 0);
431 if (MmModifiedPageListByColor
[0].Total
!= 0)
432 DPRINT1("There are %u pages ready to be paged out in the modified list.\n", MmModifiedPageListByColor
[0].Total
);
436 DPRINT1("KeWaitForMultipleObjects failed, status = %x\n", Status
);
437 KeBugCheck(MEMORY_MANAGEMENT
);
445 MiInitBalancerThread(VOID
)
449 #if !defined(__GNUC__)
451 LARGE_INTEGER dummyJunkNeeded
;
452 dummyJunkNeeded
.QuadPart
= -20000000; /* 2 sec */
457 KeInitializeEvent(&MiBalancerEvent
, SynchronizationEvent
, FALSE
);
458 KeInitializeTimerEx(&MiBalancerTimer
, SynchronizationTimer
);
459 KeSetTimerEx(&MiBalancerTimer
,
460 #if defined(__GNUC__)
461 (LARGE_INTEGER
)(LONGLONG
)-20000000LL, /* 2 sec */
468 Status
= PsCreateSystemThread(&MiBalancerThreadHandle
,
475 if (!NT_SUCCESS(Status
))
477 KeBugCheck(MEMORY_MANAGEMENT
);
480 Priority
= LOW_REALTIME_PRIORITY
+ 1;
481 NtSetInformationThread(MiBalancerThreadHandle
,