c90d447547bdac5801068433e55cca3c49f9f888
[reactos.git] / reactos / ntoskrnl / mm / balance.c
1 /*
2 * COPYRIGHT: See COPYING in the top level directory
3 * PROJECT: ReactOS kernel
4 * FILE: ntoskrnl/mm/balance.c
5 * PURPOSE: kernel memory managment functions
6 *
7 * PROGRAMMERS: David Welch (welch@cwcom.net)
8 */
9
10 /* INCLUDES *****************************************************************/
11
12 #include <ntoskrnl.h>
13 #define NDEBUG
14 #include <debug.h>
15
16 #if defined (ALLOC_PRAGMA)
17 #pragma alloc_text(INIT, MmInitializeBalancer)
18 #pragma alloc_text(INIT, MmInitializeMemoryConsumer)
19 #pragma alloc_text(INIT, MiInitBalancerThread)
20 #endif
21
22
23 /* TYPES ********************************************************************/
24 typedef struct _MM_ALLOCATION_REQUEST
25 {
26 PFN_NUMBER Page;
27 LIST_ENTRY ListEntry;
28 KEVENT Event;
29 }
30 MM_ALLOCATION_REQUEST, *PMM_ALLOCATION_REQUEST;
31
32 /* GLOBALS ******************************************************************/
33
34 MM_MEMORY_CONSUMER MiMemoryConsumers[MC_MAXIMUM];
35 static ULONG MiMinimumAvailablePages;
36 static ULONG MiNrTotalPages;
37 static LIST_ENTRY AllocationListHead;
38 static KSPIN_LOCK AllocationListLock;
39 static ULONG MiPagesRequired = 0;
40 static ULONG MiMinimumPagesPerRun = 10;
41
42 static CLIENT_ID MiBalancerThreadId;
43 static HANDLE MiBalancerThreadHandle = NULL;
44 static KEVENT MiBalancerEvent;
45 static KTIMER MiBalancerTimer;
46 static LONG MiBalancerWork = 0;
47
48 /* FUNCTIONS ****************************************************************/
49
50 VOID
51 INIT_FUNCTION
52 NTAPI
53 MmInitializeBalancer(ULONG NrAvailablePages, ULONG NrSystemPages)
54 {
55 memset(MiMemoryConsumers, 0, sizeof(MiMemoryConsumers));
56 InitializeListHead(&AllocationListHead);
57 KeInitializeSpinLock(&AllocationListLock);
58
59 MiNrTotalPages = NrAvailablePages;
60
61 /* Set up targets. */
62 MiMinimumAvailablePages = 64;
63 if ((NrAvailablePages + NrSystemPages) >= 8192)
64 {
65 MiMemoryConsumers[MC_CACHE].PagesTarget = NrAvailablePages / 4 * 3;
66 }
67 else if ((NrAvailablePages + NrSystemPages) >= 4096)
68 {
69 MiMemoryConsumers[MC_CACHE].PagesTarget = NrAvailablePages / 3 * 2;
70 }
71 else
72 {
73 MiMemoryConsumers[MC_CACHE].PagesTarget = NrAvailablePages / 8;
74 }
75 MiMemoryConsumers[MC_USER].PagesTarget = NrAvailablePages - MiMinimumAvailablePages;
76 }
77
78 VOID
79 INIT_FUNCTION
80 NTAPI
81 MmInitializeMemoryConsumer(ULONG Consumer,
82 NTSTATUS (*Trim)(ULONG Target, ULONG Priority,
83 PULONG NrFreed))
84 {
85 MiMemoryConsumers[Consumer].Trim = Trim;
86 }
87
88 VOID
89 NTAPI
90 MiZeroPhysicalPage(
91 IN PFN_NUMBER PageFrameIndex
92 );
93
94 NTSTATUS
95 NTAPI
96 MmReleasePageMemoryConsumer(ULONG Consumer, PFN_NUMBER Page)
97 {
98 PMM_ALLOCATION_REQUEST Request;
99 PLIST_ENTRY Entry;
100 KIRQL OldIrql;
101
102 if (Page == 0)
103 {
104 DPRINT1("Tried to release page zero.\n");
105 KeBugCheck(MEMORY_MANAGEMENT);
106 }
107
108 KeAcquireSpinLock(&AllocationListLock, &OldIrql);
109 if (MmGetReferenceCountPage(Page) == 1)
110 {
111 (void)InterlockedDecrementUL(&MiMemoryConsumers[Consumer].PagesUsed);
112 if (IsListEmpty(&AllocationListHead) || MmAvailablePages < MiMinimumAvailablePages)
113 {
114 KeReleaseSpinLock(&AllocationListLock, OldIrql);
115 if(Consumer == MC_USER) MmRemoveLRUUserPage(Page);
116 OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock);
117 MmDereferencePage(Page);
118 KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql);
119 }
120 else
121 {
122 Entry = RemoveHeadList(&AllocationListHead);
123 Request = CONTAINING_RECORD(Entry, MM_ALLOCATION_REQUEST, ListEntry);
124 KeReleaseSpinLock(&AllocationListLock, OldIrql);
125 if(Consumer == MC_USER) MmRemoveLRUUserPage(Page);
126 MiZeroPhysicalPage(Page);
127 Request->Page = Page;
128 KeSetEvent(&Request->Event, IO_NO_INCREMENT, FALSE);
129 }
130 }
131 else
132 {
133 KeReleaseSpinLock(&AllocationListLock, OldIrql);
134 OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock);
135 MmDereferencePage(Page);
136 KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql);
137 }
138
139 return(STATUS_SUCCESS);
140 }
141
142 VOID
143 NTAPI
144 MiTrimMemoryConsumer(ULONG Consumer)
145 {
146 LONG Target;
147 ULONG NrFreedPages;
148
149 Target = MiMemoryConsumers[Consumer].PagesUsed -
150 MiMemoryConsumers[Consumer].PagesTarget;
151 if (Target < 1)
152 {
153 Target = 1;
154 }
155
156 if (MiMemoryConsumers[Consumer].Trim != NULL)
157 {
158 MiMemoryConsumers[Consumer].Trim(Target, 0, &NrFreedPages);
159 }
160 }
161
162 NTSTATUS
163 MmTrimUserMemory(ULONG Target, ULONG Priority, PULONG NrFreedPages)
164 {
165 PFN_NUMBER CurrentPage;
166 PFN_NUMBER NextPage;
167 NTSTATUS Status;
168
169 (*NrFreedPages) = 0;
170
171 CurrentPage = MmGetLRUFirstUserPage();
172 while (CurrentPage != 0 && Target > 0)
173 {
174 Status = MmPageOutPhysicalAddress(CurrentPage);
175 if (NT_SUCCESS(Status))
176 {
177 DPRINT("Succeeded\n");
178 Target--;
179 (*NrFreedPages)++;
180 }
181
182 NextPage = MmGetLRUNextUserPage(CurrentPage);
183 if (NextPage <= CurrentPage)
184 {
185 /* We wrapped around, so we're done */
186 break;
187 }
188 CurrentPage = NextPage;
189 }
190
191 return STATUS_SUCCESS;
192 }
193
194 VOID
195 NTAPI
196 MmRebalanceMemoryConsumers(VOID)
197 {
198 LONG Target;
199 ULONG i;
200 ULONG NrFreedPages;
201 NTSTATUS Status;
202
203 Target = (ULONG)(MiMinimumAvailablePages - MmAvailablePages) + MiPagesRequired;
204 Target = max(Target, (LONG) MiMinimumPagesPerRun);
205
206 for (i = 0; i < MC_MAXIMUM && Target > 0; i++)
207 {
208 if (MiMemoryConsumers[i].Trim != NULL)
209 {
210 Status = MiMemoryConsumers[i].Trim(Target, 0, &NrFreedPages);
211 if (!NT_SUCCESS(Status))
212 {
213 KeBugCheck(MEMORY_MANAGEMENT);
214 }
215 Target = Target - NrFreedPages;
216 }
217 }
218 }
219
220 static BOOLEAN
221 MiIsBalancerThread(VOID)
222 {
223 return (MiBalancerThreadHandle != NULL) &&
224 (PsGetCurrentThreadId() == MiBalancerThreadId.UniqueThread);
225 }
226
227 NTSTATUS
228 NTAPI
229 MmRequestPageMemoryConsumer(ULONG Consumer, BOOLEAN CanWait,
230 PPFN_NUMBER AllocatedPage)
231 {
232 ULONG OldUsed;
233 PFN_NUMBER Page;
234 KIRQL OldIrql;
235
236 /*
237 * Make sure we don't exceed our individual target.
238 */
239 OldUsed = InterlockedIncrementUL(&MiMemoryConsumers[Consumer].PagesUsed);
240 if (OldUsed >= (MiMemoryConsumers[Consumer].PagesTarget - 1) &&
241 !MiIsBalancerThread())
242 {
243 if (!CanWait)
244 {
245 (void)InterlockedDecrementUL(&MiMemoryConsumers[Consumer].PagesUsed);
246 return(STATUS_NO_MEMORY);
247 }
248 MiTrimMemoryConsumer(Consumer);
249 }
250
251 /*
252 * Allocate always memory for the non paged pool and for the pager thread.
253 */
254 if ((Consumer == MC_SYSTEM) || MiIsBalancerThread())
255 {
256 OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock);
257 Page = MmAllocPage(Consumer);
258 KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql);
259 if (Page == 0)
260 {
261 KeBugCheck(NO_PAGES_AVAILABLE);
262 }
263 if (Consumer == MC_USER) MmInsertLRULastUserPage(Page);
264 *AllocatedPage = Page;
265 if (MmAvailablePages <= MiMinimumAvailablePages &&
266 MiBalancerThreadHandle != NULL &&
267 !MiIsBalancerThread())
268 {
269 KeSetEvent(&MiBalancerEvent, IO_NO_INCREMENT, FALSE);
270 }
271 return(STATUS_SUCCESS);
272 }
273
274 /*
275 * Make sure we don't exceed global targets.
276 */
277 if (MmAvailablePages <= MiMinimumAvailablePages)
278 {
279 MM_ALLOCATION_REQUEST Request;
280
281 if (!CanWait)
282 {
283 (void)InterlockedDecrementUL(&MiMemoryConsumers[Consumer].PagesUsed);
284 return(STATUS_NO_MEMORY);
285 }
286
287 /* Insert an allocation request. */
288 Request.Page = 0;
289
290 KeInitializeEvent(&Request.Event, NotificationEvent, FALSE);
291 (void)InterlockedIncrementUL(&MiPagesRequired);
292
293 KeAcquireSpinLock(&AllocationListLock, &OldIrql);
294
295 if (MiBalancerThreadHandle != NULL)
296 {
297 KeSetEvent(&MiBalancerEvent, IO_NO_INCREMENT, FALSE);
298 }
299 InsertTailList(&AllocationListHead, &Request.ListEntry);
300 KeReleaseSpinLock(&AllocationListLock, OldIrql);
301
302 KeWaitForSingleObject(&Request.Event,
303 0,
304 KernelMode,
305 FALSE,
306 NULL);
307
308 Page = Request.Page;
309 if (Page == 0)
310 {
311 KeBugCheck(NO_PAGES_AVAILABLE);
312 }
313
314 if(Consumer == MC_USER) MmInsertLRULastUserPage(Page);
315 *AllocatedPage = Page;
316 (void)InterlockedDecrementUL(&MiPagesRequired);
317
318 if (MmAvailablePages <= MiMinimumAvailablePages &&
319 MiBalancerThreadHandle != NULL &&
320 !MiIsBalancerThread())
321 {
322 KeSetEvent(&MiBalancerEvent, IO_NO_INCREMENT, FALSE);
323 }
324
325 return(STATUS_SUCCESS);
326 }
327
328 /*
329 * Actually allocate the page.
330 */
331 OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock);
332 Page = MmAllocPage(Consumer);
333 KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql);
334 if (Page == 0)
335 {
336 KeBugCheck(NO_PAGES_AVAILABLE);
337 }
338 if(Consumer == MC_USER) MmInsertLRULastUserPage(Page);
339 *AllocatedPage = Page;
340
341 if (MmAvailablePages <= MiMinimumAvailablePages &&
342 MiBalancerThreadHandle != NULL &&
343 !MiIsBalancerThread())
344 {
345 KeSetEvent(&MiBalancerEvent, IO_NO_INCREMENT, FALSE);
346 }
347
348 return(STATUS_SUCCESS);
349 }
350
351 VOID NTAPI
352 MiBalancerThread(PVOID Unused)
353 {
354 PVOID WaitObjects[2];
355 NTSTATUS Status;
356 ULONG i;
357 ULONG NrFreedPages;
358 ULONG NrPagesUsed;
359 ULONG Target;
360 BOOLEAN ShouldRun;
361
362
363 WaitObjects[0] = &MiBalancerEvent;
364 WaitObjects[1] = &MiBalancerTimer;
365
366 while (1)
367 {
368 Status = KeWaitForMultipleObjects(2,
369 WaitObjects,
370 WaitAny,
371 Executive,
372 KernelMode,
373 FALSE,
374 NULL,
375 NULL);
376
377 if (Status == STATUS_SUCCESS)
378 {
379 /* MiBalancerEvent */
380 while (MmAvailablePages < MiMinimumAvailablePages + 5)
381 {
382 for (i = 0; i < MC_MAXIMUM; i++)
383 {
384 if (MiMemoryConsumers[i].Trim != NULL)
385 {
386 NrFreedPages = 0;
387 Status = MiMemoryConsumers[i].Trim(MiMinimumPagesPerRun, 0, &NrFreedPages);
388 if (!NT_SUCCESS(Status))
389 {
390 KeBugCheck(MEMORY_MANAGEMENT);
391 }
392 }
393 }
394 }
395 InterlockedExchange(&MiBalancerWork, 0);
396 }
397 else if (Status == STATUS_SUCCESS + 1)
398 {
399 /* MiBalancerTimer */
400 ShouldRun = MmAvailablePages < MiMinimumAvailablePages + 5 ? TRUE : FALSE;
401 for (i = 0; i < MC_MAXIMUM; i++)
402 {
403 if (MiMemoryConsumers[i].Trim != NULL)
404 {
405 NrPagesUsed = MiMemoryConsumers[i].PagesUsed;
406 if (NrPagesUsed > MiMemoryConsumers[i].PagesTarget || ShouldRun)
407 {
408 if (NrPagesUsed > MiMemoryConsumers[i].PagesTarget)
409 {
410 Target = max (NrPagesUsed - MiMemoryConsumers[i].PagesTarget,
411 MiMinimumPagesPerRun);
412 }
413 else
414 {
415 Target = MiMinimumPagesPerRun;
416 }
417 NrFreedPages = 0;
418 Status = MiMemoryConsumers[i].Trim(Target, 0, &NrFreedPages);
419 if (!NT_SUCCESS(Status))
420 {
421 KeBugCheck(MEMORY_MANAGEMENT);
422 }
423 }
424 }
425 }
426 }
427 else
428 {
429 DPRINT1("KeWaitForMultipleObjects failed, status = %x\n", Status);
430 KeBugCheck(MEMORY_MANAGEMENT);
431 }
432 }
433 }
434
435 VOID
436 INIT_FUNCTION
437 NTAPI
438 MiInitBalancerThread(VOID)
439 {
440 KPRIORITY Priority;
441 NTSTATUS Status;
442 #if !defined(__GNUC__)
443
444 LARGE_INTEGER dummyJunkNeeded;
445 dummyJunkNeeded.QuadPart = -20000000; /* 2 sec */
446 ;
447 #endif
448
449
450 KeInitializeEvent(&MiBalancerEvent, SynchronizationEvent, FALSE);
451 KeInitializeTimerEx(&MiBalancerTimer, SynchronizationTimer);
452 KeSetTimerEx(&MiBalancerTimer,
453 #if defined(__GNUC__)
454 (LARGE_INTEGER)(LONGLONG)-20000000LL, /* 2 sec */
455 #else
456 dummyJunkNeeded,
457 #endif
458 2000, /* 2 sec */
459 NULL);
460
461 Status = PsCreateSystemThread(&MiBalancerThreadHandle,
462 THREAD_ALL_ACCESS,
463 NULL,
464 NULL,
465 &MiBalancerThreadId,
466 (PKSTART_ROUTINE) MiBalancerThread,
467 NULL);
468 if (!NT_SUCCESS(Status))
469 {
470 KeBugCheck(MEMORY_MANAGEMENT);
471 }
472
473 Priority = LOW_REALTIME_PRIORITY + 1;
474 NtSetInformationThread(MiBalancerThreadHandle,
475 ThreadPriority,
476 &Priority,
477 sizeof(Priority));
478
479 }
480
481
482 /* EOF */