Sync to trunk head(r38096)
[reactos.git] / reactos / ntoskrnl / ke / thrdschd.c
1 /*
2 * PROJECT: ReactOS Kernel
3 * LICENSE: GPL - See COPYING in the top level directory
4 * FILE: ntoskrnl/ke/thrdschd.c
5 * PURPOSE: Kernel Thread Scheduler (Affinity, Priority, Scheduling)
6 * PROGRAMMERS: Alex Ionescu (alex.ionescu@reactos.org)
7 */
8
9 /* INCLUDES ******************************************************************/
10
11 #include <ntoskrnl.h>
12 #define NDEBUG
13 #include <debug.h>
14
15 /* GLOBALS *******************************************************************/
16
17 ULONG KiIdleSummary;
18 ULONG KiIdleSMTSummary;
19
20 /* FUNCTIONS *****************************************************************/
21
22 PKTHREAD
23 FASTCALL
24 KiIdleSchedule(IN PKPRCB Prcb)
25 {
26 /* FIXME: TODO */
27 ASSERTMSG("Not yet implemented\n", FALSE);
28 return NULL;
29 }
30
31 VOID
32 FASTCALL
33 KiProcessDeferredReadyList(IN PKPRCB Prcb)
34 {
35 /* FIXME: TODO */
36 ASSERTMSG("Not yet implemented\n", FALSE);
37 }
38
39 VOID
40 FASTCALL
41 KiQueueReadyThread(IN PKTHREAD Thread,
42 IN PKPRCB Prcb)
43 {
44 /* Call the macro. We keep the API for compatibility with ASM code */
45 KxQueueReadyThread(Thread, Prcb);
46 }
47
48 VOID
49 NTAPI
50 KiDeferredReadyThread(IN PKTHREAD Thread)
51 {
52 PKPRCB Prcb;
53 BOOLEAN Preempted;
54 ULONG Processor = 0;
55 KPRIORITY OldPriority;
56 PKTHREAD NextThread;
57
58 /* Sanity checks */
59 ASSERT(Thread->State == DeferredReady);
60 ASSERT((Thread->Priority >= 0) && (Thread->Priority <= HIGH_PRIORITY));
61
62 /* Check if we have any adjusts to do */
63 if (Thread->AdjustReason == AdjustBoost)
64 {
65 /* Lock the thread */
66 KiAcquireThreadLock(Thread);
67
68 /* Check if the priority is low enough to qualify for boosting */
69 if ((Thread->Priority <= Thread->AdjustIncrement) &&
70 (Thread->Priority < (LOW_REALTIME_PRIORITY - 3)) &&
71 !(Thread->DisableBoost))
72 {
73 /* Calculate the new priority based on the adjust increment */
74 OldPriority = min(Thread->AdjustIncrement + 1,
75 LOW_REALTIME_PRIORITY - 3);
76
77 /* Make sure we're not decreasing outside of the priority range */
78 ASSERT((Thread->PriorityDecrement >= 0) &&
79 (Thread->PriorityDecrement <= Thread->Priority));
80
81 /* Calculate the new priority decrement based on the boost */
82 Thread->PriorityDecrement += ((SCHAR)OldPriority - Thread->Priority);
83
84 /* Again verify that this decrement is valid */
85 ASSERT((Thread->PriorityDecrement >= 0) &&
86 (Thread->PriorityDecrement <= OldPriority));
87
88 /* Set the new priority */
89 Thread->Priority = (SCHAR)OldPriority;
90 }
91
92 /* We need 4 quanta, make sure we have them, then decrease by one */
93 if (Thread->Quantum < 4) Thread->Quantum = 4;
94 Thread->Quantum--;
95
96 /* Make sure the priority is still valid */
97 ASSERT((Thread->Priority >= 0) && (Thread->Priority <= HIGH_PRIORITY));
98
99 /* Release the lock and clear the adjust reason */
100 KiReleaseThreadLock(Thread);
101 Thread->AdjustReason = AdjustNone;
102 }
103 else if (Thread->AdjustReason == AdjustUnwait)
104 {
105 /* Acquire the thread lock and check if this is a real-time thread */
106 KiAcquireThreadLock(Thread);
107 if (Thread->Priority < LOW_REALTIME_PRIORITY)
108 {
109 /* It's not real time, but is it time critical? */
110 if (Thread->BasePriority >= (LOW_REALTIME_PRIORITY - 2))
111 {
112 /* It is, so simply reset its quantum */
113 Thread->Quantum = Thread->QuantumReset;
114 }
115 else
116 {
117 /* Has the priority been adjusted previously? */
118 if (!(Thread->PriorityDecrement) && (Thread->AdjustIncrement))
119 {
120 /* Yes, reset its quantum */
121 Thread->Quantum = Thread->QuantumReset;
122 }
123
124 /* Wait code already handles quantum adjustment during APCs */
125 if (Thread->WaitStatus != STATUS_KERNEL_APC)
126 {
127 /* Decrease the quantum by one and check if we're out */
128 if (--Thread->Quantum <= 0)
129 {
130 /* We are, reset the quantum and get a new priority */
131 Thread->Quantum = Thread->QuantumReset;
132 Thread->Priority = KiComputeNewPriority(Thread, 1);
133 }
134 }
135 }
136
137 /* Now check if we have no decrement and boosts are enabled */
138 if (!(Thread->PriorityDecrement) && !(Thread->DisableBoost))
139 {
140 /* Make sure we have an increment */
141 ASSERT(Thread->AdjustIncrement >= 0);
142
143 /* Calculate the new priority after the increment */
144 OldPriority = Thread->BasePriority + Thread->AdjustIncrement;
145
146 /* Check if this new priority is higher */
147 if (OldPriority > Thread->Priority)
148 {
149 /* Make sure we don't go into the real time range */
150 if (OldPriority >= LOW_REALTIME_PRIORITY)
151 {
152 /* Normalize it back down one notch */
153 OldPriority = LOW_REALTIME_PRIORITY - 1;
154 }
155
156 /* Check if the priority is higher then the boosted base */
157 if (OldPriority > (Thread->BasePriority +
158 Thread->AdjustIncrement))
159 {
160 /* Setup a priority decrement to nullify the boost */
161 Thread->PriorityDecrement = ((SCHAR)OldPriority -
162 Thread->BasePriority -
163 Thread->AdjustIncrement);
164 }
165
166 /* Make sure that the priority decrement is valid */
167 ASSERT((Thread->PriorityDecrement >= 0) &&
168 (Thread->PriorityDecrement <= OldPriority));
169
170 /* Set this new priority */
171 Thread->Priority = (SCHAR)OldPriority;
172 }
173 }
174 }
175 else
176 {
177 /* It's a real-time thread, so just reset its quantum */
178 Thread->Quantum = Thread->QuantumReset;
179 }
180
181 /* Make sure the priority makes sense */
182 ASSERT((Thread->Priority >= 0) && (Thread->Priority <= HIGH_PRIORITY));
183
184 /* Release the thread lock and reset the adjust reason */
185 KiReleaseThreadLock(Thread);
186 Thread->AdjustReason = AdjustNone;
187 }
188
189 /* Clear thread preemption status and save current values */
190 Preempted = Thread->Preempted;
191 OldPriority = Thread->Priority;
192 Thread->Preempted = FALSE;
193
194 /* Queue the thread on CPU 0 and get the PRCB */
195 Thread->NextProcessor = 0;
196 Prcb = KiProcessorBlock[0];
197
198 /* Check if we have an idle summary */
199 if (KiIdleSummary)
200 {
201 /* Clear it and set this thread as the next one */
202 KiIdleSummary = 0;
203 Thread->State = Standby;
204 Prcb->NextThread = Thread;
205 return;
206 }
207
208 /* Set the CPU number */
209 Thread->NextProcessor = (UCHAR)Processor;
210
211 /* Get the next scheduled thread */
212 NextThread = Prcb->NextThread;
213 if (NextThread)
214 {
215 /* Sanity check */
216 ASSERT(NextThread->State == Standby);
217
218 /* Check if priority changed */
219 if (OldPriority > NextThread->Priority)
220 {
221 /* Preempt the thread */
222 NextThread->Preempted = TRUE;
223
224 /* Put this one as the next one */
225 Thread->State = Standby;
226 Prcb->NextThread = Thread;
227
228 /* Set it in deferred ready mode */
229 NextThread->State = DeferredReady;
230 NextThread->DeferredProcessor = Prcb->Number;
231 KiReleasePrcbLock(Prcb);
232 KiDeferredReadyThread(NextThread);
233 return;
234 }
235 }
236 else
237 {
238 /* Set the next thread as the current thread */
239 NextThread = Prcb->CurrentThread;
240 if (OldPriority > NextThread->Priority)
241 {
242 /* Preempt it if it's already running */
243 if (NextThread->State == Running) NextThread->Preempted = TRUE;
244
245 /* Set the thread on standby and as the next thread */
246 Thread->State = Standby;
247 Prcb->NextThread = Thread;
248
249 /* Release the lock */
250 KiReleasePrcbLock(Prcb);
251
252 /* Check if we're running on another CPU */
253 if (KeGetCurrentProcessorNumber() != Thread->NextProcessor)
254 {
255 /* We are, send an IPI */
256 KiIpiSend(AFFINITY_MASK(Thread->NextProcessor), IPI_DPC);
257 }
258 return;
259 }
260 }
261
262 /* Sanity check */
263 ASSERT((OldPriority >= 0) && (OldPriority <= HIGH_PRIORITY));
264
265 /* Set this thread as ready */
266 Thread->State = Ready;
267 Thread->WaitTime = KeTickCount.LowPart;
268
269 /* Insert this thread in the appropriate order */
270 Preempted ? InsertHeadList(&Prcb->DispatcherReadyListHead[OldPriority],
271 &Thread->WaitListEntry) :
272 InsertTailList(&Prcb->DispatcherReadyListHead[OldPriority],
273 &Thread->WaitListEntry);
274
275 /* Update the ready summary */
276 Prcb->ReadySummary |= PRIORITY_MASK(OldPriority);
277
278 /* Sanity check */
279 ASSERT(OldPriority == Thread->Priority);
280
281 /* Release the lock */
282 KiReleasePrcbLock(Prcb);
283 }
284
285 PKTHREAD
286 FASTCALL
287 KiSelectNextThread(IN PKPRCB Prcb)
288 {
289 PKTHREAD Thread;
290
291 /* Select a ready thread */
292 Thread = KiSelectReadyThread(0, Prcb);
293 if (!Thread)
294 {
295 /* Didn't find any, get the current idle thread */
296 Thread = Prcb->IdleThread;
297
298 /* Enable idle scheduling */
299 InterlockedOr((PLONG) &KiIdleSummary, Prcb->SetMember);
300 Prcb->IdleSchedule = TRUE;
301
302 /* FIXME: SMT support */
303 }
304
305 /* Sanity checks and return the thread */
306 ASSERT(Thread != NULL);
307 ASSERT((Thread->BasePriority == 0) || (Thread->Priority != 0));
308 return Thread;
309 }
310
311 LONG_PTR
312 FASTCALL
313 KiSwapThread(IN PKTHREAD CurrentThread,
314 IN PKPRCB Prcb)
315 {
316 BOOLEAN ApcState = FALSE;
317 KIRQL WaitIrql;
318 LONG_PTR WaitStatus;
319 PKTHREAD NextThread;
320 ASSERT(KeGetCurrentIrql() >= DISPATCH_LEVEL);
321
322 /* Acquire the PRCB lock */
323 KiAcquirePrcbLock(Prcb);
324
325 /* Get the next thread */
326 NextThread = Prcb->NextThread;
327 if (NextThread)
328 {
329 /* Already got a thread, set it up */
330 Prcb->NextThread = NULL;
331 Prcb->CurrentThread = NextThread;
332 NextThread->State = Running;
333 }
334 else
335 {
336 /* Try to find a ready thread */
337 NextThread = KiSelectReadyThread(0, Prcb);
338 if (NextThread)
339 {
340 /* Switch to it */
341 Prcb->CurrentThread = NextThread;
342 NextThread->State = Running;
343 }
344 else
345 {
346 /* Set the idle summary */
347 InterlockedOr((PLONG)&KiIdleSummary, Prcb->SetMember);
348
349 /* Schedule the idle thread */
350 NextThread = Prcb->IdleThread;
351 Prcb->CurrentThread = NextThread;
352 NextThread->State = Running;
353 }
354 }
355
356 /* Sanity check and release the PRCB */
357 ASSERT(CurrentThread != Prcb->IdleThread);
358 KiReleasePrcbLock(Prcb);
359
360 /* Save the wait IRQL */
361 WaitIrql = CurrentThread->WaitIrql;
362
363 /* REACTOS Mm Hack of Doom */
364 MiSyncForContextSwitch(NextThread);
365
366 /* Swap contexts */
367 ApcState = KiSwapContext(CurrentThread, NextThread);
368
369 /* Get the wait status */
370 WaitStatus = CurrentThread->WaitStatus;
371
372 /* Check if we need to deliver APCs */
373 if (ApcState)
374 {
375 /* Lower to APC_LEVEL */
376 KeLowerIrql(APC_LEVEL);
377
378 /* Deliver APCs */
379 KiDeliverApc(KernelMode, NULL, NULL);
380 ASSERT(WaitIrql == 0);
381 }
382
383 /* Lower IRQL back to what it was and return the wait status */
384 KeLowerIrql(WaitIrql);
385 return WaitStatus;
386 }
387
388 VOID
389 NTAPI
390 KiReadyThread(IN PKTHREAD Thread)
391 {
392 IN PKPROCESS Process = Thread->ApcState.Process;
393
394 /* Check if the process is paged out */
395 if (Process->State != ProcessInMemory)
396 {
397 /* We don't page out processes in ROS */
398 ASSERT(FALSE);
399 }
400 else if (!Thread->KernelStackResident)
401 {
402 /* Increase the stack count */
403 ASSERT(Process->StackCount != MAXULONG_PTR);
404 Process->StackCount++;
405
406 /* Set the thread to transition */
407 ASSERT(Thread->State != Transition);
408 Thread->State = Transition;
409
410 /* The stack is always resident in ROS */
411 ASSERT(FALSE);
412 }
413 else
414 {
415 /* Insert the thread on the deferred ready list */
416 KiInsertDeferredReadyList(Thread);
417 }
418 }
419
420 VOID
421 NTAPI
422 KiAdjustQuantumThread(IN PKTHREAD Thread)
423 {
424 PKPRCB Prcb = KeGetCurrentPrcb();
425 PKTHREAD NextThread;
426
427 /* Acquire thread and PRCB lock */
428 KiAcquireThreadLock(Thread);
429 KiAcquirePrcbLock(Prcb);
430
431 /* Don't adjust for RT threads */
432 if ((Thread->Priority < LOW_REALTIME_PRIORITY) &&
433 (Thread->BasePriority < (LOW_REALTIME_PRIORITY - 2)))
434 {
435 /* Decrease Quantum by one and see if we've ran out */
436 if (--Thread->Quantum <= 0)
437 {
438 /* Return quantum */
439 Thread->Quantum = Thread->QuantumReset;
440
441 /* Calculate new Priority */
442 Thread->Priority = KiComputeNewPriority(Thread, 1);
443
444 /* Check if there's no next thread scheduled */
445 if (!Prcb->NextThread)
446 {
447 /* Select a ready thread and check if we found one */
448 NextThread = KiSelectReadyThread(Thread->Priority, Prcb);
449 if (NextThread)
450 {
451 /* Set it on standby and switch to it */
452 NextThread->State = Standby;
453 Prcb->NextThread = NextThread;
454 }
455 }
456 else
457 {
458 /* This thread can be preempted again */
459 Thread->Preempted = FALSE;
460 }
461 }
462 }
463
464 /* Release locks */
465 KiReleasePrcbLock(Prcb);
466 KiReleaseThreadLock(Thread);
467 KiExitDispatcher(Thread->WaitIrql);
468 }
469
470 VOID
471 FASTCALL
472 KiSetPriorityThread(IN PKTHREAD Thread,
473 IN KPRIORITY Priority)
474 {
475 PKPRCB Prcb;
476 ULONG Processor;
477 BOOLEAN RequestInterrupt = FALSE;
478 KPRIORITY OldPriority;
479 PKTHREAD NewThread;
480 ASSERT((Priority >= 0) && (Priority <= HIGH_PRIORITY));
481
482 /* Check if priority changed */
483 if (Thread->Priority != Priority)
484 {
485 /* Loop priority setting in case we need to start over */
486 for (;;)
487 {
488 /* Choose action based on thread's state */
489 if (Thread->State == Ready)
490 {
491 /* Make sure we're not on the ready queue */
492 if (!Thread->ProcessReadyQueue)
493 {
494 /* Get the PRCB for the thread and lock it */
495 Processor = Thread->NextProcessor;
496 Prcb = KiProcessorBlock[Processor];
497 KiAcquirePrcbLock(Prcb);
498
499 /* Make sure the thread is still ready and on this CPU */
500 if ((Thread->State == Ready) &&
501 (Thread->NextProcessor == Prcb->Number))
502 {
503 /* Sanity check */
504 ASSERT((Prcb->ReadySummary &
505 PRIORITY_MASK(Thread->Priority)));
506
507 /* Remove it from the current queue */
508 if (RemoveEntryList(&Thread->WaitListEntry))
509 {
510 /* Update the ready summary */
511 Prcb->ReadySummary ^= PRIORITY_MASK(Thread->
512 Priority);
513 }
514
515 /* Update priority */
516 Thread->Priority = (SCHAR)Priority;
517
518 /* Re-insert it at its current priority */
519 KiInsertDeferredReadyList(Thread);
520
521 /* Release the PRCB Lock */
522 KiReleasePrcbLock(Prcb);
523 }
524 else
525 {
526 /* Release the lock and loop again */
527 KiReleasePrcbLock(Prcb);
528 continue;
529 }
530 }
531 else
532 {
533 /* It's already on the ready queue, just update priority */
534 Thread->Priority = (SCHAR)Priority;
535 }
536 }
537 else if (Thread->State == Standby)
538 {
539 /* Get the PRCB for the thread and lock it */
540 Processor = Thread->NextProcessor;
541 Prcb = KiProcessorBlock[Processor];
542 KiAcquirePrcbLock(Prcb);
543
544 /* Check if we're still the next thread to run */
545 if (Thread == Prcb->NextThread)
546 {
547 /* Get the old priority and update ours */
548 OldPriority = Thread->Priority;
549 Thread->Priority = (SCHAR)Priority;
550
551 /* Check if there was a change */
552 if (Priority < OldPriority)
553 {
554 /* Find a new thread */
555 NewThread = KiSelectReadyThread(Priority + 1, Prcb);
556 if (NewThread)
557 {
558 /* Found a new one, set it on standby */
559 NewThread->State = Standby;
560 Prcb->NextThread = NewThread;
561
562 /* Dispatch our thread */
563 KiInsertDeferredReadyList(Thread);
564 }
565 }
566
567 /* Release the PRCB lock */
568 KiReleasePrcbLock(Prcb);
569 }
570 else
571 {
572 /* Release the lock and try again */
573 KiReleasePrcbLock(Prcb);
574 continue;
575 }
576 }
577 else if (Thread->State == Running)
578 {
579 /* Get the PRCB for the thread and lock it */
580 Processor = Thread->NextProcessor;
581 Prcb = KiProcessorBlock[Processor];
582 KiAcquirePrcbLock(Prcb);
583
584 /* Check if we're still the current thread running */
585 if (Thread == Prcb->CurrentThread)
586 {
587 /* Get the old priority and update ours */
588 OldPriority = Thread->Priority;
589 Thread->Priority = (SCHAR)Priority;
590
591 /* Check if there was a change and there's no new thread */
592 if ((Priority < OldPriority) && !(Prcb->NextThread))
593 {
594 /* Find a new thread */
595 NewThread = KiSelectReadyThread(Priority + 1, Prcb);
596 if (NewThread)
597 {
598 /* Found a new one, set it on standby */
599 NewThread->State = Standby;
600 Prcb->NextThread = NewThread;
601
602 /* Request an interrupt */
603 RequestInterrupt = TRUE;
604 }
605 }
606
607 /* Release the lock and check if we need an interrupt */
608 KiReleasePrcbLock(Prcb);
609 if (RequestInterrupt)
610 {
611 /* Check if we're running on another CPU */
612 if (KeGetCurrentProcessorNumber() != Processor)
613 {
614 /* We are, send an IPI */
615 KiIpiSend(AFFINITY_MASK(Processor), IPI_DPC);
616 }
617 }
618 }
619 else
620 {
621 /* Thread changed, release lock and restart */
622 KiReleasePrcbLock(Prcb);
623 continue;
624 }
625 }
626 else if (Thread->State == DeferredReady)
627 {
628 /* FIXME: TODO */
629 DPRINT1("Deferred state not yet supported\n");
630 ASSERT(FALSE);
631 }
632 else
633 {
634 /* Any other state, just change priority */
635 Thread->Priority = (SCHAR)Priority;
636 }
637
638 /* If we got here, then thread state was consistent, so bail out */
639 break;
640 }
641 }
642 }
643
644 KAFFINITY
645 FASTCALL
646 KiSetAffinityThread(IN PKTHREAD Thread,
647 IN KAFFINITY Affinity)
648 {
649 KAFFINITY OldAffinity;
650
651 /* Get the current affinity */
652 OldAffinity = Thread->UserAffinity;
653
654 /* Make sure that the affinity is valid */
655 if (((Affinity & Thread->ApcState.Process->Affinity) != (Affinity)) ||
656 (!Affinity))
657 {
658 /* Bugcheck the system */
659 KeBugCheck(INVALID_AFFINITY_SET);
660 }
661
662 /* Update the new affinity */
663 Thread->UserAffinity = Affinity;
664
665 /* Check if system affinity is disabled */
666 if (!Thread->SystemAffinityActive)
667 {
668 /* FIXME: TODO */
669 DPRINT1("Affinity support disabled!\n");
670 }
671
672 /* Return the old affinity */
673 return OldAffinity;
674 }
675
676 /*
677 * @implemented
678 */
679 NTSTATUS
680 NTAPI
681 NtYieldExecution(VOID)
682 {
683 NTSTATUS Status = STATUS_NO_YIELD_PERFORMED;
684 KIRQL OldIrql;
685 PKPRCB Prcb = KeGetCurrentPrcb();
686 PKTHREAD Thread = KeGetCurrentThread(), NextThread;
687
688 /* Fail if there's no ready summary */
689 if (!Prcb->ReadySummary) return Status;
690
691 /* Raise IRQL to synch */
692 OldIrql = KeRaiseIrqlToSynchLevel();
693
694 /* Now check if there's still a ready summary */
695 if (Prcb->ReadySummary)
696 {
697 /* Acquire thread and PRCB lock */
698 KiAcquireThreadLock(Thread);
699 KiAcquirePrcbLock(Prcb);
700
701 /* Find a new thread to run if none was selected */
702 if (!Prcb->NextThread) Prcb->NextThread = KiSelectReadyThread(1, Prcb);
703
704 /* Make sure we still have a next thread to schedule */
705 NextThread = Prcb->NextThread;
706 if (NextThread)
707 {
708 /* Reset quantum and recalculate priority */
709 Thread->Quantum = Thread->QuantumReset;
710 Thread->Priority = KiComputeNewPriority(Thread, 1);
711
712 /* Release the thread lock */
713 KiReleaseThreadLock(Thread);
714
715 /* Set context swap busy */
716 KiSetThreadSwapBusy(Thread);
717
718 /* Set the new thread as running */
719 Prcb->NextThread = NULL;
720 Prcb->CurrentThread = NextThread;
721 NextThread->State = Running;
722
723 /* Setup a yield wait and queue the thread */
724 Thread->WaitReason = WrYieldExecution;
725 KxQueueReadyThread(Thread, Prcb);
726
727 /* Make it wait at APC_LEVEL */
728 Thread->WaitIrql = APC_LEVEL;
729
730 /* Sanity check */
731 ASSERT(OldIrql <= DISPATCH_LEVEL);
732
733 /* REACTOS Mm Hack of Doom */
734 MiSyncForContextSwitch(NextThread);
735
736 /* Swap to new thread */
737 KiSwapContext(Thread, NextThread);
738 Status = STATUS_SUCCESS;
739 }
740 else
741 {
742 /* Release the PRCB and thread lock */
743 KiReleasePrcbLock(Prcb);
744 KiReleaseThreadLock(Thread);
745 }
746 }
747
748 /* Lower IRQL and return */
749 KeLowerIrql(OldIrql);
750 return Status;
751 }