Merge from amd64-branch:
[reactos.git] / reactos / ntoskrnl / include / internal / ke_x.h
1 /*
2 * PROJECT: ReactOS Kernel
3 * LICENSE: GPL - See COPYING in the top level directory
4 * FILE: ntoskrnl/include/ke_x.h
5 * PURPOSE: Internal Inlined Functions for the Kernel
6 * PROGRAMMERS: Alex Ionescu (alex.ionescu@reactos.org)
7 */
8
9 #ifndef _M_ARM
10 FORCEINLINE
11 UCHAR
12 KeGetPreviousMode(VOID)
13 {
14 /* Return the current mode */
15 return KeGetCurrentThread()->PreviousMode;
16 }
17 #endif
18
19 //
20 // Enters a Guarded Region
21 //
22 #define KeEnterGuardedRegion() \
23 { \
24 PKTHREAD _Thread = KeGetCurrentThread(); \
25 \
26 /* Sanity checks */ \
27 ASSERT(KeGetCurrentIrql() <= APC_LEVEL); \
28 ASSERT(_Thread == KeGetCurrentThread()); \
29 ASSERT((_Thread->SpecialApcDisable <= 0) && \
30 (_Thread->SpecialApcDisable != -32768)); \
31 \
32 /* Disable Special APCs */ \
33 _Thread->SpecialApcDisable--; \
34 }
35
36 //
37 // Leaves a Guarded Region
38 //
39 #define KeLeaveGuardedRegion() \
40 { \
41 PKTHREAD _Thread = KeGetCurrentThread(); \
42 \
43 /* Sanity checks */ \
44 ASSERT(KeGetCurrentIrql() <= APC_LEVEL); \
45 ASSERT(_Thread == KeGetCurrentThread()); \
46 ASSERT(_Thread->SpecialApcDisable < 0); \
47 \
48 /* Leave region and check if APCs are OK now */ \
49 if (!(++_Thread->SpecialApcDisable)) \
50 { \
51 /* Check for Kernel APCs on the list */ \
52 if (!IsListEmpty(&_Thread->ApcState. \
53 ApcListHead[KernelMode])) \
54 { \
55 /* Check for APC Delivery */ \
56 KiCheckForKernelApcDelivery(); \
57 } \
58 } \
59 }
60
61 //
62 // Enters a Critical Region
63 //
64 #define KeEnterCriticalRegion() \
65 { \
66 PKTHREAD _Thread = KeGetCurrentThread(); \
67 \
68 /* Sanity checks */ \
69 ASSERT(_Thread == KeGetCurrentThread()); \
70 ASSERT((_Thread->KernelApcDisable <= 0) && \
71 (_Thread->KernelApcDisable != -32768)); \
72 \
73 /* Disable Kernel APCs */ \
74 _Thread->KernelApcDisable--; \
75 }
76
77 //
78 // Leaves a Critical Region
79 //
80 #define KeLeaveCriticalRegion() \
81 { \
82 PKTHREAD _Thread = KeGetCurrentThread(); \
83 \
84 /* Sanity checks */ \
85 ASSERT(_Thread == KeGetCurrentThread()); \
86 ASSERT(_Thread->KernelApcDisable < 0); \
87 \
88 /* Enable Kernel APCs */ \
89 _Thread->KernelApcDisable++; \
90 \
91 /* Check if Kernel APCs are now enabled */ \
92 if (!(_Thread->KernelApcDisable)) \
93 { \
94 /* Check if we need to request an APC Delivery */ \
95 if (!(IsListEmpty(&_Thread->ApcState.ApcListHead[KernelMode])) && \
96 !(_Thread->SpecialApcDisable)) \
97 { \
98 /* Check for the right environment */ \
99 KiCheckForKernelApcDelivery(); \
100 } \
101 } \
102 }
103
104 #ifndef CONFIG_SMP
105
106 //
107 // This routine protects against multiple CPU acquires, it's meaningless on UP.
108 //
109 FORCEINLINE
110 VOID
111 KiAcquireDispatcherObject(IN DISPATCHER_HEADER* Object)
112 {
113 UNREFERENCED_PARAMETER(Object);
114 }
115
116 //
117 // This routine protects against multiple CPU acquires, it's meaningless on UP.
118 //
119 FORCEINLINE
120 VOID
121 KiReleaseDispatcherObject(IN DISPATCHER_HEADER* Object)
122 {
123 UNREFERENCED_PARAMETER(Object);
124 }
125
126 FORCEINLINE
127 KIRQL
128 KiAcquireDispatcherLock(VOID)
129 {
130 /* Raise to DPC level */
131 return KeRaiseIrqlToDpcLevel();
132 }
133
134 FORCEINLINE
135 VOID
136 KiReleaseDispatcherLock(IN KIRQL OldIrql)
137 {
138 /* Just exit the dispatcher */
139 KiExitDispatcher(OldIrql);
140 }
141
142 FORCEINLINE
143 VOID
144 KiAcquireDispatcherLockAtDpcLevel(VOID)
145 {
146 /* This is a no-op at DPC Level for UP systems */
147 return;
148 }
149
150 FORCEINLINE
151 VOID
152 KiReleaseDispatcherLockFromDpcLevel(VOID)
153 {
154 /* This is a no-op at DPC Level for UP systems */
155 return;
156 }
157
158 //
159 // This routine makes the thread deferred ready on the boot CPU.
160 //
161 FORCEINLINE
162 VOID
163 KiInsertDeferredReadyList(IN PKTHREAD Thread)
164 {
165 /* Set the thread to deferred state and boot CPU */
166 Thread->State = DeferredReady;
167 Thread->DeferredProcessor = 0;
168
169 /* Make the thread ready immediately */
170 KiDeferredReadyThread(Thread);
171 }
172
173 FORCEINLINE
174 VOID
175 KiRescheduleThread(IN BOOLEAN NewThread,
176 IN ULONG Cpu)
177 {
178 /* This is meaningless on UP systems */
179 UNREFERENCED_PARAMETER(NewThread);
180 UNREFERENCED_PARAMETER(Cpu);
181 }
182
183 //
184 // This routine protects against multiple CPU acquires, it's meaningless on UP.
185 //
186 FORCEINLINE
187 VOID
188 KiSetThreadSwapBusy(IN PKTHREAD Thread)
189 {
190 UNREFERENCED_PARAMETER(Thread);
191 }
192
193 //
194 // This routine protects against multiple CPU acquires, it's meaningless on UP.
195 //
196 FORCEINLINE
197 VOID
198 KiAcquirePrcbLock(IN PKPRCB Prcb)
199 {
200 UNREFERENCED_PARAMETER(Prcb);
201 }
202
203 //
204 // This routine protects against multiple CPU acquires, it's meaningless on UP.
205 //
206 FORCEINLINE
207 VOID
208 KiReleasePrcbLock(IN PKPRCB Prcb)
209 {
210 UNREFERENCED_PARAMETER(Prcb);
211 }
212
213 //
214 // This routine protects against multiple CPU acquires, it's meaningless on UP.
215 //
216 FORCEINLINE
217 VOID
218 KiAcquireThreadLock(IN PKTHREAD Thread)
219 {
220 UNREFERENCED_PARAMETER(Thread);
221 }
222
223 //
224 // This routine protects against multiple CPU acquires, it's meaningless on UP.
225 //
226 FORCEINLINE
227 VOID
228 KiReleaseThreadLock(IN PKTHREAD Thread)
229 {
230 UNREFERENCED_PARAMETER(Thread);
231 }
232
233 //
234 // This routine protects against multiple CPU acquires, it's meaningless on UP.
235 //
236 FORCEINLINE
237 BOOLEAN
238 KiTryThreadLock(IN PKTHREAD Thread)
239 {
240 UNREFERENCED_PARAMETER(Thread);
241 return FALSE;
242 }
243
244 FORCEINLINE
245 VOID
246 KiCheckDeferredReadyList(IN PKPRCB Prcb)
247 {
248 /* There are no deferred ready lists on UP systems */
249 UNREFERENCED_PARAMETER(Prcb);
250 }
251
252 FORCEINLINE
253 VOID
254 KiRequestApcInterrupt(IN BOOLEAN NeedApc,
255 IN UCHAR Processor)
256 {
257 /* We deliver instantly on UP */
258 UNREFERENCED_PARAMETER(NeedApc);
259 UNREFERENCED_PARAMETER(Processor);
260 }
261
262 FORCEINLINE
263 PKSPIN_LOCK_QUEUE
264 KiAcquireTimerLock(IN ULONG Hand)
265 {
266 ASSERT(KeGetCurrentIrql() >= DISPATCH_LEVEL);
267
268 /* Nothing to do on UP */
269 UNREFERENCED_PARAMETER(Hand);
270 return NULL;
271 }
272
273 FORCEINLINE
274 VOID
275 KiReleaseTimerLock(IN PKSPIN_LOCK_QUEUE LockQueue)
276 {
277 ASSERT(KeGetCurrentIrql() >= DISPATCH_LEVEL);
278
279 /* Nothing to do on UP */
280 UNREFERENCED_PARAMETER(LockQueue);
281 }
282
283 #else
284
285 FORCEINLINE
286 VOID
287 KiAcquireDispatcherObject(IN DISPATCHER_HEADER* Object)
288 {
289 LONG OldValue;
290
291 /* Make sure we're at a safe level to touch the lock */
292 ASSERT(KeGetCurrentIrql() >= DISPATCH_LEVEL);
293
294 /* Start acquire loop */
295 do
296 {
297 /* Loop until the other CPU releases it */
298 while (TRUE)
299 {
300 /* Check if it got released */
301 OldValue = Object->Lock;
302 if ((OldValue & KOBJECT_LOCK_BIT) == 0) break;
303
304 /* Let the CPU know that this is a loop */
305 YieldProcessor();
306 }
307
308 /* Try acquiring the lock now */
309 } while (InterlockedCompareExchange(&Object->Lock,
310 OldValue | KOBJECT_LOCK_BIT,
311 OldValue) != OldValue);
312 }
313
314 FORCEINLINE
315 VOID
316 KiReleaseDispatcherObject(IN DISPATCHER_HEADER* Object)
317 {
318 /* Make sure we're at a safe level to touch the lock */
319 ASSERT(KeGetCurrentIrql() >= DISPATCH_LEVEL);
320
321 /* Release it */
322 InterlockedAnd(&Object->Lock, ~KOBJECT_LOCK_BIT);
323 }
324
325 FORCEINLINE
326 KIRQL
327 KiAcquireDispatcherLock(VOID)
328 {
329 /* Raise to synchronization level and acquire the dispatcher lock */
330 return KeAcquireQueuedSpinLockRaiseToSynch(LockQueueDispatcherLock);
331 }
332
333 FORCEINLINE
334 VOID
335 KiReleaseDispatcherLock(IN KIRQL OldIrql)
336 {
337 /* First release the lock */
338 KeReleaseQueuedSpinLockFromDpcLevel(&KeGetCurrentPrcb()->
339 LockQueue[LockQueueDispatcherLock]);
340
341 /* Then exit the dispatcher */
342 KiExitDispatcher(OldIrql);
343 }
344
345 FORCEINLINE
346 VOID
347 KiAcquireDispatcherLockAtDpcLevel(VOID)
348 {
349 /* Acquire the dispatcher lock */
350 KeAcquireQueuedSpinLockAtDpcLevel(&KeGetCurrentPrcb()->
351 LockQueue[LockQueueDispatcherLock]);
352 }
353
354 FORCEINLINE
355 VOID
356 KiReleaseDispatcherLockFromDpcLevel(VOID)
357 {
358 /* Release the dispatcher lock */
359 KeReleaseQueuedSpinLockFromDpcLevel(&KeGetCurrentPrcb()->
360 LockQueue[LockQueueDispatcherLock]);
361 }
362
363 //
364 // This routine inserts a thread into the deferred ready list of the current CPU
365 //
366 FORCEINLINE
367 VOID
368 KiInsertDeferredReadyList(IN PKTHREAD Thread)
369 {
370 PKPRCB Prcb = KeGetCurrentPrcb();
371
372 /* Set the thread to deferred state and CPU */
373 Thread->State = DeferredReady;
374 Thread->DeferredProcessor = Prcb->Number;
375
376 /* Add it on the list */
377 PushEntryList(&Prcb->DeferredReadyListHead, &Thread->SwapListEntry);
378 }
379
380 FORCEINLINE
381 VOID
382 KiRescheduleThread(IN BOOLEAN NewThread,
383 IN ULONG Cpu)
384 {
385 /* Check if a new thread needs to be scheduled on a different CPU */
386 if ((NewThread) && !(KeGetPcr()->Number == Cpu))
387 {
388 /* Send an IPI to request delivery */
389 KiIpiSend(AFFINITY_MASK(Cpu), IPI_DPC);
390 }
391 }
392
393 //
394 // This routine sets the current thread in a swap busy state, which ensure that
395 // nobody else tries to swap it concurrently.
396 //
397 FORCEINLINE
398 VOID
399 KiSetThreadSwapBusy(IN PKTHREAD Thread)
400 {
401 /* Make sure nobody already set it */
402 ASSERT(Thread->SwapBusy == FALSE);
403
404 /* Set it ourselves */
405 Thread->SwapBusy = TRUE;
406 }
407
408 //
409 // This routine acquires the PRCB lock so that only one caller can touch
410 // volatile PRCB data.
411 //
412 // Since this is a simple optimized spin-lock, it must only be acquired
413 // at dispatcher level or higher!
414 //
415 FORCEINLINE
416 VOID
417 KiAcquirePrcbLock(IN PKPRCB Prcb)
418 {
419 /* Make sure we're at a safe level to touch the PRCB lock */
420 ASSERT(KeGetCurrentIrql() >= DISPATCH_LEVEL);
421
422 /* Start acquire loop */
423 for (;;)
424 {
425 /* Acquire the lock and break out if we acquired it first */
426 if (!InterlockedExchange((PLONG)&Prcb->PrcbLock, 1)) break;
427
428 /* Loop until the other CPU releases it */
429 do
430 {
431 /* Let the CPU know that this is a loop */
432 YieldProcessor();
433 } while (Prcb->PrcbLock);
434 }
435 }
436
437 //
438 // This routine releases the PRCB lock so that other callers can touch
439 // volatile PRCB data.
440 //
441 // Since this is a simple optimized spin-lock, it must be be only acquired
442 // at dispatcher level or higher!
443 //
444 FORCEINLINE
445 VOID
446 KiReleasePrcbLock(IN PKPRCB Prcb)
447 {
448 /* Make sure we are above dispatch and the lock is acquired! */
449 ASSERT(KeGetCurrentIrql() >= DISPATCH_LEVEL);
450 ASSERT(Prcb->PrcbLock != 0);
451
452 /* Release it */
453 InterlockedAnd((PLONG)&Prcb->PrcbLock, 0);
454 }
455
456 //
457 // This routine acquires the thread lock so that only one caller can touch
458 // volatile thread data.
459 //
460 // Since this is a simple optimized spin-lock, it must be be only acquired
461 // at dispatcher level or higher!
462 //
463 FORCEINLINE
464 VOID
465 KiAcquireThreadLock(IN PKTHREAD Thread)
466 {
467 /* Make sure we're at a safe level to touch the thread lock */
468 ASSERT(KeGetCurrentIrql() >= DISPATCH_LEVEL);
469
470 /* Start acquire loop */
471 for (;;)
472 {
473 /* Acquire the lock and break out if we acquired it first */
474 if (!InterlockedExchange((PLONG)&Thread->ThreadLock, 1)) break;
475
476 /* Loop until the other CPU releases it */
477 do
478 {
479 /* Let the CPU know that this is a loop */
480 YieldProcessor();
481 } while (Thread->ThreadLock);
482 }
483 }
484
485 //
486 // This routine releases the thread lock so that other callers can touch
487 // volatile thread data.
488 //
489 // Since this is a simple optimized spin-lock, it must be be only acquired
490 // at dispatcher level or higher!
491 //
492 FORCEINLINE
493 VOID
494 KiReleaseThreadLock(IN PKTHREAD Thread)
495 {
496 /* Make sure we are still above dispatch */
497 ASSERT(KeGetCurrentIrql() >= DISPATCH_LEVEL);
498
499 /* Release it */
500 InterlockedAnd((PLONG)&Thread->ThreadLock, 0);
501 }
502
503 FORCEINLINE
504 BOOLEAN
505 KiTryThreadLock(IN PKTHREAD Thread)
506 {
507 LONG Value;
508
509 /* If the lock isn't acquired, return false */
510 if (!Thread->ThreadLock) return FALSE;
511
512 /* Otherwise, try to acquire it and check the result */
513 Value = 1;
514 Value = InterlockedExchange((PLONG)&Thread->ThreadLock, Value);
515
516 /* Return the lock state */
517 return (Value == TRUE);
518 }
519
520 FORCEINLINE
521 VOID
522 KiCheckDeferredReadyList(IN PKPRCB Prcb)
523 {
524 /* Scan the deferred ready lists if required */
525 if (Prcb->DeferredReadyListHead.Next) KiProcessDeferredReadyList(Prcb);
526 }
527
528 FORCEINLINE
529 VOID
530 KiRequestApcInterrupt(IN BOOLEAN NeedApc,
531 IN UCHAR Processor)
532 {
533 /* Check if we need to request APC delivery */
534 if (NeedApc)
535 {
536 /* Check if it's on another CPU */
537 if (KeGetPcr()->Number != Processor)
538 {
539 /* Send an IPI to request delivery */
540 KiIpiSend(AFFINITY_MASK(Processor), IPI_APC);
541 }
542 else
543 {
544 /* Request a software interrupt */
545 HalRequestSoftwareInterrupt(APC_LEVEL);
546 }
547 }
548 }
549
550 FORCEINLINE
551 PKSPIN_LOCK_QUEUE
552 KiAcquireTimerLock(IN ULONG Hand)
553 {
554 PKSPIN_LOCK_QUEUE LockQueue;
555 ULONG LockIndex;
556 ASSERT(KeGetCurrentIrql() >= DISPATCH_LEVEL);
557
558 /* Get the lock index */
559 LockIndex = Hand >> LOCK_QUEUE_TIMER_LOCK_SHIFT;
560 LockIndex &= (LOCK_QUEUE_TIMER_TABLE_LOCKS - 1);
561
562 /* Now get the lock */
563 LockQueue = &KeGetCurrentPrcb()->LockQueue[LockQueueTimerTableLock + LockIndex];
564
565 /* Acquire it and return */
566 KeAcquireQueuedSpinLockAtDpcLevel(LockQueue);
567 return LockQueue;
568 }
569
570 FORCEINLINE
571 VOID
572 KiReleaseTimerLock(IN PKSPIN_LOCK_QUEUE LockQueue)
573 {
574 ASSERT(KeGetCurrentIrql() >= DISPATCH_LEVEL);
575
576 /* Release the lock */
577 KeReleaseQueuedSpinLockFromDpcLevel(LockQueue);
578 }
579
580 #endif
581
582 FORCEINLINE
583 VOID
584 KiAcquireApcLock(IN PKTHREAD Thread,
585 IN PKLOCK_QUEUE_HANDLE Handle)
586 {
587 /* Acquire the lock and raise to synchronization level */
588 KeAcquireInStackQueuedSpinLockRaiseToSynch(&Thread->ApcQueueLock, Handle);
589 }
590
591 FORCEINLINE
592 VOID
593 KiAcquireApcLockAtDpcLevel(IN PKTHREAD Thread,
594 IN PKLOCK_QUEUE_HANDLE Handle)
595 {
596 /* Acquire the lock */
597 KeAcquireInStackQueuedSpinLockAtDpcLevel(&Thread->ApcQueueLock, Handle);
598 }
599
600 FORCEINLINE
601 VOID
602 KiAcquireApcLockAtApcLevel(IN PKTHREAD Thread,
603 IN PKLOCK_QUEUE_HANDLE Handle)
604 {
605 /* Acquire the lock */
606 KeAcquireInStackQueuedSpinLock(&Thread->ApcQueueLock, Handle);
607 }
608
609 FORCEINLINE
610 VOID
611 KiReleaseApcLock(IN PKLOCK_QUEUE_HANDLE Handle)
612 {
613 /* Release the lock */
614 KeReleaseInStackQueuedSpinLock(Handle);
615 }
616
617 FORCEINLINE
618 VOID
619 KiReleaseApcLockFromDpcLevel(IN PKLOCK_QUEUE_HANDLE Handle)
620 {
621 /* Release the lock */
622 KeReleaseInStackQueuedSpinLockFromDpcLevel(Handle);
623 }
624
625 FORCEINLINE
626 VOID
627 KiAcquireProcessLock(IN PKPROCESS Process,
628 IN PKLOCK_QUEUE_HANDLE Handle)
629 {
630 /* Acquire the lock and raise to synchronization level */
631 KeAcquireInStackQueuedSpinLockRaiseToSynch(&Process->ProcessLock, Handle);
632 }
633
634 FORCEINLINE
635 VOID
636 KiReleaseProcessLock(IN PKLOCK_QUEUE_HANDLE Handle)
637 {
638 /* Release the lock */
639 KeReleaseInStackQueuedSpinLock(Handle);
640 }
641
642 FORCEINLINE
643 VOID
644 KiReleaseProcessLockFromDpcLevel(IN PKLOCK_QUEUE_HANDLE Handle)
645 {
646 /* Release the lock */
647 KeReleaseInStackQueuedSpinLockFromDpcLevel(Handle);
648 }
649
650 FORCEINLINE
651 VOID
652 KiAcquireDeviceQueueLock(IN PKDEVICE_QUEUE DeviceQueue,
653 IN PKLOCK_QUEUE_HANDLE DeviceLock)
654 {
655 /* Check if we were called from a threaded DPC */
656 if (KeGetCurrentPrcb()->DpcThreadActive)
657 {
658 /* Lock the Queue, we're not at DPC level */
659 KeAcquireInStackQueuedSpinLock(&DeviceQueue->Lock, DeviceLock);
660 }
661 else
662 {
663 /* We must be at DPC level, acquire the lock safely */
664 ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL);
665 KeAcquireInStackQueuedSpinLockAtDpcLevel(&DeviceQueue->Lock,
666 DeviceLock);
667 }
668 }
669
670 FORCEINLINE
671 VOID
672 KiReleaseDeviceQueueLock(IN PKLOCK_QUEUE_HANDLE DeviceLock)
673 {
674 /* Check if we were called from a threaded DPC */
675 if (KeGetCurrentPrcb()->DpcThreadActive)
676 {
677 /* Unlock the Queue, we're not at DPC level */
678 KeReleaseInStackQueuedSpinLock(DeviceLock);
679 }
680 else
681 {
682 /* We must be at DPC level, release the lock safely */
683 ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL);
684 KeReleaseInStackQueuedSpinLockFromDpcLevel(DeviceLock);
685 }
686 }
687
688 //
689 // Satisfies the wait of any dispatcher object
690 //
691 #define KiSatisfyObjectWait(Object, Thread) \
692 { \
693 /* Special case for Mutants */ \
694 if ((Object)->Header.Type == MutantObject) \
695 { \
696 /* Decrease the Signal State */ \
697 (Object)->Header.SignalState--; \
698 \
699 /* Check if it's now non-signaled */ \
700 if (!(Object)->Header.SignalState) \
701 { \
702 /* Set the Owner Thread */ \
703 (Object)->OwnerThread = Thread; \
704 \
705 /* Disable APCs if needed */ \
706 Thread->KernelApcDisable = Thread->KernelApcDisable - \
707 (Object)->ApcDisable; \
708 \
709 /* Check if it's abandoned */ \
710 if ((Object)->Abandoned) \
711 { \
712 /* Unabandon it */ \
713 (Object)->Abandoned = FALSE; \
714 \
715 /* Return Status */ \
716 Thread->WaitStatus = STATUS_ABANDONED; \
717 } \
718 \
719 /* Insert it into the Mutant List */ \
720 InsertHeadList(Thread->MutantListHead.Blink, \
721 &(Object)->MutantListEntry); \
722 } \
723 } \
724 else if (((Object)->Header.Type & TIMER_OR_EVENT_TYPE) == \
725 EventSynchronizationObject) \
726 { \
727 /* Synchronization Timers and Events just get un-signaled */ \
728 (Object)->Header.SignalState = 0; \
729 } \
730 else if ((Object)->Header.Type == SemaphoreObject) \
731 { \
732 /* These ones can have multiple states, so we only decrease it */ \
733 (Object)->Header.SignalState--; \
734 } \
735 }
736
737 //
738 // Satisfies the wait of a mutant dispatcher object
739 //
740 #define KiSatisfyMutantWait(Object, Thread) \
741 { \
742 /* Decrease the Signal State */ \
743 (Object)->Header.SignalState--; \
744 \
745 /* Check if it's now non-signaled */ \
746 if (!(Object)->Header.SignalState) \
747 { \
748 /* Set the Owner Thread */ \
749 (Object)->OwnerThread = Thread; \
750 \
751 /* Disable APCs if needed */ \
752 Thread->KernelApcDisable = Thread->KernelApcDisable - \
753 (Object)->ApcDisable; \
754 \
755 /* Check if it's abandoned */ \
756 if ((Object)->Abandoned) \
757 { \
758 /* Unabandon it */ \
759 (Object)->Abandoned = FALSE; \
760 \
761 /* Return Status */ \
762 Thread->WaitStatus = STATUS_ABANDONED; \
763 } \
764 \
765 /* Insert it into the Mutant List */ \
766 InsertHeadList(Thread->MutantListHead.Blink, \
767 &(Object)->MutantListEntry); \
768 } \
769 }
770
771 //
772 // Satisfies the wait of any nonmutant dispatcher object
773 //
774 #define KiSatisfyNonMutantWait(Object) \
775 { \
776 if (((Object)->Header.Type & TIMER_OR_EVENT_TYPE) == \
777 EventSynchronizationObject) \
778 { \
779 /* Synchronization Timers and Events just get un-signaled */ \
780 (Object)->Header.SignalState = 0; \
781 } \
782 else if ((Object)->Header.Type == SemaphoreObject) \
783 { \
784 /* These ones can have multiple states, so we only decrease it */ \
785 (Object)->Header.SignalState--; \
786 } \
787 }
788
789 //
790 // Recalculates the due time
791 //
792 FORCEINLINE
793 PLARGE_INTEGER
794 KiRecalculateDueTime(IN PLARGE_INTEGER OriginalDueTime,
795 IN PLARGE_INTEGER DueTime,
796 IN OUT PLARGE_INTEGER NewDueTime)
797 {
798 /* Don't do anything for absolute waits */
799 if (OriginalDueTime->QuadPart >= 0) return OriginalDueTime;
800
801 /* Otherwise, query the interrupt time and recalculate */
802 NewDueTime->QuadPart = KeQueryInterruptTime();
803 NewDueTime->QuadPart -= DueTime->QuadPart;
804 return NewDueTime;
805 }
806
807 //
808 // Determines whether a thread should be added to the wait list
809 //
810 FORCEINLINE
811 BOOLEAN
812 KiCheckThreadStackSwap(IN PKTHREAD Thread,
813 IN KPROCESSOR_MODE WaitMode)
814 {
815 /* Check the required conditions */
816 if ((WaitMode != KernelMode) &&
817 (Thread->EnableStackSwap) &&
818 (Thread->Priority >= (LOW_REALTIME_PRIORITY + 9)))
819 {
820 /* We are go for swap */
821 return TRUE;
822 }
823 else
824 {
825 /* Don't swap the thread */
826 return FALSE;
827 }
828 }
829
830 //
831 // Adds a thread to the wait list
832 //
833 #define KiAddThreadToWaitList(Thread, Swappable) \
834 { \
835 /* Make sure it's swappable */ \
836 if (Swappable) \
837 { \
838 /* Insert it into the PRCB's List */ \
839 InsertTailList(&KeGetCurrentPrcb()->WaitListHead, \
840 &Thread->WaitListEntry); \
841 } \
842 }
843
844 //
845 // Checks if a wait in progress should be interrupted by APCs or an alertable
846 // state.
847 //
848 FORCEINLINE
849 NTSTATUS
850 KiCheckAlertability(IN PKTHREAD Thread,
851 IN BOOLEAN Alertable,
852 IN KPROCESSOR_MODE WaitMode)
853 {
854 /* Check if the wait is alertable */
855 if (Alertable)
856 {
857 /* It is, first check if the thread is alerted in this mode */
858 if (Thread->Alerted[WaitMode])
859 {
860 /* It is, so bail out of the wait */
861 Thread->Alerted[WaitMode] = FALSE;
862 return STATUS_ALERTED;
863 }
864 else if ((WaitMode != KernelMode) &&
865 (!IsListEmpty(&Thread->ApcState.ApcListHead[UserMode])))
866 {
867 /* It's isn't, but this is a user wait with queued user APCs */
868 Thread->ApcState.UserApcPending = TRUE;
869 return STATUS_USER_APC;
870 }
871 else if (Thread->Alerted[KernelMode])
872 {
873 /* It isn't that either, but we're alered in kernel mode */
874 Thread->Alerted[KernelMode] = FALSE;
875 return STATUS_ALERTED;
876 }
877 }
878 else if ((WaitMode != KernelMode) && (Thread->ApcState.UserApcPending))
879 {
880 /* Not alertable, but this is a user wait with pending user APCs */
881 return STATUS_USER_APC;
882 }
883
884 /* Otherwise, we're fine */
885 return STATUS_WAIT_0;
886 }
887
888 ULONG
889 FORCEINLINE
890 KiComputeTimerTableIndex(IN ULONGLONG DueTime)
891 {
892 return (DueTime / KeMaximumIncrement) & (TIMER_TABLE_SIZE - 1);
893 }
894
895 //
896 // Called from KiCompleteTimer, KiInsertTreeTimer, KeSetSystemTime
897 // to remove timer entries
898 // See Windows HPI blog for more information.
899 FORCEINLINE
900 VOID
901 KiRemoveEntryTimer(IN PKTIMER Timer)
902 {
903 ULONG Hand;
904 PKTIMER_TABLE_ENTRY TableEntry;
905
906 /* Remove the timer from the timer list and check if it's empty */
907 Hand = Timer->Header.Hand;
908 if (RemoveEntryList(&Timer->TimerListEntry))
909 {
910 /* Get the respective timer table entry */
911 TableEntry = &KiTimerTableListHead[Hand];
912 if (&TableEntry->Entry == TableEntry->Entry.Flink)
913 {
914 /* Set the entry to an infinite absolute time */
915 TableEntry->Time.HighPart = 0xFFFFFFFF;
916 }
917 }
918
919 /* Clear the list entries on dbg builds so we can tell the timer is gone */
920 #if DBG
921 Timer->TimerListEntry.Flink = NULL;
922 Timer->TimerListEntry.Blink = NULL;
923 #endif
924 }
925
926 //
927 // Called by Wait and Queue code to insert a timer for dispatching.
928 // Also called by KeSetTimerEx to insert a timer from the caller.
929 //
930 FORCEINLINE
931 VOID
932 KxInsertTimer(IN PKTIMER Timer,
933 IN ULONG Hand)
934 {
935 PKSPIN_LOCK_QUEUE LockQueue;
936
937 /* Acquire the lock and release the dispatcher lock */
938 LockQueue = KiAcquireTimerLock(Hand);
939 KiReleaseDispatcherLockFromDpcLevel();
940
941 /* Try to insert the timer */
942 if (KiInsertTimerTable(Timer, Hand))
943 {
944 /* Complete it */
945 KiCompleteTimer(Timer, LockQueue);
946 }
947 else
948 {
949 /* Do nothing, just release the lock */
950 KiReleaseTimerLock(LockQueue);
951 }
952 }
953
954 //
955 // Called by KeSetTimerEx and KiInsertTreeTimer to calculate Due Time
956 // See the Windows HPI Blog for more information
957 //
958 FORCEINLINE
959 BOOLEAN
960 KiComputeDueTime(IN PKTIMER Timer,
961 IN LARGE_INTEGER DueTime,
962 OUT PULONG Hand)
963 {
964 LARGE_INTEGER InterruptTime, SystemTime, DifferenceTime;
965
966 /* Convert to relative time if needed */
967 Timer->Header.Absolute = FALSE;
968 if (DueTime.HighPart >= 0)
969 {
970 /* Get System Time */
971 KeQuerySystemTime(&SystemTime);
972
973 /* Do the conversion */
974 DifferenceTime.QuadPart = SystemTime.QuadPart - DueTime.QuadPart;
975
976 /* Make sure it hasn't already expired */
977 Timer->Header.Absolute = TRUE;
978 if (DifferenceTime.HighPart >= 0)
979 {
980 /* Cancel everything */
981 Timer->Header.SignalState = TRUE;
982 Timer->Header.Hand = 0;
983 Timer->DueTime.QuadPart = 0;
984 *Hand = 0;
985 return FALSE;
986 }
987
988 /* Set the time as Absolute */
989 DueTime = DifferenceTime;
990 }
991
992 /* Get the Interrupt Time */
993 InterruptTime.QuadPart = KeQueryInterruptTime();
994
995 /* Recalculate due time */
996 Timer->DueTime.QuadPart = InterruptTime.QuadPart - DueTime.QuadPart;
997
998 /* Get the handle */
999 *Hand = KiComputeTimerTableIndex(Timer->DueTime.QuadPart);
1000 Timer->Header.Hand = (UCHAR)*Hand;
1001 Timer->Header.Inserted = TRUE;
1002 return TRUE;
1003 }
1004
1005 //
1006 // Called from Unlink and Queue Insert Code.
1007 // Also called by timer code when canceling an inserted timer.
1008 // Removes a timer from it's tree.
1009 //
1010 FORCEINLINE
1011 VOID
1012 KxRemoveTreeTimer(IN PKTIMER Timer)
1013 {
1014 ULONG Hand = Timer->Header.Hand;
1015 PKSPIN_LOCK_QUEUE LockQueue;
1016 PKTIMER_TABLE_ENTRY TimerEntry;
1017
1018 /* Acquire timer lock */
1019 LockQueue = KiAcquireTimerLock(Hand);
1020
1021 /* Set the timer as non-inserted */
1022 Timer->Header.Inserted = FALSE;
1023
1024 /* Remove it from the timer list */
1025 if (RemoveEntryList(&Timer->TimerListEntry))
1026 {
1027 /* Get the entry and check if it's empty */
1028 TimerEntry = &KiTimerTableListHead[Hand];
1029 if (IsListEmpty(&TimerEntry->Entry))
1030 {
1031 /* Clear the time then */
1032 TimerEntry->Time.HighPart = 0xFFFFFFFF;
1033 }
1034 }
1035
1036 /* Release the timer lock */
1037 KiReleaseTimerLock(LockQueue);
1038 }
1039
1040 FORCEINLINE
1041 VOID
1042 KxSetTimerForThreadWait(IN PKTIMER Timer,
1043 IN LARGE_INTEGER Interval,
1044 OUT PULONG Hand)
1045 {
1046 ULONGLONG DueTime;
1047 LARGE_INTEGER InterruptTime, SystemTime, TimeDifference;
1048
1049 /* Check the timer's interval to see if it's absolute */
1050 Timer->Header.Absolute = FALSE;
1051 if (Interval.HighPart >= 0)
1052 {
1053 /* Get the system time and calculate the relative time */
1054 KeQuerySystemTime(&SystemTime);
1055 TimeDifference.QuadPart = SystemTime.QuadPart - Interval.QuadPart;
1056 Timer->Header.Absolute = TRUE;
1057
1058 /* Check if we've already expired */
1059 if (TimeDifference.HighPart >= 0)
1060 {
1061 /* Reset everything */
1062 Timer->DueTime.QuadPart = 0;
1063 *Hand = 0;
1064 Timer->Header.Hand = 0;
1065 return;
1066 }
1067 else
1068 {
1069 /* Update the interval */
1070 Interval = TimeDifference;
1071 }
1072 }
1073
1074 /* Calculate the due time */
1075 InterruptTime.QuadPart = KeQueryInterruptTime();
1076 DueTime = InterruptTime.QuadPart - Interval.QuadPart;
1077 Timer->DueTime.QuadPart = DueTime;
1078
1079 /* Calculate the timer handle */
1080 *Hand = KiComputeTimerTableIndex(DueTime);
1081 Timer->Header.Hand = (UCHAR)*Hand;
1082 }
1083
1084 #define KxDelayThreadWait() \
1085 \
1086 /* Setup the Wait Block */ \
1087 Thread->WaitBlockList = TimerBlock; \
1088 \
1089 /* Setup the timer */ \
1090 KxSetTimerForThreadWait(Timer, *Interval, &Hand); \
1091 \
1092 /* Save the due time for the caller */ \
1093 DueTime.QuadPart = Timer->DueTime.QuadPart; \
1094 \
1095 /* Link the timer to this Wait Block */ \
1096 TimerBlock->NextWaitBlock = TimerBlock; \
1097 Timer->Header.WaitListHead.Flink = &TimerBlock->WaitListEntry; \
1098 Timer->Header.WaitListHead.Blink = &TimerBlock->WaitListEntry; \
1099 \
1100 /* Clear wait status */ \
1101 Thread->WaitStatus = STATUS_SUCCESS; \
1102 \
1103 /* Setup wait fields */ \
1104 Thread->Alertable = Alertable; \
1105 Thread->WaitReason = DelayExecution; \
1106 Thread->WaitMode = WaitMode; \
1107 \
1108 /* Check if we can swap the thread's stack */ \
1109 Thread->WaitListEntry.Flink = NULL; \
1110 Swappable = KiCheckThreadStackSwap(Thread, WaitMode); \
1111 \
1112 /* Set the wait time */ \
1113 Thread->WaitTime = KeTickCount.LowPart;
1114
1115 #define KxMultiThreadWait() \
1116 /* Link wait block array to the thread */ \
1117 Thread->WaitBlockList = WaitBlockArray; \
1118 \
1119 /* Reset the index */ \
1120 Index = 0; \
1121 \
1122 /* Loop wait blocks */ \
1123 do \
1124 { \
1125 /* Fill out the wait block */ \
1126 WaitBlock = &WaitBlockArray[Index]; \
1127 WaitBlock->Object = Object[Index]; \
1128 WaitBlock->WaitKey = (USHORT)Index; \
1129 WaitBlock->WaitType = WaitType; \
1130 WaitBlock->Thread = Thread; \
1131 \
1132 /* Link to next block */ \
1133 WaitBlock->NextWaitBlock = &WaitBlockArray[Index + 1]; \
1134 Index++; \
1135 } while (Index < Count); \
1136 \
1137 /* Link the last block */ \
1138 WaitBlock->NextWaitBlock = WaitBlockArray; \
1139 \
1140 /* Set default wait status */ \
1141 Thread->WaitStatus = STATUS_WAIT_0; \
1142 \
1143 /* Check if we have a timer */ \
1144 if (Timeout) \
1145 { \
1146 /* Link to the block */ \
1147 TimerBlock->NextWaitBlock = WaitBlockArray; \
1148 \
1149 /* Setup the timer */ \
1150 KxSetTimerForThreadWait(Timer, *Timeout, &Hand); \
1151 \
1152 /* Save the due time for the caller */ \
1153 DueTime.QuadPart = Timer->DueTime.QuadPart; \
1154 \
1155 /* Initialize the list */ \
1156 InitializeListHead(&Timer->Header.WaitListHead); \
1157 } \
1158 \
1159 /* Set wait settings */ \
1160 Thread->Alertable = Alertable; \
1161 Thread->WaitMode = WaitMode; \
1162 Thread->WaitReason = WaitReason; \
1163 \
1164 /* Check if we can swap the thread's stack */ \
1165 Thread->WaitListEntry.Flink = NULL; \
1166 Swappable = KiCheckThreadStackSwap(Thread, WaitMode); \
1167 \
1168 /* Set the wait time */ \
1169 Thread->WaitTime = KeTickCount.LowPart;
1170
1171 #define KxSingleThreadWait() \
1172 /* Setup the Wait Block */ \
1173 Thread->WaitBlockList = WaitBlock; \
1174 WaitBlock->WaitKey = STATUS_SUCCESS; \
1175 WaitBlock->Object = Object; \
1176 WaitBlock->WaitType = WaitAny; \
1177 \
1178 /* Clear wait status */ \
1179 Thread->WaitStatus = STATUS_SUCCESS; \
1180 \
1181 /* Check if we have a timer */ \
1182 if (Timeout) \
1183 { \
1184 /* Setup the timer */ \
1185 KxSetTimerForThreadWait(Timer, *Timeout, &Hand); \
1186 \
1187 /* Save the due time for the caller */ \
1188 DueTime.QuadPart = Timer->DueTime.QuadPart; \
1189 \
1190 /* Pointer to timer block */ \
1191 WaitBlock->NextWaitBlock = TimerBlock; \
1192 TimerBlock->NextWaitBlock = WaitBlock; \
1193 \
1194 /* Link the timer to this Wait Block */ \
1195 Timer->Header.WaitListHead.Flink = &TimerBlock->WaitListEntry; \
1196 Timer->Header.WaitListHead.Blink = &TimerBlock->WaitListEntry; \
1197 } \
1198 else \
1199 { \
1200 /* No timer block, just ourselves */ \
1201 WaitBlock->NextWaitBlock = WaitBlock; \
1202 } \
1203 \
1204 /* Set wait settings */ \
1205 Thread->Alertable = Alertable; \
1206 Thread->WaitMode = WaitMode; \
1207 Thread->WaitReason = WaitReason; \
1208 \
1209 /* Check if we can swap the thread's stack */ \
1210 Thread->WaitListEntry.Flink = NULL; \
1211 Swappable = KiCheckThreadStackSwap(Thread, WaitMode); \
1212 \
1213 /* Set the wait time */ \
1214 Thread->WaitTime = KeTickCount.LowPart;
1215
1216 #define KxQueueThreadWait() \
1217 /* Setup the Wait Block */ \
1218 Thread->WaitBlockList = WaitBlock; \
1219 WaitBlock->WaitKey = STATUS_SUCCESS; \
1220 WaitBlock->Object = Queue; \
1221 WaitBlock->WaitType = WaitAny; \
1222 WaitBlock->Thread = Thread; \
1223 \
1224 /* Clear wait status */ \
1225 Thread->WaitStatus = STATUS_SUCCESS; \
1226 \
1227 /* Check if we have a timer */ \
1228 if (Timeout) \
1229 { \
1230 /* Setup the timer */ \
1231 KxSetTimerForThreadWait(Timer, *Timeout, &Hand); \
1232 \
1233 /* Save the due time for the caller */ \
1234 DueTime.QuadPart = Timer->DueTime.QuadPart; \
1235 \
1236 /* Pointer to timer block */ \
1237 WaitBlock->NextWaitBlock = TimerBlock; \
1238 TimerBlock->NextWaitBlock = WaitBlock; \
1239 \
1240 /* Link the timer to this Wait Block */ \
1241 Timer->Header.WaitListHead.Flink = &TimerBlock->WaitListEntry; \
1242 Timer->Header.WaitListHead.Blink = &TimerBlock->WaitListEntry; \
1243 } \
1244 else \
1245 { \
1246 /* No timer block, just ourselves */ \
1247 WaitBlock->NextWaitBlock = WaitBlock; \
1248 } \
1249 \
1250 /* Set wait settings */ \
1251 Thread->Alertable = FALSE; \
1252 Thread->WaitMode = WaitMode; \
1253 Thread->WaitReason = WrQueue; \
1254 \
1255 /* Check if we can swap the thread's stack */ \
1256 Thread->WaitListEntry.Flink = NULL; \
1257 Swappable = KiCheckThreadStackSwap(Thread, WaitMode); \
1258 \
1259 /* Set the wait time */ \
1260 Thread->WaitTime = KeTickCount.LowPart;
1261
1262 //
1263 // Unwaits a Thread
1264 //
1265 FORCEINLINE
1266 VOID
1267 KxUnwaitThread(IN DISPATCHER_HEADER *Object,
1268 IN KPRIORITY Increment)
1269 {
1270 PLIST_ENTRY WaitEntry, WaitList;
1271 PKWAIT_BLOCK WaitBlock;
1272 PKTHREAD WaitThread;
1273 ULONG WaitKey;
1274
1275 /* Loop the Wait Entries */
1276 WaitList = &Object->WaitListHead;
1277 ASSERT(IsListEmpty(&Object->WaitListHead) == FALSE);
1278 WaitEntry = WaitList->Flink;
1279 do
1280 {
1281 /* Get the current wait block */
1282 WaitBlock = CONTAINING_RECORD(WaitEntry, KWAIT_BLOCK, WaitListEntry);
1283
1284 /* Get the waiting thread */
1285 WaitThread = WaitBlock->Thread;
1286
1287 /* Check the current Wait Mode */
1288 if (WaitBlock->WaitType == WaitAny)
1289 {
1290 /* Use the actual wait key */
1291 WaitKey = WaitBlock->WaitKey;
1292 }
1293 else
1294 {
1295 /* Otherwise, use STATUS_KERNEL_APC */
1296 WaitKey = STATUS_KERNEL_APC;
1297 }
1298
1299 /* Unwait the thread */
1300 KiUnwaitThread(WaitThread, WaitKey, Increment);
1301
1302 /* Next entry */
1303 WaitEntry = WaitList->Flink;
1304 } while (WaitEntry != WaitList);
1305 }
1306
1307 //
1308 // Unwaits a Thread waiting on an event
1309 //
1310 FORCEINLINE
1311 VOID
1312 KxUnwaitThreadForEvent(IN PKEVENT Event,
1313 IN KPRIORITY Increment)
1314 {
1315 PLIST_ENTRY WaitEntry, WaitList;
1316 PKWAIT_BLOCK WaitBlock;
1317 PKTHREAD WaitThread;
1318
1319 /* Loop the Wait Entries */
1320 WaitList = &Event->Header.WaitListHead;
1321 ASSERT(IsListEmpty(&Event->Header.WaitListHead) == FALSE);
1322 WaitEntry = WaitList->Flink;
1323 do
1324 {
1325 /* Get the current wait block */
1326 WaitBlock = CONTAINING_RECORD(WaitEntry, KWAIT_BLOCK, WaitListEntry);
1327
1328 /* Get the waiting thread */
1329 WaitThread = WaitBlock->Thread;
1330
1331 /* Check the current Wait Mode */
1332 if (WaitBlock->WaitType == WaitAny)
1333 {
1334 /* Un-signal it */
1335 Event->Header.SignalState = 0;
1336
1337 /* Un-signal the event and unwait the thread */
1338 KiUnwaitThread(WaitThread, WaitBlock->WaitKey, Increment);
1339 break;
1340 }
1341
1342 /* Unwait the thread with STATUS_KERNEL_APC */
1343 KiUnwaitThread(WaitThread, STATUS_KERNEL_APC, Increment);
1344
1345 /* Next entry */
1346 WaitEntry = WaitList->Flink;
1347 } while (WaitEntry != WaitList);
1348 }
1349
1350 //
1351 // This routine queues a thread that is ready on the PRCB's ready lists.
1352 // If this thread cannot currently run on this CPU, then the thread is
1353 // added to the deferred ready list instead.
1354 //
1355 // This routine must be entered with the PRCB lock held and it will exit
1356 // with the PRCB lock released!
1357 //
1358 FORCEINLINE
1359 VOID
1360 KxQueueReadyThread(IN PKTHREAD Thread,
1361 IN PKPRCB Prcb)
1362 {
1363 BOOLEAN Preempted;
1364 KPRIORITY Priority;
1365
1366 /* Sanity checks */
1367 ASSERT(Prcb == KeGetCurrentPrcb());
1368 ASSERT(Thread->State == Running);
1369 ASSERT(Thread->NextProcessor == Prcb->Number);
1370
1371 /* Check if this thread is allowed to run in this CPU */
1372 #ifdef CONFIG_SMP
1373 if ((Thread->Affinity) & (Prcb->SetMember))
1374 #else
1375 if (TRUE)
1376 #endif
1377 {
1378 /* Set thread ready for execution */
1379 Thread->State = Ready;
1380
1381 /* Save current priority and if someone had pre-empted it */
1382 Priority = Thread->Priority;
1383 Preempted = Thread->Preempted;
1384
1385 /* We're not pre-empting now, and set the wait time */
1386 Thread->Preempted = FALSE;
1387 Thread->WaitTime = KeTickCount.LowPart;
1388
1389 /* Sanity check */
1390 ASSERT((Priority >= 0) && (Priority <= HIGH_PRIORITY));
1391
1392 /* Insert this thread in the appropriate order */
1393 Preempted ? InsertHeadList(&Prcb->DispatcherReadyListHead[Priority],
1394 &Thread->WaitListEntry) :
1395 InsertTailList(&Prcb->DispatcherReadyListHead[Priority],
1396 &Thread->WaitListEntry);
1397
1398 /* Update the ready summary */
1399 Prcb->ReadySummary |= PRIORITY_MASK(Priority);
1400
1401 /* Sanity check */
1402 ASSERT(Priority == Thread->Priority);
1403
1404 /* Release the PRCB lock */
1405 KiReleasePrcbLock(Prcb);
1406 }
1407 else
1408 {
1409 /* Otherwise, prepare this thread to be deferred */
1410 Thread->State = DeferredReady;
1411 Thread->DeferredProcessor = Prcb->Number;
1412
1413 /* Release the lock and defer scheduling */
1414 KiReleasePrcbLock(Prcb);
1415 KiDeferredReadyThread(Thread);
1416 }
1417 }
1418
1419 //
1420 // This routine scans for an appropriate ready thread to select at the
1421 // given priority and for the given CPU.
1422 //
1423 FORCEINLINE
1424 PKTHREAD
1425 KiSelectReadyThread(IN KPRIORITY Priority,
1426 IN PKPRCB Prcb)
1427 {
1428 ULONG PrioritySet;
1429 LONG HighPriority;
1430 PLIST_ENTRY ListEntry;
1431 PKTHREAD Thread = NULL;
1432
1433 /* Save the current mask and get the priority set for the CPU */
1434 PrioritySet = Prcb->ReadySummary >> Priority;
1435 if (!PrioritySet) goto Quickie;
1436
1437 /* Get the highest priority possible */
1438 BitScanReverse((PULONG)&HighPriority, PrioritySet);
1439 ASSERT((PrioritySet & PRIORITY_MASK(HighPriority)) != 0);
1440 HighPriority += Priority;
1441
1442 /* Make sure the list isn't empty at the highest priority */
1443 ASSERT(IsListEmpty(&Prcb->DispatcherReadyListHead[HighPriority]) == FALSE);
1444
1445 /* Get the first thread on the list */
1446 ListEntry = Prcb->DispatcherReadyListHead[HighPriority].Flink;
1447 Thread = CONTAINING_RECORD(ListEntry, KTHREAD, WaitListEntry);
1448
1449 /* Make sure this thread is here for a reason */
1450 ASSERT(HighPriority == Thread->Priority);
1451 ASSERT(Thread->Affinity & AFFINITY_MASK(Prcb->Number));
1452 ASSERT(Thread->NextProcessor == Prcb->Number);
1453
1454 /* Remove it from the list */
1455 if (RemoveEntryList(&Thread->WaitListEntry))
1456 {
1457 /* The list is empty now, reset the ready summary */
1458 Prcb->ReadySummary ^= PRIORITY_MASK(HighPriority);
1459 }
1460
1461 /* Sanity check and return the thread */
1462 Quickie:
1463 ASSERT((Thread == NULL) ||
1464 (Thread->BasePriority == 0) ||
1465 (Thread->Priority != 0));
1466 return Thread;
1467 }
1468
1469 //
1470 // This routine computes the new priority for a thread. It is only valid for
1471 // threads with priorities in the dynamic priority range.
1472 //
1473 FORCEINLINE
1474 SCHAR
1475 KiComputeNewPriority(IN PKTHREAD Thread,
1476 IN SCHAR Adjustment)
1477 {
1478 SCHAR Priority;
1479
1480 /* Priority sanity checks */
1481 ASSERT((Thread->PriorityDecrement >= 0) &&
1482 (Thread->PriorityDecrement <= Thread->Priority));
1483 ASSERT((Thread->Priority < LOW_REALTIME_PRIORITY) ?
1484 TRUE : (Thread->PriorityDecrement == 0));
1485
1486 /* Get the current priority */
1487 Priority = Thread->Priority;
1488 if (Priority < LOW_REALTIME_PRIORITY)
1489 {
1490 /* Decrease priority by the priority decrement */
1491 Priority -= (Thread->PriorityDecrement + Adjustment);
1492
1493 /* Don't go out of bounds */
1494 if (Priority < Thread->BasePriority) Priority = Thread->BasePriority;
1495
1496 /* Reset the priority decrement */
1497 Thread->PriorityDecrement = 0;
1498 }
1499
1500 /* Sanity check */
1501 ASSERT((Thread->BasePriority == 0) || (Priority != 0));
1502
1503 /* Return the new priority */
1504 return Priority;
1505 }
1506
1507 //
1508 // Guarded Mutex Routines
1509 //
1510 FORCEINLINE
1511 VOID
1512 _KeInitializeGuardedMutex(OUT PKGUARDED_MUTEX GuardedMutex)
1513 {
1514 /* Setup the Initial Data */
1515 GuardedMutex->Count = GM_LOCK_BIT;
1516 GuardedMutex->Owner = NULL;
1517 GuardedMutex->Contention = 0;
1518
1519 /* Initialize the Wait Gate */
1520 KeInitializeGate(&GuardedMutex->Gate);
1521 }
1522
1523 FORCEINLINE
1524 VOID
1525 _KeAcquireGuardedMutexUnsafe(IN OUT PKGUARDED_MUTEX GuardedMutex)
1526 {
1527 PKTHREAD Thread = KeGetCurrentThread();
1528
1529 /* Sanity checks */
1530 ASSERT((KeGetCurrentIrql() == APC_LEVEL) ||
1531 (Thread->SpecialApcDisable < 0) ||
1532 (Thread->Teb == NULL) ||
1533 (Thread->Teb >= (PTEB)MM_SYSTEM_RANGE_START));
1534 ASSERT(GuardedMutex->Owner != Thread);
1535
1536 /* Remove the lock */
1537 if (!InterlockedBitTestAndReset(&GuardedMutex->Count, GM_LOCK_BIT_V))
1538 {
1539 /* The Guarded Mutex was already locked, enter contented case */
1540 KiAcquireGuardedMutex(GuardedMutex);
1541 }
1542
1543 /* Set the Owner */
1544 GuardedMutex->Owner = Thread;
1545 }
1546
1547 FORCEINLINE
1548 VOID
1549 _KeReleaseGuardedMutexUnsafe(IN OUT PKGUARDED_MUTEX GuardedMutex)
1550 {
1551 LONG OldValue, NewValue;
1552
1553 /* Sanity checks */
1554 ASSERT((KeGetCurrentIrql() == APC_LEVEL) ||
1555 (KeGetCurrentThread()->SpecialApcDisable < 0) ||
1556 (KeGetCurrentThread()->Teb == NULL) ||
1557 (KeGetCurrentThread()->Teb >= (PTEB)MM_SYSTEM_RANGE_START));
1558 ASSERT(GuardedMutex->Owner == KeGetCurrentThread());
1559
1560 /* Destroy the Owner */
1561 GuardedMutex->Owner = NULL;
1562
1563 /* Add the Lock Bit */
1564 OldValue = InterlockedExchangeAdd(&GuardedMutex->Count, GM_LOCK_BIT);
1565 ASSERT((OldValue & GM_LOCK_BIT) == 0);
1566
1567 /* Check if it was already locked, but not woken */
1568 if ((OldValue) && !(OldValue & GM_LOCK_WAITER_WOKEN))
1569 {
1570 /* Update the Oldvalue to what it should be now */
1571 OldValue += GM_LOCK_BIT;
1572
1573 /* The mutex will be woken, minus one waiter */
1574 NewValue = OldValue + GM_LOCK_WAITER_WOKEN -
1575 GM_LOCK_WAITER_INC;
1576
1577 /* Remove the Woken bit */
1578 if (InterlockedCompareExchange(&GuardedMutex->Count,
1579 NewValue,
1580 OldValue) == OldValue)
1581 {
1582 /* Signal the Gate */
1583 KeSignalGateBoostPriority(&GuardedMutex->Gate);
1584 }
1585 }
1586 }
1587
1588 FORCEINLINE
1589 VOID
1590 _KeAcquireGuardedMutex(IN PKGUARDED_MUTEX GuardedMutex)
1591 {
1592 PKTHREAD Thread = KeGetCurrentThread();
1593
1594 /* Sanity checks */
1595 ASSERT(KeGetCurrentIrql() <= APC_LEVEL);
1596 ASSERT(GuardedMutex->Owner != Thread);
1597
1598 /* Disable Special APCs */
1599 KeEnterGuardedRegion();
1600
1601 /* Remove the lock */
1602 if (!InterlockedBitTestAndReset(&GuardedMutex->Count, GM_LOCK_BIT_V))
1603 {
1604 /* The Guarded Mutex was already locked, enter contented case */
1605 KiAcquireGuardedMutex(GuardedMutex);
1606 }
1607
1608 /* Set the Owner and Special APC Disable state */
1609 GuardedMutex->Owner = Thread;
1610 GuardedMutex->SpecialApcDisable = Thread->SpecialApcDisable;
1611 }
1612
1613 FORCEINLINE
1614 VOID
1615 _KeReleaseGuardedMutex(IN OUT PKGUARDED_MUTEX GuardedMutex)
1616 {
1617 LONG OldValue, NewValue;
1618
1619 /* Sanity checks */
1620 ASSERT(KeGetCurrentIrql() <= APC_LEVEL);
1621 ASSERT(GuardedMutex->Owner == KeGetCurrentThread());
1622 ASSERT(KeGetCurrentThread()->SpecialApcDisable ==
1623 GuardedMutex->SpecialApcDisable);
1624
1625 /* Destroy the Owner */
1626 GuardedMutex->Owner = NULL;
1627
1628 /* Add the Lock Bit */
1629 OldValue = InterlockedExchangeAdd(&GuardedMutex->Count, GM_LOCK_BIT);
1630 ASSERT((OldValue & GM_LOCK_BIT) == 0);
1631
1632 /* Check if it was already locked, but not woken */
1633 if ((OldValue) && !(OldValue & GM_LOCK_WAITER_WOKEN))
1634 {
1635 /* Update the Oldvalue to what it should be now */
1636 OldValue += GM_LOCK_BIT;
1637
1638 /* The mutex will be woken, minus one waiter */
1639 NewValue = OldValue + GM_LOCK_WAITER_WOKEN -
1640 GM_LOCK_WAITER_INC;
1641
1642 /* Remove the Woken bit */
1643 if (InterlockedCompareExchange(&GuardedMutex->Count,
1644 NewValue,
1645 OldValue) == OldValue)
1646 {
1647 /* Signal the Gate */
1648 KeSignalGateBoostPriority(&GuardedMutex->Gate);
1649 }
1650 }
1651
1652 /* Re-enable APCs */
1653 KeLeaveGuardedRegion();
1654 }
1655
1656 FORCEINLINE
1657 BOOLEAN
1658 _KeTryToAcquireGuardedMutex(IN OUT PKGUARDED_MUTEX GuardedMutex)
1659 {
1660 PKTHREAD Thread = KeGetCurrentThread();
1661
1662 /* Block APCs */
1663 KeEnterGuardedRegion();
1664
1665 /* Remove the lock */
1666 if (!InterlockedBitTestAndReset(&GuardedMutex->Count, GM_LOCK_BIT_V))
1667 {
1668 /* Re-enable APCs */
1669 KeLeaveGuardedRegion();
1670 YieldProcessor();
1671
1672 /* Return failure */
1673 return FALSE;
1674 }
1675
1676 /* Set the Owner and APC State */
1677 GuardedMutex->Owner = Thread;
1678 GuardedMutex->SpecialApcDisable = Thread->SpecialApcDisable;
1679 return TRUE;
1680 }
1681
1682
1683 FORCEINLINE
1684 VOID
1685 KiAcquireNmiListLock(OUT PKIRQL OldIrql)
1686 {
1687 KeAcquireSpinLock(&KiNmiCallbackListLock, OldIrql);
1688 }
1689
1690 FORCEINLINE
1691 VOID
1692 KiReleaseNmiListLock(IN KIRQL OldIrql)
1693 {
1694 KeReleaseSpinLock(&KiNmiCallbackListLock, OldIrql);
1695 }