[USETUP] Remove the WIN32 support because usetup is a native-only application
[reactos.git] / ntoskrnl / include / internal / ke_x.h
1 /*
2 * PROJECT: ReactOS Kernel
3 * LICENSE: GPL - See COPYING in the top level directory
4 * FILE: ntoskrnl/include/internal/ke_x.h
5 * PURPOSE: Internal Inlined Functions for the Kernel
6 * PROGRAMMERS: Alex Ionescu (alex.ionescu@reactos.org)
7 */
8
9 #ifndef _M_ARM
10 FORCEINLINE
11 KPROCESSOR_MODE
12 KeGetPreviousMode(VOID)
13 {
14 /* Return the current mode */
15 return KeGetCurrentThread()->PreviousMode;
16 }
17 #endif
18
19 //
20 // Enters a Guarded Region
21 //
22 #define KeEnterGuardedRegionThread(_Thread) \
23 { \
24 /* Sanity checks */ \
25 ASSERT(KeGetCurrentIrql() <= APC_LEVEL); \
26 ASSERT(_Thread == KeGetCurrentThread()); \
27 ASSERT((_Thread->SpecialApcDisable <= 0) && \
28 (_Thread->SpecialApcDisable != -32768)); \
29 \
30 /* Disable Special APCs */ \
31 _Thread->SpecialApcDisable--; \
32 }
33
34 #define KeEnterGuardedRegion() \
35 { \
36 PKTHREAD _Thread = KeGetCurrentThread(); \
37 KeEnterGuardedRegionThread(_Thread); \
38 }
39
40 //
41 // Leaves a Guarded Region
42 //
43 #define KeLeaveGuardedRegionThread(_Thread) \
44 { \
45 /* Sanity checks */ \
46 ASSERT(KeGetCurrentIrql() <= APC_LEVEL); \
47 ASSERT(_Thread == KeGetCurrentThread()); \
48 ASSERT(_Thread->SpecialApcDisable < 0); \
49 \
50 /* Leave region and check if APCs are OK now */ \
51 if (!(++_Thread->SpecialApcDisable)) \
52 { \
53 /* Check for Kernel APCs on the list */ \
54 if (!IsListEmpty(&_Thread->ApcState. \
55 ApcListHead[KernelMode])) \
56 { \
57 /* Check for APC Delivery */ \
58 KiCheckForKernelApcDelivery(); \
59 } \
60 } \
61 }
62
63 #define KeLeaveGuardedRegion() \
64 { \
65 PKTHREAD _Thread = KeGetCurrentThread(); \
66 KeLeaveGuardedRegionThread(_Thread); \
67 }
68
69 //
70 // Enters a Critical Region
71 //
72 #define KeEnterCriticalRegionThread(_Thread) \
73 { \
74 /* Sanity checks */ \
75 ASSERT(_Thread == KeGetCurrentThread()); \
76 ASSERT((_Thread->KernelApcDisable <= 0) && \
77 (_Thread->KernelApcDisable != -32768)); \
78 \
79 /* Disable Kernel APCs */ \
80 _Thread->KernelApcDisable--; \
81 }
82
83 #define KeEnterCriticalRegion() \
84 { \
85 PKTHREAD _Thread = KeGetCurrentThread(); \
86 KeEnterCriticalRegionThread(_Thread); \
87 }
88
89 //
90 // Leaves a Critical Region
91 //
92 #define KeLeaveCriticalRegionThread(_Thread) \
93 { \
94 /* Sanity checks */ \
95 ASSERT(_Thread == KeGetCurrentThread()); \
96 ASSERT(_Thread->KernelApcDisable < 0); \
97 \
98 /* Enable Kernel APCs */ \
99 _Thread->KernelApcDisable++; \
100 \
101 /* Check if Kernel APCs are now enabled */ \
102 if (!(_Thread->KernelApcDisable)) \
103 { \
104 /* Check if we need to request an APC Delivery */ \
105 if (!(IsListEmpty(&_Thread->ApcState.ApcListHead[KernelMode])) && \
106 !(_Thread->SpecialApcDisable)) \
107 { \
108 /* Check for the right environment */ \
109 KiCheckForKernelApcDelivery(); \
110 } \
111 } \
112 }
113
114 #define KeLeaveCriticalRegion() \
115 { \
116 PKTHREAD _Thread = KeGetCurrentThread(); \
117 KeLeaveCriticalRegionThread(_Thread); \
118 }
119
120 #ifndef CONFIG_SMP
121
122 //
123 // This routine protects against multiple CPU acquires, it's meaningless on UP.
124 //
125 FORCEINLINE
126 VOID
127 KiAcquireDispatcherObject(IN DISPATCHER_HEADER* Object)
128 {
129 UNREFERENCED_PARAMETER(Object);
130 }
131
132 //
133 // This routine protects against multiple CPU acquires, it's meaningless on UP.
134 //
135 FORCEINLINE
136 VOID
137 KiReleaseDispatcherObject(IN DISPATCHER_HEADER* Object)
138 {
139 UNREFERENCED_PARAMETER(Object);
140 }
141
142 FORCEINLINE
143 KIRQL
144 KiAcquireDispatcherLock(VOID)
145 {
146 /* Raise to synch level */
147 return KfRaiseIrql(SYNCH_LEVEL);
148 }
149
150 FORCEINLINE
151 VOID
152 KiReleaseDispatcherLock(IN KIRQL OldIrql)
153 {
154 /* Just exit the dispatcher */
155 KiExitDispatcher(OldIrql);
156 }
157
158 FORCEINLINE
159 VOID
160 KiAcquireDispatcherLockAtDpcLevel(VOID)
161 {
162 /* This is a no-op at DPC Level for UP systems */
163 return;
164 }
165
166 FORCEINLINE
167 VOID
168 KiReleaseDispatcherLockFromDpcLevel(VOID)
169 {
170 /* This is a no-op at DPC Level for UP systems */
171 return;
172 }
173
174 //
175 // This routine makes the thread deferred ready on the boot CPU.
176 //
177 FORCEINLINE
178 VOID
179 KiInsertDeferredReadyList(IN PKTHREAD Thread)
180 {
181 /* Set the thread to deferred state and boot CPU */
182 Thread->State = DeferredReady;
183 Thread->DeferredProcessor = 0;
184
185 /* Make the thread ready immediately */
186 KiDeferredReadyThread(Thread);
187 }
188
189 FORCEINLINE
190 VOID
191 KiRescheduleThread(IN BOOLEAN NewThread,
192 IN ULONG Cpu)
193 {
194 /* This is meaningless on UP systems */
195 UNREFERENCED_PARAMETER(NewThread);
196 UNREFERENCED_PARAMETER(Cpu);
197 }
198
199 //
200 // This routine protects against multiple CPU acquires, it's meaningless on UP.
201 //
202 FORCEINLINE
203 VOID
204 KiSetThreadSwapBusy(IN PKTHREAD Thread)
205 {
206 UNREFERENCED_PARAMETER(Thread);
207 }
208
209 //
210 // This routine protects against multiple CPU acquires, it's meaningless on UP.
211 //
212 FORCEINLINE
213 VOID
214 KiAcquirePrcbLock(IN PKPRCB Prcb)
215 {
216 UNREFERENCED_PARAMETER(Prcb);
217 }
218
219 //
220 // This routine protects against multiple CPU acquires, it's meaningless on UP.
221 //
222 FORCEINLINE
223 VOID
224 KiReleasePrcbLock(IN PKPRCB Prcb)
225 {
226 UNREFERENCED_PARAMETER(Prcb);
227 }
228
229 //
230 // This routine protects against multiple CPU acquires, it's meaningless on UP.
231 //
232 FORCEINLINE
233 VOID
234 KiAcquireThreadLock(IN PKTHREAD Thread)
235 {
236 UNREFERENCED_PARAMETER(Thread);
237 }
238
239 //
240 // This routine protects against multiple CPU acquires, it's meaningless on UP.
241 //
242 FORCEINLINE
243 VOID
244 KiReleaseThreadLock(IN PKTHREAD Thread)
245 {
246 UNREFERENCED_PARAMETER(Thread);
247 }
248
249 //
250 // This routine protects against multiple CPU acquires, it's meaningless on UP.
251 //
252 FORCEINLINE
253 BOOLEAN
254 KiTryThreadLock(IN PKTHREAD Thread)
255 {
256 UNREFERENCED_PARAMETER(Thread);
257 return FALSE;
258 }
259
260 FORCEINLINE
261 VOID
262 KiCheckDeferredReadyList(IN PKPRCB Prcb)
263 {
264 /* There are no deferred ready lists on UP systems */
265 UNREFERENCED_PARAMETER(Prcb);
266 }
267
268 FORCEINLINE
269 VOID
270 KiRequestApcInterrupt(IN BOOLEAN NeedApc,
271 IN UCHAR Processor)
272 {
273 /* We deliver instantly on UP */
274 UNREFERENCED_PARAMETER(NeedApc);
275 UNREFERENCED_PARAMETER(Processor);
276 }
277
278 FORCEINLINE
279 PKSPIN_LOCK_QUEUE
280 KiAcquireTimerLock(IN ULONG Hand)
281 {
282 ASSERT(KeGetCurrentIrql() >= DISPATCH_LEVEL);
283
284 /* Nothing to do on UP */
285 UNREFERENCED_PARAMETER(Hand);
286 return NULL;
287 }
288
289 FORCEINLINE
290 VOID
291 KiReleaseTimerLock(IN PKSPIN_LOCK_QUEUE LockQueue)
292 {
293 ASSERT(KeGetCurrentIrql() >= DISPATCH_LEVEL);
294
295 /* Nothing to do on UP */
296 UNREFERENCED_PARAMETER(LockQueue);
297 }
298
299 #else
300
301 FORCEINLINE
302 VOID
303 KiAcquireDispatcherObject(IN DISPATCHER_HEADER* Object)
304 {
305 LONG OldValue;
306
307 /* Make sure we're at a safe level to touch the lock */
308 ASSERT(KeGetCurrentIrql() >= DISPATCH_LEVEL);
309
310 /* Start acquire loop */
311 do
312 {
313 /* Loop until the other CPU releases it */
314 while (TRUE)
315 {
316 /* Check if it got released */
317 OldValue = Object->Lock;
318 if ((OldValue & KOBJECT_LOCK_BIT) == 0) break;
319
320 /* Let the CPU know that this is a loop */
321 YieldProcessor();
322 }
323
324 /* Try acquiring the lock now */
325 } while (InterlockedCompareExchange(&Object->Lock,
326 OldValue | KOBJECT_LOCK_BIT,
327 OldValue) != OldValue);
328 }
329
330 FORCEINLINE
331 VOID
332 KiReleaseDispatcherObject(IN DISPATCHER_HEADER* Object)
333 {
334 /* Make sure we're at a safe level to touch the lock */
335 ASSERT(KeGetCurrentIrql() >= DISPATCH_LEVEL);
336
337 /* Release it */
338 InterlockedAnd(&Object->Lock, ~KOBJECT_LOCK_BIT);
339 }
340
341 FORCEINLINE
342 KIRQL
343 KiAcquireDispatcherLock(VOID)
344 {
345 /* Raise to synchronization level and acquire the dispatcher lock */
346 return KeAcquireQueuedSpinLockRaiseToSynch(LockQueueDispatcherLock);
347 }
348
349 FORCEINLINE
350 VOID
351 KiReleaseDispatcherLock(IN KIRQL OldIrql)
352 {
353 /* First release the lock */
354 KeReleaseQueuedSpinLockFromDpcLevel(&KeGetCurrentPrcb()->
355 LockQueue[LockQueueDispatcherLock]);
356
357 /* Then exit the dispatcher */
358 KiExitDispatcher(OldIrql);
359 }
360
361 FORCEINLINE
362 VOID
363 KiAcquireDispatcherLockAtDpcLevel(VOID)
364 {
365 /* Acquire the dispatcher lock */
366 KeAcquireQueuedSpinLockAtDpcLevel(&KeGetCurrentPrcb()->
367 LockQueue[LockQueueDispatcherLock]);
368 }
369
370 FORCEINLINE
371 VOID
372 KiReleaseDispatcherLockFromDpcLevel(VOID)
373 {
374 /* Release the dispatcher lock */
375 KeReleaseQueuedSpinLockFromDpcLevel(&KeGetCurrentPrcb()->
376 LockQueue[LockQueueDispatcherLock]);
377 }
378
379 //
380 // This routine inserts a thread into the deferred ready list of the current CPU
381 //
382 FORCEINLINE
383 VOID
384 KiInsertDeferredReadyList(IN PKTHREAD Thread)
385 {
386 PKPRCB Prcb = KeGetCurrentPrcb();
387
388 /* Set the thread to deferred state and CPU */
389 Thread->State = DeferredReady;
390 Thread->DeferredProcessor = Prcb->Number;
391
392 /* Add it on the list */
393 PushEntryList(&Prcb->DeferredReadyListHead, &Thread->SwapListEntry);
394 }
395
396 FORCEINLINE
397 VOID
398 KiRescheduleThread(IN BOOLEAN NewThread,
399 IN ULONG Cpu)
400 {
401 /* Check if a new thread needs to be scheduled on a different CPU */
402 if ((NewThread) && !(KeGetCurrentPrcb()->Number == Cpu))
403 {
404 /* Send an IPI to request delivery */
405 KiIpiSend(AFFINITY_MASK(Cpu), IPI_DPC);
406 }
407 }
408
409 //
410 // This routine sets the current thread in a swap busy state, which ensure that
411 // nobody else tries to swap it concurrently.
412 //
413 FORCEINLINE
414 VOID
415 KiSetThreadSwapBusy(IN PKTHREAD Thread)
416 {
417 /* Make sure nobody already set it */
418 ASSERT(Thread->SwapBusy == FALSE);
419
420 /* Set it ourselves */
421 Thread->SwapBusy = TRUE;
422 }
423
424 //
425 // This routine acquires the PRCB lock so that only one caller can touch
426 // volatile PRCB data.
427 //
428 // Since this is a simple optimized spin-lock, it must only be acquired
429 // at dispatcher level or higher!
430 //
431 FORCEINLINE
432 VOID
433 KiAcquirePrcbLock(IN PKPRCB Prcb)
434 {
435 /* Make sure we're at a safe level to touch the PRCB lock */
436 ASSERT(KeGetCurrentIrql() >= DISPATCH_LEVEL);
437
438 /* Start acquire loop */
439 for (;;)
440 {
441 /* Acquire the lock and break out if we acquired it first */
442 if (!InterlockedExchange((PLONG)&Prcb->PrcbLock, 1)) break;
443
444 /* Loop until the other CPU releases it */
445 do
446 {
447 /* Let the CPU know that this is a loop */
448 YieldProcessor();
449 } while (Prcb->PrcbLock);
450 }
451 }
452
453 //
454 // This routine releases the PRCB lock so that other callers can touch
455 // volatile PRCB data.
456 //
457 // Since this is a simple optimized spin-lock, it must be be only acquired
458 // at dispatcher level or higher!
459 //
460 FORCEINLINE
461 VOID
462 KiReleasePrcbLock(IN PKPRCB Prcb)
463 {
464 /* Make sure we are above dispatch and the lock is acquired! */
465 ASSERT(KeGetCurrentIrql() >= DISPATCH_LEVEL);
466 ASSERT(Prcb->PrcbLock != 0);
467
468 /* Release it */
469 InterlockedAnd((PLONG)&Prcb->PrcbLock, 0);
470 }
471
472 //
473 // This routine acquires the thread lock so that only one caller can touch
474 // volatile thread data.
475 //
476 // Since this is a simple optimized spin-lock, it must be be only acquired
477 // at dispatcher level or higher!
478 //
479 FORCEINLINE
480 VOID
481 KiAcquireThreadLock(IN PKTHREAD Thread)
482 {
483 /* Make sure we're at a safe level to touch the thread lock */
484 ASSERT(KeGetCurrentIrql() >= DISPATCH_LEVEL);
485
486 /* Start acquire loop */
487 for (;;)
488 {
489 /* Acquire the lock and break out if we acquired it first */
490 if (!InterlockedExchange((PLONG)&Thread->ThreadLock, 1)) break;
491
492 /* Loop until the other CPU releases it */
493 do
494 {
495 /* Let the CPU know that this is a loop */
496 YieldProcessor();
497 } while (Thread->ThreadLock);
498 }
499 }
500
501 //
502 // This routine releases the thread lock so that other callers can touch
503 // volatile thread data.
504 //
505 // Since this is a simple optimized spin-lock, it must be be only acquired
506 // at dispatcher level or higher!
507 //
508 FORCEINLINE
509 VOID
510 KiReleaseThreadLock(IN PKTHREAD Thread)
511 {
512 /* Make sure we are still above dispatch */
513 ASSERT(KeGetCurrentIrql() >= DISPATCH_LEVEL);
514
515 /* Release it */
516 InterlockedAnd((PLONG)&Thread->ThreadLock, 0);
517 }
518
519 FORCEINLINE
520 BOOLEAN
521 KiTryThreadLock(IN PKTHREAD Thread)
522 {
523 LONG Value;
524
525 /* If the lock isn't acquired, return false */
526 if (!Thread->ThreadLock) return FALSE;
527
528 /* Otherwise, try to acquire it and check the result */
529 Value = 1;
530 Value = InterlockedExchange((PLONG)&Thread->ThreadLock, Value);
531
532 /* Return the lock state */
533 return (Value == 1);
534 }
535
536 FORCEINLINE
537 VOID
538 KiCheckDeferredReadyList(IN PKPRCB Prcb)
539 {
540 /* Scan the deferred ready lists if required */
541 if (Prcb->DeferredReadyListHead.Next) KiProcessDeferredReadyList(Prcb);
542 }
543
544 FORCEINLINE
545 VOID
546 KiRequestApcInterrupt(IN BOOLEAN NeedApc,
547 IN UCHAR Processor)
548 {
549 /* Check if we need to request APC delivery */
550 if (NeedApc)
551 {
552 /* Check if it's on another CPU */
553 if (KeGetCurrentPrcb()->Number != Processor)
554 {
555 /* Send an IPI to request delivery */
556 KiIpiSend(AFFINITY_MASK(Processor), IPI_APC);
557 }
558 else
559 {
560 /* Request a software interrupt */
561 HalRequestSoftwareInterrupt(APC_LEVEL);
562 }
563 }
564 }
565
566 FORCEINLINE
567 PKSPIN_LOCK_QUEUE
568 KiAcquireTimerLock(IN ULONG Hand)
569 {
570 PKSPIN_LOCK_QUEUE LockQueue;
571 ULONG LockIndex;
572 ASSERT(KeGetCurrentIrql() >= DISPATCH_LEVEL);
573
574 /* Get the lock index */
575 LockIndex = Hand >> LOCK_QUEUE_TIMER_LOCK_SHIFT;
576 LockIndex &= (LOCK_QUEUE_TIMER_TABLE_LOCKS - 1);
577
578 /* Now get the lock */
579 LockQueue = &KeGetCurrentPrcb()->LockQueue[LockQueueTimerTableLock + LockIndex];
580
581 /* Acquire it and return */
582 KeAcquireQueuedSpinLockAtDpcLevel(LockQueue);
583 return LockQueue;
584 }
585
586 FORCEINLINE
587 VOID
588 KiReleaseTimerLock(IN PKSPIN_LOCK_QUEUE LockQueue)
589 {
590 ASSERT(KeGetCurrentIrql() >= DISPATCH_LEVEL);
591
592 /* Release the lock */
593 KeReleaseQueuedSpinLockFromDpcLevel(LockQueue);
594 }
595
596 #endif
597
598 FORCEINLINE
599 VOID
600 KiAcquireApcLock(IN PKTHREAD Thread,
601 IN PKLOCK_QUEUE_HANDLE Handle)
602 {
603 /* Acquire the lock and raise to synchronization level */
604 KeAcquireInStackQueuedSpinLockRaiseToSynch(&Thread->ApcQueueLock, Handle);
605 }
606
607 FORCEINLINE
608 VOID
609 KiAcquireApcLockAtDpcLevel(IN PKTHREAD Thread,
610 IN PKLOCK_QUEUE_HANDLE Handle)
611 {
612 /* Acquire the lock */
613 KeAcquireInStackQueuedSpinLockAtDpcLevel(&Thread->ApcQueueLock, Handle);
614 }
615
616 FORCEINLINE
617 VOID
618 KiAcquireApcLockAtApcLevel(IN PKTHREAD Thread,
619 IN PKLOCK_QUEUE_HANDLE Handle)
620 {
621 /* Acquire the lock */
622 KeAcquireInStackQueuedSpinLock(&Thread->ApcQueueLock, Handle);
623 }
624
625 FORCEINLINE
626 VOID
627 KiReleaseApcLock(IN PKLOCK_QUEUE_HANDLE Handle)
628 {
629 /* Release the lock */
630 KeReleaseInStackQueuedSpinLock(Handle);
631 }
632
633 FORCEINLINE
634 VOID
635 KiReleaseApcLockFromDpcLevel(IN PKLOCK_QUEUE_HANDLE Handle)
636 {
637 /* Release the lock */
638 KeReleaseInStackQueuedSpinLockFromDpcLevel(Handle);
639 }
640
641 FORCEINLINE
642 VOID
643 KiAcquireProcessLock(IN PKPROCESS Process,
644 IN PKLOCK_QUEUE_HANDLE Handle)
645 {
646 /* Acquire the lock and raise to synchronization level */
647 KeAcquireInStackQueuedSpinLockRaiseToSynch(&Process->ProcessLock, Handle);
648 }
649
650 FORCEINLINE
651 VOID
652 KiReleaseProcessLock(IN PKLOCK_QUEUE_HANDLE Handle)
653 {
654 /* Release the lock */
655 KeReleaseInStackQueuedSpinLock(Handle);
656 }
657
658 FORCEINLINE
659 VOID
660 KiReleaseProcessLockFromDpcLevel(IN PKLOCK_QUEUE_HANDLE Handle)
661 {
662 /* Release the lock */
663 KeReleaseInStackQueuedSpinLockFromDpcLevel(Handle);
664 }
665
666 FORCEINLINE
667 VOID
668 KiAcquireDeviceQueueLock(IN PKDEVICE_QUEUE DeviceQueue,
669 IN PKLOCK_QUEUE_HANDLE DeviceLock)
670 {
671 /* Check if we were called from a threaded DPC */
672 if (KeGetCurrentPrcb()->DpcThreadActive)
673 {
674 /* Lock the Queue, we're not at DPC level */
675 KeAcquireInStackQueuedSpinLock(&DeviceQueue->Lock, DeviceLock);
676 }
677 else
678 {
679 /* We must be at DPC level, acquire the lock safely */
680 ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL);
681 KeAcquireInStackQueuedSpinLockAtDpcLevel(&DeviceQueue->Lock,
682 DeviceLock);
683 }
684 }
685
686 FORCEINLINE
687 VOID
688 KiReleaseDeviceQueueLock(IN PKLOCK_QUEUE_HANDLE DeviceLock)
689 {
690 /* Check if we were called from a threaded DPC */
691 if (KeGetCurrentPrcb()->DpcThreadActive)
692 {
693 /* Unlock the Queue, we're not at DPC level */
694 KeReleaseInStackQueuedSpinLock(DeviceLock);
695 }
696 else
697 {
698 /* We must be at DPC level, release the lock safely */
699 ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL);
700 KeReleaseInStackQueuedSpinLockFromDpcLevel(DeviceLock);
701 }
702 }
703
704 //
705 // Satisfies the wait of a mutant dispatcher object
706 //
707 #define KiSatisfyMutantWait(Object, Thread) \
708 { \
709 /* Decrease the Signal State */ \
710 (Object)->Header.SignalState--; \
711 \
712 /* Check if it's now non-signaled */ \
713 if (!(Object)->Header.SignalState) \
714 { \
715 /* Set the Owner Thread */ \
716 (Object)->OwnerThread = Thread; \
717 \
718 /* Disable APCs if needed */ \
719 Thread->KernelApcDisable = Thread->KernelApcDisable - \
720 (Object)->ApcDisable; \
721 \
722 /* Check if it's abandoned */ \
723 if ((Object)->Abandoned) \
724 { \
725 /* Unabandon it */ \
726 (Object)->Abandoned = FALSE; \
727 \
728 /* Return Status */ \
729 Thread->WaitStatus = STATUS_ABANDONED; \
730 } \
731 \
732 /* Insert it into the Mutant List */ \
733 InsertHeadList(Thread->MutantListHead.Blink, \
734 &(Object)->MutantListEntry); \
735 } \
736 }
737
738 //
739 // Satisfies the wait of any nonmutant dispatcher object
740 //
741 #define KiSatisfyNonMutantWait(Object) \
742 { \
743 if (((Object)->Header.Type & TIMER_OR_EVENT_TYPE) == \
744 EventSynchronizationObject) \
745 { \
746 /* Synchronization Timers and Events just get un-signaled */ \
747 (Object)->Header.SignalState = 0; \
748 } \
749 else if ((Object)->Header.Type == SemaphoreObject) \
750 { \
751 /* These ones can have multiple states, so we only decrease it */ \
752 (Object)->Header.SignalState--; \
753 } \
754 }
755
756 //
757 // Satisfies the wait of any dispatcher object
758 //
759 #define KiSatisfyObjectWait(Object, Thread) \
760 { \
761 /* Special case for Mutants */ \
762 if ((Object)->Header.Type == MutantObject) \
763 { \
764 KiSatisfyMutantWait((Object), (Thread)); \
765 } \
766 else \
767 { \
768 KiSatisfyNonMutantWait(Object); \
769 } \
770 }
771
772 //
773 // Recalculates the due time
774 //
775 FORCEINLINE
776 PLARGE_INTEGER
777 KiRecalculateDueTime(IN PLARGE_INTEGER OriginalDueTime,
778 IN PLARGE_INTEGER DueTime,
779 IN OUT PLARGE_INTEGER NewDueTime)
780 {
781 /* Don't do anything for absolute waits */
782 if (OriginalDueTime->QuadPart >= 0) return OriginalDueTime;
783
784 /* Otherwise, query the interrupt time and recalculate */
785 NewDueTime->QuadPart = KeQueryInterruptTime();
786 NewDueTime->QuadPart -= DueTime->QuadPart;
787 return NewDueTime;
788 }
789
790 //
791 // Determines whether a thread should be added to the wait list
792 //
793 FORCEINLINE
794 BOOLEAN
795 KiCheckThreadStackSwap(IN PKTHREAD Thread,
796 IN KPROCESSOR_MODE WaitMode)
797 {
798 /* Check the required conditions */
799 if ((WaitMode != KernelMode) &&
800 (Thread->EnableStackSwap) &&
801 (Thread->Priority >= (LOW_REALTIME_PRIORITY + 9)))
802 {
803 /* We are go for swap */
804 return TRUE;
805 }
806 else
807 {
808 /* Don't swap the thread */
809 return FALSE;
810 }
811 }
812
813 //
814 // Adds a thread to the wait list
815 //
816 #define KiAddThreadToWaitList(Thread, Swappable) \
817 { \
818 /* Make sure it's swappable */ \
819 if (Swappable) \
820 { \
821 /* Insert it into the PRCB's List */ \
822 InsertTailList(&KeGetCurrentPrcb()->WaitListHead, \
823 &Thread->WaitListEntry); \
824 } \
825 }
826
827 //
828 // Checks if a wait in progress should be interrupted by APCs or an alertable
829 // state.
830 //
831 FORCEINLINE
832 NTSTATUS
833 KiCheckAlertability(IN PKTHREAD Thread,
834 IN BOOLEAN Alertable,
835 IN KPROCESSOR_MODE WaitMode)
836 {
837 /* Check if the wait is alertable */
838 if (Alertable)
839 {
840 /* It is, first check if the thread is alerted in this mode */
841 if (Thread->Alerted[WaitMode])
842 {
843 /* It is, so bail out of the wait */
844 Thread->Alerted[WaitMode] = FALSE;
845 return STATUS_ALERTED;
846 }
847 else if ((WaitMode != KernelMode) &&
848 (!IsListEmpty(&Thread->ApcState.ApcListHead[UserMode])))
849 {
850 /* It's isn't, but this is a user wait with queued user APCs */
851 Thread->ApcState.UserApcPending = TRUE;
852 return STATUS_USER_APC;
853 }
854 else if (Thread->Alerted[KernelMode])
855 {
856 /* It isn't that either, but we're alered in kernel mode */
857 Thread->Alerted[KernelMode] = FALSE;
858 return STATUS_ALERTED;
859 }
860 }
861 else if ((WaitMode != KernelMode) && (Thread->ApcState.UserApcPending))
862 {
863 /* Not alertable, but this is a user wait with pending user APCs */
864 return STATUS_USER_APC;
865 }
866
867 /* Otherwise, we're fine */
868 return STATUS_WAIT_0;
869 }
870
871 FORCEINLINE
872 ULONG
873 KiComputeTimerTableIndex(IN ULONGLONG DueTime)
874 {
875 return (DueTime / KeMaximumIncrement) & (TIMER_TABLE_SIZE - 1);
876 }
877
878 //
879 // Called from KiCompleteTimer, KiInsertTreeTimer, KeSetSystemTime
880 // to remove timer entries
881 // See Windows HPI blog for more information.
882 FORCEINLINE
883 VOID
884 KiRemoveEntryTimer(IN PKTIMER Timer)
885 {
886 ULONG Hand;
887 PKTIMER_TABLE_ENTRY TableEntry;
888
889 /* Remove the timer from the timer list and check if it's empty */
890 Hand = Timer->Header.Hand;
891 if (RemoveEntryList(&Timer->TimerListEntry))
892 {
893 /* Get the respective timer table entry */
894 TableEntry = &KiTimerTableListHead[Hand];
895 if (&TableEntry->Entry == TableEntry->Entry.Flink)
896 {
897 /* Set the entry to an infinite absolute time */
898 TableEntry->Time.HighPart = 0xFFFFFFFF;
899 }
900 }
901
902 /* Clear the list entries on dbg builds so we can tell the timer is gone */
903 #if DBG
904 Timer->TimerListEntry.Flink = NULL;
905 Timer->TimerListEntry.Blink = NULL;
906 #endif
907 }
908
909 //
910 // Called by Wait and Queue code to insert a timer for dispatching.
911 // Also called by KeSetTimerEx to insert a timer from the caller.
912 //
913 FORCEINLINE
914 VOID
915 KxInsertTimer(IN PKTIMER Timer,
916 IN ULONG Hand)
917 {
918 PKSPIN_LOCK_QUEUE LockQueue;
919
920 /* Acquire the lock and release the dispatcher lock */
921 LockQueue = KiAcquireTimerLock(Hand);
922 KiReleaseDispatcherLockFromDpcLevel();
923
924 /* Try to insert the timer */
925 if (KiInsertTimerTable(Timer, Hand))
926 {
927 /* Complete it */
928 KiCompleteTimer(Timer, LockQueue);
929 }
930 else
931 {
932 /* Do nothing, just release the lock */
933 KiReleaseTimerLock(LockQueue);
934 }
935 }
936
937 //
938 // Called by KeSetTimerEx and KiInsertTreeTimer to calculate Due Time
939 // See the Windows HPI Blog for more information
940 //
941 FORCEINLINE
942 BOOLEAN
943 KiComputeDueTime(IN PKTIMER Timer,
944 IN LARGE_INTEGER DueTime,
945 OUT PULONG Hand)
946 {
947 LARGE_INTEGER InterruptTime, SystemTime, DifferenceTime;
948
949 /* Convert to relative time if needed */
950 Timer->Header.Absolute = FALSE;
951 if (DueTime.HighPart >= 0)
952 {
953 /* Get System Time */
954 KeQuerySystemTime(&SystemTime);
955
956 /* Do the conversion */
957 DifferenceTime.QuadPart = SystemTime.QuadPart - DueTime.QuadPart;
958
959 /* Make sure it hasn't already expired */
960 Timer->Header.Absolute = TRUE;
961 if (DifferenceTime.HighPart >= 0)
962 {
963 /* Cancel everything */
964 Timer->Header.SignalState = TRUE;
965 Timer->Header.Hand = 0;
966 Timer->DueTime.QuadPart = 0;
967 *Hand = 0;
968 return FALSE;
969 }
970
971 /* Set the time as Absolute */
972 DueTime = DifferenceTime;
973 }
974
975 /* Get the Interrupt Time */
976 InterruptTime.QuadPart = KeQueryInterruptTime();
977
978 /* Recalculate due time */
979 Timer->DueTime.QuadPart = InterruptTime.QuadPart - DueTime.QuadPart;
980
981 /* Get the handle */
982 *Hand = KiComputeTimerTableIndex(Timer->DueTime.QuadPart);
983 Timer->Header.Hand = (UCHAR)*Hand;
984 Timer->Header.Inserted = TRUE;
985 return TRUE;
986 }
987
988 //
989 // Called from Unlink and Queue Insert Code.
990 // Also called by timer code when canceling an inserted timer.
991 // Removes a timer from it's tree.
992 //
993 FORCEINLINE
994 VOID
995 KxRemoveTreeTimer(IN PKTIMER Timer)
996 {
997 ULONG Hand = Timer->Header.Hand;
998 PKSPIN_LOCK_QUEUE LockQueue;
999 PKTIMER_TABLE_ENTRY TimerEntry;
1000
1001 /* Acquire timer lock */
1002 LockQueue = KiAcquireTimerLock(Hand);
1003
1004 /* Set the timer as non-inserted */
1005 Timer->Header.Inserted = FALSE;
1006
1007 /* Remove it from the timer list */
1008 if (RemoveEntryList(&Timer->TimerListEntry))
1009 {
1010 /* Get the entry and check if it's empty */
1011 TimerEntry = &KiTimerTableListHead[Hand];
1012 if (IsListEmpty(&TimerEntry->Entry))
1013 {
1014 /* Clear the time then */
1015 TimerEntry->Time.HighPart = 0xFFFFFFFF;
1016 }
1017 }
1018
1019 /* Release the timer lock */
1020 KiReleaseTimerLock(LockQueue);
1021 }
1022
1023 FORCEINLINE
1024 VOID
1025 KxSetTimerForThreadWait(IN PKTIMER Timer,
1026 IN LARGE_INTEGER Interval,
1027 OUT PULONG Hand)
1028 {
1029 ULONGLONG DueTime;
1030 LARGE_INTEGER InterruptTime, SystemTime, TimeDifference;
1031
1032 /* Check the timer's interval to see if it's absolute */
1033 Timer->Header.Absolute = FALSE;
1034 if (Interval.HighPart >= 0)
1035 {
1036 /* Get the system time and calculate the relative time */
1037 KeQuerySystemTime(&SystemTime);
1038 TimeDifference.QuadPart = SystemTime.QuadPart - Interval.QuadPart;
1039 Timer->Header.Absolute = TRUE;
1040
1041 /* Check if we've already expired */
1042 if (TimeDifference.HighPart >= 0)
1043 {
1044 /* Reset everything */
1045 Timer->DueTime.QuadPart = 0;
1046 *Hand = 0;
1047 Timer->Header.Hand = 0;
1048 return;
1049 }
1050 else
1051 {
1052 /* Update the interval */
1053 Interval = TimeDifference;
1054 }
1055 }
1056
1057 /* Calculate the due time */
1058 InterruptTime.QuadPart = KeQueryInterruptTime();
1059 DueTime = InterruptTime.QuadPart - Interval.QuadPart;
1060 Timer->DueTime.QuadPart = DueTime;
1061
1062 /* Calculate the timer handle */
1063 *Hand = KiComputeTimerTableIndex(DueTime);
1064 Timer->Header.Hand = (UCHAR)*Hand;
1065 }
1066
1067 #define KxDelayThreadWait() \
1068 \
1069 /* Setup the Wait Block */ \
1070 Thread->WaitBlockList = TimerBlock; \
1071 \
1072 /* Setup the timer */ \
1073 KxSetTimerForThreadWait(Timer, *Interval, &Hand); \
1074 \
1075 /* Save the due time for the caller */ \
1076 DueTime.QuadPart = Timer->DueTime.QuadPart; \
1077 \
1078 /* Link the timer to this Wait Block */ \
1079 TimerBlock->NextWaitBlock = TimerBlock; \
1080 Timer->Header.WaitListHead.Flink = &TimerBlock->WaitListEntry; \
1081 Timer->Header.WaitListHead.Blink = &TimerBlock->WaitListEntry; \
1082 \
1083 /* Clear wait status */ \
1084 Thread->WaitStatus = STATUS_SUCCESS; \
1085 \
1086 /* Setup wait fields */ \
1087 Thread->Alertable = Alertable; \
1088 Thread->WaitReason = DelayExecution; \
1089 Thread->WaitMode = WaitMode; \
1090 \
1091 /* Check if we can swap the thread's stack */ \
1092 Thread->WaitListEntry.Flink = NULL; \
1093 Swappable = KiCheckThreadStackSwap(Thread, WaitMode); \
1094 \
1095 /* Set the wait time */ \
1096 Thread->WaitTime = KeTickCount.LowPart;
1097
1098 #define KxMultiThreadWait() \
1099 /* Link wait block array to the thread */ \
1100 Thread->WaitBlockList = WaitBlockArray; \
1101 \
1102 /* Reset the index */ \
1103 Index = 0; \
1104 \
1105 /* Loop wait blocks */ \
1106 do \
1107 { \
1108 /* Fill out the wait block */ \
1109 WaitBlock = &WaitBlockArray[Index]; \
1110 WaitBlock->Object = Object[Index]; \
1111 WaitBlock->WaitKey = (USHORT)Index; \
1112 WaitBlock->WaitType = WaitType; \
1113 WaitBlock->Thread = Thread; \
1114 \
1115 /* Link to next block */ \
1116 WaitBlock->NextWaitBlock = &WaitBlockArray[Index + 1]; \
1117 Index++; \
1118 } while (Index < Count); \
1119 \
1120 /* Link the last block */ \
1121 WaitBlock->NextWaitBlock = WaitBlockArray; \
1122 \
1123 /* Set default wait status */ \
1124 Thread->WaitStatus = STATUS_WAIT_0; \
1125 \
1126 /* Check if we have a timer */ \
1127 if (Timeout) \
1128 { \
1129 /* Link to the block */ \
1130 TimerBlock->NextWaitBlock = WaitBlockArray; \
1131 \
1132 /* Setup the timer */ \
1133 KxSetTimerForThreadWait(Timer, *Timeout, &Hand); \
1134 \
1135 /* Save the due time for the caller */ \
1136 DueTime.QuadPart = Timer->DueTime.QuadPart; \
1137 \
1138 /* Initialize the list */ \
1139 InitializeListHead(&Timer->Header.WaitListHead); \
1140 } \
1141 \
1142 /* Set wait settings */ \
1143 Thread->Alertable = Alertable; \
1144 Thread->WaitMode = WaitMode; \
1145 Thread->WaitReason = WaitReason; \
1146 \
1147 /* Check if we can swap the thread's stack */ \
1148 Thread->WaitListEntry.Flink = NULL; \
1149 Swappable = KiCheckThreadStackSwap(Thread, WaitMode); \
1150 \
1151 /* Set the wait time */ \
1152 Thread->WaitTime = KeTickCount.LowPart;
1153
1154 #define KxSingleThreadWait() \
1155 /* Setup the Wait Block */ \
1156 Thread->WaitBlockList = WaitBlock; \
1157 WaitBlock->WaitKey = STATUS_SUCCESS; \
1158 WaitBlock->Object = Object; \
1159 WaitBlock->WaitType = WaitAny; \
1160 \
1161 /* Clear wait status */ \
1162 Thread->WaitStatus = STATUS_SUCCESS; \
1163 \
1164 /* Check if we have a timer */ \
1165 if (Timeout) \
1166 { \
1167 /* Setup the timer */ \
1168 KxSetTimerForThreadWait(Timer, *Timeout, &Hand); \
1169 \
1170 /* Save the due time for the caller */ \
1171 DueTime.QuadPart = Timer->DueTime.QuadPart; \
1172 \
1173 /* Pointer to timer block */ \
1174 WaitBlock->NextWaitBlock = TimerBlock; \
1175 TimerBlock->NextWaitBlock = WaitBlock; \
1176 \
1177 /* Link the timer to this Wait Block */ \
1178 Timer->Header.WaitListHead.Flink = &TimerBlock->WaitListEntry; \
1179 Timer->Header.WaitListHead.Blink = &TimerBlock->WaitListEntry; \
1180 } \
1181 else \
1182 { \
1183 /* No timer block, just ourselves */ \
1184 WaitBlock->NextWaitBlock = WaitBlock; \
1185 } \
1186 \
1187 /* Set wait settings */ \
1188 Thread->Alertable = Alertable; \
1189 Thread->WaitMode = WaitMode; \
1190 Thread->WaitReason = WaitReason; \
1191 \
1192 /* Check if we can swap the thread's stack */ \
1193 Thread->WaitListEntry.Flink = NULL; \
1194 Swappable = KiCheckThreadStackSwap(Thread, WaitMode); \
1195 \
1196 /* Set the wait time */ \
1197 Thread->WaitTime = KeTickCount.LowPart;
1198
1199 #define KxQueueThreadWait() \
1200 /* Setup the Wait Block */ \
1201 Thread->WaitBlockList = WaitBlock; \
1202 WaitBlock->WaitKey = STATUS_SUCCESS; \
1203 WaitBlock->Object = Queue; \
1204 WaitBlock->WaitType = WaitAny; \
1205 WaitBlock->Thread = Thread; \
1206 \
1207 /* Clear wait status */ \
1208 Thread->WaitStatus = STATUS_SUCCESS; \
1209 \
1210 /* Check if we have a timer */ \
1211 if (Timeout) \
1212 { \
1213 /* Setup the timer */ \
1214 KxSetTimerForThreadWait(Timer, *Timeout, &Hand); \
1215 \
1216 /* Save the due time for the caller */ \
1217 DueTime.QuadPart = Timer->DueTime.QuadPart; \
1218 \
1219 /* Pointer to timer block */ \
1220 WaitBlock->NextWaitBlock = TimerBlock; \
1221 TimerBlock->NextWaitBlock = WaitBlock; \
1222 \
1223 /* Link the timer to this Wait Block */ \
1224 Timer->Header.WaitListHead.Flink = &TimerBlock->WaitListEntry; \
1225 Timer->Header.WaitListHead.Blink = &TimerBlock->WaitListEntry; \
1226 } \
1227 else \
1228 { \
1229 /* No timer block, just ourselves */ \
1230 WaitBlock->NextWaitBlock = WaitBlock; \
1231 } \
1232 \
1233 /* Set wait settings */ \
1234 Thread->Alertable = FALSE; \
1235 Thread->WaitMode = WaitMode; \
1236 Thread->WaitReason = WrQueue; \
1237 \
1238 /* Check if we can swap the thread's stack */ \
1239 Thread->WaitListEntry.Flink = NULL; \
1240 Swappable = KiCheckThreadStackSwap(Thread, WaitMode); \
1241 \
1242 /* Set the wait time */ \
1243 Thread->WaitTime = KeTickCount.LowPart;
1244
1245 //
1246 // Unwaits a Thread
1247 //
1248 FORCEINLINE
1249 VOID
1250 KxUnwaitThread(IN DISPATCHER_HEADER *Object,
1251 IN KPRIORITY Increment)
1252 {
1253 PLIST_ENTRY WaitEntry, WaitList;
1254 PKWAIT_BLOCK WaitBlock;
1255 PKTHREAD WaitThread;
1256 ULONG WaitKey;
1257
1258 /* Loop the Wait Entries */
1259 WaitList = &Object->WaitListHead;
1260 ASSERT(IsListEmpty(&Object->WaitListHead) == FALSE);
1261 WaitEntry = WaitList->Flink;
1262 do
1263 {
1264 /* Get the current wait block */
1265 WaitBlock = CONTAINING_RECORD(WaitEntry, KWAIT_BLOCK, WaitListEntry);
1266
1267 /* Get the waiting thread */
1268 WaitThread = WaitBlock->Thread;
1269
1270 /* Check the current Wait Mode */
1271 if (WaitBlock->WaitType == WaitAny)
1272 {
1273 /* Use the actual wait key */
1274 WaitKey = WaitBlock->WaitKey;
1275 }
1276 else
1277 {
1278 /* Otherwise, use STATUS_KERNEL_APC */
1279 WaitKey = STATUS_KERNEL_APC;
1280 }
1281
1282 /* Unwait the thread */
1283 KiUnwaitThread(WaitThread, WaitKey, Increment);
1284
1285 /* Next entry */
1286 WaitEntry = WaitList->Flink;
1287 } while (WaitEntry != WaitList);
1288 }
1289
1290 //
1291 // Unwaits a Thread waiting on an event
1292 //
1293 FORCEINLINE
1294 VOID
1295 KxUnwaitThreadForEvent(IN PKEVENT Event,
1296 IN KPRIORITY Increment)
1297 {
1298 PLIST_ENTRY WaitEntry, WaitList;
1299 PKWAIT_BLOCK WaitBlock;
1300 PKTHREAD WaitThread;
1301
1302 /* Loop the Wait Entries */
1303 WaitList = &Event->Header.WaitListHead;
1304 ASSERT(IsListEmpty(&Event->Header.WaitListHead) == FALSE);
1305 WaitEntry = WaitList->Flink;
1306 do
1307 {
1308 /* Get the current wait block */
1309 WaitBlock = CONTAINING_RECORD(WaitEntry, KWAIT_BLOCK, WaitListEntry);
1310
1311 /* Get the waiting thread */
1312 WaitThread = WaitBlock->Thread;
1313
1314 /* Check the current Wait Mode */
1315 if (WaitBlock->WaitType == WaitAny)
1316 {
1317 /* Un-signal it */
1318 Event->Header.SignalState = 0;
1319
1320 /* Un-signal the event and unwait the thread */
1321 KiUnwaitThread(WaitThread, WaitBlock->WaitKey, Increment);
1322 break;
1323 }
1324
1325 /* Unwait the thread with STATUS_KERNEL_APC */
1326 KiUnwaitThread(WaitThread, STATUS_KERNEL_APC, Increment);
1327
1328 /* Next entry */
1329 WaitEntry = WaitList->Flink;
1330 } while (WaitEntry != WaitList);
1331 }
1332
1333 //
1334 // This routine queues a thread that is ready on the PRCB's ready lists.
1335 // If this thread cannot currently run on this CPU, then the thread is
1336 // added to the deferred ready list instead.
1337 //
1338 // This routine must be entered with the PRCB lock held and it will exit
1339 // with the PRCB lock released!
1340 //
1341 FORCEINLINE
1342 VOID
1343 KxQueueReadyThread(IN PKTHREAD Thread,
1344 IN PKPRCB Prcb)
1345 {
1346 BOOLEAN Preempted;
1347 KPRIORITY Priority;
1348
1349 /* Sanity checks */
1350 ASSERT(Prcb == KeGetCurrentPrcb());
1351 ASSERT(Thread->State == Running);
1352 ASSERT(Thread->NextProcessor == Prcb->Number);
1353
1354 /* Check if this thread is allowed to run in this CPU */
1355 #ifdef CONFIG_SMP
1356 if ((Thread->Affinity) & (Prcb->SetMember))
1357 #else
1358 if (TRUE)
1359 #endif
1360 {
1361 /* Set thread ready for execution */
1362 Thread->State = Ready;
1363
1364 /* Save current priority and if someone had pre-empted it */
1365 Priority = Thread->Priority;
1366 Preempted = Thread->Preempted;
1367
1368 /* We're not pre-empting now, and set the wait time */
1369 Thread->Preempted = FALSE;
1370 Thread->WaitTime = KeTickCount.LowPart;
1371
1372 /* Sanity check */
1373 ASSERT((Priority >= 0) && (Priority <= HIGH_PRIORITY));
1374
1375 /* Insert this thread in the appropriate order */
1376 Preempted ? InsertHeadList(&Prcb->DispatcherReadyListHead[Priority],
1377 &Thread->WaitListEntry) :
1378 InsertTailList(&Prcb->DispatcherReadyListHead[Priority],
1379 &Thread->WaitListEntry);
1380
1381 /* Update the ready summary */
1382 Prcb->ReadySummary |= PRIORITY_MASK(Priority);
1383
1384 /* Sanity check */
1385 ASSERT(Priority == Thread->Priority);
1386
1387 /* Release the PRCB lock */
1388 KiReleasePrcbLock(Prcb);
1389 }
1390 else
1391 {
1392 /* Otherwise, prepare this thread to be deferred */
1393 Thread->State = DeferredReady;
1394 Thread->DeferredProcessor = Prcb->Number;
1395
1396 /* Release the lock and defer scheduling */
1397 KiReleasePrcbLock(Prcb);
1398 KiDeferredReadyThread(Thread);
1399 }
1400 }
1401
1402 //
1403 // This routine scans for an appropriate ready thread to select at the
1404 // given priority and for the given CPU.
1405 //
1406 FORCEINLINE
1407 PKTHREAD
1408 KiSelectReadyThread(IN KPRIORITY Priority,
1409 IN PKPRCB Prcb)
1410 {
1411 ULONG PrioritySet;
1412 LONG HighPriority;
1413 PLIST_ENTRY ListEntry;
1414 PKTHREAD Thread = NULL;
1415
1416 /* Save the current mask and get the priority set for the CPU */
1417 PrioritySet = Prcb->ReadySummary >> Priority;
1418 if (!PrioritySet) goto Quickie;
1419
1420 /* Get the highest priority possible */
1421 BitScanReverse((PULONG)&HighPriority, PrioritySet);
1422 ASSERT((PrioritySet & PRIORITY_MASK(HighPriority)) != 0);
1423 HighPriority += Priority;
1424
1425 /* Make sure the list isn't empty at the highest priority */
1426 ASSERT(IsListEmpty(&Prcb->DispatcherReadyListHead[HighPriority]) == FALSE);
1427
1428 /* Get the first thread on the list */
1429 ListEntry = Prcb->DispatcherReadyListHead[HighPriority].Flink;
1430 Thread = CONTAINING_RECORD(ListEntry, KTHREAD, WaitListEntry);
1431
1432 /* Make sure this thread is here for a reason */
1433 ASSERT(HighPriority == Thread->Priority);
1434 ASSERT(Thread->Affinity & AFFINITY_MASK(Prcb->Number));
1435 ASSERT(Thread->NextProcessor == Prcb->Number);
1436
1437 /* Remove it from the list */
1438 if (RemoveEntryList(&Thread->WaitListEntry))
1439 {
1440 /* The list is empty now, reset the ready summary */
1441 Prcb->ReadySummary ^= PRIORITY_MASK(HighPriority);
1442 }
1443
1444 /* Sanity check and return the thread */
1445 Quickie:
1446 ASSERT((Thread == NULL) ||
1447 (Thread->BasePriority == 0) ||
1448 (Thread->Priority != 0));
1449 return Thread;
1450 }
1451
1452 //
1453 // This routine computes the new priority for a thread. It is only valid for
1454 // threads with priorities in the dynamic priority range.
1455 //
1456 FORCEINLINE
1457 SCHAR
1458 KiComputeNewPriority(IN PKTHREAD Thread,
1459 IN SCHAR Adjustment)
1460 {
1461 SCHAR Priority;
1462
1463 /* Priority sanity checks */
1464 ASSERT((Thread->PriorityDecrement >= 0) &&
1465 (Thread->PriorityDecrement <= Thread->Priority));
1466 ASSERT((Thread->Priority < LOW_REALTIME_PRIORITY) ?
1467 TRUE : (Thread->PriorityDecrement == 0));
1468
1469 /* Get the current priority */
1470 Priority = Thread->Priority;
1471 if (Priority < LOW_REALTIME_PRIORITY)
1472 {
1473 /* Decrease priority by the priority decrement */
1474 Priority -= (Thread->PriorityDecrement + Adjustment);
1475
1476 /* Don't go out of bounds */
1477 if (Priority < Thread->BasePriority) Priority = Thread->BasePriority;
1478
1479 /* Reset the priority decrement */
1480 Thread->PriorityDecrement = 0;
1481 }
1482
1483 /* Sanity check */
1484 ASSERT((Thread->BasePriority == 0) || (Priority != 0));
1485
1486 /* Return the new priority */
1487 return Priority;
1488 }
1489
1490 //
1491 // Guarded Mutex Routines
1492 //
1493 FORCEINLINE
1494 VOID
1495 _KeInitializeGuardedMutex(OUT PKGUARDED_MUTEX GuardedMutex)
1496 {
1497 /* Setup the Initial Data */
1498 GuardedMutex->Count = GM_LOCK_BIT;
1499 GuardedMutex->Owner = NULL;
1500 GuardedMutex->Contention = 0;
1501
1502 /* Initialize the Wait Gate */
1503 KeInitializeGate(&GuardedMutex->Gate);
1504 }
1505
1506 FORCEINLINE
1507 VOID
1508 _KeAcquireGuardedMutexUnsafe(IN OUT PKGUARDED_MUTEX GuardedMutex)
1509 {
1510 PKTHREAD Thread = KeGetCurrentThread();
1511
1512 /* Sanity checks */
1513 ASSERT((KeGetCurrentIrql() == APC_LEVEL) ||
1514 (Thread->SpecialApcDisable < 0) ||
1515 (Thread->Teb == NULL) ||
1516 (Thread->Teb >= (PTEB)MM_SYSTEM_RANGE_START));
1517 ASSERT(GuardedMutex->Owner != Thread);
1518
1519 /* Remove the lock */
1520 if (!InterlockedBitTestAndReset(&GuardedMutex->Count, GM_LOCK_BIT_V))
1521 {
1522 /* The Guarded Mutex was already locked, enter contented case */
1523 KiAcquireGuardedMutex(GuardedMutex);
1524 }
1525
1526 /* Set the Owner */
1527 GuardedMutex->Owner = Thread;
1528 }
1529
1530 FORCEINLINE
1531 VOID
1532 _KeReleaseGuardedMutexUnsafe(IN OUT PKGUARDED_MUTEX GuardedMutex)
1533 {
1534 LONG OldValue, NewValue;
1535
1536 /* Sanity checks */
1537 ASSERT((KeGetCurrentIrql() == APC_LEVEL) ||
1538 (KeGetCurrentThread()->SpecialApcDisable < 0) ||
1539 (KeGetCurrentThread()->Teb == NULL) ||
1540 (KeGetCurrentThread()->Teb >= (PTEB)MM_SYSTEM_RANGE_START));
1541 ASSERT(GuardedMutex->Owner == KeGetCurrentThread());
1542
1543 /* Destroy the Owner */
1544 GuardedMutex->Owner = NULL;
1545
1546 /* Add the Lock Bit */
1547 OldValue = InterlockedExchangeAdd(&GuardedMutex->Count, GM_LOCK_BIT);
1548 ASSERT((OldValue & GM_LOCK_BIT) == 0);
1549
1550 /* Check if it was already locked, but not woken */
1551 if ((OldValue) && !(OldValue & GM_LOCK_WAITER_WOKEN))
1552 {
1553 /* Update the Oldvalue to what it should be now */
1554 OldValue += GM_LOCK_BIT;
1555
1556 /* The mutex will be woken, minus one waiter */
1557 NewValue = OldValue + GM_LOCK_WAITER_WOKEN -
1558 GM_LOCK_WAITER_INC;
1559
1560 /* Remove the Woken bit */
1561 if (InterlockedCompareExchange(&GuardedMutex->Count,
1562 NewValue,
1563 OldValue) == OldValue)
1564 {
1565 /* Signal the Gate */
1566 KeSignalGateBoostPriority(&GuardedMutex->Gate);
1567 }
1568 }
1569 }
1570
1571 FORCEINLINE
1572 VOID
1573 _KeAcquireGuardedMutex(IN PKGUARDED_MUTEX GuardedMutex)
1574 {
1575 PKTHREAD Thread = KeGetCurrentThread();
1576
1577 /* Sanity checks */
1578 ASSERT(KeGetCurrentIrql() <= APC_LEVEL);
1579 ASSERT(GuardedMutex->Owner != Thread);
1580
1581 /* Disable Special APCs */
1582 KeEnterGuardedRegionThread(Thread);
1583
1584 /* Remove the lock */
1585 if (!InterlockedBitTestAndReset(&GuardedMutex->Count, GM_LOCK_BIT_V))
1586 {
1587 /* The Guarded Mutex was already locked, enter contented case */
1588 KiAcquireGuardedMutex(GuardedMutex);
1589 }
1590
1591 /* Set the Owner and Special APC Disable state */
1592 GuardedMutex->Owner = Thread;
1593 GuardedMutex->SpecialApcDisable = Thread->SpecialApcDisable;
1594 }
1595
1596 FORCEINLINE
1597 VOID
1598 _KeReleaseGuardedMutex(IN OUT PKGUARDED_MUTEX GuardedMutex)
1599 {
1600 PKTHREAD Thread = KeGetCurrentThread();
1601 LONG OldValue, NewValue;
1602
1603 /* Sanity checks */
1604 ASSERT(KeGetCurrentIrql() <= APC_LEVEL);
1605 ASSERT(GuardedMutex->Owner == Thread);
1606 ASSERT(Thread->SpecialApcDisable == GuardedMutex->SpecialApcDisable);
1607
1608 /* Destroy the Owner */
1609 GuardedMutex->Owner = NULL;
1610
1611 /* Add the Lock Bit */
1612 OldValue = InterlockedExchangeAdd(&GuardedMutex->Count, GM_LOCK_BIT);
1613 ASSERT((OldValue & GM_LOCK_BIT) == 0);
1614
1615 /* Check if it was already locked, but not woken */
1616 if ((OldValue) && !(OldValue & GM_LOCK_WAITER_WOKEN))
1617 {
1618 /* Update the Oldvalue to what it should be now */
1619 OldValue += GM_LOCK_BIT;
1620
1621 /* The mutex will be woken, minus one waiter */
1622 NewValue = OldValue + GM_LOCK_WAITER_WOKEN -
1623 GM_LOCK_WAITER_INC;
1624
1625 /* Remove the Woken bit */
1626 if (InterlockedCompareExchange(&GuardedMutex->Count,
1627 NewValue,
1628 OldValue) == OldValue)
1629 {
1630 /* Signal the Gate */
1631 KeSignalGateBoostPriority(&GuardedMutex->Gate);
1632 }
1633 }
1634
1635 /* Re-enable APCs */
1636 KeLeaveGuardedRegionThread(Thread);
1637 }
1638
1639 FORCEINLINE
1640 BOOLEAN
1641 _KeTryToAcquireGuardedMutex(IN OUT PKGUARDED_MUTEX GuardedMutex)
1642 {
1643 PKTHREAD Thread = KeGetCurrentThread();
1644
1645 /* Block APCs */
1646 KeEnterGuardedRegionThread(Thread);
1647
1648 /* Remove the lock */
1649 if (!InterlockedBitTestAndReset(&GuardedMutex->Count, GM_LOCK_BIT_V))
1650 {
1651 /* Re-enable APCs */
1652 KeLeaveGuardedRegionThread(Thread);
1653 YieldProcessor();
1654
1655 /* Return failure */
1656 return FALSE;
1657 }
1658
1659 /* Set the Owner and APC State */
1660 GuardedMutex->Owner = Thread;
1661 GuardedMutex->SpecialApcDisable = Thread->SpecialApcDisable;
1662 return TRUE;
1663 }
1664
1665
1666 FORCEINLINE
1667 VOID
1668 KiAcquireNmiListLock(OUT PKIRQL OldIrql)
1669 {
1670 KeAcquireSpinLock(&KiNmiCallbackListLock, OldIrql);
1671 }
1672
1673 FORCEINLINE
1674 VOID
1675 KiReleaseNmiListLock(IN KIRQL OldIrql)
1676 {
1677 KeReleaseSpinLock(&KiNmiCallbackListLock, OldIrql);
1678 }
1679
1680 #if defined(_M_IX86) || defined(_M_AMD64)
1681 FORCEINLINE
1682 VOID
1683 KiCpuId(
1684 PCPU_INFO CpuInfo,
1685 ULONG Function)
1686 {
1687 __cpuid((INT*)CpuInfo->AsUINT32, Function);
1688 }
1689
1690 FORCEINLINE
1691 VOID
1692 KiCpuIdEx(
1693 PCPU_INFO CpuInfo,
1694 ULONG Function,
1695 ULONG SubFunction)
1696 {
1697 __cpuidex((INT*)CpuInfo->AsUINT32, Function, SubFunction);
1698 }
1699 #endif /* _M_IX86 || _M_AMD64 */