projects / reactos.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
history | raw | HEAD
- Convert KxDelayThreadWait, KxSingleThreadWait, KxMultiThreadWait into macros.
[reactos.git] / reactos / ntoskrnl / include / internal / ke_x.h
1 /*
2 * PROJECT: ReactOS Kernel
3 * LICENSE: GPL - See COPYING in the top level directory
4 * FILE: ntoskrnl/include/ke_x.h
5 * PURPOSE: Internal Inlined Functions for the Kernel
6 * PROGRAMMERS: Alex Ionescu (alex.ionescu@reactos.org)
7 */
8
9 //
10 // Thread Dispatcher Header DebugActive Mask
11 //
12 #define DR_MASK(x) 1 << x
13 #define DR_ACTIVE_MASK 0x10
14 #define DR_REG_MASK 0x4F
15
16 //
17 // Sanitizes a selector
18 //
19 FORCEINLINE
20 ULONG
21 Ke386SanitizeSeg(IN ULONG Cs,
22 IN KPROCESSOR_MODE Mode)
23 {
24 //
25 // Check if we're in kernel-mode, and force CPL 0 if so.
26 // Otherwise, force CPL 3.
27 //
28 return ((Mode == KernelMode) ?
29 (Cs & (0xFFFF & ~RPL_MASK)) :
30 (RPL_MASK | (Cs & 0xFFFF)));
31 }
32
33 //
34 // Sanitizes EFLAGS
35 //
36 FORCEINLINE
37 ULONG
38 Ke386SanitizeFlags(IN ULONG Eflags,
39 IN KPROCESSOR_MODE Mode)
40 {
41 //
42 // Check if we're in kernel-mode, and sanitize EFLAGS if so.
43 // Otherwise, also force interrupt mask on.
44 //
45 return ((Mode == KernelMode) ?
46 (Eflags & (EFLAGS_USER_SANITIZE | EFLAGS_INTERRUPT_MASK)) :
47 (EFLAGS_INTERRUPT_MASK | (Eflags & EFLAGS_USER_SANITIZE)));
48 }
49
50 //
51 // Gets a DR register from a CONTEXT structure
52 //
53 FORCEINLINE
54 PVOID
55 KiDrFromContext(IN ULONG Dr,
56 IN PCONTEXT Context)
57 {
58 return *(PVOID*)((ULONG_PTR)Context + KiDebugRegisterContextOffsets[Dr]);
59 }
60
61 //
62 // Gets a DR register from a KTRAP_FRAME structure
63 //
64 FORCEINLINE
65 PVOID*
66 KiDrFromTrapFrame(IN ULONG Dr,
67 IN PKTRAP_FRAME TrapFrame)
68 {
69 return (PVOID*)((ULONG_PTR)TrapFrame + KiDebugRegisterTrapOffsets[Dr]);
70 }
71
72 //
73 //
74 //
75 FORCEINLINE
76 PVOID
77 Ke386SanitizeDr(IN PVOID DrAddress,
78 IN KPROCESSOR_MODE Mode)
79 {
80 //
81 // Check if we're in kernel-mode, and return the address directly if so.
82 // Otherwise, make sure it's not inside the kernel-mode address space.
83 // If it is, then clear the address.
84 //
85 return ((Mode == KernelMode) ? DrAddress :
86 (DrAddress <= MM_HIGHEST_USER_ADDRESS) ? DrAddress : 0);
87 }
88
89 //
90 // Enters a Guarded Region
91 //
92 #define KeEnterGuardedRegion() \
93 { \
94 PKTHREAD _Thread = KeGetCurrentThread(); \
95 \
96 /* Sanity checks */ \
97 ASSERT_IRQL_LESS_OR_EQUAL(APC_LEVEL); \
98 ASSERT(_Thread == KeGetCurrentThread()); \
99 ASSERT((_Thread->SpecialApcDisable <= 0) && \
100 (_Thread->SpecialApcDisable != -32768)); \
101 \
102 /* Disable Special APCs */ \
103 _Thread->SpecialApcDisable--; \
104 }
105
106 //
107 // Leaves a Guarded Region
108 //
109 #define KeLeaveGuardedRegion() \
110 { \
111 PKTHREAD _Thread = KeGetCurrentThread(); \
112 \
113 /* Sanity checks */ \
114 ASSERT_IRQL_LESS_OR_EQUAL(APC_LEVEL); \
115 ASSERT(_Thread == KeGetCurrentThread()); \
116 ASSERT(_Thread->SpecialApcDisable < 0); \
117 \
118 /* Leave region and check if APCs are OK now */ \
119 if (!(++_Thread->SpecialApcDisable)) \
120 { \
121 /* Check for Kernel APCs on the list */ \
122 if (!IsListEmpty(&_Thread->ApcState. \
123 ApcListHead[KernelMode])) \
124 { \
125 /* Check for APC Delivery */ \
126 KiCheckForKernelApcDelivery(); \
127 } \
128 } \
129 }
130
131 //
132 // TODO: Guarded Mutex Routines
133 //
134
135 //
136 // Enters a Critical Region
137 //
138 #define KeEnterCriticalRegion() \
139 { \
140 PKTHREAD _Thread = KeGetCurrentThread(); \
141 \
142 /* Sanity checks */ \
143 ASSERT(_Thread == KeGetCurrentThread()); \
144 ASSERT((_Thread->KernelApcDisable <= 0) && \
145 (_Thread->KernelApcDisable != -32768)); \
146 \
147 /* Disable Kernel APCs */ \
148 _Thread->KernelApcDisable--; \
149 }
150
151 //
152 // Leaves a Critical Region
153 //
154 #define KeLeaveCriticalRegion() \
155 { \
156 PKTHREAD _Thread = KeGetCurrentThread(); \
157 \
158 /* Sanity checks */ \
159 ASSERT(_Thread == KeGetCurrentThread()); \
160 ASSERT(_Thread->KernelApcDisable < 0); \
161 \
162 /* Enable Kernel APCs */ \
163 _Thread->KernelApcDisable++; \
164 \
165 /* Check if Kernel APCs are now enabled */ \
166 if (!(_Thread->KernelApcDisable)) \
167 { \
168 /* Check if we need to request an APC Delivery */ \
169 if (!(IsListEmpty(&_Thread->ApcState.ApcListHead[KernelMode])) && \
170 !(_Thread->SpecialApcDisable)) \
171 { \
172 /* Check for the right environment */ \
173 KiCheckForKernelApcDelivery(); \
174 } \
175 } \
176 }
177
178 //
179 // Satisfies the wait of any dispatcher object
180 //
181 #define KiSatisfyObjectWait(Object, Thread) \
182 { \
183 /* Special case for Mutants */ \
184 if ((Object)->Header.Type == MutantObject) \
185 { \
186 /* Decrease the Signal State */ \
187 (Object)->Header.SignalState--; \
188 \
189 /* Check if it's now non-signaled */ \
190 if (!(Object)->Header.SignalState) \
191 { \
192 /* Set the Owner Thread */ \
193 (Object)->OwnerThread = Thread; \
194 \
195 /* Disable APCs if needed */ \
196 Thread->KernelApcDisable -= (Object)->ApcDisable; \
197 \
198 /* Check if it's abandoned */ \
199 if ((Object)->Abandoned) \
200 { \
201 /* Unabandon it */ \
202 (Object)->Abandoned = FALSE; \
203 \
204 /* Return Status */ \
205 Thread->WaitStatus = STATUS_ABANDONED; \
206 } \
207 \
208 /* Insert it into the Mutant List */ \
209 InsertHeadList(Thread->MutantListHead.Blink, \
210 &(Object)->MutantListEntry); \
211 } \
212 } \
213 else if (((Object)->Header.Type & TIMER_OR_EVENT_TYPE) == \
214 EventSynchronizationObject) \
215 { \
216 /* Synchronization Timers and Events just get un-signaled */ \
217 (Object)->Header.SignalState = 0; \
218 } \
219 else if ((Object)->Header.Type == SemaphoreObject) \
220 { \
221 /* These ones can have multiple states, so we only decrease it */ \
222 (Object)->Header.SignalState--; \
223 } \
224 }
225
226 //
227 // Satisfies the wait of a mutant dispatcher object
228 //
229 #define KiSatisfyMutantWait(Object, Thread) \
230 { \
231 /* Decrease the Signal State */ \
232 (Object)->Header.SignalState--; \
233 \
234 /* Check if it's now non-signaled */ \
235 if (!(Object)->Header.SignalState) \
236 { \
237 /* Set the Owner Thread */ \
238 (Object)->OwnerThread = Thread; \
239 \
240 /* Disable APCs if needed */ \
241 Thread->KernelApcDisable -= (Object)->ApcDisable; \
242 \
243 /* Check if it's abandoned */ \
244 if ((Object)->Abandoned) \
245 { \
246 /* Unabandon it */ \
247 (Object)->Abandoned = FALSE; \
248 \
249 /* Return Status */ \
250 Thread->WaitStatus = STATUS_ABANDONED; \
251 } \
252 \
253 /* Insert it into the Mutant List */ \
254 InsertHeadList(Thread->MutantListHead.Blink, \
255 &(Object)->MutantListEntry); \
256 } \
257 }
258
259 //
260 // Satisfies the wait of any nonmutant dispatcher object
261 //
262 #define KiSatisfyNonMutantWait(Object, Thread) \
263 { \
264 if (((Object)->Header.Type & TIMER_OR_EVENT_TYPE) == \
265 EventSynchronizationObject) \
266 { \
267 /* Synchronization Timers and Events just get un-signaled */ \
268 (Object)->Header.SignalState = 0; \
269 } \
270 else if ((Object)->Header.Type == SemaphoreObject) \
271 { \
272 /* These ones can have multiple states, so we only decrease it */ \
273 (Object)->Header.SignalState--; \
274 } \
275 }
276
277 //
278 // Recalculates the due time
279 //
280 PLARGE_INTEGER
281 FORCEINLINE
282 KiRecalculateDueTime(IN PLARGE_INTEGER OriginalDueTime,
283 IN PLARGE_INTEGER DueTime,
284 IN OUT PLARGE_INTEGER NewDueTime)
285 {
286 /* Don't do anything for absolute waits */
287 if (OriginalDueTime->QuadPart >= 0) return OriginalDueTime;
288
289 /* Otherwise, query the interrupt time and recalculate */
290 NewDueTime->QuadPart = KeQueryInterruptTime();
291 NewDueTime->QuadPart -= DueTime->QuadPart;
292 return NewDueTime;
293 }
294
295 //
296 // Determines wether a thread should be added to the wait list
297 //
298 FORCEINLINE
299 BOOLEAN
300 KiCheckThreadStackSwap(IN PKTHREAD Thread,
301 IN KPROCESSOR_MODE WaitMode)
302 {
303 /* Check the required conditions */
304 if ((WaitMode != KernelMode) &&
305 (Thread->EnableStackSwap) &&
306 (Thread->Priority >= (LOW_REALTIME_PRIORITY + 9)))
307 {
308 /* We are go for swap */
309 return TRUE;
310 }
311 else
312 {
313 /* Don't swap the thread */
314 return FALSE;
315 }
316 }
317
318 //
319 // Adds a thread to the wait list
320 //
321 #define KiAddThreadToWaitList(Thread, Swappable) \
322 { \
323 /* Make sure it's swappable */ \
324 if (Swappable) \
325 { \
326 /* Insert it into the PRCB's List */ \
327 InsertTailList(&KeGetCurrentPrcb()->WaitListHead, \
328 &Thread->WaitListEntry); \
329 } \
330 }
331
332 //
333 // Checks if a wait in progress should be interrupted by APCs or an alertable
334 // state.
335 //
336 FORCEINLINE
337 NTSTATUS
338 KiCheckAlertability(IN PKTHREAD Thread,
339 IN BOOLEAN Alertable,
340 IN KPROCESSOR_MODE WaitMode)
341 {
342 /* Check if the wait is alertable */
343 if (Alertable)
344 {
345 /* It is, first check if the thread is alerted in this mode */
346 if (Thread->Alerted[WaitMode])
347 {
348 /* It is, so bail out of the wait */
349 Thread->Alerted[WaitMode] = FALSE;
350 return STATUS_ALERTED;
351 }
352 else if ((WaitMode != KernelMode) &&
353 (!IsListEmpty(&Thread->ApcState.ApcListHead[UserMode])))
354 {
355 /* It's isn't, but this is a user wait with queued user APCs */
356 Thread->ApcState.UserApcPending = TRUE;
357 return STATUS_USER_APC;
358 }
359 else if (Thread->Alerted[KernelMode])
360 {
361 /* It isn't that either, but we're alered in kernel mode */
362 Thread->Alerted[KernelMode] = FALSE;
363 return STATUS_ALERTED;
364 }
365 }
366 else if ((WaitMode != KernelMode) && (Thread->ApcState.UserApcPending))
367 {
368 /* Not alertable, but this is a user wait with pending user APCs */
369 return STATUS_USER_APC;
370 }
371
372 /* Otherwise, we're fine */
373 return STATUS_WAIT_0;
374 }
375
376 VOID
377 FORCEINLINE
378 KxSetTimerForThreadWait(IN PKTIMER Timer,
379 IN LARGE_INTEGER Interval)
380 {
381 LARGE_INTEGER InterruptTime, SystemTime, TimeDifference;
382
383 /* Check the timer's interval to see if it's absolute */
384 Timer->Header.Absolute = FALSE;
385 if (!Timer->Period) Timer->Header.SignalState = FALSE;
386 if (Interval.HighPart >= 0)
387 {
388 /* Get the system time and calculate the relative time */
389 KeQuerySystemTime(&SystemTime);
390 TimeDifference.QuadPart = SystemTime.QuadPart - Interval.QuadPart;
391 Timer->Header.Absolute = TRUE;
392
393 /* Check if we've already expired */
394 if (TimeDifference.HighPart >= 0)
395 {
396 /* Reset everything */
397 Timer->DueTime.QuadPart = 0;
398 Timer->Header.SignalState = TRUE;
399 return;
400 }
401 else
402 {
403 /* Update the interval */
404 Interval = TimeDifference;
405 }
406 }
407
408 /* Calculate the due time */
409 InterruptTime.QuadPart = KeQueryInterruptTime();
410 Timer->DueTime.QuadPart = InterruptTime.QuadPart - Interval.QuadPart;
411 }
412
413 #define KxDelayThreadWait() \
414 \
415 /* Setup the Wait Block */ \
416 Thread->WaitBlockList = TimerBlock; \
417 \
418 /* Setup the timer */ \
419 KxSetTimerForThreadWait(Timer, *Interval); \
420 \
421 /* Save the due time for the caller */ \
422 DueTime.QuadPart = Timer->DueTime.QuadPart; \
423 \
424 /* Link the timer to this Wait Block */ \
425 TimerBlock->NextWaitBlock = TimerBlock; \
426 Timer->Header.WaitListHead.Flink = &TimerBlock->WaitListEntry; \
427 Timer->Header.WaitListHead.Blink = &TimerBlock->WaitListEntry; \
428 \
429 /* Clear wait status */ \
430 Thread->WaitStatus = STATUS_SUCCESS; \
431 \
432 /* Setup wait fields */ \
433 Thread->Alertable = Alertable; \
434 Thread->WaitReason = DelayExecution; \
435 Thread->WaitMode = WaitMode; \
436 \
437 /* Check if we can swap the thread's stack */ \
438 Thread->WaitListEntry.Flink = NULL; \
439 Swappable = KiCheckThreadStackSwap(Thread, WaitMode); \
440 \
441 /* Set the wait time */ \
442 Thread->WaitTime = KeTickCount.LowPart;
443
444 #define KxMultiThreadWait() \
445 /* Link wait block array to the thread */ \
446 Thread->WaitBlockList = WaitBlockArray; \
447 \
448 /* Reset the index */ \
449 Index = 0; \
450 \
451 /* Loop wait blocks */ \
452 do \
453 { \
454 /* Fill out the wait block */ \
455 WaitBlock = &WaitBlockArray[Index]; \
456 WaitBlock->Object = Object[Index]; \
457 WaitBlock->WaitKey = Index; \
458 WaitBlock->WaitType = WaitType; \
459 WaitBlock->Thread = Thread; \
460 \
461 /* Link to next block */ \
462 WaitBlock->NextWaitBlock = &WaitBlockArray[Index + 1]; \
463 Index++; \
464 } while (Index < Count); \
465 \
466 /* Link the last block */ \
467 WaitBlock->NextWaitBlock = WaitBlockArray; \
468 \
469 /* Set default wait status */ \
470 Thread->WaitStatus = STATUS_WAIT_0; \
471 \
472 /* Check if we have a timer */ \
473 if (Timeout) \
474 { \
475 /* Link to the block */ \
476 TimerBlock->NextWaitBlock = WaitBlockArray; \
477 \
478 /* Setup the timer */ \
479 KxSetTimerForThreadWait(Timer, *Timeout); \
480 \
481 /* Save the due time for the caller */ \
482 DueTime.QuadPart = Timer->DueTime.QuadPart; \
483 \
484 /* Initialize the list */ \
485 InitializeListHead(&Timer->Header.WaitListHead); \
486 } \
487 \
488 /* Set wait settings */ \
489 Thread->Alertable = Alertable; \
490 Thread->WaitMode = WaitMode; \
491 Thread->WaitReason = WaitReason; \
492 \
493 /* Check if we can swap the thread's stack */ \
494 Thread->WaitListEntry.Flink = NULL; \
495 Swappable = KiCheckThreadStackSwap(Thread, WaitMode); \
496 \
497 /* Set the wait time */ \
498 Thread->WaitTime = KeTickCount.LowPart;
499
500 #define KxSingleThreadWait() \
501 /* Setup the Wait Block */ \
502 Thread->WaitBlockList = WaitBlock; \
503 WaitBlock->WaitKey = STATUS_SUCCESS; \
504 WaitBlock->Object = Object; \
505 WaitBlock->WaitType = WaitAny; \
506 \
507 /* Clear wait status */ \
508 Thread->WaitStatus = STATUS_SUCCESS; \
509 \
510 /* Check if we have a timer */ \
511 if (Timeout) \
512 { \
513 /* Setup the timer */ \
514 KxSetTimerForThreadWait(Timer, *Timeout); \
515 \
516 /* Save the due time for the caller */ \
517 DueTime.QuadPart = Timer->DueTime.QuadPart; \
518 \
519 /* Pointer to timer block */ \
520 WaitBlock->NextWaitBlock = TimerBlock; \
521 TimerBlock->NextWaitBlock = WaitBlock; \
522 \
523 /* Link the timer to this Wait Block */ \
524 Timer->Header.WaitListHead.Flink = &TimerBlock->WaitListEntry; \
525 Timer->Header.WaitListHead.Blink = &TimerBlock->WaitListEntry; \
526 } \
527 else \
528 { \
529 /* No timer block, just ourselves */ \
530 WaitBlock->NextWaitBlock = WaitBlock; \
531 } \
532 \
533 /* Set wait settings */ \
534 Thread->Alertable = Alertable; \
535 Thread->WaitMode = WaitMode; \
536 Thread->WaitReason = WaitReason; \
537 \
538 /* Check if we can swap the thread's stack */ \
539 Thread->WaitListEntry.Flink = NULL; \
540 Swappable = KiCheckThreadStackSwap(Thread, WaitMode); \
541 \
542 /* Set the wait time */ \
543 Thread->WaitTime = KeTickCount.LowPart;
544
545 #define KxQueueThreadWait() \
546 /* Setup the Wait Block */ \
547 Thread->WaitBlockList = WaitBlock; \
548 WaitBlock->WaitKey = STATUS_SUCCESS; \
549 WaitBlock->Object = Queue; \
550 WaitBlock->WaitType = WaitAny; \
551 WaitBlock->Thread = Thread; \
552 \
553 /* Clear wait status */ \
554 Thread->WaitStatus = STATUS_SUCCESS; \
555 \
556 /* Check if we have a timer */ \
557 if (Timeout) \
558 { \
559 /* Setup the timer */ \
560 KxSetTimerForThreadWait(Timer, *Timeout); \
561 \
562 /* Save the due time for the caller */ \
563 DueTime.QuadPart = Timer->DueTime.QuadPart; \
564 \
565 /* Pointer to timer block */ \
566 WaitBlock->NextWaitBlock = TimerBlock; \
567 TimerBlock->NextWaitBlock = WaitBlock; \
568 \
569 /* Link the timer to this Wait Block */ \
570 Timer->Header.WaitListHead.Flink = &TimerBlock->WaitListEntry; \
571 Timer->Header.WaitListHead.Blink = &TimerBlock->WaitListEntry; \
572 } \
573 else \
574 { \
575 /* No timer block, just ourselves */ \
576 WaitBlock->NextWaitBlock = WaitBlock; \
577 } \
578 \
579 /* Set wait settings */ \
580 Thread->Alertable = FALSE; \
581 Thread->WaitMode = WaitMode; \
582 Thread->WaitReason = WrQueue; \
583 \
584 /* Check if we can swap the thread's stack */ \
585 Thread->WaitListEntry.Flink = NULL; \
586 Swappable = KiCheckThreadStackSwap(Thread, WaitMode); \
587 \
588 /* Set the wait time */ \
589 Thread->WaitTime = KeTickCount.LowPart;
590
591 //
592 // Unwaits a Thread
593 //
594 FORCEINLINE
595 VOID
596 KxUnwaitThread(IN DISPATCHER_HEADER *Object,
597 IN KPRIORITY Increment)
598 {
599 PLIST_ENTRY WaitEntry, WaitList;
600 PKWAIT_BLOCK WaitBlock;
601 PKTHREAD WaitThread;
602 ULONG WaitKey;
603
604 /* Loop the Wait Entries */
605 WaitList = &Object->WaitListHead;
606 ASSERT(IsListEmpty(&Object->WaitListHead) == FALSE);
607 WaitEntry = WaitList->Flink;
608 do
609 {
610 /* Get the current wait block */
611 WaitBlock = CONTAINING_RECORD(WaitEntry, KWAIT_BLOCK, WaitListEntry);
612
613 /* Get the waiting thread */
614 WaitThread = WaitBlock->Thread;
615
616 /* Check the current Wait Mode */
617 if (WaitBlock->WaitType == WaitAny)
618 {
619 /* Use the actual wait key */
620 WaitKey = WaitBlock->WaitKey;
621 }
622 else
623 {
624 /* Otherwise, use STATUS_KERNEL_APC */
625 WaitKey = STATUS_KERNEL_APC;
626 }
627
628 /* Unwait the thread */
629 KiUnwaitThread(WaitThread, WaitKey, Increment);
630
631 /* Next entry */
632 WaitEntry = WaitList->Flink;
633 } while (WaitEntry != WaitList);
634 }
635
636 //
637 // Unwaits a Thread waiting on an event
638 //
639 FORCEINLINE
640 VOID
641 KxUnwaitThreadForEvent(IN PKEVENT Event,
642 IN KPRIORITY Increment)
643 {
644 PLIST_ENTRY WaitEntry, WaitList;
645 PKWAIT_BLOCK WaitBlock;
646 PKTHREAD WaitThread;
647
648 /* Loop the Wait Entries */
649 WaitList = &Event->Header.WaitListHead;
650 ASSERT(IsListEmpty(&Event->Header.WaitListHead) == FALSE);
651 WaitEntry = WaitList->Flink;
652 do
653 {
654 /* Get the current wait block */
655 WaitBlock = CONTAINING_RECORD(WaitEntry, KWAIT_BLOCK, WaitListEntry);
656
657 /* Get the waiting thread */
658 WaitThread = WaitBlock->Thread;
659
660 /* Check the current Wait Mode */
661 if (WaitBlock->WaitType == WaitAny)
662 {
663 /* Un-signal it */
664 Event->Header.SignalState = 0;
665
666 /* Un-signal the event and unwait the thread */
667 KiUnwaitThread(WaitThread, WaitBlock->WaitKey, Increment);
668 break;
669 }
670
671 /* Unwait the thread with STATUS_KERNEL_APC */
672 KiUnwaitThread(WaitThread, STATUS_KERNEL_APC, Increment);
673
674 /* Next entry */
675 WaitEntry = WaitList->Flink;
676 } while (WaitEntry != WaitList);
677 }
678
679 #ifndef _CONFIG_SMP
680 //
681 // Spinlock Acquire at IRQL >= DISPATCH_LEVEL
682 //
683 FORCEINLINE
684 VOID
685 KxAcquireSpinLock(IN PKSPIN_LOCK SpinLock)
686 {
687 /* On UP builds, spinlocks don't exist at IRQL >= DISPATCH */
688 UNREFERENCED_PARAMETER(SpinLock);
689 }
690
691 //
692 // Spinlock Release at IRQL >= DISPATCH_LEVEL
693 //
694 FORCEINLINE
695 VOID
696 KxReleaseSpinLock(IN PKSPIN_LOCK SpinLock)
697 {
698 /* On UP builds, spinlocks don't exist at IRQL >= DISPATCH */
699 UNREFERENCED_PARAMETER(SpinLock);
700 }
701
702 //
703 // This routine protects against multiple CPU acquires, it's meaningless on UP.
704 //
705 VOID
706 FORCEINLINE
707 KiAcquireDispatcherObject(IN DISPATCHER_HEADER* Object)
708 {
709 UNREFERENCED_PARAMETER(Object);
710 }
711
712 //
713 // This routine protects against multiple CPU acquires, it's meaningless on UP.
714 //
715 VOID
716 FORCEINLINE
717 KiReleaseDispatcherObject(IN DISPATCHER_HEADER* Object)
718 {
719 UNREFERENCED_PARAMETER(Object);
720 }
721
722 KIRQL
723 FORCEINLINE
724 KiAcquireDispatcherLock(VOID)
725 {
726 /* Raise to DPC level */
727 return KeRaiseIrqlToDpcLevel();
728 }
729
730 VOID
731 FORCEINLINE
732 KiReleaseDispatcherLock(IN KIRQL OldIrql)
733 {
734 /* Just exit the dispatcher */
735 KiExitDispatcher(OldIrql);
736 }
737
738 VOID
739 FORCEINLINE
740 KiAcquireDispatcherLockAtDpcLevel(VOID)
741 {
742 /* This is a no-op at DPC Level for UP systems */
743 return;
744 }
745
746 VOID
747 FORCEINLINE
748 KiReleaseDispatcherLockFromDpcLevel(VOID)
749 {
750 /* This is a no-op at DPC Level for UP systems */
751 return;
752 }
753
754 //
755 // This routine makes the thread deferred ready on the boot CPU.
756 //
757 FORCEINLINE
758 VOID
759 KiInsertDeferredReadyList(IN PKTHREAD Thread)
760 {
761 /* Set the thread to deferred state and boot CPU */
762 Thread->State = DeferredReady;
763 Thread->DeferredProcessor = 0;
764
765 /* Make the thread ready immediately */
766 KiDeferredReadyThread(Thread);
767 }
768
769 FORCEINLINE
770 VOID
771 KiRescheduleThread(IN BOOLEAN NewThread,
772 IN ULONG Cpu)
773 {
774 /* This is meaningless on UP systems */
775 UNREFERENCED_PARAMETER(NewThread);
776 UNREFERENCED_PARAMETER(Cpu);
777 }
778
779 //
780 // This routine protects against multiple CPU acquires, it's meaningless on UP.
781 //
782 FORCEINLINE
783 VOID
784 KiSetThreadSwapBusy(IN PKTHREAD Thread)
785 {
786 UNREFERENCED_PARAMETER(Thread);
787 }
788
789 //
790 // This routine protects against multiple CPU acquires, it's meaningless on UP.
791 //
792 FORCEINLINE
793 VOID
794 KiAcquirePrcbLock(IN PKPRCB Prcb)
795 {
796 UNREFERENCED_PARAMETER(Prcb);
797 }
798
799 //
800 // This routine protects against multiple CPU acquires, it's meaningless on UP.
801 //
802 FORCEINLINE
803 VOID
804 KiReleasePrcbLock(IN PKPRCB Prcb)
805 {
806 UNREFERENCED_PARAMETER(Prcb);
807 }
808
809 //
810 // This routine protects against multiple CPU acquires, it's meaningless on UP.
811 //
812 FORCEINLINE
813 VOID
814 KiAcquireThreadLock(IN PKTHREAD Thread)
815 {
816 UNREFERENCED_PARAMETER(Thread);
817 }
818
819 //
820 // This routine protects against multiple CPU acquires, it's meaningless on UP.
821 //
822 FORCEINLINE
823 VOID
824 KiReleaseThreadLock(IN PKTHREAD Thread)
825 {
826 UNREFERENCED_PARAMETER(Thread);
827 }
828
829 //
830 // This routine protects against multiple CPU acquires, it's meaningless on UP.
831 //
832 FORCEINLINE
833 BOOLEAN
834 KiTryThreadLock(IN PKTHREAD Thread)
835 {
836 UNREFERENCED_PARAMETER(Thread);
837 return FALSE;
838 }
839
840 FORCEINLINE
841 VOID
842 KiCheckDeferredReadyList(IN PKPRCB Prcb)
843 {
844 /* There are no deferred ready lists on UP systems */
845 UNREFERENCED_PARAMETER(Prcb);
846 }
847
848 FORCEINLINE
849 VOID
850 KiRundownThread(IN PKTHREAD Thread)
851 {
852 /* Check if this is the NPX Thread */
853 if (KeGetCurrentPrcb()->NpxThread == Thread)
854 {
855 /* Clear it */
856 KeGetCurrentPrcb()->NpxThread = NULL;
857 Ke386FnInit();
858 }
859 }
860
861 FORCEINLINE
862 VOID
863 KiRequestApcInterrupt(IN BOOLEAN NeedApc,
864 IN UCHAR Processor)
865 {
866 /* We deliver instantly on UP */
867 UNREFERENCED_PARAMETER(NeedApc);
868 UNREFERENCED_PARAMETER(Processor);
869 }
870
871 FORCEINLINE
872 PKSPIN_LOCK_QUEUE
873 KiAcquireTimerLock(IN ULONG Hand)
874 {
875 ASSERT(KeGetCurrentIrql() >= DISPATCH_LEVEL);
876
877 /* Nothing to do on UP */
878 UNREFERENCED_PARAMETER(Hand);
879 return NULL;
880 }
881
882 FORCEINLINE
883 VOID
884 KiReleaseTimerLock(IN PKSPIN_LOCK_QUEUE LockQueue)
885 {
886 ASSERT(KeGetCurrentIrql() >= DISPATCH_LEVEL);
887
888 /* Nothing to do on UP */
889 UNREFERENCED_PARAMETER(LockQueue);
890 }
891
892 #else
893
894 //
895 // Spinlock Acquisition at IRQL >= DISPATCH_LEVEL
896 //
897 FORCEINLINE
898 VOID
899 KxAcquireSpinLock(IN PKSPIN_LOCK SpinLock)
900 {
901 for (;;)
902 {
903 /* Try to acquire it */
904 if (InterlockedBitTestAndSet((PLONG)SpinLock, 0))
905 {
906 /* Value changed... wait until it's locked */
907 while (*(volatile KSPIN_LOCK *)SpinLock == 1)
908 {
909 #ifdef DBG
910 /* On debug builds, we use a much slower but useful routine */
911 Kii386SpinOnSpinLock(SpinLock, 5);
912 #else
913 /* Otherwise, just yield and keep looping */
914 YieldProcessor();
915 #endif
916 }
917 }
918 else
919 {
920 #ifdef DBG
921 /* On debug builds, we OR in the KTHREAD */
922 *SpinLock = KeGetCurrentThread() | 1;
923 #endif
924 /* All is well, break out */
925 break;
926 }
927 }
928 }
929
930 //
931 // Spinlock Release at IRQL >= DISPATCH_LEVEL
932 //
933 FORCEINLINE
934 VOID
935 KxReleaseSpinLock(IN PKSPIN_LOCK SpinLock)
936 {
937 #ifdef DBG
938 /* Make sure that the threads match */
939 if ((KeGetCurrentThread() | 1) != *SpinLock)
940 {
941 /* They don't, bugcheck */
942 KeBugCheckEx(SPIN_LOCK_NOT_OWNED, SpinLock, 0, 0, 0);
943 }
944 #endif
945 /* Clear the lock */
946 InterlockedAnd(SpinLock, 0);
947 }
948
949 KIRQL
950 FORCEINLINE
951 KiAcquireDispatcherObject(IN DISPATCHER_HEADER* Object)
952 {
953 LONG OldValue, NewValue;
954
955 /* Make sure we're at a safe level to touch the lock */
956 ASSERT(KeGetCurrentIrql() >= DISPATCH_LEVEL);
957
958 /* Start acquire loop */
959 do
960 {
961 /* Loop until the other CPU releases it */
962 while ((UCHAR)Object->Lock & KOBJECT_LOCK_BIT)
963 {
964 /* Let the CPU know that this is a loop */
965 YieldProcessor();
966 };
967
968 /* Try acquiring the lock now */
969 NewValue = InterlockedCompareExchange(&Object->Lock,
970 OldValue | KOBJECT_LOCK_BIT,
971 OldValue);
972 } while (NewValue != OldValue);
973 }
974
975 KIRQL
976 FORCEINLINE
977 KiReleaseDispatcherObject(IN DISPATCHER_HEADER* Object)
978 {
979 /* Make sure we're at a safe level to touch the lock */
980 ASSERT(KeGetCurrentIrql() >= DISPATCH_LEVEL);
981
982 /* Release it */
983 InterlockedAnd(&Object->Lock, ~KOBJECT_LOCK_BIT);
984 }
985
986 KIRQL
987 FORCEINLINE
988 KiAcquireDispatcherLock(VOID)
989 {
990 /* Raise to synchronization level and acquire the dispatcher lock */
991 return KeAcquireQueuedSpinLockRaiseToSynch(LockQueueDispatcherLock);
992 }
993
994 VOID
995 FORCEINLINE
996 KiReleaseDispatcherLock(IN KIRQL OldIrql)
997 {
998 /* First release the lock */
999 KeReleaseQueuedSpinLockFromDpcLevel(&KeGetCurrentPrcb()->
1000 LockQueue[LockQueueDispatcherLock]);
1001
1002 /* Then exit the dispatcher */
1003 KiExitDispatcher(OldIrql);
1004 }
1005
1006 //
1007 // This routine inserts a thread into the deferred ready list of the given CPU
1008 //
1009 FORCEINLINE
1010 VOID
1011 KiInsertDeferredReadyList(IN PKTHREAD Thread)
1012 {
1013 PKPRCB Prcb = KeGetCurrentPrcb();
1014
1015 /* Set the thread to deferred state and CPU */
1016 Thread->State = DeferredReady;
1017 Thread->DeferredProcessor = Prcb->Number;
1018
1019 /* Add it on the list */
1020 PushEntryList(&Prcb->DeferredReadyListHead, &Thread->SwapListEntry);
1021 }
1022
1023 FORCEINLINE
1024 VOID
1025 KiRescheduleThread(IN BOOLEAN NewThread,
1026 IN ULONG Cpu)
1027 {
1028 /* Check if a new thread needs to be scheduled on a different CPU */
1029 if ((NewThread) && !(KeGetPcr()->Number == Cpu))
1030 {
1031 /* Send an IPI to request delivery */
1032 KiIpiSendRequest(AFFINITY_MASK(Cpu), IPI_DPC);
1033 }
1034 }
1035
1036 //
1037 // This routine sets the current thread in a swap busy state, which ensure that
1038 // nobody else tries to swap it concurrently.
1039 //
1040 FORCEINLINE
1041 VOID
1042 KiSetThreadSwapBusy(IN PKTHREAD Thread)
1043 {
1044 /* Make sure nobody already set it */
1045 ASSERT(Thread->SwapBusy == FALSE);
1046
1047 /* Set it ourselves */
1048 Thread->SwapBusy = TRUE;
1049 }
1050
1051 //
1052 // This routine acquires the PRCB lock so that only one caller can touch
1053 // volatile PRCB data.
1054 //
1055 // Since this is a simple optimized spin-lock, it must be be only acquired
1056 // at dispatcher level or higher!
1057 //
1058 FORCEINLINE
1059 VOID
1060 KiAcquirePrcbLock(IN PKPRCB Prcb)
1061 {
1062 /* Make sure we're at a safe level to touch the PRCB lock */
1063 ASSERT(KeGetCurrentIrql() >= DISPATCH_LEVEL);
1064
1065 /* Start acquire loop */
1066 for (;;)
1067 {
1068 /* Acquire the lock and break out if we acquired it first */
1069 if (!InterlockedExchange(&Prcb->PrcbLock, 1)) break;
1070
1071 /* Loop until the other CPU releases it */
1072 do
1073 {
1074 /* Let the CPU know that this is a loop */
1075 YieldProcessor();
1076 } while (Prcb->PrcbLock);
1077 }
1078 }
1079
1080 //
1081 // This routine releases the PRCB lock so that other callers can touch
1082 // volatile PRCB data.
1083 //
1084 // Since this is a simple optimized spin-lock, it must be be only acquired
1085 // at dispatcher level or higher!
1086 //
1087 FORCEINLINE
1088 VOID
1089 KiReleasePrcbLock(IN PKPRCB Prcb)
1090 {
1091 /* Make sure it's acquired! */
1092 ASSERT(Prcb->PrcbLock != 0);
1093
1094 /* Release it */
1095 InterlockedAnd(&Prcb->PrcbLock, 0);
1096 }
1097
1098 //
1099 // This routine acquires the thread lock so that only one caller can touch
1100 // volatile thread data.
1101 //
1102 // Since this is a simple optimized spin-lock, it must be be only acquired
1103 // at dispatcher level or higher!
1104 //
1105 FORCEINLINE
1106 VOID
1107 KiAcquireThreadLock(IN PKTHREAD Thread)
1108 {
1109 /* Make sure we're at a safe level to touch the thread lock */
1110 ASSERT(KeGetCurrentIrql() >= DISPATCH_LEVEL);
1111
1112 /* Start acquire loop */
1113 for (;;)
1114 {
1115 /* Acquire the lock and break out if we acquired it first */
1116 if (!InterlockedExchange(&Thread->ThreadLock, 1)) break;
1117
1118 /* Loop until the other CPU releases it */
1119 do
1120 {
1121 /* Let the CPU know that this is a loop */
1122 YieldProcessor();
1123 } while (Thread->ThreadLock);
1124 }
1125 }
1126
1127 //
1128 // This routine releases the thread lock so that other callers can touch
1129 // volatile thread data.
1130 //
1131 // Since this is a simple optimized spin-lock, it must be be only acquired
1132 // at dispatcher level or higher!
1133 //
1134 FORCEINLINE
1135 VOID
1136 KiReleaseThreadLock(IN PKTHREAD Thread)
1137 {
1138 /* Release it */
1139 InterlockedAnd(&Thread->ThreadLock, 0);
1140 }
1141
1142 FORCEINLINE
1143 BOOLEAN
1144 KiTryThreadLock(IN PKTHREAD Thread)
1145 {
1146 LONG Value;
1147
1148 /* If the lock isn't acquired, return false */
1149 if (!Thread->ThreadLock) return FALSE;
1150
1151 /* Otherwise, try to acquire it and check the result */
1152 Value = 1;
1153 Value = InterlockedExchange(&Thread->ThreadLock, &Value);
1154
1155 /* Return the lock state */
1156 return (Value == TRUE);
1157 }
1158
1159 FORCEINLINE
1160 VOID
1161 KiCheckDeferredReadyList(IN PKPRCB Prcb)
1162 {
1163 /* Scan the deferred ready lists if required */
1164 if (Prcb->DeferredReadyListHead.Next) KiProcessDeferredReadyList(Prcb);
1165 }
1166
1167 FORCEINLINE
1168 VOID
1169 KiRequestApcInterrupt(IN BOOLEAN NeedApc,
1170 IN UCHAR Processor)
1171 {
1172 /* Check if we need to request APC delivery */
1173 if (NeedApc)
1174 {
1175 /* Check if it's on another CPU */
1176 if (KeGetPcr()->Number != Cpu)
1177 {
1178 /* Send an IPI to request delivery */
1179 KiIpiSendRequest(AFFINITY_MASK(Cpu), IPI_DPC);
1180 }
1181 else
1182 {
1183 /* Request a software interrupt */
1184 HalRequestSoftwareInterrupt(APC_LEVEL);
1185 }
1186 }
1187 }
1188
1189 #endif
1190
1191 FORCEINLINE
1192 VOID
1193 KiAcquireApcLock(IN PKTHREAD Thread,
1194 IN PKLOCK_QUEUE_HANDLE Handle)
1195 {
1196 /* Acquire the lock and raise to synchronization level */
1197 KeAcquireInStackQueuedSpinLockRaiseToSynch(&Thread->ApcQueueLock, Handle);
1198 }
1199
1200 FORCEINLINE
1201 VOID
1202 KiAcquireApcLockAtDpcLevel(IN PKTHREAD Thread,
1203 IN PKLOCK_QUEUE_HANDLE Handle)
1204 {
1205 /* Acquire the lock */
1206 KeAcquireInStackQueuedSpinLockAtDpcLevel(&Thread->ApcQueueLock, Handle);
1207 }
1208
1209 FORCEINLINE
1210 VOID
1211 KiAcquireApcLockAtApcLevel(IN PKTHREAD Thread,
1212 IN PKLOCK_QUEUE_HANDLE Handle)
1213 {
1214 /* Acquire the lock */
1215 KeAcquireInStackQueuedSpinLock(&Thread->ApcQueueLock, Handle);
1216 }
1217
1218 FORCEINLINE
1219 VOID
1220 KiReleaseApcLock(IN PKLOCK_QUEUE_HANDLE Handle)
1221 {
1222 /* Release the lock */
1223 KeReleaseInStackQueuedSpinLock(Handle);
1224 }
1225
1226 FORCEINLINE
1227 VOID
1228 KiReleaseApcLockFromDpcLevel(IN PKLOCK_QUEUE_HANDLE Handle)
1229 {
1230 /* Release the lock */
1231 KeReleaseInStackQueuedSpinLockFromDpcLevel(Handle);
1232 }
1233
1234 FORCEINLINE
1235 VOID
1236 KiAcquireProcessLock(IN PKPROCESS Process,
1237 IN PKLOCK_QUEUE_HANDLE Handle)
1238 {
1239 /* Acquire the lock and raise to synchronization level */
1240 KeAcquireInStackQueuedSpinLockRaiseToSynch(&Process->ProcessLock, Handle);
1241 }
1242
1243 FORCEINLINE
1244 VOID
1245 KiReleaseProcessLock(IN PKLOCK_QUEUE_HANDLE Handle)
1246 {
1247 /* Release the lock */
1248 KeReleaseInStackQueuedSpinLock(Handle);
1249 }
1250
1251 FORCEINLINE
1252 VOID
1253 KiReleaseProcessLockFromDpcLevel(IN PKLOCK_QUEUE_HANDLE Handle)
1254 {
1255 /* Release the lock */
1256 KeReleaseInStackQueuedSpinLockFromDpcLevel(Handle);
1257 }
1258
1259 FORCEINLINE
1260 VOID
1261 KiAcquireDeviceQueueLock(IN PKDEVICE_QUEUE DeviceQueue,
1262 IN PKLOCK_QUEUE_HANDLE DeviceLock)
1263 {
1264 /* Check if we were called from a threaded DPC */
1265 if (KeGetCurrentPrcb()->DpcThreadActive)
1266 {
1267 /* Lock the Queue, we're not at DPC level */
1268 KeAcquireInStackQueuedSpinLock(&DeviceQueue->Lock, DeviceLock);
1269 }
1270 else
1271 {
1272 /* We must be at DPC level, acquire the lock safely */
1273 ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL);
1274 KeAcquireInStackQueuedSpinLockAtDpcLevel(&DeviceQueue->Lock,
1275 DeviceLock);
1276 }
1277 }
1278
1279 FORCEINLINE
1280 VOID
1281 KiReleaseDeviceQueueLock(IN PKLOCK_QUEUE_HANDLE DeviceLock)
1282 {
1283 /* Check if we were called from a threaded DPC */
1284 if (KeGetCurrentPrcb()->DpcThreadActive)
1285 {
1286 /* Unlock the Queue, we're not at DPC level */
1287 KeReleaseInStackQueuedSpinLock(DeviceLock);
1288 }
1289 else
1290 {
1291 /* We must be at DPC level, release the lock safely */
1292 ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL);
1293 KeReleaseInStackQueuedSpinLockFromDpcLevel(DeviceLock);
1294 }
1295 }
1296
1297 //
1298 // This routine queues a thread that is ready on the PRCB's ready lists.
1299 // If this thread cannot currently run on this CPU, then the thread is
1300 // added to the deferred ready list instead.
1301 //
1302 // This routine must be entered with the PRCB lock held and it will exit
1303 // with the PRCB lock released!
1304 //
1305 FORCEINLINE
1306 VOID
1307 KxQueueReadyThread(IN PKTHREAD Thread,
1308 IN PKPRCB Prcb)
1309 {
1310 BOOLEAN Preempted;
1311 KPRIORITY Priority;
1312
1313 /* Sanity checks */
1314 ASSERT(Prcb == KeGetCurrentPrcb());
1315 ASSERT(Thread->State == Running);
1316 ASSERT(Thread->NextProcessor == Prcb->Number);
1317
1318 /* Check if this thread is allowed to run in this CPU */
1319 #ifdef _CONFIG_SMP
1320 if ((Thread->Affinity) & (Prcb->SetMember))
1321 #else
1322 if (TRUE)
1323 #endif
1324 {
1325 /* Set thread ready for execution */
1326 Thread->State = Ready;
1327
1328 /* Save current priority and if someone had pre-empted it */
1329 Priority = Thread->Priority;
1330 Preempted = Thread->Preempted;
1331
1332 /* We're not pre-empting now, and set the wait time */
1333 Thread->Preempted = FALSE;
1334 Thread->WaitTime = KeTickCount.LowPart;
1335
1336 /* Sanity check */
1337 ASSERT((Priority >= 0) && (Priority <= HIGH_PRIORITY));
1338
1339 /* Insert this thread in the appropriate order */
1340 Preempted ? InsertHeadList(&Prcb->DispatcherReadyListHead[Priority],
1341 &Thread->WaitListEntry) :
1342 InsertTailList(&Prcb->DispatcherReadyListHead[Priority],
1343 &Thread->WaitListEntry);
1344
1345 /* Update the ready summary */
1346 Prcb->ReadySummary |= PRIORITY_MASK(Priority);
1347
1348 /* Sanity check */
1349 ASSERT(Priority == Thread->Priority);
1350
1351 /* Release the PRCB lock */
1352 KiReleasePrcbLock(Prcb);
1353 }
1354 else
1355 {
1356 /* Otherwise, prepare this thread to be deferred */
1357 Thread->State = DeferredReady;
1358 Thread->DeferredProcessor = Prcb->Number;
1359
1360 /* Release the lock and defer scheduling */
1361 KiReleasePrcbLock(Prcb);
1362 KiDeferredReadyThread(Thread);
1363 }
1364 }
1365
1366 //
1367 // This routine scans for an appropriate ready thread to select at the
1368 // given priority and for the given CPU.
1369 //
1370 FORCEINLINE
1371 PKTHREAD
1372 KiSelectReadyThread(IN KPRIORITY Priority,
1373 IN PKPRCB Prcb)
1374 {
1375 ULONG PrioritySet, HighPriority;
1376 PLIST_ENTRY ListEntry;
1377 PKTHREAD Thread = NULL;
1378
1379 /* Save the current mask and get the priority set for the CPU */
1380 PrioritySet = Prcb->ReadySummary >> Priority;
1381 if (!PrioritySet) goto Quickie;
1382
1383 /* Get the highest priority possible */
1384 BitScanReverse((PULONG)&HighPriority, PrioritySet);
1385 ASSERT((PrioritySet & PRIORITY_MASK(HighPriority)) != 0);
1386 HighPriority += Priority;
1387
1388 /* Make sure the list isn't empty at the highest priority */
1389 ASSERT(IsListEmpty(&Prcb->DispatcherReadyListHead[HighPriority]) == FALSE);
1390
1391 /* Get the first thread on the list */
1392 ListEntry = Prcb->DispatcherReadyListHead[HighPriority].Flink;
1393 Thread = CONTAINING_RECORD(ListEntry, KTHREAD, WaitListEntry);
1394
1395 /* Make sure this thread is here for a reason */
1396 ASSERT(HighPriority == Thread->Priority);
1397 ASSERT(Thread->Affinity & AFFINITY_MASK(Prcb->Number));
1398 ASSERT(Thread->NextProcessor == Prcb->Number);
1399
1400 /* Remove it from the list */
1401 if (RemoveEntryList(&Thread->WaitListEntry))
1402 {
1403 /* The list is empty now, reset the ready summary */
1404 Prcb->ReadySummary ^= PRIORITY_MASK(HighPriority);
1405 }
1406
1407 /* Sanity check and return the thread */
1408 Quickie:
1409 ASSERT((Thread == NULL) ||
1410 (Thread->BasePriority == 0) ||
1411 (Thread->Priority != 0));
1412 return Thread;
1413 }
1414
1415 //
1416 // This routine computes the new priority for a thread. It is only valid for
1417 // threads with priorities in the dynamic priority range.
1418 //
1419 SCHAR
1420 FORCEINLINE
1421 KiComputeNewPriority(IN PKTHREAD Thread,
1422 IN SCHAR Adjustment)
1423 {
1424 SCHAR Priority;
1425
1426 /* Priority sanity checks */
1427 ASSERT((Thread->PriorityDecrement >= 0) &&
1428 (Thread->PriorityDecrement <= Thread->Priority));
1429 ASSERT((Thread->Priority < LOW_REALTIME_PRIORITY) ?
1430 TRUE : (Thread->PriorityDecrement == 0));
1431
1432 /* Get the current priority */
1433 Priority = Thread->Priority;
1434 if (Priority < LOW_REALTIME_PRIORITY)
1435 {
1436 /* Decrease priority by the priority decrement */
1437 Priority -= (Thread->PriorityDecrement + Adjustment);
1438
1439 /* Don't go out of bounds */
1440 if (Priority < Thread->BasePriority) Priority = Thread->BasePriority;
1441
1442 /* Reset the priority decrement */
1443 Thread->PriorityDecrement = 0;
1444 }
1445
1446 /* Sanity check */
1447 ASSERT((Thread->BasePriority == 0) || (Priority != 0));
1448
1449 /* Return the new priority */
1450 return Priority;
1451 }
1452
1453 PRKTHREAD
1454 FORCEINLINE
1455 KeGetCurrentThread(VOID)
1456 {
1457 /* Return the current thread */
1458 return ((PKIPCR)KeGetPcr())->PrcbData.CurrentThread;
1459 }
1460
1461 UCHAR
1462 FORCEINLINE
1463 KeGetPreviousMode(VOID)
1464 {
1465 /* Return the current mode */
1466 return KeGetCurrentThread()->PreviousMode;
1467 }
1468
1469 VOID
1470 FORCEINLINE
1471 KiInsertWaitTimer(IN PKTIMER Timer)
1472 {
1473 /* Set default data */
1474 Timer->Header.Inserted = TRUE;
1475 if (!Timer->Period) Timer->Header.SignalState = FALSE;
1476
1477 /* Now insert it into the Timer List */
1478 InsertAscendingList(&KiTimerListHead,
1479 Timer,
1480 KTIMER,
1481 TimerListEntry,
1482 DueTime.QuadPart);
1483 }
A free Windows-compatible Operating System