[HEADERS]
[reactos.git] / reactos / ntoskrnl / include / internal / ex.h
1 #pragma once
2
3 /* GLOBAL VARIABLES *********************************************************/
4
5 extern TIME_ZONE_INFORMATION ExpTimeZoneInfo;
6 extern LARGE_INTEGER ExpTimeZoneBias;
7 extern ULONG ExpTimeZoneId;
8 extern ULONG ExpTickCountMultiplier;
9 extern ULONG ExpLastTimeZoneBias;
10 extern POBJECT_TYPE ExEventPairObjectType;
11 extern POBJECT_TYPE _ExEventObjectType, _ExSemaphoreObjectType;
12 extern ULONG NtBuildNumber;
13 extern ULONG NtMajorVersion;
14 extern ULONG NtMinorVersion;
15 extern FAST_MUTEX ExpEnvironmentLock;
16 extern ERESOURCE ExpFirmwareTableResource;
17 extern LIST_ENTRY ExpFirmwareTableProviderListHead;
18 extern BOOLEAN ExpIsWinPEMode;
19 extern LIST_ENTRY ExpSystemResourcesList;
20 extern ULONG ExpAnsiCodePageDataOffset, ExpOemCodePageDataOffset;
21 extern ULONG ExpUnicodeCaseTableDataOffset;
22 extern PVOID ExpNlsSectionPointer;
23 extern CHAR NtBuildLab[];
24 extern ULONG CmNtCSDVersion;
25 extern ULONG NtGlobalFlag;
26 extern ULONG ExpInitializationPhase;
27 extern ULONG ExpAltTimeZoneBias;
28 extern LIST_ENTRY ExSystemLookasideListHead;
29 extern PCALLBACK_OBJECT PowerStateCallback;
30
31 typedef struct _EXHANDLE
32 {
33 union
34 {
35 struct
36 {
37 ULONG TagBits:2;
38 ULONG Index:30;
39 };
40 HANDLE GenericHandleOverlay;
41 ULONG_PTR Value;
42 };
43 } EXHANDLE, *PEXHANDLE;
44
45 typedef struct _ETIMER
46 {
47 KTIMER KeTimer;
48 KAPC TimerApc;
49 KDPC TimerDpc;
50 LIST_ENTRY ActiveTimerListEntry;
51 KSPIN_LOCK Lock;
52 LONG Period;
53 BOOLEAN ApcAssociated;
54 BOOLEAN WakeTimer;
55 LIST_ENTRY WakeTimerListEntry;
56 } ETIMER, *PETIMER;
57
58 typedef struct
59 {
60 PCALLBACK_OBJECT *CallbackObject;
61 PWSTR Name;
62 } SYSTEM_CALLBACKS;
63
64 #define MAX_FAST_REFS 7
65
66 #define ExAcquireRundownProtection _ExAcquireRundownProtection
67 #define ExReleaseRundownProtection _ExReleaseRundownProtection
68 #define ExInitializeRundownProtection _ExInitializeRundownProtection
69 #define ExWaitForRundownProtectionRelease _ExWaitForRundownProtectionRelease
70 #define ExRundownCompleted _ExRundownCompleted
71 #define ExGetPreviousMode KeGetPreviousMode
72
73
74 //
75 // Various bits tagged on the handle or handle table
76 //
77 #define EXHANDLE_TABLE_ENTRY_LOCK_BIT 1
78 #define FREE_HANDLE_MASK -1
79
80 //
81 // Number of entries in each table level
82 //
83 #define LOW_LEVEL_ENTRIES (PAGE_SIZE / sizeof(HANDLE_TABLE_ENTRY))
84 #define MID_LEVEL_ENTRIES (PAGE_SIZE / sizeof(PHANDLE_TABLE_ENTRY))
85 #define HIGH_LEVEL_ENTRIES (16777216 / (LOW_LEVEL_ENTRIES * MID_LEVEL_ENTRIES))
86
87 //
88 // Maximum index in each table level before we need another table
89 //
90 #define MAX_LOW_INDEX LOW_LEVEL_ENTRIES
91 #define MAX_MID_INDEX (MID_LEVEL_ENTRIES * LOW_LEVEL_ENTRIES)
92 #define MAX_HIGH_INDEX (MID_LEVEL_ENTRIES * MID_LEVEL_ENTRIES * LOW_LEVEL_ENTRIES)
93
94 //
95 // Detect old GCC
96 //
97 #if (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__ < 40300) || \
98 (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__ == 40303)
99
100 #define DEFINE_WAIT_BLOCK(x) \
101 struct _AlignHack \
102 { \
103 UCHAR Hack[15]; \
104 EX_PUSH_LOCK_WAIT_BLOCK UnalignedBlock; \
105 } WaitBlockBuffer; \
106 PEX_PUSH_LOCK_WAIT_BLOCK x = (PEX_PUSH_LOCK_WAIT_BLOCK) \
107 ((ULONG_PTR)&WaitBlockBuffer.UnalignedBlock &~ 0xF);
108
109 #else
110
111 //
112 // This is only for compatibility; the compiler will optimize the extra
113 // local variable (the actual pointer) away, so we don't take any perf hit
114 // by doing this.
115 //
116 #define DEFINE_WAIT_BLOCK(x) \
117 EX_PUSH_LOCK_WAIT_BLOCK WaitBlockBuffer; \
118 PEX_PUSH_LOCK_WAIT_BLOCK x = &WaitBlockBuffer;
119
120 #endif
121
122 #ifdef _WIN64
123 #define ExpChangeRundown(x, y, z) InterlockedCompareExchange64((PLONGLONG)x, y, z)
124 #define ExpChangePushlock(x, y, z) InterlockedCompareExchangePointer((PVOID*)x, (PVOID)y, (PVOID)z)
125 #define ExpSetRundown(x, y) InterlockedExchange64((PLONGLONG)x, y)
126 #else
127 #define ExpChangeRundown(x, y, z) PtrToUlong(InterlockedCompareExchange((PLONG)x, PtrToLong(y), PtrToLong(z)))
128 #define ExpChangePushlock(x, y, z) LongToPtr(InterlockedCompareExchange((PLONG)x, PtrToLong(y), PtrToLong(z)))
129 #define ExpSetRundown(x, y) InterlockedExchange((PLONG)x, y)
130 #endif
131
132 /* INITIALIZATION FUNCTIONS *************************************************/
133
134 VOID
135 NTAPI
136 ExpWin32kInit(VOID);
137
138 VOID
139 NTAPI
140 ExInit2(VOID);
141
142 VOID
143 NTAPI
144 Phase1Initialization(
145 IN PVOID Context
146 );
147
148 VOID
149 NTAPI
150 ExpInitializePushLocks(VOID);
151
152 BOOLEAN
153 NTAPI
154 ExRefreshTimeZoneInformation(
155 IN PLARGE_INTEGER SystemBootTime
156 );
157
158 VOID
159 NTAPI
160 ExpInitializeWorkerThreads(VOID);
161
162 VOID
163 NTAPI
164 ExSwapinWorkerThreads(IN BOOLEAN AllowSwap);
165
166 VOID
167 NTAPI
168 ExpInitLookasideLists(VOID);
169
170 VOID
171 NTAPI
172 ExInitializeSystemLookasideList(
173 IN PGENERAL_LOOKASIDE List,
174 IN POOL_TYPE Type,
175 IN ULONG Size,
176 IN ULONG Tag,
177 IN USHORT MaximumDepth,
178 IN PLIST_ENTRY ListHead
179 );
180
181 BOOLEAN
182 NTAPI
183 ExpInitializeCallbacks(VOID);
184
185 VOID
186 NTAPI
187 ExpInitUuids(VOID);
188
189 VOID
190 NTAPI
191 ExpInitializeExecutive(
192 IN ULONG Cpu,
193 IN PLOADER_PARAMETER_BLOCK LoaderBlock
194 );
195
196 VOID
197 NTAPI
198 ExpInitializeEventImplementation(VOID);
199
200 VOID
201 NTAPI
202 ExpInitializeEventImplementation(VOID);
203
204 VOID
205 NTAPI
206 ExpInitializeEventPairImplementation(VOID);
207
208 VOID
209 NTAPI
210 ExpInitializeSemaphoreImplementation(VOID);
211
212 VOID
213 NTAPI
214 ExpInitializeMutantImplementation(VOID);
215
216 VOID
217 NTAPI
218 ExpInitializeTimerImplementation(VOID);
219
220 VOID
221 NTAPI
222 ExpInitializeProfileImplementation(VOID);
223
224 VOID
225 NTAPI
226 ExpResourceInitialization(VOID);
227
228 VOID
229 NTAPI
230 ExInitPoolLookasidePointers(VOID);
231
232 /* Callback Functions ********************************************************/
233
234 VOID
235 NTAPI
236 ExInitializeCallBack(
237 IN OUT PEX_CALLBACK Callback
238 );
239
240 PEX_CALLBACK_ROUTINE_BLOCK
241 NTAPI
242 ExAllocateCallBack(
243 IN PEX_CALLBACK_FUNCTION Function,
244 IN PVOID Context
245 );
246
247 VOID
248 NTAPI
249 ExFreeCallBack(
250 IN PEX_CALLBACK_ROUTINE_BLOCK CallbackRoutineBlock
251 );
252
253 BOOLEAN
254 NTAPI
255 ExCompareExchangeCallBack (
256 IN OUT PEX_CALLBACK CallBack,
257 IN PEX_CALLBACK_ROUTINE_BLOCK NewBlock,
258 IN PEX_CALLBACK_ROUTINE_BLOCK OldBlock
259 );
260
261 PEX_CALLBACK_ROUTINE_BLOCK
262 NTAPI
263 ExReferenceCallBackBlock(
264 IN OUT PEX_CALLBACK CallBack
265 );
266
267 VOID
268 NTAPI
269 ExDereferenceCallBackBlock(
270 IN OUT PEX_CALLBACK CallBack,
271 IN PEX_CALLBACK_ROUTINE_BLOCK CallbackRoutineBlock
272 );
273
274 PEX_CALLBACK_FUNCTION
275 NTAPI
276 ExGetCallBackBlockRoutine(
277 IN PEX_CALLBACK_ROUTINE_BLOCK CallbackRoutineBlock
278 );
279
280 PVOID
281 NTAPI
282 ExGetCallBackBlockContext(
283 IN PEX_CALLBACK_ROUTINE_BLOCK CallbackRoutineBlock
284 );
285
286 VOID
287 NTAPI
288 ExWaitForCallBacks(
289 IN PEX_CALLBACK_ROUTINE_BLOCK CallbackRoutineBlock
290 );
291
292 /* Rundown Functions ********************************************************/
293
294 VOID
295 FASTCALL
296 ExfInitializeRundownProtection(
297 OUT PEX_RUNDOWN_REF RunRef
298 );
299
300 VOID
301 FASTCALL
302 ExfReInitializeRundownProtection(
303 OUT PEX_RUNDOWN_REF RunRef
304 );
305
306 BOOLEAN
307 FASTCALL
308 ExfAcquireRundownProtection(
309 IN OUT PEX_RUNDOWN_REF RunRef
310 );
311
312 BOOLEAN
313 FASTCALL
314 ExfAcquireRundownProtectionEx(
315 IN OUT PEX_RUNDOWN_REF RunRef,
316 IN ULONG Count
317 );
318
319 VOID
320 FASTCALL
321 ExfReleaseRundownProtection(
322 IN OUT PEX_RUNDOWN_REF RunRef
323 );
324
325 VOID
326 FASTCALL
327 ExfReleaseRundownProtectionEx(
328 IN OUT PEX_RUNDOWN_REF RunRef,
329 IN ULONG Count
330 );
331
332 VOID
333 FASTCALL
334 ExfRundownCompleted(
335 OUT PEX_RUNDOWN_REF RunRef
336 );
337
338 VOID
339 FASTCALL
340 ExfWaitForRundownProtectionRelease(
341 IN OUT PEX_RUNDOWN_REF RunRef
342 );
343
344 /* HANDLE TABLE FUNCTIONS ***************************************************/
345
346 typedef BOOLEAN
347 (NTAPI *PEX_SWEEP_HANDLE_CALLBACK)(
348 PHANDLE_TABLE_ENTRY HandleTableEntry,
349 HANDLE Handle,
350 PVOID Context
351 );
352
353 typedef BOOLEAN
354 (NTAPI *PEX_DUPLICATE_HANDLE_CALLBACK)(
355 IN PEPROCESS Process,
356 IN PHANDLE_TABLE HandleTable,
357 IN PHANDLE_TABLE_ENTRY HandleTableEntry,
358 IN PHANDLE_TABLE_ENTRY NewEntry
359 );
360
361 typedef BOOLEAN
362 (NTAPI *PEX_CHANGE_HANDLE_CALLBACK)(
363 PHANDLE_TABLE_ENTRY HandleTableEntry,
364 ULONG_PTR Context
365 );
366
367 VOID
368 NTAPI
369 ExpInitializeHandleTables(
370 VOID
371 );
372
373 PHANDLE_TABLE
374 NTAPI
375 ExCreateHandleTable(
376 IN PEPROCESS Process OPTIONAL
377 );
378
379 VOID
380 NTAPI
381 ExUnlockHandleTableEntry(
382 IN PHANDLE_TABLE HandleTable,
383 IN PHANDLE_TABLE_ENTRY HandleTableEntry
384 );
385
386 HANDLE
387 NTAPI
388 ExCreateHandle(
389 IN PHANDLE_TABLE HandleTable,
390 IN PHANDLE_TABLE_ENTRY HandleTableEntry
391 );
392
393 VOID
394 NTAPI
395 ExDestroyHandleTable(
396 IN PHANDLE_TABLE HandleTable,
397 IN PVOID DestroyHandleProcedure OPTIONAL
398 );
399
400 BOOLEAN
401 NTAPI
402 ExDestroyHandle(
403 IN PHANDLE_TABLE HandleTable,
404 IN HANDLE Handle,
405 IN PHANDLE_TABLE_ENTRY HandleTableEntry OPTIONAL
406 );
407
408 PHANDLE_TABLE_ENTRY
409 NTAPI
410 ExMapHandleToPointer(
411 IN PHANDLE_TABLE HandleTable,
412 IN HANDLE Handle
413 );
414
415 PHANDLE_TABLE
416 NTAPI
417 ExDupHandleTable(
418 IN PEPROCESS Process,
419 IN PHANDLE_TABLE HandleTable,
420 IN PEX_DUPLICATE_HANDLE_CALLBACK DupHandleProcedure,
421 IN ULONG_PTR Mask
422 );
423
424 BOOLEAN
425 NTAPI
426 ExChangeHandle(
427 IN PHANDLE_TABLE HandleTable,
428 IN HANDLE Handle,
429 IN PEX_CHANGE_HANDLE_CALLBACK ChangeRoutine,
430 IN ULONG_PTR Context
431 );
432
433 VOID
434 NTAPI
435 ExSweepHandleTable(
436 IN PHANDLE_TABLE HandleTable,
437 IN PEX_SWEEP_HANDLE_CALLBACK EnumHandleProcedure,
438 IN PVOID Context
439 );
440
441 /* PSEH EXCEPTION HANDLING **************************************************/
442
443 LONG
444 NTAPI
445 ExSystemExceptionFilter(VOID);
446
447 /* CALLBACKS *****************************************************************/
448
449 FORCEINLINE
450 VOID
451 ExDoCallBack(IN OUT PEX_CALLBACK Callback,
452 IN PVOID Context,
453 IN PVOID Argument1,
454 IN PVOID Argument2)
455 {
456 PEX_CALLBACK_ROUTINE_BLOCK CallbackBlock;
457 PEX_CALLBACK_FUNCTION Function;
458
459 /* Reference the block */
460 CallbackBlock = ExReferenceCallBackBlock(Callback);
461 if (CallbackBlock)
462 {
463 /* Get the function */
464 Function = ExGetCallBackBlockRoutine(CallbackBlock);
465
466 /* Do the callback */
467 Function(Context, Argument1, Argument2);
468
469 /* Now dereference it */
470 ExDereferenceCallBackBlock(Callback, CallbackBlock);
471 }
472 }
473
474 /* FAST REFS ******************************************************************/
475
476 FORCEINLINE
477 PVOID
478 ExGetObjectFastReference(IN EX_FAST_REF FastRef)
479 {
480 /* Return the unbiased pointer */
481 return (PVOID)(FastRef.Value & ~MAX_FAST_REFS);
482 }
483
484 FORCEINLINE
485 ULONG
486 ExGetCountFastReference(IN EX_FAST_REF FastRef)
487 {
488 /* Return the reference count */
489 return FastRef.RefCnt;
490 }
491
492 FORCEINLINE
493 VOID
494 ExInitializeFastReference(OUT PEX_FAST_REF FastRef,
495 IN OPTIONAL PVOID Object)
496 {
497 /* Sanity check */
498 ASSERT((((ULONG_PTR)Object) & MAX_FAST_REFS) == 0);
499
500 /* Check if an object is being set */
501 if (!Object)
502 {
503 /* Clear the field */
504 FastRef->Object = NULL;
505 }
506 else
507 {
508 /* Otherwise, we assume the object was referenced and is ready */
509 FastRef->Value = (ULONG_PTR)Object | MAX_FAST_REFS;
510 }
511 }
512
513 FORCEINLINE
514 EX_FAST_REF
515 ExAcquireFastReference(IN OUT PEX_FAST_REF FastRef)
516 {
517 EX_FAST_REF OldValue, NewValue;
518
519 /* Start reference loop */
520 for (;;)
521 {
522 /* Get the current reference count */
523 OldValue = *FastRef;
524 if (OldValue.RefCnt)
525 {
526 /* Increase the reference count */
527 NewValue.Value = OldValue.Value - 1;
528 NewValue.Object = ExpChangePushlock(&FastRef->Object,
529 NewValue.Object,
530 OldValue.Object);
531 if (NewValue.Object != OldValue.Object) continue;
532 }
533
534 /* We are done */
535 break;
536 }
537
538 /* Return the old value */
539 return OldValue;
540 }
541
542 FORCEINLINE
543 BOOLEAN
544 ExInsertFastReference(IN OUT PEX_FAST_REF FastRef,
545 IN PVOID Object)
546 {
547 EX_FAST_REF OldValue, NewValue;
548
549 /* Sanity checks */
550 ASSERT(!(((ULONG_PTR)Object) & MAX_FAST_REFS));
551
552 /* Start update loop */
553 for (;;)
554 {
555 /* Get the current reference count */
556 OldValue = *FastRef;
557
558 /* Check if the current count is too high or if the pointer changed */
559 if (((OldValue.RefCnt + MAX_FAST_REFS) > MAX_FAST_REFS) ||
560 ((OldValue.Value &~ MAX_FAST_REFS) != (ULONG_PTR)Object))
561 {
562 /* Fail */
563 return FALSE;
564 }
565
566 /* Update the reference count */
567 NewValue.Value = OldValue.Value + MAX_FAST_REFS;
568 NewValue.Object = ExpChangePushlock(&FastRef->Object,
569 NewValue.Object,
570 OldValue.Object);
571 if (NewValue.Object != OldValue.Object) continue;
572
573 /* We are done */
574 break;
575 }
576
577 /* Return success */
578 return TRUE;
579 }
580
581 FORCEINLINE
582 BOOLEAN
583 ExReleaseFastReference(IN PEX_FAST_REF FastRef,
584 IN PVOID Object)
585 {
586 EX_FAST_REF OldValue, NewValue;
587
588 /* Sanity checks */
589 ASSERT(Object != NULL);
590 ASSERT(!(((ULONG_PTR)Object) & MAX_FAST_REFS));
591
592 /* Start reference loop */
593 for (;;)
594 {
595 /* Get the current reference count */
596 OldValue = *FastRef;
597
598 /* Check if we're full if if the pointer changed */
599 if ((OldValue.Value ^ (ULONG_PTR)Object) >= MAX_FAST_REFS) return FALSE;
600
601 /* Decrease the reference count */
602 NewValue.Value = OldValue.Value + 1;
603 NewValue.Object = ExpChangePushlock(&FastRef->Object,
604 NewValue.Object,
605 OldValue.Object);
606 if (NewValue.Object != OldValue.Object) continue;
607
608 /* We are done */
609 break;
610 }
611
612 /* Return success */
613 return TRUE;
614 }
615
616 FORCEINLINE
617 EX_FAST_REF
618 ExSwapFastReference(IN PEX_FAST_REF FastRef,
619 IN PVOID Object)
620 {
621 EX_FAST_REF NewValue, OldValue;
622
623 /* Sanity check */
624 ASSERT((((ULONG_PTR)Object) & MAX_FAST_REFS) == 0);
625
626 /* Check if an object is being set */
627 if (!Object)
628 {
629 /* Clear the field */
630 NewValue.Object = NULL;
631 }
632 else
633 {
634 /* Otherwise, we assume the object was referenced and is ready */
635 NewValue.Value = (ULONG_PTR)Object | MAX_FAST_REFS;
636 }
637
638 /* Update the object */
639 OldValue.Object = InterlockedExchangePointer(&FastRef->Object, NewValue.Object);
640 return OldValue;
641 }
642
643 FORCEINLINE
644 EX_FAST_REF
645 ExCompareSwapFastReference(IN PEX_FAST_REF FastRef,
646 IN PVOID Object,
647 IN PVOID OldObject)
648 {
649 EX_FAST_REF OldValue, NewValue;
650
651 /* Sanity check and start swap loop */
652 ASSERT(!(((ULONG_PTR)Object) & MAX_FAST_REFS));
653 for (;;)
654 {
655 /* Get the current value */
656 OldValue = *FastRef;
657
658 /* Make sure there's enough references to swap */
659 if (!((OldValue.Value ^ (ULONG_PTR)OldObject) <= MAX_FAST_REFS)) break;
660
661 /* Check if we have an object to swap */
662 if (Object)
663 {
664 /* Set up the value with maximum fast references */
665 NewValue.Value = (ULONG_PTR)Object | MAX_FAST_REFS;
666 }
667 else
668 {
669 /* Write the object address itself (which is empty) */
670 NewValue.Value = (ULONG_PTR)Object;
671 }
672
673 /* Do the actual compare exchange */
674 NewValue.Object = ExpChangePushlock(&FastRef->Object,
675 NewValue.Object,
676 OldValue.Object);
677 if (NewValue.Object != OldValue.Object) continue;
678
679 /* All done */
680 break;
681 }
682
683 /* Return the old value */
684 return OldValue;
685 }
686
687 /* RUNDOWN *******************************************************************/
688
689 /*++
690 * @name ExfAcquireRundownProtection
691 * INTERNAL MACRO
692 *
693 * The ExfAcquireRundownProtection routine acquires rundown protection for
694 * the specified descriptor.
695 *
696 * @param RunRef
697 * Pointer to a rundown reference descriptor.
698 *
699 * @return TRUE if access to the protected structure was granted, FALSE otherwise.
700 *
701 * @remarks This is the internal macro for system use only.In case the rundown
702 * was active, then the slow-path will be called through the exported
703 * function.
704 *
705 *--*/
706 FORCEINLINE
707 BOOLEAN
708 _ExAcquireRundownProtection(IN PEX_RUNDOWN_REF RunRef)
709 {
710 ULONG_PTR Value, NewValue;
711
712 /* Get the current value and mask the active bit */
713 Value = RunRef->Count &~ EX_RUNDOWN_ACTIVE;
714
715 /* Add a reference */
716 NewValue = Value + EX_RUNDOWN_COUNT_INC;
717
718 /* Change the value */
719 NewValue = ExpChangeRundown(RunRef, NewValue, Value);
720 if (NewValue != Value)
721 {
722 /* Rundown was active, use long path */
723 return ExfAcquireRundownProtection(RunRef);
724 }
725
726 /* Success */
727 return TRUE;
728 }
729
730 /*++
731 * @name ExReleaseRundownProtection
732 * INTERNAL MACRO
733 *
734 * The ExReleaseRundownProtection routine releases rundown protection for
735 * the specified descriptor.
736 *
737 * @param RunRef
738 * Pointer to a rundown reference descriptor.
739 *
740 * @return TRUE if access to the protected structure was granted, FALSE otherwise.
741 *
742 * @remarks This is the internal macro for system use only.In case the rundown
743 * was active, then the slow-path will be called through the exported
744 * function.
745 *
746 *--*/
747 FORCEINLINE
748 VOID
749 _ExReleaseRundownProtection(IN PEX_RUNDOWN_REF RunRef)
750 {
751 ULONG_PTR Value, NewValue;
752
753 /* Get the current value and mask the active bit */
754 Value = RunRef->Count &~ EX_RUNDOWN_ACTIVE;
755
756 /* Remove a reference */
757 NewValue = Value - EX_RUNDOWN_COUNT_INC;
758
759 /* Change the value */
760 NewValue = ExpChangeRundown(RunRef, NewValue, Value);
761
762 /* Check if the rundown was active */
763 if (NewValue != Value)
764 {
765 /* Rundown was active, use long path */
766 ExfReleaseRundownProtection(RunRef);
767 }
768 else
769 {
770 /* Sanity check */
771 ASSERT((Value >= EX_RUNDOWN_COUNT_INC) || (KeNumberProcessors > 1));
772 }
773 }
774
775 /*++
776 * @name ExInitializeRundownProtection
777 * INTERNAL MACRO
778 *
779 * The ExInitializeRundownProtection routine initializes a rundown
780 * protection descriptor.
781 *
782 * @param RunRef
783 * Pointer to a rundown reference descriptor.
784 *
785 * @return None.
786 *
787 * @remarks This is the internal macro for system use only.
788 *
789 *--*/
790 FORCEINLINE
791 VOID
792 _ExInitializeRundownProtection(IN PEX_RUNDOWN_REF RunRef)
793 {
794 /* Set the count to zero */
795 RunRef->Count = 0;
796 }
797
798 /*++
799 * @name ExWaitForRundownProtectionRelease
800 * INTERNAL MACRO
801 *
802 * The ExWaitForRundownProtectionRelease routine waits until the specified
803 * rundown descriptor has been released.
804 *
805 * @param RunRef
806 * Pointer to a rundown reference descriptor.
807 *
808 * @return None.
809 *
810 * @remarks This is the internal macro for system use only. If a wait is actually
811 * necessary, then the slow path is taken through the exported function.
812 *
813 *--*/
814 FORCEINLINE
815 VOID
816 _ExWaitForRundownProtectionRelease(IN PEX_RUNDOWN_REF RunRef)
817 {
818 ULONG_PTR Value;
819
820 /* Set the active bit */
821 Value = ExpChangeRundown(RunRef, EX_RUNDOWN_ACTIVE, 0);
822 if ((Value) && (Value != EX_RUNDOWN_ACTIVE))
823 {
824 /* If the the rundown wasn't already active, then take the long path */
825 ExfWaitForRundownProtectionRelease(RunRef);
826 }
827 }
828
829 /*++
830 * @name ExRundownCompleted
831 * INTERNAL MACRO
832 *
833 * The ExRundownCompleted routine completes the rundown of the specified
834 * descriptor by setting the active bit.
835 *
836 * @param RunRef
837 * Pointer to a rundown reference descriptor.
838 *
839 * @return None.
840 *
841 * @remarks This is the internal macro for system use only.
842 *
843 *--*/
844 FORCEINLINE
845 VOID
846 _ExRundownCompleted(IN PEX_RUNDOWN_REF RunRef)
847 {
848 /* Sanity check */
849 ASSERT((RunRef->Count & EX_RUNDOWN_ACTIVE) != 0);
850
851 /* Mark the counter as active */
852 ExpSetRundown(&RunRef->Count, EX_RUNDOWN_ACTIVE);
853 }
854
855 /* PUSHLOCKS *****************************************************************/
856
857 /* FIXME: VERIFY THESE! */
858
859 VOID
860 FASTCALL
861 ExBlockPushLock(
862 IN PEX_PUSH_LOCK PushLock,
863 IN PVOID WaitBlock
864 );
865
866 VOID
867 FASTCALL
868 ExfUnblockPushLock(
869 IN PEX_PUSH_LOCK PushLock,
870 IN PVOID CurrentWaitBlock
871 );
872
873 VOID
874 FASTCALL
875 ExWaitForUnblockPushLock(
876 IN PEX_PUSH_LOCK PushLock,
877 IN PVOID WaitBlock
878 );
879
880 /*++
881 * @name _ExInitializePushLock
882 * INTERNAL MACRO
883 *
884 * The _ExInitializePushLock macro initializes a PushLock.
885 *
886 * @params PushLock
887 * Pointer to the pushlock which is to be initialized.
888 *
889 * @return None.
890 *
891 * @remarks None.
892 *
893 *--*/
894 FORCEINLINE
895 VOID
896 _ExInitializePushLock(IN PULONG_PTR PushLock)
897 {
898 /* Set the value to 0 */
899 *PushLock = 0;
900 }
901 #define ExInitializePushLock _ExInitializePushLock
902
903 /*++
904 * @name ExAcquirePushLockExclusive
905 * INTERNAL MACRO
906 *
907 * The ExAcquirePushLockExclusive macro exclusively acquires a PushLock.
908 *
909 * @params PushLock
910 * Pointer to the pushlock which is to be acquired.
911 *
912 * @return None.
913 *
914 * @remarks The function attempts the quickest route to acquire the lock, which is
915 * to simply set the lock bit.
916 * However, if the pushlock is already shared, the slower path is taken.
917 *
918 * Callers of ExAcquirePushLockShared must be running at IRQL <= APC_LEVEL.
919 * This macro should usually be paired up with KeAcquireCriticalRegion.
920 *
921 *--*/
922 FORCEINLINE
923 VOID
924 ExAcquirePushLockExclusive(PEX_PUSH_LOCK PushLock)
925 {
926 /* Try acquiring the lock */
927 if (InterlockedBitTestAndSet((PLONG)PushLock, EX_PUSH_LOCK_LOCK_V))
928 {
929 /* Someone changed it, use the slow path */
930 ExfAcquirePushLockExclusive(PushLock);
931 }
932
933 /* Sanity check */
934 ASSERT(PushLock->Locked);
935 }
936
937 /*++
938 * @name ExTryToAcquirePushLockExclusive
939 * INTERNAL MACRO
940 *
941 * The ExAcquirePushLockExclusive macro exclusively acquires a PushLock.
942 *
943 * @params PushLock
944 * Pointer to the pushlock which is to be acquired.
945 *
946 * @return None.
947 *
948 * @remarks The function attempts the quickest route to acquire the lock, which is
949 * to simply set the lock bit.
950 * However, if the pushlock is already shared, the slower path is taken.
951 *
952 * Callers of ExAcquirePushLockShared must be running at IRQL <= APC_LEVEL.
953 * This macro should usually be paired up with KeAcquireCriticalRegion.
954 *
955 *--*/
956 FORCEINLINE
957 BOOLEAN
958 ExTryToAcquirePushLockExclusive(PEX_PUSH_LOCK PushLock)
959 {
960 /* Try acquiring the lock */
961 if (InterlockedBitTestAndSet((PLONG)PushLock, EX_PUSH_LOCK_LOCK_V))
962 {
963 /* Can't acquire */
964 return FALSE;
965 }
966
967 /* Got acquired */
968 ASSERT (PushLock->Locked);
969 return TRUE;
970 }
971
972 /*++
973 * @name ExAcquirePushLockShared
974 * INTERNAL MACRO
975 *
976 * The ExAcquirePushLockShared macro acquires a shared PushLock.
977 *
978 * @params PushLock
979 * Pointer to the pushlock which is to be acquired.
980 *
981 * @return None.
982 *
983 * @remarks The function attempts the quickest route to acquire the lock, which is
984 * to simply set the lock bit and set the share count to one.
985 * However, if the pushlock is already shared, the slower path is taken.
986 *
987 * Callers of ExAcquirePushLockShared must be running at IRQL <= APC_LEVEL.
988 * This macro should usually be paired up with KeAcquireCriticalRegion.
989 *
990 *--*/
991 FORCEINLINE
992 VOID
993 ExAcquirePushLockShared(PEX_PUSH_LOCK PushLock)
994 {
995 EX_PUSH_LOCK NewValue;
996
997 /* Try acquiring the lock */
998 NewValue.Value = EX_PUSH_LOCK_LOCK | EX_PUSH_LOCK_SHARE_INC;
999 if (ExpChangePushlock(PushLock, NewValue.Ptr, 0))
1000 {
1001 /* Someone changed it, use the slow path */
1002 ExfAcquirePushLockShared(PushLock);
1003 }
1004
1005 /* Sanity checks */
1006 ASSERT(PushLock->Locked);
1007 ASSERT(PushLock->Waiting || PushLock->Shared > 0);
1008 }
1009
1010 /*++
1011 * @name ExConvertPushLockSharedToExclusive
1012 * INTERNAL MACRO
1013 *
1014 * The ExConvertPushLockSharedToExclusive macro converts an exclusive
1015 * pushlock to a shared pushlock.
1016 *
1017 * @params PushLock
1018 * Pointer to the pushlock which is to be converted.
1019 *
1020 * @return FALSE if conversion failed, TRUE otherwise.
1021 *
1022 * @remarks The function attempts the quickest route to convert the lock, which is
1023 * to simply set the lock bit and remove any other bits.
1024 *
1025 *--*/
1026 FORCEINLINE
1027 BOOLEAN
1028 ExConvertPushLockSharedToExclusive(IN PEX_PUSH_LOCK PushLock)
1029 {
1030 EX_PUSH_LOCK OldValue;
1031
1032 /* Set the expected old value */
1033 OldValue.Value = EX_PUSH_LOCK_LOCK | EX_PUSH_LOCK_SHARE_INC;
1034
1035 /* Try converting the lock */
1036 if (ExpChangePushlock(PushLock, EX_PUSH_LOCK_LOCK, OldValue.Value) !=
1037 OldValue.Ptr)
1038 {
1039 /* Conversion failed */
1040 return FALSE;
1041 }
1042
1043 /* Sanity check */
1044 ASSERT(PushLock->Locked);
1045 return TRUE;
1046 }
1047
1048 /*++
1049 * @name ExWaitOnPushLock
1050 * INTERNAL MACRO
1051 *
1052 * The ExWaitOnPushLock macro acquires and instantly releases a pushlock.
1053 *
1054 * @params PushLock
1055 * Pointer to a pushlock.
1056 *
1057 * @return None.
1058 *
1059 * @remarks The function attempts to get any exclusive waiters out of their slow
1060 * path by forcing an instant acquire/release operation.
1061 *
1062 * Callers of ExWaitOnPushLock must be running at IRQL <= APC_LEVEL.
1063 *
1064 *--*/
1065 FORCEINLINE
1066 VOID
1067 ExWaitOnPushLock(PEX_PUSH_LOCK PushLock)
1068 {
1069 /* Check if we're locked */
1070 if (PushLock->Locked)
1071 {
1072 /* Acquire the lock */
1073 ExfAcquirePushLockExclusive(PushLock);
1074 ASSERT(PushLock->Locked);
1075
1076 /* Release it */
1077 ExfReleasePushLockExclusive(PushLock);
1078 }
1079 }
1080
1081 /*++
1082 * @name ExReleasePushLockShared
1083 * INTERNAL MACRO
1084 *
1085 * The ExReleasePushLockShared macro releases a previously acquired PushLock.
1086 *
1087 * @params PushLock
1088 * Pointer to a previously acquired pushlock.
1089 *
1090 * @return None.
1091 *
1092 * @remarks The function attempts the quickest route to release the lock, which is
1093 * to simply decrease the share count and remove the lock bit.
1094 * However, if the pushlock is being waited on then the long path is taken.
1095 *
1096 * Callers of ExReleasePushLockShared must be running at IRQL <= APC_LEVEL.
1097 * This macro should usually be paired up with KeLeaveCriticalRegion.
1098 *
1099 *--*/
1100 FORCEINLINE
1101 VOID
1102 ExReleasePushLockShared(PEX_PUSH_LOCK PushLock)
1103 {
1104 EX_PUSH_LOCK OldValue;
1105
1106 /* Sanity checks */
1107 ASSERT(PushLock->Locked);
1108 ASSERT(PushLock->Waiting || PushLock->Shared > 0);
1109
1110 /* Try to clear the pushlock */
1111 OldValue.Value = EX_PUSH_LOCK_LOCK | EX_PUSH_LOCK_SHARE_INC;
1112 if (ExpChangePushlock(PushLock, 0, OldValue.Ptr) != OldValue.Ptr)
1113 {
1114 /* There are still other people waiting on it */
1115 ExfReleasePushLockShared(PushLock);
1116 }
1117 }
1118
1119 /*++
1120 * @name ExReleasePushLockExclusive
1121 * INTERNAL MACRO
1122 *
1123 * The ExReleasePushLockExclusive macro releases a previously
1124 * exclusively acquired PushLock.
1125 *
1126 * @params PushLock
1127 * Pointer to a previously acquired pushlock.
1128 *
1129 * @return None.
1130 *
1131 * @remarks The function attempts the quickest route to release the lock, which is
1132 * to simply clear the locked bit.
1133 * However, if the pushlock is being waited on, the slow path is taken
1134 * in an attempt to wake up the lock.
1135 *
1136 * Callers of ExReleasePushLockExclusive must be running at IRQL <= APC_LEVEL.
1137 * This macro should usually be paired up with KeLeaveCriticalRegion.
1138 *
1139 *--*/
1140 FORCEINLINE
1141 VOID
1142 ExReleasePushLockExclusive(PEX_PUSH_LOCK PushLock)
1143 {
1144 EX_PUSH_LOCK OldValue;
1145
1146 /* Sanity checks */
1147 ASSERT(PushLock->Locked);
1148 ASSERT(PushLock->Waiting || PushLock->Shared == 0);
1149
1150 /* Unlock the pushlock */
1151 OldValue.Value = InterlockedExchangeAddSizeT((PSIZE_T)PushLock,
1152 -(SIZE_T)EX_PUSH_LOCK_LOCK);
1153
1154 /* Sanity checks */
1155 ASSERT(OldValue.Locked);
1156 ASSERT(OldValue.Waiting || OldValue.Shared == 0);
1157
1158 /* Check if anyone is waiting on it and it's not already waking*/
1159 if ((OldValue.Waiting) && !(OldValue.Waking))
1160 {
1161 /* Wake it up */
1162 ExfTryToWakePushLock(PushLock);
1163 }
1164 }
1165
1166 /*++
1167 * @name ExReleasePushLock
1168 * INTERNAL MACRO
1169 *
1170 * The ExReleasePushLock macro releases a previously acquired PushLock.
1171 *
1172 * @params PushLock
1173 * Pointer to a previously acquired pushlock.
1174 *
1175 * @return None.
1176 *
1177 * @remarks The function attempts the quickest route to release the lock, which is
1178 * to simply clear all the fields and decrease the share count if required.
1179 * However, if the pushlock is being waited on then the long path is taken.
1180 *
1181 * Callers of ExReleasePushLock must be running at IRQL <= APC_LEVEL.
1182 * This macro should usually be paired up with KeLeaveCriticalRegion.
1183 *
1184 *--*/
1185 FORCEINLINE
1186 VOID
1187 ExReleasePushLock(PEX_PUSH_LOCK PushLock)
1188 {
1189 EX_PUSH_LOCK OldValue = *PushLock;
1190 EX_PUSH_LOCK NewValue;
1191
1192 /* Sanity checks */
1193 ASSERT(OldValue.Locked);
1194
1195 /* Check if the pushlock is shared */
1196 if (OldValue.Shared > 1)
1197 {
1198 /* Decrease the share count */
1199 NewValue.Value = OldValue.Value - EX_PUSH_LOCK_SHARE_INC;
1200 }
1201 else
1202 {
1203 /* Clear the pushlock entirely */
1204 NewValue.Value = 0;
1205 }
1206
1207 /* Check if nobody is waiting on us and try clearing the lock here */
1208 if ((OldValue.Waiting) ||
1209 (ExpChangePushlock(PushLock, NewValue.Ptr, OldValue.Ptr) !=
1210 OldValue.Ptr))
1211 {
1212 /* We have waiters, use the long path */
1213 ExfReleasePushLock(PushLock);
1214 }
1215 }
1216
1217 /* FAST MUTEX INLINES *********************************************************/
1218
1219 FORCEINLINE
1220 VOID
1221 _ExAcquireFastMutexUnsafe(IN PFAST_MUTEX FastMutex)
1222 {
1223 PKTHREAD Thread = KeGetCurrentThread();
1224
1225 /* Sanity check */
1226 ASSERT((KeGetCurrentIrql() == APC_LEVEL) ||
1227 (Thread->CombinedApcDisable != 0) ||
1228 (Thread->Teb == NULL) ||
1229 (Thread->Teb >= (PTEB)MM_SYSTEM_RANGE_START));
1230 ASSERT(FastMutex->Owner != Thread);
1231
1232 /* Decrease the count */
1233 if (InterlockedDecrement(&FastMutex->Count))
1234 {
1235 /* Someone is still holding it, use slow path */
1236 KiAcquireFastMutex(FastMutex);
1237 }
1238
1239 /* Set the owner */
1240 FastMutex->Owner = Thread;
1241 }
1242
1243 FORCEINLINE
1244 VOID
1245 _ExReleaseFastMutexUnsafe(IN OUT PFAST_MUTEX FastMutex)
1246 {
1247 ASSERT((KeGetCurrentIrql() == APC_LEVEL) ||
1248 (KeGetCurrentThread()->CombinedApcDisable != 0) ||
1249 (KeGetCurrentThread()->Teb == NULL) ||
1250 (KeGetCurrentThread()->Teb >= (PTEB)MM_SYSTEM_RANGE_START));
1251 ASSERT(FastMutex->Owner == KeGetCurrentThread());
1252
1253 /* Erase the owner */
1254 FastMutex->Owner = NULL;
1255
1256 /* Increase the count */
1257 if (InterlockedIncrement(&FastMutex->Count) <= 0)
1258 {
1259 /* Someone was waiting for it, signal the waiter */
1260 KeSetEventBoostPriority(&FastMutex->Event, NULL);
1261 }
1262 }
1263
1264 FORCEINLINE
1265 VOID
1266 _ExAcquireFastMutex(IN PFAST_MUTEX FastMutex)
1267 {
1268 KIRQL OldIrql;
1269 ASSERT(KeGetCurrentIrql() <= APC_LEVEL);
1270
1271 /* Raise IRQL to APC */
1272 KeRaiseIrql(APC_LEVEL, &OldIrql);
1273
1274 /* Decrease the count */
1275 if (InterlockedDecrement(&FastMutex->Count))
1276 {
1277 /* Someone is still holding it, use slow path */
1278 KiAcquireFastMutex(FastMutex);
1279 }
1280
1281 /* Set the owner and IRQL */
1282 FastMutex->Owner = KeGetCurrentThread();
1283 FastMutex->OldIrql = OldIrql;
1284 }
1285
1286 FORCEINLINE
1287 VOID
1288 _ExReleaseFastMutex(IN OUT PFAST_MUTEX FastMutex)
1289 {
1290 KIRQL OldIrql;
1291 ASSERT(KeGetCurrentIrql() == APC_LEVEL);
1292
1293 /* Erase the owner */
1294 FastMutex->Owner = NULL;
1295 OldIrql = (KIRQL)FastMutex->OldIrql;
1296
1297 /* Increase the count */
1298 if (InterlockedIncrement(&FastMutex->Count) <= 0)
1299 {
1300 /* Someone was waiting for it, signal the waiter */
1301 KeSetEventBoostPriority(&FastMutex->Event, NULL);
1302 }
1303
1304 /* Lower IRQL back */
1305 KeLowerIrql(OldIrql);
1306 }
1307
1308 FORCEINLINE
1309 BOOLEAN
1310 _ExTryToAcquireFastMutex(IN OUT PFAST_MUTEX FastMutex)
1311 {
1312 KIRQL OldIrql;
1313 ASSERT(KeGetCurrentIrql() <= APC_LEVEL);
1314
1315 /* Raise to APC_LEVEL */
1316 KeRaiseIrql(APC_LEVEL, &OldIrql);
1317
1318 /* Check if we can quickly acquire it */
1319 if (InterlockedCompareExchange(&FastMutex->Count, 0, 1) == 1)
1320 {
1321 /* We have, set us as owners */
1322 FastMutex->Owner = KeGetCurrentThread();
1323 FastMutex->OldIrql = OldIrql;
1324 return TRUE;
1325 }
1326 else
1327 {
1328 /* Acquire attempt failed */
1329 KeLowerIrql(OldIrql);
1330 YieldProcessor();
1331 return FALSE;
1332 }
1333 }
1334
1335 FORCEINLINE
1336 VOID
1337 _ExEnterCriticalRegionAndAcquireFastMutexUnsafe(IN OUT PFAST_MUTEX FastMutex)
1338 {
1339 /* Enter the Critical Region */
1340 KeEnterCriticalRegion();
1341
1342 /* Acquire the mutex unsafely */
1343 _ExAcquireFastMutexUnsafe(FastMutex);
1344 }
1345
1346 FORCEINLINE
1347 VOID
1348 _ExReleaseFastMutexUnsafeAndLeaveCriticalRegion(IN OUT PFAST_MUTEX FastMutex)
1349 {
1350 /* Release the mutex unsafely */
1351 _ExReleaseFastMutexUnsafe(FastMutex);
1352
1353 /* Leave the critical region */
1354 KeLeaveCriticalRegion();
1355 }
1356
1357 /* OTHER FUNCTIONS **********************************************************/
1358
1359 BOOLEAN
1360 NTAPI
1361 ExTryToAcquireResourceExclusiveLite(
1362 IN PERESOURCE Resource
1363 );
1364
1365 NTSTATUS
1366 ExpSetTimeZoneInformation(PTIME_ZONE_INFORMATION TimeZoneInformation);
1367
1368 BOOLEAN
1369 NTAPI
1370 ExAcquireTimeRefreshLock(BOOLEAN Wait);
1371
1372 VOID
1373 NTAPI
1374 ExReleaseTimeRefreshLock(VOID);
1375
1376 VOID
1377 NTAPI
1378 ExUpdateSystemTimeFromCmos(IN BOOLEAN UpdateInterruptTime,
1379 IN ULONG MaxSepInSeconds);
1380
1381 NTSTATUS
1382 NTAPI
1383 ExpAllocateLocallyUniqueId(OUT LUID *LocallyUniqueId);
1384
1385 VOID
1386 NTAPI
1387 ExTimerRundown(VOID);
1388
1389 VOID
1390 NTAPI
1391 HeadlessInit(
1392 IN PLOADER_PARAMETER_BLOCK LoaderBlock
1393 );
1394
1395 VOID
1396 NTAPI
1397 XIPInit(
1398 IN PLOADER_PARAMETER_BLOCK LoaderBlock
1399 );
1400
1401 #define InterlockedDecrementUL(Addend) \
1402 (ULONG)InterlockedDecrement((PLONG)(Addend))
1403
1404 #define InterlockedIncrementUL(Addend) \
1405 (ULONG)InterlockedIncrement((PLONG)(Addend))
1406
1407 #define InterlockedExchangeUL(Target, Value) \
1408 (ULONG)InterlockedExchange((PLONG)(Target), (LONG)(Value))
1409
1410 #define InterlockedExchangeAddUL(Addend, Value) \
1411 (ULONG)InterlockedExchangeAdd((PLONG)(Addend), (LONG)(Value))
1412
1413 #define InterlockedCompareExchangeUL(Destination, Exchange, Comperand) \
1414 (ULONG)InterlockedCompareExchange((PLONG)(Destination), (LONG)(Exchange), (LONG)(Comperand))
1415
1416 #define ExfInterlockedCompareExchange64UL(Destination, Exchange, Comperand) \
1417 (ULONGLONG)ExfInterlockedCompareExchange64((PLONGLONG)(Destination), (PLONGLONG)(Exchange), (PLONGLONG)(Comperand))