- Move ntdef.h from ddk to psdk
[reactos.git] / ntoskrnl / include / internal / ex.h
1 #pragma once
2
3 /* GLOBAL VARIABLES *********************************************************/
4
5 extern TIME_ZONE_INFORMATION ExpTimeZoneInfo;
6 extern LARGE_INTEGER ExpTimeZoneBias;
7 extern ULONG ExpTimeZoneId;
8 extern ULONG ExpTickCountMultiplier;
9 extern ULONG ExpLastTimeZoneBias;
10 extern POBJECT_TYPE ExEventPairObjectType;
11 extern POBJECT_TYPE _ExEventObjectType, _ExSemaphoreObjectType;
12 extern ULONG NtBuildNumber;
13 extern ULONG NtMajorVersion;
14 extern ULONG NtMinorVersion;
15 extern FAST_MUTEX ExpEnvironmentLock;
16 extern ERESOURCE ExpFirmwareTableResource;
17 extern LIST_ENTRY ExpFirmwareTableProviderListHead;
18 extern BOOLEAN ExpIsWinPEMode;
19 extern LIST_ENTRY ExpSystemResourcesList;
20 extern ULONG ExpAnsiCodePageDataOffset, ExpOemCodePageDataOffset;
21 extern ULONG ExpUnicodeCaseTableDataOffset;
22 extern PVOID ExpNlsSectionPointer;
23 extern CHAR NtBuildLab[];
24 extern ULONG CmNtCSDVersion;
25 extern ULONG NtGlobalFlag;
26 extern ULONG ExpInitializationPhase;
27 extern ULONG ExpAltTimeZoneBias;
28 extern LIST_ENTRY ExSystemLookasideListHead;
29
30 typedef struct _EXHANDLE
31 {
32 union
33 {
34 struct
35 {
36 ULONG TagBits:2;
37 ULONG Index:30;
38 };
39 HANDLE GenericHandleOverlay;
40 ULONG_PTR Value;
41 };
42 } EXHANDLE, *PEXHANDLE;
43
44 typedef struct _ETIMER
45 {
46 KTIMER KeTimer;
47 KAPC TimerApc;
48 KDPC TimerDpc;
49 LIST_ENTRY ActiveTimerListEntry;
50 KSPIN_LOCK Lock;
51 LONG Period;
52 BOOLEAN ApcAssociated;
53 BOOLEAN WakeTimer;
54 LIST_ENTRY WakeTimerListEntry;
55 } ETIMER, *PETIMER;
56
57 typedef struct
58 {
59 PCALLBACK_OBJECT *CallbackObject;
60 PWSTR Name;
61 } SYSTEM_CALLBACKS;
62
63 #define MAX_FAST_REFS 7
64
65 #define ExAcquireRundownProtection _ExAcquireRundownProtection
66 #define ExReleaseRundownProtection _ExReleaseRundownProtection
67 #define ExInitializeRundownProtection _ExInitializeRundownProtection
68 #define ExWaitForRundownProtectionRelease _ExWaitForRundownProtectionRelease
69 #define ExRundownCompleted _ExRundownCompleted
70 #define ExGetPreviousMode KeGetPreviousMode
71
72
73 //
74 // Various bits tagged on the handle or handle table
75 //
76 #define EXHANDLE_TABLE_ENTRY_LOCK_BIT 1
77 #define FREE_HANDLE_MASK -1
78
79 //
80 // Number of entries in each table level
81 //
82 #define LOW_LEVEL_ENTRIES (PAGE_SIZE / sizeof(HANDLE_TABLE_ENTRY))
83 #define MID_LEVEL_ENTRIES (PAGE_SIZE / sizeof(PHANDLE_TABLE_ENTRY))
84 #define HIGH_LEVEL_ENTRIES (16777216 / (LOW_LEVEL_ENTRIES * MID_LEVEL_ENTRIES))
85
86 //
87 // Maximum index in each table level before we need another table
88 //
89 #define MAX_LOW_INDEX LOW_LEVEL_ENTRIES
90 #define MAX_MID_INDEX (MID_LEVEL_ENTRIES * LOW_LEVEL_ENTRIES)
91 #define MAX_HIGH_INDEX (MID_LEVEL_ENTRIES * MID_LEVEL_ENTRIES * LOW_LEVEL_ENTRIES)
92
93 //
94 // Detect old GCC
95 //
96 #if (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__ < 40300) || \
97 (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__ == 40303)
98
99 #define DEFINE_WAIT_BLOCK(x) \
100 struct _AlignHack \
101 { \
102 UCHAR Hack[15]; \
103 EX_PUSH_LOCK_WAIT_BLOCK UnalignedBlock; \
104 } WaitBlockBuffer; \
105 PEX_PUSH_LOCK_WAIT_BLOCK x = (PEX_PUSH_LOCK_WAIT_BLOCK) \
106 ((ULONG_PTR)&WaitBlockBuffer.UnalignedBlock &~ 0xF);
107
108 #else
109
110 //
111 // This is only for compatibility; the compiler will optimize the extra
112 // local variable (the actual pointer) away, so we don't take any perf hit
113 // by doing this.
114 //
115 #define DEFINE_WAIT_BLOCK(x) \
116 EX_PUSH_LOCK_WAIT_BLOCK WaitBlockBuffer; \
117 PEX_PUSH_LOCK_WAIT_BLOCK x = &WaitBlockBuffer;
118
119 #endif
120
121 #ifdef _WIN64
122 #define ExpChangeRundown(x, y, z) InterlockedCompareExchange64((PLONGLONG)x, y, z)
123 #define ExpChangePushlock(x, y, z) InterlockedCompareExchangePointer((PVOID*)x, (PVOID)y, (PVOID)z)
124 #define ExpSetRundown(x, y) InterlockedExchange64((PLONGLONG)x, y)
125 #else
126 #define ExpChangeRundown(x, y, z) PtrToUlong(InterlockedCompareExchange((PLONG)x, PtrToLong(y), PtrToLong(z)))
127 #define ExpChangePushlock(x, y, z) LongToPtr(InterlockedCompareExchange((PLONG)x, PtrToLong(y), PtrToLong(z)))
128 #define ExpSetRundown(x, y) InterlockedExchange((PLONG)x, y)
129 #endif
130
131 /* INITIALIZATION FUNCTIONS *************************************************/
132
133 VOID
134 NTAPI
135 ExpWin32kInit(VOID);
136
137 VOID
138 NTAPI
139 ExInit2(VOID);
140
141 VOID
142 NTAPI
143 Phase1Initialization(
144 IN PVOID Context
145 );
146
147 VOID
148 NTAPI
149 ExpInitializePushLocks(VOID);
150
151 BOOLEAN
152 NTAPI
153 ExRefreshTimeZoneInformation(
154 IN PLARGE_INTEGER SystemBootTime
155 );
156
157 VOID
158 NTAPI
159 ExpInitializeWorkerThreads(VOID);
160
161 VOID
162 NTAPI
163 ExpInitLookasideLists(VOID);
164
165 VOID
166 NTAPI
167 ExInitializeSystemLookasideList(
168 IN PGENERAL_LOOKASIDE List,
169 IN POOL_TYPE Type,
170 IN ULONG Size,
171 IN ULONG Tag,
172 IN USHORT MaximumDepth,
173 IN PLIST_ENTRY ListHead
174 );
175
176 BOOLEAN
177 NTAPI
178 ExpInitializeCallbacks(VOID);
179
180 VOID
181 NTAPI
182 ExpInitUuids(VOID);
183
184 VOID
185 NTAPI
186 ExpInitializeExecutive(
187 IN ULONG Cpu,
188 IN PLOADER_PARAMETER_BLOCK LoaderBlock
189 );
190
191 VOID
192 NTAPI
193 ExpInitializeEventImplementation(VOID);
194
195 VOID
196 NTAPI
197 ExpInitializeEventImplementation(VOID);
198
199 VOID
200 NTAPI
201 ExpInitializeEventPairImplementation(VOID);
202
203 VOID
204 NTAPI
205 ExpInitializeSemaphoreImplementation(VOID);
206
207 VOID
208 NTAPI
209 ExpInitializeMutantImplementation(VOID);
210
211 VOID
212 NTAPI
213 ExpInitializeTimerImplementation(VOID);
214
215 VOID
216 NTAPI
217 ExpInitializeProfileImplementation(VOID);
218
219 VOID
220 NTAPI
221 ExpResourceInitialization(VOID);
222
223 VOID
224 NTAPI
225 ExInitPoolLookasidePointers(VOID);
226
227 /* Callback Functions ********************************************************/
228
229 VOID
230 NTAPI
231 ExInitializeCallBack(
232 IN OUT PEX_CALLBACK Callback
233 );
234
235 PEX_CALLBACK_ROUTINE_BLOCK
236 NTAPI
237 ExAllocateCallBack(
238 IN PEX_CALLBACK_FUNCTION Function,
239 IN PVOID Context
240 );
241
242 VOID
243 NTAPI
244 ExFreeCallBack(
245 IN PEX_CALLBACK_ROUTINE_BLOCK CallbackRoutineBlock
246 );
247
248 BOOLEAN
249 NTAPI
250 ExCompareExchangeCallBack (
251 IN OUT PEX_CALLBACK CallBack,
252 IN PEX_CALLBACK_ROUTINE_BLOCK NewBlock,
253 IN PEX_CALLBACK_ROUTINE_BLOCK OldBlock
254 );
255
256 PEX_CALLBACK_ROUTINE_BLOCK
257 NTAPI
258 ExReferenceCallBackBlock(
259 IN OUT PEX_CALLBACK CallBack
260 );
261
262 VOID
263 NTAPI
264 ExDereferenceCallBackBlock(
265 IN OUT PEX_CALLBACK CallBack,
266 IN PEX_CALLBACK_ROUTINE_BLOCK CallbackRoutineBlock
267 );
268
269 PEX_CALLBACK_FUNCTION
270 NTAPI
271 ExGetCallBackBlockRoutine(
272 IN PEX_CALLBACK_ROUTINE_BLOCK CallbackRoutineBlock
273 );
274
275 PVOID
276 NTAPI
277 ExGetCallBackBlockContext(
278 IN PEX_CALLBACK_ROUTINE_BLOCK CallbackRoutineBlock
279 );
280
281 VOID
282 NTAPI
283 ExWaitForCallBacks(
284 IN PEX_CALLBACK_ROUTINE_BLOCK CallbackRoutineBlock
285 );
286
287 /* Rundown Functions ********************************************************/
288
289 VOID
290 FASTCALL
291 ExfInitializeRundownProtection(
292 OUT PEX_RUNDOWN_REF RunRef
293 );
294
295 VOID
296 FASTCALL
297 ExfReInitializeRundownProtection(
298 OUT PEX_RUNDOWN_REF RunRef
299 );
300
301 BOOLEAN
302 FASTCALL
303 ExfAcquireRundownProtection(
304 IN OUT PEX_RUNDOWN_REF RunRef
305 );
306
307 BOOLEAN
308 FASTCALL
309 ExfAcquireRundownProtectionEx(
310 IN OUT PEX_RUNDOWN_REF RunRef,
311 IN ULONG Count
312 );
313
314 VOID
315 FASTCALL
316 ExfReleaseRundownProtection(
317 IN OUT PEX_RUNDOWN_REF RunRef
318 );
319
320 VOID
321 FASTCALL
322 ExfReleaseRundownProtectionEx(
323 IN OUT PEX_RUNDOWN_REF RunRef,
324 IN ULONG Count
325 );
326
327 VOID
328 FASTCALL
329 ExfRundownCompleted(
330 OUT PEX_RUNDOWN_REF RunRef
331 );
332
333 VOID
334 FASTCALL
335 ExfWaitForRundownProtectionRelease(
336 IN OUT PEX_RUNDOWN_REF RunRef
337 );
338
339 /* HANDLE TABLE FUNCTIONS ***************************************************/
340
341 typedef BOOLEAN
342 (NTAPI *PEX_SWEEP_HANDLE_CALLBACK)(
343 PHANDLE_TABLE_ENTRY HandleTableEntry,
344 HANDLE Handle,
345 PVOID Context
346 );
347
348 typedef BOOLEAN
349 (NTAPI *PEX_DUPLICATE_HANDLE_CALLBACK)(
350 IN PEPROCESS Process,
351 IN PHANDLE_TABLE HandleTable,
352 IN PHANDLE_TABLE_ENTRY HandleTableEntry,
353 IN PHANDLE_TABLE_ENTRY NewEntry
354 );
355
356 typedef BOOLEAN
357 (NTAPI *PEX_CHANGE_HANDLE_CALLBACK)(
358 PHANDLE_TABLE_ENTRY HandleTableEntry,
359 ULONG_PTR Context
360 );
361
362 VOID
363 NTAPI
364 ExpInitializeHandleTables(
365 VOID
366 );
367
368 PHANDLE_TABLE
369 NTAPI
370 ExCreateHandleTable(
371 IN PEPROCESS Process OPTIONAL
372 );
373
374 VOID
375 NTAPI
376 ExUnlockHandleTableEntry(
377 IN PHANDLE_TABLE HandleTable,
378 IN PHANDLE_TABLE_ENTRY HandleTableEntry
379 );
380
381 HANDLE
382 NTAPI
383 ExCreateHandle(
384 IN PHANDLE_TABLE HandleTable,
385 IN PHANDLE_TABLE_ENTRY HandleTableEntry
386 );
387
388 VOID
389 NTAPI
390 ExDestroyHandleTable(
391 IN PHANDLE_TABLE HandleTable,
392 IN PVOID DestroyHandleProcedure OPTIONAL
393 );
394
395 BOOLEAN
396 NTAPI
397 ExDestroyHandle(
398 IN PHANDLE_TABLE HandleTable,
399 IN HANDLE Handle,
400 IN PHANDLE_TABLE_ENTRY HandleTableEntry OPTIONAL
401 );
402
403 PHANDLE_TABLE_ENTRY
404 NTAPI
405 ExMapHandleToPointer(
406 IN PHANDLE_TABLE HandleTable,
407 IN HANDLE Handle
408 );
409
410 PHANDLE_TABLE
411 NTAPI
412 ExDupHandleTable(
413 IN PEPROCESS Process,
414 IN PHANDLE_TABLE HandleTable,
415 IN PEX_DUPLICATE_HANDLE_CALLBACK DupHandleProcedure,
416 IN ULONG_PTR Mask
417 );
418
419 BOOLEAN
420 NTAPI
421 ExChangeHandle(
422 IN PHANDLE_TABLE HandleTable,
423 IN HANDLE Handle,
424 IN PEX_CHANGE_HANDLE_CALLBACK ChangeRoutine,
425 IN ULONG_PTR Context
426 );
427
428 VOID
429 NTAPI
430 ExSweepHandleTable(
431 IN PHANDLE_TABLE HandleTable,
432 IN PEX_SWEEP_HANDLE_CALLBACK EnumHandleProcedure,
433 IN PVOID Context
434 );
435
436 /* PSEH EXCEPTION HANDLING **************************************************/
437
438 LONG
439 NTAPI
440 ExSystemExceptionFilter(VOID);
441
442 /* CALLBACKS *****************************************************************/
443
444 FORCEINLINE
445 VOID
446 ExDoCallBack(IN OUT PEX_CALLBACK Callback,
447 IN PVOID Context,
448 IN PVOID Argument1,
449 IN PVOID Argument2)
450 {
451 PEX_CALLBACK_ROUTINE_BLOCK CallbackBlock;
452 PEX_CALLBACK_FUNCTION Function;
453
454 /* Reference the block */
455 CallbackBlock = ExReferenceCallBackBlock(Callback);
456 if (CallbackBlock)
457 {
458 /* Get the function */
459 Function = ExGetCallBackBlockRoutine(CallbackBlock);
460
461 /* Do the callback */
462 Function(Context, Argument1, Argument2);
463
464 /* Now dereference it */
465 ExDereferenceCallBackBlock(Callback, CallbackBlock);
466 }
467 }
468
469 /* FAST REFS ******************************************************************/
470
471 FORCEINLINE
472 PVOID
473 ExGetObjectFastReference(IN EX_FAST_REF FastRef)
474 {
475 /* Return the unbiased pointer */
476 return (PVOID)(FastRef.Value & ~MAX_FAST_REFS);
477 }
478
479 FORCEINLINE
480 ULONG
481 ExGetCountFastReference(IN EX_FAST_REF FastRef)
482 {
483 /* Return the reference count */
484 return FastRef.RefCnt;
485 }
486
487 FORCEINLINE
488 VOID
489 ExInitializeFastReference(OUT PEX_FAST_REF FastRef,
490 IN OPTIONAL PVOID Object)
491 {
492 /* Sanity check */
493 ASSERT((((ULONG_PTR)Object) & MAX_FAST_REFS) == 0);
494
495 /* Check if an object is being set */
496 if (!Object)
497 {
498 /* Clear the field */
499 FastRef->Object = NULL;
500 }
501 else
502 {
503 /* Otherwise, we assume the object was referenced and is ready */
504 FastRef->Value = (ULONG_PTR)Object | MAX_FAST_REFS;
505 }
506 }
507
508 FORCEINLINE
509 EX_FAST_REF
510 ExAcquireFastReference(IN OUT PEX_FAST_REF FastRef)
511 {
512 EX_FAST_REF OldValue, NewValue;
513
514 /* Start reference loop */
515 for (;;)
516 {
517 /* Get the current reference count */
518 OldValue = *FastRef;
519 if (OldValue.RefCnt)
520 {
521 /* Increase the reference count */
522 NewValue.Value = OldValue.Value - 1;
523 NewValue.Object = ExpChangePushlock(&FastRef->Object,
524 NewValue.Object,
525 OldValue.Object);
526 if (NewValue.Object != OldValue.Object) continue;
527 }
528
529 /* We are done */
530 break;
531 }
532
533 /* Return the old value */
534 return OldValue;
535 }
536
537 FORCEINLINE
538 BOOLEAN
539 ExInsertFastReference(IN OUT PEX_FAST_REF FastRef,
540 IN PVOID Object)
541 {
542 EX_FAST_REF OldValue, NewValue;
543
544 /* Sanity checks */
545 ASSERT(!(((ULONG_PTR)Object) & MAX_FAST_REFS));
546
547 /* Start update loop */
548 for (;;)
549 {
550 /* Get the current reference count */
551 OldValue = *FastRef;
552
553 /* Check if the current count is too high or if the pointer changed */
554 if (((OldValue.RefCnt + MAX_FAST_REFS) > MAX_FAST_REFS) ||
555 ((OldValue.Value &~ MAX_FAST_REFS) != (ULONG_PTR)Object))
556 {
557 /* Fail */
558 return FALSE;
559 }
560
561 /* Update the reference count */
562 NewValue.Value = OldValue.Value + MAX_FAST_REFS;
563 NewValue.Object = ExpChangePushlock(&FastRef->Object,
564 NewValue.Object,
565 OldValue.Object);
566 if (NewValue.Object != OldValue.Object) continue;
567
568 /* We are done */
569 break;
570 }
571
572 /* Return success */
573 return TRUE;
574 }
575
576 FORCEINLINE
577 BOOLEAN
578 ExReleaseFastReference(IN PEX_FAST_REF FastRef,
579 IN PVOID Object)
580 {
581 EX_FAST_REF OldValue, NewValue;
582
583 /* Sanity checks */
584 ASSERT(Object != NULL);
585 ASSERT(!(((ULONG_PTR)Object) & MAX_FAST_REFS));
586
587 /* Start reference loop */
588 for (;;)
589 {
590 /* Get the current reference count */
591 OldValue = *FastRef;
592
593 /* Check if we're full if if the pointer changed */
594 if ((OldValue.Value ^ (ULONG_PTR)Object) >= MAX_FAST_REFS) return FALSE;
595
596 /* Decrease the reference count */
597 NewValue.Value = OldValue.Value + 1;
598 NewValue.Object = ExpChangePushlock(&FastRef->Object,
599 NewValue.Object,
600 OldValue.Object);
601 if (NewValue.Object != OldValue.Object) continue;
602
603 /* We are done */
604 break;
605 }
606
607 /* Return success */
608 return TRUE;
609 }
610
611 FORCEINLINE
612 EX_FAST_REF
613 ExSwapFastReference(IN PEX_FAST_REF FastRef,
614 IN PVOID Object)
615 {
616 EX_FAST_REF NewValue, OldValue;
617
618 /* Sanity check */
619 ASSERT((((ULONG_PTR)Object) & MAX_FAST_REFS) == 0);
620
621 /* Check if an object is being set */
622 if (!Object)
623 {
624 /* Clear the field */
625 NewValue.Object = NULL;
626 }
627 else
628 {
629 /* Otherwise, we assume the object was referenced and is ready */
630 NewValue.Value = (ULONG_PTR)Object | MAX_FAST_REFS;
631 }
632
633 /* Update the object */
634 OldValue.Object = InterlockedExchangePointer(&FastRef->Object, NewValue.Object);
635 return OldValue;
636 }
637
638 FORCEINLINE
639 EX_FAST_REF
640 ExCompareSwapFastReference(IN PEX_FAST_REF FastRef,
641 IN PVOID Object,
642 IN PVOID OldObject)
643 {
644 EX_FAST_REF OldValue, NewValue;
645
646 /* Sanity check and start swap loop */
647 ASSERT(!(((ULONG_PTR)Object) & MAX_FAST_REFS));
648 for (;;)
649 {
650 /* Get the current value */
651 OldValue = *FastRef;
652
653 /* Make sure there's enough references to swap */
654 if (!((OldValue.Value ^ (ULONG_PTR)OldObject) <= MAX_FAST_REFS)) break;
655
656 /* Check if we have an object to swap */
657 if (Object)
658 {
659 /* Set up the value with maximum fast references */
660 NewValue.Value = (ULONG_PTR)Object | MAX_FAST_REFS;
661 }
662 else
663 {
664 /* Write the object address itself (which is empty) */
665 NewValue.Value = (ULONG_PTR)Object;
666 }
667
668 /* Do the actual compare exchange */
669 NewValue.Object = ExpChangePushlock(&FastRef->Object,
670 NewValue.Object,
671 OldValue.Object);
672 if (NewValue.Object != OldValue.Object) continue;
673
674 /* All done */
675 break;
676 }
677
678 /* Return the old value */
679 return OldValue;
680 }
681
682 /* RUNDOWN *******************************************************************/
683
684 /*++
685 * @name ExfAcquireRundownProtection
686 * INTERNAL MACRO
687 *
688 * The ExfAcquireRundownProtection routine acquires rundown protection for
689 * the specified descriptor.
690 *
691 * @param RunRef
692 * Pointer to a rundown reference descriptor.
693 *
694 * @return TRUE if access to the protected structure was granted, FALSE otherwise.
695 *
696 * @remarks This is the internal macro for system use only.In case the rundown
697 * was active, then the slow-path will be called through the exported
698 * function.
699 *
700 *--*/
701 FORCEINLINE
702 BOOLEAN
703 _ExAcquireRundownProtection(IN PEX_RUNDOWN_REF RunRef)
704 {
705 ULONG_PTR Value, NewValue;
706
707 /* Get the current value and mask the active bit */
708 Value = RunRef->Count &~ EX_RUNDOWN_ACTIVE;
709
710 /* Add a reference */
711 NewValue = Value + EX_RUNDOWN_COUNT_INC;
712
713 /* Change the value */
714 NewValue = ExpChangeRundown(RunRef, NewValue, Value);
715 if (NewValue != Value)
716 {
717 /* Rundown was active, use long path */
718 return ExfAcquireRundownProtection(RunRef);
719 }
720
721 /* Success */
722 return TRUE;
723 }
724
725 /*++
726 * @name ExReleaseRundownProtection
727 * INTERNAL MACRO
728 *
729 * The ExReleaseRundownProtection routine releases rundown protection for
730 * the specified descriptor.
731 *
732 * @param RunRef
733 * Pointer to a rundown reference descriptor.
734 *
735 * @return TRUE if access to the protected structure was granted, FALSE otherwise.
736 *
737 * @remarks This is the internal macro for system use only.In case the rundown
738 * was active, then the slow-path will be called through the exported
739 * function.
740 *
741 *--*/
742 FORCEINLINE
743 VOID
744 _ExReleaseRundownProtection(IN PEX_RUNDOWN_REF RunRef)
745 {
746 ULONG_PTR Value, NewValue;
747
748 /* Get the current value and mask the active bit */
749 Value = RunRef->Count &~ EX_RUNDOWN_ACTIVE;
750
751 /* Remove a reference */
752 NewValue = Value - EX_RUNDOWN_COUNT_INC;
753
754 /* Change the value */
755 NewValue = ExpChangeRundown(RunRef, NewValue, Value);
756
757 /* Check if the rundown was active */
758 if (NewValue != Value)
759 {
760 /* Rundown was active, use long path */
761 ExfReleaseRundownProtection(RunRef);
762 }
763 else
764 {
765 /* Sanity check */
766 ASSERT((Value >= EX_RUNDOWN_COUNT_INC) || (KeNumberProcessors > 1));
767 }
768 }
769
770 /*++
771 * @name ExInitializeRundownProtection
772 * INTERNAL MACRO
773 *
774 * The ExInitializeRundownProtection routine initializes a rundown
775 * protection descriptor.
776 *
777 * @param RunRef
778 * Pointer to a rundown reference descriptor.
779 *
780 * @return None.
781 *
782 * @remarks This is the internal macro for system use only.
783 *
784 *--*/
785 FORCEINLINE
786 VOID
787 _ExInitializeRundownProtection(IN PEX_RUNDOWN_REF RunRef)
788 {
789 /* Set the count to zero */
790 RunRef->Count = 0;
791 }
792
793 /*++
794 * @name ExWaitForRundownProtectionRelease
795 * INTERNAL MACRO
796 *
797 * The ExWaitForRundownProtectionRelease routine waits until the specified
798 * rundown descriptor has been released.
799 *
800 * @param RunRef
801 * Pointer to a rundown reference descriptor.
802 *
803 * @return None.
804 *
805 * @remarks This is the internal macro for system use only. If a wait is actually
806 * necessary, then the slow path is taken through the exported function.
807 *
808 *--*/
809 FORCEINLINE
810 VOID
811 _ExWaitForRundownProtectionRelease(IN PEX_RUNDOWN_REF RunRef)
812 {
813 ULONG_PTR Value;
814
815 /* Set the active bit */
816 Value = ExpChangeRundown(RunRef, EX_RUNDOWN_ACTIVE, 0);
817 if ((Value) && (Value != EX_RUNDOWN_ACTIVE))
818 {
819 /* If the the rundown wasn't already active, then take the long path */
820 ExfWaitForRundownProtectionRelease(RunRef);
821 }
822 }
823
824 /*++
825 * @name ExRundownCompleted
826 * INTERNAL MACRO
827 *
828 * The ExRundownCompleted routine completes the rundown of the specified
829 * descriptor by setting the active bit.
830 *
831 * @param RunRef
832 * Pointer to a rundown reference descriptor.
833 *
834 * @return None.
835 *
836 * @remarks This is the internal macro for system use only.
837 *
838 *--*/
839 FORCEINLINE
840 VOID
841 _ExRundownCompleted(IN PEX_RUNDOWN_REF RunRef)
842 {
843 /* Sanity check */
844 ASSERT((RunRef->Count & EX_RUNDOWN_ACTIVE) != 0);
845
846 /* Mark the counter as active */
847 ExpSetRundown(&RunRef->Count, EX_RUNDOWN_ACTIVE);
848 }
849
850 /* PUSHLOCKS *****************************************************************/
851
852 /* FIXME: VERIFY THESE! */
853
854 VOID
855 FASTCALL
856 ExBlockPushLock(
857 IN PEX_PUSH_LOCK PushLock,
858 IN PVOID WaitBlock
859 );
860
861 VOID
862 FASTCALL
863 ExfUnblockPushLock(
864 IN PEX_PUSH_LOCK PushLock,
865 IN PVOID CurrentWaitBlock
866 );
867
868 VOID
869 FASTCALL
870 ExWaitForUnblockPushLock(
871 IN PEX_PUSH_LOCK PushLock,
872 IN PVOID WaitBlock
873 );
874
875 /*++
876 * @name ExInitializePushLock
877 * INTERNAL MACRO
878 *
879 * The ExInitializePushLock macro initializes a PushLock.
880 *
881 * @params PushLock
882 * Pointer to the pushlock which is to be initialized.
883 *
884 * @return None.
885 *
886 * @remarks None.
887 *
888 *--*/
889 FORCEINLINE
890 VOID
891 ExInitializePushLock(IN PULONG_PTR PushLock)
892 {
893 /* Set the value to 0 */
894 *PushLock = 0;
895 }
896
897 /*++
898 * @name ExAcquirePushLockExclusive
899 * INTERNAL MACRO
900 *
901 * The ExAcquirePushLockExclusive macro exclusively acquires a PushLock.
902 *
903 * @params PushLock
904 * Pointer to the pushlock which is to be acquired.
905 *
906 * @return None.
907 *
908 * @remarks The function attempts the quickest route to acquire the lock, which is
909 * to simply set the lock bit.
910 * However, if the pushlock is already shared, the slower path is taken.
911 *
912 * Callers of ExAcquirePushLockShared must be running at IRQL <= APC_LEVEL.
913 * This macro should usually be paired up with KeAcquireCriticalRegion.
914 *
915 *--*/
916 FORCEINLINE
917 VOID
918 ExAcquirePushLockExclusive(PEX_PUSH_LOCK PushLock)
919 {
920 /* Try acquiring the lock */
921 if (InterlockedBitTestAndSet((PLONG)PushLock, EX_PUSH_LOCK_LOCK_V))
922 {
923 /* Someone changed it, use the slow path */
924 ExfAcquirePushLockExclusive(PushLock);
925 }
926
927 /* Sanity check */
928 ASSERT(PushLock->Locked);
929 }
930
931 /*++
932 * @name ExTryToAcquirePushLockExclusive
933 * INTERNAL MACRO
934 *
935 * The ExAcquirePushLockExclusive macro exclusively acquires a PushLock.
936 *
937 * @params PushLock
938 * Pointer to the pushlock which is to be acquired.
939 *
940 * @return None.
941 *
942 * @remarks The function attempts the quickest route to acquire the lock, which is
943 * to simply set the lock bit.
944 * However, if the pushlock is already shared, the slower path is taken.
945 *
946 * Callers of ExAcquirePushLockShared must be running at IRQL <= APC_LEVEL.
947 * This macro should usually be paired up with KeAcquireCriticalRegion.
948 *
949 *--*/
950 FORCEINLINE
951 BOOLEAN
952 ExTryToAcquirePushLockExclusive(PEX_PUSH_LOCK PushLock)
953 {
954 /* Try acquiring the lock */
955 if (InterlockedBitTestAndSet((PLONG)PushLock, EX_PUSH_LOCK_LOCK_V))
956 {
957 /* Can't acquire */
958 return FALSE;
959 }
960
961 /* Got acquired */
962 ASSERT (PushLock->Locked);
963 return TRUE;
964 }
965
966 /*++
967 * @name ExAcquirePushLockShared
968 * INTERNAL MACRO
969 *
970 * The ExAcquirePushLockShared macro acquires a shared PushLock.
971 *
972 * @params PushLock
973 * Pointer to the pushlock which is to be acquired.
974 *
975 * @return None.
976 *
977 * @remarks The function attempts the quickest route to acquire the lock, which is
978 * to simply set the lock bit and set the share count to one.
979 * However, if the pushlock is already shared, the slower path is taken.
980 *
981 * Callers of ExAcquirePushLockShared must be running at IRQL <= APC_LEVEL.
982 * This macro should usually be paired up with KeAcquireCriticalRegion.
983 *
984 *--*/
985 FORCEINLINE
986 VOID
987 ExAcquirePushLockShared(PEX_PUSH_LOCK PushLock)
988 {
989 EX_PUSH_LOCK NewValue;
990
991 /* Try acquiring the lock */
992 NewValue.Value = EX_PUSH_LOCK_LOCK | EX_PUSH_LOCK_SHARE_INC;
993 if (ExpChangePushlock(PushLock, NewValue.Ptr, 0))
994 {
995 /* Someone changed it, use the slow path */
996 ExfAcquirePushLockShared(PushLock);
997 }
998
999 /* Sanity checks */
1000 ASSERT(PushLock->Locked);
1001 ASSERT(PushLock->Waiting || PushLock->Shared > 0);
1002 }
1003
1004 /*++
1005 * @name ExConvertPushLockSharedToExclusive
1006 * INTERNAL MACRO
1007 *
1008 * The ExConvertPushLockSharedToExclusive macro converts an exclusive
1009 * pushlock to a shared pushlock.
1010 *
1011 * @params PushLock
1012 * Pointer to the pushlock which is to be converted.
1013 *
1014 * @return FALSE if conversion failed, TRUE otherwise.
1015 *
1016 * @remarks The function attempts the quickest route to convert the lock, which is
1017 * to simply set the lock bit and remove any other bits.
1018 *
1019 *--*/
1020 FORCEINLINE
1021 BOOLEAN
1022 ExConvertPushLockSharedToExclusive(IN PEX_PUSH_LOCK PushLock)
1023 {
1024 EX_PUSH_LOCK OldValue;
1025
1026 /* Set the expected old value */
1027 OldValue.Value = EX_PUSH_LOCK_LOCK | EX_PUSH_LOCK_SHARE_INC;
1028
1029 /* Try converting the lock */
1030 if (ExpChangePushlock(PushLock, EX_PUSH_LOCK_LOCK, OldValue.Value) !=
1031 OldValue.Ptr)
1032 {
1033 /* Conversion failed */
1034 return FALSE;
1035 }
1036
1037 /* Sanity check */
1038 ASSERT(PushLock->Locked);
1039 return TRUE;
1040 }
1041
1042 /*++
1043 * @name ExWaitOnPushLock
1044 * INTERNAL MACRO
1045 *
1046 * The ExWaitOnPushLock macro acquires and instantly releases a pushlock.
1047 *
1048 * @params PushLock
1049 * Pointer to a pushlock.
1050 *
1051 * @return None.
1052 *
1053 * @remarks The function attempts to get any exclusive waiters out of their slow
1054 * path by forcing an instant acquire/release operation.
1055 *
1056 * Callers of ExWaitOnPushLock must be running at IRQL <= APC_LEVEL.
1057 *
1058 *--*/
1059 FORCEINLINE
1060 VOID
1061 ExWaitOnPushLock(PEX_PUSH_LOCK PushLock)
1062 {
1063 /* Check if we're locked */
1064 if (PushLock->Locked)
1065 {
1066 /* Acquire the lock */
1067 ExfAcquirePushLockExclusive(PushLock);
1068 ASSERT(PushLock->Locked);
1069
1070 /* Release it */
1071 ExfReleasePushLockExclusive(PushLock);
1072 }
1073 }
1074
1075 /*++
1076 * @name ExReleasePushLockShared
1077 * INTERNAL MACRO
1078 *
1079 * The ExReleasePushLockShared macro releases a previously acquired PushLock.
1080 *
1081 * @params PushLock
1082 * Pointer to a previously acquired pushlock.
1083 *
1084 * @return None.
1085 *
1086 * @remarks The function attempts the quickest route to release the lock, which is
1087 * to simply decrease the share count and remove the lock bit.
1088 * However, if the pushlock is being waited on then the long path is taken.
1089 *
1090 * Callers of ExReleasePushLockShared must be running at IRQL <= APC_LEVEL.
1091 * This macro should usually be paired up with KeLeaveCriticalRegion.
1092 *
1093 *--*/
1094 FORCEINLINE
1095 VOID
1096 ExReleasePushLockShared(PEX_PUSH_LOCK PushLock)
1097 {
1098 EX_PUSH_LOCK OldValue;
1099
1100 /* Sanity checks */
1101 ASSERT(PushLock->Locked);
1102 ASSERT(PushLock->Waiting || PushLock->Shared > 0);
1103
1104 /* Try to clear the pushlock */
1105 OldValue.Value = EX_PUSH_LOCK_LOCK | EX_PUSH_LOCK_SHARE_INC;
1106 if (ExpChangePushlock(PushLock, 0, OldValue.Ptr) != OldValue.Ptr)
1107 {
1108 /* There are still other people waiting on it */
1109 ExfReleasePushLockShared(PushLock);
1110 }
1111 }
1112
1113 /*++
1114 * @name ExReleasePushLockExclusive
1115 * INTERNAL MACRO
1116 *
1117 * The ExReleasePushLockExclusive macro releases a previously
1118 * exclusively acquired PushLock.
1119 *
1120 * @params PushLock
1121 * Pointer to a previously acquired pushlock.
1122 *
1123 * @return None.
1124 *
1125 * @remarks The function attempts the quickest route to release the lock, which is
1126 * to simply clear the locked bit.
1127 * However, if the pushlock is being waited on, the slow path is taken
1128 * in an attempt to wake up the lock.
1129 *
1130 * Callers of ExReleasePushLockExclusive must be running at IRQL <= APC_LEVEL.
1131 * This macro should usually be paired up with KeLeaveCriticalRegion.
1132 *
1133 *--*/
1134 FORCEINLINE
1135 VOID
1136 ExReleasePushLockExclusive(PEX_PUSH_LOCK PushLock)
1137 {
1138 EX_PUSH_LOCK OldValue;
1139
1140 /* Sanity checks */
1141 ASSERT(PushLock->Locked);
1142 ASSERT(PushLock->Waiting || PushLock->Shared == 0);
1143
1144 /* Unlock the pushlock */
1145 OldValue.Value = InterlockedExchangeAddSizeT((PSIZE_T)PushLock,
1146 -(SIZE_T)EX_PUSH_LOCK_LOCK);
1147
1148 /* Sanity checks */
1149 ASSERT(OldValue.Locked);
1150 ASSERT(OldValue.Waiting || OldValue.Shared == 0);
1151
1152 /* Check if anyone is waiting on it and it's not already waking*/
1153 if ((OldValue.Waiting) && !(OldValue.Waking))
1154 {
1155 /* Wake it up */
1156 ExfTryToWakePushLock(PushLock);
1157 }
1158 }
1159
1160 /*++
1161 * @name ExReleasePushLock
1162 * INTERNAL MACRO
1163 *
1164 * The ExReleasePushLock macro releases a previously acquired PushLock.
1165 *
1166 * @params PushLock
1167 * Pointer to a previously acquired pushlock.
1168 *
1169 * @return None.
1170 *
1171 * @remarks The function attempts the quickest route to release the lock, which is
1172 * to simply clear all the fields and decrease the share count if required.
1173 * However, if the pushlock is being waited on then the long path is taken.
1174 *
1175 * Callers of ExReleasePushLock must be running at IRQL <= APC_LEVEL.
1176 * This macro should usually be paired up with KeLeaveCriticalRegion.
1177 *
1178 *--*/
1179 FORCEINLINE
1180 VOID
1181 ExReleasePushLock(PEX_PUSH_LOCK PushLock)
1182 {
1183 EX_PUSH_LOCK OldValue = *PushLock;
1184 EX_PUSH_LOCK NewValue;
1185
1186 /* Sanity checks */
1187 ASSERT(OldValue.Locked);
1188
1189 /* Check if the pushlock is shared */
1190 if (OldValue.Shared > 1)
1191 {
1192 /* Decrease the share count */
1193 NewValue.Value = OldValue.Value - EX_PUSH_LOCK_SHARE_INC;
1194 }
1195 else
1196 {
1197 /* Clear the pushlock entirely */
1198 NewValue.Value = 0;
1199 }
1200
1201 /* Check if nobody is waiting on us and try clearing the lock here */
1202 if ((OldValue.Waiting) ||
1203 (ExpChangePushlock(PushLock, NewValue.Ptr, OldValue.Ptr) !=
1204 OldValue.Ptr))
1205 {
1206 /* We have waiters, use the long path */
1207 ExfReleasePushLock(PushLock);
1208 }
1209 }
1210
1211 /* FAST MUTEX INLINES *********************************************************/
1212
1213 FORCEINLINE
1214 VOID
1215 _ExAcquireFastMutexUnsafe(IN PFAST_MUTEX FastMutex)
1216 {
1217 PKTHREAD Thread = KeGetCurrentThread();
1218
1219 /* Sanity check */
1220 ASSERT((KeGetCurrentIrql() == APC_LEVEL) ||
1221 (Thread->CombinedApcDisable != 0) ||
1222 (Thread->Teb == NULL) ||
1223 (Thread->Teb >= (PTEB)MM_SYSTEM_RANGE_START));
1224 ASSERT(FastMutex->Owner != Thread);
1225
1226 /* Decrease the count */
1227 if (InterlockedDecrement(&FastMutex->Count))
1228 {
1229 /* Someone is still holding it, use slow path */
1230 KiAcquireFastMutex(FastMutex);
1231 }
1232
1233 /* Set the owner */
1234 FastMutex->Owner = Thread;
1235 }
1236
1237 FORCEINLINE
1238 VOID
1239 _ExReleaseFastMutexUnsafe(IN OUT PFAST_MUTEX FastMutex)
1240 {
1241 ASSERT((KeGetCurrentIrql() == APC_LEVEL) ||
1242 (KeGetCurrentThread()->CombinedApcDisable != 0) ||
1243 (KeGetCurrentThread()->Teb == NULL) ||
1244 (KeGetCurrentThread()->Teb >= (PTEB)MM_SYSTEM_RANGE_START));
1245 ASSERT(FastMutex->Owner == KeGetCurrentThread());
1246
1247 /* Erase the owner */
1248 FastMutex->Owner = NULL;
1249
1250 /* Increase the count */
1251 if (InterlockedIncrement(&FastMutex->Count) <= 0)
1252 {
1253 /* Someone was waiting for it, signal the waiter */
1254 KeSetEventBoostPriority(&FastMutex->Event, NULL);
1255 }
1256 }
1257
1258 FORCEINLINE
1259 VOID
1260 _ExAcquireFastMutex(IN PFAST_MUTEX FastMutex)
1261 {
1262 KIRQL OldIrql;
1263 ASSERT(KeGetCurrentIrql() <= APC_LEVEL);
1264
1265 /* Raise IRQL to APC */
1266 KeRaiseIrql(APC_LEVEL, &OldIrql);
1267
1268 /* Decrease the count */
1269 if (InterlockedDecrement(&FastMutex->Count))
1270 {
1271 /* Someone is still holding it, use slow path */
1272 KiAcquireFastMutex(FastMutex);
1273 }
1274
1275 /* Set the owner and IRQL */
1276 FastMutex->Owner = KeGetCurrentThread();
1277 FastMutex->OldIrql = OldIrql;
1278 }
1279
1280 FORCEINLINE
1281 VOID
1282 _ExReleaseFastMutex(IN OUT PFAST_MUTEX FastMutex)
1283 {
1284 KIRQL OldIrql;
1285 ASSERT(KeGetCurrentIrql() == APC_LEVEL);
1286
1287 /* Erase the owner */
1288 FastMutex->Owner = NULL;
1289 OldIrql = (KIRQL)FastMutex->OldIrql;
1290
1291 /* Increase the count */
1292 if (InterlockedIncrement(&FastMutex->Count) <= 0)
1293 {
1294 /* Someone was waiting for it, signal the waiter */
1295 KeSetEventBoostPriority(&FastMutex->Event, NULL);
1296 }
1297
1298 /* Lower IRQL back */
1299 KeLowerIrql(OldIrql);
1300 }
1301
1302 FORCEINLINE
1303 BOOLEAN
1304 _ExTryToAcquireFastMutex(IN OUT PFAST_MUTEX FastMutex)
1305 {
1306 KIRQL OldIrql;
1307 ASSERT(KeGetCurrentIrql() <= APC_LEVEL);
1308
1309 /* Raise to APC_LEVEL */
1310 KeRaiseIrql(APC_LEVEL, &OldIrql);
1311
1312 /* Check if we can quickly acquire it */
1313 if (InterlockedCompareExchange(&FastMutex->Count, 0, 1) == 1)
1314 {
1315 /* We have, set us as owners */
1316 FastMutex->Owner = KeGetCurrentThread();
1317 FastMutex->OldIrql = OldIrql;
1318 return TRUE;
1319 }
1320 else
1321 {
1322 /* Acquire attempt failed */
1323 KeLowerIrql(OldIrql);
1324 YieldProcessor();
1325 return FALSE;
1326 }
1327 }
1328
1329 FORCEINLINE
1330 VOID
1331 _ExEnterCriticalRegionAndAcquireFastMutexUnsafe(IN OUT PFAST_MUTEX FastMutex)
1332 {
1333 /* Enter the Critical Region */
1334 KeEnterCriticalRegion();
1335
1336 /* Acquire the mutex unsafely */
1337 _ExAcquireFastMutexUnsafe(FastMutex);
1338 }
1339
1340 FORCEINLINE
1341 VOID
1342 _ExReleaseFastMutexUnsafeAndLeaveCriticalRegion(IN OUT PFAST_MUTEX FastMutex)
1343 {
1344 /* Release the mutex unsafely */
1345 _ExReleaseFastMutexUnsafe(FastMutex);
1346
1347 /* Leave the critical region */
1348 KeLeaveCriticalRegion();
1349 }
1350
1351 /* OTHER FUNCTIONS **********************************************************/
1352
1353 BOOLEAN
1354 NTAPI
1355 ExTryToAcquireResourceExclusiveLite(
1356 IN PERESOURCE Resource
1357 );
1358
1359 NTSTATUS
1360 ExpSetTimeZoneInformation(PTIME_ZONE_INFORMATION TimeZoneInformation);
1361
1362 BOOLEAN
1363 NTAPI
1364 ExAcquireTimeRefreshLock(BOOLEAN Wait);
1365
1366 VOID
1367 NTAPI
1368 ExReleaseTimeRefreshLock(VOID);
1369
1370 VOID
1371 NTAPI
1372 ExUpdateSystemTimeFromCmos(IN BOOLEAN UpdateInterruptTime,
1373 IN ULONG MaxSepInSeconds);
1374
1375 NTSTATUS
1376 NTAPI
1377 ExpAllocateLocallyUniqueId(OUT LUID *LocallyUniqueId);
1378
1379 VOID
1380 NTAPI
1381 ExTimerRundown(VOID);
1382
1383 VOID
1384 NTAPI
1385 HeadlessInit(
1386 IN PLOADER_PARAMETER_BLOCK LoaderBlock
1387 );
1388
1389 VOID
1390 NTAPI
1391 XIPInit(
1392 IN PLOADER_PARAMETER_BLOCK LoaderBlock
1393 );
1394
1395 #define InterlockedDecrementUL(Addend) \
1396 (ULONG)InterlockedDecrement((PLONG)(Addend))
1397
1398 #define InterlockedIncrementUL(Addend) \
1399 (ULONG)InterlockedIncrement((PLONG)(Addend))
1400
1401 #define InterlockedExchangeUL(Target, Value) \
1402 (ULONG)InterlockedExchange((PLONG)(Target), (LONG)(Value))
1403
1404 #define InterlockedExchangeAddUL(Addend, Value) \
1405 (ULONG)InterlockedExchangeAdd((PLONG)(Addend), (LONG)(Value))
1406
1407 #define InterlockedCompareExchangeUL(Destination, Exchange, Comperand) \
1408 (ULONG)InterlockedCompareExchange((PLONG)(Destination), (LONG)(Exchange), (LONG)(Comperand))
1409
1410 #define ExfInterlockedCompareExchange64UL(Destination, Exchange, Comperand) \
1411 (ULONGLONG)ExfInterlockedCompareExchange64((PLONGLONG)(Destination), (PLONGLONG)(Exchange), (PLONGLONG)(Comperand))