reactos/ntoskrnl/ke/thrdobj.c

   1 /*
   2  * PROJECT:         ReactOS Kernel
   3  * LICENSE:         GPL - See COPYING in the top level directory
   4  * FILE:            ntoskrnl/ke/thrdobj.c
   5  * PURPOSE:         Implements routines to manage the Kernel Thread Object
   6  * PROGRAMMERS:     Alex Ionescu (alex.ionescu@reactos.org)
   7  */
   8
   9 /* INCLUDES ******************************************************************/
  10
  11 #include <ntoskrnl.h>
  12 #define NDEBUG
  13 #include <internal/debug.h>
  14
  15 extern EX_WORK_QUEUE ExWorkerQueue[MaximumWorkQueue];
  16 extern LIST_ENTRY PspReaperListHead;
  17
  18 ULONG KiMask32Array[MAXIMUM_PRIORITY] =
  19 {
  20     0x1,        0x2,       0x4,       0x8,       0x10,       0x20,
  21     0x40,       0x80,      0x100,     0x200,     0x4000,     0x800,
  22     0x1000,     0x2000,    0x4000,    0x8000,    0x10000,    0x20000,
  23     0x40000,    0x80000,   0x100000,  0x200000,  0x400000,   0x800000,
  24     0x1000000,  0x2000000, 0x4000000, 0x8000000, 0x10000000, 0x20000000,
  25     0x40000000, 0x80000000
  26 };
  27
  28 /* FUNCTIONS *****************************************************************/
  29
  30 UCHAR
  31 NTAPI
  32 KeFindNextRightSetAffinity(IN UCHAR Number,
  33                            IN ULONG Set)
  34 {
  35     ULONG Bit, Result;
  36     ASSERT(Set != 0);
  37
  38     /* Calculate the mask */
  39     Bit = (AFFINITY_MASK(Number) - 1) & Set;
  40
  41     /* If it's 0, use the one we got */
  42     if (!Bit) Bit = Set;
  43
  44     /* Now find the right set and return it */
  45     BitScanReverse(&Result, Bit);
  46     return (UCHAR)Result;
  47 }
  48
  49 KPRIORITY
  50 NTAPI
  51 KeQueryBasePriorityThread(IN PKTHREAD Thread)
  52 {
  53     LONG BaseIncrement;
  54     KIRQL OldIrql;
  55     PKPROCESS Process;
  56     ASSERT_THREAD(Thread);
  57     ASSERT_IRQL_LESS_OR_EQUAL(DISPATCH_LEVEL);
  58
  59     /* Raise IRQL to synch level */
  60     OldIrql = KeRaiseIrqlToSynchLevel();
  61
  62     /* Lock the thread */
  63     KiAcquireThreadLock(Thread);
  64
  65     /* Get the Process */
  66     Process = Thread->ApcStatePointer[0]->Process;
  67
  68     /* Calculate the base increment */
  69     BaseIncrement = Thread->BasePriority - Process->BasePriority;
  70
  71     /* If saturation occured, return the saturation increment instead */
  72     if (Thread->Saturation) BaseIncrement = (HIGH_PRIORITY + 1) / 2 *
  73                                             Thread->Saturation;
  74
  75     /* Release thread lock */
  76     KiReleaseThreadLock(Thread);
  77
  78     /* Lower IRQl and return Increment */
  79     KeLowerIrql(OldIrql);
  80     return BaseIncrement;
  81 }
  82
  83 ULONG
  84 NTAPI
  85 KeAlertResumeThread(IN PKTHREAD Thread)
  86 {
  87     ULONG PreviousCount;
  88     KLOCK_QUEUE_HANDLE ApcLock;
  89     ASSERT_THREAD(Thread);
  90     ASSERT_IRQL_LESS_OR_EQUAL(DISPATCH_LEVEL);
  91
  92     /* Lock the Dispatcher Database and the APC Queue */
  93     KiAcquireApcLock(Thread, &ApcLock);
  94     KiAcquireDispatcherLockAtDpcLevel();
  95
  96     /* Return if Thread is already alerted. */
  97     if (!Thread->Alerted[KernelMode])
  98     {
  99         /* If it's Blocked, unblock if it we should */
 100         if ((Thread->State == Waiting) && (Thread->Alertable))
 101         {
 102             /* Abort the wait */
 103             KiAbortWaitThread(Thread, STATUS_ALERTED, THREAD_ALERT_INCREMENT);
 104         }
 105         else
 106         {
 107             /* If not, simply Alert it */
 108             Thread->Alerted[KernelMode] = TRUE;
 109         }
 110     }
 111
 112     /* Save the old Suspend Count */
 113     PreviousCount = Thread->SuspendCount;
 114
 115     /* If the thread is suspended, decrease one of the suspend counts */
 116     if (PreviousCount)
 117     {
 118         /* Decrease count. If we are now zero, unwait it completely */
 119         if (--Thread->SuspendCount)
 120         {
 121             /* Signal and satisfy */
 122             Thread->SuspendSemaphore.Header.SignalState++;
 123             KiWaitTest(&Thread->SuspendSemaphore.Header, IO_NO_INCREMENT);
 124         }
 125     }
 126
 127     /* Release Locks and return the Old State */
 128     KiReleaseDispatcherLockFromDpcLevel();
 129     KiReleaseApcLockFromDpcLevel(&ApcLock);
 130     KiExitDispatcher(ApcLock.OldIrql);
 131     return PreviousCount;
 132 }
 133
 134 BOOLEAN
 135 NTAPI
 136 KeAlertThread(IN PKTHREAD Thread,
 137               IN KPROCESSOR_MODE AlertMode)
 138 {
 139     BOOLEAN PreviousState;
 140     KLOCK_QUEUE_HANDLE ApcLock;
 141     ASSERT_THREAD(Thread);
 142     ASSERT_IRQL_LESS_OR_EQUAL(DISPATCH_LEVEL);
 143
 144     /* Lock the Dispatcher Database and the APC Queue */
 145     KiAcquireApcLock(Thread, &ApcLock);
 146     KiAcquireDispatcherLockAtDpcLevel();
 147
 148     /* Save the Previous State */
 149     PreviousState = Thread->Alerted[AlertMode];
 150
 151     /* Check if it's already alerted */
 152     if (!PreviousState)
 153     {
 154         /* Check if the thread is alertable, and blocked in the given mode */
 155         if ((Thread->State == Waiting) &&
 156             ((AlertMode == KernelMode) || (Thread->WaitMode == AlertMode)) &&
 157             (Thread->Alertable))
 158         {
 159             /* Abort the wait to alert the thread */
 160             KiAbortWaitThread(Thread, STATUS_ALERTED, THREAD_ALERT_INCREMENT);
 161         }
 162         else
 163         {
 164             /* Otherwise, merely set the alerted state */
 165             Thread->Alerted[AlertMode] = TRUE;
 166         }
 167     }
 168
 169     /* Release the Dispatcher Lock */
 170     KiReleaseDispatcherLockFromDpcLevel();
 171     KiReleaseApcLockFromDpcLevel(&ApcLock);
 172     KiExitDispatcher(ApcLock.OldIrql);
 173
 174     /* Return the old state */
 175     return PreviousState;
 176 }
 177
 178 ULONG
 179 NTAPI
 180 KeForceResumeThread(IN PKTHREAD Thread)
 181 {
 182     KLOCK_QUEUE_HANDLE ApcLock;
 183     ULONG PreviousCount;
 184     ASSERT_THREAD(Thread);
 185     ASSERT_IRQL_LESS_OR_EQUAL(DISPATCH_LEVEL);
 186
 187     /* Lock the APC Queue */
 188     KiAcquireApcLock(Thread, &ApcLock);
 189
 190     /* Save the old Suspend Count */
 191     PreviousCount = Thread->SuspendCount + Thread->FreezeCount;
 192
 193     /* If the thread is suspended, wake it up!!! */
 194     if (PreviousCount)
 195     {
 196         /* Unwait it completely */
 197         Thread->SuspendCount = 0;
 198         Thread->FreezeCount = 0;
 199
 200         /* Lock the dispatcher */
 201         KiAcquireDispatcherLockAtDpcLevel();
 202
 203         /* Signal and satisfy */
 204         Thread->SuspendSemaphore.Header.SignalState++;
 205         KiWaitTest(&Thread->SuspendSemaphore.Header, IO_NO_INCREMENT);
 206
 207         /* Release the dispatcher */
 208         KiReleaseDispatcherLockFromDpcLevel();
 209     }
 210
 211     /* Release Lock and return the Old State */
 212     KiReleaseApcLockFromDpcLevel(&ApcLock);
 213     KiExitDispatcher(ApcLock.OldIrql);
 214     return PreviousCount;
 215 }
 216
 217 /*
 218  * Used by the debugging code to freeze all the process's threads
 219  * while the debugger is examining their state.
 220  */
 221 VOID
 222 NTAPI
 223 KeFreezeAllThreads(IN PKPROCESS Process)
 224 {
 225     KLOCK_QUEUE_HANDLE LockHandle, ApcLock;
 226     PKTHREAD Current, CurrentThread = KeGetCurrentThread();
 227     PLIST_ENTRY ListHead, NextEntry;
 228     LONG OldCount;
 229     ASSERT_IRQL_LESS_OR_EQUAL(DISPATCH_LEVEL);
 230
 231     /* Lock the process */
 232     KiAcquireProcessLock(Process, &LockHandle);
 233
 234     /* If someone is already trying to free us, try again */
 235     while (CurrentThread->FreezeCount)
 236     {
 237         /* Release and re-acquire the process lock so the APC will go through */
 238         KiReleaseProcessLock(&LockHandle);
 239         KiAcquireProcessLock(Process, &LockHandle);
 240     }
 241
 242     /* Enter a critical region */
 243     KeEnterCriticalRegion();
 244
 245     /* Loop the Process's Threads */
 246     ListHead = &Process->ThreadListHead;
 247     NextEntry = ListHead->Flink;
 248     while (NextEntry != ListHead)
 249     {
 250         /* Get the current thread */
 251         Current = CONTAINING_RECORD(NextEntry, KTHREAD, ThreadListEntry);
 252
 253         /* Lock it */
 254         KiAcquireApcLockAtDpcLevel(Current, &ApcLock);
 255
 256         /* Make sure it's not ours, and check if APCs are enabled */
 257         if ((Current != CurrentThread) && (Current->ApcQueueable))
 258         {
 259             /* Sanity check */
 260             OldCount = Current->SuspendCount;
 261             ASSERT(OldCount != MAXIMUM_SUSPEND_COUNT);
 262
 263             /* Increase the freeze count */
 264             Current->FreezeCount++;
 265
 266             /* Make sure it wasn't already suspended */
 267             if (!(OldCount) && !(Current->SuspendCount))
 268             {
 269                 /* Did we already insert it? */
 270                 if (!Current->SuspendApc.Inserted)
 271                 {
 272                     /* Insert the APC */
 273                     Current->SuspendApc.Inserted = TRUE;
 274                     KiInsertQueueApc(&Current->SuspendApc, IO_NO_INCREMENT);
 275                 }
 276                 else
 277                 {
 278                     /* Lock the dispatcher */
 279                     KiAcquireDispatcherLockAtDpcLevel();
 280
 281                     /* Unsignal the semaphore, the APC was already inserted */
 282                     Current->SuspendSemaphore.Header.SignalState--;
 283
 284                     /* Release the dispatcher */
 285                     KiReleaseDispatcherLockFromDpcLevel();
 286                 }
 287             }
 288         }
 289
 290         /* Release the APC lock */
 291         KiReleaseApcLockFromDpcLevel(&ApcLock);
 292     }
 293
 294     /* Release the process lock and exit the dispatcher */
 295     KiReleaseProcessLock(&LockHandle);
 296     KiExitDispatcher(LockHandle.OldIrql);
 297 }
 298
 299 ULONG
 300 NTAPI
 301 KeResumeThread(IN PKTHREAD Thread)
 302 {
 303     KLOCK_QUEUE_HANDLE ApcLock;
 304     ULONG PreviousCount;
 305     ASSERT_THREAD(Thread);
 306     ASSERT_IRQL_LESS_OR_EQUAL(DISPATCH_LEVEL);
 307
 308     /* Lock the APC Queue */
 309     KiAcquireApcLock(Thread, &ApcLock);
 310
 311     /* Save the Old Count */
 312     PreviousCount = Thread->SuspendCount;
 313
 314     /* Check if it existed */
 315     if (PreviousCount)
 316     {
 317         /* Decrease the suspend count */
 318         Thread->SuspendCount--;
 319
 320         /* Check if the thrad is still suspended or not */
 321         if ((!Thread->SuspendCount) && (!Thread->FreezeCount))
 322         {
 323             /* Acquire the dispatcher lock */
 324             KiAcquireDispatcherLockAtDpcLevel();
 325
 326             /* Signal the Suspend Semaphore */
 327             Thread->SuspendSemaphore.Header.SignalState++;
 328             KiWaitTest(&Thread->SuspendSemaphore.Header, IO_NO_INCREMENT);
 329
 330             /* Release the dispatcher lock */
 331             KiReleaseDispatcherLockFromDpcLevel();
 332         }
 333     }
 334
 335     /* Release APC Queue lock and return the Old State */
 336     KiReleaseApcLockFromDpcLevel(&ApcLock);
 337     KiExitDispatcher(ApcLock.OldIrql);
 338     return PreviousCount;
 339 }
 340
 341 VOID
 342 NTAPI
 343 KeRundownThread(VOID)
 344 {
 345     KIRQL OldIrql;
 346     PKTHREAD Thread = KeGetCurrentThread();
 347     PLIST_ENTRY NextEntry, ListHead;
 348     PKMUTANT Mutant;
 349
 350     /* Optimized path if nothing is on the list at the moment */
 351     if (IsListEmpty(&Thread->MutantListHead)) return;
 352
 353     /* Lock the Dispatcher Database */
 354     OldIrql = KiAcquireDispatcherLock();
 355
 356     /* Get the List Pointers */
 357     ListHead = &Thread->MutantListHead;
 358     NextEntry = ListHead->Flink;
 359     while (NextEntry != ListHead)
 360     {
 361         /* Get the Mutant */
 362         Mutant = CONTAINING_RECORD(NextEntry, KMUTANT, MutantListEntry);
 363
 364         /* Make sure it's not terminating with APCs off */
 365         if (Mutant->ApcDisable)
 366         {
 367             /* Bugcheck the system */
 368             KEBUGCHECKEX(0, //THREAD_TERMINATE_HELD_MUTEX,
 369                          (ULONG_PTR)Thread,
 370                          (ULONG_PTR)Mutant,
 371                          0,
 372                          0);
 373         }
 374
 375         /* Now we can remove it */
 376         RemoveEntryList(&Mutant->MutantListEntry);
 377
 378         /* Unconditionally abandon it */
 379         Mutant->Header.SignalState = 1;
 380         Mutant->Abandoned = TRUE;
 381         Mutant->OwnerThread = NULL;
 382
 383         /* Check if the Wait List isn't empty */
 384         if (!IsListEmpty(&Mutant->Header.WaitListHead))
 385         {
 386             /* Wake the Mutant */
 387             KiWaitTest(&Mutant->Header, MUTANT_INCREMENT);
 388         }
 389
 390         /* Move on */
 391         NextEntry = NextEntry->Flink;
 392     }
 393
 394     /* Release the Lock */
 395     KiReleaseDispatcherLock(OldIrql);
 396 }
 397
 398 VOID
 399 NTAPI
 400 KeStartThread(IN OUT PKTHREAD Thread)
 401 {
 402     KLOCK_QUEUE_HANDLE LockHandle;
 403 #ifdef CONFIG_SMP
 404     PKNODE Node;
 405     PKPRCB NodePrcb;
 406     ULONG Set, Mask;
 407 #endif
 408     UCHAR IdealProcessor = 0;
 409     PKPROCESS Process = Thread->ApcState.Process;
 410
 411     /* Setup static fields from parent */
 412     Thread->Iopl = Process->Iopl;
 413     Thread->Quantum = Process->QuantumReset;
 414     Thread->QuantumReset = Process->QuantumReset;
 415     Thread->SystemAffinityActive = FALSE;
 416
 417     /* Lock the process */
 418     KiAcquireProcessLock(Process, &LockHandle);
 419
 420     /* Setup volatile data */
 421     Thread->Priority = Process->BasePriority;
 422     Thread->BasePriority = Process->BasePriority;
 423     Thread->Affinity = Process->Affinity;
 424     Thread->UserAffinity = Process->Affinity;
 425
 426 #ifdef CONFIG_SMP
 427     /* Get the KNODE and its PRCB */
 428     Node = KeNodeBlock[Process->IdealNode];
 429     NodePrcb = (PKPRCB)(KPCR_BASE + (Process->ThreadSeed * PAGE_SIZE));
 430
 431     /* Calculate affinity mask */
 432     Set = ~NodePrcb->MultiThreadProcessorSet;
 433     Mask = (ULONG)(Node->ProcessorMask & Process->Affinity);
 434     Set &= Mask;
 435     if (Set) Mask = Set;
 436
 437     /* Get the new thread seed */
 438     IdealProcessor = KeFindNextRightSetAffinity(Process->ThreadSeed, Mask);
 439     Process->ThreadSeed = IdealProcessor;
 440
 441     /* Sanity check */
 442     ASSERT((Thread->UserAffinity & AFFINITY_MASK(IdealProcessor)));
 443 #endif
 444
 445     /* Set the Ideal Processor */
 446     Thread->IdealProcessor = IdealProcessor;
 447     Thread->UserIdealProcessor = IdealProcessor;
 448
 449     /* Lock the Dispatcher Database */
 450     KiAcquireDispatcherLockAtDpcLevel();
 451
 452     /* Insert the thread into the process list */
 453     InsertTailList(&Process->ThreadListHead, &Thread->ThreadListEntry);
 454
 455     /* Increase the stack count */
 456     ASSERT(Process->StackCount != MAXULONG_PTR);
 457     Process->StackCount++;
 458
 459     /* Release locks and return */
 460     KiReleaseDispatcherLockFromDpcLevel();
 461     KiReleaseProcessLock(&LockHandle);
 462 }
 463
 464 VOID
 465 NTAPI
 466 KiSuspendRundown(IN PKAPC Apc)
 467 {
 468     /* Does nothing */
 469     UNREFERENCED_PARAMETER(Apc);
 470 }
 471
 472 VOID
 473 NTAPI
 474 KiSuspendNop(IN PKAPC Apc,
 475              IN PKNORMAL_ROUTINE *NormalRoutine,
 476              IN PVOID *NormalContext,
 477              IN PVOID *SystemArgument1,
 478              IN PVOID *SystemArgument2)
 479 {
 480     /* Does nothing */
 481     UNREFERENCED_PARAMETER(Apc);
 482     UNREFERENCED_PARAMETER(NormalRoutine);
 483     UNREFERENCED_PARAMETER(NormalContext);
 484     UNREFERENCED_PARAMETER(SystemArgument1);
 485     UNREFERENCED_PARAMETER(SystemArgument2);
 486 }
 487
 488 VOID
 489 NTAPI
 490 KiSuspendThread(IN PVOID NormalContext,
 491                 IN PVOID SystemArgument1,
 492                 IN PVOID SystemArgument2)
 493 {
 494     /* Non-alertable kernel-mode suspended wait */
 495     KeWaitForSingleObject(&KeGetCurrentThread()->SuspendSemaphore,
 496                           Suspended,
 497                           KernelMode,
 498                           FALSE,
 499                           NULL);
 500 }
 501
 502 NTSTATUS
 503 NTAPI
 504 KeSuspendThread(PKTHREAD Thread)
 505 {
 506     KLOCK_QUEUE_HANDLE ApcLock;
 507     ULONG PreviousCount;
 508     ASSERT_THREAD(Thread);
 509     ASSERT_IRQL_LESS_OR_EQUAL(DISPATCH_LEVEL);
 510
 511     /* Lock the APC Queue */
 512     KiAcquireApcLock(Thread, &ApcLock);
 513
 514     /* Save the Old Count */
 515     PreviousCount = Thread->SuspendCount;
 516
 517     /* Handle the maximum */
 518     if (PreviousCount == MAXIMUM_SUSPEND_COUNT)
 519     {
 520         /* Raise an exception */
 521         KiReleaseApcLock(&ApcLock);
 522         RtlRaiseStatus(STATUS_SUSPEND_COUNT_EXCEEDED);
 523     }
 524
 525     /* Should we bother to queue at all? */
 526     if (Thread->ApcQueueable)
 527     {
 528         /* Increment the suspend count */
 529         Thread->SuspendCount++;
 530
 531         /* Check if we should suspend it */
 532         if (!(PreviousCount) && !(Thread->FreezeCount))
 533         {
 534             /* Is the APC already inserted? */
 535             if (!Thread->SuspendApc.Inserted)
 536             {
 537                 /* Not inserted, insert it */
 538                 Thread->SuspendApc.Inserted = TRUE;
 539                 KiInsertQueueApc(&Thread->SuspendApc, IO_NO_INCREMENT);
 540             }
 541             else
 542             {
 543                 /* Lock the dispatcher */
 544                 KiAcquireDispatcherLockAtDpcLevel();
 545
 546                 /* Unsignal the semaphore, the APC was already inserted */
 547                 Thread->SuspendSemaphore.Header.SignalState--;
 548
 549                 /* Release the dispatcher */
 550                 KiReleaseDispatcherLockFromDpcLevel();
 551             }
 552         }
 553     }
 554
 555     /* Release Lock and return the Old State */
 556     KiReleaseApcLockFromDpcLevel(&ApcLock);
 557     KiExitDispatcher(ApcLock.OldIrql);
 558     return PreviousCount;
 559 }
 560
 561 BOOLEAN
 562 NTAPI
 563 KeTestAlertThread(IN KPROCESSOR_MODE AlertMode)
 564 {
 565     PKTHREAD Thread = KeGetCurrentThread();
 566     BOOLEAN OldState;
 567     KLOCK_QUEUE_HANDLE ApcLock;
 568     ASSERT_THREAD(Thread);
 569     ASSERT_IRQL_LESS_OR_EQUAL(DISPATCH_LEVEL);
 570
 571     /* Lock the Dispatcher Database and the APC Queue */
 572     KiAcquireApcLock(Thread, &ApcLock);
 573     KiAcquireDispatcherLockAtDpcLevel();
 574
 575     /* Save the old State */
 576     OldState = Thread->Alerted[AlertMode];
 577
 578     /* Check the Thread is alerted */
 579     if (OldState)
 580     {
 581         /* Disable alert for this mode */
 582         Thread->Alerted[AlertMode] = FALSE;
 583     }
 584     else if ((AlertMode != KernelMode) &&
 585              (!IsListEmpty(&Thread->ApcState.ApcListHead[UserMode])))
 586     {
 587         /* If the mode is User and the Queue isn't empty, set Pending */
 588         Thread->ApcState.UserApcPending = TRUE;
 589     }
 590
 591     /* Release Locks and return the Old State */
 592     KiReleaseDispatcherLockFromDpcLevel();
 593     KiReleaseApcLockFromDpcLevel(&ApcLock);
 594     KiExitDispatcher(ApcLock.OldIrql);
 595     return OldState;
 596 }
 597
 598 NTSTATUS
 599 NTAPI
 600 KeInitThread(IN OUT PKTHREAD Thread,
 601              IN PVOID KernelStack,
 602              IN PKSYSTEM_ROUTINE SystemRoutine,
 603              IN PKSTART_ROUTINE StartRoutine,
 604              IN PVOID StartContext,
 605              IN PCONTEXT Context,
 606              IN PVOID Teb,
 607              IN PKPROCESS Process)
 608 {
 609     BOOLEAN AllocatedStack = FALSE;
 610     ULONG i;
 611     PKWAIT_BLOCK TimerWaitBlock;
 612     PKTIMER Timer;
 613     NTSTATUS Status;
 614
 615     /* Initalize the Dispatcher Header */
 616     KeInitializeDispatcherHeader(&Thread->DispatcherHeader,
 617                                  ThreadObject,
 618                                  sizeof(KTHREAD) / sizeof(LONG),
 619                                  FALSE);
 620
 621     /* Initialize the Mutant List */
 622     InitializeListHead(&Thread->MutantListHead);
 623
 624     /* Initialize the wait blocks */
 625     for (i = 0; i< (THREAD_WAIT_OBJECTS + 1); i++)
 626     {
 627         /* Put our pointer */
 628         Thread->WaitBlock[i].Thread = Thread;
 629     }
 630
 631     /* Set swap settings */
 632     Thread->EnableStackSwap = FALSE;//TRUE;
 633     Thread->IdealProcessor = 1;
 634     Thread->SwapBusy = FALSE;
 635     Thread->AdjustReason = 0;
 636
 637     /* Initialize the lock */
 638     KeInitializeSpinLock(&Thread->ThreadLock);
 639
 640     /* Setup the Service Descriptor Table for Native Calls */
 641     Thread->ServiceTable = KeServiceDescriptorTable;
 642
 643     /* Setup APC Fields */
 644     InitializeListHead(&Thread->ApcState.ApcListHead[0]);
 645     InitializeListHead(&Thread->ApcState.ApcListHead[1]);
 646     Thread->ApcState.Process = Process;
 647     Thread->ApcStatePointer[OriginalApcEnvironment] = &Thread->ApcState;
 648     Thread->ApcStatePointer[AttachedApcEnvironment] = &Thread->SavedApcState;
 649     Thread->ApcStateIndex = OriginalApcEnvironment;
 650     Thread->ApcQueueable = TRUE;
 651     KeInitializeSpinLock(&Thread->ApcQueueLock);
 652
 653     /* Initialize the Suspend APC */
 654     KeInitializeApc(&Thread->SuspendApc,
 655                     Thread,
 656                     OriginalApcEnvironment,
 657                     KiSuspendNop,
 658                     KiSuspendRundown,
 659                     KiSuspendThread,
 660                     KernelMode,
 661                     NULL);
 662
 663     /* Initialize the Suspend Semaphore */
 664     KeInitializeSemaphore(&Thread->SuspendSemaphore, 0, 2);
 665
 666     /* Setup the timer */
 667     Timer = &Thread->Timer;
 668     KeInitializeTimer(Timer);
 669     TimerWaitBlock = &Thread->WaitBlock[TIMER_WAIT_BLOCK];
 670     TimerWaitBlock->Object = Timer;
 671     TimerWaitBlock->WaitKey = STATUS_TIMEOUT;
 672     TimerWaitBlock->WaitType = WaitAny;
 673     TimerWaitBlock->NextWaitBlock = NULL;
 674
 675     /* Link the two wait lists together */
 676     TimerWaitBlock->WaitListEntry.Flink = &Timer->Header.WaitListHead;
 677     TimerWaitBlock->WaitListEntry.Blink = &Timer->Header.WaitListHead;
 678
 679     /* Set the TEB */
 680     Thread->Teb = Teb;
 681
 682     /* Check if we have a kernel stack */
 683     if (!KernelStack)
 684     {
 685         /* We don't, allocate one */
 686         KernelStack = (PVOID)((ULONG_PTR)MmCreateKernelStack(FALSE) +
 687                               KERNEL_STACK_SIZE);
 688         if (!KernelStack) return STATUS_INSUFFICIENT_RESOURCES;
 689
 690         /* Remember for later */
 691         AllocatedStack = TRUE;
 692     }
 693
 694     /* Set the Thread Stacks */
 695     Thread->InitialStack = (PCHAR)KernelStack;
 696     Thread->StackBase = (PCHAR)KernelStack;
 697     Thread->StackLimit = (ULONG_PTR)KernelStack - KERNEL_STACK_SIZE;
 698     Thread->KernelStackResident = TRUE;
 699
 700     /* ROS Mm HACK */
 701     MmUpdatePageDir((PEPROCESS)Process,
 702                     (PVOID)Thread->StackLimit,
 703                     KERNEL_STACK_SIZE);
 704     MmUpdatePageDir((PEPROCESS)Process, (PVOID)Thread, sizeof(ETHREAD));
 705
 706     /* Enter SEH to avoid crashes due to user mode */
 707     Status = STATUS_SUCCESS;
 708     _SEH_TRY
 709     {
 710         /* Initalize the Thread Context */
 711         Ke386InitThreadWithContext(Thread,
 712                                    SystemRoutine,
 713                                    StartRoutine,
 714                                    StartContext,
 715                                    Context);
 716     }
 717     _SEH_HANDLE
 718     {
 719         /* Set failure status */
 720         Status = STATUS_UNSUCCESSFUL;
 721
 722         /* Check if a stack was allocated */
 723         if (AllocatedStack)
 724         {
 725             /* Delete the stack */
 726             MmDeleteKernelStack(Thread->StackBase, FALSE);
 727             Thread->InitialStack = NULL;
 728         }
 729     }
 730     _SEH_END;
 731
 732     /* Set the Thread to initalized */
 733     Thread->State = Initialized;
 734     return Status;
 735 }
 736
 737 VOID
 738 NTAPI
 739 KeInitializeThread(IN PKPROCESS Process,
 740                    IN OUT PKTHREAD Thread,
 741                    IN PKSYSTEM_ROUTINE SystemRoutine,
 742                    IN PKSTART_ROUTINE StartRoutine,
 743                    IN PVOID StartContext,
 744                    IN PCONTEXT Context,
 745                    IN PVOID Teb,
 746                    IN PVOID KernelStack)
 747 {
 748     /* Initailize and start the thread on success */
 749     if (NT_SUCCESS(KeInitThread(Thread,
 750                                 KernelStack,
 751                                 SystemRoutine,
 752                                 StartRoutine,
 753                                 StartContext,
 754                                 Context,
 755                                 Teb,
 756                                 Process)))
 757     {
 758         /* Start it */
 759         KeStartThread(Thread);
 760     }
 761 }
 762
 763 VOID
 764 NTAPI
 765 KeUninitThread(IN PKTHREAD Thread)
 766 {
 767     /* Delete the stack */
 768     MmDeleteKernelStack(Thread->StackBase, FALSE);
 769     Thread->InitialStack = NULL;
 770 }
 771
 772 /* PUBLIC FUNCTIONS **********************************************************/
 773
 774 /*
 775  * @unimplemented
 776  */
 777 VOID
 778 NTAPI
 779 KeCapturePersistentThreadState(IN PVOID CurrentThread,
 780                                IN ULONG Setting1,
 781                                IN ULONG Setting2,
 782                                IN ULONG Setting3,
 783                                IN ULONG Setting4,
 784                                IN ULONG Setting5,
 785                                IN PVOID ThreadState)
 786 {
 787     UNIMPLEMENTED;
 788 }
 789
 790 /*
 791  * @implemented
 792  */
 793 #undef KeGetCurrentThread
 794 PKTHREAD
 795 NTAPI
 796 KeGetCurrentThread(VOID)
 797 {
 798     /* Return the current thread on this PCR */
 799     return ((PKIPCR)KeGetPcr())->PrcbData.CurrentThread;
 800 }
 801
 802 /*
 803  * @implemented
 804  */
 805 KPROCESSOR_MODE
 806 NTAPI
 807 KeGetPreviousMode(VOID)
 808 {
 809     /* Return the previous mode of this thread */
 810     return KeGetCurrentThread()->PreviousMode;
 811 }
 812
 813 /*
 814  * @implemented
 815  */
 816 ULONG
 817 NTAPI
 818 KeQueryRuntimeThread(IN PKTHREAD Thread,
 819                      OUT PULONG UserTime)
 820 {
 821     ASSERT_THREAD(Thread);
 822
 823     /* Return the User Time */
 824     *UserTime = Thread->UserTime;
 825
 826     /* Return the Kernel Time */
 827     return Thread->KernelTime;
 828 }
 829
 830 /*
 831  * @implemented
 832  */
 833 BOOLEAN
 834 NTAPI
 835 KeSetKernelStackSwapEnable(IN BOOLEAN Enable)
 836 {
 837     BOOLEAN PreviousState;
 838     PKTHREAD Thread = KeGetCurrentThread();
 839
 840     /* Save Old State */
 841     PreviousState = Thread->EnableStackSwap;
 842
 843     /* Set New State */
 844     Thread->EnableStackSwap = Enable;
 845
 846     /* Return Old State */
 847     return PreviousState;
 848 }
 849
 850 /*
 851  * @implemented
 852  */
 853 KPRIORITY
 854 NTAPI
 855 KeQueryPriorityThread(IN PKTHREAD Thread)
 856 {
 857     ASSERT_THREAD(Thread);
 858
 859     /* Return the current priority */
 860     return Thread->Priority;
 861 }
 862
 863 /*
 864  * @implemented
 865  */
 866 VOID
 867 NTAPI
 868 KeRevertToUserAffinityThread(VOID)
 869 {
 870     KIRQL OldIrql;
 871     PKPRCB Prcb;
 872     PKTHREAD NextThread, CurrentThread = KeGetCurrentThread();
 873     ASSERT_IRQL_LESS_OR_EQUAL(DISPATCH_LEVEL);
 874     ASSERT(CurrentThread->SystemAffinityActive != FALSE);
 875
 876     /* Lock the Dispatcher Database */
 877     OldIrql = KiAcquireDispatcherLock();
 878
 879     /* Set the user affinity and processor and disable system affinity */
 880     CurrentThread->Affinity = CurrentThread->UserAffinity;
 881     CurrentThread->IdealProcessor = CurrentThread->UserIdealProcessor;
 882     CurrentThread->SystemAffinityActive = FALSE;
 883
 884     /* Get the current PRCB and check if it doesn't match this affinity */
 885     Prcb = KeGetCurrentPrcb();
 886     if (!(Prcb->SetMember & CurrentThread->Affinity))
 887     {
 888         /* Lock the PRCB */
 889         KiAcquirePrcbLock(Prcb);
 890
 891 #ifdef NEW_SCHEDULER
 892         /* Check if there's no next thread scheduled */
 893         if (!Prcb->NextThread)
 894         {
 895             /* Select a new thread and set it on standby */
 896             NextThread = KiSelectNextThread(Prcb);
 897             NextThread->State = Standby;
 898             Prcb->NextThread = NextThread;
 899         }
 900 #else
 901         /* We need to dispatch a new thread */
 902         NextThread = NULL;
 903         CurrentThread->WaitIrql = OldIrql;
 904         KiDispatchThreadNoLock(Ready);
 905         KeLowerIrql(OldIrql);
 906         return;
 907 #endif
 908
 909         /* Release the PRCB lock */
 910         KiReleasePrcbLock(Prcb);
 911     }
 912
 913     /* Unlock dispatcher database */
 914     KiReleaseDispatcherLock(OldIrql);
 915 }
 916
 917 /*
 918  * @implemented
 919  */
 920 UCHAR
 921 NTAPI
 922 KeSetIdealProcessorThread(IN PKTHREAD Thread,
 923                           IN UCHAR Processor)
 924 {
 925     CCHAR OldIdealProcessor;
 926     KIRQL OldIrql;
 927     ASSERT(Processor <= MAXIMUM_PROCESSORS);
 928
 929     /* Lock the Dispatcher Database */
 930     OldIrql = KiAcquireDispatcherLock();
 931
 932     /* Save Old Ideal Processor */
 933     OldIdealProcessor = Thread->UserIdealProcessor;
 934
 935     /* Make sure a valid CPU was given */
 936     if (Processor < MAXIMUM_PROCESSORS)
 937     {
 938         /* Check if the user ideal CPU is in the affinity */
 939         if (Thread->UserIdealProcessor & AFFINITY_MASK(Processor))
 940         {
 941             /* Set the ideal processor */
 942             Thread->IdealProcessor = Processor;
 943
 944             /* Check if system affinity is used */
 945             if (!Thread->SystemAffinityActive)
 946             {
 947                 /* It's not, so update the user CPU too */
 948                 Thread->UserIdealProcessor = Processor;
 949             }
 950         }
 951     }
 952
 953     /* Release dispatcher lock and return the old ideal CPU */
 954     KiReleaseDispatcherLock(OldIrql);
 955     return OldIdealProcessor;
 956 }
 957
 958 /*
 959  * @implemented
 960  */
 961 VOID
 962 NTAPI
 963 KeSetSystemAffinityThread(IN KAFFINITY Affinity)
 964 {
 965     KIRQL OldIrql;
 966     PKPRCB Prcb;
 967     PKTHREAD NextThread, CurrentThread = KeGetCurrentThread();
 968     ASSERT_IRQL_LESS_OR_EQUAL(DISPATCH_LEVEL);
 969     ASSERT((Affinity & KeActiveProcessors) != 0);
 970
 971     /* Lock the Dispatcher Database */
 972     OldIrql = KiAcquireDispatcherLock();
 973
 974     /* Restore the affinity and enable system affinity */
 975     CurrentThread->Affinity = Affinity;
 976     CurrentThread->SystemAffinityActive = TRUE;
 977
 978     /* Check if the ideal processor is part of the affinity */
 979 #ifdef CONFIG_SMP
 980     if (!(Affinity & AFFINITY_MASK(CurrentThread->IdealProcessor)))
 981     {
 982         ULONG AffinitySet, NodeMask;
 983
 984         /* It's not! Get the PRCB */
 985         Prcb = KiProcessorBlock[CurrentThread->IdealProcessor];
 986
 987         /* Calculate the affinity set */
 988         AffinitySet = KeActiveProcessors & Affinity;
 989         NodeMask = Prcb->ParentNode->ProcessorMask & AffinitySet;
 990         if (NodeMask)
 991         {
 992             /* Use the Node set instead */
 993             AffinitySet = NodeMask;
 994         }
 995
 996         /* Calculate the ideal CPU from the affinity set */
 997         BitScanReverse(&NodeMask, AffinitySet);
 998         CurrentThread->IdealProcessor = (UCHAR)NodeMask;
 999     }
1000 #endif
1001
1002     /* Get the current PRCB and check if it doesn't match this affinity */
1003     Prcb = KeGetCurrentPrcb();
1004     if (!(Prcb->SetMember & CurrentThread->Affinity))
1005     {
1006         /* Lock the PRCB */
1007         KiAcquirePrcbLock(Prcb);
1008
1009 #ifdef NEW_SCHEDULER
1010         /* Check if there's no next thread scheduled */
1011         if (!Prcb->NextThread)
1012         {
1013             /* Select a new thread and set it on standby */
1014             NextThread = KiSelectNextThread(Prcb);
1015             NextThread->State = Standby;
1016             Prcb->NextThread = NextThread;
1017         }
1018 #else
1019         /* We need to dispatch a new thread */
1020         NextThread = NULL;
1021         CurrentThread->WaitIrql = OldIrql;
1022         KiDispatchThreadNoLock(Ready);
1023         KeLowerIrql(OldIrql);
1024         return;
1025 #endif
1026
1027         /* Release the PRCB lock */
1028         KiReleasePrcbLock(Prcb);
1029     }
1030
1031     /* Unlock dispatcher database */
1032     KiReleaseDispatcherLock(OldIrql);
1033 }
1034
1035 /*
1036  * @implemented
1037  */
1038 LONG
1039 STDCALL
1040 KeSetBasePriorityThread(PKTHREAD Thread,
1041                         LONG Increment)
1042 {
1043     KIRQL OldIrql;
1044     PKPROCESS Process;
1045     KPRIORITY Priority;
1046     KPRIORITY CurrentBasePriority;
1047     KPRIORITY BasePriority;
1048     BOOLEAN Released = FALSE;
1049     LONG CurrentIncrement;
1050
1051     /* Lock the Dispatcher Database */
1052     OldIrql = KeAcquireDispatcherDatabaseLock();
1053
1054     /* Get the process and calculate current BP and BPI */
1055     Process = Thread->ApcStatePointer[0]->Process;
1056     CurrentBasePriority = Thread->BasePriority;
1057     CurrentIncrement = CurrentBasePriority - Process->BasePriority;
1058
1059     /* Change to use the SI if Saturation was used */
1060     if (Thread->Saturation) CurrentIncrement = (HIGH_PRIORITY + 1) / 2 *
1061                                                Thread->Saturation;
1062
1063     /* Now check if saturation is being used for the new value */
1064     if (abs(Increment) >= ((HIGH_PRIORITY + 1) / 2))
1065     {
1066         /* Check if we need positive or negative saturation */
1067         Thread->Saturation = (Increment > 0) ? 1 : -1;
1068     }
1069
1070     /* Normalize the Base Priority */
1071     BasePriority = Process->BasePriority + Increment;
1072     if (Process->BasePriority >= LOW_REALTIME_PRIORITY)
1073     {
1074         /* Check if it's too low */
1075         if (BasePriority < LOW_REALTIME_PRIORITY)
1076             BasePriority = LOW_REALTIME_PRIORITY;
1077
1078         /* Check if it's too high */
1079         if (BasePriority > HIGH_PRIORITY) BasePriority = HIGH_PRIORITY;
1080
1081         /* We are at RTP, so use the raw BP */
1082         Priority = BasePriority;
1083     }
1084     else
1085     {
1086         /* Check if it's entering RTP */
1087         if (BasePriority >= LOW_REALTIME_PRIORITY)
1088             BasePriority = LOW_REALTIME_PRIORITY - 1;
1089
1090         /* Check if it's too low */
1091         if (BasePriority <= LOW_PRIORITY)
1092             BasePriority = 1;
1093
1094         /* If Saturation is used, then use the raw BP */
1095         if (Thread->Saturation)
1096         {
1097             Priority = BasePriority;
1098         }
1099         else
1100         {
1101             /* Calculate the new priority */
1102             Priority = Thread->Priority + (BasePriority - CurrentBasePriority)-
1103                        Thread->PriorityDecrement;
1104
1105             /* Make sure it won't enter RTP ranges */
1106             if (Priority >= LOW_REALTIME_PRIORITY)
1107                 Priority = LOW_REALTIME_PRIORITY - 1;
1108         }
1109     }
1110
1111     /* Finally set the new base priority */
1112     Thread->BasePriority = BasePriority;
1113
1114     /* Reset the decrements */
1115     Thread->PriorityDecrement = 0;
1116
1117     /* If the priority will change, reset quantum and change it for real */
1118     if (Priority != Thread->Priority)
1119     {
1120         Thread->Quantum = Thread->QuantumReset;
1121         KiSetPriorityThread(Thread, Priority, &Released);
1122     }
1123
1124     /* Release Lock if needed */
1125     if (!Released)
1126     {
1127         KeReleaseDispatcherDatabaseLock(OldIrql);
1128     }
1129     else
1130     {
1131         KeLowerIrql(OldIrql);
1132     }
1133
1134     /* Return the Old Increment */
1135     return CurrentIncrement;
1136 }
1137
1138 /*
1139  * @implemented
1140  */
1141 KAFFINITY
1142 NTAPI
1143 KeSetAffinityThread(IN PKTHREAD Thread,
1144                     IN KAFFINITY Affinity)
1145 {
1146     KIRQL OldIrql;
1147     KAFFINITY OldAffinity;
1148     BOOLEAN Released;
1149     ASSERT_THREAD(Thread);
1150     ASSERT_IRQL_LESS_OR_EQUAL(DISPATCH_LEVEL);
1151
1152     /* Lock the dispatcher database */
1153     OldIrql = KiAcquireDispatcherLock();
1154
1155     /* Call the internal function */
1156     OldAffinity = KiSetAffinityThread(Thread, Affinity, &Released);
1157
1158     /* Check if lock was released */
1159     if (!Released)
1160     {
1161         /* Release the dispatcher database */
1162         KiReleaseDispatcherLock(OldIrql);
1163     }
1164     else
1165     {
1166         /* Lower IRQL only */
1167         KeLowerIrql(OldIrql);
1168     }
1169
1170     /* Return old affinity */
1171     return OldAffinity;
1172 }
1173
1174 /*
1175  * @implemented
1176  */
1177 KPRIORITY
1178 NTAPI
1179 KeSetPriorityThread(IN PKTHREAD Thread,
1180                     IN KPRIORITY Priority)
1181 {
1182     KIRQL OldIrql;
1183     KPRIORITY OldPriority;
1184     BOOLEAN Released;
1185     ASSERT_THREAD(Thread);
1186     ASSERT_IRQL_LESS_OR_EQUAL(DISPATCH_LEVEL);
1187     ASSERT((Priority <= HIGH_PRIORITY) && (Priority >= LOW_PRIORITY));
1188
1189     /* Lock the Dispatcher Database */
1190     OldIrql = KiAcquireDispatcherLock();
1191
1192     /* Lock the thread */
1193     KiAcquireThreadLock(Thread);
1194
1195     /* Save the old Priority */
1196     OldPriority = Thread->Priority;
1197
1198     /* Make sure that an actual change is being done */
1199     if (OldPriority != Priority)
1200     {
1201         /* Reset the Quantum and Decrements */
1202         Thread->Quantum = Thread->QuantumReset;
1203         Thread->PriorityDecrement = 0;
1204
1205         /* Set the new Priority */
1206         KiSetPriorityThread(Thread, Priority, &Released);
1207     }
1208
1209     /* Release thread lock */
1210     KiReleaseThreadLock(Thread);
1211
1212     /* Check if lock was released */
1213     if (!Released)
1214     {
1215         /* Release the dispatcher database */
1216         KiReleaseDispatcherLock(OldIrql);
1217     }
1218     else
1219     {
1220         /* Lower IRQL only */
1221         KeLowerIrql(OldIrql);
1222     }
1223
1224     /* Return Old Priority */
1225     return OldPriority;
1226 }
1227
1228 /*
1229  * @implemented
1230  */
1231 VOID
1232 NTAPI
1233 KeTerminateThread(IN KPRIORITY Increment)
1234 {
1235     PLIST_ENTRY *ListHead;
1236     PETHREAD Entry, SavedEntry;
1237     PETHREAD *ThreadAddr;
1238     KLOCK_QUEUE_HANDLE LockHandle;
1239     PKTHREAD Thread = KeGetCurrentThread();
1240     PKPROCESS Process = Thread->ApcState.Process;
1241     ASSERT_IRQL_LESS_OR_EQUAL(DISPATCH_LEVEL);
1242
1243     /* Lock the process */
1244     KiAcquireProcessLock(Process, &LockHandle);
1245
1246     /* Make sure we won't get Swapped */
1247     KiSetThreadSwapBusy(Thread);
1248
1249     /* Save the Kernel and User Times */
1250     Process->KernelTime += Thread->KernelTime;
1251     Process->UserTime += Thread->UserTime;
1252
1253     /* Get the current entry and our Port */
1254     Entry = (PETHREAD)PspReaperListHead.Flink;
1255     ThreadAddr = &((PETHREAD)Thread)->ReaperLink;
1256
1257     /* Add it to the reaper's list */
1258     do
1259     {
1260         /* Get the list head */
1261         ListHead = &PspReaperListHead.Flink;
1262
1263         /* Link ourselves */
1264         *ThreadAddr = Entry;
1265         SavedEntry = Entry;
1266
1267         /* Now try to do the exchange */
1268         Entry = InterlockedCompareExchangePointer(ListHead, ThreadAddr, Entry);
1269
1270         /* Break out if the change was succesful */
1271     } while (Entry != SavedEntry);
1272
1273     /* Acquire the dispatcher lock */
1274     KiAcquireDispatcherLockAtDpcLevel();
1275
1276     /* Check if the reaper wasn't active */
1277     if (!Entry)
1278     {
1279         /* Activate it as a work item, directly through its Queue */
1280         KiInsertQueue(&ExWorkerQueue[HyperCriticalWorkQueue].WorkerQueue,
1281                       &PspReaperWorkItem.List,
1282                       FALSE);
1283     }
1284
1285     /* Check the thread has an associated queue */
1286     if (Thread->Queue)
1287     {
1288         /* Remove it from the list, and handle the queue */
1289         RemoveEntryList(&Thread->QueueListEntry);
1290         KiWakeQueue(Thread->Queue);
1291     }
1292
1293     /* Signal the thread */
1294     Thread->DispatcherHeader.SignalState = TRUE;
1295     if (IsListEmpty(&Thread->DispatcherHeader.WaitListHead) != TRUE)
1296     {
1297         /* Satisfy waits */
1298         KiWaitTest((PVOID)Thread, Increment);
1299     }
1300
1301     /* Remove the thread from the list */
1302     RemoveEntryList(&Thread->ThreadListEntry);
1303
1304     /* Release the process lock */
1305     KiReleaseProcessLock(&LockHandle);
1306
1307     /* Set us as terminated, decrease the Process's stack count */
1308     Thread->State = Terminated;
1309
1310     /* Decrease stack count */
1311     ASSERT(Process->StackCount != 0);
1312     ASSERT(Process->State == ProcessInMemory);
1313     Process->StackCount--;
1314     if (!Process->StackCount)
1315     {
1316         /* FIXME: Swap stacks */
1317     }
1318
1319     /* Rundown arch-specific parts */
1320     KiRundownThread(Thread);
1321
1322     /* Swap to a new thread */
1323     KiReleaseDispatcherLockFromDpcLevel();
1324     KiSwapThread(Thread, KeGetCurrentPrcb());
1325 }