hal/halx86/apic/apic.c

   1 /*
   2  * PROJECT:         ReactOS HAL
   3  * LICENSE:         GNU GPL - See COPYING in the top level directory
   4  * FILE:            hal/halx86/generic/apic.c
   5  * PURPOSE:         HAL APIC Management and Control Code
   6  * PROGRAMMERS:     Timo Kreuzer (timo.kreuzer@reactos.org)
   7  * REFERENCES:      http://www.joseflores.com/docs/ExploringIrql.html
   8  *                  http://www.codeproject.com/KB/system/soviet_kernel_hack.aspx
   9  *                  http://bbs.unixmap.net/thread-2022-1-1.html
  10  */
  11
  12 /* INCLUDES *******************************************************************/
  13
  14 #include <hal.h>
  15 #define NDEBUG
  16 #include <debug.h>
  17
  18 #include "apic.h"
  19 void HackEoi(void);
  20
  21 #ifndef _M_AMD64
  22 #define APIC_LAZY_IRQL
  23 #endif
  24
  25 /* GLOBALS ********************************************************************/
  26
  27 ULONG ApicVersion;
  28 UCHAR HalpVectorToIndex[256];
  29
  30 #ifndef _M_AMD64
  31 const UCHAR
  32 HalpIRQLtoTPR[32] =
  33 {
  34     0x00, /*  0 PASSIVE_LEVEL */
  35     0x3d, /*  1 APC_LEVEL */
  36     0x41, /*  2 DISPATCH_LEVEL */
  37     0x41, /*  3 \  */
  38     0x51, /*  4  \ */
  39     0x61, /*  5  | */
  40     0x71, /*  6  | */
  41     0x81, /*  7  | */
  42     0x91, /*  8  | */
  43     0xa1, /*  9  | */
  44     0xb1, /* 10  | */
  45     0xb1, /* 11  | */
  46     0xb1, /* 12  | */
  47     0xb1, /* 13  | */
  48     0xb1, /* 14  | */
  49     0xb1, /* 15 DEVICE IRQL */
  50     0xb1, /* 16  | */
  51     0xb1, /* 17  | */
  52     0xb1, /* 18  | */
  53     0xb1, /* 19  | */
  54     0xb1, /* 20  | */
  55     0xb1, /* 21  | */
  56     0xb1, /* 22  | */
  57     0xb1, /* 23  | */
  58     0xb1, /* 24  | */
  59     0xb1, /* 25  / */
  60     0xb1, /* 26 /  */
  61     0xc1, /* 27 PROFILE_LEVEL */
  62     0xd1, /* 28 CLOCK2_LEVEL */
  63     0xe1, /* 29 IPI_LEVEL */
  64     0xef, /* 30 POWER_LEVEL */
  65     0xff, /* 31 HIGH_LEVEL */
  66 };
  67
  68 const KIRQL
  69 HalVectorToIRQL[16] =
  70 {
  71        0, /* 00 PASSIVE_LEVEL */
  72     0xff, /* 10 */
  73     0xff, /* 20 */
  74        1, /* 3D APC_LEVEL */
  75        2, /* 41 DISPATCH_LEVEL */
  76        4, /* 50 \ */
  77        5, /* 60  \ */
  78        6, /* 70  | */
  79        7, /* 80 DEVICE IRQL */
  80        8, /* 90  | */
  81        9, /* A0  / */
  82       10, /* B0 /  */
  83       27, /* C1 PROFILE_LEVEL */
  84       28, /* D1 CLOCK2_LEVEL */
  85       29, /* E1 IPI_LEVEL / EF POWER_LEVEL */
  86       31, /* FF HIGH_LEVEL */
  87 };
  88 #endif
  89
  90 /* PRIVATE FUNCTIONS **********************************************************/
  91
  92 ULONG
  93 FORCEINLINE
  94 IOApicRead(UCHAR Register)
  95 {
  96     /* Select the register, then do the read */
  97     *(volatile UCHAR *)(IOAPIC_BASE + IOAPIC_IOREGSEL) = Register;
  98     return *(volatile ULONG *)(IOAPIC_BASE + IOAPIC_IOWIN);
  99 }
 100
 101 VOID
 102 FORCEINLINE
 103 IOApicWrite(UCHAR Register, ULONG Value)
 104 {
 105     /* Select the register, then do the write */
 106     *(volatile UCHAR *)(IOAPIC_BASE + IOAPIC_IOREGSEL) = Register;
 107     *(volatile ULONG *)(IOAPIC_BASE + IOAPIC_IOWIN) = Value;
 108 }
 109
 110 VOID
 111 FORCEINLINE
 112 ApicWriteIORedirectionEntry(
 113     UCHAR Index,
 114     IOAPIC_REDIRECTION_REGISTER ReDirReg)
 115 {
 116     IOApicWrite(IOAPIC_REDTBL + 2 * Index, ReDirReg.Long0);
 117     IOApicWrite(IOAPIC_REDTBL + 2 * Index + 1, ReDirReg.Long1);
 118 }
 119
 120 IOAPIC_REDIRECTION_REGISTER
 121 FORCEINLINE
 122 ApicReadIORedirectionEntry(
 123     UCHAR Index)
 124 {
 125     IOAPIC_REDIRECTION_REGISTER ReDirReg;
 126
 127     ReDirReg.Long0 = IOApicRead(IOAPIC_REDTBL + 2 * Index);
 128     ReDirReg.Long1 = IOApicRead(IOAPIC_REDTBL + 2 * Index + 1);
 129
 130     return ReDirReg;
 131 }
 132
 133 VOID
 134 FORCEINLINE
 135 ApicRequestInterrupt(IN UCHAR Vector, UCHAR TriggerMode)
 136 {
 137     APIC_COMMAND_REGISTER CommandRegister;
 138
 139     /* Setup the command register */
 140     CommandRegister.Long0 = 0;
 141     CommandRegister.Vector = Vector;
 142     CommandRegister.MessageType = APIC_MT_Fixed;
 143     CommandRegister.TriggerMode = TriggerMode;
 144     CommandRegister.DestinationShortHand = APIC_DSH_Self;
 145
 146     /* Write the low dword to send the interrupt */
 147     ApicWrite(APIC_ICR0, CommandRegister.Long0);
 148 }
 149
 150 VOID
 151 FORCEINLINE
 152 ApicSendEOI(void)
 153 {
 154     //ApicWrite(APIC_EOI, 0);
 155     HackEoi();
 156 }
 157
 158 KIRQL
 159 FORCEINLINE
 160 ApicGetProcessorIrql(VOID)
 161 {
 162     /* Read the TPR and convert it to an IRQL */
 163     return TprToIrql(ApicRead(APIC_PPR));
 164 }
 165
 166 KIRQL
 167 FORCEINLINE
 168 ApicGetCurrentIrql(VOID)
 169 {
 170 #ifdef _M_AMD64
 171     return (KIRQL)__readcr8();
 172 #elif defined(APIC_LAZY_IRQL)
 173     // HACK: some magic to Sync VBox's APIC registers
 174     ApicRead(APIC_VER);
 175
 176     /* Return the field in the PCR */
 177     return (KIRQL)__readfsbyte(FIELD_OFFSET(KPCR, Irql));
 178 #else
 179     // HACK: some magic to Sync VBox's APIC registers
 180     ApicRead(APIC_VER);
 181
 182     /* Read the TPR and convert it to an IRQL */
 183     return TprToIrql(ApicRead(APIC_TPR));
 184 #endif
 185 }
 186
 187 VOID
 188 FORCEINLINE
 189 ApicSetIrql(KIRQL Irql)
 190 {
 191 #ifdef _M_AMD64
 192     __writecr8(Irql);
 193 #elif defined(APIC_LAZY_IRQL)
 194     __writefsbyte(FIELD_OFFSET(KPCR, Irql), Irql);
 195 #else
 196     /* Convert IRQL and write the TPR */
 197     ApicWrite(APIC_TPR, IrqlToTpr(Irql));
 198 #endif
 199 }
 200 #define ApicRaiseIrql ApicSetIrql
 201
 202 #ifdef APIC_LAZY_IRQL
 203 VOID
 204 FORCEINLINE
 205 ApicLowerIrql(KIRQL Irql)
 206 {
 207     __writefsbyte(FIELD_OFFSET(KPCR, Irql), Irql);
 208
 209     /* Is the new Irql lower than set in the TPR? */
 210     if (Irql < KeGetPcr()->IRR)
 211     {
 212         /* Save the new hard IRQL in the IRR field */
 213         KeGetPcr()->IRR = Irql;
 214
 215         /* Need to lower it back */
 216         ApicWrite(APIC_TPR, IrqlToTpr(Irql));
 217     }
 218 }
 219 #else
 220 #define ApicLowerIrql ApicSetIrql
 221 #endif
 222
 223 UCHAR
 224 FASTCALL
 225 HalpIrqToVector(UCHAR Irq)
 226 {
 227     IOAPIC_REDIRECTION_REGISTER ReDirReg;
 228
 229     /* Read low dword of the redirection entry */
 230     ReDirReg.Long0 = IOApicRead(IOAPIC_REDTBL + 2 * Irq);
 231
 232     /* Return the vector */
 233     return (UCHAR)ReDirReg.Vector;
 234 }
 235
 236 KIRQL
 237 FASTCALL
 238 HalpVectorToIrql(UCHAR Vector)
 239 {
 240     return TprToIrql(Vector);
 241 }
 242
 243 UCHAR
 244 FASTCALL
 245 HalpVectorToIrq(UCHAR Vector)
 246 {
 247     return HalpVectorToIndex[Vector];
 248 }
 249
 250 VOID
 251 NTAPI
 252 HalpSendEOI(VOID)
 253 {
 254     ApicSendEOI();
 255 }
 256
 257 VOID
 258 NTAPI
 259 HalpInitializeLegacyPIC(VOID)
 260 {
 261     I8259_ICW1 Icw1;
 262     I8259_ICW2 Icw2;
 263     I8259_ICW3 Icw3;
 264     I8259_ICW4 Icw4;
 265
 266     /* Initialize ICW1 for master, interval 8, edge-triggered mode with ICW4 */
 267     Icw1.NeedIcw4 = TRUE;
 268     Icw1.OperatingMode = Cascade;
 269     Icw1.Interval = Interval8;
 270     Icw1.InterruptMode = EdgeTriggered;
 271     Icw1.Init = TRUE;
 272     Icw1.InterruptVectorAddress = 0;
 273     __outbyte(PIC1_CONTROL_PORT, Icw1.Bits);
 274
 275     /* ICW2 - interrupt vector offset */
 276     Icw2.Bits = PRIMARY_VECTOR_BASE;
 277     __outbyte(PIC1_DATA_PORT, Icw2.Bits);
 278
 279     /* Connect slave to IRQ 2 */
 280     Icw3.Bits = 0;
 281     Icw3.SlaveIrq2 = TRUE;
 282     __outbyte(PIC1_DATA_PORT, Icw3.Bits);
 283
 284     /* Enable 8086 mode, non-automatic EOI, non-buffered mode, non special fully nested mode */
 285     Icw4.SystemMode = New8086Mode;
 286     Icw4.EoiMode = NormalEoi;
 287     Icw4.BufferedMode = NonBuffered;
 288     Icw4.SpecialFullyNestedMode = FALSE;
 289     Icw4.Reserved = 0;
 290     __outbyte(PIC1_DATA_PORT, Icw4.Bits);
 291
 292     /* Mask all interrupts */
 293     __outbyte(PIC1_DATA_PORT, 0xFF);
 294
 295     /* Initialize ICW1 for slave, interval 8, edge-triggered mode with ICW4 */
 296     Icw1.NeedIcw4 = TRUE;
 297     Icw1.InterruptMode = EdgeTriggered;
 298     Icw1.OperatingMode = Cascade;
 299     Icw1.Interval = Interval8;
 300     Icw1.Init = TRUE;
 301     Icw1.InterruptVectorAddress = 0; /* This is only used in MCS80/85 mode */
 302     __outbyte(PIC2_CONTROL_PORT, Icw1.Bits);
 303
 304     /* Set interrupt vector base */
 305     Icw2.Bits = PRIMARY_VECTOR_BASE + 8;
 306     __outbyte(PIC2_DATA_PORT, Icw2.Bits);
 307
 308     /* Slave ID */
 309     Icw3.Bits = 0;
 310     Icw3.SlaveId = 2;
 311     __outbyte(PIC2_DATA_PORT, Icw3.Bits);
 312
 313     /* Enable 8086 mode, non-automatic EOI, non-buffered mode, non special fully nested mode */
 314     Icw4.SystemMode = New8086Mode;
 315     Icw4.EoiMode = NormalEoi;
 316     Icw4.BufferedMode = NonBuffered;
 317     Icw4.SpecialFullyNestedMode = FALSE;
 318     Icw4.Reserved = 0;
 319     __outbyte(PIC2_DATA_PORT, Icw4.Bits);
 320
 321     /* Mask all interrupts */
 322     __outbyte(PIC2_DATA_PORT, 0xFF);
 323 }
 324
 325 VOID
 326 NTAPI
 327 ApicInitializeLocalApic(ULONG Cpu)
 328 {
 329     APIC_BASE_ADRESS_REGISTER BaseRegister;
 330     APIC_SPURIOUS_INERRUPT_REGISTER SpIntRegister;
 331     LVT_REGISTER LvtEntry;
 332
 333     /* Enable the APIC if it wasn't yet */
 334     BaseRegister.Long = __readmsr(MSR_APIC_BASE);
 335     BaseRegister.Enable = 1;
 336     BaseRegister.BootStrapCPUCore = (Cpu == 0);
 337     __writemsr(MSR_APIC_BASE, BaseRegister.Long);
 338
 339     /* Set spurious vector and SoftwareEnable to 1 */
 340     SpIntRegister.Long = ApicRead(APIC_SIVR);
 341     SpIntRegister.Vector = APIC_SPURIOUS_VECTOR;
 342     SpIntRegister.SoftwareEnable = 1;
 343     SpIntRegister.FocusCPUCoreChecking = 0;
 344     ApicWrite(APIC_SIVR, SpIntRegister.Long);
 345
 346     /* Read the version and save it globally */
 347     if (Cpu == 0) ApicVersion = ApicRead(APIC_VER);
 348
 349     /* Set the mode to flat (max 8 CPUs supported!) */
 350     ApicWrite(APIC_DFR, APIC_DF_Flat);
 351
 352     /* Set logical apic ID */
 353     ApicWrite(APIC_LDR, ApicLogicalId(Cpu) << 24);
 354
 355     /* Set the spurious ISR */
 356     KeRegisterInterruptHandler(APIC_SPURIOUS_VECTOR, ApicSpuriousService);
 357
 358     /* Create a template LVT */
 359     LvtEntry.Long = 0;
 360     LvtEntry.Vector = 0xFF;
 361     LvtEntry.MessageType = APIC_MT_Fixed;
 362     LvtEntry.DeliveryStatus = 0;
 363     LvtEntry.RemoteIRR = 0;
 364     LvtEntry.TriggerMode = APIC_TGM_Edge;
 365     LvtEntry.Mask = 1;
 366     LvtEntry.TimerMode = 0;
 367
 368     /* Initialize and mask LVTs */
 369     ApicWrite(APIC_TMRLVTR, LvtEntry.Long);
 370     ApicWrite(APIC_THRMLVTR, LvtEntry.Long);
 371     ApicWrite(APIC_PCLVTR, LvtEntry.Long);
 372     ApicWrite(APIC_EXT0LVTR, LvtEntry.Long);
 373     ApicWrite(APIC_EXT1LVTR, LvtEntry.Long);
 374     ApicWrite(APIC_EXT2LVTR, LvtEntry.Long);
 375     ApicWrite(APIC_EXT3LVTR, LvtEntry.Long);
 376
 377     /* LINT0 */
 378     LvtEntry.Vector = APIC_SPURIOUS_VECTOR;
 379     LvtEntry.MessageType = APIC_MT_ExtInt;
 380     ApicWrite(APIC_LINT0, LvtEntry.Long);
 381
 382     /* Enable LINT1 (NMI) */
 383     LvtEntry.Mask = 0;
 384     LvtEntry.Vector = APIC_NMI_VECTOR;
 385     LvtEntry.MessageType = APIC_MT_NMI;
 386     LvtEntry.TriggerMode = APIC_TGM_Level;
 387     ApicWrite(APIC_LINT1, LvtEntry.Long);
 388
 389     /* Enable error LVTR */
 390     LvtEntry.Vector = APIC_ERROR_VECTOR;
 391     LvtEntry.MessageType = APIC_MT_Fixed;
 392     ApicWrite(APIC_ERRLVTR, LvtEntry.Long);
 393
 394     /* Set the IRQL from the PCR */
 395     ApicSetIrql(KeGetPcr()->Irql);
 396 #ifdef APIC_LAZY_IRQL
 397     /* Save the new hard IRQL in the IRR field */
 398     KeGetPcr()->IRR = KeGetPcr()->Irql;
 399 #endif
 400 }
 401
 402 UCHAR
 403 NTAPI
 404 HalpAllocateSystemInterrupt(
 405     IN UCHAR Irq,
 406     IN KIRQL Irql)
 407 {
 408     IOAPIC_REDIRECTION_REGISTER ReDirReg;
 409     IN UCHAR Vector;
 410
 411     /* Start with lowest vector */
 412     Vector = IrqlToTpr(Irql) & 0xF0;
 413
 414     /* Find an empty vector */
 415     while (HalpVectorToIndex[Vector] != 0xFF)
 416     {
 417         Vector++;
 418
 419         /* Check if we went over the edge */
 420         if (TprToIrql(Vector) > Irql)
 421         {
 422             /* Nothing free, return failure */
 423             return 0;
 424         }
 425     }
 426
 427     /* Save irq in the table */
 428     HalpVectorToIndex[Vector] = Irq;
 429
 430     /* Setup a redirection entry */
 431     ReDirReg.Vector = Vector;
 432     ReDirReg.DeliveryMode = APIC_MT_LowestPriority;
 433     ReDirReg.DestinationMode = APIC_DM_Logical;
 434     ReDirReg.DeliveryStatus = 0;
 435     ReDirReg.Polarity = 0;
 436     ReDirReg.RemoteIRR = 0;
 437     ReDirReg.TriggerMode = APIC_TGM_Edge;
 438     ReDirReg.Mask = 1;
 439     ReDirReg.Reserved = 0;
 440     ReDirReg.Destination = 0;
 441
 442     /* Initialize entry */
 443     IOApicWrite(IOAPIC_REDTBL + 2 * Irq, ReDirReg.Long0);
 444     IOApicWrite(IOAPIC_REDTBL + 2 * Irq + 1, ReDirReg.Long1);
 445
 446     return Vector;
 447 }
 448
 449 VOID
 450 NTAPI
 451 ApicInitializeIOApic(VOID)
 452 {
 453     PHARDWARE_PTE Pte;
 454     IOAPIC_REDIRECTION_REGISTER ReDirReg;
 455     UCHAR Index;
 456     ULONG Vector;
 457
 458     /* Map the I/O Apic page */
 459     Pte = HalAddressToPte(IOAPIC_BASE);
 460     Pte->PageFrameNumber = IOAPIC_PHYS_BASE / PAGE_SIZE;
 461     Pte->Valid = 1;
 462     Pte->Write = 1;
 463     Pte->Owner = 1;
 464     Pte->CacheDisable = 1;
 465     Pte->Global = 1;
 466     _ReadWriteBarrier();
 467
 468     /* Setup a redirection entry */
 469     ReDirReg.Vector = 0xFF;
 470     ReDirReg.DeliveryMode = APIC_MT_Fixed;
 471     ReDirReg.DestinationMode = APIC_DM_Physical;
 472     ReDirReg.DeliveryStatus = 0;
 473     ReDirReg.Polarity = 0;
 474     ReDirReg.RemoteIRR = 0;
 475     ReDirReg.TriggerMode = APIC_TGM_Edge;
 476     ReDirReg.Mask = 1;
 477     ReDirReg.Reserved = 0;
 478     ReDirReg.Destination = 0;
 479
 480     /* Loop all table entries */
 481     for (Index = 0; Index < 24; Index++)
 482     {
 483         /* Initialize entry */
 484         IOApicWrite(IOAPIC_REDTBL + 2 * Index, ReDirReg.Long0);
 485         IOApicWrite(IOAPIC_REDTBL + 2 * Index + 1, ReDirReg.Long1);
 486     }
 487
 488     /* Init the vactor to index table */
 489     for (Vector = 0; Vector <= 255; Vector++)
 490     {
 491         HalpVectorToIndex[Vector] = 0xFF;
 492     }
 493
 494     // HACK: Allocate all IRQs, should rather do that on demand
 495     for (Index = 0; Index <= 15; Index++)
 496     {
 497         /* Map the IRQs to IRQLs like with the PIC */
 498         HalpAllocateSystemInterrupt(Index, 27 - Index);
 499     }
 500
 501     /* Enable the timer interrupt */
 502     ReDirReg.Vector = APIC_CLOCK_VECTOR;
 503     ReDirReg.DeliveryMode = APIC_MT_Fixed;
 504     ReDirReg.DestinationMode = APIC_DM_Physical;
 505     ReDirReg.TriggerMode = APIC_TGM_Edge;
 506     ReDirReg.Mask = 0;
 507     ReDirReg.Destination = ApicRead(APIC_ID);
 508     IOApicWrite(IOAPIC_REDTBL + 2 * APIC_CLOCK_INDEX, ReDirReg.Long0);
 509
 510     ApicSendEOI();
 511 }
 512
 513 VOID
 514 NTAPI
 515 HalpInitializePICs(IN BOOLEAN EnableInterrupts)
 516 {
 517     ULONG_PTR EFlags;
 518
 519     /* Save EFlags and disable interrupts */
 520     EFlags = __readeflags();
 521     _disable();
 522
 523     /* Initialize and mask the PIC */
 524     HalpInitializeLegacyPIC();
 525
 526     /* Initialize the I/O APIC */
 527     ApicInitializeIOApic();
 528
 529     /* Manually reserve some vectors */
 530     HalpVectorToIndex[APIC_CLOCK_VECTOR] = 8;
 531     HalpVectorToIndex[APC_VECTOR] = 99;
 532     HalpVectorToIndex[DISPATCH_VECTOR] = 99;
 533
 534     /* Set interrupt handlers in the IDT */
 535     KeRegisterInterruptHandler(APIC_CLOCK_VECTOR, HalpClockInterrupt);
 536 #ifndef _M_AMD64
 537     KeRegisterInterruptHandler(APC_VECTOR, HalpApcInterrupt);
 538     KeRegisterInterruptHandler(DISPATCH_VECTOR, HalpDispatchInterrupt);
 539 #endif
 540
 541     /* Register the vectors for APC and dispatch interrupts */
 542     HalpRegisterVector(IDT_INTERNAL, 0, APC_VECTOR, APC_LEVEL);
 543     HalpRegisterVector(IDT_INTERNAL, 0, DISPATCH_VECTOR, DISPATCH_LEVEL);
 544
 545     /* Restore interrupt state */
 546     if (EnableInterrupts) EFlags |= EFLAGS_INTERRUPT_MASK;
 547     __writeeflags(EFlags);
 548 }
 549
 550
 551 /* SOFTWARE INTERRUPT TRAPS ***************************************************/
 552
 553 #ifndef _M_AMD64
 554 VOID
 555 DECLSPEC_NORETURN
 556 FASTCALL
 557 HalpApcInterruptHandler(IN PKTRAP_FRAME TrapFrame)
 558 {
 559     KPROCESSOR_MODE ProcessorMode;
 560     KIRQL OldIrql;
 561     ASSERT(ApicGetProcessorIrql() == APC_LEVEL);
 562
 563    /* Enter trap */
 564     KiEnterInterruptTrap(TrapFrame);
 565
 566 #ifdef APIC_LAZY_IRQL
 567     if (!HalBeginSystemInterrupt(APC_LEVEL, APC_VECTOR, &OldIrql))
 568     {
 569         /* "Spurious" interrupt, exit the interrupt */
 570         KiEoiHelper(TrapFrame);
 571     }
 572 #else
 573     /* Save the old IRQL */
 574     OldIrql = ApicGetCurrentIrql();
 575     ASSERT(OldIrql < APC_LEVEL);
 576 #endif
 577
 578     /* Raise to APC_LEVEL */
 579     ApicRaiseIrql(APC_LEVEL);
 580
 581     /* End the interrupt */
 582     ApicSendEOI();
 583
 584     /* Kernel or user APC? */
 585     if (KiUserTrap(TrapFrame)) ProcessorMode = UserMode;
 586     else if (TrapFrame->EFlags & EFLAGS_V86_MASK) ProcessorMode = UserMode;
 587     else ProcessorMode = KernelMode;
 588
 589     /* Enable interrupts and call the kernel's APC interrupt handler */
 590     _enable();
 591     KiDeliverApc(ProcessorMode, NULL, TrapFrame);
 592
 593     /* Disable interrupts */
 594     _disable();
 595
 596     /* Restore the old IRQL */
 597     ApicLowerIrql(OldIrql);
 598
 599     /* Exit the interrupt */
 600     KiEoiHelper(TrapFrame);
 601 }
 602
 603 VOID
 604 DECLSPEC_NORETURN
 605 FASTCALL
 606 HalpDispatchInterruptHandler(IN PKTRAP_FRAME TrapFrame)
 607 {
 608     KIRQL OldIrql;
 609     ASSERT(ApicGetProcessorIrql() == DISPATCH_LEVEL);
 610
 611    /* Enter trap */
 612     KiEnterInterruptTrap(TrapFrame);
 613
 614 #ifdef APIC_LAZY_IRQL
 615     if (!HalBeginSystemInterrupt(DISPATCH_LEVEL, DISPATCH_VECTOR, &OldIrql))
 616     {
 617         /* "Spurious" interrupt, exit the interrupt */
 618         KiEoiHelper(TrapFrame);
 619     }
 620 #else
 621     /* Get the current IRQL */
 622     OldIrql = ApicGetCurrentIrql();
 623     ASSERT(OldIrql < DISPATCH_LEVEL);
 624 #endif
 625
 626     /* Raise to DISPATCH_LEVEL */
 627     ApicRaiseIrql(DISPATCH_LEVEL);
 628
 629     /* End the interrupt */
 630     ApicSendEOI();
 631
 632     /* Enable interrupts and call the kernel's DPC interrupt handler */
 633     _enable();
 634     KiDispatchInterrupt();
 635     _disable();
 636
 637     /* Restore the old IRQL */
 638     ApicLowerIrql(OldIrql);
 639
 640     /* Exit the interrupt */
 641     KiEoiHelper(TrapFrame);
 642 }
 643 #endif
 644
 645
 646 /* SOFTWARE INTERRUPTS ********************************************************/
 647
 648
 649 VOID
 650 FASTCALL
 651 HalRequestSoftwareInterrupt(IN KIRQL Irql)
 652 {
 653     /* Convert irql to vector and request an interrupt */
 654     ApicRequestInterrupt(IrqlToSoftVector(Irql), APIC_TGM_Edge);
 655 }
 656
 657 VOID
 658 FASTCALL
 659 HalClearSoftwareInterrupt(
 660     IN KIRQL Irql)
 661 {
 662     /* Nothing to do */
 663 }
 664
 665
 666 /* SYSTEM INTERRUPTS **********************************************************/
 667
 668 BOOLEAN
 669 NTAPI
 670 HalEnableSystemInterrupt(
 671     IN ULONG Vector,
 672     IN KIRQL Irql,
 673     IN KINTERRUPT_MODE InterruptMode)
 674 {
 675     IOAPIC_REDIRECTION_REGISTER ReDirReg;
 676     PKPRCB Prcb = KeGetCurrentPrcb();
 677     UCHAR Index;
 678     ASSERT(Irql <= HIGH_LEVEL);
 679     ASSERT((IrqlToTpr(Irql) & 0xF0) == (Vector & 0xF0));
 680
 681     /* Get the irq for this vector */
 682     Index = HalpVectorToIndex[Vector];
 683
 684     /* Check if its valid */
 685     if (Index == 0xff)
 686     {
 687         /* Interrupt is not in use */
 688         return FALSE;
 689     }
 690
 691     /* Read the redirection entry */
 692     ReDirReg = ApicReadIORedirectionEntry(Index);
 693
 694     /* Check if the interrupt was unused */
 695     if (ReDirReg.Vector != Vector)
 696     {
 697         ReDirReg.Vector = Vector;
 698         ReDirReg.DeliveryMode = APIC_MT_LowestPriority;
 699         ReDirReg.DestinationMode = APIC_DM_Logical;
 700         ReDirReg.Destination = 0;
 701     }
 702
 703     /* Check if the destination is logical */
 704     if (ReDirReg.DestinationMode == APIC_DM_Logical)
 705     {
 706         /* Set the bit for this cpu */
 707         ReDirReg.Destination |= ApicLogicalId(Prcb->Number);
 708     }
 709
 710     /* Set the trigger mode */
 711     ReDirReg.TriggerMode = 1 - InterruptMode;
 712
 713     /* Now unmask it */
 714     ReDirReg.Mask = FALSE;
 715
 716     /* Write back the entry */
 717     ApicWriteIORedirectionEntry(Index, ReDirReg);
 718
 719     return TRUE;
 720 }
 721
 722 VOID
 723 NTAPI
 724 HalDisableSystemInterrupt(
 725     IN ULONG Vector,
 726     IN KIRQL Irql)
 727 {
 728     IOAPIC_REDIRECTION_REGISTER ReDirReg;
 729     UCHAR Index;
 730     ASSERT(Irql <= HIGH_LEVEL);
 731     ASSERT(Vector < RTL_NUMBER_OF(HalpVectorToIndex));
 732
 733     Index = HalpVectorToIndex[Vector];
 734
 735     /* Read lower dword of redirection entry */
 736     ReDirReg.Long0 = IOApicRead(IOAPIC_REDTBL + 2 * Index);
 737
 738     /* Mask it */
 739     ReDirReg.Mask = 1;
 740
 741     /* Write back lower dword */
 742     IOApicWrite(IOAPIC_REDTBL + 2 * Irql, ReDirReg.Long0);
 743 }
 744
 745 #ifndef _M_AMD64
 746 BOOLEAN
 747 NTAPI
 748 HalBeginSystemInterrupt(
 749     IN KIRQL Irql,
 750     IN ULONG Vector,
 751     OUT PKIRQL OldIrql)
 752 {
 753     KIRQL CurrentIrql;
 754
 755     /* Get the current IRQL */
 756     CurrentIrql = ApicGetCurrentIrql();
 757
 758 #ifdef APIC_LAZY_IRQL
 759     /* Check if this interrupt is allowed */
 760     if (CurrentIrql >= Irql)
 761     {
 762         IOAPIC_REDIRECTION_REGISTER RedirReg;
 763         UCHAR Index;
 764
 765         /* It is not, set the real Irql in the TPR! */
 766         ApicWrite(APIC_TPR, IrqlToTpr(CurrentIrql));
 767
 768         /* Save the new hard IRQL in the IRR field */
 769         KeGetPcr()->IRR = CurrentIrql;
 770
 771         /* End this interrupt */
 772         ApicSendEOI();
 773
 774         /* Get the irq for this vector */
 775         Index = HalpVectorToIndex[Vector];
 776
 777         /* Check if its valid */
 778         if (Index != 0xff)
 779         {
 780             /* Read the I/O redirection entry */
 781             RedirReg = ApicReadIORedirectionEntry(Index);
 782
 783             /* Re-request the interrupt to be handled later */
 784             ApicRequestInterrupt(Vector, (UCHAR)RedirReg.TriggerMode);
 785        }
 786        else
 787        {
 788             /* Re-request the interrupt to be handled later */
 789             ApicRequestInterrupt(Vector, APIC_TGM_Edge);
 790        }
 791
 792         /* Pretend it was a spurious interrupt */
 793         return FALSE;
 794     }
 795 #endif
 796     /* Save the current IRQL */
 797     *OldIrql = CurrentIrql;
 798
 799     /* Set the new IRQL */
 800     ApicRaiseIrql(Irql);
 801
 802     /* Turn on interrupts */
 803     _enable();
 804
 805     /* Success */
 806     return TRUE;
 807 }
 808
 809 VOID
 810 NTAPI
 811 HalEndSystemInterrupt(
 812     IN KIRQL OldIrql,
 813     IN PKTRAP_FRAME TrapFrame)
 814 {
 815     /* Send an EOI */
 816     ApicSendEOI();
 817
 818     /* Restore the old IRQL */
 819     ApicLowerIrql(OldIrql);
 820 }
 821
 822
 823 /* IRQL MANAGEMENT ************************************************************/
 824
 825 KIRQL
 826 NTAPI
 827 KeGetCurrentIrql(VOID)
 828 {
 829     /* Read the current TPR and convert it to an IRQL */
 830     return ApicGetCurrentIrql();
 831 }
 832
 833 VOID
 834 FASTCALL
 835 KfLowerIrql(
 836     IN KIRQL OldIrql)
 837 {
 838 #if DBG
 839     /* Validate correct lower */
 840     if (OldIrql > ApicGetCurrentIrql())
 841     {
 842         /* Crash system */
 843         KeBugCheck(IRQL_NOT_LESS_OR_EQUAL);
 844     }
 845 #endif
 846     /* Set the new IRQL */
 847     ApicLowerIrql(OldIrql);
 848 }
 849
 850 KIRQL
 851 FASTCALL
 852 KfRaiseIrql(
 853     IN KIRQL NewIrql)
 854 {
 855     KIRQL OldIrql;
 856
 857     /* Read the current IRQL */
 858     OldIrql = ApicGetCurrentIrql();
 859 #if DBG
 860     /* Validate correct raise */
 861     if (OldIrql > NewIrql)
 862     {
 863         /* Crash system */
 864         KeBugCheck(IRQL_NOT_GREATER_OR_EQUAL);
 865     }
 866 #endif
 867     /* Convert the new IRQL to a TPR value and write the register */
 868     ApicRaiseIrql(NewIrql);
 869
 870     /* Return old IRQL */
 871     return OldIrql;
 872 }
 873
 874 KIRQL
 875 NTAPI
 876 KeRaiseIrqlToDpcLevel(VOID)
 877 {
 878     return KfRaiseIrql(DISPATCH_LEVEL);
 879 }
 880
 881 KIRQL
 882 NTAPI
 883 KeRaiseIrqlToSynchLevel(VOID)
 884 {
 885     return KfRaiseIrql(SYNCH_LEVEL);
 886 }
 887
 888 #endif /* !_M_AMD64 */
 889