XHCI: set dcbaap

This will be needed for USB (XHCI controller driver).

Rsconscript

@@ -11,6 +11,7 @@ configure do
   check_c_compiler
   check_program "mformat", on_fail: "Install the mtools package"
   check_program "parted"
+  check_program "pkg-config"
   check_cfg package: "freetype2", on_fail: "Install libfreetype-dev", use: "freetype"
   sh %w[git submodule update --init]
 end
@@ -113,10 +114,8 @@ hulk_env = env "hulk", use: %w[ldc2 x86_64-elf-gcc] do |env|
     -avx512bw
     -avx512cd
     -avx512dq
-    -avx512er
     -avx512f
     -avx512ifma
-    -avx512pf
     -avx512vbmi
     -avx512vbmi2
     -avx512vl
@@ -132,11 +131,11 @@ hulk_env = env "hulk", use: %w[ldc2 x86_64-elf-gcc] do |env|
     -sse4a
     -ssse3
   ]
-  env["DFLAGS"] += %W[-mtriple=x86_64-unknown-elf -mattr=#{cpu_attrs.join(",")} --betterC -release -O3 --wi --enable-cross-module-inlining -code-model=large]
+  env["DFLAGS"] += %W[-g -mtriple=x86_64-unknown-elf -mattr=#{cpu_attrs.join(",")} --betterC -release -O3 --wi --enable-cross-module-inlining -code-model=kernel --disable-red-zone]
   env["D_IMPORT_PATH"] += %w[src]
   env["D_IMPORT_PATH"] << env.expand("^/src")
   env["LD"] = "x86_64-elf-gcc"
-  env["LDFLAGS"] += %w[-nostdlib -Tsrc/hulk/hulk.ld -Wl,--gc-sections -Wl,-Map,${_TARGET}.map]
+  env["LDFLAGS"] += %w[-g -nostdlib -Tsrc/hulk/hulk.ld -Wl,--gc-sections -Wl,-Map,${_TARGET}.map -Wl,--no-warn-rwx-segments]
   env["LDCMD"] = %w[${LD} -o ${_TARGET} ${LDFLAGS} ${_SOURCES} ${LIBDIRPREFIX}${LIBPATH} ${LIBLINKPREFIX}${LIBS}]
   env["OBJDUMP"] = "x86_64-elf-objdump"
   env["OBJCOPY"] = "x86_64-elf-objcopy"
@@ -171,10 +170,10 @@ hello_env = env "hello", use: %w[ldc2 x86_64-w64-mingw32-gcc] do |env|
   env.Object("^/hulk_bin.o", "^/hulk_bin.S")
   env.depends("^/hulk_bin.o", hulk_env.expand("^/hulk.bin"))
   env["sources"] << "^/hulk_bin.o"
-  env["DFLAGS"] += %w[-mtriple=x86_64-unknown-windows-coff --betterC -release -O3 --wi --enable-cross-module-inlining]
+  env["DFLAGS"] += %w[-g -mtriple=x86_64-unknown-windows-coff --betterC -release -O3 --wi --enable-cross-module-inlining --disable-red-zone]
   env["D_IMPORT_PATH"] += %w[src uefi-d/source]
   env["LD"] = "x86_64-w64-mingw32-gcc"
-  env["LDFLAGS"] += %w[-nostdlib -Wl,-dll -shared -Wl,--subsystem,10 -e efi_main -Wl,-Map,${_TARGET}.map]
+  env["LDFLAGS"] += %w[-g -nostdlib -Wl,-dll -shared -Wl,--subsystem,10 -e efi_main -Wl,-Map,${_TARGET}.map]
   env["LDCMD"] = %w[${LD} -o ${_TARGET} ${LDFLAGS} ${_SOURCES} ${LIBDIRPREFIX}${LIBPATH} ${LIBLINKPREFIX}${LIBS}]
   env["OBJDUMP"] = "x86_64-w64-mingw32-objdump"
   env.Program("^/HOS.EFI", "${sources}")
@@ -194,11 +193,15 @@ task "run", desc: "Run HOS in QEMU" do
   ovmf = "OVMF.fd"
   if File.exist?("/usr/share/edk2/x64/OVMF.fd")
     ovmf = "/usr/share/edk2/x64/OVMF.fd"
+  elsif File.exist?("/usr/share/edk2/x64/OVMF.4m.fd")
+    ovmf = "/usr/share/edk2/x64/OVMF.4m.fd"
   end
   sh %W[
     qemu-system-x86_64
     -machine q35
+    -m 128M
     -cpu max
+    -smp cpus=4
     -serial file:qemu/serial.out
     -bios #{ovmf}
     -drive file=qemu/HOS.img,format=raw

src/hulk/acpi.d

@@ -6,14 +6,14 @@ module hulk.acpi;
 import hulk.hurl;
 import hulk.klog;
 import hulk.memory;
-import hulk.list;
+import hulk.hurl.a1;
 
 struct Acpi
 {
     /**
      * List of ACPI tables.
      */
-    private static __gshared List!(Header *) tables;
+    private static __gshared Header *[] tables;
 
     enum uint XSDT_SIGNATURE = signature("XSDT");
 
@@ -65,6 +65,8 @@ struct Acpi
             map_table(acpi_xsdt_phys, xsdt.header.length);
         }
         size_t n_entries = (xsdt.header.length - xsdt.header.sizeof) / xsdt.tables[0].sizeof;
+        Header ** table_pointers = cast(Header **)A1.allocate(n_entries * (Header *).sizeof);
+        tables = table_pointers[0..n_entries];
         for (size_t i = 0u; i < n_entries; i++)
         {
             ulong address = xsdt.tables[i];
@@ -75,7 +77,7 @@ struct Acpi
             {
                 map_table(address, length);
             }
-            tables.add(header);
+            tables[i] = header;
             uint signature = header.signature;
             Klog.writefln("Found ACPI table '%c%c%c%c' at %p",
                     signature & 0xFFu,

src/hulk/apic.d

@@ -5,7 +5,7 @@ module hulk.apic;
 
 import hulk.hurl;
 import hulk.acpi;
-import hulk.idt;
+import hulk.him;
 import hulk.rtc;
 import hulk.pit;
 import hulk.klog;
@@ -13,9 +13,11 @@ import hulk.memory;
 
 struct Apic
 {
+    private enum DEBUG = false;
+    private enum DEFAULT_IO_APIC_ADDRESS = 0xFEC0_0000u;
     private enum uint PERIODIC_MODE = 0x2_0000u;
 
-    private enum ulong IRQ_PIT = 2u;
+    private enum ulong IRQ_PIT = 0u;
     private enum ulong IRQ_RTC = 8u;
 
     /**
@@ -28,16 +30,112 @@ struct Apic
          */
         static struct Entry
         {
+            enum PROCESSOR_LOCAL_APIC = 0;
+            enum IO_APIC = 1;
+            enum IO_APIC_INTERRUPT_SOURCE_OVERRIDE = 2;
+            enum IO_APIC_NMI_SOURCE = 3;
+            enum LOCAL_APIC_NMI = 4;
             enum LOCAL_APIC_ADDRESS_OVERRIDE = 5;
+            enum PROCESSOR_LOCAL_X2APIC = 9;
 
             ubyte type;
             ubyte length;
         }
 
+        /**
+         * Processor Local APIC.
+         */
+        static struct Entry0
+        {
+            Entry header;
+            ubyte acpi_processor_id;
+            ubyte apic_id;
+            uint flags;
+        }
+        static assert(Entry0.sizeof == 8);
+
+        /**
+         * I/O APIC.
+         */
+        static struct Entry1
+        {
+            Entry header;
+            ubyte io_apic_id;
+            ubyte _reserved;
+            uint io_apic_address;
+            uint global_system_interrupt_base;
+        }
+        static assert(Entry1.sizeof == 12);
+
+        /**
+         * IO/APIC Interrupt Source Override.
+         */
+        static struct Entry2
+        {
+            Entry header;
+            ubyte bus_source;
+            ubyte irq_source;
+            uint global_system_interrupt;
+            ushort flags;
+            ushort _reserved;
+        }
+        static assert(Entry2.sizeof == 12);
+
+        /**
+         * IO/APIC Non-maskable interrupt source.
+         */
+        static struct Entry3
+        {
+            Entry header;
+            ubyte nmi_source;
+            ubyte _reserved;
+            ushort flags;
+            ubyte[4] global_system_interrupt;
+        }
+        static assert(Entry3.sizeof == 10);
+
+        /**
+         * Local APIC Non-maskable interrupts.
+         */
+        static struct Entry4
+        {
+            Entry header;
+            ubyte acpi_processor_id;
+            ubyte[2] flags;
+            ubyte lint_nr;
+        }
+        static assert(Entry4.sizeof == 6);
+
+        /**
+         * Local APIC Address Override.
+         */
+        static struct Entry5
+        {
+            Entry header;
+            ushort _reserved;
+            ubyte[8] local_apic_address;
+        }
+        static assert(Entry5.sizeof == 12);
+
+        /**
+         * Processor Local x2APIC.
+         */
+        static struct Entry9
+        {
+            Entry header;
+            ushort _reserved;
+            uint local_x2apic_id;
+            uint flags;
+            uint acpi_id;
+        }
+        static assert(Entry9.sizeof == 16);
+
         Acpi.Header header;
         uint local_apic_address;
         uint flags;
 
+        static assert(MADT.sizeof == 0x2C);
+
         /**
          * Scan the MADT table.
          */
@@ -46,14 +144,107 @@ struct Apic
             apic_registers = cast(ApicRegisters *)local_apic_address;
             const(void) * end = cast(const(void) *)(&this) + header.length;
             const(Entry) * entry = cast(const(Entry) *)(cast(ulong)&this + MADT.sizeof);
+            bool first_io_apic = true;
+            size_t n_cpus = 0u;
             while (entry < end)
             {
-                entry = cast(const(Entry) *)(cast(size_t)entry + entry.length);
-                if (entry.type == Entry.LOCAL_APIC_ADDRESS_OVERRIDE)
+                if (DEBUG)
                 {
-                    /* Found a 64-bit Local APIC Address Override entry. */
-                    memcpy(cast(void *)&apic_registers, cast(const(void) *)entry + 4u, apic_registers.sizeof);
+                    Klog.writefln("MADT entry type %u, length %u", entry.type, entry.length);
                 }
+                switch (entry.type)
+                {
+                case Entry.PROCESSOR_LOCAL_APIC:
+                    Entry0 * e = cast(Entry0 *)entry;
+                    n_cpus++;
+                    if (DEBUG)
+                    {
+                        Klog.writefln("  ACPI processor ID: %u, APIC ID: %u, Flags: 0x%x",
+                                e.acpi_processor_id, e.apic_id, e.flags);
+                    }
+                    break;
+
+                case Entry.IO_APIC:
+                    Entry1 * e = cast(Entry1 *)entry;
+                    if (first_io_apic)
+                    {
+                        io_apic_registers = cast(IoApicRegisters *)e.io_apic_address;
+                        first_io_apic = false;
+                    }
+                    if (DEBUG)
+                    {
+                        Klog.writefln("  I/O APIC ID: %u, I/O APIC Address: 0x%x, GSIB: %u",
+                                e.io_apic_id, e.io_apic_address, e.global_system_interrupt_base);
+                    }
+                    break;
+
+                case Entry.IO_APIC_INTERRUPT_SOURCE_OVERRIDE:
+                    Entry2 * e = cast(Entry2 *)entry;
+                    if (e.global_system_interrupt < INT_APIC_COUNT)
+                    {
+                        irq_redirects[e.irq_source] = cast(ubyte)e.global_system_interrupt;
+                    }
+                    else
+                    {
+                        Klog.writefln("Warning: unexpected interrupt source override global system interrupt value: %u", e.global_system_interrupt);
+                    }
+                    if (DEBUG)
+                    {
+                        Klog.writefln("  Bus %u IRQ %u GSI %u Flags: 0x%x",
+                                e.bus_source, e.irq_source, e.global_system_interrupt, e.flags);
+                    }
+                    break;
+
+                case Entry.IO_APIC_NMI_SOURCE:
+                    Entry3 * e = cast(Entry3 *)entry;
+                    if (DEBUG)
+                    {
+                        uint global_system_interrupt;
+                        memcpy(&global_system_interrupt, &e.global_system_interrupt, 4u);
+                        Klog.writefln("  NMI Source: %u, Flags: 0x%x, GSI: %u",
+                                e.nmi_source, e.flags, global_system_interrupt);
+                    }
+                    break;
+
+                case Entry.LOCAL_APIC_NMI:
+                    Entry4 * e = cast(Entry4 *)entry;
+                    if (DEBUG)
+                    {
+                        ushort flags;
+                        memcpy(&flags, &e.flags, 2u);
+                        Klog.writefln("  ACPI Processor ID: %u, Flags: 0x%x, LINT#: %u",
+                                e.acpi_processor_id, flags, e.lint_nr);
+                    }
+                    break;
+
+                case Entry.LOCAL_APIC_ADDRESS_OVERRIDE:
+                    Entry5 * e = cast(Entry5 *)entry;
+                    if (DEBUG)
+                    {
+                        memcpy(&apic_registers, &e.local_apic_address, 8u);
+                        Klog.writefln("  Local APIC Address: 0x%x", apic_registers);
+                    }
+                    break;
+
+                case Entry.PROCESSOR_LOCAL_X2APIC:
+                    Entry9 * e = cast(Entry9 *)entry;
+                    if (DEBUG)
+                    {
+                        Klog.writefln("  x2APIC ID: %u, Flags: 0x%x, ACPI ID: %u",
+                                e.local_x2apic_id, e.flags, e.acpi_id);
+                    }
+                    break;
+
+                default:
+                    break;
+                }
+
+                /* Move to next entry. */
+                entry = cast(const(Entry) *)(cast(size_t)entry + entry.length);
+            }
+            if (n_cpus > 0u)
+            {
+                Klog.writefln("%u CPU(s) found", n_cpus);
             }
         }
     }
@@ -113,8 +304,11 @@ struct Apic
         ApicRegister data;
     }
 
+    alias ISRHandler = void function();
     private static __gshared ApicRegisters * apic_registers;
     private static __gshared IoApicRegisters * io_apic_registers;
+    private static __gshared ISRHandler[INT_APIC_COUNT] irq_handlers;
+    private static __gshared ubyte[INT_APIC_COUNT] irq_redirects;
 
     /**
      * Initialize APIC.
@@ -122,7 +316,11 @@ struct Apic
     public static void initialize()
     {
         Klog.writefln("\a3Initializing APIC");
-        io_apic_registers = cast(IoApicRegisters *)0xFEC0_0000u;
+        io_apic_registers = cast(IoApicRegisters *)DEFAULT_IO_APIC_ADDRESS;
+        for (ubyte i = 0u; i < INT_APIC_COUNT; i++)
+        {
+            irq_redirects[i] = i;
+        }
         MADT * madt = cast(MADT *)Acpi.get_table("APIC");
         madt.scan();
         Klog.writefln("APIC found at %p", apic_registers);
@@ -133,26 +331,28 @@ struct Apic
                 PT_WRITABLE | PT_WRITE_THROUGH | PT_DISABLE_CACHE | PT_NO_EXECUTE);
         /* Enable local APIC to receive interrupts and set spurious interrupt
          * vector to 0xFF. */
-        apic_registers.spurious_interrupt_vector.value = 0x100u | Idt.INT_APIC_SPURIOUS;
-        apic_registers.lvt_timer.value = Idt.INT_APIC_TIMER;
-        apic_registers.lvt_lint[0].value = Idt.INT_APIC_LINT0;
-        apic_registers.lvt_lint[1].value = Idt.INT_APIC_LINT1;
+        apic_registers.spurious_interrupt_vector.value = 0x100u | INT_APIC_SPURIOUS;
+        apic_registers.lvt_timer.value = INT_APIC_TIMER;
+        apic_registers.lvt_lint[0].value = INT_APIC_LINT0;
+        apic_registers.lvt_lint[1].value = INT_APIC_LINT1;
         /* Enable PIT interrupt. */
-        configure_io_apic_irq(IRQ_PIT, Idt.INT_APIC_BASE + IRQ_PIT);
+        configure_io_apic_irq(IRQ_PIT, &Pit.isr);
         /* Enable RTC interrupt. */
-        configure_io_apic_irq(IRQ_RTC, Idt.INT_APIC_BASE + IRQ_RTC);
+        configure_io_apic_irq(IRQ_RTC, &Rtc.isr);
     }
 
     /**
      * Configure routing for an I/O APIC IRQ.
      */
-    private static void configure_io_apic_irq(size_t io_apic_irq, size_t interrupt_id)
+    private static void configure_io_apic_irq(size_t orig_irq, ISRHandler isr_handler)
     {
-        ulong entry = interrupt_id;
+        size_t io_apic_irq = irq_redirects[orig_irq];
+        ulong entry = INT_APIC_BASE + io_apic_irq;
         io_apic_registers.address.value = cast(uint)(0x10u + io_apic_irq * 2u);
         io_apic_registers.data.value = entry & 0xFFFF_FFFFu;
         io_apic_registers.address.value = cast(uint)(0x10u + io_apic_irq * 2u + 1u);
         io_apic_registers.data.value = entry >> 32u;
+        irq_handlers[io_apic_irq] = isr_handler;
     }
 
     /**
@@ -168,19 +368,14 @@ struct Apic
      */
     public static void isr(ulong vector)
     {
-        switch (vector)
+        ulong io_apic_irq = vector - INT_APIC_BASE;
+        if ((io_apic_irq < INT_APIC_COUNT) && (irq_handlers[io_apic_irq] != null))
+        {
+            irq_handlers[io_apic_irq]();
+        }
+        else
         {
-        case Idt.INT_APIC_BASE + IRQ_PIT:
-            Pit.isr();
-            break;
-
-        case Idt.INT_APIC_BASE + IRQ_RTC:
-            Rtc.isr();
-            break;
-
-        default:
             Klog.writefln("Unhandled APIC ISR %u", vector);
-            break;
         }
         eoi();
     }

src/hulk/cpu.d

@@ -57,6 +57,7 @@ enum ulong XCR0_PKRU = 0x100u;
 /** @} */
 
 /** MSR register numbers. @{ */
+enum uint MSR_PAT = 0x277u;
 enum uint MSR_EFER = 0xC000_0080u;
 /** @} */
 
@@ -185,6 +186,14 @@ void write_cr4(ulong v)
     __asm("mov $0, %cr4", "r", v);
 }
 
+ulong read_rflags()
+{
+    return __asm!ulong(`
+            pushf
+            mov (%rsp), %rax
+            add $$8, %rsp`, "={rax}");
+}
+
 ulong rdmsr(uint msr)
 {
     return __asm!ulong(`
@@ -245,3 +254,8 @@ void cpuid1(uint * ebx, uint * ecx, uint * edx)
 {
     __asm("cpuid", "=*{ebx},=*{ecx},=*{edx},{eax}", ebx, ecx, edx, 1u);
 }
+
+void swint(ubyte swint_id)()
+{
+    mixin(`__asm("int $$`, cast(int)cast(uint)swint_id, `", "");`);
+}

src/hulk/him.d

@@ -0,0 +1,329 @@
+/**
+ * HULK Interrupt Manager.
+ */
+module hulk.him;
+
+import hulk.gdt;
+import ldc.llvmasm;
+import hulk.fb;
+import hulk.console;
+import hulk.klog;
+import hulk.apic;
+import hulk.cpu;
+import hulk.kfont;
+import hulk.thread;
+
+/** Number if Interrupt Service Routines. */
+private enum N_ISRS = 256;
+
+/** Interrupt stack frame indices. */
+public static enum
+{
+    ISF_RBP,
+    ISF_R15,
+    ISF_R14,
+    ISF_R13,
+    ISF_R12,
+    ISF_R11,
+    ISF_R10,
+    ISF_R9,
+    ISF_R8,
+    ISF_RDI,
+    ISF_RSI,
+    ISF_RDX,
+    ISF_RCX,
+    ISF_RBX,
+    ISF_RAX,
+    ISF_ERROR_CODE,
+    ISF_RIP,
+    ISF_CS,
+    ISF_RFLAGS,
+    ISF_RSP,
+    ISF_SS,
+    ISF_COUNT,
+}
+
+private static __gshared string[ISF_COUNT] ISF_NAMES = [
+    "RBP",
+    "R15",
+    "R14",
+    "R13",
+    "R12",
+    "R11",
+    "R10",
+    "R9",
+    "R8",
+    "RDI",
+    "RSI",
+    "RDX",
+    "RCX",
+    "RBX",
+    "RAX",
+    "Error Code",
+    "RIP",
+    "CS",
+    "RFLAGS",
+    "RSP",
+    "SS",
+];
+
+/** Interrupt IDs. @{ */
+public enum ulong INT_PAGE_FAULT = 0x0Eu;
+
+/* The I/O APIC is configured to map IRQ 0 to interrupt 64 (0x40). */
+public enum ulong INT_APIC_BASE = 0x40u; /* IRQ 0 */
+public enum ulong INT_APIC_COUNT = 24u;
+
+public enum ulong INT_APIC_TIMER = 0x70u;
+public enum ulong INT_APIC_LINT0 = 0x71u;
+public enum ulong INT_APIC_LINT1 = 0x72u;
+public enum ulong INT_KERNEL_SWINT = 0x80u;
+public enum ulong INT_APIC_SPURIOUS = 0xFFu;
+/** @} */
+
+static foreach(isr; 0 .. N_ISRS)
+{
+    mixin(`private extern(C) void isr_`, isr, `();`);
+}
+
+private string isrs_list(int index = 0)()
+{
+    static if (index < 256)
+    {
+        return "&isr_" ~ index.stringof ~ ",\n" ~ isrs_list!(index + 1)();
+    }
+    else
+    {
+        return "";
+    }
+}
+
+mixin(`private static __gshared extern(C) void function()[N_ISRS] isrs = [`,
+        isrs_list(),
+        `];`);
+
+/** Interrupt suspend level (0 when interrupts are enabled). */
+private __gshared size_t interrupt_suspend_level;
+
+/** Interrupt nest level (0 when not in an interrupt handler). */
+private __gshared size_t interrupt_level;
+
+/**
+ * Suspend interrupts.
+ *
+ * This function disables interrupts. Calls can be nested and interrupts will
+ * be enabled only when the outermost pair of
+ * suspend_interrupts()/resume_interrupts() is complete.
+ */
+void suspend_interrupts()
+{
+    if (interrupt_level == 0)
+    {
+        cli();
+        interrupt_suspend_level++;
+    }
+}
+
+/**
+ * Resume interrupts.
+ *
+ * Calls can be nested and interrupts will be enabled only when the outermost
+ * pair of suspend_interrupts()/resume_interrupts() is complete.
+ */
+void resume_interrupts()
+{
+    if (interrupt_level == 0)
+    {
+        interrupt_suspend_level--;
+        if (interrupt_suspend_level == 0)
+        {
+            sti();
+        }
+    }
+}
+
+struct Him
+{
+    /**
+     * IDT descriptor entry type.
+     */
+    struct descriptor_t
+    {
+        ulong[2] descriptor;
+
+        this(T)(size_t index, ulong dpl, bool trap, T offset)
+        {
+            descriptor[0] =
+                0x0000_8E_00_0000_0000u |
+                ((cast(ulong)offset & 0xFFFF_0000u) << 32u) |
+                (dpl << 45u) |
+                ((trap ? 1uL : 0uL) << 40u) |
+                (Gdt.SELECTOR_KERNEL_CODE << 16u) |
+                (cast(ulong)offset & 0xFFFFu);
+            descriptor[1] = cast(ulong)offset >> 32u;
+        }
+    }
+
+    /**
+     * IDTR register format.
+     */
+    struct idtr_t
+    {
+        ushort limit;
+        align(2) ulong offset;
+    }
+    static assert(idtr_t.sizeof == 10u);
+
+    private static __gshared idtr_t idtr;
+    private static __gshared align(0x10) descriptor_t[N_ISRS] idt;
+    static assert(idt.sizeof == (N_ISRS * 16u));
+
+    public static void initialize()
+    {
+        set_isrs();
+        idtr.limit = idt.sizeof - 1u;
+        idtr.offset = cast(ulong)&idt;
+        lidt(&idtr);
+    }
+
+    private static void lidt(idtr_t * idtr)
+    {
+        __asm("lidt $0", "*m", idtr);
+    }
+
+    private static void set_isrs()
+    {
+        static bool isr_has_error_code(size_t isr)
+        {
+            return (isr == 8) ||
+                ((10 <= isr) && (isr <= 14)) ||
+                (isr == 17) ||
+                (isr == 21) ||
+                (isr == 29) ||
+                (isr == 30);
+        }
+
+        static bool isr_is_trap(size_t isr)
+        {
+            return (isr < 32u) && (isr != 2u);
+        }
+
+        __asm("jmp 1f", "");
+        static foreach(isr; 0 .. N_ISRS)
+        {
+            mixin("__asm(`isr_", isr, ":
+                " ~ (isr_has_error_code(isr) ? "" : "pushq $$0") ~ "
+                pushq %rax
+                mov $$", isr, ", %rax
+                jmp isr_common`, ``);");
+        }
+
+        __asm(`isr_common:
+            pushq %rbx
+            pushq %rcx
+            pushq %rdx
+            pushq %rsi
+            pushq %rdi
+            pushq %r8
+            pushq %r9
+            pushq %r10
+            pushq %r11
+            pushq %r12
+            pushq %r13
+            pushq %r14
+            pushq %r15
+            pushq %rbp
+            mov %rsp, %rbp
+            mov %rsp, %rsi
+            and $$0xFFFFFFFFFFFFFFF0, %rsp
+            mov %rsp, %rdx
+            mov %rax, %rdi
+            movabs $$isr, %rax
+            callq *%rax
+            mov %rax, %rsp
+            popq %rbp
+            popq %r15
+            popq %r14
+            popq %r13
+            popq %r12
+            popq %r11
+            popq %r10
+            popq %r9
+            popq %r8
+            popq %rdi
+            popq %rsi
+            popq %rdx
+            popq %rcx
+            popq %rbx
+            popq %rax
+            add $$8, %rsp
+            iretq
+            1:`, ``);
+
+        for (size_t isr = 0; isr < N_ISRS; isr++)
+        {
+            idt[isr] = descriptor_t(isr, 0u, isr_is_trap(isr), isrs[isr]);
+        }
+    }
+
+}
+
+/**
+ * Interrupt service routine.
+ *
+ * This function returns the stack pointer of the thread to execute when
+ * returning from this function.
+ */
+public extern(C) ulong * isr(ulong vector, ulong * stack_frame, ulong * rsp)
+{
+    interrupt_level++;
+    if (interrupt_level == 1)
+    {
+        Thread.current_thread.stack_pointer = stack_frame;
+    }
+    if ((vector >= INT_APIC_BASE) && (vector < (INT_APIC_BASE + INT_APIC_COUNT)))
+    {
+        Apic.isr(vector);
+    }
+    else
+    {
+        switch (vector)
+        {
+        case INT_APIC_TIMER:
+        case INT_APIC_LINT0:
+        case INT_APIC_LINT1:
+        case INT_APIC_SPURIOUS:
+            Apic.isr(vector);
+            break;
+
+        case INT_KERNEL_SWINT:
+            Thread.handle_swint(stack_frame);
+            break;
+
+        default:
+            Fb.rect(0, 0, Fb.width, Kfont.line_height, 0xCC0000u);
+            Klog.writefln("\n **** Unhandled ISR %u ****", vector);
+            for (size_t i = 0u; i < ISF_COUNT; i++)
+            {
+                Klog.writef(ISF_NAMES[i]);
+                Klog.writefln(" = 0x%x", stack_frame[i]);
+            }
+            Klog.writefln("CR2 = 0x%p", read_cr2());
+            __asm("cli", "");
+            for (;;)
+            {
+                __asm("hlt", "");
+            }
+        }
+    }
+    Thread.current_thread = Thread.current_thread.list_next;
+    interrupt_level--;
+    if (interrupt_level == 0)
+    {
+        return Thread.current_thread.stack_pointer;
+    }
+    else
+    {
+        return rsp;
+    }
+}

src/hulk/hippo.d

@@ -7,21 +7,38 @@
  */
 module hulk.hippo;
 
+import hulk.hurl;
+import hulk.klog;
+
 struct Hippo
 {
     /**
-     * Linked list node entry for a physical page.
+     * Linked list node entry for a single page.
      */
     private static struct PhysicalPage
     {
         PhysicalPage * next;
     }
 
+    /**
+     * Linked list node entry for a region.
+     */
+    private static struct Region
+    {
+        ulong size;
+        Region * next;
+    }
+
     /**
      * Linked list of free physical pages.
      */
     private static __gshared PhysicalPage * free_pages;
 
+    /**
+     * Linked list of free regions.
+     */
+    private static __gshared Region * regions;
+
     /**
      * Number of free physical pages.
      */
@@ -48,22 +65,154 @@ struct Hippo
     public static void * allocate_page()
     {
         PhysicalPage * pp;
-        if (free_pages != null)
+        if (free_pages !is null)
         {
             pp = free_pages;
             free_pages = free_pages.next;
             m_n_free_pages--;
         }
+        else if (regions !is null)
+        {
+            pp = cast(PhysicalPage *)(cast(ulong)regions + regions.size - PAGE_SIZE);
+            regions.size -= PAGE_SIZE;
+            if (regions.size == 0)
+            {
+                regions = regions.next;
+            }
+            m_n_free_pages--;
+        }
         return cast(void *)pp;
     }
 
+    /**
+     * Free a physical memory region.
+     */
+    public static void free_region(T)(T start, ulong size)
+    {
+        if (size == PAGE_SIZE)
+        {
+            free_page(start);
+            return;
+        }
+
+        if (regions is null)
+        {
+            /* The free regions list is empty. Append this new region. */
+            regions = cast(Region *)start;
+            regions.size = size;
+            regions.next = null;
+        }
+        else if (size < regions.size)
+        {
+            /* The new region is smaller than the first one in the list.
+             * Add the new region to the beginning of the list. */
+            Region * new_region = cast(Region *)start;
+            new_region.size = size;
+            new_region.next = regions;
+            regions = new_region;
+        }
+        else
+        {
+            /* Find the last free region that is smaller than this one. */
+            Region * region = regions;
+            while (region.next !is null && region.next.size < size)
+            {
+                region = region.next;
+            }
+
+            /* Add the new region after this entry. */
+            Region * new_region = cast(Region *)start;
+            new_region.size = size;
+            new_region.next = region.next;
+            region.next = new_region;
+        }
+
+        m_n_free_pages += size >> 12;
+    }
+
+    /**
+     * Allocate a region with the given size and alignment.
+     *
+     * @param size Region size.
+     * @param alignment Region alignment.
+     */
+    public static void * allocate_aligned_region(ulong size, ulong alignment)
+    {
+        Region ** regionp = &regions;
+        for (;;)
+        {
+            Region * region = *regionp;
+            if (region is null)
+            {
+                break;
+            }
+
+            /* The size of the region must be large enough */
+            if (region.size >= size)
+            {
+                /* The alignment contstraint must be satisfied. */
+                if ((cast(ulong)region & (alignment - 1)) == 0)
+                {
+                    /* The region start address happens to already be aligned. */
+                    if (region.size > size)
+                    {
+                        /* Region found is larger that needed. */
+                        Region * new_region = cast(Region *)(cast(ulong)region + size);
+                        new_region.size = region.size - size;
+                        new_region.next = region.next;
+                        *regionp = new_region;
+                        return region;
+                    }
+                    else
+                    {
+                        /* Region happens to be the size requested. */
+                        *regionp = region.next;
+                        return region;
+                    }
+                }
+                else
+                {
+                    /* This region is large enough but not aligned. See if an
+                     * aligned subregion can be found at the end. */
+                    ulong start = (cast(ulong)region + region.size - size) & ~(alignment - 1);
+                    if (start > cast(ulong)region)
+                    {
+                        /* The aligned subregion does fit in this region. */
+                        region.size = (start - cast(ulong)region);
+                        ulong region_end = cast(ulong)region + region.size;
+                        if (region_end > (start + size))
+                        {
+                            /* There is another region following the aligned
+                             * subregion that we must reclaim. */
+                            free_region(start + size, region_end - (start + size));
+                        }
+                        return cast(void *)start;
+                    }
+                }
+            }
+
+            regionp = &region.next;
+        }
+        return null;
+    }
+
     /**
      * Get the number of free pages.
      *
      * @return The number of free pages.
      */
-    public @property size_t n_free_pages() const
+    public static @property size_t n_free_pages()
     {
         return m_n_free_pages;
     }
+
+    public static void dump_regions()
+    {
+        Region * region = regions;
+        while (region !is null)
+        {
+            Klog.writefln("HIPPO free region: %p %uKB", region, region.size >> 10);
+            region = region.next;
+        }
+    }
 }

src/hulk/hulk.d

@@ -14,7 +14,7 @@ import hulk.hurl;
 import hulk.hippo;
 import hulk.pci;
 import hulk.gdt;
-import hulk.idt;
+import hulk.him;
 import hulk.cpu;
 import ldc.llvmasm;
 import hulk.pic;
@@ -24,6 +24,9 @@ import hulk.rtc;
 import hulk.serial;
 import hulk.usb;
 import hulk.pit;
+import hulk.time;
+import hulk.test;
+import hulk.thread;
 
 extern extern(C) __gshared ubyte _hulk_bss_size;
 
@@ -31,7 +34,7 @@ extern extern(C) __gshared ubyte _hulk_bss_size;
 private __gshared HulkHeader hulk_header = {
     &_hulk_bss_size, /* hulk_bss_size */
     &hulk_start, /* entry */
-    16u * 1024u, /* stack_size */
+    Thread.STACK_SIZE, /* stack_size */
     Hurl.HULK_BASE, /* virt_base */
     Hurl.HULK_STACK_TOP, /* virt_stack_top */
     Hurl.HULK_FRAMEBUFFER, /* virt_fb_buffer */
@@ -57,11 +60,12 @@ private void initialize_cpu()
  */
 void hulk_start()
 {
-    cli();
+    suspend_interrupts();
+
     initialize_cpu();
     Serial.initialize();
     Gdt.initialize();
-    Idt.initialize();
+    Him.initialize();
     Fb.initialize(cast(uint *)Hurl.HULK_FRAMEBUFFER,
             cast(uint *)hulk_header.bootinfo.fb_buffer1_phys,
             hulk_header.bootinfo.fb.width,
@@ -78,17 +82,33 @@ void hulk_start()
     Acpi.initialize(hulk_header.bootinfo.acpi_xsdt_phys);
     Apic.initialize();
     Rtc.initialize();
-    /* Read the current system time. */
-    Rtc.time t = Rtc.read_rtc_time();
-    Klog.writefln("System time is 20%02u-%02u-%02u %02u:%02u:%02u",
-            t.year, t.month, t.day, t.hour, t.minute, t.second);
     Pit.initialize();
     Pci.initialize();
-    Usb.initialize();
-    sti();
+    Thread.initialize();
 
+    resume_interrupts();
+
+    Usb.initialize();
+
+    /* Run kernel tests. */
+    Test.run();
+    Klog.writefln("\a5HULK Initialization Complete!");
+//    Thread.start(&th);
+//    Time.msleep(1);
+//    Time.msleep(500);
+//    Klog.writefln("Main thread!");
+    /* Idle loop. */
     for (;;)
     {
         hlt();
     }
 }
+
+void th()
+{
+    for (;;)
+    {
+        Klog.writefln("Hello from thread.");
+        Time.msleep(1000);
+    }
+}

src/hulk/hulk.ld

@@ -1,6 +1,6 @@
 SECTIONS
 {
-    . = 0xFFFF800000000000;
+    . = 0xFFFFFFFF80000000;
     _hulk_mem_start = .;
     _hulk_header_start = .;
 

src/hulk/hurl/hurl.d

@@ -18,16 +18,16 @@ import hulk.util;
 struct Hurl
 {
     /** HULK base address. */
-    enum ulong HULK_BASE = 0xFFFF_8000_0000_0000u;
+    enum ulong HULK_BASE = 0xFFFF_FFFF_8000_0000u;
 
     /** HULK stack top address. */
-    enum ulong HULK_STACK_TOP = 0xFFFF_8001_0000_0000u;
+    enum ulong HULK_STACK_TOP = 0xFFFF_FFFF_9000_0000u;
 
     /** HULK framebuffer address. */
-    enum ulong HULK_FRAMEBUFFER = 0xFFFF_8001_0000_0000u;
+    enum ulong HULK_FRAMEBUFFER = 0xFFFF_FFFF_9000_0000u;
 
     /** A1 allocator range base address. */
-    enum ulong A1_BASE = 0xFFFF_8002_0000_0000u;
+    enum ulong A1_BASE = 0xFFFF_FFFF_A000_0000u;
 
     /**
      * Pointer to the base page table.
@@ -91,7 +91,7 @@ struct Hurl
         map_range(HULK_FRAMEBUFFER,
                 cast(ulong)header.bootinfo.fb.buffer,
                 header.bootinfo.fb.height * header.bootinfo.fb.stride * 4u,
-                PT_WRITABLE | PT_NO_EXECUTE);
+                PT_PAT | PT_WRITABLE | PT_NO_EXECUTE);
         /* Map framebuffer buffer1. */
         map_range(cast(ulong)header.bootinfo.fb_buffer1_phys,
                 cast(ulong)header.bootinfo.fb_buffer1_phys,
@@ -157,13 +157,14 @@ struct Hurl
         size_t usable_memory;
         size_t physical_address_limit;
         const(size_t) fb_size = round_up_power_2(header.bootinfo.fb.height * header.bootinfo.fb.stride * 4u, PAGE_SIZE);
-        ulong[2][2] reserved = [
+        ulong[2][2] reserved_regions = [
             [header.bootinfo.hulk_phys, LinkerAddresses.hulk_binary_size],
             [cast(ulong)header.bootinfo.fb.buffer, fb_size],
         ];
-        for (size_t ri = 0u; ri < reserved.length; ri++)
+        for (size_t ri = 0u; ri < reserved_regions.length; ri++)
         {
-            reserved[ri][1] += reserved[ri][0];
+            Klog.writefln("Reserved region %p %uKB", reserved_regions[ri][0], reserved_regions[ri][1] >> 10);
+            reserved_regions[ri][1] += reserved_regions[ri][0];
         }
         for (size_t bii = 0u; bii < header.bootinfo.memory_map_count; bii++)
         {
@@ -171,32 +172,21 @@ struct Hurl
                 (header.bootinfo.memory_map[bii].type == BootInfo.MemoryRegion.Type.Conventional))
             {
                 ulong phys = header.bootinfo.memory_map[bii].base;
-                ulong phys_end = phys + header.bootinfo.memory_map[bii].size;
+                ulong size = header.bootinfo.memory_map[bii].size;
+                ulong phys_end = phys + size;
                 if (phys_end > physical_address_limit)
                 {
                     physical_address_limit = phys_end;
                 }
                 usable_memory += header.bootinfo.memory_map[bii].size;
-                while (phys < phys_end)
-                {
-                    bool is_reserved = false;
-                    for (size_t ri = 0u; ri < reserved.length; ri++)
-                    {
-                        if ((reserved[ri][0] <= phys) && (phys < reserved[ri][1]))
-                        {
-                            is_reserved = true;
-                            break;
-                        }
-                    }
-                    if (!is_reserved)
-                    {
-                        Hippo.free_page(phys);
-                    }
-                    phys += PAGE_SIZE;
-                }
+                add_region(phys, size, reserved_regions);
             }
         }
 
+        /* Set PAT4 to indicate Write Combining so we can set the PAT bit in a
+         * page table entry to use Write Combining for that page. */
+        wrmsr(MSR_PAT, (rdmsr(MSR_PAT) & 0xFFFF_FF00_FFFF_FFFFu) | 0x0000_0001_0000_0000u);
+
         /*
          * Now that we have available physical pages to allocate from, we can
          * build new page tables to replace the bootloader page tables.
@@ -219,6 +209,60 @@ struct Hurl
         usable_memory += fb_size;
         Klog.writefln("Usable memory: %uKB", usable_memory >> 10u);
         Klog.writefln("Kernel size: %uKB", (LinkerAddresses.hulk_binary_size + LinkerAddresses.hulk_bss_size + header.stack_size) >> 10u);
+        Klog.writefln("Free pages: %u", Hippo.n_free_pages);
+//        Hippo.dump_regions();
+    }
+
+    /**
+     * Add a region of available physical memory.
+     *
+     * @param start Start address of region.
+     * @param size Size of region.
+     * @param reserved_regions List of reserved regions to exclude.
+     */
+    private static void add_region(ulong start, ulong size, ulong[2][] reserved_regions)
+    {
+        ulong end = start + size;
+        bool split;
+
+        /* Skip empty regions. */
+        if (size == 0)
+        {
+            return;
+        }
+
+        for (size_t rri = 0; rri < reserved_regions.length; rri++)
+        {
+            /* Skip this region if it is entirely within a reserved region. */
+            if (reserved_regions[rri][0] <= start && reserved_regions[rri][1] >= end)
+            {
+                return;
+            }
+
+            /* Check if the region is partly reserved but the beginning is free. */
+            if (reserved_regions[rri][0] > start && reserved_regions[rri][0] < end)
+            {
+                split = true;
+                add_region(start, reserved_regions[rri][0] - start, reserved_regions);
+            }
+
+            /* Check if the region is partly reserved but the end is free. */
+            if (reserved_regions[rri][1] > start && reserved_regions[rri][1] < end)
+            {
+                split = true;
+                add_region(reserved_regions[rri][1], end - reserved_regions[rri][1], reserved_regions);
+            }
+
+            if (split)
+            {
+                return;
+            }
+        }
+
+        /* If a region makes it here, it did not overlap with any reserved
+         * region, so the entire region can be used. */
+//        Klog.writefln("HURL: freeing region %p %uKB", start, size >> 10);
+        Hippo.free_region(start, size);
     }
 
     public static void map(ulong virtual, ulong physical, ulong flags)

src/hulk/idt.d

@@ -1,133 +0,0 @@
-/**
- * IDT (Interrupt Descriptor Table) functionality.
- */
-module hulk.idt;
-
-import hulk.gdt;
-import ldc.llvmasm;
-import hulk.fb;
-import hulk.console;
-import hulk.klog;
-import hulk.apic;
-
-struct Idt
-{
-    /* The I/O APIC is configured to map IRQ 0 to interrupt 64 (0x40). */
-    public static enum ulong INT_APIC_BASE = 0x40u; /* IRQ 0 */
-    public static enum ulong INT_APIC_COUNT = 24u;
-
-    public static enum ulong INT_APIC_TIMER = 0x70u;
-    public static enum ulong INT_APIC_LINT0 = 0x71u;
-    public static enum ulong INT_APIC_LINT1 = 0x72u;
-    public static enum ulong INT_APIC_SPURIOUS = 0xFFu;
-
-    struct idtr_t
-    {
-        ushort limit;
-        align(2) ulong offset;
-    }
-    static assert(idtr_t.sizeof == 10u);
-
-    alias descriptor_t = ulong[2];
-
-    static __gshared align(0x10) descriptor_t[256] idt;
-    static assert(idt.sizeof == (256u * 16u));
-    static __gshared idtr_t idtr;
-
-    public static void initialize()
-    {
-        set_isrs();
-        idtr.limit = idt.sizeof - 1u;
-        idtr.offset = cast(ulong)&idt;
-        lidt(&idtr);
-    }
-
-    private static void lidt(idtr_t * idtr)
-    {
-        __asm("lidt $0", "*m", idtr);
-    }
-
-    private static bool isr_has_error_code(size_t isr)
-    {
-        return (isr == 8) ||
-            ((10 <= isr) && (isr <= 14)) ||
-            (isr == 17) ||
-            (isr == 21) ||
-            (isr == 29) ||
-            (isr == 30);
-    }
-
-    private static bool isr_is_trap(size_t isr)
-    {
-        return (isr < 32u) && (isr != 2u);
-    }
-
-    private static void set_isrs()
-    {
-        ulong isr_address;
-        static foreach(isr; 0 .. 256)
-        {
-            mixin("isr_address = __asm!ulong(`
-                movabs $$1f, %rax
-                jmp 2f
-                1:
-                pushq %rdi
-                pushq %rsi
-                pushq %rax
-                mov $$", isr, ", %rdi
-                " ~ (isr_has_error_code(isr) ? "mov 0x18(%rsp), %rsi" : "") ~ "
-                movabs $$isr, %rax
-                callq *%rax
-                popq %rax
-                popq %rsi
-                popq %rdi
-                " ~ (isr_has_error_code(isr) ? "add $$8, %rsp" : "") ~ "
-                iretq
-                2:`, `={rax}`);");
-            mixin("set_descriptor(", isr, ", 0u, " ~
-                    (isr_is_trap(isr) ? "true" : "false") ~ ", isr_address);");
-        }
-    }
-
-    private static void set_descriptor(size_t index, ulong dpl, bool trap, ulong offset)
-    {
-        idt[index][0] =
-            0x0000_8E_00_0000_0000u |
-            ((offset & 0xFFFF_0000u) << 32u) |
-            (dpl << 45u) |
-            ((trap ? 1uL : 0uL) << 40u) |
-            (Gdt.SELECTOR_KERNEL_CODE << 16u) |
-            (offset & 0xFFFFu);
-        idt[index][1] = offset >> 32u;
-    }
-}
-
-public extern(C) void isr(ulong vector, ulong arg)
-{
-    if ((vector >= Idt.INT_APIC_BASE) && (vector < (Idt.INT_APIC_BASE + Idt.INT_APIC_COUNT)))
-    {
-        Apic.isr(vector);
-    }
-    else
-    {
-        switch (vector)
-        {
-        case Idt.INT_APIC_TIMER:
-        case Idt.INT_APIC_LINT0:
-        case Idt.INT_APIC_LINT1:
-        case Idt.INT_APIC_SPURIOUS:
-            Apic.isr(vector);
-            break;
-
-        default:
-            Console.clear();
-            Fb.clear(0xFF8000u);
-            Klog.writefln("ISR %u, 0x%x", vector, arg);
-            __asm("cli", "");
-            for (;;)
-            {
-                __asm("hlt", "");
-            }
-        }
-    }
-}

src/hulk/list.d

@@ -3,62 +3,98 @@
  */
 module hulk.list;
 
-import hulk.hurl.a1;
-
 /**
- * Linked list structure.
+ * Linked list mixin.
  */
-struct List(T)
+mixin template List(T)
 {
     /**
-     * Linked list item.
+     * Next item in linked list.
      */
-    T item;
+    T * list_next;
 
     /**
-     * Next list entry.
+     * Previous item in linked list.
      */
-    List!T * next;
+    T * list_prev;
 
     /**
-     * Add an item to this linked list.
+     * Initialize linked list fields.
      */
-    public void add(T item)
+    public void list_init()
     {
-        List!T * last_entry = last_entry();
-        List!T * new_entry = A1.allocate!(List!T)();
-        last_entry.item = item;
-        last_entry.next = new_entry;
+        list_next = &this;
+        list_prev = &this;
     }
 
     /**
-     * Allow foreach iteration across a List.
+     * Add this item after another in a list.
      */
-    public int opApply(scope int delegate(ref T) dg)
+    public void list_insert_after(T * item)
     {
-        List!T * entry = &this;
-        while (entry.next != null)
+        list_next = item.list_next;
+        list_prev = item;
+        item.list_next = &this;
+        list_next.list_prev = &this;
+    }
+
+    /**
+     * Add this item before another in a list.
+     */
+    public void list_insert_before(T * item)
     {
-            int result = dg(entry.item);
+        list_next = item;
+        list_prev = item.list_prev;
+        item.list_prev = &this;
+        list_prev.list_next = &this;
+    }
+
+    /**
+     * Remove this item from the list.
+     */
+    public void list_remove()
+    {
+        list_next.list_prev = list_prev;
+        list_prev.list_next = list_next;
+        list_init();
+    }
+
+    /**
+     * Get the number of items in this list.
+     */
+    public @property size_t list_count() const
+    {
+        size_t count = 1;
+        const(T) * item = &this;
+        while (item.list_next != &this)
+        {
+            count++;
+            item = item.list_next;
+        }
+        return count;
+    }
+
+    /**
+     * Allow foreach iteration through this linked list.
+     */
+    public int opApply(scope int delegate(T *) dg)
+    {
+        T * item = &this;
+        T * last = item.list_prev;
+        for (;;)
+        {
+            T * next = item.list_next;
+            int result = dg(item);
             if (result != 0)
             {
                 return result;
             }
-            entry = entry.next;
+            if (item == last)
+            {
+                break;
+            }
+            item = next;
         }
         return 0;
     }
-
-    /**
-     * Get the last linked list entry.
-     */
-    private @property List!T * last_entry()
-    {
-        List!T * result = &this;
-        while (result.next != null)
-        {
-            result = result.next;
-        }
-        return result;
-    }
 }

src/hulk/pagetable.d

@@ -11,7 +11,8 @@ enum ulong PT_WRITE_THROUGH = 0x8u;
 enum ulong PT_DISABLE_CACHE = 0x10u;
 enum ulong PT_ACCESSED = 0x20u;
 enum ulong PT_DIRTY = 0x40u;
-enum ulong PT_HUGE_PAGE = 0x80u;
+enum ulong PT_PAT = 0x80u; /* Page table entry bit. */
+enum ulong PT_HUGE_PAGE = 0x80u; /* Page directory entry bit. */
 enum ulong PT_GLOBAL = 0x100u;
 enum ulong PT_NO_EXECUTE = 0x8000_0000_0000_0000u;
 /** @} */

src/hulk/pci.d

@@ -148,8 +148,11 @@ struct Pci
         bool multifunction;
         Range[6] memory_ranges;
 
+        mixin List!Device;
+
         void initialize(Address address, Configuration * config)
         {
+            list_init();
             this.config = config;
             this.address = address;
             type = Type(config.class_id, config.subclass_id, config.interface_id);
@@ -220,7 +223,7 @@ struct Pci
                 ulong mm_region_address = (cast(ulong)r[0] & 0xFFFF_FFF0u) | (cast(ulong)r[1] << 32u);
                 ulong length = ~((cast(ulong)s[0] & 0xFFFF_FFF0u) | (cast(ulong)s[1] << 32u)) + 1u;
                 ulong flags = (r[0] & 0x8) != 0u ? PT_WRITE_THROUGH : 0u;
-                Hurl.identity_map_range(mm_region_address, length, flags);
+                Hurl.identity_map_range(mm_region_address, length, PT_WRITABLE | flags);
                 memory_ranges[range_index].address = mm_region_address;
                 memory_ranges[range_index].length = length;
                 range_index++;
@@ -233,9 +236,9 @@ struct Pci
     }
 
     /**
-     * Store a list of discovered PCI devices.
+     * Device list.
      */
-    public static __gshared List!(Device *) devices;
+    public static __gshared Device * devices;
 
     private static uint read_config_register(Address address, uint register_id)
     {
@@ -402,7 +405,14 @@ struct Pci
         if (config.vendor_id != 0xFFFFu)
         {
             Device * device = A1.allocate!Device();
-            devices.add(device);
+            if (devices == null)
+            {
+                devices = device;
+            }
+            else
+            {
+                device.list_insert_before(devices);
+            }
             device.initialize(Address(bus_nr, device_nr, function_nr), config);
             if (device.multifunction)
             {

src/hulk/rtc.d

@@ -56,6 +56,10 @@ struct Rtc
 
         /* Send EOI to enable more RTC interrupts and re-enable NMIs. */
         eoi();
+
+        time t = Rtc.read_rtc_time();
+        Klog.writefln("System time is 20%02u-%02u-%02u %02u:%02u:%02u",
+                t.year, t.month, t.day, t.hour, t.minute, t.second);
     }
 
     private static ubyte read_register(ubyte register, bool disable_nmi = false)

src/hulk/test.d

@@ -0,0 +1,139 @@
+/**
+ * Kernel tests.
+ */
+module hulk.test;
+
+import hulk.klog;
+import hulk.list;
+import hulk.time;
+import hulk.hurl.a1;
+import core.volatile;
+
+struct Test
+{
+    /**
+     * Run tests.
+     */
+    public static void run()
+    {
+        Klog.writefln("\a3Running kernel tests");
+        test_pit();
+        test_list();
+        Klog.writefln("\a3Kernel tests complete");
+    }
+
+    private static void test_pit()
+    {
+        Klog.writefln("Testing PIT...");
+        /* Check that PIT millisecond interrupt is firing. */
+        size_t uptime = Time.uptime();
+        while (Time.uptime() <= uptime)
+        {
+        }
+    }
+
+    private static void test_list()
+    {
+        Klog.writefln("Testing list...");
+
+        struct S
+        {
+            size_t v;
+            mixin List!S;
+            void initialize()
+            {
+                list_init();
+            }
+        }
+
+        S * s1 = A1.allocate!S();
+        s1.initialize();
+        s1.v = 33;
+        assert_eq(1, s1.list_count);
+        S * s2 = A1.allocate!S();
+        s2.initialize();
+        s2.v = 42;
+        s2.list_insert_after(s1);
+        S * s3 = A1.allocate!S();
+        s3.initialize();
+        s3.v = 55;
+        s3.list_insert_before(s1);
+        size_t count;
+        assert_eq(3, s1.list_count);
+        assert_eq(3, s2.list_count);
+        assert_eq(3, s3.list_count);
+        foreach (entry; *s1)
+        {
+            switch (count)
+            {
+            case 0:
+                assert_eq(33, entry.v);
+                break;
+            case 1:
+                assert_eq(42, entry.v);
+                break;
+            case 2:
+                assert_eq(55, entry.v);
+                break;
+            case 3:
+                assert_eq(0, 1);
+                break;
+            default:
+                break;
+            }
+            count++;
+        }
+        foreach (entry; *s3)
+        {
+            if (entry.v == 33)
+            {
+                entry.list_remove();
+            }
+        }
+        assert_eq(2, s3.list_count);
+        count = 0;
+        foreach (entry; *s3)
+        {
+            switch (count)
+            {
+            case 0:
+                assert_eq(55, entry.v);
+                break;
+            case 1:
+                assert_eq(42, entry.v);
+                break;
+            case 2:
+                assert_eq(0, 1);
+                break;
+            default:
+                break;
+            }
+            count++;
+        }
+        foreach (entry; *s2)
+        {
+            if (entry.v == 42)
+            {
+                entry.list_remove();
+            }
+        }
+        assert_eq(1, s2.list_count);
+        assert_eq(1, s3.list_count);
+    }
+
+    private static void assert_eq(T)(T first, T second, size_t line_number = __LINE__)
+    {
+        if (first != second)
+        {
+            Klog.fatal_error("Assertion failed! %x != %x (line %u)", first, second, line_number);
+        }
+    }
+
+    private static void assert_neq(T)(T first, T second, size_t line_number = __LINE__)
+    {
+        if (first == second)
+        {
+            Klog.fatal_error("Assertion failed! %x == %x (line %u)", first, second, line_number);
+        }
+    }
+}

src/hulk/thread.d

@@ -0,0 +1,143 @@
+/**
+ * Kernel thread functionality.
+ */
+module hulk.thread;
+
+import hulk.hurl.a1;
+import hulk.him;
+import hulk.gdt;
+import hulk.memory;
+import hulk.list;
+import hulk.cpu;
+import hulk.hurl;
+import ldc.llvmasm;
+
+struct Thread
+{
+    /* TODO: currently kernel thread stack size is fixed. */
+    enum STACK_SIZE = 16 * 1024;
+
+    enum ulong INT_TCTRL_YIELD = 0;
+
+    /** Interrupt stack template. */
+    static immutable __gshared ulong[ISF_COUNT] interrupt_stack_template = [
+        0x16161616_16161616u, /* RBP */
+        0x15151515_15151515u, /* R15 */
+        0x14141414_14141414u, /* R14 */
+        0x13131313_13131313u, /* R13 */
+        0x12121212_12121212u, /* R12 */
+        0x11111111_11111111u, /* R11 */
+        0x10101010_10101010u, /* R10 */
+        0x09090909_09090909u, /* R9 */
+        0x08080808_08080808u, /* R8 */
+        0x06060606_06060606u, /* RDI */
+        0x05050505_05050505u, /* RSI */
+        0x04040404_04040404u, /* RDX */
+        0x03030303_03030303u, /* RCX */
+        0x02020202_02020202u, /* RBX */
+        0x01010101_01010101u, /* RAX */
+        0u, /* Error Code */
+        0u, /* RIP */
+        Gdt.SELECTOR_KERNEL_CODE, /* CS */
+        0u, /* RFLAGS */
+        0u, /* RSP */
+        Gdt.SELECTOR_KERNEL_DATA, /* SS */
+    ];
+
+    /** List of all runnable kernel threads. */
+    private static __gshared Thread * runnable_threads;
+
+    /** Currently executing kernel thread. */
+    static __gshared Thread * current_thread;
+
+    /** Current kernel thread stack pointer. */
+    ulong * stack_pointer;
+    static assert(stack_pointer.offsetof == 0);
+
+    /** Base address of kernel thread stack. */
+    void * stack_addr;
+
+    mixin List!Thread;
+
+    /**
+     * Initialize a kernel thread.
+     */
+    public void initialize(void function() fn)
+    {
+        list_init();
+        suspend_interrupts();
+        list_insert_after(current_thread);
+        resume_interrupts();
+        stack_addr = A1.allocate(STACK_SIZE);
+        ulong * stack_top = cast(ulong *)(stack_addr + STACK_SIZE);
+        stack_pointer = stack_top - ISF_COUNT;
+        memcpy64(stack_pointer, &interrupt_stack_template[0], ISF_COUNT);
+        stack_pointer[ISF_RIP] = cast(ulong)fn;
+        stack_pointer[ISF_RFLAGS] = read_rflags();
+        stack_pointer[ISF_RSP] = cast(ulong)stack_top;
+    }
+
+    /**
+     * Initialize first kernel thread.
+     */
+    public void initialize_first_thread()
+    {
+        list_init();
+        runnable_threads = &this;
+        stack_addr = cast(void *)(Hurl.HULK_STACK_TOP - STACK_SIZE);
+    }
+
+    /**
+     * Initialize kernel threading.
+     */
+    public static void initialize()
+    {
+        Thread * thread = A1.allocate!Thread();
+        thread.initialize_first_thread();
+        current_thread = thread;
+    }
+
+    /**
+     * Create and start a kernel thread.
+     *
+     * TODO: fn must not return.
+     * TODO: threads are always runnable.
+     * TODO: threads cannot be destroyed.
+     */
+    public static Thread * start(void function() fn)
+    {
+        Thread * thread = A1.allocate!Thread();
+        thread.initialize(fn);
+        swint(INT_TCTRL_YIELD);
+        return thread;
+    }
+
+    /**
+     * Trigger a thread management software interrupt.
+     */
+    public static void swint(T1)(T1 v1)
+    {
+        mixin(`__asm("int $$`, cast(int)INT_KERNEL_SWINT, `", "{rax}", v1);`);
+    }
+
+    /**
+     * Handle a thread management software interrupt.
+     *
+     * This function is called in interrupt context.
+     *
+     * @param stack_frame
+     *   Pointer to the stack frame.
+     */
+    public static void handle_swint(ulong * stack_frame)
+    {
+        switch (stack_frame[ISF_RAX])
+        {
+        case INT_TCTRL_YIELD:
+            Thread.current_thread = Thread.current_thread.list_next;
+            break;
+
+        default:
+            break;
+        }
+    }
+}

src/hulk/time.d

@@ -3,12 +3,12 @@
  */
 module hulk.time;
 
-import core.volatile;
+import hulk.volatile;
 
 struct Time
 {
     /** System uptime (ms). */
-    private static __gshared ulong s_uptime;
+    private static __gshared Volatile!ulong s_uptime;
 
     /**
      * Millisecond ISR.
@@ -34,8 +34,8 @@ struct Time
      */
     public static void msleep(ulong count)
     {
-        ulong wait_for = s_uptime + count + 1;
-        while (volatileLoad(&s_uptime) < wait_for)
+        ulong wait_for = uptime() + count + 1;
+        while (uptime() < wait_for)
         {
         }
     }

src/hulk/usb/usb.d

@@ -13,7 +13,7 @@ struct Usb
     {
         Klog.writefln("\a3Initializing USB");
 
-        foreach (pci_device; Pci.devices)
+        foreach (pci_device; *Pci.devices)
         {
             if (pci_device.type == Pci.XHCI_CONTROLLER)
             {

src/hulk/usb/xhci.d

@@ -9,8 +9,12 @@
 module hulk.usb.xhci;
 
 import hulk.pci;
+import hulk.hurl;
 import hulk.hurl.a1;
+import hulk.hippo;
 import hulk.klog;
+import hulk.time;
+import hulk.volatile;
 
 struct XHCI
 {
@@ -25,22 +29,28 @@ struct XHCI
          * Offset from beginning of capability registers to beginning of
          * operational registers.
          */
-        ubyte capability_length;
+        Volatile!ubyte capability_length;
 
         ubyte _reserved01;
 
         /**
          * Interface version number supported (psuedo-BSD).
          */
-        ushort hci_version;
+        Volatile!ushort hci_version;
 
         /**
+         * HCSPARAMS1.
          * 7:0 number of device slots (MaxSlots)
          * 18:8 number of interrupters (MaxIntrs)
          * 23:19 reserved
          * 31:24 number of ports (MaxPorts)
          */
-        uint hcs_params1;
+        Volatile!uint hcs_params1;
+
+        @property ubyte max_slots()
+        {
+            return this.hcs_params1 & 0xFFu;
+        }
 
         /**
          * 3:0 isochronous scheduling threshold (IST)
@@ -50,14 +60,20 @@ struct XHCI
          * 26 scratchpad restore (SPR)
          * 31:27 max scratchpad buffers (low 5 bits)
          */
-        uint hcs_params2;
+        Volatile!uint hcs_params2;
+
+        @property size_t max_scratchpad_buffers()
+        {
+            uint v = this.hcs_params2;
+            return (v >> 27u) | ((v >> (21 - 5)) & 0x3E0u);
+        }
 
         /**
          * 7:0 U1 device exit latency
          * 15:8 reserved
          * 31:16 U2 device exit latency
          */
-        uint hcs_params3;
+        Volatile!uint hcs_params3;
 
         /**
          * 0 64-bit addressing capability (AC64)
@@ -77,7 +93,7 @@ struct XHCI
          *   offset in 32-bit words from base of capability registers to base
          *   of capabilities list
          */
-        uint hcc_params1;
+        Volatile!uint hcc_params1;
 
         /**
          * 1:0 reserved
@@ -85,13 +101,13 @@ struct XHCI
          *   offset in 32-bit words from base of capability registers to the
          *   doorbell array
          */
-        uint doorbell_offset;
+        Volatile!uint doorbell_offset;
 
         /**
          * 4:0 reserved
          * 31:5 runtime register space offset
          */
-        uint rts_offset;
+        Volatile!uint rts_offset;
 
         /**
          * 0 U3 entry capability (U3C)
@@ -106,13 +122,13 @@ struct XHCI
          * 9 virtualization based trusted I/O capability (VTC; xHCI v1.2+)
          * 31:10 reserved
          */
-        uint hcc_params2;
+        Volatile!uint hcc_params2;
 
         /**
          * 11:0 reserved
          * 31:12 VTIO register space offset (xHCI v1.2+)
          */
-        uint vtios_offset;
+        Volatile!uint vtios_offset;
     }
     static assert(CapabilityRegisters.sizeof == 0x24u);
 
@@ -139,7 +155,7 @@ struct XHCI
          * 16 VTIO enable (VTIOE; xHCI v1.2+)
          * 31:17 reserved
          */
-        uint usb_command;
+        Volatile!uint usb_command;
 
         /**
          * 0 HCHalted
@@ -155,13 +171,13 @@ struct XHCI
          * 12 Host Controller Error (HCE)
          * 31:13 reserved
          */
-        uint usb_status;
+        Volatile!uint usb_status;
 
         /**
          * 15:0 Page Size (actual page size is 2^(x+12))
          * 31:16 reserved
          */
-        uint page_size;
+        Volatile!uint page_size;
 
         uint[2] _reserved01;
 
@@ -169,7 +185,7 @@ struct XHCI
          * 15:0 Notification Enable
          * 31:16 reserved
          */
-        uint dn_ctrl;
+        Volatile!uint dn_ctrl;
 
         /**
          * 0 Ring Cycle State (RCS)
@@ -179,7 +195,7 @@ struct XHCI
          * 5:4 reserved
          * 63:6 Command Ring Pointer
          */
-        ulong cr_ctrl;
+        Volatile!ulong cr_ctrl;
 
         uint[4] _reserved02;
 
@@ -187,7 +203,7 @@ struct XHCI
          * 5:0 reserved
          * 63:6 Device Context Base Address Array Pointer
          */
-        ulong dcbaap;
+        Volatile!ulong dcbaap;
 
         /**
          * 7:0 Max Device Slots Enabled (MaxSlotsEn)
@@ -195,7 +211,7 @@ struct XHCI
          * 9 Configuration Information Enable (CIE; xHCI v1.1+)
          * 31:10 reserved
          */
-        uint config;
+        Volatile!uint config;
 
         uint _reserved03;
     }
@@ -236,22 +252,22 @@ struct XHCI
          * 30 Device Removable
          * 31 Warm Port Reset
          */
-        uint portsc;
+        Volatile!uint portsc;
 
         /**
          * Port Power Management Status and Control
          */
-        uint portpmsc;
+        Volatile!uint portpmsc;
 
         /**
          * Port Link Info
          */
-        uint portli;
+        Volatile!uint portli;
 
         /**
          * Port Hardware LPM Control (xHCI v1.1+)
          */
-        uint porthlpmc;
+        Volatile!uint porthlpmc;
     }
     static assert(PortRegisters.sizeof == 0x10u);
 
@@ -263,7 +279,7 @@ struct XHCI
         /**
          * Microframe index. Incremented by controller each microframe.
          */
-        uint mfindex;
+        Volatile!uint mfindex;
 
         uint[7] _reserved01;
 
@@ -286,21 +302,21 @@ struct XHCI
          * 1 Interrupt Enable
          * 31:2 reserved
          */
-        uint imr;
+        Volatile!uint imr;
 
         /**
          * Interrupter Moderation.
          * 15:0 Interrupt Moderation Interval
          * 31:16 Interrupt Moderation Counter
          */
-        uint im;
+        Volatile!uint im;
 
         /**
          * Event Ring Segment Table Size.
          * 15:0 Event Ring Segment Table Size
          * 31:16 reserved
          */
-        uint ersts;
+        Volatile!uint ersts;
 
         uint reserved;
 
@@ -309,7 +325,7 @@ struct XHCI
          * 5:0 reserved
          * 63:6 Event Ring Segment Table Base Address
          */
-        ulong erstba;
+        Volatile!ulong erstba;
 
         /**
          * Event Ring Dequeue Pointer.
@@ -317,14 +333,20 @@ struct XHCI
          * 3 Event Handler Busy
          * 63:4 Event Ring Dequeue Pointer
          */
-        ulong erdp;
+        Volatile!ulong erdp;
     }
 
+    /**
+     * Located at offset doorbell_offset from PCI base address.
+     *
+     * The number of these is given by HCSPARAMS1.MaxSlots + 1 (for the command
+     * ring).
+     */
     struct DoorbellRegister
     {
-        ubyte target;
+        Volatile!ubyte target;
         ubyte _reserved01;
-        ushort task_id;
+        Volatile!ushort task_id;
     }
     static assert(DoorbellRegister.sizeof == 4u);
 
@@ -344,6 +366,72 @@ struct XHCI
         m_runtime_registers = cast(RuntimeRegisters *)(base_address + m_capability_registers.rts_offset);
         m_doorbell_registers = cast(DoorbellRegister *)(base_address + m_capability_registers.doorbell_offset);
         dump_extended_capabilities();
+        if (!reset())
+        {
+            Klog.writefln("XHCI controller failed to initialize");
+            return;
+        }
+
+        /* Allocate memory for various controller data structures. */
+        ubyte * page = cast(ubyte *)Hippo.allocate_page();
+        /* Device Context Index: 2K */
+        void ** device_context_index = cast(void **)page;
+        /* Input Control Context: 64 bytes */
+        void * input_control_context = &page[2048];
+        /* Slot Context: 64 bytes */
+        void * slot_context = &page[2048 + 64];
+        /* Endpoint Context: 64 bytes */
+        void * endpoint_context = &page[2048 + 64 + 64];
+        /* Stream Context: 16 bytes */
+        void * stream_context = &page[2048 + 64 + 64 + 64];
+        /* Stream Array (Linear): 1M */
+        void * stream_array_linear = Hippo.allocate_aligned_region(1024 * 1024, PAGE_SIZE);
+        /* Stream Array (Pri/Sec): 4K */
+        void * stream_array_pri_sec = Hippo.allocate_page();
+        /* Transfer Ring Segments: 64K */
+        void * transfer_ring_segments = Hippo.allocate_aligned_region(64 * 1024, 64 * 1024);
+        /* Command Ring Segments: 64K */
+        void * command_ring_segments = Hippo.allocate_aligned_region(64 * 1024, 64 * 1024);
+        /* Event Ring Segments: 64K */
+        void * event_ring_segments = Hippo.allocate_aligned_region(64 * 1024, 64 * 1024);
+        /* Event Ring Segment Table: 512K */
+        void * event_ring_segment_table = Hippo.allocate_aligned_region(512 * 1024, PAGE_SIZE);
+        /* Scratchpad Buffer Array: 248 bytes */
+        void ** scratchpad_buffer_array = cast(void **)Hippo.allocate_aligned_region(2 * PAGE_SIZE, PAGE_SIZE);
+
+        size_t n_scratchpad_buffers = m_capability_registers.max_scratchpad_buffers;
+        for (size_t i = 0u; i < n_scratchpad_buffers; i++)
+        {
+            void * scratchpad_buffer = Hippo.allocate_page();
+            scratchpad_buffer_array[i] = scratchpad_buffer;
+        }
+
+        device_context_index[0] = scratchpad_buffer_array;
+
+        size_t max_slots = m_capability_registers.max_slots;
+        for (size_t i = 0u; i < max_slots; i++)
+        {
+            void * addr;
+            if ((i & 1) == 0)
+            {
+                page = cast(ubyte *)Hippo.allocate_page();
+                addr = page;
+            }
+            else
+            {
+                addr = &page[2048];
+            }
+            device_context_index[1 + i] = addr;
+        }
+
+        m_operational_registers.dcbaap = cast(ulong)device_context_index;
+        /* TODO: write cr_ctrl. */
+        /+ TODO:
+        m_operational_registers.config = m_capability_registers.hcs_params1 & 0xFu;
+        m_operational_registers.dn_ctrl = 1u << 1u;
+        +/
+        /* TODO: set up interrupt register set. */
+        Klog.writefln("XHCI controller initialization successful");
     }
 
     private void dump_extended_capabilities()
@@ -386,6 +474,56 @@ struct XHCI
         }
     }
 
+    private bool reset()
+    {
+        /* Clear run/stop bit. */
+        m_operational_registers.usb_command &= ~0x1u;
+
+        /* Wait for HCHalted bit to be set. */
+        enum halt_timeout = 40u;
+        for (size_t halt_timeout_counter = 0u;;)
+        {
+            Time.msleep(1u);
+
+            if ((m_operational_registers.usb_status & 1u) != 0u)
+            {
+                break;
+            }
+
+            halt_timeout_counter++;
+            if (halt_timeout_counter >= halt_timeout)
+            {
+                Klog.writefln("XHCI controller failed to halt");
+                return false;
+            }
+        }
+
+        /* Set HC reset bit. */
+        m_operational_registers.usb_command |= (1u << 1u);
+
+        /* Wait for controller to be ready. */
+        enum reset_timeout = 100u;
+        for (size_t reset_timeout_counter = 0u;;)
+        {
+            Time.msleep(1u);
+
+            if (((m_operational_registers.usb_command & (1u << 1u)) == 0u) &&
+                ((m_operational_registers.usb_status & (1u << 11u)) == 0u))
+            {
+                break;
+            }
+
+            reset_timeout_counter++;
+            if (reset_timeout_counter >= reset_timeout)
+            {
+                Klog.writefln("XHCI controller failed to reset");
+                return false;
+            }
+        }
+
+        return true;
+    }
+
     static void build(Pci.Device * pci_device)
     {
         if (pci_device.memory_ranges[0].address != null)

src/hulk/volatile.d

@@ -0,0 +1,35 @@
+module hulk.volatile;
+
+import core.volatile;
+
+struct Volatile(T)
+{
+    private T value;
+
+    public @property T read()
+    {
+        return volatileLoad(&value);
+    }
+    alias read this;
+
+    public void opAssign(T value)
+    {
+        volatileStore(&this.value, value);
+    }
+
+    public T opUnary(string s)()
+    {
+        T v = read();
+        mixin(s ~ "v;");
+        volatileStore(&this.value, v);
+        return v;
+    }
+
+    public T opOpAssign(string s)(T rhs)
+    {
+        T v = read();
+        mixin("v " ~ s ~ "= rhs;");
+        volatileStore(&this.value, v);
+        return v;
+    }
+}

x86_64-elf-gcc.rb

@@ -1,8 +1,8 @@
 project_name "x86_64-elf-gcc"
-binutils_version = "2.35"
-binutils_checksum = "1b11659fb49e20e18db460d44485f09442c8c56d5df165de9461eb09c8302f85"
-gcc_version = "10.2.0"
-gcc_checksum = "b8dd4368bb9c7f0b98188317ee0254dd8cc99d1e3a18d0ff146c855fe16c1d8c"
+binutils_version = "2.41"
+binutils_checksum = "ae9a5789e23459e59606e6714723f2d3ffc31c03174191ef0d015bdf06007450"
+gcc_version = "13.2.0"
+gcc_checksum = "e275e76442a6067341a27f04c5c6b83d8613144004c0413528863dc6b5c743da"
 install_path = File.expand_path("x86_64-elf-gcc")
 target = "x86_64-elf"
 path_prepend "#{install_path}/bin"