vm: controller: add an auto-detach flag for vm-objects

interrupts will need to be enable to allow for requesting missing pages from userspace
services.

CMakeLists.txt

@@ -1,4 +1,4 @@
-cmake_minimum_required(VERSION 3.13)
+cmake_minimum_required(VERSION 4.0)
 project(mango C ASM)
 
 if (NOT BUILD_TOOLS_DIR)
@@ -28,8 +28,6 @@ file(GLOB_RECURSE arch_sources_c arch/${kernel_arch}/*.c)
 file(GLOB_RECURSE arch_sources_asm arch/${kernel_arch}/*.S)
 file(GLOB_RECURSE arch_headers arch/${kernel_arch}/*.h)
 
-set_property(SOURCE ${arch_sources_asm} PROPERTY LANGUAGE C)
-
 add_executable(${kernel_exe_name}
 	${kernel_sources}
 	${kernel_headers}

arch/user/hwlock.c

@@ -1,5 +1,5 @@
-#include <kernel/machine/hwlock.h>
 #include <kernel/compiler.h>
+#include <kernel/machine/hwlock.h>
 
 void ml_hwlock_lock(ml_hwlock_t *lck)
 {

arch/x86_64/asm-offset.c

@@ -1,10 +1,11 @@
-#include <kernel/sched.h>
 #include <kernel/compiler.h>
+#include <kernel/sched.h>
+#include <kernel/thread.h>
 
-//size_t THREAD_sp = offsetof(struct thread, tr_sp);
+// size_t THREAD_sp = offsetof(struct thread, tr_sp);
 
-/* Use %a0 instead of %0 to prevent gcc from emitting a $ before the symbol value
+/* Use %a0 instead of %0 to prevent gcc from emitting a $ before the symbol
-   in the generated assembly.
+   value in the generated assembly.
 
    emitting
      .set TASK_sp, $56
@@ -15,13 +16,18 @@
    causes garbage symbols to be generated.
 */
 
-#define DEFINE(sym, val) \
+#define DEFINE(sym, val)                                                       \
-	asm volatile("\n.global " #sym "\n.type " #sym ", @object" "\n.set " #sym ", %a0" : : "i" (val))
+	asm volatile("\n.global " #sym "\n.type " #sym                         \
+	             ", @object"                                               \
+	             "\n.set " #sym ", %a0"                                    \
+	             :                                                         \
+	             : "i"(val))
 
-#define OFFSET(sym, str, mem) \
+#define OFFSET(sym, str, mem) DEFINE(sym, offsetof(str, mem))
-	DEFINE(sym, offsetof(str, mem))
 
 static void __used common(void)
 {
 	OFFSET(THREAD_sp, struct thread, tr_sp);
+	OFFSET(THREAD_fsbase, struct thread, tr_ml.tr_fsbase);
+	OFFSET(THREAD_gsbase, struct thread, tr_ml.tr_gsbase);
 }

arch/x86_64/config.cmake

@@ -1,10 +1,5 @@
 target_compile_options(${kernel_exe_name} PRIVATE
 	-z max-page-size=0x1000 -m64 -mcmodel=large -mno-red-zone -mno-mmx
 	-mno-sse -mno-sse2 -D_64BIT -DBYTE_ORDER=1234)
-target_link_libraries(${kernel_exe_name} "-z max-page-size=0x1000" "-T ${CMAKE_CURRENT_SOURCE_DIR}/arch/x86_64/layout.ld")
+target_link_libraries(${kernel_exe_name} "-static -z max-page-size=0x1000" "-T ${CMAKE_CURRENT_SOURCE_DIR}/arch/x86_64/layout.ld")
 
-add_custom_command(TARGET ${kernel_exe_name} POST_BUILD
-	COMMAND ${BUILD_TOOLS_DIR}/e64patch $<TARGET_FILE:${kernel_exe_name}>
-	WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
-	COMMENT "Patching kernel elf64 image"
-)

arch/x86_64/hwlock.S

@@ -11,6 +11,41 @@ ml_hwlock_lock:
 
 	mov $1, %ecx
 
+	mfence
+
+1:	mov $0, %eax
+	lock cmpxchg %ecx, (%rdi)
+	jne 1b
+
+	pop %rbp
+	ret
+
+
+	.global ml_hwlock_unlock
+	.type ml_hwlock_unlock, @function
+
+	/* %rdi = pointer to ml_hwlock_t (int) */
+ml_hwlock_unlock:
+	push %rbp
+	mov %rsp, %rbp
+
+	movl $0, (%rdi)
+	mfence
+
+	pop %rbp
+	ret
+
+
+	.global ml_hwlock_lock_irq
+	.type ml_hwlock_lock_irq, @function
+
+	/* %rdi = pointer to ml_hwlock_t (int) */
+ml_hwlock_lock_irq:
+	push %rbp
+	mov %rsp, %rbp
+
+	mov $1, %ecx
+
 	cli
 	mfence
 
@@ -21,11 +56,12 @@ ml_hwlock_lock:
 	pop %rbp
 	ret
 
-	.global ml_hwlock_unlock
+
-	.type ml_hwlock_unlock, @function
+	.global ml_hwlock_unlock_irq
+	.type ml_hwlock_unlock_irq, @function
 
 	/* %rdi = pointer to ml_hwlock_t (int) */
-ml_hwlock_unlock:
+ml_hwlock_unlock_irq:
 	push %rbp
 	mov %rsp, %rbp
 
@@ -62,6 +98,7 @@ ml_hwlock_lock_irqsave:
 	pop %rbp
 	ret
 
+
 	.global ml_hwlock_unlock_irqrestore
 	.type ml_hwlock_unlock_irqrestore, @function
 

arch/x86_64/include/arch/msr.h

@@ -3,7 +3,9 @@
 
 #include <stdint.h>
 
-#define MSR_GS_BASE 0xC0000101
+#define MSR_FS_BASE        0xC0000100
+#define MSR_GS_BASE        0xC0000101
+#define MSR_KERNEL_GS_BASE 0xC0000102
 
 #ifdef __cplusplus
 extern "C" {

arch/x86_64/include/kernel/machine/hwlock.h

@@ -12,6 +12,9 @@ typedef int ml_hwlock_t;
 extern void ml_hwlock_lock(ml_hwlock_t *lck);
 extern void ml_hwlock_unlock(ml_hwlock_t *lck);
 
+extern void ml_hwlock_lock_irq(ml_hwlock_t *lck);
+extern void ml_hwlock_unlock_irq(ml_hwlock_t *lck);
+
 extern void ml_hwlock_lock_irqsave(ml_hwlock_t *lck, unsigned long *flags);
 extern void ml_hwlock_unlock_irqrestore(ml_hwlock_t *lck, unsigned long flags);
 

arch/x86_64/include/kernel/machine/random.h

@@ -0,0 +1,10 @@
+#ifndef KERNEL_X86_64_RANDOM_H_
+#define KERNEL_X86_64_RANDOM_H_
+
+#include <stdbool.h>
+#include <stdint.h>
+
+extern bool ml_hwrng_available(void);
+extern uint64_t ml_hwrng_generate(void);
+
+#endif

arch/x86_64/include/kernel/machine/thread.h

@@ -3,6 +3,10 @@
 
 #include <kernel/sched.h>
 
+struct ml_thread {
+	virt_addr_t tr_gsbase, tr_fsbase;
+};
+
 struct ml_cpu_context;
 
 /* switch from one thread to another. the stack of the `to` thread must have
@@ -28,4 +32,15 @@ extern kern_status_t ml_thread_prepare_user_context(
 	const uintptr_t *args,
 	size_t nr_args);
 
+extern kern_status_t ml_thread_config_get(
+	struct thread *thread,
+	kern_config_key_t key,
+	void *out,
+	size_t max);
+extern kern_status_t ml_thread_config_set(
+	struct thread *thread,
+	kern_config_key_t key,
+	const void *ptr,
+	size_t len);
+
 #endif

arch/x86_64/init.c

@@ -10,15 +10,32 @@
 #include <kernel/init.h>
 #include <kernel/libc/stdio.h>
 #include <kernel/machine/cpu.h>
+#include <kernel/machine/random.h>
 #include <kernel/memblock.h>
 #include <kernel/object.h>
 #include <kernel/percpu.h>
 #include <kernel/pmap.h>
 #include <kernel/printk.h>
 #include <kernel/types.h>
+#include <kernel/util.h>
 #include <kernel/vm.h>
 
-#define PTR32(x) ((void *)((uintptr_t)(x)))
+#undef HARDWARE_RNG
+
+#define PTR32(x)            ((void *)((uintptr_t)(x)))
+
+/* the physical address of the start of the memblock heap.
+ * this is an arbirary value; the heap can start anywhere in memory.
+ * any reserved areas of memory (the kernel, bsp, bios data, etc) are
+ * automatically taken into account.
+ * HOWEVER, this value will dictate how much physical memory is required for
+ * the kernel to boot successfully.
+ * the value of 16MiB (0x1000000) means that all heap allocations will be
+ * above 16MiB, leaving the area below free for DMA operations.
+ * this value CAN be reduced all the way to zero to minimise the amount of
+ * memory required to boot, but this may leave you with no DMA memory available.
+ */
+#define MEMBLOCK_HEAP_START 0x1000000
 
 static ml_cpu_block g_bootstrap_cpu = {0};
 
@@ -33,7 +50,7 @@ static void bootstrap_cpu_init(void)
 
 static void early_vm_init(uintptr_t reserve_end)
 {
-	uintptr_t alloc_start = VM_KERNEL_VOFFSET;
+	uintptr_t alloc_start = VM_KERNEL_VOFFSET + MEMBLOCK_HEAP_START;
 	/* boot code mapped 2 GiB of memory from
 	   VM_KERNEL_VOFFSET */
 	uintptr_t alloc_end = VM_KERNEL_VOFFSET + 0x7fffffff;
@@ -110,6 +127,22 @@ int ml_init(uintptr_t arg)
 		reserve_end = bsp.mod_base + bsp.mod_size;
 	}
 
+#if defined(HARDWARE_RNG)
+	if (ml_hwrng_available()) {
+		printk("cpu: ardware RNG available");
+		uint64_t seed = ml_hwrng_generate();
+		printk("cpu: RNG seed=%zx", seed);
+		init_random(seed);
+	} else {
+#endif
+		printk("cpu: hardware RNG unavailable");
+		uint64_t seed = 0xeddc4c8a679dc23f;
+		printk("cpu: RNG seed=%zx", seed);
+		init_random(seed);
+#if defined(HARDWARE_RNG)
+	}
+#endif
+
 	early_vm_init(reserve_end);
 
 	e820_scan(PTR32(mb->mmap_addr), mb->mmap_length);

arch/x86_64/irq.c

@@ -97,7 +97,9 @@ static void pf_handler(struct ml_cpu_context *regs)
 
 	virt_addr_t fault_ptr = pf_faultptr();
 
+	ml_int_enable();
 	kern_status_t status = pmap_handle_fault(fault_ptr, fault_flags);
+	ml_int_disable();
 
 	if (status == KERN_OK) {
 		return;

arch/x86_64/irqvec.S

@@ -333,11 +333,30 @@ IRQ 223, 255
 isr_common_stub:
 	PUSH_REGS
 
+	# When ISR occurs in Ring 3, CPU sets %ss (and other non-code selectors)
+	# to 0.
+	mov %ss, %ax
+	cmp $0, %ax
+	jne isr_skipgs1
+
+	mov $0x10, %ax
+	mov %ax, %ss
+	swapgs
+
+isr_skipgs1:
 	mov %rsp, %rdi
 	call isr_dispatch
 
 	POP_REGS
 	add $16, %rsp
+
+	cmpq $0x1b, 32(%rsp)
+	jne isr_skipgs2
+
+	swapgs
+
+isr_skipgs2:
+
 	iretq
 
 
@@ -347,11 +366,31 @@ isr_common_stub:
 irq_common_stub:
 	PUSH_REGS
 
+	# When IRQ occurs in Ring 3, CPU sets %ss (and other non-code selectors)
+	# to 0.
+	mov %ss, %ax
+	cmp $0, %ax
+	jne irq_skipgs1
+
+	mov $0x10, %ax
+	mov %ax, %ss
+	swapgs
+
+irq_skipgs1:
 	mov %rsp, %rdi
 	call irq_dispatch
 
+
 	POP_REGS
 	add $16, %rsp
+
+	cmpq $0x1b, 32(%rsp)
+	jne isr_skipgs2
+
+	swapgs
+
+irq_skipgs2:
+
 	iretq
 
 
@@ -363,12 +402,12 @@ irq_common_stub:
 
 syscall_gate:
 	swapgs
-	movq %rsp, %gs:20       # GS+20 = rsp2 in the current TSS block (user stack storage)
+	movq %rsp, %gs:94       # GS+20 = rsp2 in the current TSS block (user stack storage)
-	movq %gs:4, %rsp	# GS+4  = rsp0 in the current TSS block (per-thread kstack)
+	movq %gs:78, %rsp	# GS+4  = rsp0 in the current TSS block (per-thread kstack)
 
 	# start building a ml_cpu_context
 	pushq $0x1b
-	pushq %gs:20
+	pushq %gs:94
 	push %r11
 	push $0x23
 	push %rcx
@@ -380,10 +419,6 @@ syscall_gate:
 
 	mov %rsp, %rdi
 
-	# switch back to user gs while in syscall_dispatch. Interrupts are enabled in syscall_dispatch,
-	# and if the task gets pre-empted, the incoming task will expect %gs to have its usermode value.
-	swapgs
-
 	call syscall_dispatch
 
 	POP_REGS
@@ -394,8 +429,8 @@ syscall_gate:
 	pop %r11
 	add $16, %rsp
 
-	swapgs
+	movq %gs:94, %rsp       # GS+20 = rsp2 in the current TSS block
-	movq %gs:20, %rsp       # GS+20 = rsp2 in the current TSS block
+
 	swapgs
 
 	# back to usermode

arch/x86_64/panic.c

@@ -1,8 +1,11 @@
 #include <arch/irq.h>
+#include <kernel/address-space.h>
 #include <kernel/libc/stdio.h>
 #include <kernel/machine/cpu.h>
 #include <kernel/machine/panic.h>
 #include <kernel/printk.h>
+#include <kernel/sched.h>
+#include <kernel/task.h>
 #include <kernel/vm.h>
 
 #define R_CF  0
@@ -166,36 +169,64 @@ static void print_stack_item(uintptr_t addr)
 	printk("%s", buf);
 }
 
-static void print_stack_trace(uintptr_t ip, uintptr_t *bp)
+static bool read_stack_frame(
+	struct address_space *space,
+	uintptr_t bp,
+	struct stack_frame *out)
 {
-	struct stack_frame *stk = (struct stack_frame *)bp;
+	if (bp >= VM_PAGEMAP_BASE) {
+		*out = *(struct stack_frame *)bp;
+		return true;
+	}
+
+	if (!space) {
+		return false;
+	}
+
+	size_t tmp;
+	kern_status_t status
+		= address_space_read(space, bp, sizeof *out, out, &tmp);
+	return status == KERN_OK;
+}
+
+static void print_stack_trace(
+	struct address_space *space,
+	uintptr_t ip,
+	uintptr_t bp)
+{
+	struct stack_frame stk;
+	if (!read_stack_frame(space, bp, &stk)) {
+		return;
+	}
 	printk("call trace:");
 
 	print_stack_item(ip);
 
 	int max_frames = 10, current_frame = 0;
-	while (1) {
+	while (current_frame < max_frames) {
-		if (!vm_virt_to_phys(stk) || bp == NULL
+		uintptr_t addr = stk.rip;
-		    || current_frame > max_frames) {
-			break;
-		}
-
-		uintptr_t addr = stk->rip;
 		print_stack_item(addr);
 
-		stk = (struct stack_frame *)stk->rbp;
+		bp = stk.rbp;
+		if (!read_stack_frame(space, bp, &stk)) {
+			break;
+		}
 		current_frame++;
 	}
 }
 
 void ml_print_stack_trace(uintptr_t ip)
 {
-	uintptr_t *bp;
+	struct task *task = current_task();
+	struct address_space *space = task ? task->t_address_space : NULL;
+	uintptr_t bp;
 	asm volatile("mov %%rbp, %0" : "=r"(bp));
-	print_stack_trace(ip, bp);
+	print_stack_trace(space, ip, bp);
 }
 
 void ml_print_stack_trace_irq(struct ml_cpu_context *ctx)
 {
-	print_stack_trace(ctx->rip, (uintptr_t *)ctx->rbp);
+	struct task *task = current_task();
+	struct address_space *space = task ? task->t_address_space : NULL;
+	print_stack_trace(space, ctx->rip, ctx->rbp);
 }

arch/x86_64/pmap.c

@@ -1,12 +1,13 @@
+#include <kernel/address-space.h>
 #include <kernel/compiler.h>
 #include <kernel/libc/stdio.h>
 #include <kernel/memblock.h>
 #include <kernel/pmap.h>
 #include <kernel/printk.h>
 #include <kernel/sched.h>
+#include <kernel/task.h>
 #include <kernel/types.h>
 #include <kernel/vm-object.h>
-#include <kernel/vm-region.h>
 #include <kernel/vm.h>
 #include <mango/status.h>
 
@@ -99,7 +100,7 @@ static void delete_ptab(phys_addr_t pt)
 		return;
 	}
 
-	pt &= ~VM_PAGE_MASK;
+	pt = ENTRY_TO_PTR(pt);
 	if (!pt) {
 		/* physical address of 0x0, nothing to delete */
 		return;
@@ -116,7 +117,7 @@ static void delete_pdir(phys_addr_t pd)
 		return;
 	}
 
-	pd &= ~0x1FFFFFULL;
+	pd &= ENTRY_TO_PTR(pd);
 	if (!pd) {
 		/* physical address of 0x0, nothing to delete */
 		return;
@@ -240,6 +241,104 @@ static kern_status_t do_pmap_add(
 	return KERN_OK;
 }
 
+static kern_status_t do_pmap_remove(
+	pmap_t pmap,
+	virt_addr_t pv,
+	enum page_size size)
+{
+	unsigned int pml4t_index = BAD_INDEX, pdpt_index = BAD_INDEX,
+		     pd_index = BAD_INDEX, pt_index = BAD_INDEX;
+
+	switch (size) {
+	case PS_4K:
+		pml4t_index = (pv >> 39) & 0x1FF;
+		pdpt_index = (pv >> 30) & 0x1FF;
+		pd_index = (pv >> 21) & 0x1FF;
+		pt_index = (pv >> 12) & 0x1FF;
+		break;
+	case PS_2M:
+		pml4t_index = (pv >> 39) & 0x1FF;
+		pdpt_index = (pv >> 30) & 0x1FF;
+		pd_index = (pv >> 21) & 0x1FF;
+		break;
+	case PS_1G:
+		if (!can_use_gbpages) {
+			return KERN_UNSUPPORTED;
+		}
+		pml4t_index = (pv >> 39) & 0x1FF;
+		pdpt_index = (pv >> 30) & 0x1FF;
+		break;
+	default:
+		return KERN_INVALID_ARGUMENT;
+	}
+
+	/* 1. get PML4T (mandatory) */
+	struct pml4t *pml4t = vm_phys_to_virt(ENTRY_TO_PTR(pmap));
+	if (!pml4t) {
+		return KERN_OK;
+	}
+
+	/* 2. traverse PML4T, get PDPT (mandatory) */
+	struct pdpt *pdpt = NULL;
+	if (!pml4t->p_entries[pml4t_index]) {
+		return KERN_OK;
+	} else {
+		pdpt = vm_phys_to_virt(
+			ENTRY_TO_PTR(pml4t->p_entries[pml4t_index]));
+	}
+
+	/* if we're mapping a 1GiB page, we stop here */
+	if (size == PS_1G) {
+		if (pdpt->p_entries[pdpt_index] != 0) {
+			/* this slot points to a pdir, delete it.
+			   if this slot points to a hugepage, this does nothing
+			 */
+			delete_pdir(pdpt->p_entries[pdpt_index]);
+		}
+
+		pdpt->p_pages[pdpt_index] = 0;
+
+		return KERN_OK;
+	}
+
+	/* 3. traverse PDPT, get PDIR (optional, 4K and 2M only) */
+	struct pdir *pdir = NULL;
+	if (!pdpt->p_entries[pdpt_index]
+	    || pdpt->p_pages[pdpt_index] & PTE_PAGESIZE) {
+		/* entry is null, or points to a hugepage */
+		return KERN_OK;
+	} else {
+		pdir = vm_phys_to_virt(
+			ENTRY_TO_PTR(pdpt->p_entries[pdpt_index]));
+	}
+
+	/* if we're mapping a 2MiB page, we stop here */
+	if (size == PS_2M) {
+		if (pdir->p_entries[pd_index] != 0) {
+			/* this slot points to a ptab, delete it.
+			   if this slot points to a hugepage, this does nothing
+			 */
+			delete_ptab(pdir->p_entries[pd_index]);
+		}
+
+		pdir->p_pages[pd_index] = 0;
+		return KERN_OK;
+	}
+
+	/* 4. traverse PDIR, get PTAB (optional, 4K only) */
+	struct ptab *ptab = NULL;
+	if (!pdir->p_entries[pd_index]
+	    || pdir->p_pages[pd_index] & PTE_PAGESIZE) {
+		/* entry is null, or points to a hugepage */
+		return KERN_OK;
+	} else {
+		ptab = vm_phys_to_virt(ENTRY_TO_PTR(pdir->p_entries[pd_index]));
+	}
+
+	ptab->p_pages[pt_index] = 0;
+	return KERN_OK;
+}
+
 pmap_t get_kernel_pmap(void)
 {
 	return kernel_pmap;
@@ -313,8 +412,46 @@ pmap_t pmap_create(void)
 	return pmap;
 }
 
+static void delete_pdpt(phys_addr_t pd)
+{
+	if (pd & PTE_PAGESIZE) {
+		/* this entry points to a hugepage, nothing to delete */
+		return;
+	}
+
+	pd &= ENTRY_TO_PTR(pd);
+	if (!pd) {
+		/* physical address of 0x0, nothing to delete */
+		return;
+	}
+
+	struct pdpt *pdpt = vm_phys_to_virt(ENTRY_TO_PTR(pd));
+	for (int i = 0; i < 512; i++) {
+		if (pdpt->p_pages[i] & PTE_PAGESIZE) {
+			/* this is a hugepage, there is nothing to delete */
+			continue;
+		}
+
+		if (!pdpt->p_entries[i]) {
+			continue;
+		}
+
+		delete_ptab(pdpt->p_entries[i]);
+	}
+
+	kfree(pdpt);
+}
+
 void pmap_destroy(pmap_t pmap)
 {
+	struct pml4t *pml4t = vm_phys_to_virt(ENTRY_TO_PTR(pmap));
+	for (unsigned int i = 0; i < 256; i++) {
+		if (pml4t->p_entries[i]) {
+			delete_pdpt(pml4t->p_entries[i]);
+		}
+	}
+
+	kfree(pml4t);
 }
 
 static void log_fault(virt_addr_t fault_addr, enum pmap_fault_flags flags)
@@ -362,9 +499,17 @@ kern_status_t pmap_handle_fault(
 	}
 
 	struct task *task = current_task();
-	struct vm_region *space = task->t_address_space;
+	if (!task) {
+		return KERN_FATAL_ERROR;
+	}
 
-	return vm_region_demand_map(space, fault_addr, flags);
+	struct address_space *space = task->t_address_space;
+	if (!space) {
+		return KERN_FATAL_ERROR;
+	}
+
+	/* this must be called with `space` unlocked. */
+	return address_space_demand_map(space, fault_addr, flags);
 }
 
 kern_status_t pmap_add(
@@ -395,7 +540,7 @@ kern_status_t pmap_add_block(
 
 kern_status_t pmap_remove(pmap_t pmap, virt_addr_t p)
 {
-	return KERN_OK;
+	return do_pmap_remove(pmap, p, PS_4K);
 }
 
 kern_status_t pmap_remove_range(pmap_t pmap, virt_addr_t p, size_t len)

arch/x86_64/random.S

@@ -0,0 +1,43 @@
+.code64
+
+.global ml_hwrng_available
+.type ml_hwrng_available, @function
+
+ml_hwrng_available:
+	push %rbp
+	mov %rsp, %rbp
+	push %rbx
+	push %rdx
+
+	mov $1, %eax
+	mov $0, %ecx
+
+	cpuid
+
+	shr $30, %ecx
+	and $1, %ecx
+	mov %ecx, %eax
+
+	pop %rdx
+	pop %rbx
+	pop %rbp
+	ret
+
+.global ml_hwrng_generate
+.type ml_hwrng_generate, @function
+
+ml_hwrng_generate:
+	push %rbp
+	mov %rsp, %rbp
+
+	mov $100, %rcx
+.retry:
+	rdrand %rax
+	jc .done
+	loop .retry
+
+.fail:
+	mov $0, %rax
+.done:
+	pop %rbp
+	ret

arch/x86_64/thread.c

@@ -1,5 +1,7 @@
+#include <arch/msr.h>
 #include <kernel/machine/cpu.h>
 #include <kernel/machine/thread.h>
+#include <kernel/thread.h>
 
 #define MAX_REG_ARGS 6
 #define REG_ARG_0    rdi
@@ -77,3 +79,52 @@ extern kern_status_t ml_thread_prepare_user_context(
 
 	return KERN_OK;
 }
+
+kern_status_t ml_thread_config_get(
+	struct thread *thread,
+	kern_config_key_t key,
+	void *out,
+	size_t max)
+{
+	return KERN_OK;
+}
+
+kern_status_t ml_thread_config_set(
+	struct thread *thread,
+	kern_config_key_t key,
+	const void *ptr,
+	size_t len)
+{
+	switch (key) {
+	case THREAD_CFG_FSBASE:
+		if (len != sizeof(thread->tr_ml.tr_fsbase)) {
+			return KERN_INVALID_ARGUMENT;
+		}
+
+		thread->tr_ml.tr_fsbase = *(virt_addr_t *)ptr;
+		if (thread == current_thread()) {
+			wrmsr(MSR_FS_BASE, thread->tr_ml.tr_fsbase);
+		}
+
+		break;
+	case THREAD_CFG_GSBASE:
+		if (len != sizeof(thread->tr_ml.tr_gsbase)) {
+			return KERN_INVALID_ARGUMENT;
+		}
+
+		thread->tr_ml.tr_gsbase = *(virt_addr_t *)ptr;
+		if (thread == current_thread()) {
+			/* we're in the kernel right now, so the user and kernel
+			 * gs-base registers are swapped. when we return to
+			 * usermode, this value will be swapped back into
+			 * the user gs-base register */
+			wrmsr(MSR_KERNEL_GS_BASE, thread->tr_ml.tr_gsbase);
+		}
+
+		break;
+	default:
+		return KERN_INVALID_ARGUMENT;
+	}
+
+	return KERN_OK;
+}

arch/x86_64/thread_switch.S

@@ -13,6 +13,22 @@ ml_thread_switch:
 	push %rax
 	push %rcx
 	push %rdx
+
+	// set fs-base
+	mov $0xC0000100, %rcx
+	movq THREAD_fsbase(%rsi), %rax
+	movq THREAD_fsbase(%rsi), %rdx
+	shr $32, %rdx
+	wrmsr
+
+	// set (kernel) gs-base (it will be swapped back into user-gs-base at
+	// the end of this function)
+	mov $0xC0000102, %rcx
+	movq THREAD_gsbase(%rsi), %rax
+	movq THREAD_gsbase(%rsi), %rdx
+	shr $32, %rdx
+	wrmsr
+
 	push %rbx
 	pushq $0
 	push %rbp
@@ -73,4 +89,5 @@ ml_thread_switch_user:
 	pop %rax
 
 	add $16, %rsp
+	swapgs
 	iretq

arch/x86_64/tss.c

@@ -1,5 +1,3 @@
-#include "arch/msr.h"
-
 #include <arch/gdt.h>
 #include <arch/tss.h>
 #include <kernel/libc/string.h>
@@ -22,9 +20,6 @@ void tss_init(struct tss *tss, struct tss_ptr *ptr)
 void tss_load(struct tss *tss)
 {
 	tss_flush(TSS_GDT_INDEX);
-
-	uintptr_t kernel_gs_base_reg = 0xC0000102;
-	wrmsr(kernel_gs_base_reg, (uintptr_t)tss);
 }
 
 virt_addr_t tss_get_kstack(struct tss *tss)

ds/ringbuffer.c

@@ -1,5 +1,6 @@
 #include <kernel/ringbuffer.h>
 #include <kernel/sched.h>
+#include <kernel/vm.h>
 
 size_t ringbuffer_unread(struct ringbuffer *ring_buffer)
 {
@@ -9,7 +10,7 @@ size_t ringbuffer_unread(struct ringbuffer *ring_buffer)
 
 	if (ring_buffer->r_read_ptr > ring_buffer->r_write_ptr) {
 		return (ring_buffer->r_size - ring_buffer->r_read_ptr)
-			+ ring_buffer->r_write_ptr;
+		     + ring_buffer->r_write_ptr;
 	} else {
 		return (ring_buffer->r_write_ptr - ring_buffer->r_read_ptr);
 	}
@@ -25,7 +26,7 @@ size_t ringbuffer_avail(struct ringbuffer *ring_buffer)
 		return ring_buffer->r_read_ptr - ring_buffer->r_write_ptr - 1;
 	} else {
 		return (ring_buffer->r_size - ring_buffer->r_write_ptr)
-			+ ring_buffer->r_read_ptr - 1;
+		     + ring_buffer->r_read_ptr - 1;
 	}
 }
 
@@ -45,7 +46,11 @@ static inline void increment_write(struct ringbuffer *ring_buffer)
 	}
 }
 
-size_t ringbuffer_read(struct ringbuffer *ring_buffer, size_t size, void *p, mango_flags_t flags)
+size_t ringbuffer_read(
+	struct ringbuffer *ring_buffer,
+	size_t size,
+	void *p,
+	mango_flags_t flags)
 {
 	if (!ring_buffer) {
 		return 0;
@@ -58,7 +63,9 @@ size_t ringbuffer_read(struct ringbuffer *ring_buffer, size_t size, void *p, man
 	while (collected < size) {
 		spin_lock_irqsave(&ring_buffer->r_lock, &lock_flags);
 		while (ringbuffer_unread(ring_buffer) > 0 && collected < size) {
-			buffer[collected] = ring_buffer->r_buffer[ring_buffer->r_read_ptr];
+			buffer[collected]
+				= ring_buffer
+			                  ->r_buffer[ring_buffer->r_read_ptr];
 			increment_read(ring_buffer);
 			collected++;
 		}
@@ -66,7 +73,9 @@ size_t ringbuffer_read(struct ringbuffer *ring_buffer, size_t size, void *p, man
 		wakeup_queue(&ring_buffer->r_wait_writers);
 
 		if (flags & S_NOBLOCK) {
-			spin_unlock_irqrestore(&ring_buffer->r_lock, lock_flags);
+			spin_unlock_irqrestore(
+				&ring_buffer->r_lock,
+				lock_flags);
 			break;
 		}
 
@@ -86,7 +95,11 @@ size_t ringbuffer_read(struct ringbuffer *ring_buffer, size_t size, void *p, man
 	return collected;
 }
 
-size_t ringbuffer_write(struct ringbuffer *ring_buffer, size_t size, const void *p, mango_flags_t flags)
+size_t ringbuffer_write(
+	struct ringbuffer *ring_buffer,
+	size_t size,
+	const void *p,
+	mango_flags_t flags)
 {
 	if (!ring_buffer || !size) {
 		return 0;
@@ -100,7 +113,8 @@ size_t ringbuffer_write(struct ringbuffer *ring_buffer, size_t size, const void
 		spin_lock_irqsave(&ring_buffer->r_lock, &lock_flags);
 
 		while (ringbuffer_avail(ring_buffer) > 0 && written < size) {
-			ring_buffer->r_buffer[ring_buffer->r_write_ptr] = buffer[written];
+			ring_buffer->r_buffer[ring_buffer->r_write_ptr]
+				= buffer[written];
 			increment_write(ring_buffer);
 			written++;
 		}
@@ -108,7 +122,9 @@ size_t ringbuffer_write(struct ringbuffer *ring_buffer, size_t size, const void
 		wakeup_queue(&ring_buffer->r_wait_readers);
 
 		if (flags & S_NOBLOCK) {
-			spin_unlock_irqrestore(&ring_buffer->r_lock, lock_flags);
+			spin_unlock_irqrestore(
+				&ring_buffer->r_lock,
+				lock_flags);
 			break;
 		}
 

include/kernel/address-space.h

@@ -0,0 +1,172 @@
+#ifndef KERNEL_ADDRESS_SPACE_H_
+#define KERNEL_ADDRESS_SPACE_H_
+
+#include <kernel/object.h>
+#include <kernel/pmap.h>
+#include <kernel/vm.h>
+
+#define ADDRESS_SPACE_COPY_ALL ((size_t)-1)
+#define ADDRESS_SPACE_F_
+
+struct address_space;
+struct vm_object;
+
+struct vm_area {
+	/* the vm-object mapped into this area.
+	 * if this is NULL, the vm_area represents an area of reserved memory.
+	 * it cannot be accessed, and mapping operations with MAP_ADDRESS_ANY
+	 * will avoid the area, but fixed address mappings in this area
+	 * will succeed. */
+	struct vm_object *vma_object;
+	/* the address space that this vm_area is a part of */
+	struct address_space *vma_space;
+	/* used to link to vm_object->vo_mappings */
+	struct queue_entry vma_object_entry;
+	/* the memory protection flags applied to this area */
+	vm_prot_t vma_prot;
+	/* offset in bytes to the start of the object data that was mapped */
+	off_t vma_object_offset;
+	/* used to link to address_space->s_mappings */
+	struct btree_node vma_node;
+	/* address of the first byte in this area */
+	virt_addr_t vma_base;
+	/* address of the last byte in this area */
+	virt_addr_t vma_limit;
+};
+
+struct address_space {
+	struct object s_base;
+
+	/* address of the first byte in this address space */
+	virt_addr_t s_base_address;
+	/* address of the last byte in this address space */
+	virt_addr_t s_limit_address;
+
+	/* btree of struct vm_area representing mapped vm-objects.
+	 * sibling entries cannot overlap each other. */
+	struct btree s_mappings;
+	/* btree of struct vm_area representing reserved regions of the
+	 * address space.
+	 * reserved regions will not be automatically allocated by the kernel.
+	 * sibling entries cannot overlap each other.
+	 * overlap between s_mappings and s_reserved IS allowed. */
+	struct btree s_reserved;
+
+	/* the corresponding physical address space */
+	pmap_t s_pmap;
+};
+
+extern kern_status_t address_space_type_init(void);
+extern struct address_space *address_space_cast(struct object *obj);
+
+/* create a new vm-region, optionally within a parent region.
+ * `offset` is the byte offset within the parent region where the new region
+ * should start.
+ * if no parent is specified, `offset` is the absolute virtual address of the
+ * start of the region.
+ * in both cases, `len` is the length of the new region in bytes. */
+extern kern_status_t address_space_create(
+	virt_addr_t base,
+	virt_addr_t limit,
+	struct address_space **out);
+
+/* map a vm-object into a vm-region.
+ * [region_offset,length] must fall within exactly one region, and cannot span
+ * multiple sibling regions.
+ * if [region_offset,length] falls within a child region, the map operation
+ * will be transparently redirected to the relevant region.
+ * `prot` must be allowed both by the region into which the mapping is being
+ * created AND the vm-object being mapped. */
+extern kern_status_t address_space_map(
+	struct address_space *space,
+	virt_addr_t map_address,
+	struct vm_object *object,
+	off_t object_offset,
+	size_t length,
+	vm_prot_t prot,
+	virt_addr_t *out);
+extern kern_status_t address_space_unmap(
+	struct address_space *region,
+	virt_addr_t base,
+	size_t length);
+
+/* reserve an area of the address space. the kernel will not place any
+ * new mappings in this area unless explicitly told to (i.e. by not using
+ * MAP_ADDRESS_ANY). Use MAP_ADDRESS_ANY to have the kernel allocate a region
+ * of the address space for you */
+extern kern_status_t address_space_reserve(
+	struct address_space *space,
+	virt_addr_t base,
+	size_t length,
+	virt_addr_t *out);
+/* release a previously reserved area of the address space. */
+extern kern_status_t address_space_release(
+	struct address_space *space,
+	virt_addr_t base,
+	size_t length);
+
+extern bool address_space_validate_access(
+	struct address_space *region,
+	virt_addr_t base,
+	size_t len,
+	vm_prot_t prot);
+
+/* find the mapping corresponding to the given virtual address, and page-in the
+ * necessary vm_page to allow the memory access to succeed. if the relevant
+ * vm-object page hasn't been allocated yet, it will be allocated here.
+ * this function must be called with `region` UNLOCKED and interrupts ENABLED.
+ */
+extern kern_status_t address_space_demand_map(
+	struct address_space *region,
+	virt_addr_t addr,
+	enum pmap_fault_flags flags);
+
+/* read data from the user-space area of a vm-region into a kernel-mode buffer
+ */
+extern kern_status_t address_space_read(
+	struct address_space *src_region,
+	virt_addr_t src_ptr,
+	size_t count,
+	void *dest,
+	size_t *nr_read);
+
+/* write data to the user-space area of a vm-region from a kernel-mode buffer
+ */
+extern kern_status_t address_space_write(
+	struct address_space *dst_region,
+	virt_addr_t dst_ptr,
+	size_t count,
+	const void *src,
+	size_t *nr_written);
+
+extern kern_status_t address_space_memmove(
+	struct address_space *dest_space,
+	virt_addr_t dest_ptr,
+	struct address_space *src_space,
+	virt_addr_t src_ptr,
+	size_t count,
+	size_t *nr_moved);
+
+extern kern_status_t address_space_memmove_v(
+	struct address_space *dest_space,
+	size_t dest_offset,
+	const kern_iovec_t *dest_iov,
+	size_t nr_dest_iov,
+	struct address_space *src_space,
+	size_t src_offset,
+	const kern_iovec_t *src_iov,
+	size_t nr_src_iov,
+	size_t bytes_to_move,
+	size_t *nr_bytes_moved);
+
+extern kern_status_t address_space_translate(
+	struct address_space *space,
+	virt_addr_t in,
+	phys_addr_t *out,
+	unsigned long *irq_flags);
+
+void address_space_dump(struct address_space *region);
+
+DEFINE_OBJECT_LOCK_FUNCTION(address_space, s_base)
+
+#endif

include/kernel/arg.h

@@ -1,8 +1,8 @@
 #ifndef KERNEL_ARG_H_
 #define KERNEL_ARG_H_
 
+#include <mango/types.h>
 #include <stdbool.h>
-#include <mango/status.h>
 
 #define CMDLINE_MAX 4096
 

include/kernel/atomic.h

@@ -0,0 +1,66 @@
+#ifndef KERNEL_ATOMIC_H_
+#define KERNEL_ATOMIC_H_
+
+#include <stdbool.h>
+#include <stdint.h>
+
+typedef int64_t atomic_t;
+
+/* load and return the value pointed to by `v` */
+static inline atomic_t atomic_load(atomic_t *v)
+{
+	return __atomic_load_n(v, __ATOMIC_ACQUIRE);
+}
+/* store the value `v` to the pointer `dest` */
+static inline void atomic_store(atomic_t *dest, atomic_t v)
+{
+	__atomic_store_n(dest, v, __ATOMIC_ACQUIRE);
+}
+/* store the value `v` to the pointer `dest`, and return the value previously
+ * stored at `dest` */
+static inline atomic_t atomic_exchange(atomic_t *dest, atomic_t v)
+{
+	return __atomic_exchange_n(dest, v, __ATOMIC_ACQUIRE);
+}
+/* compare the contents of `ptr` to the contents of `expected`.
+ * if they match, store the value `desired` to the pointer `ptr` and return
+ * true. if the do NOT match, store the value `*ptr` to the pointer `desired`
+ * and return false.
+ */
+static inline bool atomic_compare_exchange(
+	atomic_t *ptr,
+	atomic_t *expected,
+	atomic_t desired)
+{
+	return __atomic_compare_exchange_n(
+		ptr,
+		expected,
+		desired,
+		false,
+		__ATOMIC_ACQUIRE,
+		__ATOMIC_ACQUIRE);
+}
+
+/* perform the operation *ptr += val, and return the result */
+static inline atomic_t atomic_add_fetch(atomic_t *ptr, atomic_t val)
+{
+	return __atomic_add_fetch(ptr, val, __ATOMIC_ACQUIRE);
+}
+/* perform the operation *ptr -= val, and return the result */
+static inline atomic_t atomic_sub_fetch(atomic_t *ptr, atomic_t val)
+{
+	return __atomic_sub_fetch(ptr, val, __ATOMIC_ACQUIRE);
+}
+
+/* perform the operation *ptr += val, and return the previous value of *ptr */
+static inline atomic_t atomic_fetch_add(atomic_t *ptr, atomic_t val)
+{
+	return __atomic_fetch_add(ptr, val, __ATOMIC_ACQUIRE);
+}
+/* perform the operation *ptr -= val, and return the previous value of *ptr */
+static inline atomic_t atomic_fetch_sub(atomic_t *ptr, atomic_t val)
+{
+	return __atomic_fetch_sub(ptr, val, __ATOMIC_ACQUIRE);
+}
+
+#endif

include/kernel/channel.h

@@ -4,48 +4,50 @@
 #include <kernel/object.h>
 #include <kernel/sched.h>
 
-struct kmsg;
+struct msg;
 
 struct channel {
 	struct object c_base;
 	unsigned int c_id;
-	struct waitqueue c_wq;
+	unsigned int c_msg_waiting;
 	struct btree c_msg;
 	struct btree_node c_node;
 };
 
 extern kern_status_t channel_type_init(void);
+extern struct channel *channel_cast(struct object *obj);
 
 extern struct channel *channel_create(void);
 
 extern kern_status_t channel_enqueue_msg(
 	struct channel *channel,
-	struct kmsg *msg);
+	struct msg *msg);
 
 extern kern_status_t channel_recv_msg(
 	struct channel *channel,
-	struct msg *out_msg,
+	kern_msg_t *out_msg,
-	msgid_t *out_id,
 	unsigned long *irq_flags);
 extern kern_status_t channel_reply_msg(
 	struct channel *channel,
 	msgid_t id,
-	const struct msg *resp,
+	const kern_msg_t *reply,
 	unsigned long *irq_flags);
 
 extern kern_status_t channel_read_msg(
 	struct channel *channel,
 	msgid_t msg,
 	size_t offset,
-	void *buf,
+	struct address_space *dest_region,
-	size_t len,
+	const kern_iovec_t *dest_iov,
+	size_t dest_iov_count,
 	size_t *nr_read);
 extern kern_status_t channel_write_msg(
 	struct channel *channel,
 	msgid_t msg,
 	size_t offset,
-	const void *buf,
+	struct address_space *src_region,
-	size_t len,
+	const kern_iovec_t *src_iov,
+	size_t src_iov_count,
 	size_t *nr_written);
 
 DEFINE_OBJECT_LOCK_FUNCTION(channel, c_base)

include/kernel/console.h

@@ -14,9 +14,10 @@
    representing a serial port may allow both sending AND receiving over the
    port.
 */
-#include <kernel/queue.h>
 #include <kernel/locks.h>
+#include <kernel/queue.h>
 #include <mango/status.h>
+#include <mango/types.h>
 
 #ifdef __cplusplus
 extern "C" {

include/kernel/cpu.h

@@ -1,10 +1,11 @@
 #ifndef KERNEL_CPU_H_
 #define KERNEL_CPU_H_
 
-#include <kernel/types.h>
 #include <kernel/machine/cpu.h>
-#include <stdint.h>
 #include <kernel/sched.h>
+#include <kernel/types.h>
+#include <kernel/work.h>
+#include <stdint.h>
 
 #ifdef __cplusplus
 extern "C" {
@@ -26,7 +27,7 @@ struct cpu_data {
 
 /* maximum number of processor cores that the kernel can support.
    TODO move to build config option */
-#define CPU_MAX 128
+#define CPU_MAX    128
 
 #define this_cpu() (ml_cpu_block_get_id(ml_this_cpu()))
 
@@ -49,7 +50,7 @@ static inline cycles_t cycles_diff(cycles_t then, cycles_t now)
 	return now >= then ? now - then : (CYCLES_MAX - then) + now;
 }
 
-#define irq_enable() ml_int_enable()
+#define irq_enable()  ml_int_enable()
 #define irq_disable() ml_int_disable()
 
 extern void preempt_disable(void);

include/kernel/equeue.h

@@ -0,0 +1,34 @@
+#ifndef KERNEL_EQUEUE_H_
+#define KERNEL_EQUEUE_H_
+
+#include <kernel/locks.h>
+#include <kernel/object.h>
+#include <kernel/sched.h>
+#include <mango/types.h>
+
+#define EQUEUE_PACKET_MAX 100
+
+enum equeue_flags {
+	EQUEUE_WAIT,
+	EQUEUE_DISCARD,
+};
+
+struct equeue {
+	struct object eq_base;
+	unsigned short eq_read_ptr, eq_write_ptr;
+	equeue_packet_t eq_packets[EQUEUE_PACKET_MAX];
+	struct waitqueue eq_wq;
+};
+
+extern kern_status_t equeue_type_init(void);
+extern struct equeue *equeue_cast(struct object *obj);
+
+extern struct equeue *equeue_create(void);
+
+extern kern_status_t equeue_enqueue(
+	struct equeue *q,
+	const equeue_packet_t *pkt,
+	enum equeue_flags flags);
+extern kern_status_t equeue_dequeue(struct equeue *q, equeue_packet_t *out);
+
+#endif

include/kernel/futex.h

@@ -0,0 +1,34 @@
+#ifndef KERNEL_FUTEX_H_
+#define KERNEL_FUTEX_H_
+
+#include <kernel/btree.h>
+#include <kernel/wait.h>
+#include <mango/types.h>
+
+struct task;
+struct address_space;
+
+typedef uintptr_t futex_key_t;
+
+struct futex {
+	struct btree_node f_node;
+
+	futex_key_t f_key;
+	struct waitqueue f_waiters;
+};
+
+extern kern_status_t futex_init(void);
+extern kern_status_t futex_get(
+	kern_futex_t *futex,
+	futex_key_t *out,
+	unsigned int flags);
+extern kern_status_t futex_wait(
+	futex_key_t futex,
+	kern_futex_t new_val,
+	unsigned int flags);
+extern kern_status_t futex_wake(
+	futex_key_t futex,
+	size_t nwaiters,
+	unsigned int flags);
+
+#endif

include/kernel/handle.h

@@ -3,6 +3,7 @@
 
 #include <kernel/bitmap.h>
 #include <mango/status.h>
+#include <mango/types.h>
 #include <stddef.h>
 #include <stdint.h>
 
@@ -16,6 +17,7 @@ typedef uintptr_t handle_flags_t;
 
 struct task;
 struct object;
+struct address_space;
 struct handle_list;
 
 struct handle {
@@ -54,12 +56,14 @@ extern struct handle *handle_table_get_handle(
 	struct handle_table *tab,
 	kern_handle_t handle);
 
-extern kern_status_t handle_list_transfer(
+extern kern_status_t handle_table_transfer(
-	struct handle_table *dest,
+	struct address_space *dst_region,
-	struct handle_list *dest_list,
+	struct handle_table *dst,
-	size_t dest_list_count,
+	kern_msg_handle_t *dst_handles,
+	size_t dst_handles_max,
+	struct address_space *src_region,
 	struct handle_table *src,
-	const struct handle_list *src_list,
+	kern_msg_handle_t *src_handles,
-	size_t src_list_count);
+	size_t src_handles_count);
 
 #endif

include/kernel/iovec.h

@@ -4,8 +4,13 @@
 #include <mango/types.h>
 #include <stddef.h>
 
+struct address_space;
+
 struct iovec_iterator {
-	const struct iovec *it_vecs;
+	/* if this is set, we are iterating over a list of iovecs stored in
+	 * userspace, and must go through this region to retrieve the data. */
+	struct address_space *it_region;
+	const kern_iovec_t *it_vecs;
 	size_t it_nr_vecs;
 	size_t it_vec_ptr;
 
@@ -15,7 +20,12 @@ struct iovec_iterator {
 
 extern void iovec_iterator_begin(
 	struct iovec_iterator *it,
-	const struct iovec *vecs,
+	const kern_iovec_t *vecs,
+	size_t nr_vecs);
+extern void iovec_iterator_begin_user(
+	struct iovec_iterator *it,
+	struct address_space *address_space,
+	const kern_iovec_t *vecs,
 	size_t nr_vecs);
 
 extern void iovec_iterator_seek(struct iovec_iterator *it, size_t nr_bytes);

include/kernel/locks.h

@@ -10,13 +10,75 @@ extern "C" {
 
 typedef __aligned(8) ml_hwlock_t spin_lock_t;
 
-#define SPIN_LOCK_INIT ML_HWLOCK_INIT
+#define SPIN_LOCK_INIT                ML_HWLOCK_INIT
 
-#define spin_lock(lck) ml_hwlock_lock(lck);
+#define spin_lock(lck)                ml_hwlock_lock(lck);
-#define spin_unlock(lck) ml_hwlock_unlock(lck);
+#define spin_unlock(lck)              ml_hwlock_unlock(lck);
+
+#define spin_lock_irq(lck)            ml_hwlock_lock_irq(lck);
+#define spin_unlock_irq(lck)          ml_hwlock_unlock_irq(lck);
 
 #define spin_lock_irqsave(lck, flags) ml_hwlock_lock_irqsave(lck, flags);
-#define spin_unlock_irqrestore(lck, flags) ml_hwlock_unlock_irqrestore(lck, flags);
+#define spin_unlock_irqrestore(lck, flags)                                     \
+	ml_hwlock_unlock_irqrestore(lck, flags);
+
+static inline void spin_lock_pair(spin_lock_t *a, spin_lock_t *b)
+{
+	if (a == b) {
+		spin_lock(a);
+	} else if (a < b) {
+		spin_lock(a);
+		spin_lock(b);
+	} else {
+		spin_lock(b);
+		spin_lock(a);
+	}
+}
+
+static inline void spin_unlock_pair(spin_lock_t *a, spin_lock_t *b)
+{
+	if (a == b) {
+		spin_unlock(a);
+	} else if (a < b) {
+		spin_unlock(b);
+		spin_unlock(a);
+	} else {
+		spin_unlock(a);
+		spin_unlock(b);
+	}
+}
+
+static inline void spin_lock_pair_irqsave(
+	spin_lock_t *a,
+	spin_lock_t *b,
+	unsigned long *flags)
+{
+	if (a == b) {
+		spin_lock_irqsave(a, flags);
+	} else if (a < b) {
+		spin_lock_irqsave(a, flags);
+		spin_lock(b);
+	} else {
+		spin_lock_irqsave(b, flags);
+		spin_lock(a);
+	}
+}
+
+static inline void spin_unlock_pair_irqrestore(
+	spin_lock_t *a,
+	spin_lock_t *b,
+	unsigned long flags)
+{
+	if (a == b) {
+		spin_unlock_irqrestore(a, flags);
+	} else if (a < b) {
+		spin_unlock(b);
+		spin_unlock_irqrestore(a, flags);
+	} else {
+		spin_unlock(a);
+		spin_unlock_irqrestore(b, flags);
+	}
+}
 
 #ifdef __cplusplus
 }

include/kernel/memblock.h

@@ -22,8 +22,8 @@
 #ifndef KERNEL_MEMBLOCK_H_
 #define KERNEL_MEMBLOCK_H_
 
-#include <limits.h>
 #include <kernel/types.h>
+#include <limits.h>
 #include <stddef.h>
 
 #ifdef __cplusplus
@@ -338,6 +338,8 @@ extern void __next_memory_region(
 	phys_addr_t start,
 	phys_addr_t end);
 
+extern void memblock_dump(void);
+
 #ifdef __cplusplus
 }
 #endif

include/kernel/msg.h

@@ -13,18 +13,34 @@ enum kmsg_status {
 	KMSG_WAIT_RECEIVE,
 	KMSG_WAIT_REPLY,
 	KMSG_REPLY_SENT,
+	KMSG_ASYNC,
 };
 
-struct kmsg {
+struct msg {
 	spin_lock_t msg_lock;
 	enum kmsg_status msg_status;
 	struct btree_node msg_node;
 	msgid_t msg_id;
-	kern_status_t msg_result;
 	struct port *msg_sender_port;
 	struct thread *msg_sender_thread;
-	const struct msg *msg_req;
+	kern_status_t msg_result;
-	struct msg *msg_resp;
+	kern_msg_type_t msg_type;
+
+	union {
+		/* msg_type = KERN_MSG_TYPE_DATA */
+		struct {
+			kern_msg_t msg_req, msg_resp;
+		};
+
+		/* msg_type = KERN_MSG_TYPE_EVENT */
+		struct {
+			kern_msg_event_type_t msg_event;
+		};
+	};
 };
 
+extern void msg_init(void);
+extern struct msg *msg_alloc(void);
+extern void msg_free(struct msg *msg);
+
 #endif

include/kernel/object.h

@@ -1,10 +1,12 @@
 #ifndef KERNEL_OBJECT_H_
 #define KERNEL_OBJECT_H_
 
+#include <kernel/atomic.h>
 #include <kernel/flags.h>
 #include <kernel/locks.h>
-#include <mango/status.h>
 #include <kernel/vm.h>
+#include <kernel/wait.h>
+#include <mango/status.h>
 #include <stddef.h>
 
 #ifdef __cplusplus
@@ -31,6 +33,32 @@ extern "C" {
 		unsigned long flags)                                           \
 	{                                                                      \
 		object_unlock_irqrestore(&p->base, flags);                     \
+	}                                                                      \
+	static inline void object_name##_lock_pair(                            \
+		struct object_name *a,                                         \
+		struct object_name *b)                                         \
+	{                                                                      \
+		object_lock_pair(&a->base, &b->base);                          \
+	}                                                                      \
+	static inline void object_name##_unlock_pair(                          \
+		struct object_name *a,                                         \
+		struct object_name *b)                                         \
+	{                                                                      \
+		object_unlock_pair(&a->base, &b->base);                        \
+	}                                                                      \
+	static inline void object_name##_lock_pair_irqsave(                    \
+		struct object_name *a,                                         \
+		struct object_name *b,                                         \
+		unsigned long *flags)                                          \
+	{                                                                      \
+		object_lock_pair_irqsave(&a->base, &b->base, flags);           \
+	}                                                                      \
+	static inline void object_name##_unlock_pair_irqrestore(               \
+		struct object_name *a,                                         \
+		struct object_name *b,                                         \
+		unsigned long flags)                                           \
+	{                                                                      \
+		object_unlock_pair_irqrestore(&a->base, &b->base, flags);      \
 	}
 
 #define OBJECT_MAGIC    0xBADDCAFE
@@ -67,11 +95,13 @@ struct object_type {
 
 struct object {
 	uint32_t ob_magic;
+	koid_t ob_id;
 	struct object_type *ob_type;
 	spin_lock_t ob_lock;
-	unsigned int ob_refcount;
+	uint32_t ob_signals;
-	unsigned int ob_handles;
+	atomic_t ob_refcount;
 	struct queue_entry ob_list;
+	struct waitqueue ob_wq;
 } __aligned(sizeof(long));
 
 extern kern_status_t object_bootstrap(void);
@@ -81,13 +111,30 @@ extern kern_status_t object_type_unregister(struct object_type *p);
 extern struct object *object_create(struct object_type *type);
 extern struct object *object_ref(struct object *obj);
 extern void object_unref(struct object *obj);
-extern void object_add_handle(struct object *obj);
-extern void object_remove_handle(struct object *obj);
 extern void object_lock(struct object *obj);
 extern void object_unlock(struct object *obj);
 extern void object_lock_irqsave(struct object *obj, unsigned long *flags);
 extern void object_unlock_irqrestore(struct object *obj, unsigned long flags);
 
+extern void object_lock_pair(struct object *a, struct object *b);
+extern void object_unlock_pair(struct object *a, struct object *b);
+
+extern void object_lock_pair_irqsave(
+	struct object *a,
+	struct object *b,
+	unsigned long *flags);
+extern void object_unlock_pair_irqrestore(
+	struct object *a,
+	struct object *b,
+	unsigned long flags);
+
+extern void object_assert_signal(struct object *obj, uint32_t signals);
+extern void object_clear_signal(struct object *obj, uint32_t signals);
+extern void object_wait_signal(
+	struct object *obj,
+	uint32_t signals,
+	unsigned long *irq_flags);
+
 #ifdef __cplusplus
 }
 #endif

include/kernel/port.h

@@ -29,10 +29,12 @@ extern struct port *port_cast(struct object *obj);
 extern struct port *port_create(void);
 
 extern kern_status_t port_connect(struct port *port, struct channel *remote);
+extern kern_status_t port_disconnect(struct port *port);
 extern kern_status_t port_send_msg(
 	struct port *port,
-	const struct msg *req,
+	const kern_msg_t *msg,
-	struct msg *resp);
+	kern_msg_t *out_response,
+	unsigned long *lock_flags);
 
 DEFINE_OBJECT_LOCK_FUNCTION(port, p_base)
 

include/kernel/ringbuffer.h

@@ -1,8 +1,10 @@
 #ifndef KERNEL_RINGBUFFER_H_
 #define KERNEL_RINGBUFFER_H_
 
+#include <kernel/flags.h>
 #include <kernel/locks.h>
-#include <kernel/sched.h>
+#include <kernel/types.h>
+#include <kernel/wait.h>
 
 struct ringbuffer {
 	unsigned char *r_buffer;
@@ -22,12 +24,21 @@ extern kern_status_t ringbuffer_deinit(struct ringbuffer *buf);
 
 extern size_t ringbuffer_unread(struct ringbuffer *buf);
 extern size_t ringbuffer_avail(struct ringbuffer *buf);
-extern size_t ringbuffer_read(struct ringbuffer *buf, size_t size, void *buffer, mango_flags_t flags);
+extern size_t ringbuffer_read(
-extern size_t ringbuffer_write(struct ringbuffer *buf, size_t size, const void *buffer, mango_flags_t flags);
+	struct ringbuffer *buf,
+	size_t size,
+	void *buffer,
+	mango_flags_t flags);
+extern size_t ringbuffer_write(
+	struct ringbuffer *buf,
+	size_t size,
+	const void *buffer,
+	mango_flags_t flags);
 
 /* TODO */
-//extern size_t ringbuffer_peek(struct ringbuffer *buf, size_t at, size_t size, void *buffer);
+// extern size_t ringbuffer_peek(struct ringbuffer *buf, size_t at, size_t size,
-//extern size_t ringbuffer_skip(struct ringbuffer *buf, size_t count);
+// void *buffer); extern size_t ringbuffer_skip(struct ringbuffer *buf, size_t
+// count);
 
 extern int ringbuffer_write_would_block(struct ringbuffer *buf);
 

include/kernel/sched.h

@@ -4,32 +4,11 @@
 #include <kernel/btree.h>
 #include <kernel/handle.h>
 #include <kernel/locks.h>
-#include <kernel/msg.h>
-#include <kernel/object.h>
-#include <kernel/pmap.h>
 #include <kernel/queue.h>
+#include <kernel/types.h>
 #include <mango/status.h>
 
-#define TASK_NAME_MAX       64
+#define PRIO_MAX 32
-#define PRIO_MAX            32
-#define PID_MAX             99999
-#define THREAD_KSTACK_ORDER VM_PAGE_4K
-#define THREAD_MAX          65536
-
-#define wait_event(wq, cond)                                                   \
-	({                                                                     \
-		struct thread *self = current_thread();                        \
-		struct wait_item waiter;                                       \
-		wait_item_init(&waiter, self);                                 \
-		for (;;) {                                                     \
-			thread_wait_begin(&waiter, wq);                        \
-			if (cond) {                                            \
-				break;                                         \
-			}                                                      \
-			schedule(SCHED_NORMAL);                                \
-		}                                                              \
-		thread_wait_end(&waiter, wq);                                  \
-	})
 
 #ifdef __cplusplus
 extern "C" {
@@ -37,23 +16,6 @@ extern "C" {
 
 struct channel;
 struct runqueue;
-struct work_item;
-
-enum task_state {
-	TASK_RUNNING,
-	TASK_STOPPED,
-};
-
-enum thread_state {
-	THREAD_READY = 1,
-	THREAD_SLEEPING = 2,
-	THREAD_STOPPED = 3,
-};
-
-enum thread_flags {
-	THREAD_F_NEED_RESCHED = 0x01u,
-	THREAD_F_NO_PREEMPT = 0x02u,
-};
 
 enum sched_priority {
 	PRIO_IDLE = 4,
@@ -75,54 +37,6 @@ enum sched_mode {
 	SCHED_IRQ = 1,
 };
 
-struct task {
-	struct object t_base;
-
-	struct task *t_parent;
-	long t_id;
-	enum task_state t_state;
-	char t_name[TASK_NAME_MAX];
-
-	pmap_t t_pmap;
-	struct vm_region *t_address_space;
-	struct handle_table *t_handles;
-	struct btree b_channels;
-
-	struct btree_node t_tasklist;
-	struct queue_entry t_child_entry;
-
-	size_t t_next_thread_id;
-	struct queue t_threads;
-	struct queue t_children;
-};
-
-struct thread {
-	struct object thr_base;
-
-	enum thread_state tr_state;
-	enum thread_flags tr_flags;
-	struct task *tr_parent;
-
-	unsigned int tr_id;
-	unsigned int tr_prio;
-
-	cycles_t tr_charge_period_start;
-	cycles_t tr_quantum_cycles, tr_quantum_target;
-	cycles_t tr_total_cycles;
-
-	virt_addr_t tr_ip, tr_sp, tr_bp;
-	virt_addr_t tr_cpu_user_sp, tr_cpu_kernel_sp;
-
-	struct runqueue *tr_rq;
-	struct kmsg tr_msg;
-
-	struct queue_entry tr_parent_entry;
-	struct queue_entry tr_rqentry;
-
-	struct vm_page *tr_kstack;
-	struct vm_object *tr_ustack;
-};
-
 struct runqueue {
 	struct queue rq_queues[PRIO_MAX];
 	uint32_t rq_readybits;
@@ -140,34 +54,6 @@ struct timer {
 	void (*t_callback)(struct timer *);
 };
 
-struct wait_item {
-	struct thread *w_thread;
-	struct queue_entry w_entry;
-};
-
-struct waitqueue {
-	struct queue wq_waiters;
-	spin_lock_t wq_lock;
-};
-
-typedef void (*work_func_t)(struct work_item *);
-
-struct work_item {
-	void *w_data;
-	work_func_t w_func;
-	struct queue_entry w_head;
-};
-
-struct worker_pool {
-	struct thread **wp_workers;
-	size_t wp_nworkers;
-};
-
-struct workqueue {
-	spin_lock_t wq_lock;
-	struct queue wq_queue; /* list of struct work_item */
-};
-
 extern kern_status_t sched_init(void);
 extern void schedule(enum sched_mode mode);
 extern void preempt_disable(void);
@@ -187,38 +73,6 @@ static inline void rq_unlock(struct runqueue *rq, unsigned long flags)
 extern void rq_remove_thread(struct runqueue *rq, struct thread *thr);
 extern struct runqueue *cpu_rq(unsigned int cpu);
 
-extern struct task *task_alloc(void);
-extern struct task *task_cast(struct object *obj);
-extern struct task *task_create(const char *name, size_t name_len);
-static inline struct task *task_ref(struct task *task)
-{
-	return OBJECT_CAST(struct task, t_base, object_ref(&task->t_base));
-}
-static inline void task_unref(struct task *task)
-{
-	object_unref(&task->t_base);
-}
-extern kern_status_t task_add_child(struct task *parent, struct task *child);
-extern kern_status_t task_add_channel(
-	struct task *task,
-	struct channel *channel,
-	unsigned int id);
-extern struct channel *task_get_channel(struct task *task, unsigned int id);
-extern struct task *task_from_tid(tid_t id);
-extern kern_status_t task_open_handle(
-	struct task *task,
-	struct object *obj,
-	handle_flags_t flags,
-	kern_handle_t *out);
-extern kern_status_t task_resolve_handle(
-	struct task *task,
-	kern_handle_t handle,
-	struct object **out_obj,
-	handle_flags_t *out_flags);
-extern kern_status_t task_close_handle(struct task *task, kern_handle_t handle);
-extern struct thread *task_create_thread(struct task *parent);
-extern struct task *kernel_task(void);
-extern struct task *idle_task(void);
 extern cycles_t default_quantum(void);
 
 extern bool need_resched(void);
@@ -231,45 +85,12 @@ extern void schedule_thread_on_cpu(struct thread *thr);
 extern void start_charge_period(void);
 extern void end_charge_period(void);
 
-DEFINE_OBJECT_LOCK_FUNCTION(task, t_base)
-
-extern struct thread *thread_alloc(void);
-extern struct thread *thread_cast(struct object *obj);
-extern kern_status_t thread_init_kernel(struct thread *thr, virt_addr_t ip);
-extern kern_status_t thread_init_user(
-	struct thread *thr,
-	virt_addr_t ip,
-	virt_addr_t sp,
-	const uintptr_t *args,
-	size_t nr_args);
-extern int thread_priority(struct thread *thr);
-extern void thread_awaken(struct thread *thr);
-extern void idle(void);
-extern struct thread *create_kernel_thread(void (*fn)(void));
-extern struct thread *create_idle_thread(void);
-
 extern void add_timer(struct timer *timer);
 extern void remove_timer(struct timer *timer);
 extern unsigned long schedule_timeout(unsigned long clock_ticks);
 extern unsigned long milli_sleep(unsigned long ms);
 extern void sleep_forever(void);
 
-extern void wait_item_init(struct wait_item *item, struct thread *thr);
-extern void thread_wait_begin(struct wait_item *waiter, struct waitqueue *q);
-extern void thread_wait_end(struct wait_item *waiter, struct waitqueue *q);
-extern void wait_on_queue(struct waitqueue *q);
-extern void wakeup_queue(struct waitqueue *q);
-extern void wakeup_one(struct waitqueue *q);
-
-extern void work_item_init(work_func_t func, void *data, struct work_item *out);
-extern void workqueue_init(struct workqueue *wq);
-extern struct worker_pool *worker_pool_create(size_t nworkers);
-extern struct worker_pool *global_worker_pool(void);
-extern bool schedule_work_on(struct workqueue *wq, struct work_item *work);
-extern bool schedule_work(struct work_item *work);
-
-extern void wake_workers(struct workqueue *wq, struct worker_pool *pool);
-
 #ifdef __cplusplus
 }
 #endif

include/kernel/syscall.h

@@ -1,17 +1,15 @@
 #ifndef KERNEL_SYSCALL_H_
 #define KERNEL_SYSCALL_H_
 
+#include <kernel/address-space.h>
 #include <kernel/handle.h>
+#include <kernel/task.h>
 #include <kernel/vm.h>
 #include <mango/status.h>
 #include <mango/syscall.h>
 
 #define validate_access(task, ptr, len, flags)                                 \
-	vm_region_validate_access(                                             \
+	__validate_access(task, (const void *)ptr, len, flags)
-		task->t_address_space,                                         \
-		(virt_addr_t)ptr,                                              \
-		len,                                                           \
-		flags | VM_PROT_USER)
 #define validate_access_r(task, ptr, len)                                      \
 	validate_access(task, ptr, len, VM_PROT_READ | VM_PROT_USER)
 #define validate_access_w(task, ptr, len)                                      \
@@ -23,7 +21,25 @@
 		len,                                                           \
 		VM_PROT_READ | VM_PROT_WRITE | VM_PROT_USER)
 
+static inline bool __validate_access(
+	struct task *task,
+	const void *ptr,
+	size_t len,
+	vm_prot_t flags)
+{
+	unsigned long irq_flags;
+	address_space_lock_irqsave(task->t_address_space, &irq_flags);
+	bool result = address_space_validate_access(
+		task->t_address_space,
+		(virt_addr_t)ptr,
+		len,
+		flags | VM_PROT_USER);
+	address_space_unlock_irqrestore(task->t_address_space, irq_flags);
+	return result;
+}
+
 extern kern_status_t sys_task_exit(int status);
+extern kern_status_t sys_task_self(kern_handle_t *out);
 extern kern_status_t sys_task_create(
 	kern_handle_t parent_handle,
 	const char *name,
@@ -37,8 +53,33 @@ extern kern_status_t sys_task_create_thread(
 	uintptr_t *args,
 	size_t nr_args,
 	kern_handle_t *out_thread);
+extern kern_status_t sys_task_get_address_space(
+	kern_handle_t task,
+	kern_handle_t *out);
+extern kern_status_t sys_task_config_get(
+	kern_handle_t task,
+	kern_config_key_t key,
+	void *ptr,
+	size_t len);
+extern kern_status_t sys_task_config_set(
+	kern_handle_t task,
+	kern_config_key_t key,
+	const void *ptr,
+	size_t len);
 
+extern kern_status_t sys_thread_self(kern_handle_t *out);
 extern kern_status_t sys_thread_start(kern_handle_t thread);
+extern kern_status_t sys_thread_exit(void);
+extern kern_status_t sys_thread_config_get(
+	kern_handle_t thread,
+	kern_config_key_t key,
+	void *ptr,
+	size_t len);
+extern kern_status_t sys_thread_config_set(
+	kern_handle_t thread,
+	kern_config_key_t key,
+	const void *ptr,
+	size_t len);
 
 extern kern_status_t sys_vm_object_create(
 	const char *name,
@@ -66,28 +107,19 @@ extern kern_status_t sys_vm_object_copy(
 	size_t count,
 	size_t *nr_copied);
 
-extern kern_status_t sys_vm_region_create(
+extern kern_status_t sys_address_space_read(
-	kern_handle_t parent,
-	const char *name,
-	size_t name_len,
-	off_t offset,
-	size_t region_len,
-	vm_prot_t prot,
-	kern_handle_t *out,
-	virt_addr_t *out_base_address);
-extern kern_status_t sys_vm_region_read(
 	kern_handle_t region,
 	void *dst,
-	off_t offset,
+	virt_addr_t base,
 	size_t count,
 	size_t *nr_read);
-extern kern_status_t sys_vm_region_write(
+extern kern_status_t sys_address_space_write(
 	kern_handle_t region,
 	const void *src,
-	off_t offset,
+	virt_addr_t base,
 	size_t count,
 	size_t *nr_read);
-extern kern_status_t sys_vm_region_map_absolute(
+extern kern_status_t sys_address_space_map(
 	kern_handle_t region,
 	virt_addr_t map_address,
 	kern_handle_t object,
@@ -95,25 +127,25 @@ extern kern_status_t sys_vm_region_map_absolute(
 	size_t length,
 	vm_prot_t prot,
 	virt_addr_t *out_base_address);
-extern kern_status_t sys_vm_region_map_relative(
+extern kern_status_t sys_address_space_unmap(
 	kern_handle_t region,
-	off_t region_offset,
+	virt_addr_t base,
-	kern_handle_t object,
-	off_t object_offset,
-	size_t length,
-	vm_prot_t prot,
-	virt_addr_t *out_base_address);
-extern kern_status_t sys_vm_region_unmap_absolute(
-	kern_handle_t region,
-	virt_addr_t address,
 	size_t length);
-extern kern_status_t sys_vm_region_unmap_relative(
+extern kern_status_t sys_address_space_reserve(
 	kern_handle_t region,
-	off_t offset,
+	virt_addr_t base,
+	size_t length,
+	virt_addr_t *out_base_address);
+extern kern_status_t sys_address_space_release(
+	kern_handle_t region,
+	virt_addr_t base,
 	size_t length);
 
 extern kern_status_t sys_kern_log(const char *s);
 extern kern_status_t sys_kern_handle_close(kern_handle_t handle);
+extern kern_status_t sys_kern_handle_duplicate(
+	kern_handle_t handle,
+	kern_handle_t *out);
 extern kern_status_t sys_kern_config_get(
 	kern_config_key_t key,
 	void *ptr,
@@ -123,10 +155,7 @@ extern kern_status_t sys_kern_config_set(
 	const void *ptr,
 	size_t len);
 
-extern kern_status_t sys_channel_create(
+extern kern_status_t sys_channel_create(unsigned int id, kern_handle_t *out);
-	unsigned int id,
-	channel_flags_t flags,
-	kern_handle_t *out);
 extern kern_status_t sys_port_create(kern_handle_t *out);
 extern kern_status_t sys_port_connect(
 	kern_handle_t port,
@@ -136,47 +165,68 @@ extern kern_status_t sys_port_disconnect(kern_handle_t port);
 
 extern kern_status_t sys_msg_send(
 	kern_handle_t port,
-	msg_flags_t flags,
+	const kern_msg_t *msg,
-	const struct msg *req,
+	kern_msg_t *out_reply);
-	struct msg *resp);
+extern kern_status_t sys_msg_recv(kern_handle_t channel, kern_msg_t *out_msg);
-
-extern kern_status_t sys_msg_recv(
-	kern_handle_t channel,
-	msg_flags_t flags,
-	msgid_t *out_id,
-	struct msg *out_msg);
 
 extern kern_status_t sys_msg_reply(
 	kern_handle_t channel,
-	msg_flags_t flags,
 	msgid_t id,
-	const struct msg *reply);
+	const kern_msg_t *msg);
-
 extern kern_status_t sys_msg_read(
-	kern_handle_t channel,
+	kern_handle_t channel_handle,
 	msgid_t id,
 	size_t offset,
-	struct iovec *out,
+	const kern_iovec_t *iov,
-	size_t nr_out);
+	size_t iov_count,
-extern kern_status_t sys_msg_read_handles(
+	size_t *nr_read);
-	kern_handle_t channel,
-	msgid_t id,
-	size_t offset,
-	struct handle_list *out,
-	size_t nr_out);
-
 extern kern_status_t sys_msg_write(
 	kern_handle_t channel,
 	msgid_t id,
 	size_t offset,
-	const struct iovec *in,
+	const kern_iovec_t *in,
-	size_t nr_in);
+	size_t nr_in,
-extern kern_status_t sys_msg_write_handles(
+	size_t *nr_written);
-	kern_handle_t channel,
+
-	msgid_t id,
+extern kern_status_t sys_kern_object_wait(
-	size_t offset,
+	kern_wait_item_t *items,
-	const struct handle_list *in,
+	size_t nr_items);
-	size_t nr_in);
+
+extern kern_status_t sys_vm_controller_create(kern_handle_t *out);
+extern kern_status_t sys_vm_controller_recv(
+	kern_handle_t ctrl,
+	equeue_packet_page_request_t *out);
+extern kern_status_t sys_vm_controller_recv_async(
+	kern_handle_t ctrl,
+	kern_handle_t eq,
+	equeue_key_t key);
+extern kern_status_t sys_vm_controller_create_object(
+	kern_handle_t ctrl,
+	const char *name,
+	size_t name_len,
+	equeue_key_t key,
+	size_t data_len,
+	vm_prot_t prot,
+	kern_handle_t *out);
+extern kern_status_t sys_vm_controller_detach_object(
+	kern_handle_t ctrl,
+	kern_handle_t vmo);
+extern kern_status_t sys_vm_controller_supply_pages(
+	kern_handle_t ctrl,
+	kern_handle_t dst_vmo,
+	off_t dst_offset,
+	kern_handle_t src_vmo,
+	off_t src_offset,
+	size_t count);
+
+extern kern_status_t sys_futex_wait(
+	kern_futex_t *futex,
+	kern_futex_t new_val,
+	unsigned int flags);
+extern kern_status_t sys_futex_wake(
+	kern_futex_t *futex,
+	unsigned int nr_waiters,
+	unsigned int flags);
 
 extern virt_addr_t syscall_get_function(unsigned int sysid);
 

include/kernel/task.h

@@ -0,0 +1,76 @@
+#ifndef KERNEL_TASK_H_
+#define KERNEL_TASK_H_
+
+#include <kernel/handle.h>
+#include <kernel/object.h>
+#include <kernel/pmap.h>
+
+#define TASK_NAME_MAX 64
+#define PID_MAX       99999
+
+struct channel;
+
+enum task_state {
+	TASK_RUNNING,
+	TASK_STOPPED,
+};
+
+struct task {
+	struct object t_base;
+
+	struct task *t_parent;
+	long t_id;
+	enum task_state t_state;
+	char t_name[TASK_NAME_MAX];
+
+	pmap_t t_pmap;
+	struct address_space *t_address_space;
+	spin_lock_t t_handles_lock;
+	struct handle_table *t_handles;
+	struct btree t_channels, t_futex;
+
+	struct btree_node t_tasklist;
+	struct queue_entry t_child_entry;
+
+	size_t t_next_thread_id;
+	struct queue t_threads;
+	struct queue t_children;
+};
+
+extern struct task *task_alloc(void);
+extern struct task *task_cast(struct object *obj);
+extern struct task *task_create(const char *name, size_t name_len);
+static inline struct task *task_ref(struct task *task)
+{
+	return OBJECT_CAST(struct task, t_base, object_ref(&task->t_base));
+}
+static inline void task_unref(struct task *task)
+{
+	object_unref(&task->t_base);
+}
+extern void task_exit(int status);
+extern kern_status_t task_add_child(struct task *parent, struct task *child);
+extern kern_status_t task_add_channel(
+	struct task *task,
+	struct channel *channel,
+	unsigned int id);
+extern struct channel *task_get_channel(struct task *task, unsigned int id);
+extern struct task *task_from_tid(tid_t id);
+extern kern_status_t task_open_handle(
+	struct task *task,
+	struct object *obj,
+	handle_flags_t flags,
+	kern_handle_t *out);
+extern kern_status_t task_resolve_handle(
+	struct task *task,
+	kern_handle_t handle,
+	struct object **out_obj,
+	handle_flags_t *out_flags);
+extern kern_status_t task_close_handle(struct task *task, kern_handle_t handle);
+extern struct thread *task_create_thread(struct task *parent);
+extern struct task *kernel_task(void);
+extern struct task *idle_task(void);
+
+DEFINE_OBJECT_LOCK_FUNCTION(task, t_base)
+
+#endif

include/kernel/thread.h

@@ -0,0 +1,82 @@
+#ifndef KERNEL_THREAD_H_
+#define KERNEL_THREAD_H_
+
+#include <kernel/machine/thread.h>
+#include <kernel/msg.h>
+#include <kernel/object.h>
+#include <kernel/vm-controller.h>
+
+#define THREAD_KSTACK_ORDER VM_PAGE_4K
+
+enum thread_state {
+	THREAD_READY = 1,
+	THREAD_SLEEPING = 2,
+	THREAD_STOPPED = 3,
+};
+
+enum thread_flags {
+	/* this thread has exhausted its quantum and is due to be re-scheduled.
+	 */
+	THREAD_F_NEED_RESCHED = 0x01u,
+	/* this thread is currently scheduled (i.e. is present on a runqueue) */
+	THREAD_F_SCHEDULED = 0x04u,
+};
+
+struct thread {
+	struct object tr_base;
+
+	enum thread_state tr_state;
+	enum thread_flags tr_flags;
+	struct task *tr_parent;
+
+	unsigned int tr_id;
+	unsigned int tr_prio;
+
+	cycles_t tr_charge_period_start;
+	cycles_t tr_quantum_cycles, tr_quantum_target;
+	cycles_t tr_total_cycles;
+
+	virt_addr_t tr_ip, tr_sp, tr_bp;
+	virt_addr_t tr_cpu_user_sp, tr_cpu_kernel_sp;
+
+	struct ml_thread tr_ml;
+	struct runqueue *tr_rq;
+
+	struct queue_entry tr_parent_entry;
+	struct queue_entry tr_rqentry;
+
+	struct vm_page *tr_kstack;
+};
+
+extern struct thread *thread_alloc(void);
+extern struct thread *thread_cast(struct object *obj);
+extern kern_status_t thread_init_kernel(struct thread *thr, virt_addr_t ip);
+extern kern_status_t thread_init_user(
+	struct thread *thr,
+	virt_addr_t ip,
+	virt_addr_t sp,
+	const uintptr_t *args,
+	size_t nr_args);
+extern int thread_priority(struct thread *thr);
+extern void thread_awaken(struct thread *thr);
+extern void thread_exit(void);
+extern void thread_join(struct thread *thread, unsigned long *irq_flags);
+extern void thread_kill(struct thread *thread);
+extern void idle(void);
+extern struct thread *create_kernel_thread(void (*fn)(void));
+extern struct thread *create_idle_thread(void);
+
+extern kern_status_t thread_config_get(
+	struct thread *thread,
+	kern_config_key_t key,
+	void *out,
+	size_t max);
+extern kern_status_t thread_config_set(
+	struct thread *thread,
+	kern_config_key_t key,
+	const void *ptr,
+	size_t len);
+
+DEFINE_OBJECT_LOCK_FUNCTION(thread, tr_base)
+
+#endif

include/kernel/util.h

@@ -61,6 +61,7 @@ extern uint64_t host_to_little_u64(uint64_t v);
 extern uint64_t big_to_host_u64(uint64_t v);
 extern uint64_t little_to_host_u64(uint64_t v);
 
+extern void init_random(uint64_t seed);
 extern bool fill_random(void *buffer, unsigned int size);
 
 #ifdef __cplusplus

include/kernel/vm-controller.h

@@ -0,0 +1,83 @@
+#ifndef KERNEL_VM_CONTROLLER_H_
+#define KERNEL_VM_CONTROLLER_H_
+
+#include <kernel/locks.h>
+#include <kernel/object.h>
+#include <mango/types.h>
+
+struct thread;
+struct equeue;
+struct vm_object;
+
+enum page_request_status {
+	PAGE_REQUEST_PENDING = 0,
+	PAGE_REQUEST_IN_PROGRESS,
+	PAGE_REQUEST_COMPLETE,
+	PAGE_REQUEST_ASYNC,
+};
+
+struct vm_controller {
+	struct object vc_base;
+	/* tree of pending page requests */
+	struct btree vc_requests;
+	/* the equeue to send async page requests to */
+	struct equeue *vc_eq;
+	equeue_key_t vc_eq_key;
+	/* the number of page requests queued with status PAGE_REQUEST_PENDING.
+	 * used to assert/clear VM_CONTROLLER_SIGNAL_REQUEST_RECEIVED */
+	size_t vc_requests_waiting;
+};
+
+struct page_request {
+	uint64_t req_id;
+	unsigned int req_type;
+	enum page_request_status req_status;
+	kern_status_t req_result;
+	spin_lock_t req_lock;
+	equeue_key_t req_object;
+	struct thread *req_sender;
+	struct btree_node req_node;
+	off_t req_offset;
+	size_t req_length;
+};
+
+extern kern_status_t vm_controller_type_init(void);
+extern struct vm_controller *vm_controller_cast(struct object *obj);
+
+extern struct vm_controller *vm_controller_create(void);
+
+extern kern_status_t vm_controller_recv(
+	struct vm_controller *ctrl,
+	equeue_packet_page_request_t *out);
+extern kern_status_t vm_controller_recv_async(
+	struct vm_controller *ctrl,
+	struct equeue *eq,
+	equeue_key_t key);
+
+extern kern_status_t vm_controller_create_object(
+	struct vm_controller *ctrl,
+	const char *name,
+	size_t name_len,
+	equeue_key_t key,
+	size_t data_len,
+	vm_prot_t prot,
+	struct vm_object **out);
+extern kern_status_t vm_controller_detach_object(
+	struct vm_controller *ctrl,
+	struct vm_object *vmo);
+extern kern_status_t vm_controller_supply_pages(
+	struct vm_controller *ctrl,
+	struct vm_object *dst,
+	off_t dst_offset,
+	struct vm_object *src,
+	off_t src_offset,
+	size_t count);
+
+extern kern_status_t vm_controller_send_request(
+	struct vm_controller *ctrl,
+	struct page_request *req,
+	unsigned long *irq_flags);
+
+DEFINE_OBJECT_LOCK_FUNCTION(vm_controller, vc_base)
+
+#endif

include/kernel/vm-object.h

@@ -6,10 +6,33 @@
 
 #define VM_OBJECT_NAME_MAX 64
 
+struct vm_controller;
+
 enum vm_object_flags {
 	/* the memory behind this vm-object wasn't allocated by us, and
 	 * therefore shouldn't be freed by us */
 	VMO_IN_PLACE = 0x01u,
+	/* this vm-object is/was attached to a vm-controller */
+	VMO_CONTROLLER = 0x02u,
+	/* when a vm-object is attached to a controller, and the ref-count of
+	 * the object falls to one (i.e. the last reference is the handle to
+	 * the object held by the server that created it), the object will
+	 * be detached, allowing the server to close the last handle to the
+	 * object and dispose of it. */
+	VMO_AUTO_DETACH = 0x04u,
+
+	/* these flags are for use with vm_object_get_page */
+	/**************************************************/
+
+	/* if the relevant page hasn't been allocated yet, it will be allocated
+	 * and returned. if this flag isn't specified, NULL will be returned. */
+	VMO_ALLOCATE_MISSING_PAGE = 0x08u,
+	/* if the vm-object is attached to a vm-controller, and the relevant
+	 * page is uncommitted, send a request to the vm-controller to provide
+	 * the missing page. will result in the vm-object being unlocked and
+	 * the current thread sleeping until the request is fulfilled. the
+	 * vm-object will be re-locked before the function returns. */
+	VMO_REQUEST_MISSING_PAGE = 0x10u,
 };
 
 struct vm_object {
@@ -21,8 +44,12 @@ struct vm_object {
 	/* queue of struct vm_region_mapping */
 	struct queue vo_mappings;
 
-	/* memory protection flags. mappings of this vm_object can only use
+	struct vm_controller *vo_ctrl;
-	 * a subset of the flags set in this mask. */
+	equeue_key_t vo_key;
+	struct btree_node vo_ctrl_node;
+
+	/* memory protection flags. mappings of this vm_object can only
+	 * use a subset of the flags set in this mask. */
 	vm_prot_t vo_prot;
 
 	/* btree of vm_pages that have been allocated to this vm_object.
@@ -58,13 +85,10 @@ extern struct vm_object *vm_object_create_in_place(
 	vm_prot_t prot);
 
 extern struct vm_page *vm_object_get_page(
-	const struct vm_object *vo,
-	off_t offset);
-
-extern struct vm_page *vm_object_alloc_page(
 	struct vm_object *vo,
 	off_t offset,
-	enum vm_page_order size);
+	enum vm_object_flags flags,
+	unsigned long *irq_flags);
 
 extern kern_status_t vm_object_read(
 	struct vm_object *vo,
@@ -85,6 +109,13 @@ extern kern_status_t vm_object_copy(
 	off_t src_offset,
 	size_t count,
 	size_t *nr_copied);
+extern kern_status_t vm_object_transfer(
+	struct vm_object *dst,
+	off_t dst_offset,
+	struct vm_object *src,
+	off_t src_offset,
+	size_t count,
+	size_t *nr_moved);
 
 DEFINE_OBJECT_LOCK_FUNCTION(vm_object, vo_base)
 

include/kernel/vm-region.h

@@ -1,146 +0,0 @@
-#ifndef KERNEL_VM_REGION_H_
-#define KERNEL_VM_REGION_H_
-
-#include <kernel/object.h>
-#include <kernel/pmap.h>
-#include <kernel/vm.h>
-
-#define VM_REGION_NAME_MAX 64
-#define VM_REGION_COPY_ALL ((size_t)-1)
-
-struct vm_region;
-struct vm_object;
-
-enum vm_region_entry_type {
-	VM_REGION_ENTRY_NONE = 0,
-	VM_REGION_ENTRY_REGION,
-	VM_REGION_ENTRY_MAPPING,
-};
-
-struct vm_region_entry {
-	struct btree_node e_node;
-	struct vm_region_entry *e_parent;
-	enum vm_region_entry_type e_type;
-	/* offset in bytes of this entry within its immediate parent. */
-	off_t e_offset;
-	/* size of the entry in bytes */
-	size_t e_size;
-};
-
-struct vm_region_mapping {
-	struct vm_region_entry m_entry;
-	struct vm_object *m_object;
-
-	/* used to link to  vm_object->vo_mappings */
-	struct queue_entry m_object_entry;
-
-	vm_prot_t m_prot;
-	/* offset in bytes to the start of the object data that was mapped */
-	off_t m_object_offset;
-};
-
-struct vm_region {
-	struct object vr_base;
-	struct vm_region_entry vr_entry;
-
-	char vr_name[VM_REGION_NAME_MAX];
-
-	/* btree of struct vm_region_entry.
-	 * sibling entries cannot overlap each other, and child entries must
-	 * be entirely contained within their immediate parent entry. */
-	struct btree vr_entries;
-
-	/* memory protection restriction mask.
-	 * any mapping in this region, or any of its children, cannot use
-	 * protection flags that are not set in this mask.
-	 * for example, if VM_PROT_EXEC is /not/ set here, no mapping
-	 * can be created in this region or any child region with VM_PROT_EXEC
-	 * set. */
-	vm_prot_t vr_prot;
-
-	/* the physical address space in which mappings in this region (and
-	 * its children) are created */
-	pmap_t vr_pmap;
-};
-
-extern kern_status_t vm_region_type_init(void);
-extern struct vm_region *vm_region_cast(struct object *obj);
-
-/* create a new vm-region, optionally within a parent region.
- * `offset` is the byte offset within the parent region where the new region
- * should start.
- * if no parent is specified, `offset` is the absolute virtual address of the
- * start of the region.
- * in both cases, `len` is the length of the new region in bytes. */
-extern kern_status_t vm_region_create(
-	struct vm_region *parent,
-	const char *name,
-	size_t name_len,
-	off_t offset,
-	size_t region_len,
-	vm_prot_t prot,
-	struct vm_region **out);
-
-/* map a vm-object into a vm-region.
- * [region_offset,length] must fall within exactly one region, and cannot span
- * multiple sibling regions.
- * if [region_offset,length] falls within a child region, the map operation
- * will be transparently redirected to the relevant region.
- * `prot` must be allowed both by the region into which the mapping is being
- * created AND the vm-object being mapped. */
-extern kern_status_t vm_region_map_object(
-	struct vm_region *region,
-	off_t region_offset,
-	struct vm_object *object,
-	off_t object_offset,
-	size_t length,
-	vm_prot_t prot,
-	virt_addr_t *out);
-
-extern kern_status_t vm_region_unmap(
-	struct vm_region *region,
-	off_t region_offset,
-	size_t length);
-
-extern bool vm_region_validate_access(
-	struct vm_region *region,
-	off_t offset,
-	size_t len,
-	vm_prot_t prot);
-
-/* find the mapping corresponding to the given virtual address, and page-in the
- * necessary vm_page to allow the memory access to succeed. if the relevant
- * vm-object page hasn't been allocated yet, it will be allocated here. */
-extern kern_status_t vm_region_demand_map(
-	struct vm_region *region,
-	virt_addr_t addr,
-	enum pmap_fault_flags flags);
-
-/* get the absolute base virtual address of a region within its
- * parent/ancestors. */
-extern virt_addr_t vm_region_get_base_address(const struct vm_region *region);
-
-extern void vm_region_dump(struct vm_region *region);
-
-extern kern_status_t vm_region_memmove(
-	struct vm_region *dest_region,
-	virt_addr_t dest_ptr,
-	struct vm_region *src_region,
-	virt_addr_t src_ptr,
-	size_t count,
-	size_t *nr_moved);
-
-extern kern_status_t vm_region_memmove_v(
-	struct vm_region *dest_region,
-	size_t dest_offset,
-	struct iovec *dest,
-	size_t nr_dest,
-	struct vm_region *src_region,
-	size_t src_offset,
-	const struct iovec *src,
-	size_t nr_src,
-	size_t bytes_to_move);
-
-DEFINE_OBJECT_LOCK_FUNCTION(vm_region, vr_base)
-
-#endif

include/kernel/wait.h

@@ -0,0 +1,54 @@
+#ifndef KERNEL_WAIT_H_
+#define KERNEL_WAIT_H_
+
+#include <kernel/locks.h>
+#include <kernel/queue.h>
+
+#define wait_event(wq, cond)                                                   \
+	({                                                                     \
+		struct thread *self = current_thread();                        \
+		struct wait_item waiter;                                       \
+		wait_item_init(&waiter, self);                                 \
+		for (;;) {                                                     \
+			thread_wait_begin(&waiter, wq);                        \
+			if (cond) {                                            \
+				break;                                         \
+			}                                                      \
+			schedule(SCHED_NORMAL);                                \
+		}                                                              \
+		thread_wait_end(&waiter, wq);                                  \
+	})
+
+struct wait_item {
+	struct thread *w_thread;
+	struct queue_entry w_entry;
+};
+
+struct waitqueue {
+	struct queue wq_waiters;
+	spin_lock_t wq_lock;
+};
+
+extern void wait_item_init(struct wait_item *item, struct thread *thr);
+extern void thread_wait_begin(struct wait_item *waiter, struct waitqueue *q);
+extern void thread_wait_end(struct wait_item *waiter, struct waitqueue *q);
+extern void thread_wait_begin_nosleep(
+	struct wait_item *waiter,
+	struct waitqueue *q);
+extern void thread_wait_end_nosleep(
+	struct wait_item *waiter,
+	struct waitqueue *q);
+extern void wait_on_queue(struct waitqueue *q);
+extern void wakeup_queue(struct waitqueue *q);
+extern void wakeup_n(struct waitqueue *q, size_t n);
+extern void wakeup_one(struct waitqueue *q);
+static inline bool waitqueue_empty(struct waitqueue *wq)
+{
+	unsigned long flags;
+	spin_lock_irqsave(&wq->wq_lock, &flags);
+	bool result = queue_empty(&wq->wq_waiters);
+	spin_unlock_irqrestore(&wq->wq_lock, flags);
+	return result;
+}
+
+#endif

include/kernel/work.h

@@ -0,0 +1,37 @@
+#ifndef KERNEL_WORK_H_
+#define KERNEL_WORK_H_
+
+#include <kernel/locks.h>
+#include <kernel/queue.h>
+#include <stddef.h>
+
+struct work_item;
+
+typedef void (*work_func_t)(struct work_item *);
+
+struct work_item {
+	void *w_data;
+	work_func_t w_func;
+	struct queue_entry w_head;
+};
+
+struct worker_pool {
+	struct thread **wp_workers;
+	size_t wp_nworkers;
+};
+
+struct workqueue {
+	spin_lock_t wq_lock;
+	struct queue wq_queue; /* list of struct work_item */
+};
+
+extern void work_item_init(work_func_t func, void *data, struct work_item *out);
+extern void workqueue_init(struct workqueue *wq);
+extern struct worker_pool *worker_pool_create(size_t nworkers);
+extern struct worker_pool *global_worker_pool(void);
+extern bool schedule_work_on(struct workqueue *wq, struct work_item *work);
+extern bool schedule_work(struct work_item *work);
+
+extern void wake_workers(struct workqueue *wq, struct worker_pool *pool);
+
+#endif

init/main.c

@@ -13,7 +13,9 @@
 #include <kernel/port.h>
 #include <kernel/printk.h>
 #include <kernel/sched.h>
+#include <kernel/task.h>
 #include <kernel/test.h>
+#include <kernel/thread.h>
 #include <kernel/vm-object.h>
 #include <stdint.h>
 
@@ -76,6 +78,7 @@ void kernel_init(uintptr_t arg)
 
 	port_type_init();
 	channel_type_init();
+	msg_init();
 
 	struct boot_module bsp_image = {0};
 	bsp_get_location(&bsp_image);
@@ -110,7 +113,10 @@ void kernel_init(uintptr_t arg)
 	struct task *bootstrap_task = task_create("bootstrap", 9);
 	tracek("created bootstrap task (pid=%u)", bootstrap_task->t_id);
 
-	bsp_launch_async(&bsp, bootstrap_task);
+	status = bsp_launch_async(&bsp, bootstrap_task);
+	if (status != KERN_OK) {
+		printk("bsp launch failed with status %d", status);
+	}
 
 	hang();
 }

kernel/arg.c

@@ -1,6 +1,7 @@
 #include <kernel/arg.h>
-#include <kernel/libc/string.h>
 #include <kernel/libc/ctype.h>
+#include <kernel/libc/string.h>
+#include <mango/status.h>
 
 static char g_cmdline[CMDLINE_MAX + 1] = {0};
 
@@ -81,7 +82,6 @@ static char *advance_to_next_arg(char *s, char *max)
 	return s;
 }
 
-
 const char *arg_value(const char *arg_name)
 {
 	char *s = g_cmdline;

kernel/bsp.c

@@ -1,10 +1,12 @@
+#include <kernel/address-space.h>
 #include <kernel/bsp.h>
 #include <kernel/handle.h>
 #include <kernel/printk.h>
 #include <kernel/sched.h>
+#include <kernel/task.h>
+#include <kernel/thread.h>
 #include <kernel/util.h>
 #include <kernel/vm-object.h>
-#include <kernel/vm-region.h>
 
 #define BOOTSTRAP_STACK_SIZE 0x10000
 
@@ -69,34 +71,12 @@ kern_status_t bsp_load(struct bsp *bsp, const struct boot_module *mod)
 	return KERN_OK;
 }
 
-static kern_status_t map_executable(
+static kern_status_t map_executable_exec(
 	struct bsp *bsp,
 	struct task *task,
 	virt_addr_t *entry)
 {
 	kern_status_t status = KERN_OK;
-	size_t exec_size = 0;
-	if (bsp->bsp_trailer.bsp_text_vaddr > bsp->bsp_trailer.bsp_data_vaddr) {
-		exec_size = bsp->bsp_trailer.bsp_text_vaddr
-		          + bsp->bsp_trailer.bsp_text_size;
-	} else {
-		exec_size = bsp->bsp_trailer.bsp_data_vaddr
-		          + bsp->bsp_trailer.bsp_data_size;
-	}
-
-	struct vm_region *region;
-	status = vm_region_create(
-		task->t_address_space,
-		"exec",
-		4,
-		VM_REGION_ANY_OFFSET,
-		exec_size,
-		VM_PROT_READ | VM_PROT_WRITE | VM_PROT_EXEC | VM_PROT_USER,
-		&region);
-	if (status != KERN_OK) {
-		return status;
-	}
-
 	struct vm_object *data = vm_object_create(
 		".data",
 		5,
@@ -134,8 +114,8 @@ static kern_status_t map_executable(
 	       text_voffset,
 	       data_voffset);
 
-	status = vm_region_map_object(
+	status = address_space_map(
-		region,
+		task->t_address_space,
 		text_voffset,
 		bsp->bsp_vmo,
 		text_foffset,
@@ -146,8 +126,8 @@ static kern_status_t map_executable(
 		return status;
 	}
 
-	status = vm_region_map_object(
+	status = address_space_map(
-		region,
+		task->t_address_space,
 		data_voffset,
 		data,
 		data_foffset,
@@ -160,7 +140,7 @@ static kern_status_t map_executable(
 
 	tracek("text_base=%08llx, data_base=%08llx", text_base, data_base);
 
-	*entry = text_base + bsp->bsp_trailer.bsp_exec_entry;
+	*entry = bsp->bsp_trailer.bsp_exec_entry;
 	return KERN_OK;
 }
 
@@ -179,9 +159,9 @@ kern_status_t bsp_launch_async(struct bsp *bsp, struct task *task)
 		return KERN_NO_ENTRY;
 	}
 
-	status = vm_region_map_object(
+	status = address_space_map(
 		task->t_address_space,
-		VM_REGION_ANY_OFFSET,
+		MAP_ADDRESS_ANY,
 		user_stack,
 		0,
 		BOOTSTRAP_STACK_SIZE,
@@ -192,9 +172,9 @@ kern_status_t bsp_launch_async(struct bsp *bsp, struct task *task)
 		return status;
 	}
 
-	status = vm_region_map_object(
+	status = address_space_map(
 		task->t_address_space,
-		VM_REGION_ANY_OFFSET,
+		MAP_ADDRESS_ANY,
 		bsp->bsp_vmo,
 		0,
 		bsp->bsp_trailer.bsp_exec_offset,
@@ -205,12 +185,12 @@ kern_status_t bsp_launch_async(struct bsp *bsp, struct task *task)
 		return status;
 	}
 
-	status = map_executable(bsp, task, &entry);
+	status = map_executable_exec(bsp, task, &entry);
 	if (status != KERN_OK) {
 		return status;
 	}
 #ifdef TRACE
-	vm_region_dump(task->t_address_space);
+	address_space_dump(task->t_address_space);
 #endif
 
 	sp = stack_buffer + BOOTSTRAP_STACK_SIZE;
@@ -220,7 +200,7 @@ kern_status_t bsp_launch_async(struct bsp *bsp, struct task *task)
 	task_open_handle(task, &task->t_base, 0, &self);
 	task_open_handle(
 		task,
-		&task->t_address_space->vr_base,
+		&task->t_address_space->s_base,
 		0,
 		&self_address_space);
 

kernel/channel.c

@@ -1,7 +1,11 @@
+#include <kernel/address-space.h>
 #include <kernel/channel.h>
 #include <kernel/msg.h>
+#include <kernel/port.h>
+#include <kernel/task.h>
+#include <kernel/thread.h>
 #include <kernel/util.h>
-#include <kernel/vm-region.h>
+#include <mango/signal.h>
 
 #define CHANNEL_CAST(p) OBJECT_C_CAST(struct channel, c_base, &channel_type, p)
 
@@ -11,13 +15,18 @@ static struct object_type channel_type = {
 	.ob_header_offset = offsetof(struct channel, c_base),
 };
 
-BTREE_DEFINE_SIMPLE_GET(struct kmsg, msgid_t, msg_node, msg_id, get_msg_with_id)
+BTREE_DEFINE_SIMPLE_GET(struct msg, msgid_t, msg_node, msg_id, get_msg_with_id)
 
 kern_status_t channel_type_init(void)
 {
 	return object_type_register(&channel_type);
 }
 
+struct channel *channel_cast(struct object *obj)
+{
+	return CHANNEL_CAST(obj);
+}
+
 extern struct channel *channel_create(void)
 {
 	struct object *channel_object = object_create(&channel_type);
@@ -30,7 +39,7 @@ extern struct channel *channel_create(void)
 	return channel;
 }
 
-static bool try_enqueue(struct btree *tree, struct kmsg *msg)
+static bool try_enqueue(struct btree *tree, struct msg *msg)
 {
 	if (!tree->b_root) {
 		tree->b_root = &msg->msg_node;
@@ -40,8 +49,8 @@ static bool try_enqueue(struct btree *tree, struct kmsg *msg)
 
 	struct btree_node *cur = tree->b_root;
 	while (1) {
-		struct kmsg *cur_node
+		struct msg *cur_node
-			= BTREE_CONTAINER(struct kmsg, msg_node, cur);
+			= BTREE_CONTAINER(struct msg, msg_node, cur);
 		struct btree_node *next = NULL;
 
 		if (msg->msg_id > cur_node->msg_id) {
@@ -69,26 +78,41 @@ static bool try_enqueue(struct btree *tree, struct kmsg *msg)
 	return true;
 }
 
-static void kmsg_reply_error(struct kmsg *msg, kern_status_t status)
+static void kmsg_reply_error(
+	struct msg *msg,
+	kern_status_t status,
+	unsigned long *lock_flags)
 {
 	msg->msg_status = KMSG_REPLY_SENT;
-	msg->msg_status = status;
+	msg->msg_sender_port->p_status = PORT_READY;
+	msg->msg_result = status;
 	thread_awaken(msg->msg_sender_thread);
+	spin_unlock_irqrestore(&msg->msg_lock, *lock_flags);
 }
 
-static struct kmsg *get_next_msg(struct channel *channel)
+static struct msg *get_next_msg(
+	struct channel *channel,
+	unsigned long *lock_flags)
 {
-	unsigned long flags;
 	struct btree_node *cur = btree_first(&channel->c_msg);
 	while (cur) {
-		struct kmsg *msg = BTREE_CONTAINER(struct kmsg, msg_node, cur);
+		struct msg *msg = BTREE_CONTAINER(struct msg, msg_node, cur);
-		spin_lock_irqsave(&msg->msg_lock, &flags);
+		spin_lock_irqsave(&msg->msg_lock, lock_flags);
-		if (msg->msg_status == KMSG_WAIT_RECEIVE) {
+		switch (msg->msg_status) {
+		case KMSG_WAIT_RECEIVE:
 			msg->msg_status = KMSG_WAIT_REPLY;
+			msg->msg_sender_port->p_status = PORT_REPLY_BLOCKED;
+			channel->c_msg_waiting--;
 			return msg;
+		case KMSG_ASYNC:
+			btree_delete(&channel->c_msg, &msg->msg_node);
+			channel->c_msg_waiting--;
+			return msg;
+		default:
+			break;
 		}
 
-		spin_unlock_irqrestore(&msg->msg_lock, flags);
+		spin_unlock_irqrestore(&msg->msg_lock, *lock_flags);
 		cur = btree_next(cur);
 	}
 
@@ -97,144 +121,281 @@ static struct kmsg *get_next_msg(struct channel *channel)
 
 extern kern_status_t channel_enqueue_msg(
 	struct channel *channel,
-	struct kmsg *msg)
+	struct msg *msg)
 {
 	fill_random(&msg->msg_id, sizeof msg->msg_id);
 	while (!try_enqueue(&channel->c_msg, msg)) {
 		msg->msg_id++;
 	}
 
-	wakeup_one(&channel->c_wq);
+	channel->c_msg_waiting++;
+	object_assert_signal(&channel->c_base, CHANNEL_SIGNAL_MSG_RECEIVED);
 
 	return KERN_OK;
 }
 
 extern kern_status_t channel_recv_msg(
 	struct channel *channel,
-	struct msg *out_msg,
+	kern_msg_t *out_msg,
-	msgid_t *out_id,
 	unsigned long *irq_flags)
 {
-	struct wait_item waiter;
 	struct thread *self = current_thread();
-	struct kmsg *msg = NULL;
+	struct msg *msg = NULL;
+	unsigned long msg_lock_flags;
 
-	wait_item_init(&waiter, self);
+	msg = get_next_msg(channel, &msg_lock_flags);
-	for (;;) {
+	if (!msg) {
-		thread_wait_begin(&waiter, &channel->c_wq);
+		return KERN_NO_ENTRY;
-		msg = get_next_msg(channel);
+	}
-		if (msg) {
+
-			break;
+	if (channel->c_msg_waiting == 0) {
-		}
+		object_clear_signal(
-
+			&channel->c_base,
-		object_unlock_irqrestore(&channel->c_base, *irq_flags);
+			CHANNEL_SIGNAL_MSG_RECEIVED);
-		schedule(SCHED_NORMAL);
-		object_lock_irqsave(&channel->c_base, irq_flags);
 	}
-	thread_wait_end(&waiter, &channel->c_wq);
 
 	/* msg is now set to the next message to process */
 
+	if (msg->msg_type != KERN_MSG_TYPE_DATA) {
+		/* event messages as asynchronous */
+		out_msg->msg_id = msg->msg_id;
+		out_msg->msg_type = msg->msg_type;
+		out_msg->msg_event = msg->msg_event;
+		out_msg->msg_sender = msg->msg_sender_thread->tr_parent->t_id;
+		out_msg->msg_endpoint = msg->msg_sender_port->p_base.ob_id;
+		spin_unlock_irqrestore(&msg->msg_lock, msg_lock_flags);
+		msg_free(msg);
+
+		return KERN_OK;
+	}
+
 	struct task *sender = msg->msg_sender_thread->tr_parent;
 	struct task *receiver = self->tr_parent;
 
-	kern_status_t status = vm_region_memmove_v(
+	struct address_space *src = sender->t_address_space,
-		receiver->t_address_space,
+			     *dst = receiver->t_address_space;
+
+	unsigned long f;
+	address_space_lock_pair_irqsave(src, dst, &f);
+
+	kern_status_t status = address_space_memmove_v(
+		dst,
 		0,
 		out_msg->msg_data,
 		out_msg->msg_data_count,
-		sender->t_address_space,
+		src,
 		0,
-		msg->msg_req->msg_data,
+		msg->msg_req.msg_data,
-		msg->msg_req->msg_data_count,
+		msg->msg_req.msg_data_count,
-		VM_REGION_COPY_ALL);
+		ADDRESS_SPACE_COPY_ALL,
+		NULL);
+
 	if (status != KERN_OK) {
-		kmsg_reply_error(msg, status);
+		kmsg_reply_error(msg, status, &msg_lock_flags);
 		return status;
 	}
 
-	status = handle_list_transfer(
+	struct handle_table *src_table = sender->t_handles,
-		receiver->t_handles,
+			    *dst_table = receiver->t_handles;
+
+	spin_lock_pair_irqsave(
+		&sender->t_handles_lock,
+		&receiver->t_handles_lock,
+		&f);
+	status = handle_table_transfer(
+		dst,
+		dst_table,
 		out_msg->msg_handles,
 		out_msg->msg_handles_count,
-		sender->t_handles,
+		src,
-		msg->msg_req->msg_handles,
+		src_table,
-		msg->msg_req->msg_handles_count);
+		msg->msg_req.msg_handles,
+		msg->msg_req.msg_handles_count);
+	spin_unlock_pair_irqrestore(
+		&sender->t_handles_lock,
+		&receiver->t_handles_lock,
+		f);
+	address_space_unlock_pair_irqrestore(src, dst, f);
+
 	if (status != KERN_OK) {
-		kmsg_reply_error(msg, status);
+		kmsg_reply_error(msg, status, &msg_lock_flags);
 		return status;
 	}
 
-	kmsg_reply_error(msg, KERN_OK);
+	out_msg->msg_id = msg->msg_id;
+	out_msg->msg_type = msg->msg_type;
+	out_msg->msg_sender = msg->msg_sender_thread->tr_parent->t_id;
+	out_msg->msg_endpoint = msg->msg_sender_port->p_base.ob_id;
+
+	spin_unlock_irqrestore(&msg->msg_lock, msg_lock_flags);
+
 	return KERN_OK;
 }
 
 extern kern_status_t channel_reply_msg(
 	struct channel *channel,
 	msgid_t id,
-	const struct msg *resp,
+	const kern_msg_t *reply,
 	unsigned long *irq_flags)
 {
-	struct kmsg *msg = get_msg_with_id(&channel->c_msg, id);
+	unsigned long msg_lock_flags;
-	if (!msg || msg->msg_status != KMSG_WAIT_REPLY) {
+	struct msg *msg = get_msg_with_id(&channel->c_msg, id);
+	if (!msg) {
+		return KERN_INVALID_ARGUMENT;
+	}
+
+	spin_lock_irqsave(&msg->msg_lock, &msg_lock_flags);
+	if (msg->msg_status != KMSG_WAIT_REPLY) {
+		spin_unlock_irqrestore(&msg->msg_lock, msg_lock_flags);
 		return KERN_INVALID_ARGUMENT;
 	}
 
 	struct thread *self = current_thread();
-	struct task *sender = msg->msg_sender_thread->tr_parent;
+	/* the task that is about to receive the response */
-	struct task *receiver = self->tr_parent;
+	struct task *receiver = msg->msg_sender_thread->tr_parent;
+	/* the task that is about to send the response */
+	struct task *sender = self->tr_parent;
 
-	kern_status_t status = vm_region_memmove_v(
+	struct address_space *src = sender->t_address_space,
-		receiver->t_address_space,
+			     *dst = receiver->t_address_space;
+	unsigned long f;
+	address_space_lock_pair_irqsave(src, dst, &f);
+
+	kern_status_t status = address_space_memmove_v(
+		dst,
 		0,
-		msg->msg_resp->msg_data,
+		msg->msg_resp.msg_data,
-		msg->msg_resp->msg_data_count,
+		msg->msg_resp.msg_data_count,
-		sender->t_address_space,
+		src,
 		0,
-		resp->msg_data,
+		reply->msg_data,
-		resp->msg_data_count,
+		reply->msg_data_count,
-		VM_REGION_COPY_ALL);
+		ADDRESS_SPACE_COPY_ALL,
+		NULL);
+
 	if (status != KERN_OK) {
-		kmsg_reply_error(msg, status);
+		kmsg_reply_error(msg, status, &msg_lock_flags);
 		return status;
 	}
 
-	status = handle_list_transfer(
+	struct handle_table *src_table = sender->t_handles,
-		receiver->t_handles,
+			    *dst_table = receiver->t_handles;
-		msg->msg_resp->msg_handles,
+
-		msg->msg_resp->msg_handles_count,
+	spin_lock_pair_irqsave(
-		sender->t_handles,
+		&sender->t_handles_lock,
-		resp->msg_handles,
+		&receiver->t_handles_lock,
-		resp->msg_handles_count);
+		&f);
+	status = handle_table_transfer(
+		dst,
+		dst_table,
+		msg->msg_resp.msg_handles,
+		msg->msg_resp.msg_handles_count,
+		src,
+		src_table,
+		reply->msg_handles,
+		reply->msg_handles_count);
+	spin_unlock_pair_irqrestore(
+		&sender->t_handles_lock,
+		&receiver->t_handles_lock,
+		f);
+	address_space_unlock_pair_irqrestore(src, dst, f);
+
 	if (status != KERN_OK) {
-		kmsg_reply_error(msg, status);
+		kmsg_reply_error(msg, status, &msg_lock_flags);
 		return status;
 	}
 
-	msg->msg_status = KERN_OK;
+	kmsg_reply_error(msg, KERN_OK, &msg_lock_flags);
-	msg->msg_status = KMSG_REPLY_SENT;
 
-	return KERN_UNIMPLEMENTED;
+	return KERN_OK;
 }
 
 extern kern_status_t channel_read_msg(
 	struct channel *channel,
-	msgid_t msg,
+	msgid_t id,
 	size_t offset,
-	void *buf,
+	struct address_space *dest_region,
-	size_t len,
+	const kern_iovec_t *dest_iov,
+	size_t dest_iov_count,
 	size_t *nr_read)
 {
-	return KERN_UNIMPLEMENTED;
+	unsigned long msg_lock_flags;
+	struct msg *msg = get_msg_with_id(&channel->c_msg, id);
+	if (!msg) {
+		return KERN_INVALID_ARGUMENT;
+	}
+
+	spin_lock_irqsave(&msg->msg_lock, &msg_lock_flags);
+	if (msg->msg_status != KMSG_WAIT_REPLY) {
+		spin_unlock_irqrestore(&msg->msg_lock, msg_lock_flags);
+		return KERN_INVALID_ARGUMENT;
+	}
+
+	struct address_space *src_region
+		= msg->msg_sender_thread->tr_parent->t_address_space;
+
+	unsigned long f;
+	address_space_lock_pair_irqsave(src_region, dest_region, &f);
+
+	kern_status_t status = address_space_memmove_v(
+		dest_region,
+		0,
+		dest_iov,
+		dest_iov_count,
+		src_region,
+		offset,
+		msg->msg_req.msg_data,
+		msg->msg_req.msg_data_count,
+		ADDRESS_SPACE_COPY_ALL,
+		nr_read);
+	address_space_unlock_pair_irqrestore(src_region, dest_region, f);
+
+	spin_unlock_irqrestore(&msg->msg_lock, msg_lock_flags);
+
+	return status;
 }
 
 extern kern_status_t channel_write_msg(
 	struct channel *channel,
-	msgid_t msg,
+	msgid_t id,
 	size_t offset,
-	const void *buf,
+	struct address_space *src_region,
-	size_t len,
+	const kern_iovec_t *src_iov,
+	size_t src_iov_count,
 	size_t *nr_written)
 {
-	return KERN_UNIMPLEMENTED;
+	unsigned long msg_lock_flags;
+	struct msg *msg = get_msg_with_id(&channel->c_msg, id);
+	if (!msg) {
+		return KERN_INVALID_ARGUMENT;
+	}
+
+	spin_lock_irqsave(&msg->msg_lock, &msg_lock_flags);
+	if (msg->msg_status != KMSG_WAIT_REPLY) {
+		spin_unlock_irqrestore(&msg->msg_lock, msg_lock_flags);
+		return KERN_INVALID_ARGUMENT;
+	}
+
+	struct address_space *dest_region
+		= msg->msg_sender_thread->tr_parent->t_address_space;
+
+	unsigned long f;
+	address_space_lock_pair_irqsave(src_region, dest_region, &f);
+
+	kern_status_t status = address_space_memmove_v(
+		dest_region,
+		offset,
+		msg->msg_resp.msg_data,
+		msg->msg_resp.msg_data_count,
+		src_region,
+		0,
+		src_iov,
+		src_iov_count,
+		ADDRESS_SPACE_COPY_ALL,
+		nr_written);
+	address_space_unlock_pair_irqrestore(src_region, dest_region, f);
+
+	spin_unlock_irqrestore(&msg->msg_lock, msg_lock_flags);
+
+	return status;
 }

kernel/console.c

@@ -1,9 +1,9 @@
 #include <kernel/console.h>
-#include <kernel/queue.h>
-#include <kernel/locks.h>
 #include <kernel/libc/string.h>
+#include <kernel/locks.h>
+#include <kernel/queue.h>
 
-static struct queue consoles;
+static struct queue consoles = {0};
 static spin_lock_t consoles_lock = SPIN_LOCK_INIT;
 
 static void unregister_boot_consoles(void)
@@ -11,7 +11,8 @@ static void unregister_boot_consoles(void)
 	struct queue_entry *cur = queue_first(&consoles);
 	while (cur) {
 		struct queue_entry *next = cur->qe_next;
-		struct console *con = QUEUE_CONTAINER(struct console, c_list, cur);
+		struct console *con
+			= QUEUE_CONTAINER(struct console, c_list, cur);
 		if (con->c_flags & CON_BOOT) {
 			queue_delete(&consoles, cur);
 		}
@@ -25,7 +26,8 @@ kern_status_t console_register(struct console *con)
 	unsigned long flags;
 	spin_lock_irqsave(&consoles_lock, &flags);
 
-	queue_foreach (struct console, cur, &consoles, c_list) {
+	queue_foreach(struct console, cur, &consoles, c_list)
+	{
 		if (!strcmp(cur->c_name, con->c_name)) {
 			spin_unlock_irqrestore(&consoles_lock, flags);
 			return KERN_NAME_EXISTS;

kernel/equeue.c

@@ -0,0 +1,29 @@
+#include <kernel/equeue.h>
+
+kern_status_t equeue_type_init(void)
+{
+	return KERN_UNIMPLEMENTED;
+}
+
+struct equeue *equeue_cast(struct object *obj)
+{
+	return NULL;
+}
+
+struct equeue *equeue_create(void)
+{
+	return NULL;
+}
+
+kern_status_t equeue_enqueue(
+	struct equeue *q,
+	const equeue_packet_t *pkt,
+	enum equeue_flags flags)
+{
+	return KERN_UNIMPLEMENTED;
+}
+
+kern_status_t equeue_dequeue(struct equeue *q, equeue_packet_t *out)
+{
+	return KERN_UNIMPLEMENTED;
+}

kernel/futex.c

@@ -0,0 +1,211 @@
+#include <kernel/address-space.h>
+#include <kernel/futex.h>
+#include <kernel/sched.h>
+#include <kernel/task.h>
+#include <mango/status.h>
+
+#define FUTEX_CREATE 0x40u
+
+static struct btree shared_futex_list = {0};
+static spin_lock_t shared_futex_list_lock = SPIN_LOCK_INIT;
+
+static struct vm_cache futex_cache = {
+	.c_name = "futex",
+	.c_obj_size = sizeof(struct futex),
+};
+
+BTREE_DEFINE_SIMPLE_INSERT(struct futex, f_node, f_key, put_futex)
+BTREE_DEFINE_SIMPLE_GET(struct futex, uintptr_t, f_node, f_key, get_futex)
+
+kern_status_t futex_init(void)
+{
+	vm_cache_init(&futex_cache);
+	return KERN_OK;
+}
+
+static kern_status_t get_data(
+	futex_key_t key,
+	unsigned int flags,
+	struct futex **out,
+	spin_lock_t **out_lock,
+	unsigned long *irq_flags)
+{
+	spin_lock_t *lock = NULL;
+	struct btree *futex_list = NULL;
+	if (flags & FUTEX_PRIVATE) {
+		struct task *self = current_task();
+		lock = &self->t_base.ob_lock;
+		futex_list = &self->t_futex;
+	} else if (flags & FUTEX_SHARED) {
+		lock = &shared_futex_list_lock;
+		futex_list = &shared_futex_list;
+	} else {
+		return KERN_INVALID_ARGUMENT;
+	}
+
+	spin_lock_irqsave(lock, irq_flags);
+	struct futex *futex = get_futex(futex_list, key);
+
+	if (!futex && !(flags & FUTEX_CREATE)) {
+		spin_unlock_irqrestore(lock, *irq_flags);
+		return KERN_NO_ENTRY;
+	}
+
+	futex = vm_cache_alloc(&futex_cache, VM_NORMAL);
+	if (!futex) {
+		spin_unlock_irqrestore(lock, *irq_flags);
+		return KERN_NO_MEMORY;
+	}
+
+	futex->f_key = key;
+
+	put_futex(futex_list, futex);
+
+	*out = futex;
+	*out_lock = lock;
+	return KERN_OK;
+}
+
+static kern_status_t cleanup_data(struct futex *futex, unsigned int flags)
+{
+	struct btree *futex_list = NULL;
+	if (flags & FUTEX_PRIVATE) {
+		struct task *self = current_task();
+		futex_list = &self->t_futex;
+	} else if (flags & FUTEX_SHARED) {
+		futex_list = &shared_futex_list;
+	} else {
+		return KERN_INVALID_ARGUMENT;
+	}
+
+	btree_delete(futex_list, &futex->f_node);
+	vm_cache_free(&futex_cache, futex);
+
+	return KERN_OK;
+}
+
+static kern_status_t futex_get_shared(kern_futex_t *futex, futex_key_t *out)
+{
+	struct task *self = current_task();
+	struct address_space *space = self->t_address_space;
+
+	unsigned long flags;
+	address_space_lock_irqsave(space, &flags);
+	kern_status_t status = address_space_translate(
+		space,
+		(virt_addr_t)futex,
+		out,
+		&flags);
+	address_space_unlock_irqrestore(space, flags);
+	return status;
+}
+
+static kern_status_t futex_get_private(kern_futex_t *futex, futex_key_t *out)
+{
+	*out = (futex_key_t)futex;
+	return KERN_OK;
+}
+
+kern_status_t futex_get(
+	kern_futex_t *futex,
+	futex_key_t *out,
+	unsigned int flags)
+{
+	if (flags & FUTEX_PRIVATE) {
+		return futex_get_private(futex, out);
+	}
+
+	if (flags & FUTEX_SHARED) {
+		return futex_get_shared(futex, out);
+	}
+
+	return KERN_INVALID_ARGUMENT;
+}
+
+static kern_status_t futex_read(
+	struct futex *futex,
+	unsigned int flags,
+	kern_futex_t *out)
+{
+	if (flags & FUTEX_PRIVATE) {
+		virt_addr_t addr = futex->f_key;
+		*out = *(kern_futex_t *)addr;
+		return KERN_OK;
+	}
+
+	if (flags & FUTEX_SHARED) {
+		phys_addr_t paddr = futex->f_key;
+		virt_addr_t vaddr = (virt_addr_t)vm_phys_to_virt(paddr);
+		if (!vaddr) {
+			return KERN_MEMORY_FAULT;
+		}
+
+		*out = *(kern_futex_t *)vaddr;
+		return KERN_OK;
+	}
+
+	return KERN_INVALID_ARGUMENT;
+}
+
+kern_status_t futex_wait(
+	futex_key_t key,
+	kern_futex_t new_val,
+	unsigned int flags)
+{
+	spin_lock_t *lock = NULL;
+	unsigned long irq_flags = 0;
+	struct futex *futex = NULL;
+	kern_status_t status = get_data(key, flags, &futex, &lock, &irq_flags);
+	if (status != KERN_OK) {
+		return status;
+	}
+
+	kern_futex_t current_val = 0;
+	status = futex_read(futex, flags, &current_val);
+	if (status != KERN_OK) {
+		spin_unlock_irqrestore(lock, irq_flags);
+		return status;
+	}
+
+	if (current_val != new_val) {
+		spin_unlock_irqrestore(lock, irq_flags);
+		return KERN_BAD_STATE;
+	}
+
+	struct wait_item waiter;
+	thread_wait_begin(&waiter, &futex->f_waiters);
+	spin_unlock_irqrestore(lock, irq_flags);
+
+	schedule(SCHED_NORMAL);
+
+	spin_lock_irqsave(lock, &irq_flags);
+	thread_wait_end(&waiter, &futex->f_waiters);
+
+	if (waitqueue_empty(&futex->f_waiters)) {
+		cleanup_data(futex, flags);
+	}
+
+	spin_unlock_irqrestore(lock, irq_flags);
+
+	return KERN_OK;
+}
+
+kern_status_t futex_wake(futex_key_t key, size_t nwaiters, unsigned int flags)
+{
+	spin_lock_t *lock = NULL;
+	unsigned long irq_flags = 0;
+	struct futex *futex = NULL;
+	kern_status_t status = get_data(key, flags, &futex, &lock, &irq_flags);
+	if (status != KERN_OK) {
+		return status;
+	}
+
+	if (nwaiters == FUTEX_WAKE_ALL) {
+		wakeup_queue(&futex->f_waiters);
+	} else {
+		wakeup_n(&futex->f_waiters, nwaiters);
+	}
+
+	spin_unlock_irqrestore(lock, irq_flags);
+	return KERN_OK;
+}

kernel/handle.c

@@ -1,16 +1,18 @@
+#include <kernel/address-space.h>
 #include <kernel/handle.h>
 #include <kernel/libc/string.h>
 #include <kernel/object.h>
 #include <kernel/sched.h>
 #include <kernel/util.h>
 #include <kernel/vm.h>
+#include <mango/types.h>
 
 /* depth=3 gives a maximum of ~66.6 million handles */
 #define MAX_TABLE_DEPTH  3
 #define RESERVED_HANDLES 64
 
 static struct vm_cache handle_table_cache = {
-	.c_name = "handle_table",
+	.c_name = "handle-table",
 	.c_obj_size = sizeof(struct handle_table),
 };
 
@@ -31,8 +33,48 @@ struct handle_table *handle_table_create(void)
 	return out;
 }
 
+static void do_handle_table_destroy_leaf(struct handle_table *tab)
+{
+	while (1) {
+		unsigned int index = bitmap_lowest_set(
+			tab->t_handles.t_handle_map,
+			HANDLES_PER_TABLE);
+		if (index == BITMAP_NPOS) {
+			break;
+		}
+
+		struct handle *child = &tab->t_handles.t_handle_list[index];
+		bitmap_clear(tab->t_subtables.t_subtable_map, index);
+		if (child->h_object) {
+			object_unref(child->h_object);
+			child->h_object = NULL;
+		}
+	}
+}
+
+static void do_handle_table_destroy(
+	struct handle_table *tab,
+	unsigned int depth)
+{
+	if (depth == MAX_TABLE_DEPTH - 1) {
+		do_handle_table_destroy_leaf(tab);
+		return;
+	}
+
+	for (size_t i = 0; i < REFS_PER_TABLE; i++) {
+		struct handle_table *child
+			= tab->t_subtables.t_subtable_list[i];
+		if (child) {
+			do_handle_table_destroy(child, depth + 1);
+		}
+	}
+
+	vm_cache_free(&handle_table_cache, tab);
+}
+
 void handle_table_destroy(struct handle_table *tab)
 {
+	do_handle_table_destroy(tab, 0);
 }
 
 static kern_status_t decode_handle_indices(
@@ -153,7 +195,7 @@ kern_status_t handle_table_free_handle(
 		= &tab->t_handles.t_handle_list[handle_index];
 
 	if (handle_entry->h_object) {
-		object_remove_handle(handle_entry->h_object);
+		object_unref(handle_entry->h_object);
 	}
 
 	memset(handle_entry, 0x0, sizeof *handle_entry);
@@ -192,122 +234,147 @@ struct handle *handle_table_get_handle(
 	return &tab->t_handles.t_handle_list[handle_index];
 }
 
-struct handle_list_iterator {
+kern_status_t handle_table_transfer(
-	struct handle_list *it_list;
+	struct address_space *dst_region,
-	size_t it_list_count;
+	struct handle_table *dst,
-	size_t it_list_ptr;
+	kern_msg_handle_t *dst_handles,
-
+	size_t dst_handles_max,
-	kern_handle_t *it_handles;
+	struct address_space *src_region,
-	size_t it_nr_handles;
+	struct handle_table *src,
-};
+	kern_msg_handle_t *src_handles,
-
+	size_t src_handles_count)
-static void handle_list_iterator_begin(
-	struct handle_list_iterator *it,
-	struct handle_list *list,
-	size_t list_count)
 {
-	memset(it, 0x0, sizeof *it);
+	kern_status_t status = KERN_OK;
-	it->it_list = list;
+	size_t to_transfer = MIN(dst_handles_max, src_handles_count);
-	it->it_list_count = list_count;
 
-	while (it->it_list_ptr < list_count) {
+	size_t i = 0;
-		if (list[it->it_list_ptr].l_nr_handles > 0) {
+	for (size_t i = 0; i < to_transfer; i++) {
+		kern_msg_handle_t src_handle = {0}, dst_handle = {0};
+		virt_addr_t src_handle_addr
+			= (virt_addr_t)src_handles + (i * sizeof src_handle);
+		virt_addr_t dst_handle_addr
+			= (virt_addr_t)dst_handles + (i * sizeof dst_handle);
+		status = address_space_read(
+			src_region,
+			src_handle_addr,
+			sizeof src_handle,
+			&src_handle,
+			NULL);
+
+		if (status != KERN_OK) {
+			src_handle.hnd_result = KERN_OK;
+			address_space_write(
+				src_region,
+				src_handle_addr,
+				sizeof src_handle,
+				&src_handle,
+				NULL);
 			break;
 		}
 
-		it->it_list_ptr++;
+		if (src_handle.hnd_value == KERN_HANDLE_INVALID) {
-	}
+			continue;
+		}
 
-	if (it->it_list_ptr >= list_count) {
+		struct handle *src_entry
-		return;
+			= handle_table_get_handle(src, src_handle.hnd_value);
-	}
+		struct handle *dst_entry = NULL;
+		kern_handle_t dst_value = KERN_HANDLE_INVALID;
 
-	it->it_handles = list[it->it_list_ptr].l_handles;
+		if (!src_entry) {
-	it->it_nr_handles = list[it->it_list_ptr].l_nr_handles;
+			status = KERN_INVALID_ARGUMENT;
-}
+			src_handle.hnd_result = KERN_INVALID_ARGUMENT;
-
+			address_space_write(
-static void handle_list_iterator_seek(
+				src_region,
-	struct handle_list_iterator *it,
+				src_handle_addr,
-	size_t nr_handles)
+				sizeof src_handle,
-{
+				&src_handle,
-	if (nr_handles > it->it_nr_handles) {
+				NULL);
-		nr_handles = it->it_nr_handles;
-	}
-
-	if (nr_handles < it->it_nr_handles) {
-		it->it_handles += nr_handles;
-		it->it_nr_handles -= nr_handles;
-		return;
-	}
-
-	it->it_list_ptr++;
-	while (it->it_list_ptr < it->it_list_count) {
-		if (it->it_list[it->it_list_ptr].l_nr_handles > 0) {
 			break;
 		}
 
-		it->it_list_ptr++;
+		switch (src_handle.hnd_mode) {
-	}
+		case KERN_MSG_HANDLE_IGNORE:
-
+			break;
-	if (it->it_list_ptr >= it->it_list_count) {
+		case KERN_MSG_HANDLE_MOVE:
-		return;
+			status = handle_table_alloc_handle(
-	}
+				dst,
-
+				&dst_entry,
-	it->it_handles = it->it_list[it->it_list_ptr].l_handles;
+				&dst_value);
-	it->it_nr_handles = it->it_list[it->it_list_ptr].l_nr_handles;
-}
-
-kern_status_t handle_list_transfer(
-	struct handle_table *dest_table,
-	struct handle_list *dest_list,
-	size_t dest_list_count,
-	struct handle_table *src_table,
-	const struct handle_list *src_list,
-	size_t src_list_count)
-{
-	struct handle_list_iterator src, dest;
-	handle_list_iterator_begin(
-		&src,
-		(struct handle_list *)src_list,
-		src_list_count);
-	handle_list_iterator_begin(&dest, dest_list, dest_list_count);
-
-	while (src.it_nr_handles && dest.it_nr_handles) {
-		size_t to_copy = MIN(src.it_nr_handles, dest.it_nr_handles);
-		for (size_t i = 0; i < to_copy; i++) {
-			kern_handle_t handle_v = src.it_handles[i];
-			struct handle *handle
-				= handle_table_get_handle(src_table, handle_v);
-			if (!handle) {
-				return KERN_HANDLE_INVALID;
-			}
-
-			struct object *obj = object_ref(handle->h_object);
-			handle_flags_t flags = handle->h_flags;
-
-			handle_table_free_handle(src_table, handle_v);
-
-			struct handle *dest_slot = NULL;
-			kern_status_t status = handle_table_alloc_handle(
-				dest_table,
-				&dest_slot,
-				&handle_v);
 			if (status != KERN_OK) {
-				return status;
+				break;
 			}
 
-			dest_slot->h_object = obj;
+			dst_entry->h_object = src_entry->h_object;
-			dest_slot->h_flags = flags;
+			dst_entry->h_flags = src_entry->h_flags;
+			object_ref(dst_entry->h_object);
 
-			object_add_handle(obj);
+			handle_table_free_handle(src, src_handles[i].hnd_value);
-			object_unref(obj);
 
-			dest.it_handles[i] = handle_v;
+			dst_handle.hnd_mode = src_handles[i].hnd_mode;
+			dst_handle.hnd_value = dst_value;
+			dst_handle.hnd_result = KERN_OK;
+			break;
+		case KERN_MSG_HANDLE_COPY:
+			status = handle_table_alloc_handle(
+				dst,
+				&dst_entry,
+				&dst_value);
+			if (status != KERN_OK) {
+				break;
+			}
+
+			dst_entry->h_object = src_entry->h_object;
+			dst_entry->h_flags = src_entry->h_flags;
+			object_ref(dst_entry->h_object);
+
+			dst_handle.hnd_mode = src_handles[i].hnd_mode;
+			dst_handle.hnd_value = dst_value;
+			dst_handle.hnd_result = KERN_OK;
+			break;
+		default:
+			status = KERN_INVALID_ARGUMENT;
+			break;
 		}
 
-		handle_list_iterator_seek(&src, to_copy);
+		src_handle.hnd_result = status;
-		handle_list_iterator_seek(&dest, to_copy);
+
+		address_space_write(
+			src_region,
+			src_handle_addr,
+			sizeof src_handle,
+			&src_handle,
+			NULL);
+		address_space_write(
+			dst_region,
+			dst_handle_addr,
+			sizeof dst_handle,
+			&dst_handle,
+			NULL);
 	}
 
-	return KERN_OK;
+	for (; i < src_handles_count; i++) {
+		kern_msg_handle_t handle = {0};
+		virt_addr_t handle_addr
+			= (virt_addr_t)src_handles + (i * sizeof handle);
+		address_space_read(
+			src_region,
+			handle_addr,
+			sizeof handle,
+			&handle,
+			NULL);
+
+		if (handle.hnd_mode != KERN_MSG_HANDLE_MOVE) {
+			continue;
+		}
+
+		struct handle *src_entry
+			= handle_table_get_handle(src, handle.hnd_value);
+		if (src_entry) {
+			object_unref(src_entry->h_object);
+			handle_table_free_handle(src, handle.hnd_value);
+		}
+	}
+
+	return status;
 }

kernel/iovec.c

@@ -1,10 +1,66 @@
+#include <kernel/address-space.h>
 #include <kernel/iovec.h>
 #include <kernel/libc/string.h>
 #include <kernel/util.h>
 
+static bool read_iovec(
+	struct iovec_iterator *it,
+	size_t index,
+	kern_iovec_t *out)
+{
+	if (index >= it->it_nr_vecs) {
+		return false;
+	}
+
+	if (!it->it_region) {
+		memcpy(out, &it->it_vecs[index], sizeof *out);
+		return true;
+	}
+
+	size_t nr_read = 0;
+	kern_status_t status = address_space_read(
+		it->it_region,
+		(virt_addr_t)it->it_vecs + (index * sizeof(kern_iovec_t)),
+		sizeof(kern_iovec_t),
+		out,
+		&nr_read);
+
+	return (status == KERN_OK && nr_read != sizeof(kern_iovec_t));
+}
+
+void iovec_iterator_begin_user(
+	struct iovec_iterator *it,
+	struct address_space *region,
+	const kern_iovec_t *vecs,
+	size_t nr_vecs)
+{
+	memset(it, 0x0, sizeof *it);
+	it->it_region = region;
+	it->it_vecs = vecs;
+	it->it_nr_vecs = nr_vecs;
+
+	kern_iovec_t iov;
+
+	while (it->it_vec_ptr < nr_vecs) {
+		read_iovec(it, it->it_vec_ptr, &iov);
+		if (iov.io_len > 0) {
+			break;
+		}
+
+		it->it_vec_ptr++;
+	}
+
+	if (it->it_vec_ptr >= nr_vecs) {
+		return;
+	}
+
+	it->it_base = iov.io_base;
+	it->it_len = iov.io_len;
+}
+
 void iovec_iterator_begin(
 	struct iovec_iterator *it,
-	const struct iovec *vecs,
+	const kern_iovec_t *vecs,
 	size_t nr_vecs)
 {
 	memset(it, 0x0, sizeof *it);
@@ -20,6 +76,8 @@ void iovec_iterator_begin(
 	}
 
 	if (it->it_vec_ptr >= nr_vecs) {
+		it->it_len = 0;
+		it->it_base = 0;
 		return;
 	}
 
@@ -39,10 +97,12 @@ void iovec_iterator_seek(struct iovec_iterator *it, size_t nr_bytes)
 		}
 
 		nr_bytes -= to_seek;
+		kern_iovec_t iov;
 
 		it->it_vec_ptr++;
 		while (it->it_vec_ptr < it->it_nr_vecs) {
-			if (it->it_vecs[it->it_vec_ptr].io_len > 0) {
+			read_iovec(it, it->it_vec_ptr, &iov);
+			if (iov.io_len > 0) {
 				break;
 			}
 
@@ -50,10 +110,12 @@ void iovec_iterator_seek(struct iovec_iterator *it, size_t nr_bytes)
 		}
 
 		if (it->it_vec_ptr >= it->it_nr_vecs) {
+			it->it_len = 0;
+			it->it_base = 0;
 			return;
 		}
 
-		it->it_base = it->it_vecs[it->it_vec_ptr].io_base;
+		it->it_base = iov.io_base;
-		it->it_len = it->it_vecs[it->it_vec_ptr].io_len;
+		it->it_len = iov.io_len;
 	}
 }

kernel/msg.c

@@ -0,0 +1,22 @@
+#include <kernel/msg.h>
+#include <kernel/vm.h>
+
+static struct vm_cache msg_cache = {
+	.c_name = "msg",
+	.c_obj_size = sizeof(struct msg),
+};
+
+void msg_init(void)
+{
+	vm_cache_init(&msg_cache);
+}
+
+struct msg *msg_alloc(void)
+{
+	return vm_cache_alloc(&msg_cache, VM_NORMAL);
+}
+
+void msg_free(struct msg *msg)
+{
+	vm_cache_free(&msg_cache, msg);
+}

kernel/object.c

@@ -1,12 +1,28 @@
 #include <kernel/locks.h>
 #include <kernel/object.h>
 #include <kernel/queue.h>
+#include <kernel/sched.h>
+#include <kernel/thread.h>
 
 #define HAS_OP(obj, opname) ((obj)->ob_type->ob_ops.opname)
 
 static struct queue object_types;
 static spin_lock_t object_types_lock = SPIN_LOCK_INIT;
 
+static koid_t koid_alloc(void)
+{
+	static koid_t counter = 0;
+	static spin_lock_t lock = SPIN_LOCK_INIT;
+
+	unsigned long flags;
+	spin_lock_irqsave(&lock, &flags);
+	koid_t result = counter;
+	counter++;
+	spin_unlock_irqrestore(&lock, flags);
+
+	return result;
+}
+
 kern_status_t object_bootstrap(void)
 {
 	return KERN_OK;
@@ -50,30 +66,28 @@ struct object *object_create(struct object_type *type)
 		return NULL;
 	}
 
-	memset(obj_buf, 0x00, type->ob_size);
-
 	struct object *obj = (struct object *)((unsigned char *)obj_buf
 	                                       + type->ob_header_offset);
 
+	obj->ob_id = koid_alloc();
 	obj->ob_type = type;
 	obj->ob_lock = SPIN_LOCK_INIT;
 	obj->ob_magic = OBJECT_MAGIC;
 	obj->ob_refcount = 1;
-	obj->ob_handles = 0;
 
 	return obj;
 }
 
 struct object *object_ref(struct object *obj)
 {
-	obj->ob_refcount++;
+	atomic_add_fetch(&obj->ob_refcount, 1);
 	return obj;
 }
 
-static void object_cleanup(struct object *obj, unsigned long flags)
+void object_unref(struct object *obj)
 {
-	if (obj->ob_refcount > 0 || obj->ob_handles > 0) {
+	int ref = atomic_sub_fetch(&obj->ob_refcount, 1);
-		spin_unlock_irqrestore(&obj->ob_lock, flags);
+	if (ref > 0) {
 		return;
 	}
 
@@ -84,39 +98,6 @@ static void object_cleanup(struct object *obj, unsigned long flags)
 	vm_cache_free(&obj->ob_type->ob_cache, obj);
 }
 
-void object_unref(struct object *obj)
-{
-	unsigned long flags;
-	spin_lock_irqsave(&obj->ob_lock, &flags);
-
-	if (obj->ob_refcount == 0) {
-		spin_unlock_irqrestore(&obj->ob_lock, flags);
-		return;
-	}
-
-	obj->ob_refcount--;
-	object_cleanup(obj, flags);
-}
-
-void object_add_handle(struct object *obj)
-{
-	obj->ob_handles++;
-}
-
-void object_remove_handle(struct object *obj)
-{
-	unsigned long flags;
-	spin_lock_irqsave(&obj->ob_lock, &flags);
-
-	if (obj->ob_handles == 0) {
-		spin_unlock_irqrestore(&obj->ob_lock, flags);
-		return;
-	}
-
-	obj->ob_handles--;
-	object_cleanup(obj, flags);
-}
-
 void object_lock(struct object *obj)
 {
 	spin_lock(&obj->ob_lock);
@@ -137,6 +118,32 @@ void object_unlock_irqrestore(struct object *obj, unsigned long flags)
 	spin_unlock_irqrestore(&obj->ob_lock, flags);
 }
 
+void object_lock_pair(struct object *a, struct object *b)
+{
+	spin_lock_pair(&a->ob_lock, &b->ob_lock);
+}
+
+void object_unlock_pair(struct object *a, struct object *b)
+{
+	spin_unlock_pair(&a->ob_lock, &b->ob_lock);
+}
+
+void object_lock_pair_irqsave(
+	struct object *a,
+	struct object *b,
+	unsigned long *flags)
+{
+	spin_lock_pair_irqsave(&a->ob_lock, &b->ob_lock, flags);
+}
+
+void object_unlock_pair_irqrestore(
+	struct object *a,
+	struct object *b,
+	unsigned long flags)
+{
+	spin_unlock_pair_irqrestore(&a->ob_lock, &b->ob_lock, flags);
+}
+
 void *object_data(struct object *obj)
 {
 	return (char *)obj + sizeof *obj;
@@ -151,3 +158,35 @@ struct object *object_header(void *p)
 
 	return obj;
 }
+
+void object_assert_signal(struct object *obj, uint32_t signals)
+{
+	obj->ob_signals |= signals;
+	wakeup_queue(&obj->ob_wq);
+}
+
+void object_clear_signal(struct object *obj, uint32_t signals)
+{
+	obj->ob_signals &= ~signals;
+}
+
+void object_wait_signal(
+	struct object *obj,
+	uint32_t signals,
+	unsigned long *irq_flags)
+{
+	struct thread *self = current_thread();
+	struct wait_item waiter;
+	wait_item_init(&waiter, self);
+	for (;;) {
+		thread_wait_begin(&waiter, &obj->ob_wq);
+		if (obj->ob_signals & signals) {
+			break;
+		}
+
+		object_unlock_irqrestore(obj, *irq_flags);
+		schedule(SCHED_NORMAL);
+		object_lock_irqsave(obj, irq_flags);
+	}
+	thread_wait_end(&waiter, &obj->ob_wq);
+}

kernel/panic.c

@@ -2,7 +2,8 @@
 #include <kernel/libc/stdio.h>
 #include <kernel/machine/panic.h>
 #include <kernel/printk.h>
-#include <kernel/sched.h>
+#include <kernel/task.h>
+#include <kernel/thread.h>
 #include <stdarg.h>
 
 static int has_panicked = 0;

kernel/port.c

@@ -1,15 +1,29 @@
 #include <kernel/channel.h>
 #include <kernel/port.h>
+#include <kernel/printk.h>
+#include <kernel/thread.h>
 #include <kernel/util.h>
 
 #define PORT_CAST(p) OBJECT_C_CAST(struct port, p_base, &port_type, p)
 
+static kern_status_t port_cleanup(struct object *obj);
+
 static struct object_type port_type = {
 	.ob_name = "port",
 	.ob_size = sizeof(struct port),
 	.ob_header_offset = offsetof(struct port, p_base),
+	.ob_ops = {
+		.destroy = port_cleanup,
+	},
 };
 
+static kern_status_t port_cleanup(struct object *obj)
+{
+	struct port *port = PORT_CAST(obj);
+	port_disconnect(port);
+	return KERN_OK;
+}
+
 kern_status_t port_type_init(void)
 {
 	return object_type_register(&port_type);
@@ -20,8 +34,24 @@ struct port *port_cast(struct object *obj)
 	return PORT_CAST(obj);
 }
 
-static void wait_for_reply(struct port *port)
+static void wait_for_reply(struct msg *msg, unsigned long *lock_flags)
 {
+	struct wait_item waiter;
+	struct thread *self = current_thread();
+
+	wait_item_init(&waiter, self);
+	for (;;) {
+		self->tr_state = THREAD_SLEEPING;
+		if (msg->msg_status == KMSG_REPLY_SENT) {
+			break;
+		}
+
+		port_unlock_irqrestore(msg->msg_sender_port, *lock_flags);
+		schedule(SCHED_NORMAL);
+		port_lock_irqsave(msg->msg_sender_port, lock_flags);
+	}
+
+	self->tr_state = THREAD_READY;
 }
 
 struct port *port_create(void)
@@ -41,38 +71,92 @@ struct port *port_create(void)
 kern_status_t port_connect(struct port *port, struct channel *remote)
 {
 	if (port->p_status != PORT_OFFLINE) {
+		tracek("port_connect: port in bad state (%d)", port->p_status);
 		return KERN_BAD_STATE;
 	}
 
+	struct msg *msg = msg_alloc();
+	if (!msg) {
+		return KERN_NO_MEMORY;
+	}
+
+	msg->msg_status = KMSG_ASYNC;
+	msg->msg_type = KERN_MSG_TYPE_EVENT;
+	msg->msg_event = KERN_MSG_EVENT_CONNECTION;
+	msg->msg_sender_thread = current_thread();
+	msg->msg_sender_port = port;
+
+	unsigned long flags;
+	channel_lock_irqsave(remote, &flags);
+	channel_enqueue_msg(remote, msg);
+	channel_unlock_irqrestore(remote, flags);
+
 	port->p_remote = remote;
 	port->p_status = PORT_READY;
 	return KERN_OK;
 }
 
-kern_status_t port_send_msg(
+kern_status_t port_disconnect(struct port *port)
-	struct port *port,
-	const struct msg *req,
-	struct msg *resp)
 {
 	if (port->p_status != PORT_READY) {
+		tracek("port_disconnect: port in bad state (%d)",
+		       port->p_status);
 		return KERN_BAD_STATE;
 	}
 
-	struct thread *self = current_thread();
+	struct msg *msg = msg_alloc();
-	struct kmsg *msg = &self->tr_msg;
+	if (!msg) {
-	msg->msg_sender_thread = self;
+		return KERN_NO_MEMORY;
+	}
+
+	msg->msg_status = KMSG_ASYNC;
+	msg->msg_type = KERN_MSG_TYPE_EVENT;
+	msg->msg_event = KERN_MSG_EVENT_DISCONNECTION;
+	msg->msg_sender_thread = current_thread();
 	msg->msg_sender_port = port;
-	msg->msg_req = req;
-	msg->msg_resp = resp;
 
 	unsigned long flags;
 	channel_lock_irqsave(port->p_remote, &flags);
 	channel_enqueue_msg(port->p_remote, msg);
 	channel_unlock_irqrestore(port->p_remote, flags);
 
-	port->p_status = PORT_SEND_BLOCKED;
+	port->p_remote = NULL;
-
+	port->p_status = PORT_OFFLINE;
-	wait_for_reply(port);
+	return KERN_OK;
-
+}
-	return msg->msg_result;
+
+kern_status_t port_send_msg(
+	struct port *port,
+	const kern_msg_t *in_msg,
+	kern_msg_t *out_reply,
+	unsigned long *lock_flags)
+{
+	if (port->p_status != PORT_READY) {
+		tracek("port_send_msg: port in bad state (%d)", port->p_status);
+		return KERN_BAD_STATE;
+	}
+
+	struct thread *self = current_thread();
+	struct msg msg;
+	memset(&msg, 0x0, sizeof msg);
+	msg.msg_type = KERN_MSG_TYPE_DATA;
+	msg.msg_status = KMSG_WAIT_RECEIVE;
+	msg.msg_sender_thread = self;
+	msg.msg_sender_port = port;
+	memcpy(&msg.msg_req, in_msg, sizeof msg.msg_req);
+	memcpy(&msg.msg_resp, out_reply, sizeof msg.msg_req);
+
+	unsigned long flags;
+	channel_lock_irqsave(port->p_remote, &flags);
+	port->p_status = PORT_SEND_BLOCKED;
+	channel_enqueue_msg(port->p_remote, &msg);
+	channel_unlock_irqrestore(port->p_remote, flags);
+
+	wait_for_reply(&msg, lock_flags);
+
+	channel_lock_irqsave(port->p_remote, &flags);
+	btree_delete(&port->p_remote->c_msg, &msg.msg_node);
+	channel_unlock_irqrestore(port->p_remote, flags);
+
+	return msg.msg_result;
 }

kernel/printk.c

@@ -1,7 +1,7 @@
-#include <kernel/printk.h>
-#include <kernel/locks.h>
 #include <kernel/console.h>
 #include <kernel/libc/stdio.h>
+#include <kernel/locks.h>
+#include <kernel/printk.h>
 #include <stdarg.h>
 
 #define LOG_BUFFER_SIZE 0x40000
@@ -23,16 +23,21 @@ static void flush_log_buffer(void)
 {
 	/*
 	if (!early_console) {
-		return;
+	        return;
 	}
 
-	console_write(early_console, log_buffer + log_buffer_readp, log_buffer_writep - log_buffer_readp);
+	console_write(early_console, log_buffer + log_buffer_readp,
+	log_buffer_writep - log_buffer_readp);
 	*/
 
 	unsigned long flags;
 	struct queue *consoles = get_consoles(&flags);
-	queue_foreach(struct console, con, consoles, c_list) {
+	queue_foreach(struct console, con, consoles, c_list)
-		console_write(con, log_buffer + log_buffer_readp, log_buffer_writep - log_buffer_readp);
+	{
+		console_write(
+			con,
+			log_buffer + log_buffer_readp,
+			log_buffer_writep - log_buffer_readp);
 	}
 
 	put_consoles(consoles, flags);
@@ -61,20 +66,33 @@ void early_printk_init(struct console *con)
 	early_console = con;
 }
 
+static void print_msg_direct(const char *s, size_t len)
+{
+	unsigned long flags;
+	struct queue *consoles = get_consoles(&flags);
+	queue_foreach(struct console, con, consoles, c_list)
+	{
+		console_write(con, s, len);
+	}
+
+	put_consoles(consoles, flags);
+}
+
 int printk(const char *format, ...)
 {
 	char msg[LOG_MSG_SIZE];
 
 	va_list arg;
 	va_start(arg, format);
-        int len = vsnprintf(msg, sizeof msg - 1, format, arg);
+	int len = vsnprintf(msg, sizeof msg - 1, format, arg);
 	va_end(arg);
 
 	msg[len] = '\n';
 	msg[len + 1] = '\0';
 
 	if (log_buffer_writep == LOG_BUFFER_SIZE - 1) {
-		console_write(early_console, msg, len + 1);
+		print_msg_direct(msg, len + 1);
+		return 0;
 	}
 
 	unsigned long flags;
@@ -84,5 +102,5 @@ int printk(const char *format, ...)
 
 	flush_log_buffer();
 
-        return 0;
+	return 0;
 }

kernel/status.c

@@ -1,8 +1,9 @@
 #include <mango/status.h>
+#include <mango/types.h>
 
-#define ERROR_STRING_CASE(code) \
+#define ERROR_STRING_CASE(code)                                                \
-	case code: \
+	case code:                                                             \
-	return #code
+		return #code
 
 const char *kern_status_string(kern_status_t status)
 {

libmango/CMakeLists.txt

@@ -1,7 +1,6 @@
 file(GLOB headers ${CMAKE_CURRENT_SOURCE_DIR}/include/mango/*.h)
 file(GLOB asm_sources
 	${CMAKE_CURRENT_SOURCE_DIR}/arch/${CMAKE_SYSTEM_PROCESSOR}/*.S)
-set_property(SOURCE ${asm_sources} PROPERTY LANGUAGE C)
 
 set(public_include_dirs
 	${CMAKE_CURRENT_SOURCE_DIR}/include

libmango/arch/x86_64/syscall.S

@@ -56,38 +56,56 @@
 .endm
 
 SYSCALL_GATE task_exit SYS_TASK_EXIT 1
+SYSCALL_GATE task_self SYS_TASK_SELF 1
 SYSCALL_GATE task_create SYS_TASK_CREATE 5
 SYSCALL_GATE task_create_thread SYS_TASK_CREATE_THREAD 6
+SYSCALL_GATE task_get_address_space SYS_TASK_GET_ADDRESS_SPACE 1
+SYSCALL_GATE task_config_get SYS_TASK_CONFIG_GET 4
+SYSCALL_GATE task_config_set SYS_TASK_CONFIG_SET 4
 
+SYSCALL_GATE thread_self SYS_THREAD_SELF 1
 SYSCALL_GATE thread_start SYS_THREAD_START 1
+SYSCALL_GATE thread_exit SYS_THREAD_EXIT 0
+SYSCALL_GATE thread_config_get SYS_THREAD_CONFIG_GET 4
+SYSCALL_GATE thread_config_set SYS_THREAD_CONFIG_SET 4
 
 SYSCALL_GATE vm_object_create SYS_VM_OBJECT_CREATE 5
 SYSCALL_GATE vm_object_read SYS_VM_OBJECT_READ 5
 SYSCALL_GATE vm_object_write SYS_VM_OBJECT_WRITE 5
 SYSCALL_GATE vm_object_copy SYS_VM_OBJECT_COPY 6
 
-SYSCALL_GATE vm_region_create SYS_VM_REGION_CREATE 8
+SYSCALL_GATE address_space_read SYS_ADDRESS_SPACE_READ 5
-SYSCALL_GATE vm_region_read SYS_VM_REGION_READ 5
+SYSCALL_GATE address_space_write SYS_ADDRESS_SPACE_WRITE 5
-SYSCALL_GATE vm_region_write SYS_VM_REGION_WRITE 5
+SYSCALL_GATE address_space_map SYS_ADDRESS_SPACE_MAP 7
-SYSCALL_GATE vm_region_map_absolute SYS_VM_REGION_MAP_ABSOLUTE 7
+SYSCALL_GATE address_space_unmap SYS_ADDRESS_SPACE_UNMAP 3
-SYSCALL_GATE vm_region_map_relative SYS_VM_REGION_MAP_RELATIVE 7
+SYSCALL_GATE address_space_reserve SYS_ADDRESS_SPACE_RESERVE 4
-SYSCALL_GATE vm_region_unmap_absolute SYS_VM_REGION_UNMAP_ABSOLUTE 3
+SYSCALL_GATE address_space_release SYS_ADDRESS_SPACE_RELEASE 3
-SYSCALL_GATE vm_region_unmap_relative SYS_VM_REGION_UNMAP_RELATIVE 3
 
 SYSCALL_GATE kern_log SYS_KERN_LOG 1
 SYSCALL_GATE kern_handle_close SYS_KERN_HANDLE_CLOSE 1
+SYSCALL_GATE kern_handle_duplicate SYS_KERN_HANDLE_DUPLICATE 2
 SYSCALL_GATE kern_config_get SYS_KERN_CONFIG_GET 3
 SYSCALL_GATE kern_config_set SYS_KERN_CONFIG_SET 3
 
-SYSCALL_GATE channel_create SYS_CHANNEL_CREATE 3
+SYSCALL_GATE channel_create SYS_CHANNEL_CREATE 2
 SYSCALL_GATE port_create SYS_PORT_CREATE 1
 SYSCALL_GATE port_connect SYS_PORT_CONNECT 3
 SYSCALL_GATE port_disconnect SYS_PORT_DISCONNECT 1
-SYSCALL_GATE msg_send SYS_MSG_SEND 4
+SYSCALL_GATE msg_send SYS_MSG_SEND 5
 SYSCALL_GATE msg_recv SYS_MSG_RECV 4
 SYSCALL_GATE msg_reply SYS_MSG_REPLY 4
-SYSCALL_GATE msg_read SYS_MSG_READ 5
+SYSCALL_GATE msg_read SYS_MSG_READ 6
-SYSCALL_GATE msg_read_handles SYS_MSG_READ_HANDLES 5
+SYSCALL_GATE msg_write SYS_MSG_WRITE 6
-SYSCALL_GATE msg_write SYS_MSG_WRITE 5
+
-SYSCALL_GATE msg_write_handles SYS_MSG_WRITE_HANDLES 5
+SYSCALL_GATE vm_controller_create SYS_VM_CONTROLLER_CREATE 1
+SYSCALL_GATE vm_controller_recv SYS_VM_CONTROLLER_RECV 2
+SYSCALL_GATE vm_controller_recv_async SYS_VM_CONTROLLER_RECV_ASYNC 3
+SYSCALL_GATE vm_controller_create_object SYS_VM_CONTROLLER_CREATE_OBJECT 7
+SYSCALL_GATE vm_controller_detach_object SYS_VM_CONTROLLER_DETACH_OBJECT 2
+SYSCALL_GATE vm_controller_supply_pages SYS_VM_CONTROLLER_SUPPLY_PAGES 6
+
+SYSCALL_GATE kern_object_wait SYS_KERN_OBJECT_WAIT 2
+
+SYSCALL_GATE futex_wait SYS_FUTEX_WAIT 3
+SYSCALL_GATE futex_wake SYS_FUTEX_WAKE 3
 

libmango/include-user/mango/equeue.h

@@ -0,0 +1,9 @@
+#ifndef MANGO_EQUEUE_H_
+#define MANGO_EQUEUE_H_
+
+#include <mango/types.h>
+
+extern kern_status_t equeue_create(kern_handle_t *out);
+extern kern_status_t equeue_dequeue(kern_handle_t eq, equeue_packet_t *out);
+
+#endif

libmango/include-user/mango/futex.h

@@ -0,0 +1,16 @@
+#ifndef MANGO_FUTEX_H_
+#define MANGO_FUTEX_H_
+
+#include <mango/status.h>
+#include <mango/types.h>
+
+extern kern_status_t futex_wait(
+	kern_futex_t *futex,
+	kern_futex_t new_val,
+	unsigned int flags);
+extern kern_status_t futex_wake(
+	kern_futex_t *futex,
+	unsigned int nr_waiters,
+	unsigned int flags);
+
+#endif

libmango/include-user/mango/handle.h

@@ -5,5 +5,8 @@
 #include <mango/types.h>
 
 extern kern_status_t kern_handle_close(kern_handle_t handle);
+extern kern_status_t kern_handle_duplicate(
+	kern_handle_t handle,
+	kern_handle_t *out);
 
 #endif

libmango/include-user/mango/log.h

@@ -2,16 +2,26 @@
 #define MANGO_LOG_H_
 
 #include <mango/status.h>
+#include <mango/types.h>
+
+#undef TRACE
 
 extern kern_status_t kern_log(const char *s);
 
+#define kern_logf(...)                                                         \
+	do {                                                                   \
+		char __logbuf[128];                                            \
+		snprintf(__logbuf, sizeof __logbuf, __VA_ARGS__);              \
+		kern_log(__logbuf);                                            \
+	} while (0)
+
 #ifdef TRACE
 #define kern_trace(...) kern_log(__VA_ARGS__)
 #define kern_tracef(...)                                                       \
 	do {                                                                   \
-		char s[128];                                                   \
+		char __logbuf[128];                                            \
-		snprintf(s, sizeof s, __VA_ARGS__);                            \
+		snprintf(__logbuf, sizeof __logbuf, __VA_ARGS__);              \
-		kern_log(s);                                                   \
+		kern_log(__logbuf);                                            \
 	} while (0)
 #else
 #define kern_trace(...)

libmango/include-user/mango/msg.h

@@ -4,10 +4,7 @@
 #include <mango/status.h>
 #include <mango/types.h>
 
-extern kern_status_t channel_create(
+extern kern_status_t channel_create(unsigned int id, kern_handle_t *out);
-	unsigned int id,
-	channel_flags_t flags,
-	kern_handle_t *out);
 extern kern_status_t port_create(kern_handle_t *out);
 extern kern_status_t port_connect(
 	kern_handle_t port,
@@ -17,46 +14,30 @@ extern kern_status_t port_disconnect(kern_handle_t port);
 
 extern kern_status_t msg_send(
 	kern_handle_t port,
-	msg_flags_t flags,
+	const kern_msg_t *msg,
-	const struct msg *req,
+	kern_msg_t *out_response);
-	struct msg *resp);
 
-extern kern_status_t msg_recv(
+extern kern_status_t msg_recv(kern_handle_t channel, kern_msg_t *out);
-	kern_handle_t channel,
-	msg_flags_t flags,
-	msgid_t *out_id,
-	struct msg *out_msg);
 
 extern kern_status_t msg_reply(
 	kern_handle_t channel,
-	msg_flags_t flags,
 	msgid_t id,
-	const struct msg *reply);
+	const kern_msg_t *response);
 
 extern kern_status_t msg_read(
 	kern_handle_t channel,
 	msgid_t id,
 	size_t offset,
-	struct iovec *out,
+	kern_iovec_t *out,
-	size_t nr_out);
+	size_t out_count,
-extern kern_status_t msg_read_handles(
+	size_t *nr_read);
-	kern_handle_t channel,
-	msgid_t id,
-	size_t offset,
-	struct handle_list *out,
-	size_t nr_out);
 
 extern kern_status_t msg_write(
 	kern_handle_t channel,
 	msgid_t id,
 	size_t offset,
-	const struct iovec *in,
+	const kern_iovec_t *in,
-	size_t nr_in);
+	size_t nr_in,
-extern kern_status_t msg_write_handles(
+	size_t *nr_written);
-	kern_handle_t channel,
-	msgid_t id,
-	size_t offset,
-	const struct handle_list *in,
-	size_t nr_in);
 
 #endif

libmango/include-user/mango/object.h

@@ -0,0 +1,8 @@
+#ifndef MANGO_OBJECT_H_
+#define MANGO_OBJECT_H_
+
+#include <mango/types.h>
+
+extern kern_status_t kern_object_wait(kern_wait_item_t *items, size_t nr_items);
+
+#endif

libmango/include-user/mango/task.h

@@ -5,6 +5,7 @@
 #include <mango/types.h>
 
 extern kern_status_t task_exit(int status);
+extern kern_status_t task_self(kern_handle_t *out);
 
 extern kern_status_t task_create(
 	kern_handle_t parent,
@@ -19,7 +20,32 @@ extern kern_status_t task_create_thread(
 	uintptr_t *args,
 	size_t nr_args,
 	kern_handle_t *out_thread);
+extern kern_status_t task_get_address_space(
+	kern_handle_t task,
+	kern_handle_t *out);
+extern kern_status_t task_config_get(
+	kern_handle_t task,
+	kern_config_key_t key,
+	void *ptr,
+	size_t len);
+extern kern_status_t task_config_set(
+	kern_handle_t task,
+	kern_config_key_t key,
+	const void *ptr,
+	size_t len);
 
+extern kern_status_t thread_self(kern_handle_t *out);
 extern kern_status_t thread_start(kern_handle_t thread);
+extern kern_status_t thread_exit(void);
+extern kern_status_t thread_config_get(
+	kern_handle_t thread,
+	kern_config_key_t key,
+	void *ptr,
+	size_t len);
+extern kern_status_t thread_config_set(
+	kern_handle_t thread,
+	kern_config_key_t key,
+	const void *ptr,
+	size_t len);
 
 #endif

libmango/include-user/mango/vm.h

@@ -30,28 +30,19 @@ extern kern_status_t vm_object_copy(
 	size_t count,
 	size_t *nr_copied);
 
-extern kern_status_t vm_region_create(
+extern kern_status_t address_space_read(
-	kern_handle_t parent,
-	const char *name,
-	size_t name_len,
-	off_t offset,
-	size_t region_len,
-	vm_prot_t prot,
-	kern_handle_t *out,
-	virt_addr_t *out_base_address);
-extern kern_status_t vm_region_read(
 	kern_handle_t region,
 	void *dst,
-	off_t offset,
+	virt_addr_t base,
 	size_t count,
 	size_t *nr_read);
-extern kern_status_t vm_region_write(
+extern kern_status_t address_space_write(
 	kern_handle_t region,
 	const void *src,
-	off_t offset,
+	virt_addr_t base,
 	size_t count,
 	size_t *nr_read);
-extern kern_status_t vm_region_map_absolute(
+extern kern_status_t address_space_map(
 	kern_handle_t region,
 	virt_addr_t map_address,
 	kern_handle_t object,
@@ -59,21 +50,45 @@ extern kern_status_t vm_region_map_absolute(
 	size_t length,
 	vm_prot_t prot,
 	virt_addr_t *out_base_address);
-extern kern_status_t vm_region_map_relative(
+extern kern_status_t address_space_unmap(
 	kern_handle_t region,
-	off_t region_offset,
+	virt_addr_t base,
-	kern_handle_t object,
-	off_t object_offset,
-	size_t length,
-	vm_prot_t prot,
-	virt_addr_t *out_base_address);
-extern kern_status_t vm_region_unmap_absolute(
-	kern_handle_t region,
-	virt_addr_t address,
 	size_t length);
-extern kern_status_t vm_region_unmap_relative(
+extern kern_status_t address_space_reserve(
 	kern_handle_t region,
-	off_t offset,
+	virt_addr_t base,
+	size_t length,
+	virt_addr_t *out_base_address);
+extern kern_status_t address_space_release(
+	kern_handle_t region,
+	virt_addr_t base,
+	size_t length);
+
+extern kern_status_t vm_controller_create(kern_handle_t *out);
+extern kern_status_t vm_controller_recv(
+	kern_handle_t ctrl,
+	equeue_packet_page_request_t *out);
+extern kern_status_t vm_controller_recv_async(
+	kern_handle_t ctrl,
+	kern_handle_t eq,
+	equeue_key_t key);
+extern kern_status_t vm_controller_create_object(
+	kern_handle_t ctrl,
+	const char *name,
+	size_t name_len,
+	equeue_key_t key,
+	size_t data_len,
+	vm_prot_t prot,
+	kern_handle_t *out);
+extern kern_status_t vm_controller_detach_object(
+	kern_handle_t ctrl,
+	kern_handle_t vmo);
+extern kern_status_t vm_controller_supply_pages(
+	kern_handle_t ctrl,
+	kern_handle_t dst_vmo,
+	off_t dst_offset,
+	kern_handle_t src_vmo,
+	off_t src_offset,
 	size_t length);
 
 #endif

libmango/include/mango/signal.h

@@ -0,0 +1,12 @@
+#ifndef MANGO_SIGNAL_H_
+#define MANGO_SIGNAL_H_
+
+#define THREAD_SIGNAL_STOPPED                 0x01u
+
+#define CHANNEL_SIGNAL_MSG_RECEIVED           0x01u
+
+#define VM_CONTROLLER_SIGNAL_REQUEST_RECEIVED 0x01u
+
+#define EQUEUE_SIGNAL_PACKET_RECEIVED         0x01u
+
+#endif

libmango/include/mango/status.h

@@ -1,8 +1,6 @@
 #ifndef MANGO_STATUS_H_
 #define MANGO_STATUS_H_
 
-typedef unsigned int kern_status_t;
-
 #define KERN_OK               (0)
 #define KERN_UNIMPLEMENTED    (1)
 #define KERN_NAME_EXISTS      (2)

libmango/include/mango/syscall.h

@@ -1,35 +1,53 @@
 #ifndef MANGO_SYSCALL_H_
 #define MANGO_SYSCALL_H_
 
-#define SYS_TASK_EXIT                1
+#define SYS_KERN_LOG                    1
-#define SYS_TASK_CREATE              2
+#define SYS_KERN_HANDLE_CLOSE           2
-#define SYS_TASK_CREATE_THREAD       3
+#define SYS_KERN_HANDLE_DUPLICATE       3
-#define SYS_THREAD_START             30
+#define SYS_KERN_CONFIG_GET             4
-#define SYS_VM_OBJECT_CREATE         4
+#define SYS_KERN_CONFIG_SET             5
-#define SYS_VM_OBJECT_READ           5
+#define SYS_KERN_OBJECT_WAIT            6
-#define SYS_VM_OBJECT_WRITE          6
+#define SYS_KERN_OBJECT_WAIT_ASYNC      7
-#define SYS_VM_OBJECT_COPY           29
+#define SYS_TASK_EXIT                   8
-#define SYS_VM_REGION_CREATE         7
+#define SYS_TASK_SELF                   9
-#define SYS_VM_REGION_READ           8
+#define SYS_TASK_CREATE                 10
-#define SYS_VM_REGION_WRITE          9
+#define SYS_TASK_CREATE_THREAD          11
-#define SYS_VM_REGION_MAP_ABSOLUTE   10
+#define SYS_TASK_GET_ADDRESS_SPACE      12
-#define SYS_VM_REGION_MAP_RELATIVE   11
+#define SYS_TASK_CONFIG_GET             13
-#define SYS_VM_REGION_UNMAP_ABSOLUTE 12
+#define SYS_TASK_CONFIG_SET             14
-#define SYS_VM_REGION_UNMAP_RELATIVE 13
+#define SYS_THREAD_SELF                 15
-#define SYS_KERN_LOG                 14
+#define SYS_THREAD_START                16
-#define SYS_KERN_HANDLE_CLOSE        15
+#define SYS_THREAD_EXIT                 17
-#define SYS_KERN_CONFIG_GET          16
+#define SYS_THREAD_CONFIG_GET           18
-#define SYS_KERN_CONFIG_SET          17
+#define SYS_THREAD_CONFIG_SET           19
-#define SYS_MSG_SEND                 18
+#define SYS_VM_OBJECT_CREATE            20
-#define SYS_MSG_RECV                 19
+#define SYS_VM_OBJECT_READ              21
-#define SYS_MSG_REPLY                20
+#define SYS_VM_OBJECT_WRITE             22
-#define SYS_MSG_READ                 21
+#define SYS_VM_OBJECT_COPY              23
-#define SYS_MSG_READ_HANDLES         22
+#define SYS_ADDRESS_SPACE_READ          24
-#define SYS_MSG_WRITE                23
+#define SYS_ADDRESS_SPACE_WRITE         25
-#define SYS_MSG_WRITE_HANDLES        24
+#define SYS_ADDRESS_SPACE_MAP           26
-#define SYS_CHANNEL_CREATE           25
+#define SYS_ADDRESS_SPACE_UNMAP         27
-#define SYS_PORT_CREATE              26
+#define SYS_ADDRESS_SPACE_RESERVE       28
-#define SYS_PORT_CONNECT             27
+#define SYS_ADDRESS_SPACE_RELEASE       29
-#define SYS_PORT_DISCONNECT          28
+#define SYS_MSG_SEND                    30
+#define SYS_MSG_RECV                    31
+#define SYS_MSG_REPLY                   32
+#define SYS_MSG_READ                    33
+#define SYS_MSG_WRITE                   34
+#define SYS_CHANNEL_CREATE              35
+#define SYS_PORT_CREATE                 36
+#define SYS_PORT_CONNECT                37
+#define SYS_PORT_DISCONNECT             38
+#define SYS_EQUEUE_CREATE               39
+#define SYS_EQUEUE_DEQUEUE              40
+#define SYS_VM_CONTROLLER_CREATE        41
+#define SYS_VM_CONTROLLER_RECV          42
+#define SYS_VM_CONTROLLER_RECV_ASYNC    43
+#define SYS_VM_CONTROLLER_CREATE_OBJECT 44
+#define SYS_VM_CONTROLLER_DETACH_OBJECT 45
+#define SYS_VM_CONTROLLER_SUPPLY_PAGES  46
+#define SYS_FUTEX_WAIT                  47
+#define SYS_FUTEX_WAKE                  48
 
 #endif

libmango/include/mango/types.h

@@ -4,59 +4,189 @@
 #include <stddef.h>
 #include <stdint.h>
 
-#define VM_PROT_READ                       0x01u
+#define VM_PROT_READ                 0x01u
-#define VM_PROT_WRITE                      0x02u
+#define VM_PROT_WRITE                0x02u
-#define VM_PROT_EXEC                       0x04u
+#define VM_PROT_EXEC                 0x04u
-#define VM_PROT_USER                       0x08u
+#define VM_PROT_USER                 0x08u
-#define VM_PROT_SVR                        0x10u
+#define VM_PROT_SVR                  0x10u
-#define VM_PROT_NOCACHE                    0x10u
+#define VM_PROT_NOCACHE              0x10u
-#define VM_PROT_MAP_SPECIFIC               0x40u
+#define VM_PROT_MAP_SPECIFIC         0x40u
 
-/* if this flag is set, other tasks can connect to this channel using
+#define MAP_ADDRESS_ANY              ((virt_addr_t) - 1)
- * the port_connect_* syscalls.
+#define MAP_ADDRESS_INVALID          ((virt_addr_t)0)
- * if this flag is NOT set, only threads in the task that owns the channel
+#define KERN_HANDLE_INVALID          ((kern_handle_t)0xFFFFFFFF)
- * can create ports connecting to it. */
-#define CHANNEL_F_ALLOW_DIRECT_CONNECTIONS 0x01u
 
-/* msg_reply: once the reply has been sent, disconnect the port that sent the
+/* config keys for use with kern_config_get/kern_config_set */
- * original message */
+#define KERN_CFG_INVALID             0x00000u
-#define MSG_F_DISCONNECT_AFTER_REPLY       0x01u
+#define KERN_CFG_PAGE_SIZE           0x00001u
 
-#define VM_REGION_ANY_OFFSET               ((off_t) - 1)
+/* config keys for use with task_config_get/task_config_set */
-#define KERN_HANDLE_INVALID                ((kern_handle_t)0xFFFFFFFF)
+#define TASK_CFG_INVALID             0x00000u
 
-#define KERN_CFG_INVALID                   0x00u
+/* config keys for use with thread_config_get/thread_config_set */
-#define KERN_CFG_PAGE_SIZE                 0x01u
+#define THREAD_CFG_INVALID           0x00000u
+#define THREAD_CFG_FSBASE            0x20001u
+#define THREAD_CFG_GSBASE            0x20002u
+
+/* maximum number of handles that can be sent in a single message */
+#define KERN_MSG_MAX_HANDLES         64
+
+/* the corresponding handle should be ignored */
+#define KERN_MSG_HANDLE_IGNORE       0
+/* the corresponding handle should be moved to the recipient task. the handle
+ * will be closed. */
+#define KERN_MSG_HANDLE_MOVE         1
+/* the corresponding handle should be copied to the recipient task. the handle
+ * will remain valid for the sending task. */
+#define KERN_MSG_HANDLE_COPY         2
+
+/* maximum number of objects that can be waited on in a single call to
+ * kern_object_wait */
+#define KERN_WAIT_MAX_ITEMS          64
+
+/* message types */
+#define KERN_MSG_TYPE_NONE           0
+#define KERN_MSG_TYPE_DATA           1
+#define KERN_MSG_TYPE_EVENT          2
+
+/* event message types */
+#define KERN_MSG_EVENT_NONE          0
+#define KERN_MSG_EVENT_CONNECTION    1
+#define KERN_MSG_EVENT_DISCONNECTION 2
+
+/* equeue packet types */
+#define EQUEUE_PKT_PAGE_REQUEST      0x01u
+#define EQUEUE_PKT_ASYNC_SIGNAL      0x02u
+
+/* page request types */
+#define PAGE_REQUEST_READ            0x01u
+#define PAGE_REQUEST_DIRTY           0x02u
+#define PAGE_REQUEST_DETACH          0x03u
+
+/* futex special values */
+#define FUTEX_WAKE_ALL               ((size_t)-1)
+
+/* futex flags */
+#define FUTEX_PRIVATE                0x01u
+#define FUTEX_SHARED                 0x02u
+
+#define IOVEC(p, len)                                                          \
+	{                                                                      \
+		.io_base = (virt_addr_t)(p),                                   \
+		.io_len = (len),                                               \
+	}
+#define MSG_HANDLE(mode, value)                                                \
+	{                                                                      \
+		.hnd_mode = (mode),                                            \
+		.hnd_value = (value),                                          \
+	}
+#define MSG(data, data_count, handles, handles_len)                            \
+	{                                                                      \
+		.msg_data = (data),                                            \
+		.msg_data_count = (data_count),                                \
+		.msg_handles = (handles),                                      \
+		.msg_handles_count = (handles_len),                            \
+	}
 
 typedef uintptr_t phys_addr_t;
 typedef uintptr_t virt_addr_t;
 typedef uint64_t msgid_t;
 typedef uint64_t off_t;
+typedef uint64_t koid_t;
+typedef uintptr_t equeue_key_t;
 typedef unsigned int tid_t;
+typedef unsigned int vm_controller_packet_type_t;
+typedef unsigned int kern_status_t;
 typedef uint32_t kern_handle_t;
 typedef uint32_t kern_config_key_t;
 typedef uint32_t vm_prot_t;
-typedef uint32_t channel_flags_t;
+typedef int64_t ssize_t;
-typedef uint32_t msg_flags_t;
+typedef uint32_t kern_futex_t;
+typedef uint32_t kern_msg_type_t;
+typedef uint32_t kern_msg_event_type_t;
+
+typedef unsigned short equeue_packet_type_t;
 
 typedef unsigned int umode_t;
 
-struct iovec {
+typedef struct {
 	virt_addr_t io_base;
 	size_t io_len;
-};
+} kern_iovec_t;
 
-struct handle_list {
+typedef struct {
-	kern_handle_t *l_handles;
+	kern_handle_t w_handle;
-	size_t l_nr_handles;
+	uint32_t w_waitfor;
-};
+	uint32_t w_observed;
+} kern_wait_item_t;
 
-struct msg {
+typedef struct {
-	struct iovec *msg_data;
+	unsigned int hnd_mode;
-	size_t msg_data_count;
+	kern_handle_t hnd_value;
+	kern_status_t hnd_result;
+} kern_msg_handle_t;
 
-	struct handle_list *msg_handles;
+typedef struct {
-	size_t msg_handles_count;
+	/* transaction id. identifies a particular request/response exchange.
-};
+	 * used when replying to a particular message. */
+	msgid_t msg_id;
+	/* the id of the task that sent a particular message. */
+	tid_t msg_sender;
+	/* the id of the port or channel used to send a particular message. */
+	koid_t msg_endpoint;
+	/* the message type */
+	kern_msg_type_t msg_type;
+
+	union {
+		/* msg_type = KERN_MSG_TYPE_DATA */
+		struct {
+			/* a list of iovecs that point to the buffers that make
+			 * up the main message data. */
+			kern_iovec_t *msg_data;
+			size_t msg_data_count;
+			/* a list of handle entries that contain the kernel
+			 * handles included in a message. */
+			kern_msg_handle_t *msg_handles;
+			size_t msg_handles_count;
+		};
+
+		/* msg_type = KERN_MSG_TYPE_EVENT */
+		struct {
+			kern_msg_event_type_t msg_event;
+		};
+	};
+} kern_msg_t;
+
+typedef struct {
+	uint32_t s_observed;
+} equeue_packet_async_signal_t;
+
+typedef struct {
+	/* the key of the vm-object for which the page request relates, as
+	 * specified when the vm-object was created */
+	equeue_key_t req_vmo;
+	/* page request type. one of PAGE_REQUEST_* */
+	unsigned short req_type;
+	/* of the offset into the vm-object for which pages are being requested
+	 */
+	off_t req_offset;
+	/* the length in bytes of the region being requested */
+	size_t req_length;
+} equeue_packet_page_request_t;
+
+typedef struct {
+	/* the type of packet. one of EQUEUE_PKT_* */
+	equeue_packet_type_t p_type;
+	/* the key of the object that is responsible for the event, as specified
+	 * when the event was first subscribed to */
+	equeue_key_t p_key;
+
+	union {
+		/* p_type = EQUEUE_PKT_PAGE_REQUEST */
+		equeue_packet_page_request_t page_request;
+		/* p_type = EQUEUE_PKT_ASYNC_SIGNAL */
+		equeue_packet_async_signal_t async_signal;
+	};
+} equeue_packet_t;
 
 #endif

sched/core.c

@@ -4,7 +4,8 @@
 #include <kernel/object.h>
 #include <kernel/printk.h>
 #include <kernel/sched.h>
-#include <kernel/vm-region.h>
+#include <kernel/task.h>
+#include <kernel/thread.h>
 
 extern kern_status_t setup_kernel_task(void);
 extern kern_status_t setup_idle_task(void);
@@ -91,7 +92,9 @@ void context_switch(struct thread *old, struct thread *new)
 
 void __schedule(enum sched_mode mode)
 {
-	ml_int_disable();
+	if (mode != SCHED_IRQ) {
+		ml_int_disable();
+	}
 
 	struct cpu_data *this_cpu = get_this_cpu();
 	struct runqueue *rq = &this_cpu->c_rq;
@@ -120,8 +123,16 @@ void __schedule(enum sched_mode mode)
 
 	enum thread_state prev_state = READ_ONCE(prev->tr_state);
 
-	if ((mode == SCHED_IRQ || prev_state == THREAD_READY)
+	bool reschedule = false;
-	    && prev != rq->rq_idle) {
+	if (prev_state == THREAD_READY || mode == SCHED_IRQ) {
+		reschedule = true;
+	}
+
+	if (prev == rq->rq_idle || prev_state == THREAD_STOPPED) {
+		reschedule = false;
+	}
+
+	if (reschedule) {
 		rq_enqueue(rq, prev);
 	}
 
@@ -142,7 +153,9 @@ void __schedule(enum sched_mode mode)
 		context_switch(prev, next);
 	}
 
-	ml_int_enable();
+	if (mode != SCHED_IRQ) {
+		ml_int_enable();
+	}
 }
 
 void schedule(enum sched_mode mode)

sched/runqueue.c

@@ -1,8 +1,10 @@
-#include <kernel/sched.h>
-#include <kernel/percpu.h>
 #include <kernel/cpu.h>
+#include <kernel/percpu.h>
+#include <kernel/sched.h>
+#include <kernel/task.h>
+#include <kernel/thread.h>
 
-#define PRIO_MASK(p) (((uint32_t)1) << (p))
+#define PRIO_MASK(p)  (((uint32_t)1) << (p))
 #define FIRST_PRIO(m) (m > 0 ? (PRIO_MAX - __builtin_clz(m) - 1) : -1)
 
 void rq_init(struct runqueue *rq)
@@ -19,6 +21,7 @@ struct thread *rq_dequeue(struct runqueue *rq)
 	}
 
 	struct queue *q = &rq->rq_queues[prio];
+
 	struct queue_entry *qe = queue_pop_front(q);
 	if (!qe) {
 		rq->rq_readybits &= ~PRIO_MASK(prio);
@@ -26,6 +29,7 @@ struct thread *rq_dequeue(struct runqueue *rq)
 	}
 
 	struct thread *thr = QUEUE_CONTAINER(struct thread, tr_rqentry, qe);
+	thr->tr_flags &= ~THREAD_F_SCHEDULED;
 
 	if (rq->rq_nthreads > 0) {
 		rq->rq_nthreads--;
@@ -40,17 +44,24 @@ struct thread *rq_dequeue(struct runqueue *rq)
 
 void rq_enqueue(struct runqueue *rq, struct thread *thr)
 {
+	if (thr->tr_flags & THREAD_F_SCHEDULED) {
+		return;
+	}
+
 	int prio = thread_priority(thr);
 	if (prio < 0 || prio > PRIO_MAX) {
 		return;
 	}
 
 	struct queue *q = &rq->rq_queues[prio];
+
 	queue_push_back(q, &thr->tr_rqentry);
+
 	rq->rq_nthreads++;
 
 	rq->rq_readybits |= PRIO_MASK(thread_priority(thr));
 	thr->tr_rq = rq;
+	thr->tr_flags |= THREAD_F_SCHEDULED;
 }
 
 struct runqueue *cpu_rq(unsigned int cpu)
@@ -64,6 +75,10 @@ struct runqueue *cpu_rq(unsigned int cpu)
 
 void rq_remove_thread(struct runqueue *rq, struct thread *thr)
 {
+	if (!(thr->tr_flags & THREAD_F_SCHEDULED)) {
+		return;
+	}
+
 	int prio = thread_priority(thr);
 	if (prio < 0 || prio > PRIO_MAX) {
 		return;
@@ -71,6 +86,7 @@ void rq_remove_thread(struct runqueue *rq, struct thread *thr)
 
 	struct queue *q = &rq->rq_queues[prio];
 	queue_delete(q, &thr->tr_rqentry);
+	thr->tr_flags &= ~THREAD_F_SCHEDULED;
 
 	if (rq->rq_nthreads > 0) {
 		rq->rq_nthreads--;

sched/task.c

@@ -1,3 +1,4 @@
+#include <kernel/address-space.h>
 #include <kernel/channel.h>
 #include <kernel/clock.h>
 #include <kernel/cpu.h>
@@ -7,8 +8,9 @@
 #include <kernel/object.h>
 #include <kernel/printk.h>
 #include <kernel/sched.h>
+#include <kernel/task.h>
+#include <kernel/thread.h>
 #include <kernel/util.h>
-#include <kernel/vm-region.h>
 
 #define TASK_CAST(p) OBJECT_C_CAST(struct task, t_base, &task_type, p)
 
@@ -93,15 +95,6 @@ kern_status_t setup_kernel_task(void)
 	__kernel_task->t_state = TASK_RUNNING;
 	__kernel_task->t_pmap = get_kernel_pmap();
 
-	vm_region_create(
-		NULL,
-		"root",
-		4,
-		VM_KERNEL_BASE,
-		VM_KERNEL_LIMIT - VM_KERNEL_BASE,
-		VM_PROT_READ | VM_PROT_WRITE | VM_PROT_EXEC | VM_PROT_SVR,
-		&__kernel_task->t_address_space);
-
 	snprintf(
 		__kernel_task->t_name,
 		sizeof __kernel_task->t_name,
@@ -193,16 +186,12 @@ struct task *task_create(const char *name, size_t name_len)
 
 	task->t_id = pid_alloc();
 	task->t_pmap = pmap;
-	vm_region_create(
+	address_space_create(
-		NULL,
-		"root",
-		4,
 		VM_USER_BASE,
-		VM_USER_LIMIT - VM_USER_BASE,
+		VM_USER_LIMIT,
-		VM_PROT_READ | VM_PROT_WRITE | VM_PROT_EXEC | VM_PROT_USER,
 		&task->t_address_space);
 
-	task->t_address_space->vr_pmap = pmap;
+	task->t_address_space->s_pmap = pmap;
 	task->t_state = TASK_RUNNING;
 	task->t_handles = handle_table_create();
 
@@ -233,13 +222,13 @@ kern_status_t task_add_channel(
 {
 	channel->c_id = id;
 
-	if (!task->b_channels.b_root) {
+	if (!task->t_channels.b_root) {
-		task->b_channels.b_root = &channel->c_node;
+		task->t_channels.b_root = &channel->c_node;
-		btree_insert_fixup(&task->b_channels, &channel->c_node);
+		btree_insert_fixup(&task->t_channels, &channel->c_node);
 		return KERN_OK;
 	}
 
-	struct btree_node *cur = task->b_channels.b_root;
+	struct btree_node *cur = task->t_channels.b_root;
 	while (1) {
 		struct channel *cur_node
 			= BTREE_CONTAINER(struct channel, c_node, cur);
@@ -266,7 +255,7 @@ kern_status_t task_add_channel(
 		cur = next;
 	}
 
-	btree_insert_fixup(&task->b_channels, &channel->c_node);
+	btree_insert_fixup(&task->t_channels, &channel->c_node);
 	return KERN_OK;
 }
 
@@ -279,7 +268,7 @@ BTREE_DEFINE_SIMPLE_GET(
 
 struct channel *task_get_channel(struct task *task, unsigned int id)
 {
-	return get_channel_with_id(&task->b_channels, id);
+	return get_channel_with_id(&task->t_channels, id);
 }
 
 struct task *task_from_tid(tid_t id)
@@ -291,6 +280,71 @@ struct task *task_from_tid(tid_t id)
 	return t;
 }
 
+void task_exit(int status)
+{
+	struct task *self = current_task();
+	unsigned long flags;
+	task_lock_irqsave(self, &flags);
+	struct task *parent = self->t_parent;
+
+	if (parent) {
+		task_unlock_irqrestore(self, flags);
+		task_lock_irqsave(parent, &flags);
+		task_lock(self);
+		queue_delete(&parent->t_children, &self->t_child_entry);
+		task_unlock(parent);
+	}
+
+	struct thread *cur_thread = current_thread();
+
+	self->t_state = TASK_STOPPED;
+	cur_thread->tr_state = THREAD_STOPPED;
+
+	struct queue_entry *cur = queue_first(&self->t_threads);
+	while (cur) {
+		struct queue_entry *next = queue_next(cur);
+		struct thread *thread
+			= QUEUE_CONTAINER(struct thread, tr_parent_entry, cur);
+
+		if (thread == cur_thread) {
+			cur = next;
+			continue;
+		}
+
+		thread_lock(thread);
+		thread_kill(thread);
+		queue_delete(&self->t_threads, cur);
+		thread_unlock(thread);
+
+		cur = next;
+	}
+
+	object_unref(&self->t_address_space->s_base);
+	spin_lock_t *handles_lock = &self->t_handles_lock;
+	struct handle_table *handles = self->t_handles;
+	spin_lock(&self->t_handles_lock);
+
+	pmap_switch(get_kernel_pmap());
+	pmap_destroy(self->t_pmap);
+
+	task_unlock(self);
+	handle_table_destroy(handles);
+
+	tracek("thread %s[%u.%u] killed",
+	       self->t_name,
+	       self->t_id,
+	       cur_thread->tr_id);
+	tracek("task %s[%u] killed (%u)",
+	       self->t_name,
+	       self->t_id,
+	       self->t_base.ob_refcount);
+	spin_unlock_irqrestore(handles_lock, flags);
+
+	while (1) {
+		schedule(SCHED_NORMAL);
+	}
+}
+
 kern_status_t task_open_handle(
 	struct task *task,
 	struct object *obj,
@@ -304,7 +358,7 @@ kern_status_t task_open_handle(
 		return status;
 	}
 
-	object_add_handle(obj);
+	object_ref(obj);
 	handle_data->h_object = obj;
 	handle_data->h_flags = flags;
 
@@ -324,7 +378,7 @@ kern_status_t task_resolve_handle(
 	}
 
 	if (out_obj) {
-		*out_obj = handle_data->h_object;
+		*out_obj = object_ref(handle_data->h_object);
 	}
 
 	if (out_flags) {

sched/thread.c

@@ -2,14 +2,17 @@
 #include <kernel/cpu.h>
 #include <kernel/machine/thread.h>
 #include <kernel/object.h>
-#include <kernel/sched.h>
+#include <kernel/printk.h>
+#include <kernel/task.h>
+#include <kernel/thread.h>
+#include <mango/signal.h>
 
-#define THREAD_CAST(p) OBJECT_C_CAST(struct thread, thr_base, &thread_type, p)
+#define THREAD_CAST(p) OBJECT_C_CAST(struct thread, tr_base, &thread_type, p)
 
 static struct object_type thread_type = {
 	.ob_name = "thread",
 	.ob_size = sizeof(struct thread),
-	.ob_header_offset = offsetof(struct thread, thr_base),
+	.ob_header_offset = offsetof(struct thread, tr_base),
 };
 
 kern_status_t thread_object_type_init(void)
@@ -62,9 +65,6 @@ kern_status_t thread_init_user(
 	const uintptr_t *args,
 	size_t nr_args)
 {
-	thr->tr_id = thr->tr_parent->t_next_thread_id++;
-
-	thr->tr_prio = PRIO_NORMAL;
 	thr->tr_state = THREAD_READY;
 	thr->tr_quantum_target = default_quantum();
 
@@ -135,7 +135,59 @@ void thread_awaken(struct thread *thr)
 	}
 
 	thr->tr_state = THREAD_READY;
+	unsigned long flags;
+	rq_lock(rq, &flags);
 	rq_enqueue(rq, thr);
+	rq_unlock(rq, flags);
+}
+
+void thread_exit(void)
+{
+	struct thread *self = current_thread();
+	unsigned long flags;
+	thread_lock_irqsave(self, &flags);
+	self->tr_state = THREAD_STOPPED;
+	object_assert_signal(&self->tr_base, THREAD_SIGNAL_STOPPED);
+	tracek("thread %s[%u.%u] exited",
+	       self->tr_parent->t_name,
+	       self->tr_parent->t_id,
+	       self->tr_id);
+	thread_unlock_irqrestore(self, flags);
+
+	while (1) {
+		schedule(SCHED_NORMAL);
+	}
+}
+
+void thread_join(struct thread *thread, unsigned long *irq_flags)
+{
+	while (1) {
+		if (thread->tr_state == THREAD_STOPPED) {
+			break;
+		}
+
+		object_wait_signal(
+			&thread->tr_base,
+			THREAD_SIGNAL_STOPPED,
+			irq_flags);
+	}
+}
+
+void thread_kill(struct thread *thread)
+{
+	thread->tr_state = THREAD_STOPPED;
+	if (thread->tr_rq) {
+		unsigned long flags;
+		rq_lock(thread->tr_rq, &flags);
+		rq_remove_thread(thread->tr_rq, thread);
+		rq_unlock(thread->tr_rq, flags);
+	}
+
+	object_assert_signal(&thread->tr_base, THREAD_SIGNAL_STOPPED);
+	tracek("thread %s[%u.%u] killed",
+	       thread->tr_parent->t_name,
+	       thread->tr_parent->t_id,
+	       thread->tr_id);
 }
 
 struct thread *create_kernel_thread(void (*fn)(void))
@@ -181,3 +233,31 @@ struct thread *create_idle_thread(void)
 
 	return thr;
 }
+
+kern_status_t thread_config_get(
+	struct thread *thread,
+	kern_config_key_t key,
+	void *out,
+	size_t max)
+{
+	switch (key) {
+	default:
+		break;
+	}
+
+	return ml_thread_config_get(thread, key, out, max);
+}
+
+kern_status_t thread_config_set(
+	struct thread *thread,
+	kern_config_key_t key,
+	const void *ptr,
+	size_t len)
+{
+	switch (key) {
+	default:
+		break;
+	}
+
+	return ml_thread_config_set(thread, key, ptr, len);
+}

sched/timer.c

@@ -1,7 +1,8 @@
-#include <kernel/sched.h>
-#include <kernel/printk.h>
-#include <kernel/cpu.h>
 #include <kernel/clock.h>
+#include <kernel/cpu.h>
+#include <kernel/printk.h>
+#include <kernel/sched.h>
+#include <kernel/thread.h>
 
 static void timeout_expiry(struct timer *timer)
 {

sched/wait.c

@@ -1,5 +1,7 @@
 #include <kernel/cpu.h>
 #include <kernel/sched.h>
+#include <kernel/thread.h>
+#include <kernel/wait.h>
 
 void wait_item_init(struct wait_item *item, struct thread *thr)
 {
@@ -30,6 +32,26 @@ void thread_wait_end(struct wait_item *waiter, struct waitqueue *q)
 	spin_unlock_irqrestore(&q->wq_lock, flags);
 }
 
+void thread_wait_begin_nosleep(struct wait_item *waiter, struct waitqueue *q)
+{
+	unsigned long flags;
+	spin_lock_irqsave(&q->wq_lock, &flags);
+
+	queue_push_back(&q->wq_waiters, &waiter->w_entry);
+
+	spin_unlock_irqrestore(&q->wq_lock, flags);
+}
+
+void thread_wait_end_nosleep(struct wait_item *waiter, struct waitqueue *q)
+{
+	unsigned long flags;
+	spin_lock_irqsave(&q->wq_lock, &flags);
+
+	queue_delete(&q->wq_waiters, &waiter->w_entry);
+
+	spin_unlock_irqrestore(&q->wq_lock, flags);
+}
+
 void wakeup_queue(struct waitqueue *q)
 {
 	unsigned long flags;
@@ -47,6 +69,24 @@ void wakeup_queue(struct waitqueue *q)
 	spin_unlock_irqrestore(&q->wq_lock, flags);
 }
 
+void wakeup_n(struct waitqueue *q, size_t nr_waiters)
+{
+	unsigned long flags;
+	spin_lock_irqsave(&q->wq_lock, &flags);
+	struct queue_entry *ent = queue_pop_front(&q->wq_waiters);
+	while (ent && nr_waiters > 0) {
+		struct wait_item *waiter
+			= QUEUE_CONTAINER(struct wait_item, w_entry, ent);
+		struct thread *thr = waiter->w_thread;
+		thread_awaken(thr);
+
+		ent = queue_pop_front(&q->wq_waiters);
+		nr_waiters--;
+	}
+
+	spin_unlock_irqrestore(&q->wq_lock, flags);
+}
+
 void wakeup_one(struct waitqueue *q)
 {
 	unsigned long flags;

sched/workqueue.c

@@ -1,7 +1,8 @@
-#include <kernel/sched.h>
-#include <kernel/vm.h>
-#include <kernel/util.h>
 #include <kernel/cpu.h>
+#include <kernel/sched.h>
+#include <kernel/thread.h>
+#include <kernel/util.h>
+#include <kernel/vm.h>
 
 static struct worker_pool *__global_worker_pool = NULL;
 
@@ -48,8 +49,12 @@ static void worker_func()
 			continue;
 		}
 
-		struct queue_entry *work_item_qe = queue_pop_front(&this_cpu->c_wq.wq_queue);
+		struct queue_entry *work_item_qe
-		struct work_item *work_item = QUEUE_CONTAINER(struct work_item, w_head, work_item_qe);
+			= queue_pop_front(&this_cpu->c_wq.wq_queue);
+		struct work_item *work_item = QUEUE_CONTAINER(
+			struct work_item,
+			w_head,
+			work_item_qe);
 		spin_unlock_irqrestore(&this_cpu->c_wq.wq_lock, flags);
 		put_cpu(this_cpu);
 

syscall/address-space.c

@@ -1,101 +1,13 @@
+#include <kernel/address-space.h>
 #include <kernel/printk.h>
 #include <kernel/sched.h>
 #include <kernel/syscall.h>
 #include <kernel/vm-object.h>
-#include <kernel/vm-region.h>
 
-kern_status_t sys_vm_region_create(
+kern_status_t sys_address_space_read(
-	kern_handle_t parent,
-	const char *name,
-	size_t name_len,
-	off_t offset,
-	size_t region_len,
-	vm_prot_t prot,
-	kern_handle_t *out,
-	virt_addr_t *out_base_address)
-{
-	struct task *self = current_task();
-
-	if (name_len && !validate_access_r(self, name, name_len)) {
-		return KERN_MEMORY_FAULT;
-	}
-
-	if (!validate_access_w(self, out, sizeof *out)) {
-		return KERN_MEMORY_FAULT;
-	}
-
-	if (!validate_access_w(
-		    self,
-		    out_base_address,
-		    sizeof *out_base_address)) {
-		return KERN_MEMORY_FAULT;
-	}
-
-	unsigned long flags;
-	task_lock_irqsave(self, &flags);
-
-	struct object *obj = NULL;
-	handle_flags_t handle_flags = 0;
-	kern_status_t status
-		= task_resolve_handle(self, parent, &obj, &handle_flags);
-	if (status != KERN_OK) {
-		task_unlock_irqrestore(self, flags);
-		return status;
-	}
-
-	struct vm_region *parent_region = vm_region_cast(obj);
-	if (!parent_region) {
-		task_unlock_irqrestore(self, flags);
-		return KERN_INVALID_ARGUMENT;
-	}
-
-	struct handle *child_handle_slot = NULL;
-	kern_handle_t child_handle = KERN_HANDLE_INVALID;
-
-	status = handle_table_alloc_handle(
-		self->t_handles,
-		&child_handle_slot,
-		&child_handle);
-	if (status != KERN_OK) {
-		task_unlock_irqrestore(self, flags);
-		return status;
-	}
-
-	object_ref(obj);
-	task_unlock_irqrestore(self, flags);
-
-	struct vm_region *child = NULL;
-	status = vm_region_create(
-		parent_region,
-		name,
-		name_len,
-		offset,
-		region_len,
-		prot,
-		&child);
-	object_unref(obj);
-
-	if (status != KERN_OK) {
-		task_lock_irqsave(self, &flags);
-		handle_table_free_handle(self->t_handles, child_handle);
-		task_unlock_irqrestore(self, flags);
-		return status;
-	}
-
-	child_handle_slot->h_object = &child->vr_base;
-	object_add_handle(&child->vr_base);
-	object_unref(&child->vr_base);
-
-	*out = child_handle;
-	*out_base_address = vm_region_get_base_address(child);
-
-	return KERN_OK;
-}
-
-kern_status_t sys_vm_region_read(
 	kern_handle_t region_handle,
 	void *dst,
-	off_t offset,
+	virt_addr_t base,
 	size_t count,
 	size_t *nr_read)
 {
@@ -121,32 +33,33 @@ kern_status_t sys_vm_region_read(
 		return status;
 	}
 
-	struct vm_region *region = vm_region_cast(obj);
+	struct address_space *region = address_space_cast(obj);
 	if (!region) {
 		task_unlock_irqrestore(self, flags);
 		return KERN_INVALID_ARGUMENT;
 	}
 
-	object_ref(obj);
 	task_unlock_irqrestore(self, flags);
 
-	virt_addr_t src_address = vm_region_get_base_address(region) + offset;
+	address_space_lock_irqsave(region, &flags);
-	status = vm_region_memmove(
+	status = address_space_memmove(
 		self->t_address_space,
 		(virt_addr_t)dst,
 		region,
-		src_address,
+		base,
 		count,
 		nr_read);
+	address_space_unlock_irqrestore(region, flags);
+
 	object_unref(obj);
 
 	return status;
 }
 
-kern_status_t sys_vm_region_write(
+kern_status_t sys_address_space_write(
 	kern_handle_t region_handle,
 	const void *src,
-	off_t offset,
+	virt_addr_t base,
 	size_t count,
 	size_t *nr_written)
 {
@@ -173,29 +86,30 @@ kern_status_t sys_vm_region_write(
 		return status;
 	}
 
-	struct vm_region *region = vm_region_cast(obj);
+	struct address_space *region = address_space_cast(obj);
 	if (!region) {
 		task_unlock_irqrestore(self, flags);
 		return KERN_INVALID_ARGUMENT;
 	}
 
-	object_ref(obj);
 	task_unlock_irqrestore(self, flags);
 
-	virt_addr_t dst_address = vm_region_get_base_address(region) + offset;
+	address_space_lock_irqsave(region, &flags);
-	status = vm_region_memmove(
+	status = address_space_memmove(
 		region,
-		dst_address,
+		base,
 		self->t_address_space,
 		(virt_addr_t)src,
 		count,
 		nr_written);
+	address_space_unlock_irqrestore(region, flags);
+
 	object_unref(obj);
 
 	return status;
 }
 
-kern_status_t sys_vm_region_map_absolute(
+kern_status_t sys_address_space_map(
 	kern_handle_t region_handle,
 	virt_addr_t map_address,
 	kern_handle_t object_handle,
@@ -236,7 +150,7 @@ kern_status_t sys_vm_region_map_absolute(
 		return status;
 	}
 
-	struct vm_region *region = vm_region_cast(region_obj);
+	struct address_space *region = address_space_cast(region_obj);
 	if (!region) {
 		task_unlock_irqrestore(self, flags);
 		return KERN_INVALID_ARGUMENT;
@@ -248,24 +162,18 @@ kern_status_t sys_vm_region_map_absolute(
 		return KERN_INVALID_ARGUMENT;
 	}
 
-	object_ref(vmo_obj);
-	object_ref(region_obj);
 	task_unlock_irqrestore(self, flags);
-
+	address_space_lock_irqsave(region, &flags);
-	off_t region_offset = VM_REGION_ANY_OFFSET;
+	/* address_space_map will take care of locking `vmo` */
-	if (map_address != VM_REGION_ANY_OFFSET) {
+	status = address_space_map(
-		region_offset
-			= map_address - vm_region_get_base_address(region);
-	}
-
-	status = vm_region_map_object(
 		region,
-		region_offset,
+		map_address,
 		vmo,
 		object_offset,
 		length,
 		prot,
 		out_base_address);
+	address_space_unlock_irqrestore(region, flags);
 
 	object_unref(vmo_obj);
 	object_unref(region_obj);
@@ -273,23 +181,50 @@ kern_status_t sys_vm_region_map_absolute(
 	return status;
 }
 
-kern_status_t sys_vm_region_map_relative(
+kern_status_t sys_address_space_unmap(
 	kern_handle_t region_handle,
-	off_t region_offset,
+	virt_addr_t base,
-	kern_handle_t object_handle,
+	size_t length)
-	off_t object_offset,
+{
+	struct task *self = current_task();
+
+	kern_status_t status = KERN_OK;
+	unsigned long flags;
+	task_lock_irqsave(self, &flags);
+
+	struct object *region_obj = NULL;
+	handle_flags_t region_flags = 0;
+	status = task_resolve_handle(
+		self,
+		region_handle,
+		&region_obj,
+		&region_flags);
+	if (status != KERN_OK) {
+		task_unlock_irqrestore(self, flags);
+		return status;
+	}
+
+	struct address_space *region = address_space_cast(region_obj);
+	if (!region) {
+		task_unlock_irqrestore(self, flags);
+		return KERN_INVALID_ARGUMENT;
+	}
+
+	task_unlock_irqrestore(self, flags);
+
+	status = address_space_unmap(region, base, length);
+
+	object_unref(region_obj);
+
+	return status;
+}
+
+kern_status_t sys_address_space_reserve(
+	kern_handle_t region_handle,
+	virt_addr_t map_address,
 	size_t length,
-	vm_prot_t prot,
 	virt_addr_t *out_base_address)
 {
-	tracek("vm_region_map_relative(%x, %x, %x, %x, %x, %x, %p)",
-	       region_handle,
-	       region_offset,
-	       object_handle,
-	       object_offset,
-	       length,
-	       prot,
-	       out_base_address);
 	struct task *self = current_task();
 
 	if (out_base_address
@@ -304,8 +239,8 @@ kern_status_t sys_vm_region_map_relative(
 	unsigned long flags;
 	task_lock_irqsave(self, &flags);
 
-	struct object *region_obj = NULL, *vmo_obj = NULL;
+	struct object *region_obj = NULL;
-	handle_flags_t region_flags = 0, vmo_flags = 0;
+	handle_flags_t region_flags = 0;
 	status = task_resolve_handle(
 		self,
 		region_handle,
@@ -316,47 +251,30 @@ kern_status_t sys_vm_region_map_relative(
 		return status;
 	}
 
-	status = task_resolve_handle(self, object_handle, &vmo_obj, &vmo_flags);
+	struct address_space *region = address_space_cast(region_obj);
-	if (status != KERN_OK) {
-		task_unlock_irqrestore(self, flags);
-		return status;
-	}
-
-	struct vm_region *region = vm_region_cast(region_obj);
 	if (!region) {
 		task_unlock_irqrestore(self, flags);
 		return KERN_INVALID_ARGUMENT;
 	}
 
-	struct vm_object *vmo = vm_object_cast(vmo_obj);
-	if (!vmo) {
-		task_unlock_irqrestore(self, flags);
-		return KERN_INVALID_ARGUMENT;
-	}
-
-	object_ref(vmo_obj);
-	object_ref(region_obj);
 	task_unlock_irqrestore(self, flags);
 
-	status = vm_region_map_object(
+	address_space_lock_irqsave(region, &flags);
+	status = address_space_reserve(
 		region,
-		region_offset,
+		map_address,
-		vmo,
-		object_offset,
 		length,
-		prot,
 		out_base_address);
+	address_space_unlock_irqrestore(region, flags);
 
-	object_unref(vmo_obj);
 	object_unref(region_obj);
 
-	tracek("result: %u", status);
 	return status;
 }
 
-kern_status_t sys_vm_region_unmap_absolute(
+kern_status_t sys_address_space_release(
 	kern_handle_t region_handle,
-	virt_addr_t address,
+	virt_addr_t base,
 	size_t length)
 {
 	struct task *self = current_task();
@@ -377,56 +295,17 @@ kern_status_t sys_vm_region_unmap_absolute(
 		return status;
 	}
 
-	struct vm_region *region = vm_region_cast(region_obj);
+	struct address_space *region = address_space_cast(region_obj);
 	if (!region) {
 		task_unlock_irqrestore(self, flags);
 		return KERN_INVALID_ARGUMENT;
 	}
 
-	object_ref(region_obj);
 	task_unlock_irqrestore(self, flags);
 
-	off_t region_offset = address - vm_region_get_base_address(region);
+	address_space_lock_irqsave(region, &flags);
-	status = vm_region_unmap(region, region_offset, length);
+	status = address_space_unmap(region, base, length);
-
+	address_space_unlock_irqrestore(region, flags);
-	object_unref(region_obj);
-
-	return status;
-}
-
-kern_status_t sys_vm_region_unmap_relative(
-	kern_handle_t region_handle,
-	off_t offset,
-	size_t length)
-{
-	struct task *self = current_task();
-
-	kern_status_t status = KERN_OK;
-	unsigned long flags;
-	task_lock_irqsave(self, &flags);
-
-	struct object *region_obj = NULL;
-	handle_flags_t region_flags = 0;
-	status = task_resolve_handle(
-		self,
-		region_handle,
-		&region_obj,
-		&region_flags);
-	if (status != KERN_OK) {
-		task_unlock_irqrestore(self, flags);
-		return status;
-	}
-
-	struct vm_region *region = vm_region_cast(region_obj);
-	if (!region) {
-		task_unlock_irqrestore(self, flags);
-		return KERN_INVALID_ARGUMENT;
-	}
-
-	object_ref(region_obj);
-	task_unlock_irqrestore(self, flags);
-
-	status = vm_region_unmap(region, offset, length);
 
 	object_unref(region_obj);
 

syscall/config.c

@@ -1,6 +1,5 @@
 #include <kernel/sched.h>
 #include <kernel/syscall.h>
-#include <kernel/vm-region.h>
 
 kern_status_t sys_kern_config_get(kern_config_key_t key, void *ptr, size_t len)
 {

syscall/dispatch.c

@@ -6,22 +6,28 @@
 
 static const virt_addr_t syscall_table[] = {
 	SYSCALL_TABLE_ENTRY(TASK_EXIT, task_exit),
+	SYSCALL_TABLE_ENTRY(TASK_SELF, task_self),
 	SYSCALL_TABLE_ENTRY(TASK_CREATE, task_create),
 	SYSCALL_TABLE_ENTRY(TASK_CREATE_THREAD, task_create_thread),
+	SYSCALL_TABLE_ENTRY(TASK_GET_ADDRESS_SPACE, task_get_address_space),
+	SYSCALL_TABLE_ENTRY(THREAD_SELF, thread_self),
 	SYSCALL_TABLE_ENTRY(THREAD_START, thread_start),
+	SYSCALL_TABLE_ENTRY(THREAD_EXIT, thread_exit),
+	SYSCALL_TABLE_ENTRY(THREAD_CONFIG_GET, thread_config_get),
+	SYSCALL_TABLE_ENTRY(THREAD_CONFIG_SET, thread_config_set),
 	SYSCALL_TABLE_ENTRY(VM_OBJECT_CREATE, vm_object_create),
 	SYSCALL_TABLE_ENTRY(VM_OBJECT_READ, vm_object_read),
 	SYSCALL_TABLE_ENTRY(VM_OBJECT_WRITE, vm_object_write),
 	SYSCALL_TABLE_ENTRY(VM_OBJECT_COPY, vm_object_copy),
-	SYSCALL_TABLE_ENTRY(VM_REGION_CREATE, vm_region_create),
+	SYSCALL_TABLE_ENTRY(ADDRESS_SPACE_READ, address_space_read),
-	SYSCALL_TABLE_ENTRY(VM_REGION_READ, vm_region_read),
+	SYSCALL_TABLE_ENTRY(ADDRESS_SPACE_WRITE, address_space_write),
-	SYSCALL_TABLE_ENTRY(VM_REGION_WRITE, vm_region_write),
+	SYSCALL_TABLE_ENTRY(ADDRESS_SPACE_MAP, address_space_map),
-	SYSCALL_TABLE_ENTRY(VM_REGION_MAP_ABSOLUTE, vm_region_map_absolute),
+	SYSCALL_TABLE_ENTRY(ADDRESS_SPACE_UNMAP, address_space_unmap),
-	SYSCALL_TABLE_ENTRY(VM_REGION_MAP_RELATIVE, vm_region_map_relative),
+	SYSCALL_TABLE_ENTRY(ADDRESS_SPACE_RESERVE, address_space_reserve),
-	SYSCALL_TABLE_ENTRY(VM_REGION_UNMAP_ABSOLUTE, vm_region_unmap_absolute),
+	SYSCALL_TABLE_ENTRY(ADDRESS_SPACE_RELEASE, address_space_release),
-	SYSCALL_TABLE_ENTRY(VM_REGION_UNMAP_RELATIVE, vm_region_unmap_relative),
 	SYSCALL_TABLE_ENTRY(KERN_LOG, kern_log),
 	SYSCALL_TABLE_ENTRY(KERN_HANDLE_CLOSE, kern_handle_close),
+	SYSCALL_TABLE_ENTRY(KERN_HANDLE_DUPLICATE, kern_handle_duplicate),
 	SYSCALL_TABLE_ENTRY(KERN_CONFIG_GET, kern_config_get),
 	SYSCALL_TABLE_ENTRY(KERN_CONFIG_SET, kern_config_set),
 	SYSCALL_TABLE_ENTRY(CHANNEL_CREATE, channel_create),
@@ -32,9 +38,22 @@ static const virt_addr_t syscall_table[] = {
 	SYSCALL_TABLE_ENTRY(MSG_RECV, msg_recv),
 	SYSCALL_TABLE_ENTRY(MSG_REPLY, msg_reply),
 	SYSCALL_TABLE_ENTRY(MSG_READ, msg_read),
-	SYSCALL_TABLE_ENTRY(MSG_READ_HANDLES, msg_read_handles),
 	SYSCALL_TABLE_ENTRY(MSG_WRITE, msg_write),
-	SYSCALL_TABLE_ENTRY(MSG_WRITE_HANDLES, msg_write_handles),
+	SYSCALL_TABLE_ENTRY(VM_CONTROLLER_CREATE, vm_controller_create),
+	SYSCALL_TABLE_ENTRY(VM_CONTROLLER_RECV, vm_controller_recv),
+	SYSCALL_TABLE_ENTRY(VM_CONTROLLER_RECV_ASYNC, vm_controller_recv_async),
+	SYSCALL_TABLE_ENTRY(
+		VM_CONTROLLER_CREATE_OBJECT,
+		vm_controller_create_object),
+	SYSCALL_TABLE_ENTRY(
+		VM_CONTROLLER_DETACH_OBJECT,
+		vm_controller_detach_object),
+	SYSCALL_TABLE_ENTRY(
+		VM_CONTROLLER_SUPPLY_PAGES,
+		vm_controller_supply_pages),
+	SYSCALL_TABLE_ENTRY(KERN_OBJECT_WAIT, kern_object_wait),
+	SYSCALL_TABLE_ENTRY(FUTEX_WAIT, futex_wait),
+	SYSCALL_TABLE_ENTRY(FUTEX_WAKE, futex_wake),
 };
 static const size_t syscall_table_count
 	= sizeof syscall_table / sizeof syscall_table[0];

syscall/futex.c

@@ -0,0 +1,37 @@
+#include <kernel/futex.h>
+#include <kernel/sched.h>
+#include <kernel/syscall.h>
+#include <kernel/task.h>
+
+kern_status_t sys_futex_wait(
+	kern_futex_t *futex,
+	kern_futex_t new_val,
+	unsigned int flags)
+{
+	struct task *self = current_task();
+	if (!validate_access_r(self, futex, sizeof *futex)) {
+		return KERN_MEMORY_FAULT;
+	}
+
+	futex_key_t key;
+	kern_status_t status = futex_get(futex, &key, flags);
+	if (status != KERN_OK) {
+		return status;
+	}
+
+	return futex_wait(key, new_val, flags);
+}
+
+kern_status_t sys_futex_wake(
+	kern_futex_t *futex,
+	unsigned int nr_waiters,
+	unsigned int flags)
+{
+	futex_key_t key;
+	kern_status_t status = futex_get(futex, &key, flags);
+	if (status != KERN_OK) {
+		return status;
+	}
+
+	return futex_wake(key, nr_waiters, flags);
+}

syscall/handle.c

@@ -4,5 +4,36 @@
 kern_status_t sys_kern_handle_close(kern_handle_t handle)
 {
 	struct task *self = current_task();
+
 	return task_close_handle(self, handle);
 }
+
+kern_status_t sys_kern_handle_duplicate(
+	kern_handle_t handle,
+	kern_handle_t *out)
+{
+	struct task *self = current_task();
+
+	if (!validate_access_w(self, out, sizeof *out)) {
+		return KERN_MEMORY_FAULT;
+	}
+
+	unsigned long flags;
+	task_lock_irqsave(self, &flags);
+
+	struct object *obj = NULL;
+	handle_flags_t handle_flags = 0;
+	kern_status_t status
+		= task_resolve_handle(self, handle, &obj, &handle_flags);
+	if (status != KERN_OK) {
+		task_unlock_irqrestore(self, flags);
+		return status;
+	}
+
+	status = task_open_handle(self, obj, handle_flags, out);
+	object_unref(obj);
+
+	task_unlock_irqrestore(self, flags);
+
+	return status;
+}

syscall/log.c

@@ -1,9 +1,12 @@
 #include <kernel/printk.h>
 #include <kernel/sched.h>
+#include <kernel/task.h>
+#include <kernel/thread.h>
 
 kern_status_t sys_kern_log(const char *s)
 {
 	struct task *task = current_task();
-	printk("%s: %s", task->t_name, s);
+	struct thread *thread = current_thread();
+	printk("%s[%d.%d]: %s", task->t_name, task->t_id, thread->tr_id, s);
 	return KERN_OK;
 }

syscall/msg.c

@@ -3,12 +3,9 @@
 #include <kernel/printk.h>
 #include <kernel/sched.h>
 #include <kernel/syscall.h>
-#include <kernel/vm-region.h>
+#include <kernel/task.h>
 
-kern_status_t sys_channel_create(
+kern_status_t sys_channel_create(unsigned int id, kern_handle_t *out)
-	unsigned int id,
-	channel_flags_t flags,
-	kern_handle_t *out)
 {
 	struct task *self = current_task();
 	if (!validate_access_w(self, out, sizeof *out)) {
@@ -62,14 +59,13 @@ kern_status_t sys_port_create(kern_handle_t *out)
 	kern_handle_t handle;
 	kern_status_t status
 		= task_open_handle(self, &port->p_base, 0, &handle);
+	task_unlock_irqrestore(self, irq_flags);
+	object_unref(&port->p_base);
+
 	if (status != KERN_OK) {
-		task_unlock_irqrestore(self, irq_flags);
-		object_unref(&port->p_base);
 		return status;
 	}
 
-	task_unlock_irqrestore(self, irq_flags);
-
 	*out = handle;
 	return KERN_OK;
 }
@@ -95,9 +91,7 @@ kern_status_t sys_port_connect(
 		return status;
 	}
 
-	/* add a reference to the port object to make sure it isn't deleted
+	struct port *port = port_cast(port_obj);
-	 * while we're using it */
-	object_ref(port_obj);
 	task_unlock_irqrestore(self, flags);
 
 	struct task *remote_task = task_from_tid(task_id);
@@ -116,81 +110,346 @@ kern_status_t sys_port_connect(
 	object_ref(&remote->c_base);
 	task_unlock_irqrestore(remote_task, flags);
 
-	status = port_connect(port_cast(port_obj), remote);
+	port_lock_irqsave(port, &flags);
-	object_unref(port_obj);
+	status = port_connect(port, remote);
+	port_unlock_irqrestore(port, flags);
 	object_unref(&remote->c_base);
+	object_unref(port_obj);
 
 	return KERN_OK;
 }
 
-kern_status_t sys_port_disconnect(kern_handle_t port)
+kern_status_t sys_port_disconnect(kern_handle_t port_handle)
 {
-	return KERN_UNIMPLEMENTED;
+	unsigned long flags;
+
+	struct task *self = current_task();
+	task_lock_irqsave(self, &flags);
+
+	struct object *port_obj = NULL;
+	handle_flags_t port_handle_flags = 0;
+	kern_status_t status = task_resolve_handle(
+		self,
+		port_handle,
+		&port_obj,
+		&port_handle_flags);
+	if (status != KERN_OK) {
+		return status;
+	}
+
+	task_unlock_irqrestore(self, flags);
+
+	struct port *port = port_cast(port_obj);
+	if (!port) {
+		object_unref(port_obj);
+		return KERN_INVALID_ARGUMENT;
+	}
+
+	object_unref(port_obj);
+	port_lock_irqsave(port, &flags);
+	status = port_disconnect(port);
+	port_unlock_irqrestore(port, flags);
+
+	return status;
+}
+
+static bool validate_iovec(
+	struct task *task,
+	const kern_iovec_t *iov,
+	size_t count,
+	bool rw)
+{
+	if (!validate_access_r(task, iov, count * sizeof(*iov))) {
+		return false;
+	}
+
+	for (size_t i = 0; i < count; i++) {
+		bool ok = false;
+		const kern_iovec_t *vec = &iov[i];
+		if (rw) {
+			ok = validate_access_w(task, vec->io_base, vec->io_len);
+		} else {
+			ok = validate_access_r(task, vec->io_base, vec->io_len);
+		}
+
+		if (!ok) {
+			return false;
+		}
+	}
+
+	return true;
+}
+
+static bool validate_msg(struct task *task, const kern_msg_t *msg, bool rw)
+{
+	if (!msg) {
+		return false;
+	}
+
+	vm_prot_t flags;
+	if (rw) {
+		flags = VM_PROT_WRITE | VM_PROT_USER;
+	} else {
+		flags = VM_PROT_READ | VM_PROT_USER;
+	}
+
+	if (!validate_access(task, msg, sizeof *msg, flags)) {
+		return false;
+	}
+
+	if (!validate_iovec(task, msg->msg_data, msg->msg_data_count, rw)) {
+		return false;
+	}
+
+	size_t handle_buffer_len
+		= msg->msg_handles_count * sizeof(*msg->msg_handles);
+	if (!validate_access(
+		    task,
+		    msg->msg_handles,
+		    handle_buffer_len,
+		    flags)) {
+		return false;
+	}
+
+	return true;
 }
 
 kern_status_t sys_msg_send(
-	kern_handle_t port,
+	kern_handle_t port_handle,
-	msg_flags_t flags,
+	const kern_msg_t *msg,
-	const struct msg *req,
+	kern_msg_t *out_reply)
-	struct msg *resp)
 {
-	return KERN_UNIMPLEMENTED;
+	struct task *self = current_task();
+
+	if (!validate_msg(self, msg, false)) {
+		return KERN_MEMORY_FAULT;
+	}
+
+	if (!validate_msg(self, out_reply, true)) {
+		return KERN_MEMORY_FAULT;
+	}
+
+	unsigned long flags;
+
+	task_lock_irqsave(self, &flags);
+
+	struct object *port_obj = NULL;
+	handle_flags_t port_handle_flags = 0;
+	kern_status_t status = task_resolve_handle(
+		self,
+		port_handle,
+		&port_obj,
+		&port_handle_flags);
+	if (status != KERN_OK) {
+		return status;
+	}
+
+	task_unlock_irqrestore(self, flags);
+
+	struct port *port = port_cast(port_obj);
+	if (!port) {
+		object_unref(port_obj);
+		return KERN_INVALID_ARGUMENT;
+	}
+
+	port_lock_irqsave(port, &flags);
+	status = port_send_msg(port, msg, out_reply, &flags);
+	port_unlock_irqrestore(port, flags);
+	object_unref(port_obj);
+
+	return status;
 }
 
-kern_status_t sys_msg_recv(
+kern_status_t sys_msg_recv(kern_handle_t channel_handle, kern_msg_t *out_msg)
-	kern_handle_t channel,
-	msg_flags_t flags,
-	msgid_t *out_id,
-	struct msg *out_msg)
 {
-	return KERN_UNIMPLEMENTED;
+	struct task *self = current_task();
+
+	if (!validate_msg(self, out_msg, true)) {
+		return KERN_MEMORY_FAULT;
+	}
+
+	unsigned long flags;
+
+	task_lock_irqsave(self, &flags);
+
+	struct object *channel_obj = NULL;
+	handle_flags_t channel_handle_flags = 0;
+	kern_status_t status = task_resolve_handle(
+		self,
+		channel_handle,
+		&channel_obj,
+		&channel_handle_flags);
+	if (status != KERN_OK) {
+		return status;
+	}
+
+	task_unlock_irqrestore(self, flags);
+
+	struct channel *channel = channel_cast(channel_obj);
+	if (!channel) {
+		object_unref(channel_obj);
+		return KERN_INVALID_ARGUMENT;
+	}
+
+	channel_lock_irqsave(channel, &flags);
+	status = channel_recv_msg(channel, out_msg, &flags);
+	channel_unlock_irqrestore(channel, flags);
+	object_unref(channel_obj);
+
+	return status;
 }
 
 kern_status_t sys_msg_reply(
-	kern_handle_t channel,
+	kern_handle_t channel_handle,
-	msg_flags_t flags,
 	msgid_t id,
-	const struct msg *reply)
+	const kern_msg_t *reply)
 {
-	return KERN_UNIMPLEMENTED;
+	struct task *self = current_task();
+
+	if (!validate_msg(self, reply, true)) {
+		return KERN_MEMORY_FAULT;
+	}
+
+	unsigned long flags;
+
+	task_lock_irqsave(self, &flags);
+
+	struct object *channel_obj = NULL;
+	handle_flags_t channel_handle_flags = 0;
+	kern_status_t status = task_resolve_handle(
+		self,
+		channel_handle,
+		&channel_obj,
+		&channel_handle_flags);
+	if (status != KERN_OK) {
+		return status;
+	}
+
+	task_unlock_irqrestore(self, flags);
+
+	struct channel *channel = channel_cast(channel_obj);
+	if (!channel) {
+		object_unref(channel_obj);
+		return KERN_INVALID_ARGUMENT;
+	}
+
+	channel_lock_irqsave(channel, &flags);
+	status = channel_reply_msg(channel, id, reply, &flags);
+	channel_unlock_irqrestore(channel, flags);
+	object_unref(channel_obj);
+
+	return status;
 }
 
 kern_status_t sys_msg_read(
-	kern_handle_t channel,
+	kern_handle_t channel_handle,
 	msgid_t id,
 	size_t offset,
-	struct iovec *out,
+	const kern_iovec_t *iov,
-	size_t nr_out)
+	size_t iov_count,
+	size_t *nr_read)
 {
-	return KERN_UNIMPLEMENTED;
+	struct task *self = current_task();
-}
 
-kern_status_t sys_msg_read_handles(
+	if (nr_read && !validate_access_w(self, nr_read, sizeof *nr_read)) {
-	kern_handle_t channel,
+		return KERN_MEMORY_FAULT;
-	msgid_t id,
+	}
-	size_t offset,
+
-	struct handle_list *out,
+	if (!validate_iovec(self, iov, iov_count, true)) {
-	size_t nr_out)
+		return KERN_MEMORY_FAULT;
-{
+	}
-	return KERN_UNIMPLEMENTED;
+
+	unsigned long flags;
+
+	task_lock_irqsave(self, &flags);
+
+	struct object *channel_obj = NULL;
+	handle_flags_t channel_handle_flags = 0;
+	kern_status_t status = task_resolve_handle(
+		self,
+		channel_handle,
+		&channel_obj,
+		&channel_handle_flags);
+	if (status != KERN_OK) {
+		return status;
+	}
+
+	task_unlock_irqrestore(self, flags);
+
+	struct channel *channel = channel_cast(channel_obj);
+	if (!channel) {
+		object_unref(channel_obj);
+		return KERN_INVALID_ARGUMENT;
+	}
+
+	channel_lock_irqsave(channel, &flags);
+	status = channel_read_msg(
+		channel,
+		id,
+		offset,
+		self->t_address_space,
+		iov,
+		iov_count,
+		nr_read);
+	channel_unlock_irqrestore(channel, flags);
+	object_unref(channel_obj);
+
+	return status;
 }
 
 kern_status_t sys_msg_write(
-	kern_handle_t channel,
+	kern_handle_t channel_handle,
 	msgid_t id,
 	size_t offset,
-	const struct iovec *in,
+	const kern_iovec_t *iov,
-	size_t nr_in)
+	size_t iov_count,
+	size_t *nr_written)
 {
-	return KERN_UNIMPLEMENTED;
+	struct task *self = current_task();
-}
 
-kern_status_t sys_msg_write_handles(
+	if (nr_written
-	kern_handle_t channel,
+	    && !validate_access_w(self, nr_written, sizeof *nr_written)) {
-	msgid_t id,
+		return KERN_MEMORY_FAULT;
-	size_t offset,
+	}
-	const struct handle_list *in,
+
-	size_t nr_in)
+	if (!validate_iovec(self, iov, iov_count, false)) {
-{
+		return KERN_MEMORY_FAULT;
-	return KERN_UNIMPLEMENTED;
+	}
+
+	unsigned long flags;
+
+	task_lock_irqsave(self, &flags);
+
+	struct object *channel_obj = NULL;
+	handle_flags_t channel_handle_flags = 0;
+	kern_status_t status = task_resolve_handle(
+		self,
+		channel_handle,
+		&channel_obj,
+		&channel_handle_flags);
+	if (status != KERN_OK) {
+		return status;
+	}
+
+	task_unlock_irqrestore(self, flags);
+
+	struct channel *channel = channel_cast(channel_obj);
+	if (!channel) {
+		object_unref(channel_obj);
+		return KERN_INVALID_ARGUMENT;
+	}
+
+	channel_lock_irqsave(channel, &flags);
+	status = channel_write_msg(
+		channel,
+		id,
+		offset,
+		self->t_address_space,
+		iov,
+		iov_count,
+		nr_written);
+	channel_unlock_irqrestore(channel, flags);
+	object_unref(channel_obj);
+
+	return status;
 }

syscall/object.c

@@ -0,0 +1,82 @@
+#include <kernel/object.h>
+#include <kernel/sched.h>
+#include <kernel/syscall.h>
+#include <kernel/task.h>
+#include <kernel/thread.h>
+#include <kernel/wait.h>
+#include <mango/status.h>
+#include <mango/types.h>
+
+kern_status_t sys_kern_object_wait(kern_wait_item_t *items, size_t nr_items)
+{
+	if (nr_items > KERN_WAIT_MAX_ITEMS) {
+		return KERN_INVALID_ARGUMENT;
+	}
+
+	struct task *self = current_task();
+	struct thread *self_thread = current_thread();
+	if (!validate_access_rw(self, items, nr_items * sizeof *items)) {
+		return KERN_MEMORY_FAULT;
+	}
+
+	self_thread->tr_state = THREAD_SLEEPING;
+
+	kern_status_t status = KERN_OK;
+	struct object *objects[KERN_WAIT_MAX_ITEMS];
+	struct wait_item waiters[KERN_WAIT_MAX_ITEMS];
+	unsigned long irq_flags = 0;
+
+	size_t nr_retained = 0;
+	bool signals_observed = false;
+	for (nr_retained = 0; nr_retained < nr_items; nr_retained++) {
+		kern_handle_t handle = items[nr_retained].w_handle;
+		handle_flags_t flags;
+		struct object *object = NULL;
+
+		status = task_resolve_handle(self, handle, &object, &flags);
+		if (status != KERN_OK) {
+			break;
+		}
+
+		objects[nr_retained] = object;
+
+		object_lock_irqsave(object, &irq_flags);
+		wait_item_init(&waiters[nr_retained], self_thread);
+		thread_wait_begin_nosleep(
+			&waiters[nr_retained],
+			&object->ob_wq);
+
+		if (object->ob_signals & items[nr_retained].w_waitfor) {
+			signals_observed = true;
+			items[nr_retained].w_observed = object->ob_signals;
+		}
+
+		object_unlock_irqrestore(object, irq_flags);
+	}
+
+	if (signals_observed || status != KERN_OK) {
+		goto cleanup;
+	}
+
+	schedule(SCHED_NORMAL);
+
+	for (size_t i = 0; i < nr_retained; i++) {
+		object_lock_irqsave(objects[i], &irq_flags);
+
+		if (objects[i]->ob_signals & items[i].w_waitfor) {
+			signals_observed = true;
+			items[i].w_observed = objects[i]->ob_signals;
+		}
+
+		object_unlock_irqrestore(objects[i], irq_flags);
+	}
+
+cleanup:
+	for (size_t i = 0; i < nr_retained; i++) {
+		thread_wait_end_nosleep(&waiters[i], &objects[i]->ob_wq);
+		object_unref(objects[i]);
+	}
+
+	self_thread->tr_state = THREAD_READY;
+	return status;
+}

syscall/task.c

@@ -1,17 +1,48 @@
+#include <kernel/address-space.h>
 #include <kernel/machine/cpu.h>
 #include <kernel/printk.h>
 #include <kernel/sched.h>
 #include <kernel/syscall.h>
-#include <kernel/vm-region.h>
+#include <kernel/task.h>
+#include <kernel/thread.h>
 
 extern kern_status_t sys_task_exit(int status)
 {
-	while (1) {
+#if defined(TRACE)
-		printk("sys_exit(%d)", status);
+	struct task *self = current_task();
-		milli_sleep(1000);
+	printk("%s[%d]: task_exit(%d)", self->t_name, self->t_id, status);
+#endif
+	task_exit(status);
+	return KERN_FATAL_ERROR;
+}
+
+kern_status_t sys_task_self(kern_handle_t *out)
+{
+	struct task *self = current_task();
+	if (!validate_access_w(self, out, sizeof *out)) {
+		return KERN_MEMORY_FAULT;
 	}
 
-	return KERN_UNIMPLEMENTED;
+	unsigned long flags;
+	task_lock_irqsave(self, &flags);
+
+	struct handle *handle_slot = NULL;
+	kern_handle_t handle;
+	kern_status_t status = handle_table_alloc_handle(
+		self->t_handles,
+		&handle_slot,
+		&handle);
+	task_unlock_irqrestore(self, flags);
+
+	if (status != KERN_OK) {
+		return status;
+	}
+
+	object_ref(&self->t_base);
+	handle_slot->h_object = &self->t_base;
+
+	*out = handle;
+	return KERN_OK;
 }
 
 kern_status_t sys_task_create(
@@ -52,7 +83,6 @@ kern_status_t sys_task_create(
 		return status;
 	}
 
-	object_ref(parent_obj);
 	struct task *parent = task_cast(parent_obj);
 
 	struct handle *child_handle_slot = NULL, *space_handle_slot = NULL;
@@ -62,6 +92,7 @@ kern_status_t sys_task_create(
 		&child_handle_slot,
 		&child_handle);
 	if (status != KERN_OK) {
+		object_unref(parent_obj);
 		task_unlock_irqrestore(self, flags);
 		return status;
 	}
@@ -71,6 +102,7 @@ kern_status_t sys_task_create(
 		&space_handle_slot,
 		&space_handle);
 	if (status != KERN_OK) {
+		object_unref(parent_obj);
 		handle_table_free_handle(self->t_handles, child_handle);
 		task_unlock_irqrestore(self, flags);
 		return status;
@@ -95,10 +127,10 @@ kern_status_t sys_task_create(
 	task_unlock_irqrestore(parent, flags);
 
 	child_handle_slot->h_object = &child->t_base;
-	space_handle_slot->h_object = &child->t_address_space->vr_base;
+	space_handle_slot->h_object = &child->t_address_space->s_base;
 
-	object_add_handle(&child->t_base);
+	object_ref(&child->t_base);
-	object_add_handle(&child->t_address_space->vr_base);
+	object_ref(&child->t_address_space->s_base);
 
 	object_unref(parent_obj);
 
@@ -137,7 +169,6 @@ kern_status_t sys_task_create_thread(
 		return status;
 	}
 
-	object_ref(target_obj);
 	struct task *target = task_cast(target_obj);
 
 	struct handle *target_handle = NULL;
@@ -147,6 +178,7 @@ kern_status_t sys_task_create_thread(
 		&target_handle,
 		&out_handle);
 	if (status != KERN_OK) {
+		object_unref(target_obj);
 		task_unlock_irqrestore(self, flags);
 		return status;
 	}
@@ -166,15 +198,101 @@ kern_status_t sys_task_create_thread(
 	}
 
 	thread_init_user(thread, ip, sp, args, nr_args);
-	target_handle->h_object = &thread->thr_base;
+	target_handle->h_object = &thread->tr_base;
-	object_add_handle(&thread->thr_base);
+	object_ref(&thread->tr_base);
 
 	task_unlock_irqrestore(target, flags);
+	object_unref(target_obj);
 
 	*out_thread = out_handle;
 	return KERN_OK;
 }
 
+kern_status_t sys_task_get_address_space(
+	kern_handle_t task_handle,
+	kern_handle_t *out)
+{
+	struct task *self = current_task();
+	if (!validate_access_w(self, out, sizeof *out)) {
+		return KERN_MEMORY_FAULT;
+	}
+
+	unsigned long flags;
+	task_lock_irqsave(self, &flags);
+
+	struct handle *handle_slot = NULL;
+	kern_handle_t handle;
+	struct object *task_obj = NULL;
+	handle_flags_t handle_flags = 0;
+	kern_status_t status = task_resolve_handle(
+		self,
+		task_handle,
+		&task_obj,
+		&handle_flags);
+	if (status != KERN_OK) {
+		task_unlock_irqrestore(self, flags);
+		return status;
+	}
+
+	status = handle_table_alloc_handle(
+		self->t_handles,
+		&handle_slot,
+		&handle);
+	if (status != KERN_OK) {
+		object_unref(task_obj);
+		task_unlock_irqrestore(self, flags);
+		return status;
+	}
+
+	struct task *task = task_cast(task_obj);
+
+	if (!task) {
+		object_unref(task_obj);
+		handle_table_free_handle(self->t_handles, handle);
+		task_unlock_irqrestore(self, flags);
+		return KERN_INVALID_ARGUMENT;
+	}
+
+	handle_slot->h_object = &task->t_address_space->s_base;
+	object_ref(&task->t_address_space->s_base);
+	task_unlock_irqrestore(self, flags);
+	object_unref(task_obj);
+
+	*out = handle;
+	return KERN_OK;
+}
+
+kern_status_t sys_thread_self(kern_handle_t *out)
+{
+	struct task *self = current_task();
+	if (!validate_access_w(self, out, sizeof *out)) {
+		return KERN_MEMORY_FAULT;
+	}
+
+	struct thread *self_thread = current_thread();
+
+	unsigned long flags;
+	task_lock_irqsave(self, &flags);
+
+	struct handle *handle_slot = NULL;
+	kern_handle_t handle;
+	kern_status_t status = handle_table_alloc_handle(
+		self->t_handles,
+		&handle_slot,
+		&handle);
+	task_unlock_irqrestore(self, flags);
+
+	if (status != KERN_OK) {
+		return status;
+	}
+
+	object_ref(&self_thread->tr_base);
+	handle_slot->h_object = &self_thread->tr_base;
+
+	*out = handle;
+	return KERN_OK;
+}
+
 kern_status_t sys_thread_start(kern_handle_t thread_handle)
 {
 	unsigned long flags;
@@ -193,11 +311,90 @@ kern_status_t sys_thread_start(kern_handle_t thread_handle)
 		return status;
 	}
 
-	object_ref(thread_obj);
 	struct thread *thread = thread_cast(thread_obj);
 	task_unlock_irqrestore(self, flags);
 
 	schedule_thread_on_cpu(thread);
+	object_unref(thread_obj);
 
 	return KERN_OK;
 }
+
+kern_status_t sys_thread_exit(void)
+{
+	thread_exit();
+	/* unreachable */
+	return KERN_FATAL_ERROR;
+}
+
+kern_status_t sys_thread_config_get(
+	kern_handle_t thread_handle,
+	kern_config_key_t key,
+	void *ptr,
+	size_t len)
+{
+	unsigned long flags;
+	struct task *self = current_task();
+
+	if (!validate_access_w(self, ptr, len)) {
+		return KERN_MEMORY_FAULT;
+	}
+
+	struct object *thread_obj;
+	handle_flags_t thread_flags;
+	task_lock_irqsave(self, &flags);
+	kern_status_t status = task_resolve_handle(
+		self,
+		thread_handle,
+		&thread_obj,
+		&thread_flags);
+	if (status != KERN_OK) {
+		task_unlock_irqrestore(self, flags);
+		return status;
+	}
+
+	struct thread *thread = thread_cast(thread_obj);
+	task_unlock_irqrestore(self, flags);
+
+	status = thread_config_get(thread, key, ptr, len);
+
+	object_unref(thread_obj);
+
+	return status;
+}
+
+kern_status_t sys_thread_config_set(
+	kern_handle_t thread_handle,
+	kern_config_key_t key,
+	const void *ptr,
+	size_t len)
+{
+	unsigned long flags;
+	struct task *self = current_task();
+
+	if (!validate_access_w(self, ptr, len)) {
+		return KERN_MEMORY_FAULT;
+	}
+
+	struct object *thread_obj;
+	handle_flags_t thread_flags;
+	task_lock_irqsave(self, &flags);
+	kern_status_t status = task_resolve_handle(
+		self,
+		thread_handle,
+		&thread_obj,
+		&thread_flags);
+	if (status != KERN_OK) {
+		task_unlock_irqrestore(self, flags);
+		return status;
+	}
+
+	struct thread *thread = thread_cast(thread_obj);
+	task_unlock_irqrestore(self, flags);
+
+	status = thread_config_set(thread, key, ptr, len);
+
+	object_unref(thread_obj);
+
+	return status;
+}

syscall/vm-controller.c

@@ -0,0 +1,307 @@
+#include <kernel/equeue.h>
+#include <kernel/sched.h>
+#include <kernel/syscall.h>
+#include <kernel/task.h>
+#include <kernel/vm-controller.h>
+#include <kernel/vm-object.h>
+
+kern_status_t sys_vm_controller_create(kern_handle_t *out)
+{
+	struct task *self = current_task();
+
+	if (!validate_access_w(self, out, sizeof *out)) {
+		return KERN_MEMORY_FAULT;
+	}
+
+	struct vm_controller *ctrl = vm_controller_create();
+	if (!ctrl) {
+		return KERN_NO_MEMORY;
+	}
+
+	kern_status_t status = task_open_handle(self, &ctrl->vc_base, 0, out);
+	if (status != KERN_OK) {
+		object_unref(&ctrl->vc_base);
+		return status;
+	}
+
+	return KERN_OK;
+}
+
+kern_status_t sys_vm_controller_recv(
+	kern_handle_t ctrl_handle,
+	equeue_packet_page_request_t *out)
+{
+	struct task *self = current_task();
+
+	if (!validate_access_w(self, out, sizeof *out)) {
+		return KERN_MEMORY_FAULT;
+	}
+
+	kern_status_t status = KERN_OK;
+	unsigned long flags;
+	task_lock_irqsave(self, &flags);
+
+	struct object *ctrl_obj = NULL;
+	handle_flags_t handle_flags = 0;
+	status = task_resolve_handle(
+		self,
+		ctrl_handle,
+		&ctrl_obj,
+		&handle_flags);
+	if (status != KERN_OK) {
+		task_unlock_irqrestore(self, flags);
+		return status;
+	}
+
+	struct vm_controller *ctrl = vm_controller_cast(ctrl_obj);
+	task_unlock_irqrestore(self, flags);
+	if (!ctrl) {
+		object_unref(ctrl_obj);
+		return KERN_INVALID_ARGUMENT;
+	}
+
+	vm_controller_lock_irqsave(ctrl, &flags);
+	status = vm_controller_recv(ctrl, out);
+	vm_controller_unlock_irqrestore(ctrl, flags);
+
+	object_unref(ctrl_obj);
+
+	return status;
+}
+
+kern_status_t sys_vm_controller_recv_async(
+	kern_handle_t ctrl_handle,
+	kern_handle_t eq_handle,
+	equeue_key_t key)
+{
+	struct task *self = current_task();
+
+	kern_status_t status = KERN_OK;
+	unsigned long flags;
+	task_lock_irqsave(self, &flags);
+
+	struct object *ctrl_obj = NULL, *eq_obj = NULL;
+	handle_flags_t ctrl_flags = 0, eq_flags = 0;
+	status = task_resolve_handle(self, ctrl_handle, &ctrl_obj, &ctrl_flags);
+	if (status != KERN_OK) {
+		task_unlock_irqrestore(self, flags);
+		return status;
+	}
+
+	status = task_resolve_handle(self, eq_handle, &eq_obj, &eq_flags);
+	if (status != KERN_OK) {
+		object_unref(ctrl_obj);
+		task_unlock_irqrestore(self, flags);
+		return status;
+	}
+
+	struct vm_controller *ctrl = vm_controller_cast(ctrl_obj);
+	struct equeue *eq = equeue_cast(eq_obj);
+	task_unlock_irqrestore(self, flags);
+
+	if (!ctrl || !eq) {
+		object_unref(ctrl_obj);
+		object_unref(eq_obj);
+		return KERN_INVALID_ARGUMENT;
+	}
+
+	vm_controller_lock_irqsave(ctrl, &flags);
+	status = vm_controller_recv_async(ctrl, eq, key);
+	vm_controller_unlock_irqrestore(ctrl, flags);
+
+	object_unref(ctrl_obj);
+	object_unref(eq_obj);
+
+	return status;
+}
+
+kern_status_t sys_vm_controller_create_object(
+	kern_handle_t ctrl_handle,
+	const char *name,
+	size_t name_len,
+	equeue_key_t key,
+	size_t data_len,
+	vm_prot_t prot,
+	kern_handle_t *out)
+{
+	struct task *self = current_task();
+
+	if (!validate_access_r(self, name, name_len)) {
+		return KERN_MEMORY_FAULT;
+	}
+
+	if (!validate_access_w(self, out, sizeof *out)) {
+		return KERN_MEMORY_FAULT;
+	}
+
+	kern_status_t status = KERN_OK;
+	unsigned long flags;
+	task_lock_irqsave(self, &flags);
+
+	struct object *ctrl_obj = NULL;
+	handle_flags_t handle_flags = 0;
+	status = task_resolve_handle(
+		self,
+		ctrl_handle,
+		&ctrl_obj,
+		&handle_flags);
+	if (status != KERN_OK) {
+		task_unlock_irqrestore(self, flags);
+		return status;
+	}
+
+	struct handle *out_slot = NULL;
+	kern_handle_t out_handle = KERN_HANDLE_INVALID;
+	status = handle_table_alloc_handle(
+		self->t_handles,
+		&out_slot,
+		&out_handle);
+
+	struct vm_controller *ctrl = vm_controller_cast(ctrl_obj);
+	task_unlock_irqrestore(self, flags);
+	if (!ctrl) {
+		object_unref(ctrl_obj);
+		return KERN_INVALID_ARGUMENT;
+	}
+
+	vm_controller_lock_irqsave(ctrl, &flags);
+	struct vm_object *out_vmo = NULL;
+	status = vm_controller_create_object(
+		ctrl,
+		name,
+		name_len,
+		key,
+		data_len,
+		prot,
+		&out_vmo);
+	vm_controller_unlock_irqrestore(ctrl, flags);
+
+	object_unref(ctrl_obj);
+
+	if (status != KERN_OK) {
+		task_lock_irqsave(self, &flags);
+		handle_table_free_handle(self->t_handles, out_handle);
+		task_unlock_irqrestore(self, flags);
+		return status;
+	}
+
+	out_slot->h_object = &out_vmo->vo_base;
+
+	*out = out_handle;
+	return KERN_OK;
+}
+
+kern_status_t sys_vm_controller_detach_object(
+	kern_handle_t ctrl_handle,
+	kern_handle_t vmo_handle)
+{
+	struct task *self = current_task();
+
+	kern_status_t status = KERN_OK;
+	unsigned long flags;
+	task_lock_irqsave(self, &flags);
+
+	struct object *ctrl_obj = NULL, *vmo_obj = NULL;
+	handle_flags_t ctrl_flags = 0, vmo_flags = 0;
+	status = task_resolve_handle(self, ctrl_handle, &ctrl_obj, &ctrl_flags);
+	if (status != KERN_OK) {
+		task_unlock_irqrestore(self, flags);
+		return status;
+	}
+
+	status = task_resolve_handle(self, vmo_handle, &vmo_obj, &vmo_flags);
+	if (status != KERN_OK) {
+		object_unref(ctrl_obj);
+		task_unlock_irqrestore(self, flags);
+		return status;
+	}
+
+	struct vm_controller *ctrl = vm_controller_cast(ctrl_obj);
+	struct vm_object *vmo = vm_object_cast(vmo_obj);
+	task_unlock_irqrestore(self, flags);
+
+	if (!ctrl || !vmo) {
+		object_unref(ctrl_obj);
+		object_unref(vmo_obj);
+		return KERN_INVALID_ARGUMENT;
+	}
+
+	vm_controller_lock_irqsave(ctrl, &flags);
+	vm_object_lock(vmo);
+	status = vm_controller_detach_object(ctrl, vmo);
+	vm_object_unlock(vmo);
+	vm_controller_unlock_irqrestore(ctrl, flags);
+
+	object_unref(ctrl_obj);
+	object_unref(vmo_obj);
+
+	return status;
+}
+
+kern_status_t sys_vm_controller_supply_pages(
+	kern_handle_t ctrl_handle,
+	kern_handle_t dst_handle,
+	off_t dst_offset,
+	kern_handle_t src_handle,
+	off_t src_offset,
+	size_t count)
+{
+	struct task *self = current_task();
+
+	kern_status_t status = KERN_OK;
+	unsigned long flags;
+	task_lock_irqsave(self, &flags);
+
+	struct object *ctrl_obj = NULL, *src_obj = NULL, *dst_obj = NULL;
+	handle_flags_t ctrl_flags = 0, src_flags = 0, dst_flags = 0;
+	status = task_resolve_handle(self, ctrl_handle, &ctrl_obj, &ctrl_flags);
+	if (status != KERN_OK) {
+		task_unlock_irqrestore(self, flags);
+		return status;
+	}
+
+	status = task_resolve_handle(self, dst_handle, &dst_obj, &dst_flags);
+	if (status != KERN_OK) {
+		object_unref(ctrl_obj);
+		task_unlock_irqrestore(self, flags);
+		return status;
+	}
+
+	status = task_resolve_handle(self, src_handle, &src_obj, &src_flags);
+	if (status != KERN_OK) {
+		object_unref(ctrl_obj);
+		object_unref(dst_obj);
+		task_unlock_irqrestore(self, flags);
+		return status;
+	}
+
+	struct vm_controller *ctrl = vm_controller_cast(ctrl_obj);
+	struct vm_object *dst = vm_object_cast(dst_obj);
+	struct vm_object *src = vm_object_cast(src_obj);
+	task_unlock_irqrestore(self, flags);
+
+	if (!ctrl || !dst || !src) {
+		object_unref(ctrl_obj);
+		object_unref(dst_obj);
+		object_unref(src_obj);
+		return KERN_INVALID_ARGUMENT;
+	}
+
+	vm_controller_lock_irqsave(ctrl, &flags);
+	vm_object_lock_pair(src, dst);
+	status = vm_controller_supply_pages(
+		ctrl,
+		dst,
+		dst_offset,
+		src,
+		src_offset,
+		count);
+	vm_object_unlock_pair(src, dst);
+	vm_controller_unlock_irqrestore(ctrl, flags);
+
+	object_unref(ctrl_obj);
+	object_unref(dst_obj);
+	object_unref(src_obj);
+
+	return status;
+}

syscall/vm-object.c

@@ -3,7 +3,6 @@
 #include <kernel/sched.h>
 #include <kernel/syscall.h>
 #include <kernel/vm-object.h>
-#include <kernel/vm-region.h>
 
 kern_status_t sys_vm_object_create(
 	const char *name,
@@ -30,12 +29,9 @@ kern_status_t sys_vm_object_create(
 
 	kern_status_t status
 		= task_open_handle(self, &obj->vo_base, 0, out_handle);
-	if (status != KERN_OK) {
+	object_unref(&obj->vo_base);
-		object_unref(&obj->vo_base);
-		return status;
-	}
 
-	return KERN_OK;
+	return status;
 }
 
 kern_status_t sys_vm_object_read(
@@ -111,6 +107,13 @@ kern_status_t sys_vm_object_copy(
 	size_t count,
 	size_t *nr_copied)
 {
+	tracek("vm_object_copy(%x, %zx, %x, %zx, %zx, %p)",
+	       dst,
+	       dst_offset,
+	       src,
+	       src_offset,
+	       count,
+	       nr_copied);
 	struct task *self = current_task();
 
 	if (nr_copied
@@ -137,9 +140,6 @@ kern_status_t sys_vm_object_copy(
 		return status;
 	}
 
-	object_ref(src_obj);
-	object_ref(dst_obj);
-
 	task_unlock_irqrestore(self, flags);
 
 	struct vm_object *dst_vmo = vm_object_cast(dst_obj);

util/random.c

@@ -1,35 +1,81 @@
 #include <kernel/util.h>
 
-static unsigned int random_seed = 53455346;
+typedef uint64_t word_t;
+#define STATE_SIZE 312
+#define MIDDLE     156
+#define INIT_SHIFT 62
+#define TWIST_MASK 0xb5026f5aa96619e9
+#define INIT_FACT  6364136223846793005
+#define SHIFT1     29
+#define MASK1      0x5555555555555555
+#define SHIFT2     17
+#define MASK2      0x71d67fffeda60000
+#define SHIFT3     37
+#define MASK3      0xfff7eee000000000
+#define SHIFT4     43
+
+#define LOWER_MASK 0x7fffffff
+#define UPPER_MASK (~(word_t)LOWER_MASK)
+static word_t state[STATE_SIZE];
+static size_t index = STATE_SIZE + 1;
+
+static void seed(word_t s)
+{
+	index = STATE_SIZE;
+	state[0] = s;
+	for (size_t i = 1; i < STATE_SIZE; i++) {
+		state[i] = (INIT_FACT
+		            * (state[i - 1] ^ (state[i - 1] >> INIT_SHIFT)))
+		         + i;
+	}
+}
+
+static void twist(void)
+{
+	for (size_t i = 0; i < STATE_SIZE; i++) {
+		word_t x = (state[i] & UPPER_MASK)
+		         | (state[(i + 1) % STATE_SIZE] & LOWER_MASK);
+		x = (x >> 1) ^ (x & 1 ? TWIST_MASK : 0);
+		state[i] = state[(i + MIDDLE) % STATE_SIZE] ^ x;
+	}
+
+	index = 0;
+}
+
+static word_t mt_random(void)
+{
+	if (index >= STATE_SIZE) {
+		twist();
+	}
+
+	word_t y = state[index];
+	y ^= (y >> SHIFT1) & MASK1;
+	y ^= (y << SHIFT2) & MASK2;
+	y ^= (y << SHIFT3) & MASK3;
+	y ^= y >> SHIFT4;
+
+	index++;
+	return y;
+}
+
+void init_random(uint64_t seedvalue)
+{
+	seed(seedvalue);
+}
 
 bool fill_random(void *p, unsigned int size)
 {
-	unsigned char *buffer = p;
+	unsigned char *dst = p;
+	uint64_t w = mt_random();
+	unsigned char *src = (unsigned char *)&w;
 
-	if (!buffer || !size) {
+	for (size_t i = 0, j = 0; i < size; i++, j++) {
-		return false;
+		dst[i] = src[j];
-	}
 
-	for (uint32_t i = 0; i < size; i++) {
+		if (j == (sizeof w) - 1) {
-		uint32_t next = random_seed;
+			w = mt_random();
-		uint32_t result;
+			j = 0;
-
+		}
-		next *= 1103515245;
-		next += 12345;
-		result = (uint32_t)(next / 65536) % 2048;
-
-		next *= 1103515245;
-		next += 12345;
-		result <<= 10;
-		result ^= (uint32_t)(next / 65536) % 1024;
-
-		next *= 1103515245;
-		next += 12345;
-		result <<= 10;
-		result ^= (uint32_t)(next / 65536) % 1024;
-		random_seed = next;
-
-		buffer[i] = (uint8_t)(result % 256);
 	}
 
 	return true;

vm/bootstrap.c

@@ -1,11 +1,12 @@
-#include <limits.h>
+#include <kernel/address-space.h>
 #include <kernel/machine/cpu.h>
 #include <kernel/memblock.h>
 #include <kernel/printk.h>
-#include <mango/status.h>
+#include <kernel/vm-controller.h>
 #include <kernel/vm-object.h>
-#include <kernel/vm-region.h>
 #include <kernel/vm.h>
+#include <limits.h>
+#include <mango/status.h>
 #include <stddef.h>
 #include <stdint.h>
 
@@ -42,7 +43,8 @@ kern_status_t vm_bootstrap(
 
 	kmalloc_init();
 	vm_object_type_init();
-	vm_region_type_init();
+	vm_controller_type_init();
+	address_space_type_init();
 	return KERN_OK;
 }
 

vm/cache.c

@@ -198,6 +198,8 @@ void *vm_cache_alloc(struct vm_cache *cache, enum vm_flags flags)
 	}
 
 	spin_unlock_irqrestore(&cache->c_lock, irq_flags);
+
+	memset(p, 0x0, cache->c_obj_size);
 	return p;
 }
 

vm/memblock.c

@@ -19,10 +19,11 @@
     contributors may be used to endorse or promote products derived from this
     software without specific prior written permission.
  */
-#include <limits.h>
 #include <kernel/libc/string.h>
 #include <kernel/memblock.h>
+#include <kernel/printk.h>
 #include <kernel/types.h>
+#include <limits.h>
 #include <stdbool.h>
 
 #define MIN(a, b)  ((a) < (b) ? (a) : (b))
@@ -447,10 +448,10 @@ void __next_memory_region(
 			}
 
 			/* we want the area that is overlapped by both
-			        region M (m_start - m_end) : The region defined
+			    - region M (m_start - m_end) : The region defined
-			   as system memory. region R (r_start - r_end) : The
+			      as system memory.
-			   region defined as free / outside of any reserved
+			    - region R (r_start - r_end) : The region defined as
-			   regions.
+			      free / outside of any reserved regions.
 			 */
 			it->it_base = MAX(m_start, r_start);
 			it->it_limit = MIN(m_end, r_end);
@@ -497,3 +498,28 @@ void *memblock_phys_to_virt(phys_addr_t p)
 {
 	return (void *)(p + memblock.m_voffset);
 }
+
+#ifdef TRACE
+static void memblock_type_dump(struct memblock_type *type)
+{
+	tracek("%s:", type->name);
+	for (size_t i = 0; i < type->count; i++) {
+		tracek("  [%zx-%zx]",
+		       type->regions[i].base,
+		       type->regions[i].limit);
+	}
+}
+
+extern void memblock_dump(void)
+{
+	memblock_type_dump(&memblock.memory);
+	memblock_type_dump(&memblock.reserved);
+
+	tracek("free:");
+	struct memblock_iter it;
+	for_each_free_mem_range(&it, 0, ADDR_MAX)
+	{
+		tracek("  [%zx-%zx]", it.it_base, it.it_limit);
+	}
+}
+#endif

vm/vm-controller.c

@@ -0,0 +1,347 @@
+#include <kernel/equeue.h>
+#include <kernel/sched.h>
+#include <kernel/thread.h>
+#include <kernel/util.h>
+#include <kernel/vm-controller.h>
+#include <kernel/vm-object.h>
+#include <mango/signal.h>
+
+#define VM_CONTROLLER_CAST(p)                                                  \
+	OBJECT_C_CAST(struct vm_controller, vc_base, &vm_controller_type, p)
+
+BTREE_DEFINE_SIMPLE_INSERT(struct vm_object, vo_ctrl_node, vo_key, put_object)
+BTREE_DEFINE_SIMPLE_GET(
+	struct vm_object,
+	equeue_key_t,
+	vo_ctrl_node,
+	vo_key,
+	get_object)
+
+static struct object_type vm_controller_type = {
+	.ob_name = "vm-controller",
+	.ob_size = sizeof(struct vm_controller),
+	.ob_header_offset = offsetof(struct vm_controller, vc_base),
+};
+
+static struct vm_cache page_request_cache = {
+	.c_name = "page-request",
+	.c_obj_size = sizeof(struct page_request),
+};
+
+kern_status_t vm_controller_type_init(void)
+{
+	vm_cache_init(&page_request_cache);
+	return object_type_register(&vm_controller_type);
+}
+
+struct vm_controller *vm_controller_cast(struct object *obj)
+{
+	return VM_CONTROLLER_CAST(obj);
+}
+
+struct vm_controller *vm_controller_create(void)
+{
+	struct object *ctrl_object = object_create(&vm_controller_type);
+	if (!ctrl_object) {
+		return NULL;
+	}
+
+	struct vm_controller *ctrl = VM_CONTROLLER_CAST(ctrl_object);
+
+	return ctrl;
+}
+
+static struct page_request *get_next_request(struct vm_controller *ctrl)
+{
+	struct btree_node *cur = btree_first(&ctrl->vc_requests);
+	while (cur) {
+		struct page_request *req
+			= BTREE_CONTAINER(struct page_request, req_node, cur);
+		spin_lock(&req->req_lock);
+		switch (req->req_status) {
+		case PAGE_REQUEST_PENDING:
+			req->req_status = PAGE_REQUEST_IN_PROGRESS;
+			ctrl->vc_requests_waiting--;
+			return req;
+		case PAGE_REQUEST_ASYNC:
+			btree_delete(&ctrl->vc_requests, &req->req_node);
+			ctrl->vc_requests_waiting--;
+			return req;
+		default:
+			break;
+		}
+
+		spin_unlock(&req->req_lock);
+		cur = btree_next(cur);
+	}
+
+	return NULL;
+}
+
+static kern_status_t try_enqueue(struct btree *tree, struct page_request *req)
+{
+	if (!tree->b_root) {
+		tree->b_root = &req->req_node;
+		btree_insert_fixup(tree, &req->req_node);
+		return true;
+	}
+
+	struct btree_node *cur = tree->b_root;
+	while (1) {
+		struct page_request *cur_node
+			= BTREE_CONTAINER(struct page_request, req_node, cur);
+		struct btree_node *next = NULL;
+
+		if (req->req_id > cur_node->req_id) {
+			next = btree_right(cur);
+
+			if (!next) {
+				btree_put_right(cur, &req->req_node);
+				break;
+			}
+		} else if (req->req_id < cur_node->req_id) {
+			next = btree_left(cur);
+
+			if (!next) {
+				btree_put_left(cur, &req->req_node);
+				break;
+			}
+		} else {
+			return false;
+		}
+
+		cur = next;
+	}
+
+	btree_insert_fixup(tree, &req->req_node);
+	return true;
+}
+
+static kern_status_t send_request_async(
+	struct vm_controller *ctrl,
+	struct page_request *req)
+{
+	fill_random(&req->req_id, sizeof req->req_id);
+	while (!try_enqueue(&ctrl->vc_requests, req)) {
+		req->req_id++;
+	}
+
+	ctrl->vc_requests_waiting++;
+	object_assert_signal(
+		&ctrl->vc_base,
+		VM_CONTROLLER_SIGNAL_REQUEST_RECEIVED);
+
+	return KERN_OK;
+}
+
+kern_status_t vm_controller_recv(
+	struct vm_controller *ctrl,
+	equeue_packet_page_request_t *out)
+{
+	struct page_request *req = NULL;
+
+	req = get_next_request(ctrl);
+	if (!req) {
+		return KERN_NO_ENTRY;
+	}
+
+	if (ctrl->vc_requests_waiting == 0) {
+		object_clear_signal(
+			&ctrl->vc_base,
+			VM_CONTROLLER_SIGNAL_REQUEST_RECEIVED);
+	}
+
+	out->req_vmo = req->req_object;
+	out->req_type = req->req_type;
+	out->req_offset = req->req_offset;
+	out->req_length = req->req_length;
+
+	spin_unlock(&req->req_lock);
+
+	if (req->req_status == PAGE_REQUEST_ASYNC) {
+		vm_cache_free(&page_request_cache, req);
+	}
+
+	return KERN_OK;
+}
+
+kern_status_t vm_controller_recv_async(
+	struct vm_controller *ctrl,
+	struct equeue *eq,
+	equeue_key_t key)
+{
+	if (ctrl->vc_eq) {
+		object_unref(&ctrl->vc_eq->eq_base);
+	}
+
+	object_ref(&eq->eq_base);
+	ctrl->vc_eq = eq;
+	ctrl->vc_eq_key = key;
+
+	return KERN_OK;
+}
+
+kern_status_t vm_controller_create_object(
+	struct vm_controller *ctrl,
+	const char *name,
+	size_t name_len,
+	equeue_key_t key,
+	size_t data_len,
+	vm_prot_t prot,
+	struct vm_object **out)
+{
+	struct vm_object *vmo
+		= vm_object_create(name, name_len, data_len, prot);
+	if (!vmo) {
+		return KERN_NO_MEMORY;
+	}
+
+	object_ref(&ctrl->vc_base);
+
+	/* TODO expose the VMO_AUTO_DETACH flag to userspace */
+	vmo->vo_flags |= VMO_CONTROLLER | VMO_AUTO_DETACH;
+	vmo->vo_ctrl = ctrl;
+	vmo->vo_key = key;
+
+	*out = vmo;
+	return KERN_OK;
+}
+
+kern_status_t vm_controller_detach_object(
+	struct vm_controller *ctrl,
+	struct vm_object *vmo)
+{
+	if (vmo->vo_ctrl != ctrl) {
+		return KERN_INVALID_ARGUMENT;
+	}
+
+	struct page_request *req
+		= vm_cache_alloc(&page_request_cache, VM_NORMAL);
+	req->req_type = PAGE_REQUEST_DETACH;
+	req->req_status = PAGE_REQUEST_ASYNC;
+	req->req_object = vmo->vo_key;
+	req->req_sender = current_thread();
+	send_request_async(ctrl, req);
+
+	vmo->vo_ctrl = NULL;
+	vmo->vo_key = 0;
+	object_unref(&ctrl->vc_base);
+
+	return KERN_OK;
+}
+
+static void wait_for_reply(
+	struct vm_controller *ctrl,
+	struct page_request *req,
+	unsigned long *lock_flags)
+{
+	struct wait_item waiter;
+	struct thread *self = current_thread();
+
+	wait_item_init(&waiter, self);
+	for (;;) {
+		self->tr_state = THREAD_SLEEPING;
+		if (req->req_status == PAGE_REQUEST_COMPLETE) {
+			break;
+		}
+
+		spin_unlock_irqrestore(&req->req_lock, *lock_flags);
+		schedule(SCHED_NORMAL);
+		spin_lock_irqsave(&req->req_lock, lock_flags);
+	}
+
+	self->tr_state = THREAD_READY;
+}
+
+static void fulfill_requests(
+	struct vm_controller *ctrl,
+	equeue_key_t object,
+	off_t offset,
+	size_t length,
+	kern_status_t result)
+{
+	off_t limit = offset + length - 1;
+	struct btree_node *cur = btree_first(&ctrl->vc_requests);
+	while (cur) {
+		struct page_request *req
+			= BTREE_CONTAINER(struct page_request, req_node, cur);
+		spin_lock(&req->req_lock);
+		bool match = false;
+		off_t req_base = req->req_offset;
+		off_t req_limit = req->req_offset + req->req_length - 1;
+
+		if (req_base >= offset && req_base <= limit) {
+			match = true;
+		} else if (req_limit >= offset && req_limit <= limit) {
+			match = true;
+		}
+
+		if (req->req_object != object) {
+			match = false;
+		}
+
+		if (match) {
+			req->req_status = PAGE_REQUEST_COMPLETE;
+			req->req_result = result;
+			thread_awaken(req->req_sender);
+		}
+
+		spin_unlock(&req->req_lock);
+		cur = btree_next(cur);
+	}
+}
+
+kern_status_t vm_controller_supply_pages(
+	struct vm_controller *ctrl,
+	struct vm_object *dst,
+	off_t dst_offset,
+	struct vm_object *src,
+	off_t src_offset,
+	size_t count)
+{
+	if (src->vo_flags & VMO_CONTROLLER) {
+		return KERN_INVALID_ARGUMENT;
+	}
+
+	if (dst->vo_ctrl != ctrl) {
+		return KERN_INVALID_ARGUMENT;
+	}
+
+	kern_status_t status = vm_object_transfer(
+		dst,
+		dst_offset,
+		src,
+		src_offset,
+		count,
+		NULL);
+	fulfill_requests(ctrl, dst->vo_key, dst_offset, count, status);
+
+	return status;
+}
+
+kern_status_t vm_controller_send_request(
+	struct vm_controller *ctrl,
+	struct page_request *req,
+	unsigned long *irq_flags)
+{
+	fill_random(&req->req_id, sizeof req->req_id);
+	while (!try_enqueue(&ctrl->vc_requests, req)) {
+		req->req_id++;
+	}
+
+	ctrl->vc_requests_waiting++;
+	object_assert_signal(
+		&ctrl->vc_base,
+		VM_CONTROLLER_SIGNAL_REQUEST_RECEIVED);
+
+	vm_controller_unlock(ctrl);
+	wait_for_reply(ctrl, req, irq_flags);
+
+	spin_unlock_irqrestore(&req->req_lock, *irq_flags);
+	vm_controller_lock_irqsave(ctrl, irq_flags);
+	spin_lock(&req->req_lock);
+
+	btree_delete(&ctrl->vc_requests, &req->req_node);
+
+	return KERN_OK;
+}

vm/vm-object.c

@@ -1,5 +1,8 @@
+#include <kernel/address-space.h>
 #include <kernel/printk.h>
+#include <kernel/sched.h>
 #include <kernel/util.h>
+#include <kernel/vm-controller.h>
 #include <kernel/vm-object.h>
 
 #define VM_OBJECT_CAST(p)                                                      \
@@ -12,10 +15,37 @@
 		(p) += VM_PAGE_SIZE;                                           \
 	}
 
+static kern_status_t vm_object_cleanup(struct object *obj)
+{
+	struct vm_object *vmo = vm_object_cast(obj);
+	struct btree_node *cur = btree_first(&vmo->vo_pages);
+	while (cur) {
+		struct vm_page *pg
+			= BTREE_CONTAINER(struct vm_page, p_bnode, cur);
+		struct btree_node *next = btree_next(cur);
+		btree_delete(&vmo->vo_pages, cur);
+
+		vm_page_free(pg);
+		cur = next;
+	}
+
+	if (vmo->vo_ctrl) {
+		unsigned long flags;
+		vm_controller_lock_irqsave(vmo->vo_ctrl, &flags);
+		vm_controller_detach_object(vmo->vo_ctrl, vmo);
+		vm_controller_unlock_irqrestore(vmo->vo_ctrl, flags);
+	}
+
+	return KERN_OK;
+}
+
 static struct object_type vm_object_type = {
 	.ob_name = "vm-object",
 	.ob_size = sizeof(struct vm_object),
 	.ob_header_offset = offsetof(struct vm_object, vo_base),
+	.ob_ops = {
+		.destroy = vm_object_cleanup,
+	},
 };
 
 struct object_iterator {
@@ -39,15 +69,16 @@ static kern_status_t object_iterator_begin(
 
 	it->it_obj = obj;
 	it->it_alloc = alloc;
+	enum vm_object_flags flags = 0;
 
 	if (alloc) {
-		it->it_pg = vm_object_alloc_page(obj, 0, VM_PAGE_4K);
+		flags |= VMO_ALLOCATE_MISSING_PAGE;
+	}
 
-		if (!it->it_pg) {
+	it->it_pg = vm_object_get_page(obj, 0, flags, NULL);
-			return KERN_NO_MEMORY;
+
-		}
+	if (alloc && !it->it_pg) {
-	} else {
+		return KERN_NO_MEMORY;
-		it->it_pg = vm_object_get_page(obj, 0);
 	}
 
 	if (it->it_pg) {
@@ -82,22 +113,25 @@ static kern_status_t object_iterator_seek(
 		return KERN_OK;
 	}
 
-	if (it->it_alloc) {
+	enum vm_object_flags flags = 0;
-		it->it_pg = vm_object_alloc_page(
-			it->it_obj,
-			it->it_offset,
-			VM_PAGE_4K);
 
-		if (!it->it_pg) {
+	if (it->it_alloc) {
-			return KERN_NO_MEMORY;
+		flags |= VMO_ALLOCATE_MISSING_PAGE;
-		}
+	}
-	} else {
+
-		it->it_pg = vm_object_get_page(it->it_obj, it->it_offset);
+	it->it_pg = vm_object_get_page(it->it_obj, it->it_offset, flags, NULL);
+
+	if (it->it_alloc && !it->it_pg) {
+		return KERN_NO_MEMORY;
 	}
 
 	if (it->it_pg) {
-		it->it_buf = vm_page_get_vaddr(it->it_pg);
+		virt_addr_t vaddr = (virt_addr_t)vm_page_get_vaddr(it->it_pg);
+		vaddr += (it->it_offset & VM_PAGE_MASK);
+
+		it->it_buf = (void *)vaddr;
 		it->it_max = vm_page_get_size_bytes(it->it_pg);
+		it->it_max -= (it->it_offset & VM_PAGE_MASK);
 	} else {
 		struct btree_node *n = btree_first(&it->it_obj->vo_pages);
 		struct vm_page *pg
@@ -112,8 +146,9 @@ static kern_status_t object_iterator_seek(
 		}
 
 		it->it_buf = NULL;
-		it->it_max = pg ? pg->p_vmo_offset
+		it->it_max
-		                : it->it_obj->vo_size - it->it_offset;
+			= pg ? pg->p_vmo_offset - (it->it_offset & VM_PAGE_MASK)
+		             : it->it_obj->vo_size - it->it_offset;
 	}
 
 	return KERN_OK;
@@ -229,7 +264,6 @@ extern struct vm_object *vm_object_create_in_place(
 	     i += VM_PAGE_SIZE, offset += VM_PAGE_SIZE) {
 		struct vm_page *pg = vm_page_get(i);
 		if (!pg) {
-			printk("vm-object: invalid physical address %08llx", i);
 			object_unref(&vmo->vo_base);
 			return NULL;
 		}
@@ -242,36 +276,7 @@ extern struct vm_object *vm_object_create_in_place(
 	return vmo;
 }
 
-extern struct vm_page *vm_object_get_page(
+static struct vm_page *alloc_page(struct vm_object *vo, off_t offset)
-	const struct vm_object *vo,
-	off_t offset)
-{
-	struct btree_node *cur = vo->vo_pages.b_root;
-	while (cur) {
-		struct vm_page *page
-			= BTREE_CONTAINER(struct vm_page, p_bnode, cur);
-		struct btree_node *next = NULL;
-
-		off_t base = page->p_vmo_offset;
-		off_t limit = base + vm_page_get_size_bytes(page);
-		if (offset < base) {
-			next = btree_left(cur);
-		} else if (offset >= limit) {
-			next = btree_right(cur);
-		} else {
-			return page;
-		}
-
-		cur = next;
-	}
-
-	return NULL;
-}
-
-extern struct vm_page *vm_object_alloc_page(
-	struct vm_object *vo,
-	off_t offset,
-	enum vm_page_order size)
 {
 	struct vm_page *page = NULL;
 	struct btree_node *cur = vo->vo_pages.b_root;
@@ -281,6 +286,14 @@ extern struct vm_page *vm_object_alloc_page(
 			return NULL;
 		}
 
+		void *page_buf = vm_page_get_vaddr(page);
+		memset(page_buf, 0x0, vm_page_get_size_bytes(page));
+#if 0
+		printk("vm-object: [%s] alloc offset %zx -> page %zx",
+		       vo->vo_name,
+		       offset,
+		       vm_page_get_paddr(page));
+#endif
 		page->p_vmo_offset = offset;
 		vo->vo_pages.b_root = &page->p_bnode;
 		btree_insert_fixup(&vo->vo_pages, &page->p_bnode);
@@ -328,6 +341,89 @@ extern struct vm_page *vm_object_alloc_page(
 	return NULL;
 }
 
+static struct vm_page *get_page(struct vm_object *vo, off_t offset)
+{
+	struct btree_node *cur = vo->vo_pages.b_root;
+	while (cur) {
+		struct vm_page *page
+			= BTREE_CONTAINER(struct vm_page, p_bnode, cur);
+		struct btree_node *next = NULL;
+
+		off_t base = page->p_vmo_offset;
+		off_t limit = base + vm_page_get_size_bytes(page);
+		if (offset < base) {
+			next = btree_left(cur);
+		} else if (offset >= limit) {
+			next = btree_right(cur);
+		} else {
+			return page;
+		}
+
+		cur = next;
+	}
+
+	return NULL;
+}
+
+static kern_status_t request_page(
+	struct vm_object *vo,
+	off_t offset,
+	unsigned long *irq_flags)
+{
+	struct vm_controller *ctrl = vo->vo_ctrl;
+	struct page_request req = {0};
+	req.req_status = PAGE_REQUEST_PENDING;
+	req.req_type = PAGE_REQUEST_READ;
+	req.req_offset = offset;
+	req.req_length = vm_page_order_to_bytes(VM_PAGE_4K);
+	req.req_sender = current_thread();
+
+	object_ref(&vo->vo_base);
+	req.req_object = vo->vo_key;
+
+	vm_object_unlock_irqrestore(vo, *irq_flags);
+	vm_controller_lock_irqsave(ctrl, irq_flags);
+	spin_lock(&req.req_lock);
+
+	kern_status_t status
+		= vm_controller_send_request(ctrl, &req, irq_flags);
+
+	spin_unlock(&req.req_lock);
+	vm_controller_unlock_irqrestore(ctrl, *irq_flags);
+	object_unref(&vo->vo_base);
+	vm_object_lock_irqsave(vo, irq_flags);
+
+	return status;
+}
+
+struct vm_page *vm_object_get_page(
+	struct vm_object *vo,
+	off_t offset,
+	enum vm_object_flags flags,
+	unsigned long *irq_flags)
+{
+	offset &= ~VM_PAGE_MASK;
+	if (!vo->vo_ctrl && (flags & VMO_ALLOCATE_MISSING_PAGE)) {
+		return alloc_page(vo, offset);
+	}
+
+	struct vm_page *pg = get_page(vo, offset);
+	if (pg) {
+		return pg;
+	}
+
+	if (!vo->vo_ctrl) {
+		return NULL;
+	}
+
+	kern_status_t status = request_page(vo, offset, irq_flags);
+	if (status != KERN_OK) {
+		return NULL;
+	}
+
+	return get_page(vo, offset);
+}
+
 #if 0
 /* read data from a vm-object, where [offset, offset+count] is confined to
  * a single page */
@@ -768,3 +864,77 @@ kern_status_t vm_object_copy(
 
 	return KERN_OK;
 }
+
+kern_status_t vm_object_transfer(
+	struct vm_object *dst,
+	off_t dst_offset,
+	struct vm_object *src,
+	off_t src_offset,
+	size_t count,
+	size_t *nr_moved)
+{
+	dst_offset &= ~VM_PAGE_MASK;
+	src_offset &= ~VM_PAGE_MASK;
+
+	if (count & VM_PAGE_MASK) {
+		count &= ~VM_PAGE_MASK;
+		count += VM_PAGE_SIZE;
+	}
+
+	size_t moved = 0;
+	for (size_t i = 0; i < count; i += VM_PAGE_SIZE) {
+		struct vm_page *src_pg
+			= vm_object_get_page(src, src_offset + i, 0, NULL);
+		if (!src_pg) {
+			continue;
+		}
+
+		btree_delete(&src->vo_pages, &src_pg->p_bnode);
+
+		struct vm_page *dst_pg
+			= vm_object_get_page(src, dst_offset + i, 0, NULL);
+		if (dst_pg) {
+			vm_page_free(src_pg);
+			continue;
+		}
+
+		put_page(dst, src_pg, dst_offset + i);
+		moved += VM_PAGE_SIZE;
+	}
+
+	struct queue_entry *cur = queue_first(&src->vo_mappings);
+	off_t src_limit = src_offset + count - 1;
+	while (cur) {
+		struct vm_area *area = QUEUE_CONTAINER(
+			struct vm_area,
+			vma_object_entry,
+			cur);
+		off_t area_offset = area->vma_object_offset;
+		off_t area_limit
+			= area_offset + (area->vma_limit - area->vma_base);
+
+		if (src_offset > area_limit || src_limit < area_offset) {
+			cur = queue_next(cur);
+			continue;
+		}
+
+		off_t unmap_offset = MAX(area_offset, src_offset);
+		off_t unmap_limit = MIN(area_limit, src_limit);
+
+		virt_addr_t base
+			= area->vma_base + (unmap_offset - area_offset);
+		virt_addr_t limit = base + (unmap_limit - unmap_offset);
+
+		for (virt_addr_t i = base; i < limit; i += VM_PAGE_SIZE) {
+			pmap_remove(area->vma_space->s_pmap, i);
+		}
+
+		cur = queue_next(cur);
+	}
+
+	if (nr_moved) {
+		*nr_moved = moved;
+	}
+
+	return KERN_OK;
+}