meta: rename kernel to Magenta

whether a mapping is private or shared determines how the mapping is handled
when a task is duplicated.

CMakeLists.txt

@@ -1,5 +1,5 @@
-cmake_minimum_required(VERSION 4.0)
+cmake_minimum_required(VERSION 3.31)
-project(mango C ASM)
+project(magenta C ASM)
 
 if (NOT BUILD_TOOLS_DIR)
 	message(FATAL_ERROR "No build tools directory specified. Please run build.sh")
@@ -9,8 +9,8 @@ set(CMAKE_C_STANDARD 17)
 
 set(kernel_arch x86_64)
 
-set(kernel_name "Mango")
+set(kernel_name "Magenta")
-set(kernel_exe_name "mango_kernel")
+set(kernel_exe_name "magenta_kernel")
 
 set(generic_src_dirs ds init kernel libc sched util vm syscall)
 set(kernel_sources "")
@@ -38,7 +38,7 @@ add_executable(${kernel_exe_name}
 target_include_directories(${kernel_exe_name} PRIVATE
 	include
 	libc/include
-	libmango/include
+	libmagenta/include
 	arch/${kernel_arch}/include)
 target_compile_options(${kernel_exe_name} PRIVATE
 	-nostdlib -ffreestanding

README

@@ -1,4 +1,4 @@
-Mango
+Magenta
 =====
 
 It's a kernel!

arch/user/include/mango/machine/cpu.h

@@ -1,5 +1,5 @@
-#ifndef MANGO_USER_CPU_H_
+#ifndef MAGENTA_USER_CPU_H_
-#define MANGO_USER_CPU_H_
+#define MAGENTA_USER_CPU_H_
 
 #ifdef __cplusplus
 extern "C" {

arch/user/include/mango/machine/hwlock.h

@@ -1,5 +1,5 @@
-#ifndef MANGO_USER_HWLOCK_H_
+#ifndef MAGENTA_USER_HWLOCK_H_
-#define MANGO_USER_HWLOCK_H_
+#define MAGENTA_USER_HWLOCK_H_
 
 #define ML_HWLOCK_INIT (0)
 

arch/user/include/mango/machine/init.h

@@ -1,5 +1,5 @@
-#ifndef MANGO_X86_64_INIT_H_
+#ifndef MAGENTA_X86_64_INIT_H_
-#define MANGO_X86_64_INIT_H_
+#define MAGENTA_X86_64_INIT_H_
 
 #include <stddef.h>
 #include <stdint.h>

arch/user/include/mango/machine/irq.h

@@ -1,5 +1,5 @@
-#ifndef MANGO_X86_64_IRQ_H_
+#ifndef MAGENTA_X86_64_IRQ_H_
-#define MANGO_X86_64_IRQ_H_
+#define MAGENTA_X86_64_IRQ_H_
 
 
 #endif

arch/user/include/mango/machine/pmap.h

@@ -1,5 +1,5 @@
-#ifndef MANGO_USER_PMAP_H_
+#ifndef MAGENTA_USER_PMAP_H_
-#define MANGO_USER_PMAP_H_
+#define MAGENTA_USER_PMAP_H_
 
 #include <stdint.h>
 

arch/user/include/mango/machine/vm.h

@@ -1,5 +1,5 @@
-#ifndef MANGO_USER_VM_H_
+#ifndef MAGENTA_USER_VM_H_
-#define MANGO_USER_VM_H_
+#define MAGENTA_USER_VM_H_
 
 #include <stdint.h>
 

arch/x86_64/include/arch/paging.h

@@ -1,8 +1,8 @@
 #ifndef ARCH_PAGING_H_
 #define ARCH_PAGING_H_
 
-#include <kernel/types.h>
 #include <kernel/compiler.h>
+#include <kernel/types.h>
 
 #ifdef __cplusplus
 extern "C" {
@@ -55,6 +55,7 @@ enum page_size {
    defined in pmap_ctrl.S */
 extern int gigabyte_pages(void);
 extern int enable_nx(void);
+extern int enable_wp(void);
 
 #ifdef __cplusplus
 }

arch/x86_64/include/kernel/machine/thread.h

@@ -31,6 +31,14 @@ extern kern_status_t ml_thread_prepare_user_context(
 	virt_addr_t *kernel_sp,
 	const uintptr_t *args,
 	size_t nr_args);
+/* prepare the stack so that ml_thread_switch_user can jump to usermode
+ * with the specified register context */
+extern kern_status_t ml_thread_clone_user_context(
+	const struct ml_cpu_context *src_regs,
+	const struct ml_thread *src_ml,
+	struct ml_thread *dest_ml,
+	uintptr_t return_value,
+	virt_addr_t *kernel_sp);
 
 extern kern_status_t ml_thread_config_get(
 	struct thread *thread,

arch/x86_64/init.c

@@ -20,7 +20,7 @@
 #include <kernel/util.h>
 #include <kernel/vm.h>
 
-#undef HARDWARE_RNG
+#define HARDWARE_RNG
 
 #define PTR32(x)            ((void *)((uintptr_t)(x)))
 

arch/x86_64/irq.c

@@ -8,6 +8,7 @@
 #include <kernel/panic.h>
 #include <kernel/sched.h>
 #include <kernel/syscall.h>
+#include <kernel/thread.h>
 #include <stddef.h>
 
 #define MAX_ISR_HANDLERS  16
@@ -166,6 +167,12 @@ int idt_load(struct idt_ptr *ptr)
 
 void isr_dispatch(struct ml_cpu_context *regs)
 {
+	struct thread *thr = get_current_thread();
+	if (thr) {
+		thr->tr_irqctx = regs;
+		put_current_thread(thr);
+	}
+
 	int_hook h = isr_handlers[regs->int_no];
 	if (h) {
 		h(regs);
@@ -188,6 +195,13 @@ void irq_dispatch(struct ml_cpu_context *regs)
 	end_charge_period();
 
 	irq_ack(regs->int_no);
+
+	struct thread *thr = get_current_thread();
+	if (thr) {
+		thr->tr_irqctx = regs;
+		put_current_thread(thr);
+	}
+
 	struct queue *hooks = &irq_hooks[regs->int_no - IRQ0];
 	queue_foreach(struct irq_hook, hook, hooks, irq_entry)
 	{
@@ -203,6 +217,12 @@ void irq_dispatch(struct ml_cpu_context *regs)
 
 void syscall_dispatch(struct ml_cpu_context *regs)
 {
+	struct thread *thr = get_current_thread();
+	if (thr) {
+		thr->tr_irqctx = regs;
+		put_current_thread(thr);
+	}
+
 	unsigned int sysid = regs->rax;
 	virt_addr_t syscall_impl = syscall_get_function(sysid);
 

arch/x86_64/panic.c

@@ -217,16 +217,18 @@ static void print_stack_trace(
 
 void ml_print_stack_trace(uintptr_t ip)
 {
-	struct task *task = current_task();
+	struct task *task = get_current_task();
 	struct address_space *space = task ? task->t_address_space : NULL;
 	uintptr_t bp;
 	asm volatile("mov %%rbp, %0" : "=r"(bp));
 	print_stack_trace(space, ip, bp);
+	put_current_task(task);
 }
 
 void ml_print_stack_trace_irq(struct ml_cpu_context *ctx)
 {
-	struct task *task = current_task();
+	struct task *task = get_current_task();
 	struct address_space *space = task ? task->t_address_space : NULL;
 	print_stack_trace(space, ctx->rip, ctx->rbp);
+	put_current_task(task);
 }

arch/x86_64/pmap.c

@@ -9,7 +9,7 @@
 #include <kernel/types.h>
 #include <kernel/vm-object.h>
 #include <kernel/vm.h>
-#include <mango/status.h>
+#include <magenta/status.h>
 
 /* some helpful datasize constants */
 #define C_1GiB          0x40000000ULL
@@ -19,7 +19,7 @@
 #define PTR_TO_ENTRY(x) (((x) & ~VM_PAGE_MASK) | PTE_PRESENT | PTE_RW | PTE_USR)
 #define ENTRY_TO_PTR(x) ((x) & ~VM_PAGE_MASK)
 
-#define PFN(x)          ((x) >> VM_PAGE_SHIFT)
+#define PFN(x)          (((x) >> VM_PAGE_SHIFT) & 0xFFFFFFFFFF)
 
 static int can_use_gbpages = 0;
 static pmap_t kernel_pmap;
@@ -136,6 +136,81 @@ static void delete_pdir(phys_addr_t pd)
 	kfree(pdir);
 }
 
+kern_status_t pmap_get(
+	pmap_t pmap,
+	virt_addr_t pv,
+	pfn_t *out_pfn,
+	vm_prot_t *out_prot)
+{
+	unsigned int pml4t_index = BAD_INDEX, pdpt_index = BAD_INDEX,
+		     pd_index = BAD_INDEX, pt_index = BAD_INDEX;
+
+	pml4t_index = (pv >> 39) & 0x1FF;
+	pdpt_index = (pv >> 30) & 0x1FF;
+	pd_index = (pv >> 21) & 0x1FF;
+	pt_index = (pv >> 12) & 0x1FF;
+
+	/* 1. get PML4T (mandatory) */
+	struct pml4t *pml4t = vm_phys_to_virt(ENTRY_TO_PTR(pmap));
+	if (!pml4t) {
+		return KERN_INVALID_ARGUMENT;
+	}
+
+	/* 2. traverse PML4T, get PDPT (mandatory) */
+	struct pdpt *pdpt = NULL;
+	if (!pml4t->p_entries[pml4t_index]) {
+		return KERN_NO_ENTRY;
+	} else {
+		pdpt = vm_phys_to_virt(
+			ENTRY_TO_PTR(pml4t->p_entries[pml4t_index]));
+	}
+
+	/* 3. traverse PDPT, get PDIR (optional, 4K and 2M only) */
+	struct pdir *pdir = NULL;
+	if (!pdpt->p_entries[pdpt_index]
+	    || pdpt->p_pages[pdpt_index] & PTE_PAGESIZE) {
+		return KERN_NO_ENTRY;
+	} else {
+		pdir = vm_phys_to_virt(
+			ENTRY_TO_PTR(pdpt->p_entries[pdpt_index]));
+	}
+
+	/* 4. traverse PDIR, get PTAB (optional, 4K only) */
+	struct ptab *ptab = NULL;
+	if (!pdir->p_entries[pd_index]
+	    || pdir->p_pages[pd_index] & PTE_PAGESIZE) {
+		/* entry is null, or points to a hugepage */
+		return KERN_NO_ENTRY;
+	} else {
+		ptab = vm_phys_to_virt(ENTRY_TO_PTR(pdir->p_entries[pd_index]));
+	}
+
+	uint64_t pte = ptab->p_pages[pt_index];
+	if (out_pfn) {
+		*out_pfn = PFN(pte);
+	}
+
+	if (out_prot) {
+		if (pte & PTE_PRESENT) {
+			*out_prot |= VM_PROT_USER;
+		}
+
+		if (pte & PTE_RW) {
+			*out_prot |= (VM_PROT_READ | VM_PROT_WRITE);
+		}
+
+		if (pte & PTE_USR) {
+			*out_prot |= VM_PROT_USER;
+		}
+
+		if (!(pte & PTE_NX)) {
+			*out_prot |= VM_PROT_EXEC;
+		}
+	}
+
+	return KERN_OK;
+}
+
 static kern_status_t do_pmap_add(
 	pmap_t pmap,
 	virt_addr_t pv,
@@ -246,6 +321,10 @@ static kern_status_t do_pmap_remove(
 	virt_addr_t pv,
 	enum page_size size)
 {
+	if (pmap == PMAP_INVALID) {
+		return KERN_OK;
+	}
+
 	unsigned int pml4t_index = BAD_INDEX, pdpt_index = BAD_INDEX,
 		     pd_index = BAD_INDEX, pt_index = BAD_INDEX;
 
@@ -351,6 +430,8 @@ void pmap_bootstrap(void)
 	       can_use_gbpages == 1 ? "en" : "dis");
 	enable_nx();
 	printk("pmap: NX protection enabled");
+	enable_wp();
+	printk("pmap: kernel-mode write protection enabled");
 
 	enum page_size hugepage = PS_2M;
 	if (can_use_gbpages) {
@@ -494,11 +575,7 @@ kern_status_t pmap_handle_fault(
 {
 	// log_fault(fault_addr, flags);
 
-	if (flags & PMAP_FAULT_PRESENT) {
+	struct task *task = get_current_task();
-		return KERN_FATAL_ERROR;
-	}
-
-	struct task *task = current_task();
 	if (!task) {
 		return KERN_FATAL_ERROR;
 	}
@@ -509,7 +586,11 @@ kern_status_t pmap_handle_fault(
 	}
 
 	/* this must be called with `space` unlocked. */
-	return address_space_demand_map(space, fault_addr, flags);
+	kern_status_t status
+		= address_space_demand_map(space, fault_addr, flags);
+
+	put_current_task(task);
+	return status;
 }
 
 kern_status_t pmap_add(
@@ -545,5 +626,13 @@ kern_status_t pmap_remove(pmap_t pmap, virt_addr_t p)
 
 kern_status_t pmap_remove_range(pmap_t pmap, virt_addr_t p, size_t len)
 {
+	if (pmap == PMAP_INVALID) {
+		return KERN_OK;
+	}
+
+	for (size_t i = p; i < p + len; i += VM_PAGE_SIZE) {
+		pmap_remove(pmap, i);
+	}
+
 	return KERN_OK;
 }

arch/x86_64/pmap_ctrl.S

@@ -1,3 +1,11 @@
+	.global pmap_flush
+	.type pmap_flush, @function
+
+pmap_flush:
+	mov %cr3, %rax
+	mov %rax, %cr3
+	ret
+
 
 	.global pmap_switch
 	.type pmap_switch, @function
@@ -6,6 +14,7 @@ pmap_switch:
 	mov %rdi, %cr3
 	ret
 
+
 	.global gigabyte_pages
 	.type gigabyte_pages, @function
 
@@ -30,6 +39,7 @@ gigabyte_pages:
 	pop %rbp
 	ret
 
+
 	.global enable_nx
 	.type enable_nx, @function
 
@@ -40,3 +50,14 @@ enable_nx:
 	wrmsr
 
 	ret
+
+
+	.global enable_wp
+	.type enable_wp, @function
+
+enable_wp:
+	mov %cr0, %rax
+	or $0x10000, %rax
+	mov %rax, %cr0
+
+	ret

arch/x86_64/serial.c

@@ -28,6 +28,10 @@ void serial_send_byte(int device, char out)
 
 	outportb(device, out);
 
+	if (device == COM1) {
+		outportb(0xe9, out);
+	}
+
 	while (!transmit_empty(device)) {
 		_count++;
 	}

arch/x86_64/thread.c

@@ -80,6 +80,26 @@ extern kern_status_t ml_thread_prepare_user_context(
 	return KERN_OK;
 }
 
+kern_status_t ml_thread_clone_user_context(
+	const struct ml_cpu_context *src_regs,
+	const struct ml_thread *src_ml,
+	struct ml_thread *dest_ml,
+	uintptr_t return_value,
+	virt_addr_t *kernel_sp)
+{
+	(*kernel_sp) -= sizeof(struct ml_cpu_context);
+
+	struct ml_cpu_context *regs = (struct ml_cpu_context *)(*kernel_sp);
+	memcpy(regs, src_regs, sizeof *regs);
+
+	regs->rax = return_value;
+
+	dest_ml->tr_fsbase = src_ml->tr_fsbase;
+	dest_ml->tr_gsbase = src_ml->tr_gsbase;
+
+	return KERN_OK;
+}
+
 kern_status_t ml_thread_config_get(
 	struct thread *thread,
 	kern_config_key_t key,
@@ -95,25 +115,30 @@ kern_status_t ml_thread_config_set(
 	const void *ptr,
 	size_t len)
 {
+	struct thread *self = get_current_thread();
+	kern_status_t status = KERN_OK;
+
 	switch (key) {
 	case THREAD_CFG_FSBASE:
 		if (len != sizeof(thread->tr_ml.tr_fsbase)) {
-			return KERN_INVALID_ARGUMENT;
+			status = KERN_INVALID_ARGUMENT;
+			break;
 		}
 
 		thread->tr_ml.tr_fsbase = *(virt_addr_t *)ptr;
-		if (thread == current_thread()) {
+		if (thread == self) {
 			wrmsr(MSR_FS_BASE, thread->tr_ml.tr_fsbase);
 		}
 
 		break;
 	case THREAD_CFG_GSBASE:
 		if (len != sizeof(thread->tr_ml.tr_gsbase)) {
-			return KERN_INVALID_ARGUMENT;
+			status = KERN_INVALID_ARGUMENT;
+			break;
 		}
 
 		thread->tr_ml.tr_gsbase = *(virt_addr_t *)ptr;
-		if (thread == current_thread()) {
+		if (thread == self) {
 			/* we're in the kernel right now, so the user and kernel
 			 * gs-base registers are swapped. when we return to
 			 * usermode, this value will be swapped back into
@@ -123,8 +148,10 @@ kern_status_t ml_thread_config_set(
 
 		break;
 	default:
-		return KERN_INVALID_ARGUMENT;
+		status = KERN_INVALID_ARGUMENT;
+		break;
 	}
 
-	return KERN_OK;
+	put_current_thread(self);
+	return status;
 }

arch/x86_64/toolchain.cmake

@@ -1,5 +1,5 @@
 # the name of the target operating system
-set(CMAKE_SYSTEM_NAME Mango)
+set(CMAKE_SYSTEM_NAME Magenta)
 
 # which compilers to use for C and C++
 set(CMAKE_C_COMPILER   x86_64-elf-gcc)

include/kernel/address-space.h

@@ -22,6 +22,8 @@ struct vm_area {
 	struct address_space *vma_space;
 	/* used to link to vm_object->vo_mappings */
 	struct queue_entry vma_object_entry;
+	/* the memory control flags applied to this area */
+	vm_flags_t vma_flags;
 	/* the memory protection flags applied to this area */
 	vm_prot_t vma_prot;
 	/* offset in bytes to the start of the object data that was mapped */
@@ -83,6 +85,7 @@ extern kern_status_t address_space_map(
 	struct vm_object *object,
 	off_t object_offset,
 	size_t length,
+	vm_flags_t flags,
 	vm_prot_t prot,
 	virt_addr_t *out);
 extern kern_status_t address_space_unmap(
@@ -105,6 +108,12 @@ extern kern_status_t address_space_release(
 	virt_addr_t base,
 	size_t length);
 
+/* duplicate all of the mappings in `src` within `dest. the duplication will use
+ * copy-on-write; page data will not be copied until it is written to. */
+extern kern_status_t address_space_duplicate(
+	struct address_space *dest,
+	struct address_space *src);
+
 extern bool address_space_validate_access(
 	struct address_space *region,
 	virt_addr_t base,

include/kernel/arg.h

@@ -1,7 +1,7 @@
 #ifndef KERNEL_ARG_H_
 #define KERNEL_ARG_H_
 
-#include <mango/types.h>
+#include <magenta/types.h>
 #include <stdbool.h>
 
 #define CMDLINE_MAX 4096

include/kernel/atomic.h

@@ -4,7 +4,7 @@
 #include <stdbool.h>
 #include <stdint.h>
 
-typedef int64_t atomic_t;
+typedef int32_t atomic_t;
 
 /* load and return the value pointed to by `v` */
 static inline atomic_t atomic_load(atomic_t *v)

include/kernel/bsp.h

@@ -2,7 +2,7 @@
 #define KERNEL_BSP_H_
 
 #include <kernel/compiler.h>
-#include <mango/status.h>
+#include <magenta/status.h>
 #include <kernel/types.h>
 #include <stddef.h>
 #include <stdint.h>

include/kernel/console.h

@@ -16,8 +16,8 @@
 */
 #include <kernel/locks.h>
 #include <kernel/queue.h>
-#include <mango/status.h>
+#include <magenta/status.h>
-#include <mango/types.h>
+#include <magenta/types.h>
 
 #ifdef __cplusplus
 extern "C" {

include/kernel/equeue.h

@@ -4,7 +4,7 @@
 #include <kernel/locks.h>
 #include <kernel/object.h>
 #include <kernel/sched.h>
-#include <mango/types.h>
+#include <magenta/types.h>
 
 #define EQUEUE_PACKET_MAX 100
 

include/kernel/futex.h

@@ -3,7 +3,7 @@
 
 #include <kernel/btree.h>
 #include <kernel/wait.h>
-#include <mango/types.h>
+#include <magenta/types.h>
 
 struct task;
 struct address_space;

include/kernel/handle.h

@@ -2,8 +2,8 @@
 #define KERNEL_HANDLE_H_
 
 #include <kernel/bitmap.h>
-#include <mango/status.h>
+#include <magenta/status.h>
-#include <mango/types.h>
+#include <magenta/types.h>
 #include <stddef.h>
 #include <stdint.h>
 
@@ -44,6 +44,9 @@ struct handle_table {
 
 extern struct handle_table *handle_table_create(void);
 extern void handle_table_destroy(struct handle_table *tab);
+extern kern_status_t handle_table_duplicate(
+	struct handle_table *src,
+	struct handle_table **dest);
 
 extern kern_status_t handle_table_alloc_handle(
 	struct handle_table *tab,

include/kernel/iovec.h

@@ -1,7 +1,7 @@
 #ifndef KERNEL_IOVEC_H_
 #define KERNEL_IOVEC_H_
 
-#include <mango/types.h>
+#include <magenta/types.h>
 #include <stddef.h>
 
 struct address_space;

include/kernel/msg.h

@@ -3,8 +3,8 @@
 
 #include <kernel/btree.h>
 #include <kernel/locks.h>
-#include <mango/status.h>
+#include <magenta/status.h>
-#include <mango/types.h>
+#include <magenta/types.h>
 
 struct port;
 struct thread;

include/kernel/object.h

@@ -5,7 +5,7 @@
 #include <kernel/locks.h>
 #include <kernel/vm.h>
 #include <kernel/wait.h>
-#include <mango/status.h>
+#include <magenta/status.h>
 #include <stddef.h>
 
 #ifdef __cplusplus
@@ -65,7 +65,7 @@ extern "C" {
 #define OBJECT_PATH_MAX 256
 
 #define OBJECT_CAST(to_type, to_type_member, p)                                \
-	((to_type *)((uintptr_t)p) - offsetof(to_type, to_type_member))
+	((to_type *)(((uintptr_t)p) - offsetof(to_type, to_type_member)))
 #define OBJECT_C_CAST(c_type, c_type_member, obj_type, objp)                   \
 	OBJECT_IS_TYPE(objp, obj_type)                                         \
 	? OBJECT_CAST(c_type, c_type_member, (objp)) : NULL

include/kernel/percpu.h

@@ -1,7 +1,7 @@
 #ifndef KERNEL_PERCPU_H_
 #define KERNEL_PERCPU_H_
 
-#include <mango/status.h>
+#include <magenta/status.h>
 #include <kernel/compiler.h>
 #include <kernel/sched.h>
 

include/kernel/pmap.h

@@ -5,11 +5,10 @@
 
 #include <kernel/machine/pmap.h>
 #include <kernel/vm.h>
-#include <mango/status.h>
+#include <magenta/status.h>
 #include <stddef.h>
 
 #define PMAP_INVALID ML_PMAP_INVALID
-#define PFN(x)       ((x) >> VM_PAGE_SHIFT)
 
 #ifdef __cplusplus
 extern "C" {
@@ -53,10 +52,19 @@ extern pmap_t pmap_create(void);
 extern void pmap_destroy(pmap_t pmap);
 extern void pmap_switch(pmap_t pmap);
 
+extern void pmap_flush(void);
+extern void pmap_flush_page(virt_addr_t p);
+extern void pmap_flush_range(virt_addr_t start, size_t length);
+
 extern kern_status_t pmap_handle_fault(
 	virt_addr_t fault_addr,
 	enum pmap_fault_flags flags);
 
+extern kern_status_t pmap_get(
+	pmap_t pmap,
+	virt_addr_t p,
+	pfn_t *out_pfn,
+	vm_prot_t *out_prot);
 extern kern_status_t pmap_add(
 	pmap_t pmap,
 	virt_addr_t p,

include/kernel/sched.h

@@ -6,7 +6,7 @@
 #include <kernel/locks.h>
 #include <kernel/queue.h>
 #include <kernel/types.h>
-#include <mango/status.h>
+#include <magenta/status.h>
 
 #define PRIO_MAX 32
 
@@ -76,8 +76,10 @@ extern struct runqueue *cpu_rq(unsigned int cpu);
 extern cycles_t default_quantum(void);
 
 extern bool need_resched(void);
-extern struct task *current_task(void);
+extern struct task *get_current_task(void);
-extern struct thread *current_thread(void);
+extern struct thread *get_current_thread(void);
+extern void put_current_task(struct task *task);
+extern void put_current_thread(struct thread *thread);
 
 extern struct runqueue *select_rq_for_thread(struct thread *thr);
 extern void schedule_thread_on_cpu(struct thread *thr);

include/kernel/syscall.h

@@ -5,8 +5,8 @@
 #include <kernel/handle.h>
 #include <kernel/task.h>
 #include <kernel/vm.h>
-#include <mango/status.h>
+#include <magenta/status.h>
-#include <mango/syscall.h>
+#include <magenta/syscall.h>
 
 #define validate_access(task, ptr, len, flags)                                 \
 	__validate_access(task, (const void *)ptr, len, flags)
@@ -42,6 +42,7 @@ extern kern_status_t sys_task_exit(int status);
 extern kern_status_t sys_task_self(kern_handle_t *out);
 extern kern_status_t sys_task_create(
 	kern_handle_t parent_handle,
+	task_flags_t flags,
 	const char *name,
 	size_t name_len,
 	kern_handle_t *out_task,
@@ -66,6 +67,9 @@ extern kern_status_t sys_task_config_set(
 	kern_config_key_t key,
 	const void *ptr,
 	size_t len);
+extern kern_status_t sys_task_duplicate(
+	kern_handle_t *out_task,
+	kern_handle_t *out_address_space);
 
 extern kern_status_t sys_thread_self(kern_handle_t *out);
 extern kern_status_t sys_thread_start(kern_handle_t thread);
@@ -125,6 +129,7 @@ extern kern_status_t sys_address_space_map(
 	kern_handle_t object,
 	off_t object_offset,
 	size_t length,
+	vm_flags_t flags,
 	vm_prot_t prot,
 	virt_addr_t *out_base_address);
 extern kern_status_t sys_address_space_unmap(
@@ -150,6 +155,12 @@ extern kern_status_t sys_kern_handle_transfer(
 	kern_handle_t dest_handle,
 	unsigned int mode,
 	kern_handle_t *out_handle);
+extern kern_status_t sys_kern_handle_control(
+	kern_handle_t task,
+	kern_handle_t handle,
+	uint32_t set_mask,
+	uint32_t clear_mask,
+	uint32_t *out_flags);
 extern kern_status_t sys_kern_config_get(
 	kern_config_key_t key,
 	void *ptr,
@@ -202,7 +213,7 @@ extern kern_status_t sys_kern_object_query(
 extern kern_status_t sys_vm_controller_create(kern_handle_t *out);
 extern kern_status_t sys_vm_controller_recv(
 	kern_handle_t ctrl,
-	equeue_packet_page_request_t *out);
+	equeue_packet_vm_request_t *out);
 extern kern_status_t sys_vm_controller_recv_async(
 	kern_handle_t ctrl,
 	kern_handle_t eq,
@@ -215,6 +226,14 @@ extern kern_status_t sys_vm_controller_create_object(
 	size_t data_len,
 	vm_prot_t prot,
 	kern_handle_t *out);
+extern kern_status_t sys_vm_controller_prepare_attach(
+	kern_handle_t ctrl,
+	uint64_t req_id,
+	kern_handle_t *out_vmo);
+extern kern_status_t sys_vm_controller_finish_attach(
+	kern_handle_t ctrl,
+	uint64_t req_id,
+	equeue_key_t new_key);
 extern kern_status_t sys_vm_controller_detach_object(
 	kern_handle_t ctrl,
 	kern_handle_t vmo);

include/kernel/task.h

@@ -39,7 +39,11 @@ struct task {
 
 extern struct task *task_alloc(void);
 extern struct task *task_cast(struct object *obj);
-extern struct task *task_create(const char *name, size_t name_len);
+extern struct task *task_create(
+	struct task *parent,
+	task_flags_t flags,
+	const char *name,
+	size_t name_len);
 static inline struct task *task_ref(struct task *task)
 {
 	return OBJECT_CAST(struct task, t_base, object_ref(&task->t_base));
@@ -66,6 +70,16 @@ extern kern_status_t task_resolve_handle(
 	kern_handle_t handle,
 	struct object **out_obj,
 	handle_flags_t *out_flags);
+extern kern_status_t task_config_get(
+	struct task *task,
+	kern_config_key_t key,
+	void *out,
+	size_t max);
+extern kern_status_t task_config_set(
+	struct task *task,
+	kern_config_key_t key,
+	const void *ptr,
+	size_t len);
 extern kern_status_t task_close_handle(struct task *task, kern_handle_t handle);
 extern struct thread *task_create_thread(struct task *parent);
 extern struct task *kernel_task(void);

include/kernel/thread.h

@@ -8,6 +8,8 @@
 
 #define THREAD_KSTACK_ORDER VM_PAGE_4K
 
+struct ml_cpu_context;
+
 enum thread_state {
 	THREAD_READY = 1,
 	THREAD_SLEEPING = 2,
@@ -39,6 +41,9 @@ struct thread {
 	virt_addr_t tr_ip, tr_sp, tr_bp;
 	virt_addr_t tr_cpu_user_sp, tr_cpu_kernel_sp;
 
+	/* only valid within an interrupt or syscall context */
+	struct ml_cpu_context *tr_irqctx;
+
 	struct ml_thread tr_ml;
 	struct runqueue *tr_rq;
 
@@ -57,6 +62,10 @@ extern kern_status_t thread_init_user(
 	virt_addr_t sp,
 	const uintptr_t *args,
 	size_t nr_args);
+extern kern_status_t thread_init_user_clone(
+	struct thread *thr,
+	const struct thread *src,
+	uintptr_t return_value);
 extern int thread_priority(struct thread *thr);
 extern void thread_awaken(struct thread *thr);
 extern void thread_exit(void);

include/kernel/types.h

@@ -1,7 +1,7 @@
 #ifndef KERNEL_TYPES_H_
 #define KERNEL_TYPES_H_
 
-#include <mango/types.h>
+#include <magenta/types.h>
 #include <stddef.h>
 #include <stdint.h>
 

include/kernel/util.h

@@ -1,7 +1,7 @@
 #ifndef KERNEL_UTIL_H_
 #define KERNEL_UTIL_H_
 
-#include <mango/types.h>
+#include <magenta/types.h>
 #include <stdbool.h>
 #include <stddef.h>
 #include <stdint.h>

include/kernel/vm-controller.h

@@ -3,40 +3,41 @@
 
 #include <kernel/locks.h>
 #include <kernel/object.h>
-#include <mango/types.h>
+#include <magenta/types.h>
 
 struct thread;
 struct equeue;
 struct vm_object;
 
-enum page_request_status {
+enum vm_request_status {
-	PAGE_REQUEST_PENDING = 0,
+	VM_REQUEST_PENDING = 0,
-	PAGE_REQUEST_IN_PROGRESS,
+	VM_REQUEST_IN_PROGRESS,
-	PAGE_REQUEST_COMPLETE,
+	VM_REQUEST_COMPLETE,
-	PAGE_REQUEST_ASYNC,
+	VM_REQUEST_ASYNC,
 };
 
 struct vm_controller {
 	struct object vc_base;
-	/* tree of pending page requests */
+	/* tree of pending vm requests */
 	struct btree vc_requests;
-	/* the equeue to send async page requests to */
 	struct equeue *vc_eq;
 	equeue_key_t vc_eq_key;
-	/* the number of page requests queued with status PAGE_REQUEST_PENDING.
+	/* the number of page requests queued with status VM_REQUEST_PENDING.
 	 * used to assert/clear VM_CONTROLLER_SIGNAL_REQUEST_RECEIVED */
 	size_t vc_requests_waiting;
 };
 
-struct page_request {
+struct vm_request {
 	uint64_t req_id;
 	unsigned int req_type;
-	enum page_request_status req_status;
+	enum vm_request_status req_status;
 	kern_status_t req_result;
 	spin_lock_t req_lock;
-	equeue_key_t req_object;
+	struct vm_object *req_object;
 	struct thread *req_sender;
+	/* this node is added to vm-controller vc_requests list */
 	struct btree_node req_node;
+	/* these values are used for VM_REQUEST_READ and VM_REQUEST_DIRTY */
 	off_t req_offset;
 	size_t req_length;
 };
@@ -48,7 +49,7 @@ extern struct vm_controller *vm_controller_create(void);
 
 extern kern_status_t vm_controller_recv(
 	struct vm_controller *ctrl,
-	equeue_packet_page_request_t *out);
+	equeue_packet_vm_request_t *out);
 extern kern_status_t vm_controller_recv_async(
 	struct vm_controller *ctrl,
 	struct equeue *eq,
@@ -62,6 +63,14 @@ extern kern_status_t vm_controller_create_object(
 	size_t data_len,
 	vm_prot_t prot,
 	struct vm_object **out);
+extern kern_status_t vm_controller_prepare_attach(
+	struct vm_controller *ctrl,
+	uint64_t req_id,
+	struct vm_object **out_vmo);
+extern kern_status_t vm_controller_finish_attach(
+	struct vm_controller *ctrl,
+	uint64_t req_id,
+	equeue_key_t new_key);
 extern kern_status_t vm_controller_detach_object(
 	struct vm_controller *ctrl,
 	struct vm_object *vmo);
@@ -72,10 +81,16 @@ extern kern_status_t vm_controller_supply_pages(
 	struct vm_object *src,
 	off_t src_offset,
 	size_t count);
+extern void vm_controller_fulfill_requests(
+	struct vm_controller *ctrl,
+	equeue_key_t object,
+	off_t offset,
+	size_t length,
+	kern_status_t result);
 
 extern kern_status_t vm_controller_send_request(
 	struct vm_controller *ctrl,
-	struct page_request *req,
+	struct vm_request *req,
 	unsigned long *irq_flags);
 
 DEFINE_OBJECT_LOCK_FUNCTION(vm_controller, vc_base)

include/kernel/vm-object.h

@@ -20,19 +20,25 @@ enum vm_object_flags {
 	 * be detached, allowing the server to close the last handle to the
 	 * object and dispose of it. */
 	VMO_AUTO_DETACH = 0x04u,
+	/* this vmo is a duplicate of a vmo that is attached to a vm-controller.
+	 * the duplicate vmo is scheduled to be attached to the same controller,
+	 * but this won't actually happen until the controller is needed to
+	 * fulfill a page request. once the duplicate vmo has been attached to
+	 * the controller, this flag will be cleared. */
+	VMO_LAZY_ATTACH = 0x08u,
 
 	/* these flags are for use with vm_object_get_page */
 	/**************************************************/
 
 	/* if the relevant page hasn't been allocated yet, it will be allocated
 	 * and returned. if this flag isn't specified, NULL will be returned. */
-	VMO_ALLOCATE_MISSING_PAGE = 0x08u,
+	VMO_ALLOCATE_MISSING_PAGE = 0x0100u,
 	/* if the vm-object is attached to a vm-controller, and the relevant
 	 * page is uncommitted, send a request to the vm-controller to provide
 	 * the missing page. will result in the vm-object being unlocked and
 	 * the current thread sleeping until the request is fulfilled. the
 	 * vm-object will be re-locked before the function returns. */
-	VMO_REQUEST_MISSING_PAGE = 0x10u,
+	VMO_REQUEST_MISSING_PAGE = 0x0200u,
 };
 
 struct vm_object {
@@ -84,11 +90,33 @@ extern struct vm_object *vm_object_create_in_place(
 	size_t data_len,
 	vm_prot_t prot);
 
+/* create a copy-on-write duplicate of a vm-object */
+extern struct vm_object *vm_object_duplicate_cow(struct vm_object *vmo);
+/* attach a copy-on-write duplicate of a vm-object to the vm-controller that
+ * controlled the original vm-object */
+extern kern_status_t vm_object_attach_cow(
+	struct vm_object *vmo,
+	unsigned long *irq_flags);
+
+/* prefetch any missing pages in the specified range of a vm-object.
+ * any lazy-allocated pages will be allocated.
+ * any missing pages will be requested from the vm-controller, if one is
+ * attached. */
+extern kern_status_t vm_object_prefetch(
+	struct vm_object *vo,
+	off_t offset,
+	size_t length,
+	unsigned long *irq_flags);
+
 extern struct vm_page *vm_object_get_page(
 	struct vm_object *vo,
 	off_t offset,
 	enum vm_object_flags flags,
 	unsigned long *irq_flags);
+extern kern_status_t vm_object_put_page(
+	struct vm_object *vo,
+	off_t offset,
+	struct vm_page *pg);
 
 extern kern_status_t vm_object_read(
 	struct vm_object *vo,

include/kernel/vm.h

@@ -1,13 +1,14 @@
 #ifndef KERNEL_VM_H_
 #define KERNEL_VM_H_
 
+#include <kernel/atomic.h>
 #include <kernel/bitmap.h>
 #include <kernel/btree.h>
 #include <kernel/locks.h>
 #include <kernel/machine/vm.h>
 #include <kernel/queue.h>
 #include <kernel/types.h>
-#include <mango/status.h>
+#include <magenta/status.h>
 #include <stddef.h>
 
 #ifdef __cplusplus
@@ -221,6 +222,9 @@ struct vm_page {
 	};
 
 	union {
+		/* how many vm_areas reference this vm_page. if >0, the page is
+		 * subject to copy-on-write. */
+		atomic_t p_cow_ref;
 		uint32_t p_priv2;
 	};
 

init/main.c

@@ -23,7 +23,7 @@ extern char __pstart[], __pend[];
 
 void print_kernel_banner(void)
 {
-	printk("Mango kernel version " BUILD_ID);
+	printk("Magenta kernel version " BUILD_ID);
 }
 
 static void hang(void)
@@ -44,8 +44,8 @@ static void hang(void)
 
 void background_thread(void)
 {
-	struct task *self = current_task();
+	struct task *self = get_current_task();
-	struct thread *thread = current_thread();
+	struct thread *thread = get_current_thread();
 	printk("background_thread() running on processor %u", this_cpu());
 	milli_sleep(1000);
 
@@ -108,7 +108,8 @@ void kernel_init(uintptr_t arg)
 	       bsp.bsp_trailer.bsp_exec_entry,
 	       bsp.bsp_vmo);
 
-	struct task *bootstrap_task = task_create("bootstrap", 9);
+	struct task *bootstrap_task
+		= task_create(NULL, TASK_F_DEFAULT, "bootstrap", 9);
 	tracek("created bootstrap task (pid=%u)", bootstrap_task->t_id);
 
 	status = bsp_launch_async(&bsp, bootstrap_task);

kernel/arg.c

@@ -1,7 +1,7 @@
 #include <kernel/arg.h>
 #include <kernel/libc/ctype.h>
 #include <kernel/libc/string.h>
-#include <mango/status.h>
+#include <magenta/status.h>
 
 static char g_cmdline[CMDLINE_MAX + 1] = {0};
 

kernel/bsp.c

@@ -120,6 +120,7 @@ static kern_status_t map_executable_exec(
 		bsp->bsp_vmo,
 		text_foffset,
 		bsp->bsp_trailer.bsp_text_size,
+		VM_SHARED,
 		VM_PROT_READ | VM_PROT_EXEC | VM_PROT_USER,
 		&text_base);
 	if (status != KERN_OK) {
@@ -132,6 +133,7 @@ static kern_status_t map_executable_exec(
 		data,
 		data_foffset,
 		bsp->bsp_trailer.bsp_data_size,
+		VM_PRIVATE,
 		VM_PROT_READ | VM_PROT_WRITE | VM_PROT_USER,
 		&data_base);
 	if (status != KERN_OK) {
@@ -165,6 +167,7 @@ kern_status_t bsp_launch_async(struct bsp *bsp, struct task *task)
 		user_stack,
 		0,
 		BOOTSTRAP_STACK_SIZE,
+		VM_PRIVATE,
 		VM_PROT_READ | VM_PROT_WRITE | VM_PROT_USER,
 		&stack_buffer);
 
@@ -178,6 +181,7 @@ kern_status_t bsp_launch_async(struct bsp *bsp, struct task *task)
 		bsp->bsp_vmo,
 		0,
 		bsp->bsp_trailer.bsp_exec_offset,
+		VM_PRIVATE,
 		VM_PROT_READ | VM_PROT_USER,
 		&bsp_data_base);
 

kernel/channel.c

@@ -6,7 +6,7 @@
 #include <kernel/task.h>
 #include <kernel/thread.h>
 #include <kernel/util.h>
-#include <mango/signal.h>
+#include <magenta/signal.h>
 
 #define CHANNEL_CAST(p) OBJECT_C_CAST(struct channel, c_base, &channel_type, p)
 
@@ -139,7 +139,6 @@ extern kern_status_t channel_recv_msg(
 	kern_msg_t *out_msg,
 	unsigned long *irq_flags)
 {
-	struct thread *self = current_thread();
 	struct msg *msg = NULL;
 	unsigned long msg_lock_flags;
 
@@ -170,7 +169,7 @@ extern kern_status_t channel_recv_msg(
 	}
 
 	struct task *sender = msg->msg_sender_thread->tr_parent;
-	struct task *receiver = self->tr_parent;
+	struct task *receiver = get_current_task();
 
 	struct address_space *src = sender->t_address_space,
 			     *dst = receiver->t_address_space;
@@ -192,6 +191,7 @@ extern kern_status_t channel_recv_msg(
 
 	if (status != KERN_OK) {
 		kmsg_reply_error(msg, status, &msg_lock_flags);
+		put_current_task(receiver);
 		return status;
 	}
 
@@ -216,6 +216,7 @@ extern kern_status_t channel_recv_msg(
 		&receiver->t_handles_lock,
 		f);
 	address_space_unlock_pair_irqrestore(src, dst, f);
+	put_current_task(receiver);
 
 	if (status != KERN_OK) {
 		kmsg_reply_error(msg, status, &msg_lock_flags);
@@ -250,11 +251,10 @@ extern kern_status_t channel_reply_msg(
 		return KERN_INVALID_ARGUMENT;
 	}
 
-	struct thread *self = current_thread();
 	/* the task that is about to receive the response */
 	struct task *receiver = msg->msg_sender_thread->tr_parent;
 	/* the task that is about to send the response */
-	struct task *sender = self->tr_parent;
+	struct task *sender = get_current_task();
 
 	struct address_space *src = sender->t_address_space,
 			     *dst = receiver->t_address_space;
@@ -275,6 +275,7 @@ extern kern_status_t channel_reply_msg(
 
 	if (status != KERN_OK) {
 		kmsg_reply_error(msg, status, &msg_lock_flags);
+		put_current_task(sender);
 		return status;
 	}
 
@@ -299,6 +300,7 @@ extern kern_status_t channel_reply_msg(
 		&receiver->t_handles_lock,
 		f);
 	address_space_unlock_pair_irqrestore(src, dst, f);
+	put_current_task(sender);
 
 	if (status != KERN_OK) {
 		kmsg_reply_error(msg, status, &msg_lock_flags);

kernel/futex.c

@@ -2,7 +2,7 @@
 #include <kernel/futex.h>
 #include <kernel/sched.h>
 #include <kernel/task.h>
-#include <mango/status.h>
+#include <magenta/status.h>
 
 #define FUTEX_CREATE 0x40u
 
@@ -32,8 +32,10 @@ static kern_status_t get_data(
 {
 	spin_lock_t *lock = NULL;
 	struct btree *futex_list = NULL;
+	struct task *self = NULL;
+
 	if (flags & FUTEX_PRIVATE) {
-		struct task *self = current_task();
+		self = get_current_task();
 		lock = &self->t_base.ob_lock;
 		futex_list = &self->t_futex;
 	} else if (flags & FUTEX_SHARED) {
@@ -48,12 +50,20 @@ static kern_status_t get_data(
 
 	if (!futex && !(flags & FUTEX_CREATE)) {
 		spin_unlock_irqrestore(lock, *irq_flags);
+		if (self) {
+			put_current_task(self);
+		}
+
 		return KERN_NO_ENTRY;
 	}
 
 	futex = vm_cache_alloc(&futex_cache, VM_NORMAL);
 	if (!futex) {
 		spin_unlock_irqrestore(lock, *irq_flags);
+		if (self) {
+			put_current_task(self);
+		}
+
 		return KERN_NO_MEMORY;
 	}
 
@@ -61,6 +71,10 @@ static kern_status_t get_data(
 
 	put_futex(futex_list, futex);
 
+	if (self) {
+		put_current_task(self);
+	}
+
 	*out = futex;
 	*out_lock = lock;
 	return KERN_OK;
@@ -68,9 +82,10 @@ static kern_status_t get_data(
 
 static kern_status_t cleanup_data(struct futex *futex, unsigned int flags)
 {
+	struct task *self = NULL;
 	struct btree *futex_list = NULL;
 	if (flags & FUTEX_PRIVATE) {
-		struct task *self = current_task();
+		self = get_current_task();
 		futex_list = &self->t_futex;
 	} else if (flags & FUTEX_SHARED) {
 		futex_list = &shared_futex_list;
@@ -81,12 +96,16 @@ static kern_status_t cleanup_data(struct futex *futex, unsigned int flags)
 	btree_delete(futex_list, &futex->f_node);
 	vm_cache_free(&futex_cache, futex);
 
+	if (self) {
+		put_current_task(self);
+	}
+
 	return KERN_OK;
 }
 
 static kern_status_t futex_get_shared(kern_futex_t *futex, futex_key_t *out)
 {
-	struct task *self = current_task();
+	struct task *self = get_current_task();
 	struct address_space *space = self->t_address_space;
 
 	unsigned long flags;
@@ -97,6 +116,7 @@ static kern_status_t futex_get_shared(kern_futex_t *futex, futex_key_t *out)
 		out,
 		&flags);
 	address_space_unlock_irqrestore(space, flags);
+	put_current_task(self);
 	return status;
 }
 

kernel/handle.c

@@ -5,7 +5,7 @@
 #include <kernel/sched.h>
 #include <kernel/util.h>
 #include <kernel/vm.h>
-#include <mango/types.h>
+#include <magenta/types.h>
 
 /* depth=3 gives a maximum of ~66.6 million handles */
 #define MAX_TABLE_DEPTH  3
@@ -77,6 +77,81 @@ void handle_table_destroy(struct handle_table *tab)
 	do_handle_table_destroy(tab, 0);
 }
 
+static kern_status_t do_handle_table_duplicate_leaf(
+	struct handle_table *src,
+	struct handle_table **dest)
+{
+	struct handle_table *out
+		= vm_cache_alloc(&handle_table_cache, VM_NORMAL);
+	if (!out) {
+		return KERN_NO_MEMORY;
+	}
+
+	memcpy(out, src, sizeof *out);
+
+	for (size_t i = 0; i < HANDLES_PER_TABLE; i++) {
+		struct object *obj = src->t_handles.t_handle_list[i].h_object;
+		if (obj) {
+			object_ref(obj);
+		}
+	}
+
+	*dest = out;
+	return KERN_OK;
+}
+
+static kern_status_t do_handle_table_duplicate(
+	struct handle_table *src,
+	struct handle_table **dest,
+	unsigned int depth)
+{
+	if (depth == MAX_TABLE_DEPTH - 1) {
+		return do_handle_table_duplicate_leaf(src, dest);
+	}
+
+	struct handle_table *out
+		= vm_cache_alloc(&handle_table_cache, VM_NORMAL);
+	if (!out) {
+		return KERN_NO_MEMORY;
+	}
+
+	memcpy(out->t_subtables.t_subtable_map,
+	       src->t_subtables.t_subtable_map,
+	       sizeof out->t_subtables.t_subtable_map);
+	memset(out->t_subtables.t_subtable_list,
+	       0x0,
+	       sizeof out->t_subtables.t_subtable_list);
+
+	for (size_t i = 0; i < REFS_PER_TABLE; i++) {
+		struct handle_table *child
+			= src->t_subtables.t_subtable_list[i];
+		struct handle_table *dup = NULL;
+		kern_status_t status = KERN_OK;
+		if (child) {
+			status = do_handle_table_duplicate(
+				child,
+				&dup,
+				depth + 1);
+		}
+
+		if (status == KERN_OK) {
+			out->t_subtables.t_subtable_list[i] = dup;
+		} else {
+			return status;
+		}
+	}
+
+	*dest = out;
+	return KERN_OK;
+}
+
+kern_status_t handle_table_duplicate(
+	struct handle_table *src,
+	struct handle_table **dest)
+{
+	return do_handle_table_duplicate(src, dest, 0);
+}
+
 static kern_status_t decode_handle_indices(
 	kern_handle_t handle,
 	unsigned int indices[MAX_TABLE_DEPTH])

kernel/object.c

@@ -175,7 +175,7 @@ void object_wait_signal(
 	uint32_t signals,
 	unsigned long *irq_flags)
 {
-	struct thread *self = current_thread();
+	struct thread *self = get_current_thread();
 	struct wait_item waiter;
 	wait_item_init(&waiter, self);
 	for (;;) {
@@ -189,4 +189,5 @@ void object_wait_signal(
 		object_lock_irqsave(obj, irq_flags);
 	}
 	thread_wait_end(&waiter, &obj->ob_wq);
+	put_current_thread(self);
 }

kernel/panic.c

@@ -19,8 +19,8 @@ void panic_irq(struct ml_cpu_context *ctx, const char *fmt, ...)
 	printk("---[ kernel panic: %s", buf);
 	printk("kernel: " BUILD_ID ", compiler version: " __VERSION__);
 
-	struct task *task = current_task();
+	struct task *task = get_current_task();
-	struct thread *thr = current_thread();
+	struct thread *thr = get_current_thread();
 
 	if (task && thr) {
 		printk("task: %s (id: %d, thread: %d)",
@@ -31,6 +31,9 @@ void panic_irq(struct ml_cpu_context *ctx, const char *fmt, ...)
 		printk("task: [bootstrap]");
 	}
 
+	put_current_thread(thr);
+	put_current_task(task);
+
 	printk("cpu: %u", this_cpu());
 
 	ml_print_cpu_state(ctx);

kernel/port.c

@@ -37,7 +37,7 @@ struct port *port_cast(struct object *obj)
 static void wait_for_reply(struct msg *msg, unsigned long *lock_flags)
 {
 	struct wait_item waiter;
-	struct thread *self = current_thread();
+	struct thread *self = get_current_thread();
 
 	wait_item_init(&waiter, self);
 	for (;;) {
@@ -52,6 +52,7 @@ static void wait_for_reply(struct msg *msg, unsigned long *lock_flags)
 	}
 
 	self->tr_state = THREAD_READY;
+	put_current_thread(self);
 }
 
 struct port *port_create(void)
@@ -83,9 +84,12 @@ kern_status_t port_connect(struct port *port, struct channel *remote)
 	msg->msg_status = KMSG_ASYNC;
 	msg->msg_type = KERN_MSG_TYPE_EVENT;
 	msg->msg_event = KERN_MSG_EVENT_CONNECTION;
-	msg->msg_sender_thread_id = current_thread()->tr_id;
 	msg->msg_sender_port_id = port->p_base.ob_id;
 
+	struct thread *self = get_current_thread();
+	msg->msg_sender_thread_id = self->tr_id;
+	put_current_thread(self);
+
 	unsigned long flags;
 	channel_lock_irqsave(remote, &flags);
 	channel_enqueue_msg(remote, msg);
@@ -112,9 +116,12 @@ kern_status_t port_disconnect(struct port *port)
 	msg->msg_status = KMSG_ASYNC;
 	msg->msg_type = KERN_MSG_TYPE_EVENT;
 	msg->msg_event = KERN_MSG_EVENT_DISCONNECTION;
-	msg->msg_sender_thread_id = current_thread()->tr_id;
 	msg->msg_sender_port_id = port->p_base.ob_id;
 
+	struct thread *self = get_current_thread();
+	msg->msg_sender_thread_id = self->tr_id;
+	put_current_thread(self);
+
 	unsigned long flags;
 	channel_lock_irqsave(port->p_remote, &flags);
 	channel_enqueue_msg(port->p_remote, msg);
@@ -136,7 +143,7 @@ kern_status_t port_send_msg(
 		return KERN_BAD_STATE;
 	}
 
-	struct thread *self = current_thread();
+	struct thread *self = get_current_thread();
 	struct msg msg;
 	memset(&msg, 0x0, sizeof msg);
 	msg.msg_type = KERN_MSG_TYPE_DATA;
@@ -156,6 +163,7 @@ kern_status_t port_send_msg(
 	channel_lock_irqsave(port->p_remote, &flags);
 	btree_delete(&port->p_remote->c_msg, &msg.msg_node);
 	channel_unlock_irqrestore(port->p_remote, flags);
+	put_current_thread(self);
 
 	return msg.msg_result;
 }

kernel/printk.c

@@ -98,9 +98,8 @@ int printk(const char *format, ...)
 	unsigned long flags;
 	spin_lock_irqsave(&log_buffer_lock, &flags);
 	save_log_message(msg);
-	spin_unlock_irqrestore(&log_buffer_lock, flags);
-
 	flush_log_buffer();
+	spin_unlock_irqrestore(&log_buffer_lock, flags);
 
 	return 0;
 }

kernel/status.c

@@ -1,5 +1,5 @@
-#include <mango/status.h>
+#include <magenta/status.h>
-#include <mango/types.h>
+#include <magenta/types.h>
 
 #define ERROR_STRING_CASE(code)                                                \
 	case code:                                                             \

libmagenta/CMakeLists.txt

@@ -1,4 +1,4 @@
-file(GLOB headers ${CMAKE_CURRENT_SOURCE_DIR}/include/mango/*.h)
+file(GLOB headers ${CMAKE_CURRENT_SOURCE_DIR}/include/magenta/*.h)
 file(GLOB asm_sources
 	${CMAKE_CURRENT_SOURCE_DIR}/arch/${CMAKE_SYSTEM_PROCESSOR}/*.S)
 
@@ -6,7 +6,7 @@ set(public_include_dirs
 	${CMAKE_CURRENT_SOURCE_DIR}/include
 	${CMAKE_CURRENT_SOURCE_DIR}/include-user)
 
-add_library(libmango STATIC ${asm_sources})
+add_library(libmagenta STATIC ${asm_sources})
-target_include_directories(libmango PUBLIC
+target_include_directories(libmagenta PUBLIC
 	${CMAKE_CURRENT_SOURCE_DIR}/include
 	${CMAKE_CURRENT_SOURCE_DIR}/include-user)

libmagenta/arch/x86_64/syscall.S

@@ -1,4 +1,4 @@
-#include "mango/syscall.h"
+#include "magenta/syscall.h"
 
 # Registers:
 #  rax = syscall ID + return value
@@ -57,7 +57,8 @@
 
 SYSCALL_GATE task_exit SYS_TASK_EXIT 1
 SYSCALL_GATE task_self SYS_TASK_SELF 1
-SYSCALL_GATE task_create SYS_TASK_CREATE 5
+SYSCALL_GATE task_create SYS_TASK_CREATE 6
+SYSCALL_GATE task_duplicate SYS_TASK_DUPLICATE 2
 SYSCALL_GATE task_create_thread SYS_TASK_CREATE_THREAD 6
 SYSCALL_GATE task_get_address_space SYS_TASK_GET_ADDRESS_SPACE 1
 SYSCALL_GATE task_config_get SYS_TASK_CONFIG_GET 4
@@ -76,7 +77,7 @@ SYSCALL_GATE vm_object_copy SYS_VM_OBJECT_COPY 6
 
 SYSCALL_GATE address_space_read SYS_ADDRESS_SPACE_READ 5
 SYSCALL_GATE address_space_write SYS_ADDRESS_SPACE_WRITE 5
-SYSCALL_GATE address_space_map SYS_ADDRESS_SPACE_MAP 7
+SYSCALL_GATE address_space_map SYS_ADDRESS_SPACE_MAP 8
 SYSCALL_GATE address_space_unmap SYS_ADDRESS_SPACE_UNMAP 3
 SYSCALL_GATE address_space_reserve SYS_ADDRESS_SPACE_RESERVE 4
 SYSCALL_GATE address_space_release SYS_ADDRESS_SPACE_RELEASE 3
@@ -101,6 +102,8 @@ SYSCALL_GATE vm_controller_create SYS_VM_CONTROLLER_CREATE 1
 SYSCALL_GATE vm_controller_recv SYS_VM_CONTROLLER_RECV 2
 SYSCALL_GATE vm_controller_recv_async SYS_VM_CONTROLLER_RECV_ASYNC 3
 SYSCALL_GATE vm_controller_create_object SYS_VM_CONTROLLER_CREATE_OBJECT 7
+SYSCALL_GATE vm_controller_prepare_attach SYS_VM_CONTROLLER_PREPARE_ATTACH 3
+SYSCALL_GATE vm_controller_finish_attach SYS_VM_CONTROLLER_FINISH_ATTACH 3
 SYSCALL_GATE vm_controller_detach_object SYS_VM_CONTROLLER_DETACH_OBJECT 2
 SYSCALL_GATE vm_controller_supply_pages SYS_VM_CONTROLLER_SUPPLY_PAGES 6
 

libmagenta/include-user/magenta/config.h

@@ -1,8 +1,8 @@
-#ifndef MANGO_CONFIG_H_
+#ifndef MAGENTA_CONFIG_H_
-#define MANGO_CONFIG_H_
+#define MAGENTA_CONFIG_H_
 
-#include <mango/status.h>
+#include <magenta/status.h>
-#include <mango/types.h>
+#include <magenta/types.h>
 #include <stddef.h>
 
 extern kern_status_t kern_config_get(

libmagenta/include-user/magenta/equeue.h

@@ -1,7 +1,7 @@
-#ifndef MANGO_EQUEUE_H_
+#ifndef MAGENTA_EQUEUE_H_
-#define MANGO_EQUEUE_H_
+#define MAGENTA_EQUEUE_H_
 
-#include <mango/types.h>
+#include <magenta/types.h>
 
 extern kern_status_t equeue_create(kern_handle_t *out);
 extern kern_status_t equeue_dequeue(kern_handle_t eq, equeue_packet_t *out);

libmagenta/include-user/magenta/futex.h

@@ -1,8 +1,8 @@
-#ifndef MANGO_FUTEX_H_
+#ifndef MAGENTA_FUTEX_H_
-#define MANGO_FUTEX_H_
+#define MAGENTA_FUTEX_H_
 
-#include <mango/status.h>
+#include <magenta/status.h>
-#include <mango/types.h>
+#include <magenta/types.h>
 
 extern kern_status_t futex_wait(
 	kern_futex_t *futex,

libmagenta/include-user/magenta/handle.h

@@ -1,8 +1,8 @@
-#ifndef MANGO_HANDLE_H_
+#ifndef MAGENTA_HANDLE_H_
-#define MANGO_HANDLE_H_
+#define MAGENTA_HANDLE_H_
 
-#include <mango/status.h>
+#include <magenta/status.h>
-#include <mango/types.h>
+#include <magenta/types.h>
 
 extern kern_status_t kern_handle_close(kern_handle_t handle);
 extern kern_status_t kern_handle_transfer(
@@ -12,5 +12,11 @@ extern kern_status_t kern_handle_transfer(
 	kern_handle_t dest_handle,
 	unsigned int mode,
 	kern_handle_t *out_handle);
+extern kern_status_t kern_handle_control(
+	kern_handle_t task,
+	kern_handle_t handle,
+	uint32_t set_mask,
+	uint32_t clear_mask,
+	uint32_t *out_flags);
 
 #endif

libmagenta/include-user/magenta/log.h

@@ -1,8 +1,8 @@
-#ifndef MANGO_LOG_H_
+#ifndef MAGENTA_LOG_H_
-#define MANGO_LOG_H_
+#define MAGENTA_LOG_H_
 
-#include <mango/status.h>
+#include <magenta/status.h>
-#include <mango/types.h>
+#include <magenta/types.h>
 
 #undef TRACE
 

libmagenta/include-user/magenta/msg.h

@@ -1,8 +1,8 @@
-#ifndef MANGO_MSG_H_
+#ifndef MAGENTA_MSG_H_
-#define MANGO_MSG_H_
+#define MAGENTA_MSG_H_
 
-#include <mango/status.h>
+#include <magenta/status.h>
-#include <mango/types.h>
+#include <magenta/types.h>
 
 extern kern_status_t channel_create(unsigned int id, kern_handle_t *out);
 extern kern_status_t port_create(kern_handle_t *out);

libmagenta/include-user/magenta/object.h

@@ -1,7 +1,7 @@
-#ifndef MANGO_OBJECT_H_
+#ifndef MAGENTA_OBJECT_H_
-#define MANGO_OBJECT_H_
+#define MAGENTA_OBJECT_H_
 
-#include <mango/types.h>
+#include <magenta/types.h>
 
 extern kern_status_t kern_object_wait(kern_wait_item_t *items, size_t nr_items);
 extern kern_status_t kern_object_query(

libmagenta/include-user/magenta/task.h

@@ -1,14 +1,15 @@
-#ifndef MANGO_TASK_H_
+#ifndef MAGENTA_TASK_H_
-#define MANGO_TASK_H_
+#define MAGENTA_TASK_H_
 
-#include <mango/status.h>
+#include <magenta/status.h>
-#include <mango/types.h>
+#include <magenta/types.h>
 
 extern kern_status_t task_exit(int status);
 extern kern_status_t task_self(kern_handle_t *out);
 
 extern kern_status_t task_create(
 	kern_handle_t parent,
+	task_flags_t flags,
 	const char *name,
 	size_t name_len,
 	kern_handle_t *out_task,
@@ -33,6 +34,16 @@ extern kern_status_t task_config_set(
 	kern_config_key_t key,
 	const void *ptr,
 	size_t len);
+extern kern_status_t task_duplicate(
+	kern_handle_t *out_task,
+	kern_handle_t *out_address_space);
+extern kern_status_t task_reset(
+	virt_addr_t ip,
+	virt_addr_t sp,
+	uintptr_t *args,
+	size_t nr_args,
+	virt_addr_t unmap_base,
+	size_t unmap_length);
 
 extern kern_status_t thread_self(kern_handle_t *out);
 extern kern_status_t thread_start(kern_handle_t thread);

libmagenta/include-user/magenta/vm.h

@@ -1,8 +1,8 @@
-#ifndef MANGO_VM_H_
+#ifndef MAGENTA_VM_H_
-#define MANGO_VM_H_
+#define MAGENTA_VM_H_
 
-#include <mango/status.h>
+#include <magenta/status.h>
-#include <mango/types.h>
+#include <magenta/types.h>
 
 extern kern_status_t vm_object_create(
 	const char *name,
@@ -48,6 +48,7 @@ extern kern_status_t address_space_map(
 	kern_handle_t object,
 	off_t object_offset,
 	size_t length,
+	vm_flags_t flags,
 	vm_prot_t prot,
 	virt_addr_t *out_base_address);
 extern kern_status_t address_space_unmap(
@@ -67,7 +68,7 @@ extern kern_status_t address_space_release(
 extern kern_status_t vm_controller_create(kern_handle_t *out);
 extern kern_status_t vm_controller_recv(
 	kern_handle_t ctrl,
-	equeue_packet_page_request_t *out);
+	equeue_packet_vm_request_t *out);
 extern kern_status_t vm_controller_recv_async(
 	kern_handle_t ctrl,
 	kern_handle_t eq,
@@ -80,6 +81,14 @@ extern kern_status_t vm_controller_create_object(
 	size_t data_len,
 	vm_prot_t prot,
 	kern_handle_t *out);
+extern kern_status_t vm_controller_prepare_attach(
+	kern_handle_t ctrl,
+	uint64_t req_id,
+	kern_handle_t *out_vmo);
+extern kern_status_t vm_controller_finish_attach(
+	kern_handle_t ctrl,
+	uint64_t req_id,
+	equeue_key_t new_key);
 extern kern_status_t vm_controller_detach_object(
 	kern_handle_t ctrl,
 	kern_handle_t vmo);

libmagenta/include/magenta/signal.h

@@ -1,5 +1,5 @@
-#ifndef MANGO_SIGNAL_H_
+#ifndef MAGENTA_SIGNAL_H_
-#define MANGO_SIGNAL_H_
+#define MAGENTA_SIGNAL_H_
 
 #define THREAD_SIGNAL_STOPPED                 0x01u
 

libmagenta/include/magenta/status.h

@@ -1,5 +1,5 @@
-#ifndef MANGO_STATUS_H_
+#ifndef MAGENTA_STATUS_H_
-#define MANGO_STATUS_H_
+#define MAGENTA_STATUS_H_
 
 #define KERN_OK               (0)
 #define KERN_UNIMPLEMENTED    (1)

libmagenta/include/magenta/syscall.h

@@ -0,0 +1,58 @@
+#ifndef MAGENTA_SYSCALL_H_
+#define MAGENTA_SYSCALL_H_
+
+#define SYS_KERN_LOG                     1
+#define SYS_KERN_HANDLE_CLOSE            2
+#define SYS_KERN_HANDLE_TRANSFER         3
+#define SYS_KERN_HANDLE_CONTROL          4
+#define SYS_KERN_CONFIG_GET              5
+#define SYS_KERN_CONFIG_SET              6
+#define SYS_KERN_OBJECT_WAIT             7
+#define SYS_KERN_OBJECT_WAIT_ASYNC       8
+#define SYS_TASK_EXIT                    9
+#define SYS_TASK_SELF                    10
+#define SYS_TASK_CREATE                  11
+#define SYS_TASK_CREATE_THREAD           12
+#define SYS_TASK_GET_ADDRESS_SPACE       13
+#define SYS_TASK_CONFIG_GET              14
+#define SYS_TASK_CONFIG_SET              15
+#define SYS_TASK_DUPLICATE               16
+#define SYS_THREAD_SELF                  17
+#define SYS_THREAD_START                 18
+#define SYS_THREAD_EXIT                  19
+#define SYS_THREAD_CONFIG_GET            20
+#define SYS_THREAD_CONFIG_SET            21
+#define SYS_VM_OBJECT_CREATE             22
+#define SYS_VM_OBJECT_READ               23
+#define SYS_VM_OBJECT_WRITE              24
+#define SYS_VM_OBJECT_COPY               25
+#define SYS_ADDRESS_SPACE_READ           26
+#define SYS_ADDRESS_SPACE_WRITE          27
+#define SYS_ADDRESS_SPACE_MAP            28
+#define SYS_ADDRESS_SPACE_UNMAP          29
+#define SYS_ADDRESS_SPACE_RESERVE        30
+#define SYS_ADDRESS_SPACE_RELEASE        31
+#define SYS_MSG_SEND                     32
+#define SYS_MSG_RECV                     33
+#define SYS_MSG_REPLY                    34
+#define SYS_MSG_READ                     35
+#define SYS_MSG_WRITE                    36
+#define SYS_CHANNEL_CREATE               37
+#define SYS_PORT_CREATE                  38
+#define SYS_PORT_CONNECT                 39
+#define SYS_PORT_DISCONNECT              40
+#define SYS_EQUEUE_CREATE                41
+#define SYS_EQUEUE_DEQUEUE               42
+#define SYS_VM_CONTROLLER_CREATE         43
+#define SYS_VM_CONTROLLER_RECV           44
+#define SYS_VM_CONTROLLER_RECV_ASYNC     45
+#define SYS_VM_CONTROLLER_CREATE_OBJECT  46
+#define SYS_VM_CONTROLLER_PREPARE_ATTACH 47
+#define SYS_VM_CONTROLLER_FINISH_ATTACH  48
+#define SYS_VM_CONTROLLER_DETACH_OBJECT  49
+#define SYS_VM_CONTROLLER_SUPPLY_PAGES   50
+#define SYS_FUTEX_WAIT                   51
+#define SYS_FUTEX_WAKE                   52
+#define SYS_KERN_OBJECT_QUERY            53
+
+#endif

libmagenta/include/magenta/types.h

@@ -1,5 +1,5 @@
-#ifndef MANGO_TYPES_H_
+#ifndef MAGENTA_TYPES_H_
-#define MANGO_TYPES_H_
+#define MAGENTA_TYPES_H_
 
 #include <stddef.h>
 #include <stdint.h>
@@ -16,18 +16,39 @@
 #define MAP_ADDRESS_INVALID          ((virt_addr_t)0)
 #define KERN_HANDLE_INVALID          ((kern_handle_t)0xFFFFFFFF)
 
+/* task creation flags */
+#define TASK_F_DEFAULT               0x0000u
+#define TASK_F_CLONE_ALL_HANDLES     0x0001u
+
 /* config keys for use with kern_config_get/kern_config_set */
 #define KERN_CFG_INVALID             0x00000u
 #define KERN_CFG_PAGE_SIZE           0x00001u
 
 /* config keys for use with task_config_get/task_config_set */
 #define TASK_CFG_INVALID             0x00000u
+#define TASK_CFG_ID                  0x10001u
 
 /* config keys for use with thread_config_get/thread_config_set */
 #define THREAD_CFG_INVALID           0x00000u
 #define THREAD_CFG_FSBASE            0x20001u
 #define THREAD_CFG_GSBASE            0x20002u
 
+/* user-defined flags that can be set on handles */
+#define KERN_HANDLE_FLAG0            0x10000000UL
+#define KERN_HANDLE_FLAG1            0x20000000UL
+#define KERN_HANDLE_FLAG2            0x40000000UL
+#define KERN_HANDLE_FLAG3            0x80000000UL
+
+/* flags to specify when creating address-space mappings */
+/* this mapping is private. if a task with this mapping is duplicated,
+   the duplicate task will receive a copy-on-write mapping. changes to one
+   mapping will not be visible to the other. */
+#define VM_PRIVATE                   0x0000u
+/* this mapping is shared. if a task with this mapping is duplicated,
+ * the duplicate will receive a mapping of the same data. changes to one mapping
+ * will be visibile to the other */
+#define VM_SHARED                    0x0001u
+
 /* maximum number of handles that can be sent in a single message */
 #define KERN_MSG_MAX_HANDLES         64
 
@@ -55,13 +76,14 @@
 #define KERN_MSG_EVENT_DISCONNECTION 2
 
 /* equeue packet types */
-#define EQUEUE_PKT_PAGE_REQUEST      0x01u
+#define EQUEUE_PKT_VM_REQUEST        0x01u
 #define EQUEUE_PKT_ASYNC_SIGNAL      0x02u
 
-/* page request types */
+/* vm request types */
-#define PAGE_REQUEST_READ            0x01u
+#define VM_REQUEST_READ              0x01u
-#define PAGE_REQUEST_DIRTY           0x02u
+#define VM_REQUEST_DIRTY             0x02u
-#define PAGE_REQUEST_DETACH          0x03u
+#define VM_REQUEST_ATTACH            0x03u
+#define VM_REQUEST_DETACH            0x04u
 
 /* futex special values */
 #define FUTEX_WAKE_ALL               ((size_t)-1)
@@ -100,6 +122,8 @@ typedef unsigned int kern_status_t;
 typedef uint32_t kern_handle_t;
 typedef uint32_t kern_config_key_t;
 typedef uint32_t vm_prot_t;
+typedef uint32_t vm_flags_t;
+typedef uint32_t task_flags_t;
 typedef int64_t ssize_t;
 typedef uint32_t kern_futex_t;
 typedef uint32_t kern_msg_type_t;
@@ -169,14 +193,33 @@ typedef struct {
 	/* the key of the vm-object for which the page request relates, as
 	 * specified when the vm-object was created */
 	equeue_key_t req_vmo;
-	/* page request type. one of PAGE_REQUEST_* */
+	/* page request type. one of VM_REQUEST_* */
 	unsigned short req_type;
-	/* of the offset into the vm-object for which pages are being requested
+	/* the offset into the vm-object for which pages are being requested */
-	 */
+	union {
-	off_t req_offset;
+		/* used for:
-	/* the length in bytes of the region being requested */
+		 * 	VM_REQUEST_READ
-	size_t req_length;
+		 * 	VM_REQUEST_DIRTY
-} equeue_packet_page_request_t;
+		 */
+		struct {
+			off_t req_offset;
+			/* the length in bytes of the region being requested */
+			size_t req_length;
+		};
+
+		/* used for:
+		 * 	VM_REQUEST_ATTACH
+		 */
+		struct {
+			/* the key of the original/source vmo. */
+			equeue_key_t req_src_vmo;
+			/* a request ID. used to retrieve information about
+			 * the newly-attached object, as the server won't know
+			 * about it yet, and won't have a handle to it. */
+			uint64_t req_id;
+		};
+	};
+} equeue_packet_vm_request_t;
 
 typedef struct {
 	/* the type of packet. one of EQUEUE_PKT_* */
@@ -186,8 +229,8 @@ typedef struct {
 	equeue_key_t p_key;
 
 	union {
-		/* p_type = EQUEUE_PKT_PAGE_REQUEST */
+		/* p_type = EQUEUE_PKT_VM_REQUEST */
-		equeue_packet_page_request_t page_request;
+		equeue_packet_vm_request_t vm_request;
 		/* p_type = EQUEUE_PKT_ASYNC_SIGNAL */
 		equeue_packet_async_signal_t async_signal;
 	};

libmango/include/mango/syscall.h

@@ -1,54 +0,0 @@
-#ifndef MANGO_SYSCALL_H_
-#define MANGO_SYSCALL_H_
-
-#define SYS_KERN_LOG                    1
-#define SYS_KERN_HANDLE_CLOSE           2
-#define SYS_KERN_HANDLE_TRANSFER        3
-#define SYS_KERN_CONFIG_GET             4
-#define SYS_KERN_CONFIG_SET             5
-#define SYS_KERN_OBJECT_WAIT            6
-#define SYS_KERN_OBJECT_WAIT_ASYNC      7
-#define SYS_TASK_EXIT                   8
-#define SYS_TASK_SELF                   9
-#define SYS_TASK_CREATE                 10
-#define SYS_TASK_CREATE_THREAD          11
-#define SYS_TASK_GET_ADDRESS_SPACE      12
-#define SYS_TASK_CONFIG_GET             13
-#define SYS_TASK_CONFIG_SET             14
-#define SYS_THREAD_SELF                 15
-#define SYS_THREAD_START                16
-#define SYS_THREAD_EXIT                 17
-#define SYS_THREAD_CONFIG_GET           18
-#define SYS_THREAD_CONFIG_SET           19
-#define SYS_VM_OBJECT_CREATE            20
-#define SYS_VM_OBJECT_READ              21
-#define SYS_VM_OBJECT_WRITE             22
-#define SYS_VM_OBJECT_COPY              23
-#define SYS_ADDRESS_SPACE_READ          24
-#define SYS_ADDRESS_SPACE_WRITE         25
-#define SYS_ADDRESS_SPACE_MAP           26
-#define SYS_ADDRESS_SPACE_UNMAP         27
-#define SYS_ADDRESS_SPACE_RESERVE       28
-#define SYS_ADDRESS_SPACE_RELEASE       29
-#define SYS_MSG_SEND                    30
-#define SYS_MSG_RECV                    31
-#define SYS_MSG_REPLY                   32
-#define SYS_MSG_READ                    33
-#define SYS_MSG_WRITE                   34
-#define SYS_CHANNEL_CREATE              35
-#define SYS_PORT_CREATE                 36
-#define SYS_PORT_CONNECT                37
-#define SYS_PORT_DISCONNECT             38
-#define SYS_EQUEUE_CREATE               39
-#define SYS_EQUEUE_DEQUEUE              40
-#define SYS_VM_CONTROLLER_CREATE        41
-#define SYS_VM_CONTROLLER_RECV          42
-#define SYS_VM_CONTROLLER_RECV_ASYNC    43
-#define SYS_VM_CONTROLLER_CREATE_OBJECT 44
-#define SYS_VM_CONTROLLER_DETACH_OBJECT 45
-#define SYS_VM_CONTROLLER_SUPPLY_PAGES  46
-#define SYS_FUTEX_WAIT                  47
-#define SYS_FUTEX_WAKE                  48
-#define SYS_KERN_OBJECT_QUERY           49
-
-#endif

sched/core.c

@@ -215,18 +215,19 @@ void schedule_thread_on_cpu(struct thread *thr)
 
 void start_charge_period(void)
 {
-	struct thread *self = current_thread();
+	struct thread *self = get_current_thread();
 	if (!self) {
 		return;
 	}
 
 	self->tr_charge_period_start = get_cycles();
+	put_current_thread(self);
 }
 
 void end_charge_period(void)
 {
 	preempt_disable();
-	struct thread *self = current_thread();
+	struct thread *self = get_current_thread();
 	if (!self) {
 		return;
 	}
@@ -244,6 +245,7 @@ void end_charge_period(void)
 	}
 
 	self->tr_charge_period_start = 0;
+	put_current_thread(self);
 
 	// printk("%llu cycles charged to %s/%u", charge,
 	// self->tr_parent->t_name, self->tr_parent->t_id);

sched/task.c

@@ -171,8 +171,13 @@ struct task *task_alloc(void)
 	return t;
 }
 
-struct task *task_create(const char *name, size_t name_len)
+struct task *task_create(
+	struct task *parent,
+	task_flags_t task_flags,
+	const char *name,
+	size_t name_len)
 {
+	kern_status_t status = KERN_OK;
 	struct task *task = task_alloc();
 	if (!task) {
 		return NULL;
@@ -191,9 +196,21 @@ struct task *task_create(const char *name, size_t name_len)
 		VM_USER_LIMIT,
 		&task->t_address_space);
 
+	if (task_flags & TASK_F_CLONE_ALL_HANDLES) {
+		status = handle_table_duplicate(
+			parent->t_handles,
+			&task->t_handles);
+	} else {
+		task->t_handles = handle_table_create();
+	}
+
+	if (status != KERN_OK) {
+		object_unref(&task->t_base);
+		return NULL;
+	}
+
 	task->t_address_space->s_pmap = pmap;
 	task->t_state = TASK_RUNNING;
-	task->t_handles = handle_table_create();
 
 	if (name) {
 		name_len = MIN(name_len, sizeof task->t_name - 1);
@@ -280,9 +297,43 @@ struct task *task_from_tid(tid_t id)
 	return t;
 }
 
+kern_status_t task_config_get(
+	struct task *task,
+	kern_config_key_t key,
+	void *out,
+	size_t max)
+{
+	switch (key) {
+	case TASK_CFG_ID: {
+		if (max != sizeof(tid_t)) {
+			return KERN_INVALID_ARGUMENT;
+		}
+
+		tid_t *value = out;
+		*value = task->t_id;
+		return KERN_OK;
+	}
+
+	default:
+		return KERN_INVALID_ARGUMENT;
+	}
+}
+
+kern_status_t task_config_set(
+	struct task *task,
+	kern_config_key_t key,
+	const void *ptr,
+	size_t len)
+{
+	switch (key) {
+	default:
+		return KERN_INVALID_ARGUMENT;
+	}
+}
+
 void task_exit(int status)
 {
-	struct task *self = current_task();
+	struct task *self = get_current_task();
 	unsigned long flags;
 	task_lock_irqsave(self, &flags);
 	struct task *parent = self->t_parent;
@@ -295,7 +346,7 @@ void task_exit(int status)
 		task_unlock(parent);
 	}
 
-	struct thread *cur_thread = current_thread();
+	struct thread *cur_thread = get_current_thread();
 
 	self->t_state = TASK_STOPPED;
 	cur_thread->tr_state = THREAD_STOPPED;
@@ -325,6 +376,7 @@ void task_exit(int status)
 	spin_lock(&self->t_handles_lock);
 
 	pmap_switch(get_kernel_pmap());
+	self->t_address_space->s_pmap = PMAP_INVALID;
 	pmap_destroy(self->t_pmap);
 
 	task_unlock(self);
@@ -340,6 +392,9 @@ void task_exit(int status)
 	       self->t_base.ob_refcount);
 	spin_unlock_irqrestore(handles_lock, flags);
 
+	put_current_thread(cur_thread);
+	put_current_task(self);
+
 	while (1) {
 		schedule(SCHED_NORMAL);
 	}
@@ -410,8 +465,19 @@ struct thread *task_create_thread(struct task *parent)
 	return thread;
 }
 
-struct task *current_task(void)
+struct task *get_current_task(void)
 {
-	struct thread *thr = current_thread();
+	struct thread *thr = get_current_thread();
-	return thr ? thr->tr_parent : NULL;
+	if (!thr) {
+		return NULL;
+	}
+
+	struct task *out = task_ref(thr->tr_parent);
+	put_current_thread(thr);
+	return out;
+}
+
+void put_current_task(struct task *task)
+{
+	task_unref(task);
 }

sched/thread.c

@@ -5,7 +5,7 @@
 #include <kernel/printk.h>
 #include <kernel/task.h>
 #include <kernel/thread.h>
-#include <mango/signal.h>
+#include <magenta/signal.h>
 
 #define THREAD_CAST(p) OBJECT_C_CAST(struct thread, tr_base, &thread_type, p)
 
@@ -101,11 +101,57 @@ kern_status_t thread_init_user(
 	return KERN_OK;
 }
 
+kern_status_t thread_init_user_clone(
+	struct thread *thr,
+	const struct thread *src,
+	uintptr_t return_value)
+{
+	thr->tr_state = THREAD_READY;
+	thr->tr_quantum_target = default_quantum();
+
+	thr->tr_kstack = vm_page_alloc(THREAD_KSTACK_ORDER, VM_NORMAL);
+	if (!thr->tr_kstack) {
+		return KERN_NO_MEMORY;
+	}
+
+	thr->tr_sp = (uintptr_t)vm_page_get_vaddr(thr->tr_kstack)
+	           + vm_page_order_to_bytes(THREAD_KSTACK_ORDER);
+	thr->tr_bp = thr->tr_sp;
+	thr->tr_cpu_kernel_sp = thr->tr_sp;
+
+	/* the new thread needs two contextx:
+	 *   1) to get the thread running in kernel mode, so that it can
+	 *   execute ml_thread_switch_user
+	 *   2) to allow ml_thread_switch_user to jump to the correct place
+	 *   in usermode (and with the correct stack).
+	 *
+	 * these two contexts are constructed on the thread's kernel stack
+	 * in reverse order.
+	 */
+
+	/* this context will be used by ml_user_return to jump to userspace
+	 * with the specified instruction pointer and user stack */
+	ml_thread_clone_user_context(
+		src->tr_irqctx,
+		&src->tr_ml,
+		&thr->tr_ml,
+		return_value,
+		&thr->tr_sp);
+	/* this context will be used by the scheduler and ml_thread_switch to
+	 * jump to ml_user_return in kernel mode with the thread's kernel stack.
+	 */
+	ml_thread_prepare_kernel_context(
+		(uintptr_t)ml_thread_switch_user,
+		&thr->tr_sp);
+
+	return KERN_OK;
+}
+
 void thread_free(struct thread *thr)
 {
 }
 
-struct thread *current_thread(void)
+struct thread *get_current_thread(void)
 {
 	struct cpu_data *cpu = get_this_cpu();
 	if (!cpu) {
@@ -113,13 +159,24 @@ struct thread *current_thread(void)
 	}
 
 	struct thread *out = cpu->c_rq.rq_cur;
+	object_ref(&out->tr_base);
+
 	put_cpu(cpu);
 	return out;
 }
 
+void put_current_thread(struct thread *thr)
+{
+	object_unref(&thr->tr_base);
+}
+
 bool need_resched(void)
 {
-	return (current_thread()->tr_flags & THREAD_F_NEED_RESCHED) != 0;
+	struct thread *thr = get_current_thread();
+	bool result = (thr->tr_flags & THREAD_F_NEED_RESCHED) != 0;
+	put_current_thread(thr);
+
+	return result;
 }
 
 int thread_priority(struct thread *thr)
@@ -143,7 +200,7 @@ void thread_awaken(struct thread *thr)
 
 void thread_exit(void)
 {
-	struct thread *self = current_thread();
+	struct thread *self = get_current_thread();
 	unsigned long flags;
 	thread_lock_irqsave(self, &flags);
 	self->tr_state = THREAD_STOPPED;
@@ -153,6 +210,7 @@ void thread_exit(void)
 	       self->tr_parent->t_id,
 	       self->tr_id);
 	thread_unlock_irqrestore(self, flags);
+	put_current_thread(self);
 
 	while (1) {
 		schedule(SCHED_NORMAL);

sched/timer.c

@@ -44,7 +44,7 @@ unsigned long schedule_timeout(unsigned long ticks)
 {
 	struct timer timer;
 
-	struct thread *self = current_thread();
+	struct thread *self = get_current_thread();
 
 	timer.t_entry = QUEUE_ENTRY_INIT;
 	timer.t_expiry = clock_ticks + ticks;
@@ -58,6 +58,7 @@ unsigned long schedule_timeout(unsigned long ticks)
 	schedule(SCHED_NORMAL);
 
 	remove_timer(&timer);
+	put_current_thread(self);
 
 	return 0;
 }

sched/wait.c

@@ -110,7 +110,7 @@ void wakeup_one(struct waitqueue *q)
 
 void sleep_forever(void)
 {
-	struct thread *thr = current_thread();
+	struct thread *thr = get_current_thread();
 	struct runqueue *rq = thr->tr_rq;
 
 	unsigned long flags;
@@ -121,7 +121,17 @@ void sleep_forever(void)
 
 	rq_unlock(rq, flags);
 
-	while (thr->tr_state == THREAD_SLEEPING) {
+	put_current_thread(thr);
+
+	while (1) {
+		thr = get_current_thread();
+		bool sleep = (thr->tr_state == THREAD_SLEEPING);
+		put_current_thread(thr);
+
+		if (!sleep) {
+			break;
+		}
+
 		schedule(SCHED_NORMAL);
 	}
 }

syscall/address-space.c

@@ -11,13 +11,15 @@ kern_status_t sys_address_space_read(
 	size_t count,
 	size_t *nr_read)
 {
-	struct task *self = current_task();
+	struct task *self = get_current_task();
 
 	if (!validate_access_w(self, dst, count)) {
+		put_current_task(self);
 		return KERN_MEMORY_FAULT;
 	}
 
 	if (nr_read && !validate_access_w(self, nr_read, sizeof *nr_read)) {
+		put_current_task(self);
 		return KERN_MEMORY_FAULT;
 	}
 
@@ -30,12 +32,14 @@ kern_status_t sys_address_space_read(
 		= task_resolve_handle(self, region_handle, &obj, &handle_flags);
 	if (status != KERN_OK) {
 		task_unlock_irqrestore(self, flags);
+		put_current_task(self);
 		return status;
 	}
 
 	struct address_space *region = address_space_cast(obj);
 	if (!region) {
 		task_unlock_irqrestore(self, flags);
+		put_current_task(self);
 		return KERN_INVALID_ARGUMENT;
 	}
 
@@ -52,6 +56,7 @@ kern_status_t sys_address_space_read(
 	address_space_unlock_irqrestore(region, flags);
 
 	object_unref(obj);
+	put_current_task(self);
 
 	return status;
 }
@@ -63,14 +68,16 @@ kern_status_t sys_address_space_write(
 	size_t count,
 	size_t *nr_written)
 {
-	struct task *self = current_task();
+	struct task *self = get_current_task();
 
 	if (!validate_access_r(self, src, count)) {
+		put_current_task(self);
 		return KERN_MEMORY_FAULT;
 	}
 
 	if (nr_written
 	    && !validate_access_w(self, nr_written, sizeof *nr_written)) {
+		put_current_task(self);
 		return KERN_MEMORY_FAULT;
 	}
 
@@ -83,12 +90,14 @@ kern_status_t sys_address_space_write(
 		= task_resolve_handle(self, region_handle, &obj, &handle_flags);
 	if (status != KERN_OK) {
 		task_unlock_irqrestore(self, flags);
+		put_current_task(self);
 		return status;
 	}
 
 	struct address_space *region = address_space_cast(obj);
 	if (!region) {
 		task_unlock_irqrestore(self, flags);
+		put_current_task(self);
 		return KERN_INVALID_ARGUMENT;
 	}
 
@@ -105,6 +114,7 @@ kern_status_t sys_address_space_write(
 	address_space_unlock_irqrestore(region, flags);
 
 	object_unref(obj);
+	put_current_task(self);
 
 	return status;
 }
@@ -115,22 +125,24 @@ kern_status_t sys_address_space_map(
 	kern_handle_t object_handle,
 	off_t object_offset,
 	size_t length,
+	vm_flags_t flags,
 	vm_prot_t prot,
 	virt_addr_t *out_base_address)
 {
-	struct task *self = current_task();
+	struct task *self = get_current_task();
 
 	if (out_base_address
 	    && !validate_access_r(
 		    self,
 		    out_base_address,
 		    sizeof *out_base_address)) {
+		put_current_task(self);
 		return KERN_MEMORY_FAULT;
 	}
 
 	kern_status_t status = KERN_OK;
-	unsigned long flags;
+	unsigned long irq_flags;
-	task_lock_irqsave(self, &flags);
+	task_lock_irqsave(self, &irq_flags);
 
 	struct object *region_obj = NULL, *vmo_obj = NULL;
 	handle_flags_t region_flags = 0, vmo_flags = 0;
@@ -140,30 +152,34 @@ kern_status_t sys_address_space_map(
 		&region_obj,
 		&region_flags);
 	if (status != KERN_OK) {
-		task_unlock_irqrestore(self, flags);
+		task_unlock_irqrestore(self, irq_flags);
+		put_current_task(self);
 		return status;
 	}
 
 	status = task_resolve_handle(self, object_handle, &vmo_obj, &vmo_flags);
 	if (status != KERN_OK) {
-		task_unlock_irqrestore(self, flags);
+		task_unlock_irqrestore(self, irq_flags);
+		put_current_task(self);
 		return status;
 	}
 
 	struct address_space *region = address_space_cast(region_obj);
 	if (!region) {
-		task_unlock_irqrestore(self, flags);
+		task_unlock_irqrestore(self, irq_flags);
+		put_current_task(self);
 		return KERN_INVALID_ARGUMENT;
 	}
 
 	struct vm_object *vmo = vm_object_cast(vmo_obj);
 	if (!vmo) {
-		task_unlock_irqrestore(self, flags);
+		task_unlock_irqrestore(self, irq_flags);
+		put_current_task(self);
 		return KERN_INVALID_ARGUMENT;
 	}
 
-	task_unlock_irqrestore(self, flags);
+	task_unlock_irqrestore(self, irq_flags);
-	address_space_lock_irqsave(region, &flags);
+	address_space_lock_irqsave(region, &irq_flags);
 	/* address_space_map will take care of locking `vmo` */
 	status = address_space_map(
 		region,
@@ -171,12 +187,14 @@ kern_status_t sys_address_space_map(
 		vmo,
 		object_offset,
 		length,
+		flags,
 		prot,
 		out_base_address);
-	address_space_unlock_irqrestore(region, flags);
+	address_space_unlock_irqrestore(region, irq_flags);
 
 	object_unref(vmo_obj);
 	object_unref(region_obj);
+	put_current_task(self);
 
 	return status;
 }
@@ -186,7 +204,7 @@ kern_status_t sys_address_space_unmap(
 	virt_addr_t base,
 	size_t length)
 {
-	struct task *self = current_task();
+	struct task *self = get_current_task();
 
 	kern_status_t status = KERN_OK;
 	unsigned long flags;
@@ -201,12 +219,14 @@ kern_status_t sys_address_space_unmap(
 		&region_flags);
 	if (status != KERN_OK) {
 		task_unlock_irqrestore(self, flags);
+		put_current_task(self);
 		return status;
 	}
 
 	struct address_space *region = address_space_cast(region_obj);
 	if (!region) {
 		task_unlock_irqrestore(self, flags);
+		put_current_task(self);
 		return KERN_INVALID_ARGUMENT;
 	}
 
@@ -215,6 +235,7 @@ kern_status_t sys_address_space_unmap(
 	status = address_space_unmap(region, base, length);
 
 	object_unref(region_obj);
+	put_current_task(self);
 
 	return status;
 }
@@ -225,13 +246,14 @@ kern_status_t sys_address_space_reserve(
 	size_t length,
 	virt_addr_t *out_base_address)
 {
-	struct task *self = current_task();
+	struct task *self = get_current_task();
 
 	if (out_base_address
 	    && !validate_access_r(
 		    self,
 		    out_base_address,
 		    sizeof *out_base_address)) {
+		put_current_task(self);
 		return KERN_MEMORY_FAULT;
 	}
 
@@ -248,12 +270,14 @@ kern_status_t sys_address_space_reserve(
 		&region_flags);
 	if (status != KERN_OK) {
 		task_unlock_irqrestore(self, flags);
+		put_current_task(self);
 		return status;
 	}
 
 	struct address_space *region = address_space_cast(region_obj);
 	if (!region) {
 		task_unlock_irqrestore(self, flags);
+		put_current_task(self);
 		return KERN_INVALID_ARGUMENT;
 	}
 
@@ -268,6 +292,7 @@ kern_status_t sys_address_space_reserve(
 	address_space_unlock_irqrestore(region, flags);
 
 	object_unref(region_obj);
+	put_current_task(self);
 
 	return status;
 }
@@ -277,7 +302,7 @@ kern_status_t sys_address_space_release(
 	virt_addr_t base,
 	size_t length)
 {
-	struct task *self = current_task();
+	struct task *self = get_current_task();
 
 	kern_status_t status = KERN_OK;
 	unsigned long flags;
@@ -292,12 +317,14 @@ kern_status_t sys_address_space_release(
 		&region_flags);
 	if (status != KERN_OK) {
 		task_unlock_irqrestore(self, flags);
+		put_current_task(self);
 		return status;
 	}
 
 	struct address_space *region = address_space_cast(region_obj);
 	if (!region) {
 		task_unlock_irqrestore(self, flags);
+		put_current_task(self);
 		return KERN_INVALID_ARGUMENT;
 	}
 
@@ -308,6 +335,7 @@ kern_status_t sys_address_space_release(
 	address_space_unlock_irqrestore(region, flags);
 
 	object_unref(region_obj);
+	put_current_task(self);
 
 	return status;
 }

syscall/config.c

@@ -3,21 +3,26 @@
 
 kern_status_t sys_kern_config_get(kern_config_key_t key, void *ptr, size_t len)
 {
-	struct task *self = current_task();
+	struct task *self = get_current_task();
+	kern_status_t status = KERN_OK;
 
 	switch (key) {
 	case KERN_CFG_PAGE_SIZE:
 		if (!validate_access_w(self, ptr, sizeof(uintptr_t))) {
-			return KERN_MEMORY_FAULT;
+			status = KERN_MEMORY_FAULT;
+			break;
 		}
 
 		*(uint32_t *)ptr = VM_PAGE_SIZE;
-		return KERN_OK;
+		status = KERN_OK;
+		break;
 	default:
-		return KERN_INVALID_ARGUMENT;
+		status = KERN_INVALID_ARGUMENT;
+		break;
 	}
 
-	return KERN_UNSUPPORTED;
+	put_current_task(self);
+	return status;
 }
 
 kern_status_t sys_kern_config_set(

syscall/dispatch.c

@@ -10,6 +10,9 @@ static const virt_addr_t syscall_table[] = {
 	SYSCALL_TABLE_ENTRY(TASK_CREATE, task_create),
 	SYSCALL_TABLE_ENTRY(TASK_CREATE_THREAD, task_create_thread),
 	SYSCALL_TABLE_ENTRY(TASK_GET_ADDRESS_SPACE, task_get_address_space),
+	SYSCALL_TABLE_ENTRY(TASK_DUPLICATE, task_duplicate),
+	SYSCALL_TABLE_ENTRY(TASK_CONFIG_GET, task_config_get),
+	SYSCALL_TABLE_ENTRY(TASK_CONFIG_SET, task_config_set),
 	SYSCALL_TABLE_ENTRY(THREAD_SELF, thread_self),
 	SYSCALL_TABLE_ENTRY(THREAD_START, thread_start),
 	SYSCALL_TABLE_ENTRY(THREAD_EXIT, thread_exit),
@@ -28,6 +31,7 @@ static const virt_addr_t syscall_table[] = {
 	SYSCALL_TABLE_ENTRY(KERN_LOG, kern_log),
 	SYSCALL_TABLE_ENTRY(KERN_HANDLE_CLOSE, kern_handle_close),
 	SYSCALL_TABLE_ENTRY(KERN_HANDLE_TRANSFER, kern_handle_transfer),
+	SYSCALL_TABLE_ENTRY(KERN_HANDLE_CONTROL, kern_handle_control),
 	SYSCALL_TABLE_ENTRY(KERN_CONFIG_GET, kern_config_get),
 	SYSCALL_TABLE_ENTRY(KERN_CONFIG_SET, kern_config_set),
 	SYSCALL_TABLE_ENTRY(CHANNEL_CREATE, channel_create),
@@ -45,6 +49,12 @@ static const virt_addr_t syscall_table[] = {
 	SYSCALL_TABLE_ENTRY(
 		VM_CONTROLLER_CREATE_OBJECT,
 		vm_controller_create_object),
+	SYSCALL_TABLE_ENTRY(
+		VM_CONTROLLER_PREPARE_ATTACH,
+		vm_controller_prepare_attach),
+	SYSCALL_TABLE_ENTRY(
+		VM_CONTROLLER_FINISH_ATTACH,
+		vm_controller_finish_attach),
 	SYSCALL_TABLE_ENTRY(
 		VM_CONTROLLER_DETACH_OBJECT,
 		vm_controller_detach_object),

syscall/futex.c

@@ -8,18 +8,22 @@ kern_status_t sys_futex_wait(
 	kern_futex_t new_val,
 	unsigned int flags)
 {
-	struct task *self = current_task();
+	struct task *self = get_current_task();
 	if (!validate_access_r(self, futex, sizeof *futex)) {
+		put_current_task(self);
 		return KERN_MEMORY_FAULT;
 	}
 
 	futex_key_t key;
 	kern_status_t status = futex_get(futex, &key, flags);
 	if (status != KERN_OK) {
+		put_current_task(self);
 		return status;
 	}
 
-	return futex_wait(key, new_val, flags);
+	status = futex_wait(key, new_val, flags);
+	put_current_task(self);
+	return status;
 }
 
 kern_status_t sys_futex_wake(

syscall/handle.c

@@ -3,9 +3,13 @@
 
 kern_status_t sys_kern_handle_close(kern_handle_t handle)
 {
-	struct task *self = current_task();
+	struct task *self = get_current_task();
 
-	return task_close_handle(self, handle);
+	kern_status_t status = task_close_handle(self, handle);
+
+	put_current_task(self);
+
+	return status;
 }
 
 kern_status_t sys_kern_handle_transfer(
@@ -24,10 +28,11 @@ kern_status_t sys_kern_handle_transfer(
 		return KERN_INVALID_ARGUMENT;
 	}
 
-	struct task *self = current_task();
+	struct task *self = get_current_task();
 
 	if (out_handle
 	    && !validate_access_w(self, out_handle, sizeof *out_handle)) {
+		put_current_task(self);
 		return KERN_MEMORY_FAULT;
 	}
 
@@ -56,6 +61,7 @@ kern_status_t sys_kern_handle_transfer(
 		src_task = task_cast(obj);
 		if (!src_task) {
 			status = KERN_INVALID_ARGUMENT;
+			task_unlock_irqrestore(self, flags);
 			goto cleanup;
 		}
 	} else {
@@ -76,6 +82,7 @@ kern_status_t sys_kern_handle_transfer(
 		dest_task = task_cast(obj);
 		if (!dest_task) {
 			status = KERN_INVALID_ARGUMENT;
+			task_unlock_irqrestore(self, flags);
 			goto cleanup;
 		}
 	} else {
@@ -87,12 +94,12 @@ kern_status_t sys_kern_handle_transfer(
 		src_handle,
 		&src_object,
 		&handle_flags);
+	task_unlock_irqrestore(self, flags);
+
 	if (status != KERN_OK) {
 		goto cleanup;
 	}
 
-	task_unlock_irqrestore(self, flags);
-
 	struct handle *dest = NULL;
 	task_lock_irqsave(dest_task, &flags);
 	status = handle_table_alloc_handle(
@@ -120,6 +127,8 @@ kern_status_t sys_kern_handle_transfer(
 		*out_handle = dest_handle;
 	}
 
+	put_current_task(self);
+
 	return KERN_OK;
 
 cleanup:
@@ -135,5 +144,17 @@ cleanup:
 		object_unref(src_object);
 	}
 
+	put_current_task(self);
+
 	return status;
 }
+
+kern_status_t sys_kern_handle_control(
+	kern_handle_t task,
+	kern_handle_t handle,
+	uint32_t set_mask,
+	uint32_t clear_mask,
+	uint32_t *out_flags)
+{
+	return KERN_UNIMPLEMENTED;
+}

syscall/log.c

@@ -6,9 +6,11 @@
 kern_status_t sys_kern_log(const char *s)
 {
 #ifdef TRACE
-	struct task *task = current_task();
+	struct task *task = get_current_task();
-	struct thread *thread = current_thread();
+	struct thread *thread = get_current_thread();
 	printk("%s[%d.%d]: %s", task->t_name, task->t_id, thread->tr_id, s);
+	put_current_thread(thread);
+	put_current_task(task);
 #else
 	printk("%s", s);
 #endif

syscall/msg.c

@@ -7,13 +7,15 @@
 
 kern_status_t sys_channel_create(unsigned int id, kern_handle_t *out)
 {
-	struct task *self = current_task();
+	struct task *self = get_current_task();
 	if (!validate_access_w(self, out, sizeof *out)) {
+		put_current_task(self);
 		return KERN_MEMORY_FAULT;
 	}
 
 	struct channel *channel = channel_create();
 	if (!channel) {
+		put_current_task(self);
 		return KERN_NO_MEMORY;
 	}
 
@@ -22,6 +24,7 @@ kern_status_t sys_channel_create(unsigned int id, kern_handle_t *out)
 
 	if (task_get_channel(self, id)) {
 		task_unlock_irqrestore(self, irq_flags);
+		put_current_task(self);
 		return KERN_NAME_EXISTS;
 	}
 
@@ -31,11 +34,13 @@ kern_status_t sys_channel_create(unsigned int id, kern_handle_t *out)
 	if (status != KERN_OK) {
 		task_unlock_irqrestore(self, irq_flags);
 		object_unref(&channel->c_base);
+		put_current_task(self);
 		return status;
 	}
 
 	task_add_channel(self, channel, id);
 	task_unlock_irqrestore(self, irq_flags);
+	put_current_task(self);
 
 	*out = handle;
 	return KERN_OK;
@@ -43,13 +48,15 @@ kern_status_t sys_channel_create(unsigned int id, kern_handle_t *out)
 
 kern_status_t sys_port_create(kern_handle_t *out)
 {
-	struct task *self = current_task();
+	struct task *self = get_current_task();
 	if (!validate_access_w(self, out, sizeof *out)) {
+		put_current_task(self);
 		return KERN_MEMORY_FAULT;
 	}
 
 	struct port *port = port_create();
 	if (!port) {
+		put_current_task(self);
 		return KERN_NO_MEMORY;
 	}
 
@@ -61,6 +68,7 @@ kern_status_t sys_port_create(kern_handle_t *out)
 		= task_open_handle(self, &port->p_base, 0, &handle);
 	task_unlock_irqrestore(self, irq_flags);
 	object_unref(&port->p_base);
+	put_current_task(self);
 
 	if (status != KERN_OK) {
 		return status;
@@ -77,7 +85,7 @@ kern_status_t sys_port_connect(
 {
 	unsigned long flags;
 
-	struct task *self = current_task();
+	struct task *self = get_current_task();
 	task_lock_irqsave(self, &flags);
 
 	struct object *port_obj = NULL;
@@ -88,6 +96,7 @@ kern_status_t sys_port_connect(
 		&port_obj,
 		&port_handle_flags);
 	if (status != KERN_OK) {
+		put_current_task(self);
 		return status;
 	}
 
@@ -96,6 +105,7 @@ kern_status_t sys_port_connect(
 
 	struct task *remote_task = task_from_tid(task_id);
 	if (!remote_task) {
+		put_current_task(self);
 		return KERN_NO_ENTRY;
 	}
 
@@ -104,6 +114,7 @@ kern_status_t sys_port_connect(
 	struct channel *remote = task_get_channel(remote_task, channel_id);
 	if (!remote) {
 		task_unlock_irqrestore(remote_task, flags);
+		put_current_task(self);
 		return KERN_NO_ENTRY;
 	}
 
@@ -115,6 +126,7 @@ kern_status_t sys_port_connect(
 	port_unlock_irqrestore(port, flags);
 	object_unref(&remote->c_base);
 	object_unref(port_obj);
+	put_current_task(self);
 
 	return KERN_OK;
 }
@@ -123,7 +135,7 @@ kern_status_t sys_port_disconnect(kern_handle_t port_handle)
 {
 	unsigned long flags;
 
-	struct task *self = current_task();
+	struct task *self = get_current_task();
 	task_lock_irqsave(self, &flags);
 
 	struct object *port_obj = NULL;
@@ -134,6 +146,7 @@ kern_status_t sys_port_disconnect(kern_handle_t port_handle)
 		&port_obj,
 		&port_handle_flags);
 	if (status != KERN_OK) {
+		put_current_task(self);
 		return status;
 	}
 
@@ -142,6 +155,7 @@ kern_status_t sys_port_disconnect(kern_handle_t port_handle)
 	struct port *port = port_cast(port_obj);
 	if (!port) {
 		object_unref(port_obj);
+		put_current_task(self);
 		return KERN_INVALID_ARGUMENT;
 	}
 
@@ -149,6 +163,7 @@ kern_status_t sys_port_disconnect(kern_handle_t port_handle)
 	port_lock_irqsave(port, &flags);
 	status = port_disconnect(port);
 	port_unlock_irqrestore(port, flags);
+	put_current_task(self);
 
 	return status;
 }
@@ -219,13 +234,15 @@ kern_status_t sys_msg_send(
 	const kern_msg_t *msg,
 	kern_msg_t *out_reply)
 {
-	struct task *self = current_task();
+	struct task *self = get_current_task();
 
 	if (!validate_msg(self, msg, false)) {
+		put_current_task(self);
 		return KERN_MEMORY_FAULT;
 	}
 
 	if (!validate_msg(self, out_reply, true)) {
+		put_current_task(self);
 		return KERN_MEMORY_FAULT;
 	}
 
@@ -243,12 +260,14 @@ kern_status_t sys_msg_send(
 	task_unlock_irqrestore(self, flags);
 
 	if (status != KERN_OK) {
+		put_current_task(self);
 		return status;
 	}
 
 	struct port *port = port_cast(port_obj);
 	if (!port) {
 		object_unref(port_obj);
+		put_current_task(self);
 		return KERN_INVALID_ARGUMENT;
 	}
 
@@ -256,15 +275,17 @@ kern_status_t sys_msg_send(
 	status = port_send_msg(port, msg, out_reply, &flags);
 	port_unlock_irqrestore(port, flags);
 	object_unref(port_obj);
+	put_current_task(self);
 
 	return status;
 }
 
 kern_status_t sys_msg_recv(kern_handle_t channel_handle, kern_msg_t *out_msg)
 {
-	struct task *self = current_task();
+	struct task *self = get_current_task();
 
 	if (!validate_msg(self, out_msg, true)) {
+		put_current_task(self);
 		return KERN_MEMORY_FAULT;
 	}
 
@@ -280,6 +301,7 @@ kern_status_t sys_msg_recv(kern_handle_t channel_handle, kern_msg_t *out_msg)
 		&channel_obj,
 		&channel_handle_flags);
 	if (status != KERN_OK) {
+		put_current_task(self);
 		return status;
 	}
 
@@ -288,6 +310,7 @@ kern_status_t sys_msg_recv(kern_handle_t channel_handle, kern_msg_t *out_msg)
 	struct channel *channel = channel_cast(channel_obj);
 	if (!channel) {
 		object_unref(channel_obj);
+		put_current_task(self);
 		return KERN_INVALID_ARGUMENT;
 	}
 
@@ -295,6 +318,7 @@ kern_status_t sys_msg_recv(kern_handle_t channel_handle, kern_msg_t *out_msg)
 	status = channel_recv_msg(channel, out_msg, &flags);
 	channel_unlock_irqrestore(channel, flags);
 	object_unref(channel_obj);
+	put_current_task(self);
 
 	return status;
 }
@@ -304,9 +328,10 @@ kern_status_t sys_msg_reply(
 	msgid_t id,
 	const kern_msg_t *reply)
 {
-	struct task *self = current_task();
+	struct task *self = get_current_task();
 
 	if (!validate_msg(self, reply, true)) {
+		put_current_task(self);
 		return KERN_MEMORY_FAULT;
 	}
 
@@ -322,6 +347,7 @@ kern_status_t sys_msg_reply(
 		&channel_obj,
 		&channel_handle_flags);
 	if (status != KERN_OK) {
+		put_current_task(self);
 		return status;
 	}
 
@@ -330,6 +356,7 @@ kern_status_t sys_msg_reply(
 	struct channel *channel = channel_cast(channel_obj);
 	if (!channel) {
 		object_unref(channel_obj);
+		put_current_task(self);
 		return KERN_INVALID_ARGUMENT;
 	}
 
@@ -337,6 +364,7 @@ kern_status_t sys_msg_reply(
 	status = channel_reply_msg(channel, id, reply, &flags);
 	channel_unlock_irqrestore(channel, flags);
 	object_unref(channel_obj);
+	put_current_task(self);
 
 	return status;
 }
@@ -349,13 +377,15 @@ kern_status_t sys_msg_read(
 	size_t iov_count,
 	size_t *nr_read)
 {
-	struct task *self = current_task();
+	struct task *self = get_current_task();
 
 	if (nr_read && !validate_access_w(self, nr_read, sizeof *nr_read)) {
+		put_current_task(self);
 		return KERN_MEMORY_FAULT;
 	}
 
 	if (!validate_iovec(self, iov, iov_count, true)) {
+		put_current_task(self);
 		return KERN_MEMORY_FAULT;
 	}
 
@@ -371,6 +401,7 @@ kern_status_t sys_msg_read(
 		&channel_obj,
 		&channel_handle_flags);
 	if (status != KERN_OK) {
+		put_current_task(self);
 		return status;
 	}
 
@@ -379,6 +410,7 @@ kern_status_t sys_msg_read(
 	struct channel *channel = channel_cast(channel_obj);
 	if (!channel) {
 		object_unref(channel_obj);
+		put_current_task(self);
 		return KERN_INVALID_ARGUMENT;
 	}
 
@@ -393,6 +425,7 @@ kern_status_t sys_msg_read(
 		nr_read);
 	channel_unlock_irqrestore(channel, flags);
 	object_unref(channel_obj);
+	put_current_task(self);
 
 	return status;
 }
@@ -405,14 +438,16 @@ kern_status_t sys_msg_write(
 	size_t iov_count,
 	size_t *nr_written)
 {
-	struct task *self = current_task();
+	struct task *self = get_current_task();
 
 	if (nr_written
 	    && !validate_access_w(self, nr_written, sizeof *nr_written)) {
+		put_current_task(self);
 		return KERN_MEMORY_FAULT;
 	}
 
 	if (!validate_iovec(self, iov, iov_count, false)) {
+		put_current_task(self);
 		return KERN_MEMORY_FAULT;
 	}
 
@@ -428,6 +463,7 @@ kern_status_t sys_msg_write(
 		&channel_obj,
 		&channel_handle_flags);
 	if (status != KERN_OK) {
+		put_current_task(self);
 		return status;
 	}
 
@@ -436,6 +472,7 @@ kern_status_t sys_msg_write(
 	struct channel *channel = channel_cast(channel_obj);
 	if (!channel) {
 		object_unref(channel_obj);
+		put_current_task(self);
 		return KERN_INVALID_ARGUMENT;
 	}
 
@@ -450,6 +487,7 @@ kern_status_t sys_msg_write(
 		nr_written);
 	channel_unlock_irqrestore(channel, flags);
 	object_unref(channel_obj);
+	put_current_task(self);
 
 	return status;
 }

syscall/object.c

@@ -4,8 +4,8 @@
 #include <kernel/task.h>
 #include <kernel/thread.h>
 #include <kernel/wait.h>
-#include <mango/status.h>
+#include <magenta/status.h>
-#include <mango/types.h>
+#include <magenta/types.h>
 
 kern_status_t sys_kern_object_wait(kern_wait_item_t *items, size_t nr_items)
 {
@@ -13,12 +13,13 @@ kern_status_t sys_kern_object_wait(kern_wait_item_t *items, size_t nr_items)
 		return KERN_INVALID_ARGUMENT;
 	}
 
-	struct task *self = current_task();
+	struct task *self = get_current_task();
-	struct thread *self_thread = current_thread();
 	if (!validate_access_rw(self, items, nr_items * sizeof *items)) {
+		put_current_task(self);
 		return KERN_MEMORY_FAULT;
 	}
 
+	struct thread *self_thread = get_current_thread();
 	self_thread->tr_state = THREAD_SLEEPING;
 
 	kern_status_t status = KERN_OK;
@@ -78,6 +79,9 @@ cleanup:
 	}
 
 	self_thread->tr_state = THREAD_READY;
+	put_current_thread(self_thread);
+	put_current_task(self);
+
 	return status;
 }
 
@@ -85,13 +89,15 @@ kern_status_t sys_kern_object_query(
 	kern_handle_t object_handle,
 	kern_object_info_t *out)
 {
-	struct task *self = current_task();
+	struct task *self = get_current_task();
 
 	if (!out) {
+		put_current_task(self);
 		return KERN_INVALID_ARGUMENT;
 	}
 
 	if (!validate_access_w(self, out, sizeof *out)) {
+		put_current_task(self);
 		return KERN_MEMORY_FAULT;
 	}
 
@@ -100,12 +106,14 @@ kern_status_t sys_kern_object_query(
 	kern_status_t status
 		= task_resolve_handle(self, object_handle, &obj, &flags);
 	if (status != KERN_OK) {
+		put_current_task(self);
 		return status;
 	}
 
 	out->obj_id = obj->ob_id;
 
 	object_unref(obj);
+	put_current_task(self);
 
 	return KERN_OK;
 }

syscall/task.c

@@ -1,5 +1,6 @@
 #include <kernel/address-space.h>
 #include <kernel/machine/cpu.h>
+#include <kernel/panic.h>
 #include <kernel/printk.h>
 #include <kernel/sched.h>
 #include <kernel/syscall.h>
@@ -9,8 +10,9 @@
 extern kern_status_t sys_task_exit(int status)
 {
 #if defined(TRACE)
-	struct task *self = current_task();
+	struct task *self = get_current_task();
 	printk("%s[%d]: task_exit(%d)", self->t_name, self->t_id, status);
+	put_current_task(self);
 #endif
 	task_exit(status);
 	return KERN_FATAL_ERROR;
@@ -18,8 +20,9 @@ extern kern_status_t sys_task_exit(int status)
 
 kern_status_t sys_task_self(kern_handle_t *out)
 {
-	struct task *self = current_task();
+	struct task *self = get_current_task();
 	if (!validate_access_w(self, out, sizeof *out)) {
+		put_current_task(self);
 		return KERN_MEMORY_FAULT;
 	}
 
@@ -36,11 +39,13 @@ kern_status_t sys_task_self(kern_handle_t *out)
 	task_unlock_irqrestore(self, flags);
 
 	if (status != KERN_OK) {
+		put_current_task(self);
 		return status;
 	}
 
 	object_ref(&self->t_base);
 	handle_slot->h_object = &self->t_base;
+	put_current_task(self);
 
 	*out = handle;
 	return KERN_OK;
@@ -48,19 +53,22 @@ kern_status_t sys_task_self(kern_handle_t *out)
 
 kern_status_t sys_task_create(
 	kern_handle_t parent_handle,
+	task_flags_t task_flags,
 	const char *name,
 	size_t name_len,
 	kern_handle_t *out_task,
 	kern_handle_t *out_address_space)
 {
 	unsigned long flags;
-	struct task *self = current_task();
+	struct task *self = get_current_task();
 
 	if (name_len && !validate_access_r(self, name, name_len)) {
+		put_current_task(self);
 		return KERN_MEMORY_FAULT;
 	}
 
 	if (!validate_access_w(self, out_task, sizeof *out_task)) {
+		put_current_task(self);
 		return KERN_MEMORY_FAULT;
 	}
 
@@ -68,6 +76,7 @@ kern_status_t sys_task_create(
 		    self,
 		    out_address_space,
 		    sizeof *out_address_space)) {
+		put_current_task(self);
 		return KERN_MEMORY_FAULT;
 	}
 
@@ -81,6 +90,7 @@ kern_status_t sys_task_create(
 		&parent_flags);
 	if (status != KERN_OK) {
 		task_unlock_irqrestore(self, flags);
+		put_current_task(self);
 		return status;
 	}
 
@@ -96,6 +106,7 @@ kern_status_t sys_task_create(
 	if (status != KERN_OK) {
 		object_unref(parent_obj);
 		task_unlock_irqrestore(self, flags);
+		put_current_task(self);
 		return status;
 	}
 
@@ -108,12 +119,13 @@ kern_status_t sys_task_create(
 		object_unref(parent_obj);
 		handle_table_free_handle(self->t_handles, child_handle);
 		task_unlock_irqrestore(self, flags);
+		put_current_task(self);
 		return status;
 	}
 
 	task_unlock_irqrestore(self, flags);
 
-	struct task *child = task_create(name, name_len);
+	struct task *child = task_create(parent, task_flags, name, name_len);
 	if (!child) {
 		object_unref(parent_obj);
 
@@ -121,6 +133,7 @@ kern_status_t sys_task_create(
 		handle_table_free_handle(self->t_handles, child_handle);
 		handle_table_free_handle(self->t_handles, space_handle);
 		task_unlock_irqrestore(self, flags);
+		put_current_task(self);
 
 		return KERN_NO_MEMORY;
 	}
@@ -136,6 +149,7 @@ kern_status_t sys_task_create(
 	object_ref(&child->t_address_space->s_base);
 
 	object_unref(parent_obj);
+	put_current_task(self);
 
 	*out_task = child_handle;
 	*out_address_space = space_handle;
@@ -152,13 +166,15 @@ kern_status_t sys_task_create_thread(
 	kern_handle_t *out_thread)
 {
 	unsigned long flags;
-	struct task *self = current_task();
+	struct task *self = get_current_task();
 
 	if (!validate_access_r(self, args, nr_args * sizeof(uintptr_t))) {
+		put_current_task(self);
 		return KERN_MEMORY_FAULT;
 	}
 
 	if (!validate_access_w(self, out_thread, sizeof *out_thread)) {
+		put_current_task(self);
 		return KERN_MEMORY_FAULT;
 	}
 
@@ -169,6 +185,7 @@ kern_status_t sys_task_create_thread(
 		= task_resolve_handle(self, task, &target_obj, &target_flags);
 	if (status != KERN_OK) {
 		task_unlock_irqrestore(self, flags);
+		put_current_task(self);
 		return status;
 	}
 
@@ -184,6 +201,7 @@ kern_status_t sys_task_create_thread(
 	if (status != KERN_OK) {
 		object_unref(target_obj);
 		task_unlock_irqrestore(self, flags);
+		put_current_task(self);
 		return status;
 	}
 
@@ -198,6 +216,7 @@ kern_status_t sys_task_create_thread(
 		task_lock_irqsave(self, &flags);
 		handle_table_free_handle(self->t_handles, out_handle);
 		task_unlock_irqrestore(self, flags);
+		put_current_task(self);
 		return KERN_NO_MEMORY;
 	}
 
@@ -207,6 +226,7 @@ kern_status_t sys_task_create_thread(
 
 	task_unlock_irqrestore(target, flags);
 	object_unref(target_obj);
+	put_current_task(self);
 
 	*out_thread = out_handle;
 	return KERN_OK;
@@ -216,8 +236,9 @@ kern_status_t sys_task_get_address_space(
 	kern_handle_t task_handle,
 	kern_handle_t *out)
 {
-	struct task *self = current_task();
+	struct task *self = get_current_task();
 	if (!validate_access_w(self, out, sizeof *out)) {
+		put_current_task(self);
 		return KERN_MEMORY_FAULT;
 	}
 
@@ -235,6 +256,7 @@ kern_status_t sys_task_get_address_space(
 		&handle_flags);
 	if (status != KERN_OK) {
 		task_unlock_irqrestore(self, flags);
+		put_current_task(self);
 		return status;
 	}
 
@@ -246,6 +268,7 @@ kern_status_t sys_task_get_address_space(
 	if (status != KERN_OK) {
 		object_unref(task_obj);
 		task_unlock_irqrestore(self, flags);
+		put_current_task(self);
 		return status;
 	}
 
@@ -255,6 +278,7 @@ kern_status_t sys_task_get_address_space(
 		object_unref(task_obj);
 		handle_table_free_handle(self->t_handles, handle);
 		task_unlock_irqrestore(self, flags);
+		put_current_task(self);
 		return KERN_INVALID_ARGUMENT;
 	}
 
@@ -262,19 +286,104 @@ kern_status_t sys_task_get_address_space(
 	object_ref(&task->t_address_space->s_base);
 	task_unlock_irqrestore(self, flags);
 	object_unref(task_obj);
+	put_current_task(self);
 
 	*out = handle;
 	return KERN_OK;
 }
 
-kern_status_t sys_thread_self(kern_handle_t *out)
+kern_status_t sys_task_config_get(
+	kern_handle_t task_handle,
+	kern_config_key_t key,
+	void *ptr,
+	size_t len)
 {
-	struct task *self = current_task();
+	unsigned long flags;
-	if (!validate_access_w(self, out, sizeof *out)) {
+	struct task *self = get_current_task();
+
+	if (!validate_access_w(self, ptr, len)) {
+		put_current_task(self);
 		return KERN_MEMORY_FAULT;
 	}
 
-	struct thread *self_thread = current_thread();
+	struct object *task_obj;
+	handle_flags_t task_flags;
+	task_lock_irqsave(self, &flags);
+	kern_status_t status = task_resolve_handle(
+		self,
+		task_handle,
+		&task_obj,
+		&task_flags);
+	put_current_task(self);
+	task_unlock_irqrestore(self, flags);
+
+	if (status != KERN_OK) {
+		return status;
+	}
+
+	struct task *task = task_cast(task_obj);
+	task_unlock_irqrestore(self, flags);
+
+	if (task) {
+		status = task_config_get(task, key, ptr, len);
+	} else {
+		status = KERN_INVALID_ARGUMENT;
+	}
+
+	object_unref(task_obj);
+
+	return status;
+}
+
+kern_status_t sys_task_config_set(
+	kern_handle_t task_handle,
+	kern_config_key_t key,
+	const void *ptr,
+	size_t len)
+{
+	unsigned long flags;
+	struct task *self = get_current_task();
+
+	if (!validate_access_w(self, ptr, len)) {
+		put_current_task(self);
+		return KERN_MEMORY_FAULT;
+	}
+
+	struct object *task_obj;
+	handle_flags_t task_flags;
+	task_lock_irqsave(self, &flags);
+	kern_status_t status = task_resolve_handle(
+		self,
+		task_handle,
+		&task_obj,
+		&task_flags);
+	task_unlock_irqrestore(self, flags);
+	put_current_task(self);
+	if (status != KERN_OK) {
+		return status;
+	}
+
+	struct task *task = task_cast(task_obj);
+	if (task) {
+		status = task_config_set(task, key, ptr, len);
+	} else {
+		status = KERN_INVALID_ARGUMENT;
+	}
+
+	object_unref(task_obj);
+
+	return status;
+}
+
+kern_status_t sys_thread_self(kern_handle_t *out)
+{
+	struct task *self = get_current_task();
+	if (!validate_access_w(self, out, sizeof *out)) {
+		put_current_task(self);
+		return KERN_MEMORY_FAULT;
+	}
+
+	struct thread *self_thread = get_current_thread();
 
 	unsigned long flags;
 	task_lock_irqsave(self, &flags);
@@ -289,11 +398,15 @@ kern_status_t sys_thread_self(kern_handle_t *out)
 	task_unlock_irqrestore(self, flags);
 
 	if (status != KERN_OK) {
+		put_current_thread(self_thread);
+		put_current_task(self);
 		return status;
 	}
 
 	object_ref(&self_thread->tr_base);
 	handle_slot->h_object = &self_thread->tr_base;
+	put_current_thread(self_thread);
+	put_current_task(self);
 
 	*out = handle;
 	return KERN_OK;
@@ -302,7 +415,7 @@ kern_status_t sys_thread_self(kern_handle_t *out)
 kern_status_t sys_thread_start(kern_handle_t thread_handle)
 {
 	unsigned long flags;
-	struct task *self = current_task();
+	struct task *self = get_current_task();
 
 	struct object *thread_obj;
 	handle_flags_t thread_flags;
@@ -314,6 +427,7 @@ kern_status_t sys_thread_start(kern_handle_t thread_handle)
 		&thread_flags);
 	if (status != KERN_OK) {
 		task_unlock_irqrestore(self, flags);
+		put_current_task(self);
 		return status;
 	}
 
@@ -322,6 +436,7 @@ kern_status_t sys_thread_start(kern_handle_t thread_handle)
 
 	schedule_thread_on_cpu(thread);
 	object_unref(thread_obj);
+	put_current_task(self);
 
 	return KERN_OK;
 }
@@ -340,9 +455,10 @@ kern_status_t sys_thread_config_get(
 	size_t len)
 {
 	unsigned long flags;
-	struct task *self = current_task();
+	struct task *self = get_current_task();
 
 	if (!validate_access_w(self, ptr, len)) {
+		put_current_task(self);
 		return KERN_MEMORY_FAULT;
 	}
 
@@ -356,6 +472,7 @@ kern_status_t sys_thread_config_get(
 		&thread_flags);
 	if (status != KERN_OK) {
 		task_unlock_irqrestore(self, flags);
+		put_current_task(self);
 		return status;
 	}
 
@@ -365,6 +482,7 @@ kern_status_t sys_thread_config_get(
 	status = thread_config_get(thread, key, ptr, len);
 
 	object_unref(thread_obj);
+	put_current_task(self);
 
 	return status;
 }
@@ -376,9 +494,10 @@ kern_status_t sys_thread_config_set(
 	size_t len)
 {
 	unsigned long flags;
-	struct task *self = current_task();
+	struct task *self = get_current_task();
 
 	if (!validate_access_w(self, ptr, len)) {
+		put_current_task(self);
 		return KERN_MEMORY_FAULT;
 	}
 
@@ -392,6 +511,7 @@ kern_status_t sys_thread_config_set(
 		&thread_flags);
 	if (status != KERN_OK) {
 		task_unlock_irqrestore(self, flags);
+		put_current_task(self);
 		return status;
 	}
 
@@ -401,6 +521,112 @@ kern_status_t sys_thread_config_set(
 	status = thread_config_set(thread, key, ptr, len);
 
 	object_unref(thread_obj);
+	put_current_task(self);
 
 	return status;
 }
+
+kern_status_t sys_task_duplicate(
+	kern_handle_t *out_task,
+	kern_handle_t *out_address_space)
+{
+	struct task *self = get_current_task();
+
+	if (!validate_access_w(self, out_task, sizeof *out_task)) {
+		put_current_task(self);
+		return KERN_MEMORY_FAULT;
+	}
+
+	if (!validate_access_w(
+		    self,
+		    out_address_space,
+		    sizeof *out_address_space)) {
+		put_current_task(self);
+		return KERN_MEMORY_FAULT;
+	}
+
+	*out_task = KERN_HANDLE_INVALID;
+	*out_address_space = KERN_HANDLE_INVALID;
+
+	kern_status_t status = KERN_OK;
+	unsigned long flags;
+	task_lock_irqsave(self, &flags);
+
+	struct handle *child_handle_slot = NULL, *space_handle_slot = NULL;
+	kern_handle_t child_handle, space_handle;
+	status = handle_table_alloc_handle(
+		self->t_handles,
+		KERN_HANDLE_INVALID,
+		&child_handle_slot,
+		&child_handle);
+	if (status != KERN_OK) {
+		task_unlock_irqrestore(self, flags);
+		put_current_task(self);
+		return status;
+	}
+
+	status = handle_table_alloc_handle(
+		self->t_handles,
+		KERN_HANDLE_INVALID,
+		&space_handle_slot,
+		&space_handle);
+	if (status != KERN_OK) {
+		handle_table_free_handle(self->t_handles, child_handle);
+		task_unlock_irqrestore(self, flags);
+		put_current_task(self);
+		return status;
+	}
+
+	struct task *new_task = task_create(
+		self,
+		TASK_F_CLONE_ALL_HANDLES,
+		self->t_name,
+		strlen(self->t_name));
+	if (!new_task) {
+		put_current_task(self);
+		return KERN_NO_MEMORY;
+	}
+
+	struct thread *new_thread = task_create_thread(new_task);
+	if (!new_thread) {
+		handle_table_free_handle(self->t_handles, child_handle);
+		handle_table_free_handle(self->t_handles, space_handle);
+		task_unlock_irqrestore(self, flags);
+		object_unref(&new_task->t_base);
+		put_current_task(self);
+		return KERN_NO_MEMORY;
+	}
+
+	struct thread *self_thread = get_current_thread();
+	thread_init_user_clone(new_thread, self_thread, KERN_OK);
+	put_current_thread(self_thread);
+
+	status = address_space_duplicate(
+		new_task->t_address_space,
+		self->t_address_space);
+	if (status != KERN_OK) {
+		handle_table_free_handle(self->t_handles, child_handle);
+		handle_table_free_handle(self->t_handles, space_handle);
+		task_unlock_irqrestore(self, flags);
+		object_unref(&new_thread->tr_base);
+		object_unref(&new_task->t_base);
+		put_current_task(self);
+		return status;
+	}
+
+	child_handle_slot->h_object = &new_task->t_base;
+	space_handle_slot->h_object
+		= object_ref(&new_task->t_address_space->s_base);
+	task_unlock_irqrestore(self, flags);
+
+	/* clear TLB */
+	pmap_flush();
+	put_current_task(self);
+
+	*out_task = child_handle;
+	*out_address_space = space_handle;
+
+	schedule_thread_on_cpu(new_thread);
+
+	return KERN_OK;
+}

syscall/vm-controller.c

@@ -7,18 +7,21 @@
 
 kern_status_t sys_vm_controller_create(kern_handle_t *out)
 {
-	struct task *self = current_task();
+	struct task *self = get_current_task();
 
 	if (!validate_access_w(self, out, sizeof *out)) {
+		put_current_task(self);
 		return KERN_MEMORY_FAULT;
 	}
 
 	struct vm_controller *ctrl = vm_controller_create();
 	if (!ctrl) {
+		put_current_task(self);
 		return KERN_NO_MEMORY;
 	}
 
 	kern_status_t status = task_open_handle(self, &ctrl->vc_base, 0, out);
+	put_current_task(self);
 	if (status != KERN_OK) {
 		object_unref(&ctrl->vc_base);
 		return status;
@@ -29,11 +32,12 @@ kern_status_t sys_vm_controller_create(kern_handle_t *out)
 
 kern_status_t sys_vm_controller_recv(
 	kern_handle_t ctrl_handle,
-	equeue_packet_page_request_t *out)
+	equeue_packet_vm_request_t *out)
 {
-	struct task *self = current_task();
+	struct task *self = get_current_task();
 
 	if (!validate_access_w(self, out, sizeof *out)) {
+		put_current_task(self);
 		return KERN_MEMORY_FAULT;
 	}
 
@@ -50,6 +54,7 @@ kern_status_t sys_vm_controller_recv(
 		&handle_flags);
 	if (status != KERN_OK) {
 		task_unlock_irqrestore(self, flags);
+		put_current_task(self);
 		return status;
 	}
 
@@ -57,6 +62,7 @@ kern_status_t sys_vm_controller_recv(
 	task_unlock_irqrestore(self, flags);
 	if (!ctrl) {
 		object_unref(ctrl_obj);
+		put_current_task(self);
 		return KERN_INVALID_ARGUMENT;
 	}
 
@@ -65,6 +71,7 @@ kern_status_t sys_vm_controller_recv(
 	vm_controller_unlock_irqrestore(ctrl, flags);
 
 	object_unref(ctrl_obj);
+	put_current_task(self);
 
 	return status;
 }
@@ -74,7 +81,7 @@ kern_status_t sys_vm_controller_recv_async(
 	kern_handle_t eq_handle,
 	equeue_key_t key)
 {
-	struct task *self = current_task();
+	struct task *self = get_current_task();
 
 	kern_status_t status = KERN_OK;
 	unsigned long flags;
@@ -85,6 +92,7 @@ kern_status_t sys_vm_controller_recv_async(
 	status = task_resolve_handle(self, ctrl_handle, &ctrl_obj, &ctrl_flags);
 	if (status != KERN_OK) {
 		task_unlock_irqrestore(self, flags);
+		put_current_task(self);
 		return status;
 	}
 
@@ -92,6 +100,7 @@ kern_status_t sys_vm_controller_recv_async(
 	if (status != KERN_OK) {
 		object_unref(ctrl_obj);
 		task_unlock_irqrestore(self, flags);
+		put_current_task(self);
 		return status;
 	}
 
@@ -102,6 +111,7 @@ kern_status_t sys_vm_controller_recv_async(
 	if (!ctrl || !eq) {
 		object_unref(ctrl_obj);
 		object_unref(eq_obj);
+		put_current_task(self);
 		return KERN_INVALID_ARGUMENT;
 	}
 
@@ -111,6 +121,7 @@ kern_status_t sys_vm_controller_recv_async(
 
 	object_unref(ctrl_obj);
 	object_unref(eq_obj);
+	put_current_task(self);
 
 	return status;
 }
@@ -124,19 +135,21 @@ kern_status_t sys_vm_controller_create_object(
 	vm_prot_t prot,
 	kern_handle_t *out)
 {
-	struct task *self = current_task();
+	struct task *self = get_current_task();
 
 	if (!validate_access_r(self, name, name_len)) {
+		put_current_task(self);
 		return KERN_MEMORY_FAULT;
 	}
 
 	if (!validate_access_w(self, out, sizeof *out)) {
+		put_current_task(self);
 		return KERN_MEMORY_FAULT;
 	}
 
 	kern_status_t status = KERN_OK;
-	unsigned long flags;
+	unsigned long irq_flags;
-	task_lock_irqsave(self, &flags);
+	task_lock_irqsave(self, &irq_flags);
 
 	struct object *ctrl_obj = NULL;
 	handle_flags_t handle_flags = 0;
@@ -146,7 +159,8 @@ kern_status_t sys_vm_controller_create_object(
 		&ctrl_obj,
 		&handle_flags);
 	if (status != KERN_OK) {
-		task_unlock_irqrestore(self, flags);
+		task_unlock_irqrestore(self, irq_flags);
+		put_current_task(self);
 		return status;
 	}
 
@@ -159,13 +173,14 @@ kern_status_t sys_vm_controller_create_object(
 		&out_handle);
 
 	struct vm_controller *ctrl = vm_controller_cast(ctrl_obj);
-	task_unlock_irqrestore(self, flags);
+	task_unlock_irqrestore(self, irq_flags);
 	if (!ctrl) {
 		object_unref(ctrl_obj);
+		put_current_task(self);
 		return KERN_INVALID_ARGUMENT;
 	}
 
-	vm_controller_lock_irqsave(ctrl, &flags);
+	vm_controller_lock_irqsave(ctrl, &irq_flags);
 	struct vm_object *out_vmo = NULL;
 	status = vm_controller_create_object(
 		ctrl,
@@ -175,28 +190,140 @@ kern_status_t sys_vm_controller_create_object(
 		data_len,
 		prot,
 		&out_vmo);
-	vm_controller_unlock_irqrestore(ctrl, flags);
+	vm_controller_unlock_irqrestore(ctrl, irq_flags);
 
 	object_unref(ctrl_obj);
 
 	if (status != KERN_OK) {
-		task_lock_irqsave(self, &flags);
+		task_lock_irqsave(self, &irq_flags);
 		handle_table_free_handle(self->t_handles, out_handle);
-		task_unlock_irqrestore(self, flags);
+		task_unlock_irqrestore(self, irq_flags);
+		put_current_task(self);
 		return status;
 	}
 
 	out_slot->h_object = &out_vmo->vo_base;
+	put_current_task(self);
 
 	*out = out_handle;
 	return KERN_OK;
 }
 
+kern_status_t sys_vm_controller_prepare_attach(
+	kern_handle_t ctrl_handle,
+	uint64_t req_id,
+	kern_handle_t *out_vmo)
+{
+	struct task *self = get_current_task();
+	if (!out_vmo || !validate_access_w(self, out_vmo, sizeof *out_vmo)) {
+		return KERN_MEMORY_FAULT;
+	}
+
+	kern_status_t status = KERN_OK;
+	unsigned long flags;
+	task_lock_irqsave(self, &flags);
+
+	struct object *ctrl_obj = NULL;
+	handle_flags_t ctrl_flags = 0;
+	status = task_resolve_handle(self, ctrl_handle, &ctrl_obj, &ctrl_flags);
+	if (status != KERN_OK) {
+		task_unlock_irqrestore(self, flags);
+		put_current_task(self);
+		return status;
+	}
+
+	struct handle *out_slot = NULL;
+	kern_handle_t out_handle = KERN_HANDLE_INVALID;
+	status = handle_table_alloc_handle(
+		self->t_handles,
+		KERN_HANDLE_INVALID,
+		&out_slot,
+		&out_handle);
+	if (status != KERN_OK) {
+		object_unref(ctrl_obj);
+		task_unlock_irqrestore(self, flags);
+		put_current_task(self);
+		return status;
+	}
+
+	struct vm_controller *ctrl = vm_controller_cast(ctrl_obj);
+	task_unlock_irqrestore(self, flags);
+
+	if (!ctrl) {
+		object_unref(ctrl_obj);
+		return KERN_INVALID_ARGUMENT;
+	}
+
+	vm_controller_lock_irqsave(ctrl, &flags);
+
+	struct vm_object *vmo = NULL;
+	status = vm_controller_prepare_attach(ctrl, req_id, &vmo);
+	vm_controller_unlock_irqrestore(ctrl, flags);
+
+	object_unref(ctrl_obj);
+	if (status != KERN_OK) {
+		task_lock_irqsave(self, &flags);
+		handle_table_free_handle(self->t_handles, out_handle);
+		task_unlock_irqrestore(self, flags);
+
+		return status;
+	}
+
+	out_slot->h_object = &vmo->vo_base;
+	put_current_task(self);
+
+	*out_vmo = out_handle;
+
+	return KERN_OK;
+}
+
+kern_status_t sys_vm_controller_finish_attach(
+	kern_handle_t ctrl_handle,
+	uint64_t req_id,
+	equeue_key_t new_key)
+{
+	struct task *self = get_current_task();
+
+	kern_status_t status = KERN_OK;
+	unsigned long flags;
+	task_lock_irqsave(self, &flags);
+
+	struct object *ctrl_obj = NULL;
+	handle_flags_t ctrl_flags = 0;
+	status = task_resolve_handle(self, ctrl_handle, &ctrl_obj, &ctrl_flags);
+	if (status != KERN_OK) {
+		task_unlock_irqrestore(self, flags);
+		put_current_task(self);
+		return status;
+	}
+
+	struct vm_controller *ctrl = vm_controller_cast(ctrl_obj);
+	task_unlock_irqrestore(self, flags);
+	put_current_task(self);
+
+	if (!ctrl) {
+		object_unref(ctrl_obj);
+		return KERN_INVALID_ARGUMENT;
+	}
+
+	vm_controller_lock_irqsave(ctrl, &flags);
+
+	status = vm_controller_finish_attach(ctrl, req_id, new_key);
+	vm_controller_unlock_irqrestore(ctrl, flags);
+
+	object_unref(ctrl_obj);
+	if (status != KERN_OK) {
+		return status;
+	}
+
+	return KERN_OK;
+}
+
 kern_status_t sys_vm_controller_detach_object(
 	kern_handle_t ctrl_handle,
 	kern_handle_t vmo_handle)
 {
-	struct task *self = current_task();
+	struct task *self = get_current_task();
 
 	kern_status_t status = KERN_OK;
 	unsigned long flags;
@@ -207,6 +334,7 @@ kern_status_t sys_vm_controller_detach_object(
 	status = task_resolve_handle(self, ctrl_handle, &ctrl_obj, &ctrl_flags);
 	if (status != KERN_OK) {
 		task_unlock_irqrestore(self, flags);
+		put_current_task(self);
 		return status;
 	}
 
@@ -214,6 +342,7 @@ kern_status_t sys_vm_controller_detach_object(
 	if (status != KERN_OK) {
 		object_unref(ctrl_obj);
 		task_unlock_irqrestore(self, flags);
+		put_current_task(self);
 		return status;
 	}
 
@@ -224,6 +353,7 @@ kern_status_t sys_vm_controller_detach_object(
 	if (!ctrl || !vmo) {
 		object_unref(ctrl_obj);
 		object_unref(vmo_obj);
+		put_current_task(self);
 		return KERN_INVALID_ARGUMENT;
 	}
 
@@ -235,6 +365,7 @@ kern_status_t sys_vm_controller_detach_object(
 
 	object_unref(ctrl_obj);
 	object_unref(vmo_obj);
+	put_current_task(self);
 
 	return status;
 }
@@ -247,7 +378,7 @@ kern_status_t sys_vm_controller_supply_pages(
 	off_t src_offset,
 	size_t count)
 {
-	struct task *self = current_task();
+	struct task *self = get_current_task();
 
 	kern_status_t status = KERN_OK;
 	unsigned long flags;
@@ -258,6 +389,7 @@ kern_status_t sys_vm_controller_supply_pages(
 	status = task_resolve_handle(self, ctrl_handle, &ctrl_obj, &ctrl_flags);
 	if (status != KERN_OK) {
 		task_unlock_irqrestore(self, flags);
+		put_current_task(self);
 		return status;
 	}
 
@@ -265,6 +397,7 @@ kern_status_t sys_vm_controller_supply_pages(
 	if (status != KERN_OK) {
 		object_unref(ctrl_obj);
 		task_unlock_irqrestore(self, flags);
+		put_current_task(self);
 		return status;
 	}
 
@@ -273,6 +406,7 @@ kern_status_t sys_vm_controller_supply_pages(
 		object_unref(ctrl_obj);
 		object_unref(dst_obj);
 		task_unlock_irqrestore(self, flags);
+		put_current_task(self);
 		return status;
 	}
 
@@ -285,11 +419,14 @@ kern_status_t sys_vm_controller_supply_pages(
 		object_unref(ctrl_obj);
 		object_unref(dst_obj);
 		object_unref(src_obj);
+		put_current_task(self);
 		return KERN_INVALID_ARGUMENT;
 	}
 
 	vm_controller_lock_irqsave(ctrl, &flags);
 	vm_object_lock_pair(src, dst);
+	equeue_key_t requester_key = dst->vo_key;
+
 	status = vm_controller_supply_pages(
 		ctrl,
 		dst,
@@ -298,11 +435,21 @@ kern_status_t sys_vm_controller_supply_pages(
 		src_offset,
 		count);
 	vm_object_unlock_pair(src, dst);
+	vm_controller_fulfill_requests(
+		ctrl,
+		requester_key,
+		dst_offset,
+		count,
+		status);
 	vm_controller_unlock_irqrestore(ctrl, flags);
 
 	object_unref(ctrl_obj);
 	object_unref(dst_obj);
 	object_unref(src_obj);
+	put_current_task(self);
+
+	/* TODO flush individual pages in vm_object_transfer */
+	pmap_flush();
 
 	return status;
 }

syscall/vm-object.c

@@ -11,13 +11,15 @@ kern_status_t sys_vm_object_create(
 	vm_prot_t prot,
 	kern_handle_t *out_handle)
 {
-	struct task *self = current_task();
+	struct task *self = get_current_task();
 
 	if ((name || name_len) && !validate_access_r(self, name, name_len)) {
+		put_current_task(self);
 		return KERN_MEMORY_FAULT;
 	}
 
 	if (!validate_access_w(self, out_handle, sizeof *out_handle)) {
+		put_current_task(self);
 		return KERN_MEMORY_FAULT;
 	}
 
@@ -30,6 +32,7 @@ kern_status_t sys_vm_object_create(
 	kern_status_t status
 		= task_open_handle(self, &obj->vo_base, 0, out_handle);
 	object_unref(&obj->vo_base);
+	put_current_task(self);
 
 	return status;
 }
@@ -41,29 +44,46 @@ kern_status_t sys_vm_object_read(
 	size_t count,
 	size_t *nr_read)
 {
-	struct task *self = current_task();
+	struct task *self = get_current_task();
 
 	if (!validate_access_w(self, dst, count)) {
+		put_current_task(self);
 		return KERN_MEMORY_FAULT;
 	}
 
 	if (nr_read && !validate_access_w(self, nr_read, sizeof *nr_read)) {
+		put_current_task(self);
 		return KERN_MEMORY_FAULT;
 	}
 
 	struct object *obj = NULL;
 	handle_flags_t flags = 0;
+	unsigned long irq_flags = 0;
+	task_lock_irqsave(self, &irq_flags);
 	kern_status_t status = task_resolve_handle(self, object, &obj, &flags);
+	task_unlock_irqrestore(self, irq_flags);
+	put_current_task(self);
+
 	if (status != KERN_OK) {
 		return status;
 	}
 
 	struct vm_object *vmo = vm_object_cast(obj);
 	if (!vmo) {
+		object_unref(obj);
 		return KERN_INVALID_ARGUMENT;
 	}
 
-	return vm_object_read(vmo, dst, offset, count, nr_read);
+	vm_object_lock_irqsave(vmo, &irq_flags);
+	status = vm_object_prefetch(vmo, offset, count, &irq_flags);
+	if (status == KERN_OK) {
+		status = vm_object_read(vmo, dst, offset, count, nr_read);
+	}
+
+	vm_object_unlock_irqrestore(vmo, irq_flags);
+	object_unref(obj);
+
+	return status;
 }
 
 kern_status_t sys_vm_object_write(
@@ -73,20 +93,27 @@ kern_status_t sys_vm_object_write(
 	size_t count,
 	size_t *nr_written)
 {
-	struct task *self = current_task();
+	struct task *self = get_current_task();
 
 	if (!validate_access_r(self, src, count)) {
+		put_current_task(self);
 		return KERN_MEMORY_FAULT;
 	}
 
 	if (nr_written
 	    && !validate_access_w(self, nr_written, sizeof *nr_written)) {
+		put_current_task(self);
 		return KERN_MEMORY_FAULT;
 	}
 
 	struct object *obj = NULL;
 	handle_flags_t flags = 0;
+	unsigned long irq_flags = 0;
+	task_lock_irqsave(self, &irq_flags);
 	kern_status_t status = task_resolve_handle(self, object, &obj, &flags);
+	task_unlock_irqrestore(self, irq_flags);
+	put_current_task(self);
+
 	if (status != KERN_OK) {
 		return status;
 	}
@@ -96,7 +123,11 @@ kern_status_t sys_vm_object_write(
 		return KERN_INVALID_ARGUMENT;
 	}
 
-	return vm_object_write(vmo, src, offset, count, nr_written);
+	vm_object_lock_irqsave(vmo, &irq_flags);
+	status = vm_object_write(vmo, src, offset, count, nr_written);
+	vm_object_unlock_irqrestore(vmo, irq_flags);
+
+	return status;
 }
 
 kern_status_t sys_vm_object_copy(
@@ -114,10 +145,11 @@ kern_status_t sys_vm_object_copy(
 	       src_offset,
 	       count,
 	       nr_copied);
-	struct task *self = current_task();
+	struct task *self = get_current_task();
 
 	if (nr_copied
 	    && !validate_access_w(self, nr_copied, sizeof *nr_copied)) {
+		put_current_task(self);
 		return KERN_MEMORY_FAULT;
 	}
 
@@ -131,25 +163,43 @@ kern_status_t sys_vm_object_copy(
 	status = task_resolve_handle(self, dst, &dst_obj, &dst_flags);
 	if (status != KERN_OK) {
 		task_unlock_irqrestore(self, flags);
+		put_current_task(self);
 		return status;
 	}
 
 	status = task_resolve_handle(self, src, &src_obj, &src_flags);
 	if (status != KERN_OK) {
 		task_unlock_irqrestore(self, flags);
+		put_current_task(self);
 		return status;
 	}
 
 	task_unlock_irqrestore(self, flags);
+	put_current_task(self);
 
 	struct vm_object *dst_vmo = vm_object_cast(dst_obj);
+	if (!dst_vmo) {
+		return KERN_INVALID_ARGUMENT;
+	}
+
 	struct vm_object *src_vmo = vm_object_cast(src_obj);
-	if (!dst_vmo || !src_vmo) {
+	if (!src_vmo) {
-		object_unref(src_obj);
 		object_unref(dst_obj);
 		return KERN_INVALID_ARGUMENT;
 	}
 
+	unsigned long irq_flags = 0;
+	vm_object_lock_irqsave(src_vmo, &irq_flags);
+	status = vm_object_prefetch(src_vmo, src_offset, count, &irq_flags);
+	vm_object_unlock_irqrestore(src_vmo, irq_flags);
+
+	if (status != KERN_OK) {
+		object_unref(src_obj);
+		object_unref(dst_obj);
+		return status;
+	}
+
+	vm_object_lock_pair_irqsave(src_vmo, dst_vmo, &irq_flags);
 	status = vm_object_copy(
 		dst_vmo,
 		dst_offset,
@@ -157,6 +207,7 @@ kern_status_t sys_vm_object_copy(
 		src_offset,
 		count,
 		nr_copied);
+	vm_object_unlock_pair_irqrestore(src_vmo, dst_vmo, irq_flags);
 
 	object_unref(src_obj);
 	object_unref(dst_obj);

tools/CMakeLists.txt

@@ -1,5 +1,5 @@
 cmake_minimum_required(VERSION 3.13)
-project(mango-tools C)
+project(magenta-tools C)
 
 set(tool_dirs e64patch)
 

tools/bochsrc.bxrc

@@ -10,7 +10,7 @@ floppy_bootsig_check: disabled=0
 floppya: type=1_44
 # no floppyb
 ata0: enabled=true, ioaddr1=0x1f0, ioaddr2=0x3f0, irq=14
-ata0-master: type=cdrom, path="build/mango-kernel.iso", status=inserted, model="Generic 1234", biosdetect=auto
+ata0-master: type=cdrom, path="build/magenta-kernel.iso", status=inserted, model="Generic 1234", biosdetect=auto
 ata0-slave: type=none
 ata1: enabled=true, ioaddr1=0x170, ioaddr2=0x370, irq=15
 ata1-master: type=none

tools/boot-image/grub.cfg

@@ -1,4 +1,4 @@
-menuentry "Mango Kernel" {
+menuentry "Magenta Kernel" {
-	multiboot /boot/mango_kernel
+	multiboot /boot/magenta_kernel
 	boot
 }

tools/kernel-debug/debug_session.sh

@@ -5,9 +5,9 @@ lldb_cfg=$2
 shift 2
 
 if command -v gdb &> /dev/null; then
-	printf "    \033[93;1mGDB\033[0m    boot/mango_kernel\n"
+	printf "    \033[93;1mGDB\033[0m    boot/magenta_kernel\n"
 	tmux \
-		new-session -d -s mango-debug "sleep 0.3; gdb -tui -x $gdb_cfg" \; \
+		new-session -d -s magenta-debug "sleep 0.3; gdb -tui -x $gdb_cfg" \; \
 		split-window -h -l 80 \; \
 		split-window -v -l 25 "$@"\; \
 		select-pane -t 1 \; \
@@ -15,9 +15,9 @@ if command -v gdb &> /dev/null; then
 		select-pane -t 0
 
 elif command -v lldb &> /dev/null; then
-	printf "    \033[93;1mLLDB\033[0m   boot/mango_kernel\n"
+	printf "    \033[93;1mLLDB\033[0m   boot/magenta_kernel\n"
 	tmux \
-		new-session -d -s mango-debug "sleep 0.1; lldb --source $lldb_cfg" \; \
+		new-session -d -s magenta-debug "sleep 0.1; lldb --source $lldb_cfg" \; \
 		split-window -h -l 160 \; \
 		split-window -v -l 25 "$@"\; \
 		select-pane -t 1 \; \
@@ -28,6 +28,6 @@ else
 	exit -1
 fi
 
-tmux a -t mango-debug
+tmux a -t magenta-debug
-tmux kill-session -t mango-debug
+tmux kill-session -t magenta-debug
 

tools/kernel-debug/gdb_session_init

@@ -1,4 +1,4 @@
 set confirm off
-symbol-file mango_kernel.debug
+symbol-file magenta_kernel.debug
 target remote localhost:1234
 set confirm on

tools/kernel-debug/lldb_session_init

@@ -1,2 +1,2 @@
-file mango_kernel.debug
+file magenta_kernel.debug
 gdb-remote localhost:1234

tools/magenta.install-cd

@@ -116,7 +116,7 @@ def choice_options(num_devices):
     return '1-{}'.format(num_devices)
 
 
-if not os.path.isfile('build/mango-kernel.iso'):
+if not os.path.isfile('build/magenta-kernel.iso'):
     print('No system ISO image found.')
     print('Please run \'make cd\' to generate an ISO image')
     exit(-1)
@@ -157,7 +157,7 @@ disk_prewrite(devices[choice][0])
 dd_args = [
     'sudo',
     'dd',
-    'if=build/mango-kernel.iso',
+    'if=build/magenta-kernel.iso',
     'of={}'.format(devices[choice][0]),
     'bs=1{}'.format('m' if sys.platform == 'darwin' else 'M')
 ]

tools/magenta.mkrescue

@@ -7,22 +7,22 @@ import sys
 import os
 import shutil
 
-kernel_src_path = os.path.join('build', 'mango_kernel')
+kernel_src_path = os.path.join('build', 'magenta_kernel')
 grub_cfg_src_path = os.path.join('tools', 'boot-image', 'grub.cfg')
-iso_build_dir = os.path.join('build', 'mango-kernel.iso-build')
+iso_build_dir = os.path.join('build', 'magenta-kernel.iso-build')
-iso_path = os.path.join('build', 'mango-kernel.iso')
+iso_path = os.path.join('build', 'magenta-kernel.iso')
 
 
 def in_source_tree():
-    return os.path.isfile('tools/mango.sync')
+    return os.path.isfile('tools/magenta.sync')
 
 
 if not in_source_tree():
-    print('This script must be executed from the root of the Mango source tree')
+    print('This script must be executed from the root of the Magenta source tree')
     exit(-1)
 
 if not os.path.isdir('build'):
-    print('Please build the Mango kernel before using this tool')
+    print('Please build the Magenta kernel before using this tool')
     exit(-1)
 
 if os.path.isdir(iso_build_dir):
@@ -33,7 +33,7 @@ os.mkdir(iso_build_dir)
 os.mkdir(os.path.join(iso_build_dir, 'boot'))
 os.mkdir(os.path.join(iso_build_dir, 'boot', 'grub'))
 
-shutil.copyfile(kernel_src_path, os.path.join(iso_build_dir, 'boot', 'mango_kernel'))
+shutil.copyfile(kernel_src_path, os.path.join(iso_build_dir, 'boot', 'magenta_kernel'))
 shutil.copyfile(grub_cfg_src_path, os.path.join(iso_build_dir, 'boot', 'grub', 'grub.cfg'))
 
 subprocess.run(['grub-mkrescue', '-o', iso_path, iso_build_dir])

tools/magenta.sync

@@ -16,7 +16,7 @@ rsync_path = os.getenv('RSYNC_PATH', default = 'rsync')
 
 
 def in_source_tree():
-    return os.path.isfile('tools/mango.sync')
+    return os.path.isfile('tools/magenta.sync')
 
 
 def create_sample_config():
@@ -78,7 +78,7 @@ def print_available_targets(targets):
 
 
 if not in_source_tree():
-    print('This script must be executed from the root of the Mango source tree')
+    print('This script must be executed from the root of the Magenta source tree')
     exit(-1)
 
 config = read_config()

vm/address-space.c

@@ -6,10 +6,11 @@
 #include <kernel/printk.h>
 #include <kernel/sched.h>
 #include <kernel/task.h>
+#include <kernel/thread.h>
 #include <kernel/util.h>
 #include <kernel/vm-controller.h>
 #include <kernel/vm-object.h>
-#include <mango/status.h>
+#include <magenta/status.h>
 
 /*** STATIC DATA + MACROS *****************************************************/
 
@@ -660,9 +661,8 @@ static void area_unmap(struct vm_area *area)
 	pmap_t pmap = area->vma_space->s_pmap;
 	virt_addr_t base = area->vma_base;
 	virt_addr_t limit = area->vma_limit;
-	for (virt_addr_t i = base; i < limit; i += VM_PAGE_SIZE) {
+	pmap_remove_range(pmap, base, limit - base);
-		pmap_remove(pmap, i);
+	pmap_flush();
-	}
 }
 
 static kern_status_t address_space_cleanup(struct object *obj)
@@ -705,6 +705,7 @@ kern_status_t address_space_map(
 	struct vm_object *object,
 	off_t object_offset,
 	size_t length,
+	vm_flags_t flags,
 	vm_prot_t prot,
 	virt_addr_t *out)
 {
@@ -762,6 +763,7 @@ kern_status_t address_space_map(
 	area->vma_space = root;
 	area->vma_object = object;
 	area->vma_prot = prot;
+	area->vma_flags = flags;
 	area->vma_object_offset = object_offset;
 	area->vma_base = map_address;
 	area->vma_limit = map_address + length - 1;
@@ -841,11 +843,9 @@ static kern_status_t split_area(
 	}
 
 	put_entry(&root->s_mappings, right);
-	for (size_t i = unmap_base; i < unmap_limit; i += VM_PAGE_SIZE) {
-		tracek("pmap_remove %zx", i);
-		pmap_remove(root->s_pmap, i);
-	}
 
+	pmap_remove_range(root->s_pmap, unmap_base, unmap_limit - unmap_base);
+	pmap_flush();
 	return KERN_OK;
 }
 
@@ -880,10 +880,8 @@ static kern_status_t left_reduce_area(
 	}
 
 	tracek(" pmap_remove %zx-%zx (%zx bytes)", base, base + length, length);
-	for (size_t i = base; i < limit; i += VM_PAGE_SIZE) {
+	pmap_remove_range(root->s_pmap, base, length);
-		pmap_remove(root->s_pmap, i);
+	pmap_flush();
-	}
-
 	return KERN_OK;
 }
 
@@ -917,10 +915,8 @@ static kern_status_t right_reduce_area(
 	}
 
 	tracek(" pmap_remove %zx-%zx (%zx bytes)", base, base + length, length);
-	for (size_t i = base; i < limit; i += VM_PAGE_SIZE) {
+	pmap_remove_range(root->s_pmap, base, limit - base);
-		pmap_remove(root->s_pmap, i);
+	pmap_flush();
-	}
-
 	return KERN_OK;
 }
 
@@ -939,10 +935,11 @@ static kern_status_t delete_area(
 	       mapping->vma_limit,
 	       mapping->vma_limit - mapping->vma_base);
 
-	for (size_t i = mapping->vma_base; i < mapping->vma_limit;
+	pmap_remove_range(
-	     i += VM_PAGE_SIZE) {
+		root->s_pmap,
-		pmap_remove(root->s_pmap, i);
+		mapping->vma_base,
-	}
+		mapping->vma_limit - mapping->vma_base);
+	pmap_flush();
 
 	struct vm_object *object = mapping->vma_object;
 	unsigned long flags;
@@ -1206,6 +1203,267 @@ kern_status_t address_space_release(
 	return status;
 }
 
+static struct vm_area *area_duplicate(struct vm_area *area)
+{
+	struct vm_area *out = vm_cache_alloc(&vm_area_cache, VM_NORMAL);
+	if (!out) {
+		return NULL;
+	}
+
+	out->vma_prot = area->vma_prot;
+	out->vma_flags = area->vma_flags;
+	out->vma_object_offset = area->vma_object_offset;
+	out->vma_base = area->vma_base;
+	out->vma_limit = area->vma_limit;
+
+	return out;
+}
+
+static kern_status_t update_area_pte_cow(
+	struct address_space *src,
+	struct address_space *dest,
+	struct vm_area *area)
+{
+	if (!area->vma_object) {
+		return KERN_OK;
+	}
+
+	for (virt_addr_t i = area->vma_base; i < area->vma_limit;
+	     i += VM_PAGE_SIZE) {
+		off_t pg_offset = i - area->vma_base + area->vma_object_offset;
+		struct vm_page *pg = vm_object_get_page(
+			area->vma_object,
+			pg_offset,
+			0,
+			NULL);
+		vm_prot_t temp_prot = area->vma_prot;
+		temp_prot &= ~VM_PROT_WRITE;
+
+		if (pg) {
+			pmap_add(
+				src->s_pmap,
+				i,
+				vm_page_get_pfn(pg),
+				temp_prot,
+				PMAP_NORMAL);
+			pmap_add(
+				dest->s_pmap,
+				i,
+				vm_page_get_pfn(pg),
+				temp_prot,
+				PMAP_NORMAL);
+			tracek("PTE %zx -> %zx [%x]",
+			       i,
+			       vm_page_get_paddr(pg),
+			       temp_prot);
+		}
+	}
+
+	return KERN_OK;
+}
+
+static kern_status_t prepare_duplicate_areas(
+	struct address_space *src,
+	struct address_space *dest)
+{
+	struct btree_node *cur_node = btree_first(&src->s_mappings);
+	while (cur_node) {
+		struct vm_area *tmp_area
+			= BTREE_CONTAINER(struct vm_area, vma_node, cur_node);
+		if (!tmp_area->vma_object) {
+			cur_node = btree_next(cur_node);
+			continue;
+		}
+
+		struct vm_object *src_vmo = tmp_area->vma_object;
+		vm_object_lock(src_vmo);
+
+		struct vm_object *dest_vmo_link = NULL;
+		struct vm_object *dest_vmo_cow = NULL;
+		struct queue_entry *cur_entry
+			= queue_first(&src_vmo->vo_mappings);
+
+		while (cur_entry) {
+			struct vm_area *src_area = QUEUE_CONTAINER(
+				struct vm_area,
+				vma_object_entry,
+				cur_entry);
+			if (src_area->vma_space != src) {
+				cur_entry = queue_next(cur_entry);
+				continue;
+			}
+
+			struct vm_area *dest_area = get_entry(
+				&dest->s_mappings,
+				src_area->vma_base,
+				GET_ENTRY_EXACT);
+			if (!dest_area) {
+				/* this shouldn't happen. the duplicate vm_areas
+				 * were already created by
+				 * address_space_duplicate */
+				panic("create_duplicate_vmo: corresponding "
+				      "vm_area is missing");
+			}
+
+			if (dest_area->vma_object) {
+				cur_entry = queue_next(cur_entry);
+				continue;
+			}
+
+			struct vm_object *dest_vmo = NULL;
+			if (src_area->vma_flags & VM_SHARED) {
+				dest_vmo = dest_vmo_link;
+			} else {
+				dest_vmo = dest_vmo_cow;
+			}
+
+			if (!dest_vmo) {
+				tracek("[%zx-%zx %x] creating COW duplicate of "
+				       "vmo %p",
+				       src_area->vma_base,
+				       src_area->vma_limit,
+				       src_area->vma_prot,
+				       src_vmo);
+
+				if (src_area->vma_flags & VM_SHARED) {
+					dest_vmo_link = src_vmo;
+					object_ref(&dest_vmo_link->vo_base);
+
+					dest_vmo = dest_vmo_link;
+				} else {
+					dest_vmo_cow = vm_object_duplicate_cow(
+						src_vmo);
+					dest_vmo = dest_vmo_cow;
+				}
+
+				tracek("[%zx-%zx %x] created COW duplicate of "
+				       "vmo %p -> %p",
+				       src_area->vma_base,
+				       src_area->vma_limit,
+				       src_area->vma_prot,
+				       src_vmo,
+				       dest_vmo);
+			}
+
+			object_ref(&dest_vmo->vo_base);
+			dest_area->vma_object = dest_vmo;
+			update_area_pte_cow(src, dest, src_area);
+			cur_entry = queue_next(cur_entry);
+		}
+
+		if (dest_vmo_link) {
+			object_unref(&dest_vmo_link->vo_base);
+		}
+
+		if (dest_vmo_cow) {
+			object_unref(&dest_vmo_cow->vo_base);
+		}
+
+		vm_object_unlock(src_vmo);
+
+		cur_node = btree_next(cur_node);
+	}
+
+	return KERN_OK;
+}
+
+kern_status_t address_space_duplicate(
+	struct address_space *dest,
+	struct address_space *src)
+{
+	// address_space_dump(src);
+	/* clang-format off
+	 * strategy for COW address space duplication:
+	 * 1. duplicate each vm_area in the address space
+	 *    a. all details except for the vm_object pointer are copied.
+	 *    b. create a duplicate vm_object, where all the details are the
+	 *       same, but don't copy the pages or vm_page pointers.
+	 *    c. if the vm_object is attached to a vm_controller, don't inform
+	 *       the controller yet.
+	 *    d. for both the original and duplicate vm_area, duplicate the PTE
+	 *       entries, changing all of them to read-only. increment the
+	 *       p_cow_ref counters for all committed vm_pages.
+	 *    e. use the vm_object's vm_area list, and the vm_area's vma_space
+	 *       pointer, to ensure that only one duplicate is created for each
+	 *       unique vm-object referenced by an address-space.
+	 * 2. when a page fault occurs:
+	 *    a. find the relevant vm_area as normal.
+	 *    b. if the faulted page is present and the vm_area's prot flags
+	 *       should allow the access, a COW is required.
+	 *    c. if the relevant page is already present in the vm_area's
+	 *       vm_object, this is the original vm_area. otherwise, this is the
+	 *       clone vm_area.
+	 *    d. if this is the source vm_area:
+	 *       i. decrement p_cow_ref in the page. if it is 0, skip to step v.
+	 *       ii. remove the relevant page from the vm_area
+	 *       iii. allocate a new page and copy the data.
+	 *       iv. add the new page to the vm_object at the same offset.
+	 *       v. change the PTE entry to the proper protection flags.
+	 *       vi. resume the faulting task.
+	 *    e. otherwise, if this is the clone vm_area:
+	 *       i. if the vm-object has a controller, send
+	 *          PAGE_REQUEST_DUPLICATE to it. the controller needs to
+	 *          prepare itself to receive page requests from this vm-object,
+	 *          which includes priving it an equeue_key_t.
+	 *       i. use the physical address stored in the PTE to find the
+	 *          relevant vm_page.
+	 *       ii. decrement p_cow_ref in the page.
+	 *       iii. if p_cow_ref is > 0, allocate a new page and copy the data.
+	 *            otherwise, use the existing page as-is.
+	 *       iv. add the page from step iii to the vm_object at the correct
+	 *           offset.
+	 *       v. change the PTE entry to the proper protection flags.
+	 *       vi. resume the faulting task.
+	 * 3. when destroying a vm_area:
+	 *    a. for pages already present in a vm-object, handle as normal.
+	 *    b. for pages not present in a vm-object, but for which a valid PTE
+	 *       exists, use the PTE physical address to find the vm_page.
+	 *    c. decrement p_cow_ref in this page.
+	 *    d. if p_cow_ref == 0, de-allocate the page.
+	 * clang-format on
+	 */
+	struct btree_node *cur = btree_first(&src->s_mappings);
+	while (cur) {
+		struct vm_area *src_area
+			= BTREE_CONTAINER(struct vm_area, vma_node, cur);
+		struct vm_area *dest_area = area_duplicate(src_area);
+		tracek("duplicated vm_area [%zx-%zx] %p -> %p",
+		       src_area->vma_base,
+		       src_area->vma_limit,
+		       src_area,
+		       dest_area);
+		/* TODO handle OOM */
+		put_entry(&dest->s_mappings, dest_area);
+
+		cur = btree_next(cur);
+	}
+
+	cur = btree_first(&src->s_reserved);
+	while (cur) {
+		struct vm_area *src_area
+			= BTREE_CONTAINER(struct vm_area, vma_node, cur);
+		struct vm_area *dest_area = area_duplicate(src_area);
+		tracek("duplicated vm_area [r] [%zx-%zx] %p -> %p",
+		       src_area->vma_base,
+		       src_area->vma_limit,
+		       src_area,
+		       dest_area);
+		/* TODO handle OOM */
+		put_entry(&dest->s_reserved, dest_area);
+
+		cur = btree_next(cur);
+	}
+
+	tracek("preparing duplicate areas");
+	kern_status_t status = prepare_duplicate_areas(src, dest);
+	tracek("prepared duplicate areas");
+	if (status != KERN_OK) {
+		return status;
+	}
+
+	return KERN_OK;
+}
+
 bool address_space_validate_access(
 	struct address_space *region,
 	virt_addr_t ptr,
@@ -1261,13 +1519,26 @@ static kern_status_t request_missing_page(
 	/* here:
 	 * `region` is locked.
 	 * `object` is unlocked.
-	 * `irq_flags` must be restored when `region` is unlocked.
+	 * `irq_flags` must be restored when `region` is
-	 * the relevant page in `object` may or may not be committed.
+	 * unlocked.
-	 * if it isn't, it needs to be requested.
+	 * the relevant page in `object` may or may
+	 * not be committed. if it isn't, it needs to be
+	 * requested.
 	 */
 	vm_object_lock(object);
 	address_space_unlock(region);
 
+	kern_status_t status = KERN_OK;
+
+	if (object->vo_flags & VMO_LAZY_ATTACH) {
+		status = vm_object_attach_cow(object, irq_flags);
+	}
+
+	if (status != KERN_OK) {
+		vm_object_unlock_irqrestore(object, *irq_flags);
+		return status;
+	}
+
 	struct vm_page *pg = vm_object_get_page(
 		object,
 		object_offset,
@@ -1281,7 +1552,7 @@ static kern_status_t request_missing_page(
 
 	/* now: `region` is unlocked, and `object` is locked */
 
-	kern_status_t status = pmap_add(
+	status = pmap_add(
 		region->s_pmap,
 		addr,
 		vm_page_get_pfn(pg),
@@ -1292,7 +1563,126 @@ static kern_status_t request_missing_page(
 	return status;
 }
 
-/* this function must be called with `region` locked */
+/* handle a write to a page that is present and /should be/ writeable, but is
+ * currently read-only due to COW. */
+static kern_status_t handle_cow_access(
+	struct address_space *region,
+	virt_addr_t addr,
+	enum pmap_fault_flags flags,
+	unsigned long *irq_flags)
+{
+	struct vm_area *area
+		= get_entry(&region->s_mappings, addr, GET_ENTRY_EXACT);
+	if (!area || !area->vma_object) {
+		/* no mapping exists (this shouldn't happen) */
+		address_space_unlock_irqrestore(region, *irq_flags);
+		return KERN_NO_ENTRY;
+	}
+
+	if ((area->vma_prot & (VM_PROT_WRITE | VM_PROT_USER))
+	    != (VM_PROT_WRITE | VM_PROT_USER)) {
+		/* access denied */
+		address_space_unlock_irqrestore(region, *irq_flags);
+		return KERN_ACCESS_DENIED;
+	}
+
+	tracek("cow access %zx", addr);
+
+	off_t object_offset = addr - area->vma_base + area->vma_object_offset;
+	vm_object_lock(area->vma_object);
+	struct vm_page *pg
+		= vm_object_get_page(area->vma_object, object_offset, 0, NULL);
+
+	bool add_page = false;
+	if (!pg) {
+		/* the page hasn't been added to the vmo yet. */
+		pfn_t pfn;
+		vm_prot_t prot;
+		kern_status_t status
+			= pmap_get(region->s_pmap, addr, &pfn, &prot);
+		if (status != KERN_OK) {
+			vm_object_unlock(area->vma_object);
+			address_space_unlock_irqrestore(region, *irq_flags);
+			return status;
+		}
+
+		add_page = true;
+		pg = vm_page_get(pfn * VM_PAGE_SIZE);
+		tracek("recovered page %zx (%p, %u) from the aether",
+		       vm_page_get_paddr(pg),
+		       pg,
+		       pg->p_cow_ref);
+
+		if (!pg) {
+			/* still can't find the page */
+			panic("cannot resolve cow-reference to page %zx",
+			      pfn * VM_PAGE_SIZE);
+		}
+
+		if (!pg->p_cow_ref) {
+			/* still can't find the page */
+			panic("cow-reference page %zx cow-ref=0",
+			      pfn * VM_PAGE_SIZE);
+		}
+	}
+
+	atomic_t cow_ref = atomic_sub_fetch(&pg->p_cow_ref, 1);
+	tracek("decrement cow-ref page %zx -> now %u",
+	       vm_page_get_paddr(pg),
+	       pg->p_cow_ref);
+	if (cow_ref > 0) {
+		/* another task is (or will be) referencing this page. create
+		 * a copy */
+		struct vm_page *dup = vm_page_alloc(VM_PAGE_4K, VM_NORMAL);
+		if (!dup) {
+			vm_object_unlock(area->vma_object);
+			address_space_unlock_irqrestore(region, *irq_flags);
+			return KERN_NO_MEMORY;
+		}
+
+		void *src = vm_page_get_vaddr(pg);
+		void *dest = vm_page_get_vaddr(dup);
+		memcpy(dest, src, vm_page_get_size_bytes(dup));
+
+		if (!add_page) {
+			btree_delete(&area->vma_object->vo_pages, &pg->p_bnode);
+		}
+
+		vm_object_put_page(area->vma_object, object_offset, dup);
+		tracek("splitting cow page %zx -> %zx",
+		       vm_page_get_paddr(pg),
+		       vm_page_get_paddr(dup));
+		tracek("put page %zx into area [%zx-%zx] at %zx",
+		       vm_page_get_paddr(dup),
+		       area->vma_base,
+		       area->vma_limit,
+		       object_offset);
+		pg = dup;
+	} else if (add_page) {
+		/* we are the last task referencing this page. add it to our
+		 * vmo and continue */
+		tracek("claimed cow page %zx", vm_page_get_paddr(pg));
+		vm_object_put_page(area->vma_object, object_offset, pg);
+	} else {
+		/* we are the last task referencing this page, and it is already
+		 * in our vmo. */
+		tracek("reclaimed cow page %zx", vm_page_get_paddr(pg));
+	}
+
+	kern_status_t status = pmap_add(
+		region->s_pmap,
+		addr,
+		vm_page_get_pfn(pg),
+		area->vma_prot,
+		PMAP_NORMAL);
+
+	vm_object_unlock(area->vma_object);
+	address_space_unlock_irqrestore(region, *irq_flags);
+
+	return status;
+}
+
+/* this function must be called with `region` unlocked */
 kern_status_t address_space_demand_map(
 	struct address_space *region,
 	virt_addr_t addr,
@@ -1306,6 +1696,13 @@ kern_status_t address_space_demand_map(
 	unsigned long irq_flags;
 	address_space_lock_irqsave(region, &irq_flags);
 
+	const enum pmap_fault_flags cow_flags
+		= PMAP_FAULT_WRITE | PMAP_FAULT_PRESENT;
+
+	if ((flags & cow_flags) == cow_flags) {
+		return handle_cow_access(region, addr, flags, &irq_flags);
+	}
+
 	struct vm_area *area
 		= get_entry(&region->s_mappings, addr, GET_ENTRY_EXACT);
 	if (!area || !area->vma_object) {
@@ -1348,6 +1745,8 @@ kern_status_t address_space_demand_map(
 #endif
 
 	if (!pg) {
+		vm_object_unlock(area->vma_object);
+		address_space_unlock_irqrestore(region, irq_flags);
 		return KERN_FATAL_ERROR;
 	}
 

vm/bootstrap.c

@@ -6,7 +6,7 @@
 #include <kernel/vm-object.h>
 #include <kernel/vm.h>
 #include <limits.h>
-#include <mango/status.h>
+#include <magenta/status.h>
 #include <stddef.h>
 #include <stdint.h>
 

vm/vm-controller.c

@@ -4,18 +4,17 @@
 #include <kernel/util.h>
 #include <kernel/vm-controller.h>
 #include <kernel/vm-object.h>
-#include <mango/signal.h>
+#include <magenta/signal.h>
 
 #define VM_CONTROLLER_CAST(p)                                                  \
 	OBJECT_C_CAST(struct vm_controller, vc_base, &vm_controller_type, p)
 
-BTREE_DEFINE_SIMPLE_INSERT(struct vm_object, vo_ctrl_node, vo_key, put_object)
 BTREE_DEFINE_SIMPLE_GET(
-	struct vm_object,
+	struct vm_request,
-	equeue_key_t,
+	uint64_t,
-	vo_ctrl_node,
+	req_node,
-	vo_key,
+	req_id,
-	get_object)
+	get_request)
 
 static struct object_type vm_controller_type = {
 	.ob_name = "vm-controller",
@@ -23,14 +22,14 @@ static struct object_type vm_controller_type = {
 	.ob_header_offset = offsetof(struct vm_controller, vc_base),
 };
 
-static struct vm_cache page_request_cache = {
+static struct vm_cache vm_request_cache = {
-	.c_name = "page-request",
+	.c_name = "vm-request",
-	.c_obj_size = sizeof(struct page_request),
+	.c_obj_size = sizeof(struct vm_request),
 };
 
 kern_status_t vm_controller_type_init(void)
 {
-	vm_cache_init(&page_request_cache);
+	vm_cache_init(&vm_request_cache);
 	return object_type_register(&vm_controller_type);
 }
 
@@ -51,19 +50,19 @@ struct vm_controller *vm_controller_create(void)
 	return ctrl;
 }
 
-static struct page_request *get_next_request(struct vm_controller *ctrl)
+static struct vm_request *get_next_request(struct vm_controller *ctrl)
 {
 	struct btree_node *cur = btree_first(&ctrl->vc_requests);
 	while (cur) {
-		struct page_request *req
+		struct vm_request *req
-			= BTREE_CONTAINER(struct page_request, req_node, cur);
+			= BTREE_CONTAINER(struct vm_request, req_node, cur);
 		spin_lock(&req->req_lock);
 		switch (req->req_status) {
-		case PAGE_REQUEST_PENDING:
+		case VM_REQUEST_PENDING:
-			req->req_status = PAGE_REQUEST_IN_PROGRESS;
+			req->req_status = VM_REQUEST_IN_PROGRESS;
 			ctrl->vc_requests_waiting--;
 			return req;
-		case PAGE_REQUEST_ASYNC:
+		case VM_REQUEST_ASYNC:
 			btree_delete(&ctrl->vc_requests, &req->req_node);
 			ctrl->vc_requests_waiting--;
 			return req;
@@ -78,7 +77,7 @@ static struct page_request *get_next_request(struct vm_controller *ctrl)
 	return NULL;
 }
 
-static kern_status_t try_enqueue(struct btree *tree, struct page_request *req)
+static kern_status_t try_enqueue(struct btree *tree, struct vm_request *req)
 {
 	if (!tree->b_root) {
 		tree->b_root = &req->req_node;
@@ -88,8 +87,8 @@ static kern_status_t try_enqueue(struct btree *tree, struct page_request *req)
 
 	struct btree_node *cur = tree->b_root;
 	while (1) {
-		struct page_request *cur_node
+		struct vm_request *cur_node
-			= BTREE_CONTAINER(struct page_request, req_node, cur);
+			= BTREE_CONTAINER(struct vm_request, req_node, cur);
 		struct btree_node *next = NULL;
 
 		if (req->req_id > cur_node->req_id) {
@@ -119,7 +118,7 @@ static kern_status_t try_enqueue(struct btree *tree, struct page_request *req)
 
 static kern_status_t send_request_async(
 	struct vm_controller *ctrl,
-	struct page_request *req)
+	struct vm_request *req)
 {
 	fill_random(&req->req_id, sizeof req->req_id);
 	while (!try_enqueue(&ctrl->vc_requests, req)) {
@@ -136,9 +135,9 @@ static kern_status_t send_request_async(
 
 kern_status_t vm_controller_recv(
 	struct vm_controller *ctrl,
-	equeue_packet_page_request_t *out)
+	equeue_packet_vm_request_t *out)
 {
-	struct page_request *req = NULL;
+	struct vm_request *req = NULL;
 
 	req = get_next_request(ctrl);
 	if (!req) {
@@ -151,15 +150,30 @@ kern_status_t vm_controller_recv(
 			VM_CONTROLLER_SIGNAL_REQUEST_RECEIVED);
 	}
 
-	out->req_vmo = req->req_object;
+	vm_object_lock(req->req_object);
+	out->req_id = req->req_id;
+	out->req_vmo = req->req_object->vo_key;
 	out->req_type = req->req_type;
-	out->req_offset = req->req_offset;
-	out->req_length = req->req_length;
 
+	switch (req->req_type) {
+	case VM_REQUEST_READ:
+	case VM_REQUEST_DIRTY:
+		out->req_offset = req->req_offset;
+		out->req_length = req->req_length;
+		break;
+	case VM_REQUEST_ATTACH:
+		out->req_src_vmo = req->req_object->vo_key;
+		break;
+	default:
+		break;
+	}
+
+	vm_object_unlock(req->req_object);
 	spin_unlock(&req->req_lock);
 
-	if (req->req_status == PAGE_REQUEST_ASYNC) {
+	if (req->req_status == VM_REQUEST_ASYNC) {
-		vm_cache_free(&page_request_cache, req);
+		put_current_thread(req->req_sender);
+		vm_cache_free(&vm_request_cache, req);
 	}
 
 	return KERN_OK;
@@ -207,6 +221,52 @@ kern_status_t vm_controller_create_object(
 	return KERN_OK;
 }
 
+kern_status_t vm_controller_prepare_attach(
+	struct vm_controller *ctrl,
+	uint64_t req_id,
+	struct vm_object **out_vmo)
+{
+	struct vm_request *req = get_request(&ctrl->vc_requests, req_id);
+	if (!req) {
+		return KERN_INVALID_ARGUMENT;
+	}
+
+	spin_lock(&req->req_lock);
+	req->req_status = VM_REQUEST_IN_PROGRESS;
+	*out_vmo = req->req_object;
+	spin_unlock(&req->req_lock);
+
+	return KERN_OK;
+}
+
+kern_status_t vm_controller_finish_attach(
+	struct vm_controller *ctrl,
+	uint64_t req_id,
+	equeue_key_t new_key)
+{
+	struct vm_request *req = get_request(&ctrl->vc_requests, req_id);
+	if (!req) {
+		return KERN_INVALID_ARGUMENT;
+	}
+
+	spin_lock(&req->req_lock);
+	struct vm_object *vmo = req->req_object;
+	spin_unlock(&req->req_lock);
+
+	vm_object_lock(vmo);
+	vmo->vo_key = new_key;
+	vmo->vo_flags &= ~VMO_LAZY_ATTACH;
+	vm_object_unlock(vmo);
+
+	spin_lock(&req->req_lock);
+	req->req_status = VM_REQUEST_COMPLETE;
+	req->req_result = KERN_OK;
+	thread_awaken(req->req_sender);
+	spin_unlock(&req->req_lock);
+
+	return KERN_OK;
+}
+
 kern_status_t vm_controller_detach_object(
 	struct vm_controller *ctrl,
 	struct vm_object *vmo)
@@ -215,16 +275,23 @@ kern_status_t vm_controller_detach_object(
 		return KERN_INVALID_ARGUMENT;
 	}
 
-	struct page_request *req
+	if (vmo->vo_flags & VMO_LAZY_ATTACH) {
-		= vm_cache_alloc(&page_request_cache, VM_NORMAL);
+		/* this vmo isn't actually attached to this controller yet.
-	req->req_type = PAGE_REQUEST_DETACH;
+		 * this can happen if a controller-attached vmo was duplicated
-	req->req_status = PAGE_REQUEST_ASYNC;
+		 * via copy-on-write, and the duplicate vmo has not yet been
-	req->req_object = vmo->vo_key;
+		 * accessed. */
-	req->req_sender = current_thread();
+		vmo->vo_ctrl = NULL;
+		return KERN_OK;
+	}
+
+	struct vm_request *req = vm_cache_alloc(&vm_request_cache, VM_NORMAL);
+	req->req_type = VM_REQUEST_DETACH;
+	req->req_status = VM_REQUEST_ASYNC;
+	req->req_object = vmo;
+	req->req_sender = get_current_thread();
 	send_request_async(ctrl, req);
 
 	vmo->vo_ctrl = NULL;
-	vmo->vo_key = 0;
 	object_unref(&ctrl->vc_base);
 
 	return KERN_OK;
@@ -232,16 +299,16 @@ kern_status_t vm_controller_detach_object(
 
 static void wait_for_reply(
 	struct vm_controller *ctrl,
-	struct page_request *req,
+	struct vm_request *req,
 	unsigned long *lock_flags)
 {
 	struct wait_item waiter;
-	struct thread *self = current_thread();
+	struct thread *self = get_current_thread();
 
 	wait_item_init(&waiter, self);
 	for (;;) {
 		self->tr_state = THREAD_SLEEPING;
-		if (req->req_status == PAGE_REQUEST_COMPLETE) {
+		if (req->req_status == VM_REQUEST_COMPLETE) {
 			break;
 		}
 
@@ -251,9 +318,10 @@ static void wait_for_reply(
 	}
 
 	self->tr_state = THREAD_READY;
+	put_current_thread(self);
 }
 
-static void fulfill_requests(
+void vm_controller_fulfill_requests(
 	struct vm_controller *ctrl,
 	equeue_key_t object,
 	off_t offset,
@@ -263,8 +331,8 @@ static void fulfill_requests(
 	off_t limit = offset + length - 1;
 	struct btree_node *cur = btree_first(&ctrl->vc_requests);
 	while (cur) {
-		struct page_request *req
+		struct vm_request *req
-			= BTREE_CONTAINER(struct page_request, req_node, cur);
+			= BTREE_CONTAINER(struct vm_request, req_node, cur);
 		spin_lock(&req->req_lock);
 		bool match = false;
 		off_t req_base = req->req_offset;
@@ -276,12 +344,14 @@ static void fulfill_requests(
 			match = true;
 		}
 
-		if (req->req_object != object) {
+		vm_object_lock(req->req_object);
+		if (req->req_object->vo_key != object) {
 			match = false;
 		}
+		vm_object_unlock(req->req_object);
 
 		if (match) {
-			req->req_status = PAGE_REQUEST_COMPLETE;
+			req->req_status = VM_REQUEST_COMPLETE;
 			req->req_result = result;
 			thread_awaken(req->req_sender);
 		}
@@ -314,14 +384,13 @@ kern_status_t vm_controller_supply_pages(
 		src_offset,
 		count,
 		NULL);
-	fulfill_requests(ctrl, dst->vo_key, dst_offset, count, status);
 
 	return status;
 }
 
 kern_status_t vm_controller_send_request(
 	struct vm_controller *ctrl,
-	struct page_request *req,
+	struct vm_request *req,
 	unsigned long *irq_flags)
 {
 	fill_random(&req->req_id, sizeof req->req_id);

vm/vm-object.c

@@ -1,4 +1,5 @@
 #include <kernel/address-space.h>
+#include <kernel/panic.h>
 #include <kernel/printk.h>
 #include <kernel/sched.h>
 #include <kernel/util.h>
@@ -31,9 +32,10 @@ static kern_status_t vm_object_cleanup(struct object *obj)
 
 	if (vmo->vo_ctrl) {
 		unsigned long flags;
-		vm_controller_lock_irqsave(vmo->vo_ctrl, &flags);
+		struct vm_controller *ctrl = vmo->vo_ctrl;
+		vm_controller_lock_irqsave(ctrl, &flags);
 		vm_controller_detach_object(vmo->vo_ctrl, vmo);
-		vm_controller_unlock_irqrestore(vmo->vo_ctrl, flags);
+		vm_controller_unlock_irqrestore(ctrl, flags);
 	}
 
 	return KERN_OK;
@@ -276,6 +278,75 @@ extern struct vm_object *vm_object_create_in_place(
 	return vmo;
 }
 
+struct vm_object *vm_object_duplicate_cow(struct vm_object *vmo)
+{
+	struct object *obj = object_create(&vm_object_type);
+	if (!obj) {
+		return NULL;
+	}
+
+	struct vm_object *out = VM_OBJECT_CAST(obj);
+
+	memcpy(out->vo_name, vmo->vo_name, sizeof out->vo_name);
+	out->vo_flags = vmo->vo_flags | VMO_LAZY_ATTACH;
+	out->vo_ctrl = vmo->vo_ctrl;
+	out->vo_key = vmo->vo_key;
+	out->vo_prot = vmo->vo_prot;
+	out->vo_size = vmo->vo_size;
+	memcpy(out->vo_name, vmo->vo_name, sizeof vmo->vo_name);
+
+	struct btree_node *cur = btree_first(&vmo->vo_pages);
+	while (cur) {
+		struct vm_page *pg
+			= BTREE_CONTAINER(struct vm_page, p_bnode, cur);
+		int x = atomic_fetch_add(&pg->p_cow_ref, 1);
+		if (x == 0) {
+			/* this is the first cow-reference for this page, so
+			 * the address-space it belongs to will need one too */
+			atomic_fetch_add(&pg->p_cow_ref, 1);
+		}
+
+		tracek("convert page %zx to cow %u",
+		       vm_page_get_paddr(pg),
+		       pg->p_cow_ref);
+
+		cur = btree_next(cur);
+	}
+
+	return out;
+}
+
+kern_status_t vm_object_attach_cow(
+	struct vm_object *vmo,
+	unsigned long *irq_flags)
+{
+	struct vm_controller *ctrl = vmo->vo_ctrl;
+	struct vm_request req = {0};
+	req.req_status = VM_REQUEST_PENDING;
+	req.req_type = VM_REQUEST_ATTACH;
+	req.req_length = vm_page_order_to_bytes(VM_PAGE_4K);
+	req.req_sender = get_current_thread();
+
+	object_ref(&vmo->vo_base);
+	req.req_object = vmo;
+
+	vm_object_unlock_irqrestore(vmo, *irq_flags);
+	vm_controller_lock_irqsave(ctrl, irq_flags);
+
+	spin_lock(&req.req_lock);
+
+	kern_status_t status
+		= vm_controller_send_request(ctrl, &req, irq_flags);
+
+	put_current_thread(req.req_sender);
+	spin_unlock(&req.req_lock);
+	vm_controller_unlock_irqrestore(ctrl, *irq_flags);
+	object_unref(&vmo->vo_base);
+	vm_object_lock_irqsave(vmo, irq_flags);
+
+	return status;
+}
+
 static struct vm_page *alloc_page(struct vm_object *vo, off_t offset)
 {
 	struct vm_page *page = NULL;
@@ -341,6 +412,54 @@ static struct vm_page *alloc_page(struct vm_object *vo, off_t offset)
 	return NULL;
 }
 
+kern_status_t vm_object_put_page(
+	struct vm_object *vo,
+	off_t offset,
+	struct vm_page *pg)
+{
+	struct btree_node *cur = vo->vo_pages.b_root;
+	if (!vo->vo_pages.b_root) {
+		vo->vo_pages.b_root = &pg->p_bnode;
+		return KERN_OK;
+	}
+
+	while (cur) {
+		struct vm_page *page
+			= BTREE_CONTAINER(struct vm_page, p_bnode, cur);
+		struct btree_node *next = NULL;
+
+		off_t base = page->p_vmo_offset;
+		off_t limit = base + vm_page_get_size_bytes(page);
+		if (offset < base) {
+			next = btree_left(cur);
+		} else if (offset >= limit) {
+			next = btree_right(cur);
+		} else {
+			panic("vm_object_put_page: page already exists");
+			return KERN_NAME_EXISTS;
+		}
+
+		if (next) {
+			cur = next;
+			continue;
+		}
+
+		pg->p_vmo_offset = offset;
+
+		if (offset < base) {
+			btree_put_left(cur, &pg->p_bnode);
+		} else {
+			btree_put_right(cur, &pg->p_bnode);
+		}
+
+		btree_insert_fixup(&vo->vo_pages, &pg->p_bnode);
+
+		return KERN_OK;
+	}
+
+	return KERN_FATAL_ERROR;
+}
+
 static struct vm_page *get_page(struct vm_object *vo, off_t offset)
 {
 	struct btree_node *cur = vo->vo_pages.b_root;
@@ -371,15 +490,15 @@ static kern_status_t request_page(
 	unsigned long *irq_flags)
 {
 	struct vm_controller *ctrl = vo->vo_ctrl;
-	struct page_request req = {0};
+	struct vm_request req = {0};
-	req.req_status = PAGE_REQUEST_PENDING;
+	req.req_status = VM_REQUEST_PENDING;
-	req.req_type = PAGE_REQUEST_READ;
+	req.req_type = VM_REQUEST_READ;
 	req.req_offset = offset;
 	req.req_length = vm_page_order_to_bytes(VM_PAGE_4K);
-	req.req_sender = current_thread();
+	req.req_sender = get_current_thread();
 
 	object_ref(&vo->vo_base);
-	req.req_object = vo->vo_key;
+	req.req_object = vo;
 
 	vm_object_unlock_irqrestore(vo, *irq_flags);
 	vm_controller_lock_irqsave(ctrl, irq_flags);
@@ -388,6 +507,7 @@ static kern_status_t request_page(
 	kern_status_t status
 		= vm_controller_send_request(ctrl, &req, irq_flags);
 
+	put_current_thread(req.req_sender);
 	spin_unlock(&req.req_lock);
 	vm_controller_unlock_irqrestore(ctrl, *irq_flags);
 	object_unref(&vo->vo_base);
@@ -396,6 +516,33 @@ static kern_status_t request_page(
 	return status;
 }
 
+kern_status_t vm_object_prefetch(
+	struct vm_object *vo,
+	off_t offset,
+	size_t length,
+	unsigned long *irq_flags)
+{
+	offset &= ~VM_PAGE_MASK;
+	if (length & VM_PAGE_MASK) {
+		length &= ~VM_PAGE_MASK;
+		length += VM_PAGE_SIZE;
+	}
+
+	for (off_t i = offset; i < offset + length; i += VM_PAGE_SIZE) {
+		struct vm_page *pg = vm_object_get_page(
+			vo,
+			i,
+			VMO_ALLOCATE_MISSING_PAGE | VMO_REQUEST_MISSING_PAGE,
+			irq_flags);
+		if (!pg) {
+			/* TODO get error code from vm_object_get_page */
+			return KERN_NO_MEMORY;
+		}
+	}
+
+	return KERN_OK;
+}
+
 struct vm_page *vm_object_get_page(
 	struct vm_object *vo,
 	off_t offset,
@@ -412,7 +559,7 @@ struct vm_page *vm_object_get_page(
 		return pg;
 	}
 
-	if (!vo->vo_ctrl) {
+	if (!vo->vo_ctrl || !(flags & VMO_REQUEST_MISSING_PAGE)) {
 		return NULL;
 	}