vm: implement private and shared address space mappings

whether a mapping is private or shared determines how the mapping is handled
when a task is duplicated.

CMakeLists.txt

@@ -1,4 +1,4 @@
-cmake_minimum_required(VERSION 4.0)
+cmake_minimum_required(VERSION 3.31)
 project(mango C ASM)
 
 if (NOT BUILD_TOOLS_DIR)

arch/x86_64/include/kernel/machine/thread.h

@@ -34,7 +34,9 @@ extern kern_status_t ml_thread_prepare_user_context(
 /* prepare the stack so that ml_thread_switch_user can jump to usermode
  * with the specified register context */
 extern kern_status_t ml_thread_clone_user_context(
-	const struct ml_cpu_context *ctx,
+	const struct ml_cpu_context *src_regs,
+	const struct ml_thread *src_ml,
+	struct ml_thread *dest_ml,
 	uintptr_t return_value,
 	virt_addr_t *kernel_sp);
 

arch/x86_64/init.c

@@ -20,7 +20,7 @@
 #include <kernel/util.h>
 #include <kernel/vm.h>
 
-#undef HARDWARE_RNG
+#define HARDWARE_RNG
 
 #define PTR32(x)            ((void *)((uintptr_t)(x)))
 
@@ -56,9 +56,7 @@ static void early_vm_init(uintptr_t reserve_end)
 	uintptr_t alloc_end = VM_KERNEL_VOFFSET + 0x7fffffff;
 
 	memblock_init(alloc_start, alloc_end, VM_KERNEL_VOFFSET);
-	printk("memblock: allocating from [0x%llx-0x%llx]",
+	printk("memblock: allocating from [0x%llx-0x%llx]", alloc_start, alloc_end);
-	       alloc_start,
-	       alloc_end);
 
 	memblock_reserve(0x00, reserve_end);
 	printk("memblock: reserved bios+kernel at [0x%016llx-0x%016llx]",

arch/x86_64/irq.c

@@ -217,6 +217,12 @@ void irq_dispatch(struct ml_cpu_context *regs)
 
 void syscall_dispatch(struct ml_cpu_context *regs)
 {
+	struct thread *thr = get_current_thread();
+	if (thr) {
+		thr->tr_irqctx = regs;
+		put_current_thread(thr);
+	}
+
 	unsigned int sysid = regs->rax;
 	virt_addr_t syscall_impl = syscall_get_function(sysid);
 

arch/x86_64/pmap.c

@@ -196,15 +196,15 @@ kern_status_t pmap_get(
 		}
 
 		if (pte & PTE_RW) {
-			*out_pfn |= (VM_PROT_READ | VM_PROT_WRITE);
+			*out_prot |= (VM_PROT_READ | VM_PROT_WRITE);
 		}
 
 		if (pte & PTE_USR) {
-			*out_pfn |= VM_PROT_USER;
+			*out_prot |= VM_PROT_USER;
 		}
 
 		if (!(pte & PTE_NX)) {
-			*out_pfn |= VM_PROT_EXEC;
+			*out_prot |= VM_PROT_EXEC;
 		}
 	}
 

arch/x86_64/serial.c

@@ -28,6 +28,10 @@ void serial_send_byte(int device, char out)
 
 	outportb(device, out);
 
+	if (device == COM1) {
+		outportb(0xe9, out);
+	}
+
 	while (!transmit_empty(device)) {
 		_count++;
 	}

arch/x86_64/thread.c

@@ -81,16 +81,21 @@ extern kern_status_t ml_thread_prepare_user_context(
 }
 
 kern_status_t ml_thread_clone_user_context(
-	const struct ml_cpu_context *src_ctx,
+	const struct ml_cpu_context *src_regs,
+	const struct ml_thread *src_ml,
+	struct ml_thread *dest_ml,
 	uintptr_t return_value,
 	virt_addr_t *kernel_sp)
 {
 	(*kernel_sp) -= sizeof(struct ml_cpu_context);
 
-	struct ml_cpu_context *ctx = (struct ml_cpu_context *)(*kernel_sp);
+	struct ml_cpu_context *regs = (struct ml_cpu_context *)(*kernel_sp);
-	memcpy(ctx, src_ctx, sizeof *ctx);
+	memcpy(regs, src_regs, sizeof *regs);
 
-	ctx->rax = return_value;
+	regs->rax = return_value;
+
+	dest_ml->tr_fsbase = src_ml->tr_fsbase;
+	dest_ml->tr_gsbase = src_ml->tr_gsbase;
 
 	return KERN_OK;
 }

include/kernel/address-space.h

@@ -22,6 +22,8 @@ struct vm_area {
 	struct address_space *vma_space;
 	/* used to link to vm_object->vo_mappings */
 	struct queue_entry vma_object_entry;
+	/* the memory control flags applied to this area */
+	vm_flags_t vma_flags;
 	/* the memory protection flags applied to this area */
 	vm_prot_t vma_prot;
 	/* offset in bytes to the start of the object data that was mapped */
@@ -83,6 +85,7 @@ extern kern_status_t address_space_map(
 	struct vm_object *object,
 	off_t object_offset,
 	size_t length,
+	vm_flags_t flags,
 	vm_prot_t prot,
 	virt_addr_t *out);
 extern kern_status_t address_space_unmap(

include/kernel/handle.h

@@ -44,6 +44,9 @@ struct handle_table {
 
 extern struct handle_table *handle_table_create(void);
 extern void handle_table_destroy(struct handle_table *tab);
+extern kern_status_t handle_table_duplicate(
+	struct handle_table *src,
+	struct handle_table **dest);
 
 extern kern_status_t handle_table_alloc_handle(
 	struct handle_table *tab,

include/kernel/object.h

@@ -65,7 +65,7 @@ extern "C" {
 #define OBJECT_PATH_MAX 256
 
 #define OBJECT_CAST(to_type, to_type_member, p)                                \
-	((to_type *)((uintptr_t)p) - offsetof(to_type, to_type_member))
+	((to_type *)(((uintptr_t)p) - offsetof(to_type, to_type_member)))
 #define OBJECT_C_CAST(c_type, c_type_member, obj_type, objp)                   \
 	OBJECT_IS_TYPE(objp, obj_type)                                         \
 	? OBJECT_CAST(c_type, c_type_member, (objp)) : NULL

include/kernel/syscall.h

@@ -128,6 +128,7 @@ extern kern_status_t sys_address_space_map(
 	kern_handle_t object,
 	off_t object_offset,
 	size_t length,
+	vm_flags_t flags,
 	vm_prot_t prot,
 	virt_addr_t *out_base_address);
 extern kern_status_t sys_address_space_unmap(
@@ -211,7 +212,7 @@ extern kern_status_t sys_kern_object_query(
 extern kern_status_t sys_vm_controller_create(kern_handle_t *out);
 extern kern_status_t sys_vm_controller_recv(
 	kern_handle_t ctrl,
-	equeue_packet_page_request_t *out);
+	equeue_packet_vm_request_t *out);
 extern kern_status_t sys_vm_controller_recv_async(
 	kern_handle_t ctrl,
 	kern_handle_t eq,
@@ -224,6 +225,14 @@ extern kern_status_t sys_vm_controller_create_object(
 	size_t data_len,
 	vm_prot_t prot,
 	kern_handle_t *out);
+extern kern_status_t sys_vm_controller_prepare_attach(
+	kern_handle_t ctrl,
+	uint64_t req_id,
+	kern_handle_t *out_vmo);
+extern kern_status_t sys_vm_controller_finish_attach(
+	kern_handle_t ctrl,
+	uint64_t req_id,
+	equeue_key_t new_key);
 extern kern_status_t sys_vm_controller_detach_object(
 	kern_handle_t ctrl,
 	kern_handle_t vmo);

include/kernel/task.h

@@ -39,7 +39,10 @@ struct task {
 
 extern struct task *task_alloc(void);
 extern struct task *task_cast(struct object *obj);
-extern struct task *task_create(const char *name, size_t name_len);
+extern struct task *task_create(
+	const char *name,
+	size_t name_len,
+	struct handle_table *handles);
 static inline struct task *task_ref(struct task *task)
 {
 	return OBJECT_CAST(struct task, t_base, object_ref(&task->t_base));

include/kernel/vm-controller.h

@@ -9,34 +9,35 @@ struct thread;
 struct equeue;
 struct vm_object;
 
-enum page_request_status {
+enum vm_request_status {
-	PAGE_REQUEST_PENDING = 0,
+	VM_REQUEST_PENDING = 0,
-	PAGE_REQUEST_IN_PROGRESS,
+	VM_REQUEST_IN_PROGRESS,
-	PAGE_REQUEST_COMPLETE,
+	VM_REQUEST_COMPLETE,
-	PAGE_REQUEST_ASYNC,
+	VM_REQUEST_ASYNC,
 };
 
 struct vm_controller {
 	struct object vc_base;
-	/* tree of pending page requests */
+	/* tree of pending vm requests */
 	struct btree vc_requests;
-	/* the equeue to send async page requests to */
 	struct equeue *vc_eq;
 	equeue_key_t vc_eq_key;
-	/* the number of page requests queued with status PAGE_REQUEST_PENDING.
+	/* the number of page requests queued with status VM_REQUEST_PENDING.
 	 * used to assert/clear VM_CONTROLLER_SIGNAL_REQUEST_RECEIVED */
 	size_t vc_requests_waiting;
 };
 
-struct page_request {
+struct vm_request {
 	uint64_t req_id;
 	unsigned int req_type;
-	enum page_request_status req_status;
+	enum vm_request_status req_status;
 	kern_status_t req_result;
 	spin_lock_t req_lock;
-	equeue_key_t req_object;
+	struct vm_object *req_object;
 	struct thread *req_sender;
+	/* this node is added to vm-controller vc_requests list */
 	struct btree_node req_node;
+	/* these values are used for VM_REQUEST_READ and VM_REQUEST_DIRTY */
 	off_t req_offset;
 	size_t req_length;
 };
@@ -48,7 +49,7 @@ extern struct vm_controller *vm_controller_create(void);
 
 extern kern_status_t vm_controller_recv(
 	struct vm_controller *ctrl,
-	equeue_packet_page_request_t *out);
+	equeue_packet_vm_request_t *out);
 extern kern_status_t vm_controller_recv_async(
 	struct vm_controller *ctrl,
 	struct equeue *eq,
@@ -62,6 +63,14 @@ extern kern_status_t vm_controller_create_object(
 	size_t data_len,
 	vm_prot_t prot,
 	struct vm_object **out);
+extern kern_status_t vm_controller_prepare_attach(
+	struct vm_controller *ctrl,
+	uint64_t req_id,
+	struct vm_object **out_vmo);
+extern kern_status_t vm_controller_finish_attach(
+	struct vm_controller *ctrl,
+	uint64_t req_id,
+	equeue_key_t new_key);
 extern kern_status_t vm_controller_detach_object(
 	struct vm_controller *ctrl,
 	struct vm_object *vmo);
@@ -72,10 +81,16 @@ extern kern_status_t vm_controller_supply_pages(
 	struct vm_object *src,
 	off_t src_offset,
 	size_t count);
+extern void vm_controller_fulfill_requests(
+	struct vm_controller *ctrl,
+	equeue_key_t object,
+	off_t offset,
+	size_t length,
+	kern_status_t result);
 
 extern kern_status_t vm_controller_send_request(
 	struct vm_controller *ctrl,
-	struct page_request *req,
+	struct vm_request *req,
 	unsigned long *irq_flags);
 
 DEFINE_OBJECT_LOCK_FUNCTION(vm_controller, vc_base)

include/kernel/vm-object.h

@@ -20,19 +20,25 @@ enum vm_object_flags {
 	 * be detached, allowing the server to close the last handle to the
 	 * object and dispose of it. */
 	VMO_AUTO_DETACH = 0x04u,
+	/* this vmo is a duplicate of a vmo that is attached to a vm-controller.
+	 * the duplicate vmo is scheduled to be attached to the same controller,
+	 * but this won't actually happen until the controller is needed to
+	 * fulfill a page request. once the duplicate vmo has been attached to
+	 * the controller, this flag will be cleared. */
+	VMO_LAZY_ATTACH = 0x08u,
 
 	/* these flags are for use with vm_object_get_page */
 	/**************************************************/
 
 	/* if the relevant page hasn't been allocated yet, it will be allocated
 	 * and returned. if this flag isn't specified, NULL will be returned. */
-	VMO_ALLOCATE_MISSING_PAGE = 0x08u,
+	VMO_ALLOCATE_MISSING_PAGE = 0x0100u,
 	/* if the vm-object is attached to a vm-controller, and the relevant
 	 * page is uncommitted, send a request to the vm-controller to provide
 	 * the missing page. will result in the vm-object being unlocked and
 	 * the current thread sleeping until the request is fulfilled. the
 	 * vm-object will be re-locked before the function returns. */
-	VMO_REQUEST_MISSING_PAGE = 0x10u,
+	VMO_REQUEST_MISSING_PAGE = 0x0200u,
 };
 
 struct vm_object {
@@ -86,6 +92,11 @@ extern struct vm_object *vm_object_create_in_place(
 
 /* create a copy-on-write duplicate of a vm-object */
 extern struct vm_object *vm_object_duplicate_cow(struct vm_object *vmo);
+/* attach a copy-on-write duplicate of a vm-object to the vm-controller that
+ * controlled the original vm-object */
+extern kern_status_t vm_object_attach_cow(
+	struct vm_object *vmo,
+	unsigned long *irq_flags);
 
 extern struct vm_page *vm_object_get_page(
 	struct vm_object *vo,

init/main.c

@@ -108,7 +108,7 @@ void kernel_init(uintptr_t arg)
 	       bsp.bsp_trailer.bsp_exec_entry,
 	       bsp.bsp_vmo);
 
-	struct task *bootstrap_task = task_create("bootstrap", 9);
+	struct task *bootstrap_task = task_create("bootstrap", 9, NULL);
 	tracek("created bootstrap task (pid=%u)", bootstrap_task->t_id);
 
 	status = bsp_launch_async(&bsp, bootstrap_task);

kernel/bsp.c

@@ -120,6 +120,7 @@ static kern_status_t map_executable_exec(
 		bsp->bsp_vmo,
 		text_foffset,
 		bsp->bsp_trailer.bsp_text_size,
+		VM_SHARED,
 		VM_PROT_READ | VM_PROT_EXEC | VM_PROT_USER,
 		&text_base);
 	if (status != KERN_OK) {
@@ -132,6 +133,7 @@ static kern_status_t map_executable_exec(
 		data,
 		data_foffset,
 		bsp->bsp_trailer.bsp_data_size,
+		VM_PRIVATE,
 		VM_PROT_READ | VM_PROT_WRITE | VM_PROT_USER,
 		&data_base);
 	if (status != KERN_OK) {
@@ -165,6 +167,7 @@ kern_status_t bsp_launch_async(struct bsp *bsp, struct task *task)
 		user_stack,
 		0,
 		BOOTSTRAP_STACK_SIZE,
+		VM_PRIVATE,
 		VM_PROT_READ | VM_PROT_WRITE | VM_PROT_USER,
 		&stack_buffer);
 
@@ -178,6 +181,7 @@ kern_status_t bsp_launch_async(struct bsp *bsp, struct task *task)
 		bsp->bsp_vmo,
 		0,
 		bsp->bsp_trailer.bsp_exec_offset,
+		VM_PRIVATE,
 		VM_PROT_READ | VM_PROT_USER,
 		&bsp_data_base);
 

kernel/handle.c

@@ -77,6 +77,81 @@ void handle_table_destroy(struct handle_table *tab)
 	do_handle_table_destroy(tab, 0);
 }
 
+static kern_status_t do_handle_table_duplicate_leaf(
+	struct handle_table *src,
+	struct handle_table **dest)
+{
+	struct handle_table *out
+		= vm_cache_alloc(&handle_table_cache, VM_NORMAL);
+	if (!out) {
+		return KERN_NO_MEMORY;
+	}
+
+	memcpy(out, src, sizeof *out);
+
+	for (size_t i = 0; i < HANDLES_PER_TABLE; i++) {
+		struct object *obj = src->t_handles.t_handle_list[i].h_object;
+		if (obj) {
+			object_ref(obj);
+		}
+	}
+
+	*dest = out;
+	return KERN_OK;
+}
+
+static kern_status_t do_handle_table_duplicate(
+	struct handle_table *src,
+	struct handle_table **dest,
+	unsigned int depth)
+{
+	if (depth == MAX_TABLE_DEPTH - 1) {
+		return do_handle_table_duplicate_leaf(src, dest);
+	}
+
+	struct handle_table *out
+		= vm_cache_alloc(&handle_table_cache, VM_NORMAL);
+	if (!out) {
+		return KERN_NO_MEMORY;
+	}
+
+	memcpy(out->t_subtables.t_subtable_map,
+	       src->t_subtables.t_subtable_map,
+	       sizeof out->t_subtables.t_subtable_map);
+	memset(out->t_subtables.t_subtable_list,
+	       0x0,
+	       sizeof out->t_subtables.t_subtable_list);
+
+	for (size_t i = 0; i < REFS_PER_TABLE; i++) {
+		struct handle_table *child
+			= src->t_subtables.t_subtable_list[i];
+		struct handle_table *dup = NULL;
+		kern_status_t status = KERN_OK;
+		if (child) {
+			status = do_handle_table_duplicate(
+				child,
+				&dup,
+				depth + 1);
+		}
+
+		if (status == KERN_OK) {
+			out->t_subtables.t_subtable_list[i] = dup;
+		} else {
+			return status;
+		}
+	}
+
+	*dest = out;
+	return KERN_OK;
+}
+
+kern_status_t handle_table_duplicate(
+	struct handle_table *src,
+	struct handle_table **dest)
+{
+	return do_handle_table_duplicate(src, dest, 0);
+}
+
 static kern_status_t decode_handle_indices(
 	kern_handle_t handle,
 	unsigned int indices[MAX_TABLE_DEPTH])

kernel/printk.c

@@ -98,9 +98,8 @@ int printk(const char *format, ...)
 	unsigned long flags;
 	spin_lock_irqsave(&log_buffer_lock, &flags);
 	save_log_message(msg);
-	spin_unlock_irqrestore(&log_buffer_lock, flags);
-
 	flush_log_buffer();
+	spin_unlock_irqrestore(&log_buffer_lock, flags);
 
 	return 0;
 }

libmango/arch/x86_64/syscall.S

@@ -77,7 +77,7 @@ SYSCALL_GATE vm_object_copy SYS_VM_OBJECT_COPY 6
 
 SYSCALL_GATE address_space_read SYS_ADDRESS_SPACE_READ 5
 SYSCALL_GATE address_space_write SYS_ADDRESS_SPACE_WRITE 5
-SYSCALL_GATE address_space_map SYS_ADDRESS_SPACE_MAP 7
+SYSCALL_GATE address_space_map SYS_ADDRESS_SPACE_MAP 8
 SYSCALL_GATE address_space_unmap SYS_ADDRESS_SPACE_UNMAP 3
 SYSCALL_GATE address_space_reserve SYS_ADDRESS_SPACE_RESERVE 4
 SYSCALL_GATE address_space_release SYS_ADDRESS_SPACE_RELEASE 3
@@ -102,6 +102,8 @@ SYSCALL_GATE vm_controller_create SYS_VM_CONTROLLER_CREATE 1
 SYSCALL_GATE vm_controller_recv SYS_VM_CONTROLLER_RECV 2
 SYSCALL_GATE vm_controller_recv_async SYS_VM_CONTROLLER_RECV_ASYNC 3
 SYSCALL_GATE vm_controller_create_object SYS_VM_CONTROLLER_CREATE_OBJECT 7
+SYSCALL_GATE vm_controller_prepare_attach SYS_VM_CONTROLLER_PREPARE_ATTACH 3
+SYSCALL_GATE vm_controller_finish_attach SYS_VM_CONTROLLER_FINISH_ATTACH 3
 SYSCALL_GATE vm_controller_detach_object SYS_VM_CONTROLLER_DETACH_OBJECT 2
 SYSCALL_GATE vm_controller_supply_pages SYS_VM_CONTROLLER_SUPPLY_PAGES 6
 

libmango/include-user/mango/vm.h

@@ -48,6 +48,7 @@ extern kern_status_t address_space_map(
 	kern_handle_t object,
 	off_t object_offset,
 	size_t length,
+	vm_flags_t flags,
 	vm_prot_t prot,
 	virt_addr_t *out_base_address);
 extern kern_status_t address_space_unmap(
@@ -67,7 +68,7 @@ extern kern_status_t address_space_release(
 extern kern_status_t vm_controller_create(kern_handle_t *out);
 extern kern_status_t vm_controller_recv(
 	kern_handle_t ctrl,
-	equeue_packet_page_request_t *out);
+	equeue_packet_vm_request_t *out);
 extern kern_status_t vm_controller_recv_async(
 	kern_handle_t ctrl,
 	kern_handle_t eq,
@@ -80,6 +81,14 @@ extern kern_status_t vm_controller_create_object(
 	size_t data_len,
 	vm_prot_t prot,
 	kern_handle_t *out);
+extern kern_status_t vm_controller_prepare_attach(
+	kern_handle_t ctrl,
+	uint64_t req_id,
+	kern_handle_t *out_vmo);
+extern kern_status_t vm_controller_finish_attach(
+	kern_handle_t ctrl,
+	uint64_t req_id,
+	equeue_key_t new_key);
 extern kern_status_t vm_controller_detach_object(
 	kern_handle_t ctrl,
 	kern_handle_t vmo);

libmango/include/mango/syscall.h

@@ -4,53 +4,55 @@
 #define SYS_KERN_LOG                     1
 #define SYS_KERN_HANDLE_CLOSE            2
 #define SYS_KERN_HANDLE_TRANSFER         3
-#define SYS_KERN_CONFIG_GET             4
+#define SYS_KERN_HANDLE_CONTROL          4
-#define SYS_KERN_CONFIG_SET             5
+#define SYS_KERN_CONFIG_GET              5
-#define SYS_KERN_OBJECT_WAIT            6
+#define SYS_KERN_CONFIG_SET              6
-#define SYS_KERN_OBJECT_WAIT_ASYNC      7
+#define SYS_KERN_OBJECT_WAIT             7
-#define SYS_TASK_EXIT                   8
+#define SYS_KERN_OBJECT_WAIT_ASYNC       8
-#define SYS_TASK_SELF                   9
+#define SYS_TASK_EXIT                    9
-#define SYS_TASK_CREATE                 10
+#define SYS_TASK_SELF                    10
-#define SYS_TASK_CREATE_THREAD          11
+#define SYS_TASK_CREATE                  11
-#define SYS_TASK_GET_ADDRESS_SPACE      12
+#define SYS_TASK_CREATE_THREAD           12
-#define SYS_TASK_CONFIG_GET             13
+#define SYS_TASK_GET_ADDRESS_SPACE       13
-#define SYS_TASK_CONFIG_SET             14
+#define SYS_TASK_CONFIG_GET              14
-#define SYS_THREAD_SELF                 15
+#define SYS_TASK_CONFIG_SET              15
-#define SYS_THREAD_START                16
+#define SYS_TASK_DUPLICATE               16
-#define SYS_THREAD_EXIT                 17
+#define SYS_THREAD_SELF                  17
-#define SYS_THREAD_CONFIG_GET           18
+#define SYS_THREAD_START                 18
-#define SYS_THREAD_CONFIG_SET           19
+#define SYS_THREAD_EXIT                  19
-#define SYS_VM_OBJECT_CREATE            20
+#define SYS_THREAD_CONFIG_GET            20
-#define SYS_VM_OBJECT_READ              21
+#define SYS_THREAD_CONFIG_SET            21
-#define SYS_VM_OBJECT_WRITE             22
+#define SYS_VM_OBJECT_CREATE             22
-#define SYS_VM_OBJECT_COPY              23
+#define SYS_VM_OBJECT_READ               23
-#define SYS_ADDRESS_SPACE_READ          24
+#define SYS_VM_OBJECT_WRITE              24
-#define SYS_ADDRESS_SPACE_WRITE         25
+#define SYS_VM_OBJECT_COPY               25
-#define SYS_ADDRESS_SPACE_MAP           26
+#define SYS_ADDRESS_SPACE_READ           26
-#define SYS_ADDRESS_SPACE_UNMAP         27
+#define SYS_ADDRESS_SPACE_WRITE          27
-#define SYS_ADDRESS_SPACE_RESERVE       28
+#define SYS_ADDRESS_SPACE_MAP            28
-#define SYS_ADDRESS_SPACE_RELEASE       29
+#define SYS_ADDRESS_SPACE_UNMAP          29
-#define SYS_MSG_SEND                    30
+#define SYS_ADDRESS_SPACE_RESERVE        30
-#define SYS_MSG_RECV                    31
+#define SYS_ADDRESS_SPACE_RELEASE        31
-#define SYS_MSG_REPLY                   32
+#define SYS_MSG_SEND                     32
-#define SYS_MSG_READ                    33
+#define SYS_MSG_RECV                     33
-#define SYS_MSG_WRITE                   34
+#define SYS_MSG_REPLY                    34
-#define SYS_CHANNEL_CREATE              35
+#define SYS_MSG_READ                     35
-#define SYS_PORT_CREATE                 36
+#define SYS_MSG_WRITE                    36
-#define SYS_PORT_CONNECT                37
+#define SYS_CHANNEL_CREATE               37
-#define SYS_PORT_DISCONNECT             38
+#define SYS_PORT_CREATE                  38
-#define SYS_EQUEUE_CREATE               39
+#define SYS_PORT_CONNECT                 39
-#define SYS_EQUEUE_DEQUEUE              40
+#define SYS_PORT_DISCONNECT              40
-#define SYS_VM_CONTROLLER_CREATE        41
+#define SYS_EQUEUE_CREATE                41
-#define SYS_VM_CONTROLLER_RECV          42
+#define SYS_EQUEUE_DEQUEUE               42
-#define SYS_VM_CONTROLLER_RECV_ASYNC    43
+#define SYS_VM_CONTROLLER_CREATE         43
-#define SYS_VM_CONTROLLER_CREATE_OBJECT 44
+#define SYS_VM_CONTROLLER_RECV           44
-#define SYS_VM_CONTROLLER_DETACH_OBJECT 45
+#define SYS_VM_CONTROLLER_RECV_ASYNC     45
-#define SYS_VM_CONTROLLER_SUPPLY_PAGES  46
+#define SYS_VM_CONTROLLER_CREATE_OBJECT  46
-#define SYS_FUTEX_WAIT                  47
+#define SYS_VM_CONTROLLER_PREPARE_ATTACH 47
-#define SYS_FUTEX_WAKE                  48
+#define SYS_VM_CONTROLLER_FINISH_ATTACH  48
-#define SYS_KERN_OBJECT_QUERY           49
+#define SYS_VM_CONTROLLER_DETACH_OBJECT  49
-#define SYS_TASK_DUPLICATE              50
+#define SYS_VM_CONTROLLER_SUPPLY_PAGES   50
-#define SYS_KERN_HANDLE_CONTROL         51
+#define SYS_FUTEX_WAIT                   51
+#define SYS_FUTEX_WAKE                   52
+#define SYS_KERN_OBJECT_QUERY            53
 
 #endif

libmango/include/mango/types.h

@@ -34,6 +34,16 @@
 #define KERN_HANDLE_FLAG2            0x40000000UL
 #define KERN_HANDLE_FLAG3            0x80000000UL
 
+/* flags to specify when creating address-space mappings */
+/* this mapping is private. if a task with this mapping is duplicated,
+   the duplicate task will receive a copy-on-write mapping. changes to one
+   mapping will not be visible to the other. */
+#define VM_PRIVATE                   0x0000u
+/* this mapping is shared. if a task with this mapping is duplicated,
+ * the duplicate will receive a mapping of the same data. changes to one mapping
+ * will be visibile to the other */
+#define VM_SHARED                    0x0001u
+
 /* maximum number of handles that can be sent in a single message */
 #define KERN_MSG_MAX_HANDLES         64
 
@@ -61,13 +71,14 @@
 #define KERN_MSG_EVENT_DISCONNECTION 2
 
 /* equeue packet types */
-#define EQUEUE_PKT_PAGE_REQUEST      0x01u
+#define EQUEUE_PKT_VM_REQUEST        0x01u
 #define EQUEUE_PKT_ASYNC_SIGNAL      0x02u
 
-/* page request types */
+/* vm request types */
-#define PAGE_REQUEST_READ            0x01u
+#define VM_REQUEST_READ              0x01u
-#define PAGE_REQUEST_DIRTY           0x02u
+#define VM_REQUEST_DIRTY             0x02u
-#define PAGE_REQUEST_DETACH          0x03u
+#define VM_REQUEST_ATTACH            0x03u
+#define VM_REQUEST_DETACH            0x04u
 
 /* futex special values */
 #define FUTEX_WAKE_ALL               ((size_t)-1)
@@ -106,6 +117,7 @@ typedef unsigned int kern_status_t;
 typedef uint32_t kern_handle_t;
 typedef uint32_t kern_config_key_t;
 typedef uint32_t vm_prot_t;
+typedef uint32_t vm_flags_t;
 typedef int64_t ssize_t;
 typedef uint32_t kern_futex_t;
 typedef uint32_t kern_msg_type_t;
@@ -175,14 +187,33 @@ typedef struct {
 	/* the key of the vm-object for which the page request relates, as
 	 * specified when the vm-object was created */
 	equeue_key_t req_vmo;
-	/* page request type. one of PAGE_REQUEST_* */
+	/* page request type. one of VM_REQUEST_* */
 	unsigned short req_type;
-	/* of the offset into the vm-object for which pages are being requested
+	/* the offset into the vm-object for which pages are being requested */
+	union {
+		/* used for:
+		 * 	VM_REQUEST_READ
+		 * 	VM_REQUEST_DIRTY
 		 */
+		struct {
 			off_t req_offset;
 			/* the length in bytes of the region being requested */
 			size_t req_length;
-} equeue_packet_page_request_t;
+		};
+
+		/* used for:
+		 * 	VM_REQUEST_ATTACH
+		 */
+		struct {
+			/* the key of the original/source vmo. */
+			equeue_key_t req_src_vmo;
+			/* a request ID. used to retrieve information about
+			 * the newly-attached object, as the server won't know
+			 * about it yet, and won't have a handle to it. */
+			uint64_t req_id;
+		};
+	};
+} equeue_packet_vm_request_t;
 
 typedef struct {
 	/* the type of packet. one of EQUEUE_PKT_* */
@@ -192,8 +223,8 @@ typedef struct {
 	equeue_key_t p_key;
 
 	union {
-		/* p_type = EQUEUE_PKT_PAGE_REQUEST */
+		/* p_type = EQUEUE_PKT_VM_REQUEST */
-		equeue_packet_page_request_t page_request;
+		equeue_packet_vm_request_t vm_request;
 		/* p_type = EQUEUE_PKT_ASYNC_SIGNAL */
 		equeue_packet_async_signal_t async_signal;
 	};

sched/task.c

@@ -171,7 +171,10 @@ struct task *task_alloc(void)
 	return t;
 }
 
-struct task *task_create(const char *name, size_t name_len)
+struct task *task_create(
+	const char *name,
+	size_t name_len,
+	struct handle_table *handles)
 {
 	struct task *task = task_alloc();
 	if (!task) {
@@ -191,9 +194,13 @@ struct task *task_create(const char *name, size_t name_len)
 		VM_USER_LIMIT,
 		&task->t_address_space);
 
+	if (!handles) {
+		handles = handle_table_create();
+	}
+
 	task->t_address_space->s_pmap = pmap;
 	task->t_state = TASK_RUNNING;
-	task->t_handles = handle_table_create();
+	task->t_handles = handles;
 
 	if (name) {
 		name_len = MIN(name_len, sizeof task->t_name - 1);

sched/thread.c

@@ -131,7 +131,12 @@ kern_status_t thread_init_user_clone(
 
 	/* this context will be used by ml_user_return to jump to userspace
 	 * with the specified instruction pointer and user stack */
-	ml_thread_clone_user_context(src->tr_irqctx, return_value, &thr->tr_sp);
+	ml_thread_clone_user_context(
+		src->tr_irqctx,
+		&src->tr_ml,
+		&thr->tr_ml,
+		return_value,
+		&thr->tr_sp);
 	/* this context will be used by the scheduler and ml_thread_switch to
 	 * jump to ml_user_return in kernel mode with the thread's kernel stack.
 	 */

syscall/address-space.c

@@ -125,6 +125,7 @@ kern_status_t sys_address_space_map(
 	kern_handle_t object_handle,
 	off_t object_offset,
 	size_t length,
+	vm_flags_t flags,
 	vm_prot_t prot,
 	virt_addr_t *out_base_address)
 {
@@ -140,8 +141,8 @@ kern_status_t sys_address_space_map(
 	}
 
 	kern_status_t status = KERN_OK;
-	unsigned long flags;
+	unsigned long irq_flags;
-	task_lock_irqsave(self, &flags);
+	task_lock_irqsave(self, &irq_flags);
 
 	struct object *region_obj = NULL, *vmo_obj = NULL;
 	handle_flags_t region_flags = 0, vmo_flags = 0;
@@ -151,34 +152,34 @@ kern_status_t sys_address_space_map(
 		&region_obj,
 		&region_flags);
 	if (status != KERN_OK) {
-		task_unlock_irqrestore(self, flags);
+		task_unlock_irqrestore(self, irq_flags);
 		put_current_task(self);
 		return status;
 	}
 
 	status = task_resolve_handle(self, object_handle, &vmo_obj, &vmo_flags);
 	if (status != KERN_OK) {
-		task_unlock_irqrestore(self, flags);
+		task_unlock_irqrestore(self, irq_flags);
 		put_current_task(self);
 		return status;
 	}
 
 	struct address_space *region = address_space_cast(region_obj);
 	if (!region) {
-		task_unlock_irqrestore(self, flags);
+		task_unlock_irqrestore(self, irq_flags);
 		put_current_task(self);
 		return KERN_INVALID_ARGUMENT;
 	}
 
 	struct vm_object *vmo = vm_object_cast(vmo_obj);
 	if (!vmo) {
-		task_unlock_irqrestore(self, flags);
+		task_unlock_irqrestore(self, irq_flags);
 		put_current_task(self);
 		return KERN_INVALID_ARGUMENT;
 	}
 
-	task_unlock_irqrestore(self, flags);
+	task_unlock_irqrestore(self, irq_flags);
-	address_space_lock_irqsave(region, &flags);
+	address_space_lock_irqsave(region, &irq_flags);
 	/* address_space_map will take care of locking `vmo` */
 	status = address_space_map(
 		region,
@@ -186,9 +187,10 @@ kern_status_t sys_address_space_map(
 		vmo,
 		object_offset,
 		length,
+		flags,
 		prot,
 		out_base_address);
-	address_space_unlock_irqrestore(region, flags);
+	address_space_unlock_irqrestore(region, irq_flags);
 
 	object_unref(vmo_obj);
 	object_unref(region_obj);

syscall/dispatch.c

@@ -47,6 +47,12 @@ static const virt_addr_t syscall_table[] = {
 	SYSCALL_TABLE_ENTRY(
 		VM_CONTROLLER_CREATE_OBJECT,
 		vm_controller_create_object),
+	SYSCALL_TABLE_ENTRY(
+		VM_CONTROLLER_PREPARE_ATTACH,
+		vm_controller_prepare_attach),
+	SYSCALL_TABLE_ENTRY(
+		VM_CONTROLLER_FINISH_ATTACH,
+		vm_controller_finish_attach),
 	SYSCALL_TABLE_ENTRY(
 		VM_CONTROLLER_DETACH_OBJECT,
 		vm_controller_detach_object),

syscall/task.c

@@ -124,7 +124,7 @@ kern_status_t sys_task_create(
 
 	task_unlock_irqrestore(self, flags);
 
-	struct task *child = task_create(name, name_len);
+	struct task *child = task_create(name, name_len, NULL);
 	if (!child) {
 		object_unref(parent_obj);
 
@@ -468,6 +468,14 @@ kern_status_t sys_task_duplicate(
 	unsigned long flags;
 	task_lock_irqsave(self, &flags);
 
+	struct handle_table *child_handle_table = NULL;
+	status = handle_table_duplicate(self->t_handles, &child_handle_table);
+	if (status != KERN_OK) {
+		task_unlock_irqrestore(self, flags);
+		put_current_task(self);
+		return status;
+	}
+
 	struct handle *child_handle_slot = NULL, *space_handle_slot = NULL;
 	kern_handle_t child_handle, space_handle;
 	status = handle_table_alloc_handle(
@@ -476,6 +484,7 @@ kern_status_t sys_task_duplicate(
 		&child_handle_slot,
 		&child_handle);
 	if (status != KERN_OK) {
+		handle_table_destroy(child_handle_table);
 		task_unlock_irqrestore(self, flags);
 		put_current_task(self);
 		return status;
@@ -487,14 +496,19 @@ kern_status_t sys_task_duplicate(
 		&space_handle_slot,
 		&space_handle);
 	if (status != KERN_OK) {
+		handle_table_destroy(child_handle_table);
 		handle_table_free_handle(self->t_handles, child_handle);
 		task_unlock_irqrestore(self, flags);
 		put_current_task(self);
 		return status;
 	}
 
-	struct task *new_task = task_create(self->t_name, strlen(self->t_name));
+	struct task *new_task = task_create(
+		self->t_name,
+		strlen(self->t_name),
+		child_handle_table);
 	if (!new_task) {
+		handle_table_destroy(child_handle_table);
 		put_current_task(self);
 		return KERN_NO_MEMORY;
 	}
@@ -526,18 +540,19 @@ kern_status_t sys_task_duplicate(
 		return status;
 	}
 
-	schedule_thread_on_cpu(new_thread);
-
 	child_handle_slot->h_object = &new_task->t_base;
-	space_handle_slot->h_object = &new_task->t_address_space->s_base;
+	space_handle_slot->h_object
+		= object_ref(&new_task->t_address_space->s_base);
 	task_unlock_irqrestore(self, flags);
 
-	*out_task = child_handle;
-	*out_address_space = space_handle;
-
 	/* clear TLB */
 	pmap_switch(self->t_pmap);
 	put_current_task(self);
 
+	*out_task = child_handle;
+	*out_address_space = space_handle;
+
+	schedule_thread_on_cpu(new_thread);
+
 	return KERN_OK;
 }

syscall/vm-controller.c

@@ -32,7 +32,7 @@ kern_status_t sys_vm_controller_create(kern_handle_t *out)
 
 kern_status_t sys_vm_controller_recv(
 	kern_handle_t ctrl_handle,
-	equeue_packet_page_request_t *out)
+	equeue_packet_vm_request_t *out)
 {
 	struct task *self = get_current_task();
 
@@ -148,8 +148,8 @@ kern_status_t sys_vm_controller_create_object(
 	}
 
 	kern_status_t status = KERN_OK;
-	unsigned long flags;
+	unsigned long irq_flags;
-	task_lock_irqsave(self, &flags);
+	task_lock_irqsave(self, &irq_flags);
 
 	struct object *ctrl_obj = NULL;
 	handle_flags_t handle_flags = 0;
@@ -159,7 +159,7 @@ kern_status_t sys_vm_controller_create_object(
 		&ctrl_obj,
 		&handle_flags);
 	if (status != KERN_OK) {
-		task_unlock_irqrestore(self, flags);
+		task_unlock_irqrestore(self, irq_flags);
 		put_current_task(self);
 		return status;
 	}
@@ -173,14 +173,14 @@ kern_status_t sys_vm_controller_create_object(
 		&out_handle);
 
 	struct vm_controller *ctrl = vm_controller_cast(ctrl_obj);
-	task_unlock_irqrestore(self, flags);
+	task_unlock_irqrestore(self, irq_flags);
 	if (!ctrl) {
 		object_unref(ctrl_obj);
 		put_current_task(self);
 		return KERN_INVALID_ARGUMENT;
 	}
 
-	vm_controller_lock_irqsave(ctrl, &flags);
+	vm_controller_lock_irqsave(ctrl, &irq_flags);
 	struct vm_object *out_vmo = NULL;
 	status = vm_controller_create_object(
 		ctrl,
@@ -190,14 +190,14 @@ kern_status_t sys_vm_controller_create_object(
 		data_len,
 		prot,
 		&out_vmo);
-	vm_controller_unlock_irqrestore(ctrl, flags);
+	vm_controller_unlock_irqrestore(ctrl, irq_flags);
 
 	object_unref(ctrl_obj);
 
 	if (status != KERN_OK) {
-		task_lock_irqsave(self, &flags);
+		task_lock_irqsave(self, &irq_flags);
 		handle_table_free_handle(self->t_handles, out_handle);
-		task_unlock_irqrestore(self, flags);
+		task_unlock_irqrestore(self, irq_flags);
 		put_current_task(self);
 		return status;
 	}
@@ -209,6 +209,116 @@ kern_status_t sys_vm_controller_create_object(
 	return KERN_OK;
 }
 
+kern_status_t sys_vm_controller_prepare_attach(
+	kern_handle_t ctrl_handle,
+	uint64_t req_id,
+	kern_handle_t *out_vmo)
+{
+	struct task *self = get_current_task();
+	if (!out_vmo || !validate_access_w(self, out_vmo, sizeof *out_vmo)) {
+		return KERN_MEMORY_FAULT;
+	}
+
+	kern_status_t status = KERN_OK;
+	unsigned long flags;
+	task_lock_irqsave(self, &flags);
+
+	struct object *ctrl_obj = NULL;
+	handle_flags_t ctrl_flags = 0;
+	status = task_resolve_handle(self, ctrl_handle, &ctrl_obj, &ctrl_flags);
+	if (status != KERN_OK) {
+		task_unlock_irqrestore(self, flags);
+		put_current_task(self);
+		return status;
+	}
+
+	struct handle *out_slot = NULL;
+	kern_handle_t out_handle = KERN_HANDLE_INVALID;
+	status = handle_table_alloc_handle(
+		self->t_handles,
+		KERN_HANDLE_INVALID,
+		&out_slot,
+		&out_handle);
+	if (status != KERN_OK) {
+		object_unref(ctrl_obj);
+		task_unlock_irqrestore(self, flags);
+		put_current_task(self);
+		return status;
+	}
+
+	struct vm_controller *ctrl = vm_controller_cast(ctrl_obj);
+	task_unlock_irqrestore(self, flags);
+
+	if (!ctrl) {
+		object_unref(ctrl_obj);
+		return KERN_INVALID_ARGUMENT;
+	}
+
+	vm_controller_lock_irqsave(ctrl, &flags);
+
+	struct vm_object *vmo = NULL;
+	status = vm_controller_prepare_attach(ctrl, req_id, &vmo);
+	vm_controller_unlock_irqrestore(ctrl, flags);
+
+	object_unref(ctrl_obj);
+	if (status != KERN_OK) {
+		task_lock_irqsave(self, &flags);
+		handle_table_free_handle(self->t_handles, out_handle);
+		task_unlock_irqrestore(self, flags);
+
+		return status;
+	}
+
+	out_slot->h_object = &vmo->vo_base;
+	put_current_task(self);
+
+	*out_vmo = out_handle;
+
+	return KERN_OK;
+}
+
+kern_status_t sys_vm_controller_finish_attach(
+	kern_handle_t ctrl_handle,
+	uint64_t req_id,
+	equeue_key_t new_key)
+{
+	struct task *self = get_current_task();
+
+	kern_status_t status = KERN_OK;
+	unsigned long flags;
+	task_lock_irqsave(self, &flags);
+
+	struct object *ctrl_obj = NULL;
+	handle_flags_t ctrl_flags = 0;
+	status = task_resolve_handle(self, ctrl_handle, &ctrl_obj, &ctrl_flags);
+	if (status != KERN_OK) {
+		task_unlock_irqrestore(self, flags);
+		put_current_task(self);
+		return status;
+	}
+
+	struct vm_controller *ctrl = vm_controller_cast(ctrl_obj);
+	task_unlock_irqrestore(self, flags);
+	put_current_task(self);
+
+	if (!ctrl) {
+		object_unref(ctrl_obj);
+		return KERN_INVALID_ARGUMENT;
+	}
+
+	vm_controller_lock_irqsave(ctrl, &flags);
+
+	status = vm_controller_finish_attach(ctrl, req_id, new_key);
+	vm_controller_unlock_irqrestore(ctrl, flags);
+
+	object_unref(ctrl_obj);
+	if (status != KERN_OK) {
+		return status;
+	}
+
+	return KERN_OK;
+}
+
 kern_status_t sys_vm_controller_detach_object(
 	kern_handle_t ctrl_handle,
 	kern_handle_t vmo_handle)
@@ -315,6 +425,8 @@ kern_status_t sys_vm_controller_supply_pages(
 
 	vm_controller_lock_irqsave(ctrl, &flags);
 	vm_object_lock_pair(src, dst);
+	equeue_key_t requester_key = dst->vo_key;
+
 	status = vm_controller_supply_pages(
 		ctrl,
 		dst,
@@ -323,6 +435,12 @@ kern_status_t sys_vm_controller_supply_pages(
 		src_offset,
 		count);
 	vm_object_unlock_pair(src, dst);
+	vm_controller_fulfill_requests(
+		ctrl,
+		requester_key,
+		dst_offset,
+		count,
+		status);
 	vm_controller_unlock_irqrestore(ctrl, flags);
 
 	object_unref(ctrl_obj);

vm/address-space.c

@@ -6,6 +6,7 @@
 #include <kernel/printk.h>
 #include <kernel/sched.h>
 #include <kernel/task.h>
+#include <kernel/thread.h>
 #include <kernel/util.h>
 #include <kernel/vm-controller.h>
 #include <kernel/vm-object.h>
@@ -705,6 +706,7 @@ kern_status_t address_space_map(
 	struct vm_object *object,
 	off_t object_offset,
 	size_t length,
+	vm_flags_t flags,
 	vm_prot_t prot,
 	virt_addr_t *out)
 {
@@ -762,6 +764,7 @@ kern_status_t address_space_map(
 	area->vma_space = root;
 	area->vma_object = object;
 	area->vma_prot = prot;
+	area->vma_flags = flags;
 	area->vma_object_offset = object_offset;
 	area->vma_base = map_address;
 	area->vma_limit = map_address + length - 1;
@@ -1214,6 +1217,7 @@ static struct vm_area *area_duplicate(struct vm_area *area)
 	}
 
 	out->vma_prot = area->vma_prot;
+	out->vma_flags = area->vma_flags;
 	out->vma_object_offset = area->vma_object_offset;
 	out->vma_base = area->vma_base;
 	out->vma_limit = area->vma_limit;
@@ -1280,7 +1284,8 @@ static kern_status_t prepare_duplicate_areas(
 		struct vm_object *src_vmo = tmp_area->vma_object;
 		vm_object_lock(src_vmo);
 
-		struct vm_object *dest_vmo = NULL;
+		struct vm_object *dest_vmo_link = NULL;
+		struct vm_object *dest_vmo_cow = NULL;
 		struct queue_entry *cur_entry
 			= queue_first(&src_vmo->vo_mappings);
 
@@ -1311,6 +1316,13 @@ static kern_status_t prepare_duplicate_areas(
 				continue;
 			}
 
+			struct vm_object *dest_vmo = NULL;
+			if (src_area->vma_flags & VM_SHARED) {
+				dest_vmo = dest_vmo_link;
+			} else {
+				dest_vmo = dest_vmo_cow;
+			}
+
 			if (!dest_vmo) {
 				tracek("[%zx-%zx %x] creating COW duplicate of "
 				       "vmo %p",
@@ -1318,7 +1330,18 @@ static kern_status_t prepare_duplicate_areas(
 				       src_area->vma_limit,
 				       src_area->vma_prot,
 				       src_vmo);
-				dest_vmo = vm_object_duplicate_cow(src_vmo);
+
+				if (src_area->vma_flags & VM_SHARED) {
+					dest_vmo_link = src_vmo;
+					object_ref(&dest_vmo_link->vo_base);
+
+					dest_vmo = dest_vmo_link;
+				} else {
+					dest_vmo_cow = vm_object_duplicate_cow(
+						src_vmo);
+					dest_vmo = dest_vmo_cow;
+				}
+
 				tracek("[%zx-%zx %x] created COW duplicate of "
 				       "vmo %p -> %p",
 				       src_area->vma_base,
@@ -1328,11 +1351,20 @@ static kern_status_t prepare_duplicate_areas(
 				       dest_vmo);
 			}
 
+			object_ref(&dest_vmo->vo_base);
 			dest_area->vma_object = dest_vmo;
 			update_area_pte_cow(src, dest, src_area);
 			cur_entry = queue_next(cur_entry);
 		}
 
+		if (dest_vmo_link) {
+			object_unref(&dest_vmo_link->vo_base);
+		}
+
+		if (dest_vmo_cow) {
+			object_unref(&dest_vmo_cow->vo_base);
+		}
+
 		vm_object_unlock(src_vmo);
 
 		cur_node = btree_next(cur_node);
@@ -1502,6 +1534,17 @@ static kern_status_t request_missing_page(
 	vm_object_lock(object);
 	address_space_unlock(region);
 
+	kern_status_t status = KERN_OK;
+
+	if (object->vo_flags & VMO_LAZY_ATTACH) {
+		status = vm_object_attach_cow(object, irq_flags);
+	}
+
+	if (status != KERN_OK) {
+		vm_object_unlock_irqrestore(object, *irq_flags);
+		return status;
+	}
+
 	struct vm_page *pg = vm_object_get_page(
 		object,
 		object_offset,
@@ -1515,7 +1558,7 @@ static kern_status_t request_missing_page(
 
 	/* now: `region` is unlocked, and `object` is locked */
 
-	kern_status_t status = pmap_add(
+	status = pmap_add(
 		region->s_pmap,
 		addr,
 		vm_page_get_pfn(pg),
@@ -1550,9 +1593,6 @@ static kern_status_t handle_cow_access(
 	}
 
 	tracek("cow access %zx", addr);
-	if (area->vma_object->vo_ctrl) {
-		panic("COW on controlled vm-object");
-	}
 
 	off_t object_offset = addr - area->vma_base + area->vma_object_offset;
 	vm_object_lock(area->vma_object);
@@ -1663,7 +1703,7 @@ kern_status_t address_space_demand_map(
 	address_space_lock_irqsave(region, &irq_flags);
 
 	const enum pmap_fault_flags cow_flags
-		= PMAP_FAULT_WRITE | PMAP_FAULT_PRESENT | PMAP_FAULT_USER;
+		= PMAP_FAULT_WRITE | PMAP_FAULT_PRESENT;
 
 	if ((flags & cow_flags) == cow_flags) {
 		return handle_cow_access(region, addr, flags, &irq_flags);

vm/vm-controller.c

@@ -9,13 +9,12 @@
 #define VM_CONTROLLER_CAST(p)                                                  \
 	OBJECT_C_CAST(struct vm_controller, vc_base, &vm_controller_type, p)
 
-BTREE_DEFINE_SIMPLE_INSERT(struct vm_object, vo_ctrl_node, vo_key, put_object)
 BTREE_DEFINE_SIMPLE_GET(
-	struct vm_object,
+	struct vm_request,
-	equeue_key_t,
+	uint64_t,
-	vo_ctrl_node,
+	req_node,
-	vo_key,
+	req_id,
-	get_object)
+	get_request)
 
 static struct object_type vm_controller_type = {
 	.ob_name = "vm-controller",
@@ -23,14 +22,14 @@ static struct object_type vm_controller_type = {
 	.ob_header_offset = offsetof(struct vm_controller, vc_base),
 };
 
-static struct vm_cache page_request_cache = {
+static struct vm_cache vm_request_cache = {
-	.c_name = "page-request",
+	.c_name = "vm-request",
-	.c_obj_size = sizeof(struct page_request),
+	.c_obj_size = sizeof(struct vm_request),
 };
 
 kern_status_t vm_controller_type_init(void)
 {
-	vm_cache_init(&page_request_cache);
+	vm_cache_init(&vm_request_cache);
 	return object_type_register(&vm_controller_type);
 }
 
@@ -51,19 +50,19 @@ struct vm_controller *vm_controller_create(void)
 	return ctrl;
 }
 
-static struct page_request *get_next_request(struct vm_controller *ctrl)
+static struct vm_request *get_next_request(struct vm_controller *ctrl)
 {
 	struct btree_node *cur = btree_first(&ctrl->vc_requests);
 	while (cur) {
-		struct page_request *req
+		struct vm_request *req
-			= BTREE_CONTAINER(struct page_request, req_node, cur);
+			= BTREE_CONTAINER(struct vm_request, req_node, cur);
 		spin_lock(&req->req_lock);
 		switch (req->req_status) {
-		case PAGE_REQUEST_PENDING:
+		case VM_REQUEST_PENDING:
-			req->req_status = PAGE_REQUEST_IN_PROGRESS;
+			req->req_status = VM_REQUEST_IN_PROGRESS;
 			ctrl->vc_requests_waiting--;
 			return req;
-		case PAGE_REQUEST_ASYNC:
+		case VM_REQUEST_ASYNC:
 			btree_delete(&ctrl->vc_requests, &req->req_node);
 			ctrl->vc_requests_waiting--;
 			return req;
@@ -78,7 +77,7 @@ static struct page_request *get_next_request(struct vm_controller *ctrl)
 	return NULL;
 }
 
-static kern_status_t try_enqueue(struct btree *tree, struct page_request *req)
+static kern_status_t try_enqueue(struct btree *tree, struct vm_request *req)
 {
 	if (!tree->b_root) {
 		tree->b_root = &req->req_node;
@@ -88,8 +87,8 @@ static kern_status_t try_enqueue(struct btree *tree, struct page_request *req)
 
 	struct btree_node *cur = tree->b_root;
 	while (1) {
-		struct page_request *cur_node
+		struct vm_request *cur_node
-			= BTREE_CONTAINER(struct page_request, req_node, cur);
+			= BTREE_CONTAINER(struct vm_request, req_node, cur);
 		struct btree_node *next = NULL;
 
 		if (req->req_id > cur_node->req_id) {
@@ -119,7 +118,7 @@ static kern_status_t try_enqueue(struct btree *tree, struct page_request *req)
 
 static kern_status_t send_request_async(
 	struct vm_controller *ctrl,
-	struct page_request *req)
+	struct vm_request *req)
 {
 	fill_random(&req->req_id, sizeof req->req_id);
 	while (!try_enqueue(&ctrl->vc_requests, req)) {
@@ -136,9 +135,9 @@ static kern_status_t send_request_async(
 
 kern_status_t vm_controller_recv(
 	struct vm_controller *ctrl,
-	equeue_packet_page_request_t *out)
+	equeue_packet_vm_request_t *out)
 {
-	struct page_request *req = NULL;
+	struct vm_request *req = NULL;
 
 	req = get_next_request(ctrl);
 	if (!req) {
@@ -151,16 +150,30 @@ kern_status_t vm_controller_recv(
 			VM_CONTROLLER_SIGNAL_REQUEST_RECEIVED);
 	}
 
-	out->req_vmo = req->req_object;
+	vm_object_lock(req->req_object);
+	out->req_id = req->req_id;
+	out->req_vmo = req->req_object->vo_key;
 	out->req_type = req->req_type;
+
+	switch (req->req_type) {
+	case VM_REQUEST_READ:
+	case VM_REQUEST_DIRTY:
 		out->req_offset = req->req_offset;
 		out->req_length = req->req_length;
+		break;
+	case VM_REQUEST_ATTACH:
+		out->req_src_vmo = req->req_object->vo_key;
+		break;
+	default:
+		break;
+	}
 
+	vm_object_unlock(req->req_object);
 	spin_unlock(&req->req_lock);
 
-	if (req->req_status == PAGE_REQUEST_ASYNC) {
+	if (req->req_status == VM_REQUEST_ASYNC) {
 		put_current_thread(req->req_sender);
-		vm_cache_free(&page_request_cache, req);
+		vm_cache_free(&vm_request_cache, req);
 	}
 
 	return KERN_OK;
@@ -208,6 +221,52 @@ kern_status_t vm_controller_create_object(
 	return KERN_OK;
 }
 
+kern_status_t vm_controller_prepare_attach(
+	struct vm_controller *ctrl,
+	uint64_t req_id,
+	struct vm_object **out_vmo)
+{
+	struct vm_request *req = get_request(&ctrl->vc_requests, req_id);
+	if (!req) {
+		return KERN_INVALID_ARGUMENT;
+	}
+
+	spin_lock(&req->req_lock);
+	req->req_status = VM_REQUEST_IN_PROGRESS;
+	*out_vmo = req->req_object;
+	spin_unlock(&req->req_lock);
+
+	return KERN_OK;
+}
+
+kern_status_t vm_controller_finish_attach(
+	struct vm_controller *ctrl,
+	uint64_t req_id,
+	equeue_key_t new_key)
+{
+	struct vm_request *req = get_request(&ctrl->vc_requests, req_id);
+	if (!req) {
+		return KERN_INVALID_ARGUMENT;
+	}
+
+	spin_lock(&req->req_lock);
+	struct vm_object *vmo = req->req_object;
+	spin_unlock(&req->req_lock);
+
+	vm_object_lock(vmo);
+	vmo->vo_key = new_key;
+	vmo->vo_flags &= ~VMO_LAZY_ATTACH;
+	vm_object_unlock(vmo);
+
+	spin_lock(&req->req_lock);
+	req->req_status = VM_REQUEST_COMPLETE;
+	req->req_result = KERN_OK;
+	thread_awaken(req->req_sender);
+	spin_unlock(&req->req_lock);
+
+	return KERN_OK;
+}
+
 kern_status_t vm_controller_detach_object(
 	struct vm_controller *ctrl,
 	struct vm_object *vmo)
@@ -216,7 +275,7 @@ kern_status_t vm_controller_detach_object(
 		return KERN_INVALID_ARGUMENT;
 	}
 
-	if (vmo->vo_key == 0) {
+	if (vmo->vo_flags & VMO_LAZY_ATTACH) {
 		/* this vmo isn't actually attached to this controller yet.
 		 * this can happen if a controller-attached vmo was duplicated
 		 * via copy-on-write, and the duplicate vmo has not yet been
@@ -225,16 +284,14 @@ kern_status_t vm_controller_detach_object(
 		return KERN_OK;
 	}
 
-	struct page_request *req
+	struct vm_request *req = vm_cache_alloc(&vm_request_cache, VM_NORMAL);
-		= vm_cache_alloc(&page_request_cache, VM_NORMAL);
+	req->req_type = VM_REQUEST_DETACH;
-	req->req_type = PAGE_REQUEST_DETACH;
+	req->req_status = VM_REQUEST_ASYNC;
-	req->req_status = PAGE_REQUEST_ASYNC;
+	req->req_object = vmo;
-	req->req_object = vmo->vo_key;
 	req->req_sender = get_current_thread();
 	send_request_async(ctrl, req);
 
 	vmo->vo_ctrl = NULL;
-	vmo->vo_key = 0;
 	object_unref(&ctrl->vc_base);
 
 	return KERN_OK;
@@ -242,7 +299,7 @@ kern_status_t vm_controller_detach_object(
 
 static void wait_for_reply(
 	struct vm_controller *ctrl,
-	struct page_request *req,
+	struct vm_request *req,
 	unsigned long *lock_flags)
 {
 	struct wait_item waiter;
@@ -251,7 +308,7 @@ static void wait_for_reply(
 	wait_item_init(&waiter, self);
 	for (;;) {
 		self->tr_state = THREAD_SLEEPING;
-		if (req->req_status == PAGE_REQUEST_COMPLETE) {
+		if (req->req_status == VM_REQUEST_COMPLETE) {
 			break;
 		}
 
@@ -264,7 +321,7 @@ static void wait_for_reply(
 	put_current_thread(self);
 }
 
-static void fulfill_requests(
+void vm_controller_fulfill_requests(
 	struct vm_controller *ctrl,
 	equeue_key_t object,
 	off_t offset,
@@ -274,8 +331,8 @@ static void fulfill_requests(
 	off_t limit = offset + length - 1;
 	struct btree_node *cur = btree_first(&ctrl->vc_requests);
 	while (cur) {
-		struct page_request *req
+		struct vm_request *req
-			= BTREE_CONTAINER(struct page_request, req_node, cur);
+			= BTREE_CONTAINER(struct vm_request, req_node, cur);
 		spin_lock(&req->req_lock);
 		bool match = false;
 		off_t req_base = req->req_offset;
@@ -287,12 +344,14 @@ static void fulfill_requests(
 			match = true;
 		}
 
-		if (req->req_object != object) {
+		vm_object_lock(req->req_object);
+		if (req->req_object->vo_key != object) {
 			match = false;
 		}
+		vm_object_unlock(req->req_object);
 
 		if (match) {
-			req->req_status = PAGE_REQUEST_COMPLETE;
+			req->req_status = VM_REQUEST_COMPLETE;
 			req->req_result = result;
 			thread_awaken(req->req_sender);
 		}
@@ -325,14 +384,13 @@ kern_status_t vm_controller_supply_pages(
 		src_offset,
 		count,
 		NULL);
-	fulfill_requests(ctrl, dst->vo_key, dst_offset, count, status);
 
 	return status;
 }
 
 kern_status_t vm_controller_send_request(
 	struct vm_controller *ctrl,
-	struct page_request *req,
+	struct vm_request *req,
 	unsigned long *irq_flags)
 {
 	fill_random(&req->req_id, sizeof req->req_id);

vm/vm-object.c

@@ -288,8 +288,9 @@ struct vm_object *vm_object_duplicate_cow(struct vm_object *vmo)
 	struct vm_object *out = VM_OBJECT_CAST(obj);
 
 	memcpy(out->vo_name, vmo->vo_name, sizeof out->vo_name);
-	out->vo_flags = vmo->vo_flags;
+	out->vo_flags = vmo->vo_flags | VMO_LAZY_ATTACH;
 	out->vo_ctrl = vmo->vo_ctrl;
+	out->vo_key = vmo->vo_key;
 	out->vo_prot = vmo->vo_prot;
 	out->vo_size = vmo->vo_size;
 	memcpy(out->vo_name, vmo->vo_name, sizeof vmo->vo_name);
@@ -315,6 +316,37 @@ struct vm_object *vm_object_duplicate_cow(struct vm_object *vmo)
 	return out;
 }
 
+kern_status_t vm_object_attach_cow(
+	struct vm_object *vmo,
+	unsigned long *irq_flags)
+{
+	struct vm_controller *ctrl = vmo->vo_ctrl;
+	struct vm_request req = {0};
+	req.req_status = VM_REQUEST_PENDING;
+	req.req_type = VM_REQUEST_ATTACH;
+	req.req_length = vm_page_order_to_bytes(VM_PAGE_4K);
+	req.req_sender = get_current_thread();
+
+	object_ref(&vmo->vo_base);
+	req.req_object = vmo;
+
+	vm_object_unlock_irqrestore(vmo, *irq_flags);
+	vm_controller_lock_irqsave(ctrl, irq_flags);
+
+	spin_lock(&req.req_lock);
+
+	kern_status_t status
+		= vm_controller_send_request(ctrl, &req, irq_flags);
+
+	put_current_thread(req.req_sender);
+	spin_unlock(&req.req_lock);
+	vm_controller_unlock_irqrestore(ctrl, *irq_flags);
+	object_unref(&vmo->vo_base);
+	vm_object_lock_irqsave(vmo, irq_flags);
+
+	return status;
+}
+
 static struct vm_page *alloc_page(struct vm_object *vo, off_t offset)
 {
 	struct vm_page *page = NULL;
@@ -458,15 +490,15 @@ static kern_status_t request_page(
 	unsigned long *irq_flags)
 {
 	struct vm_controller *ctrl = vo->vo_ctrl;
-	struct page_request req = {0};
+	struct vm_request req = {0};
-	req.req_status = PAGE_REQUEST_PENDING;
+	req.req_status = VM_REQUEST_PENDING;
-	req.req_type = PAGE_REQUEST_READ;
+	req.req_type = VM_REQUEST_READ;
 	req.req_offset = offset;
 	req.req_length = vm_page_order_to_bytes(VM_PAGE_4K);
 	req.req_sender = get_current_thread();
 
 	object_ref(&vo->vo_base);
-	req.req_object = vo->vo_key;
+	req.req_object = vo;
 
 	vm_object_unlock_irqrestore(vo, *irq_flags);
 	vm_controller_lock_irqsave(ctrl, irq_flags);