vm: implement private and shared address space mappings

whether a mapping is private or shared determines how the mapping is handled
when a task is duplicated.

include/kernel/address-space.h

@@ -22,6 +22,8 @@ struct vm_area {
 	struct address_space *vma_space;
 	/* used to link to vm_object->vo_mappings */
 	struct queue_entry vma_object_entry;
+	/* the memory control flags applied to this area */
+	vm_flags_t vma_flags;
 	/* the memory protection flags applied to this area */
 	vm_prot_t vma_prot;
 	/* offset in bytes to the start of the object data that was mapped */
@@ -83,6 +85,7 @@ extern kern_status_t address_space_map(
 	struct vm_object *object,
 	off_t object_offset,
 	size_t length,
+	vm_flags_t flags,
 	vm_prot_t prot,
 	virt_addr_t *out);
 extern kern_status_t address_space_unmap(

include/kernel/syscall.h

@@ -128,6 +128,7 @@ extern kern_status_t sys_address_space_map(
 	kern_handle_t object,
 	off_t object_offset,
 	size_t length,
+	vm_flags_t flags,
 	vm_prot_t prot,
 	virt_addr_t *out_base_address);
 extern kern_status_t sys_address_space_unmap(

include/kernel/vm-object.h

@@ -51,7 +51,7 @@ struct vm_object {
 	struct queue vo_mappings;
 
 	struct vm_controller *vo_ctrl;
-	equeue_key_t vo_key, vo_src_key;
+	equeue_key_t vo_key;
 	struct btree_node vo_ctrl_node;
 
 	/* memory protection flags. mappings of this vm_object can only

kernel/bsp.c

@@ -120,6 +120,7 @@ static kern_status_t map_executable_exec(
 		bsp->bsp_vmo,
 		text_foffset,
 		bsp->bsp_trailer.bsp_text_size,
+		VM_SHARED,
 		VM_PROT_READ | VM_PROT_EXEC | VM_PROT_USER,
 		&text_base);
 	if (status != KERN_OK) {
@@ -132,6 +133,7 @@ static kern_status_t map_executable_exec(
 		data,
 		data_foffset,
 		bsp->bsp_trailer.bsp_data_size,
+		VM_PRIVATE,
 		VM_PROT_READ | VM_PROT_WRITE | VM_PROT_USER,
 		&data_base);
 	if (status != KERN_OK) {
@@ -165,6 +167,7 @@ kern_status_t bsp_launch_async(struct bsp *bsp, struct task *task)
 		user_stack,
 		0,
 		BOOTSTRAP_STACK_SIZE,
+		VM_PRIVATE,
 		VM_PROT_READ | VM_PROT_WRITE | VM_PROT_USER,
 		&stack_buffer);
 
@@ -178,6 +181,7 @@ kern_status_t bsp_launch_async(struct bsp *bsp, struct task *task)
 		bsp->bsp_vmo,
 		0,
 		bsp->bsp_trailer.bsp_exec_offset,
+		VM_PRIVATE,
 		VM_PROT_READ | VM_PROT_USER,
 		&bsp_data_base);
 

syscall/address-space.c

@@ -125,6 +125,7 @@ kern_status_t sys_address_space_map(
 	kern_handle_t object_handle,
 	off_t object_offset,
 	size_t length,
+	vm_flags_t flags,
 	vm_prot_t prot,
 	virt_addr_t *out_base_address)
 {
@@ -140,8 +141,8 @@ kern_status_t sys_address_space_map(
 	}
 
 	kern_status_t status = KERN_OK;
-	unsigned long flags;
-	task_lock_irqsave(self, &flags);
+	unsigned long irq_flags;
+	task_lock_irqsave(self, &irq_flags);
 
 	struct object *region_obj = NULL, *vmo_obj = NULL;
 	handle_flags_t region_flags = 0, vmo_flags = 0;
@@ -151,34 +152,34 @@ kern_status_t sys_address_space_map(
 		&region_obj,
 		&region_flags);
 	if (status != KERN_OK) {
-		task_unlock_irqrestore(self, flags);
+		task_unlock_irqrestore(self, irq_flags);
 		put_current_task(self);
 		return status;
 	}
 
 	status = task_resolve_handle(self, object_handle, &vmo_obj, &vmo_flags);
 	if (status != KERN_OK) {
-		task_unlock_irqrestore(self, flags);
+		task_unlock_irqrestore(self, irq_flags);
 		put_current_task(self);
 		return status;
 	}
 
 	struct address_space *region = address_space_cast(region_obj);
 	if (!region) {
-		task_unlock_irqrestore(self, flags);
+		task_unlock_irqrestore(self, irq_flags);
 		put_current_task(self);
 		return KERN_INVALID_ARGUMENT;
 	}
 
 	struct vm_object *vmo = vm_object_cast(vmo_obj);
 	if (!vmo) {
-		task_unlock_irqrestore(self, flags);
+		task_unlock_irqrestore(self, irq_flags);
 		put_current_task(self);
 		return KERN_INVALID_ARGUMENT;
 	}
 
-	task_unlock_irqrestore(self, flags);
-	address_space_lock_irqsave(region, &flags);
+	task_unlock_irqrestore(self, irq_flags);
+	address_space_lock_irqsave(region, &irq_flags);
 	/* address_space_map will take care of locking `vmo` */
 	status = address_space_map(
 		region,
@@ -186,9 +187,10 @@ kern_status_t sys_address_space_map(
 		vmo,
 		object_offset,
 		length,
+		flags,
 		prot,
 		out_base_address);
-	address_space_unlock_irqrestore(region, flags);
+	address_space_unlock_irqrestore(region, irq_flags);
 
 	object_unref(vmo_obj);
 	object_unref(region_obj);

syscall/vm-controller.c

@@ -148,8 +148,8 @@ kern_status_t sys_vm_controller_create_object(
 	}
 
 	kern_status_t status = KERN_OK;
-	unsigned long flags;
-	task_lock_irqsave(self, &flags);
+	unsigned long irq_flags;
+	task_lock_irqsave(self, &irq_flags);
 
 	struct object *ctrl_obj = NULL;
 	handle_flags_t handle_flags = 0;
@@ -159,7 +159,7 @@ kern_status_t sys_vm_controller_create_object(
 		&ctrl_obj,
 		&handle_flags);
 	if (status != KERN_OK) {
-		task_unlock_irqrestore(self, flags);
+		task_unlock_irqrestore(self, irq_flags);
 		put_current_task(self);
 		return status;
 	}
@@ -173,14 +173,14 @@ kern_status_t sys_vm_controller_create_object(
 		&out_handle);
 
 	struct vm_controller *ctrl = vm_controller_cast(ctrl_obj);
-	task_unlock_irqrestore(self, flags);
+	task_unlock_irqrestore(self, irq_flags);
 	if (!ctrl) {
 		object_unref(ctrl_obj);
 		put_current_task(self);
 		return KERN_INVALID_ARGUMENT;
 	}
 
-	vm_controller_lock_irqsave(ctrl, &flags);
+	vm_controller_lock_irqsave(ctrl, &irq_flags);
 	struct vm_object *out_vmo = NULL;
 	status = vm_controller_create_object(
 		ctrl,
@@ -190,14 +190,14 @@ kern_status_t sys_vm_controller_create_object(
 		data_len,
 		prot,
 		&out_vmo);
-	vm_controller_unlock_irqrestore(ctrl, flags);
+	vm_controller_unlock_irqrestore(ctrl, irq_flags);
 
 	object_unref(ctrl_obj);
 
 	if (status != KERN_OK) {
-		task_lock_irqsave(self, &flags);
+		task_lock_irqsave(self, &irq_flags);
 		handle_table_free_handle(self->t_handles, out_handle);
-		task_unlock_irqrestore(self, flags);
+		task_unlock_irqrestore(self, irq_flags);
 		put_current_task(self);
 		return status;
 	}

vm/address-space.c

@@ -706,6 +706,7 @@ kern_status_t address_space_map(
 	struct vm_object *object,
 	off_t object_offset,
 	size_t length,
+	vm_flags_t flags,
 	vm_prot_t prot,
 	virt_addr_t *out)
 {
@@ -763,6 +764,7 @@ kern_status_t address_space_map(
 	area->vma_space = root;
 	area->vma_object = object;
 	area->vma_prot = prot;
+	area->vma_flags = flags;
 	area->vma_object_offset = object_offset;
 	area->vma_base = map_address;
 	area->vma_limit = map_address + length - 1;
@@ -1215,6 +1217,7 @@ static struct vm_area *area_duplicate(struct vm_area *area)
 	}
 
 	out->vma_prot = area->vma_prot;
+	out->vma_flags = area->vma_flags;
 	out->vma_object_offset = area->vma_object_offset;
 	out->vma_base = area->vma_base;
 	out->vma_limit = area->vma_limit;
@@ -1281,7 +1284,8 @@ static kern_status_t prepare_duplicate_areas(
 		struct vm_object *src_vmo = tmp_area->vma_object;
 		vm_object_lock(src_vmo);
 
-		struct vm_object *dest_vmo = NULL;
+		struct vm_object *dest_vmo_link = NULL;
+		struct vm_object *dest_vmo_cow = NULL;
 		struct queue_entry *cur_entry
 			= queue_first(&src_vmo->vo_mappings);
 
@@ -1312,6 +1316,13 @@ static kern_status_t prepare_duplicate_areas(
 				continue;
 			}
 
+			struct vm_object *dest_vmo = NULL;
+			if (src_area->vma_flags & VM_SHARED) {
+				dest_vmo = dest_vmo_link;
+			} else {
+				dest_vmo = dest_vmo_cow;
+			}
+
 			if (!dest_vmo) {
 				tracek("[%zx-%zx %x] creating COW duplicate of "
 				       "vmo %p",
@@ -1319,7 +1330,18 @@ static kern_status_t prepare_duplicate_areas(
 				       src_area->vma_limit,
 				       src_area->vma_prot,
 				       src_vmo);
-				dest_vmo = vm_object_duplicate_cow(src_vmo);
+
+				if (src_area->vma_flags & VM_SHARED) {
+					dest_vmo_link = src_vmo;
+					object_ref(&dest_vmo_link->vo_base);
+
+					dest_vmo = dest_vmo_link;
+				} else {
+					dest_vmo_cow = vm_object_duplicate_cow(
+						src_vmo);
+					dest_vmo = dest_vmo_cow;
+				}
+
 				tracek("[%zx-%zx %x] created COW duplicate of "
 				       "vmo %p -> %p",
 				       src_area->vma_base,
@@ -1329,11 +1351,20 @@ static kern_status_t prepare_duplicate_areas(
 				       dest_vmo);
 			}
 
+			object_ref(&dest_vmo->vo_base);
 			dest_area->vma_object = dest_vmo;
 			update_area_pte_cow(src, dest, src_area);
 			cur_entry = queue_next(cur_entry);
 		}
 
+		if (dest_vmo_link) {
+			object_unref(&dest_vmo_link->vo_base);
+		}
+
+		if (dest_vmo_cow) {
+			object_unref(&dest_vmo_cow->vo_base);
+		}
+
 		vm_object_unlock(src_vmo);
 
 		cur_node = btree_next(cur_node);

vm/vm-controller.c

@@ -162,7 +162,7 @@ kern_status_t vm_controller_recv(
 		out->req_length = req->req_length;
 		break;
 	case VM_REQUEST_ATTACH:
-		out->req_src_vmo = req->req_object->vo_src_key;
+		out->req_src_vmo = req->req_object->vo_key;
 		break;
 	default:
 		break;
@@ -255,7 +255,6 @@ kern_status_t vm_controller_finish_attach(
 
 	vm_object_lock(vmo);
 	vmo->vo_key = new_key;
-	vmo->vo_src_key = 0;
 	vmo->vo_flags &= ~VMO_LAZY_ATTACH;
 	vm_object_unlock(vmo);
 

vm/vm-object.c

@@ -290,8 +290,7 @@ struct vm_object *vm_object_duplicate_cow(struct vm_object *vmo)
 	memcpy(out->vo_name, vmo->vo_name, sizeof out->vo_name);
 	out->vo_flags = vmo->vo_flags | VMO_LAZY_ATTACH;
 	out->vo_ctrl = vmo->vo_ctrl;
-	out->vo_key = 0;
-	out->vo_src_key = vmo->vo_key;
+	out->vo_key = vmo->vo_key;
 	out->vo_prot = vmo->vo_prot;
 	out->vo_size = vmo->vo_size;
 	memcpy(out->vo_name, vmo->vo_name, sizeof vmo->vo_name);