vm: implement lazy-attach cow-duplication of vm-objects attached to a controller

include/kernel/syscall.h

@@ -211,7 +211,7 @@ extern kern_status_t sys_kern_object_query(
 extern kern_status_t sys_vm_controller_create(kern_handle_t *out);
 extern kern_status_t sys_vm_controller_recv(
 	kern_handle_t ctrl,
-	equeue_packet_page_request_t *out);
+	equeue_packet_vm_request_t *out);
 extern kern_status_t sys_vm_controller_recv_async(
 	kern_handle_t ctrl,
 	kern_handle_t eq,
@@ -224,6 +224,14 @@ extern kern_status_t sys_vm_controller_create_object(
 	size_t data_len,
 	vm_prot_t prot,
 	kern_handle_t *out);
+extern kern_status_t sys_vm_controller_prepare_attach(
+	kern_handle_t ctrl,
+	uint64_t req_id,
+	kern_handle_t *out_vmo);
+extern kern_status_t sys_vm_controller_finish_attach(
+	kern_handle_t ctrl,
+	uint64_t req_id,
+	equeue_key_t new_key);
 extern kern_status_t sys_vm_controller_detach_object(
 	kern_handle_t ctrl,
 	kern_handle_t vmo);

include/kernel/vm-controller.h

@@ -9,34 +9,35 @@ struct thread;
 struct equeue;
 struct vm_object;
 
-enum page_request_status {
-	PAGE_REQUEST_PENDING = 0,
-	PAGE_REQUEST_IN_PROGRESS,
-	PAGE_REQUEST_COMPLETE,
-	PAGE_REQUEST_ASYNC,
+enum vm_request_status {
+	VM_REQUEST_PENDING = 0,
+	VM_REQUEST_IN_PROGRESS,
+	VM_REQUEST_COMPLETE,
+	VM_REQUEST_ASYNC,
 };
 
 struct vm_controller {
 	struct object vc_base;
-	/* tree of pending page requests */
+	/* tree of pending vm requests */
 	struct btree vc_requests;
-	/* the equeue to send async page requests to */
 	struct equeue *vc_eq;
 	equeue_key_t vc_eq_key;
-	/* the number of page requests queued with status PAGE_REQUEST_PENDING.
+	/* the number of page requests queued with status VM_REQUEST_PENDING.
 	 * used to assert/clear VM_CONTROLLER_SIGNAL_REQUEST_RECEIVED */
 	size_t vc_requests_waiting;
 };
 
-struct page_request {
+struct vm_request {
 	uint64_t req_id;
 	unsigned int req_type;
-	enum page_request_status req_status;
+	enum vm_request_status req_status;
 	kern_status_t req_result;
 	spin_lock_t req_lock;
-	equeue_key_t req_object;
+	struct vm_object *req_object;
 	struct thread *req_sender;
+	/* this node is added to vm-controller vc_requests list */
 	struct btree_node req_node;
+	/* these values are used for VM_REQUEST_READ and VM_REQUEST_DIRTY */
 	off_t req_offset;
 	size_t req_length;
 };
@@ -48,7 +49,7 @@ extern struct vm_controller *vm_controller_create(void);
 
 extern kern_status_t vm_controller_recv(
 	struct vm_controller *ctrl,
-	equeue_packet_page_request_t *out);
+	equeue_packet_vm_request_t *out);
 extern kern_status_t vm_controller_recv_async(
 	struct vm_controller *ctrl,
 	struct equeue *eq,
@@ -62,6 +63,14 @@ extern kern_status_t vm_controller_create_object(
 	size_t data_len,
 	vm_prot_t prot,
 	struct vm_object **out);
+extern kern_status_t vm_controller_prepare_attach(
+	struct vm_controller *ctrl,
+	uint64_t req_id,
+	struct vm_object **out_vmo);
+extern kern_status_t vm_controller_finish_attach(
+	struct vm_controller *ctrl,
+	uint64_t req_id,
+	equeue_key_t new_key);
 extern kern_status_t vm_controller_detach_object(
 	struct vm_controller *ctrl,
 	struct vm_object *vmo);
@@ -72,10 +81,16 @@ extern kern_status_t vm_controller_supply_pages(
 	struct vm_object *src,
 	off_t src_offset,
 	size_t count);
+extern void vm_controller_fulfill_requests(
+	struct vm_controller *ctrl,
+	equeue_key_t object,
+	off_t offset,
+	size_t length,
+	kern_status_t result);
 
 extern kern_status_t vm_controller_send_request(
 	struct vm_controller *ctrl,
-	struct page_request *req,
+	struct vm_request *req,
 	unsigned long *irq_flags);
 
 DEFINE_OBJECT_LOCK_FUNCTION(vm_controller, vc_base)

include/kernel/vm-object.h

@@ -20,19 +20,25 @@ enum vm_object_flags {
 	 * be detached, allowing the server to close the last handle to the
 	 * object and dispose of it. */
 	VMO_AUTO_DETACH = 0x04u,
+	/* this vmo is a duplicate of a vmo that is attached to a vm-controller.
+	 * the duplicate vmo is scheduled to be attached to the same controller,
+	 * but this won't actually happen until the controller is needed to
+	 * fulfill a page request. once the duplicate vmo has been attached to
+	 * the controller, this flag will be cleared. */
+	VMO_LAZY_ATTACH = 0x08u,
 
 	/* these flags are for use with vm_object_get_page */
 	/**************************************************/
 
 	/* if the relevant page hasn't been allocated yet, it will be allocated
 	 * and returned. if this flag isn't specified, NULL will be returned. */
-	VMO_ALLOCATE_MISSING_PAGE = 0x08u,
+	VMO_ALLOCATE_MISSING_PAGE = 0x0100u,
 	/* if the vm-object is attached to a vm-controller, and the relevant
 	 * page is uncommitted, send a request to the vm-controller to provide
 	 * the missing page. will result in the vm-object being unlocked and
 	 * the current thread sleeping until the request is fulfilled. the
 	 * vm-object will be re-locked before the function returns. */
-	VMO_REQUEST_MISSING_PAGE = 0x10u,
+	VMO_REQUEST_MISSING_PAGE = 0x0200u,
 };
 
 struct vm_object {
@@ -45,7 +51,7 @@ struct vm_object {
 	struct queue vo_mappings;
 
 	struct vm_controller *vo_ctrl;
-	equeue_key_t vo_key;
+	equeue_key_t vo_key, vo_src_key;
 	struct btree_node vo_ctrl_node;
 
 	/* memory protection flags. mappings of this vm_object can only
@@ -86,6 +92,11 @@ extern struct vm_object *vm_object_create_in_place(
 
 /* create a copy-on-write duplicate of a vm-object */
 extern struct vm_object *vm_object_duplicate_cow(struct vm_object *vmo);
+/* attach a copy-on-write duplicate of a vm-object to the vm-controller that
+ * controlled the original vm-object */
+extern kern_status_t vm_object_attach_cow(
+	struct vm_object *vmo,
+	unsigned long *irq_flags);
 
 extern struct vm_page *vm_object_get_page(
 	struct vm_object *vo,