diff --git a/include/kernel/syscall.h b/include/kernel/syscall.h index 3dde6f5..4152e64 100644 --- a/include/kernel/syscall.h +++ b/include/kernel/syscall.h @@ -211,7 +211,7 @@ extern kern_status_t sys_kern_object_query( extern kern_status_t sys_vm_controller_create(kern_handle_t *out); extern kern_status_t sys_vm_controller_recv( kern_handle_t ctrl, - equeue_packet_page_request_t *out); + equeue_packet_vm_request_t *out); extern kern_status_t sys_vm_controller_recv_async( kern_handle_t ctrl, kern_handle_t eq, @@ -224,6 +224,14 @@ extern kern_status_t sys_vm_controller_create_object( size_t data_len, vm_prot_t prot, kern_handle_t *out); +extern kern_status_t sys_vm_controller_prepare_attach( + kern_handle_t ctrl, + uint64_t req_id, + kern_handle_t *out_vmo); +extern kern_status_t sys_vm_controller_finish_attach( + kern_handle_t ctrl, + uint64_t req_id, + equeue_key_t new_key); extern kern_status_t sys_vm_controller_detach_object( kern_handle_t ctrl, kern_handle_t vmo); diff --git a/include/kernel/vm-controller.h b/include/kernel/vm-controller.h index 65e3b2a..23d9ceb 100644 --- a/include/kernel/vm-controller.h +++ b/include/kernel/vm-controller.h @@ -9,34 +9,35 @@ struct thread; struct equeue; struct vm_object; -enum page_request_status { - PAGE_REQUEST_PENDING = 0, - PAGE_REQUEST_IN_PROGRESS, - PAGE_REQUEST_COMPLETE, - PAGE_REQUEST_ASYNC, +enum vm_request_status { + VM_REQUEST_PENDING = 0, + VM_REQUEST_IN_PROGRESS, + VM_REQUEST_COMPLETE, + VM_REQUEST_ASYNC, }; struct vm_controller { struct object vc_base; - /* tree of pending page requests */ + /* tree of pending vm requests */ struct btree vc_requests; - /* the equeue to send async page requests to */ struct equeue *vc_eq; equeue_key_t vc_eq_key; - /* the number of page requests queued with status PAGE_REQUEST_PENDING. + /* the number of page requests queued with status VM_REQUEST_PENDING. * used to assert/clear VM_CONTROLLER_SIGNAL_REQUEST_RECEIVED */ size_t vc_requests_waiting; }; -struct page_request { +struct vm_request { uint64_t req_id; unsigned int req_type; - enum page_request_status req_status; + enum vm_request_status req_status; kern_status_t req_result; spin_lock_t req_lock; - equeue_key_t req_object; + struct vm_object *req_object; struct thread *req_sender; + /* this node is added to vm-controller vc_requests list */ struct btree_node req_node; + /* these values are used for VM_REQUEST_READ and VM_REQUEST_DIRTY */ off_t req_offset; size_t req_length; }; @@ -48,7 +49,7 @@ extern struct vm_controller *vm_controller_create(void); extern kern_status_t vm_controller_recv( struct vm_controller *ctrl, - equeue_packet_page_request_t *out); + equeue_packet_vm_request_t *out); extern kern_status_t vm_controller_recv_async( struct vm_controller *ctrl, struct equeue *eq, @@ -62,6 +63,14 @@ extern kern_status_t vm_controller_create_object( size_t data_len, vm_prot_t prot, struct vm_object **out); +extern kern_status_t vm_controller_prepare_attach( + struct vm_controller *ctrl, + uint64_t req_id, + struct vm_object **out_vmo); +extern kern_status_t vm_controller_finish_attach( + struct vm_controller *ctrl, + uint64_t req_id, + equeue_key_t new_key); extern kern_status_t vm_controller_detach_object( struct vm_controller *ctrl, struct vm_object *vmo); @@ -72,10 +81,16 @@ extern kern_status_t vm_controller_supply_pages( struct vm_object *src, off_t src_offset, size_t count); +extern void vm_controller_fulfill_requests( + struct vm_controller *ctrl, + equeue_key_t object, + off_t offset, + size_t length, + kern_status_t result); extern kern_status_t vm_controller_send_request( struct vm_controller *ctrl, - struct page_request *req, + struct vm_request *req, unsigned long *irq_flags); DEFINE_OBJECT_LOCK_FUNCTION(vm_controller, vc_base) diff --git a/include/kernel/vm-object.h b/include/kernel/vm-object.h index d921a0e..d5a43c3 100644 --- a/include/kernel/vm-object.h +++ b/include/kernel/vm-object.h @@ -20,19 +20,25 @@ enum vm_object_flags { * be detached, allowing the server to close the last handle to the * object and dispose of it. */ VMO_AUTO_DETACH = 0x04u, + /* this vmo is a duplicate of a vmo that is attached to a vm-controller. + * the duplicate vmo is scheduled to be attached to the same controller, + * but this won't actually happen until the controller is needed to + * fulfill a page request. once the duplicate vmo has been attached to + * the controller, this flag will be cleared. */ + VMO_LAZY_ATTACH = 0x08u, /* these flags are for use with vm_object_get_page */ /**************************************************/ /* if the relevant page hasn't been allocated yet, it will be allocated * and returned. if this flag isn't specified, NULL will be returned. */ - VMO_ALLOCATE_MISSING_PAGE = 0x08u, + VMO_ALLOCATE_MISSING_PAGE = 0x0100u, /* if the vm-object is attached to a vm-controller, and the relevant * page is uncommitted, send a request to the vm-controller to provide * the missing page. will result in the vm-object being unlocked and * the current thread sleeping until the request is fulfilled. the * vm-object will be re-locked before the function returns. */ - VMO_REQUEST_MISSING_PAGE = 0x10u, + VMO_REQUEST_MISSING_PAGE = 0x0200u, }; struct vm_object { @@ -45,7 +51,7 @@ struct vm_object { struct queue vo_mappings; struct vm_controller *vo_ctrl; - equeue_key_t vo_key; + equeue_key_t vo_key, vo_src_key; struct btree_node vo_ctrl_node; /* memory protection flags. mappings of this vm_object can only @@ -86,6 +92,11 @@ extern struct vm_object *vm_object_create_in_place( /* create a copy-on-write duplicate of a vm-object */ extern struct vm_object *vm_object_duplicate_cow(struct vm_object *vmo); +/* attach a copy-on-write duplicate of a vm-object to the vm-controller that + * controlled the original vm-object */ +extern kern_status_t vm_object_attach_cow( + struct vm_object *vmo, + unsigned long *irq_flags); extern struct vm_page *vm_object_get_page( struct vm_object *vo, diff --git a/syscall/dispatch.c b/syscall/dispatch.c index fcb838c..6261c5e 100644 --- a/syscall/dispatch.c +++ b/syscall/dispatch.c @@ -47,6 +47,12 @@ static const virt_addr_t syscall_table[] = { SYSCALL_TABLE_ENTRY( VM_CONTROLLER_CREATE_OBJECT, vm_controller_create_object), + SYSCALL_TABLE_ENTRY( + VM_CONTROLLER_PREPARE_ATTACH, + vm_controller_prepare_attach), + SYSCALL_TABLE_ENTRY( + VM_CONTROLLER_FINISH_ATTACH, + vm_controller_finish_attach), SYSCALL_TABLE_ENTRY( VM_CONTROLLER_DETACH_OBJECT, vm_controller_detach_object), diff --git a/syscall/task.c b/syscall/task.c index 421ef84..e164d44 100644 --- a/syscall/task.c +++ b/syscall/task.c @@ -506,8 +506,9 @@ kern_status_t sys_task_duplicate( struct task *new_task = task_create( self->t_name, strlen(self->t_name), - NULL); + child_handle_table); if (!new_task) { + handle_table_destroy(child_handle_table); put_current_task(self); return KERN_NO_MEMORY; } diff --git a/syscall/vm-controller.c b/syscall/vm-controller.c index 5c3ddf2..546daba 100644 --- a/syscall/vm-controller.c +++ b/syscall/vm-controller.c @@ -32,7 +32,7 @@ kern_status_t sys_vm_controller_create(kern_handle_t *out) kern_status_t sys_vm_controller_recv( kern_handle_t ctrl_handle, - equeue_packet_page_request_t *out) + equeue_packet_vm_request_t *out) { struct task *self = get_current_task(); @@ -209,6 +209,116 @@ kern_status_t sys_vm_controller_create_object( return KERN_OK; } +kern_status_t sys_vm_controller_prepare_attach( + kern_handle_t ctrl_handle, + uint64_t req_id, + kern_handle_t *out_vmo) +{ + struct task *self = get_current_task(); + if (!out_vmo || !validate_access_w(self, out_vmo, sizeof *out_vmo)) { + return KERN_MEMORY_FAULT; + } + + kern_status_t status = KERN_OK; + unsigned long flags; + task_lock_irqsave(self, &flags); + + struct object *ctrl_obj = NULL; + handle_flags_t ctrl_flags = 0; + status = task_resolve_handle(self, ctrl_handle, &ctrl_obj, &ctrl_flags); + if (status != KERN_OK) { + task_unlock_irqrestore(self, flags); + put_current_task(self); + return status; + } + + struct handle *out_slot = NULL; + kern_handle_t out_handle = KERN_HANDLE_INVALID; + status = handle_table_alloc_handle( + self->t_handles, + KERN_HANDLE_INVALID, + &out_slot, + &out_handle); + if (status != KERN_OK) { + object_unref(ctrl_obj); + task_unlock_irqrestore(self, flags); + put_current_task(self); + return status; + } + + struct vm_controller *ctrl = vm_controller_cast(ctrl_obj); + task_unlock_irqrestore(self, flags); + + if (!ctrl) { + object_unref(ctrl_obj); + return KERN_INVALID_ARGUMENT; + } + + vm_controller_lock_irqsave(ctrl, &flags); + + struct vm_object *vmo = NULL; + status = vm_controller_prepare_attach(ctrl, req_id, &vmo); + vm_controller_unlock_irqrestore(ctrl, flags); + + object_unref(ctrl_obj); + if (status != KERN_OK) { + task_lock_irqsave(self, &flags); + handle_table_free_handle(self->t_handles, out_handle); + task_unlock_irqrestore(self, flags); + + return status; + } + + out_slot->h_object = &vmo->vo_base; + put_current_task(self); + + *out_vmo = out_handle; + + return KERN_OK; +} + +kern_status_t sys_vm_controller_finish_attach( + kern_handle_t ctrl_handle, + uint64_t req_id, + equeue_key_t new_key) +{ + struct task *self = get_current_task(); + + kern_status_t status = KERN_OK; + unsigned long flags; + task_lock_irqsave(self, &flags); + + struct object *ctrl_obj = NULL; + handle_flags_t ctrl_flags = 0; + status = task_resolve_handle(self, ctrl_handle, &ctrl_obj, &ctrl_flags); + if (status != KERN_OK) { + task_unlock_irqrestore(self, flags); + put_current_task(self); + return status; + } + + struct vm_controller *ctrl = vm_controller_cast(ctrl_obj); + task_unlock_irqrestore(self, flags); + put_current_task(self); + + if (!ctrl) { + object_unref(ctrl_obj); + return KERN_INVALID_ARGUMENT; + } + + vm_controller_lock_irqsave(ctrl, &flags); + + status = vm_controller_finish_attach(ctrl, req_id, new_key); + vm_controller_unlock_irqrestore(ctrl, flags); + + object_unref(ctrl_obj); + if (status != KERN_OK) { + return status; + } + + return KERN_OK; +} + kern_status_t sys_vm_controller_detach_object( kern_handle_t ctrl_handle, kern_handle_t vmo_handle) @@ -315,6 +425,8 @@ kern_status_t sys_vm_controller_supply_pages( vm_controller_lock_irqsave(ctrl, &flags); vm_object_lock_pair(src, dst); + equeue_key_t requester_key = dst->vo_key; + status = vm_controller_supply_pages( ctrl, dst, @@ -323,6 +435,12 @@ kern_status_t sys_vm_controller_supply_pages( src_offset, count); vm_object_unlock_pair(src, dst); + vm_controller_fulfill_requests( + ctrl, + requester_key, + dst_offset, + count, + status); vm_controller_unlock_irqrestore(ctrl, flags); object_unref(ctrl_obj); diff --git a/vm/address-space.c b/vm/address-space.c index 5af9041..044618f 100644 --- a/vm/address-space.c +++ b/vm/address-space.c @@ -6,6 +6,7 @@ #include #include #include +#include #include #include #include @@ -1502,6 +1503,17 @@ static kern_status_t request_missing_page( vm_object_lock(object); address_space_unlock(region); + kern_status_t status = KERN_OK; + + if (object->vo_flags & VMO_LAZY_ATTACH) { + status = vm_object_attach_cow(object, irq_flags); + } + + if (status != KERN_OK) { + vm_object_unlock_irqrestore(object, *irq_flags); + return status; + } + struct vm_page *pg = vm_object_get_page( object, object_offset, @@ -1515,7 +1527,7 @@ static kern_status_t request_missing_page( /* now: `region` is unlocked, and `object` is locked */ - kern_status_t status = pmap_add( + status = pmap_add( region->s_pmap, addr, vm_page_get_pfn(pg), @@ -1550,9 +1562,6 @@ static kern_status_t handle_cow_access( } tracek("cow access %zx", addr); - if (area->vma_object->vo_ctrl) { - panic("COW on controlled vm-object"); - } off_t object_offset = addr - area->vma_base + area->vma_object_offset; vm_object_lock(area->vma_object); @@ -1663,7 +1672,7 @@ kern_status_t address_space_demand_map( address_space_lock_irqsave(region, &irq_flags); const enum pmap_fault_flags cow_flags - = PMAP_FAULT_WRITE | PMAP_FAULT_PRESENT | PMAP_FAULT_USER; + = PMAP_FAULT_WRITE | PMAP_FAULT_PRESENT; if ((flags & cow_flags) == cow_flags) { return handle_cow_access(region, addr, flags, &irq_flags); diff --git a/vm/vm-controller.c b/vm/vm-controller.c index 12ce88b..718dffa 100644 --- a/vm/vm-controller.c +++ b/vm/vm-controller.c @@ -9,13 +9,12 @@ #define VM_CONTROLLER_CAST(p) \ OBJECT_C_CAST(struct vm_controller, vc_base, &vm_controller_type, p) -BTREE_DEFINE_SIMPLE_INSERT(struct vm_object, vo_ctrl_node, vo_key, put_object) BTREE_DEFINE_SIMPLE_GET( - struct vm_object, - equeue_key_t, - vo_ctrl_node, - vo_key, - get_object) + struct vm_request, + uint64_t, + req_node, + req_id, + get_request) static struct object_type vm_controller_type = { .ob_name = "vm-controller", @@ -23,14 +22,14 @@ static struct object_type vm_controller_type = { .ob_header_offset = offsetof(struct vm_controller, vc_base), }; -static struct vm_cache page_request_cache = { - .c_name = "page-request", - .c_obj_size = sizeof(struct page_request), +static struct vm_cache vm_request_cache = { + .c_name = "vm-request", + .c_obj_size = sizeof(struct vm_request), }; kern_status_t vm_controller_type_init(void) { - vm_cache_init(&page_request_cache); + vm_cache_init(&vm_request_cache); return object_type_register(&vm_controller_type); } @@ -51,19 +50,19 @@ struct vm_controller *vm_controller_create(void) return ctrl; } -static struct page_request *get_next_request(struct vm_controller *ctrl) +static struct vm_request *get_next_request(struct vm_controller *ctrl) { struct btree_node *cur = btree_first(&ctrl->vc_requests); while (cur) { - struct page_request *req - = BTREE_CONTAINER(struct page_request, req_node, cur); + struct vm_request *req + = BTREE_CONTAINER(struct vm_request, req_node, cur); spin_lock(&req->req_lock); switch (req->req_status) { - case PAGE_REQUEST_PENDING: - req->req_status = PAGE_REQUEST_IN_PROGRESS; + case VM_REQUEST_PENDING: + req->req_status = VM_REQUEST_IN_PROGRESS; ctrl->vc_requests_waiting--; return req; - case PAGE_REQUEST_ASYNC: + case VM_REQUEST_ASYNC: btree_delete(&ctrl->vc_requests, &req->req_node); ctrl->vc_requests_waiting--; return req; @@ -78,7 +77,7 @@ static struct page_request *get_next_request(struct vm_controller *ctrl) return NULL; } -static kern_status_t try_enqueue(struct btree *tree, struct page_request *req) +static kern_status_t try_enqueue(struct btree *tree, struct vm_request *req) { if (!tree->b_root) { tree->b_root = &req->req_node; @@ -88,8 +87,8 @@ static kern_status_t try_enqueue(struct btree *tree, struct page_request *req) struct btree_node *cur = tree->b_root; while (1) { - struct page_request *cur_node - = BTREE_CONTAINER(struct page_request, req_node, cur); + struct vm_request *cur_node + = BTREE_CONTAINER(struct vm_request, req_node, cur); struct btree_node *next = NULL; if (req->req_id > cur_node->req_id) { @@ -119,7 +118,7 @@ static kern_status_t try_enqueue(struct btree *tree, struct page_request *req) static kern_status_t send_request_async( struct vm_controller *ctrl, - struct page_request *req) + struct vm_request *req) { fill_random(&req->req_id, sizeof req->req_id); while (!try_enqueue(&ctrl->vc_requests, req)) { @@ -136,9 +135,9 @@ static kern_status_t send_request_async( kern_status_t vm_controller_recv( struct vm_controller *ctrl, - equeue_packet_page_request_t *out) + equeue_packet_vm_request_t *out) { - struct page_request *req = NULL; + struct vm_request *req = NULL; req = get_next_request(ctrl); if (!req) { @@ -151,16 +150,30 @@ kern_status_t vm_controller_recv( VM_CONTROLLER_SIGNAL_REQUEST_RECEIVED); } - out->req_vmo = req->req_object; + vm_object_lock(req->req_object); + out->req_id = req->req_id; + out->req_vmo = req->req_object->vo_key; out->req_type = req->req_type; - out->req_offset = req->req_offset; - out->req_length = req->req_length; + switch (req->req_type) { + case VM_REQUEST_READ: + case VM_REQUEST_DIRTY: + out->req_offset = req->req_offset; + out->req_length = req->req_length; + break; + case VM_REQUEST_ATTACH: + out->req_src_vmo = req->req_object->vo_src_key; + break; + default: + break; + } + + vm_object_unlock(req->req_object); spin_unlock(&req->req_lock); - if (req->req_status == PAGE_REQUEST_ASYNC) { + if (req->req_status == VM_REQUEST_ASYNC) { put_current_thread(req->req_sender); - vm_cache_free(&page_request_cache, req); + vm_cache_free(&vm_request_cache, req); } return KERN_OK; @@ -208,6 +221,53 @@ kern_status_t vm_controller_create_object( return KERN_OK; } +kern_status_t vm_controller_prepare_attach( + struct vm_controller *ctrl, + uint64_t req_id, + struct vm_object **out_vmo) +{ + struct vm_request *req = get_request(&ctrl->vc_requests, req_id); + if (!req) { + return KERN_INVALID_ARGUMENT; + } + + spin_lock(&req->req_lock); + req->req_status = VM_REQUEST_IN_PROGRESS; + *out_vmo = req->req_object; + spin_unlock(&req->req_lock); + + return KERN_OK; +} + +kern_status_t vm_controller_finish_attach( + struct vm_controller *ctrl, + uint64_t req_id, + equeue_key_t new_key) +{ + struct vm_request *req = get_request(&ctrl->vc_requests, req_id); + if (!req) { + return KERN_INVALID_ARGUMENT; + } + + spin_lock(&req->req_lock); + struct vm_object *vmo = req->req_object; + spin_unlock(&req->req_lock); + + vm_object_lock(vmo); + vmo->vo_key = new_key; + vmo->vo_src_key = 0; + vmo->vo_flags &= ~VMO_LAZY_ATTACH; + vm_object_unlock(vmo); + + spin_lock(&req->req_lock); + req->req_status = VM_REQUEST_COMPLETE; + req->req_result = KERN_OK; + thread_awaken(req->req_sender); + spin_unlock(&req->req_lock); + + return KERN_OK; +} + kern_status_t vm_controller_detach_object( struct vm_controller *ctrl, struct vm_object *vmo) @@ -216,7 +276,7 @@ kern_status_t vm_controller_detach_object( return KERN_INVALID_ARGUMENT; } - if (vmo->vo_key == 0) { + if (vmo->vo_flags & VMO_LAZY_ATTACH) { /* this vmo isn't actually attached to this controller yet. * this can happen if a controller-attached vmo was duplicated * via copy-on-write, and the duplicate vmo has not yet been @@ -225,16 +285,14 @@ kern_status_t vm_controller_detach_object( return KERN_OK; } - struct page_request *req - = vm_cache_alloc(&page_request_cache, VM_NORMAL); - req->req_type = PAGE_REQUEST_DETACH; - req->req_status = PAGE_REQUEST_ASYNC; - req->req_object = vmo->vo_key; + struct vm_request *req = vm_cache_alloc(&vm_request_cache, VM_NORMAL); + req->req_type = VM_REQUEST_DETACH; + req->req_status = VM_REQUEST_ASYNC; + req->req_object = vmo; req->req_sender = get_current_thread(); send_request_async(ctrl, req); vmo->vo_ctrl = NULL; - vmo->vo_key = 0; object_unref(&ctrl->vc_base); return KERN_OK; @@ -242,7 +300,7 @@ kern_status_t vm_controller_detach_object( static void wait_for_reply( struct vm_controller *ctrl, - struct page_request *req, + struct vm_request *req, unsigned long *lock_flags) { struct wait_item waiter; @@ -251,7 +309,7 @@ static void wait_for_reply( wait_item_init(&waiter, self); for (;;) { self->tr_state = THREAD_SLEEPING; - if (req->req_status == PAGE_REQUEST_COMPLETE) { + if (req->req_status == VM_REQUEST_COMPLETE) { break; } @@ -264,7 +322,7 @@ static void wait_for_reply( put_current_thread(self); } -static void fulfill_requests( +void vm_controller_fulfill_requests( struct vm_controller *ctrl, equeue_key_t object, off_t offset, @@ -274,8 +332,8 @@ static void fulfill_requests( off_t limit = offset + length - 1; struct btree_node *cur = btree_first(&ctrl->vc_requests); while (cur) { - struct page_request *req - = BTREE_CONTAINER(struct page_request, req_node, cur); + struct vm_request *req + = BTREE_CONTAINER(struct vm_request, req_node, cur); spin_lock(&req->req_lock); bool match = false; off_t req_base = req->req_offset; @@ -287,12 +345,14 @@ static void fulfill_requests( match = true; } - if (req->req_object != object) { + vm_object_lock(req->req_object); + if (req->req_object->vo_key != object) { match = false; } + vm_object_unlock(req->req_object); if (match) { - req->req_status = PAGE_REQUEST_COMPLETE; + req->req_status = VM_REQUEST_COMPLETE; req->req_result = result; thread_awaken(req->req_sender); } @@ -325,14 +385,13 @@ kern_status_t vm_controller_supply_pages( src_offset, count, NULL); - fulfill_requests(ctrl, dst->vo_key, dst_offset, count, status); return status; } kern_status_t vm_controller_send_request( struct vm_controller *ctrl, - struct page_request *req, + struct vm_request *req, unsigned long *irq_flags) { fill_random(&req->req_id, sizeof req->req_id); diff --git a/vm/vm-object.c b/vm/vm-object.c index 4b7e0df..4a9745f 100644 --- a/vm/vm-object.c +++ b/vm/vm-object.c @@ -288,8 +288,10 @@ struct vm_object *vm_object_duplicate_cow(struct vm_object *vmo) struct vm_object *out = VM_OBJECT_CAST(obj); memcpy(out->vo_name, vmo->vo_name, sizeof out->vo_name); - out->vo_flags = vmo->vo_flags; + out->vo_flags = vmo->vo_flags | VMO_LAZY_ATTACH; out->vo_ctrl = vmo->vo_ctrl; + out->vo_key = 0; + out->vo_src_key = vmo->vo_key; out->vo_prot = vmo->vo_prot; out->vo_size = vmo->vo_size; memcpy(out->vo_name, vmo->vo_name, sizeof vmo->vo_name); @@ -315,6 +317,37 @@ struct vm_object *vm_object_duplicate_cow(struct vm_object *vmo) return out; } +kern_status_t vm_object_attach_cow( + struct vm_object *vmo, + unsigned long *irq_flags) +{ + struct vm_controller *ctrl = vmo->vo_ctrl; + struct vm_request req = {0}; + req.req_status = VM_REQUEST_PENDING; + req.req_type = VM_REQUEST_ATTACH; + req.req_length = vm_page_order_to_bytes(VM_PAGE_4K); + req.req_sender = get_current_thread(); + + object_ref(&vmo->vo_base); + req.req_object = vmo; + + vm_object_unlock_irqrestore(vmo, *irq_flags); + vm_controller_lock_irqsave(ctrl, irq_flags); + + spin_lock(&req.req_lock); + + kern_status_t status + = vm_controller_send_request(ctrl, &req, irq_flags); + + put_current_thread(req.req_sender); + spin_unlock(&req.req_lock); + vm_controller_unlock_irqrestore(ctrl, *irq_flags); + object_unref(&vmo->vo_base); + vm_object_lock_irqsave(vmo, irq_flags); + + return status; +} + static struct vm_page *alloc_page(struct vm_object *vo, off_t offset) { struct vm_page *page = NULL; @@ -458,15 +491,15 @@ static kern_status_t request_page( unsigned long *irq_flags) { struct vm_controller *ctrl = vo->vo_ctrl; - struct page_request req = {0}; - req.req_status = PAGE_REQUEST_PENDING; - req.req_type = PAGE_REQUEST_READ; + struct vm_request req = {0}; + req.req_status = VM_REQUEST_PENDING; + req.req_type = VM_REQUEST_READ; req.req_offset = offset; req.req_length = vm_page_order_to_bytes(VM_PAGE_4K); req.req_sender = get_current_thread(); object_ref(&vo->vo_base); - req.req_object = vo->vo_key; + req.req_object = vo; vm_object_unlock_irqrestore(vo, *irq_flags); vm_controller_lock_irqsave(ctrl, irq_flags);