wash/magenta
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

vm: implement lazy-attach cow-duplication of vm-objects attached to a controller

Browse Source
This commit is contained in:
max 🍓
2026-04-19 20:16:19 +01:00
parent b3be4c541b
commit 4a9e907a75
9 changed files with 332 additions and 72 deletions
Download Patch File Download Diff File Expand all files Collapse all files
vm/address-space.c
+14 -5
View File
@@ -6,6 +6,7 @@
#include <kernel/printk.h>
#include <kernel/sched.h>
#include <kernel/task.h>
#include <kernel/thread.h>
#include <kernel/util.h>
#include <kernel/vm-controller.h>
#include <kernel/vm-object.h>
@@ -1502,6 +1503,17 @@ static kern_status_t request_missing_page(
vm_object_lock(object);
address_space_unlock(region);
kern_status_t status = KERN_OK;
if (object->vo_flags & VMO_LAZY_ATTACH) {
status = vm_object_attach_cow(object, irq_flags);
}
if (status != KERN_OK) {
vm_object_unlock_irqrestore(object, *irq_flags);
return status;
}
struct vm_page *pg = vm_object_get_page(
object,
object_offset,
@@ -1515,7 +1527,7 @@ static kern_status_t request_missing_page(
/* now: `region` is unlocked, and `object` is locked */
kern_status_t status = pmap_add(
status = pmap_add(
region->s_pmap,
addr,
vm_page_get_pfn(pg),
@@ -1550,9 +1562,6 @@ static kern_status_t handle_cow_access(
}
tracek("cow access %zx", addr);
if (area->vma_object->vo_ctrl) {
panic("COW on controlled vm-object");
}
off_t object_offset = addr - area->vma_base + area->vma_object_offset;
vm_object_lock(area->vma_object);
@@ -1663,7 +1672,7 @@ kern_status_t address_space_demand_map(
address_space_lock_irqsave(region, &irq_flags);
const enum pmap_fault_flags cow_flags
= PMAP_FAULT_WRITE | PMAP_FAULT_PRESENT | PMAP_FAULT_USER;
= PMAP_FAULT_WRITE | PMAP_FAULT_PRESENT;
if ((flags & cow_flags) == cow_flags) {
return handle_cow_access(region, addr, flags, &irq_flags);
vm/vm-controller.c
+102 -43
View File
@@ -9,13 +9,12 @@
#define VM_CONTROLLER_CAST(p) \
OBJECT_C_CAST(struct vm_controller, vc_base, &vm_controller_type, p)
BTREE_DEFINE_SIMPLE_INSERT(struct vm_object, vo_ctrl_node, vo_key, put_object)
BTREE_DEFINE_SIMPLE_GET(
struct vm_object,
equeue_key_t,
vo_ctrl_node,
vo_key,
get_object)
struct vm_request,
uint64_t,
req_node,
req_id,
get_request)
static struct object_type vm_controller_type = {
.ob_name = "vm-controller",
@@ -23,14 +22,14 @@ static struct object_type vm_controller_type = {
.ob_header_offset = offsetof(struct vm_controller, vc_base),
};
static struct vm_cache page_request_cache = {
.c_name = "page-request",
.c_obj_size = sizeof(struct page_request),
static struct vm_cache vm_request_cache = {
.c_name = "vm-request",
.c_obj_size = sizeof(struct vm_request),
};
kern_status_t vm_controller_type_init(void)
{
vm_cache_init(&page_request_cache);
vm_cache_init(&vm_request_cache);
return object_type_register(&vm_controller_type);
}
@@ -51,19 +50,19 @@ struct vm_controller *vm_controller_create(void)
return ctrl;
}
static struct page_request *get_next_request(struct vm_controller *ctrl)
static struct vm_request *get_next_request(struct vm_controller *ctrl)
{
struct btree_node *cur = btree_first(&ctrl->vc_requests);
while (cur) {
struct page_request *req
= BTREE_CONTAINER(struct page_request, req_node, cur);
struct vm_request *req
= BTREE_CONTAINER(struct vm_request, req_node, cur);
spin_lock(&req->req_lock);
switch (req->req_status) {
case PAGE_REQUEST_PENDING:
req->req_status = PAGE_REQUEST_IN_PROGRESS;
case VM_REQUEST_PENDING:
req->req_status = VM_REQUEST_IN_PROGRESS;
ctrl->vc_requests_waiting--;
return req;
case PAGE_REQUEST_ASYNC:
case VM_REQUEST_ASYNC:
btree_delete(&ctrl->vc_requests, &req->req_node);
ctrl->vc_requests_waiting--;
return req;
@@ -78,7 +77,7 @@ static struct page_request *get_next_request(struct vm_controller *ctrl)
return NULL;
}
static kern_status_t try_enqueue(struct btree *tree, struct page_request *req)
static kern_status_t try_enqueue(struct btree *tree, struct vm_request *req)
{
if (!tree->b_root) {
tree->b_root = &req->req_node;
@@ -88,8 +87,8 @@ static kern_status_t try_enqueue(struct btree *tree, struct page_request *req)
struct btree_node *cur = tree->b_root;
while (1) {
struct page_request *cur_node
= BTREE_CONTAINER(struct page_request, req_node, cur);
struct vm_request *cur_node
= BTREE_CONTAINER(struct vm_request, req_node, cur);
struct btree_node *next = NULL;
if (req->req_id > cur_node->req_id) {
@@ -119,7 +118,7 @@ static kern_status_t try_enqueue(struct btree *tree, struct page_request *req)
static kern_status_t send_request_async(
struct vm_controller *ctrl,
struct page_request *req)
struct vm_request *req)
{
fill_random(&req->req_id, sizeof req->req_id);
while (!try_enqueue(&ctrl->vc_requests, req)) {
@@ -136,9 +135,9 @@ static kern_status_t send_request_async(
kern_status_t vm_controller_recv(
struct vm_controller *ctrl,
equeue_packet_page_request_t *out)
equeue_packet_vm_request_t *out)
{
struct page_request *req = NULL;
struct vm_request *req = NULL;
req = get_next_request(ctrl);
if (!req) {
@@ -151,16 +150,30 @@ kern_status_t vm_controller_recv(
VM_CONTROLLER_SIGNAL_REQUEST_RECEIVED);
}
out->req_vmo = req->req_object;
vm_object_lock(req->req_object);
out->req_id = req->req_id;
out->req_vmo = req->req_object->vo_key;
out->req_type = req->req_type;
out->req_offset = req->req_offset;
out->req_length = req->req_length;
switch (req->req_type) {
case VM_REQUEST_READ:
case VM_REQUEST_DIRTY:
out->req_offset = req->req_offset;
out->req_length = req->req_length;
break;
case VM_REQUEST_ATTACH:
out->req_src_vmo = req->req_object->vo_src_key;
break;
default:
break;
}
vm_object_unlock(req->req_object);
spin_unlock(&req->req_lock);
if (req->req_status == PAGE_REQUEST_ASYNC) {
if (req->req_status == VM_REQUEST_ASYNC) {
put_current_thread(req->req_sender);
vm_cache_free(&page_request_cache, req);
vm_cache_free(&vm_request_cache, req);
}
return KERN_OK;
@@ -208,6 +221,53 @@ kern_status_t vm_controller_create_object(
return KERN_OK;
}
kern_status_t vm_controller_prepare_attach(
struct vm_controller *ctrl,
uint64_t req_id,
struct vm_object **out_vmo)
{
struct vm_request *req = get_request(&ctrl->vc_requests, req_id);
if (!req) {
return KERN_INVALID_ARGUMENT;
}
spin_lock(&req->req_lock);
req->req_status = VM_REQUEST_IN_PROGRESS;
*out_vmo = req->req_object;
spin_unlock(&req->req_lock);
return KERN_OK;
}
kern_status_t vm_controller_finish_attach(
struct vm_controller *ctrl,
uint64_t req_id,
equeue_key_t new_key)
{
struct vm_request *req = get_request(&ctrl->vc_requests, req_id);
if (!req) {
return KERN_INVALID_ARGUMENT;
}
spin_lock(&req->req_lock);
struct vm_object *vmo = req->req_object;
spin_unlock(&req->req_lock);
vm_object_lock(vmo);
vmo->vo_key = new_key;
vmo->vo_src_key = 0;
vmo->vo_flags &= ~VMO_LAZY_ATTACH;
vm_object_unlock(vmo);
spin_lock(&req->req_lock);
req->req_status = VM_REQUEST_COMPLETE;
req->req_result = KERN_OK;
thread_awaken(req->req_sender);
spin_unlock(&req->req_lock);
return KERN_OK;
}
kern_status_t vm_controller_detach_object(
struct vm_controller *ctrl,
struct vm_object *vmo)
@@ -216,7 +276,7 @@ kern_status_t vm_controller_detach_object(
return KERN_INVALID_ARGUMENT;
}
if (vmo->vo_key == 0) {
if (vmo->vo_flags & VMO_LAZY_ATTACH) {
/* this vmo isn't actually attached to this controller yet.
* this can happen if a controller-attached vmo was duplicated
* via copy-on-write, and the duplicate vmo has not yet been
@@ -225,16 +285,14 @@ kern_status_t vm_controller_detach_object(
return KERN_OK;
}
struct page_request *req
= vm_cache_alloc(&page_request_cache, VM_NORMAL);
req->req_type = PAGE_REQUEST_DETACH;
req->req_status = PAGE_REQUEST_ASYNC;
req->req_object = vmo->vo_key;
struct vm_request *req = vm_cache_alloc(&vm_request_cache, VM_NORMAL);
req->req_type = VM_REQUEST_DETACH;
req->req_status = VM_REQUEST_ASYNC;
req->req_object = vmo;
req->req_sender = get_current_thread();
send_request_async(ctrl, req);
vmo->vo_ctrl = NULL;
vmo->vo_key = 0;
object_unref(&ctrl->vc_base);
return KERN_OK;
@@ -242,7 +300,7 @@ kern_status_t vm_controller_detach_object(
static void wait_for_reply(
struct vm_controller *ctrl,
struct page_request *req,
struct vm_request *req,
unsigned long *lock_flags)
{
struct wait_item waiter;
@@ -251,7 +309,7 @@ static void wait_for_reply(
wait_item_init(&waiter, self);
for (;;) {
self->tr_state = THREAD_SLEEPING;
if (req->req_status == PAGE_REQUEST_COMPLETE) {
if (req->req_status == VM_REQUEST_COMPLETE) {
break;
}
@@ -264,7 +322,7 @@ static void wait_for_reply(
put_current_thread(self);
}
static void fulfill_requests(
void vm_controller_fulfill_requests(
struct vm_controller *ctrl,
equeue_key_t object,
off_t offset,
@@ -274,8 +332,8 @@ static void fulfill_requests(
off_t limit = offset + length - 1;
struct btree_node *cur = btree_first(&ctrl->vc_requests);
while (cur) {
struct page_request *req
= BTREE_CONTAINER(struct page_request, req_node, cur);
struct vm_request *req
= BTREE_CONTAINER(struct vm_request, req_node, cur);
spin_lock(&req->req_lock);
bool match = false;
off_t req_base = req->req_offset;
@@ -287,12 +345,14 @@ static void fulfill_requests(
match = true;
}
if (req->req_object != object) {
vm_object_lock(req->req_object);
if (req->req_object->vo_key != object) {
match = false;
}
vm_object_unlock(req->req_object);
if (match) {
req->req_status = PAGE_REQUEST_COMPLETE;
req->req_status = VM_REQUEST_COMPLETE;
req->req_result = result;
thread_awaken(req->req_sender);
}
@@ -325,14 +385,13 @@ kern_status_t vm_controller_supply_pages(
src_offset,
count,
NULL);
fulfill_requests(ctrl, dst->vo_key, dst_offset, count, status);
return status;
}
kern_status_t vm_controller_send_request(
struct vm_controller *ctrl,
struct page_request *req,
struct vm_request *req,
unsigned long *irq_flags)
{
fill_random(&req->req_id, sizeof req->req_id);
vm/vm-object.c
+38 -5
View File
@@ -288,8 +288,10 @@ struct vm_object *vm_object_duplicate_cow(struct vm_object *vmo)
struct vm_object *out = VM_OBJECT_CAST(obj);
memcpy(out->vo_name, vmo->vo_name, sizeof out->vo_name);
out->vo_flags = vmo->vo_flags;
out->vo_flags = vmo->vo_flags | VMO_LAZY_ATTACH;
out->vo_ctrl = vmo->vo_ctrl;
out->vo_key = 0;
out->vo_src_key = vmo->vo_key;
out->vo_prot = vmo->vo_prot;
out->vo_size = vmo->vo_size;
memcpy(out->vo_name, vmo->vo_name, sizeof vmo->vo_name);
@@ -315,6 +317,37 @@ struct vm_object *vm_object_duplicate_cow(struct vm_object *vmo)
return out;
}
kern_status_t vm_object_attach_cow(
struct vm_object *vmo,
unsigned long *irq_flags)
{
struct vm_controller *ctrl = vmo->vo_ctrl;
struct vm_request req = {0};
req.req_status = VM_REQUEST_PENDING;
req.req_type = VM_REQUEST_ATTACH;
req.req_length = vm_page_order_to_bytes(VM_PAGE_4K);
req.req_sender = get_current_thread();
object_ref(&vmo->vo_base);
req.req_object = vmo;
vm_object_unlock_irqrestore(vmo, *irq_flags);
vm_controller_lock_irqsave(ctrl, irq_flags);
spin_lock(&req.req_lock);
kern_status_t status
= vm_controller_send_request(ctrl, &req, irq_flags);
put_current_thread(req.req_sender);
spin_unlock(&req.req_lock);
vm_controller_unlock_irqrestore(ctrl, *irq_flags);
object_unref(&vmo->vo_base);
vm_object_lock_irqsave(vmo, irq_flags);
return status;
}
static struct vm_page *alloc_page(struct vm_object *vo, off_t offset)
{
struct vm_page *page = NULL;
@@ -458,15 +491,15 @@ static kern_status_t request_page(
unsigned long *irq_flags)
{
struct vm_controller *ctrl = vo->vo_ctrl;
struct page_request req = {0};
req.req_status = PAGE_REQUEST_PENDING;
req.req_type = PAGE_REQUEST_READ;
struct vm_request req = {0};
req.req_status = VM_REQUEST_PENDING;
req.req_type = VM_REQUEST_READ;
req.req_offset = offset;
req.req_length = vm_page_order_to_bytes(VM_PAGE_4K);
req.req_sender = get_current_thread();
object_ref(&vo->vo_base);
req.req_object = vo->vo_key;
req.req_object = vo;
vm_object_unlock_irqrestore(vo, *irq_flags);
vm_controller_lock_irqsave(ctrl, irq_flags);