diff --git a/vm/address-space.c b/vm/address-space.c index 92730b0..5af9041 100644 --- a/vm/address-space.c +++ b/vm/address-space.c @@ -1526,7 +1526,129 @@ static kern_status_t request_missing_page( return status; } -/* this function must be called with `region` locked */ +/* handle a write to a page that is present and /should be/ writeable, but is + * currently read-only due to COW. */ +static kern_status_t handle_cow_access( + struct address_space *region, + virt_addr_t addr, + enum pmap_fault_flags flags, + unsigned long *irq_flags) +{ + struct vm_area *area + = get_entry(®ion->s_mappings, addr, GET_ENTRY_EXACT); + if (!area || !area->vma_object) { + /* no mapping exists (this shouldn't happen) */ + address_space_unlock_irqrestore(region, *irq_flags); + return KERN_NO_ENTRY; + } + + if ((area->vma_prot & (VM_PROT_WRITE | VM_PROT_USER)) + != (VM_PROT_WRITE | VM_PROT_USER)) { + /* access denied */ + address_space_unlock_irqrestore(region, *irq_flags); + return KERN_ACCESS_DENIED; + } + + tracek("cow access %zx", addr); + if (area->vma_object->vo_ctrl) { + panic("COW on controlled vm-object"); + } + + off_t object_offset = addr - area->vma_base + area->vma_object_offset; + vm_object_lock(area->vma_object); + struct vm_page *pg + = vm_object_get_page(area->vma_object, object_offset, 0, NULL); + + bool add_page = false; + if (!pg) { + /* the page hasn't been added to the vmo yet. */ + pfn_t pfn; + vm_prot_t prot; + kern_status_t status + = pmap_get(region->s_pmap, addr, &pfn, &prot); + if (status != KERN_OK) { + vm_object_unlock(area->vma_object); + address_space_unlock_irqrestore(region, *irq_flags); + return status; + } + + add_page = true; + pg = vm_page_get(pfn * VM_PAGE_SIZE); + tracek("recovered page %zx (%p, %u) from the aether", + vm_page_get_paddr(pg), + pg, + pg->p_cow_ref); + + if (!pg) { + /* still can't find the page */ + panic("cannot resolve cow-reference to page %zx", + pfn * VM_PAGE_SIZE); + } + + if (!pg->p_cow_ref) { + /* still can't find the page */ + panic("cow-reference page %zx cow-ref=0", + pfn * VM_PAGE_SIZE); + } + } + + atomic_t cow_ref = atomic_sub_fetch(&pg->p_cow_ref, 1); + tracek("decrement cow-ref page %zx -> now %u", + vm_page_get_paddr(pg), + pg->p_cow_ref); + if (cow_ref > 0) { + /* another task is (or will be) referencing this page. create + * a copy */ + struct vm_page *dup = vm_page_alloc(VM_PAGE_4K, VM_NORMAL); + if (!dup) { + vm_object_unlock(area->vma_object); + address_space_unlock_irqrestore(region, *irq_flags); + return KERN_NO_MEMORY; + } + + void *src = vm_page_get_vaddr(pg); + void *dest = vm_page_get_vaddr(dup); + memcpy(dest, src, vm_page_get_size_bytes(dup)); + + if (!add_page) { + btree_delete(&area->vma_object->vo_pages, &pg->p_bnode); + } + + vm_object_put_page(area->vma_object, object_offset, dup); + tracek("splitting cow page %zx -> %zx", + vm_page_get_paddr(pg), + vm_page_get_paddr(dup)); + tracek("put page %zx into area [%zx-%zx] at %zx", + vm_page_get_paddr(dup), + area->vma_base, + area->vma_limit, + object_offset); + pg = dup; + } else if (add_page) { + /* we are the last task referencing this page. add it to our + * vmo and continue */ + tracek("claimed cow page %zx", vm_page_get_paddr(pg)); + vm_object_put_page(area->vma_object, object_offset, pg); + } else { + /* we are the last task referencing this page, and it is already + * in our vmo. */ + tracek("reclaimed cow page %zx", vm_page_get_paddr(pg)); + } + + kern_status_t status = pmap_add( + region->s_pmap, + addr, + vm_page_get_pfn(pg), + area->vma_prot, + PMAP_NORMAL); + + vm_object_unlock(area->vma_object); + address_space_unlock_irqrestore(region, *irq_flags); + + return status; +} + +/* this function must be called with `region` unlocked */ kern_status_t address_space_demand_map( struct address_space *region, virt_addr_t addr, @@ -1540,6 +1662,13 @@ kern_status_t address_space_demand_map( unsigned long irq_flags; address_space_lock_irqsave(region, &irq_flags); + const enum pmap_fault_flags cow_flags + = PMAP_FAULT_WRITE | PMAP_FAULT_PRESENT | PMAP_FAULT_USER; + + if ((flags & cow_flags) == cow_flags) { + return handle_cow_access(region, addr, flags, &irq_flags); + } + struct vm_area *area = get_entry(®ion->s_mappings, addr, GET_ENTRY_EXACT); if (!area || !area->vma_object) {