vm: address-space: implement resolving accesses to copy-on-write pages

vm/address-space.c

@@ -1526,7 +1526,129 @@ static kern_status_t request_missing_page(
 	return status;
 }
 
-/* this function must be called with `region` locked */
+/* handle a write to a page that is present and /should be/ writeable, but is
+ * currently read-only due to COW. */
+static kern_status_t handle_cow_access(
+	struct address_space *region,
+	virt_addr_t addr,
+	enum pmap_fault_flags flags,
+	unsigned long *irq_flags)
+{
+	struct vm_area *area
+		= get_entry(&region->s_mappings, addr, GET_ENTRY_EXACT);
+	if (!area || !area->vma_object) {
+		/* no mapping exists (this shouldn't happen) */
+		address_space_unlock_irqrestore(region, *irq_flags);
+		return KERN_NO_ENTRY;
+	}
+
+	if ((area->vma_prot & (VM_PROT_WRITE | VM_PROT_USER))
+	    != (VM_PROT_WRITE | VM_PROT_USER)) {
+		/* access denied */
+		address_space_unlock_irqrestore(region, *irq_flags);
+		return KERN_ACCESS_DENIED;
+	}
+
+	tracek("cow access %zx", addr);
+	if (area->vma_object->vo_ctrl) {
+		panic("COW on controlled vm-object");
+	}
+
+	off_t object_offset = addr - area->vma_base + area->vma_object_offset;
+	vm_object_lock(area->vma_object);
+	struct vm_page *pg
+		= vm_object_get_page(area->vma_object, object_offset, 0, NULL);
+
+	bool add_page = false;
+	if (!pg) {
+		/* the page hasn't been added to the vmo yet. */
+		pfn_t pfn;
+		vm_prot_t prot;
+		kern_status_t status
+			= pmap_get(region->s_pmap, addr, &pfn, &prot);
+		if (status != KERN_OK) {
+			vm_object_unlock(area->vma_object);
+			address_space_unlock_irqrestore(region, *irq_flags);
+			return status;
+		}
+
+		add_page = true;
+		pg = vm_page_get(pfn * VM_PAGE_SIZE);
+		tracek("recovered page %zx (%p, %u) from the aether",
+		       vm_page_get_paddr(pg),
+		       pg,
+		       pg->p_cow_ref);
+
+		if (!pg) {
+			/* still can't find the page */
+			panic("cannot resolve cow-reference to page %zx",
+			      pfn * VM_PAGE_SIZE);
+		}
+
+		if (!pg->p_cow_ref) {
+			/* still can't find the page */
+			panic("cow-reference page %zx cow-ref=0",
+			      pfn * VM_PAGE_SIZE);
+		}
+	}
+
+	atomic_t cow_ref = atomic_sub_fetch(&pg->p_cow_ref, 1);
+	tracek("decrement cow-ref page %zx -> now %u",
+	       vm_page_get_paddr(pg),
+	       pg->p_cow_ref);
+	if (cow_ref > 0) {
+		/* another task is (or will be) referencing this page. create
+		 * a copy */
+		struct vm_page *dup = vm_page_alloc(VM_PAGE_4K, VM_NORMAL);
+		if (!dup) {
+			vm_object_unlock(area->vma_object);
+			address_space_unlock_irqrestore(region, *irq_flags);
+			return KERN_NO_MEMORY;
+		}
+
+		void *src = vm_page_get_vaddr(pg);
+		void *dest = vm_page_get_vaddr(dup);
+		memcpy(dest, src, vm_page_get_size_bytes(dup));
+
+		if (!add_page) {
+			btree_delete(&area->vma_object->vo_pages, &pg->p_bnode);
+		}
+
+		vm_object_put_page(area->vma_object, object_offset, dup);
+		tracek("splitting cow page %zx -> %zx",
+		       vm_page_get_paddr(pg),
+		       vm_page_get_paddr(dup));
+		tracek("put page %zx into area [%zx-%zx] at %zx",
+		       vm_page_get_paddr(dup),
+		       area->vma_base,
+		       area->vma_limit,
+		       object_offset);
+		pg = dup;
+	} else if (add_page) {
+		/* we are the last task referencing this page. add it to our
+		 * vmo and continue */
+		tracek("claimed cow page %zx", vm_page_get_paddr(pg));
+		vm_object_put_page(area->vma_object, object_offset, pg);
+	} else {
+		/* we are the last task referencing this page, and it is already
+		 * in our vmo. */
+		tracek("reclaimed cow page %zx", vm_page_get_paddr(pg));
+	}
+
+	kern_status_t status = pmap_add(
+		region->s_pmap,
+		addr,
+		vm_page_get_pfn(pg),
+		area->vma_prot,
+		PMAP_NORMAL);
+
+	vm_object_unlock(area->vma_object);
+	address_space_unlock_irqrestore(region, *irq_flags);
+
+	return status;
+}
+
+/* this function must be called with `region` unlocked */
 kern_status_t address_space_demand_map(
 	struct address_space *region,
 	virt_addr_t addr,
@@ -1540,6 +1662,13 @@ kern_status_t address_space_demand_map(
 	unsigned long irq_flags;
 	address_space_lock_irqsave(region, &irq_flags);
 
+	const enum pmap_fault_flags cow_flags
+		= PMAP_FAULT_WRITE | PMAP_FAULT_PRESENT | PMAP_FAULT_USER;
+
+	if ((flags & cow_flags) == cow_flags) {
+		return handle_cow_access(region, addr, flags, &irq_flags);
+	}
+
 	struct vm_area *area
 		= get_entry(&region->s_mappings, addr, GET_ENTRY_EXACT);
 	if (!area || !area->vma_object) {