libfs: improve cleanup of shared file mappings

lib/libfs/context.c

@@ -211,14 +211,29 @@ static enum fs_status handle_vm_request_detach(
 	}
 
 	kern_tracef(
-		"received vm request (detach) for file %s",
+		"received vm request (detach) for file %s (ref=%zu)",
-		mapping->m_file->f_dent->d_name);
+		mapping->m_file->f_dent->d_name,
+		mapping->m_ref);
+
+	if (mapping->m_ref > 1) {
+		mapping->m_ref--;
+		return FS_SUCCESS;
+	}
 
 	struct fs_file *f = mapping->m_file;
 	switch (mapping->m_type) {
 	case FILE_MAPPING_PRIVATE:
+		kern_tracef(
+			"destroying private mapping of %s",
+			f->f_dent->d_name);
 		queue_delete(&f->f_mappings, &mapping->m_entry);
 		break;
+	case FILE_MAPPING_SHARED:
+		kern_tracef(
+			"destroying shared mapping of %s",
+			f->f_dent->d_name);
+		f->f_inode->i_shared_mapping = NULL;
+		break;
 	default:
 		break;
 	}

lib/libfs/interface/map.c

@@ -8,6 +8,7 @@
 #include <magenta/handle.h>
 #include <magenta/log.h>
 #include <magenta/vm.h>
+#include <stdio.h>
 #include <sys/mman.h>
 
 static int create_file_mapping(
@@ -57,6 +58,7 @@ static int create_file_mapping(
 		return __errno_from_kern_status(status);
 	}
 
+	mapping->m_ref = 1;
 	mapping->m_file = f;
 	mapping->m_vmo = vmo;
 
@@ -102,14 +104,19 @@ extern kern_status_t fs_msg_map(
 		(flags & MAP_SHARED) ? "shared" : "private",
 		mapping->m_vmo);
 
-	kern_handle_t vmo;
+	kern_handle_t vmo = KERN_HANDLE_INVALID;
-	kern_handle_transfer(
+	kern_status_t status = kern_handle_transfer(
 		KERN_HANDLE_INVALID,
 		mapping->m_vmo,
 		KERN_HANDLE_INVALID,
 		KERN_HANDLE_INVALID,
 		HANDLE_TRANSFER_COPY,
 		&vmo);
+	if (status != KERN_OK) {
+		kern_tracef("kern_handle_transfer failed %d", status);
+		return status;
+	}
+
 	f->f_ref++;
 
 	*out_err = SUCCESS;

lib/libfs/mapping.h

@@ -14,6 +14,10 @@ enum file_mapping_type {
 
 struct file_mapping {
 	enum file_mapping_type m_type;
+
+	/* for shared mappings */
+	size_t m_ref;
+
 	struct fs_file *m_file;
 	kern_handle_t m_vmo;
 	struct queue_entry m_entry;