/* * (MPSAFE) * * Copyright (c) 1991, 1993 * The Regents of the University of California. All rights reserved. * * This code is derived from software contributed to Berkeley by * The Mach Operating System project at Carnegie-Mellon University. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. Neither the name of the University nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * from: @(#)vm_kern.c 8.3 (Berkeley) 1/12/94 * * * Copyright (c) 1987, 1990 Carnegie-Mellon University. * All rights reserved. * * Authors: Avadis Tevanian, Jr., Michael Wayne Young * * Permission to use, copy, modify and distribute this software and * its documentation is hereby granted, provided that both the copyright * notice and this permission notice appear in all copies of the * software, derivative works or modified versions, and any portions * thereof, and that both notices appear in supporting documentation. * * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS" * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND * FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE. * * Carnegie Mellon requests users of this software to return to * * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU * School of Computer Science * Carnegie Mellon University * Pittsburgh PA 15213-3890 * * any improvements or extensions that they make and grant Carnegie the * rights to redistribute these changes. * * $FreeBSD: src/sys/vm/vm_kern.c,v 1.61.2.2 2002/03/12 18:25:26 tegge Exp $ */ /* * Kernel memory management. */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include static struct vm_map kernel_map_store; static struct vm_map clean_map_store; static struct vm_map buffer_map_store; struct vm_map *kernel_map = &kernel_map_store; struct vm_map *clean_map = &clean_map_store; struct vm_map *buffer_map = &buffer_map_store; static __inline int KMVMCPU(int kmflags) { if ((kmflags & KM_CPU_SPEC) == 0) return 0; return VM_ALLOC_CPU(KM_GETCPU(kmflags)); } /* * Allocate pageable swap-backed anonymous memory */ void * kmem_alloc_swapbacked(kmem_anon_desc_t *kp, vm_size_t size, vm_subsys_t id) { int error; vm_pindex_t npages; size = round_page(size); npages = size / PAGE_SIZE; if (kp->map == NULL) kp->map = kernel_map; kp->data = vm_map_min(kernel_map); kp->size = size; kp->object = vm_object_allocate(OBJT_DEFAULT, npages); error = vm_map_find(kp->map, kp->object, NULL, 0, &kp->data, size, PAGE_SIZE, TRUE, VM_MAPTYPE_NORMAL, id, VM_PROT_ALL, VM_PROT_ALL, 0); if (error) { kprintf("kmem_alloc_swapbacked: %zd bytes failed %d\n", size, error); kp->data = (vm_offset_t)0; kmem_free_swapbacked(kp); return NULL; } return ((void *)(intptr_t)kp->data); } void kmem_free_swapbacked(kmem_anon_desc_t *kp) { if (kp->data) { /* * The object will be deallocated by kmem_free(). */ kmem_free(kp->map, kp->data, kp->size); kp->data = (vm_offset_t)0; } else { /* * Failure during allocation, object must be deallocated * manually. */ vm_object_deallocate(kp->object); } kp->object = NULL; } /* * Allocate pageable memory to the kernel's address map. "map" must * be kernel_map or a submap of kernel_map. Caller must adjust map or * enter VM pages itself. * * No requirements. */ vm_offset_t kmem_alloc_pageable(vm_map_t map, vm_size_t size, vm_subsys_t id) { vm_offset_t addr; int result; size = round_page(size); addr = vm_map_min(map); result = vm_map_find(map, NULL, NULL, (vm_offset_t) 0, &addr, size, PAGE_SIZE, TRUE, VM_MAPTYPE_NORMAL, id, VM_PROT_ALL, VM_PROT_ALL, 0); if (result != KERN_SUCCESS) return (0); return (addr); } /* * Same as kmem_alloc_pageable, except that it create a nofault entry. * * No requirements. */ vm_offset_t kmem_alloc_nofault(vm_map_t map, vm_size_t size, vm_subsys_t id, vm_size_t align) { vm_offset_t addr; int result; size = round_page(size); addr = vm_map_min(map); result = vm_map_find(map, NULL, NULL, (vm_offset_t) 0, &addr, size, align, TRUE, VM_MAPTYPE_NORMAL, id, VM_PROT_ALL, VM_PROT_ALL, MAP_NOFAULT); if (result != KERN_SUCCESS) return (0); return (addr); } /* * Allocate wired-down memory in the kernel's address map or a submap. * * No requirements. */ vm_offset_t kmem_alloc3(vm_map_t map, vm_size_t size, vm_subsys_t id, int kmflags) { vm_offset_t addr; vm_offset_t gstart; vm_offset_t i; int count; int cow; size = round_page(size); if (kmflags & KM_KRESERVE) count = vm_map_entry_kreserve(MAP_RESERVE_COUNT); else count = vm_map_entry_reserve(MAP_RESERVE_COUNT); if (kmflags & KM_STACK) { cow = MAP_IS_KSTACK; gstart = PAGE_SIZE; } else { cow = 0; gstart = 0; } /* * Use the kernel object for wired-down kernel pages. Assume that no * region of the kernel object is referenced more than once. * * Locate sufficient space in the map. This will give us the final * virtual address for the new memory, and thus will tell us the * offset within the kernel map. */ vm_map_lock(map); if (vm_map_findspace(map, vm_map_min(map), size, PAGE_SIZE, 0, &addr)) { vm_map_unlock(map); if (kmflags & KM_KRESERVE) vm_map_entry_krelease(count); else vm_map_entry_release(count); return (0); } vm_object_hold(kernel_object); vm_object_reference_locked(kernel_object); vm_map_insert(map, &count, kernel_object, NULL, addr, NULL, addr, addr + size, VM_MAPTYPE_NORMAL, id, VM_PROT_ALL, VM_PROT_ALL, cow); vm_object_drop(kernel_object); vm_map_unlock(map); if (kmflags & KM_KRESERVE) vm_map_entry_krelease(count); else vm_map_entry_release(count); /* * Guarantee that there are pages already in this object before * calling vm_map_kernel_wiring(). This is to prevent the following * scenario: * * 1) Threads have swapped out, so that there is a pager for the * kernel_object. 2) The kmsg zone is empty, and so we are * kmem_allocing a new page for it. 3) vm_map_kernel_wiring() calls * vm_fault(); there is no page, but there is a pager, so we call * pager_data_request. But the kmsg zone is empty, so we must * kmem_alloc. 4) goto 1 5) Even if the kmsg zone is not empty: when * we get the data back from the pager, it will be (very stale) * non-zero data. kmem_alloc is defined to return zero-filled memory. * * We're intentionally not activating the pages we allocate to prevent a * race with page-out. vm_map_kernel_wiring() will wire the pages. */ vm_object_hold(kernel_object); for (i = gstart; i < size; i += PAGE_SIZE) { vm_page_t mem; mem = vm_page_grab(kernel_object, OFF_TO_IDX(addr + i), VM_ALLOC_FORCE_ZERO | VM_ALLOC_NORMAL | VM_ALLOC_RETRY | KMVMCPU(kmflags)); vm_page_unqueue_nowakeup(mem); vm_page_wakeup(mem); } vm_object_drop(kernel_object); /* * And finally, mark the data as pageable or non-pageable (unwiring * or wiring the pages), according to the passed-in kmflags. * * NOTE: vm_map_kernel_wiring() handles any kstack guard. */ vm_map_kernel_wiring(map, addr, addr + size, kmflags); return (addr); } /* * Release a region of kernel virtual memory allocated with kmem_alloc, * and return the physical pages associated with that region. * * WARNING! If the caller entered pages into the region using pmap_kenter() * it must remove the pages using pmap_kremove[_quick]() before freeing the * underlying kmem, otherwise resident_count will be mistabulated. * * No requirements. */ void kmem_free(vm_map_t map, vm_offset_t addr, vm_size_t size) { vm_map_remove(map, trunc_page(addr), round_page(addr + size)); } /* * Used to break a system map into smaller maps, usually to reduce * contention and to provide large KVA spaces for subsystems like the * buffer cache. * * parent Map to take range from * result * size Size of range to find * min, max Returned endpoints of map * pageable Can the region be paged * * No requirements. */ void kmem_suballoc(vm_map_t parent, vm_map_t result, vm_offset_t *min, vm_offset_t *max, vm_size_t size) { int ret; size = round_page(size); *min = (vm_offset_t) vm_map_min(parent); ret = vm_map_find(parent, NULL, NULL, (vm_offset_t) 0, min, size, PAGE_SIZE, TRUE, VM_MAPTYPE_UNSPECIFIED, VM_SUBSYS_SYSMAP, VM_PROT_ALL, VM_PROT_ALL, 0); if (ret != KERN_SUCCESS) { kprintf("kmem_suballoc: bad status return of %d.\n", ret); panic("kmem_suballoc"); } *max = *min + size; pmap_reference(vm_map_pmap(parent)); vm_map_init(result, *min, *max, vm_map_pmap(parent)); if ((ret = vm_map_submap(parent, *min, *max, result)) != KERN_SUCCESS) panic("kmem_suballoc: unable to change range to submap"); } /* * Allocates pageable memory from a sub-map of the kernel. If the submap * has no room, the caller sleeps waiting for more memory in the submap. * * No requirements. */ vm_offset_t kmem_alloc_wait(vm_map_t map, vm_size_t size, vm_subsys_t id) { vm_offset_t addr; int count; size = round_page(size); count = vm_map_entry_reserve(MAP_RESERVE_COUNT); for (;;) { /* * To make this work for more than one map, use the map's lock * to lock out sleepers/wakers. */ vm_map_lock(map); if (vm_map_findspace(map, vm_map_min(map), size, PAGE_SIZE, 0, &addr) == 0) { break; } /* no space now; see if we can ever get space */ if (vm_map_max(map) - vm_map_min(map) < size) { vm_map_entry_release(count); vm_map_unlock(map); return (0); } vm_map_unlock(map); tsleep(map, 0, "kmaw", 0); } vm_map_insert(map, &count, NULL, NULL, (vm_offset_t)0, NULL, addr, addr + size, VM_MAPTYPE_NORMAL, id, VM_PROT_ALL, VM_PROT_ALL, 0); vm_map_unlock(map); vm_map_entry_release(count); return (addr); } /* * Allocates a region from the kernel address map and physical pages * within the specified address range to the kernel object. Creates a * wired mapping from this region to these pages, and returns the * region's starting virtual address. The allocated pages are not * necessarily physically contiguous. If M_ZERO is specified through the * given flags, then the pages are zeroed before they are mapped. */ vm_offset_t kmem_alloc_attr(vm_map_t map, vm_size_t size, vm_subsys_t id, int flags, vm_paddr_t low, vm_paddr_t high, vm_memattr_t memattr) { vm_offset_t addr, i, offset; vm_page_t m; int count; size = round_page(size); count = vm_map_entry_reserve(MAP_RESERVE_COUNT); vm_map_lock(map); if (vm_map_findspace(map, vm_map_min(map), size, PAGE_SIZE, flags, &addr)) { vm_map_unlock(map); vm_map_entry_release(count); return (0); } offset = addr - vm_map_min(kernel_map); vm_object_hold(kernel_object); vm_object_reference_locked(kernel_object); vm_map_insert(map, &count, kernel_object, NULL, offset, NULL, addr, addr + size, VM_MAPTYPE_NORMAL, id, VM_PROT_ALL, VM_PROT_ALL, 0); vm_map_unlock(map); vm_map_entry_release(count); vm_object_drop(kernel_object); for (i = 0; i < size; i += PAGE_SIZE) { m = vm_page_alloc_contig(low, high, PAGE_SIZE, 0, PAGE_SIZE, memattr); if (!m) { return (0); } vm_object_hold(kernel_object); vm_page_insert(m, kernel_object, OFF_TO_IDX(offset + i)); vm_object_drop(kernel_object); if (flags & M_ZERO) pmap_zero_page(VM_PAGE_TO_PHYS(m)); m->valid = VM_PAGE_BITS_ALL; } /* wire the pages */ vm_map_kernel_wiring(map, addr, addr + size, 0); return (addr); } /* * Returns memory to a submap of the kernel, and wakes up any processes * waiting for memory in that map. * * No requirements. */ void kmem_free_wakeup(vm_map_t map, vm_offset_t addr, vm_size_t size) { int count; count = vm_map_entry_reserve(MAP_RESERVE_COUNT); vm_map_lock(map); vm_map_delete(map, trunc_page(addr), round_page(addr + size), &count); wakeup(map); vm_map_unlock(map); vm_map_entry_release(count); } /* * Create the kernel_ma for (KvaStart,KvaEnd) and insert mappings to * cover areas already allocated or reserved thus far. * * The areas (virtual_start, virtual_end) and (virtual2_start, virtual2_end) * are available so the cutouts are the areas around these ranges between * KvaStart and KvaEnd. * * Depend on the zalloc bootstrap cache to get our vm_map_entry_t. * Called from the low level boot code only. */ void kmem_init(void) { vm_offset_t addr; vm_map_t m; int count; m = kernel_map; vm_map_init(m, KvaStart, KvaEnd, kernel_pmap); vm_map_lock(m); /* N.B.: cannot use kgdb to debug, starting with this assignment ... */ m->system_map = 1; count = vm_map_entry_reserve(MAP_RESERVE_COUNT); addr = KvaStart; if (virtual2_start) { if (addr < virtual2_start) { vm_map_insert(m, &count, NULL, NULL, (vm_offset_t) 0, NULL, addr, virtual2_start, VM_MAPTYPE_NORMAL, VM_SUBSYS_RESERVED, VM_PROT_ALL, VM_PROT_ALL, 0); } addr = virtual2_end; } if (addr < virtual_start) { vm_map_insert(m, &count, NULL, NULL, (vm_offset_t) 0, NULL, addr, virtual_start, VM_MAPTYPE_NORMAL, VM_SUBSYS_RESERVED, VM_PROT_ALL, VM_PROT_ALL, 0); } addr = virtual_end; if (addr < KvaEnd) { vm_map_insert(m, &count, NULL, NULL, (vm_offset_t) 0, NULL, addr, KvaEnd, VM_MAPTYPE_NORMAL, VM_SUBSYS_RESERVED, VM_PROT_ALL, VM_PROT_ALL, 0); } /* ... and ending with the completion of the above `insert' */ vm_map_unlock(m); vm_map_entry_release(count); } /* * No requirements. */ static int kvm_size(SYSCTL_HANDLER_ARGS) { unsigned long ksize = KvaSize; return sysctl_handle_long(oidp, &ksize, 0, req); } SYSCTL_PROC(_vm, OID_AUTO, kvm_size, CTLTYPE_ULONG|CTLFLAG_RD, 0, 0, kvm_size, "LU", "Size of KVM"); /* * No requirements. */ static int kvm_free(SYSCTL_HANDLER_ARGS) { unsigned long kfree = virtual_end - kernel_vm_end; return sysctl_handle_long(oidp, &kfree, 0, req); } SYSCTL_PROC(_vm, OID_AUTO, kvm_free, CTLTYPE_ULONG|CTLFLAG_RD, 0, 0, kvm_free, "LU", "Amount of KVM free");