Commit ee46872e authored by Colin Finck's avatar Colin Finck
Browse files

Implement physical and virtual memory managers, a FreeList for both, an even...

Implement physical and virtual memory managers, a FreeList for both, an even more generic DoublyLinkedList in Safe Rust, and a custom allocator.

The data structures used to manage heap memory require dynamic memory allocations themselves.
To solve this chicken-egg problem, I have introduced a "Bootstrap Allocator". This is a simple
single-threaded implementation of a bump allocator using some preallocated space.
As soon as all required data structures have been set up, the more sophisticated "System Allocator"
is used.

This code finally breaks compatibility with the HermitCore C implementation.
It compiles, but applications can only be run again when the remaining kernel components have been ported.
parent 78a29dc5
......@@ -10,6 +10,7 @@ crate-type = ["staticlib"]
[dependencies]
bitflags = "1.0.0"
multiboot = "0.3.0"
spin = "0.4.6"
[dependencies.lazy_static]
......
......@@ -158,8 +158,8 @@ pub fn install()
}
}
#[no_mangle]
pub unsafe extern "C" fn set_tss(rsp: u64/*, ist: u64*/)
/*#[no_mangle]
pub unsafe extern "C" fn set_tss(rsp: u64, ist: u64)
{
let core_id = __core_id.per_core() as usize;
TSS_BUFFER.tss[core_id].rsp[0] = rsp;
......@@ -170,4 +170,4 @@ pub unsafe extern "C" fn set_tss(rsp: u64/*, ist: u64*/)
pub extern "C" fn gdt_install()
{
install();
}
}*/
......@@ -69,7 +69,7 @@ pub fn set_gate(index: u8, handler: unsafe extern "C" fn(), ist_index: u8)
unsafe { IDT[index as usize] = entry; }
}
#[no_mangle]
/*#[no_mangle]
pub unsafe extern "C" fn idt_install() {
install();
}
}*/
......@@ -22,3 +22,12 @@
// WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
pub mod paging;
pub mod physicalmem;
pub mod virtualmem;
pub fn init() {
paging::init();
physicalmem::init();
virtualmem::init();
}
......@@ -22,11 +22,13 @@
// WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
use arch::x86_64::irq;
use arch::x86_64::mm;
use arch::x86_64::percore::*;
use arch::x86_64::processor;
use core::fmt;
use core::{fmt, ptr};
use core::marker::PhantomData;
use logging::*;
use multiboot;
use synch::spinlock::*;
use tasks::*;
use x86::shared::*;
......@@ -44,22 +46,18 @@ extern "C" {
static cmdline: *const u8;
static cmdsize: usize;
static mb_info: usize;
static image_size: usize;
static kernel_start: u8;
static mb_info: multiboot::PAddr;
fn get_pages(npages: usize) -> usize;
fn get_zeroed_page() -> usize;
fn ipi_tlb_flush() -> i32;
}
lazy_static! {
static ref ROOT_PAGETABLE: SpinlockIrqSave<&'static mut PageTable<PML4>> =
SpinlockIrqSave::new(unsafe { &mut *(0xFFFF_FFFF_FFFF_F000 as *mut PageTable<PML4>) });
}
/// Number of Offset bits of a virtual address for a 4KiB page, which are shifted away to get its Page Frame Number (PFN).
/// Number of Offset bits of a virtual address for a 4 KiB page, which are shifted away to get its Page Frame Number (PFN).
const PAGE_BITS: usize = 12;
/// Number of bits of the index in each table (PML4, PDPT, PDT, PGT).
......@@ -73,7 +71,7 @@ bitflags! {
/// Possible flags for an entry in either table (PML4, PDPT, PDT, PGT)
///
/// See Intel Vol. 3A, Tables 4-14 through 4-19
struct PageTableEntryFlags: usize {
pub struct PageTableEntryFlags: usize {
/// Set if this entry is valid and points to a page or table.
const PRESENT = 1 << 0;
......@@ -96,7 +94,7 @@ bitflags! {
/// Only for page entries: Set if software has written to the memory referenced by this entry.
const DIRTY = 1 << 6;
/// Only for page entries in PDPT or PDT: Set if this entry references a 1GiB (PDPT) or 2MiB (PDT) page.
/// Only for page entries in PDPT or PDT: Set if this entry references a 1 GiB (PDPT) or 2 MiB (PDT) page.
const HUGE_PAGE = 1 << 7;
/// Only for page entries: Set if this address translation is global for all tasks and does not need to
......@@ -116,14 +114,14 @@ impl PageTableEntryFlags {
/// An entry in either table (PML4, PDPT, PDT, PGT)
#[derive(Clone, Copy)]
struct PageTableEntry {
pub struct PageTableEntry {
/// Physical memory address this entry refers, combined with flags from PageTableEntryFlags.
physical_address_and_flags: usize
}
impl PageTableEntry {
/// Return the stored physical address.
fn address(&self) -> usize {
pub fn address(&self) -> usize {
self.physical_address_and_flags & !(BasePageSize::SIZE - 1) & !(PageTableEntryFlags::EXECUTE_DISABLE).bits()
}
......@@ -133,7 +131,7 @@ impl PageTableEntry {
}
/// Returns whether this entry is valid (present).
fn is_present(&self) -> bool {
pub fn is_present(&self) -> bool {
(self.physical_address_and_flags & PageTableEntryFlags::PRESENT.bits()) != 0
}
......@@ -145,12 +143,12 @@ impl PageTableEntry {
/// * `flags` - Flags from PageTableEntryFlags (note that the PRESENT and ACCESSED flags are set automatically)
fn set(&mut self, physical_address: usize, flags: PageTableEntryFlags) {
if flags.contains(PageTableEntryFlags::HUGE_PAGE) {
// HUGE_PAGE may indicate a 2MiB or 1GiB page.
// We don't know this here, so we can only verify that at least the offset bits for a 2MiB page are zero.
assert!(physical_address & (LargePageSize::SIZE - 1) == 0, "Physical address not on 2MiB page boundary (physical_address = {:#X})", physical_address);
// HUGE_PAGE may indicate a 2 MiB or 1 GiB page.
// We don't know this here, so we can only verify that at least the offset bits for a 2 MiB page are zero.
assert!(physical_address & (LargePageSize::SIZE - 1) == 0, "Physical address not on 2 MiB page boundary (physical_address = {:#X})", physical_address);
} else {
// Verify that the offset bits for a 4KiB page are zero.
assert!(physical_address & (BasePageSize::SIZE - 1) == 0, "Physical address not on 4KiB page boundary (physical_address = {:#X})", physical_address);
// Verify that the offset bits for a 4 KiB page are zero.
assert!(physical_address & (BasePageSize::SIZE - 1) == 0, "Physical address not on 4 KiB page boundary (physical_address = {:#X})", physical_address);
}
// Verify that the physical address does not exceed the CPU's physical address width.
......@@ -164,7 +162,7 @@ impl PageTableEntry {
///
/// This is defined as a subtrait of Copy to enable #[derive(Clone, Copy)] for Page.
/// Currently, deriving implementations for these traits only works if all dependent types implement it as well.
trait PageSize: Copy {
pub trait PageSize: Copy {
/// The page size in bytes.
const SIZE: usize;
......@@ -177,27 +175,27 @@ trait PageSize: Copy {
const MAP_EXTRA_FLAG: PageTableEntryFlags;
}
/// A 4KiB page mapped in the PGT.
/// A 4 KiB page mapped in the PGT.
#[derive(Clone, Copy)]
enum BasePageSize {}
pub enum BasePageSize {}
impl PageSize for BasePageSize {
const SIZE: usize = 4096;
const MAP_LEVEL: usize = 0;
const MAP_EXTRA_FLAG: PageTableEntryFlags = PageTableEntryFlags::BLANK;
}
/// A 2MiB page mapped in the PDT.
/// A 2 MiB page mapped in the PDT.
#[derive(Clone, Copy)]
enum LargePageSize {}
pub enum LargePageSize {}
impl PageSize for LargePageSize {
const SIZE: usize = 2 * 1024 * 1024;
const MAP_LEVEL: usize = 1;
const MAP_EXTRA_FLAG: PageTableEntryFlags = PageTableEntryFlags::HUGE_PAGE;
}
/// A 1GiB page mapped in the PDPT.
/// A 1 GiB page mapped in the PDPT.
#[derive(Clone, Copy)]
enum HugePageSize {}
pub enum HugePageSize {}
impl PageSize for HugePageSize {
const SIZE: usize = 1024 * 1024 * 1024;
const MAP_LEVEL: usize = 2;
......@@ -249,11 +247,19 @@ impl<S: PageSize> Page<S> {
}
Self {
virtual_address: virtual_address & !(S::SIZE - 1),
virtual_address: align_down!(virtual_address, S::SIZE),
size: PhantomData,
}
}
/// Returns a Page after the given virtual address.
/// That means, the address is rounded up to a page size boundary.
fn after_address(virtual_address: usize) -> Self {
let mut page = Self::including_address(virtual_address);
page.virtual_address += S::SIZE;
page
}
/// Returns a PageIter to iterate from the given first Page to the given last Page (inclusive).
fn range(first: Self, last: Self) -> PageIter<S> {
assert!(first.virtual_address <= last.virtual_address);
......@@ -457,8 +463,8 @@ impl<L: PageTableLevelWithSubtables> PageTableMethods for PageTable<L> where L::
// Does the table exist yet?
if !self.entries[index].is_present() {
// Allocate a single 4KiB page for the new entry and mark it as a valid, writable subtable.
let physical_address = unsafe { get_pages(1) };
// Allocate a single 4 KiB page for the new entry and mark it as a valid, writable subtable.
let physical_address = mm::physicalmem::allocate(BasePageSize::SIZE);
self.entries[index].set(physical_address, PageTableEntryFlags::WRITABLE);
// Mark all entries as unused in the newly created table.
......@@ -580,33 +586,37 @@ impl fmt::Display for PageFaultError {
}
#[inline]
fn get_page_range(viraddr: usize, npages: usize) -> PageIter<BasePageSize> {
let first_page = Page::<BasePageSize>::including_address(viraddr);
let last_page = Page::<BasePageSize>::including_address(viraddr + (npages - 1) * BasePageSize::SIZE);
Page::<BasePageSize>::range(first_page, last_page)
}
#[no_mangle]
pub extern "C" fn getpagesize() -> i32 {
BasePageSize::SIZE as i32
}
fn page_fault_handler(state_ref: &irq::state) {
debug!("page_fault_handler{:#X}", state_ref as *const _ as usize);
let virtual_address = unsafe { control_regs::cr2() };
let task = unsafe { current_task.per_core().as_ref().expect("task is NULL!") };
// Is there a heap associated to the current task?
if let Some(ref heap) = unsafe { task.heap.as_ref() } {
// Is the requested virtual address within the boundary of that heap.
if virtual_address >= heap.start && virtual_address < heap.end {
// Then the task may access the page at that virtual address.
let mut locked_root_table = ROOT_PAGETABLE.lock();
let page = Page::<BasePageSize>::including_address(virtual_address);
// Is the page already mapped in the page table?
if locked_root_table.get_page_table_entry(page).is_none() {
let physical_address = unsafe { if runtime_osinit.is_null() { get_pages(1) } else { get_zeroed_page() } };
locked_root_table.map_page::<BasePageSize>(page, physical_address, PageTableEntryFlags::WRITABLE | PageTableEntryFlags::EXECUTE_DISABLE);
// No, then create a mapping.
let physical_address = mm::physicalmem::allocate(BasePageSize::SIZE);
locked_root_table.map_page::<BasePageSize>(
page,
physical_address,
PageTableEntryFlags::WRITABLE | PageTableEntryFlags::EXECUTE_DISABLE
);
// If our application is a Go application (detected by the presence of the
// weak symbol "runtime_osinit"), we have to return a zeroed page.
unsafe {
if !runtime_osinit.is_null() {
ptr::write_bytes(page.address() as *mut u8, 0, BasePageSize::SIZE);
}
}
}
return;
......@@ -625,16 +635,33 @@ fn page_fault_handler(state_ref: &irq::state) {
panic!();
}
#[no_mangle]
pub extern "C" fn __page_map(viraddr: usize, phyaddr: usize, npages: usize, bits: usize, do_ipi: u8) -> i32 {
debug!("__page_map({:#X}, {:#X}, {}, {:#X}, {})", viraddr, phyaddr, npages, bits, do_ipi);
#[inline]
fn get_page_range<S: PageSize>(virtual_address: usize, count: usize) -> PageIter<S> {
let first_page = Page::<S>::including_address(virtual_address);
let last_page = Page::<S>::including_address(virtual_address + (count - 1) * S::SIZE);
Page::range(first_page, last_page)
}
let range = get_page_range(viraddr, npages);
ROOT_PAGETABLE.lock().map_pages(range, phyaddr, PageTableEntryFlags::from_bits_truncate(bits), do_ipi > 0);
0
pub fn map<S: PageSize>(virtual_address: usize, physical_address: usize, count: usize, flags: PageTableEntryFlags, do_ipi: bool) {
let range = get_page_range::<S>(virtual_address, count);
ROOT_PAGETABLE.lock().map_pages(range, physical_address, flags, do_ipi);
}
pub fn page_table_entry<S: PageSize>(virtual_address: usize) -> Option<PageTableEntry> {
let page = Page::<S>::including_address(virtual_address);
ROOT_PAGETABLE.lock().get_page_table_entry(page)
}
#[no_mangle]
pub extern "C" fn getpagesize() -> i32 {
BasePageSize::SIZE as i32
}
/*#[no_mangle]
pub extern "C" fn page_unmap(viraddr: usize, npages: usize) -> i32 {
debug!("page_unmap({:#X}, {})", viraddr, npages);
......@@ -644,14 +671,13 @@ pub extern "C" fn page_unmap(viraddr: usize, npages: usize) -> i32 {
}
0
}
}*/
/// Add read-only, execute-disable identity page mappings for the supplied
/// Multiboot information and command line.
pub fn map_boot_info() {
pub fn init() {
// Add read-only, execute-disable identity page mappings for the supplied Multiboot information and command line.
unsafe {
if mb_info > 0 {
let page = Page::<BasePageSize>::including_address(mb_info);
let page = Page::<BasePageSize>::including_address(mb_info as usize);
ROOT_PAGETABLE.lock().map_page(page, page.address(), PageTableEntryFlags::EXECUTE_DISABLE);
}
......@@ -662,26 +688,17 @@ pub fn map_boot_info() {
ROOT_PAGETABLE.lock().map_pages(range, first_page.address(), PageTableEntryFlags::EXECUTE_DISABLE, false);
}
}
}
#[no_mangle]
pub unsafe extern "C" fn page_init() -> i32 {
if !runtime_osinit.is_null() {
info!("Detected Go runtime! HermitCore will return zeroed pages.");
}
// Install the Page Fault handler.
irq::set_handler(14, page_fault_handler);
0
}
#[no_mangle]
/*#[no_mangle]
pub unsafe extern "C" fn virt_to_phys(addr: usize) -> usize {
debug!("virt_to_phys({:#X})", addr);
// HACK: Currently, we use 2MiB pages only for the kernel.
let kernel_end: usize = Page::<LargePageSize>::including_address(&kernel_start as *const u8 as usize + image_size).address().saturating_add(LargePageSize::SIZE);
if addr >= (&kernel_start as *const u8 as usize) && addr < kernel_end {
// HACK: Currently, we use 2 MiB pages only for the kernel.
if addr >= mm::kernel_start_address() && addr < mm::kernel_end_address() {
let page = Page::<LargePageSize>::including_address(addr);
let address = ROOT_PAGETABLE.lock().get_page_table_entry(page).expect("Entry not present").address();
let offset = addr & (LargePageSize::SIZE - 1);
......@@ -692,4 +709,4 @@ pub unsafe extern "C" fn virt_to_phys(addr: usize) -> usize {
let offset = addr & (BasePageSize::SIZE - 1);
address | offset
}
}
}*/
// Copyright (c) 2017 Colin Finck, RWTH Aachen University
//
// MIT License
//
// Permission is hereby granted, free of charge, to any person obtaining
// a copy of this software and associated documentation files (the
// "Software"), to deal in the Software without restriction, including
// without limitation the rights to use, copy, modify, merge, publish,
// distribute, sublicense, and/or sell copies of the Software, and to
// permit persons to whom the Software is furnished to do so, subject to
// the following conditions:
//
// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
// LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
// OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
// WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
use arch::x86_64::mm::paging::{BasePageSize, PageSize};
use collections::{FreeList, FreeListEntry};
use core::{mem, slice};
use mm;
use multiboot;
use synch::spinlock::*;
extern "C" {
static limit: usize;
static mb_info: multiboot::PAddr;
}
static PHYSICAL_FREE_LIST: SpinlockIrqSave<FreeList> = SpinlockIrqSave::new(FreeList::new());
fn paddr_to_slice<'a>(p: multiboot::PAddr, sz: usize) -> Option<&'a [u8]> {
unsafe {
let ptr = mem::transmute(p);
Some(slice::from_raw_parts(ptr, sz))
}
}
fn detect_from_multiboot_info() -> bool {
if unsafe { mb_info } == 0 {
return false;
}
let mb = unsafe { multiboot::Multiboot::new(mb_info, paddr_to_slice).unwrap() };
let all_regions = mb.memory_regions().expect("No memory regions supplied by multiboot information!");
let ram_regions = all_regions.filter(|m|
m.memory_type() == multiboot::MemoryType::Available &&
m.base_address() + m.length() > mm::kernel_end_address() as u64
);
let mut locked_list = PHYSICAL_FREE_LIST.lock();
for m in ram_regions {
let start_address = if m.base_address() <= mm::kernel_start_address() as u64 {
mm::kernel_end_address()
} else {
m.base_address() as usize
};
let entry = FreeListEntry {
start: start_address,
end: (m.base_address() + m.length()) as usize
};
locked_list.list.push(entry);
}
true
}
fn detect_from_limits() -> bool {
if unsafe { limit } == 0 {
return false;
}
let entry = FreeListEntry {
start: mm::kernel_end_address(),
end: unsafe { limit }
};
PHYSICAL_FREE_LIST.lock().list.push(entry);
true
}
pub fn init() {
detect_from_multiboot_info() || detect_from_limits() || panic!("Could not detect RAM");
}
pub fn allocate(size: usize) -> usize {
assert!(size & (BasePageSize::SIZE - 1) == 0, "Size is not a multiple of 4 KiB (size = {:#X})", size);
let result = PHYSICAL_FREE_LIST.lock().allocate(size);
assert!(result.is_ok(), "Could not allocate {:#X} bytes of physical memory", size);
result.unwrap()
}
pub fn deallocate(physical_address: usize, size: usize) {
assert!(physical_address >= mm::kernel_end_address(), "Physical address {:#X} < KERNEL_END_ADDRESS", physical_address);
assert!(size & (BasePageSize::SIZE - 1) == 0, "Size is not a multiple of 4 KiB (size = {:#X})", size);
let result = PHYSICAL_FREE_LIST.lock().deallocate(physical_address, size);
assert!(result.is_ok(), "Could not deallocate physical memory address {:#X} with size {:#X}", physical_address, size);
}
// Copyright (c) 2017 Colin Finck, RWTH Aachen University
//
// MIT License
//
// Permission is hereby granted, free of charge, to any person obtaining
// a copy of this software and associated documentation files (the
// "Software"), to deal in the Software without restriction, including
// without limitation the rights to use, copy, modify, merge, publish,
// distribute, sublicense, and/or sell copies of the Software, and to
// permit persons to whom the Software is furnished to do so, subject to
// the following conditions:
//
// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
// LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
// OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
// WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
use arch::x86_64::mm::paging::{BasePageSize, PageSize};
use collections::{FreeList, FreeListEntry};
use mm;
use synch::spinlock::*;
static KERNEL_FREE_LIST: SpinlockIrqSave<FreeList> = SpinlockIrqSave::new(FreeList::new());
/// End of the virtual memory address space reserved for kernel memory (1 GiB).
/// This also marks the start of the virtual memory address space reserved for the task heap.
const KERNEL_VIRTUAL_MEMORY_END: usize = 0x4000_0000;
/// End of the virtual memory address space reserved for task memory (128 TiB).
/// This is the maximum contiguous virtual memory area possible with current x86-64 CPUs, which only support 48-bit
/// linear addressing (in two 47-bit areas).
const TASK_VIRTUAL_MEMORY_END: usize = 0x8000_0000_0000;
pub fn init() {
let entry = FreeListEntry {
start: mm::kernel_end_address(),
end: KERNEL_VIRTUAL_MEMORY_END
};
KERNEL_FREE_LIST.lock().list.push(entry);
}
pub fn allocate(size: usize) -> usize {
assert!(size & (BasePageSize::SIZE - 1) == 0, "Size is not a multiple of 4 KiB (size = {:#X})", size);
let result = KERNEL_FREE_LIST.lock().allocate(size);
assert!(result.is_ok(), "Could not allocate {:#X} bytes of virtual memory", size);
result.unwrap()
}
pub fn deallocate(virtual_address: usize, size: usize) {
assert!(virtual_address >= mm::kernel_end_address(), "Virtual address {:#X} < KERNEL_END_ADDRESS", virtual_address);
assert!(virtual_address < KERNEL_VIRTUAL_MEMORY_END, "Virtual address {:#X} >= KERNEL_VIRTUAL_MEMORY_END", virtual_address);
assert!(size & (BasePageSize::SIZE - 1) == 0, "Size is not a multiple of 4 KiB (size = {:#X})", size);
let result = KERNEL_FREE_LIST.lock().deallocate(virtual_address, size);
assert!(result.is_ok(), "Could not deallocate virtual memory address {:#X} with size {:#X}", virtual_address, size);
}
......@@ -22,9 +22,6 @@
// OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
// WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
use logging::*;
// MODULES
pub mod gdt;
pub mod idt;
pub mod irq;
......@@ -35,18 +32,25 @@ pub mod pic;
pub mod pit;
pub mod processor;
use logging::*;
extern "C" {
static image_size: usize;
static kernel_start: u8;
fn memory_init() -> i32;
fn signal_init();
}
// FUNCTIONS
pub fn system_init() {
gdt::install();
idt::install();
processor::detect_features();
processor::configure();
mm::paging::map_boot_info();
::mm::init();
pic::remap();
pit::deinit();
isr::install();
......@@ -61,7 +65,6 @@ pub fn system_init() {
}
unsafe {
memory_init();
signal_init();
}
}
......@@ -658,18 +658,18 @@ pub fn update_ticks() {
}
}
#[no_mangle]
/*#[no_mangle]
pub extern "C" fn cpu_detection() -> i32 {
configure();
0
}
}*/
#[no_mangle]
pub extern "C" fn get_cpu_frequency() -> u32 {
unsafe { CPU_FREQUENCY.get() as u32 }