diff --git a/0C_virtual_memory/README.md b/0C_virtual_memory/README.md index 2c323e2a..2d713220 100644 --- a/0C_virtual_memory/README.md +++ b/0C_virtual_memory/README.md @@ -65,3 +65,18 @@ kernel8::mmu::init::h53df3fab6e51e098: 80778: 0d a2 18 d5 msr MAIR_EL1, x13 ... ``` + +## Output + +```console +ferris@box:~$ make raspboot + +[0] UART is live! +[1] Press a key to continue booting... Greetings fellow Rustacean! +[i] MMU: 4 KiB granule supported! +[i] MMU: Up to 40 Bit physical address range supported! +[2] MMU online. + +Writing through the virtual mapping at 0x00000000001FF000. + +``` diff --git a/0C_virtual_memory/kernel8 b/0C_virtual_memory/kernel8 index 8d326c65..44757979 100755 Binary files a/0C_virtual_memory/kernel8 and b/0C_virtual_memory/kernel8 differ diff --git a/0C_virtual_memory/kernel8.img b/0C_virtual_memory/kernel8.img index 69b1b215..93ff1e1c 100755 Binary files a/0C_virtual_memory/kernel8.img and b/0C_virtual_memory/kernel8.img differ diff --git a/0C_virtual_memory/src/gpio.rs b/0C_virtual_memory/src/gpio.rs index 608ba532..7affea08 100644 --- a/0C_virtual_memory/src/gpio.rs +++ b/0C_virtual_memory/src/gpio.rs @@ -1,7 +1,7 @@ /* * MIT License * - * Copyright (c) 2018 Andre Richter + * Copyright (c) 2018-2019 Andre Richter * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -22,7 +22,6 @@ * SOFTWARE. */ -use super::MMIO_BASE; use core::ops; use register::{mmio::ReadWrite, register_bitfields}; @@ -67,8 +66,6 @@ register_bitfields! { ] } -const GPIO_BASE: u32 = MMIO_BASE + 0x200_000; - #[allow(non_snake_case)] #[repr(C)] pub struct RegisterBlock { @@ -99,23 +96,25 @@ pub struct RegisterBlock { } /// Public interface to the GPIO MMIO area -pub struct GPIO; +pub struct GPIO { + base_addr: usize, +} impl ops::Deref for GPIO { type Target = RegisterBlock; fn deref(&self) -> &Self::Target { - unsafe { &*Self::ptr() } + unsafe { &*self.ptr() } } } impl GPIO { - pub fn new() -> GPIO { - GPIO + pub fn new(base_addr: usize) -> GPIO { + GPIO { base_addr } } /// Returns a pointer to the register block - fn ptr() -> *const RegisterBlock { - GPIO_BASE as *const _ + fn ptr(&self) -> *const RegisterBlock { + self.base_addr as *const _ } } diff --git a/0C_virtual_memory/src/main.rs b/0C_virtual_memory/src/main.rs index bd646dce..45ad5cb9 100644 --- a/0C_virtual_memory/src/main.rs +++ b/0C_virtual_memory/src/main.rs @@ -26,21 +26,19 @@ #![no_main] #![feature(range_contains)] -const MMIO_BASE: u32 = 0x3F00_0000; - mod delays; mod gpio; mod mbox; -mod mmu; +mod memory; mod uart; fn kernel_entry() -> ! { - let gpio = gpio::GPIO::new(); - let mut mbox = mbox::Mbox::new(); + let gpio = gpio::GPIO::new(memory::map::physical::GPIO_BASE); + let mut mbox = mbox::Mbox::new(memory::map::physical::VIDEOCORE_MBOX_BASE); { // Before the MMU is live, instantiate a UART driver with the physical address - let uart = uart::Uart::new(uart::UART_PHYS_BASE); + let uart = uart::Uart::new(memory::map::physical::UART_BASE); // set up serial console match uart.init(&mut mbox, &gpio) { @@ -54,20 +52,24 @@ fn kernel_entry() -> ! { uart.getc(); uart.puts("Greetings fellow Rustacean!\n"); - mmu::print_features(&uart); + memory::mmu::print_features(&uart); - uart.puts("[2] Switching MMU on now... "); + match unsafe { memory::mmu::init() } { + Err(s) => { + uart.puts("[2][Error] MMU: "); + uart.puts(s); + uart.puts("\n"); + } + Ok(()) => uart.puts("[2] MMU online.\n"), + } } // After this closure, the UART instance is not valid anymore. - unsafe { mmu::init() }; - - // Instantiate a new UART using the virtual mapping in the second 2 MiB - // block. No need to init() again, though. - const UART_VIRT_BASE: u32 = 2 * 1024 * 1024 + 0x1000; - let uart = uart::Uart::new(UART_VIRT_BASE); + // Instantiate a new UART using the remapped address. No need to init() + // again, though. + let uart = uart::Uart::new(memory::map::virt::REMAPPED_UART_BASE); - uart.puts("MMU is live \\o/\n\nWriting through the virtual mapping at 0x"); - uart.hex(u64::from(UART_VIRT_BASE)); + uart.puts("\nWriting through the virtual mapping at 0x"); + uart.hex(memory::map::virt::REMAPPED_UART_BASE as u64); uart.puts(".\n"); // echo everything back diff --git a/0C_virtual_memory/src/mbox.rs b/0C_virtual_memory/src/mbox.rs index aeae88bb..2e4bf0ad 100644 --- a/0C_virtual_memory/src/mbox.rs +++ b/0C_virtual_memory/src/mbox.rs @@ -1,7 +1,7 @@ /* * MIT License * - * Copyright (c) 2018 Andre Richter + * Copyright (c) 2018-2019 Andre Richter * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -22,7 +22,6 @@ * SOFTWARE. */ -use super::MMIO_BASE; use core::ops; use cortex_a::asm; use register::{ @@ -39,8 +38,6 @@ register_bitfields! { ] } -const VIDEOCORE_MBOX: u32 = MMIO_BASE + 0xB880; - #[allow(non_snake_case)] #[repr(C)] pub struct RegisterBlock { @@ -89,6 +86,7 @@ pub struct Mbox { // The address for buffer needs to be 16-byte aligned so that the // Videcore can handle it properly. pub buffer: [u32; 36], + base_addr: usize, } /// Deref to RegisterBlock @@ -105,18 +103,21 @@ impl ops::Deref for Mbox { type Target = RegisterBlock; fn deref(&self) -> &Self::Target { - unsafe { &*Self::ptr() } + unsafe { &*self.ptr() } } } impl Mbox { - pub fn new() -> Mbox { - Mbox { buffer: [0; 36] } + pub fn new(base_addr: usize) -> Mbox { + Mbox { + buffer: [0; 36], + base_addr, + } } /// Returns a pointer to the register block - fn ptr() -> *const RegisterBlock { - VIDEOCORE_MBOX as *const _ + fn ptr(&self) -> *const RegisterBlock { + self.base_addr as *const _ } /// Make a mailbox call. Returns Err(MboxError) on failure, Ok(()) success diff --git a/0C_virtual_memory/src/memory.rs b/0C_virtual_memory/src/memory.rs new file mode 100644 index 00000000..c2135539 --- /dev/null +++ b/0C_virtual_memory/src/memory.rs @@ -0,0 +1,221 @@ +/* + * MIT License + * + * Copyright (c) 2019 Andre Richter + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +use core::ops::RangeInclusive; + +pub mod mmu; + +/// System memory map. +#[rustfmt::skip] +pub mod map { + pub const START: usize = 0x0000_0000; + pub const END: usize = 0x3FFF_FFFF; + + pub mod physical { + pub const MMIO_BASE: usize = 0x3F00_0000; + pub const VIDEOCORE_MBOX_BASE: usize = MMIO_BASE + 0x0000_B880; + pub const GPIO_BASE: usize = MMIO_BASE + 0x0020_0000; + pub const UART_BASE: usize = MMIO_BASE + 0x0020_1000; + pub const MMIO_END: usize = super::END; + } + + pub mod virt { + pub const KERN_STACK_START: usize = super::START; + pub const KERN_STACK_END: usize = 0x0007_FFFF; + + // The last 4 KiB slot in the first 2 MiB + pub const REMAPPED_UART_BASE: usize = 0x001F_F000; + pub const REMAPPED_UART_END: usize = 0x001F_FFFF; + } +} + +/// Types used for compiling the virtual memory layout of the kernel using +/// address ranges. +pub mod kernel_mem_range { + use core::ops::RangeInclusive; + + #[derive(Copy, Clone)] + pub enum MemAttributes { + CacheableDRAM, + Device, + } + + #[derive(Copy, Clone)] + pub enum AccessPermissions { + ReadOnly, + ReadWrite, + } + + #[derive(Copy, Clone)] + pub enum Translation { + Identity, + Offset(usize), + } + + #[derive(Copy, Clone)] + pub struct AttributeFields { + pub mem_attributes: MemAttributes, + pub acc_perms: AccessPermissions, + pub execute_never: bool, + } + + impl Default for AttributeFields { + fn default() -> AttributeFields { + AttributeFields { + mem_attributes: MemAttributes::CacheableDRAM, + acc_perms: AccessPermissions::ReadWrite, + execute_never: true, + } + } + } + + pub struct Descriptor { + pub virtual_range: fn() -> RangeInclusive, + pub translation: Translation, + pub attribute_fields: AttributeFields, + } +} + +use kernel_mem_range::*; + +/// A virtual memory layout that is agnostic of the paging granularity that the +/// hardware MMU will use. +/// +/// Contains only special ranges, aka anything that is _not_ normal cacheable +/// DRAM. +static KERNEL_VIRTUAL_LAYOUT: [Descriptor; 5] = [ + // Kernel stack + Descriptor { + virtual_range: || { + RangeInclusive::new(map::virt::KERN_STACK_START, map::virt::KERN_STACK_END) + }, + translation: Translation::Identity, + attribute_fields: AttributeFields { + mem_attributes: MemAttributes::CacheableDRAM, + acc_perms: AccessPermissions::ReadWrite, + execute_never: true, + }, + }, + // Kernel code and RO data + Descriptor { + virtual_range: || { + // Using the linker script, we ensure that the RO area is consecutive and 4 + // KiB aligned, and we export the boundaries via symbols: + // + // [__ro_start, __ro_end) + extern "C" { + // The inclusive start of the read-only area, aka the address of the + // first byte of the area. + static __ro_start: u64; + + // The exclusive end of the read-only area, aka the address of + // the first byte _after_ the RO area. + static __ro_end: u64; + } + + unsafe { + // Notice the subtraction to turn the exclusive end into an + // inclusive end + RangeInclusive::new( + &__ro_start as *const _ as usize, + &__ro_end as *const _ as usize - 1, + ) + } + }, + translation: Translation::Identity, + attribute_fields: AttributeFields { + mem_attributes: MemAttributes::CacheableDRAM, + acc_perms: AccessPermissions::ReadOnly, + execute_never: false, + }, + }, + // Kernel data and BSS + Descriptor { + virtual_range: || { + extern "C" { + static __ro_end: u64; + static __bss_end: u64; + } + + unsafe { + RangeInclusive::new( + &__ro_end as *const _ as usize, + &__bss_end as *const _ as usize - 1, + ) + } + }, + translation: Translation::Identity, + attribute_fields: AttributeFields { + mem_attributes: MemAttributes::CacheableDRAM, + acc_perms: AccessPermissions::ReadWrite, + execute_never: true, + }, + }, + // Remapped UART + Descriptor { + virtual_range: || { + RangeInclusive::new(map::virt::REMAPPED_UART_BASE, map::virt::REMAPPED_UART_END) + }, + translation: Translation::Offset(map::physical::UART_BASE), + attribute_fields: AttributeFields { + mem_attributes: MemAttributes::Device, + acc_perms: AccessPermissions::ReadWrite, + execute_never: true, + }, + }, + // Device MMIO + Descriptor { + virtual_range: || RangeInclusive::new(map::physical::MMIO_BASE, map::physical::MMIO_END), + translation: Translation::Identity, + attribute_fields: AttributeFields { + mem_attributes: MemAttributes::Device, + acc_perms: AccessPermissions::ReadWrite, + execute_never: true, + }, + }, +]; + +/// For a given virtual address, find and return the output address and +/// according attributes. +/// +/// If the address is not covered in VIRTUAL_LAYOUT, return a default for normal +/// cacheable DRAM. +fn get_virt_addr_properties(virt_addr: usize) -> Result<(usize, AttributeFields), &'static str> { + if virt_addr > map::END { + return Err("Address out of range."); + } + + for i in KERNEL_VIRTUAL_LAYOUT.iter() { + if (i.virtual_range)().contains(&virt_addr) { + let output_addr = match i.translation { + Translation::Identity => virt_addr, + Translation::Offset(a) => a + (virt_addr - (i.virtual_range)().start()), + }; + + return Ok((output_addr, i.attribute_fields)); + } + } + + Ok((virt_addr, AttributeFields::default())) +} diff --git a/0C_virtual_memory/src/memory/mmu.rs b/0C_virtual_memory/src/memory/mmu.rs new file mode 100644 index 00000000..e6628990 --- /dev/null +++ b/0C_virtual_memory/src/memory/mmu.rs @@ -0,0 +1,355 @@ +/* + * MIT License + * + * Copyright (c) 2018-2019 Andre Richter + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +use crate::memory::{get_virt_addr_properties, AttributeFields}; +use crate::uart; +use cortex_a::{barrier, regs::*}; +use register::register_bitfields; + +/// Parse the ID_AA64MMFR0_EL1 register for runtime information about supported +/// MMU features. +pub fn print_features(uart: &uart::Uart) { + let mmfr = ID_AA64MMFR0_EL1.extract(); + + if let Some(ID_AA64MMFR0_EL1::TGran4::Value::Supported) = + mmfr.read_as_enum(ID_AA64MMFR0_EL1::TGran4) + { + uart.puts("[i] MMU: 4 KiB granule supported!\n"); + } + + if let Some(ID_AA64MMFR0_EL1::PARange::Value::Bits_40) = + mmfr.read_as_enum(ID_AA64MMFR0_EL1::PARange) + { + uart.puts("[i] MMU: Up to 40 Bit physical address range supported!\n"); + } +} + +register_bitfields! {u64, + // AArch64 Reference Manual page 2150 + STAGE1_DESCRIPTOR [ + /// Privileged execute-never + PXN OFFSET(53) NUMBITS(1) [ + False = 0, + True = 1 + ], + + /// Various address fields, depending on use case + LVL2_OUTPUT_ADDR_4KiB OFFSET(21) NUMBITS(27) [], // [47:21] + NEXT_LVL_TABLE_ADDR_4KiB OFFSET(12) NUMBITS(36) [], // [47:12] + + /// Access flag + AF OFFSET(10) NUMBITS(1) [ + False = 0, + True = 1 + ], + + /// Shareability field + SH OFFSET(8) NUMBITS(2) [ + OuterShareable = 0b10, + InnerShareable = 0b11 + ], + + /// Access Permissions + AP OFFSET(6) NUMBITS(2) [ + RW_EL1 = 0b00, + RW_EL1_EL0 = 0b01, + RO_EL1 = 0b10, + RO_EL1_EL0 = 0b11 + ], + + /// Memory attributes index into the MAIR_EL1 register + AttrIndx OFFSET(2) NUMBITS(3) [], + + TYPE OFFSET(1) NUMBITS(1) [ + Block = 0, + Table = 1 + ], + + VALID OFFSET(0) NUMBITS(1) [ + False = 0, + True = 1 + ] + ] +} + +const FOUR_KIB: usize = 4 * 1024; +const FOUR_KIB_SHIFT: usize = 12; // log2(4 * 1024) + +const TWO_MIB: usize = 2 * 1024 * 1024; +const TWO_MIB_SHIFT: usize = 21; // log2(2 * 1024 * 1024) + +/// A descriptor pointing to the next page table. +struct TableDescriptor(register::FieldValue); + +impl TableDescriptor { + fn new(next_lvl_table_addr: usize) -> Result { + if next_lvl_table_addr % FOUR_KIB != 0 { + return Err("TableDescriptor: Address is not 4 KiB aligned."); + } + + let shifted = next_lvl_table_addr >> FOUR_KIB_SHIFT; + + Ok(TableDescriptor( + STAGE1_DESCRIPTOR::VALID::True + + STAGE1_DESCRIPTOR::TYPE::Table + + STAGE1_DESCRIPTOR::NEXT_LVL_TABLE_ADDR_4KiB.val(shifted as u64), + )) + } + + fn value(&self) -> u64 { + self.0.value + } +} + +/// A function that maps the generic memory range attributes to HW-specific +/// attributes of the MMU. +fn into_mmu_attributes( + attribute_fields: AttributeFields, +) -> register::FieldValue { + use crate::memory::{AccessPermissions, MemAttributes}; + + // Memory attributes + let mut desc = match attribute_fields.mem_attributes { + MemAttributes::CacheableDRAM => { + STAGE1_DESCRIPTOR::SH::InnerShareable + STAGE1_DESCRIPTOR::AttrIndx.val(mair::NORMAL) + } + MemAttributes::Device => { + STAGE1_DESCRIPTOR::SH::OuterShareable + STAGE1_DESCRIPTOR::AttrIndx.val(mair::DEVICE) + } + }; + + // Access Permissions + desc += match attribute_fields.acc_perms { + AccessPermissions::ReadOnly => STAGE1_DESCRIPTOR::AP::RO_EL1, + AccessPermissions::ReadWrite => STAGE1_DESCRIPTOR::AP::RW_EL1, + }; + + // Execute Never + desc += if attribute_fields.execute_never { + STAGE1_DESCRIPTOR::PXN::True + } else { + STAGE1_DESCRIPTOR::PXN::False + }; + + desc +} + +/// A Level2 block descriptor with 2 MiB aperture. +/// +/// The output points to physical memory. +struct Lvl2BlockDescriptor(register::FieldValue); + +impl Lvl2BlockDescriptor { + fn new( + output_addr: usize, + attribute_fields: AttributeFields, + ) -> Result { + if output_addr % TWO_MIB != 0 { + return Err("BlockDescriptor: Address is not 2 MiB aligned."); + } + + let shifted = output_addr >> TWO_MIB_SHIFT; + + Ok(Lvl2BlockDescriptor( + STAGE1_DESCRIPTOR::VALID::True + + STAGE1_DESCRIPTOR::AF::True + + into_mmu_attributes(attribute_fields) + + STAGE1_DESCRIPTOR::TYPE::Block + + STAGE1_DESCRIPTOR::LVL2_OUTPUT_ADDR_4KiB.val(shifted as u64), + )) + } + + fn value(&self) -> u64 { + self.0.value + } +} + +/// A page descriptor with 4 KiB aperture. +/// +/// The output points to physical memory. +struct PageDescriptor(register::FieldValue); + +impl PageDescriptor { + fn new( + output_addr: usize, + attribute_fields: AttributeFields, + ) -> Result { + if output_addr % FOUR_KIB != 0 { + return Err("PageDescriptor: Address is not 4 KiB aligned."); + } + + let shifted = output_addr >> FOUR_KIB_SHIFT; + + Ok(PageDescriptor( + STAGE1_DESCRIPTOR::VALID::True + + STAGE1_DESCRIPTOR::AF::True + + into_mmu_attributes(attribute_fields) + + STAGE1_DESCRIPTOR::TYPE::Table + + STAGE1_DESCRIPTOR::NEXT_LVL_TABLE_ADDR_4KiB.val(shifted as u64), + )) + } + + fn value(&self) -> u64 { + self.0.value + } +} + +/// Constants for indexing the MAIR_EL1. +#[allow(dead_code)] +mod mair { + pub const DEVICE: u64 = 0; + pub const NORMAL: u64 = 1; +} + +/// Setup function for the MAIR_EL1 register. +fn set_up_mair() { + // Define the three memory types that we will map. Cacheable and + // non-cacheable normal DRAM, and device. + MAIR_EL1.write( + // Attribute 1 + MAIR_EL1::Attr1_HIGH::Memory_OuterWriteBack_NonTransient_ReadAlloc_WriteAlloc + + MAIR_EL1::Attr1_LOW_MEMORY::InnerWriteBack_NonTransient_ReadAlloc_WriteAlloc + + // Attribute 0 + + MAIR_EL1::Attr0_HIGH::Device + + MAIR_EL1::Attr0_LOW_DEVICE::Device_nGnRE, + ); +} + +trait BaseAddr { + fn base_addr_u64(&self) -> u64; + fn base_addr_usize(&self) -> usize; +} + +impl BaseAddr for [u64; 512] { + fn base_addr_u64(&self) -> u64 { + self as *const u64 as u64 + } + + fn base_addr_usize(&self) -> usize { + self as *const u64 as usize + } +} + +const NUM_ENTRIES_4KIB: usize = 512; + +// A wrapper struct is needed here so that the align attribute can be used. +#[repr(C)] +#[repr(align(4096))] +struct PageTable { + entries: [u64; NUM_ENTRIES_4KIB], +} + +/// The LVL2 page table containng the 2 MiB entries. +static mut LVL2_TABLE: PageTable = PageTable { + entries: [0; NUM_ENTRIES_4KIB], +}; + +/// The LVL3 page table containing the 4 KiB entries. +/// +/// The first entry of the LVL2_TABLE will forward to this table. +static mut LVL3_TABLE: PageTable = PageTable { + entries: [0; NUM_ENTRIES_4KIB], +}; + +/// Set up identity mapped page tables for the first 1 GiB of address space. +/// +/// The first 2 MiB are 4 KiB granule, the rest 2 MiB. +pub unsafe fn init() -> Result<(), &'static str> { + // Prepare the memory attribute indirection register. + set_up_mair(); + + // Point the first 2 MiB of virtual addresses to the follow-up LVL3 + // page-table. + LVL2_TABLE.entries[0] = match TableDescriptor::new(LVL3_TABLE.entries.base_addr_usize()) { + Err(s) => return Err(s), + Ok(d) => d.value(), + }; + + // Fill the rest of the LVL2 (2 MiB) entries as block descriptors. + // + // Notice the skip(1) which makes the iteration start at the second 2 MiB + // block (0x20_0000). + for (block_descriptor_nr, entry) in LVL2_TABLE.entries.iter_mut().enumerate().skip(1) { + let virt_addr = block_descriptor_nr << TWO_MIB_SHIFT; + + let (output_addr, attribute_fields) = match get_virt_addr_properties(virt_addr) { + Err(s) => return Err(s), + Ok((a, b)) => (a, b), + }; + + let block_desc = match Lvl2BlockDescriptor::new(output_addr, attribute_fields) { + Err(s) => return Err(s), + Ok(desc) => desc, + }; + + *entry = block_desc.value(); + } + + // Finally, fill the single LVL3 table (4 KiB granule). + for (page_descriptor_nr, entry) in LVL3_TABLE.entries.iter_mut().enumerate() { + let virt_addr = page_descriptor_nr << FOUR_KIB_SHIFT; + + let (output_addr, attribute_fields) = match get_virt_addr_properties(virt_addr) { + Err(s) => return Err(s), + Ok((a, b)) => (a, b), + }; + + let page_desc = match PageDescriptor::new(output_addr, attribute_fields) { + Err(s) => return Err(s), + Ok(desc) => desc, + }; + + *entry = page_desc.value(); + } + + // Point to the LVL2 table base address in TTBR0. + TTBR0_EL1.set_baddr(LVL2_TABLE.entries.base_addr_u64()); + + // Configure various settings of stage 1 of the EL1 translation regime. + let ips = ID_AA64MMFR0_EL1.read(ID_AA64MMFR0_EL1::PARange); + TCR_EL1.write( + TCR_EL1::TBI0::Ignored + + TCR_EL1::IPS.val(ips) + + TCR_EL1::TG0::KiB_4 // 4 KiB granule + + TCR_EL1::SH0::Inner + + TCR_EL1::ORGN0::WriteBack_ReadAlloc_WriteAlloc_Cacheable + + TCR_EL1::IRGN0::WriteBack_ReadAlloc_WriteAlloc_Cacheable + + TCR_EL1::EPD0::EnableTTBR0Walks + + TCR_EL1::T0SZ.val(34), // Start walks at level 2 + ); + + // Switch the MMU on. + // + // First, force all previous changes to be seen before the MMU is enabled. + barrier::isb(barrier::SY); + + // Enable the MMU and turn on data and instruction caching. + SCTLR_EL1.modify(SCTLR_EL1::M::Enable + SCTLR_EL1::C::Cacheable + SCTLR_EL1::I::Cacheable); + + // Force MMU init to complete before next instruction + barrier::isb(barrier::SY); + + Ok(()) +} diff --git a/0C_virtual_memory/src/mmu.rs b/0C_virtual_memory/src/mmu.rs deleted file mode 100644 index ef352461..00000000 --- a/0C_virtual_memory/src/mmu.rs +++ /dev/null @@ -1,273 +0,0 @@ -/* - * MIT License - * - * Copyright (c) 2018-2019 Andre Richter - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -use super::uart; -use cortex_a::{barrier, regs::*}; -use register::register_bitfields; - -/// Parse the ID_AA64MMFR0_EL1 register for runtime information about supported -/// MMU features. -pub fn print_features(uart: &uart::Uart) { - let mmfr = ID_AA64MMFR0_EL1.extract(); - - if let Some(ID_AA64MMFR0_EL1::TGran4::Value::Supported) = - mmfr.read_as_enum(ID_AA64MMFR0_EL1::TGran4) - { - uart.puts("[i] MMU: 4 KiB granule supported!\n"); - } - - if let Some(ID_AA64MMFR0_EL1::PARange::Value::Bits_40) = - mmfr.read_as_enum(ID_AA64MMFR0_EL1::PARange) - { - uart.puts("[i] MMU: Up to 40 Bit physical address range supported!\n"); - } -} - -register_bitfields! {u64, - // AArch64 Reference Manual page 2150 - STAGE1_DESCRIPTOR [ - /// Execute-never - XN OFFSET(54) NUMBITS(1) [ - False = 0, - True = 1 - ], - - /// Various address fields, depending on use case - LVL2_OUTPUT_ADDR_4KiB OFFSET(21) NUMBITS(27) [], // [47:21] - NEXT_LVL_TABLE_ADDR_4KiB OFFSET(12) NUMBITS(36) [], // [47:12] - - /// Access flag - AF OFFSET(10) NUMBITS(1) [ - False = 0, - True = 1 - ], - - /// Shareability field - SH OFFSET(8) NUMBITS(2) [ - OuterShareable = 0b10, - InnerShareable = 0b11 - ], - - /// Access Permissions - AP OFFSET(6) NUMBITS(2) [ - RW_EL1 = 0b00, - RW_EL1_EL0 = 0b01, - RO_EL1 = 0b10, - RO_EL1_EL0 = 0b11 - ], - - /// Memory attributes index into the MAIR_EL1 register - AttrIndx OFFSET(2) NUMBITS(3) [], - - TYPE OFFSET(1) NUMBITS(1) [ - Block = 0, - Table = 1 - ], - - VALID OFFSET(0) NUMBITS(1) [ - False = 0, - True = 1 - ] - ] -} - -trait BaseAddr { - fn base_addr(&self) -> u64; -} - -impl BaseAddr for [u64; 512] { - fn base_addr(&self) -> u64 { - self as *const u64 as u64 - } -} - -const NUM_ENTRIES_4KIB: usize = 512; - -// We need a wrapper struct here so that we can make use of the align attribute. -#[repr(C)] -#[repr(align(4096))] -struct PageTable { - entries: [u64; NUM_ENTRIES_4KIB], -} - -static mut LVL2_TABLE: PageTable = PageTable { - entries: [0; NUM_ENTRIES_4KIB], -}; -static mut SINGLE_LVL3_TABLE: PageTable = PageTable { - entries: [0; NUM_ENTRIES_4KIB], -}; - -/// Set up identity mapped page tables for the first 1 GiB of address space. -pub unsafe fn init() { - // First, define the two memory types that we will map. Cacheable normal DRAM and - // device. - MAIR_EL1.write( - // Attribute 1 - MAIR_EL1::Attr1_HIGH::Device - + MAIR_EL1::Attr1_LOW_DEVICE::Device_nGnRE - // Attribute 0 - + MAIR_EL1::Attr0_HIGH::Memory_OuterWriteBack_NonTransient_ReadAlloc_WriteAlloc - + MAIR_EL1::Attr0_LOW_MEMORY::InnerWriteBack_NonTransient_ReadAlloc_WriteAlloc, - ); - - // Descriptive consts for indexing into the correct MAIR_EL1 attributes. - mod mair { - pub const NORMAL: u64 = 0; - pub const DEVICE: u64 = 1; - } - - // The first 2 MiB. - // - // Set up the first LVL2 entry, pointing to the base address of a follow-up - // table containing 4 KiB pages. - // - // 0x0000_0000_0000_0000 | - // |> 2 MiB - // 0x0000_0000_001F_FFFF | - let lvl3_base: u64 = SINGLE_LVL3_TABLE.entries.base_addr() >> 12; - LVL2_TABLE.entries[0] = (STAGE1_DESCRIPTOR::VALID::True - + STAGE1_DESCRIPTOR::TYPE::Table - + STAGE1_DESCRIPTOR::NEXT_LVL_TABLE_ADDR_4KiB.val(lvl3_base)) - .value; - - // For educational purposes and fun, let the start of the second 2 MiB block - // point to the 2 MiB aperture which contains the UART's base address. - // - // 0x0000_0000_0020_0000 | - // |> 2 MiB - // 0x0000_0000_003F_FFFF | - let uart_phys_base: u64 = (uart::UART_PHYS_BASE >> 21).into(); - LVL2_TABLE.entries[1] = (STAGE1_DESCRIPTOR::VALID::True - + STAGE1_DESCRIPTOR::TYPE::Block - + STAGE1_DESCRIPTOR::AttrIndx.val(mair::DEVICE) - + STAGE1_DESCRIPTOR::AP::RW_EL1 - + STAGE1_DESCRIPTOR::SH::OuterShareable - + STAGE1_DESCRIPTOR::AF::True - + STAGE1_DESCRIPTOR::LVL2_OUTPUT_ADDR_4KiB.val(uart_phys_base) - + STAGE1_DESCRIPTOR::XN::True) - .value; - - // Fill the rest of the LVL2 (2 MiB) entries as block descriptors. - // - // Differentiate between - // - cacheable DRAM - // - device memory - // - // Ranges are stored in memory.rs. - // - // 0x0000_0000_0040_0000 | - // |> 1004 MiB cacheable DRAM - // 0x0000_0000_3EFF_FFFF | - // 0x0000_0000_3F00_0000 | - // |> 16 MiB device (MMIO) - // 0x0000_0000_4000_0000 | - let mmio_first_block_index: u64 = (super::MMIO_BASE >> 21).into(); - - let common = STAGE1_DESCRIPTOR::VALID::True - + STAGE1_DESCRIPTOR::TYPE::Block - + STAGE1_DESCRIPTOR::AP::RW_EL1 - + STAGE1_DESCRIPTOR::AF::True - + STAGE1_DESCRIPTOR::XN::True; - - // Notice the skip(2) which makes the iteration start at the third 2 MiB - // block (0x40_0000). - for (i, entry) in LVL2_TABLE.entries.iter_mut().enumerate().skip(2) { - let j: u64 = i as u64; - - let mem_attr = if j >= mmio_first_block_index { - STAGE1_DESCRIPTOR::SH::OuterShareable + STAGE1_DESCRIPTOR::AttrIndx.val(mair::DEVICE) - } else { - STAGE1_DESCRIPTOR::SH::InnerShareable + STAGE1_DESCRIPTOR::AttrIndx.val(mair::NORMAL) - }; - - *entry = (common + mem_attr + STAGE1_DESCRIPTOR::LVL2_OUTPUT_ADDR_4KiB.val(j)).value; - } - - // Finally, fill the single LVL3 table (4 KiB granule). Differentiate - // between code+RO and RW pages. - // - // Using the linker script, we ensure that the RO area is consecutive and 4 - // KiB aligned, and we export the boundaries via symbols. - extern "C" { - // The inclusive start of the read-only area, aka the address of the - // first byte of the area. - static mut __ro_start: u64; - - // The non-inclusive end of the read-only area, aka the address of the - // first byte _after_ the RO area. - static mut __ro_end: u64; - } - - const PAGESIZE: u64 = 4096; - let ro_first_page_index: u64 = &__ro_start as *const _ as u64 / PAGESIZE; - - // Notice the subtraction to calculate the last page index of the RO area - // and not the first page index after the RO area. - let ro_last_page_index: u64 = (&__ro_end as *const _ as u64 / PAGESIZE) - 1; - - let common = STAGE1_DESCRIPTOR::VALID::True - + STAGE1_DESCRIPTOR::TYPE::Table - + STAGE1_DESCRIPTOR::AttrIndx.val(mair::NORMAL) - + STAGE1_DESCRIPTOR::SH::InnerShareable - + STAGE1_DESCRIPTOR::AF::True; - - for (i, entry) in SINGLE_LVL3_TABLE.entries.iter_mut().enumerate() { - let j: u64 = i as u64; - - let mem_attr = if (ro_first_page_index..=ro_last_page_index).contains(&j) { - STAGE1_DESCRIPTOR::AP::RO_EL1 + STAGE1_DESCRIPTOR::XN::False - } else { - STAGE1_DESCRIPTOR::AP::RW_EL1 + STAGE1_DESCRIPTOR::XN::True - }; - - *entry = (common + mem_attr + STAGE1_DESCRIPTOR::NEXT_LVL_TABLE_ADDR_4KiB.val(j)).value; - } - - // Point to the LVL2 table base address in TTBR0. - TTBR0_EL1.set_baddr(LVL2_TABLE.entries.base_addr()); - - // Configure various settings of stage 1 of the EL1 translation regime. - let ips = ID_AA64MMFR0_EL1.read(ID_AA64MMFR0_EL1::PARange); - TCR_EL1.write( - TCR_EL1::TBI0::Ignored - + TCR_EL1::IPS.val(ips) - + TCR_EL1::TG0::KiB_4 // 4 KiB granule - + TCR_EL1::SH0::Inner - + TCR_EL1::ORGN0::WriteBack_ReadAlloc_WriteAlloc_Cacheable - + TCR_EL1::IRGN0::WriteBack_ReadAlloc_WriteAlloc_Cacheable - + TCR_EL1::EPD0::EnableTTBR0Walks - + TCR_EL1::T0SZ.val(34), // Start walks at level 2 - ); - - // Switch the MMU on. - // - // First, force all previous changes to be seen before the MMU is enabled. - barrier::isb(barrier::SY); - - // Enable the MMU and turn on data and instruction caching. - SCTLR_EL1.modify(SCTLR_EL1::M::Enable + SCTLR_EL1::C::Cacheable + SCTLR_EL1::I::Cacheable); - - // Force MMU init to complete before next instruction - barrier::isb(barrier::SY); -} diff --git a/0C_virtual_memory/src/uart.rs b/0C_virtual_memory/src/uart.rs index 2c0e8a03..9505be3a 100644 --- a/0C_virtual_memory/src/uart.rs +++ b/0C_virtual_memory/src/uart.rs @@ -1,7 +1,7 @@ /* * MIT License * - * Copyright (c) 2018 Andre Richter + * Copyright (c) 2018-2019 Andre Richter * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -22,7 +22,6 @@ * SOFTWARE. */ -use super::MMIO_BASE; use crate::delays; use crate::gpio; use crate::mbox; @@ -120,8 +119,6 @@ register_bitfields! { ] } -pub const UART_PHYS_BASE: u32 = MMIO_BASE + 0x20_1000; - #[allow(non_snake_case)] #[repr(C)] pub struct RegisterBlock { @@ -143,7 +140,7 @@ pub enum UartError { pub type Result = ::core::result::Result; pub struct Uart { - uart_base: u32, + base_addr: usize, } impl ops::Deref for Uart { @@ -155,13 +152,13 @@ impl ops::Deref for Uart { } impl Uart { - pub fn new(uart_base: u32) -> Uart { - Uart { uart_base } + pub fn new(base_addr: usize) -> Uart { + Uart { base_addr } } /// Returns a pointer to the register block fn ptr(&self) -> *const RegisterBlock { - self.uart_base as *const _ + self.base_addr as *const _ } ///Set baud rate and characteristics (115200 8N1) and map to GPIO diff --git a/0D_cache_performance/README.md b/0D_cache_performance/README.md index 6248a0a7..bcb7fd90 100644 --- a/0D_cache_performance/README.md +++ b/0D_cache_performance/README.md @@ -18,13 +18,12 @@ operating with data on the same DRAM with caching enabled and disabled. ### mmu.rs Therefore, we will map the same physical memory via two different virtual -addresses. We set up our pagetables such that the virtual address `0x200000` -points to the physical DRAM at `0x400000`, and we configure it as +addresses. We set up our pagetables such that the virtual address `0x400000` +points to the physical DRAM at `0x200000`, and we configure it as `non-cacheable` in the page tables. -We are still using a `2 MiB` granule, and set up the next block, which starts at -virtual `0x400000`, to point at physical `0x400000` (this is an identity mapped -block). This time, the block is configured as cacheable. +There is also an identity mapped block, which starts at virtual `0x200000` and +points at physical `0x200000`. This time, the block is configured as cacheable. ### benchmark.rs @@ -38,15 +37,20 @@ non-cacheable virtual addresses. Remember that both virtual addresses point to the _same_ physical DRAM, so the difference in time that we will see will showcase how much faster it is to operate on DRAM with caching enabled. -## Results +## Output On my Raspberry, I get the following results: -```text -Benchmarking non-cacheable DRAM modifications at virtual 0x0000000000200000, physical 0x0000000000400000: +```console +ferris@box:~$ make raspboot + +[0] UART is live! +[1] Press a key to continue booting... Greetings fellow Rustacean! +[2] MMU online. +Benchmarking non-cacheable DRAM modifications at virtual 0x0000000000400000, physical 0x0000000000200000: 1040 miliseconds. -Benchmarking cacheable DRAM modifications at virtual 0x0000000000400000, physical 0x0000000000400000: +Benchmarking cacheable DRAM modifications at virtual 0x0000000000200000, physical 0x0000000000200000: 53 miliseconds. With caching, the function is 1800% faster! diff --git a/0D_cache_performance/kernel8 b/0D_cache_performance/kernel8 index 673897fd..a19e827b 100755 Binary files a/0D_cache_performance/kernel8 and b/0D_cache_performance/kernel8 differ diff --git a/0D_cache_performance/kernel8.img b/0D_cache_performance/kernel8.img index 06cee929..149556d6 100755 Binary files a/0D_cache_performance/kernel8.img and b/0D_cache_performance/kernel8.img differ diff --git a/0D_cache_performance/src/benchmark.rs b/0D_cache_performance/src/benchmark.rs index 86af6d62..346399f5 100644 --- a/0D_cache_performance/src/benchmark.rs +++ b/0D_cache_performance/src/benchmark.rs @@ -22,26 +22,26 @@ * SOFTWARE. */ -use super::uart; +use crate::uart; use core::sync::atomic::{compiler_fence, Ordering}; use cortex_a::{barrier, regs::*}; /// We assume that addr is cacheline aligned -fn batch_modify_time(addr: u64) -> Option { +fn batch_modify_time(addr: usize) -> Option { const CACHELINE_SIZE_BYTES: usize = 64; // TODO: retrieve this from a system register const NUM_CACHELINES_TOUCHED: usize = 5; const NUM_BENCH_ITERATIONS: usize = 20_000; const NUM_BYTES_TOUCHED: usize = CACHELINE_SIZE_BYTES * NUM_CACHELINES_TOUCHED; - let mem = unsafe { core::slice::from_raw_parts_mut(addr as *mut u64, NUM_BYTES_TOUCHED) }; + let mem = unsafe { core::slice::from_raw_parts_mut(addr as *mut usize, NUM_BYTES_TOUCHED) }; // Benchmark starts here let t1 = CNTPCT_EL0.get(); compiler_fence(Ordering::SeqCst); - let mut temp: u64; + let mut temp: usize; for _ in 0..NUM_BENCH_ITERATIONS { for qword in mem.iter_mut() { unsafe { @@ -65,24 +65,17 @@ fn batch_modify_time(addr: u64) -> Option { } pub fn run(uart: &uart::Uart) { - const SIZE_2MIB: u64 = 2 * 1024 * 1024; - const ERROR_STRING: &str = "Something went wrong!"; - - // Start of the __SECOND__ virtual 2 MiB block (counting starts at zero). - // NON-cacheable DRAM memory. - let non_cacheable_addr: u64 = SIZE_2MIB; + use crate::memory::map; - // Start of the __THIRD__ virtual 2 MiB block. - // Cacheable DRAM memory - let cacheable_addr: u64 = 2 * SIZE_2MIB; + const ERROR_STRING: &str = "Something went wrong!"; uart.puts("Benchmarking non-cacheable DRAM modifications at virtual 0x"); - uart.hex(non_cacheable_addr); + uart.hex(map::virt::NON_CACHEABLE_START as u64); uart.puts(", physical 0x"); - uart.hex(2 * SIZE_2MIB); + uart.hex(map::virt::CACHEABLE_START as u64); uart.puts(":\n"); - let result_nc = match batch_modify_time(non_cacheable_addr) { + let result_nc = match batch_modify_time(map::virt::NON_CACHEABLE_START) { Some(t) => { uart.dec(t as u32); uart.puts(" miliseconds.\n\n"); @@ -95,12 +88,12 @@ pub fn run(uart: &uart::Uart) { }; uart.puts("Benchmarking cacheable DRAM modifications at virtual 0x"); - uart.hex(cacheable_addr); + uart.hex(map::virt::CACHEABLE_START as u64); uart.puts(", physical 0x"); - uart.hex(2 * SIZE_2MIB); + uart.hex(map::virt::CACHEABLE_START as u64); uart.puts(":\n"); - let result_c = match batch_modify_time(cacheable_addr) { + let result_c = match batch_modify_time(map::virt::CACHEABLE_START) { Some(t) => { uart.dec(t as u32); uart.puts(" miliseconds.\n\n"); diff --git a/0D_cache_performance/src/gpio.rs b/0D_cache_performance/src/gpio.rs index 608ba532..7affea08 100644 --- a/0D_cache_performance/src/gpio.rs +++ b/0D_cache_performance/src/gpio.rs @@ -1,7 +1,7 @@ /* * MIT License * - * Copyright (c) 2018 Andre Richter + * Copyright (c) 2018-2019 Andre Richter * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -22,7 +22,6 @@ * SOFTWARE. */ -use super::MMIO_BASE; use core::ops; use register::{mmio::ReadWrite, register_bitfields}; @@ -67,8 +66,6 @@ register_bitfields! { ] } -const GPIO_BASE: u32 = MMIO_BASE + 0x200_000; - #[allow(non_snake_case)] #[repr(C)] pub struct RegisterBlock { @@ -99,23 +96,25 @@ pub struct RegisterBlock { } /// Public interface to the GPIO MMIO area -pub struct GPIO; +pub struct GPIO { + base_addr: usize, +} impl ops::Deref for GPIO { type Target = RegisterBlock; fn deref(&self) -> &Self::Target { - unsafe { &*Self::ptr() } + unsafe { &*self.ptr() } } } impl GPIO { - pub fn new() -> GPIO { - GPIO + pub fn new(base_addr: usize) -> GPIO { + GPIO { base_addr } } /// Returns a pointer to the register block - fn ptr() -> *const RegisterBlock { - GPIO_BASE as *const _ + fn ptr(&self) -> *const RegisterBlock { + self.base_addr as *const _ } } diff --git a/0D_cache_performance/src/main.rs b/0D_cache_performance/src/main.rs index 0a02da69..45a8521f 100644 --- a/0D_cache_performance/src/main.rs +++ b/0D_cache_performance/src/main.rs @@ -26,19 +26,17 @@ #![no_main] #![feature(range_contains)] -const MMIO_BASE: u32 = 0x3F00_0000; - mod benchmark; mod delays; mod gpio; mod mbox; -mod mmu; +mod memory; mod uart; fn kernel_entry() -> ! { - let gpio = gpio::GPIO::new(); - let mut mbox = mbox::Mbox::new(); - let uart = uart::Uart::new(uart::UART_PHYS_BASE); + let gpio = gpio::GPIO::new(memory::map::physical::GPIO_BASE); + let mut mbox = mbox::Mbox::new(memory::map::physical::VIDEOCORE_MBOX_BASE); + let uart = uart::Uart::new(memory::map::physical::UART_BASE); // set up serial console match uart.init(&mut mbox, &gpio) { @@ -52,11 +50,14 @@ fn kernel_entry() -> ! { uart.getc(); uart.puts("Greetings fellow Rustacean!\n"); - uart.puts("[2] Switching MMU on now... "); - - unsafe { mmu::init() }; - - uart.puts("MMU is live \\o/\n\n"); + match unsafe { memory::mmu::init() } { + Err(s) => { + uart.puts("[2][Error] MMU: "); + uart.puts(s); + uart.puts("\n"); + } + Ok(()) => uart.puts("[2] MMU online.\n"), + } benchmark::run(&uart); diff --git a/0D_cache_performance/src/mbox.rs b/0D_cache_performance/src/mbox.rs index aeae88bb..2e4bf0ad 100644 --- a/0D_cache_performance/src/mbox.rs +++ b/0D_cache_performance/src/mbox.rs @@ -1,7 +1,7 @@ /* * MIT License * - * Copyright (c) 2018 Andre Richter + * Copyright (c) 2018-2019 Andre Richter * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -22,7 +22,6 @@ * SOFTWARE. */ -use super::MMIO_BASE; use core::ops; use cortex_a::asm; use register::{ @@ -39,8 +38,6 @@ register_bitfields! { ] } -const VIDEOCORE_MBOX: u32 = MMIO_BASE + 0xB880; - #[allow(non_snake_case)] #[repr(C)] pub struct RegisterBlock { @@ -89,6 +86,7 @@ pub struct Mbox { // The address for buffer needs to be 16-byte aligned so that the // Videcore can handle it properly. pub buffer: [u32; 36], + base_addr: usize, } /// Deref to RegisterBlock @@ -105,18 +103,21 @@ impl ops::Deref for Mbox { type Target = RegisterBlock; fn deref(&self) -> &Self::Target { - unsafe { &*Self::ptr() } + unsafe { &*self.ptr() } } } impl Mbox { - pub fn new() -> Mbox { - Mbox { buffer: [0; 36] } + pub fn new(base_addr: usize) -> Mbox { + Mbox { + buffer: [0; 36], + base_addr, + } } /// Returns a pointer to the register block - fn ptr() -> *const RegisterBlock { - VIDEOCORE_MBOX as *const _ + fn ptr(&self) -> *const RegisterBlock { + self.base_addr as *const _ } /// Make a mailbox call. Returns Err(MboxError) on failure, Ok(()) success diff --git a/0D_cache_performance/src/memory.rs b/0D_cache_performance/src/memory.rs new file mode 100644 index 00000000..72db92dd --- /dev/null +++ b/0D_cache_performance/src/memory.rs @@ -0,0 +1,225 @@ +/* + * MIT License + * + * Copyright (c) 2019 Andre Richter + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +use core::ops::RangeInclusive; + +pub mod mmu; + +/// System memory map. +#[rustfmt::skip] +pub mod map { + pub const START: usize = 0x0000_0000; + pub const END: usize = 0x3FFF_FFFF; + + pub mod physical { + pub const MMIO_BASE: usize = 0x3F00_0000; + pub const VIDEOCORE_MBOX_BASE: usize = MMIO_BASE + 0x0000_B880; + pub const GPIO_BASE: usize = MMIO_BASE + 0x0020_0000; + pub const UART_BASE: usize = MMIO_BASE + 0x0020_1000; + pub const MMIO_END: usize = super::END; + } + + pub mod virt { + pub const KERN_STACK_START: usize = super::START; + pub const KERN_STACK_END: usize = 0x0007_FFFF; + + // The second 2 MiB block + pub const CACHEABLE_START: usize = 0x0020_0000; + + // The third 2 MiB block + pub const NON_CACHEABLE_START: usize = 0x0040_0000; + pub const NON_CACHEABLE_END: usize = 0x005F_FFFF; + } +} + +/// Types used for compiling the virtual memory layout of the kernel using +/// address ranges. +pub mod kernel_mem_range { + use core::ops::RangeInclusive; + + #[derive(Copy, Clone)] + pub enum MemAttributes { + CacheableDRAM, + NonCacheableDRAM, + Device, + } + + #[derive(Copy, Clone)] + pub enum AccessPermissions { + ReadOnly, + ReadWrite, + } + + #[derive(Copy, Clone)] + pub enum Translation { + Identity, + Offset(usize), + } + + #[derive(Copy, Clone)] + pub struct AttributeFields { + pub mem_attributes: MemAttributes, + pub acc_perms: AccessPermissions, + pub execute_never: bool, + } + + impl Default for AttributeFields { + fn default() -> AttributeFields { + AttributeFields { + mem_attributes: MemAttributes::CacheableDRAM, + acc_perms: AccessPermissions::ReadWrite, + execute_never: true, + } + } + } + + pub struct Descriptor { + pub virtual_range: fn() -> RangeInclusive, + pub translation: Translation, + pub attribute_fields: AttributeFields, + } +} + +use kernel_mem_range::*; + +/// A virtual memory layout that is agnostic of the paging granularity that the +/// hardware MMU will use. +/// +/// Contains only special ranges, aka anything that is _not_ normal cacheable +/// DRAM. +static KERNEL_VIRTUAL_LAYOUT: [Descriptor; 5] = [ + // Kernel stack + Descriptor { + virtual_range: || { + RangeInclusive::new(map::virt::KERN_STACK_START, map::virt::KERN_STACK_END) + }, + translation: Translation::Identity, + attribute_fields: AttributeFields { + mem_attributes: MemAttributes::CacheableDRAM, + acc_perms: AccessPermissions::ReadWrite, + execute_never: true, + }, + }, + // Kernel code and RO data + Descriptor { + virtual_range: || { + // Using the linker script, we ensure that the RO area is consecutive and 4 + // KiB aligned, and we export the boundaries via symbols: + // + // [__ro_start, __ro_end) + extern "C" { + // The inclusive start of the read-only area, aka the address of the + // first byte of the area. + static __ro_start: u64; + + // The exclusive end of the read-only area, aka the address of + // the first byte _after_ the RO area. + static __ro_end: u64; + } + + unsafe { + // Notice the subtraction to turn the exclusive end into an + // inclusive end + RangeInclusive::new( + &__ro_start as *const _ as usize, + &__ro_end as *const _ as usize - 1, + ) + } + }, + translation: Translation::Identity, + attribute_fields: AttributeFields { + mem_attributes: MemAttributes::CacheableDRAM, + acc_perms: AccessPermissions::ReadOnly, + execute_never: false, + }, + }, + // Kernel data and BSS + Descriptor { + virtual_range: || { + extern "C" { + static __ro_end: u64; + static __bss_end: u64; + } + + unsafe { + RangeInclusive::new( + &__ro_end as *const _ as usize, + &__bss_end as *const _ as usize - 1, + ) + } + }, + translation: Translation::Identity, + attribute_fields: AttributeFields { + mem_attributes: MemAttributes::CacheableDRAM, + acc_perms: AccessPermissions::ReadWrite, + execute_never: true, + }, + }, + // Non-cacheable DRAM + Descriptor { + virtual_range: || { + RangeInclusive::new(map::virt::NON_CACHEABLE_START, map::virt::NON_CACHEABLE_END) + }, + translation: Translation::Offset(map::virt::CACHEABLE_START), + attribute_fields: AttributeFields { + mem_attributes: MemAttributes::NonCacheableDRAM, + acc_perms: AccessPermissions::ReadWrite, + execute_never: true, + }, + }, + // Device MMIO + Descriptor { + virtual_range: || RangeInclusive::new(map::physical::MMIO_BASE, map::physical::MMIO_END), + translation: Translation::Identity, + attribute_fields: AttributeFields { + mem_attributes: MemAttributes::Device, + acc_perms: AccessPermissions::ReadWrite, + execute_never: true, + }, + }, +]; + +/// For a given virtual address, find and return the output address and +/// according attributes. +/// +/// If the address is not covered in VIRTUAL_LAYOUT, return a default for normal +/// cacheable DRAM. +fn get_virt_addr_properties(virt_addr: usize) -> Result<(usize, AttributeFields), &'static str> { + if virt_addr > map::END { + return Err("Address out of range."); + } + + for i in KERNEL_VIRTUAL_LAYOUT.iter() { + if (i.virtual_range)().contains(&virt_addr) { + let output_addr = match i.translation { + Translation::Identity => virt_addr, + Translation::Offset(a) => a + (virt_addr - (i.virtual_range)().start()), + }; + + return Ok((output_addr, i.attribute_fields)); + } + } + + Ok((virt_addr, AttributeFields::default())) +} diff --git a/0D_cache_performance/src/memory/mmu.rs b/0D_cache_performance/src/memory/mmu.rs new file mode 100644 index 00000000..72405e74 --- /dev/null +++ b/0D_cache_performance/src/memory/mmu.rs @@ -0,0 +1,345 @@ +/* + * MIT License + * + * Copyright (c) 2018-2019 Andre Richter + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +use crate::memory::{get_virt_addr_properties, AttributeFields}; +use cortex_a::{barrier, regs::*}; +use register::register_bitfields; + +register_bitfields! {u64, + // AArch64 Reference Manual page 2150 + STAGE1_DESCRIPTOR [ + /// Privileged execute-never + PXN OFFSET(53) NUMBITS(1) [ + False = 0, + True = 1 + ], + + /// Various address fields, depending on use case + LVL2_OUTPUT_ADDR_4KiB OFFSET(21) NUMBITS(27) [], // [47:21] + NEXT_LVL_TABLE_ADDR_4KiB OFFSET(12) NUMBITS(36) [], // [47:12] + + /// Access flag + AF OFFSET(10) NUMBITS(1) [ + False = 0, + True = 1 + ], + + /// Shareability field + SH OFFSET(8) NUMBITS(2) [ + OuterShareable = 0b10, + InnerShareable = 0b11 + ], + + /// Access Permissions + AP OFFSET(6) NUMBITS(2) [ + RW_EL1 = 0b00, + RW_EL1_EL0 = 0b01, + RO_EL1 = 0b10, + RO_EL1_EL0 = 0b11 + ], + + /// Memory attributes index into the MAIR_EL1 register + AttrIndx OFFSET(2) NUMBITS(3) [], + + TYPE OFFSET(1) NUMBITS(1) [ + Block = 0, + Table = 1 + ], + + VALID OFFSET(0) NUMBITS(1) [ + False = 0, + True = 1 + ] + ] +} + +const FOUR_KIB: usize = 4 * 1024; +const FOUR_KIB_SHIFT: usize = 12; // log2(4 * 1024) + +const TWO_MIB: usize = 2 * 1024 * 1024; +const TWO_MIB_SHIFT: usize = 21; // log2(2 * 1024 * 1024) + +/// A descriptor pointing to the next page table. +struct TableDescriptor(register::FieldValue); + +impl TableDescriptor { + fn new(next_lvl_table_addr: usize) -> Result { + if next_lvl_table_addr % FOUR_KIB != 0 { + return Err("TableDescriptor: Address is not 4 KiB aligned."); + } + + let shifted = next_lvl_table_addr >> FOUR_KIB_SHIFT; + + Ok(TableDescriptor( + STAGE1_DESCRIPTOR::VALID::True + + STAGE1_DESCRIPTOR::TYPE::Table + + STAGE1_DESCRIPTOR::NEXT_LVL_TABLE_ADDR_4KiB.val(shifted as u64), + )) + } + + fn value(&self) -> u64 { + self.0.value + } +} + +/// A function that maps the generic memory range attributes to HW-specific +/// attributes of the MMU. +fn into_mmu_attributes( + attribute_fields: AttributeFields, +) -> register::FieldValue { + use crate::memory::{AccessPermissions, MemAttributes}; + + // Memory attributes + let mut desc = match attribute_fields.mem_attributes { + MemAttributes::CacheableDRAM => { + STAGE1_DESCRIPTOR::SH::InnerShareable + STAGE1_DESCRIPTOR::AttrIndx.val(mair::NORMAL) + } + MemAttributes::NonCacheableDRAM => { + STAGE1_DESCRIPTOR::SH::InnerShareable + + STAGE1_DESCRIPTOR::AttrIndx.val(mair::NORMAL_NON_CACHEABLE) + } + MemAttributes::Device => { + STAGE1_DESCRIPTOR::SH::OuterShareable + STAGE1_DESCRIPTOR::AttrIndx.val(mair::DEVICE) + } + }; + + // Access Permissions + desc += match attribute_fields.acc_perms { + AccessPermissions::ReadOnly => STAGE1_DESCRIPTOR::AP::RO_EL1, + AccessPermissions::ReadWrite => STAGE1_DESCRIPTOR::AP::RW_EL1, + }; + + // Execute Never + desc += if attribute_fields.execute_never { + STAGE1_DESCRIPTOR::PXN::True + } else { + STAGE1_DESCRIPTOR::PXN::False + }; + + desc +} + +/// A Level2 block descriptor with 2 MiB aperture. +/// +/// The output points to physical memory. +struct Lvl2BlockDescriptor(register::FieldValue); + +impl Lvl2BlockDescriptor { + fn new( + output_addr: usize, + attribute_fields: AttributeFields, + ) -> Result { + if output_addr % TWO_MIB != 0 { + return Err("BlockDescriptor: Address is not 2 MiB aligned."); + } + + let shifted = output_addr >> TWO_MIB_SHIFT; + + Ok(Lvl2BlockDescriptor( + STAGE1_DESCRIPTOR::VALID::True + + STAGE1_DESCRIPTOR::AF::True + + into_mmu_attributes(attribute_fields) + + STAGE1_DESCRIPTOR::TYPE::Block + + STAGE1_DESCRIPTOR::LVL2_OUTPUT_ADDR_4KiB.val(shifted as u64), + )) + } + + fn value(&self) -> u64 { + self.0.value + } +} + +/// A page descriptor with 4 KiB aperture. +/// +/// The output points to physical memory. +struct PageDescriptor(register::FieldValue); + +impl PageDescriptor { + fn new( + output_addr: usize, + attribute_fields: AttributeFields, + ) -> Result { + if output_addr % FOUR_KIB != 0 { + return Err("PageDescriptor: Address is not 4 KiB aligned."); + } + + let shifted = output_addr >> FOUR_KIB_SHIFT; + + Ok(PageDescriptor( + STAGE1_DESCRIPTOR::VALID::True + + STAGE1_DESCRIPTOR::AF::True + + into_mmu_attributes(attribute_fields) + + STAGE1_DESCRIPTOR::TYPE::Table + + STAGE1_DESCRIPTOR::NEXT_LVL_TABLE_ADDR_4KiB.val(shifted as u64), + )) + } + + fn value(&self) -> u64 { + self.0.value + } +} + +/// Constants for indexing the MAIR_EL1. +#[allow(dead_code)] +mod mair { + pub const DEVICE: u64 = 0; + pub const NORMAL: u64 = 1; + pub const NORMAL_NON_CACHEABLE: u64 = 2; +} + +/// Setup function for the MAIR_EL1 register. +fn set_up_mair() { + // Define the three memory types that we will map. Cacheable and + // non-cacheable normal DRAM, and device. + MAIR_EL1.write( + // Attribute 2 + MAIR_EL1::Attr2_HIGH::Memory_OuterNonCacheable + + MAIR_EL1::Attr2_LOW_MEMORY::InnerNonCacheable + + // Attribute 1 + + MAIR_EL1::Attr1_HIGH::Memory_OuterWriteBack_NonTransient_ReadAlloc_WriteAlloc + + MAIR_EL1::Attr1_LOW_MEMORY::InnerWriteBack_NonTransient_ReadAlloc_WriteAlloc + + // Attribute 0 + + MAIR_EL1::Attr0_HIGH::Device + + MAIR_EL1::Attr0_LOW_DEVICE::Device_nGnRE, + ); +} + +trait BaseAddr { + fn base_addr_u64(&self) -> u64; + fn base_addr_usize(&self) -> usize; +} + +impl BaseAddr for [u64; 512] { + fn base_addr_u64(&self) -> u64 { + self as *const u64 as u64 + } + + fn base_addr_usize(&self) -> usize { + self as *const u64 as usize + } +} + +const NUM_ENTRIES_4KIB: usize = 512; + +// A wrapper struct is needed here so that the align attribute can be used. +#[repr(C)] +#[repr(align(4096))] +struct PageTable { + entries: [u64; NUM_ENTRIES_4KIB], +} + +/// The LVL2 page table containng the 2 MiB entries. +static mut LVL2_TABLE: PageTable = PageTable { + entries: [0; NUM_ENTRIES_4KIB], +}; + +/// The LVL3 page table containing the 4 KiB entries. +/// +/// The first entry of the LVL2_TABLE will forward to this table. +static mut LVL3_TABLE: PageTable = PageTable { + entries: [0; NUM_ENTRIES_4KIB], +}; + +/// Set up identity mapped page tables for the first 1 GiB of address space. +/// +/// The first 2 MiB are 4 KiB granule, the rest 2 MiB. +pub unsafe fn init() -> Result<(), &'static str> { + // Prepare the memory attribute indirection register. + set_up_mair(); + + // Point the first 2 MiB of virtual addresses to the follow-up LVL3 + // page-table. + LVL2_TABLE.entries[0] = match TableDescriptor::new(LVL3_TABLE.entries.base_addr_usize()) { + Err(s) => return Err(s), + Ok(d) => d.value(), + }; + + // Fill the rest of the LVL2 (2 MiB) entries as block descriptors. + // + // Notice the skip(1) which makes the iteration start at the second 2 MiB + // block (0x20_0000). + for (block_descriptor_nr, entry) in LVL2_TABLE.entries.iter_mut().enumerate().skip(1) { + let virt_addr = block_descriptor_nr << TWO_MIB_SHIFT; + + let (output_addr, attribute_fields) = match get_virt_addr_properties(virt_addr) { + Err(s) => return Err(s), + Ok((a, b)) => (a, b), + }; + + let block_desc = match Lvl2BlockDescriptor::new(output_addr, attribute_fields) { + Err(s) => return Err(s), + Ok(desc) => desc, + }; + + *entry = block_desc.value(); + } + + // Finally, fill the single LVL3 table (4 KiB granule). + for (page_descriptor_nr, entry) in LVL3_TABLE.entries.iter_mut().enumerate() { + let virt_addr = page_descriptor_nr << FOUR_KIB_SHIFT; + + let (output_addr, attribute_fields) = match get_virt_addr_properties(virt_addr) { + Err(s) => return Err(s), + Ok((a, b)) => (a, b), + }; + + let page_desc = match PageDescriptor::new(output_addr, attribute_fields) { + Err(s) => return Err(s), + Ok(desc) => desc, + }; + + *entry = page_desc.value(); + } + + // Point to the LVL2 table base address in TTBR0. + TTBR0_EL1.set_baddr(LVL2_TABLE.entries.base_addr_u64()); + + // Configure various settings of stage 1 of the EL1 translation regime. + let ips = ID_AA64MMFR0_EL1.read(ID_AA64MMFR0_EL1::PARange); + TCR_EL1.write( + TCR_EL1::TBI0::Ignored + + TCR_EL1::IPS.val(ips) + + TCR_EL1::TG0::KiB_4 // 4 KiB granule + + TCR_EL1::SH0::Inner + + TCR_EL1::ORGN0::WriteBack_ReadAlloc_WriteAlloc_Cacheable + + TCR_EL1::IRGN0::WriteBack_ReadAlloc_WriteAlloc_Cacheable + + TCR_EL1::EPD0::EnableTTBR0Walks + + TCR_EL1::T0SZ.val(34), // Start walks at level 2 + ); + + // Switch the MMU on. + // + // First, force all previous changes to be seen before the MMU is enabled. + barrier::isb(barrier::SY); + + // Enable the MMU and turn on data and instruction caching. + SCTLR_EL1.modify(SCTLR_EL1::M::Enable + SCTLR_EL1::C::Cacheable + SCTLR_EL1::I::Cacheable); + + // Force MMU init to complete before next instruction + barrier::isb(barrier::SY); + + Ok(()) +} diff --git a/0D_cache_performance/src/mmu.rs b/0D_cache_performance/src/mmu.rs deleted file mode 100644 index 6198bb02..00000000 --- a/0D_cache_performance/src/mmu.rs +++ /dev/null @@ -1,265 +0,0 @@ -/* - * MIT License - * - * Copyright (c) 2018-2019 Andre Richter - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -use cortex_a::{barrier, regs::*}; -use register::register_bitfields; - -register_bitfields! {u64, - // AArch64 Reference Manual page 2150 - STAGE1_DESCRIPTOR [ - /// Execute-never - XN OFFSET(54) NUMBITS(1) [ - False = 0, - True = 1 - ], - - /// Various address fields, depending on use case - LVL2_OUTPUT_ADDR_4KiB OFFSET(21) NUMBITS(27) [], // [47:21] - NEXT_LVL_TABLE_ADDR_4KiB OFFSET(12) NUMBITS(36) [], // [47:12] - - /// Access flag - AF OFFSET(10) NUMBITS(1) [ - False = 0, - True = 1 - ], - - /// Shareability field - SH OFFSET(8) NUMBITS(2) [ - OuterShareable = 0b10, - InnerShareable = 0b11 - ], - - /// Access Permissions - AP OFFSET(6) NUMBITS(2) [ - RW_EL1 = 0b00, - RW_EL1_EL0 = 0b01, - RO_EL1 = 0b10, - RO_EL1_EL0 = 0b11 - ], - - /// Memory attributes index into the MAIR_EL1 register - AttrIndx OFFSET(2) NUMBITS(3) [], - - TYPE OFFSET(1) NUMBITS(1) [ - Block = 0, - Table = 1 - ], - - VALID OFFSET(0) NUMBITS(1) [ - False = 0, - True = 1 - ] - ] -} - -trait BaseAddr { - fn base_addr(&self) -> u64; -} - -impl BaseAddr for [u64; 512] { - fn base_addr(&self) -> u64 { - self as *const u64 as u64 - } -} - -const NUM_ENTRIES_4KIB: usize = 512; - -// We need a wrapper struct here so that we can make use of the align attribute. -#[repr(C)] -#[repr(align(4096))] -struct PageTable { - entries: [u64; NUM_ENTRIES_4KIB], -} - -static mut LVL2_TABLE: PageTable = PageTable { - entries: [0; NUM_ENTRIES_4KIB], -}; -static mut SINGLE_LVL3_TABLE: PageTable = PageTable { - entries: [0; NUM_ENTRIES_4KIB], -}; - -/// Set up identity mapped page tables for the first 1 GiB of address space. -pub unsafe fn init() { - // First, define the three memory types that we will map. Cacheable and - // non-cacheable normal DRAM, and device. - MAIR_EL1.write( - // Attribute 2 - MAIR_EL1::Attr2_HIGH::Memory_OuterNonCacheable - + MAIR_EL1::Attr2_LOW_MEMORY::InnerNonCacheable - - // Attribute 1 - + MAIR_EL1::Attr1_HIGH::Memory_OuterWriteBack_NonTransient_ReadAlloc_WriteAlloc - + MAIR_EL1::Attr1_LOW_MEMORY::InnerWriteBack_NonTransient_ReadAlloc_WriteAlloc - - // Attribute 0 - + MAIR_EL1::Attr0_HIGH::Device - + MAIR_EL1::Attr0_LOW_DEVICE::Device_nGnRE, - ); - - // Descriptive consts for indexing into the correct MAIR_EL1 attributes. - mod mair { - pub const DEVICE: u64 = 0; - pub const NORMAL: u64 = 1; - pub const NORMAL_NON_CACHEABLE: u64 = 2; - } - - // The first 2 MiB. - // - // Set up the first LVL2 entry, pointing to the base address of a follow-up - // table containing 4 KiB pages. - // - // 0x0000_0000_0000_0000 | - // |> 2 MiB - // 0x0000_0000_001F_FFFF | - let lvl3_base: u64 = SINGLE_LVL3_TABLE.entries.base_addr() >> 12; - LVL2_TABLE.entries[0] = (STAGE1_DESCRIPTOR::VALID::True - + STAGE1_DESCRIPTOR::TYPE::Table - + STAGE1_DESCRIPTOR::NEXT_LVL_TABLE_ADDR_4KiB.val(lvl3_base)) - .value; - - // The second 2 MiB as block entry. - // - // Mapped as non-cacheable. - // - // 0x0000_0000_0020_0000 | - // |> 2 MiB - // 0x0000_0000_003F_FFFF | - LVL2_TABLE.entries[1] = (STAGE1_DESCRIPTOR::VALID::True - + STAGE1_DESCRIPTOR::TYPE::Block - + STAGE1_DESCRIPTOR::AttrIndx.val(mair::NORMAL_NON_CACHEABLE) - + STAGE1_DESCRIPTOR::AP::RW_EL1 - + STAGE1_DESCRIPTOR::SH::OuterShareable - + STAGE1_DESCRIPTOR::AF::True - // This translation is accessed for virtual 0x200000. Point to physical - // 0x400000, aka the third phyiscal 2 MiB DRAM block (third block == 2, - // because we start counting at 0). - // - // Here, we configure it non-cacheable. - + STAGE1_DESCRIPTOR::LVL2_OUTPUT_ADDR_4KiB.val(2) - + STAGE1_DESCRIPTOR::XN::True) - .value; - - // Fill the rest of the LVL2 (2 MiB) entries as block descriptors. - // - // Differentiate between - // - cacheable DRAM - // - device memory - // - // Ranges are stored in memory.rs. - // - // 0x0000_0000_0040_0000 | - // |> 1004 MiB cacheable DRAM - // 0x0000_0000_3EFF_FFFF | - // 0x0000_0000_3F00_0000 | - // |> 16 MiB device (MMIO) - // 0x0000_0000_4000_0000 | - let mmio_first_block_index: u64 = (super::MMIO_BASE >> 21).into(); - - let common = STAGE1_DESCRIPTOR::VALID::True - + STAGE1_DESCRIPTOR::TYPE::Block - + STAGE1_DESCRIPTOR::AP::RW_EL1 - + STAGE1_DESCRIPTOR::AF::True - + STAGE1_DESCRIPTOR::XN::True; - - // Notice the skip(2). Start at the third 2 MiB DRAM block, which will point - // virtual 0x400000 to physical 0x400000, configured as cacheable memory. - for (i, entry) in LVL2_TABLE.entries.iter_mut().enumerate().skip(2) { - let j: u64 = i as u64; - - let mem_attr = if j >= mmio_first_block_index { - STAGE1_DESCRIPTOR::SH::OuterShareable + STAGE1_DESCRIPTOR::AttrIndx.val(mair::DEVICE) - } else { - STAGE1_DESCRIPTOR::SH::InnerShareable + STAGE1_DESCRIPTOR::AttrIndx.val(mair::NORMAL) - }; - - *entry = (common + mem_attr + STAGE1_DESCRIPTOR::LVL2_OUTPUT_ADDR_4KiB.val(j)).value; - } - - // Finally, fill the single LVL3 table (4 KiB granule). Differentiate - // between code+RO and RW pages. - // - // Using the linker script, we ensure that the RO area is consecutive and 4 - // KiB aligned, and we export the boundaries via symbols. - extern "C" { - // The inclusive start of the read-only area, aka the address of the - // first byte of the area. - static mut __ro_start: u64; - - // The non-inclusive end of the read-only area, aka the address of the - // first byte _after_ the RO area. - static mut __ro_end: u64; - } - - const PAGESIZE: u64 = 4096; - let ro_first_page_index: u64 = &__ro_start as *const _ as u64 / PAGESIZE; - - // Notice the subtraction to calculate the last page index of the RO area - // and not the first page index after the RO area. - let ro_last_page_index: u64 = (&__ro_end as *const _ as u64 / PAGESIZE) - 1; - - let common = STAGE1_DESCRIPTOR::VALID::True - + STAGE1_DESCRIPTOR::TYPE::Table - + STAGE1_DESCRIPTOR::AttrIndx.val(mair::NORMAL) - + STAGE1_DESCRIPTOR::SH::InnerShareable - + STAGE1_DESCRIPTOR::AF::True; - - for (i, entry) in SINGLE_LVL3_TABLE.entries.iter_mut().enumerate() { - let j: u64 = i as u64; - - let mem_attr = if (ro_first_page_index..=ro_last_page_index).contains(&j) { - STAGE1_DESCRIPTOR::AP::RO_EL1 + STAGE1_DESCRIPTOR::XN::False - } else { - STAGE1_DESCRIPTOR::AP::RW_EL1 + STAGE1_DESCRIPTOR::XN::True - }; - - *entry = (common + mem_attr + STAGE1_DESCRIPTOR::NEXT_LVL_TABLE_ADDR_4KiB.val(j)).value; - } - - // Point to the LVL2 table base address in TTBR0. - TTBR0_EL1.set_baddr(LVL2_TABLE.entries.base_addr()); - - // Configure various settings of stage 1 of the EL1 translation regime. - let ips = ID_AA64MMFR0_EL1.read(ID_AA64MMFR0_EL1::PARange); - TCR_EL1.write( - TCR_EL1::TBI0::Ignored - + TCR_EL1::IPS.val(ips) - + TCR_EL1::TG0::KiB_4 // 4 KiB granule - + TCR_EL1::SH0::Inner - + TCR_EL1::ORGN0::WriteBack_ReadAlloc_WriteAlloc_Cacheable - + TCR_EL1::IRGN0::WriteBack_ReadAlloc_WriteAlloc_Cacheable - + TCR_EL1::EPD0::EnableTTBR0Walks - + TCR_EL1::T0SZ.val(34), // Start walks at level 2 - ); - - // Switch the MMU on. - // - // First, force all previous changes to be seen before the MMU is enabled. - barrier::isb(barrier::SY); - - // Enable the MMU and turn on data and instruction caching. - SCTLR_EL1.modify(SCTLR_EL1::M::Enable + SCTLR_EL1::C::Cacheable + SCTLR_EL1::I::Cacheable); - - // Force MMU init to complete before next instruction - barrier::isb(barrier::SY); -} diff --git a/0D_cache_performance/src/uart.rs b/0D_cache_performance/src/uart.rs index 7287ec75..093ef6a7 100644 --- a/0D_cache_performance/src/uart.rs +++ b/0D_cache_performance/src/uart.rs @@ -1,7 +1,7 @@ /* * MIT License * - * Copyright (c) 2018 Andre Richter + * Copyright (c) 2018-2019 Andre Richter * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -22,7 +22,6 @@ * SOFTWARE. */ -use super::MMIO_BASE; use crate::delays; use crate::gpio; use crate::mbox; @@ -120,8 +119,6 @@ register_bitfields! { ] } -pub const UART_PHYS_BASE: u32 = MMIO_BASE + 0x20_1000; - #[allow(non_snake_case)] #[repr(C)] pub struct RegisterBlock { @@ -143,7 +140,7 @@ pub enum UartError { pub type Result = ::core::result::Result; pub struct Uart { - uart_base: u32, + base_addr: usize, } impl ops::Deref for Uart { @@ -155,13 +152,13 @@ impl ops::Deref for Uart { } impl Uart { - pub fn new(uart_base: u32) -> Uart { - Uart { uart_base } + pub fn new(base_addr: usize) -> Uart { + Uart { base_addr } } /// Returns a pointer to the register block fn ptr(&self) -> *const RegisterBlock { - self.uart_base as *const _ + self.base_addr as *const _ } ///Set baud rate and characteristics (115200 8N1) and map to GPIO diff --git a/0E_global_println/kernel8 b/0E_global_println/kernel8 index 1af89f94..c078e488 100755 Binary files a/0E_global_println/kernel8 and b/0E_global_println/kernel8 differ diff --git a/0E_global_println/kernel8.img b/0E_global_println/kernel8.img index f892fc1d..f3f884fb 100755 Binary files a/0E_global_println/kernel8.img and b/0E_global_println/kernel8.img differ diff --git a/0E_global_println/src/devices/hw.rs b/0E_global_println/src/devices/hw.rs index 877622ed..d9684419 100644 --- a/0E_global_println/src/devices/hw.rs +++ b/0E_global_println/src/devices/hw.rs @@ -23,9 +23,9 @@ */ mod gpio; -mod pl011_uart; +mod uart; mod videocore_mbox; pub use gpio::GPIO; -pub use pl011_uart::PL011Uart; +pub use uart::Uart; pub use videocore_mbox::VideocoreMbox; diff --git a/0E_global_println/src/devices/hw/gpio.rs b/0E_global_println/src/devices/hw/gpio.rs index 0d06dfa3..7affea08 100644 --- a/0E_global_println/src/devices/hw/gpio.rs +++ b/0E_global_println/src/devices/hw/gpio.rs @@ -97,7 +97,7 @@ pub struct RegisterBlock { /// Public interface to the GPIO MMIO area pub struct GPIO { - base_addr: u32, + base_addr: usize, } impl ops::Deref for GPIO { @@ -109,7 +109,7 @@ impl ops::Deref for GPIO { } impl GPIO { - pub fn new(base_addr: u32) -> GPIO { + pub fn new(base_addr: usize) -> GPIO { GPIO { base_addr } } diff --git a/0E_global_println/src/devices/hw/pl011_uart.rs b/0E_global_println/src/devices/hw/uart.rs similarity index 95% rename from 0E_global_println/src/devices/hw/pl011_uart.rs rename to 0E_global_println/src/devices/hw/uart.rs index f98534e4..c27b62d6 100644 --- a/0E_global_println/src/devices/hw/pl011_uart.rs +++ b/0E_global_println/src/devices/hw/uart.rs @@ -135,16 +135,16 @@ pub struct RegisterBlock { ICR: WriteOnly, // 0x44 } -pub enum PL011UartError { +pub enum UartError { MailboxError, } -pub type Result = ::core::result::Result; +pub type Result = ::core::result::Result; -pub struct PL011Uart { - base_addr: u32, +pub struct Uart { + base_addr: usize, } -impl ops::Deref for PL011Uart { +impl ops::Deref for Uart { type Target = RegisterBlock; fn deref(&self) -> &Self::Target { @@ -152,9 +152,9 @@ impl ops::Deref for PL011Uart { } } -impl PL011Uart { - pub fn new(base_addr: u32) -> PL011Uart { - PL011Uart { base_addr } +impl Uart { + pub fn new(base_addr: usize) -> Uart { + Uart { base_addr } } /// Returns a pointer to the register block @@ -188,7 +188,7 @@ impl PL011Uart { compiler_fence(Ordering::Release); if v_mbox.call(videocore_mbox::channel::PROP).is_err() { - return Err(PL011UartError::MailboxError); // Abort if UART clocks couldn't be set + return Err(UartError::MailboxError); // Abort if UART clocks couldn't be set }; // map UART0 to GPIO pins @@ -217,14 +217,14 @@ impl PL011Uart { } } -impl Drop for PL011Uart { +impl Drop for Uart { fn drop(&mut self) { self.CR .write(CR::UARTEN::Disabled + CR::TXE::Disabled + CR::RXE::Disabled); } } -impl ConsoleOps for PL011Uart { +impl ConsoleOps for Uart { /// Send a character fn putc(&self, c: char) { // wait until we can send diff --git a/0E_global_println/src/devices/hw/videocore_mbox.rs b/0E_global_println/src/devices/hw/videocore_mbox.rs index 1409f54a..0f6e294b 100644 --- a/0E_global_println/src/devices/hw/videocore_mbox.rs +++ b/0E_global_println/src/devices/hw/videocore_mbox.rs @@ -87,7 +87,7 @@ pub struct VideocoreMbox { // can handle it properly. Hence, put it at the start of the struct so that // the align attribute is effective. pub buffer: [u32; 36], - base_addr: u32, + base_addr: usize, } /// Deref to RegisterBlock @@ -109,7 +109,7 @@ impl ops::Deref for VideocoreMbox { } impl VideocoreMbox { - pub fn new(base_addr: u32) -> VideocoreMbox { + pub fn new(base_addr: usize) -> VideocoreMbox { VideocoreMbox { buffer: [0; 36], base_addr, diff --git a/0E_global_println/src/devices/virt/console.rs b/0E_global_println/src/devices/virt/console.rs index 088971c6..84244da8 100644 --- a/0E_global_println/src/devices/virt/console.rs +++ b/0E_global_println/src/devices/virt/console.rs @@ -47,12 +47,12 @@ impl ConsoleOps for NullConsole {} /// Possible outputs which the console can store. pub enum Output { None(NullConsole), - PL011Uart(hw::PL011Uart), + Uart(hw::Uart), } -impl From for Output { - fn from(instance: hw::PL011Uart) -> Self { - Output::PL011Uart(instance) +impl From for Output { + fn from(instance: hw::Uart) -> Self { + Output::Uart(instance) } } @@ -71,7 +71,7 @@ impl Console { fn current_ptr(&self) -> &dyn ConsoleOps { match &self.output { Output::None(i) => i, - Output::PL011Uart(i) => i, + Output::Uart(i) => i, } } diff --git a/0E_global_println/src/main.rs b/0E_global_println/src/main.rs index bd9fb8e3..58534756 100644 --- a/0E_global_println/src/main.rs +++ b/0E_global_println/src/main.rs @@ -50,17 +50,17 @@ fn kernel_entry() -> ! { //------------------------------------------------------------ // Instantiate GPIO device //------------------------------------------------------------ - let gpio = hw::GPIO::new(memory::map::GPIO_BASE); + let gpio = hw::GPIO::new(memory::map::physical::GPIO_BASE); //------------------------------------------------------------ // Instantiate Videocore Mailbox //------------------------------------------------------------ - let mut v_mbox = hw::VideocoreMbox::new(memory::map::VIDEOCORE_MBOX_BASE); + let mut v_mbox = hw::VideocoreMbox::new(memory::map::physical::VIDEOCORE_MBOX_BASE); //------------------------------------------------------------ // Instantiate PL011 UART and put it in CONSOLE //------------------------------------------------------------ - let uart = hw::PL011Uart::new(memory::map::PL011_UART_BASE); + let uart = hw::Uart::new(memory::map::physical::UART_BASE); match uart.init(&mut v_mbox, &gpio) { Ok(_) => { @@ -89,9 +89,11 @@ fn kernel_entry() -> ! { //------------------------------------------------------------ // Bring up memory subsystem //------------------------------------------------------------ - print!("[2] Switching MMU on now... "); - unsafe { memory::mmu::init() }; - println!("MMU online."); + if unsafe { memory::mmu::init() }.is_err() { + println!("[2][Error] Could not set up MMU. Aborting."); + } else { + println!("[2] MMU online."); + } memory::print_layout(); diff --git a/0E_global_println/src/memory.rs b/0E_global_println/src/memory.rs index ee828c9d..dd00a8f4 100644 --- a/0E_global_println/src/memory.rs +++ b/0E_global_println/src/memory.rs @@ -23,91 +23,246 @@ */ use crate::println; +use core::fmt; +use core::ops::RangeInclusive; pub mod mmu; -/// The system memory map. +/// System memory map. #[rustfmt::skip] pub mod map { - pub const KERN_STACK_BOT: u32 = 0x0000_0000; - pub const KERN_STACK_TOP: u32 = 0x0007_FFFF; + pub const START: usize = 0x0000_0000; + pub const END: usize = 0x3FFF_FFFF; - pub const MMIO_BASE: u32 = 0x3F00_0000; - pub const VIDEOCORE_MBOX_BASE: u32 = MMIO_BASE + 0x0000_B880; - pub const GPIO_BASE: u32 = MMIO_BASE + 0x0020_0000; - pub const PL011_UART_BASE: u32 = MMIO_BASE + 0x0020_1000; + pub mod physical { + pub const MMIO_BASE: usize = 0x3F00_0000; + pub const VIDEOCORE_MBOX_BASE: usize = MMIO_BASE + 0x0000_B880; + pub const GPIO_BASE: usize = MMIO_BASE + 0x0020_0000; + pub const UART_BASE: usize = MMIO_BASE + 0x0020_1000; + pub const MMIO_END: usize = super::END; + } - pub const PHYS_ADDR_MAX: u32 = 0x3FFF_FFFF; + pub mod virt { + pub const KERN_STACK_START: usize = super::START; + pub const KERN_STACK_END: usize = 0x0007_FFFF; + } } -const PAGESIZE: u64 = 4096; +/// Types used for compiling the virtual memory layout of the kernel using +/// address ranges. +pub mod kernel_mem_range { + use core::ops::RangeInclusive; -fn get_ro_start_end() -> (u64, u64) { - // Using the linker script, we ensure that the RO area is consecutive and 4 - // KiB aligned, and we export the boundaries via symbols. - extern "C" { - // The inclusive start of the read-only area, aka the address of the - // first byte of the area. - static __ro_start: u64; + #[allow(dead_code)] + #[derive(Copy, Clone)] + pub enum MemAttributes { + CacheableDRAM, + NonCacheableDRAM, + Device, + } - // The non-inclusive end of the read-only area, aka the address of the - // first byte _after_ the RO area. - static __ro_end: u64; + #[derive(Copy, Clone)] + pub enum AccessPermissions { + ReadOnly, + ReadWrite, } - unsafe { - // Notice the subtraction to calculate the last page index of the RO - // area and not the first page index after the RO area. - ( - &__ro_start as *const _ as u64, - &__ro_end as *const _ as u64 - 1, - ) + #[allow(dead_code)] + #[derive(Copy, Clone)] + pub enum Translation { + Identity, + Offset(usize), + } + + #[derive(Copy, Clone)] + pub struct AttributeFields { + pub mem_attributes: MemAttributes, + pub acc_perms: AccessPermissions, + pub execute_never: bool, + } + + impl Default for AttributeFields { + fn default() -> AttributeFields { + AttributeFields { + mem_attributes: MemAttributes::CacheableDRAM, + acc_perms: AccessPermissions::ReadWrite, + execute_never: true, + } + } + } + + pub struct Descriptor { + pub name: &'static str, + pub virtual_range: fn() -> RangeInclusive, + pub translation: Translation, + pub attribute_fields: AttributeFields, } } -pub fn print_layout() { - use crate::memory::map::*; +use kernel_mem_range::*; - // log2(1024) - const KIB_RSHIFT: u32 = 10; +/// A virtual memory layout that is agnostic of the paging granularity that the +/// hardware MMU will use. +/// +/// Contains only special ranges, aka anything that is _not_ normal cacheable +/// DRAM. +static KERNEL_VIRTUAL_LAYOUT: [Descriptor; 4] = [ + Descriptor { + name: "Kernel stack", + virtual_range: || { + RangeInclusive::new(map::virt::KERN_STACK_START, map::virt::KERN_STACK_END) + }, + translation: Translation::Identity, + attribute_fields: AttributeFields { + mem_attributes: MemAttributes::CacheableDRAM, + acc_perms: AccessPermissions::ReadWrite, + execute_never: true, + }, + }, + Descriptor { + name: "Kernel code and RO data", + virtual_range: || { + // Using the linker script, we ensure that the RO area is consecutive and 4 + // KiB aligned, and we export the boundaries via symbols: + // + // [__ro_start, __ro_end) + extern "C" { + // The inclusive start of the read-only area, aka the address of the + // first byte of the area. + static __ro_start: u64; - // log2(1024 * 1024) - const MIB_RSHIFT: u32 = 20; + // The exclusive end of the read-only area, aka the address of + // the first byte _after_ the RO area. + static __ro_end: u64; + } - println!("[i] Memory layout:"); + unsafe { + // Notice the subtraction to turn the exclusive end into an + // inclusive end + RangeInclusive::new( + &__ro_start as *const _ as usize, + &__ro_end as *const _ as usize - 1, + ) + } + }, + translation: Translation::Identity, + attribute_fields: AttributeFields { + mem_attributes: MemAttributes::CacheableDRAM, + acc_perms: AccessPermissions::ReadOnly, + execute_never: false, + }, + }, + Descriptor { + name: "Kernel data and BSS", + virtual_range: || { + extern "C" { + static __ro_end: u64; + static __bss_end: u64; + } - println!( - " {:#010X} - {:#010X} | {: >4} KiB | Kernel stack", - KERN_STACK_BOT, - KERN_STACK_TOP, - (KERN_STACK_TOP - KERN_STACK_BOT + 1) >> KIB_RSHIFT - ); + unsafe { + RangeInclusive::new( + &__ro_end as *const _ as usize, + &__bss_end as *const _ as usize - 1, + ) + } + }, + translation: Translation::Identity, + attribute_fields: AttributeFields { + mem_attributes: MemAttributes::CacheableDRAM, + acc_perms: AccessPermissions::ReadWrite, + execute_never: true, + }, + }, + Descriptor { + name: "Device MMIO", + virtual_range: || RangeInclusive::new(map::physical::MMIO_BASE, map::physical::MMIO_END), + translation: Translation::Identity, + attribute_fields: AttributeFields { + mem_attributes: MemAttributes::Device, + acc_perms: AccessPermissions::ReadWrite, + execute_never: true, + }, + }, +]; - let (ro_start, ro_end) = get_ro_start_end(); - println!( - " {:#010X} - {:#010X} | {: >4} KiB | Kernel code and RO data", - ro_start, - ro_end, - (ro_end - ro_start + 1) >> KIB_RSHIFT - ); +/// For a given virtual address, find and return the output address and +/// according attributes. +/// +/// If the address is not covered in VIRTUAL_LAYOUT, return a default for normal +/// cacheable DRAM. +fn get_virt_addr_properties(virt_addr: usize) -> Result<(usize, AttributeFields), &'static str> { + if virt_addr > map::END { + return Err("Address out of range."); + } - extern "C" { - static __bss_end: u64; + for i in KERNEL_VIRTUAL_LAYOUT.iter() { + if (i.virtual_range)().contains(&virt_addr) { + let output_addr = match i.translation { + Translation::Identity => virt_addr, + Translation::Offset(a) => a + (virt_addr - (i.virtual_range)().start()), + }; + + return Ok((output_addr, i.attribute_fields)); + } } - let start = ro_end + 1; - let end = unsafe { &__bss_end as *const _ as u64 } - 1; - println!( - " {:#010X} - {:#010X} | {: >4} KiB | Kernel data and BSS", - start, - end, - (end - start + 1) >> KIB_RSHIFT - ); - - println!( - " {:#010X} - {:#010X} | {: >4} MiB | Device MMIO", - MMIO_BASE, - PHYS_ADDR_MAX, - (PHYS_ADDR_MAX - MMIO_BASE + 1) >> MIB_RSHIFT - ); + Ok((virt_addr, AttributeFields::default())) +} + +/// Human-readable output of a Descriptor. +impl fmt::Display for Descriptor { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + // Call the function to which self.range points, and dereference the + // result, which causes Rust to copy the value. + let start = *(self.virtual_range)().start(); + let end = *(self.virtual_range)().end(); + let size = end - start + 1; + + // log2(1024) + const KIB_RSHIFT: u32 = 10; + + // log2(1024 * 1024) + const MIB_RSHIFT: u32 = 20; + + let (size, unit) = if (size >> MIB_RSHIFT) > 0 { + (size >> MIB_RSHIFT, "MiB") + } else if (size >> KIB_RSHIFT) > 0 { + (size >> KIB_RSHIFT, "KiB") + } else { + (size, "Byte") + }; + + let attr = match self.attribute_fields.mem_attributes { + MemAttributes::CacheableDRAM => "C", + MemAttributes::NonCacheableDRAM => "NC", + MemAttributes::Device => "Dev", + }; + + let acc_p = match self.attribute_fields.acc_perms { + AccessPermissions::ReadOnly => "RO", + AccessPermissions::ReadWrite => "RW", + }; + + let xn = if self.attribute_fields.execute_never { + "PXN" + } else { + "PX" + }; + + write!( + f, + " {:#010X} - {:#010X} | {: >3} {} | {: <3} {} {: <3} | {}", + start, end, size, unit, attr, acc_p, xn, self.name + ) + } +} + +/// Print the kernel memory layout. +pub fn print_layout() { + println!("[i] Kernel memory layout:"); + + for i in KERNEL_VIRTUAL_LAYOUT.iter() { + println!("{}", i); + } } diff --git a/0E_global_println/src/memory/mmu.rs b/0E_global_println/src/memory/mmu.rs index e44283cc..72405e74 100644 --- a/0E_global_println/src/memory/mmu.rs +++ b/0E_global_println/src/memory/mmu.rs @@ -22,14 +22,15 @@ * SOFTWARE. */ +use crate::memory::{get_virt_addr_properties, AttributeFields}; use cortex_a::{barrier, regs::*}; use register::register_bitfields; register_bitfields! {u64, // AArch64 Reference Manual page 2150 STAGE1_DESCRIPTOR [ - /// Execute-never - XN OFFSET(54) NUMBITS(1) [ + /// Privileged execute-never + PXN OFFSET(53) NUMBITS(1) [ False = 0, True = 1 ], @@ -73,42 +74,150 @@ register_bitfields! {u64, ] } -trait BaseAddr { - fn base_addr(&self) -> u64; +const FOUR_KIB: usize = 4 * 1024; +const FOUR_KIB_SHIFT: usize = 12; // log2(4 * 1024) + +const TWO_MIB: usize = 2 * 1024 * 1024; +const TWO_MIB_SHIFT: usize = 21; // log2(2 * 1024 * 1024) + +/// A descriptor pointing to the next page table. +struct TableDescriptor(register::FieldValue); + +impl TableDescriptor { + fn new(next_lvl_table_addr: usize) -> Result { + if next_lvl_table_addr % FOUR_KIB != 0 { + return Err("TableDescriptor: Address is not 4 KiB aligned."); + } + + let shifted = next_lvl_table_addr >> FOUR_KIB_SHIFT; + + Ok(TableDescriptor( + STAGE1_DESCRIPTOR::VALID::True + + STAGE1_DESCRIPTOR::TYPE::Table + + STAGE1_DESCRIPTOR::NEXT_LVL_TABLE_ADDR_4KiB.val(shifted as u64), + )) + } + + fn value(&self) -> u64 { + self.0.value + } } -impl BaseAddr for [u64; 512] { - fn base_addr(&self) -> u64 { - self as *const u64 as u64 +/// A function that maps the generic memory range attributes to HW-specific +/// attributes of the MMU. +fn into_mmu_attributes( + attribute_fields: AttributeFields, +) -> register::FieldValue { + use crate::memory::{AccessPermissions, MemAttributes}; + + // Memory attributes + let mut desc = match attribute_fields.mem_attributes { + MemAttributes::CacheableDRAM => { + STAGE1_DESCRIPTOR::SH::InnerShareable + STAGE1_DESCRIPTOR::AttrIndx.val(mair::NORMAL) + } + MemAttributes::NonCacheableDRAM => { + STAGE1_DESCRIPTOR::SH::InnerShareable + + STAGE1_DESCRIPTOR::AttrIndx.val(mair::NORMAL_NON_CACHEABLE) + } + MemAttributes::Device => { + STAGE1_DESCRIPTOR::SH::OuterShareable + STAGE1_DESCRIPTOR::AttrIndx.val(mair::DEVICE) + } + }; + + // Access Permissions + desc += match attribute_fields.acc_perms { + AccessPermissions::ReadOnly => STAGE1_DESCRIPTOR::AP::RO_EL1, + AccessPermissions::ReadWrite => STAGE1_DESCRIPTOR::AP::RW_EL1, + }; + + // Execute Never + desc += if attribute_fields.execute_never { + STAGE1_DESCRIPTOR::PXN::True + } else { + STAGE1_DESCRIPTOR::PXN::False + }; + + desc +} + +/// A Level2 block descriptor with 2 MiB aperture. +/// +/// The output points to physical memory. +struct Lvl2BlockDescriptor(register::FieldValue); + +impl Lvl2BlockDescriptor { + fn new( + output_addr: usize, + attribute_fields: AttributeFields, + ) -> Result { + if output_addr % TWO_MIB != 0 { + return Err("BlockDescriptor: Address is not 2 MiB aligned."); + } + + let shifted = output_addr >> TWO_MIB_SHIFT; + + Ok(Lvl2BlockDescriptor( + STAGE1_DESCRIPTOR::VALID::True + + STAGE1_DESCRIPTOR::AF::True + + into_mmu_attributes(attribute_fields) + + STAGE1_DESCRIPTOR::TYPE::Block + + STAGE1_DESCRIPTOR::LVL2_OUTPUT_ADDR_4KiB.val(shifted as u64), + )) + } + + fn value(&self) -> u64 { + self.0.value } } -const NUM_ENTRIES_4KIB: usize = 512; +/// A page descriptor with 4 KiB aperture. +/// +/// The output points to physical memory. +struct PageDescriptor(register::FieldValue); + +impl PageDescriptor { + fn new( + output_addr: usize, + attribute_fields: AttributeFields, + ) -> Result { + if output_addr % FOUR_KIB != 0 { + return Err("PageDescriptor: Address is not 4 KiB aligned."); + } + + let shifted = output_addr >> FOUR_KIB_SHIFT; + + Ok(PageDescriptor( + STAGE1_DESCRIPTOR::VALID::True + + STAGE1_DESCRIPTOR::AF::True + + into_mmu_attributes(attribute_fields) + + STAGE1_DESCRIPTOR::TYPE::Table + + STAGE1_DESCRIPTOR::NEXT_LVL_TABLE_ADDR_4KiB.val(shifted as u64), + )) + } -// We need a wrapper struct here so that we can make use of the align attribute. -#[repr(C)] -#[repr(align(4096))] -struct PageTable { - entries: [u64; NUM_ENTRIES_4KIB], + fn value(&self) -> u64 { + self.0.value + } } -static mut LVL2_TABLE: PageTable = PageTable { - entries: [0; NUM_ENTRIES_4KIB], -}; -static mut SINGLE_LVL3_TABLE: PageTable = PageTable { - entries: [0; NUM_ENTRIES_4KIB], -}; +/// Constants for indexing the MAIR_EL1. +#[allow(dead_code)] +mod mair { + pub const DEVICE: u64 = 0; + pub const NORMAL: u64 = 1; + pub const NORMAL_NON_CACHEABLE: u64 = 2; +} -/// Set up identity mapped page tables for the first 1 GiB of address space. -pub unsafe fn init() { - // First, define the three memory types that we will map. Cacheable and +/// Setup function for the MAIR_EL1 register. +fn set_up_mair() { + // Define the three memory types that we will map. Cacheable and // non-cacheable normal DRAM, and device. MAIR_EL1.write( // Attribute 2 MAIR_EL1::Attr2_HIGH::Memory_OuterNonCacheable + MAIR_EL1::Attr2_LOW_MEMORY::InnerNonCacheable - // Attribute 1 + // Attribute 1 + MAIR_EL1::Attr1_HIGH::Memory_OuterWriteBack_NonTransient_ReadAlloc_WriteAlloc + MAIR_EL1::Attr1_LOW_MEMORY::InnerWriteBack_NonTransient_ReadAlloc_WriteAlloc @@ -116,92 +225,97 @@ pub unsafe fn init() { + MAIR_EL1::Attr0_HIGH::Device + MAIR_EL1::Attr0_LOW_DEVICE::Device_nGnRE, ); +} + +trait BaseAddr { + fn base_addr_u64(&self) -> u64; + fn base_addr_usize(&self) -> usize; +} - // Descriptive consts for indexing into the correct MAIR_EL1 attributes. - #[allow(dead_code)] - mod mair { - pub const DEVICE: u64 = 0; - pub const NORMAL: u64 = 1; - pub const NORMAL_NON_CACHEABLE: u64 = 2; +impl BaseAddr for [u64; 512] { + fn base_addr_u64(&self) -> u64 { + self as *const u64 as u64 } - // The first 2 MiB. - // - // Set up the first LVL2 entry, pointing to the base address of a follow-up - // table containing 4 KiB pages. - // - // 0x0000_0000_0000_0000 | - // |> 2 MiB - // 0x0000_0000_001F_FFFF | - let lvl3_base: u64 = SINGLE_LVL3_TABLE.entries.base_addr() >> 12; - LVL2_TABLE.entries[0] = (STAGE1_DESCRIPTOR::VALID::True - + STAGE1_DESCRIPTOR::TYPE::Table - + STAGE1_DESCRIPTOR::NEXT_LVL_TABLE_ADDR_4KiB.val(lvl3_base)) - .value; + fn base_addr_usize(&self) -> usize { + self as *const u64 as usize + } +} + +const NUM_ENTRIES_4KIB: usize = 512; + +// A wrapper struct is needed here so that the align attribute can be used. +#[repr(C)] +#[repr(align(4096))] +struct PageTable { + entries: [u64; NUM_ENTRIES_4KIB], +} + +/// The LVL2 page table containng the 2 MiB entries. +static mut LVL2_TABLE: PageTable = PageTable { + entries: [0; NUM_ENTRIES_4KIB], +}; + +/// The LVL3 page table containing the 4 KiB entries. +/// +/// The first entry of the LVL2_TABLE will forward to this table. +static mut LVL3_TABLE: PageTable = PageTable { + entries: [0; NUM_ENTRIES_4KIB], +}; + +/// Set up identity mapped page tables for the first 1 GiB of address space. +/// +/// The first 2 MiB are 4 KiB granule, the rest 2 MiB. +pub unsafe fn init() -> Result<(), &'static str> { + // Prepare the memory attribute indirection register. + set_up_mair(); + + // Point the first 2 MiB of virtual addresses to the follow-up LVL3 + // page-table. + LVL2_TABLE.entries[0] = match TableDescriptor::new(LVL3_TABLE.entries.base_addr_usize()) { + Err(s) => return Err(s), + Ok(d) => d.value(), + }; // Fill the rest of the LVL2 (2 MiB) entries as block descriptors. // - // Differentiate between - // - cacheable DRAM - // - device memory - // - // Ranges are stored in memory.rs. - // - // 0x0000_0000_0020_0000 | - // |> 1006 MiB cacheable DRAM - // 0x0000_0000_3EFF_FFFF | - // 0x0000_0000_3F00_0000 | - // |> 16 MiB device (MMIO) - // 0x0000_0000_4000_0000 | - let mmio_first_block_index: u64 = (crate::memory::map::MMIO_BASE >> 21).into(); - - let common = STAGE1_DESCRIPTOR::VALID::True - + STAGE1_DESCRIPTOR::TYPE::Block - + STAGE1_DESCRIPTOR::AP::RW_EL1 - + STAGE1_DESCRIPTOR::AF::True - + STAGE1_DESCRIPTOR::XN::True; - // Notice the skip(1) which makes the iteration start at the second 2 MiB // block (0x20_0000). - for (i, entry) in LVL2_TABLE.entries.iter_mut().enumerate().skip(1) { - let j: u64 = i as u64; + for (block_descriptor_nr, entry) in LVL2_TABLE.entries.iter_mut().enumerate().skip(1) { + let virt_addr = block_descriptor_nr << TWO_MIB_SHIFT; - let mem_attr = if j >= mmio_first_block_index { - STAGE1_DESCRIPTOR::SH::OuterShareable + STAGE1_DESCRIPTOR::AttrIndx.val(mair::DEVICE) - } else { - STAGE1_DESCRIPTOR::SH::InnerShareable + STAGE1_DESCRIPTOR::AttrIndx.val(mair::NORMAL) + let (output_addr, attribute_fields) = match get_virt_addr_properties(virt_addr) { + Err(s) => return Err(s), + Ok((a, b)) => (a, b), }; - *entry = (common + mem_attr + STAGE1_DESCRIPTOR::LVL2_OUTPUT_ADDR_4KiB.val(j)).value; - } - - // Finally, fill the single LVL3 table (4 KiB granule). Differentiate - // between code+RO and RW pages. - let (ro_start_addr, ro_end_addr) = crate::memory::get_ro_start_end(); + let block_desc = match Lvl2BlockDescriptor::new(output_addr, attribute_fields) { + Err(s) => return Err(s), + Ok(desc) => desc, + }; - let ro_first_page_index = ro_start_addr / crate::memory::PAGESIZE; - let ro_last_page_index = ro_end_addr / crate::memory::PAGESIZE; + *entry = block_desc.value(); + } - let common = STAGE1_DESCRIPTOR::VALID::True - + STAGE1_DESCRIPTOR::TYPE::Table - + STAGE1_DESCRIPTOR::AttrIndx.val(mair::NORMAL) - + STAGE1_DESCRIPTOR::SH::InnerShareable - + STAGE1_DESCRIPTOR::AF::True; + // Finally, fill the single LVL3 table (4 KiB granule). + for (page_descriptor_nr, entry) in LVL3_TABLE.entries.iter_mut().enumerate() { + let virt_addr = page_descriptor_nr << FOUR_KIB_SHIFT; - for (i, entry) in SINGLE_LVL3_TABLE.entries.iter_mut().enumerate() { - let j: u64 = i as u64; + let (output_addr, attribute_fields) = match get_virt_addr_properties(virt_addr) { + Err(s) => return Err(s), + Ok((a, b)) => (a, b), + }; - let mem_attr = if (ro_first_page_index..=ro_last_page_index).contains(&j) { - STAGE1_DESCRIPTOR::AP::RO_EL1 + STAGE1_DESCRIPTOR::XN::False - } else { - STAGE1_DESCRIPTOR::AP::RW_EL1 + STAGE1_DESCRIPTOR::XN::True + let page_desc = match PageDescriptor::new(output_addr, attribute_fields) { + Err(s) => return Err(s), + Ok(desc) => desc, }; - *entry = (common + mem_attr + STAGE1_DESCRIPTOR::NEXT_LVL_TABLE_ADDR_4KiB.val(j)).value; + *entry = page_desc.value(); } // Point to the LVL2 table base address in TTBR0. - TTBR0_EL1.set_baddr(LVL2_TABLE.entries.base_addr()); + TTBR0_EL1.set_baddr(LVL2_TABLE.entries.base_addr_u64()); // Configure various settings of stage 1 of the EL1 translation regime. let ips = ID_AA64MMFR0_EL1.read(ID_AA64MMFR0_EL1::PARange); @@ -226,4 +340,6 @@ pub unsafe fn init() { // Force MMU init to complete before next instruction barrier::isb(barrier::SY); + + Ok(()) } diff --git a/0F_DMA_memory/README.md b/0F_DMA_memory/README.md index 6e01a2d7..4d20bbed 100644 --- a/0F_DMA_memory/README.md +++ b/0F_DMA_memory/README.md @@ -8,16 +8,21 @@ This lesson will teach about: Uart later (which now needs the memory allocator that theoretically could fail - which the MiniUart could then print). + +## Output + ```console +ferris@box:~$ make raspboot + [0] MiniUart online. [1] Press a key to continue booting... Greetings fellow Rustacean! -[2] Switching MMU on now... MMU online. -[i] Memory layout: - 0x00000000 - 0x0007FFFF | 512 KiB | Kernel stack - 0x00080000 - 0x00083FFF | 16 KiB | Kernel code and RO data - 0x00084000 - 0x00087007 | 12 KiB | Kernel data and BSS - 0x00200000 - 0x005FFFFF | 4 MiB | DMA heap pool - 0x3F000000 - 0x3FFFFFFF | 16 MiB | Device MMIO +[2] MMU online. +[i] Kernel memory layout: + 0x00000000 - 0x0007FFFF | 512 KiB | C RW PXN | Kernel stack + 0x00080000 - 0x00083FFF | 16 KiB | C RO PX | Kernel code and RO data + 0x00084000 - 0x0008700F | 12 KiB | C RW PXN | Kernel data and BSS + 0x00200000 - 0x005FFFFF | 4 MiB | NC RW PXN | DMA heap pool + 0x3F000000 - 0x3FFFFFFF | 16 MiB | Dev RW PXN | Device MMIO [i] Global DMA Allocator: Allocated Addr 0x00200000 Size 0x90 [3] Videocore Mailbox set up (DMA mem heap allocation successful). diff --git a/0F_DMA_memory/kernel8 b/0F_DMA_memory/kernel8 index 48adf027..45c05e25 100755 Binary files a/0F_DMA_memory/kernel8 and b/0F_DMA_memory/kernel8 differ diff --git a/0F_DMA_memory/kernel8.img b/0F_DMA_memory/kernel8.img index 6b55311e..194369ce 100755 Binary files a/0F_DMA_memory/kernel8.img and b/0F_DMA_memory/kernel8.img differ diff --git a/0F_DMA_memory/src/devices/hw/gpio.rs b/0F_DMA_memory/src/devices/hw/gpio.rs index 0d06dfa3..7affea08 100644 --- a/0F_DMA_memory/src/devices/hw/gpio.rs +++ b/0F_DMA_memory/src/devices/hw/gpio.rs @@ -97,7 +97,7 @@ pub struct RegisterBlock { /// Public interface to the GPIO MMIO area pub struct GPIO { - base_addr: u32, + base_addr: usize, } impl ops::Deref for GPIO { @@ -109,7 +109,7 @@ impl ops::Deref for GPIO { } impl GPIO { - pub fn new(base_addr: u32) -> GPIO { + pub fn new(base_addr: usize) -> GPIO { GPIO { base_addr } } diff --git a/0F_DMA_memory/src/devices/hw/mini_uart.rs b/0F_DMA_memory/src/devices/hw/mini_uart.rs index 4f814268..a03ed8e4 100644 --- a/0F_DMA_memory/src/devices/hw/mini_uart.rs +++ b/0F_DMA_memory/src/devices/hw/mini_uart.rs @@ -123,7 +123,7 @@ pub struct RegisterBlock { } pub struct MiniUart { - base_addr: u32, + base_addr: usize, } /// Deref to RegisterBlock @@ -145,7 +145,7 @@ impl ops::Deref for MiniUart { } impl MiniUart { - pub fn new(base_addr: u32) -> MiniUart { + pub fn new(base_addr: usize) -> MiniUart { MiniUart { base_addr } } diff --git a/0F_DMA_memory/src/devices/hw/pl011_uart.rs b/0F_DMA_memory/src/devices/hw/pl011_uart.rs index f98534e4..44580d35 100644 --- a/0F_DMA_memory/src/devices/hw/pl011_uart.rs +++ b/0F_DMA_memory/src/devices/hw/pl011_uart.rs @@ -141,7 +141,7 @@ pub enum PL011UartError { pub type Result = ::core::result::Result; pub struct PL011Uart { - base_addr: u32, + base_addr: usize, } impl ops::Deref for PL011Uart { @@ -153,7 +153,7 @@ impl ops::Deref for PL011Uart { } impl PL011Uart { - pub fn new(base_addr: u32) -> PL011Uart { + pub fn new(base_addr: usize) -> PL011Uart { PL011Uart { base_addr } } diff --git a/0F_DMA_memory/src/devices/hw/videocore_mbox.rs b/0F_DMA_memory/src/devices/hw/videocore_mbox.rs index fad36461..729777aa 100644 --- a/0F_DMA_memory/src/devices/hw/videocore_mbox.rs +++ b/0F_DMA_memory/src/devices/hw/videocore_mbox.rs @@ -87,7 +87,7 @@ const MBOX_SIZE: usize = 36; // Public interface to the mailbox pub struct VideocoreMbox<'a> { pub buffer: &'a mut [u32], - base_addr: u32, + base_addr: usize, } /// Deref to RegisterBlock @@ -109,7 +109,7 @@ impl<'a> ops::Deref for VideocoreMbox<'a> { } impl<'a> VideocoreMbox<'a> { - pub fn new(base_addr: u32) -> ::core::result::Result, ()> { + pub fn new(base_addr: usize) -> ::core::result::Result, ()> { let ret = crate::DMA_ALLOCATOR.lock(|d| d.alloc_slice_zeroed(MBOX_SIZE, MBOX_ALIGNMENT)); if ret.is_err() { diff --git a/0F_DMA_memory/src/main.rs b/0F_DMA_memory/src/main.rs index a72c9a13..4f4b2405 100644 --- a/0F_DMA_memory/src/main.rs +++ b/0F_DMA_memory/src/main.rs @@ -45,8 +45,8 @@ static CONSOLE: sync::NullLock = /// non-cacheable in the page tables. static DMA_ALLOCATOR: sync::NullLock = sync::NullLock::new(memory::BumpAllocator::new( - memory::map::DMA_HEAP_START as usize, - memory::map::DMA_HEAP_END as usize, + memory::map::virt::DMA_HEAP_START as usize, + memory::map::virt::DMA_HEAP_END as usize, "Global DMA Allocator", // Try the following arguments instead to see the PL011 UART init // fail. It will cause the allocator to use memory that are marked @@ -66,12 +66,12 @@ fn kernel_entry() -> ! { //------------------------------------------------------------ // Instantiate GPIO device //------------------------------------------------------------ - let gpio = hw::GPIO::new(memory::map::GPIO_BASE); + let gpio = hw::GPIO::new(memory::map::physical::GPIO_BASE); //------------------------------------------------------------ // Instantiate MiniUart //------------------------------------------------------------ - let mini_uart = hw::MiniUart::new(memory::map::MINI_UART_BASE); + let mini_uart = hw::MiniUart::new(memory::map::physical::MINI_UART_BASE); mini_uart.init(&gpio); CONSOLE.lock(|c| { @@ -90,24 +90,26 @@ fn kernel_entry() -> ! { }); println!("Greetings fellow Rustacean!"); - //------------------------------------------------------------ - // Bring up memory subsystem - //------------------------------------------------------------ - print!("[2] Switching MMU on now... "); - unsafe { memory::mmu::init() }; - println!("MMU online."); - - memory::print_layout(); - // We are now in a state where every next step can fail, but we can handle // the error with feedback for the user and fall through to our UART // loopback. 'init: { + //------------------------------------------------------------ + // Bring up memory subsystem + //------------------------------------------------------------ + if unsafe { memory::mmu::init() }.is_err() { + println!("[2][Error] Could not set up MMU. Aborting."); + break 'init; + }; + println!("[2] MMU online."); + + memory::print_layout(); + //------------------------------------------------------------ // Instantiate Videocore Mailbox //------------------------------------------------------------ let mut v_mbox; - match hw::VideocoreMbox::new(memory::map::VIDEOCORE_MBOX_BASE) { + match hw::VideocoreMbox::new(memory::map::physical::VIDEOCORE_MBOX_BASE) { Ok(i) => { println!("[3] Videocore Mailbox set up (DMA mem heap allocation successful)."); v_mbox = i; @@ -122,7 +124,7 @@ fn kernel_entry() -> ! { //------------------------------------------------------------ // Instantiate PL011 UART and replace MiniUart with it in CONSOLE //------------------------------------------------------------ - let pl011_uart = hw::PL011Uart::new(memory::map::PL011_UART_BASE); + let pl011_uart = hw::PL011Uart::new(memory::map::physical::PL011_UART_BASE); // uart.init() will reconfigure the GPIO, which causes a race against // the MiniUart that is still putting out characters on the physical diff --git a/0F_DMA_memory/src/memory.rs b/0F_DMA_memory/src/memory.rs index 42535974..f1da0c87 100644 --- a/0F_DMA_memory/src/memory.rs +++ b/0F_DMA_memory/src/memory.rs @@ -23,111 +23,270 @@ */ use crate::println; +use core::fmt; +use core::ops::RangeInclusive; mod bump_allocator; pub use bump_allocator::BumpAllocator; pub mod mmu; -/// The system memory map. +/// System memory map. #[rustfmt::skip] pub mod map { - pub const KERN_STACK_BOT: u32 = 0x0000_0000; - pub const KERN_STACK_TOP: u32 = 0x0007_FFFF; + pub const START: usize = 0x0000_0000; + pub const END: usize = 0x3FFF_FFFF; - /// The second 2 MiB block. - pub const DMA_HEAP_START: u32 = 0x0020_0000; - pub const DMA_HEAP_END: u32 = 0x005F_FFFF; + pub mod physical { + pub const MMIO_BASE: usize = 0x3F00_0000; + pub const VIDEOCORE_MBOX_BASE: usize = MMIO_BASE + 0x0000_B880; + pub const GPIO_BASE: usize = MMIO_BASE + 0x0020_0000; + pub const PL011_UART_BASE: usize = MMIO_BASE + 0x0020_1000; + pub const MINI_UART_BASE: usize = MMIO_BASE + 0x0021_5000; + pub const MMIO_END: usize = super::END; + } - pub const MMIO_BASE: u32 = 0x3F00_0000; - pub const VIDEOCORE_MBOX_BASE: u32 = MMIO_BASE + 0x0000_B880; - pub const GPIO_BASE: u32 = MMIO_BASE + 0x0020_0000; - pub const PL011_UART_BASE: u32 = MMIO_BASE + 0x0020_1000; - pub const MINI_UART_BASE: u32 = MMIO_BASE + 0x0021_5000; + pub mod virt { + pub const KERN_STACK_START: usize = super::START; + pub const KERN_STACK_END: usize = 0x0007_FFFF; - pub const PHYS_ADDR_MAX: u32 = 0x3FFF_FFFF; + // The second 2 MiB block. + pub const DMA_HEAP_START: usize = 0x0020_0000; + pub const DMA_HEAP_END: usize = 0x005F_FFFF; + } } -const PAGESIZE: u64 = 4096; +/// Types used for compiling the virtual memory layout of the kernel using +/// address ranges. +pub mod kernel_mem_range { + use core::ops::RangeInclusive; -#[inline] -fn aligned_addr_unchecked(addr: usize, alignment: usize) -> usize { - (addr + (alignment - 1)) & !(alignment - 1) + #[derive(Copy, Clone)] + pub enum MemAttributes { + CacheableDRAM, + NonCacheableDRAM, + Device, + } + + #[derive(Copy, Clone)] + pub enum AccessPermissions { + ReadOnly, + ReadWrite, + } + + #[allow(dead_code)] + #[derive(Copy, Clone)] + pub enum Translation { + Identity, + Offset(usize), + } + + #[derive(Copy, Clone)] + pub struct AttributeFields { + pub mem_attributes: MemAttributes, + pub acc_perms: AccessPermissions, + pub execute_never: bool, + } + + impl Default for AttributeFields { + fn default() -> AttributeFields { + AttributeFields { + mem_attributes: MemAttributes::CacheableDRAM, + acc_perms: AccessPermissions::ReadWrite, + execute_never: true, + } + } + } + + pub struct Descriptor { + pub name: &'static str, + pub virtual_range: fn() -> RangeInclusive, + pub translation: Translation, + pub attribute_fields: AttributeFields, + } } -fn get_ro_start_end() -> (u64, u64) { - // Using the linker script, we ensure that the RO area is consecutive and 4 - // KiB aligned, and we export the boundaries via symbols. - extern "C" { - // The inclusive start of the read-only area, aka the address of the - // first byte of the area. - static __ro_start: u64; - - // The non-inclusive end of the read-only area, aka the address of the - // first byte _after_ the RO area. - static __ro_end: u64; +use kernel_mem_range::*; + +/// A virtual memory layout that is agnostic of the paging granularity that the +/// hardware MMU will use. +/// +/// Contains only special ranges, aka anything that is _not_ normal cacheable +/// DRAM. +static KERNEL_VIRTUAL_LAYOUT: [Descriptor; 5] = [ + Descriptor { + name: "Kernel stack", + virtual_range: || { + RangeInclusive::new(map::virt::KERN_STACK_START, map::virt::KERN_STACK_END) + }, + translation: Translation::Identity, + attribute_fields: AttributeFields { + mem_attributes: MemAttributes::CacheableDRAM, + acc_perms: AccessPermissions::ReadWrite, + execute_never: true, + }, + }, + Descriptor { + name: "Kernel code and RO data", + virtual_range: || { + // Using the linker script, we ensure that the RO area is consecutive and 4 + // KiB aligned, and we export the boundaries via symbols: + // + // [__ro_start, __ro_end) + extern "C" { + // The inclusive start of the read-only area, aka the address of the + // first byte of the area. + static __ro_start: u64; + + // The exclusive end of the read-only area, aka the address of + // the first byte _after_ the RO area. + static __ro_end: u64; + } + + unsafe { + // Notice the subtraction to turn the exclusive end into an + // inclusive end + RangeInclusive::new( + &__ro_start as *const _ as usize, + &__ro_end as *const _ as usize - 1, + ) + } + }, + translation: Translation::Identity, + attribute_fields: AttributeFields { + mem_attributes: MemAttributes::CacheableDRAM, + acc_perms: AccessPermissions::ReadOnly, + execute_never: false, + }, + }, + Descriptor { + name: "Kernel data and BSS", + virtual_range: || { + extern "C" { + static __ro_end: u64; + static __bss_end: u64; + } + + unsafe { + RangeInclusive::new( + &__ro_end as *const _ as usize, + &__bss_end as *const _ as usize - 1, + ) + } + }, + translation: Translation::Identity, + attribute_fields: AttributeFields { + mem_attributes: MemAttributes::CacheableDRAM, + acc_perms: AccessPermissions::ReadWrite, + execute_never: true, + }, + }, + Descriptor { + name: "DMA heap pool", + virtual_range: || RangeInclusive::new(map::virt::DMA_HEAP_START, map::virt::DMA_HEAP_END), + translation: Translation::Identity, + attribute_fields: AttributeFields { + mem_attributes: MemAttributes::NonCacheableDRAM, + acc_perms: AccessPermissions::ReadWrite, + execute_never: true, + }, + }, + Descriptor { + name: "Device MMIO", + virtual_range: || RangeInclusive::new(map::physical::MMIO_BASE, map::physical::MMIO_END), + translation: Translation::Identity, + attribute_fields: AttributeFields { + mem_attributes: MemAttributes::Device, + acc_perms: AccessPermissions::ReadWrite, + execute_never: true, + }, + }, +]; + +/// For a given virtual address, find and return the output address and +/// according attributes. +/// +/// If the address is not covered in VIRTUAL_LAYOUT, return a default for normal +/// cacheable DRAM. +fn get_virt_addr_properties(virt_addr: usize) -> Result<(usize, AttributeFields), &'static str> { + if virt_addr > map::END { + return Err("Address out of range."); } - unsafe { - // Notice the subtraction to calculate the last page index of the RO - // area and not the first page index after the RO area. - ( - &__ro_start as *const _ as u64, - &__ro_end as *const _ as u64 - 1, - ) + for i in KERNEL_VIRTUAL_LAYOUT.iter() { + if (i.virtual_range)().contains(&virt_addr) { + let output_addr = match i.translation { + Translation::Identity => virt_addr, + Translation::Offset(a) => a + (virt_addr - (i.virtual_range)().start()), + }; + + return Ok((output_addr, i.attribute_fields)); + } } + + Ok((virt_addr, AttributeFields::default())) } -pub fn print_layout() { - use crate::memory::map::*; +/// Human-readable output of a Descriptor. +impl fmt::Display for Descriptor { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + // Call the function to which self.range points, and dereference the + // result, which causes Rust to copy the value. + let start = *(self.virtual_range)().start(); + let end = *(self.virtual_range)().end(); + let size = end - start + 1; + + // log2(1024) + const KIB_RSHIFT: u32 = 10; + + // log2(1024 * 1024) + const MIB_RSHIFT: u32 = 20; - // log2(1024) - const KIB_RSHIFT: u32 = 10; + let (size, unit) = if (size >> MIB_RSHIFT) > 0 { + (size >> MIB_RSHIFT, "MiB") + } else if (size >> KIB_RSHIFT) > 0 { + (size >> KIB_RSHIFT, "KiB") + } else { + (size, "Byte") + }; - // log2(1024 * 1024) - const MIB_RSHIFT: u32 = 20; + let attr = match self.attribute_fields.mem_attributes { + MemAttributes::CacheableDRAM => "C", + MemAttributes::NonCacheableDRAM => "NC", + MemAttributes::Device => "Dev", + }; - println!("[i] Memory layout:"); + let acc_p = match self.attribute_fields.acc_perms { + AccessPermissions::ReadOnly => "RO", + AccessPermissions::ReadWrite => "RW", + }; - println!( - " {:#010X} - {:#010X} | {: >4} KiB | Kernel stack", - KERN_STACK_BOT, - KERN_STACK_TOP, - (KERN_STACK_TOP - KERN_STACK_BOT + 1) >> KIB_RSHIFT - ); + let xn = if self.attribute_fields.execute_never { + "PXN" + } else { + "PX" + }; - let (ro_start, ro_end) = get_ro_start_end(); - println!( - " {:#010X} - {:#010X} | {: >4} KiB | Kernel code and RO data", - ro_start, - ro_end, - (ro_end - ro_start + 1) >> KIB_RSHIFT - ); + write!( + f, + " {:#010X} - {:#010X} | {: >3} {} | {: <3} {} {: <3} | {}", + start, end, size, unit, attr, acc_p, xn, self.name + ) + } +} + +/// Print the kernel memory layout. +pub fn print_layout() { + println!("[i] Kernel memory layout:"); - extern "C" { - static __bss_end: u64; + for i in KERNEL_VIRTUAL_LAYOUT.iter() { + println!("{}", i); } +} - let start = ro_end + 1; - let end = unsafe { &__bss_end as *const _ as u64 } - 1; - println!( - " {:#010X} - {:#010X} | {: >4} KiB | Kernel data and BSS", - start, - end, - (end - start + 1) >> KIB_RSHIFT - ); - - println!( - " {:#010X} - {:#010X} | {: >4} MiB | DMA heap pool", - DMA_HEAP_START, - DMA_HEAP_END, - (DMA_HEAP_END - DMA_HEAP_START + 1) >> MIB_RSHIFT - ); - - println!( - " {:#010X} - {:#010X} | {: >4} MiB | Device MMIO", - MMIO_BASE, - PHYS_ADDR_MAX, - (PHYS_ADDR_MAX - MMIO_BASE + 1) >> MIB_RSHIFT - ); +/// Calculate the next possible aligned address without sanity checking the +/// input parameters. +#[inline] +fn aligned_addr_unchecked(addr: usize, alignment: usize) -> usize { + (addr + (alignment - 1)) & !(alignment - 1) } diff --git a/0F_DMA_memory/src/memory/mmu.rs b/0F_DMA_memory/src/memory/mmu.rs index 1ead372f..72405e74 100644 --- a/0F_DMA_memory/src/memory/mmu.rs +++ b/0F_DMA_memory/src/memory/mmu.rs @@ -22,14 +22,15 @@ * SOFTWARE. */ +use crate::memory::{get_virt_addr_properties, AttributeFields}; use cortex_a::{barrier, regs::*}; use register::register_bitfields; register_bitfields! {u64, // AArch64 Reference Manual page 2150 STAGE1_DESCRIPTOR [ - /// Execute-never - XN OFFSET(54) NUMBITS(1) [ + /// Privileged execute-never + PXN OFFSET(53) NUMBITS(1) [ False = 0, True = 1 ], @@ -73,42 +74,150 @@ register_bitfields! {u64, ] } -trait BaseAddr { - fn base_addr(&self) -> u64; +const FOUR_KIB: usize = 4 * 1024; +const FOUR_KIB_SHIFT: usize = 12; // log2(4 * 1024) + +const TWO_MIB: usize = 2 * 1024 * 1024; +const TWO_MIB_SHIFT: usize = 21; // log2(2 * 1024 * 1024) + +/// A descriptor pointing to the next page table. +struct TableDescriptor(register::FieldValue); + +impl TableDescriptor { + fn new(next_lvl_table_addr: usize) -> Result { + if next_lvl_table_addr % FOUR_KIB != 0 { + return Err("TableDescriptor: Address is not 4 KiB aligned."); + } + + let shifted = next_lvl_table_addr >> FOUR_KIB_SHIFT; + + Ok(TableDescriptor( + STAGE1_DESCRIPTOR::VALID::True + + STAGE1_DESCRIPTOR::TYPE::Table + + STAGE1_DESCRIPTOR::NEXT_LVL_TABLE_ADDR_4KiB.val(shifted as u64), + )) + } + + fn value(&self) -> u64 { + self.0.value + } } -impl BaseAddr for [u64; 512] { - fn base_addr(&self) -> u64 { - self as *const u64 as u64 +/// A function that maps the generic memory range attributes to HW-specific +/// attributes of the MMU. +fn into_mmu_attributes( + attribute_fields: AttributeFields, +) -> register::FieldValue { + use crate::memory::{AccessPermissions, MemAttributes}; + + // Memory attributes + let mut desc = match attribute_fields.mem_attributes { + MemAttributes::CacheableDRAM => { + STAGE1_DESCRIPTOR::SH::InnerShareable + STAGE1_DESCRIPTOR::AttrIndx.val(mair::NORMAL) + } + MemAttributes::NonCacheableDRAM => { + STAGE1_DESCRIPTOR::SH::InnerShareable + + STAGE1_DESCRIPTOR::AttrIndx.val(mair::NORMAL_NON_CACHEABLE) + } + MemAttributes::Device => { + STAGE1_DESCRIPTOR::SH::OuterShareable + STAGE1_DESCRIPTOR::AttrIndx.val(mair::DEVICE) + } + }; + + // Access Permissions + desc += match attribute_fields.acc_perms { + AccessPermissions::ReadOnly => STAGE1_DESCRIPTOR::AP::RO_EL1, + AccessPermissions::ReadWrite => STAGE1_DESCRIPTOR::AP::RW_EL1, + }; + + // Execute Never + desc += if attribute_fields.execute_never { + STAGE1_DESCRIPTOR::PXN::True + } else { + STAGE1_DESCRIPTOR::PXN::False + }; + + desc +} + +/// A Level2 block descriptor with 2 MiB aperture. +/// +/// The output points to physical memory. +struct Lvl2BlockDescriptor(register::FieldValue); + +impl Lvl2BlockDescriptor { + fn new( + output_addr: usize, + attribute_fields: AttributeFields, + ) -> Result { + if output_addr % TWO_MIB != 0 { + return Err("BlockDescriptor: Address is not 2 MiB aligned."); + } + + let shifted = output_addr >> TWO_MIB_SHIFT; + + Ok(Lvl2BlockDescriptor( + STAGE1_DESCRIPTOR::VALID::True + + STAGE1_DESCRIPTOR::AF::True + + into_mmu_attributes(attribute_fields) + + STAGE1_DESCRIPTOR::TYPE::Block + + STAGE1_DESCRIPTOR::LVL2_OUTPUT_ADDR_4KiB.val(shifted as u64), + )) + } + + fn value(&self) -> u64 { + self.0.value } } -const NUM_ENTRIES_4KIB: usize = 512; +/// A page descriptor with 4 KiB aperture. +/// +/// The output points to physical memory. +struct PageDescriptor(register::FieldValue); + +impl PageDescriptor { + fn new( + output_addr: usize, + attribute_fields: AttributeFields, + ) -> Result { + if output_addr % FOUR_KIB != 0 { + return Err("PageDescriptor: Address is not 4 KiB aligned."); + } + + let shifted = output_addr >> FOUR_KIB_SHIFT; + + Ok(PageDescriptor( + STAGE1_DESCRIPTOR::VALID::True + + STAGE1_DESCRIPTOR::AF::True + + into_mmu_attributes(attribute_fields) + + STAGE1_DESCRIPTOR::TYPE::Table + + STAGE1_DESCRIPTOR::NEXT_LVL_TABLE_ADDR_4KiB.val(shifted as u64), + )) + } -// We need a wrapper struct here so that we can make use of the align attribute. -#[repr(C)] -#[repr(align(4096))] -struct PageTable { - entries: [u64; NUM_ENTRIES_4KIB], + fn value(&self) -> u64 { + self.0.value + } } -static mut LVL2_TABLE: PageTable = PageTable { - entries: [0; NUM_ENTRIES_4KIB], -}; -static mut SINGLE_LVL3_TABLE: PageTable = PageTable { - entries: [0; NUM_ENTRIES_4KIB], -}; +/// Constants for indexing the MAIR_EL1. +#[allow(dead_code)] +mod mair { + pub const DEVICE: u64 = 0; + pub const NORMAL: u64 = 1; + pub const NORMAL_NON_CACHEABLE: u64 = 2; +} -/// Set up identity mapped page tables for the first 1 GiB of address space. -pub unsafe fn init() { - // First, define the three memory types that we will map. Cacheable and +/// Setup function for the MAIR_EL1 register. +fn set_up_mair() { + // Define the three memory types that we will map. Cacheable and // non-cacheable normal DRAM, and device. MAIR_EL1.write( // Attribute 2 MAIR_EL1::Attr2_HIGH::Memory_OuterNonCacheable + MAIR_EL1::Attr2_LOW_MEMORY::InnerNonCacheable - // Attribute 1 + // Attribute 1 + MAIR_EL1::Attr1_HIGH::Memory_OuterWriteBack_NonTransient_ReadAlloc_WriteAlloc + MAIR_EL1::Attr1_LOW_MEMORY::InnerWriteBack_NonTransient_ReadAlloc_WriteAlloc @@ -116,101 +225,97 @@ pub unsafe fn init() { + MAIR_EL1::Attr0_HIGH::Device + MAIR_EL1::Attr0_LOW_DEVICE::Device_nGnRE, ); +} + +trait BaseAddr { + fn base_addr_u64(&self) -> u64; + fn base_addr_usize(&self) -> usize; +} - // Descriptive consts for indexing into the correct MAIR_EL1 attributes. - #[allow(dead_code)] - mod mair { - pub const DEVICE: u64 = 0; - pub const NORMAL: u64 = 1; - pub const NORMAL_NON_CACHEABLE: u64 = 2; +impl BaseAddr for [u64; 512] { + fn base_addr_u64(&self) -> u64 { + self as *const u64 as u64 } - // The first 2 MiB. - // - // Set up the first LVL2 entry, pointing to the base address of a follow-up - // table containing 4 KiB pages. - // - // 0x0000_0000_0000_0000 | - // |> 2 MiB - // 0x0000_0000_001F_FFFF | - let lvl3_base: u64 = SINGLE_LVL3_TABLE.entries.base_addr() >> 12; - LVL2_TABLE.entries[0] = (STAGE1_DESCRIPTOR::VALID::True - + STAGE1_DESCRIPTOR::TYPE::Table - + STAGE1_DESCRIPTOR::NEXT_LVL_TABLE_ADDR_4KiB.val(lvl3_base)) - .value; + fn base_addr_usize(&self) -> usize { + self as *const u64 as usize + } +} + +const NUM_ENTRIES_4KIB: usize = 512; + +// A wrapper struct is needed here so that the align attribute can be used. +#[repr(C)] +#[repr(align(4096))] +struct PageTable { + entries: [u64; NUM_ENTRIES_4KIB], +} + +/// The LVL2 page table containng the 2 MiB entries. +static mut LVL2_TABLE: PageTable = PageTable { + entries: [0; NUM_ENTRIES_4KIB], +}; + +/// The LVL3 page table containing the 4 KiB entries. +/// +/// The first entry of the LVL2_TABLE will forward to this table. +static mut LVL3_TABLE: PageTable = PageTable { + entries: [0; NUM_ENTRIES_4KIB], +}; + +/// Set up identity mapped page tables for the first 1 GiB of address space. +/// +/// The first 2 MiB are 4 KiB granule, the rest 2 MiB. +pub unsafe fn init() -> Result<(), &'static str> { + // Prepare the memory attribute indirection register. + set_up_mair(); + + // Point the first 2 MiB of virtual addresses to the follow-up LVL3 + // page-table. + LVL2_TABLE.entries[0] = match TableDescriptor::new(LVL3_TABLE.entries.base_addr_usize()) { + Err(s) => return Err(s), + Ok(d) => d.value(), + }; // Fill the rest of the LVL2 (2 MiB) entries as block descriptors. // - // Differentiate between - // - non-cacheable DRAM - // - cacheable DRAM - // - device memory - // - // Ranges are stored in memory.rs. - // - // 0x0000_0000_0020_0000 | - // |> 4 MiB non-cacheable DRAM - // 0x0000_0000_005F_FFFF | - // 0x0000_0000_0060_0000 | - // |> 1002 MiB cacheable DRAM - // 0x0000_0000_3EFF_FFFF | - // 0x0000_0000_3F00_0000 | - // |> 16 MiB device (MMIO) - // 0x0000_0000_4000_0000 | - let dma_first_block_index: u64 = (crate::memory::map::DMA_HEAP_START >> 21).into(); - let dma_last_block_index: u64 = (crate::memory::map::DMA_HEAP_END >> 21).into(); - let mmio_first_block_index: u64 = (crate::memory::map::MMIO_BASE >> 21).into(); - - let common = STAGE1_DESCRIPTOR::VALID::True - + STAGE1_DESCRIPTOR::TYPE::Block - + STAGE1_DESCRIPTOR::AP::RW_EL1 - + STAGE1_DESCRIPTOR::AF::True - + STAGE1_DESCRIPTOR::XN::True; - // Notice the skip(1) which makes the iteration start at the second 2 MiB // block (0x20_0000). - for (i, entry) in LVL2_TABLE.entries.iter_mut().enumerate().skip(1) { - let j: u64 = i as u64; + for (block_descriptor_nr, entry) in LVL2_TABLE.entries.iter_mut().enumerate().skip(1) { + let virt_addr = block_descriptor_nr << TWO_MIB_SHIFT; - let mem_attr = if (dma_first_block_index..=dma_last_block_index).contains(&j) { - STAGE1_DESCRIPTOR::SH::InnerShareable - + STAGE1_DESCRIPTOR::AttrIndx.val(mair::NORMAL_NON_CACHEABLE) - } else if j >= mmio_first_block_index { - STAGE1_DESCRIPTOR::SH::OuterShareable + STAGE1_DESCRIPTOR::AttrIndx.val(mair::DEVICE) - } else { - STAGE1_DESCRIPTOR::SH::InnerShareable + STAGE1_DESCRIPTOR::AttrIndx.val(mair::NORMAL) + let (output_addr, attribute_fields) = match get_virt_addr_properties(virt_addr) { + Err(s) => return Err(s), + Ok((a, b)) => (a, b), }; - *entry = (common + mem_attr + STAGE1_DESCRIPTOR::LVL2_OUTPUT_ADDR_4KiB.val(j)).value; - } - - // Finally, fill the single LVL3 table (4 KiB granule). Differentiate - // between code+RO and RW pages. - let (ro_start_addr, ro_end_addr) = crate::memory::get_ro_start_end(); + let block_desc = match Lvl2BlockDescriptor::new(output_addr, attribute_fields) { + Err(s) => return Err(s), + Ok(desc) => desc, + }; - let ro_first_page_index = ro_start_addr / crate::memory::PAGESIZE; - let ro_last_page_index = ro_end_addr / crate::memory::PAGESIZE; + *entry = block_desc.value(); + } - let common = STAGE1_DESCRIPTOR::VALID::True - + STAGE1_DESCRIPTOR::TYPE::Table - + STAGE1_DESCRIPTOR::AttrIndx.val(mair::NORMAL) - + STAGE1_DESCRIPTOR::SH::InnerShareable - + STAGE1_DESCRIPTOR::AF::True; + // Finally, fill the single LVL3 table (4 KiB granule). + for (page_descriptor_nr, entry) in LVL3_TABLE.entries.iter_mut().enumerate() { + let virt_addr = page_descriptor_nr << FOUR_KIB_SHIFT; - for (i, entry) in SINGLE_LVL3_TABLE.entries.iter_mut().enumerate() { - let j: u64 = i as u64; + let (output_addr, attribute_fields) = match get_virt_addr_properties(virt_addr) { + Err(s) => return Err(s), + Ok((a, b)) => (a, b), + }; - let mem_attr = if (ro_first_page_index..=ro_last_page_index).contains(&j) { - STAGE1_DESCRIPTOR::AP::RO_EL1 + STAGE1_DESCRIPTOR::XN::False - } else { - STAGE1_DESCRIPTOR::AP::RW_EL1 + STAGE1_DESCRIPTOR::XN::True + let page_desc = match PageDescriptor::new(output_addr, attribute_fields) { + Err(s) => return Err(s), + Ok(desc) => desc, }; - *entry = (common + mem_attr + STAGE1_DESCRIPTOR::NEXT_LVL_TABLE_ADDR_4KiB.val(j)).value; + *entry = page_desc.value(); } // Point to the LVL2 table base address in TTBR0. - TTBR0_EL1.set_baddr(LVL2_TABLE.entries.base_addr()); + TTBR0_EL1.set_baddr(LVL2_TABLE.entries.base_addr_u64()); // Configure various settings of stage 1 of the EL1 translation regime. let ips = ID_AA64MMFR0_EL1.read(ID_AA64MMFR0_EL1::PARange); @@ -235,4 +340,6 @@ pub unsafe fn init() { // Force MMU init to complete before next instruction barrier::isb(barrier::SY); + + Ok(()) } diff --git a/10_exceptions_groundwork/README.md b/10_exceptions_groundwork/README.md index 7edd68db..4f250146 100644 --- a/10_exceptions_groundwork/README.md +++ b/10_exceptions_groundwork/README.md @@ -258,29 +258,35 @@ unsafe { core::ptr::read_volatile(big_addr as *mut u64) }; ``` Finally, this triggers our exception code, because we try to read from a virtual address for which no address translations have been installed. Remember, we only installed identity-mapped page tables for the first 1 GiB of address space in lesson `0C`. -After the exception handler is finished, it returns to the first instruction after the memory read that caused the exception: +After the exception handler is finished, it returns to the first instruction +after the memory read that caused the exception. + +## Output ```console ferris@box:~$ make raspboot + [0] MiniUart online. [1] Press a key to continue booting... Greetings fellow Rustacean! -[2] Switching MMU on now... MMU online. -[i] Memory layout: - 0x00000000 - 0x0007FFFF | 512 KiB | Kernel stack - 0x00080000 - 0x00084FFF | 20 KiB | Kernel code and RO data - 0x00085000 - 0x00088007 | 12 KiB | Kernel data and BSS - 0x00200000 - 0x005FFFFF | 4 MiB | DMA heap pool - 0x3F000000 - 0x3FFFFFFF | 16 MiB | Device MMIO +[2] MMU online. +[i] Kernel memory layout: + 0x00000000 - 0x0007FFFF | 512 KiB | C RW PXN | Kernel stack + 0x00080000 - 0x00084FFF | 20 KiB | C RO PX | Kernel code and RO data + 0x00085000 - 0x0008800F | 12 KiB | C RW PXN | Kernel data and BSS + 0x00200000 - 0x005FFFFF | 4 MiB | NC RW PXN | DMA heap pool + 0x3F000000 - 0x3FFFFFFF | 16 MiB | Dev RW PXN | Device MMIO [i] Global DMA Allocator: Allocated Addr 0x00200000 Size 0x90 [3] Videocore Mailbox set up (DMA mem heap allocation successful). [4] PL011 UART online. Output switched to it. [5] Exception vectors are set up. [!] A synchronous exception happened. - ELR_EL1: 0x000809C0 + ELR_EL1: 0x00080C20 Incrementing ELR_EL1 by 4 now to continue with the first instruction after the exception! - ELR_EL1 modified: 0x000809C4 + ELR_EL1 modified: 0x00080C24 Returning from exception... [i] Whoa! We recovered from an exception. + +$> ``` diff --git a/10_exceptions_groundwork/kernel8 b/10_exceptions_groundwork/kernel8 index cb515e3e..8210cc7c 100755 Binary files a/10_exceptions_groundwork/kernel8 and b/10_exceptions_groundwork/kernel8 differ diff --git a/10_exceptions_groundwork/kernel8.img b/10_exceptions_groundwork/kernel8.img index 9b81de8d..bf33dae4 100755 Binary files a/10_exceptions_groundwork/kernel8.img and b/10_exceptions_groundwork/kernel8.img differ diff --git a/10_exceptions_groundwork/src/devices/hw/gpio.rs b/10_exceptions_groundwork/src/devices/hw/gpio.rs index 0d06dfa3..7affea08 100644 --- a/10_exceptions_groundwork/src/devices/hw/gpio.rs +++ b/10_exceptions_groundwork/src/devices/hw/gpio.rs @@ -97,7 +97,7 @@ pub struct RegisterBlock { /// Public interface to the GPIO MMIO area pub struct GPIO { - base_addr: u32, + base_addr: usize, } impl ops::Deref for GPIO { @@ -109,7 +109,7 @@ impl ops::Deref for GPIO { } impl GPIO { - pub fn new(base_addr: u32) -> GPIO { + pub fn new(base_addr: usize) -> GPIO { GPIO { base_addr } } diff --git a/10_exceptions_groundwork/src/devices/hw/mini_uart.rs b/10_exceptions_groundwork/src/devices/hw/mini_uart.rs index 4f814268..a03ed8e4 100644 --- a/10_exceptions_groundwork/src/devices/hw/mini_uart.rs +++ b/10_exceptions_groundwork/src/devices/hw/mini_uart.rs @@ -123,7 +123,7 @@ pub struct RegisterBlock { } pub struct MiniUart { - base_addr: u32, + base_addr: usize, } /// Deref to RegisterBlock @@ -145,7 +145,7 @@ impl ops::Deref for MiniUart { } impl MiniUart { - pub fn new(base_addr: u32) -> MiniUart { + pub fn new(base_addr: usize) -> MiniUart { MiniUart { base_addr } } diff --git a/10_exceptions_groundwork/src/devices/hw/pl011_uart.rs b/10_exceptions_groundwork/src/devices/hw/pl011_uart.rs index f98534e4..44580d35 100644 --- a/10_exceptions_groundwork/src/devices/hw/pl011_uart.rs +++ b/10_exceptions_groundwork/src/devices/hw/pl011_uart.rs @@ -141,7 +141,7 @@ pub enum PL011UartError { pub type Result = ::core::result::Result; pub struct PL011Uart { - base_addr: u32, + base_addr: usize, } impl ops::Deref for PL011Uart { @@ -153,7 +153,7 @@ impl ops::Deref for PL011Uart { } impl PL011Uart { - pub fn new(base_addr: u32) -> PL011Uart { + pub fn new(base_addr: usize) -> PL011Uart { PL011Uart { base_addr } } diff --git a/10_exceptions_groundwork/src/devices/hw/videocore_mbox.rs b/10_exceptions_groundwork/src/devices/hw/videocore_mbox.rs index fad36461..729777aa 100644 --- a/10_exceptions_groundwork/src/devices/hw/videocore_mbox.rs +++ b/10_exceptions_groundwork/src/devices/hw/videocore_mbox.rs @@ -87,7 +87,7 @@ const MBOX_SIZE: usize = 36; // Public interface to the mailbox pub struct VideocoreMbox<'a> { pub buffer: &'a mut [u32], - base_addr: u32, + base_addr: usize, } /// Deref to RegisterBlock @@ -109,7 +109,7 @@ impl<'a> ops::Deref for VideocoreMbox<'a> { } impl<'a> VideocoreMbox<'a> { - pub fn new(base_addr: u32) -> ::core::result::Result, ()> { + pub fn new(base_addr: usize) -> ::core::result::Result, ()> { let ret = crate::DMA_ALLOCATOR.lock(|d| d.alloc_slice_zeroed(MBOX_SIZE, MBOX_ALIGNMENT)); if ret.is_err() { diff --git a/10_exceptions_groundwork/src/main.rs b/10_exceptions_groundwork/src/main.rs index efcdd8ee..aa37ad13 100644 --- a/10_exceptions_groundwork/src/main.rs +++ b/10_exceptions_groundwork/src/main.rs @@ -47,8 +47,8 @@ static CONSOLE: sync::NullLock = /// non-cacheable in the page tables. static DMA_ALLOCATOR: sync::NullLock = sync::NullLock::new(memory::BumpAllocator::new( - memory::map::DMA_HEAP_START as usize, - memory::map::DMA_HEAP_END as usize, + memory::map::virt::DMA_HEAP_START as usize, + memory::map::virt::DMA_HEAP_END as usize, "Global DMA Allocator", )); @@ -63,12 +63,12 @@ fn kernel_entry() -> ! { //------------------------------------------------------------ // Instantiate GPIO device //------------------------------------------------------------ - let gpio = hw::GPIO::new(memory::map::GPIO_BASE); + let gpio = hw::GPIO::new(memory::map::physical::GPIO_BASE); //------------------------------------------------------------ // Instantiate MiniUart //------------------------------------------------------------ - let mini_uart = hw::MiniUart::new(memory::map::MINI_UART_BASE); + let mini_uart = hw::MiniUart::new(memory::map::physical::MINI_UART_BASE); mini_uart.init(&gpio); CONSOLE.lock(|c| { @@ -87,24 +87,26 @@ fn kernel_entry() -> ! { }); println!("Greetings fellow Rustacean!"); - //------------------------------------------------------------ - // Bring up memory subsystem - //------------------------------------------------------------ - print!("[2] Switching MMU on now... "); - unsafe { memory::mmu::init() }; - println!("MMU online."); - - memory::print_layout(); - // We are now in a state where every next step can fail, but we can handle // the error with feedback for the user and fall through to our UART // loopback. 'init: { + //------------------------------------------------------------ + // Bring up memory subsystem + //------------------------------------------------------------ + if unsafe { memory::mmu::init() }.is_err() { + println!("[2][Error] Could not set up MMU. Aborting."); + break 'init; + }; + println!("[2] MMU online."); + + memory::print_layout(); + //------------------------------------------------------------ // Instantiate Videocore Mailbox //------------------------------------------------------------ let mut v_mbox; - match hw::VideocoreMbox::new(memory::map::VIDEOCORE_MBOX_BASE) { + match hw::VideocoreMbox::new(memory::map::physical::VIDEOCORE_MBOX_BASE) { Ok(i) => { println!("[3] Videocore Mailbox set up (DMA mem heap allocation successful)."); v_mbox = i; @@ -119,7 +121,7 @@ fn kernel_entry() -> ! { //------------------------------------------------------------ // Instantiate PL011 UART and replace MiniUart with it in CONSOLE //------------------------------------------------------------ - let pl011_uart = hw::PL011Uart::new(memory::map::PL011_UART_BASE); + let pl011_uart = hw::PL011Uart::new(memory::map::physical::PL011_UART_BASE); // uart.init() will reconfigure the GPIO, which causes a race against // the MiniUart that is still putting out characters on the physical diff --git a/10_exceptions_groundwork/src/memory.rs b/10_exceptions_groundwork/src/memory.rs index 42535974..f1da0c87 100644 --- a/10_exceptions_groundwork/src/memory.rs +++ b/10_exceptions_groundwork/src/memory.rs @@ -23,111 +23,270 @@ */ use crate::println; +use core::fmt; +use core::ops::RangeInclusive; mod bump_allocator; pub use bump_allocator::BumpAllocator; pub mod mmu; -/// The system memory map. +/// System memory map. #[rustfmt::skip] pub mod map { - pub const KERN_STACK_BOT: u32 = 0x0000_0000; - pub const KERN_STACK_TOP: u32 = 0x0007_FFFF; + pub const START: usize = 0x0000_0000; + pub const END: usize = 0x3FFF_FFFF; - /// The second 2 MiB block. - pub const DMA_HEAP_START: u32 = 0x0020_0000; - pub const DMA_HEAP_END: u32 = 0x005F_FFFF; + pub mod physical { + pub const MMIO_BASE: usize = 0x3F00_0000; + pub const VIDEOCORE_MBOX_BASE: usize = MMIO_BASE + 0x0000_B880; + pub const GPIO_BASE: usize = MMIO_BASE + 0x0020_0000; + pub const PL011_UART_BASE: usize = MMIO_BASE + 0x0020_1000; + pub const MINI_UART_BASE: usize = MMIO_BASE + 0x0021_5000; + pub const MMIO_END: usize = super::END; + } - pub const MMIO_BASE: u32 = 0x3F00_0000; - pub const VIDEOCORE_MBOX_BASE: u32 = MMIO_BASE + 0x0000_B880; - pub const GPIO_BASE: u32 = MMIO_BASE + 0x0020_0000; - pub const PL011_UART_BASE: u32 = MMIO_BASE + 0x0020_1000; - pub const MINI_UART_BASE: u32 = MMIO_BASE + 0x0021_5000; + pub mod virt { + pub const KERN_STACK_START: usize = super::START; + pub const KERN_STACK_END: usize = 0x0007_FFFF; - pub const PHYS_ADDR_MAX: u32 = 0x3FFF_FFFF; + // The second 2 MiB block. + pub const DMA_HEAP_START: usize = 0x0020_0000; + pub const DMA_HEAP_END: usize = 0x005F_FFFF; + } } -const PAGESIZE: u64 = 4096; +/// Types used for compiling the virtual memory layout of the kernel using +/// address ranges. +pub mod kernel_mem_range { + use core::ops::RangeInclusive; -#[inline] -fn aligned_addr_unchecked(addr: usize, alignment: usize) -> usize { - (addr + (alignment - 1)) & !(alignment - 1) + #[derive(Copy, Clone)] + pub enum MemAttributes { + CacheableDRAM, + NonCacheableDRAM, + Device, + } + + #[derive(Copy, Clone)] + pub enum AccessPermissions { + ReadOnly, + ReadWrite, + } + + #[allow(dead_code)] + #[derive(Copy, Clone)] + pub enum Translation { + Identity, + Offset(usize), + } + + #[derive(Copy, Clone)] + pub struct AttributeFields { + pub mem_attributes: MemAttributes, + pub acc_perms: AccessPermissions, + pub execute_never: bool, + } + + impl Default for AttributeFields { + fn default() -> AttributeFields { + AttributeFields { + mem_attributes: MemAttributes::CacheableDRAM, + acc_perms: AccessPermissions::ReadWrite, + execute_never: true, + } + } + } + + pub struct Descriptor { + pub name: &'static str, + pub virtual_range: fn() -> RangeInclusive, + pub translation: Translation, + pub attribute_fields: AttributeFields, + } } -fn get_ro_start_end() -> (u64, u64) { - // Using the linker script, we ensure that the RO area is consecutive and 4 - // KiB aligned, and we export the boundaries via symbols. - extern "C" { - // The inclusive start of the read-only area, aka the address of the - // first byte of the area. - static __ro_start: u64; - - // The non-inclusive end of the read-only area, aka the address of the - // first byte _after_ the RO area. - static __ro_end: u64; +use kernel_mem_range::*; + +/// A virtual memory layout that is agnostic of the paging granularity that the +/// hardware MMU will use. +/// +/// Contains only special ranges, aka anything that is _not_ normal cacheable +/// DRAM. +static KERNEL_VIRTUAL_LAYOUT: [Descriptor; 5] = [ + Descriptor { + name: "Kernel stack", + virtual_range: || { + RangeInclusive::new(map::virt::KERN_STACK_START, map::virt::KERN_STACK_END) + }, + translation: Translation::Identity, + attribute_fields: AttributeFields { + mem_attributes: MemAttributes::CacheableDRAM, + acc_perms: AccessPermissions::ReadWrite, + execute_never: true, + }, + }, + Descriptor { + name: "Kernel code and RO data", + virtual_range: || { + // Using the linker script, we ensure that the RO area is consecutive and 4 + // KiB aligned, and we export the boundaries via symbols: + // + // [__ro_start, __ro_end) + extern "C" { + // The inclusive start of the read-only area, aka the address of the + // first byte of the area. + static __ro_start: u64; + + // The exclusive end of the read-only area, aka the address of + // the first byte _after_ the RO area. + static __ro_end: u64; + } + + unsafe { + // Notice the subtraction to turn the exclusive end into an + // inclusive end + RangeInclusive::new( + &__ro_start as *const _ as usize, + &__ro_end as *const _ as usize - 1, + ) + } + }, + translation: Translation::Identity, + attribute_fields: AttributeFields { + mem_attributes: MemAttributes::CacheableDRAM, + acc_perms: AccessPermissions::ReadOnly, + execute_never: false, + }, + }, + Descriptor { + name: "Kernel data and BSS", + virtual_range: || { + extern "C" { + static __ro_end: u64; + static __bss_end: u64; + } + + unsafe { + RangeInclusive::new( + &__ro_end as *const _ as usize, + &__bss_end as *const _ as usize - 1, + ) + } + }, + translation: Translation::Identity, + attribute_fields: AttributeFields { + mem_attributes: MemAttributes::CacheableDRAM, + acc_perms: AccessPermissions::ReadWrite, + execute_never: true, + }, + }, + Descriptor { + name: "DMA heap pool", + virtual_range: || RangeInclusive::new(map::virt::DMA_HEAP_START, map::virt::DMA_HEAP_END), + translation: Translation::Identity, + attribute_fields: AttributeFields { + mem_attributes: MemAttributes::NonCacheableDRAM, + acc_perms: AccessPermissions::ReadWrite, + execute_never: true, + }, + }, + Descriptor { + name: "Device MMIO", + virtual_range: || RangeInclusive::new(map::physical::MMIO_BASE, map::physical::MMIO_END), + translation: Translation::Identity, + attribute_fields: AttributeFields { + mem_attributes: MemAttributes::Device, + acc_perms: AccessPermissions::ReadWrite, + execute_never: true, + }, + }, +]; + +/// For a given virtual address, find and return the output address and +/// according attributes. +/// +/// If the address is not covered in VIRTUAL_LAYOUT, return a default for normal +/// cacheable DRAM. +fn get_virt_addr_properties(virt_addr: usize) -> Result<(usize, AttributeFields), &'static str> { + if virt_addr > map::END { + return Err("Address out of range."); } - unsafe { - // Notice the subtraction to calculate the last page index of the RO - // area and not the first page index after the RO area. - ( - &__ro_start as *const _ as u64, - &__ro_end as *const _ as u64 - 1, - ) + for i in KERNEL_VIRTUAL_LAYOUT.iter() { + if (i.virtual_range)().contains(&virt_addr) { + let output_addr = match i.translation { + Translation::Identity => virt_addr, + Translation::Offset(a) => a + (virt_addr - (i.virtual_range)().start()), + }; + + return Ok((output_addr, i.attribute_fields)); + } } + + Ok((virt_addr, AttributeFields::default())) } -pub fn print_layout() { - use crate::memory::map::*; +/// Human-readable output of a Descriptor. +impl fmt::Display for Descriptor { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + // Call the function to which self.range points, and dereference the + // result, which causes Rust to copy the value. + let start = *(self.virtual_range)().start(); + let end = *(self.virtual_range)().end(); + let size = end - start + 1; + + // log2(1024) + const KIB_RSHIFT: u32 = 10; + + // log2(1024 * 1024) + const MIB_RSHIFT: u32 = 20; - // log2(1024) - const KIB_RSHIFT: u32 = 10; + let (size, unit) = if (size >> MIB_RSHIFT) > 0 { + (size >> MIB_RSHIFT, "MiB") + } else if (size >> KIB_RSHIFT) > 0 { + (size >> KIB_RSHIFT, "KiB") + } else { + (size, "Byte") + }; - // log2(1024 * 1024) - const MIB_RSHIFT: u32 = 20; + let attr = match self.attribute_fields.mem_attributes { + MemAttributes::CacheableDRAM => "C", + MemAttributes::NonCacheableDRAM => "NC", + MemAttributes::Device => "Dev", + }; - println!("[i] Memory layout:"); + let acc_p = match self.attribute_fields.acc_perms { + AccessPermissions::ReadOnly => "RO", + AccessPermissions::ReadWrite => "RW", + }; - println!( - " {:#010X} - {:#010X} | {: >4} KiB | Kernel stack", - KERN_STACK_BOT, - KERN_STACK_TOP, - (KERN_STACK_TOP - KERN_STACK_BOT + 1) >> KIB_RSHIFT - ); + let xn = if self.attribute_fields.execute_never { + "PXN" + } else { + "PX" + }; - let (ro_start, ro_end) = get_ro_start_end(); - println!( - " {:#010X} - {:#010X} | {: >4} KiB | Kernel code and RO data", - ro_start, - ro_end, - (ro_end - ro_start + 1) >> KIB_RSHIFT - ); + write!( + f, + " {:#010X} - {:#010X} | {: >3} {} | {: <3} {} {: <3} | {}", + start, end, size, unit, attr, acc_p, xn, self.name + ) + } +} + +/// Print the kernel memory layout. +pub fn print_layout() { + println!("[i] Kernel memory layout:"); - extern "C" { - static __bss_end: u64; + for i in KERNEL_VIRTUAL_LAYOUT.iter() { + println!("{}", i); } +} - let start = ro_end + 1; - let end = unsafe { &__bss_end as *const _ as u64 } - 1; - println!( - " {:#010X} - {:#010X} | {: >4} KiB | Kernel data and BSS", - start, - end, - (end - start + 1) >> KIB_RSHIFT - ); - - println!( - " {:#010X} - {:#010X} | {: >4} MiB | DMA heap pool", - DMA_HEAP_START, - DMA_HEAP_END, - (DMA_HEAP_END - DMA_HEAP_START + 1) >> MIB_RSHIFT - ); - - println!( - " {:#010X} - {:#010X} | {: >4} MiB | Device MMIO", - MMIO_BASE, - PHYS_ADDR_MAX, - (PHYS_ADDR_MAX - MMIO_BASE + 1) >> MIB_RSHIFT - ); +/// Calculate the next possible aligned address without sanity checking the +/// input parameters. +#[inline] +fn aligned_addr_unchecked(addr: usize, alignment: usize) -> usize { + (addr + (alignment - 1)) & !(alignment - 1) } diff --git a/10_exceptions_groundwork/src/memory/mmu.rs b/10_exceptions_groundwork/src/memory/mmu.rs index 1ead372f..72405e74 100644 --- a/10_exceptions_groundwork/src/memory/mmu.rs +++ b/10_exceptions_groundwork/src/memory/mmu.rs @@ -22,14 +22,15 @@ * SOFTWARE. */ +use crate::memory::{get_virt_addr_properties, AttributeFields}; use cortex_a::{barrier, regs::*}; use register::register_bitfields; register_bitfields! {u64, // AArch64 Reference Manual page 2150 STAGE1_DESCRIPTOR [ - /// Execute-never - XN OFFSET(54) NUMBITS(1) [ + /// Privileged execute-never + PXN OFFSET(53) NUMBITS(1) [ False = 0, True = 1 ], @@ -73,42 +74,150 @@ register_bitfields! {u64, ] } -trait BaseAddr { - fn base_addr(&self) -> u64; +const FOUR_KIB: usize = 4 * 1024; +const FOUR_KIB_SHIFT: usize = 12; // log2(4 * 1024) + +const TWO_MIB: usize = 2 * 1024 * 1024; +const TWO_MIB_SHIFT: usize = 21; // log2(2 * 1024 * 1024) + +/// A descriptor pointing to the next page table. +struct TableDescriptor(register::FieldValue); + +impl TableDescriptor { + fn new(next_lvl_table_addr: usize) -> Result { + if next_lvl_table_addr % FOUR_KIB != 0 { + return Err("TableDescriptor: Address is not 4 KiB aligned."); + } + + let shifted = next_lvl_table_addr >> FOUR_KIB_SHIFT; + + Ok(TableDescriptor( + STAGE1_DESCRIPTOR::VALID::True + + STAGE1_DESCRIPTOR::TYPE::Table + + STAGE1_DESCRIPTOR::NEXT_LVL_TABLE_ADDR_4KiB.val(shifted as u64), + )) + } + + fn value(&self) -> u64 { + self.0.value + } } -impl BaseAddr for [u64; 512] { - fn base_addr(&self) -> u64 { - self as *const u64 as u64 +/// A function that maps the generic memory range attributes to HW-specific +/// attributes of the MMU. +fn into_mmu_attributes( + attribute_fields: AttributeFields, +) -> register::FieldValue { + use crate::memory::{AccessPermissions, MemAttributes}; + + // Memory attributes + let mut desc = match attribute_fields.mem_attributes { + MemAttributes::CacheableDRAM => { + STAGE1_DESCRIPTOR::SH::InnerShareable + STAGE1_DESCRIPTOR::AttrIndx.val(mair::NORMAL) + } + MemAttributes::NonCacheableDRAM => { + STAGE1_DESCRIPTOR::SH::InnerShareable + + STAGE1_DESCRIPTOR::AttrIndx.val(mair::NORMAL_NON_CACHEABLE) + } + MemAttributes::Device => { + STAGE1_DESCRIPTOR::SH::OuterShareable + STAGE1_DESCRIPTOR::AttrIndx.val(mair::DEVICE) + } + }; + + // Access Permissions + desc += match attribute_fields.acc_perms { + AccessPermissions::ReadOnly => STAGE1_DESCRIPTOR::AP::RO_EL1, + AccessPermissions::ReadWrite => STAGE1_DESCRIPTOR::AP::RW_EL1, + }; + + // Execute Never + desc += if attribute_fields.execute_never { + STAGE1_DESCRIPTOR::PXN::True + } else { + STAGE1_DESCRIPTOR::PXN::False + }; + + desc +} + +/// A Level2 block descriptor with 2 MiB aperture. +/// +/// The output points to physical memory. +struct Lvl2BlockDescriptor(register::FieldValue); + +impl Lvl2BlockDescriptor { + fn new( + output_addr: usize, + attribute_fields: AttributeFields, + ) -> Result { + if output_addr % TWO_MIB != 0 { + return Err("BlockDescriptor: Address is not 2 MiB aligned."); + } + + let shifted = output_addr >> TWO_MIB_SHIFT; + + Ok(Lvl2BlockDescriptor( + STAGE1_DESCRIPTOR::VALID::True + + STAGE1_DESCRIPTOR::AF::True + + into_mmu_attributes(attribute_fields) + + STAGE1_DESCRIPTOR::TYPE::Block + + STAGE1_DESCRIPTOR::LVL2_OUTPUT_ADDR_4KiB.val(shifted as u64), + )) + } + + fn value(&self) -> u64 { + self.0.value } } -const NUM_ENTRIES_4KIB: usize = 512; +/// A page descriptor with 4 KiB aperture. +/// +/// The output points to physical memory. +struct PageDescriptor(register::FieldValue); + +impl PageDescriptor { + fn new( + output_addr: usize, + attribute_fields: AttributeFields, + ) -> Result { + if output_addr % FOUR_KIB != 0 { + return Err("PageDescriptor: Address is not 4 KiB aligned."); + } + + let shifted = output_addr >> FOUR_KIB_SHIFT; + + Ok(PageDescriptor( + STAGE1_DESCRIPTOR::VALID::True + + STAGE1_DESCRIPTOR::AF::True + + into_mmu_attributes(attribute_fields) + + STAGE1_DESCRIPTOR::TYPE::Table + + STAGE1_DESCRIPTOR::NEXT_LVL_TABLE_ADDR_4KiB.val(shifted as u64), + )) + } -// We need a wrapper struct here so that we can make use of the align attribute. -#[repr(C)] -#[repr(align(4096))] -struct PageTable { - entries: [u64; NUM_ENTRIES_4KIB], + fn value(&self) -> u64 { + self.0.value + } } -static mut LVL2_TABLE: PageTable = PageTable { - entries: [0; NUM_ENTRIES_4KIB], -}; -static mut SINGLE_LVL3_TABLE: PageTable = PageTable { - entries: [0; NUM_ENTRIES_4KIB], -}; +/// Constants for indexing the MAIR_EL1. +#[allow(dead_code)] +mod mair { + pub const DEVICE: u64 = 0; + pub const NORMAL: u64 = 1; + pub const NORMAL_NON_CACHEABLE: u64 = 2; +} -/// Set up identity mapped page tables for the first 1 GiB of address space. -pub unsafe fn init() { - // First, define the three memory types that we will map. Cacheable and +/// Setup function for the MAIR_EL1 register. +fn set_up_mair() { + // Define the three memory types that we will map. Cacheable and // non-cacheable normal DRAM, and device. MAIR_EL1.write( // Attribute 2 MAIR_EL1::Attr2_HIGH::Memory_OuterNonCacheable + MAIR_EL1::Attr2_LOW_MEMORY::InnerNonCacheable - // Attribute 1 + // Attribute 1 + MAIR_EL1::Attr1_HIGH::Memory_OuterWriteBack_NonTransient_ReadAlloc_WriteAlloc + MAIR_EL1::Attr1_LOW_MEMORY::InnerWriteBack_NonTransient_ReadAlloc_WriteAlloc @@ -116,101 +225,97 @@ pub unsafe fn init() { + MAIR_EL1::Attr0_HIGH::Device + MAIR_EL1::Attr0_LOW_DEVICE::Device_nGnRE, ); +} + +trait BaseAddr { + fn base_addr_u64(&self) -> u64; + fn base_addr_usize(&self) -> usize; +} - // Descriptive consts for indexing into the correct MAIR_EL1 attributes. - #[allow(dead_code)] - mod mair { - pub const DEVICE: u64 = 0; - pub const NORMAL: u64 = 1; - pub const NORMAL_NON_CACHEABLE: u64 = 2; +impl BaseAddr for [u64; 512] { + fn base_addr_u64(&self) -> u64 { + self as *const u64 as u64 } - // The first 2 MiB. - // - // Set up the first LVL2 entry, pointing to the base address of a follow-up - // table containing 4 KiB pages. - // - // 0x0000_0000_0000_0000 | - // |> 2 MiB - // 0x0000_0000_001F_FFFF | - let lvl3_base: u64 = SINGLE_LVL3_TABLE.entries.base_addr() >> 12; - LVL2_TABLE.entries[0] = (STAGE1_DESCRIPTOR::VALID::True - + STAGE1_DESCRIPTOR::TYPE::Table - + STAGE1_DESCRIPTOR::NEXT_LVL_TABLE_ADDR_4KiB.val(lvl3_base)) - .value; + fn base_addr_usize(&self) -> usize { + self as *const u64 as usize + } +} + +const NUM_ENTRIES_4KIB: usize = 512; + +// A wrapper struct is needed here so that the align attribute can be used. +#[repr(C)] +#[repr(align(4096))] +struct PageTable { + entries: [u64; NUM_ENTRIES_4KIB], +} + +/// The LVL2 page table containng the 2 MiB entries. +static mut LVL2_TABLE: PageTable = PageTable { + entries: [0; NUM_ENTRIES_4KIB], +}; + +/// The LVL3 page table containing the 4 KiB entries. +/// +/// The first entry of the LVL2_TABLE will forward to this table. +static mut LVL3_TABLE: PageTable = PageTable { + entries: [0; NUM_ENTRIES_4KIB], +}; + +/// Set up identity mapped page tables for the first 1 GiB of address space. +/// +/// The first 2 MiB are 4 KiB granule, the rest 2 MiB. +pub unsafe fn init() -> Result<(), &'static str> { + // Prepare the memory attribute indirection register. + set_up_mair(); + + // Point the first 2 MiB of virtual addresses to the follow-up LVL3 + // page-table. + LVL2_TABLE.entries[0] = match TableDescriptor::new(LVL3_TABLE.entries.base_addr_usize()) { + Err(s) => return Err(s), + Ok(d) => d.value(), + }; // Fill the rest of the LVL2 (2 MiB) entries as block descriptors. // - // Differentiate between - // - non-cacheable DRAM - // - cacheable DRAM - // - device memory - // - // Ranges are stored in memory.rs. - // - // 0x0000_0000_0020_0000 | - // |> 4 MiB non-cacheable DRAM - // 0x0000_0000_005F_FFFF | - // 0x0000_0000_0060_0000 | - // |> 1002 MiB cacheable DRAM - // 0x0000_0000_3EFF_FFFF | - // 0x0000_0000_3F00_0000 | - // |> 16 MiB device (MMIO) - // 0x0000_0000_4000_0000 | - let dma_first_block_index: u64 = (crate::memory::map::DMA_HEAP_START >> 21).into(); - let dma_last_block_index: u64 = (crate::memory::map::DMA_HEAP_END >> 21).into(); - let mmio_first_block_index: u64 = (crate::memory::map::MMIO_BASE >> 21).into(); - - let common = STAGE1_DESCRIPTOR::VALID::True - + STAGE1_DESCRIPTOR::TYPE::Block - + STAGE1_DESCRIPTOR::AP::RW_EL1 - + STAGE1_DESCRIPTOR::AF::True - + STAGE1_DESCRIPTOR::XN::True; - // Notice the skip(1) which makes the iteration start at the second 2 MiB // block (0x20_0000). - for (i, entry) in LVL2_TABLE.entries.iter_mut().enumerate().skip(1) { - let j: u64 = i as u64; + for (block_descriptor_nr, entry) in LVL2_TABLE.entries.iter_mut().enumerate().skip(1) { + let virt_addr = block_descriptor_nr << TWO_MIB_SHIFT; - let mem_attr = if (dma_first_block_index..=dma_last_block_index).contains(&j) { - STAGE1_DESCRIPTOR::SH::InnerShareable - + STAGE1_DESCRIPTOR::AttrIndx.val(mair::NORMAL_NON_CACHEABLE) - } else if j >= mmio_first_block_index { - STAGE1_DESCRIPTOR::SH::OuterShareable + STAGE1_DESCRIPTOR::AttrIndx.val(mair::DEVICE) - } else { - STAGE1_DESCRIPTOR::SH::InnerShareable + STAGE1_DESCRIPTOR::AttrIndx.val(mair::NORMAL) + let (output_addr, attribute_fields) = match get_virt_addr_properties(virt_addr) { + Err(s) => return Err(s), + Ok((a, b)) => (a, b), }; - *entry = (common + mem_attr + STAGE1_DESCRIPTOR::LVL2_OUTPUT_ADDR_4KiB.val(j)).value; - } - - // Finally, fill the single LVL3 table (4 KiB granule). Differentiate - // between code+RO and RW pages. - let (ro_start_addr, ro_end_addr) = crate::memory::get_ro_start_end(); + let block_desc = match Lvl2BlockDescriptor::new(output_addr, attribute_fields) { + Err(s) => return Err(s), + Ok(desc) => desc, + }; - let ro_first_page_index = ro_start_addr / crate::memory::PAGESIZE; - let ro_last_page_index = ro_end_addr / crate::memory::PAGESIZE; + *entry = block_desc.value(); + } - let common = STAGE1_DESCRIPTOR::VALID::True - + STAGE1_DESCRIPTOR::TYPE::Table - + STAGE1_DESCRIPTOR::AttrIndx.val(mair::NORMAL) - + STAGE1_DESCRIPTOR::SH::InnerShareable - + STAGE1_DESCRIPTOR::AF::True; + // Finally, fill the single LVL3 table (4 KiB granule). + for (page_descriptor_nr, entry) in LVL3_TABLE.entries.iter_mut().enumerate() { + let virt_addr = page_descriptor_nr << FOUR_KIB_SHIFT; - for (i, entry) in SINGLE_LVL3_TABLE.entries.iter_mut().enumerate() { - let j: u64 = i as u64; + let (output_addr, attribute_fields) = match get_virt_addr_properties(virt_addr) { + Err(s) => return Err(s), + Ok((a, b)) => (a, b), + }; - let mem_attr = if (ro_first_page_index..=ro_last_page_index).contains(&j) { - STAGE1_DESCRIPTOR::AP::RO_EL1 + STAGE1_DESCRIPTOR::XN::False - } else { - STAGE1_DESCRIPTOR::AP::RW_EL1 + STAGE1_DESCRIPTOR::XN::True + let page_desc = match PageDescriptor::new(output_addr, attribute_fields) { + Err(s) => return Err(s), + Ok(desc) => desc, }; - *entry = (common + mem_attr + STAGE1_DESCRIPTOR::NEXT_LVL_TABLE_ADDR_4KiB.val(j)).value; + *entry = page_desc.value(); } // Point to the LVL2 table base address in TTBR0. - TTBR0_EL1.set_baddr(LVL2_TABLE.entries.base_addr()); + TTBR0_EL1.set_baddr(LVL2_TABLE.entries.base_addr_u64()); // Configure various settings of stage 1 of the EL1 translation regime. let ips = ID_AA64MMFR0_EL1.read(ID_AA64MMFR0_EL1::PARange); @@ -235,4 +340,6 @@ pub unsafe fn init() { // Force MMU init to complete before next instruction barrier::isb(barrier::SY); + + Ok(()) }