diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index cf8ed89309..bfac929bc3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -42,28 +42,28 @@ jobs: - run: rustup component add clippy - name: cargo hack clippy (x86_64) run: | - cargo hack clippy --package hermit-kernel --each-feature --no-dev-deps --target x86_64-unknown-none --exclude-features dhcpv4,dns,gem-net,net,rtl8139,virtio-net - cargo hack clippy --package hermit-kernel --each-feature --no-dev-deps --target x86_64-unknown-none --exclude-features dhcpv4,dns,gem-net,net,rtl8139,virtio-net --features pci - cargo hack clippy --package hermit-kernel --each-feature --no-dev-deps --target x86_64-unknown-none --exclude-features gem-net,rtl8139 --features tcp,virtio-net + cargo hack clippy --package hermit-kernel --each-feature --no-dev-deps --target x86_64-unknown-none --exclude-features dhcpv4,dns,gem-net,net,rtl8139,virtio-net,msix + cargo hack clippy --package hermit-kernel --each-feature --no-dev-deps --target x86_64-unknown-none --exclude-features dhcpv4,dns,gem-net,net,rtl8139,virtio-net,msix --features pci + cargo hack clippy --package hermit-kernel --each-feature --no-dev-deps --target x86_64-unknown-none --exclude-features gem-net,rtl8139,msix --features tcp,virtio-net cargo hack clippy --package hermit-kernel --each-feature --no-dev-deps --target x86_64-unknown-none --exclude-features gem-net,rtl8139 --features pci,tcp,virtio-net - cargo hack clippy --package hermit-kernel --each-feature --no-dev-deps --target x86_64-unknown-none --exclude-features gem-net,virtio-net --features tcp,rtl8139 - cargo hack clippy --package hermit-kernel --each-feature --no-dev-deps --target x86_64-unknown-none --exclude-features gem-net,virtio-net --features pci,tcp,rtl8139 + cargo hack clippy --package hermit-kernel --each-feature --no-dev-deps --target x86_64-unknown-none --exclude-features gem-net,virtio-net,msix --features tcp,rtl8139 + cargo hack clippy --package hermit-kernel --each-feature --no-dev-deps --target x86_64-unknown-none --exclude-features gem-net,virtio-net,msix --features pci,tcp,rtl8139 - name: cargo hack clippy (aarch64) run: | - cargo hack clippy --package hermit-kernel --each-feature --no-dev-deps --target aarch64-unknown-none-softfloat --exclude-features dhcpv4,dns,gem-net,net,rtl8139,virtio-net - cargo hack clippy --package hermit-kernel --each-feature --no-dev-deps --target aarch64-unknown-none-softfloat --exclude-features dhcpv4,dns,gem-net,net,rtl8139,virtio-net --features pci - cargo hack clippy --package hermit-kernel --each-feature --no-dev-deps --target aarch64-unknown-none-softfloat --exclude-features gem-net,rtl8139 --features tcp,virtio-net - cargo hack clippy --package hermit-kernel --each-feature --no-dev-deps --target aarch64-unknown-none-softfloat --exclude-features gem-net,rtl8139 --features pci,tcp,virtio-net - cargo hack clippy --package hermit-kernel --each-feature --no-dev-deps --target aarch64-unknown-none-softfloat --exclude-features gem-net,virtio-net --features tcp,rtl8139 - cargo hack clippy --package hermit-kernel --each-feature --no-dev-deps --target aarch64-unknown-none-softfloat --exclude-features gem-net,virtio-net --features pci,tcp,rtl8139 + cargo hack clippy --package hermit-kernel --each-feature --no-dev-deps --target aarch64-unknown-none-softfloat --exclude-features dhcpv4,dns,gem-net,net,rtl8139,virtio-net,msix + cargo hack clippy --package hermit-kernel --each-feature --no-dev-deps --target aarch64-unknown-none-softfloat --exclude-features dhcpv4,dns,gem-net,net,rtl8139,virtio-net,msix --features pci + cargo hack clippy --package hermit-kernel --each-feature --no-dev-deps --target aarch64-unknown-none-softfloat --exclude-features gem-net,rtl8139,msix --features tcp,virtio-net + cargo hack clippy --package hermit-kernel --each-feature --no-dev-deps --target aarch64-unknown-none-softfloat --exclude-features gem-net,rtl8139,msix --features pci,tcp,virtio-net + cargo hack clippy --package hermit-kernel --each-feature --no-dev-deps --target aarch64-unknown-none-softfloat --exclude-features gem-net,virtio-net,msix --features tcp,rtl8139 + cargo hack clippy --package hermit-kernel --each-feature --no-dev-deps --target aarch64-unknown-none-softfloat --exclude-features gem-net,virtio-net,msix --features pci,tcp,rtl8139 - name: cargo hack clippy (riscv64) run: | - cargo hack clippy --package hermit-kernel --each-feature --no-dev-deps --target riscv64gc-unknown-none-elf --exclude-features dhcpv4,dns,gem-net,net,rtl8139,virtio-net - cargo hack clippy --package hermit-kernel --each-feature --no-dev-deps --target riscv64gc-unknown-none-elf --exclude-features dhcpv4,dns,gem-net,net,rtl8139,virtio-net --features pci - cargo hack clippy --package hermit-kernel --each-feature --no-dev-deps --target riscv64gc-unknown-none-elf --exclude-features gem-net,rtl8139 --features tcp,virtio-net - cargo hack clippy --package hermit-kernel --each-feature --no-dev-deps --target riscv64gc-unknown-none-elf --exclude-features gem-net,rtl8139 --features pci,tcp,virtio-net - cargo hack clippy --package hermit-kernel --each-feature --no-dev-deps --target riscv64gc-unknown-none-elf --exclude-features rtl8139,virtio-net --features tcp,gem-net - cargo hack clippy --package hermit-kernel --each-feature --no-dev-deps --target riscv64gc-unknown-none-elf --exclude-features rtl8139,virtio-net --features pci,tcp,gem-net + cargo hack clippy --package hermit-kernel --each-feature --no-dev-deps --target riscv64gc-unknown-none-elf --exclude-features dhcpv4,dns,gem-net,net,rtl8139,virtio-net,msix + cargo hack clippy --package hermit-kernel --each-feature --no-dev-deps --target riscv64gc-unknown-none-elf --exclude-features dhcpv4,dns,gem-net,net,rtl8139,virtio-net,msix --features pci + cargo hack clippy --package hermit-kernel --each-feature --no-dev-deps --target riscv64gc-unknown-none-elf --exclude-features gem-net,rtl8139,msix --features tcp,virtio-net + cargo hack clippy --package hermit-kernel --each-feature --no-dev-deps --target riscv64gc-unknown-none-elf --exclude-features gem-net,rtl8139,msix --features pci,tcp,virtio-net + cargo hack clippy --package hermit-kernel --each-feature --no-dev-deps --target riscv64gc-unknown-none-elf --exclude-features rtl8139,virtio-net,msix --features tcp,gem-net + cargo hack clippy --package hermit-kernel --each-feature --no-dev-deps --target riscv64gc-unknown-none-elf --exclude-features rtl8139,virtio-net,msix --features pci,tcp,gem-net format: name: Format diff --git a/Cargo.toml b/Cargo.toml index a99e36fd7d..79597f9a78 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -118,6 +118,12 @@ pci = ["virtio?/pci"] ## [PCI ID]: https://pci-ids.ucw.cz/ pci-ids = ["dep:pci-ids"] +## Causes the drivers to use _MSI-X (Message Signaled Interrupt)_ when +## the PCI device and the driver (currently only virtio-net) supports it. +## It is only supported on x86_64 and is known to not work with all virtual +## devices. +msix = ["volatile/derive"] + ## Enables semihosting support. ## ## Semihosting allows communicating with the host for diff --git a/src/drivers/net/virtio/mmio.rs b/src/drivers/net/virtio/mmio.rs index ea21245e3c..152b037b8e 100644 --- a/src/drivers/net/virtio/mmio.rs +++ b/src/drivers/net/virtio/mmio.rs @@ -39,7 +39,7 @@ impl VirtioNetDriver { notif_cfg, inner: Uninit, num_vqs: 0, - irq, + irq: Some(irq), checksums: ChecksumCapabilities::default(), }) } diff --git a/src/drivers/net/virtio/mod.rs b/src/drivers/net/virtio/mod.rs index e587e69342..b61152b1ac 100644 --- a/src/drivers/net/virtio/mod.rs +++ b/src/drivers/net/virtio/mod.rs @@ -33,6 +33,8 @@ use self::error::VirtioNetError; use crate::config::VIRTIO_MAX_QUEUE_SIZE; use crate::drivers::net::virtio::constants::BUFF_PER_PACKET; use crate::drivers::net::{NetworkDriver, mtu}; +#[cfg(feature = "msix")] +use crate::drivers::pci::MsixTableEntry; use crate::drivers::virtio::ControlRegisters; #[cfg(not(feature = "pci"))] use crate::drivers::virtio::transport::mmio::{ComCfg, IsrStatus, NotifCfg}; @@ -241,11 +243,13 @@ pub(crate) struct VirtioNetDriver { pub(super) com_cfg: ComCfg, pub(super) isr_stat: IsrStatus, pub(super) notif_cfg: NotifCfg, + #[cfg(feature = "msix")] + pub(super) msix_table: Option>, pub(super) inner: T, pub(super) num_vqs: u16, - pub(super) irq: InterruptLine, + pub(super) irq: Option, pub(super) checksums: ChecksumCapabilities, } @@ -485,7 +489,7 @@ impl smoltcp::phy::Device for VirtioNetDriver { impl Driver for VirtioNetDriver { fn get_interrupt_number(&self) -> InterruptLine { - self.irq + self.irq.unwrap() } fn get_name(&self) -> &'static str { @@ -741,11 +745,34 @@ impl VirtioNetDriver { } debug!("{:?}", self.checksums); + #[cfg(feature = "msix")] + if let Some(msix_table) = self.msix_table.as_mut() { + warn!( + "Setting up message signaled interrupts. MSI-X is known to not work on all platforms (e.g. QEMU with TAP)." + ); + // Chosen arbitatrily. Ideally does not clash with another interrupt line. + const MSIX_VECTOR: u8 = 112; + let msix_entry = unsafe { + msix_table + .as_mut_ptr() + .map(|table| table.get_unchecked_mut(0)) + }; + MsixTableEntry::configure(msix_entry, MSIX_VECTOR); + self.com_cfg.select_vq(0).unwrap().set_msix_table_index(0); + self.irq = Some(MSIX_VECTOR); + }; + + if self.irq.is_none() { + warn!("No interrupt methods found for virtio-net."); + } + Ok(VirtioNetDriver { dev_cfg: self.dev_cfg, com_cfg: self.com_cfg, isr_stat: self.isr_stat, notif_cfg: self.notif_cfg, + #[cfg(feature = "msix")] + msix_table: self.msix_table, inner, num_vqs: self.num_vqs, irq: self.irq, diff --git a/src/drivers/net/virtio/pci.rs b/src/drivers/net/virtio/pci.rs index 104ab84060..42eb51c8a8 100644 --- a/src/drivers/net/virtio/pci.rs +++ b/src/drivers/net/virtio/pci.rs @@ -36,6 +36,8 @@ impl VirtioNetDriver { notif_cfg, isr_cfg, dev_cfg_list, + #[cfg(feature = "msix")] + msix_table, .. } = caps_coll; @@ -44,14 +46,18 @@ impl VirtioNetDriver { return Err(error::VirtioNetError::NoDevCfg(device_id)); }; + let irq = device.get_irq(); + Ok(VirtioNetDriver { dev_cfg, com_cfg, isr_stat: isr_cfg, notif_cfg, + #[cfg(feature = "msix")] + msix_table, inner: Uninit, num_vqs: 0, - irq: device.get_irq().unwrap(), + irq, checksums: ChecksumCapabilities::default(), }) } diff --git a/src/drivers/pci.rs b/src/drivers/pci.rs index bfd574c6b3..f52089908f 100644 --- a/src/drivers/pci.rs +++ b/src/drivers/pci.rs @@ -537,6 +537,49 @@ pub(crate) fn init() { }); } +/// MSI-X Table entry. +#[cfg(feature = "msix")] +#[repr(C)] +#[derive(volatile::VolatileFieldAccess)] +pub(crate) struct MsixTableEntry { + /// Message Address + message_address: u32, + + /// Message Upper Address + message_upper_address: u32, + + /// Message Data + message_data: u32, + + /// Vector Control + vector_control: u32, +} + +#[cfg(feature = "msix")] +impl MsixTableEntry { + #[cfg(target_arch = "x86_64")] + pub fn configure(msix_entry: volatile::VolatilePtr<'_, Self>, vector: u8) { + use MsixTableEntryVolatileFieldAccess; + use bit_field::BitField; + + // Mask the entry because "[s]oftware must not modify the Address, Data, or Steering Tag fields + // of an entry while it is unmasked." (PCIe spec. 6.1.4.2) + msix_entry + .vector_control() + .update(|mut control| *control.set_bit(0, true)); + + msix_entry + .message_address() + .update(|mut addr_low| *addr_low.set_bits(20..32, 0xfee)); + msix_entry + .message_data() + .update(|mut data| *data.set_bits(0..8, u32::from(vector) + 32)); + msix_entry + .vector_control() + .update(|mut control| *control.set_bit(0, false)); + } +} + /// A module containing PCI specific errors /// /// Errors include... diff --git a/src/drivers/virtio/transport/pci.rs b/src/drivers/virtio/transport/pci.rs index a16aba68a6..4cb58e5e86 100644 --- a/src/drivers/virtio/transport/pci.rs +++ b/src/drivers/virtio/transport/pci.rs @@ -181,6 +181,8 @@ pub struct UniCapsColl { pub(crate) notif_cfg: NotifCfg, pub(crate) isr_cfg: IsrStatus, pub(crate) dev_cfg_list: Vec, + #[cfg(feature = "msix")] + pub(crate) msix_table: Option>, } /// Wraps a [`CommonCfg`] in order to preserve /// the original structure. @@ -256,6 +258,15 @@ impl VqCfgHandler<'_> { .write(addr.as_u64().into()); } + #[cfg(feature = "msix")] + pub fn set_msix_table_index(&mut self, index: u16) { + self.select_queue(); + self.raw + .as_mut_ptr() + .queue_msix_vector() + .write(index.into()); + } + pub fn notif_off(&mut self) -> u16 { self.select_queue(); self.raw.as_mut_ptr().queue_notify_off().read().to_ne() @@ -509,43 +520,6 @@ impl PciBar { } } -/// Reads all PCI capabilities, starting at the capabilities list pointer from the -/// PCI device. -/// -/// Returns ONLY Virtio specific capabilities, which allow to locate the actual capability -/// structures inside the memory areas, indicated by the BaseAddressRegisters (BAR's). -fn read_caps(device: &PciDevice) -> Result, PciError> { - let device_id = device.device_id(); - - let capabilities = device - .capabilities() - .unwrap() - .filter_map(|capability| match capability { - PciCapability::Vendor(capability) => Some(capability), - _ => None, - }) - .map(|addr| CapData::read(addr, device.access()).unwrap()) - .filter(|cap| cap.cfg_type != CapCfgType::Pci) - .flat_map(|cap| { - let slot = cap.bar; - device - .memory_map_bar(slot, true) - .map(|(addr, size)| PciCap { - bar: VirtioPciBar::new(slot, addr.as_u64(), size.try_into().unwrap()), - dev_id: device_id, - cap, - }) - }) - .collect::>(); - - if capabilities.is_empty() { - error!("No virtio capability found for device {device_id:x}"); - return Err(PciError::NoVirtioCaps(device_id)); - } - - Ok(capabilities) -} - pub(crate) fn map_caps(device: &PciDevice) -> Result { let device_id = device.device_id(); @@ -555,58 +529,94 @@ pub(crate) fn map_caps(device: &PciDevice) -> Result list, - Err(pci_error) => return Err(VirtioError::FromPci(pci_error)), - }; - let mut com_cfg = None; let mut notif_cfg = None; let mut isr_cfg = None; let mut dev_cfg_list = Vec::new(); - // Map Caps in virtual memory - for pci_cap in cap_list { - match pci_cap.cap.cfg_type { - CapCfgType::Common => { - if com_cfg.is_none() { - match pci_cap.map_common_cfg() { - Some(cap) => com_cfg = Some(ComCfg::new(cap)), - None => error!( - "Common config capability of device {device_id:x} could not be mapped!" - ), - } + #[cfg(feature = "msix")] + let mut msix_table = None; + + // Reads all PCI capabilities, starting at the capabilities list pointer from the + // PCI device. + // + // Maps ONLY Virtio specific capabilities and the MSI-X capability , which allow to locate the actual capability + // structures inside the memory areas, indicated by the BaseAddressRegisters (BAR's). + for capability in device.capabilities().unwrap() { + match capability { + PciCapability::Vendor(addr) => { + let cap = CapData::read(addr, device.access()).unwrap(); + if cap.cfg_type == CapCfgType::Pci { + continue; } - } - CapCfgType::Notify => { - if notif_cfg.is_none() { - match NotifCfg::new(&pci_cap) { - Some(notif) => notif_cfg = Some(notif), - None => error!( - "Notification config capability of device {device_id:x} could not be used!" - ), + let slot = cap.bar; + let Some((addr, size)) = device.memory_map_bar(slot, true) else { + continue; + }; + let pci_cap = PciCap { + bar: VirtioPciBar::new(slot, addr.as_u64(), size.try_into().unwrap()), + dev_id: device_id, + cap, + }; + match pci_cap.cap.cfg_type { + CapCfgType::Common => { + if com_cfg.is_none() { + match pci_cap.map_common_cfg() { + Some(cap) => com_cfg = Some(ComCfg::new(cap)), + None => error!( + "Common config capability of device {device_id:x} could not be mapped!" + ), + } + } } - } - } - CapCfgType::Isr => { - if isr_cfg.is_none() { - match pci_cap.map_isr_status() { - Some(isr_stat) => isr_cfg = Some(IsrStatus::new(isr_stat)), - None => error!( - "ISR status config capability of device {device_id:x} could not be used!" - ), + CapCfgType::Notify => { + if notif_cfg.is_none() { + match NotifCfg::new(&pci_cap) { + Some(notif) => notif_cfg = Some(notif), + None => error!( + "Notification config capability of device {device_id:x} could not be used!" + ), + } + } + } + CapCfgType::Isr => { + if isr_cfg.is_none() { + match pci_cap.map_isr_status() { + Some(isr_stat) => isr_cfg = Some(IsrStatus::new(isr_stat)), + None => error!( + "ISR status config capability of device {device_id:x} could not be used!" + ), + } + } } + CapCfgType::SharedMemory => { + let cap_id = pci_cap.cap.id; + error!( + "Shared Memory config capability with id {cap_id} of device {device_id:x} could not be used!" + ); + } + CapCfgType::Device => dev_cfg_list.push(pci_cap), + _ => continue, } } - CapCfgType::SharedMemory => { - let cap_id = pci_cap.cap.id; - error!( - "Shared Memory config capability with id {cap_id} of device {device_id:x} could not be used!" + #[cfg(feature = "msix")] + PciCapability::MsiX(mut msix_capability) => { + msix_capability.set_enabled(true, device.access()); + let (base_addr, _) = device + .memory_map_bar(msix_capability.table_bar(), true) + .unwrap(); + let table_ptr = NonNull::slice_from_raw_parts( + NonNull::with_exposed_provenance( + core::num::NonZero::new( + base_addr.as_usize() + + usize::try_from(msix_capability.table_offset()).unwrap(), + ) + .unwrap(), + ), + msix_capability.table_size().into(), ); + msix_table = Some(unsafe { VolatileRef::new(table_ptr) }); } - CapCfgType::Device => dev_cfg_list.push(pci_cap), - - // PCI's configuration space is allowed to hold other structures, which are not virtio specific and are therefore ignored + // PCI's configuration space is allowed to hold other structures, which are not useful for us and are therefore ignored // in the following _ => continue, } @@ -617,6 +627,8 @@ pub(crate) fn map_caps(device: &PciDevice) -> Result match VirtioNetDriver::init(device) { Ok(virt_net_drv) => { + use crate::drivers::Driver; info!("Virtio network driver initialized."); - let irq = device.get_irq().unwrap(); + let irq = virt_net_drv.get_interrupt_number(); crate::arch::interrupts::add_irq_name(irq, "virtio"); info!("Virtio interrupt handler at line {irq}");