diff --git a/Cargo.lock b/Cargo.lock index 9fbafa57..e0b263a9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2541,6 +2541,7 @@ dependencies = [ "rand 0.10.2", "rattan-log", "rtnetlink", + "rvnic", "serde", "test-log", "thiserror 2.0.18", @@ -2792,6 +2793,15 @@ version = "1.0.22" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" +[[package]] +name = "rvnic" +version = "0.1.0" +dependencies = [ + "libc", + "nix 0.30.1", + "thiserror 2.0.18", +] + [[package]] name = "ryu" version = "1.0.23" diff --git a/Cargo.toml b/Cargo.toml index d89b6086..c0eea69b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -15,6 +15,7 @@ default-run = "rattan" [workspace] resolver = "2" members = [".", "rattan-core", "rattan-env", "rattan-log"] +exclude = ["rvnic"] [workspace.package] version = "0.1.1" @@ -67,6 +68,7 @@ parking_lot = "0.12.5" pcap-file = { version = "2.0.0" } rand = "0.10.0" rtnetlink = "0.20.0" +rvnic.path = "./rvnic/librvnic/" serde = { version = "1.0.160", features = ["derive"] } serde_json = "1.0.149" tempfile = "3" @@ -88,6 +90,11 @@ tracing-subscriber = { version = "0.3.17", features = ["env-filter"] } name = "rattan" path = "src/main.rs" +[[bin]] +name = "rattan-rv" +path = "src/main-rv.rs" +required-features = ["rvnic"] + [[bin]] name = "rattan-xdp" path = "src/main-xdp.rs" @@ -131,12 +138,14 @@ shadow-rs = "1.7.0" default = ["first-packet"] drift-stat = ["rattan-core/drift-stat"] drift-log = ["rattan-core/drift-log"] +drop-stat = ["rattan-env/drop-stat"] first-packet = ["rattan-core/first-packet"] first-payload = ["rattan-core/first-payload"] http = ["rattan-core/http"] nat = ["dep:mnl", "dep:nftnl"] packet-dump = ["rattan-core/packet-dump"] xdp = ["rattan-env/xdp"] +rvnic = ["rattan-env/rvnic"] # required for cargo flamegraph [profile.dev] diff --git a/rattan-core/src/cells/external.rs b/rattan-core/src/cells/external.rs index 54a80738..be611060 100644 --- a/rattan-core/src/cells/external.rs +++ b/rattan-core/src/cells/external.rs @@ -30,7 +30,8 @@ use crate::{ radix::{PacketLogMode, BASE_TS, PKT_LOG_MODE}, }; -const SEND_CHANNEL_PACKETS: usize = 16384; +const SEND_CHANNEL_PACKETS: usize = 32768; +const RECV_CHANNEL_PACKETS: usize = 32768; #[derive(Debug, Clone, Copy)] #[repr(transparent)] @@ -119,7 +120,7 @@ where tokio::spawn(Self::send(rx, s.clone())); senders.push(tx); } - tokio::spawn(Self::demux(dev_rx, senders.clone())); + tokio::spawn(Self::demux(dev_rx, senders)); } Self { @@ -244,11 +245,18 @@ where log_tx: Option>, base_ts: i64, ) -> Result { - let (tx, rx) = tokio::sync::mpsc::channel(1024); + let (tx, rx) = tokio::sync::mpsc::channel(RECV_CHANNEL_PACKETS); let flow_map = Arc::new(FlowMap::new(id)); - for d in driver.into_iter() { - let notify = AsyncFd::new(d.raw_fd())?; + if D::use_blocking_recv() { + tracing::warn!("Recv in thread on Driver's request"); + let tx_clone = tx.clone(); + std::thread::spawn(|| D::recv_thread(d.into_receiver(), tx_clone)); + continue; + } + + let notify = AsyncFd::new(d.raw_fd()) + .inspect_err(|e| tracing::error!(?e, "Failed to epoll on raw fd {}", d.raw_fd()))?; tokio::spawn(Self::recv( flow_map.clone(), notify, @@ -278,6 +286,7 @@ where let mut _guard = notify.readable().await.unwrap(); let Ok(packet) = _guard.try_io(|_fd| receiver.receive()) else { + tokio::task::yield_now().await; continue; // would block }; diff --git a/rattan-core/src/config/mod.rs b/rattan-core/src/config/mod.rs index f770d428..ed88ebf0 100644 --- a/rattan-core/src/config/mod.rs +++ b/rattan-core/src/config/mod.rs @@ -1,4 +1,4 @@ -use std::{collections::HashMap, path::PathBuf}; +use std::{collections::HashMap, env, path::PathBuf}; use rattan_env::env::RattanEnvConfig; #[cfg(feature = "serde")] @@ -78,6 +78,31 @@ impl RattanResourceConfig { pub fn new() -> Self { Default::default() } + + pub fn working_threads(&self) -> usize { + // If env `RATTAN_WORKING_THREADS` is set, use it; otherwise use cpu cores + if let Some(threads) = env::var("RATTAN_WORKING_THREADS") + .ok() + .and_then(|v| v.parse().ok()) + { + return threads; + } + if let Some(cpu) = &self.cpu { + cpu.len() + } else { + 4 + } + } + + pub fn get_cpu(&self) -> Option> { + // If env `RATTAN_CPU` is set, use it; otherwise use the cpu cores from the config + if let Ok(cpu_list) = env::var("RATTAN_CPU") { + // split by comma and parse each core + let cpu: Vec = cpu_list.split(',').filter_map(|s| s.parse().ok()).collect(); + return Some(cpu); + } + self.cpu.clone() + } } #[cfg_attr( diff --git a/rattan-core/src/radix/mod.rs b/rattan-core/src/radix/mod.rs index 1d6263ec..35ee4631 100644 --- a/rattan-core/src/radix/mod.rs +++ b/rattan-core/src/radix/mod.rs @@ -1,12 +1,16 @@ #[cfg(feature = "http")] use std::net::IpAddr; use std::{ - sync::{mpsc, Arc}, + sync::{atomic::AtomicUsize, mpsc, Arc}, thread, time::{SystemTime, UNIX_EPOCH}, }; use backon::{BlockingRetryable, ExponentialBuilder}; +use nix::{ + sched::{sched_setaffinity, CpuSet}, + unistd::Pid, +}; use once_cell::sync::{Lazy, OnceCell}; use rattan_env::{env::standard::AfPacketDriver, InterfaceDriver, StdNetEnv}; use rattan_env::{ @@ -18,10 +22,6 @@ use rattan_log::{file_logging_thread, RattanLogOp, LOGGING_TX}; use tokio::{runtime::Runtime, time::Instant}; use tokio_util::sync::CancellationToken; use tracing::{debug, error, info, span, warn, Level}; -// use nix::{ -// sched::{sched_setaffinity, CpuSet}, -// unistd::Pid, -// }; use crate::{ cells::{ @@ -52,6 +52,44 @@ pub static BASE_TS: Lazy<(i64, u64, tokio::time::Instant)> = Lazy::new(|| { (machine_time, unix_time as u64, Instant::now()) }); +// Upon worker thread startup, this counter is incremented. So that the working threads +// are assigned unique CPU affinity from the given user-specified CPU set. +static AFFINITY_COUNTER: AtomicUsize = AtomicUsize::new(0); + +// Utility function to set CPU affinity for the current working thread. +// If anything failes, an error is printed, and the worker thread will continue. +fn set_cpu_affinity(available_cpus: &[u32]) { + let index = AFFINITY_COUNTER.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + if available_cpus.is_empty() { + return; + } + let cpu = available_cpus[index % available_cpus.len()] as usize; + + let mut cpu_set = CpuSet::new(); + if cpu_set.set(cpu).is_err() { + error!( + target : "working_thread", + "Failed to set CPU affinity: CPU {} is not available", cpu + ); + return; + } + + info!( + target : "working_thread", + "Trying to set CPU affinity for working thread to CPU {}", cpu + ); + + if let Err(e) = sched_setaffinity( + Pid::from_raw(0), // current thread + &cpu_set, + ) { + error!( + target : "working_thread", + "Failed to set CPU affinity to CPU {}: {}", cpu, e + ); + } +} + #[derive(Clone, Copy, Debug, clap::ValueEnum, PartialEq, Eq)] #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] pub enum PacketLogMode { @@ -128,7 +166,14 @@ where ) .call()?; - let running_core = config.env.get_running_cpu(); + let running_core = config.resource.get_cpu().unwrap_or_default(); + let worker_thread_cnt = config.resource.working_threads().max(1); + + if running_core.is_empty() { + info!(target : "working_thread", "Try to start {} working threads without setting CPU affinity", worker_thread_cnt); + } else { + info!(target : "working_thread", "Try to start {} working threads with CPU affinity on CPUs: {:?}", worker_thread_cnt, running_core); + }; // Log env creation to kmsg for system-level observability if let Ok(mut kmesg_logger) = std::fs::OpenOptions::new().write(true).open("/dev/kmsg") { @@ -165,15 +210,11 @@ where // TODO(enhancement): need to handle panic due to affinity setting let runtime = tokio::runtime::Builder::new_multi_thread() - .worker_threads(running_core.len()) + .worker_threads(worker_thread_cnt) .enable_all() - // .on_thread_start(move || { - // let mut cpuset = CpuSet::new(); - // for core in running_core.iter() { - // cpuset.set(*core as usize).unwrap(); - // } - // sched_setaffinity(Pid::from_raw(0), &cpuset).unwrap(); - // }) + .on_thread_start(move || { + set_cpu_affinity(running_core.as_slice()); + }) .build() .map(Arc::new); @@ -269,9 +310,15 @@ where http_thread_handle, }; // This could be veth pairs, or rvnic - radix.init_iterfaces()?; - radix.load_cells_config(config.cells)?; - radix.link_cells(config.links)?; + radix + .init_iterfaces() + .inspect_err(|e| tracing::error!(?e, "Failed to init interfaces"))?; + radix + .load_cells_config(config.cells) + .inspect_err(|e| tracing::error!(?e, "Failed to load cell config"))?; + radix + .link_cells(config.links) + .inspect_err(|e| tracing::error!(?e, "Failed to link cells"))?; Ok(radix) } @@ -299,13 +346,16 @@ where drivers, } = interface; + let name_clone = name.clone(); + self.build_cell(name.clone(), move |rt| { let _guard = rt.enter(); let mut id = VirtualEthernetId::new(); id.set_ns_id(ns_id); id.set_veth_id_copied(veth_id); VirtualEthernet::::new(drivers, id) - })?; + }) + .inspect_err(|e| tracing::error!(?e, "Failed to init interface {}", name_clone))?; Ok(()) } @@ -316,9 +366,11 @@ where let interfaces = self .env .build_interfaces(self.rattan.get_runtime_handle())?; + info!("{} interfaces to be registered as cells.", interfaces.len()); for interface in interfaces.into_iter() { self.init_interface(interface)?; } + info!("Interfaces registered as cells"); Ok(()) } diff --git a/rattan-env/Cargo.toml b/rattan-env/Cargo.toml index 3fd8c5d3..0226527d 100644 --- a/rattan-env/Cargo.toml +++ b/rattan-env/Cargo.toml @@ -16,11 +16,12 @@ futures = { workspace = true } ipnet = { workspace = true } lazy_static = "1.5.0" libc = { workspace = true } -nix = { workspace = true } +nix.workspace = true once_cell = { workspace = true } parking_lot = { workspace = true } rand = { workspace = true } rtnetlink = { workspace = true } +rvnic = { workspace = true , optional = true} serde = { workspace = true, optional = true } thiserror = { workspace = true } tokio = { workspace = true } @@ -33,6 +34,8 @@ rattan-log = { path = "../rattan-log" } test-log = { workspace = true } [features] -default = ["xdp"] +default = [] +drop-stat = [] serde = ["dep:serde"] xdp = ["dep:camellia-net"] +rvnic = ["dep:rvnic"] diff --git a/rattan-env/src/env/mod.rs b/rattan-env/src/env/mod.rs index bd13ed89..6e3725df 100644 --- a/rattan-env/src/env/mod.rs +++ b/rattan-env/src/env/mod.rs @@ -1,10 +1,12 @@ +use std::fs::create_dir_all; use std::net::IpAddr; use std::sync::Arc; +use futures::TryStreamExt; +use rtnetlink::packet_route::address::AddressAttribute; use tokio::runtime::Handle; use crate::InterfaceDriver; - use crate::{error::MetalError, InterfaceBuildArtifact, NetNs}; #[cfg(feature = "serde")] @@ -21,6 +23,8 @@ pub trait SerdeBounds {} #[cfg(not(feature = "serde"))] impl SerdeBounds for T {} +#[cfg(feature = "rvnic")] +pub mod rvnic; pub mod standard; pub trait RattanEnv { @@ -70,12 +74,68 @@ pub trait RattanEnvConfig: SerdeBounds + Default { // args: Self::BuildArgs, ) -> Result; - /// Returns the CPU for the working threads of the async runtime to run on. - fn get_running_cpu(&self) -> Vec { - vec![1] - } - fn default_with_mode(mode: >::Mode) -> Self; fn default_compatible() -> Self; fn default_isolated() -> Self; } + +const IP_LOCK_DIR: &str = "/tmp/rattan/ip_lock"; + +struct IpAddrLock { + file_dir: String, +} + +impl IpAddrLock { + /// Lock an IP address by creating a file with the same name as the it + fn new(ip: IpAddr) -> Result, std::io::Error> { + // for ipv6, ':' may be illegal as a filename + let ip_str = format!("{ip}").replace(':', "_"); + let file_dir = format!("{IP_LOCK_DIR}/{ip_str}"); + + create_dir_all(IP_LOCK_DIR)?; + match std::fs::File::create_new(&file_dir) { + // Lock successfully + Ok(_) => Ok(Some(IpAddrLock { file_dir })), + // Lock file already exists + Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => Ok(None), + // Other error + Err(e) => Err(e), + } + } +} + +impl Drop for IpAddrLock { + /// Unlock by removing the lock file + fn drop(&mut self) { + let _ = std::fs::remove_file(&self.file_dir); + } +} + +fn get_addresses_in_use() -> Result, crate::error::Error> { + tracing::debug!("Get addresses in use"); + let rt = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .map_err(|e| { + tracing::error!("Failed to build rtnetlink runtime"); + crate::error::Error::TokioRuntimeError(e.into()) + })?; + let _guard = rt.enter(); + let (conn, rtnl_handle, _) = rtnetlink::new_connection()?; + rt.spawn(conn); + + let mut addresses = vec![]; + rt.block_on(async { + let mut links = rtnl_handle.address().get().execute(); + while let Ok(Some(address_msg)) = links.try_next().await { + for address_attr in address_msg.attributes { + if let AddressAttribute::Address(address) = address_attr { + tracing::trace!(?address, ?address_msg.header.prefix_len, "Get address"); + addresses.push(address); + } + } + } + }); + tracing::debug!(?addresses, "Addresses in use"); + Ok(addresses) +} diff --git a/rattan-env/src/env/rvnic/af_rvnic.rs b/rattan-env/src/env/rvnic/af_rvnic.rs new file mode 100644 index 00000000..8d78fcc5 --- /dev/null +++ b/rattan-env/src/env/rvnic/af_rvnic.rs @@ -0,0 +1,417 @@ +use std::{os::fd::AsRawFd, sync::Arc}; + +use etherparse::{Ethernet2Header, Ipv4Header}; + +use rvnic::{sys::RattanDesc, RxRing, TxRing}; +use tokio::{ + runtime::Handle, + sync::mpsc, + time::{Duration, Instant}, +}; +use tracing::instrument; + +use super::constants::*; +use super::rvnic_env::{QueueID, RVNIC_PACKET_RECYCLE_TX, UMEM}; +use super::DriverMetaData; +use crate::{ + common::*, + env::rvnic::utils::{set_cpu_affinity, Epoll}, + timer::Timer, + Packet, +}; + +#[derive(Debug)] +pub struct RvnicPacket { + desc: RattanDesc, + timestamp: Instant, + // This tells when we drop this packet in Rattan, to which FILL Ring + // should we return the buffer! + received_from: QueueID, + // If this is false, we need to recycle the token + sent: bool, + + header: Option>, +} + +impl Drop for RvnicPacket { + fn drop(&mut self) { + if !self.sent { + if let Some(tx) = RVNIC_PACKET_RECYCLE_TX.get() { + tx.send((self.received_from, self.desc.token)).ok(); + tracing::trace!( + target = "RVNIC", + "dropped packet on {:?}, recycling token {}", + self.received_from, + self.desc.token + ) + } + } + } +} + +fn try_get_slice_from_umem(desc: &RattanDesc) -> Option<&[u8]> { + let umem = UMEM.get()?; + let chunk_index = umem.offset_to_index(desc.addr)?; + let data_ptr = umem.data_ptr(chunk_index)?; + let data_len = desc.len as usize; + unsafe { Some(std::slice::from_raw_parts(data_ptr, data_len)) } +} + +fn try_copy_to_umem(desc: &RattanDesc, header: &[u8]) -> Option<()> { + let umem = UMEM.get()?; + let chunk_index = umem.offset_to_index(desc.addr)?; + let dest_ptr = umem.data_ptr(chunk_index)?; + let copy_len = std::cmp::min(desc.len as usize, header.len()); + unsafe { + std::ptr::copy_nonoverlapping(header.as_ptr(), dest_ptr, copy_len); + } + Some(()) +} + +impl Packet for RvnicPacket { + type PacketGenerator = (); + fn empty(_maximum: usize, _generator: &Self::PacketGenerator) -> Self { + unimplemented!("Rvnic packet can not be built directly") + } + fn from_raw_buffer(_buf: &[u8]) -> Self { + unimplemented!("Rvnic packet can not be built directly") + } + + fn as_raw_buffer(&mut self) -> &mut [u8] { + // Copy data from UMEM to header buffer. It is the rvnic driver's responsibility + // to copy back any modification to the skb in kernel from the UMEM, and we need + // to copy back from `self.header` to the UMEM when the packet leaves Rattan + // and was sent on the TX path, which is done in the function `try_copy_to_umem`. + let header = self + .header + .get_or_insert_with(|| try_get_slice_from_umem(&self.desc).unwrap_or(&[]).to_vec()); + header.as_mut_slice() + } + + fn length(&self) -> usize { + self.desc.get_skb_len() as usize + } + fn l2_length(&self) -> usize { + self.desc.get_skb_len() as usize + } + fn l3_length(&self) -> usize { + self.desc.get_skb_len().saturating_sub(14) as usize + } + fn as_slice(&self) -> &[u8] { + try_get_slice_from_umem(&self.desc).unwrap_or(&[]) + } + + fn ether_hdr(&self) -> Option { + None + } + fn ip_hdr(&self) -> Option { + None + } + fn get_timestamp(&self) -> Instant { + self.timestamp + } + fn set_timestamp(&mut self, timestamp: Instant) { + self.delay_until(timestamp); + } + fn delay_by(&mut self, delay: Duration) { + self.timestamp += delay; + } + fn delay_until(&mut self, timestamp: Instant) { + self.timestamp = timestamp; + } +} + +pub struct RvnicSender { + sender: mpsc::Sender, +} + +impl RvnicSender { + pub fn new(sender: mpsc::Sender) -> RvnicSender { + Self { sender } + } +} + +impl InterfaceSender for RvnicSender { + fn send(&self, packet: RvnicPacket) -> std::io::Result<()> { + // TODO(minhuw): handle errors more carefully here + let _ = self.sender.try_send(packet); + Ok(()) + } + + fn send_bulk(&self, packets: Iter) -> std::io::Result + where + T: Into, + Iter: IntoIterator, + Iter::IntoIter: ExactSizeIterator, + { + let packets = packets.into_iter().map(|packet| packet.into()); + + let len = packets.len(); + + //TODO(minhuw): currently we return error even if part of packets are sent + // maybe we should distinguish between partial success and total failure + for packet in packets { + self.sender + .blocking_send(packet) + .map_err(|_| std::io::Error::other("send error"))?; + } + + Ok(len) + } +} + +pub struct RvnicReceiver { + id: QueueID, + rx_ring: RxRing, + meta_data: DriverMetaData, +} + +impl RvnicReceiver { + fn new(id: QueueID, rx_ring: RxRing, meta_data: DriverMetaData) -> Self { + Self { + id, + rx_ring, + meta_data, + } + } +} + +impl InterfaceReceiver for RvnicReceiver { + // Rvnic has native support for batch receiving, so we + // are using `receive_bulk` to implement `receive`. + fn receive(&mut self) -> std::io::Result> { + unimplemented!("Separate thread is used for receiving from rvnic") + } + + fn receive_bulk(&mut self) -> std::io::Result> { + unimplemented!("Separate thread is used for receiving from rvnic") + } +} + +pub struct RvnicDriver { + receiver: RvnicReceiver, + sender: Arc, + queue_id: QueueID, + #[allow(unused)] // To be done: error handling + forwarder: tokio::task::JoinHandle<()>, + // forwarder: std::thread::JoinHandle<()>, +} + +/// Two things are done here: +/// 1. Mark the `sent` as true, which would be checked during the `drop()` of the `RvnicPacket`, +/// so that the packet would be ignored in the recycling token procedure. +/// 2. Copy back the `self.header` back to the umem. +fn packet_send_prepare(packet: impl Into) -> (RattanDesc, (QueueID, u64)) { + let mut packet = packet.into(); + // Avoid recycling tokens! + packet.sent = true; + let token = packet.desc.token; + let desc = packet.desc; + + // Make the constant compare first, make it easier for the compiler's optimizier. + // Warning: As no cells we currently have modifies the header, this function is never tested! + if RATTAN_HEADER_SIZE != 0 { + if let Some(header) = packet.header.take() { + // XXX: Ignore error handling here + try_copy_to_umem(&desc, header.as_slice()); + } + } + + (desc, (packet.received_from, token)) +} + +impl RvnicDriver { + #[instrument(name = "Rvnic send", skip_all)] + fn send( + tx: &mut TxRing, + packets: Iter, + meta: &DriverMetaData, + ) -> std::io::Result + where + T: Into, + Iter: IntoIterator, + Iter::IntoIter: ExactSizeIterator, + { + let (packets, loss_report): (Vec<_>, Vec<_>) = + packets.into_iter().map(packet_send_prepare).unzip(); + + let sent = tx.produce(&packets); + tracing::debug!(target: "rvnic", "[{}]Send batch {}/{}", meta.device_fd, packets.len(), sent); + meta.kick_self(); + if sent < packets.len() { + for to_report in loss_report.into_iter().skip(sent) { + if let Some(tx) = RVNIC_PACKET_RECYCLE_TX.get() { + tx.send(to_report).ok(); + } + } + tracing::warn!( + target = "RVNIC", + "{} packets dropped on tx ring unexpectedly", + packets.len() - sent + ); + } + Ok(sent) + } + + async fn buffered_send( + mut receiver: mpsc::Receiver, + mut tx_ring: TxRing, + meta: DriverMetaData, + ) { + let mut timer = Timer::new().expect("Failed to create timer for batch sending"); + let mut packets = Vec::with_capacity(BATCH_SIZE); + + // TODO(minhuw): it should not leave forever. But let is be now. + // we should stop the buffered send task when the cell exists. + loop { + let target = (tx_ring.available() as usize).min(BATCH_SIZE); + let ddl = Instant::now() + Duration::from_micros(SEND_BATCH_MAX_TIME_US); + while packets.len() < target { + let to_wait = ddl.duration_since(Instant::now()); + if to_wait < Duration::from_micros(SEND_BATCH_MIN_SLEEP_US) { + break; + } + tokio::select! { + packet = receiver.recv() => { + if let Some(packet) = packet { + packets.push(packet); + // break; + } + }, + _ = timer.sleep(to_wait) => {break;} + } + } + if packets.is_empty() { + tokio::task::yield_now().await; + continue; + } + let mut send_packets = Vec::with_capacity(BATCH_SIZE); + std::mem::swap(&mut packets, &mut send_packets); + let _ = Self::send(&mut tx_ring, send_packets, &meta); + } + } +} + +impl RvnicDriver { + pub(super) fn new( + queue_id: QueueID, + rx_ring: RxRing, + tx_ring: TxRing, + runtime: &Handle, + meta: DriverMetaData, + ) -> Self { + let (tx, rx) = mpsc::channel(SEND_BUFFER_SIZE); + let sender = Arc::new(RvnicSender { sender: tx }); + let receiver = RvnicReceiver::new(queue_id, rx_ring, meta.clone()); + tracing::info!(target: "RVNIC", "Built driver for {:?}", queue_id); + + Self { + receiver, + sender, + queue_id, + forwarder: runtime.spawn(RvnicDriver::buffered_send(rx, tx_ring, meta)), + // forwarder: std::thread::spawn(|| RvnicDriver::sending_thread(rx, tx_ring, meta)), + } + } +} + +impl InterfaceDriver for RvnicDriver { + type Packet = RvnicPacket; + type Sender = RvnicSender; + type Receiver = RvnicReceiver; + + fn raw_fd(&self) -> i32 { + self.queue_id.device_fd + } + + fn sender(&self) -> Arc { + self.sender.clone() + } + + fn receiver(&mut self) -> &mut Self::Receiver { + &mut self.receiver + } + + fn into_receiver(self) -> Self::Receiver { + self.receiver + } + + fn use_blocking_recv() -> bool { + true + } + #[instrument(name = "Rvnic recv", skip_all)] + fn recv_thread(mut receiver: Self::Receiver, tx: tokio::sync::mpsc::Sender) { + // Set core affinity + let cpu = receiver.meta_data.receive_thread_core; + tracing::info!("Trying to set CPU affinity to {}", cpu); + if set_cpu_affinity(cpu).is_err() { + tracing::warn!("Failed to set CPU affinity"); + } + + let self_raw_fd = receiver.meta_data.self_device.as_raw_fd(); + + let epoll_fd = Epoll::new() + .inspect_err(|e| { + tracing::error!( + ?e, + "Failed to get Rvnic receiving epoll fd for fd {}", + self_raw_fd + ) + }) + .expect("Failed to get Rvnic receiving epoll fd"); + + epoll_fd + .add(self_raw_fd, self_raw_fd as u64) + .inspect_err(|e| tracing::error!(?e, "Failed to add epoll fd")) + .expect("Failed to add epoll fd"); + + let mut events = vec![libc::epoll_event { events: 0, u64: 0 }; 32]; + + for loop_cnt in 0..u64::MAX { + let Ok(nfds) = epoll_fd + .wait(&mut events, RECV_EPOLL_MAX_WAIT_MS) + .inspect_err(|e| tracing::error!(?e, "Failed to wait on epoll")) + else { + break; + }; + + // Make it simple to avoid subtle performance drawback. + #[allow(clippy::needless_range_loop)] + for i in 0..nfds { + let ev = events[i]; + // Always check event flags! + if ev.events & (libc::EPOLLERR as u32) != 0 { + let error_fd = ev.u64; + tracing::error!(?error_fd, "Failed on raw fd "); + continue; + } + // We only added one fd. So just drain all the events on the rx ring! + + let mut batch_count = 0; + let mut to_receive = [RattanDesc::default(); BATCH_SIZE]; + loop { + let filled = receiver.rx_ring.consume(&mut to_receive); + if filled == 0 { + break; + } + batch_count += filled; + + let timestamp = Instant::now(); + let result = to_receive.into_iter().take(filled).map(|desc| RvnicPacket { + desc, + header: None, + timestamp, + received_from: receiver.id, + sent: false, + }); + for packet in result { + let _ = tx.blocking_send(packet); + } + } + tracing::debug!(target: "rvnic", "[{}]Recv batch {} @ {}", self_raw_fd, batch_count, loop_cnt); + } + } + + tracing::warn!("Rvnic recv thread for {} exited!", self_raw_fd); + } +} diff --git a/rattan-env/src/env/rvnic/mod.rs b/rattan-env/src/env/rvnic/mod.rs new file mode 100644 index 00000000..e5f4a949 --- /dev/null +++ b/rattan-env/src/env/rvnic/mod.rs @@ -0,0 +1,63 @@ +mod af_rvnic; +mod rvnic_env; +mod utils; + +use std::sync::Arc; + +pub use af_rvnic::{RvnicDriver, RvnicPacket, RvnicReceiver, RvnicSender}; +use rvnic::RvnicDevice; +pub use rvnic_env::{RvnicEnv, RvnicEnvConfig, RvnicEnvMode}; + +pub(crate) mod constants { + pub const RVNIC_DEVICES: u32 = 2; + pub const BATCH_SIZE: usize = 32; + // Size of internal driver in RvnicDriver + pub const SEND_BUFFER_SIZE: usize = 32768; + + // Waiting at most `SEND_BATCH_MAX_TIME_US` microseconds until trying to send the current batch which does not meet + // the desired batch size. + pub const SEND_BATCH_MAX_TIME_US: u64 = 1000; + // Avoid sleeping for an ultra-short period in a coroutine. + pub const SEND_BATCH_MIN_SLEEP_US: u64 = 300; + // Used in epoll call within receiving thread + pub const RECV_EPOLL_MAX_WAIT_MS: i32 = 1; + + pub const fn default_left_recv_core() -> u32 { + 0 + } + pub const fn default_right_recv_core() -> u32 { + 2 + } + pub const fn default_left_napi_core() -> i32 { + 0 + } + pub const fn default_right_napi_core() -> i32 { + 2 + } + + pub const RATTAN_HEADROOM: u32 = 64; + pub const fn default_num_queues() -> usize { + 1 + } + + // Re import from rvnic + pub use rvnic::{RATTAN_DEFAULT_CHUNK_SIZE, RATTAN_HEADER_SIZE, RATTAN_RING_SIZE}; +} + +#[derive(Clone)] +struct DriverMetaData { + pub self_device: Arc, + #[allow(unused)] + pub other_device: Arc, + pub queue_id: u32, + pub device_fd: i32, + + pub receive_thread_core: u32, +} + +impl DriverMetaData { + // The sender shall call this after producing packets into the TX Ring + pub fn kick_self(&self) { + self.self_device.kick_rx_queue(self.queue_id).ok(); + } +} diff --git a/rattan-env/src/env/rvnic/rvnic_env.rs b/rattan-env/src/env/rvnic/rvnic_env.rs new file mode 100644 index 00000000..5b75b04f --- /dev/null +++ b/rattan-env/src/env/rvnic/rvnic_env.rs @@ -0,0 +1,666 @@ +use std::collections::{HashMap, HashSet}; +use std::net::{IpAddr, Ipv4Addr}; +#[cfg(feature = "drop-stat")] +use std::ops::AddAssign; +use std::sync::Arc; + +use rand::distr::Alphanumeric; +use rand::{rng, RngExt}; +use rvnic::{DropRing, RvnicDevice, RxRing, TxRing, Umem, UmemBuilder}; +#[cfg(feature = "serde")] +use serde::{Deserialize, Serialize}; +use tokio::sync::{ + mpsc::{self}, + OnceCell, +}; +use tokio::time::{timeout, Duration, Instant}; +use tracing::{info, instrument, warn}; + +use super::constants::*; +use super::utils::*; +use super::DriverMetaData; +use crate::env::rvnic::RvnicDriver; +use crate::env::{get_addresses_in_use, IpAddrLock}; +use crate::error::MetalError; +use crate::{Error, InterfaceBuildArtifact, NetNs, RattanEnv, RattanEnvConfig, StdNetEnvMode}; + +lazy_static::lazy_static! { + static ref RVNIC_ENV_LOCK: Arc> = Arc::new(parking_lot::Mutex::new(())); +} + +pub type RvnicEnvMode = StdNetEnvMode; + +// Packets that are meant to be dropped by rattan are sent to this channel. +pub static RVNIC_PACKET_RECYCLE_TX: OnceCell> = + OnceCell::const_new(); + +pub static UMEM: OnceCell> = OnceCell::const_new(); + +struct RvnicPacketRecycler { + drop_rings: HashMap)>, + rx: mpsc::UnboundedReceiver<(QueueID, u64)>, + #[cfg(feature = "drop-stat")] + drop_stat: HashMap, +} + +#[cfg(feature = "drop-stat")] +impl Drop for RvnicPacketRecycler { + fn drop(&mut self) { + let mut drop_total = 0; + for (queue_id, drop_stat) in self.drop_stat.iter() { + eprintln!("[rvnic] Recycled {} tokens on {}", drop_stat, queue_id); + drop_total += drop_stat; + } + eprintln!("[rvnic] Recycled {} tokens in total", drop_total); + } +} + +// If token is Some, try to batch the tokens until there are BATCH_SIZE tokens +// If token is None, eagerly recycle any token +// Return true if the drop is deffered. +fn recycle_token_on_queue( + token: Option, + queue_id: QueueID, + drop_ring: &mut DropRing, + buffer: &mut Vec, +) -> bool { + if let Some(token) = token { + buffer.push(token); + if buffer.len() < BATCH_SIZE { + return true; + } + } + let original_len = buffer.len(); + + // We do not care the order that the tokens are recycled, so we push from the rear + // of the buffer, and also send a send-able part from back + let available = drop_ring.available() as usize; + let to_send = available.min(BATCH_SIZE).min(original_len); + if to_send == 0 { + return false; + } + let new_length = original_len - to_send; + let (_remain, to_send_buffer) = buffer.as_slice().split_at(new_length); + + let actual_sent = drop_ring.produce(to_send_buffer); + if actual_sent == to_send { + // This is the common case without any memory allocation + buffer.truncate(new_length); + } else { + // It is new_length..(new_length + actual_sent) that is sent, + // we have to remove them from the buffer + let not_sent_tail = buffer.split_off(new_length + actual_sent); + buffer.truncate(new_length); + buffer.extend_from_slice(¬_sent_tail); + } + + tracing::debug!( + target = "rvnic", + "On {}, tried to drop {} tokens, dropped {}, remain {}", + queue_id, + to_send, + actual_sent, + buffer.len() + ); + false +} + +struct ReceiveTimeout {} + +async fn recv_with_timeout( + rx: &mut mpsc::UnboundedReceiver, + dur: Duration, +) -> Result, ReceiveTimeout> { + match timeout(dur, rx.recv()).await { + Ok(msg) => Ok(msg), + Err(_) => Err(ReceiveTimeout {}), + } +} + +impl RvnicPacketRecycler { + async fn run(mut self) { + info!(target = "rvnic", "Packet Recycler start"); + + let mut defered_schedule: HashMap = HashMap::new(); + + loop { + // There won't be many queues, so a simlpe O(n) iteration would work. + let (waiting_queue, timeout) = if let Some((&queue_id, &instant)) = + defered_schedule.iter().min_by_key(|&(_queue, time)| time) + { + (Some(queue_id), instant.duration_since(Instant::now())) + } else { + (None, Duration::from_secs(10)) + }; + + let (queue_id, token) = match recv_with_timeout(&mut self.rx, timeout).await { + Ok(Some((queue_id, token))) => { + #[cfg(feature = "drop-stat")] + self.drop_stat.entry(queue_id).or_default().add_assign(1); + (Some(queue_id), Some(token)) + } + Ok(None) => { + // The other side of the channel has been closed. + break; + } + Err(_) => (waiting_queue, None), + }; + + // Nothing we can do if we can not specific a queue to recycle on + let Some(queue_id) = queue_id else { + continue; + }; + + let mut delayed = false; + if let Some((drop_ring, buffer)) = self.drop_rings.get_mut(&queue_id) { + delayed = recycle_token_on_queue(token, queue_id, drop_ring, buffer); + } + + if delayed + && defered_schedule + .get(&queue_id) + .is_some_and(|t| t.elapsed().is_zero()) + { + continue; + } + + defered_schedule.insert(queue_id, Instant::now() + Duration::from_secs(5)); + } + } +} +#[derive(PartialEq, Eq, Clone, Copy, Debug, Hash)] +pub struct QueueID { + pub device_fd: i32, + pub queue_id: u32, +} + +impl std::fmt::Display for QueueID { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str("d")?; + self.device_fd.fmt(f)?; + f.write_str("q")?; + self.queue_id.fmt(f) + } +} + +#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] +#[derive(Debug, Clone)] +pub struct RvnicEnvConfig { + #[cfg_attr(feature = "serde", serde(default))] + pub mode: RvnicEnvMode, + + #[cfg_attr(feature = "serde", serde(default = "default_num_queues"))] + pub num_queues: usize, + + // These settings are not supported for RvnicEnv now + // #[cfg_attr(feature = "serde", serde(default = "default_veth_count"))] + // pub left_veth_count: usize, + // #[cfg_attr(feature = "serde", serde(default = "default_veth_count"))] + // pub right_veth_count: usize, + + // #[cfg_attr(feature = "serde", serde(default))] + // pub left_external: Option, + + // #[cfg_attr(feature = "serde", serde(default))] + // pub right_external: Option, + + // This should be consistent with the kernel module! + + // TODO(minhuw): pretty sure these two configs should not be here + // but let it be for now + #[cfg_attr(feature = "serde", serde(default))] + pub server_cores: Vec, + + #[cfg_attr(feature = "serde", serde(default))] + pub client_cores: Vec, + + /// Rvnic uses a separate recv core for each end. This can be overwritten in the config. + #[cfg_attr(feature = "serde", serde(default = "default_left_recv_core"))] + pub left_recv_core: u32, + /// Rvnic uses a separate recv core for each end. This can be overwritten in the config. + #[cfg_attr(feature = "serde", serde(default = "default_right_recv_core"))] + pub right_recv_core: u32, + /// Core to do NAPI polling on the left rvnic. It can be -1 to use the core that kicked the rx. This can be overritten in the config. + #[cfg_attr(feature = "serde", serde(default = "default_left_napi_core"))] + pub left_napi_core: i32, + /// Core to do NAPI polling on the right rvnic. It can be -1 to use the core that kicked the rx. + #[cfg_attr(feature = "serde", serde(default = "default_right_napi_core"))] + pub right_napi_core: i32, +} + +impl Default for RvnicEnvConfig { + fn default() -> Self { + Self { + mode: RvnicEnvMode::default(), + num_queues: default_num_queues(), + server_cores: Vec::new(), + client_cores: Vec::new(), + left_recv_core: default_left_recv_core(), + right_recv_core: default_right_recv_core(), + left_napi_core: default_left_napi_core(), + right_napi_core: default_right_napi_core(), + } + } +} + +impl RattanEnvConfig for RvnicEnvConfig { + type BuildError = crate::error::Error; + type Driver = RvnicDriver; + type BuildOutput = RvnicEnv; + + fn build(&self) -> Result { + let RvnicEnvBuildArtifact { + env, + umem, + packet_recycler_tx, + } = get_rvnic_env(self)?; + + // These should not happen, as we only try to set these global OnceCells after we have successfully + // created the RvnicEnv. + if UMEM.set(umem).is_err() { + unreachable!("Failed to set UMEM"); + } + if RVNIC_PACKET_RECYCLE_TX.set(packet_recycler_tx).is_err() { + unreachable!("Failed to set RVNIC_PACKET_RECYCLE_TX"); + } + + Ok(env) + } + fn default_with_mode(mode: >::Mode) -> Self { + RvnicEnvConfig { + mode, + num_queues: default_num_queues(), + server_cores: vec![1], + client_cores: vec![3], + left_recv_core: default_left_recv_core(), + right_recv_core: default_right_recv_core(), + left_napi_core: default_left_napi_core(), + right_napi_core: default_right_napi_core(), + } + } + + fn default_compatible() -> Self { + Self::default_with_mode(RvnicEnvMode::Compatible) + } + + fn default_isolated() -> Self { + Self::default_with_mode(RvnicEnvMode::Isolated) + } +} + +pub struct RvnicEnv { + pub mode: RvnicEnvMode, + pub left_ns: Arc, + pub right_ns: Arc, + pub left_ip: IpAddr, + pub right_ip: IpAddr, + pub rattan_id: String, + packet_recycler: Option, + left_data: Option>, + right_data: Option>, + left_device: Arc, + right_device: Arc, + left_recv_core: u32, + right_recv_core: u32, +} + +impl RattanEnv for RvnicEnv { + type Mode = RvnicEnvMode; + fn get_rattan_id(&self) -> &str { + &self.rattan_id + } + fn get_rattan_ns(&self) -> Option> { + None + } + fn get_left_ns(&self) -> Arc { + self.left_ns.clone() + } + fn get_right_ns(&self) -> Arc { + self.right_ns.clone() + } + fn get_mode(&self) -> Self::Mode { + self.mode + } + fn left_ip(&self, i: usize) -> IpAddr { + assert_eq!(i, 1, "Multiple Rvnic devices are not supported"); + self.left_ip + } + fn right_ip(&self, i: usize) -> IpAddr { + assert_eq!(i, 1, "Multiple Rvnic devices are not supported"); + self.right_ip + } + + // For legacy reasons, 0 is reserved for `external` connection, which we do not have + fn left_ip_list(&self) -> Vec<(usize, IpAddr)> { + vec![(1, self.left_ip)] + } + fn right_ip_list(&self) -> Vec<(usize, IpAddr)> { + vec![(1, self.right_ip)] + } + + // Multipath is not support for rvnic for now + fn left_max_id(&self) -> usize { + 1 + } + fn right_max_id(&self) -> usize { + 1 + } + + fn build_interfaces( + &mut self, + runtime: &tokio::runtime::Handle, + ) -> Result>, MetalError> { + if let Some(packet_recycler) = self.packet_recycler.take() { + runtime.spawn(packet_recycler.run()); + } + + let (Some(left_data), Some(right_data)) = (self.left_data.take(), self.right_data.take()) + else { + unreachable!("This function should be called exactly once"); + }; + + let to_driver = |data_path: RvnicQueueDataPath, + receive_thread_core, + self_device: Arc, + other_device: Arc| { + let device_fd = self_device.fd(); + let meta = DriverMetaData { + self_device, + other_device, + device_fd, + queue_id: data_path.id.queue_id, + receive_thread_core, + }; + RvnicDriver::new(data_path.id, data_path.rx, data_path.tx, runtime, meta) + }; + + let left = left_data.into_iter().map(|data| { + to_driver( + data, + self.left_recv_core, + self.left_device.clone(), + self.right_device.clone(), + ) + }); + let right = right_data.into_iter().map(|data| { + to_driver( + data, + self.right_recv_core, + self.right_device.clone(), + self.left_device.clone(), + ) + }); + + let result = vec![ + InterfaceBuildArtifact { + ns_id: 1, + veth_id: 1, + name: "left".to_string(), + drivers: left.collect(), + }, + InterfaceBuildArtifact { + ns_id: 2, + veth_id: 1, + name: "right".to_string(), + drivers: right.collect(), + }, + ]; + + Ok(result) + } +} + +struct RvnicQueueDataPath { + id: QueueID, + rx: RxRing, + tx: TxRing, +} + +struct RvnicBuildArtifact { + left_device: RvnicDevice, + right_device: RvnicDevice, + left_data: Vec, + right_data: Vec, + umem: Arc, + packet_recycler: RvnicPacketRecycler, + packet_recycler_tx: mpsc::UnboundedSender<(QueueID, u64)>, +} + +fn build_rvnic_pair(queue_nums: u32) -> rvnic::Result { + let mut dev0 = RvnicDevice::open()?; + let mut dev1 = RvnicDevice::open()?; + + let chunks_per_device = queue_nums * RATTAN_RING_SIZE; + + let umem = UmemBuilder::new() + .chunk_size(RATTAN_DEFAULT_CHUNK_SIZE) + .headroom(RATTAN_HEADROOM) + // `RATTAN_RING_SIZE` chunks per queue, `RVNIC_DEVICES` devices, `queue_nums` queues per device + .num_chunks(chunks_per_device * RVNIC_DEVICES) + .build()?; + + let num_chunks = umem.num_chunks(); + let chunk_size = umem.chunk_size(); + + tracing::info!( + "umem size: {} chunks x {}Bytes", + umem.num_chunks(), + umem.chunk_size() + ); + + let umem = dev0.register_umem(umem)?; + dev1.share_umem(&dev0)?; + tracing::info!(" UMEM registered and shared"); + + let dev0_configs = build_queue_configs(queue_nums)?; + let dev1_configs = build_queue_configs(queue_nums)?; + + let mut dev0_queues = dev0.register_queues(dev0_configs)?; + let mut dev1_queues = dev1.register_queues(dev1_configs)?; + tracing::info!(" Registered {} queues on each device", queue_nums); + + let dev0_chunks = chunks_per_device; + let dev1_chunks = num_chunks - dev0_chunks; + + let dev0_fill = prefill_device_queues(&mut dev0_queues, 0, dev0_chunks, chunk_size); + let dev1_fill = prefill_device_queues(&mut dev1_queues, dev0_chunks, dev1_chunks, chunk_size); + + tracing::info!( + "Pre-filled FILL rings: {} chunks to dev0, {} chunks to dev1 ", + dev0_fill, + dev1_fill, + ); + + let mut drop_rings = vec![]; + let mut rx_tx_rings = vec![]; + + let raw_fds = [dev0.fd(), dev1.fd()]; + + for (device_id, queues) in raw_fds.into_iter().zip([dev0_queues, dev1_queues]) { + for queue_ring in queues { + let queue_id = QueueID { + device_fd: device_id, + queue_id: queue_ring.queue_id, + }; + drop_rings.push((queue_id, queue_ring.drop)); + rx_tx_rings.push((queue_id, queue_ring.rx, queue_ring.tx)); + // We do not need to use the other 2 rings, and they are dropped here + } + } + + let (tx, rx) = mpsc::unbounded_channel::<(QueueID, u64)>(); + + let packet_recycler = RvnicPacketRecycler { + rx, + drop_rings: HashMap::from_iter( + drop_rings + .into_iter() + .map(|(id, ring)| (id, (ring, Vec::with_capacity(BATCH_SIZE)))), + ), + #[cfg(feature = "drop-stat")] + drop_stat: HashMap::new(), + }; + + let (left, right): (Vec<_>, Vec<_>) = rx_tx_rings + .into_iter() + .partition(|(id, _, _)| id.device_fd == dev0.fd()); + + let left_data = left + .into_iter() + .map(|(id, rx, tx)| RvnicQueueDataPath { id, rx, tx }) + .collect(); + + let right_data = right + .into_iter() + .map(|(id, rx, tx)| RvnicQueueDataPath { id, rx, tx }) + .collect(); + + Ok(RvnicBuildArtifact { + left_device: dev0, + right_device: dev1, + left_data, + right_data, + umem, + packet_recycler, + packet_recycler_tx: tx, + }) +} + +// Get 10.x.y.1 (right) and 10.x.y.2(left), that 10.x.y.1 is not occupied. +pub fn get_addrs(mode: RvnicEnvMode) -> Result<(IpAddr, (IpAddr, Option)), Error> { + if !matches!(mode, RvnicEnvMode::Compatible) { + let left_addr = IpAddr::from(Ipv4Addr::new(10, 99, 0, 1)); + let right_addr = IpAddr::from(Ipv4Addr::new(10, 99, 0, 2)); + return Ok((left_addr, (right_addr, None))); + } + + let address_in_use: HashSet = HashSet::from_iter(get_addresses_in_use()?); + for x in 0..255u16 { + let x = (x + 99) % 255; + let x = x as u8; + for y in 0..255 { + let left_addr = IpAddr::from(Ipv4Addr::new(10, x, y, 1)); + let right_addr = IpAddr::from(Ipv4Addr::new(10, x, y, 2)); + if address_in_use.contains(&right_addr) { + continue; + } + if let Some(lock) = IpAddrLock::new(right_addr)? { + return Ok((left_addr, (right_addr, Some(lock)))); + } + } + } + Err(Error::IoError(std::io::Error::other( + "Failed to get available ip addr for rvnic in host netns", + ))) +} + +struct RvnicEnvBuildArtifact { + env: RvnicEnv, + umem: Arc, + packet_recycler_tx: mpsc::UnboundedSender<(QueueID, u64)>, +} + +#[instrument(skip_all, level = "debug", name = "RvnicEnv")] +fn get_rvnic_env(config: &RvnicEnvConfig) -> Result { + // Create network namespaces + info!(?config); + let _guard = RVNIC_ENV_LOCK.lock(); + let rand_string: String = rng() + .sample_iter(&Alphanumeric) + .take(6) + .map(char::from) + .collect(); + + let left_netns_name = format!("ns-left-{rand_string}"); + let left_netns = NetNs::new(&left_netns_name)?; + info!(?left_netns, "Left netns {left_netns_name} created"); + + let (right_netns, right_netns_name) = match config.mode { + RvnicEnvMode::Compatible => (NetNs::current()?, "".to_string()), + RvnicEnvMode::Isolated => { + let right_netns_name = format!("ns-right-{rand_string}"); + let right_netns = NetNs::new(&right_netns_name)?; + info!( + ?right_netns, + "Right netns {} created", + right_netns_name.as_str() + ); + (right_netns, right_netns_name) + } + RvnicEnvMode::Container => { + return Err(Error::ConfigError( + "Rvnic does not support container mode".to_string(), + )) + } + }; + + // Ignore the `external` veth pairs for now! + // + let RvnicBuildArtifact { + mut left_device, + mut right_device, + left_data, + right_data, + umem, + packet_recycler, + packet_recycler_tx, + } = build_rvnic_pair(config.num_queues as u32)?; + + // Start the devices + left_device + .start(config.left_napi_core) + .map_err(MetalError::RvnicError)?; + right_device + .start(config.right_napi_core) + .map_err(MetalError::RvnicError)?; + tracing::info!("RVNIC devices started"); + + // Move the devices into the netns + move_to_netns(&left_device.device_name()?, &left_netns_name)?; + move_to_netns(&right_device.device_name()?, &right_netns_name)?; + + let (left_ip, (right_ip, right_ip_lock)) = get_addrs(config.mode)?; + + info!( + "trying to set up rvnic device interface {} in {} with ip {}", + left_device.device_name()?, + left_netns_name, + left_ip + ); + configure_interface_in_netns(&left_netns_name, &left_device.device_name()?, left_ip)?; + + if right_netns_name.is_empty() { + info!( + "trying to set up rvnic device interface {} with ip {}", + right_device.device_name()?, + right_ip + ); + configure_interface_in_current_netns(&right_device.device_name()?, right_ip)?; + } else { + configure_interface_in_netns(&right_netns_name, &right_device.device_name()?, right_ip)?; + } + + let env = RvnicEnv { + left_device: Arc::new(left_device), + right_device: Arc::new(right_device), + mode: config.mode, + left_ns: left_netns, + right_ns: right_netns, + left_ip, + right_ip, + rattan_id: rand_string, + // Will be consumed when `build_interface` is called for the first time. + packet_recycler: Some(packet_recycler), + left_data: Some(left_data), + right_data: Some(right_data), + left_recv_core: config.left_recv_core, + right_recv_core: config.right_recv_core, + }; + + drop(right_ip_lock); + Ok(RvnicEnvBuildArtifact { + env, + umem, + packet_recycler_tx, + }) +} diff --git a/rattan-env/src/env/rvnic/utils.rs b/rattan-env/src/env/rvnic/utils.rs new file mode 100644 index 00000000..0fc9aa41 --- /dev/null +++ b/rattan-env/src/env/rvnic/utils.rs @@ -0,0 +1,361 @@ +use rtnetlink::{LinkMessageBuilder, LinkUnspec}; +use rvnic::{QueueConfig, QueueRings, Rings}; +use std::io; +use std::os::fd::{AsRawFd, RawFd}; +use tracing::info; + +use super::constants::BATCH_SIZE; + +/// A thin wrapper around Linux epoll. +/// +/// Design goals: +/// - Minimal abstraction over libc epoll API +/// - Keep full control over flags (EPOLLET, etc.) +/// - Localize unsafe usage +/// - Use u64 token to identify fds (like mio::Token) +/// +/// This is suitable for high-performance, single-threaded +/// event loops (e.g., AF_XDP, custom network stacks). +pub struct Epoll { + epfd: RawFd, +} + +impl Epoll { + /// Create a new epoll instance. + /// + /// Internally calls epoll_create1. + pub fn new() -> io::Result { + let epfd = unsafe { libc::epoll_create1(0) }; + if epfd < 0 { + return Err(io::Error::last_os_error()); + } + Ok(Self { epfd }) + } + + /// Add a file descriptor to the epoll set. + /// + /// # Arguments + /// + /// - `fd`: file descriptor to monitor + /// - `token`: user-defined identifier (stored in epoll_event.u64) + /// + /// # Behavior + /// + /// - Registers for READABLE events (EPOLLIN) + /// - Uses EDGE-TRIGGERED mode (EPOLLET) + /// + /// # Important (Edge-triggered semantics!) + /// + /// Once you receive a readiness event: + /// - You MUST drain the fd completely (read until EAGAIN) + /// - Otherwise, you may never receive another event + /// + pub fn add(&self, fd: RawFd, token: u64) -> io::Result<()> { + let mut ev = libc::epoll_event { + events: (libc::EPOLLIN | libc::EPOLLET) as u32, + u64: token, + }; + + let ret = unsafe { libc::epoll_ctl(self.epfd, libc::EPOLL_CTL_ADD, fd, &mut ev) }; + + if ret < 0 { + return Err(io::Error::last_os_error()); + } + Ok(()) + } + + /// Wait for events. + /// + /// # Arguments + /// + /// - `events`: preallocated buffer + /// - `timeout_ms`: timeout in milliseconds + /// - -1: block indefinitely + /// - 0: non-blocking poll + /// + /// # Returns + /// + /// Number of ready events + /// + /// # EINTR handling + /// + /// If interrupted by a signal (e.g., SIGWINCH), + /// this function will automatically retry. + /// + pub fn wait(&self, events: &mut [libc::epoll_event], timeout_ms: i32) -> io::Result { + loop { + let nfds = unsafe { + libc::epoll_wait( + self.epfd, + events.as_mut_ptr(), + events.len() as i32, + timeout_ms, + ) + }; + + if nfds < 0 { + let err = io::Error::last_os_error(); + + // Retry on EINTR (signal interruption) + if err.kind() == io::ErrorKind::Interrupted { + continue; + } + + return Err(err); + } + + return Ok(nfds as usize); + } + } +} + +impl Drop for Epoll { + /// Ensure epoll fd is closed when dropped. + fn drop(&mut self) { + unsafe { + libc::close(self.epfd); + } + } +} + +// Utility functions + +/// Move a network interface into a specific network namespace using native netlink +pub fn move_to_netns(iface: &str, netns: &str) -> std::io::Result<()> { + if netns.is_empty() { + return Ok(()); + } + + // Initialize a current-thread Tokio runtime to execute netlink commands + let rt = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build()?; + + rt.block_on(async { + // Open the target network namespace file to get its file descriptor + let ns_path = format!("/var/run/netns/{}", netns); + let fd = File::open(&ns_path).map_err(|e| { + std::io::Error::other(format!("Failed to open netns file {}: {}", ns_path, e)) + })?; + + let (connection, handle, _) = + rtnetlink::new_connection().map_err(|e| std::io::Error::other(e.to_string()))?; + tokio::spawn(connection); + + // Retrieve the interface index (ifindex) of the link in the current namespace + let mut links = handle.link().get().match_name(iface.to_string()).execute(); + let link = links + .try_next() + .await + .map_err(|e| std::io::Error::other(e.to_string()))? + .ok_or_else(|| { + std::io::Error::other(format!("Interface {} not found in current netns", iface)) + })?; + let ifindex = link.header.index; + + let msg = LinkMessageBuilder::::new() + .index(ifindex) + .setns_by_fd(fd.as_raw_fd()) + .build(); + + // Move the interface into the target network namespace using its file descriptor + handle.link().change(msg).execute().await.map_err(|e| { + std::io::Error::other(format!( + "Failed to move interface {} to netns {}: {}", + iface, netns, e + )) + })?; + + Ok::<(), std::io::Error>(()) + })?; + + // (Ensure your logging crate like `log` or `tracing` is imported in the file) + info!("Moved RVNIC {} into network namespace {}", iface, netns); + Ok(()) +} + +pub fn build_queue_configs(num_queues: u32) -> rvnic::Result> { + let mut queues = Vec::with_capacity(num_queues as usize); + for queue_id in 0..num_queues { + queues.push(QueueConfig { + queue_id, + rings: Rings::new()?, + }); + } + Ok(queues) +} + +pub fn prefill_device_queues( + queues: &mut [QueueRings], + start_chunk: u32, + num_chunks: u32, + chunk_size: u32, +) -> usize { + let mut total = 0usize; + let mut addrs = [0u64; BATCH_SIZE]; + let end_chunk = start_chunk + num_chunks; + let mut queue_index = (0..queues.len()).cycle(); + + for batch_chunk_start in (start_chunk..end_chunk).step_by(BATCH_SIZE) { + let batch_chunk_end = (batch_chunk_start + BATCH_SIZE as u32).min(end_chunk); + let batch_size = (batch_chunk_end - batch_chunk_start) as usize; + for (i, addr) in addrs.iter_mut().enumerate().take(batch_size) { + *addr = (batch_chunk_start as u64 + i as u64) * chunk_size as u64; + } + let queue = queue_index.next().unwrap(); + let produced = queues[queue].fill.produce(&addrs[..batch_size]); + total += produced; + } + + total +} + +use futures::stream::TryStreamExt; +use nix::sched::{setns, CloneFlags}; +use std::fs::File; +use std::net::IpAddr; + +/// Configure interface inside a specific network namespace using native netlink +pub fn configure_interface_in_netns( + netns: &str, + iface: &str, + ip_addr: IpAddr, +) -> std::io::Result<()> { + let netns = netns.to_string(); + let iface = iface.to_string(); + + let prefix_len = if ip_addr.is_ipv4() { 24 } else { 64 }; + + // Spawn a dedicated OS thread because setns changes the network namespace + // of the calling thread only, preventing namespace pollution of the main thread. + std::thread::spawn(move || { + let ns_path = format!("/var/run/netns/{}", netns); + let fd = File::open(&ns_path).map_err(|e| { + std::io::Error::other(format!("Failed to open netns file {}: {}", ns_path, e)) + })?; + + setns(&fd, CloneFlags::CLONE_NEWNET) + .map_err(|e| std::io::Error::other(format!("Failed to setns to {}: {}", netns, e)))?; + + // Initialize a current-thread Tokio runtime inside the target namespace + let rt = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build()?; + + rt.block_on(async { + let (connection, handle, _) = + rtnetlink::new_connection().map_err(|e| std::io::Error::other(e.to_string()))?; + // Drive the netlink socket connection in the background + tokio::spawn(connection); + + // Retrieve the interface index (ifindex) by its name + let mut links = handle.link().get().match_name(iface.clone()).execute(); + let link = links + .try_next() + .await + .map_err(|e| std::io::Error::other(e.to_string()))? + .ok_or_else(|| { + std::io::Error::other(format!( + "Interface {} not found in netns {}", + iface, netns + )) + })?; + let ifindex = link.header.index; + + // Assign the IP address to the interface + handle + .address() + .add(ifindex, ip_addr, prefix_len) + .execute() + .await + .map_err(|e| { + std::io::Error::other(format!("Failed to add IP via netlink: {}", e)) + })?; + + let msg = LinkMessageBuilder::::new() + .index(ifindex) + .up() + .build(); + + // Bring up the interface (IFF_UP) using the property handle + handle.link().change(msg).execute().await.map_err(|e| { + std::io::Error::other(format!("Failed to bring up {} via netlink: {}", iface, e)) + })?; + + // Bring up the loopback interface as well + let mut lo_links = handle.link().get().match_name("lo".to_string()).execute(); + if let Ok(Some(lo_link)) = lo_links.try_next().await { + let msg = LinkMessageBuilder::::new() + .index(lo_link.header.index) + .up() + .build(); + handle.link().change(msg).execute().await.map_err(|e| { + std::io::Error::other(format!("Failed to bring lo via netlink: {}", e)) + })?; + } + + Ok(()) + }) + }) + .join() + .map_err(|_| std::io::Error::other("Netns configuration thread panicked"))? +} + +/// Configure interface in the current network namespace using native netlink +pub fn configure_interface_in_current_netns(iface: &str, ip_addr: IpAddr) -> std::io::Result<()> { + let iface = iface.to_string(); + + let prefix_len = if ip_addr.is_ipv4() { 24 } else { 64 }; + + // Directly initialize and block on the runtime since nested runtimes are not a concern + let rt = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build()?; + + rt.block_on(async { + let (connection, handle, _) = + rtnetlink::new_connection().map_err(|e| std::io::Error::other(e.to_string()))?; + tokio::spawn(connection); + + let mut links = handle.link().get().match_name(iface.clone()).execute(); + let link = links + .try_next() + .await + .map_err(|e| std::io::Error::other(e.to_string()))? + .ok_or_else(|| std::io::Error::other(format!("Interface {} not found", iface)))?; + let ifindex = link.header.index; + + handle + .address() + .add(ifindex, ip_addr, prefix_len) + .execute() + .await + .map_err(|e| std::io::Error::other(format!("Failed to add IP: {}", e)))?; + + let msg = LinkMessageBuilder::::new() + .index(ifindex) + .up() + .build(); + + // Bring up the interface (IFF_UP) using the property handle + handle.link().change(msg).execute().await.map_err(|e| { + std::io::Error::other(format!("Failed to bring up {} via netlink: {}", iface, e)) + })?; + + Ok(()) + }) +} + +pub fn set_cpu_affinity(cpu: u32) -> std::io::Result<()> { + unsafe { + let mut cpuset: libc::cpu_set_t = std::mem::zeroed(); + libc::CPU_ZERO(&mut cpuset); + libc::CPU_SET(cpu as usize, &mut cpuset); + + let ret = libc::sched_setaffinity(0, std::mem::size_of::(), &cpuset); + if ret != 0 { + return Err(std::io::Error::last_os_error()); + } + } + Ok(()) +} diff --git a/rattan-env/src/env/standard/std_env.rs b/rattan-env/src/env/standard/std_env.rs index f06b3a0a..c4a07c24 100644 --- a/rattan-env/src/env/standard/std_env.rs +++ b/rattan-env/src/env/standard/std_env.rs @@ -1,11 +1,9 @@ -use crate::{ - env::standard::{AfPacketDriver, VethLikeDriver}, - error::{Error, VethError}, - netns::NetNs, - route::{add_arp_entry_with_netns, add_route_with_netns, set_loopback_up_with_netns}, - veth::{MacAddr, VethCell, VethPair, VethPairBuilder}, - NetNsGuard, +use std::{collections::BTreeMap, sync::Arc}; +use std::{ + net::{IpAddr, Ipv4Addr}, + str::FromStr, }; + use futures::TryStreamExt; use once_cell::sync::OnceCell; use rand::distr::Alphanumeric; @@ -13,17 +11,21 @@ use rand::{rng, RngExt}; use rtnetlink::packet_route::{address::AddressAttribute, link::LinkAttribute, route::RouteScope}; #[cfg(feature = "serde")] use serde::{Deserialize, Serialize}; -use std::{collections::BTreeMap, sync::Arc}; -use std::{ - net::{IpAddr, Ipv4Addr}, - str::FromStr, -}; use tokio::runtime::Handle; use tracing::{debug, error, info, instrument, trace}; +use super::{AfPacketDriver, VethLikeDriver}; use crate::env::{RattanEnv, RattanEnvConfig}; use crate::error::MetalError; use crate::InterfaceBuildArtifact; +use crate::{ + env::{get_addresses_in_use, IpAddrLock, IP_LOCK_DIR}, + error::{Error, VethError}, + netns::NetNs, + route::{add_arp_entry_with_netns, add_route_with_netns, set_loopback_up_with_netns}, + veth::{MacAddr, VethCell, VethPair, VethPairBuilder}, + NetNsGuard, +}; // ns-left ns-right // +-----------+ [Internet] [Internet] +-----------+ @@ -70,65 +72,6 @@ fn get_veth_mac_address(pair_group: VethPairGroup, pair_id: usize, suffix: u8) - [0x38, 0x7e, 0x58, pair_group as u8, pair_id as u8, suffix].into() } -fn get_addresses_in_use() -> Result, Error> { - debug!("Get addresses in use"); - let rt = tokio::runtime::Builder::new_current_thread() - .enable_all() - .build() - .map_err(|e| { - error!("Failed to build rtnetlink runtime"); - Error::TokioRuntimeError(e.into()) - })?; - let _guard = rt.enter(); - let (conn, rtnl_handle, _) = rtnetlink::new_connection()?; - rt.spawn(conn); - - let mut addresses = vec![]; - rt.block_on(async { - let mut links = rtnl_handle.address().get().execute(); - while let Ok(Some(address_msg)) = links.try_next().await { - for address_attr in address_msg.attributes { - if let AddressAttribute::Address(address) = address_attr { - trace!(?address, ?address_msg.header.prefix_len, "Get address"); - addresses.push(address); - } - } - } - }); - debug!(?addresses, "Addresses in use"); - Ok(addresses) -} - -const IP_LOCK_DIR: &str = "/tmp/rattan/ip_lock"; - -struct IpAddrLock { - file_dir: String, -} - -impl IpAddrLock { - /// Lock an IP address by creating a file with the same name as the it - fn new(ip: IpAddr) -> Result, std::io::Error> { - // for ipv6, ':' may be illegal as a filename - let ip_str = format!("{ip}").replace(':', "_"); - let file_dir = format!("{IP_LOCK_DIR}/{ip_str}"); - match std::fs::File::create_new(&file_dir) { - // Lock successfully - Ok(_) => Ok(Some(IpAddrLock { file_dir })), - // Lock file already exists - Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => Ok(None), - // Other error - Err(e) => Err(e), - } - } -} - -impl Drop for IpAddrLock { - /// Unlock by removing the lock file - fn drop(&mut self) { - let _ = std::fs::remove_file(&self.file_dir); - } -} - lazy_static::lazy_static! { static ref STD_ENV_LOCK: Arc> = Arc::new(parking_lot::Mutex::new(())); } diff --git a/rattan-env/src/error.rs b/rattan-env/src/error.rs index 6c15f34f..ec2348ff 100644 --- a/rattan-env/src/error.rs +++ b/rattan-env/src/error.rs @@ -2,6 +2,8 @@ use std::process::{ExitCode, Termination}; #[cfg(feature = "xdp")] use camellia_net::error::CamelliaError; +#[cfg(feature = "rvnic")] +use rvnic::Error as RvnicError; pub type Result = std::result::Result; @@ -21,6 +23,8 @@ pub enum Error { TokioRuntimeError(#[from] TokioRuntimeError), #[error("Rtnetlink error: {0}")] RtnetlinkError(#[from] rtnetlink::Error), + #[error("Metal error: {0}")] + MetalError(#[from] MetalError), } #[derive(Debug, thiserror::Error)] @@ -56,10 +60,20 @@ pub enum MetalError { #[cfg_attr(feature = "xdp", error("Encounter XDP error, {0}"))] #[cfg(feature = "xdp")] XDPError(#[from] CamelliaError), + #[cfg_attr(feature = "rvnic", error("Encounter Rvnic error, {0}"))] + #[cfg(feature = "rvnic")] + RvnicError(#[from] RvnicError), #[error("not interested packet")] NotInterestedPacket, } +#[cfg(feature = "rvnic")] +impl From for Error { + fn from(value: RvnicError) -> Self { + Error::MetalError(MetalError::RvnicError(value)) + } +} + /// An error that may occur when parsing a MAC address string. #[derive(Copy, Clone, Debug, Eq, PartialEq, Hash, thiserror::Error)] pub enum MacParseError { @@ -105,6 +119,7 @@ impl Termination for Error { Error::NsError(_) => ExitCode::from(71), Error::VethError(_) => ExitCode::from(71), Error::MacParseError(_) => ExitCode::from(78), + Error::MetalError(_) => ExitCode::from(71), } } } diff --git a/rattan-env/src/interface.rs b/rattan-env/src/interface.rs index add2679c..21c8cdd6 100644 --- a/rattan-env/src/interface.rs +++ b/rattan-env/src/interface.rs @@ -25,6 +25,14 @@ pub trait InterfaceDriver: Send + 'static { fn sender(&self) -> Arc; fn receiver(&mut self) -> &mut Self::Receiver; fn into_receiver(self) -> Self::Receiver; + + // Let the driver to decide use async recving or blocking recv + fn use_blocking_recv() -> bool { + false + } + fn recv_thread(_receiver: Self::Receiver, _tx: tokio::sync::mpsc::Sender) { + unimplemented!("Driver decided to use blocking recv but did not implement it") + } } pub struct InterfaceBuildArtifact { diff --git a/rvnic/kernel/.clang-format b/rvnic/kernel/.clang-format new file mode 100644 index 00000000..5c0bf894 --- /dev/null +++ b/rvnic/kernel/.clang-format @@ -0,0 +1,11 @@ +# yaml-language-server: $schema=https://json.schemastore.org/clang-format-21.x.json + +BasedOnStyle: LLVM + +IndentWidth: 4 +ContinuationIndentWidth: 4 +TabWidth: 4 +UseTab: Never + +BreakBeforeBraces: Attach +ColumnLimit: 100 \ No newline at end of file diff --git a/rvnic/kernel/.clangd b/rvnic/kernel/.clangd new file mode 100644 index 00000000..966da82f --- /dev/null +++ b/rvnic/kernel/.clangd @@ -0,0 +1,37 @@ +CompileFlags: + Remove: + # Remove GCC-specific flags that clang doesn't understand + - -mpreferred-stack-boundary=* + - -mindirect-branch=* + - -mindirect-branch-register + - -mindirect-branch-cs-prefix + - -fno-allow-store-data-races + - -fmin-function-alignment=* + - -fconserve-stack + - -mrecord-mcount + - -mfentry + - -fsanitize=bounds-strict + - -fno-var-tracking-assignments + - -mfunction-return=* + - -fzero-call-used-regs=* + - -fno-stack-clash-protection + - -ftrivial-auto-var-init=* + - -mharden-sls=* + - -fstrict-flex-arrays=* + - -fpatchable-function-entry=* + - -DCC_USING_FENTRY + Add: + # Help clang find kernel headers + - -D__KERNEL__ + - -DMODULE + - -Wno-unknown-warning-option + +Diagnostics: + Suppress: + - drv_unknown_argument + - drv_unsupported_option_argument + - drv_unsupported_opt_for_target + ClangTidy: + Remove: + # RCU macros intentionally use sizeof on pointers for type checking + - bugprone-sizeof-expression diff --git a/rvnic/kernel/.gitignore b/rvnic/kernel/.gitignore new file mode 100644 index 00000000..f316d744 --- /dev/null +++ b/rvnic/kernel/.gitignore @@ -0,0 +1,23 @@ +# Kernel module build artifacts +*.o +*.ko +*.mod +*.mod.c +*.cmd +.tmp_versions/ +modules.order +Module.symvers +*.o.d +compile_commands.json + +# Vagrant +.vagrant/ + +# Editor backup files +*~ +*.swp +*.swo +.*.sw? + +# Clangd +.cache/ diff --git a/rvnic/kernel/Makefile b/rvnic/kernel/Makefile new file mode 100644 index 00000000..ec329bf9 --- /dev/null +++ b/rvnic/kernel/Makefile @@ -0,0 +1,41 @@ +# Makefile for Rattan Virtual NIC kernel module + +obj-m += rattan_vnic.o + +KDIR ?= /lib/modules/$(shell uname -r)/build +PWD := $(shell pwd) + +# Default target +all: + $(MAKE) -C $(KDIR) M=$(PWD) modules + +clean: + $(MAKE) -C $(KDIR) M=$(PWD) clean + rm -f Module.symvers modules.order + +install: + $(MAKE) -C $(KDIR) M=$(PWD) modules_install + depmod -a + +uninstall: + rm -f /lib/modules/$(shell uname -r)/extra/rattan_vnic.ko + depmod -a + +load: + sudo insmod rattan_vnic.ko + +unload: + sudo rmmod rattan_vnic + +reload: unload load + +# Generate compile_commands.json for IDE/LSP support +compile_commands.json: + $(MAKE) -C $(KDIR) M=$(PWD) compile_commands.json + +# Development helpers +check: + @echo "Checking module info..." + @modinfo rattan_vnic.ko 2>/dev/null || echo "Module not built" + +.PHONY: all clean install uninstall load unload reload check diff --git a/rvnic/kernel/Vagrantfile b/rvnic/kernel/Vagrantfile new file mode 100644 index 00000000..5c24c668 --- /dev/null +++ b/rvnic/kernel/Vagrantfile @@ -0,0 +1,148 @@ +# -*- mode: ruby -*- +# vi: set ft=ruby : +# +# Rattan vNIC Kernel Module Development VM (libvirt) +# +# Prerequisites: +# sudo apt install vagrant libvirt-daemon-system vagrant-libvirt +# sudo usermod -aG libvirt $USER +# newgrp libvirt # or logout/login +# +# Usage: +# cd rvnic/kernel/ +# vagrant up - Create and provision the VM (default: debian13) +# vagrant ssh - SSH into the VM +# vagrant halt - Stop the VM +# vagrant destroy - Remove the VM +# vagrant rsync-auto - Auto-sync on file changes (if using rsync) +# +# Crash capture: +# The guest kernel console goes to the serial port, which is written to +# SERIAL_LOG on the host (default .vagrant/serial.log next to this file, +# override with RVNIC_SERIAL_LOG, but not under /tmp, see below). An oops +# panics and reboots the guest after 5 s instead of wedging it, so the BUG +# text, RIP and call trace are always left in that file: +# tail -f rvnic/kernel/.vagrant/serial.log +# The serial port is part of the domain definition: an existing VM needs +# `vagrant destroy && vagrant up` to pick it up. The guest side can be +# (re)applied alone with `vagrant provision --provision-with debug-console` +# followed by `vagrant reload`. +# +# Inside VM: +# cd /rattan - Access the full rattan project +# cd /rattan/rvnic/kernel - Kernel module directory +# make - Build the kernel module +# sudo insmod rattan_vnic.ko - Load the module +# sudo rmmod rattan_vnic - Unload the module +# dmesg | tail - Check kernel messages +# + +SERIAL_LOG = ENV.fetch("RVNIC_SERIAL_LOG", File.join(__dir__, ".vagrant", "serial.log")) + +# libvirt opens the serial file as root with O_CREAT. Pre-create it here, as +# the invoking user, so it stays readable without sudo. Keep it out of sticky +# dirs such as /tmp: with fs.protected_regular=2 root may not O_CREAT-open an +# existing file there that someone else owns, and the domain fails to start. +unless File.exist?(SERIAL_LOG) + require "fileutils" + FileUtils.mkdir_p(File.dirname(SERIAL_LOG)) + File.open(SERIAL_LOG, "a", 0o644) {} +end + +Vagrant.configure("2") do |config| + # Base box + config.vm.box = "behemoth/debian13" + + config.vm.hostname = "rattan-dev" + + config.vm.synced_folder ".", "/vagrant", disabled: true + + config.vm.synced_folder ".", "/home/vagrant/rvnic", + type: "rsync", + rsync__auto: true, + rsync__include: ["rattan_vnic.ko"] + + config.vm.synced_folder "../librvnic/target/release/examples", "/home/vagrant/examples", + type: "rsync", + rsync__auto: true, + rsync__include: ["test_forward", "test_forward_drop", "test_forward_multiqueue", "test_umem"] + + # Libvirt provider configuration + config.vm.provider "libvirt" do |lv| + + lv.memory = 4096 + lv.cpus = 16 + + lv.driver = "kvm" + lv.nested = true + + lv.cpu_mode = "host-passthrough" + + # Kernel console -> host file (see "Crash capture" above) + lv.serial :type => "file", :source => { :path => SERIAL_LOG } + # lv.machine_virtual_size = 128 # 128GB + + end + + # Provisioning script + config.vm.provision "shell", inline: <<-SHELL + # Update package lists + apt-get update + + # 1. System Essentials (Keep for stability) + apt-get install -y libpam-systemd libnss-systemd systemd-timesyncd + + # 2. Network Experiment Tools + + apt-get install -y \ + bash-completion \ + bridge-utils \ + bzip2 \ + bpftrace \ + ca-certificates \ + dbus \ + ethtool \ + file \ + gettext-base \ + gnupg \ + htop \ + iproute2 \ + iputils-ping \ + liblockfile-bin \ + lsof \ + ltrace \ + lvm2 \ + media-types \ + ncurses-term \ + netcat-traditional \ + openssh-client \ + perl \ + psmisc \ + rsync \ + socat \ + ssl-cert \ + strace \ + tcpdump \ + traceroute \ + util-linux-extra \ + xz-utils + + DEBIAN_FRONTEND=noninteractive apt-get install -y iperf3 + + # 3. Update to the newest kernel + apt-get install -y linux-image-amd64 linux-headers-amd64 + + SHELL + + # Kernel console on ttyS0, and panic + reboot on oops (see "Crash capture") + config.vm.provision "debug-console", type: "shell", inline: <<-SHELL + set -e + if ! grep -q 'console=ttyS0' /etc/default/grub; then + sed -i 's/^GRUB_CMDLINE_LINUX="\\(.*\\)"/GRUB_CMDLINE_LINUX="\\1 console=tty0 console=ttyS0,115200n8"/' /etc/default/grub + update-grub + fi + printf 'kernel.panic_on_oops = 1\\nkernel.panic = 5\\n' > /etc/sysctl.d/90-rattan-crash.conf + sysctl -q -p /etc/sysctl.d/90-rattan-crash.conf + SHELL + +end diff --git a/rvnic/kernel/rattan_vnic.c b/rvnic/kernel/rattan_vnic.c new file mode 100644 index 00000000..8445cf4d --- /dev/null +++ b/rvnic/kernel/rattan_vnic.c @@ -0,0 +1,2601 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Rattan Virtual NIC + * + * Copyright (C) 2025 Rattan Project + * + * This module implements a virtual network device with: + * - Miscdevice interface for userspace control + * - Dynamic netdev creation on open + * - AF_XDP-style memory management + */ + +#include "rattan_vnic_uapi.h" +#define pr_fmt(fmt) "rvnic: " fmt + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "rattan_vnic.h" + +/* + * Kernel version compatibility for page pinning APIs + * pin_user_pages_fast was introduced in 5.6 + */ +#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 6, 0) +#define rattan_pin_pages(addr, nr, flags, pages) pin_user_pages_fast(addr, nr, flags, pages) +#define rattan_unpin_pages(pages, nr) unpin_user_pages(pages, nr) +#else +#define rattan_pin_pages(addr, nr, flags, pages) get_user_pages_fast(addr, nr, flags, pages) +static void rattan_unpin_pages(struct page **pages, unsigned long nr) { + unsigned long i; + + for (i = 0; i < nr; i++) + put_page(pages[i]); +} +#endif + +/* + * Kernel version compatibility for netif_napi_add + * The weight parameter was removed in kernel 6.1 + */ +#if LINUX_VERSION_CODE < KERNEL_VERSION(6, 1, 0) +#define rattan_netif_napi_add(dev, napi, poll) netif_napi_add(dev, napi, poll, NAPI_POLL_WEIGHT) +#else +#define rattan_netif_napi_add(dev, napi, poll) netif_napi_add(dev, napi, poll) +#endif + +/* + * Kernel version compatibility for skb_gso_segment + * Moved from netdevice.h to net/gso.h in kernel 6.4.10 + */ +#if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 4, 10) +#include +#endif + +/* + * Kernel version compatibility for per-cpu device stats + * - The core allocates dev->tstats for NETDEV_PCPU_STAT_TSTATS since 6.7; + * before that the driver does it in ndo_init/ndo_uninit. + * - Per-cpu drop counters (dev->core_stats) since 5.18; before that the + * atomic dev->{rx,tx}_dropped. + */ +#if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 7, 0) +#define RATTAN_CORE_TSTATS +#endif + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 18, 0) +#define rattan_tx_dropped_inc(dev) dev_core_stats_tx_dropped_inc(dev) +#define rattan_rx_dropped_inc(dev) dev_core_stats_rx_dropped_inc(dev) +#else +#define rattan_tx_dropped_inc(dev) atomic_long_inc(&(dev)->tx_dropped) +#define rattan_rx_dropped_inc(dev) atomic_long_inc(&(dev)->rx_dropped) +#endif + +/* + * Kernel version compatibility for hrtimer_init / hrtimer_setup + * hrtimer_setup was introduced since 6.13 and hrtimer_init was removed from 6.15 + * 6.14 is a version that these two functions coexist + */ +#if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 14, 0) +#define rattan_hrtimer_setup(timer, restart, clock, mode) hrtimer_setup(timer, restart, clock, mode) +#else +static inline void rattan_hrtimer_setup(struct hrtimer *timer, + enum hrtimer_restart (*function)(struct hrtimer *), + clockid_t which_clock, + enum hrtimer_mode mode) +{ + hrtimer_init(timer, which_clock, mode); + timer->function = function; +} +#endif + +static int max_devices = 256; +module_param(max_devices, int, 0644); +MODULE_PARM_DESC(max_devices, "Maximum number of rattan devices (default: 256)"); + +static int num_queues = 1; +module_param(num_queues, int, 0644); +MODULE_PARM_DESC(num_queues, "Number of TX/RX queues per rattan device (default: 1)"); + +static unsigned int debug_stats_interval_ms = 0; +module_param(debug_stats_interval_ms, uint, 0644); +MODULE_PARM_DESC(debug_stats_interval_ms, + "Periodic per-queue debug stats dump interval in ms (0 = disabled). " + "The counters only count while this is nonzero."); + +/* + * Per-cpu debug counters (struct rattan_queue_dbg). A plain branch on a + * read-mostly global when the dump is off; no locked op or shared line when on. + */ +#define RATTAN_DBG_ADD(queue, field, n) \ + do { \ + if (unlikely(READ_ONCE(debug_stats_interval_ms))) \ + this_cpu_add((queue)->dbg->field, (n)); \ + } while (0) +#define RATTAN_DBG_INC(queue, field) RATTAN_DBG_ADD(queue, field, 1) + +/* ================== Debug: ndo_start_xmit latency bisection ================== + * + * Splits the xmit path into stages and warns, with a per-stage breakdown, when + * the total exceeds slow_tx_ns. Set slow_tx_ns=0 to disable: each checkpoint + * then costs one branch and nothing else. + * + * Remove this block (and its call sites) once the slow-TX question is settled. + */ +static unsigned int slow_tx_ns = 0; +module_param(slow_tx_ns, uint, 0644); +MODULE_PARM_DESC(slow_tx_ns, "Warn when ndo_start_xmit exceeds this many ns (0 = disabled)"); + +enum { + XT_START, /* entry to ndo_start_xmit */ + XT_RESOLVE, /* queue + rings + umem resolved */ + XT_ALLOC, /* unused since the slab entry went away; kept for log compatibility */ + XT_CHUNK, /* chunk obtained: COMP/FILL pop + DROP-ring token release */ + XT_ENTRY, /* header copied to UMEM */ + XT_ORPHAN, /* skb_orphan + nf_reset_ct, i.e. the socket destructor */ + XT_HASH, /* bound to the chunk's pending slot, i.e. the token insert */ + XT_PUSH, /* descriptor published to the RX ring */ + XT_NOTIFY, /* rattan_rx_notify returned, i.e. the userspace wake-up */ + XT_NR +}; + +static const char *const xt_names[XT_NR] = { + "start", "resolve", "alloc", "chunk", "entry", "orphan", "hash", "push", "notify", +}; + +/* Which branch rattan_rx_notify() took */ +#define RATTAN_NOTIFY_NONE 0 /* not reached (packet dropped) */ +#define RATTAN_NOTIFY_ARM 1 /* first of a batch: hrtimer armed */ +#define RATTAN_NOTIFY_MID 2 /* mid-batch: nothing done */ +#define RATTAN_NOTIFY_WAKE 3 /* batch full: wake_up performed */ + +/* + * local_clock() is only comparable within one CPU, which is fine: ndo_start_xmit + * runs with BH (hence preemption) disabled, so every sample below is taken on + * the same CPU. For a GSO skb the per-stage fields describe the LAST segment, + * while nsegs counts all of them. + */ +struct rattan_xmit_trace { + u64 t[XT_NR]; + unsigned long irqs_start; + u32 nsegs; + u8 notify_path; + bool recycled; /* a DROP-ring token was released during this call */ +}; + +#define XT(tr, idx) \ + do { \ + if (tr) \ + (tr)->t[idx] = local_clock(); \ + } while (0) + +/* 200 lines/s is enough to see a distribution without flooding the log */ +static DEFINE_RATELIMIT_STATE(slow_tx_rs, HZ, 200); + +/* + * Format " name=delta" for every recorded stage in t[1..last], each delta taken + * against the nearest earlier recorded stage. t[0] is the start timestamp. + */ +static void rattan_format_stages(const u64 *t, int last, const char *const *names, char *buf, + size_t size) { + size_t n = 0; + int i, j; + + buf[0] = '\0'; + for (i = 1; i <= last; i++) { + if (!t[i]) + continue; + for (j = i - 1; j >= 0; j--) { + if (t[j]) { + n += scnprintf(buf + n, size - n, " %s=%llu", names[i], t[i] - t[j]); + break; + } + } + } +} + +static void rattan_report_slow_tx(struct rattan_xmit_trace *tr, struct rattan_queue *queue, + const char *path) { + unsigned long irqs; + char buf[192]; + u64 total; + int i, last = XT_START; + + if (!tr || !tr->t[XT_START]) + return; + + irqs = kstat_this_cpu->irqs_sum; + + for (i = XT_START + 1; i < XT_NR; i++) + if (tr->t[i]) + last = i; + + /* An early drop records no stages; still report the total. */ + total = (last == XT_START ? local_clock() : tr->t[last]) - tr->t[XT_START]; + if (total < slow_tx_ns) + return; + + rattan_format_stages(tr->t, last, xt_names, buf, sizeof(buf)); + + if (!__ratelimit(&slow_tx_rs)) + return; + + pr_warn("slow tx[%s] rattan%d:q%u %llu ns cpu=%d segs=%u notify=%u " + "recycled=%d hardirqs=%lu |%s\n", + path, queue ? queue->ctx->dev_id : -1, queue ? queue->qid : 0, total, + smp_processor_id(), tr->nsegs, tr->notify_path, tr->recycled, + irqs - tr->irqs_start, buf); +} +/* ============== End debug: ndo_start_xmit latency bisection ============== */ + +/* ================== Debug: NAPI per-packet latency bisection ================== + * + * Same idea as the xmit trace, for one iteration of the rattan_napi_poll loop. + * Stage names are distinct from the xmit ones so the two can be aggregated from + * one log. Set slow_rx_ns=1 to sample every packet (subject to the ratelimit). + */ +static unsigned int slow_rx_ns = 0; +module_param(slow_rx_ns, uint, 0644); +MODULE_PARM_DESC(slow_rx_ns, "Warn when one NAPI packet exceeds this many ns (0 = disabled)"); + +enum { + NT_START, /* top of the loop iteration */ + NT_PEEK, /* COMP space check + TX descriptor peeked */ + NT_LOOKUP, /* token's pending slot claimed */ + NT_UNLINK, /* descriptor consumed, slot emptied */ + NT_PREP, /* header copy-back + eth_type_trans */ + NT_STACK, /* netif_receive_skb returned */ + NT_RELEASE, /* chunk(s) completed, rings reference dropped */ + NT_NR +}; + +static const char *const nt_names[NT_NR] = { + "nstart", "peek", "lookup", "unlink", "prep", "stack", "release", +}; + +struct rattan_napi_trace { + u64 t[NT_NR]; + unsigned long irqs_start; +}; + +#define NT(tr, idx) XT(tr, idx) + +static DEFINE_RATELIMIT_STATE(slow_rx_rs, HZ, 200); + +static void rattan_napi_trace_begin(struct rattan_napi_trace *tr) { + memset(tr, 0, sizeof(*tr)); + tr->irqs_start = kstat_this_cpu->irqs_sum; + tr->t[NT_START] = local_clock(); +} + +static void rattan_report_slow_rx(struct rattan_napi_trace *tr, struct rattan_queue *queue, + const char *path) { + char buf[192]; + u64 total; + int i, last = NT_START; + + if (!tr || !tr->t[NT_START]) + return; + + for (i = NT_START + 1; i < NT_NR; i++) + if (tr->t[i]) + last = i; + if (last == NT_START) + return; + + total = tr->t[last] - tr->t[NT_START]; + if (total < slow_rx_ns) + return; + + if (!__ratelimit(&slow_rx_rs)) + return; + + rattan_format_stages(tr->t, last, nt_names, buf, sizeof(buf)); + pr_warn("slow rx[%s] rattan%d:q%u %llu ns cpu=%d hardirqs=%lu |%s\n", path, + queue->ctx->dev_id, queue->qid, total, smp_processor_id(), + kstat_this_cpu->irqs_sum - tr->irqs_start, buf); +} +/* ============== End debug: NAPI per-packet latency bisection ============== */ + +/* Device management - use IDA for unique device IDs */ +static DEFINE_IDA(rattan_ida); + +/* Every live UMEM, so GC can find the pending slots. Protected by rattan_umems_lock. */ +static LIST_HEAD(rattan_umems); +static DEFINE_MUTEX(rattan_umems_lock); + +/* + * Live token count, for the debug dump only. Per-cpu so that the xmit CPU + * (insert) and the NAPI CPU (release) do not bounce a shared counter. + */ +static DEFINE_PER_CPU(long, rattan_pending_tokens); + +/* Periodic GC workqueue */ +static struct delayed_work gc_work; + +/* Deferred rattan_rings cleanup; drained on module exit */ +static struct workqueue_struct *rattan_cleanup_wq; +static bool gc_work_initialized; + +static const struct file_operations rattan_vnic_fops; + +static int rattan_vnic_open(struct net_device *dev); +static int rattan_vnic_stop(struct net_device *dev); +static netdev_tx_t rattan_vnic_start_xmit(struct sk_buff *skb, struct net_device *dev); + +static bool fill_ring_pop(struct rattan_fill_ring *fill, u64 *addr); +static bool rx_ring_has_space(struct rattan_rx_ring *rx); +static int rx_ring_push(struct rattan_rx_ring *rx, struct rattan_desc *desc); +static u32 comp_ring_push_many(struct rattan_comp_ring *comp, const u64 *addrs, u32 n); +static u32 comp_ring_free(struct rattan_comp_ring *comp); +static bool comp_ring_pop(struct rattan_comp_ring *comp, u64 *addr); +static bool release_token(struct rattan_umem *umem, u64 token); +static bool rattan_recycle_drop_token(struct rattan_umem *umem, struct rattan_drop_ring *drop); +static bool try_get_free_addr(struct rattan_queue *queue, struct rattan_rings *rings, + struct rattan_umem *umem, u64 *addr, + struct rattan_xmit_trace *tr); +static bool tx_ring_peek(struct rattan_tx_ring *tx, struct rattan_desc *desc); +static void tx_ring_advance(struct rattan_tx_ring *tx); +static int rattan_napi_poll(struct napi_struct *napi, int budget); +static enum hrtimer_restart rattan_rx_timer_cb(struct hrtimer *timer); +static struct rattan_queue *rattan_get_queue(struct rattan_vnic_ctx *ctx, u32 qid); +static struct rattan_queue *rattan_get_xmit_queue(struct rattan_vnic_ctx *ctx, struct sk_buff *skb); +static struct rattan_rings *rattan_get_rings(struct rattan_queue *queue); +static void rattan_put_rings(struct rattan_rings *rings); +static bool rattan_complete_chunk(struct rattan_queue *queue, struct rattan_rings *rings, u64 addr); +static void rattan_debug_dump_queue_stats(struct rattan_queue *queue); +static void rattan_umem_drain(struct rattan_umem *umem); +static u64 rattan_slot_insert(struct rattan_umem *umem, u32 idx, struct sk_buff *skb, + struct rattan_rings *rings, u16 qid); + +/* + * Is addr (a UMEM offset from userspace: FILL, TX descriptors, CLONE) the + * start of a chunk? umem->len is a multiple of chunk_size, so an aligned + * addr below len names a whole chunk. Never test "addr + chunk_size <= len": + * a u64 addr near 2^64 wraps that sum around to pass. + */ +static inline bool rattan_chunk_valid(struct rattan_umem *umem, u64 addr) { + return (addr & (u64)(umem->chunk_size - 1)) == 0 && addr < umem->len; +} + +/* ========== Net Device Operations ========== */ + +#ifndef RATTAN_CORE_TSTATS +static int rattan_vnic_ndo_init(struct net_device *dev) { + dev->tstats = netdev_alloc_pcpu_stats(struct pcpu_sw_netstats); + return dev->tstats ? 0 : -ENOMEM; +} + +static void rattan_vnic_ndo_uninit(struct net_device *dev) { + free_percpu(dev->tstats); +} +#endif + +static const struct net_device_ops rattan_netdev_ops = { +#ifndef RATTAN_CORE_TSTATS + .ndo_init = rattan_vnic_ndo_init, + .ndo_uninit = rattan_vnic_ndo_uninit, +#endif + .ndo_open = rattan_vnic_open, + .ndo_stop = rattan_vnic_stop, + .ndo_start_xmit = rattan_vnic_start_xmit, + .ndo_get_stats64 = dev_get_tstats64, +}; + +static int rattan_vnic_open(struct net_device *dev) { + struct rattan_vnic_ctx *ctx = netdev_priv(dev); + u16 qid; + + for (qid = 0; qid < ctx->num_queues; qid++) + netif_start_subqueue(dev, qid); + + return 0; +} + +static int rattan_vnic_stop(struct net_device *dev) { + struct rattan_vnic_ctx *ctx = netdev_priv(dev); + u16 qid; + + for (qid = 0; qid < ctx->num_queues; qid++) + netif_stop_subqueue(dev, qid); + + return 0; +} + +/* + * RX wake-up coalescing timer callback + * + * Called when the coalescing timeout expires. Wakes up userspace if there + * are pending packets that haven't been notified yet. + */ +static enum hrtimer_restart rattan_rx_timer_cb(struct hrtimer *timer) { + struct rattan_queue *queue = container_of(timer, struct rattan_queue, rx_timer); + struct rattan_vnic_ctx *ctx = queue->ctx; + + /* Wake up if there are pending packets */ + if (atomic_xchg(&queue->rx_pending, 0) > 0) { + if (waitqueue_active(&queue->wait)) + wake_up_interruptible_poll(&queue->wait, EPOLLIN | EPOLLRDNORM); + if (waitqueue_active(&ctx->wait)) + wake_up_interruptible_poll(&ctx->wait, EPOLLIN | EPOLLRDNORM); + } + + return HRTIMER_NORESTART; +} + +/* + * Try to wake up userspace with coalescing + * + * Called after pushing packets to the RX ring. Implements batching: + * - If batch threshold reached, wake immediately + * - Otherwise, arm timer to wake after timeout (if not already armed) + */ +static void rattan_rx_notify(struct rattan_queue *queue, struct rattan_xmit_trace *tr) { + struct rattan_vnic_ctx *ctx = queue->ctx; + int pending; + + RATTAN_DBG_INC(queue, rx_notify); + pending = atomic_inc_return(&queue->rx_pending); + + if (tr) + tr->notify_path = pending >= RATTAN_RX_BATCH_SIZE ? RATTAN_NOTIFY_WAKE + : pending == 1 ? RATTAN_NOTIFY_ARM + : RATTAN_NOTIFY_MID; + + if (pending >= RATTAN_RX_BATCH_SIZE) { + /* Batch full - wake immediately */ + atomic_set(&queue->rx_pending, 0); + hrtimer_try_to_cancel(&queue->rx_timer); + if (waitqueue_active(&queue->wait)) + wake_up_interruptible_poll(&queue->wait, EPOLLIN | EPOLLRDNORM); + if (waitqueue_active(&ctx->wait)) + wake_up_interruptible_poll(&ctx->wait, EPOLLIN | EPOLLRDNORM); + } else if (pending == 1) { + /* First packet in new batch - arm timer */ + hrtimer_start(&queue->rx_timer, ns_to_ktime(RATTAN_RX_TIMEOUT_NS), HRTIMER_MODE_REL_SOFT); + } + /* Otherwise timer is already armed, just wait */ +} + +/* + * Transmit a single packet (or GSO segment) to userspace via RX ring + * Returns: + * < 0: error + * 0: success + */ +static int rattan_xmit_one(struct sk_buff *skb, struct net_device *dev, struct rattan_queue *queue, + struct rattan_rings *rings, struct rattan_umem *umem, + struct rattan_xmit_trace *tr) { + struct rattan_desc desc; + u64 chunk_addr; + u64 token; + void *chunk_ptr; + u32 copy_len; + u32 pkt_len; + + /* Check RX ring has space */ + if (!rx_ring_has_space(rings->rx)) { + RATTAN_DBG_INC(queue, xmit_rx_full); + pr_warn_ratelimited("rattan%d:q%u xmit blocked: RX ring full\n", queue->ctx->dev_id, + queue->qid); + return -ENOSPC; + } + + /* Get chunk from COMP or FILL ring */ + if (!try_get_free_addr(queue, rings, umem, &chunk_addr, tr)) { + RATTAN_DBG_INC(queue, xmit_fill_empty); + pr_warn_ratelimited("rattan%d:q%u xmit blocked: no chunk available\n", queue->ctx->dev_id, + queue->qid); + return -ENOBUFS; + } + XT(tr, XT_CHUNK); + + /* Validate chunk address */ + if (!rattan_chunk_valid(umem, chunk_addr)) { + RATTAN_DBG_INC(queue, xmit_invalid_addr); + pr_warn_ratelimited("rattan%d:q%u invalid chunk addr 0x%llx\n", queue->ctx->dev_id, + queue->qid, chunk_addr); + return -EINVAL; + } + + /* + * Copy header to UMEM chunk. RATTAN_HEADER_SIZE is 0 by design; the + * guard lets the compiler drop the (out-of-line) copy entirely. + */ + copy_len = 0; + if (RATTAN_HEADER_SIZE) { + copy_len = min_t(u32, skb->len, RATTAN_HEADER_SIZE); + chunk_ptr = (u8 *)umem->mapped + chunk_addr + umem->headroom; + skb_copy_bits(skb, 0, chunk_ptr, copy_len); + } + XT(tr, XT_ENTRY); + + /* + * Finish with the SKB *before* it becomes reachable through a token. + * + * Once the slot is LIVE, whoever presents the token (normally NAPI on + * another CPU, after userspace hands it back via the TX ring) may pass + * the SKB to netif_receive_skb(), after which the stack owns it and may + * free it. Any access to skb after the insert races with that and can be + * a use-after-free. + * + * Orphan the skb, or it will still occupy the TSQ credit + * on the sender side. + * + * Copied from https://elixir.bootlin.com/linux/v6.12.74/source/drivers/net/tun.c#L1121-L1126 + */ + pkt_len = skb->len; + skb_orphan(skb); + nf_reset_ct(skb); + XT(tr, XT_ORPHAN); + + /* Bind the SKB to the chunk; the slot owns this rings reference */ + refcount_inc(&rings->refcount); + token = rattan_slot_insert(umem, chunk_addr >> umem->chunk_shift, skb, rings, queue->qid); + if (!token) { + rattan_put_rings(rings); + RATTAN_DBG_INC(queue, xmit_invalid_addr); + pr_warn_ratelimited("rattan%d:q%u chunk 0x%llx is still bound to a live token\n", + queue->ctx->dev_id, queue->qid, chunk_addr); + return -EBUSY; + } + XT(tr, XT_HASH); + + /* Build descriptor for userspace */ + desc.addr = chunk_addr; + desc.len = copy_len; + desc.options = pkt_len & 0xffff; + desc.token = token; + desc.timestamp = ktime_get_ns(); + + dev_sw_netstats_tx_add(dev, 1, pkt_len); + + rx_ring_push(rings->rx, &desc); + RATTAN_DBG_INC(queue, xmit_ok); + XT(tr, XT_PUSH); + + return 0; +} + +static netdev_tx_t rattan_vnic_start_xmit(struct sk_buff *skb, struct net_device *dev) { + struct rattan_vnic_ctx *ctx = netdev_priv(dev); + struct rattan_queue *queue = NULL; + struct rattan_rings *rings; + struct rattan_umem *umem; + struct sk_buff *segs, *next; + struct rattan_xmit_trace trace; + struct rattan_xmit_trace *tr = NULL; + int ret; + bool wake = false; + + if (slow_tx_ns) { + memset(&trace, 0, sizeof(trace)); + tr = &trace; + tr->irqs_start = kstat_this_cpu->irqs_sum; + tr->t[XT_START] = local_clock(); + } + + rcu_read_lock(); + + /* Check if device is started */ + if (!ctx->started) + goto drop; + + /* + * Queue selection currently follows skb->queue_mapping. + * Custom steering can be added later via ndo_select_queue or a + * driver-specific hashing policy if needed. + */ + queue = rattan_get_xmit_queue(ctx, skb); + if (!queue) + goto drop; + + /* + * No reference on rings here: we are inside rcu_read_lock(), and + * fop_release unpublishes the rings and waits for RCU before dropping + * the queue's own reference. Each slot takes its own reference below. + */ + rings = rcu_dereference(queue->rings); + umem = rcu_dereference(ctx->umem); + if (!rings || !umem) + goto drop; + XT(tr, XT_RESOLVE); + + /* + * Handle GSO packets: segment them into individual MTU-sized SKBs. + * This ensures each entry in the hash table is a standard packet, + * which simplifies forwarding and allows per-packet processing. + */ + if (skb_is_gso(skb)) { + segs = skb_gso_segment(skb, 0); + if (IS_ERR_OR_NULL(segs)) { + dev_kfree_skb_any(skb); + rattan_tx_dropped_inc(dev); + goto out; + } + + /* Free the original GSO SKB */ + consume_skb(skb); + + /* Process each segment */ + while (segs) { + next = segs->next; + segs->next = NULL; + + ret = rattan_xmit_one(segs, dev, queue, rings, umem, tr); + if (tr) + tr->nsegs++; + if (ret < 0) { + /* Drop this and remaining segments */ + dev_kfree_skb_any(segs); + rattan_tx_dropped_inc(dev); + while (next) { + segs = next; + next = segs->next; + dev_kfree_skb_any(segs); + rattan_tx_dropped_inc(dev); + } + goto out; + } + wake = true; + segs = next; + } + } else { + /* Non-GSO packet: process directly */ + ret = rattan_xmit_one(skb, dev, queue, rings, umem, tr); + if (tr) + tr->nsegs = 1; + if (ret < 0) { + dev_kfree_skb_any(skb); + rattan_tx_dropped_inc(dev); + goto out; + } + wake = true; + } + +out: + rcu_read_unlock(); + if (wake) { + rattan_rx_notify(queue, tr); + XT(tr, XT_NOTIFY); + } + + rattan_report_slow_tx(tr, queue, "xmit"); + return NETDEV_TX_OK; + +drop: + rcu_read_unlock(); + dev_kfree_skb_any(skb); + rattan_tx_dropped_inc(dev); + + rattan_report_slow_tx(tr, queue, "drop"); + return NETDEV_TX_OK; +} + +/* + * Net device setup - called by alloc_netdev + */ +static void rattan_vnic_setup(struct net_device *dev) { + ether_setup(dev); + + dev->netdev_ops = &rattan_netdev_ops; +#ifdef RATTAN_CORE_TSTATS + /* Per-cpu tx/rx packet and byte counters, allocated by the core */ + dev->pcpu_stat_type = NETDEV_PCPU_STAT_TSTATS; +#endif + + /* Virtual device flags */ + dev->flags &= ~IFF_MULTICAST; + + /* use no queue */ + dev->priv_flags |= IFF_NO_QUEUE; + + pr_info("RVnic flags: 0x%x 0x%x\n", dev->flags, dev->priv_flags); + + /* Generate random MAC address */ + eth_hw_addr_random(dev); + + /* Standard MTU */ + dev->mtu = ETH_DATA_LEN; + dev->min_mtu = 68; + dev->max_mtu = 65535; + + /* GSO/GRO features - large packets get segmented in start_xmit */ + dev->features |= + NETIF_F_GSO | NETIF_F_GRO | NETIF_F_SG | NETIF_F_HIGHDMA | NETIF_F_GSO_SOFTWARE; + + dev->hw_features |= + NETIF_F_GSO | NETIF_F_GRO | NETIF_F_SG | NETIF_F_HIGHDMA | NETIF_F_GSO_SOFTWARE; + + /* No carrier until started */ + netif_carrier_off(dev); +} + +/* ========== Cleanup Helpers ========== */ + +/* + * UMEM cleanup helper - decrements refcount, frees if last user + */ +static void rattan_put_umem(struct rattan_umem *umem) { + if (!umem) + return; + + if (!refcount_dec_and_test(&umem->refcount)) + return; /* Still in use by other devices */ + + /* Last user - actually free the UMEM */ + mutex_lock(&rattan_umems_lock); + list_del(&umem->node); + mutex_unlock(&rattan_umems_lock); + + rattan_umem_drain(umem); + vfree(umem->pending); + + if (umem->mapped) + vunmap(umem->mapped); + + if (umem->pages) { + rattan_unpin_pages(umem->pages, umem->nr_pages); + kvfree(umem->pages); + } + + kfree(umem); +} + +static struct rattan_queue *rattan_get_queue(struct rattan_vnic_ctx *ctx, u32 qid) { + if (!ctx || qid >= ctx->num_queues) + return NULL; + + return &ctx->queues[qid]; +} + +static struct rattan_queue *rattan_get_xmit_queue(struct rattan_vnic_ctx *ctx, + struct sk_buff *skb) { + u16 qid; + + if (!ctx || !ctx->num_queues) + return NULL; + + qid = skb_get_queue_mapping(skb); + if (qid >= ctx->num_queues) + qid %= ctx->num_queues; + + return &ctx->queues[qid]; +} + +static struct rattan_rings *rattan_get_rings(struct rattan_queue *queue) { + struct rattan_rings *rings; + + if (!queue) + return NULL; + + rings = rcu_dereference(queue->rings); + if (rings) + refcount_inc(&rings->refcount); + + return rings; +} + +static void rattan_cleanup_rings(struct rattan_rings *rings) { + if (!rings) + return; + + might_sleep(); + + if (rings->mapped) + vunmap(rings->mapped); + + if (rings->pages) { + rattan_unpin_pages(rings->pages, rings->nr_pages); + kvfree(rings->pages); + } + + // kfree(rings->recycle); + kfree(rings); +} + +static void rattan_cleanup_rings_work(struct work_struct *work) { + rattan_cleanup_rings(container_of(work, struct rattan_rings, cleanup_work)); +} + +/* + * Drop a reference on rings. Safe from any context: the last put only queues + * the cleanup, which sleeps, to process context. + */ +static void rattan_put_rings(struct rattan_rings *rings) { + if (!rings) + return; + + if (refcount_dec_and_test(&rings->refcount)) + queue_work(rattan_cleanup_wq, &rings->cleanup_work); +} + +/* + * Push n chunk addresses to rings' COMP ring under one lock and one producer + * update. Returns how many fit; the rest are dropped (warned and counted). + */ +static u32 rattan_complete_chunks(struct rattan_queue *queue, struct rattan_rings *rings, + const u64 *addrs, u32 n) { + u32 pushed; + + if (!rings || !n) + return 0; + /* + * Concurrency model: + * + * This comp ring is used as MPSC (multi-producer, single-consumer): + * + * Producers: + * - napi_poll() context (softirq) + * - process context (e.g., ioctl) + * -> may run concurrently on different CPUs + * + * Consumer: + * - ndo_start_xmit() + * -> single-consumer per TX queue (no concurrent pop) + * + * Therefore, producers must be serialized. + * + * We use spin_lock_bh() here because: + * 1. It provides mutual exclusion across CPUs (spinlock) + * 2. It disables softirqs on the local CPU, preventing reentry from + * napi_poll() while running in process context + * + * Note: + * - We do NOT use spin_lock_irqsave() because there is no hard IRQ + * context accessing this ring (no real NIC / interrupt handler). + * - If this changes in the future (e.g., push from hard IRQ), this + * must be upgraded to spin_lock_irqsave(). + * + * Also note: + * - The ring itself is implemented as SPSC (lockless) between + * producer and consumer, using acquire/release barriers. + * - This lock only serializes multiple producers (MPSC -> SPSC). + */ + + spin_lock_bh(&rings->comp_lock); + pushed = comp_ring_push_many(rings->comp, addrs, n); + spin_unlock_bh(&rings->comp_lock); + + if (queue) { + RATTAN_DBG_ADD(queue, comp_ok, pushed); + if (pushed < n) + RATTAN_DBG_ADD(queue, comp_full, n - pushed); + } + + if (pushed < n) + pr_warn_ratelimited("rattan%d:q%u completion ring full, dropping %u chunks from 0x%llx\n", + queue ? queue->ctx->dev_id : -1, queue ? queue->qid : 0, n - pushed, + addrs[pushed]); + + return pushed; +} + +static bool rattan_complete_chunk(struct rattan_queue *queue, struct rattan_rings *rings, + u64 addr) { + return rattan_complete_chunks(queue, rings, &addr, 1) == 1; +} + +/* + * Per-NAPI-poll completion batching. + * + * Every forwarded packet returns its chunk to the COMP ring of the queue that + * supplied it (the owner, usually another device's queue on another CPU), and + * drops the owner reference its slot held. Doing that per packet bounces the + * owner's comp_lock/refcount line and COMP producer line between cores on + * every packet. Instead, NAPI collects addresses per destination rings here + * and pushes each group with one lock, one producer store and one + * refcount_sub. + * + * Kept small: this lives on the NAPI stack frame, underneath + * netif_receive_skb() and everything the stack does from there. + */ +#define RATTAN_COMP_GROUPS 2 +#define RATTAN_COMP_BATCH 32 + +struct rattan_comp_batch { + struct rattan_queue *queue; /* For statistics */ + struct { + struct rattan_rings *rings; + u32 n; /* Addresses in addrs[] */ + u32 refs; /* Slot references on rings to drop at flush */ + u64 addrs[RATTAN_COMP_BATCH]; + } g[RATTAN_COMP_GROUPS]; +}; + +static void rattan_comp_batch_init(struct rattan_comp_batch *b, struct rattan_queue *queue) { + int i; + + b->queue = queue; + for (i = 0; i < RATTAN_COMP_GROUPS; i++) { + b->g[i].rings = NULL; + b->g[i].n = 0; + b->g[i].refs = 0; + } +} + +static void rattan_comp_batch_flush_group(struct rattan_comp_batch *b, int i) { + struct rattan_rings *rings = b->g[i].rings; + + rattan_complete_chunks(b->queue, rings, b->g[i].addrs, b->g[i].n); + if (b->g[i].refs && refcount_sub_and_test(b->g[i].refs, &rings->refcount)) + queue_work(rattan_cleanup_wq, &rings->cleanup_work); + + b->g[i].rings = NULL; + b->g[i].n = 0; + b->g[i].refs = 0; +} + +static void rattan_comp_batch_flush(struct rattan_comp_batch *b) { + int i; + + for (i = 0; i < RATTAN_COMP_GROUPS; i++) + if (b->g[i].rings) + rattan_comp_batch_flush_group(b, i); +} + +/* Addresses queued for rings but not yet on its COMP ring */ +static u32 rattan_comp_batch_pending(struct rattan_comp_batch *b, struct rattan_rings *rings) { + int i; + + for (i = 0; i < RATTAN_COMP_GROUPS; i++) + if (b->g[i].rings == rings) + return b->g[i].n; + return 0; +} + +/* + * Queue addr for rings' COMP ring. With put_ref, the batch also takes over + * one reference on rings (the one a slot held) and drops it at flush; the + * caller must not touch rings after that unless it holds its own. + */ +static void rattan_comp_batch_add(struct rattan_comp_batch *b, struct rattan_rings *rings, + u64 addr, bool put_ref) { + int i, slot = -1; + + for (i = 0; i < RATTAN_COMP_GROUPS; i++) { + if (b->g[i].rings == rings) { + slot = i; + break; + } + if (!b->g[i].rings && slot < 0) + slot = i; + } + + if (slot < 0) { + /* Every group holds other rings: evict the last one */ + slot = RATTAN_COMP_GROUPS - 1; + rattan_comp_batch_flush_group(b, slot); + } else if (b->g[slot].n == RATTAN_COMP_BATCH) { + rattan_comp_batch_flush_group(b, slot); + } + + b->g[slot].rings = rings; + b->g[slot].addrs[b->g[slot].n++] = addr; + if (put_ref) + b->g[slot].refs++; +} + +/* ========== Ring Helper Functions ========== */ + +/* Check if ring is empty (producer == consumer) */ +static inline bool ring_is_empty(struct rattan_ring *ring) { + return READ_ONCE(ring->producer) == READ_ONCE(ring->consumer); +} + +/* Check if ring is full */ +static inline bool ring_is_full(struct rattan_ring *ring) { + u32 prod = READ_ONCE(ring->producer); + u32 cons = READ_ONCE(ring->consumer); + return ((prod - cons) >= RATTAN_RING_SIZE); +} + +/* Get number of entries in ring */ +static inline u32 ring_count(struct rattan_ring *ring) { + u32 prod = READ_ONCE(ring->producer); + u32 cons = READ_ONCE(ring->consumer); + return prod - cons; +} + +/* + * Pop chunk address from FILL ring (kernel is consumer) + * Returns true on success, false if ring empty + */ +static bool fill_ring_pop(struct rattan_fill_ring *fill, u64 *addr) { + u32 cons, prod; + + prod = READ_ONCE(fill->ring.producer); + cons = READ_ONCE(fill->ring.consumer); + + /* Memory barrier to ensure we see updated producer */ + smp_rmb(); + + if (cons == prod) + return false; /* Ring empty */ + + *addr = fill->addrs[cons & RATTAN_RING_MASK]; + + /* Memory barrier before updating consumer */ + smp_wmb(); + WRITE_ONCE(fill->ring.consumer, cons + 1); + + return true; +} + +/* + * Get a chunk address, trying COMP ring first, then FILL ring + * The recycle ring is protected by rings->recycle_lock. + * Returns true on success, false if both rings empty + */ +static bool try_get_free_addr(struct rattan_queue *queue, struct rattan_rings *rings, + struct rattan_umem *umem, u64 *addr, + struct rattan_xmit_trace *tr) { + bool ok; + + // As long as we only pop in `rattan_xmit_one`, we do not need to lock + // spin_lock_bh(&rings->comp_lock); + ok = comp_ring_pop(rings->comp, addr); + // spin_unlock_bh(&rings->comp_lock); + + if (rattan_recycle_drop_token(umem, rings->drop)) { + RATTAN_DBG_INC(queue, recycle_token); + if (tr) + tr->recycled = true; + } + + if (ok) + return true; + + return fill_ring_pop(rings->fill, addr); +} + +/* + * Check if RX ring has space for at least one entry + * Used for backpressure - check before consuming from FILL ring + */ +static bool rx_ring_has_space(struct rattan_rx_ring *rx) { + u32 cons, prod; + + prod = READ_ONCE(rx->ring.producer); + cons = READ_ONCE(rx->ring.consumer); + + return (prod - cons) < RATTAN_RING_SIZE; +} + +/* + * Push descriptor to RX ring (kernel is producer) + * Returns: + * < 0: error (ring full) + * 0: success + */ +static int rx_ring_push(struct rattan_rx_ring *rx, struct rattan_desc *desc) { + u32 cons, prod; + + prod = READ_ONCE(rx->ring.producer); + cons = READ_ONCE(rx->ring.consumer); + + if ((prod - cons) >= RATTAN_RING_SIZE) + return -ENOSPC; /* Ring full */ + + rx->descs[prod & RATTAN_RING_MASK] = *desc; + + /* Memory barrier before updating producer */ + smp_wmb(); + WRITE_ONCE(rx->ring.producer, prod + 1); + + return 0; +} + +/* + * Free entries in the COMP ring + * Used for backpressure - check before consuming from TX ring + */ +static u32 comp_ring_free(struct rattan_comp_ring *comp) { + u32 cons, prod; + + prod = READ_ONCE(comp->ring.producer); + cons = READ_ONCE(comp->ring.consumer); + + return RATTAN_RING_SIZE - (prod - cons); +} + +/* + * Push up to n chunk addresses to the COMP ring with a single producer + * update. Returns how many fit. + */ +static u32 comp_ring_push_many(struct rattan_comp_ring *comp, const u64 *addrs, u32 n) { + u32 cons, prod, i; + + prod = READ_ONCE(comp->ring.producer); + // Make sure that we see the updated value + cons = smp_load_acquire(&comp->ring.consumer); + + n = min_t(u32, n, RATTAN_RING_SIZE - (prod - cons)); + for (i = 0; i < n; i++) + comp->addrs[(prod + i) & RATTAN_RING_MASK] = addrs[i]; + + if (n) + smp_store_release(&comp->ring.producer, prod + n); + + return n; +} + +/* + * Pop chunk address from COMP ring + * Returns true on success, false if ring empty + */ +static bool comp_ring_pop(struct rattan_comp_ring *comp, u64 *addr) { + u32 cons, prod; + + cons = READ_ONCE(comp->ring.consumer); + prod = smp_load_acquire(&comp->ring.producer); + + if (cons == prod) + return false; + + *addr = comp->addrs[cons & RATTAN_RING_MASK]; + + smp_store_release(&comp->ring.consumer, cons + 1); + + return true; +} + +/* + * Pop token from DROP ring, and release them. + */ +static bool rattan_recycle_drop_token(struct rattan_umem *umem, struct rattan_drop_ring *drop) { + u32 cons, prod; + u64 token; + + cons = READ_ONCE(drop->ring.consumer); + prod = smp_load_acquire(&drop->ring.producer); + + if (cons == prod) + return false; + + token = drop->tokens[cons & RATTAN_RING_MASK]; + + smp_store_release(&drop->ring.consumer, cons + 1); + + return release_token(umem, token); +} + +/* + * Peek at next descriptor in TX ring without consuming + * Returns true if descriptor available, false if ring empty + */ +static bool tx_ring_peek(struct rattan_tx_ring *tx, struct rattan_desc *desc) { + u32 cons, prod; + + prod = READ_ONCE(tx->ring.producer); + cons = READ_ONCE(tx->ring.consumer); + + /* Memory barrier to ensure we see updated producer */ + smp_rmb(); + + if (cons == prod) + return false; /* Ring empty */ + + *desc = tx->descs[cons & RATTAN_RING_MASK]; + return true; +} + +/* + * Advance TX ring consumer after peeking + * Call this after tx_ring_peek() to consume the descriptor + */ +static void tx_ring_advance(struct rattan_tx_ring *tx) { + u32 cons = READ_ONCE(tx->ring.consumer); + + /* Memory barrier before updating consumer */ + smp_wmb(); + WRITE_ONCE(tx->ring.consumer, cons + 1); +} + +/* ========== Token/SKB Management ========== */ + +static long rattan_pending_count(void) { + long sum = 0; + int cpu; + + for_each_possible_cpu(cpu) + sum += per_cpu(rattan_pending_tokens, cpu); + return sum; +} + +static inline u64 rattan_chunk_addr(struct rattan_umem *umem, u32 idx) { + return (u64)idx << umem->chunk_shift; +} + +/* + * The UMEM of a device, from an ioctl on its fd. ctx->umem only goes from NULL + * to a UMEM in ioctls and back to NULL in fop_release, which cannot run while + * an ioctl on the same file is in flight, so no RCU read section is needed. + */ +static inline struct rattan_umem *rattan_ctx_umem(struct rattan_vnic_ctx *ctx) { + return rcu_dereference_check(ctx->umem, 1); +} + +/* + * Bind skb to the (just obtained) chunk idx: FREE(g) -> BUSY(g) -> LIVE(g+1). + * The slot takes over the caller's reference on rings. + * + * Returns the new token, or 0 if the slot is not free, i.e. userspace handed + * out a chunk that is still bound to a live token. + */ +static u64 rattan_slot_insert(struct rattan_umem *umem, u32 idx, struct sk_buff *skb, + struct rattan_rings *rings, u16 qid) { + struct rattan_pending *p; + u64 old; + u32 gen; + + /* Callers validate the chunk first; this guards the array regardless */ + if (WARN_ON_ONCE(idx >= umem->nr_chunks)) + return 0; + + p = &umem->pending[idx]; + old = READ_ONCE(p->state); + if ((old & RATTAN_SLOT_TAG_MASK) != RATTAN_SLOT_FREE || + cmpxchg(&p->state, old, old | RATTAN_SLOT_BUSY) != old) + return 0; + + gen = RATTAN_SLOT_GEN(old) + 1; + if (unlikely(!gen)) + gen = 1; + + p->skb = skb; + p->rings = rings; + p->stamp = (u32)jiffies; + p->qid = qid; + + /* Publish the fields together with LIVE */ + smp_store_release(&p->state, RATTAN_SLOT_STATE(gen, RATTAN_SLOT_LIVE)); + this_cpu_inc(rattan_pending_tokens); + + return RATTAN_TOKEN(idx, gen); +} + +/* + * Claim the slot of a live token: LIVE(gen) -> BUSY(gen). The one cmpxchg + * checks liveness and generation together, so a stale, replayed or + * concurrently claimed token misses. Returns NULL on a miss. + * + * The caller must end the claim with rattan_slot_take() or + * rattan_slot_unclaim() without sleeping; process-context callers disable BH + * so that NAPI on the same CPU never finds the slot BUSY for long. + */ +static struct rattan_pending *rattan_slot_claim(struct rattan_umem *umem, u64 token) { + u32 idx = RATTAN_TOKEN_IDX(token); + u32 gen = RATTAN_TOKEN_GEN(token); + u64 live = RATTAN_SLOT_STATE(gen, RATTAN_SLOT_LIVE); + struct rattan_pending *p; + + if (unlikely(idx >= umem->nr_chunks)) + return NULL; + + p = &umem->pending[idx]; + if (cmpxchg(&p->state, live, RATTAN_SLOT_STATE(gen, RATTAN_SLOT_BUSY)) != live) + return NULL; + + return p; +} + +/* BUSY(gen) -> LIVE(gen): hand a claimed slot back unchanged */ +static inline void rattan_slot_unclaim(struct rattan_pending *p, u64 token) { + smp_store_release(&p->state, RATTAN_SLOT_STATE(RATTAN_TOKEN_GEN(token), RATTAN_SLOT_LIVE)); +} + +/* + * BUSY(gen) -> FREE(gen): empty a claimed slot. The caller inherits the skb + * and the rings reference, and still owes the chunk a completion. + */ +static struct sk_buff *rattan_slot_take(struct rattan_pending *p, u64 token, + struct rattan_rings **rings) { + struct sk_buff *skb = p->skb; + + *rings = p->rings; + p->skb = NULL; + p->rings = NULL; + + /* The reads above must complete before the slot can be reused */ + smp_store_release(&p->state, RATTAN_SLOT_STATE(RATTAN_TOKEN_GEN(token), RATTAN_SLOT_FREE)); + this_cpu_dec(rattan_pending_tokens); + + return skb; +} + +/* Claim and empty a token's slot in one go. Returns NULL on a miss. */ +static struct sk_buff *rattan_token_take(struct rattan_umem *umem, u64 token, + struct rattan_rings **rings) { + struct rattan_pending *p = rattan_slot_claim(umem, token); + + return p ? rattan_slot_take(p, token, rings) : NULL; +} + +/* + * Finish releasing a taken token: return its chunk to the owning COMP ring and + * free the SKB. + */ +static void rattan_token_drop(struct rattan_umem *umem, u64 token, struct sk_buff *skb, + struct rattan_rings *rings) { + /* The queue may have been released here, if the fd has been closed, + * since the pointer to the queue is used for statistics and debugging only, + * we pass NULL to avoid possible use-after-free. + */ + rattan_complete_chunk(NULL, rings, rattan_chunk_addr(umem, RATTAN_TOKEN_IDX(token))); + rattan_put_rings(rings); + dev_kfree_skb_any(skb); +} + +/* + * Release token - free the SKB associated with a token and return its chunk + * to the owning COMP ring. Softirq context (the DROP ring); process context + * uses rattan_release_token_process(). + */ +static bool release_token(struct rattan_umem *umem, u64 token) { + struct rattan_rings *rings; + struct sk_buff *skb; + + skb = rattan_token_take(umem, token, &rings); + if (!skb) + return false; + + rattan_token_drop(umem, token, skb, rings); + return true; +} + +/* + * release_token() for process context. BH is disabled only while the slot is + * claimed, so NAPI on this CPU never finds it BUSY for longer than that. + */ +static bool rattan_release_token_process(struct rattan_umem *umem, u64 token) { + struct rattan_rings *rings; + struct sk_buff *skb; + + local_bh_disable(); + skb = rattan_token_take(umem, token, &rings); + local_bh_enable(); + if (!skb) + return false; + + rattan_token_drop(umem, token, skb, rings); + return true; +} + +/* + * Free whatever is still pending on a UMEM that has no users left. Nothing + * can race with this: every device that used the UMEM has unpublished it and + * waited for RCU, and it is off rattan_umems so GC cannot see it. The chunks + * are not completed, since the UMEM itself is going away. + */ +static void rattan_umem_drain(struct rattan_umem *umem) { + u32 idx, leaked = 0; + + for (idx = 0; idx < umem->nr_chunks; idx++) { + struct rattan_pending *p = &umem->pending[idx]; + + if ((p->state & RATTAN_SLOT_TAG_MASK) != RATTAN_SLOT_LIVE) + continue; + + rattan_put_rings(p->rings); + dev_kfree_skb_any(p->skb); + this_cpu_dec(rattan_pending_tokens); + leaked++; + } + + if (leaked) + pr_info("released %u pending SKBs with their UMEM\n", leaked); +} + +/* + * Read payload from tracked SKB + * The token resolves through this device's UMEM, so it must have been minted + * on a device sharing it. + */ +static int rattan_read_payload(struct rattan_vnic_ctx *ctx, + struct rattan_payload_req __user *argp) { + struct rattan_payload_req req; + struct rattan_payload_resp resp; + struct rattan_umem *umem; + struct rattan_pending *p; + struct sk_buff *skb; + void *kbuf; + u32 copy_len; + int err; + + if (copy_from_user(&req, argp, sizeof(req))) + return -EFAULT; + + umem = rattan_ctx_umem(ctx); + if (!umem) + return -ENOENT; + + local_bh_disable(); + p = rattan_slot_claim(umem, req.token); + if (!p) { + local_bh_enable(); + return -ENOENT; + } + + skb = p->skb; + resp.total_len = skb->len; + + /* Calculate how much to copy */ + if (req.offset >= skb->len) { + rattan_slot_unclaim(p, req.token); + local_bh_enable(); + copy_len = 0; + resp.len = 0; + goto write_resp; + } + + copy_len = min_t(u32, req.len, skb->len - req.offset); + resp.len = copy_len; + + /* Take reference and give the slot back */ + skb_get(skb); + rattan_slot_unclaim(p, req.token); + local_bh_enable(); + + /* Allocate kernel buffer for copy */ + kbuf = kmalloc(copy_len, GFP_KERNEL); + if (!kbuf) { + kfree_skb(skb); + return -ENOMEM; + } + + /* Copy from SKB to kernel buffer */ + err = skb_copy_bits(skb, req.offset, kbuf, copy_len); + kfree_skb(skb); + + if (err) { + kfree(kbuf); + return err; + } + + /* Copy to userspace */ + if (copy_to_user((void __user *)req.buf, kbuf, copy_len)) { + kfree(kbuf); + return -EFAULT; + } + + kfree(kbuf); + +write_resp: + /* Write response */ + if (copy_to_user(argp, &resp, sizeof(resp))) + return -EFAULT; + + return 0; +} + +/* + * Release token, ioctl handler + */ +static int rattan_release_token(struct rattan_vnic_ctx *ctx, + struct rattan_release_req __user *argp) { + struct rattan_release_req req; + struct rattan_umem *umem; + bool ok; + + if (copy_from_user(&req, argp, sizeof(req))) + return -EFAULT; + + umem = rattan_ctx_umem(ctx); + if (!umem) + return -ENOENT; + + ok = rattan_release_token_process(umem, req.token); + + return ok ? 0 : -ENOENT; +} + +/* + * Clone a token - bind a copy of its SKB to a chunk supplied by userspace + */ +static int rattan_clone_token(struct rattan_vnic_ctx *ctx, struct rattan_clone_req __user *argp) { + struct rattan_clone_req req; + struct rattan_clone_resp resp; + struct rattan_umem *umem; + struct rattan_queue *queue; + struct rattan_rings *rings, *old_rings; + struct rattan_pending *p; + struct sk_buff *new_skb, *old_skb; + u64 new_token; + int err; + + if (copy_from_user(&req, argp, sizeof(req))) + return -EFAULT; + + umem = rattan_ctx_umem(ctx); + queue = rattan_get_queue(ctx, req.queue_id); + if (!umem || !queue) + return -EINVAL; + + if (!rattan_chunk_valid(umem, req.chunk_addr)) + return -EINVAL; + + /* The new slot owns this reference */ + rcu_read_lock(); + rings = rattan_get_rings(queue); + rcu_read_unlock(); + if (!rings) + return -EINVAL; + + local_bh_disable(); + p = rattan_slot_claim(umem, req.token); + if (!p) { + err = -ENOENT; + goto out_unlock; + } + + /* Copy the SKB (not clone - we need independent data for modifications) */ + new_skb = skb_copy(p->skb, GFP_ATOMIC); + rattan_slot_unclaim(p, req.token); + if (!new_skb) { + err = -ENOMEM; + goto out_unlock; + } + + new_token = rattan_slot_insert(umem, req.chunk_addr >> umem->chunk_shift, new_skb, rings, + queue->qid); + if (!new_token) { + kfree_skb(new_skb); + err = -EBUSY; + goto out_unlock; + } + local_bh_enable(); + + resp.new_token = new_token; + if (copy_to_user(argp, &resp, sizeof(resp))) { + /* + * Userspace never learns the token, so undo the insert. The chunk + * stays with userspace, which still owns it as far as it knows. + */ + local_bh_disable(); + old_skb = rattan_token_take(umem, new_token, &old_rings); + local_bh_enable(); + if (old_skb) { + rattan_put_rings(old_rings); + kfree_skb(old_skb); + } + return -EFAULT; + } + + return 0; + +out_unlock: + local_bh_enable(); + rattan_put_rings(rings); + return err; +} + +static int rattan_device_id(struct rattan_vnic_ctx *ctx, void __user *argp) { + int dev_id = ctx->dev_id; + if (copy_to_user(argp, &dev_id, sizeof(dev_id))) { + return -EFAULT; + } + return 0; +} + +static int rattan_napi_poll(struct napi_struct *napi, int budget) { + struct rattan_queue *queue = container_of(napi, struct rattan_queue, napi); + struct rattan_vnic_ctx *ctx = queue->ctx; + struct rattan_rings *rings; + struct rattan_umem *umem; + struct net_device *dev = ctx->netdev; + struct rattan_desc desc; + struct rattan_pending *p; + struct rattan_rings *owner; + struct sk_buff *skb; + u64 chunk_addr; + int processed = 0; + int pkt_len; + struct rattan_comp_batch comp; + struct rattan_napi_trace trace; + struct rattan_napi_trace *tr = NULL; + + RATTAN_DBG_INC(queue, napi_polls); + rattan_debug_dump_queue_stats(queue); + + rcu_read_lock(); + + if (!ctx->started) { + rcu_read_unlock(); + goto complete; + } + + rings = rcu_dereference(queue->rings); + umem = rcu_dereference(ctx->umem); + if (!rings || !umem) { + rcu_read_unlock(); + goto complete; + } + + rattan_comp_batch_init(&comp, queue); + + while (processed < budget) { + if (slow_rx_ns) { + tr = &trace; + rattan_napi_trace_begin(tr); + } + + /* + * Backpressure: queue-local COMP ring must have space, counting what + * this poll has batched for it but not pushed yet. + */ + if (comp_ring_free(rings->comp) <= rattan_comp_batch_pending(&comp, rings)) { + RATTAN_DBG_INC(queue, napi_comp_full); + pr_warn_ratelimited("rattan%d:q%u NAPI blocked: COMP ring full\n", ctx->dev_id, + queue->qid); + break; + } + + /* Peek at next TX descriptor */ + if (!tx_ring_peek(rings->tx, &desc)) { + RATTAN_DBG_INC(queue, napi_tx_empty); + break; /* TX ring empty */ + } + NT(tr, NT_PEEK); + + /* Claim the token's pending slot */ + p = rattan_slot_claim(umem, desc.token); + if (!p) { + /* Token not found - consume descriptor, return TX chunk */ + tx_ring_advance(rings->tx); + RATTAN_DBG_INC(queue, napi_token_miss); + rattan_comp_batch_add(&comp, rings, desc.addr, false); + pr_warn_ratelimited("rattan%d:q%u NAPI token miss: 0x%llx\n", ctx->dev_id, queue->qid, + desc.token); + processed++; + NT(tr, NT_RELEASE); + rattan_report_slow_rx(tr, queue, "miss"); + continue; + } + NT(tr, NT_LOOKUP); + + /* Consume descriptor and empty the slot */ + tx_ring_advance(rings->tx); + skb = rattan_slot_take(p, desc.token, &owner); + NT(tr, NT_UNLINK); + + /* + * Copy modified header back to SKB. Compiled out while + * RATTAN_HEADER_SIZE is 0, and not correct as it stands. Before + * raising it: + * - skb_ensure_writable(skb, copy_len) first: TCP transmits clones, + * so writing in place corrupts the sender's retransmit copy (and + * with sendfile, page-cache pages); + * - settle a checksum contract (userspace patches incrementally, + * RFC 1624, or the kernel fixes them up here); + * - the header size is part of the ABI: it is overwritten in place + * at offset 0. + */ + if (RATTAN_HEADER_SIZE && desc.len > 0) { + if (rattan_chunk_valid(umem, desc.addr) && + umem->headroom + desc.len <= umem->chunk_size) { + void *header_ptr = (u8 *)umem->mapped + desc.addr + umem->headroom; + u32 copy_len = min3(desc.len, skb->len, (u32)RATTAN_HEADER_SIZE); + skb_store_bits(skb, 0, header_ptr, copy_len); + } + } + + /* Record length and prepare SKB for injection */ + pkt_len = skb->len; + skb->dev = dev; + skb_record_rx_queue(skb, queue->qid); + skb->protocol = eth_type_trans(skb, dev); + NT(tr, NT_PREP); + + /* + * Use netif_receive_skb() instead of netif_rx() since we're + * already in NAPI/softirq context on the target CPU. + */ + if (netif_receive_skb(skb) == NET_RX_SUCCESS) { + RATTAN_DBG_INC(queue, napi_rx_ok); + dev_sw_netstats_rx_add(dev, pkt_len); + } else { + RATTAN_DBG_INC(queue, napi_rx_drop); + rattan_rx_dropped_inc(dev); + } + NT(tr, NT_STACK); + + /* + * The token names the original chunk; desc.addr is whichever chunk + * userspace sent back, which may differ. Both go home. + */ + chunk_addr = rattan_chunk_addr(umem, RATTAN_TOKEN_IDX(desc.token)); + if (desc.addr != chunk_addr) + rattan_comp_batch_add(&comp, rings, desc.addr, false); + /* Hands the slot's owner reference to the batch */ + rattan_comp_batch_add(&comp, owner, chunk_addr, true); + processed++; + NT(tr, NT_RELEASE); + rattan_report_slow_rx(tr, queue, "ok"); + } + + /* Before rcu_read_unlock(): groups for rings hold no reference */ + rattan_comp_batch_flush(&comp); + rcu_read_unlock(); + + if (processed < budget) { + complete: + /* All work done - exit polling mode */ + RATTAN_DBG_INC(queue, napi_complete); + napi_complete_done(napi, processed); + atomic_set(&queue->napi_scheduled, 0); + } else { + RATTAN_DBG_INC(queue, napi_budget_stop); + } + + return processed; +} + +/* + * IPI callback to schedule NAPI on target CPU + */ +static void rattan_schedule_napi_ipi(void *data) { + struct rattan_queue *queue = data; + + if (napi_schedule_prep(&queue->napi)) + __napi_schedule(&queue->napi); +} + +/* + * Inject packets into kernel receive path via NAPI + * + * Always uses NAPI for packet injection. If napi_cpu is not set (-1), + * schedules NAPI on the current CPU. Otherwise schedules on the + * configured target CPU via IPI if needed. + */ +static int rattan_kick_rx(struct rattan_queue *queue) { + int target_cpu; + int current_cpu; + + if (!queue || !queue->ctx->started) + return -EINVAL; + + RATTAN_DBG_INC(queue, kick_rx); + rattan_debug_dump_queue_stats(queue); + + current_cpu = get_cpu(); + target_cpu = READ_ONCE(queue->napi_cpu); + + /* If napi_cpu not configured, use current CPU */ + if (target_cpu < 0) + target_cpu = current_cpu; + + /* Validate target CPU is still online, fallback to current */ + if (!cpu_online(target_cpu)) + target_cpu = current_cpu; + + if (target_cpu == current_cpu) { + /* Target is current CPU - schedule directly */ + if (napi_schedule_prep(&queue->napi)) + __napi_schedule(&queue->napi); + put_cpu(); + } else { + put_cpu(); + + /* + * Target is remote CPU - send IPI. + * Use napi_scheduled to avoid unnecessary IPIs when NAPI + * is already scheduled. The IPI callback will do its own + * napi_schedule_prep() check, but avoiding the IPI is cheaper. + * + * Per-queue CPU affinity can be customized further later if + * custom steering is introduced. + */ + if (atomic_cmpxchg(&queue->napi_scheduled, 0, 1) == 0) { + RATTAN_DBG_INC(queue, kick_rx_remote); + smp_call_function_single(target_cpu, rattan_schedule_napi_ipi, queue, 0); + } + } + + return 0; +} + +/* + * Internal GC helper - releases tokens older than timeout_ms on every UMEM + * + * A linear scan of each UMEM's pending slots. Each release is an ordinary + * claim, so a token that is concurrently transmitted or released is simply + * skipped (or makes the other side miss), never freed twice. + * + * Returns number of tokens collected. + */ +static u32 rattan_do_gc(u32 timeout_ms) { + u32 timeout = msecs_to_jiffies(timeout_ms); + struct rattan_umem *umem; + u32 collected = 0; + u32 idx; + + mutex_lock(&rattan_umems_lock); + list_for_each_entry(umem, &rattan_umems, node) { + for (idx = 0; idx < umem->nr_chunks; idx++) { + struct rattan_pending *p = &umem->pending[idx]; + u64 state = READ_ONCE(p->state); + + if ((state & RATTAN_SLOT_TAG_MASK) != RATTAN_SLOT_LIVE) + continue; + if ((u32)jiffies - READ_ONCE(p->stamp) < timeout) + continue; + + if (rattan_release_token_process(umem, RATTAN_TOKEN(idx, RATTAN_SLOT_GEN(state)))) + collected++; + + if ((idx & 4095) == 0) + cond_resched(); + } + } + mutex_unlock(&rattan_umems_lock); + + return collected; +} + +/* + * Periodic GC work function + * + * Runs every RATTAN_GC_INTERVAL_MS (10 minutes) to clean up orphaned tokens. + * This handles cases where userspace dropped packets without forwarding them. + */ +static void rattan_gc_work_fn(struct work_struct *work) { + u32 collected; + + collected = rattan_do_gc(RATTAN_TOKEN_TIMEOUT_MS); + + if (collected > 0) + pr_info("periodic GC: collected %u orphaned tokens, %ld remaining\n", collected, + rattan_pending_count()); + + /* Reschedule for next interval */ + schedule_delayed_work(&gc_work, msecs_to_jiffies(RATTAN_GC_INTERVAL_MS)); +} + +/* ========== IOCTL Handlers ========== */ + +/* + * UMEM registration + */ +static int rattan_reg_umem(struct rattan_vnic_ctx *ctx, struct rattan_umem_reg __user *argp) { + struct rattan_umem_reg params; + struct rattan_umem *umem; + unsigned long nr_pages; + int ret; + + if (rcu_access_pointer(ctx->umem)) + return -EBUSY; + + if (copy_from_user(¶ms, argp, sizeof(params))) + return -EFAULT; + + /* Validate parameters */ + if (params.len > RATTAN_MAX_UMEM_SIZE) { + pr_warn("UMEM len %llu exceeds max %llu\n", params.len, (u64)RATTAN_MAX_UMEM_SIZE); + return -EINVAL; + } + + if (params.chunk_size < RATTAN_MIN_CHUNK_SIZE || params.chunk_size > RATTAN_MAX_CHUNK_SIZE) + return -EINVAL; + + /* chunk_size must be power of 2 for efficient address validation */ + if (!is_power_of_2(params.chunk_size)) { + pr_warn("chunk_size %u must be power of 2\n", params.chunk_size); + return -EINVAL; + } + + /* headroom + max header size must fit in chunk */ + if (params.headroom + RATTAN_HEADER_SIZE > params.chunk_size) { + pr_warn("headroom %u + header %u > chunk_size %u\n", params.headroom, RATTAN_HEADER_SIZE, + params.chunk_size); + return -EINVAL; + } + + if (params.len < params.chunk_size) + return -EINVAL; + + /* UMEM length must be multiple of chunk_size (no partial chunks) */ + if (params.len % params.chunk_size != 0) { + pr_warn("UMEM len %llu must be multiple of chunk_size %u\n", params.len, params.chunk_size); + return -EINVAL; + } + + /* One pending slot per chunk, so the chunk count bounds its memory */ + if (params.len / params.chunk_size > RATTAN_MAX_CHUNKS) { + pr_warn("UMEM has %llu chunks, max %u\n", params.len / params.chunk_size, + RATTAN_MAX_CHUNKS); + return -EINVAL; + } + + /* Userspace address must be page-aligned for correct vmap mapping */ + if (params.addr & (PAGE_SIZE - 1)) { + pr_warn("UMEM addr 0x%llx not page-aligned\n", params.addr); + return -EINVAL; + } + + umem = kzalloc(sizeof(*umem), GFP_KERNEL); + if (!umem) + return -ENOMEM; + + umem->len = params.len; + umem->chunk_size = params.chunk_size; + umem->chunk_shift = ilog2(params.chunk_size); + umem->headroom = params.headroom; + umem->nr_chunks = params.len / params.chunk_size; + INIT_LIST_HEAD(&umem->node); + + umem->pending = vzalloc(array_size(umem->nr_chunks, sizeof(*umem->pending))); + if (!umem->pending) { + ret = -ENOMEM; + goto err_free; + } + + nr_pages = (params.len + PAGE_SIZE - 1) / PAGE_SIZE; + umem->pages = kvmalloc_array(nr_pages, sizeof(struct page *), GFP_KERNEL); + if (!umem->pages) { + ret = -ENOMEM; + goto err_free; + } + + ret = rattan_pin_pages(params.addr, nr_pages, FOLL_WRITE, umem->pages); + if (ret < 0) + goto err_free; + + if (ret != nr_pages) { + rattan_unpin_pages(umem->pages, ret); + ret = -ENOMEM; + goto err_free; + } + + umem->nr_pages = nr_pages; + + umem->mapped = vmap(umem->pages, nr_pages, VM_MAP, PAGE_KERNEL); + if (!umem->mapped) { + rattan_unpin_pages(umem->pages, nr_pages); + ret = -ENOMEM; + goto err_free; + } + + /* Initialize refcount to 1 (this device owns it) */ + refcount_set(&umem->refcount, 1); + + mutex_lock(&rattan_umems_lock); + list_add(&umem->node, &rattan_umems); + mutex_unlock(&rattan_umems_lock); + + rcu_assign_pointer(ctx->umem, umem); + pr_info("rattan%d: UMEM registered: %u chunks of %u bytes (headroom=%u)\n", ctx->dev_id, + umem->nr_chunks, umem->chunk_size, umem->headroom); + + return 0; + +err_free: + kvfree(umem->pages); + vfree(umem->pending); + kfree(umem); + return ret; +} + +/* + * Share UMEM from another device + * + * Instead of registering new UMEM, share an existing one from another device. + * This enables zero-copy forwarding between devices. + */ +static int rattan_share_umem(struct rattan_vnic_ctx *ctx, + struct rattan_share_umem_req __user *argp) { + struct rattan_share_umem_req req; + struct rattan_vnic_ctx *source_ctx; + struct rattan_umem *umem; + struct file *source_file; + + if (rcu_access_pointer(ctx->umem)) + return -EBUSY; /* Already have UMEM */ + + if (copy_from_user(&req, argp, sizeof(req))) + return -EFAULT; + + /* Get the source file from fd */ + source_file = fget(req.source_fd); + if (!source_file) + return -EBADF; + + /* Verify it's a rattan-vnic file */ + if (source_file->f_op != &rattan_vnic_fops) { + fput(source_file); + return -EINVAL; + } + + source_ctx = source_file->private_data; + if (!source_ctx) { + fput(source_file); + return -EINVAL; + } + + umem = rcu_access_pointer(source_ctx->umem); + if (!umem) { + fput(source_file); + return -EINVAL; /* Source has no UMEM */ + } + + /* Increment refcount */ + refcount_inc(&umem->refcount); + + rcu_assign_pointer(ctx->umem, umem); + fput(source_file); + + pr_info("rattan%d: sharing UMEM from rattan%d (refcount=%u)\n", ctx->dev_id, source_ctx->dev_id, + refcount_read(&umem->refcount)); + + return 0; +} + +/* + * Rings registration + */ +static int rattan_reg_rings_queue(struct rattan_vnic_ctx *ctx, u32 qid, u64 addr, u64 len) { + struct rattan_queue *queue; + struct rattan_rings *rings; + unsigned long nr_pages; + int ret; + + queue = rattan_get_queue(ctx, qid); + if (!queue) + return -EINVAL; + + if (rcu_access_pointer(queue->rings)) + return -EBUSY; + + if (!rcu_access_pointer(ctx->umem)) + return -EINVAL; /* UMEM must be registered first */ + + /* Validate size */ + if (len < RATTAN_RINGS_SIZE) { + pr_warn("rings len %llu too small (need %zu)\n", len, (size_t)RATTAN_RINGS_SIZE); + return -EINVAL; + } + + /* Userspace address must be page-aligned for correct vmap mapping */ + if (addr & (PAGE_SIZE - 1)) { + pr_warn("rings addr 0x%llx not page-aligned\n", addr); + return -EINVAL; + } + + rings = kzalloc(sizeof(*rings), GFP_KERNEL); + if (!rings) + return -ENOMEM; + + refcount_set(&rings->refcount, 1); + spin_lock_init(&rings->comp_lock); + INIT_WORK(&rings->cleanup_work, rattan_cleanup_rings_work); + rings->len = len; + + nr_pages = (len + PAGE_SIZE - 1) / PAGE_SIZE; + rings->pages = kvmalloc_array(nr_pages, sizeof(struct page *), GFP_KERNEL); + if (!rings->pages) { + kfree(rings); + return -ENOMEM; + } + + ret = rattan_pin_pages(addr, nr_pages, FOLL_WRITE, rings->pages); + if (ret < 0) { + kvfree(rings->pages); + kfree(rings); + return ret; + } + + if (ret != nr_pages) { + rattan_unpin_pages(rings->pages, ret); + kvfree(rings->pages); + kfree(rings); + return -ENOMEM; + } + + rings->nr_pages = nr_pages; + + rings->mapped = vmap(rings->pages, nr_pages, VM_MAP, PAGE_KERNEL); + if (!rings->mapped) { + rattan_unpin_pages(rings->pages, nr_pages); + kvfree(rings->pages); + kfree(rings); + return -ENOMEM; + } + + /* Set up ring pointers */ + rings->rx = (struct rattan_rx_ring *)((char *)rings->mapped + RATTAN_RX_RING_OFFSET); + rings->tx = (struct rattan_tx_ring *)((char *)rings->mapped + RATTAN_TX_RING_OFFSET); + rings->fill = (struct rattan_fill_ring *)((char *)rings->mapped + RATTAN_FILL_RING_OFFSET); + rings->comp = (struct rattan_comp_ring *)((char *)rings->mapped + RATTAN_COMP_RING_OFFSET); + rings->drop = (struct rattan_drop_ring *)((char *)rings->mapped + RATTAN_DROP_RING_OFFSET); + + /* Allocate kernel-private recycle ring */ + // rings->recycle = kzalloc(sizeof(struct rattan_comp_ring), GFP_KERNEL); + // if (!rings->recycle) { + // vunmap(rings->mapped); + // rattan_unpin_pages(rings->pages, rings->nr_pages); + // kvfree(rings->pages); + // kfree(rings); + // return -ENOMEM; + // } + // /* Initialize producer/consumer to zero */ + // rings->recycle->ring.producer = 0; + // rings->recycle->ring.consumer = 0; + + rcu_assign_pointer(queue->rings, rings); + pr_info("rattan%d:q%u rings registered: rx=%p tx=%p fill=%p comp=%p drop=%p\n", ctx->dev_id, + qid, rings->rx, rings->tx, rings->fill, rings->comp, rings->drop); + + return 0; +} + +static int rattan_reg_rings(struct rattan_vnic_ctx *ctx, struct rattan_rings_reg __user *argp) { + struct rattan_rings_reg params; + + if (copy_from_user(¶ms, argp, sizeof(params))) + return -EFAULT; + + return rattan_reg_rings_queue(ctx, 0, params.addr, params.len); +} + +/* Sum a queue's per-cpu debug counters. Racy snapshot, fine for a dump. */ +static void rattan_queue_dbg_sum(struct rattan_queue *queue, struct rattan_queue_dbg *sum) { + const size_t n = sizeof(*sum) / sizeof(u64); + u64 *dst = (u64 *)sum; + int cpu; + size_t i; + + BUILD_BUG_ON(sizeof(*sum) % sizeof(u64)); + memset(sum, 0, sizeof(*sum)); + for_each_possible_cpu(cpu) { + const u64 *src = (const u64 *)per_cpu_ptr(queue->dbg, cpu); + + for (i = 0; i < n; i++) + dst[i] += READ_ONCE(src[i]); + } +} + +static void rattan_debug_dump_queue_stats(struct rattan_queue *queue) { + struct rattan_queue_dbg d; + unsigned long interval; + + if (!debug_stats_interval_ms || !queue) + return; + + interval = msecs_to_jiffies(debug_stats_interval_ms); + if (!interval) + interval = 1; + + if (!time_after_eq(jiffies, queue->dbg_last_dump_jiffies + interval)) + return; + + queue->dbg_last_dump_jiffies = jiffies; + + rattan_queue_dbg_sum(queue, &d); + + pr_info("rattan%d:q%u stats: xmit_ok=%llu rx_full=%llu fill_empty=%llu nomem=%llu " + "invalid_addr=%llu notify=%llu kick_rx=%llu kick_rx_remote=%llu napi_polls=%llu " + "napi_complete=%llu napi_budget_stop=%llu napi_tx_empty=%llu napi_comp_full=%llu " + "napi_token_miss=%llu rx_ok=%llu rx_drop=%llu comp_ok=%llu comp_full=%llu " + "recycled_token=%llu pending_tokens=%ld\n", + queue->ctx->dev_id, queue->qid, d.xmit_ok, d.xmit_rx_full, + d.xmit_fill_empty, d.xmit_nomem, d.xmit_invalid_addr, d.rx_notify, d.kick_rx, + d.kick_rx_remote, d.napi_polls, d.napi_complete, d.napi_budget_stop, d.napi_tx_empty, + d.napi_comp_full, d.napi_token_miss, d.napi_rx_ok, d.napi_rx_drop, d.comp_ok, + d.comp_full, d.recycle_token, rattan_pending_count()); + + pr_info("rattan%d:q%u ring: R%u T%u C%u F%u D%u", queue->ctx->dev_id, queue->qid, + ring_count(&queue->rings->rx->ring), ring_count(&queue->rings->tx->ring), + ring_count(&queue->rings->comp->ring), ring_count(&queue->rings->fill->ring), + ring_count(&queue->rings->drop->ring)); +} + +/* ========== File Operations ========== */ + +static long rattan_vnic_ioctl(struct file *file, unsigned int cmd, unsigned long arg) { + struct rattan_vnic_ctx *ctx = file->private_data; + void __user *argp = (void __user *)arg; + + if (!ctx || !ctx->netdev) + return -ENODEV; + + switch (cmd) { + case RATTAN_VNIC_REG_UMEM: + return rattan_reg_umem(ctx, argp); + + case RATTAN_VNIC_SHARE_UMEM: + return rattan_share_umem(ctx, argp); + + case RATTAN_VNIC_REG_RINGS: + return rattan_reg_rings(ctx, argp); + + case RATTAN_VNIC_REG_RINGS_Q: { + struct rattan_rings_reg_q params; + + if (copy_from_user(¶ms, argp, sizeof(params))) + return -EFAULT; + + return rattan_reg_rings_queue(ctx, params.queue_id, params.addr, params.len); + } + + case RATTAN_VNIC_START: { + struct rattan_start_config config; + u16 qid; + + if (!rcu_access_pointer(ctx->umem)) + return -EINVAL; + + for (qid = 0; qid < ctx->num_queues; qid++) { + if (!rcu_access_pointer(ctx->queues[qid].rings)) + return -EINVAL; + } + + if (copy_from_user(&config, argp, sizeof(config))) + return -EFAULT; + + /* Validate CPU number if specified */ + if (config.napi_cpu >= 0) { + if (config.napi_cpu >= nr_cpu_ids) + return -EINVAL; + if (!cpu_online(config.napi_cpu)) + return -ENODEV; + } + + /* + * One NAPI CPU setting is applied to all queues for now. + * Per-queue CPU placement can be added later if custom steering + * or queue affinity control is introduced. + */ + for (qid = 0; qid < ctx->num_queues; qid++) { + ctx->queues[qid].napi_cpu = config.napi_cpu; + napi_enable(&ctx->queues[qid].napi); + } + + ctx->started = true; + netif_carrier_on(ctx->netdev); + pr_info("rattan%d: started (%u queues, napi_cpu=%d)\n", ctx->dev_id, ctx->num_queues, + config.napi_cpu); + return 0; + } + + case RATTAN_VNIC_STOP: { + u16 qid; + + ctx->started = false; + netif_carrier_off(ctx->netdev); + for (qid = 0; qid < ctx->num_queues; qid++) + napi_disable(&ctx->queues[qid].napi); + pr_info("rattan%d: stopped\n", ctx->dev_id); + return 0; + } + + case RATTAN_VNIC_READ_PAYLOAD: + return rattan_read_payload(ctx, argp); + + case RATTAN_VNIC_RELEASE: + return rattan_release_token(ctx, argp); + + case RATTAN_VNIC_KICK_RX: + return rattan_kick_rx(rattan_get_queue(ctx, 0)); + + case RATTAN_VNIC_KICK_RX_Q: { + __u32 qid; + + if (copy_from_user(&qid, argp, sizeof(qid))) + return -EFAULT; + + return rattan_kick_rx(rattan_get_queue(ctx, qid)); + } + + case RATTAN_VNIC_CLONE: + return rattan_clone_token(ctx, argp); + + case RATTAN_VNIC_DEVICE_ID: { + return rattan_device_id(ctx, argp); + } + + default: + pr_debug("rattan%d: unknown ioctl 0x%x\n", ctx->dev_id, cmd); + return -ENOTTY; + } +} + +/* + * File operations - open + * Creates a new rattan network device + */ +static int rattan_vnic_fop_open(struct inode *inode, struct file *file) { + struct rattan_vnic_ctx *ctx; + struct net_device *netdev; + u16 qid, queues; + char name[IFNAMSIZ]; + int dev_id; + int err; + + /* Allocate device ID using IDA (reuses freed IDs) */ + dev_id = ida_alloc_max(&rattan_ida, max_devices - 1, GFP_KERNEL); + if (dev_id < 0) + return dev_id; + + queues = num_queues > 0 ? num_queues : 1; + + /* Create multiqueue network device with embedded ctx */ + snprintf(name, IFNAMSIZ, "rattan%d", dev_id); + netdev = alloc_netdev_mqs(sizeof(struct rattan_vnic_ctx), name, NET_NAME_UNKNOWN, + rattan_vnic_setup, queues, queues); + if (!netdev) { + err = -ENOMEM; + goto err_alloc_netdev; + } + + /* Initialize ctx (embedded in netdev priv area) */ + ctx = netdev_priv(netdev); + ctx->netdev = netdev; + ctx->dev_id = dev_id; + ctx->started = false; + ctx->num_queues = queues; + ctx->queues = kcalloc(ctx->num_queues, sizeof(*ctx->queues), GFP_KERNEL); + if (!ctx->queues) { + err = -ENOMEM; + goto err_alloc_queues; + } + + RCU_INIT_POINTER(ctx->umem, NULL); + init_waitqueue_head(&ctx->wait); + + for (qid = 0; qid < ctx->num_queues; qid++) { + struct rattan_queue *queue = &ctx->queues[qid]; + + queue->ctx = ctx; + queue->qid = qid; + RCU_INIT_POINTER(queue->rings, NULL); + init_waitqueue_head(&queue->wait); + + rattan_hrtimer_setup(&queue->rx_timer, rattan_rx_timer_cb, CLOCK_MONOTONIC, HRTIMER_MODE_REL_SOFT); + atomic_set(&queue->rx_pending, 0); + + queue->napi_cpu = -1; + atomic_set(&queue->napi_scheduled, 0); + queue->dbg = alloc_percpu(struct rattan_queue_dbg); + if (!queue->dbg) { + err = -ENOMEM; + goto err_register; + } + queue->dbg_last_dump_jiffies = jiffies; + rattan_netif_napi_add(netdev, &queue->napi, rattan_napi_poll); + } + + err = netif_set_real_num_tx_queues(netdev, ctx->num_queues); + if (err) + goto err_register; + + err = netif_set_real_num_rx_queues(netdev, ctx->num_queues); + if (err) + goto err_register; + + /* Register network device */ + err = register_netdev(netdev); + if (err) { + pr_err("failed to register netdev %s: %d\n", name, err); + goto err_register; + } + + file->private_data = ctx; + + pr_info("created %s with %u queues\n", name, ctx->num_queues); + return 0; + +err_register: + while (qid--) { + netif_napi_del(&ctx->queues[qid].napi); + free_percpu(ctx->queues[qid].dbg); + } + kfree(ctx->queues); +err_alloc_queues: + free_netdev(netdev); +err_alloc_netdev: + ida_free(&rattan_ida, dev_id); + return err; +} + +/* + * File operations - release + * Destroys the rattan network device + */ +static int rattan_vnic_fop_release(struct inode *inode, struct file *file) { + struct rattan_vnic_ctx *ctx = file->private_data; + struct rattan_umem *old_umem; + struct rattan_rings **old_rings; + u16 qid; + + if (!ctx) + return 0; + + if (ctx->netdev) { + int dev_id = ctx->dev_id; + + pr_info("destroying rattan%d\n", dev_id); + + if (ctx->started) { + ctx->started = false; + for (qid = 0; qid < ctx->num_queues; qid++) + napi_disable(&ctx->queues[qid].napi); + } + + for (qid = 0; qid < ctx->num_queues; qid++) { + hrtimer_cancel(&ctx->queues[qid].rx_timer); + netif_napi_del(&ctx->queues[qid].napi); + } + + old_rings = kcalloc(ctx->num_queues, sizeof(*old_rings), GFP_KERNEL); + if (!old_rings) + return -ENOMEM; + + old_umem = rcu_access_pointer(ctx->umem); + rcu_assign_pointer(ctx->umem, NULL); + + for (qid = 0; qid < ctx->num_queues; qid++) { + old_rings[qid] = rcu_access_pointer(ctx->queues[qid].rings); + rcu_assign_pointer(ctx->queues[qid].rings, NULL); + } + + synchronize_rcu(); + + /* + * Note: We don't clean up pending SKBs here. They can still be + * forwarded by other devices sharing the same UMEM, and are released + * together with the UMEM once its last user is gone. + */ + + for (qid = 0; qid < ctx->num_queues; qid++) + rattan_put_rings(old_rings[qid]); + + kfree(old_rings); + rattan_put_umem(old_umem); + + unregister_netdev(ctx->netdev); + for (qid = 0; qid < ctx->num_queues; qid++) + free_percpu(ctx->queues[qid].dbg); + kfree(ctx->queues); + free_netdev(ctx->netdev); + ida_free(&rattan_ida, dev_id); + } + + return 0; +} + +/* + * Poll implementation - allows userspace to block until RX data is available + * + * Returns: + * EPOLLIN | EPOLLRDNORM - RX ring has data (producer != consumer) + * EPOLLERR - device not configured + */ +static __poll_t rattan_vnic_poll(struct file *file, poll_table *wait) { + struct rattan_vnic_ctx *ctx = file->private_data; + __poll_t mask = 0; + bool configured = false; + u16 qid; + + if (!ctx) + return EPOLLERR; + + poll_wait(file, &ctx->wait, wait); + + rcu_read_lock(); + for (qid = 0; qid < ctx->num_queues; qid++) { + struct rattan_rings *rings; + u32 rx_prod, rx_cons; + + rings = rcu_dereference(ctx->queues[qid].rings); + if (!rings || !rings->rx) + continue; + + configured = true; + + /* Check RX ring: kernel produces, userspace consumes */ + rx_prod = smp_load_acquire(&rings->rx->ring.producer); + rx_cons = READ_ONCE(rings->rx->ring.consumer); + if (rx_prod != rx_cons) { + mask |= EPOLLIN | EPOLLRDNORM; + break; + } + } + rcu_read_unlock(); + + if (!configured) + mask = EPOLLERR; + + return mask; +} + +static const struct file_operations rattan_vnic_fops = { + .owner = THIS_MODULE, + .open = rattan_vnic_fop_open, + .release = rattan_vnic_fop_release, + .unlocked_ioctl = rattan_vnic_ioctl, + .poll = rattan_vnic_poll, +}; + +static struct miscdevice rattan_vnic_misc = { + .minor = MISC_DYNAMIC_MINOR, + .name = "rattan-vnic", + .fops = &rattan_vnic_fops, + .mode = 0666, +}; + +static int __init rattan_vnic_init(void) { + int err; + + rattan_cleanup_wq = alloc_workqueue("rattan_vnic_cleanup", 0, 0); + if (!rattan_cleanup_wq) + return -ENOMEM; + + err = misc_register(&rattan_vnic_misc); + if (err) { + pr_err("failed to register misc device: %d\n", err); + destroy_workqueue(rattan_cleanup_wq); + return err; + } + + /* Initialize periodic GC workqueue */ + INIT_DELAYED_WORK(&gc_work, rattan_gc_work_fn); + schedule_delayed_work(&gc_work, msecs_to_jiffies(RATTAN_GC_INTERVAL_MS)); + gc_work_initialized = true; + + pr_info("loaded (max_devices=%d, gc_interval=%ds)\n", max_devices, + RATTAN_GC_INTERVAL_MS / 1000); + return 0; +} + +static void __exit rattan_vnic_exit(void) { + /* Cancel periodic GC work before cleaning up */ + if (gc_work_initialized) { + cancel_delayed_work_sync(&gc_work); + gc_work_initialized = false; + } + + misc_deregister(&rattan_vnic_misc); + + /* + * Every fd is closed by now (the module is pinned while one is open), so + * every UMEM has been freed and its pending SKBs with it. + */ + WARN_ON(!list_empty(&rattan_umems)); + + /* Drains any rings cleanup still queued by the last puts */ + destroy_workqueue(rattan_cleanup_wq); + + pr_info("unloaded\n"); +} + +module_init(rattan_vnic_init); +module_exit(rattan_vnic_exit); + +MODULE_LICENSE("GPL"); +MODULE_AUTHOR("Rattan Project"); +MODULE_DESCRIPTION("Rattan Virtual NIC Driver"); +MODULE_VERSION(DRV_VERSION); diff --git a/rvnic/kernel/rattan_vnic.h b/rvnic/kernel/rattan_vnic.h new file mode 100644 index 00000000..1bb2f500 --- /dev/null +++ b/rvnic/kernel/rattan_vnic.h @@ -0,0 +1,206 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Rattan Virtual NIC - Kernel Internal Header + * + * Copyright (C) 2025 Rattan Project + * + * This header contains kernel-internal definitions. + * For the userspace API, see rattan_vnic_uapi.h. + */ + +#ifndef _RATTAN_VNIC_INTERNAL_H +#define _RATTAN_VNIC_INTERNAL_H + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "rattan_vnic_uapi.h" + +/* + * Driver identification + */ +#define DRV_NAME "rattan_vnic" +#define DRV_VERSION "0.1.0" + +/* + * Internal configuration constants + */ +#define RATTAN_TOKEN_TIMEOUT_MS 30000 /* Token timeout for GC (30 seconds) */ +#define RATTAN_GC_INTERVAL_MS 600000 /* GC workqueue interval (10 minutes) */ +#define RATTAN_RX_BATCH_SIZE 32 /* Wake up every N packets */ +#define RATTAN_RX_TIMEOUT_NS 64000 /* Or after N nanoseconds (64µs) */ + +/* + * Pending-packet slots - one per UMEM chunk + * + * Every packet handed to userspace is bound to exactly one UMEM chunk, so the + * chunk index doubles as the slot index and the token is simply + * + * token = gen << 32 | chunk_index + * + * where gen is a per-slot generation, bumped on every insert, that makes stale + * or replayed tokens miss instead of aliasing onto whatever packet occupies the + * chunk now. gen is never 0, so neither is a valid token. + * + * The slot state word packs gen with a FREE/LIVE/BUSY tag and every ownership + * change is a single cmpxchg on it. A correct userspace never makes two CPUs + * contend for one slot (the chunk's owner is also the slot's owner), but the + * device node is world-accessible, so a duplicated token or a live chunk put + * back on FILL must fail the cmpxchg rather than double-free the skb. + * + * FREE(g) --insert--> BUSY(g) --publish--> LIVE(g+1) + * LIVE(g) --claim---> BUSY(g) --release--> FREE(g) (or back to LIVE(g)) + * + * BUSY is only ever held for a few non-sleeping instructions, and a claimer that + * finds it treats the token as a miss rather than waiting. + * + * Chunks are returned to the COMP ring of the queue that originally supplied + * them (slot->rings), which preserves per-queue buffer ownership. + */ +#define RATTAN_SLOT_FREE 0 +#define RATTAN_SLOT_LIVE 1 +#define RATTAN_SLOT_BUSY 2 +#define RATTAN_SLOT_TAG_MASK 3 + +#define RATTAN_SLOT_STATE(gen, tag) (((u64)(u32)(gen) << 2) | (tag)) +#define RATTAN_SLOT_GEN(state) ((u32)((state) >> 2)) +#define RATTAN_TOKEN(idx, gen) (((u64)(u32)(gen) << 32) | (u32)(idx)) +#define RATTAN_TOKEN_IDX(token) ((u32)(token)) +#define RATTAN_TOKEN_GEN(token) ((u32)((token) >> 32)) + +struct rattan_rings; + +struct rattan_pending { + u64 state; /* RATTAN_SLOT_STATE(gen, tag) */ + struct sk_buff *skb; /* The actual packet */ + struct rattan_rings *rings; /* Rings that own completion for this chunk */ + u32 stamp; /* jiffies at insert, for GC */ + u16 qid; /* Queue that provided the chunk */ +}; + +/* + * UMEM context - tracks pinned userspace memory + * + * UMEM can be shared between multiple devices (AF_XDP style). + * The owner (first device to register) pins the pages and creates the mapping. + * Other devices can share via SHARE_UMEM ioctl, incrementing the refcount. + */ +struct rattan_umem { + refcount_t refcount; /* Number of devices using this UMEM */ + struct list_head node; /* On rattan_umems, for GC */ + struct page **pages; + unsigned long nr_pages; + void *mapped; /* Kernel virtual address */ + u64 len; /* Total length */ + u32 chunk_size; /* Chunk size */ + u32 headroom; /* Headroom in each chunk */ + u32 nr_chunks; /* Number of chunks */ + u8 chunk_shift; /* ilog2(chunk_size) */ + + struct rattan_pending *pending; /* nr_chunks slots, indexed by chunk */ +}; + +/* + * Rings context - tracks pinned ring memory + */ +struct rattan_rings { + refcount_t refcount; /* References from queue + outstanding tokens */ + spinlock_t comp_lock; /* Serialize completion ring producers */ + + struct page **pages; + unsigned long nr_pages; + void *mapped; + u64 len; + + struct rattan_rx_ring *rx; + struct rattan_tx_ring *tx; + struct rattan_fill_ring *fill; + struct rattan_comp_ring *comp; + struct rattan_drop_ring *drop; + + /* + * The last put can come from softirq (a token completed by NAPI, or + * released via the DROP ring, after this queue's device closed), but + * vunmap() and unpinning must not run there. The cleanup is deferred. + */ + struct work_struct cleanup_work; +}; + +/* + * Debug/instrumentation counters, per CPU so that the xmit and NAPI CPUs of a + * queue never share a line. Only counted while debug_stats_interval_ms != 0. + */ +struct rattan_queue_dbg { + u64 xmit_ok; /* Packets enqueued to RX ring for userspace */ + u64 xmit_rx_full; /* start_xmit blocked by RX ring full */ + u64 xmit_fill_empty; /* start_xmit blocked by empty FILL ring */ + u64 xmit_nomem; /* start_xmit blocked by allocation failure */ + u64 xmit_invalid_addr; /* start_xmit saw invalid chunk address */ + u64 rx_notify; /* Userspace wake notifications */ + u64 kick_rx; /* RX kick requests from userspace */ + u64 kick_rx_remote; /* RX kicks sent via remote IPI */ + u64 napi_polls; /* NAPI poll invocations */ + u64 napi_complete; /* NAPI completion events */ + u64 napi_budget_stop; /* NAPI stopped due to budget exhaustion */ + u64 napi_tx_empty; /* NAPI found TX ring empty */ + u64 napi_comp_full; /* NAPI blocked by COMP ring full */ + u64 napi_token_miss; /* NAPI saw TX token lookup miss */ + u64 napi_rx_ok; /* Packets injected into kernel RX path */ + u64 napi_rx_drop; /* Packets dropped by kernel RX path */ + u64 comp_ok; /* Chunk completions pushed to COMP ring */ + u64 comp_full; /* Chunk completions dropped due to full COMP */ + u64 recycle_token; /* Tokens recycled successfully */ +}; + +/* + * Per-device context - embedded in netdev private area + */ +struct rattan_queue { + struct rattan_vnic_ctx *ctx; + u16 qid; + struct rattan_rings __rcu *rings; + + /* + * Poll wait queue - for blocking waits on RX availability. + * The device still keeps a device-global wait queue so a single fd can + * be polled for activity from any queue. + */ + wait_queue_head_t wait; + + /* + * RX wake-up coalescing - batch wake-ups to reduce overhead + * Similar to NIC interrupt coalescing (ethtool -C rx-frames/rx-usecs) + */ + struct hrtimer rx_timer; /* Timer for coalescing timeout */ + atomic_t rx_pending; /* Packets since last wake-up */ + + struct napi_struct napi; /* NAPI structure for RX injection */ + int napi_cpu; /* Target CPU (-1 = use caller's CPU) */ + atomic_t napi_scheduled; /* Avoid redundant IPIs for remote NAPI */ + + /* Debug counters (struct rattan_queue_dbg), see RATTAN_DBG_INC */ + struct rattan_queue_dbg __percpu *dbg; + unsigned long dbg_last_dump_jiffies; /* Last periodic stats dump time */ +}; + +struct rattan_vnic_ctx { + struct net_device *netdev; + int dev_id; + bool started; + + struct rattan_umem __rcu *umem; + u16 num_queues; + struct rattan_queue *queues; + + /* Device-global poll wait queue for activity from any queue */ + wait_queue_head_t wait; +}; + +#endif /* _RATTAN_VNIC_INTERNAL_H */ diff --git a/rvnic/kernel/rattan_vnic_uapi.h b/rvnic/kernel/rattan_vnic_uapi.h new file mode 100644 index 00000000..0bb13640 --- /dev/null +++ b/rvnic/kernel/rattan_vnic_uapi.h @@ -0,0 +1,278 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ +/* + * Rattan Virtual NIC - User API Header + * + * Copyright (C) 2025 Rattan Project + * + * This header defines the interface between kernel and userspace. + * It should be kept in sync with userspace libraries (e.g., librvnic). + */ + +#ifndef _RATTAN_VNIC_UAPI_H +#define _RATTAN_VNIC_UAPI_H + +#include +#include + +/* + * Configuration constants + */ +#define RATTAN_RING_SIZE 262144 /* Must be power of 2 */ +#define RATTAN_RING_MASK (RATTAN_RING_SIZE - 1) +#define RATTAN_DEFAULT_CHUNK_SIZE 256 /* Default chunk size */ +#define RATTAN_MIN_CHUNK_SIZE 256 /* Minimum chunk size */ +#define RATTAN_MAX_CHUNK_SIZE 65536 /* Maximum chunk size (64KB) */ +#define RATTAN_MAX_UMEM_SIZE (128ULL << 30) /* Maximum UMEM size (128 GiB) */ +#define RATTAN_MAX_CHUNKS (1U << 22) /* Maximum chunks per UMEM */ +#define RATTAN_HEADER_SIZE 0 /* Bytes eagerly copied to chunk */ + +/* + * UMEM registration parameters + * + * UMEM - Userspace Memory Pool + * + * Userspace allocates a contiguous memory region and registers it with + * the kernel. The region is divided into fixed-size chunks. + * + * Layout: + * +----------+----------+----------+----------+----------+ + * | chunk 0 | chunk 1 | chunk 2 | ... | chunk N | + * +----------+----------+----------+----------+----------+ + * + * Each chunk layout (with headroom): + * +----------+------------------------------------------+ + * | headroom | packet data | + * +----------+------------------------------------------+ + */ +struct rattan_umem_reg { + __u64 addr; /* Userspace address of UMEM */ + __u64 len; /* Total length of UMEM */ + __u32 chunk_size; /* Size of each chunk (must be power of 2) */ + __u32 headroom; /* Headroom before packet data in each chunk */ +}; + +/* + * Ring header - producer/consumer indices (cache-line aligned) + */ +struct rattan_ring { + __u32 producer __attribute__((aligned(64))); + __u32 consumer __attribute__((aligned(64))); + __u32 flags; + __u32 _pad[13]; /* Pad to cache line boundary */ +}; + +/* Ring flags (reserved for future use) */ +#define RATTAN_RING_NEED_WAKEUP (1 << 0) + +/* + * Packet descriptor - used in both RX and TX rings + */ +struct rattan_desc { + __u64 addr; /* Chunk address (offset in UMEM) */ + __u32 len; /* Packet/header length */ + __u32 options; /* Reserved for future use */ + __u64 token; /* Token for payload retrieval / forwarding */ + __u64 timestamp; /* Timestamp (ns) - RX: arrival, TX: reserved */ +}; + +/* + * FILL Ring - Userspace provides free chunks for kernel to use + * + * Flow: + * 1. Userspace writes chunk addresses to fill ring (producer) + * 2. Kernel reads chunk addresses when RX packet arrives (consumer) + * 3. Kernel writes packet header to chunk, puts descriptor in RX ring + * + * Producer: Userspace + * Consumer: Kernel + */ +struct rattan_fill_ring { + struct rattan_ring ring; + __u64 addrs[RATTAN_RING_SIZE]; /* Chunk addresses (offset in UMEM) */ +}; + +struct rattan_drop_ring { + struct rattan_ring ring; + __u64 tokens[RATTAN_RING_SIZE]; /* tokens */ +}; + +/* + * COMP Ring - Kernel returns chunks that are done + * + * Flow: + * 1. When token is released (via RELEASE ioctl or TX with release flag) + * 2. Kernel writes chunk address to completion ring (producer) + * 3. Userspace reads to reclaim chunks (consumer) + * + * Producer: Kernel + * Consumer: Userspace + */ +struct rattan_comp_ring { + struct rattan_ring ring; + __u64 addrs[RATTAN_RING_SIZE]; /* Completed chunk addresses */ +}; + +/* + * RX Ring - Kernel produces packet descriptors, userspace consumes + * + * Producer: Kernel + * Consumer: Userspace + */ +struct rattan_rx_ring { + struct rattan_ring ring; + struct rattan_desc descs[RATTAN_RING_SIZE]; +}; + +/* + * TX Ring - Userspace produces descriptors to forward, kernel consumes + * + * Producer: Userspace + * Consumer: Kernel + */ +struct rattan_tx_ring { + struct rattan_ring ring; + struct rattan_desc descs[RATTAN_RING_SIZE]; +}; + +/* + * Ring memory layout + * + * Userspace allocates a contiguous region containing all five rings: + * +--------------+--------------+---------------+---------------+--------------+ + * | RX Ring | TX Ring | FILL Ring | COMP Ring | DROP Ring | + * +--------------+--------------+---------------+---------------+--------------+ + */ +#define RATTAN_RINGS_SIZE \ + (sizeof(struct rattan_rx_ring) + sizeof(struct rattan_tx_ring) + \ + sizeof(struct rattan_fill_ring) + sizeof(struct rattan_comp_ring) + \ + sizeof(struct rattan_drop_ring)) + +#define RATTAN_RX_RING_OFFSET 0 +#define RATTAN_TX_RING_OFFSET sizeof(struct rattan_rx_ring) +#define RATTAN_FILL_RING_OFFSET (RATTAN_TX_RING_OFFSET + sizeof(struct rattan_tx_ring)) +#define RATTAN_COMP_RING_OFFSET (RATTAN_FILL_RING_OFFSET + sizeof(struct rattan_fill_ring)) +#define RATTAN_DROP_RING_OFFSET (RATTAN_COMP_RING_OFFSET + sizeof(struct rattan_comp_ring)) + +/* + * Ring registration parameters + * + * The original REG_RINGS ioctl continues to target queue 0 for backward + * compatibility. Use struct rattan_rings_reg_q with REG_RINGS_Q to register + * rings for a specific queue. + */ +struct rattan_rings_reg { + __u64 addr; /* Userspace address of rings */ + __u64 len; /* Total length (must be >= RATTAN_RINGS_SIZE) */ +}; + +struct rattan_rings_reg_q { + __u64 addr; /* Userspace address of rings */ + __u64 len; /* Total length (must be >= RATTAN_RINGS_SIZE) */ + __u32 queue_id; /* Queue to register rings for */ + __u32 _pad; +}; + +/* + * Share UMEM from another device + * + * Instead of registering new UMEM, share an existing UMEM from another + * rattan device. Pass the file descriptor of the device that owns the UMEM. + * + * Each device still needs its own rings (FILL/RX/TX/COMP), but chunks + * can freely flow between devices sharing the same UMEM. + */ +struct rattan_share_umem_req { + __s32 source_fd; /* fd of device that owns the UMEM */ + __u32 _pad; +}; + +/* + * Payload read request/response + */ +struct rattan_payload_req { + __u64 token; /* Token from RX descriptor */ + __u64 buf; /* Userspace buffer address */ + __u32 len; /* Buffer length */ + __u32 offset; /* Offset within payload */ +}; + +struct rattan_payload_resp { + __u32 len; /* Actual length copied */ + __u32 total_len; /* Total packet length */ +}; + +/* + * Token release request + */ +struct rattan_release_req { + __u64 token; /* Token to release */ +}; + +/* + * Token clone request/response + * + * Copy the SKB behind a token into a new token, e.g. to forward one packet to + * several destinations. + * + * A token is bound to exactly one UMEM chunk, so the clone needs one too: + * userspace supplies a free chunk (chunk_addr) from its own pool, exactly as it + * would on FILL, and queue_id names the queue of this device whose COMP ring + * gets the chunk back when the clone is released or transmitted. + * + * Fails with -EBUSY if chunk_addr is still bound to a live token. + */ +struct rattan_clone_req { + __u64 token; /* Token to clone */ + __u64 chunk_addr; /* Free chunk to bind the new token to */ + __u32 queue_id; /* Queue whose COMP ring owns chunk_addr */ + __u32 _pad; +}; + +struct rattan_clone_resp { + __u64 new_token; /* New token for cloned SKB */ +}; + +struct rattan_start_config { + __s32 napi_cpu; /* Target CPU for NAPI (-1 = use caller's CPU) */ + __u32 _pad; +}; + +/* + * ioctl commands + */ +#define RATTAN_VNIC_MAGIC 'R' + +/* Setup commands */ +#define RATTAN_VNIC_REG_UMEM _IOW(RATTAN_VNIC_MAGIC, 1, struct rattan_umem_reg) + +#define RATTAN_VNIC_REG_RINGS _IOW(RATTAN_VNIC_MAGIC, 2, struct rattan_rings_reg) + +#define RATTAN_VNIC_SHARE_UMEM _IOW(RATTAN_VNIC_MAGIC, 3, struct rattan_share_umem_req) + +/* Queue-aware setup commands */ +#define RATTAN_VNIC_REG_RINGS_Q _IOW(RATTAN_VNIC_MAGIC, 6, struct rattan_rings_reg_q) + +/* Control commands */ +#define RATTAN_VNIC_START _IOW(RATTAN_VNIC_MAGIC, 4, struct rattan_start_config) + +#define RATTAN_VNIC_STOP _IO(RATTAN_VNIC_MAGIC, 5) + +/* Data path commands */ +#define RATTAN_VNIC_READ_PAYLOAD _IOWR(RATTAN_VNIC_MAGIC, 10, struct rattan_payload_req) + +#define RATTAN_VNIC_RELEASE _IOW(RATTAN_VNIC_MAGIC, 11, struct rattan_release_req) + +/* + * Backward-compatible queue-0 RX kick. + * Use KICK_RX_Q to target a specific queue. + */ +#define RATTAN_VNIC_KICK_RX _IO(RATTAN_VNIC_MAGIC, 12) + +#define RATTAN_VNIC_CLONE _IOWR(RATTAN_VNIC_MAGIC, 13, struct rattan_clone_req) + +#define RATTAN_VNIC_KICK_RX_Q _IOW(RATTAN_VNIC_MAGIC, 14, __u32) + +// Get the device ID of the VNIC +#define RATTAN_VNIC_DEVICE_ID _IOR(RATTAN_VNIC_MAGIC, 15, __u32) + +#endif /* _RATTAN_VNIC_UAPI_H */ diff --git a/rvnic/kernel/readme.md b/rvnic/kernel/readme.md new file mode 100644 index 00000000..1b64f96a --- /dev/null +++ b/rvnic/kernel/readme.md @@ -0,0 +1,6 @@ +## For developers + +If you want enable `clang` to find the kernel-only headers in your editor (for example, `#include `). + +1. `sudo apt install bear` +2. `bear -- make all`, and `compile_commands.json` will be created. `clang` can find the compile commands now! \ No newline at end of file diff --git a/rvnic/librvnic/.gitignore b/rvnic/librvnic/.gitignore new file mode 100644 index 00000000..c41cc9e3 --- /dev/null +++ b/rvnic/librvnic/.gitignore @@ -0,0 +1 @@ +/target \ No newline at end of file diff --git a/rvnic/librvnic/Cargo.lock b/rvnic/librvnic/Cargo.lock new file mode 100644 index 00000000..b555b466 --- /dev/null +++ b/rvnic/librvnic/Cargo.lock @@ -0,0 +1,103 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "bitflags" +version = "2.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "843867be96c8daad0d758b57df9392b6d8d271134fce549de6ce169ff98a92af" + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "cfg_aliases" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" + +[[package]] +name = "libc" +version = "0.2.183" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5b646652bf6661599e1da8901b3b9522896f01e736bad5f723fe7a3a27f899d" + +[[package]] +name = "nix" +version = "0.30.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "74523f3a35e05aba87a1d978330aef40f67b0304ac79c1c00b294c9830543db6" +dependencies = [ + "bitflags", + "cfg-if", + "cfg_aliases", + "libc", +] + +[[package]] +name = "proc-macro2" +version = "1.0.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "rvnic" +version = "0.1.0" +dependencies = [ + "libc", + "nix", + "thiserror", +] + +[[package]] +name = "syn" +version = "2.0.117" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e665b8803e7b1d2a727f4023456bbbbe74da67099c585258af0ad9c5013b9b99" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "thiserror" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" diff --git a/rvnic/librvnic/Cargo.toml b/rvnic/librvnic/Cargo.toml new file mode 100644 index 00000000..050dce3a --- /dev/null +++ b/rvnic/librvnic/Cargo.toml @@ -0,0 +1,14 @@ +[package] +name = "rvnic" +version = "0.1.0" +edition = "2024" +description = "Rattan Virtual NIC userspace library" +license = "MIT OR Apache-2.0" + +[dependencies] +libc = "0.2" +nix = { version = "0.30.1", features = ["fs", "ioctl", "mman"] } +thiserror = "2" + +[dev-dependencies] +# For running tests diff --git a/rvnic/librvnic/examples/test_forward.rs b/rvnic/librvnic/examples/test_forward.rs new file mode 100644 index 00000000..6ece4769 --- /dev/null +++ b/rvnic/librvnic/examples/test_forward.rs @@ -0,0 +1,662 @@ +//! Test packet forwarding between two vNICs with iperf +//! +//! This test creates two vNIC devices sharing UMEM and forwards packets between them. +//! Each vNIC is placed in a separate network namespace to simulate a real deployment. +//! +//! Topology: +//! [ns_client] [ns_server] +//! iperf3 -c 10.0.0.1 --> rattan0 --> [forward] --> rattan1 --> iperf3 -s +//! 10.0.0.2 10.0.0.1 10.0.0.1 (binds 0.0.0.0) +//! +//! This example uses the queue-0 convenience API and therefore assumes the +//! kernel module is loaded with `num_queues=1`. +//! +//! If you want to test multiqueue forwarding, use `test_forward_multiqueue.rs` +//! instead and load the kernel module with a matching `num_queues` value. +//! +//! Run this with: sudo ./target/debug/examples/test_forward +//! +//! Prerequisites: +//! 1. Build and load the kernel module in single-queue mode: +//! cd rvnic/kernel && make && sudo insmod rattan_vnic.ko num_queues=1 +//! 2. Install iperf3: sudo apt-get install iperf3 + +use rvnic::sys::RattanDesc; +use rvnic::{ + // CompRing, FillRing, + RATTAN_RING_SIZE, + Rings, + RvnicDevice, + RxRing, + TxRing, + UmemBuilder, +}; +use std::collections::VecDeque; +use std::os::unix::io::AsRawFd; +use std::process::{Child, Command, Stdio}; +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; +use std::time::{Duration, Instant}; + +// Batch size for ring operations +const BATCH_SIZE: usize = 256; + +// Initial UMEM chunk split between the two devices. +// +// Chunks are owned by the device/queue that originally supplied them and are +// recycled back to that same device/queue. Because the TCP data direction and +// ACK direction are often asymmetric, keep this as a tunable constant so the +// initial pool sizing can be adjusted during performance testing. +// +// Device 0 gets DEV0_CHUNK_SHARE_PERCENT percent of the chunks. +// Device 1 gets the remainder. +const DEV0_CHUNK_SHARE_PERCENT: u32 = 50; + +// CPU cores for pinning +// +// Topology (4-core system): +// CPU 0: iperf client + NAPI for rattan0 (client-side packet injection) +// CPU 1: forwarder thread (ring management only) +// CPU 2: iperf server + NAPI for rattan1 (server-side packet injection) +// CPU 3: spare +// +// This distributes work so the forwarder only does ring operations while +// packet injection (softirq) runs on the same CPU as the application. +const CPU_FORWARDER: u32 = 1; +const CPU_IPERF_SERVER: u32 = 2; +const CPU_IPERF_CLIENT: u32 = 0; +// NAPI CPU for each device (-1 = use caller's CPU at kick_rx time) +// Spread NAPI to client/server CPUs to parallelize with forwarder +const CPU_NAPI_DEV0: i32 = 0; // Client CPU (parallel with forwarder) +const CPU_NAPI_DEV1: i32 = 2; // Server CPU (parallel with forwarder) + +/// RAII guard for network namespace cleanup +struct NetnsGuard { + names: Vec<&'static str>, +} + +impl NetnsGuard { + fn new() -> Self { + Self { names: Vec::new() } + } + + fn add(&mut self, name: &'static str) -> std::io::Result<()> { + // Try to delete first in case it exists from a previous run + let _ = Command::new("ip").args(["netns", "delete", name]).status(); + let status = Command::new("ip").args(["netns", "add", name]).status()?; + if !status.success() { + return Err(std::io::Error::other(format!( + "Failed to create netns {}", + name + ))); + } + self.names.push(name); + Ok(()) + } +} + +impl Drop for NetnsGuard { + fn drop(&mut self) { + for name in &self.names { + let _ = Command::new("ip").args(["netns", "delete", name]).status(); + } + } +} + +// Direct transfer from COMP to FILL ring (single batch, zero-copy recycling) +// #[inline] +// fn recycle_chunks(comp: &mut CompRing, fill: &mut FillRing, buf: &mut [u64]) -> usize { +// let n = comp.consume(buf); +// if n > 0 { fill.produce(&buf[..n]) } else { 0 } +// } + +/// Move interface to network namespace +fn move_to_netns(iface: &str, netns: &str) -> std::io::Result<()> { + let status = Command::new("ip") + .args(["link", "set", iface, "netns", netns]) + .status()?; + if !status.success() { + return Err(std::io::Error::other(format!( + "Failed to move {} to netns {}", + iface, netns + ))); + } + Ok(()) +} + +/// Configure interface inside a namespace +fn configure_interface_in_netns(netns: &str, iface: &str, ip: &str) -> std::io::Result<()> { + let status = Command::new("ip") + .args([ + "netns", "exec", netns, "ip", "addr", "add", ip, "dev", iface, + ]) + .status()?; + if !status.success() { + return Err(std::io::Error::other(format!( + "Failed to add IP {} to {} in {}", + ip, iface, netns + ))); + } + + let status = Command::new("ip") + .args(["netns", "exec", netns, "ip", "link", "set", iface, "up"]) + .status()?; + if !status.success() { + return Err(std::io::Error::other(format!( + "Failed to bring up {} in {}", + iface, netns + ))); + } + + let _ = Command::new("ip") + .args(["netns", "exec", netns, "ip", "link", "set", "lo", "up"]) + .status(); + + Ok(()) +} + +/// Start iperf3 server in a namespace, pinned to a CPU +fn start_iperf_server_in_netns(netns: &str, cpu: u32) -> std::io::Result { + Command::new("taskset") + .args([ + "-c", + &cpu.to_string(), + "ip", + "netns", + "exec", + netns, + "iperf3", + "-s", + "-1", + ]) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() +} + +/// Run iperf3 client in a namespace, pinned to a CPU +fn run_iperf_client_in_netns( + netns: &str, + server_addr: &str, + duration: u32, + parallel: u32, + cpu: u32, +) -> std::io::Result { + let cpu_str = cpu.to_string(); + let duration_str = duration.to_string(); + let parallel_str = parallel.to_string(); + + let mut args = vec![ + "-c", + &cpu_str, + "ip", + "netns", + "exec", + netns, + "iperf3", + "-c", + server_addr, + "-t", + &duration_str, + "-i", + "1", + "--forceflush", + ]; + + if parallel > 1 { + args.extend(["-P", ¶llel_str]); + } + + let output = Command::new("taskset").args(&args).output()?; + let stdout = String::from_utf8_lossy(&output.stdout); + let stderr = String::from_utf8_lossy(&output.stderr); + + if !output.status.success() { + return Err(std::io::Error::other(format!( + "iperf3 client failed: {}", + stderr + ))); + } + + Ok(stdout.to_string()) +} + +/// Set CPU affinity for the current thread +fn set_cpu_affinity(cpu: u32) -> std::io::Result<()> { + unsafe { + let mut cpuset: libc::cpu_set_t = std::mem::zeroed(); + libc::CPU_ZERO(&mut cpuset); + libc::CPU_SET(cpu as usize, &mut cpuset); + + let ret = libc::sched_setaffinity(0, std::mem::size_of::(), &cpuset); + if ret != 0 { + return Err(std::io::Error::last_os_error()); + } + } + Ok(()) +} + +/// Forward packets from one RX ring to another TX ring (single batch) +#[inline] +/// Transmit `n` UDP datagrams out of `iface` (still in the root netns) and +/// keep their descriptors, so their tokens stay live until the devices close. +fn hold_tokens(iface: &str, rx: &mut RxRing, n: usize) -> std::io::Result> { + let mut held = Vec::with_capacity(n); + if n == 0 { + return Ok(held); + } + let ip = |args: &[&str]| Command::new("ip").args(args).status().map(|_| ()); + ip(&["addr", "add", "10.99.0.1/24", "dev", iface])?; + ip(&["link", "set", iface, "up"])?; + ip(&[ + "neigh", + "replace", + "10.99.0.2", + "lladdr", + "02:00:00:00:00:02", + "dev", + iface, + "nud", + "permanent", + ])?; + + let sock = std::net::UdpSocket::bind("10.99.0.1:0")?; + let mut descs = [RattanDesc::default(); BATCH_SIZE]; + let mut sent = 0; + while held.len() < n { + while sent < n && sent < held.len() + BATCH_SIZE { + sock.send_to(b"hold", "10.99.0.2:9")?; + sent += 1; + } + let got = rx.consume(&mut descs); + held.extend_from_slice(&descs[..got]); + if got == 0 && sent >= n { + std::thread::sleep(Duration::from_millis(1)); + } + } + + ip(&["addr", "flush", "dev", iface])?; + ip(&["link", "set", iface, "down"])?; + println!(" Holding {} live tokens for the whole test", held.len()); + Ok(held) +} + +fn forward_batch(rx: &mut RxRing, tx: &mut TxRing, descs: &mut [RattanDesc]) -> usize { + let received = rx.consume(descs); + if received > 0 { + tx.produce(&descs[..received]) + } else { + 0 + } +} + +fn main() -> Result<(), Box> { + println!("=== Rattan vNIC Forwarding Performance Test ===\n"); + + // Step 1: Create network namespaces (with RAII cleanup) + println!("1. Creating network namespaces..."); + let mut netns_guard = NetnsGuard::new(); + netns_guard.add("ns_client")?; + netns_guard.add("ns_server")?; + println!(" Created ns_client and ns_server"); + + // Step 2: Setup devices with shared UMEM + println!("\n2. Setting up vNIC devices..."); + + let mut dev0 = RvnicDevice::open()?; + let device_name_0 = dev0.device_name()?; + println!(" {} opened (fd={})", device_name_0, dev0.fd()); + + let mut dev1 = RvnicDevice::open()?; + let device_name_1 = dev1.device_name()?; + println!(" {} opened (fd={})", device_name_1, dev1.fd()); + + // Allocate shared UMEM - use 2x ring size total to allow buffering across + // both devices. Chunks are ingress-owned and are recycled back to the + // device that originally supplied them, so the initial split between the + // two devices is configurable below. + let chunks_per_device = RATTAN_RING_SIZE; + let total_chunks = chunks_per_device * 2; + + let umem = UmemBuilder::new() + .chunk_size(256) + .headroom(64) + .num_chunks(total_chunks) + .build()?; + + println!( + " UMEM allocated: {} chunks x {} bytes ({} chunks/device worth total capacity)", + umem.num_chunks(), + umem.chunk_size(), + chunks_per_device + ); + + let chunk_size = umem.chunk_size() as u64; + let num_chunks = umem.num_chunks(); + + dev0.register_umem(umem)?; + dev1.share_umem(&dev0)?; + println!(" UMEM registered and shared"); + + // Setup rings and get ring handles + let rings0 = Rings::new()?; + let (mut fill0, mut _drop0, mut rx0, mut tx0) = dev0.register_rings_for_queue(0, rings0)?; + + let rings1 = Rings::new()?; + let (mut fill1, mut _drop1, mut rx1, mut tx1) = dev1.register_rings_for_queue(0, rings1)?; + println!(" Rings registered"); + + // Distribute chunks between the two devices. Chunks are ingress-owned and + // return to the same device/queue that originally supplied them, so this + // initial split controls the steady-state buffer budget available to each + // direction. + let mut buf = [0u64; BATCH_SIZE]; + let mut dist0 = 0usize; + let mut dist1 = 0usize; + let dev0_chunks = ((num_chunks * DEV0_CHUNK_SHARE_PERCENT) / 100) as usize; + let dev1_start = dev0_chunks; + let dev1_chunks = num_chunks as usize - dev0_chunks; + + // Fill ring0 with the device 0 share + for chunk_start in (0..dev0_chunks).step_by(BATCH_SIZE) { + let chunk_end = (chunk_start + BATCH_SIZE).min(dev0_chunks); + let batch_size = chunk_end - chunk_start; + + for (i, addr) in buf.iter_mut().enumerate().take(batch_size) { + *addr = ((chunk_start + i) as u64) * chunk_size; + } + + dist0 += fill0.produce(&buf[..batch_size]); + } + + // Fill ring1 with the remaining share for device 1 + for chunk_start in (dev1_start..(dev1_start + dev1_chunks)).step_by(BATCH_SIZE) { + let chunk_end = (chunk_start + BATCH_SIZE).min(dev1_start + dev1_chunks); + let batch_size = chunk_end - chunk_start; + + for (i, addr) in buf.iter_mut().enumerate().take(batch_size) { + *addr = ((chunk_start + i) as u64) * chunk_size; + } + + dist1 += fill1.produce(&buf[..batch_size]); + } + + println!(" Chunks allocated: {}", num_chunks); + println!( + " FILL rings populated ({} to ring0, {} to ring1, dev0 share={}%)", + dist0, dist1, DEV0_CHUNK_SHARE_PERCENT + ); + + // Start devices with NAPI CPU configuration + dev0.start(CPU_NAPI_DEV0)?; + dev1.start(CPU_NAPI_DEV1)?; + println!(" Devices started"); + if CPU_NAPI_DEV0 >= 0 { + println!( + " {} NAPI CPU: {} (client-side injection)", + device_name_0, CPU_NAPI_DEV0 + ); + } else { + println!(" {} NAPI: use caller's CPU", device_name_0); + } + if CPU_NAPI_DEV1 >= 0 { + println!( + " {} NAPI CPU: {} (server-side injection)", + device_name_1, CPU_NAPI_DEV1 + ); + } else { + println!(" rattan1 NAPI: use caller's CPU"); + } + + // Optional: delay the client->server direction by DELAY_MS, so tokens stay + // in flight for that long, as they do behind an emulated link. + let delay = Duration::from_millis( + std::env::var("DELAY_MS") + .ok() + .and_then(|v| v.parse().ok()) + .unwrap_or(0), + ); + + // Optional: park HOLD_TOKENS live tokens for the whole test, to model the + // packets an emulated link keeps in flight (its bandwidth-delay product). + let hold: usize = std::env::var("HOLD_TOKENS") + .ok() + .and_then(|v| v.parse().ok()) + .unwrap_or(0); + let _held = hold_tokens(device_name_0.as_str(), &mut rx0, hold)?; + + // Step 3: Move interfaces to namespaces and configure + println!("\n3. Configuring network namespaces..."); + + move_to_netns(device_name_0.as_str(), "ns_client")?; + println!(" Moved {} to ns_client", device_name_0); + + move_to_netns(device_name_1.as_str(), "ns_server")?; + println!(" Moved {} to ns_server", device_name_1); + + configure_interface_in_netns("ns_client", device_name_0.as_str(), "10.0.0.2/24")?; + println!(" ns_client: {} = 10.0.0.2/24", device_name_0); + + configure_interface_in_netns("ns_server", device_name_1.as_str(), "10.0.0.1/24")?; + println!(" ns_server: {} = 10.0.0.1/24", device_name_1); + + if !delay.is_zero() { + // Let TCP autotuning grow the window to cover the added delay + for ns in ["ns_client", "ns_server"] { + for key in ["net.ipv4.tcp_rmem", "net.ipv4.tcp_wmem"] { + let _ = Command::new("ip") + .args(["netns", "exec", ns, "sysctl", "-qw"]) + .arg(format!("{}=4096 131072 268435456", key)) + .status(); + } + } + println!(" client->server delay: {:?}", delay); + } + + // Step 4: Start forwarding loop + println!("\n4. Starting forwarding loop..."); + + let running = Arc::new(AtomicBool::new(true)); + let running_clone = running.clone(); + + let packets_forwarded = Arc::new(AtomicU64::new(0)); + let packets_forwarded_clone = packets_forwarded.clone(); + + // Get file descriptors for epoll + let dev0_fd = dev0.as_raw_fd(); + let dev1_fd = dev1.as_raw_fd(); + + // Single-threaded forwarder using epoll, pinned to CPU + let forward_thread = std::thread::spawn(move || { + // Pin to dedicated CPU core + if let Err(e) = set_cpu_affinity(CPU_FORWARDER) { + eprintln!("Warning: failed to set CPU affinity: {}", e); + } + + let mut descs = [RattanDesc::default(); BATCH_SIZE]; + // let mut addrs = [0u64; BATCH_SIZE]; + + // Create epoll instance + let epfd = unsafe { libc::epoll_create1(0) }; + if epfd < 0 { + eprintln!("epoll_create1 failed"); + return (dev0, dev1); + } + + // Add dev0 to epoll (data.u32 = 0) + let mut ev = libc::epoll_event { + events: libc::EPOLLIN as u32, + u64: 0, + }; + if unsafe { libc::epoll_ctl(epfd, libc::EPOLL_CTL_ADD, dev0_fd, &mut ev) } < 0 { + eprintln!("epoll_ctl add dev0 failed"); + unsafe { libc::close(epfd) }; + return (dev0, dev1); + } + + // Add dev1 to epoll (data.u32 = 1) + ev.u64 = 1; + if unsafe { libc::epoll_ctl(epfd, libc::EPOLL_CTL_ADD, dev1_fd, &mut ev) } < 0 { + eprintln!("epoll_ctl add dev1 failed"); + unsafe { libc::close(epfd) }; + return (dev0, dev1); + } + + let mut events = [libc::epoll_event { events: 0, u64: 0 }; 2]; + let mut delay_line: VecDeque<(Instant, RattanDesc)> = VecDeque::new(); + + while running_clone.load(Ordering::Relaxed) { + // Wait for events on either device (10ms timeout, 1ms with a delay line) + let timeout = if delay.is_zero() { 10 } else { 1 }; + let nfds = unsafe { libc::epoll_wait(epfd, events.as_mut_ptr(), 2, timeout) }; + + if nfds < 0 { + let err = std::io::Error::last_os_error(); + if err.kind() != std::io::ErrorKind::Interrupted { + eprintln!("epoll_wait failed: {}", err); + break; + } + continue; + } + + // Process events - drain RX rings completely + for event in events.iter().take(nfds as usize) { + let dev_idx = event.u64; + + if dev_idx == 0 && !delay.is_zero() { + // dev0 has RX data: queue it on the delay line + loop { + let got = rx0.consume(&mut descs); + let now = Instant::now(); + delay_line.extend(descs[..got].iter().map(|d| (now, *d))); + if got < BATCH_SIZE { + break; + } + } + } else if dev_idx == 0 { + // dev0 has RX data: forward rx0 → tx1 + // Drain all available packets (not just one batch) + loop { + let fwd = forward_batch(&mut rx0, &mut tx1, &mut descs); + if fwd > 0 { + packets_forwarded_clone.fetch_add(fwd as u64, Ordering::Relaxed); + let _ = dev1.kick_rx_queue(0); + } + if fwd < BATCH_SIZE { + break; // Ring drained + } + } + } else { + // dev1 has RX data: forward rx1 → tx0 + // Drain all available packets (not just one batch) + loop { + let fwd = forward_batch(&mut rx1, &mut tx0, &mut descs); + if fwd > 0 { + packets_forwarded_clone.fetch_add(fwd as u64, Ordering::Relaxed); + let _ = dev0.kick_rx_queue(0); + } + if fwd < BATCH_SIZE { + break; // Ring drained + } + } + } + } + + // Release everything that has spent `delay` on the delay line + if !delay_line.is_empty() { + let now = Instant::now(); + let mut n = 0; + while let Some(&(t, d)) = delay_line.front() { + if now.duration_since(t) < delay || n == BATCH_SIZE { + break; + } + descs[n] = d; + n += 1; + delay_line.pop_front(); + } + if n > 0 { + let fwd = tx1.produce(&descs[..n]); + // TX full: put the rest back in order + for d in descs[fwd..n].iter().rev() { + delay_line.push_front((now - delay, *d)); + } + packets_forwarded_clone.fetch_add(fwd as u64, Ordering::Relaxed); + let _ = dev1.kick_rx_queue(0); + } + } + + // Always recycle chunks back to the device that originally supplied + // them. With current kernel completion routing, chunks are + // ingress-owned and completions return to the originating + // device/queue, not the forwarding destination. + // loop { + // let r0 = recycle_chunks(&mut comp0, &mut fill0, &mut addrs); + // let r1 = recycle_chunks(&mut comp1, &mut fill1, &mut addrs); + // if r0 == 0 && r1 == 0 { + // break; + // } + // } + } + + unsafe { libc::close(epfd) }; + (dev0, dev1) + }); + + println!( + " Forwarding loop started (single thread, epoll, CPU {})", + CPU_FORWARDER + ); + + // Step 5: Run iperf test + println!("\n5. Running iperf3 performance test..."); + println!( + " Starting iperf3 server in ns_server (CPU {})...", + CPU_IPERF_SERVER + ); + let mut server = start_iperf_server_in_netns("ns_server", CPU_IPERF_SERVER)?; + + std::thread::sleep(Duration::from_millis(500)); + + let parallel_streams = 4; + println!( + " Running iperf3 client from ns_client (10 seconds, {} parallel streams, CPU {})...\n", + parallel_streams, CPU_IPERF_CLIENT + ); + let start = Instant::now(); + + match run_iperf_client_in_netns( + "ns_client", + "10.0.0.1", + 10, + parallel_streams, + CPU_IPERF_CLIENT, + ) { + Ok(output) => println!("{}", output), + Err(e) => eprintln!(" iperf3 client error: {}", e), + } + + let elapsed = start.elapsed(); + println!("\n Test completed in {:.2}s", elapsed.as_secs_f64()); + + let _ = server.wait(); + + // Step 6: Print statistics + println!("\n6. Statistics:"); + let total_forwarded = packets_forwarded.load(Ordering::Relaxed); + println!(" Packets forwarded: {}", total_forwarded); + + // Stop forwarding + println!("\n7. Cleaning up..."); + running.store(false, Ordering::Relaxed); + + let (_dev0, _dev1) = forward_thread.join().unwrap(); + + // netns_guard will clean up namespaces on drop + drop(netns_guard); + println!(" Namespaces deleted"); + + println!(" Done"); + println!("\n=== Test Complete ==="); + + Ok(()) +} diff --git a/rvnic/librvnic/examples/test_forward_drop.rs b/rvnic/librvnic/examples/test_forward_drop.rs new file mode 100644 index 00000000..c7b3cebf --- /dev/null +++ b/rvnic/librvnic/examples/test_forward_drop.rs @@ -0,0 +1,588 @@ +//! Test packet forwarding between two vNICs with iperf +//! +//! This test creates two vNIC devices sharing UMEM and forwards packets between them. +//! Each vNIC is placed in a separate network namespace to simulate a real deployment. +//! +//! Topology: +//! [ns_client] [ns_server] +//! iperf3 -c 10.0.0.1 --> rattan0 --> [forward] --> rattan1 --> iperf3 -s +//! 10.0.0.2 10.0.0.1 10.0.0.1 (binds 0.0.0.0) +//! +//! This example uses the queue-0 convenience API and therefore assumes the +//! kernel module is loaded with `num_queues=1`. +//! +//! If you want to test multiqueue forwarding, use `test_forward_multiqueue.rs` +//! instead and load the kernel module with a matching `num_queues` value. +//! +//! Run this with: sudo ./target/debug/examples/test_forward +//! +//! Prerequisites: +//! 1. Build and load the kernel module in single-queue mode: +//! cd rvnic/kernel && make && sudo insmod rattan_vnic.ko num_queues=1 +//! 2. Install iperf3: sudo apt-get install iperf3 + +use rvnic::sys::RattanDesc; +use rvnic::{ + DropRing, + // CompRing, FillRing, + RATTAN_RING_SIZE, + Rings, + RvnicDevice, + RxRing, + TxRing, + UmemBuilder, +}; +use std::os::unix::io::AsRawFd; +use std::process::{Child, Command, Stdio}; +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; +use std::time::{Duration, Instant}; + +// Batch size for ring operations +const BATCH_SIZE: usize = 256; + +// Initial UMEM chunk split between the two devices. +// +// Chunks are owned by the device/queue that originally supplied them and are +// recycled back to that same device/queue. Because the TCP data direction and +// ACK direction are often asymmetric, keep this as a tunable constant so the +// initial pool sizing can be adjusted during performance testing. +// +// Device 0 gets DEV0_CHUNK_SHARE_PERCENT percent of the chunks. +// Device 1 gets the remainder. +const DEV0_CHUNK_SHARE_PERCENT: u32 = 50; + +// CPU cores for pinning +// +// Topology (4-core system): +// CPU 0: iperf client + NAPI for rattan0 (client-side packet injection) +// CPU 1: forwarder thread (ring management only) +// CPU 2: iperf server + NAPI for rattan1 (server-side packet injection) +// CPU 3: spare +// +// This distributes work so the forwarder only does ring operations while +// packet injection (softirq) runs on the same CPU as the application. +const CPU_FORWARDER: u32 = 1; +const CPU_IPERF_SERVER: u32 = 2; +const CPU_IPERF_CLIENT: u32 = 0; +// NAPI CPU for each device (-1 = use caller's CPU at kick_rx time) +// Spread NAPI to client/server CPUs to parallelize with forwarder +const CPU_NAPI_DEV0: i32 = 0; // Client CPU (parallel with forwarder) +const CPU_NAPI_DEV1: i32 = 2; // Server CPU (parallel with forwarder) + +/// RAII guard for network namespace cleanup +struct NetnsGuard { + names: Vec<&'static str>, +} + +impl NetnsGuard { + fn new() -> Self { + Self { names: Vec::new() } + } + + fn add(&mut self, name: &'static str) -> std::io::Result<()> { + // Try to delete first in case it exists from a previous run + let _ = Command::new("ip").args(["netns", "delete", name]).status(); + let status = Command::new("ip").args(["netns", "add", name]).status()?; + if !status.success() { + return Err(std::io::Error::other(format!( + "Failed to create netns {}", + name + ))); + } + self.names.push(name); + Ok(()) + } +} + +impl Drop for NetnsGuard { + fn drop(&mut self) { + for name in &self.names { + let _ = Command::new("ip").args(["netns", "delete", name]).status(); + } + } +} + +// Direct transfer from COMP to FILL ring (single batch, zero-copy recycling) +// #[inline] +// fn recycle_chunks(comp: &mut CompRing, fill: &mut FillRing, buf: &mut [u64]) -> usize { +// let n = comp.consume(buf); +// if n > 0 { fill.produce(&buf[..n]) } else { 0 } +// } + +/// Move interface to network namespace +fn move_to_netns(iface: &str, netns: &str) -> std::io::Result<()> { + let status = Command::new("ip") + .args(["link", "set", iface, "netns", netns]) + .status()?; + if !status.success() { + return Err(std::io::Error::other(format!( + "Failed to move {} to netns {}", + iface, netns + ))); + } + Ok(()) +} + +/// Configure interface inside a namespace +fn configure_interface_in_netns(netns: &str, iface: &str, ip: &str) -> std::io::Result<()> { + let status = Command::new("ip") + .args([ + "netns", "exec", netns, "ip", "addr", "add", ip, "dev", iface, + ]) + .status()?; + if !status.success() { + return Err(std::io::Error::other(format!( + "Failed to add IP {} to {} in {}", + ip, iface, netns + ))); + } + + let status = Command::new("ip") + .args(["netns", "exec", netns, "ip", "link", "set", iface, "up"]) + .status()?; + if !status.success() { + return Err(std::io::Error::other(format!( + "Failed to bring up {} in {}", + iface, netns + ))); + } + + let _ = Command::new("ip") + .args(["netns", "exec", netns, "ip", "link", "set", "lo", "up"]) + .status(); + + Ok(()) +} + +/// Start iperf3 server in a namespace, pinned to a CPU +fn start_iperf_server_in_netns(netns: &str, cpu: u32) -> std::io::Result { + Command::new("taskset") + .args([ + "-c", + &cpu.to_string(), + "ip", + "netns", + "exec", + netns, + "iperf3", + "-s", + "-1", + ]) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() +} + +/// Run iperf3 client in a namespace, pinned to a CPU +fn run_iperf_client_in_netns( + netns: &str, + server_addr: &str, + duration: u32, + parallel: u32, + cpu: u32, +) -> std::io::Result { + let cpu_str = cpu.to_string(); + let duration_str = duration.to_string(); + let parallel_str = parallel.to_string(); + + let mut args = vec![ + "-c", + &cpu_str, + "ip", + "netns", + "exec", + netns, + "iperf3", + "-c", + server_addr, + "-t", + &duration_str, + "-i", + "1", + "-u", + "-b", + "2G", + "-Z", + "--forceflush", + ]; + + if parallel > 1 { + args.extend(["-P", ¶llel_str]); + } + + let output = Command::new("taskset").args(&args).output()?; + let stdout = String::from_utf8_lossy(&output.stdout); + let stderr = String::from_utf8_lossy(&output.stderr); + + if !output.status.success() { + return Err(std::io::Error::other(format!( + "iperf3 client failed: {}", + stderr + ))); + } + + Ok(stdout.to_string()) +} + +/// Set CPU affinity for the current thread +fn set_cpu_affinity(cpu: u32) -> std::io::Result<()> { + unsafe { + let mut cpuset: libc::cpu_set_t = std::mem::zeroed(); + libc::CPU_ZERO(&mut cpuset); + libc::CPU_SET(cpu as usize, &mut cpuset); + + let ret = libc::sched_setaffinity(0, std::mem::size_of::(), &cpuset); + if ret != 0 { + return Err(std::io::Error::last_os_error()); + } + } + Ok(()) +} + +/// Forward packets from one RX ring to another TX ring (single batch) +#[inline] +fn forward_batch(rx: &mut RxRing, tx: &mut TxRing, descs: &mut [RattanDesc]) -> usize { + let received = rx.consume(descs); + if received > 0 { + tx.produce(&descs[..received]) + } else { + 0 + } +} + +#[inline] +fn drop_batch(rx: &mut RxRing, drop: &mut DropRing, descs: &mut [RattanDesc]) -> usize { + let received = rx.consume(descs); + + let tokens: &mut [u64] = &mut descs[..received] + .iter() + .map(|d| d.token) + .collect::>(); + + if received > 0 { + drop.produce(tokens) + } else { + 0 + } +} + +fn main() -> Result<(), Box> { + println!("=== Rattan vNIC Forwarding Performance Test ===\n"); + + // Step 1: Create network namespaces (with RAII cleanup) + println!("1. Creating network namespaces..."); + let mut netns_guard = NetnsGuard::new(); + netns_guard.add("ns_client")?; + netns_guard.add("ns_server")?; + println!(" Created ns_client and ns_server"); + + // Step 2: Setup devices with shared UMEM + println!("\n2. Setting up vNIC devices..."); + + let mut dev0 = RvnicDevice::open()?; + println!(" rattan0 opened (fd={})", dev0.fd()); + + let mut dev1 = RvnicDevice::open()?; + println!(" rattan1 opened (fd={})", dev1.fd()); + + // Allocate shared UMEM - use 2x ring size total to allow buffering across + // both devices. Chunks are ingress-owned and are recycled back to the + // device that originally supplied them, so the initial split between the + // two devices is configurable below. + let chunks_per_device = RATTAN_RING_SIZE; + let total_chunks = chunks_per_device * 2; + + let umem = UmemBuilder::new() + .chunk_size(2048) + .headroom(128) + .num_chunks(total_chunks) + .build()?; + + println!( + " UMEM allocated: {} chunks x {} bytes ({} chunks/device worth total capacity)", + umem.num_chunks(), + umem.chunk_size(), + chunks_per_device + ); + + let chunk_size = umem.chunk_size() as u64; + let num_chunks = umem.num_chunks(); + + dev0.register_umem(umem)?; + dev1.share_umem(&dev0)?; + println!(" UMEM registered and shared"); + + // Setup rings and get ring handles + let rings0 = Rings::new()?; + let (mut fill0, mut drop0, mut rx0, mut tx0) = dev0.register_rings_for_queue(0, rings0)?; + + let rings1 = Rings::new()?; + let (mut fill1, mut _drop1, mut rx1, mut tx1) = dev1.register_rings_for_queue(0, rings1)?; + println!(" Rings registered"); + + // Distribute chunks between the two devices. Chunks are ingress-owned and + // return to the same device/queue that originally supplied them, so this + // initial split controls the steady-state buffer budget available to each + // direction. + let mut buf = [0u64; BATCH_SIZE]; + let mut dist0 = 0usize; + let mut dist1 = 0usize; + let dev0_chunks = ((num_chunks * DEV0_CHUNK_SHARE_PERCENT) / 100) as usize; + let dev1_start = dev0_chunks; + let dev1_chunks = num_chunks as usize - dev0_chunks; + + // Fill ring0 with the device 0 share + for chunk_start in (0..dev0_chunks).step_by(BATCH_SIZE) { + let chunk_end = (chunk_start + BATCH_SIZE).min(dev0_chunks); + let batch_size = chunk_end - chunk_start; + + for (i, addr) in buf.iter_mut().enumerate().take(batch_size) { + *addr = ((chunk_start + i) as u64) * chunk_size; + } + + dist0 += fill0.produce(&buf[..batch_size]); + } + + // Fill ring1 with the remaining share for device 1 + for chunk_start in (dev1_start..(dev1_start + dev1_chunks)).step_by(BATCH_SIZE) { + let chunk_end = (chunk_start + BATCH_SIZE).min(dev1_start + dev1_chunks); + let batch_size = chunk_end - chunk_start; + + for (i, addr) in buf.iter_mut().enumerate().take(batch_size) { + *addr = ((chunk_start + i) as u64) * chunk_size; + } + + dist1 += fill1.produce(&buf[..batch_size]); + } + + println!(" Chunks allocated: {}", num_chunks); + println!( + " FILL rings populated ({} to ring0, {} to ring1, dev0 share={}%)", + dist0, dist1, DEV0_CHUNK_SHARE_PERCENT + ); + + // Start devices with NAPI CPU configuration + dev0.start(CPU_NAPI_DEV0)?; + dev1.start(CPU_NAPI_DEV1)?; + println!(" Devices started"); + if CPU_NAPI_DEV0 >= 0 { + println!( + " rattan0 NAPI CPU: {} (client-side injection)", + CPU_NAPI_DEV0 + ); + } else { + println!(" rattan0 NAPI: use caller's CPU"); + } + if CPU_NAPI_DEV1 >= 0 { + println!( + " rattan1 NAPI CPU: {} (server-side injection)", + CPU_NAPI_DEV1 + ); + } else { + println!(" rattan1 NAPI: use caller's CPU"); + } + + // Step 3: Move interfaces to namespaces and configure + println!("\n3. Configuring network namespaces..."); + + move_to_netns("rattan0", "ns_client")?; + println!(" Moved rattan0 to ns_client"); + + move_to_netns("rattan1", "ns_server")?; + println!(" Moved rattan1 to ns_server"); + + configure_interface_in_netns("ns_client", "rattan0", "10.0.0.2/24")?; + println!(" ns_client: rattan0 = 10.0.0.2/24"); + + configure_interface_in_netns("ns_server", "rattan1", "10.0.0.1/24")?; + println!(" ns_server: rattan1 = 10.0.0.1/24"); + + // Step 4: Start forwarding loop + println!("\n4. Starting forwarding loop..."); + + let running = Arc::new(AtomicBool::new(true)); + let running_clone = running.clone(); + + let packets_forwarded = Arc::new(AtomicU64::new(0)); + let packets_forwarded_clone = packets_forwarded.clone(); + let packets_dropped = Arc::new(AtomicU64::new(0)); + let packets_dropped_clone = packets_dropped.clone(); + + // Get file descriptors for epoll + let dev0_fd = dev0.as_raw_fd(); + let dev1_fd = dev1.as_raw_fd(); + + let mut round_count = 0; + + // Single-threaded forwarder using epoll, pinned to CPU + let forward_thread = std::thread::spawn(move || { + // Pin to dedicated CPU core + if let Err(e) = set_cpu_affinity(CPU_FORWARDER) { + eprintln!("Warning: failed to set CPU affinity: {}", e); + } + + let mut descs = [RattanDesc::default(); BATCH_SIZE]; + // let mut addrs = [0u64; BATCH_SIZE]; + + // Create epoll instance + let epfd = unsafe { libc::epoll_create1(0) }; + if epfd < 0 { + eprintln!("epoll_create1 failed"); + return (dev0, dev1); + } + + // Add dev0 to epoll (data.u32 = 0) + let mut ev = libc::epoll_event { + events: libc::EPOLLIN as u32, + u64: 0, + }; + if unsafe { libc::epoll_ctl(epfd, libc::EPOLL_CTL_ADD, dev0_fd, &mut ev) } < 0 { + eprintln!("epoll_ctl add dev0 failed"); + unsafe { libc::close(epfd) }; + return (dev0, dev1); + } + + // Add dev1 to epoll (data.u32 = 1) + ev.u64 = 1; + if unsafe { libc::epoll_ctl(epfd, libc::EPOLL_CTL_ADD, dev1_fd, &mut ev) } < 0 { + eprintln!("epoll_ctl add dev1 failed"); + unsafe { libc::close(epfd) }; + return (dev0, dev1); + } + + let mut events = [libc::epoll_event { events: 0, u64: 0 }; 2]; + + while running_clone.load(Ordering::Relaxed) { + // Wait for events on either device (10ms timeout) + let nfds = unsafe { libc::epoll_wait(epfd, events.as_mut_ptr(), 2, 10) }; + + if nfds < 0 { + let err = std::io::Error::last_os_error(); + if err.kind() != std::io::ErrorKind::Interrupted { + eprintln!("epoll_wait failed: {}", err); + break; + } + continue; + } + + // Process events - drain RX rings completely + for event in events.iter().take(nfds as usize) { + let dev_idx = event.u64; + + if dev_idx == 0 { + // dev0 has RX data: forward rx0 → tx1 + // Drain all available packets (not just one batch) + loop { + round_count += 1; + + if round_count & 15 == 0 && round_count > 100 { + // Receive and drop! + let drop = drop_batch(&mut rx0, &mut drop0, &mut descs); + packets_dropped_clone.fetch_add(drop as u64, Ordering::Relaxed); + continue; + } + + let fwd = forward_batch(&mut rx0, &mut tx1, &mut descs); + if fwd > 0 { + packets_forwarded_clone.fetch_add(fwd as u64, Ordering::Relaxed); + let _ = dev1.kick_rx_queue(0); + } + if fwd < BATCH_SIZE { + break; // Ring drained + } + } + } else { + // dev1 has RX data: forward rx1 → tx0 + // Drain all available packets (not just one batch) + loop { + let fwd = forward_batch(&mut rx1, &mut tx0, &mut descs); + if fwd > 0 { + packets_forwarded_clone.fetch_add(fwd as u64, Ordering::Relaxed); + let _ = dev0.kick_rx_queue(0); + } + if fwd < BATCH_SIZE { + break; // Ring drained + } + } + } + } + + // Always recycle chunks back to the device that originally supplied + // them. With current kernel completion routing, chunks are + // ingress-owned and completions return to the originating + // device/queue, not the forwarding destination. + // loop { + // let r0 = recycle_chunks(&mut comp0, &mut fill0, &mut addrs); + // let r1 = recycle_chunks(&mut comp1, &mut fill1, &mut addrs); + // if r0 == 0 && r1 == 0 { + // break; + // } + // } + } + + unsafe { libc::close(epfd) }; + (dev0, dev1) + }); + + println!( + " Forwarding loop started (single thread, epoll, CPU {})", + CPU_FORWARDER + ); + + // Step 5: Run iperf test + println!("\n5. Running iperf3 performance test..."); + println!( + " Starting iperf3 server in ns_server (CPU {})...", + CPU_IPERF_SERVER + ); + let mut server = start_iperf_server_in_netns("ns_server", CPU_IPERF_SERVER)?; + + std::thread::sleep(Duration::from_millis(500)); + + let parallel_streams = 4; + println!( + " Running iperf3 client from ns_client (10 seconds, {} parallel streams, CPU {})...\n", + parallel_streams, CPU_IPERF_CLIENT + ); + let start = Instant::now(); + + match run_iperf_client_in_netns( + "ns_client", + "10.0.0.1", + 10, + parallel_streams, + CPU_IPERF_CLIENT, + ) { + Ok(output) => println!("{}", output), + Err(e) => eprintln!(" iperf3 client error: {}", e), + } + + let elapsed = start.elapsed(); + println!("\n Test completed in {:.2}s", elapsed.as_secs_f64()); + + let _ = server.wait(); + + // Step 6: Print statistics + println!("\n6. Statistics:"); + let total_forwarded = packets_forwarded.load(Ordering::Relaxed); + println!(" Packets forwarded: {}", total_forwarded); + + let total_dropped = packets_dropped.load(Ordering::Relaxed); + println!(" Packets dropped: {}", total_dropped); + + // Stop forwarding + println!("\n7. Cleaning up..."); + running.store(false, Ordering::Relaxed); + + let (_dev0, _dev1) = forward_thread.join().unwrap(); + + // netns_guard will clean up namespaces on drop + drop(netns_guard); + println!(" Namespaces deleted"); + + println!(" Done"); + println!("\n=== Test Complete ==="); + + Ok(()) +} diff --git a/rvnic/librvnic/examples/test_forward_multiqueue.rs b/rvnic/librvnic/examples/test_forward_multiqueue.rs new file mode 100644 index 00000000..54f47552 --- /dev/null +++ b/rvnic/librvnic/examples/test_forward_multiqueue.rs @@ -0,0 +1,537 @@ +//! Test multiqueue packet forwarding between two vNICs with shared UMEM. +//! +//! This example demonstrates the ergonomic multiqueue-facing `librvnic` API: +//! +//! - one shared UMEM +//! - one ring set per queue +//! - `QueueConfig` / `register_queues(...)` +//! - `QueueRings` bundles for per-queue operation +//! - queue-specific `kick_rx_queue(...)` +//! - one queue-worker thread per queue +//! +//! Topology: +//! [ns_client] [ns_server] +//! iperf3 -c 10.0.0.1 --> rattan0[q0..q3] --> [forward] --> rattan1[q0..q3] --> iperf3 -s +//! +//! Chunk ownership model: +//! - chunks are owned by the ingress device/queue that originally supplied them +//! - completions return to that same queue's COMP ring +//! - userspace should recycle `comp -> fill` on the same queue +//! +//! Notes: +//! - This example assumes the kernel module is loaded with `num_queues=4`. +//! - Queue count is not yet discoverable from userspace, so it is configured here +//! explicitly via `NUM_QUEUES`. +//! - The initial UMEM chunk split between device 0 and device 1 is controlled by +//! `DEV0_CHUNK_SHARE_PERCENT` below so it can be tuned later. +//! - Queue workers use `napi_cpu = -1`, so each `kick_rx_queue(...)` schedules +//! NAPI on the calling queue worker's CPU. +//! - This is a manual example. Do not run it automatically in tests. +//! +//! Example module load: +//! sudo insmod rattan_vnic.ko num_queues=4 +//! +//! Run manually with: +//! sudo cargo run --example test_forward_multiqueue +//! +//! Prerequisites: +//! 1. Build and load the kernel module with `num_queues=4` +//! 2. Install iperf3: sudo apt-get install iperf3 +//! 3. Ensure `/dev/rattan-vnic` exists + +use rvnic::sys::{RattanDesc, ioctl_kick_rx_q}; +use rvnic::{ + QueueConfig, QueueRings, RATTAN_DEFAULT_CHUNK_SIZE, RATTAN_RING_SIZE, Rings, RvnicDevice, + UmemBuilder, +}; + +use std::process::{Child, Command, Stdio}; +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; +use std::time::{Duration, Instant}; + +const NUM_QUEUES: u32 = 4; +const BATCH_SIZE: usize = 256; + +// CPU layout for a 16-core machine: +// CPU 0-3: queue workers for q0..q3 (also drive NAPI via kick_rx_queue) +// CPU 4-9: iperf client CPU set +// CPU 10-15: iperf server CPU set +const QUEUE_WORKER_CPUS: [u32; NUM_QUEUES as usize] = [0, 1, 2, 3]; +const CPUSET_IPERF_CLIENT: &str = "4-9"; +const CPUSET_IPERF_SERVER: &str = "10-15"; +const NAPI_CPU: i32 = -1; + +struct NetnsGuard { + names: Vec<&'static str>, +} + +impl NetnsGuard { + fn new() -> Self { + Self { names: Vec::new() } + } + + fn add(&mut self, name: &'static str) -> std::io::Result<()> { + let _ = Command::new("ip").args(["netns", "delete", name]).status(); + let status = Command::new("ip").args(["netns", "add", name]).status()?; + if !status.success() { + return Err(std::io::Error::other(format!( + "failed to create netns {}", + name + ))); + } + self.names.push(name); + Ok(()) + } +} + +impl Drop for NetnsGuard { + fn drop(&mut self) { + for name in &self.names { + let _ = Command::new("ip").args(["netns", "delete", name]).status(); + } + } +} + +fn move_to_netns(iface: &str, netns: &str) -> std::io::Result<()> { + let status = Command::new("ip") + .args(["link", "set", iface, "netns", netns]) + .status()?; + if !status.success() { + return Err(std::io::Error::other(format!( + "failed to move {} to {}", + iface, netns + ))); + } + Ok(()) +} + +fn configure_interface_in_netns(netns: &str, iface: &str, ip: &str) -> std::io::Result<()> { + let status = Command::new("ip") + .args([ + "netns", "exec", netns, "ip", "addr", "add", ip, "dev", iface, + ]) + .status()?; + if !status.success() { + return Err(std::io::Error::other(format!( + "failed to add IP {} to {} in {}", + ip, iface, netns + ))); + } + + let status = Command::new("ip") + .args(["netns", "exec", netns, "ip", "link", "set", iface, "up"]) + .status()?; + if !status.success() { + return Err(std::io::Error::other(format!( + "failed to bring up {} in {}", + iface, netns + ))); + } + + let _ = Command::new("ip") + .args(["netns", "exec", netns, "ip", "link", "set", "lo", "up"]) + .status(); + + Ok(()) +} + +fn start_iperf_server_in_netns(netns: &str, cpuset: &str) -> std::io::Result { + Command::new("taskset") + .args([ + "-c", cpuset, "ip", "netns", "exec", netns, "iperf3", "-s", "-1", + ]) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() +} + +fn run_iperf_client_in_netns( + netns: &str, + server_addr: &str, + duration: u32, + parallel: u32, + cpuset: &str, +) -> std::io::Result { + let duration_str = duration.to_string(); + let parallel_str = parallel.to_string(); + + let mut args = vec![ + "-c", + cpuset, + "ip", + "netns", + "exec", + netns, + "iperf3", + "-c", + server_addr, + "-t", + &duration_str, + "-i", + "1", + "--forceflush", + ]; + + if parallel > 1 { + args.extend(["-P", ¶llel_str]); + } + + if parallel == 4 { + args.extend(["--cport", "3333"]); + } + + let output = Command::new("taskset").args(&args).output()?; + let stdout = String::from_utf8_lossy(&output.stdout); + let stderr = String::from_utf8_lossy(&output.stderr); + + if !output.status.success() { + return Err(std::io::Error::other(format!( + "iperf3 client failed: {}", + stderr + ))); + } + + Ok(stdout.to_string()) +} + +fn set_cpu_affinity(cpu: u32) -> std::io::Result<()> { + unsafe { + let mut cpuset: libc::cpu_set_t = std::mem::zeroed(); + libc::CPU_ZERO(&mut cpuset); + libc::CPU_SET(cpu as usize, &mut cpuset); + + let ret = libc::sched_setaffinity(0, std::mem::size_of::(), &cpuset); + if ret != 0 { + return Err(std::io::Error::last_os_error()); + } + } + Ok(()) +} + +#[inline] +fn forward_batch( + rx: &mut rvnic::RxRing, + tx: &mut rvnic::TxRing, + descs: &mut [RattanDesc], +) -> usize { + let received = rx.consume(descs); + if received > 0 { + tx.produce(&descs[..received]) + } else { + 0 + } +} + +// #[inline] +// fn recycle_chunks( +// comp: &mut rvnic::CompRing, +// fill: &mut rvnic::FillRing, +// buf: &mut [u64], +// ) -> usize { +// let n = comp.consume(buf); +// if n > 0 { fill.produce(&buf[..n]) } else { 0 } +// } + +fn build_queue_configs(num_queues: u32) -> Result, Box> { + let mut queues = Vec::with_capacity(num_queues as usize); + for queue_id in 0..num_queues { + queues.push(QueueConfig { + queue_id, + rings: Rings::new()?, + }); + } + Ok(queues) +} + +fn prefill_device_queues( + queues: &mut [QueueRings], + start_chunk: u32, + num_chunks: u32, + chunk_size: u64, +) -> usize { + let mut total = 0usize; + let mut addrs = [0u64; BATCH_SIZE]; + + let end_chunk = start_chunk + num_chunks; + + let mut queue_index = (0..queues.len()).cycle(); + + for batch_chunk_start in (start_chunk..end_chunk).step_by(BATCH_SIZE) { + let batch_chunk_end = (batch_chunk_start + BATCH_SIZE as u32).min(end_chunk); + let batch_size = (batch_chunk_end - batch_chunk_start) as usize; + + for (i, addr) in addrs.iter_mut().enumerate().take(batch_size) { + *addr = (batch_chunk_start + i as u32) as u64 * chunk_size; + } + + let queue = queue_index.next().unwrap(); + + let produced = queues[queue].fill.produce(&addrs[..batch_size]); + total += produced; + } + + assert_eq!(num_chunks, total as u32); + + total +} + +fn drain_one_direction( + src: &mut QueueRings, + dst: &mut QueueRings, + dst_fd: i32, + descs: &mut [RattanDesc], + packets_forwarded: &AtomicU64, +) -> usize { + let mut total = 0usize; + + loop { + let forwarded = forward_batch(&mut src.rx, &mut dst.tx, descs); + if forwarded > 0 { + packets_forwarded.fetch_add(forwarded as u64, Ordering::Relaxed); + unsafe { + let _ = ioctl_kick_rx_q(dst_fd, dst.queue_id); + } + total += forwarded; + } + if forwarded < BATCH_SIZE { + break; + } + } + + total +} + +// fn drain_one_queue_recycle(queue: &mut QueueRings, addrs: &mut [u64]) -> usize { +// let mut recycled = 0usize; + +// loop { +// let n = recycle_chunks(&mut queue.comp, &mut queue.fill, addrs); +// recycled += n; +// if n < addrs.len() { +// break; +// } +// } + +// recycled +// } + +fn main() -> Result<(), Box> { + println!("=== Rattan vNIC Multiqueue Forwarding Test ===\n"); + println!("Configured queue count: {}", NUM_QUEUES); + println!( + "This example assumes the kernel module was loaded with num_queues={}.\n", + NUM_QUEUES + ); + println!( + "CPU layout: queue workers={:?}, client cpuset={}, server cpuset={}\n", + QUEUE_WORKER_CPUS, CPUSET_IPERF_CLIENT, CPUSET_IPERF_SERVER + ); + + println!("1. Creating network namespaces..."); + let mut netns_guard = NetnsGuard::new(); + netns_guard.add("ns_client")?; + netns_guard.add("ns_server")?; + println!(" Created ns_client and ns_server"); + + println!("\n2. Setting up multiqueue vNIC devices..."); + + let mut dev0 = RvnicDevice::open()?; + println!(" rattan0 opened (fd={})", dev0.fd()); + + let mut dev1 = RvnicDevice::open()?; + println!(" rattan1 opened (fd={})", dev1.fd()); + + let chunks_per_queue = RATTAN_RING_SIZE; + + let chunks_per_dev = chunks_per_queue * NUM_QUEUES; + let total_chunks = chunks_per_dev * 2; + + let umem = UmemBuilder::new() + .chunk_size(RATTAN_DEFAULT_CHUNK_SIZE) + .headroom(128) + .num_chunks(total_chunks) + .build()?; + + println!( + " UMEM allocated: {} chunks x {} bytes", + umem.num_chunks(), + umem.chunk_size() + ); + + let chunk_size = umem.chunk_size() as u64; + let num_chunks = umem.num_chunks(); + + dev0.register_umem(umem)?; + dev1.share_umem(&dev0)?; + println!(" UMEM registered and shared"); + + let dev0_configs = build_queue_configs(NUM_QUEUES)?; + let dev1_configs = build_queue_configs(NUM_QUEUES)?; + + let mut dev0_queues = dev0.register_queues(dev0_configs)?; + let mut dev1_queues = dev1.register_queues(dev1_configs)?; + println!(" Registered {} queues on each device", NUM_QUEUES); + + let dev0_chunks = chunks_per_dev; + let dev1_chunks = num_chunks - dev0_chunks; + let dev0_fill = prefill_device_queues(&mut dev0_queues, 0, dev0_chunks, chunk_size); + let dev1_fill = prefill_device_queues(&mut dev1_queues, dev0_chunks, dev1_chunks, chunk_size); + + println!( + " Pre-filled FILL rings: {} chunks to dev0, {} chunks to dev1 ", + dev0_fill, dev1_fill, + ); + + dev0.start(NAPI_CPU)?; + dev1.start(NAPI_CPU)?; + println!(" Devices started"); + println!(" dev0 NAPI CPU: {}", NAPI_CPU); + println!(" dev1 NAPI CPU: {}", NAPI_CPU); + + println!("\n3. Configuring network namespaces..."); + + move_to_netns("rattan0", "ns_client")?; + println!(" Moved rattan0 to ns_client"); + + move_to_netns("rattan1", "ns_server")?; + println!(" Moved rattan1 to ns_server"); + + configure_interface_in_netns("ns_client", "rattan0", "10.0.0.2/24")?; + println!(" ns_client: rattan0 = 10.0.0.2/24"); + + configure_interface_in_netns("ns_server", "rattan1", "10.0.0.1/24")?; + println!(" ns_server: rattan1 = 10.0.0.1/24"); + + println!("\n4. Starting multiqueue forwarding loop..."); + + let running = Arc::new(AtomicBool::new(true)); + let running_clone = Arc::clone(&running); + + let packets_forwarded_cnts: Vec<_> = (0..NUM_QUEUES) + .map(|_| Arc::new(AtomicU64::new(0))) + .collect(); + + let dev0_fd = dev0.fd(); + let dev1_fd = dev1.fd(); + + let mut dev0_workers: Vec> = dev0_queues.into_iter().map(Some).collect(); + let mut dev1_workers: Vec> = dev1_queues.into_iter().map(Some).collect(); + let mut forward_threads = Vec::with_capacity(NUM_QUEUES as usize); + + for (queue_idx, cpu) in QUEUE_WORKER_CPUS.iter().copied().enumerate() { + let mut dev0_queue = dev0_workers[queue_idx].take().unwrap(); + let mut dev1_queue = dev1_workers[queue_idx].take().unwrap(); + let running = Arc::clone(&running_clone); + let packets_forwarded = packets_forwarded_cnts[queue_idx].clone(); + + let handle = std::thread::spawn(move || { + if let Err(e) = set_cpu_affinity(cpu) { + eprintln!( + "warning: failed to set CPU affinity for queue {}: {}", + queue_idx, e + ); + } + + let mut descs = [RattanDesc::default(); BATCH_SIZE]; + let mut _addrs = [0u64; BATCH_SIZE]; + + while running.load(Ordering::Relaxed) { + let mut progressed = 0usize; + + progressed += drain_one_direction( + &mut dev0_queue, + &mut dev1_queue, + dev1_fd, + &mut descs, + &packets_forwarded, + ); + + progressed += drain_one_direction( + &mut dev1_queue, + &mut dev0_queue, + dev0_fd, + &mut descs, + &packets_forwarded, + ); + + // progressed += drain_one_queue_recycle(&mut dev0_queue, &mut addrs); + // progressed += drain_one_queue_recycle(&mut dev1_queue, &mut addrs); + + if progressed == 0 { + std::thread::yield_now(); + } + } + + (dev0_queue, dev1_queue) + }); + + forward_threads.push(handle); + } + + println!( + " Forwarding started ({} queue-worker threads on CPUs {:?})", + NUM_QUEUES, QUEUE_WORKER_CPUS + ); + + println!("\n5. Running iperf3 performance test..."); + println!( + " Starting iperf3 server in ns_server (CPU set {})...", + CPUSET_IPERF_SERVER + ); + let mut server = start_iperf_server_in_netns("ns_server", CPUSET_IPERF_SERVER)?; + + std::thread::sleep(Duration::from_millis(500)); + + let parallel_streams = 4; + println!( + " Running iperf3 client from ns_client (10 seconds, {} parallel streams, CPU set {})...\n", + parallel_streams, CPUSET_IPERF_CLIENT + ); + let start = Instant::now(); + + match run_iperf_client_in_netns( + "ns_client", + "10.0.0.1", + 10, + parallel_streams, + CPUSET_IPERF_CLIENT, + ) { + Ok(output) => println!("{}", output), + Err(e) => eprintln!(" iperf3 client error: {}", e), + } + + let elapsed = start.elapsed(); + println!("\n Test completed in {:.2}s", elapsed.as_secs_f64()); + + let _ = server.wait(); + + println!("\n6. Statistics:"); + + for queue_index in 0..NUM_QUEUES { + let packets_forwarded = packets_forwarded_cnts + .get(queue_index as usize) + .unwrap() + .clone(); + let total_forwarded = packets_forwarded.load(Ordering::Relaxed); + println!( + " Packets forwarded on queue {}: {}", + queue_index, total_forwarded + ); + } + + println!("\n7. Cleaning up..."); + running.store(false, Ordering::Relaxed); + + for handle in forward_threads { + let (_dev0_queue, _dev1_queue) = handle.join().unwrap(); + } + + drop(netns_guard); + println!(" Namespaces deleted"); + + println!(" Done"); + println!("\n=== Test Complete ==="); + + Ok(()) +} diff --git a/rvnic/librvnic/examples/test_token.rs b/rvnic/librvnic/examples/test_token.rs new file mode 100644 index 00000000..b305a25c --- /dev/null +++ b/rvnic/librvnic/examples/test_token.rs @@ -0,0 +1,325 @@ +//! Test token semantics: chunk-bound tokens, CLONE, and misbehaving userspace +//! +//! Tokens are `gen << 32 | chunk_index`, one per UMEM chunk. This test checks +//! that encoding, and that every way userspace can misuse a token or a chunk +//! fails cleanly instead of corrupting kernel state: +//! +//! - READ_PAYLOAD / RELEASE / CLONE on live and on stale tokens +//! - CLONE onto a chunk that is still live (EBUSY) or misaligned (EINVAL) +//! - replaying a token, and putting one token on the TX ring twice +//! - releasing a token via DROP and transmitting it at the same time +//! - putting a live chunk back on FILL +//! - closing a device while its tokens are pending, then releasing them +//! through the device that shares the UMEM +//! +//! Both devices stay in the root netns. Packets are UDP datagrams sent to a +//! permanent neighbour, so nothing but our own traffic reaches the rings. +//! +//! Run with: sudo ./target/release/examples/test_token +//! Assumes the kernel module is loaded with `num_queues=1`. + +use rvnic::sys::RattanDesc; +use rvnic::{Rings, RvnicDevice, RxRing, UmemBuilder}; +use std::net::UdpSocket; +use std::process::Command; +use std::thread::sleep; +use std::time::{Duration, Instant}; + +const CHUNK_SIZE: u64 = 2048; +const NUM_CHUNKS: u32 = 64; +const MARKER: &[u8] = b"rvnic-token-test"; + +// Chunks 0..28 go to rattan0's FILL. 30 and 31 are kept back for CLONE. +// rattan1's FILL is left empty until the live-chunk test. +const DEV0_FILL: u32 = 28; +const CLONE_CHUNK: u64 = 30 * CHUNK_SIZE; + +fn sh(args: &[&str]) { + let status = Command::new(args[0]).args(&args[1..]).status().unwrap(); + assert!(status.success(), "command failed: {:?}", args); +} + +fn stat(iface: &str, name: &str) -> u64 { + std::fs::read_to_string(format!("/sys/class/net/{}/statistics/{}", iface, name)) + .unwrap() + .trim() + .parse() + .unwrap() +} + +/// Packets NAPI handed to the stack on `iface`, accepted or not +fn delivered(iface: &str) -> u64 { + stat(iface, "rx_packets") + stat(iface, "rx_dropped") +} + +fn setup_iface(iface: &str, addr: &str, peer: &str) { + sh(&[ + "sysctl", + "-q", + &format!("net.ipv6.conf.{}.disable_ipv6=1", iface), + ]); + sh(&["ip", "addr", "add", addr, "dev", iface]); + sh(&["ip", "link", "set", iface, "up"]); + sh(&[ + "ip", + "neigh", + "replace", + peer, + "lladdr", + "02:00:00:00:00:02", + "dev", + iface, + "nud", + "permanent", + ]); +} + +fn send(src: &str, dst: &str, seq: u32) { + let sock = UdpSocket::bind(format!("{}:0", src)).unwrap(); + let mut payload = MARKER.to_vec(); + payload.extend_from_slice(&seq.to_le_bytes()); + sock.send_to(&payload, format!("{}:9999", dst)).unwrap(); +} + +fn payload(dev: &RvnicDevice, token: u64) -> std::io::Result> { + let mut buf = vec![0u8; 2048]; + let (len, _total) = dev + .read_payload(token, &mut buf, 0) + .map_err(|e| std::io::Error::other(format!("{:?}", e)))?; + buf.truncate(len as usize); + Ok(buf) +} + +fn is_ours(pkt: &[u8]) -> bool { + pkt.windows(MARKER.len()).any(|w| w == MARKER) +} + +/// Wait for `n` of our packets on `rx`, releasing anything else that shows up +fn recv(dev: &RvnicDevice, rx: &mut RxRing, n: usize) -> Vec { + let mut out = Vec::new(); + let mut descs = [RattanDesc::default(); 16]; + let deadline = Instant::now() + Duration::from_secs(2); + while out.len() < n { + assert!( + Instant::now() < deadline, + "timed out, got {}/{} packets", + out.len(), + n + ); + let got = rx.consume(&mut descs); + for d in &descs[..got] { + if is_ours(&payload(dev, d.token).unwrap()) { + out.push(*d); + } else { + dev.release(d.token).unwrap(); + } + } + if got == 0 { + sleep(Duration::from_millis(5)); + } + } + out +} + +fn errno(r: rvnic::Result) -> i32 { + match r { + Err(rvnic::Error::Io(e)) => e.raw_os_error().unwrap(), + other => panic!("expected an errno, got {:?}", other), + } +} + +fn main() -> Result<(), Box> { + println!("=== Rattan vNIC token test ===\n"); + + let mut dev0 = RvnicDevice::open()?; + let mut dev1 = RvnicDevice::open()?; + let umem = UmemBuilder::new() + .chunk_size(CHUNK_SIZE as u32) + .headroom(0) + .num_chunks(NUM_CHUNKS) + .build()?; + dev0.register_umem(umem)?; + dev1.share_umem(&dev0)?; + let (mut fill0, mut drop0, mut rx0, _tx0) = dev0.register_rings_for_queue(0, Rings::new()?)?; + let (mut fill1, mut drop1, mut rx1, mut tx1) = + dev1.register_rings_for_queue(0, Rings::new()?)?; + + let addrs: Vec = (0..DEV0_FILL as u64).map(|i| i * CHUNK_SIZE).collect(); + assert_eq!(fill0.produce(&addrs), addrs.len()); + + dev0.start(-1)?; + dev1.start(-1)?; + let if0 = dev0.device_name()?; + let if1 = dev1.device_name()?; + setup_iface(&if0, "10.9.0.1/24", "10.9.0.2"); + setup_iface(&if1, "10.9.1.1/24", "10.9.1.2"); + sleep(Duration::from_millis(100)); + + // 1. Encoding + for seq in 0..5 { + send("10.9.0.1", "10.9.0.2", seq); + } + let pkts = recv(&dev0, &mut rx0, 5); + for d in &pkts { + assert_eq!( + d.token as u32 as u64, + d.addr / CHUNK_SIZE, + "token index != chunk index" + ); + assert_ne!(d.token >> 32, 0, "generation 0 issued"); + } + println!("1. tokens are gen<<32 | chunk index: ok"); + let [t1, t2, t3, t4, t5] = [0, 1, 2, 3, 4].map(|i| pkts[i]); + + // 2. CLONE + assert!(errno(dev0.clone_token(t1.token, t2.addr, 0)) == libc::EBUSY); + assert!(errno(dev0.clone_token(t1.token, CLONE_CHUNK + 1, 0)) == libc::EINVAL); + assert!(errno(dev0.clone_token(t1.token, CLONE_CHUNK, 7)) == libc::EINVAL); + assert!(errno(dev0.clone_token(t1.token + (1 << 32), CLONE_CHUNK, 0)) == libc::ENOENT); + let c1 = dev0.clone_token(t1.token, CLONE_CHUNK, 0)?; + assert_eq!(c1 as u32 as u64, CLONE_CHUNK / CHUNK_SIZE); + assert_eq!(payload(&dev0, c1)?, payload(&dev0, t1.token)?); + // The token also resolves through the device sharing the UMEM + assert_eq!(payload(&dev1, c1)?, payload(&dev0, t1.token)?); + println!("2. CLONE: EBUSY/EINVAL/ENOENT on bad input, copy bound to the given chunk: ok"); + + // 3. RELEASE, then everything with the stale token misses + dev0.release(t2.token)?; + assert!(errno(dev0.release(t2.token)) == libc::ENOENT); + assert!(payload(&dev0, t2.token).is_err()); + assert!(errno(dev0.clone_token(t2.token, CLONE_CHUNK + CHUNK_SIZE, 0)) == libc::ENOENT); + println!("3. released token is stale for RELEASE/READ_PAYLOAD/CLONE: ok"); + + // 4. A live chunk on FILL: the xmit must be refused, the live token untouched. + // Runs before any miss below can complete stray chunks into rattan1's COMP, + // which xmit would pop ahead of FILL. + let before = payload(&dev0, t5.token)?; + let dropped = stat(&if1, "tx_dropped"); + assert_eq!(fill1.produce(&[t5.addr]), 1); + send("10.9.1.1", "10.9.1.2", 200); + sleep(Duration::from_millis(100)); + assert_eq!( + stat(&if1, "tx_dropped") - dropped, + 1, + "xmit onto a live chunk not refused" + ); + assert_eq!(payload(&dev0, t5.token)?, before, "live token clobbered"); + let mut descs = [RattanDesc::default(); 4]; + assert_eq!( + rx1.consume(&mut descs), + 0, + "packet published on a live chunk" + ); + println!("4. live chunk on FILL refused, live token intact: ok"); + + // 4b. Addresses past the UMEM, including ones where addr + chunk_size + // wraps around u64, must be rejected rather than become a slot index. + // rattan1's COMP is still empty, so xmit takes them from FILL. + let wrap = 0u64.wrapping_sub(CHUNK_SIZE); + let end = CHUNK_SIZE * NUM_CHUNKS as u64; + for bad in [wrap, end] { + assert_eq!(errno(dev0.clone_token(t1.token, bad, 0)), libc::EINVAL); + } + let dropped = stat(&if1, "tx_dropped"); + assert_eq!(fill1.produce(&[wrap, end]), 2); + send("10.9.1.1", "10.9.1.2", 202); + send("10.9.1.1", "10.9.1.2", 203); + sleep(Duration::from_millis(100)); + assert_eq!( + stat(&if1, "tx_dropped") - dropped, + 2, + "out-of-range chunk not refused" + ); + assert_eq!( + rx1.consume(&mut descs), + 0, + "packet published on an out-of-range chunk" + ); + println!("4b. out-of-range and wrapping chunk addresses refused: ok"); + + // 5. Forward t1 and its clone through rattan1, then replay t1, and put t3 on TX twice + let base = delivered(&if1); + assert_eq!( + tx1.produce(&[ + t1, + RattanDesc { + token: c1, + addr: CLONE_CHUNK, + ..t1 + } + ]), + 2 + ); + dev1.kick_rx_queue(0)?; + sleep(Duration::from_millis(100)); + assert_eq!(delivered(&if1) - base, 2, "forwarded packets not delivered"); + assert!( + payload(&dev0, t1.token).is_err(), + "forwarded token still live" + ); + + let base = delivered(&if1); + assert_eq!(tx1.produce(&[t1, t3, t3]), 3); + dev1.kick_rx_queue(0)?; + sleep(Duration::from_millis(100)); + assert_eq!( + delivered(&if1) - base, + 1, + "replayed or duplicated token delivered" + ); + println!("5. forward ok; replayed token and duplicate TX entry miss: ok"); + + // 6. The same token on DROP and TX: exactly one of them takes it + let base = delivered(&if1); + assert_eq!(drop0.produce(&[t4.token]), 1); + assert_eq!(tx1.produce(&[t4]), 1); + dev1.kick_rx_queue(0)?; + send("10.9.0.1", "10.9.0.2", 100); // an xmit on rattan0 drains its DROP ring + sleep(Duration::from_millis(100)); + let fwd = delivered(&if1) - base; + assert!(fwd <= 1); + assert!( + payload(&dev0, t4.token).is_err(), + "token survived both DROP and TX" + ); + let extra = recv(&dev0, &mut rx0, 1); + println!( + "6. DROP + TX of one token: taken once (forwarded={}): ok", + fwd + ); + + // 7. Close rattan0 while three of its tokens are pending. They stay usable + // through rattan1, which shares the UMEM. Finish them via the RELEASE + // ioctl, the DROP ring (softirq) and finally NAPI (softirq), which drops + // the last reference on rattan0's rings: the cleanup, which sleeps, + // must be deferred rather than run there. + send("10.9.0.1", "10.9.0.2", 300); + let last = recv(&dev0, &mut rx0, 1)[0]; + drop((fill0, drop0, rx0)); + drop(dev0); + assert_eq!(payload(&dev1, t5.token)?, before); + dev1.release(t5.token)?; + + assert_eq!(drop1.produce(&[extra[0].token]), 1); + send("10.9.1.1", "10.9.1.2", 301); // an xmit on rattan1 drains its DROP ring + sleep(Duration::from_millis(100)); + assert!( + payload(&dev1, extra[0].token).is_err(), + "DROP ring did not release" + ); + + let base = delivered(&if1); + assert_eq!(tx1.produce(&[last]), 1); + dev1.kick_rx_queue(0)?; + sleep(Duration::from_millis(100)); + assert_eq!(delivered(&if1) - base, 1, "orphaned token not forwarded"); + println!("7. tokens outlive their device; last one freed by NAPI: ok"); + + // 8. Close the rest + drop((fill1, drop1, rx1, tx1)); + drop(dev1); + println!("8. closed (dmesg must show no BUG/WARN)"); + + println!("\n=== Test Passed ==="); + Ok(()) +} diff --git a/rvnic/librvnic/examples/test_umem.rs b/rvnic/librvnic/examples/test_umem.rs new file mode 100644 index 00000000..f1ada99f --- /dev/null +++ b/rvnic/librvnic/examples/test_umem.rs @@ -0,0 +1,147 @@ +//! Test UMEM and rings registration with the rattan-vnic kernel module +//! +//! This test creates TWO vNIC devices (rattan0 and rattan1) sharing a single UMEM. +//! This validates the shared UMEM functionality required for packet forwarding. +//! +//! This example uses the queue-0 convenience API: +//! - `register_rings(...)` +//! - `kick_rx()` +//! +//! So it assumes the kernel module is loaded in single-queue mode: +//! sudo insmod rattan_vnic.ko num_queues=1 +//! +//! For a multiqueue setup, see: +//! examples/test_forward_multiqueue.rs +//! +//! Run this with: sudo ./target/debug/examples/test_umem +//! +//! Prerequisites: +//! 1. Build and load the kernel module in single-queue mode: +//! cd rvnic/kernel && make && sudo insmod rattan_vnic.ko num_queues=1 +//! 2. Verify device exists: +//! ls -la /dev/rattan-vnic + +use rvnic::{Rings, RvnicDevice, UmemBuilder}; + +const DEV0_CHUNK_SHARE_PERCENT: u32 = 50; +const DEV1_CHUNK_SHARE_PERCENT: u32 = 100 - DEV0_CHUNK_SHARE_PERCENT; + +fn main() -> Result<(), Box> { + println!("=== Rattan vNIC Shared UMEM Test ===\n"); + println!("This test creates two vNICs sharing a single UMEM region."); + println!( + "Chunk split: dev0={}%, dev1={}%\n", + DEV0_CHUNK_SHARE_PERCENT, DEV1_CHUNK_SHARE_PERCENT + ); + + // Step 1: Open first device (rattan0) + println!("1. Opening first device (rattan0)..."); + let mut dev0 = RvnicDevice::open()?; + println!(" rattan0 opened (fd={})", dev0.fd()); + + // Step 2: Open second device (rattan1) + println!("\n2. Opening second device (rattan1)..."); + let mut dev1 = RvnicDevice::open()?; + println!(" rattan1 opened (fd={})", dev1.fd()); + + // Step 3: Allocate UMEM (will be shared between both devices) + println!("\n3. Allocating shared UMEM..."); + let umem = UmemBuilder::new() + .chunk_size(2048) + .headroom(128) + .num_chunks(128) // 128 * 2048 = 256KB, enough for both devices + .build()?; + + println!(" UMEM allocated:"); + println!(" - Address: 0x{:x}", umem.addr()); + println!(" - Length: {} bytes", umem.len()); + println!(" - Chunk size: {} bytes", umem.chunk_size()); + println!(" - Headroom: {} bytes", umem.headroom()); + println!(" - Num chunks: {}", umem.num_chunks()); + + // Step 4: Register UMEM with rattan0 (owner) + println!("\n4. Registering UMEM with rattan0 (owner)..."); + dev0.register_umem(umem)?; + println!(" UMEM registered with rattan0"); + println!(" has_umem: {}", dev0.has_umem()); + + // Step 5: Share UMEM with rattan1 + println!("\n5. Sharing UMEM with rattan1..."); + dev1.share_umem(&dev0)?; + println!(" UMEM shared with rattan1"); + println!(" dev1.has_umem: {}", dev1.has_umem()); + + // Step 6: Allocate rings for rattan0 + println!("\n6. Allocating rings for rattan0..."); + let rings0 = Rings::new()?; + println!( + " Rings allocated: addr=0x{:x}, len={}", + rings0.addr(), + rings0.len() + ); + let (mut fill0, _, _, _) = dev0.register_rings_for_queue(0, rings0)?; + println!(" Rings registered with rattan0"); + + // Step 7: Allocate rings for rattan1 + println!("\n7. Allocating rings for rattan1..."); + let rings1 = Rings::new()?; + println!( + " Rings allocated: addr=0x{:x}, len={}", + rings1.addr(), + rings1.len() + ); + let (mut fill1, _, _, _) = dev1.register_rings_for_queue(0, rings1)?; + println!(" Rings registered with rattan1"); + + // Step 8: Pre-fill FILL rings + // Split chunks between the two devices according to the configured + // directional chunk share. Chunks should be recycled back to the same + // device that originally supplied them, so this split models the initial + // per-device buffer budget. + println!("\n8. Pre-filling FILL rings..."); + let umem = dev0.umem().unwrap(); + let chunk_size = umem.chunk_size() as u64; + let num_chunks = umem.num_chunks(); + let dev0_chunks = num_chunks * DEV0_CHUNK_SHARE_PERCENT / 100; + let dev1_chunks = num_chunks - dev0_chunks; + + // Device 0 gets the first portion of chunks + let addrs0: Vec = (0..dev0_chunks).map(|i| (i as u64) * chunk_size).collect(); + let count0 = fill0.produce(&addrs0); + println!( + " Pushed {} chunks to rattan0 FILL ring (configured share: {}%)", + count0, DEV0_CHUNK_SHARE_PERCENT + ); + + // Device 1 gets the remainder + let addrs1: Vec = (dev0_chunks..num_chunks) + .map(|i| (i as u64) * chunk_size) + .collect(); + let count1 = fill1.produce(&addrs1); + println!( + " Pushed {} chunks to rattan1 FILL ring (configured share: {}%)", + count1, DEV1_CHUNK_SHARE_PERCENT + ); + + println!( + " Initial chunk budget: dev0={} chunks, dev1={} chunks", + dev0_chunks, dev1_chunks + ); + + // Step 9: Summary + println!("\n9. Setup complete!"); + println!(); + println!(" Summary:"); + println!(" - rattan0: owns UMEM, {} chunks in FILL ring", count0); + println!( + " - rattan1: shares UMEM, {} chunks in FILL ring", + count1 + ); + println!(); + println!(" Both devices have rings registered and are ready."); + println!(" Devices will be cleaned up on exit."); + println!(); + println!("=== Test Passed ==="); + + Ok(()) +} diff --git a/rvnic/librvnic/src/device.rs b/rvnic/librvnic/src/device.rs new file mode 100644 index 00000000..4cc9cd13 --- /dev/null +++ b/rvnic/librvnic/src/device.rs @@ -0,0 +1,547 @@ +//! RvnicDevice - Handle to a rattan virtual NIC +//! +//! Opening a device creates a new `rattanX` network interface. The device +//! must be configured with UMEM and rings before starting. +//! +//! # Example +//! +//! ```no_run +//! use rvnic::{RvnicDevice, UmemBuilder, Rings}; +//! +//! // Open device (creates rattan0 interface) +//! let mut dev = RvnicDevice::open().expect("failed to open device"); +//! +//! // Allocate and register UMEM +//! let umem = UmemBuilder::new() +//! .chunk_size(2048) +//! .num_chunks(4096) +//! .build() +//! .expect("failed to allocate UMEM"); +//! +//! dev.register_umem(umem).expect("failed to register UMEM"); +//! +//! // Allocate and register rings +//! let rings = Rings::new().expect("failed to allocate rings"); +//! dev.register_rings_for_queue(0, rings).expect("failed to register rings"); +//! +//! // Start the device (napi_cpu = -1 means use caller's CPU) +//! dev.start(-1).expect("failed to start device"); +//! +//! // ... use the device ... +//! +//! // Stop the device +//! dev.stop().expect("failed to stop device"); +//! +//! // Device, UMEM, and rings are cleaned up on drop +//! ``` + +use std::collections::BTreeMap; +use std::fs::{File, OpenOptions}; +use std::os::unix::io::{AsRawFd, RawFd}; +use std::sync::Arc; + +use crate::rings::{CompRing, DropRing, FillRing, Rings, RxRing, TxRing}; +use crate::sys::{ + RattanPayloadReq, RattanRingsReg, RattanRingsRegQ, RattanUmemReg, ioctl_kick_rx_q, + ioctl_read_payload, ioctl_reg_rings, ioctl_reg_rings_q, ioctl_reg_umem, ioctl_release, + ioctl_share_umem, ioctl_start, ioctl_stop, +}; +use crate::umem::Umem; +use crate::{Error, Result}; + +/// Path to the rattan-vnic device file +pub const DEVICE_PATH: &str = "/dev/rattan-vnic"; + +/// Owned ring handles for one queue. +/// +/// This bundles the four queue-local rings returned after registration so +/// multiqueue applications can manage one queue at a time. +pub struct QueueRings { + /// Queue identifier + pub queue_id: u32, + /// Fill ring for providing free UMEM chunks to the kernel + pub fill: FillRing, + /// Drop ring for release resources dropped by the userspace + pub drop: DropRing, + /// Completion ring for reclaiming finished UMEM chunks + pub comp: CompRing, + /// RX ring for consuming packet descriptors from the kernel + pub rx: RxRing, + /// TX ring for submitting forwarding descriptors to the kernel + pub tx: TxRing, +} + +/// Ring memory plus queue id used for batch registration. +/// +/// This is useful for ergonomic multiqueue setup where userspace allocates a +/// ring set per queue first and then registers them in one call. +pub struct QueueConfig { + /// Queue identifier + pub queue_id: u32, + /// Queue-local ring memory + pub rings: Rings, +} + +/// Handle to a rattan virtual NIC device +/// +/// Opening this device creates a new `rattanX` network interface. +/// The interface is destroyed when this handle is dropped. +pub struct RvnicDevice { + file: File, + /// UMEM memory (kept alive via Arc, may be shared between devices) + umem: Option>, + /// Rings memory per queue (kept alive via Arc) + rings: BTreeMap>, + started: bool, +} + +impl RvnicDevice { + /// Open a new rattan virtual NIC device + /// + /// This creates a new `rattanX` network interface (e.g., `rattan0`). + /// The interface name is assigned by the kernel based on availability. + /// + /// # Errors + /// Returns an error if: + /// - The device file doesn't exist (module not loaded) + /// - Permission denied + /// - Maximum number of devices reached + pub fn open() -> Result { + Self::open_path(DEVICE_PATH) + } + + /// Open a rattan virtual NIC device at a custom path + /// + /// This is mainly useful for testing. + pub fn open_path(path: &str) -> Result { + let file = OpenOptions::new().read(true).write(true).open(path)?; + + Ok(Self { + file, + umem: None, + rings: BTreeMap::new(), + started: false, + }) + } + + /// Get the raw file descriptor + /// + /// This can be used for poll/epoll or other low-level operations. + #[inline] + pub fn fd(&self) -> RawFd { + self.file.as_raw_fd() + } + + /// Check if UMEM is configured + #[inline] + pub fn has_umem(&self) -> bool { + self.umem.is_some() + } + + /// Check if rings are registered + #[inline] + pub fn has_rings(&self) -> bool { + self.has_rings_queue(0) + } + + /// Check if rings are registered for a specific queue + #[inline] + pub fn has_rings_queue(&self, queue_id: u32) -> bool { + self.rings.contains_key(&queue_id) + } + + /// Check if the device is started + #[inline] + pub fn is_started(&self) -> bool { + self.started + } + + /// Get a reference to the registered UMEM + #[inline] + pub fn umem(&self) -> Option<&Umem> { + self.umem.as_deref() + } + + /// Register a UMEM region with the kernel + /// + /// The UMEM memory will be pinned by the kernel and used for packet data. + /// UMEM can only be registered once per device. + /// + /// Returns an `Arc` that can be used to share UMEM with other devices. + /// + /// # Errors + /// Returns an error if: + /// - UMEM is already registered + /// - The ioctl fails + pub fn register_umem(&mut self, umem: Umem) -> Result> { + if self.umem.is_some() { + return Err(Error::UmemAlreadyRegistered); + } + + let reg = RattanUmemReg { + addr: umem.addr(), + len: umem.len() as u64, + chunk_size: umem.chunk_size(), + headroom: umem.headroom(), + }; + + // SAFETY: fd is valid (we own the File), reg is valid on stack + unsafe { + ioctl_reg_umem(self.fd(), ®)?; + } + + let arc = Arc::new(umem); + self.umem = Some(Arc::clone(&arc)); + Ok(arc) + } + + /// Share UMEM from another device + /// + /// # Arguments + /// * `source` - The device that owns the UMEM + /// + /// # Errors + /// Returns an error if: + /// - This device already has UMEM + /// - Source device has no UMEM + /// - The ioctl fails + /// + /// # Example + /// ```no_run + /// use rvnic::{RvnicDevice, UmemBuilder, Rings}; + /// + /// let mut dev0 = RvnicDevice::open().unwrap(); + /// let umem = UmemBuilder::new().build().unwrap(); + /// dev0.register_umem(umem).unwrap(); + /// + /// let mut dev1 = RvnicDevice::open().unwrap(); + /// dev1.share_umem(&dev0).unwrap(); // Share UMEM from dev0 + /// + /// // Both devices can now use the same UMEM + /// // Each still needs its own rings + /// ``` + pub fn share_umem(&mut self, source: &RvnicDevice) -> Result<()> { + if self.umem.is_some() { + return Err(Error::UmemAlreadyRegistered); + } + + let source_umem = source + .umem + .as_ref() + .ok_or(Error::NotConfigured("source device has no UMEM"))?; + + // SAFETY: fd is valid (we own the File), source fd is valid + unsafe { + ioctl_share_umem(self.fd(), source.fd())?; + } + + self.umem = Some(Arc::clone(source_umem)); + Ok(()) + } + + /// Register ring buffers with the kernel for a specific queue + /// + /// The rings memory will be pinned by the kernel and used for packet + /// descriptors. Each queue can only be registered once per device. + /// + /// Returns the ring handles for packet I/O operations. + /// + /// # Errors + /// Returns an error if: + /// - UMEM is not registered (must register UMEM first) + /// - Rings are already registered for this queue + /// - The ioctl fails + pub fn register_rings_for_queue( + &mut self, + queue_id: u32, + rings: Rings, + ) -> Result<(FillRing, DropRing, RxRing, TxRing)> { + let queue = self.register_queue(queue_id, rings)?; + // In current version, user space should not use the comp ring + Ok((queue.fill, queue.drop, queue.rx, queue.tx)) + } + + /// Register ring buffers and return an ergonomic queue bundle + /// + /// This is the queue-oriented variant of [`Self::register_rings_for_queue`]. + /// It returns a [`QueueRings`] bundle instead of a tuple. + /// + /// # Errors + /// Returns an error if: + /// - UMEM is not registered + /// - Rings are already registered for this queue + /// - The ioctl fails + pub fn register_queue(&mut self, queue_id: u32, rings: Rings) -> Result { + if self.umem.is_none() { + return Err(Error::NotConfigured("UMEM not registered")); + } + if self.rings.contains_key(&queue_id) { + return Err(Error::RingsAlreadyRegistered); + } + + if queue_id == 0 { + let reg = RattanRingsReg { + addr: rings.addr(), + len: rings.len() as u64, + }; + + // SAFETY: fd is valid (we own the File), reg is valid on stack + unsafe { + ioctl_reg_rings(self.fd(), ®)?; + } + } else { + let reg = RattanRingsRegQ { + addr: rings.addr(), + len: rings.len() as u64, + queue_id, + _pad: 0, + }; + + // SAFETY: fd is valid (we own the File), reg is valid on stack + unsafe { + ioctl_reg_rings_q(self.fd(), ®)?; + } + } + + let (arc, fill, drop, comp, rx, tx) = rings.split(); + self.rings.insert(queue_id, arc); + Ok(QueueRings { + queue_id, + fill, + drop, + comp, + rx, + tx, + }) + } + + /// Register multiple queues in one call + /// + /// The provided queue configurations are registered sequentially. If a + /// later queue registration fails, previously registered queues remain + /// registered. + /// + /// # Errors + /// Returns an error if: + /// - UMEM is not registered + /// - Any queue is already registered + /// - Any ioctl fails + pub fn register_queues(&mut self, queues: Vec) -> Result> { + let mut registered = Vec::with_capacity(queues.len()); + + for queue in queues { + registered.push(self.register_queue(queue.queue_id, queue.rings)?); + } + + Ok(registered) + } + + /// Start the device + /// + /// This enables packet processing on the device. The network interface + /// will show carrier on after this call. + /// + /// # Arguments + /// * `napi_cpu` - Target CPU for NAPI polling, or -1 to use caller's CPU + /// + /// # Errors + /// Returns an error if: + /// - UMEM is not registered + /// - Rings are not registered + /// - The device is already started + /// - The ioctl fails + pub fn start(&mut self, napi_cpu: i32) -> Result<()> { + if self.umem.is_none() { + return Err(Error::NotConfigured("UMEM not registered")); + } + if self.rings.is_empty() { + return Err(Error::NotConfigured("rings not registered")); + } + if self.started { + return Err(Error::AlreadyStarted); + } + + // SAFETY: fd is valid + unsafe { + ioctl_start(self.fd(), napi_cpu)?; + } + + self.started = true; + Ok(()) + } + + /// Stop the device + /// + /// This disables packet processing on the device. The network interface + /// will show carrier off after this call. + /// + /// # Errors + /// Returns an error if the ioctl fails + pub fn stop(&mut self) -> Result<()> { + if !self.started { + return Ok(()); // Already stopped, not an error + } + + // SAFETY: fd is valid + unsafe { + ioctl_stop(self.fd())?; + } + + self.started = false; + Ok(()) + } + + /// Read payload from a tracked packet + /// + /// Given a token from an RX descriptor, reads the full packet payload + /// into the provided buffer. + /// + /// # Arguments + /// * `token` - The token from the RX descriptor + /// * `buf` - Buffer to read payload into + /// * `offset` - Offset within the packet to start reading + /// + /// # Returns + /// A tuple of (bytes_copied, total_packet_length) + /// + /// # Errors + /// Returns an error if: + /// - Token is not found (already released or invalid) + /// - The ioctl fails + pub fn read_payload(&self, token: u64, buf: &mut [u8], offset: u32) -> Result<(u32, u32)> { + let req = RattanPayloadReq { + token, + buf: buf.as_mut_ptr() as u64, + len: buf.len() as u32, + offset, + }; + + // SAFETY: fd is valid, buf is valid for the length specified + let resp = unsafe { ioctl_read_payload(self.fd(), &req)? }; + + Ok((resp.len, resp.total_len)) + } + + /// Release a token - free the SKB in kernel + /// + /// # Arguments + /// * `token` - The token to release + /// + /// # Errors + /// Returns an error if: + /// - Token is not found (already released or invalid) + /// - The ioctl fails + pub fn release(&self, token: u64) -> Result<()> { + // SAFETY: fd is valid + unsafe { + ioctl_release(self.fd(), token)?; + } + + Ok(()) + } + + /// Kick the RX path for a specific queue + /// + /// Processes all pending descriptors in the selected queue's TX ring, + /// injecting the associated packets into the kernel's RX path. + /// + /// # Returns + /// The number of packets processed + /// + /// # Errors + /// Returns an error if: + /// - Device is not started + /// - The ioctl fails + pub fn kick_rx_queue(&self, queue_id: u32) -> Result { + // SAFETY: fd is valid + let processed = unsafe { ioctl_kick_rx_q(self.fd(), queue_id)? }; + Ok(processed) + } + + /// Kick a batch of queues and return packets processed per queue + /// + /// The returned vector preserves the input queue order. + /// + /// # Errors + /// Returns an error if any queue kick ioctl fails. + pub fn kick_rx_queues(&self, queue_ids: &[u32]) -> Result> { + let mut processed = Vec::with_capacity(queue_ids.len()); + + for &queue_id in queue_ids { + processed.push((queue_id, self.kick_rx_queue(queue_id)?)); + } + + Ok(processed) + } + + /// Clone a token to get a new token with a cloned SKB + /// + /// Every token is bound to one UMEM chunk, so the clone needs its own: + /// pass a free chunk from the same pool you refill FILL from. It comes back + /// on the COMP ring of `queue_id` on this device once the clone is released + /// or transmitted, like any chunk the kernel took from that queue's FILL. + /// + /// # Arguments + /// * `token` - The token to clone + /// * `chunk_addr` - Free chunk (UMEM offset) to bind the new token to + /// * `queue_id` - Queue of this device that owns `chunk_addr` + /// + /// # Returns + /// The new token for the cloned SKB + /// + /// # Errors + /// Returns an error if: + /// - Token is not found (already released or invalid): `ENOENT` + /// - `chunk_addr` or `queue_id` is invalid: `EINVAL` + /// - `chunk_addr` is still bound to a live token: `EBUSY` + /// - Memory allocation fails: `ENOMEM` + pub fn clone_token(&self, token: u64, chunk_addr: u64, queue_id: u32) -> Result { + // SAFETY: fd is valid + let new_token = unsafe { crate::sys::ioctl_clone(self.fd(), token, chunk_addr, queue_id)? }; + Ok(new_token) + } + + /// Get the device ID + /// If the device id is `x`, then the name of the device is rattan`x` + /// + /// # Returns + /// The device ID + /// + /// # Errors + /// Returns an error if the ioctl fails + pub fn device_id(&self) -> Result { + // SAFETY: fd is valid + let device_id = unsafe { crate::sys::ioctl_device_id(self.fd()) }?; + Ok(device_id) + } + + /// Get the device name, which is `rattan` followed by the device ID + /// + /// # Returns + /// The device name + /// + /// # Errors + /// Returns an error if the ioctl fails + pub fn device_name(&self) -> Result { + let device_id = self.device_id()?; + Ok(format!("rattan{}", device_id)) + } +} + +impl AsRawFd for RvnicDevice { + fn as_raw_fd(&self) -> RawFd { + self.file.as_raw_fd() + } +} + +impl Drop for RvnicDevice { + fn drop(&mut self) { + // Try to stop the device if it's running + // Ignore errors - we're in drop + if self.started { + let _ = self.stop(); + } + // UMEM and file are dropped automatically + // The kernel will unpin pages and destroy the interface + } +} diff --git a/rvnic/librvnic/src/lib.rs b/rvnic/librvnic/src/lib.rs new file mode 100644 index 00000000..db02d72b --- /dev/null +++ b/rvnic/librvnic/src/lib.rs @@ -0,0 +1,125 @@ +//! # rvnic - Rattan Virtual NIC Userspace Library +//! +//! This crate provides a safe Rust interface to the `rattan-vnic` kernel module. +//! +//! It supports both: +//! - simple single-queue setups via queue-0 convenience helpers +//! - multiqueue setups with one ring set per queue +//! +//! ## Quick Start +//! +//! ```no_run +//! use rvnic::{RvnicDevice, UmemBuilder, Rings}; +//! +//! // Open device (creates rattan0 interface) +//! let mut dev = RvnicDevice::open()?; +//! +//! // Allocate UMEM (128KB = 64 chunks × 2KB) +//! let umem = UmemBuilder::new() +//! .chunk_size(2048) +//! .headroom(128) +//! .num_chunks(64) +//! .build()?; +//! +//! // Allocate rings +//! let rings = Rings::new()?; +//! +//! // Register UMEM and queue-0 rings with kernel +//! dev.register_umem(umem)?; +//! dev.register_rings_for_queue(0, rings)?; +//! +//! // Start the device (napi_cpu = -1 means use caller's CPU) +//! dev.start(-1)?; +//! +//! // ... process packets ... +//! +//! // Cleanup happens automatically on drop +//! # Ok::<(), rvnic::Error>(()) +//! ``` +//! +//! ## Multiqueue Setup +//! +//! ```no_run +//! use rvnic::{QueueConfig, Rings, RvnicDevice, UmemBuilder}; +//! +//! let mut dev = RvnicDevice::open()?; +//! let umem = UmemBuilder::new().build()?; +//! dev.register_umem(umem)?; +//! +//! let queues = vec![ +//! QueueConfig { +//! queue_id: 0, +//! rings: Rings::new()?, +//! }, +//! QueueConfig { +//! queue_id: 1, +//! rings: Rings::new()?, +//! }, +//! ]; +//! +//! let mut queue_rings = dev.register_queues(queues)?; +//! dev.start(-1)?; +//! +//! // Kick queue 0 +//! let _processed = dev.kick_rx_queue(queue_rings[0].queue_id)?; +//! +//! // Access per-queue rings +//! let q0 = &mut queue_rings[0]; +//! let _free_slots = q0.fill.available(); +//! let _pending_rx = q0.rx.available(); +//! # Ok::<(), rvnic::Error>(()) +//! ``` + +#![warn(missing_docs)] +#![warn(rust_2018_idioms)] + +mod device; +mod rings; +mod umem; + +// Low-level types (public for advanced users) +pub mod sys; + +// Re-export public types +pub use device::{DEVICE_PATH, QueueConfig, QueueRings, RvnicDevice}; +pub use rings::{CompRing, DropRing, FillRing, Rings, RxRing, TxRing}; +pub use umem::{Umem, UmemBuilder}; + +// Re-export constants that users might need +pub use sys::{ + RATTAN_DEFAULT_CHUNK_SIZE, RATTAN_HEADER_SIZE, RATTAN_MAX_CHUNK_SIZE, RATTAN_MIN_CHUNK_SIZE, + RATTAN_RING_SIZE, +}; + +use std::io; + +/// Error type for rvnic operations +#[derive(Debug, thiserror::Error)] +pub enum Error { + /// I/O error (file operations, ioctl failures) + #[error("I/O error: {0}")] + Io(#[from] io::Error), + + /// UMEM is already registered on this device + #[error("UMEM already registered")] + UmemAlreadyRegistered, + + /// Rings are already registered on this device + #[error("rings already registered")] + RingsAlreadyRegistered, + + /// Device is not properly configured + #[error("device not configured: {0}")] + NotConfigured(&'static str), + + /// Device is already started + #[error("device already started")] + AlreadyStarted, + + /// Invalid parameter provided + #[error("invalid parameter: {0}")] + InvalidParam(&'static str), +} + +/// Result type for rvnic operations +pub type Result = std::result::Result; diff --git a/rvnic/librvnic/src/rings.rs b/rvnic/librvnic/src/rings.rs new file mode 100644 index 00000000..ef82a5ba --- /dev/null +++ b/rvnic/librvnic/src/rings.rs @@ -0,0 +1,577 @@ +//! Ring buffer abstractions for packet I/O +//! +//! This module provides the memory management and high-level interfaces for the four ring types: +//! +//! - [`FillRing`]: Userspace provides free UMEM chunks to kernel (producer) +//! - [`CompRing`]: Kernel returns completed chunks to userspace (consumer) +//! - [`RxRing`]: Kernel delivers received packet descriptors (consumer) +//! - [`TxRing`]: Userspace submits packet descriptors for forwarding (producer) + +use std::ptr::NonNull; +use std::sync::Arc; +use std::sync::atomic::{AtomicU32, Ordering}; + +use crate::sys::{ + RATTAN_COMP_RING_OFFSET, RATTAN_DROP_RING_OFFSET, RATTAN_FILL_RING_OFFSET, RATTAN_RING_SIZE, + RATTAN_RINGS_SIZE, RATTAN_RX_RING_OFFSET, RATTAN_TX_RING_OFFSET, RattanCompRing, RattanDesc, + RattanDropRing, RattanFillRing, RattanRxRing, RattanTxRing, +}; +use crate::{Error, Result}; + +const RING_MASK: u32 = RATTAN_RING_SIZE - 1; + +// ============================================================================ +// Rings - Memory management for all ring buffers +// ============================================================================ + +/// Ring buffers for packet descriptors +/// +/// Contains four rings: +/// - RX: Kernel writes packet descriptors, userspace reads +/// - TX: Userspace writes forward requests, kernel reads +/// - FILL: Userspace provides free chunks, kernel consumes +/// - COMP: Kernel returns completed chunks, userspace recycles +pub struct Rings { + /// Pointer to mmap'd memory + ptr: NonNull, + /// Total length of the mapping + len: usize, +} + +impl Rings { + /// Allocate ring buffers + /// + /// The memory is page-aligned and populated immediately. + pub fn new() -> Result { + let len = RATTAN_RINGS_SIZE; + + // Round up to page size + let page_size = unsafe { libc::sysconf(libc::_SC_PAGESIZE) as usize }; + let len = (len + page_size - 1) & !(page_size - 1); + + // mmap anonymous memory + let ptr = unsafe { + libc::mmap( + std::ptr::null_mut(), + len, + libc::PROT_READ | libc::PROT_WRITE, + libc::MAP_PRIVATE | libc::MAP_ANONYMOUS | libc::MAP_POPULATE, + -1, + 0, + ) + }; + + if ptr == libc::MAP_FAILED { + return Err(Error::Io(std::io::Error::last_os_error())); + } + + // Zero-initialize the memory + unsafe { + std::ptr::write_bytes(ptr as *mut u8, 0, len); + } + + Ok(Self { + ptr: NonNull::new(ptr as *mut u8).unwrap(), + len, + }) + } + + /// Get the address of the rings memory + #[inline] + pub fn addr(&self) -> u64 { + self.ptr.as_ptr() as u64 + } + + /// Get the total length of the rings memory in bytes + #[inline] + #[allow(clippy::len_without_is_empty)] // Rings are never "empty" - they're fixed-size + pub fn len(&self) -> usize { + self.len + } + + #[inline] + fn rx_ptr(&self) -> *mut RattanRxRing { + unsafe { self.ptr.as_ptr().add(RATTAN_RX_RING_OFFSET) as *mut RattanRxRing } + } + + #[inline] + fn tx_ptr(&self) -> *mut RattanTxRing { + unsafe { self.ptr.as_ptr().add(RATTAN_TX_RING_OFFSET) as *mut RattanTxRing } + } + + #[inline] + fn fill_ptr(&self) -> *mut RattanFillRing { + unsafe { self.ptr.as_ptr().add(RATTAN_FILL_RING_OFFSET) as *mut RattanFillRing } + } + + #[inline] + fn drop_ptr(&self) -> *mut RattanDropRing { + unsafe { self.ptr.as_ptr().add(RATTAN_DROP_RING_OFFSET) as *mut RattanDropRing } + } + + #[inline] + fn comp_ptr(&self) -> *mut RattanCompRing { + unsafe { self.ptr.as_ptr().add(RATTAN_COMP_RING_OFFSET) as *mut RattanCompRing } + } + + /// Split into individual ring handles, consuming the Rings struct + /// + /// Returns an `Arc` along with the ring handles. Each ring handle + /// holds an `Arc` reference to the underlying memory, so the memory is + /// automatically freed when all handles (and the returned Arc) are dropped. + /// + /// The ring handles can be freely moved to different threads. + /// + /// # Example + /// + /// ```no_run + /// # use rvnic::Rings; + /// let rings = Rings::new()?; + /// let (arc, mut fill, mut drop, mut comp, mut rx, mut tx) = rings.split(); + /// + /// // Provide chunks to kernel + /// fill.produce(&[0, 2048, 4096]); + /// + /// // Process received packets + /// let mut descs = [Default::default(); 32]; + /// let n = rx.consume(&mut descs); + /// # Ok::<(), rvnic::Error>(()) + /// ``` + #[inline] + pub fn split(self) -> (Arc, FillRing, DropRing, CompRing, RxRing, TxRing) { + let arc = Arc::new(self); + ( + Arc::clone(&arc), + FillRing::new(arc.fill_ptr(), Arc::clone(&arc)), + DropRing::new(arc.drop_ptr(), Arc::clone(&arc)), + CompRing::new(arc.comp_ptr(), Arc::clone(&arc)), + RxRing::new(arc.rx_ptr(), Arc::clone(&arc)), + TxRing::new(arc.tx_ptr(), arc), + ) + } +} + +impl Drop for Rings { + fn drop(&mut self) { + unsafe { + libc::munmap(self.ptr.as_ptr() as *mut libc::c_void, self.len); + } + } +} + +// SAFETY: Rings can be sent between threads (the memory is owned). +// Rings is Sync because after split(), each ring handle accesses a different +// region of the memory. +unsafe impl Send for Rings {} +unsafe impl Sync for Rings {} + +// ============================================================================ +// Fill Ring operations (shared between owned and borrowed versions) +// ============================================================================ + +#[inline] +fn fill_produce(ring: *mut RattanFillRing, addrs: &[u64]) -> usize { + if addrs.is_empty() { + return 0; + } + + unsafe { + let ring = &mut *ring; + let prod_ptr = &ring.ring.producer as *const u32 as *const AtomicU32; + let cons_ptr = &ring.ring.consumer as *const u32 as *const AtomicU32; + + let prod = (*prod_ptr).load(Ordering::Relaxed); + let cons = (*cons_ptr).load(Ordering::Acquire); + + let free_slots = RATTAN_RING_SIZE.wrapping_sub(prod.wrapping_sub(cons)) as usize; + let n = addrs.len().min(free_slots); + if n == 0 { + return 0; + } + + let mut idx = prod; + for &addr in &addrs[..n] { + ring.addrs[(idx & RING_MASK) as usize] = addr; + idx = idx.wrapping_add(1); + } + + (*prod_ptr).store(prod.wrapping_add(n as u32), Ordering::Release); + n + } +} + +#[inline] +fn fill_available(ring: *mut RattanFillRing) -> u32 { + unsafe { + let ring = &*ring; + let prod_ptr = &ring.ring.producer as *const u32 as *const AtomicU32; + let cons_ptr = &ring.ring.consumer as *const u32 as *const AtomicU32; + + let prod = (*prod_ptr).load(Ordering::Relaxed); + let cons = (*cons_ptr).load(Ordering::Acquire); + RATTAN_RING_SIZE.wrapping_sub(prod.wrapping_sub(cons)) + } +} + +// ============================================================================ +// Drop Ring operations (shared between owned and borrowed versions) +// ============================================================================ + +#[inline] +fn drop_produce(ring: *mut RattanDropRing, tokens: &[u64]) -> usize { + if tokens.is_empty() { + return 0; + } + + unsafe { + let ring = &mut *ring; + let prod_ptr = &ring.ring.producer as *const u32 as *const AtomicU32; + let cons_ptr = &ring.ring.consumer as *const u32 as *const AtomicU32; + + let prod = (*prod_ptr).load(Ordering::Relaxed); + let cons = (*cons_ptr).load(Ordering::Acquire); + + let free_slots = RATTAN_RING_SIZE.wrapping_sub(prod.wrapping_sub(cons)) as usize; + let n = tokens.len().min(free_slots); + if n == 0 { + return 0; + } + + let mut idx = prod; + for &addr in &tokens[..n] { + ring.tokens[(idx & RING_MASK) as usize] = addr; + idx = idx.wrapping_add(1); + } + + (*prod_ptr).store(prod.wrapping_add(n as u32), Ordering::Release); + n + } +} + +#[inline] +fn drop_available(ring: *mut RattanDropRing) -> u32 { + unsafe { + let ring = &*ring; + let prod_ptr = &ring.ring.producer as *const u32 as *const AtomicU32; + let cons_ptr = &ring.ring.consumer as *const u32 as *const AtomicU32; + + let prod = (*prod_ptr).load(Ordering::Relaxed); + let cons = (*cons_ptr).load(Ordering::Acquire); + RATTAN_RING_SIZE.wrapping_sub(prod.wrapping_sub(cons)) + } +} + +// ============================================================================ +// Comp Ring operations +// ============================================================================ + +#[inline] +fn comp_consume(ring: *mut RattanCompRing, addrs: &mut [u64]) -> usize { + if addrs.is_empty() { + return 0; + } + + unsafe { + let ring = &*ring; + let prod_ptr = &ring.ring.producer as *const u32 as *const AtomicU32; + let cons_ptr = &ring.ring.consumer as *const u32 as *const AtomicU32; + + let prod = (*prod_ptr).load(Ordering::Acquire); + let cons = (*cons_ptr).load(Ordering::Relaxed); + + let avail = prod.wrapping_sub(cons) as usize; + let n = addrs.len().min(avail); + if n == 0 { + return 0; + } + + let mut idx = cons; + for addr in &mut addrs[..n] { + *addr = ring.addrs[(idx & RING_MASK) as usize]; + idx = idx.wrapping_add(1); + } + + (*cons_ptr).store(cons.wrapping_add(n as u32), Ordering::Release); + n + } +} + +#[inline] +fn comp_available(ring: *mut RattanCompRing) -> u32 { + unsafe { + let ring = &*ring; + let prod_ptr = &ring.ring.producer as *const u32 as *const AtomicU32; + let cons_ptr = &ring.ring.consumer as *const u32 as *const AtomicU32; + + let prod = (*prod_ptr).load(Ordering::Acquire); + let cons = (*cons_ptr).load(Ordering::Relaxed); + prod.wrapping_sub(cons) + } +} + +// ============================================================================ +// RX Ring operations +// ============================================================================ + +#[inline] +fn rx_consume(ring: *mut RattanRxRing, descs: &mut [RattanDesc]) -> usize { + if descs.is_empty() { + return 0; + } + + unsafe { + let ring = &*ring; + let prod_ptr = &ring.ring.producer as *const u32 as *const AtomicU32; + let cons_ptr = &ring.ring.consumer as *const u32 as *const AtomicU32; + + let prod = (*prod_ptr).load(Ordering::Acquire); + let cons = (*cons_ptr).load(Ordering::Relaxed); + + let avail = prod.wrapping_sub(cons) as usize; + let n = descs.len().min(avail); + if n == 0 { + return 0; + } + + let mut idx = cons; + for desc in &mut descs[..n] { + *desc = ring.descs[(idx & RING_MASK) as usize]; + idx = idx.wrapping_add(1); + } + + (*cons_ptr).store(cons.wrapping_add(n as u32), Ordering::Release); + n + } +} + +#[inline] +fn rx_available(ring: *mut RattanRxRing) -> u32 { + unsafe { + let ring = &*ring; + let prod_ptr = &ring.ring.producer as *const u32 as *const AtomicU32; + let cons_ptr = &ring.ring.consumer as *const u32 as *const AtomicU32; + + let prod = (*prod_ptr).load(Ordering::Acquire); + let cons = (*cons_ptr).load(Ordering::Relaxed); + prod.wrapping_sub(cons) + } +} + +// ============================================================================ +// TX Ring operations +// ============================================================================ + +#[inline] +fn tx_produce(ring: *mut RattanTxRing, descs: &[RattanDesc]) -> usize { + if descs.is_empty() { + return 0; + } + + unsafe { + let ring = &mut *ring; + let prod_ptr = &ring.ring.producer as *const u32 as *const AtomicU32; + let cons_ptr = &ring.ring.consumer as *const u32 as *const AtomicU32; + + let prod = (*prod_ptr).load(Ordering::Relaxed); + let cons = (*cons_ptr).load(Ordering::Acquire); + + let free_slots = RATTAN_RING_SIZE.wrapping_sub(prod.wrapping_sub(cons)) as usize; + let n = descs.len().min(free_slots); + if n == 0 { + return 0; + } + + let mut idx = prod; + for desc in &descs[..n] { + ring.descs[(idx & RING_MASK) as usize] = *desc; + idx = idx.wrapping_add(1); + } + + (*prod_ptr).store(prod.wrapping_add(n as u32), Ordering::Release); + n + } +} + +#[inline] +fn tx_available(ring: *mut RattanTxRing) -> u32 { + unsafe { + let ring = &*ring; + let prod_ptr = &ring.ring.producer as *const u32 as *const AtomicU32; + let cons_ptr = &ring.ring.consumer as *const u32 as *const AtomicU32; + + let prod = (*prod_ptr).load(Ordering::Relaxed); + let cons = (*cons_ptr).load(Ordering::Acquire); + RATTAN_RING_SIZE.wrapping_sub(prod.wrapping_sub(cons)) + } +} + +// ============================================================================ +// FillRing - Owned version (holds Arc) +// ============================================================================ + +/// Fill Ring - Userspace provides free UMEM chunks to kernel +/// +/// This is the owned version that holds an `Arc` reference. +/// The underlying memory is freed when all ring handles are dropped. +pub struct FillRing { + ring: *mut RattanFillRing, + _owner: Arc, +} + +impl FillRing { + fn new(ring: *mut RattanFillRing, owner: Arc) -> Self { + Self { + ring, + _owner: owner, + } + } + + /// Produce chunk addresses to the fill ring + #[inline] + pub fn produce(&mut self, addrs: &[u64]) -> usize { + fill_produce(self.ring, addrs) + } + + /// Get available slots + #[inline] + pub fn available(&self) -> u32 { + fill_available(self.ring) + } +} + +unsafe impl Send for FillRing {} + +// ============================================================================ +// DropRing - Owned version (holds Arc) +// ============================================================================ + +/// Drop Ring - User space drops +/// +/// This is the owned version that holds an `Arc` reference. +/// The underlying memory is freed when all ring handles are dropped. +pub struct DropRing { + ring: *mut RattanDropRing, + _owner: Arc, +} + +impl DropRing { + fn new(ring: *mut RattanDropRing, owner: Arc) -> Self { + Self { + ring, + _owner: owner, + } + } + + /// Produce chunk addresses to the fill ring + #[inline] + pub fn produce(&mut self, addrs: &[u64]) -> usize { + drop_produce(self.ring, addrs) + } + + /// Get available slots + #[inline] + pub fn available(&self) -> u32 { + drop_available(self.ring) + } +} + +unsafe impl Send for DropRing {} + +// ============================================================================ +// CompRing - Owned version +// ============================================================================ + +/// Completion Ring - Kernel returns completed chunks to userspace +pub struct CompRing { + ring: *mut RattanCompRing, + _owner: Arc, +} + +impl CompRing { + fn new(ring: *mut RattanCompRing, owner: Arc) -> Self { + Self { + ring, + _owner: owner, + } + } + + /// Consume completed chunk addresses from the ring + #[inline] + pub fn consume(&mut self, addrs: &mut [u64]) -> usize { + comp_consume(self.ring, addrs) + } + + /// Get available items + #[inline] + pub fn available(&self) -> u32 { + comp_available(self.ring) + } +} + +unsafe impl Send for CompRing {} + +// ============================================================================ +// RxRing - Owned version +// ============================================================================ + +/// RX Ring - Kernel delivers received packet descriptors +pub struct RxRing { + ring: *mut RattanRxRing, + _owner: Arc, +} + +impl RxRing { + fn new(ring: *mut RattanRxRing, owner: Arc) -> Self { + Self { + ring, + _owner: owner, + } + } + + /// Consume received packet descriptors from the ring + #[inline] + pub fn consume(&mut self, descs: &mut [RattanDesc]) -> usize { + rx_consume(self.ring, descs) + } + + /// Get available items + #[inline] + pub fn available(&self) -> u32 { + rx_available(self.ring) + } +} + +unsafe impl Send for RxRing {} + +// ============================================================================ +// TxRing - Owned version +// ============================================================================ + +/// TX Ring - Userspace submits packet descriptors for forwarding +pub struct TxRing { + ring: *mut RattanTxRing, + _owner: Arc, +} + +impl TxRing { + fn new(ring: *mut RattanTxRing, owner: Arc) -> Self { + Self { + ring, + _owner: owner, + } + } + + /// Produce packet descriptors to the TX ring for forwarding + #[inline] + pub fn produce(&mut self, descs: &[RattanDesc]) -> usize { + tx_produce(self.ring, descs) + } + + /// Get available slots + #[inline] + pub fn available(&self) -> u32 { + tx_available(self.ring) + } +} + +unsafe impl Send for TxRing {} diff --git a/rvnic/librvnic/src/sys.rs b/rvnic/librvnic/src/sys.rs new file mode 100644 index 00000000..4121bd29 --- /dev/null +++ b/rvnic/librvnic/src/sys.rs @@ -0,0 +1,438 @@ +//! Raw system types and ioctl definitions for rattan-vnic. +//! Must match `kernel/rattan_vnic_uapi.h`. + +#![allow(missing_docs)] + +use std::io; +use std::os::unix::io::RawFd; + +// Configuration constants +pub const RATTAN_RING_SIZE: u32 = 262144; +pub const RATTAN_RING_MASK: u32 = RATTAN_RING_SIZE - 1; +pub const RATTAN_DEFAULT_CHUNK_SIZE: u32 = 256; +pub const RATTAN_MIN_CHUNK_SIZE: u32 = 256; +pub const RATTAN_MAX_CHUNK_SIZE: u32 = 65536; +pub const RATTAN_MAX_UMEM_SIZE: u64 = 128 << 30; // 128 GiB +pub const RATTAN_MAX_CHUNKS: u32 = 1 << 22; +pub const RATTAN_HEADER_SIZE: u32 = 0; + +// UMEM registration parameters +#[repr(C)] +#[derive(Debug, Clone, Copy, Default)] +pub struct RattanUmemReg { + pub addr: u64, + pub len: u64, + pub chunk_size: u32, + pub headroom: u32, +} + +// Ring header - 128 bytes, cache-line aligned fields +#[repr(C)] +#[derive(Debug)] +pub struct RattanRing { + pub producer: u32, + _pad1: [u8; 60], + pub consumer: u32, + pub flags: u32, + _pad2: [u32; 14], +} + +pub const RATTAN_RING_NEED_WAKEUP: u32 = 1 << 0; + +// Packet descriptor - 32 bytes +#[repr(C)] +#[derive(Debug, Clone, Copy, Default)] +pub struct RattanDesc { + pub addr: u64, + pub len: u32, + pub options: u32, + pub token: u64, + pub timestamp: u64, +} + +impl RattanDesc { + /// Packet length of the underlying skb in kernel space + pub fn get_skb_len(&self) -> u32 { + self.options & 0xffff + } + + /// Length of copied packet headers in UMEM + pub fn get_umem_len(&self) -> u32 { + self.len + } +} + +#[repr(C)] +pub struct RattanFillRing { + pub ring: RattanRing, + pub addrs: [u64; RATTAN_RING_SIZE as usize], +} + +#[repr(C)] +pub struct RattanDropRing { + pub ring: RattanRing, + pub tokens: [u64; RATTAN_RING_SIZE as usize], +} + +#[repr(C)] +pub struct RattanCompRing { + pub ring: RattanRing, + pub addrs: [u64; RATTAN_RING_SIZE as usize], +} + +#[repr(C)] +pub struct RattanRxRing { + pub ring: RattanRing, + pub descs: [RattanDesc; RATTAN_RING_SIZE as usize], +} + +#[repr(C)] +pub struct RattanTxRing { + pub ring: RattanRing, + pub descs: [RattanDesc; RATTAN_RING_SIZE as usize], +} + +// Ring memory layout: RX | TX | FILL | COMP | DROP +pub const RATTAN_RINGS_SIZE: usize = std::mem::size_of::() + + std::mem::size_of::() + + std::mem::size_of::() + + std::mem::size_of::() + + std::mem::size_of::(); + +pub const RATTAN_RX_RING_OFFSET: usize = 0; +pub const RATTAN_TX_RING_OFFSET: usize = std::mem::size_of::(); +pub const RATTAN_FILL_RING_OFFSET: usize = + RATTAN_TX_RING_OFFSET + std::mem::size_of::(); +pub const RATTAN_COMP_RING_OFFSET: usize = + RATTAN_FILL_RING_OFFSET + std::mem::size_of::(); +pub const RATTAN_DROP_RING_OFFSET: usize = + RATTAN_COMP_RING_OFFSET + std::mem::size_of::(); + +#[repr(C)] +#[derive(Debug, Clone, Copy, Default)] +pub struct RattanRingsReg { + pub addr: u64, + pub len: u64, +} + +#[repr(C)] +#[derive(Debug, Clone, Copy, Default)] +pub struct RattanRingsRegQ { + pub addr: u64, + pub len: u64, + pub queue_id: u32, + pub _pad: u32, +} + +#[repr(C)] +#[derive(Debug, Clone, Copy, Default)] +pub struct RattanShareUmemReq { + pub source_fd: i32, + pub _pad: u32, +} + +#[repr(C)] +#[derive(Debug, Clone, Copy, Default)] +pub struct RattanPayloadReq { + pub token: u64, + pub buf: u64, + pub len: u32, + pub offset: u32, +} + +#[repr(C)] +#[derive(Debug, Clone, Copy, Default)] +pub struct RattanPayloadResp { + pub len: u32, + pub total_len: u32, +} + +/// Union for read_payload ioctl - kernel overwrites request with response +#[repr(C)] +pub union RattanPayloadIoctl { + pub req: RattanPayloadReq, + pub resp: RattanPayloadResp, +} + +#[repr(C)] +#[derive(Debug, Clone, Copy, Default)] +pub struct RattanReleaseReq { + pub token: u64, +} + +#[repr(C)] +#[derive(Debug, Clone, Copy, Default)] +pub struct RattanCloneReq { + pub token: u64, + pub chunk_addr: u64, + pub queue_id: u32, + pub _pad: u32, +} + +#[repr(C)] +#[derive(Debug, Clone, Copy, Default)] +pub struct RattanCloneResp { + pub new_token: u64, +} + +/// Start configuration +#[repr(C)] +#[derive(Debug, Clone, Copy, Default)] +pub struct RattanStartConfig { + /// Target CPU for NAPI (-1 = use caller's CPU) + pub napi_cpu: i32, + pub _pad: u32, +} + +// ioctl definitions +pub const RATTAN_VNIC_MAGIC: u8 = b'R'; + +const NR_REG_UMEM: u8 = 1; +const NR_REG_RINGS: u8 = 2; +const NR_SHARE_UMEM: u8 = 3; +const NR_START: u8 = 4; +const NR_STOP: u8 = 5; +const NR_REG_RINGS_Q: u8 = 6; +const NR_READ_PAYLOAD: u8 = 10; +const NR_RELEASE: u8 = 11; +const NR_KICK_RX: u8 = 12; +const NR_CLONE: u8 = 13; +const NR_KICK_RX_Q: u8 = 14; +const NR_DEVICE_ID: u8 = 15; + +const fn iow(magic: u8, nr: u8) -> libc::c_ulong { + let size = std::mem::size_of::() as libc::c_ulong; + (1 << 30) | (size << 16) | ((magic as libc::c_ulong) << 8) | (nr as libc::c_ulong) +} + +const fn ior(magic: u8, nr: u8) -> libc::c_ulong { + let size = std::mem::size_of::() as libc::c_ulong; + (2 << 30) | (size << 16) | ((magic as libc::c_ulong) << 8) | (nr as libc::c_ulong) +} + +const fn iowr(magic: u8, nr: u8) -> libc::c_ulong { + let size = std::mem::size_of::() as libc::c_ulong; + (3 << 30) | (size << 16) | ((magic as libc::c_ulong) << 8) | (nr as libc::c_ulong) +} + +const fn io(magic: u8, nr: u8) -> libc::c_ulong { + ((magic as libc::c_ulong) << 8) | (nr as libc::c_ulong) +} + +const IOCTL_REG_UMEM: libc::c_ulong = iow::(RATTAN_VNIC_MAGIC, NR_REG_UMEM); +const IOCTL_REG_RINGS: libc::c_ulong = iow::(RATTAN_VNIC_MAGIC, NR_REG_RINGS); +const IOCTL_SHARE_UMEM: libc::c_ulong = iow::(RATTAN_VNIC_MAGIC, NR_SHARE_UMEM); +const IOCTL_START: libc::c_ulong = iow::(RATTAN_VNIC_MAGIC, NR_START); +const IOCTL_STOP: libc::c_ulong = io(RATTAN_VNIC_MAGIC, NR_STOP); +const IOCTL_REG_RINGS_Q: libc::c_ulong = iow::(RATTAN_VNIC_MAGIC, NR_REG_RINGS_Q); +const IOCTL_READ_PAYLOAD: libc::c_ulong = + iowr::(RATTAN_VNIC_MAGIC, NR_READ_PAYLOAD); +const IOCTL_RELEASE: libc::c_ulong = iow::(RATTAN_VNIC_MAGIC, NR_RELEASE); +const IOCTL_KICK_RX: libc::c_ulong = io(RATTAN_VNIC_MAGIC, NR_KICK_RX); +const IOCTL_CLONE: libc::c_ulong = iowr::(RATTAN_VNIC_MAGIC, NR_CLONE); +const IOCTL_KICK_RX_Q: libc::c_ulong = iow::(RATTAN_VNIC_MAGIC, NR_KICK_RX_Q); +const IOCTL_DEVICE_ID: libc::c_ulong = ior::(RATTAN_VNIC_MAGIC, NR_DEVICE_ID); + +// Raw ioctl wrappers + +/// # Safety +/// `fd` must be valid, memory region must remain valid until unregistered. +#[inline] +pub unsafe fn ioctl_reg_umem(fd: RawFd, reg: &RattanUmemReg) -> io::Result<()> { + let ret = unsafe { libc::ioctl(fd, IOCTL_REG_UMEM, reg as *const RattanUmemReg) }; + if ret < 0 { + return Err(io::Error::last_os_error()); + } + Ok(()) +} + +/// # Safety +/// `fd` must be valid, UMEM must be registered first. +#[inline] +pub unsafe fn ioctl_reg_rings(fd: RawFd, reg: &RattanRingsReg) -> io::Result<()> { + let ret = unsafe { libc::ioctl(fd, IOCTL_REG_RINGS, reg as *const RattanRingsReg) }; + if ret < 0 { + return Err(io::Error::last_os_error()); + } + Ok(()) +} + +/// # Safety +/// `fd` must be valid, UMEM must be registered first, and `queue_id` must be valid. +#[inline] +pub unsafe fn ioctl_reg_rings_q(fd: RawFd, reg: &RattanRingsRegQ) -> io::Result<()> { + let ret = unsafe { libc::ioctl(fd, IOCTL_REG_RINGS_Q, reg as *const RattanRingsRegQ) }; + if ret < 0 { + return Err(io::Error::last_os_error()); + } + Ok(()) +} + +/// # Safety +/// Both `fd` and `source_fd` must be valid rattan-vnic devices. +#[inline] +pub unsafe fn ioctl_share_umem(fd: RawFd, source_fd: RawFd) -> io::Result<()> { + let req = RattanShareUmemReq { source_fd, _pad: 0 }; + let ret = unsafe { libc::ioctl(fd, IOCTL_SHARE_UMEM, &req as *const RattanShareUmemReq) }; + if ret < 0 { + return Err(io::Error::last_os_error()); + } + Ok(()) +} + +/// # Safety +/// `fd` must be valid, UMEM and rings must be registered. +#[inline] +pub unsafe fn ioctl_start(fd: RawFd, napi_cpu: i32) -> io::Result<()> { + let config = RattanStartConfig { napi_cpu, _pad: 0 }; + let ret = unsafe { libc::ioctl(fd, IOCTL_START, &config as *const RattanStartConfig) }; + if ret < 0 { + return Err(io::Error::last_os_error()); + } + Ok(()) +} + +/// # Safety +/// `fd` must be valid. +#[inline] +pub unsafe fn ioctl_stop(fd: RawFd) -> io::Result<()> { + let ret = unsafe { libc::ioctl(fd, IOCTL_STOP) }; + if ret < 0 { + return Err(io::Error::last_os_error()); + } + Ok(()) +} + +/// # Safety +/// `fd` must be valid, `req.buf` must point to valid buffer of at least `req.len` bytes. +#[inline] +pub unsafe fn ioctl_read_payload( + fd: RawFd, + req: &RattanPayloadReq, +) -> io::Result { + let mut ioctl_data = RattanPayloadIoctl { req: *req }; + let ret = unsafe { + libc::ioctl( + fd, + IOCTL_READ_PAYLOAD, + &mut ioctl_data as *mut RattanPayloadIoctl, + ) + }; + if ret < 0 { + return Err(io::Error::last_os_error()); + } + // SAFETY: After successful ioctl, kernel has written response into the union + Ok(unsafe { ioctl_data.resp }) +} + +/// # Safety +/// `fd` must be valid, `token` must be valid. +#[inline] +pub unsafe fn ioctl_release(fd: RawFd, token: u64) -> io::Result<()> { + let req = RattanReleaseReq { token }; + let ret = unsafe { libc::ioctl(fd, IOCTL_RELEASE, &req as *const RattanReleaseReq) }; + if ret < 0 { + return Err(io::Error::last_os_error()); + } + Ok(()) +} + +/// # Safety +/// `fd` must be valid. +#[inline] +pub unsafe fn ioctl_kick_rx(fd: RawFd) -> io::Result { + let ret = unsafe { libc::ioctl(fd, IOCTL_KICK_RX) }; + if ret < 0 { + return Err(io::Error::last_os_error()); + } + Ok(ret) +} + +/// # Safety +/// `fd` must be valid and `queue_id` must be valid. +#[inline] +pub unsafe fn ioctl_kick_rx_q(fd: RawFd, queue_id: u32) -> io::Result { + let mut qid = queue_id; + let ret = unsafe { libc::ioctl(fd, IOCTL_KICK_RX_Q, &mut qid as *mut u32) }; + if ret < 0 { + return Err(io::Error::last_os_error()); + } + Ok(ret) +} + +#[repr(C)] +pub union RattanCloneIoctl { + pub req: RattanCloneReq, + pub resp: RattanCloneResp, +} + +/// # Safety +/// `fd` must be valid, `token` must be valid. +#[inline] +pub unsafe fn ioctl_clone( + fd: RawFd, + token: u64, + chunk_addr: u64, + queue_id: u32, +) -> io::Result { + let mut buf = RattanCloneIoctl { + req: RattanCloneReq { + token, + chunk_addr, + queue_id, + _pad: 0, + }, + }; + let ret = unsafe { libc::ioctl(fd, IOCTL_CLONE, &mut buf as *mut RattanCloneIoctl) }; + if ret < 0 { + return Err(io::Error::last_os_error()); + } + // SAFETY: the kernel wrote the response on success + Ok(unsafe { buf.resp.new_token }) +} + +/// # Safety +/// `fd` must be valid. +#[inline] +pub unsafe fn ioctl_device_id(fd: RawFd) -> io::Result { + let mut buf: u32 = 0; + let ret = unsafe { libc::ioctl(fd, IOCTL_DEVICE_ID, &mut buf as *mut u32) }; + if ret < 0 { + return Err(io::Error::last_os_error()); + } + Ok(buf) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_struct_sizes() { + assert_eq!(std::mem::size_of::(), 24); + assert_eq!(std::mem::size_of::(), 16); + assert_eq!(std::mem::size_of::(), 24); + assert_eq!(std::mem::size_of::(), 32); + assert_eq!(std::mem::size_of::(), 128); + } + + #[test] + fn test_ring_offsets() { + let ring: RattanRing = unsafe { std::mem::zeroed() }; + let base = &ring as *const _ as usize; + assert_eq!(&ring.producer as *const _ as usize - base, 0); + assert_eq!(&ring.consumer as *const _ as usize - base, 64); + assert_eq!(&ring.flags as *const _ as usize - base, 68); + } + + #[test] + fn test_ioctl_codes() { + assert_eq!(IOCTL_REG_UMEM, 0x40185201); + assert_eq!(IOCTL_REG_RINGS, 0x40105202); + assert_eq!(IOCTL_REG_RINGS_Q, 0x40185206); + assert_eq!(IOCTL_START, 0x40085204); + assert_eq!(IOCTL_STOP, 0x5205); + assert_eq!(IOCTL_KICK_RX_Q, 0x4004520e); + assert_eq!(IOCTL_DEVICE_ID, 0x8004520f) + } +} diff --git a/rvnic/librvnic/src/umem.rs b/rvnic/librvnic/src/umem.rs new file mode 100644 index 00000000..43b3c056 --- /dev/null +++ b/rvnic/librvnic/src/umem.rs @@ -0,0 +1,352 @@ +//! UMEM - Userspace Memory Pool +//! +//! UMEM is a contiguous memory region allocated by userspace and registered +//! with the kernel. The region is divided into fixed-size chunks that are +//! used for packet data. +//! +//! # Example +//! +//! ```no_run +//! use rvnic::{Umem, UmemBuilder}; +//! +//! let umem = UmemBuilder::new() +//! .chunk_size(2048) +//! .headroom(128) +//! .num_chunks(4096) +//! .build() +//! .expect("failed to allocate UMEM"); +//! +//! println!("UMEM: {} chunks of {} bytes", umem.num_chunks(), umem.chunk_size()); +//! ``` + +use std::io; +use std::ptr::NonNull; + +use crate::Result; +use crate::sys::{RATTAN_DEFAULT_CHUNK_SIZE, RATTAN_MAX_CHUNK_SIZE, RATTAN_MIN_CHUNK_SIZE}; + +/// Builder for creating UMEM regions +#[derive(Debug, Clone)] +pub struct UmemBuilder { + chunk_size: u32, + headroom: u32, + num_chunks: u32, +} + +impl Default for UmemBuilder { + fn default() -> Self { + Self::new() + } +} + +impl UmemBuilder { + /// Create a new UMEM builder with default settings + pub fn new() -> Self { + Self { + chunk_size: RATTAN_DEFAULT_CHUNK_SIZE, + headroom: 0, + num_chunks: 4096, + } + } + + /// Set the chunk size (must be power of 2, 256-65536) + pub fn chunk_size(mut self, size: u32) -> Self { + self.chunk_size = size; + self + } + + /// Set the headroom before packet data in each chunk + pub fn headroom(mut self, headroom: u32) -> Self { + self.headroom = headroom; + self + } + + /// Set the number of chunks + pub fn num_chunks(mut self, count: u32) -> Self { + self.num_chunks = count; + self + } + + /// Build the UMEM region + pub fn build(self) -> Result { + Umem::new(self.chunk_size, self.headroom, self.num_chunks) + } +} + +/// UMEM - Userspace Memory Pool +/// +/// A contiguous memory region divided into fixed-size chunks for packet data. +/// The memory is allocated via mmap and will be pinned by the kernel when +/// registered with a device. +pub struct Umem { + /// Pointer to the mmap'd memory + ptr: NonNull, + /// Total size of the memory region + len: usize, + /// Size of each chunk + chunk_size: u32, + /// Headroom before packet data + headroom: u32, + /// Number of chunks + num_chunks: u32, +} + +impl Umem { + /// Create a new UMEM region + /// + /// # Arguments + /// - `chunk_size`: Size of each chunk (must be power of 2, 256-65536) + /// - `headroom`: Headroom before packet data in each chunk + /// - `num_chunks`: Number of chunks to allocate + /// + /// # Errors + /// Returns an error if: + /// - `chunk_size` is not a power of 2 + /// - `chunk_size` is outside the valid range (256-65536) + /// - `headroom` >= `chunk_size` + /// - `num_chunks` is 0 + /// - mmap fails + pub fn new(chunk_size: u32, headroom: u32, num_chunks: u32) -> Result { + // Validate chunk_size + if !chunk_size.is_power_of_two() { + return Err(crate::Error::InvalidParam("chunk_size must be power of 2")); + } + if !(RATTAN_MIN_CHUNK_SIZE..=RATTAN_MAX_CHUNK_SIZE).contains(&chunk_size) { + return Err(crate::Error::InvalidParam("chunk_size must be 256-65536")); + } + + // Validate headroom + if headroom >= chunk_size { + return Err(crate::Error::InvalidParam("headroom must be < chunk_size")); + } + + // Validate num_chunks + if num_chunks == 0 { + return Err(crate::Error::InvalidParam("num_chunks must be > 0")); + } + if num_chunks > crate::sys::RATTAN_MAX_CHUNKS { + return Err(crate::Error::InvalidParam( + "num_chunks exceeds RATTAN_MAX_CHUNKS", + )); + } + + let len_u64 = (chunk_size as u64) + .checked_mul(num_chunks as u64) + .ok_or(crate::Error::InvalidParam("UMEM size overflow"))?; + if len_u64 > crate::sys::RATTAN_MAX_UMEM_SIZE { + return Err(crate::Error::InvalidParam( + "UMEM size exceeds RATTAN_MAX_UMEM_SIZE", + )); + } + if len_u64 > usize::MAX as u64 { + return Err(crate::Error::InvalidParam( + "UMEM too large for this platform", + )); + } + let len = len_u64 as usize; + + // Allocate memory via mmap + // MAP_POPULATE pre-faults the pages to avoid page faults during operation + let ptr = unsafe { + libc::mmap( + std::ptr::null_mut(), + len, + libc::PROT_READ | libc::PROT_WRITE, + libc::MAP_PRIVATE | libc::MAP_ANONYMOUS | libc::MAP_POPULATE, + -1, + 0, + ) + }; + + if ptr == libc::MAP_FAILED { + return Err(crate::Error::Io(io::Error::last_os_error())); + } + + Ok(Self { + ptr: NonNull::new(ptr as *mut u8).expect("mmap returned null"), + len, + chunk_size, + headroom, + num_chunks, + }) + } + + /// Get the base address of the UMEM region + #[inline] + pub fn addr(&self) -> u64 { + self.ptr.as_ptr() as u64 + } + + /// Get the total length of the UMEM region in bytes + #[inline] + pub fn len(&self) -> usize { + self.len + } + + /// Check if UMEM is empty (always false for valid UMEM) + #[inline] + pub fn is_empty(&self) -> bool { + self.len == 0 + } + + /// Get the chunk size in bytes + #[inline] + pub fn chunk_size(&self) -> u32 { + self.chunk_size + } + + /// Get the headroom in bytes + #[inline] + pub fn headroom(&self) -> u32 { + self.headroom + } + + /// Get the number of chunks + #[inline] + pub fn num_chunks(&self) -> u32 { + self.num_chunks + } + + /// Get a raw pointer to a chunk by index + /// + /// Returns `None` if the index is out of bounds. + #[inline] + pub fn chunk_ptr(&self, index: u32) -> Option<*mut u8> { + if index >= self.num_chunks { + return None; + } + let offset = index as usize * self.chunk_size as usize; + Some(unsafe { self.ptr.as_ptr().add(offset) }) + } + + /// Get a raw pointer to the data area of a chunk (after headroom) + /// + /// Returns `None` if the index is out of bounds. + #[inline] + pub fn data_ptr(&self, index: u32) -> Option<*mut u8> { + self.chunk_ptr(index) + .map(|p| unsafe { p.add(self.headroom as usize) }) + } + + /// Get the offset of a chunk from the UMEM base address + #[inline] + pub fn chunk_offset(&self, index: u32) -> Option { + if index >= self.num_chunks { + return None; + } + Some(index as u64 * self.chunk_size as u64) + } + + /// Convert an offset to a chunk index + /// + /// Returns `None` if the offset is not chunk-aligned or out of bounds. + #[inline] + pub fn offset_to_index(&self, offset: u64) -> Option { + // Check alignment (chunk_size is power of 2) + if offset & (self.chunk_size as u64 - 1) != 0 { + return None; + } + let index = offset / self.chunk_size as u64; + if index >= self.num_chunks as u64 { + return None; + } + Some(index as u32) + } +} + +impl Drop for Umem { + fn drop(&mut self) { + unsafe { + libc::munmap(self.ptr.as_ptr() as *mut libc::c_void, self.len); + } + } +} + +// SAFETY: Umem is just a pointer to mmap'd memory with no interior mutability +// concerns beyond what the raw pointer provides +unsafe impl Send for Umem {} +unsafe impl Sync for Umem {} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_umem_builder_default() { + let builder = UmemBuilder::new(); + assert_eq!(builder.chunk_size, 256); + assert_eq!(builder.headroom, 0); + assert_eq!(builder.num_chunks, 4096); + } + + #[test] + fn test_umem_builder_custom() { + let builder = UmemBuilder::new() + .chunk_size(4096) + .headroom(256) + .num_chunks(1024); + assert_eq!(builder.chunk_size, 4096); + assert_eq!(builder.headroom, 256); + assert_eq!(builder.num_chunks, 1024); + } + + #[test] + fn test_umem_creation() { + let umem = Umem::new(2048, 128, 64).expect("failed to create UMEM"); + assert_eq!(umem.chunk_size(), 2048); + assert_eq!(umem.headroom(), 128); + assert_eq!(umem.num_chunks(), 64); + assert_eq!(umem.len(), 2048 * 64); + } + + #[test] + fn test_umem_chunk_access() { + let umem = Umem::new(2048, 128, 64).expect("failed to create UMEM"); + + // Valid indices + assert!(umem.chunk_ptr(0).is_some()); + assert!(umem.chunk_ptr(63).is_some()); + + // Invalid index + assert!(umem.chunk_ptr(64).is_none()); + + // Check offsets + assert_eq!(umem.chunk_offset(0), Some(0)); + assert_eq!(umem.chunk_offset(1), Some(2048)); + assert_eq!(umem.chunk_offset(63), Some(63 * 2048)); + } + + #[test] + fn test_umem_offset_to_index() { + let umem = Umem::new(2048, 0, 64).expect("failed to create UMEM"); + + // Valid offsets + assert_eq!(umem.offset_to_index(0), Some(0)); + assert_eq!(umem.offset_to_index(2048), Some(1)); + assert_eq!(umem.offset_to_index(63 * 2048), Some(63)); + + // Unaligned offset + assert_eq!(umem.offset_to_index(100), None); + + // Out of bounds + assert_eq!(umem.offset_to_index(64 * 2048), None); + } + + #[test] + fn test_umem_invalid_params() { + // Not power of 2 + assert!(Umem::new(1000, 0, 64).is_err()); + + // Too small + assert!(Umem::new(128, 0, 64).is_err()); + + // Too large + assert!(Umem::new(128 * 1024, 0, 64).is_err()); + + // Headroom >= chunk_size + assert!(Umem::new(2048, 2048, 64).is_err()); + + // Zero chunks + assert!(Umem::new(2048, 0, 0).is_err()); + } +} diff --git a/scripts/.gitignore b/scripts/.gitignore index a6c57f5f..f6d53f17 100644 --- a/scripts/.gitignore +++ b/scripts/.gitignore @@ -1 +1,2 @@ *.json +__pycache__/ diff --git a/scripts/install.sh b/scripts/install.sh index 83b1c9c9..4378e6cf 100755 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -191,11 +191,12 @@ download_rattan() { # --- Install steps --- install_binary() { - sudo install -m 755 "$1" /usr/local/bin/rattan - sudo setcap 'cap_dac_override,cap_dac_read_search,cap_sys_ptrace,cap_net_admin,cap_sys_admin,cap_net_raw+ep' /usr/local/bin/rattan + local binary_name=$(basename $1) + sudo install -m 755 "$1" /usr/local/bin/${binary_name} + sudo setcap 'cap_dac_override,cap_dac_read_search,cap_sys_ptrace,cap_net_admin,cap_sys_admin,cap_net_raw+ep' /usr/local/bin/${binary_name} } -# Config systemd-networkd to not change MAC address of veth interfaces +# Config systemd-networkd to not change MAC address of veth and rvnic interfaces # Ref: https://github.com/stack-rs/rattan/issues/42 config_networkd() { if [ -d "/lib/systemd/network" ]; then @@ -211,16 +212,41 @@ Driver=veth MACAddressPolicy=none EOF " - sudo systemctl daemon-reload - sudo systemctl restart systemd-networkd.service - fi + sudo sh -c "cat </lib/systemd/network/80-rattan-rv.link +[Match] +OriginalName=rattan* + +[Link] +MACAddressPolicy=none +EOF +" + sudo systemctl daemon-reload + sudo systemctl restart systemd-networkd.service + fi +} + +config_nm() { + sudo mkdir -p /etc/NetworkManager/conf.d + sudo sh -c "cat </etc/NetworkManager/conf.d/80-rattan-unmanaged.conf +# Do not manage interfaces created and owned by rattan. +# These interfaces manage their own lifecycle (addresses, up/down state). +[keyfile] +unmanaged-devices=interface-name:ns*-v*-*;interface-name:rattan* +EOF +" + sudo systemctl reload NetworkManager.service } install_rattan() { - local _binary_path="$1" - install_binary "$_binary_path" - config_networkd - echo "$APP_NAME installed successfully to /usr/local/bin/$APP_NAME" + local binary_path=${1:-"rattan"} + install_binary "$binary_path" + + if systemctl cat systemd-networkd.service >/dev/null 2>&1; then + config_networkd + fi + if systemctl cat NetworkManager.service >/dev/null 2>&1; then + config_nm + fi } # --- Entry point --- @@ -239,8 +265,9 @@ Options: --help|-h Print this help message Examples: - $myname # Download and install latest release - $myname target/release/rattan # Install a locally-built binary + $myname # Download and install latest release + $myname target/release/rattan # Install a locally-built binary + $myname target/release/rattan-rv # Install a locally-built binary using rattan vnic EOL exit 1 diff --git a/src/channel.rs b/src/channel.rs index 0e1d36a2..836c1538 100644 --- a/src/channel.rs +++ b/src/channel.rs @@ -335,7 +335,7 @@ impl ChannelArgs { links: links_config, resource: RattanResourceConfig { memory: None, - cpu: Some(vec![2]), + cpu: None, }, ..Default::default() }) diff --git a/src/main-rv.rs b/src/main-rv.rs new file mode 100644 index 00000000..c4d8ef46 --- /dev/null +++ b/src/main-rv.rs @@ -0,0 +1,803 @@ +use std::{ + borrow::Cow, + fs::File, + path::PathBuf, + process::{ExitCode, Stdio, Termination}, + sync::{atomic::AtomicBool, Arc}, + time::Duration, +}; + +use clap::{Args, Parser, Subcommand, ValueEnum}; +use figment::{ + providers::{Env, Format, Serialized, Toml}, + Figment, +}; +use nix::sys::signal::{self, Signal}; +use nix::unistd::Pid; +use once_cell::sync::OnceCell; + +use rattan_env::env::rvnic::{ + RvnicDriver as RattanPacketDriver, RvnicEnv as RattanNetEnv, RvnicEnvConfig as RattanEnvConfig, + RvnicPacket as RattanPacket, +}; +use rattan_env::StdNetEnvMode; + +use rattan_core::radix::PacketLogMode; +use rattan_core::radix::RattanRadix; +use rattan_core::{config::RattanConfig, radix::TaskResultNotify}; +use rattan_env::RattanEnv; +use rattan_log::convert_log_to_pcapng; +use serde::{Deserialize, Serialize}; +use shadow_rs::shadow; +use tracing::warn; +use tracing_subscriber::Layer; +use tracing_subscriber::{layer::SubscriberExt, util::SubscriberInitExt}; + +use crate::{ + build::{CARGO_FEATURES, CLAP_LONG_VERSION}, + visualize_trace::{write_visualize_trace, OutputMode}, +}; + +mod channel; +mod env_var; +mod visualize_trace; +// mod log_converter; +#[cfg(feature = "nat")] +mod nat; +// mod docker; + +use env_var::add_runtime_env_var; + +// const CONFIG_PORT_BASE: u16 = 8086; + +shadow!(build); + +static LEFT_PID: OnceCell = OnceCell::new(); +static RIGHT_PID: OnceCell = OnceCell::new(); + +const RATTAN_LONG_VERSION: &str = + shadow_rs::formatcp!("{}\nbuild_features:{}", CLAP_LONG_VERSION, CARGO_FEATURES); + +fn parse_duration(delay: &str) -> Result { + let span: jiff::Span = delay.parse()?; + Duration::try_from(span) +} + +#[derive(Debug, Parser, Clone)] +#[command(rename_all = "kebab-case")] +#[command(version, propagate_version = true, long_version = RATTAN_LONG_VERSION)] +pub struct Arguments { + // Verbose debug output + // #[arg(short, long)] + // verbose: bool, + // Run in docker mode + // #[arg(long)] + // docker: bool, + #[command(subcommand)] + subcommand: CliCommand, + + /// Generate config file instead of running a instance + /// + /// If this flag is set, the program will only generate the config to stdout and exit. + #[arg(long, global = true)] + generate: bool, + + /// Generate visualized trace until `visualize_trace` since the trace starts + /// If this is set, the program will only generate the csv to stdout and exit. + /// Two output modes are supported: CSV for human and Parquet for scripts. + #[arg( + long, + global = true, + value_parser = parse_duration, + value_name = "End Time" + )] + visualize_trace: Option, + + /// Start time of visualize_trace, 0s as default. + #[arg(long, requires = "visualize_trace", global = true, value_name = "Start Time", value_parser = parse_duration)] + visualize_trace_start: Option, + + /// Path to output the visualized trace. If not set, output to stdout. + /// The extension name would be changed automatically according to `visualize_trace_mode` + #[arg(long, requires = "visualize_trace", global = true, value_name = "File")] + visualize_trace_output: Option, + + /// Mode to output the visualize_trace. + #[arg( + long, + requires = "visualize_trace", + global = true, + value_name = "Mode", + default_value = "human-json" + )] + visualize_trace_mode: OutputMode, + + /// Used in isolated mode only. If set, stdout of left is passed to output of this program. + #[arg(long, global = true)] + left_stdout: bool, + + /// Used in isolated mode only. If set, stdout of rihgt is passed to output of this program. + #[arg(long, global = true)] + right_stdout: bool, + + /// Used in isolated mode only. If set, stderr of left is passed to output of this program. + #[arg(long, global = true)] + left_stderr: bool, + + /// Used in isolated mode only. If set, stderr of right is passed to output of this program. + #[arg(long, global = true)] + right_stderr: bool, + + /// Generate config file to the specified path instead of stdout + #[arg(long, requires = "generate", global = true, value_name = "File")] + generate_path: Option, + + #[cfg(feature = "http")] + /// Enable HTTP control server (overwrite config) + #[arg(long, global = true)] + http: bool, + #[cfg(feature = "http")] + /// HTTP control server port (overwrite config) (default: 8086) + #[arg(short, long, value_name = "Port", global = true)] + port: Option, + + /// The file to store compressed packet log (overwrite config) (default: None) + #[arg(long, value_name = "File", global = true)] + packet_log: Option, + + /// If this flag is set, raw packet header would be recorded for packet_log. + #[arg( + long, + requires = "packet_log", + value_name = "Mode", + global = true, + default_value = "compact-tcp" + )] + packet_log_mode: PacketLogMode, + + /// Enable logging to file + #[arg(long, global = true)] + file_log: bool, + // This "requires" field uses an underscore '_' instead of a dash '-' since "file_log" is a + // field name instead of a group name + /// File log path, default to $CACHE_DIR/rattan/core.log + #[arg(long, value_name = "File", requires = "file_log", global = true)] + file_log_path: Option, + + #[cfg(feature = "nat")] + /// Disable NAT in compatible mode + #[arg(long, global = true)] + no_nat: bool, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct TaskCommands { + #[serde(skip_serializing_if = "::std::option::Option::is_none")] + pub left: Option>, + #[serde(skip_serializing_if = "::std::option::Option::is_none")] + pub right: Option>, + #[serde(skip_serializing_if = "::std::option::Option::is_none")] + pub shell: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct DisplayTaskCommands { + pub commands: TaskCommands, +} + +/// Convert Rattan Packet Log file to pcapng file for each side +#[derive(Args, Debug, Default, Clone)] +#[command(rename_all = "kebab-case")] +pub struct ConvertLogArgs { + /// Input Rattan Packet Log file path + pub input: PathBuf, + /// Output pcapng file name prefix. take the input as default value + pub output: Option, +} + +#[derive(Subcommand, Debug, Clone)] +enum CliCommand { + /// Run a templated channel with command line arguments. + Link(channel::ChannelArgs), + /// Run the instance according to the config. + Run(RunArgs), + /// Convert Rattan packet log into .pcapng file. + Convert(ConvertLogArgs), +} + +#[derive(Args, Debug, Default, Clone)] +#[command(rename_all = "kebab-case")] +pub struct RunArgs { + /// Use config file to run a instance. + #[arg(short, long, value_name = "Config File")] + pub config: PathBuf, + /// Command to run in left ns. Can be used in compatible and isolated mode + #[arg(long = "left", num_args = 0..)] + left_command: Option>, + /// Command to run in right ns. Only used in isolated mode + #[arg(long = "right", num_args = 0..)] + right_command: Option>, +} + +#[derive(Debug, Serialize, Deserialize, Clone, ValueEnum, Copy, Default)] +pub enum TaskShell { + #[default] + Default, + Sh, + Bash, + Zsh, + Fish, +} + +impl TaskShell { + fn shell(&self) -> Cow<'_, str> { + match self { + TaskShell::Default => { + let shell = std::env::var("SHELL").unwrap_or("/bin/sh".to_string()); + Cow::Owned(shell) + } + TaskShell::Sh => Cow::Borrowed("/bin/sh"), + TaskShell::Bash => Cow::Borrowed("/bin/bash"), + TaskShell::Zsh => Cow::Borrowed("/bin/zsh"), + TaskShell::Fish => Cow::Borrowed("/bin/fish"), + } + } +} + +fn main() -> ExitCode { + // Parse Arguments + let opts = Arguments::parse(); + tracing::debug!("{:?}", opts); + // if opts.docker { + // docker::docker_main(opts).unwrap(); + // return; + // } + + // Install Tracing Subscriber + let subscriber = + tracing_subscriber::registry().with(tracing_subscriber::fmt::layer().with_filter( + tracing_subscriber::EnvFilter::try_from_default_env().unwrap_or_else(|_| "warn".into()), + )); + let _guard: Option<_> = if opts.file_log { + if let Some((log_dir, file_name)) = opts + .file_log_path + .and_then(|path| { + let file_name = path.file_name().and_then(|f| f.to_str()); + let log_dir = path + .parent() + .map(|p| p.to_path_buf()) + .and_then(|p| file_name.map(|f| (p, f.to_string()))); + log_dir + }) + .or_else(|| { + dirs::cache_dir() + .map(|mut p| { + p.push("rattan"); + p + }) + .map(|log_dir| (log_dir, "core.log".to_string())) + }) + { + if let Err(e) = std::fs::create_dir_all(&log_dir) { + tracing::error!("Failed to create log directory: {:?}", e); + return ExitCode::from(74); + } + let file_logger = match std::fs::OpenOptions::new() + .create(true) + .append(true) + .open(log_dir.join(file_name)) + { + Ok(f) => f, + Err(e) => { + tracing::error!("Failed to open log file: {:?}", e); + return ExitCode::from(74); + } + }; + // let file_logger = tracing_appender::rolling::daily(log_dir, file_name_prefix); + let (non_blocking, guard) = tracing_appender::non_blocking(file_logger); + let env_filter = tracing_subscriber::EnvFilter::try_from_env("RATTAN_LOG") + .unwrap_or_else(|_| "info".into()); + subscriber + .with( + tracing_subscriber::fmt::layer() + .with_writer(non_blocking) + .with_filter(env_filter), + ) + .init(); + Some(guard) + } else { + subscriber.init(); + None + } + } else { + subscriber.init(); + None + }; + + let left_handle_finished = Arc::new(AtomicBool::new(false)); + let left_handle_finished_inner = left_handle_finished.clone(); + let right_handle_finished = Arc::new(AtomicBool::new(false)); + let right_handle_finished_inner = right_handle_finished.clone(); + // Install Ctrl-C Handler + ctrlc::set_handler(move || { + if !left_handle_finished_inner.load(std::sync::atomic::Ordering::Relaxed) { + if let Some(pid) = LEFT_PID.get() { + let _ = signal::kill(Pid::from_raw(*pid), Signal::SIGTERM).inspect_err(|e| { + tracing::error!("Failed to send SIGTERM to left task: {}", e); + }); + } + } + if !right_handle_finished_inner.load(std::sync::atomic::Ordering::Relaxed) { + if let Some(pid) = RIGHT_PID.get() { + let _ = signal::kill(Pid::from_raw(*pid), Signal::SIGTERM).inspect_err(|e| { + tracing::error!("Failed to send SIGTERM to right task: {}", e); + }); + } + } + }) + .expect("unable to install ctrl+c handler"); + + // Main CLI + let main_cli = || -> rattan_core::error::Result<()> { + let (mut config, commands) = match opts.subcommand { + CliCommand::Run(mut args) => { + tracing::info!("Loading config from {}", args.config.display()); + if !args.config.exists() { + return Err(rattan_core::error::Error::ConfigError(format!( + "Config file {} does not exist", + args.config.display() + ))); + } + let config: RattanConfig = Figment::new() + .merge(Toml::file(&args.config)) + .merge(Env::prefixed("RATTAN_")) + .extract() + .map_err(|e| rattan_core::error::Error::ConfigError(e.to_string()))?; + let commands = Figment::new() + .merge(Toml::file(args.config).nested()) + .merge(Serialized::from( + TaskCommands { + left: args.left_command.take(), + right: args.right_command.take(), + shell: None, + }, + "commands", + )) + .merge(Env::prefixed("RATTAN_").profile("commands")) + .select("commands") + .extract::() + .map_err(|e| rattan_core::error::Error::ConfigError(e.to_string()))?; + (config, commands) + } + CliCommand::Link(mut args) => { + let commands = Figment::new() + .merge(Serialized::from( + TaskCommands { + left: args.command.take(), + right: None, + shell: Some(args.shell), + }, + "commands", + )) + .merge(Env::prefixed("RATTAN_").profile("commands")) + .select("commands") + .extract::() + .map_err(|e| rattan_core::error::Error::ConfigError(e.to_string()))?; + let config = args.build_rattan_config::()?; + (config, commands) + } + CliCommand::Convert(args) => { + convert_log_to_pcapng(args.input, args.output)?; + return Ok(()); + } + }; + + // Overwrite config with CLI options + #[cfg(feature = "http")] + if opts.http { + config.http.enable = true; + } + #[cfg(feature = "http")] + if let Some(port) = opts.port { + config.http.port = port; + } + if let Some(packet_log) = opts.packet_log { + config.general.packet_log = Some(packet_log); + config.general.packet_log_mode = opts.packet_log_mode.into(); + } + + tracing::debug!(?config); + + // Generate config + if opts.generate { + // DONE: generate commands as well + let mut toml_string = toml::to_string_pretty(&config) + .map_err(|e| rattan_core::error::Error::ConfigError(e.to_string()))?; + let display_commands = DisplayTaskCommands { commands }; + let command_string = toml::to_string_pretty(&display_commands) + .map_err(|e| rattan_core::error::Error::ConfigError(e.to_string()))?; + + toml_string.push_str(format!("\n{command_string}").as_str()); + if let Some(output) = opts.generate_path { + std::fs::write(output, toml_string)?; + } else { + println!("{toml_string}"); + } + return Ok(()); + } + + if let Some(visualize_trace_length) = opts.visualize_trace { + let mode = opts.visualize_trace_mode; + if let Some(mut output_path) = opts.visualize_trace_output { + output_path.set_extension(mode.get_extension_name()); + + return write_visualize_trace( + mode, + File::create(output_path)?, + config.cells, + opts.visualize_trace_start, + visualize_trace_length, + ); + } else { + return write_visualize_trace( + mode, + std::io::stdout(), + config.cells, + opts.visualize_trace_start, + visualize_trace_length, + ); + } + } + + // Check if the config can correctly spawn + if config.cells.is_empty() { + tracing::warn!("No cells specified in config"); + } + if config.links.is_empty() { + tracing::warn!("No links specified in config"); + } + if config.env.mode == StdNetEnvMode::Container { + return Err(rattan_core::error::Error::ConfigError( + "Container mode is not supported yet".to_string(), + )); + } + if config.env.mode == StdNetEnvMode::Isolated + && (commands.left.is_none() || commands.right.is_none()) + { + return Err(rattan_core::error::Error::ConfigError( + "Isolated mode requires commands to be set in both sides".to_string(), + )); + } + + // Start Rattan + let mut radix = RattanRadix::::new(config)?; + radix.spawn_rattan()?; + tracing::info!("Radix spawned"); + radix.start_rattan()?; + tracing::info!("Rattan started"); + let rattan_id = + >::get_rattan_id(&radix).to_string(); + + let (tx_left, rx) = std::sync::mpsc::channel(); + let tx_right = tx_left.clone(); + + match >::get_mode(&radix) { + StdNetEnvMode::Compatible => { + if opts.left_stdout | opts.left_stderr | opts.right_stdout | opts.right_stderr { + warn!( + "--left-stdout, --left-stderr, --right-stdout, --right-stderr \ + are for isolated mode only and thus ignored." + ); + } + + #[cfg(feature = "nat")] + let left_ip_list = + >::left_ip_list(&radix); + #[cfg(feature = "nat")] + let _nat = if !opts.no_nat { + left_ip_list + .iter() + .find_map(|&(id, ip)| (id == 1).then_some(ip)) + .map(nat::Nat::new) + } else { + None + }; + + let right_ip_list = + >::right_ip_list(&radix); + let left_handle = radix.left_spawn(None, move || { + let mut client_handle = std::process::Command::new("/usr/bin/env"); + add_runtime_env_var(&mut client_handle, right_ip_list, rattan_id); + if let Some(arguments) = commands.left { + client_handle.args(arguments); + } else { + let shell = commands.shell.unwrap_or_default(); + client_handle.arg(shell.shell().as_ref()); + if shell.shell().ends_with("/bash") { + client_handle + .env("PROMPT_COMMAND", "PS1=\"[rattan] $PS1\" PROMPT_COMMAND="); + } + } + tracing::info!("Running {:?}", client_handle); + let mut client_handle = client_handle + .stdin(Stdio::inherit()) + .stdout(Stdio::inherit()) + .stderr(Stdio::inherit()) + .spawn()?; + let pid = client_handle.id() as i32; + tracing::debug!("Left pid: {}", pid); + let _ = LEFT_PID.set(pid); + let status = client_handle.wait()?; + left_handle_finished.store(true, std::sync::atomic::Ordering::Relaxed); + Ok(status) + })?; + match left_handle.join() { + Ok(Ok(status)) => { + if let Some(code) = status.code() { + if code == 0 { + tracing::info!("Left handle {status}"); + Ok(()) + } else { + Err(rattan_core::error::Error::Custom(format!( + "Left handle {status}" + ))) + } + } else { + Err(rattan_core::error::Error::Custom(format!( + "Left handle {status}" + ))) + } + } + Ok(Err(e)) => Err(rattan_core::error::Error::Custom(format!( + "Left handle exited with error: {e:?}" + ))), + Err(e) => Err(rattan_core::error::Error::Custom(format!( + "Fail to join left handle: {e:?}" + ))), + } + } + StdNetEnvMode::Isolated => { + #[cfg(feature = "nat")] + if opts.no_nat { + warn!("--no-nat is only for compatible mode and thus ignored in current isolated mode."); + } + + let ip_list = >::left_ip_list(&radix); + let rattan_id_right = rattan_id.clone(); + let right_handle = radix.right_spawn(Some(tx_right), move || { + let mut server_handle = std::process::Command::new("/usr/bin/env"); + add_runtime_env_var(&mut server_handle, ip_list, rattan_id_right); + if let Some(arguments) = commands.right { + server_handle.args(arguments); + } + tracing::info!("Running in right NS {server_handle:?}"); + let mut server_handle = server_handle + .stdin(Stdio::null()) + .stdout(if opts.right_stdout { + Stdio::inherit() + } else { + Stdio::null() + }) + .stderr(if opts.right_stderr { + Stdio::inherit() + } else { + Stdio::null() + }) + .spawn()?; + let pid = server_handle.id() as i32; + tracing::debug!("Right pid: {pid}"); + let _ = RIGHT_PID.set(pid); + let status = server_handle.wait()?; + right_handle_finished.store(true, std::sync::atomic::Ordering::Relaxed); + Ok(status) + })?; + let ip_list = + >::right_ip_list(&radix); + let left_handle = radix.left_spawn(Some(tx_left), move || { + let mut client_handle = std::process::Command::new("/usr/bin/env"); + add_runtime_env_var(&mut client_handle, ip_list, rattan_id); + if let Some(arguments) = commands.left { + client_handle.args(arguments); + } + tracing::info!("Running in left NS {client_handle:?}"); + let mut client_handle = client_handle + .stdin(Stdio::null()) + .stdout(if opts.left_stdout { + Stdio::inherit() + } else { + Stdio::null() + }) + .stderr(if opts.left_stderr { + Stdio::inherit() + } else { + Stdio::null() + }) + .spawn()?; + let pid = client_handle.id() as i32; + tracing::debug!("Left pid: {pid}"); + let _ = LEFT_PID.set(pid); + let status = client_handle.wait()?; + left_handle_finished.store(true, std::sync::atomic::Ordering::Relaxed); + Ok(status) + })?; + let mut left_res = Ok(()); + let mut right_res = Ok(()); + match rx.recv() { + Ok(notify) => match notify { + TaskResultNotify::Left => { + match left_handle.join() { + Ok(Ok(status)) => { + if let Some(code) = status.code() { + if code == 0 { + tracing::info!("Left handle {status}"); + } else { + left_res = left_res + .and_then(|_| Err(format!("Left handle {status}"))); + } + } else { + left_res = left_res + .and_then(|_| Err(format!("Left handle {status}"))); + } + } + Ok(Err(e)) => { + left_res = left_res.and_then(|_| { + Err(format!("Left handle exited with error: {e:?}")) + }); + // TODO: we may also add other arguments to disable the killing of the other half + if right_handle.is_finished() { + tracing::warn!("Right handle is already finished"); + } else { + tracing::warn!( + "Try to send SIGTERM to right spawned thread" + ); + if let Some(pid) = RIGHT_PID.get() { + let _ = + signal::kill(Pid::from_raw(*pid), Signal::SIGTERM) + .inspect_err(|e| { + tracing::error!( + "Failed to send SIGTERM: {e}" + ); + }); + } + // TODO: we may wait for a while before sending SIGKILL + } + } + Err(e) => { + left_res = left_res.and_then(|_| { + Err(format!("Fail to join left handle: {e:?}")) + }); + } + } + match right_handle.join() { + Ok(Ok(status)) => { + if let Some(code) = status.code() { + if code == 0 { + tracing::info!("Right handle {status}"); + } else { + right_res = right_res.and_then(|_| { + Err(format!("Right handle {status}")) + }); + } + } else { + right_res = right_res + .and_then(|_| Err(format!("Right handle {status}"))); + } + } + Ok(Err(e)) => { + right_res = right_res.and_then(|_| { + Err(format!("Right handle exited with error: {e:?}")) + }); + } + Err(e) => { + right_res = right_res.and_then(|_| { + Err(format!("Fail to join right handle: {e:?}")) + }); + } + } + } + TaskResultNotify::Right => { + match right_handle.join() { + Ok(Ok(status)) => { + if let Some(code) = status.code() { + if code == 0 { + tracing::info!("Right handle {status}"); + } else { + right_res = right_res.and_then(|_| { + Err(format!("Right handle {status}")) + }); + } + } else { + right_res = right_res + .and_then(|_| Err(format!("Right handle {status}"))); + } + } + Ok(Err(e)) => { + right_res = right_res.and_then(|_| { + Err(format!("Right handle exited with error: {e:?}")) + }); + // TODO: we may also add other arguments to disable the killing of the other half + if left_handle.is_finished() { + tracing::warn!("Left handle is already finished"); + } else { + tracing::warn!( + "Try to send SIGTERM to left spawned thread" + ); + if let Some(pid) = LEFT_PID.get() { + let _ = + signal::kill(Pid::from_raw(*pid), Signal::SIGTERM) + .inspect_err(|e| { + tracing::error!( + "Failed to send SIGTERM: {e}" + ); + }); + } + // TODO: we may wait for a while before sending SIGKILL + } + } + Err(e) => { + right_res = right_res.and_then(|_| { + Err(format!("Fail to join right handle: {e:?}")) + }); + } + } + match left_handle.join() { + Ok(Ok(status)) => { + if let Some(code) = status.code() { + if code == 0 { + tracing::info!("Left handle {status}"); + } else { + left_res = left_res + .and_then(|_| Err(format!("Left handle {status}"))); + } + } else { + left_res = left_res + .and_then(|_| Err(format!("Left handle {status}"))); + } + } + Ok(Err(e)) => { + left_res = left_res.and_then(|_| { + Err(format!("Left handle exited with error: {e:?}")) + }); + } + Err(e) => { + left_res = left_res.and_then(|_| { + Err(format!("Fail to join left handle: {e:?}")) + }); + } + } + } + }, + Err(e) => { + return Err(rattan_core::error::Error::ChannelError(format!( + "Fail to receive from channel: {e:?}" + ))); + } + } + match (left_res, right_res) { + (Ok(()), Ok(())) => Ok(()), + (Err(e), Ok(())) => Err(rattan_core::error::Error::Custom(e)), + (Ok(()), Err(e)) => Err(rattan_core::error::Error::Custom(e)), + (Err(e1), Err(e2)) => { + Err(rattan_core::error::Error::Custom(format!("{e1}. {e2}"))) + } + } + } + StdNetEnvMode::Container => Ok(()), + } + + // get the last byte of rattan_base as the port number + // let port = CONFIG_PORT_BASE - 1 + // + match rattan_base { + // std::net::IpAddr::V4(ip) => ip.octets()[3], + // std::net::IpAddr::V6(ip) => ip.octets()[15], + // } as u16; + // let config = RattanMachineConfig { original_ns, port }; + }; + match main_cli() { + Ok(()) => ExitCode::SUCCESS, + Err(e) => { + tracing::error!("{e}"); + e.report() + } + } +}