From 061a8ac6deab9c9043e59dcceea2e4de17dc450b Mon Sep 17 00:00:00 2001 From: Tamir Duberstein Date: Sat, 18 Oct 2025 15:16:22 -0400 Subject: [PATCH 01/53] samples: rust: platform: remove trailing commas This prepares for a later commit in which we introduce a custom formatting macro; that macro doesn't handle trailing commas so just remove them. Acked-by: Danilo Krummrich Reviewed-by: Benno Lossin Signed-off-by: Tamir Duberstein Link: https://patch.msgid.link/20251018-cstr-core-v18-1-9378a54385f8@gmail.com Signed-off-by: Miguel Ojeda --- samples/rust/rust_driver_platform.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/samples/rust/rust_driver_platform.rs b/samples/rust/rust_driver_platform.rs index 6473baf4f120..8a82e251820f 100644 --- a/samples/rust/rust_driver_platform.rs +++ b/samples/rust/rust_driver_platform.rs @@ -146,7 +146,7 @@ impl SampleDriver { let name = c_str!("test,u32-optional-prop"); let prop = fwnode.property_read::(name).or(0x12); - dev_info!(dev, "'{name}'='{prop:#x}' (default = 0x12)\n",); + dev_info!(dev, "'{name}'='{prop:#x}' (default = 0x12)\n"); // A missing required property will print an error. Discard the error to // prevent properties_parse from failing in that case. @@ -161,7 +161,7 @@ impl SampleDriver { let prop: [i16; 4] = fwnode.property_read(name).required_by(dev)?; dev_info!(dev, "'{name}'='{prop:?}'\n"); let len = fwnode.property_count_elem::(name)?; - dev_info!(dev, "'{name}' length is {len}\n",); + dev_info!(dev, "'{name}' length is {len}\n"); let name = c_str!("test,i16-array"); let prop: KVec = fwnode.property_read_array_vec(name, 4)?.required_by(dev)?; From d9252f1be2f91c61e43e056ed1420ebdb68ca12f Mon Sep 17 00:00:00 2001 From: Tamir Duberstein Date: Sat, 18 Oct 2025 15:16:23 -0400 Subject: [PATCH 02/53] rust_binder: remove trailing comma This prepares for a later commit in which we introduce a custom formatting macro; that macro doesn't handle trailing commas so just remove this one. Reviewed-by: Alice Ryhl Signed-off-by: Tamir Duberstein Link: https://patch.msgid.link/20251018-cstr-core-v18-2-9378a54385f8@gmail.com Signed-off-by: Miguel Ojeda --- drivers/android/binder/process.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/android/binder/process.rs b/drivers/android/binder/process.rs index f13a747e784c..d8111c990f21 100644 --- a/drivers/android/binder/process.rs +++ b/drivers/android/binder/process.rs @@ -596,7 +596,7 @@ impl Process { " ref {}: desc {} {}node {debug_id} s {strong} w {weak}", r.debug_id, r.handle, - if dead { "dead " } else { "" }, + if dead { "dead " } else { "" } ); } } From 5aed9677e507fdf43510fc62774fe40a03c47f31 Mon Sep 17 00:00:00 2001 From: Tamir Duberstein Date: Sat, 18 Oct 2025 15:16:24 -0400 Subject: [PATCH 03/53] rust_binder: use `kernel::fmt` Reduce coupling to implementation details of the formatting machinery by avoiding direct use for `core`'s formatting traits and macros. This backslid in commit eafedbc7c050 ("rust_binder: add Rust Binder driver"). Reviewed-by: Alice Ryhl Signed-off-by: Tamir Duberstein Link: https://patch.msgid.link/20251018-cstr-core-v18-3-9378a54385f8@gmail.com Signed-off-by: Miguel Ojeda --- drivers/android/binder/error.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/drivers/android/binder/error.rs b/drivers/android/binder/error.rs index 9921827267d0..b24497cfa292 100644 --- a/drivers/android/binder/error.rs +++ b/drivers/android/binder/error.rs @@ -2,6 +2,7 @@ // Copyright (C) 2025 Google LLC. +use kernel::fmt; use kernel::prelude::*; use crate::defs::*; @@ -76,8 +77,8 @@ impl From for BinderError { } } -impl core::fmt::Debug for BinderError { - fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { +impl fmt::Debug for BinderError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self.reply { BR_FAILED_REPLY => match self.source.as_ref() { Some(source) => f From 0dac8cf44b2d82fbd6b57d961dcbacccf0ae190a Mon Sep 17 00:00:00 2001 From: Tamir Duberstein Date: Sat, 18 Oct 2025 15:16:25 -0400 Subject: [PATCH 04/53] rust_binder: use `core::ffi::CStr` method names Prepare for `core::ffi::CStr` taking the place of `kernel::str::CStr` by avoiding methods that only exist on the latter. This backslid in commit eafedbc7c050 ("rust_binder: add Rust Binder driver"). Reviewed-by: Alice Ryhl Signed-off-by: Tamir Duberstein Link: https://patch.msgid.link/20251018-cstr-core-v18-4-9378a54385f8@gmail.com Signed-off-by: Miguel Ojeda --- drivers/android/binder/stats.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/android/binder/stats.rs b/drivers/android/binder/stats.rs index a83ec111d2cb..10c43679d5c3 100644 --- a/drivers/android/binder/stats.rs +++ b/drivers/android/binder/stats.rs @@ -72,7 +72,7 @@ mod strings { // SAFETY: Accessing `binder_command_strings` is always safe. let c_str_ptr = unsafe { binder_command_strings[i] }; // SAFETY: The `binder_command_strings` array only contains nul-terminated strings. - let bytes = unsafe { CStr::from_char_ptr(c_str_ptr) }.as_bytes(); + let bytes = unsafe { CStr::from_char_ptr(c_str_ptr) }.to_bytes(); // SAFETY: The `binder_command_strings` array only contains strings with ascii-chars. unsafe { from_utf8_unchecked(bytes) } } @@ -81,7 +81,7 @@ mod strings { // SAFETY: Accessing `binder_return_strings` is always safe. let c_str_ptr = unsafe { binder_return_strings[i] }; // SAFETY: The `binder_command_strings` array only contains nul-terminated strings. - let bytes = unsafe { CStr::from_char_ptr(c_str_ptr) }.as_bytes(); + let bytes = unsafe { CStr::from_char_ptr(c_str_ptr) }.to_bytes(); // SAFETY: The `binder_command_strings` array only contains strings with ascii-chars. unsafe { from_utf8_unchecked(bytes) } } From 5cc5d805e3ca44831ee668762dc6ee8876d14a37 Mon Sep 17 00:00:00 2001 From: Tamir Duberstein Date: Sat, 18 Oct 2025 15:16:26 -0400 Subject: [PATCH 05/53] rnull: use `kernel::fmt` Reduce coupling to implementation details of the formatting machinery by avoiding direct use for `core`'s formatting traits and macros. This backslid in commit d969d504bc13 ("rnull: enable configuration via `configfs`") and commit 34585dc649fb ("rnull: add soft-irq completion support"). Acked-by: Andreas Hindborg Signed-off-by: Tamir Duberstein Link: https://patch.msgid.link/20251018-cstr-core-v18-5-9378a54385f8@gmail.com Signed-off-by: Miguel Ojeda --- drivers/block/rnull/configfs.rs | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/drivers/block/rnull/configfs.rs b/drivers/block/rnull/configfs.rs index 8498e9bae6fd..6713a6d92391 100644 --- a/drivers/block/rnull/configfs.rs +++ b/drivers/block/rnull/configfs.rs @@ -1,12 +1,13 @@ // SPDX-License-Identifier: GPL-2.0 use super::{NullBlkDevice, THIS_MODULE}; -use core::fmt::{Display, Write}; use kernel::{ block::mq::gen_disk::{GenDisk, GenDiskBuilder}, c_str, configfs::{self, AttributeOperations}, - configfs_attrs, new_mutex, + configfs_attrs, + fmt::{self, Write as _}, + new_mutex, page::PAGE_SIZE, prelude::*, str::{kstrtobool_bytes, CString}, @@ -99,8 +100,8 @@ impl TryFrom for IRQMode { } } -impl Display for IRQMode { - fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { +impl fmt::Display for IRQMode { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self { Self::None => f.write_str("0")?, Self::Soft => f.write_str("1")?, From b0af4f9142d046e6564479a3afb1b35479d81039 Mon Sep 17 00:00:00 2001 From: Tamir Duberstein Date: Sat, 18 Oct 2025 15:16:27 -0400 Subject: [PATCH 06/53] rust: alloc: use `kernel::fmt` Reduce coupling to implementation details of the formatting machinery by avoiding direct use for `core`'s formatting traits and macros. This backslid in commit 9def0d0a2a1c ("rust: alloc: add Vec::push_within_capacity"). Reviewed-by: Alice Ryhl Acked-by: Danilo Krummrich Signed-off-by: Tamir Duberstein Link: https://patch.msgid.link/20251018-cstr-core-v18-6-9378a54385f8@gmail.com Signed-off-by: Miguel Ojeda --- rust/kernel/alloc/kvec/errors.rs | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/rust/kernel/alloc/kvec/errors.rs b/rust/kernel/alloc/kvec/errors.rs index 21a920a4b09b..e7de5049ee47 100644 --- a/rust/kernel/alloc/kvec/errors.rs +++ b/rust/kernel/alloc/kvec/errors.rs @@ -2,14 +2,14 @@ //! Errors for the [`Vec`] type. -use kernel::fmt::{self, Debug, Formatter}; +use kernel::fmt; use kernel::prelude::*; /// Error type for [`Vec::push_within_capacity`]. pub struct PushError(pub T); -impl Debug for PushError { - fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { +impl fmt::Debug for PushError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "Not enough capacity") } } @@ -25,8 +25,8 @@ impl From> for Error { /// Error type for [`Vec::remove`]. pub struct RemoveError; -impl Debug for RemoveError { - fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { +impl fmt::Debug for RemoveError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "Index out of bounds") } } @@ -45,8 +45,8 @@ pub enum InsertError { OutOfCapacity(T), } -impl Debug for InsertError { - fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { +impl fmt::Debug for InsertError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self { InsertError::IndexOutOfBounds(_) => write!(f, "Index out of bounds"), InsertError::OutOfCapacity(_) => write!(f, "Not enough capacity"), From 3f0dd5fad9ac275648bd7407f032a8b7adaf204f Mon Sep 17 00:00:00 2001 From: Tamir Duberstein Date: Sat, 18 Oct 2025 15:16:28 -0400 Subject: [PATCH 07/53] rust: debugfs: use `kernel::fmt` Reduce coupling to implementation details of the formatting machinery by avoiding direct use for `core`'s formatting traits and macros. This backslid in commit 40ecc49466c8 ("rust: debugfs: Add support for callback-based files") and commit 5e40b591cb46 ("rust: debugfs: Add support for read-only files"). Acked-by: Danilo Krummrich Reviewed-by: Matthew Maurer Signed-off-by: Tamir Duberstein Link: https://patch.msgid.link/20251018-cstr-core-v18-7-9378a54385f8@gmail.com Signed-off-by: Miguel Ojeda --- rust/kernel/debugfs.rs | 2 +- rust/kernel/debugfs/callback_adapters.rs | 7 +++---- rust/kernel/debugfs/file_ops.rs | 6 +++--- rust/kernel/debugfs/traits.rs | 10 +++++----- 4 files changed, 12 insertions(+), 13 deletions(-) diff --git a/rust/kernel/debugfs.rs b/rust/kernel/debugfs.rs index 381c23b3dd83..8c35d032acfe 100644 --- a/rust/kernel/debugfs.rs +++ b/rust/kernel/debugfs.rs @@ -8,12 +8,12 @@ // When DebugFS is disabled, many parameters are dead. Linting for this isn't helpful. #![cfg_attr(not(CONFIG_DEBUG_FS), allow(unused_variables))] +use crate::fmt; use crate::prelude::*; use crate::str::CStr; #[cfg(CONFIG_DEBUG_FS)] use crate::sync::Arc; use crate::uaccess::UserSliceReader; -use core::fmt; use core::marker::PhantomData; use core::marker::PhantomPinned; #[cfg(CONFIG_DEBUG_FS)] diff --git a/rust/kernel/debugfs/callback_adapters.rs b/rust/kernel/debugfs/callback_adapters.rs index 6c024230f676..a260d8dee051 100644 --- a/rust/kernel/debugfs/callback_adapters.rs +++ b/rust/kernel/debugfs/callback_adapters.rs @@ -5,10 +5,9 @@ //! than a trait implementation. If provided, it will override the trait implementation. use super::{Reader, Writer}; +use crate::fmt; use crate::prelude::*; use crate::uaccess::UserSliceReader; -use core::fmt; -use core::fmt::Formatter; use core::marker::PhantomData; use core::ops::Deref; @@ -76,9 +75,9 @@ impl Deref for FormatAdapter { impl Writer for FormatAdapter where - F: Fn(&D, &mut Formatter<'_>) -> fmt::Result + 'static, + F: Fn(&D, &mut fmt::Formatter<'_>) -> fmt::Result + 'static, { - fn write(&self, fmt: &mut Formatter<'_>) -> fmt::Result { + fn write(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { // SAFETY: FormatAdapter<_, F> can only be constructed if F is inhabited let f: &F = unsafe { materialize_zst() }; f(&self.inner, fmt) diff --git a/rust/kernel/debugfs/file_ops.rs b/rust/kernel/debugfs/file_ops.rs index 50fead17b6f3..9ad5e3fa6f69 100644 --- a/rust/kernel/debugfs/file_ops.rs +++ b/rust/kernel/debugfs/file_ops.rs @@ -3,11 +3,11 @@ use super::{Reader, Writer}; use crate::debugfs::callback_adapters::Adapter; +use crate::fmt; use crate::prelude::*; use crate::seq_file::SeqFile; use crate::seq_print; use crate::uaccess::UserSlice; -use core::fmt::{Display, Formatter, Result}; use core::marker::PhantomData; #[cfg(CONFIG_DEBUG_FS)] @@ -65,8 +65,8 @@ impl Deref for FileOps { struct WriterAdapter(T); -impl<'a, T: Writer> Display for WriterAdapter<&'a T> { - fn fmt(&self, f: &mut Formatter<'_>) -> Result { +impl<'a, T: Writer> fmt::Display for WriterAdapter<&'a T> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { self.0.write(f) } } diff --git a/rust/kernel/debugfs/traits.rs b/rust/kernel/debugfs/traits.rs index ab009eb254b3..ad33bfbc7669 100644 --- a/rust/kernel/debugfs/traits.rs +++ b/rust/kernel/debugfs/traits.rs @@ -3,10 +3,10 @@ //! Traits for rendering or updating values exported to DebugFS. +use crate::fmt; use crate::prelude::*; use crate::sync::Mutex; use crate::uaccess::UserSliceReader; -use core::fmt::{self, Debug, Formatter}; use core::str::FromStr; use core::sync::atomic::{ AtomicI16, AtomicI32, AtomicI64, AtomicI8, AtomicIsize, AtomicU16, AtomicU32, AtomicU64, @@ -24,17 +24,17 @@ use core::sync::atomic::{ /// explicitly instead. pub trait Writer { /// Formats the value using the given formatter. - fn write(&self, f: &mut Formatter<'_>) -> fmt::Result; + fn write(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result; } impl Writer for Mutex { - fn write(&self, f: &mut Formatter<'_>) -> fmt::Result { + fn write(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { self.lock().write(f) } } -impl Writer for T { - fn write(&self, f: &mut Formatter<'_>) -> fmt::Result { +impl Writer for T { + fn write(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { writeln!(f, "{self:?}") } } From 7b0c32cbed761335b0fa7f3db232ad69bc23ce69 Mon Sep 17 00:00:00 2001 From: Miguel Ojeda Date: Sun, 19 Oct 2025 22:13:49 +0200 Subject: [PATCH 08/53] samples: rust: debugfs: use `core::ffi::CStr` method names Prepare for `core::ffi::CStr` taking the place of `kernel::str::CStr` by avoiding methods that only exist on the latter. This backslid in commit d4a5d397c7fb ("samples: rust: Add scoped debugfs sample driver"). Link: https://patch.msgid.link/20251019213049.2060970-1-ojeda@kernel.org Signed-off-by: Miguel Ojeda --- samples/rust/rust_debugfs_scoped.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/rust/rust_debugfs_scoped.rs b/samples/rust/rust_debugfs_scoped.rs index b0c4e76b123e..eb870e9986b8 100644 --- a/samples/rust/rust_debugfs_scoped.rs +++ b/samples/rust/rust_debugfs_scoped.rs @@ -38,7 +38,7 @@ fn remove_file_write( mod_data .devices .lock() - .retain(|device| device.name.as_bytes() != to_remove.as_bytes()); + .retain(|device| device.name.to_bytes() != to_remove.to_bytes()); Ok(()) } From 0e947bc22b0bdac7f708a0c6856f5999baff7b5d Mon Sep 17 00:00:00 2001 From: Tamir Duberstein Date: Sat, 18 Oct 2025 15:16:29 -0400 Subject: [PATCH 09/53] rust: pci: use `kernel::fmt` Reduce coupling to implementation details of the formatting machinery by avoiding direct use for `core`'s formatting traits and macros. This backslid in commit ed78a01887e2 ("rust: pci: provide access to PCI Class and Class-related items"). Acked-by: Danilo Krummrich Signed-off-by: Tamir Duberstein Link: https://patch.msgid.link/20251018-cstr-core-v18-8-9378a54385f8@gmail.com Signed-off-by: Miguel Ojeda --- rust/kernel/pci/id.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/rust/kernel/pci/id.rs b/rust/kernel/pci/id.rs index 7f2a7f57507f..5f5d59ff49fc 100644 --- a/rust/kernel/pci/id.rs +++ b/rust/kernel/pci/id.rs @@ -4,8 +4,7 @@ //! //! This module contains PCI class codes, Vendor IDs, and supporting types. -use crate::{bindings, error::code::EINVAL, error::Error, prelude::*}; -use core::fmt; +use crate::{bindings, error::code::EINVAL, error::Error, fmt, prelude::*}; /// PCI device class codes. /// From 5b60cde74bb77aaec72d47fc208e4b057f618336 Mon Sep 17 00:00:00 2001 From: Tamir Duberstein Date: Sat, 18 Oct 2025 15:16:30 -0400 Subject: [PATCH 10/53] rust: remove spurious `use core::fmt::Debug` We want folks to use `kernel::fmt` but this is only used for `derive` so can be removed entirely. This backslid in commit ea60cea07d8c ("rust: add `Alignment` type"). Reviewed-by: Alice Ryhl Signed-off-by: Tamir Duberstein Link: https://patch.msgid.link/20251018-cstr-core-v18-9-9378a54385f8@gmail.com Signed-off-by: Miguel Ojeda --- rust/kernel/ptr.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/rust/kernel/ptr.rs b/rust/kernel/ptr.rs index 2e5e2a090480..e3893ed04049 100644 --- a/rust/kernel/ptr.rs +++ b/rust/kernel/ptr.rs @@ -2,7 +2,6 @@ //! Types and functions to work with pointers and addresses. -use core::fmt::Debug; use core::mem::align_of; use core::num::NonZero; From 965a39a9627bfaee5a4f7471097439da19008338 Mon Sep 17 00:00:00 2001 From: Tamir Duberstein Date: Sat, 18 Oct 2025 15:16:31 -0400 Subject: [PATCH 11/53] rust: opp: use `CStr::as_char_ptr` Replace the use of `as_ptr` which works through `::deref()` in preparation for replacing `kernel::str::CStr` with `core::ffi::CStr` as the latter does not implement `Deref`. Signed-off-by: Tamir Duberstein Acked-by: Viresh Kumar Link: https://patch.msgid.link/20251018-cstr-core-v18-10-9378a54385f8@gmail.com Signed-off-by: Miguel Ojeda --- rust/kernel/opp.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/rust/kernel/opp.rs b/rust/kernel/opp.rs index 2c763fa9276d..9d6c58178a6f 100644 --- a/rust/kernel/opp.rs +++ b/rust/kernel/opp.rs @@ -13,7 +13,7 @@ use crate::{ cpumask::{Cpumask, CpumaskVar}, device::Device, error::{code::*, from_err_ptr, from_result, to_result, Result, VTABLE_DEFAULT_ERROR}, - ffi::c_ulong, + ffi::{c_char, c_ulong}, prelude::*, str::CString, sync::aref::{ARef, AlwaysRefCounted}, @@ -88,12 +88,12 @@ use core::{marker::PhantomData, ptr}; use macros::vtable; /// Creates a null-terminated slice of pointers to [`Cstring`]s. -fn to_c_str_array(names: &[CString]) -> Result> { +fn to_c_str_array(names: &[CString]) -> Result> { // Allocated a null-terminated vector of pointers. let mut list = KVec::with_capacity(names.len() + 1, GFP_KERNEL)?; for name in names.iter() { - list.push(name.as_ptr().cast(), GFP_KERNEL)?; + list.push(name.as_char_ptr(), GFP_KERNEL)?; } list.push(ptr::null(), GFP_KERNEL)?; From 3b46f65355c9627efd2dd8180c96b006a44d30f8 Mon Sep 17 00:00:00 2001 From: Tamir Duberstein Date: Sat, 18 Oct 2025 15:16:33 -0400 Subject: [PATCH 12/53] rust: configfs: use `CStr::as_char_ptr` Replace the use of `as_ptr` which works through `::deref()` in preparation for replacing `kernel::str::CStr` with `core::ffi::CStr` as the latter does not implement `Deref`. Signed-off-by: Tamir Duberstein Acked-by: Andreas Hindborg Link: https://patch.msgid.link/20251018-cstr-core-v18-12-9378a54385f8@gmail.com Signed-off-by: Miguel Ojeda --- rust/kernel/configfs.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rust/kernel/configfs.rs b/rust/kernel/configfs.rs index 10f1547ca9f1..466fb7f40762 100644 --- a/rust/kernel/configfs.rs +++ b/rust/kernel/configfs.rs @@ -157,7 +157,7 @@ impl Subsystem { unsafe { bindings::config_group_init_type_name( &mut (*place.get()).su_group, - name.as_ptr(), + name.as_char_ptr(), item_type.as_ptr(), ) }; From 9ce084e579bf550ee92b1ecbadf1c29a76c1062c Mon Sep 17 00:00:00 2001 From: Tamir Duberstein Date: Sat, 18 Oct 2025 15:16:34 -0400 Subject: [PATCH 13/53] rust: regulator: use `CStr::as_char_ptr` Replace the use of `as_ptr` which works through `::deref()` in preparation for replacing `kernel::str::CStr` with `core::ffi::CStr` as the latter does not implement `Deref`. Signed-off-by: Tamir Duberstein Acked-by: Mark Brown Reviewed-by: Daniel Almeida Link: https://patch.msgid.link/20251018-cstr-core-v18-13-9378a54385f8@gmail.com [ Move safety comment below to support older Clippy. - Miguel ] Signed-off-by: Miguel Ojeda --- rust/kernel/regulator.rs | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/rust/kernel/regulator.rs b/rust/kernel/regulator.rs index b55a201e5029..2c44827ad0b7 100644 --- a/rust/kernel/regulator.rs +++ b/rust/kernel/regulator.rs @@ -84,7 +84,7 @@ pub struct Error { pub fn devm_enable(dev: &Device, name: &CStr) -> Result { // SAFETY: `dev` is a valid and bound device, while `name` is a valid C // string. - to_result(unsafe { bindings::devm_regulator_get_enable(dev.as_raw(), name.as_ptr()) }) + to_result(unsafe { bindings::devm_regulator_get_enable(dev.as_raw(), name.as_char_ptr()) }) } /// Same as [`devm_enable`], but calls `devm_regulator_get_enable_optional` @@ -102,7 +102,9 @@ pub fn devm_enable(dev: &Device, name: &CStr) -> Result { pub fn devm_enable_optional(dev: &Device, name: &CStr) -> Result { // SAFETY: `dev` is a valid and bound device, while `name` is a valid C // string. - to_result(unsafe { bindings::devm_regulator_get_enable_optional(dev.as_raw(), name.as_ptr()) }) + to_result(unsafe { + bindings::devm_regulator_get_enable_optional(dev.as_raw(), name.as_char_ptr()) + }) } /// A `struct regulator` abstraction. @@ -266,9 +268,10 @@ impl Regulator { } fn get_internal(dev: &Device, name: &CStr) -> Result> { - // SAFETY: It is safe to call `regulator_get()`, on a device pointer - // received from the C code. - let inner = from_err_ptr(unsafe { bindings::regulator_get(dev.as_raw(), name.as_ptr()) })?; + let inner = + // SAFETY: It is safe to call `regulator_get()`, on a device pointer + // received from the C code. + from_err_ptr(unsafe { bindings::regulator_get(dev.as_raw(), name.as_char_ptr()) })?; // SAFETY: We can safely trust `inner` to be a pointer to a valid // regulator if `ERR_PTR` was not returned. From 1dcd763ba1f62c1305a03bc7d5bd1d44c20a4f5e Mon Sep 17 00:00:00 2001 From: Tamir Duberstein Date: Sat, 18 Oct 2025 15:16:35 -0400 Subject: [PATCH 14/53] rust: clk: use `CStr::as_char_ptr` Replace the use of `as_ptr` which works through `::deref()` in preparation for replacing `kernel::str::CStr` with `core::ffi::CStr` as the latter does not implement `Deref`. Signed-off-by: Tamir Duberstein Acked-by: Viresh Kumar Link: https://patch.msgid.link/20251018-cstr-core-v18-14-9378a54385f8@gmail.com Signed-off-by: Miguel Ojeda --- rust/kernel/clk.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rust/kernel/clk.rs b/rust/kernel/clk.rs index 1e6c8c42fb3a..c1cfaeaa36a2 100644 --- a/rust/kernel/clk.rs +++ b/rust/kernel/clk.rs @@ -136,7 +136,7 @@ mod common_clk { /// /// [`clk_get`]: https://docs.kernel.org/core-api/kernel-api.html#c.clk_get pub fn get(dev: &Device, name: Option<&CStr>) -> Result { - let con_id = name.map_or(ptr::null(), |n| n.as_ptr()); + let con_id = name.map_or(ptr::null(), |n| n.as_char_ptr()); // SAFETY: It is safe to call [`clk_get`] for a valid device pointer. // @@ -304,7 +304,7 @@ mod common_clk { /// [`clk_get_optional`]: /// https://docs.kernel.org/core-api/kernel-api.html#c.clk_get_optional pub fn get(dev: &Device, name: Option<&CStr>) -> Result { - let con_id = name.map_or(ptr::null(), |n| n.as_ptr()); + let con_id = name.map_or(ptr::null(), |n| n.as_char_ptr()); // SAFETY: It is safe to call [`clk_get_optional`] for a valid device pointer. // From c5cf01ba8dfe9c0f631ace6f64ec436303783000 Mon Sep 17 00:00:00 2001 From: Tamir Duberstein Date: Sat, 18 Oct 2025 15:16:36 -0400 Subject: [PATCH 15/53] rust: support formatting of foreign types Introduce a `fmt!` macro which wraps all arguments in `kernel::fmt::Adapter` and a `kernel::fmt::Display` trait. This enables formatting of foreign types (like `core::ffi::CStr`) that do not implement `core::fmt::Display` due to concerns around lossy conversions which do not apply in the kernel. Suggested-by: Alice Ryhl Link: https://rust-for-linux.zulipchat.com/#narrow/channel/288089-General/topic/Custom.20formatting/with/516476467 Acked-by: Greg Kroah-Hartman Reviewed-by: Alice Ryhl Acked-by: Danilo Krummrich Reviewed-by: Benno Lossin Signed-off-by: Tamir Duberstein Link: https://patch.msgid.link/20251018-cstr-core-v18-15-9378a54385f8@gmail.com Signed-off-by: Miguel Ojeda --- rust/kernel/fmt.rs | 87 +++++++++++++++++++++++++++++++++++++- rust/kernel/prelude.rs | 3 +- rust/macros/fmt.rs | 94 ++++++++++++++++++++++++++++++++++++++++++ rust/macros/lib.rs | 19 +++++++++ rust/macros/quote.rs | 7 ++++ 5 files changed, 207 insertions(+), 3 deletions(-) create mode 100644 rust/macros/fmt.rs diff --git a/rust/kernel/fmt.rs b/rust/kernel/fmt.rs index 0306e8388968..84d634201d90 100644 --- a/rust/kernel/fmt.rs +++ b/rust/kernel/fmt.rs @@ -4,4 +4,89 @@ //! //! This module is intended to be used in place of `core::fmt` in kernel code. -pub use core::fmt::{Arguments, Debug, Display, Error, Formatter, Result, Write}; +pub use core::fmt::{Arguments, Debug, Error, Formatter, Result, Write}; + +/// Internal adapter used to route allow implementations of formatting traits for foreign types. +/// +/// It is inserted automatically by the [`fmt!`] macro and is not meant to be used directly. +/// +/// [`fmt!`]: crate::prelude::fmt! +#[doc(hidden)] +pub struct Adapter(pub T); + +macro_rules! impl_fmt_adapter_forward { + ($($trait:ident),* $(,)?) => { + $( + impl $trait for Adapter { + fn fmt(&self, f: &mut Formatter<'_>) -> Result { + let Self(t) = self; + $trait::fmt(t, f) + } + } + )* + }; +} + +use core::fmt::{Binary, LowerExp, LowerHex, Octal, Pointer, UpperExp, UpperHex}; +impl_fmt_adapter_forward!(Debug, LowerHex, UpperHex, Octal, Binary, Pointer, LowerExp, UpperExp); + +/// A copy of [`core::fmt::Display`] that allows us to implement it for foreign types. +/// +/// Types should implement this trait rather than [`core::fmt::Display`]. Together with the +/// [`Adapter`] type and [`fmt!`] macro, it allows for formatting foreign types (e.g. types from +/// core) which do not implement [`core::fmt::Display`] directly. +/// +/// [`fmt!`]: crate::prelude::fmt! +pub trait Display { + /// Same as [`core::fmt::Display::fmt`]. + fn fmt(&self, f: &mut Formatter<'_>) -> Result; +} + +impl Display for &T { + fn fmt(&self, f: &mut Formatter<'_>) -> Result { + Display::fmt(*self, f) + } +} + +impl core::fmt::Display for Adapter<&T> { + fn fmt(&self, f: &mut Formatter<'_>) -> Result { + let Self(t) = self; + Display::fmt(t, f) + } +} + +macro_rules! impl_display_forward { + ($( + $( { $($generics:tt)* } )? $ty:ty $( { where $($where:tt)* } )? + ),* $(,)?) => { + $( + impl$($($generics)*)? Display for $ty $(where $($where)*)? { + fn fmt(&self, f: &mut Formatter<'_>) -> Result { + core::fmt::Display::fmt(self, f) + } + } + )* + }; +} + +impl_display_forward!( + bool, + char, + core::panic::PanicInfo<'_>, + Arguments<'_>, + i128, + i16, + i32, + i64, + i8, + isize, + str, + u128, + u16, + u32, + u64, + u8, + usize, + {} crate::sync::Arc {where crate::sync::Arc: core::fmt::Display}, + {} crate::sync::UniqueArc {where crate::sync::UniqueArc: core::fmt::Display}, +); diff --git a/rust/kernel/prelude.rs b/rust/kernel/prelude.rs index 198d09a31449..26424ad7e989 100644 --- a/rust/kernel/prelude.rs +++ b/rust/kernel/prelude.rs @@ -25,7 +25,7 @@ pub use ::ffi::{ pub use crate::alloc::{flags::*, Box, KBox, KVBox, KVVec, KVec, VBox, VVec, Vec}; #[doc(no_inline)] -pub use macros::{export, kunit_tests, module, vtable}; +pub use macros::{export, fmt, kunit_tests, module, vtable}; pub use pin_init::{init, pin_data, pin_init, pinned_drop, InPlaceWrite, Init, PinInit, Zeroable}; @@ -36,7 +36,6 @@ pub use super::{build_assert, build_error}; pub use super::dbg; pub use super::{dev_alert, dev_crit, dev_dbg, dev_emerg, dev_err, dev_info, dev_notice, dev_warn}; pub use super::{pr_alert, pr_crit, pr_debug, pr_emerg, pr_err, pr_info, pr_notice, pr_warn}; -pub use core::format_args as fmt; pub use super::{try_init, try_pin_init}; diff --git a/rust/macros/fmt.rs b/rust/macros/fmt.rs new file mode 100644 index 000000000000..2f4b9f6e2211 --- /dev/null +++ b/rust/macros/fmt.rs @@ -0,0 +1,94 @@ +// SPDX-License-Identifier: GPL-2.0 + +use proc_macro::{Ident, TokenStream, TokenTree}; +use std::collections::BTreeSet; + +/// Please see [`crate::fmt`] for documentation. +pub(crate) fn fmt(input: TokenStream) -> TokenStream { + let mut input = input.into_iter(); + + let first_opt = input.next(); + let first_owned_str; + let mut names = BTreeSet::new(); + let first_span = { + let Some((mut first_str, first_span)) = (match first_opt.as_ref() { + Some(TokenTree::Literal(first_lit)) => { + first_owned_str = first_lit.to_string(); + Some(first_owned_str.as_str()).and_then(|first| { + let first = first.strip_prefix('"')?; + let first = first.strip_suffix('"')?; + Some((first, first_lit.span())) + }) + } + _ => None, + }) else { + return first_opt.into_iter().chain(input).collect(); + }; + + // Parse `identifier`s from the format string. + // + // See https://doc.rust-lang.org/std/fmt/index.html#syntax. + while let Some((_, rest)) = first_str.split_once('{') { + first_str = rest; + if let Some(rest) = first_str.strip_prefix('{') { + first_str = rest; + continue; + } + if let Some((name, rest)) = first_str.split_once('}') { + first_str = rest; + let name = name.split_once(':').map_or(name, |(name, _)| name); + if !name.is_empty() && !name.chars().all(|c| c.is_ascii_digit()) { + names.insert(name); + } + } + } + first_span + }; + + let adapter = quote_spanned!(first_span => ::kernel::fmt::Adapter); + + let mut args = TokenStream::from_iter(first_opt); + { + let mut flush = |args: &mut TokenStream, current: &mut TokenStream| { + let current = std::mem::take(current); + if !current.is_empty() { + let (lhs, rhs) = (|| { + let mut current = current.into_iter(); + let mut acc = TokenStream::new(); + while let Some(tt) = current.next() { + // Split on `=` only once to handle cases like `a = b = c`. + if matches!(&tt, TokenTree::Punct(p) if p.as_char() == '=') { + names.remove(acc.to_string().as_str()); + // Include the `=` itself to keep the handling below uniform. + acc.extend([tt]); + return (Some(acc), current.collect::()); + } + acc.extend([tt]); + } + (None, acc) + })(); + args.extend(quote_spanned!(first_span => #lhs #adapter(&#rhs))); + } + }; + + let mut current = TokenStream::new(); + for tt in input { + match &tt { + TokenTree::Punct(p) if p.as_char() == ',' => { + flush(&mut args, &mut current); + &mut args + } + _ => &mut current, + } + .extend([tt]); + } + flush(&mut args, &mut current); + } + + for name in names { + let name = Ident::new(name, first_span); + args.extend(quote_spanned!(first_span => , #name = #adapter(&#name))); + } + + quote_spanned!(first_span => ::core::format_args!(#args)) +} diff --git a/rust/macros/lib.rs b/rust/macros/lib.rs index fa847cf3a9b5..793f712dbf7c 100644 --- a/rust/macros/lib.rs +++ b/rust/macros/lib.rs @@ -15,6 +15,7 @@ mod quote; mod concat_idents; mod export; +mod fmt; mod helpers; mod kunit; mod module; @@ -201,6 +202,24 @@ pub fn export(attr: TokenStream, ts: TokenStream) -> TokenStream { export::export(attr, ts) } +/// Like [`core::format_args!`], but automatically wraps arguments in [`kernel::fmt::Adapter`]. +/// +/// This macro allows generating `fmt::Arguments` while ensuring that each argument is wrapped with +/// `::kernel::fmt::Adapter`, which customizes formatting behavior for kernel logging. +/// +/// Named arguments used in the format string (e.g. `{foo}`) are detected and resolved from local +/// bindings. All positional and named arguments are automatically wrapped. +/// +/// This macro is an implementation detail of other kernel logging macros like [`pr_info!`] and +/// should not typically be used directly. +/// +/// [`kernel::fmt::Adapter`]: ../kernel/fmt/struct.Adapter.html +/// [`pr_info!`]: ../kernel/macro.pr_info.html +#[proc_macro] +pub fn fmt(input: TokenStream) -> TokenStream { + fmt::fmt(input) +} + /// Concatenate two identifiers. /// /// This is useful in macros that need to declare or reference items with names diff --git a/rust/macros/quote.rs b/rust/macros/quote.rs index acc140c18653..ddfc21577539 100644 --- a/rust/macros/quote.rs +++ b/rust/macros/quote.rs @@ -48,6 +48,7 @@ macro_rules! quote_spanned { ($span:expr => $($tt:tt)*) => {{ let mut tokens = ::proc_macro::TokenStream::new(); { + #[allow(unused_variables)] let span = $span; quote_spanned!(@proc tokens span $($tt)*); } @@ -146,6 +147,12 @@ macro_rules! quote_spanned { )]); quote_spanned!(@proc $v $span $($tt)*); }; + (@proc $v:ident $span:ident & $($tt:tt)*) => { + $v.extend([::proc_macro::TokenTree::Punct( + ::proc_macro::Punct::new('&', ::proc_macro::Spacing::Alone), + )]); + quote_spanned!(@proc $v $span $($tt)*); + }; (@proc $v:ident $span:ident _ $($tt:tt)*) => { $v.extend([::proc_macro::TokenTree::Ident( ::proc_macro::Ident::new("_", $span), From 3b83f5d5e78ac5cddd811a5e431af73959864390 Mon Sep 17 00:00:00 2001 From: Tamir Duberstein Date: Sat, 18 Oct 2025 15:16:37 -0400 Subject: [PATCH 16/53] rust: replace `CStr` with `core::ffi::CStr` `kernel::ffi::CStr` was introduced in commit d126d2380131 ("rust: str: add `CStr` type") in November 2022 as an upstreaming of earlier work that was done in May 2021[0]. That earlier work, having predated the inclusion of `CStr` in `core`, largely duplicated the implementation of `std::ffi::CStr`. `std::ffi::CStr` was moved to `core::ffi::CStr` in Rust 1.64 in September 2022. Hence replace `kernel::str::CStr` with `core::ffi::CStr` to reduce our custom code footprint, and retain needed custom functionality through an extension trait. Add `CStr` to `ffi` and the kernel prelude. Link: https://github.com/Rust-for-Linux/linux/commit/faa3cbcca03d0dec8f8e43f1d8d5c0860d98a23f [0] Acked-by: Greg Kroah-Hartman Reviewed-by: Alice Ryhl Acked-by: Danilo Krummrich Reviewed-by: Benno Lossin Signed-off-by: Tamir Duberstein Link: https://patch.msgid.link/20251018-cstr-core-v18-16-9378a54385f8@gmail.com [ Removed assert that would now depend on the Rust version. - Miguel ] Signed-off-by: Miguel Ojeda --- drivers/android/binder/stats.rs | 2 +- rust/ffi.rs | 2 + rust/kernel/debugfs/entry.rs | 2 +- rust/kernel/device.rs | 1 + rust/kernel/drm/ioctl.rs | 4 +- rust/kernel/error.rs | 2 + rust/kernel/firmware.rs | 9 +- rust/kernel/prelude.rs | 4 +- rust/kernel/seq_file.rs | 2 +- rust/kernel/str.rs | 423 ++++++++------------------------ rust/kernel/sync/condvar.rs | 2 +- rust/kernel/sync/lock.rs | 2 +- rust/kernel/sync/lock/global.rs | 2 +- 13 files changed, 127 insertions(+), 330 deletions(-) diff --git a/drivers/android/binder/stats.rs b/drivers/android/binder/stats.rs index 10c43679d5c3..037002651941 100644 --- a/drivers/android/binder/stats.rs +++ b/drivers/android/binder/stats.rs @@ -61,7 +61,7 @@ impl BinderStats { mod strings { use core::str::from_utf8_unchecked; - use kernel::str::CStr; + use kernel::str::{CStr, CStrExt as _}; extern "C" { static binder_command_strings: [*const u8; super::BC_COUNT]; diff --git a/rust/ffi.rs b/rust/ffi.rs index d60aad792af4..f961e9728f59 100644 --- a/rust/ffi.rs +++ b/rust/ffi.rs @@ -46,3 +46,5 @@ alias! { } pub use core::ffi::c_void; + +pub use core::ffi::CStr; diff --git a/rust/kernel/debugfs/entry.rs b/rust/kernel/debugfs/entry.rs index f99402cd3ba0..706cb7f73d6c 100644 --- a/rust/kernel/debugfs/entry.rs +++ b/rust/kernel/debugfs/entry.rs @@ -3,7 +3,7 @@ use crate::debugfs::file_ops::FileOps; use crate::ffi::c_void; -use crate::str::CStr; +use crate::str::{CStr, CStrExt as _}; use crate::sync::Arc; use core::marker::PhantomData; diff --git a/rust/kernel/device.rs b/rust/kernel/device.rs index 1321e6f0b53c..0b13aa60b685 100644 --- a/rust/kernel/device.rs +++ b/rust/kernel/device.rs @@ -13,6 +13,7 @@ use core::{marker::PhantomData, ptr}; #[cfg(CONFIG_PRINTK)] use crate::c_str; +use crate::str::CStrExt as _; pub mod property; diff --git a/rust/kernel/drm/ioctl.rs b/rust/kernel/drm/ioctl.rs index 69efbdb4c85a..cf328101dde4 100644 --- a/rust/kernel/drm/ioctl.rs +++ b/rust/kernel/drm/ioctl.rs @@ -156,7 +156,9 @@ macro_rules! declare_drm_ioctls { Some($cmd) }, flags: $flags, - name: $crate::c_str!(::core::stringify!($cmd)).as_char_ptr(), + name: $crate::str::as_char_ptr_in_const_context( + $crate::c_str!(::core::stringify!($cmd)), + ), } ),*]; ioctls diff --git a/rust/kernel/error.rs b/rust/kernel/error.rs index 1c0e0e241daa..258b12afdcba 100644 --- a/rust/kernel/error.rs +++ b/rust/kernel/error.rs @@ -182,6 +182,8 @@ impl Error { if ptr.is_null() { None } else { + use crate::str::CStrExt as _; + // SAFETY: The string returned by `errname` is static and `NUL`-terminated. Some(unsafe { CStr::from_char_ptr(ptr) }) } diff --git a/rust/kernel/firmware.rs b/rust/kernel/firmware.rs index 94e6bb88b903..376e7e77453f 100644 --- a/rust/kernel/firmware.rs +++ b/rust/kernel/firmware.rs @@ -4,7 +4,14 @@ //! //! C header: [`include/linux/firmware.h`](srctree/include/linux/firmware.h) -use crate::{bindings, device::Device, error::Error, error::Result, ffi, str::CStr}; +use crate::{ + bindings, + device::Device, + error::Error, + error::Result, + ffi, + str::{CStr, CStrExt as _}, +}; use core::ptr::NonNull; /// # Invariants diff --git a/rust/kernel/prelude.rs b/rust/kernel/prelude.rs index 26424ad7e989..33fa8404c5c6 100644 --- a/rust/kernel/prelude.rs +++ b/rust/kernel/prelude.rs @@ -19,7 +19,7 @@ pub use core::{ pub use ::ffi::{ c_char, c_int, c_long, c_longlong, c_schar, c_short, c_uchar, c_uint, c_ulong, c_ulonglong, - c_ushort, c_void, + c_ushort, c_void, CStr, }; pub use crate::alloc::{flags::*, Box, KBox, KVBox, KVVec, KVec, VBox, VVec, Vec}; @@ -43,7 +43,7 @@ pub use super::static_assert; pub use super::error::{code::*, Error, Result}; -pub use super::{str::CStr, ThisModule}; +pub use super::{str::CStrExt as _, ThisModule}; pub use super::init::InPlaceInit; diff --git a/rust/kernel/seq_file.rs b/rust/kernel/seq_file.rs index 59fbfc2473f8..855e533813a6 100644 --- a/rust/kernel/seq_file.rs +++ b/rust/kernel/seq_file.rs @@ -4,7 +4,7 @@ //! //! C header: [`include/linux/seq_file.h`](srctree/include/linux/seq_file.h) -use crate::{bindings, c_str, fmt, types::NotThreadSafe, types::Opaque}; +use crate::{bindings, c_str, fmt, str::CStrExt as _, types::NotThreadSafe, types::Opaque}; /// A utility for generating the contents of a seq file. #[repr(transparent)] diff --git a/rust/kernel/str.rs b/rust/kernel/str.rs index 5c74e5f77601..7593d758fbb7 100644 --- a/rust/kernel/str.rs +++ b/rust/kernel/str.rs @@ -10,9 +10,11 @@ use crate::{ }; use core::{ marker::PhantomData, - ops::{self, Deref, DerefMut, Index}, + ops::{Deref, DerefMut, Index}, }; +pub use crate::prelude::CStr; + /// Byte string without UTF-8 validity guarantee. #[repr(transparent)] pub struct BStr([u8]); @@ -186,58 +188,17 @@ macro_rules! b_str { // - error[E0379]: functions in trait impls cannot be declared const #[inline] pub const fn as_char_ptr_in_const_context(c_str: &CStr) -> *const c_char { - c_str.0.as_ptr() + c_str.as_ptr().cast() } -/// Possible errors when using conversion functions in [`CStr`]. -#[derive(Debug, Clone, Copy)] -pub enum CStrConvertError { - /// Supplied bytes contain an interior `NUL`. - InteriorNul, +mod private { + pub trait Sealed {} - /// Supplied bytes are not terminated by `NUL`. - NotNulTerminated, + impl Sealed for super::CStr {} } -impl From for Error { - #[inline] - fn from(_: CStrConvertError) -> Error { - EINVAL - } -} - -/// A string that is guaranteed to have exactly one `NUL` byte, which is at the -/// end. -/// -/// Used for interoperability with kernel APIs that take C strings. -#[repr(transparent)] -pub struct CStr([u8]); - -impl CStr { - /// Returns the length of this string excluding `NUL`. - #[inline] - pub const fn len(&self) -> usize { - self.len_with_nul() - 1 - } - - /// Returns the length of this string with `NUL`. - #[inline] - pub const fn len_with_nul(&self) -> usize { - if self.0.is_empty() { - // SAFETY: This is one of the invariant of `CStr`. - // We add a `unreachable_unchecked` here to hint the optimizer that - // the value returned from this function is non-zero. - unsafe { core::hint::unreachable_unchecked() }; - } - self.0.len() - } - - /// Returns `true` if the string only includes `NUL`. - #[inline] - pub const fn is_empty(&self) -> bool { - self.len() == 0 - } - +/// Extensions to [`CStr`]. +pub trait CStrExt: private::Sealed { /// Wraps a raw C string pointer. /// /// # Safety @@ -245,54 +206,9 @@ impl CStr { /// `ptr` must be a valid pointer to a `NUL`-terminated C string, and it must /// last at least `'a`. When `CStr` is alive, the memory pointed by `ptr` /// must not be mutated. - #[inline] - pub unsafe fn from_char_ptr<'a>(ptr: *const c_char) -> &'a Self { - // SAFETY: The safety precondition guarantees `ptr` is a valid pointer - // to a `NUL`-terminated C string. - let len = unsafe { bindings::strlen(ptr) } + 1; - // SAFETY: Lifetime guaranteed by the safety precondition. - let bytes = unsafe { core::slice::from_raw_parts(ptr.cast(), len) }; - // SAFETY: As `len` is returned by `strlen`, `bytes` does not contain interior `NUL`. - // As we have added 1 to `len`, the last byte is known to be `NUL`. - unsafe { Self::from_bytes_with_nul_unchecked(bytes) } - } - - /// Creates a [`CStr`] from a `[u8]`. - /// - /// The provided slice must be `NUL`-terminated, does not contain any - /// interior `NUL` bytes. - pub const fn from_bytes_with_nul(bytes: &[u8]) -> Result<&Self, CStrConvertError> { - if bytes.is_empty() { - return Err(CStrConvertError::NotNulTerminated); - } - if bytes[bytes.len() - 1] != 0 { - return Err(CStrConvertError::NotNulTerminated); - } - let mut i = 0; - // `i + 1 < bytes.len()` allows LLVM to optimize away bounds checking, - // while it couldn't optimize away bounds checks for `i < bytes.len() - 1`. - while i + 1 < bytes.len() { - if bytes[i] == 0 { - return Err(CStrConvertError::InteriorNul); - } - i += 1; - } - // SAFETY: We just checked that all properties hold. - Ok(unsafe { Self::from_bytes_with_nul_unchecked(bytes) }) - } - - /// Creates a [`CStr`] from a `[u8]` without performing any additional - /// checks. - /// - /// # Safety - /// - /// `bytes` *must* end with a `NUL` byte, and should only have a single - /// `NUL` byte (or the string will be truncated). - #[inline] - pub const unsafe fn from_bytes_with_nul_unchecked(bytes: &[u8]) -> &CStr { - // SAFETY: Properties of `bytes` guaranteed by the safety precondition. - unsafe { core::mem::transmute(bytes) } - } + // This function exists to paper over the fact that `CStr::from_ptr` takes a `*const + // core::ffi::c_char` rather than a `*const crate::ffi::c_char`. + unsafe fn from_char_ptr<'a>(ptr: *const c_char) -> &'a Self; /// Creates a mutable [`CStr`] from a `[u8]` without performing any /// additional checks. @@ -301,99 +217,16 @@ impl CStr { /// /// `bytes` *must* end with a `NUL` byte, and should only have a single /// `NUL` byte (or the string will be truncated). - #[inline] - pub unsafe fn from_bytes_with_nul_unchecked_mut(bytes: &mut [u8]) -> &mut CStr { - // SAFETY: Properties of `bytes` guaranteed by the safety precondition. - unsafe { &mut *(core::ptr::from_mut(bytes) as *mut CStr) } - } + unsafe fn from_bytes_with_nul_unchecked_mut(bytes: &mut [u8]) -> &mut Self; /// Returns a C pointer to the string. - /// - /// Using this function in a const context is deprecated in favor of - /// [`as_char_ptr_in_const_context`] in preparation for replacing `CStr` with `core::ffi::CStr` - /// which does not have this method. - #[inline] - pub const fn as_char_ptr(&self) -> *const c_char { - as_char_ptr_in_const_context(self) - } - - /// Convert the string to a byte slice without the trailing `NUL` byte. - #[inline] - pub fn to_bytes(&self) -> &[u8] { - &self.0[..self.len()] - } - - /// Convert the string to a byte slice without the trailing `NUL` byte. - /// - /// This function is deprecated in favor of [`Self::to_bytes`] in preparation for replacing - /// `CStr` with `core::ffi::CStr` which does not have this method. - #[inline] - pub fn as_bytes(&self) -> &[u8] { - self.to_bytes() - } - - /// Convert the string to a byte slice containing the trailing `NUL` byte. - #[inline] - pub const fn to_bytes_with_nul(&self) -> &[u8] { - &self.0 - } - - /// Convert the string to a byte slice containing the trailing `NUL` byte. - /// - /// This function is deprecated in favor of [`Self::to_bytes_with_nul`] in preparation for - /// replacing `CStr` with `core::ffi::CStr` which does not have this method. - #[inline] - pub const fn as_bytes_with_nul(&self) -> &[u8] { - self.to_bytes_with_nul() - } - - /// Yields a [`&str`] slice if the [`CStr`] contains valid UTF-8. - /// - /// If the contents of the [`CStr`] are valid UTF-8 data, this - /// function will return the corresponding [`&str`] slice. Otherwise, - /// it will return an error with details of where UTF-8 validation failed. - /// - /// # Examples - /// - /// ``` - /// # use kernel::str::CStr; - /// let cstr = CStr::from_bytes_with_nul(b"foo\0")?; - /// assert_eq!(cstr.to_str(), Ok("foo")); - /// # Ok::<(), kernel::error::Error>(()) - /// ``` - #[inline] - pub fn to_str(&self) -> Result<&str, core::str::Utf8Error> { - core::str::from_utf8(self.as_bytes()) - } - - /// Unsafely convert this [`CStr`] into a [`&str`], without checking for - /// valid UTF-8. - /// - /// # Safety - /// - /// The contents must be valid UTF-8. - /// - /// # Examples - /// - /// ``` - /// # use kernel::c_str; - /// # use kernel::str::CStr; - /// let bar = c_str!("ツ"); - /// // SAFETY: String literals are guaranteed to be valid UTF-8 - /// // by the Rust compiler. - /// assert_eq!(unsafe { bar.as_str_unchecked() }, "ツ"); - /// ``` - #[inline] - pub unsafe fn as_str_unchecked(&self) -> &str { - // SAFETY: TODO. - unsafe { core::str::from_utf8_unchecked(self.as_bytes()) } - } + // This function exists to paper over the fact that `CStr::as_ptr` returns a `*const + // core::ffi::c_char` rather than a `*const crate::ffi::c_char`. + fn as_char_ptr(&self) -> *const c_char; /// Convert this [`CStr`] into a [`CString`] by allocating memory and /// copying over the string data. - pub fn to_cstring(&self) -> Result { - CString::try_from(self) - } + fn to_cstring(&self) -> Result; /// Converts this [`CStr`] to its ASCII lower case equivalent in-place. /// @@ -404,11 +237,7 @@ impl CStr { /// [`to_ascii_lowercase()`]. /// /// [`to_ascii_lowercase()`]: #method.to_ascii_lowercase - pub fn make_ascii_lowercase(&mut self) { - // INVARIANT: This doesn't introduce or remove NUL bytes in the C - // string. - self.0.make_ascii_lowercase(); - } + fn make_ascii_lowercase(&mut self); /// Converts this [`CStr`] to its ASCII upper case equivalent in-place. /// @@ -419,11 +248,7 @@ impl CStr { /// [`to_ascii_uppercase()`]. /// /// [`to_ascii_uppercase()`]: #method.to_ascii_uppercase - pub fn make_ascii_uppercase(&mut self) { - // INVARIANT: This doesn't introduce or remove NUL bytes in the C - // string. - self.0.make_ascii_uppercase(); - } + fn make_ascii_uppercase(&mut self); /// Returns a copy of this [`CString`] where each character is mapped to its /// ASCII lower case equivalent. @@ -434,13 +259,7 @@ impl CStr { /// To lowercase the value in-place, use [`make_ascii_lowercase`]. /// /// [`make_ascii_lowercase`]: str::make_ascii_lowercase - pub fn to_ascii_lowercase(&self) -> Result { - let mut s = self.to_cstring()?; - - s.make_ascii_lowercase(); - - Ok(s) - } + fn to_ascii_lowercase(&self) -> Result; /// Returns a copy of this [`CString`] where each character is mapped to its /// ASCII upper case equivalent. @@ -451,13 +270,7 @@ impl CStr { /// To uppercase the value in-place, use [`make_ascii_uppercase`]. /// /// [`make_ascii_uppercase`]: str::make_ascii_uppercase - pub fn to_ascii_uppercase(&self) -> Result { - let mut s = self.to_cstring()?; - - s.make_ascii_uppercase(); - - Ok(s) - } + fn to_ascii_uppercase(&self) -> Result; } impl fmt::Display for CStr { @@ -490,98 +303,75 @@ impl fmt::Display for CStr { } } -impl fmt::Debug for CStr { - /// Formats printable ASCII characters with a double quote on either end, escaping the rest. - /// - /// ``` - /// # use kernel::c_str; - /// # use kernel::prelude::fmt; - /// # use kernel::str::CStr; - /// # use kernel::str::CString; - /// let penguin = c_str!("🐧"); - /// let s = CString::try_from_fmt(fmt!("{penguin:?}"))?; - /// assert_eq!(s.as_bytes_with_nul(), "\"\\xf0\\x9f\\x90\\xa7\"\0".as_bytes()); - /// - /// // Embedded double quotes are escaped. - /// let ascii = c_str!("so \"cool\""); - /// let s = CString::try_from_fmt(fmt!("{ascii:?}"))?; - /// assert_eq!(s.as_bytes_with_nul(), "\"so \\\"cool\\\"\"\0".as_bytes()); - /// # Ok::<(), kernel::error::Error>(()) - /// ``` - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.write_str("\"")?; - for &c in self.as_bytes() { - match c { - // Printable characters. - b'\"' => f.write_str("\\\"")?, - 0x20..=0x7e => f.write_char(c as char)?, - _ => write!(f, "\\x{c:02x}")?, - } - } - f.write_str("\"") +/// Converts a mutable C string to a mutable byte slice. +/// +/// # Safety +/// +/// The caller must ensure that the slice ends in a NUL byte and contains no other NUL bytes before +/// the borrow ends and the underlying [`CStr`] is used. +unsafe fn to_bytes_mut(s: &mut CStr) -> &mut [u8] { + // SAFETY: the cast from `&CStr` to `&[u8]` is safe since `CStr` has the same layout as `&[u8]` + // (this is technically not guaranteed, but we rely on it here). The pointer dereference is + // safe since it comes from a mutable reference which is guaranteed to be valid for writes. + unsafe { &mut *(core::ptr::from_mut(s) as *mut [u8]) } +} + +impl CStrExt for CStr { + #[inline] + unsafe fn from_char_ptr<'a>(ptr: *const c_char) -> &'a Self { + // SAFETY: The safety preconditions are the same as for `CStr::from_ptr`. + unsafe { CStr::from_ptr(ptr.cast()) } + } + + #[inline] + unsafe fn from_bytes_with_nul_unchecked_mut(bytes: &mut [u8]) -> &mut Self { + // SAFETY: the cast from `&[u8]` to `&CStr` is safe since the properties of `bytes` are + // guaranteed by the safety precondition and `CStr` has the same layout as `&[u8]` (this is + // technically not guaranteed, but we rely on it here). The pointer dereference is safe + // since it comes from a mutable reference which is guaranteed to be valid for writes. + unsafe { &mut *(core::ptr::from_mut(bytes) as *mut CStr) } + } + + #[inline] + fn as_char_ptr(&self) -> *const c_char { + self.as_ptr().cast() + } + + fn to_cstring(&self) -> Result { + CString::try_from(self) + } + + fn make_ascii_lowercase(&mut self) { + // SAFETY: This doesn't introduce or remove NUL bytes in the C string. + unsafe { to_bytes_mut(self) }.make_ascii_lowercase(); + } + + fn make_ascii_uppercase(&mut self) { + // SAFETY: This doesn't introduce or remove NUL bytes in the C string. + unsafe { to_bytes_mut(self) }.make_ascii_uppercase(); + } + + fn to_ascii_lowercase(&self) -> Result { + let mut s = self.to_cstring()?; + + s.make_ascii_lowercase(); + + Ok(s) + } + + fn to_ascii_uppercase(&self) -> Result { + let mut s = self.to_cstring()?; + + s.make_ascii_uppercase(); + + Ok(s) } } impl AsRef for CStr { #[inline] fn as_ref(&self) -> &BStr { - BStr::from_bytes(self.as_bytes()) - } -} - -impl Deref for CStr { - type Target = BStr; - - #[inline] - fn deref(&self) -> &Self::Target { - self.as_ref() - } -} - -impl Index> for CStr { - type Output = CStr; - - #[inline] - fn index(&self, index: ops::RangeFrom) -> &Self::Output { - // Delegate bounds checking to slice. - // Assign to _ to mute clippy's unnecessary operation warning. - let _ = &self.as_bytes()[index.start..]; - // SAFETY: We just checked the bounds. - unsafe { Self::from_bytes_with_nul_unchecked(&self.0[index.start..]) } - } -} - -impl Index for CStr { - type Output = CStr; - - #[inline] - fn index(&self, _index: ops::RangeFull) -> &Self::Output { - self - } -} - -mod private { - use core::ops; - - // Marker trait for index types that can be forward to `BStr`. - pub trait CStrIndex {} - - impl CStrIndex for usize {} - impl CStrIndex for ops::Range {} - impl CStrIndex for ops::RangeInclusive {} - impl CStrIndex for ops::RangeToInclusive {} -} - -impl Index for CStr -where - Idx: private::CStrIndex, - BStr: Index, -{ - type Output = >::Output; - - #[inline] - fn index(&self, index: Idx) -> &Self::Output { - &self.as_ref()[index] + BStr::from_bytes(self.to_bytes()) } } @@ -612,6 +402,13 @@ macro_rules! c_str { mod tests { use super::*; + impl From for Error { + #[inline] + fn from(_: core::ffi::FromBytesWithNulError) -> Error { + EINVAL + } + } + macro_rules! format { ($($f:tt)*) => ({ CString::try_from_fmt(fmt!($($f)*))?.to_str()? @@ -634,40 +431,28 @@ mod tests { #[test] fn test_cstr_to_str() -> Result { - let good_bytes = b"\xf0\x9f\xa6\x80\0"; - let checked_cstr = CStr::from_bytes_with_nul(good_bytes)?; - let checked_str = checked_cstr.to_str()?; + let cstr = c"\xf0\x9f\xa6\x80"; + let checked_str = cstr.to_str()?; assert_eq!(checked_str, "🦀"); Ok(()) } #[test] fn test_cstr_to_str_invalid_utf8() -> Result { - let bad_bytes = b"\xc3\x28\0"; - let checked_cstr = CStr::from_bytes_with_nul(bad_bytes)?; - assert!(checked_cstr.to_str().is_err()); - Ok(()) - } - - #[test] - fn test_cstr_as_str_unchecked() -> Result { - let good_bytes = b"\xf0\x9f\x90\xA7\0"; - let checked_cstr = CStr::from_bytes_with_nul(good_bytes)?; - // SAFETY: The contents come from a string literal which contains valid UTF-8. - let unchecked_str = unsafe { checked_cstr.as_str_unchecked() }; - assert_eq!(unchecked_str, "🐧"); + let cstr = c"\xc3\x28"; + assert!(cstr.to_str().is_err()); Ok(()) } #[test] fn test_cstr_display() -> Result { - let hello_world = CStr::from_bytes_with_nul(b"hello, world!\0")?; + let hello_world = c"hello, world!"; assert_eq!(format!("{hello_world}"), "hello, world!"); - let non_printables = CStr::from_bytes_with_nul(b"\x01\x09\x0a\0")?; + let non_printables = c"\x01\x09\x0a"; assert_eq!(format!("{non_printables}"), "\\x01\\x09\\x0a"); - let non_ascii = CStr::from_bytes_with_nul(b"d\xe9j\xe0 vu\0")?; + let non_ascii = c"d\xe9j\xe0 vu"; assert_eq!(format!("{non_ascii}"), "d\\xe9j\\xe0 vu"); - let good_bytes = CStr::from_bytes_with_nul(b"\xf0\x9f\xa6\x80\0")?; + let good_bytes = c"\xf0\x9f\xa6\x80"; assert_eq!(format!("{good_bytes}"), "\\xf0\\x9f\\xa6\\x80"); Ok(()) } @@ -686,14 +471,12 @@ mod tests { #[test] fn test_cstr_debug() -> Result { - let hello_world = CStr::from_bytes_with_nul(b"hello, world!\0")?; + let hello_world = c"hello, world!"; assert_eq!(format!("{hello_world:?}"), "\"hello, world!\""); - let non_printables = CStr::from_bytes_with_nul(b"\x01\x09\x0a\0")?; - assert_eq!(format!("{non_printables:?}"), "\"\\x01\\x09\\x0a\""); - let non_ascii = CStr::from_bytes_with_nul(b"d\xe9j\xe0 vu\0")?; + let non_printables = c"\x01\x09\x0a"; + assert_eq!(format!("{non_printables:?}"), "\"\\x01\\t\\n\""); + let non_ascii = c"d\xe9j\xe0 vu"; assert_eq!(format!("{non_ascii:?}"), "\"d\\xe9j\\xe0 vu\""); - let good_bytes = CStr::from_bytes_with_nul(b"\xf0\x9f\xa6\x80\0")?; - assert_eq!(format!("{good_bytes:?}"), "\"\\xf0\\x9f\\xa6\\x80\""); Ok(()) } diff --git a/rust/kernel/sync/condvar.rs b/rust/kernel/sync/condvar.rs index c6ec64295c9f..a24e25a690ee 100644 --- a/rust/kernel/sync/condvar.rs +++ b/rust/kernel/sync/condvar.rs @@ -8,7 +8,7 @@ use super::{lock::Backend, lock::Guard, LockClassKey}; use crate::{ ffi::{c_int, c_long}, - str::CStr, + str::{CStr, CStrExt as _}, task::{ MAX_SCHEDULE_TIMEOUT, TASK_FREEZABLE, TASK_INTERRUPTIBLE, TASK_NORMAL, TASK_UNINTERRUPTIBLE, }, diff --git a/rust/kernel/sync/lock.rs b/rust/kernel/sync/lock.rs index 27202beef90c..5d7991e6d373 100644 --- a/rust/kernel/sync/lock.rs +++ b/rust/kernel/sync/lock.rs @@ -7,7 +7,7 @@ use super::LockClassKey; use crate::{ - str::CStr, + str::{CStr, CStrExt as _}, types::{NotThreadSafe, Opaque, ScopeGuard}, }; use core::{cell::UnsafeCell, marker::PhantomPinned, pin::Pin}; diff --git a/rust/kernel/sync/lock/global.rs b/rust/kernel/sync/lock/global.rs index d65f94b5caf2..79d0ef7fda86 100644 --- a/rust/kernel/sync/lock/global.rs +++ b/rust/kernel/sync/lock/global.rs @@ -5,7 +5,7 @@ //! Support for defining statics containing locks. use crate::{ - str::CStr, + str::{CStr, CStrExt as _}, sync::lock::{Backend, Guard, Lock}, sync::{LockClassKey, LockedBy}, types::Opaque, From 5935461b458463ee51aac8d95c25d7a5e1de8c4d Mon Sep 17 00:00:00 2001 From: Miguel Ojeda Date: Mon, 13 Oct 2025 00:46:45 +0200 Subject: [PATCH 17/53] docs: rust: quick-start: add Debian 13 (Trixie) Debian 13 (released 2025-08-09) packages Rust 1.85.0 [1], which is recent enough to build Linux. Thus document it. In fact, we are planning to propose that the minimum supported Rust version in Linux follows Debian Stable releases, with Debian 13 being the first one we upgrade to, i.e. Rust 1.85. Link: https://www.debian.org/News/2025/20250809 [1] Link: https://patch.msgid.link/20251012224645.1148411-1-ojeda@kernel.org Signed-off-by: Miguel Ojeda --- Documentation/rust/quick-start.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Documentation/rust/quick-start.rst b/Documentation/rust/quick-start.rst index 155f7107329a..152289f0bed2 100644 --- a/Documentation/rust/quick-start.rst +++ b/Documentation/rust/quick-start.rst @@ -39,8 +39,8 @@ of the box, e.g.:: Debian ****** -Debian Testing and Debian Unstable (Sid), outside of the freeze period, provide -recent Rust releases and thus they should generally work out of the box, e.g.:: +Debian 13 (Trixie), as well as Testing and Debian Unstable (Sid) provide recent +Rust releases and thus they should generally work out of the box, e.g.:: apt install rustc rust-src bindgen rustfmt rust-clippy From f56b131723826dfdbf829d014591d831d649ae4a Mon Sep 17 00:00:00 2001 From: Vitaly Wool Date: Tue, 14 Oct 2025 14:33:39 +0200 Subject: [PATCH 18/53] rust: rbtree: add immutable cursor Sometimes we may need to iterate over, or find an element in a read only (or read mostly) red-black tree, and in that case we don't need a mutable reference to the tree, which we'll however have to take to be able to use the current (mutable) cursor implementation. This patch adds a simple immutable cursor implementation to RBTree, which enables us to use an immutable tree reference. The existing (fully featured) cursor implementation is renamed to CursorMut, while retaining its functionality. The only existing user of the [mutable] cursor for RBTrees (binder) is updated to match the changes. Signed-off-by: Vitaly Wool Reviewed-by: Alice Ryhl Link: https://patch.msgid.link/20251014123339.2492210-1-vitaly.wool@konsulko.se [ Applied `rustfmt`. Added intra-doc link. Fixed unclosed example. Fixed docs description. Fixed typo and other formatting nits. - Miguel ] Signed-off-by: Miguel Ojeda --- drivers/android/binder/freeze.rs | 4 +- drivers/android/binder/process.rs | 2 +- drivers/android/binder/range_alloc/tree.rs | 2 +- rust/kernel/rbtree.rs | 244 +++++++++++++++++---- 4 files changed, 201 insertions(+), 51 deletions(-) diff --git a/drivers/android/binder/freeze.rs b/drivers/android/binder/freeze.rs index e68c3c8bc55a..87bebef53d94 100644 --- a/drivers/android/binder/freeze.rs +++ b/drivers/android/binder/freeze.rs @@ -321,7 +321,7 @@ impl Process { KVVec::with_capacity(8, GFP_KERNEL).unwrap_or_else(|_err| KVVec::new()); let mut inner = self.lock_with_nodes(); - let mut curr = inner.nodes.cursor_front(); + let mut curr = inner.nodes.cursor_front_mut(); while let Some(cursor) = curr { let (key, node) = cursor.current(); let key = *key; @@ -335,7 +335,7 @@ impl Process { // Find the node we were looking at and try again. If the set of nodes was changed, // then just proceed to the next node. This is ok because we don't guarantee the // inclusion of nodes that are added or removed in parallel with this operation. - curr = inner.nodes.cursor_lower_bound(&key); + curr = inner.nodes.cursor_lower_bound_mut(&key); continue; } diff --git a/drivers/android/binder/process.rs b/drivers/android/binder/process.rs index d8111c990f21..c406f8b48d82 100644 --- a/drivers/android/binder/process.rs +++ b/drivers/android/binder/process.rs @@ -1293,7 +1293,7 @@ impl Process { { while let Some(node) = { let mut lock = self.inner.lock(); - lock.nodes.cursor_front().map(|c| c.remove_current().1) + lock.nodes.cursor_front_mut().map(|c| c.remove_current().1) } { node.to_key_value().1.release(); } diff --git a/drivers/android/binder/range_alloc/tree.rs b/drivers/android/binder/range_alloc/tree.rs index 7b1a248fcb02..838fdd2b47ea 100644 --- a/drivers/android/binder/range_alloc/tree.rs +++ b/drivers/android/binder/range_alloc/tree.rs @@ -207,7 +207,7 @@ impl TreeRangeAllocator { } pub(crate) fn reservation_abort(&mut self, offset: usize) -> Result { - let mut cursor = self.tree.cursor_lower_bound(&offset).ok_or_else(|| { + let mut cursor = self.tree.cursor_lower_bound_mut(&offset).ok_or_else(|| { pr_warn!( "EINVAL from range_alloc.reservation_abort - offset: {}", offset diff --git a/rust/kernel/rbtree.rs b/rust/kernel/rbtree.rs index b8fe6be6fcc4..4729eb56827a 100644 --- a/rust/kernel/rbtree.rs +++ b/rust/kernel/rbtree.rs @@ -243,14 +243,44 @@ impl RBTree { } /// Returns a cursor over the tree nodes, starting with the smallest key. - pub fn cursor_front(&mut self) -> Option> { + pub fn cursor_front_mut(&mut self) -> Option> { let root = addr_of_mut!(self.root); - // SAFETY: `self.root` is always a valid root node + // SAFETY: `self.root` is always a valid root node. + let current = unsafe { bindings::rb_first(root) }; + NonNull::new(current).map(|current| { + // INVARIANT: + // - `current` is a valid node in the [`RBTree`] pointed to by `self`. + CursorMut { + current, + tree: self, + } + }) + } + + /// Returns an immutable cursor over the tree nodes, starting with the smallest key. + pub fn cursor_front(&self) -> Option> { + let root = &raw const self.root; + // SAFETY: `self.root` is always a valid root node. let current = unsafe { bindings::rb_first(root) }; NonNull::new(current).map(|current| { // INVARIANT: // - `current` is a valid node in the [`RBTree`] pointed to by `self`. Cursor { + current, + _tree: PhantomData, + } + }) + } + + /// Returns a cursor over the tree nodes, starting with the largest key. + pub fn cursor_back_mut(&mut self) -> Option> { + let root = addr_of_mut!(self.root); + // SAFETY: `self.root` is always a valid root node. + let current = unsafe { bindings::rb_last(root) }; + NonNull::new(current).map(|current| { + // INVARIANT: + // - `current` is a valid node in the [`RBTree`] pointed to by `self`. + CursorMut { current, tree: self, } @@ -258,16 +288,16 @@ impl RBTree { } /// Returns a cursor over the tree nodes, starting with the largest key. - pub fn cursor_back(&mut self) -> Option> { - let root = addr_of_mut!(self.root); - // SAFETY: `self.root` is always a valid root node + pub fn cursor_back(&self) -> Option> { + let root = &raw const self.root; + // SAFETY: `self.root` is always a valid root node. let current = unsafe { bindings::rb_last(root) }; NonNull::new(current).map(|current| { // INVARIANT: // - `current` is a valid node in the [`RBTree`] pointed to by `self`. Cursor { current, - tree: self, + _tree: PhantomData, } }) } @@ -421,12 +451,47 @@ where /// If the given key exists, the cursor starts there. /// Otherwise it starts with the first larger key in sort order. /// If there is no larger key, it returns [`None`]. - pub fn cursor_lower_bound(&mut self, key: &K) -> Option> + pub fn cursor_lower_bound_mut(&mut self, key: &K) -> Option> where K: Ord, { + let best = self.find_best_match(key)?; + + NonNull::new(best.as_ptr()).map(|current| { + // INVARIANT: + // - `current` is a valid node in the [`RBTree`] pointed to by `self`. + CursorMut { + current, + tree: self, + } + }) + } + + /// Returns a cursor over the tree nodes based on the given key. + /// + /// If the given key exists, the cursor starts there. + /// Otherwise it starts with the first larger key in sort order. + /// If there is no larger key, it returns [`None`]. + pub fn cursor_lower_bound(&self, key: &K) -> Option> + where + K: Ord, + { + let best = self.find_best_match(key)?; + + NonNull::new(best.as_ptr()).map(|current| { + // INVARIANT: + // - `current` is a valid node in the [`RBTree`] pointed to by `self`. + Cursor { + current, + _tree: PhantomData, + } + }) + } + + fn find_best_match(&self, key: &K) -> Option> { let mut node = self.root.rb_node; - let mut best_match: Option>> = None; + let mut best_key: Option<&K> = None; + let mut best_links: Option> = None; while !node.is_null() { // SAFETY: By the type invariant of `Self`, all non-null `rb_node` pointers stored in `self` // point to the links field of `Node` objects. @@ -439,42 +504,28 @@ where let right_child = unsafe { (*node).rb_right }; match key.cmp(this_key) { Ordering::Equal => { - best_match = NonNull::new(this); + // SAFETY: `this` is a non-null node so it is valid by the type invariants. + best_links = Some(unsafe { NonNull::new_unchecked(&mut (*this).links) }); break; } Ordering::Greater => { node = right_child; } Ordering::Less => { - let is_better_match = match best_match { + let is_better_match = match best_key { None => true, - Some(best) => { - // SAFETY: `best` is a non-null node so it is valid by the type invariants. - let best_key = unsafe { &(*best.as_ptr()).key }; - best_key > this_key - } + Some(best) => best > this_key, }; if is_better_match { - best_match = NonNull::new(this); + best_key = Some(this_key); + // SAFETY: `this` is a non-null node so it is valid by the type invariants. + best_links = Some(unsafe { NonNull::new_unchecked(&mut (*this).links) }); } node = left_child; } }; } - - let best = best_match?; - - // SAFETY: `best` is a non-null node so it is valid by the type invariants. - let links = unsafe { addr_of_mut!((*best.as_ptr()).links) }; - - NonNull::new(links).map(|current| { - // INVARIANT: - // - `current` is a valid node in the [`RBTree`] pointed to by `self`. - Cursor { - current, - tree: self, - } - }) + best_links } } @@ -507,7 +558,7 @@ impl Drop for RBTree { } } -/// A bidirectional cursor over the tree nodes, sorted by key. +/// A bidirectional mutable cursor over the tree nodes, sorted by key. /// /// # Examples /// @@ -526,7 +577,7 @@ impl Drop for RBTree { /// tree.try_create_and_insert(30, 300, flags::GFP_KERNEL)?; /// /// // Get a cursor to the first element. -/// let mut cursor = tree.cursor_front().unwrap(); +/// let mut cursor = tree.cursor_front_mut().unwrap(); /// let mut current = cursor.current(); /// assert_eq!(current, (&10, &100)); /// @@ -564,7 +615,7 @@ impl Drop for RBTree { /// tree.try_create_and_insert(20, 200, flags::GFP_KERNEL)?; /// tree.try_create_and_insert(30, 300, flags::GFP_KERNEL)?; /// -/// let mut cursor = tree.cursor_back().unwrap(); +/// let mut cursor = tree.cursor_back_mut().unwrap(); /// let current = cursor.current(); /// assert_eq!(current, (&30, &300)); /// @@ -577,7 +628,7 @@ impl Drop for RBTree { /// use kernel::rbtree::RBTree; /// /// let mut tree: RBTree = RBTree::new(); -/// assert!(tree.cursor_front().is_none()); +/// assert!(tree.cursor_front_mut().is_none()); /// /// # Ok::<(), Error>(()) /// ``` @@ -628,7 +679,7 @@ impl Drop for RBTree { /// tree.try_create_and_insert(30, 300, flags::GFP_KERNEL)?; /// /// // Retrieve a cursor. -/// let mut cursor = tree.cursor_front().unwrap(); +/// let mut cursor = tree.cursor_front_mut().unwrap(); /// /// // Get a mutable reference to the current value. /// let (k, v) = cursor.current_mut(); @@ -655,7 +706,7 @@ impl Drop for RBTree { /// tree.try_create_and_insert(30, 300, flags::GFP_KERNEL)?; /// /// // Remove the first element. -/// let mut cursor = tree.cursor_front().unwrap(); +/// let mut cursor = tree.cursor_front_mut().unwrap(); /// let mut current = cursor.current(); /// assert_eq!(current, (&10, &100)); /// cursor = cursor.remove_current().0.unwrap(); @@ -665,7 +716,7 @@ impl Drop for RBTree { /// assert_eq!(current, (&20, &200)); /// /// // Get a cursor to the last element, and remove it. -/// cursor = tree.cursor_back().unwrap(); +/// cursor = tree.cursor_back_mut().unwrap(); /// current = cursor.current(); /// assert_eq!(current, (&30, &300)); /// @@ -694,7 +745,7 @@ impl Drop for RBTree { /// tree.try_create_and_insert(30, 300, flags::GFP_KERNEL)?; /// /// // Get a cursor to the first element. -/// let mut cursor = tree.cursor_front().unwrap(); +/// let mut cursor = tree.cursor_front_mut().unwrap(); /// let mut current = cursor.current(); /// assert_eq!(current, (&10, &100)); /// @@ -702,7 +753,7 @@ impl Drop for RBTree { /// assert!(cursor.remove_prev().is_none()); /// /// // Get a cursor to the last element. -/// cursor = tree.cursor_back().unwrap(); +/// cursor = tree.cursor_back_mut().unwrap(); /// current = cursor.current(); /// assert_eq!(current, (&30, &300)); /// @@ -726,18 +777,48 @@ impl Drop for RBTree { /// /// # Invariants /// - `current` points to a node that is in the same [`RBTree`] as `tree`. -pub struct Cursor<'a, K, V> { +pub struct CursorMut<'a, K, V> { tree: &'a mut RBTree, current: NonNull, } -// SAFETY: The [`Cursor`] has exclusive access to both `K` and `V`, so it is sufficient to require them to be `Send`. -// The cursor only gives out immutable references to the keys, but since it has excusive access to those same -// keys, `Send` is sufficient. `Sync` would be okay, but it is more restrictive to the user. -unsafe impl<'a, K: Send, V: Send> Send for Cursor<'a, K, V> {} +/// A bidirectional immutable cursor over the tree nodes, sorted by key. This is a simpler +/// variant of [`CursorMut`] that is basically providing read only access. +/// +/// # Examples +/// +/// In the following example, we obtain a cursor to the first element in the tree. +/// The cursor allows us to iterate bidirectionally over key/value pairs in the tree. +/// +/// ``` +/// use kernel::{alloc::flags, rbtree::RBTree}; +/// +/// // Create a new tree. +/// let mut tree = RBTree::new(); +/// +/// // Insert three elements. +/// tree.try_create_and_insert(10, 100, flags::GFP_KERNEL)?; +/// tree.try_create_and_insert(20, 200, flags::GFP_KERNEL)?; +/// tree.try_create_and_insert(30, 300, flags::GFP_KERNEL)?; +/// +/// // Get a cursor to the first element. +/// let cursor = tree.cursor_front().unwrap(); +/// let current = cursor.current(); +/// assert_eq!(current, (&10, &100)); +/// +/// # Ok::<(), Error>(()) +/// ``` +pub struct Cursor<'a, K, V> { + _tree: PhantomData<&'a RBTree>, + current: NonNull, +} -// SAFETY: The [`Cursor`] gives out immutable references to K and mutable references to V, -// so it has the same thread safety requirements as mutable references. +// SAFETY: The immutable cursor gives out shared access to `K` and `V` so if `K` and `V` can be +// shared across threads, then it's safe to share the cursor. +unsafe impl<'a, K: Sync, V: Sync> Send for Cursor<'a, K, V> {} + +// SAFETY: The immutable cursor gives out shared access to `K` and `V` so if `K` and `V` can be +// shared across threads, then it's safe to share the cursor. unsafe impl<'a, K: Sync, V: Sync> Sync for Cursor<'a, K, V> {} impl<'a, K, V> Cursor<'a, K, V> { @@ -749,6 +830,75 @@ impl<'a, K, V> Cursor<'a, K, V> { unsafe { Self::to_key_value(self.current) } } + /// # Safety + /// + /// - `node` must be a valid pointer to a node in an [`RBTree`]. + /// - The caller has immutable access to `node` for the duration of `'b`. + unsafe fn to_key_value<'b>(node: NonNull) -> (&'b K, &'b V) { + // SAFETY: By the type invariant of `Self`, all non-null `rb_node` pointers stored in `self` + // point to the links field of `Node` objects. + let this = unsafe { container_of!(node.as_ptr(), Node, links) }; + // SAFETY: The passed `node` is the current node or a non-null neighbor, + // thus `this` is valid by the type invariants. + let k = unsafe { &(*this).key }; + // SAFETY: The passed `node` is the current node or a non-null neighbor, + // thus `this` is valid by the type invariants. + let v = unsafe { &(*this).value }; + (k, v) + } + + /// Access the previous node without moving the cursor. + pub fn peek_prev(&self) -> Option<(&K, &V)> { + self.peek(Direction::Prev) + } + + /// Access the next node without moving the cursor. + pub fn peek_next(&self) -> Option<(&K, &V)> { + self.peek(Direction::Next) + } + + fn peek(&self, direction: Direction) -> Option<(&K, &V)> { + self.get_neighbor_raw(direction).map(|neighbor| { + // SAFETY: + // - `neighbor` is a valid tree node. + // - By the function signature, we have an immutable reference to `self`. + unsafe { Self::to_key_value(neighbor) } + }) + } + + fn get_neighbor_raw(&self, direction: Direction) -> Option> { + // SAFETY: `self.current` is valid by the type invariants. + let neighbor = unsafe { + match direction { + Direction::Prev => bindings::rb_prev(self.current.as_ptr()), + Direction::Next => bindings::rb_next(self.current.as_ptr()), + } + }; + + NonNull::new(neighbor) + } +} + +// SAFETY: The [`CursorMut`] has exclusive access to both `K` and `V`, so it is sufficient to +// require them to be `Send`. +// The cursor only gives out immutable references to the keys, but since it has exclusive access to +// those same keys, `Send` is sufficient. `Sync` would be okay, but it is more restrictive to the +// user. +unsafe impl<'a, K: Send, V: Send> Send for CursorMut<'a, K, V> {} + +// SAFETY: The [`CursorMut`] gives out immutable references to `K` and mutable references to `V`, +// so it has the same thread safety requirements as mutable references. +unsafe impl<'a, K: Sync, V: Sync> Sync for CursorMut<'a, K, V> {} + +impl<'a, K, V> CursorMut<'a, K, V> { + /// The current node. + pub fn current(&self) -> (&K, &V) { + // SAFETY: + // - `self.current` is a valid node by the type invariants. + // - We have an immutable reference by the function signature. + unsafe { Self::to_key_value(self.current) } + } + /// The current node, with a mutable value pub fn current_mut(&mut self) -> (&K, &mut V) { // SAFETY: @@ -920,7 +1070,7 @@ impl<'a, K, V> Cursor<'a, K, V> { } } -/// Direction for [`Cursor`] operations. +/// Direction for [`Cursor`] and [`CursorMut`] operations. enum Direction { /// the node immediately before, in sort order Prev, From 305b795730684796f3f4bbb1aa8f04d52cd2005d Mon Sep 17 00:00:00 2001 From: Tamir Duberstein Date: Thu, 13 Nov 2025 17:58:24 -0500 Subject: [PATCH 19/53] rust: firmware: replace `kernel::c_str!` with C-Strings C-String literals were added in Rust 1.77. Replace instances of `kernel::c_str!` with C-String literals where possible. Acked-by: Greg Kroah-Hartman Reviewed-by: Alice Ryhl Reviewed-by: Benno Lossin Acked-by: Danilo Krummrich Signed-off-by: Tamir Duberstein Link: https://patch.msgid.link/20251113-core-cstr-cstrings-v3-1-411b34002774@gmail.com Signed-off-by: Miguel Ojeda --- rust/kernel/firmware.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/rust/kernel/firmware.rs b/rust/kernel/firmware.rs index 376e7e77453f..71168d8004e2 100644 --- a/rust/kernel/firmware.rs +++ b/rust/kernel/firmware.rs @@ -51,13 +51,13 @@ impl FwFunc { /// # Examples /// /// ```no_run -/// # use kernel::{c_str, device::Device, firmware::Firmware}; +/// # use kernel::{device::Device, firmware::Firmware}; /// /// # fn no_run() -> Result<(), Error> { /// # // SAFETY: *NOT* safe, just for the example to get an `ARef` instance /// # let dev = unsafe { Device::get_device(core::ptr::null_mut()) }; /// -/// let fw = Firmware::request(c_str!("path/to/firmware.bin"), &dev)?; +/// let fw = Firmware::request(c"path/to/firmware.bin", &dev)?; /// let blob = fw.data(); /// /// # Ok(()) @@ -204,7 +204,7 @@ macro_rules! module_firmware { ($($builder:tt)*) => { const _: () = { const __MODULE_FIRMWARE_PREFIX: &'static $crate::str::CStr = if cfg!(MODULE) { - $crate::c_str!("") + c"" } else { ::NAME }; From ab8a6c7b34513e48eb4fa766daaa81e75e4306ed Mon Sep 17 00:00:00 2001 From: Tamir Duberstein Date: Thu, 13 Nov 2025 17:58:26 -0500 Subject: [PATCH 20/53] rust: str: replace `kernel::c_str!` with C-Strings C-String literals were added in Rust 1.77. Replace instances of `kernel::c_str!` with C-String literals where possible. Acked-by: Greg Kroah-Hartman Reviewed-by: Alice Ryhl Reviewed-by: Benno Lossin Signed-off-by: Tamir Duberstein Link: https://patch.msgid.link/20251113-core-cstr-cstrings-v3-3-411b34002774@gmail.com [ Removed unused `c_str` import in doctest. - Miguel ] Signed-off-by: Miguel Ojeda --- rust/kernel/str.rs | 59 +++++++++++++++++++++++----------------------- 1 file changed, 29 insertions(+), 30 deletions(-) diff --git a/rust/kernel/str.rs b/rust/kernel/str.rs index 7593d758fbb7..6fcc9d47f12e 100644 --- a/rust/kernel/str.rs +++ b/rust/kernel/str.rs @@ -277,15 +277,14 @@ impl fmt::Display for CStr { /// Formats printable ASCII characters, escaping the rest. /// /// ``` - /// # use kernel::c_str; /// # use kernel::prelude::fmt; /// # use kernel::str::CStr; /// # use kernel::str::CString; - /// let penguin = c_str!("🐧"); + /// let penguin = c"🐧"; /// let s = CString::try_from_fmt(fmt!("{penguin}"))?; /// assert_eq!(s.to_bytes_with_nul(), "\\xf0\\x9f\\x90\\xa7\0".as_bytes()); /// - /// let ascii = c_str!("so \"cool\""); + /// let ascii = c"so \"cool\""; /// let s = CString::try_from_fmt(fmt!("{ascii}"))?; /// assert_eq!(s.to_bytes_with_nul(), "so \"cool\"\0".as_bytes()); /// # Ok::<(), kernel::error::Error>(()) @@ -724,43 +723,43 @@ unsafe fn kstrtobool_raw(string: *const u8) -> Result { /// # Examples /// /// ``` -/// # use kernel::{c_str, str::kstrtobool}; +/// # use kernel::str::kstrtobool; /// /// // Lowercase -/// assert_eq!(kstrtobool(c_str!("true")), Ok(true)); -/// assert_eq!(kstrtobool(c_str!("tr")), Ok(true)); -/// assert_eq!(kstrtobool(c_str!("t")), Ok(true)); -/// assert_eq!(kstrtobool(c_str!("twrong")), Ok(true)); -/// assert_eq!(kstrtobool(c_str!("false")), Ok(false)); -/// assert_eq!(kstrtobool(c_str!("f")), Ok(false)); -/// assert_eq!(kstrtobool(c_str!("yes")), Ok(true)); -/// assert_eq!(kstrtobool(c_str!("no")), Ok(false)); -/// assert_eq!(kstrtobool(c_str!("on")), Ok(true)); -/// assert_eq!(kstrtobool(c_str!("off")), Ok(false)); +/// assert_eq!(kstrtobool(c"true"), Ok(true)); +/// assert_eq!(kstrtobool(c"tr"), Ok(true)); +/// assert_eq!(kstrtobool(c"t"), Ok(true)); +/// assert_eq!(kstrtobool(c"twrong"), Ok(true)); +/// assert_eq!(kstrtobool(c"false"), Ok(false)); +/// assert_eq!(kstrtobool(c"f"), Ok(false)); +/// assert_eq!(kstrtobool(c"yes"), Ok(true)); +/// assert_eq!(kstrtobool(c"no"), Ok(false)); +/// assert_eq!(kstrtobool(c"on"), Ok(true)); +/// assert_eq!(kstrtobool(c"off"), Ok(false)); /// /// // Camel case -/// assert_eq!(kstrtobool(c_str!("True")), Ok(true)); -/// assert_eq!(kstrtobool(c_str!("False")), Ok(false)); -/// assert_eq!(kstrtobool(c_str!("Yes")), Ok(true)); -/// assert_eq!(kstrtobool(c_str!("No")), Ok(false)); -/// assert_eq!(kstrtobool(c_str!("On")), Ok(true)); -/// assert_eq!(kstrtobool(c_str!("Off")), Ok(false)); +/// assert_eq!(kstrtobool(c"True"), Ok(true)); +/// assert_eq!(kstrtobool(c"False"), Ok(false)); +/// assert_eq!(kstrtobool(c"Yes"), Ok(true)); +/// assert_eq!(kstrtobool(c"No"), Ok(false)); +/// assert_eq!(kstrtobool(c"On"), Ok(true)); +/// assert_eq!(kstrtobool(c"Off"), Ok(false)); /// /// // All caps -/// assert_eq!(kstrtobool(c_str!("TRUE")), Ok(true)); -/// assert_eq!(kstrtobool(c_str!("FALSE")), Ok(false)); -/// assert_eq!(kstrtobool(c_str!("YES")), Ok(true)); -/// assert_eq!(kstrtobool(c_str!("NO")), Ok(false)); -/// assert_eq!(kstrtobool(c_str!("ON")), Ok(true)); -/// assert_eq!(kstrtobool(c_str!("OFF")), Ok(false)); +/// assert_eq!(kstrtobool(c"TRUE"), Ok(true)); +/// assert_eq!(kstrtobool(c"FALSE"), Ok(false)); +/// assert_eq!(kstrtobool(c"YES"), Ok(true)); +/// assert_eq!(kstrtobool(c"NO"), Ok(false)); +/// assert_eq!(kstrtobool(c"ON"), Ok(true)); +/// assert_eq!(kstrtobool(c"OFF"), Ok(false)); /// /// // Numeric -/// assert_eq!(kstrtobool(c_str!("1")), Ok(true)); -/// assert_eq!(kstrtobool(c_str!("0")), Ok(false)); +/// assert_eq!(kstrtobool(c"1"), Ok(true)); +/// assert_eq!(kstrtobool(c"0"), Ok(false)); /// /// // Invalid input -/// assert_eq!(kstrtobool(c_str!("invalid")), Err(EINVAL)); -/// assert_eq!(kstrtobool(c_str!("2")), Err(EINVAL)); +/// assert_eq!(kstrtobool(c"invalid"), Err(EINVAL)); +/// assert_eq!(kstrtobool(c"2"), Err(EINVAL)); /// ``` pub fn kstrtobool(string: &CStr) -> Result { // SAFETY: From bf724be7f00cf2cbafc105422476f6242e917a90 Mon Sep 17 00:00:00 2001 From: Tamir Duberstein Date: Thu, 13 Nov 2025 17:58:29 -0500 Subject: [PATCH 21/53] rust: macros: replace `kernel::c_str!` with C-Strings C-String literals were added in Rust 1.77. Replace instances of `kernel::c_str!` with C-String literals where possible. Acked-by: Greg Kroah-Hartman Reviewed-by: Alice Ryhl Reviewed-by: Benno Lossin Signed-off-by: Tamir Duberstein Link: https://patch.msgid.link/20251113-core-cstr-cstrings-v3-6-411b34002774@gmail.com Signed-off-by: Miguel Ojeda --- rust/macros/module.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rust/macros/module.rs b/rust/macros/module.rs index 5ee54a00c0b6..8cef6cc958b5 100644 --- a/rust/macros/module.rs +++ b/rust/macros/module.rs @@ -228,7 +228,7 @@ pub(crate) fn module(ts: TokenStream) -> TokenStream { type LocalModule = {type_}; impl ::kernel::ModuleMetadata for {type_} {{ - const NAME: &'static ::kernel::str::CStr = ::kernel::c_str!(\"{name}\"); + const NAME: &'static ::kernel::str::CStr = c\"{name}\"; }} // Double nested modules, since then nobody can access the public items inside. From ab844cf32058d6ba9bd98a921afa2256085e21fc Mon Sep 17 00:00:00 2001 From: Miguel Ojeda Date: Mon, 10 Nov 2025 12:35:27 +0100 Subject: [PATCH 22/53] rust: allow `unreachable_pub` for doctests Examples (i.e. doctests) may want to show public items such as structs, thus the `unreachable_pub` warning is not very helpful. Thus allow it for all doctests. In addition, remove it from the existing `expect`s we have in a couple doctests. Suggested-by: Alice Ryhl Link: https://lore.kernel.org/rust-for-linux/aRG9VjsaCjsvAwUn@google.com/ Reviewed-by: David Gow Acked-by: Benno Lossin Reviewed-by: Alice Ryhl Link: https://patch.msgid.link/20251110113528.1658238-1-ojeda@kernel.org Signed-off-by: Miguel Ojeda --- rust/kernel/init.rs | 2 +- rust/kernel/types.rs | 2 +- scripts/rustdoc_test_gen.rs | 1 + 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/rust/kernel/init.rs b/rust/kernel/init.rs index 4949047af8d7..e476d81c1a27 100644 --- a/rust/kernel/init.rs +++ b/rust/kernel/init.rs @@ -67,7 +67,7 @@ //! ``` //! //! ```rust -//! # #![expect(unreachable_pub, clippy::disallowed_names)] +//! # #![expect(clippy::disallowed_names)] //! use kernel::{prelude::*, types::Opaque}; //! use core::{ptr::addr_of_mut, marker::PhantomPinned, pin::Pin}; //! # mod bindings { diff --git a/rust/kernel/types.rs b/rust/kernel/types.rs index dc0a02f5c3cf..835824788506 100644 --- a/rust/kernel/types.rs +++ b/rust/kernel/types.rs @@ -289,7 +289,7 @@ impl Drop for ScopeGuard { /// # Examples /// /// ``` -/// # #![expect(unreachable_pub, clippy::disallowed_names)] +/// # #![expect(clippy::disallowed_names)] /// use kernel::types::Opaque; /// # // Emulate a C struct binding which is from C, maybe uninitialized or not, only the C side /// # // knows. diff --git a/scripts/rustdoc_test_gen.rs b/scripts/rustdoc_test_gen.rs index c8f9dc2ab976..0e6a0542d1bd 100644 --- a/scripts/rustdoc_test_gen.rs +++ b/scripts/rustdoc_test_gen.rs @@ -208,6 +208,7 @@ pub extern "C" fn {kunit_name}(__kunit_test: *mut ::kernel::bindings::kunit) {{ #[allow(unused)] static __DOCTEST_ANCHOR: i32 = ::core::line!() as i32 + {body_offset} + 1; {{ + #![allow(unreachable_pub)] {body} main(); }} From e5d330e13f67d574f683c052c9a342814fd8fa39 Mon Sep 17 00:00:00 2001 From: Miguel Ojeda Date: Mon, 17 Nov 2025 09:07:13 +0100 Subject: [PATCH 23/53] rust: allow `clippy::disallowed_names` for doctests Examples (i.e. doctests) may want to use names such as `foo`, thus the `clippy::disallowed_names` lint [1] gets in the way. Thus allow it for all doctests. In addition, remove it from the existing `expect`s we have in a few doctests. This does not mean that we should stop trying to find good names for our examples, though. Link: https://rust-lang.github.io/rust-clippy/stable/index.html#disallowed_names [1] Suggested-by: Alice Ryhl Link: https://lore.kernel.org/rust-for-linux/aRHSLChi5HYXW4-9@google.com/ Reviewed-by: Alice Ryhl Acked-by: Benno Lossin Link: https://patch.msgid.link/20251117080714.876978-1-ojeda@kernel.org Signed-off-by: Miguel Ojeda --- rust/kernel/init.rs | 3 +-- rust/kernel/types.rs | 1 - scripts/rustdoc_test_gen.rs | 2 +- 3 files changed, 2 insertions(+), 4 deletions(-) diff --git a/rust/kernel/init.rs b/rust/kernel/init.rs index e476d81c1a27..899b9a962762 100644 --- a/rust/kernel/init.rs +++ b/rust/kernel/init.rs @@ -30,7 +30,7 @@ //! ## General Examples //! //! ```rust -//! # #![expect(clippy::disallowed_names, clippy::undocumented_unsafe_blocks)] +//! # #![expect(clippy::undocumented_unsafe_blocks)] //! use kernel::types::Opaque; //! use pin_init::pin_init_from_closure; //! @@ -67,7 +67,6 @@ //! ``` //! //! ```rust -//! # #![expect(clippy::disallowed_names)] //! use kernel::{prelude::*, types::Opaque}; //! use core::{ptr::addr_of_mut, marker::PhantomPinned, pin::Pin}; //! # mod bindings { diff --git a/rust/kernel/types.rs b/rust/kernel/types.rs index 835824788506..9c5e7dbf1632 100644 --- a/rust/kernel/types.rs +++ b/rust/kernel/types.rs @@ -289,7 +289,6 @@ impl Drop for ScopeGuard { /// # Examples /// /// ``` -/// # #![expect(clippy::disallowed_names)] /// use kernel::types::Opaque; /// # // Emulate a C struct binding which is from C, maybe uninitialized or not, only the C side /// # // knows. diff --git a/scripts/rustdoc_test_gen.rs b/scripts/rustdoc_test_gen.rs index 0e6a0542d1bd..be0561049660 100644 --- a/scripts/rustdoc_test_gen.rs +++ b/scripts/rustdoc_test_gen.rs @@ -208,7 +208,7 @@ pub extern "C" fn {kunit_name}(__kunit_test: *mut ::kernel::bindings::kunit) {{ #[allow(unused)] static __DOCTEST_ANCHOR: i32 = ::core::line!() as i32 + {body_offset} + 1; {{ - #![allow(unreachable_pub)] + #![allow(unreachable_pub, clippy::disallowed_names)] {body} main(); }} From 90f3df4fdfb682e2394ee3f97dfe91a402d5c46a Mon Sep 17 00:00:00 2001 From: Alexandre Courbot Date: Sat, 8 Nov 2025 11:23:47 +0900 Subject: [PATCH 24/53] rust: add num module and Integer trait Introduce the `num` module, which will provide numerical extensions and utilities for the kernel. For now, introduce the `Integer` trait, which is implemented for all primitive integer types to provides their core properties to generic code. Signed-off-by: Alexandre Courbot Reviewed-by: Alice Ryhl Link: https://patch.msgid.link/20251108-bounded_ints-v4-1-c9342ac7ebd1@nvidia.com Signed-off-by: Miguel Ojeda --- rust/kernel/lib.rs | 1 + rust/kernel/num.rs | 76 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 77 insertions(+) create mode 100644 rust/kernel/num.rs diff --git a/rust/kernel/lib.rs b/rust/kernel/lib.rs index 3dd7bebe7888..235d0d8b1eff 100644 --- a/rust/kernel/lib.rs +++ b/rust/kernel/lib.rs @@ -109,6 +109,7 @@ pub mod miscdevice; pub mod mm; #[cfg(CONFIG_NET)] pub mod net; +pub mod num; pub mod of; #[cfg(CONFIG_PM_OPP)] pub mod opp; diff --git a/rust/kernel/num.rs b/rust/kernel/num.rs new file mode 100644 index 000000000000..c8c91cb9e682 --- /dev/null +++ b/rust/kernel/num.rs @@ -0,0 +1,76 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Additional numerical features for the kernel. + +use core::ops; + +/// Designates unsigned primitive types. +pub enum Unsigned {} + +/// Designates signed primitive types. +pub enum Signed {} + +/// Describes core properties of integer types. +pub trait Integer: + Sized + + Copy + + Clone + + PartialEq + + Eq + + PartialOrd + + Ord + + ops::Add + + ops::AddAssign + + ops::Sub + + ops::SubAssign + + ops::Mul + + ops::MulAssign + + ops::Div + + ops::DivAssign + + ops::Rem + + ops::RemAssign + + ops::BitAnd + + ops::BitAndAssign + + ops::BitOr + + ops::BitOrAssign + + ops::BitXor + + ops::BitXorAssign + + ops::Shl + + ops::ShlAssign + + ops::Shr + + ops::ShrAssign + + ops::Not +{ + /// Whether this type is [`Signed`] or [`Unsigned`]. + type Signedness; + + /// Number of bits used for value representation. + const BITS: u32; +} + +macro_rules! impl_integer { + ($($type:ty: $signedness:ty), *) => { + $( + impl Integer for $type { + type Signedness = $signedness; + + const BITS: u32 = <$type>::BITS; + } + )* + }; +} + +impl_integer!( + u8: Unsigned, + u16: Unsigned, + u32: Unsigned, + u64: Unsigned, + u128: Unsigned, + usize: Unsigned, + i8: Signed, + i16: Signed, + i32: Signed, + i64: Signed, + i128: Signed, + isize: Signed +); From 01e345e82ec3a5a7edeb9fa0dcb7fd4b0e5c534e Mon Sep 17 00:00:00 2001 From: Alexandre Courbot Date: Sat, 8 Nov 2025 11:23:48 +0900 Subject: [PATCH 25/53] rust: num: add Bounded integer wrapping type Add the `Bounded` integer wrapper type, which restricts the number of bits allowed to represent of value. This is useful to e.g. enforce guarantees when working with bitfields that have an arbitrary number of bits. Alongside this type, provide many `From` and `TryFrom` implementations are to reduce friction when using with regular integer types. Proxy implementations of common integer operations are also provided. Signed-off-by: Alexandre Courbot Reviewed-by: Alice Ryhl Link: https://patch.msgid.link/20251108-bounded_ints-v4-2-c9342ac7ebd1@nvidia.com [ Added intra-doc link. Fixed a few other nits. - Miguel ] Signed-off-by: Miguel Ojeda --- rust/kernel/num.rs | 3 + rust/kernel/num/bounded.rs | 1056 ++++++++++++++++++++++++++++++++++++ 2 files changed, 1059 insertions(+) create mode 100644 rust/kernel/num/bounded.rs diff --git a/rust/kernel/num.rs b/rust/kernel/num.rs index c8c91cb9e682..8532b511384c 100644 --- a/rust/kernel/num.rs +++ b/rust/kernel/num.rs @@ -4,6 +4,9 @@ use core::ops; +pub mod bounded; +pub use bounded::*; + /// Designates unsigned primitive types. pub enum Unsigned {} diff --git a/rust/kernel/num/bounded.rs b/rust/kernel/num/bounded.rs new file mode 100644 index 000000000000..18e45bf6f84b --- /dev/null +++ b/rust/kernel/num/bounded.rs @@ -0,0 +1,1056 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Implementation of [`Bounded`], a wrapper around integer types limiting the number of bits +//! usable for value representation. + +use core::{ + cmp, + fmt, + ops::{ + self, + Deref, // + }, //, +}; + +use kernel::{ + num::Integer, + prelude::*, // +}; + +/// Evaluates to `true` if `$value` can be represented using at most `$n` bits in a `$type`. +/// +/// `expr` must be of type `type`, or the result will be incorrect. +/// +/// Can be used in const context. +macro_rules! fits_within { + ($value:expr, $type:ty, $n:expr) => {{ + let shift: u32 = <$type>::BITS - $n; + + // `value` fits within `$n` bits if shifting it left by the number of unused bits, then + // right by the same number, doesn't change it. + // + // This method has the benefit of working for both unsigned and signed values. + ($value << shift) >> shift == $value + }}; +} + +/// Returns `true` if `value` can be represented with at most `N` bits in a `T`. +fn fits_within(value: T, num_bits: u32) -> bool { + fits_within!(value, T, num_bits) +} + +/// An integer value that requires only the `N` less significant bits of the wrapped type to be +/// encoded. +/// +/// This limits the number of usable bits in the wrapped integer type, and thus the stored value to +/// a narrower range, which provides guarantees that can be useful when working with in e.g. +/// bitfields. +/// +/// # Invariants +/// +/// - `N` is greater than `0`. +/// - `N` is less than or equal to `T::BITS`. +/// - Stored values can be represented with at most `N` bits. +/// +/// # Examples +/// +/// The preferred way to create values is through constants and the [`Bounded::new`] family of +/// constructors, as they trigger a build error if the type invariants cannot be withheld. +/// +/// ``` +/// use kernel::num::Bounded; +/// +/// // An unsigned 8-bit integer, of which only the 4 LSBs are used. +/// // The value `15` is statically validated to fit that constraint at build time. +/// let v = Bounded::::new::<15>(); +/// assert_eq!(v.get(), 15); +/// +/// // Same using signed values. +/// let v = Bounded::::new::<-8>(); +/// assert_eq!(v.get(), -8); +/// +/// // This doesn't build: a `u8` is smaller than the requested 9 bits. +/// // let _ = Bounded::::new::<10>(); +/// +/// // This also doesn't build: the requested value doesn't fit within 4 signed bits. +/// // let _ = Bounded::::new::<8>(); +/// ``` +/// +/// Values can also be validated at runtime with [`Bounded::try_new`]. +/// +/// ``` +/// use kernel::num::Bounded; +/// +/// // This succeeds because `15` can be represented with 4 unsigned bits. +/// assert!(Bounded::::try_new(15).is_some()); +/// +/// // This fails because `16` cannot be represented with 4 unsigned bits. +/// assert!(Bounded::::try_new(16).is_none()); +/// ``` +/// +/// Non-constant expressions can be validated at build-time thanks to compiler optimizations. This +/// should be used with caution, on simple expressions only. +/// +/// ``` +/// use kernel::num::Bounded; +/// # fn some_number() -> u32 { 0xffffffff } +/// +/// // Here the compiler can infer from the mask that the type invariants are not violated, even +/// // though the value returned by `some_number` is not statically known. +/// let v = Bounded::::from_expr(some_number() & 0xf); +/// ``` +/// +/// Comparison and arithmetic operations are supported on [`Bounded`]s with a compatible backing +/// type, regardless of their number of valid bits. +/// +/// ``` +/// use kernel::num::Bounded; +/// +/// let v1 = Bounded::::new::<4>(); +/// let v2 = Bounded::::new::<15>(); +/// +/// assert!(v1 != v2); +/// assert!(v1 < v2); +/// assert_eq!(v1 + v2, 19); +/// assert_eq!(v2 % v1, 3); +/// ``` +/// +/// These operations are also supported between a [`Bounded`] and its backing type. +/// +/// ``` +/// use kernel::num::Bounded; +/// +/// let v = Bounded::::new::<15>(); +/// +/// assert!(v == 15); +/// assert!(v > 12); +/// assert_eq!(v + 5, 20); +/// assert_eq!(v / 3, 5); +/// ``` +/// +/// A change of backing types is possible using [`Bounded::cast`], and the number of valid bits can +/// be extended or reduced with [`Bounded::extend`] and [`Bounded::try_shrink`]. +/// +/// ``` +/// use kernel::num::Bounded; +/// +/// let v = Bounded::::new::<127>(); +/// +/// // Changes backing type from `u32` to `u16`. +/// let _: Bounded = v.cast(); +/// +/// // This does not build, as `u8` is smaller than 12 bits. +/// // let _: Bounded = v.cast(); +/// +/// // We can safely extend the number of bits... +/// let _ = v.extend::<15>(); +/// +/// // ... to the limits of the backing type. This doesn't build as a `u32` cannot contain 33 bits. +/// // let _ = v.extend::<33>(); +/// +/// // Reducing the number of bits is validated at runtime. This works because `127` can be +/// // represented with 8 bits. +/// assert!(v.try_shrink::<8>().is_some()); +/// +/// // ... but not with 6, so this fails. +/// assert!(v.try_shrink::<6>().is_none()); +/// ``` +/// +/// Infallible conversions from a primitive integer to a large-enough [`Bounded`] are supported. +/// +/// ``` +/// use kernel::num::Bounded; +/// +/// // This unsigned `Bounded` has 8 bits, so it can represent any `u8`. +/// let v = Bounded::::from(128u8); +/// assert_eq!(v.get(), 128); +/// +/// // This signed `Bounded` has 8 bits, so it can represent any `i8`. +/// let v = Bounded::::from(-128i8); +/// assert_eq!(v.get(), -128); +/// +/// // This doesn't build, as this 6-bit `Bounded` does not have enough capacity to represent a +/// // `u8` (regardless of the passed value). +/// // let _ = Bounded::::from(10u8); +/// +/// // Booleans can be converted into single-bit `Bounded`s. +/// +/// let v = Bounded::::from(false); +/// assert_eq!(v.get(), 0); +/// +/// let v = Bounded::::from(true); +/// assert_eq!(v.get(), 1); +/// ``` +/// +/// Infallible conversions from a [`Bounded`] to a primitive integer are also supported, and +/// dependent on the number of bits used for value representation, not on the backing type. +/// +/// ``` +/// use kernel::num::Bounded; +/// +/// // Even though its backing type is `u32`, this `Bounded` only uses 6 bits and thus can safely +/// // be converted to a `u8`. +/// let v = Bounded::::new::<63>(); +/// assert_eq!(u8::from(v), 63); +/// +/// // Same using signed values. +/// let v = Bounded::::new::<-128>(); +/// assert_eq!(i8::from(v), -128); +/// +/// // This however does not build, as 10 bits won't fit into a `u8` (regardless of the actually +/// // contained value). +/// let _v = Bounded::::new::<10>(); +/// // assert_eq!(u8::from(_v), 10); +/// +/// // Single-bit `Bounded`s can be converted into a boolean. +/// let v = Bounded::::new::<1>(); +/// assert_eq!(bool::from(v), true); +/// +/// let v = Bounded::::new::<0>(); +/// assert_eq!(bool::from(v), false); +/// ``` +/// +/// Fallible conversions from any primitive integer to any [`Bounded`] are also supported using the +/// [`TryIntoBounded`] trait. +/// +/// ``` +/// use kernel::num::{Bounded, TryIntoBounded}; +/// +/// // Succeeds because `128` fits into 8 bits. +/// let v: Option> = 128u32.try_into_bitint(); +/// assert_eq!(v.as_deref().copied(), Some(128)); +/// +/// // Fails because `128` doesn't fits into 6 bits. +/// let v: Option> = 128u32.try_into_bitint(); +/// assert_eq!(v, None); +/// ``` +#[repr(transparent)] +#[derive(Clone, Copy, Debug, Default, Hash)] +pub struct Bounded(T); + +/// Validating the value as a const expression cannot be done as a regular method, as the +/// arithmetic operations we rely on to check the bounds are not const. Thus, implement +/// [`Bounded::new`] using a macro. +macro_rules! impl_const_new { + ($($type:ty)*) => { + $( + impl Bounded<$type, N> { + /// Creates a [`Bounded`] for the constant `VALUE`. + /// + /// Fails at build time if `VALUE` cannot be represented with `N` bits. + /// + /// This method should be preferred to [`Self::from_expr`] whenever possible. + /// + /// # Examples + /// + /// ``` + /// use kernel::num::Bounded; + /// + #[doc = ::core::concat!( + "let v = Bounded::<", + ::core::stringify!($type), + ", 4>::new::<7>();")] + /// assert_eq!(v.get(), 7); + /// ``` + pub const fn new() -> Self { + // Statically assert that `VALUE` fits within the set number of bits. + const { + assert!(fits_within!(VALUE, $type, N)); + } + + // INVARIANT: `fits_within` confirmed that `VALUE` can be represented within + // `N` bits. + Self::__new(VALUE) + } + } + )* + }; +} + +impl_const_new!( + u8 u16 u32 u64 usize + i8 i16 i32 i64 isize +); + +impl Bounded +where + T: Integer, +{ + /// Private constructor enforcing the type invariants. + /// + /// All instances of [`Bounded`] must be created through this method as it enforces most of the + /// type invariants. + /// + /// The caller remains responsible for checking, either statically or dynamically, that `value` + /// can be represented as a `T` using at most `N` bits. + const fn __new(value: T) -> Self { + // Enforce the type invariants. + const { + // `N` cannot be zero. + assert!(N != 0); + // The backing type is at least as large as `N` bits. + assert!(N <= T::BITS); + } + + Self(value) + } + + /// Attempts to turn `value` into a `Bounded` using `N` bits. + /// + /// Returns [`None`] if `value` doesn't fit within `N` bits. + /// + /// # Examples + /// + /// ``` + /// use kernel::num::Bounded; + /// + /// let v = Bounded::::try_new(1); + /// assert_eq!(v.as_deref().copied(), Some(1)); + /// + /// let v = Bounded::::try_new(-2); + /// assert_eq!(v.as_deref().copied(), Some(-2)); + /// + /// // `0x1ff` doesn't fit into 8 unsigned bits. + /// let v = Bounded::::try_new(0x1ff); + /// assert_eq!(v, None); + /// + /// // The range of values representable with 4 bits is `[-8..=7]`. The following tests these + /// // limits. + /// let v = Bounded::::try_new(-8); + /// assert_eq!(v.map(Bounded::get), Some(-8)); + /// let v = Bounded::::try_new(-9); + /// assert_eq!(v, None); + /// let v = Bounded::::try_new(7); + /// assert_eq!(v.map(Bounded::get), Some(7)); + /// let v = Bounded::::try_new(8); + /// assert_eq!(v, None); + /// ``` + pub fn try_new(value: T) -> Option { + fits_within(value, N).then(|| { + // INVARIANT: `fits_within` confirmed that `value` can be represented within `N` bits. + Self::__new(value) + }) + } + + /// Checks that `expr` is valid for this type at compile-time and build a new value. + /// + /// This relies on [`build_assert!`] and guaranteed optimization to perform validation at + /// compile-time. If `expr` cannot be proved to be within the requested bounds at compile-time, + /// use the fallible [`Self::try_new`] instead. + /// + /// Limit this to simple, easily provable expressions, and prefer one of the [`Self::new`] + /// constructors whenever possible as they statically validate the value instead of relying on + /// compiler optimizations. + /// + /// # Examples + /// + /// ``` + /// use kernel::num::Bounded; + /// # fn some_number() -> u32 { 0xffffffff } + /// + /// // Some undefined number. + /// let v: u32 = some_number(); + /// + /// // Triggers a build error as `v` cannot be asserted to fit within 4 bits... + /// // let _ = Bounded::::from_expr(v); + /// + /// // ... but this works as the compiler can assert the range from the mask. + /// let _ = Bounded::::from_expr(v & 0xf); + /// + /// // These expressions are simple enough to be proven correct, but since they are static the + /// // `new` constructor should be preferred. + /// assert_eq!(Bounded::::from_expr(1).get(), 1); + /// assert_eq!(Bounded::::from_expr(0xff).get(), 0xff); + /// ``` + pub fn from_expr(expr: T) -> Self { + crate::build_assert!( + fits_within(expr, N), + "Requested value larger than maximal representable value." + ); + + // INVARIANT: `fits_within` confirmed that `expr` can be represented within `N` bits. + Self::__new(expr) + } + + /// Returns the wrapped value as the backing type. + /// + /// # Examples + /// + /// ``` + /// use kernel::num::Bounded; + /// + /// let v = Bounded::::new::<7>(); + /// assert_eq!(v.get(), 7u32); + /// ``` + pub fn get(self) -> T { + *self.deref() + } + + /// Increases the number of bits usable for `self`. + /// + /// This operation cannot fail. + /// + /// # Examples + /// + /// ``` + /// use kernel::num::Bounded; + /// + /// let v = Bounded::::new::<7>(); + /// let larger_v = v.extend::<12>(); + /// // The contained values are equal even though `larger_v` has a bigger capacity. + /// assert_eq!(larger_v, v); + /// ``` + pub const fn extend(self) -> Bounded { + const { + assert!( + M >= N, + "Requested number of bits is less than the current representation." + ); + } + + // INVARIANT: The value did fit within `N` bits, so it will all the more fit within + // the larger `M` bits. + Bounded::__new(self.0) + } + + /// Attempts to shrink the number of bits usable for `self`. + /// + /// Returns [`None`] if the value of `self` cannot be represented within `M` bits. + /// + /// # Examples + /// + /// ``` + /// use kernel::num::Bounded; + /// + /// let v = Bounded::::new::<7>(); + /// + /// // `7` can be represented using 3 unsigned bits... + /// let smaller_v = v.try_shrink::<3>(); + /// assert_eq!(smaller_v.as_deref().copied(), Some(7)); + /// + /// // ... but doesn't fit within `2` bits. + /// assert_eq!(v.try_shrink::<2>(), None); + /// ``` + pub fn try_shrink(self) -> Option> { + Bounded::::try_new(self.get()) + } + + /// Casts `self` into a [`Bounded`] backed by a different storage type, but using the same + /// number of valid bits. + /// + /// Both `T` and `U` must be of same signedness, and `U` must be at least as large as + /// `N` bits, or a build error will occur. + /// + /// # Examples + /// + /// ``` + /// use kernel::num::Bounded; + /// + /// let v = Bounded::::new::<127>(); + /// + /// let u16_v: Bounded = v.cast(); + /// assert_eq!(u16_v.get(), 127); + /// + /// // This won't build: a `u8` is smaller than the required 12 bits. + /// // let _: Bounded = v.cast(); + /// ``` + pub fn cast(self) -> Bounded + where + U: TryFrom + Integer, + T: Integer, + U: Integer, + { + // SAFETY: The converted value is represented using `N` bits, `U` can contain `N` bits, and + // `U` and `T` have the same sign, hence this conversion cannot fail. + let value = unsafe { U::try_from(self.get()).unwrap_unchecked() }; + + // INVARIANT: Although the backing type has changed, the value is still represented within + // `N` bits, and with the same signedness. + Bounded::__new(value) + } +} + +impl Deref for Bounded +where + T: Integer, +{ + type Target = T; + + fn deref(&self) -> &Self::Target { + // Enforce the invariant to inform the compiler of the bounds of the value. + if !fits_within(self.0, N) { + // SAFETY: Per the `Bounded` invariants, `fits_within` can never return `false` on the + // value of a valid instance. + unsafe { core::hint::unreachable_unchecked() } + } + + &self.0 + } +} + +/// Trait similar to [`TryInto`] but for [`Bounded`], to avoid conflicting implementations. +/// +/// # Examples +/// +/// ``` +/// use kernel::num::{Bounded, TryIntoBounded}; +/// +/// // Succeeds because `128` fits into 8 bits. +/// let v: Option> = 128u32.try_into_bitint(); +/// assert_eq!(v.as_deref().copied(), Some(128)); +/// +/// // Fails because `128` doesn't fits into 6 bits. +/// let v: Option> = 128u32.try_into_bitint(); +/// assert_eq!(v, None); +/// ``` +pub trait TryIntoBounded { + /// Attempts to convert `self` into a [`Bounded`] using `N` bits. + /// + /// Returns [`None`] if `self` does not fit into the target type. + fn try_into_bitint(self) -> Option>; +} + +/// Any integer value can be attempted to be converted into a [`Bounded`] of any size. +impl TryIntoBounded for U +where + T: Integer, + U: TryInto, +{ + fn try_into_bitint(self) -> Option> { + self.try_into().ok().and_then(Bounded::try_new) + } +} + +// Comparisons between `Bounded`s. + +impl PartialEq> for Bounded +where + T: Integer, + U: Integer, + T: PartialEq, +{ + fn eq(&self, other: &Bounded) -> bool { + self.get() == other.get() + } +} + +impl Eq for Bounded where T: Integer {} + +impl PartialOrd> for Bounded +where + T: Integer, + U: Integer, + T: PartialOrd, +{ + fn partial_cmp(&self, other: &Bounded) -> Option { + self.get().partial_cmp(&other.get()) + } +} + +impl Ord for Bounded +where + T: Integer, + T: Ord, +{ + fn cmp(&self, other: &Self) -> cmp::Ordering { + self.get().cmp(&other.get()) + } +} + +// Comparisons between a `Bounded` and its backing type. + +impl PartialEq for Bounded +where + T: Integer, + T: PartialEq, +{ + fn eq(&self, other: &T) -> bool { + self.get() == *other + } +} + +impl PartialOrd for Bounded +where + T: Integer, + T: PartialOrd, +{ + fn partial_cmp(&self, other: &T) -> Option { + self.get().partial_cmp(other) + } +} + +// Implementations of `core::ops` for two `Bounded` with the same backing type. + +impl ops::Add> for Bounded +where + T: Integer, + T: ops::Add, +{ + type Output = T; + + fn add(self, rhs: Bounded) -> Self::Output { + self.get() + rhs.get() + } +} + +impl ops::BitAnd> for Bounded +where + T: Integer, + T: ops::BitAnd, +{ + type Output = T; + + fn bitand(self, rhs: Bounded) -> Self::Output { + self.get() & rhs.get() + } +} + +impl ops::BitOr> for Bounded +where + T: Integer, + T: ops::BitOr, +{ + type Output = T; + + fn bitor(self, rhs: Bounded) -> Self::Output { + self.get() | rhs.get() + } +} + +impl ops::BitXor> for Bounded +where + T: Integer, + T: ops::BitXor, +{ + type Output = T; + + fn bitxor(self, rhs: Bounded) -> Self::Output { + self.get() ^ rhs.get() + } +} + +impl ops::Div> for Bounded +where + T: Integer, + T: ops::Div, +{ + type Output = T; + + fn div(self, rhs: Bounded) -> Self::Output { + self.get() / rhs.get() + } +} + +impl ops::Mul> for Bounded +where + T: Integer, + T: ops::Mul, +{ + type Output = T; + + fn mul(self, rhs: Bounded) -> Self::Output { + self.get() * rhs.get() + } +} + +impl ops::Rem> for Bounded +where + T: Integer, + T: ops::Rem, +{ + type Output = T; + + fn rem(self, rhs: Bounded) -> Self::Output { + self.get() % rhs.get() + } +} + +impl ops::Sub> for Bounded +where + T: Integer, + T: ops::Sub, +{ + type Output = T; + + fn sub(self, rhs: Bounded) -> Self::Output { + self.get() - rhs.get() + } +} + +// Implementations of `core::ops` between a `Bounded` and its backing type. + +impl ops::Add for Bounded +where + T: Integer, + T: ops::Add, +{ + type Output = T; + + fn add(self, rhs: T) -> Self::Output { + self.get() + rhs + } +} + +impl ops::BitAnd for Bounded +where + T: Integer, + T: ops::BitAnd, +{ + type Output = T; + + fn bitand(self, rhs: T) -> Self::Output { + self.get() & rhs + } +} + +impl ops::BitOr for Bounded +where + T: Integer, + T: ops::BitOr, +{ + type Output = T; + + fn bitor(self, rhs: T) -> Self::Output { + self.get() | rhs + } +} + +impl ops::BitXor for Bounded +where + T: Integer, + T: ops::BitXor, +{ + type Output = T; + + fn bitxor(self, rhs: T) -> Self::Output { + self.get() ^ rhs + } +} + +impl ops::Div for Bounded +where + T: Integer, + T: ops::Div, +{ + type Output = T; + + fn div(self, rhs: T) -> Self::Output { + self.get() / rhs + } +} + +impl ops::Mul for Bounded +where + T: Integer, + T: ops::Mul, +{ + type Output = T; + + fn mul(self, rhs: T) -> Self::Output { + self.get() * rhs + } +} + +impl ops::Neg for Bounded +where + T: Integer, + T: ops::Neg, +{ + type Output = T; + + fn neg(self) -> Self::Output { + -self.get() + } +} + +impl ops::Not for Bounded +where + T: Integer, + T: ops::Not, +{ + type Output = T; + + fn not(self) -> Self::Output { + !self.get() + } +} + +impl ops::Rem for Bounded +where + T: Integer, + T: ops::Rem, +{ + type Output = T; + + fn rem(self, rhs: T) -> Self::Output { + self.get() % rhs + } +} + +impl ops::Sub for Bounded +where + T: Integer, + T: ops::Sub, +{ + type Output = T; + + fn sub(self, rhs: T) -> Self::Output { + self.get() - rhs + } +} + +// Proxy implementations of `core::fmt`. + +impl fmt::Display for Bounded +where + T: Integer, + T: fmt::Display, +{ + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + self.get().fmt(f) + } +} + +impl fmt::Binary for Bounded +where + T: Integer, + T: fmt::Binary, +{ + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + self.get().fmt(f) + } +} + +impl fmt::LowerExp for Bounded +where + T: Integer, + T: fmt::LowerExp, +{ + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + self.get().fmt(f) + } +} + +impl fmt::LowerHex for Bounded +where + T: Integer, + T: fmt::LowerHex, +{ + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + self.get().fmt(f) + } +} + +impl fmt::Octal for Bounded +where + T: Integer, + T: fmt::Octal, +{ + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + self.get().fmt(f) + } +} + +impl fmt::UpperExp for Bounded +where + T: Integer, + T: fmt::UpperExp, +{ + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + self.get().fmt(f) + } +} + +impl fmt::UpperHex for Bounded +where + T: Integer, + T: fmt::UpperHex, +{ + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + self.get().fmt(f) + } +} + +/// Implements `$trait` for all [`Bounded`] types represented using `$num_bits`. +/// +/// This is used to declare size properties as traits that we can constrain against in impl blocks. +macro_rules! impl_size_rule { + ($trait:ty, $($num_bits:literal)*) => { + $( + impl $trait for Bounded where T: Integer {} + )* + }; +} + +/// Local trait expressing the fact that a given [`Bounded`] has at least `N` bits used for value +/// representation. +trait AtLeastXBits {} + +/// Implementations for infallibly converting a primitive type into a [`Bounded`] that can contain +/// it. +/// +/// Put into their own module for readability, and to avoid cluttering the rustdoc of the parent +/// module. +mod atleast_impls { + use super::*; + + // Number of bits at least as large as 64. + impl_size_rule!(AtLeastXBits<64>, 64); + + // Anything 64 bits or more is also larger than 32. + impl AtLeastXBits<32> for T where T: AtLeastXBits<64> {} + // Other numbers of bits at least as large as 32. + impl_size_rule!(AtLeastXBits<32>, + 32 33 34 35 36 37 38 39 + 40 41 42 43 44 45 46 47 + 48 49 50 51 52 53 54 55 + 56 57 58 59 60 61 62 63 + ); + + // Anything 32 bits or more is also larger than 16. + impl AtLeastXBits<16> for T where T: AtLeastXBits<32> {} + // Other numbers of bits at least as large as 16. + impl_size_rule!(AtLeastXBits<16>, + 16 17 18 19 20 21 22 23 + 24 25 26 27 28 29 30 31 + ); + + // Anything 16 bits or more is also larger than 8. + impl AtLeastXBits<8> for T where T: AtLeastXBits<16> {} + // Other numbers of bits at least as large as 8. + impl_size_rule!(AtLeastXBits<8>, 8 9 10 11 12 13 14 15); +} + +/// Generates `From` implementations from a primitive type into a [`Bounded`] with +/// enough bits to store any value of that type. +/// +/// Note: The only reason for having this macro is that if we pass `$type` as a generic +/// parameter, we cannot use it in the const context of [`AtLeastXBits`]'s generic parameter. This +/// can be fixed once the `generic_const_exprs` feature is usable, and this macro replaced by a +/// regular `impl` block. +macro_rules! impl_from_primitive { + ($($type:ty)*) => { + $( + #[doc = ::core::concat!( + "Conversion from a [`", + ::core::stringify!($type), + "`] into a [`Bounded`] of same signedness with enough bits to store it.")] + impl From<$type> for Bounded + where + $type: Integer, + T: Integer::Signedness> + From<$type>, + Self: AtLeastXBits<{ <$type as Integer>::BITS as usize }>, + { + fn from(value: $type) -> Self { + // INVARIANT: The trait bound on `Self` guarantees that `N` bits is + // enough to hold any value of the source type. + Self::__new(T::from(value)) + } + } + )* + } +} + +impl_from_primitive!( + u8 u16 u32 u64 usize + i8 i16 i32 i64 isize +); + +/// Local trait expressing the fact that a given [`Bounded`] fits into a primitive type of `N` bits, +/// provided they have the same signedness. +trait FitsInXBits {} + +/// Implementations for infallibly converting a [`Bounded`] into a primitive type that can contain +/// it. +/// +/// Put into their own module for readability, and to avoid cluttering the rustdoc of the parent +/// module. +mod fits_impls { + use super::*; + + // Number of bits that fit into a 8-bits primitive. + impl_size_rule!(FitsInXBits<8>, 1 2 3 4 5 6 7 8); + + // Anything that fits into 8 bits also fits into 16. + impl FitsInXBits<16> for T where T: FitsInXBits<8> {} + // Other number of bits that fit into a 16-bits primitive. + impl_size_rule!(FitsInXBits<16>, 9 10 11 12 13 14 15 16); + + // Anything that fits into 16 bits also fits into 32. + impl FitsInXBits<32> for T where T: FitsInXBits<16> {} + // Other number of bits that fit into a 32-bits primitive. + impl_size_rule!(FitsInXBits<32>, + 17 18 19 20 21 22 23 24 + 25 26 27 28 29 30 31 32 + ); + + // Anything that fits into 32 bits also fits into 64. + impl FitsInXBits<64> for T where T: FitsInXBits<32> {} + // Other number of bits that fit into a 64-bits primitive. + impl_size_rule!(FitsInXBits<64>, + 33 34 35 36 37 38 39 40 + 41 42 43 44 45 46 47 48 + 49 50 51 52 53 54 55 56 + 57 58 59 60 61 62 63 64 + ); +} + +/// Generates [`From`] implementations from a [`Bounded`] into a primitive type that is +/// guaranteed to contain it. +/// +/// Note: The only reason for having this macro is that if we pass `$type` as a generic +/// parameter, we cannot use it in the const context of `AtLeastXBits`'s generic parameter. This +/// can be fixed once the `generic_const_exprs` feature is usable, and this macro replaced by a +/// regular `impl` block. +macro_rules! impl_into_primitive { + ($($type:ty)*) => { + $( + #[doc = ::core::concat!( + "Conversion from a [`Bounded`] with no more bits than a [`", + ::core::stringify!($type), + "`] and of same signedness into [`", + ::core::stringify!($type), + "`]")] + impl From> for $type + where + $type: Integer + TryFrom, + T: Integer::Signedness>, + Bounded: FitsInXBits<{ <$type as Integer>::BITS as usize }>, + { + fn from(value: Bounded) -> $type { + // SAFETY: The trait bound on `Bounded` ensures that any value it holds (which + // is constrained to `N` bits) can fit into the destination type, so this + // conversion cannot fail. + unsafe { <$type>::try_from(value.get()).unwrap_unchecked() } + } + } + )* + } +} + +impl_into_primitive!( + u8 u16 u32 u64 usize + i8 i16 i32 i64 isize +); + +// Single-bit `Bounded`s can be converted from/to a boolean. + +impl From> for bool +where + T: Integer + Zeroable, +{ + fn from(value: Bounded) -> Self { + value.get() != Zeroable::zeroed() + } +} + +impl From for Bounded +where + T: Integer + From, +{ + fn from(value: bool) -> Self { + // INVARIANT: A boolean can be represented using a single bit, and thus fits within any + // integer type for any `N` > 0. + Self::__new(T::from(value)) + } +} From 1aaa5cfbd489e02d18973c7721a00613de544a63 Mon Sep 17 00:00:00 2001 From: Alexandre Courbot Date: Sat, 8 Nov 2025 11:23:49 +0900 Subject: [PATCH 26/53] MAINTAINERS: add entry for the Rust `num` module This new module provides numerical features useful for the kernel, such as the `Integer` trait and the `Bounded` integer wrapper. Signed-off-by: Alexandre Courbot Acked-by: Yury Norov (NVIDIA) Reviewed-by: Alice Ryhl Link: https://patch.msgid.link/20251108-bounded_ints-v4-3-c9342ac7ebd1@nvidia.com Signed-off-by: Miguel Ojeda --- MAINTAINERS | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/MAINTAINERS b/MAINTAINERS index 545a4776795e..9a892c08edab 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -22520,6 +22520,14 @@ T: git https://github.com/Rust-for-Linux/linux.git alloc-next F: rust/kernel/alloc.rs F: rust/kernel/alloc/ +RUST [NUM] +M: Alexandre Courbot +R: Yury Norov +L: rust-for-linux@vger.kernel.org +S: Maintained +F: rust/kernel/num.rs +F: rust/kernel/num/ + RUST [PIN-INIT] M: Benno Lossin L: rust-for-linux@vger.kernel.org From 53870c679e4933d11d9488db61caa3e670621528 Mon Sep 17 00:00:00 2001 From: Benno Lossin Date: Thu, 16 Oct 2025 23:17:38 +0200 Subject: [PATCH 27/53] rust: pin-init: fix broken rust doc link MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Rust 1.92.0 warns when building the documentation that [`PinnedDrop`] is an invalid reference. This is correct and it's weird that it didn't warn before, so fix the link. [ The reason is that it is hidden -- I had asked about that in the upstream PR that changed the behavior because I wasn't sure it was intentional (and thus whether we needed to fix this and other cases): https://github.com/rust-lang/rust/pull/147153#issuecomment-3395484636 It turns out it was not, and it has been fixed for 1.92.0's upcoming release thanks to Guillaume and León. So we do not strictly need this patch and the other changes anymore: https://github.com/rust-lang/rust/pull/147809 However, checking hidden/private items or, even better, a runtime toggle to be able to see those on the fly, is something that I think would be quite nice so I have had it in our usual lists for a while. Guillaume is open to the idea and perhaps experimenting with an implementation on our side first -- he asked me to open issues upstream: https://github.com/rust-lang/rust/issues/149105 https://github.com/rust-lang/rust/issues/149106 - Miguel ] Signed-off-by: Benno Lossin Link: https://patch.msgid.link/20251016211740.653599-1-lossin@kernel.org Signed-off-by: Miguel Ojeda --- rust/pin-init/src/macros.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/rust/pin-init/src/macros.rs b/rust/pin-init/src/macros.rs index d6acf2cd291e..682c61a587a0 100644 --- a/rust/pin-init/src/macros.rs +++ b/rust/pin-init/src/macros.rs @@ -506,6 +506,8 @@ pub use ::paste::paste; /// Creates a `unsafe impl<...> PinnedDrop for $type` block. /// /// See [`PinnedDrop`] for more information. +/// +/// [`PinnedDrop`]: crate::PinnedDrop #[doc(hidden)] #[macro_export] macro_rules! __pinned_drop { From 26866b6bb10b8f71e64f2f7a564e52a540c98335 Mon Sep 17 00:00:00 2001 From: Brian Harring Date: Thu, 16 Oct 2025 23:17:39 +0200 Subject: [PATCH 28/53] rust: pin-init: fix typo in docs Signed-off-by: Brian Harring Signed-off-by: Benno Lossin Link: https://patch.msgid.link/20251016211740.653599-2-lossin@kernel.org Signed-off-by: Miguel Ojeda --- rust/pin-init/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rust/pin-init/README.md b/rust/pin-init/README.md index 723e275445d4..74bbb4e0a2f7 100644 --- a/rust/pin-init/README.md +++ b/rust/pin-init/README.md @@ -9,7 +9,7 @@ > [!NOTE] > > This crate was originally named [`pinned-init`], but the migration to -> `pin-init` is not yet complete. The `legcay` branch contains the current +> `pin-init` is not yet complete. The `legacy` branch contains the current > version of the `pinned-init` crate & the `main` branch already incorporates > the rename to `pin-init`. > From 494de8f67b1e586b0190eb7f835e97c97f6b81b1 Mon Sep 17 00:00:00 2001 From: Tamir Duberstein Date: Mon, 17 Nov 2025 11:38:07 -0500 Subject: [PATCH 29/53] rust: sync: replace `kernel::c_str!` with C-Strings C-String literals were added in Rust 1.77. Replace instances of `kernel::c_str!` with C-String literals where possible. Acked-by: Greg Kroah-Hartman Reviewed-by: Alice Ryhl Reviewed-by: Benno Lossin Signed-off-by: Tamir Duberstein Link: https://patch.msgid.link/20251117-core-cstr-cstrings-v4-1-924886ad9f75@gmail.com Signed-off-by: Miguel Ojeda --- rust/kernel/sync.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/rust/kernel/sync.rs b/rust/kernel/sync.rs index cf5b638a097d..c94753d6413e 100644 --- a/rust/kernel/sync.rs +++ b/rust/kernel/sync.rs @@ -48,7 +48,6 @@ impl LockClassKey { /// /// # Examples /// ``` - /// # use kernel::c_str; /// # use kernel::alloc::KBox; /// # use kernel::types::ForeignOwnable; /// # use kernel::sync::{LockClassKey, SpinLock}; @@ -60,7 +59,7 @@ impl LockClassKey { /// { /// stack_pin_init!(let num: SpinLock = SpinLock::new( /// 0, - /// c_str!("my_spinlock"), + /// c"my_spinlock", /// // SAFETY: `key_ptr` is returned by the above `into_foreign()`, whose /// // `from_foreign()` has not yet been called. /// unsafe { > as ForeignOwnable>::borrow(key_ptr) } From bc197e24a3acd13dd0b7b07c1448c5c225946546 Mon Sep 17 00:00:00 2001 From: Alexandre Courbot Date: Sat, 22 Nov 2025 13:00:49 +0900 Subject: [PATCH 30/53] rust: num: bounded: Always inline fits_within and from_expr `from_expr` relies on `build_assert` to infer that the passed expression fits the type's boundaries at build time. That inference can only be successful its code (and that of `fits_within`, which performs the check) is inlined, as a dedicated function would need to work with a variable and cannot verify that property. While inlining happens as expected in most cases, it is not guaranteed. In particular, kernel options that optimize for size like `CONFIG_CC_OPTIMIZE_FOR_SIZE` can result in `from_expr` not being inlined. Add `#[inline(always)]` attributes to both `fits_within` and `from_expr` to make the compiler inline these functions more aggressively, as it does not make sense to use them non-inlined anyway. [ For reference, the errors look like: ld.lld: error: undefined symbol: rust_build_error >>> referenced by build_assert.rs:83 (rust/kernel/build_assert.rs:83) >>> rust/doctests_kernel_generated.o:(>::from_expr) in archive vmlinux.a - Miguel ] Fixes: 01e345e82ec3 ("rust: num: add Bounded integer wrapping type") Reported-by: kernel test robot Closes: https://lore.kernel.org/oe-kbuild-all/202511210055.RUsFNku1-lkp@intel.com/ Suggested-by: Gary Guo Signed-off-by: Alexandre Courbot Link: https://patch.msgid.link/20251122-bounded_ints_fix-v1-1-1e07589d4955@nvidia.com Signed-off-by: Miguel Ojeda --- rust/kernel/num/bounded.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/rust/kernel/num/bounded.rs b/rust/kernel/num/bounded.rs index 18e45bf6f84b..92c41b2eb760 100644 --- a/rust/kernel/num/bounded.rs +++ b/rust/kernel/num/bounded.rs @@ -35,6 +35,7 @@ macro_rules! fits_within { } /// Returns `true` if `value` can be represented with at most `N` bits in a `T`. +#[inline(always)] fn fits_within(value: T, num_bits: u32) -> bool { fits_within!(value, T, num_bits) } @@ -362,6 +363,7 @@ where /// assert_eq!(Bounded::::from_expr(1).get(), 1); /// assert_eq!(Bounded::::from_expr(0xff).get(), 0xff); /// ``` + #[inline(always)] pub fn from_expr(expr: T) -> Self { crate::build_assert!( fits_within(expr, N), From 841f31d298693e82aaa9e7ac09f5fa9c8dcfdbc3 Mon Sep 17 00:00:00 2001 From: Alexandre Courbot Date: Mon, 24 Nov 2025 22:50:02 +0900 Subject: [PATCH 31/53] rust: num: bounded: rename `try_into_bitint` to `try_into_bounded` This is a remnant from when `Bounded` was called `BitInt` which I didn't rename. Fix this. Fixes: 01e345e82ec3 ("rust: num: add Bounded integer wrapping type") Signed-off-by: Alexandre Courbot Link: https://patch.msgid.link/20251124-bounded_fix-v1-1-d8e34e1c727f@nvidia.com Signed-off-by: Miguel Ojeda --- rust/kernel/num/bounded.rs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/rust/kernel/num/bounded.rs b/rust/kernel/num/bounded.rs index 92c41b2eb760..f870080af8ac 100644 --- a/rust/kernel/num/bounded.rs +++ b/rust/kernel/num/bounded.rs @@ -218,11 +218,11 @@ fn fits_within(value: T, num_bits: u32) -> bool { /// use kernel::num::{Bounded, TryIntoBounded}; /// /// // Succeeds because `128` fits into 8 bits. -/// let v: Option> = 128u32.try_into_bitint(); +/// let v: Option> = 128u32.try_into_bounded(); /// assert_eq!(v.as_deref().copied(), Some(128)); /// /// // Fails because `128` doesn't fits into 6 bits. -/// let v: Option> = 128u32.try_into_bitint(); +/// let v: Option> = 128u32.try_into_bounded(); /// assert_eq!(v, None); /// ``` #[repr(transparent)] @@ -498,18 +498,18 @@ where /// use kernel::num::{Bounded, TryIntoBounded}; /// /// // Succeeds because `128` fits into 8 bits. -/// let v: Option> = 128u32.try_into_bitint(); +/// let v: Option> = 128u32.try_into_bounded(); /// assert_eq!(v.as_deref().copied(), Some(128)); /// /// // Fails because `128` doesn't fits into 6 bits. -/// let v: Option> = 128u32.try_into_bitint(); +/// let v: Option> = 128u32.try_into_bounded(); /// assert_eq!(v, None); /// ``` pub trait TryIntoBounded { /// Attempts to convert `self` into a [`Bounded`] using `N` bits. /// /// Returns [`None`] if `self` does not fit into the target type. - fn try_into_bitint(self) -> Option>; + fn try_into_bounded(self) -> Option>; } /// Any integer value can be attempted to be converted into a [`Bounded`] of any size. @@ -518,7 +518,7 @@ where T: Integer, U: TryInto, { - fn try_into_bitint(self) -> Option> { + fn try_into_bounded(self) -> Option> { self.try_into().ok().and_then(Bounded::try_new) } } From f3f9f42232dee596d15491ca3f611d02174db49c Mon Sep 17 00:00:00 2001 From: Zheng Yejian Date: Fri, 11 Oct 2024 22:38:53 +0800 Subject: [PATCH 32/53] kallsyms: Fix wrong "big" kernel symbol type read from procfs Currently when the length of a symbol is longer than 0x7f characters, its type shown in /proc/kallsyms can be incorrect. I found this issue when reading the code, but it can be reproduced by following steps: 1. Define a function which symbol length is 130 characters: #define X13(x) x##x##x##x##x##x##x##x##x##x##x##x##x static noinline void X13(x123456789)(void) { printk("hello world\n"); } 2. The type in vmlinux is 't': $ nm vmlinux | grep x123456 ffffffff816290f0 t x123456789x123456789x123456789x12[...] 3. Then boot the kernel, the type shown in /proc/kallsyms becomes 'g' instead of the expected 't': # cat /proc/kallsyms | grep x123456 ffffffff816290f0 g x123456789x123456789x123456789x12[...] The root cause is that, after commit 73bbb94466fd ("kallsyms: support "big" kernel symbols"), ULEB128 was used to encode symbol name length. That is, for "big" kernel symbols of which name length is longer than 0x7f characters, the length info is encoded into 2 bytes. kallsyms_get_symbol_type() expects to read the first char of the symbol name which indicates the symbol type. However, due to the "big" symbol case not being handled, the symbol type read from /proc/kallsyms may be wrong, so handle it properly. Cc: stable@vger.kernel.org Fixes: 73bbb94466fd ("kallsyms: support "big" kernel symbols") Signed-off-by: Zheng Yejian Acked-by: Gary Guo Link: https://patch.msgid.link/20241011143853.3022643-1-zhengyejian@huaweicloud.com Signed-off-by: Miguel Ojeda --- kernel/kallsyms.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/kernel/kallsyms.c b/kernel/kallsyms.c index 1e7635864124..049e296f586c 100644 --- a/kernel/kallsyms.c +++ b/kernel/kallsyms.c @@ -103,8 +103,11 @@ static char kallsyms_get_symbol_type(unsigned int off) { /* * Get just the first code, look it up in the token table, - * and return the first char from this token. + * and return the first char from this token. If MSB of length + * is 1, it is a "big" symbol, so needs an additional byte. */ + if (kallsyms_names[off] & 0x80) + off++; return kallsyms_token_table[kallsyms_token_index[kallsyms_names[off + 1]]]; } From 7a0eae4d43d265c56e9d0b136ec08e35b83525b8 Mon Sep 17 00:00:00 2001 From: Alex Gaynor Date: Sun, 26 Oct 2025 10:29:52 -0400 Subject: [PATCH 33/53] MAINTAINERS: Remove Alex Gaynor as Rust maintainer I've long since stopped having the time to contribute code or reviews, this acknowledges that. Geoffrey Thomas and I created the "linux-kernel-module-rust" project at PyCon in 2018, as an experiment to see if we could make it possible to write kernel modules in Rust. The Rust for Linux effort has far exceeded anything we could have expected at the time. I want to thank all the Rust for Linux contributors, past and present, who have helped make this a reality -- and in particularly Miguel, who really transformed this project from an interesting demo to something that could really land in mainline. Signed-off-by: Alex Gaynor Link: https://patch.msgid.link/CAFRnB2U1uvg1vyZe1kDi7L3P4kTFowfOo6Hfo9WJED4qve4ZZw@mail.gmail.com [ Reflowed. - Miguel ] Signed-off-by: Miguel Ojeda --- MAINTAINERS | 1 - 1 file changed, 1 deletion(-) diff --git a/MAINTAINERS b/MAINTAINERS index 9a892c08edab..acd2b0540b20 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -22483,7 +22483,6 @@ F: tools/verification/ RUST M: Miguel Ojeda -M: Alex Gaynor R: Boqun Feng R: Gary Guo R: Björn Roy Baron From 46e58a9637ec6493b142b788595eed627b4973b7 Mon Sep 17 00:00:00 2001 From: Miguel Ojeda Date: Mon, 24 Nov 2025 16:18:13 +0100 Subject: [PATCH 34/53] rust: kbuild: introduce `core-flags` and `core-skip_flags` In the next commits we are introducing `*-{cfgs,skip_flags,flags}` variables for other crates. Thus do so here for `core`, which simplifies a bit the `Makefile` (including the next commit) and makes it more consistent. This means we stop passing `-Wrustdoc::unescaped_backticks` to `rustc` and `-Wunreachable_pub` to `rustdoc`, i.e. we skip more, which is fine since it shouldn't have an effect. In addition, use `:=` for `core-cfgs` to make it consistent with the upcoming additions. Reviewed-by: Alice Ryhl Reviewed-by: Gary Guo Tested-by: Gary Guo Tested-by: Jesung Yang Link: https://patch.msgid.link/20251124151837.2184382-2-ojeda@kernel.org Signed-off-by: Miguel Ojeda --- rust/Makefile | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/rust/Makefile b/rust/Makefile index 23c7ae905bd2..ce1853a09d3d 100644 --- a/rust/Makefile +++ b/rust/Makefile @@ -60,11 +60,20 @@ rustdoc_test_quiet=--test-args -q rustdoc_test_kernel_quiet=>/dev/null endif -core-cfgs = \ +core-cfgs := \ --cfg no_fp_fmt_parse core-edition := $(if $(call rustc-min-version,108700),2024,2021) +core-skip_flags := \ + --edition=2021 \ + -Wunreachable_pub \ + -Wrustdoc::unescaped_backticks + +core-flags := \ + --edition=$(core-edition) \ + $(core-cfgs) + # `rustdoc` did not save the target modifiers, thus workaround for # the time being (https://github.com/rust-lang/rust/issues/144521). rustdoc_modifiers_workaround := $(if $(call rustc-min-version,108800),-Cunsafe-allow-abi-mismatch=fixed-x18) @@ -122,8 +131,8 @@ rustdoc-macros: $(src)/macros/lib.rs rustdoc-clean FORCE # Starting with Rust 1.82.0, skipping `-Wrustdoc::unescaped_backticks` should # not be needed -- see https://github.com/rust-lang/rust/pull/128307. -rustdoc-core: private skip_flags = --edition=2021 -Wrustdoc::unescaped_backticks -rustdoc-core: private rustc_target_flags = --edition=$(core-edition) $(core-cfgs) +rustdoc-core: private skip_flags = $(core-skip_flags) +rustdoc-core: private rustc_target_flags = $(core-flags) rustdoc-core: $(RUST_LIB_SRC)/core/src/lib.rs rustdoc-clean FORCE +$(call if_changed,rustdoc) @@ -499,9 +508,9 @@ $(obj)/helpers/helpers.o: $(src)/helpers/helpers.c $(recordmcount_source) FORCE $(obj)/exports.o: private skip_gendwarfksyms = 1 $(obj)/core.o: private skip_clippy = 1 -$(obj)/core.o: private skip_flags = --edition=2021 -Wunreachable_pub +$(obj)/core.o: private skip_flags = $(core-skip_flags) $(obj)/core.o: private rustc_objcopy = $(foreach sym,$(redirect-intrinsics),--redefine-sym $(sym)=__rust$(sym)) -$(obj)/core.o: private rustc_target_flags = --edition=$(core-edition) $(core-cfgs) +$(obj)/core.o: private rustc_target_flags = $(core-flags) $(obj)/core.o: $(RUST_LIB_SRC)/core/src/lib.rs \ $(wildcard $(objtree)/include/config/RUSTC_VERSION_TEXT) FORCE +$(call if_changed_rule,rustc_library) From 1181c974421818ff7318e3a211c87b5dd437c13e Mon Sep 17 00:00:00 2001 From: Miguel Ojeda Date: Mon, 24 Nov 2025 16:18:14 +0100 Subject: [PATCH 35/53] rust: kbuild: simplify `--cfg` handling We need to handle `cfg`s in both `rustc` and `rust-analyzer`, and in future commits some of those contain double quotes, which complicates things further. Thus, instead of removing the `--cfg ` part in the rust-analyzer generation script, have the `*-cfgs` variables contain just the actual `cfg`, and use that to generate the actual flags in `*-flags`. Reviewed-by: Alice Ryhl Reviewed-by: Gary Guo Tested-by: Gary Guo Tested-by: Jesung Yang Link: https://patch.msgid.link/20251124151837.2184382-3-ojeda@kernel.org Signed-off-by: Miguel Ojeda --- rust/Makefile | 6 ++++-- scripts/generate_rust_analyzer.py | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/rust/Makefile b/rust/Makefile index ce1853a09d3d..9967f3457d44 100644 --- a/rust/Makefile +++ b/rust/Makefile @@ -60,8 +60,10 @@ rustdoc_test_quiet=--test-args -q rustdoc_test_kernel_quiet=>/dev/null endif +cfgs-to-flags = $(patsubst %,--cfg='%',$1) + core-cfgs := \ - --cfg no_fp_fmt_parse + no_fp_fmt_parse core-edition := $(if $(call rustc-min-version,108700),2024,2021) @@ -72,7 +74,7 @@ core-skip_flags := \ core-flags := \ --edition=$(core-edition) \ - $(core-cfgs) + $(call cfgs-to-flags,$(core-cfgs)) # `rustdoc` did not save the target modifiers, thus workaround for # the time being (https://github.com/rust-lang/rust/issues/144521). diff --git a/scripts/generate_rust_analyzer.py b/scripts/generate_rust_analyzer.py index fc27f0cca752..dedca470adc1 100755 --- a/scripts/generate_rust_analyzer.py +++ b/scripts/generate_rust_analyzer.py @@ -15,7 +15,7 @@ def args_crates_cfgs(cfgs): crates_cfgs = {} for cfg in cfgs: crate, vals = cfg.split("=", 1) - crates_cfgs[crate] = vals.replace("--cfg", "").split() + crates_cfgs[crate] = vals.split() return crates_cfgs From 7dbe46c0b11dded16cf9c5f85c1a3f260422cee5 Mon Sep 17 00:00:00 2001 From: Miguel Ojeda Date: Mon, 24 Nov 2025 16:18:15 +0100 Subject: [PATCH 36/53] rust: kbuild: add proc macro library support Add the proc macro library rule that produces `.rlib` files to be used by proc macros such as the `macros` crate. Reviewed-by: Gary Guo Tested-by: Gary Guo Tested-by: Jesung Yang Link: https://patch.msgid.link/20251124151837.2184382-4-ojeda@kernel.org Signed-off-by: Miguel Ojeda --- .gitignore | 1 + rust/Makefile | 10 ++++++++++ 2 files changed, 11 insertions(+) diff --git a/.gitignore b/.gitignore index 86a1ba0d9035..3a7241c941f5 100644 --- a/.gitignore +++ b/.gitignore @@ -41,6 +41,7 @@ *.o.* *.patch *.pyc +*.rlib *.rmeta *.rpm *.rsi diff --git a/rust/Makefile b/rust/Makefile index 9967f3457d44..45ef84513b6c 100644 --- a/rust/Makefile +++ b/rust/Makefile @@ -421,6 +421,16 @@ $(obj)/exports_bindings_generated.h: $(obj)/bindings.o FORCE $(obj)/exports_kernel_generated.h: $(obj)/kernel.o FORCE $(call if_changed,exports) +quiet_cmd_rustc_procmacrolibrary = $(RUSTC_OR_CLIPPY_QUIET) PL $@ + cmd_rustc_procmacrolibrary = \ + $(if $(skip_clippy),$(RUSTC),$(RUSTC_OR_CLIPPY)) \ + $(filter-out $(skip_flags),$(rust_common_flags) $(rustc_target_flags)) \ + --emit=dep-info,link --crate-type rlib -O \ + --out-dir $(objtree)/$(obj) -L$(objtree)/$(obj) \ + --crate-name $(patsubst lib%.rlib,%,$(notdir $@)) $<; \ + mv $(objtree)/$(obj)/$(patsubst lib%.rlib,%,$(notdir $@)).d $(depfile); \ + sed -i '/^\#/d' $(depfile) + quiet_cmd_rustc_procmacro = $(RUSTC_OR_CLIPPY_QUIET) P $@ cmd_rustc_procmacro = \ $(RUSTC_OR_CLIPPY) $(rust_common_flags) $(rustc_target_flags) \ From d4e7307b1f3535c19ff6ca24d3e4aec32cebb120 Mon Sep 17 00:00:00 2001 From: Miguel Ojeda Date: Mon, 24 Nov 2025 16:18:16 +0100 Subject: [PATCH 37/53] rust: kbuild: support skipping flags in `rustc_test_library` Crates like `quote` (added later) will need the ability to skip flags in the `rustc_test_library` command. Thus add the support for it. Reviewed-by: Alice Ryhl Reviewed-by: Gary Guo Tested-by: Gary Guo Tested-by: Jesung Yang Link: https://patch.msgid.link/20251124151837.2184382-5-ojeda@kernel.org Signed-off-by: Miguel Ojeda --- rust/Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rust/Makefile b/rust/Makefile index 45ef84513b6c..6fdab341fc48 100644 --- a/rust/Makefile +++ b/rust/Makefile @@ -172,8 +172,8 @@ rustdoc-clean: FORCE quiet_cmd_rustc_test_library = $(RUSTC_OR_CLIPPY_QUIET) TL $< cmd_rustc_test_library = \ OBJTREE=$(abspath $(objtree)) \ - $(RUSTC_OR_CLIPPY) $(rust_common_flags) \ - @$(objtree)/include/generated/rustc_cfg $(rustc_target_flags) \ + $(RUSTC_OR_CLIPPY) $(filter-out $(skip_flags),$(rust_common_flags) $(rustc_target_flags)) \ + @$(objtree)/include/generated/rustc_cfg \ --crate-type $(if $(rustc_test_library_proc),proc-macro,rlib) \ --out-dir $(objtree)/$(obj)/test --cfg testlib \ -L$(objtree)/$(obj)/test \ From c46b34f1d484c11c2a0cbd663168c2213175ca5c Mon Sep 17 00:00:00 2001 From: Miguel Ojeda Date: Mon, 24 Nov 2025 16:18:17 +0100 Subject: [PATCH 38/53] rust: kbuild: support using libraries in `rustc_procmacro` Proc macros such as `macros` and `pin-init` will need the ability to use libraries such as `syn` (added later) in the `rustc_procmacro` command. Thus add the support for it. Reviewed-by: Alice Ryhl Reviewed-by: Gary Guo Tested-by: Gary Guo Tested-by: Jesung Yang Link: https://patch.msgid.link/20251124151837.2184382-6-ojeda@kernel.org Signed-off-by: Miguel Ojeda --- rust/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rust/Makefile b/rust/Makefile index 6fdab341fc48..0288ca8d270c 100644 --- a/rust/Makefile +++ b/rust/Makefile @@ -437,7 +437,7 @@ quiet_cmd_rustc_procmacro = $(RUSTC_OR_CLIPPY_QUIET) P $@ -Clinker-flavor=gcc -Clinker=$(HOSTCC) \ -Clink-args='$(call escsq,$(KBUILD_PROCMACROLDFLAGS))' \ --emit=dep-info=$(depfile) --emit=link=$@ --extern proc_macro \ - --crate-type proc-macro \ + --crate-type proc-macro -L$(objtree)/$(obj) \ --crate-name $(patsubst lib%.$(libmacros_extension),%,$(notdir $@)) \ @$(objtree)/include/generated/rustc_cfg $< From 3a8b546a2786e54fbfff4d368ae45e65e1e43d21 Mon Sep 17 00:00:00 2001 From: Miguel Ojeda Date: Mon, 24 Nov 2025 16:18:18 +0100 Subject: [PATCH 39/53] rust: proc-macro2: import crate This is a subset of the Rust `proc-macro2` crate, version 1.0.101 (released 2025-08-16), licensed under "Apache-2.0 OR MIT", from: https://github.com/dtolnay/proc-macro2/raw/1.0.101/src The files are copied as-is, with no modifications whatsoever (not even adding the SPDX identifiers). For copyright details, please see: https://github.com/dtolnay/proc-macro2/blob/1.0.101/README.md#license https://github.com/dtolnay/proc-macro2/blob/1.0.101/LICENSE-APACHE https://github.com/dtolnay/proc-macro2/blob/1.0.101/LICENSE-MIT The next two patches modify these files as needed for use within the kernel. This patch split allows reviewers to double-check the import and to clearly see the differences introduced. The following script may be used to verify the contents: for path in $(cd rust/proc-macro2/ && find . -type f -name '*.rs'); do curl --silent --show-error --location \ https://github.com/dtolnay/proc-macro2/raw/1.0.101/src/$path \ | diff --unified rust/proc-macro2/$path - && echo $path: OK done Reviewed-by: Gary Guo Tested-by: Gary Guo Tested-by: Jesung Yang Link: https://patch.msgid.link/20251124151837.2184382-7-ojeda@kernel.org Signed-off-by: Miguel Ojeda --- rust/proc-macro2/detection.rs | 75 + rust/proc-macro2/extra.rs | 151 ++ rust/proc-macro2/fallback.rs | 1256 +++++++++++++++ rust/proc-macro2/lib.rs | 1349 +++++++++++++++++ rust/proc-macro2/location.rs | 29 + rust/proc-macro2/marker.rs | 17 + rust/proc-macro2/parse.rs | 995 ++++++++++++ rust/proc-macro2/probe.rs | 10 + rust/proc-macro2/probe/proc_macro_span.rs | 51 + .../proc-macro2/probe/proc_macro_span_file.rs | 14 + .../probe/proc_macro_span_location.rs | 21 + rust/proc-macro2/rcvec.rs | 146 ++ rust/proc-macro2/wrapper.rs | 984 ++++++++++++ 13 files changed, 5098 insertions(+) create mode 100644 rust/proc-macro2/detection.rs create mode 100644 rust/proc-macro2/extra.rs create mode 100644 rust/proc-macro2/fallback.rs create mode 100644 rust/proc-macro2/lib.rs create mode 100644 rust/proc-macro2/location.rs create mode 100644 rust/proc-macro2/marker.rs create mode 100644 rust/proc-macro2/parse.rs create mode 100644 rust/proc-macro2/probe.rs create mode 100644 rust/proc-macro2/probe/proc_macro_span.rs create mode 100644 rust/proc-macro2/probe/proc_macro_span_file.rs create mode 100644 rust/proc-macro2/probe/proc_macro_span_location.rs create mode 100644 rust/proc-macro2/rcvec.rs create mode 100644 rust/proc-macro2/wrapper.rs diff --git a/rust/proc-macro2/detection.rs b/rust/proc-macro2/detection.rs new file mode 100644 index 000000000000..beba7b237395 --- /dev/null +++ b/rust/proc-macro2/detection.rs @@ -0,0 +1,75 @@ +use core::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::Once; + +static WORKS: AtomicUsize = AtomicUsize::new(0); +static INIT: Once = Once::new(); + +pub(crate) fn inside_proc_macro() -> bool { + match WORKS.load(Ordering::Relaxed) { + 1 => return false, + 2 => return true, + _ => {} + } + + INIT.call_once(initialize); + inside_proc_macro() +} + +pub(crate) fn force_fallback() { + WORKS.store(1, Ordering::Relaxed); +} + +pub(crate) fn unforce_fallback() { + initialize(); +} + +#[cfg(not(no_is_available))] +fn initialize() { + let available = proc_macro::is_available(); + WORKS.store(available as usize + 1, Ordering::Relaxed); +} + +// Swap in a null panic hook to avoid printing "thread panicked" to stderr, +// then use catch_unwind to determine whether the compiler's proc_macro is +// working. When proc-macro2 is used from outside of a procedural macro all +// of the proc_macro crate's APIs currently panic. +// +// The Once is to prevent the possibility of this ordering: +// +// thread 1 calls take_hook, gets the user's original hook +// thread 1 calls set_hook with the null hook +// thread 2 calls take_hook, thinks null hook is the original hook +// thread 2 calls set_hook with the null hook +// thread 1 calls set_hook with the actual original hook +// thread 2 calls set_hook with what it thinks is the original hook +// +// in which the user's hook has been lost. +// +// There is still a race condition where a panic in a different thread can +// happen during the interval that the user's original panic hook is +// unregistered such that their hook is incorrectly not called. This is +// sufficiently unlikely and less bad than printing panic messages to stderr +// on correct use of this crate. Maybe there is a libstd feature request +// here. For now, if a user needs to guarantee that this failure mode does +// not occur, they need to call e.g. `proc_macro2::Span::call_site()` from +// the main thread before launching any other threads. +#[cfg(no_is_available)] +fn initialize() { + use std::panic::{self, PanicInfo}; + + type PanicHook = dyn Fn(&PanicInfo) + Sync + Send + 'static; + + let null_hook: Box = Box::new(|_panic_info| { /* ignore */ }); + let sanity_check = &*null_hook as *const PanicHook; + let original_hook = panic::take_hook(); + panic::set_hook(null_hook); + + let works = panic::catch_unwind(proc_macro::Span::call_site).is_ok(); + WORKS.store(works as usize + 1, Ordering::Relaxed); + + let hopefully_null_hook = panic::take_hook(); + panic::set_hook(original_hook); + if sanity_check != &*hopefully_null_hook { + panic!("observed race condition in proc_macro2::inside_proc_macro"); + } +} diff --git a/rust/proc-macro2/extra.rs b/rust/proc-macro2/extra.rs new file mode 100644 index 000000000000..522a90e136be --- /dev/null +++ b/rust/proc-macro2/extra.rs @@ -0,0 +1,151 @@ +//! Items which do not have a correspondence to any API in the proc_macro crate, +//! but are necessary to include in proc-macro2. + +use crate::fallback; +use crate::imp; +use crate::marker::{ProcMacroAutoTraits, MARKER}; +use crate::Span; +use core::fmt::{self, Debug}; + +/// Invalidate any `proc_macro2::Span` that exist on the current thread. +/// +/// The implementation of `Span` uses thread-local data structures and this +/// function clears them. Calling any method on a `Span` on the current thread +/// created prior to the invalidation will return incorrect values or crash. +/// +/// This function is useful for programs that process more than 232 +/// bytes of Rust source code on the same thread. Just like rustc, proc-macro2 +/// uses 32-bit source locations, and these wrap around when the total source +/// code processed by the same thread exceeds 232 bytes (4 +/// gigabytes). After a wraparound, `Span` methods such as `source_text()` can +/// return wrong data. +/// +/// # Example +/// +/// As of late 2023, there is 200 GB of Rust code published on crates.io. +/// Looking at just the newest version of every crate, it is 16 GB of code. So a +/// workload that involves parsing it all would overflow a 32-bit source +/// location unless spans are being invalidated. +/// +/// ``` +/// use flate2::read::GzDecoder; +/// use std::ffi::OsStr; +/// use std::io::{BufReader, Read}; +/// use std::str::FromStr; +/// use tar::Archive; +/// +/// rayon::scope(|s| { +/// for krate in every_version_of_every_crate() { +/// s.spawn(move |_| { +/// proc_macro2::extra::invalidate_current_thread_spans(); +/// +/// let reader = BufReader::new(krate); +/// let tar = GzDecoder::new(reader); +/// let mut archive = Archive::new(tar); +/// for entry in archive.entries().unwrap() { +/// let mut entry = entry.unwrap(); +/// let path = entry.path().unwrap(); +/// if path.extension() != Some(OsStr::new("rs")) { +/// continue; +/// } +/// let mut content = String::new(); +/// entry.read_to_string(&mut content).unwrap(); +/// match proc_macro2::TokenStream::from_str(&content) { +/// Ok(tokens) => {/* ... */}, +/// Err(_) => continue, +/// } +/// } +/// }); +/// } +/// }); +/// # +/// # fn every_version_of_every_crate() -> Vec { +/// # Vec::new() +/// # } +/// ``` +/// +/// # Panics +/// +/// This function is not applicable to and will panic if called from a +/// procedural macro. +#[cfg(span_locations)] +#[cfg_attr(docsrs, doc(cfg(feature = "span-locations")))] +pub fn invalidate_current_thread_spans() { + crate::imp::invalidate_current_thread_spans(); +} + +/// An object that holds a [`Group`]'s `span_open()` and `span_close()` together +/// in a more compact representation than holding those 2 spans individually. +/// +/// [`Group`]: crate::Group +#[derive(Copy, Clone)] +pub struct DelimSpan { + inner: DelimSpanEnum, + _marker: ProcMacroAutoTraits, +} + +#[derive(Copy, Clone)] +enum DelimSpanEnum { + #[cfg(wrap_proc_macro)] + Compiler { + join: proc_macro::Span, + open: proc_macro::Span, + close: proc_macro::Span, + }, + Fallback(fallback::Span), +} + +impl DelimSpan { + pub(crate) fn new(group: &imp::Group) -> Self { + #[cfg(wrap_proc_macro)] + let inner = match group { + imp::Group::Compiler(group) => DelimSpanEnum::Compiler { + join: group.span(), + open: group.span_open(), + close: group.span_close(), + }, + imp::Group::Fallback(group) => DelimSpanEnum::Fallback(group.span()), + }; + + #[cfg(not(wrap_proc_macro))] + let inner = DelimSpanEnum::Fallback(group.span()); + + DelimSpan { + inner, + _marker: MARKER, + } + } + + /// Returns a span covering the entire delimited group. + pub fn join(&self) -> Span { + match &self.inner { + #[cfg(wrap_proc_macro)] + DelimSpanEnum::Compiler { join, .. } => Span::_new(imp::Span::Compiler(*join)), + DelimSpanEnum::Fallback(span) => Span::_new_fallback(*span), + } + } + + /// Returns a span for the opening punctuation of the group only. + pub fn open(&self) -> Span { + match &self.inner { + #[cfg(wrap_proc_macro)] + DelimSpanEnum::Compiler { open, .. } => Span::_new(imp::Span::Compiler(*open)), + DelimSpanEnum::Fallback(span) => Span::_new_fallback(span.first_byte()), + } + } + + /// Returns a span for the closing punctuation of the group only. + pub fn close(&self) -> Span { + match &self.inner { + #[cfg(wrap_proc_macro)] + DelimSpanEnum::Compiler { close, .. } => Span::_new(imp::Span::Compiler(*close)), + DelimSpanEnum::Fallback(span) => Span::_new_fallback(span.last_byte()), + } + } +} + +impl Debug for DelimSpan { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + Debug::fmt(&self.join(), f) + } +} diff --git a/rust/proc-macro2/fallback.rs b/rust/proc-macro2/fallback.rs new file mode 100644 index 000000000000..1560105cfd25 --- /dev/null +++ b/rust/proc-macro2/fallback.rs @@ -0,0 +1,1256 @@ +#[cfg(wrap_proc_macro)] +use crate::imp; +#[cfg(span_locations)] +use crate::location::LineColumn; +use crate::parse::{self, Cursor}; +use crate::rcvec::{RcVec, RcVecBuilder, RcVecIntoIter, RcVecMut}; +use crate::{Delimiter, Spacing, TokenTree}; +#[cfg(all(span_locations, not(fuzzing)))] +use alloc::collections::BTreeMap; +#[cfg(all(span_locations, not(fuzzing)))] +use core::cell::RefCell; +#[cfg(span_locations)] +use core::cmp; +#[cfg(all(span_locations, not(fuzzing)))] +use core::cmp::Ordering; +use core::fmt::{self, Debug, Display, Write}; +use core::mem::ManuallyDrop; +#[cfg(span_locations)] +use core::ops::Range; +use core::ops::RangeBounds; +use core::ptr; +use core::str; +#[cfg(feature = "proc-macro")] +use core::str::FromStr; +use std::ffi::CStr; +#[cfg(wrap_proc_macro)] +use std::panic; +#[cfg(span_locations)] +use std::path::PathBuf; + +/// Force use of proc-macro2's fallback implementation of the API for now, even +/// if the compiler's implementation is available. +pub fn force() { + #[cfg(wrap_proc_macro)] + crate::detection::force_fallback(); +} + +/// Resume using the compiler's implementation of the proc macro API if it is +/// available. +pub fn unforce() { + #[cfg(wrap_proc_macro)] + crate::detection::unforce_fallback(); +} + +#[derive(Clone)] +pub(crate) struct TokenStream { + inner: RcVec, +} + +#[derive(Debug)] +pub(crate) struct LexError { + pub(crate) span: Span, +} + +impl LexError { + pub(crate) fn span(&self) -> Span { + self.span + } + + pub(crate) fn call_site() -> Self { + LexError { + span: Span::call_site(), + } + } +} + +impl TokenStream { + pub(crate) fn new() -> Self { + TokenStream { + inner: RcVecBuilder::new().build(), + } + } + + pub(crate) fn from_str_checked(src: &str) -> Result { + // Create a dummy file & add it to the source map + let mut cursor = get_cursor(src); + + // Strip a byte order mark if present + const BYTE_ORDER_MARK: &str = "\u{feff}"; + if cursor.starts_with(BYTE_ORDER_MARK) { + cursor = cursor.advance(BYTE_ORDER_MARK.len()); + } + + parse::token_stream(cursor) + } + + #[cfg(feature = "proc-macro")] + pub(crate) fn from_str_unchecked(src: &str) -> Self { + Self::from_str_checked(src).unwrap() + } + + pub(crate) fn is_empty(&self) -> bool { + self.inner.len() == 0 + } + + fn take_inner(self) -> RcVecBuilder { + let nodrop = ManuallyDrop::new(self); + unsafe { ptr::read(&nodrop.inner) }.make_owned() + } +} + +fn push_token_from_proc_macro(mut vec: RcVecMut, token: TokenTree) { + // https://github.com/dtolnay/proc-macro2/issues/235 + match token { + TokenTree::Literal(crate::Literal { + #[cfg(wrap_proc_macro)] + inner: crate::imp::Literal::Fallback(literal), + #[cfg(not(wrap_proc_macro))] + inner: literal, + .. + }) if literal.repr.starts_with('-') => { + push_negative_literal(vec, literal); + } + _ => vec.push(token), + } + + #[cold] + fn push_negative_literal(mut vec: RcVecMut, mut literal: Literal) { + literal.repr.remove(0); + let mut punct = crate::Punct::new('-', Spacing::Alone); + punct.set_span(crate::Span::_new_fallback(literal.span)); + vec.push(TokenTree::Punct(punct)); + vec.push(TokenTree::Literal(crate::Literal::_new_fallback(literal))); + } +} + +// Nonrecursive to prevent stack overflow. +impl Drop for TokenStream { + fn drop(&mut self) { + let mut stack = Vec::new(); + let mut current = match self.inner.get_mut() { + Some(inner) => inner.take().into_iter(), + None => return, + }; + loop { + while let Some(token) = current.next() { + let group = match token { + TokenTree::Group(group) => group.inner, + _ => continue, + }; + #[cfg(wrap_proc_macro)] + let group = match group { + crate::imp::Group::Fallback(group) => group, + crate::imp::Group::Compiler(_) => continue, + }; + let mut group = group; + if let Some(inner) = group.stream.inner.get_mut() { + stack.push(current); + current = inner.take().into_iter(); + } + } + match stack.pop() { + Some(next) => current = next, + None => return, + } + } + } +} + +pub(crate) struct TokenStreamBuilder { + inner: RcVecBuilder, +} + +impl TokenStreamBuilder { + pub(crate) fn new() -> Self { + TokenStreamBuilder { + inner: RcVecBuilder::new(), + } + } + + pub(crate) fn with_capacity(cap: usize) -> Self { + TokenStreamBuilder { + inner: RcVecBuilder::with_capacity(cap), + } + } + + pub(crate) fn push_token_from_parser(&mut self, tt: TokenTree) { + self.inner.push(tt); + } + + pub(crate) fn build(self) -> TokenStream { + TokenStream { + inner: self.inner.build(), + } + } +} + +#[cfg(span_locations)] +fn get_cursor(src: &str) -> Cursor { + #[cfg(fuzzing)] + return Cursor { rest: src, off: 1 }; + + // Create a dummy file & add it to the source map + #[cfg(not(fuzzing))] + SOURCE_MAP.with(|sm| { + let mut sm = sm.borrow_mut(); + let span = sm.add_file(src); + Cursor { + rest: src, + off: span.lo, + } + }) +} + +#[cfg(not(span_locations))] +fn get_cursor(src: &str) -> Cursor { + Cursor { rest: src } +} + +impl Display for LexError { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + f.write_str("cannot parse string into token stream") + } +} + +impl Display for TokenStream { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + let mut joint = false; + for (i, tt) in self.inner.iter().enumerate() { + if i != 0 && !joint { + write!(f, " ")?; + } + joint = false; + match tt { + TokenTree::Group(tt) => Display::fmt(tt, f), + TokenTree::Ident(tt) => Display::fmt(tt, f), + TokenTree::Punct(tt) => { + joint = tt.spacing() == Spacing::Joint; + Display::fmt(tt, f) + } + TokenTree::Literal(tt) => Display::fmt(tt, f), + }?; + } + + Ok(()) + } +} + +impl Debug for TokenStream { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + f.write_str("TokenStream ")?; + f.debug_list().entries(self.clone()).finish() + } +} + +#[cfg(feature = "proc-macro")] +impl From for TokenStream { + fn from(inner: proc_macro::TokenStream) -> Self { + TokenStream::from_str_unchecked(&inner.to_string()) + } +} + +#[cfg(feature = "proc-macro")] +impl From for proc_macro::TokenStream { + fn from(inner: TokenStream) -> Self { + proc_macro::TokenStream::from_str_unchecked(&inner.to_string()) + } +} + +impl From for TokenStream { + fn from(tree: TokenTree) -> Self { + let mut stream = RcVecBuilder::new(); + push_token_from_proc_macro(stream.as_mut(), tree); + TokenStream { + inner: stream.build(), + } + } +} + +impl FromIterator for TokenStream { + fn from_iter>(tokens: I) -> Self { + let mut stream = TokenStream::new(); + stream.extend(tokens); + stream + } +} + +impl FromIterator for TokenStream { + fn from_iter>(streams: I) -> Self { + let mut v = RcVecBuilder::new(); + + for stream in streams { + v.extend(stream.take_inner()); + } + + TokenStream { inner: v.build() } + } +} + +impl Extend for TokenStream { + fn extend>(&mut self, tokens: I) { + let mut vec = self.inner.make_mut(); + tokens + .into_iter() + .for_each(|token| push_token_from_proc_macro(vec.as_mut(), token)); + } +} + +impl Extend for TokenStream { + fn extend>(&mut self, streams: I) { + self.inner.make_mut().extend(streams.into_iter().flatten()); + } +} + +pub(crate) type TokenTreeIter = RcVecIntoIter; + +impl IntoIterator for TokenStream { + type Item = TokenTree; + type IntoIter = TokenTreeIter; + + fn into_iter(self) -> TokenTreeIter { + self.take_inner().into_iter() + } +} + +#[cfg(all(span_locations, not(fuzzing)))] +thread_local! { + static SOURCE_MAP: RefCell = RefCell::new(SourceMap { + // Start with a single dummy file which all call_site() and def_site() + // spans reference. + files: vec![FileInfo { + source_text: String::new(), + span: Span { lo: 0, hi: 0 }, + lines: vec![0], + char_index_to_byte_offset: BTreeMap::new(), + }], + }); +} + +#[cfg(span_locations)] +pub(crate) fn invalidate_current_thread_spans() { + #[cfg(not(fuzzing))] + SOURCE_MAP.with(|sm| sm.borrow_mut().files.truncate(1)); +} + +#[cfg(all(span_locations, not(fuzzing)))] +struct FileInfo { + source_text: String, + span: Span, + lines: Vec, + char_index_to_byte_offset: BTreeMap, +} + +#[cfg(all(span_locations, not(fuzzing)))] +impl FileInfo { + fn offset_line_column(&self, offset: usize) -> LineColumn { + assert!(self.span_within(Span { + lo: offset as u32, + hi: offset as u32, + })); + let offset = offset - self.span.lo as usize; + match self.lines.binary_search(&offset) { + Ok(found) => LineColumn { + line: found + 1, + column: 0, + }, + Err(idx) => LineColumn { + line: idx, + column: offset - self.lines[idx - 1], + }, + } + } + + fn span_within(&self, span: Span) -> bool { + span.lo >= self.span.lo && span.hi <= self.span.hi + } + + fn byte_range(&mut self, span: Span) -> Range { + let lo_char = (span.lo - self.span.lo) as usize; + + // Look up offset of the largest already-computed char index that is + // less than or equal to the current requested one. We resume counting + // chars from that point. + let (&last_char_index, &last_byte_offset) = self + .char_index_to_byte_offset + .range(..=lo_char) + .next_back() + .unwrap_or((&0, &0)); + + let lo_byte = if last_char_index == lo_char { + last_byte_offset + } else { + let total_byte_offset = match self.source_text[last_byte_offset..] + .char_indices() + .nth(lo_char - last_char_index) + { + Some((additional_offset, _ch)) => last_byte_offset + additional_offset, + None => self.source_text.len(), + }; + self.char_index_to_byte_offset + .insert(lo_char, total_byte_offset); + total_byte_offset + }; + + let trunc_lo = &self.source_text[lo_byte..]; + let char_len = (span.hi - span.lo) as usize; + lo_byte..match trunc_lo.char_indices().nth(char_len) { + Some((offset, _ch)) => lo_byte + offset, + None => self.source_text.len(), + } + } + + fn source_text(&mut self, span: Span) -> String { + let byte_range = self.byte_range(span); + self.source_text[byte_range].to_owned() + } +} + +/// Computes the offsets of each line in the given source string +/// and the total number of characters +#[cfg(all(span_locations, not(fuzzing)))] +fn lines_offsets(s: &str) -> (usize, Vec) { + let mut lines = vec![0]; + let mut total = 0; + + for ch in s.chars() { + total += 1; + if ch == '\n' { + lines.push(total); + } + } + + (total, lines) +} + +#[cfg(all(span_locations, not(fuzzing)))] +struct SourceMap { + files: Vec, +} + +#[cfg(all(span_locations, not(fuzzing)))] +impl SourceMap { + fn next_start_pos(&self) -> u32 { + // Add 1 so there's always space between files. + // + // We'll always have at least 1 file, as we initialize our files list + // with a dummy file. + self.files.last().unwrap().span.hi + 1 + } + + fn add_file(&mut self, src: &str) -> Span { + let (len, lines) = lines_offsets(src); + let lo = self.next_start_pos(); + let span = Span { + lo, + hi: lo + (len as u32), + }; + + self.files.push(FileInfo { + source_text: src.to_owned(), + span, + lines, + // Populated lazily by source_text(). + char_index_to_byte_offset: BTreeMap::new(), + }); + + span + } + + fn find(&self, span: Span) -> usize { + match self.files.binary_search_by(|file| { + if file.span.hi < span.lo { + Ordering::Less + } else if file.span.lo > span.hi { + Ordering::Greater + } else { + assert!(file.span_within(span)); + Ordering::Equal + } + }) { + Ok(i) => i, + Err(_) => unreachable!("Invalid span with no related FileInfo!"), + } + } + + fn filepath(&self, span: Span) -> String { + let i = self.find(span); + if i == 0 { + "".to_owned() + } else { + format!("", i) + } + } + + fn fileinfo(&self, span: Span) -> &FileInfo { + let i = self.find(span); + &self.files[i] + } + + fn fileinfo_mut(&mut self, span: Span) -> &mut FileInfo { + let i = self.find(span); + &mut self.files[i] + } +} + +#[derive(Clone, Copy, PartialEq, Eq)] +pub(crate) struct Span { + #[cfg(span_locations)] + pub(crate) lo: u32, + #[cfg(span_locations)] + pub(crate) hi: u32, +} + +impl Span { + #[cfg(not(span_locations))] + pub(crate) fn call_site() -> Self { + Span {} + } + + #[cfg(span_locations)] + pub(crate) fn call_site() -> Self { + Span { lo: 0, hi: 0 } + } + + pub(crate) fn mixed_site() -> Self { + Span::call_site() + } + + #[cfg(procmacro2_semver_exempt)] + pub(crate) fn def_site() -> Self { + Span::call_site() + } + + pub(crate) fn resolved_at(&self, _other: Span) -> Span { + // Stable spans consist only of line/column information, so + // `resolved_at` and `located_at` only select which span the + // caller wants line/column information from. + *self + } + + pub(crate) fn located_at(&self, other: Span) -> Span { + other + } + + #[cfg(span_locations)] + pub(crate) fn byte_range(&self) -> Range { + #[cfg(fuzzing)] + return 0..0; + + #[cfg(not(fuzzing))] + { + if self.is_call_site() { + 0..0 + } else { + SOURCE_MAP.with(|sm| sm.borrow_mut().fileinfo_mut(*self).byte_range(*self)) + } + } + } + + #[cfg(span_locations)] + pub(crate) fn start(&self) -> LineColumn { + #[cfg(fuzzing)] + return LineColumn { line: 0, column: 0 }; + + #[cfg(not(fuzzing))] + SOURCE_MAP.with(|sm| { + let sm = sm.borrow(); + let fi = sm.fileinfo(*self); + fi.offset_line_column(self.lo as usize) + }) + } + + #[cfg(span_locations)] + pub(crate) fn end(&self) -> LineColumn { + #[cfg(fuzzing)] + return LineColumn { line: 0, column: 0 }; + + #[cfg(not(fuzzing))] + SOURCE_MAP.with(|sm| { + let sm = sm.borrow(); + let fi = sm.fileinfo(*self); + fi.offset_line_column(self.hi as usize) + }) + } + + #[cfg(span_locations)] + pub(crate) fn file(&self) -> String { + #[cfg(fuzzing)] + return "".to_owned(); + + #[cfg(not(fuzzing))] + SOURCE_MAP.with(|sm| { + let sm = sm.borrow(); + sm.filepath(*self) + }) + } + + #[cfg(span_locations)] + pub(crate) fn local_file(&self) -> Option { + None + } + + #[cfg(not(span_locations))] + pub(crate) fn join(&self, _other: Span) -> Option { + Some(Span {}) + } + + #[cfg(span_locations)] + pub(crate) fn join(&self, other: Span) -> Option { + #[cfg(fuzzing)] + return { + let _ = other; + None + }; + + #[cfg(not(fuzzing))] + SOURCE_MAP.with(|sm| { + let sm = sm.borrow(); + // If `other` is not within the same FileInfo as us, return None. + if !sm.fileinfo(*self).span_within(other) { + return None; + } + Some(Span { + lo: cmp::min(self.lo, other.lo), + hi: cmp::max(self.hi, other.hi), + }) + }) + } + + #[cfg(not(span_locations))] + pub(crate) fn source_text(&self) -> Option { + None + } + + #[cfg(span_locations)] + pub(crate) fn source_text(&self) -> Option { + #[cfg(fuzzing)] + return None; + + #[cfg(not(fuzzing))] + { + if self.is_call_site() { + None + } else { + Some(SOURCE_MAP.with(|sm| sm.borrow_mut().fileinfo_mut(*self).source_text(*self))) + } + } + } + + #[cfg(not(span_locations))] + pub(crate) fn first_byte(self) -> Self { + self + } + + #[cfg(span_locations)] + pub(crate) fn first_byte(self) -> Self { + Span { + lo: self.lo, + hi: cmp::min(self.lo.saturating_add(1), self.hi), + } + } + + #[cfg(not(span_locations))] + pub(crate) fn last_byte(self) -> Self { + self + } + + #[cfg(span_locations)] + pub(crate) fn last_byte(self) -> Self { + Span { + lo: cmp::max(self.hi.saturating_sub(1), self.lo), + hi: self.hi, + } + } + + #[cfg(span_locations)] + fn is_call_site(&self) -> bool { + self.lo == 0 && self.hi == 0 + } +} + +impl Debug for Span { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + #[cfg(span_locations)] + return write!(f, "bytes({}..{})", self.lo, self.hi); + + #[cfg(not(span_locations))] + write!(f, "Span") + } +} + +pub(crate) fn debug_span_field_if_nontrivial(debug: &mut fmt::DebugStruct, span: Span) { + #[cfg(span_locations)] + { + if span.is_call_site() { + return; + } + } + + if cfg!(span_locations) { + debug.field("span", &span); + } +} + +#[derive(Clone)] +pub(crate) struct Group { + delimiter: Delimiter, + stream: TokenStream, + span: Span, +} + +impl Group { + pub(crate) fn new(delimiter: Delimiter, stream: TokenStream) -> Self { + Group { + delimiter, + stream, + span: Span::call_site(), + } + } + + pub(crate) fn delimiter(&self) -> Delimiter { + self.delimiter + } + + pub(crate) fn stream(&self) -> TokenStream { + self.stream.clone() + } + + pub(crate) fn span(&self) -> Span { + self.span + } + + pub(crate) fn span_open(&self) -> Span { + self.span.first_byte() + } + + pub(crate) fn span_close(&self) -> Span { + self.span.last_byte() + } + + pub(crate) fn set_span(&mut self, span: Span) { + self.span = span; + } +} + +impl Display for Group { + // We attempt to match libproc_macro's formatting. + // Empty parens: () + // Nonempty parens: (...) + // Empty brackets: [] + // Nonempty brackets: [...] + // Empty braces: { } + // Nonempty braces: { ... } + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + let (open, close) = match self.delimiter { + Delimiter::Parenthesis => ("(", ")"), + Delimiter::Brace => ("{ ", "}"), + Delimiter::Bracket => ("[", "]"), + Delimiter::None => ("", ""), + }; + + f.write_str(open)?; + Display::fmt(&self.stream, f)?; + if self.delimiter == Delimiter::Brace && !self.stream.inner.is_empty() { + f.write_str(" ")?; + } + f.write_str(close)?; + + Ok(()) + } +} + +impl Debug for Group { + fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result { + let mut debug = fmt.debug_struct("Group"); + debug.field("delimiter", &self.delimiter); + debug.field("stream", &self.stream); + debug_span_field_if_nontrivial(&mut debug, self.span); + debug.finish() + } +} + +#[derive(Clone)] +pub(crate) struct Ident { + sym: Box, + span: Span, + raw: bool, +} + +impl Ident { + #[track_caller] + pub(crate) fn new_checked(string: &str, span: Span) -> Self { + validate_ident(string); + Ident::new_unchecked(string, span) + } + + pub(crate) fn new_unchecked(string: &str, span: Span) -> Self { + Ident { + sym: Box::from(string), + span, + raw: false, + } + } + + #[track_caller] + pub(crate) fn new_raw_checked(string: &str, span: Span) -> Self { + validate_ident_raw(string); + Ident::new_raw_unchecked(string, span) + } + + pub(crate) fn new_raw_unchecked(string: &str, span: Span) -> Self { + Ident { + sym: Box::from(string), + span, + raw: true, + } + } + + pub(crate) fn span(&self) -> Span { + self.span + } + + pub(crate) fn set_span(&mut self, span: Span) { + self.span = span; + } +} + +pub(crate) fn is_ident_start(c: char) -> bool { + c == '_' || unicode_ident::is_xid_start(c) +} + +pub(crate) fn is_ident_continue(c: char) -> bool { + unicode_ident::is_xid_continue(c) +} + +#[track_caller] +fn validate_ident(string: &str) { + if string.is_empty() { + panic!("Ident is not allowed to be empty; use Option"); + } + + if string.bytes().all(|digit| b'0' <= digit && digit <= b'9') { + panic!("Ident cannot be a number; use Literal instead"); + } + + fn ident_ok(string: &str) -> bool { + let mut chars = string.chars(); + let first = chars.next().unwrap(); + if !is_ident_start(first) { + return false; + } + for ch in chars { + if !is_ident_continue(ch) { + return false; + } + } + true + } + + if !ident_ok(string) { + panic!("{:?} is not a valid Ident", string); + } +} + +#[track_caller] +fn validate_ident_raw(string: &str) { + validate_ident(string); + + match string { + "_" | "super" | "self" | "Self" | "crate" => { + panic!("`r#{}` cannot be a raw identifier", string); + } + _ => {} + } +} + +impl PartialEq for Ident { + fn eq(&self, other: &Ident) -> bool { + self.sym == other.sym && self.raw == other.raw + } +} + +impl PartialEq for Ident +where + T: ?Sized + AsRef, +{ + fn eq(&self, other: &T) -> bool { + let other = other.as_ref(); + if self.raw { + other.starts_with("r#") && *self.sym == other[2..] + } else { + *self.sym == *other + } + } +} + +impl Display for Ident { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + if self.raw { + f.write_str("r#")?; + } + Display::fmt(&self.sym, f) + } +} + +#[allow(clippy::missing_fields_in_debug)] +impl Debug for Ident { + // Ident(proc_macro), Ident(r#union) + #[cfg(not(span_locations))] + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + let mut debug = f.debug_tuple("Ident"); + debug.field(&format_args!("{}", self)); + debug.finish() + } + + // Ident { + // sym: proc_macro, + // span: bytes(128..138) + // } + #[cfg(span_locations)] + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + let mut debug = f.debug_struct("Ident"); + debug.field("sym", &format_args!("{}", self)); + debug_span_field_if_nontrivial(&mut debug, self.span); + debug.finish() + } +} + +#[derive(Clone)] +pub(crate) struct Literal { + pub(crate) repr: String, + span: Span, +} + +macro_rules! suffixed_numbers { + ($($name:ident => $kind:ident,)*) => ($( + pub(crate) fn $name(n: $kind) -> Literal { + Literal::_new(format!(concat!("{}", stringify!($kind)), n)) + } + )*) +} + +macro_rules! unsuffixed_numbers { + ($($name:ident => $kind:ident,)*) => ($( + pub(crate) fn $name(n: $kind) -> Literal { + Literal::_new(n.to_string()) + } + )*) +} + +impl Literal { + pub(crate) fn _new(repr: String) -> Self { + Literal { + repr, + span: Span::call_site(), + } + } + + pub(crate) fn from_str_checked(repr: &str) -> Result { + let mut cursor = get_cursor(repr); + #[cfg(span_locations)] + let lo = cursor.off; + + let negative = cursor.starts_with_char('-'); + if negative { + cursor = cursor.advance(1); + if !cursor.starts_with_fn(|ch| ch.is_ascii_digit()) { + return Err(LexError::call_site()); + } + } + + if let Ok((rest, mut literal)) = parse::literal(cursor) { + if rest.is_empty() { + if negative { + literal.repr.insert(0, '-'); + } + literal.span = Span { + #[cfg(span_locations)] + lo, + #[cfg(span_locations)] + hi: rest.off, + }; + return Ok(literal); + } + } + Err(LexError::call_site()) + } + + pub(crate) unsafe fn from_str_unchecked(repr: &str) -> Self { + Literal::_new(repr.to_owned()) + } + + suffixed_numbers! { + u8_suffixed => u8, + u16_suffixed => u16, + u32_suffixed => u32, + u64_suffixed => u64, + u128_suffixed => u128, + usize_suffixed => usize, + i8_suffixed => i8, + i16_suffixed => i16, + i32_suffixed => i32, + i64_suffixed => i64, + i128_suffixed => i128, + isize_suffixed => isize, + + f32_suffixed => f32, + f64_suffixed => f64, + } + + unsuffixed_numbers! { + u8_unsuffixed => u8, + u16_unsuffixed => u16, + u32_unsuffixed => u32, + u64_unsuffixed => u64, + u128_unsuffixed => u128, + usize_unsuffixed => usize, + i8_unsuffixed => i8, + i16_unsuffixed => i16, + i32_unsuffixed => i32, + i64_unsuffixed => i64, + i128_unsuffixed => i128, + isize_unsuffixed => isize, + } + + pub(crate) fn f32_unsuffixed(f: f32) -> Literal { + let mut s = f.to_string(); + if !s.contains('.') { + s.push_str(".0"); + } + Literal::_new(s) + } + + pub(crate) fn f64_unsuffixed(f: f64) -> Literal { + let mut s = f.to_string(); + if !s.contains('.') { + s.push_str(".0"); + } + Literal::_new(s) + } + + pub(crate) fn string(string: &str) -> Literal { + let mut repr = String::with_capacity(string.len() + 2); + repr.push('"'); + escape_utf8(string, &mut repr); + repr.push('"'); + Literal::_new(repr) + } + + pub(crate) fn character(ch: char) -> Literal { + let mut repr = String::new(); + repr.push('\''); + if ch == '"' { + // escape_debug turns this into '\"' which is unnecessary. + repr.push(ch); + } else { + repr.extend(ch.escape_debug()); + } + repr.push('\''); + Literal::_new(repr) + } + + pub(crate) fn byte_character(byte: u8) -> Literal { + let mut repr = "b'".to_string(); + #[allow(clippy::match_overlapping_arm)] + match byte { + b'\0' => repr.push_str(r"\0"), + b'\t' => repr.push_str(r"\t"), + b'\n' => repr.push_str(r"\n"), + b'\r' => repr.push_str(r"\r"), + b'\'' => repr.push_str(r"\'"), + b'\\' => repr.push_str(r"\\"), + b'\x20'..=b'\x7E' => repr.push(byte as char), + _ => { + let _ = write!(repr, r"\x{:02X}", byte); + } + } + repr.push('\''); + Literal::_new(repr) + } + + pub(crate) fn byte_string(bytes: &[u8]) -> Literal { + let mut repr = "b\"".to_string(); + let mut bytes = bytes.iter(); + while let Some(&b) = bytes.next() { + #[allow(clippy::match_overlapping_arm)] + match b { + b'\0' => repr.push_str(match bytes.as_slice().first() { + // circumvent clippy::octal_escapes lint + Some(b'0'..=b'7') => r"\x00", + _ => r"\0", + }), + b'\t' => repr.push_str(r"\t"), + b'\n' => repr.push_str(r"\n"), + b'\r' => repr.push_str(r"\r"), + b'"' => repr.push_str("\\\""), + b'\\' => repr.push_str(r"\\"), + b'\x20'..=b'\x7E' => repr.push(b as char), + _ => { + let _ = write!(repr, r"\x{:02X}", b); + } + } + } + repr.push('"'); + Literal::_new(repr) + } + + pub(crate) fn c_string(string: &CStr) -> Literal { + let mut repr = "c\"".to_string(); + let mut bytes = string.to_bytes(); + while !bytes.is_empty() { + let (valid, invalid) = match str::from_utf8(bytes) { + Ok(all_valid) => { + bytes = b""; + (all_valid, bytes) + } + Err(utf8_error) => { + let (valid, rest) = bytes.split_at(utf8_error.valid_up_to()); + let valid = str::from_utf8(valid).unwrap(); + let invalid = utf8_error + .error_len() + .map_or(rest, |error_len| &rest[..error_len]); + bytes = &bytes[valid.len() + invalid.len()..]; + (valid, invalid) + } + }; + escape_utf8(valid, &mut repr); + for &byte in invalid { + let _ = write!(repr, r"\x{:02X}", byte); + } + } + repr.push('"'); + Literal::_new(repr) + } + + pub(crate) fn span(&self) -> Span { + self.span + } + + pub(crate) fn set_span(&mut self, span: Span) { + self.span = span; + } + + pub(crate) fn subspan>(&self, range: R) -> Option { + #[cfg(not(span_locations))] + { + let _ = range; + None + } + + #[cfg(span_locations)] + { + use core::ops::Bound; + + let lo = match range.start_bound() { + Bound::Included(start) => { + let start = u32::try_from(*start).ok()?; + self.span.lo.checked_add(start)? + } + Bound::Excluded(start) => { + let start = u32::try_from(*start).ok()?; + self.span.lo.checked_add(start)?.checked_add(1)? + } + Bound::Unbounded => self.span.lo, + }; + let hi = match range.end_bound() { + Bound::Included(end) => { + let end = u32::try_from(*end).ok()?; + self.span.lo.checked_add(end)?.checked_add(1)? + } + Bound::Excluded(end) => { + let end = u32::try_from(*end).ok()?; + self.span.lo.checked_add(end)? + } + Bound::Unbounded => self.span.hi, + }; + if lo <= hi && hi <= self.span.hi { + Some(Span { lo, hi }) + } else { + None + } + } + } +} + +impl Display for Literal { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + Display::fmt(&self.repr, f) + } +} + +impl Debug for Literal { + fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result { + let mut debug = fmt.debug_struct("Literal"); + debug.field("lit", &format_args!("{}", self.repr)); + debug_span_field_if_nontrivial(&mut debug, self.span); + debug.finish() + } +} + +fn escape_utf8(string: &str, repr: &mut String) { + let mut chars = string.chars(); + while let Some(ch) = chars.next() { + if ch == '\0' { + repr.push_str( + if chars + .as_str() + .starts_with(|next| '0' <= next && next <= '7') + { + // circumvent clippy::octal_escapes lint + r"\x00" + } else { + r"\0" + }, + ); + } else if ch == '\'' { + // escape_debug turns this into "\'" which is unnecessary. + repr.push(ch); + } else { + repr.extend(ch.escape_debug()); + } + } +} + +#[cfg(feature = "proc-macro")] +pub(crate) trait FromStr2: FromStr { + #[cfg(wrap_proc_macro)] + fn valid(src: &str) -> bool; + + #[cfg(wrap_proc_macro)] + fn from_str_checked(src: &str) -> Result { + // Validate using fallback parser, because rustc is incapable of + // returning a recoverable Err for certain invalid token streams, and + // will instead permanently poison the compilation. + if !Self::valid(src) { + return Err(imp::LexError::CompilerPanic); + } + + // Catch panic to work around https://github.com/rust-lang/rust/issues/58736. + match panic::catch_unwind(|| Self::from_str(src)) { + Ok(Ok(ok)) => Ok(ok), + Ok(Err(lex)) => Err(imp::LexError::Compiler(lex)), + Err(_panic) => Err(imp::LexError::CompilerPanic), + } + } + + fn from_str_unchecked(src: &str) -> Self { + Self::from_str(src).unwrap() + } +} + +#[cfg(feature = "proc-macro")] +impl FromStr2 for proc_macro::TokenStream { + #[cfg(wrap_proc_macro)] + fn valid(src: &str) -> bool { + TokenStream::from_str_checked(src).is_ok() + } +} + +#[cfg(feature = "proc-macro")] +impl FromStr2 for proc_macro::Literal { + #[cfg(wrap_proc_macro)] + fn valid(src: &str) -> bool { + Literal::from_str_checked(src).is_ok() + } +} diff --git a/rust/proc-macro2/lib.rs b/rust/proc-macro2/lib.rs new file mode 100644 index 000000000000..2984af335adc --- /dev/null +++ b/rust/proc-macro2/lib.rs @@ -0,0 +1,1349 @@ +//! [![github]](https://github.com/dtolnay/proc-macro2) [![crates-io]](https://crates.io/crates/proc-macro2) [![docs-rs]](crate) +//! +//! [github]: https://img.shields.io/badge/github-8da0cb?style=for-the-badge&labelColor=555555&logo=github +//! [crates-io]: https://img.shields.io/badge/crates.io-fc8d62?style=for-the-badge&labelColor=555555&logo=rust +//! [docs-rs]: https://img.shields.io/badge/docs.rs-66c2a5?style=for-the-badge&labelColor=555555&logo=docs.rs +//! +//!
+//! +//! A wrapper around the procedural macro API of the compiler's [`proc_macro`] +//! crate. This library serves two purposes: +//! +//! - **Bring proc-macro-like functionality to other contexts like build.rs and +//! main.rs.** Types from `proc_macro` are entirely specific to procedural +//! macros and cannot ever exist in code outside of a procedural macro. +//! Meanwhile `proc_macro2` types may exist anywhere including non-macro code. +//! By developing foundational libraries like [syn] and [quote] against +//! `proc_macro2` rather than `proc_macro`, the procedural macro ecosystem +//! becomes easily applicable to many other use cases and we avoid +//! reimplementing non-macro equivalents of those libraries. +//! +//! - **Make procedural macros unit testable.** As a consequence of being +//! specific to procedural macros, nothing that uses `proc_macro` can be +//! executed from a unit test. In order for helper libraries or components of +//! a macro to be testable in isolation, they must be implemented using +//! `proc_macro2`. +//! +//! [syn]: https://github.com/dtolnay/syn +//! [quote]: https://github.com/dtolnay/quote +//! +//! # Usage +//! +//! The skeleton of a typical procedural macro typically looks like this: +//! +//! ``` +//! extern crate proc_macro; +//! +//! # const IGNORE: &str = stringify! { +//! #[proc_macro_derive(MyDerive)] +//! # }; +//! # #[cfg(wrap_proc_macro)] +//! pub fn my_derive(input: proc_macro::TokenStream) -> proc_macro::TokenStream { +//! let input = proc_macro2::TokenStream::from(input); +//! +//! let output: proc_macro2::TokenStream = { +//! /* transform input */ +//! # input +//! }; +//! +//! proc_macro::TokenStream::from(output) +//! } +//! ``` +//! +//! If parsing with [Syn], you'll use [`parse_macro_input!`] instead to +//! propagate parse errors correctly back to the compiler when parsing fails. +//! +//! [`parse_macro_input!`]: https://docs.rs/syn/2.0/syn/macro.parse_macro_input.html +//! +//! # Unstable features +//! +//! The default feature set of proc-macro2 tracks the most recent stable +//! compiler API. Functionality in `proc_macro` that is not yet stable is not +//! exposed by proc-macro2 by default. +//! +//! To opt into the additional APIs available in the most recent nightly +//! compiler, the `procmacro2_semver_exempt` config flag must be passed to +//! rustc. We will polyfill those nightly-only APIs back to Rust 1.56.0. As +//! these are unstable APIs that track the nightly compiler, minor versions of +//! proc-macro2 may make breaking changes to them at any time. +//! +//! ```sh +//! RUSTFLAGS='--cfg procmacro2_semver_exempt' cargo build +//! ``` +//! +//! Note that this must not only be done for your crate, but for any crate that +//! depends on your crate. This infectious nature is intentional, as it serves +//! as a reminder that you are outside of the normal semver guarantees. +//! +//! Semver exempt methods are marked as such in the proc-macro2 documentation. +//! +//! # Thread-Safety +//! +//! Most types in this crate are `!Sync` because the underlying compiler +//! types make use of thread-local memory, meaning they cannot be accessed from +//! a different thread. + +// Proc-macro2 types in rustdoc of other crates get linked to here. +#![doc(html_root_url = "https://docs.rs/proc-macro2/1.0.101")] +#![cfg_attr(any(proc_macro_span, super_unstable), feature(proc_macro_span))] +#![cfg_attr(super_unstable, feature(proc_macro_def_site))] +#![cfg_attr(docsrs, feature(doc_cfg))] +#![deny(unsafe_op_in_unsafe_fn)] +#![allow( + clippy::cast_lossless, + clippy::cast_possible_truncation, + clippy::checked_conversions, + clippy::doc_markdown, + clippy::elidable_lifetime_names, + clippy::incompatible_msrv, + clippy::items_after_statements, + clippy::iter_without_into_iter, + clippy::let_underscore_untyped, + clippy::manual_assert, + clippy::manual_range_contains, + clippy::missing_panics_doc, + clippy::missing_safety_doc, + clippy::must_use_candidate, + clippy::needless_doctest_main, + clippy::needless_lifetimes, + clippy::new_without_default, + clippy::return_self_not_must_use, + clippy::shadow_unrelated, + clippy::trivially_copy_pass_by_ref, + clippy::unnecessary_wraps, + clippy::unused_self, + clippy::used_underscore_binding, + clippy::vec_init_then_push +)] +#![allow(unknown_lints, mismatched_lifetime_syntaxes)] + +#[cfg(all(procmacro2_semver_exempt, wrap_proc_macro, not(super_unstable)))] +compile_error! {"\ + Something is not right. If you've tried to turn on \ + procmacro2_semver_exempt, you need to ensure that it \ + is turned on for the compilation of the proc-macro2 \ + build script as well. +"} + +#[cfg(all( + procmacro2_nightly_testing, + feature = "proc-macro", + not(proc_macro_span) +))] +compile_error! {"\ + Build script probe failed to compile. +"} + +extern crate alloc; + +#[cfg(feature = "proc-macro")] +extern crate proc_macro; + +mod marker; +mod parse; +mod probe; +mod rcvec; + +#[cfg(wrap_proc_macro)] +mod detection; + +// Public for proc_macro2::fallback::force() and unforce(), but those are quite +// a niche use case so we omit it from rustdoc. +#[doc(hidden)] +pub mod fallback; + +pub mod extra; + +#[cfg(not(wrap_proc_macro))] +use crate::fallback as imp; +#[path = "wrapper.rs"] +#[cfg(wrap_proc_macro)] +mod imp; + +#[cfg(span_locations)] +mod location; + +use crate::extra::DelimSpan; +use crate::marker::{ProcMacroAutoTraits, MARKER}; +use core::cmp::Ordering; +use core::fmt::{self, Debug, Display}; +use core::hash::{Hash, Hasher}; +#[cfg(span_locations)] +use core::ops::Range; +use core::ops::RangeBounds; +use core::str::FromStr; +use std::error::Error; +use std::ffi::CStr; +#[cfg(span_locations)] +use std::path::PathBuf; + +#[cfg(span_locations)] +#[cfg_attr(docsrs, doc(cfg(feature = "span-locations")))] +pub use crate::location::LineColumn; + +/// An abstract stream of tokens, or more concretely a sequence of token trees. +/// +/// This type provides interfaces for iterating over token trees and for +/// collecting token trees into one stream. +/// +/// Token stream is both the input and output of `#[proc_macro]`, +/// `#[proc_macro_attribute]` and `#[proc_macro_derive]` definitions. +#[derive(Clone)] +pub struct TokenStream { + inner: imp::TokenStream, + _marker: ProcMacroAutoTraits, +} + +/// Error returned from `TokenStream::from_str`. +pub struct LexError { + inner: imp::LexError, + _marker: ProcMacroAutoTraits, +} + +impl TokenStream { + fn _new(inner: imp::TokenStream) -> Self { + TokenStream { + inner, + _marker: MARKER, + } + } + + fn _new_fallback(inner: fallback::TokenStream) -> Self { + TokenStream { + inner: imp::TokenStream::from(inner), + _marker: MARKER, + } + } + + /// Returns an empty `TokenStream` containing no token trees. + pub fn new() -> Self { + TokenStream::_new(imp::TokenStream::new()) + } + + /// Checks if this `TokenStream` is empty. + pub fn is_empty(&self) -> bool { + self.inner.is_empty() + } +} + +/// `TokenStream::default()` returns an empty stream, +/// i.e. this is equivalent with `TokenStream::new()`. +impl Default for TokenStream { + fn default() -> Self { + TokenStream::new() + } +} + +/// Attempts to break the string into tokens and parse those tokens into a token +/// stream. +/// +/// May fail for a number of reasons, for example, if the string contains +/// unbalanced delimiters or characters not existing in the language. +/// +/// NOTE: Some errors may cause panics instead of returning `LexError`. We +/// reserve the right to change these errors into `LexError`s later. +impl FromStr for TokenStream { + type Err = LexError; + + fn from_str(src: &str) -> Result { + match imp::TokenStream::from_str_checked(src) { + Ok(tokens) => Ok(TokenStream::_new(tokens)), + Err(lex) => Err(LexError { + inner: lex, + _marker: MARKER, + }), + } + } +} + +#[cfg(feature = "proc-macro")] +#[cfg_attr(docsrs, doc(cfg(feature = "proc-macro")))] +impl From for TokenStream { + fn from(inner: proc_macro::TokenStream) -> Self { + TokenStream::_new(imp::TokenStream::from(inner)) + } +} + +#[cfg(feature = "proc-macro")] +#[cfg_attr(docsrs, doc(cfg(feature = "proc-macro")))] +impl From for proc_macro::TokenStream { + fn from(inner: TokenStream) -> Self { + proc_macro::TokenStream::from(inner.inner) + } +} + +impl From for TokenStream { + fn from(token: TokenTree) -> Self { + TokenStream::_new(imp::TokenStream::from(token)) + } +} + +impl Extend for TokenStream { + fn extend>(&mut self, streams: I) { + self.inner.extend(streams); + } +} + +impl Extend for TokenStream { + fn extend>(&mut self, streams: I) { + self.inner + .extend(streams.into_iter().map(|stream| stream.inner)); + } +} + +/// Collects a number of token trees into a single stream. +impl FromIterator for TokenStream { + fn from_iter>(streams: I) -> Self { + TokenStream::_new(streams.into_iter().collect()) + } +} +impl FromIterator for TokenStream { + fn from_iter>(streams: I) -> Self { + TokenStream::_new(streams.into_iter().map(|i| i.inner).collect()) + } +} + +/// Prints the token stream as a string that is supposed to be losslessly +/// convertible back into the same token stream (modulo spans), except for +/// possibly `TokenTree::Group`s with `Delimiter::None` delimiters and negative +/// numeric literals. +impl Display for TokenStream { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + Display::fmt(&self.inner, f) + } +} + +/// Prints token in a form convenient for debugging. +impl Debug for TokenStream { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + Debug::fmt(&self.inner, f) + } +} + +impl LexError { + pub fn span(&self) -> Span { + Span::_new(self.inner.span()) + } +} + +impl Debug for LexError { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + Debug::fmt(&self.inner, f) + } +} + +impl Display for LexError { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + Display::fmt(&self.inner, f) + } +} + +impl Error for LexError {} + +/// A region of source code, along with macro expansion information. +#[derive(Copy, Clone)] +pub struct Span { + inner: imp::Span, + _marker: ProcMacroAutoTraits, +} + +impl Span { + fn _new(inner: imp::Span) -> Self { + Span { + inner, + _marker: MARKER, + } + } + + fn _new_fallback(inner: fallback::Span) -> Self { + Span { + inner: imp::Span::from(inner), + _marker: MARKER, + } + } + + /// The span of the invocation of the current procedural macro. + /// + /// Identifiers created with this span will be resolved as if they were + /// written directly at the macro call location (call-site hygiene) and + /// other code at the macro call site will be able to refer to them as well. + pub fn call_site() -> Self { + Span::_new(imp::Span::call_site()) + } + + /// The span located at the invocation of the procedural macro, but with + /// local variables, labels, and `$crate` resolved at the definition site + /// of the macro. This is the same hygiene behavior as `macro_rules`. + pub fn mixed_site() -> Self { + Span::_new(imp::Span::mixed_site()) + } + + /// A span that resolves at the macro definition site. + /// + /// This method is semver exempt and not exposed by default. + #[cfg(procmacro2_semver_exempt)] + #[cfg_attr(docsrs, doc(cfg(procmacro2_semver_exempt)))] + pub fn def_site() -> Self { + Span::_new(imp::Span::def_site()) + } + + /// Creates a new span with the same line/column information as `self` but + /// that resolves symbols as though it were at `other`. + pub fn resolved_at(&self, other: Span) -> Span { + Span::_new(self.inner.resolved_at(other.inner)) + } + + /// Creates a new span with the same name resolution behavior as `self` but + /// with the line/column information of `other`. + pub fn located_at(&self, other: Span) -> Span { + Span::_new(self.inner.located_at(other.inner)) + } + + /// Convert `proc_macro2::Span` to `proc_macro::Span`. + /// + /// This method is available when building with a nightly compiler, or when + /// building with rustc 1.29+ *without* semver exempt features. + /// + /// # Panics + /// + /// Panics if called from outside of a procedural macro. Unlike + /// `proc_macro2::Span`, the `proc_macro::Span` type can only exist within + /// the context of a procedural macro invocation. + #[cfg(wrap_proc_macro)] + pub fn unwrap(self) -> proc_macro::Span { + self.inner.unwrap() + } + + // Soft deprecated. Please use Span::unwrap. + #[cfg(wrap_proc_macro)] + #[doc(hidden)] + pub fn unstable(self) -> proc_macro::Span { + self.unwrap() + } + + /// Returns the span's byte position range in the source file. + /// + /// This method requires the `"span-locations"` feature to be enabled. + /// + /// When executing in a procedural macro context, the returned range is only + /// accurate if compiled with a nightly toolchain. The stable toolchain does + /// not have this information available. When executing outside of a + /// procedural macro, such as main.rs or build.rs, the byte range is always + /// accurate regardless of toolchain. + #[cfg(span_locations)] + #[cfg_attr(docsrs, doc(cfg(feature = "span-locations")))] + pub fn byte_range(&self) -> Range { + self.inner.byte_range() + } + + /// Get the starting line/column in the source file for this span. + /// + /// This method requires the `"span-locations"` feature to be enabled. + /// + /// When executing in a procedural macro context, the returned line/column + /// are only meaningful if compiled with a nightly toolchain. The stable + /// toolchain does not have this information available. When executing + /// outside of a procedural macro, such as main.rs or build.rs, the + /// line/column are always meaningful regardless of toolchain. + #[cfg(span_locations)] + #[cfg_attr(docsrs, doc(cfg(feature = "span-locations")))] + pub fn start(&self) -> LineColumn { + self.inner.start() + } + + /// Get the ending line/column in the source file for this span. + /// + /// This method requires the `"span-locations"` feature to be enabled. + /// + /// When executing in a procedural macro context, the returned line/column + /// are only meaningful if compiled with a nightly toolchain. The stable + /// toolchain does not have this information available. When executing + /// outside of a procedural macro, such as main.rs or build.rs, the + /// line/column are always meaningful regardless of toolchain. + #[cfg(span_locations)] + #[cfg_attr(docsrs, doc(cfg(feature = "span-locations")))] + pub fn end(&self) -> LineColumn { + self.inner.end() + } + + /// The path to the source file in which this span occurs, for display + /// purposes. + /// + /// This might not correspond to a valid file system path. It might be + /// remapped, or might be an artificial path such as `""`. + #[cfg(span_locations)] + #[cfg_attr(docsrs, doc(cfg(feature = "span-locations")))] + pub fn file(&self) -> String { + self.inner.file() + } + + /// The path to the source file in which this span occurs on disk. + /// + /// This is the actual path on disk. It is unaffected by path remapping. + /// + /// This path should not be embedded in the output of the macro; prefer + /// `file()` instead. + #[cfg(span_locations)] + #[cfg_attr(docsrs, doc(cfg(feature = "span-locations")))] + pub fn local_file(&self) -> Option { + self.inner.local_file() + } + + /// Create a new span encompassing `self` and `other`. + /// + /// Returns `None` if `self` and `other` are from different files. + /// + /// Warning: the underlying [`proc_macro::Span::join`] method is + /// nightly-only. When called from within a procedural macro not using a + /// nightly compiler, this method will always return `None`. + pub fn join(&self, other: Span) -> Option { + self.inner.join(other.inner).map(Span::_new) + } + + /// Compares two spans to see if they're equal. + /// + /// This method is semver exempt and not exposed by default. + #[cfg(procmacro2_semver_exempt)] + #[cfg_attr(docsrs, doc(cfg(procmacro2_semver_exempt)))] + pub fn eq(&self, other: &Span) -> bool { + self.inner.eq(&other.inner) + } + + /// Returns the source text behind a span. This preserves the original + /// source code, including spaces and comments. It only returns a result if + /// the span corresponds to real source code. + /// + /// Note: The observable result of a macro should only rely on the tokens + /// and not on this source text. The result of this function is a best + /// effort to be used for diagnostics only. + pub fn source_text(&self) -> Option { + self.inner.source_text() + } +} + +/// Prints a span in a form convenient for debugging. +impl Debug for Span { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + Debug::fmt(&self.inner, f) + } +} + +/// A single token or a delimited sequence of token trees (e.g. `[1, (), ..]`). +#[derive(Clone)] +pub enum TokenTree { + /// A token stream surrounded by bracket delimiters. + Group(Group), + /// An identifier. + Ident(Ident), + /// A single punctuation character (`+`, `,`, `$`, etc.). + Punct(Punct), + /// A literal character (`'a'`), string (`"hello"`), number (`2.3`), etc. + Literal(Literal), +} + +impl TokenTree { + /// Returns the span of this tree, delegating to the `span` method of + /// the contained token or a delimited stream. + pub fn span(&self) -> Span { + match self { + TokenTree::Group(t) => t.span(), + TokenTree::Ident(t) => t.span(), + TokenTree::Punct(t) => t.span(), + TokenTree::Literal(t) => t.span(), + } + } + + /// Configures the span for *only this token*. + /// + /// Note that if this token is a `Group` then this method will not configure + /// the span of each of the internal tokens, this will simply delegate to + /// the `set_span` method of each variant. + pub fn set_span(&mut self, span: Span) { + match self { + TokenTree::Group(t) => t.set_span(span), + TokenTree::Ident(t) => t.set_span(span), + TokenTree::Punct(t) => t.set_span(span), + TokenTree::Literal(t) => t.set_span(span), + } + } +} + +impl From for TokenTree { + fn from(g: Group) -> Self { + TokenTree::Group(g) + } +} + +impl From for TokenTree { + fn from(g: Ident) -> Self { + TokenTree::Ident(g) + } +} + +impl From for TokenTree { + fn from(g: Punct) -> Self { + TokenTree::Punct(g) + } +} + +impl From for TokenTree { + fn from(g: Literal) -> Self { + TokenTree::Literal(g) + } +} + +/// Prints the token tree as a string that is supposed to be losslessly +/// convertible back into the same token tree (modulo spans), except for +/// possibly `TokenTree::Group`s with `Delimiter::None` delimiters and negative +/// numeric literals. +impl Display for TokenTree { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + match self { + TokenTree::Group(t) => Display::fmt(t, f), + TokenTree::Ident(t) => Display::fmt(t, f), + TokenTree::Punct(t) => Display::fmt(t, f), + TokenTree::Literal(t) => Display::fmt(t, f), + } + } +} + +/// Prints token tree in a form convenient for debugging. +impl Debug for TokenTree { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + // Each of these has the name in the struct type in the derived debug, + // so don't bother with an extra layer of indirection + match self { + TokenTree::Group(t) => Debug::fmt(t, f), + TokenTree::Ident(t) => { + let mut debug = f.debug_struct("Ident"); + debug.field("sym", &format_args!("{}", t)); + imp::debug_span_field_if_nontrivial(&mut debug, t.span().inner); + debug.finish() + } + TokenTree::Punct(t) => Debug::fmt(t, f), + TokenTree::Literal(t) => Debug::fmt(t, f), + } + } +} + +/// A delimited token stream. +/// +/// A `Group` internally contains a `TokenStream` which is surrounded by +/// `Delimiter`s. +#[derive(Clone)] +pub struct Group { + inner: imp::Group, +} + +/// Describes how a sequence of token trees is delimited. +#[derive(Copy, Clone, Debug, Eq, PartialEq)] +pub enum Delimiter { + /// `( ... )` + Parenthesis, + /// `{ ... }` + Brace, + /// `[ ... ]` + Bracket, + /// `∅ ... ∅` + /// + /// An invisible delimiter, that may, for example, appear around tokens + /// coming from a "macro variable" `$var`. It is important to preserve + /// operator priorities in cases like `$var * 3` where `$var` is `1 + 2`. + /// Invisible delimiters may not survive roundtrip of a token stream through + /// a string. + /// + ///
+ /// + /// Note: rustc currently can ignore the grouping of tokens delimited by `None` in the output + /// of a proc_macro. Only `None`-delimited groups created by a macro_rules macro in the input + /// of a proc_macro macro are preserved, and only in very specific circumstances. + /// Any `None`-delimited groups (re)created by a proc_macro will therefore not preserve + /// operator priorities as indicated above. The other `Delimiter` variants should be used + /// instead in this context. This is a rustc bug. For details, see + /// [rust-lang/rust#67062](https://github.com/rust-lang/rust/issues/67062). + /// + ///
+ None, +} + +impl Group { + fn _new(inner: imp::Group) -> Self { + Group { inner } + } + + fn _new_fallback(inner: fallback::Group) -> Self { + Group { + inner: imp::Group::from(inner), + } + } + + /// Creates a new `Group` with the given delimiter and token stream. + /// + /// This constructor will set the span for this group to + /// `Span::call_site()`. To change the span you can use the `set_span` + /// method below. + pub fn new(delimiter: Delimiter, stream: TokenStream) -> Self { + Group { + inner: imp::Group::new(delimiter, stream.inner), + } + } + + /// Returns the punctuation used as the delimiter for this group: a set of + /// parentheses, square brackets, or curly braces. + pub fn delimiter(&self) -> Delimiter { + self.inner.delimiter() + } + + /// Returns the `TokenStream` of tokens that are delimited in this `Group`. + /// + /// Note that the returned token stream does not include the delimiter + /// returned above. + pub fn stream(&self) -> TokenStream { + TokenStream::_new(self.inner.stream()) + } + + /// Returns the span for the delimiters of this token stream, spanning the + /// entire `Group`. + /// + /// ```text + /// pub fn span(&self) -> Span { + /// ^^^^^^^ + /// ``` + pub fn span(&self) -> Span { + Span::_new(self.inner.span()) + } + + /// Returns the span pointing to the opening delimiter of this group. + /// + /// ```text + /// pub fn span_open(&self) -> Span { + /// ^ + /// ``` + pub fn span_open(&self) -> Span { + Span::_new(self.inner.span_open()) + } + + /// Returns the span pointing to the closing delimiter of this group. + /// + /// ```text + /// pub fn span_close(&self) -> Span { + /// ^ + /// ``` + pub fn span_close(&self) -> Span { + Span::_new(self.inner.span_close()) + } + + /// Returns an object that holds this group's `span_open()` and + /// `span_close()` together (in a more compact representation than holding + /// those 2 spans individually). + pub fn delim_span(&self) -> DelimSpan { + DelimSpan::new(&self.inner) + } + + /// Configures the span for this `Group`'s delimiters, but not its internal + /// tokens. + /// + /// This method will **not** set the span of all the internal tokens spanned + /// by this group, but rather it will only set the span of the delimiter + /// tokens at the level of the `Group`. + pub fn set_span(&mut self, span: Span) { + self.inner.set_span(span.inner); + } +} + +/// Prints the group as a string that should be losslessly convertible back +/// into the same group (modulo spans), except for possibly `TokenTree::Group`s +/// with `Delimiter::None` delimiters. +impl Display for Group { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + Display::fmt(&self.inner, formatter) + } +} + +impl Debug for Group { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + Debug::fmt(&self.inner, formatter) + } +} + +/// A `Punct` is a single punctuation character like `+`, `-` or `#`. +/// +/// Multicharacter operators like `+=` are represented as two instances of +/// `Punct` with different forms of `Spacing` returned. +#[derive(Clone)] +pub struct Punct { + ch: char, + spacing: Spacing, + span: Span, +} + +/// Whether a `Punct` is followed immediately by another `Punct` or followed by +/// another token or whitespace. +#[derive(Copy, Clone, Debug, Eq, PartialEq)] +pub enum Spacing { + /// E.g. `+` is `Alone` in `+ =`, `+ident` or `+()`. + Alone, + /// E.g. `+` is `Joint` in `+=` or `'` is `Joint` in `'#`. + /// + /// Additionally, single quote `'` can join with identifiers to form + /// lifetimes `'ident`. + Joint, +} + +impl Punct { + /// Creates a new `Punct` from the given character and spacing. + /// + /// The `ch` argument must be a valid punctuation character permitted by the + /// language, otherwise the function will panic. + /// + /// The returned `Punct` will have the default span of `Span::call_site()` + /// which can be further configured with the `set_span` method below. + pub fn new(ch: char, spacing: Spacing) -> Self { + if let '!' | '#' | '$' | '%' | '&' | '\'' | '*' | '+' | ',' | '-' | '.' | '/' | ':' | ';' + | '<' | '=' | '>' | '?' | '@' | '^' | '|' | '~' = ch + { + Punct { + ch, + spacing, + span: Span::call_site(), + } + } else { + panic!("unsupported proc macro punctuation character {:?}", ch); + } + } + + /// Returns the value of this punctuation character as `char`. + pub fn as_char(&self) -> char { + self.ch + } + + /// Returns the spacing of this punctuation character, indicating whether + /// it's immediately followed by another `Punct` in the token stream, so + /// they can potentially be combined into a multicharacter operator + /// (`Joint`), or it's followed by some other token or whitespace (`Alone`) + /// so the operator has certainly ended. + pub fn spacing(&self) -> Spacing { + self.spacing + } + + /// Returns the span for this punctuation character. + pub fn span(&self) -> Span { + self.span + } + + /// Configure the span for this punctuation character. + pub fn set_span(&mut self, span: Span) { + self.span = span; + } +} + +/// Prints the punctuation character as a string that should be losslessly +/// convertible back into the same character. +impl Display for Punct { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + Display::fmt(&self.ch, f) + } +} + +impl Debug for Punct { + fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result { + let mut debug = fmt.debug_struct("Punct"); + debug.field("char", &self.ch); + debug.field("spacing", &self.spacing); + imp::debug_span_field_if_nontrivial(&mut debug, self.span.inner); + debug.finish() + } +} + +/// A word of Rust code, which may be a keyword or legal variable name. +/// +/// An identifier consists of at least one Unicode code point, the first of +/// which has the XID_Start property and the rest of which have the XID_Continue +/// property. +/// +/// - The empty string is not an identifier. Use `Option`. +/// - A lifetime is not an identifier. Use `syn::Lifetime` instead. +/// +/// An identifier constructed with `Ident::new` is permitted to be a Rust +/// keyword, though parsing one through its [`Parse`] implementation rejects +/// Rust keywords. Use `input.call(Ident::parse_any)` when parsing to match the +/// behaviour of `Ident::new`. +/// +/// [`Parse`]: https://docs.rs/syn/2.0/syn/parse/trait.Parse.html +/// +/// # Examples +/// +/// A new ident can be created from a string using the `Ident::new` function. +/// A span must be provided explicitly which governs the name resolution +/// behavior of the resulting identifier. +/// +/// ``` +/// use proc_macro2::{Ident, Span}; +/// +/// fn main() { +/// let call_ident = Ident::new("calligraphy", Span::call_site()); +/// +/// println!("{}", call_ident); +/// } +/// ``` +/// +/// An ident can be interpolated into a token stream using the `quote!` macro. +/// +/// ``` +/// use proc_macro2::{Ident, Span}; +/// use quote::quote; +/// +/// fn main() { +/// let ident = Ident::new("demo", Span::call_site()); +/// +/// // Create a variable binding whose name is this ident. +/// let expanded = quote! { let #ident = 10; }; +/// +/// // Create a variable binding with a slightly different name. +/// let temp_ident = Ident::new(&format!("new_{}", ident), Span::call_site()); +/// let expanded = quote! { let #temp_ident = 10; }; +/// } +/// ``` +/// +/// A string representation of the ident is available through the `to_string()` +/// method. +/// +/// ``` +/// # use proc_macro2::{Ident, Span}; +/// # +/// # let ident = Ident::new("another_identifier", Span::call_site()); +/// # +/// // Examine the ident as a string. +/// let ident_string = ident.to_string(); +/// if ident_string.len() > 60 { +/// println!("Very long identifier: {}", ident_string) +/// } +/// ``` +#[derive(Clone)] +pub struct Ident { + inner: imp::Ident, + _marker: ProcMacroAutoTraits, +} + +impl Ident { + fn _new(inner: imp::Ident) -> Self { + Ident { + inner, + _marker: MARKER, + } + } + + fn _new_fallback(inner: fallback::Ident) -> Self { + Ident { + inner: imp::Ident::from(inner), + _marker: MARKER, + } + } + + /// Creates a new `Ident` with the given `string` as well as the specified + /// `span`. + /// + /// The `string` argument must be a valid identifier permitted by the + /// language, otherwise the function will panic. + /// + /// Note that `span`, currently in rustc, configures the hygiene information + /// for this identifier. + /// + /// As of this time `Span::call_site()` explicitly opts-in to "call-site" + /// hygiene meaning that identifiers created with this span will be resolved + /// as if they were written directly at the location of the macro call, and + /// other code at the macro call site will be able to refer to them as well. + /// + /// Later spans like `Span::def_site()` will allow to opt-in to + /// "definition-site" hygiene meaning that identifiers created with this + /// span will be resolved at the location of the macro definition and other + /// code at the macro call site will not be able to refer to them. + /// + /// Due to the current importance of hygiene this constructor, unlike other + /// tokens, requires a `Span` to be specified at construction. + /// + /// # Panics + /// + /// Panics if the input string is neither a keyword nor a legal variable + /// name. If you are not sure whether the string contains an identifier and + /// need to handle an error case, use + /// syn::parse_str::<Ident> + /// rather than `Ident::new`. + #[track_caller] + pub fn new(string: &str, span: Span) -> Self { + Ident::_new(imp::Ident::new_checked(string, span.inner)) + } + + /// Same as `Ident::new`, but creates a raw identifier (`r#ident`). The + /// `string` argument must be a valid identifier permitted by the language + /// (including keywords, e.g. `fn`). Keywords which are usable in path + /// segments (e.g. `self`, `super`) are not supported, and will cause a + /// panic. + #[track_caller] + pub fn new_raw(string: &str, span: Span) -> Self { + Ident::_new(imp::Ident::new_raw_checked(string, span.inner)) + } + + /// Returns the span of this `Ident`. + pub fn span(&self) -> Span { + Span::_new(self.inner.span()) + } + + /// Configures the span of this `Ident`, possibly changing its hygiene + /// context. + pub fn set_span(&mut self, span: Span) { + self.inner.set_span(span.inner); + } +} + +impl PartialEq for Ident { + fn eq(&self, other: &Ident) -> bool { + self.inner == other.inner + } +} + +impl PartialEq for Ident +where + T: ?Sized + AsRef, +{ + fn eq(&self, other: &T) -> bool { + self.inner == other + } +} + +impl Eq for Ident {} + +impl PartialOrd for Ident { + fn partial_cmp(&self, other: &Ident) -> Option { + Some(self.cmp(other)) + } +} + +impl Ord for Ident { + fn cmp(&self, other: &Ident) -> Ordering { + self.to_string().cmp(&other.to_string()) + } +} + +impl Hash for Ident { + fn hash(&self, hasher: &mut H) { + self.to_string().hash(hasher); + } +} + +/// Prints the identifier as a string that should be losslessly convertible back +/// into the same identifier. +impl Display for Ident { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + Display::fmt(&self.inner, f) + } +} + +impl Debug for Ident { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + Debug::fmt(&self.inner, f) + } +} + +/// A literal string (`"hello"`), byte string (`b"hello"`), character (`'a'`), +/// byte character (`b'a'`), an integer or floating point number with or without +/// a suffix (`1`, `1u8`, `2.3`, `2.3f32`). +/// +/// Boolean literals like `true` and `false` do not belong here, they are +/// `Ident`s. +#[derive(Clone)] +pub struct Literal { + inner: imp::Literal, + _marker: ProcMacroAutoTraits, +} + +macro_rules! suffixed_int_literals { + ($($name:ident => $kind:ident,)*) => ($( + /// Creates a new suffixed integer literal with the specified value. + /// + /// This function will create an integer like `1u32` where the integer + /// value specified is the first part of the token and the integral is + /// also suffixed at the end. Literals created from negative numbers may + /// not survive roundtrips through `TokenStream` or strings and may be + /// broken into two tokens (`-` and positive literal). + /// + /// Literals created through this method have the `Span::call_site()` + /// span by default, which can be configured with the `set_span` method + /// below. + pub fn $name(n: $kind) -> Literal { + Literal::_new(imp::Literal::$name(n)) + } + )*) +} + +macro_rules! unsuffixed_int_literals { + ($($name:ident => $kind:ident,)*) => ($( + /// Creates a new unsuffixed integer literal with the specified value. + /// + /// This function will create an integer like `1` where the integer + /// value specified is the first part of the token. No suffix is + /// specified on this token, meaning that invocations like + /// `Literal::i8_unsuffixed(1)` are equivalent to + /// `Literal::u32_unsuffixed(1)`. Literals created from negative numbers + /// may not survive roundtrips through `TokenStream` or strings and may + /// be broken into two tokens (`-` and positive literal). + /// + /// Literals created through this method have the `Span::call_site()` + /// span by default, which can be configured with the `set_span` method + /// below. + pub fn $name(n: $kind) -> Literal { + Literal::_new(imp::Literal::$name(n)) + } + )*) +} + +impl Literal { + fn _new(inner: imp::Literal) -> Self { + Literal { + inner, + _marker: MARKER, + } + } + + fn _new_fallback(inner: fallback::Literal) -> Self { + Literal { + inner: imp::Literal::from(inner), + _marker: MARKER, + } + } + + suffixed_int_literals! { + u8_suffixed => u8, + u16_suffixed => u16, + u32_suffixed => u32, + u64_suffixed => u64, + u128_suffixed => u128, + usize_suffixed => usize, + i8_suffixed => i8, + i16_suffixed => i16, + i32_suffixed => i32, + i64_suffixed => i64, + i128_suffixed => i128, + isize_suffixed => isize, + } + + unsuffixed_int_literals! { + u8_unsuffixed => u8, + u16_unsuffixed => u16, + u32_unsuffixed => u32, + u64_unsuffixed => u64, + u128_unsuffixed => u128, + usize_unsuffixed => usize, + i8_unsuffixed => i8, + i16_unsuffixed => i16, + i32_unsuffixed => i32, + i64_unsuffixed => i64, + i128_unsuffixed => i128, + isize_unsuffixed => isize, + } + + /// Creates a new unsuffixed floating-point literal. + /// + /// This constructor is similar to those like `Literal::i8_unsuffixed` where + /// the float's value is emitted directly into the token but no suffix is + /// used, so it may be inferred to be a `f64` later in the compiler. + /// Literals created from negative numbers may not survive round-trips + /// through `TokenStream` or strings and may be broken into two tokens (`-` + /// and positive literal). + /// + /// # Panics + /// + /// This function requires that the specified float is finite, for example + /// if it is infinity or NaN this function will panic. + pub fn f64_unsuffixed(f: f64) -> Literal { + assert!(f.is_finite()); + Literal::_new(imp::Literal::f64_unsuffixed(f)) + } + + /// Creates a new suffixed floating-point literal. + /// + /// This constructor will create a literal like `1.0f64` where the value + /// specified is the preceding part of the token and `f64` is the suffix of + /// the token. This token will always be inferred to be an `f64` in the + /// compiler. Literals created from negative numbers may not survive + /// round-trips through `TokenStream` or strings and may be broken into two + /// tokens (`-` and positive literal). + /// + /// # Panics + /// + /// This function requires that the specified float is finite, for example + /// if it is infinity or NaN this function will panic. + pub fn f64_suffixed(f: f64) -> Literal { + assert!(f.is_finite()); + Literal::_new(imp::Literal::f64_suffixed(f)) + } + + /// Creates a new unsuffixed floating-point literal. + /// + /// This constructor is similar to those like `Literal::i8_unsuffixed` where + /// the float's value is emitted directly into the token but no suffix is + /// used, so it may be inferred to be a `f64` later in the compiler. + /// Literals created from negative numbers may not survive round-trips + /// through `TokenStream` or strings and may be broken into two tokens (`-` + /// and positive literal). + /// + /// # Panics + /// + /// This function requires that the specified float is finite, for example + /// if it is infinity or NaN this function will panic. + pub fn f32_unsuffixed(f: f32) -> Literal { + assert!(f.is_finite()); + Literal::_new(imp::Literal::f32_unsuffixed(f)) + } + + /// Creates a new suffixed floating-point literal. + /// + /// This constructor will create a literal like `1.0f32` where the value + /// specified is the preceding part of the token and `f32` is the suffix of + /// the token. This token will always be inferred to be an `f32` in the + /// compiler. Literals created from negative numbers may not survive + /// round-trips through `TokenStream` or strings and may be broken into two + /// tokens (`-` and positive literal). + /// + /// # Panics + /// + /// This function requires that the specified float is finite, for example + /// if it is infinity or NaN this function will panic. + pub fn f32_suffixed(f: f32) -> Literal { + assert!(f.is_finite()); + Literal::_new(imp::Literal::f32_suffixed(f)) + } + + /// String literal. + pub fn string(string: &str) -> Literal { + Literal::_new(imp::Literal::string(string)) + } + + /// Character literal. + pub fn character(ch: char) -> Literal { + Literal::_new(imp::Literal::character(ch)) + } + + /// Byte character literal. + pub fn byte_character(byte: u8) -> Literal { + Literal::_new(imp::Literal::byte_character(byte)) + } + + /// Byte string literal. + pub fn byte_string(bytes: &[u8]) -> Literal { + Literal::_new(imp::Literal::byte_string(bytes)) + } + + /// C string literal. + pub fn c_string(string: &CStr) -> Literal { + Literal::_new(imp::Literal::c_string(string)) + } + + /// Returns the span encompassing this literal. + pub fn span(&self) -> Span { + Span::_new(self.inner.span()) + } + + /// Configures the span associated for this literal. + pub fn set_span(&mut self, span: Span) { + self.inner.set_span(span.inner); + } + + /// Returns a `Span` that is a subset of `self.span()` containing only + /// the source bytes in range `range`. Returns `None` if the would-be + /// trimmed span is outside the bounds of `self`. + /// + /// Warning: the underlying [`proc_macro::Literal::subspan`] method is + /// nightly-only. When called from within a procedural macro not using a + /// nightly compiler, this method will always return `None`. + pub fn subspan>(&self, range: R) -> Option { + self.inner.subspan(range).map(Span::_new) + } + + // Intended for the `quote!` macro to use when constructing a proc-macro2 + // token out of a macro_rules $:literal token, which is already known to be + // a valid literal. This avoids reparsing/validating the literal's string + // representation. This is not public API other than for quote. + #[doc(hidden)] + pub unsafe fn from_str_unchecked(repr: &str) -> Self { + Literal::_new(unsafe { imp::Literal::from_str_unchecked(repr) }) + } +} + +impl FromStr for Literal { + type Err = LexError; + + fn from_str(repr: &str) -> Result { + match imp::Literal::from_str_checked(repr) { + Ok(lit) => Ok(Literal::_new(lit)), + Err(lex) => Err(LexError { + inner: lex, + _marker: MARKER, + }), + } + } +} + +impl Debug for Literal { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + Debug::fmt(&self.inner, f) + } +} + +impl Display for Literal { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + Display::fmt(&self.inner, f) + } +} + +/// Public implementation details for the `TokenStream` type, such as iterators. +pub mod token_stream { + use crate::marker::{ProcMacroAutoTraits, MARKER}; + use crate::{imp, TokenTree}; + use core::fmt::{self, Debug}; + + pub use crate::TokenStream; + + /// An iterator over `TokenStream`'s `TokenTree`s. + /// + /// The iteration is "shallow", e.g. the iterator doesn't recurse into + /// delimited groups, and returns whole groups as token trees. + #[derive(Clone)] + pub struct IntoIter { + inner: imp::TokenTreeIter, + _marker: ProcMacroAutoTraits, + } + + impl Iterator for IntoIter { + type Item = TokenTree; + + fn next(&mut self) -> Option { + self.inner.next() + } + + fn size_hint(&self) -> (usize, Option) { + self.inner.size_hint() + } + } + + impl Debug for IntoIter { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + f.write_str("TokenStream ")?; + f.debug_list().entries(self.clone()).finish() + } + } + + impl IntoIterator for TokenStream { + type Item = TokenTree; + type IntoIter = IntoIter; + + fn into_iter(self) -> IntoIter { + IntoIter { + inner: self.inner.into_iter(), + _marker: MARKER, + } + } + } +} diff --git a/rust/proc-macro2/location.rs b/rust/proc-macro2/location.rs new file mode 100644 index 000000000000..7190e2d05255 --- /dev/null +++ b/rust/proc-macro2/location.rs @@ -0,0 +1,29 @@ +use core::cmp::Ordering; + +/// A line-column pair representing the start or end of a `Span`. +/// +/// This type is semver exempt and not exposed by default. +#[cfg_attr(docsrs, doc(cfg(feature = "span-locations")))] +#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)] +pub struct LineColumn { + /// The 1-indexed line in the source file on which the span starts or ends + /// (inclusive). + pub line: usize, + /// The 0-indexed column (in UTF-8 characters) in the source file on which + /// the span starts or ends (inclusive). + pub column: usize, +} + +impl Ord for LineColumn { + fn cmp(&self, other: &Self) -> Ordering { + self.line + .cmp(&other.line) + .then(self.column.cmp(&other.column)) + } +} + +impl PartialOrd for LineColumn { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} diff --git a/rust/proc-macro2/marker.rs b/rust/proc-macro2/marker.rs new file mode 100644 index 000000000000..23b94ce6fa85 --- /dev/null +++ b/rust/proc-macro2/marker.rs @@ -0,0 +1,17 @@ +use alloc::rc::Rc; +use core::marker::PhantomData; +use core::panic::{RefUnwindSafe, UnwindSafe}; + +// Zero sized marker with the correct set of autotrait impls we want all proc +// macro types to have. +#[derive(Copy, Clone)] +#[cfg_attr( + all(procmacro2_semver_exempt, any(not(wrap_proc_macro), super_unstable)), + derive(PartialEq, Eq) +)] +pub(crate) struct ProcMacroAutoTraits(PhantomData>); + +pub(crate) const MARKER: ProcMacroAutoTraits = ProcMacroAutoTraits(PhantomData); + +impl UnwindSafe for ProcMacroAutoTraits {} +impl RefUnwindSafe for ProcMacroAutoTraits {} diff --git a/rust/proc-macro2/parse.rs b/rust/proc-macro2/parse.rs new file mode 100644 index 000000000000..b8be403f842f --- /dev/null +++ b/rust/proc-macro2/parse.rs @@ -0,0 +1,995 @@ +use crate::fallback::{ + self, is_ident_continue, is_ident_start, Group, Ident, LexError, Literal, Span, TokenStream, + TokenStreamBuilder, +}; +use crate::{Delimiter, Punct, Spacing, TokenTree}; +use core::char; +use core::str::{Bytes, CharIndices, Chars}; + +#[derive(Copy, Clone, Eq, PartialEq)] +pub(crate) struct Cursor<'a> { + pub(crate) rest: &'a str, + #[cfg(span_locations)] + pub(crate) off: u32, +} + +impl<'a> Cursor<'a> { + pub(crate) fn advance(&self, bytes: usize) -> Cursor<'a> { + let (_front, rest) = self.rest.split_at(bytes); + Cursor { + rest, + #[cfg(span_locations)] + off: self.off + _front.chars().count() as u32, + } + } + + pub(crate) fn starts_with(&self, s: &str) -> bool { + self.rest.starts_with(s) + } + + pub(crate) fn starts_with_char(&self, ch: char) -> bool { + self.rest.starts_with(ch) + } + + pub(crate) fn starts_with_fn(&self, f: Pattern) -> bool + where + Pattern: FnMut(char) -> bool, + { + self.rest.starts_with(f) + } + + pub(crate) fn is_empty(&self) -> bool { + self.rest.is_empty() + } + + fn len(&self) -> usize { + self.rest.len() + } + + fn as_bytes(&self) -> &'a [u8] { + self.rest.as_bytes() + } + + fn bytes(&self) -> Bytes<'a> { + self.rest.bytes() + } + + fn chars(&self) -> Chars<'a> { + self.rest.chars() + } + + fn char_indices(&self) -> CharIndices<'a> { + self.rest.char_indices() + } + + fn parse(&self, tag: &str) -> Result, Reject> { + if self.starts_with(tag) { + Ok(self.advance(tag.len())) + } else { + Err(Reject) + } + } +} + +pub(crate) struct Reject; +type PResult<'a, O> = Result<(Cursor<'a>, O), Reject>; + +fn skip_whitespace(input: Cursor) -> Cursor { + let mut s = input; + + while !s.is_empty() { + let byte = s.as_bytes()[0]; + if byte == b'/' { + if s.starts_with("//") + && (!s.starts_with("///") || s.starts_with("////")) + && !s.starts_with("//!") + { + let (cursor, _) = take_until_newline_or_eof(s); + s = cursor; + continue; + } else if s.starts_with("/**/") { + s = s.advance(4); + continue; + } else if s.starts_with("/*") + && (!s.starts_with("/**") || s.starts_with("/***")) + && !s.starts_with("/*!") + { + match block_comment(s) { + Ok((rest, _)) => { + s = rest; + continue; + } + Err(Reject) => return s, + } + } + } + match byte { + b' ' | 0x09..=0x0d => { + s = s.advance(1); + continue; + } + b if b.is_ascii() => {} + _ => { + let ch = s.chars().next().unwrap(); + if is_whitespace(ch) { + s = s.advance(ch.len_utf8()); + continue; + } + } + } + return s; + } + s +} + +fn block_comment(input: Cursor) -> PResult<&str> { + if !input.starts_with("/*") { + return Err(Reject); + } + + let mut depth = 0usize; + let bytes = input.as_bytes(); + let mut i = 0usize; + let upper = bytes.len() - 1; + + while i < upper { + if bytes[i] == b'/' && bytes[i + 1] == b'*' { + depth += 1; + i += 1; // eat '*' + } else if bytes[i] == b'*' && bytes[i + 1] == b'/' { + depth -= 1; + if depth == 0 { + return Ok((input.advance(i + 2), &input.rest[..i + 2])); + } + i += 1; // eat '/' + } + i += 1; + } + + Err(Reject) +} + +fn is_whitespace(ch: char) -> bool { + // Rust treats left-to-right mark and right-to-left mark as whitespace + ch.is_whitespace() || ch == '\u{200e}' || ch == '\u{200f}' +} + +fn word_break(input: Cursor) -> Result { + match input.chars().next() { + Some(ch) if is_ident_continue(ch) => Err(Reject), + Some(_) | None => Ok(input), + } +} + +// Rustc's representation of a macro expansion error in expression position or +// type position. +const ERROR: &str = "(/*ERROR*/)"; + +pub(crate) fn token_stream(mut input: Cursor) -> Result { + let mut trees = TokenStreamBuilder::new(); + let mut stack = Vec::new(); + + loop { + input = skip_whitespace(input); + + if let Ok((rest, ())) = doc_comment(input, &mut trees) { + input = rest; + continue; + } + + #[cfg(span_locations)] + let lo = input.off; + + let first = match input.bytes().next() { + Some(first) => first, + None => match stack.last() { + None => return Ok(trees.build()), + #[cfg(span_locations)] + Some((lo, _frame)) => { + return Err(LexError { + span: Span { lo: *lo, hi: *lo }, + }) + } + #[cfg(not(span_locations))] + Some(_frame) => return Err(LexError { span: Span {} }), + }, + }; + + if let Some(open_delimiter) = match first { + b'(' if !input.starts_with(ERROR) => Some(Delimiter::Parenthesis), + b'[' => Some(Delimiter::Bracket), + b'{' => Some(Delimiter::Brace), + _ => None, + } { + input = input.advance(1); + let frame = (open_delimiter, trees); + #[cfg(span_locations)] + let frame = (lo, frame); + stack.push(frame); + trees = TokenStreamBuilder::new(); + } else if let Some(close_delimiter) = match first { + b')' => Some(Delimiter::Parenthesis), + b']' => Some(Delimiter::Bracket), + b'}' => Some(Delimiter::Brace), + _ => None, + } { + let frame = match stack.pop() { + Some(frame) => frame, + None => return Err(lex_error(input)), + }; + #[cfg(span_locations)] + let (lo, frame) = frame; + let (open_delimiter, outer) = frame; + if open_delimiter != close_delimiter { + return Err(lex_error(input)); + } + input = input.advance(1); + let mut g = Group::new(open_delimiter, trees.build()); + g.set_span(Span { + #[cfg(span_locations)] + lo, + #[cfg(span_locations)] + hi: input.off, + }); + trees = outer; + trees.push_token_from_parser(TokenTree::Group(crate::Group::_new_fallback(g))); + } else { + let (rest, mut tt) = match leaf_token(input) { + Ok((rest, tt)) => (rest, tt), + Err(Reject) => return Err(lex_error(input)), + }; + tt.set_span(crate::Span::_new_fallback(Span { + #[cfg(span_locations)] + lo, + #[cfg(span_locations)] + hi: rest.off, + })); + trees.push_token_from_parser(tt); + input = rest; + } + } +} + +fn lex_error(cursor: Cursor) -> LexError { + #[cfg(not(span_locations))] + let _ = cursor; + LexError { + span: Span { + #[cfg(span_locations)] + lo: cursor.off, + #[cfg(span_locations)] + hi: cursor.off, + }, + } +} + +fn leaf_token(input: Cursor) -> PResult { + if let Ok((input, l)) = literal(input) { + // must be parsed before ident + Ok((input, TokenTree::Literal(crate::Literal::_new_fallback(l)))) + } else if let Ok((input, p)) = punct(input) { + Ok((input, TokenTree::Punct(p))) + } else if let Ok((input, i)) = ident(input) { + Ok((input, TokenTree::Ident(i))) + } else if input.starts_with(ERROR) { + let rest = input.advance(ERROR.len()); + let repr = crate::Literal::_new_fallback(Literal::_new(ERROR.to_owned())); + Ok((rest, TokenTree::Literal(repr))) + } else { + Err(Reject) + } +} + +fn ident(input: Cursor) -> PResult { + if [ + "r\"", "r#\"", "r##", "b\"", "b\'", "br\"", "br#", "c\"", "cr\"", "cr#", + ] + .iter() + .any(|prefix| input.starts_with(prefix)) + { + Err(Reject) + } else { + ident_any(input) + } +} + +fn ident_any(input: Cursor) -> PResult { + let raw = input.starts_with("r#"); + let rest = input.advance((raw as usize) << 1); + + let (rest, sym) = ident_not_raw(rest)?; + + if !raw { + let ident = + crate::Ident::_new_fallback(Ident::new_unchecked(sym, fallback::Span::call_site())); + return Ok((rest, ident)); + } + + match sym { + "_" | "super" | "self" | "Self" | "crate" => return Err(Reject), + _ => {} + } + + let ident = + crate::Ident::_new_fallback(Ident::new_raw_unchecked(sym, fallback::Span::call_site())); + Ok((rest, ident)) +} + +fn ident_not_raw(input: Cursor) -> PResult<&str> { + let mut chars = input.char_indices(); + + match chars.next() { + Some((_, ch)) if is_ident_start(ch) => {} + _ => return Err(Reject), + } + + let mut end = input.len(); + for (i, ch) in chars { + if !is_ident_continue(ch) { + end = i; + break; + } + } + + Ok((input.advance(end), &input.rest[..end])) +} + +pub(crate) fn literal(input: Cursor) -> PResult { + let rest = literal_nocapture(input)?; + let end = input.len() - rest.len(); + Ok((rest, Literal::_new(input.rest[..end].to_string()))) +} + +fn literal_nocapture(input: Cursor) -> Result { + if let Ok(ok) = string(input) { + Ok(ok) + } else if let Ok(ok) = byte_string(input) { + Ok(ok) + } else if let Ok(ok) = c_string(input) { + Ok(ok) + } else if let Ok(ok) = byte(input) { + Ok(ok) + } else if let Ok(ok) = character(input) { + Ok(ok) + } else if let Ok(ok) = float(input) { + Ok(ok) + } else if let Ok(ok) = int(input) { + Ok(ok) + } else { + Err(Reject) + } +} + +fn literal_suffix(input: Cursor) -> Cursor { + match ident_not_raw(input) { + Ok((input, _)) => input, + Err(Reject) => input, + } +} + +fn string(input: Cursor) -> Result { + if let Ok(input) = input.parse("\"") { + cooked_string(input) + } else if let Ok(input) = input.parse("r") { + raw_string(input) + } else { + Err(Reject) + } +} + +fn cooked_string(mut input: Cursor) -> Result { + let mut chars = input.char_indices(); + + while let Some((i, ch)) = chars.next() { + match ch { + '"' => { + let input = input.advance(i + 1); + return Ok(literal_suffix(input)); + } + '\r' => match chars.next() { + Some((_, '\n')) => {} + _ => break, + }, + '\\' => match chars.next() { + Some((_, 'x')) => { + backslash_x_char(&mut chars)?; + } + Some((_, 'n' | 'r' | 't' | '\\' | '\'' | '"' | '0')) => {} + Some((_, 'u')) => { + backslash_u(&mut chars)?; + } + Some((newline, ch @ ('\n' | '\r'))) => { + input = input.advance(newline + 1); + trailing_backslash(&mut input, ch as u8)?; + chars = input.char_indices(); + } + _ => break, + }, + _ch => {} + } + } + Err(Reject) +} + +fn raw_string(input: Cursor) -> Result { + let (input, delimiter) = delimiter_of_raw_string(input)?; + let mut bytes = input.bytes().enumerate(); + while let Some((i, byte)) = bytes.next() { + match byte { + b'"' if input.rest[i + 1..].starts_with(delimiter) => { + let rest = input.advance(i + 1 + delimiter.len()); + return Ok(literal_suffix(rest)); + } + b'\r' => match bytes.next() { + Some((_, b'\n')) => {} + _ => break, + }, + _ => {} + } + } + Err(Reject) +} + +fn byte_string(input: Cursor) -> Result { + if let Ok(input) = input.parse("b\"") { + cooked_byte_string(input) + } else if let Ok(input) = input.parse("br") { + raw_byte_string(input) + } else { + Err(Reject) + } +} + +fn cooked_byte_string(mut input: Cursor) -> Result { + let mut bytes = input.bytes().enumerate(); + while let Some((offset, b)) = bytes.next() { + match b { + b'"' => { + let input = input.advance(offset + 1); + return Ok(literal_suffix(input)); + } + b'\r' => match bytes.next() { + Some((_, b'\n')) => {} + _ => break, + }, + b'\\' => match bytes.next() { + Some((_, b'x')) => { + backslash_x_byte(&mut bytes)?; + } + Some((_, b'n' | b'r' | b't' | b'\\' | b'0' | b'\'' | b'"')) => {} + Some((newline, b @ (b'\n' | b'\r'))) => { + input = input.advance(newline + 1); + trailing_backslash(&mut input, b)?; + bytes = input.bytes().enumerate(); + } + _ => break, + }, + b if b.is_ascii() => {} + _ => break, + } + } + Err(Reject) +} + +fn delimiter_of_raw_string(input: Cursor) -> PResult<&str> { + for (i, byte) in input.bytes().enumerate() { + match byte { + b'"' => { + if i > 255 { + // https://github.com/rust-lang/rust/pull/95251 + return Err(Reject); + } + return Ok((input.advance(i + 1), &input.rest[..i])); + } + b'#' => {} + _ => break, + } + } + Err(Reject) +} + +fn raw_byte_string(input: Cursor) -> Result { + let (input, delimiter) = delimiter_of_raw_string(input)?; + let mut bytes = input.bytes().enumerate(); + while let Some((i, byte)) = bytes.next() { + match byte { + b'"' if input.rest[i + 1..].starts_with(delimiter) => { + let rest = input.advance(i + 1 + delimiter.len()); + return Ok(literal_suffix(rest)); + } + b'\r' => match bytes.next() { + Some((_, b'\n')) => {} + _ => break, + }, + other => { + if !other.is_ascii() { + break; + } + } + } + } + Err(Reject) +} + +fn c_string(input: Cursor) -> Result { + if let Ok(input) = input.parse("c\"") { + cooked_c_string(input) + } else if let Ok(input) = input.parse("cr") { + raw_c_string(input) + } else { + Err(Reject) + } +} + +fn raw_c_string(input: Cursor) -> Result { + let (input, delimiter) = delimiter_of_raw_string(input)?; + let mut bytes = input.bytes().enumerate(); + while let Some((i, byte)) = bytes.next() { + match byte { + b'"' if input.rest[i + 1..].starts_with(delimiter) => { + let rest = input.advance(i + 1 + delimiter.len()); + return Ok(literal_suffix(rest)); + } + b'\r' => match bytes.next() { + Some((_, b'\n')) => {} + _ => break, + }, + b'\0' => break, + _ => {} + } + } + Err(Reject) +} + +fn cooked_c_string(mut input: Cursor) -> Result { + let mut chars = input.char_indices(); + + while let Some((i, ch)) = chars.next() { + match ch { + '"' => { + let input = input.advance(i + 1); + return Ok(literal_suffix(input)); + } + '\r' => match chars.next() { + Some((_, '\n')) => {} + _ => break, + }, + '\\' => match chars.next() { + Some((_, 'x')) => { + backslash_x_nonzero(&mut chars)?; + } + Some((_, 'n' | 'r' | 't' | '\\' | '\'' | '"')) => {} + Some((_, 'u')) => { + if backslash_u(&mut chars)? == '\0' { + break; + } + } + Some((newline, ch @ ('\n' | '\r'))) => { + input = input.advance(newline + 1); + trailing_backslash(&mut input, ch as u8)?; + chars = input.char_indices(); + } + _ => break, + }, + '\0' => break, + _ch => {} + } + } + Err(Reject) +} + +fn byte(input: Cursor) -> Result { + let input = input.parse("b'")?; + let mut bytes = input.bytes().enumerate(); + let ok = match bytes.next().map(|(_, b)| b) { + Some(b'\\') => match bytes.next().map(|(_, b)| b) { + Some(b'x') => backslash_x_byte(&mut bytes).is_ok(), + Some(b'n' | b'r' | b't' | b'\\' | b'0' | b'\'' | b'"') => true, + _ => false, + }, + b => b.is_some(), + }; + if !ok { + return Err(Reject); + } + let (offset, _) = bytes.next().ok_or(Reject)?; + if !input.chars().as_str().is_char_boundary(offset) { + return Err(Reject); + } + let input = input.advance(offset).parse("'")?; + Ok(literal_suffix(input)) +} + +fn character(input: Cursor) -> Result { + let input = input.parse("'")?; + let mut chars = input.char_indices(); + let ok = match chars.next().map(|(_, ch)| ch) { + Some('\\') => match chars.next().map(|(_, ch)| ch) { + Some('x') => backslash_x_char(&mut chars).is_ok(), + Some('u') => backslash_u(&mut chars).is_ok(), + Some('n' | 'r' | 't' | '\\' | '0' | '\'' | '"') => true, + _ => false, + }, + ch => ch.is_some(), + }; + if !ok { + return Err(Reject); + } + let (idx, _) = chars.next().ok_or(Reject)?; + let input = input.advance(idx).parse("'")?; + Ok(literal_suffix(input)) +} + +macro_rules! next_ch { + ($chars:ident @ $pat:pat) => { + match $chars.next() { + Some((_, ch)) => match ch { + $pat => ch, + _ => return Err(Reject), + }, + None => return Err(Reject), + } + }; +} + +fn backslash_x_char(chars: &mut I) -> Result<(), Reject> +where + I: Iterator, +{ + next_ch!(chars @ '0'..='7'); + next_ch!(chars @ '0'..='9' | 'a'..='f' | 'A'..='F'); + Ok(()) +} + +fn backslash_x_byte(chars: &mut I) -> Result<(), Reject> +where + I: Iterator, +{ + next_ch!(chars @ b'0'..=b'9' | b'a'..=b'f' | b'A'..=b'F'); + next_ch!(chars @ b'0'..=b'9' | b'a'..=b'f' | b'A'..=b'F'); + Ok(()) +} + +fn backslash_x_nonzero(chars: &mut I) -> Result<(), Reject> +where + I: Iterator, +{ + let first = next_ch!(chars @ '0'..='9' | 'a'..='f' | 'A'..='F'); + let second = next_ch!(chars @ '0'..='9' | 'a'..='f' | 'A'..='F'); + if first == '0' && second == '0' { + Err(Reject) + } else { + Ok(()) + } +} + +fn backslash_u(chars: &mut I) -> Result +where + I: Iterator, +{ + next_ch!(chars @ '{'); + let mut value = 0; + let mut len = 0; + for (_, ch) in chars { + let digit = match ch { + '0'..='9' => ch as u8 - b'0', + 'a'..='f' => 10 + ch as u8 - b'a', + 'A'..='F' => 10 + ch as u8 - b'A', + '_' if len > 0 => continue, + '}' if len > 0 => return char::from_u32(value).ok_or(Reject), + _ => break, + }; + if len == 6 { + break; + } + value *= 0x10; + value += u32::from(digit); + len += 1; + } + Err(Reject) +} + +fn trailing_backslash(input: &mut Cursor, mut last: u8) -> Result<(), Reject> { + let mut whitespace = input.bytes().enumerate(); + loop { + if last == b'\r' && whitespace.next().map_or(true, |(_, b)| b != b'\n') { + return Err(Reject); + } + match whitespace.next() { + Some((_, b @ (b' ' | b'\t' | b'\n' | b'\r'))) => { + last = b; + } + Some((offset, _)) => { + *input = input.advance(offset); + return Ok(()); + } + None => return Err(Reject), + } + } +} + +fn float(input: Cursor) -> Result { + let mut rest = float_digits(input)?; + if let Some(ch) = rest.chars().next() { + if is_ident_start(ch) { + rest = ident_not_raw(rest)?.0; + } + } + word_break(rest) +} + +fn float_digits(input: Cursor) -> Result { + let mut chars = input.chars().peekable(); + match chars.next() { + Some(ch) if '0' <= ch && ch <= '9' => {} + _ => return Err(Reject), + } + + let mut len = 1; + let mut has_dot = false; + let mut has_exp = false; + while let Some(&ch) = chars.peek() { + match ch { + '0'..='9' | '_' => { + chars.next(); + len += 1; + } + '.' => { + if has_dot { + break; + } + chars.next(); + if chars + .peek() + .map_or(false, |&ch| ch == '.' || is_ident_start(ch)) + { + return Err(Reject); + } + len += 1; + has_dot = true; + } + 'e' | 'E' => { + chars.next(); + len += 1; + has_exp = true; + break; + } + _ => break, + } + } + + if !(has_dot || has_exp) { + return Err(Reject); + } + + if has_exp { + let token_before_exp = if has_dot { + Ok(input.advance(len - 1)) + } else { + Err(Reject) + }; + let mut has_sign = false; + let mut has_exp_value = false; + while let Some(&ch) = chars.peek() { + match ch { + '+' | '-' => { + if has_exp_value { + break; + } + if has_sign { + return token_before_exp; + } + chars.next(); + len += 1; + has_sign = true; + } + '0'..='9' => { + chars.next(); + len += 1; + has_exp_value = true; + } + '_' => { + chars.next(); + len += 1; + } + _ => break, + } + } + if !has_exp_value { + return token_before_exp; + } + } + + Ok(input.advance(len)) +} + +fn int(input: Cursor) -> Result { + let mut rest = digits(input)?; + if let Some(ch) = rest.chars().next() { + if is_ident_start(ch) { + rest = ident_not_raw(rest)?.0; + } + } + word_break(rest) +} + +fn digits(mut input: Cursor) -> Result { + let base = if input.starts_with("0x") { + input = input.advance(2); + 16 + } else if input.starts_with("0o") { + input = input.advance(2); + 8 + } else if input.starts_with("0b") { + input = input.advance(2); + 2 + } else { + 10 + }; + + let mut len = 0; + let mut empty = true; + for b in input.bytes() { + match b { + b'0'..=b'9' => { + let digit = (b - b'0') as u64; + if digit >= base { + return Err(Reject); + } + } + b'a'..=b'f' => { + let digit = 10 + (b - b'a') as u64; + if digit >= base { + break; + } + } + b'A'..=b'F' => { + let digit = 10 + (b - b'A') as u64; + if digit >= base { + break; + } + } + b'_' => { + if empty && base == 10 { + return Err(Reject); + } + len += 1; + continue; + } + _ => break, + } + len += 1; + empty = false; + } + if empty { + Err(Reject) + } else { + Ok(input.advance(len)) + } +} + +fn punct(input: Cursor) -> PResult { + let (rest, ch) = punct_char(input)?; + if ch == '\'' { + let (after_lifetime, _ident) = ident_any(rest)?; + if after_lifetime.starts_with_char('\'') + || (after_lifetime.starts_with_char('#') && !rest.starts_with("r#")) + { + Err(Reject) + } else { + Ok((rest, Punct::new('\'', Spacing::Joint))) + } + } else { + let kind = match punct_char(rest) { + Ok(_) => Spacing::Joint, + Err(Reject) => Spacing::Alone, + }; + Ok((rest, Punct::new(ch, kind))) + } +} + +fn punct_char(input: Cursor) -> PResult { + if input.starts_with("//") || input.starts_with("/*") { + // Do not accept `/` of a comment as a punct. + return Err(Reject); + } + + let mut chars = input.chars(); + let first = match chars.next() { + Some(ch) => ch, + None => { + return Err(Reject); + } + }; + let recognized = "~!@#$%^&*-=+|;:,<.>/?'"; + if recognized.contains(first) { + Ok((input.advance(first.len_utf8()), first)) + } else { + Err(Reject) + } +} + +fn doc_comment<'a>(input: Cursor<'a>, trees: &mut TokenStreamBuilder) -> PResult<'a, ()> { + #[cfg(span_locations)] + let lo = input.off; + let (rest, (comment, inner)) = doc_comment_contents(input)?; + let fallback_span = Span { + #[cfg(span_locations)] + lo, + #[cfg(span_locations)] + hi: rest.off, + }; + let span = crate::Span::_new_fallback(fallback_span); + + let mut scan_for_bare_cr = comment; + while let Some(cr) = scan_for_bare_cr.find('\r') { + let rest = &scan_for_bare_cr[cr + 1..]; + if !rest.starts_with('\n') { + return Err(Reject); + } + scan_for_bare_cr = rest; + } + + let mut pound = Punct::new('#', Spacing::Alone); + pound.set_span(span); + trees.push_token_from_parser(TokenTree::Punct(pound)); + + if inner { + let mut bang = Punct::new('!', Spacing::Alone); + bang.set_span(span); + trees.push_token_from_parser(TokenTree::Punct(bang)); + } + + let doc_ident = crate::Ident::_new_fallback(Ident::new_unchecked("doc", fallback_span)); + let mut equal = Punct::new('=', Spacing::Alone); + equal.set_span(span); + let mut literal = crate::Literal::_new_fallback(Literal::string(comment)); + literal.set_span(span); + let mut bracketed = TokenStreamBuilder::with_capacity(3); + bracketed.push_token_from_parser(TokenTree::Ident(doc_ident)); + bracketed.push_token_from_parser(TokenTree::Punct(equal)); + bracketed.push_token_from_parser(TokenTree::Literal(literal)); + let group = Group::new(Delimiter::Bracket, bracketed.build()); + let mut group = crate::Group::_new_fallback(group); + group.set_span(span); + trees.push_token_from_parser(TokenTree::Group(group)); + + Ok((rest, ())) +} + +fn doc_comment_contents(input: Cursor) -> PResult<(&str, bool)> { + if input.starts_with("//!") { + let input = input.advance(3); + let (input, s) = take_until_newline_or_eof(input); + Ok((input, (s, true))) + } else if input.starts_with("/*!") { + let (input, s) = block_comment(input)?; + Ok((input, (&s[3..s.len() - 2], true))) + } else if input.starts_with("///") { + let input = input.advance(3); + if input.starts_with_char('/') { + return Err(Reject); + } + let (input, s) = take_until_newline_or_eof(input); + Ok((input, (s, false))) + } else if input.starts_with("/**") && !input.rest[3..].starts_with('*') { + let (input, s) = block_comment(input)?; + Ok((input, (&s[3..s.len() - 2], false))) + } else { + Err(Reject) + } +} + +fn take_until_newline_or_eof(input: Cursor) -> (Cursor, &str) { + let chars = input.char_indices(); + + for (i, ch) in chars { + if ch == '\n' { + return (input.advance(i), &input.rest[..i]); + } else if ch == '\r' && input.rest[i + 1..].starts_with('\n') { + return (input.advance(i + 1), &input.rest[..i]); + } + } + + (input.advance(input.len()), input.rest) +} diff --git a/rust/proc-macro2/probe.rs b/rust/proc-macro2/probe.rs new file mode 100644 index 000000000000..b67f52036218 --- /dev/null +++ b/rust/proc-macro2/probe.rs @@ -0,0 +1,10 @@ +#![allow(dead_code)] + +#[cfg(proc_macro_span)] +pub(crate) mod proc_macro_span; + +#[cfg(proc_macro_span_file)] +pub(crate) mod proc_macro_span_file; + +#[cfg(proc_macro_span_location)] +pub(crate) mod proc_macro_span_location; diff --git a/rust/proc-macro2/probe/proc_macro_span.rs b/rust/proc-macro2/probe/proc_macro_span.rs new file mode 100644 index 000000000000..2d7d44e07708 --- /dev/null +++ b/rust/proc-macro2/probe/proc_macro_span.rs @@ -0,0 +1,51 @@ +// This code exercises the surface area that we expect of Span's unstable API. +// If the current toolchain is able to compile it, then proc-macro2 is able to +// offer these APIs too. + +#![cfg_attr(procmacro2_build_probe, feature(proc_macro_span))] + +extern crate proc_macro; + +use core::ops::{Range, RangeBounds}; +use proc_macro::{Literal, Span}; +use std::path::PathBuf; + +pub fn byte_range(this: &Span) -> Range { + this.byte_range() +} + +pub fn start(this: &Span) -> Span { + this.start() +} + +pub fn end(this: &Span) -> Span { + this.end() +} + +pub fn line(this: &Span) -> usize { + this.line() +} + +pub fn column(this: &Span) -> usize { + this.column() +} + +pub fn file(this: &Span) -> String { + this.file() +} + +pub fn local_file(this: &Span) -> Option { + this.local_file() +} + +pub fn join(this: &Span, other: Span) -> Option { + this.join(other) +} + +pub fn subspan>(this: &Literal, range: R) -> Option { + this.subspan(range) +} + +// Include in sccache cache key. +#[cfg(procmacro2_build_probe)] +const _: Option<&str> = option_env!("RUSTC_BOOTSTRAP"); diff --git a/rust/proc-macro2/probe/proc_macro_span_file.rs b/rust/proc-macro2/probe/proc_macro_span_file.rs new file mode 100644 index 000000000000..8b76bdf5007b --- /dev/null +++ b/rust/proc-macro2/probe/proc_macro_span_file.rs @@ -0,0 +1,14 @@ +// The subset of Span's API stabilized in Rust 1.88. + +extern crate proc_macro; + +use proc_macro::Span; +use std::path::PathBuf; + +pub fn file(this: &Span) -> String { + this.file() +} + +pub fn local_file(this: &Span) -> Option { + this.local_file() +} diff --git a/rust/proc-macro2/probe/proc_macro_span_location.rs b/rust/proc-macro2/probe/proc_macro_span_location.rs new file mode 100644 index 000000000000..79da34af54af --- /dev/null +++ b/rust/proc-macro2/probe/proc_macro_span_location.rs @@ -0,0 +1,21 @@ +// The subset of Span's API stabilized in Rust 1.88. + +extern crate proc_macro; + +use proc_macro::Span; + +pub fn start(this: &Span) -> Span { + this.start() +} + +pub fn end(this: &Span) -> Span { + this.end() +} + +pub fn line(this: &Span) -> usize { + this.line() +} + +pub fn column(this: &Span) -> usize { + this.column() +} diff --git a/rust/proc-macro2/rcvec.rs b/rust/proc-macro2/rcvec.rs new file mode 100644 index 000000000000..23edc77d597f --- /dev/null +++ b/rust/proc-macro2/rcvec.rs @@ -0,0 +1,146 @@ +use alloc::rc::Rc; +use alloc::vec; +use core::mem; +use core::panic::RefUnwindSafe; +use core::slice; + +pub(crate) struct RcVec { + inner: Rc>, +} + +pub(crate) struct RcVecBuilder { + inner: Vec, +} + +pub(crate) struct RcVecMut<'a, T> { + inner: &'a mut Vec, +} + +#[derive(Clone)] +pub(crate) struct RcVecIntoIter { + inner: vec::IntoIter, +} + +impl RcVec { + pub(crate) fn is_empty(&self) -> bool { + self.inner.is_empty() + } + + pub(crate) fn len(&self) -> usize { + self.inner.len() + } + + pub(crate) fn iter(&self) -> slice::Iter { + self.inner.iter() + } + + pub(crate) fn make_mut(&mut self) -> RcVecMut + where + T: Clone, + { + RcVecMut { + inner: Rc::make_mut(&mut self.inner), + } + } + + pub(crate) fn get_mut(&mut self) -> Option> { + let inner = Rc::get_mut(&mut self.inner)?; + Some(RcVecMut { inner }) + } + + pub(crate) fn make_owned(mut self) -> RcVecBuilder + where + T: Clone, + { + let vec = if let Some(owned) = Rc::get_mut(&mut self.inner) { + mem::take(owned) + } else { + Vec::clone(&self.inner) + }; + RcVecBuilder { inner: vec } + } +} + +impl RcVecBuilder { + pub(crate) fn new() -> Self { + RcVecBuilder { inner: Vec::new() } + } + + pub(crate) fn with_capacity(cap: usize) -> Self { + RcVecBuilder { + inner: Vec::with_capacity(cap), + } + } + + pub(crate) fn push(&mut self, element: T) { + self.inner.push(element); + } + + pub(crate) fn extend(&mut self, iter: impl IntoIterator) { + self.inner.extend(iter); + } + + pub(crate) fn as_mut(&mut self) -> RcVecMut { + RcVecMut { + inner: &mut self.inner, + } + } + + pub(crate) fn build(self) -> RcVec { + RcVec { + inner: Rc::new(self.inner), + } + } +} + +impl<'a, T> RcVecMut<'a, T> { + pub(crate) fn push(&mut self, element: T) { + self.inner.push(element); + } + + pub(crate) fn extend(&mut self, iter: impl IntoIterator) { + self.inner.extend(iter); + } + + pub(crate) fn as_mut(&mut self) -> RcVecMut { + RcVecMut { inner: self.inner } + } + + pub(crate) fn take(self) -> RcVecBuilder { + let vec = mem::take(self.inner); + RcVecBuilder { inner: vec } + } +} + +impl Clone for RcVec { + fn clone(&self) -> Self { + RcVec { + inner: Rc::clone(&self.inner), + } + } +} + +impl IntoIterator for RcVecBuilder { + type Item = T; + type IntoIter = RcVecIntoIter; + + fn into_iter(self) -> Self::IntoIter { + RcVecIntoIter { + inner: self.inner.into_iter(), + } + } +} + +impl Iterator for RcVecIntoIter { + type Item = T; + + fn next(&mut self) -> Option { + self.inner.next() + } + + fn size_hint(&self) -> (usize, Option) { + self.inner.size_hint() + } +} + +impl RefUnwindSafe for RcVec where T: RefUnwindSafe {} diff --git a/rust/proc-macro2/wrapper.rs b/rust/proc-macro2/wrapper.rs new file mode 100644 index 000000000000..2e3eb5b4d04e --- /dev/null +++ b/rust/proc-macro2/wrapper.rs @@ -0,0 +1,984 @@ +use crate::detection::inside_proc_macro; +use crate::fallback::{self, FromStr2 as _}; +#[cfg(span_locations)] +use crate::location::LineColumn; +#[cfg(proc_macro_span)] +use crate::probe::proc_macro_span; +#[cfg(all(span_locations, proc_macro_span_file))] +use crate::probe::proc_macro_span_file; +#[cfg(all(span_locations, proc_macro_span_location))] +use crate::probe::proc_macro_span_location; +use crate::{Delimiter, Punct, Spacing, TokenTree}; +use core::fmt::{self, Debug, Display}; +#[cfg(span_locations)] +use core::ops::Range; +use core::ops::RangeBounds; +use std::ffi::CStr; +#[cfg(span_locations)] +use std::path::PathBuf; + +#[derive(Clone)] +pub(crate) enum TokenStream { + Compiler(DeferredTokenStream), + Fallback(fallback::TokenStream), +} + +// Work around https://github.com/rust-lang/rust/issues/65080. +// In `impl Extend for TokenStream` which is used heavily by quote, +// we hold on to the appended tokens and do proc_macro::TokenStream::extend as +// late as possible to batch together consecutive uses of the Extend impl. +#[derive(Clone)] +pub(crate) struct DeferredTokenStream { + stream: proc_macro::TokenStream, + extra: Vec, +} + +pub(crate) enum LexError { + Compiler(proc_macro::LexError), + Fallback(fallback::LexError), + + // Rustc was supposed to return a LexError, but it panicked instead. + // https://github.com/rust-lang/rust/issues/58736 + CompilerPanic, +} + +#[cold] +fn mismatch(line: u32) -> ! { + #[cfg(procmacro2_backtrace)] + { + let backtrace = std::backtrace::Backtrace::force_capture(); + panic!("compiler/fallback mismatch L{}\n\n{}", line, backtrace) + } + #[cfg(not(procmacro2_backtrace))] + { + panic!("compiler/fallback mismatch L{}", line) + } +} + +impl DeferredTokenStream { + fn new(stream: proc_macro::TokenStream) -> Self { + DeferredTokenStream { + stream, + extra: Vec::new(), + } + } + + fn is_empty(&self) -> bool { + self.stream.is_empty() && self.extra.is_empty() + } + + fn evaluate_now(&mut self) { + // If-check provides a fast short circuit for the common case of `extra` + // being empty, which saves a round trip over the proc macro bridge. + // Improves macro expansion time in winrt by 6% in debug mode. + if !self.extra.is_empty() { + self.stream.extend(self.extra.drain(..)); + } + } + + fn into_token_stream(mut self) -> proc_macro::TokenStream { + self.evaluate_now(); + self.stream + } +} + +impl TokenStream { + pub(crate) fn new() -> Self { + if inside_proc_macro() { + TokenStream::Compiler(DeferredTokenStream::new(proc_macro::TokenStream::new())) + } else { + TokenStream::Fallback(fallback::TokenStream::new()) + } + } + + pub(crate) fn from_str_checked(src: &str) -> Result { + if inside_proc_macro() { + Ok(TokenStream::Compiler(DeferredTokenStream::new( + proc_macro::TokenStream::from_str_checked(src)?, + ))) + } else { + Ok(TokenStream::Fallback( + fallback::TokenStream::from_str_checked(src)?, + )) + } + } + + pub(crate) fn is_empty(&self) -> bool { + match self { + TokenStream::Compiler(tts) => tts.is_empty(), + TokenStream::Fallback(tts) => tts.is_empty(), + } + } + + fn unwrap_nightly(self) -> proc_macro::TokenStream { + match self { + TokenStream::Compiler(s) => s.into_token_stream(), + TokenStream::Fallback(_) => mismatch(line!()), + } + } + + fn unwrap_stable(self) -> fallback::TokenStream { + match self { + TokenStream::Compiler(_) => mismatch(line!()), + TokenStream::Fallback(s) => s, + } + } +} + +impl Display for TokenStream { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + match self { + TokenStream::Compiler(tts) => Display::fmt(&tts.clone().into_token_stream(), f), + TokenStream::Fallback(tts) => Display::fmt(tts, f), + } + } +} + +impl From for TokenStream { + fn from(inner: proc_macro::TokenStream) -> Self { + TokenStream::Compiler(DeferredTokenStream::new(inner)) + } +} + +impl From for proc_macro::TokenStream { + fn from(inner: TokenStream) -> Self { + match inner { + TokenStream::Compiler(inner) => inner.into_token_stream(), + TokenStream::Fallback(inner) => { + proc_macro::TokenStream::from_str_unchecked(&inner.to_string()) + } + } + } +} + +impl From for TokenStream { + fn from(inner: fallback::TokenStream) -> Self { + TokenStream::Fallback(inner) + } +} + +// Assumes inside_proc_macro(). +fn into_compiler_token(token: TokenTree) -> proc_macro::TokenTree { + match token { + TokenTree::Group(tt) => proc_macro::TokenTree::Group(tt.inner.unwrap_nightly()), + TokenTree::Punct(tt) => { + let spacing = match tt.spacing() { + Spacing::Joint => proc_macro::Spacing::Joint, + Spacing::Alone => proc_macro::Spacing::Alone, + }; + let mut punct = proc_macro::Punct::new(tt.as_char(), spacing); + punct.set_span(tt.span().inner.unwrap_nightly()); + proc_macro::TokenTree::Punct(punct) + } + TokenTree::Ident(tt) => proc_macro::TokenTree::Ident(tt.inner.unwrap_nightly()), + TokenTree::Literal(tt) => proc_macro::TokenTree::Literal(tt.inner.unwrap_nightly()), + } +} + +impl From for TokenStream { + fn from(token: TokenTree) -> Self { + if inside_proc_macro() { + TokenStream::Compiler(DeferredTokenStream::new(proc_macro::TokenStream::from( + into_compiler_token(token), + ))) + } else { + TokenStream::Fallback(fallback::TokenStream::from(token)) + } + } +} + +impl FromIterator for TokenStream { + fn from_iter>(trees: I) -> Self { + if inside_proc_macro() { + TokenStream::Compiler(DeferredTokenStream::new( + trees.into_iter().map(into_compiler_token).collect(), + )) + } else { + TokenStream::Fallback(trees.into_iter().collect()) + } + } +} + +impl FromIterator for TokenStream { + fn from_iter>(streams: I) -> Self { + let mut streams = streams.into_iter(); + match streams.next() { + Some(TokenStream::Compiler(mut first)) => { + first.evaluate_now(); + first.stream.extend(streams.map(|s| match s { + TokenStream::Compiler(s) => s.into_token_stream(), + TokenStream::Fallback(_) => mismatch(line!()), + })); + TokenStream::Compiler(first) + } + Some(TokenStream::Fallback(mut first)) => { + first.extend(streams.map(|s| match s { + TokenStream::Fallback(s) => s, + TokenStream::Compiler(_) => mismatch(line!()), + })); + TokenStream::Fallback(first) + } + None => TokenStream::new(), + } + } +} + +impl Extend for TokenStream { + fn extend>(&mut self, stream: I) { + match self { + TokenStream::Compiler(tts) => { + // Here is the reason for DeferredTokenStream. + for token in stream { + tts.extra.push(into_compiler_token(token)); + } + } + TokenStream::Fallback(tts) => tts.extend(stream), + } + } +} + +impl Extend for TokenStream { + fn extend>(&mut self, streams: I) { + match self { + TokenStream::Compiler(tts) => { + tts.evaluate_now(); + tts.stream + .extend(streams.into_iter().map(TokenStream::unwrap_nightly)); + } + TokenStream::Fallback(tts) => { + tts.extend(streams.into_iter().map(TokenStream::unwrap_stable)); + } + } + } +} + +impl Debug for TokenStream { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + match self { + TokenStream::Compiler(tts) => Debug::fmt(&tts.clone().into_token_stream(), f), + TokenStream::Fallback(tts) => Debug::fmt(tts, f), + } + } +} + +impl LexError { + pub(crate) fn span(&self) -> Span { + match self { + LexError::Compiler(_) | LexError::CompilerPanic => Span::call_site(), + LexError::Fallback(e) => Span::Fallback(e.span()), + } + } +} + +impl From for LexError { + fn from(e: proc_macro::LexError) -> Self { + LexError::Compiler(e) + } +} + +impl From for LexError { + fn from(e: fallback::LexError) -> Self { + LexError::Fallback(e) + } +} + +impl Debug for LexError { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + match self { + LexError::Compiler(e) => Debug::fmt(e, f), + LexError::Fallback(e) => Debug::fmt(e, f), + LexError::CompilerPanic => { + let fallback = fallback::LexError::call_site(); + Debug::fmt(&fallback, f) + } + } + } +} + +impl Display for LexError { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + match self { + LexError::Compiler(e) => Display::fmt(e, f), + LexError::Fallback(e) => Display::fmt(e, f), + LexError::CompilerPanic => { + let fallback = fallback::LexError::call_site(); + Display::fmt(&fallback, f) + } + } + } +} + +#[derive(Clone)] +pub(crate) enum TokenTreeIter { + Compiler(proc_macro::token_stream::IntoIter), + Fallback(fallback::TokenTreeIter), +} + +impl IntoIterator for TokenStream { + type Item = TokenTree; + type IntoIter = TokenTreeIter; + + fn into_iter(self) -> TokenTreeIter { + match self { + TokenStream::Compiler(tts) => { + TokenTreeIter::Compiler(tts.into_token_stream().into_iter()) + } + TokenStream::Fallback(tts) => TokenTreeIter::Fallback(tts.into_iter()), + } + } +} + +impl Iterator for TokenTreeIter { + type Item = TokenTree; + + fn next(&mut self) -> Option { + let token = match self { + TokenTreeIter::Compiler(iter) => iter.next()?, + TokenTreeIter::Fallback(iter) => return iter.next(), + }; + Some(match token { + proc_macro::TokenTree::Group(tt) => { + TokenTree::Group(crate::Group::_new(Group::Compiler(tt))) + } + proc_macro::TokenTree::Punct(tt) => { + let spacing = match tt.spacing() { + proc_macro::Spacing::Joint => Spacing::Joint, + proc_macro::Spacing::Alone => Spacing::Alone, + }; + let mut o = Punct::new(tt.as_char(), spacing); + o.set_span(crate::Span::_new(Span::Compiler(tt.span()))); + TokenTree::Punct(o) + } + proc_macro::TokenTree::Ident(s) => { + TokenTree::Ident(crate::Ident::_new(Ident::Compiler(s))) + } + proc_macro::TokenTree::Literal(l) => { + TokenTree::Literal(crate::Literal::_new(Literal::Compiler(l))) + } + }) + } + + fn size_hint(&self) -> (usize, Option) { + match self { + TokenTreeIter::Compiler(tts) => tts.size_hint(), + TokenTreeIter::Fallback(tts) => tts.size_hint(), + } + } +} + +#[derive(Copy, Clone)] +pub(crate) enum Span { + Compiler(proc_macro::Span), + Fallback(fallback::Span), +} + +impl Span { + pub(crate) fn call_site() -> Self { + if inside_proc_macro() { + Span::Compiler(proc_macro::Span::call_site()) + } else { + Span::Fallback(fallback::Span::call_site()) + } + } + + pub(crate) fn mixed_site() -> Self { + if inside_proc_macro() { + Span::Compiler(proc_macro::Span::mixed_site()) + } else { + Span::Fallback(fallback::Span::mixed_site()) + } + } + + #[cfg(super_unstable)] + pub(crate) fn def_site() -> Self { + if inside_proc_macro() { + Span::Compiler(proc_macro::Span::def_site()) + } else { + Span::Fallback(fallback::Span::def_site()) + } + } + + pub(crate) fn resolved_at(&self, other: Span) -> Span { + match (self, other) { + (Span::Compiler(a), Span::Compiler(b)) => Span::Compiler(a.resolved_at(b)), + (Span::Fallback(a), Span::Fallback(b)) => Span::Fallback(a.resolved_at(b)), + (Span::Compiler(_), Span::Fallback(_)) => mismatch(line!()), + (Span::Fallback(_), Span::Compiler(_)) => mismatch(line!()), + } + } + + pub(crate) fn located_at(&self, other: Span) -> Span { + match (self, other) { + (Span::Compiler(a), Span::Compiler(b)) => Span::Compiler(a.located_at(b)), + (Span::Fallback(a), Span::Fallback(b)) => Span::Fallback(a.located_at(b)), + (Span::Compiler(_), Span::Fallback(_)) => mismatch(line!()), + (Span::Fallback(_), Span::Compiler(_)) => mismatch(line!()), + } + } + + pub(crate) fn unwrap(self) -> proc_macro::Span { + match self { + Span::Compiler(s) => s, + Span::Fallback(_) => panic!("proc_macro::Span is only available in procedural macros"), + } + } + + #[cfg(span_locations)] + pub(crate) fn byte_range(&self) -> Range { + match self { + #[cfg(proc_macro_span)] + Span::Compiler(s) => proc_macro_span::byte_range(s), + #[cfg(not(proc_macro_span))] + Span::Compiler(_) => 0..0, + Span::Fallback(s) => s.byte_range(), + } + } + + #[cfg(span_locations)] + pub(crate) fn start(&self) -> LineColumn { + match self { + #[cfg(proc_macro_span_location)] + Span::Compiler(s) => LineColumn { + line: proc_macro_span_location::line(s), + column: proc_macro_span_location::column(s).saturating_sub(1), + }, + #[cfg(not(proc_macro_span_location))] + Span::Compiler(_) => LineColumn { line: 0, column: 0 }, + Span::Fallback(s) => s.start(), + } + } + + #[cfg(span_locations)] + pub(crate) fn end(&self) -> LineColumn { + match self { + #[cfg(proc_macro_span_location)] + Span::Compiler(s) => { + let end = proc_macro_span_location::end(s); + LineColumn { + line: proc_macro_span_location::line(&end), + column: proc_macro_span_location::column(&end).saturating_sub(1), + } + } + #[cfg(not(proc_macro_span_location))] + Span::Compiler(_) => LineColumn { line: 0, column: 0 }, + Span::Fallback(s) => s.end(), + } + } + + #[cfg(span_locations)] + pub(crate) fn file(&self) -> String { + match self { + #[cfg(proc_macro_span_file)] + Span::Compiler(s) => proc_macro_span_file::file(s), + #[cfg(not(proc_macro_span_file))] + Span::Compiler(_) => "".to_owned(), + Span::Fallback(s) => s.file(), + } + } + + #[cfg(span_locations)] + pub(crate) fn local_file(&self) -> Option { + match self { + #[cfg(proc_macro_span_file)] + Span::Compiler(s) => proc_macro_span_file::local_file(s), + #[cfg(not(proc_macro_span_file))] + Span::Compiler(_) => None, + Span::Fallback(s) => s.local_file(), + } + } + + pub(crate) fn join(&self, other: Span) -> Option { + let ret = match (self, other) { + #[cfg(proc_macro_span)] + (Span::Compiler(a), Span::Compiler(b)) => Span::Compiler(proc_macro_span::join(a, b)?), + (Span::Fallback(a), Span::Fallback(b)) => Span::Fallback(a.join(b)?), + _ => return None, + }; + Some(ret) + } + + #[cfg(super_unstable)] + pub(crate) fn eq(&self, other: &Span) -> bool { + match (self, other) { + (Span::Compiler(a), Span::Compiler(b)) => a.eq(b), + (Span::Fallback(a), Span::Fallback(b)) => a.eq(b), + _ => false, + } + } + + pub(crate) fn source_text(&self) -> Option { + match self { + #[cfg(not(no_source_text))] + Span::Compiler(s) => s.source_text(), + #[cfg(no_source_text)] + Span::Compiler(_) => None, + Span::Fallback(s) => s.source_text(), + } + } + + fn unwrap_nightly(self) -> proc_macro::Span { + match self { + Span::Compiler(s) => s, + Span::Fallback(_) => mismatch(line!()), + } + } +} + +impl From for crate::Span { + fn from(proc_span: proc_macro::Span) -> Self { + crate::Span::_new(Span::Compiler(proc_span)) + } +} + +impl From for Span { + fn from(inner: fallback::Span) -> Self { + Span::Fallback(inner) + } +} + +impl Debug for Span { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + match self { + Span::Compiler(s) => Debug::fmt(s, f), + Span::Fallback(s) => Debug::fmt(s, f), + } + } +} + +pub(crate) fn debug_span_field_if_nontrivial(debug: &mut fmt::DebugStruct, span: Span) { + match span { + Span::Compiler(s) => { + debug.field("span", &s); + } + Span::Fallback(s) => fallback::debug_span_field_if_nontrivial(debug, s), + } +} + +#[derive(Clone)] +pub(crate) enum Group { + Compiler(proc_macro::Group), + Fallback(fallback::Group), +} + +impl Group { + pub(crate) fn new(delimiter: Delimiter, stream: TokenStream) -> Self { + match stream { + TokenStream::Compiler(tts) => { + let delimiter = match delimiter { + Delimiter::Parenthesis => proc_macro::Delimiter::Parenthesis, + Delimiter::Bracket => proc_macro::Delimiter::Bracket, + Delimiter::Brace => proc_macro::Delimiter::Brace, + Delimiter::None => proc_macro::Delimiter::None, + }; + Group::Compiler(proc_macro::Group::new(delimiter, tts.into_token_stream())) + } + TokenStream::Fallback(stream) => { + Group::Fallback(fallback::Group::new(delimiter, stream)) + } + } + } + + pub(crate) fn delimiter(&self) -> Delimiter { + match self { + Group::Compiler(g) => match g.delimiter() { + proc_macro::Delimiter::Parenthesis => Delimiter::Parenthesis, + proc_macro::Delimiter::Bracket => Delimiter::Bracket, + proc_macro::Delimiter::Brace => Delimiter::Brace, + proc_macro::Delimiter::None => Delimiter::None, + }, + Group::Fallback(g) => g.delimiter(), + } + } + + pub(crate) fn stream(&self) -> TokenStream { + match self { + Group::Compiler(g) => TokenStream::Compiler(DeferredTokenStream::new(g.stream())), + Group::Fallback(g) => TokenStream::Fallback(g.stream()), + } + } + + pub(crate) fn span(&self) -> Span { + match self { + Group::Compiler(g) => Span::Compiler(g.span()), + Group::Fallback(g) => Span::Fallback(g.span()), + } + } + + pub(crate) fn span_open(&self) -> Span { + match self { + Group::Compiler(g) => Span::Compiler(g.span_open()), + Group::Fallback(g) => Span::Fallback(g.span_open()), + } + } + + pub(crate) fn span_close(&self) -> Span { + match self { + Group::Compiler(g) => Span::Compiler(g.span_close()), + Group::Fallback(g) => Span::Fallback(g.span_close()), + } + } + + pub(crate) fn set_span(&mut self, span: Span) { + match (self, span) { + (Group::Compiler(g), Span::Compiler(s)) => g.set_span(s), + (Group::Fallback(g), Span::Fallback(s)) => g.set_span(s), + (Group::Compiler(_), Span::Fallback(_)) => mismatch(line!()), + (Group::Fallback(_), Span::Compiler(_)) => mismatch(line!()), + } + } + + fn unwrap_nightly(self) -> proc_macro::Group { + match self { + Group::Compiler(g) => g, + Group::Fallback(_) => mismatch(line!()), + } + } +} + +impl From for Group { + fn from(g: fallback::Group) -> Self { + Group::Fallback(g) + } +} + +impl Display for Group { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + match self { + Group::Compiler(group) => Display::fmt(group, formatter), + Group::Fallback(group) => Display::fmt(group, formatter), + } + } +} + +impl Debug for Group { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + match self { + Group::Compiler(group) => Debug::fmt(group, formatter), + Group::Fallback(group) => Debug::fmt(group, formatter), + } + } +} + +#[derive(Clone)] +pub(crate) enum Ident { + Compiler(proc_macro::Ident), + Fallback(fallback::Ident), +} + +impl Ident { + #[track_caller] + pub(crate) fn new_checked(string: &str, span: Span) -> Self { + match span { + Span::Compiler(s) => Ident::Compiler(proc_macro::Ident::new(string, s)), + Span::Fallback(s) => Ident::Fallback(fallback::Ident::new_checked(string, s)), + } + } + + #[track_caller] + pub(crate) fn new_raw_checked(string: &str, span: Span) -> Self { + match span { + Span::Compiler(s) => Ident::Compiler(proc_macro::Ident::new_raw(string, s)), + Span::Fallback(s) => Ident::Fallback(fallback::Ident::new_raw_checked(string, s)), + } + } + + pub(crate) fn span(&self) -> Span { + match self { + Ident::Compiler(t) => Span::Compiler(t.span()), + Ident::Fallback(t) => Span::Fallback(t.span()), + } + } + + pub(crate) fn set_span(&mut self, span: Span) { + match (self, span) { + (Ident::Compiler(t), Span::Compiler(s)) => t.set_span(s), + (Ident::Fallback(t), Span::Fallback(s)) => t.set_span(s), + (Ident::Compiler(_), Span::Fallback(_)) => mismatch(line!()), + (Ident::Fallback(_), Span::Compiler(_)) => mismatch(line!()), + } + } + + fn unwrap_nightly(self) -> proc_macro::Ident { + match self { + Ident::Compiler(s) => s, + Ident::Fallback(_) => mismatch(line!()), + } + } +} + +impl From for Ident { + fn from(inner: fallback::Ident) -> Self { + Ident::Fallback(inner) + } +} + +impl PartialEq for Ident { + fn eq(&self, other: &Ident) -> bool { + match (self, other) { + (Ident::Compiler(t), Ident::Compiler(o)) => t.to_string() == o.to_string(), + (Ident::Fallback(t), Ident::Fallback(o)) => t == o, + (Ident::Compiler(_), Ident::Fallback(_)) => mismatch(line!()), + (Ident::Fallback(_), Ident::Compiler(_)) => mismatch(line!()), + } + } +} + +impl PartialEq for Ident +where + T: ?Sized + AsRef, +{ + fn eq(&self, other: &T) -> bool { + let other = other.as_ref(); + match self { + Ident::Compiler(t) => t.to_string() == other, + Ident::Fallback(t) => t == other, + } + } +} + +impl Display for Ident { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + match self { + Ident::Compiler(t) => Display::fmt(t, f), + Ident::Fallback(t) => Display::fmt(t, f), + } + } +} + +impl Debug for Ident { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + match self { + Ident::Compiler(t) => Debug::fmt(t, f), + Ident::Fallback(t) => Debug::fmt(t, f), + } + } +} + +#[derive(Clone)] +pub(crate) enum Literal { + Compiler(proc_macro::Literal), + Fallback(fallback::Literal), +} + +macro_rules! suffixed_numbers { + ($($name:ident => $kind:ident,)*) => ($( + pub(crate) fn $name(n: $kind) -> Literal { + if inside_proc_macro() { + Literal::Compiler(proc_macro::Literal::$name(n)) + } else { + Literal::Fallback(fallback::Literal::$name(n)) + } + } + )*) +} + +macro_rules! unsuffixed_integers { + ($($name:ident => $kind:ident,)*) => ($( + pub(crate) fn $name(n: $kind) -> Literal { + if inside_proc_macro() { + Literal::Compiler(proc_macro::Literal::$name(n)) + } else { + Literal::Fallback(fallback::Literal::$name(n)) + } + } + )*) +} + +impl Literal { + pub(crate) fn from_str_checked(repr: &str) -> Result { + if inside_proc_macro() { + let literal = proc_macro::Literal::from_str_checked(repr)?; + Ok(Literal::Compiler(literal)) + } else { + let literal = fallback::Literal::from_str_checked(repr)?; + Ok(Literal::Fallback(literal)) + } + } + + pub(crate) unsafe fn from_str_unchecked(repr: &str) -> Self { + if inside_proc_macro() { + Literal::Compiler(proc_macro::Literal::from_str_unchecked(repr)) + } else { + Literal::Fallback(unsafe { fallback::Literal::from_str_unchecked(repr) }) + } + } + + suffixed_numbers! { + u8_suffixed => u8, + u16_suffixed => u16, + u32_suffixed => u32, + u64_suffixed => u64, + u128_suffixed => u128, + usize_suffixed => usize, + i8_suffixed => i8, + i16_suffixed => i16, + i32_suffixed => i32, + i64_suffixed => i64, + i128_suffixed => i128, + isize_suffixed => isize, + + f32_suffixed => f32, + f64_suffixed => f64, + } + + unsuffixed_integers! { + u8_unsuffixed => u8, + u16_unsuffixed => u16, + u32_unsuffixed => u32, + u64_unsuffixed => u64, + u128_unsuffixed => u128, + usize_unsuffixed => usize, + i8_unsuffixed => i8, + i16_unsuffixed => i16, + i32_unsuffixed => i32, + i64_unsuffixed => i64, + i128_unsuffixed => i128, + isize_unsuffixed => isize, + } + + pub(crate) fn f32_unsuffixed(f: f32) -> Literal { + if inside_proc_macro() { + Literal::Compiler(proc_macro::Literal::f32_unsuffixed(f)) + } else { + Literal::Fallback(fallback::Literal::f32_unsuffixed(f)) + } + } + + pub(crate) fn f64_unsuffixed(f: f64) -> Literal { + if inside_proc_macro() { + Literal::Compiler(proc_macro::Literal::f64_unsuffixed(f)) + } else { + Literal::Fallback(fallback::Literal::f64_unsuffixed(f)) + } + } + + pub(crate) fn string(string: &str) -> Literal { + if inside_proc_macro() { + Literal::Compiler(proc_macro::Literal::string(string)) + } else { + Literal::Fallback(fallback::Literal::string(string)) + } + } + + pub(crate) fn character(ch: char) -> Literal { + if inside_proc_macro() { + Literal::Compiler(proc_macro::Literal::character(ch)) + } else { + Literal::Fallback(fallback::Literal::character(ch)) + } + } + + pub(crate) fn byte_character(byte: u8) -> Literal { + if inside_proc_macro() { + Literal::Compiler({ + #[cfg(not(no_literal_byte_character))] + { + proc_macro::Literal::byte_character(byte) + } + + #[cfg(no_literal_byte_character)] + { + let fallback = fallback::Literal::byte_character(byte); + proc_macro::Literal::from_str_unchecked(&fallback.repr) + } + }) + } else { + Literal::Fallback(fallback::Literal::byte_character(byte)) + } + } + + pub(crate) fn byte_string(bytes: &[u8]) -> Literal { + if inside_proc_macro() { + Literal::Compiler(proc_macro::Literal::byte_string(bytes)) + } else { + Literal::Fallback(fallback::Literal::byte_string(bytes)) + } + } + + pub(crate) fn c_string(string: &CStr) -> Literal { + if inside_proc_macro() { + Literal::Compiler({ + #[cfg(not(no_literal_c_string))] + { + proc_macro::Literal::c_string(string) + } + + #[cfg(no_literal_c_string)] + { + let fallback = fallback::Literal::c_string(string); + proc_macro::Literal::from_str_unchecked(&fallback.repr) + } + }) + } else { + Literal::Fallback(fallback::Literal::c_string(string)) + } + } + + pub(crate) fn span(&self) -> Span { + match self { + Literal::Compiler(lit) => Span::Compiler(lit.span()), + Literal::Fallback(lit) => Span::Fallback(lit.span()), + } + } + + pub(crate) fn set_span(&mut self, span: Span) { + match (self, span) { + (Literal::Compiler(lit), Span::Compiler(s)) => lit.set_span(s), + (Literal::Fallback(lit), Span::Fallback(s)) => lit.set_span(s), + (Literal::Compiler(_), Span::Fallback(_)) => mismatch(line!()), + (Literal::Fallback(_), Span::Compiler(_)) => mismatch(line!()), + } + } + + pub(crate) fn subspan>(&self, range: R) -> Option { + match self { + #[cfg(proc_macro_span)] + Literal::Compiler(lit) => proc_macro_span::subspan(lit, range).map(Span::Compiler), + #[cfg(not(proc_macro_span))] + Literal::Compiler(_lit) => None, + Literal::Fallback(lit) => lit.subspan(range).map(Span::Fallback), + } + } + + fn unwrap_nightly(self) -> proc_macro::Literal { + match self { + Literal::Compiler(s) => s, + Literal::Fallback(_) => mismatch(line!()), + } + } +} + +impl From for Literal { + fn from(s: fallback::Literal) -> Self { + Literal::Fallback(s) + } +} + +impl Display for Literal { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + match self { + Literal::Compiler(t) => Display::fmt(t, f), + Literal::Fallback(t) => Display::fmt(t, f), + } + } +} + +impl Debug for Literal { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + match self { + Literal::Compiler(t) => Debug::fmt(t, f), + Literal::Fallback(t) => Debug::fmt(t, f), + } + } +} + +#[cfg(span_locations)] +pub(crate) fn invalidate_current_thread_spans() { + if inside_proc_macro() { + panic!( + "proc_macro2::extra::invalidate_current_thread_spans is not available in procedural macros" + ); + } else { + crate::fallback::invalidate_current_thread_spans(); + } +} From a9acfceb9614f18d799c135bbb225bd0f55a81ab Mon Sep 17 00:00:00 2001 From: Miguel Ojeda Date: Mon, 24 Nov 2025 16:18:19 +0100 Subject: [PATCH 40/53] rust: proc-macro2: add SPDX License Identifiers Originally, when the Rust upstream `alloc` standard library crate was vendored in commit 057b8d257107 ("rust: adapt `alloc` crate to the kernel"), the SPDX License Identifiers were added to every file so that the license on those was clear. Thus do the same for the `proc-macro2` crate. This makes `scripts/spdxcheck.py` pass. Reviewed-by: Gary Guo Tested-by: Gary Guo Tested-by: Jesung Yang Link: https://patch.msgid.link/20251124151837.2184382-8-ojeda@kernel.org Signed-off-by: Miguel Ojeda --- rust/proc-macro2/detection.rs | 2 ++ rust/proc-macro2/extra.rs | 2 ++ rust/proc-macro2/fallback.rs | 2 ++ rust/proc-macro2/lib.rs | 2 ++ rust/proc-macro2/location.rs | 2 ++ rust/proc-macro2/marker.rs | 2 ++ rust/proc-macro2/parse.rs | 2 ++ rust/proc-macro2/probe.rs | 2 ++ rust/proc-macro2/probe/proc_macro_span.rs | 2 ++ rust/proc-macro2/probe/proc_macro_span_file.rs | 2 ++ rust/proc-macro2/probe/proc_macro_span_location.rs | 2 ++ rust/proc-macro2/rcvec.rs | 2 ++ rust/proc-macro2/wrapper.rs | 2 ++ 13 files changed, 26 insertions(+) diff --git a/rust/proc-macro2/detection.rs b/rust/proc-macro2/detection.rs index beba7b237395..3de448cb2dde 100644 --- a/rust/proc-macro2/detection.rs +++ b/rust/proc-macro2/detection.rs @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: Apache-2.0 OR MIT + use core::sync::atomic::{AtomicUsize, Ordering}; use std::sync::Once; diff --git a/rust/proc-macro2/extra.rs b/rust/proc-macro2/extra.rs index 522a90e136be..55feb5ec7526 100644 --- a/rust/proc-macro2/extra.rs +++ b/rust/proc-macro2/extra.rs @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: Apache-2.0 OR MIT + //! Items which do not have a correspondence to any API in the proc_macro crate, //! but are necessary to include in proc-macro2. diff --git a/rust/proc-macro2/fallback.rs b/rust/proc-macro2/fallback.rs index 1560105cfd25..9e005d67f7f5 100644 --- a/rust/proc-macro2/fallback.rs +++ b/rust/proc-macro2/fallback.rs @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: Apache-2.0 OR MIT + #[cfg(wrap_proc_macro)] use crate::imp; #[cfg(span_locations)] diff --git a/rust/proc-macro2/lib.rs b/rust/proc-macro2/lib.rs index 2984af335adc..7b78d065d51c 100644 --- a/rust/proc-macro2/lib.rs +++ b/rust/proc-macro2/lib.rs @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: Apache-2.0 OR MIT + //! [![github]](https://github.com/dtolnay/proc-macro2) [![crates-io]](https://crates.io/crates/proc-macro2) [![docs-rs]](crate) //! //! [github]: https://img.shields.io/badge/github-8da0cb?style=for-the-badge&labelColor=555555&logo=github diff --git a/rust/proc-macro2/location.rs b/rust/proc-macro2/location.rs index 7190e2d05255..3a11871e2943 100644 --- a/rust/proc-macro2/location.rs +++ b/rust/proc-macro2/location.rs @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: Apache-2.0 OR MIT + use core::cmp::Ordering; /// A line-column pair representing the start or end of a `Span`. diff --git a/rust/proc-macro2/marker.rs b/rust/proc-macro2/marker.rs index 23b94ce6fa85..6f322424808c 100644 --- a/rust/proc-macro2/marker.rs +++ b/rust/proc-macro2/marker.rs @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: Apache-2.0 OR MIT + use alloc::rc::Rc; use core::marker::PhantomData; use core::panic::{RefUnwindSafe, UnwindSafe}; diff --git a/rust/proc-macro2/parse.rs b/rust/proc-macro2/parse.rs index b8be403f842f..a005dea1fe88 100644 --- a/rust/proc-macro2/parse.rs +++ b/rust/proc-macro2/parse.rs @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: Apache-2.0 OR MIT + use crate::fallback::{ self, is_ident_continue, is_ident_start, Group, Ident, LexError, Literal, Span, TokenStream, TokenStreamBuilder, diff --git a/rust/proc-macro2/probe.rs b/rust/proc-macro2/probe.rs index b67f52036218..d68aa8cfd85e 100644 --- a/rust/proc-macro2/probe.rs +++ b/rust/proc-macro2/probe.rs @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: Apache-2.0 OR MIT + #![allow(dead_code)] #[cfg(proc_macro_span)] diff --git a/rust/proc-macro2/probe/proc_macro_span.rs b/rust/proc-macro2/probe/proc_macro_span.rs index 2d7d44e07708..892a7eb3e5a0 100644 --- a/rust/proc-macro2/probe/proc_macro_span.rs +++ b/rust/proc-macro2/probe/proc_macro_span.rs @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: Apache-2.0 OR MIT + // This code exercises the surface area that we expect of Span's unstable API. // If the current toolchain is able to compile it, then proc-macro2 is able to // offer these APIs too. diff --git a/rust/proc-macro2/probe/proc_macro_span_file.rs b/rust/proc-macro2/probe/proc_macro_span_file.rs index 8b76bdf5007b..f2dbc4056ebe 100644 --- a/rust/proc-macro2/probe/proc_macro_span_file.rs +++ b/rust/proc-macro2/probe/proc_macro_span_file.rs @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: Apache-2.0 OR MIT + // The subset of Span's API stabilized in Rust 1.88. extern crate proc_macro; diff --git a/rust/proc-macro2/probe/proc_macro_span_location.rs b/rust/proc-macro2/probe/proc_macro_span_location.rs index 79da34af54af..ae19c93394fa 100644 --- a/rust/proc-macro2/probe/proc_macro_span_location.rs +++ b/rust/proc-macro2/probe/proc_macro_span_location.rs @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: Apache-2.0 OR MIT + // The subset of Span's API stabilized in Rust 1.88. extern crate proc_macro; diff --git a/rust/proc-macro2/rcvec.rs b/rust/proc-macro2/rcvec.rs index 23edc77d597f..e224ebe277ef 100644 --- a/rust/proc-macro2/rcvec.rs +++ b/rust/proc-macro2/rcvec.rs @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: Apache-2.0 OR MIT + use alloc::rc::Rc; use alloc::vec; use core::mem; diff --git a/rust/proc-macro2/wrapper.rs b/rust/proc-macro2/wrapper.rs index 2e3eb5b4d04e..6792e9834111 100644 --- a/rust/proc-macro2/wrapper.rs +++ b/rust/proc-macro2/wrapper.rs @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: Apache-2.0 OR MIT + use crate::detection::inside_proc_macro; use crate::fallback::{self, FromStr2 as _}; #[cfg(span_locations)] From c2af0e5f02b9aebfc53987c6e935d266101f0ce9 Mon Sep 17 00:00:00 2001 From: Miguel Ojeda Date: Mon, 24 Nov 2025 16:18:20 +0100 Subject: [PATCH 41/53] rust: proc-macro2: remove `unicode_ident` dependency The `proc-macro2` crate depends on the `unicode-ident` crate to determine whether characters have the XID_Start or XID_Continue properties according to Unicode Standard Annex #31. However, we only need ASCII identifiers in the kernel, thus we can simplify the check and remove completely that dependency. Reviewed-by: Alice Ryhl Reviewed-by: Gary Guo Tested-by: Gary Guo Tested-by: Jesung Yang Link: https://patch.msgid.link/20251124151837.2184382-9-ojeda@kernel.org Signed-off-by: Miguel Ojeda --- rust/proc-macro2/fallback.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rust/proc-macro2/fallback.rs b/rust/proc-macro2/fallback.rs index 9e005d67f7f5..9b43c97df97a 100644 --- a/rust/proc-macro2/fallback.rs +++ b/rust/proc-macro2/fallback.rs @@ -818,11 +818,11 @@ impl Ident { } pub(crate) fn is_ident_start(c: char) -> bool { - c == '_' || unicode_ident::is_xid_start(c) + c == '_' || c.is_ascii_alphabetic() } pub(crate) fn is_ident_continue(c: char) -> bool { - unicode_ident::is_xid_continue(c) + c == '_' || c.is_ascii_alphanumeric() } #[track_caller] From bc1565efc358553d7be23a09fe012560402b7151 Mon Sep 17 00:00:00 2001 From: Miguel Ojeda Date: Mon, 24 Nov 2025 16:18:21 +0100 Subject: [PATCH 42/53] rust: proc-macro2: add `README.md` Originally, when the Rust upstream `alloc` standard library crate was vendored in commit 057b8d257107 ("rust: adapt `alloc` crate to the kernel"), a `README.md` file was added to explain the provenance and licensing of the source files. Thus do the same for the `proc-macro2` crate. Reviewed-by: Alice Ryhl Reviewed-by: Gary Guo Tested-by: Gary Guo Tested-by: Jesung Yang Link: https://patch.msgid.link/20251124151837.2184382-10-ojeda@kernel.org Signed-off-by: Miguel Ojeda --- rust/proc-macro2/README.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 rust/proc-macro2/README.md diff --git a/rust/proc-macro2/README.md b/rust/proc-macro2/README.md new file mode 100644 index 000000000000..af044fee4f59 --- /dev/null +++ b/rust/proc-macro2/README.md @@ -0,0 +1,13 @@ +# `proc-macro2` + +These source files come from the Rust `proc-macro2` crate, version +1.0.101 (released 2025-08-16), hosted in the + repository, licensed under +"Apache-2.0 OR MIT" and only modified to add the SPDX license +identifiers and to remove the `unicode-ident` dependency. + +For copyright details, please see: + + https://github.com/dtolnay/proc-macro2/blob/1.0.101/README.md#license + https://github.com/dtolnay/proc-macro2/blob/1.0.101/LICENSE-APACHE + https://github.com/dtolnay/proc-macro2/blob/1.0.101/LICENSE-MIT From 158a3b72118a4dab7e7bf2d89afbab9b96eddc1c Mon Sep 17 00:00:00 2001 From: Miguel Ojeda Date: Mon, 24 Nov 2025 16:18:22 +0100 Subject: [PATCH 43/53] rust: proc-macro2: enable support in kbuild With all the new files in place and ready from the new crate, enable the support for it in the build system. `proc_macro_byte_character` and `proc_macro_c_str_literals` were stabilized in Rust 1.79.0 [1] and were implemented earlier than our minimum Rust version (1.78) [2][3]. Thus just enable them instead of using the `cfg` that `proc-macro2` uses to emulate them in older compilers. In addition, skip formatting for this vendored crate and take the chance to add a comment mentioning this. Link: https://github.com/rust-lang/rust/pull/123431 [1] Link: https://github.com/rust-lang/rust/pull/112711 [2] Link: https://github.com/rust-lang/rust/pull/119651 [3] Reviewed-by: Gary Guo Tested-by: Gary Guo Tested-by: Jesung Yang Link: https://patch.msgid.link/20251124151837.2184382-11-ojeda@kernel.org Signed-off-by: Miguel Ojeda --- Makefile | 5 +++++ rust/Makefile | 32 +++++++++++++++++++++++++++++-- scripts/generate_rust_analyzer.py | 7 +++++++ 3 files changed, 42 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index d14824792227..589bcfe3bf75 100644 --- a/Makefile +++ b/Makefile @@ -1826,10 +1826,15 @@ rusttest: prepare $(Q)$(MAKE) $(build)=rust $@ # Formatting targets +# +# Generated files as well as vendored crates are skipped. PHONY += rustfmt rustfmtcheck rustfmt: $(Q)find $(srctree) $(RCS_FIND_IGNORE) \ + \( \ + -path $(srctree)/rust/proc-macro2 \ + \) -prune -o \ -type f -a -name '*.rs' -a ! -name '*generated*' -print \ | xargs $(RUSTFMT) $(rustfmt_flags) diff --git a/rust/Makefile b/rust/Makefile index 0288ca8d270c..7dd1261ad98f 100644 --- a/rust/Makefile +++ b/rust/Makefile @@ -27,6 +27,8 @@ endif obj-$(CONFIG_RUST) += exports.o +always-$(CONFIG_RUST) += libproc_macro2.rlib + always-$(CONFIG_RUST_KERNEL_DOCTESTS) += doctests_kernel_generated.rs always-$(CONFIG_RUST_KERNEL_DOCTESTS) += doctests_kernel_generated_kunit.c @@ -76,6 +78,17 @@ core-flags := \ --edition=$(core-edition) \ $(call cfgs-to-flags,$(core-cfgs)) +proc_macro2-cfgs := \ + feature="proc-macro" \ + wrap_proc_macro \ + $(if $(call rustc-min-version,108800),proc_macro_span_file proc_macro_span_location) + +# Stable since Rust 1.79.0: `feature(proc_macro_byte_character,proc_macro_c_str_literals)`. +proc_macro2-flags := \ + --cap-lints=allow \ + -Zcrate-attr='feature(proc_macro_byte_character,proc_macro_c_str_literals)' \ + $(call cfgs-to-flags,$(proc_macro2-cfgs)) + # `rustdoc` did not save the target modifiers, thus workaround for # the time being (https://github.com/rust-lang/rust/issues/144521). rustdoc_modifiers_workaround := $(if $(call rustc-min-version,108800),-Cunsafe-allow-abi-mismatch=fixed-x18) @@ -125,10 +138,15 @@ rustdoc: rustdoc-core rustdoc-macros rustdoc-compiler_builtins \ $(Q)for f in $(rustdoc_output)/static.files/rustdoc-*.css; do \ echo ".logo-container > img { object-fit: contain; }" >> $$f; done +rustdoc-proc_macro2: private rustdoc_host = yes +rustdoc-proc_macro2: private rustc_target_flags = $(proc_macro2-flags) +rustdoc-proc_macro2: $(src)/proc-macro2/lib.rs rustdoc-clean FORCE + +$(call if_changed,rustdoc) + rustdoc-macros: private rustdoc_host = yes rustdoc-macros: private rustc_target_flags = --crate-type proc-macro \ --extern proc_macro -rustdoc-macros: $(src)/macros/lib.rs rustdoc-clean FORCE +rustdoc-macros: $(src)/macros/lib.rs rustdoc-clean rustdoc-proc_macro2 FORCE +$(call if_changed,rustdoc) # Starting with Rust 1.82.0, skipping `-Wrustdoc::unescaped_backticks` should @@ -185,6 +203,10 @@ rusttestlib-build_error: $(src)/build_error.rs FORCE rusttestlib-ffi: $(src)/ffi.rs FORCE +$(call if_changed,rustc_test_library) +rusttestlib-proc_macro2: private rustc_target_flags = $(proc_macro2-flags) +rusttestlib-proc_macro2: $(src)/proc-macro2/lib.rs FORCE + +$(call if_changed,rustc_test_library) + rusttestlib-macros: private rustc_target_flags = --extern proc_macro rusttestlib-macros: private rustc_test_library_proc = yes rusttestlib-macros: $(src)/macros/lib.rs FORCE @@ -431,6 +453,11 @@ quiet_cmd_rustc_procmacrolibrary = $(RUSTC_OR_CLIPPY_QUIET) PL $@ mv $(objtree)/$(obj)/$(patsubst lib%.rlib,%,$(notdir $@)).d $(depfile); \ sed -i '/^\#/d' $(depfile) +$(obj)/libproc_macro2.rlib: private skip_clippy = 1 +$(obj)/libproc_macro2.rlib: private rustc_target_flags = $(proc_macro2-flags) +$(obj)/libproc_macro2.rlib: $(src)/proc-macro2/lib.rs FORCE + +$(call if_changed_dep,rustc_procmacrolibrary) + quiet_cmd_rustc_procmacro = $(RUSTC_OR_CLIPPY_QUIET) P $@ cmd_rustc_procmacro = \ $(RUSTC_OR_CLIPPY) $(rust_common_flags) $(rustc_target_flags) \ @@ -442,7 +469,7 @@ quiet_cmd_rustc_procmacro = $(RUSTC_OR_CLIPPY_QUIET) P $@ @$(objtree)/include/generated/rustc_cfg $< # Procedural macros can only be used with the `rustc` that compiled it. -$(obj)/$(libmacros_name): $(src)/macros/lib.rs FORCE +$(obj)/$(libmacros_name): $(src)/macros/lib.rs $(obj)/libproc_macro2.rlib FORCE +$(call if_changed_dep,rustc_procmacro) $(obj)/$(libpin_init_internal_name): private rustc_target_flags = --cfg kernel @@ -465,6 +492,7 @@ quiet_cmd_rustc_library = $(if $(skip_clippy),RUSTC,$(RUSTC_OR_CLIPPY_QUIET)) L rust-analyzer: $(Q)MAKEFLAGS= $(srctree)/scripts/generate_rust_analyzer.py \ --cfgs='core=$(core-cfgs)' $(core-edition) \ + --cfgs='proc_macro2=$(proc_macro2-cfgs)' \ $(realpath $(srctree)) $(realpath $(objtree)) \ $(rustc_sysroot) $(RUST_LIB_SRC) $(if $(KBUILD_EXTMOD),$(srcroot)) \ > rust-project.json diff --git a/scripts/generate_rust_analyzer.py b/scripts/generate_rust_analyzer.py index dedca470adc1..00c6b7cc94b7 100755 --- a/scripts/generate_rust_analyzer.py +++ b/scripts/generate_rust_analyzer.py @@ -86,6 +86,13 @@ def generate_crates(srctree, objtree, sysroot_src, external_src, cfgs, core_edit [], ) + append_crate( + "proc_macro2", + srctree / "rust" / "proc-macro2" / "lib.rs", + ["core", "alloc", "std", "proc_macro"], + cfg=crates_cfgs["proc_macro2"], + ) + append_crate( "macros", srctree / "rust" / "macros" / "lib.rs", From a4851eeef3e7cbcd89b5fd0234c04ce408a9ae81 Mon Sep 17 00:00:00 2001 From: Miguel Ojeda Date: Mon, 24 Nov 2025 16:18:23 +0100 Subject: [PATCH 44/53] rust: quote: import crate This is a subset of the Rust `quote` crate, version 1.0.40 (released 2025-03-12), licensed under "Apache-2.0 OR MIT", from: https://github.com/dtolnay/quote/raw/1.0.40/src The files are copied as-is, with no modifications whatsoever (not even adding the SPDX identifiers). For copyright details, please see: https://github.com/dtolnay/quote/blob/1.0.40/README.md#license https://github.com/dtolnay/quote/blob/1.0.40/LICENSE-APACHE https://github.com/dtolnay/quote/blob/1.0.40/LICENSE-MIT The next patch modifies these files as needed for use within the kernel. This patch split allows reviewers to double-check the import and to clearly see the differences introduced. The following script may be used to verify the contents: for path in $(cd rust/quote/ && find . -type f -name '*.rs'); do curl --silent --show-error --location \ https://github.com/dtolnay/quote/raw/1.0.40/src/$path \ | diff --unified rust/quote/$path - && echo $path: OK done Reviewed-by: Gary Guo Tested-by: Gary Guo Tested-by: Jesung Yang Link: https://patch.msgid.link/20251124151837.2184382-12-ojeda@kernel.org Signed-off-by: Miguel Ojeda --- rust/quote/ext.rs | 110 +++ rust/quote/format.rs | 168 ++++ rust/quote/ident_fragment.rs | 88 ++ rust/quote/lib.rs | 1454 ++++++++++++++++++++++++++++++++++ rust/quote/runtime.rs | 492 ++++++++++++ rust/quote/spanned.rs | 50 ++ rust/quote/to_tokens.rs | 271 +++++++ 7 files changed, 2633 insertions(+) create mode 100644 rust/quote/ext.rs create mode 100644 rust/quote/format.rs create mode 100644 rust/quote/ident_fragment.rs create mode 100644 rust/quote/lib.rs create mode 100644 rust/quote/runtime.rs create mode 100644 rust/quote/spanned.rs create mode 100644 rust/quote/to_tokens.rs diff --git a/rust/quote/ext.rs b/rust/quote/ext.rs new file mode 100644 index 000000000000..92c2315b182d --- /dev/null +++ b/rust/quote/ext.rs @@ -0,0 +1,110 @@ +use super::ToTokens; +use core::iter; +use proc_macro2::{TokenStream, TokenTree}; + +/// TokenStream extension trait with methods for appending tokens. +/// +/// This trait is sealed and cannot be implemented outside of the `quote` crate. +pub trait TokenStreamExt: private::Sealed { + /// For use by `ToTokens` implementations. + /// + /// Appends the token specified to this list of tokens. + fn append(&mut self, token: U) + where + U: Into; + + /// For use by `ToTokens` implementations. + /// + /// ``` + /// # use quote::{quote, TokenStreamExt, ToTokens}; + /// # use proc_macro2::TokenStream; + /// # + /// struct X; + /// + /// impl ToTokens for X { + /// fn to_tokens(&self, tokens: &mut TokenStream) { + /// tokens.append_all(&[true, false]); + /// } + /// } + /// + /// let tokens = quote!(#X); + /// assert_eq!(tokens.to_string(), "true false"); + /// ``` + fn append_all(&mut self, iter: I) + where + I: IntoIterator, + I::Item: ToTokens; + + /// For use by `ToTokens` implementations. + /// + /// Appends all of the items in the iterator `I`, separated by the tokens + /// `U`. + fn append_separated(&mut self, iter: I, op: U) + where + I: IntoIterator, + I::Item: ToTokens, + U: ToTokens; + + /// For use by `ToTokens` implementations. + /// + /// Appends all tokens in the iterator `I`, appending `U` after each + /// element, including after the last element of the iterator. + fn append_terminated(&mut self, iter: I, term: U) + where + I: IntoIterator, + I::Item: ToTokens, + U: ToTokens; +} + +impl TokenStreamExt for TokenStream { + fn append(&mut self, token: U) + where + U: Into, + { + self.extend(iter::once(token.into())); + } + + fn append_all(&mut self, iter: I) + where + I: IntoIterator, + I::Item: ToTokens, + { + for token in iter { + token.to_tokens(self); + } + } + + fn append_separated(&mut self, iter: I, op: U) + where + I: IntoIterator, + I::Item: ToTokens, + U: ToTokens, + { + for (i, token) in iter.into_iter().enumerate() { + if i > 0 { + op.to_tokens(self); + } + token.to_tokens(self); + } + } + + fn append_terminated(&mut self, iter: I, term: U) + where + I: IntoIterator, + I::Item: ToTokens, + U: ToTokens, + { + for token in iter { + token.to_tokens(self); + term.to_tokens(self); + } + } +} + +mod private { + use proc_macro2::TokenStream; + + pub trait Sealed {} + + impl Sealed for TokenStream {} +} diff --git a/rust/quote/format.rs b/rust/quote/format.rs new file mode 100644 index 000000000000..ec0bbf38ba37 --- /dev/null +++ b/rust/quote/format.rs @@ -0,0 +1,168 @@ +/// Formatting macro for constructing `Ident`s. +/// +///
+/// +/// # Syntax +/// +/// Syntax is copied from the [`format!`] macro, supporting both positional and +/// named arguments. +/// +/// Only a limited set of formatting traits are supported. The current mapping +/// of format types to traits is: +/// +/// * `{}` ⇒ [`IdentFragment`] +/// * `{:o}` ⇒ [`Octal`](std::fmt::Octal) +/// * `{:x}` ⇒ [`LowerHex`](std::fmt::LowerHex) +/// * `{:X}` ⇒ [`UpperHex`](std::fmt::UpperHex) +/// * `{:b}` ⇒ [`Binary`](std::fmt::Binary) +/// +/// See [`std::fmt`] for more information. +/// +///
+/// +/// # IdentFragment +/// +/// Unlike `format!`, this macro uses the [`IdentFragment`] formatting trait by +/// default. This trait is like `Display`, with a few differences: +/// +/// * `IdentFragment` is only implemented for a limited set of types, such as +/// unsigned integers and strings. +/// * [`Ident`] arguments will have their `r#` prefixes stripped, if present. +/// +/// [`IdentFragment`]: crate::IdentFragment +/// [`Ident`]: proc_macro2::Ident +/// +///
+/// +/// # Hygiene +/// +/// The [`Span`] of the first `Ident` argument is used as the span of the final +/// identifier, falling back to [`Span::call_site`] when no identifiers are +/// provided. +/// +/// ``` +/// # use quote::format_ident; +/// # let ident = format_ident!("Ident"); +/// // If `ident` is an Ident, the span of `my_ident` will be inherited from it. +/// let my_ident = format_ident!("My{}{}", ident, "IsCool"); +/// assert_eq!(my_ident, "MyIdentIsCool"); +/// ``` +/// +/// Alternatively, the span can be overridden by passing the `span` named +/// argument. +/// +/// ``` +/// # use quote::format_ident; +/// # const IGNORE_TOKENS: &'static str = stringify! { +/// let my_span = /* ... */; +/// # }; +/// # let my_span = proc_macro2::Span::call_site(); +/// format_ident!("MyIdent", span = my_span); +/// ``` +/// +/// [`Span`]: proc_macro2::Span +/// [`Span::call_site`]: proc_macro2::Span::call_site +/// +///


+/// +/// # Panics +/// +/// This method will panic if the resulting formatted string is not a valid +/// identifier. +/// +///
+/// +/// # Examples +/// +/// Composing raw and non-raw identifiers: +/// ``` +/// # use quote::format_ident; +/// let my_ident = format_ident!("My{}", "Ident"); +/// assert_eq!(my_ident, "MyIdent"); +/// +/// let raw = format_ident!("r#Raw"); +/// assert_eq!(raw, "r#Raw"); +/// +/// let my_ident_raw = format_ident!("{}Is{}", my_ident, raw); +/// assert_eq!(my_ident_raw, "MyIdentIsRaw"); +/// ``` +/// +/// Integer formatting options: +/// ``` +/// # use quote::format_ident; +/// let num: u32 = 10; +/// +/// let decimal = format_ident!("Id_{}", num); +/// assert_eq!(decimal, "Id_10"); +/// +/// let octal = format_ident!("Id_{:o}", num); +/// assert_eq!(octal, "Id_12"); +/// +/// let binary = format_ident!("Id_{:b}", num); +/// assert_eq!(binary, "Id_1010"); +/// +/// let lower_hex = format_ident!("Id_{:x}", num); +/// assert_eq!(lower_hex, "Id_a"); +/// +/// let upper_hex = format_ident!("Id_{:X}", num); +/// assert_eq!(upper_hex, "Id_A"); +/// ``` +#[macro_export] +macro_rules! format_ident { + ($fmt:expr) => { + $crate::format_ident_impl!([ + $crate::__private::Option::None, + $fmt + ]) + }; + + ($fmt:expr, $($rest:tt)*) => { + $crate::format_ident_impl!([ + $crate::__private::Option::None, + $fmt + ] $($rest)*) + }; +} + +#[macro_export] +#[doc(hidden)] +macro_rules! format_ident_impl { + // Final state + ([$span:expr, $($fmt:tt)*]) => { + $crate::__private::mk_ident( + &$crate::__private::format!($($fmt)*), + $span, + ) + }; + + // Span argument + ([$old:expr, $($fmt:tt)*] span = $span:expr) => { + $crate::format_ident_impl!([$old, $($fmt)*] span = $span,) + }; + ([$old:expr, $($fmt:tt)*] span = $span:expr, $($rest:tt)*) => { + $crate::format_ident_impl!([ + $crate::__private::Option::Some::<$crate::__private::Span>($span), + $($fmt)* + ] $($rest)*) + }; + + // Named argument + ([$span:expr, $($fmt:tt)*] $name:ident = $arg:expr) => { + $crate::format_ident_impl!([$span, $($fmt)*] $name = $arg,) + }; + ([$span:expr, $($fmt:tt)*] $name:ident = $arg:expr, $($rest:tt)*) => { + match $crate::__private::IdentFragmentAdapter(&$arg) { + arg => $crate::format_ident_impl!([$span.or(arg.span()), $($fmt)*, $name = arg] $($rest)*), + } + }; + + // Positional argument + ([$span:expr, $($fmt:tt)*] $arg:expr) => { + $crate::format_ident_impl!([$span, $($fmt)*] $arg,) + }; + ([$span:expr, $($fmt:tt)*] $arg:expr, $($rest:tt)*) => { + match $crate::__private::IdentFragmentAdapter(&$arg) { + arg => $crate::format_ident_impl!([$span.or(arg.span()), $($fmt)*, arg] $($rest)*), + } + }; +} diff --git a/rust/quote/ident_fragment.rs b/rust/quote/ident_fragment.rs new file mode 100644 index 000000000000..6c2a9a87acb4 --- /dev/null +++ b/rust/quote/ident_fragment.rs @@ -0,0 +1,88 @@ +use alloc::borrow::Cow; +use core::fmt; +use proc_macro2::{Ident, Span}; + +/// Specialized formatting trait used by `format_ident!`. +/// +/// [`Ident`] arguments formatted using this trait will have their `r#` prefix +/// stripped, if present. +/// +/// See [`format_ident!`] for more information. +/// +/// [`format_ident!`]: crate::format_ident +pub trait IdentFragment { + /// Format this value as an identifier fragment. + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result; + + /// Span associated with this `IdentFragment`. + /// + /// If non-`None`, may be inherited by formatted identifiers. + fn span(&self) -> Option { + None + } +} + +impl IdentFragment for &T { + fn span(&self) -> Option { + ::span(*self) + } + + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + IdentFragment::fmt(*self, f) + } +} + +impl IdentFragment for &mut T { + fn span(&self) -> Option { + ::span(*self) + } + + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + IdentFragment::fmt(*self, f) + } +} + +impl IdentFragment for Ident { + fn span(&self) -> Option { + Some(self.span()) + } + + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + let id = self.to_string(); + if let Some(id) = id.strip_prefix("r#") { + fmt::Display::fmt(id, f) + } else { + fmt::Display::fmt(&id[..], f) + } + } +} + +impl IdentFragment for Cow<'_, T> +where + T: IdentFragment + ToOwned + ?Sized, +{ + fn span(&self) -> Option { + T::span(self) + } + + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + T::fmt(self, f) + } +} + +// Limited set of types which this is implemented for, as we want to avoid types +// which will often include non-identifier characters in their `Display` impl. +macro_rules! ident_fragment_display { + ($($T:ty),*) => { + $( + impl IdentFragment for $T { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + fmt::Display::fmt(self, f) + } + } + )* + }; +} + +ident_fragment_display!(bool, str, String, char); +ident_fragment_display!(u8, u16, u32, u64, u128, usize); diff --git a/rust/quote/lib.rs b/rust/quote/lib.rs new file mode 100644 index 000000000000..0a12d607f279 --- /dev/null +++ b/rust/quote/lib.rs @@ -0,0 +1,1454 @@ +//! [![github]](https://github.com/dtolnay/quote) [![crates-io]](https://crates.io/crates/quote) [![docs-rs]](https://docs.rs/quote) +//! +//! [github]: https://img.shields.io/badge/github-8da0cb?style=for-the-badge&labelColor=555555&logo=github +//! [crates-io]: https://img.shields.io/badge/crates.io-fc8d62?style=for-the-badge&labelColor=555555&logo=rust +//! [docs-rs]: https://img.shields.io/badge/docs.rs-66c2a5?style=for-the-badge&labelColor=555555&logo=docs.rs +//! +//!
+//! +//! This crate provides the [`quote!`] macro for turning Rust syntax tree data +//! structures into tokens of source code. +//! +//! Procedural macros in Rust receive a stream of tokens as input, execute +//! arbitrary Rust code to determine how to manipulate those tokens, and produce +//! a stream of tokens to hand back to the compiler to compile into the caller's +//! crate. Quasi-quoting is a solution to one piece of that — producing +//! tokens to return to the compiler. +//! +//! The idea of quasi-quoting is that we write *code* that we treat as *data*. +//! Within the `quote!` macro, we can write what looks like code to our text +//! editor or IDE. We get all the benefits of the editor's brace matching, +//! syntax highlighting, indentation, and maybe autocompletion. But rather than +//! compiling that as code into the current crate, we can treat it as data, pass +//! it around, mutate it, and eventually hand it back to the compiler as tokens +//! to compile into the macro caller's crate. +//! +//! This crate is motivated by the procedural macro use case, but is a +//! general-purpose Rust quasi-quoting library and is not specific to procedural +//! macros. +//! +//! ```toml +//! [dependencies] +//! quote = "1.0" +//! ``` +//! +//!
+//! +//! # Example +//! +//! The following quasi-quoted block of code is something you might find in [a] +//! procedural macro having to do with data structure serialization. The `#var` +//! syntax performs interpolation of runtime variables into the quoted tokens. +//! Check out the documentation of the [`quote!`] macro for more detail about +//! the syntax. See also the [`quote_spanned!`] macro which is important for +//! implementing hygienic procedural macros. +//! +//! [a]: https://serde.rs/ +//! +//! ``` +//! # use quote::quote; +//! # +//! # let generics = ""; +//! # let where_clause = ""; +//! # let field_ty = ""; +//! # let item_ty = ""; +//! # let path = ""; +//! # let value = ""; +//! # +//! let tokens = quote! { +//! struct SerializeWith #generics #where_clause { +//! value: &'a #field_ty, +//! phantom: core::marker::PhantomData<#item_ty>, +//! } +//! +//! impl #generics serde::Serialize for SerializeWith #generics #where_clause { +//! fn serialize(&self, serializer: S) -> Result +//! where +//! S: serde::Serializer, +//! { +//! #path(self.value, serializer) +//! } +//! } +//! +//! SerializeWith { +//! value: #value, +//! phantom: core::marker::PhantomData::<#item_ty>, +//! } +//! }; +//! ``` +//! +//!
+//! +//! # Non-macro code generators +//! +//! When using `quote` in a build.rs or main.rs and writing the output out to a +//! file, consider having the code generator pass the tokens through +//! [prettyplease] before writing. This way if an error occurs in the generated +//! code it is convenient for a human to read and debug. +//! +//! [prettyplease]: https://github.com/dtolnay/prettyplease + +// Quote types in rustdoc of other crates get linked to here. +#![doc(html_root_url = "https://docs.rs/quote/1.0.40")] +#![allow( + clippy::doc_markdown, + clippy::elidable_lifetime_names, + clippy::missing_errors_doc, + clippy::missing_panics_doc, + clippy::module_name_repetitions, + clippy::needless_lifetimes, + // false positive https://github.com/rust-lang/rust-clippy/issues/6983 + clippy::wrong_self_convention, +)] + +extern crate alloc; + +#[cfg(feature = "proc-macro")] +extern crate proc_macro; + +mod ext; +mod format; +mod ident_fragment; +mod to_tokens; + +// Not public API. +#[doc(hidden)] +#[path = "runtime.rs"] +pub mod __private; + +pub use crate::ext::TokenStreamExt; +pub use crate::ident_fragment::IdentFragment; +pub use crate::to_tokens::ToTokens; + +// Not public API. +#[doc(hidden)] +pub mod spanned; + +macro_rules! __quote { + ($quote:item) => { + /// The whole point. + /// + /// Performs variable interpolation against the input and produces it as + /// [`proc_macro2::TokenStream`]. + /// + /// Note: for returning tokens to the compiler in a procedural macro, use + /// `.into()` on the result to convert to [`proc_macro::TokenStream`]. + /// + ///
+ /// + /// # Interpolation + /// + /// Variable interpolation is done with `#var` (similar to `$var` in + /// `macro_rules!` macros). This grabs the `var` variable that is currently in + /// scope and inserts it in that location in the output tokens. Any type + /// implementing the [`ToTokens`] trait can be interpolated. This includes most + /// Rust primitive types as well as most of the syntax tree types from the [Syn] + /// crate. + /// + /// [Syn]: https://github.com/dtolnay/syn + /// + /// Repetition is done using `#(...)*` or `#(...),*` again similar to + /// `macro_rules!`. This iterates through the elements of any variable + /// interpolated within the repetition and inserts a copy of the repetition body + /// for each one. The variables in an interpolation may be a `Vec`, slice, + /// `BTreeSet`, or any `Iterator`. + /// + /// - `#(#var)*` — no separators + /// - `#(#var),*` — the character before the asterisk is used as a separator + /// - `#( struct #var; )*` — the repetition can contain other tokens + /// - `#( #k => println!("{}", #v), )*` — even multiple interpolations + /// + ///
+ /// + /// # Hygiene + /// + /// Any interpolated tokens preserve the `Span` information provided by their + /// `ToTokens` implementation. Tokens that originate within the `quote!` + /// invocation are spanned with [`Span::call_site()`]. + /// + /// [`Span::call_site()`]: proc_macro2::Span::call_site + /// + /// A different span can be provided through the [`quote_spanned!`] macro. + /// + ///
+ /// + /// # Return type + /// + /// The macro evaluates to an expression of type `proc_macro2::TokenStream`. + /// Meanwhile Rust procedural macros are expected to return the type + /// `proc_macro::TokenStream`. + /// + /// The difference between the two types is that `proc_macro` types are entirely + /// specific to procedural macros and cannot ever exist in code outside of a + /// procedural macro, while `proc_macro2` types may exist anywhere including + /// tests and non-macro code like main.rs and build.rs. This is why even the + /// procedural macro ecosystem is largely built around `proc_macro2`, because + /// that ensures the libraries are unit testable and accessible in non-macro + /// contexts. + /// + /// There is a [`From`]-conversion in both directions so returning the output of + /// `quote!` from a procedural macro usually looks like `tokens.into()` or + /// `proc_macro::TokenStream::from(tokens)`. + /// + ///
+ /// + /// # Examples + /// + /// ### Procedural macro + /// + /// The structure of a basic procedural macro is as follows. Refer to the [Syn] + /// crate for further useful guidance on using `quote!` as part of a procedural + /// macro. + /// + /// [Syn]: https://github.com/dtolnay/syn + /// + /// ``` + /// # #[cfg(any())] + /// extern crate proc_macro; + /// # extern crate proc_macro2; + /// + /// # #[cfg(any())] + /// use proc_macro::TokenStream; + /// # use proc_macro2::TokenStream; + /// use quote::quote; + /// + /// # const IGNORE_TOKENS: &'static str = stringify! { + /// #[proc_macro_derive(HeapSize)] + /// # }; + /// pub fn derive_heap_size(input: TokenStream) -> TokenStream { + /// // Parse the input and figure out what implementation to generate... + /// # const IGNORE_TOKENS: &'static str = stringify! { + /// let name = /* ... */; + /// let expr = /* ... */; + /// # }; + /// # + /// # let name = 0; + /// # let expr = 0; + /// + /// let expanded = quote! { + /// // The generated impl. + /// impl heapsize::HeapSize for #name { + /// fn heap_size_of_children(&self) -> usize { + /// #expr + /// } + /// } + /// }; + /// + /// // Hand the output tokens back to the compiler. + /// TokenStream::from(expanded) + /// } + /// ``` + /// + ///


+ /// + /// ### Combining quoted fragments + /// + /// Usually you don't end up constructing an entire final `TokenStream` in one + /// piece. Different parts may come from different helper functions. The tokens + /// produced by `quote!` themselves implement `ToTokens` and so can be + /// interpolated into later `quote!` invocations to build up a final result. + /// + /// ``` + /// # use quote::quote; + /// # + /// let type_definition = quote! {...}; + /// let methods = quote! {...}; + /// + /// let tokens = quote! { + /// #type_definition + /// #methods + /// }; + /// ``` + /// + ///


+ /// + /// ### Constructing identifiers + /// + /// Suppose we have an identifier `ident` which came from somewhere in a macro + /// input and we need to modify it in some way for the macro output. Let's + /// consider prepending the identifier with an underscore. + /// + /// Simply interpolating the identifier next to an underscore will not have the + /// behavior of concatenating them. The underscore and the identifier will + /// continue to be two separate tokens as if you had written `_ x`. + /// + /// ``` + /// # use proc_macro2::{self as syn, Span}; + /// # use quote::quote; + /// # + /// # let ident = syn::Ident::new("i", Span::call_site()); + /// # + /// // incorrect + /// quote! { + /// let mut _#ident = 0; + /// } + /// # ; + /// ``` + /// + /// The solution is to build a new identifier token with the correct value. As + /// this is such a common case, the [`format_ident!`] macro provides a + /// convenient utility for doing so correctly. + /// + /// ``` + /// # use proc_macro2::{Ident, Span}; + /// # use quote::{format_ident, quote}; + /// # + /// # let ident = Ident::new("i", Span::call_site()); + /// # + /// let varname = format_ident!("_{}", ident); + /// quote! { + /// let mut #varname = 0; + /// } + /// # ; + /// ``` + /// + /// Alternatively, the APIs provided by Syn and proc-macro2 can be used to + /// directly build the identifier. This is roughly equivalent to the above, but + /// will not handle `ident` being a raw identifier. + /// + /// ``` + /// # use proc_macro2::{self as syn, Span}; + /// # use quote::quote; + /// # + /// # let ident = syn::Ident::new("i", Span::call_site()); + /// # + /// let concatenated = format!("_{}", ident); + /// let varname = syn::Ident::new(&concatenated, ident.span()); + /// quote! { + /// let mut #varname = 0; + /// } + /// # ; + /// ``` + /// + ///


+ /// + /// ### Making method calls + /// + /// Let's say our macro requires some type specified in the macro input to have + /// a constructor called `new`. We have the type in a variable called + /// `field_type` of type `syn::Type` and want to invoke the constructor. + /// + /// ``` + /// # use quote::quote; + /// # + /// # let field_type = quote!(...); + /// # + /// // incorrect + /// quote! { + /// let value = #field_type::new(); + /// } + /// # ; + /// ``` + /// + /// This works only sometimes. If `field_type` is `String`, the expanded code + /// contains `String::new()` which is fine. But if `field_type` is something + /// like `Vec` then the expanded code is `Vec::new()` which is invalid + /// syntax. Ordinarily in handwritten Rust we would write `Vec::::new()` + /// but for macros often the following is more convenient. + /// + /// ``` + /// # use quote::quote; + /// # + /// # let field_type = quote!(...); + /// # + /// quote! { + /// let value = <#field_type>::new(); + /// } + /// # ; + /// ``` + /// + /// This expands to `>::new()` which behaves correctly. + /// + /// A similar pattern is appropriate for trait methods. + /// + /// ``` + /// # use quote::quote; + /// # + /// # let field_type = quote!(...); + /// # + /// quote! { + /// let value = <#field_type as core::default::Default>::default(); + /// } + /// # ; + /// ``` + /// + ///


+ /// + /// ### Interpolating text inside of doc comments + /// + /// Neither doc comments nor string literals get interpolation behavior in + /// quote: + /// + /// ```compile_fail + /// quote! { + /// /// try to interpolate: #ident + /// /// + /// /// ... + /// } + /// ``` + /// + /// ```compile_fail + /// quote! { + /// #[doc = "try to interpolate: #ident"] + /// } + /// ``` + /// + /// Instead the best way to build doc comments that involve variables is by + /// formatting the doc string literal outside of quote. + /// + /// ```rust + /// # use proc_macro2::{Ident, Span}; + /// # use quote::quote; + /// # + /// # const IGNORE: &str = stringify! { + /// let msg = format!(...); + /// # }; + /// # + /// # let ident = Ident::new("var", Span::call_site()); + /// # let msg = format!("try to interpolate: {}", ident); + /// quote! { + /// #[doc = #msg] + /// /// + /// /// ... + /// } + /// # ; + /// ``` + /// + ///


+ /// + /// ### Indexing into a tuple struct + /// + /// When interpolating indices of a tuple or tuple struct, we need them not to + /// appears suffixed as integer literals by interpolating them as [`syn::Index`] + /// instead. + /// + /// [`syn::Index`]: https://docs.rs/syn/2.0/syn/struct.Index.html + /// + /// ```compile_fail + /// let i = 0usize..self.fields.len(); + /// + /// // expands to 0 + self.0usize.heap_size() + self.1usize.heap_size() + ... + /// // which is not valid syntax + /// quote! { + /// 0 #( + self.#i.heap_size() )* + /// } + /// ``` + /// + /// ``` + /// # use proc_macro2::{Ident, TokenStream}; + /// # use quote::quote; + /// # + /// # mod syn { + /// # use proc_macro2::{Literal, TokenStream}; + /// # use quote::{ToTokens, TokenStreamExt}; + /// # + /// # pub struct Index(usize); + /// # + /// # impl From for Index { + /// # fn from(i: usize) -> Self { + /// # Index(i) + /// # } + /// # } + /// # + /// # impl ToTokens for Index { + /// # fn to_tokens(&self, tokens: &mut TokenStream) { + /// # tokens.append(Literal::usize_unsuffixed(self.0)); + /// # } + /// # } + /// # } + /// # + /// # struct Struct { + /// # fields: Vec, + /// # } + /// # + /// # impl Struct { + /// # fn example(&self) -> TokenStream { + /// let i = (0..self.fields.len()).map(syn::Index::from); + /// + /// // expands to 0 + self.0.heap_size() + self.1.heap_size() + ... + /// quote! { + /// 0 #( + self.#i.heap_size() )* + /// } + /// # } + /// # } + /// ``` + $quote + }; +} + +#[cfg(doc)] +__quote![ + #[macro_export] + macro_rules! quote { + ($($tt:tt)*) => { + ... + }; + } +]; + +#[cfg(not(doc))] +__quote![ + #[macro_export] + macro_rules! quote { + () => { + $crate::__private::TokenStream::new() + }; + + // Special case rule for a single tt, for performance. + ($tt:tt) => {{ + let mut _s = $crate::__private::TokenStream::new(); + $crate::quote_token!{$tt _s} + _s + }}; + + // Special case rules for two tts, for performance. + (# $var:ident) => {{ + let mut _s = $crate::__private::TokenStream::new(); + $crate::ToTokens::to_tokens(&$var, &mut _s); + _s + }}; + ($tt1:tt $tt2:tt) => {{ + let mut _s = $crate::__private::TokenStream::new(); + $crate::quote_token!{$tt1 _s} + $crate::quote_token!{$tt2 _s} + _s + }}; + + // Rule for any other number of tokens. + ($($tt:tt)*) => {{ + let mut _s = $crate::__private::TokenStream::new(); + $crate::quote_each_token!{_s $($tt)*} + _s + }}; + } +]; + +macro_rules! __quote_spanned { + ($quote_spanned:item) => { + /// Same as `quote!`, but applies a given span to all tokens originating within + /// the macro invocation. + /// + ///
+ /// + /// # Syntax + /// + /// A span expression of type [`Span`], followed by `=>`, followed by the tokens + /// to quote. The span expression should be brief — use a variable for + /// anything more than a few characters. There should be no space before the + /// `=>` token. + /// + /// [`Span`]: proc_macro2::Span + /// + /// ``` + /// # use proc_macro2::Span; + /// # use quote::quote_spanned; + /// # + /// # const IGNORE_TOKENS: &'static str = stringify! { + /// let span = /* ... */; + /// # }; + /// # let span = Span::call_site(); + /// # let init = 0; + /// + /// // On one line, use parentheses. + /// let tokens = quote_spanned!(span=> Box::into_raw(Box::new(#init))); + /// + /// // On multiple lines, place the span at the top and use braces. + /// let tokens = quote_spanned! {span=> + /// Box::into_raw(Box::new(#init)) + /// }; + /// ``` + /// + /// The lack of space before the `=>` should look jarring to Rust programmers + /// and this is intentional. The formatting is designed to be visibly + /// off-balance and draw the eye a particular way, due to the span expression + /// being evaluated in the context of the procedural macro and the remaining + /// tokens being evaluated in the generated code. + /// + ///
+ /// + /// # Hygiene + /// + /// Any interpolated tokens preserve the `Span` information provided by their + /// `ToTokens` implementation. Tokens that originate within the `quote_spanned!` + /// invocation are spanned with the given span argument. + /// + ///
+ /// + /// # Example + /// + /// The following procedural macro code uses `quote_spanned!` to assert that a + /// particular Rust type implements the [`Sync`] trait so that references can be + /// safely shared between threads. + /// + /// ``` + /// # use quote::{quote_spanned, TokenStreamExt, ToTokens}; + /// # use proc_macro2::{Span, TokenStream}; + /// # + /// # struct Type; + /// # + /// # impl Type { + /// # fn span(&self) -> Span { + /// # Span::call_site() + /// # } + /// # } + /// # + /// # impl ToTokens for Type { + /// # fn to_tokens(&self, _tokens: &mut TokenStream) {} + /// # } + /// # + /// # let ty = Type; + /// # let call_site = Span::call_site(); + /// # + /// let ty_span = ty.span(); + /// let assert_sync = quote_spanned! {ty_span=> + /// struct _AssertSync where #ty: Sync; + /// }; + /// ``` + /// + /// If the assertion fails, the user will see an error like the following. The + /// input span of their type is highlighted in the error. + /// + /// ```text + /// error[E0277]: the trait bound `*const (): std::marker::Sync` is not satisfied + /// --> src/main.rs:10:21 + /// | + /// 10 | static ref PTR: *const () = &(); + /// | ^^^^^^^^^ `*const ()` cannot be shared between threads safely + /// ``` + /// + /// In this example it is important for the where-clause to be spanned with the + /// line/column information of the user's input type so that error messages are + /// placed appropriately by the compiler. + $quote_spanned + }; +} + +#[cfg(doc)] +__quote_spanned![ + #[macro_export] + macro_rules! quote_spanned { + ($span:expr=> $($tt:tt)*) => { + ... + }; + } +]; + +#[cfg(not(doc))] +__quote_spanned![ + #[macro_export] + macro_rules! quote_spanned { + ($span:expr=>) => {{ + let _: $crate::__private::Span = $crate::__private::get_span($span).__into_span(); + $crate::__private::TokenStream::new() + }}; + + // Special case rule for a single tt, for performance. + ($span:expr=> $tt:tt) => {{ + let mut _s = $crate::__private::TokenStream::new(); + let _span: $crate::__private::Span = $crate::__private::get_span($span).__into_span(); + $crate::quote_token_spanned!{$tt _s _span} + _s + }}; + + // Special case rules for two tts, for performance. + ($span:expr=> # $var:ident) => {{ + let mut _s = $crate::__private::TokenStream::new(); + let _: $crate::__private::Span = $crate::__private::get_span($span).__into_span(); + $crate::ToTokens::to_tokens(&$var, &mut _s); + _s + }}; + ($span:expr=> $tt1:tt $tt2:tt) => {{ + let mut _s = $crate::__private::TokenStream::new(); + let _span: $crate::__private::Span = $crate::__private::get_span($span).__into_span(); + $crate::quote_token_spanned!{$tt1 _s _span} + $crate::quote_token_spanned!{$tt2 _s _span} + _s + }}; + + // Rule for any other number of tokens. + ($span:expr=> $($tt:tt)*) => {{ + let mut _s = $crate::__private::TokenStream::new(); + let _span: $crate::__private::Span = $crate::__private::get_span($span).__into_span(); + $crate::quote_each_token_spanned!{_s _span $($tt)*} + _s + }}; + } +]; + +// Extract the names of all #metavariables and pass them to the $call macro. +// +// in: pounded_var_names!(then!(...) a #b c #( #d )* #e) +// out: then!(... b); +// then!(... d); +// then!(... e); +#[macro_export] +#[doc(hidden)] +macro_rules! pounded_var_names { + ($call:ident! $extra:tt $($tts:tt)*) => { + $crate::pounded_var_names_with_context!{$call! $extra + (@ $($tts)*) + ($($tts)* @) + } + }; +} + +#[macro_export] +#[doc(hidden)] +macro_rules! pounded_var_names_with_context { + ($call:ident! $extra:tt ($($b1:tt)*) ($($curr:tt)*)) => { + $( + $crate::pounded_var_with_context!{$call! $extra $b1 $curr} + )* + }; +} + +#[macro_export] +#[doc(hidden)] +macro_rules! pounded_var_with_context { + ($call:ident! $extra:tt $b1:tt ( $($inner:tt)* )) => { + $crate::pounded_var_names!{$call! $extra $($inner)*} + }; + + ($call:ident! $extra:tt $b1:tt [ $($inner:tt)* ]) => { + $crate::pounded_var_names!{$call! $extra $($inner)*} + }; + + ($call:ident! $extra:tt $b1:tt { $($inner:tt)* }) => { + $crate::pounded_var_names!{$call! $extra $($inner)*} + }; + + ($call:ident!($($extra:tt)*) # $var:ident) => { + $crate::$call!($($extra)* $var); + }; + + ($call:ident! $extra:tt $b1:tt $curr:tt) => {}; +} + +#[macro_export] +#[doc(hidden)] +macro_rules! quote_bind_into_iter { + ($has_iter:ident $var:ident) => { + // `mut` may be unused if $var occurs multiple times in the list. + #[allow(unused_mut)] + let (mut $var, i) = $var.quote_into_iter(); + let $has_iter = $has_iter | i; + }; +} + +#[macro_export] +#[doc(hidden)] +macro_rules! quote_bind_next_or_break { + ($var:ident) => { + let $var = match $var.next() { + Some(_x) => $crate::__private::RepInterp(_x), + None => break, + }; + }; +} + +// The obvious way to write this macro is as a tt muncher. This implementation +// does something more complex for two reasons. +// +// - With a tt muncher it's easy to hit Rust's built-in recursion_limit, which +// this implementation avoids because it isn't tail recursive. +// +// - Compile times for a tt muncher are quadratic relative to the length of +// the input. This implementation is linear, so it will be faster +// (potentially much faster) for big inputs. However, the constant factors +// of this implementation are higher than that of a tt muncher, so it is +// somewhat slower than a tt muncher if there are many invocations with +// short inputs. +// +// An invocation like this: +// +// quote_each_token!(_s a b c d e f g h i j); +// +// expands to this: +// +// quote_tokens_with_context!(_s +// (@ @ @ @ @ @ a b c d e f g h i j) +// (@ @ @ @ @ a b c d e f g h i j @) +// (@ @ @ @ a b c d e f g h i j @ @) +// (@ @ @ (a) (b) (c) (d) (e) (f) (g) (h) (i) (j) @ @ @) +// (@ @ a b c d e f g h i j @ @ @ @) +// (@ a b c d e f g h i j @ @ @ @ @) +// (a b c d e f g h i j @ @ @ @ @ @) +// ); +// +// which gets transposed and expanded to this: +// +// quote_token_with_context!(_s @ @ @ @ @ @ a); +// quote_token_with_context!(_s @ @ @ @ @ a b); +// quote_token_with_context!(_s @ @ @ @ a b c); +// quote_token_with_context!(_s @ @ @ (a) b c d); +// quote_token_with_context!(_s @ @ a (b) c d e); +// quote_token_with_context!(_s @ a b (c) d e f); +// quote_token_with_context!(_s a b c (d) e f g); +// quote_token_with_context!(_s b c d (e) f g h); +// quote_token_with_context!(_s c d e (f) g h i); +// quote_token_with_context!(_s d e f (g) h i j); +// quote_token_with_context!(_s e f g (h) i j @); +// quote_token_with_context!(_s f g h (i) j @ @); +// quote_token_with_context!(_s g h i (j) @ @ @); +// quote_token_with_context!(_s h i j @ @ @ @); +// quote_token_with_context!(_s i j @ @ @ @ @); +// quote_token_with_context!(_s j @ @ @ @ @ @); +// +// Without having used muncher-style recursion, we get one invocation of +// quote_token_with_context for each original tt, with three tts of context on +// either side. This is enough for the longest possible interpolation form (a +// repetition with separator, as in `# (#var) , *`) to be fully represented with +// the first or last tt in the middle. +// +// The middle tt (surrounded by parentheses) is the tt being processed. +// +// - When it is a `#`, quote_token_with_context can do an interpolation. The +// interpolation kind will depend on the three subsequent tts. +// +// - When it is within a later part of an interpolation, it can be ignored +// because the interpolation has already been done. +// +// - When it is not part of an interpolation it can be pushed as a single +// token into the output. +// +// - When the middle token is an unparenthesized `@`, that call is one of the +// first 3 or last 3 calls of quote_token_with_context and does not +// correspond to one of the original input tokens, so turns into nothing. +#[macro_export] +#[doc(hidden)] +macro_rules! quote_each_token { + ($tokens:ident $($tts:tt)*) => { + $crate::quote_tokens_with_context!{$tokens + (@ @ @ @ @ @ $($tts)*) + (@ @ @ @ @ $($tts)* @) + (@ @ @ @ $($tts)* @ @) + (@ @ @ $(($tts))* @ @ @) + (@ @ $($tts)* @ @ @ @) + (@ $($tts)* @ @ @ @ @) + ($($tts)* @ @ @ @ @ @) + } + }; +} + +// See the explanation on quote_each_token. +#[macro_export] +#[doc(hidden)] +macro_rules! quote_each_token_spanned { + ($tokens:ident $span:ident $($tts:tt)*) => { + $crate::quote_tokens_with_context_spanned!{$tokens $span + (@ @ @ @ @ @ $($tts)*) + (@ @ @ @ @ $($tts)* @) + (@ @ @ @ $($tts)* @ @) + (@ @ @ $(($tts))* @ @ @) + (@ @ $($tts)* @ @ @ @) + (@ $($tts)* @ @ @ @ @) + ($($tts)* @ @ @ @ @ @) + } + }; +} + +// See the explanation on quote_each_token. +#[macro_export] +#[doc(hidden)] +macro_rules! quote_tokens_with_context { + ($tokens:ident + ($($b3:tt)*) ($($b2:tt)*) ($($b1:tt)*) + ($($curr:tt)*) + ($($a1:tt)*) ($($a2:tt)*) ($($a3:tt)*) + ) => { + $( + $crate::quote_token_with_context!{$tokens $b3 $b2 $b1 $curr $a1 $a2 $a3} + )* + }; +} + +// See the explanation on quote_each_token. +#[macro_export] +#[doc(hidden)] +macro_rules! quote_tokens_with_context_spanned { + ($tokens:ident $span:ident + ($($b3:tt)*) ($($b2:tt)*) ($($b1:tt)*) + ($($curr:tt)*) + ($($a1:tt)*) ($($a2:tt)*) ($($a3:tt)*) + ) => { + $( + $crate::quote_token_with_context_spanned!{$tokens $span $b3 $b2 $b1 $curr $a1 $a2 $a3} + )* + }; +} + +// See the explanation on quote_each_token. +#[macro_export] +#[doc(hidden)] +macro_rules! quote_token_with_context { + // Unparenthesized `@` indicates this call does not correspond to one of the + // original input tokens. Ignore it. + ($tokens:ident $b3:tt $b2:tt $b1:tt @ $a1:tt $a2:tt $a3:tt) => {}; + + // A repetition with no separator. + ($tokens:ident $b3:tt $b2:tt $b1:tt (#) ( $($inner:tt)* ) * $a3:tt) => {{ + use $crate::__private::ext::*; + let has_iter = $crate::__private::ThereIsNoIteratorInRepetition; + $crate::pounded_var_names!{quote_bind_into_iter!(has_iter) () $($inner)*} + let _: $crate::__private::HasIterator = has_iter; + // This is `while true` instead of `loop` because if there are no + // iterators used inside of this repetition then the body would not + // contain any `break`, so the compiler would emit unreachable code + // warnings on anything below the loop. We use has_iter to detect and + // fail to compile when there are no iterators, so here we just work + // around the unneeded extra warning. + while true { + $crate::pounded_var_names!{quote_bind_next_or_break!() () $($inner)*} + $crate::quote_each_token!{$tokens $($inner)*} + } + }}; + // ... and one step later. + ($tokens:ident $b3:tt $b2:tt # (( $($inner:tt)* )) * $a2:tt $a3:tt) => {}; + // ... and one step later. + ($tokens:ident $b3:tt # ( $($inner:tt)* ) (*) $a1:tt $a2:tt $a3:tt) => {}; + + // A repetition with separator. + ($tokens:ident $b3:tt $b2:tt $b1:tt (#) ( $($inner:tt)* ) $sep:tt *) => {{ + use $crate::__private::ext::*; + let mut _i = 0usize; + let has_iter = $crate::__private::ThereIsNoIteratorInRepetition; + $crate::pounded_var_names!{quote_bind_into_iter!(has_iter) () $($inner)*} + let _: $crate::__private::HasIterator = has_iter; + while true { + $crate::pounded_var_names!{quote_bind_next_or_break!() () $($inner)*} + if _i > 0 { + $crate::quote_token!{$sep $tokens} + } + _i += 1; + $crate::quote_each_token!{$tokens $($inner)*} + } + }}; + // ... and one step later. + ($tokens:ident $b3:tt $b2:tt # (( $($inner:tt)* )) $sep:tt * $a3:tt) => {}; + // ... and one step later. + ($tokens:ident $b3:tt # ( $($inner:tt)* ) ($sep:tt) * $a2:tt $a3:tt) => {}; + // (A special case for `#(var)**`, where the first `*` is treated as the + // repetition symbol and the second `*` is treated as an ordinary token.) + ($tokens:ident # ( $($inner:tt)* ) * (*) $a1:tt $a2:tt $a3:tt) => { + // https://github.com/dtolnay/quote/issues/130 + $crate::quote_token!{* $tokens} + }; + // ... and one step later. + ($tokens:ident # ( $($inner:tt)* ) $sep:tt (*) $a1:tt $a2:tt $a3:tt) => {}; + + // A non-repetition interpolation. + ($tokens:ident $b3:tt $b2:tt $b1:tt (#) $var:ident $a2:tt $a3:tt) => { + $crate::ToTokens::to_tokens(&$var, &mut $tokens); + }; + // ... and one step later. + ($tokens:ident $b3:tt $b2:tt # ($var:ident) $a1:tt $a2:tt $a3:tt) => {}; + + // An ordinary token, not part of any interpolation. + ($tokens:ident $b3:tt $b2:tt $b1:tt ($curr:tt) $a1:tt $a2:tt $a3:tt) => { + $crate::quote_token!{$curr $tokens} + }; +} + +// See the explanation on quote_each_token, and on the individual rules of +// quote_token_with_context. +#[macro_export] +#[doc(hidden)] +macro_rules! quote_token_with_context_spanned { + ($tokens:ident $span:ident $b3:tt $b2:tt $b1:tt @ $a1:tt $a2:tt $a3:tt) => {}; + + ($tokens:ident $span:ident $b3:tt $b2:tt $b1:tt (#) ( $($inner:tt)* ) * $a3:tt) => {{ + use $crate::__private::ext::*; + let has_iter = $crate::__private::ThereIsNoIteratorInRepetition; + $crate::pounded_var_names!{quote_bind_into_iter!(has_iter) () $($inner)*} + let _: $crate::__private::HasIterator = has_iter; + while true { + $crate::pounded_var_names!{quote_bind_next_or_break!() () $($inner)*} + $crate::quote_each_token_spanned!{$tokens $span $($inner)*} + } + }}; + ($tokens:ident $span:ident $b3:tt $b2:tt # (( $($inner:tt)* )) * $a2:tt $a3:tt) => {}; + ($tokens:ident $span:ident $b3:tt # ( $($inner:tt)* ) (*) $a1:tt $a2:tt $a3:tt) => {}; + + ($tokens:ident $span:ident $b3:tt $b2:tt $b1:tt (#) ( $($inner:tt)* ) $sep:tt *) => {{ + use $crate::__private::ext::*; + let mut _i = 0usize; + let has_iter = $crate::__private::ThereIsNoIteratorInRepetition; + $crate::pounded_var_names!{quote_bind_into_iter!(has_iter) () $($inner)*} + let _: $crate::__private::HasIterator = has_iter; + while true { + $crate::pounded_var_names!{quote_bind_next_or_break!() () $($inner)*} + if _i > 0 { + $crate::quote_token_spanned!{$sep $tokens $span} + } + _i += 1; + $crate::quote_each_token_spanned!{$tokens $span $($inner)*} + } + }}; + ($tokens:ident $span:ident $b3:tt $b2:tt # (( $($inner:tt)* )) $sep:tt * $a3:tt) => {}; + ($tokens:ident $span:ident $b3:tt # ( $($inner:tt)* ) ($sep:tt) * $a2:tt $a3:tt) => {}; + ($tokens:ident $span:ident # ( $($inner:tt)* ) * (*) $a1:tt $a2:tt $a3:tt) => { + // https://github.com/dtolnay/quote/issues/130 + $crate::quote_token_spanned!{* $tokens $span} + }; + ($tokens:ident $span:ident # ( $($inner:tt)* ) $sep:tt (*) $a1:tt $a2:tt $a3:tt) => {}; + + ($tokens:ident $span:ident $b3:tt $b2:tt $b1:tt (#) $var:ident $a2:tt $a3:tt) => { + $crate::ToTokens::to_tokens(&$var, &mut $tokens); + }; + ($tokens:ident $span:ident $b3:tt $b2:tt # ($var:ident) $a1:tt $a2:tt $a3:tt) => {}; + + ($tokens:ident $span:ident $b3:tt $b2:tt $b1:tt ($curr:tt) $a1:tt $a2:tt $a3:tt) => { + $crate::quote_token_spanned!{$curr $tokens $span} + }; +} + +// These rules are ordered by approximate token frequency, at least for the +// first 10 or so, to improve compile times. Having `ident` first is by far the +// most important because it's typically 2-3x more common than the next most +// common token. +// +// Separately, we put the token being matched in the very front so that failing +// rules may fail to match as quickly as possible. +#[macro_export] +#[doc(hidden)] +macro_rules! quote_token { + ($ident:ident $tokens:ident) => { + $crate::__private::push_ident(&mut $tokens, stringify!($ident)); + }; + + (:: $tokens:ident) => { + $crate::__private::push_colon2(&mut $tokens); + }; + + (( $($inner:tt)* ) $tokens:ident) => { + $crate::__private::push_group( + &mut $tokens, + $crate::__private::Delimiter::Parenthesis, + $crate::quote!($($inner)*), + ); + }; + + ([ $($inner:tt)* ] $tokens:ident) => { + $crate::__private::push_group( + &mut $tokens, + $crate::__private::Delimiter::Bracket, + $crate::quote!($($inner)*), + ); + }; + + ({ $($inner:tt)* } $tokens:ident) => { + $crate::__private::push_group( + &mut $tokens, + $crate::__private::Delimiter::Brace, + $crate::quote!($($inner)*), + ); + }; + + (# $tokens:ident) => { + $crate::__private::push_pound(&mut $tokens); + }; + + (, $tokens:ident) => { + $crate::__private::push_comma(&mut $tokens); + }; + + (. $tokens:ident) => { + $crate::__private::push_dot(&mut $tokens); + }; + + (; $tokens:ident) => { + $crate::__private::push_semi(&mut $tokens); + }; + + (: $tokens:ident) => { + $crate::__private::push_colon(&mut $tokens); + }; + + (+ $tokens:ident) => { + $crate::__private::push_add(&mut $tokens); + }; + + (+= $tokens:ident) => { + $crate::__private::push_add_eq(&mut $tokens); + }; + + (& $tokens:ident) => { + $crate::__private::push_and(&mut $tokens); + }; + + (&& $tokens:ident) => { + $crate::__private::push_and_and(&mut $tokens); + }; + + (&= $tokens:ident) => { + $crate::__private::push_and_eq(&mut $tokens); + }; + + (@ $tokens:ident) => { + $crate::__private::push_at(&mut $tokens); + }; + + (! $tokens:ident) => { + $crate::__private::push_bang(&mut $tokens); + }; + + (^ $tokens:ident) => { + $crate::__private::push_caret(&mut $tokens); + }; + + (^= $tokens:ident) => { + $crate::__private::push_caret_eq(&mut $tokens); + }; + + (/ $tokens:ident) => { + $crate::__private::push_div(&mut $tokens); + }; + + (/= $tokens:ident) => { + $crate::__private::push_div_eq(&mut $tokens); + }; + + (.. $tokens:ident) => { + $crate::__private::push_dot2(&mut $tokens); + }; + + (... $tokens:ident) => { + $crate::__private::push_dot3(&mut $tokens); + }; + + (..= $tokens:ident) => { + $crate::__private::push_dot_dot_eq(&mut $tokens); + }; + + (= $tokens:ident) => { + $crate::__private::push_eq(&mut $tokens); + }; + + (== $tokens:ident) => { + $crate::__private::push_eq_eq(&mut $tokens); + }; + + (>= $tokens:ident) => { + $crate::__private::push_ge(&mut $tokens); + }; + + (> $tokens:ident) => { + $crate::__private::push_gt(&mut $tokens); + }; + + (<= $tokens:ident) => { + $crate::__private::push_le(&mut $tokens); + }; + + (< $tokens:ident) => { + $crate::__private::push_lt(&mut $tokens); + }; + + (*= $tokens:ident) => { + $crate::__private::push_mul_eq(&mut $tokens); + }; + + (!= $tokens:ident) => { + $crate::__private::push_ne(&mut $tokens); + }; + + (| $tokens:ident) => { + $crate::__private::push_or(&mut $tokens); + }; + + (|= $tokens:ident) => { + $crate::__private::push_or_eq(&mut $tokens); + }; + + (|| $tokens:ident) => { + $crate::__private::push_or_or(&mut $tokens); + }; + + (? $tokens:ident) => { + $crate::__private::push_question(&mut $tokens); + }; + + (-> $tokens:ident) => { + $crate::__private::push_rarrow(&mut $tokens); + }; + + (<- $tokens:ident) => { + $crate::__private::push_larrow(&mut $tokens); + }; + + (% $tokens:ident) => { + $crate::__private::push_rem(&mut $tokens); + }; + + (%= $tokens:ident) => { + $crate::__private::push_rem_eq(&mut $tokens); + }; + + (=> $tokens:ident) => { + $crate::__private::push_fat_arrow(&mut $tokens); + }; + + (<< $tokens:ident) => { + $crate::__private::push_shl(&mut $tokens); + }; + + (<<= $tokens:ident) => { + $crate::__private::push_shl_eq(&mut $tokens); + }; + + (>> $tokens:ident) => { + $crate::__private::push_shr(&mut $tokens); + }; + + (>>= $tokens:ident) => { + $crate::__private::push_shr_eq(&mut $tokens); + }; + + (* $tokens:ident) => { + $crate::__private::push_star(&mut $tokens); + }; + + (- $tokens:ident) => { + $crate::__private::push_sub(&mut $tokens); + }; + + (-= $tokens:ident) => { + $crate::__private::push_sub_eq(&mut $tokens); + }; + + ($lifetime:lifetime $tokens:ident) => { + $crate::__private::push_lifetime(&mut $tokens, stringify!($lifetime)); + }; + + (_ $tokens:ident) => { + $crate::__private::push_underscore(&mut $tokens); + }; + + ($other:tt $tokens:ident) => { + $crate::__private::parse(&mut $tokens, stringify!($other)); + }; +} + +// See the comment above `quote_token!` about the rule ordering. +#[macro_export] +#[doc(hidden)] +macro_rules! quote_token_spanned { + ($ident:ident $tokens:ident $span:ident) => { + $crate::__private::push_ident_spanned(&mut $tokens, $span, stringify!($ident)); + }; + + (:: $tokens:ident $span:ident) => { + $crate::__private::push_colon2_spanned(&mut $tokens, $span); + }; + + (( $($inner:tt)* ) $tokens:ident $span:ident) => { + $crate::__private::push_group_spanned( + &mut $tokens, + $span, + $crate::__private::Delimiter::Parenthesis, + $crate::quote_spanned!($span=> $($inner)*), + ); + }; + + ([ $($inner:tt)* ] $tokens:ident $span:ident) => { + $crate::__private::push_group_spanned( + &mut $tokens, + $span, + $crate::__private::Delimiter::Bracket, + $crate::quote_spanned!($span=> $($inner)*), + ); + }; + + ({ $($inner:tt)* } $tokens:ident $span:ident) => { + $crate::__private::push_group_spanned( + &mut $tokens, + $span, + $crate::__private::Delimiter::Brace, + $crate::quote_spanned!($span=> $($inner)*), + ); + }; + + (# $tokens:ident $span:ident) => { + $crate::__private::push_pound_spanned(&mut $tokens, $span); + }; + + (, $tokens:ident $span:ident) => { + $crate::__private::push_comma_spanned(&mut $tokens, $span); + }; + + (. $tokens:ident $span:ident) => { + $crate::__private::push_dot_spanned(&mut $tokens, $span); + }; + + (; $tokens:ident $span:ident) => { + $crate::__private::push_semi_spanned(&mut $tokens, $span); + }; + + (: $tokens:ident $span:ident) => { + $crate::__private::push_colon_spanned(&mut $tokens, $span); + }; + + (+ $tokens:ident $span:ident) => { + $crate::__private::push_add_spanned(&mut $tokens, $span); + }; + + (+= $tokens:ident $span:ident) => { + $crate::__private::push_add_eq_spanned(&mut $tokens, $span); + }; + + (& $tokens:ident $span:ident) => { + $crate::__private::push_and_spanned(&mut $tokens, $span); + }; + + (&& $tokens:ident $span:ident) => { + $crate::__private::push_and_and_spanned(&mut $tokens, $span); + }; + + (&= $tokens:ident $span:ident) => { + $crate::__private::push_and_eq_spanned(&mut $tokens, $span); + }; + + (@ $tokens:ident $span:ident) => { + $crate::__private::push_at_spanned(&mut $tokens, $span); + }; + + (! $tokens:ident $span:ident) => { + $crate::__private::push_bang_spanned(&mut $tokens, $span); + }; + + (^ $tokens:ident $span:ident) => { + $crate::__private::push_caret_spanned(&mut $tokens, $span); + }; + + (^= $tokens:ident $span:ident) => { + $crate::__private::push_caret_eq_spanned(&mut $tokens, $span); + }; + + (/ $tokens:ident $span:ident) => { + $crate::__private::push_div_spanned(&mut $tokens, $span); + }; + + (/= $tokens:ident $span:ident) => { + $crate::__private::push_div_eq_spanned(&mut $tokens, $span); + }; + + (.. $tokens:ident $span:ident) => { + $crate::__private::push_dot2_spanned(&mut $tokens, $span); + }; + + (... $tokens:ident $span:ident) => { + $crate::__private::push_dot3_spanned(&mut $tokens, $span); + }; + + (..= $tokens:ident $span:ident) => { + $crate::__private::push_dot_dot_eq_spanned(&mut $tokens, $span); + }; + + (= $tokens:ident $span:ident) => { + $crate::__private::push_eq_spanned(&mut $tokens, $span); + }; + + (== $tokens:ident $span:ident) => { + $crate::__private::push_eq_eq_spanned(&mut $tokens, $span); + }; + + (>= $tokens:ident $span:ident) => { + $crate::__private::push_ge_spanned(&mut $tokens, $span); + }; + + (> $tokens:ident $span:ident) => { + $crate::__private::push_gt_spanned(&mut $tokens, $span); + }; + + (<= $tokens:ident $span:ident) => { + $crate::__private::push_le_spanned(&mut $tokens, $span); + }; + + (< $tokens:ident $span:ident) => { + $crate::__private::push_lt_spanned(&mut $tokens, $span); + }; + + (*= $tokens:ident $span:ident) => { + $crate::__private::push_mul_eq_spanned(&mut $tokens, $span); + }; + + (!= $tokens:ident $span:ident) => { + $crate::__private::push_ne_spanned(&mut $tokens, $span); + }; + + (| $tokens:ident $span:ident) => { + $crate::__private::push_or_spanned(&mut $tokens, $span); + }; + + (|= $tokens:ident $span:ident) => { + $crate::__private::push_or_eq_spanned(&mut $tokens, $span); + }; + + (|| $tokens:ident $span:ident) => { + $crate::__private::push_or_or_spanned(&mut $tokens, $span); + }; + + (? $tokens:ident $span:ident) => { + $crate::__private::push_question_spanned(&mut $tokens, $span); + }; + + (-> $tokens:ident $span:ident) => { + $crate::__private::push_rarrow_spanned(&mut $tokens, $span); + }; + + (<- $tokens:ident $span:ident) => { + $crate::__private::push_larrow_spanned(&mut $tokens, $span); + }; + + (% $tokens:ident $span:ident) => { + $crate::__private::push_rem_spanned(&mut $tokens, $span); + }; + + (%= $tokens:ident $span:ident) => { + $crate::__private::push_rem_eq_spanned(&mut $tokens, $span); + }; + + (=> $tokens:ident $span:ident) => { + $crate::__private::push_fat_arrow_spanned(&mut $tokens, $span); + }; + + (<< $tokens:ident $span:ident) => { + $crate::__private::push_shl_spanned(&mut $tokens, $span); + }; + + (<<= $tokens:ident $span:ident) => { + $crate::__private::push_shl_eq_spanned(&mut $tokens, $span); + }; + + (>> $tokens:ident $span:ident) => { + $crate::__private::push_shr_spanned(&mut $tokens, $span); + }; + + (>>= $tokens:ident $span:ident) => { + $crate::__private::push_shr_eq_spanned(&mut $tokens, $span); + }; + + (* $tokens:ident $span:ident) => { + $crate::__private::push_star_spanned(&mut $tokens, $span); + }; + + (- $tokens:ident $span:ident) => { + $crate::__private::push_sub_spanned(&mut $tokens, $span); + }; + + (-= $tokens:ident $span:ident) => { + $crate::__private::push_sub_eq_spanned(&mut $tokens, $span); + }; + + ($lifetime:lifetime $tokens:ident $span:ident) => { + $crate::__private::push_lifetime_spanned(&mut $tokens, $span, stringify!($lifetime)); + }; + + (_ $tokens:ident $span:ident) => { + $crate::__private::push_underscore_spanned(&mut $tokens, $span); + }; + + ($other:tt $tokens:ident $span:ident) => { + $crate::__private::parse_spanned(&mut $tokens, $span, stringify!($other)); + }; +} diff --git a/rust/quote/runtime.rs b/rust/quote/runtime.rs new file mode 100644 index 000000000000..c704ca89411f --- /dev/null +++ b/rust/quote/runtime.rs @@ -0,0 +1,492 @@ +use self::get_span::{GetSpan, GetSpanBase, GetSpanInner}; +use crate::{IdentFragment, ToTokens, TokenStreamExt}; +use core::fmt; +use core::iter; +use core::ops::BitOr; +use proc_macro2::{Group, Ident, Punct, Spacing, TokenTree}; + +#[doc(hidden)] +pub use alloc::format; +#[doc(hidden)] +pub use core::option::Option; + +#[doc(hidden)] +pub type Delimiter = proc_macro2::Delimiter; +#[doc(hidden)] +pub type Span = proc_macro2::Span; +#[doc(hidden)] +pub type TokenStream = proc_macro2::TokenStream; + +#[doc(hidden)] +pub struct HasIterator; // True +#[doc(hidden)] +pub struct ThereIsNoIteratorInRepetition; // False + +impl BitOr for ThereIsNoIteratorInRepetition { + type Output = ThereIsNoIteratorInRepetition; + fn bitor(self, _rhs: ThereIsNoIteratorInRepetition) -> ThereIsNoIteratorInRepetition { + ThereIsNoIteratorInRepetition + } +} + +impl BitOr for HasIterator { + type Output = HasIterator; + fn bitor(self, _rhs: ThereIsNoIteratorInRepetition) -> HasIterator { + HasIterator + } +} + +impl BitOr for ThereIsNoIteratorInRepetition { + type Output = HasIterator; + fn bitor(self, _rhs: HasIterator) -> HasIterator { + HasIterator + } +} + +impl BitOr for HasIterator { + type Output = HasIterator; + fn bitor(self, _rhs: HasIterator) -> HasIterator { + HasIterator + } +} + +/// Extension traits used by the implementation of `quote!`. These are defined +/// in separate traits, rather than as a single trait due to ambiguity issues. +/// +/// These traits expose a `quote_into_iter` method which should allow calling +/// whichever impl happens to be applicable. Calling that method repeatedly on +/// the returned value should be idempotent. +#[doc(hidden)] +pub mod ext { + use super::RepInterp; + use super::{HasIterator as HasIter, ThereIsNoIteratorInRepetition as DoesNotHaveIter}; + use crate::ToTokens; + use alloc::collections::btree_set::{self, BTreeSet}; + use core::slice; + + /// Extension trait providing the `quote_into_iter` method on iterators. + #[doc(hidden)] + pub trait RepIteratorExt: Iterator + Sized { + fn quote_into_iter(self) -> (Self, HasIter) { + (self, HasIter) + } + } + + impl RepIteratorExt for T {} + + /// Extension trait providing the `quote_into_iter` method for + /// non-iterable types. These types interpolate the same value in each + /// iteration of the repetition. + #[doc(hidden)] + pub trait RepToTokensExt { + /// Pretend to be an iterator for the purposes of `quote_into_iter`. + /// This allows repeated calls to `quote_into_iter` to continue + /// correctly returning DoesNotHaveIter. + fn next(&self) -> Option<&Self> { + Some(self) + } + + fn quote_into_iter(&self) -> (&Self, DoesNotHaveIter) { + (self, DoesNotHaveIter) + } + } + + impl RepToTokensExt for T {} + + /// Extension trait providing the `quote_into_iter` method for types that + /// can be referenced as an iterator. + #[doc(hidden)] + pub trait RepAsIteratorExt<'q> { + type Iter: Iterator; + + fn quote_into_iter(&'q self) -> (Self::Iter, HasIter); + } + + impl<'q, T: RepAsIteratorExt<'q> + ?Sized> RepAsIteratorExt<'q> for &T { + type Iter = T::Iter; + + fn quote_into_iter(&'q self) -> (Self::Iter, HasIter) { + ::quote_into_iter(*self) + } + } + + impl<'q, T: RepAsIteratorExt<'q> + ?Sized> RepAsIteratorExt<'q> for &mut T { + type Iter = T::Iter; + + fn quote_into_iter(&'q self) -> (Self::Iter, HasIter) { + ::quote_into_iter(*self) + } + } + + impl<'q, T: 'q> RepAsIteratorExt<'q> for [T] { + type Iter = slice::Iter<'q, T>; + + fn quote_into_iter(&'q self) -> (Self::Iter, HasIter) { + (self.iter(), HasIter) + } + } + + impl<'q, T: 'q, const N: usize> RepAsIteratorExt<'q> for [T; N] { + type Iter = slice::Iter<'q, T>; + + fn quote_into_iter(&'q self) -> (Self::Iter, HasIter) { + (self.iter(), HasIter) + } + } + + impl<'q, T: 'q> RepAsIteratorExt<'q> for Vec { + type Iter = slice::Iter<'q, T>; + + fn quote_into_iter(&'q self) -> (Self::Iter, HasIter) { + (self.iter(), HasIter) + } + } + + impl<'q, T: 'q> RepAsIteratorExt<'q> for BTreeSet { + type Iter = btree_set::Iter<'q, T>; + + fn quote_into_iter(&'q self) -> (Self::Iter, HasIter) { + (self.iter(), HasIter) + } + } + + impl<'q, T: RepAsIteratorExt<'q>> RepAsIteratorExt<'q> for RepInterp { + type Iter = T::Iter; + + fn quote_into_iter(&'q self) -> (Self::Iter, HasIter) { + self.0.quote_into_iter() + } + } +} + +// Helper type used within interpolations to allow for repeated binding names. +// Implements the relevant traits, and exports a dummy `next()` method. +#[derive(Copy, Clone)] +#[doc(hidden)] +pub struct RepInterp(pub T); + +impl RepInterp { + // This method is intended to look like `Iterator::next`, and is called when + // a name is bound multiple times, as the previous binding will shadow the + // original `Iterator` object. This allows us to avoid advancing the + // iterator multiple times per iteration. + pub fn next(self) -> Option { + Some(self.0) + } +} + +impl Iterator for RepInterp { + type Item = T::Item; + + fn next(&mut self) -> Option { + self.0.next() + } +} + +impl ToTokens for RepInterp { + fn to_tokens(&self, tokens: &mut TokenStream) { + self.0.to_tokens(tokens); + } +} + +#[doc(hidden)] +#[inline] +pub fn get_span(span: T) -> GetSpan { + GetSpan(GetSpanInner(GetSpanBase(span))) +} + +mod get_span { + use core::ops::Deref; + use proc_macro2::extra::DelimSpan; + use proc_macro2::Span; + + pub struct GetSpan(pub(crate) GetSpanInner); + + pub struct GetSpanInner(pub(crate) GetSpanBase); + + pub struct GetSpanBase(pub(crate) T); + + impl GetSpan { + #[inline] + pub fn __into_span(self) -> Span { + ((self.0).0).0 + } + } + + impl GetSpanInner { + #[inline] + pub fn __into_span(&self) -> Span { + (self.0).0.join() + } + } + + impl GetSpanBase { + #[allow(clippy::unused_self)] + pub fn __into_span(&self) -> T { + unreachable!() + } + } + + impl Deref for GetSpan { + type Target = GetSpanInner; + + #[inline] + fn deref(&self) -> &Self::Target { + &self.0 + } + } + + impl Deref for GetSpanInner { + type Target = GetSpanBase; + + #[inline] + fn deref(&self) -> &Self::Target { + &self.0 + } + } +} + +#[doc(hidden)] +pub fn push_group(tokens: &mut TokenStream, delimiter: Delimiter, inner: TokenStream) { + tokens.append(Group::new(delimiter, inner)); +} + +#[doc(hidden)] +pub fn push_group_spanned( + tokens: &mut TokenStream, + span: Span, + delimiter: Delimiter, + inner: TokenStream, +) { + let mut g = Group::new(delimiter, inner); + g.set_span(span); + tokens.append(g); +} + +#[doc(hidden)] +pub fn parse(tokens: &mut TokenStream, s: &str) { + let s: TokenStream = s.parse().expect("invalid token stream"); + tokens.extend(iter::once(s)); +} + +#[doc(hidden)] +pub fn parse_spanned(tokens: &mut TokenStream, span: Span, s: &str) { + let s: TokenStream = s.parse().expect("invalid token stream"); + tokens.extend(s.into_iter().map(|t| respan_token_tree(t, span))); +} + +// Token tree with every span replaced by the given one. +fn respan_token_tree(mut token: TokenTree, span: Span) -> TokenTree { + match &mut token { + TokenTree::Group(g) => { + let stream = g + .stream() + .into_iter() + .map(|token| respan_token_tree(token, span)) + .collect(); + *g = Group::new(g.delimiter(), stream); + g.set_span(span); + } + other => other.set_span(span), + } + token +} + +#[doc(hidden)] +pub fn push_ident(tokens: &mut TokenStream, s: &str) { + let span = Span::call_site(); + push_ident_spanned(tokens, span, s); +} + +#[doc(hidden)] +pub fn push_ident_spanned(tokens: &mut TokenStream, span: Span, s: &str) { + tokens.append(ident_maybe_raw(s, span)); +} + +#[doc(hidden)] +pub fn push_lifetime(tokens: &mut TokenStream, lifetime: &str) { + tokens.extend([ + TokenTree::Punct(Punct::new('\'', Spacing::Joint)), + TokenTree::Ident(Ident::new(&lifetime[1..], Span::call_site())), + ]); +} + +#[doc(hidden)] +pub fn push_lifetime_spanned(tokens: &mut TokenStream, span: Span, lifetime: &str) { + tokens.extend([ + TokenTree::Punct({ + let mut apostrophe = Punct::new('\'', Spacing::Joint); + apostrophe.set_span(span); + apostrophe + }), + TokenTree::Ident(Ident::new(&lifetime[1..], span)), + ]); +} + +macro_rules! push_punct { + ($name:ident $spanned:ident $char1:tt) => { + #[doc(hidden)] + pub fn $name(tokens: &mut TokenStream) { + tokens.append(Punct::new($char1, Spacing::Alone)); + } + #[doc(hidden)] + pub fn $spanned(tokens: &mut TokenStream, span: Span) { + let mut punct = Punct::new($char1, Spacing::Alone); + punct.set_span(span); + tokens.append(punct); + } + }; + ($name:ident $spanned:ident $char1:tt $char2:tt) => { + #[doc(hidden)] + pub fn $name(tokens: &mut TokenStream) { + tokens.append(Punct::new($char1, Spacing::Joint)); + tokens.append(Punct::new($char2, Spacing::Alone)); + } + #[doc(hidden)] + pub fn $spanned(tokens: &mut TokenStream, span: Span) { + let mut punct = Punct::new($char1, Spacing::Joint); + punct.set_span(span); + tokens.append(punct); + let mut punct = Punct::new($char2, Spacing::Alone); + punct.set_span(span); + tokens.append(punct); + } + }; + ($name:ident $spanned:ident $char1:tt $char2:tt $char3:tt) => { + #[doc(hidden)] + pub fn $name(tokens: &mut TokenStream) { + tokens.append(Punct::new($char1, Spacing::Joint)); + tokens.append(Punct::new($char2, Spacing::Joint)); + tokens.append(Punct::new($char3, Spacing::Alone)); + } + #[doc(hidden)] + pub fn $spanned(tokens: &mut TokenStream, span: Span) { + let mut punct = Punct::new($char1, Spacing::Joint); + punct.set_span(span); + tokens.append(punct); + let mut punct = Punct::new($char2, Spacing::Joint); + punct.set_span(span); + tokens.append(punct); + let mut punct = Punct::new($char3, Spacing::Alone); + punct.set_span(span); + tokens.append(punct); + } + }; +} + +push_punct!(push_add push_add_spanned '+'); +push_punct!(push_add_eq push_add_eq_spanned '+' '='); +push_punct!(push_and push_and_spanned '&'); +push_punct!(push_and_and push_and_and_spanned '&' '&'); +push_punct!(push_and_eq push_and_eq_spanned '&' '='); +push_punct!(push_at push_at_spanned '@'); +push_punct!(push_bang push_bang_spanned '!'); +push_punct!(push_caret push_caret_spanned '^'); +push_punct!(push_caret_eq push_caret_eq_spanned '^' '='); +push_punct!(push_colon push_colon_spanned ':'); +push_punct!(push_colon2 push_colon2_spanned ':' ':'); +push_punct!(push_comma push_comma_spanned ','); +push_punct!(push_div push_div_spanned '/'); +push_punct!(push_div_eq push_div_eq_spanned '/' '='); +push_punct!(push_dot push_dot_spanned '.'); +push_punct!(push_dot2 push_dot2_spanned '.' '.'); +push_punct!(push_dot3 push_dot3_spanned '.' '.' '.'); +push_punct!(push_dot_dot_eq push_dot_dot_eq_spanned '.' '.' '='); +push_punct!(push_eq push_eq_spanned '='); +push_punct!(push_eq_eq push_eq_eq_spanned '=' '='); +push_punct!(push_ge push_ge_spanned '>' '='); +push_punct!(push_gt push_gt_spanned '>'); +push_punct!(push_le push_le_spanned '<' '='); +push_punct!(push_lt push_lt_spanned '<'); +push_punct!(push_mul_eq push_mul_eq_spanned '*' '='); +push_punct!(push_ne push_ne_spanned '!' '='); +push_punct!(push_or push_or_spanned '|'); +push_punct!(push_or_eq push_or_eq_spanned '|' '='); +push_punct!(push_or_or push_or_or_spanned '|' '|'); +push_punct!(push_pound push_pound_spanned '#'); +push_punct!(push_question push_question_spanned '?'); +push_punct!(push_rarrow push_rarrow_spanned '-' '>'); +push_punct!(push_larrow push_larrow_spanned '<' '-'); +push_punct!(push_rem push_rem_spanned '%'); +push_punct!(push_rem_eq push_rem_eq_spanned '%' '='); +push_punct!(push_fat_arrow push_fat_arrow_spanned '=' '>'); +push_punct!(push_semi push_semi_spanned ';'); +push_punct!(push_shl push_shl_spanned '<' '<'); +push_punct!(push_shl_eq push_shl_eq_spanned '<' '<' '='); +push_punct!(push_shr push_shr_spanned '>' '>'); +push_punct!(push_shr_eq push_shr_eq_spanned '>' '>' '='); +push_punct!(push_star push_star_spanned '*'); +push_punct!(push_sub push_sub_spanned '-'); +push_punct!(push_sub_eq push_sub_eq_spanned '-' '='); + +#[doc(hidden)] +pub fn push_underscore(tokens: &mut TokenStream) { + push_underscore_spanned(tokens, Span::call_site()); +} + +#[doc(hidden)] +pub fn push_underscore_spanned(tokens: &mut TokenStream, span: Span) { + tokens.append(Ident::new("_", span)); +} + +// Helper method for constructing identifiers from the `format_ident!` macro, +// handling `r#` prefixes. +#[doc(hidden)] +pub fn mk_ident(id: &str, span: Option) -> Ident { + let span = span.unwrap_or_else(Span::call_site); + ident_maybe_raw(id, span) +} + +fn ident_maybe_raw(id: &str, span: Span) -> Ident { + if let Some(id) = id.strip_prefix("r#") { + Ident::new_raw(id, span) + } else { + Ident::new(id, span) + } +} + +// Adapts from `IdentFragment` to `fmt::Display` for use by the `format_ident!` +// macro, and exposes span information from these fragments. +// +// This struct also has forwarding implementations of the formatting traits +// `Octal`, `LowerHex`, `UpperHex`, and `Binary` to allow for their use within +// `format_ident!`. +#[derive(Copy, Clone)] +#[doc(hidden)] +pub struct IdentFragmentAdapter(pub T); + +impl IdentFragmentAdapter { + pub fn span(&self) -> Option { + self.0.span() + } +} + +impl fmt::Display for IdentFragmentAdapter { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + IdentFragment::fmt(&self.0, f) + } +} + +impl fmt::Octal for IdentFragmentAdapter { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + fmt::Octal::fmt(&self.0, f) + } +} + +impl fmt::LowerHex for IdentFragmentAdapter { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + fmt::LowerHex::fmt(&self.0, f) + } +} + +impl fmt::UpperHex for IdentFragmentAdapter { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + fmt::UpperHex::fmt(&self.0, f) + } +} + +impl fmt::Binary for IdentFragmentAdapter { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + fmt::Binary::fmt(&self.0, f) + } +} diff --git a/rust/quote/spanned.rs b/rust/quote/spanned.rs new file mode 100644 index 000000000000..6eba64445d89 --- /dev/null +++ b/rust/quote/spanned.rs @@ -0,0 +1,50 @@ +use crate::ToTokens; +use proc_macro2::extra::DelimSpan; +use proc_macro2::{Span, TokenStream}; + +// Not public API other than via the syn crate. Use syn::spanned::Spanned. +pub trait Spanned: private::Sealed { + fn __span(&self) -> Span; +} + +impl Spanned for Span { + fn __span(&self) -> Span { + *self + } +} + +impl Spanned for DelimSpan { + fn __span(&self) -> Span { + self.join() + } +} + +impl Spanned for T { + fn __span(&self) -> Span { + join_spans(self.into_token_stream()) + } +} + +fn join_spans(tokens: TokenStream) -> Span { + let mut iter = tokens.into_iter().map(|tt| tt.span()); + + let first = match iter.next() { + Some(span) => span, + None => return Span::call_site(), + }; + + iter.fold(None, |_prev, next| Some(next)) + .and_then(|last| first.join(last)) + .unwrap_or(first) +} + +mod private { + use crate::ToTokens; + use proc_macro2::extra::DelimSpan; + use proc_macro2::Span; + + pub trait Sealed {} + impl Sealed for Span {} + impl Sealed for DelimSpan {} + impl Sealed for T {} +} diff --git a/rust/quote/to_tokens.rs b/rust/quote/to_tokens.rs new file mode 100644 index 000000000000..f373092b650f --- /dev/null +++ b/rust/quote/to_tokens.rs @@ -0,0 +1,271 @@ +use super::TokenStreamExt; +use alloc::borrow::Cow; +use alloc::rc::Rc; +use core::iter; +use proc_macro2::{Group, Ident, Literal, Punct, Span, TokenStream, TokenTree}; +use std::ffi::{CStr, CString}; + +/// Types that can be interpolated inside a `quote!` invocation. +pub trait ToTokens { + /// Write `self` to the given `TokenStream`. + /// + /// The token append methods provided by the [`TokenStreamExt`] extension + /// trait may be useful for implementing `ToTokens`. + /// + /// # Example + /// + /// Example implementation for a struct representing Rust paths like + /// `std::cmp::PartialEq`: + /// + /// ``` + /// use proc_macro2::{TokenTree, Spacing, Span, Punct, TokenStream}; + /// use quote::{TokenStreamExt, ToTokens}; + /// + /// pub struct Path { + /// pub global: bool, + /// pub segments: Vec, + /// } + /// + /// impl ToTokens for Path { + /// fn to_tokens(&self, tokens: &mut TokenStream) { + /// for (i, segment) in self.segments.iter().enumerate() { + /// if i > 0 || self.global { + /// // Double colon `::` + /// tokens.append(Punct::new(':', Spacing::Joint)); + /// tokens.append(Punct::new(':', Spacing::Alone)); + /// } + /// segment.to_tokens(tokens); + /// } + /// } + /// } + /// # + /// # pub struct PathSegment; + /// # + /// # impl ToTokens for PathSegment { + /// # fn to_tokens(&self, tokens: &mut TokenStream) { + /// # unimplemented!() + /// # } + /// # } + /// ``` + fn to_tokens(&self, tokens: &mut TokenStream); + + /// Convert `self` directly into a `TokenStream` object. + /// + /// This method is implicitly implemented using `to_tokens`, and acts as a + /// convenience method for consumers of the `ToTokens` trait. + fn to_token_stream(&self) -> TokenStream { + let mut tokens = TokenStream::new(); + self.to_tokens(&mut tokens); + tokens + } + + /// Convert `self` directly into a `TokenStream` object. + /// + /// This method is implicitly implemented using `to_tokens`, and acts as a + /// convenience method for consumers of the `ToTokens` trait. + fn into_token_stream(self) -> TokenStream + where + Self: Sized, + { + self.to_token_stream() + } +} + +impl ToTokens for &T { + fn to_tokens(&self, tokens: &mut TokenStream) { + (**self).to_tokens(tokens); + } +} + +impl ToTokens for &mut T { + fn to_tokens(&self, tokens: &mut TokenStream) { + (**self).to_tokens(tokens); + } +} + +impl<'a, T: ?Sized + ToOwned + ToTokens> ToTokens for Cow<'a, T> { + fn to_tokens(&self, tokens: &mut TokenStream) { + (**self).to_tokens(tokens); + } +} + +impl ToTokens for Box { + fn to_tokens(&self, tokens: &mut TokenStream) { + (**self).to_tokens(tokens); + } +} + +impl ToTokens for Rc { + fn to_tokens(&self, tokens: &mut TokenStream) { + (**self).to_tokens(tokens); + } +} + +impl ToTokens for Option { + fn to_tokens(&self, tokens: &mut TokenStream) { + if let Some(t) = self { + t.to_tokens(tokens); + } + } +} + +impl ToTokens for str { + fn to_tokens(&self, tokens: &mut TokenStream) { + tokens.append(Literal::string(self)); + } +} + +impl ToTokens for String { + fn to_tokens(&self, tokens: &mut TokenStream) { + self.as_str().to_tokens(tokens); + } +} + +impl ToTokens for i8 { + fn to_tokens(&self, tokens: &mut TokenStream) { + tokens.append(Literal::i8_suffixed(*self)); + } +} + +impl ToTokens for i16 { + fn to_tokens(&self, tokens: &mut TokenStream) { + tokens.append(Literal::i16_suffixed(*self)); + } +} + +impl ToTokens for i32 { + fn to_tokens(&self, tokens: &mut TokenStream) { + tokens.append(Literal::i32_suffixed(*self)); + } +} + +impl ToTokens for i64 { + fn to_tokens(&self, tokens: &mut TokenStream) { + tokens.append(Literal::i64_suffixed(*self)); + } +} + +impl ToTokens for i128 { + fn to_tokens(&self, tokens: &mut TokenStream) { + tokens.append(Literal::i128_suffixed(*self)); + } +} + +impl ToTokens for isize { + fn to_tokens(&self, tokens: &mut TokenStream) { + tokens.append(Literal::isize_suffixed(*self)); + } +} + +impl ToTokens for u8 { + fn to_tokens(&self, tokens: &mut TokenStream) { + tokens.append(Literal::u8_suffixed(*self)); + } +} + +impl ToTokens for u16 { + fn to_tokens(&self, tokens: &mut TokenStream) { + tokens.append(Literal::u16_suffixed(*self)); + } +} + +impl ToTokens for u32 { + fn to_tokens(&self, tokens: &mut TokenStream) { + tokens.append(Literal::u32_suffixed(*self)); + } +} + +impl ToTokens for u64 { + fn to_tokens(&self, tokens: &mut TokenStream) { + tokens.append(Literal::u64_suffixed(*self)); + } +} + +impl ToTokens for u128 { + fn to_tokens(&self, tokens: &mut TokenStream) { + tokens.append(Literal::u128_suffixed(*self)); + } +} + +impl ToTokens for usize { + fn to_tokens(&self, tokens: &mut TokenStream) { + tokens.append(Literal::usize_suffixed(*self)); + } +} + +impl ToTokens for f32 { + fn to_tokens(&self, tokens: &mut TokenStream) { + tokens.append(Literal::f32_suffixed(*self)); + } +} + +impl ToTokens for f64 { + fn to_tokens(&self, tokens: &mut TokenStream) { + tokens.append(Literal::f64_suffixed(*self)); + } +} + +impl ToTokens for char { + fn to_tokens(&self, tokens: &mut TokenStream) { + tokens.append(Literal::character(*self)); + } +} + +impl ToTokens for bool { + fn to_tokens(&self, tokens: &mut TokenStream) { + let word = if *self { "true" } else { "false" }; + tokens.append(Ident::new(word, Span::call_site())); + } +} + +impl ToTokens for CStr { + fn to_tokens(&self, tokens: &mut TokenStream) { + tokens.append(Literal::c_string(self)); + } +} + +impl ToTokens for CString { + fn to_tokens(&self, tokens: &mut TokenStream) { + tokens.append(Literal::c_string(self)); + } +} + +impl ToTokens for Group { + fn to_tokens(&self, tokens: &mut TokenStream) { + tokens.append(self.clone()); + } +} + +impl ToTokens for Ident { + fn to_tokens(&self, tokens: &mut TokenStream) { + tokens.append(self.clone()); + } +} + +impl ToTokens for Punct { + fn to_tokens(&self, tokens: &mut TokenStream) { + tokens.append(self.clone()); + } +} + +impl ToTokens for Literal { + fn to_tokens(&self, tokens: &mut TokenStream) { + tokens.append(self.clone()); + } +} + +impl ToTokens for TokenTree { + fn to_tokens(&self, tokens: &mut TokenStream) { + tokens.append(self.clone()); + } +} + +impl ToTokens for TokenStream { + fn to_tokens(&self, tokens: &mut TokenStream) { + tokens.extend(iter::once(self.clone())); + } + + fn into_token_stream(self) -> TokenStream { + self + } +} From ddfa1b279d088edfb059b6936919faf2070fdd14 Mon Sep 17 00:00:00 2001 From: Miguel Ojeda Date: Mon, 24 Nov 2025 16:18:24 +0100 Subject: [PATCH 45/53] rust: quote: add SPDX License Identifiers Originally, when the Rust upstream `alloc` standard library crate was vendored in commit 057b8d257107 ("rust: adapt `alloc` crate to the kernel"), the SPDX License Identifiers were added to every file so that the license on those was clear. Thus do the same for the `quote` crate. This makes `scripts/spdxcheck.py` pass. Reviewed-by: Gary Guo Tested-by: Gary Guo Tested-by: Jesung Yang Link: https://patch.msgid.link/20251124151837.2184382-13-ojeda@kernel.org Signed-off-by: Miguel Ojeda --- rust/quote/ext.rs | 2 ++ rust/quote/format.rs | 2 ++ rust/quote/ident_fragment.rs | 2 ++ rust/quote/lib.rs | 2 ++ rust/quote/runtime.rs | 2 ++ rust/quote/spanned.rs | 2 ++ rust/quote/to_tokens.rs | 2 ++ 7 files changed, 14 insertions(+) diff --git a/rust/quote/ext.rs b/rust/quote/ext.rs index 92c2315b182d..977d2f0c5919 100644 --- a/rust/quote/ext.rs +++ b/rust/quote/ext.rs @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: Apache-2.0 OR MIT + use super::ToTokens; use core::iter; use proc_macro2::{TokenStream, TokenTree}; diff --git a/rust/quote/format.rs b/rust/quote/format.rs index ec0bbf38ba37..6e3d55b6e427 100644 --- a/rust/quote/format.rs +++ b/rust/quote/format.rs @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: Apache-2.0 OR MIT + /// Formatting macro for constructing `Ident`s. /// ///
diff --git a/rust/quote/ident_fragment.rs b/rust/quote/ident_fragment.rs index 6c2a9a87acb4..d98106f17666 100644 --- a/rust/quote/ident_fragment.rs +++ b/rust/quote/ident_fragment.rs @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: Apache-2.0 OR MIT + use alloc::borrow::Cow; use core::fmt; use proc_macro2::{Ident, Span}; diff --git a/rust/quote/lib.rs b/rust/quote/lib.rs index 0a12d607f279..cc1637660a75 100644 --- a/rust/quote/lib.rs +++ b/rust/quote/lib.rs @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: Apache-2.0 OR MIT + //! [![github]](https://github.com/dtolnay/quote) [![crates-io]](https://crates.io/crates/quote) [![docs-rs]](https://docs.rs/quote) //! //! [github]: https://img.shields.io/badge/github-8da0cb?style=for-the-badge&labelColor=555555&logo=github diff --git a/rust/quote/runtime.rs b/rust/quote/runtime.rs index c704ca89411f..09a94f5dd4fe 100644 --- a/rust/quote/runtime.rs +++ b/rust/quote/runtime.rs @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: Apache-2.0 OR MIT + use self::get_span::{GetSpan, GetSpanBase, GetSpanInner}; use crate::{IdentFragment, ToTokens, TokenStreamExt}; use core::fmt; diff --git a/rust/quote/spanned.rs b/rust/quote/spanned.rs index 6eba64445d89..54ce9177f45e 100644 --- a/rust/quote/spanned.rs +++ b/rust/quote/spanned.rs @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: Apache-2.0 OR MIT + use crate::ToTokens; use proc_macro2::extra::DelimSpan; use proc_macro2::{Span, TokenStream}; diff --git a/rust/quote/to_tokens.rs b/rust/quote/to_tokens.rs index f373092b650f..1af1089e1423 100644 --- a/rust/quote/to_tokens.rs +++ b/rust/quote/to_tokens.rs @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: Apache-2.0 OR MIT + use super::TokenStreamExt; use alloc::borrow::Cow; use alloc::rc::Rc; From 51177f023ce82c6d7d59bc5c64d94eea317d445d Mon Sep 17 00:00:00 2001 From: Miguel Ojeda Date: Mon, 24 Nov 2025 16:18:25 +0100 Subject: [PATCH 46/53] rust: quote: add `README.md` Originally, when the Rust upstream `alloc` standard library crate was vendored in commit 057b8d257107 ("rust: adapt `alloc` crate to the kernel"), a `README.md` file was added to explain the provenance and licensing of the source files. Thus do the same for the `quote` crate. Reviewed-by: Gary Guo Tested-by: Gary Guo Tested-by: Jesung Yang Link: https://patch.msgid.link/20251124151837.2184382-14-ojeda@kernel.org Signed-off-by: Miguel Ojeda --- rust/quote/README.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 rust/quote/README.md diff --git a/rust/quote/README.md b/rust/quote/README.md new file mode 100644 index 000000000000..186123eff1cb --- /dev/null +++ b/rust/quote/README.md @@ -0,0 +1,12 @@ +# `quote` + +These source files come from the Rust `quote` crate, version 1.0.40 +(released 2025-03-12), hosted in the +repository, licensed under "Apache-2.0 OR MIT" and only modified to add +the SPDX license identifiers. + +For copyright details, please see: + + https://github.com/dtolnay/quote/blob/1.0.40/README.md#license + https://github.com/dtolnay/quote/blob/1.0.40/LICENSE-APACHE + https://github.com/dtolnay/quote/blob/1.0.40/LICENSE-MIT From 88de91cc1ce7b3069ccabc1a5fbe16d41c663093 Mon Sep 17 00:00:00 2001 From: Miguel Ojeda Date: Mon, 24 Nov 2025 16:18:26 +0100 Subject: [PATCH 47/53] rust: quote: enable support in kbuild With all the new files in place and ready from the new crate, enable the support for it in the build system. Reviewed-by: Alice Ryhl Reviewed-by: Gary Guo Tested-by: Gary Guo Tested-by: Jesung Yang Link: https://patch.msgid.link/20251124151837.2184382-15-ojeda@kernel.org Signed-off-by: Miguel Ojeda --- Makefile | 1 + rust/Makefile | 38 ++++++++++++++++++++++++++++--- scripts/generate_rust_analyzer.py | 7 ++++++ 3 files changed, 43 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 589bcfe3bf75..85da055e4f00 100644 --- a/Makefile +++ b/Makefile @@ -1834,6 +1834,7 @@ rustfmt: $(Q)find $(srctree) $(RCS_FIND_IGNORE) \ \( \ -path $(srctree)/rust/proc-macro2 \ + -o -path $(srctree)/rust/quote \ \) -prune -o \ -type f -a -name '*.rs' -a ! -name '*generated*' -print \ | xargs $(RUSTFMT) $(rustfmt_flags) diff --git a/rust/Makefile b/rust/Makefile index 7dd1261ad98f..4f4a00594142 100644 --- a/rust/Makefile +++ b/rust/Makefile @@ -27,7 +27,7 @@ endif obj-$(CONFIG_RUST) += exports.o -always-$(CONFIG_RUST) += libproc_macro2.rlib +always-$(CONFIG_RUST) += libproc_macro2.rlib libquote.rlib always-$(CONFIG_RUST_KERNEL_DOCTESTS) += doctests_kernel_generated.rs always-$(CONFIG_RUST_KERNEL_DOCTESTS) += doctests_kernel_generated_kunit.c @@ -89,6 +89,18 @@ proc_macro2-flags := \ -Zcrate-attr='feature(proc_macro_byte_character,proc_macro_c_str_literals)' \ $(call cfgs-to-flags,$(proc_macro2-cfgs)) +quote-cfgs := \ + feature="proc-macro" + +quote-skip_flags := \ + --edition=2021 + +quote-flags := \ + --edition=2018 \ + --cap-lints=allow \ + --extern proc_macro2 \ + $(call cfgs-to-flags,$(quote-cfgs)) + # `rustdoc` did not save the target modifiers, thus workaround for # the time being (https://github.com/rust-lang/rust/issues/144521). rustdoc_modifiers_workaround := $(if $(call rustc-min-version,108800),-Cunsafe-allow-abi-mismatch=fixed-x18) @@ -143,10 +155,17 @@ rustdoc-proc_macro2: private rustc_target_flags = $(proc_macro2-flags) rustdoc-proc_macro2: $(src)/proc-macro2/lib.rs rustdoc-clean FORCE +$(call if_changed,rustdoc) +rustdoc-quote: private rustdoc_host = yes +rustdoc-quote: private rustc_target_flags = $(quote-flags) +rustdoc-quote: private skip_flags = $(quote-skip_flags) +rustdoc-quote: $(src)/quote/lib.rs rustdoc-clean rustdoc-proc_macro2 FORCE + +$(call if_changed,rustdoc) + rustdoc-macros: private rustdoc_host = yes rustdoc-macros: private rustc_target_flags = --crate-type proc-macro \ --extern proc_macro -rustdoc-macros: $(src)/macros/lib.rs rustdoc-clean rustdoc-proc_macro2 FORCE +rustdoc-macros: $(src)/macros/lib.rs rustdoc-clean rustdoc-proc_macro2 \ + rustdoc-quote FORCE +$(call if_changed,rustdoc) # Starting with Rust 1.82.0, skipping `-Wrustdoc::unescaped_backticks` should @@ -207,6 +226,11 @@ rusttestlib-proc_macro2: private rustc_target_flags = $(proc_macro2-flags) rusttestlib-proc_macro2: $(src)/proc-macro2/lib.rs FORCE +$(call if_changed,rustc_test_library) +rusttestlib-quote: private skip_flags = $(quote-skip_flags) +rusttestlib-quote: private rustc_target_flags = $(quote-flags) +rusttestlib-quote: $(src)/quote/lib.rs rusttestlib-proc_macro2 FORCE + +$(call if_changed,rustc_test_library) + rusttestlib-macros: private rustc_target_flags = --extern proc_macro rusttestlib-macros: private rustc_test_library_proc = yes rusttestlib-macros: $(src)/macros/lib.rs FORCE @@ -458,6 +482,12 @@ $(obj)/libproc_macro2.rlib: private rustc_target_flags = $(proc_macro2-flags) $(obj)/libproc_macro2.rlib: $(src)/proc-macro2/lib.rs FORCE +$(call if_changed_dep,rustc_procmacrolibrary) +$(obj)/libquote.rlib: private skip_clippy = 1 +$(obj)/libquote.rlib: private skip_flags = $(quote-skip_flags) +$(obj)/libquote.rlib: private rustc_target_flags = $(quote-flags) +$(obj)/libquote.rlib: $(src)/quote/lib.rs $(obj)/libproc_macro2.rlib FORCE + +$(call if_changed_dep,rustc_procmacrolibrary) + quiet_cmd_rustc_procmacro = $(RUSTC_OR_CLIPPY_QUIET) P $@ cmd_rustc_procmacro = \ $(RUSTC_OR_CLIPPY) $(rust_common_flags) $(rustc_target_flags) \ @@ -469,7 +499,8 @@ quiet_cmd_rustc_procmacro = $(RUSTC_OR_CLIPPY_QUIET) P $@ @$(objtree)/include/generated/rustc_cfg $< # Procedural macros can only be used with the `rustc` that compiled it. -$(obj)/$(libmacros_name): $(src)/macros/lib.rs $(obj)/libproc_macro2.rlib FORCE +$(obj)/$(libmacros_name): $(src)/macros/lib.rs $(obj)/libproc_macro2.rlib \ + $(obj)/libquote.rlib FORCE +$(call if_changed_dep,rustc_procmacro) $(obj)/$(libpin_init_internal_name): private rustc_target_flags = --cfg kernel @@ -493,6 +524,7 @@ rust-analyzer: $(Q)MAKEFLAGS= $(srctree)/scripts/generate_rust_analyzer.py \ --cfgs='core=$(core-cfgs)' $(core-edition) \ --cfgs='proc_macro2=$(proc_macro2-cfgs)' \ + --cfgs='quote=$(quote-cfgs)' \ $(realpath $(srctree)) $(realpath $(objtree)) \ $(rustc_sysroot) $(RUST_LIB_SRC) $(if $(KBUILD_EXTMOD),$(srcroot)) \ > rust-project.json diff --git a/scripts/generate_rust_analyzer.py b/scripts/generate_rust_analyzer.py index 00c6b7cc94b7..4faf153ed2ee 100755 --- a/scripts/generate_rust_analyzer.py +++ b/scripts/generate_rust_analyzer.py @@ -93,6 +93,13 @@ def generate_crates(srctree, objtree, sysroot_src, external_src, cfgs, core_edit cfg=crates_cfgs["proc_macro2"], ) + append_crate( + "quote", + srctree / "rust" / "quote" / "lib.rs", + ["alloc", "proc_macro", "proc_macro2"], + cfg=crates_cfgs["quote"], + ) + append_crate( "macros", srctree / "rust" / "macros" / "lib.rs", From 808c999fc9e7c366fd47da564e69d579c1dc8279 Mon Sep 17 00:00:00 2001 From: Miguel Ojeda Date: Mon, 24 Nov 2025 16:18:27 +0100 Subject: [PATCH 48/53] rust: syn: import crate This is a subset of the Rust `syn` crate, version 2.0.106 (released 2025-08-16), licensed under "Apache-2.0 OR MIT", from: https://github.com/dtolnay/syn/raw/2.0.106/src The files are copied as-is, with no modifications whatsoever (not even adding the SPDX identifiers). For copyright details, please see: https://github.com/dtolnay/syn/blob/2.0.106/README.md#license https://github.com/dtolnay/syn/blob/2.0.106/LICENSE-APACHE https://github.com/dtolnay/syn/blob/2.0.106/LICENSE-MIT The next two patches modify these files as needed for use within the kernel. This patch split allows reviewers to double-check the import and to clearly see the differences introduced. The following script may be used to verify the contents: for path in $(cd rust/syn/ && find . -type f -name '*.rs'); do curl --silent --show-error --location \ https://github.com/dtolnay/syn/raw/2.0.106/src/$path \ | diff --unified rust/syn/$path - && echo $path: OK done Reviewed-by: Gary Guo Tested-by: Gary Guo Tested-by: Jesung Yang Link: https://patch.msgid.link/20251124151837.2184382-16-ojeda@kernel.org Signed-off-by: Miguel Ojeda --- rust/syn/attr.rs | 836 +++++++ rust/syn/bigint.rs | 66 + rust/syn/buffer.rs | 434 ++++ rust/syn/classify.rs | 311 +++ rust/syn/custom_keyword.rs | 260 ++ rust/syn/custom_punctuation.rs | 304 +++ rust/syn/data.rs | 424 ++++ rust/syn/derive.rs | 259 ++ rust/syn/discouraged.rs | 225 ++ rust/syn/drops.rs | 58 + rust/syn/error.rs | 467 ++++ rust/syn/export.rs | 73 + rust/syn/expr.rs | 4173 ++++++++++++++++++++++++++++++++ rust/syn/ext.rs | 136 ++ rust/syn/file.rs | 125 + rust/syn/fixup.rs | 773 ++++++ rust/syn/gen/clone.rs | 2267 +++++++++++++++++ rust/syn/gen/debug.rs | 3238 +++++++++++++++++++++++++ rust/syn/gen/eq.rs | 2306 ++++++++++++++++++ rust/syn/gen/fold.rs | 3902 +++++++++++++++++++++++++++++ rust/syn/gen/hash.rs | 2876 ++++++++++++++++++++++ rust/syn/gen/visit.rs | 3941 ++++++++++++++++++++++++++++++ rust/syn/gen/visit_mut.rs | 3759 ++++++++++++++++++++++++++++ rust/syn/generics.rs | 1477 +++++++++++ rust/syn/group.rs | 291 +++ rust/syn/ident.rs | 108 + rust/syn/item.rs | 3490 ++++++++++++++++++++++++++ rust/syn/lib.rs | 1011 ++++++++ rust/syn/lifetime.rs | 156 ++ rust/syn/lit.rs | 1860 ++++++++++++++ rust/syn/lookahead.rs | 332 +++ rust/syn/mac.rs | 225 ++ rust/syn/macros.rs | 182 ++ rust/syn/meta.rs | 427 ++++ rust/syn/op.rs | 219 ++ rust/syn/parse.rs | 1419 +++++++++++ rust/syn/parse_macro_input.rs | 128 + rust/syn/parse_quote.rs | 240 ++ rust/syn/pat.rs | 955 ++++++++ rust/syn/path.rs | 966 ++++++++ rust/syn/precedence.rs | 210 ++ rust/syn/print.rs | 16 + rust/syn/punctuated.rs | 1155 +++++++++ rust/syn/restriction.rs | 178 ++ rust/syn/scan_expr.rs | 265 ++ rust/syn/sealed.rs | 4 + rust/syn/span.rs | 63 + rust/syn/spanned.rs | 118 + rust/syn/stmt.rs | 484 ++++ rust/syn/thread.rs | 60 + rust/syn/token.rs | 1096 +++++++++ rust/syn/tt.rs | 107 + rust/syn/ty.rs | 1271 ++++++++++ rust/syn/verbatim.rs | 33 + rust/syn/whitespace.rs | 65 + 55 files changed, 49824 insertions(+) create mode 100644 rust/syn/attr.rs create mode 100644 rust/syn/bigint.rs create mode 100644 rust/syn/buffer.rs create mode 100644 rust/syn/classify.rs create mode 100644 rust/syn/custom_keyword.rs create mode 100644 rust/syn/custom_punctuation.rs create mode 100644 rust/syn/data.rs create mode 100644 rust/syn/derive.rs create mode 100644 rust/syn/discouraged.rs create mode 100644 rust/syn/drops.rs create mode 100644 rust/syn/error.rs create mode 100644 rust/syn/export.rs create mode 100644 rust/syn/expr.rs create mode 100644 rust/syn/ext.rs create mode 100644 rust/syn/file.rs create mode 100644 rust/syn/fixup.rs create mode 100644 rust/syn/gen/clone.rs create mode 100644 rust/syn/gen/debug.rs create mode 100644 rust/syn/gen/eq.rs create mode 100644 rust/syn/gen/fold.rs create mode 100644 rust/syn/gen/hash.rs create mode 100644 rust/syn/gen/visit.rs create mode 100644 rust/syn/gen/visit_mut.rs create mode 100644 rust/syn/generics.rs create mode 100644 rust/syn/group.rs create mode 100644 rust/syn/ident.rs create mode 100644 rust/syn/item.rs create mode 100644 rust/syn/lib.rs create mode 100644 rust/syn/lifetime.rs create mode 100644 rust/syn/lit.rs create mode 100644 rust/syn/lookahead.rs create mode 100644 rust/syn/mac.rs create mode 100644 rust/syn/macros.rs create mode 100644 rust/syn/meta.rs create mode 100644 rust/syn/op.rs create mode 100644 rust/syn/parse.rs create mode 100644 rust/syn/parse_macro_input.rs create mode 100644 rust/syn/parse_quote.rs create mode 100644 rust/syn/pat.rs create mode 100644 rust/syn/path.rs create mode 100644 rust/syn/precedence.rs create mode 100644 rust/syn/print.rs create mode 100644 rust/syn/punctuated.rs create mode 100644 rust/syn/restriction.rs create mode 100644 rust/syn/scan_expr.rs create mode 100644 rust/syn/sealed.rs create mode 100644 rust/syn/span.rs create mode 100644 rust/syn/spanned.rs create mode 100644 rust/syn/stmt.rs create mode 100644 rust/syn/thread.rs create mode 100644 rust/syn/token.rs create mode 100644 rust/syn/tt.rs create mode 100644 rust/syn/ty.rs create mode 100644 rust/syn/verbatim.rs create mode 100644 rust/syn/whitespace.rs diff --git a/rust/syn/attr.rs b/rust/syn/attr.rs new file mode 100644 index 000000000000..2bdf96ee7fa9 --- /dev/null +++ b/rust/syn/attr.rs @@ -0,0 +1,836 @@ +#[cfg(feature = "parsing")] +use crate::error::Error; +#[cfg(feature = "parsing")] +use crate::error::Result; +use crate::expr::Expr; +use crate::mac::MacroDelimiter; +#[cfg(feature = "parsing")] +use crate::meta::{self, ParseNestedMeta}; +#[cfg(feature = "parsing")] +use crate::parse::{Parse, ParseStream, Parser}; +use crate::path::Path; +use crate::token; +use proc_macro2::TokenStream; +#[cfg(feature = "printing")] +use std::iter; +#[cfg(feature = "printing")] +use std::slice; + +ast_struct! { + /// An attribute, like `#[repr(transparent)]`. + /// + ///
+ /// + /// # Syntax + /// + /// Rust has six types of attributes. + /// + /// - Outer attributes like `#[repr(transparent)]`. These appear outside or + /// in front of the item they describe. + /// + /// - Inner attributes like `#![feature(proc_macro)]`. These appear inside + /// of the item they describe, usually a module. + /// + /// - Outer one-line doc comments like `/// Example`. + /// + /// - Inner one-line doc comments like `//! Please file an issue`. + /// + /// - Outer documentation blocks `/** Example */`. + /// + /// - Inner documentation blocks `/*! Please file an issue */`. + /// + /// The `style` field of type `AttrStyle` distinguishes whether an attribute + /// is outer or inner. + /// + /// Every attribute has a `path` that indicates the intended interpretation + /// of the rest of the attribute's contents. The path and the optional + /// additional contents are represented together in the `meta` field of the + /// attribute in three possible varieties: + /// + /// - Meta::Path — attributes whose information content conveys just a + /// path, for example the `#[test]` attribute. + /// + /// - Meta::List — attributes that carry arbitrary tokens after the + /// path, surrounded by a delimiter (parenthesis, bracket, or brace). For + /// example `#[derive(Copy)]` or `#[precondition(x < 5)]`. + /// + /// - Meta::NameValue — attributes with an `=` sign after the path, + /// followed by a Rust expression. For example `#[path = + /// "sys/windows.rs"]`. + /// + /// All doc comments are represented in the NameValue style with a path of + /// "doc", as this is how they are processed by the compiler and by + /// `macro_rules!` macros. + /// + /// ```text + /// #[derive(Copy, Clone)] + /// ~~~~~~Path + /// ^^^^^^^^^^^^^^^^^^^Meta::List + /// + /// #[path = "sys/windows.rs"] + /// ~~~~Path + /// ^^^^^^^^^^^^^^^^^^^^^^^Meta::NameValue + /// + /// #[test] + /// ^^^^Meta::Path + /// ``` + /// + ///
+ /// + /// # Parsing from tokens to Attribute + /// + /// This type does not implement the [`Parse`] trait and thus cannot be + /// parsed directly by [`ParseStream::parse`]. Instead use + /// [`ParseStream::call`] with one of the two parser functions + /// [`Attribute::parse_outer`] or [`Attribute::parse_inner`] depending on + /// which you intend to parse. + /// + /// [`Parse`]: crate::parse::Parse + /// [`ParseStream::parse`]: crate::parse::ParseBuffer::parse + /// [`ParseStream::call`]: crate::parse::ParseBuffer::call + /// + /// ``` + /// use syn::{Attribute, Ident, Result, Token}; + /// use syn::parse::{Parse, ParseStream}; + /// + /// // Parses a unit struct with attributes. + /// // + /// // #[path = "s.tmpl"] + /// // struct S; + /// struct UnitStruct { + /// attrs: Vec, + /// struct_token: Token![struct], + /// name: Ident, + /// semi_token: Token![;], + /// } + /// + /// impl Parse for UnitStruct { + /// fn parse(input: ParseStream) -> Result { + /// Ok(UnitStruct { + /// attrs: input.call(Attribute::parse_outer)?, + /// struct_token: input.parse()?, + /// name: input.parse()?, + /// semi_token: input.parse()?, + /// }) + /// } + /// } + /// ``` + /// + ///


+ /// + /// # Parsing from Attribute to structured arguments + /// + /// The grammar of attributes in Rust is very flexible, which makes the + /// syntax tree not that useful on its own. In particular, arguments of the + /// `Meta::List` variety of attribute are held in an arbitrary `tokens: + /// TokenStream`. Macros are expected to check the `path` of the attribute, + /// decide whether they recognize it, and then parse the remaining tokens + /// according to whatever grammar they wish to require for that kind of + /// attribute. Use [`parse_args()`] to parse those tokens into the expected + /// data structure. + /// + /// [`parse_args()`]: Attribute::parse_args + /// + ///


+ /// + /// # Doc comments + /// + /// The compiler transforms doc comments, such as `/// comment` and `/*! + /// comment */`, into attributes before macros are expanded. Each comment is + /// expanded into an attribute of the form `#[doc = r"comment"]`. + /// + /// As an example, the following `mod` items are expanded identically: + /// + /// ``` + /// # use syn::{ItemMod, parse_quote}; + /// let doc: ItemMod = parse_quote! { + /// /// Single line doc comments + /// /// We write so many! + /// /** + /// * Multi-line comments... + /// * May span many lines + /// */ + /// mod example { + /// //! Of course, they can be inner too + /// /*! And fit in a single line */ + /// } + /// }; + /// let attr: ItemMod = parse_quote! { + /// #[doc = r" Single line doc comments"] + /// #[doc = r" We write so many!"] + /// #[doc = r" + /// * Multi-line comments... + /// * May span many lines + /// "] + /// mod example { + /// #![doc = r" Of course, they can be inner too"] + /// #![doc = r" And fit in a single line "] + /// } + /// }; + /// assert_eq!(doc, attr); + /// ``` + #[cfg_attr(docsrs, doc(cfg(any(feature = "full", feature = "derive"))))] + pub struct Attribute { + pub pound_token: Token![#], + pub style: AttrStyle, + pub bracket_token: token::Bracket, + pub meta: Meta, + } +} + +impl Attribute { + /// Returns the path that identifies the interpretation of this attribute. + /// + /// For example this would return the `test` in `#[test]`, the `derive` in + /// `#[derive(Copy)]`, and the `path` in `#[path = "sys/windows.rs"]`. + pub fn path(&self) -> &Path { + self.meta.path() + } + + /// Parse the arguments to the attribute as a syntax tree. + /// + /// This is similar to pulling out the `TokenStream` from `Meta::List` and + /// doing `syn::parse2::(meta_list.tokens)`, except that using + /// `parse_args` the error message has a more useful span when `tokens` is + /// empty. + /// + /// The surrounding delimiters are *not* included in the input to the + /// parser. + /// + /// ```text + /// #[my_attr(value < 5)] + /// ^^^^^^^^^ what gets parsed + /// ``` + /// + /// # Example + /// + /// ``` + /// use syn::{parse_quote, Attribute, Expr}; + /// + /// let attr: Attribute = parse_quote! { + /// #[precondition(value < 5)] + /// }; + /// + /// if attr.path().is_ident("precondition") { + /// let precondition: Expr = attr.parse_args()?; + /// // ... + /// } + /// # anyhow::Ok(()) + /// ``` + #[cfg(feature = "parsing")] + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + pub fn parse_args(&self) -> Result { + self.parse_args_with(T::parse) + } + + /// Parse the arguments to the attribute using the given parser. + /// + /// # Example + /// + /// ``` + /// use syn::{parse_quote, Attribute}; + /// + /// let attr: Attribute = parse_quote! { + /// #[inception { #[brrrrrrraaaaawwwwrwrrrmrmrmmrmrmmmmm] }] + /// }; + /// + /// let bwom = attr.parse_args_with(Attribute::parse_outer)?; + /// + /// // Attribute does not have a Parse impl, so we couldn't directly do: + /// // let bwom: Attribute = attr.parse_args()?; + /// # anyhow::Ok(()) + /// ``` + #[cfg(feature = "parsing")] + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + pub fn parse_args_with(&self, parser: F) -> Result { + match &self.meta { + Meta::Path(path) => Err(crate::error::new2( + path.segments.first().unwrap().ident.span(), + path.segments.last().unwrap().ident.span(), + format!( + "expected attribute arguments in parentheses: {}[{}(...)]", + parsing::DisplayAttrStyle(&self.style), + parsing::DisplayPath(path), + ), + )), + Meta::NameValue(meta) => Err(Error::new( + meta.eq_token.span, + format_args!( + "expected parentheses: {}[{}(...)]", + parsing::DisplayAttrStyle(&self.style), + parsing::DisplayPath(&meta.path), + ), + )), + Meta::List(meta) => meta.parse_args_with(parser), + } + } + + /// Parse the arguments to the attribute, expecting it to follow the + /// conventional structure used by most of Rust's built-in attributes. + /// + /// The [*Meta Item Attribute Syntax*][syntax] section in the Rust reference + /// explains the convention in more detail. Not all attributes follow this + /// convention, so [`parse_args()`][Self::parse_args] is available if you + /// need to parse arbitrarily goofy attribute syntax. + /// + /// [syntax]: https://doc.rust-lang.org/reference/attributes.html#meta-item-attribute-syntax + /// + /// # Example + /// + /// We'll parse a struct, and then parse some of Rust's `#[repr]` attribute + /// syntax. + /// + /// ``` + /// use syn::{parenthesized, parse_quote, token, ItemStruct, LitInt}; + /// + /// let input: ItemStruct = parse_quote! { + /// #[repr(C, align(4))] + /// pub struct MyStruct(u16, u32); + /// }; + /// + /// let mut repr_c = false; + /// let mut repr_transparent = false; + /// let mut repr_align = None::; + /// let mut repr_packed = None::; + /// for attr in &input.attrs { + /// if attr.path().is_ident("repr") { + /// attr.parse_nested_meta(|meta| { + /// // #[repr(C)] + /// if meta.path.is_ident("C") { + /// repr_c = true; + /// return Ok(()); + /// } + /// + /// // #[repr(transparent)] + /// if meta.path.is_ident("transparent") { + /// repr_transparent = true; + /// return Ok(()); + /// } + /// + /// // #[repr(align(N))] + /// if meta.path.is_ident("align") { + /// let content; + /// parenthesized!(content in meta.input); + /// let lit: LitInt = content.parse()?; + /// let n: usize = lit.base10_parse()?; + /// repr_align = Some(n); + /// return Ok(()); + /// } + /// + /// // #[repr(packed)] or #[repr(packed(N))], omitted N means 1 + /// if meta.path.is_ident("packed") { + /// if meta.input.peek(token::Paren) { + /// let content; + /// parenthesized!(content in meta.input); + /// let lit: LitInt = content.parse()?; + /// let n: usize = lit.base10_parse()?; + /// repr_packed = Some(n); + /// } else { + /// repr_packed = Some(1); + /// } + /// return Ok(()); + /// } + /// + /// Err(meta.error("unrecognized repr")) + /// })?; + /// } + /// } + /// # anyhow::Ok(()) + /// ``` + /// + /// # Alternatives + /// + /// In some cases, for attributes which have nested layers of structured + /// content, the following less flexible approach might be more convenient: + /// + /// ``` + /// # use syn::{parse_quote, ItemStruct}; + /// # + /// # let input: ItemStruct = parse_quote! { + /// # #[repr(C, align(4))] + /// # pub struct MyStruct(u16, u32); + /// # }; + /// # + /// use syn::punctuated::Punctuated; + /// use syn::{parenthesized, token, Error, LitInt, Meta, Token}; + /// + /// let mut repr_c = false; + /// let mut repr_transparent = false; + /// let mut repr_align = None::; + /// let mut repr_packed = None::; + /// for attr in &input.attrs { + /// if attr.path().is_ident("repr") { + /// let nested = attr.parse_args_with(Punctuated::::parse_terminated)?; + /// for meta in nested { + /// match meta { + /// // #[repr(C)] + /// Meta::Path(path) if path.is_ident("C") => { + /// repr_c = true; + /// } + /// + /// // #[repr(align(N))] + /// Meta::List(meta) if meta.path.is_ident("align") => { + /// let lit: LitInt = meta.parse_args()?; + /// let n: usize = lit.base10_parse()?; + /// repr_align = Some(n); + /// } + /// + /// /* ... */ + /// + /// _ => { + /// return Err(Error::new_spanned(meta, "unrecognized repr")); + /// } + /// } + /// } + /// } + /// } + /// # Ok(()) + /// ``` + #[cfg(feature = "parsing")] + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + pub fn parse_nested_meta( + &self, + logic: impl FnMut(ParseNestedMeta) -> Result<()>, + ) -> Result<()> { + self.parse_args_with(meta::parser(logic)) + } + + /// Parses zero or more outer attributes from the stream. + /// + /// # Example + /// + /// See + /// [*Parsing from tokens to Attribute*](#parsing-from-tokens-to-attribute). + #[cfg(feature = "parsing")] + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + pub fn parse_outer(input: ParseStream) -> Result> { + let mut attrs = Vec::new(); + while input.peek(Token![#]) { + attrs.push(input.call(parsing::single_parse_outer)?); + } + Ok(attrs) + } + + /// Parses zero or more inner attributes from the stream. + /// + /// # Example + /// + /// See + /// [*Parsing from tokens to Attribute*](#parsing-from-tokens-to-attribute). + #[cfg(feature = "parsing")] + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + pub fn parse_inner(input: ParseStream) -> Result> { + let mut attrs = Vec::new(); + parsing::parse_inner(input, &mut attrs)?; + Ok(attrs) + } +} + +ast_enum! { + /// Distinguishes between attributes that decorate an item and attributes + /// that are contained within an item. + /// + /// # Outer attributes + /// + /// - `#[repr(transparent)]` + /// - `/// # Example` + /// - `/** Please file an issue */` + /// + /// # Inner attributes + /// + /// - `#![feature(proc_macro)]` + /// - `//! # Example` + /// - `/*! Please file an issue */` + #[cfg_attr(docsrs, doc(cfg(any(feature = "full", feature = "derive"))))] + pub enum AttrStyle { + Outer, + Inner(Token![!]), + } +} + +ast_enum! { + /// Content of a compile-time structured attribute. + /// + /// ## Path + /// + /// A meta path is like the `test` in `#[test]`. + /// + /// ## List + /// + /// A meta list is like the `derive(Copy)` in `#[derive(Copy)]`. + /// + /// ## NameValue + /// + /// A name-value meta is like the `path = "..."` in `#[path = + /// "sys/windows.rs"]`. + /// + /// # Syntax tree enum + /// + /// This type is a [syntax tree enum]. + /// + /// [syntax tree enum]: crate::expr::Expr#syntax-tree-enums + #[cfg_attr(docsrs, doc(cfg(any(feature = "full", feature = "derive"))))] + pub enum Meta { + Path(Path), + + /// A structured list within an attribute, like `derive(Copy, Clone)`. + List(MetaList), + + /// A name-value pair within an attribute, like `feature = "nightly"`. + NameValue(MetaNameValue), + } +} + +ast_struct! { + /// A structured list within an attribute, like `derive(Copy, Clone)`. + #[cfg_attr(docsrs, doc(cfg(any(feature = "full", feature = "derive"))))] + pub struct MetaList { + pub path: Path, + pub delimiter: MacroDelimiter, + pub tokens: TokenStream, + } +} + +ast_struct! { + /// A name-value pair within an attribute, like `feature = "nightly"`. + #[cfg_attr(docsrs, doc(cfg(any(feature = "full", feature = "derive"))))] + pub struct MetaNameValue { + pub path: Path, + pub eq_token: Token![=], + pub value: Expr, + } +} + +impl Meta { + /// Returns the path that begins this structured meta item. + /// + /// For example this would return the `test` in `#[test]`, the `derive` in + /// `#[derive(Copy)]`, and the `path` in `#[path = "sys/windows.rs"]`. + pub fn path(&self) -> &Path { + match self { + Meta::Path(path) => path, + Meta::List(meta) => &meta.path, + Meta::NameValue(meta) => &meta.path, + } + } + + /// Error if this is a `Meta::List` or `Meta::NameValue`. + #[cfg(feature = "parsing")] + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + pub fn require_path_only(&self) -> Result<&Path> { + let error_span = match self { + Meta::Path(path) => return Ok(path), + Meta::List(meta) => meta.delimiter.span().open(), + Meta::NameValue(meta) => meta.eq_token.span, + }; + Err(Error::new(error_span, "unexpected token in attribute")) + } + + /// Error if this is a `Meta::Path` or `Meta::NameValue`. + #[cfg(feature = "parsing")] + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + pub fn require_list(&self) -> Result<&MetaList> { + match self { + Meta::List(meta) => Ok(meta), + Meta::Path(path) => Err(crate::error::new2( + path.segments.first().unwrap().ident.span(), + path.segments.last().unwrap().ident.span(), + format!( + "expected attribute arguments in parentheses: `{}(...)`", + parsing::DisplayPath(path), + ), + )), + Meta::NameValue(meta) => Err(Error::new(meta.eq_token.span, "expected `(`")), + } + } + + /// Error if this is a `Meta::Path` or `Meta::List`. + #[cfg(feature = "parsing")] + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + pub fn require_name_value(&self) -> Result<&MetaNameValue> { + match self { + Meta::NameValue(meta) => Ok(meta), + Meta::Path(path) => Err(crate::error::new2( + path.segments.first().unwrap().ident.span(), + path.segments.last().unwrap().ident.span(), + format!( + "expected a value for this attribute: `{} = ...`", + parsing::DisplayPath(path), + ), + )), + Meta::List(meta) => Err(Error::new(meta.delimiter.span().open(), "expected `=`")), + } + } +} + +impl MetaList { + /// See [`Attribute::parse_args`]. + #[cfg(feature = "parsing")] + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + pub fn parse_args(&self) -> Result { + self.parse_args_with(T::parse) + } + + /// See [`Attribute::parse_args_with`]. + #[cfg(feature = "parsing")] + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + pub fn parse_args_with(&self, parser: F) -> Result { + let scope = self.delimiter.span().close(); + crate::parse::parse_scoped(parser, scope, self.tokens.clone()) + } + + /// See [`Attribute::parse_nested_meta`]. + #[cfg(feature = "parsing")] + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + pub fn parse_nested_meta( + &self, + logic: impl FnMut(ParseNestedMeta) -> Result<()>, + ) -> Result<()> { + self.parse_args_with(meta::parser(logic)) + } +} + +#[cfg(feature = "printing")] +pub(crate) trait FilterAttrs<'a> { + type Ret: Iterator; + + fn outer(self) -> Self::Ret; + #[cfg(feature = "full")] + fn inner(self) -> Self::Ret; +} + +#[cfg(feature = "printing")] +impl<'a> FilterAttrs<'a> for &'a [Attribute] { + type Ret = iter::Filter, fn(&&Attribute) -> bool>; + + fn outer(self) -> Self::Ret { + fn is_outer(attr: &&Attribute) -> bool { + match attr.style { + AttrStyle::Outer => true, + AttrStyle::Inner(_) => false, + } + } + self.iter().filter(is_outer) + } + + #[cfg(feature = "full")] + fn inner(self) -> Self::Ret { + fn is_inner(attr: &&Attribute) -> bool { + match attr.style { + AttrStyle::Inner(_) => true, + AttrStyle::Outer => false, + } + } + self.iter().filter(is_inner) + } +} + +impl From for Meta { + fn from(meta: Path) -> Meta { + Meta::Path(meta) + } +} + +impl From for Meta { + fn from(meta: MetaList) -> Meta { + Meta::List(meta) + } +} + +impl From for Meta { + fn from(meta: MetaNameValue) -> Meta { + Meta::NameValue(meta) + } +} + +#[cfg(feature = "parsing")] +pub(crate) mod parsing { + use crate::attr::{AttrStyle, Attribute, Meta, MetaList, MetaNameValue}; + use crate::error::Result; + use crate::expr::{Expr, ExprLit}; + use crate::lit::Lit; + use crate::parse::discouraged::Speculative as _; + use crate::parse::{Parse, ParseStream}; + use crate::path::Path; + use crate::{mac, token}; + use proc_macro2::Ident; + use std::fmt::{self, Display}; + + pub(crate) fn parse_inner(input: ParseStream, attrs: &mut Vec) -> Result<()> { + while input.peek(Token![#]) && input.peek2(Token![!]) { + attrs.push(input.call(single_parse_inner)?); + } + Ok(()) + } + + pub(crate) fn single_parse_inner(input: ParseStream) -> Result { + let content; + Ok(Attribute { + pound_token: input.parse()?, + style: AttrStyle::Inner(input.parse()?), + bracket_token: bracketed!(content in input), + meta: content.parse()?, + }) + } + + pub(crate) fn single_parse_outer(input: ParseStream) -> Result { + let content; + Ok(Attribute { + pound_token: input.parse()?, + style: AttrStyle::Outer, + bracket_token: bracketed!(content in input), + meta: content.parse()?, + }) + } + + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + impl Parse for Meta { + fn parse(input: ParseStream) -> Result { + let path = parse_outermost_meta_path(input)?; + parse_meta_after_path(path, input) + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + impl Parse for MetaList { + fn parse(input: ParseStream) -> Result { + let path = parse_outermost_meta_path(input)?; + parse_meta_list_after_path(path, input) + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + impl Parse for MetaNameValue { + fn parse(input: ParseStream) -> Result { + let path = parse_outermost_meta_path(input)?; + parse_meta_name_value_after_path(path, input) + } + } + + // Unlike meta::parse_meta_path which accepts arbitrary keywords in the path, + // only the `unsafe` keyword is accepted as an attribute's outermost path. + fn parse_outermost_meta_path(input: ParseStream) -> Result { + if input.peek(Token![unsafe]) { + let unsafe_token: Token![unsafe] = input.parse()?; + Ok(Path::from(Ident::new("unsafe", unsafe_token.span))) + } else { + Path::parse_mod_style(input) + } + } + + pub(crate) fn parse_meta_after_path(path: Path, input: ParseStream) -> Result { + if input.peek(token::Paren) || input.peek(token::Bracket) || input.peek(token::Brace) { + parse_meta_list_after_path(path, input).map(Meta::List) + } else if input.peek(Token![=]) { + parse_meta_name_value_after_path(path, input).map(Meta::NameValue) + } else { + Ok(Meta::Path(path)) + } + } + + fn parse_meta_list_after_path(path: Path, input: ParseStream) -> Result { + let (delimiter, tokens) = mac::parse_delimiter(input)?; + Ok(MetaList { + path, + delimiter, + tokens, + }) + } + + fn parse_meta_name_value_after_path(path: Path, input: ParseStream) -> Result { + let eq_token: Token![=] = input.parse()?; + let ahead = input.fork(); + let lit: Option = ahead.parse()?; + let value = if let (Some(lit), true) = (lit, ahead.is_empty()) { + input.advance_to(&ahead); + Expr::Lit(ExprLit { + attrs: Vec::new(), + lit, + }) + } else if input.peek(Token![#]) && input.peek2(token::Bracket) { + return Err(input.error("unexpected attribute inside of attribute")); + } else { + input.parse()? + }; + Ok(MetaNameValue { + path, + eq_token, + value, + }) + } + + pub(super) struct DisplayAttrStyle<'a>(pub &'a AttrStyle); + + impl<'a> Display for DisplayAttrStyle<'a> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str(match self.0 { + AttrStyle::Outer => "#", + AttrStyle::Inner(_) => "#!", + }) + } + } + + pub(super) struct DisplayPath<'a>(pub &'a Path); + + impl<'a> Display for DisplayPath<'a> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + for (i, segment) in self.0.segments.iter().enumerate() { + if i > 0 || self.0.leading_colon.is_some() { + formatter.write_str("::")?; + } + write!(formatter, "{}", segment.ident)?; + } + Ok(()) + } + } +} + +#[cfg(feature = "printing")] +mod printing { + use crate::attr::{AttrStyle, Attribute, Meta, MetaList, MetaNameValue}; + use crate::path; + use crate::path::printing::PathStyle; + use proc_macro2::TokenStream; + use quote::ToTokens; + + #[cfg_attr(docsrs, doc(cfg(feature = "printing")))] + impl ToTokens for Attribute { + fn to_tokens(&self, tokens: &mut TokenStream) { + self.pound_token.to_tokens(tokens); + if let AttrStyle::Inner(b) = &self.style { + b.to_tokens(tokens); + } + self.bracket_token.surround(tokens, |tokens| { + self.meta.to_tokens(tokens); + }); + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "printing")))] + impl ToTokens for Meta { + fn to_tokens(&self, tokens: &mut TokenStream) { + match self { + Meta::Path(path) => path::printing::print_path(tokens, path, PathStyle::Mod), + Meta::List(meta_list) => meta_list.to_tokens(tokens), + Meta::NameValue(meta_name_value) => meta_name_value.to_tokens(tokens), + } + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "printing")))] + impl ToTokens for MetaList { + fn to_tokens(&self, tokens: &mut TokenStream) { + path::printing::print_path(tokens, &self.path, PathStyle::Mod); + self.delimiter.surround(tokens, self.tokens.clone()); + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "printing")))] + impl ToTokens for MetaNameValue { + fn to_tokens(&self, tokens: &mut TokenStream) { + path::printing::print_path(tokens, &self.path, PathStyle::Mod); + self.eq_token.to_tokens(tokens); + self.value.to_tokens(tokens); + } + } +} diff --git a/rust/syn/bigint.rs b/rust/syn/bigint.rs new file mode 100644 index 000000000000..66aaa9372540 --- /dev/null +++ b/rust/syn/bigint.rs @@ -0,0 +1,66 @@ +use std::ops::{AddAssign, MulAssign}; + +// For implementing base10_digits() accessor on LitInt. +pub(crate) struct BigInt { + digits: Vec, +} + +impl BigInt { + pub(crate) fn new() -> Self { + BigInt { digits: Vec::new() } + } + + pub(crate) fn to_string(&self) -> String { + let mut repr = String::with_capacity(self.digits.len()); + + let mut has_nonzero = false; + for digit in self.digits.iter().rev() { + has_nonzero |= *digit != 0; + if has_nonzero { + repr.push((*digit + b'0') as char); + } + } + + if repr.is_empty() { + repr.push('0'); + } + + repr + } + + fn reserve_two_digits(&mut self) { + let len = self.digits.len(); + let desired = + len + !self.digits.ends_with(&[0, 0]) as usize + !self.digits.ends_with(&[0]) as usize; + self.digits.resize(desired, 0); + } +} + +impl AddAssign for BigInt { + // Assumes increment <16. + fn add_assign(&mut self, mut increment: u8) { + self.reserve_two_digits(); + + let mut i = 0; + while increment > 0 { + let sum = self.digits[i] + increment; + self.digits[i] = sum % 10; + increment = sum / 10; + i += 1; + } + } +} + +impl MulAssign for BigInt { + // Assumes base <=16. + fn mul_assign(&mut self, base: u8) { + self.reserve_two_digits(); + + let mut carry = 0; + for digit in &mut self.digits { + let prod = *digit * base + carry; + *digit = prod % 10; + carry = prod / 10; + } + } +} diff --git a/rust/syn/buffer.rs b/rust/syn/buffer.rs new file mode 100644 index 000000000000..7b6a504eeb7c --- /dev/null +++ b/rust/syn/buffer.rs @@ -0,0 +1,434 @@ +//! A stably addressed token buffer supporting efficient traversal based on a +//! cheaply copyable cursor. + +// This module is heavily commented as it contains most of the unsafe code in +// Syn, and caution should be used when editing it. The public-facing interface +// is 100% safe but the implementation is fragile internally. + +use crate::Lifetime; +use proc_macro2::extra::DelimSpan; +use proc_macro2::{Delimiter, Group, Ident, Literal, Punct, Spacing, Span, TokenStream, TokenTree}; +use std::cmp::Ordering; +use std::marker::PhantomData; +use std::ptr; + +/// Internal type which is used instead of `TokenTree` to represent a token tree +/// within a `TokenBuffer`. +enum Entry { + // Mimicking types from proc-macro. + // Group entries contain the offset to the matching End entry. + Group(Group, usize), + Ident(Ident), + Punct(Punct), + Literal(Literal), + // End entries contain the offset (negative) to the start of the buffer, and + // offset (negative) to the matching Group entry. + End(isize, isize), +} + +/// A buffer that can be efficiently traversed multiple times, unlike +/// `TokenStream` which requires a deep copy in order to traverse more than +/// once. +pub struct TokenBuffer { + // NOTE: Do not implement clone on this - while the current design could be + // cloned, other designs which could be desirable may not be cloneable. + entries: Box<[Entry]>, +} + +impl TokenBuffer { + fn recursive_new(entries: &mut Vec, stream: TokenStream) { + for tt in stream { + match tt { + TokenTree::Ident(ident) => entries.push(Entry::Ident(ident)), + TokenTree::Punct(punct) => entries.push(Entry::Punct(punct)), + TokenTree::Literal(literal) => entries.push(Entry::Literal(literal)), + TokenTree::Group(group) => { + let group_start_index = entries.len(); + entries.push(Entry::End(0, 0)); // we replace this below + Self::recursive_new(entries, group.stream()); + let group_end_index = entries.len(); + let group_offset = group_end_index - group_start_index; + entries.push(Entry::End( + -(group_end_index as isize), + -(group_offset as isize), + )); + entries[group_start_index] = Entry::Group(group, group_offset); + } + } + } + } + + /// Creates a `TokenBuffer` containing all the tokens from the input + /// `proc_macro::TokenStream`. + #[cfg(feature = "proc-macro")] + #[cfg_attr(docsrs, doc(cfg(feature = "proc-macro")))] + pub fn new(stream: proc_macro::TokenStream) -> Self { + Self::new2(stream.into()) + } + + /// Creates a `TokenBuffer` containing all the tokens from the input + /// `proc_macro2::TokenStream`. + pub fn new2(stream: TokenStream) -> Self { + let mut entries = Vec::new(); + Self::recursive_new(&mut entries, stream); + entries.push(Entry::End(-(entries.len() as isize), 0)); + Self { + entries: entries.into_boxed_slice(), + } + } + + /// Creates a cursor referencing the first token in the buffer and able to + /// traverse until the end of the buffer. + pub fn begin(&self) -> Cursor { + let ptr = self.entries.as_ptr(); + unsafe { Cursor::create(ptr, ptr.add(self.entries.len() - 1)) } + } +} + +/// A cheaply copyable cursor into a `TokenBuffer`. +/// +/// This cursor holds a shared reference into the immutable data which is used +/// internally to represent a `TokenStream`, and can be efficiently manipulated +/// and copied around. +/// +/// An empty `Cursor` can be created directly, or one may create a `TokenBuffer` +/// object and get a cursor to its first token with `begin()`. +pub struct Cursor<'a> { + // The current entry which the `Cursor` is pointing at. + ptr: *const Entry, + // This is the only `Entry::End` object which this cursor is allowed to + // point at. All other `End` objects are skipped over in `Cursor::create`. + scope: *const Entry, + // Cursor is covariant in 'a. This field ensures that our pointers are still + // valid. + marker: PhantomData<&'a Entry>, +} + +impl<'a> Cursor<'a> { + /// Creates a cursor referencing a static empty TokenStream. + pub fn empty() -> Self { + // It's safe in this situation for us to put an `Entry` object in global + // storage, despite it not actually being safe to send across threads + // (`Ident` is a reference into a thread-local table). This is because + // this entry never includes a `Ident` object. + // + // This wrapper struct allows us to break the rules and put a `Sync` + // object in global storage. + struct UnsafeSyncEntry(Entry); + unsafe impl Sync for UnsafeSyncEntry {} + static EMPTY_ENTRY: UnsafeSyncEntry = UnsafeSyncEntry(Entry::End(0, 0)); + + Cursor { + ptr: &EMPTY_ENTRY.0, + scope: &EMPTY_ENTRY.0, + marker: PhantomData, + } + } + + /// This create method intelligently exits non-explicitly-entered + /// `None`-delimited scopes when the cursor reaches the end of them, + /// allowing for them to be treated transparently. + unsafe fn create(mut ptr: *const Entry, scope: *const Entry) -> Self { + // NOTE: If we're looking at a `End`, we want to advance the cursor + // past it, unless `ptr == scope`, which means that we're at the edge of + // our cursor's scope. We should only have `ptr != scope` at the exit + // from None-delimited groups entered with `ignore_none`. + while let Entry::End(..) = unsafe { &*ptr } { + if ptr::eq(ptr, scope) { + break; + } + ptr = unsafe { ptr.add(1) }; + } + + Cursor { + ptr, + scope, + marker: PhantomData, + } + } + + /// Get the current entry. + fn entry(self) -> &'a Entry { + unsafe { &*self.ptr } + } + + /// Bump the cursor to point at the next token after the current one. This + /// is undefined behavior if the cursor is currently looking at an + /// `Entry::End`. + /// + /// If the cursor is looking at an `Entry::Group`, the bumped cursor will + /// point at the first token in the group (with the same scope end). + unsafe fn bump_ignore_group(self) -> Cursor<'a> { + unsafe { Cursor::create(self.ptr.offset(1), self.scope) } + } + + /// While the cursor is looking at a `None`-delimited group, move it to look + /// at the first token inside instead. If the group is empty, this will move + /// the cursor past the `None`-delimited group. + /// + /// WARNING: This mutates its argument. + fn ignore_none(&mut self) { + while let Entry::Group(group, _) = self.entry() { + if group.delimiter() == Delimiter::None { + unsafe { *self = self.bump_ignore_group() }; + } else { + break; + } + } + } + + /// Checks whether the cursor is currently pointing at the end of its valid + /// scope. + pub fn eof(self) -> bool { + // We're at eof if we're at the end of our scope. + ptr::eq(self.ptr, self.scope) + } + + /// If the cursor is pointing at a `Ident`, returns it along with a cursor + /// pointing at the next `TokenTree`. + pub fn ident(mut self) -> Option<(Ident, Cursor<'a>)> { + self.ignore_none(); + match self.entry() { + Entry::Ident(ident) => Some((ident.clone(), unsafe { self.bump_ignore_group() })), + _ => None, + } + } + + /// If the cursor is pointing at a `Punct`, returns it along with a cursor + /// pointing at the next `TokenTree`. + pub fn punct(mut self) -> Option<(Punct, Cursor<'a>)> { + self.ignore_none(); + match self.entry() { + Entry::Punct(punct) if punct.as_char() != '\'' => { + Some((punct.clone(), unsafe { self.bump_ignore_group() })) + } + _ => None, + } + } + + /// If the cursor is pointing at a `Literal`, return it along with a cursor + /// pointing at the next `TokenTree`. + pub fn literal(mut self) -> Option<(Literal, Cursor<'a>)> { + self.ignore_none(); + match self.entry() { + Entry::Literal(literal) => Some((literal.clone(), unsafe { self.bump_ignore_group() })), + _ => None, + } + } + + /// If the cursor is pointing at a `Lifetime`, returns it along with a + /// cursor pointing at the next `TokenTree`. + pub fn lifetime(mut self) -> Option<(Lifetime, Cursor<'a>)> { + self.ignore_none(); + match self.entry() { + Entry::Punct(punct) if punct.as_char() == '\'' && punct.spacing() == Spacing::Joint => { + let next = unsafe { self.bump_ignore_group() }; + let (ident, rest) = next.ident()?; + let lifetime = Lifetime { + apostrophe: punct.span(), + ident, + }; + Some((lifetime, rest)) + } + _ => None, + } + } + + /// If the cursor is pointing at a `Group` with the given delimiter, returns + /// a cursor into that group and one pointing to the next `TokenTree`. + pub fn group(mut self, delim: Delimiter) -> Option<(Cursor<'a>, DelimSpan, Cursor<'a>)> { + // If we're not trying to enter a none-delimited group, we want to + // ignore them. We have to make sure to _not_ ignore them when we want + // to enter them, of course. For obvious reasons. + if delim != Delimiter::None { + self.ignore_none(); + } + + if let Entry::Group(group, end_offset) = self.entry() { + if group.delimiter() == delim { + let span = group.delim_span(); + let end_of_group = unsafe { self.ptr.add(*end_offset) }; + let inside_of_group = unsafe { Cursor::create(self.ptr.add(1), end_of_group) }; + let after_group = unsafe { Cursor::create(end_of_group, self.scope) }; + return Some((inside_of_group, span, after_group)); + } + } + + None + } + + /// If the cursor is pointing at a `Group`, returns a cursor into the group + /// and one pointing to the next `TokenTree`. + pub fn any_group(self) -> Option<(Cursor<'a>, Delimiter, DelimSpan, Cursor<'a>)> { + if let Entry::Group(group, end_offset) = self.entry() { + let delimiter = group.delimiter(); + let span = group.delim_span(); + let end_of_group = unsafe { self.ptr.add(*end_offset) }; + let inside_of_group = unsafe { Cursor::create(self.ptr.add(1), end_of_group) }; + let after_group = unsafe { Cursor::create(end_of_group, self.scope) }; + return Some((inside_of_group, delimiter, span, after_group)); + } + + None + } + + pub(crate) fn any_group_token(self) -> Option<(Group, Cursor<'a>)> { + if let Entry::Group(group, end_offset) = self.entry() { + let end_of_group = unsafe { self.ptr.add(*end_offset) }; + let after_group = unsafe { Cursor::create(end_of_group, self.scope) }; + return Some((group.clone(), after_group)); + } + + None + } + + /// Copies all remaining tokens visible from this cursor into a + /// `TokenStream`. + pub fn token_stream(self) -> TokenStream { + let mut tts = Vec::new(); + let mut cursor = self; + while let Some((tt, rest)) = cursor.token_tree() { + tts.push(tt); + cursor = rest; + } + tts.into_iter().collect() + } + + /// If the cursor is pointing at a `TokenTree`, returns it along with a + /// cursor pointing at the next `TokenTree`. + /// + /// Returns `None` if the cursor has reached the end of its stream. + /// + /// This method does not treat `None`-delimited groups as transparent, and + /// will return a `Group(None, ..)` if the cursor is looking at one. + pub fn token_tree(self) -> Option<(TokenTree, Cursor<'a>)> { + let (tree, len) = match self.entry() { + Entry::Group(group, end_offset) => (group.clone().into(), *end_offset), + Entry::Literal(literal) => (literal.clone().into(), 1), + Entry::Ident(ident) => (ident.clone().into(), 1), + Entry::Punct(punct) => (punct.clone().into(), 1), + Entry::End(..) => return None, + }; + + let rest = unsafe { Cursor::create(self.ptr.add(len), self.scope) }; + Some((tree, rest)) + } + + /// Returns the `Span` of the current token, or `Span::call_site()` if this + /// cursor points to eof. + pub fn span(mut self) -> Span { + match self.entry() { + Entry::Group(group, _) => group.span(), + Entry::Literal(literal) => literal.span(), + Entry::Ident(ident) => ident.span(), + Entry::Punct(punct) => punct.span(), + Entry::End(_, offset) => { + self.ptr = unsafe { self.ptr.offset(*offset) }; + if let Entry::Group(group, _) = self.entry() { + group.span_close() + } else { + Span::call_site() + } + } + } + } + + /// Returns the `Span` of the token immediately prior to the position of + /// this cursor, or of the current token if there is no previous one. + #[cfg(any(feature = "full", feature = "derive"))] + pub(crate) fn prev_span(mut self) -> Span { + if start_of_buffer(self) < self.ptr { + self.ptr = unsafe { self.ptr.offset(-1) }; + } + self.span() + } + + /// Skip over the next token that is not a None-delimited group, without + /// cloning it. Returns `None` if this cursor points to eof. + /// + /// This method treats `'lifetimes` as a single token. + pub(crate) fn skip(mut self) -> Option> { + self.ignore_none(); + + let len = match self.entry() { + Entry::End(..) => return None, + + // Treat lifetimes as a single tt for the purposes of 'skip'. + Entry::Punct(punct) if punct.as_char() == '\'' && punct.spacing() == Spacing::Joint => { + match unsafe { &*self.ptr.add(1) } { + Entry::Ident(_) => 2, + _ => 1, + } + } + + Entry::Group(_, end_offset) => *end_offset, + _ => 1, + }; + + Some(unsafe { Cursor::create(self.ptr.add(len), self.scope) }) + } + + pub(crate) fn scope_delimiter(self) -> Delimiter { + match unsafe { &*self.scope } { + Entry::End(_, offset) => match unsafe { &*self.scope.offset(*offset) } { + Entry::Group(group, _) => group.delimiter(), + _ => Delimiter::None, + }, + _ => unreachable!(), + } + } +} + +impl<'a> Copy for Cursor<'a> {} + +impl<'a> Clone for Cursor<'a> { + fn clone(&self) -> Self { + *self + } +} + +impl<'a> Eq for Cursor<'a> {} + +impl<'a> PartialEq for Cursor<'a> { + fn eq(&self, other: &Self) -> bool { + ptr::eq(self.ptr, other.ptr) + } +} + +impl<'a> PartialOrd for Cursor<'a> { + fn partial_cmp(&self, other: &Self) -> Option { + if same_buffer(*self, *other) { + Some(cmp_assuming_same_buffer(*self, *other)) + } else { + None + } + } +} + +pub(crate) fn same_scope(a: Cursor, b: Cursor) -> bool { + ptr::eq(a.scope, b.scope) +} + +pub(crate) fn same_buffer(a: Cursor, b: Cursor) -> bool { + ptr::eq(start_of_buffer(a), start_of_buffer(b)) +} + +fn start_of_buffer(cursor: Cursor) -> *const Entry { + unsafe { + match &*cursor.scope { + Entry::End(offset, _) => cursor.scope.offset(*offset), + _ => unreachable!(), + } + } +} + +pub(crate) fn cmp_assuming_same_buffer(a: Cursor, b: Cursor) -> Ordering { + a.ptr.cmp(&b.ptr) +} + +pub(crate) fn open_span_of_group(cursor: Cursor) -> Span { + match cursor.entry() { + Entry::Group(group, _) => group.span_open(), + _ => cursor.span(), + } +} diff --git a/rust/syn/classify.rs b/rust/syn/classify.rs new file mode 100644 index 000000000000..8eab19dbc37c --- /dev/null +++ b/rust/syn/classify.rs @@ -0,0 +1,311 @@ +#[cfg(feature = "full")] +use crate::expr::Expr; +#[cfg(any(feature = "printing", feature = "full"))] +use crate::generics::TypeParamBound; +#[cfg(any(feature = "printing", feature = "full"))] +use crate::path::{Path, PathArguments}; +#[cfg(any(feature = "printing", feature = "full"))] +use crate::punctuated::Punctuated; +#[cfg(any(feature = "printing", feature = "full"))] +use crate::ty::{ReturnType, Type}; +#[cfg(feature = "full")] +use proc_macro2::{Delimiter, TokenStream, TokenTree}; +#[cfg(any(feature = "printing", feature = "full"))] +use std::ops::ControlFlow; + +#[cfg(feature = "full")] +pub(crate) fn requires_semi_to_be_stmt(expr: &Expr) -> bool { + match expr { + Expr::Macro(expr) => !expr.mac.delimiter.is_brace(), + _ => requires_comma_to_be_match_arm(expr), + } +} + +#[cfg(feature = "full")] +pub(crate) fn requires_comma_to_be_match_arm(expr: &Expr) -> bool { + match expr { + Expr::If(_) + | Expr::Match(_) + | Expr::Block(_) | Expr::Unsafe(_) // both under ExprKind::Block in rustc + | Expr::While(_) + | Expr::Loop(_) + | Expr::ForLoop(_) + | Expr::TryBlock(_) + | Expr::Const(_) => false, + + Expr::Array(_) + | Expr::Assign(_) + | Expr::Async(_) + | Expr::Await(_) + | Expr::Binary(_) + | Expr::Break(_) + | Expr::Call(_) + | Expr::Cast(_) + | Expr::Closure(_) + | Expr::Continue(_) + | Expr::Field(_) + | Expr::Group(_) + | Expr::Index(_) + | Expr::Infer(_) + | Expr::Let(_) + | Expr::Lit(_) + | Expr::Macro(_) + | Expr::MethodCall(_) + | Expr::Paren(_) + | Expr::Path(_) + | Expr::Range(_) + | Expr::RawAddr(_) + | Expr::Reference(_) + | Expr::Repeat(_) + | Expr::Return(_) + | Expr::Struct(_) + | Expr::Try(_) + | Expr::Tuple(_) + | Expr::Unary(_) + | Expr::Yield(_) + | Expr::Verbatim(_) => true, + } +} + +#[cfg(feature = "printing")] +pub(crate) fn trailing_unparameterized_path(mut ty: &Type) -> bool { + loop { + match ty { + Type::BareFn(t) => match &t.output { + ReturnType::Default => return false, + ReturnType::Type(_, ret) => ty = ret, + }, + Type::ImplTrait(t) => match last_type_in_bounds(&t.bounds) { + ControlFlow::Break(trailing_path) => return trailing_path, + ControlFlow::Continue(t) => ty = t, + }, + Type::Path(t) => match last_type_in_path(&t.path) { + ControlFlow::Break(trailing_path) => return trailing_path, + ControlFlow::Continue(t) => ty = t, + }, + Type::Ptr(t) => ty = &t.elem, + Type::Reference(t) => ty = &t.elem, + Type::TraitObject(t) => match last_type_in_bounds(&t.bounds) { + ControlFlow::Break(trailing_path) => return trailing_path, + ControlFlow::Continue(t) => ty = t, + }, + + Type::Array(_) + | Type::Group(_) + | Type::Infer(_) + | Type::Macro(_) + | Type::Never(_) + | Type::Paren(_) + | Type::Slice(_) + | Type::Tuple(_) + | Type::Verbatim(_) => return false, + } + } + + fn last_type_in_path(path: &Path) -> ControlFlow { + match &path.segments.last().unwrap().arguments { + PathArguments::None => ControlFlow::Break(true), + PathArguments::AngleBracketed(_) => ControlFlow::Break(false), + PathArguments::Parenthesized(arg) => match &arg.output { + ReturnType::Default => ControlFlow::Break(false), + ReturnType::Type(_, ret) => ControlFlow::Continue(ret), + }, + } + } + + fn last_type_in_bounds( + bounds: &Punctuated, + ) -> ControlFlow { + match bounds.last().unwrap() { + TypeParamBound::Trait(t) => last_type_in_path(&t.path), + TypeParamBound::Lifetime(_) + | TypeParamBound::PreciseCapture(_) + | TypeParamBound::Verbatim(_) => ControlFlow::Break(false), + } + } +} + +/// Whether the expression's first token is the label of a loop/block. +#[cfg(all(feature = "printing", feature = "full"))] +pub(crate) fn expr_leading_label(mut expr: &Expr) -> bool { + loop { + match expr { + Expr::Block(e) => return e.label.is_some(), + Expr::ForLoop(e) => return e.label.is_some(), + Expr::Loop(e) => return e.label.is_some(), + Expr::While(e) => return e.label.is_some(), + + Expr::Assign(e) => expr = &e.left, + Expr::Await(e) => expr = &e.base, + Expr::Binary(e) => expr = &e.left, + Expr::Call(e) => expr = &e.func, + Expr::Cast(e) => expr = &e.expr, + Expr::Field(e) => expr = &e.base, + Expr::Index(e) => expr = &e.expr, + Expr::MethodCall(e) => expr = &e.receiver, + Expr::Range(e) => match &e.start { + Some(start) => expr = start, + None => return false, + }, + Expr::Try(e) => expr = &e.expr, + + Expr::Array(_) + | Expr::Async(_) + | Expr::Break(_) + | Expr::Closure(_) + | Expr::Const(_) + | Expr::Continue(_) + | Expr::Group(_) + | Expr::If(_) + | Expr::Infer(_) + | Expr::Let(_) + | Expr::Lit(_) + | Expr::Macro(_) + | Expr::Match(_) + | Expr::Paren(_) + | Expr::Path(_) + | Expr::RawAddr(_) + | Expr::Reference(_) + | Expr::Repeat(_) + | Expr::Return(_) + | Expr::Struct(_) + | Expr::TryBlock(_) + | Expr::Tuple(_) + | Expr::Unary(_) + | Expr::Unsafe(_) + | Expr::Verbatim(_) + | Expr::Yield(_) => return false, + } + } +} + +/// Whether the expression's last token is `}`. +#[cfg(feature = "full")] +pub(crate) fn expr_trailing_brace(mut expr: &Expr) -> bool { + loop { + match expr { + Expr::Async(_) + | Expr::Block(_) + | Expr::Const(_) + | Expr::ForLoop(_) + | Expr::If(_) + | Expr::Loop(_) + | Expr::Match(_) + | Expr::Struct(_) + | Expr::TryBlock(_) + | Expr::Unsafe(_) + | Expr::While(_) => return true, + + Expr::Assign(e) => expr = &e.right, + Expr::Binary(e) => expr = &e.right, + Expr::Break(e) => match &e.expr { + Some(e) => expr = e, + None => return false, + }, + Expr::Cast(e) => return type_trailing_brace(&e.ty), + Expr::Closure(e) => expr = &e.body, + Expr::Let(e) => expr = &e.expr, + Expr::Macro(e) => return e.mac.delimiter.is_brace(), + Expr::Range(e) => match &e.end { + Some(end) => expr = end, + None => return false, + }, + Expr::RawAddr(e) => expr = &e.expr, + Expr::Reference(e) => expr = &e.expr, + Expr::Return(e) => match &e.expr { + Some(e) => expr = e, + None => return false, + }, + Expr::Unary(e) => expr = &e.expr, + Expr::Verbatim(e) => return tokens_trailing_brace(e), + Expr::Yield(e) => match &e.expr { + Some(e) => expr = e, + None => return false, + }, + + Expr::Array(_) + | Expr::Await(_) + | Expr::Call(_) + | Expr::Continue(_) + | Expr::Field(_) + | Expr::Group(_) + | Expr::Index(_) + | Expr::Infer(_) + | Expr::Lit(_) + | Expr::MethodCall(_) + | Expr::Paren(_) + | Expr::Path(_) + | Expr::Repeat(_) + | Expr::Try(_) + | Expr::Tuple(_) => return false, + } + } + + fn type_trailing_brace(mut ty: &Type) -> bool { + loop { + match ty { + Type::BareFn(t) => match &t.output { + ReturnType::Default => return false, + ReturnType::Type(_, ret) => ty = ret, + }, + Type::ImplTrait(t) => match last_type_in_bounds(&t.bounds) { + ControlFlow::Break(trailing_brace) => return trailing_brace, + ControlFlow::Continue(t) => ty = t, + }, + Type::Macro(t) => return t.mac.delimiter.is_brace(), + Type::Path(t) => match last_type_in_path(&t.path) { + Some(t) => ty = t, + None => return false, + }, + Type::Ptr(t) => ty = &t.elem, + Type::Reference(t) => ty = &t.elem, + Type::TraitObject(t) => match last_type_in_bounds(&t.bounds) { + ControlFlow::Break(trailing_brace) => return trailing_brace, + ControlFlow::Continue(t) => ty = t, + }, + Type::Verbatim(t) => return tokens_trailing_brace(t), + + Type::Array(_) + | Type::Group(_) + | Type::Infer(_) + | Type::Never(_) + | Type::Paren(_) + | Type::Slice(_) + | Type::Tuple(_) => return false, + } + } + } + + fn last_type_in_path(path: &Path) -> Option<&Type> { + match &path.segments.last().unwrap().arguments { + PathArguments::None | PathArguments::AngleBracketed(_) => None, + PathArguments::Parenthesized(arg) => match &arg.output { + ReturnType::Default => None, + ReturnType::Type(_, ret) => Some(ret), + }, + } + } + + fn last_type_in_bounds( + bounds: &Punctuated, + ) -> ControlFlow { + match bounds.last().unwrap() { + TypeParamBound::Trait(t) => match last_type_in_path(&t.path) { + Some(t) => ControlFlow::Continue(t), + None => ControlFlow::Break(false), + }, + TypeParamBound::Lifetime(_) | TypeParamBound::PreciseCapture(_) => { + ControlFlow::Break(false) + } + TypeParamBound::Verbatim(t) => ControlFlow::Break(tokens_trailing_brace(t)), + } + } + + fn tokens_trailing_brace(tokens: &TokenStream) -> bool { + if let Some(TokenTree::Group(last)) = tokens.clone().into_iter().last() { + last.delimiter() == Delimiter::Brace + } else { + false + } + } +} diff --git a/rust/syn/custom_keyword.rs b/rust/syn/custom_keyword.rs new file mode 100644 index 000000000000..cc4f632c981a --- /dev/null +++ b/rust/syn/custom_keyword.rs @@ -0,0 +1,260 @@ +/// Define a type that supports parsing and printing a given identifier as if it +/// were a keyword. +/// +/// # Usage +/// +/// As a convention, it is recommended that this macro be invoked within a +/// module called `kw` or `keyword` and that the resulting parser be invoked +/// with a `kw::` or `keyword::` prefix. +/// +/// ``` +/// mod kw { +/// syn::custom_keyword!(whatever); +/// } +/// ``` +/// +/// The generated syntax tree node supports the following operations just like +/// any built-in keyword token. +/// +/// - [Peeking] — `input.peek(kw::whatever)` +/// +/// - [Parsing] — `input.parse::()?` +/// +/// - [Printing] — `quote!( ... #whatever_token ... )` +/// +/// - Construction from a [`Span`] — `let whatever_token = kw::whatever(sp)` +/// +/// - Field access to its span — `let sp = whatever_token.span` +/// +/// [Peeking]: crate::parse::ParseBuffer::peek +/// [Parsing]: crate::parse::ParseBuffer::parse +/// [Printing]: quote::ToTokens +/// [`Span`]: proc_macro2::Span +/// +/// # Example +/// +/// This example parses input that looks like `bool = true` or `str = "value"`. +/// The key must be either the identifier `bool` or the identifier `str`. If +/// `bool`, the value may be either `true` or `false`. If `str`, the value may +/// be any string literal. +/// +/// The symbols `bool` and `str` are not reserved keywords in Rust so these are +/// not considered keywords in the `syn::token` module. Like any other +/// identifier that is not a keyword, these can be declared as custom keywords +/// by crates that need to use them as such. +/// +/// ``` +/// use syn::{LitBool, LitStr, Result, Token}; +/// use syn::parse::{Parse, ParseStream}; +/// +/// mod kw { +/// syn::custom_keyword!(bool); +/// syn::custom_keyword!(str); +/// } +/// +/// enum Argument { +/// Bool { +/// bool_token: kw::bool, +/// eq_token: Token![=], +/// value: LitBool, +/// }, +/// Str { +/// str_token: kw::str, +/// eq_token: Token![=], +/// value: LitStr, +/// }, +/// } +/// +/// impl Parse for Argument { +/// fn parse(input: ParseStream) -> Result { +/// let lookahead = input.lookahead1(); +/// if lookahead.peek(kw::bool) { +/// Ok(Argument::Bool { +/// bool_token: input.parse::()?, +/// eq_token: input.parse()?, +/// value: input.parse()?, +/// }) +/// } else if lookahead.peek(kw::str) { +/// Ok(Argument::Str { +/// str_token: input.parse::()?, +/// eq_token: input.parse()?, +/// value: input.parse()?, +/// }) +/// } else { +/// Err(lookahead.error()) +/// } +/// } +/// } +/// ``` +#[macro_export] +macro_rules! custom_keyword { + ($ident:ident) => { + #[allow(non_camel_case_types)] + pub struct $ident { + #[allow(dead_code)] + pub span: $crate::__private::Span, + } + + #[doc(hidden)] + #[allow(dead_code, non_snake_case)] + pub fn $ident<__S: $crate::__private::IntoSpans<$crate::__private::Span>>( + span: __S, + ) -> $ident { + $ident { + span: $crate::__private::IntoSpans::into_spans(span), + } + } + + const _: () = { + impl $crate::__private::Default for $ident { + fn default() -> Self { + $ident { + span: $crate::__private::Span::call_site(), + } + } + } + + $crate::impl_parse_for_custom_keyword!($ident); + $crate::impl_to_tokens_for_custom_keyword!($ident); + $crate::impl_clone_for_custom_keyword!($ident); + $crate::impl_extra_traits_for_custom_keyword!($ident); + }; + }; +} + +// Not public API. +#[cfg(feature = "parsing")] +#[doc(hidden)] +#[macro_export] +macro_rules! impl_parse_for_custom_keyword { + ($ident:ident) => { + // For peek. + impl $crate::__private::CustomToken for $ident { + fn peek(cursor: $crate::buffer::Cursor) -> $crate::__private::bool { + if let $crate::__private::Some((ident, _rest)) = cursor.ident() { + ident == $crate::__private::stringify!($ident) + } else { + false + } + } + + fn display() -> &'static $crate::__private::str { + $crate::__private::concat!("`", $crate::__private::stringify!($ident), "`") + } + } + + impl $crate::parse::Parse for $ident { + fn parse(input: $crate::parse::ParseStream) -> $crate::parse::Result<$ident> { + input.step(|cursor| { + if let $crate::__private::Some((ident, rest)) = cursor.ident() { + if ident == $crate::__private::stringify!($ident) { + return $crate::__private::Ok(($ident { span: ident.span() }, rest)); + } + } + $crate::__private::Err(cursor.error($crate::__private::concat!( + "expected `", + $crate::__private::stringify!($ident), + "`", + ))) + }) + } + } + }; +} + +// Not public API. +#[cfg(not(feature = "parsing"))] +#[doc(hidden)] +#[macro_export] +macro_rules! impl_parse_for_custom_keyword { + ($ident:ident) => {}; +} + +// Not public API. +#[cfg(feature = "printing")] +#[doc(hidden)] +#[macro_export] +macro_rules! impl_to_tokens_for_custom_keyword { + ($ident:ident) => { + impl $crate::__private::ToTokens for $ident { + fn to_tokens(&self, tokens: &mut $crate::__private::TokenStream2) { + let ident = $crate::Ident::new($crate::__private::stringify!($ident), self.span); + $crate::__private::TokenStreamExt::append(tokens, ident); + } + } + }; +} + +// Not public API. +#[cfg(not(feature = "printing"))] +#[doc(hidden)] +#[macro_export] +macro_rules! impl_to_tokens_for_custom_keyword { + ($ident:ident) => {}; +} + +// Not public API. +#[cfg(feature = "clone-impls")] +#[doc(hidden)] +#[macro_export] +macro_rules! impl_clone_for_custom_keyword { + ($ident:ident) => { + impl $crate::__private::Copy for $ident {} + + #[allow(clippy::expl_impl_clone_on_copy)] + impl $crate::__private::Clone for $ident { + fn clone(&self) -> Self { + *self + } + } + }; +} + +// Not public API. +#[cfg(not(feature = "clone-impls"))] +#[doc(hidden)] +#[macro_export] +macro_rules! impl_clone_for_custom_keyword { + ($ident:ident) => {}; +} + +// Not public API. +#[cfg(feature = "extra-traits")] +#[doc(hidden)] +#[macro_export] +macro_rules! impl_extra_traits_for_custom_keyword { + ($ident:ident) => { + impl $crate::__private::Debug for $ident { + fn fmt(&self, f: &mut $crate::__private::Formatter) -> $crate::__private::FmtResult { + $crate::__private::Formatter::write_str( + f, + $crate::__private::concat!( + "Keyword [", + $crate::__private::stringify!($ident), + "]", + ), + ) + } + } + + impl $crate::__private::Eq for $ident {} + + impl $crate::__private::PartialEq for $ident { + fn eq(&self, _other: &Self) -> $crate::__private::bool { + true + } + } + + impl $crate::__private::Hash for $ident { + fn hash<__H: $crate::__private::Hasher>(&self, _state: &mut __H) {} + } + }; +} + +// Not public API. +#[cfg(not(feature = "extra-traits"))] +#[doc(hidden)] +#[macro_export] +macro_rules! impl_extra_traits_for_custom_keyword { + ($ident:ident) => {}; +} diff --git a/rust/syn/custom_punctuation.rs b/rust/syn/custom_punctuation.rs new file mode 100644 index 000000000000..eef5f5458459 --- /dev/null +++ b/rust/syn/custom_punctuation.rs @@ -0,0 +1,304 @@ +/// Define a type that supports parsing and printing a multi-character symbol +/// as if it were a punctuation token. +/// +/// # Usage +/// +/// ``` +/// syn::custom_punctuation!(LeftRightArrow, <=>); +/// ``` +/// +/// The generated syntax tree node supports the following operations just like +/// any built-in punctuation token. +/// +/// - [Peeking] — `input.peek(LeftRightArrow)` +/// +/// - [Parsing] — `input.parse::()?` +/// +/// - [Printing] — `quote!( ... #lrarrow ... )` +/// +/// - Construction from a [`Span`] — `let lrarrow = LeftRightArrow(sp)` +/// +/// - Construction from multiple [`Span`] — `let lrarrow = LeftRightArrow([sp, sp, sp])` +/// +/// - Field access to its spans — `let spans = lrarrow.spans` +/// +/// [Peeking]: crate::parse::ParseBuffer::peek +/// [Parsing]: crate::parse::ParseBuffer::parse +/// [Printing]: quote::ToTokens +/// [`Span`]: proc_macro2::Span +/// +/// # Example +/// +/// ``` +/// use proc_macro2::{TokenStream, TokenTree}; +/// use syn::parse::{Parse, ParseStream, Peek, Result}; +/// use syn::punctuated::Punctuated; +/// use syn::Expr; +/// +/// syn::custom_punctuation!(PathSeparator, ); +/// +/// // expr expr expr ... +/// struct PathSegments { +/// segments: Punctuated, +/// } +/// +/// impl Parse for PathSegments { +/// fn parse(input: ParseStream) -> Result { +/// let mut segments = Punctuated::new(); +/// +/// let first = parse_until(input, PathSeparator)?; +/// segments.push_value(syn::parse2(first)?); +/// +/// while input.peek(PathSeparator) { +/// segments.push_punct(input.parse()?); +/// +/// let next = parse_until(input, PathSeparator)?; +/// segments.push_value(syn::parse2(next)?); +/// } +/// +/// Ok(PathSegments { segments }) +/// } +/// } +/// +/// fn parse_until(input: ParseStream, end: E) -> Result { +/// let mut tokens = TokenStream::new(); +/// while !input.is_empty() && !input.peek(end) { +/// let next: TokenTree = input.parse()?; +/// tokens.extend(Some(next)); +/// } +/// Ok(tokens) +/// } +/// +/// fn main() { +/// let input = r#" a::b c::d::e "#; +/// let _: PathSegments = syn::parse_str(input).unwrap(); +/// } +/// ``` +#[macro_export] +macro_rules! custom_punctuation { + ($ident:ident, $($tt:tt)+) => { + pub struct $ident { + #[allow(dead_code)] + pub spans: $crate::custom_punctuation_repr!($($tt)+), + } + + #[doc(hidden)] + #[allow(dead_code, non_snake_case)] + pub fn $ident<__S: $crate::__private::IntoSpans<$crate::custom_punctuation_repr!($($tt)+)>>( + spans: __S, + ) -> $ident { + let _validate_len = 0 $(+ $crate::custom_punctuation_len!(strict, $tt))*; + $ident { + spans: $crate::__private::IntoSpans::into_spans(spans) + } + } + + const _: () = { + impl $crate::__private::Default for $ident { + fn default() -> Self { + $ident($crate::__private::Span::call_site()) + } + } + + $crate::impl_parse_for_custom_punctuation!($ident, $($tt)+); + $crate::impl_to_tokens_for_custom_punctuation!($ident, $($tt)+); + $crate::impl_clone_for_custom_punctuation!($ident, $($tt)+); + $crate::impl_extra_traits_for_custom_punctuation!($ident, $($tt)+); + }; + }; +} + +// Not public API. +#[cfg(feature = "parsing")] +#[doc(hidden)] +#[macro_export] +macro_rules! impl_parse_for_custom_punctuation { + ($ident:ident, $($tt:tt)+) => { + impl $crate::__private::CustomToken for $ident { + fn peek(cursor: $crate::buffer::Cursor) -> $crate::__private::bool { + $crate::__private::peek_punct(cursor, $crate::stringify_punct!($($tt)+)) + } + + fn display() -> &'static $crate::__private::str { + $crate::__private::concat!("`", $crate::stringify_punct!($($tt)+), "`") + } + } + + impl $crate::parse::Parse for $ident { + fn parse(input: $crate::parse::ParseStream) -> $crate::parse::Result<$ident> { + let spans: $crate::custom_punctuation_repr!($($tt)+) = + $crate::__private::parse_punct(input, $crate::stringify_punct!($($tt)+))?; + Ok($ident(spans)) + } + } + }; +} + +// Not public API. +#[cfg(not(feature = "parsing"))] +#[doc(hidden)] +#[macro_export] +macro_rules! impl_parse_for_custom_punctuation { + ($ident:ident, $($tt:tt)+) => {}; +} + +// Not public API. +#[cfg(feature = "printing")] +#[doc(hidden)] +#[macro_export] +macro_rules! impl_to_tokens_for_custom_punctuation { + ($ident:ident, $($tt:tt)+) => { + impl $crate::__private::ToTokens for $ident { + fn to_tokens(&self, tokens: &mut $crate::__private::TokenStream2) { + $crate::__private::print_punct($crate::stringify_punct!($($tt)+), &self.spans, tokens) + } + } + }; +} + +// Not public API. +#[cfg(not(feature = "printing"))] +#[doc(hidden)] +#[macro_export] +macro_rules! impl_to_tokens_for_custom_punctuation { + ($ident:ident, $($tt:tt)+) => {}; +} + +// Not public API. +#[cfg(feature = "clone-impls")] +#[doc(hidden)] +#[macro_export] +macro_rules! impl_clone_for_custom_punctuation { + ($ident:ident, $($tt:tt)+) => { + impl $crate::__private::Copy for $ident {} + + #[allow(clippy::expl_impl_clone_on_copy)] + impl $crate::__private::Clone for $ident { + fn clone(&self) -> Self { + *self + } + } + }; +} + +// Not public API. +#[cfg(not(feature = "clone-impls"))] +#[doc(hidden)] +#[macro_export] +macro_rules! impl_clone_for_custom_punctuation { + ($ident:ident, $($tt:tt)+) => {}; +} + +// Not public API. +#[cfg(feature = "extra-traits")] +#[doc(hidden)] +#[macro_export] +macro_rules! impl_extra_traits_for_custom_punctuation { + ($ident:ident, $($tt:tt)+) => { + impl $crate::__private::Debug for $ident { + fn fmt(&self, f: &mut $crate::__private::Formatter) -> $crate::__private::FmtResult { + $crate::__private::Formatter::write_str(f, $crate::__private::stringify!($ident)) + } + } + + impl $crate::__private::Eq for $ident {} + + impl $crate::__private::PartialEq for $ident { + fn eq(&self, _other: &Self) -> $crate::__private::bool { + true + } + } + + impl $crate::__private::Hash for $ident { + fn hash<__H: $crate::__private::Hasher>(&self, _state: &mut __H) {} + } + }; +} + +// Not public API. +#[cfg(not(feature = "extra-traits"))] +#[doc(hidden)] +#[macro_export] +macro_rules! impl_extra_traits_for_custom_punctuation { + ($ident:ident, $($tt:tt)+) => {}; +} + +// Not public API. +#[doc(hidden)] +#[macro_export] +macro_rules! custom_punctuation_repr { + ($($tt:tt)+) => { + [$crate::__private::Span; 0 $(+ $crate::custom_punctuation_len!(lenient, $tt))+] + }; +} + +// Not public API. +#[doc(hidden)] +#[macro_export] +#[rustfmt::skip] +macro_rules! custom_punctuation_len { + ($mode:ident, &) => { 1 }; + ($mode:ident, &&) => { 2 }; + ($mode:ident, &=) => { 2 }; + ($mode:ident, @) => { 1 }; + ($mode:ident, ^) => { 1 }; + ($mode:ident, ^=) => { 2 }; + ($mode:ident, :) => { 1 }; + ($mode:ident, ,) => { 1 }; + ($mode:ident, $) => { 1 }; + ($mode:ident, .) => { 1 }; + ($mode:ident, ..) => { 2 }; + ($mode:ident, ...) => { 3 }; + ($mode:ident, ..=) => { 3 }; + ($mode:ident, =) => { 1 }; + ($mode:ident, ==) => { 2 }; + ($mode:ident, =>) => { 2 }; + ($mode:ident, >=) => { 2 }; + ($mode:ident, >) => { 1 }; + ($mode:ident, <-) => { 2 }; + ($mode:ident, <=) => { 2 }; + ($mode:ident, <) => { 1 }; + ($mode:ident, -) => { 1 }; + ($mode:ident, -=) => { 2 }; + ($mode:ident, !=) => { 2 }; + ($mode:ident, !) => { 1 }; + ($mode:ident, |) => { 1 }; + ($mode:ident, |=) => { 2 }; + ($mode:ident, ||) => { 2 }; + ($mode:ident, ::) => { 2 }; + ($mode:ident, %) => { 1 }; + ($mode:ident, %=) => { 2 }; + ($mode:ident, +) => { 1 }; + ($mode:ident, +=) => { 2 }; + ($mode:ident, #) => { 1 }; + ($mode:ident, ?) => { 1 }; + ($mode:ident, ->) => { 2 }; + ($mode:ident, ;) => { 1 }; + ($mode:ident, <<) => { 2 }; + ($mode:ident, <<=) => { 3 }; + ($mode:ident, >>) => { 2 }; + ($mode:ident, >>=) => { 3 }; + ($mode:ident, /) => { 1 }; + ($mode:ident, /=) => { 2 }; + ($mode:ident, *) => { 1 }; + ($mode:ident, *=) => { 2 }; + ($mode:ident, ~) => { 1 }; + (lenient, $tt:tt) => { 0 }; + (strict, $tt:tt) => {{ $crate::custom_punctuation_unexpected!($tt); 0 }}; +} + +// Not public API. +#[doc(hidden)] +#[macro_export] +macro_rules! custom_punctuation_unexpected { + () => {}; +} + +// Not public API. +#[doc(hidden)] +#[macro_export] +macro_rules! stringify_punct { + ($($tt:tt)+) => { + $crate::__private::concat!($($crate::__private::stringify!($tt)),+) + }; +} diff --git a/rust/syn/data.rs b/rust/syn/data.rs new file mode 100644 index 000000000000..96db2a0b7c6f --- /dev/null +++ b/rust/syn/data.rs @@ -0,0 +1,424 @@ +use crate::attr::Attribute; +use crate::expr::{Expr, Index, Member}; +use crate::ident::Ident; +use crate::punctuated::{self, Punctuated}; +use crate::restriction::{FieldMutability, Visibility}; +use crate::token; +use crate::ty::Type; + +ast_struct! { + /// An enum variant. + #[cfg_attr(docsrs, doc(cfg(any(feature = "full", feature = "derive"))))] + pub struct Variant { + pub attrs: Vec, + + /// Name of the variant. + pub ident: Ident, + + /// Content stored in the variant. + pub fields: Fields, + + /// Explicit discriminant: `Variant = 1` + pub discriminant: Option<(Token![=], Expr)>, + } +} + +ast_enum_of_structs! { + /// Data stored within an enum variant or struct. + /// + /// # Syntax tree enum + /// + /// This type is a [syntax tree enum]. + /// + /// [syntax tree enum]: crate::expr::Expr#syntax-tree-enums + #[cfg_attr(docsrs, doc(cfg(any(feature = "full", feature = "derive"))))] + pub enum Fields { + /// Named fields of a struct or struct variant such as `Point { x: f64, + /// y: f64 }`. + Named(FieldsNamed), + + /// Unnamed fields of a tuple struct or tuple variant such as `Some(T)`. + Unnamed(FieldsUnnamed), + + /// Unit struct or unit variant such as `None`. + Unit, + } +} + +ast_struct! { + /// Named fields of a struct or struct variant such as `Point { x: f64, + /// y: f64 }`. + #[cfg_attr(docsrs, doc(cfg(any(feature = "full", feature = "derive"))))] + pub struct FieldsNamed { + pub brace_token: token::Brace, + pub named: Punctuated, + } +} + +ast_struct! { + /// Unnamed fields of a tuple struct or tuple variant such as `Some(T)`. + #[cfg_attr(docsrs, doc(cfg(any(feature = "full", feature = "derive"))))] + pub struct FieldsUnnamed { + pub paren_token: token::Paren, + pub unnamed: Punctuated, + } +} + +impl Fields { + /// Get an iterator over the borrowed [`Field`] items in this object. This + /// iterator can be used to iterate over a named or unnamed struct or + /// variant's fields uniformly. + pub fn iter(&self) -> punctuated::Iter { + match self { + Fields::Unit => crate::punctuated::empty_punctuated_iter(), + Fields::Named(f) => f.named.iter(), + Fields::Unnamed(f) => f.unnamed.iter(), + } + } + + /// Get an iterator over the mutably borrowed [`Field`] items in this + /// object. This iterator can be used to iterate over a named or unnamed + /// struct or variant's fields uniformly. + pub fn iter_mut(&mut self) -> punctuated::IterMut { + match self { + Fields::Unit => crate::punctuated::empty_punctuated_iter_mut(), + Fields::Named(f) => f.named.iter_mut(), + Fields::Unnamed(f) => f.unnamed.iter_mut(), + } + } + + /// Returns the number of fields. + pub fn len(&self) -> usize { + match self { + Fields::Unit => 0, + Fields::Named(f) => f.named.len(), + Fields::Unnamed(f) => f.unnamed.len(), + } + } + + /// Returns `true` if there are zero fields. + pub fn is_empty(&self) -> bool { + match self { + Fields::Unit => true, + Fields::Named(f) => f.named.is_empty(), + Fields::Unnamed(f) => f.unnamed.is_empty(), + } + } + + return_impl_trait! { + /// Get an iterator over the fields of a struct or variant as [`Member`]s. + /// This iterator can be used to iterate over a named or unnamed struct or + /// variant's fields uniformly. + /// + /// # Example + /// + /// The following is a simplistic [`Clone`] derive for structs. (A more + /// complete implementation would additionally want to infer trait bounds on + /// the generic type parameters.) + /// + /// ``` + /// # use quote::quote; + /// # + /// fn derive_clone(input: &syn::ItemStruct) -> proc_macro2::TokenStream { + /// let ident = &input.ident; + /// let members = input.fields.members(); + /// let (impl_generics, ty_generics, where_clause) = input.generics.split_for_impl(); + /// quote! { + /// impl #impl_generics Clone for #ident #ty_generics #where_clause { + /// fn clone(&self) -> Self { + /// Self { + /// #(#members: self.#members.clone()),* + /// } + /// } + /// } + /// } + /// } + /// ``` + /// + /// For structs with named fields, it produces an expression like `Self { a: + /// self.a.clone() }`. For structs with unnamed fields, `Self { 0: + /// self.0.clone() }`. And for unit structs, `Self {}`. + pub fn members(&self) -> impl Iterator + Clone + '_ [Members] { + Members { + fields: self.iter(), + index: 0, + } + } + } +} + +impl IntoIterator for Fields { + type Item = Field; + type IntoIter = punctuated::IntoIter; + + fn into_iter(self) -> Self::IntoIter { + match self { + Fields::Unit => Punctuated::::new().into_iter(), + Fields::Named(f) => f.named.into_iter(), + Fields::Unnamed(f) => f.unnamed.into_iter(), + } + } +} + +impl<'a> IntoIterator for &'a Fields { + type Item = &'a Field; + type IntoIter = punctuated::Iter<'a, Field>; + + fn into_iter(self) -> Self::IntoIter { + self.iter() + } +} + +impl<'a> IntoIterator for &'a mut Fields { + type Item = &'a mut Field; + type IntoIter = punctuated::IterMut<'a, Field>; + + fn into_iter(self) -> Self::IntoIter { + self.iter_mut() + } +} + +ast_struct! { + /// A field of a struct or enum variant. + #[cfg_attr(docsrs, doc(cfg(any(feature = "full", feature = "derive"))))] + pub struct Field { + pub attrs: Vec, + + pub vis: Visibility, + + pub mutability: FieldMutability, + + /// Name of the field, if any. + /// + /// Fields of tuple structs have no names. + pub ident: Option, + + pub colon_token: Option, + + pub ty: Type, + } +} + +pub struct Members<'a> { + fields: punctuated::Iter<'a, Field>, + index: u32, +} + +impl<'a> Iterator for Members<'a> { + type Item = Member; + + fn next(&mut self) -> Option { + let field = self.fields.next()?; + let member = match &field.ident { + Some(ident) => Member::Named(ident.clone()), + None => { + #[cfg(all(feature = "parsing", feature = "printing"))] + let span = crate::spanned::Spanned::span(&field.ty); + #[cfg(not(all(feature = "parsing", feature = "printing")))] + let span = proc_macro2::Span::call_site(); + Member::Unnamed(Index { + index: self.index, + span, + }) + } + }; + self.index += 1; + Some(member) + } +} + +impl<'a> Clone for Members<'a> { + fn clone(&self) -> Self { + Members { + fields: self.fields.clone(), + index: self.index, + } + } +} + +#[cfg(feature = "parsing")] +pub(crate) mod parsing { + use crate::attr::Attribute; + use crate::data::{Field, Fields, FieldsNamed, FieldsUnnamed, Variant}; + use crate::error::Result; + use crate::expr::Expr; + use crate::ext::IdentExt as _; + use crate::ident::Ident; + #[cfg(not(feature = "full"))] + use crate::parse::discouraged::Speculative as _; + use crate::parse::{Parse, ParseStream}; + use crate::restriction::{FieldMutability, Visibility}; + #[cfg(not(feature = "full"))] + use crate::scan_expr::scan_expr; + use crate::token; + use crate::ty::Type; + use crate::verbatim; + + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + impl Parse for Variant { + fn parse(input: ParseStream) -> Result { + let attrs = input.call(Attribute::parse_outer)?; + let _visibility: Visibility = input.parse()?; + let ident: Ident = input.parse()?; + let fields = if input.peek(token::Brace) { + Fields::Named(input.parse()?) + } else if input.peek(token::Paren) { + Fields::Unnamed(input.parse()?) + } else { + Fields::Unit + }; + let discriminant = if input.peek(Token![=]) { + let eq_token: Token![=] = input.parse()?; + #[cfg(feature = "full")] + let discriminant: Expr = input.parse()?; + #[cfg(not(feature = "full"))] + let discriminant = { + let begin = input.fork(); + let ahead = input.fork(); + let mut discriminant: Result = ahead.parse(); + if discriminant.is_ok() { + input.advance_to(&ahead); + } else if scan_expr(input).is_ok() { + discriminant = Ok(Expr::Verbatim(verbatim::between(&begin, input))); + } + discriminant? + }; + Some((eq_token, discriminant)) + } else { + None + }; + Ok(Variant { + attrs, + ident, + fields, + discriminant, + }) + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + impl Parse for FieldsNamed { + fn parse(input: ParseStream) -> Result { + let content; + Ok(FieldsNamed { + brace_token: braced!(content in input), + named: content.parse_terminated(Field::parse_named, Token![,])?, + }) + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + impl Parse for FieldsUnnamed { + fn parse(input: ParseStream) -> Result { + let content; + Ok(FieldsUnnamed { + paren_token: parenthesized!(content in input), + unnamed: content.parse_terminated(Field::parse_unnamed, Token![,])?, + }) + } + } + + impl Field { + /// Parses a named (braced struct) field. + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + pub fn parse_named(input: ParseStream) -> Result { + let attrs = input.call(Attribute::parse_outer)?; + let vis: Visibility = input.parse()?; + + let unnamed_field = cfg!(feature = "full") && input.peek(Token![_]); + let ident = if unnamed_field { + input.call(Ident::parse_any) + } else { + input.parse() + }?; + + let colon_token: Token![:] = input.parse()?; + + let ty: Type = if unnamed_field + && (input.peek(Token![struct]) + || input.peek(Token![union]) && input.peek2(token::Brace)) + { + let begin = input.fork(); + input.call(Ident::parse_any)?; + input.parse::()?; + Type::Verbatim(verbatim::between(&begin, input)) + } else { + input.parse()? + }; + + Ok(Field { + attrs, + vis, + mutability: FieldMutability::None, + ident: Some(ident), + colon_token: Some(colon_token), + ty, + }) + } + + /// Parses an unnamed (tuple struct) field. + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + pub fn parse_unnamed(input: ParseStream) -> Result { + Ok(Field { + attrs: input.call(Attribute::parse_outer)?, + vis: input.parse()?, + mutability: FieldMutability::None, + ident: None, + colon_token: None, + ty: input.parse()?, + }) + } + } +} + +#[cfg(feature = "printing")] +mod printing { + use crate::data::{Field, FieldsNamed, FieldsUnnamed, Variant}; + use crate::print::TokensOrDefault; + use proc_macro2::TokenStream; + use quote::{ToTokens, TokenStreamExt}; + + #[cfg_attr(docsrs, doc(cfg(feature = "printing")))] + impl ToTokens for Variant { + fn to_tokens(&self, tokens: &mut TokenStream) { + tokens.append_all(&self.attrs); + self.ident.to_tokens(tokens); + self.fields.to_tokens(tokens); + if let Some((eq_token, disc)) = &self.discriminant { + eq_token.to_tokens(tokens); + disc.to_tokens(tokens); + } + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "printing")))] + impl ToTokens for FieldsNamed { + fn to_tokens(&self, tokens: &mut TokenStream) { + self.brace_token.surround(tokens, |tokens| { + self.named.to_tokens(tokens); + }); + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "printing")))] + impl ToTokens for FieldsUnnamed { + fn to_tokens(&self, tokens: &mut TokenStream) { + self.paren_token.surround(tokens, |tokens| { + self.unnamed.to_tokens(tokens); + }); + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "printing")))] + impl ToTokens for Field { + fn to_tokens(&self, tokens: &mut TokenStream) { + tokens.append_all(&self.attrs); + self.vis.to_tokens(tokens); + if let Some(ident) = &self.ident { + ident.to_tokens(tokens); + TokensOrDefault(&self.colon_token).to_tokens(tokens); + } + self.ty.to_tokens(tokens); + } + } +} diff --git a/rust/syn/derive.rs b/rust/syn/derive.rs new file mode 100644 index 000000000000..3443ecfc05cb --- /dev/null +++ b/rust/syn/derive.rs @@ -0,0 +1,259 @@ +use crate::attr::Attribute; +use crate::data::{Fields, FieldsNamed, Variant}; +use crate::generics::Generics; +use crate::ident::Ident; +use crate::punctuated::Punctuated; +use crate::restriction::Visibility; +use crate::token; + +ast_struct! { + /// Data structure sent to a `proc_macro_derive` macro. + #[cfg_attr(docsrs, doc(cfg(feature = "derive")))] + pub struct DeriveInput { + pub attrs: Vec, + pub vis: Visibility, + pub ident: Ident, + pub generics: Generics, + pub data: Data, + } +} + +ast_enum! { + /// The storage of a struct, enum or union data structure. + /// + /// # Syntax tree enum + /// + /// This type is a [syntax tree enum]. + /// + /// [syntax tree enum]: crate::expr::Expr#syntax-tree-enums + #[cfg_attr(docsrs, doc(cfg(feature = "derive")))] + pub enum Data { + Struct(DataStruct), + Enum(DataEnum), + Union(DataUnion), + } +} + +ast_struct! { + /// A struct input to a `proc_macro_derive` macro. + #[cfg_attr(docsrs, doc(cfg(feature = "derive")))] + pub struct DataStruct { + pub struct_token: Token![struct], + pub fields: Fields, + pub semi_token: Option, + } +} + +ast_struct! { + /// An enum input to a `proc_macro_derive` macro. + #[cfg_attr(docsrs, doc(cfg(feature = "derive")))] + pub struct DataEnum { + pub enum_token: Token![enum], + pub brace_token: token::Brace, + pub variants: Punctuated, + } +} + +ast_struct! { + /// An untagged union input to a `proc_macro_derive` macro. + #[cfg_attr(docsrs, doc(cfg(feature = "derive")))] + pub struct DataUnion { + pub union_token: Token![union], + pub fields: FieldsNamed, + } +} + +#[cfg(feature = "parsing")] +pub(crate) mod parsing { + use crate::attr::Attribute; + use crate::data::{Fields, FieldsNamed, Variant}; + use crate::derive::{Data, DataEnum, DataStruct, DataUnion, DeriveInput}; + use crate::error::Result; + use crate::generics::{Generics, WhereClause}; + use crate::ident::Ident; + use crate::parse::{Parse, ParseStream}; + use crate::punctuated::Punctuated; + use crate::restriction::Visibility; + use crate::token; + + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + impl Parse for DeriveInput { + fn parse(input: ParseStream) -> Result { + let attrs = input.call(Attribute::parse_outer)?; + let vis = input.parse::()?; + + let lookahead = input.lookahead1(); + if lookahead.peek(Token![struct]) { + let struct_token = input.parse::()?; + let ident = input.parse::()?; + let generics = input.parse::()?; + let (where_clause, fields, semi) = data_struct(input)?; + Ok(DeriveInput { + attrs, + vis, + ident, + generics: Generics { + where_clause, + ..generics + }, + data: Data::Struct(DataStruct { + struct_token, + fields, + semi_token: semi, + }), + }) + } else if lookahead.peek(Token![enum]) { + let enum_token = input.parse::()?; + let ident = input.parse::()?; + let generics = input.parse::()?; + let (where_clause, brace, variants) = data_enum(input)?; + Ok(DeriveInput { + attrs, + vis, + ident, + generics: Generics { + where_clause, + ..generics + }, + data: Data::Enum(DataEnum { + enum_token, + brace_token: brace, + variants, + }), + }) + } else if lookahead.peek(Token![union]) { + let union_token = input.parse::()?; + let ident = input.parse::()?; + let generics = input.parse::()?; + let (where_clause, fields) = data_union(input)?; + Ok(DeriveInput { + attrs, + vis, + ident, + generics: Generics { + where_clause, + ..generics + }, + data: Data::Union(DataUnion { + union_token, + fields, + }), + }) + } else { + Err(lookahead.error()) + } + } + } + + pub(crate) fn data_struct( + input: ParseStream, + ) -> Result<(Option, Fields, Option)> { + let mut lookahead = input.lookahead1(); + let mut where_clause = None; + if lookahead.peek(Token![where]) { + where_clause = Some(input.parse()?); + lookahead = input.lookahead1(); + } + + if where_clause.is_none() && lookahead.peek(token::Paren) { + let fields = input.parse()?; + + lookahead = input.lookahead1(); + if lookahead.peek(Token![where]) { + where_clause = Some(input.parse()?); + lookahead = input.lookahead1(); + } + + if lookahead.peek(Token![;]) { + let semi = input.parse()?; + Ok((where_clause, Fields::Unnamed(fields), Some(semi))) + } else { + Err(lookahead.error()) + } + } else if lookahead.peek(token::Brace) { + let fields = input.parse()?; + Ok((where_clause, Fields::Named(fields), None)) + } else if lookahead.peek(Token![;]) { + let semi = input.parse()?; + Ok((where_clause, Fields::Unit, Some(semi))) + } else { + Err(lookahead.error()) + } + } + + pub(crate) fn data_enum( + input: ParseStream, + ) -> Result<( + Option, + token::Brace, + Punctuated, + )> { + let where_clause = input.parse()?; + + let content; + let brace = braced!(content in input); + let variants = content.parse_terminated(Variant::parse, Token![,])?; + + Ok((where_clause, brace, variants)) + } + + pub(crate) fn data_union(input: ParseStream) -> Result<(Option, FieldsNamed)> { + let where_clause = input.parse()?; + let fields = input.parse()?; + Ok((where_clause, fields)) + } +} + +#[cfg(feature = "printing")] +mod printing { + use crate::attr::FilterAttrs; + use crate::data::Fields; + use crate::derive::{Data, DeriveInput}; + use crate::print::TokensOrDefault; + use proc_macro2::TokenStream; + use quote::ToTokens; + + #[cfg_attr(docsrs, doc(cfg(feature = "printing")))] + impl ToTokens for DeriveInput { + fn to_tokens(&self, tokens: &mut TokenStream) { + for attr in self.attrs.outer() { + attr.to_tokens(tokens); + } + self.vis.to_tokens(tokens); + match &self.data { + Data::Struct(d) => d.struct_token.to_tokens(tokens), + Data::Enum(d) => d.enum_token.to_tokens(tokens), + Data::Union(d) => d.union_token.to_tokens(tokens), + } + self.ident.to_tokens(tokens); + self.generics.to_tokens(tokens); + match &self.data { + Data::Struct(data) => match &data.fields { + Fields::Named(fields) => { + self.generics.where_clause.to_tokens(tokens); + fields.to_tokens(tokens); + } + Fields::Unnamed(fields) => { + fields.to_tokens(tokens); + self.generics.where_clause.to_tokens(tokens); + TokensOrDefault(&data.semi_token).to_tokens(tokens); + } + Fields::Unit => { + self.generics.where_clause.to_tokens(tokens); + TokensOrDefault(&data.semi_token).to_tokens(tokens); + } + }, + Data::Enum(data) => { + self.generics.where_clause.to_tokens(tokens); + data.brace_token.surround(tokens, |tokens| { + data.variants.to_tokens(tokens); + }); + } + Data::Union(data) => { + self.generics.where_clause.to_tokens(tokens); + data.fields.to_tokens(tokens); + } + } + } + } +} diff --git a/rust/syn/discouraged.rs b/rust/syn/discouraged.rs new file mode 100644 index 000000000000..c8d6bfe89a14 --- /dev/null +++ b/rust/syn/discouraged.rs @@ -0,0 +1,225 @@ +//! Extensions to the parsing API with niche applicability. + +use crate::buffer::Cursor; +use crate::error::Result; +use crate::parse::{inner_unexpected, ParseBuffer, Unexpected}; +use proc_macro2::extra::DelimSpan; +use proc_macro2::Delimiter; +use std::cell::Cell; +use std::mem; +use std::rc::Rc; + +/// Extensions to the `ParseStream` API to support speculative parsing. +pub trait Speculative { + /// Advance this parse stream to the position of a forked parse stream. + /// + /// This is the opposite operation to [`ParseStream::fork`]. You can fork a + /// parse stream, perform some speculative parsing, then join the original + /// stream to the fork to "commit" the parsing from the fork to the main + /// stream. + /// + /// If you can avoid doing this, you should, as it limits the ability to + /// generate useful errors. That said, it is often the only way to parse + /// syntax of the form `A* B*` for arbitrary syntax `A` and `B`. The problem + /// is that when the fork fails to parse an `A`, it's impossible to tell + /// whether that was because of a syntax error and the user meant to provide + /// an `A`, or that the `A`s are finished and it's time to start parsing + /// `B`s. Use with care. + /// + /// Also note that if `A` is a subset of `B`, `A* B*` can be parsed by + /// parsing `B*` and removing the leading members of `A` from the + /// repetition, bypassing the need to involve the downsides associated with + /// speculative parsing. + /// + /// [`ParseStream::fork`]: ParseBuffer::fork + /// + /// # Example + /// + /// There has been chatter about the possibility of making the colons in the + /// turbofish syntax like `path::to::` no longer required by accepting + /// `path::to` in expression position. Specifically, according to [RFC + /// 2544], [`PathSegment`] parsing should always try to consume a following + /// `<` token as the start of generic arguments, and reset to the `<` if + /// that fails (e.g. the token is acting as a less-than operator). + /// + /// This is the exact kind of parsing behavior which requires the "fork, + /// try, commit" behavior that [`ParseStream::fork`] discourages. With + /// `advance_to`, we can avoid having to parse the speculatively parsed + /// content a second time. + /// + /// This change in behavior can be implemented in syn by replacing just the + /// `Parse` implementation for `PathSegment`: + /// + /// ``` + /// # use syn::ext::IdentExt; + /// use syn::parse::discouraged::Speculative; + /// # use syn::parse::{Parse, ParseStream}; + /// # use syn::{Ident, PathArguments, Result, Token}; + /// + /// pub struct PathSegment { + /// pub ident: Ident, + /// pub arguments: PathArguments, + /// } + /// # + /// # impl From for PathSegment + /// # where + /// # T: Into, + /// # { + /// # fn from(ident: T) -> Self { + /// # PathSegment { + /// # ident: ident.into(), + /// # arguments: PathArguments::None, + /// # } + /// # } + /// # } + /// + /// impl Parse for PathSegment { + /// fn parse(input: ParseStream) -> Result { + /// if input.peek(Token![super]) + /// || input.peek(Token![self]) + /// || input.peek(Token![Self]) + /// || input.peek(Token![crate]) + /// { + /// let ident = input.call(Ident::parse_any)?; + /// return Ok(PathSegment::from(ident)); + /// } + /// + /// let ident = input.parse()?; + /// if input.peek(Token![::]) && input.peek3(Token![<]) { + /// return Ok(PathSegment { + /// ident, + /// arguments: PathArguments::AngleBracketed(input.parse()?), + /// }); + /// } + /// if input.peek(Token![<]) && !input.peek(Token![<=]) { + /// let fork = input.fork(); + /// if let Ok(arguments) = fork.parse() { + /// input.advance_to(&fork); + /// return Ok(PathSegment { + /// ident, + /// arguments: PathArguments::AngleBracketed(arguments), + /// }); + /// } + /// } + /// Ok(PathSegment::from(ident)) + /// } + /// } + /// + /// # syn::parse_str::("a").unwrap(); + /// ``` + /// + /// # Drawbacks + /// + /// The main drawback of this style of speculative parsing is in error + /// presentation. Even if the lookahead is the "correct" parse, the error + /// that is shown is that of the "fallback" parse. To use the same example + /// as the turbofish above, take the following unfinished "turbofish": + /// + /// ```text + /// let _ = f<&'a fn(), for<'a> serde::>(); + /// ``` + /// + /// If this is parsed as generic arguments, we can provide the error message + /// + /// ```text + /// error: expected identifier + /// --> src.rs:L:C + /// | + /// L | let _ = f<&'a fn(), for<'a> serde::>(); + /// | ^ + /// ``` + /// + /// but if parsed using the above speculative parsing, it falls back to + /// assuming that the `<` is a less-than when it fails to parse the generic + /// arguments, and tries to interpret the `&'a` as the start of a labelled + /// loop, resulting in the much less helpful error + /// + /// ```text + /// error: expected `:` + /// --> src.rs:L:C + /// | + /// L | let _ = f<&'a fn(), for<'a> serde::>(); + /// | ^^ + /// ``` + /// + /// This can be mitigated with various heuristics (two examples: show both + /// forks' parse errors, or show the one that consumed more tokens), but + /// when you can control the grammar, sticking to something that can be + /// parsed LL(3) and without the LL(*) speculative parsing this makes + /// possible, displaying reasonable errors becomes much more simple. + /// + /// [RFC 2544]: https://github.com/rust-lang/rfcs/pull/2544 + /// [`PathSegment`]: crate::PathSegment + /// + /// # Performance + /// + /// This method performs a cheap fixed amount of work that does not depend + /// on how far apart the two streams are positioned. + /// + /// # Panics + /// + /// The forked stream in the argument of `advance_to` must have been + /// obtained by forking `self`. Attempting to advance to any other stream + /// will cause a panic. + fn advance_to(&self, fork: &Self); +} + +impl<'a> Speculative for ParseBuffer<'a> { + fn advance_to(&self, fork: &Self) { + if !crate::buffer::same_scope(self.cursor(), fork.cursor()) { + panic!("fork was not derived from the advancing parse stream"); + } + + let (self_unexp, self_sp) = inner_unexpected(self); + let (fork_unexp, fork_sp) = inner_unexpected(fork); + if !Rc::ptr_eq(&self_unexp, &fork_unexp) { + match (fork_sp, self_sp) { + // Unexpected set on the fork, but not on `self`, copy it over. + (Some((span, delimiter)), None) => { + self_unexp.set(Unexpected::Some(span, delimiter)); + } + // Unexpected unset. Use chain to propagate errors from fork. + (None, None) => { + fork_unexp.set(Unexpected::Chain(self_unexp)); + + // Ensure toplevel 'unexpected' tokens from the fork don't + // propagate up the chain by replacing the root `unexpected` + // pointer, only 'unexpected' tokens from existing group + // parsers should propagate. + fork.unexpected + .set(Some(Rc::new(Cell::new(Unexpected::None)))); + } + // Unexpected has been set on `self`. No changes needed. + (_, Some(_)) => {} + } + } + + // See comment on `cell` in the struct definition. + self.cell + .set(unsafe { mem::transmute::>(fork.cursor()) }); + } +} + +/// Extensions to the `ParseStream` API to support manipulating invisible +/// delimiters the same as if they were visible. +pub trait AnyDelimiter { + /// Returns the delimiter, the span of the delimiter token, and the nested + /// contents for further parsing. + fn parse_any_delimiter(&self) -> Result<(Delimiter, DelimSpan, ParseBuffer)>; +} + +impl<'a> AnyDelimiter for ParseBuffer<'a> { + fn parse_any_delimiter(&self) -> Result<(Delimiter, DelimSpan, ParseBuffer)> { + self.step(|cursor| { + if let Some((content, delimiter, span, rest)) = cursor.any_group() { + let scope = span.close(); + let nested = crate::parse::advance_step_cursor(cursor, content); + let unexpected = crate::parse::get_unexpected(self); + let content = crate::parse::new_parse_buffer(scope, nested, unexpected); + Ok(((delimiter, span, content), rest)) + } else { + Err(cursor.error("expected any delimiter")) + } + }) + } +} diff --git a/rust/syn/drops.rs b/rust/syn/drops.rs new file mode 100644 index 000000000000..c54308f02c13 --- /dev/null +++ b/rust/syn/drops.rs @@ -0,0 +1,58 @@ +use std::iter; +use std::mem::ManuallyDrop; +use std::ops::{Deref, DerefMut}; +use std::option; +use std::slice; + +#[repr(transparent)] +pub(crate) struct NoDrop(ManuallyDrop); + +impl NoDrop { + pub(crate) fn new(value: T) -> Self + where + T: TrivialDrop, + { + NoDrop(ManuallyDrop::new(value)) + } +} + +impl Deref for NoDrop { + type Target = T; + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl DerefMut for NoDrop { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.0 + } +} + +pub(crate) trait TrivialDrop {} + +impl TrivialDrop for iter::Empty {} +impl TrivialDrop for slice::Iter<'_, T> {} +impl TrivialDrop for slice::IterMut<'_, T> {} +impl TrivialDrop for option::IntoIter<&T> {} +impl TrivialDrop for option::IntoIter<&mut T> {} + +#[test] +fn test_needs_drop() { + use std::mem::needs_drop; + + struct NeedsDrop; + + impl Drop for NeedsDrop { + fn drop(&mut self) {} + } + + assert!(needs_drop::()); + + // Test each of the types with a handwritten TrivialDrop impl above. + assert!(!needs_drop::>()); + assert!(!needs_drop::>()); + assert!(!needs_drop::>()); + assert!(!needs_drop::>()); + assert!(!needs_drop::>()); +} diff --git a/rust/syn/error.rs b/rust/syn/error.rs new file mode 100644 index 000000000000..63310543a3b4 --- /dev/null +++ b/rust/syn/error.rs @@ -0,0 +1,467 @@ +#[cfg(feature = "parsing")] +use crate::buffer::Cursor; +use crate::thread::ThreadBound; +use proc_macro2::{ + Delimiter, Group, Ident, LexError, Literal, Punct, Spacing, Span, TokenStream, TokenTree, +}; +#[cfg(feature = "printing")] +use quote::ToTokens; +use std::fmt::{self, Debug, Display}; +use std::slice; +use std::vec; + +/// The result of a Syn parser. +pub type Result = std::result::Result; + +/// Error returned when a Syn parser cannot parse the input tokens. +/// +/// # Error reporting in proc macros +/// +/// The correct way to report errors back to the compiler from a procedural +/// macro is by emitting an appropriately spanned invocation of +/// [`compile_error!`] in the generated code. This produces a better diagnostic +/// message than simply panicking the macro. +/// +/// [`compile_error!`]: std::compile_error! +/// +/// When parsing macro input, the [`parse_macro_input!`] macro handles the +/// conversion to `compile_error!` automatically. +/// +/// [`parse_macro_input!`]: crate::parse_macro_input! +/// +/// ``` +/// # extern crate proc_macro; +/// # +/// use proc_macro::TokenStream; +/// use syn::parse::{Parse, ParseStream, Result}; +/// use syn::{parse_macro_input, ItemFn}; +/// +/// # const IGNORE: &str = stringify! { +/// #[proc_macro_attribute] +/// # }; +/// pub fn my_attr(args: TokenStream, input: TokenStream) -> TokenStream { +/// let args = parse_macro_input!(args as MyAttrArgs); +/// let input = parse_macro_input!(input as ItemFn); +/// +/// /* ... */ +/// # TokenStream::new() +/// } +/// +/// struct MyAttrArgs { +/// # _k: [(); { stringify! { +/// ... +/// # }; 0 }] +/// } +/// +/// impl Parse for MyAttrArgs { +/// fn parse(input: ParseStream) -> Result { +/// # stringify! { +/// ... +/// # }; +/// # unimplemented!() +/// } +/// } +/// ``` +/// +/// For errors that arise later than the initial parsing stage, the +/// [`.to_compile_error()`] or [`.into_compile_error()`] methods can be used to +/// perform an explicit conversion to `compile_error!`. +/// +/// [`.to_compile_error()`]: Error::to_compile_error +/// [`.into_compile_error()`]: Error::into_compile_error +/// +/// ``` +/// # extern crate proc_macro; +/// # +/// # use proc_macro::TokenStream; +/// # use syn::{parse_macro_input, DeriveInput}; +/// # +/// # const IGNORE: &str = stringify! { +/// #[proc_macro_derive(MyDerive)] +/// # }; +/// pub fn my_derive(input: TokenStream) -> TokenStream { +/// let input = parse_macro_input!(input as DeriveInput); +/// +/// // fn(DeriveInput) -> syn::Result +/// expand::my_derive(input) +/// .unwrap_or_else(syn::Error::into_compile_error) +/// .into() +/// } +/// # +/// # mod expand { +/// # use proc_macro2::TokenStream; +/// # use syn::{DeriveInput, Result}; +/// # +/// # pub fn my_derive(input: DeriveInput) -> Result { +/// # unimplemented!() +/// # } +/// # } +/// ``` +pub struct Error { + messages: Vec, +} + +struct ErrorMessage { + // Span is implemented as an index into a thread-local interner to keep the + // size small. It is not safe to access from a different thread. We want + // errors to be Send and Sync to play nicely with ecosystem crates for error + // handling, so pin the span we're given to its original thread and assume + // it is Span::call_site if accessed from any other thread. + span: ThreadBound, + message: String, +} + +// Cannot use std::ops::Range because that does not implement Copy, +// whereas ThreadBound requires a Copy impl as a way to ensure no Drop impls +// are involved. +struct SpanRange { + start: Span, + end: Span, +} + +#[cfg(test)] +struct _Test +where + Error: Send + Sync; + +impl Error { + /// Usually the [`ParseStream::error`] method will be used instead, which + /// automatically uses the correct span from the current position of the + /// parse stream. + /// + /// Use `Error::new` when the error needs to be triggered on some span other + /// than where the parse stream is currently positioned. + /// + /// [`ParseStream::error`]: crate::parse::ParseBuffer::error + /// + /// # Example + /// + /// ``` + /// use syn::{Error, Ident, LitStr, Result, Token}; + /// use syn::parse::ParseStream; + /// + /// // Parses input that looks like `name = "string"` where the key must be + /// // the identifier `name` and the value may be any string literal. + /// // Returns the string literal. + /// fn parse_name(input: ParseStream) -> Result { + /// let name_token: Ident = input.parse()?; + /// if name_token != "name" { + /// // Trigger an error not on the current position of the stream, + /// // but on the position of the unexpected identifier. + /// return Err(Error::new(name_token.span(), "expected `name`")); + /// } + /// input.parse::()?; + /// let s: LitStr = input.parse()?; + /// Ok(s) + /// } + /// ``` + pub fn new(span: Span, message: T) -> Self { + return new(span, message.to_string()); + + fn new(span: Span, message: String) -> Error { + Error { + messages: vec![ErrorMessage { + span: ThreadBound::new(SpanRange { + start: span, + end: span, + }), + message, + }], + } + } + } + + /// Creates an error with the specified message spanning the given syntax + /// tree node. + /// + /// Unlike the `Error::new` constructor, this constructor takes an argument + /// `tokens` which is a syntax tree node. This allows the resulting `Error` + /// to attempt to span all tokens inside of `tokens`. While you would + /// typically be able to use the `Spanned` trait with the above `Error::new` + /// constructor, implementation limitations today mean that + /// `Error::new_spanned` may provide a higher-quality error message on + /// stable Rust. + /// + /// When in doubt it's recommended to stick to `Error::new` (or + /// `ParseStream::error`)! + #[cfg(feature = "printing")] + #[cfg_attr(docsrs, doc(cfg(feature = "printing")))] + pub fn new_spanned(tokens: T, message: U) -> Self { + return new_spanned(tokens.into_token_stream(), message.to_string()); + + fn new_spanned(tokens: TokenStream, message: String) -> Error { + let mut iter = tokens.into_iter(); + let start = iter.next().map_or_else(Span::call_site, |t| t.span()); + let end = iter.last().map_or(start, |t| t.span()); + Error { + messages: vec![ErrorMessage { + span: ThreadBound::new(SpanRange { start, end }), + message, + }], + } + } + } + + /// The source location of the error. + /// + /// Spans are not thread-safe so this function returns `Span::call_site()` + /// if called from a different thread than the one on which the `Error` was + /// originally created. + pub fn span(&self) -> Span { + let SpanRange { start, end } = match self.messages[0].span.get() { + Some(span) => *span, + None => return Span::call_site(), + }; + start.join(end).unwrap_or(start) + } + + /// Render the error as an invocation of [`compile_error!`]. + /// + /// The [`parse_macro_input!`] macro provides a convenient way to invoke + /// this method correctly in a procedural macro. + /// + /// [`compile_error!`]: std::compile_error! + /// [`parse_macro_input!`]: crate::parse_macro_input! + pub fn to_compile_error(&self) -> TokenStream { + self.messages + .iter() + .map(ErrorMessage::to_compile_error) + .collect() + } + + /// Render the error as an invocation of [`compile_error!`]. + /// + /// [`compile_error!`]: std::compile_error! + /// + /// # Example + /// + /// ``` + /// # extern crate proc_macro; + /// # + /// use proc_macro::TokenStream; + /// use syn::{parse_macro_input, DeriveInput, Error}; + /// + /// # const _: &str = stringify! { + /// #[proc_macro_derive(MyTrait)] + /// # }; + /// pub fn derive_my_trait(input: TokenStream) -> TokenStream { + /// let input = parse_macro_input!(input as DeriveInput); + /// my_trait::expand(input) + /// .unwrap_or_else(Error::into_compile_error) + /// .into() + /// } + /// + /// mod my_trait { + /// use proc_macro2::TokenStream; + /// use syn::{DeriveInput, Result}; + /// + /// pub(crate) fn expand(input: DeriveInput) -> Result { + /// /* ... */ + /// # unimplemented!() + /// } + /// } + /// ``` + pub fn into_compile_error(self) -> TokenStream { + self.to_compile_error() + } + + /// Add another error message to self such that when `to_compile_error()` is + /// called, both errors will be emitted together. + pub fn combine(&mut self, another: Error) { + self.messages.extend(another.messages); + } +} + +impl ErrorMessage { + fn to_compile_error(&self) -> TokenStream { + let (start, end) = match self.span.get() { + Some(range) => (range.start, range.end), + None => (Span::call_site(), Span::call_site()), + }; + + // ::core::compile_error!($message) + TokenStream::from_iter([ + TokenTree::Punct({ + let mut punct = Punct::new(':', Spacing::Joint); + punct.set_span(start); + punct + }), + TokenTree::Punct({ + let mut punct = Punct::new(':', Spacing::Alone); + punct.set_span(start); + punct + }), + TokenTree::Ident(Ident::new("core", start)), + TokenTree::Punct({ + let mut punct = Punct::new(':', Spacing::Joint); + punct.set_span(start); + punct + }), + TokenTree::Punct({ + let mut punct = Punct::new(':', Spacing::Alone); + punct.set_span(start); + punct + }), + TokenTree::Ident(Ident::new("compile_error", start)), + TokenTree::Punct({ + let mut punct = Punct::new('!', Spacing::Alone); + punct.set_span(start); + punct + }), + TokenTree::Group({ + let mut group = Group::new(Delimiter::Brace, { + TokenStream::from_iter([TokenTree::Literal({ + let mut string = Literal::string(&self.message); + string.set_span(end); + string + })]) + }); + group.set_span(end); + group + }), + ]) + } +} + +#[cfg(feature = "parsing")] +pub(crate) fn new_at(scope: Span, cursor: Cursor, message: T) -> Error { + if cursor.eof() { + Error::new(scope, format!("unexpected end of input, {}", message)) + } else { + let span = crate::buffer::open_span_of_group(cursor); + Error::new(span, message) + } +} + +#[cfg(all(feature = "parsing", any(feature = "full", feature = "derive")))] +pub(crate) fn new2(start: Span, end: Span, message: T) -> Error { + return new2(start, end, message.to_string()); + + fn new2(start: Span, end: Span, message: String) -> Error { + Error { + messages: vec![ErrorMessage { + span: ThreadBound::new(SpanRange { start, end }), + message, + }], + } + } +} + +impl Debug for Error { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + if self.messages.len() == 1 { + formatter + .debug_tuple("Error") + .field(&self.messages[0]) + .finish() + } else { + formatter + .debug_tuple("Error") + .field(&self.messages) + .finish() + } + } +} + +impl Debug for ErrorMessage { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + Debug::fmt(&self.message, formatter) + } +} + +impl Display for Error { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str(&self.messages[0].message) + } +} + +impl Clone for Error { + fn clone(&self) -> Self { + Error { + messages: self.messages.clone(), + } + } +} + +impl Clone for ErrorMessage { + fn clone(&self) -> Self { + ErrorMessage { + span: self.span, + message: self.message.clone(), + } + } +} + +impl Clone for SpanRange { + fn clone(&self) -> Self { + *self + } +} + +impl Copy for SpanRange {} + +impl std::error::Error for Error {} + +impl From for Error { + fn from(err: LexError) -> Self { + Error::new(err.span(), err) + } +} + +impl IntoIterator for Error { + type Item = Error; + type IntoIter = IntoIter; + + fn into_iter(self) -> Self::IntoIter { + IntoIter { + messages: self.messages.into_iter(), + } + } +} + +pub struct IntoIter { + messages: vec::IntoIter, +} + +impl Iterator for IntoIter { + type Item = Error; + + fn next(&mut self) -> Option { + Some(Error { + messages: vec![self.messages.next()?], + }) + } +} + +impl<'a> IntoIterator for &'a Error { + type Item = Error; + type IntoIter = Iter<'a>; + + fn into_iter(self) -> Self::IntoIter { + Iter { + messages: self.messages.iter(), + } + } +} + +pub struct Iter<'a> { + messages: slice::Iter<'a, ErrorMessage>, +} + +impl<'a> Iterator for Iter<'a> { + type Item = Error; + + fn next(&mut self) -> Option { + Some(Error { + messages: vec![self.messages.next()?.clone()], + }) + } +} + +impl Extend for Error { + fn extend>(&mut self, iter: T) { + for err in iter { + self.combine(err); + } + } +} diff --git a/rust/syn/export.rs b/rust/syn/export.rs new file mode 100644 index 000000000000..b9ea5c747b75 --- /dev/null +++ b/rust/syn/export.rs @@ -0,0 +1,73 @@ +#[doc(hidden)] +pub use std::clone::Clone; +#[doc(hidden)] +pub use std::cmp::{Eq, PartialEq}; +#[doc(hidden)] +pub use std::concat; +#[doc(hidden)] +pub use std::default::Default; +#[doc(hidden)] +pub use std::fmt::Debug; +#[doc(hidden)] +pub use std::hash::{Hash, Hasher}; +#[doc(hidden)] +pub use std::marker::Copy; +#[doc(hidden)] +pub use std::option::Option::{None, Some}; +#[doc(hidden)] +pub use std::result::Result::{Err, Ok}; +#[doc(hidden)] +pub use std::stringify; + +#[doc(hidden)] +pub type Formatter<'a> = std::fmt::Formatter<'a>; +#[doc(hidden)] +pub type FmtResult = std::fmt::Result; + +#[doc(hidden)] +pub type bool = std::primitive::bool; +#[doc(hidden)] +pub type str = std::primitive::str; + +#[cfg(feature = "printing")] +#[doc(hidden)] +pub use quote; + +#[doc(hidden)] +pub type Span = proc_macro2::Span; +#[doc(hidden)] +pub type TokenStream2 = proc_macro2::TokenStream; + +#[cfg(feature = "parsing")] +#[doc(hidden)] +pub use crate::group::{parse_braces, parse_brackets, parse_parens}; + +#[doc(hidden)] +pub use crate::span::IntoSpans; + +#[cfg(all(feature = "parsing", feature = "printing"))] +#[doc(hidden)] +pub use crate::parse_quote::parse as parse_quote; + +#[cfg(feature = "parsing")] +#[doc(hidden)] +pub use crate::token::parsing::{peek_punct, punct as parse_punct}; + +#[cfg(feature = "printing")] +#[doc(hidden)] +pub use crate::token::printing::punct as print_punct; + +#[cfg(feature = "parsing")] +#[doc(hidden)] +pub use crate::token::private::CustomToken; + +#[cfg(feature = "proc-macro")] +#[doc(hidden)] +pub type TokenStream = proc_macro::TokenStream; + +#[cfg(feature = "printing")] +#[doc(hidden)] +pub use quote::{ToTokens, TokenStreamExt}; + +#[doc(hidden)] +pub struct private(pub(crate) ()); diff --git a/rust/syn/expr.rs b/rust/syn/expr.rs new file mode 100644 index 000000000000..1e49d9a6633e --- /dev/null +++ b/rust/syn/expr.rs @@ -0,0 +1,4173 @@ +use crate::attr::Attribute; +#[cfg(all(feature = "parsing", feature = "full"))] +use crate::error::Result; +#[cfg(feature = "parsing")] +use crate::ext::IdentExt as _; +#[cfg(feature = "full")] +use crate::generics::BoundLifetimes; +use crate::ident::Ident; +#[cfg(any(feature = "parsing", feature = "full"))] +use crate::lifetime::Lifetime; +use crate::lit::Lit; +use crate::mac::Macro; +use crate::op::{BinOp, UnOp}; +#[cfg(feature = "parsing")] +use crate::parse::ParseStream; +#[cfg(feature = "full")] +use crate::pat::Pat; +use crate::path::{AngleBracketedGenericArguments, Path, QSelf}; +use crate::punctuated::Punctuated; +#[cfg(feature = "full")] +use crate::stmt::Block; +use crate::token; +#[cfg(feature = "full")] +use crate::ty::ReturnType; +use crate::ty::Type; +use proc_macro2::{Span, TokenStream}; +#[cfg(feature = "printing")] +use quote::IdentFragment; +#[cfg(feature = "printing")] +use std::fmt::{self, Display}; +use std::hash::{Hash, Hasher}; +#[cfg(all(feature = "parsing", feature = "full"))] +use std::mem; + +ast_enum_of_structs! { + /// A Rust expression. + /// + /// *This type is available only if Syn is built with the `"derive"` or `"full"` + /// feature, but most of the variants are not available unless "full" is enabled.* + /// + /// # Syntax tree enums + /// + /// This type is a syntax tree enum. In Syn this and other syntax tree enums + /// are designed to be traversed using the following rebinding idiom. + /// + /// ``` + /// # use syn::Expr; + /// # + /// # fn example(expr: Expr) { + /// # const IGNORE: &str = stringify! { + /// let expr: Expr = /* ... */; + /// # }; + /// match expr { + /// Expr::MethodCall(expr) => { + /// /* ... */ + /// } + /// Expr::Cast(expr) => { + /// /* ... */ + /// } + /// Expr::If(expr) => { + /// /* ... */ + /// } + /// + /// /* ... */ + /// # _ => {} + /// # } + /// # } + /// ``` + /// + /// We begin with a variable `expr` of type `Expr` that has no fields + /// (because it is an enum), and by matching on it and rebinding a variable + /// with the same name `expr` we effectively imbue our variable with all of + /// the data fields provided by the variant that it turned out to be. So for + /// example above if we ended up in the `MethodCall` case then we get to use + /// `expr.receiver`, `expr.args` etc; if we ended up in the `If` case we get + /// to use `expr.cond`, `expr.then_branch`, `expr.else_branch`. + /// + /// This approach avoids repeating the variant names twice on every line. + /// + /// ``` + /// # use syn::{Expr, ExprMethodCall}; + /// # + /// # fn example(expr: Expr) { + /// // Repetitive; recommend not doing this. + /// match expr { + /// Expr::MethodCall(ExprMethodCall { method, args, .. }) => { + /// # } + /// # _ => {} + /// # } + /// # } + /// ``` + /// + /// In general, the name to which a syntax tree enum variant is bound should + /// be a suitable name for the complete syntax tree enum type. + /// + /// ``` + /// # use syn::{Expr, ExprField}; + /// # + /// # fn example(discriminant: ExprField) { + /// // Binding is called `base` which is the name I would use if I were + /// // assigning `*discriminant.base` without an `if let`. + /// if let Expr::Tuple(base) = *discriminant.base { + /// # } + /// # } + /// ``` + /// + /// A sign that you may not be choosing the right variable names is if you + /// see names getting repeated in your code, like accessing + /// `receiver.receiver` or `pat.pat` or `cond.cond`. + #[cfg_attr(docsrs, doc(cfg(any(feature = "full", feature = "derive"))))] + #[non_exhaustive] + pub enum Expr { + /// A slice literal expression: `[a, b, c, d]`. + Array(ExprArray), + + /// An assignment expression: `a = compute()`. + Assign(ExprAssign), + + /// An async block: `async { ... }`. + Async(ExprAsync), + + /// An await expression: `fut.await`. + Await(ExprAwait), + + /// A binary operation: `a + b`, `a += b`. + Binary(ExprBinary), + + /// A blocked scope: `{ ... }`. + Block(ExprBlock), + + /// A `break`, with an optional label to break and an optional + /// expression. + Break(ExprBreak), + + /// A function call expression: `invoke(a, b)`. + Call(ExprCall), + + /// A cast expression: `foo as f64`. + Cast(ExprCast), + + /// A closure expression: `|a, b| a + b`. + Closure(ExprClosure), + + /// A const block: `const { ... }`. + Const(ExprConst), + + /// A `continue`, with an optional label. + Continue(ExprContinue), + + /// Access of a named struct field (`obj.k`) or unnamed tuple struct + /// field (`obj.0`). + Field(ExprField), + + /// A for loop: `for pat in expr { ... }`. + ForLoop(ExprForLoop), + + /// An expression contained within invisible delimiters. + /// + /// This variant is important for faithfully representing the precedence + /// of expressions and is related to `None`-delimited spans in a + /// `TokenStream`. + Group(ExprGroup), + + /// An `if` expression with an optional `else` block: `if expr { ... } + /// else { ... }`. + /// + /// The `else` branch expression may only be an `If` or `Block` + /// expression, not any of the other types of expression. + If(ExprIf), + + /// A square bracketed indexing expression: `vector[2]`. + Index(ExprIndex), + + /// The inferred value of a const generic argument, denoted `_`. + Infer(ExprInfer), + + /// A `let` guard: `let Some(x) = opt`. + Let(ExprLet), + + /// A literal in place of an expression: `1`, `"foo"`. + Lit(ExprLit), + + /// Conditionless loop: `loop { ... }`. + Loop(ExprLoop), + + /// A macro invocation expression: `format!("{}", q)`. + Macro(ExprMacro), + + /// A `match` expression: `match n { Some(n) => {}, None => {} }`. + Match(ExprMatch), + + /// A method call expression: `x.foo::(a, b)`. + MethodCall(ExprMethodCall), + + /// A parenthesized expression: `(a + b)`. + Paren(ExprParen), + + /// A path like `std::mem::replace` possibly containing generic + /// parameters and a qualified self-type. + /// + /// A plain identifier like `x` is a path of length 1. + Path(ExprPath), + + /// A range expression: `1..2`, `1..`, `..2`, `1..=2`, `..=2`. + Range(ExprRange), + + /// Address-of operation: `&raw const place` or `&raw mut place`. + RawAddr(ExprRawAddr), + + /// A referencing operation: `&a` or `&mut a`. + Reference(ExprReference), + + /// An array literal constructed from one repeated element: `[0u8; N]`. + Repeat(ExprRepeat), + + /// A `return`, with an optional value to be returned. + Return(ExprReturn), + + /// A struct literal expression: `Point { x: 1, y: 1 }`. + /// + /// The `rest` provides the value of the remaining fields as in `S { a: + /// 1, b: 1, ..rest }`. + Struct(ExprStruct), + + /// A try-expression: `expr?`. + Try(ExprTry), + + /// A try block: `try { ... }`. + TryBlock(ExprTryBlock), + + /// A tuple expression: `(a, b, c, d)`. + Tuple(ExprTuple), + + /// A unary operation: `!x`, `*x`. + Unary(ExprUnary), + + /// An unsafe block: `unsafe { ... }`. + Unsafe(ExprUnsafe), + + /// Tokens in expression position not interpreted by Syn. + Verbatim(TokenStream), + + /// A while loop: `while expr { ... }`. + While(ExprWhile), + + /// A yield expression: `yield expr`. + Yield(ExprYield), + + // For testing exhaustiveness in downstream code, use the following idiom: + // + // match expr { + // #![cfg_attr(test, deny(non_exhaustive_omitted_patterns))] + // + // Expr::Array(expr) => {...} + // Expr::Assign(expr) => {...} + // ... + // Expr::Yield(expr) => {...} + // + // _ => { /* some sane fallback */ } + // } + // + // This way we fail your tests but don't break your library when adding + // a variant. You will be notified by a test failure when a variant is + // added, so that you can add code to handle it, but your library will + // continue to compile and work for downstream users in the interim. + } +} + +ast_struct! { + /// A slice literal expression: `[a, b, c, d]`. + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + pub struct ExprArray #full { + pub attrs: Vec, + pub bracket_token: token::Bracket, + pub elems: Punctuated, + } +} + +ast_struct! { + /// An assignment expression: `a = compute()`. + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + pub struct ExprAssign #full { + pub attrs: Vec, + pub left: Box, + pub eq_token: Token![=], + pub right: Box, + } +} + +ast_struct! { + /// An async block: `async { ... }`. + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + pub struct ExprAsync #full { + pub attrs: Vec, + pub async_token: Token![async], + pub capture: Option, + pub block: Block, + } +} + +ast_struct! { + /// An await expression: `fut.await`. + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + pub struct ExprAwait #full { + pub attrs: Vec, + pub base: Box, + pub dot_token: Token![.], + pub await_token: Token![await], + } +} + +ast_struct! { + /// A binary operation: `a + b`, `a += b`. + #[cfg_attr(docsrs, doc(cfg(any(feature = "full", feature = "derive"))))] + pub struct ExprBinary { + pub attrs: Vec, + pub left: Box, + pub op: BinOp, + pub right: Box, + } +} + +ast_struct! { + /// A blocked scope: `{ ... }`. + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + pub struct ExprBlock #full { + pub attrs: Vec, + pub label: Option