From 9762dca54a4fec433b50eb83fdd8ff0a876cccf2 Mon Sep 17 00:00:00 2001 From: Benno Lossin Date: Sat, 9 Mar 2024 15:53:25 +0000 Subject: rust: macros: add `decl_generics` to `parse_generics()` The generic parameters on a type definition can specify default values. Currently `parse_generics()` cannot handle this though. For example when parsing the following generics: The `impl_generics` will be set to `T: Clone, const N: usize = 0` and `ty_generics` will be set to `T, N`. Now using the `impl_generics` on an impl block: impl<$($impl_generics)*> Foo {} will result in invalid Rust code, because default values are only available on type definitions. Therefore add parsing support for generic parameter default values using a new kind of generics called `decl_generics` and change the old behavior of `impl_generics` to not contain the generic parameter default values. Now `Generics` has three fields: - `impl_generics`: the generics with bounds (e.g. `T: Clone, const N: usize`) - `decl_generics`: the generics with bounds and default values (e.g. `T: Clone, const N: usize = 0`) - `ty_generics`: contains the generics without bounds and without default values (e.g. `T, N`) `impl_generics` is designed to be used on `impl<$impl_generics>`, `decl_generics` for the type definition, so `struct Foo<$decl_generics>` and `ty_generics` whenever you use the type, so `Foo<$ty_generics>`. Here is an example that uses all three different types of generics: let (Generics { decl_generics, impl_generics, ty_generics }, rest) = parse_generics(input); quote! { struct Foo<$($decl_generics)*> { // ... } impl<$impl_generics> Foo<$ty_generics> { fn foo() { // ... } } } The next commit contains a fix to the `#[pin_data]` macro making it compatible with generic parameter default values by relying on this new behavior. Signed-off-by: Benno Lossin Reviewed-by: Alice Ryhl Link: https://lore.kernel.org/r/20240309155243.482334-1-benno.lossin@proton.me Signed-off-by: Miguel Ojeda --- rust/macros/helpers.rs | 123 +++++++++++++++++++++++++++++++++++++------------ 1 file changed, 93 insertions(+), 30 deletions(-) (limited to 'rust/macros/helpers.rs') diff --git a/rust/macros/helpers.rs b/rust/macros/helpers.rs index afb0f2e3a36a..2497d6519f2f 100644 --- a/rust/macros/helpers.rs +++ b/rust/macros/helpers.rs @@ -1,6 +1,6 @@ // SPDX-License-Identifier: GPL-2.0 -use proc_macro::{token_stream, Group, Punct, Spacing, TokenStream, TokenTree}; +use proc_macro::{token_stream, Group, TokenStream, TokenTree}; pub(crate) fn try_ident(it: &mut token_stream::IntoIter) -> Option { if let Some(TokenTree::Ident(ident)) = it.next() { @@ -70,8 +70,41 @@ pub(crate) fn expect_end(it: &mut token_stream::IntoIter) { } } +/// Parsed generics. +/// +/// See the field documentation for an explanation what each of the fields represents. +/// +/// # Examples +/// +/// ```rust,ignore +/// # let input = todo!(); +/// let (Generics { decl_generics, impl_generics, ty_generics }, rest) = parse_generics(input); +/// quote! { +/// struct Foo<$($decl_generics)*> { +/// // ... +/// } +/// +/// impl<$impl_generics> Foo<$ty_generics> { +/// fn foo() { +/// // ... +/// } +/// } +/// } +/// ``` pub(crate) struct Generics { + /// The generics with bounds and default values (e.g. `T: Clone, const N: usize = 0`). + /// + /// Use this on type definitions e.g. `struct Foo<$decl_generics> ...` (or `union`/`enum`). + #[allow(dead_code)] + pub(crate) decl_generics: Vec, + /// The generics with bounds (e.g. `T: Clone, const N: usize`). + /// + /// Use this on `impl` blocks e.g. `impl<$impl_generics> Trait for ...`. pub(crate) impl_generics: Vec, + /// The generics without bounds and without default values (e.g. `T, N`). + /// + /// Use this when you use the type that is declared with these generics e.g. + /// `Foo<$ty_generics>`. pub(crate) ty_generics: Vec, } @@ -79,6 +112,8 @@ pub(crate) struct Generics { /// /// The generics are not present in the rest, but a where clause might remain. pub(crate) fn parse_generics(input: TokenStream) -> (Generics, Vec) { + // The generics with bounds and default values. + let mut decl_generics = vec![]; // `impl_generics`, the declared generics with their bounds. let mut impl_generics = vec![]; // Only the names of the generics, without any bounds. @@ -90,10 +125,17 @@ pub(crate) fn parse_generics(input: TokenStream) -> (Generics, Vec) { let mut toks = input.into_iter(); // If we are at the beginning of a generic parameter. let mut at_start = true; - for tt in &mut toks { + let mut skip_until_comma = false; + while let Some(tt) = toks.next() { + if nesting == 1 && matches!(&tt, TokenTree::Punct(p) if p.as_char() == '>') { + // Found the end of the generics. + break; + } else if nesting >= 1 { + decl_generics.push(tt.clone()); + } match tt.clone() { TokenTree::Punct(p) if p.as_char() == '<' => { - if nesting >= 1 { + if nesting >= 1 && !skip_until_comma { // This is inside of the generics and part of some bound. impl_generics.push(tt); } @@ -105,49 +147,70 @@ pub(crate) fn parse_generics(input: TokenStream) -> (Generics, Vec) { break; } else { nesting -= 1; - if nesting >= 1 { + if nesting >= 1 && !skip_until_comma { // We are still inside of the generics and part of some bound. impl_generics.push(tt); } - if nesting == 0 { - break; - } } } - tt => { + TokenTree::Punct(p) if skip_until_comma && p.as_char() == ',' => { if nesting == 1 { - // Here depending on the token, it might be a generic variable name. - match &tt { - // Ignore const. - TokenTree::Ident(i) if i.to_string() == "const" => {} - TokenTree::Ident(_) if at_start => { - ty_generics.push(tt.clone()); - // We also already push the `,` token, this makes it easier to append - // generics. - ty_generics.push(TokenTree::Punct(Punct::new(',', Spacing::Alone))); - at_start = false; - } - TokenTree::Punct(p) if p.as_char() == ',' => at_start = true, - // Lifetimes begin with `'`. - TokenTree::Punct(p) if p.as_char() == '\'' && at_start => { - ty_generics.push(tt.clone()); - } - _ => {} - } - } - if nesting >= 1 { + impl_generics.push(tt.clone()); impl_generics.push(tt); - } else if nesting == 0 { + skip_until_comma = false; + } + } + _ if !skip_until_comma => { + match nesting { // If we haven't entered the generics yet, we still want to keep these tokens. - rest.push(tt); + 0 => rest.push(tt), + 1 => { + // Here depending on the token, it might be a generic variable name. + match tt.clone() { + TokenTree::Ident(i) if at_start && i.to_string() == "const" => { + let Some(name) = toks.next() else { + // Parsing error. + break; + }; + impl_generics.push(tt); + impl_generics.push(name.clone()); + ty_generics.push(name.clone()); + decl_generics.push(name); + at_start = false; + } + TokenTree::Ident(_) if at_start => { + impl_generics.push(tt.clone()); + ty_generics.push(tt); + at_start = false; + } + TokenTree::Punct(p) if p.as_char() == ',' => { + impl_generics.push(tt.clone()); + ty_generics.push(tt); + at_start = true; + } + // Lifetimes begin with `'`. + TokenTree::Punct(p) if p.as_char() == '\'' && at_start => { + impl_generics.push(tt.clone()); + ty_generics.push(tt); + } + // Generics can have default values, we skip these. + TokenTree::Punct(p) if p.as_char() == '=' => { + skip_until_comma = true; + } + _ => impl_generics.push(tt), + } + } + _ => impl_generics.push(tt), } } + _ => {} } } rest.extend(toks); ( Generics { impl_generics, + decl_generics, ty_generics, }, rest, -- cgit v1.2.3 From 22eed6068d76d1d9672f33334740657208a91483 Mon Sep 17 00:00:00 2001 From: Benno Lossin Date: Sat, 9 Mar 2024 15:54:04 +0000 Subject: rust: macros: allow generic parameter default values in `#[pin_data]` Add support for generic parameters defaults in `#[pin_data]` by using the newly introduced `decl_generics` instead of the `impl_generics`. Before this would not compile: #[pin_data] struct Foo { // ... } because it would be expanded to this: struct Foo { // ... } const _: () = { struct __ThePinData { __phantom: ::core::marker::PhantomData) -> Foo>, } impl ::core::clone::Clone for __ThePinData { fn clone(&self) -> Self { *self } } // [...] rest of expansion omitted }; The problem is with the `impl`, since that is invalid Rust syntax. It should not mention the default value at all, since default values only make sense on type definitions. The new `impl_generics` do not contain the default values, thus generating correct Rust code. This is used by the next commit that puts `#[pin_data]` on `kernel::workqueue::Work`. Signed-off-by: Benno Lossin Reviewed-by: Martin Rodriguez Reboredo Reviewed-by: Gary Guo Reviewed-by: Alice Ryhl Tested-by: Alice Ryhl Link: https://lore.kernel.org/r/20240309155243.482334-2-benno.lossin@proton.me Signed-off-by: Miguel Ojeda --- rust/kernel/init/macros.rs | 19 ++++++++++++++++++- rust/macros/helpers.rs | 1 - rust/macros/pin_data.rs | 3 ++- 3 files changed, 20 insertions(+), 3 deletions(-) (limited to 'rust/macros/helpers.rs') diff --git a/rust/kernel/init/macros.rs b/rust/kernel/init/macros.rs index cb6e61b6c50b..624e9108e3b4 100644 --- a/rust/kernel/init/macros.rs +++ b/rust/kernel/init/macros.rs @@ -538,6 +538,7 @@ macro_rules! __pin_data { ), @impl_generics($($impl_generics:tt)*), @ty_generics($($ty_generics:tt)*), + @decl_generics($($decl_generics:tt)*), @body({ $($fields:tt)* }), ) => { // We now use token munching to iterate through all of the fields. While doing this we @@ -560,6 +561,9 @@ macro_rules! __pin_data { @impl_generics($($impl_generics)*), // The 'ty generics', the generics that will need to be specified on the impl blocks. @ty_generics($($ty_generics)*), + // The 'decl generics', the generics that need to be specified on the struct + // definition. + @decl_generics($($decl_generics)*), // The where clause of any impl block and the declaration. @where($($($whr)*)?), // The remaining fields tokens that need to be processed. @@ -585,6 +589,7 @@ macro_rules! __pin_data { @name($name:ident), @impl_generics($($impl_generics:tt)*), @ty_generics($($ty_generics:tt)*), + @decl_generics($($decl_generics:tt)*), @where($($whr:tt)*), // We found a PhantomPinned field, this should generally be pinned! @fields_munch($field:ident : $($($(::)?core::)?marker::)?PhantomPinned, $($rest:tt)*), @@ -607,6 +612,7 @@ macro_rules! __pin_data { @name($name), @impl_generics($($impl_generics)*), @ty_generics($($ty_generics)*), + @decl_generics($($decl_generics)*), @where($($whr)*), @fields_munch($($rest)*), @pinned($($pinned)* $($accum)* $field: ::core::marker::PhantomPinned,), @@ -623,6 +629,7 @@ macro_rules! __pin_data { @name($name:ident), @impl_generics($($impl_generics:tt)*), @ty_generics($($ty_generics:tt)*), + @decl_generics($($decl_generics:tt)*), @where($($whr:tt)*), // We reached the field declaration. @fields_munch($field:ident : $type:ty, $($rest:tt)*), @@ -640,6 +647,7 @@ macro_rules! __pin_data { @name($name), @impl_generics($($impl_generics)*), @ty_generics($($ty_generics)*), + @decl_generics($($decl_generics)*), @where($($whr)*), @fields_munch($($rest)*), @pinned($($pinned)* $($accum)* $field: $type,), @@ -656,6 +664,7 @@ macro_rules! __pin_data { @name($name:ident), @impl_generics($($impl_generics:tt)*), @ty_generics($($ty_generics:tt)*), + @decl_generics($($decl_generics:tt)*), @where($($whr:tt)*), // We reached the field declaration. @fields_munch($field:ident : $type:ty, $($rest:tt)*), @@ -673,6 +682,7 @@ macro_rules! __pin_data { @name($name), @impl_generics($($impl_generics)*), @ty_generics($($ty_generics)*), + @decl_generics($($decl_generics)*), @where($($whr)*), @fields_munch($($rest)*), @pinned($($pinned)*), @@ -689,6 +699,7 @@ macro_rules! __pin_data { @name($name:ident), @impl_generics($($impl_generics:tt)*), @ty_generics($($ty_generics:tt)*), + @decl_generics($($decl_generics:tt)*), @where($($whr:tt)*), // We found the `#[pin]` attr. @fields_munch(#[pin] $($rest:tt)*), @@ -705,6 +716,7 @@ macro_rules! __pin_data { @name($name), @impl_generics($($impl_generics)*), @ty_generics($($ty_generics)*), + @decl_generics($($decl_generics)*), @where($($whr)*), @fields_munch($($rest)*), // We do not include `#[pin]` in the list of attributes, since it is not actually an @@ -724,6 +736,7 @@ macro_rules! __pin_data { @name($name:ident), @impl_generics($($impl_generics:tt)*), @ty_generics($($ty_generics:tt)*), + @decl_generics($($decl_generics:tt)*), @where($($whr:tt)*), // We reached the field declaration with visibility, for simplicity we only munch the // visibility and put it into `$accum`. @@ -741,6 +754,7 @@ macro_rules! __pin_data { @name($name), @impl_generics($($impl_generics)*), @ty_generics($($ty_generics)*), + @decl_generics($($decl_generics)*), @where($($whr)*), @fields_munch($field $($rest)*), @pinned($($pinned)*), @@ -757,6 +771,7 @@ macro_rules! __pin_data { @name($name:ident), @impl_generics($($impl_generics:tt)*), @ty_generics($($ty_generics:tt)*), + @decl_generics($($decl_generics:tt)*), @where($($whr:tt)*), // Some other attribute, just put it into `$accum`. @fields_munch(#[$($attr:tt)*] $($rest:tt)*), @@ -773,6 +788,7 @@ macro_rules! __pin_data { @name($name), @impl_generics($($impl_generics)*), @ty_generics($($ty_generics)*), + @decl_generics($($decl_generics)*), @where($($whr)*), @fields_munch($($rest)*), @pinned($($pinned)*), @@ -789,6 +805,7 @@ macro_rules! __pin_data { @name($name:ident), @impl_generics($($impl_generics:tt)*), @ty_generics($($ty_generics:tt)*), + @decl_generics($($decl_generics:tt)*), @where($($whr:tt)*), // We reached the end of the fields, plus an optional additional comma, since we added one // before and the user is also allowed to put a trailing comma. @@ -802,7 +819,7 @@ macro_rules! __pin_data { ) => { // Declare the struct with all fields in the correct order. $($struct_attrs)* - $vis struct $name <$($impl_generics)*> + $vis struct $name <$($decl_generics)*> where $($whr)* { $($fields)* diff --git a/rust/macros/helpers.rs b/rust/macros/helpers.rs index 2497d6519f2f..563dcd2b7ace 100644 --- a/rust/macros/helpers.rs +++ b/rust/macros/helpers.rs @@ -95,7 +95,6 @@ pub(crate) struct Generics { /// The generics with bounds and default values (e.g. `T: Clone, const N: usize = 0`). /// /// Use this on type definitions e.g. `struct Foo<$decl_generics> ...` (or `union`/`enum`). - #[allow(dead_code)] pub(crate) decl_generics: Vec, /// The generics with bounds (e.g. `T: Clone, const N: usize`). /// diff --git a/rust/macros/pin_data.rs b/rust/macros/pin_data.rs index 022e68e9720d..1d4a3547c684 100644 --- a/rust/macros/pin_data.rs +++ b/rust/macros/pin_data.rs @@ -10,7 +10,7 @@ pub(crate) fn pin_data(args: TokenStream, input: TokenStream) -> TokenStream { let ( Generics { impl_generics, - decl_generics: _, + decl_generics, ty_generics, }, rest, @@ -77,6 +77,7 @@ pub(crate) fn pin_data(args: TokenStream, input: TokenStream) -> TokenStream { @sig(#(#rest)*), @impl_generics(#(#impl_generics)*), @ty_generics(#(#ty_generics)*), + @decl_generics(#(#decl_generics)*), @body(#last), }); quoted.extend(errs); -- cgit v1.2.3