From d9f2fdfaed8d18ede6d113426279c86d84d92445 Mon Sep 17 00:00:00 2001 From: Nicholas Nethercote Date: Mon, 31 Jul 2023 15:47:19 +1000 Subject: [PATCH 1/8] `parse_all_token_trees` cannot fail. --- compiler/rustc_parse/src/parser/mod.rs | 4 ++-- src/librustdoc/clean/render_macro_matchers.rs | 8 +------- 2 files changed, 3 insertions(+), 9 deletions(-) diff --git a/compiler/rustc_parse/src/parser/mod.rs b/compiler/rustc_parse/src/parser/mod.rs index 57778d670980a..b452eb6a401f2 100644 --- a/compiler/rustc_parse/src/parser/mod.rs +++ b/compiler/rustc_parse/src/parser/mod.rs @@ -1288,12 +1288,12 @@ impl<'a> Parser<'a> { } /// Parses a stream of tokens into a list of `TokenTree`s, up to EOF. - pub fn parse_all_token_trees(&mut self) -> PResult<'a, Vec> { + pub fn parse_all_token_trees(&mut self) -> Vec { let mut tts = Vec::new(); while self.token != token::Eof { tts.push(self.parse_token_tree()); } - Ok(tts) + tts } pub fn parse_tokens(&mut self) -> TokenStream { diff --git a/src/librustdoc/clean/render_macro_matchers.rs b/src/librustdoc/clean/render_macro_matchers.rs index ef38ca3c16c1a..b6120e814dfff 100644 --- a/src/librustdoc/clean/render_macro_matchers.rs +++ b/src/librustdoc/clean/render_macro_matchers.rs @@ -76,13 +76,7 @@ fn snippet_equal_to_token(tcx: TyCtxt<'_>, matcher: &TokenTree) -> Option reparsed_trees, - Err(diagnostic) => { - diagnostic.cancel(); - return None; - } - }; + let mut reparsed_trees = parser.parse_all_token_trees(); if reparsed_trees.len() != 1 { return None; } From db7b07aa028b65b4d85610e81fecb0d2382b8c76 Mon Sep 17 00:00:00 2001 From: Nicholas Nethercote Date: Mon, 31 Jul 2023 15:51:08 +1000 Subject: [PATCH 2/8] Inline and remove `parse_all_token_trees`. It has a single call site. --- compiler/rustc_parse/src/parser/mod.rs | 11 +---------- src/librustdoc/clean/render_macro_matchers.rs | 5 ++++- 2 files changed, 5 insertions(+), 11 deletions(-) diff --git a/compiler/rustc_parse/src/parser/mod.rs b/compiler/rustc_parse/src/parser/mod.rs index b452eb6a401f2..4b23f5a6de12e 100644 --- a/compiler/rustc_parse/src/parser/mod.rs +++ b/compiler/rustc_parse/src/parser/mod.rs @@ -1251,7 +1251,7 @@ impl<'a> Parser<'a> { } /// Parses a single token tree from the input. - pub(crate) fn parse_token_tree(&mut self) -> TokenTree { + pub fn parse_token_tree(&mut self) -> TokenTree { match self.token.kind { token::OpenDelim(..) => { // Grab the tokens within the delimiters. @@ -1287,15 +1287,6 @@ impl<'a> Parser<'a> { } } - /// Parses a stream of tokens into a list of `TokenTree`s, up to EOF. - pub fn parse_all_token_trees(&mut self) -> Vec { - let mut tts = Vec::new(); - while self.token != token::Eof { - tts.push(self.parse_token_tree()); - } - tts - } - pub fn parse_tokens(&mut self) -> TokenStream { let mut result = Vec::new(); loop { diff --git a/src/librustdoc/clean/render_macro_matchers.rs b/src/librustdoc/clean/render_macro_matchers.rs index b6120e814dfff..b3dc0f63e4505 100644 --- a/src/librustdoc/clean/render_macro_matchers.rs +++ b/src/librustdoc/clean/render_macro_matchers.rs @@ -76,7 +76,10 @@ fn snippet_equal_to_token(tcx: TyCtxt<'_>, matcher: &TokenTree) -> Option Date: Mon, 31 Jul 2023 15:53:46 +1000 Subject: [PATCH 3/8] Only call `parse_token_tree` once. This code currently uses a `while` loop and gathers token trees into a vector, but it only succeeds in the case where there is a single token tree. We can instead just call `parse_token_tree` once and then look for `Eof` to detect the success case. --- src/librustdoc/clean/render_macro_matchers.rs | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/librustdoc/clean/render_macro_matchers.rs b/src/librustdoc/clean/render_macro_matchers.rs index b3dc0f63e4505..66d10f2368b22 100644 --- a/src/librustdoc/clean/render_macro_matchers.rs +++ b/src/librustdoc/clean/render_macro_matchers.rs @@ -76,14 +76,13 @@ fn snippet_equal_to_token(tcx: TyCtxt<'_>, matcher: &TokenTree) -> Option Date: Mon, 31 Jul 2023 16:15:54 +1000 Subject: [PATCH 4/8] Move `TokenCursor::num_next_calls` into `Parser` and rename it. It's more of a `Parser`-level concern than a `TokenCursor`-level concern. Also, `num_bump_calls` is a more accurate name, because it's incremented in `Parser::bump`. --- compiler/rustc_parse/src/parser/attr.rs | 6 +++--- compiler/rustc_parse/src/parser/attr_wrapper.rs | 13 ++++++------- compiler/rustc_parse/src/parser/mod.rs | 13 +++++-------- 3 files changed, 14 insertions(+), 18 deletions(-) diff --git a/compiler/rustc_parse/src/parser/attr.rs b/compiler/rustc_parse/src/parser/attr.rs index ee0abba1c1753..104de47b97dd1 100644 --- a/compiler/rustc_parse/src/parser/attr.rs +++ b/compiler/rustc_parse/src/parser/attr.rs @@ -36,7 +36,7 @@ impl<'a> Parser<'a> { pub(super) fn parse_outer_attributes(&mut self) -> PResult<'a, AttrWrapper> { let mut outer_attrs = ast::AttrVec::new(); let mut just_parsed_doc_comment = false; - let start_pos = self.token_cursor.num_next_calls; + let start_pos = self.num_bump_calls; loop { let attr = if self.check(&token::Pound) { let prev_outer_attr_sp = outer_attrs.last().map(|attr| attr.span); @@ -277,7 +277,7 @@ impl<'a> Parser<'a> { pub(crate) fn parse_inner_attributes(&mut self) -> PResult<'a, ast::AttrVec> { let mut attrs = ast::AttrVec::new(); loop { - let start_pos: u32 = self.token_cursor.num_next_calls.try_into().unwrap(); + let start_pos: u32 = self.num_bump_calls.try_into().unwrap(); // Only try to parse if it is an inner attribute (has `!`). let attr = if self.check(&token::Pound) && self.look_ahead(1, |t| t == &token::Not) { Some(self.parse_attribute(InnerAttrPolicy::Permitted)?) @@ -298,7 +298,7 @@ impl<'a> Parser<'a> { None }; if let Some(attr) = attr { - let end_pos: u32 = self.token_cursor.num_next_calls.try_into().unwrap(); + let end_pos: u32 = self.num_bump_calls.try_into().unwrap(); // If we are currently capturing tokens, mark the location of this inner attribute. // If capturing ends up creating a `LazyAttrTokenStream`, we will include // this replace range with it, removing the inner attribute from the final diff --git a/compiler/rustc_parse/src/parser/attr_wrapper.rs b/compiler/rustc_parse/src/parser/attr_wrapper.rs index 158ab2a295667..10c2066ef6c22 100644 --- a/compiler/rustc_parse/src/parser/attr_wrapper.rs +++ b/compiler/rustc_parse/src/parser/attr_wrapper.rs @@ -213,6 +213,7 @@ impl<'a> Parser<'a> { let start_token = (self.token.clone(), self.token_spacing); let cursor_snapshot = self.token_cursor.clone(); + let start_pos = self.num_bump_calls; let has_outer_attrs = !attrs.attrs.is_empty(); let prev_capturing = std::mem::replace(&mut self.capture_state.capturing, Capturing::Yes); @@ -273,8 +274,7 @@ impl<'a> Parser<'a> { let replace_ranges_end = self.capture_state.replace_ranges.len(); - let cursor_snapshot_next_calls = cursor_snapshot.num_next_calls; - let mut end_pos = self.token_cursor.num_next_calls; + let mut end_pos = self.num_bump_calls; let mut captured_trailing = false; @@ -306,7 +306,7 @@ impl<'a> Parser<'a> { end_pos += 1; } - let num_calls = end_pos - cursor_snapshot_next_calls; + let num_calls = end_pos - start_pos; // If we have no attributes, then we will never need to // use any replace ranges. @@ -316,7 +316,7 @@ impl<'a> Parser<'a> { // Grab any replace ranges that occur *inside* the current AST node. // We will perform the actual replacement when we convert the `LazyAttrTokenStream` // to an `AttrTokenStream`. - let start_calls: u32 = cursor_snapshot_next_calls.try_into().unwrap(); + let start_calls: u32 = start_pos.try_into().unwrap(); self.capture_state.replace_ranges[replace_ranges_start..replace_ranges_end] .iter() .cloned() @@ -359,8 +359,7 @@ impl<'a> Parser<'a> { // with a `FlatToken::AttrTarget`. If this AST node is inside an item // that has `#[derive]`, then this will allow us to cfg-expand this // AST node. - let start_pos = - if has_outer_attrs { attrs.start_pos } else { cursor_snapshot_next_calls }; + let start_pos = if has_outer_attrs { attrs.start_pos } else { start_pos }; let new_tokens = vec![(FlatToken::AttrTarget(attr_data), Spacing::Alone)]; assert!( @@ -464,6 +463,6 @@ mod size_asserts { use rustc_data_structures::static_assert_size; // tidy-alphabetical-start static_assert_size!(AttrWrapper, 16); - static_assert_size!(LazyAttrTokenStreamImpl, 120); + static_assert_size!(LazyAttrTokenStreamImpl, 112); // tidy-alphabetical-end } diff --git a/compiler/rustc_parse/src/parser/mod.rs b/compiler/rustc_parse/src/parser/mod.rs index 4b23f5a6de12e..2a1524f108a9e 100644 --- a/compiler/rustc_parse/src/parser/mod.rs +++ b/compiler/rustc_parse/src/parser/mod.rs @@ -135,9 +135,9 @@ pub struct Parser<'a> { pub capture_cfg: bool, restrictions: Restrictions, expected_tokens: Vec, - // Important: This must only be advanced from `bump` to ensure that - // `token_cursor.num_next_calls` is updated properly. token_cursor: TokenCursor, + // The number of calls to `bump`, i.e. the position in the token stream. + num_bump_calls: usize, /// This field is used to keep track of how many left angle brackets we have seen. This is /// required in order to detect extra leading left angle brackets (`<` characters) and error /// appropriately. @@ -224,9 +224,6 @@ struct TokenCursor { // because it's the outermost token stream which never has delimiters. stack: Vec<(TokenTreeCursor, Delimiter, DelimSpan)>, - // Counts the number of calls to `{,inlined_}next`. - num_next_calls: usize, - // During parsing, we may sometimes need to 'unglue' a // glued token into two component tokens // (e.g. '>>' into '>' and '>), so that the parser @@ -402,9 +399,9 @@ impl<'a> Parser<'a> { token_cursor: TokenCursor { tree_cursor: stream.into_trees(), stack: Vec::new(), - num_next_calls: 0, break_last_token: false, }, + num_bump_calls: 0, unmatched_angle_bracket_count: 0, max_angle_bracket_count: 0, last_unexpected_token_span: None, @@ -1049,7 +1046,7 @@ impl<'a> Parser<'a> { // Note: destructuring here would give nicer code, but it was found in #96210 to be slower // than `.0`/`.1` access. let mut next = self.token_cursor.inlined_next(); - self.token_cursor.num_next_calls += 1; + self.num_bump_calls += 1; // We've retrieved an token from the underlying // cursor, so we no longer need to worry about // an unglued token. See `break_and_eat` for more details @@ -1446,7 +1443,7 @@ impl<'a> Parser<'a> { } pub fn approx_token_stream_pos(&self) -> usize { - self.token_cursor.num_next_calls + self.num_bump_calls } } From 6fc2c481e5a7daf1873ccc34926f1c0387979569 Mon Sep 17 00:00:00 2001 From: Nicholas Nethercote Date: Mon, 31 Jul 2023 16:36:27 +1000 Subject: [PATCH 5/8] Move `TokenCursor::break_last_token` into `Parser`. Similar to the last commit, it's more of a `Parser`-level concern than a `TokenCursor`-level concern. And the struct size reductions are nice. After this change, `TokenCursor` is as minimal as possible (two fields and two methods) which is nice. --- .../rustc_parse/src/parser/attr_wrapper.rs | 11 ++-- compiler/rustc_parse/src/parser/expr.rs | 2 +- compiler/rustc_parse/src/parser/mod.rs | 51 ++++++++----------- 3 files changed, 25 insertions(+), 39 deletions(-) diff --git a/compiler/rustc_parse/src/parser/attr_wrapper.rs b/compiler/rustc_parse/src/parser/attr_wrapper.rs index 10c2066ef6c22..5d6c574baa612 100644 --- a/compiler/rustc_parse/src/parser/attr_wrapper.rs +++ b/compiler/rustc_parse/src/parser/attr_wrapper.rs @@ -301,7 +301,7 @@ impl<'a> Parser<'a> { // then extend the range of captured tokens to include it, since the parser // was not actually bumped past it. When the `LazyAttrTokenStream` gets converted // into an `AttrTokenStream`, we will create the proper token. - if self.token_cursor.break_last_token { + if self.break_last_token { assert!(!captured_trailing, "Cannot set break_last_token and have trailing token"); end_pos += 1; } @@ -331,7 +331,7 @@ impl<'a> Parser<'a> { start_token, num_calls, cursor_snapshot, - break_last_token: self.token_cursor.break_last_token, + break_last_token: self.break_last_token, replace_ranges, }); @@ -362,10 +362,7 @@ impl<'a> Parser<'a> { let start_pos = if has_outer_attrs { attrs.start_pos } else { start_pos }; let new_tokens = vec![(FlatToken::AttrTarget(attr_data), Spacing::Alone)]; - assert!( - !self.token_cursor.break_last_token, - "Should not have unglued last token with cfg attr" - ); + assert!(!self.break_last_token, "Should not have unglued last token with cfg attr"); let range: Range = (start_pos.try_into().unwrap())..(end_pos.try_into().unwrap()); self.capture_state.replace_ranges.push((range, new_tokens)); self.capture_state.replace_ranges.extend(inner_attr_replace_ranges); @@ -463,6 +460,6 @@ mod size_asserts { use rustc_data_structures::static_assert_size; // tidy-alphabetical-start static_assert_size!(AttrWrapper, 16); - static_assert_size!(LazyAttrTokenStreamImpl, 112); + static_assert_size!(LazyAttrTokenStreamImpl, 104); // tidy-alphabetical-end } diff --git a/compiler/rustc_parse/src/parser/expr.rs b/compiler/rustc_parse/src/parser/expr.rs index 55f857aa31cda..975dafecb1088 100644 --- a/compiler/rustc_parse/src/parser/expr.rs +++ b/compiler/rustc_parse/src/parser/expr.rs @@ -1167,7 +1167,7 @@ impl<'a> Parser<'a> { DestructuredFloat::TrailingDot(sym, sym_span, dot_span) => { assert!(suffix.is_none()); // Analogous to `Self::break_and_eat` - self.token_cursor.break_last_token = true; + self.break_last_token = true; // This might work, in cases like `1. 2`, and might not, // in cases like `offset_of!(Ty, 1.)`. It depends on what comes // after the float-like token, and therefore we have to make diff --git a/compiler/rustc_parse/src/parser/mod.rs b/compiler/rustc_parse/src/parser/mod.rs index 2a1524f108a9e..27a8af630ce7e 100644 --- a/compiler/rustc_parse/src/parser/mod.rs +++ b/compiler/rustc_parse/src/parser/mod.rs @@ -138,6 +138,21 @@ pub struct Parser<'a> { token_cursor: TokenCursor, // The number of calls to `bump`, i.e. the position in the token stream. num_bump_calls: usize, + // During parsing we may sometimes need to 'unglue' a glued token into two + // component tokens (e.g. '>>' into '>' and '>), so the parser can consume + // them one at a time. This process bypasses the normal capturing mechanism + // (e.g. `num_bump_calls` will not be incremented), since the 'unglued' + // tokens due not exist in the original `TokenStream`. + // + // If we end up consuming both unglued tokens, this is not an issue. We'll + // end up capturing the single 'glued' token. + // + // However, sometimes we may want to capture just the first 'unglued' + // token. For example, capturing the `Vec` in `Option>` + // requires us to unglue the trailing `>>` token. The `break_last_token` + // field is used to track this token. It gets appended to the captured + // stream when we evaluate a `LazyAttrTokenStream`. + break_last_token: bool, /// This field is used to keep track of how many left angle brackets we have seen. This is /// required in order to detect extra leading left angle brackets (`<` characters) and error /// appropriately. @@ -161,7 +176,7 @@ pub struct Parser<'a> { // This type is used a lot, e.g. it's cloned when matching many declarative macro rules with nonterminals. Make sure // it doesn't unintentionally get bigger. #[cfg(all(target_arch = "x86_64", target_pointer_width = "64"))] -rustc_data_structures::static_assert_size!(Parser<'_>, 272); +rustc_data_structures::static_assert_size!(Parser<'_>, 264); /// Stores span information about a closure. #[derive(Clone)] @@ -223,29 +238,6 @@ struct TokenCursor { // tokens are in `stack[n-1]`. `stack[0]` (when present) has no delimiters // because it's the outermost token stream which never has delimiters. stack: Vec<(TokenTreeCursor, Delimiter, DelimSpan)>, - - // During parsing, we may sometimes need to 'unglue' a - // glued token into two component tokens - // (e.g. '>>' into '>' and '>), so that the parser - // can consume them one at a time. This process - // bypasses the normal capturing mechanism - // (e.g. `num_next_calls` will not be incremented), - // since the 'unglued' tokens due not exist in - // the original `TokenStream`. - // - // If we end up consuming both unglued tokens, - // then this is not an issue - we'll end up - // capturing the single 'glued' token. - // - // However, in certain circumstances, we may - // want to capture just the first 'unglued' token. - // For example, capturing the `Vec` - // in `Option>` requires us to unglue - // the trailing `>>` token. The `break_last_token` - // field is used to track this token - it gets - // appended to the captured stream when - // we evaluate a `LazyAttrTokenStream`. - break_last_token: bool, } impl TokenCursor { @@ -396,12 +388,9 @@ impl<'a> Parser<'a> { capture_cfg: false, restrictions: Restrictions::empty(), expected_tokens: Vec::new(), - token_cursor: TokenCursor { - tree_cursor: stream.into_trees(), - stack: Vec::new(), - break_last_token: false, - }, + token_cursor: TokenCursor { tree_cursor: stream.into_trees(), stack: Vec::new() }, num_bump_calls: 0, + break_last_token: false, unmatched_angle_bracket_count: 0, max_angle_bracket_count: 0, last_unexpected_token_span: None, @@ -704,7 +693,7 @@ impl<'a> Parser<'a> { // If we consume any additional tokens, then this token // is not needed (we'll capture the entire 'glued' token), // and `bump` will set this field to `None` - self.token_cursor.break_last_token = true; + self.break_last_token = true; // Use the spacing of the glued token as the spacing // of the unglued second token. self.bump_with((Token::new(second, second_span), self.token_spacing)); @@ -1050,7 +1039,7 @@ impl<'a> Parser<'a> { // We've retrieved an token from the underlying // cursor, so we no longer need to worry about // an unglued token. See `break_and_eat` for more details - self.token_cursor.break_last_token = false; + self.break_last_token = false; if next.0.span.is_dummy() { // Tweak the location for better diagnostics, but keep syntactic context intact. let fallback_span = self.token.span; From ac64a53e17097f87de8913e372a0baa99e58e31e Mon Sep 17 00:00:00 2001 From: Nicholas Nethercote Date: Mon, 31 Jul 2023 16:57:35 +1000 Subject: [PATCH 6/8] Avoid an unnecessary local variable. --- compiler/rustc_parse/src/parser/mod.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/compiler/rustc_parse/src/parser/mod.rs b/compiler/rustc_parse/src/parser/mod.rs index 27a8af630ce7e..f0d2411a7a60f 100644 --- a/compiler/rustc_parse/src/parser/mod.rs +++ b/compiler/rustc_parse/src/parser/mod.rs @@ -1241,8 +1241,7 @@ impl<'a> Parser<'a> { match self.token.kind { token::OpenDelim(..) => { // Grab the tokens within the delimiters. - let tree_cursor = &self.token_cursor.tree_cursor; - let stream = tree_cursor.stream.clone(); + let stream = self.token_cursor.tree_cursor.stream.clone(); let (_, delim, span) = *self.token_cursor.stack.last().unwrap(); // Advance the token cursor through the entire delimited From ba294a816bd51526b8bf99cb21050f669b8397b0 Mon Sep 17 00:00:00 2001 From: Nicholas Nethercote Date: Wed, 2 Aug 2023 09:28:44 +1000 Subject: [PATCH 7/8] Fix an erroneous comment. The `None` variant has already been renamed `Invisible`. --- compiler/rustc_ast/src/token.rs | 2 -- 1 file changed, 2 deletions(-) diff --git a/compiler/rustc_ast/src/token.rs b/compiler/rustc_ast/src/token.rs index 4c920e84f8679..f4ad0efa42344 100644 --- a/compiler/rustc_ast/src/token.rs +++ b/compiler/rustc_ast/src/token.rs @@ -41,8 +41,6 @@ pub enum BinOpToken { /// Describes how a sequence of token trees is delimited. /// Cannot use `proc_macro::Delimiter` directly because this /// structure should implement some additional traits. -/// The `None` variant is also renamed to `Invisible` to be -/// less confusing and better convey the semantics. #[derive(Copy, Clone, Debug, PartialEq, Eq)] #[derive(Encodable, Decodable, Hash, HashStable_Generic)] pub enum Delimiter { From d75ee2a6bcfc2ec9b21c0bb1ffaf34ff6b7161f1 Mon Sep 17 00:00:00 2001 From: Nicholas Nethercote Date: Wed, 2 Aug 2023 09:56:26 +1000 Subject: [PATCH 8/8] Remove `MacDelimiter`. It's the same as `Delimiter`, minus the `Invisible` variant. I'm generally in favour of using types to make impossible states unrepresentable, but this one feels very low-value, and the conversions between the two types are annoying and confusing. Look at the change in `src/tools/rustfmt/src/expr.rs` for an example: the old code converted from `MacDelimiter` to `Delimiter` and back again, for no good reason. This suggests the author was confused about the types. --- compiler/rustc_ast/src/ast.rs | 32 ++----------------- compiler/rustc_ast/src/attr/mod.rs | 14 ++++---- compiler/rustc_ast_pretty/src/pprust/state.rs | 6 ++-- compiler/rustc_builtin_macros/src/assert.rs | 5 +-- .../src/assert/context.rs | 7 ++-- .../rustc_builtin_macros/src/edition_panic.rs | 3 +- compiler/rustc_expand/src/placeholders.rs | 3 +- compiler/rustc_parse/src/parser/expr.rs | 2 +- compiler/rustc_parse/src/parser/item.rs | 4 +-- compiler/rustc_parse/src/parser/mod.rs | 6 ++-- compiler/rustc_parse/src/parser/stmt.rs | 3 +- compiler/rustc_parse/src/validate_attr.rs | 11 ++++--- src/tools/rustfmt/src/expr.rs | 8 ++--- 13 files changed, 37 insertions(+), 67 deletions(-) diff --git a/compiler/rustc_ast/src/ast.rs b/compiler/rustc_ast/src/ast.rs index dd4321bea1b2e..2a268c2da85c3 100644 --- a/compiler/rustc_ast/src/ast.rs +++ b/compiler/rustc_ast/src/ast.rs @@ -14,7 +14,7 @@ //! - [`Generics`], [`GenericParam`], [`WhereClause`]: Metadata associated with generic parameters. //! - [`EnumDef`] and [`Variant`]: Enum declaration. //! - [`MetaItemLit`] and [`LitKind`]: Literal expressions. -//! - [`MacroDef`], [`MacStmtStyle`], [`MacCall`], [`MacDelimiter`]: Macro definition and invocation. +//! - [`MacroDef`], [`MacStmtStyle`], [`MacCall`]: Macro definition and invocation. //! - [`Attribute`]: Metadata associated with item. //! - [`UnOp`], [`BinOp`], and [`BinOpKind`]: Unary and binary operators. @@ -1693,7 +1693,7 @@ where #[derive(Clone, Encodable, Decodable, Debug)] pub struct DelimArgs { pub dspan: DelimSpan, - pub delim: MacDelimiter, + pub delim: Delimiter, // Note: `Delimiter::Invisible` never occurs pub tokens: TokenStream, } @@ -1701,7 +1701,7 @@ impl DelimArgs { /// Whether a macro with these arguments needs a semicolon /// when used as a standalone item or statement. pub fn need_semicolon(&self) -> bool { - !matches!(self, DelimArgs { delim: MacDelimiter::Brace, .. }) + !matches!(self, DelimArgs { delim: Delimiter::Brace, .. }) } } @@ -1717,32 +1717,6 @@ where } } -#[derive(Copy, Clone, PartialEq, Eq, Encodable, Decodable, Debug, HashStable_Generic)] -pub enum MacDelimiter { - Parenthesis, - Bracket, - Brace, -} - -impl MacDelimiter { - pub fn to_token(self) -> Delimiter { - match self { - MacDelimiter::Parenthesis => Delimiter::Parenthesis, - MacDelimiter::Bracket => Delimiter::Bracket, - MacDelimiter::Brace => Delimiter::Brace, - } - } - - pub fn from_token(delim: Delimiter) -> Option { - match delim { - Delimiter::Parenthesis => Some(MacDelimiter::Parenthesis), - Delimiter::Bracket => Some(MacDelimiter::Bracket), - Delimiter::Brace => Some(MacDelimiter::Brace), - Delimiter::Invisible => None, - } - } -} - /// Represents a macro definition. #[derive(Clone, Encodable, Decodable, Debug, HashStable_Generic)] pub struct MacroDef { diff --git a/compiler/rustc_ast/src/attr/mod.rs b/compiler/rustc_ast/src/attr/mod.rs index 8b9bb1df5dc97..19a2b3017bc53 100644 --- a/compiler/rustc_ast/src/attr/mod.rs +++ b/compiler/rustc_ast/src/attr/mod.rs @@ -2,7 +2,7 @@ use crate::ast::{AttrArgs, AttrArgsEq, AttrId, AttrItem, AttrKind, AttrStyle, AttrVec, Attribute}; use crate::ast::{DelimArgs, Expr, ExprKind, LitKind, MetaItemLit}; -use crate::ast::{MacDelimiter, MetaItem, MetaItemKind, NestedMetaItem, NormalAttr}; +use crate::ast::{MetaItem, MetaItemKind, NestedMetaItem, NormalAttr}; use crate::ast::{Path, PathSegment, DUMMY_NODE_ID}; use crate::ptr::P; use crate::token::{self, CommentKind, Delimiter, Token}; @@ -196,7 +196,7 @@ impl AttrItem { fn meta_item_list(&self) -> Option> { match &self.args { - AttrArgs::Delimited(args) if args.delim == MacDelimiter::Parenthesis => { + AttrArgs::Delimited(args) if args.delim == Delimiter::Parenthesis => { MetaItemKind::list_from_tokens(args.tokens.clone()) } AttrArgs::Delimited(_) | AttrArgs::Eq(..) | AttrArgs::Empty => None, @@ -402,11 +402,9 @@ impl MetaItemKind { fn from_attr_args(args: &AttrArgs) -> Option { match args { AttrArgs::Empty => Some(MetaItemKind::Word), - AttrArgs::Delimited(DelimArgs { - dspan: _, - delim: MacDelimiter::Parenthesis, - tokens, - }) => MetaItemKind::list_from_tokens(tokens.clone()).map(MetaItemKind::List), + AttrArgs::Delimited(DelimArgs { dspan: _, delim: Delimiter::Parenthesis, tokens }) => { + MetaItemKind::list_from_tokens(tokens.clone()).map(MetaItemKind::List) + } AttrArgs::Delimited(..) => None, AttrArgs::Eq(_, AttrArgsEq::Ast(expr)) => match expr.kind { ExprKind::Lit(token_lit) => { @@ -578,7 +576,7 @@ pub fn mk_attr_nested_word( let path = Path::from_ident(outer_ident); let attr_args = AttrArgs::Delimited(DelimArgs { dspan: DelimSpan::from_single(span), - delim: MacDelimiter::Parenthesis, + delim: Delimiter::Parenthesis, tokens: inner_tokens, }); mk_attr(g, style, path, attr_args, span) diff --git a/compiler/rustc_ast_pretty/src/pprust/state.rs b/compiler/rustc_ast_pretty/src/pprust/state.rs index 59239b49eddd8..068b255e9f28b 100644 --- a/compiler/rustc_ast_pretty/src/pprust/state.rs +++ b/compiler/rustc_ast_pretty/src/pprust/state.rs @@ -476,7 +476,7 @@ pub trait PrintState<'a>: std::ops::Deref + std::ops::Dere Some(MacHeader::Path(&item.path)), false, None, - delim.to_token(), + *delim, tokens, true, span, @@ -640,7 +640,7 @@ pub trait PrintState<'a>: std::ops::Deref + std::ops::Dere Some(MacHeader::Keyword(kw)), has_bang, Some(*ident), - macro_def.body.delim.to_token(), + macro_def.body.delim, ¯o_def.body.tokens.clone(), true, sp, @@ -1240,7 +1240,7 @@ impl<'a> State<'a> { Some(MacHeader::Path(&m.path)), true, None, - m.args.delim.to_token(), + m.args.delim, &m.args.tokens.clone(), true, m.span(), diff --git a/compiler/rustc_builtin_macros/src/assert.rs b/compiler/rustc_builtin_macros/src/assert.rs index ab4ea9c8c2052..3e90ae6907f3a 100644 --- a/compiler/rustc_builtin_macros/src/assert.rs +++ b/compiler/rustc_builtin_macros/src/assert.rs @@ -4,8 +4,9 @@ use crate::edition_panic::use_panic_2021; use crate::errors; use rustc_ast::ptr::P; use rustc_ast::token; +use rustc_ast::token::Delimiter; use rustc_ast::tokenstream::{DelimSpan, TokenStream}; -use rustc_ast::{DelimArgs, Expr, ExprKind, MacCall, MacDelimiter, Path, PathSegment, UnOp}; +use rustc_ast::{DelimArgs, Expr, ExprKind, MacCall, Path, PathSegment, UnOp}; use rustc_ast_pretty::pprust; use rustc_errors::PResult; use rustc_expand::base::{DummyResult, ExtCtxt, MacEager, MacResult}; @@ -58,7 +59,7 @@ pub fn expand_assert<'cx>( path: panic_path(), args: P(DelimArgs { dspan: DelimSpan::from_single(call_site_span), - delim: MacDelimiter::Parenthesis, + delim: Delimiter::Parenthesis, tokens, }), })), diff --git a/compiler/rustc_builtin_macros/src/assert/context.rs b/compiler/rustc_builtin_macros/src/assert/context.rs index 902c1e40c7541..cbf115127cf3f 100644 --- a/compiler/rustc_builtin_macros/src/assert/context.rs +++ b/compiler/rustc_builtin_macros/src/assert/context.rs @@ -1,9 +1,10 @@ use rustc_ast::{ ptr::P, token, + token::Delimiter, tokenstream::{DelimSpan, TokenStream, TokenTree}, - BinOpKind, BorrowKind, DelimArgs, Expr, ExprKind, ItemKind, MacCall, MacDelimiter, MethodCall, - Mutability, Path, PathSegment, Stmt, StructRest, UnOp, UseTree, UseTreeKind, DUMMY_NODE_ID, + BinOpKind, BorrowKind, DelimArgs, Expr, ExprKind, ItemKind, MacCall, MethodCall, Mutability, + Path, PathSegment, Stmt, StructRest, UnOp, UseTree, UseTreeKind, DUMMY_NODE_ID, }; use rustc_ast_pretty::pprust; use rustc_data_structures::fx::FxHashSet; @@ -179,7 +180,7 @@ impl<'cx, 'a> Context<'cx, 'a> { path: panic_path, args: P(DelimArgs { dspan: DelimSpan::from_single(self.span), - delim: MacDelimiter::Parenthesis, + delim: Delimiter::Parenthesis, tokens: initial.into_iter().chain(captures).collect::(), }), })), diff --git a/compiler/rustc_builtin_macros/src/edition_panic.rs b/compiler/rustc_builtin_macros/src/edition_panic.rs index ef0db23ff2f2d..1e1dadab48067 100644 --- a/compiler/rustc_builtin_macros/src/edition_panic.rs +++ b/compiler/rustc_builtin_macros/src/edition_panic.rs @@ -1,4 +1,5 @@ use rustc_ast::ptr::P; +use rustc_ast::token::Delimiter; use rustc_ast::tokenstream::{DelimSpan, TokenStream}; use rustc_ast::*; use rustc_expand::base::*; @@ -60,7 +61,7 @@ fn expand<'cx>( }, args: P(DelimArgs { dspan: DelimSpan::from_single(sp), - delim: MacDelimiter::Parenthesis, + delim: Delimiter::Parenthesis, tokens: tts, }), })), diff --git a/compiler/rustc_expand/src/placeholders.rs b/compiler/rustc_expand/src/placeholders.rs index e9af688ee2b61..82cac229284d8 100644 --- a/compiler/rustc_expand/src/placeholders.rs +++ b/compiler/rustc_expand/src/placeholders.rs @@ -2,6 +2,7 @@ use crate::expand::{AstFragment, AstFragmentKind}; use rustc_ast as ast; use rustc_ast::mut_visit::*; use rustc_ast::ptr::P; +use rustc_ast::token::Delimiter; use rustc_data_structures::fx::FxHashMap; use rustc_span::source_map::DUMMY_SP; use rustc_span::symbol::Ident; @@ -18,7 +19,7 @@ pub fn placeholder( path: ast::Path { span: DUMMY_SP, segments: ThinVec::new(), tokens: None }, args: P(ast::DelimArgs { dspan: ast::tokenstream::DelimSpan::dummy(), - delim: ast::MacDelimiter::Parenthesis, + delim: Delimiter::Parenthesis, tokens: ast::tokenstream::TokenStream::new(Vec::new()), }), }) diff --git a/compiler/rustc_parse/src/parser/expr.rs b/compiler/rustc_parse/src/parser/expr.rs index 975dafecb1088..dc3b131e7f273 100644 --- a/compiler/rustc_parse/src/parser/expr.rs +++ b/compiler/rustc_parse/src/parser/expr.rs @@ -2599,7 +2599,7 @@ impl<'a> Parser<'a> { // Recover from missing expression in `for` loop if matches!(expr.kind, ExprKind::Block(..)) - && !matches!(self.token.kind, token::OpenDelim(token::Delimiter::Brace)) + && !matches!(self.token.kind, token::OpenDelim(Delimiter::Brace)) && self.may_recover() { self.sess diff --git a/compiler/rustc_parse/src/parser/item.rs b/compiler/rustc_parse/src/parser/item.rs index 1301ed3e3882e..e9cc858c8c53f 100644 --- a/compiler/rustc_parse/src/parser/item.rs +++ b/compiler/rustc_parse/src/parser/item.rs @@ -9,12 +9,12 @@ use rustc_ast::ptr::P; use rustc_ast::token::{self, Delimiter, TokenKind}; use rustc_ast::tokenstream::{DelimSpan, TokenStream, TokenTree}; use rustc_ast::util::case::Case; +use rustc_ast::MacCall; use rustc_ast::{self as ast, AttrVec, Attribute, DUMMY_NODE_ID}; use rustc_ast::{Async, Const, Defaultness, IsAuto, Mutability, Unsafe, UseTree, UseTreeKind}; use rustc_ast::{BindingAnnotation, Block, FnDecl, FnSig, Param, SelfKind}; use rustc_ast::{EnumDef, FieldDef, Generics, TraitRef, Ty, TyKind, Variant, VariantData}; use rustc_ast::{FnHeader, ForeignItem, Path, PathSegment, Visibility, VisibilityKind}; -use rustc_ast::{MacCall, MacDelimiter}; use rustc_ast_pretty::pprust; use rustc_errors::{ struct_span_err, Applicability, DiagnosticBuilder, ErrorGuaranteed, IntoDiagnostic, PResult, @@ -1968,7 +1968,7 @@ impl<'a> Parser<'a> { let arrow = TokenTree::token_alone(token::FatArrow, pspan.between(bspan)); // `=>` let tokens = TokenStream::new(vec![params, arrow, body]); let dspan = DelimSpan::from_pair(pspan.shrink_to_lo(), bspan.shrink_to_hi()); - P(DelimArgs { dspan, delim: MacDelimiter::Brace, tokens }) + P(DelimArgs { dspan, delim: Delimiter::Brace, tokens }) } else { return self.unexpected(); }; diff --git a/compiler/rustc_parse/src/parser/mod.rs b/compiler/rustc_parse/src/parser/mod.rs index f0d2411a7a60f..c5b46b809b1b1 100644 --- a/compiler/rustc_parse/src/parser/mod.rs +++ b/compiler/rustc_parse/src/parser/mod.rs @@ -25,7 +25,7 @@ use rustc_ast::util::case::Case; use rustc_ast::AttrId; use rustc_ast::DUMMY_NODE_ID; use rustc_ast::{self as ast, AnonConst, Const, DelimArgs, Extern}; -use rustc_ast::{Async, AttrArgs, AttrArgsEq, Expr, ExprKind, MacDelimiter, Mutability, StrLit}; +use rustc_ast::{Async, AttrArgs, AttrArgsEq, Expr, ExprKind, Mutability, StrLit}; use rustc_ast::{HasAttrs, HasTokens, Unsafe, Visibility, VisibilityKind}; use rustc_ast_pretty::pprust; use rustc_data_structures::fx::FxHashMap; @@ -1216,12 +1216,10 @@ impl<'a> Parser<'a> { || self.check(&token::OpenDelim(Delimiter::Brace)); delimited.then(|| { - // We've confirmed above that there is a delimiter so unwrapping is OK. let TokenTree::Delimited(dspan, delim, tokens) = self.parse_token_tree() else { unreachable!() }; - - DelimArgs { dspan, delim: MacDelimiter::from_token(delim).unwrap(), tokens } + DelimArgs { dspan, delim, tokens } }) } diff --git a/compiler/rustc_parse/src/parser/stmt.rs b/compiler/rustc_parse/src/parser/stmt.rs index 1cdf2efa76445..12c267351b9aa 100644 --- a/compiler/rustc_parse/src/parser/stmt.rs +++ b/compiler/rustc_parse/src/parser/stmt.rs @@ -193,10 +193,9 @@ impl<'a> Parser<'a> { /// At this point, the `!` token after the path has already been eaten. fn parse_stmt_mac(&mut self, lo: Span, attrs: AttrVec, path: ast::Path) -> PResult<'a, Stmt> { let args = self.parse_delim_args()?; - let delim = args.delim.to_token(); let hi = self.prev_token.span; - let style = match delim { + let style = match args.delim { Delimiter::Brace => MacStmtStyle::Braces, _ => MacStmtStyle::NoBraces, }; diff --git a/compiler/rustc_parse/src/validate_attr.rs b/compiler/rustc_parse/src/validate_attr.rs index 2011083019c8d..f739659822047 100644 --- a/compiler/rustc_parse/src/validate_attr.rs +++ b/compiler/rustc_parse/src/validate_attr.rs @@ -2,9 +2,10 @@ use crate::{errors, parse_in}; +use rustc_ast::token::Delimiter; use rustc_ast::tokenstream::DelimSpan; use rustc_ast::MetaItemKind; -use rustc_ast::{self as ast, AttrArgs, AttrArgsEq, Attribute, DelimArgs, MacDelimiter, MetaItem}; +use rustc_ast::{self as ast, AttrArgs, AttrArgsEq, Attribute, DelimArgs, MetaItem}; use rustc_ast_pretty::pprust; use rustc_errors::{Applicability, FatalError, PResult}; use rustc_feature::{AttributeTemplate, BuiltinAttribute, BUILTIN_ATTRIBUTE_MAP}; @@ -84,8 +85,8 @@ pub fn parse_meta<'a>(sess: &'a ParseSess, attr: &Attribute) -> PResult<'a, Meta }) } -pub fn check_meta_bad_delim(sess: &ParseSess, span: DelimSpan, delim: MacDelimiter) { - if let ast::MacDelimiter::Parenthesis = delim { +pub fn check_meta_bad_delim(sess: &ParseSess, span: DelimSpan, delim: Delimiter) { + if let Delimiter::Parenthesis = delim { return; } sess.emit_err(errors::MetaBadDelim { @@ -94,8 +95,8 @@ pub fn check_meta_bad_delim(sess: &ParseSess, span: DelimSpan, delim: MacDelimit }); } -pub fn check_cfg_attr_bad_delim(sess: &ParseSess, span: DelimSpan, delim: MacDelimiter) { - if let ast::MacDelimiter::Parenthesis = delim { +pub fn check_cfg_attr_bad_delim(sess: &ParseSess, span: DelimSpan, delim: Delimiter) { + if let Delimiter::Parenthesis = delim { return; } sess.emit_err(errors::CfgAttrBadDelim { diff --git a/src/tools/rustfmt/src/expr.rs b/src/tools/rustfmt/src/expr.rs index 5b1b4fbd491c3..739afb4e0ac03 100644 --- a/src/tools/rustfmt/src/expr.rs +++ b/src/tools/rustfmt/src/expr.rs @@ -1382,12 +1382,8 @@ pub(crate) fn can_be_overflowed_expr( || (context.use_block_indent() && args_len == 1) } ast::ExprKind::MacCall(ref mac) => { - match ( - rustc_ast::ast::MacDelimiter::from_token(mac.args.delim.to_token()), - context.config.overflow_delimited_expr(), - ) { - (Some(ast::MacDelimiter::Bracket), true) - | (Some(ast::MacDelimiter::Brace), true) => true, + match (mac.args.delim, context.config.overflow_delimited_expr()) { + (Delimiter::Bracket, true) | (Delimiter::Brace, true) => true, _ => context.use_block_indent() && args_len == 1, } }