Remove token::Lit from ast::Lit.

`token::Lit` contains a `kind` field that indicates what kind of literal
it is. `ast::Lit` currently wraps a `token::Lit` but also has its own
`kind` field. Which means that `ast::Lit` encodes the literal kind in
two different ways.

This commit changes `ast::Lit` so it no longer wraps `token::Lit`. It
now contains the `symbol` and `suffix` fields from `token::Lit`, but not
the `kind` field, eliminating the redundancy. This requires extending
`ast::LitKind::ByteStr` to include a `StrStyle` to properly distinguish
between cooked and raw byte string literals at the `ast::Lit` level.

Author: nnethercote

compiler/rustc_ast/src/ast.rs

@@ -23,7 +23,7 @@ pub use GenericArgs::*;
 pub use UnsafeSource::*;
 
 use crate::ptr::P;
-use crate::token::{self, CommentKind, Delimiter};
+use crate::token::{CommentKind, Delimiter};
 use crate::tokenstream::{DelimSpan, LazyTokenStream, TokenStream};
 use rustc_data_structures::stable_hasher::{HashStable, StableHasher};
 use rustc_data_structures::stack::ensure_sufficient_stack;
@@ -1719,15 +1719,49 @@ pub enum StrStyle {
 /// An AST literal.
 #[derive(Clone, Encodable, Decodable, Debug, HashStable_Generic)]
 pub struct Lit {
-    /// The original literal token as written in source code.
-    pub token_lit: token::Lit,
+    /// The original literal as written in the source code.
+    pub symbol: Symbol,
+    /// The original suffix as written in the source code.
+    pub suffix: Option<Symbol>,
     /// The "semantic" representation of the literal lowered from the original tokens.
     /// Strings are unescaped, hexadecimal forms are eliminated, etc.
-    /// FIXME: Remove this and only create the semantic representation during lowering to HIR.
     pub kind: LitKind,
     pub span: Span,
 }
 
+impl fmt::Display for Lit {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        let Lit { kind, symbol, suffix, .. } = self;
+        match kind {
+            LitKind::Byte(_) => write!(f, "b'{}'", symbol)?,
+            LitKind::Char(_) => write!(f, "'{}'", symbol)?,
+            LitKind::Str(_, StrStyle::Cooked) => write!(f, "\"{}\"", symbol)?,
+            LitKind::Str(_, StrStyle::Raw(n)) => write!(
+                f,
+                "r{delim}\"{string}\"{delim}",
+                delim = "#".repeat(*n as usize),
+                string = symbol
+            )?,
+            LitKind::ByteStr(_, StrStyle::Cooked) => write!(f, "b\"{}\"", symbol)?,
+            LitKind::ByteStr(_, StrStyle::Raw(n)) => write!(
+                f,
+                "br{delim}\"{string}\"{delim}",
+                delim = "#".repeat(*n as usize),
+                string = symbol
+            )?,
+            LitKind::Int(..) | LitKind::Float(..) | LitKind::Bool(..) | LitKind::Err => {
+                write!(f, "{}", symbol)?
+            }
+        }
+
+        if let Some(suffix) = suffix {
+            write!(f, "{}", suffix)?;
+        }
+
+        Ok(())
+    }
+}
+
 /// Same as `Lit`, but restricted to string literals.
 #[derive(Clone, Copy, Encodable, Decodable, Debug)]
 pub struct StrLit {
@@ -1737,18 +1771,14 @@ pub struct StrLit {
     pub suffix: Option<Symbol>,
     pub span: Span,
     /// The unescaped "semantic" representation of the literal lowered from the original token.
-    /// FIXME: Remove this and only create the semantic representation during lowering to HIR.
     pub symbol_unescaped: Symbol,
 }
 
 impl StrLit {
     pub fn as_lit(&self) -> Lit {
-        let token_kind = match self.style {
-            StrStyle::Cooked => token::Str,
-            StrStyle::Raw(n) => token::StrRaw(n),
-        };
         Lit {
-            token_lit: token::Lit::new(token_kind, self.symbol, self.suffix),
+            symbol: self.symbol,
+            suffix: self.suffix,
             span: self.span,
             kind: LitKind::Str(self.symbol_unescaped, self.style),
         }
@@ -1785,8 +1815,9 @@ pub enum LitKind {
     /// A string literal (`"foo"`). The symbol is unescaped, and so may differ
     /// from the original token's symbol.
     Str(Symbol, StrStyle),
-    /// A byte string (`b"foo"`).
-    ByteStr(Lrc<[u8]>),
+    /// A byte string (`b"foo"`). Not stored as a symbol because it might be
+    /// non-utf8, and symbols only allow utf8 strings.
+    ByteStr(Lrc<[u8]>, StrStyle),
     /// A byte char (`b'f'`).
     Byte(u8),
     /// A character literal (`'a'`).
@@ -1810,7 +1841,7 @@ impl LitKind {
 
     /// Returns `true` if this literal is byte literal string.
     pub fn is_bytestr(&self) -> bool {
-        matches!(self, LitKind::ByteStr(_))
+        matches!(self, LitKind::ByteStr(..))
     }
 
     /// Returns `true` if this is a numeric literal.
@@ -3084,7 +3115,7 @@ mod size_asserts {
     static_assert_size!(Impl, 200);
     static_assert_size!(Item, 184);
     static_assert_size!(ItemKind, 112);
-    static_assert_size!(Lit, 48);
+    static_assert_size!(Lit, 40);
     static_assert_size!(LitKind, 24);
     static_assert_size!(Local, 72);
     static_assert_size!(Param, 40);

compiler/rustc_ast/src/util/literal.rs

@@ -1,13 +1,11 @@
 //! Code related to parsing literals.
 
-use crate::ast::{self, Lit, LitKind};
+use crate::ast::{self, Lit, LitKind, StrStyle};
 use crate::token::{self, Token};
-
 use rustc_lexer::unescape::{unescape_byte, unescape_char};
 use rustc_lexer::unescape::{unescape_byte_literal, unescape_literal, Mode};
 use rustc_span::symbol::{kw, sym, Symbol};
 use rustc_span::Span;
-
 use std::ascii;
 
 pub enum LitError {
@@ -120,9 +118,9 @@ impl LitKind {
                     }
                 });
                 error?;
-                LitKind::ByteStr(buf.into())
+                LitKind::ByteStr(buf.into(), StrStyle::Cooked)
             }
-            token::ByteStrRaw(_) => {
+            token::ByteStrRaw(n) => {
                 let s = symbol.as_str();
                 let bytes = if s.contains('\r') {
                     let mut buf = Vec::with_capacity(s.len());
@@ -143,7 +141,7 @@ impl LitKind {
                     symbol.to_string().into_bytes()
                 };
 
-                LitKind::ByteStr(bytes.into())
+                LitKind::ByteStr(bytes.into(), StrStyle::Raw(n))
             }
             token::Err => LitKind::Err,
         })
@@ -162,15 +160,19 @@ impl LitKind {
                 (token::Str, symbol, None)
             }
             LitKind::Str(symbol, ast::StrStyle::Raw(n)) => (token::StrRaw(n), symbol, None),
-            LitKind::ByteStr(ref bytes) => {
+            LitKind::ByteStr(ref bytes, ast::StrStyle::Cooked) => {
                 let string = bytes
                     .iter()
-                    .cloned()
+                    .copied()
                     .flat_map(ascii::escape_default)
                     .map(Into::<char>::into)
                     .collect::<String>();
                 (token::ByteStr, Symbol::intern(&string), None)
             }
+            LitKind::ByteStr(ref bytes, ast::StrStyle::Raw(n)) => {
+                let string = bytes.iter().copied().map(Into::<char>::into).collect::<String>();
+                (token::ByteStrRaw(n), Symbol::intern(&string), None)
+            }
             LitKind::Byte(byte) => {
                 let string: String = ascii::escape_default(byte).map(Into::<char>::into).collect();
                 (token::Byte, Symbol::intern(&string), None)
@@ -210,7 +212,12 @@ impl LitKind {
 impl Lit {
     /// Converts literal token into an AST literal.
     pub fn from_token_lit(token_lit: token::Lit, span: Span) -> Result<Lit, LitError> {
-        Ok(Lit { token_lit, kind: LitKind::from_token_lit(token_lit)?, span })
+        Ok(Lit {
+            symbol: token_lit.symbol,
+            suffix: token_lit.suffix,
+            kind: LitKind::from_token_lit(token_lit)?,
+            span,
+        })
     }
 
     /// Converts arbitrary token into an AST literal.
@@ -240,16 +247,27 @@ impl Lit {
     /// This function is used when the original token doesn't exist (e.g. the literal is created
     /// by an AST-based macro) or unavailable (e.g. from HIR pretty-printing).
     pub fn from_lit_kind(kind: LitKind, span: Span) -> Lit {
-        Lit { token_lit: kind.to_token_lit(), kind, span }
+        let token_lit = kind.to_token_lit();
+        Lit { symbol: token_lit.symbol, suffix: token_lit.suffix, kind, span }
     }
 
     /// Losslessly convert an AST literal into a token.
     pub fn to_token(&self) -> Token {
-        let kind = match self.token_lit.kind {
-            token::Bool => token::Ident(self.token_lit.symbol, false),
-            _ => token::Literal(self.token_lit),
+        let symbol = self.symbol;
+        let suffix = self.suffix;
+        let kind = match self.kind {
+            LitKind::Bool(_) => return Token::new(token::Ident(symbol, false), self.span),
+            LitKind::Str(_, StrStyle::Cooked) => token::LitKind::Str,
+            LitKind::Str(_, StrStyle::Raw(n)) => token::LitKind::StrRaw(n),
+            LitKind::ByteStr(_, StrStyle::Cooked) => token::LitKind::ByteStr,
+            LitKind::ByteStr(_, StrStyle::Raw(n)) => token::LitKind::ByteStrRaw(n),
+            LitKind::Byte(_) => token::LitKind::Byte,
+            LitKind::Char(_) => token::LitKind::Char,
+            LitKind::Int(..) => token::LitKind::Integer,
+            LitKind::Float(..) => token::LitKind::Float,
+            LitKind::Err => token::LitKind::Err,
         };
-        Token::new(kind, self.span)
+        Token::new(token::Literal(token::Lit { kind, symbol, suffix }), self.span)
     }
 }
 

compiler/rustc_ast_lowering/src/lib.rs

@@ -925,11 +925,7 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
                 let lit = if let ExprKind::Lit(lit) = &expr.kind {
                     lit.clone()
                 } else {
-                    Lit {
-                        token_lit: token::Lit::new(token::LitKind::Err, kw::Empty, None),
-                        kind: LitKind::Err,
-                        span: DUMMY_SP,
-                    }
+                    Lit { symbol: kw::Empty, suffix: None, kind: LitKind::Err, span: DUMMY_SP }
                 };
                 MacArgs::Eq(eq_span, MacArgsEq::Hir(lit))
             }

compiler/rustc_ast_pretty/src/pprust/state.rs

@@ -372,7 +372,7 @@ pub trait PrintState<'a>: std::ops::Deref<Target = pp::Printer> + std::ops::Dere
 
     fn print_literal(&mut self, lit: &ast::Lit) {
         self.maybe_print_comment(lit.span.lo());
-        self.word(lit.token_lit.to_string())
+        self.word(lit.to_string())
     }
 
     fn print_string(&mut self, st: &str, style: ast::StrStyle) {

compiler/rustc_builtin_macros/src/concat_bytes.rs

@@ -90,7 +90,7 @@ fn handle_array_element(
             ) if val <= u8::MAX.into() => Some(val as u8),
 
             ast::LitKind::Byte(val) => Some(val),
-            ast::LitKind::ByteStr(_) => {
+            ast::LitKind::ByteStr(..) => {
                 if !*has_errors {
                     cx.struct_span_err(expr.span, "cannot concatenate doubly nested array")
                         .note("byte strings are treated as arrays of bytes")
@@ -157,7 +157,7 @@ pub fn expand_concat_bytes(
                 ast::LitKind::Byte(val) => {
                     accumulator.push(val);
                 }
-                ast::LitKind::ByteStr(ref bytes) => {
+                ast::LitKind::ByteStr(ref bytes, _) => {
                     accumulator.extend_from_slice(&bytes);
                 }
                 _ => {

compiler/rustc_builtin_macros/src/derive.rs

@@ -1,7 +1,7 @@
 use crate::cfg_eval::cfg_eval;
 
 use rustc_ast as ast;
-use rustc_ast::{attr, token, GenericParamKind, ItemKind, MetaItemKind, NestedMetaItem, StmtKind};
+use rustc_ast::{attr, GenericParamKind, ItemKind, MetaItemKind, NestedMetaItem, StmtKind};
 use rustc_errors::{struct_span_err, Applicability};
 use rustc_expand::base::{Annotatable, ExpandResult, ExtCtxt, Indeterminate, MultiItemModifier};
 use rustc_feature::AttributeTemplate;
@@ -126,9 +126,11 @@ fn report_bad_target(sess: &Session, item: &Annotatable, span: Span) -> bool {
 }
 
 fn report_unexpected_literal(sess: &Session, lit: &ast::Lit) {
-    let help_msg = match lit.token_lit.kind {
-        token::Str if rustc_lexer::is_ident(lit.token_lit.symbol.as_str()) => {
-            format!("try using `#[derive({})]`", lit.token_lit.symbol)
+    let help_msg = match lit.kind {
+        ast::LitKind::Str(_, ast::StrStyle::Cooked)
+            if rustc_lexer::is_ident(lit.symbol.as_str()) =>
+        {
+            format!("try using `#[derive({})]`", lit.symbol)
         }
         _ => "for example, write `#[derive(Debug)]` for `Debug`".to_string(),
     };

compiler/rustc_expand/src/base.rs

@@ -1217,7 +1217,7 @@ pub fn expr_to_spanned_string<'a>(
     Err(match expr.kind {
         ast::ExprKind::Lit(ref l) => match l.kind {
             ast::LitKind::Str(s, style) => return Ok((s, style, expr.span)),
-            ast::LitKind::ByteStr(_) => {
+            ast::LitKind::ByteStr(..) => {
                 let mut err = cx.struct_span_err(l.span, err_msg);
                 err.span_suggestion(
                     expr.span.shrink_to_lo(),

compiler/rustc_expand/src/build.rs

@@ -358,7 +358,7 @@ impl<'a> ExtCtxt<'a> {
     }
 
     pub fn expr_byte_str(&self, sp: Span, bytes: Vec<u8>) -> P<ast::Expr> {
-        self.expr_lit(sp, ast::LitKind::ByteStr(Lrc::from(bytes)))
+        self.expr_lit(sp, ast::LitKind::ByteStr(Lrc::from(bytes), ast::StrStyle::Cooked))
     }
 
     /// `[expr1, expr2, ...]`

compiler/rustc_expand/src/proc_macro_server.rs

@@ -499,26 +499,19 @@ impl server::TokenStream for Rustc<'_, '_> {
         // We don't use `TokenStream::from_ast` as the tokenstream currently cannot
         // be recovered in the general case.
         match &expr.kind {
-            ast::ExprKind::Lit(l) if l.token_lit.kind == token::Bool => {
-                Ok(tokenstream::TokenStream::token_alone(
-                    token::Ident(l.token_lit.symbol, false),
-                    l.span,
-                ))
-            }
             ast::ExprKind::Lit(l) => {
-                Ok(tokenstream::TokenStream::token_alone(token::Literal(l.token_lit), l.span))
+                let token = l.to_token();
+                Ok(tokenstream::TokenStream::token_alone(token.kind, token.span))
             }
             ast::ExprKind::Unary(ast::UnOp::Neg, e) => match &e.kind {
-                ast::ExprKind::Lit(l) => match l.token_lit {
-                    token::Lit { kind: token::Integer | token::Float, .. } => {
+                ast::ExprKind::Lit(l) => match l.kind {
+                    ast::LitKind::Int(..) | ast::LitKind::Float(..) => {
+                        let token = l.to_token();
                         Ok(Self::TokenStream::from_iter([
                             // FIXME: The span of the `-` token is lost when
                             // parsing, so we cannot faithfully recover it here.
                             tokenstream::TokenTree::token_alone(token::BinOp(token::Minus), e.span),
-                            tokenstream::TokenTree::token_alone(
-                                token::Literal(l.token_lit),
-                                l.span,
-                            ),
+                            tokenstream::TokenTree::token_alone(token.kind, token.span),
                         ]))
                     }
                     _ => Err(()),

compiler/rustc_lint/src/hidden_unicode_codepoints.rs

@@ -120,8 +120,8 @@ impl EarlyLintPass for HiddenUnicodeCodepoints {
     fn check_expr(&mut self, cx: &EarlyContext<'_>, expr: &ast::Expr) {
         // byte strings are already handled well enough by `EscapeError::NonAsciiCharInByteString`
         let (text, span, padding) = match &expr.kind {
-            ast::ExprKind::Lit(ast::Lit { token_lit, kind, span }) => {
-                let text = token_lit.symbol;
+            ast::ExprKind::Lit(ast::Lit { symbol, kind, span, .. }) => {
+                let text = symbol;
                 if !contains_text_flow_control_chars(text.as_str()) {
                     return;
                 }
@@ -136,6 +136,6 @@ impl EarlyLintPass for HiddenUnicodeCodepoints {
             }
             _ => return,
         };
-        self.lint_text_direction_codepoint(cx, text, *span, padding, true, "literal");
+        self.lint_text_direction_codepoint(cx, *text, *span, padding, true, "literal");
     }
 }