Make retags an implicit part of typed copies

compiler/rustc_borrowck/src/dataflow.rs

@@ -578,7 +578,6 @@ impl<'tcx> rustc_mir_dataflow::Analysis<'tcx> for Borrows<'_, 'tcx> {
             mir::StatementKind::FakeRead(..)
             | mir::StatementKind::SetDiscriminant { .. }
             | mir::StatementKind::StorageLive(..)
-            | mir::StatementKind::Retag { .. }
             | mir::StatementKind::PlaceMention(..)
             | mir::StatementKind::AscribeUserType(..)
             | mir::StatementKind::Coverage(..)

compiler/rustc_borrowck/src/diagnostics/conflict_errors.rs

@@ -4238,7 +4238,7 @@ impl<'infcx, 'tcx> MirBorrowckCtxt<'_, 'infcx, 'tcx> {
                     // Otherwise, look at other types of assignment.
                     let assigned_from = match rvalue {
                         Rvalue::Ref(_, _, assigned_from) => assigned_from,
-                        Rvalue::Use(operand) => match operand {
+                        Rvalue::Use(operand, _) => match operand {
                             Operand::Copy(assigned_from) | Operand::Move(assigned_from) => {
                                 assigned_from
                             }

compiler/rustc_borrowck/src/diagnostics/explain_borrow.rs

@@ -875,7 +875,7 @@ impl<'tcx> MirBorrowckCtxt<'_, '_, 'tcx> {
                     match rvalue {
                         // If we see a use, we should check whether it is our data, and if so
                         // update the place that we're looking for to that new place.
-                        Rvalue::Use(operand) => match operand {
+                        Rvalue::Use(operand, _) => match operand {
                             Operand::Copy(place) | Operand::Move(place) => {
                                 if let Some(from) = place.as_local() {
                                     if from == target {

compiler/rustc_borrowck/src/diagnostics/mod.rs

@@ -243,7 +243,7 @@ impl<'infcx, 'tcx> MirBorrowckCtxt<'_, 'infcx, 'tcx> {
         let mut target = place.local_or_deref_local();
         for stmt in &self.body[location.block].statements[location.statement_index..] {
             debug!("add_moved_or_invoked_closure_note: stmt={:?} target={:?}", stmt, target);
-            if let StatementKind::Assign(box (into, Rvalue::Use(from))) = &stmt.kind {
+            if let StatementKind::Assign(box (into, Rvalue::Use(from, _))) = &stmt.kind {
                 debug!("add_fnonce_closure_note: into={:?} from={:?}", into, from);
                 match from {
                     Operand::Copy(place) | Operand::Move(place)

compiler/rustc_borrowck/src/diagnostics/move_errors.rs

@@ -120,7 +120,7 @@ impl<'infcx, 'tcx> MirBorrowckCtxt<'_, 'infcx, 'tcx> {
         // to a user variable is when initializing it.
         // If that ever stops being the case, then the ever initialized
         // flow could be used.
-        if let Some(StatementKind::Assign(box (place, Rvalue::Use(Operand::Move(move_from))))) =
+        if let Some(StatementKind::Assign(box (place, Rvalue::Use(Operand::Move(move_from), _)))) =
             self.body.basic_blocks[location.block]
                 .statements
                 .get(location.statement_index)

compiler/rustc_borrowck/src/diagnostics/mutability_errors.rs

@@ -1282,7 +1282,10 @@ impl<'infcx, 'tcx> MirBorrowckCtxt<'_, 'infcx, 'tcx> {
                 if let Some(mir::Statement {
                     source_info: _,
                     kind:
-                        mir::StatementKind::Assign(box (_, mir::Rvalue::Use(mir::Operand::Copy(place)))),
+                        mir::StatementKind::Assign(box (
+                            _,
+                            mir::Rvalue::Use(mir::Operand::Copy(place), _),
+                        )),
                     ..
                 }) = first_assignment_stmt
                 {

compiler/rustc_borrowck/src/lib.rs

@@ -853,7 +853,6 @@ impl<'a, 'tcx> ResultsVisitor<'tcx, Borrowck<'a, 'tcx>> for MirBorrowckCtxt<'a,
                 );
             }
             StatementKind::Nop
-            | StatementKind::Retag { .. }
             | StatementKind::SetDiscriminant { .. } => {
                 bug!("Statement not allowed in this MIR phase")
             }
@@ -1540,7 +1539,7 @@ impl<'a, 'tcx> MirBorrowckCtxt<'a, '_, 'tcx> {
 
             Rvalue::ThreadLocalRef(_) => {}
 
-            Rvalue::Use(operand)
+            Rvalue::Use(operand, _)
             | Rvalue::Repeat(operand, _)
             | Rvalue::UnaryOp(_ /*un_op*/, operand)
             | Rvalue::Cast(_ /*cast_kind*/, operand, _ /*ty*/) => {
@@ -1693,7 +1692,7 @@ impl<'a, 'tcx> MirBorrowckCtxt<'a, '_, 'tcx> {
                             StatementKind::Assign(box (
                                 _,
                                 Rvalue::Ref(_, _, source)
-                                | Rvalue::Use(Operand::Copy(source) | Operand::Move(source)),
+                                | Rvalue::Use(Operand::Copy(source) | Operand::Move(source), _),
                             )) => {
                                 propagate_closure_used_mut_place(self, source);
                             }

compiler/rustc_borrowck/src/polonius/legacy/loan_invalidations.rs

@@ -83,7 +83,6 @@ impl<'a, 'tcx> Visitor<'tcx> for LoanInvalidationsGenerator<'a, 'tcx> {
             }
             StatementKind::ConstEvalCounter
             | StatementKind::Nop
-            | StatementKind::Retag { .. }
             | StatementKind::BackwardIncompatibleDropHint { .. }
             | StatementKind::SetDiscriminant { .. } => {
                 bug!("Statement not allowed in this MIR phase")
@@ -294,7 +293,7 @@ impl<'a, 'tcx> LoanInvalidationsGenerator<'a, 'tcx> {
 
             Rvalue::ThreadLocalRef(_) => {}
 
-            Rvalue::Use(operand)
+            Rvalue::Use(operand, _)
             | Rvalue::Repeat(operand, _)
             | Rvalue::UnaryOp(_ /*un_op*/, operand)
             | Rvalue::Cast(_ /*cast_kind*/, operand, _ /*ty*/) => {

compiler/rustc_borrowck/src/type_check/mod.rs

@@ -694,7 +694,6 @@ impl<'a, 'tcx> Visitor<'tcx> for TypeChecker<'a, 'tcx> {
             | StatementKind::FakeRead(..)
             | StatementKind::StorageLive(..)
             | StatementKind::StorageDead(..)
-            | StatementKind::Retag { .. }
             | StatementKind::Coverage(..)
             | StatementKind::ConstEvalCounter
             | StatementKind::PlaceMention(..)
@@ -1649,7 +1648,7 @@ impl<'a, 'tcx> Visitor<'tcx> for TypeChecker<'a, 'tcx> {
                 .unwrap();
             }
 
-            Rvalue::Use(_)
+            Rvalue::Use(_, _)
             | Rvalue::UnaryOp(_, _)
             | Rvalue::CopyForDeref(_)
             | Rvalue::BinaryOp(..)
@@ -2210,8 +2209,8 @@ impl<'a, 'tcx> TypeChecker<'a, 'tcx> {
     /// rvalue and will be unified with the inferred type.
     fn rvalue_user_ty(&self, rvalue: &Rvalue<'tcx>) -> Option<UserTypeAnnotationIndex> {
         match rvalue {
-            Rvalue::Use(_)
-            | Rvalue::ThreadLocalRef(_)
+            Rvalue::Use(..)
+            | Rvalue::ThreadLocalRef(..)
             | Rvalue::Repeat(..)
             | Rvalue::Ref(..)
             | Rvalue::RawPtr(..)

compiler/rustc_codegen_cranelift/src/base.rs

@@ -620,7 +620,7 @@ fn codegen_stmt<'tcx>(fx: &mut FunctionCx<'_, '_, 'tcx>, cur_block: Block, stmt:
             let lval = codegen_place(fx, to_place_and_rval.0);
             let dest_layout = lval.layout();
             match to_place_and_rval.1 {
-                Rvalue::Use(ref operand) => {
+                Rvalue::Use(ref operand, _) => {
                     let val = codegen_operand(fx, operand);
                     lval.write_cvalue(fx, val);
                 }
@@ -909,7 +909,6 @@ fn codegen_stmt<'tcx>(fx: &mut FunctionCx<'_, '_, 'tcx>, cur_block: Block, stmt:
         | StatementKind::ConstEvalCounter
         | StatementKind::Nop
         | StatementKind::FakeRead(..)
-        | StatementKind::Retag { .. }
         | StatementKind::PlaceMention(..)
         | StatementKind::BackwardIncompatibleDropHint { .. }
         | StatementKind::AscribeUserType(..) => {}

compiler/rustc_codegen_cranelift/src/constant.rs

@@ -592,7 +592,7 @@ pub(crate) fn mir_operand_get_const_val<'tcx>(
                                         };
                                     computed_scalar_int = Some(scalar_int);
                                 }
-                                Rvalue::Use(operand) => {
+                                Rvalue::Use(operand, _) => {
                                     computed_scalar_int = mir_operand_get_const_val(fx, operand)
                                 }
                                 _ => return None,
@@ -613,7 +613,6 @@ pub(crate) fn mir_operand_get_const_val<'tcx>(
                         | StatementKind::SetDiscriminant { .. }
                         | StatementKind::StorageLive(_)
                         | StatementKind::StorageDead(_)
-                        | StatementKind::Retag(_, _)
                         | StatementKind::AscribeUserType(_, _)
                         | StatementKind::PlaceMention(..)
                         | StatementKind::Coverage(_)

compiler/rustc_codegen_ssa/src/mir/mod.rs

@@ -366,9 +366,12 @@ fn optimize_use_clone<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
                 bb.terminator().source_info,
                 mir::StatementKind::Assign(Box::new((
                     *destination,
-                    mir::Rvalue::Use(mir::Operand::Copy(
-                        arg_place.project_deeper(&[mir::ProjectionElem::Deref], tcx),
-                    )),
+                    mir::Rvalue::Use(
+                        mir::Operand::Copy(
+                            arg_place.project_deeper(&[mir::ProjectionElem::Deref], tcx),
+                        ),
+                        mir::WithRetag::Yes,
+                    ),
                 ))),
             ));
 

compiler/rustc_codegen_ssa/src/mir/rvalue.rs

@@ -23,7 +23,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
         rvalue: &mir::Rvalue<'tcx>,
     ) {
         match *rvalue {
-            mir::Rvalue::Use(ref operand) => {
+            mir::Rvalue::Use(ref operand, _) => {
                 if let mir::Operand::Constant(const_op) = operand {
                     let val = self.eval_mir_constant(&const_op);
                     if val.all_bytes_uninit(self.cx.tcx()) {
@@ -650,7 +650,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                 };
                 OperandRef { val: OperandValue::Immediate(static_), layout, move_annotation: None }
             }
-            mir::Rvalue::Use(ref operand) => self.codegen_operand(bx, operand),
+            mir::Rvalue::Use(ref operand, _) => self.codegen_operand(bx, operand),
             mir::Rvalue::Repeat(ref elem, len_const) => {
                 // All arrays have `BackendRepr::Memory`, so only the ZST cases
                 // end up here. Anything else forces the destination local to be

compiler/rustc_codegen_ssa/src/mir/statement.rs

@@ -95,7 +95,6 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                 bx.memcpy(dst, align, src, align, bytes, crate::MemFlags::empty(), None);
             }
             mir::StatementKind::FakeRead(..)
-            | mir::StatementKind::Retag { .. }
             | mir::StatementKind::AscribeUserType(..)
             | mir::StatementKind::ConstEvalCounter
             | mir::StatementKind::PlaceMention(..)

compiler/rustc_const_eval/src/check_consts/check.rs

@@ -569,7 +569,7 @@ impl<'tcx> Visitor<'tcx> for Checker<'_, 'tcx> {
         match rvalue {
             Rvalue::ThreadLocalRef(_) => self.check_op(ops::ThreadLocalAccess),
 
-            Rvalue::Use(_)
+            Rvalue::Use(..)
             | Rvalue::CopyForDeref(..)
             | Rvalue::Repeat(..)
             | Rvalue::Discriminant(..) => {}
@@ -724,7 +724,6 @@ impl<'tcx> Visitor<'tcx> for Checker<'_, 'tcx> {
             | StatementKind::FakeRead(..)
             | StatementKind::StorageLive(_)
             | StatementKind::StorageDead(_)
-            | StatementKind::Retag { .. }
             | StatementKind::PlaceMention(..)
             | StatementKind::AscribeUserType(..)
             | StatementKind::Coverage(..)

compiler/rustc_const_eval/src/check_consts/qualifs.rs

@@ -234,7 +234,7 @@ where
 
         Rvalue::CopyForDeref(place) => in_place::<Q, _>(cx, in_local, place.as_ref()),
 
-        Rvalue::Use(operand)
+        Rvalue::Use(operand, _)
         | Rvalue::Repeat(operand, _)
         | Rvalue::UnaryOp(_, operand)
         | Rvalue::Cast(_, operand, _) => in_operand::<Q, _>(cx, in_local, operand),

compiler/rustc_const_eval/src/const_eval/machine.rs

@@ -1,6 +1,6 @@
 use std::borrow::{Borrow, Cow};
-use std::fmt;
 use std::hash::Hash;
+use std::{fmt, mem};
 
 use rustc_abi::{Align, FIRST_VARIANT, Size};
 use rustc_ast::Mutability;
@@ -21,8 +21,8 @@ use super::error::*;
 use crate::errors::{LongRunning, LongRunningWarn};
 use crate::interpret::{
     self, AllocId, AllocInit, AllocRange, ConstAllocation, CtfeProvenance, FnArg, Frame,
-    GlobalAlloc, ImmTy, InterpCx, InterpResult, OpTy, PlaceTy, Pointer, RangeSet, Scalar,
-    compile_time_machine, ensure_monomorphic_enough, err_inval, interp_ok, throw_exhaust,
+    GlobalAlloc, ImmTy, InterpCx, InterpResult, OpTy, PlaceTy, Pointer, RangeSet, RetagMode,
+    Scalar, compile_time_machine, ensure_monomorphic_enough, err_inval, interp_ok, throw_exhaust,
     throw_inval, throw_ub, throw_ub_format, throw_unsup, throw_unsup_format,
     type_implements_dyn_trait,
 };
@@ -67,6 +67,9 @@ pub struct CompileTimeMachine<'tcx> {
 
     /// A cache of "data range" computations for unions (i.e., the offsets of non-padding bytes).
     union_data_ranges: FxHashMap<Ty<'tcx>, RangeSet>,
+
+    /// The current retag mode.
+    retag_mode: RetagMode,
 }
 
 #[derive(Copy, Clone)]
@@ -102,6 +105,7 @@ impl<'tcx> CompileTimeMachine<'tcx> {
             check_alignment,
             static_root_ids: None,
             union_data_ranges: FxHashMap::default(),
+            retag_mode: RetagMode::Default,
         }
     }
 }
@@ -824,9 +828,12 @@ impl<'tcx> interpret::Machine<'tcx> for CompileTimeMachine<'tcx> {
 
     fn retag_ptr_value(
         ecx: &mut InterpCx<'tcx, Self>,
-        _kind: mir::RetagKind,
         val: &ImmTy<'tcx, CtfeProvenance>,
-    ) -> InterpResult<'tcx, ImmTy<'tcx, CtfeProvenance>> {
+        _ty: Ty<'tcx>,
+    ) -> InterpResult<'tcx, Option<ImmTy<'tcx, CtfeProvenance>>> {
+        if matches!(ecx.machine.retag_mode, RetagMode::None | RetagMode::Raw) {
+            return interp_ok(None);
+        }
         // If it's a frozen shared reference that's not already immutable, potentially make it immutable.
         // (Do nothing on `None` provenance, that cannot store immutability anyway.)
         if let ty::Ref(_, ty, mutbl) = val.layout.ty.kind()
@@ -850,12 +857,23 @@ impl<'tcx> interpret::Machine<'tcx> for CompileTimeMachine<'tcx> {
                 // even when there is interior mutability.)
                 place.map_provenance(CtfeProvenance::as_shared_ref)
             };
-            interp_ok(ImmTy::from_immediate(new_place.to_ref(ecx), val.layout))
+            interp_ok(Some(ImmTy::from_immediate(new_place.to_ref(ecx), val.layout)))
         } else {
-            interp_ok(val.clone())
+            interp_ok(None)
         }
     }
 
+    fn with_retag_mode<T>(
+        ecx: &mut InterpCx<'tcx, Self>,
+        mode: RetagMode,
+        f: impl FnOnce(&mut InterpCx<'tcx, Self>) -> InterpResult<'tcx, T>,
+    ) -> InterpResult<'tcx, T> {
+        let old_mode = mem::replace(&mut ecx.machine.retag_mode, mode);
+        let ret = f(ecx);
+        ecx.machine.retag_mode = old_mode;
+        ret
+    }
+
     fn before_memory_write(
         _tcx: TyCtxtAt<'tcx>,
         _machine: &mut Self,