[region-isolation] When assigning RValues into memory, use tuple_addr_constructor instead of doing it in pieces.

docs/SIL.rst

@@ -4799,6 +4799,71 @@ eliminated. However, a memory location ``%a`` must not be accessed
 after ``destroy_addr %a`` (which has not yet been eliminated)
 regardless of its type.
 
+tuple_addr_constructor
+``````````````````````
+
+::
+
+   sil-instruction ::= 'tuple_addr_constructor' sil-tuple-addr-constructor-init sil-operand 'with' sil-tuple-addr-constructor-elements
+   sil-tuple-addr-constructor-init ::= init|assign
+   sil-tuple-addr-constructor-elements ::= '(' (sil-operand (',' sil-operand)*)? ')'
+
+   // %destAddr has the type $*(Type1, Type2, Type3). Note how we convert all of the types
+   // to their address form.
+   %1 = tuple_addr_constructor [init] %destAddr : $*(Type1, Type2, Type3) with (%a : $Type1, %b : $*Type2, %c : $Type3)
+
+Creates a new tuple in memory from an exploded list of object and address
+values. The SSA values form the leaf elements of the exploded tuple. So for a
+simple tuple that only has top level tuple elements, then the instruction lowers
+as follows::
+
+  %1 = tuple_addr_constructor [init] %destAddr : $*(Type1, Type2, Type3) with (%a : $Type1, %b : $*Type2, %c : $Type3)
+
+  -->
+
+  %0 = tuple_element_addr %destAddr : $*(Type1, Type2, Type3), 0
+  store %a to [init] %0 : $*Type1
+  %1 = tuple_element_addr %destAddr : $*(Type1, Type2, Type3), 1
+  copy_addr %b to [init] %1 : $*Type2
+  %2 = tuple_element_addr %destAddr : $*(Type1, Type2, Type3), 2
+  store %2 to [init] %2 : $*Type3
+
+A ``tuple_addr_constructor`` is lowered similarly with each store/copy_addr
+being changed to their dest assign form.
+
+In contrast, if we have a more complicated form of tuple with sub-tuples, then
+we read one element from the list as we process the tuple recursively from left
+to right. So for instance we would lower as follows a more complicated tuple::
+
+  %1 = tuple_addr_constructor [init] %destAddr : $*((), (Type1, ((), Type2)), Type3) with (%a : $Type1, %b : $*Type2, %c : $Type3)
+
+  ->
+
+  %0 = tuple_element_addr %destAddr : $*((), (Type1, ((), Type2)), Type3), 1
+  %1 = tuple_element_addr %0 : $*(Type1, ((), Type2)), 0
+  store %a to [init] %1 : $*Type1
+  %2 = tuple_element_addr %0 : $*(Type1, ((), Type2)), 1
+  %3 = tuple_element_addr %2 : $*((), Type2), 1
+  copy_addr %b to [init] %3 : $*Type2
+  %4 = tuple_element_addr %destAddr : $*((), (Type1, ((), Type2)), Type3), 2
+  store %c to [init] %4 : $*Type3
+
+This instruction exists to enable for SILGen to init and assign RValues into
+tuples with a single instruction. Since an RValue is a potentially exploded
+tuple, we are forced to use our representation here. If SILGen instead just uses
+separate address projections and stores when it sees such an aggregate,
+diagnostic SIL passes can not tell the difference semantically in between
+initializing a tuple in parts or at once::
+
+  var arg = (Type1(), Type2())
+
+  // This looks the same at the SIL level...
+  arg = (a, b)
+
+  // to assigning in pieces even though we have formed a new tuple.
+  arg.0 = a
+  arg.1 = a
+
 index_addr
 ``````````
 ::

include/swift/SIL/AddressWalker.h

@@ -199,7 +199,8 @@ TransitiveAddressWalker<Impl>::walk(SILValue projectedAddress) && {
         isa<UncheckedRefCastAddrInst>(user) || isa<KeyPathInst>(user) ||
         isa<RetainValueAddrInst>(user) || isa<ReleaseValueAddrInst>(user) ||
         isa<PackElementSetInst>(user) || isa<PackElementGetInst>(user) ||
-        isa<DeinitExistentialAddrInst>(user) || isa<LoadBorrowInst>(user)) {
+        isa<DeinitExistentialAddrInst>(user) || isa<LoadBorrowInst>(user) ||
+        isa<TupleAddrConstructorInst>(user)) {
       callVisitUse(op);
       continue;
     }

include/swift/SIL/InstructionUtils.h

@@ -205,6 +205,23 @@ struct PolymorphicBuiltinSpecializedOverloadInfo {
 /// polymorphic builtin or does not have any available overload for these types,
 /// return SILValue().
 SILValue getStaticOverloadForSpecializedPolymorphicBuiltin(BuiltinInst *bi);
+
+/// Visit the exploded leaf elements of a tuple type that contains potentially
+/// a tree of tuples.
+///
+/// If visitor returns false, we stop processing early. We return true if we
+/// visited all of the tuple elements without the visitor returing false.
+bool visitExplodedTupleType(SILType type,
+                            llvm::function_ref<bool(SILType)> callback);
+
+/// Visit the exploded leaf elements of a tuple type that contains potentially
+/// a tree of tuples.
+///
+/// If visitor returns false, we stop processing early. We return true if we
+/// visited all of the tuple elements without the visitor returing false.
+bool visitExplodedTupleValue(SILValue value,
+                             llvm::function_ref<SILValue(SILValue, std::optional<unsigned>)> callback);
+
 } // end namespace swift
 
 #endif

include/swift/SIL/SILBuilder.h

@@ -1644,6 +1644,15 @@ class SILBuilder {
 
   TupleInst *createTuple(SILLocation loc, ArrayRef<SILValue> elts);
 
+  TupleAddrConstructorInst *
+  createTupleAddrConstructor(SILLocation Loc, SILValue DestAddr,
+                             ArrayRef<SILValue> Elements,
+                             IsInitialization_t IsInitOfDest) {
+    return insert(TupleAddrConstructorInst::create(getSILDebugLocation(Loc),
+                                                   DestAddr, Elements,
+                                                   IsInitOfDest, getModule()));
+  }
+
   EnumInst *createEnum(SILLocation Loc, SILValue Operand,
                        EnumElementDecl *Element, SILType Ty) {
     return createEnum(Loc, Operand, Element, Ty,

include/swift/SIL/SILCloner.h

@@ -2181,6 +2181,35 @@ SILCloner<ImplClass>::visitTupleInst(TupleInst *Inst) {
                                    : ValueOwnershipKind(OwnershipKind::None)));
 }
 
+template <typename ImplClass>
+void SILCloner<ImplClass>::visitTupleAddrConstructorInst(
+    TupleAddrConstructorInst *Inst) {
+  SmallVector<SILValue, 8> Elements;
+  for (auto e : Inst->getElements()) {
+    SILValue mappedValue = getOpValue(e);
+
+    // Check if mappedValue only consists of empty tuple elements. If it does,
+    // then we do not add it to our result. This is because we know that the
+    // corresponding elements in getOpValue(Inst->getDest()) will also change
+    // into an empty exploded tuple. Since we only have leaf non-empty non-tuple
+    // elements as operands, these are not represented.
+    bool FoundNonTuple = false;
+    mappedValue->getType().getASTType().visit(
+        [&](CanType ty) { FoundNonTuple |= !ty->is<TupleType>(); });
+    if (FoundNonTuple)
+      Elements.push_back(mappedValue);
+  }
+
+  if (Elements.empty())
+    return;
+
+  getBuilder().setCurrentDebugScope(getOpScope(Inst->getDebugScope()));
+  recordClonedInstruction(Inst, getBuilder().createTupleAddrConstructor(
+                                    getOpLocation(Inst->getLoc()),
+                                    getOpValue(Inst->getDest()), Elements,
+                                    Inst->isInitializationOfDest()));
+}
+
 template<typename ImplClass>
 void
 SILCloner<ImplClass>::visitEnumInst(EnumInst *Inst) {

include/swift/SIL/SILInstruction.h

@@ -6339,6 +6339,80 @@ class TupleInst final : public InstructionBaseWithTrailingOperands<
   }
 };
 
+/// TupleAddrConstructorInst - a constructor for address tuples. Can take
+/// objects and addresses. Intended only to be used with diagnostics and be
+/// lowered after diagnostics run. Once we have opaque values this will not be
+/// necessary.
+///
+/// tuple_addr_constructor [init] dest with (operands)
+///
+/// This always consumes its operands but will either init or assign into dest.
+class TupleAddrConstructorInst final
+    : public InstructionBaseWithTrailingOperands<
+          SILInstructionKind::TupleAddrConstructorInst,
+          TupleAddrConstructorInst, NonValueInstruction> {
+  friend SILBuilder;
+  USE_SHARED_UINT8;
+
+  TupleAddrConstructorInst(SILDebugLocation DebugLoc, ArrayRef<SILValue> Elts,
+                           IsInitialization_t IsInitOfDest)
+      : InstructionBaseWithTrailingOperands(Elts, DebugLoc) {
+    sharedUInt8().TupleAddrConstructorInst.isInitializationOfDest =
+        bool(IsInitOfDest);
+  }
+
+  static TupleAddrConstructorInst *create(SILDebugLocation DebugLoc,
+                                          SILValue DestAddr,
+                                          ArrayRef<SILValue> Elements,
+                                          IsInitialization_t IsInitOfDest,
+                                          SILModule &Mod);
+
+public:
+  enum {
+    Dest = 0,
+  };
+
+  Operand &getDestOperand() { return getAllOperands().front(); }
+  const Operand &getDestOperand() const { return getAllOperands().front(); }
+
+  SILValue getDest() const { return getDestOperand().get(); }
+
+  /// The elements referenced by this TupleInst.
+  MutableArrayRef<Operand> getElementOperands() {
+    return getAllOperands().drop_front();
+  }
+
+  /// The elements referenced by this TupleInst.
+  OperandValueArrayRef getElements() const {
+    return OperandValueArrayRef(getAllOperands().drop_front());
+  }
+
+  /// Return the i'th value referenced by this TupleInst.
+  SILValue getElement(unsigned i) const { return getElements()[i]; }
+
+  unsigned getElementIndex(Operand *operand) {
+    assert(operand->getUser() == this);
+    assert(operand != &getDestOperand() && "Cannot pass in the destination");
+    return operand->getOperandNumber() + 1;
+  }
+
+  unsigned getNumElements() const { return getTupleType()->getNumElements(); }
+
+  TupleType *getTupleType() const {
+    // We use getASTType() since we want to look through a wrapped noncopyable
+    // type to get to the underlying tuple type.
+    return getDest()->getType().getASTType()->castTo<TupleType>();
+  }
+
+  IsInitialization_t isInitializationOfDest() const {
+    return IsInitialization_t(
+        sharedUInt8().TupleAddrConstructorInst.isInitializationOfDest);
+  }
+  void setIsInitializationOfDest(IsInitialization_t I) {
+    sharedUInt8().TupleAddrConstructorInst.isInitializationOfDest = (bool)I;
+  }
+};
+
 /// Represents a loadable enum constructed from one of its
 /// elements.
 class EnumInst

include/swift/SIL/SILNode.h

@@ -251,6 +251,9 @@ class alignas(8) SILNode :
       isTakeOfSrc : 1,
       isInitializationOfDest : 1);
 
+    SHARED_FIELD(TupleAddrConstructorInst, uint8_t
+      isInitializationOfDest : 1);
+
     SHARED_FIELD(PointerToAddressInst, uint8_t
       isStrict : 1,
       isInvariant : 1);

include/swift/SIL/SILNodes.def

@@ -848,6 +848,8 @@ NON_VALUE_INST(CopyAddrInst, copy_addr,
                SILInstruction, MayHaveSideEffects, MayRelease)
 NON_VALUE_INST(ExplicitCopyAddrInst, explicit_copy_addr,
                SILInstruction, MayHaveSideEffects, MayRelease)
+NON_VALUE_INST(TupleAddrConstructorInst, tuple_addr_constructor,
+               SILInstruction, MayHaveSideEffects, MayRelease)
 NON_VALUE_INST(DestroyAddrInst, destroy_addr,
                SILInstruction, MayHaveSideEffects, MayRelease)
 NON_VALUE_INST(EndLifetimeInst, end_lifetime,

include/swift/SILOptimizer/PassManager/Passes.def

@@ -369,6 +369,8 @@ PASS(IRGenPrepare, "irgen-prepare",
      "Cleanup SIL in preparation for IRGen")
 PASS(TransferNonSendable, "transfer-non-sendable",
      "Checks calls that send non-sendable values between isolation domains")
+PASS(LowerTupleAddrConstructor, "lower-tuple-addr-constructor",
+     "Lower tuple addr constructor to tuple_element_addr+copy_addr")
 PASS(SILGenCleanup, "silgen-cleanup",
      "Cleanup SIL in preparation for diagnostics")
 PASS(SILCombine, "sil-combine",

lib/IRGen/IRGenSIL.cpp

@@ -1417,6 +1417,9 @@ class IRGenSILFunction :
   void visitMarkUnresolvedMoveAddrInst(MarkUnresolvedMoveAddrInst *mai) {
     llvm_unreachable("Valid only when ownership is enabled");
   }
+  void visitTupleAddrConstructorInst(TupleAddrConstructorInst *i) {
+    llvm_unreachable("Valid only in raw SIL");
+  }
   void visitDestroyAddrInst(DestroyAddrInst *i);
 
   void visitBindMemoryInst(BindMemoryInst *i);

lib/SIL/IR/OperandOwnership.cpp

@@ -676,6 +676,15 @@ OperandOwnership OperandOwnershipClassifier::visitKeyPathInst(KeyPathInst *I) {
   return OperandOwnership::ForwardingConsume;
 }
 
+OperandOwnership OperandOwnershipClassifier::visitTupleAddrConstructorInst(
+    TupleAddrConstructorInst *inst) {
+  // If we have an object, then we have an instantaneous use...
+  if (getValue()->getType().isObject())
+    return OperandOwnership::DestroyingConsume;
+  // Otherwise, we have a trivial use since we have an address.
+  return OperandOwnership::TrivialUse;
+}
+
 //===----------------------------------------------------------------------===//
 //                            Builtin Use Checker
 //===----------------------------------------------------------------------===//

lib/SIL/IR/SILInstructions.cpp

@@ -1460,6 +1460,18 @@ TupleInst *TupleInst::create(SILDebugLocation Loc, SILType Ty,
   return ::new (Buffer) TupleInst(Loc, Ty, Elements, forwardingOwnershipKind);
 }
 
+TupleAddrConstructorInst *TupleAddrConstructorInst::create(
+    SILDebugLocation Loc, SILValue DestAddr, ArrayRef<SILValue> Elements,
+    IsInitialization_t IsInitOfDest, SILModule &M) {
+  assert(DestAddr->getType().isAddress());
+  auto Size = totalSizeToAlloc<swift::Operand>(Elements.size() + 1);
+  auto Buffer = M.allocateInst(Size, alignof(TupleAddrConstructorInst));
+  llvm::SmallVector<SILValue, 16> Data;
+  Data.push_back(DestAddr);
+  copy(Elements, std::back_inserter(Data));
+  return ::new (Buffer) TupleAddrConstructorInst(Loc, Data, IsInitOfDest);
+}
+
 bool TupleExtractInst::isTrivialEltOfOneRCIDTuple() const {
   auto *F = getFunction();
 

lib/SIL/IR/SILPrinter.cpp

@@ -2215,7 +2215,25 @@ class SILPrinter : public SILInstructionVisitor<SILPrinter> {
       *this << ')';
     }
   }
-
+
+  void visitTupleAddrConstructorInst(TupleAddrConstructorInst *TI) {
+    // First print out our dest.
+    if (TI->isInitializationOfDest()) {
+      *this << "[init] ";
+    } else {
+      *this << "[assign] ";
+    }
+    *this << getIDAndType(TI->getDest());
+
+    *this << " with (";
+
+    llvm::interleave(
+        TI->getElements(), [&](const SILValue &V) { *this << getIDAndType(V); },
+        [&] { *this << ", "; });
+
+    *this << ')';
+  }
+
   void visitEnumInst(EnumInst *UI) {
     *this << UI->getType() << ", "
           << SILDeclRef(UI->getElement(), SILDeclRef::Kind::EnumElement);