OSSA: simplify-cfg support for trivial block arguments.

include/swift/SIL/SILInstruction.h

@@ -1025,6 +1025,16 @@ class OwnershipForwardingMixin {
   }
 
 public:
+  /// Forwarding ownership is determined by the forwarding instruction's
+  /// constant ownership attribute. If forwarding ownership is owned, then the
+  /// instruction moves an owned operand to its result, ending its lifetime. If
+  /// forwarding ownership is guaranteed, then the instruction propagates the
+  /// lifetime of its borrows operand through its result.
+  ///
+  /// The resulting forwarded value's ownership, returned by getOwnershipKind(),
+  /// is not identical to the forwarding ownership. It differs when the result
+  /// is trivial type. e.g. an owned or guaranteed value can be cast to a
+  /// trivial type using owned or guaranteed forwarding.
   ValueOwnershipKind getForwardingOwnershipKind() const {
     return ownershipKind;
   }

include/swift/SIL/SILValue.h

@@ -515,6 +515,16 @@ class ValueBase : public SILNode, public SILAllocated<ValueBase> {
   /// result index, or None if it is not defined by an instruction.
   Optional<DefiningInstructionResult> getDefiningInstructionResult();
 
+  /// Returns the ValueOwnershipKind that describes this SILValue's ownership
+  /// semantics if the SILValue has ownership semantics. Returns is a value
+  /// without any Ownership Semantics.
+  ///
+  /// An example of a SILValue without ownership semantics is a
+  /// struct_element_addr.
+  ///
+  /// NOTE: This is implemented in ValueOwnership.cpp not SILValue.cpp.
+  ValueOwnershipKind getOwnershipKind() const;
+
   static bool classof(SILNodePointer node) {
     return node->getKind() >= SILNodeKind::First_ValueBase &&
            node->getKind() <= SILNodeKind::Last_ValueBase;
@@ -592,6 +602,8 @@ class SILValue {
 
   /// If this SILValue is a result of an instruction, return its
   /// defining instruction. Returns nullptr otherwise.
+  ///
+  /// FIXME: remove this redundant API from SILValue.
   SILInstruction *getDefiningInstruction() {
     return Value->getDefiningInstruction();
   }
@@ -610,7 +622,11 @@ class SILValue {
   /// struct_element_addr.
   ///
   /// NOTE: This is implemented in ValueOwnership.cpp not SILValue.cpp.
-  ValueOwnershipKind getOwnershipKind() const;
+  ///
+  /// FIXME: remove this redundant API from SILValue.
+  ValueOwnershipKind getOwnershipKind() const {
+    return Value->getOwnershipKind();
+  }
 
   /// Verify that this SILValue and its uses respects ownership invariants.
   void verifyOwnership(DeadEndBlocks *DEBlocks) const;
@@ -660,6 +676,11 @@ class OwnershipConstraint {
     return lifetimeConstraint;
   }
 
+  bool isConsuming() const {
+    return ownershipKind == OwnershipKind::Owned
+      && lifetimeConstraint == UseLifetimeConstraint::LifetimeEnding;
+  }
+
   bool satisfiedBy(const Operand *use) const;
 
   bool satisfiesConstraint(ValueOwnershipKind testKind) const {
@@ -880,15 +901,20 @@ inline bool canAcceptUnownedValue(OperandOwnership operandOwnership) {
   }
 }
 
-/// Return the OperandOwnership for a forwarded operand when the forwarded
-/// result has this ValueOwnershipKind. \p allowUnowned is true for a subset
-/// of forwarding operations that are allowed to propagate Unowned values.
+/// Return the OperandOwnership for a forwarded operand when the forwarding
+/// operation has this "forwarding ownership" (as returned by
+/// getForwardingOwnershipKind()). \p allowUnowned is true for a subset of
+/// forwarding operations that are allowed to propagate Unowned values.
+///
+/// Forwarding ownership is determined by the forwarding instruction's constant
+/// ownership attribute. If forwarding ownership is owned, then the instruction
+/// moves owned operand to its result, ending its lifetime. If forwarding
+/// ownership is guaranteed, then the instruction propagates the lifetime of its
+/// borrows operand through its result.
 ///
-/// The ownership of a forwarded value is derived from the forwarding
-/// instruction's constant ownership attribute. If the result is owned, then the
-/// instruction moves owned operand to its result, ending its lifetime. If the
-/// result is guaranteed value, then the instruction propagates the lifetime of
-/// its borrows operand through its result.
+/// The resulting forwarded value typically has forwarding ownership, but may
+/// differ when the result is trivial type. e.g. an owned or guaranteed value
+/// can be cast to a trivial type using owned or guaranteed forwarding.
 inline OperandOwnership
 ValueOwnershipKind::getForwardingOperandOwnership(bool allowUnowned) const {
   switch (value) {

include/swift/SILOptimizer/Utils/InstOptUtils.h

@@ -47,6 +47,14 @@ inline TransformRange<Range, UserTransform> makeUserRange(Range range) {
   return makeTransformRange(range, UserTransform(toUser));
 }
 
+/// Transform a use_iterator range (Operand*) into an llvm::iterator_range
+/// of users (SILInstruction *)
+inline iterator_range<llvm::mapped_iterator<ValueBase::use_iterator, UserTransform>>
+makeUserIteratorRange(iterator_range<ValueBase::use_iterator> useRange) {
+  auto toUser = [](Operand *operand) { return operand->getUser(); };
+  return llvm::map_range(useRange, UserTransform(toUser));
+}
+
 using DeadInstructionSet = llvm::SmallSetVector<SILInstruction *, 8>;
 
 /// Create a retain of \p Ptr before the \p InsertPt.
@@ -265,9 +273,15 @@ SILValue getConcreteValueOfExistentialBoxAddr(SILValue addr,
 /// - a type of the return value is a subclass of the expected return type.
 /// - actual return type and expected return type differ in optionality.
 /// - both types are tuple-types and some of the elements need to be casted.
+///
+/// \p usePoints is required when \p value has guaranteed ownership. It must be
+/// the last users of the returned, casted value. A usePoint cannot be a
+/// BranchInst (a phi is never the last guaranteed user). \p builder's current
+/// insertion point must dominate all \p usePoints.
 std::pair<SILValue, bool /* changedCFG */>
 castValueToABICompatibleType(SILBuilder *builder, SILLocation Loc,
-                             SILValue value, SILType srcTy, SILType destTy);
+                             SILValue value, SILType srcTy, SILType destTy,
+                             ArrayRef<SILInstruction *> usePoints);
 /// Peek through trivial Enum initialization, typically for pointless
 /// Optionals.
 ///
@@ -700,10 +714,18 @@ makeCopiedValueAvailable(SILValue value, SILBasicBlock *inBlock);
 /// Given a newly created @owned value \p value without any uses, this utility
 /// inserts control equivalent copy and destroy at leaking blocks to adjust
 /// ownership and make \p value available for use at \p inBlock.
+///
+/// inBlock must be the only point at which \p value will be consumed. If this
+/// consuming point is within a loop, this will create and return a copy of \p
+/// value inside \p inBlock.
 SILValue
 makeNewValueAvailable(SILValue value, SILBasicBlock *inBlock);
 
 /// Given an ssa value \p value, create destroy_values at leaking blocks
+///
+/// Warning: This does not properly cleanup an OSSA lifetime with a consuming
+/// use blocks inside a loop relative to \p value. The client must create
+/// separate copies for any uses within the loop.
 void endLifetimeAtLeakingBlocks(SILValue value,
                                 ArrayRef<SILBasicBlock *> userBBs);
 

include/swift/SILOptimizer/Utils/OwnershipOptUtils.h

@@ -27,6 +27,13 @@
 
 namespace swift {
 
+/// Returns true if this value requires OSSA cleanups.
+inline bool requiresOSSACleanup(SILValue v) {
+  return v->getFunction()->hasOwnership()
+    && v.getOwnershipKind() != OwnershipKind::None
+    && v.getOwnershipKind() != OwnershipKind::Unowned;
+}
+
 // Defined in BasicBlockUtils.h
 struct JointPostDominanceSetComputer;
 

lib/SIL/IR/OperandOwnership.cpp

@@ -443,9 +443,24 @@ OperandOwnershipClassifier::visitFullApply(FullApplySite apply) {
     ? SILArgumentConvention(apply.getSubstCalleeType()->getCalleeConvention())
     : apply.getArgumentConvention(op);
 
-  return getFunctionArgOwnership(
-      argConv,
-      /*hasScopeInCaller*/ apply.beginsCoroutineEvaluation());
+  auto argOwnership = getFunctionArgOwnership(
+    argConv, /*hasScopeInCaller*/ apply.beginsCoroutineEvaluation());
+
+  // OSSA cleanup needs to handle each of these callee ownership cases.
+  //
+  // OperandOwnership::ForwardingConsume is only for thick @callee_owned.
+  //
+  // OperandOwnership::Borrow would only happen for a coroutine closure, which
+  // isn't yet possible.
+  if (apply.isCalleeOperand(op)) {
+    assert((argOwnership == OperandOwnership::TrivialUse
+            || argOwnership == OperandOwnership::UnownedInstantaneousUse
+            || argOwnership == OperandOwnership::InstantaneousUse
+            || argOwnership == OperandOwnership::ForwardingConsume
+            || argOwnership == OperandOwnership::Borrow) &&
+           "unsupported callee ownership");
+  }
+  return argOwnership;
 }
 
 OperandOwnership

lib/SIL/IR/SILFunctionType.cpp

@@ -4319,6 +4319,9 @@ TypeConverter::getLoweredFormalTypes(SILDeclRef constant,
 // match exactly.
 // TODO: More sophisticated param and return ABI compatibility rules could
 // diverge.
+//
+// Note: all cases recognized here must be handled in the SILOptimizer's
+// castValueToABICompatibleType().
 static bool areABICompatibleParamsOrReturns(SILType a, SILType b,
                                             SILFunction *inFunction) {
   // Address parameters are all ABI-compatible, though the referenced

lib/SIL/IR/ValueOwnership.cpp

@@ -586,15 +586,15 @@ ValueOwnershipKindClassifier::visitBuiltinInst(BuiltinInst *BI) {
 //                            Top Level Entrypoint
 //===----------------------------------------------------------------------===//
 
-ValueOwnershipKind SILValue::getOwnershipKind() const {
+ValueOwnershipKind ValueBase::getOwnershipKind() const {
   // If we do not have an undef, we should always be able to get to our function
   // here. If we do not have ownership enabled, just return none for everything
   // to short circuit ownership optimizations. Since SILUndef in either case
   // will be ValueOwnershipKind::None, we will not get any wonky behavior here.
   //
   // We assume that any time we are in SILBuilder and call this without having a
   // value in a block yet, ossa is enabled.
-  if (auto *block = Value->getParentBlock()) {
+  if (auto *block = getParentBlock()) {
     auto *f = block->getParent();
     // If our block isn't in a function, then it must be in a global
     // variable. We don't verify ownership there so just return
@@ -609,7 +609,7 @@ ValueOwnershipKind SILValue::getOwnershipKind() const {
   }
 
   ValueOwnershipKindClassifier Classifier;
-  auto result = Classifier.visit(const_cast<ValueBase *>(Value));
+  auto result = Classifier.visit(const_cast<ValueBase *>(this));
   assert(result && "Returned ownership kind invalid on values");
   return result;
 }

lib/SILGen/SILGenThunk.cpp

@@ -260,7 +260,6 @@ SILGenModule::getOrCreateForeignAsyncCompletionHandlerImplFunction(
       auto flagIndex = convention.completionHandlerFlagParamIndex();
 
       FuncDecl *resumeIntrinsic;
-      Type replacementTypes[] = {resumeType};
 
       SILBasicBlock *returnBB = nullptr;
       if (errorIndex) {

lib/SILOptimizer/Transforms/EagerSpecializer.cpp

@@ -888,7 +888,11 @@ void EagerSpecializerTransform::run() {
   // removed.
   for (auto *SA : attrsToRemove)
     F.removeSpecializeAttr(SA);
-  F.verify();
+
+  // If any specializations were created, reverify the original body now that it
+  // has checks.
+  if (!newFunctions.empty())
+    F.verify();
 
   for (SILFunction *newF : newFunctions) {
     addFunctionToPassManagerWorklist(newF, nullptr);