From 1ac8c8af402c0c3618e6d023af249da7cd5157c9 Mon Sep 17 00:00:00 2001 From: Aaron Lehmann Date: Tue, 25 Jul 2017 13:06:27 -0700 Subject: [PATCH 1/8] Timestamp task status updates on the manager side as well This timestamp will be useful for tracking restart history, since the manager clocks may more trustworthy than agent clocks. Signed-off-by: Aaron Lehmann --- api/types.pb.go | 795 +++++++++++++++----------- api/types.proto | 9 + manager/dispatcher/dispatcher.go | 7 +- manager/dispatcher/dispatcher_test.go | 2 +- manager/manager.go | 2 +- 5 files changed, 465 insertions(+), 350 deletions(-) diff --git a/api/types.pb.go b/api/types.pb.go index 4a615b2425..8ffc000e17 100644 --- a/api/types.pb.go +++ b/api/types.pb.go @@ -1292,6 +1292,13 @@ type TaskStatus struct { // HostPorts provides a list of ports allocated at the host // level. PortStatus *PortStatus `protobuf:"bytes,6,opt,name=port_status,json=portStatus" json:"port_status,omitempty"` + // AppliedBy gives the node ID of the manager that applied this task + // status update to the Task object. + AppliedBy string `protobuf:"bytes,7,opt,name=applied_by,json=appliedBy,proto3" json:"applied_by,omitempty"` + // AppliedAt gives a timestamp of when this status update was applied to + // the Task object. + // Note: can't use stdtime because this field is nullable. + AppliedAt *google_protobuf.Timestamp `protobuf:"bytes,8,opt,name=applied_at,json=appliedAt" json:"applied_at,omitempty"` } func (m *TaskStatus) Reset() { *m = TaskStatus{} } @@ -2893,6 +2900,10 @@ func (m *TaskStatus) CopyFrom(src interface{}) { m.PortStatus = &PortStatus{} github_com_docker_swarmkit_api_deepcopy.Copy(m.PortStatus, o.PortStatus) } + if o.AppliedAt != nil { + m.AppliedAt = &google_protobuf.Timestamp{} + github_com_docker_swarmkit_api_deepcopy.Copy(m.AppliedAt, o.AppliedAt) + } if o.RuntimeStatus != nil { switch o.RuntimeStatus.(type) { case *TaskStatus_Container: @@ -4756,6 +4767,22 @@ func (m *TaskStatus) MarshalTo(dAtA []byte) (int, error) { } i += n22 } + if len(m.AppliedBy) > 0 { + dAtA[i] = 0x3a + i++ + i = encodeVarintTypes(dAtA, i, uint64(len(m.AppliedBy))) + i += copy(dAtA[i:], m.AppliedBy) + } + if m.AppliedAt != nil { + dAtA[i] = 0x42 + i++ + i = encodeVarintTypes(dAtA, i, uint64(m.AppliedAt.Size())) + n23, err := m.AppliedAt.MarshalTo(dAtA[i:]) + if err != nil { + return 0, err + } + i += n23 + } return i, nil } @@ -4765,11 +4792,11 @@ func (m *TaskStatus_Container) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0x2a i++ i = encodeVarintTypes(dAtA, i, uint64(m.Container.Size())) - n23, err := m.Container.MarshalTo(dAtA[i:]) + n24, err := m.Container.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n23 + i += n24 } return i, nil } @@ -5006,11 +5033,11 @@ func (m *IPAMOptions) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0xa i++ i = encodeVarintTypes(dAtA, i, uint64(m.Driver.Size())) - n24, err := m.Driver.MarshalTo(dAtA[i:]) + n25, err := m.Driver.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n24 + i += n25 } if len(m.Configs) > 0 { for _, msg := range m.Configs { @@ -5076,11 +5103,11 @@ func (m *WeightedPeer) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0xa i++ i = encodeVarintTypes(dAtA, i, uint64(m.Peer.Size())) - n25, err := m.Peer.MarshalTo(dAtA[i:]) + n26, err := m.Peer.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n25 + i += n26 } if m.Weight != 0 { dAtA[i] = 0x10 @@ -5183,11 +5210,11 @@ func (m *AcceptancePolicy_RoleAdmissionPolicy) MarshalTo(dAtA []byte) (int, erro dAtA[i] = 0x1a i++ i = encodeVarintTypes(dAtA, i, uint64(m.Secret.Size())) - n26, err := m.Secret.MarshalTo(dAtA[i:]) + n27, err := m.Secret.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n26 + i += n27 } return i, nil } @@ -5293,11 +5320,11 @@ func (m *CAConfig) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0xa i++ i = encodeVarintTypes(dAtA, i, uint64(m.NodeCertExpiry.Size())) - n27, err := m.NodeCertExpiry.MarshalTo(dAtA[i:]) + n28, err := m.NodeCertExpiry.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n27 + i += n28 } if len(m.ExternalCAs) > 0 { for _, msg := range m.ExternalCAs { @@ -5373,11 +5400,11 @@ func (m *TaskDefaults) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0xa i++ i = encodeVarintTypes(dAtA, i, uint64(m.LogDriver.Size())) - n28, err := m.LogDriver.MarshalTo(dAtA[i:]) + n29, err := m.LogDriver.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n28 + i += n29 } return i, nil } @@ -5401,11 +5428,11 @@ func (m *DispatcherConfig) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0xa i++ i = encodeVarintTypes(dAtA, i, uint64(m.HeartbeatPeriod.Size())) - n29, err := m.HeartbeatPeriod.MarshalTo(dAtA[i:]) + n30, err := m.HeartbeatPeriod.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n29 + i += n30 } return i, nil } @@ -5521,11 +5548,11 @@ func (m *PlacementPreference) MarshalTo(dAtA []byte) (int, error) { var l int _ = l if m.Preference != nil { - nn30, err := m.Preference.MarshalTo(dAtA[i:]) + nn31, err := m.Preference.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += nn30 + i += nn31 } return i, nil } @@ -5536,11 +5563,11 @@ func (m *PlacementPreference_Spread) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0xa i++ i = encodeVarintTypes(dAtA, i, uint64(m.Spread.Size())) - n31, err := m.Spread.MarshalTo(dAtA[i:]) + n32, err := m.Spread.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n31 + i += n32 } return i, nil } @@ -5667,20 +5694,20 @@ func (m *RootCA) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0x22 i++ i = encodeVarintTypes(dAtA, i, uint64(m.JoinTokens.Size())) - n32, err := m.JoinTokens.MarshalTo(dAtA[i:]) + n33, err := m.JoinTokens.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n32 + i += n33 if m.RootRotation != nil { dAtA[i] = 0x2a i++ i = encodeVarintTypes(dAtA, i, uint64(m.RootRotation.Size())) - n33, err := m.RootRotation.MarshalTo(dAtA[i:]) + n34, err := m.RootRotation.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n33 + i += n34 } if m.LastForcedRotation != 0 { dAtA[i] = 0x30 @@ -5719,11 +5746,11 @@ func (m *Certificate) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0x1a i++ i = encodeVarintTypes(dAtA, i, uint64(m.Status.Size())) - n34, err := m.Status.MarshalTo(dAtA[i:]) + n35, err := m.Status.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n34 + i += n35 if len(m.Certificate) > 0 { dAtA[i] = 0x22 i++ @@ -5892,11 +5919,11 @@ func (m *SecretReference) MarshalTo(dAtA []byte) (int, error) { i += copy(dAtA[i:], m.SecretName) } if m.Target != nil { - nn35, err := m.Target.MarshalTo(dAtA[i:]) + nn36, err := m.Target.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += nn35 + i += nn36 } return i, nil } @@ -5907,11 +5934,11 @@ func (m *SecretReference_File) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0x1a i++ i = encodeVarintTypes(dAtA, i, uint64(m.File.Size())) - n36, err := m.File.MarshalTo(dAtA[i:]) + n37, err := m.File.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n36 + i += n37 } return i, nil } @@ -5943,11 +5970,11 @@ func (m *ConfigReference) MarshalTo(dAtA []byte) (int, error) { i += copy(dAtA[i:], m.ConfigName) } if m.Target != nil { - nn37, err := m.Target.MarshalTo(dAtA[i:]) + nn38, err := m.Target.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += nn37 + i += nn38 } return i, nil } @@ -5958,11 +5985,11 @@ func (m *ConfigReference_File) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0x1a i++ i = encodeVarintTypes(dAtA, i, uint64(m.File.Size())) - n38, err := m.File.MarshalTo(dAtA[i:]) + n39, err := m.File.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n38 + i += n39 } return i, nil } @@ -5985,11 +6012,11 @@ func (m *BlacklistedCertificate) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0xa i++ i = encodeVarintTypes(dAtA, i, uint64(m.Expiry.Size())) - n39, err := m.Expiry.MarshalTo(dAtA[i:]) + n40, err := m.Expiry.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n39 + i += n40 } return i, nil } @@ -6028,21 +6055,21 @@ func (m *HealthConfig) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0x12 i++ i = encodeVarintTypes(dAtA, i, uint64(m.Interval.Size())) - n40, err := m.Interval.MarshalTo(dAtA[i:]) + n41, err := m.Interval.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n40 + i += n41 } if m.Timeout != nil { dAtA[i] = 0x1a i++ i = encodeVarintTypes(dAtA, i, uint64(m.Timeout.Size())) - n41, err := m.Timeout.MarshalTo(dAtA[i:]) + n42, err := m.Timeout.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n41 + i += n42 } if m.Retries != 0 { dAtA[i] = 0x20 @@ -6053,11 +6080,11 @@ func (m *HealthConfig) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0x2a i++ i = encodeVarintTypes(dAtA, i, uint64(m.StartPeriod.Size())) - n42, err := m.StartPeriod.MarshalTo(dAtA[i:]) + n43, err := m.StartPeriod.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n42 + i += n43 } return i, nil } @@ -6152,21 +6179,21 @@ func (m *Privileges) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0xa i++ i = encodeVarintTypes(dAtA, i, uint64(m.CredentialSpec.Size())) - n43, err := m.CredentialSpec.MarshalTo(dAtA[i:]) + n44, err := m.CredentialSpec.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n43 + i += n44 } if m.SELinuxContext != nil { dAtA[i] = 0x12 i++ i = encodeVarintTypes(dAtA, i, uint64(m.SELinuxContext.Size())) - n44, err := m.SELinuxContext.MarshalTo(dAtA[i:]) + n45, err := m.SELinuxContext.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n44 + i += n45 } return i, nil } @@ -6187,11 +6214,11 @@ func (m *Privileges_CredentialSpec) MarshalTo(dAtA []byte) (int, error) { var l int _ = l if m.Source != nil { - nn45, err := m.Source.MarshalTo(dAtA[i:]) + nn46, err := m.Source.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += nn45 + i += nn46 } return i, nil } @@ -6756,6 +6783,14 @@ func (m *TaskStatus) Size() (n int) { l = m.PortStatus.Size() n += 1 + l + sovTypes(uint64(l)) } + l = len(m.AppliedBy) + if l > 0 { + n += 1 + l + sovTypes(uint64(l)) + } + if m.AppliedAt != nil { + l = m.AppliedAt.Size() + n += 1 + l + sovTypes(uint64(l)) + } return n } @@ -7806,6 +7841,8 @@ func (this *TaskStatus) String() string { `Err:` + fmt.Sprintf("%v", this.Err) + `,`, `RuntimeStatus:` + fmt.Sprintf("%v", this.RuntimeStatus) + `,`, `PortStatus:` + strings.Replace(fmt.Sprintf("%v", this.PortStatus), "PortStatus", "PortStatus", 1) + `,`, + `AppliedBy:` + fmt.Sprintf("%v", this.AppliedBy) + `,`, + `AppliedAt:` + strings.Replace(fmt.Sprintf("%v", this.AppliedAt), "Timestamp", "google_protobuf.Timestamp", 1) + `,`, `}`, }, "") return s @@ -11950,6 +11987,68 @@ func (m *TaskStatus) Unmarshal(dAtA []byte) error { return err } iNdEx = postIndex + case 7: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field AppliedBy", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowTypes + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthTypes + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.AppliedBy = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 8: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field AppliedAt", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowTypes + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthTypes + } + postIndex := iNdEx + msglen + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.AppliedAt == nil { + m.AppliedAt = &google_protobuf.Timestamp{} + } + if err := m.AppliedAt.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipTypes(dAtA[iNdEx:]) @@ -17087,309 +17186,311 @@ var ( func init() { proto.RegisterFile("types.proto", fileDescriptorTypes) } var fileDescriptorTypes = []byte{ - // 4854 bytes of a gzipped FileDescriptorProto + // 4887 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x7a, 0x4d, 0x6c, 0x24, 0xd7, 0x56, 0xbf, 0xfb, 0xd3, 0xdd, 0xa7, 0xdb, 0x76, 0xf9, 0x8e, 0x33, 0xf1, 0x74, 0x26, 0xb6, 0x53, 0xc9, 0xbc, 0x24, 0xf3, 0xf2, 0xef, 0xcc, 0x47, 0x12, 0x4d, 0x92, 0x7f, 0x3e, 0xfa, 0xcb, 0xe3, 0x7e, 0x63, 0x77, 0xb7, 0x6e, 0xb7, 0x67, 0x5e, 0x90, 0xa0, 0x54, 0xae, 0xba, 0x6e, 0x57, 0x5c, - 0x5d, 0xb7, 0xa9, 0xaa, 0xb6, 0xa7, 0x79, 0x20, 0x46, 0x2c, 0x00, 0x79, 0x05, 0xbb, 0x27, 0x21, - 0xb3, 0x81, 0x15, 0x42, 0x62, 0x01, 0x12, 0x82, 0x0d, 0x41, 0x62, 0x91, 0x1d, 0x0f, 0x90, 0xd0, - 0x13, 0x48, 0x86, 0x78, 0xc1, 0x0e, 0xc1, 0xe6, 0x89, 0x0d, 0x48, 0xe8, 0x7e, 0x54, 0x75, 0x75, - 0x4f, 0xd9, 0x9e, 0x90, 0xb7, 0xb1, 0xeb, 0x9e, 0xf3, 0x3b, 0xe7, 0x9e, 0x7b, 0xee, 0xbd, 0xe7, - 0x9e, 0x73, 0x6f, 0x43, 0xc1, 0x1f, 0x0f, 0x89, 0x57, 0x1e, 0xba, 0xd4, 0xa7, 0x08, 0x99, 0xd4, - 0x38, 0x24, 0x6e, 0xd9, 0x3b, 0xd6, 0xdd, 0xc1, 0xa1, 0xe5, 0x97, 0x8f, 0xee, 0x96, 0xd6, 0xfb, - 0x94, 0xf6, 0x6d, 0xf2, 0x2e, 0x47, 0xec, 0x8d, 0xf6, 0xdf, 0xf5, 0xad, 0x01, 0xf1, 0x7c, 0x7d, - 0x30, 0x14, 0x42, 0xa5, 0xb5, 0x59, 0x80, 0x39, 0x72, 0x75, 0xdf, 0xa2, 0x8e, 0xe4, 0xaf, 0xf4, - 0x69, 0x9f, 0xf2, 0xcf, 0x77, 0xd9, 0x97, 0xa0, 0xaa, 0xeb, 0x30, 0xff, 0x98, 0xb8, 0x9e, 0x45, - 0x1d, 0xb4, 0x02, 0x19, 0xcb, 0x31, 0xc9, 0xd3, 0xd5, 0xc4, 0x46, 0xe2, 0xad, 0x34, 0x16, 0x0d, - 0xf5, 0x0e, 0x40, 0x93, 0x7d, 0x34, 0x1c, 0xdf, 0x1d, 0x23, 0x05, 0x52, 0x87, 0x64, 0xcc, 0x11, - 0x79, 0xcc, 0x3e, 0x19, 0xe5, 0x48, 0xb7, 0x57, 0x93, 0x82, 0x72, 0xa4, 0xdb, 0xea, 0x37, 0x09, - 0x28, 0x54, 0x1c, 0x87, 0xfa, 0xbc, 0x77, 0x0f, 0x21, 0x48, 0x3b, 0xfa, 0x80, 0x48, 0x21, 0xfe, - 0x8d, 0x6a, 0x90, 0xb5, 0xf5, 0x3d, 0x62, 0x7b, 0xab, 0xc9, 0x8d, 0xd4, 0x5b, 0x85, 0x7b, 0xdf, - 0x2f, 0x3f, 0x3f, 0xe4, 0x72, 0x44, 0x49, 0x79, 0x9b, 0xa3, 0xb9, 0x11, 0x58, 0x8a, 0xa2, 0x4f, - 0x61, 0xde, 0x72, 0x4c, 0xcb, 0x20, 0xde, 0x6a, 0x9a, 0x6b, 0x59, 0x8b, 0xd3, 0x32, 0xb1, 0xbe, - 0x9a, 0xfe, 0xfa, 0x6c, 0x7d, 0x0e, 0x07, 0x42, 0xa5, 0x0f, 0xa1, 0x10, 0x51, 0x1b, 0x33, 0xb6, - 0x15, 0xc8, 0x1c, 0xe9, 0xf6, 0x88, 0xc8, 0xd1, 0x89, 0xc6, 0x47, 0xc9, 0x07, 0x09, 0xf5, 0x73, - 0x58, 0x69, 0xe9, 0x03, 0x62, 0x3e, 0x24, 0x0e, 0x71, 0x2d, 0x03, 0x13, 0x8f, 0x8e, 0x5c, 0x83, - 0xb0, 0xb1, 0x1e, 0x5a, 0x8e, 0x19, 0x8c, 0x95, 0x7d, 0xc7, 0x6b, 0x51, 0x6b, 0xf0, 0x72, 0xdd, - 0xf2, 0x0c, 0x97, 0xf8, 0xe4, 0x5b, 0x2b, 0x49, 0x05, 0x4a, 0xce, 0x12, 0xb0, 0x34, 0x2b, 0xfd, - 0x0b, 0x70, 0x8d, 0xb9, 0xd8, 0xd4, 0x5c, 0x49, 0xd1, 0xbc, 0x21, 0x31, 0xb8, 0xb2, 0xc2, 0xbd, - 0xb7, 0xe2, 0x3c, 0x14, 0x37, 0x92, 0xad, 0x39, 0xbc, 0xcc, 0xd5, 0x04, 0x84, 0xee, 0x90, 0x18, - 0xc8, 0x80, 0xeb, 0xa6, 0x34, 0x7a, 0x46, 0x7d, 0x92, 0xab, 0x8f, 0x9d, 0xc6, 0x0b, 0x86, 0xb9, - 0x35, 0x87, 0x57, 0x02, 0x65, 0xd1, 0x4e, 0xaa, 0x00, 0xb9, 0x40, 0xb7, 0xfa, 0xe3, 0x04, 0xe4, - 0x03, 0xa6, 0x87, 0xde, 0x86, 0xbc, 0xa3, 0x3b, 0x54, 0x33, 0x86, 0x23, 0x8f, 0x0f, 0x28, 0x55, - 0x2d, 0x9e, 0x9f, 0xad, 0xe7, 0x5a, 0xba, 0x43, 0x6b, 0x9d, 0x5d, 0x0f, 0xe7, 0x18, 0xbb, 0x36, - 0x1c, 0x79, 0xe8, 0x35, 0x28, 0x0e, 0xc8, 0x80, 0xba, 0x63, 0x6d, 0x6f, 0xec, 0x13, 0x4f, 0xba, - 0xad, 0x20, 0x68, 0x55, 0x46, 0x42, 0x9f, 0xc0, 0x7c, 0x5f, 0x98, 0xb4, 0x9a, 0xe2, 0xcb, 0xe7, - 0xf5, 0x38, 0xeb, 0x67, 0xac, 0xc6, 0x81, 0x8c, 0xfa, 0x3b, 0x09, 0x58, 0x09, 0xa9, 0xe4, 0x97, - 0x47, 0x96, 0x4b, 0x06, 0xc4, 0xf1, 0x3d, 0xf4, 0x3e, 0x64, 0x6d, 0x6b, 0x60, 0xf9, 0x9e, 0xf4, - 0xf9, 0xab, 0x71, 0x6a, 0xc3, 0x41, 0x61, 0x09, 0x46, 0x15, 0x28, 0xba, 0xc4, 0x23, 0xee, 0x91, - 0x58, 0xf1, 0xd2, 0xa3, 0x57, 0x08, 0x4f, 0x89, 0xa8, 0x9b, 0x90, 0xeb, 0xd8, 0xba, 0xbf, 0x4f, - 0xdd, 0x01, 0x52, 0xa1, 0xa8, 0xbb, 0xc6, 0x81, 0xe5, 0x13, 0xc3, 0x1f, 0xb9, 0xc1, 0xee, 0x9b, - 0xa2, 0xa1, 0xeb, 0x90, 0xa4, 0xa2, 0xa3, 0x7c, 0x35, 0x7b, 0x7e, 0xb6, 0x9e, 0x6c, 0x77, 0x71, - 0x92, 0x7a, 0xea, 0xc7, 0xb0, 0xdc, 0xb1, 0x47, 0x7d, 0xcb, 0xa9, 0x13, 0xcf, 0x70, 0xad, 0x21, - 0xd3, 0xce, 0x56, 0x25, 0x8b, 0x51, 0xc1, 0xaa, 0x64, 0xdf, 0xe1, 0xd6, 0x4e, 0x4e, 0xb6, 0xb6, - 0xfa, 0x5b, 0x49, 0x58, 0x6e, 0x38, 0x7d, 0xcb, 0x21, 0x51, 0xe9, 0x5b, 0xb0, 0x48, 0x38, 0x51, - 0x3b, 0x12, 0xe1, 0x46, 0xea, 0x59, 0x10, 0xd4, 0x20, 0x06, 0x35, 0x67, 0xe2, 0xc2, 0xdd, 0xb8, - 0xe1, 0x3f, 0xa7, 0x3d, 0x36, 0x3a, 0x34, 0x60, 0x7e, 0xc8, 0x07, 0xe1, 0xc9, 0xe9, 0xbd, 0x15, - 0xa7, 0xeb, 0xb9, 0x71, 0x06, 0x41, 0x42, 0xca, 0x7e, 0x97, 0x20, 0xf1, 0xc7, 0x49, 0x58, 0x6a, - 0x51, 0x73, 0xca, 0x0f, 0x25, 0xc8, 0x1d, 0x50, 0xcf, 0x8f, 0x04, 0xc4, 0xb0, 0x8d, 0x1e, 0x40, - 0x6e, 0x28, 0xa7, 0x4f, 0xce, 0xfe, 0xcd, 0x78, 0x93, 0x05, 0x06, 0x87, 0x68, 0xf4, 0x31, 0xe4, - 0x83, 0x2d, 0xc3, 0x46, 0xfb, 0x02, 0x0b, 0x67, 0x82, 0x47, 0x9f, 0x40, 0x56, 0x4c, 0xc2, 0x6a, - 0x9a, 0x4b, 0xde, 0x7a, 0x21, 0x9f, 0x63, 0x29, 0x84, 0x1e, 0x42, 0xce, 0xb7, 0x3d, 0xcd, 0x72, - 0xf6, 0xe9, 0x6a, 0x86, 0x2b, 0x58, 0x8f, 0x0d, 0x32, 0xd4, 0x24, 0xbd, 0xed, 0x6e, 0xd3, 0xd9, - 0xa7, 0xd5, 0xc2, 0xf9, 0xd9, 0xfa, 0xbc, 0x6c, 0xe0, 0x79, 0xdf, 0xf6, 0xd8, 0x87, 0xfa, 0xbb, - 0x09, 0x28, 0x44, 0x50, 0xe8, 0x55, 0x00, 0xdf, 0x1d, 0x79, 0xbe, 0xe6, 0x52, 0xea, 0x73, 0x67, - 0x15, 0x71, 0x9e, 0x53, 0x30, 0xa5, 0x3e, 0x2a, 0xc3, 0x35, 0x83, 0xb8, 0xbe, 0x66, 0x79, 0xde, - 0x88, 0xb8, 0x9a, 0x37, 0xda, 0xfb, 0x92, 0x18, 0x3e, 0x77, 0x5c, 0x11, 0x2f, 0x33, 0x56, 0x93, - 0x73, 0xba, 0x82, 0x81, 0xee, 0xc3, 0xf5, 0x28, 0x7e, 0x38, 0xda, 0xb3, 0x2d, 0x43, 0x63, 0x93, - 0x99, 0xe2, 0x22, 0xd7, 0x26, 0x22, 0x1d, 0xce, 0x7b, 0x44, 0xc6, 0xea, 0x4f, 0x13, 0xa0, 0x60, - 0x7d, 0xdf, 0xdf, 0x21, 0x83, 0x3d, 0xe2, 0x76, 0x7d, 0xdd, 0x1f, 0x79, 0xe8, 0x3a, 0x64, 0x6d, - 0xa2, 0x9b, 0xc4, 0xe5, 0x46, 0xe5, 0xb0, 0x6c, 0xa1, 0x5d, 0xb6, 0x83, 0x75, 0xe3, 0x40, 0xdf, - 0xb3, 0x6c, 0xcb, 0x1f, 0x73, 0x53, 0x16, 0xe3, 0x97, 0xf0, 0xac, 0xce, 0x32, 0x8e, 0x08, 0xe2, - 0x29, 0x35, 0x68, 0x15, 0xe6, 0x07, 0xc4, 0xf3, 0xf4, 0x3e, 0xe1, 0x96, 0xe6, 0x71, 0xd0, 0x54, - 0x3f, 0x86, 0x62, 0x54, 0x0e, 0x15, 0x60, 0x7e, 0xb7, 0xf5, 0xa8, 0xd5, 0x7e, 0xd2, 0x52, 0xe6, - 0xd0, 0x12, 0x14, 0x76, 0x5b, 0xb8, 0x51, 0xa9, 0x6d, 0x55, 0xaa, 0xdb, 0x0d, 0x25, 0x81, 0x16, - 0x20, 0x3f, 0x69, 0x26, 0xd5, 0x3f, 0x4d, 0x00, 0x30, 0x77, 0xcb, 0x41, 0x7d, 0x04, 0x19, 0xcf, - 0xd7, 0x7d, 0xb1, 0x2a, 0x17, 0xef, 0xbd, 0x71, 0xd1, 0x1c, 0x4a, 0x7b, 0xd9, 0x3f, 0x82, 0x85, - 0x48, 0xd4, 0xc2, 0xe4, 0x94, 0x85, 0x2c, 0x40, 0xe8, 0xa6, 0xe9, 0x4a, 0xc3, 0xf9, 0xb7, 0xfa, - 0x31, 0x64, 0xb8, 0xf4, 0xb4, 0xb9, 0x39, 0x48, 0xd7, 0xd9, 0x57, 0x02, 0xe5, 0x21, 0x83, 0x1b, - 0x95, 0xfa, 0x17, 0x4a, 0x12, 0x29, 0x50, 0xac, 0x37, 0xbb, 0xb5, 0x76, 0xab, 0xd5, 0xa8, 0xf5, - 0x1a, 0x75, 0x25, 0xa5, 0xde, 0x82, 0x4c, 0x73, 0xc0, 0x34, 0xdf, 0x64, 0x4b, 0x7e, 0x9f, 0xb8, - 0xc4, 0x31, 0x82, 0x9d, 0x34, 0x21, 0xa8, 0x3f, 0xc9, 0x43, 0x66, 0x87, 0x8e, 0x1c, 0x1f, 0xdd, - 0x8b, 0x84, 0xad, 0xc5, 0xf8, 0x0c, 0x81, 0x03, 0xcb, 0xbd, 0xf1, 0x90, 0xc8, 0xb0, 0x76, 0x1d, - 0xb2, 0x62, 0x73, 0xc8, 0xe1, 0xc8, 0x16, 0xa3, 0xfb, 0xba, 0xdb, 0x27, 0xbe, 0x1c, 0x8f, 0x6c, - 0xa1, 0xb7, 0xd8, 0x89, 0xa5, 0x9b, 0xd4, 0xb1, 0xc7, 0x7c, 0x0f, 0xe5, 0xc4, 0xb1, 0x84, 0x89, - 0x6e, 0xb6, 0x1d, 0x7b, 0x8c, 0x43, 0x2e, 0xda, 0x82, 0xe2, 0x9e, 0xe5, 0x98, 0x1a, 0x1d, 0x8a, - 0x20, 0x9f, 0xb9, 0x78, 0xc7, 0x09, 0xab, 0xaa, 0x96, 0x63, 0xb6, 0x05, 0x18, 0x17, 0xf6, 0x26, - 0x0d, 0xd4, 0x82, 0xc5, 0x23, 0x6a, 0x8f, 0x06, 0x24, 0xd4, 0x95, 0xe5, 0xba, 0xde, 0xbc, 0x58, - 0xd7, 0x63, 0x8e, 0x0f, 0xb4, 0x2d, 0x1c, 0x45, 0x9b, 0xe8, 0x11, 0x2c, 0xf8, 0x83, 0xe1, 0xbe, - 0x17, 0xaa, 0x9b, 0xe7, 0xea, 0xbe, 0x77, 0x89, 0xc3, 0x18, 0x3c, 0xd0, 0x56, 0xf4, 0x23, 0xad, - 0xd2, 0x6f, 0xa4, 0xa0, 0x10, 0xb1, 0x1c, 0x75, 0xa1, 0x30, 0x74, 0xe9, 0x50, 0xef, 0xf3, 0x83, - 0x4a, 0xce, 0xc5, 0xdd, 0x17, 0x1a, 0x75, 0xb9, 0x33, 0x11, 0xc4, 0x51, 0x2d, 0xea, 0x69, 0x12, - 0x0a, 0x11, 0x26, 0xba, 0x0d, 0x39, 0xdc, 0xc1, 0xcd, 0xc7, 0x95, 0x5e, 0x43, 0x99, 0x2b, 0xdd, - 0x3c, 0x39, 0xdd, 0x58, 0xe5, 0xda, 0xa2, 0x0a, 0x3a, 0xae, 0x75, 0xc4, 0x96, 0xde, 0x5b, 0x30, - 0x1f, 0x40, 0x13, 0xa5, 0x57, 0x4e, 0x4e, 0x37, 0x5e, 0x9e, 0x85, 0x46, 0x90, 0xb8, 0xbb, 0x55, - 0xc1, 0x8d, 0xba, 0x92, 0x8c, 0x47, 0xe2, 0xee, 0x81, 0xee, 0x12, 0x13, 0x7d, 0x0f, 0xb2, 0x12, - 0x98, 0x2a, 0x95, 0x4e, 0x4e, 0x37, 0xae, 0xcf, 0x02, 0x27, 0x38, 0xdc, 0xdd, 0xae, 0x3c, 0x6e, - 0x28, 0xe9, 0x78, 0x1c, 0xee, 0xda, 0xfa, 0x11, 0x41, 0x6f, 0x40, 0x46, 0xc0, 0x32, 0xa5, 0x1b, - 0x27, 0xa7, 0x1b, 0x2f, 0x3d, 0xa7, 0x8e, 0xa1, 0x4a, 0xab, 0xbf, 0xfd, 0x07, 0x6b, 0x73, 0x7f, - 0xf9, 0x87, 0x6b, 0xca, 0x2c, 0xbb, 0xf4, 0xdf, 0x09, 0x58, 0x98, 0x9a, 0x72, 0xa4, 0x42, 0xd6, - 0xa1, 0x06, 0x1d, 0x8a, 0xf3, 0x2b, 0x57, 0x85, 0xf3, 0xb3, 0xf5, 0x6c, 0x8b, 0xd6, 0xe8, 0x70, - 0x8c, 0x25, 0x07, 0x3d, 0x9a, 0x39, 0x81, 0xef, 0xbf, 0xe0, 0x7a, 0x8a, 0x3d, 0x83, 0x3f, 0x83, - 0x05, 0xd3, 0xb5, 0x8e, 0x88, 0xab, 0x19, 0xd4, 0xd9, 0xb7, 0xfa, 0xf2, 0x6c, 0x2a, 0xc5, 0xa6, - 0x89, 0x1c, 0x88, 0x8b, 0x42, 0xa0, 0xc6, 0xf1, 0xdf, 0xe1, 0xf4, 0x2d, 0x3d, 0x86, 0x62, 0x74, - 0x85, 0xb2, 0xe3, 0xc4, 0xb3, 0x7e, 0x85, 0xc8, 0x7c, 0x90, 0x67, 0x8f, 0x38, 0xcf, 0x28, 0x22, - 0x1b, 0x7c, 0x13, 0xd2, 0x03, 0x6a, 0x0a, 0x3d, 0x0b, 0xd5, 0x6b, 0x2c, 0x09, 0xf8, 0xa7, 0xb3, - 0xf5, 0x02, 0xf5, 0xca, 0x9b, 0x96, 0x4d, 0x76, 0xa8, 0x49, 0x30, 0x07, 0xa8, 0x47, 0x90, 0x66, - 0xa1, 0x02, 0xbd, 0x02, 0xe9, 0x6a, 0xb3, 0x55, 0x57, 0xe6, 0x4a, 0xcb, 0x27, 0xa7, 0x1b, 0x0b, - 0xdc, 0x25, 0x8c, 0xc1, 0xd6, 0x2e, 0x5a, 0x87, 0xec, 0xe3, 0xf6, 0xf6, 0xee, 0x0e, 0x5b, 0x5e, - 0xd7, 0x4e, 0x4e, 0x37, 0x96, 0x42, 0xb6, 0x70, 0x1a, 0x7a, 0x15, 0x32, 0xbd, 0x9d, 0xce, 0x66, - 0x57, 0x49, 0x96, 0xd0, 0xc9, 0xe9, 0xc6, 0x62, 0xc8, 0xe7, 0x36, 0x97, 0x96, 0xe5, 0xac, 0xe6, - 0x43, 0xba, 0xfa, 0xb3, 0x24, 0x2c, 0x60, 0x56, 0xf1, 0xb9, 0x7e, 0x87, 0xda, 0x96, 0x31, 0x46, - 0x1d, 0xc8, 0x1b, 0xd4, 0x31, 0xad, 0xc8, 0x9e, 0xba, 0x77, 0xc1, 0xa9, 0x3f, 0x91, 0x0a, 0x5a, - 0xb5, 0x40, 0x12, 0x4f, 0x94, 0xa0, 0x77, 0x21, 0x63, 0x12, 0x5b, 0x1f, 0xcb, 0xf4, 0xe3, 0x46, - 0x59, 0xd4, 0x94, 0xe5, 0xa0, 0xa6, 0x2c, 0xd7, 0x65, 0x4d, 0x89, 0x05, 0x8e, 0xa7, 0xd9, 0xfa, - 0x53, 0x4d, 0xf7, 0x7d, 0x32, 0x18, 0xfa, 0x22, 0xf7, 0x48, 0xe3, 0xc2, 0x40, 0x7f, 0x5a, 0x91, - 0x24, 0x74, 0x17, 0xb2, 0xc7, 0x96, 0x63, 0xd2, 0x63, 0x99, 0x5e, 0x5c, 0xa2, 0x54, 0x02, 0xd5, - 0x13, 0x76, 0xea, 0xce, 0x98, 0xc9, 0xfc, 0xdd, 0x6a, 0xb7, 0x1a, 0x81, 0xbf, 0x25, 0xbf, 0xed, - 0xb4, 0xa8, 0xc3, 0xf6, 0x0a, 0xb4, 0x5b, 0xda, 0x66, 0xa5, 0xb9, 0xbd, 0x8b, 0x99, 0xcf, 0x57, - 0x4e, 0x4e, 0x37, 0x94, 0x10, 0xb2, 0xa9, 0x5b, 0x36, 0xcb, 0x77, 0x6f, 0x40, 0xaa, 0xd2, 0xfa, - 0x42, 0x49, 0x96, 0x94, 0x93, 0xd3, 0x8d, 0x62, 0xc8, 0xae, 0x38, 0xe3, 0xc9, 0x36, 0x9a, 0xed, - 0x57, 0xfd, 0xdb, 0x14, 0x14, 0x77, 0x87, 0xa6, 0xee, 0x13, 0xb1, 0x26, 0xd1, 0x06, 0x14, 0x86, - 0xba, 0xab, 0xdb, 0x36, 0xb1, 0x2d, 0x6f, 0x20, 0xab, 0xe5, 0x28, 0x09, 0x7d, 0xf8, 0xa2, 0x6e, - 0xac, 0xe6, 0xd8, 0x3a, 0xfb, 0xf1, 0xbf, 0xac, 0x27, 0x02, 0x87, 0xee, 0xc2, 0xe2, 0xbe, 0xb0, - 0x56, 0xd3, 0x0d, 0x3e, 0xb1, 0x29, 0x3e, 0xb1, 0xe5, 0xb8, 0x89, 0x8d, 0x9a, 0x55, 0x96, 0x83, - 0xac, 0x70, 0x29, 0xbc, 0xb0, 0x1f, 0x6d, 0xa2, 0xfb, 0x30, 0x3f, 0xa0, 0x8e, 0xe5, 0x53, 0xf7, - 0xea, 0x59, 0x08, 0x90, 0xe8, 0x36, 0x2c, 0xb3, 0xc9, 0x0d, 0xec, 0xe1, 0x6c, 0x7e, 0x62, 0x25, - 0xf1, 0xd2, 0x40, 0x7f, 0x2a, 0x3b, 0xc4, 0x8c, 0x8c, 0xaa, 0x90, 0xa1, 0x2e, 0x4b, 0x89, 0xb2, - 0xdc, 0xdc, 0x77, 0xae, 0x34, 0x57, 0x34, 0xda, 0x4c, 0x06, 0x0b, 0x51, 0xf5, 0x03, 0x58, 0x98, - 0x1a, 0x04, 0xcb, 0x04, 0x3a, 0x95, 0xdd, 0x6e, 0x43, 0x99, 0x43, 0x45, 0xc8, 0xd5, 0xda, 0xad, - 0x5e, 0xb3, 0xb5, 0xcb, 0x52, 0x99, 0x22, 0xe4, 0x70, 0x7b, 0x7b, 0xbb, 0x5a, 0xa9, 0x3d, 0x52, - 0x92, 0x6a, 0x19, 0x0a, 0x11, 0x6d, 0x68, 0x11, 0xa0, 0xdb, 0x6b, 0x77, 0xb4, 0xcd, 0x26, 0xee, - 0xf6, 0x44, 0x22, 0xd4, 0xed, 0x55, 0x70, 0x4f, 0x12, 0x12, 0xea, 0x7f, 0x24, 0x83, 0x19, 0x95, - 0xb9, 0x4f, 0x75, 0x3a, 0xf7, 0xb9, 0xc4, 0x78, 0x99, 0xfd, 0x4c, 0x1a, 0x61, 0x0e, 0xf4, 0x21, - 0x00, 0x5f, 0x38, 0xc4, 0xd4, 0x74, 0x5f, 0x4e, 0x7c, 0xe9, 0x39, 0x27, 0xf7, 0x82, 0x4b, 0x1b, - 0x9c, 0x97, 0xe8, 0x8a, 0x8f, 0x3e, 0x81, 0xa2, 0x41, 0x07, 0x43, 0x9b, 0x48, 0xe1, 0xd4, 0x95, - 0xc2, 0x85, 0x10, 0x5f, 0xf1, 0xa3, 0xd9, 0x57, 0x7a, 0x3a, 0x3f, 0xfc, 0xcd, 0x44, 0xe0, 0x99, - 0x98, 0x84, 0xab, 0x08, 0xb9, 0xdd, 0x4e, 0xbd, 0xd2, 0x6b, 0xb6, 0x1e, 0x2a, 0x09, 0x04, 0x90, - 0xe5, 0xae, 0xae, 0x2b, 0x49, 0x96, 0x28, 0xd6, 0xda, 0x3b, 0x9d, 0xed, 0x06, 0x4f, 0xb9, 0xd0, - 0x0a, 0x28, 0x81, 0xb3, 0x35, 0xee, 0xc8, 0x46, 0x5d, 0x49, 0xa3, 0x6b, 0xb0, 0x14, 0x52, 0xa5, - 0x64, 0x06, 0x5d, 0x07, 0x14, 0x12, 0x27, 0x2a, 0xb2, 0xea, 0xaf, 0xc1, 0x52, 0x8d, 0x3a, 0xbe, - 0x6e, 0x39, 0x61, 0x12, 0x7d, 0x8f, 0x0d, 0x5a, 0x92, 0x34, 0x4b, 0x5e, 0x76, 0x54, 0x97, 0xce, - 0xcf, 0xd6, 0x0b, 0x21, 0xb4, 0x59, 0x67, 0x23, 0x0d, 0x1a, 0x26, 0xdb, 0xbf, 0x43, 0xcb, 0xe4, - 0xce, 0xcd, 0x54, 0xe7, 0xcf, 0xcf, 0xd6, 0x53, 0x9d, 0x66, 0x1d, 0x33, 0x1a, 0x7a, 0x05, 0xf2, - 0xe4, 0xa9, 0xe5, 0x6b, 0x06, 0x8b, 0xe1, 0xcc, 0x81, 0x19, 0x9c, 0x63, 0x84, 0x1a, 0x0b, 0xd9, - 0x55, 0x80, 0x0e, 0x75, 0x7d, 0xd9, 0xf3, 0x7b, 0x90, 0x19, 0x52, 0x97, 0x97, 0xe7, 0x17, 0x5e, - 0x1a, 0x31, 0xb8, 0x58, 0xa8, 0x58, 0x80, 0xd5, 0xbf, 0x4a, 0x02, 0xf4, 0x74, 0xef, 0x50, 0x2a, - 0x79, 0x00, 0xf9, 0xf0, 0x02, 0x4e, 0xd6, 0xf9, 0x97, 0xce, 0x76, 0x08, 0x46, 0xf7, 0x83, 0xc5, - 0x26, 0xca, 0x83, 0xd8, 0x3a, 0x2d, 0xe8, 0x28, 0x2e, 0xc3, 0x9e, 0xae, 0x01, 0xd8, 0x91, 0x48, - 0x5c, 0x57, 0xce, 0x3c, 0xfb, 0x44, 0x35, 0x7e, 0x2c, 0x08, 0xa7, 0xc9, 0x04, 0x33, 0xf6, 0x66, - 0x63, 0x66, 0x46, 0xb6, 0xe6, 0xf0, 0x44, 0x0e, 0x7d, 0x06, 0x05, 0x36, 0x6e, 0xcd, 0xe3, 0x3c, - 0x99, 0x5b, 0x5e, 0xe8, 0x2a, 0xa1, 0x01, 0xc3, 0x30, 0xfc, 0xae, 0x2a, 0xb0, 0xe8, 0x8e, 0x1c, - 0x36, 0x6c, 0xa9, 0x43, 0xfd, 0x93, 0x24, 0xbc, 0xdc, 0x22, 0xfe, 0x31, 0x75, 0x0f, 0x2b, 0xbe, - 0xaf, 0x1b, 0x07, 0x03, 0xe2, 0x48, 0x27, 0x47, 0x32, 0xeb, 0xc4, 0x54, 0x66, 0xbd, 0x0a, 0xf3, - 0xba, 0x6d, 0xe9, 0x1e, 0x11, 0xe9, 0x48, 0x1e, 0x07, 0x4d, 0x96, 0xff, 0xb3, 0x6a, 0x82, 0x78, - 0x1e, 0x11, 0x05, 0x7e, 0x1e, 0x4f, 0x08, 0xe8, 0x47, 0x70, 0x5d, 0x26, 0x1e, 0x7a, 0xd8, 0x15, - 0xcb, 0x6c, 0x83, 0x9b, 0xc2, 0x46, 0x6c, 0x79, 0x13, 0x6f, 0x9c, 0xcc, 0x4c, 0x26, 0xe4, 0xf6, - 0xd0, 0x97, 0x79, 0xce, 0x8a, 0x19, 0xc3, 0x2a, 0x3d, 0x84, 0x1b, 0x17, 0x8a, 0x7c, 0xab, 0x0b, - 0x84, 0x7f, 0x48, 0x02, 0x34, 0x3b, 0x95, 0x1d, 0xe9, 0xa4, 0x3a, 0x64, 0xf7, 0xf5, 0x81, 0x65, - 0x8f, 0x2f, 0x8b, 0x53, 0x13, 0x7c, 0xb9, 0x22, 0xdc, 0xb1, 0xc9, 0x65, 0xb0, 0x94, 0xe5, 0xc5, - 0xcd, 0x68, 0xcf, 0x21, 0x7e, 0x58, 0xdc, 0xf0, 0x16, 0x33, 0xc3, 0xd5, 0x9d, 0x70, 0x81, 0x89, - 0x06, 0x9b, 0x80, 0xbe, 0xee, 0x93, 0x63, 0x7d, 0x1c, 0x04, 0x17, 0xd9, 0x44, 0x5b, 0xfc, 0x9a, - 0x8e, 0xb8, 0x47, 0xc4, 0x5c, 0xcd, 0x70, 0xa7, 0x5e, 0x65, 0x0f, 0x96, 0x70, 0xe1, 0xbb, 0x50, - 0xba, 0xf4, 0x31, 0x4f, 0x6c, 0x26, 0xac, 0x6f, 0xe5, 0xa3, 0x3b, 0xb0, 0x30, 0x35, 0xce, 0xe7, - 0xaa, 0xca, 0x66, 0xe7, 0xf1, 0x7b, 0x4a, 0x5a, 0x7e, 0x7d, 0xa0, 0x64, 0xd5, 0x3f, 0x4a, 0x89, - 0x70, 0x20, 0xbd, 0x1a, 0x7f, 0x3d, 0x9d, 0xe3, 0x9b, 0xd8, 0xa0, 0xb6, 0xdc, 0xa6, 0x6f, 0x5e, - 0x1e, 0x25, 0x58, 0x95, 0xc2, 0xe1, 0x38, 0x14, 0x44, 0xeb, 0x50, 0x10, 0xab, 0x58, 0x63, 0xdb, - 0x82, 0xbb, 0x75, 0x01, 0x83, 0x20, 0x31, 0x49, 0x74, 0x0b, 0x16, 0xf9, 0x2d, 0x84, 0x77, 0x40, - 0x4c, 0x81, 0x49, 0x73, 0xcc, 0x42, 0x48, 0xe5, 0xb0, 0x1d, 0x28, 0x4a, 0x82, 0xc6, 0x33, 0xd4, - 0x0c, 0x37, 0xe8, 0xf6, 0x55, 0x06, 0x09, 0x11, 0x9e, 0xb8, 0x16, 0x86, 0x93, 0x86, 0x5a, 0x87, - 0x5c, 0x60, 0x2c, 0x5a, 0x85, 0x54, 0xaf, 0xd6, 0x51, 0xe6, 0x4a, 0x4b, 0x27, 0xa7, 0x1b, 0x85, - 0x80, 0xdc, 0xab, 0x75, 0x18, 0x67, 0xb7, 0xde, 0x51, 0x12, 0xd3, 0x9c, 0xdd, 0x7a, 0xa7, 0x94, - 0x66, 0x99, 0x92, 0xba, 0x0f, 0x85, 0x48, 0x0f, 0xe8, 0x75, 0x98, 0x6f, 0xb6, 0x1e, 0xe2, 0x46, - 0xb7, 0xab, 0xcc, 0x95, 0xae, 0x9f, 0x9c, 0x6e, 0xa0, 0x08, 0xb7, 0xe9, 0xf4, 0xd9, 0xfc, 0xa0, - 0x57, 0x21, 0xbd, 0xd5, 0x66, 0x27, 0xb0, 0x48, 0x89, 0x23, 0x88, 0x2d, 0xea, 0xf9, 0xa5, 0x6b, - 0x32, 0x05, 0x8b, 0x2a, 0x56, 0x7f, 0x2f, 0x01, 0x59, 0xb1, 0x99, 0x62, 0x27, 0xaa, 0x02, 0xf3, - 0x41, 0xbd, 0x2a, 0xca, 0x95, 0x37, 0x2f, 0x2e, 0x2d, 0xca, 0xb2, 0x12, 0x10, 0xcb, 0x2f, 0x90, - 0x2b, 0x7d, 0x04, 0xc5, 0x28, 0xe3, 0x5b, 0x2d, 0xbe, 0x1f, 0x41, 0x81, 0xad, 0xef, 0xa0, 0xc4, - 0xb8, 0x07, 0x59, 0x11, 0x10, 0xc2, 0x13, 0xe1, 0xe2, 0x3a, 0x47, 0x22, 0xd1, 0x03, 0x98, 0x17, - 0xb5, 0x51, 0x70, 0x4d, 0xb9, 0x76, 0xf9, 0x2e, 0xc2, 0x01, 0x5c, 0xfd, 0x0c, 0xd2, 0x1d, 0x42, - 0x5c, 0xe6, 0x7b, 0x87, 0x9a, 0x64, 0x72, 0x88, 0xca, 0xb2, 0xce, 0x24, 0xcd, 0x3a, 0x2b, 0xeb, - 0x4c, 0xd2, 0x34, 0xc3, 0x8b, 0x98, 0x64, 0xe4, 0x22, 0xa6, 0x07, 0xc5, 0x27, 0xc4, 0xea, 0x1f, - 0xf8, 0xc4, 0xe4, 0x8a, 0xde, 0x81, 0xf4, 0x90, 0x84, 0xc6, 0xaf, 0xc6, 0x2e, 0x30, 0x42, 0x5c, - 0xcc, 0x51, 0x2c, 0x8e, 0x1c, 0x73, 0x69, 0x79, 0xb7, 0x2e, 0x5b, 0xea, 0xdf, 0x27, 0x61, 0xb1, - 0xe9, 0x79, 0x23, 0xdd, 0x31, 0x82, 0xfc, 0xea, 0xd3, 0xe9, 0xfc, 0x2a, 0xf6, 0x11, 0x62, 0x5a, - 0x64, 0xfa, 0x7e, 0x49, 0x9e, 0x71, 0xc9, 0xf0, 0x8c, 0x53, 0xff, 0x3d, 0x11, 0x5c, 0x22, 0xdd, - 0x8a, 0x6c, 0xf7, 0xd2, 0xea, 0xc9, 0xe9, 0xc6, 0x4a, 0x54, 0x13, 0xd9, 0x75, 0x0e, 0x1d, 0x7a, - 0xec, 0xa0, 0xd7, 0x20, 0x83, 0x1b, 0xad, 0xc6, 0x13, 0x25, 0x21, 0x96, 0xe7, 0x14, 0x08, 0x13, - 0x87, 0x1c, 0x33, 0x4d, 0x9d, 0x46, 0xab, 0xce, 0xf2, 0xa1, 0x64, 0x8c, 0xa6, 0x0e, 0x71, 0x4c, - 0xcb, 0xe9, 0xa3, 0xd7, 0x21, 0xdb, 0xec, 0x76, 0x77, 0x79, 0x99, 0xff, 0xf2, 0xc9, 0xe9, 0xc6, - 0xb5, 0x29, 0x14, 0xbf, 0x40, 0x34, 0x19, 0x88, 0x15, 0x23, 0x2c, 0x53, 0x8a, 0x01, 0xb1, 0x2c, - 0x57, 0x80, 0x70, 0xbb, 0x57, 0xe9, 0xb1, 0x0a, 0xff, 0x79, 0x10, 0xa6, 0xec, 0xaf, 0xdc, 0x6e, - 0xff, 0x9c, 0x04, 0xa5, 0x62, 0x18, 0x64, 0xe8, 0x33, 0xbe, 0xac, 0xff, 0x7a, 0x90, 0x1b, 0xb2, - 0x2f, 0x8b, 0x04, 0xb9, 0xcc, 0x83, 0xd8, 0x67, 0xb4, 0x19, 0xb9, 0x32, 0xa6, 0x36, 0xa9, 0x98, - 0x03, 0xcb, 0xf3, 0x2c, 0xea, 0x08, 0x1a, 0x0e, 0x35, 0x95, 0xfe, 0x33, 0x01, 0xd7, 0x62, 0x10, - 0xe8, 0x0e, 0xa4, 0x5d, 0x6a, 0x07, 0x73, 0x78, 0xf3, 0xa2, 0xfb, 0x41, 0x26, 0x8a, 0x39, 0x12, - 0xad, 0x01, 0xe8, 0x23, 0x9f, 0xea, 0xbc, 0x7f, 0x3e, 0x7b, 0x39, 0x1c, 0xa1, 0xa0, 0x27, 0x90, - 0xf5, 0x88, 0xe1, 0x92, 0x20, 0xe3, 0xfd, 0xec, 0xff, 0x6a, 0x7d, 0xb9, 0xcb, 0xd5, 0x60, 0xa9, - 0xae, 0x54, 0x86, 0xac, 0xa0, 0xb0, 0x65, 0x6f, 0xea, 0xbe, 0x2e, 0x6f, 0x8f, 0xf9, 0x37, 0x5b, - 0x4d, 0xba, 0xdd, 0x0f, 0x56, 0x93, 0x6e, 0xf7, 0xd5, 0xbf, 0x49, 0x02, 0x34, 0x9e, 0xfa, 0xc4, - 0x75, 0x74, 0xbb, 0x56, 0x41, 0x8d, 0x48, 0xf4, 0x17, 0xa3, 0x7d, 0x3b, 0xf6, 0x4a, 0x3c, 0x94, - 0x28, 0xd7, 0x2a, 0x31, 0xf1, 0xff, 0x06, 0xa4, 0x46, 0xae, 0x7c, 0x19, 0x15, 0xd9, 0xea, 0x2e, - 0xde, 0xc6, 0x8c, 0x86, 0x1a, 0x93, 0xb0, 0x95, 0xba, 0xf8, 0xfd, 0x33, 0xd2, 0x41, 0x6c, 0xe8, - 0x62, 0x3b, 0xdf, 0xd0, 0x35, 0x83, 0xc8, 0x93, 0xa3, 0x28, 0x76, 0x7e, 0xad, 0x52, 0x23, 0xae, - 0x8f, 0xb3, 0x86, 0xce, 0xfe, 0x7f, 0xa7, 0xf8, 0xf6, 0x0e, 0xc0, 0x64, 0x68, 0x68, 0x0d, 0x32, - 0xb5, 0xcd, 0x6e, 0x77, 0x5b, 0x99, 0x13, 0x01, 0x7c, 0xc2, 0xe2, 0x64, 0xf5, 0x2f, 0x92, 0x90, - 0xab, 0x55, 0xe4, 0xb1, 0x5a, 0x03, 0x85, 0x47, 0x25, 0x7e, 0xe7, 0x4e, 0x9e, 0x0e, 0x2d, 0x77, - 0x2c, 0x03, 0xcb, 0x25, 0xa5, 0xe7, 0x22, 0x13, 0x61, 0x56, 0x37, 0xb8, 0x00, 0xc2, 0x50, 0x24, - 0xd2, 0x09, 0x9a, 0xa1, 0x07, 0x31, 0x7e, 0xed, 0x72, 0x67, 0x89, 0x22, 0x62, 0xd2, 0xf6, 0x70, - 0x21, 0x50, 0x52, 0xd3, 0x3d, 0xf4, 0x21, 0x2c, 0x79, 0x56, 0xdf, 0xb1, 0x9c, 0xbe, 0x16, 0x38, - 0x8f, 0x3f, 0x00, 0x54, 0x97, 0xcf, 0xcf, 0xd6, 0x17, 0xba, 0x82, 0x25, 0x7d, 0xb8, 0x20, 0x91, - 0x35, 0xee, 0x4a, 0xf4, 0x01, 0x2c, 0x46, 0x44, 0x99, 0x17, 0x85, 0xdb, 0x95, 0xf3, 0xb3, 0xf5, - 0x62, 0x28, 0xf9, 0x88, 0x8c, 0x71, 0x31, 0x14, 0x7c, 0x44, 0xf8, 0x2d, 0xc9, 0x3e, 0x75, 0x0d, - 0xa2, 0xb9, 0x7c, 0x4f, 0xf3, 0x13, 0x3c, 0x8d, 0x0b, 0x9c, 0x26, 0xb6, 0xb9, 0xfa, 0x18, 0xae, - 0xb5, 0x5d, 0xe3, 0x80, 0x78, 0xbe, 0x70, 0x85, 0xf4, 0xe2, 0x67, 0x70, 0xd3, 0xd7, 0xbd, 0x43, - 0xed, 0xc0, 0xf2, 0x7c, 0xea, 0x8e, 0x35, 0x97, 0xf8, 0xc4, 0x61, 0x7c, 0x8d, 0xbf, 0x1a, 0xca, - 0x6b, 0xac, 0x1b, 0x0c, 0xb3, 0x25, 0x20, 0x38, 0x40, 0x6c, 0x33, 0x80, 0xda, 0x84, 0x22, 0x2b, - 0x26, 0xea, 0x64, 0x5f, 0x1f, 0xd9, 0x3e, 0x1b, 0x3d, 0xd8, 0xb4, 0xaf, 0xbd, 0xf0, 0x31, 0x95, - 0xb7, 0x69, 0x5f, 0x7c, 0xaa, 0x3f, 0x04, 0xa5, 0x6e, 0x79, 0x43, 0xdd, 0x37, 0x0e, 0x82, 0xfb, - 0x39, 0x54, 0x07, 0xe5, 0x80, 0xe8, 0xae, 0xbf, 0x47, 0x74, 0x5f, 0x1b, 0x12, 0xd7, 0xa2, 0xe6, - 0xd5, 0xb3, 0xbc, 0x14, 0x8a, 0x74, 0xb8, 0x84, 0xfa, 0x5f, 0x09, 0x00, 0xac, 0xef, 0x07, 0x19, - 0xd9, 0xf7, 0x61, 0xd9, 0x73, 0xf4, 0xa1, 0x77, 0x40, 0x7d, 0xcd, 0x72, 0x7c, 0xe2, 0x1e, 0xe9, - 0xb6, 0xbc, 0x66, 0x51, 0x02, 0x46, 0x53, 0xd2, 0xd1, 0x3b, 0x80, 0x0e, 0x09, 0x19, 0x6a, 0xd4, - 0x36, 0xb5, 0x80, 0x29, 0xde, 0x34, 0xd3, 0x58, 0x61, 0x9c, 0xb6, 0x6d, 0x76, 0x03, 0x3a, 0xaa, - 0xc2, 0x1a, 0x1b, 0x3e, 0x71, 0x7c, 0xd7, 0x22, 0x9e, 0xb6, 0x4f, 0x5d, 0xcd, 0xb3, 0xe9, 0xb1, - 0xb6, 0x4f, 0x6d, 0x9b, 0x1e, 0x13, 0x37, 0xb8, 0xc1, 0x2a, 0xd9, 0xb4, 0xdf, 0x10, 0xa0, 0x4d, - 0xea, 0x76, 0x6d, 0x7a, 0xbc, 0x19, 0x20, 0x58, 0xda, 0x36, 0x19, 0xb3, 0x6f, 0x19, 0x87, 0x41, - 0xda, 0x16, 0x52, 0x7b, 0x96, 0x71, 0x88, 0x5e, 0x87, 0x05, 0x62, 0x13, 0x7e, 0x91, 0x21, 0x50, - 0x19, 0x8e, 0x2a, 0x06, 0x44, 0x06, 0x52, 0x3f, 0x07, 0xa5, 0xe1, 0x18, 0xee, 0x78, 0x18, 0x99, - 0xf3, 0x77, 0x00, 0xb1, 0x20, 0xa9, 0xd9, 0xd4, 0x38, 0xd4, 0x06, 0xba, 0xa3, 0xf7, 0x99, 0x5d, - 0xe2, 0xa9, 0x49, 0x61, 0x9c, 0x6d, 0x6a, 0x1c, 0xee, 0x48, 0xba, 0xfa, 0x21, 0x40, 0x77, 0xe8, - 0x12, 0xdd, 0x6c, 0xb3, 0x6c, 0x82, 0xb9, 0x8e, 0xb7, 0x34, 0x53, 0x3e, 0xd5, 0x51, 0x57, 0x6e, - 0x75, 0x45, 0x30, 0xea, 0x21, 0x5d, 0xfd, 0x45, 0xb8, 0xd6, 0xb1, 0x75, 0x83, 0x3f, 0x5b, 0x77, - 0xc2, 0xb7, 0x13, 0xf4, 0x00, 0xb2, 0x02, 0x2a, 0x67, 0x32, 0x76, 0xbb, 0x4d, 0xfa, 0xdc, 0x9a, - 0xc3, 0x12, 0x5f, 0x2d, 0x02, 0x4c, 0xf4, 0xa8, 0x7f, 0x96, 0x80, 0x7c, 0xa8, 0x1f, 0x6d, 0x00, - 0x2b, 0xe5, 0xd9, 0xf2, 0xb6, 0x1c, 0x59, 0x7b, 0xe7, 0x71, 0x94, 0x84, 0x9a, 0x50, 0x18, 0x86, - 0xd2, 0x97, 0xe6, 0x73, 0x31, 0x56, 0xe3, 0xa8, 0x2c, 0xfa, 0x08, 0xf2, 0xc1, 0xdb, 0x68, 0x10, - 0x61, 0x2f, 0x7f, 0x4a, 0x9d, 0xc0, 0xd5, 0x4f, 0x01, 0x7e, 0x40, 0x2d, 0xa7, 0x47, 0x0f, 0x89, - 0xc3, 0xdf, 0xfa, 0x58, 0x4d, 0x48, 0x02, 0x2f, 0xca, 0x16, 0x2f, 0xc8, 0xc5, 0x14, 0x84, 0x4f, - 0x5e, 0xa2, 0xa9, 0xfe, 0x75, 0x12, 0xb2, 0x98, 0x52, 0xbf, 0x56, 0x41, 0x1b, 0x90, 0x95, 0x71, - 0x82, 0x9f, 0x3f, 0xd5, 0xfc, 0xf9, 0xd9, 0x7a, 0x46, 0x04, 0x88, 0x8c, 0xc1, 0x23, 0x43, 0x24, - 0x82, 0x27, 0x2f, 0x8a, 0xe0, 0xe8, 0x0e, 0x14, 0x25, 0x48, 0x3b, 0xd0, 0xbd, 0x03, 0x51, 0xa0, - 0x55, 0x17, 0xcf, 0xcf, 0xd6, 0x41, 0x20, 0xb7, 0x74, 0xef, 0x00, 0x83, 0x40, 0xb3, 0x6f, 0xd4, - 0x80, 0xc2, 0x97, 0xd4, 0x72, 0x34, 0x9f, 0x0f, 0x42, 0x5e, 0xf9, 0xc5, 0xce, 0xe3, 0x64, 0xa8, - 0xf2, 0xe1, 0x1b, 0xbe, 0x9c, 0x0c, 0xbe, 0x01, 0x0b, 0x2e, 0xa5, 0xbe, 0x08, 0x5b, 0x16, 0x75, - 0xe4, 0x6d, 0xc2, 0x46, 0xec, 0x25, 0x33, 0xa5, 0x3e, 0x96, 0x38, 0x5c, 0x74, 0x23, 0x2d, 0x74, - 0x07, 0x56, 0x6c, 0xdd, 0xf3, 0x35, 0x1e, 0xef, 0xcc, 0x89, 0xb6, 0x2c, 0xdf, 0x6a, 0x88, 0xf1, - 0x36, 0x39, 0x2b, 0x90, 0x50, 0xff, 0x31, 0x01, 0x05, 0x36, 0x18, 0x6b, 0xdf, 0x32, 0x58, 0x92, - 0xf7, 0xed, 0x73, 0x8f, 0x1b, 0x90, 0x32, 0x3c, 0x57, 0x3a, 0x95, 0x1f, 0xbe, 0xb5, 0x2e, 0xc6, - 0x8c, 0x86, 0x3e, 0x87, 0xac, 0xbc, 0xd5, 0x10, 0x69, 0x87, 0x7a, 0x75, 0x3a, 0x2a, 0x7d, 0x23, - 0xe5, 0xf8, 0x5a, 0x9e, 0x58, 0x27, 0x0e, 0x01, 0x1c, 0x25, 0xa1, 0xeb, 0x90, 0x34, 0x84, 0xbb, - 0xe4, 0x2f, 0x2b, 0x6a, 0x2d, 0x9c, 0x34, 0x1c, 0xf5, 0xef, 0x12, 0xb0, 0x30, 0xd9, 0xf0, 0x6c, - 0x05, 0xdc, 0x84, 0xbc, 0x37, 0xda, 0xf3, 0xc6, 0x9e, 0x4f, 0x06, 0xc1, 0x3b, 0x66, 0x48, 0x40, - 0x4d, 0xc8, 0xeb, 0x76, 0x9f, 0xba, 0x96, 0x7f, 0x30, 0x90, 0x95, 0x68, 0x7c, 0xaa, 0x10, 0xd5, - 0x59, 0xae, 0x04, 0x22, 0x78, 0x22, 0x1d, 0x9c, 0xfb, 0xe2, 0xb1, 0x9b, 0x9f, 0xfb, 0xaf, 0x41, - 0xd1, 0xd6, 0x07, 0xfc, 0x9a, 0xc7, 0xb7, 0x06, 0x62, 0x1c, 0x69, 0x5c, 0x90, 0xb4, 0x9e, 0x35, - 0x20, 0xaa, 0x0a, 0xf9, 0x50, 0x19, 0x5a, 0x82, 0x42, 0xa5, 0xd1, 0xd5, 0xee, 0xde, 0x7b, 0xa0, - 0x3d, 0xac, 0xed, 0x28, 0x73, 0x32, 0x37, 0xfd, 0xf3, 0x04, 0x2c, 0xc8, 0x70, 0x24, 0xf3, 0xfd, - 0xd7, 0x61, 0xde, 0xd5, 0xf7, 0xfd, 0xa0, 0x22, 0x49, 0x8b, 0x55, 0xcd, 0x22, 0x3c, 0xab, 0x48, - 0x18, 0x2b, 0xbe, 0x22, 0x89, 0xbc, 0xac, 0xa7, 0x2e, 0x7d, 0x59, 0x4f, 0xff, 0x5c, 0x5e, 0xd6, - 0xd5, 0x5f, 0x07, 0xd8, 0xb4, 0x6c, 0xd2, 0x13, 0x77, 0x4d, 0x71, 0xf5, 0x25, 0xcb, 0xe1, 0xe4, - 0x8d, 0x63, 0x90, 0xc3, 0x35, 0xeb, 0x98, 0xd1, 0x18, 0xab, 0x6f, 0x99, 0x72, 0x33, 0x72, 0xd6, - 0x43, 0xc6, 0xea, 0x5b, 0x66, 0xf8, 0x96, 0x94, 0xbe, 0xea, 0x2d, 0xe9, 0x34, 0x01, 0x4b, 0x32, - 0x77, 0x0d, 0xc3, 0xef, 0xdb, 0x90, 0x17, 0x69, 0xec, 0xa4, 0xa0, 0xe3, 0xaf, 0xc9, 0x02, 0xd7, - 0xac, 0xe3, 0x9c, 0x60, 0x37, 0x4d, 0xb4, 0x0e, 0x05, 0x09, 0x8d, 0xfc, 0x0a, 0x07, 0x04, 0xa9, - 0xc5, 0xcc, 0x7f, 0x0f, 0xd2, 0xfb, 0x96, 0x4d, 0xe4, 0x42, 0x8f, 0x0d, 0x00, 0x13, 0x07, 0x6c, - 0xcd, 0x61, 0x8e, 0xae, 0xe6, 0x82, 0xcb, 0x38, 0x6e, 0x9f, 0x2c, 0x3b, 0xa3, 0xf6, 0x89, 0x0a, - 0x74, 0xc6, 0x3e, 0x81, 0x63, 0xf6, 0x09, 0xb6, 0xb0, 0x4f, 0x42, 0xa3, 0xf6, 0x09, 0xd2, 0xcf, - 0xc5, 0xbe, 0x6d, 0xb8, 0x5e, 0xb5, 0x75, 0xe3, 0xd0, 0xb6, 0x3c, 0x9f, 0x98, 0xd1, 0x88, 0x71, - 0x0f, 0xb2, 0x53, 0x49, 0xe7, 0x65, 0x97, 0xb3, 0x12, 0xa9, 0xfe, 0x5b, 0x02, 0x8a, 0x5b, 0x44, - 0xb7, 0xfd, 0x83, 0xc9, 0xd5, 0x90, 0x4f, 0x3c, 0x5f, 0x1e, 0x56, 0xfc, 0x1b, 0xbd, 0x0f, 0xb9, - 0x30, 0x27, 0xb9, 0xf2, 0x95, 0x2c, 0x84, 0xa2, 0xfb, 0x30, 0xcf, 0xf6, 0x18, 0x1d, 0x05, 0xc5, - 0xce, 0x65, 0x0f, 0x30, 0x12, 0xc9, 0x0e, 0x19, 0x97, 0xf0, 0x24, 0x84, 0x2f, 0xa5, 0x0c, 0x0e, - 0x9a, 0xe8, 0xff, 0x43, 0x91, 0xbf, 0x1f, 0x04, 0x39, 0x57, 0xe6, 0x2a, 0x9d, 0x05, 0xf1, 0x04, - 0x28, 0xf2, 0xad, 0xff, 0x49, 0xc0, 0xca, 0x8e, 0x3e, 0xde, 0x23, 0x32, 0x6c, 0x10, 0x13, 0x13, - 0x83, 0xba, 0x26, 0xea, 0x44, 0xc3, 0xcd, 0x25, 0x2f, 0x8a, 0x71, 0xc2, 0xf1, 0x51, 0x27, 0x28, - 0xc0, 0x92, 0x91, 0x02, 0x6c, 0x05, 0x32, 0x0e, 0x75, 0x0c, 0x22, 0x63, 0x91, 0x68, 0xa8, 0x56, - 0x34, 0xd4, 0x94, 0xc2, 0xc7, 0x3e, 0xfe, 0x54, 0xd7, 0xa2, 0x7e, 0xd8, 0x1b, 0xfa, 0x1c, 0x4a, - 0xdd, 0x46, 0x0d, 0x37, 0x7a, 0xd5, 0xf6, 0x0f, 0xb5, 0x6e, 0x65, 0xbb, 0x5b, 0xb9, 0x77, 0x47, - 0xeb, 0xb4, 0xb7, 0xbf, 0xb8, 0x7b, 0xff, 0xce, 0xfb, 0x4a, 0xa2, 0xb4, 0x71, 0x72, 0xba, 0x71, - 0xb3, 0x55, 0xa9, 0x6d, 0x8b, 0x1d, 0xb3, 0x47, 0x9f, 0x76, 0x75, 0xdb, 0xd3, 0xef, 0xdd, 0xe9, - 0x50, 0x7b, 0xcc, 0x30, 0x6c, 0x59, 0x17, 0xa3, 0xe7, 0x55, 0xf4, 0x18, 0x4e, 0x5c, 0x78, 0x0c, - 0x4f, 0x4e, 0xf3, 0xe4, 0x05, 0xa7, 0xf9, 0x26, 0xac, 0x18, 0x2e, 0xf5, 0x3c, 0x8d, 0x65, 0xff, - 0xc4, 0x9c, 0xa9, 0x2f, 0x5e, 0x3a, 0x3f, 0x5b, 0x5f, 0xae, 0x31, 0x7e, 0x97, 0xb3, 0xa5, 0xfa, - 0x65, 0x23, 0x42, 0xe2, 0x3d, 0xa9, 0xbf, 0x9f, 0x62, 0x89, 0x94, 0x75, 0x64, 0xd9, 0xa4, 0x4f, - 0x3c, 0xf4, 0x18, 0x96, 0x0c, 0x97, 0x98, 0x2c, 0xad, 0xd7, 0xed, 0xe8, 0xaf, 0x39, 0xff, 0x5f, - 0x6c, 0x4e, 0x13, 0x0a, 0x96, 0x6b, 0xa1, 0x54, 0x77, 0x48, 0x0c, 0xbc, 0x68, 0x4c, 0xb5, 0xd1, - 0x97, 0xb0, 0xe4, 0x11, 0xdb, 0x72, 0x46, 0x4f, 0x35, 0x83, 0x3a, 0x3e, 0x79, 0x1a, 0xbc, 0x5b, - 0x5d, 0xa5, 0xb7, 0xdb, 0xd8, 0x66, 0x52, 0x35, 0x21, 0x54, 0x45, 0xe7, 0x67, 0xeb, 0x8b, 0xd3, - 0x34, 0xbc, 0x28, 0x35, 0xcb, 0x76, 0xa9, 0x05, 0x8b, 0xd3, 0xd6, 0xa0, 0x15, 0xb9, 0xf7, 0x79, - 0x08, 0x09, 0xf6, 0x36, 0xba, 0x09, 0x39, 0x97, 0xf4, 0x2d, 0xcf, 0x77, 0x85, 0x9b, 0x19, 0x27, - 0xa4, 0xb0, 0x9d, 0x2f, 0x7e, 0x8a, 0x53, 0xfa, 0x55, 0x98, 0xe9, 0x91, 0x6d, 0x16, 0xd3, 0xf2, - 0xf4, 0x3d, 0xa9, 0x32, 0x87, 0x83, 0x26, 0x5b, 0x83, 0x23, 0x2f, 0x4c, 0xd4, 0xf8, 0x37, 0xa3, - 0xf1, 0x8c, 0x42, 0xfe, 0x30, 0x89, 0xe7, 0x0c, 0xc1, 0x2f, 0x1c, 0xd3, 0x91, 0x5f, 0x38, 0xae, - 0x40, 0xc6, 0x26, 0x47, 0xc4, 0x16, 0x67, 0x39, 0x16, 0x8d, 0xdb, 0x77, 0xa0, 0x18, 0xfc, 0x94, - 0x8e, 0xff, 0x16, 0x20, 0x07, 0xe9, 0x5e, 0xa5, 0xfb, 0x48, 0x99, 0x43, 0x00, 0x59, 0xb1, 0x38, - 0xc5, 0x9b, 0x5a, 0xad, 0xdd, 0xda, 0x6c, 0x3e, 0x54, 0x92, 0xb7, 0x7f, 0x96, 0x82, 0x7c, 0xf8, - 0xaa, 0xc3, 0xce, 0x8e, 0x56, 0xe3, 0x49, 0xb0, 0xba, 0x43, 0x7a, 0x8b, 0x1c, 0xa3, 0xd7, 0x26, - 0xb7, 0x50, 0x9f, 0x8b, 0x67, 0xec, 0x90, 0x1d, 0xdc, 0x40, 0xbd, 0x01, 0xb9, 0x4a, 0xb7, 0xdb, - 0x7c, 0xd8, 0x6a, 0xd4, 0x95, 0xaf, 0x12, 0xa5, 0x97, 0x4e, 0x4e, 0x37, 0x96, 0x43, 0x50, 0xc5, - 0x13, 0x8b, 0x8f, 0xa3, 0x6a, 0xb5, 0x46, 0xa7, 0xd7, 0xa8, 0x2b, 0xcf, 0x92, 0xb3, 0x28, 0x7e, - 0xab, 0xc2, 0x7f, 0x8c, 0x92, 0xef, 0xe0, 0x46, 0xa7, 0x82, 0x59, 0x87, 0x5f, 0x25, 0xc5, 0xe5, - 0xd8, 0xa4, 0x47, 0x97, 0x0c, 0x75, 0x97, 0xf5, 0xb9, 0x16, 0xfc, 0x28, 0xeb, 0x59, 0x4a, 0xfc, - 0x60, 0x61, 0xf2, 0x44, 0x45, 0x74, 0x73, 0xcc, 0x7a, 0xe3, 0x6f, 0x83, 0x5c, 0x4d, 0x6a, 0xa6, - 0xb7, 0x2e, 0x8b, 0x3d, 0x4c, 0x8b, 0x0a, 0xf3, 0x78, 0xb7, 0xd5, 0x62, 0xa0, 0x67, 0xe9, 0x99, - 0xd1, 0xe1, 0x91, 0xc3, 0x2a, 0x66, 0x74, 0x0b, 0x72, 0xc1, 0xd3, 0xa1, 0xf2, 0x55, 0x7a, 0xc6, - 0xa0, 0x5a, 0xf0, 0xee, 0xc9, 0x3b, 0xdc, 0xda, 0xed, 0xf1, 0xdf, 0x8c, 0x3d, 0xcb, 0xcc, 0x76, - 0x78, 0x30, 0xf2, 0x4d, 0x7a, 0xec, 0xb0, 0x3d, 0x2b, 0xef, 0xe1, 0xbe, 0xca, 0x88, 0x4b, 0x8b, - 0x10, 0x23, 0x2f, 0xe1, 0xde, 0x80, 0x1c, 0x6e, 0xfc, 0x40, 0xfc, 0xbc, 0xec, 0x59, 0x76, 0x46, - 0x0f, 0x26, 0x5f, 0x12, 0x43, 0xf6, 0xd6, 0xc6, 0x9d, 0xad, 0x0a, 0x77, 0xf9, 0x2c, 0xaa, 0xed, - 0x0e, 0x0f, 0x74, 0x87, 0x98, 0x93, 0x5f, 0x6d, 0x84, 0xac, 0xdb, 0xbf, 0x04, 0xb9, 0x20, 0x33, - 0x45, 0x6b, 0x90, 0x7d, 0xd2, 0xc6, 0x8f, 0x1a, 0x58, 0x99, 0x13, 0x3e, 0x0c, 0x38, 0x4f, 0x44, - 0x4d, 0xb1, 0x01, 0xf3, 0x3b, 0x95, 0x56, 0xe5, 0x61, 0x03, 0x07, 0x57, 0xe4, 0x01, 0x40, 0xa6, - 0x57, 0x25, 0x45, 0x76, 0x10, 0xea, 0xac, 0xae, 0x7e, 0xfd, 0xcd, 0xda, 0xdc, 0x4f, 0xbf, 0x59, - 0x9b, 0x7b, 0x76, 0xbe, 0x96, 0xf8, 0xfa, 0x7c, 0x2d, 0xf1, 0x93, 0xf3, 0xb5, 0xc4, 0xbf, 0x9e, - 0xaf, 0x25, 0xf6, 0xb2, 0xfc, 0x10, 0xb8, 0xff, 0xbf, 0x01, 0x00, 0x00, 0xff, 0xff, 0xbf, 0x83, - 0x64, 0x72, 0x5a, 0x30, 0x00, 0x00, + 0x5d, 0xb7, 0xa9, 0xaa, 0xb6, 0xa7, 0x79, 0x20, 0x46, 0x2c, 0x00, 0x79, 0x05, 0xbb, 0x87, 0x90, + 0xd9, 0xc0, 0x0a, 0x21, 0xb1, 0x00, 0x09, 0xc1, 0x2a, 0x48, 0x2c, 0xb2, 0xe3, 0x01, 0x12, 0x7a, + 0x02, 0xc9, 0x10, 0x2f, 0xd8, 0x21, 0xd8, 0x3c, 0xb1, 0x01, 0x09, 0xdd, 0x8f, 0xaa, 0xae, 0xee, + 0x29, 0xdb, 0x13, 0xf2, 0x36, 0x76, 0xdd, 0x73, 0x7e, 0xe7, 0xdc, 0x73, 0xcf, 0xbd, 0xf7, 0xdc, + 0x73, 0xee, 0x6d, 0x28, 0xf8, 0xe3, 0x21, 0xf1, 0xca, 0x43, 0x97, 0xfa, 0x14, 0x21, 0x93, 0x1a, + 0x87, 0xc4, 0x2d, 0x7b, 0xc7, 0xba, 0x3b, 0x38, 0xb4, 0xfc, 0xf2, 0xd1, 0xdd, 0xd2, 0x7a, 0x9f, + 0xd2, 0xbe, 0x4d, 0xde, 0xe5, 0x88, 0xbd, 0xd1, 0xfe, 0xbb, 0xbe, 0x35, 0x20, 0x9e, 0xaf, 0x0f, + 0x86, 0x42, 0xa8, 0xb4, 0x36, 0x0b, 0x30, 0x47, 0xae, 0xee, 0x5b, 0xd4, 0x91, 0xfc, 0x95, 0x3e, + 0xed, 0x53, 0xfe, 0xf9, 0x2e, 0xfb, 0x12, 0x54, 0x75, 0x1d, 0xe6, 0x1f, 0x13, 0xd7, 0xb3, 0xa8, + 0x83, 0x56, 0x20, 0x63, 0x39, 0x26, 0x79, 0xba, 0x9a, 0xd8, 0x48, 0xbc, 0x95, 0xc6, 0xa2, 0xa1, + 0xde, 0x01, 0x68, 0xb2, 0x8f, 0x86, 0xe3, 0xbb, 0x63, 0xa4, 0x40, 0xea, 0x90, 0x8c, 0x39, 0x22, + 0x8f, 0xd9, 0x27, 0xa3, 0x1c, 0xe9, 0xf6, 0x6a, 0x52, 0x50, 0x8e, 0x74, 0x5b, 0xfd, 0x26, 0x01, + 0x85, 0x8a, 0xe3, 0x50, 0x9f, 0xf7, 0xee, 0x21, 0x04, 0x69, 0x47, 0x1f, 0x10, 0x29, 0xc4, 0xbf, + 0x51, 0x0d, 0xb2, 0xb6, 0xbe, 0x47, 0x6c, 0x6f, 0x35, 0xb9, 0x91, 0x7a, 0xab, 0x70, 0xef, 0xfb, + 0xe5, 0xe7, 0x87, 0x5c, 0x8e, 0x28, 0x29, 0x6f, 0x73, 0x34, 0x37, 0x02, 0x4b, 0x51, 0xf4, 0x29, + 0xcc, 0x5b, 0x8e, 0x69, 0x19, 0xc4, 0x5b, 0x4d, 0x73, 0x2d, 0x6b, 0x71, 0x5a, 0x26, 0xd6, 0x57, + 0xd3, 0x5f, 0x9f, 0xad, 0xcf, 0xe1, 0x40, 0xa8, 0xf4, 0x21, 0x14, 0x22, 0x6a, 0x63, 0xc6, 0xb6, + 0x02, 0x99, 0x23, 0xdd, 0x1e, 0x11, 0x39, 0x3a, 0xd1, 0xf8, 0x28, 0xf9, 0x20, 0xa1, 0x7e, 0x0e, + 0x2b, 0x2d, 0x7d, 0x40, 0xcc, 0x87, 0xc4, 0x21, 0xae, 0x65, 0x60, 0xe2, 0xd1, 0x91, 0x6b, 0x10, + 0x36, 0xd6, 0x43, 0xcb, 0x31, 0x83, 0xb1, 0xb2, 0xef, 0x78, 0x2d, 0x6a, 0x0d, 0x5e, 0xae, 0x5b, + 0x9e, 0xe1, 0x12, 0x9f, 0x7c, 0x6b, 0x25, 0xa9, 0x40, 0xc9, 0x59, 0x02, 0x96, 0x66, 0xa5, 0x7f, + 0x01, 0xae, 0x31, 0x17, 0x9b, 0x9a, 0x2b, 0x29, 0x9a, 0x37, 0x24, 0x06, 0x57, 0x56, 0xb8, 0xf7, + 0x56, 0x9c, 0x87, 0xe2, 0x46, 0xb2, 0x35, 0x87, 0x97, 0xb9, 0x9a, 0x80, 0xd0, 0x1d, 0x12, 0x03, + 0x19, 0x70, 0xdd, 0x94, 0x46, 0xcf, 0xa8, 0x4f, 0x72, 0xf5, 0xb1, 0xd3, 0x78, 0xc1, 0x30, 0xb7, + 0xe6, 0xf0, 0x4a, 0xa0, 0x2c, 0xda, 0x49, 0x15, 0x20, 0x17, 0xe8, 0x56, 0x7f, 0x9c, 0x80, 0x7c, + 0xc0, 0xf4, 0xd0, 0xdb, 0x90, 0x77, 0x74, 0x87, 0x6a, 0xc6, 0x70, 0xe4, 0xf1, 0x01, 0xa5, 0xaa, + 0xc5, 0xf3, 0xb3, 0xf5, 0x5c, 0x4b, 0x77, 0x68, 0xad, 0xb3, 0xeb, 0xe1, 0x1c, 0x63, 0xd7, 0x86, + 0x23, 0x0f, 0xbd, 0x06, 0xc5, 0x01, 0x19, 0x50, 0x77, 0xac, 0xed, 0x8d, 0x7d, 0xe2, 0x49, 0xb7, + 0x15, 0x04, 0xad, 0xca, 0x48, 0xe8, 0x13, 0x98, 0xef, 0x0b, 0x93, 0x56, 0x53, 0x7c, 0xf9, 0xbc, + 0x1e, 0x67, 0xfd, 0x8c, 0xd5, 0x38, 0x90, 0x51, 0x7f, 0x27, 0x01, 0x2b, 0x21, 0x95, 0xfc, 0xf2, + 0xc8, 0x72, 0xc9, 0x80, 0x38, 0xbe, 0x87, 0xde, 0x87, 0xac, 0x6d, 0x0d, 0x2c, 0xdf, 0x93, 0x3e, + 0x7f, 0x35, 0x4e, 0x6d, 0x38, 0x28, 0x2c, 0xc1, 0xa8, 0x02, 0x45, 0x97, 0x78, 0xc4, 0x3d, 0x12, + 0x2b, 0x5e, 0x7a, 0xf4, 0x0a, 0xe1, 0x29, 0x11, 0x75, 0x13, 0x72, 0x1d, 0x5b, 0xf7, 0xf7, 0xa9, + 0x3b, 0x40, 0x2a, 0x14, 0x75, 0xd7, 0x38, 0xb0, 0x7c, 0x62, 0xf8, 0x23, 0x37, 0xd8, 0x7d, 0x53, + 0x34, 0x74, 0x1d, 0x92, 0x54, 0x74, 0x94, 0xaf, 0x66, 0xcf, 0xcf, 0xd6, 0x93, 0xed, 0x2e, 0x4e, + 0x52, 0x4f, 0xfd, 0x18, 0x96, 0x3b, 0xf6, 0xa8, 0x6f, 0x39, 0x75, 0xe2, 0x19, 0xae, 0x35, 0x64, + 0xda, 0xd9, 0xaa, 0x64, 0x31, 0x2a, 0x58, 0x95, 0xec, 0x3b, 0xdc, 0xda, 0xc9, 0xc9, 0xd6, 0x56, + 0x7f, 0x2b, 0x09, 0xcb, 0x0d, 0xa7, 0x6f, 0x39, 0x24, 0x2a, 0x7d, 0x0b, 0x16, 0x09, 0x27, 0x6a, + 0x47, 0x22, 0xdc, 0x48, 0x3d, 0x0b, 0x82, 0x1a, 0xc4, 0xa0, 0xe6, 0x4c, 0x5c, 0xb8, 0x1b, 0x37, + 0xfc, 0xe7, 0xb4, 0xc7, 0x46, 0x87, 0x06, 0xcc, 0x0f, 0xf9, 0x20, 0x3c, 0x39, 0xbd, 0xb7, 0xe2, + 0x74, 0x3d, 0x37, 0xce, 0x20, 0x48, 0x48, 0xd9, 0xef, 0x12, 0x24, 0xfe, 0x24, 0x09, 0x4b, 0x2d, + 0x6a, 0x4e, 0xf9, 0xa1, 0x04, 0xb9, 0x03, 0xea, 0xf9, 0x91, 0x80, 0x18, 0xb6, 0xd1, 0x03, 0xc8, + 0x0d, 0xe5, 0xf4, 0xc9, 0xd9, 0xbf, 0x19, 0x6f, 0xb2, 0xc0, 0xe0, 0x10, 0x8d, 0x3e, 0x86, 0x7c, + 0xb0, 0x65, 0xd8, 0x68, 0x5f, 0x60, 0xe1, 0x4c, 0xf0, 0xe8, 0x13, 0xc8, 0x8a, 0x49, 0x58, 0x4d, + 0x73, 0xc9, 0x5b, 0x2f, 0xe4, 0x73, 0x2c, 0x85, 0xd0, 0x43, 0xc8, 0xf9, 0xb6, 0xa7, 0x59, 0xce, + 0x3e, 0x5d, 0xcd, 0x70, 0x05, 0xeb, 0xb1, 0x41, 0x86, 0x9a, 0xa4, 0xb7, 0xdd, 0x6d, 0x3a, 0xfb, + 0xb4, 0x5a, 0x38, 0x3f, 0x5b, 0x9f, 0x97, 0x0d, 0x3c, 0xef, 0xdb, 0x1e, 0xfb, 0x50, 0x7f, 0x37, + 0x01, 0x85, 0x08, 0x0a, 0xbd, 0x0a, 0xe0, 0xbb, 0x23, 0xcf, 0xd7, 0x5c, 0x4a, 0x7d, 0xee, 0xac, + 0x22, 0xce, 0x73, 0x0a, 0xa6, 0xd4, 0x47, 0x65, 0xb8, 0x66, 0x10, 0xd7, 0xd7, 0x2c, 0xcf, 0x1b, + 0x11, 0x57, 0xf3, 0x46, 0x7b, 0x5f, 0x12, 0xc3, 0xe7, 0x8e, 0x2b, 0xe2, 0x65, 0xc6, 0x6a, 0x72, + 0x4e, 0x57, 0x30, 0xd0, 0x7d, 0xb8, 0x1e, 0xc5, 0x0f, 0x47, 0x7b, 0xb6, 0x65, 0x68, 0x6c, 0x32, + 0x53, 0x5c, 0xe4, 0xda, 0x44, 0xa4, 0xc3, 0x79, 0x8f, 0xc8, 0x58, 0xfd, 0x69, 0x02, 0x14, 0xac, + 0xef, 0xfb, 0x3b, 0x64, 0xb0, 0x47, 0xdc, 0xae, 0xaf, 0xfb, 0x23, 0x0f, 0x5d, 0x87, 0xac, 0x4d, + 0x74, 0x93, 0xb8, 0xdc, 0xa8, 0x1c, 0x96, 0x2d, 0xb4, 0xcb, 0x76, 0xb0, 0x6e, 0x1c, 0xe8, 0x7b, + 0x96, 0x6d, 0xf9, 0x63, 0x6e, 0xca, 0x62, 0xfc, 0x12, 0x9e, 0xd5, 0x59, 0xc6, 0x11, 0x41, 0x3c, + 0xa5, 0x06, 0xad, 0xc2, 0xfc, 0x80, 0x78, 0x9e, 0xde, 0x27, 0xdc, 0xd2, 0x3c, 0x0e, 0x9a, 0xea, + 0xc7, 0x50, 0x8c, 0xca, 0xa1, 0x02, 0xcc, 0xef, 0xb6, 0x1e, 0xb5, 0xda, 0x4f, 0x5a, 0xca, 0x1c, + 0x5a, 0x82, 0xc2, 0x6e, 0x0b, 0x37, 0x2a, 0xb5, 0xad, 0x4a, 0x75, 0xbb, 0xa1, 0x24, 0xd0, 0x02, + 0xe4, 0x27, 0xcd, 0xa4, 0xfa, 0x67, 0x09, 0x00, 0xe6, 0x6e, 0x39, 0xa8, 0x8f, 0x20, 0xe3, 0xf9, + 0xba, 0x2f, 0x56, 0xe5, 0xe2, 0xbd, 0x37, 0x2e, 0x9a, 0x43, 0x69, 0x2f, 0xfb, 0x47, 0xb0, 0x10, + 0x89, 0x5a, 0x98, 0x9c, 0xb2, 0x90, 0x05, 0x08, 0xdd, 0x34, 0x5d, 0x69, 0x38, 0xff, 0x56, 0x3f, + 0x86, 0x0c, 0x97, 0x9e, 0x36, 0x37, 0x07, 0xe9, 0x3a, 0xfb, 0x4a, 0xa0, 0x3c, 0x64, 0x70, 0xa3, + 0x52, 0xff, 0x42, 0x49, 0x22, 0x05, 0x8a, 0xf5, 0x66, 0xb7, 0xd6, 0x6e, 0xb5, 0x1a, 0xb5, 0x5e, + 0xa3, 0xae, 0xa4, 0xd4, 0x5b, 0x90, 0x69, 0x0e, 0x98, 0xe6, 0x9b, 0x6c, 0xc9, 0xef, 0x13, 0x97, + 0x38, 0x46, 0xb0, 0x93, 0x26, 0x04, 0xf5, 0x27, 0x79, 0xc8, 0xec, 0xd0, 0x91, 0xe3, 0xa3, 0x7b, + 0x91, 0xb0, 0xb5, 0x18, 0x9f, 0x21, 0x70, 0x60, 0xb9, 0x37, 0x1e, 0x12, 0x19, 0xd6, 0xae, 0x43, + 0x56, 0x6c, 0x0e, 0x39, 0x1c, 0xd9, 0x62, 0x74, 0x5f, 0x77, 0xfb, 0xc4, 0x97, 0xe3, 0x91, 0x2d, + 0xf4, 0x16, 0x3b, 0xb1, 0x74, 0x93, 0x3a, 0xf6, 0x98, 0xef, 0xa1, 0x9c, 0x38, 0x96, 0x30, 0xd1, + 0xcd, 0xb6, 0x63, 0x8f, 0x71, 0xc8, 0x45, 0x5b, 0x50, 0xdc, 0xb3, 0x1c, 0x53, 0xa3, 0x43, 0x11, + 0xe4, 0x33, 0x17, 0xef, 0x38, 0x61, 0x55, 0xd5, 0x72, 0xcc, 0xb6, 0x00, 0xe3, 0xc2, 0xde, 0xa4, + 0x81, 0x5a, 0xb0, 0x78, 0x44, 0xed, 0xd1, 0x80, 0x84, 0xba, 0xb2, 0x5c, 0xd7, 0x9b, 0x17, 0xeb, + 0x7a, 0xcc, 0xf1, 0x81, 0xb6, 0x85, 0xa3, 0x68, 0x13, 0x3d, 0x82, 0x05, 0x7f, 0x30, 0xdc, 0xf7, + 0x42, 0x75, 0xf3, 0x5c, 0xdd, 0xf7, 0x2e, 0x71, 0x18, 0x83, 0x07, 0xda, 0x8a, 0x7e, 0xa4, 0x55, + 0xfa, 0x8d, 0x14, 0x14, 0x22, 0x96, 0xa3, 0x2e, 0x14, 0x86, 0x2e, 0x1d, 0xea, 0x7d, 0x7e, 0x50, + 0xc9, 0xb9, 0xb8, 0xfb, 0x42, 0xa3, 0x2e, 0x77, 0x26, 0x82, 0x38, 0xaa, 0x45, 0x3d, 0x4d, 0x42, + 0x21, 0xc2, 0x44, 0xb7, 0x21, 0x87, 0x3b, 0xb8, 0xf9, 0xb8, 0xd2, 0x6b, 0x28, 0x73, 0xa5, 0x9b, + 0x27, 0xa7, 0x1b, 0xab, 0x5c, 0x5b, 0x54, 0x41, 0xc7, 0xb5, 0x8e, 0xd8, 0xd2, 0x7b, 0x0b, 0xe6, + 0x03, 0x68, 0xa2, 0xf4, 0xca, 0xc9, 0xe9, 0xc6, 0xcb, 0xb3, 0xd0, 0x08, 0x12, 0x77, 0xb7, 0x2a, + 0xb8, 0x51, 0x57, 0x92, 0xf1, 0x48, 0xdc, 0x3d, 0xd0, 0x5d, 0x62, 0xa2, 0xef, 0x41, 0x56, 0x02, + 0x53, 0xa5, 0xd2, 0xc9, 0xe9, 0xc6, 0xf5, 0x59, 0xe0, 0x04, 0x87, 0xbb, 0xdb, 0x95, 0xc7, 0x0d, + 0x25, 0x1d, 0x8f, 0xc3, 0x5d, 0x5b, 0x3f, 0x22, 0xe8, 0x0d, 0xc8, 0x08, 0x58, 0xa6, 0x74, 0xe3, + 0xe4, 0x74, 0xe3, 0xa5, 0xe7, 0xd4, 0x31, 0x54, 0x69, 0xf5, 0xb7, 0xff, 0x70, 0x6d, 0xee, 0xaf, + 0xfe, 0x68, 0x4d, 0x99, 0x65, 0x97, 0xfe, 0x3b, 0x01, 0x0b, 0x53, 0x53, 0x8e, 0x54, 0xc8, 0x3a, + 0xd4, 0xa0, 0x43, 0x71, 0x7e, 0xe5, 0xaa, 0x70, 0x7e, 0xb6, 0x9e, 0x6d, 0xd1, 0x1a, 0x1d, 0x8e, + 0xb1, 0xe4, 0xa0, 0x47, 0x33, 0x27, 0xf0, 0xfd, 0x17, 0x5c, 0x4f, 0xb1, 0x67, 0xf0, 0x67, 0xb0, + 0x60, 0xba, 0xd6, 0x11, 0x71, 0x35, 0x83, 0x3a, 0xfb, 0x56, 0x5f, 0x9e, 0x4d, 0xa5, 0xd8, 0x34, + 0x91, 0x03, 0x71, 0x51, 0x08, 0xd4, 0x38, 0xfe, 0x3b, 0x9c, 0xbe, 0xa5, 0xc7, 0x50, 0x8c, 0xae, + 0x50, 0x76, 0x9c, 0x78, 0xd6, 0xaf, 0x10, 0x99, 0x0f, 0xf2, 0xec, 0x11, 0xe7, 0x19, 0x45, 0x64, + 0x83, 0x6f, 0x42, 0x7a, 0x40, 0x4d, 0xa1, 0x67, 0xa1, 0x7a, 0x8d, 0x25, 0x01, 0xff, 0x74, 0xb6, + 0x5e, 0xa0, 0x5e, 0x79, 0xd3, 0xb2, 0xc9, 0x0e, 0x35, 0x09, 0xe6, 0x00, 0xf5, 0x08, 0xd2, 0x2c, + 0x54, 0xa0, 0x57, 0x20, 0x5d, 0x6d, 0xb6, 0xea, 0xca, 0x5c, 0x69, 0xf9, 0xe4, 0x74, 0x63, 0x81, + 0xbb, 0x84, 0x31, 0xd8, 0xda, 0x45, 0xeb, 0x90, 0x7d, 0xdc, 0xde, 0xde, 0xdd, 0x61, 0xcb, 0xeb, + 0xda, 0xc9, 0xe9, 0xc6, 0x52, 0xc8, 0x16, 0x4e, 0x43, 0xaf, 0x42, 0xa6, 0xb7, 0xd3, 0xd9, 0xec, + 0x2a, 0xc9, 0x12, 0x3a, 0x39, 0xdd, 0x58, 0x0c, 0xf9, 0xdc, 0xe6, 0xd2, 0xb2, 0x9c, 0xd5, 0x7c, + 0x48, 0x57, 0x7f, 0x96, 0x84, 0x05, 0xcc, 0x2a, 0x3e, 0xd7, 0xef, 0x50, 0xdb, 0x32, 0xc6, 0xa8, + 0x03, 0x79, 0x83, 0x3a, 0xa6, 0x15, 0xd9, 0x53, 0xf7, 0x2e, 0x38, 0xf5, 0x27, 0x52, 0x41, 0xab, + 0x16, 0x48, 0xe2, 0x89, 0x12, 0xf4, 0x2e, 0x64, 0x4c, 0x62, 0xeb, 0x63, 0x99, 0x7e, 0xdc, 0x28, + 0x8b, 0x9a, 0xb2, 0x1c, 0xd4, 0x94, 0xe5, 0xba, 0xac, 0x29, 0xb1, 0xc0, 0xf1, 0x34, 0x5b, 0x7f, + 0xaa, 0xe9, 0xbe, 0x4f, 0x06, 0x43, 0x5f, 0xe4, 0x1e, 0x69, 0x5c, 0x18, 0xe8, 0x4f, 0x2b, 0x92, + 0x84, 0xee, 0x42, 0xf6, 0xd8, 0x72, 0x4c, 0x7a, 0x2c, 0xd3, 0x8b, 0x4b, 0x94, 0x4a, 0xa0, 0x7a, + 0xc2, 0x4e, 0xdd, 0x19, 0x33, 0x99, 0xbf, 0x5b, 0xed, 0x56, 0x23, 0xf0, 0xb7, 0xe4, 0xb7, 0x9d, + 0x16, 0x75, 0xd8, 0x5e, 0x81, 0x76, 0x4b, 0xdb, 0xac, 0x34, 0xb7, 0x77, 0x31, 0xf3, 0xf9, 0xca, + 0xc9, 0xe9, 0x86, 0x12, 0x42, 0x36, 0x75, 0xcb, 0x66, 0xf9, 0xee, 0x0d, 0x48, 0x55, 0x5a, 0x5f, + 0x28, 0xc9, 0x92, 0x72, 0x72, 0xba, 0x51, 0x0c, 0xd9, 0x15, 0x67, 0x3c, 0xd9, 0x46, 0xb3, 0xfd, + 0xaa, 0x7f, 0x9b, 0x82, 0xe2, 0xee, 0xd0, 0xd4, 0x7d, 0x22, 0xd6, 0x24, 0xda, 0x80, 0xc2, 0x50, + 0x77, 0x75, 0xdb, 0x26, 0xb6, 0xe5, 0x0d, 0x64, 0xb5, 0x1c, 0x25, 0xa1, 0x0f, 0x5f, 0xd4, 0x8d, + 0xd5, 0x1c, 0x5b, 0x67, 0x3f, 0xfe, 0x97, 0xf5, 0x44, 0xe0, 0xd0, 0x5d, 0x58, 0xdc, 0x17, 0xd6, + 0x6a, 0xba, 0xc1, 0x27, 0x36, 0xc5, 0x27, 0xb6, 0x1c, 0x37, 0xb1, 0x51, 0xb3, 0xca, 0x72, 0x90, + 0x15, 0x2e, 0x85, 0x17, 0xf6, 0xa3, 0x4d, 0x74, 0x1f, 0xe6, 0x07, 0xd4, 0xb1, 0x7c, 0xea, 0x5e, + 0x3d, 0x0b, 0x01, 0x12, 0xdd, 0x86, 0x65, 0x36, 0xb9, 0x81, 0x3d, 0x9c, 0xcd, 0x4f, 0xac, 0x24, + 0x5e, 0x1a, 0xe8, 0x4f, 0x65, 0x87, 0x98, 0x91, 0x51, 0x15, 0x32, 0xd4, 0x65, 0x29, 0x51, 0x96, + 0x9b, 0xfb, 0xce, 0x95, 0xe6, 0x8a, 0x46, 0x9b, 0xc9, 0x60, 0x21, 0xaa, 0x7e, 0x00, 0x0b, 0x53, + 0x83, 0x60, 0x99, 0x40, 0xa7, 0xb2, 0xdb, 0x6d, 0x28, 0x73, 0xa8, 0x08, 0xb9, 0x5a, 0xbb, 0xd5, + 0x6b, 0xb6, 0x76, 0x59, 0x2a, 0x53, 0x84, 0x1c, 0x6e, 0x6f, 0x6f, 0x57, 0x2b, 0xb5, 0x47, 0x4a, + 0x52, 0x2d, 0x43, 0x21, 0xa2, 0x0d, 0x2d, 0x02, 0x74, 0x7b, 0xed, 0x8e, 0xb6, 0xd9, 0xc4, 0xdd, + 0x9e, 0x48, 0x84, 0xba, 0xbd, 0x0a, 0xee, 0x49, 0x42, 0x42, 0xfd, 0x8f, 0x64, 0x30, 0xa3, 0x32, + 0xf7, 0xa9, 0x4e, 0xe7, 0x3e, 0x97, 0x18, 0x2f, 0xb3, 0x9f, 0x49, 0x23, 0xcc, 0x81, 0x3e, 0x04, + 0xe0, 0x0b, 0x87, 0x98, 0x9a, 0xee, 0xcb, 0x89, 0x2f, 0x3d, 0xe7, 0xe4, 0x5e, 0x70, 0x69, 0x83, + 0xf3, 0x12, 0x5d, 0xf1, 0xd1, 0x27, 0x50, 0x34, 0xe8, 0x60, 0x68, 0x13, 0x29, 0x9c, 0xba, 0x52, + 0xb8, 0x10, 0xe2, 0x2b, 0x7e, 0x34, 0xfb, 0x4a, 0x4f, 0xe7, 0x87, 0xbf, 0x99, 0x08, 0x3c, 0x13, + 0x93, 0x70, 0x15, 0x21, 0xb7, 0xdb, 0xa9, 0x57, 0x7a, 0xcd, 0xd6, 0x43, 0x25, 0x81, 0x00, 0xb2, + 0xdc, 0xd5, 0x75, 0x25, 0xc9, 0x12, 0xc5, 0x5a, 0x7b, 0xa7, 0xb3, 0xdd, 0xe0, 0x29, 0x17, 0x5a, + 0x01, 0x25, 0x70, 0xb6, 0xc6, 0x1d, 0xd9, 0xa8, 0x2b, 0x69, 0x74, 0x0d, 0x96, 0x42, 0xaa, 0x94, + 0xcc, 0xa0, 0xeb, 0x80, 0x42, 0xe2, 0x44, 0x45, 0x56, 0xfd, 0x35, 0x58, 0xaa, 0x51, 0xc7, 0xd7, + 0x2d, 0x27, 0x4c, 0xa2, 0xef, 0xb1, 0x41, 0x4b, 0x92, 0x66, 0xc9, 0xcb, 0x8e, 0xea, 0xd2, 0xf9, + 0xd9, 0x7a, 0x21, 0x84, 0x36, 0xeb, 0x6c, 0xa4, 0x41, 0xc3, 0x64, 0xfb, 0x77, 0x68, 0x99, 0xdc, + 0xb9, 0x99, 0xea, 0xfc, 0xf9, 0xd9, 0x7a, 0xaa, 0xd3, 0xac, 0x63, 0x46, 0x43, 0xaf, 0x40, 0x9e, + 0x3c, 0xb5, 0x7c, 0xcd, 0x60, 0x31, 0x9c, 0x39, 0x30, 0x83, 0x73, 0x8c, 0x50, 0x63, 0x21, 0xbb, + 0x0a, 0xd0, 0xa1, 0xae, 0x2f, 0x7b, 0x7e, 0x0f, 0x32, 0x43, 0xea, 0xf2, 0xf2, 0xfc, 0xc2, 0x4b, + 0x23, 0x06, 0x17, 0x0b, 0x15, 0x0b, 0xb0, 0xfa, 0x7b, 0x29, 0x80, 0x9e, 0xee, 0x1d, 0x4a, 0x25, + 0x0f, 0x20, 0x1f, 0x5e, 0xc0, 0xc9, 0x3a, 0xff, 0xd2, 0xd9, 0x0e, 0xc1, 0xe8, 0x7e, 0xb0, 0xd8, + 0x44, 0x79, 0x10, 0x5b, 0xa7, 0x05, 0x1d, 0xc5, 0x65, 0xd8, 0xd3, 0x35, 0x00, 0x3b, 0x12, 0x89, + 0xeb, 0xca, 0x99, 0x67, 0x9f, 0xa8, 0xc6, 0x8f, 0x05, 0xe1, 0x34, 0x99, 0x60, 0xc6, 0xde, 0x6c, + 0xcc, 0xcc, 0xc8, 0xd6, 0x1c, 0x9e, 0xc8, 0xa1, 0xcf, 0xa0, 0xc0, 0xc6, 0xad, 0x79, 0x9c, 0x27, + 0x73, 0xcb, 0x0b, 0x5d, 0x25, 0x34, 0x60, 0x18, 0x4e, 0xbc, 0xfc, 0x2a, 0x80, 0x3e, 0x1c, 0xda, + 0x16, 0x31, 0xb5, 0xbd, 0x31, 0x4f, 0x26, 0xf3, 0x38, 0x2f, 0x29, 0xd5, 0x31, 0xdb, 0x2e, 0x01, + 0x5b, 0xf7, 0x57, 0x73, 0x57, 0x3b, 0x50, 0xa2, 0x2b, 0x7e, 0x55, 0x81, 0x45, 0x77, 0xe4, 0x30, + 0x87, 0x4a, 0xeb, 0xd4, 0x3f, 0x4d, 0xc2, 0xcb, 0x2d, 0xe2, 0x1f, 0x53, 0xf7, 0xb0, 0xe2, 0xfb, + 0xba, 0x71, 0x30, 0x20, 0x8e, 0x9c, 0xbe, 0x48, 0xce, 0x9e, 0x98, 0xca, 0xd9, 0x57, 0x61, 0x5e, + 0xb7, 0x2d, 0xdd, 0x23, 0x22, 0xd1, 0xc9, 0xe3, 0xa0, 0xc9, 0x2a, 0x0b, 0x56, 0xa7, 0x10, 0xcf, + 0x23, 0xe2, 0xea, 0x80, 0x19, 0x1e, 0x10, 0xd0, 0x8f, 0xe0, 0xba, 0x4c, 0x69, 0xf4, 0xb0, 0x2b, + 0x96, 0x33, 0x07, 0x77, 0x90, 0x8d, 0xd8, 0xc2, 0x29, 0xde, 0x38, 0x99, 0xf3, 0x4c, 0xc8, 0xed, + 0xa1, 0x2f, 0x33, 0xa8, 0x15, 0x33, 0x86, 0x55, 0x7a, 0x08, 0x37, 0x2e, 0x14, 0xf9, 0x56, 0x57, + 0x13, 0xff, 0x90, 0x04, 0x68, 0x76, 0x2a, 0x3b, 0xd2, 0x49, 0x75, 0xc8, 0xee, 0xeb, 0x03, 0xcb, + 0x1e, 0x5f, 0x16, 0x01, 0x27, 0xf8, 0x72, 0x45, 0xb8, 0x63, 0x93, 0xcb, 0x60, 0x29, 0xcb, 0xcb, + 0xa6, 0xd1, 0x9e, 0x43, 0xfc, 0xb0, 0x6c, 0xe2, 0x2d, 0x66, 0x86, 0xab, 0x3b, 0xe1, 0xd2, 0x15, + 0x0d, 0x36, 0x01, 0x7d, 0xdd, 0x27, 0xc7, 0xfa, 0x38, 0x08, 0x5b, 0xb2, 0x89, 0xb6, 0xf8, 0x05, + 0x20, 0x71, 0x8f, 0x88, 0xb9, 0x9a, 0xe1, 0x4e, 0xbd, 0xca, 0x1e, 0x2c, 0xe1, 0xc2, 0x77, 0xa1, + 0x74, 0xe9, 0x63, 0x9e, 0x32, 0x4d, 0x58, 0xdf, 0xca, 0x47, 0x77, 0x60, 0x61, 0x6a, 0x9c, 0xcf, + 0xd5, 0xab, 0xcd, 0xce, 0xe3, 0xf7, 0x94, 0xb4, 0xfc, 0xfa, 0x40, 0xc9, 0xaa, 0x7f, 0x9c, 0x12, + 0x81, 0x46, 0x7a, 0x35, 0xfe, 0xe2, 0x3b, 0xc7, 0x57, 0xb7, 0x41, 0x6d, 0x19, 0x00, 0xde, 0xbc, + 0x3c, 0xfe, 0xb0, 0xfa, 0x87, 0xc3, 0x71, 0x28, 0x88, 0xd6, 0xa1, 0x20, 0x56, 0xb1, 0xc6, 0x36, + 0x1c, 0x77, 0xeb, 0x02, 0x06, 0x41, 0x62, 0x92, 0xe8, 0x16, 0x2c, 0xf2, 0xfb, 0x0d, 0xef, 0x80, + 0x98, 0x02, 0x93, 0xe6, 0x98, 0x85, 0x90, 0xca, 0x61, 0x3b, 0x50, 0x94, 0x04, 0x8d, 0xe7, 0xbe, + 0x19, 0x6e, 0xd0, 0xed, 0xab, 0x0c, 0x12, 0x22, 0x3c, 0x25, 0x2e, 0x0c, 0x27, 0x0d, 0xb5, 0x0e, + 0xb9, 0xc0, 0x58, 0xb4, 0x0a, 0xa9, 0x5e, 0xad, 0xa3, 0xcc, 0x95, 0x96, 0x4e, 0x4e, 0x37, 0x0a, + 0x01, 0xb9, 0x57, 0xeb, 0x30, 0xce, 0x6e, 0xbd, 0xa3, 0x24, 0xa6, 0x39, 0xbb, 0xf5, 0x4e, 0x29, + 0xcd, 0x72, 0x30, 0x75, 0x1f, 0x0a, 0x91, 0x1e, 0xd0, 0xeb, 0x30, 0xdf, 0x6c, 0x3d, 0xc4, 0x8d, + 0x6e, 0x57, 0x99, 0x2b, 0x5d, 0x3f, 0x39, 0xdd, 0x40, 0x11, 0x6e, 0xd3, 0xe9, 0xb3, 0xf9, 0x41, + 0xaf, 0x42, 0x7a, 0xab, 0xcd, 0xce, 0x76, 0x91, 0x6c, 0x47, 0x10, 0x5b, 0xd4, 0xf3, 0x4b, 0xd7, + 0x64, 0x72, 0x17, 0x55, 0xac, 0xfe, 0x7e, 0x02, 0xb2, 0x62, 0x33, 0xc5, 0x4e, 0x54, 0x05, 0xe6, + 0x83, 0x4a, 0x58, 0x14, 0x42, 0x6f, 0x5e, 0x5c, 0xb4, 0x94, 0x65, 0x8d, 0x21, 0x96, 0x5f, 0x20, + 0x57, 0xfa, 0x08, 0x8a, 0x51, 0xc6, 0xb7, 0x5a, 0x7c, 0x3f, 0x82, 0x02, 0x5b, 0xdf, 0x41, 0xf1, + 0x72, 0x0f, 0xb2, 0x22, 0x20, 0x84, 0x67, 0xcd, 0xc5, 0x15, 0x94, 0x44, 0xa2, 0x07, 0x30, 0x2f, + 0xaa, 0xae, 0xe0, 0x02, 0x74, 0xed, 0xf2, 0x5d, 0x84, 0x03, 0xb8, 0xfa, 0x19, 0xa4, 0x3b, 0x84, + 0xb8, 0xcc, 0xf7, 0x0e, 0x35, 0xc9, 0xe4, 0x78, 0x96, 0x05, 0xa3, 0x49, 0x9a, 0x75, 0x56, 0x30, + 0x9a, 0xa4, 0x69, 0x86, 0x57, 0x3c, 0xc9, 0xc8, 0x15, 0x4f, 0x0f, 0x8a, 0x4f, 0x88, 0xd5, 0x3f, + 0xf0, 0x89, 0xc9, 0x15, 0xbd, 0x03, 0xe9, 0x21, 0x09, 0x8d, 0x5f, 0x8d, 0x5d, 0x60, 0x84, 0xb8, + 0x98, 0xa3, 0x58, 0x1c, 0x39, 0xe6, 0xd2, 0xf2, 0xd6, 0x5e, 0xb6, 0xd4, 0xbf, 0x4f, 0xc2, 0x62, + 0xd3, 0xf3, 0x46, 0xba, 0x63, 0x04, 0x99, 0xdb, 0xa7, 0xd3, 0x99, 0x5b, 0xec, 0xf3, 0xc6, 0xb4, + 0xc8, 0xf4, 0xcd, 0x95, 0x3c, 0x3d, 0x93, 0xe1, 0xe9, 0xa9, 0xfe, 0x7b, 0x22, 0xb8, 0x9e, 0xba, + 0x15, 0xd9, 0xee, 0xa5, 0xd5, 0x93, 0xd3, 0x8d, 0x95, 0xa8, 0x26, 0xb2, 0xeb, 0x1c, 0x3a, 0xf4, + 0xd8, 0x41, 0xaf, 0x41, 0x06, 0x37, 0x5a, 0x8d, 0x27, 0x4a, 0x42, 0x2c, 0xcf, 0x29, 0x10, 0x26, + 0x0e, 0x39, 0x66, 0x9a, 0x3a, 0x8d, 0x56, 0x9d, 0x65, 0x5a, 0xc9, 0x18, 0x4d, 0x1d, 0xe2, 0x98, + 0x96, 0xd3, 0x47, 0xaf, 0x43, 0xb6, 0xd9, 0xed, 0xee, 0xf2, 0x0b, 0x84, 0x97, 0x4f, 0x4e, 0x37, + 0xae, 0x4d, 0xa1, 0xf8, 0xd5, 0xa4, 0xc9, 0x40, 0xac, 0xcc, 0x61, 0x39, 0x58, 0x0c, 0x88, 0xe5, + 0xcf, 0x02, 0x84, 0xdb, 0xbd, 0x4a, 0xaf, 0xa1, 0x64, 0x62, 0x40, 0x98, 0xb2, 0xbf, 0x72, 0xbb, + 0xfd, 0x73, 0x12, 0x94, 0x8a, 0x61, 0x90, 0xa1, 0xcf, 0xf8, 0xb2, 0xb2, 0xec, 0x41, 0x6e, 0xc8, + 0xbe, 0x2c, 0x12, 0x64, 0x49, 0x0f, 0x62, 0x1f, 0xe8, 0x66, 0xe4, 0xca, 0x98, 0xda, 0xa4, 0x62, + 0x0e, 0x2c, 0xcf, 0xb3, 0xa8, 0x23, 0x68, 0x38, 0xd4, 0x54, 0xfa, 0xcf, 0x04, 0x5c, 0x8b, 0x41, + 0xa0, 0x3b, 0x90, 0x76, 0xa9, 0x1d, 0xcc, 0xe1, 0xcd, 0x8b, 0x6e, 0x1e, 0x99, 0x28, 0xe6, 0x48, + 0xb4, 0x06, 0xa0, 0x8f, 0x7c, 0xaa, 0xf3, 0xfe, 0xf9, 0xec, 0xe5, 0x70, 0x84, 0x82, 0x9e, 0x40, + 0xd6, 0x23, 0x86, 0x4b, 0x82, 0x5c, 0xfa, 0xb3, 0xff, 0xab, 0xf5, 0xe5, 0x2e, 0x57, 0x83, 0xa5, + 0xba, 0x52, 0x19, 0xb2, 0x82, 0xc2, 0x96, 0xbd, 0xa9, 0xfb, 0xba, 0xbc, 0x97, 0xe6, 0xdf, 0x6c, + 0x35, 0xe9, 0x76, 0x3f, 0x58, 0x4d, 0xba, 0xdd, 0x57, 0xff, 0x26, 0x09, 0xd0, 0x78, 0xea, 0x13, + 0xd7, 0xd1, 0xed, 0x5a, 0x05, 0x35, 0x22, 0xd1, 0x5f, 0x8c, 0xf6, 0xed, 0xd8, 0xcb, 0xf6, 0x50, + 0xa2, 0x5c, 0xab, 0xc4, 0xc4, 0xff, 0x1b, 0x90, 0x1a, 0xb9, 0xf2, 0xcd, 0x55, 0xe4, 0xc1, 0xbb, + 0x78, 0x1b, 0x33, 0x1a, 0x6a, 0x4c, 0xc2, 0x56, 0xea, 0xe2, 0x97, 0xd5, 0x48, 0x07, 0xb1, 0xa1, + 0x8b, 0xed, 0x7c, 0x43, 0xd7, 0x0c, 0x22, 0x4f, 0x8e, 0xa2, 0xd8, 0xf9, 0xb5, 0x4a, 0x8d, 0xb8, + 0x3e, 0xce, 0x1a, 0x3a, 0xfb, 0xff, 0x9d, 0xe2, 0xdb, 0x3b, 0x00, 0x93, 0xa1, 0xa1, 0x35, 0xc8, + 0xd4, 0x36, 0xbb, 0xdd, 0x6d, 0x65, 0x4e, 0x04, 0xf0, 0x09, 0x8b, 0x93, 0xd5, 0xbf, 0x4c, 0x42, + 0xae, 0x56, 0x91, 0xc7, 0x6a, 0x0d, 0x14, 0x1e, 0x95, 0xf8, 0x6d, 0x3e, 0x79, 0x3a, 0xb4, 0xdc, + 0xb1, 0x0c, 0x2c, 0x97, 0x14, 0xb5, 0x8b, 0x4c, 0x84, 0x59, 0xdd, 0xe0, 0x02, 0x08, 0x43, 0x91, + 0x48, 0x27, 0x68, 0x86, 0x1e, 0xc4, 0xf8, 0xb5, 0xcb, 0x9d, 0x25, 0xca, 0x93, 0x49, 0xdb, 0xc3, + 0x85, 0x40, 0x49, 0x4d, 0xf7, 0xd0, 0x87, 0xb0, 0xe4, 0x59, 0x7d, 0xc7, 0x72, 0xfa, 0x5a, 0xe0, + 0x3c, 0xfe, 0xb4, 0x50, 0x5d, 0x3e, 0x3f, 0x5b, 0x5f, 0xe8, 0x0a, 0x96, 0xf4, 0xe1, 0x82, 0x44, + 0xd6, 0xb8, 0x2b, 0xd1, 0x07, 0xb0, 0x18, 0x11, 0x65, 0x5e, 0x14, 0x6e, 0x57, 0xce, 0xcf, 0xd6, + 0x8b, 0xa1, 0xe4, 0x23, 0x32, 0xc6, 0xc5, 0x50, 0xf0, 0x11, 0xe1, 0xf7, 0x2f, 0xfb, 0xd4, 0x35, + 0x88, 0xe6, 0xf2, 0x3d, 0xcd, 0x4f, 0xf0, 0x34, 0x2e, 0x70, 0x9a, 0xd8, 0xe6, 0xea, 0x63, 0xb8, + 0xd6, 0x76, 0x8d, 0x03, 0xe2, 0xf9, 0xc2, 0x15, 0xd2, 0x8b, 0x9f, 0xc1, 0x4d, 0x5f, 0xf7, 0x0e, + 0xb5, 0x03, 0xcb, 0xf3, 0xa9, 0x3b, 0xd6, 0x5c, 0xe2, 0x13, 0x87, 0xf1, 0x35, 0xfe, 0x1e, 0x29, + 0x2f, 0xc8, 0x6e, 0x30, 0xcc, 0x96, 0x80, 0xe0, 0x00, 0xb1, 0xcd, 0x00, 0x6a, 0x13, 0x8a, 0xac, + 0x4c, 0xa9, 0x93, 0x7d, 0x7d, 0x64, 0xfb, 0x6c, 0xf4, 0x60, 0xd3, 0xbe, 0xf6, 0xc2, 0xc7, 0x54, + 0xde, 0xa6, 0x7d, 0xf1, 0xa9, 0xfe, 0x10, 0x94, 0xba, 0xe5, 0x0d, 0x75, 0xdf, 0x38, 0x08, 0x6e, + 0xfe, 0x50, 0x1d, 0x94, 0x03, 0xa2, 0xbb, 0xfe, 0x1e, 0xd1, 0x7d, 0x6d, 0x48, 0x5c, 0x8b, 0x9a, + 0x57, 0xcf, 0xf2, 0x52, 0x28, 0xd2, 0xe1, 0x12, 0xea, 0x7f, 0x25, 0x00, 0xb0, 0xbe, 0x1f, 0x64, + 0x64, 0xdf, 0x87, 0x65, 0xcf, 0xd1, 0x87, 0xde, 0x01, 0xf5, 0x35, 0xcb, 0xf1, 0x89, 0x7b, 0xa4, + 0xdb, 0xf2, 0x02, 0x47, 0x09, 0x18, 0x4d, 0x49, 0x47, 0xef, 0x00, 0x3a, 0x24, 0x64, 0xa8, 0x51, + 0xdb, 0xd4, 0x02, 0xa6, 0x78, 0x2d, 0x4d, 0x63, 0x85, 0x71, 0xda, 0xb6, 0xd9, 0x0d, 0xe8, 0xa8, + 0x0a, 0x6b, 0x6c, 0xf8, 0xc4, 0xf1, 0x5d, 0x8b, 0x78, 0xda, 0x3e, 0x75, 0x35, 0xcf, 0xa6, 0xc7, + 0xda, 0x3e, 0xb5, 0x6d, 0x7a, 0x4c, 0xdc, 0xe0, 0x6e, 0xac, 0x64, 0xd3, 0x7e, 0x43, 0x80, 0x36, + 0xa9, 0xdb, 0xb5, 0xe9, 0xf1, 0x66, 0x80, 0x60, 0x69, 0xdb, 0x64, 0xcc, 0xbe, 0x65, 0x1c, 0x06, + 0x69, 0x5b, 0x48, 0xed, 0x59, 0xc6, 0x21, 0x7a, 0x1d, 0x16, 0x88, 0x4d, 0xf8, 0x15, 0x89, 0x40, + 0x65, 0x38, 0xaa, 0x18, 0x10, 0x19, 0x48, 0xfd, 0x1c, 0x94, 0x86, 0x63, 0xb8, 0xe3, 0x61, 0x64, + 0xce, 0xdf, 0x01, 0xc4, 0x82, 0xa4, 0x66, 0x53, 0xe3, 0x50, 0x1b, 0xe8, 0x8e, 0xde, 0x67, 0x76, + 0x89, 0x47, 0x2c, 0x85, 0x71, 0xb6, 0xa9, 0x71, 0xb8, 0x23, 0xe9, 0xea, 0x87, 0x00, 0xdd, 0xa1, + 0x4b, 0x74, 0xb3, 0xcd, 0xb2, 0x09, 0xe6, 0x3a, 0xde, 0xd2, 0x4c, 0xf9, 0x08, 0x48, 0x5d, 0xb9, + 0xd5, 0x15, 0xc1, 0xa8, 0x87, 0x74, 0xf5, 0x17, 0xe1, 0x5a, 0xc7, 0xd6, 0x0d, 0xfe, 0x20, 0xde, + 0x09, 0x5f, 0x65, 0xd0, 0x03, 0xc8, 0x0a, 0xa8, 0x9c, 0xc9, 0xd8, 0xed, 0x36, 0xe9, 0x73, 0x6b, + 0x0e, 0x4b, 0x7c, 0xb5, 0x08, 0x30, 0xd1, 0xa3, 0xfe, 0x79, 0x02, 0xf2, 0xa1, 0x7e, 0xb4, 0x01, + 0x05, 0x83, 0x3a, 0x6c, 0x79, 0x5b, 0x8e, 0xac, 0xea, 0xf3, 0x38, 0x4a, 0x42, 0x4d, 0x28, 0x0c, + 0x43, 0xe9, 0x4b, 0xf3, 0xb9, 0x18, 0xab, 0x71, 0x54, 0x16, 0x7d, 0x04, 0xf9, 0xe0, 0xd5, 0x35, + 0x88, 0xb0, 0x97, 0x3f, 0xd2, 0x4e, 0xe0, 0xea, 0xa7, 0x00, 0x3f, 0xa0, 0x96, 0xd3, 0xa3, 0x87, + 0xc4, 0xe1, 0xaf, 0x88, 0xac, 0x26, 0x24, 0x81, 0x17, 0x65, 0x8b, 0x97, 0xfa, 0x62, 0x0a, 0xc2, + 0xc7, 0x34, 0xd1, 0x54, 0xff, 0x3a, 0x09, 0x59, 0x4c, 0xa9, 0x5f, 0xab, 0xa0, 0x0d, 0xc8, 0xca, + 0x38, 0xc1, 0xcf, 0x9f, 0x6a, 0xfe, 0xfc, 0x6c, 0x3d, 0x23, 0x02, 0x44, 0xc6, 0xe0, 0x91, 0x21, + 0x12, 0xc1, 0x93, 0x17, 0x45, 0x70, 0x74, 0x07, 0x8a, 0x12, 0xa4, 0x1d, 0xe8, 0xde, 0x81, 0x28, + 0xd0, 0xaa, 0x8b, 0xe7, 0x67, 0xeb, 0x20, 0x90, 0x5b, 0xba, 0x77, 0x80, 0x41, 0xa0, 0xd9, 0x37, + 0x6a, 0x40, 0xe1, 0x4b, 0x6a, 0x39, 0x9a, 0xcf, 0x07, 0x21, 0x2f, 0x13, 0x63, 0xe7, 0x71, 0x32, + 0x54, 0xf9, 0xa4, 0x0e, 0x5f, 0x4e, 0x06, 0xdf, 0x80, 0x05, 0x97, 0x52, 0x5f, 0x84, 0x2d, 0x8b, + 0x3a, 0xf2, 0x9e, 0x62, 0x23, 0xf6, 0xfa, 0x9a, 0x52, 0x1f, 0x4b, 0x1c, 0x2e, 0xba, 0x91, 0x16, + 0xba, 0x03, 0x2b, 0xb6, 0xee, 0xf9, 0x1a, 0x8f, 0x77, 0xe6, 0x44, 0x5b, 0x96, 0x6f, 0x35, 0xc4, + 0x78, 0x9b, 0x9c, 0x15, 0x48, 0xa8, 0xff, 0x98, 0x80, 0x02, 0x1b, 0x8c, 0xb5, 0x6f, 0x19, 0x2c, + 0xc9, 0xfb, 0xf6, 0xb9, 0xc7, 0x0d, 0x48, 0x19, 0x9e, 0x2b, 0x9d, 0xca, 0x0f, 0xdf, 0x5a, 0x17, + 0x63, 0x46, 0x43, 0x9f, 0x43, 0x56, 0xde, 0x97, 0x88, 0xb4, 0x43, 0xbd, 0x3a, 0x1d, 0x95, 0xbe, + 0x91, 0x72, 0x7c, 0x2d, 0x4f, 0xac, 0x13, 0x87, 0x00, 0x8e, 0x92, 0xd0, 0x75, 0x48, 0x1a, 0xc2, + 0x5d, 0xf2, 0x37, 0x1b, 0xb5, 0x16, 0x4e, 0x1a, 0x8e, 0xfa, 0x77, 0x09, 0x58, 0x98, 0x6c, 0x78, + 0xb6, 0x02, 0x6e, 0x42, 0xde, 0x1b, 0xed, 0x79, 0x63, 0xcf, 0x27, 0x83, 0xe0, 0x85, 0x34, 0x24, + 0xa0, 0x26, 0xe4, 0x75, 0xbb, 0x4f, 0x5d, 0xcb, 0x3f, 0x18, 0xc8, 0x4a, 0x34, 0x3e, 0x55, 0x88, + 0xea, 0x2c, 0x57, 0x02, 0x11, 0x3c, 0x91, 0x0e, 0xce, 0x7d, 0xf1, 0x8c, 0xce, 0xcf, 0xfd, 0xd7, + 0xa0, 0x68, 0xeb, 0x03, 0x7e, 0x81, 0xe4, 0x5b, 0x03, 0x31, 0x8e, 0x34, 0x2e, 0x48, 0x5a, 0xcf, + 0x1a, 0x10, 0x55, 0x85, 0x7c, 0xa8, 0x0c, 0x2d, 0x41, 0xa1, 0xd2, 0xe8, 0x6a, 0x77, 0xef, 0x3d, + 0xd0, 0x1e, 0xd6, 0x76, 0x94, 0x39, 0x99, 0x9b, 0xfe, 0x45, 0x02, 0x16, 0x64, 0x38, 0x92, 0xf9, + 0xfe, 0xeb, 0x30, 0xef, 0xea, 0xfb, 0x7e, 0x50, 0x91, 0xa4, 0xc5, 0xaa, 0x66, 0x11, 0x9e, 0x55, + 0x24, 0x8c, 0x15, 0x5f, 0x91, 0x44, 0xde, 0xec, 0x53, 0x97, 0xbe, 0xd9, 0xa7, 0x7f, 0x2e, 0x6f, + 0xf6, 0xea, 0xaf, 0x03, 0x6c, 0x5a, 0x36, 0xe9, 0x89, 0xbb, 0xa6, 0xb8, 0xfa, 0x92, 0xe5, 0x70, + 0xf2, 0x2e, 0x33, 0xc8, 0xe1, 0x9a, 0x75, 0xcc, 0x68, 0x8c, 0xd5, 0xb7, 0x4c, 0xb9, 0x19, 0x39, + 0xeb, 0x21, 0x63, 0xf5, 0x2d, 0x33, 0x7c, 0xa5, 0x4a, 0x5f, 0xf5, 0x4a, 0x75, 0x9a, 0x80, 0x25, + 0x99, 0xbb, 0x86, 0xe1, 0xf7, 0x6d, 0xc8, 0x8b, 0x34, 0x76, 0x52, 0xd0, 0xf1, 0x77, 0x6a, 0x81, + 0x6b, 0xd6, 0x71, 0x4e, 0xb0, 0x9b, 0x26, 0x5a, 0x87, 0x82, 0x84, 0x46, 0x7e, 0xdf, 0x03, 0x82, + 0xd4, 0x62, 0xe6, 0xbf, 0x07, 0xe9, 0x7d, 0xcb, 0x26, 0x72, 0xa1, 0xc7, 0x06, 0x80, 0x89, 0x03, + 0xb6, 0xe6, 0x30, 0x47, 0x57, 0x73, 0xc1, 0x65, 0x1c, 0xb7, 0x4f, 0x96, 0x9d, 0x51, 0xfb, 0x44, + 0x05, 0x3a, 0x63, 0x9f, 0xc0, 0x31, 0xfb, 0x04, 0x5b, 0xd8, 0x27, 0xa1, 0x51, 0xfb, 0x04, 0xe9, + 0xe7, 0x62, 0xdf, 0x36, 0x5c, 0xaf, 0xda, 0xba, 0x71, 0x68, 0x5b, 0x9e, 0x4f, 0xcc, 0x68, 0xc4, + 0xb8, 0x07, 0xd9, 0xa9, 0xa4, 0xf3, 0xb2, 0x5b, 0x4b, 0x89, 0x54, 0xff, 0x2d, 0x01, 0xc5, 0x2d, + 0xa2, 0xdb, 0xfe, 0xc1, 0xe4, 0x6a, 0xc8, 0x27, 0x9e, 0x2f, 0x0f, 0x2b, 0xfe, 0x8d, 0xde, 0x87, + 0x5c, 0x98, 0x93, 0x5c, 0xf9, 0xfe, 0x16, 0x42, 0xd1, 0x7d, 0x98, 0x67, 0x7b, 0x8c, 0x8e, 0x82, + 0x62, 0xe7, 0xb2, 0xa7, 0x1d, 0x89, 0x64, 0x87, 0x8c, 0x4b, 0x78, 0x12, 0xc2, 0x97, 0x52, 0x06, + 0x07, 0x4d, 0xf4, 0xff, 0xa1, 0xc8, 0x5f, 0x26, 0x82, 0x9c, 0x2b, 0x73, 0x95, 0xce, 0x82, 0x78, + 0x5c, 0x14, 0xf9, 0xd6, 0xff, 0x24, 0x60, 0x65, 0x47, 0x1f, 0xef, 0x11, 0x19, 0x36, 0x88, 0x89, + 0x89, 0x41, 0x5d, 0x13, 0x75, 0xa2, 0xe1, 0xe6, 0x92, 0xb7, 0xca, 0x38, 0xe1, 0xf8, 0xa8, 0x13, + 0x14, 0x60, 0xc9, 0x48, 0x01, 0xb6, 0x02, 0x19, 0x87, 0x3a, 0x06, 0x91, 0xb1, 0x48, 0x34, 0x54, + 0x2b, 0x1a, 0x6a, 0x4a, 0xe1, 0x33, 0x22, 0x7f, 0x04, 0x6c, 0x51, 0x3f, 0xec, 0x0d, 0x7d, 0x0e, + 0xa5, 0x6e, 0xa3, 0x86, 0x1b, 0xbd, 0x6a, 0xfb, 0x87, 0x5a, 0xb7, 0xb2, 0xdd, 0xad, 0xdc, 0xbb, + 0xa3, 0x75, 0xda, 0xdb, 0x5f, 0xdc, 0xbd, 0x7f, 0xe7, 0x7d, 0x25, 0x51, 0xda, 0x38, 0x39, 0xdd, + 0xb8, 0xd9, 0xaa, 0xd4, 0xb6, 0xc5, 0x8e, 0xd9, 0xa3, 0x4f, 0xbb, 0xba, 0xed, 0xe9, 0xf7, 0xee, + 0x74, 0xa8, 0x3d, 0x66, 0x18, 0xb6, 0xac, 0x8b, 0xd1, 0xf3, 0x2a, 0x7a, 0x0c, 0x27, 0x2e, 0x3c, + 0x86, 0x27, 0xa7, 0x79, 0xf2, 0x82, 0xd3, 0x7c, 0x13, 0x56, 0x0c, 0x97, 0x7a, 0x9e, 0xc6, 0xb2, + 0x7f, 0x62, 0xce, 0xd4, 0x17, 0x2f, 0x9d, 0x9f, 0xad, 0x2f, 0xd7, 0x18, 0xbf, 0xcb, 0xd9, 0x52, + 0xfd, 0xb2, 0x11, 0x21, 0xf1, 0x9e, 0xd4, 0x3f, 0x48, 0xb1, 0x44, 0xca, 0x3a, 0xb2, 0x6c, 0xd2, + 0x27, 0x1e, 0x7a, 0x0c, 0x4b, 0x86, 0x4b, 0x4c, 0x96, 0xd6, 0xeb, 0x76, 0xf4, 0x77, 0xa2, 0xff, + 0x2f, 0x36, 0xa7, 0x09, 0x05, 0xcb, 0xb5, 0x50, 0xaa, 0x3b, 0x24, 0x06, 0x5e, 0x34, 0xa6, 0xda, + 0xe8, 0x4b, 0x58, 0xf2, 0x88, 0x6d, 0x39, 0xa3, 0xa7, 0x9a, 0x41, 0x1d, 0x9f, 0x3c, 0x0d, 0x5e, + 0xc4, 0xae, 0xd2, 0xdb, 0x6d, 0x6c, 0x33, 0xa9, 0x9a, 0x10, 0xaa, 0xa2, 0xf3, 0xb3, 0xf5, 0xc5, + 0x69, 0x1a, 0x5e, 0x94, 0x9a, 0x65, 0xbb, 0xd4, 0x82, 0xc5, 0x69, 0x6b, 0xd0, 0x8a, 0xdc, 0xfb, + 0x3c, 0x84, 0x04, 0x7b, 0x1b, 0xdd, 0x84, 0x9c, 0x4b, 0xfa, 0x96, 0xe7, 0xbb, 0xc2, 0xcd, 0x8c, + 0x13, 0x52, 0xd8, 0xce, 0x17, 0x3f, 0xf2, 0x29, 0xfd, 0x2a, 0xcc, 0xf4, 0xc8, 0x36, 0x8b, 0x69, + 0x79, 0xfa, 0x9e, 0x54, 0x99, 0xc3, 0x41, 0x93, 0xad, 0xc1, 0x91, 0x17, 0x26, 0x6a, 0xfc, 0x9b, + 0xd1, 0x78, 0x46, 0x21, 0x7f, 0xf2, 0xc4, 0x73, 0x86, 0xe0, 0xb7, 0x93, 0xe9, 0xc8, 0x6f, 0x27, + 0x57, 0x20, 0x63, 0x93, 0x23, 0x62, 0x8b, 0xb3, 0x1c, 0x8b, 0xc6, 0xed, 0x3b, 0x50, 0x0c, 0x7e, + 0xa4, 0xc7, 0x7f, 0x65, 0x90, 0x83, 0x74, 0xaf, 0xd2, 0x7d, 0xa4, 0xcc, 0x21, 0x80, 0xac, 0x58, + 0x9c, 0xe2, 0xb5, 0xae, 0xd6, 0x6e, 0x6d, 0x36, 0x1f, 0x2a, 0xc9, 0xdb, 0x3f, 0x4b, 0x41, 0x3e, + 0x7c, 0x2f, 0x62, 0x67, 0x47, 0xab, 0xf1, 0x24, 0x58, 0xdd, 0x21, 0xbd, 0x45, 0x8e, 0xd1, 0x6b, + 0x93, 0x5b, 0xa8, 0xcf, 0xc5, 0x03, 0x79, 0xc8, 0x0e, 0x6e, 0xa0, 0xde, 0x80, 0x5c, 0xa5, 0xdb, + 0x6d, 0x3e, 0x6c, 0x35, 0xea, 0xca, 0x57, 0x89, 0xd2, 0x4b, 0x27, 0xa7, 0x1b, 0xcb, 0x21, 0xa8, + 0xe2, 0x89, 0xc5, 0xc7, 0x51, 0xb5, 0x5a, 0xa3, 0xd3, 0x6b, 0xd4, 0x95, 0x67, 0xc9, 0x59, 0x14, + 0xbf, 0x55, 0xe1, 0x3f, 0x73, 0xc9, 0x77, 0x70, 0xa3, 0x53, 0xc1, 0xac, 0xc3, 0xaf, 0x92, 0xe2, + 0x72, 0x6c, 0xd2, 0xa3, 0x4b, 0x86, 0xba, 0xcb, 0xfa, 0x5c, 0x0b, 0x7e, 0xee, 0xf5, 0x2c, 0x25, + 0x7e, 0x0a, 0x31, 0x79, 0xfc, 0x22, 0xba, 0x39, 0x66, 0xbd, 0xf1, 0x57, 0x47, 0xae, 0x26, 0x35, + 0xd3, 0x5b, 0x97, 0xc5, 0x1e, 0xa6, 0x45, 0x85, 0x79, 0xbc, 0xdb, 0x6a, 0x31, 0xd0, 0xb3, 0xf4, + 0xcc, 0xe8, 0xf0, 0xc8, 0x61, 0x15, 0x33, 0xba, 0x05, 0xb9, 0xe0, 0x51, 0x52, 0xf9, 0x2a, 0x3d, + 0x63, 0x50, 0x2d, 0x78, 0x51, 0xe5, 0x1d, 0x6e, 0xed, 0xf6, 0xf8, 0xaf, 0xd1, 0x9e, 0x65, 0x66, + 0x3b, 0x3c, 0x18, 0xf9, 0x26, 0x3d, 0x76, 0xd8, 0x9e, 0x95, 0xf7, 0x70, 0x5f, 0x65, 0xc4, 0xa5, + 0x45, 0x88, 0x91, 0x97, 0x70, 0x6f, 0x40, 0x0e, 0x37, 0x7e, 0x20, 0x7e, 0xb8, 0xf6, 0x2c, 0x3b, + 0xa3, 0x07, 0x93, 0x2f, 0x89, 0x21, 0x7b, 0x6b, 0xe3, 0xce, 0x56, 0x85, 0xbb, 0x7c, 0x16, 0xd5, + 0x76, 0x87, 0x07, 0xba, 0x43, 0xcc, 0xc9, 0xef, 0x41, 0x42, 0xd6, 0xed, 0x5f, 0x82, 0x5c, 0x90, + 0x99, 0xa2, 0x35, 0xc8, 0x3e, 0x69, 0xe3, 0x47, 0x0d, 0xac, 0xcc, 0x09, 0x1f, 0x06, 0x9c, 0x27, + 0xa2, 0xa6, 0xd8, 0x80, 0xf9, 0x9d, 0x4a, 0xab, 0xf2, 0xb0, 0x81, 0x83, 0x2b, 0xf2, 0x00, 0x20, + 0xd3, 0xab, 0x92, 0x22, 0x3b, 0x08, 0x75, 0x56, 0x57, 0xbf, 0xfe, 0x66, 0x6d, 0xee, 0xa7, 0xdf, + 0xac, 0xcd, 0x3d, 0x3b, 0x5f, 0x4b, 0x7c, 0x7d, 0xbe, 0x96, 0xf8, 0xc9, 0xf9, 0x5a, 0xe2, 0x5f, + 0xcf, 0xd7, 0x12, 0x7b, 0x59, 0x7e, 0x08, 0xdc, 0xff, 0xdf, 0x00, 0x00, 0x00, 0xff, 0xff, 0xd0, + 0xd1, 0x79, 0x14, 0xb4, 0x30, 0x00, 0x00, } diff --git a/api/types.proto b/api/types.proto index db7fb746ef..eaf037e77d 100644 --- a/api/types.proto +++ b/api/types.proto @@ -514,6 +514,15 @@ message TaskStatus { // HostPorts provides a list of ports allocated at the host // level. PortStatus port_status = 6; + + // AppliedBy gives the node ID of the manager that applied this task + // status update to the Task object. + string applied_by = 7; + + // AppliedAt gives a timestamp of when this status update was applied to + // the Task object. + // Note: can't use stdtime because this field is nullable. + google.protobuf.Timestamp applied_at = 8; } // NetworkAttachmentConfig specifies how a service should be attached to a particular network. diff --git a/manager/dispatcher/dispatcher.go b/manager/dispatcher/dispatcher.go index 8ec3ae27df..c93554778d 100644 --- a/manager/dispatcher/dispatcher.go +++ b/manager/dispatcher/dispatcher.go @@ -19,6 +19,7 @@ import ( "github.com/docker/swarmkit/log" "github.com/docker/swarmkit/manager/drivers" "github.com/docker/swarmkit/manager/state/store" + "github.com/docker/swarmkit/protobuf/ptypes" "github.com/docker/swarmkit/remotes" "github.com/docker/swarmkit/watch" gogotypes "github.com/gogo/protobuf/types" @@ -127,6 +128,7 @@ type Dispatcher struct { cancel context.CancelFunc clusterUpdateQueue *watch.Queue dp *drivers.DriverProvider + securityConfig *ca.SecurityConfig taskUpdates map[string]*api.TaskStatus // indexed by task ID taskUpdatesLock sync.Mutex @@ -144,7 +146,7 @@ type Dispatcher struct { } // New returns Dispatcher with cluster interface(usually raft.Node). -func New(cluster Cluster, c *Config, dp *drivers.DriverProvider) *Dispatcher { +func New(cluster Cluster, c *Config, dp *drivers.DriverProvider, securityConfig *ca.SecurityConfig) *Dispatcher { d := &Dispatcher{ dp: dp, nodes: newNodeStore(c.HeartbeatPeriod, c.HeartbeatEpsilon, c.GracePeriodMultiplier, c.RateLimitPeriod), @@ -153,6 +155,7 @@ func New(cluster Cluster, c *Config, dp *drivers.DriverProvider) *Dispatcher { cluster: cluster, processUpdatesTrigger: make(chan struct{}, 1), config: c, + securityConfig: securityConfig, } d.processUpdatesCond = sync.NewCond(&d.processUpdatesLock) @@ -630,6 +633,8 @@ func (d *Dispatcher) processUpdates(ctx context.Context) { } task.Status = *status + task.Status.AppliedBy = d.securityConfig.ClientTLSCreds.NodeID() + task.Status.AppliedAt = ptypes.MustTimestampProto(time.Now()) if err := store.UpdateTask(tx, task); err != nil { logger.WithError(err).Error("failed to update task status") return nil diff --git a/manager/dispatcher/dispatcher_test.go b/manager/dispatcher/dispatcher_test.go index 086a4f249f..09dcf0d684 100644 --- a/manager/dispatcher/dispatcher_test.go +++ b/manager/dispatcher/dispatcher_test.go @@ -155,7 +155,7 @@ func startDispatcher(c *Config) (*grpcDispatcher, error) { s := grpc.NewServer(serverOpts...) tc := newTestCluster(l.Addr().String(), tca.MemoryStore) driverGetter := &mockPluginGetter{} - d := New(tc, c, drivers.New(driverGetter)) + d := New(tc, c, drivers.New(driverGetter), managerSecurityConfig) authorize := func(ctx context.Context, roles []string) error { _, err := ca.AuthorizeForwardedRoleAndOrg(ctx, roles, []string{ca.ManagerRole}, tca.Organization, nil) diff --git a/manager/manager.go b/manager/manager.go index 7c3facff3a..a9bda3be35 100644 --- a/manager/manager.go +++ b/manager/manager.go @@ -225,7 +225,7 @@ func New(config *Config) (*Manager, error) { m := &Manager{ config: *config, caserver: ca.NewServer(raftNode.MemoryStore(), config.SecurityConfig, config.RootCAPaths), - dispatcher: dispatcher.New(raftNode, dispatcher.DefaultConfig(), drivers.New(config.PluginGetter)), + dispatcher: dispatcher.New(raftNode, dispatcher.DefaultConfig(), drivers.New(config.PluginGetter), config.SecurityConfig), logbroker: logbroker.New(raftNode.MemoryStore()), watchServer: watchapi.NewServer(raftNode.MemoryStore()), server: grpc.NewServer(opts...), From 0072dffff4044cc37105d965c9d779b106d8f363 Mon Sep 17 00:00:00 2001 From: Aaron Lehmann Date: Fri, 21 Jul 2017 14:47:11 -0700 Subject: [PATCH 2/8] orchestrator: Use task history to evaluate restart policy Previously, restart conditions other than "OnAny" were honored on a best-effort basis. A service-level reconciliation, for example after a leader election, would see that not enough tasks were running, and start replacement tasks regardless of the restart policy. This limited the usefulness of the other restart conditions. This change makes the orchestrators check historic tasks to figure out if a task should be restarted or not, on service reconciliation. Signed-off-by: Aaron Lehmann --- manager/orchestrator/global/global.go | 27 +-- manager/orchestrator/global/global_test.go | 220 ++++++++++++++++- .../orchestrator/replicated/restart_test.go | 224 +++++++++++------- manager/orchestrator/replicated/services.go | 2 +- manager/orchestrator/replicated/slot.go | 35 +++ manager/orchestrator/restart/restart.go | 63 ++++- manager/orchestrator/slot.go | 34 --- manager/orchestrator/update/updater.go | 6 +- manager/orchestrator/update/updater_test.go | 17 +- 9 files changed, 474 insertions(+), 154 deletions(-) diff --git a/manager/orchestrator/global/global.go b/manager/orchestrator/global/global.go index bced3dd189..b850c8cf95 100644 --- a/manager/orchestrator/global/global.go +++ b/manager/orchestrator/global/global.go @@ -265,8 +265,15 @@ func (g *Orchestrator) reconcileServices(ctx context.Context, serviceIDs []strin nodeTasks[serviceID] = make(map[string][]*api.Task) for _, t := range tasks { - if t.DesiredState <= api.TaskStateRunning { - // Collect all running instances of this service + service := g.globalServices[serviceID].Service + if service == nil { + continue + } + // Collect all runnable instances of this service, + // and instances that were not be restarted due + // to restart policy but may be updated if the + // service spec changed. + if g.restarts.IsTaskUpdatable(ctx, t, service) { nodeTasks[serviceID][t.NodeID] = append(nodeTasks[serviceID][t.NodeID], t) } } @@ -374,11 +381,6 @@ func (g *Orchestrator) reconcileOneNode(ctx context.Context, node *api.Node) { return } - var serviceIDs []string - for id := range g.globalServices { - serviceIDs = append(serviceIDs, id) - } - node, exists := g.nodes[node.ID] if !exists { return @@ -400,24 +402,19 @@ func (g *Orchestrator) reconcileOneNode(ctx context.Context, node *api.Node) { return } - for _, serviceID := range serviceIDs { + for serviceID, service := range g.globalServices { for _, t := range tasksOnNode { if t.ServiceID != serviceID { continue } - if t.DesiredState <= api.TaskStateRunning { + if g.restarts.IsTaskUpdatable(ctx, t, service.Service) { tasks[serviceID] = append(tasks[serviceID], t) } } } err = g.store.Batch(func(batch *store.Batch) error { - for _, serviceID := range serviceIDs { - service, exists := g.globalServices[serviceID] - if !exists { - continue - } - + for serviceID, service := range g.globalServices { if !constraint.NodeMatches(service.constraints, node) { continue } diff --git a/manager/orchestrator/global/global_test.go b/manager/orchestrator/global/global_test.go index 51f4eda376..b22f96da28 100644 --- a/manager/orchestrator/global/global_test.go +++ b/manager/orchestrator/global/global_test.go @@ -213,7 +213,7 @@ func TestNodeAvailability(t *testing.T) { testutils.Expect(t, watch, state.EventCommit{}) // updating the service shouldn't restart the task - updateService(t, store, service1) + updateService(t, store, service1, true) testutils.Expect(t, watch, api.EventUpdateService{}) testutils.Expect(t, watch, state.EventCommit{}) select { @@ -241,7 +241,7 @@ func TestNodeAvailability(t *testing.T) { testutils.Expect(t, watch, state.EventCommit{}) // updating the service shouldn't restart the task - updateService(t, store, service1) + updateService(t, store, service1, true) testutils.Expect(t, watch, api.EventUpdateService{}) testutils.Expect(t, watch, state.EventCommit{}) select { @@ -277,7 +277,7 @@ func TestNodeState(t *testing.T) { testutils.Expect(t, watch, state.EventCommit{}) // updating the service shouldn't restart the task - updateService(t, store, service1) + updateService(t, store, service1, true) testutils.Expect(t, watch, api.EventUpdateService{}) testutils.Expect(t, watch, state.EventCommit{}) select { @@ -425,8 +425,20 @@ func TestTaskFailure(t *testing.T) { case <-time.After(100 * time.Millisecond): } - // update the service. now the task should be recreated. - updateService(t, store, serviceNoRestart) + // update the service with no spec changes, to trigger a + // reconciliation. the task should still not be updated. + updateService(t, store, serviceNoRestart, false) + testutils.Expect(t, watch, api.EventUpdateService{}) + testutils.Expect(t, watch, state.EventCommit{}) + + select { + case event := <-watch: + t.Fatalf("got unexpected event %T: %+v", event, event) + case <-time.After(100 * time.Millisecond): + } + + // update the service with spec changes. now the task should be recreated. + updateService(t, store, serviceNoRestart, true) testutils.Expect(t, watch, api.EventUpdateService{}) testutils.Expect(t, watch, state.EventCommit{}) @@ -444,11 +456,13 @@ func addService(t *testing.T, s *store.MemoryStore, service *api.Service) { }) } -func updateService(t *testing.T, s *store.MemoryStore, service *api.Service) { +func updateService(t *testing.T, s *store.MemoryStore, service *api.Service, force bool) { s.Update(func(tx store.Tx) error { service := store.GetService(tx, service.ID) require.NotNil(t, service) - service.Spec.Task.ForceUpdate++ + if force { + service.Spec.Task.ForceUpdate++ + } assert.NoError(t, store.UpdateService(tx, service)) return nil }) @@ -1097,3 +1111,195 @@ func TestInitializationTaskOnNonexistentNode(t *testing.T) { assert.Equal(t, deadCnt, 1) assert.Equal(t, liveCnt, 0) } + +func TestInitializationRestartHistory(t *testing.T) { + ctx := context.Background() + s := store.NewMemoryStore(nil) + assert.NotNil(t, s) + defer s.Close() + + // create nodes, services and tasks in store directly + addNode(t, s, node1) + + service := &api.Service{ + ID: "serviceid1", + SpecVersion: &api.Version{ + Index: 2, + }, + Spec: api.ServiceSpec{ + Annotations: api.Annotations{ + Name: "name1", + }, + Task: api.TaskSpec{ + Runtime: &api.TaskSpec_Container{ + Container: &api.ContainerSpec{}, + }, + Restart: &api.RestartPolicy{ + Condition: api.RestartOnAny, + Delay: gogotypes.DurationProto(restartDelay), + MaxAttempts: 3, + Window: gogotypes.DurationProto(10 * time.Minute), + }, + }, + Mode: &api.ServiceSpec_Global{ + Global: &api.GlobalService{}, + }, + }, + } + addService(t, s, service) + + now := time.Now() + + tasks := []*api.Task{ + // old spec versions should be ignored for restart tracking + { + ID: "oldspec", + Meta: api.Meta{ + CreatedAt: ptypes.MustTimestampProto(now.Add(-5 * time.Minute)), + }, + DesiredState: api.TaskStateShutdown, + SpecVersion: &api.Version{ + Index: 1, + }, + Status: api.TaskStatus{ + State: api.TaskStateShutdown, + Timestamp: ptypes.MustTimestampProto(now.Add(-5 * time.Minute)), + }, + Spec: service.Spec.Task, + ServiceID: "serviceid1", + NodeID: "nodeid1", + }, + // this is the first task with the current spec version + { + ID: "firstcurrent", + Meta: api.Meta{ + CreatedAt: ptypes.MustTimestampProto(now.Add(-12 * time.Minute)), + }, + DesiredState: api.TaskStateShutdown, + SpecVersion: &api.Version{ + Index: 2, + }, + Status: api.TaskStatus{ + State: api.TaskStateFailed, + Timestamp: ptypes.MustTimestampProto(now.Add(-12 * time.Minute)), + }, + Spec: service.Spec.Task, + ServiceID: "serviceid1", + NodeID: "nodeid1", + }, + + // this task falls outside the restart window + { + ID: "outsidewindow", + Meta: api.Meta{ + CreatedAt: ptypes.MustTimestampProto(now.Add(-11 * time.Minute)), + }, + DesiredState: api.TaskStateShutdown, + SpecVersion: &api.Version{ + Index: 2, + }, + Status: api.TaskStatus{ + State: api.TaskStateFailed, + Timestamp: ptypes.MustTimestampProto(now.Add(-11 * time.Minute)), + }, + Spec: service.Spec.Task, + ServiceID: "serviceid1", + NodeID: "nodeid1", + }, + // first task inside restart window + { + ID: "firstinside", + Meta: api.Meta{ + CreatedAt: ptypes.MustTimestampProto(now.Add(-9 * time.Minute)), + }, + DesiredState: api.TaskStateShutdown, + SpecVersion: &api.Version{ + Index: 2, + }, + Status: api.TaskStatus{ + State: api.TaskStateFailed, + Timestamp: ptypes.MustTimestampProto(now.Add(-9 * time.Minute)), + }, + Spec: service.Spec.Task, + ServiceID: "serviceid1", + NodeID: "nodeid1", + }, + // second task inside restart window, currently running + { + ID: "secondinside", + Meta: api.Meta{ + CreatedAt: ptypes.MustTimestampProto(now.Add(-8 * time.Minute)), + }, + DesiredState: api.TaskStateRunning, + SpecVersion: &api.Version{ + Index: 2, + }, + Status: api.TaskStatus{ + State: api.TaskStateRunning, + Timestamp: ptypes.MustTimestampProto(now.Add(-8 * time.Minute)), + }, + Spec: service.Spec.Task, + ServiceID: "serviceid1", + NodeID: "nodeid1", + }, + } + for _, task := range tasks { + addTask(t, s, task) + } + + // watch orchestration events + watch, cancel := state.Watch(s.WatchQueue(), api.EventCreateTask{}, api.EventUpdateTask{}, api.EventDeleteTask{}) + defer cancel() + + orchestrator := NewGlobalOrchestrator(s) + defer orchestrator.Stop() + + go func() { + assert.NoError(t, orchestrator.Run(ctx)) + }() + + // Fail the running task + s.Update(func(tx store.Tx) error { + task := store.GetTask(tx, "secondinside") + require.NotNil(t, task) + task.Status.State = api.TaskStateFailed + task.Status.Timestamp = ptypes.MustTimestampProto(time.Now()) + assert.NoError(t, store.UpdateTask(tx, task)) + return nil + }) + testutils.Expect(t, watch, api.EventUpdateTask{}) + + // It should restart, because this will only be the third restart + // attempt within the time window. + observedTask1 := testutils.WatchTaskUpdate(t, watch) + assert.Equal(t, "secondinside", observedTask1.ID) + assert.Equal(t, api.TaskStateFailed, observedTask1.Status.State) + + observedTask2 := testutils.WatchTaskCreate(t, watch) + assert.Equal(t, observedTask2.NodeID, "nodeid1") + assert.Equal(t, api.TaskStateNew, observedTask2.Status.State) + assert.Equal(t, api.TaskStateReady, observedTask2.DesiredState) + + observedTask3 := testutils.WatchTaskUpdate(t, watch) + assert.Equal(t, observedTask2.ID, observedTask3.ID) + assert.Equal(t, api.TaskStateRunning, observedTask3.DesiredState) + + // Reject the new task + s.Update(func(tx store.Tx) error { + task := store.GetTask(tx, observedTask2.ID) + require.NotNil(t, task) + task.Status.State = api.TaskStateRejected + task.Status.Timestamp = ptypes.MustTimestampProto(time.Now()) + assert.NoError(t, store.UpdateTask(tx, task)) + return nil + }) + testutils.Expect(t, watch, api.EventUpdateTask{}) // our update + testutils.Expect(t, watch, api.EventUpdateTask{}) // orchestrator changes desired state + + // It shouldn't restart - that would exceed MaxAttempts + select { + case event := <-watch: + t.Fatalf("got unexpected event %T: %+v", event, event) + case <-time.After(100 * time.Millisecond): + } +} diff --git a/manager/orchestrator/replicated/restart_test.go b/manager/orchestrator/replicated/restart_test.go index 2d621c36a8..b90268db94 100644 --- a/manager/orchestrator/replicated/restart_test.go +++ b/manager/orchestrator/replicated/restart_test.go @@ -8,6 +8,7 @@ import ( "github.com/docker/swarmkit/manager/orchestrator/testutils" "github.com/docker/swarmkit/manager/state" "github.com/docker/swarmkit/manager/state/store" + "github.com/docker/swarmkit/protobuf/ptypes" gogotypes "github.com/gogo/protobuf/types" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -72,7 +73,7 @@ func TestOrchestratorRestartOnAny(t *testing.T) { // Fail the first task. Confirm that it gets restarted. updatedTask1 := observedTask1.Copy() - updatedTask1.Status = api.TaskStatus{State: api.TaskStateFailed} + updatedTask1.Status = api.TaskStatus{State: api.TaskStateFailed, Timestamp: ptypes.MustTimestampProto(time.Now())} err = s.Update(func(tx store.Tx) error { assert.NoError(t, store.UpdateTask(tx, updatedTask1)) return nil @@ -95,7 +96,7 @@ func TestOrchestratorRestartOnAny(t *testing.T) { // Mark the second task as completed. Confirm that it gets restarted. updatedTask2 := observedTask2.Copy() - updatedTask2.Status = api.TaskStatus{State: api.TaskStateCompleted} + updatedTask2.Status = api.TaskStatus{State: api.TaskStateCompleted, Timestamp: ptypes.MustTimestampProto(time.Now())} err = s.Update(func(tx store.Tx) error { assert.NoError(t, store.UpdateTask(tx, updatedTask2)) return nil @@ -118,6 +119,8 @@ func TestOrchestratorRestartOnAny(t *testing.T) { } func TestOrchestratorRestartOnFailure(t *testing.T) { + t.Parallel() + ctx := context.Background() s := store.NewMemoryStore(nil) assert.NotNil(t, s) @@ -126,7 +129,7 @@ func TestOrchestratorRestartOnFailure(t *testing.T) { orchestrator := NewReplicatedOrchestrator(s) defer orchestrator.Stop() - watch, cancel := state.Watch(s.WatchQueue() /*api.EventCreateTask{}, api.EventUpdateTask{}*/) + watch, cancel := state.Watch(s.WatchQueue(), api.EventCreateTask{}, api.EventUpdateTask{}) defer cancel() // Create a service with two instances specified before the orchestrator is @@ -175,15 +178,13 @@ func TestOrchestratorRestartOnFailure(t *testing.T) { // Fail the first task. Confirm that it gets restarted. updatedTask1 := observedTask1.Copy() - updatedTask1.Status = api.TaskStatus{State: api.TaskStateFailed} + updatedTask1.Status = api.TaskStatus{State: api.TaskStateFailed, Timestamp: ptypes.MustTimestampProto(time.Now())} err = s.Update(func(tx store.Tx) error { assert.NoError(t, store.UpdateTask(tx, updatedTask1)) return nil }) assert.NoError(t, err) - testutils.Expect(t, watch, state.EventCommit{}) testutils.Expect(t, watch, api.EventUpdateTask{}) - testutils.Expect(t, watch, state.EventCommit{}) testutils.Expect(t, watch, api.EventUpdateTask{}) observedTask3 := testutils.WatchTaskCreate(t, watch) @@ -191,34 +192,59 @@ func TestOrchestratorRestartOnFailure(t *testing.T) { assert.Equal(t, observedTask3.DesiredState, api.TaskStateReady) assert.Equal(t, observedTask3.ServiceAnnotations.Name, "name1") - testutils.Expect(t, watch, state.EventCommit{}) - observedTask4 := testutils.WatchTaskUpdate(t, watch) assert.Equal(t, observedTask4.DesiredState, api.TaskStateRunning) assert.Equal(t, observedTask4.ServiceAnnotations.Name, "name1") // Mark the second task as completed. Confirm that it does not get restarted. updatedTask2 := observedTask2.Copy() - updatedTask2.Status = api.TaskStatus{State: api.TaskStateCompleted} + updatedTask2.Status = api.TaskStatus{State: api.TaskStateCompleted, Timestamp: ptypes.MustTimestampProto(time.Now())} err = s.Update(func(tx store.Tx) error { assert.NoError(t, store.UpdateTask(tx, updatedTask2)) return nil }) assert.NoError(t, err) - testutils.Expect(t, watch, state.EventCommit{}) testutils.Expect(t, watch, api.EventUpdateTask{}) - testutils.Expect(t, watch, state.EventCommit{}) testutils.Expect(t, watch, api.EventUpdateTask{}) - testutils.Expect(t, watch, state.EventCommit{}) select { case <-watch: t.Fatal("got unexpected event") case <-time.After(100 * time.Millisecond): } + + // Update the service, but don't change anything in the spec. The + // second instance instance should not be restarted. + err = s.Update(func(tx store.Tx) error { + service := store.GetService(tx, "id1") + require.NotNil(t, service) + assert.NoError(t, store.UpdateService(tx, service)) + return nil + }) + assert.NoError(t, err) + + select { + case <-watch: + t.Fatal("got unexpected event") + case <-time.After(100 * time.Millisecond): + } + + // Update the service, and change the TaskSpec. Now the second instance + // should be restarted. + err = s.Update(func(tx store.Tx) error { + service := store.GetService(tx, "id1") + require.NotNil(t, service) + service.Spec.Task.ForceUpdate++ + assert.NoError(t, store.UpdateService(tx, service)) + return nil + }) + assert.NoError(t, err) + testutils.Expect(t, watch, api.EventCreateTask{}) } func TestOrchestratorRestartOnNone(t *testing.T) { + t.Parallel() + ctx := context.Background() s := store.NewMemoryStore(nil) assert.NotNil(t, s) @@ -227,7 +253,7 @@ func TestOrchestratorRestartOnNone(t *testing.T) { orchestrator := NewReplicatedOrchestrator(s) defer orchestrator.Stop() - watch, cancel := state.Watch(s.WatchQueue() /*api.EventCreateTask{}, api.EventUpdateTask{}*/) + watch, cancel := state.Watch(s.WatchQueue(), api.EventCreateTask{}, api.EventUpdateTask{}) defer cancel() // Create a service with two instances specified before the orchestrator is @@ -281,11 +307,8 @@ func TestOrchestratorRestartOnNone(t *testing.T) { return nil }) assert.NoError(t, err) - testutils.Expect(t, watch, state.EventCommit{}) testutils.Expect(t, watch, api.EventUpdateTask{}) - testutils.Expect(t, watch, state.EventCommit{}) testutils.Expect(t, watch, api.EventUpdateTask{}) - testutils.Expect(t, watch, state.EventCommit{}) select { case <-watch: @@ -295,25 +318,66 @@ func TestOrchestratorRestartOnNone(t *testing.T) { // Mark the second task as completed. Confirm that it does not get restarted. updatedTask2 := observedTask2.Copy() - updatedTask2.Status = api.TaskStatus{State: api.TaskStateCompleted} + updatedTask2.Status = api.TaskStatus{State: api.TaskStateCompleted, Timestamp: ptypes.MustTimestampProto(time.Now())} err = s.Update(func(tx store.Tx) error { assert.NoError(t, store.UpdateTask(tx, updatedTask2)) return nil }) assert.NoError(t, err) testutils.Expect(t, watch, api.EventUpdateTask{}) - testutils.Expect(t, watch, state.EventCommit{}) testutils.Expect(t, watch, api.EventUpdateTask{}) - testutils.Expect(t, watch, state.EventCommit{}) select { case <-watch: t.Fatal("got unexpected event") case <-time.After(100 * time.Millisecond): } + + // Update the service, but don't change anything in the spec. Neither + // instance should be restarted. + err = s.Update(func(tx store.Tx) error { + service := store.GetService(tx, "id1") + require.NotNil(t, service) + assert.NoError(t, store.UpdateService(tx, service)) + return nil + }) + assert.NoError(t, err) + + select { + case <-watch: + t.Fatal("got unexpected event") + case <-time.After(100 * time.Millisecond): + } + + // Update the service, and change the TaskSpec. Both instances should + // be restarted. + err = s.Update(func(tx store.Tx) error { + service := store.GetService(tx, "id1") + require.NotNil(t, service) + service.Spec.Task.ForceUpdate++ + assert.NoError(t, store.UpdateService(tx, service)) + return nil + }) + assert.NoError(t, err) + testutils.Expect(t, watch, api.EventCreateTask{}) + newTask := testutils.WatchTaskUpdate(t, watch) + assert.Equal(t, api.TaskStateRunning, newTask.DesiredState) + err = s.Update(func(tx store.Tx) error { + newTask := store.GetTask(tx, newTask.ID) + require.NotNil(t, newTask) + newTask.Status.State = api.TaskStateRunning + assert.NoError(t, store.UpdateTask(tx, newTask)) + return nil + }) + assert.NoError(t, err) + testutils.Expect(t, watch, api.EventUpdateTask{}) + + testutils.Expect(t, watch, api.EventCreateTask{}) } func TestOrchestratorRestartDelay(t *testing.T) { + t.Parallel() + ctx := context.Background() s := store.NewMemoryStore(nil) assert.NotNil(t, s) @@ -371,7 +435,7 @@ func TestOrchestratorRestartDelay(t *testing.T) { // Fail the first task. Confirm that it gets restarted. updatedTask1 := observedTask1.Copy() - updatedTask1.Status = api.TaskStatus{State: api.TaskStateFailed} + updatedTask1.Status = api.TaskStatus{State: api.TaskStateFailed, Timestamp: ptypes.MustTimestampProto(time.Now())} before := time.Now() err = s.Update(func(tx store.Tx) error { assert.NoError(t, store.UpdateTask(tx, updatedTask1)) @@ -414,7 +478,7 @@ func TestOrchestratorRestartMaxAttempts(t *testing.T) { orchestrator := NewReplicatedOrchestrator(s) defer orchestrator.Stop() - watch, cancel := state.Watch(s.WatchQueue() /*api.EventCreateTask{}, api.EventUpdateTask{}*/) + watch, cancel := state.Watch(s.WatchQueue(), api.EventCreateTask{}, api.EventUpdateTask{}) defer cancel() // Create a service with two instances specified before the orchestrator is @@ -457,38 +521,65 @@ func TestOrchestratorRestartMaxAttempts(t *testing.T) { assert.NoError(t, orchestrator.Run(ctx)) }() - testRestart := func() { + failTask := func(task *api.Task, expectRestart bool) { + task = task.Copy() + task.Status = api.TaskStatus{State: api.TaskStateFailed, Timestamp: ptypes.MustTimestampProto(time.Now())} + err = s.Update(func(tx store.Tx) error { + assert.NoError(t, store.UpdateTask(tx, task)) + return nil + }) + assert.NoError(t, err) + testutils.Expect(t, watch, api.EventUpdateTask{}) + task = testutils.WatchShutdownTask(t, watch) + if expectRestart { + createdTask := testutils.WatchTaskCreate(t, watch) + assert.Equal(t, createdTask.Status.State, api.TaskStateNew) + assert.Equal(t, createdTask.DesiredState, api.TaskStateReady) + assert.Equal(t, createdTask.ServiceAnnotations.Name, "name1") + } + err = s.Update(func(tx store.Tx) error { + task := task.Copy() + task.Status.State = api.TaskStateShutdown + assert.NoError(t, store.UpdateTask(tx, task)) + return nil + }) + assert.NoError(t, err) + testutils.Expect(t, watch, api.EventUpdateTask{}) + } + + testRestart := func(serviceUpdated bool) { observedTask1 := testutils.WatchTaskCreate(t, watch) assert.Equal(t, observedTask1.Status.State, api.TaskStateNew) assert.Equal(t, observedTask1.ServiceAnnotations.Name, "name1") + if serviceUpdated { + runnableTask := testutils.WatchTaskUpdate(t, watch) + assert.Equal(t, observedTask1.ID, runnableTask.ID) + assert.Equal(t, api.TaskStateRunning, runnableTask.DesiredState) + err = s.Update(func(tx store.Tx) error { + task := runnableTask.Copy() + task.Status.State = api.TaskStateRunning + assert.NoError(t, store.UpdateTask(tx, task)) + return nil + }) + assert.NoError(t, err) + + testutils.Expect(t, watch, api.EventUpdateTask{}) + } + observedTask2 := testutils.WatchTaskCreate(t, watch) assert.Equal(t, observedTask2.Status.State, api.TaskStateNew) assert.Equal(t, observedTask2.ServiceAnnotations.Name, "name1") - testutils.Expect(t, watch, state.EventCommit{}) + if serviceUpdated { + testutils.Expect(t, watch, api.EventUpdateTask{}) + } // Fail the first task. Confirm that it gets restarted. - updatedTask1 := observedTask1.Copy() - updatedTask1.Status = api.TaskStatus{State: api.TaskStateFailed} before := time.Now() - err = s.Update(func(tx store.Tx) error { - assert.NoError(t, store.UpdateTask(tx, updatedTask1)) - return nil - }) - assert.NoError(t, err) - testutils.Expect(t, watch, api.EventUpdateTask{}) - testutils.Expect(t, watch, state.EventCommit{}) - testutils.Expect(t, watch, api.EventUpdateTask{}) - - observedTask3 := testutils.WatchTaskCreate(t, watch) - testutils.Expect(t, watch, state.EventCommit{}) - assert.Equal(t, observedTask3.Status.State, api.TaskStateNew) - assert.Equal(t, observedTask3.DesiredState, api.TaskStateReady) - assert.Equal(t, observedTask3.ServiceAnnotations.Name, "name1") + failTask(observedTask1, true) observedTask4 := testutils.WatchTaskUpdate(t, watch) - testutils.Expect(t, watch, state.EventCommit{}) after := time.Now() // At least 100 ms should have elapsed. Only check the lower bound, @@ -502,40 +593,15 @@ func TestOrchestratorRestartMaxAttempts(t *testing.T) { assert.Equal(t, observedTask4.ServiceAnnotations.Name, "name1") // Fail the second task. Confirm that it gets restarted. - updatedTask2 := observedTask2.Copy() - updatedTask2.Status = api.TaskStatus{State: api.TaskStateFailed} - err = s.Update(func(tx store.Tx) error { - assert.NoError(t, store.UpdateTask(tx, updatedTask2)) - return nil - }) - assert.NoError(t, err) - testutils.Expect(t, watch, api.EventUpdateTask{}) - testutils.Expect(t, watch, state.EventCommit{}) - testutils.Expect(t, watch, api.EventUpdateTask{}) - - observedTask5 := testutils.WatchTaskCreate(t, watch) - testutils.Expect(t, watch, state.EventCommit{}) - assert.Equal(t, observedTask5.Status.State, api.TaskStateNew) - assert.Equal(t, observedTask5.DesiredState, api.TaskStateReady) + failTask(observedTask2, true) observedTask6 := testutils.WatchTaskUpdate(t, watch) // task gets started after a delay - testutils.Expect(t, watch, state.EventCommit{}) assert.Equal(t, observedTask6.Status.State, api.TaskStateNew) assert.Equal(t, observedTask6.DesiredState, api.TaskStateRunning) assert.Equal(t, observedTask6.ServiceAnnotations.Name, "name1") // Fail the first instance again. It should not be restarted. - updatedTask1 = observedTask3.Copy() - updatedTask1.Status = api.TaskStatus{State: api.TaskStateFailed} - err = s.Update(func(tx store.Tx) error { - assert.NoError(t, store.UpdateTask(tx, updatedTask1)) - return nil - }) - assert.NoError(t, err) - testutils.Expect(t, watch, api.EventUpdateTask{}) - testutils.Expect(t, watch, state.EventCommit{}) - testutils.Expect(t, watch, api.EventUpdateTask{}) - testutils.Expect(t, watch, state.EventCommit{}) + failTask(observedTask4, false) select { case <-watch: @@ -544,17 +610,7 @@ func TestOrchestratorRestartMaxAttempts(t *testing.T) { } // Fail the second instance again. It should not be restarted. - updatedTask2 = observedTask5.Copy() - updatedTask2.Status = api.TaskStatus{State: api.TaskStateFailed} - err = s.Update(func(tx store.Tx) error { - assert.NoError(t, store.UpdateTask(tx, updatedTask2)) - return nil - }) - assert.NoError(t, err) - testutils.Expect(t, watch, api.EventUpdateTask{}) - testutils.Expect(t, watch, state.EventCommit{}) - testutils.Expect(t, watch, api.EventUpdateTask{}) - testutils.Expect(t, watch, state.EventCommit{}) + failTask(observedTask6, false) select { case <-watch: @@ -563,7 +619,7 @@ func TestOrchestratorRestartMaxAttempts(t *testing.T) { } } - testRestart() + testRestart(false) // Update the service spec err = s.Update(func(tx store.Tx) error { @@ -576,7 +632,7 @@ func TestOrchestratorRestartMaxAttempts(t *testing.T) { }) assert.NoError(t, err) - testRestart() + testRestart(true) } func TestOrchestratorRestartWindow(t *testing.T) { @@ -638,7 +694,7 @@ func TestOrchestratorRestartWindow(t *testing.T) { // Fail the first task. Confirm that it gets restarted. updatedTask1 := observedTask1.Copy() - updatedTask1.Status = api.TaskStatus{State: api.TaskStateFailed} + updatedTask1.Status = api.TaskStatus{State: api.TaskStateFailed, Timestamp: ptypes.MustTimestampProto(time.Now())} before := time.Now() err = s.Update(func(tx store.Tx) error { assert.NoError(t, store.UpdateTask(tx, updatedTask1)) @@ -671,7 +727,7 @@ func TestOrchestratorRestartWindow(t *testing.T) { // Fail the second task. Confirm that it gets restarted. updatedTask2 := observedTask2.Copy() - updatedTask2.Status = api.TaskStatus{State: api.TaskStateFailed} + updatedTask2.Status = api.TaskStatus{State: api.TaskStateFailed, Timestamp: ptypes.MustTimestampProto(time.Now())} err = s.Update(func(tx store.Tx) error { assert.NoError(t, store.UpdateTask(tx, updatedTask2)) return nil @@ -696,7 +752,7 @@ func TestOrchestratorRestartWindow(t *testing.T) { // Fail the first instance again. It should not be restarted. updatedTask1 = observedTask3.Copy() - updatedTask1.Status = api.TaskStatus{State: api.TaskStateFailed} + updatedTask1.Status = api.TaskStatus{State: api.TaskStateFailed, Timestamp: ptypes.MustTimestampProto(time.Now())} err = s.Update(func(tx store.Tx) error { assert.NoError(t, store.UpdateTask(tx, updatedTask1)) return nil @@ -718,7 +774,7 @@ func TestOrchestratorRestartWindow(t *testing.T) { // Fail the second instance again. It should get restarted because // enough time has elapsed since the last restarts. updatedTask2 = observedTask5.Copy() - updatedTask2.Status = api.TaskStatus{State: api.TaskStateFailed} + updatedTask2.Status = api.TaskStatus{State: api.TaskStateFailed, Timestamp: ptypes.MustTimestampProto(time.Now())} before = time.Now() err = s.Update(func(tx store.Tx) error { assert.NoError(t, store.UpdateTask(tx, updatedTask2)) diff --git a/manager/orchestrator/replicated/services.go b/manager/orchestrator/replicated/services.go index 101976d96e..f4d0511f80 100644 --- a/manager/orchestrator/replicated/services.go +++ b/manager/orchestrator/replicated/services.go @@ -87,7 +87,7 @@ func (r *Orchestrator) resolveService(ctx context.Context, task *api.Task) *api. } func (r *Orchestrator) reconcile(ctx context.Context, service *api.Service) { - runningSlots, deadSlots, err := orchestrator.GetRunnableAndDeadSlots(r.store, service.ID) + runningSlots, deadSlots, err := r.updatableAndDeadSlots(ctx, service) if err != nil { log.G(ctx).WithError(err).Errorf("reconcile failed finding tasks") return diff --git a/manager/orchestrator/replicated/slot.go b/manager/orchestrator/replicated/slot.go index 1f1fd3eca4..15b11c0ff0 100644 --- a/manager/orchestrator/replicated/slot.go +++ b/manager/orchestrator/replicated/slot.go @@ -3,6 +3,8 @@ package replicated import ( "github.com/docker/swarmkit/api" "github.com/docker/swarmkit/manager/orchestrator" + "github.com/docker/swarmkit/manager/state/store" + "golang.org/x/net/context" ) type slotsByRunningState []orchestrator.Slot @@ -53,3 +55,36 @@ func (is slotsByIndex) Less(i, j int) bool { } return is[i].index < is[j].index } + +// updatableAndDeadSlots returns two maps of slots. The first contains slots +// that have at least one task with a desired state above NEW and lesser or +// equal to RUNNING, or a task that shouldn't be restarted. The second contains +// all other slots with at least one task. +func (r *Orchestrator) updatableAndDeadSlots(ctx context.Context, service *api.Service) (map[uint64]orchestrator.Slot, map[uint64]orchestrator.Slot, error) { + var ( + tasks []*api.Task + err error + ) + r.store.View(func(tx store.ReadTx) { + tasks, err = store.FindTasks(tx, store.ByServiceID(service.ID)) + }) + if err != nil { + return nil, nil, err + } + + updatableSlots := make(map[uint64]orchestrator.Slot) + for _, t := range tasks { + if r.restarts.IsTaskUpdatable(ctx, t, service) { + updatableSlots[t.Slot] = append(updatableSlots[t.Slot], t) + } + } + + deadSlots := make(map[uint64]orchestrator.Slot) + for _, t := range tasks { + if _, exists := updatableSlots[t.Slot]; !exists { + deadSlots[t.Slot] = append(deadSlots[t.Slot], t) + } + } + + return updatableSlots, deadSlots, nil +} diff --git a/manager/orchestrator/restart/restart.go b/manager/orchestrator/restart/restart.go index eed28f8202..98d8c9676c 100644 --- a/manager/orchestrator/restart/restart.go +++ b/manager/orchestrator/restart/restart.go @@ -138,7 +138,7 @@ func (r *Supervisor) Restart(ctx context.Context, tx store.Tx, cluster *api.Clus return err } - if !r.shouldRestart(ctx, &t, service) { + if !r.ShouldRestart(ctx, &t, service) { return nil } @@ -191,9 +191,10 @@ func (r *Supervisor) Restart(ctx context.Context, tx store.Tx, cluster *api.Clus return nil } -func (r *Supervisor) shouldRestart(ctx context.Context, t *api.Task, service *api.Service) bool { +// ShouldRestart returns true if a task should be restarted according to the +// restart policy. +func (r *Supervisor) ShouldRestart(ctx context.Context, t *api.Task, service *api.Service) bool { // TODO(aluzzardi): This function should not depend on `service`. - condition := orchestrator.RestartCondition(t) if condition != api.RestartOnAny && @@ -237,27 +238,69 @@ func (r *Supervisor) shouldRestart(ctx context.Context, t *api.Task, service *ap log.G(ctx).WithError(err).Error("invalid restart lookback window") return restartInfo.totalRestarts < t.Spec.Restart.MaxAttempts } - lookback := time.Now().Add(-window) + + var timestamp time.Time + // Prefer the manager's timestamp over the agent's, since manager + // clocks are more trustworthy. + if t.Status.AppliedAt != nil { + timestamp, err = gogotypes.TimestampFromProto(t.Status.AppliedAt) + if err != nil { + log.G(ctx).WithError(err).Error("invalid task status AppliedAt timestamp") + return restartInfo.totalRestarts < t.Spec.Restart.MaxAttempts + } + } else { + // It's safe to call TimestampFromProto with a nil timestamp + timestamp, err = gogotypes.TimestampFromProto(t.Status.Timestamp) + if t.Status.Timestamp == nil || err != nil { + log.G(ctx).WithError(err).Error("invalid task completion timestamp") + return restartInfo.totalRestarts < t.Spec.Restart.MaxAttempts + } + } + lookback := timestamp.Add(-window) + + numRestarts := uint64(restartInfo.restartedInstances.Len()) var next *list.Element for e := restartInfo.restartedInstances.Front(); e != nil; e = next { next = e.Next() if e.Value.(restartedInstance).timestamp.After(lookback) { + for e2 := restartInfo.restartedInstances.Back(); e2 != nil; e2 = e2.Prev() { + // Ignore restarts that didn't happen before the + // task we're looking at. + if e.Value.(restartedInstance).timestamp.Before(timestamp) { + break + } + numRestarts-- + } break } restartInfo.restartedInstances.Remove(e) + numRestarts-- } - numRestarts := uint64(restartInfo.restartedInstances.Len()) - - if numRestarts == 0 { + if restartInfo.restartedInstances.Len() == 0 { restartInfo.restartedInstances = nil } return numRestarts < t.Spec.Restart.MaxAttempts } +// IsTaskUpdatable determines whether the task should be passed to the updater +// or not. An updatable task is either a task that with desired state <= +// RUNNING, or a task which has stopped running and should not be restarted. The +// latter case is for making sure that tasks that shouldn't normally be +// restarted will still be handled by rolling updates when they become outdated. +// There is a special case for rollbacks to make sure that a rollback always +// takes the service to a converged state, instead of ignoring tasks with the +// original spec that stopped running and shouldn't be restarted according to +// the restart policy. +func (r *Supervisor) IsTaskUpdatable(ctx context.Context, t *api.Task, service *api.Service) bool { + return t.DesiredState <= api.TaskStateRunning || + ((service.UpdateStatus == nil || service.UpdateStatus.State != api.UpdateStatus_ROLLBACK_STARTED) && + !r.ShouldRestart(ctx, t, service)) +} + func (r *Supervisor) recordRestartHistory(restartTask *api.Task) { if restartTask.Spec.Restart == nil || restartTask.Spec.Restart.MaxAttempts == 0 { // No limit on the number of restarts, so no need to record @@ -337,7 +380,9 @@ func (r *Supervisor) DelayStart(ctx context.Context, _ store.Tx, oldTask *api.Ta var watch chan events.Event cancelWatch := func() {} - if waitStop && oldTask != nil { + waitForTask := waitStop && oldTask != nil && oldTask.Status.State <= api.TaskStateRunning + + if waitForTask { // Wait for either the old task to complete, or the old task's // node to become unavailable. watch, cancelWatch = state.Watch( @@ -378,7 +423,7 @@ func (r *Supervisor) DelayStart(ctx context.Context, _ store.Tx, oldTask *api.Ta } } - if waitStop && oldTask != nil { + if waitForTask { select { case <-watch: case <-oldTaskTimer.C: diff --git a/manager/orchestrator/slot.go b/manager/orchestrator/slot.go index ce347136ba..732933f1a5 100644 --- a/manager/orchestrator/slot.go +++ b/manager/orchestrator/slot.go @@ -2,7 +2,6 @@ package orchestrator import ( "github.com/docker/swarmkit/api" - "github.com/docker/swarmkit/manager/state/store" ) // Slot is a list of the running tasks occupying a certain slot. Generally this @@ -11,36 +10,3 @@ import ( // "slot" is more generic than the Slot number for replicated services - a node // is also considered a slot for global services. type Slot []*api.Task - -// GetRunnableAndDeadSlots returns two maps of slots. The first contains slots -// that have at least one task with a desired state above NEW and lesser or -// equal to RUNNING. The second is for slots that only contain tasks with a -// desired state above RUNNING. -func GetRunnableAndDeadSlots(s *store.MemoryStore, serviceID string) (map[uint64]Slot, map[uint64]Slot, error) { - var ( - tasks []*api.Task - err error - ) - s.View(func(tx store.ReadTx) { - tasks, err = store.FindTasks(tx, store.ByServiceID(serviceID)) - }) - if err != nil { - return nil, nil, err - } - - runningSlots := make(map[uint64]Slot) - for _, t := range tasks { - if t.DesiredState <= api.TaskStateRunning { - runningSlots[t.Slot] = append(runningSlots[t.Slot], t) - } - } - - deadSlots := make(map[uint64]Slot) - for _, t := range tasks { - if _, exists := runningSlots[t.Slot]; !exists { - deadSlots[t.Slot] = append(deadSlots[t.Slot], t) - } - } - - return runningSlots, deadSlots, nil -} diff --git a/manager/orchestrator/update/updater.go b/manager/orchestrator/update/updater.go index 349bccabb9..fc2b6472a8 100644 --- a/manager/orchestrator/update/updater.go +++ b/manager/orchestrator/update/updater.go @@ -502,6 +502,9 @@ func (u *Updater) removeOldTasks(ctx context.Context, batch *store.Batch, remove return fmt.Errorf("task %s not found while trying to shut it down", original.ID) } if t.DesiredState > api.TaskStateRunning { + if !u.restarts.ShouldRestart(ctx, t, u.newService) { + return nil + } return fmt.Errorf("task %s was already shut down when reached by updater", original.ID) } t.DesiredState = api.TaskStateShutdown @@ -583,9 +586,8 @@ func (u *Updater) pauseUpdate(ctx context.Context, serviceID, message string) { func (u *Updater) rollbackUpdate(ctx context.Context, serviceID, message string) { log.G(ctx).Debugf("starting rollback of service %s", serviceID) - var service *api.Service err := u.store.Update(func(tx store.Tx) error { - service = store.GetService(tx, serviceID) + service := store.GetService(tx, serviceID) if service == nil { return nil } diff --git a/manager/orchestrator/update/updater_test.go b/manager/orchestrator/update/updater_test.go index 505973633e..e5e476243d 100644 --- a/manager/orchestrator/update/updater_test.go +++ b/manager/orchestrator/update/updater_test.go @@ -16,11 +16,24 @@ import ( ) func getRunnableSlotSlice(t *testing.T, s *store.MemoryStore, service *api.Service) []orchestrator.Slot { - runnable, _, err := orchestrator.GetRunnableAndDeadSlots(s, service.ID) + var ( + tasks []*api.Task + err error + ) + s.View(func(tx store.ReadTx) { + tasks, err = store.FindTasks(tx, store.ByServiceID(service.ID)) + }) require.NoError(t, err) + runningSlots := make(map[uint64]orchestrator.Slot) + for _, t := range tasks { + if t.DesiredState <= api.TaskStateRunning { + runningSlots[t.Slot] = append(runningSlots[t.Slot], t) + } + } + var runnableSlice []orchestrator.Slot - for _, slot := range runnable { + for _, slot := range runningSlots { runnableSlice = append(runnableSlice, slot) } return runnableSlice From a7a78fbe793190e47bb3ac78c9c5f089abd1ed60 Mon Sep 17 00:00:00 2001 From: Aaron Lehmann Date: Tue, 25 Jul 2017 14:38:41 -0700 Subject: [PATCH 3/8] Reconstruct task restart history based on historic tasks Signed-off-by: Aaron Lehmann --- manager/orchestrator/global/global.go | 8 ++ manager/orchestrator/replicated/slot.go | 8 ++ manager/orchestrator/restart/restart.go | 61 ++++++++------- manager/orchestrator/slot.go | 9 +++ manager/orchestrator/taskinit/init.go | 75 ++++++++++++++++++- .../orchestrator/taskreaper/task_reaper.go | 27 +++---- 6 files changed, 141 insertions(+), 47 deletions(-) diff --git a/manager/orchestrator/global/global.go b/manager/orchestrator/global/global.go index b850c8cf95..37a27ac867 100644 --- a/manager/orchestrator/global/global.go +++ b/manager/orchestrator/global/global.go @@ -557,6 +557,14 @@ func (g *Orchestrator) IsRelatedService(service *api.Service) bool { return orchestrator.IsGlobalService(service) } +// SlotTuple returns a slot tuple for the global service task. +func (g *Orchestrator) SlotTuple(t *api.Task) orchestrator.SlotTuple { + return orchestrator.SlotTuple{ + ServiceID: t.ServiceID, + NodeID: t.NodeID, + } +} + func isTaskCompleted(t *api.Task, restartPolicy api.RestartPolicy_RestartCondition) bool { if t == nil || t.DesiredState <= api.TaskStateRunning { return false diff --git a/manager/orchestrator/replicated/slot.go b/manager/orchestrator/replicated/slot.go index 15b11c0ff0..ab13e9f4aa 100644 --- a/manager/orchestrator/replicated/slot.go +++ b/manager/orchestrator/replicated/slot.go @@ -88,3 +88,11 @@ func (r *Orchestrator) updatableAndDeadSlots(ctx context.Context, service *api.S return updatableSlots, deadSlots, nil } + +// SlotTuple returns a slot tuple for the replicated service task. +func (r *Orchestrator) SlotTuple(t *api.Task) orchestrator.SlotTuple { + return orchestrator.SlotTuple{ + ServiceID: t.ServiceID, + Slot: t.Slot, + } +} diff --git a/manager/orchestrator/restart/restart.go b/manager/orchestrator/restart/restart.go index 98d8c9676c..1daa8fd197 100644 --- a/manager/orchestrator/restart/restart.go +++ b/manager/orchestrator/restart/restart.go @@ -49,19 +49,13 @@ type delayedStart struct { waiter bool } -type instanceTuple struct { - instance uint64 // unset for global tasks - serviceID string - nodeID string // unset for replicated tasks -} - // Supervisor initiates and manages restarts. It's responsible for // delaying restarts when applicable. type Supervisor struct { mu sync.Mutex store *store.MemoryStore delays map[string]*delayedStart - historyByService map[string]map[instanceTuple]*instanceRestartInfo + historyByService map[string]map[orchestrator.SlotTuple]*instanceRestartInfo TaskTimeout time.Duration } @@ -70,7 +64,7 @@ func NewSupervisor(store *store.MemoryStore) *Supervisor { return &Supervisor{ store: store, delays: make(map[string]*delayedStart), - historyByService: make(map[string]map[instanceTuple]*instanceRestartInfo), + historyByService: make(map[string]map[orchestrator.SlotTuple]*instanceRestartInfo), TaskTimeout: defaultOldTaskTimeout, } } @@ -185,7 +179,12 @@ func (r *Supervisor) Restart(ctx context.Context, tx store.Tx, cluster *api.Clus return err } - r.recordRestartHistory(restartTask) + tuple := orchestrator.SlotTuple{ + Slot: restartTask.Slot, + ServiceID: restartTask.ServiceID, + NodeID: restartTask.NodeID, + } + r.RecordRestartHistory(tuple, restartTask) r.DelayStart(ctx, tx, &t, restartTask.ID, restartDelay, waitStop) return nil @@ -206,15 +205,15 @@ func (r *Supervisor) ShouldRestart(ctx context.Context, t *api.Task, service *ap return true } - instanceTuple := instanceTuple{ - instance: t.Slot, - serviceID: t.ServiceID, + instanceTuple := orchestrator.SlotTuple{ + Slot: t.Slot, + ServiceID: t.ServiceID, } // Instance is not meaningful for "global" tasks, so they need to be // indexed by NodeID. if orchestrator.IsGlobalService(service) { - instanceTuple.nodeID = t.NodeID + instanceTuple.NodeID = t.NodeID } r.mu.Lock() @@ -301,49 +300,53 @@ func (r *Supervisor) IsTaskUpdatable(ctx context.Context, t *api.Task, service * !r.ShouldRestart(ctx, t, service)) } -func (r *Supervisor) recordRestartHistory(restartTask *api.Task) { - if restartTask.Spec.Restart == nil || restartTask.Spec.Restart.MaxAttempts == 0 { +// RecordRestartHistory updates the historyByService map to reflect the restart +// of restartedTask. +func (r *Supervisor) RecordRestartHistory(tuple orchestrator.SlotTuple, replacementTask *api.Task) { + if replacementTask.Spec.Restart == nil || replacementTask.Spec.Restart.MaxAttempts == 0 { // No limit on the number of restarts, so no need to record // history. return } - tuple := instanceTuple{ - instance: restartTask.Slot, - serviceID: restartTask.ServiceID, - nodeID: restartTask.NodeID, - } r.mu.Lock() defer r.mu.Unlock() - if r.historyByService[restartTask.ServiceID] == nil { - r.historyByService[restartTask.ServiceID] = make(map[instanceTuple]*instanceRestartInfo) + serviceID := replacementTask.ServiceID + if r.historyByService[serviceID] == nil { + r.historyByService[serviceID] = make(map[orchestrator.SlotTuple]*instanceRestartInfo) } - if r.historyByService[restartTask.ServiceID][tuple] == nil { - r.historyByService[restartTask.ServiceID][tuple] = &instanceRestartInfo{} + if r.historyByService[serviceID][tuple] == nil { + r.historyByService[serviceID][tuple] = &instanceRestartInfo{} } - restartInfo := r.historyByService[restartTask.ServiceID][tuple] + restartInfo := r.historyByService[serviceID][tuple] - if restartTask.SpecVersion != nil && *restartTask.SpecVersion != restartInfo.specVersion { + if replacementTask.SpecVersion != nil && *replacementTask.SpecVersion != restartInfo.specVersion { // This task has a different SpecVersion from the one we're // tracking. Most likely, the service was updated. Past failures // shouldn't count against the new service definition, so clear // the history for this instance. *restartInfo = instanceRestartInfo{ - specVersion: *restartTask.SpecVersion, + specVersion: *replacementTask.SpecVersion, } } restartInfo.totalRestarts++ - if restartTask.Spec.Restart.Window != nil && (restartTask.Spec.Restart.Window.Seconds != 0 || restartTask.Spec.Restart.Window.Nanos != 0) { + if replacementTask.Spec.Restart.Window != nil && (replacementTask.Spec.Restart.Window.Seconds != 0 || replacementTask.Spec.Restart.Window.Nanos != 0) { if restartInfo.restartedInstances == nil { restartInfo.restartedInstances = list.New() } + // it's okay to call TimestampFromProto with a nil argument + timestamp, err := gogotypes.TimestampFromProto(replacementTask.Meta.CreatedAt) + if replacementTask.Meta.CreatedAt == nil || err != nil { + timestamp = time.Now() + } + restartedInstance := restartedInstance{ - timestamp: time.Now(), + timestamp: timestamp, } restartInfo.restartedInstances.PushBack(restartedInstance) diff --git a/manager/orchestrator/slot.go b/manager/orchestrator/slot.go index 732933f1a5..7839a7509c 100644 --- a/manager/orchestrator/slot.go +++ b/manager/orchestrator/slot.go @@ -10,3 +10,12 @@ import ( // "slot" is more generic than the Slot number for replicated services - a node // is also considered a slot for global services. type Slot []*api.Task + +// SlotTuple identifies a unique slot, in the broad sense described above. It's +// a combination of either a service ID and a slot number (replicated services), +// or a service ID and a node ID (global services). +type SlotTuple struct { + Slot uint64 // unset for global service tasks + ServiceID string + NodeID string // unset for replicated service tasks +} diff --git a/manager/orchestrator/taskinit/init.go b/manager/orchestrator/taskinit/init.go index 33558a43c9..b893428d51 100644 --- a/manager/orchestrator/taskinit/init.go +++ b/manager/orchestrator/taskinit/init.go @@ -1,11 +1,13 @@ package taskinit import ( + "sort" "time" "github.com/docker/swarmkit/api" "github.com/docker/swarmkit/api/defaults" "github.com/docker/swarmkit/log" + "github.com/docker/swarmkit/manager/orchestrator" "github.com/docker/swarmkit/manager/orchestrator/restart" "github.com/docker/swarmkit/manager/state/store" gogotypes "github.com/gogo/protobuf/types" @@ -16,11 +18,13 @@ import ( type InitHandler interface { IsRelatedService(service *api.Service) bool FixTask(ctx context.Context, batch *store.Batch, t *api.Task) + SlotTuple(t *api.Task) orchestrator.SlotTuple } // CheckTasks fixes tasks in the store before orchestrator runs. The previous leader might // not have finished processing their updates and left them in an inconsistent state. func CheckTasks(ctx context.Context, s *store.MemoryStore, readTx store.ReadTx, initHandler InitHandler, startSupervisor *restart.Supervisor) error { + instances := make(map[orchestrator.SlotTuple][]*api.Task) err := s.Batch(func(batch *store.Batch) error { tasks, err := store.FindTasks(readTx, store.All) if err != nil { @@ -47,6 +51,9 @@ func CheckTasks(ctx context.Context, s *store.MemoryStore, readTx store.ReadTx, continue } + tuple := initHandler.SlotTuple(t) + instances[tuple] = append(instances[tuple], t) + // handle task updates from agent which should have been triggered by task update events initHandler.FixTask(ctx, batch, t) @@ -65,7 +72,12 @@ func CheckTasks(ctx context.Context, s *store.MemoryStore, readTx store.ReadTx, } } if restartDelay != 0 { - timestamp, err := gogotypes.TimestampFromProto(t.Status.Timestamp) + var timestamp time.Time + if t.Status.AppliedAt != nil { + timestamp, err = gogotypes.TimestampFromProto(t.Status.AppliedAt) + } else { + timestamp, err = gogotypes.TimestampFromProto(t.Status.Timestamp) + } if err == nil { restartTime := timestamp.Add(restartDelay) calculatedRestartDelay := restartTime.Sub(time.Now()) @@ -99,5 +111,64 @@ func CheckTasks(ctx context.Context, s *store.MemoryStore, readTx store.ReadTx, } return nil }) - return err + if err != nil { + return err + } + + for tuple, instance := range instances { + // Find the most current spec version. That's the only one + // we care about for the purpose of reconstructing restart + // history. + maxVersion := uint64(0) + for _, t := range instance { + if t.SpecVersion != nil && t.SpecVersion.Index > maxVersion { + maxVersion = t.SpecVersion.Index + } + } + + // Create a new slice with just the current spec version tasks. + var upToDate []*api.Task + for _, t := range instance { + if t.SpecVersion != nil && t.SpecVersion.Index == maxVersion { + upToDate = append(upToDate, t) + } + } + + // Sort by creation timestamp + sort.Sort(tasksByCreationTimestamp(upToDate)) + + // All up-to-date tasks in this instance except the first one + // should be considered restarted. + if len(upToDate) < 2 { + continue + } + for _, t := range upToDate[1:] { + startSupervisor.RecordRestartHistory(tuple, t) + } + } + return nil +} + +type tasksByCreationTimestamp []*api.Task + +func (t tasksByCreationTimestamp) Len() int { + return len(t) +} +func (t tasksByCreationTimestamp) Swap(i, j int) { + t[i], t[j] = t[j], t[i] +} +func (t tasksByCreationTimestamp) Less(i, j int) bool { + if t[i].Meta.CreatedAt == nil { + return true + } + if t[j].Meta.CreatedAt == nil { + return false + } + if t[i].Meta.CreatedAt.Seconds < t[j].Meta.CreatedAt.Seconds { + return true + } + if t[i].Meta.CreatedAt.Seconds > t[j].Meta.CreatedAt.Seconds { + return false + } + return t[i].Meta.CreatedAt.Nanos < t[j].Meta.CreatedAt.Nanos } diff --git a/manager/orchestrator/taskreaper/task_reaper.go b/manager/orchestrator/taskreaper/task_reaper.go index edb771c699..35ac714b81 100644 --- a/manager/orchestrator/taskreaper/task_reaper.go +++ b/manager/orchestrator/taskreaper/task_reaper.go @@ -6,6 +6,7 @@ import ( "github.com/docker/swarmkit/api" "github.com/docker/swarmkit/log" + "github.com/docker/swarmkit/manager/orchestrator" "github.com/docker/swarmkit/manager/state" "github.com/docker/swarmkit/manager/state/store" "golang.org/x/net/context" @@ -18,19 +19,13 @@ const ( reaperBatchingInterval = 250 * time.Millisecond ) -type instanceTuple struct { - instance uint64 // unset for global tasks - serviceID string - nodeID string // unset for replicated tasks -} - // A TaskReaper deletes old tasks when more than TaskHistoryRetentionLimit tasks // exist for the same service/instance or service/nodeid combination. type TaskReaper struct { store *store.MemoryStore // taskHistory is the number of tasks to keep taskHistory int64 - dirty map[instanceTuple]struct{} + dirty map[orchestrator.SlotTuple]struct{} orphaned []string stopChan chan struct{} doneChan chan struct{} @@ -40,7 +35,7 @@ type TaskReaper struct { func New(store *store.MemoryStore) *TaskReaper { return &TaskReaper{ store: store, - dirty: make(map[instanceTuple]struct{}), + dirty: make(map[orchestrator.SlotTuple]struct{}), stopChan: make(chan struct{}), doneChan: make(chan struct{}), } @@ -93,10 +88,10 @@ func (tr *TaskReaper) Run(ctx context.Context) { switch v := event.(type) { case api.EventCreateTask: t := v.Task - tr.dirty[instanceTuple{ - instance: t.Slot, - serviceID: t.ServiceID, - nodeID: t.NodeID, + tr.dirty[orchestrator.SlotTuple{ + Slot: t.Slot, + ServiceID: t.ServiceID, + NodeID: t.NodeID, }] = struct{}{} case api.EventUpdateTask: t := v.Task @@ -138,7 +133,7 @@ func (tr *TaskReaper) tick() { } tr.store.View(func(tx store.ReadTx) { for dirty := range tr.dirty { - service := store.GetService(tx, dirty.serviceID) + service := store.GetService(tx, dirty.ServiceID) if service == nil { continue } @@ -154,19 +149,19 @@ func (tr *TaskReaper) tick() { switch service.Spec.GetMode().(type) { case *api.ServiceSpec_Replicated: var err error - historicTasks, err = store.FindTasks(tx, store.BySlot(dirty.serviceID, dirty.instance)) + historicTasks, err = store.FindTasks(tx, store.BySlot(dirty.ServiceID, dirty.Slot)) if err != nil { continue } case *api.ServiceSpec_Global: - tasksByNode, err := store.FindTasks(tx, store.ByNodeID(dirty.nodeID)) + tasksByNode, err := store.FindTasks(tx, store.ByNodeID(dirty.NodeID)) if err != nil { continue } for _, t := range tasksByNode { - if t.ServiceID == dirty.serviceID { + if t.ServiceID == dirty.ServiceID { historicTasks = append(historicTasks, t) } } From 4f6cd4d9cd3500dbcba8b7b8f7becf83016555ef Mon Sep 17 00:00:00 2001 From: Aaron Lehmann Date: Tue, 25 Jul 2017 14:52:39 -0700 Subject: [PATCH 4/8] Change the task reaper to always retained at least MaxAttempts+1 tasks Signed-off-by: Aaron Lehmann --- manager/orchestrator/taskreaper/task_reaper.go | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/manager/orchestrator/taskreaper/task_reaper.go b/manager/orchestrator/taskreaper/task_reaper.go index 35ac714b81..fa9af1b605 100644 --- a/manager/orchestrator/taskreaper/task_reaper.go +++ b/manager/orchestrator/taskreaper/task_reaper.go @@ -140,6 +140,22 @@ func (tr *TaskReaper) tick() { taskHistory := tr.taskHistory + // If MaxAttempts is set, keep at least one more than + // that number of tasks. This is necessary reconstruct + // restart history when the orchestrator starts up. + // TODO(aaronl): Consider hiding tasks beyond the normal + // retention limit in the UI. + // TODO(aaronl): There are some ways to cut down the + // number of retained tasks at the cost of more + // complexity: + // - Don't force retention of tasks with an older spec + // version. + // - Don't force retention of tasks outside of the + // time window configured for restart lookback. + if service.Spec.Task.Restart != nil && service.Spec.Task.Restart.MaxAttempts > 0 { + taskHistory = int64(service.Spec.Task.Restart.MaxAttempts) + 1 + } + if taskHistory < 0 { continue } @@ -173,6 +189,8 @@ func (tr *TaskReaper) tick() { // TODO(aaronl): This could filter for non-running tasks and use quickselect // instead of sorting the whole slice. + // TODO(aaronl): This sort should really use lamport time instead of wall + // clock time. We should store a Version in the Status field. sort.Sort(tasksByTimestamp(historicTasks)) runningTasks := 0 From 0976bbd23604be170b84a0a371cff33d9462261d Mon Sep 17 00:00:00 2001 From: Aaron Lehmann Date: Wed, 26 Jul 2017 17:04:54 -0700 Subject: [PATCH 5/8] updater: Simpler and more reliable skipping of already-shutdown tasks Signed-off-by: Aaron Lehmann --- manager/orchestrator/update/updater.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/manager/orchestrator/update/updater.go b/manager/orchestrator/update/updater.go index fc2b6472a8..0bda96e9a6 100644 --- a/manager/orchestrator/update/updater.go +++ b/manager/orchestrator/update/updater.go @@ -496,15 +496,15 @@ func (u *Updater) removeOldTasks(ctx context.Context, batch *store.Batch, remove removedTask *api.Task ) for _, original := range removeTasks { + if original.DesiredState > api.TaskStateRunning { + continue + } err := batch.Update(func(tx store.Tx) error { t := store.GetTask(tx, original.ID) if t == nil { return fmt.Errorf("task %s not found while trying to shut it down", original.ID) } if t.DesiredState > api.TaskStateRunning { - if !u.restarts.ShouldRestart(ctx, t, u.newService) { - return nil - } return fmt.Errorf("task %s was already shut down when reached by updater", original.ID) } t.DesiredState = api.TaskStateShutdown From 112794b22be3761401266364e3cd6283e65fc547 Mon Sep 17 00:00:00 2001 From: Aaron Lehmann Date: Wed, 26 Jul 2017 17:09:58 -0700 Subject: [PATCH 6/8] restart: Clean up for loops Signed-off-by: Aaron Lehmann --- manager/orchestrator/restart/restart.go | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/manager/orchestrator/restart/restart.go b/manager/orchestrator/restart/restart.go index 1daa8fd197..db3024c0bd 100644 --- a/manager/orchestrator/restart/restart.go +++ b/manager/orchestrator/restart/restart.go @@ -259,25 +259,29 @@ func (r *Supervisor) ShouldRestart(ctx context.Context, t *api.Task, service *ap numRestarts := uint64(restartInfo.restartedInstances.Len()) + // Disregard any restarts that happened before the lookback window, + // and remove them from the linked list since they will no longer + // be relevant to figuring out if tasks should be restarted going + // forward. var next *list.Element for e := restartInfo.restartedInstances.Front(); e != nil; e = next { next = e.Next() if e.Value.(restartedInstance).timestamp.After(lookback) { - for e2 := restartInfo.restartedInstances.Back(); e2 != nil; e2 = e2.Prev() { - // Ignore restarts that didn't happen before the - // task we're looking at. - if e.Value.(restartedInstance).timestamp.Before(timestamp) { - break - } - numRestarts-- - } break } restartInfo.restartedInstances.Remove(e) numRestarts-- } + // Ignore restarts that didn't happen before the task we're looking at. + for e2 := restartInfo.restartedInstances.Back(); e2 != nil; e2 = e2.Prev() { + if e.Value.(restartedInstance).timestamp.Before(timestamp) { + break + } + numRestarts-- + } + if restartInfo.restartedInstances.Len() == 0 { restartInfo.restartedInstances = nil } From d489fa549b860d65b656800ae613e25de00d8bb2 Mon Sep 17 00:00:00 2001 From: Aaron Lehmann Date: Wed, 26 Jul 2017 18:03:26 -0700 Subject: [PATCH 7/8] orchestrator: Don't get confused by historic tasks when making reconciliation decisions Only look at the most recent task to see if it should be restarted. Signed-off-by: Aaron Lehmann --- manager/orchestrator/global/global.go | 45 +++++++++---- manager/orchestrator/replicated/slot.go | 14 +++-- manager/orchestrator/restart/restart.go | 63 ++++++++++++++----- manager/orchestrator/task.go | 41 ++++++++++++ .../orchestrator/taskreaper/task_reaper.go | 26 +------- 5 files changed, 129 insertions(+), 60 deletions(-) diff --git a/manager/orchestrator/global/global.go b/manager/orchestrator/global/global.go index 37a27ac867..a1d2873e80 100644 --- a/manager/orchestrator/global/global.go +++ b/manager/orchestrator/global/global.go @@ -255,6 +255,11 @@ func (g *Orchestrator) reconcileServices(ctx context.Context, serviceIDs []strin g.store.View(func(tx store.ReadTx) { for _, serviceID := range serviceIDs { + service := g.globalServices[serviceID].Service + if service == nil { + continue + } + tasks, err := store.FindTasks(tx, store.ByServiceID(serviceID)) if err != nil { log.G(ctx).WithError(err).Errorf("global orchestrator: reconcileServices failed finding tasks for service %s", serviceID) @@ -265,18 +270,22 @@ func (g *Orchestrator) reconcileServices(ctx context.Context, serviceIDs []strin nodeTasks[serviceID] = make(map[string][]*api.Task) for _, t := range tasks { - service := g.globalServices[serviceID].Service - if service == nil { - continue - } - // Collect all runnable instances of this service, - // and instances that were not be restarted due - // to restart policy but may be updated if the - // service spec changed. - if g.restarts.IsTaskUpdatable(ctx, t, service) { - nodeTasks[serviceID][t.NodeID] = append(nodeTasks[serviceID][t.NodeID], t) + nodeTasks[serviceID][t.NodeID] = append(nodeTasks[serviceID][t.NodeID], t) + } + + // Keep all runnable instances of this service, + // and instances that were not be restarted due + // to restart policy but may be updated if the + // service spec changed. + for nodeID, slot := range nodeTasks[serviceID] { + updatable := g.restarts.UpdatableTasksInSlot(ctx, slot, g.globalServices[serviceID].Service) + if len(updatable) != 0 { + nodeTasks[serviceID][nodeID] = updatable + } else { + delete(nodeTasks[serviceID], nodeID) } } + } }) @@ -407,8 +416,20 @@ func (g *Orchestrator) reconcileOneNode(ctx context.Context, node *api.Node) { if t.ServiceID != serviceID { continue } - if g.restarts.IsTaskUpdatable(ctx, t, service.Service) { - tasks[serviceID] = append(tasks[serviceID], t) + tasks[serviceID] = append(tasks[serviceID], t) + } + + // Keep all runnable instances of this service, + // and instances that were not be restarted due + // to restart policy but may be updated if the + // service spec changed. + for serviceID, slot := range tasks { + updatable := g.restarts.UpdatableTasksInSlot(ctx, slot, service.Service) + + if len(updatable) != 0 { + tasks[serviceID] = updatable + } else { + delete(tasks, serviceID) } } } diff --git a/manager/orchestrator/replicated/slot.go b/manager/orchestrator/replicated/slot.go index ab13e9f4aa..bdc25d9d76 100644 --- a/manager/orchestrator/replicated/slot.go +++ b/manager/orchestrator/replicated/slot.go @@ -74,15 +74,17 @@ func (r *Orchestrator) updatableAndDeadSlots(ctx context.Context, service *api.S updatableSlots := make(map[uint64]orchestrator.Slot) for _, t := range tasks { - if r.restarts.IsTaskUpdatable(ctx, t, service) { - updatableSlots[t.Slot] = append(updatableSlots[t.Slot], t) - } + updatableSlots[t.Slot] = append(updatableSlots[t.Slot], t) } deadSlots := make(map[uint64]orchestrator.Slot) - for _, t := range tasks { - if _, exists := updatableSlots[t.Slot]; !exists { - deadSlots[t.Slot] = append(deadSlots[t.Slot], t) + for slotID, slot := range updatableSlots { + updatable := r.restarts.UpdatableTasksInSlot(ctx, slot, service) + if len(updatable) != 0 { + updatableSlots[slotID] = updatable + } else { + delete(updatableSlots, slotID) + deadSlots[slotID] = slot } } diff --git a/manager/orchestrator/restart/restart.go b/manager/orchestrator/restart/restart.go index db3024c0bd..0c739bef0d 100644 --- a/manager/orchestrator/restart/restart.go +++ b/manager/orchestrator/restart/restart.go @@ -132,7 +132,7 @@ func (r *Supervisor) Restart(ctx context.Context, tx store.Tx, cluster *api.Clus return err } - if !r.ShouldRestart(ctx, &t, service) { + if !r.shouldRestart(ctx, &t, service) { return nil } @@ -190,9 +190,9 @@ func (r *Supervisor) Restart(ctx context.Context, tx store.Tx, cluster *api.Clus return nil } -// ShouldRestart returns true if a task should be restarted according to the +// shouldRestart returns true if a task should be restarted according to the // restart policy. -func (r *Supervisor) ShouldRestart(ctx context.Context, t *api.Task, service *api.Service) bool { +func (r *Supervisor) shouldRestart(ctx context.Context, t *api.Task, service *api.Service) bool { // TODO(aluzzardi): This function should not depend on `service`. condition := orchestrator.RestartCondition(t) @@ -276,7 +276,7 @@ func (r *Supervisor) ShouldRestart(ctx context.Context, t *api.Task, service *ap // Ignore restarts that didn't happen before the task we're looking at. for e2 := restartInfo.restartedInstances.Back(); e2 != nil; e2 = e2.Prev() { - if e.Value.(restartedInstance).timestamp.Before(timestamp) { + if e2.Value.(restartedInstance).timestamp.Before(timestamp) { break } numRestarts-- @@ -289,19 +289,48 @@ func (r *Supervisor) ShouldRestart(ctx context.Context, t *api.Task, service *ap return numRestarts < t.Spec.Restart.MaxAttempts } -// IsTaskUpdatable determines whether the task should be passed to the updater -// or not. An updatable task is either a task that with desired state <= -// RUNNING, or a task which has stopped running and should not be restarted. The -// latter case is for making sure that tasks that shouldn't normally be -// restarted will still be handled by rolling updates when they become outdated. -// There is a special case for rollbacks to make sure that a rollback always -// takes the service to a converged state, instead of ignoring tasks with the -// original spec that stopped running and shouldn't be restarted according to -// the restart policy. -func (r *Supervisor) IsTaskUpdatable(ctx context.Context, t *api.Task, service *api.Service) bool { - return t.DesiredState <= api.TaskStateRunning || - ((service.UpdateStatus == nil || service.UpdateStatus.State != api.UpdateStatus_ROLLBACK_STARTED) && - !r.ShouldRestart(ctx, t, service)) +// UpdatableTasksInSlot returns the set of tasks that should be passed to the +// updater from this slot, or an empty slice if none should be. An updatable +// slot has either at least one task that with desired state <= RUNNING, or its +// most recent task has stopped running and should not be restarted. The latter +// case is for making sure that tasks that shouldn't normally be restarted will +// still be handled by rolling updates when they become outdated. There is a +// special case for rollbacks to make sure that a rollback always takes the +// service to a converged state, instead of ignoring tasks with the original +// spec that stopped running and shouldn't be restarted according to the +// restart policy. +func (r *Supervisor) UpdatableTasksInSlot(ctx context.Context, slot orchestrator.Slot, service *api.Service) orchestrator.Slot { + if len(slot) < 1 { + return nil + } + + var updatable orchestrator.Slot + for _, t := range slot { + if t.DesiredState <= api.TaskStateRunning { + updatable = append(updatable, t) + } + } + if len(updatable) > 0 { + return updatable + } + + if service.UpdateStatus != nil && service.UpdateStatus.State == api.UpdateStatus_ROLLBACK_STARTED { + return nil + } + + // Find most recent task + byTimestamp := orchestrator.TasksByTimestamp(slot) + newestIndex := 0 + for i := 1; i != len(slot); i++ { + if byTimestamp.Less(newestIndex, i) { + newestIndex = i + } + } + + if !r.shouldRestart(ctx, slot[newestIndex], service) { + return orchestrator.Slot{slot[newestIndex]} + } + return nil } // RecordRestartHistory updates the historyByService map to reflect the restart diff --git a/manager/orchestrator/task.go b/manager/orchestrator/task.go index 32a22d5f5a..8ae3a4cc26 100644 --- a/manager/orchestrator/task.go +++ b/manager/orchestrator/task.go @@ -77,3 +77,44 @@ func InvalidNode(n *api.Node) bool { n.Status.State == api.NodeStatus_DOWN || n.Spec.Availability == api.NodeAvailabilityDrain } + +// TasksByTimestamp sorts tasks by applied timestamp if available, otherwise +// status timestamp. +type TasksByTimestamp []*api.Task + +// Len implements the Len method for sorting. +func (t TasksByTimestamp) Len() int { + return len(t) +} + +// Swap implements the Swap method for sorting. +func (t TasksByTimestamp) Swap(i, j int) { + t[i], t[j] = t[j], t[i] +} + +// Less implements the Less method for sorting. +func (t TasksByTimestamp) Less(i, j int) bool { + iTimestamp := t[i].Status.Timestamp + if t[i].Status.AppliedAt != nil { + iTimestamp = t[i].Status.AppliedAt + } + + jTimestamp := t[j].Status.Timestamp + if t[j].Status.AppliedAt != nil { + iTimestamp = t[j].Status.AppliedAt + } + + if iTimestamp == nil { + return true + } + if jTimestamp == nil { + return false + } + if iTimestamp.Seconds < jTimestamp.Seconds { + return true + } + if iTimestamp.Seconds > jTimestamp.Seconds { + return false + } + return iTimestamp.Nanos < jTimestamp.Nanos +} diff --git a/manager/orchestrator/taskreaper/task_reaper.go b/manager/orchestrator/taskreaper/task_reaper.go index fa9af1b605..577319c8e8 100644 --- a/manager/orchestrator/taskreaper/task_reaper.go +++ b/manager/orchestrator/taskreaper/task_reaper.go @@ -191,7 +191,7 @@ func (tr *TaskReaper) tick() { // instead of sorting the whole slice. // TODO(aaronl): This sort should really use lamport time instead of wall // clock time. We should store a Version in the Status field. - sort.Sort(tasksByTimestamp(historicTasks)) + sort.Sort(orchestrator.TasksByTimestamp(historicTasks)) runningTasks := 0 for _, t := range historicTasks { @@ -232,27 +232,3 @@ func (tr *TaskReaper) Stop() { close(tr.stopChan) <-tr.doneChan } - -type tasksByTimestamp []*api.Task - -func (t tasksByTimestamp) Len() int { - return len(t) -} -func (t tasksByTimestamp) Swap(i, j int) { - t[i], t[j] = t[j], t[i] -} -func (t tasksByTimestamp) Less(i, j int) bool { - if t[i].Status.Timestamp == nil { - return true - } - if t[j].Status.Timestamp == nil { - return false - } - if t[i].Status.Timestamp.Seconds < t[j].Status.Timestamp.Seconds { - return true - } - if t[i].Status.Timestamp.Seconds > t[j].Status.Timestamp.Seconds { - return false - } - return t[i].Status.Timestamp.Nanos < t[j].Status.Timestamp.Nanos -} From 1b7b99d00f09d572abd61d82951c5899aac1211a Mon Sep 17 00:00:00 2001 From: Aaron Lehmann Date: Wed, 26 Jul 2017 18:26:28 -0700 Subject: [PATCH 8/8] restart: Instance -> Slot Signed-off-by: Aaron Lehmann --- manager/orchestrator/restart/restart.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/manager/orchestrator/restart/restart.go b/manager/orchestrator/restart/restart.go index 0c739bef0d..6af44b734c 100644 --- a/manager/orchestrator/restart/restart.go +++ b/manager/orchestrator/restart/restart.go @@ -210,7 +210,7 @@ func (r *Supervisor) shouldRestart(ctx context.Context, t *api.Task, service *ap ServiceID: t.ServiceID, } - // Instance is not meaningful for "global" tasks, so they need to be + // Slot is not meaningful for "global" tasks, so they need to be // indexed by NodeID. if orchestrator.IsGlobalService(service) { instanceTuple.NodeID = t.NodeID