From 1ecf4c6bfe5ecdb2f0847da5a27cdced0ea95770 Mon Sep 17 00:00:00 2001 From: Jean Rouge Date: Sun, 28 Oct 2018 16:59:54 -0700 Subject: [PATCH] Adding a new `Deallocator` component This patch adds a new manager/deallocator package. The deallocator's job is to take over deletion of services and service-level resources (as of right now, only networks, but e.g. volumes would also fall in the same category. The deallocator relies on the reaper correctly deleting tasks for services that are shutting down, and only when no tasks are left within a service does it then proceed to delete that service and its resources. This patch does not yet actually put that new component to use; that will be done in a future, separate patch, for the sake of easier reviews. Special care has been taken to separate the deallocator itself from each of the service-level resource implementation. Includes unit tests. Signed-off-by: Jean Rouge --- api/api.pb.txt | 14 + api/objects.pb.go | 292 ++++++++----- api/objects.proto | 24 ++ manager/deallocator/deallocator.go | 291 +++++++++++++ manager/deallocator/deallocator_test.go | 406 ++++++++++++++++++ .../orchestrator/taskreaper/task_reaper.go | 12 +- manager/state/store/networks.go | 2 +- manager/state/store/services.go | 2 +- 8 files changed, 934 insertions(+), 109 deletions(-) create mode 100644 manager/deallocator/deallocator.go create mode 100644 manager/deallocator/deallocator_test.go diff --git a/api/api.pb.txt b/api/api.pb.txt index e88f39315b..826eedaf90 100755 --- a/api/api.pb.txt +++ b/api/api.pb.txt @@ -5839,6 +5839,13 @@ file { type_name: ".docker.swarmkit.v1.UpdateStatus" json_name: "updateStatus" } + field { + name: "pending_delete" + number: 7 + label: LABEL_OPTIONAL + type: TYPE_BOOL + json_name: "pendingDelete" + } options { 70001 { 1 { @@ -6152,6 +6159,13 @@ file { } json_name: "ipam" } + field { + name: "pending_delete" + number: 6 + label: LABEL_OPTIONAL + type: TYPE_BOOL + json_name: "pendingDelete" + } options { 70001 { 1 { diff --git a/api/objects.pb.go b/api/objects.pb.go index 4200eda49b..d700c3f92c 100644 --- a/api/objects.pb.go +++ b/api/objects.pb.go @@ -102,6 +102,15 @@ type Service struct { // UpdateStatus contains the status of an update, if one is in // progress. UpdateStatus *UpdateStatus `protobuf:"bytes,5,opt,name=update_status,json=updateStatus" json:"update_status,omitempty"` + // PendingDelete indicates that this service's deletion has been requested. + // Services, as well as all service-level resources, can only be deleted + // after all of the service's containers have properly shut down. + // When a user requests a deletion, we just flip this flag + // the deallocator will take it from there - it will start monitoring + // this service's tasks, and proceed to delete the service itself (and + // potentially its associated resources also marked for deletion) when + // all of its tasks are gone + PendingDelete bool `protobuf:"varint,7,opt,name=pending_delete,json=pendingDelete,proto3" json:"pending_delete,omitempty"` } func (m *Service) Reset() { *m = Service{} } @@ -239,6 +248,19 @@ type Network struct { // Runtime state of IPAM options. This may not reflect the // ipam options from NetworkSpec. IPAM *IPAMOptions `protobuf:"bytes,5,opt,name=ipam" json:"ipam,omitempty"` + // PendingDelete indicates that this network's deletion has been requested. + // Services, as well as all service-level resources, can only be deleted + // after all the service's containers have properly shut down + // when a user requests a deletion, we just flip this flag + // the deallocator will take it from there + // PendingDelete indicates that this network's deletion has been requested. + // Services, as well as all service-level resources, can only be deleted + // after all of the service's containers have properly shut down. + // When a user requests a deletion of this network, we just flip this flag + // the deallocator will take it from there - it will start monitoring + // the services that still use this service, and proceed to delete + // this network when all of these services are gone + PendingDelete bool `protobuf:"varint,6,opt,name=pending_delete,json=pendingDelete,proto3" json:"pending_delete,omitempty"` } func (m *Network) Reset() { *m = Network{} } @@ -960,6 +982,16 @@ func (m *Service) MarshalTo(dAtA []byte) (int, error) { } i += n15 } + if m.PendingDelete { + dAtA[i] = 0x38 + i++ + if m.PendingDelete { + dAtA[i] = 1 + } else { + dAtA[i] = 0 + } + i++ + } if m.SpecVersion != nil { dAtA[i] = 0x52 i++ @@ -1337,6 +1369,16 @@ func (m *Network) MarshalTo(dAtA []byte) (int, error) { } i += n31 } + if m.PendingDelete { + dAtA[i] = 0x30 + i++ + if m.PendingDelete { + dAtA[i] = 1 + } else { + dAtA[i] = 0 + } + i++ + } return i, nil } @@ -1755,6 +1797,9 @@ func (m *Service) Size() (n int) { l = m.PreviousSpec.Size() n += 1 + l + sovObjects(uint64(l)) } + if m.PendingDelete { + n += 2 + } if m.SpecVersion != nil { l = m.SpecVersion.Size() n += 1 + l + sovObjects(uint64(l)) @@ -1909,6 +1954,9 @@ func (m *Network) Size() (n int) { l = m.IPAM.Size() n += 1 + l + sovObjects(uint64(l)) } + if m.PendingDelete { + n += 2 + } return n } @@ -4731,6 +4779,7 @@ func (this *Service) String() string { `Endpoint:` + strings.Replace(fmt.Sprintf("%v", this.Endpoint), "Endpoint", "Endpoint", 1) + `,`, `UpdateStatus:` + strings.Replace(fmt.Sprintf("%v", this.UpdateStatus), "UpdateStatus", "UpdateStatus", 1) + `,`, `PreviousSpec:` + strings.Replace(fmt.Sprintf("%v", this.PreviousSpec), "ServiceSpec", "ServiceSpec", 1) + `,`, + `PendingDelete:` + fmt.Sprintf("%v", this.PendingDelete) + `,`, `SpecVersion:` + strings.Replace(fmt.Sprintf("%v", this.SpecVersion), "Version", "Version", 1) + `,`, `PreviousSpecVersion:` + strings.Replace(fmt.Sprintf("%v", this.PreviousSpecVersion), "Version", "Version", 1) + `,`, `}`, @@ -4817,6 +4866,7 @@ func (this *Network) String() string { `Spec:` + strings.Replace(strings.Replace(this.Spec.String(), "NetworkSpec", "NetworkSpec", 1), `&`, ``, 1) + `,`, `DriverState:` + strings.Replace(fmt.Sprintf("%v", this.DriverState), "Driver", "Driver", 1) + `,`, `IPAM:` + strings.Replace(fmt.Sprintf("%v", this.IPAM), "IPAMOptions", "IPAMOptions", 1) + `,`, + `PendingDelete:` + fmt.Sprintf("%v", this.PendingDelete) + `,`, `}`, }, "") return s @@ -5622,6 +5672,26 @@ func (m *Service) Unmarshal(dAtA []byte) error { return err } iNdEx = postIndex + case 7: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field PendingDelete", wireType) + } + var v int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowObjects + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + m.PendingDelete = bool(v != 0) case 10: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field SpecVersion", wireType) @@ -6891,6 +6961,26 @@ func (m *Network) Unmarshal(dAtA []byte) error { return err } iNdEx = postIndex + case 6: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field PendingDelete", wireType) + } + var v int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowObjects + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + m.PendingDelete = bool(v != 0) default: iNdEx = preIndex skippy, err := skipObjects(dAtA[iNdEx:]) @@ -8128,104 +8218,106 @@ var ( func init() { proto.RegisterFile("github.com/docker/swarmkit/api/objects.proto", fileDescriptorObjects) } var fileDescriptorObjects = []byte{ - // 1581 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xcc, 0x58, 0x4b, 0x73, 0x1b, 0x4b, - 0x15, 0xce, 0x48, 0x63, 0x3d, 0x8e, 0x6c, 0x61, 0xfa, 0x1a, 0x33, 0x11, 0x46, 0x32, 0xba, 0x05, - 0x75, 0xeb, 0x56, 0x4a, 0xbe, 0x98, 0x0b, 0x38, 0x86, 0xcb, 0x8d, 0x64, 0x9b, 0x44, 0x15, 0x42, - 0x5c, 0xed, 0x90, 0xb0, 0x1b, 0x5a, 0x33, 0x6d, 0x65, 0xd0, 0x68, 0x7a, 0x6a, 0xba, 0xa5, 0x20, - 0x56, 0x59, 0x9b, 0x1f, 0xe0, 0x1d, 0x8b, 0xfc, 0x0b, 0x36, 0x2c, 0x58, 0x65, 0xc9, 0x8a, 0x62, - 0xe5, 0x22, 0xfa, 0x17, 0x54, 0xb1, 0xa0, 0xba, 0xa7, 0x47, 0x1a, 0x5b, 0xe3, 0x17, 0x95, 0x72, - 0xb1, 0x72, 0x3f, 0xbe, 0xef, 0xf4, 0x39, 0x67, 0xce, 0xcb, 0x82, 0x07, 0x7d, 0x4f, 0xbc, 0x1e, - 0xf5, 0x5a, 0x0e, 0x1b, 0x6e, 0xb9, 0xcc, 0x19, 0xd0, 0x68, 0x8b, 0xbf, 0x21, 0xd1, 0x70, 0xe0, - 0x89, 0x2d, 0x12, 0x7a, 0x5b, 0xac, 0xf7, 0x7b, 0xea, 0x08, 0xde, 0x0a, 0x23, 0x26, 0x18, 0x42, - 0x31, 0xa4, 0x95, 0x40, 0x5a, 0xe3, 0x1f, 0xd6, 0x3e, 0xbf, 0x46, 0x82, 0x98, 0x84, 0x54, 0xf3, - 0xaf, 0xc5, 0xf2, 0x90, 0x3a, 0x09, 0xb6, 0xd1, 0x67, 0xac, 0xef, 0xd3, 0x2d, 0xb5, 0xeb, 0x8d, - 0x8e, 0xb7, 0x84, 0x37, 0xa4, 0x5c, 0x90, 0x61, 0xa8, 0x01, 0x6b, 0x7d, 0xd6, 0x67, 0x6a, 0xb9, - 0x25, 0x57, 0xfa, 0xf4, 0xfe, 0x45, 0x1a, 0x09, 0x26, 0xfa, 0xea, 0xa7, 0x57, 0xbc, 0x3e, 0x83, - 0x87, 0xfe, 0xa8, 0xef, 0x05, 0xfa, 0x4f, 0x4c, 0x6c, 0xfe, 0xc5, 0x00, 0xf3, 0x19, 0x15, 0x04, - 0xfd, 0x0c, 0x8a, 0x63, 0x1a, 0x71, 0x8f, 0x05, 0x96, 0xb1, 0x69, 0x7c, 0x56, 0xd9, 0xfe, 0x4e, - 0x6b, 0xd1, 0x23, 0xad, 0x97, 0x31, 0xa4, 0x63, 0xbe, 0x3f, 0x6b, 0xdc, 0xc3, 0x09, 0x03, 0x3d, - 0x04, 0x70, 0x22, 0x4a, 0x04, 0x75, 0x6d, 0x22, 0xac, 0x9c, 0xe2, 0xd7, 0x5a, 0xb1, 0xba, 0xad, - 0xe4, 0xfd, 0xd6, 0x8b, 0xc4, 0x4a, 0x5c, 0xd6, 0xe8, 0xb6, 0x90, 0xd4, 0x51, 0xe8, 0x26, 0xd4, - 0xfc, 0xf5, 0x54, 0x8d, 0x6e, 0x8b, 0xe6, 0xdb, 0x25, 0x30, 0x7f, 0xcd, 0x5c, 0x8a, 0xd6, 0x21, - 0xe7, 0xb9, 0x4a, 0xed, 0x72, 0xa7, 0x30, 0x3d, 0x6b, 0xe4, 0xba, 0xfb, 0x38, 0xe7, 0xb9, 0x68, - 0x1b, 0xcc, 0x21, 0x15, 0x44, 0x2b, 0x64, 0x65, 0x19, 0x24, 0x6d, 0xd7, 0xd6, 0x28, 0x2c, 0xfa, - 0x09, 0x98, 0xf2, 0x53, 0x69, 0x4d, 0x36, 0xb2, 0x38, 0xf2, 0xcd, 0xa3, 0x90, 0x3a, 0x09, 0x4f, - 0xe2, 0xd1, 0x01, 0x54, 0x5c, 0xca, 0x9d, 0xc8, 0x0b, 0x85, 0xf4, 0xa1, 0xa9, 0xe8, 0x9f, 0x5e, - 0x46, 0xdf, 0x9f, 0x43, 0x71, 0x9a, 0x87, 0x7e, 0x0e, 0x05, 0x2e, 0x88, 0x18, 0x71, 0x6b, 0x49, - 0x49, 0xa8, 0x5f, 0xaa, 0x80, 0x42, 0x69, 0x15, 0x34, 0x07, 0x3d, 0x81, 0xea, 0x90, 0x04, 0xa4, - 0x4f, 0x23, 0x5b, 0x4b, 0x29, 0x28, 0x29, 0xdf, 0xcb, 0x34, 0x3d, 0x46, 0xc6, 0x82, 0xf0, 0xca, - 0x30, 0xbd, 0x45, 0x5d, 0x00, 0x22, 0x04, 0x71, 0x5e, 0x0f, 0x69, 0x20, 0xac, 0xa2, 0x92, 0xf2, - 0xfd, 0x4c, 0x5d, 0xa8, 0x78, 0xc3, 0xa2, 0x41, 0x7b, 0x06, 0xee, 0xe4, 0x2c, 0x03, 0xa7, 0xc8, - 0xe8, 0x31, 0x54, 0x1c, 0x1a, 0x09, 0xef, 0xd8, 0x73, 0x88, 0xa0, 0x56, 0x49, 0xc9, 0x6a, 0x64, - 0xc9, 0xda, 0x9b, 0xc3, 0xb4, 0x61, 0x69, 0x26, 0xfa, 0x02, 0xcc, 0x88, 0xf9, 0xd4, 0x2a, 0x6f, - 0x1a, 0x9f, 0x55, 0x2f, 0xff, 0x34, 0x98, 0xf9, 0x14, 0x2b, 0xa4, 0x7c, 0x7a, 0xae, 0x08, 0xb7, - 0x60, 0x33, 0x7f, 0x63, 0x33, 0x70, 0x9a, 0xb9, 0xbb, 0x7e, 0x72, 0xda, 0x44, 0xb0, 0x5a, 0x32, - 0x56, 0x0d, 0x15, 0x67, 0xc6, 0x17, 0xc6, 0x6f, 0x8d, 0xdf, 0x19, 0xcd, 0xff, 0xe4, 0xa1, 0x78, - 0x44, 0xa3, 0xb1, 0xe7, 0x7c, 0xdc, 0x28, 0x7c, 0x78, 0x2e, 0x0a, 0x33, 0x9d, 0xa5, 0x9f, 0x5d, - 0x08, 0xc4, 0x1d, 0x28, 0xd1, 0xc0, 0x0d, 0x99, 0x17, 0x08, 0x1d, 0x85, 0x99, 0x9e, 0x3a, 0xd0, - 0x18, 0x3c, 0x43, 0xa3, 0x03, 0x58, 0x89, 0x93, 0xcb, 0x3e, 0x17, 0x82, 0x9b, 0x59, 0xf4, 0xdf, - 0x28, 0xa0, 0x8e, 0x9d, 0xe5, 0x51, 0x6a, 0x87, 0xf6, 0x61, 0x25, 0x8c, 0xe8, 0xd8, 0x63, 0x23, - 0x6e, 0x2b, 0x23, 0x0a, 0x37, 0x32, 0x02, 0x2f, 0x27, 0x2c, 0xb9, 0x43, 0xbf, 0x80, 0x65, 0x49, - 0xb6, 0x93, 0xa2, 0x04, 0xd7, 0x16, 0x25, 0x5c, 0x91, 0x04, 0xbd, 0x41, 0xcf, 0xe1, 0x5b, 0xe7, - 0xb4, 0x98, 0x09, 0xaa, 0x5c, 0x2f, 0xe8, 0x93, 0xb4, 0x26, 0xfa, 0x70, 0x17, 0x9d, 0x9c, 0x36, - 0xab, 0xb0, 0x9c, 0x0e, 0x81, 0xe6, 0x9f, 0x73, 0x50, 0x4a, 0x1c, 0x89, 0xbe, 0xd4, 0xdf, 0xcc, - 0xb8, 0xdc, 0x6b, 0x09, 0x56, 0xd9, 0x1b, 0x7f, 0xae, 0x2f, 0x61, 0x29, 0x64, 0x91, 0xe0, 0x56, - 0x4e, 0x05, 0x67, 0x66, 0xbe, 0x1f, 0xb2, 0x48, 0xec, 0xb1, 0xe0, 0xd8, 0xeb, 0xe3, 0x18, 0x8c, - 0x5e, 0x41, 0x65, 0xec, 0x45, 0x62, 0x44, 0x7c, 0xdb, 0x0b, 0xb9, 0x95, 0x57, 0xdc, 0x1f, 0x5c, - 0xf5, 0x64, 0xeb, 0x65, 0x8c, 0xef, 0x1e, 0x76, 0xaa, 0xd3, 0xb3, 0x06, 0xcc, 0xb6, 0x1c, 0x83, - 0x16, 0xd5, 0x0d, 0x79, 0xed, 0x19, 0x94, 0x67, 0x37, 0xe8, 0x01, 0x40, 0x10, 0xe7, 0x85, 0x3d, - 0x8b, 0xec, 0x95, 0xe9, 0x59, 0xa3, 0xac, 0xb3, 0xa5, 0xbb, 0x8f, 0xcb, 0x1a, 0xd0, 0x75, 0x11, - 0x02, 0x93, 0xb8, 0x6e, 0xa4, 0xe2, 0xbc, 0x8c, 0xd5, 0xba, 0xf9, 0xa7, 0x22, 0x98, 0x2f, 0x08, - 0x1f, 0xdc, 0x75, 0x89, 0x96, 0x6f, 0x2e, 0x64, 0xc6, 0x03, 0x00, 0x1e, 0xc7, 0x9b, 0x34, 0xc7, - 0x9c, 0x9b, 0xa3, 0xa3, 0x50, 0x9a, 0xa3, 0x01, 0xb1, 0x39, 0xdc, 0x67, 0x42, 0x25, 0x81, 0x89, - 0xd5, 0x1a, 0x7d, 0x0a, 0xc5, 0x80, 0xb9, 0x8a, 0x5e, 0x50, 0x74, 0x98, 0x9e, 0x35, 0x0a, 0xb2, - 0xe8, 0x74, 0xf7, 0x71, 0x41, 0x5e, 0x75, 0x5d, 0x55, 0x74, 0x82, 0x80, 0x09, 0x22, 0x0b, 0x3a, - 0xd7, 0xb5, 0x33, 0x33, 0xfa, 0xdb, 0x73, 0x58, 0x52, 0xef, 0x52, 0x4c, 0xf4, 0x12, 0x3e, 0x49, - 0xf4, 0x4d, 0x0b, 0x2c, 0xdd, 0x46, 0x20, 0xd2, 0x12, 0x52, 0x37, 0xa9, 0x1e, 0x53, 0xbe, 0xbc, - 0xc7, 0x28, 0x0f, 0x66, 0xf5, 0x98, 0x0e, 0xac, 0xb8, 0x94, 0x7b, 0x11, 0x75, 0x55, 0x99, 0xa0, - 0x2a, 0x33, 0xab, 0xdb, 0xdf, 0xbd, 0x4a, 0x08, 0xc5, 0xcb, 0x9a, 0xa3, 0x76, 0xa8, 0x0d, 0x25, - 0x1d, 0x37, 0xdc, 0xaa, 0xdc, 0xa6, 0x28, 0xcf, 0x68, 0xe7, 0xca, 0xdc, 0xf2, 0xad, 0xca, 0xdc, - 0x43, 0x00, 0x9f, 0xf5, 0x6d, 0x37, 0xf2, 0xc6, 0x34, 0xb2, 0x56, 0xf4, 0xc4, 0x91, 0xc1, 0xdd, - 0x57, 0x08, 0x5c, 0xf6, 0x59, 0x3f, 0x5e, 0x2e, 0x14, 0xa5, 0xea, 0x2d, 0x8b, 0x12, 0x81, 0x1a, - 0xe1, 0xdc, 0xeb, 0x07, 0xd4, 0xb5, 0xfb, 0x34, 0xa0, 0x91, 0xe7, 0xd8, 0x11, 0xe5, 0x6c, 0x14, - 0x39, 0x94, 0x5b, 0xdf, 0x50, 0x9e, 0xc8, 0x9c, 0x19, 0x1e, 0xc7, 0x60, 0xac, 0xb1, 0xd8, 0x4a, - 0xc4, 0x5c, 0xb8, 0xe0, 0xbb, 0xb5, 0x93, 0xd3, 0xe6, 0x3a, 0xac, 0xa5, 0xcb, 0xd4, 0x8e, 0xf1, - 0xc8, 0x78, 0x62, 0x1c, 0x1a, 0xcd, 0xbf, 0xe5, 0xe0, 0x9b, 0x0b, 0x3e, 0x45, 0x3f, 0x86, 0xa2, - 0xf6, 0xea, 0x55, 0x93, 0x9f, 0xe6, 0xe1, 0x04, 0x8b, 0x36, 0xa0, 0x2c, 0x53, 0x9c, 0x72, 0x4e, - 0xe3, 0xe2, 0x55, 0xc6, 0xf3, 0x03, 0x64, 0x41, 0x91, 0xf8, 0x1e, 0x91, 0x77, 0x79, 0x75, 0x97, - 0x6c, 0xd1, 0x08, 0xd6, 0x63, 0xd7, 0xdb, 0xf3, 0x06, 0x6b, 0xb3, 0x50, 0x70, 0xcb, 0x54, 0xf6, - 0x7f, 0x7d, 0xa3, 0x48, 0xd0, 0x1f, 0x67, 0x7e, 0xf0, 0x3c, 0x14, 0xfc, 0x20, 0x10, 0xd1, 0x04, - 0xaf, 0xb9, 0x19, 0x57, 0xb5, 0xc7, 0x70, 0xff, 0x52, 0x0a, 0x5a, 0x85, 0xfc, 0x80, 0x4e, 0xe2, - 0xf2, 0x84, 0xe5, 0x12, 0xad, 0xc1, 0xd2, 0x98, 0xf8, 0x23, 0xaa, 0xab, 0x59, 0xbc, 0xd9, 0xcd, - 0xed, 0x18, 0xcd, 0x77, 0x39, 0x28, 0x6a, 0x75, 0xee, 0xba, 0xe5, 0xeb, 0x67, 0x17, 0x0a, 0xdb, - 0x57, 0xb0, 0xac, 0x5d, 0x1a, 0x67, 0xa4, 0x79, 0x6d, 0x4c, 0x57, 0x62, 0x7c, 0x9c, 0x8d, 0x5f, - 0x81, 0xe9, 0x85, 0x64, 0xa8, 0xdb, 0x7d, 0xe6, 0xcb, 0xdd, 0xc3, 0xf6, 0xb3, 0xe7, 0x61, 0x5c, - 0x58, 0x4a, 0xd3, 0xb3, 0x86, 0x29, 0x0f, 0xb0, 0xa2, 0x65, 0x36, 0xc6, 0x77, 0x05, 0x28, 0xee, - 0xf9, 0x23, 0x2e, 0x68, 0x74, 0xd7, 0x4e, 0xd2, 0xcf, 0x2e, 0x38, 0x69, 0x0f, 0x8a, 0x11, 0x63, - 0xc2, 0x76, 0xc8, 0x55, 0xfe, 0xc1, 0x8c, 0x89, 0xbd, 0x76, 0xa7, 0x2a, 0x89, 0xb2, 0xb6, 0xc7, - 0x7b, 0x5c, 0x90, 0xd4, 0x3d, 0x82, 0x5e, 0xc1, 0x7a, 0xd2, 0x11, 0x7b, 0x8c, 0x09, 0x2e, 0x22, - 0x12, 0xda, 0x03, 0x3a, 0x91, 0xb3, 0x52, 0xfe, 0xb2, 0x41, 0xfb, 0x20, 0x70, 0xa2, 0x89, 0x72, - 0xde, 0x53, 0x3a, 0xc1, 0x6b, 0x5a, 0x40, 0x27, 0xe1, 0x3f, 0xa5, 0x13, 0x8e, 0xbe, 0x86, 0x0d, - 0x3a, 0x83, 0x49, 0x89, 0xb6, 0x4f, 0x86, 0xb2, 0xd7, 0xdb, 0x8e, 0xcf, 0x9c, 0x81, 0x6a, 0x37, - 0x26, 0xbe, 0x4f, 0xd3, 0xa2, 0x7e, 0x15, 0x23, 0xf6, 0x24, 0x00, 0x71, 0xb0, 0x7a, 0x3e, 0x71, - 0x06, 0xbe, 0xc7, 0xe5, 0xff, 0x52, 0xa9, 0xb9, 0x59, 0x76, 0x0c, 0xa9, 0xdb, 0xce, 0x15, 0xde, - 0x6a, 0x75, 0xe6, 0xdc, 0xd4, 0x14, 0xae, 0x33, 0xea, 0xdb, 0xbd, 0xec, 0x5b, 0xd4, 0x81, 0xca, - 0x28, 0x90, 0xcf, 0xc7, 0x3e, 0x28, 0xdf, 0xd4, 0x07, 0x10, 0xb3, 0x94, 0xe5, 0x1b, 0x60, 0x1e, - 0xcb, 0x19, 0x46, 0xb6, 0x91, 0x52, 0x1c, 0x5c, 0xbf, 0xec, 0x1e, 0x1e, 0x61, 0x75, 0x8a, 0x5a, - 0x80, 0x5c, 0x7a, 0x4c, 0x46, 0xbe, 0x68, 0xc7, 0xb5, 0xe5, 0x90, 0x31, 0x5f, 0xf5, 0x8c, 0x32, - 0xce, 0xb8, 0x41, 0x75, 0x00, 0x3e, 0xea, 0x05, 0x54, 0x1c, 0x79, 0x7f, 0xa4, 0xaa, 0x31, 0xac, - 0xe0, 0xd4, 0x49, 0x6d, 0x0c, 0x1b, 0x57, 0x99, 0x9a, 0x51, 0x09, 0x1e, 0xa5, 0x2b, 0x41, 0x65, - 0xfb, 0xf3, 0x2c, 0xeb, 0xb2, 0x45, 0xa6, 0xaa, 0x46, 0x66, 0x92, 0xfc, 0xd5, 0x80, 0xc2, 0x11, - 0x75, 0x22, 0x2a, 0x3e, 0x6a, 0x8e, 0xec, 0x9c, 0xcb, 0x91, 0x7a, 0xf6, 0xd8, 0x2d, 0x5f, 0x5d, - 0x48, 0x91, 0x1a, 0x94, 0xbc, 0x40, 0xd0, 0x28, 0x20, 0xbe, 0xca, 0x91, 0x12, 0x9e, 0xed, 0xb3, - 0xb3, 0xdc, 0x80, 0x42, 0x3c, 0x97, 0xde, 0xb5, 0x01, 0xf1, 0xab, 0x17, 0x0d, 0xc8, 0x54, 0xf2, - 0xdf, 0x06, 0x94, 0x92, 0xf6, 0xf8, 0x51, 0xd5, 0xbc, 0x30, 0xe7, 0xe5, 0xff, 0xe7, 0x39, 0x0f, - 0x81, 0x39, 0xf0, 0x02, 0x3d, 0x91, 0x62, 0xb5, 0x46, 0x2d, 0x28, 0x86, 0x64, 0xe2, 0x33, 0xe2, - 0xea, 0xb2, 0xbc, 0xb6, 0xf0, 0x9b, 0x48, 0x3b, 0x98, 0xe0, 0x04, 0xb4, 0xbb, 0x76, 0x72, 0xda, - 0x5c, 0x85, 0x6a, 0xda, 0xf2, 0xd7, 0x46, 0xf3, 0x1f, 0x06, 0x94, 0x0f, 0xfe, 0x20, 0x68, 0xa0, - 0xa6, 0x8f, 0xff, 0x4b, 0xe3, 0x37, 0x17, 0x7f, 0x37, 0x29, 0x9f, 0xfb, 0x49, 0x24, 0xeb, 0xa3, - 0x76, 0xac, 0xf7, 0x1f, 0xea, 0xf7, 0xfe, 0xf9, 0xa1, 0x7e, 0xef, 0xed, 0xb4, 0x6e, 0xbc, 0x9f, - 0xd6, 0x8d, 0xbf, 0x4f, 0xeb, 0xc6, 0xbf, 0xa6, 0x75, 0xa3, 0x57, 0x50, 0xfe, 0xf9, 0xd1, 0x7f, - 0x03, 0x00, 0x00, 0xff, 0xff, 0xbc, 0x6b, 0x9b, 0xd8, 0xfe, 0x13, 0x00, 0x00, + // 1610 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xcc, 0x58, 0xcd, 0x73, 0x1b, 0x49, + 0x15, 0xcf, 0xc8, 0x63, 0x7d, 0x3c, 0x59, 0xc2, 0xf4, 0x1a, 0x33, 0x11, 0x46, 0x32, 0xda, 0x5a, + 0x6a, 0x6b, 0x2b, 0x25, 0x2f, 0x66, 0x01, 0xc7, 0xb0, 0x6c, 0x24, 0xdb, 0x24, 0xaa, 0x10, 0xe2, + 0x6a, 0x87, 0x84, 0xdb, 0xd0, 0x9a, 0x69, 0x2b, 0x83, 0x46, 0xd3, 0x53, 0xd3, 0x2d, 0x05, 0x71, + 0xca, 0xd9, 0xfc, 0x01, 0xbe, 0x71, 0x80, 0xbf, 0x82, 0x0b, 0x07, 0x4e, 0xe1, 0xc6, 0x89, 0xe2, + 0xe4, 0x22, 0xfa, 0x2f, 0xb8, 0x51, 0xdd, 0xd3, 0x23, 0x8d, 0xad, 0xf1, 0x17, 0x95, 0x72, 0xed, + 0xc9, 0xfd, 0xf1, 0xfb, 0xbd, 0x7e, 0xef, 0xcd, 0xfb, 0xb2, 0xe0, 0x41, 0xdf, 0x13, 0xaf, 0x47, + 0xbd, 0x96, 0xc3, 0x86, 0x5b, 0x2e, 0x73, 0x06, 0x34, 0xda, 0xe2, 0x6f, 0x48, 0x34, 0x1c, 0x78, + 0x62, 0x8b, 0x84, 0xde, 0x16, 0xeb, 0xfd, 0x8e, 0x3a, 0x82, 0xb7, 0xc2, 0x88, 0x09, 0x86, 0x50, + 0x0c, 0x69, 0x25, 0x90, 0xd6, 0xf8, 0x07, 0xb5, 0xcf, 0xae, 0x91, 0x20, 0x26, 0x21, 0xd5, 0xfc, + 0x6b, 0xb1, 0x3c, 0xa4, 0x4e, 0x82, 0x6d, 0xf4, 0x19, 0xeb, 0xfb, 0x74, 0x4b, 0xed, 0x7a, 0xa3, + 0xe3, 0x2d, 0xe1, 0x0d, 0x29, 0x17, 0x64, 0x18, 0x6a, 0xc0, 0x5a, 0x9f, 0xf5, 0x99, 0x5a, 0x6e, + 0xc9, 0x95, 0x3e, 0xbd, 0x7f, 0x91, 0x46, 0x82, 0x89, 0xbe, 0xfa, 0xc9, 0x15, 0xaf, 0xcf, 0xe0, + 0xa1, 0x3f, 0xea, 0x7b, 0x81, 0xfe, 0x13, 0x13, 0x9b, 0x7f, 0x35, 0xc0, 0x7c, 0x46, 0x05, 0x41, + 0x3f, 0x85, 0xc2, 0x98, 0x46, 0xdc, 0x63, 0x81, 0x65, 0x6c, 0x1a, 0x9f, 0x96, 0xb7, 0xbf, 0xd3, + 0x5a, 0xf4, 0x48, 0xeb, 0x65, 0x0c, 0xe9, 0x98, 0xef, 0xce, 0x1a, 0xf7, 0x70, 0xc2, 0x40, 0x0f, + 0x01, 0x9c, 0x88, 0x12, 0x41, 0x5d, 0x9b, 0x08, 0x2b, 0xa7, 0xf8, 0xb5, 0x56, 0xac, 0x6e, 0x2b, + 0x79, 0xbf, 0xf5, 0x22, 0xb1, 0x12, 0x97, 0x34, 0xba, 0x2d, 0x24, 0x75, 0x14, 0xba, 0x09, 0x75, + 0xe9, 0x7a, 0xaa, 0x46, 0xb7, 0x45, 0xf3, 0xed, 0x32, 0x98, 0xbf, 0x62, 0x2e, 0x45, 0xeb, 0x90, + 0xf3, 0x5c, 0xa5, 0x76, 0xa9, 0x93, 0x9f, 0x9e, 0x35, 0x72, 0xdd, 0x7d, 0x9c, 0xf3, 0x5c, 0xb4, + 0x0d, 0xe6, 0x90, 0x0a, 0xa2, 0x15, 0xb2, 0xb2, 0x0c, 0x92, 0xb6, 0x6b, 0x6b, 0x14, 0x16, 0xfd, + 0x18, 0x4c, 0xf9, 0xa9, 0xb4, 0x26, 0x1b, 0x59, 0x1c, 0xf9, 0xe6, 0x51, 0x48, 0x9d, 0x84, 0x27, + 0xf1, 0xe8, 0x00, 0xca, 0x2e, 0xe5, 0x4e, 0xe4, 0x85, 0x42, 0xfa, 0xd0, 0x54, 0xf4, 0x8f, 0x2f, + 0xa3, 0xef, 0xcf, 0xa1, 0x38, 0xcd, 0x43, 0x3f, 0x83, 0x3c, 0x17, 0x44, 0x8c, 0xb8, 0xb5, 0xac, + 0x24, 0xd4, 0x2f, 0x55, 0x40, 0xa1, 0xb4, 0x0a, 0x9a, 0x83, 0x9e, 0x40, 0x75, 0x48, 0x02, 0xd2, + 0xa7, 0x91, 0xad, 0xa5, 0xe4, 0x95, 0x94, 0xef, 0x65, 0x9a, 0x1e, 0x23, 0x63, 0x41, 0xb8, 0x32, + 0x4c, 0x6f, 0x51, 0x17, 0x80, 0x08, 0x41, 0x9c, 0xd7, 0x43, 0x1a, 0x08, 0xab, 0xa0, 0xa4, 0x7c, + 0x92, 0xa9, 0x0b, 0x15, 0x6f, 0x58, 0x34, 0x68, 0xcf, 0xc0, 0x9d, 0x9c, 0x65, 0xe0, 0x14, 0x19, + 0x3d, 0x86, 0xb2, 0x43, 0x23, 0xe1, 0x1d, 0x7b, 0x0e, 0x11, 0xd4, 0x2a, 0x2a, 0x59, 0x8d, 0x2c, + 0x59, 0x7b, 0x73, 0x98, 0x36, 0x2c, 0xcd, 0x44, 0x9f, 0x83, 0x19, 0x31, 0x9f, 0x5a, 0xa5, 0x4d, + 0xe3, 0xd3, 0xea, 0xe5, 0x9f, 0x06, 0x33, 0x9f, 0x62, 0x85, 0x94, 0x4f, 0xcf, 0x15, 0xe1, 0x16, + 0x6c, 0x2e, 0xdd, 0xd8, 0x0c, 0x9c, 0x66, 0xee, 0xae, 0x9f, 0x9c, 0x36, 0x11, 0xac, 0x16, 0x8d, + 0x55, 0x43, 0xc5, 0x99, 0xf1, 0xb9, 0xf1, 0x1b, 0xe3, 0xb7, 0x46, 0xf3, 0x2f, 0x26, 0x14, 0x8e, + 0x68, 0x34, 0xf6, 0x9c, 0x0f, 0x1b, 0x85, 0x0f, 0xcf, 0x45, 0x61, 0xa6, 0xb3, 0xf4, 0xb3, 0x0b, + 0x81, 0xb8, 0x03, 0x45, 0x1a, 0xb8, 0x21, 0xf3, 0x02, 0xa1, 0xa3, 0x30, 0xd3, 0x53, 0x07, 0x1a, + 0x83, 0x67, 0x68, 0x74, 0x00, 0x95, 0x38, 0xb9, 0xec, 0x73, 0x21, 0xb8, 0x99, 0x45, 0xff, 0xb5, + 0x02, 0xea, 0xd8, 0x59, 0x19, 0xa5, 0x76, 0x68, 0x1f, 0x2a, 0x61, 0x44, 0xc7, 0x1e, 0x1b, 0x71, + 0x5b, 0x19, 0x91, 0xbf, 0x91, 0x11, 0x78, 0x25, 0x61, 0xc9, 0x1d, 0xfa, 0x04, 0xaa, 0x21, 0x0d, + 0x5c, 0x2f, 0xe8, 0xdb, 0x2e, 0xf5, 0xa9, 0xa0, 0x2a, 0x08, 0x8b, 0xb8, 0xa2, 0x4f, 0xf7, 0xd5, + 0x21, 0xfa, 0x39, 0xac, 0xc8, 0x37, 0xec, 0xa4, 0x76, 0xc1, 0xb5, 0xb5, 0x0b, 0x97, 0x25, 0x41, + 0x6f, 0xd0, 0x73, 0xf8, 0xd6, 0x39, 0x65, 0x67, 0x82, 0xca, 0xd7, 0x0b, 0xfa, 0x28, 0xad, 0xb0, + 0x3e, 0xdc, 0x45, 0x27, 0xa7, 0xcd, 0x2a, 0xac, 0xa4, 0x23, 0xa5, 0xf9, 0xa7, 0x1c, 0x14, 0x13, + 0x7f, 0xa3, 0x2f, 0xf4, 0xa7, 0x35, 0x2e, 0x77, 0x6e, 0x82, 0x55, 0x6e, 0x89, 0xbf, 0xea, 0x17, + 0xb0, 0x1c, 0xb2, 0x48, 0x70, 0x2b, 0xa7, 0x62, 0x38, 0xb3, 0x2c, 0x1c, 0xb2, 0x48, 0xec, 0xb1, + 0xe0, 0xd8, 0xeb, 0xe3, 0x18, 0x8c, 0x5e, 0x41, 0x79, 0xec, 0x45, 0x62, 0x44, 0x7c, 0xdb, 0x0b, + 0xb9, 0xb5, 0xa4, 0xb8, 0xdf, 0xbf, 0xea, 0xc9, 0xd6, 0xcb, 0x18, 0xdf, 0x3d, 0xec, 0x54, 0xa7, + 0x67, 0x0d, 0x98, 0x6d, 0x39, 0x06, 0x2d, 0xaa, 0x1b, 0xf2, 0xda, 0x33, 0x28, 0xcd, 0x6e, 0xd0, + 0x03, 0x80, 0x20, 0x4e, 0x1f, 0x7b, 0x96, 0x00, 0x95, 0xe9, 0x59, 0xa3, 0xa4, 0x93, 0xaa, 0xbb, + 0x8f, 0x4b, 0x1a, 0xd0, 0x75, 0x11, 0x02, 0x93, 0xb8, 0x6e, 0xa4, 0xd2, 0xa1, 0x84, 0xd5, 0xba, + 0xf9, 0xc7, 0x02, 0x98, 0x2f, 0x08, 0x1f, 0xdc, 0x75, 0x25, 0x97, 0x6f, 0x2e, 0x24, 0xd0, 0x03, + 0x00, 0x1e, 0x87, 0xa5, 0x34, 0xc7, 0x9c, 0x9b, 0xa3, 0x83, 0x55, 0x9a, 0xa3, 0x01, 0xb1, 0x39, + 0xdc, 0x67, 0x42, 0xe5, 0x8a, 0x89, 0xd5, 0x1a, 0x7d, 0x0c, 0x85, 0x80, 0xb9, 0x8a, 0x9e, 0x57, + 0x74, 0x98, 0x9e, 0x35, 0xf2, 0xb2, 0x36, 0x75, 0xf7, 0x71, 0x5e, 0x5e, 0x75, 0x5d, 0x55, 0x9b, + 0x82, 0x80, 0x09, 0x22, 0xeb, 0x3e, 0xd7, 0x25, 0x36, 0x33, 0x49, 0xda, 0x73, 0x58, 0x52, 0x16, + 0x53, 0x4c, 0xf4, 0x12, 0x3e, 0x4a, 0xf4, 0x4d, 0x0b, 0x2c, 0xde, 0x46, 0x20, 0xd2, 0x12, 0x52, + 0x37, 0xa9, 0x56, 0x54, 0xba, 0xbc, 0x15, 0x29, 0x0f, 0x66, 0xb5, 0xa2, 0x0e, 0x54, 0x5c, 0xca, + 0xbd, 0x88, 0xba, 0xaa, 0x9a, 0x50, 0x95, 0x99, 0xd5, 0xed, 0xef, 0x5e, 0x25, 0x84, 0xe2, 0x15, + 0xcd, 0x51, 0x3b, 0xd4, 0x86, 0xa2, 0x8e, 0x1b, 0x6e, 0x95, 0x6f, 0x53, 0xbb, 0x67, 0xb4, 0x73, + 0xd5, 0x70, 0xe5, 0x56, 0xd5, 0xf0, 0x21, 0x80, 0xcf, 0xfa, 0xb6, 0x1b, 0x79, 0x63, 0x1a, 0x59, + 0x15, 0x3d, 0x98, 0x64, 0x70, 0xf7, 0x15, 0x02, 0x97, 0x7c, 0xd6, 0x8f, 0x97, 0x0b, 0x45, 0xa9, + 0x7a, 0xcb, 0xa2, 0x44, 0xa0, 0x46, 0x38, 0xf7, 0xfa, 0x01, 0x75, 0xed, 0x3e, 0x0d, 0x68, 0xe4, + 0x39, 0x76, 0x44, 0x39, 0x1b, 0x45, 0x0e, 0xe5, 0xd6, 0x37, 0x94, 0x27, 0x32, 0x47, 0x8b, 0xc7, + 0x31, 0x18, 0x6b, 0x2c, 0xb6, 0x12, 0x31, 0x17, 0x2e, 0xf8, 0x6e, 0xed, 0xe4, 0xb4, 0xb9, 0x0e, + 0x6b, 0xe9, 0x32, 0xb5, 0x63, 0x3c, 0x32, 0x9e, 0x18, 0x87, 0x46, 0xf3, 0xef, 0x39, 0xf8, 0xe6, + 0x82, 0x4f, 0xd1, 0x8f, 0xa0, 0xa0, 0xbd, 0x7a, 0xd5, 0x80, 0xa8, 0x79, 0x38, 0xc1, 0xa2, 0x0d, + 0x28, 0xc9, 0x14, 0xa7, 0x9c, 0xd3, 0xb8, 0x78, 0x95, 0xf0, 0xfc, 0x00, 0x59, 0x50, 0x20, 0xbe, + 0x47, 0xe4, 0xdd, 0x92, 0xba, 0x4b, 0xb6, 0x68, 0x04, 0xeb, 0xb1, 0xeb, 0xed, 0x79, 0x1f, 0xb6, + 0x59, 0x28, 0xb8, 0x65, 0x2a, 0xfb, 0xbf, 0xba, 0x51, 0x24, 0xe8, 0x8f, 0x33, 0x3f, 0x78, 0x1e, + 0x0a, 0x7e, 0x10, 0x88, 0x68, 0x82, 0xd7, 0xdc, 0x8c, 0xab, 0xda, 0x63, 0xb8, 0x7f, 0x29, 0x05, + 0xad, 0xc2, 0xd2, 0x80, 0x4e, 0xe2, 0xf2, 0x84, 0xe5, 0x12, 0xad, 0xc1, 0xf2, 0x98, 0xf8, 0x23, + 0xaa, 0xab, 0x59, 0xbc, 0xd9, 0xcd, 0xed, 0x18, 0xcd, 0x7f, 0xe4, 0xa0, 0xa0, 0xd5, 0xb9, 0xeb, + 0xc9, 0x40, 0x3f, 0xbb, 0x50, 0xd8, 0xbe, 0x84, 0x15, 0xed, 0xd2, 0x38, 0x23, 0xcd, 0x6b, 0x63, + 0xba, 0x1c, 0xe3, 0xe3, 0x6c, 0xfc, 0x12, 0x4c, 0x2f, 0x24, 0x43, 0x3d, 0x15, 0x64, 0xbe, 0xdc, + 0x3d, 0x6c, 0x3f, 0x7b, 0x1e, 0xc6, 0x85, 0xa5, 0x38, 0x3d, 0x6b, 0x98, 0xf2, 0x00, 0x2b, 0x5a, + 0x46, 0x43, 0xcf, 0x67, 0x34, 0xf4, 0xcc, 0xfe, 0xf9, 0xe7, 0x3c, 0x14, 0xf6, 0xfc, 0x11, 0x17, + 0x34, 0xba, 0x6b, 0x5f, 0xea, 0x67, 0x17, 0x7c, 0xb9, 0x07, 0x85, 0x88, 0x31, 0x61, 0x3b, 0xe4, + 0x2a, 0x37, 0x62, 0xc6, 0xc4, 0x5e, 0xbb, 0x53, 0x95, 0x44, 0xd9, 0x02, 0xe2, 0x3d, 0xce, 0x4b, + 0xea, 0x1e, 0x41, 0xaf, 0x60, 0x3d, 0x69, 0x9c, 0x3d, 0xc6, 0x04, 0x17, 0x11, 0x09, 0xed, 0x01, + 0x9d, 0xc8, 0xc9, 0x6b, 0xe9, 0xb2, 0xb1, 0xfd, 0x20, 0x70, 0xa2, 0x89, 0xf2, 0xf1, 0x53, 0x3a, + 0xc1, 0x6b, 0x5a, 0x40, 0x27, 0xe1, 0x3f, 0xa5, 0x13, 0x8e, 0xbe, 0x82, 0x0d, 0x3a, 0x83, 0x49, + 0x89, 0xb6, 0x4f, 0x86, 0x72, 0x24, 0xb0, 0x1d, 0x9f, 0x39, 0x03, 0xe5, 0x79, 0x13, 0xdf, 0xa7, + 0x69, 0x51, 0xbf, 0x8c, 0x11, 0x7b, 0x12, 0x80, 0x38, 0x58, 0x3d, 0x9f, 0x38, 0x03, 0xdf, 0xe3, + 0xf2, 0x3f, 0xb3, 0xd4, 0x14, 0x2e, 0x1b, 0x8b, 0xd4, 0x6d, 0xe7, 0x0a, 0x6f, 0xb5, 0x3a, 0x73, + 0x6e, 0x6a, 0xa6, 0xd7, 0x89, 0xf7, 0xed, 0x5e, 0xf6, 0x2d, 0xea, 0x40, 0x79, 0x14, 0xc8, 0xe7, + 0x63, 0x1f, 0x94, 0x6e, 0xea, 0x03, 0x88, 0x59, 0xca, 0xf2, 0x0d, 0x30, 0x8f, 0xe5, 0xa8, 0x23, + 0xbb, 0x4d, 0x31, 0x8e, 0xc1, 0x5f, 0x74, 0x0f, 0x8f, 0xb0, 0x3a, 0x45, 0x2d, 0x40, 0x2e, 0x3d, + 0x26, 0x23, 0x5f, 0xb4, 0xe3, 0x12, 0x74, 0xc8, 0x98, 0xaf, 0x5a, 0x4b, 0x09, 0x67, 0xdc, 0xa0, + 0x3a, 0x00, 0x1f, 0xf5, 0x02, 0x2a, 0x8e, 0xbc, 0x3f, 0x50, 0xd5, 0x3f, 0x2a, 0x38, 0x75, 0x52, + 0x1b, 0xc3, 0xc6, 0x55, 0xa6, 0x66, 0x14, 0x8c, 0x47, 0xe9, 0x82, 0x51, 0xde, 0xfe, 0x2c, 0xcb, + 0xba, 0x6c, 0x91, 0xa9, 0xe2, 0x92, 0x99, 0x24, 0x7f, 0x33, 0x20, 0x7f, 0x44, 0x9d, 0x88, 0x8a, + 0x0f, 0x9a, 0x23, 0x3b, 0xe7, 0x72, 0xa4, 0x9e, 0x3d, 0xc4, 0xcb, 0x57, 0x17, 0x52, 0xa4, 0x06, + 0x45, 0x2f, 0x10, 0x34, 0x0a, 0x88, 0xaf, 0x72, 0xa4, 0x88, 0x67, 0xfb, 0xec, 0x2c, 0x37, 0x20, + 0x1f, 0x8f, 0xaf, 0x77, 0x6d, 0x40, 0xfc, 0xea, 0x45, 0x03, 0x32, 0x95, 0xfc, 0xaf, 0x01, 0xc5, + 0xa4, 0x8b, 0x7e, 0x50, 0x35, 0x2f, 0x8c, 0x83, 0x4b, 0xff, 0xf7, 0x38, 0x88, 0xc0, 0x1c, 0x78, + 0x81, 0x1e, 0x5c, 0xb1, 0x5a, 0xa3, 0x16, 0x14, 0x42, 0x32, 0xf1, 0x19, 0x71, 0x75, 0xf5, 0x5e, + 0x5b, 0xf8, 0x85, 0xa5, 0x1d, 0x4c, 0x70, 0x02, 0xda, 0x5d, 0x3b, 0x39, 0x6d, 0xae, 0x42, 0x35, + 0x6d, 0xf9, 0x6b, 0xa3, 0xf9, 0x2f, 0x03, 0x4a, 0x07, 0xbf, 0x17, 0x34, 0x50, 0x43, 0xca, 0xd7, + 0xd2, 0xf8, 0xcd, 0xc5, 0x5f, 0x61, 0x4a, 0xe7, 0x7e, 0x60, 0xc9, 0xfa, 0xa8, 0x1d, 0xeb, 0xdd, + 0xfb, 0xfa, 0xbd, 0x7f, 0xbf, 0xaf, 0xdf, 0x7b, 0x3b, 0xad, 0x1b, 0xef, 0xa6, 0x75, 0xe3, 0x9f, + 0xd3, 0xba, 0xf1, 0x9f, 0x69, 0xdd, 0xe8, 0xe5, 0x95, 0x7f, 0x7e, 0xf8, 0xbf, 0x00, 0x00, 0x00, + 0xff, 0xff, 0xea, 0x67, 0xde, 0xa7, 0x4c, 0x14, 0x00, 0x00, } diff --git a/api/objects.proto b/api/objects.proto index 2211395bf5..1c523dfe36 100644 --- a/api/objects.proto +++ b/api/objects.proto @@ -122,6 +122,16 @@ message Service { // UpdateStatus contains the status of an update, if one is in // progress. UpdateStatus update_status = 5; + + // PendingDelete indicates that this service's deletion has been requested. + // Services, as well as all service-level resources, can only be deleted + // after all of the service's containers have properly shut down. + // When a user requests a deletion, we just flip this flag + // the deallocator will take it from there - it will start monitoring + // this service's tasks, and proceed to delete the service itself (and + // potentially its associated resources also marked for deletion) when + // all of its tasks are gone + bool pending_delete = 7; } // Endpoint specified all the network parameters required to @@ -292,6 +302,20 @@ message Network { // Runtime state of IPAM options. This may not reflect the // ipam options from NetworkSpec. IPAMOptions ipam = 5 [(gogoproto.customname) = "IPAM"]; + + // PendingDelete indicates that this network's deletion has been requested. + // Services, as well as all service-level resources, can only be deleted + // after all the service's containers have properly shut down + // when a user requests a deletion, we just flip this flag + // the deallocator will take it from there + // PendingDelete indicates that this network's deletion has been requested. + // Services, as well as all service-level resources, can only be deleted + // after all of the service's containers have properly shut down. + // When a user requests a deletion of this network, we just flip this flag + // the deallocator will take it from there - it will start monitoring + // the services that still use this service, and proceed to delete + // this network when all of these services are gone + bool pending_delete = 6; } // Cluster provides global cluster settings. diff --git a/manager/deallocator/deallocator.go b/manager/deallocator/deallocator.go new file mode 100644 index 0000000000..8144152dce --- /dev/null +++ b/manager/deallocator/deallocator.go @@ -0,0 +1,291 @@ +package deallocator + +import ( + "context" + + "github.com/docker/go-events" + "github.com/docker/swarmkit/api" + "github.com/docker/swarmkit/log" + "github.com/docker/swarmkit/manager/state/store" +) + +// Deallocator waits for services to fully shutdown (ie no containers left) +// and then proceeds to deallocate service-level resources (e.g. networks), +// and finally services themselves +// in particular, the Deallocator should be the only place where services, or +// service-level resources, are ever deleted! +// +// It’s worth noting that this new component’s role is quite different from +// the task reaper’s: tasks are purely internal to Swarmkit, and their status +// is entirely managed by the system itself. In contrast, the deallocator is +// responsible for safely deleting entities that are directly controlled by the +// user. +// +// NOTE: since networks are the only service-level resources as of now, +// it has been deemed over-engineered to have a generic way to +// handle other types of service-level resources; if we ever start +// having more of those and thus want to reconsider this choice, it +// might be worth having a look at this archived branch, that does +// implement a way of separating the code for the deallocator itself +// from each resource-speficic way of handling it +// https://github.com/docker/swarmkit/compare/a84c01f49091167dd086c26b45dc18b38d52e4d9...wk8:wk8/generic_deallocator#diff-75f4f75eee6a6a7a7268c672203ea0ac +type Deallocator struct { + store *store.MemoryStore + + // for services that are shutting down, we keep track of how many + // tasks still exist for them + services map[string]*serviceWithTaskCounts + + // mainly used for tests, so that we can peek + // into the DB state in between events + // the bool notifies whether any DB update was actually performed + eventChan chan bool + + stopChan chan struct{} + doneChan chan struct{} +} + +// used in our internal state's `services` right above +type serviceWithTaskCounts struct { + service *api.Service + taskCount int +} + +// New creates a new deallocator +func New(store *store.MemoryStore) *Deallocator { + return &Deallocator{ + store: store, + services: make(map[string]*serviceWithTaskCounts), + + stopChan: make(chan struct{}), + doneChan: make(chan struct{}), + } +} + +// Run starts the deallocator, which then starts cleaning up services +// and their resources when relevant (ie when no tasks still exist +// for a given service) +// This is a blocking function +func (deallocator *Deallocator) Run(ctx context.Context) error { + var ( + allServices []*api.Service + allNetworks []*api.Network + ) + + eventsChan, _, err := store.ViewAndWatch(deallocator.store, + func(readTx store.ReadTx) (err error) { + // look for services that are marked for deletion + // there's no index on the `PendingDelete` field in the store, + // so we just iterate over all of them and filter manually + // this is okay since we only do this at leadership change + allServices, err = store.FindServices(readTx, store.All) + + if err != nil { + log.G(ctx).WithError(err).Error("failed to list services in deallocator init") + return err + } + + // now we also need to look at all existing service-level networks + // that may be marked for deletion + if allNetworks, err = store.FindNetworks(readTx, store.All); err != nil { + log.G(ctx).WithError(err).Error("failed to list networks in deallocator init") + return err + } + + return + }, + api.EventDeleteTask{}, + api.EventUpdateService{}, + api.EventUpdateNetwork{}) + + if err != nil { + // if we have an error here, we can't proceed any further + log.G(ctx).WithError(err).Error("failed to initialize the deallocator") + return err + } + + defer func() { + // eventsChanCancel() + close(deallocator.doneChan) + }() + + anyUpdated := false + // now let's populate our internal taskCounts + for _, service := range allServices { + if updated, _ := deallocator.processService(ctx, service); updated { + anyUpdated = true + } + } + + // and deallocate networks that may be marked for deletion and aren't used any more + for _, network := range allNetworks { + if updated, _ := deallocator.processNetwork(ctx, nil, network, nil); updated { + anyUpdated = true + } + } + + // now we just need to wait for events + deallocator.notifyEventChan(anyUpdated) + for { + select { + case event := <-eventsChan: + if updated, err := deallocator.processNewEvent(ctx, event); err == nil { + deallocator.notifyEventChan(updated) + } else { + log.G(ctx).WithError(err).Errorf("error processing deallocator event %#v", event) + } + case <-deallocator.stopChan: + return nil + case <-ctx.Done(): + return ctx.Err() + } + } +} + +// Stop stops the deallocator's routine +// FIXME (jrouge): see the comment on TaskReaper.Stop() and see when to properly stop this +// plus unit test on this! +func (deallocator *Deallocator) Stop() { + close(deallocator.stopChan) + <-deallocator.doneChan +} + +// always a bno-op, except when running tests tests +// see the comment about `Deallocator`s' `eventChan` field +func (deallocator *Deallocator) notifyEventChan(updated bool) { + if deallocator.eventChan != nil { + deallocator.eventChan <- updated + } +} + +// if a service is marked for deletion, this checks whether it's ready to be +// deleted yet, and does it if relevant +func (deallocator *Deallocator) processService(ctx context.Context, service *api.Service) (bool, error) { + if !service.PendingDelete { + return false, nil + } + + var ( + tasks []*api.Task + err error + ) + + deallocator.store.View(func(tx store.ReadTx) { + tasks, err = store.FindTasks(tx, store.ByServiceID(service.ID)) + }) + + if err != nil { + log.G(ctx).WithError(err).Errorf("failed to retrieve the list of tasks for service %v", service.ID) + // if in doubt, let's proceed to clean up the service anyway + // better to clean up resources that shouldn't be cleaned up yet + // than ending up with a service and some resources lost in limbo forever + return true, deallocator.deallocateService(ctx, service) + } else if len(tasks) == 0 { + // no tasks remaining for this service, we can clean it up + return true, deallocator.deallocateService(ctx, service) + } + deallocator.services[service.ID] = &serviceWithTaskCounts{service: service, taskCount: len(tasks)} + return false, nil +} + +func (deallocator *Deallocator) deallocateService(ctx context.Context, service *api.Service) (err error) { + err = deallocator.store.Update(func(tx store.Tx) error { + // first, let's delete the service + var ignoreServiceID *string + if err := store.DeleteService(tx, service.ID); err != nil { + // all errors are just for logging here, we do a best effort at cleaning up everything we can + log.G(ctx).WithError(err).Errorf("failed to delete service record ID %v", service.ID) + ignoreServiceID = &service.ID + } + + // then all of its networks, provided no other service uses them + spec := service.Spec + // see https://github.com/docker/swarmkit/blob/e2aafdd3453d2ab103dd97364f79ea6b857f9446/api/specs.proto#L80-L84 + // we really should have a helper function on services to do this... + networkConfigs := spec.Task.Networks + if len(networkConfigs) == 0 { + networkConfigs = spec.Networks + } + for _, networkConfig := range networkConfigs { + if network := store.GetNetwork(tx, networkConfig.Target); network != nil { + deallocator.processNetwork(ctx, tx, network, ignoreServiceID) + } + } + + return nil + }) + + if err != nil { + log.G(ctx).WithError(err).Errorf("DB error when deallocating service %v", service.ID) + } + return +} + +// proceeds to deallocate a network if it's pending deletion and there no +// longer are any services using it +// actually deletes the network if it's marked for deletion and no services are +// using it any more (or the only one using it has ID `ignoreServiceID`, if not +// nil - this comes in handy when there's been an error deleting a service) +// This function can be called either when deallocating a whole service, or +// because there was an `EventUpdateNetwork` event - in the former case, the +// transaction will be that of the service deallocation, in the latter it will be nil +func (deallocator *Deallocator) processNetwork(ctx context.Context, tx store.Tx, network *api.Network, ignoreServiceID *string) (updated bool, err error) { + if !network.PendingDelete { + return + } + + updateFunc := func(t store.Tx) error { + services, err := store.FindServices(t, store.ByReferencedNetworkID(network.ID)) + + if err != nil { + log.G(ctx).WithError(err).Errorf("could not fetch services using network ID %v", network.ID) + return err + } + + noMoreServices := len(services) == 0 || + len(services) == 1 && ignoreServiceID != nil && services[0].ID == *ignoreServiceID + + if noMoreServices { + return store.DeleteNetwork(t, network.ID) + } + return nil + } + + if tx == nil { + err = deallocator.store.Update(updateFunc) + } else { + err = updateFunc(tx) + } + + if err != nil { + log.G(ctx).WithError(err).Errorf("DB error when deallocating network ID %v", network.ID) + } + return +} + +// Processes new events, and dispatches to the right method depending on what +// type of event it is. +// The boolean part of the return tuple indicates whether anything was actually +// removed from the store +func (deallocator *Deallocator) processNewEvent(ctx context.Context, event events.Event) (bool, error) { + switch typedEvent := event.(type) { + case api.EventDeleteTask: + serviceID := typedEvent.Task.ServiceID + + if serviceWithCount, present := deallocator.services[serviceID]; present { + if serviceWithCount.taskCount <= 1 { + delete(deallocator.services, serviceID) + return deallocator.processService(ctx, serviceWithCount.service) + } + serviceWithCount.taskCount-- + } + + return false, nil + case api.EventUpdateService: + return deallocator.processService(ctx, typedEvent.Service) + case api.EventUpdateNetwork: + return deallocator.processNetwork(ctx, nil, typedEvent.Network, nil) + default: + return false, nil + } +} diff --git a/manager/deallocator/deallocator_test.go b/manager/deallocator/deallocator_test.go new file mode 100644 index 0000000000..1703d993cb --- /dev/null +++ b/manager/deallocator/deallocator_test.go @@ -0,0 +1,406 @@ +package deallocator + +import ( + "context" + "strconv" + "testing" + "time" + + "github.com/docker/swarmkit/api" + "github.com/docker/swarmkit/manager/state/store" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestDeallocatorInit(t *testing.T) { + // start up the memory store + s := store.NewMemoryStore(nil) + require.NotNil(t, s) + defer s.Close() + + // create a service that's pending deletion, with no tasks remaining + // this one should be deleted by the deallocator + // additionally, that service is using a network that's also marked for + // deletion, and another that's not + network1 := newNetwork("network1", true) + network2 := newNetwork("network2", false) + service1 := newService("service1", true, network1, network2) + + // now let's create another service that's also pending deletion, but still + // has one task associated with it (in any state) - and also uses a network + // that's also marked for deletion + // none of those should get deleted + network3 := newNetwork("network3", true) + service2 := newService("service2", true, network3) + task1 := newTask("task1", service2) + + // let's also have a network that's pending deletion, + // but isn't used by any existing service + // this one should be gone after the init + network4 := newNetwork("network4", true) + + // and finally a network that's not pending deletion, not + // used by any service + network5 := newNetwork("network5", false) + + createDBObjects(t, s, service1, service2, + network1, network2, network3, network4, network5, task1) + + // create and start the deallocator + deallocator, ran := startNewDeallocator(t, s, true) + + // and then stop it immediately - we're just interested in the init + // phase for this test + stopDeallocator(t, deallocator, ran) + + // now let's check that the DB is in the state we expect + s.View(func(tx store.ReadTx) { + assert.Nil(t, store.GetService(tx, service1.ID)) + assert.Nil(t, store.GetNetwork(tx, network1.ID)) + assert.NotNil(t, store.GetNetwork(tx, network2.ID)) + + assert.NotNil(t, store.GetService(tx, service2.ID)) + assert.NotNil(t, store.GetNetwork(tx, network3.ID)) + + assert.Nil(t, store.GetNetwork(tx, network4.ID)) + + assert.NotNil(t, store.GetNetwork(tx, network5.ID)) + }) +} + +// this tests what happens when a service is marked for deletion +func TestServiceDelete(t *testing.T) { + // we test services with respectively 1, 2, 5 and 10 tasks + for _, taskCount := range []int{1, 2, 5, 10} { + t.Run("service delete with "+strconv.Itoa(taskCount)+" tasks", + func(t *testing.T) { + // start up the memory store + s := store.NewMemoryStore(nil) + require.NotNil(t, s) + defer s.Close() + + // let's create the task and services + service := newService("service", false) + createDBObjects(t, s, service) + + taskIDs := make([]string, taskCount) + tasks := make([]interface{}, taskCount) + for i := 0; i < taskCount; i++ { + taskIDs[i] = "task" + strconv.Itoa(i+1) + tasks[i] = newTask(taskIDs[i], service) + } + createDBObjects(t, s, tasks...) + + // now let's start the deallocator + deallocator, ran := startNewDeallocator(t, s, false) + defer stopDeallocator(t, deallocator, ran) + + // then let's mark the service for deletion... + updateStoreAndWaitForEvent(t, deallocator, func(tx store.Tx) { + service.PendingDelete = true + require.NoError(t, store.UpdateService(tx, service)) + }, false) + + // and then let's remove all tasks + for i, taskID := range taskIDs { + lastTask := i == len(taskIDs)-1 + + updateStoreAndWaitForEvent(t, deallocator, func(tx store.Tx) { + require.NoError(t, store.DeleteTask(tx, taskID)) + }, lastTask) + + // and after each but the last one, the service should still + // be there - after the last one it should be gone + s.View(func(tx store.ReadTx) { + if lastTask { + require.Nil(t, store.GetService(tx, service.ID)) + } else { + require.NotNil(t, store.GetService(tx, service.ID)) + } + }) + + } + }) + } +} + +// this tests what happens when a service is marked for deletion, +// along with its network, _before_ the service has had time to +// fully shut down +func TestServiceAndNetworkDelete(t *testing.T) { + s := store.NewMemoryStore(nil) + require.NotNil(t, s) + defer s.Close() + + // let's create a couple of networks, a service, and a couple of tasks + network1 := newNetwork("network1", false) + network2 := newNetwork("network2", false) + service := newService("service", false, network1, network2) + task1 := newTask("task1", service) + task2 := newTask("task2", service) + + createDBObjects(t, s, network1, network2, service, task1, task2) + + deallocator, ran := startNewDeallocator(t, s, false) + defer stopDeallocator(t, deallocator, ran) + + // then let's mark the service and network2 for deletion + updateStoreAndWaitForEvent(t, deallocator, func(tx store.Tx) { + service.PendingDelete = true + require.NoError(t, store.UpdateService(tx, service)) + }, false) + + updateStoreAndWaitForEvent(t, deallocator, func(tx store.Tx) { + network2.PendingDelete = true + require.NoError(t, store.UpdateNetwork(tx, network2)) + }, false) + + // then let's delete one task + updateStoreAndWaitForEvent(t, deallocator, func(tx store.Tx) { + require.NoError(t, store.DeleteTask(tx, task2.ID)) + }, false) + + // the service and network2 should still exist + s.View(func(tx store.ReadTx) { + require.NotNil(t, store.GetService(tx, service.ID)) + require.NotNil(t, store.GetNetwork(tx, network2.ID)) + }) + + // now let's delete the other task + updateStoreAndWaitForEvent(t, deallocator, func(tx store.Tx) { + require.NoError(t, store.DeleteTask(tx, task1.ID)) + }, true) + + // now the service and network2 should be gone + s.View(func(tx store.ReadTx) { + require.Nil(t, store.GetService(tx, service.ID)) + require.Nil(t, store.GetNetwork(tx, network2.ID)) + + // quick sanity check, the first service should be + // unaffected + require.NotNil(t, store.GetNetwork(tx, network1.ID)) + }) +} + +// this tests that an update to a service that is _not_ marked it for deletion +// doesn't do anything +func TestServiceNotMarkedForDeletion(t *testing.T) { + s := store.NewMemoryStore(nil) + require.NotNil(t, s) + defer s.Close() + + service := newService("service", false) + createDBObjects(t, s, service) + + deallocator, ran := startNewDeallocator(t, s, false) + defer stopDeallocator(t, deallocator, ran) + + updateStoreAndWaitForEvent(t, deallocator, func(tx store.Tx) { + service.Meta = api.Meta{Version: api.Version{Index: 12}} + require.NoError(t, store.UpdateService(tx, service)) + }, + // the deallocator shouldn't do any DB updates based on this event + false) +} + +// this tests that an update to a network that is _not_ marked it for deletion +// doesn't do anything +func TestNetworkNotMarkedForDeletion(t *testing.T) { + s := store.NewMemoryStore(nil) + require.NotNil(t, s) + defer s.Close() + + network := newNetwork("network", false) + service := newService("service", false, network) + createDBObjects(t, s, network, service) + + deallocator, ran := startNewDeallocator(t, s, false) + defer stopDeallocator(t, deallocator, ran) + + updateStoreAndWaitForEvent(t, deallocator, func(tx store.Tx) { + network.IPAM = &api.IPAMOptions{Driver: &api.Driver{Name: "test_driver"}} + require.NoError(t, store.UpdateNetwork(tx, network)) + }, + // the deallocator shouldn't do any DB updates based on this event + false) +} + +// this test that the deallocator also works with the "old" style of storing +// networks directly on the service spec (instead of the task spec) +// TODO: as said in the source file, we should really add a helper +// on services objects itself, and test it there instead +func TestDeallocatorWithOldStyleNetworks(t *testing.T) { + s := store.NewMemoryStore(nil) + require.NotNil(t, s) + defer s.Close() + + service := newService("service", true) + + // add a couple of networks with the old style + network1 := newNetwork("network1", true) + network2 := newNetwork("network2", false) + service.Spec.Networks = append(service.Spec.Networks, newNetworkConfigs(network1, network2)...) + task := newTask("task", service) + + createDBObjects(t, s, service, network1, network2, task) + + deallocator, ran := startNewDeallocator(t, s, + // nothing should have been deleted + false) + defer stopDeallocator(t, deallocator, ran) + + // now let's delete the one task saving it all from oblivion + updateStoreAndWaitForEvent(t, deallocator, func(tx store.Tx) { + require.NoError(t, store.DeleteTask(tx, task.ID)) + }, true) + + // the deallocator should have removed both the service and + // the first network, but not the second + s.View(func(tx store.ReadTx) { + require.Nil(t, store.GetService(tx, service.ID)) + require.Nil(t, store.GetNetwork(tx, network1.ID)) + require.NotNil(t, store.GetNetwork(tx, network2.ID)) + }) +} + +// Helpers below + +// starts a new deallocator, and also creates a channel to retrieve the return +// value, so that we can check later than there was no error +func startNewDeallocator(t *testing.T, s *store.MemoryStore, expectedUpdates bool) (deallocator *Deallocator, ran chan error) { + deallocator = New(s) + deallocator.eventChan = make(chan bool) + ran = make(chan error) + + go func() { + returnValue := deallocator.Run(context.Background()) + // allows checking that `Run` does return after we've stopped + ran <- returnValue + close(ran) + }() + waitForDeallocatorEvent(t, deallocator, expectedUpdates) + + return +} + +// stops the deallocator started by `startNewDeallocator` above +func stopDeallocator(t *testing.T, deallocator *Deallocator, ran chan error) { + stopped := make(chan struct{}) + go func() { + deallocator.Stop() + close(stopped) + }() + + // it shouldn't take too long to stop + select { + case <-stopped: + case <-time.After(time.Second): + t.Fatal("Waited for too long for the deallocator to stop") + } + + // `Run` should have returned, too + select { + case returnValue := <-ran: + require.NoError(t, returnValue) + case <-time.After(time.Second): + t.Fatal("Run hasn't returned") + } + + ensureNoDeallocatorEvent(t, deallocator) +} + +func waitForDeallocatorEvent(t *testing.T, deallocator *Deallocator, expectedUpdates bool) { + select { + case updates := <-deallocator.eventChan: + if updates != expectedUpdates { + t.Errorf("Expected updates %v VS actual %v", expectedUpdates, updates) + } + ensureNoDeallocatorEvent(t, deallocator) + case <-time.After(time.Second): + t.Fatal("Waited for too long for the deallocator to process new events") + } +} + +func ensureNoDeallocatorEvent(t *testing.T, deallocator *Deallocator) { + select { + case <-deallocator.eventChan: + t.Fatal("Found unexpected event") + default: + } +} + +func createDBObjects(t *testing.T, s *store.MemoryStore, objects ...interface{}) { + err := s.Update(func(tx store.Tx) (e error) { + for _, object := range objects { + switch typedObject := object.(type) { + case *api.Service: + e = store.CreateService(tx, typedObject) + case *api.Task: + e = store.CreateTask(tx, typedObject) + case *api.Network: + e = store.CreateNetwork(tx, typedObject) + } + require.NoError(t, e, "Unable to create DB object %v", object) + } + return + }) + require.NoError(t, err, "Error setting up test fixtures") +} + +func updateStore(t *testing.T, s *store.MemoryStore, cb func(x store.Tx)) { + require.NoError(t, s.Update(func(tx store.Tx) error { + cb(tx) + return nil + })) +} + +func updateStoreAndWaitForEvent(t *testing.T, deallocator *Deallocator, cb func(x store.Tx), expectedUpdates bool) { + updateStore(t, deallocator.store, cb) + waitForDeallocatorEvent(t, deallocator, expectedUpdates) +} + +func newService(id string, pendingDelete bool, networks ...*api.Network) *api.Service { + return &api.Service{ + ID: id, + Spec: api.ServiceSpec{ + Annotations: api.Annotations{ + Name: id, + }, + Task: api.TaskSpec{ + Networks: newNetworkConfigs(networks...), + }, + }, + PendingDelete: pendingDelete, + } +} + +func newNetwork(id string, pendingDelete bool) *api.Network { + return &api.Network{ + ID: id, + Spec: api.NetworkSpec{ + Annotations: api.Annotations{ + Name: id, + }, + }, + PendingDelete: pendingDelete, + } +} + +func newNetworkConfigs(networks ...*api.Network) []*api.NetworkAttachmentConfig { + networkConfigs := make([]*api.NetworkAttachmentConfig, len(networks)) + + for i := 0; i < len(networks); i++ { + networkConfigs[i] = &api.NetworkAttachmentConfig{ + Target: networks[i].ID, + } + } + + return networkConfigs +} + +func newTask(id string, service *api.Service) *api.Task { + return &api.Task{ + ID: id, + ServiceID: service.ID, + } +} diff --git a/manager/orchestrator/taskreaper/task_reaper.go b/manager/orchestrator/taskreaper/task_reaper.go index 5d2d2c7490..7fdb0a65d0 100644 --- a/manager/orchestrator/taskreaper/task_reaper.go +++ b/manager/orchestrator/taskreaper/task_reaper.go @@ -31,7 +31,7 @@ type TaskReaper struct { // taskHistory is the number of tasks to keep taskHistory int64 - // List of slot tubles to be inspected for task history cleanup. + // List of slot tuples to be inspected for task history cleanup. dirty map[orchestrator.SlotTuple]struct{} // List of tasks collected for cleanup, which includes two kinds of tasks @@ -61,7 +61,7 @@ func New(store *store.MemoryStore) *TaskReaper { // Run is the TaskReaper's watch loop which collects candidates for cleanup. // Task history is mainly used in task restarts but is also available for administrative purposes. // Note that the task history is stored per-slot-per-service for replicated services -// and per-node-per-service for global services. History does not apply to serviceless +// and per-node-per-service for global services. History does not apply to serviceless tasks // since they are not attached to a service. In addition, the TaskReaper watch loop is also // responsible for cleaning up tasks associated with slots that were removed as part of // service scale down or service removal. @@ -196,11 +196,9 @@ func (tr *TaskReaper) Run(ctx context.Context) { } isTimerStopped = true tr.tick() - } else { - if isTimerStopped { - timer.Reset(reaperBatchingInterval) - isTimerStopped = false - } + } else if isTimerStopped { + timer.Reset(reaperBatchingInterval) + isTimerStopped = false } case <-timer.C: // we can safely ignore draining off of the timer channel, because diff --git a/manager/state/store/networks.go b/manager/state/store/networks.go index 3042def1bf..fa887b3b11 100644 --- a/manager/state/store/networks.go +++ b/manager/state/store/networks.go @@ -105,7 +105,7 @@ func GetNetwork(tx ReadTx, id string) *api.Network { func FindNetworks(tx ReadTx, by By) ([]*api.Network, error) { checkType := func(by By) error { switch by.(type) { - case byName, byNamePrefix, byIDPrefix, byCustom, byCustomPrefix: + case byName, byNamePrefix, byIDPrefix, byCustom, byCustomPrefix, byAll: return nil default: return ErrInvalidFindBy diff --git a/manager/state/store/services.go b/manager/state/store/services.go index 1adbb87fe4..a581737369 100644 --- a/manager/state/store/services.go +++ b/manager/state/store/services.go @@ -126,7 +126,7 @@ func GetService(tx ReadTx, id string) *api.Service { func FindServices(tx ReadTx, by By) ([]*api.Service, error) { checkType := func(by By) error { switch by.(type) { - case byName, byNamePrefix, byIDPrefix, byRuntime, byReferencedNetworkID, byReferencedSecretID, byReferencedConfigID, byCustom, byCustomPrefix: + case byName, byNamePrefix, byIDPrefix, byRuntime, byReferencedNetworkID, byReferencedSecretID, byReferencedConfigID, byCustom, byCustomPrefix, byAll: return nil default: return ErrInvalidFindBy