diff --git a/jsonapi-resources.gemspec b/jsonapi-resources.gemspec
index 2f2044aa..65372c9f 100644
--- a/jsonapi-resources.gemspec
+++ b/jsonapi-resources.gemspec
@@ -31,5 +31,4 @@ Gem::Specification.new do |spec|
   spec.add_dependency 'activerecord', '>= 5.1'
   spec.add_dependency 'railties', '>= 5.1'
   spec.add_dependency 'concurrent-ruby'
-  spec.add_dependency 'sorted_set'
 end
diff --git a/lib/jsonapi/resource_fragment.rb b/lib/jsonapi/resource_fragment.rb
index fb923714..c354f9aa 100644
--- a/lib/jsonapi/resource_fragment.rb
+++ b/lib/jsonapi/resource_fragment.rb
@@ -25,7 +25,7 @@ def initialize(identity, resource: nil, cache: nil, primary: false)
       @primary = primary
 
       @related = {}
-      @related_from = SortedSet.new
+      @related_from = Set.new
     end
 
     def initialize_related(relationship_name)
diff --git a/lib/jsonapi/resource_set.rb b/lib/jsonapi/resource_set.rb
index 894d8557..f4d6186f 100644
--- a/lib/jsonapi/resource_set.rb
+++ b/lib/jsonapi/resource_set.rb
@@ -180,7 +180,7 @@ def flatten_resource_tree(resource_tree, flattened_tree = {})
         flattened_tree[resource_klass][id][:resource] ||= fragment.resource if fragment.resource
 
         fragment.related.try(:each_pair) do |relationship_name, related_rids|
-          flattened_tree[resource_klass][id][:relationships][relationship_name] ||= SortedSet.new
+          flattened_tree[resource_klass][id][:relationships][relationship_name] ||= Set.new
           flattened_tree[resource_klass][id][:relationships][relationship_name].merge(related_rids)
         end
       end