diff --git a/lib/active_model/serializer/associations.rb b/lib/active_model/serializer/associations.rb
index 76574dd7..0c961fbc 100644
--- a/lib/active_model/serializer/associations.rb
+++ b/lib/active_model/serializer/associations.rb
@@ -53,7 +53,7 @@ module ActiveModel
         end
 
         def build_serializer(object, options = {})
-          if object.respond_to?(:to_ary)
+          if object.respond_to?(:to_ary) && !(@serializer_class && @serializer_class <= ArraySerializer)
             use_array_serializer!
           else
             @serializer_class ||= Serializer.serializer_for(object) || DefaultSerializer
diff --git a/test/unit/active_model/serializer/associations/build_serializer_test.rb b/test/unit/active_model/serializer/associations/build_serializer_test.rb
new file mode 100644
index 00000000..c2d84efc
--- /dev/null
+++ b/test/unit/active_model/serializer/associations/build_serializer_test.rb
@@ -0,0 +1,24 @@
+require 'test_helper'
+
+module ActiveModel
+  class Serializer
+    class Association
+      class BuildSerializerTest < ActiveModel::TestCase
+        def setup
+          @association = Association::HasOne.new('post', serializer: PostSerializer)
+          @post = Post.new({ title: 'Title 1', body: 'Body 1', date: '1/1/2000' })
+        end
+
+        def test_build_serializer_for_array_called_twice
+          2.times do
+            serializer      = @association.build_serializer([@post])
+            each_serializer = serializer.serializer_for(@post)
+
+            assert_instance_of(ArraySerializer, serializer)
+            assert_instance_of(PostSerializer,  each_serializer)
+          end
+        end
+      end
+    end
+  end
+end