diff --git a/lib/buildkite/config/rake_command.rb b/lib/buildkite/config/rake_command.rb
index fceb4ee..67129d1 100644
--- a/lib/buildkite/config/rake_command.rb
+++ b/lib/buildkite/config/rake_command.rb
@@ -106,16 +106,17 @@ def rake(dir, task: "test", label: nil, service: "default", pre_steps: nil, env:
 
           artifact_paths build_context.artifact_paths
 
-          if retry_on
-            automatic_retry_on(**retry_on)
-          else
-            automatic_retry_on(**build_context.automatic_retry_on)
+          [retry_on].flatten.compact.each do |retry_args|
+            automatic_retry_on(**retry_args)
           end
+          automatic_retry_on(**build_context.automatic_retry_on)
 
           timeout_in_minutes build_context.timeout_in_minutes
 
-          if soft_fail || build_context.ruby.soft_fail?
+          if soft_fail.is_a?(TrueClass) || build_context.ruby.soft_fail?
             soft_fail true
+          else
+            soft_fail([soft_fail].flatten.compact) if soft_fail
           end
 
           if parallelism
diff --git a/pipelines/rails-ci/pipeline.rb b/pipelines/rails-ci/pipeline.rb
index 5102c73..c633a38 100644
--- a/pipelines/rails-ci/pipeline.rb
+++ b/pipelines/rails-ci/pipeline.rb
@@ -160,7 +160,7 @@
       end
 
       # ActionCable and ActiveJob integration tests
-      rake "actioncable", task: "test:integration", retry_on: { exit_status: -1, limit: 3 }
+      rake "actioncable", task: "test:integration", retry_on: { exit_status: 3, limit: 1 }, soft_fail: { exit_status: 3 }
 
       if ruby == build_context.default_ruby
         if build_context.rails_root.join("actionview/Rakefile").read.include?("task :ujs")
diff --git a/test/buildkite_config/test_rake_command.rb b/test/buildkite_config/test_rake_command.rb
index 102ec17..1cea39c 100644
--- a/test/buildkite_config/test_rake_command.rb
+++ b/test/buildkite_config/test_rake_command.rb
@@ -328,7 +328,41 @@ def test_automatic_retry_on
     end
 
     assert_includes pipeline.to_h["steps"][0], "retry"
-    assert_equal({ "automatic" => [{ "limit" => 1, "exit_status" => 127 }] }, pipeline.to_h["steps"][0]["retry"])
+    assert_equal({ "automatic" => [{ "limit" => 1, "exit_status" => 127 }, { "limit" => 2, "exit_status" => -1 }] }, pipeline.to_h["steps"][0]["retry"])
+  end
+
+  def test_automatic_retry_on_default
+    pipeline = PipelineFixture.new do
+      build_context.ruby = Buildkite::Config::RubyConfig.new(version: Gem::Version.new("3.2"))
+      use Buildkite::Config::RakeCommand
+
+      build_context.stub(:rails_version, Gem::Version.new("7.1")) do
+        rake "test_automatic_retry_on"
+      end
+    end
+
+    assert_includes pipeline.to_h["steps"][0], "retry"
+    assert_equal({ "automatic" => [{ "limit" => 2, "exit_status" => -1 } ] }, pipeline.to_h["steps"][0]["retry"])
+  end
+
+  def test_automatic_retry_on_array
+    pipeline = PipelineFixture.new do
+      build_context.ruby = Buildkite::Config::RubyConfig.new(version: Gem::Version.new("3.2"))
+      use Buildkite::Config::RakeCommand
+
+      build_context.stub(:rails_version, Gem::Version.new("7.1")) do
+        rake "test_automatic_retry_on", retry_on: [{ limit: 1, exit_status: 127 }, { limit: 2, exit_status: 2 }, { limit: 3, exit_status: 3 }]
+      end
+    end
+
+    assert_includes pipeline.to_h["steps"][0], "retry"
+    assert_equal(
+      { "automatic" =>
+        [{ "limit" => 1, "exit_status" => 127 },
+         { "limit" => 2, "exit_status" => 2 },
+         { "limit" => 3, "exit_status" => 3 },
+         { "limit" => 2, "exit_status" => -1 }]
+      }, pipeline.to_h["steps"][0]["retry"])
   end
 
   def test_soft_fail
@@ -345,6 +379,34 @@ def test_soft_fail
     assert_equal true, pipeline.to_h["steps"][0]["soft_fail"]
   end
 
+  def test_soft_fail_hash
+    pipeline = PipelineFixture.new do
+      build_context.ruby = Buildkite::Config::RubyConfig.new(version: Gem::Version.new("3.2"))
+      use Buildkite::Config::RakeCommand
+
+      build_context.stub(:rails_version, Gem::Version.new("7.1")) do
+        rake "test_soft_fail", soft_fail: { exit_status: 127 }
+      end
+    end
+
+    assert_includes pipeline.to_h["steps"][0], "soft_fail"
+    assert_equal [{ "exit_status" => 127 }], pipeline.to_h["steps"][0]["soft_fail"]
+  end
+
+  def test_soft_fail_array
+    pipeline = PipelineFixture.new do
+      build_context.ruby = Buildkite::Config::RubyConfig.new(version: Gem::Version.new("3.2"))
+      use Buildkite::Config::RakeCommand
+
+      build_context.stub(:rails_version, Gem::Version.new("7.1")) do
+        rake "test_soft_fail", soft_fail: [{ exit_status: 127 }, { exit_status: 2 }]
+      end
+    end
+
+    assert_includes pipeline.to_h["steps"][0], "soft_fail"
+    assert_equal [{ "exit_status" => 127 }, { "exit_status" => 2 }], pipeline.to_h["steps"][0]["soft_fail"]
+  end
+
   def test_soft_fail_ruby
     pipeline = PipelineFixture.new do
       build_context.ruby = Buildkite::Config::RubyConfig.new(version: Gem::Version.new("3.3"), soft_fail: true)