Java generator: call as_json only once for invalid strings

CHANGES.md

@@ -5,6 +5,7 @@
 * Fix `JSON::ParserError#json_path` raising `NoMethodError` when the document was parsed with `symbolize_names: true`.
 * Fix `JSON.generate` not detecting duplicate keys such as `{ "1" => 1, 1 => 2 }`.
 * JRuby: raise `TypeError` when a String subclass used as a Hash key has a `to_s` that doesn't return a String, like the C extension.
+* JRuby and TruffleRuby: fix a stack overflow in `JSON::Coder` when `as_json` returns a Symbol for a string with invalid encoding.
 
 ### 2026-10-13 (3.1.0)
 

java/src/json/ext/Generator.java

@@ -713,12 +713,21 @@ void generate(ThreadContext context, Session session, RubyString object, OutputS
 
             if (state.strict() && !encoder.hasValidEncoding(object) && state.getAsJSON() != null) {
                 IRubyObject value = state.getAsJSON().call(context, object, context.getRuntime().getFalse());
-                if (value instanceof RubyString) {
+                if (value != object) {
+                    // C: convert_invalid_encoding. as_json is only called once, so a new
+                    // string that is still invalid raises instead of calling it again.
+                    if (value instanceof RubySymbol) {
+                        value = ((RubySymbol) value).asString();
+                    }
+                    if (!(value instanceof RubyString)) {
+                        Handler handler = getHandlerFor(context.runtime, value);
+                        handler.generate(context, session, value, buffer);
+                        return;
+                    }
+                    if (!encoder.hasValidEncoding((RubyString) value)) {
+                        throw Utils.buildGeneratorError(context, object, "source sequence is illegal/malformed utf-8").toThrowable();
+                    }
                     object = (RubyString)value;
-                } else {
-                    Handler handler = getHandlerFor(context.runtime, value);
-                    handler.generate(context, session, value, buffer);
-                    return;
                 }
             }
             generateString(context, session, object, buffer);

lib/json/truffle_ruby/generator.rb

@@ -791,9 +791,18 @@ def to_json(state = nil, *args)
 
             if state.strict? && state.as_json
               unless Generator.valid_encoding?(string)
-                string = state.as_json.call(string, false)
-                unless string.is_a?(::String)
-                  return string.to_json(state, *args)
+                coerced = state.as_json.call(string, false)
+                unless coerced.equal?(string)
+                  # as_json is only called once, so a new string that is still
+                  # invalid raises instead of calling it again.
+                  coerced = coerced.name if coerced.is_a?(::Symbol)
+                  unless coerced.is_a?(::String)
+                    return coerced.to_json(state, *args)
+                  end
+                  unless Generator.valid_encoding?(coerced)
+                    raise GeneratorError.new("source sequence is illegal/malformed utf-8", self)
+                  end
+                  string = coerced
                 end
               end
             end

test/json/json_coder_test.rb

@@ -238,13 +238,11 @@ def test_json_coder_symbol_invalid_encoding
     coder = JSON::Coder.new { |object, is_key| Object.new }
     assert_raise(JSON::GeneratorError) { coder.dump([symbol]) }
 
-    if RUBY_ENGINE == "ruby"
-      coder = JSON::Coder.new { |object, is_key| symbol }
-      error = assert_raise JSON::GeneratorError do
-        coder.dump([symbol])
-      end
-      assert_equal "source sequence is illegal/malformed utf-8", error.message
+    coder = JSON::Coder.new { |object, is_key| symbol }
+    error = assert_raise JSON::GeneratorError do
+      coder.dump([symbol])
     end
+    assert_equal "source sequence is illegal/malformed utf-8", error.message
   end
 
   def test_depth

test/json/json_parser_test.rb

@@ -481,11 +481,9 @@ def test_parse_duplicate_key
     assert_raise(ParserError) { parse('{"a": 1, "a": 2}', allow_duplicate_key: false) }
     assert_raise(ParserError) { parse('{"a": 1, "a": 2}', allow_duplicate_key: false, symbolize_names: true) }
 
-    unless RUBY_ENGINE == 'jruby'
-      assert_raise(ParserError) do
-        fake_key = Object.new
-        JSON.load('{"a": 1, "a": 2}', -> (obj) { obj == "a" ? fake_key : obj }, allow_duplicate_key: false)
-      end
+    assert_raise(ParserError) do
+      fake_key = Object.new
+      JSON.load('{"a": 1, "a": 2}', -> (obj) { obj == "a" ? fake_key : obj }, allow_duplicate_key: false)
     end
   end