diff --git a/.duel.rb b/.duel.rb new file mode 100644 index 00000000..84f8d3f0 --- /dev/null +++ b/.duel.rb @@ -0,0 +1,43 @@ +# frozen_string_literal: true + +# Duel's configuration: https://github.com/sferik/duel-ruby/blob/main/docs/configuration.md +# +# Duel follows conventions, so set only what differs: benchmarks are in +# duel/**/*_duel.rb, each version's lib/ is first on the load path, a +# change is compared when it touches lib/ or ext/ (or the benchmarks, or +# this file), and releases are tagged v*. +Duel.configure do |c| + # The library each version's workers require (from the gemspec, unless + # this says otherwise). + # c.library = "json" + + # Builds each version's C extension, so that each is measured with its + # own build. + c.build = "bundle exec rake compile" + + # The documents the benchmarks parse and generate (duel/duel_helper.rb + # reads them from benchmark/data) decide what's measured too. + c.watch += %w[benchmark/data] + + # When the workflow comments on a pull request with the report: :always, + # :results (passed or failed), :failures (failed, or couldn't compare), + # or :never. + # c.comment = :failures + + # Which runs a regression fails: :every_run, or :pull_requests, for a + # project where people push to main directly (on GitHub Actions, another + # run warns and passes, and a pushed commit gets the comment instead, + # from a workflow job written only where this is set: see docs/ci.md). + # c.fail_on = :every_run + + # A label that accepts a regression in a pull request, for a change + # that's slower on purpose (nil for none). + c.accept_label = "duel: accept" + + # What the workflow compares a push with: :before (the commit before it), + # or :last_passed (the last commit Duel passed, which a branch, + # duel/passed/main, marks), so small slowdowns add up until they fail. + # (A workflow job moves the branch, written only where this is set: see + # docs/ci.md.) + # c.push_base = :before +end diff --git a/.github/workflows/perf-comment.yml b/.github/workflows/perf-comment.yml new file mode 100644 index 00000000..5a144853 --- /dev/null +++ b/.github/workflows/perf-comment.yml @@ -0,0 +1,75 @@ +name: Perf comment + +# Duel's report, as a new comment on the pull request, when Duel says to: +# always (post), or only when its last comment's status differs from the +# run's (change), so the latest comment is never stale. It never edits or +# deletes a comment. It runs once the Perf workflow has, with this +# repository's own workflow and a token that can write, so a pull request +# from a fork gets a comment too. It checks nothing out and runs none of +# the pull request's code, and of what the Duel job kept (where that code +# ran), it trusts only what that job could have said: the pull request's +# number, if it names the pull request the run compared, and the comment +# and the status, if they're ones Duel says. A fork's report, which the +# fork's code wrote, is shown as text. (The workflow must be on the +# default branch before it runs at all.) +on: + workflow_run: + workflows: ["Perf"] + types: [completed] + +permissions: {} + +jobs: + comment: + name: Comment + if: ${{ github.event.workflow_run.event == 'pull_request' && github.event.workflow_run.conclusion != 'cancelled' }} + runs-on: ubuntu-latest + permissions: + actions: read # to get the report from the Perf run + pull-requests: write # to comment on the pull request with the report + steps: + - name: Get the report + id: report + continue-on-error: true # there's none when the Duel job failed before comparing + uses: actions/download-artifact@v8 + with: + name: duel-report + path: ${{ runner.temp }}/report + run-id: ${{ github.event.workflow_run.id }} + github-token: ${{ github.token }} + - name: Comment on the pull request + if: steps.report.outcome == 'success' + env: + GH_TOKEN: ${{ github.token }} + HEAD_SHA: ${{ github.event.workflow_run.head_sha }} + run: | + said() { grep -xE "$1=($2)" "$RUNNER_TEMP/report/comment.env" | tail -n 1 | cut -d= -f2; } + PR=$(said pr '[0-9]+') + COMMENT=$(said comment 'post|change') + STATUS=$(said status 'passed|failed|neutral') + [ -n "$PR" ] && [ -n "$COMMENT" ] && [ -n "$STATUS" ] || exit 0 + # (The pull request's head commit, and the repository it's from.) + head=$(gh api "repos/$GITHUB_REPOSITORY/pulls/$PR" --jq '"\(.head.sha) \(.head.repo.full_name)"') + [ "${head%% *}" = "$HEAD_SHA" ] || exit 0 + # The report, as Markdown for a branch of this repository (whose + # author can push here anyway), but for a fork's, as text: its run + # ran the fork's code, which could have written anything in it. In + # a code block, fenced with more backticks than any run of them in + # the report, it can't mention anyone, link anywhere or show an + # image. (And it's cut short where a comment couldn't hold it.) + report() { + if [ "${head#* }" = "$GITHUB_REPOSITORY" ]; then cat "$RUNNER_TEMP/report/duel.md" && return; fi + fence=$(grep -o '``*' "$RUNNER_TEMP/report/duel.md" | awk '{ if (length > n) n = length } END { n = (n < 3) ? 3 : n + 1; while (n-- > 0) printf "`" }') + printf 'From a fork, so shown as text:\n\n%s\n' "$fence" + head -c 60000 "$RUNNER_TEMP/report/duel.md" + printf '\n%s\n' "$fence" + } + marker="\")" | tail -n 1) + if [ "$COMMENT" = post ] || { [ -n "$last" ] && [ "$last" != "$marker status=$STATUS -->" ]; }; then + { echo "$marker status=$STATUS -->"; report; } > "$RUNNER_TEMP/comment.md" + gh api --method POST "repos/$GITHUB_REPOSITORY/issues/$PR/comments" -F body=@"$RUNNER_TEMP/comment.md" > /dev/null + fi diff --git a/.github/workflows/perf.yml b/.github/workflows/perf.yml new file mode 100644 index 00000000..a3de6b30 --- /dev/null +++ b/.github/workflows/perf.yml @@ -0,0 +1,115 @@ +name: Perf + +on: + pull_request: + merge_group: # (a merge queue's) + push: + branches: [master] + schedule: + - cron: "0 4 * * *" + workflow_dispatch: + +# A pull request's newer push cancels its run, if that's still going, as +# only its newest commit's result counts. Every other run (a push's, say) +# runs to the end, so that each push is compared. +concurrency: + group: ${{ github.workflow }}-${{ github.event_name == 'pull_request' && github.event.pull_request.number || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +# The Duel job runs the branch's code (its benchmarks too), so it only +# reads. Commenting on a commit is the Comment on the commit job's, which +# runs none of it, and commenting on a pull request is the Perf comment +# workflow's. +permissions: + contents: read + pull-requests: read # for the pull request's labels (see c.accept_label) + +jobs: + duel: + name: Duel + runs-on: ubuntu-latest + env: + BUNDLE_WITHOUT: release + # Whether to comment (post, change or none), the run's status (passed, + # failed or neutral), and whether to mark a push passed (true, or + # nothing), from Duel. + outputs: + comment: ${{ steps.compare.outputs.comment }} + status: ${{ steps.compare.outputs.status }} + passed: ${{ steps.compare.outputs.passed }} + steps: + - uses: actions/checkout@v7 + with: + fetch-depth: 0 # the baseline comes from history, tags included + persist-credentials: false + - name: Set up Ruby + uses: ruby/setup-ruby@v1 + with: + ruby-version: "4.0" # the newest Ruby, which those who care most about speed run + bundler-cache: true + # The labels of the pull request (or for a push, of the one that + # merged it), for c.accept_label in .duel.rb, as a JSON array. They're + # read as the job runs, so after adding the label, run it again. + - name: Find the pull request's labels + id: labels + if: github.event_name == 'pull_request' || github.event_name == 'merge_group' || github.event_name == 'push' + continue-on-error: true # (without them, nothing's accepted) + env: + GH_TOKEN: ${{ github.token }} + PR: ${{ github.event.pull_request.number }} + QUEUED: ${{ github.event.merge_group.head_ref }} + run: | + # (In a merge queue, the last pull request queued, which the queue's + # branch names: gh-readonly-queue/master/pr-123-.) + if [ -n "$QUEUED" ]; then + PR=${QUEUED##*/pr-} + PR=${PR%%-*} + fi + if [ -n "$PR" ]; then + labels=$(gh api "repos/$GITHUB_REPOSITORY/issues/$PR/labels" --jq '[.[].name]') + else + labels=$(gh api "repos/$GITHUB_REPOSITORY/commits/$GITHUB_SHA/pulls" --jq '[.[0].labels[]?.name]') + fi + echo "labels=$labels" >> "$GITHUB_OUTPUT" + # Duel itself requires json as it starts, which the bundle finds here, + # in this checkout, so its extensions are built first. (Duel then + # builds each version's own: c.build in .duel.rb.) + - name: Compile the extensions + run: bundle exec rake compile + # Against the branch it targets for a pull request, the queue's base + # for a merge queue, what was there before for a push (or as + # c.push_base in .duel.rb says, the last commit Duel passed), and the + # last release on a schedule (slow creep). Duel compiles each + # version's extensions (c.build in .duel.rb), adds its report to + # the job's summary, writes it to duel.md for the comment, and sets + # the step's outputs to say whether to comment, as c.comment in + # .duel.rb says, whatever the outcome. + - name: Compare + id: compare + env: + DUEL_BASE: ${{ github.event_name == 'schedule' && 'last-release' || github.event_name == 'push' && github.event.before || github.event_name == 'merge_group' && github.event.merge_group.base_sha || '' }} + # (Run by hand, it compares anyway: with nothing changed, the code + # with itself, which checks the setup.) + DUEL_FORCE: ${{ github.event_name == 'workflow_dispatch' && '1' || '' }} + DUEL_LABELS: ${{ steps.labels.outputs.labels }} + run: bundle exec duel --markdown "$RUNNER_TEMP/duel/duel.md" + # For a pull request, what Duel said, for the Perf comment workflow, + # which can't see this job's outputs. + - name: Say whether to comment + if: ${{ !cancelled() && github.event_name == 'pull_request' }} + env: + PR: ${{ github.event.pull_request.number }} + COMMENT: ${{ steps.compare.outputs.comment }} + STATUS: ${{ steps.compare.outputs.status }} + run: | + mkdir -p "$RUNNER_TEMP/duel" + printf 'pr=%s\ncomment=%s\nstatus=%s\n' "$PR" "$COMMENT" "$STATUS" > "$RUNNER_TEMP/duel/comment.env" + # The report, for the comments. + - name: Keep the report + if: ${{ !cancelled() && (github.event_name == 'pull_request' || github.event_name == 'push') }} + uses: actions/upload-artifact@v7 + with: + name: duel-report + path: ${{ runner.temp }}/duel/ + if-no-files-found: ignore # none when Duel didn't get to compare + retention-days: 1 diff --git a/Gemfile b/Gemfile index 407a3f9b..f1895db9 100644 --- a/Gemfile +++ b/Gemfile @@ -11,6 +11,10 @@ group :development do gem "test-unit-ruby-core" gem "simplecov", require: false + # Duel requires Ruby 3.4 or later + gem "duel", "~> 0.1.5" if RUBY_VERSION >= "3.4" + gem "stackprof", platforms: :mri + if RUBY_VERSION < "3.1." # Ref: https://github.com/test-unit/test-unit/pull/329 gem 'power_assert', '< 3.0.0' diff --git a/duel/duel_helper.rb b/duel/duel_helper.rb new file mode 100644 index 00000000..43097000 --- /dev/null +++ b/duel/duel_helper.rb @@ -0,0 +1,70 @@ +# frozen_string_literal: true + +# The documents in benchmark/data, the same ones that benchmark/parser.rb +# and benchmark/encoder.rb compare against other parsers with. +def fixture(name) + File.read(File.expand_path("../benchmark/data/#{name}", __dir__), encoding: Encoding::UTF_8) +end + +# The first +count+ points of canada.json's polygon, as a document of its +# own, with the numbers exactly as the file writes them (most with 17 +# significant digits). The whole file takes several milliseconds to +# generate, too long for one iteration. +def canada_points(count) + points = fixture("canada.json").scan(/\[-?\d+(?:\.\d+)?,-?\d+(?:\.\d+)?\]/).first(count) + %({"type":"Polygon","coordinates":[[#{points.join(',')}]]}) +end + +# A typical response from a JSON API, about 1 KB: a record with a few +# nested records, short and escaped strings, integers, floats, booleans +# and a null. Written out, so every version parses the same bytes. +API_RESPONSE = <<~'JSON' + {"id":1234567,"login":"octocat","name":"The Octocat","email":null,"site_admin":false, + "score":98.6,"created_at":"2026-01-21T08:41:44.286Z","tags":["ruby","json","api"], + "plan":{"name":"pro","space":976562499,"private_repos":9999,"collaborators":0}, + "repos":[ + {"id":1000,"name":"repo-0","fork":false,"stars":0,"price":19.99,"description":"A \"quoted\" description.\nLine two."}, + {"id":1001,"name":"repo-1","fork":true,"stars":42,"price":4.5,"description":"Caf\u00e9 au lait, served in cafés."}, + {"id":1002,"name":"repo-2","fork":false,"stars":84,"price":0.99,"description":"Tabs\tand slashes: C:\\Users\\octocat"}, + {"id":1003,"name":"repo-3","fork":true,"stars":126,"price":120.0,"description":"Plain ASCII text, nothing to escape here."}, + {"id":1004,"name":"repo-4","fork":false,"stars":168,"price":1e-3,"description":"Emoji: \ud83d\ude00 and ☃"} + ]} +JSON + +# The small payloads that benchmark/parser.rb and benchmark/encoder.rb use. +SMALL_HASH = { "username" => "jhawthorn", "id" => 123, "event" => "wrote json serializer" }.freeze +SMALL_MIXED = [1, "string", { a: 1, b: 2 }, [3, 4, 5]].freeze + +# Prices and the like: +count+ decimals with up to two places, the same +# every time. +def short_decimals(count) + random = Random.new(42) + Array.new(count) { random.rand(1..100_000) / 100.0 } +end + +# Strings as they come from users: quotes, newlines, tabs, backslashes, +# a control character, and non-ASCII text, which a generator escapes. +def strings_to_escape(count) + Array.new(count) do |i| + "Line #{i}: \"quoted\"\n\tC:\\Users\\octocat \u0001 caf\u00e9 \u{1F600}" + end +end + +# A value that generates by calling its own #to_json. +class Event + def initialize(id, name) + @id = id + @name = name + end + + def to_json(*args) + { "id" => @id, "name" => @name }.to_json(*args) + end +end + +# A value without #to_json of its own, which generates as its #to_s. +class Tag + def to_s + "tag" + end +end diff --git a/duel/generator_duel.rb b/duel/generator_duel.rb new file mode 100644 index 00000000..e94fe74f --- /dev/null +++ b/duel/generator_duel.rb @@ -0,0 +1,124 @@ +# frozen_string_literal: true + +require "date" +require "stringio" +require_relative "duel_helper" + +# Generating, through JSON.generate and the other ways in: the documents +# from benchmark/encoder.rb, pretty and not, small payloads where the +# per-call overhead counts, and the string escaping and number +# formatting paths on their own, with the options that change them. + +duel "generate: twitter.json" do + once { @object = JSON.parse(fixture("twitter.json")) } + JSON.generate(@object) +end + +duel "generate: twitter.json pretty" do + once { @object = JSON.parse(fixture("twitter.json")) } + JSON.pretty_generate(@object) +end + +duel "generate: citm_catalog.json" do + once { @object = JSON.parse(fixture("citm_catalog.json")) } + JSON.generate(@object) +end + +duel "generate: activitypub.json" do + once { @object = JSON.parse(fixture("activitypub.json")) } + JSON.generate(@object) +end + +duel "generate: 10,000 points from canada.json (floats)" do + once { @object = JSON.parse(canada_points(10_000)) } + JSON.generate(@object) +end + +duel "generate: a small hash" do + JSON.generate(SMALL_HASH) +end + +duel "generate: a small array with symbol keys" do + JSON.generate(SMALL_MIXED) +end + +duel "generate: a 1 KB API response" do + once { @object = JSON.parse(API_RESPONSE) } + JSON.generate(@object) +end + +duel "generate: a 1 KB API response pretty" do + once { @object = JSON.parse(API_RESPONSE) } + JSON.pretty_generate(@object) +end + +duel "generate: a 1 KB API response with Hash#to_json" do + once { @object = JSON.parse(API_RESPONSE) } + @object.to_json +end + +duel "generate: a 1 KB API response with JSON::Coder" do + once do + @object = JSON.parse(API_RESPONSE) + @coder = JSON::Coder.new + end + @coder.dump(@object) +end + +duel "generate: a 1 KB API response into an IO with JSON.dump" do + once do + @object = JSON.parse(API_RESPONSE) + @io = StringIO.new(+"") + end + @io.truncate(0) + @io.rewind + JSON.dump(@object, @io) +end + +duel "generate: 1,000 integers" do + once { @object = (1_000_000..1_000_999).to_a } + JSON.generate(@object) +end + +duel "generate: 1,000 short decimals" do + once { @object = short_decimals(1000) } + JSON.generate(@object) +end + +duel "generate: 100 strings with escapes" do + once { @object = strings_to_escape(100) } + JSON.generate(@object) +end + +duel "generate: 10 strings of 4 KB, ASCII but for one character" do + once { @object = Array.new(10) { ("a" * 2000) + "€" + ("a" * 2000) } } + JSON.generate(@object) +end + +duel "generate: 10 strings of 4 KB, all multibyte" do + once { @object = Array.new(10) { "€" * 1333 } } + JSON.generate(@object) +end + +duel "generate: 100 strings with script_safe" do + once { @object = Array.new(100) { |i| " \u2028 https://example.com/a/b" } } + JSON.generate(@object, script_safe: true) +end + +duel "generate: 100 strings with ascii_only" do + once { @object = strings_to_escape(100) } + JSON.generate(@object, ascii_only: true) +end + +# Objects that aren't JSON types: with a #to_json of their own, or +# generated as their #to_s (Time, Date, a class), as benchmark/encoder.rb's +# "many #to_json calls" does, with fixed values. +duel "generate: objects through #to_json and #to_s" do + once do + @object = Array.new(20) do |i| + { event: Event.new(i, "signup"), tag: Tag.new, int: 12, float: 54.3, class: Float, + time: Time.utc(2026, 1, 21, 8, 41, 44), date: Date.new(2026, 1, 21) } + end + end + JSON.generate(@object) +end diff --git a/duel/parser_duel.rb b/duel/parser_duel.rb new file mode 100644 index 00000000..6360374f --- /dev/null +++ b/duel/parser_duel.rb @@ -0,0 +1,103 @@ +# frozen_string_literal: true + +require_relative "duel_helper" + +# Parsing, through JSON.parse, as most callers do: the documents from +# benchmark/parser.rb, the options that change the parser's hot loop, +# small payloads where the per-call overhead counts, and the string and +# number paths on their own. + +duel "parse: twitter.json" do + once { @source = fixture("twitter.json") } + JSON.parse(@source) +end + +duel "parse: twitter.json with symbolize_names" do + once { @source = fixture("twitter.json") } + JSON.parse(@source, symbolize_names: true) +end + +duel "parse: twitter.json with freeze" do + once { @source = fixture("twitter.json") } + JSON.parse(@source, freeze: true) +end + +duel "parse: citm_catalog.json" do + once { @source = fixture("citm_catalog.json") } + JSON.parse(@source) +end + +duel "parse: canada.json (floats)" do + once { @source = fixture("canada.json") } + JSON.parse(@source) +end + +duel "parse: activitypub.json" do + once { @source = fixture("activitypub.json") } + JSON.parse(@source) +end + +duel "parse: a small hash" do + once { @source = '{"username":"jhawthorn","id":123,"event":"wrote json serializer"}' } + JSON.parse(@source) +end + +duel "parse: a 1 KB API response" do + JSON.parse(API_RESPONSE) +end + +duel "parse: a 1 KB API response with symbolize_names" do + JSON.parse(API_RESPONSE, symbolize_names: true) +end + +duel "parse: a 1 KB API response with JSON::Coder" do + once { @coder = JSON::Coder.new } + @coder.load(API_RESPONSE) +end + +duel "parse: the nested document from oj's tests" do + once do + @source = <<~'JSON' + {"a":"Alpha","b":true,"c":12345,"d":[true,[false,[-123456789,null],3.9676,["Something else.",false],null]], + "e":{"zero":null,"one":1,"two":2,"three":[3],"four":[0,1,2,3,4]},"f":null, + "h":{"a":{"b":{"c":{"d":{"e":{"f":{"g":null}}}}}}},"i":[[[[[[[null]]]]]]]} + JSON + end + JSON.parse(@source) +end + +duel "parse: 1,000 integers" do + once { @source = "[#{(1_000_000..1_000_999).to_a.join(',')}]" } + JSON.parse(@source) +end + +duel "parse: 1,000 short decimals" do + once { @source = "[#{short_decimals(1000).join(',')}]" } + JSON.parse(@source) +end + +duel "parse: 100 strings with escapes" do + once do + @source = "[#{Array.new(100) { |i| %("Line #{i}: \\"quoted\\"\\n\\tC:\\\\Users\\\\octocat \\u0001 caf\\u00e9 \\ud83d\\ude00") }.join(',')}]" + end + JSON.parse(@source) +end + +duel "parse: 100 ASCII strings of 1 KB" do + once { @source = "[#{Array.new(100) { |i| %("#{i} #{'The quick brown fox jumps over the lazy dog. ' * 22}") }.join(',')}]" } + JSON.parse(@source) +end + +# A stream of documents, one per line, fed to a ResumableParser in 4 KB +# chunks of bytes, as a socket reads them. +duel "parse: a stream of 50 API responses with ResumableParser" do + once do + stream = "#{API_RESPONSE.delete("\n")}\n" * 50 + @chunks = stream.b.scan(/.{1,4096}/mn) + end + parser = JSON::ResumableParser.new + @chunks.each do |chunk| + parser << chunk + parser.value while parser.parse + end +end