Class: Raif::CLI::Evals

Inherits:
Base
  • Object
show all
Defined in:
lib/raif/cli/evals.rb

Instance Attribute Summary

Attributes inherited from Base

#args, #options

Instance Method Summary collapse

Methods inherited from Base

#initialize

Constructor Details

This class inherits a constructor from Raif::CLI::Base

Instance Method Details

#run ⇒ Object 



9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
 
# File 'lib/raif/cli/evals.rb', line 9

def run
  ENV["RAILS_ENV"] ||= "test"
  ENV["RAIF_RUNNING_EVALS"] = "true"

  repeats = ENV.fetch("RAIF_EVAL_REPEATS", 1).to_i
  cases = ENV["RAIF_EVAL_CASES"].to_s.split(",").map(&:strip).reject(&:empty?)
  sample = ENV["RAIF_EVAL_SAMPLE"]&.to_i
  seed = ENV["RAIF_EVAL_SEED"]&.to_i
  resume_path = ENV["RAIF_EVAL_RESUME"].to_s.empty? ? nil : ENV["RAIF_EVAL_RESUME"]
  # nil leaves Raif.config.evals_concurrency alone, so an app that sets it in its
  # initializer does not have to pass the flag on every run.
  concurrency = ENV["RAIF_EVAL_CONCURRENCY"].to_s.empty? ? nil : ENV["RAIF_EVAL_CONCURRENCY"].to_i
  # nil leaves Raif.config.evals_verbose_output alone. --no-verbose has to be able to
  # win, so an app whose initializer turns verbose output on can still get back to the
  # compact dataset output.
  verbose = case ENV["RAIF_EVAL_VERBOSE"]
  when nil, "" then nil
  when "0", "false" then false
  else true
  end
  # nil leaves Raif.config.evals_live_report alone, for the same reason as verbose above.
  live_report = case ENV["RAIF_EVAL_LIVE_REPORT"]
  when nil, "" then nil
  when "0", "false" then false
  else true
  end
  open_live_report = case ENV["RAIF_EVAL_OPEN_LIVE_REPORT"]
  when nil, "" then nil
  when "0", "false" then false
  else true
  end

  parser = OptionParser.new do |opts|
    opts.banner = "Usage: raif evals [options] [FILE_PATHS]"

    opts.on("-e", "--environment ENV", "Rails environment (default: test)") do |env|
      ENV["RAILS_ENV"] = env
    end

    opts.on("-r", "--repeat N", Integer, "Run each eval N times and report a pass rate (default: 1)") do |n|
      repeats = n
    end

    opts.on("--concurrency N", Integer, "Run N evals at once (default: Raif.config.evals_concurrency)") do |n|
      concurrency = n
    end

    opts.on("--cases a,b,c", Array, "Run only these dataset cases") do |ids|
      cases = ids.map(&:strip).reject(&:empty?)
    end

    opts.on("--sample N", Integer, "Run a random N cases from each dataset") do |n|
      sample = n
    end

    opts.on("--seed N", Integer, "Seed for --sample, so the same cases can be drawn again") do |n|
      seed = n
    end

    opts.on("--[no-]verbose", "Print every expectation for every dataset case (default: Raif.config.evals_verbose_output)") do |value|
      verbose = value
    end

    opts.on("--[no-]live-report", "Write an HTML page that shows the run's progress (default: Raif.config.evals_live_report)") do |value|
      live_report = value
    end

    opts.on("--[no-]open-live-report", "Open the live report in a browser at start (default: Raif.config.evals_open_live_report)") do |value|
      open_live_report = value
    end

    opts.on("--resume PATH", "Resume an interrupted run from its .partial.jsonl log, skipping results it already holds") do |path|
      resume_path = path
    end

    opts.on("-h", "--help", "Show this help message") do
      puts opts
      exit
    end
  end

  parse_options!(parser)

  # A sample of zero or fewer cases is a typo, not a selection: zero would run nothing
  # and report an empty suite as a pass, and a negative count raises in Array#take.
  if sample && sample < 1
    puts "Error: --sample must be 1 or greater (got #{sample})"
    exit 1
  end

  if concurrency && concurrency < 1
    puts "Error: --concurrency must be 1 or greater (got #{concurrency})"
    exit 1
  end

  # Before Rails boots, which is the slowest thing this command does and entirely wasted
  # on a mistyped log path.
  if resume_path && !File.exist?(resume_path)
    puts "Error: Resume log not found: #{resume_path}"
    exit 1
  end

  # Parse file paths with optional line numbers
  file_paths = args.map do |arg|
    if arg.include?(":")
      file_path, line_number = arg.split(":", 2)
      { file_path: file_path, line_number: line_number.to_i }
    else
      { file_path: arg, line_number: nil }
    end
  end if args.any?

  # Find and load Rails application
  load_rails_application

  require "raif/evals"

  Raif.config.evals_verbose_output = verbose unless verbose.nil?
  Raif.config.evals_live_report = live_report unless live_report.nil?
  Raif.config.evals_open_live_report = open_live_report unless open_live_report.nil?

  run = Raif::Evals::Run.new(
    file_paths: file_paths,
    repeats: repeats,
    cases: (cases if cases.any?),
    sample: sample,
    seed: seed,
    resume_path: resume_path,
    concurrency: concurrency
  )
  run.execute
end