Class: Raif::Generators::Evals::SetupGenerator

Inherits:
Rails::Generators::Base
  • Object
show all
Defined in:
lib/generators/raif/evals/setup/setup_generator.rb

Constant Summary collapse

RESULTS_GITIGNORE_PATH = 
"raif_evals/results/.gitignore"
RUN_LOG_IGNORE = 
<<~EOS
  # Eval run logs are transient: the run each belongs to either finishes and replaces it
  # with a results file, or gets resumed. Keep this even if you delete the `*` above to
  # put result files in version control.
  *.partial.jsonl
EOS

Instance Method Summary collapse

Instance Method Details

#create_directories ⇒ Object 



20
21
22
23
24
25
26
 
# File 'lib/generators/raif/evals/setup/setup_generator.rb', line 20

def create_directories
  empty_directory "raif_evals"
  empty_directory "raif_evals/eval_sets"
  empty_directory "raif_evals/files"
  empty_directory "raif_evals/datasets"
  empty_directory "raif_evals/results"
end

#create_gitignore ⇒ Object

Appends to an existing file rather than raising a conflict prompt, so an install made before the run log pattern existed picks it up on a re-run.



40
41
42
43
44
45
46
47
48
49
50
 
# File 'lib/generators/raif/evals/setup/setup_generator.rb', line 40

def create_gitignore
  existing_gitignore = File.join(destination_root, RESULTS_GITIGNORE_PATH)

  unless File.exist?(existing_gitignore)
    return create_file RESULTS_GITIGNORE_PATH, "*\n!.gitignore\n\n#{RUN_LOG_IGNORE}"
  end

  return if File.read(existing_gitignore).match?(/^\s*\*\.partial\.jsonl\s*$/)

  append_to_file RESULTS_GITIGNORE_PATH, "\n#{RUN_LOG_IGNORE}"
end

#create_setup_file ⇒ Object 



28
29
30
31
32
33
34
35
36
 
# File 'lib/generators/raif/evals/setup/setup_generator.rb', line 28

def create_setup_file
  create_file "raif_evals/setup.rb", <<~EOS
    #
    # This file is loaded at the start of a run of your evals.
    #
    # Add any setup code that should run before your evals.
    #
  EOS
end

#show_instructions ⇒ Object 



52
53
54
55
56
57
58
59
60
 
# File 'lib/generators/raif/evals/setup/setup_generator.rb', line 52

def show_instructions
  say "\nRaif evals setup complete!", :green
  say "You can create evals with: rails g raif:eval_set ExampleName"
  say ""
  say "Run evals with:"
  say "  bundle exec raif evals                                           # Run all evals"
  say "  bundle exec raif evals ./raif_evals/eval_sets/my_eval_set.rb     # Run specific eval set"
  say ""
end