Class: Raif::TokenEstimator

Inherits:
Object
  • Object
show all
Defined in:
lib/raif/token_estimator.rb

Class Method Summary collapse

Class Method Details

.available? ⇒ Boolean

Returns:

  • (Boolean) 


5
6
7
8
9
10
11
12
 
# File 'lib/raif/token_estimator.rb', line 5

def self.available?
  return true if defined?(::Tiktoken)

  require "tiktoken_ruby"
  !!defined?(::Tiktoken)
rescue LoadError
  false
end

.encoder_for_model(model) ⇒ Object 



23
24
25
26
 
# File 'lib/raif/token_estimator.rb', line 23

def self.encoder_for_model(model)
  @encoders ||= {}
  @encoders[model] ||= ::Tiktoken.encoding_for_model(model)
end

.estimate_tokens(*texts) ⇒ Object

Estimates the total token count for a prompt + system prompt combination. Returns nil if tiktoken_ruby is not installed.



16
17
18
19
20
21
 
# File 'lib/raif/token_estimator.rb', line 16

def self.estimate_tokens(*texts)
  return unless available?

  encoder = encoder_for_model("gpt-4")
  texts.compact.sum { |text| encoder.encode(text).length }
end