-
Notifications
You must be signed in to change notification settings - Fork 10
Expand file tree
/
Copy pathRakefile
More file actions
374 lines (318 loc) · 10.7 KB
/
Copy pathRakefile
File metadata and controls
374 lines (318 loc) · 10.7 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
# frozen_string_literal: true
require "rake/testtask"
desc "Run tests (optionally with seed: rake test[12345])"
task :test, [:seed] do |t, args|
seed_opt = args[:seed] ? " -- --seed=#{args[:seed]}" : ""
sh "ruby -Ilib:test -e \"Dir.glob('test/**/*_test.rb').each { |f| require_relative f }\"#{seed_opt}"
end
desc "Run Standard linter"
task :lint do
sh "bundle exec standardrb"
end
desc "Run linter with auto-fix"
task :"lint:fix" do
sh "bundle exec standardrb --fix"
end
desc "Remove Ruby build artifacts"
task :clean do
FileUtils.rm_rf("pkg")
FileUtils.rm_rf("coverage")
FileUtils.rm_rf("doc")
FileUtils.rm_rf(".yardoc")
FileUtils.rm_rf("tmp")
FileUtils.rm_f(Dir.glob("*.gem"))
FileUtils.rm_f("changelog.md")
end
def appraisal_for(example)
case example
when /ruby_llm/ then "ruby_llm"
when /ruby-openai/, /ruby_openai/, /alexrudall/ then "ruby-openai"
when /anthropic/ then "anthropic"
when /openai/, /kitchen-sink/ then "openai"
end
end
def run_example(example)
appraisal = appraisal_for(example)
prefix = appraisal ? "bundle exec appraisal #{appraisal}" : "bundle exec"
sh "#{prefix} ruby #{example}"
end
desc "Run a single example with the correct gemfile"
task :example, [:path] do |t, args|
example = args[:path]
raise "Usage: rake example[path/to/example.rb]" unless example
puts "Running #{example}..."
run_example(example)
end
desc "Run all examples"
task :examples do
examples = FileList["examples/**/*.rb"].exclude("examples/**/README.md")
puts "Running #{examples.length} examples..."
examples.each do |example|
puts "\n=== Running #{example} ==="
run_example(example)
end
end
desc "Build the gem"
task build: [:clean] do
# Output to pkg/ (Bundler convention). The release workflow's build-and-ship step
# globs pkg/*.gem and requires exactly one match, so the gem must land there —
# not the repo root.
require_relative "lib/braintrust/version"
sh "gem build braintrust.gemspec"
mkdir_p "pkg"
mv "braintrust-#{Braintrust::VERSION}.gem", "pkg/"
end
desc "Generate YARD documentation"
task :yard do
sh "bundle exec yard doc"
end
desc "Run tests and open coverage report"
task coverage: :test do
coverage_file = "coverage/index.html"
case RbConfig::CONFIG["host_os"]
when /darwin/i
sh "open #{coverage_file}"
when /linux/i
sh "xdg-open #{coverage_file}"
when /mswin|mingw|cygwin/i
sh "start #{coverage_file}"
else
puts "Coverage report available at: #{File.expand_path(coverage_file)}"
end
end
desc "Verify CI (lint + test all appraisal scenarios)"
task ci: [:lint, :"test:appraisal"]
task default: :ci
# Test-related tasks
namespace :test do
desc "Run only contrib framework tests"
task :contrib do
sh "bundle exec ruby -Ilib:test -e \"Dir.glob('test/braintrust/contrib{_test.rb,/**/*_test.rb}').each { |f| require_relative f }\""
end
namespace :contrib do
# Tasks per integration
[
{name: :anthropic},
{name: :openai},
{name: :ruby_llm},
{name: :ruby_openai, appraisal: "ruby-openai"}
].each do |integration|
name = integration[:name]
appraisal = integration[:appraisal] || integration[:name]
folder = integration[:name]
desc "Run #{name} contrib tests"
task name do
sh "bundle exec appraisal #{appraisal} ruby -Ilib:test -e \"Dir.glob('test/braintrust/contrib/#{folder}/**/*_test.rb').each { |f| require_relative f }\""
end
end
end
desc "Run tests with verbose timing output"
task :verbose do
ENV["MT_VERBOSE"] = "1"
Rake::Task[:test].invoke
end
desc "Install optional test dependencies (e.g., openai gem)"
task :install do
puts "Installing optional test dependencies..."
sh "gem install openai -v '~> 0.34'"
puts "✓ Optional dependencies installed"
puts ""
puts "Now run 'rake test' to run tests with OpenAI integration"
end
# Appraisal tasks for testing with/without optional dependencies
# Run directly: bundle exec appraisal [scenario] rake test
# List scenarios: bundle exec appraisal list
desc "Run tests against different dependencies (with install)"
task appraisal: :"appraisal:install" do
sh "bundle exec appraisal rake test"
end
desc "Run tests against all dependency scenarios (without install)"
task :all do
sh "bundle exec appraisal rake test"
end
namespace :appraisal do
desc "Show help for appraisal scenarios and usage"
task :help do
puts "\n=== Appraisal Test Scenarios ==="
puts "\nAvailable scenarios:"
sh "bundle exec appraisal list"
puts "\n=== Usage ==="
puts "Run specific scenario:"
puts " bundle exec appraisal <scenario> rake test"
puts ""
puts "Example:"
puts " bundle exec appraisal openai-0.34 rake test"
puts ""
puts "Run all scenarios:"
puts " bundle exec appraisal rake test"
puts " or: rake test:appraisal"
puts ""
end
desc "Install all appraisal gemfiles"
task :install do
sh "bundle exec appraisal install"
end
end
# VCR tasks for managing HTTP cassettes
namespace :vcr do
desc "Re-record all VCR cassettes"
task :record_all do
ENV["VCR_MODE"] = "all"
Rake::Task["test"].invoke
end
desc "Record new VCR cassettes only"
task :record_new do
ENV["VCR_MODE"] = "new_episodes"
Rake::Task["test"].invoke
end
desc "Run tests without VCR"
task :off do
ENV["VCR_OFF"] = "true"
Rake::Task["test"].invoke
end
end
end
# Contrib tasks
namespace :contrib do
desc "Generate a new integration (NAME=name [GEM_NAMES=gem1,gem2] [REQUIRE_PATHS=path1,path2] [MIN_VERSION=1.0.0] [MAX_VERSION=2.0.0] [AUTO_REGISTER=true])"
task :generate do
require "erb"
require "fileutils"
# Parse parameters
name = ENV["NAME"]
unless name
puts "Error: NAME is required"
puts "Usage: rake contrib:generate NAME=trustybrain_llm [GEM_NAMES=trustybrain_llm] [AUTO_REGISTER=true]"
exit 1
end
# Convert name to snake_case if it's PascalCase
snake_case_name = name.gsub(/([A-Z]+)([A-Z][a-z])/, '\1_\2')
.gsub(/([a-z\d])([A-Z])/, '\1_\2')
.downcase
# Convert to PascalCase for module name
module_name = snake_case_name.split("_").map(&:capitalize).join
integration_name = snake_case_name.to_sym
# Parse optional parameters
gem_names = ENV["GEM_NAMES"]&.split(",") || [snake_case_name]
require_paths = ENV["REQUIRE_PATHS"]&.split(",") || gem_names
min_version = ENV["MIN_VERSION"]
max_version = ENV["MAX_VERSION"]
auto_register = ENV.fetch("AUTO_REGISTER", "false").downcase == "true"
# Display what will be generated
puts "\n=== Generating Integration ==="
puts "Name: #{module_name}"
puts "Integration name: :#{integration_name}"
puts "Gem names: #{gem_names.inspect}"
puts "Require paths: #{require_paths.inspect}" if require_paths != gem_names
puts "Min version: #{min_version}" if min_version
puts "Max version: #{max_version}" if max_version
puts
# Template binding
template_binding = binding
# Paths
integration_dir = "lib/braintrust/contrib/#{snake_case_name}"
test_dir = "test/braintrust/contrib/#{snake_case_name}"
# Create directories
FileUtils.mkdir_p(integration_dir)
FileUtils.mkdir_p(test_dir)
# Generate files
templates = {
"templates/contrib/integration.rb.erb" => "#{integration_dir}/integration.rb",
"templates/contrib/patcher.rb.erb" => "#{integration_dir}/patcher.rb",
"templates/contrib/integration_test.rb.erb" => "#{test_dir}/integration_test.rb",
"templates/contrib/patcher_test.rb.erb" => "#{test_dir}/patcher_test.rb"
}
templates.each do |template_path, output_path|
template = ERB.new(File.read(template_path), trim_mode: "-")
content = template.result(template_binding)
File.write(output_path, content)
puts "✓ Created #{output_path}"
end
# Auto-register if requested
if auto_register
contrib_file = "lib/braintrust/contrib.rb"
contrib_content = File.read(contrib_file)
# Find the position to insert (before the last "end" or after the last require)
insertion_point = if /^# Load integration stubs/.match?(contrib_content)
contrib_content.index("# Load integration stubs")
else
# Insert before the final module end
contrib_content.rindex("end")
end
require_line = "require_relative \"contrib/#{snake_case_name}/integration\""
register_line = "Contrib::#{module_name}::Integration.register!"
# Check if already registered
if contrib_content.include?(require_line)
puts "⚠ #{contrib_file} already contains this integration"
else
lines_to_add = [
"",
"# #{module_name}",
require_line,
register_line
].join("\n")
contrib_content.insert(insertion_point, lines_to_add + "\n")
File.write(contrib_file, contrib_content)
puts "✓ Updated #{contrib_file}"
end
end
# Display next steps
puts "\n=== Next Steps ==="
unless auto_register
puts "1. Add to lib/braintrust/contrib.rb:"
puts " require_relative \"contrib/#{snake_case_name}/integration\""
puts " Contrib::#{module_name}::Integration.register!"
puts
end
puts "#{auto_register ? "1" : "2"}. Implement the patcher in:"
puts " #{integration_dir}/patcher.rb"
puts
puts "#{auto_register ? "2" : "3"}. Add tests in:"
puts " #{test_dir}/"
puts
puts "#{auto_register ? "3" : "4"}. Run tests:"
puts " bundle exec rake test TEST=#{test_dir}/**/*_test.rb"
puts
end
end
# Version bump tasks
def bump_version(type)
version_file = "lib/braintrust/version.rb"
content = File.read(version_file)
current = content.match(/VERSION = "(\d+)\.(\d+)\.(\d+)"/)
raise "Could not parse version from #{version_file}" unless current
major, minor, patch = current[1].to_i, current[2].to_i, current[3].to_i
old_version = "#{major}.#{minor}.#{patch}"
case type
when :major
major += 1
minor = 0
patch = 0
when :minor
minor += 1
patch = 0
when :patch
patch += 1
end
new_version = "#{major}.#{minor}.#{patch}"
new_content = content.gsub(/VERSION = "#{old_version}"/, "VERSION = \"#{new_version}\"")
File.write(version_file, new_content)
puts "#{old_version} → #{new_version}"
new_version
end
namespace :version do
namespace :bump do
desc "Bump patch version (0.0.5 → 0.0.6)"
task :patch do
bump_version(:patch)
end
desc "Bump minor version (0.0.5 → 0.1.0)"
task :minor do
bump_version(:minor)
end
desc "Bump major version (0.0.5 → 1.0.0)"
task :major do
bump_version(:major)
end
end
end