1# 2# Ruby Benchmark driver 3# 4 5first = true 6 7begin 8 require 'optparse' 9rescue LoadError 10 if first 11 first = false 12 $:.unshift File.join(File.dirname(__FILE__), '../lib') 13 retry 14 else 15 raise 16 end 17end 18 19require 'benchmark' 20require 'pp' 21 22class BenchmarkDriver 23 def self.benchmark(opt) 24 driver = self.new(opt[:execs], opt[:dir], opt) 25 begin 26 driver.run 27 ensure 28 driver.show_results 29 end 30 end 31 32 def output *args 33 puts(*args) 34 @output and @output.puts(*args) 35 end 36 37 def message *args 38 output(*args) if @verbose 39 end 40 41 def message_print *args 42 if @verbose 43 print(*args) 44 STDOUT.flush 45 @output and @output.print(*args) 46 end 47 end 48 49 def progress_message *args 50 unless STDOUT.tty? 51 STDERR.print(*args) 52 STDERR.flush 53 end 54 end 55 56 def initialize execs, dir, opt = {} 57 @execs = execs.map{|e| 58 e.strip! 59 next if e.empty? 60 61 if /(.+)::(.+)/ =~ e 62 # ex) ruby-a::/path/to/ruby-a 63 label = $1.strip 64 path = $2 65 version = `#{path} -v`.chomp 66 else 67 path = e 68 version = label = `#{path} -v`.chomp 69 end 70 [path, label, version] 71 }.compact 72 73 @dir = dir 74 @repeat = opt[:repeat] || 1 75 @repeat = 1 if @repeat < 1 76 @pattern = opt[:pattern] || nil 77 @exclude = opt[:exclude] || nil 78 @verbose = opt[:quiet] ? false : (opt[:verbose] || false) 79 @output = opt[:output] ? open(opt[:output], 'w') : nil 80 @loop_wl1 = @loop_wl2 = nil 81 @ruby_arg = opt[:ruby_arg] || nil 82 @opt = opt 83 84 # [[name, [[r-1-1, r-1-2, ...], [r-2-1, r-2-2, ...]]], ...] 85 @results = [] 86 87 if @verbose 88 @start_time = Time.now 89 message @start_time 90 @execs.each_with_index{|(path, label, version), i| 91 message "target #{i}: " + (label == version ? "#{label}" : "#{label} (#{version})") + " at \"#{path}\"" 92 } 93 end 94 end 95 96 def adjusted_results name, results 97 s = nil 98 results.each_with_index{|e, i| 99 r = e.min 100 case name 101 when /^vm1_/ 102 if @loop_wl1 103 r -= @loop_wl1[i] 104 r = 0 if r < 0 105 s = '*' 106 end 107 when /^vm2_/ 108 if @loop_wl2 109 r -= @loop_wl2[i] 110 r = 0 if r < 0 111 s = '*' 112 end 113 end 114 yield r 115 } 116 s 117 end 118 119 def show_results 120 output 121 122 if @verbose 123 message '-----------------------------------------------------------' 124 message 'raw data:' 125 message 126 message PP.pp(@results, "", 79) 127 message 128 message "Elapsed time: #{Time.now - @start_time} (sec)" 129 end 130 131 output '-----------------------------------------------------------' 132 output 'benchmark results:' 133 134 if @verbose and @repeat > 1 135 output "minimum results in each #{@repeat} measurements." 136 end 137 138 output "Execution time (sec)" 139 output "name\t#{@execs.map{|(_, v)| v}.join("\t")}" 140 @results.each{|v, result| 141 rets = [] 142 s = adjusted_results(v, result){|r| 143 rets << sprintf("%.3f", r) 144 } 145 output "#{v}#{s}\t#{rets.join("\t")}" 146 } 147 148 if @execs.size > 1 149 output 150 output "Speedup ratio: compare with the result of `#{@execs[0][1]}' (greater is better)" 151 output "name\t#{@execs[1..-1].map{|(_, v)| v}.join("\t")}" 152 @results.each{|v, result| 153 rets = [] 154 first_value = nil 155 s = adjusted_results(v, result){|r| 156 if first_value 157 if r == 0 158 rets << "Error" 159 else 160 rets << sprintf("%.3f", first_value/r) 161 end 162 else 163 first_value = r 164 end 165 } 166 output "#{v}#{s}\t#{rets.join("\t")}" 167 } 168 end 169 170 if @opt[:output] 171 output 172 output "Log file: #{@opt[:output]}" 173 end 174 end 175 176 def files 177 flag = {} 178 @files = Dir.glob(File.join(@dir, 'bm*.rb')).map{|file| 179 next if @pattern && /#{@pattern}/ !~ File.basename(file) 180 next if @exclude && /#{@exclude}/ =~ File.basename(file) 181 case file 182 when /bm_(vm[12])_/, /bm_loop_(whileloop2?).rb/ 183 flag[$1] = true 184 end 185 file 186 }.compact 187 188 if flag['vm1'] && !flag['whileloop'] 189 @files << File.join(@dir, 'bm_loop_whileloop.rb') 190 elsif flag['vm2'] && !flag['whileloop2'] 191 @files << File.join(@dir, 'bm_loop_whileloop2.rb') 192 end 193 194 @files.sort! 195 progress_message "total: #{@files.size * @repeat} trial(s) (#{@repeat} trial(s) for #{@files.size} benchmark(s))\n" 196 @files 197 end 198 199 def run 200 files.each_with_index{|file, i| 201 @i = i 202 r = measure_file(file) 203 204 if /bm_loop_whileloop.rb/ =~ file 205 @loop_wl1 = r[1].map{|e| e.min} 206 elsif /bm_loop_whileloop2.rb/ =~ file 207 @loop_wl2 = r[1].map{|e| e.min} 208 end 209 } 210 end 211 212 def measure_file file 213 name = File.basename(file, '.rb').sub(/^bm_/, '') 214 prepare_file = File.join(File.dirname(file), "prepare_#{name}.rb") 215 load prepare_file if FileTest.exist?(prepare_file) 216 217 if @verbose 218 output 219 output '-----------------------------------------------------------' 220 output name 221 output 222 output File.read(file) 223 output 224 end 225 226 result = [name] 227 result << @execs.map{|(e, v)| 228 (0...@repeat).map{ 229 message_print "#{v}\t" 230 progress_message '.' 231 232 m = measure(e, file) 233 message "#{m}" 234 m 235 } 236 } 237 @results << result 238 result 239 end 240 241 def measure executable, file 242 cmd = "#{executable} #{@ruby_arg} #{file}" 243 244 m = Benchmark.measure{ 245 system(cmd, out: File::NULL) 246 } 247 248 if $? != 0 249 output "\`#{cmd}\' exited with abnormal status (#{$?})" 250 0 251 else 252 m.real 253 end 254 end 255end 256 257if __FILE__ == $0 258 opt = { 259 :execs => [], 260 :dir => File.dirname(__FILE__), 261 :repeat => 1, 262 :output => "bmlog-#{Time.now.strftime('%Y%m%d-%H%M%S')}.#{$$}", 263 } 264 265 parser = OptionParser.new{|o| 266 o.on('-e', '--executables [EXECS]', 267 "Specify benchmark one or more targets (e1::path1; e2::path2; e3::path3;...)"){|e| 268 e.split(/;/).each{|path| 269 opt[:execs] << path 270 } 271 } 272 o.on('-d', '--directory [DIRECTORY]', "Benchmark suites directory"){|d| 273 opt[:dir] = d 274 } 275 o.on('-p', '--pattern [PATTERN]', "Benchmark name pattern"){|p| 276 opt[:pattern] = p 277 } 278 o.on('-x', '--exclude [PATTERN]', "Benchmark exclude pattern"){|e| 279 opt[:exclude] = e 280 } 281 o.on('-r', '--repeat-count [NUM]', "Repeat count"){|n| 282 opt[:repeat] = n.to_i 283 } 284 o.on('-o', '--output-file [FILE]', "Output file"){|f| 285 opt[:output] = f 286 } 287 o.on('--ruby-arg [ARG]', "Optional argument for ruby"){|a| 288 opt[:ruby_arg] = a 289 } 290 o.on('-q', '--quiet', "Run without notify information except result table."){|q| 291 opt[:quiet] = q 292 } 293 o.on('-v', '--verbose'){|v| 294 opt[:verbose] = v 295 } 296 } 297 298 parser.parse!(ARGV) 299 BenchmarkDriver.benchmark(opt) 300end 301 302