1#
2# Ruby Benchmark driver
3#
4
5first = true
6
7begin
8  require 'optparse'
9rescue LoadError
10  if first
11    first = false
12    $:.unshift File.join(File.dirname(__FILE__), '../lib')
13    retry
14  else
15    raise
16  end
17end
18
19require 'benchmark'
20require 'pp'
21
22class BenchmarkDriver
23  def self.benchmark(opt)
24    driver = self.new(opt[:execs], opt[:dir], opt)
25    begin
26      driver.run
27    ensure
28      driver.show_results
29    end
30  end
31
32  def output *args
33    puts(*args)
34    @output and @output.puts(*args)
35  end
36
37  def message *args
38    output(*args) if @verbose
39  end
40
41  def message_print *args
42    if @verbose
43      print(*args)
44      STDOUT.flush
45      @output and @output.print(*args)
46    end
47  end
48
49  def progress_message *args
50    unless STDOUT.tty?
51      STDERR.print(*args)
52      STDERR.flush
53    end
54  end
55
56  def initialize execs, dir, opt = {}
57    @execs = execs.map{|e|
58      e.strip!
59      next if e.empty?
60
61      if /(.+)::(.+)/ =~ e
62        # ex) ruby-a::/path/to/ruby-a
63        label = $1.strip
64        path = $2
65        version = `#{path} -v`.chomp
66      else
67        path = e
68        version = label = `#{path} -v`.chomp
69      end
70      [path, label, version]
71    }.compact
72
73    @dir = dir
74    @repeat = opt[:repeat] || 1
75    @repeat = 1 if @repeat < 1
76    @pattern = opt[:pattern] || nil
77    @exclude = opt[:exclude] || nil
78    @verbose = opt[:quiet] ? false : (opt[:verbose] || false)
79    @output = opt[:output] ? open(opt[:output], 'w') : nil
80    @loop_wl1 = @loop_wl2 = nil
81    @ruby_arg = opt[:ruby_arg] || nil
82    @opt = opt
83
84    # [[name, [[r-1-1, r-1-2, ...], [r-2-1, r-2-2, ...]]], ...]
85    @results = []
86
87    if @verbose
88      @start_time = Time.now
89      message @start_time
90      @execs.each_with_index{|(path, label, version), i|
91        message "target #{i}: " + (label == version ? "#{label}" : "#{label} (#{version})") + " at \"#{path}\""
92      }
93    end
94  end
95
96  def adjusted_results name, results
97    s = nil
98    results.each_with_index{|e, i|
99      r = e.min
100      case name
101      when /^vm1_/
102        if @loop_wl1
103          r -= @loop_wl1[i]
104          r = 0 if r < 0
105          s = '*'
106        end
107      when /^vm2_/
108        if @loop_wl2
109          r -= @loop_wl2[i]
110          r = 0 if r < 0
111          s = '*'
112        end
113      end
114      yield r
115    }
116    s
117  end
118
119  def show_results
120    output
121
122    if @verbose
123      message '-----------------------------------------------------------'
124      message 'raw data:'
125      message
126      message PP.pp(@results, "", 79)
127      message
128      message "Elapsed time: #{Time.now - @start_time} (sec)"
129    end
130
131    output '-----------------------------------------------------------'
132    output 'benchmark results:'
133
134    if @verbose and @repeat > 1
135      output "minimum results in each #{@repeat} measurements."
136    end
137
138    output "Execution time (sec)"
139    output "name\t#{@execs.map{|(_, v)| v}.join("\t")}"
140    @results.each{|v, result|
141      rets = []
142      s = adjusted_results(v, result){|r|
143        rets << sprintf("%.3f", r)
144      }
145      output "#{v}#{s}\t#{rets.join("\t")}"
146    }
147
148    if @execs.size > 1
149      output
150      output "Speedup ratio: compare with the result of `#{@execs[0][1]}' (greater is better)"
151      output "name\t#{@execs[1..-1].map{|(_, v)| v}.join("\t")}"
152      @results.each{|v, result|
153        rets = []
154        first_value = nil
155        s = adjusted_results(v, result){|r|
156          if first_value
157            if r == 0
158              rets << "Error"
159            else
160              rets << sprintf("%.3f", first_value/r)
161            end
162          else
163            first_value = r
164          end
165        }
166        output "#{v}#{s}\t#{rets.join("\t")}"
167      }
168    end
169
170    if @opt[:output]
171      output
172      output "Log file: #{@opt[:output]}"
173    end
174  end
175
176  def files
177    flag = {}
178    @files = Dir.glob(File.join(@dir, 'bm*.rb')).map{|file|
179      next if @pattern && /#{@pattern}/ !~ File.basename(file)
180      next if @exclude && /#{@exclude}/ =~ File.basename(file)
181      case file
182      when /bm_(vm[12])_/, /bm_loop_(whileloop2?).rb/
183        flag[$1] = true
184      end
185      file
186    }.compact
187
188    if flag['vm1'] && !flag['whileloop']
189      @files << File.join(@dir, 'bm_loop_whileloop.rb')
190    elsif flag['vm2'] && !flag['whileloop2']
191      @files << File.join(@dir, 'bm_loop_whileloop2.rb')
192    end
193
194    @files.sort!
195    progress_message "total: #{@files.size * @repeat} trial(s) (#{@repeat} trial(s) for #{@files.size} benchmark(s))\n"
196    @files
197  end
198
199  def run
200    files.each_with_index{|file, i|
201      @i = i
202      r = measure_file(file)
203
204      if /bm_loop_whileloop.rb/ =~ file
205        @loop_wl1 = r[1].map{|e| e.min}
206      elsif /bm_loop_whileloop2.rb/ =~ file
207        @loop_wl2 = r[1].map{|e| e.min}
208      end
209    }
210  end
211
212  def measure_file file
213    name = File.basename(file, '.rb').sub(/^bm_/, '')
214    prepare_file = File.join(File.dirname(file), "prepare_#{name}.rb")
215    load prepare_file if FileTest.exist?(prepare_file)
216
217    if @verbose
218      output
219      output '-----------------------------------------------------------'
220      output name
221      output
222      output File.read(file)
223      output
224    end
225
226    result = [name]
227    result << @execs.map{|(e, v)|
228      (0...@repeat).map{
229        message_print "#{v}\t"
230        progress_message '.'
231
232        m = measure(e, file)
233        message "#{m}"
234        m
235      }
236    }
237    @results << result
238    result
239  end
240
241  def measure executable, file
242    cmd = "#{executable} #{@ruby_arg} #{file}"
243
244    m = Benchmark.measure{
245      system(cmd, out: File::NULL)
246    }
247
248    if $? != 0
249      output "\`#{cmd}\' exited with abnormal status (#{$?})"
250      0
251    else
252      m.real
253    end
254  end
255end
256
257if __FILE__ == $0
258  opt = {
259    :execs => [],
260    :dir => File.dirname(__FILE__),
261    :repeat => 1,
262    :output => "bmlog-#{Time.now.strftime('%Y%m%d-%H%M%S')}.#{$$}",
263  }
264
265  parser = OptionParser.new{|o|
266    o.on('-e', '--executables [EXECS]',
267      "Specify benchmark one or more targets (e1::path1; e2::path2; e3::path3;...)"){|e|
268       e.split(/;/).each{|path|
269         opt[:execs] << path
270       }
271    }
272    o.on('-d', '--directory [DIRECTORY]', "Benchmark suites directory"){|d|
273      opt[:dir] = d
274    }
275    o.on('-p', '--pattern [PATTERN]', "Benchmark name pattern"){|p|
276      opt[:pattern] = p
277    }
278    o.on('-x', '--exclude [PATTERN]', "Benchmark exclude pattern"){|e|
279      opt[:exclude] = e
280    }
281    o.on('-r', '--repeat-count [NUM]', "Repeat count"){|n|
282      opt[:repeat] = n.to_i
283    }
284    o.on('-o', '--output-file [FILE]', "Output file"){|f|
285      opt[:output] = f
286    }
287    o.on('--ruby-arg [ARG]', "Optional argument for ruby"){|a|
288      opt[:ruby_arg] = a
289    }
290    o.on('-q', '--quiet', "Run without notify information except result table."){|q|
291      opt[:quiet] = q
292    }
293    o.on('-v', '--verbose'){|v|
294      opt[:verbose] = v
295    }
296  }
297
298  parser.parse!(ARGV)
299  BenchmarkDriver.benchmark(opt)
300end
301
302