1require 'rubygems' 2require 'rubygems/package' 3require 'time' 4 5begin 6 gem 'builder' 7 require 'builder/xchar' 8rescue LoadError 9end 10 11## 12# Top level class for building the gem repository index. 13 14class Gem::Indexer 15 16 include Gem::UserInteraction 17 18 ## 19 # Build indexes for RubyGems 1.2.0 and newer when true 20 21 attr_accessor :build_modern 22 23 ## 24 # Index install location 25 26 attr_reader :dest_directory 27 28 ## 29 # Specs index install location 30 31 attr_reader :dest_specs_index 32 33 ## 34 # Latest specs index install location 35 36 attr_reader :dest_latest_specs_index 37 38 ## 39 # Prerelease specs index install location 40 41 attr_reader :dest_prerelease_specs_index 42 43 ## 44 # Index build directory 45 46 attr_reader :directory 47 48 ## 49 # Create an indexer that will index the gems in +directory+. 50 51 def initialize(directory, options = {}) 52 require 'fileutils' 53 require 'tmpdir' 54 require 'zlib' 55 56 unless defined?(Builder::XChar) then 57 raise "Gem::Indexer requires that the XML Builder library be installed:" + 58 "\n\tgem install builder" 59 end 60 61 options = { :build_modern => true }.merge options 62 63 @build_modern = options[:build_modern] 64 65 @dest_directory = directory 66 @directory = File.join(Dir.tmpdir, "gem_generate_index_#{$$}") 67 68 marshal_name = "Marshal.#{Gem.marshal_version}" 69 70 @master_index = File.join @directory, 'yaml' 71 @marshal_index = File.join @directory, marshal_name 72 73 @quick_dir = File.join @directory, 'quick' 74 @quick_marshal_dir = File.join @quick_dir, marshal_name 75 @quick_marshal_dir_base = File.join "quick", marshal_name # FIX: UGH 76 77 @quick_index = File.join @quick_dir, 'index' 78 @latest_index = File.join @quick_dir, 'latest_index' 79 80 @specs_index = File.join @directory, "specs.#{Gem.marshal_version}" 81 @latest_specs_index = 82 File.join(@directory, "latest_specs.#{Gem.marshal_version}") 83 @prerelease_specs_index = 84 File.join(@directory, "prerelease_specs.#{Gem.marshal_version}") 85 @dest_specs_index = 86 File.join(@dest_directory, "specs.#{Gem.marshal_version}") 87 @dest_latest_specs_index = 88 File.join(@dest_directory, "latest_specs.#{Gem.marshal_version}") 89 @dest_prerelease_specs_index = 90 File.join(@dest_directory, "prerelease_specs.#{Gem.marshal_version}") 91 92 @files = [] 93 end 94 95 ## 96 # Abbreviate the spec for downloading. Abbreviated specs are only used for 97 # searching, downloading and related activities and do not need deployment 98 # specific information (e.g. list of files). So we abbreviate the spec, 99 # making it much smaller for quicker downloads. 100 #-- 101 # TODO move to Gem::Specification 102 103 def abbreviate(spec) 104 spec.files = [] 105 spec.test_files = [] 106 spec.rdoc_options = [] 107 spec.extra_rdoc_files = [] 108 spec.cert_chain = [] 109 spec 110 end 111 112 ## 113 # Build various indicies 114 115 def build_indicies 116 Gem::Specification.dirs = [] 117 Gem::Specification.add_specs(*map_gems_to_specs(gem_file_list)) 118 119 build_marshal_gemspecs 120 build_modern_indicies if @build_modern 121 122 compress_indicies 123 end 124 125 ## 126 # Builds Marshal quick index gemspecs. 127 128 def build_marshal_gemspecs 129 count = Gem::Specification.count 130 progress = ui.progress_reporter count, 131 "Generating Marshal quick index gemspecs for #{count} gems", 132 "Complete" 133 134 files = [] 135 136 Gem.time 'Generated Marshal quick index gemspecs' do 137 Gem::Specification.each do |spec| 138 spec_file_name = "#{spec.original_name}.gemspec.rz" 139 marshal_name = File.join @quick_marshal_dir, spec_file_name 140 141 marshal_zipped = Gem.deflate Marshal.dump(spec) 142 open marshal_name, 'wb' do |io| io.write marshal_zipped end 143 144 files << marshal_name 145 146 progress.updated spec.original_name 147 end 148 149 progress.done 150 end 151 152 @files << @quick_marshal_dir 153 154 files 155 end 156 157 ## 158 # Build a single index for RubyGems 1.2 and newer 159 160 def build_modern_index(index, file, name) 161 say "Generating #{name} index" 162 163 Gem.time "Generated #{name} index" do 164 open(file, 'wb') do |io| 165 specs = index.map do |*spec| 166 # We have to splat here because latest_specs is an array, while the 167 # others are hashes. 168 spec = spec.flatten.last 169 platform = spec.original_platform 170 171 # win32-api-1.0.4-x86-mswin32-60 172 unless String === platform then 173 alert_warning "Skipping invalid platform in gem: #{spec.full_name}" 174 next 175 end 176 177 platform = Gem::Platform::RUBY if platform.nil? or platform.empty? 178 [spec.name, spec.version, platform] 179 end 180 181 specs = compact_specs(specs) 182 Marshal.dump(specs, io) 183 end 184 end 185 end 186 187 ## 188 # Builds indicies for RubyGems 1.2 and newer. Handles full, latest, prerelease 189 190 def build_modern_indicies 191 prerelease, released = Gem::Specification.partition { |s| 192 s.version.prerelease? 193 } 194 latest_specs = Gem::Specification.latest_specs 195 196 build_modern_index(released.sort, @specs_index, 'specs') 197 build_modern_index(latest_specs.sort, @latest_specs_index, 'latest specs') 198 build_modern_index(prerelease.sort, @prerelease_specs_index, 199 'prerelease specs') 200 201 @files += [@specs_index, 202 "#{@specs_index}.gz", 203 @latest_specs_index, 204 "#{@latest_specs_index}.gz", 205 @prerelease_specs_index, 206 "#{@prerelease_specs_index}.gz"] 207 end 208 209 def map_gems_to_specs gems 210 gems.map { |gemfile| 211 if File.size(gemfile) == 0 then 212 alert_warning "Skipping zero-length gem: #{gemfile}" 213 next 214 end 215 216 begin 217 spec = Gem::Package.new(gemfile).spec 218 spec.loaded_from = gemfile 219 220 # HACK: fuck this shit - borks all tests that use pl1 221 # if File.basename(gemfile, ".gem") != spec.original_name then 222 # exp = spec.full_name 223 # exp << " (#{spec.original_name})" if 224 # spec.original_name != spec.full_name 225 # msg = "Skipping misnamed gem: #{gemfile} should be named #{exp}" 226 # alert_warning msg 227 # next 228 # end 229 230 abbreviate spec 231 sanitize spec 232 233 spec 234 rescue SignalException => e 235 alert_error "Received signal, exiting" 236 raise 237 rescue Exception => e 238 msg = ["Unable to process #{gemfile}", 239 "#{e.message} (#{e.class})", 240 "\t#{e.backtrace.join "\n\t"}"].join("\n") 241 alert_error msg 242 end 243 }.compact 244 end 245 246 ## 247 # Compresses indicies on disk 248 #-- 249 # All future files should be compressed using gzip, not deflate 250 251 def compress_indicies 252 say "Compressing indicies" 253 254 Gem.time 'Compressed indicies' do 255 if @build_modern then 256 gzip @specs_index 257 gzip @latest_specs_index 258 gzip @prerelease_specs_index 259 end 260 end 261 end 262 263 ## 264 # Compacts Marshal output for the specs index data source by using identical 265 # objects as much as possible. 266 267 def compact_specs(specs) 268 names = {} 269 versions = {} 270 platforms = {} 271 272 specs.map do |(name, version, platform)| 273 names[name] = name unless names.include? name 274 versions[version] = version unless versions.include? version 275 platforms[platform] = platform unless platforms.include? platform 276 277 [names[name], versions[version], platforms[platform]] 278 end 279 end 280 281 ## 282 # Compress +filename+ with +extension+. 283 284 def compress(filename, extension) 285 data = Gem.read_binary filename 286 287 zipped = Gem.deflate data 288 289 open "#{filename}.#{extension}", 'wb' do |io| 290 io.write zipped 291 end 292 end 293 294 ## 295 # List of gem file names to index. 296 297 def gem_file_list 298 Dir[File.join(@dest_directory, "gems", '*.gem')] 299 end 300 301 ## 302 # Builds and installs indicies. 303 304 def generate_index 305 make_temp_directories 306 build_indicies 307 install_indicies 308 rescue SignalException 309 ensure 310 FileUtils.rm_rf @directory 311 end 312 313 ## 314 # Zlib::GzipWriter wrapper that gzips +filename+ on disk. 315 316 def gzip(filename) 317 Zlib::GzipWriter.open "#{filename}.gz" do |io| 318 io.write Gem.read_binary(filename) 319 end 320 end 321 322 ## 323 # Install generated indicies into the destination directory. 324 325 def install_indicies 326 verbose = Gem.configuration.really_verbose 327 328 say "Moving index into production dir #{@dest_directory}" if verbose 329 330 files = @files 331 files.delete @quick_marshal_dir if files.include? @quick_dir 332 333 if files.include? @quick_marshal_dir and not files.include? @quick_dir then 334 files.delete @quick_marshal_dir 335 336 dst_name = File.join(@dest_directory, @quick_marshal_dir_base) 337 338 FileUtils.mkdir_p File.dirname(dst_name), :verbose => verbose 339 FileUtils.rm_rf dst_name, :verbose => verbose 340 FileUtils.mv(@quick_marshal_dir, dst_name, 341 :verbose => verbose, :force => true) 342 end 343 344 files = files.map do |path| 345 path.sub(/^#{Regexp.escape @directory}\/?/, '') # HACK? 346 end 347 348 files.each do |file| 349 src_name = File.join @directory, file 350 dst_name = File.join @dest_directory, file 351 352 FileUtils.rm_rf dst_name, :verbose => verbose 353 FileUtils.mv(src_name, @dest_directory, 354 :verbose => verbose, :force => true) 355 end 356 end 357 358 ## 359 # Make directories for index generation 360 361 def make_temp_directories 362 FileUtils.rm_rf @directory 363 FileUtils.mkdir_p @directory, :mode => 0700 364 FileUtils.mkdir_p @quick_marshal_dir 365 end 366 367 ## 368 # Ensure +path+ and path with +extension+ are identical. 369 370 def paranoid(path, extension) 371 data = Gem.read_binary path 372 compressed_data = Gem.read_binary "#{path}.#{extension}" 373 374 unless data == Gem.inflate(compressed_data) then 375 raise "Compressed file #{compressed_path} does not match uncompressed file #{path}" 376 end 377 end 378 379 ## 380 # Sanitize the descriptive fields in the spec. Sometimes non-ASCII 381 # characters will garble the site index. Non-ASCII characters will 382 # be replaced by their XML entity equivalent. 383 384 def sanitize(spec) 385 spec.summary = sanitize_string(spec.summary) 386 spec.description = sanitize_string(spec.description) 387 spec.post_install_message = sanitize_string(spec.post_install_message) 388 spec.authors = spec.authors.collect { |a| sanitize_string(a) } 389 390 spec 391 end 392 393 ## 394 # Sanitize a single string. 395 396 def sanitize_string(string) 397 return string unless string 398 399 # HACK the #to_s is in here because RSpec has an Array of Arrays of 400 # Strings for authors. Need a way to disallow bad values on gemspec 401 # generation. (Probably won't happen.) 402 string = string.to_s 403 404 begin 405 Builder::XChar.encode string 406 rescue NameError, NoMethodError 407 string.to_xs 408 end 409 end 410 411 ## 412 # Perform an in-place update of the repository from newly added gems. 413 414 def update_index 415 make_temp_directories 416 417 specs_mtime = File.stat(@dest_specs_index).mtime 418 newest_mtime = Time.at 0 419 420 updated_gems = gem_file_list.select do |gem| 421 gem_mtime = File.stat(gem).mtime 422 newest_mtime = gem_mtime if gem_mtime > newest_mtime 423 gem_mtime >= specs_mtime 424 end 425 426 if updated_gems.empty? then 427 say 'No new gems' 428 terminate_interaction 0 429 end 430 431 specs = map_gems_to_specs updated_gems 432 prerelease, released = specs.partition { |s| s.version.prerelease? } 433 434 Gem::Specification.dirs = [] 435 Gem::Specification.add_specs(*specs) 436 437 files = build_marshal_gemspecs 438 439 Gem.time 'Updated indexes' do 440 update_specs_index released, @dest_specs_index, @specs_index 441 update_specs_index released, @dest_latest_specs_index, @latest_specs_index 442 update_specs_index(prerelease, 443 @dest_prerelease_specs_index, 444 @prerelease_specs_index) 445 end 446 447 compress_indicies 448 449 verbose = Gem.configuration.really_verbose 450 451 say "Updating production dir #{@dest_directory}" if verbose 452 453 files << @specs_index 454 files << "#{@specs_index}.gz" 455 files << @latest_specs_index 456 files << "#{@latest_specs_index}.gz" 457 files << @prerelease_specs_index 458 files << "#{@prerelease_specs_index}.gz" 459 460 files = files.map do |path| 461 path.sub(/^#{Regexp.escape @directory}\/?/, '') # HACK? 462 end 463 464 files.each do |file| 465 src_name = File.join @directory, file 466 dst_name = File.join @dest_directory, file # REFACTOR: duped above 467 468 FileUtils.mv src_name, dst_name, :verbose => verbose, 469 :force => true 470 471 File.utime newest_mtime, newest_mtime, dst_name 472 end 473 end 474 475 ## 476 # Combines specs in +index+ and +source+ then writes out a new copy to 477 # +dest+. For a latest index, does not ensure the new file is minimal. 478 479 def update_specs_index(index, source, dest) 480 specs_index = Marshal.load Gem.read_binary(source) 481 482 index.each do |spec| 483 platform = spec.original_platform 484 platform = Gem::Platform::RUBY if platform.nil? or platform.empty? 485 specs_index << [spec.name, spec.version, platform] 486 end 487 488 specs_index = compact_specs specs_index.uniq.sort 489 490 open dest, 'wb' do |io| 491 Marshal.dump specs_index, io 492 end 493 end 494end 495