reduce-headers revision 1.1.1.1 
1#! /usr/bin/python2
2import os.path
3import sys
4import shlex
5import re
6import tempfile
7import copy
8
9from headerutils import *
10
11requires = { }
12provides = { }
13
14no_remove = [ "system.h", "coretypes.h", "config.h" , "bconfig.h", "backend.h" ]
15
16# These targets are the ones which provide "coverage".  Typically, if any
17# target is going to fail compilation, it's one of these.  This was determined
18# during the initial runs of reduce-headers... On a full set of target builds,
19# every failure which occured was triggered by one of these.  
20# This list is used during target-list construction simply to put any of these
21# *first* in the candidate list, increasing the probability that a failure is 
22# found quickly.
23target_priority = [
24    "aarch64-linux-gnu",
25    "arm-netbsdelf",
26    "avr-rtems",
27    "c6x-elf",
28    "epiphany-elf",
29    "hppa2.0-hpux10.1",
30    "i686-mingw32crt",
31    "i686-pc-msdosdjgpp",
32    "mipsel-elf",
33    "powerpc-eabisimaltivec",
34    "rs6000-ibm-aix5.1.0",
35    "sh-superh-elf",
36    "sparc64-elf",
37    "spu-elf"
38]
39
40
41target_dir = ""
42build_dir = ""
43ignore_list = list()
44target_builds = list()
45
46target_dict = { }
47header_dict = { }
48search_path = [ ".", "../include", "../libcpp/include" ]
49
50remove_count = { }
51
52
53# Given a header name, normalize it.  ie.  cp/cp-tree.h could be in gcc, while
54# the same header could be referenced from within the cp subdirectory as
55# just cp-tree.h
56# for now, just assume basenames are unique
57
58def normalize_header (header):
59  return os.path.basename (header)
60
61
62# Adds a header file and its sub-includes to the global dictionary if they
63# aren't already there.  Specify s_path since different build directories may
64# append themselves on demand to the global list.
65# return entry for the specified header, knowing all sub entries are completed
66
67def get_header_info (header, s_path):
68  global header_dict
69  global empty_iinfo
70  process_list = list ()
71  location = ""
72  bname = ""
73  bname_iinfo = empty_iinfo
74  for path in s_path:
75    if os.path.exists (path + "/" + header):
76      location = path + "/" + header
77      break
78
79  if location:
80    bname = normalize_header (location)
81    if header_dict.get (bname):
82      bname_iinfo = header_dict[bname]
83      loc2 = ii_path (bname_iinfo)+ "/" + bname
84      if loc2[:2] == "./":
85        loc2 = loc2[2:]
86      if location[:2] == "./":
87        location = location[2:]
88      if loc2 != location:
89        # Don't use the cache if it isnt the right one.
90        bname_iinfo = process_ii_macro (location)
91      return bname_iinfo
92
93    bname_iinfo = process_ii_macro (location)
94    header_dict[bname] = bname_iinfo
95    # now decend into the include tree
96    for i in ii_include_list (bname_iinfo):
97      get_header_info (i, s_path)
98  else:
99    # if the file isnt in the source directories, look in the build and target
100    # directories. If it is here, then aggregate all the versions.
101    location = build_dir + "/gcc/" + header
102    build_inc = target_inc = False
103    if os.path.exists (location):
104      build_inc = True
105    for x in target_dict:
106      location = target_dict[x] + "/gcc/" + header
107      if os.path.exists (location):
108        target_inc = True
109        break
110
111    if (build_inc or target_inc):
112      bname = normalize_header(header)
113      defines = set()
114      consumes = set()
115      incl = set()
116      if build_inc:
117        iinfo = process_ii_macro (build_dir + "/gcc/" + header)
118        defines = set (ii_macro_define (iinfo))
119        consumes = set (ii_macro_consume (iinfo))
120        incl = set (ii_include_list (iinfo))
121
122      if (target_inc):
123        for x in target_dict:
124          location = target_dict[x] + "/gcc/" + header
125          if os.path.exists (location):
126            iinfo = process_ii_macro (location)
127            defines.update (ii_macro_define (iinfo))
128            consumes.update (ii_macro_consume (iinfo))
129            incl.update (ii_include_list (iinfo))
130
131      bname_iinfo = (header, "build", list(incl), list(), list(consumes), list(defines), list(), list())
132
133      header_dict[bname] = bname_iinfo
134      for i in incl:
135        get_header_info (i, s_path)
136
137  return bname_iinfo
138
139
140# return a list of all headers brought in by this header
141def all_headers (fname):
142  global header_dict
143  headers_stack = list()
144  headers_list = list()
145  if header_dict.get (fname) == None:
146    return list ()
147  for y in ii_include_list (header_dict[fname]):
148    headers_stack.append (y)
149
150  while headers_stack:
151    h = headers_stack.pop ()
152    hn = normalize_header (h)
153    if hn not in headers_list:
154      headers_list.append (hn)
155      if header_dict.get(hn):
156        for y in ii_include_list (header_dict[hn]):
157          if normalize_header (y) not in headers_list:
158            headers_stack.append (y)
159
160  return headers_list
161
162
163
164
165# Search bld_dir for all target tuples, confirm that they have a build path with
166# bld_dir/target-tuple/gcc, and build a dictionary of build paths indexed by
167# target tuple..
168
169def build_target_dict (bld_dir, just_these):
170  global target_dict
171  target_doct = { }
172  error = False
173  if os.path.exists (bld_dir):
174    if just_these:
175      ls = just_these
176    else:
177      ls = os.listdir(bld_dir)
178    for t in ls:
179      if t.find("-") != -1:
180        target = t.strip()
181        tpath = bld_dir + "/" + target
182        if not os.path.exists (tpath + "/gcc"):
183          print "Error: gcc build directory for target " + t + " Does not exist: " + tpath + "/gcc"
184          error = True
185        else:
186          target_dict[target] = tpath
187
188  if error:
189    target_dict = { }
190
191def get_obj_name (src_file):
192  if src_file[-2:] == ".c":
193    return src_file.replace (".c", ".o")
194  elif src_file[-3:] == ".cc":
195    return src_file.replace (".cc", ".o")
196  return ""
197
198def target_obj_exists (target, obj_name):
199  global target_dict
200  # look in a subdir if src has a subdir, then check gcc base directory.
201  if target_dict.get(target):
202    obj = target_dict[target] + "/gcc/" + obj_name
203    if not os.path.exists (obj):
204      obj = target_dict[target] + "/gcc/" + os.path.basename(obj_name)
205    if os.path.exists (obj):
206      return True
207  return False
208 
209# Given a src file, return a list of targets which may build this file.
210def find_targets (src_file):
211  global target_dict
212  targ_list = list()
213  obj_name = get_obj_name (src_file)
214  if not obj_name:
215    print "Error: " + src_file + " - Cannot determine object name."
216    return list()
217
218  # Put the high priority targets which tend to trigger failures first
219  for target in target_priority:
220    if target_obj_exists (target, obj_name):
221      targ_list.append ((target, target_dict[target]))
222
223  for target in target_dict:
224    if target not in target_priority and target_obj_exists (target, obj_name):
225      targ_list.append ((target, target_dict[target]))
226        
227  return targ_list
228
229
230def try_to_remove (src_file, h_list, verbose):
231  global target_dict
232  global header_dict
233  global build_dir
234
235  # build from scratch each time
236  header_dict = { }
237  summary = ""
238  rmcount = 0
239
240  because = { }
241  src_info = process_ii_macro_src (src_file)
242  src_data = ii_src (src_info)
243  if src_data:
244    inclist = ii_include_list_non_cond (src_info)
245    # work is done if there are no includes to check
246    if not inclist:
247      return src_file + ": No include files to attempt to remove"
248
249    # work on the include list in reverse.
250    inclist.reverse()
251
252    # Get the target list 
253    targ_list = list()
254    targ_list = find_targets (src_file)
255
256    spath = search_path
257    if os.path.dirname (src_file):
258      spath.append (os.path.dirname (src_file))
259
260    hostbuild = True
261    if src_file.find("config/") != -1:
262      # config files dont usually build on the host
263      hostbuild = False
264      obn = get_obj_name (os.path.basename (src_file))
265      if obn and os.path.exists (build_dir + "/gcc/" + obn):
266        hostbuild = True
267      if not target_dict:
268        summary = src_file + ": Target builds are required for config files.  None found."
269        print summary
270        return summary
271      if not targ_list:
272        summary =src_file + ": Cannot find any targets which build this file."
273        print summary
274        return summary
275
276    if hostbuild:
277      # confirm it actually builds before we do anything
278      print "Confirming source file builds"
279      res = get_make_output (build_dir + "/gcc", "all")
280      if res[0] != 0:
281        message = "Error: " + src_file + " does not build currently."
282        summary = src_file + " does not build on host."
283        print message
284        print res[1]
285        if verbose:
286          verbose.write (message + "\n")
287          verbose.write (res[1]+ "\n")
288        return summary
289
290    src_requires = set (ii_macro_consume (src_info))
291    for macro in src_requires:
292      because[macro] = src_file
293    header_seen = list ()
294
295    os.rename (src_file, src_file + ".bak")
296    src_orig = copy.deepcopy (src_data)
297    src_tmp = copy.deepcopy (src_data)
298
299    try:
300      # process the includes from bottom to top.  This is because we know that
301      # later includes have are known to be needed, so any dependency from this 
302      # header is a true dependency
303      for inc_file in inclist:
304        inc_file_norm = normalize_header (inc_file)
305        
306        if inc_file in no_remove:
307          continue
308        if len (h_list) != 0 and inc_file_norm not in h_list:
309          continue
310        if inc_file_norm[0:3] == "gt-":
311          continue
312        if inc_file_norm[0:6] == "gtype-":
313          continue
314        if inc_file_norm.replace(".h",".c") == os.path.basename(src_file):
315          continue
316             
317        lookfor = ii_src_line(src_info)[inc_file]
318        src_tmp.remove (lookfor)
319        message = "Trying " + src_file + " without " + inc_file
320        print message
321        if verbose:
322          verbose.write (message + "\n")
323        out = open(src_file, "w")
324        for line in src_tmp:
325          out.write (line)
326        out.close()
327          
328        keep = False
329        if hostbuild:
330          res = get_make_output (build_dir + "/gcc", "all")
331        else:
332          res = (0, "")
333
334        rc = res[0]
335        message = "Passed Host build"
336        if (rc != 0):
337          # host build failed
338          message  = "Compilation failed:\n";
339          keep = True
340        else:
341          if targ_list:
342            objfile = get_obj_name (src_file)
343            t1 = targ_list[0]
344            if objfile and os.path.exists(t1[1] +"/gcc/"+objfile):
345              res = get_make_output_parallel (targ_list, objfile, 0)
346            else:
347              res = get_make_output_parallel (targ_list, "all-gcc", 0)
348            rc = res[0]
349            if rc != 0:
350              message = "Compilation failed on TARGET : " + res[2]
351              keep = True
352            else:
353              message = "Passed host and target builds"
354
355        if keep:
356          print message + "\n"
357
358        if (rc != 0):
359          if verbose:
360            verbose.write (message + "\n");
361            verbose.write (res[1])
362            verbose.write ("\n");
363            if os.path.exists (inc_file):
364              ilog = open(inc_file+".log","a")
365              ilog.write (message + " for " + src_file + ":\n\n");
366              ilog.write ("============================================\n");
367              ilog.write (res[1])
368              ilog.write ("\n");
369              ilog.close()
370            if os.path.exists (src_file):
371              ilog = open(src_file+".log","a")
372              ilog.write (message + " for " +inc_file + ":\n\n");
373              ilog.write ("============================================\n");
374              ilog.write (res[1])
375              ilog.write ("\n");
376              ilog.close()
377
378        # Given a sequence where :
379        # #include "tm.h"
380        # #include "target.h"  // includes tm.h
381
382        # target.h was required, and when attempting to remove tm.h we'd see that
383        # all the macro defintions are "required" since they all look like:
384        # #ifndef HAVE_blah
385        # #define HAVE_blah
386        # endif
387
388        # when target.h was found to be required, tm.h will be tagged as included.
389        # so when we get this far, we know we dont have to check the macros for
390        # tm.h since we know it is already been included.
391
392        if inc_file_norm not in header_seen:
393          iinfo = get_header_info (inc_file, spath)
394          newlist = all_headers (inc_file_norm)
395          if ii_path(iinfo) == "build" and not target_dict:
396            keep = True
397            text = message + " : Will not remove a build file without some targets."
398            print text
399            ilog = open(src_file+".log","a")
400            ilog.write (text +"\n")
401            ilog.write ("============================================\n");
402            ilog.close()
403            ilog = open("reduce-headers-kept.log","a")
404            ilog.write (src_file + " " + text +"\n")
405            ilog.close()
406        else:
407          newlist = list()
408        if not keep and inc_file_norm not in header_seen:
409          # now look for any macro requirements.
410          for h in newlist:
411            if not h in header_seen:
412              if header_dict.get(h):
413                defined = ii_macro_define (header_dict[h])
414                for dep in defined:
415                  if dep in src_requires and dep not in ignore_list:
416                    keep = True;
417                    text = message + ", but must keep " + inc_file + " because it provides " + dep 
418                    if because.get(dep) != None:
419                      text = text + " Possibly required by " + because[dep]
420                    print text
421                    ilog = open(inc_file+".log","a")
422                    ilog.write (because[dep]+": Requires [dep] in "+src_file+"\n")
423                    ilog.write ("============================================\n");
424                    ilog.close()
425                    ilog = open(src_file+".log","a")
426                    ilog.write (text +"\n")
427                    ilog.write ("============================================\n");
428                    ilog.close()
429                    ilog = open("reduce-headers-kept.log","a")
430                    ilog.write (src_file + " " + text +"\n")
431                    ilog.close()
432                    if verbose:
433                      verbose.write (text + "\n")
434
435        if keep:
436          # add all headers 'consumes' to src_requires list, and mark as seen
437          for h in newlist:
438            if not h in header_seen:
439              header_seen.append (h)
440              if header_dict.get(h):
441                consume = ii_macro_consume (header_dict[h])
442                for dep in consume:
443                  if dep not in src_requires:
444                    src_requires.add (dep)
445                    if because.get(dep) == None:
446                      because[dep] = inc_file
447
448          src_tmp = copy.deepcopy (src_data)
449        else:
450          print message + "  --> removing " + inc_file + "\n"
451          rmcount += 1
452          if verbose:
453            verbose.write (message + "  --> removing " + inc_file + "\n")
454          if remove_count.get(inc_file) == None:
455            remove_count[inc_file] = 1
456          else:
457            remove_count[inc_file] += 1
458          src_data = copy.deepcopy (src_tmp)
459    except:
460      print "Interuption: restoring original file"
461      out = open(src_file, "w")
462      for line in src_orig:
463        out.write (line)
464      out.close()
465      raise
466
467    # copy current version, since it is the "right" one now.
468    out = open(src_file, "w")
469    for line in src_data:
470      out.write (line)
471    out.close()
472    
473    # Try a final host bootstrap build to make sure everything is kosher.
474    if hostbuild:
475      res = get_make_output (build_dir, "all")
476      rc = res[0]
477      if (rc != 0):
478        # host build failed! return to original version
479        print "Error: " + src_file + " Failed to bootstrap at end!!! restoring."
480        print "        Bad version at " + src_file + ".bad"
481        os.rename (src_file, src_file + ".bad")
482        out = open(src_file, "w")
483        for line in src_orig:
484          out.write (line)
485        out.close()
486        return src_file + ": failed to build after reduction.  Restored original"
487
488    if src_data == src_orig:
489      summary = src_file + ": No change."
490    else:
491      summary = src_file + ": Reduction performed, "+str(rmcount)+" includes removed."
492  print summary
493  return summary
494
495only_h = list ()
496ignore_cond = False
497
498usage = False
499src = list()
500only_targs = list ()
501for x in sys.argv[1:]:
502  if x[0:2] == "-b":
503    build_dir = x[2:]
504  elif x[0:2] == "-f":
505    fn = normalize_header (x[2:])
506    if fn not in only_h:
507      only_h.append (fn)
508  elif x[0:2] == "-h":
509    usage = True
510  elif x[0:2] == "-d":
511    ignore_cond = True
512  elif x[0:2] == "-D":
513    ignore_list.append(x[2:])
514  elif x[0:2] == "-T":
515    only_targs.append(x[2:])
516  elif x[0:2] == "-t":
517    target_dir = x[2:]
518  elif x[0] == "-":
519    print "Error:  Unrecognized option " + x
520    usgae = True
521  else:
522    if not os.path.exists (x):
523      print "Error: specified file " + x + " does not exist."
524      usage = True
525    else:
526      src.append (x)
527
528if target_dir:
529  build_target_dict (target_dir, only_targs)
530
531if build_dir == "" and target_dir == "":
532  print "Error: Must specify a build directory, and/or a target directory."
533  usage = True
534
535if build_dir and not os.path.exists (build_dir):
536    print "Error: specified build directory does not exist : " + build_dir
537    usage = True
538
539if target_dir and not os.path.exists (target_dir):
540    print "Error: specified target directory does not exist : " + target_dir
541    usage = True
542
543if usage:
544  print "Attempts to remove extraneous include files from source files."
545  print " "
546  print "Should be run from the main gcc source directory, and works on a target"
547  print "directory, as we attempt to make the 'all' target."
548  print " "
549  print "By default, gcc-reorder-includes is run on each file before attempting"
550  print "to remove includes. this removes duplicates and puts some headers in a"
551  print "canonical ordering"
552  print " "
553  print "The build directory should be ready to compile via make. Time is saved"
554  print "if the build is already complete, so that only changes need to be built."
555  print " "
556  print "Usage: [options] file1.c [file2.c] ... [filen.c]"
557  print "      -bdir    : the root build directory to attempt buiding .o files."
558  print "      -tdir    : the target build directory"
559  print "      -d       : Ignore conditional macro dependencies."
560  print " "
561  print "      -Dmacro  : Ignore a specific macro for dependencies"
562  print "      -Ttarget : Only consider target in target directory."
563  print "      -fheader : Specifies a specific .h file to be considered."
564  print " "
565  print "      -D, -T, and -f can be specified mulitple times and are aggregated."
566  print " "
567  print "  The original file will be in filen.bak"
568  print " "
569  sys.exit (0)
570 
571if only_h:
572  print "Attempting to remove only these files:"
573  for x in only_h:
574    print x
575  print " "
576
577logfile = open("reduce-headers.log","w")
578
579for x in src:
580  msg = try_to_remove (x, only_h, logfile)
581  ilog = open("reduce-headers.sum","a")
582  ilog.write (msg + "\n")
583  ilog.close()
584
585ilog = open("reduce-headers.sum","a")
586ilog.write ("===============================================================\n")
587for x in remove_count:
588  msg = x + ": Removed " + str(remove_count[x]) + " times."
589  print msg
590  logfile.write (msg + "\n")
591  ilog.write (msg + "\n")
592
593
594
595
596
597