1#! /usr/bin/python2
2import os.path
3import sys
4import shlex
5import re
6import tempfile
7import copy
8
9from headerutils import *
10
11requires = { }
12provides = { }
13
14no_remove = [ "system.h", "coretypes.h", "config.h" , "bconfig.h", "backend.h" ]
15
16# These targets are the ones which provide "coverage".  Typically, if any
17# target is going to fail compilation, it's one of these.  This was determined
18# during the initial runs of reduce-headers... On a full set of target builds,
19# every failure which occured was triggered by one of these.  
20# This list is used during target-list construction simply to put any of these
21# *first* in the candidate list, increasing the probability that a failure is 
22# found quickly.
23target_priority = [
24    "aarch64-linux-gnu",
25    "arm-netbsdelf",
26    "c6x-elf",
27    "epiphany-elf",
28    "hppa2.0-hpux10.1",
29    "i686-mingw32crt",
30    "i686-pc-msdosdjgpp",
31    "mipsel-elf",
32    "powerpc-eabisimaltivec",
33    "rs6000-ibm-aix5.1.0",
34    "sh-superh-elf",
35    "sparc64-elf"
36]
37
38
39target_dir = ""
40build_dir = ""
41ignore_list = list()
42target_builds = list()
43
44target_dict = { }
45header_dict = { }
46search_path = [ ".", "../include", "../libcpp/include" ]
47
48remove_count = { }
49
50
51# Given a header name, normalize it.  ie.  cp/cp-tree.h could be in gcc, while
52# the same header could be referenced from within the cp subdirectory as
53# just cp-tree.h
54# for now, just assume basenames are unique
55
56def normalize_header (header):
57  return os.path.basename (header)
58
59
60# Adds a header file and its sub-includes to the global dictionary if they
61# aren't already there.  Specify s_path since different build directories may
62# append themselves on demand to the global list.
63# return entry for the specified header, knowing all sub entries are completed
64
65def get_header_info (header, s_path):
66  global header_dict
67  global empty_iinfo
68  process_list = list ()
69  location = ""
70  bname = ""
71  bname_iinfo = empty_iinfo
72  for path in s_path:
73    if os.path.exists (path + "/" + header):
74      location = path + "/" + header
75      break
76
77  if location:
78    bname = normalize_header (location)
79    if header_dict.get (bname):
80      bname_iinfo = header_dict[bname]
81      loc2 = ii_path (bname_iinfo)+ "/" + bname
82      if loc2[:2] == "./":
83        loc2 = loc2[2:]
84      if location[:2] == "./":
85        location = location[2:]
86      if loc2 != location:
87        # Don't use the cache if it isnt the right one.
88        bname_iinfo = process_ii_macro (location)
89      return bname_iinfo
90
91    bname_iinfo = process_ii_macro (location)
92    header_dict[bname] = bname_iinfo
93    # now decend into the include tree
94    for i in ii_include_list (bname_iinfo):
95      get_header_info (i, s_path)
96  else:
97    # if the file isnt in the source directories, look in the build and target
98    # directories. If it is here, then aggregate all the versions.
99    location = build_dir + "/gcc/" + header
100    build_inc = target_inc = False
101    if os.path.exists (location):
102      build_inc = True
103    for x in target_dict:
104      location = target_dict[x] + "/gcc/" + header
105      if os.path.exists (location):
106        target_inc = True
107        break
108
109    if (build_inc or target_inc):
110      bname = normalize_header(header)
111      defines = set()
112      consumes = set()
113      incl = set()
114      if build_inc:
115        iinfo = process_ii_macro (build_dir + "/gcc/" + header)
116        defines = set (ii_macro_define (iinfo))
117        consumes = set (ii_macro_consume (iinfo))
118        incl = set (ii_include_list (iinfo))
119
120      if (target_inc):
121        for x in target_dict:
122          location = target_dict[x] + "/gcc/" + header
123          if os.path.exists (location):
124            iinfo = process_ii_macro (location)
125            defines.update (ii_macro_define (iinfo))
126            consumes.update (ii_macro_consume (iinfo))
127            incl.update (ii_include_list (iinfo))
128
129      bname_iinfo = (header, "build", list(incl), list(), list(consumes), list(defines), list(), list())
130
131      header_dict[bname] = bname_iinfo
132      for i in incl:
133        get_header_info (i, s_path)
134
135  return bname_iinfo
136
137
138# return a list of all headers brought in by this header
139def all_headers (fname):
140  global header_dict
141  headers_stack = list()
142  headers_list = list()
143  if header_dict.get (fname) == None:
144    return list ()
145  for y in ii_include_list (header_dict[fname]):
146    headers_stack.append (y)
147
148  while headers_stack:
149    h = headers_stack.pop ()
150    hn = normalize_header (h)
151    if hn not in headers_list:
152      headers_list.append (hn)
153      if header_dict.get(hn):
154        for y in ii_include_list (header_dict[hn]):
155          if normalize_header (y) not in headers_list:
156            headers_stack.append (y)
157
158  return headers_list
159
160
161
162
163# Search bld_dir for all target tuples, confirm that they have a build path with
164# bld_dir/target-tuple/gcc, and build a dictionary of build paths indexed by
165# target tuple..
166
167def build_target_dict (bld_dir, just_these):
168  global target_dict
169  target_doct = { }
170  error = False
171  if os.path.exists (bld_dir):
172    if just_these:
173      ls = just_these
174    else:
175      ls = os.listdir(bld_dir)
176    for t in ls:
177      if t.find("-") != -1:
178        target = t.strip()
179        tpath = bld_dir + "/" + target
180        if not os.path.exists (tpath + "/gcc"):
181          print "Error: gcc build directory for target " + t + " Does not exist: " + tpath + "/gcc"
182          error = True
183        else:
184          target_dict[target] = tpath
185
186  if error:
187    target_dict = { }
188
189def get_obj_name (src_file):
190  if src_file[-2:] == ".c":
191    return src_file.replace (".c", ".o")
192  elif src_file[-3:] == ".cc":
193    return src_file.replace (".cc", ".o")
194  return ""
195
196def target_obj_exists (target, obj_name):
197  global target_dict
198  # look in a subdir if src has a subdir, then check gcc base directory.
199  if target_dict.get(target):
200    obj = target_dict[target] + "/gcc/" + obj_name
201    if not os.path.exists (obj):
202      obj = target_dict[target] + "/gcc/" + os.path.basename(obj_name)
203    if os.path.exists (obj):
204      return True
205  return False
206 
207# Given a src file, return a list of targets which may build this file.
208def find_targets (src_file):
209  global target_dict
210  targ_list = list()
211  obj_name = get_obj_name (src_file)
212  if not obj_name:
213    print "Error: " + src_file + " - Cannot determine object name."
214    return list()
215
216  # Put the high priority targets which tend to trigger failures first
217  for target in target_priority:
218    if target_obj_exists (target, obj_name):
219      targ_list.append ((target, target_dict[target]))
220
221  for target in target_dict:
222    if target not in target_priority and target_obj_exists (target, obj_name):
223      targ_list.append ((target, target_dict[target]))
224        
225  return targ_list
226
227
228def try_to_remove (src_file, h_list, verbose):
229  global target_dict
230  global header_dict
231  global build_dir
232
233  # build from scratch each time
234  header_dict = { }
235  summary = ""
236  rmcount = 0
237
238  because = { }
239  src_info = process_ii_macro_src (src_file)
240  src_data = ii_src (src_info)
241  if src_data:
242    inclist = ii_include_list_non_cond (src_info)
243    # work is done if there are no includes to check
244    if not inclist:
245      return src_file + ": No include files to attempt to remove"
246
247    # work on the include list in reverse.
248    inclist.reverse()
249
250    # Get the target list 
251    targ_list = list()
252    targ_list = find_targets (src_file)
253
254    spath = search_path
255    if os.path.dirname (src_file):
256      spath.append (os.path.dirname (src_file))
257
258    hostbuild = True
259    if src_file.find("config/") != -1:
260      # config files dont usually build on the host
261      hostbuild = False
262      obn = get_obj_name (os.path.basename (src_file))
263      if obn and os.path.exists (build_dir + "/gcc/" + obn):
264        hostbuild = True
265      if not target_dict:
266        summary = src_file + ": Target builds are required for config files.  None found."
267        print summary
268        return summary
269      if not targ_list:
270        summary =src_file + ": Cannot find any targets which build this file."
271        print summary
272        return summary
273
274    if hostbuild:
275      # confirm it actually builds before we do anything
276      print "Confirming source file builds"
277      res = get_make_output (build_dir + "/gcc", "all")
278      if res[0] != 0:
279        message = "Error: " + src_file + " does not build currently."
280        summary = src_file + " does not build on host."
281        print message
282        print res[1]
283        if verbose:
284          verbose.write (message + "\n")
285          verbose.write (res[1]+ "\n")
286        return summary
287
288    src_requires = set (ii_macro_consume (src_info))
289    for macro in src_requires:
290      because[macro] = src_file
291    header_seen = list ()
292
293    os.rename (src_file, src_file + ".bak")
294    src_orig = copy.deepcopy (src_data)
295    src_tmp = copy.deepcopy (src_data)
296
297    try:
298      # process the includes from bottom to top.  This is because we know that
299      # later includes have are known to be needed, so any dependency from this 
300      # header is a true dependency
301      for inc_file in inclist:
302        inc_file_norm = normalize_header (inc_file)
303        
304        if inc_file in no_remove:
305          continue
306        if len (h_list) != 0 and inc_file_norm not in h_list:
307          continue
308        if inc_file_norm[0:3] == "gt-":
309          continue
310        if inc_file_norm[0:6] == "gtype-":
311          continue
312        if inc_file_norm.replace(".h",".c") == os.path.basename(src_file):
313          continue
314             
315        lookfor = ii_src_line(src_info)[inc_file]
316        src_tmp.remove (lookfor)
317        message = "Trying " + src_file + " without " + inc_file
318        print message
319        if verbose:
320          verbose.write (message + "\n")
321        out = open(src_file, "w")
322        for line in src_tmp:
323          out.write (line)
324        out.close()
325          
326        keep = False
327        if hostbuild:
328          res = get_make_output (build_dir + "/gcc", "all")
329        else:
330          res = (0, "")
331
332        rc = res[0]
333        message = "Passed Host build"
334        if (rc != 0):
335          # host build failed
336          message  = "Compilation failed:\n";
337          keep = True
338        else:
339          if targ_list:
340            objfile = get_obj_name (src_file)
341            t1 = targ_list[0]
342            if objfile and os.path.exists(t1[1] +"/gcc/"+objfile):
343              res = get_make_output_parallel (targ_list, objfile, 0)
344            else:
345              res = get_make_output_parallel (targ_list, "all-gcc", 0)
346            rc = res[0]
347            if rc != 0:
348              message = "Compilation failed on TARGET : " + res[2]
349              keep = True
350            else:
351              message = "Passed host and target builds"
352
353        if keep:
354          print message + "\n"
355
356        if (rc != 0):
357          if verbose:
358            verbose.write (message + "\n");
359            verbose.write (res[1])
360            verbose.write ("\n");
361            if os.path.exists (inc_file):
362              ilog = open(inc_file+".log","a")
363              ilog.write (message + " for " + src_file + ":\n\n");
364              ilog.write ("============================================\n");
365              ilog.write (res[1])
366              ilog.write ("\n");
367              ilog.close()
368            if os.path.exists (src_file):
369              ilog = open(src_file+".log","a")
370              ilog.write (message + " for " +inc_file + ":\n\n");
371              ilog.write ("============================================\n");
372              ilog.write (res[1])
373              ilog.write ("\n");
374              ilog.close()
375
376        # Given a sequence where :
377        # #include "tm.h"
378        # #include "target.h"  // includes tm.h
379
380        # target.h was required, and when attempting to remove tm.h we'd see that
381        # all the macro defintions are "required" since they all look like:
382        # #ifndef HAVE_blah
383        # #define HAVE_blah
384        # endif
385
386        # when target.h was found to be required, tm.h will be tagged as included.
387        # so when we get this far, we know we dont have to check the macros for
388        # tm.h since we know it is already been included.
389
390        if inc_file_norm not in header_seen:
391          iinfo = get_header_info (inc_file, spath)
392          newlist = all_headers (inc_file_norm)
393          if ii_path(iinfo) == "build" and not target_dict:
394            keep = True
395            text = message + " : Will not remove a build file without some targets."
396            print text
397            ilog = open(src_file+".log","a")
398            ilog.write (text +"\n")
399            ilog.write ("============================================\n");
400            ilog.close()
401            ilog = open("reduce-headers-kept.log","a")
402            ilog.write (src_file + " " + text +"\n")
403            ilog.close()
404        else:
405          newlist = list()
406        if not keep and inc_file_norm not in header_seen:
407          # now look for any macro requirements.
408          for h in newlist:
409            if not h in header_seen:
410              if header_dict.get(h):
411                defined = ii_macro_define (header_dict[h])
412                for dep in defined:
413                  if dep in src_requires and dep not in ignore_list:
414                    keep = True;
415                    text = message + ", but must keep " + inc_file + " because it provides " + dep 
416                    if because.get(dep) != None:
417                      text = text + " Possibly required by " + because[dep]
418                    print text
419                    ilog = open(inc_file+".log","a")
420                    ilog.write (because[dep]+": Requires [dep] in "+src_file+"\n")
421                    ilog.write ("============================================\n");
422                    ilog.close()
423                    ilog = open(src_file+".log","a")
424                    ilog.write (text +"\n")
425                    ilog.write ("============================================\n");
426                    ilog.close()
427                    ilog = open("reduce-headers-kept.log","a")
428                    ilog.write (src_file + " " + text +"\n")
429                    ilog.close()
430                    if verbose:
431                      verbose.write (text + "\n")
432
433        if keep:
434          # add all headers 'consumes' to src_requires list, and mark as seen
435          for h in newlist:
436            if not h in header_seen:
437              header_seen.append (h)
438              if header_dict.get(h):
439                consume = ii_macro_consume (header_dict[h])
440                for dep in consume:
441                  if dep not in src_requires:
442                    src_requires.add (dep)
443                    if because.get(dep) == None:
444                      because[dep] = inc_file
445
446          src_tmp = copy.deepcopy (src_data)
447        else:
448          print message + "  --> removing " + inc_file + "\n"
449          rmcount += 1
450          if verbose:
451            verbose.write (message + "  --> removing " + inc_file + "\n")
452          if remove_count.get(inc_file) == None:
453            remove_count[inc_file] = 1
454          else:
455            remove_count[inc_file] += 1
456          src_data = copy.deepcopy (src_tmp)
457    except:
458      print "Interuption: restoring original file"
459      out = open(src_file, "w")
460      for line in src_orig:
461        out.write (line)
462      out.close()
463      raise
464
465    # copy current version, since it is the "right" one now.
466    out = open(src_file, "w")
467    for line in src_data:
468      out.write (line)
469    out.close()
470    
471    # Try a final host bootstrap build to make sure everything is kosher.
472    if hostbuild:
473      res = get_make_output (build_dir, "all")
474      rc = res[0]
475      if (rc != 0):
476        # host build failed! return to original version
477        print "Error: " + src_file + " Failed to bootstrap at end!!! restoring."
478        print "        Bad version at " + src_file + ".bad"
479        os.rename (src_file, src_file + ".bad")
480        out = open(src_file, "w")
481        for line in src_orig:
482          out.write (line)
483        out.close()
484        return src_file + ": failed to build after reduction.  Restored original"
485
486    if src_data == src_orig:
487      summary = src_file + ": No change."
488    else:
489      summary = src_file + ": Reduction performed, "+str(rmcount)+" includes removed."
490  print summary
491  return summary
492
493only_h = list ()
494ignore_cond = False
495
496usage = False
497src = list()
498only_targs = list ()
499for x in sys.argv[1:]:
500  if x[0:2] == "-b":
501    build_dir = x[2:]
502  elif x[0:2] == "-f":
503    fn = normalize_header (x[2:])
504    if fn not in only_h:
505      only_h.append (fn)
506  elif x[0:2] == "-h":
507    usage = True
508  elif x[0:2] == "-d":
509    ignore_cond = True
510  elif x[0:2] == "-D":
511    ignore_list.append(x[2:])
512  elif x[0:2] == "-T":
513    only_targs.append(x[2:])
514  elif x[0:2] == "-t":
515    target_dir = x[2:]
516  elif x[0] == "-":
517    print "Error:  Unrecognized option " + x
518    usgae = True
519  else:
520    if not os.path.exists (x):
521      print "Error: specified file " + x + " does not exist."
522      usage = True
523    else:
524      src.append (x)
525
526if target_dir:
527  build_target_dict (target_dir, only_targs)
528
529if build_dir == "" and target_dir == "":
530  print "Error: Must specify a build directory, and/or a target directory."
531  usage = True
532
533if build_dir and not os.path.exists (build_dir):
534    print "Error: specified build directory does not exist : " + build_dir
535    usage = True
536
537if target_dir and not os.path.exists (target_dir):
538    print "Error: specified target directory does not exist : " + target_dir
539    usage = True
540
541if usage:
542  print "Attempts to remove extraneous include files from source files."
543  print " "
544  print "Should be run from the main gcc source directory, and works on a target"
545  print "directory, as we attempt to make the 'all' target."
546  print " "
547  print "By default, gcc-reorder-includes is run on each file before attempting"
548  print "to remove includes. this removes duplicates and puts some headers in a"
549  print "canonical ordering"
550  print " "
551  print "The build directory should be ready to compile via make. Time is saved"
552  print "if the build is already complete, so that only changes need to be built."
553  print " "
554  print "Usage: [options] file1.c [file2.c] ... [filen.c]"
555  print "      -bdir    : the root build directory to attempt buiding .o files."
556  print "      -tdir    : the target build directory"
557  print "      -d       : Ignore conditional macro dependencies."
558  print " "
559  print "      -Dmacro  : Ignore a specific macro for dependencies"
560  print "      -Ttarget : Only consider target in target directory."
561  print "      -fheader : Specifies a specific .h file to be considered."
562  print " "
563  print "      -D, -T, and -f can be specified mulitple times and are aggregated."
564  print " "
565  print "  The original file will be in filen.bak"
566  print " "
567  sys.exit (0)
568 
569if only_h:
570  print "Attempting to remove only these files:"
571  for x in only_h:
572    print x
573  print " "
574
575logfile = open("reduce-headers.log","w")
576
577for x in src:
578  msg = try_to_remove (x, only_h, logfile)
579  ilog = open("reduce-headers.sum","a")
580  ilog.write (msg + "\n")
581  ilog.close()
582
583ilog = open("reduce-headers.sum","a")
584ilog.write ("===============================================================\n")
585for x in remove_count:
586  msg = x + ": Removed " + str(remove_count[x]) + " times."
587  print msg
588  logfile.write (msg + "\n")
589  ilog.write (msg + "\n")
590
591
592
593
594
595