1#!/usr/bin/env python
2
3from __future__ import print_function
4
5"""
6This script parses each "meta" file and extracts the
7information needed to deduce build and src dependencies.
8
--- 23 unchanged lines hidden (view full) ---
32
33'V' the filemon version, this record is used as a clue
34 that we have reached the interesting bit.
35
36"""
37
38"""
39RCSid:
40 $Id: meta2deps.py,v 1.17 2014/04/05 22:56:54 sjg Exp $
41
42 Copyright (c) 2011-2013, Juniper Networks, Inc.
43 All rights reserved.
44
45 Redistribution and use in source and binary forms, with or without
46 modification, are permitted provided that the following conditions
47 are met:
48 1. Redistributions of source code must retain the above copyright
--- 58 unchanged lines hidden (view full) ---
107 """
108 Return an absolute path, resolving via cwd or last_dir if needed.
109 this gets called a lot, so we try to avoid calling realpath
110 until we know we have something.
111 """
112 rpath = resolve(path, cwd, last_dir, debug, debug_out)
113 if rpath:
114 path = rpath
115 if (path.find('./') > 0 or
116 path.endswith('/..') or
117 os.path.islink(path)):
118 return os.path.realpath(path)
119 return path
120
121def sort_unique(list, cmp=None, key=None, reverse=False):
122 list.sort(cmp, key, reverse)
123 nl = []
--- 13 unchanged lines hidden (view full) ---
137class MetaFile:
138 """class to parse meta files generated by bmake."""
139
140 conf = None
141 dirdep_re = None
142 host_target = None
143 srctops = []
144 objroots = []
145
146 seen = {}
147 obj_deps = []
148 src_deps = []
149 file_deps = []
150
151 def __init__(self, name, conf={}):
152 """if name is set we will parse it now.
153 conf can have the follwing keys:
--- 20 unchanged lines hidden (view full) ---
174
175 DPDEPS names an optional file to which per file dependencies
176 will be appended.
177 For example if 'some/path/foo.h' is read from SRCTOP
178 then 'DPDEPS_some/path/foo.h +=' "RELDIR" is output.
179 This can allow 'bmake' to learn all the dirs within
180 the tree that depend on 'foo.h'
181
182 debug desired debug level
183
184 debug_out open file to send debug output to (sys.stderr)
185
186 """
187
188 self.name = name
189 self.debug = getv(conf, 'debug', 0)
--- 41 unchanged lines hidden (view full) ---
231 if objroot[-1] == '/':
232 _objroot += '/'
233 if not _objroot in self.objroots:
234 self.objroots.append(_objroot)
235
236 # we want the longest match
237 self.srctops.sort(reverse=True)
238 self.objroots.sort(reverse=True)
239
240 if self.debug:
241 print("host_target=", self.host_target, file=self.debug_out)
242 print("srctops=", self.srctops, file=self.debug_out)
243 print("objroots=", self.objroots, file=self.debug_out)
244
245 self.dirdep_re = re.compile(r'([^/]+)/(.+)')
246
247 if self.dpdeps and not self.reldir:
248 if self.debug:
249 print("need reldir:", end=' ', file=self.debug_out)
250 if self.curdir:
251 srctop = self.find_top(self.curdir, self.srctops)
252 if srctop:
253 self.reldir = self.curdir.replace(srctop,'')
254 if self.debug:
255 print(self.reldir, file=self.debug_out)
256 if not self.reldir:
257 self.dpdeps = None # we cannot do it?
258
259 self.cwd = os.getcwd() # make sure this is initialized
260
261 if name:
262 self.try_parse()
263
264 def reset(self):
265 """reset state if we are being passed meta files from multiple directories."""
266 self.seen = {}
267 self.obj_deps = []
--- 87 unchanged lines hidden (view full) ---
355 CWD "cwd"
356 TARGET "target"
357 -- command output --
358 -- filemon acquired metadata --
359 # buildmon version 3
360 V 3
361 C "pid" "cwd"
362 E "pid" "path"
363 F "pid" "child"
364 R "pid" "path"
365 W "pid" "path"
366 X "pid" "status"
367 D "pid" "path"
368 L "pid" "src" "target"
369 M "pid" "old" "new"
370 S "pid" "path"
371 # Bye bye
372
373 We go to some effort to avoid processing a dependency more than once.
374 Of the above record types only C,E,F,L,R,V and W are of interest.
375 """
376
377 version = 0 # unknown
378 if name:
379 self.name = name;
380 if file:
381 f = file
382 cwd = last_dir = self.cwd
383 else:
384 f = open(self.name, 'r')
385 skip = True
386 pid_cwd = {}
387 pid_last_dir = {}
388 last_pid = 0
389
390 self.line = 0
--- 16 unchanged lines hidden (view full) ---
407 version = int(w[1])
408 """
409 if version < 4:
410 # we cannot ignore 'W' records
411 # as they may be 'rw'
412 interesting += 'W'
413 """
414 elif w[0] == 'CWD':
415 self.cwd = cwd = last_dir = w[1]
416 self.seenit(cwd) # ignore this
417 if self.debug:
418 print("%s: CWD=%s" % (self.name, cwd), file=self.debug_out)
419 continue
420
421 pid = int(w[1])
422 if pid != last_pid:
423 if last_pid:
424 pid_cwd[last_pid] = cwd
425 pid_last_dir[last_pid] = last_dir
426 cwd = getv(pid_cwd, pid, self.cwd)
427 last_dir = getv(pid_last_dir, pid, self.cwd)
428 last_pid = pid
429
430 # process operations
431 if w[0] == 'F':
432 npid = int(w[2])
433 pid_cwd[npid] = cwd
434 pid_last_dir[npid] = cwd
435 last_pid = npid
436 continue
437 elif w[0] == 'C':
438 cwd = abspath(w[2], cwd, None, self.debug, self.debug_out)
439 if cwd.endswith('/.'):
440 cwd = cwd[0:-2]
441 last_dir = cwd
442 if self.debug > 1:
443 print("cwd=", cwd, file=self.debug_out)
444 continue
445
446 if w[2] in self.seen:
447 if self.debug > 2:
448 print("seen:", w[2], file=self.debug_out)
449 continue
450 # file operations
451 if w[0] in 'ML':
452 path = w[2].strip("'")
453 else:
454 path = w[2]
455 # we are never interested in .dirdep files as dependencies
456 if path.endswith('.dirdep'):
457 continue
458 # we don't want to resolve the last component if it is
459 # a symlink
460 path = resolve(path, cwd, last_dir, self.debug, self.debug_out)
461 if not path:
462 continue
463 dir,base = os.path.split(path)
464 if dir in self.seen:
465 if self.debug > 2:
466 print("seen:", dir, file=self.debug_out)
467 continue
468 # we can have a path in an objdir which is a link
469 # to the src dir, we may need to add dependencies for each
470 rdir = dir
471 dir = abspath(dir, cwd, last_dir, self.debug, self.debug_out)
472 if rdir == dir or rdir.find('./') > 0:
473 rdir = None
474 # now put path back together
475 path = '/'.join([dir,base])
476 if self.debug > 1:
477 print("raw=%s rdir=%s dir=%s path=%s" % (w[2], rdir, dir, path), file=self.debug_out)
478 if w[0] in 'SRWL':
479 if w[0] == 'W' and path.endswith('.dirdep'):
480 continue
481 if path in [last_dir, cwd, self.cwd, self.curdir]:
482 if self.debug > 1:
483 print("skipping:", path, file=self.debug_out)
484 continue
485 if os.path.isdir(path):
486 if w[0] in 'RW':
487 last_dir = path;
488 if self.debug > 1:
489 print("ldir=", last_dir, file=self.debug_out)
490 continue
491
492 if w[0] in 'REWML':
493 # finally, we get down to it
494 if dir == self.cwd or dir == self.curdir:
495 continue
496 srctop = self.find_top(path, self.srctops)
497 if srctop:
498 if self.dpdeps:
499 self.add(self.file_deps, path.replace(srctop,''), 'file')
500 self.add(self.src_deps, dir.replace(srctop,''), 'src')
501 self.seenit(w[2])
502 self.seenit(dir)
503 if rdir and not rdir.startswith(srctop):
504 dir = rdir # for below
505 rdir = None
506 else:
507 continue
508
509 objroot = None
510 for dir in [dir,rdir]:
511 if not dir:
512 continue
513 objroot = self.find_top(dir, self.objroots)
514 if objroot:
515 break
516 if objroot:
517 ddep = self.find_obj(objroot, dir, path, w[2])
518 if ddep:
519 self.add(self.obj_deps, ddep, 'obj')
520 else:
521 # don't waste time looking again
522 self.seenit(w[2])
523 self.seenit(dir)
524 if not file:
525 f.close()
526
527
528def main(argv, klass=MetaFile, xopts='', xoptf=None):
529 """Simple driver for class MetaFile.
530
531 Usage:
532 script [options] [key=value ...] "meta" ...
533
534 Options and key=value pairs contribute to the
535 dictionary passed to MetaFile.
536
537 -S "SRCTOP"
538 add "SRCTOP" to the "SRCTOPS" list.
539
540 -C "CURDIR"
541
542 -O "OBJROOT"
543 add "OBJROOT" to the "OBJROOTS" list.
544
545 -m "MACHINE"
546
547 -a "MACHINE_ARCH"
548
549 -H "HOST_TARGET"
550
551 -D "DPDEPS"
552
553 -d bumps debug level
554
555 """
556 import getopt
557
558 # import Psyco if we can
559 # it can speed things up quite a bit
560 have_psyco = 0
561 try:
562 import psyco
563 psyco.full()
564 have_psyco = 1
565 except:
566 pass
567
568 conf = {
569 'SRCTOPS': [],
570 'OBJROOTS': [],
571 }
572
573 try:
574 machine = os.environ['MACHINE']
575 if machine:
576 conf['MACHINE'] = machine
577 machine_arch = os.environ['MACHINE_ARCH']
578 if machine_arch:
--- 5 unchanged lines hidden (view full) ---
584 if objroot:
585 conf['OBJROOTS'].append(objroot)
586 except:
587 pass
588
589 debug = 0
590 output = True
591
592 opts, args = getopt.getopt(argv[1:], 'a:dS:C:O:R:m:D:H:qT:' + xopts)
593 for o, a in opts:
594 if o == '-a':
595 conf['MACHINE_ARCH'] = a
596 elif o == '-d':
597 debug += 1
598 elif o == '-q':
599 output = False
600 elif o == '-H':
--- 9 unchanged lines hidden (view full) ---
610 elif o == '-R':
611 conf['RELDIR'] = a
612 elif o == '-D':
613 conf['DPDEPS'] = a
614 elif o == '-m':
615 conf['MACHINE'] = a
616 elif o == '-T':
617 conf['TARGET_SPEC'] = a
618 elif xoptf:
619 xoptf(o, a, conf)
620
621 conf['debug'] = debug
622
623 # get any var=val assignments
624 eaten = []
625 for a in args:
--- 18 unchanged lines hidden (view full) ---
644 debug_out = getv(conf, 'debug_out', sys.stderr)
645
646 if debug:
647 print("config:", file=debug_out)
648 print("psyco=", have_psyco, file=debug_out)
649 for k,v in list(conf.items()):
650 print("%s=%s" % (k,v), file=debug_out)
651
652 for a in args:
653 if a.endswith('.meta'):
654 m = klass(a, conf)
655 elif a.startswith('@'):
656 # there can actually multiple files per line
657 for line in open(a[1:]):
658 for f in line.strip().split():
659 m = klass(f, conf)
660
661 if output:
662 print(m.dirdeps())
663
664 print(m.src_dirdeps('\nsrc:'))
665
666 dpdeps = getv(conf, 'DPDEPS')
667 if dpdeps:
668 m.file_depends(open(dpdeps, 'wb'))
669
670 return m
671
672if __name__ == '__main__':
673 try:
674 main(sys.argv)
675 except:
676 # yes, this goes to stdout
677 print("ERROR: ", sys.exc_info()[1])
678 raise
679
2
3from __future__ import print_function
4
5"""
6This script parses each "meta" file and extracts the
7information needed to deduce build and src dependencies.
8
--- 23 unchanged lines hidden (view full) ---
32
33'V' the filemon version, this record is used as a clue
34 that we have reached the interesting bit.
35
36"""
37
38"""
39RCSid:
40 $Id: meta2deps.py,v 1.17 2014/04/05 22:56:54 sjg Exp $
41
42 Copyright (c) 2011-2013, Juniper Networks, Inc.
43 All rights reserved.
44
45 Redistribution and use in source and binary forms, with or without
46 modification, are permitted provided that the following conditions
47 are met:
48 1. Redistributions of source code must retain the above copyright
--- 58 unchanged lines hidden (view full) ---
107 """
108 Return an absolute path, resolving via cwd or last_dir if needed.
109 this gets called a lot, so we try to avoid calling realpath
110 until we know we have something.
111 """
112 rpath = resolve(path, cwd, last_dir, debug, debug_out)
113 if rpath:
114 path = rpath
115 if (path.find('./') > 0 or
116 path.endswith('/..') or
117 os.path.islink(path)):
118 return os.path.realpath(path)
119 return path
120
121def sort_unique(list, cmp=None, key=None, reverse=False):
122 list.sort(cmp, key, reverse)
123 nl = []
--- 13 unchanged lines hidden (view full) ---
137class MetaFile:
138 """class to parse meta files generated by bmake."""
139
140 conf = None
141 dirdep_re = None
142 host_target = None
143 srctops = []
144 objroots = []
145
146 seen = {}
147 obj_deps = []
148 src_deps = []
149 file_deps = []
150
151 def __init__(self, name, conf={}):
152 """if name is set we will parse it now.
153 conf can have the follwing keys:
--- 20 unchanged lines hidden (view full) ---
174
175 DPDEPS names an optional file to which per file dependencies
176 will be appended.
177 For example if 'some/path/foo.h' is read from SRCTOP
178 then 'DPDEPS_some/path/foo.h +=' "RELDIR" is output.
179 This can allow 'bmake' to learn all the dirs within
180 the tree that depend on 'foo.h'
181
182 debug desired debug level
183
184 debug_out open file to send debug output to (sys.stderr)
185
186 """
187
188 self.name = name
189 self.debug = getv(conf, 'debug', 0)
--- 41 unchanged lines hidden (view full) ---
231 if objroot[-1] == '/':
232 _objroot += '/'
233 if not _objroot in self.objroots:
234 self.objroots.append(_objroot)
235
236 # we want the longest match
237 self.srctops.sort(reverse=True)
238 self.objroots.sort(reverse=True)
239
240 if self.debug:
241 print("host_target=", self.host_target, file=self.debug_out)
242 print("srctops=", self.srctops, file=self.debug_out)
243 print("objroots=", self.objroots, file=self.debug_out)
244
245 self.dirdep_re = re.compile(r'([^/]+)/(.+)')
246
247 if self.dpdeps and not self.reldir:
248 if self.debug:
249 print("need reldir:", end=' ', file=self.debug_out)
250 if self.curdir:
251 srctop = self.find_top(self.curdir, self.srctops)
252 if srctop:
253 self.reldir = self.curdir.replace(srctop,'')
254 if self.debug:
255 print(self.reldir, file=self.debug_out)
256 if not self.reldir:
257 self.dpdeps = None # we cannot do it?
258
259 self.cwd = os.getcwd() # make sure this is initialized
260
261 if name:
262 self.try_parse()
263
264 def reset(self):
265 """reset state if we are being passed meta files from multiple directories."""
266 self.seen = {}
267 self.obj_deps = []
--- 87 unchanged lines hidden (view full) ---
355 CWD "cwd"
356 TARGET "target"
357 -- command output --
358 -- filemon acquired metadata --
359 # buildmon version 3
360 V 3
361 C "pid" "cwd"
362 E "pid" "path"
363 F "pid" "child"
364 R "pid" "path"
365 W "pid" "path"
366 X "pid" "status"
367 D "pid" "path"
368 L "pid" "src" "target"
369 M "pid" "old" "new"
370 S "pid" "path"
371 # Bye bye
372
373 We go to some effort to avoid processing a dependency more than once.
374 Of the above record types only C,E,F,L,R,V and W are of interest.
375 """
376
377 version = 0 # unknown
378 if name:
379 self.name = name;
380 if file:
381 f = file
382 cwd = last_dir = self.cwd
383 else:
384 f = open(self.name, 'r')
385 skip = True
386 pid_cwd = {}
387 pid_last_dir = {}
388 last_pid = 0
389
390 self.line = 0
--- 16 unchanged lines hidden (view full) ---
407 version = int(w[1])
408 """
409 if version < 4:
410 # we cannot ignore 'W' records
411 # as they may be 'rw'
412 interesting += 'W'
413 """
414 elif w[0] == 'CWD':
415 self.cwd = cwd = last_dir = w[1]
416 self.seenit(cwd) # ignore this
417 if self.debug:
418 print("%s: CWD=%s" % (self.name, cwd), file=self.debug_out)
419 continue
420
421 pid = int(w[1])
422 if pid != last_pid:
423 if last_pid:
424 pid_cwd[last_pid] = cwd
425 pid_last_dir[last_pid] = last_dir
426 cwd = getv(pid_cwd, pid, self.cwd)
427 last_dir = getv(pid_last_dir, pid, self.cwd)
428 last_pid = pid
429
430 # process operations
431 if w[0] == 'F':
432 npid = int(w[2])
433 pid_cwd[npid] = cwd
434 pid_last_dir[npid] = cwd
435 last_pid = npid
436 continue
437 elif w[0] == 'C':
438 cwd = abspath(w[2], cwd, None, self.debug, self.debug_out)
439 if cwd.endswith('/.'):
440 cwd = cwd[0:-2]
441 last_dir = cwd
442 if self.debug > 1:
443 print("cwd=", cwd, file=self.debug_out)
444 continue
445
446 if w[2] in self.seen:
447 if self.debug > 2:
448 print("seen:", w[2], file=self.debug_out)
449 continue
450 # file operations
451 if w[0] in 'ML':
452 path = w[2].strip("'")
453 else:
454 path = w[2]
455 # we are never interested in .dirdep files as dependencies
456 if path.endswith('.dirdep'):
457 continue
458 # we don't want to resolve the last component if it is
459 # a symlink
460 path = resolve(path, cwd, last_dir, self.debug, self.debug_out)
461 if not path:
462 continue
463 dir,base = os.path.split(path)
464 if dir in self.seen:
465 if self.debug > 2:
466 print("seen:", dir, file=self.debug_out)
467 continue
468 # we can have a path in an objdir which is a link
469 # to the src dir, we may need to add dependencies for each
470 rdir = dir
471 dir = abspath(dir, cwd, last_dir, self.debug, self.debug_out)
472 if rdir == dir or rdir.find('./') > 0:
473 rdir = None
474 # now put path back together
475 path = '/'.join([dir,base])
476 if self.debug > 1:
477 print("raw=%s rdir=%s dir=%s path=%s" % (w[2], rdir, dir, path), file=self.debug_out)
478 if w[0] in 'SRWL':
479 if w[0] == 'W' and path.endswith('.dirdep'):
480 continue
481 if path in [last_dir, cwd, self.cwd, self.curdir]:
482 if self.debug > 1:
483 print("skipping:", path, file=self.debug_out)
484 continue
485 if os.path.isdir(path):
486 if w[0] in 'RW':
487 last_dir = path;
488 if self.debug > 1:
489 print("ldir=", last_dir, file=self.debug_out)
490 continue
491
492 if w[0] in 'REWML':
493 # finally, we get down to it
494 if dir == self.cwd or dir == self.curdir:
495 continue
496 srctop = self.find_top(path, self.srctops)
497 if srctop:
498 if self.dpdeps:
499 self.add(self.file_deps, path.replace(srctop,''), 'file')
500 self.add(self.src_deps, dir.replace(srctop,''), 'src')
501 self.seenit(w[2])
502 self.seenit(dir)
503 if rdir and not rdir.startswith(srctop):
504 dir = rdir # for below
505 rdir = None
506 else:
507 continue
508
509 objroot = None
510 for dir in [dir,rdir]:
511 if not dir:
512 continue
513 objroot = self.find_top(dir, self.objroots)
514 if objroot:
515 break
516 if objroot:
517 ddep = self.find_obj(objroot, dir, path, w[2])
518 if ddep:
519 self.add(self.obj_deps, ddep, 'obj')
520 else:
521 # don't waste time looking again
522 self.seenit(w[2])
523 self.seenit(dir)
524 if not file:
525 f.close()
526
527
528def main(argv, klass=MetaFile, xopts='', xoptf=None):
529 """Simple driver for class MetaFile.
530
531 Usage:
532 script [options] [key=value ...] "meta" ...
533
534 Options and key=value pairs contribute to the
535 dictionary passed to MetaFile.
536
537 -S "SRCTOP"
538 add "SRCTOP" to the "SRCTOPS" list.
539
540 -C "CURDIR"
541
542 -O "OBJROOT"
543 add "OBJROOT" to the "OBJROOTS" list.
544
545 -m "MACHINE"
546
547 -a "MACHINE_ARCH"
548
549 -H "HOST_TARGET"
550
551 -D "DPDEPS"
552
553 -d bumps debug level
554
555 """
556 import getopt
557
558 # import Psyco if we can
559 # it can speed things up quite a bit
560 have_psyco = 0
561 try:
562 import psyco
563 psyco.full()
564 have_psyco = 1
565 except:
566 pass
567
568 conf = {
569 'SRCTOPS': [],
570 'OBJROOTS': [],
571 }
572
573 try:
574 machine = os.environ['MACHINE']
575 if machine:
576 conf['MACHINE'] = machine
577 machine_arch = os.environ['MACHINE_ARCH']
578 if machine_arch:
--- 5 unchanged lines hidden (view full) ---
584 if objroot:
585 conf['OBJROOTS'].append(objroot)
586 except:
587 pass
588
589 debug = 0
590 output = True
591
592 opts, args = getopt.getopt(argv[1:], 'a:dS:C:O:R:m:D:H:qT:' + xopts)
593 for o, a in opts:
594 if o == '-a':
595 conf['MACHINE_ARCH'] = a
596 elif o == '-d':
597 debug += 1
598 elif o == '-q':
599 output = False
600 elif o == '-H':
--- 9 unchanged lines hidden (view full) ---
610 elif o == '-R':
611 conf['RELDIR'] = a
612 elif o == '-D':
613 conf['DPDEPS'] = a
614 elif o == '-m':
615 conf['MACHINE'] = a
616 elif o == '-T':
617 conf['TARGET_SPEC'] = a
618 elif xoptf:
619 xoptf(o, a, conf)
620
621 conf['debug'] = debug
622
623 # get any var=val assignments
624 eaten = []
625 for a in args:
--- 18 unchanged lines hidden (view full) ---
644 debug_out = getv(conf, 'debug_out', sys.stderr)
645
646 if debug:
647 print("config:", file=debug_out)
648 print("psyco=", have_psyco, file=debug_out)
649 for k,v in list(conf.items()):
650 print("%s=%s" % (k,v), file=debug_out)
651
652 for a in args:
653 if a.endswith('.meta'):
654 m = klass(a, conf)
655 elif a.startswith('@'):
656 # there can actually multiple files per line
657 for line in open(a[1:]):
658 for f in line.strip().split():
659 m = klass(f, conf)
660
661 if output:
662 print(m.dirdeps())
663
664 print(m.src_dirdeps('\nsrc:'))
665
666 dpdeps = getv(conf, 'DPDEPS')
667 if dpdeps:
668 m.file_depends(open(dpdeps, 'wb'))
669
670 return m
671
672if __name__ == '__main__':
673 try:
674 main(sys.argv)
675 except:
676 # yes, this goes to stdout
677 print("ERROR: ", sys.exc_info()[1])
678 raise
679