#!/usr/bin/python2.7 # Copyright 2016 The Chromium Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. """Prints all non-system dependencies for the given module. The primary use-case for this script is to generate the list of python modules required for .isolate files. This script should be compatible with Python 2 and Python 3. """ import argparse import fnmatch import os import pipes import sys # Don't use any helper modules, or else they will end up in the results. _SRC_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir)) def ComputePythonDependencies(): """Gets the paths of imported non-system python modules. A path is assumed to be a "system" import if it is outside of chromium's src/. The paths will be relative to the current directory. """ module_paths = (m.__file__ for m in sys.modules.values() if m and hasattr(m, '__file__')) src_paths = set() for path in module_paths: if path == __file__: continue path = os.path.abspath(path) if not path.startswith(_SRC_ROOT): continue if (path.endswith('.pyc') or (path.endswith('c') and not os.path.splitext(path)[1])): path = path[:-1] src_paths.add(path) return src_paths def _NormalizeCommandLine(options): """Returns a string that when run from SRC_ROOT replicates the command.""" args = ['build/print_python_deps.py'] root = os.path.relpath(options.root, _SRC_ROOT) if root != '.': args.extend(('--root', root)) if options.output: args.extend(('--output', os.path.relpath(options.output, _SRC_ROOT))) if options.gn_paths: args.extend(('--gn-paths',)) for whitelist in sorted(options.whitelists): args.extend(('--whitelist', os.path.relpath(whitelist, _SRC_ROOT))) args.append(os.path.relpath(options.module, _SRC_ROOT)) return ' '.join(pipes.quote(x) for x in args) def _FindPythonInDirectory(directory, allow_test): """Returns an iterable of all non-test python files in the given directory.""" files = [] for root, _dirnames, filenames in os.walk(directory): for filename in filenames: if filename.endswith('.py') and (allow_test or not filename.endswith('_test.py')): yield os.path.join(root, filename) def _GetTargetPythonVersion(module): """Heuristically determines the target module's Python version.""" with open(module) as f: shebang = f.readline().strip() default_version = 2 if shebang.startswith('#!'): # Examples: # '#!/usr/bin/python' # '#!/usr/bin/python2.7' # '#!/usr/bin/python3' # '#!/usr/bin/env python3' # '#!/usr/bin/env vpython' # '#!/usr/bin/env vpython3' exec_name = os.path.basename(shebang[2:].split(' ')[-1]) for python_prefix in ['python', 'vpython']: if exec_name.startswith(python_prefix): version_string = exec_name[len(python_prefix):] break else: raise ValueError('Invalid shebang: ' + shebang) if version_string: return int(float(version_string)) return default_version def _ImportModuleByPath(module_path): """Imports a module by its source file.""" # Replace the path entry for print_python_deps.py with the one for the given # module. sys.path[0] = os.path.dirname(module_path) if sys.version_info[0] == 2: import imp # Python 2 only, since it's deprecated in Python 3. imp.load_source('NAME', module_path) else: # https://docs.python.org/3/library/importlib.html#importing-a-source-file-directly module_name = os.path.splitext(os.path.basename(module_path))[0] import importlib.util # Python 3 only, since it's unavailable in Python 2. spec = importlib.util.spec_from_file_location(module_name, module_path) module = importlib.util.module_from_spec(spec) sys.modules[module_name] = module spec.loader.exec_module(module) def main(): parser = argparse.ArgumentParser( description='Prints all non-system dependencies for the given module.') parser.add_argument('module', help='The python module to analyze.') parser.add_argument('--root', default='.', help='Directory to make paths relative to.') parser.add_argument('--output', help='Write output to a file rather than stdout.') parser.add_argument('--inplace', action='store_true', help='Write output to a file with the same path as the ' 'module, but with a .pydeps extension. Also sets the ' 'root to the module\'s directory.') parser.add_argument('--no-header', action='store_true', help='Do not write the "# Generated by" header.') parser.add_argument('--gn-paths', action='store_true', help='Write paths as //foo/bar/baz.py') parser.add_argument('--did-relaunch', action='store_true', help=argparse.SUPPRESS) parser.add_argument('--whitelist', default=[], action='append', dest='whitelists', help='Recursively include all non-test python files ' 'within this directory. May be specified multiple times.') options = parser.parse_args() if options.inplace: if options.output: parser.error('Cannot use --inplace and --output at the same time!') if not options.module.endswith('.py'): parser.error('Input module path should end with .py suffix!') options.output = options.module + 'deps' options.root = os.path.dirname(options.module) modules = [options.module] if os.path.isdir(options.module): modules = list(_FindPythonInDirectory(options.module, allow_test=True)) if not modules: parser.error('Input directory does not contain any python files!') target_versions = [_GetTargetPythonVersion(m) for m in modules] target_version = target_versions[0] assert target_version in [2, 3] assert all(v == target_version for v in target_versions) current_version = sys.version_info[0] # Trybots run with vpython as default Python, but with a different config # from //.vpython. To make the is_vpython test work, and to match the behavior # of dev machines, the shebang line must be run with python2.7. # # E.g. $HOME/.vpython-root/dd50d3/bin/python # E.g. /b/s/w/ir/cache/vpython/ab5c79/bin/python is_vpython = 'vpython' in sys.executable if not is_vpython or target_version != current_version: # Prevent infinite relaunch if something goes awry. assert not options.did_relaunch # Re-launch using vpython will cause us to pick up modules specified in # //.vpython, but does not cause it to pick up modules defined inline via # [VPYTHON:BEGIN] ... [VPYTHON:END] comments. # TODO(agrieve): Add support for this if the need ever arises. vpython_to_use = {2: 'vpython', 3: 'vpython3'}[target_version] os.execvp(vpython_to_use, [vpython_to_use] + sys.argv + ['--did-relaunch']) paths_set = set() try: for module in modules: _ImportModuleByPath(module) paths_set.update(ComputePythonDependencies()) except Exception: # Output extra diagnostics when loading the script fails. sys.stderr.write('Error running print_python_deps.py.\n') sys.stderr.write('is_vpython={}\n'.format(is_vpython)) sys.stderr.write('did_relanuch={}\n'.format(options.did_relaunch)) sys.stderr.write('python={}\n'.format(sys.executable)) raise for path in options.whitelists: paths_set.update( os.path.abspath(p) for p in _FindPythonInDirectory(path, allow_test=False)) paths = [os.path.relpath(p, options.root) for p in paths_set] normalized_cmdline = _NormalizeCommandLine(options) out = open(options.output, 'w') if options.output else sys.stdout with out: if not options.no_header: out.write('# Generated by running:\n') out.write('# %s\n' % normalized_cmdline) prefix = '//' if options.gn_paths else '' for path in sorted(paths): out.write(prefix + path + '\n') if __name__ == '__main__': sys.exit(main())