compileall.py 13 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334
  1. """Module/script to byte-compile all .py files to .pyc files.
  2. When called as a script with arguments, this compiles the directories
  3. given as arguments recursively; the -l option prevents it from
  4. recursing into directories.
  5. Without arguments, if compiles all modules on sys.path, without
  6. recursing into subdirectories. (Even though it should do so for
  7. packages -- for now, you'll have to deal with packages separately.)
  8. See module py_compile for details of the actual byte-compilation.
  9. """
  10. import os
  11. import sys
  12. import importlib.util
  13. import py_compile
  14. import struct
  15. from functools import partial
  16. __all__ = ["compile_dir","compile_file","compile_path"]
  17. def _walk_dir(dir, ddir=None, maxlevels=10, quiet=0):
  18. if quiet < 2 and isinstance(dir, os.PathLike):
  19. dir = os.fspath(dir)
  20. if not quiet:
  21. print('Listing {!r}...'.format(dir))
  22. try:
  23. names = os.listdir(dir)
  24. except OSError:
  25. if quiet < 2:
  26. print("Can't list {!r}".format(dir))
  27. names = []
  28. names.sort()
  29. for name in names:
  30. if name == '__pycache__':
  31. continue
  32. fullname = os.path.join(dir, name)
  33. if ddir is not None:
  34. dfile = os.path.join(ddir, name)
  35. else:
  36. dfile = None
  37. if not os.path.isdir(fullname):
  38. yield fullname, ddir
  39. elif (maxlevels > 0 and name != os.curdir and name != os.pardir and
  40. os.path.isdir(fullname) and not os.path.islink(fullname)):
  41. yield from _walk_dir(fullname, ddir=dfile,
  42. maxlevels=maxlevels - 1, quiet=quiet)
  43. def compile_dir(dir, maxlevels=10, ddir=None, force=False, rx=None,
  44. quiet=0, legacy=False, optimize=-1, workers=1,
  45. invalidation_mode=None):
  46. """Byte-compile all modules in the given directory tree.
  47. Arguments (only dir is required):
  48. dir: the directory to byte-compile
  49. maxlevels: maximum recursion level (default 10)
  50. ddir: the directory that will be prepended to the path to the
  51. file as it is compiled into each byte-code file.
  52. force: if True, force compilation, even if timestamps are up-to-date
  53. quiet: full output with False or 0, errors only with 1,
  54. no output with 2
  55. legacy: if True, produce legacy pyc paths instead of PEP 3147 paths
  56. optimize: optimization level or -1 for level of the interpreter
  57. workers: maximum number of parallel workers
  58. invalidation_mode: how the up-to-dateness of the pyc will be checked
  59. """
  60. ProcessPoolExecutor = None
  61. if workers < 0:
  62. raise ValueError('workers must be greater or equal to 0')
  63. if workers != 1:
  64. try:
  65. # Only import when needed, as low resource platforms may
  66. # fail to import it
  67. from concurrent.futures import ProcessPoolExecutor
  68. except ImportError:
  69. workers = 1
  70. files_and_ddirs = _walk_dir(dir, quiet=quiet, maxlevels=maxlevels,
  71. ddir=ddir)
  72. success = True
  73. if workers != 1 and ProcessPoolExecutor is not None:
  74. # If workers == 0, let ProcessPoolExecutor choose
  75. workers = workers or None
  76. with ProcessPoolExecutor(max_workers=workers) as executor:
  77. results = executor.map(
  78. partial(_compile_file_tuple,
  79. force=force, rx=rx, quiet=quiet,
  80. legacy=legacy, optimize=optimize,
  81. invalidation_mode=invalidation_mode,
  82. ),
  83. files_and_ddirs)
  84. success = min(results, default=True)
  85. else:
  86. for file, dfile in files_and_ddirs:
  87. if not compile_file(file, dfile, force, rx, quiet,
  88. legacy, optimize, invalidation_mode):
  89. success = False
  90. return success
  91. def _compile_file_tuple(file_and_dfile, **kwargs):
  92. """Needs to be toplevel for ProcessPoolExecutor."""
  93. file, dfile = file_and_dfile
  94. return compile_file(file, dfile, **kwargs)
  95. def compile_file(fullname, ddir=None, force=False, rx=None, quiet=0,
  96. legacy=False, optimize=-1,
  97. invalidation_mode=None):
  98. """Byte-compile one file.
  99. Arguments (only fullname is required):
  100. fullname: the file to byte-compile
  101. ddir: if given, the directory name compiled in to the
  102. byte-code file.
  103. force: if True, force compilation, even if timestamps are up-to-date
  104. quiet: full output with False or 0, errors only with 1,
  105. no output with 2
  106. legacy: if True, produce legacy pyc paths instead of PEP 3147 paths
  107. optimize: optimization level or -1 for level of the interpreter
  108. invalidation_mode: how the up-to-dateness of the pyc will be checked
  109. """
  110. success = True
  111. if quiet < 2 and isinstance(fullname, os.PathLike):
  112. fullname = os.fspath(fullname)
  113. name = os.path.basename(fullname)
  114. if ddir is not None:
  115. dfile = os.path.join(ddir, name)
  116. else:
  117. dfile = None
  118. if rx is not None:
  119. mo = rx.search(fullname)
  120. if mo:
  121. return success
  122. if os.path.isfile(fullname):
  123. if legacy:
  124. cfile = fullname + 'c'
  125. else:
  126. if optimize >= 0:
  127. opt = optimize if optimize >= 1 else ''
  128. cfile = importlib.util.cache_from_source(
  129. fullname, optimization=opt)
  130. else:
  131. cfile = importlib.util.cache_from_source(fullname)
  132. cache_dir = os.path.dirname(cfile)
  133. head, tail = name[:-3], name[-3:]
  134. if tail == '.py':
  135. if not force:
  136. try:
  137. mtime = int(os.stat(fullname).st_mtime)
  138. expect = struct.pack('<4sll', importlib.util.MAGIC_NUMBER,
  139. 0, mtime)
  140. with open(cfile, 'rb') as chandle:
  141. actual = chandle.read(12)
  142. if expect == actual:
  143. return success
  144. except OSError:
  145. pass
  146. if not quiet:
  147. print('Compiling {!r}...'.format(fullname))
  148. try:
  149. ok = py_compile.compile(fullname, cfile, dfile, True,
  150. optimize=optimize,
  151. invalidation_mode=invalidation_mode)
  152. except py_compile.PyCompileError as err:
  153. success = False
  154. if quiet >= 2:
  155. return success
  156. elif quiet:
  157. print('*** Error compiling {!r}...'.format(fullname))
  158. else:
  159. print('*** ', end='')
  160. # escape non-printable characters in msg
  161. msg = err.msg.encode(sys.stdout.encoding,
  162. errors='backslashreplace')
  163. msg = msg.decode(sys.stdout.encoding)
  164. print(msg)
  165. except (SyntaxError, UnicodeError, OSError) as e:
  166. success = False
  167. if quiet >= 2:
  168. return success
  169. elif quiet:
  170. print('*** Error compiling {!r}...'.format(fullname))
  171. else:
  172. print('*** ', end='')
  173. print(e.__class__.__name__ + ':', e)
  174. else:
  175. if ok == 0:
  176. success = False
  177. return success
  178. def compile_path(skip_curdir=1, maxlevels=0, force=False, quiet=0,
  179. legacy=False, optimize=-1,
  180. invalidation_mode=None):
  181. """Byte-compile all module on sys.path.
  182. Arguments (all optional):
  183. skip_curdir: if true, skip current directory (default True)
  184. maxlevels: max recursion level (default 0)
  185. force: as for compile_dir() (default False)
  186. quiet: as for compile_dir() (default 0)
  187. legacy: as for compile_dir() (default False)
  188. optimize: as for compile_dir() (default -1)
  189. invalidation_mode: as for compiler_dir()
  190. """
  191. success = True
  192. for dir in sys.path:
  193. if (not dir or dir == os.curdir) and skip_curdir:
  194. if quiet < 2:
  195. print('Skipping current directory')
  196. else:
  197. success = success and compile_dir(
  198. dir,
  199. maxlevels,
  200. None,
  201. force,
  202. quiet=quiet,
  203. legacy=legacy,
  204. optimize=optimize,
  205. invalidation_mode=invalidation_mode,
  206. )
  207. return success
  208. def main():
  209. """Script main program."""
  210. import argparse
  211. parser = argparse.ArgumentParser(
  212. description='Utilities to support installing Python libraries.')
  213. parser.add_argument('-l', action='store_const', const=0,
  214. default=10, dest='maxlevels',
  215. help="don't recurse into subdirectories")
  216. parser.add_argument('-r', type=int, dest='recursion',
  217. help=('control the maximum recursion level. '
  218. 'if `-l` and `-r` options are specified, '
  219. 'then `-r` takes precedence.'))
  220. parser.add_argument('-f', action='store_true', dest='force',
  221. help='force rebuild even if timestamps are up to date')
  222. parser.add_argument('-q', action='count', dest='quiet', default=0,
  223. help='output only error messages; -qq will suppress '
  224. 'the error messages as well.')
  225. parser.add_argument('-b', action='store_true', dest='legacy',
  226. help='use legacy (pre-PEP3147) compiled file locations')
  227. parser.add_argument('-d', metavar='DESTDIR', dest='ddir', default=None,
  228. help=('directory to prepend to file paths for use in '
  229. 'compile-time tracebacks and in runtime '
  230. 'tracebacks in cases where the source file is '
  231. 'unavailable'))
  232. parser.add_argument('-x', metavar='REGEXP', dest='rx', default=None,
  233. help=('skip files matching the regular expression; '
  234. 'the regexp is searched for in the full path '
  235. 'of each file considered for compilation'))
  236. parser.add_argument('-i', metavar='FILE', dest='flist',
  237. help=('add all the files and directories listed in '
  238. 'FILE to the list considered for compilation; '
  239. 'if "-", names are read from stdin'))
  240. parser.add_argument('compile_dest', metavar='FILE|DIR', nargs='*',
  241. help=('zero or more file and directory names '
  242. 'to compile; if no arguments given, defaults '
  243. 'to the equivalent of -l sys.path'))
  244. parser.add_argument('-j', '--workers', default=1,
  245. type=int, help='Run compileall concurrently')
  246. invalidation_modes = [mode.name.lower().replace('_', '-')
  247. for mode in py_compile.PycInvalidationMode]
  248. parser.add_argument('--invalidation-mode',
  249. choices=sorted(invalidation_modes),
  250. help=('set .pyc invalidation mode; defaults to '
  251. '"checked-hash" if the SOURCE_DATE_EPOCH '
  252. 'environment variable is set, and '
  253. '"timestamp" otherwise.'))
  254. args = parser.parse_args()
  255. compile_dests = args.compile_dest
  256. if args.rx:
  257. import re
  258. args.rx = re.compile(args.rx)
  259. if args.recursion is not None:
  260. maxlevels = args.recursion
  261. else:
  262. maxlevels = args.maxlevels
  263. # if flist is provided then load it
  264. if args.flist:
  265. try:
  266. with (sys.stdin if args.flist=='-' else open(args.flist)) as f:
  267. for line in f:
  268. compile_dests.append(line.strip())
  269. except OSError:
  270. if args.quiet < 2:
  271. print("Error reading file list {}".format(args.flist))
  272. return False
  273. if args.invalidation_mode:
  274. ivl_mode = args.invalidation_mode.replace('-', '_').upper()
  275. invalidation_mode = py_compile.PycInvalidationMode[ivl_mode]
  276. else:
  277. invalidation_mode = None
  278. success = True
  279. try:
  280. if compile_dests:
  281. for dest in compile_dests:
  282. if os.path.isfile(dest):
  283. if not compile_file(dest, args.ddir, args.force, args.rx,
  284. args.quiet, args.legacy,
  285. invalidation_mode=invalidation_mode):
  286. success = False
  287. else:
  288. if not compile_dir(dest, maxlevels, args.ddir,
  289. args.force, args.rx, args.quiet,
  290. args.legacy, workers=args.workers,
  291. invalidation_mode=invalidation_mode):
  292. success = False
  293. return success
  294. else:
  295. return compile_path(legacy=args.legacy, force=args.force,
  296. quiet=args.quiet,
  297. invalidation_mode=invalidation_mode)
  298. except KeyboardInterrupt:
  299. if args.quiet < 2:
  300. print("\n[interrupted]")
  301. return False
  302. return True
  303. if __name__ == '__main__':
  304. exit_status = int(not main())
  305. sys.exit(exit_status)