util.py 7.7 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287
  1. from collections import namedtuple
  2. from hashlib import sha256
  3. import os
  4. import shutil
  5. import sys
  6. import fnmatch
  7. from sympy.testing.pytest import XFAIL
  8. def may_xfail(func):
  9. if sys.platform.lower() == 'darwin' or os.name == 'nt':
  10. # sympy.utilities._compilation needs more testing on Windows and macOS
  11. # once those two platforms are reliably supported this xfail decorator
  12. # may be removed.
  13. return XFAIL(func)
  14. else:
  15. return func
  16. class CompilerNotFoundError(FileNotFoundError):
  17. pass
  18. class CompileError (Exception):
  19. """Failure to compile one or more C/C++ source files."""
  20. def get_abspath(path, cwd='.'):
  21. """ Returns the absolute path.
  22. Parameters
  23. ==========
  24. path : str
  25. (relative) path.
  26. cwd : str
  27. Path to root of relative path.
  28. """
  29. if os.path.isabs(path):
  30. return path
  31. else:
  32. if not os.path.isabs(cwd):
  33. cwd = os.path.abspath(cwd)
  34. return os.path.abspath(
  35. os.path.join(cwd, path)
  36. )
  37. def make_dirs(path):
  38. """ Create directories (equivalent of ``mkdir -p``). """
  39. if path[-1] == '/':
  40. parent = os.path.dirname(path[:-1])
  41. else:
  42. parent = os.path.dirname(path)
  43. if len(parent) > 0:
  44. if not os.path.exists(parent):
  45. make_dirs(parent)
  46. if not os.path.exists(path):
  47. os.mkdir(path, 0o777)
  48. else:
  49. assert os.path.isdir(path)
  50. def copy(src, dst, only_update=False, copystat=True, cwd=None,
  51. dest_is_dir=False, create_dest_dirs=False):
  52. """ Variation of ``shutil.copy`` with extra options.
  53. Parameters
  54. ==========
  55. src : str
  56. Path to source file.
  57. dst : str
  58. Path to destination.
  59. only_update : bool
  60. Only copy if source is newer than destination
  61. (returns None if it was newer), default: ``False``.
  62. copystat : bool
  63. See ``shutil.copystat``. default: ``True``.
  64. cwd : str
  65. Path to working directory (root of relative paths).
  66. dest_is_dir : bool
  67. Ensures that dst is treated as a directory. default: ``False``
  68. create_dest_dirs : bool
  69. Creates directories if needed.
  70. Returns
  71. =======
  72. Path to the copied file.
  73. """
  74. if cwd: # Handle working directory
  75. if not os.path.isabs(src):
  76. src = os.path.join(cwd, src)
  77. if not os.path.isabs(dst):
  78. dst = os.path.join(cwd, dst)
  79. if not os.path.exists(src): # Make sure source file extists
  80. raise FileNotFoundError("Source: `{}` does not exist".format(src))
  81. # We accept both (re)naming destination file _or_
  82. # passing a (possible non-existent) destination directory
  83. if dest_is_dir:
  84. if not dst[-1] == '/':
  85. dst = dst+'/'
  86. else:
  87. if os.path.exists(dst) and os.path.isdir(dst):
  88. dest_is_dir = True
  89. if dest_is_dir:
  90. dest_dir = dst
  91. dest_fname = os.path.basename(src)
  92. dst = os.path.join(dest_dir, dest_fname)
  93. else:
  94. dest_dir = os.path.dirname(dst)
  95. if not os.path.exists(dest_dir):
  96. if create_dest_dirs:
  97. make_dirs(dest_dir)
  98. else:
  99. raise FileNotFoundError("You must create directory first.")
  100. if only_update:
  101. # This function is not defined:
  102. # XXX: This branch is clearly not tested!
  103. if not missing_or_other_newer(dst, src): # noqa
  104. return
  105. if os.path.islink(dst):
  106. dst = os.path.abspath(os.path.realpath(dst), cwd=cwd)
  107. shutil.copy(src, dst)
  108. if copystat:
  109. shutil.copystat(src, dst)
  110. return dst
  111. Glob = namedtuple('Glob', 'pathname')
  112. ArbitraryDepthGlob = namedtuple('ArbitraryDepthGlob', 'filename')
  113. def glob_at_depth(filename_glob, cwd=None):
  114. if cwd is not None:
  115. cwd = '.'
  116. globbed = []
  117. for root, dirs, filenames in os.walk(cwd):
  118. for fn in filenames:
  119. # This is not tested:
  120. if fnmatch.fnmatch(fn, filename_glob):
  121. globbed.append(os.path.join(root, fn))
  122. return globbed
  123. def sha256_of_file(path, nblocks=128):
  124. """ Computes the SHA256 hash of a file.
  125. Parameters
  126. ==========
  127. path : string
  128. Path to file to compute hash of.
  129. nblocks : int
  130. Number of blocks to read per iteration.
  131. Returns
  132. =======
  133. hashlib sha256 hash object. Use ``.digest()`` or ``.hexdigest()``
  134. on returned object to get binary or hex encoded string.
  135. """
  136. sh = sha256()
  137. with open(path, 'rb') as f:
  138. for chunk in iter(lambda: f.read(nblocks*sh.block_size), b''):
  139. sh.update(chunk)
  140. return sh
  141. def sha256_of_string(string):
  142. """ Computes the SHA256 hash of a string. """
  143. sh = sha256()
  144. sh.update(string)
  145. return sh
  146. def pyx_is_cplus(path):
  147. """
  148. Inspect a Cython source file (.pyx) and look for comment line like:
  149. # distutils: language = c++
  150. Returns True if such a file is present in the file, else False.
  151. """
  152. with open(path) as fh:
  153. for line in fh:
  154. if line.startswith('#') and '=' in line:
  155. splitted = line.split('=')
  156. if len(splitted) != 2:
  157. continue
  158. lhs, rhs = splitted
  159. if lhs.strip().split()[-1].lower() == 'language' and \
  160. rhs.strip().split()[0].lower() == 'c++':
  161. return True
  162. return False
  163. def import_module_from_file(filename, only_if_newer_than=None):
  164. """ Imports Python extension (from shared object file)
  165. Provide a list of paths in `only_if_newer_than` to check
  166. timestamps of dependencies. import_ raises an ImportError
  167. if any is newer.
  168. Word of warning: The OS may cache shared objects which makes
  169. reimporting same path of an shared object file very problematic.
  170. It will not detect the new time stamp, nor new checksum, but will
  171. instead silently use old module. Use unique names for this reason.
  172. Parameters
  173. ==========
  174. filename : str
  175. Path to shared object.
  176. only_if_newer_than : iterable of strings
  177. Paths to dependencies of the shared object.
  178. Raises
  179. ======
  180. ``ImportError`` if any of the files specified in ``only_if_newer_than`` are newer
  181. than the file given by filename.
  182. """
  183. path, name = os.path.split(filename)
  184. name, ext = os.path.splitext(name)
  185. name = name.split('.')[0]
  186. if sys.version_info[0] == 2:
  187. from imp import find_module, load_module
  188. fobj, filename, data = find_module(name, [path])
  189. if only_if_newer_than:
  190. for dep in only_if_newer_than:
  191. if os.path.getmtime(filename) < os.path.getmtime(dep):
  192. raise ImportError("{} is newer than {}".format(dep, filename))
  193. mod = load_module(name, fobj, filename, data)
  194. else:
  195. import importlib.util
  196. spec = importlib.util.spec_from_file_location(name, filename)
  197. if spec is None:
  198. raise ImportError("Failed to import: '%s'" % filename)
  199. mod = importlib.util.module_from_spec(spec)
  200. spec.loader.exec_module(mod)
  201. return mod
  202. def find_binary_of_command(candidates):
  203. """ Finds binary first matching name among candidates.
  204. Calls ``which`` from shutils for provided candidates and returns
  205. first hit.
  206. Parameters
  207. ==========
  208. candidates : iterable of str
  209. Names of candidate commands
  210. Raises
  211. ======
  212. CompilerNotFoundError if no candidates match.
  213. """
  214. from shutil import which
  215. for c in candidates:
  216. binary_path = which(c)
  217. if c and binary_path:
  218. return c, binary_path
  219. raise CompilerNotFoundError('No binary located for candidates: {}'.format(candidates))
  220. def unique_list(l):
  221. """ Uniquify a list (skip duplicate items). """
  222. result = []
  223. for x in l:
  224. if x not in result:
  225. result.append(x)
  226. return result