#!/usr/bin/env python """ runtests.py [OPTIONS] [-- ARGS] Run tests, building the project first. Examples:: $ python runtests.py $ python runtests.py -s {SAMPLE_SUBMODULE} $ python runtests.py -t {SAMPLE_TEST} $ python runtests.py --ipython $ python runtests.py --python somescript.py $ python runtests.py --bench $ python runtests.py --no-build --bench signal.LTI Run a debugger: $ gdb --args python runtests.py [...other args...] Generate C code coverage listing under build/lcov/: (requires http://ltp.sourceforge.net/coverage/lcov.php) $ python runtests.py --gcov [...other args...] $ python runtests.py --lcov-html """ # # This is a generic test runner script for projects using NumPy's test # framework. Change the following values to adapt to your project: # PROJECT_MODULE = "scipy" PROJECT_ROOT_FILES = ['scipy', 'LICENSE.txt', 'setup.py'] SAMPLE_TEST = "scipy.fftpack.tests.test_real_transforms::TestIDSTIIIInt" SAMPLE_SUBMODULE = "optimize" EXTRA_PATH = ['/usr/lib/ccache', '/usr/lib/f90cache', '/usr/local/lib/ccache', '/usr/local/lib/f90cache'] # --------------------------------------------------------------------- if __doc__ is None: __doc__ = "Run without -OO if you want usage info" else: __doc__ = __doc__.format(**globals()) import sys import os import errno # the following multiprocessing import is necessary to prevent tests that use # multiprocessing from hanging on >= Python3.8 (macOS) using pytest. Just the # import is enough... import multiprocessing # In case we are run from the source directory, we don't want to import the # project from there: sys.path.pop(0) from argparse import ArgumentParser, REMAINDER import contextlib import shutil import subprocess import time import datetime from types import ModuleType as new_module # noqa: E402 ROOT_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__))) def main(argv): parser = ArgumentParser(usage=__doc__.lstrip()) parser.add_argument("--verbose", "-v", action="count", default=1, help="more verbosity") parser.add_argument("--no-build", "-n", action="store_true", default=False, help="do not build the project (use system installed version)") parser.add_argument("--build-only", "-b", action="store_true", default=False, help="just build, do not run any tests") parser.add_argument("--doctests", action="store_true", default=False, help="Run doctests in module") parser.add_argument("--refguide-check", action="store_true", default=False, help="Run refguide check (do not run regular tests.)") parser.add_argument("--coverage", action="store_true", default=False, help=("report coverage of project code. HTML output" " goes under build/coverage")) parser.add_argument("--gcov", action="store_true", default=False, help=("enable C code coverage via gcov (requires GCC)." " gcov output goes to build/**/*.gc*")) parser.add_argument("--lcov-html", action="store_true", default=False, help=("produce HTML for C code coverage information " "from a previous run with --gcov. " "HTML output goes to build/lcov/")) parser.add_argument("--mode", "-m", default="fast", help="'fast', 'full', or something that could be " "passed to nosetests -A [default: fast]") parser.add_argument("--submodule", "-s", default=None, help="Submodule whose tests to run (cluster," " constants, ...)") parser.add_argument("--pythonpath", "-p", default=None, help="Paths to prepend to PYTHONPATH") parser.add_argument("--tests", "-t", action='append', help="Specify tests to run") parser.add_argument("--python", action="store_true", help="Start a Python shell with PYTHONPATH set") parser.add_argument("--ipython", "-i", action="store_true", help="Start IPython shell with PYTHONPATH set") parser.add_argument("--shell", action="store_true", help="Start Unix shell with PYTHONPATH set") parser.add_argument("--debug", "-g", action="store_true", help="Debug build") parser.add_argument("--parallel", "-j", type=int, default=1, help="Number of parallel jobs for build and testing") parser.add_argument("--show-build-log", action="store_true", help="Show build output rather than using a log file") parser.add_argument("--bench", action="store_true", help="Run benchmark suite instead of test suite") parser.add_argument("--bench-compare", action="append", metavar="BEFORE", help=("Compare benchmark results of current HEAD to" " BEFORE. Use an additional " "--bench-compare=COMMIT to override HEAD with" " COMMIT. Note that you need to commit your " "changes first!" )) parser.add_argument("args", metavar="ARGS", default=[], nargs=REMAINDER, help="Arguments to pass to Nose, Python or shell") parser.add_argument("--pep8", action="store_true", default=False, help="Perform pep8 check with flake8.") parser.add_argument("--mypy", action="store_true", default=False, help="Run mypy on the codebase") parser.add_argument("--doc", action="append", nargs="?", const="html-scipyorg", help="Build documentation") args = parser.parse_args(argv) if args.pep8: # Lint the source using the configuration in tox.ini. os.system("flake8 scipy benchmarks/benchmarks") # Lint just the diff since branching off of main using a # stricter configuration. lint_diff = os.path.join(ROOT_DIR, 'tools', 'lint_diff.py') os.system(lint_diff) sys.exit(0) if args.mypy: sys.exit(run_mypy(args)) if args.bench_compare: args.bench = True args.no_build = True # ASV does the building if args.lcov_html: # generate C code coverage output lcov_generate() sys.exit(0) if args.pythonpath: for p in reversed(args.pythonpath.split(os.pathsep)): sys.path.insert(0, p) if args.gcov: gcov_reset_counters() if args.debug and args.bench: print("*** Benchmarks should not be run against debug version; " "remove -g flag ***") if not args.no_build: site_dir = build_project(args) sys.path.insert(0, site_dir) os.environ['PYTHONPATH'] = \ os.pathsep.join((site_dir, os.environ.get('PYTHONPATH', ''))) extra_argv = args.args[:] if extra_argv and extra_argv[0] == '--': extra_argv = extra_argv[1:] if args.python: if extra_argv: # Don't use subprocess, since we don't want to include the # current path in PYTHONPATH. sys.argv = extra_argv with open(extra_argv[0], 'r') as f: script = f.read() sys.modules['__main__'] = new_module('__main__') ns = dict(__name__='__main__', __file__=extra_argv[0]) exec(script, ns) sys.exit(0) else: import code code.interact() sys.exit(0) if args.ipython: import IPython IPython.embed(user_ns={}) sys.exit(0) if args.shell: shell = os.environ.get('SHELL', 'sh') print("Spawning a Unix shell...") os.execv(shell, [shell] + extra_argv) sys.exit(1) if args.doc: cmd = ["make", "-Cdoc", 'PYTHON="{}"'.format(sys.executable)] cmd += args.doc if args.parallel: cmd.append('SPHINXOPTS="-j{}"'.format(args.parallel)) subprocess.run(cmd, check=True) sys.exit(0) if args.coverage: dst_dir = os.path.join(ROOT_DIR, 'build', 'coverage') fn = os.path.join(dst_dir, 'coverage_html.js') if os.path.isdir(dst_dir) and os.path.isfile(fn): shutil.rmtree(dst_dir) extra_argv += ['--cov-report=html:' + dst_dir] if args.refguide_check: cmd = [os.path.join(ROOT_DIR, 'tools', 'refguide_check.py'), '--doctests'] if args.verbose: cmd += ['-' + 'v'*args.verbose] if args.submodule: cmd += [args.submodule] os.execv(sys.executable, [sys.executable] + cmd) sys.exit(0) if args.bench: # Run ASV items = extra_argv if args.tests: items += args.tests if args.submodule: items += [args.submodule] bench_args = [] for a in items: bench_args.extend(['--bench', a]) if not args.bench_compare: import scipy print("Running benchmarks for Scipy version %s at %s" % (scipy.__version__, scipy.__file__)) cmd = ['asv', 'run', '--dry-run', '--show-stderr', '--python=same'] + bench_args retval = run_asv(cmd) sys.exit(retval) else: if len(args.bench_compare) == 1: commit_a = args.bench_compare[0] commit_b = 'HEAD' elif len(args.bench_compare) == 2: commit_a, commit_b = args.bench_compare else: p.error("Too many commits to compare benchmarks for") # Check for uncommitted files if commit_b == 'HEAD': r1 = subprocess.call(['git', 'diff-index', '--quiet', '--cached', 'HEAD']) r2 = subprocess.call(['git', 'diff-files', '--quiet']) if r1 != 0 or r2 != 0: print("*"*80) print("WARNING: you have uncommitted changes --- " "these will NOT be benchmarked!") print("*"*80) # Fix commit ids (HEAD is local to current repo) p = subprocess.Popen(['git', 'rev-parse', commit_b], stdout=subprocess.PIPE) out, err = p.communicate() commit_b = out.strip() p = subprocess.Popen(['git', 'rev-parse', commit_a], stdout=subprocess.PIPE) out, err = p.communicate() commit_a = out.strip() cmd = ['asv', 'continuous', '--show-stderr', '--factor', '1.05', commit_a, commit_b] + bench_args run_asv(cmd) sys.exit(1) if args.build_only: sys.exit(0) else: try: test, version, mod_path = import_module() except ImportError: # this may fail when running with --no-build, so try to detect # an installed scipy in a subdir inside a repo dst_dir = os.path.join(ROOT_DIR, 'build', 'testenv') from sysconfig import get_path py_path = get_path('platlib') site_dir = os.path.join(dst_dir, get_path_suffix(py_path, 3)) print("Trying to import scipy from development installed path at:", site_dir) sys.path.insert(0, site_dir) os.environ['PYTHONPATH'] = \ os.pathsep.join((site_dir, os.environ.get('PYTHONPATH', ''))) test, version, mod_path = import_module() if args.submodule: tests = [PROJECT_MODULE + "." + args.submodule] elif args.tests: tests = args.tests else: tests = None # Run the tests if not args.no_build: test_dir = site_dir else: test_dir = os.path.join(ROOT_DIR, 'build', 'test') if not os.path.isdir(test_dir): os.makedirs(test_dir) shutil.copyfile(os.path.join(ROOT_DIR, '.coveragerc'), os.path.join(test_dir, '.coveragerc')) cwd = os.getcwd() try: os.chdir(test_dir) print("Running tests for {} version:{}, installed at:{}".format( PROJECT_MODULE, version, mod_path)) result = test(args.mode, verbose=args.verbose, extra_argv=extra_argv, doctests=args.doctests, coverage=args.coverage, tests=tests, parallel=args.parallel) finally: os.chdir(cwd) if isinstance(result, bool): sys.exit(0 if result else 1) elif result.wasSuccessful(): sys.exit(0) else: sys.exit(1) def import_module(): """ Function of import project module. """ __import__(PROJECT_MODULE) test = sys.modules[PROJECT_MODULE].test version = sys.modules[PROJECT_MODULE].__version__ mod_path = sys.modules[PROJECT_MODULE].__file__ mod_path = os.path.abspath(os.path.join(os.path.dirname(mod_path))) return test, version, mod_path def get_path_suffix(current_path, levels=3): """ This utility function is only needed for a single use further down, in order to grab the last `levels` subdirs from the input path. It'll always resolve to something like ``lib/python3.X/site-packages``. Site-packages is actually 3 levels deep on all platforms, so this function should suffice. """ current_new = current_path for i in range(levels): current_new = os.path.dirname(current_new) return os.path.relpath(current_path, current_new) def build_project(args): """ Build a dev version of the project. Returns ------- site_dir site-packages directory where it was installed """ root_ok = [os.path.exists(os.path.join(ROOT_DIR, fn)) for fn in PROJECT_ROOT_FILES] if not all(root_ok): print("To build the project, run runtests.py in " "git checkout or unpacked source") sys.exit(1) dst_dir = os.path.join(ROOT_DIR, 'build', 'testenv') env = dict(os.environ) cmd = [sys.executable, 'setup.py'] # Always use ccache, if installed env['PATH'] = os.pathsep.join(EXTRA_PATH + env.get('PATH', '').split(os.pathsep)) if args.debug or args.gcov: # assume everyone uses gcc/gfortran env['OPT'] = '-O0 -ggdb' env['FOPT'] = '-O0 -ggdb' if args.gcov: from sysconfig import get_config_vars cvars = get_config_vars() env['OPT'] = '-O0 -ggdb' env['FOPT'] = '-O0 -ggdb' env['CC'] = env.get('CC', cvars['CC']) + ' --coverage' env['CXX'] = env.get('CXX', cvars['CXX']) + ' --coverage' env['F77'] = 'gfortran --coverage ' env['F90'] = 'gfortran --coverage ' env['LDSHARED'] = cvars['LDSHARED'] + ' --coverage' env['LDFLAGS'] = " ".join(cvars['LDSHARED'].split()[1:]) +\ ' --coverage' cmd += ['build'] if args.parallel > 1: cmd += ['-j', str(args.parallel)] # Install; avoid producing eggs so SciPy can be imported from dst_dir. cmd += ['install', '--prefix=' + dst_dir, '--single-version-externally-managed', '--record=' + dst_dir + 'tmp_install_log.txt'] from sysconfig import get_path py_path = get_path('platlib') site_dir = os.path.join(dst_dir, get_path_suffix(py_path, 3)) # easy_install won't install to a path that Python by default cannot see # and isn't on the PYTHONPATH. Plus, it has to exist. if not os.path.exists(site_dir): os.makedirs(site_dir) env['PYTHONPATH'] = os.pathsep.join((site_dir, env.get('PYTHONPATH', ''))) log_filename = os.path.join(ROOT_DIR, 'build.log') start_time = datetime.datetime.now() if args.show_build_log: ret = subprocess.call(cmd, env=env, cwd=ROOT_DIR) else: log_filename = os.path.join(ROOT_DIR, 'build.log') print("Building, see build.log...") with open(log_filename, 'w') as log: p = subprocess.Popen(cmd, env=env, stdout=log, stderr=log, cwd=ROOT_DIR) try: # Wait for it to finish, and print something to indicate the # process is alive, but only if the log file has grown (to # allow continuous integration environments kill a hanging # process accurately if it produces no output) last_blip = time.time() last_log_size = os.stat(log_filename).st_size while p.poll() is None: time.sleep(0.5) if time.time() - last_blip > 60: log_size = os.stat(log_filename).st_size if log_size > last_log_size: elapsed = datetime.datetime.now() - start_time print(" ... build in progress ({0} " "elapsed)".format(elapsed)) last_blip = time.time() last_log_size = log_size ret = p.wait() except: # noqa: E722 p.terminate() raise elapsed = datetime.datetime.now() - start_time if ret == 0: print("Build OK ({0} elapsed)".format(elapsed)) else: if not args.show_build_log: with open(log_filename, 'r') as f: print(f.read()) print("Build failed! ({0} elapsed)".format(elapsed)) sys.exit(1) return site_dir # # GCOV support # def gcov_reset_counters(): print("Removing previous GCOV .gcda files...") build_dir = os.path.join(ROOT_DIR, 'build') for dirpath, dirnames, filenames in os.walk(build_dir): for fn in filenames: if fn.endswith('.gcda') or fn.endswith('.da'): pth = os.path.join(dirpath, fn) os.unlink(pth) # # LCOV support # LCOV_OUTPUT_FILE = os.path.join(ROOT_DIR, 'build', 'lcov.out') LCOV_HTML_DIR = os.path.join(ROOT_DIR, 'build', 'lcov') def lcov_generate(): try: os.unlink(LCOV_OUTPUT_FILE) except OSError: pass try: shutil.rmtree(LCOV_HTML_DIR) except OSError: pass print("Capturing lcov info...") subprocess.call(['lcov', '-q', '-c', '-d', os.path.join(ROOT_DIR, 'build'), '-b', ROOT_DIR, '--output-file', LCOV_OUTPUT_FILE]) print("Generating lcov HTML output...") ret = subprocess.call(['genhtml', '-q', LCOV_OUTPUT_FILE, '--output-directory', LCOV_HTML_DIR, '--legend', '--highlight']) if ret != 0: print("genhtml failed!") else: print("HTML output generated under build/lcov/") @contextlib.contextmanager def working_dir(new_dir): current_dir = os.getcwd() try: os.chdir(new_dir) yield finally: os.chdir(current_dir) def run_mypy(args): if args.no_build: raise ValueError('Cannot run mypy with --no-build') try: import mypy.api except ImportError as e: raise RuntimeError( "Mypy not found. Please install it by running " "pip install -r mypy_requirements.txt from the repo root" ) from e site_dir = build_project(args) config = os.path.join( os.path.dirname(os.path.abspath(__file__)), "mypy.ini", ) with working_dir(site_dir): # By default mypy won't color the output since it isn't being # invoked from a tty. os.environ['MYPY_FORCE_COLOR'] = '1' # Change to the site directory to make sure mypy doesn't pick # up any type stubs in the source tree. report, errors, status = mypy.api.run([ "--config-file", config, PROJECT_MODULE, ]) print(report, end='') print(errors, end='', file=sys.stderr) return status def run_asv(cmd): cwd = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'benchmarks') # Always use ccache, if installed env = dict(os.environ) env['PATH'] = os.pathsep.join(EXTRA_PATH + env.get('PATH', '').split(os.pathsep)) # Control BLAS/LAPACK threads env['OPENBLAS_NUM_THREADS'] = '1' env['MKL_NUM_THREADS'] = '1' # Limit memory usage sys.path.insert(0, cwd) from benchmarks.common import set_mem_rlimit try: set_mem_rlimit() except (ImportError, RuntimeError): pass # Run try: return subprocess.call(cmd, env=env, cwd=cwd) except OSError as err: if err.errno == errno.ENOENT: print("Error when running '%s': %s\n" % (" ".join(cmd), str(err),)) print("You need to install Airspeed Velocity (https://airspeed-velocity.github.io/asv/)") print("to run Scipy benchmarks") return 1 raise if __name__ == "__main__": main(argv=sys.argv[1:])