Stem Docs

stem.util.test_tools

Source code for stem.util.test_tools

# Copyright 2015-2016, Damian Johnson and The Tor Project
# See LICENSE for licensing information

"""
Helper functions for testing.

.. versionadded:: 1.2.0

::

  clean_orphaned_pyc - delete *.pyc files without corresponding *.py

  is_pyflakes_available - checks if pyflakes is available
  is_pycodestyle_available - checks if pycodestyle is available

  stylistic_issues - checks for PEP8 and other stylistic issues
  pyflakes_issues - static checks for problems via pyflakes
"""

import collections
import linecache
import os
import re

import stem.util.conf
import stem.util.system

CONFIG = stem.util.conf.config_dict('test', {
  'pep8.ignore': [],  # TODO: drop with stem 2.x, legacy alias for pycodestyle.ignore
  'pycodestyle.ignore': [],
  'pyflakes.ignore': [],
  'exclude_paths': [],
})


[docs]class Issue(collections.namedtuple('Issue', ['line_number', 'message', 'line'])): """ Issue encountered by pyflakes or pycodestyle. :var int line_number: line number the issue occured on :var str message: description of the issue :var str line: content of the line the issue is about """
[docs]def clean_orphaned_pyc(paths): """ Deletes any file with a \*.pyc extention without a corresponding \*.py. This helps to address a common gotcha when deleting python files... * You delete module 'foo.py' and run the tests to ensure that you haven't broken anything. They pass, however there *are* still some 'import foo' statements that still work because the bytecode (foo.pyc) is still around. * You push your change. * Another developer clones our repository and is confused because we have a bunch of ImportErrors. :param list paths: paths to search for orphaned pyc files :returns: list of absolute paths that were deleted """ orphaned_pyc = [] for path in paths: for pyc_path in stem.util.system.files_with_suffix(path, '.pyc'): py_path = pyc_path[:-1] # If we're running python 3 then the *.pyc files are no longer bundled # with the *.py. Rather, they're in a __pycache__ directory. pycache = '%s__pycache__%s' % (os.path.sep, os.path.sep) if pycache in pyc_path: directory, pycache_filename = pyc_path.split(pycache, 1) if not pycache_filename.endswith('.pyc'): continue # should look like 'test_tools.cpython-32.pyc' py_path = os.path.join(directory, pycache_filename.split('.')[0] + '.py') if not os.path.exists(py_path): orphaned_pyc.append(pyc_path) os.remove(pyc_path) return orphaned_pyc
[docs]def is_pyflakes_available(): """ Checks if pyflakes is availalbe. :returns: **True** if we can use pyflakes and **False** otherwise """ return _module_exists('pyflakes.api') and _module_exists('pyflakes.reporter')
[docs]def is_pycodestyle_available(): """ Checks if pycodestyle is availalbe. :returns: **True** if we can use pycodestyle and **False** otherwise """ if _module_exists('pycodestyle'): import pycodestyle elif _module_exists('pep8'): import pep8 as pycodestyle else: return False if not hasattr(pycodestyle, 'BaseReport'): return False else: return True
[docs]def stylistic_issues(paths, check_newlines = False, check_exception_keyword = False, prefer_single_quotes = False): """ Checks for stylistic issues that are an issue according to the parts of PEP8 we conform to. You can suppress pycodestyle issues by making a 'test' configuration that sets 'pycodestyle.ignore'. For example, with a 'test/settings.cfg' of... :: # pycodestyle compliance issues that we're ignoreing... # # * E111 and E121 four space indentations # * E501 line is over 79 characters pycodestyle.ignore E111 pycodestyle.ignore E121 pycodestyle.ignore E501 pycodestyle.ignore run_tests.py => E402: import stem.util.enum ... you can then run tests with... :: import stem.util.conf test_config = stem.util.conf.get_config('test') test_config.load('test/settings.cfg') issues = stylistic_issues('my_project') If a 'exclude_paths' was set in our test config then we exclude any absolute paths matching those regexes. .. versionchanged:: 1.3.0 Renamed from get_stylistic_issues() to stylistic_issues(). The old name still works as an alias, but will be dropped in Stem version 2.0.0. .. versionchanged:: 1.4.0 Changing tuples in return value to be namedtuple instances, and adding the line that had the issue. .. versionchanged:: 1.4.0 Added the prefer_single_quotes option. :param list paths: paths to search for stylistic issues :param bool check_newlines: check that we have standard newlines (\\n), not windows (\\r\\n) nor classic mac (\\r) :param bool check_exception_keyword: checks that we're using 'as' for exceptions rather than a comma :param bool prefer_single_quotes: standardize on using single rather than double quotes for strings, when reasonable :returns: dict of paths list of :class:`stem.util.test_tools.Issue` instances """ issues = {} ignore_rules = [] ignore_for_file = [] for rule in CONFIG['pycodestyle.ignore'] + CONFIG['pep8.ignore']: if '=>' in rule: path, rule_entry = rule.split('=>', 1) if ':' in rule_entry: rule, code = rule_entry.split(':', 1) ignore_for_file.append((path.strip(), rule.strip(), code.strip())) else: ignore_rules.append(rule) def is_ignored(path, rule, code): for ignored_path, ignored_rule, ignored_code in ignore_for_file: if path.endswith(ignored_path) and ignored_rule == rule and ignored_code == code.strip(): return True return False if is_pycodestyle_available(): if _module_exists('pep8'): import pep8 as pycodestyle else: import pycodestyle class StyleReport(pycodestyle.BaseReport): def __init__(self, options): super(StyleReport, self).__init__(options) def error(self, line_number, offset, text, check): code = super(StyleReport, self).error(line_number, offset, text, check) if code: line = linecache.getline(self.filename, line_number) if not is_ignored(self.filename, code, line): issues.setdefault(self.filename, []).append(Issue(line_number, text, line)) style_checker = pycodestyle.StyleGuide(ignore = ignore_rules, reporter = StyleReport) style_checker.check_files(list(_python_files(paths))) if check_newlines or check_exception_keyword: for path in _python_files(paths): with open(path) as f: file_contents = f.read() lines = file_contents.split('\n') is_block_comment = False for index, line in enumerate(lines): if '"""' in line: is_block_comment = not is_block_comment if check_newlines and '\r' in line: issues.setdefault(path, []).append(Issue(index + 1, 'contains a windows newline', line)) elif check_exception_keyword and line.strip().startswith('except') and line.strip().endswith(', exc:'): # Python 2.6 - 2.7 supports two forms for exceptions... # # except ValueError, exc: # except ValueError as exc: # # The former is the old method and no longer supported in python 3 # going forward. # TODO: This check only works if the exception variable is called # 'exc'. We should generalize this via a regex so other names work # too. issues.setdefault(path, []).append(Issue(index + 1, "except clause should use 'as', not comma", line)) if prefer_single_quotes and line and not is_block_comment: content = line.strip().split('#', 1)[0] if '"' in content and "'" not in content and '"""' not in content and not content.endswith('\\'): # Checking if the line already has any single quotes since that # usually means double quotes are preferable for the content (for # instance "I'm hungry"). Also checking for '\' at the end since # that can indicate a multi-line string. issues.setdefault(path, []).append(Issue(index + 1, 'use single rather than double quotes', line)) return issues
[docs]def pyflakes_issues(paths): """ Performs static checks via pyflakes. False positives can be ignored via 'pyflakes.ignore' entries in our 'test' config. For instance... :: pyflakes.ignore stem/util/test_tools.py => 'pyflakes' imported but unused pyflakes.ignore stem/util/test_tools.py => 'pycodestyle' imported but unused If a 'exclude_paths' was set in our test config then we exclude any absolute paths matching those regexes. Issue strings can start or end with an asterisk to match just against the prefix or suffix. .. versionchanged:: 1.3.0 Renamed from get_pyflakes_issues() to pyflakes_issues(). The old name still works as an alias, but will be dropped in Stem version 2.0.0. .. versionchanged:: 1.4.0 Changing tuples in return value to be namedtuple instances, and adding the line that had the issue. .. versionchanged:: 1.5.0 Support matching against prefix or suffix issue strings. :param list paths: paths to search for problems :returns: dict of paths list of :class:`stem.util.test_tools.Issue` instances """ issues = {} if is_pyflakes_available(): import pyflakes.api import pyflakes.reporter class Reporter(pyflakes.reporter.Reporter): def __init__(self): self._ignored_issues = {} for line in CONFIG['pyflakes.ignore']: path, issue = line.split('=>') self._ignored_issues.setdefault(path.strip(), []).append(issue.strip()) def unexpectedError(self, filename, msg): self._register_issue(filename, None, msg, None) def syntaxError(self, filename, msg, lineno, offset, text): self._register_issue(filename, lineno, msg, text) def flake(self, msg): self._register_issue(msg.filename, msg.lineno, msg.message % msg.message_args, None) def _is_ignored(self, path, issue): # Paths in pyflakes_ignore are relative, so we need to check to see if our # path ends with any of them. for ignored_path, ignored_issues in self._ignored_issues.items(): if path.endswith(ignored_path): is_match = issue in ignored_issues for prefix in [i[:1] for i in ignored_issues if i.endswith('*')]: if issue.startswith(prefix): is_match = True for suffix in [i[1:] for i in ignored_issues if i.startswith('*')]: if issue.endswith(suffix): is_match = True if is_match: return True return False def _register_issue(self, path, line_number, issue, line): if not self._is_ignored(path, issue): if path and line_number and not line: line = linecache.getline(path, line_number).strip() issues.setdefault(path, []).append(Issue(line_number, issue, line)) reporter = Reporter() for path in _python_files(paths): pyflakes.api.checkPath(path, reporter) return issues
def _module_exists(module_name): """ Checks if a module exists. :param str module_name: module to check existance of :returns: **True** if module exists and **False** otherwise """ try: __import__(module_name) return True except ImportError: return False def _python_files(paths): for path in paths: for file_path in stem.util.system.files_with_suffix(path, '.py'): skip = False for exclude_path in CONFIG['exclude_paths']: if re.match(exclude_path, file_path): skip = True break if not skip: yield file_path # TODO: drop with stem 2.x # We renamed our methods to drop a redundant 'get_*' prefix, so alias the old # names for backward compatability, and account for pep8 being renamed to # pycodestyle. get_stylistic_issues = stylistic_issues get_pyflakes_issues = pyflakes_issues is_pep8_available = is_pycodestyle_available