diff --git a/analyzer/codechecker_analyzer/analysis_manager.py b/analyzer/codechecker_analyzer/analysis_manager.py index 9de00df99e..36aad1188a 100644 --- a/analyzer/codechecker_analyzer/analysis_manager.py +++ b/analyzer/codechecker_analyzer/analysis_manager.py @@ -19,7 +19,7 @@ from functools import lru_cache from threading import Timer -import multiprocess +import multiprocess # type: ignore from codechecker_common.logger import get_logger from codechecker_common.process import kill_process_tree diff --git a/analyzer/codechecker_analyzer/analyzer.py b/analyzer/codechecker_analyzer/analyzer.py index 89861a6c45..990ec8eeb5 100644 --- a/analyzer/codechecker_analyzer/analyzer.py +++ b/analyzer/codechecker_analyzer/analyzer.py @@ -18,7 +18,7 @@ import sys import time -from multiprocess.managers import SyncManager +from multiprocess.managers import SyncManager # type: ignore from codechecker_common.logger import get_logger, DEBUG from codechecker_common.review_status_handler import ReviewStatusHandler diff --git a/analyzer/codechecker_analyzer/analyzers/analyzer_base.py b/analyzer/codechecker_analyzer/analyzers/analyzer_base.py index 4022ca6320..ab14c9ed15 100644 --- a/analyzer/codechecker_analyzer/analyzers/analyzer_base.py +++ b/analyzer/codechecker_analyzer/analyzers/analyzer_base.py @@ -151,6 +151,13 @@ def get_analyzer_checkers(cls): """ raise NotImplementedError("Subclasses should implement this!") + @classmethod + def analyzer_binary(cls) -> str: + """ + Return path to the analyzer binary. + """ + raise NotImplementedError("Subclasses should implement this!") + @classmethod def get_analyzer_config(cls) -> List[AnalyzerConfig]: return [] diff --git a/analyzer/codechecker_analyzer/analyzers/clangsa/analyzer.py b/analyzer/codechecker_analyzer/analyzers/clangsa/analyzer.py index f5e4195920..269f022887 100644 --- a/analyzer/codechecker_analyzer/analyzers/clangsa/analyzer.py +++ b/analyzer/codechecker_analyzer/analyzers/clangsa/analyzer.py @@ -361,7 +361,7 @@ def get_analyzer_checkers( cls, alpha: bool = True, debug: bool = False - ) -> List[str]: + ) -> List[Tuple[str, str]]: """ Return the list of the supported checkers. @@ -425,7 +425,8 @@ def get_checker_config(cls) -> List[analyzer_base.CheckerConfig]: result = [] for cfg, doc in parse_clang_help_page(command, 'OPTIONS:'): - result.append(analyzer_base.CheckerConfig(*cfg.split(':', 1), doc)) + checker, opt = cfg.split(':', 1) + result.append(analyzer_base.CheckerConfig(checker, opt, doc)) return result @@ -439,11 +440,11 @@ def get_analyzer_config(cls) -> List[analyzer_base.AnalyzerConfig]: command.append("-analyzer-config-help") native_config = parse_clang_help_page(command, 'OPTIONS:') - native_config = map( + analyzer_config_list: List[analyzer_base.AnalyzerConfig] = list(map( lambda cfg: analyzer_base.AnalyzerConfig(cfg[0], cfg[1], str), - native_config) + native_config)) - return list(native_config) + list(cls.__additional_analyzer_config) + return analyzer_config_list + list(cls.__additional_analyzer_config) def post_analyze(self, result_handler): """ diff --git a/analyzer/codechecker_analyzer/analyzers/clangtidy/analyzer.py b/analyzer/codechecker_analyzer/analyzers/clangtidy/analyzer.py index 4f32c045a5..b6aac3f9a7 100644 --- a/analyzer/codechecker_analyzer/analyzers/clangtidy/analyzer.py +++ b/analyzer/codechecker_analyzer/analyzers/clangtidy/analyzer.py @@ -351,7 +351,8 @@ def get_checker_config(cls) -> List[analyzer_base.CheckerConfig]: result = [] for cfg, doc in parse_checker_config(help_page): - result.append(analyzer_base.CheckerConfig(*cfg.split(':', 1), doc)) + checker, opt = cfg.split(':', 1) + result.append(analyzer_base.CheckerConfig(checker, opt, doc)) return result diff --git a/analyzer/codechecker_analyzer/analyzers/cppcheck/result_handler.py b/analyzer/codechecker_analyzer/analyzers/cppcheck/result_handler.py index ad3a5939da..def0ecf304 100644 --- a/analyzer/codechecker_analyzer/analyzers/cppcheck/result_handler.py +++ b/analyzer/codechecker_analyzer/analyzers/cppcheck/result_handler.py @@ -21,6 +21,8 @@ from codechecker_common.skiplist_handler import SkipListHandlers from codechecker_common.review_status_handler import ReviewStatusHandler +from codechecker_analyzer.analyzers.cppcheck.analyzer import Cppcheck + from ..result_handler_base import ResultHandler LOG = get_logger('analyzer.cppcheck') @@ -31,6 +33,10 @@ class CppcheckResultHandler(ResultHandler): Create analyzer result file for Cppcheck output. """ + # This attribute is set dynamically when constructing + # this class. + analyzer: Cppcheck + def __init__(self, *args, **kwargs): self.analyzer_info = AnalyzerInfo(name=AnalyzerResult.TOOL_NAME) diff --git a/analyzer/codechecker_analyzer/buildlog/host_check.py b/analyzer/codechecker_analyzer/buildlog/host_check.py index e73187b17a..3774acd958 100644 --- a/analyzer/codechecker_analyzer/buildlog/host_check.py +++ b/analyzer/codechecker_analyzer/buildlog/host_check.py @@ -32,7 +32,7 @@ def check_intercept(env) -> bool: stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, encoding="utf-8", - errors="ignore") + errors="ignore") # type: ignore if not res: return True @@ -74,4 +74,4 @@ def check_ldlogger(env) -> bool: alternative_logger_lib_dir_path = os.path.join( lib_dir_path, 'codechecker_analyzer', 'ld_logger', 'lib', '**', 'ldlogger.so') - return glob.glob(alternative_logger_lib_dir_path, recursive=True) + return bool(glob.glob(alternative_logger_lib_dir_path, recursive=True)) diff --git a/analyzer/codechecker_analyzer/buildlog/log_parser.py b/analyzer/codechecker_analyzer/buildlog/log_parser.py index f0a325ff2e..38c412ad58 100644 --- a/analyzer/codechecker_analyzer/buildlog/log_parser.py +++ b/analyzer/codechecker_analyzer/buildlog/log_parser.py @@ -22,7 +22,7 @@ import sys import tempfile import traceback -from typing import Dict, List, Optional +from typing import Dict, List, Optional, Any from codechecker_analyzer.analyzers.clangsa.analyzer import ClangSA @@ -49,7 +49,7 @@ # The compilation flags of which the prefix is any of these regular expressions # will not be included in the output Clang command. # These flags should be ignored only in case the original compiler is clang. -IGNORED_OPTIONS_CLANG = [ +IGNORED_OPTIONS_LIST_CLANG = [ # Clang gives different warnings than GCC. Thus if these flags are kept, # '-Werror', '-pedantic-errors' the analysis with Clang can fail even # if the compilation passes with GCC. @@ -66,7 +66,7 @@ # The compilation flags of which the prefix is any of these regular expressions # will not be included in the output Clang command. # These flags should be ignored only in case the original compiler is gcc. -IGNORED_OPTIONS_GCC = [ +IGNORED_OPTIONS_LIST_GCC = [ # --- UNKNOWN BY CLANG --- # '-fallow-fetchr-insn', '-fcall-saved-', @@ -191,8 +191,8 @@ '-mabi' ] -IGNORED_OPTIONS_GCC = re.compile('|'.join(IGNORED_OPTIONS_GCC)) -IGNORED_OPTIONS_CLANG = re.compile('|'.join(IGNORED_OPTIONS_CLANG)) +IGNORED_OPTIONS_GCC = re.compile('|'.join(IGNORED_OPTIONS_LIST_GCC)) +IGNORED_OPTIONS_CLANG = re.compile('|'.join(IGNORED_OPTIONS_LIST_CLANG)) # The compilation flags of which the prefix is any of these regular expressions # will not be included in the output Clang command. These flags have further @@ -218,7 +218,7 @@ } -COMPILE_OPTIONS = [ +COMPILE_OPTIONS_LIST = [ '-nostdinc', r'-nostdinc\+\+', '-pedantic', @@ -235,9 +235,9 @@ '-pthread' ] -COMPILE_OPTIONS = re.compile('|'.join(COMPILE_OPTIONS)) +COMPILE_OPTIONS = re.compile('|'.join(COMPILE_OPTIONS_LIST)) -COMPILE_OPTIONS_MERGED = [ +COMPILE_OPTIONS_LIST_MERGED = [ '--sysroot', '-sdkroot', '--include', @@ -253,7 +253,7 @@ '-iwithprefixbefore' ] -INCLUDE_OPTIONS_MERGED = [ +INCLUDE_OPTIONS_LIST_MERGED = [ '-iquote', '-[IF]', '-isystem', @@ -271,10 +271,10 @@ '-rewrite-objc'] COMPILE_OPTIONS_MERGED = \ - re.compile('(' + '|'.join(COMPILE_OPTIONS_MERGED) + ')') + re.compile('(' + '|'.join(COMPILE_OPTIONS_LIST_MERGED) + ')') INCLUDE_OPTIONS_MERGED = \ - re.compile('(' + '|'.join(INCLUDE_OPTIONS_MERGED) + ')') + re.compile('(' + '|'.join(INCLUDE_OPTIONS_LIST_MERGED) + ')') PRECOMPILATION_OPTION = re.compile('-(E|M[G|T|Q|F|J|P|V|M]*)$') @@ -344,14 +344,14 @@ def __hash__(self): (self.compiler, self.language, tuple(self.compiler_flags))) compiler_info: Dict[ImplicitInfoSpecifierKey, dict] = {} - compiler_isexecutable = {} + compiler_isexecutable: Dict[str, bool] = {} # Store the already detected compiler version information. # If the value is False the compiler is not clang otherwise the value # should be a clang version information object. - compiler_versions = {} + compiler_versions: Dict[str, Any] = {} @staticmethod - def is_executable_compiler(compiler): + def is_executable_compiler(compiler: str): if compiler not in ImplicitCompilerInfo.compiler_isexecutable: ImplicitCompilerInfo.compiler_isexecutable[compiler] = \ which(compiler) is not None @@ -405,7 +405,7 @@ def __parse_compiler_includes(compile_cmd: List[str]): start_mark = "#include <...> search starts here:" end_mark = "End of search list." - include_paths = [] + include_paths: List[str] = [] lines = ImplicitCompilerInfo.__get_compiler_err(compile_cmd) if not lines: @@ -571,7 +571,7 @@ def load_compiler_info(file_path: str): for k, v in contents.items(): k = json.loads(k) ICI.compiler_info[ - ICI.ImplicitInfoSpecifierKey(k[0], k[1], tuple(k[2]))] = v + ICI.ImplicitInfoSpecifierKey(k[0], k[1], list(k[2]))] = v @staticmethod def set(details, compiler_info_file=None): @@ -739,7 +739,7 @@ def __get_installed_dir(clang_binary) -> Optional[str]: if 'clang' not in clang_path.name: return None - return clang_path.parent + return str(clang_path.parent) if clang_path.parent else None def __collect_clang_compile_opts(flag_iterator, details): diff --git a/analyzer/codechecker_analyzer/cli/checkers.py b/analyzer/codechecker_analyzer/cli/checkers.py index cbe393925d..5262445548 100644 --- a/analyzer/codechecker_analyzer/cli/checkers.py +++ b/analyzer/codechecker_analyzer/cli/checkers.py @@ -15,7 +15,7 @@ import os import sys from collections import defaultdict -from typing import Dict, Iterable, List, Tuple +from typing import Dict, Iterable, List, Tuple, Union from codechecker_report_converter import twodim @@ -280,7 +280,7 @@ def __print_profiles(args: argparse.Namespace, cl: CheckerLabels): if 'details' in args: header = ['Profile name', 'Description'] - rows = cl.get_description('profile').items() + rows: List[Tuple] = list(cl.get_description('profile').items()) else: header = ['Profile name'] rows = [(key,) for key in cl.get_description('profile')] @@ -301,7 +301,7 @@ def __print_severities(args: argparse.Namespace, cl: CheckerLabels): if 'details' in args: header = ['Severity', 'Description'] - rows = cl.get_description('severity').items() + rows: List[Tuple] = list(cl.get_description('severity').items()) else: header = ['Severity'] rows = [(key,) for key in cl.get_description('severity')] @@ -330,7 +330,7 @@ def __print_guidelines(args: argparse.Namespace, cl: CheckerLabels): header = list(map(__uglify, header)) if args.output_format == 'json': - rows = [(g, sorted(list(r))) for g, r in result.items()] + rows: List[Tuple] = [(g, sorted(list(r))) for g, r in result.items()] else: rows = [(g, ', '.join(sorted(r))) for g, r in result.items()] @@ -475,7 +475,7 @@ def __print_checkers(args: argparse.Namespace, cl: CheckerLabels): checker_info = __get_detailed_checker_info(args, cl) - result = [] + result: List[Tuple] = [] for analyzer in args.analyzers: if labels: checkers = cl.checkers_by_labels(labels, analyzer) @@ -501,7 +501,7 @@ def __print_checkers(args: argparse.Namespace, cl: CheckerLabels): if 'details' in args: header = ['Status', 'Name', 'Analyzer', 'Description', 'Labels'] - rows = list(map(__format_row, result)) + rows: Union[List[List], List[Tuple]] = list(map(__format_row, result)) else: header = ['Name'] rows = [[r[1]] for r in result] @@ -535,7 +535,7 @@ def __print_checker_config(args: argparse.Namespace): if args.output_format in ['csv', 'json']: header = list(map(__uglify, header)) - rows = [] + rows: List[Tuple] = [] analyzer_failures = [] for analyzer in working_analyzers: analyzer_class = analyzer_types.supported_analyzers[analyzer] diff --git a/analyzer/codechecker_analyzer/cli/parse.py b/analyzer/codechecker_analyzer/cli/parse.py index 523426172c..ce67b37dcf 100644 --- a/analyzer/codechecker_analyzer/cli/parse.py +++ b/analyzer/codechecker_analyzer/cli/parse.py @@ -14,7 +14,7 @@ import argparse import os import sys -from typing import Dict, Optional, Set, List +from typing import Dict, Optional, Set, List, Any import json import fnmatch @@ -285,7 +285,11 @@ def get_report_dir_status(compile_commands: List[dict[str, str]], report_dir: str, detailed_flag: bool): - recent, old, failed, missing, analyzed_actions = {}, {}, {}, {}, {} + recent: Dict[str, Dict[str, int]] = {} + old: Dict[str, Dict[str, int]] = {} + failed: Dict[str, Dict[str, int]] = {} + missing: Dict[str, Dict[str, int]] = {} + analyzed_actions: Dict[str, int] = {} for analyzer in supported_analyzers: recent[analyzer] = {} @@ -331,7 +335,7 @@ def get_report_dir_status(compile_commands: List[dict[str, str]], "failed": failed } - out_analyzers = {} + out_analyzers: Dict[str, Any] = {} for analyzer in supported_analyzers: detailed, summary = {}, {} diff --git a/analyzer/codechecker_analyzer/pre_analysis_manager.py b/analyzer/codechecker_analyzer/pre_analysis_manager.py index 73321c0072..0f164abd03 100644 --- a/analyzer/codechecker_analyzer/pre_analysis_manager.py +++ b/analyzer/codechecker_analyzer/pre_analysis_manager.py @@ -17,7 +17,7 @@ import traceback import uuid -import multiprocess +import multiprocess # type: ignore from codechecker_common.logger import get_logger diff --git a/analyzer/requirements.txt b/analyzer/requirements.txt index 1c75e34b7a..4d80ba611a 100644 --- a/analyzer/requirements.txt +++ b/analyzer/requirements.txt @@ -5,6 +5,7 @@ # to let users select a specific compatible version. lxml~=6.0 +lxml-stubs~=0.5.0 portalocker~=3.0 psutil~=7.0 PyYAML~=6.0 @@ -12,5 +13,6 @@ types-PyYAML~=6.0 sarif-tools~=3.0 multiprocess~=0.70 setuptools~=80.0 +types-setuptools~=80.0 semver~=3.0 argcomplete~=3.0 diff --git a/analyzer/tools/merge_clang_extdef_mappings/codechecker_merge_clang_extdef_mappings/cli.py b/analyzer/tools/merge_clang_extdef_mappings/codechecker_merge_clang_extdef_mappings/cli.py index cf3189d21a..ba427d27ea 100644 --- a/analyzer/tools/merge_clang_extdef_mappings/codechecker_merge_clang_extdef_mappings/cli.py +++ b/analyzer/tools/merge_clang_extdef_mappings/codechecker_merge_clang_extdef_mappings/cli.py @@ -21,7 +21,7 @@ # dependencies. if __name__ == '__main__': current_dir = os.path.dirname(os.path.realpath(__file__)) - os.sys.path.append(os.path.dirname(current_dir)) + sys.path.append(os.path.dirname(current_dir)) from codechecker_merge_clang_extdef_mappings import \ merge_clang_extdef_mappings # noqa diff --git a/analyzer/tools/statistics_collector/codechecker_statistics_collector/cli.py b/analyzer/tools/statistics_collector/codechecker_statistics_collector/cli.py index c068a83d4c..ba49fc89cc 100644 --- a/analyzer/tools/statistics_collector/codechecker_statistics_collector/cli.py +++ b/analyzer/tools/statistics_collector/codechecker_statistics_collector/cli.py @@ -11,6 +11,7 @@ import argparse import logging import os +import sys # If we run this script in an environment where # 'codechecker_statistics_collector' module is not available we should add the @@ -20,7 +21,7 @@ # dependencies. if __name__ == '__main__': current_dir = os.path.dirname(os.path.realpath(__file__)) - os.sys.path.append(os.path.dirname(current_dir)) + sys.path.append(os.path.dirname(current_dir)) from codechecker_statistics_collector import post_process_stats # noqa diff --git a/codechecker_common/checker_labels.py b/codechecker_common/checker_labels.py index 7f1f630a85..0d170084a4 100644 --- a/codechecker_common/checker_labels.py +++ b/codechecker_common/checker_labels.py @@ -252,11 +252,11 @@ def labels_of_checker( # cover this case properly. return list(set(labels)) - def get_description(self, label: str) -> Optional[Dict[str, str]]: + def get_description(self, label: str) -> Dict[str, str]: """ Returns the descriptions of the given label's values. """ - return self.__descriptions.get(label) + return self.__descriptions.get(label, {}) def checkers(self, analyzer: Optional[str] = None) -> List[str]: """ diff --git a/pyproject.toml b/pyproject.toml index b547b1b141..593799bf48 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -2,12 +2,34 @@ verbosity = 1 show_error_codes = true +# Packages are explicitly defined at mypy_path/, +# rather than discovered via __init__.py. +explicit_package_bases = true + files = [ - "codechecker_common/" + "codechecker_common/", + "tools/", + "analyzer/" ] mypy_path = [ "analyzer/", + "analyzer/tools/statistics_collector/", + "analyzer/tools/merge_clang_extdef_mappings/", "web/", - "tools/report-converter/" + "tools/report-converter/", + "tools/tu_collector/" +] + +exclude = [ + # Ignore tests + "/tests/libtest/", + "/tests/functional/", + "/tests/unit/", + + # Ignore setup.py files + "/setup.py$", + + # Ignore build directories + "/build/", ] diff --git a/tools/report-converter/codechecker_report_converter/report/__init__.py b/tools/report-converter/codechecker_report_converter/report/__init__.py index 525ced65be..6ae48f3dd5 100644 --- a/tools/report-converter/codechecker_report_converter/report/__init__.py +++ b/tools/report-converter/codechecker_report_converter/report/__init__.py @@ -14,7 +14,8 @@ import logging import os -from typing import Callable, Dict, List, Optional, Protocol, Set, Tuple +from typing import Dict, List, Optional, Set, Tuple +from codechecker_common.skiplist_handler import SkipListHandlers from .. import util @@ -24,11 +25,6 @@ FakeChecker: Tuple[str, str] = ("__FAKE__", "__FAKE__") UnknownChecker: Tuple[str, str] = ("UNKNOWN", "NOT FOUND") - -class SkipListHandlers(Protocol): - should_skip: Callable[[str], bool] - - InvalidFileContentMsg: str = \ "WARNING: source file content is changed or missing. Please re-analyze " \ "your project to update the reports." diff --git a/tools/report-converter/codechecker_report_converter/report/parser/sarif.py b/tools/report-converter/codechecker_report_converter/report/parser/sarif.py index 376403dc86..c727affe13 100644 --- a/tools/report-converter/codechecker_report_converter/report/parser/sarif.py +++ b/tools/report-converter/codechecker_report_converter/report/parser/sarif.py @@ -10,7 +10,7 @@ import logging import os -from sarif import loader +from sarif import loader # type: ignore from typing import Any, Dict, List, Optional, Tuple diff --git a/tools/report-converter/codechecker_report_converter/report/reports.py b/tools/report-converter/codechecker_report_converter/report/reports.py index 65b0bf1c72..2a162d1df5 100644 --- a/tools/report-converter/codechecker_report_converter/report/reports.py +++ b/tools/report-converter/codechecker_report_converter/report/reports.py @@ -10,7 +10,8 @@ from typing import Callable, Iterable, List, Optional, Set -from codechecker_report_converter.report import Report, SkipListHandlers +from codechecker_common.skiplist_handler import SkipListHandlers +from codechecker_report_converter.report import Report from codechecker_report_converter.report.hash import get_report_path_hash LOG = logging.getLogger('report-converter') diff --git a/tools/tu_collector/tu_collector/tu_collector.py b/tools/tu_collector/tu_collector/tu_collector.py index 04fa784442..677c0d38c7 100755 --- a/tools/tu_collector/tu_collector/tu_collector.py +++ b/tools/tu_collector/tu_collector/tu_collector.py @@ -218,7 +218,7 @@ def __eliminate_argument( except subprocess.CalledProcessError as ex: output, rc = ex.output, ex.returncode except OSError as oerr: - output, rc = oerr.strerror, oerr.errno + output, rc = oerr.strerror or "", oerr.errno or 1 if rc != 0: raise IOError(output)