Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
29 changes: 29 additions & 0 deletions analyzer/codechecker_analyzer/cli/analyze.py
Original file line number Diff line number Diff line change
Expand Up @@ -1094,6 +1094,34 @@ def __update_review_status_config(args):
os.symlink(args.review_status_config, rs_config_to_send)


def __update_analysis_config_files(args):
"""
Copy analysis related configuration files (e.g. skipfile)
to report_dir/conf/.
This directory will be included in the ZIP file,
which will be stored on the server.
"""
conf_dir = os.path.join(args.output_path, "conf")

# Remove any config files used during previous analysis
if os.path.isdir(conf_dir):
shutil.rmtree(conf_dir)

# Create a new conf directory
os.makedirs(conf_dir)

def add_file_to_conf_dir(file_path: str):
if not os.path.isfile(file_path):
return

file_path = os.path.abspath(file_path)
filename = os.path.basename(file_path)
shutil.copyfile(file_path, os.path.join(conf_dir, filename))

if 'skipfile' in args:
add_file_to_conf_dir(args.skipfile)


def __cleanup_metadata(metadata_prev, metadata):
""" Cleanup metadata.

Expand Down Expand Up @@ -1455,6 +1483,7 @@ def main(args):

__update_skip_file(args)
__update_review_status_config(args)
__update_analysis_config_files(args)

LOG.debug("Cleanup metadata file started.")
__cleanup_metadata(metadata_prev, metadata)
Expand Down
7 changes: 7 additions & 0 deletions web/client/codechecker_client/cli/store.py
Original file line number Diff line number Diff line change
Expand Up @@ -505,6 +505,13 @@ def assemble_zip(inputs,
files_to_compress[os.path.dirname(review_status_file_path)]\
.add(review_status_file_path)

# Add files from report_dir/conf/ directory
conf_dir = os.path.join(dir_path, "conf")
if os.path.isdir(conf_dir):
for file in os.listdir(os.fsencode(conf_dir)):
conf_file = os.path.join(conf_dir, os.fsdecode(file))
files_to_compress[conf_dir].add(conf_file)

LOG.debug(f"Processing {len(analyzer_result_file_paths)} report files ...")

analyzer_result_file_reports = parse_analyzer_result_files(
Expand Down
91 changes: 57 additions & 34 deletions web/server/codechecker_server/api/mass_store_run.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,11 +15,13 @@
from collections import defaultdict
from datetime import datetime, timedelta
import fnmatch
import hashlib
from hashlib import sha256
import json
import os
from pathlib import Path
import sqlalchemy
from sqlalchemy.orm import Session as SA_Session
import tempfile
import time
from typing import Any, Callable, Dict, List, Optional, Set, Tuple, Union, \
Expand All @@ -46,7 +48,7 @@
from ..database.config_db_model import Product
from ..database.database import DBSession
from ..database.run_db_model import \
AnalysisInfo, AnalysisInfoChecker, AnalyzerStatistic, \
AnalysisInfo, AnalysisInfoChecker, AnalysisInfoFile, AnalyzerStatistic, \
BugPathEvent, BugReportPoint, \
Checker, \
ExtendedReportData, \
Expand Down Expand Up @@ -814,8 +816,8 @@ def __add_file_content(
self,
session: DBSession,
source_file_name: str,
content_hash: Optional[str]
):
content_hash: Optional[str] = None
) -> str:
"""
Add the necessary file contents. If content_hash in None then this
function calculates the content hash. Or if it's available at the
Expand Down Expand Up @@ -871,6 +873,8 @@ def __add_file_content(
# the meantime.
session.rollback()

return content_hash

def __store_checker_identifiers(self, checkers: Set[Tuple[str, str]]):
"""
Stores the identifiers "(analyzer, checker_name)" in the database into
Expand Down Expand Up @@ -1000,6 +1004,32 @@ def __store_analysis_statistics(

session.add(analyzer_statistics)

def __store_analysis_info_files(
self,
session: SA_Session,
analysis_info_id: int,
report_dir_path: str
):
""" Store analyzer related config files (e.g. skipfile) """
conf_dir_path = os.path.join(report_dir_path, "conf")
zip_conf_dir = os.path.join(
self._zip_dir, "reports",
hashlib.md5(conf_dir_path.encode('utf-8')).hexdigest())

if not os.path.isdir(zip_conf_dir):
return

for file in os.listdir(os.fsencode(zip_conf_dir)):
conf_file = os.path.join(zip_conf_dir, os.fsdecode(file))
content_hash = self.__add_file_content(session, conf_file)

if (not session.get(AnalysisInfoFile,
(analysis_info_id, content_hash))):
session.add(AnalysisInfoFile(
analysis_info_id=analysis_info_id,
filename=os.path.basename(conf_file),
content_hash=content_hash))

def __store_analysis_info(
self,
session: DBSession,
Expand All @@ -1012,37 +1042,30 @@ def __store_analysis_info(
analyzer_command.encode("utf-8"),
zlib.Z_BEST_COMPRESSION)

analysis_info_rows = session \
.query(AnalysisInfo) \
.filter(AnalysisInfo.analyzer_command == cmd) \
.all()

if analysis_info_rows:
# It is possible when multiple runs are stored
# simultaneously to the server with the same analysis
# command that multiple entries are stored into the
# database. In this case we will select the first one.
analysis_info = analysis_info_rows[0]
else:
analysis_info = AnalysisInfo(analyzer_command=cmd)

# Obtain the ID eagerly to be able to use the M-to-N table.
session.add(analysis_info)
session.flush()
session.refresh(analysis_info, ["id"])

for analyzer in mip.analyzers:
q = session \
.query(Checker) \
.filter(Checker.analyzer_name == analyzer)
db_checkers = {r.checker_name: r for r in q.all()}

connection_rows = [AnalysisInfoChecker(
analysis_info, db_checkers[chk], is_enabled)
for chk, is_enabled
in mip.checkers.get(analyzer, {}).items()]
for r in connection_rows:
session.add(r)
analysis_info = AnalysisInfo(analyzer_command=cmd)

# Obtain the ID eagerly to be able to use the M-to-N table.
session.add(analysis_info)
session.flush()
session.refresh(analysis_info, ["id"])

for analyzer in mip.analyzers:
q = session \
.query(Checker) \
.filter(Checker.analyzer_name == analyzer)
db_checkers = {r.checker_name: r for r in q.all()}

connection_rows = [AnalysisInfoChecker(
analysis_info, db_checkers[chk], is_enabled)
for chk, is_enabled
in mip.checkers.get(analyzer, {}).items()]
for r in connection_rows:
session.add(r)

if mip.report_dir_path:
self.__store_analysis_info_files(session,
analysis_info.id,
mip.report_dir_path)

run_history.analysis_info.append(analysis_info)
self.__analysis_info[src_dir_path] = analysis_info
Expand Down
16 changes: 15 additions & 1 deletion web/server/codechecker_server/api/report_server.py
Original file line number Diff line number Diff line change
Expand Up @@ -63,7 +63,8 @@
from ..database.config_db_model import Product
from ..database.database import conv, DBSession, escape_like
from ..database.run_db_model import \
AnalysisInfo, AnalysisInfoChecker as DB_AnalysisInfoChecker, \
AnalysisInfo, \
AnalysisInfoChecker as DB_AnalysisInfoChecker, AnalysisInfoFile, \
AnalyzerStatistic, \
BugPathEvent, BugReportPoint, \
CleanupPlan, CleanupPlanReportHash, Checker, Comment, \
Expand Down Expand Up @@ -1723,6 +1724,19 @@ def getAnalysisInfo(self, analysis_info_filter, limit, offset):
checkers[analyzer][checker] = API_AnalysisInfoChecker(
enabled=enabled)

analysis_config_files = session \
.query(AnalysisInfoFile.filename,
FileContent.content) \
.join(FileContent, AnalysisInfoFile.content_hash
== FileContent.content_hash) \
.filter(AnalysisInfoFile.analysis_info_id
== cmd.id).all()

# Append analysis files to the command string
for filename, content in analysis_config_files:
command += f"\n\n{filename}:\n"
command += zlib.decompress(content).decode("utf-8")

res.append(ttypes.AnalysisInfo(
analyzerCommand=html.escape(command),
checkers=checkers))
Expand Down
8 changes: 5 additions & 3 deletions web/server/codechecker_server/database/db_cleanup.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
from typing import Dict

import sqlalchemy
from sqlalchemy import union

from codechecker_api.codeCheckerDBAccess_v6.ttypes import Severity

Expand All @@ -21,7 +22,7 @@

from .database import DBSession
from .run_db_model import \
AnalysisInfo, \
AnalysisInfo, AnalysisInfoFile, \
BugPathEvent, BugReportPoint, \
Comment, Checker, \
File, FileContent, \
Expand Down Expand Up @@ -108,8 +109,9 @@ def remove_unused_files(product):
if total_count:
LOG.debug("%d dangling files deleted.", total_count)

files = session.query(File.content_hash) \
.group_by(File.content_hash)
files = union(
session.query(File.content_hash),
session.query(AnalysisInfoFile.content_hash))

session.query(FileContent) \
.filter(FileContent.content_hash.notin_(files)) \
Expand Down
29 changes: 29 additions & 0 deletions web/server/codechecker_server/database/run_db_model.py
Original file line number Diff line number Diff line change
Expand Up @@ -79,12 +79,41 @@ def __init__(self,
self.enabled = is_enabled


class AnalysisInfoFile(Base):
__tablename__ = "analysis_info_files"

analysis_info_id = Column(Integer,
ForeignKey("analysis_info.id",
deferrable=True,
initially="DEFERRED",
ondelete="CASCADE"),
primary_key=True)

filename = Column(String, nullable=False)

content_hash = Column(String,
ForeignKey("file_contents.content_hash",
deferrable=True,
initially="DEFERRED",
ondelete="CASCADE"),
primary_key=True)

def __init__(self,
analysis_info_id: int,
filename: str,
content_hash: str):
self.analysis_info_id = analysis_info_id
self.filename = filename
self.content_hash = content_hash


class AnalysisInfo(Base):
__tablename__ = "analysis_info"

id = Column(Integer, autoincrement=True, primary_key=True)
analyzer_command = Column(LargeBinary)
available_checkers = relationship(AnalysisInfoChecker, uselist=True)
analyzer_files = relationship(AnalysisInfoFile, uselist=True)

def __init__(self, analyzer_command: bytes):
self.analyzer_command = analyzer_command
Expand Down
5 changes: 5 additions & 0 deletions web/server/codechecker_server/metadata.py
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,8 @@ def __init__(self, metadata_file_path):
self.disabled_checkers: DisabledCheckers = set()
self.checker_to_analyzer: CheckerToAnalyzer = {}

self.report_dir_path = None

self.__metadata_dict: Dict[str, Any] = {}
if os.path.isfile(metadata_file_path):
self.__metadata_dict = cast(Dict[str, Any],
Expand Down Expand Up @@ -184,6 +186,9 @@ def __process_metadata_info_v2(self):
if tool['name'] == 'codechecker' and 'version' in tool:
cc_versions.add(tool['version'])

if tool['name'] == 'codechecker':
self.report_dir_path = tool.get('output_path')

if 'command' in tool:
check_commands.add(' '.join(tool['command']))

Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,51 @@
"""
Add analysis_info_files table

Revision ID: 29e5047b6513
Revises: 198654dac219
Create Date: 2026-03-05 17:35:36.286847
"""

from logging import getLogger

from alembic import op
import sqlalchemy as sa


# Revision identifiers, used by Alembic.
revision = '29e5047b6513'
down_revision = '198654dac219'
branch_labels = None
depends_on = None


def upgrade():
LOG = getLogger("migration/report")
# ### commands auto generated by Alembic - please adjust! ###
op.create_table(
'analysis_info_files',
sa.Column('analysis_info_id', sa.Integer(), nullable=False),
sa.Column('filename', sa.String(), nullable=False),
sa.Column('content_hash', sa.String(), nullable=False),
sa.ForeignKeyConstraint(
['analysis_info_id'], ['analysis_info.id'],
name=op.f(
'fk_analysis_info_files_analysis_info_id_analysis_info'),
ondelete='CASCADE', initially='DEFERRED', deferrable=True),
sa.ForeignKeyConstraint(
['content_hash'], ['file_contents.content_hash'],
name=op.f(
'fk_analysis_info_files_content_hash_file_contents'),
ondelete='CASCADE', initially='DEFERRED', deferrable=True),
sa.PrimaryKeyConstraint(
'analysis_info_id', 'content_hash',
name=op.f('pk_analysis_info_files'))
)
# ### end Alembic commands ###


def downgrade():
LOG = getLogger("migration/report")
# ### commands auto generated by Alembic - please adjust! ###
op.drop_table('analysis_info_files')
# ### end Alembic commands ###
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
-*.txt
Loading
Loading