Skip to content

bm25

code_context_agent.tools.search.bm25

BM25 full-text search index for codebase analysis.

BM25Index 

BM25Index()

BM25 full-text search index built from file contents.

Source code in packages/code-context-agent/src/code_context_agent/tools/search/bm25.py 
17
18
19
def __init__(self) -> None:  # noqa: D107
    self._documents: list[dict[str, Any]] = []  # [{path, content, tokens}]
    self._bm25: BM25Okapi | None = None

document_count property 

document_count

Return the number of indexed documents.

from_files classmethod 

from_files(file_paths, repo_path)

Build index from file paths relative to repo root.

Source code in packages/code-context-agent/src/code_context_agent/tools/search/bm25.py 
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
@classmethod
def from_files(cls, file_paths: list[str], repo_path: Path) -> BM25Index:
    """Build index from file paths relative to repo root."""
    index = cls()
    for fp in file_paths:
        full = repo_path / fp
        if not full.is_file():
            continue
        try:
            content = full.read_text(errors="replace")
        except (OSError, UnicodeDecodeError):
            continue
        tokens = _tokenize(content)
        if tokens:
            index._documents.append({"path": fp, "content": content, "tokens": tokens})

    if index._documents:
        corpus = [doc["tokens"] for doc in index._documents]
        index._bm25 = BM25Okapi(corpus)
    return index

search 

search(query, top_k=20)

Search the index with a text query.

Source code in packages/code-context-agent/src/code_context_agent/tools/search/bm25.py 
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
def search(self, query: str, top_k: int = 20) -> list[dict[str, Any]]:
    """Search the index with a text query."""
    if not self._bm25 or not self._documents:
        return []

    query_tokens = _tokenize(query)
    if not query_tokens:
        return []

    scores = self._bm25.get_scores(query_tokens)
    ranked = sorted(enumerate(scores), key=lambda x: x[1], reverse=True)

    results = []
    for idx, score in ranked[:top_k]:
        if score <= 0:
            break
        doc = self._documents[idx]
        matching_lines = _find_matching_lines(doc["content"], query_tokens, max_lines=3)
        results.append(
            {
                "path": doc["path"],
                "score": round(float(score), 4),
                "matching_lines": matching_lines,
            },
        )
    return results