Skip to content

Index

code_context_agent.tools.search

BM25Index 

BM25Index()

BM25 full-text search index built from file contents.

Source code in packages/code-context-agent/src/code_context_agent/tools/search/bm25.py 
17
18
19
def __init__(self) -> None:  # noqa: D107
    self._documents: list[dict[str, Any]] = []  # [{path, content, tokens}]
    self._bm25: BM25Okapi | None = None

document_count property 

document_count

Return the number of indexed documents.

from_files classmethod 

from_files(file_paths, repo_path)

Build index from file paths relative to repo root.

Source code in packages/code-context-agent/src/code_context_agent/tools/search/bm25.py 
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
@classmethod
def from_files(cls, file_paths: list[str], repo_path: Path) -> BM25Index:
    """Build index from file paths relative to repo root."""
    index = cls()
    for fp in file_paths:
        full = repo_path / fp
        if not full.is_file():
            continue
        try:
            content = full.read_text(errors="replace")
        except (OSError, UnicodeDecodeError):
            continue
        tokens = _tokenize(content)
        if tokens:
            index._documents.append({"path": fp, "content": content, "tokens": tokens})

    if index._documents:
        corpus = [doc["tokens"] for doc in index._documents]
        index._bm25 = BM25Okapi(corpus)
    return index

search 

search(query, top_k=20)

Search the index with a text query.

Source code in packages/code-context-agent/src/code_context_agent/tools/search/bm25.py 
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
def search(self, query: str, top_k: int = 20) -> list[dict[str, Any]]:
    """Search the index with a text query."""
    if not self._bm25 or not self._documents:
        return []

    query_tokens = _tokenize(query)
    if not query_tokens:
        return []

    scores = self._bm25.get_scores(query_tokens)
    ranked = sorted(enumerate(scores), key=lambda x: x[1], reverse=True)

    results = []
    for idx, score in ranked[:top_k]:
        if score <= 0:
            break
        doc = self._documents[idx]
        matching_lines = _find_matching_lines(doc["content"], query_tokens, max_lines=3)
        results.append(
            {
                "path": doc["path"],
                "score": round(float(score), 4),
                "matching_lines": matching_lines,
            },
        )
    return results

bm25_search(query, repo_path, top_k=20, rebuild=False)

Ranked text search using BM25 algorithm.

Unlike ripgrep (exact pattern matching), BM25 ranks results by relevance — terms appearing in fewer files score higher (TF-IDF-like).

Use for: finding code related to a concept (e.g., "authentication flow"), semantic-like search without embeddings, ranked results when ripgrep returns too many matches.

Parameters:

Name Type Description Default
query str

Search query (natural language or code terms).

 required
repo_path str

Absolute path to the repository.

 required
top_k int

Maximum results to return.

 20
rebuild bool

Force rebuild the index (default: reuse cached).

 False

Returns:

Type Description
str

JSON with ranked results: [{path, score, matching_lines}]
Source code in packages/code-context-agent/src/code_context_agent/tools/search/tools.py 
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
@tool
def bm25_search(query: str, repo_path: str, top_k: int = 20, rebuild: bool = False) -> str:
    """Ranked text search using BM25 algorithm.

    Unlike ripgrep (exact pattern matching), BM25 ranks results by relevance —
    terms appearing in fewer files score higher (TF-IDF-like).

    Use for: finding code related to a concept (e.g., "authentication flow"),
    semantic-like search without embeddings, ranked results when ripgrep
    returns too many matches.

    Args:
        query: Search query (natural language or code terms).
        repo_path: Absolute path to the repository.
        top_k: Maximum results to return.
        rebuild: Force rebuild the index (default: reuse cached).

    Returns:
        JSON with ranked results: [{path, score, matching_lines}]
    """
    repo = Path(repo_path).resolve()

    # Build or reuse cached index
    cache_key = str(repo)
    if rebuild or cache_key not in _indexes:
        # Get file list
        manifest_path = repo / ".code-context" / "files.all.txt"
        if manifest_path.exists():
            files = [line.strip() for line in manifest_path.read_text().splitlines() if line.strip()]
        else:
            # Fallback: glob for files, skipping known non-text paths
            files = [str(p.relative_to(repo)) for p in repo.rglob("*") if p.is_file() and not _should_skip(p)]
            files = files[:5000]  # Cap for safety

        _indexes[cache_key] = BM25Index.from_files(files, repo)

    index = _indexes[cache_key]
    results = index.search(query, top_k=top_k)

    return json.dumps(
        {
            "status": "success",
            "query": query,
            "total_documents": index.document_count,
            "results": results,
            "count": len(results),
        },
    )