grep -rn "def fibonacci" .
ctags -R .
ack --python "TODO"
ripgrep "^class \w+\(.*Mixin\)" --type py        # 现代化的 grep，trigram 索引

# 伪代码：第一代代码搜索的核心
def code_search(query: str, files: list[str]) -> list[str]:
    pattern = re.compile(query)
    hits = []
    for f in files:
        for i, line in enumerate(open(f)):
            if pattern.search(line):
                hits.append((f, i, line))
    return rank_by_tfidf(hits, query)

# 伪代码：Sourcegraph-style 跨仓库 "find references"
def find_references(symbol_id: str, scip_index: Graph) -> list[Location]:
    # symbol_id 是跨仓库稳定的全局符号，例如 "github.com/foo/bar/pkg.HandleRequest"
    # SCIP 图里每个节点是一个符号，每条边是 "definition" / "reference" / "implementation"
    return [edge.location for edge in scip_index.edges_to(symbol_id)
            if edge.kind in {"reference", "implementation"}]

// 找所有 "未经验证就被传给 sql.Exec 的字符串"——SQL 注入
import go

from DataFlow::Node source, DataFlow::Node sink
where
  source instanceof UntrustedFlowSource and
  sink.asExpr().getParent*() = any(SqlExec call).getQuery() and
  TaintTracking::localTaint(source, sink)
select sink, "tainted SQL query from $@", source, "untrusted input"

// Joern: 找所有 "从 read_input 流到 system 调用"
cpg.call("read_input").reachableBy(cpg.call("system")).l

# Word2Vec Skip-gram 的核心 —— 不到 10 行 PyTorch
import torch, torch.nn as nn, torch.nn.functional as F

class SkipGram(nn.Module):
    def __init__(self, vocab_size, dim=128):
        super().__init__()
        self.in_emb  = nn.Embedding(vocab_size, dim)   # 中心词 embedding
        self.out_emb = nn.Embedding(vocab_size, dim)   # 上下文 embedding

    def forward(self, center, context):
        # logits[i] = <in_emb[center], out_emb[i]>
        v = self.in_emb(center)                # (B, d)
        logits = v @ self.out_emb.weight.T     # (B, vocab)
        return F.cross_entropy(logits, context)

# 代码向量化搜索：把整个仓库的函数都 encode 成 (N, d) 的矩阵
import faiss, numpy as np
from sentence_transformers import SentenceTransformer

encoder = SentenceTransformer('microsoft/codebert-base')
funcs = [open(p).read() for p in glob('repo/**/*.py')]   # 假装 N = 1e5
V = encoder.encode(funcs, normalize_embeddings=True)     # (N, 768)

index = faiss.IndexHNSWFlat(768, M=32)
index.add(V.astype('float32'))

q = encoder.encode(["read a file line by line"], normalize_embeddings=True)
D, I = index.search(q.astype('float32'), k=5)            # 找 top-5 最相似函数

"sum a list"  →  [Encoder LSTM]  →  c  →  [Decoder LSTM]  →  "def sum_list(xs): return sum(xs)"
                  压缩信息              定长向量            自回归展开

# 训练样本（CodeSearchNet, CONALA 等数据集都是这种结构）
{"comment": "Compute factorial of n recursively",
 "code":    "def fact(n):\n    return 1 if n <= 1 else n * fact(n - 1)"}

"""
Minimal seq2seq demo: train on (comment, code) pairs, sample code from a comment.

Run:
    python seq2seq_demo.py            # train ~30s on CPU, then sample
"""
import torch, torch.nn as nn, torch.nn.functional as F
import tiktoken

# ---------- 1. 玩具数据集：(注释, 代码) 配对 ----------
PAIRS = [
    ("add two numbers",                "def add(a, b):\n    return a + b"),
    ("multiply two numbers",           "def mul(a, b):\n    return a * b"),
    ("subtract b from a",              "def sub(a, b):\n    return a - b"),
    ("square a number",                "def square(x):\n    return x * x"),
    ("compute factorial recursively",  "def fact(n):\n    return 1 if n <= 1 else n * fact(n-1)"),
    ("return the maximum of two",      "def max2(a, b):\n    return a if a > b else b"),
    ("check if number is even",        "def is_even(n):\n    return n % 2 == 0"),
    ("sum a list",                     "def sum_list(xs):\n    return sum(xs)"),
    ("length of a list",               "def length(xs):\n    return len(xs)"),
    ("reverse a list",                 "def reverse(xs):\n    return xs[::-1]"),
] * 80   # 重复多次让小模型学得动

# 复用本项目的 tiktoken GPT-2 BPE，再加一个 <bos>/<eos>/<pad>
enc = tiktoken.get_encoding("gpt2")
BOS, EOS, PAD = 50256, 50257, 50258      # 只在本 demo 内有效
VOCAB = 50259

def encode(s, add_eos=True):
    ids = enc.encode_ordinary(s)
    return ids + ([EOS] if add_eos else [])

# 把每条样本变成 (src_ids, tgt_ids)
data = [(torch.tensor(encode(c)), torch.tensor([BOS] + encode(k))) for c, k in PAIRS]

# ---------- 2. 模型：Encoder LSTM + Bahdanau Attention + Decoder LSTM ----------
class Encoder(nn.Module):
    def __init__(self, vocab, d=128):
        super().__init__()
        self.emb = nn.Embedding(vocab, d)
        self.rnn = nn.LSTM(d, d, batch_first=True, bidirectional=True)
        self.proj = nn.Linear(2*d, d)             # 把 BiLSTM 输出降回 d 维

    def forward(self, src):
        h, _ = self.rnn(self.emb(src))            # (B, T, 2d)
        return self.proj(h)                       # (B, T, d)

class Attention(nn.Module):
    """Bahdanau (additive) attention —— seq2seq 时代的标志性发明。"""
    def __init__(self, d=128):
        super().__init__()
        self.W = nn.Linear(2*d, d); self.v = nn.Linear(d, 1)

    def forward(self, dec_h, enc_h):
        # dec_h: (B, d)   enc_h: (B, T, d)
        T = enc_h.size(1)
        cat = torch.cat([dec_h.unsqueeze(1).expand(-1, T, -1), enc_h], dim=-1)
        score = self.v(torch.tanh(self.W(cat))).squeeze(-1)   # (B, T)
        a = F.softmax(score, dim=-1)
        ctx = (a.unsqueeze(-1) * enc_h).sum(dim=1)            # (B, d)
        return ctx

class Decoder(nn.Module):
    def __init__(self, vocab, d=128):
        super().__init__()
        self.emb = nn.Embedding(vocab, d)
        self.rnn = nn.LSTMCell(2*d, d)            # 输入 = emb + context
        self.attn = Attention(d)
        self.out = nn.Linear(d, vocab)

    def step(self, prev_tok, h, c, enc_h):
        ctx = self.attn(h, enc_h)                              # (B, d)
        x = torch.cat([self.emb(prev_tok), ctx], dim=-1)       # (B, 2d)
        h, c = self.rnn(x, (h, c))
        return self.out(h), h, c                               # logits, new state

class Seq2Seq(nn.Module):
    def __init__(self, vocab, d=128):
        super().__init__()
        self.enc = Encoder(vocab, d); self.dec = Decoder(vocab, d); self.d = d

    def forward(self, src, tgt):
        enc_h = self.enc(src)                                  # (1, T_src, d)
        h = enc_h.mean(dim=1); c = torch.zeros_like(h)         # 初始化 decoder state
        loss = 0.0
        for t in range(tgt.size(1) - 1):
            logits, h, c = self.dec.step(tgt[:, t], h, c, enc_h)
            loss = loss + F.cross_entropy(logits, tgt[:, t+1])
        return loss / (tgt.size(1) - 1)

    @torch.no_grad()
    def generate(self, src, max_len=80):
        enc_h = self.enc(src)
        h = enc_h.mean(dim=1); c = torch.zeros_like(h)
        cur = torch.tensor([[BOS]])
        out = []
        for _ in range(max_len):
            logits, h, c = self.dec.step(cur[:, -1], h, c, enc_h)
            nxt = logits.argmax(dim=-1)
            if nxt.item() == EOS: break
            out.append(nxt.item())
            cur = torch.cat([cur, nxt.unsqueeze(0)], dim=1)
        return enc.decode(out)

# ---------- 3. 训练 ----------
torch.manual_seed(0)
model = Seq2Seq(VOCAB)
opt = torch.optim.AdamW(model.parameters(), lr=3e-3)
for epoch in range(8):
    total = 0.0
    for src, tgt in data:
        loss = model(src.unsqueeze(0), tgt.unsqueeze(0))
        opt.zero_grad(); loss.backward(); opt.step()
        total += loss.item()
    print(f"epoch {epoch}  loss={total/len(data):.3f}")

# ---------- 4. 推理：自然语言 → 代码 ----------
for prompt in ["add two numbers", "reverse a list", "check if number is even"]:
    src = torch.tensor(encode(prompt)).unsqueeze(0)
    print(f"\n>>> {prompt}\n{model.generate(src)}")

# 这就是 model.py:46-49 + 67 那个 tril mask 的全部意义
torch.tril(torch.ones(T, T))             # 下三角为 1
att = att.masked_fill(mask == 0, -inf)   # 上三角设 -inf —— 看不到未来

src: "compute factorial recursively"
tgt: "def fact(n): return 1 if n<=1 else n*fact(n-1)"

def fact(n):
    """Compute factorial of n recursively."""   # 这就是注释
    return 1 if n <= 1 else n * fact(n - 1)     # 这就是代码

代际    搜索空间                                      维度类型              单次查询的"操作"
─────  ────────────────────────────────────────────  ────────────────────  ──────────────────────────────────
1.     字符串                                         1 维离散              re.search / 倒排索引查表
2.     字符串 + AST + 引用 + CFG + DFG                多维离散结构          图遍历 / Datalog / Cypher
3.     字符串 + 结构 + ℝ^d                            + 1 个连续维度        q @ V.T  内积
4.     字符串 + 结构 + ℝ^d + 时间轴 T                  + 1 个时序维度        encode → ctx → decode 自回归
5.     字符串 + 结构 + ℝ^d × T × L 层 × H 头           + 深度 + 多头并行     一次 forward = L·H 次高维 attention
                                                       (CodeGPT: L=12, H=12,
                                                        即 144 次/token)

# 五代搜索的统一形式
def search(query, space, metric, k=5):
    """
    space:  被搜索的对象集合（已建索引）
    metric: 衡量 query 和 space 中元素相似度的函数
    """
    return top_k(space, key=lambda x: metric(query, x), k=k)

# 第一代：space = 倒排索引里的文档；metric = TF-IDF / 子串匹配
# 第二代：space = SCIP/CPG 图节点；   metric = 图模式合一 / Datalog 规则可满足性
# 第三代：space = ANN 索引里的向量；  metric = q @ v  (内积 / 余弦)
# 第四代:  space 在每一步动态变化，   metric = decoder_state · encoder_h  (Bahdanau attention)
#         "搜索"沿时间轴展开 T 次
# 第五代:  space = 历史 token 的 K, V 矩阵； metric = q @ k.T （softmax）
#         在 L 层 × H 头的高维空间里反复搜索 L·H·T 次

# model.py:66-70 —— 第五代每一层每一头都在做的搜索
att = (q @ k.transpose(-2, -1)) * (1.0 / math.sqrt(k.size(-1)))   # q · K^T = "和谁相似？"
att = att.masked_fill(self.bias[:, :, :T, :T] == 0, float('-inf'))  # 因果掩码：只搜过去
att = F.softmax(att, dim=-1)                                       # 归一化为软分布
y = att @ v                                                        # 按相似度加权拷贝 V

第一代  shape = (T_query,)                    char level
第二代  shape = (N_nodes, K_edge_types)       discrete graph
第三代  shape = (N_docs, d)                   first continuous axis
第四代  shape = (T, d)                        + time axis
第五代  shape = (T, d, L, H)                  + depth + heads