mirror of
https://bitbucket.org/siakitem/my-pi.git
synced 2026-08-28 08:35:57 +00:00
feat: vendor grouped tool search
This commit is contained in:
云服务部-叶林立
@@ -0,0 +1,366 @@
|
||||
import { createHash } from "node:crypto";
|
||||
import { chmodSync, mkdirSync, readFileSync, renameSync, writeFileSync } from "node:fs";
|
||||
import { dirname } from "node:path";
|
||||
|
||||
import type { ToolInfo } from "@earendil-works/pi-coding-agent";
|
||||
|
||||
export const CATALOG_VERSION = 1;
|
||||
|
||||
export interface ToolCard {
|
||||
name: string;
|
||||
summary: string;
|
||||
useWhen: string[];
|
||||
avoidWhen: string[];
|
||||
keywords: string[];
|
||||
primaryGroup: string;
|
||||
}
|
||||
|
||||
export interface GroupCard {
|
||||
id: string;
|
||||
title: string;
|
||||
summary: string;
|
||||
useWhen: string[];
|
||||
avoidWhen: string[];
|
||||
tools: string[];
|
||||
}
|
||||
|
||||
export interface ToolCatalog {
|
||||
version: number;
|
||||
manifestHash: string;
|
||||
generatedAt: string;
|
||||
generatedBy?: string;
|
||||
groups: GroupCard[];
|
||||
tools: ToolCard[];
|
||||
}
|
||||
|
||||
export interface CatalogConstraints {
|
||||
maxToolsPerGroup: number;
|
||||
groupOverrides: Record<string, string[]>;
|
||||
}
|
||||
|
||||
interface StoredCatalog {
|
||||
version: number;
|
||||
catalog: ToolCatalog;
|
||||
}
|
||||
|
||||
function compactText(value: unknown, maxLength: number): string {
|
||||
if (typeof value !== "string") return "";
|
||||
return value.replace(/\s+/gu, " ").trim().slice(0, maxLength);
|
||||
}
|
||||
|
||||
function compactStrings(value: unknown, maxItems: number, maxLength: number): string[] {
|
||||
if (!Array.isArray(value)) return [];
|
||||
return value
|
||||
.filter((item): item is string => typeof item === "string")
|
||||
.map((item) => compactText(item, maxLength))
|
||||
.filter(Boolean)
|
||||
.slice(0, maxItems);
|
||||
}
|
||||
|
||||
function normalizeGroupId(value: string): string {
|
||||
return value
|
||||
.trim()
|
||||
.toLowerCase()
|
||||
.replace(/[^a-z0-9]+/gu, "-")
|
||||
.replace(/^-+|-+$/gu, "")
|
||||
.slice(0, 64);
|
||||
}
|
||||
|
||||
function stableValue(value: unknown, seen = new WeakSet<object>()): unknown {
|
||||
if (value === null || typeof value === "string" || typeof value === "number" || typeof value === "boolean") {
|
||||
return value;
|
||||
}
|
||||
if (Array.isArray(value)) return value.map((item) => stableValue(item, seen));
|
||||
if (typeof value !== "object") return undefined;
|
||||
if (seen.has(value)) return "[Circular]";
|
||||
seen.add(value);
|
||||
const output: Record<string, unknown> = {};
|
||||
for (const key of Object.keys(value as Record<string, unknown>).sort()) {
|
||||
const child = stableValue((value as Record<string, unknown>)[key], seen);
|
||||
if (child !== undefined) output[key] = child;
|
||||
}
|
||||
seen.delete(value);
|
||||
return output;
|
||||
}
|
||||
|
||||
export function buildManifestHash(tools: ToolInfo[], constraints: CatalogConstraints): string {
|
||||
const manifest = tools
|
||||
.map((tool) => ({
|
||||
name: tool.name,
|
||||
description: tool.description,
|
||||
parameters: stableValue(tool.parameters),
|
||||
promptGuidelines: tool.promptGuidelines,
|
||||
sourceInfo: stableValue(tool.sourceInfo),
|
||||
}))
|
||||
.sort((left, right) => left.name.localeCompare(right.name));
|
||||
const payload = stableValue({ manifest, constraints });
|
||||
return createHash("sha256").update(JSON.stringify(payload)).digest("hex");
|
||||
}
|
||||
|
||||
export function buildCatalogModelInput(tools: ToolInfo[]): string {
|
||||
const input = tools
|
||||
.map((tool) => ({
|
||||
name: tool.name,
|
||||
description: tool.description,
|
||||
parameters: stableValue(tool.parameters),
|
||||
promptGuidelines: tool.promptGuidelines,
|
||||
sourceInfo: stableValue(tool.sourceInfo),
|
||||
}))
|
||||
.sort((left, right) => left.name.localeCompare(right.name));
|
||||
return JSON.stringify(input);
|
||||
}
|
||||
|
||||
function fallbackGroupKey(tool: ToolInfo): string {
|
||||
const prefix = tool.name.includes("_") ? tool.name.split("_", 1)[0] : "";
|
||||
if (prefix && prefix.length > 1) return normalizeGroupId(prefix);
|
||||
|
||||
const source = stableValue(tool.sourceInfo);
|
||||
if (source && typeof source === "object") {
|
||||
const record = source as Record<string, unknown>;
|
||||
for (const key of ["package", "name", "path", "source"]) {
|
||||
const candidate = record[key];
|
||||
if (typeof candidate !== "string") continue;
|
||||
const parts = candidate.split(/[\\/]/u);
|
||||
const normalized = normalizeGroupId(parts.at(-1)?.replace(/\.[^.]+$/u, "") ?? candidate);
|
||||
if (normalized) return normalized;
|
||||
}
|
||||
}
|
||||
return "other-tools";
|
||||
}
|
||||
|
||||
export function applyCatalogOverrides(
|
||||
groups: GroupCard[],
|
||||
toolCards: ToolCard[],
|
||||
overrides: Record<string, string[]>,
|
||||
): { groups: GroupCard[]; tools: ToolCard[] } {
|
||||
const knownNames = new Set(toolCards.map((tool) => tool.name));
|
||||
const overridden = new Map<string, string>();
|
||||
for (const [rawId, names] of Object.entries(overrides)) {
|
||||
const id = normalizeGroupId(rawId);
|
||||
if (!id) continue;
|
||||
for (const name of names) {
|
||||
if (knownNames.has(name)) overridden.set(name, id);
|
||||
}
|
||||
}
|
||||
if (overridden.size === 0) return { groups, tools: toolCards };
|
||||
|
||||
const byId = new Map<string, GroupCard>();
|
||||
for (const group of groups) {
|
||||
const remaining = group.tools.filter((name) => !overridden.has(name));
|
||||
if (remaining.length > 0) byId.set(group.id, { ...group, tools: remaining });
|
||||
}
|
||||
for (const [name, id] of overridden) {
|
||||
const existing = byId.get(id);
|
||||
if (existing) existing.tools.push(name);
|
||||
else {
|
||||
byId.set(id, {
|
||||
id,
|
||||
title: id.replace(/-/gu, " "),
|
||||
summary: `Configured tool group for ${id.replace(/-/gu, " ")}.`,
|
||||
useWhen: [],
|
||||
avoidWhen: [],
|
||||
tools: [name],
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
const updatedTools = toolCards.map((tool) => ({
|
||||
...tool,
|
||||
primaryGroup: overridden.get(tool.name) ?? tool.primaryGroup,
|
||||
}));
|
||||
return { groups: [...byId.values()], tools: updatedTools };
|
||||
}
|
||||
|
||||
export function createFallbackCatalog(
|
||||
tools: ToolInfo[],
|
||||
manifestHash: string,
|
||||
constraints: CatalogConstraints,
|
||||
): ToolCatalog {
|
||||
const cards = tools.map<ToolCard>((tool) => ({
|
||||
name: tool.name,
|
||||
summary: compactText(tool.description, 240) || `Use the ${tool.name} tool.`,
|
||||
useWhen: [],
|
||||
avoidWhen: [],
|
||||
keywords: tool.name.split(/[_-]/u).filter(Boolean),
|
||||
primaryGroup: fallbackGroupKey(tool),
|
||||
}));
|
||||
|
||||
const buckets = new Map<string, ToolCard[]>();
|
||||
for (const card of cards) {
|
||||
const bucket = buckets.get(card.primaryGroup) ?? [];
|
||||
bucket.push(card);
|
||||
buckets.set(card.primaryGroup, bucket);
|
||||
}
|
||||
|
||||
const groups: GroupCard[] = [];
|
||||
for (const [baseId, bucket] of [...buckets].sort(([left], [right]) => left.localeCompare(right))) {
|
||||
for (let index = 0; index < bucket.length; index += constraints.maxToolsPerGroup) {
|
||||
const chunk = bucket.slice(index, index + constraints.maxToolsPerGroup);
|
||||
const suffix = index === 0 ? "" : `-${Math.floor(index / constraints.maxToolsPerGroup) + 1}`;
|
||||
const id = `${baseId}${suffix}`;
|
||||
for (const card of chunk) card.primaryGroup = id;
|
||||
groups.push({
|
||||
id,
|
||||
title: id.replace(/-/gu, " "),
|
||||
summary: compactText(chunk.map((tool) => tool.summary).join("; "), 280),
|
||||
useWhen: [],
|
||||
avoidWhen: [],
|
||||
tools: chunk.map((tool) => tool.name),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
const overridden = applyCatalogOverrides(groups, cards, constraints.groupOverrides);
|
||||
return {
|
||||
version: CATALOG_VERSION,
|
||||
manifestHash,
|
||||
generatedAt: new Date().toISOString(),
|
||||
groups: overridden.groups,
|
||||
tools: overridden.tools,
|
||||
};
|
||||
}
|
||||
|
||||
function parseJsonObject(text: string): unknown {
|
||||
const unfenced = text.replace(/^\s*```(?:json)?\s*/iu, "").replace(/\s*```\s*$/u, "");
|
||||
const start = unfenced.indexOf("{");
|
||||
const end = unfenced.lastIndexOf("}");
|
||||
if (start < 0 || end <= start) throw new Error("catalog response did not contain a JSON object");
|
||||
return JSON.parse(unfenced.slice(start, end + 1));
|
||||
}
|
||||
|
||||
function isRecord(value: unknown): value is Record<string, unknown> {
|
||||
return value !== null && typeof value === "object" && !Array.isArray(value);
|
||||
}
|
||||
|
||||
export function parseGeneratedCatalog(
|
||||
text: string,
|
||||
tools: ToolInfo[],
|
||||
manifestHash: string,
|
||||
constraints: CatalogConstraints,
|
||||
generatedBy: string,
|
||||
): ToolCatalog {
|
||||
const parsed = parseJsonObject(text);
|
||||
if (!isRecord(parsed) || !Array.isArray(parsed.groups) || !Array.isArray(parsed.tools)) {
|
||||
throw new Error("catalog response must contain groups and tools arrays");
|
||||
}
|
||||
|
||||
const expectedNames = new Set(tools.map((tool) => tool.name));
|
||||
const cards: ToolCard[] = [];
|
||||
const cardNames = new Set<string>();
|
||||
for (const value of parsed.tools) {
|
||||
if (!isRecord(value) || typeof value.name !== "string" || typeof value.summary !== "string" || typeof value.primaryGroup !== "string") {
|
||||
throw new Error("every generated tool card needs name, summary, and primaryGroup");
|
||||
}
|
||||
if (!expectedNames.has(value.name) || cardNames.has(value.name)) throw new Error(`invalid or duplicate tool card: ${value.name}`);
|
||||
const primaryGroup = normalizeGroupId(value.primaryGroup);
|
||||
if (!primaryGroup) throw new Error(`invalid primary group for ${value.name}`);
|
||||
cardNames.add(value.name);
|
||||
cards.push({
|
||||
name: value.name,
|
||||
summary: compactText(value.summary, 240),
|
||||
useWhen: compactStrings(value.useWhen, 5, 160),
|
||||
avoidWhen: compactStrings(value.avoidWhen, 5, 160),
|
||||
keywords: compactStrings(value.keywords, 16, 60),
|
||||
primaryGroup,
|
||||
});
|
||||
}
|
||||
if (cardNames.size !== expectedNames.size) throw new Error("generated tool cards omitted one or more tools");
|
||||
|
||||
const groups: GroupCard[] = [];
|
||||
const groupIds = new Set<string>();
|
||||
const assignedNames = new Set<string>();
|
||||
for (const value of parsed.groups) {
|
||||
if (!isRecord(value) || typeof value.id !== "string" || typeof value.title !== "string" || typeof value.summary !== "string" || !Array.isArray(value.tools)) {
|
||||
throw new Error("every generated group needs id, title, summary, and tools");
|
||||
}
|
||||
const id = normalizeGroupId(value.id);
|
||||
if (!id || groupIds.has(id)) throw new Error(`invalid or duplicate group: ${value.id}`);
|
||||
const names = value.tools.filter((name): name is string => typeof name === "string");
|
||||
if (names.length === 0 || names.length > constraints.maxToolsPerGroup) throw new Error(`group ${id} has an invalid size`);
|
||||
for (const name of names) {
|
||||
if (!expectedNames.has(name) || assignedNames.has(name)) throw new Error(`invalid or duplicate group assignment: ${name}`);
|
||||
assignedNames.add(name);
|
||||
}
|
||||
groupIds.add(id);
|
||||
groups.push({
|
||||
id,
|
||||
title: compactText(value.title, 80),
|
||||
summary: compactText(value.summary, 280),
|
||||
useWhen: compactStrings(value.useWhen, 5, 160),
|
||||
avoidWhen: compactStrings(value.avoidWhen, 5, 160),
|
||||
tools: names,
|
||||
});
|
||||
}
|
||||
if (assignedNames.size !== expectedNames.size) throw new Error("generated groups omitted one or more tools");
|
||||
for (const card of cards) {
|
||||
const group = groups.find((candidate) => candidate.id === card.primaryGroup);
|
||||
if (!group?.tools.includes(card.name)) throw new Error(`tool card/group mismatch for ${card.name}`);
|
||||
}
|
||||
|
||||
const overridden = applyCatalogOverrides(groups, cards, constraints.groupOverrides);
|
||||
return {
|
||||
version: CATALOG_VERSION,
|
||||
manifestHash,
|
||||
generatedAt: new Date().toISOString(),
|
||||
generatedBy,
|
||||
groups: overridden.groups,
|
||||
tools: overridden.tools,
|
||||
};
|
||||
}
|
||||
|
||||
export function readCachedCatalog(
|
||||
cachePath: string,
|
||||
tools: ToolInfo[],
|
||||
manifestHash: string,
|
||||
constraints: CatalogConstraints,
|
||||
): ToolCatalog | undefined {
|
||||
try {
|
||||
const stored: unknown = JSON.parse(readFileSync(cachePath, "utf8"));
|
||||
if (!isRecord(stored) || stored.version !== CATALOG_VERSION || !isRecord(stored.catalog)) return undefined;
|
||||
const catalog = stored.catalog as unknown as ToolCatalog;
|
||||
if (catalog.manifestHash !== manifestHash) return undefined;
|
||||
return parseGeneratedCatalog(JSON.stringify(catalog), tools, manifestHash, constraints, catalog.generatedBy ?? "cached model");
|
||||
} catch {
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
|
||||
export function writeCachedCatalog(cachePath: string, catalog: ToolCatalog): void {
|
||||
mkdirSync(dirname(cachePath), { recursive: true, mode: 0o700 });
|
||||
const temporaryPath = `${cachePath}.${process.pid}.tmp`;
|
||||
const stored: StoredCatalog = { version: CATALOG_VERSION, catalog };
|
||||
writeFileSync(temporaryPath, `${JSON.stringify(stored, null, 2)}\n`, { encoding: "utf8", mode: 0o600 });
|
||||
chmodSync(temporaryPath, 0o600);
|
||||
renameSync(temporaryPath, cachePath);
|
||||
}
|
||||
|
||||
function queryTerms(value: string): Set<string> {
|
||||
const normalized = value.toLowerCase();
|
||||
const terms = new Set(normalized.match(/[\p{L}\p{N}]+/gu) ?? []);
|
||||
const cjk = [...normalized].filter((character) => /[\p{Script=Han}\p{Script=Hiragana}\p{Script=Katakana}]/u.test(character));
|
||||
for (const character of cjk) terms.add(character);
|
||||
for (let index = 0; index + 1 < cjk.length; index += 1) terms.add(`${cjk[index]}${cjk[index + 1]}`);
|
||||
return terms;
|
||||
}
|
||||
|
||||
export function rankGroups(catalog: ToolCatalog, query: string): Array<{ group: GroupCard; score: number }> {
|
||||
const querySet = queryTerms(query);
|
||||
return catalog.groups
|
||||
.map((group) => {
|
||||
const cards = catalog.tools.filter((tool) => tool.primaryGroup === group.id);
|
||||
const searchable = [
|
||||
group.id,
|
||||
group.title,
|
||||
group.summary,
|
||||
...group.useWhen,
|
||||
...group.avoidWhen,
|
||||
...cards.flatMap((tool) => [tool.name, tool.summary, ...tool.useWhen, ...tool.keywords]),
|
||||
].join(" ");
|
||||
const terms = queryTerms(searchable);
|
||||
let score = 0;
|
||||
for (const term of querySet) if (terms.has(term)) score += term.length > 1 ? 3 : 1;
|
||||
if (group.id === query.trim().toLowerCase()) score += 100;
|
||||
return { group, score };
|
||||
})
|
||||
.sort((left, right) => right.score - left.score || left.group.id.localeCompare(right.group.id));
|
||||
}
|
||||
Reference in New Issue
Block a user