Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
114 changes: 94 additions & 20 deletions src/core/skill-search.ts
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
import { parseSkillMetadata } from '../validators/skill.js';

/**
* GitHub Code Search wrapper for `allagents skill search`.
*
Expand Down Expand Up @@ -36,7 +38,7 @@ export interface SkillSearchItem {
repo: string;
/** Path to SKILL.md inside the repo. */
path: string;
/** Repo description (often empty for Code Search results). */
/** Skill description, enriched from SKILL.md frontmatter when available. */
description: string;
/** File blob SHA. */
sha: string;
Expand All @@ -57,6 +59,8 @@ export interface SkillSearchOptions {
limit?: number;
}

const ENRICHMENT_CONCURRENCY = 10;

export class SkillSearchError extends Error {
constructor(message: string, public readonly kind: 'validation' | 'rate-limit' | 'api') {
super(message);
Expand Down Expand Up @@ -434,7 +438,10 @@ export async function searchSkills(
const firstSegment = item.path.split('/')[0] ?? '';
return !firstSegment.startsWith('.');
});
await fetchStarsForItems(visible, token, fetchFn);
await Promise.all([
fetchStarsForItems(visible, token, fetchFn),
enrichDescriptionsForItems(visible, token, fetchFn),
]);
visible.sort((a, b) => b.stars - a.stars);
// Apply the limit to the merged output so `--limit N` caps total results,
// not just per-query results (each query runs with the same limit).
Expand Down Expand Up @@ -478,29 +485,96 @@ async function fetchStarsForItems(
fetchFn: typeof fetch,
): Promise<void> {
const uniqueRepos = [...new Set(items.map((i) => i.repo))];
const headers: Record<string, string> = {
Accept: 'application/vnd.github+json',
'X-GitHub-Api-Version': '2022-11-28',
'User-Agent': 'allagents-cli',
};
if (token) headers.Authorization = `token ${token}`;
const headers = buildGitHubApiHeaders(token);

const starsMap = new Map<string, number>();
await Promise.allSettled(
uniqueRepos.map(async (repo) => {
try {
const res = await fetchFn(`https://api.github.com/repos/${repo}`, { headers });
if (!res.ok) return;
const body = await res.json() as { stargazers_count?: number };
starsMap.set(repo, body.stargazers_count ?? 0);
} catch {
// ignore — stars stay 0
}
}),
);
await forEachWithConcurrency(uniqueRepos, ENRICHMENT_CONCURRENCY, async (repo) => {
try {
const res = await fetchFn(`https://api.github.com/repos/${repo}`, { headers });
if (!res.ok) return;
const body = await res.json() as { stargazers_count?: number };
starsMap.set(repo, body.stargazers_count ?? 0);
} catch {
// ignore — stars stay 0
}
});

for (const item of items) {
const s = starsMap.get(item.repo);
if (s !== undefined) item.stars = s;
}
}

async function enrichDescriptionsForItems(
items: SkillSearchItem[],
token: string | undefined,
fetchFn: typeof fetch,
): Promise<void> {
const headers = buildGitHubApiHeaders(token);

const descriptionMap = new Map<string, string>();
const uniqueSkills = [...new Set(items.map((item) => `${item.repo}#${item.sha}`))];

await forEachWithConcurrency(uniqueSkills, ENRICHMENT_CONCURRENCY, async (key) => {
const [repo, sha] = key.split('#');
if (!repo || !sha) return;

try {
const res = await fetchFn(`https://api.github.com/repos/${repo}/git/blobs/${sha}`, { headers });
if (!res.ok) return;

const body = await res.json() as { content?: string; encoding?: string };
const content = decodeGitBlob(body.content, body.encoding);
if (!content) return;

const metadata = parseSkillMetadata(content);
if (!metadata?.description) return;

descriptionMap.set(key, metadata.description);
} catch {
// Ignore metadata fetch failures and keep the repo description fallback.
}
});

for (const item of items) {
const description = descriptionMap.get(`${item.repo}#${item.sha}`);
if (description) item.description = description;
}
}

async function forEachWithConcurrency<T>(
items: T[],
concurrency: number,
worker: (item: T) => Promise<void>,
): Promise<void> {
if (items.length === 0) return;

let nextIndex = 0;
const runWorker = async () => {
while (true) {
const currentIndex = nextIndex++;
if (currentIndex >= items.length) return;
await worker(items[currentIndex] as T);
}
};

const workerCount = Math.min(concurrency, items.length);
await Promise.all(Array.from({ length: workerCount }, () => runWorker()));
}

function buildGitHubApiHeaders(token: string | undefined): Record<string, string> {
const headers: Record<string, string> = {
Accept: 'application/vnd.github+json',
'X-GitHub-Api-Version': '2022-11-28',
'User-Agent': 'allagents-cli',
};
if (token) headers.Authorization = `token ${token}`;
return headers;
}

function decodeGitBlob(content: string | undefined, encoding: string | undefined): string | undefined {
if (!content) return undefined;
if (!encoding || encoding === 'utf-8') return content;
if (encoding !== 'base64') return undefined;
return Buffer.from(content.replace(/\s+/g, ''), 'base64').toString('utf8');
}
112 changes: 112 additions & 0 deletions tests/unit/core/skill-search.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -338,6 +338,118 @@ describe('searchSkills error mapping', () => {
});
});

describe('searchSkills description enrichment', () => {
it('prefers SKILL.md frontmatter description over the repository description', async () => {
const fakeFetch = (async (url: string) => {
const u = new URL(url);

if (u.pathname === '/search/code') {
return {
ok: true,
status: 200,
json: async () => ({
total_count: 1,
incomplete_results: false,
items: [
{
path: 'plugins/wzg/skills/skill-source-mapping/SKILL.md',
sha: 'blob-sha',
repository: {
full_name: 'WiseTechGlobal/WZG.Playbook.Content',
description: 'Walter Zhang\'s personal engineering playbook',
},
},
],
}),
};
}

if (u.pathname === '/repos/WiseTechGlobal/WZG.Playbook.Content') {
return {
ok: true,
status: 200,
json: async () => ({ stargazers_count: 1 }),
};
}

if (u.pathname === '/repos/WiseTechGlobal/WZG.Playbook.Content/git/blobs/blob-sha') {
return {
ok: true,
status: 200,
json: async () => ({
encoding: 'base64',
content: Buffer.from(
'---\nname: skill-source-mapping\ndescription: Locate source repositories for AI skills.\n---\n',
).toString('base64'),
}),
};
}

throw new Error(`Unexpected URL: ${url}`);
}) as unknown as typeof fetch;

const result = await searchSkills(
'skill-source-mapping',
{},
{ fetch: fakeFetch, logger: silentLogger },
);

expect(result.items[0]?.description).toBe('Locate source repositories for AI skills.');
});

it('falls back to the repository description when SKILL.md metadata cannot be parsed', async () => {
const fakeFetch = (async (url: string) => {
const u = new URL(url);

if (u.pathname === '/search/code') {
return {
ok: true,
status: 200,
json: async () => ({
total_count: 1,
incomplete_results: false,
items: [
{
path: 'skills/docs-writer/SKILL.md',
sha: 'blob-sha',
repository: {
full_name: 'org/repo',
description: 'Repository description fallback',
},
},
],
}),
};
}

if (u.pathname === '/repos/org/repo') {
return {
ok: true,
status: 200,
json: async () => ({ stargazers_count: 0 }),
};
}

if (u.pathname === '/repos/org/repo/git/blobs/blob-sha') {
return {
ok: true,
status: 200,
json: async () => ({
encoding: 'base64',
content: Buffer.from('# No frontmatter here\n').toString('base64'),
}),
};
}

throw new Error(`Unexpected URL: ${url}`);
}) as unknown as typeof fetch;

const result = await searchSkills('docs', {}, { fetch: fakeFetch, logger: silentLogger });

expect(result.items[0]?.description).toBe('Repository description fallback');
});
});

describe('namespace extraction', () => {
it('parses skills/<ns>/<name>/SKILL.md into namespace + name', async () => {
const fakeFetch = makeFakeFetch([
Expand Down