pippinmole/ghost

public

Source code of Ghost - On Ghost

api: list up to 2000 matching files per code searchf35c3fc8 days ago426 lines · 11.1 KB

1	import { spawn, execFileSync, type ChildProcess } from 'node:child_process';
2	import { mkdtempSync, rmSync
3	import { tmpdir }
4	import path from 'node:path
5	import {
6	afterAll,
7	afterEach,
8	beforeAll,
9	describe,
10	expect,
11	it,
12	vi,
13	} from 'vitest';
14
15	import {
16	CodeSearchUnavailableError,
17	InvalidSearchQueryError,
18	} from './code-search.errors.js';
19	import { indexRepository, listMatchingFiles
20
21	const hasZoekt = (
22	try {
23	execFileSync('zoekt-webserver',
24	execFileSync('zoekt-git-index',
25	return true;
26	} catch {
27	return false;
28	}
29	})();
30
31	const b64 = (
32
33	const URL = '
34
35	const replyWith = (
36	const fetch =
37	async () =>
38	new Response(typeof
39	status,
40	}),
41	);
42	vi.stubGlobal('fetch
43	return fetch;
44	};
45	const sentQuery = (
46	JSON.parse(
47	(fetch.mock.calls
48	) as { Q
49
50	const zoektFile = (
51	repository: string,
52	fileName: string,
53	extra: Record
54	) => ({
55	FileName: fileName,
56	Repository: repository,
57	Version: 'c1',
58	Language: 'TypeScript',
59	Score: 1,
60	...extra,
61	});
62
63	const pageFile = (
64	repositoryId,
65	path,
66	commit: 'c1',
67	language: 'TypeScript',
68	});
69
70	const lineMatch = (
71	lineNumber: number,
72	line: string,
73	offset = 0,
74	length = 1,
75	) => ({
76	Line: b64(`${
77	LineNumber: lineNumber,
78	FileName: false,
79	LineFragments: [{ LineOffset:
80	});
81
82	describe('listMatchingFiles', ()
83	afterEach(() =>
84
85	it('asks for names only and sorts by score, then repository and path',
86	const fetch =
87	Result: {
88	Files: [
89	zoektFile('repo_b',
90	zoektFile('repo_a',
91	zoektFile('repo_a',
92	zoektFile('repo_c',
93	],
94	},
95	});
96
97	await expect(listMatchingFiles
98	pageFile('repo_c',
99	pageFile('repo_a',
100	pageFile('repo_a',
101	pageFile('repo_b',
102	]);
103	expect(sentQuery(fetch
104	Q: 'type:filename (x)',
105	Opts: {
106	MaxDocDisplayCount: 2000,
107	TotalMaxMatchCount: 10_000,
108	MaxWallTime: 3_000_000_000,
109	},
110	});
111	});
112
113	it('returns no files when zoekt reports none',
114	replyWith({ Result:
115
116	await expect(listMatchingFiles
117	[],
118	);
119	});
120
121	it('rejects a query zoekt cannot parse',
122	replyWith({ Error:
123
124	await expect(
125	listMatchingFiles({ url:
126	).rejects.toBeInstanceOf(
127	});
128
129	it('is unavailable when zoekt fails',
130	replyWith('oops',
131
132	await expect(
133	listMatchingFiles({ url:
134	).rejects.toMatchObject(
135	constructor: CodeSearchUnavailableError,
136	cause: new Error
137	});
138	});
139
140	it('is unavailable when zoekt cannot be reached',
141	const refused =
142	vi.stubGlobal('fetch
143
144	await expect(
145	listMatchingFiles({ url:
146	).rejects.toMatchObject(
147	constructor: CodeSearchUnavailableError,
148	cause: refused,
149	});
150	});
151	});
152
153	describe('searchFiles', ()
154	afterEach(() =>
155
156	const searchOne =
157	searchFiles({ url:
158
159	it('asks nothing of zoekt for an empty page',
160	const fetch =
161
162	await expect(
163	searchFiles({ url:
164	).resolves.toEqual([])
165	expect(fetch).not
166	});
167
168	it('asks once per file, capping its matches and escaping every path character outside a safe set',
169	const fetch =
170
171	await searchFiles({
172	url: URL,
173	query: 'x',
174	files: [pageFile('
175	});
176
177	const limits =
178	MaxDocDisplayCount: 1,
179	MaxMatchDisplayCount: 10,
180	TotalMaxMatchCount: 10_000,
181	MaxWallTime: 3_000_000_000,
182	};
183	expect(sentQuery(fetch
184	Q: String.raw
185	Opts: limits,
186	});
187	expect(sentQuery(fetch
188	Q: String.raw
189	Opts: limits,
190	});
191	});
192
193	it('answers only for the exact file asked about',
194	replyWith({
195	Result: {
196	Files: [
197	zoektFile('repo_other',
198	LineMatches: [lineMatch(1
199	}),
200	],
201	MatchCount: 1,
202	},
203	});
204
205	await expect(searchOne
206	{ ...pageFile('
207	]);
208	});
209
210	it('decodes lines, turns byte offsets into string offsets, and reports zoekt’s count',
211	const line =
212	replyWith({
213	Result: {
214	Files: [
215	zoektFile('repo_a',
216	LineMatches: [
217	lineMatch(9, 'later
218	{
219	Line: b64('a.ts
220	LineNumber: 0,
221	FileName: true,
222	LineFragments: [{ LineOffset:
223	},
224	lineMatch(3, line
225	],
226	}),
227	],
228	MatchCount: 42,
229	},
230	});
231
232	await expect(searchOne
233	{
234	...pageFile('repo_a'
235	matchCount: 42,
236	lines: [
237	{ lineNumber: 3, line
238	{ lineNumber: 9, line
239	],
240	},
241	]);
242	});
243
244	it('refuses an answer past the byte budget rather than read it',
245	replyWith({
246	Result: {
247	Files: [
248	zoektFile('repo_a',
249	LineMatches: [lineMatch(1
250	}),
251	],
252	MatchCount: 1,
253	},
254	});
255
256	await expect(searchOne
257	constructor: CodeSearchUnavailableError,
258	cause: new Error
259	});
260	});
261
262	it('keeps at most twenty ranges on a line',
263	replyWith({
264	Result: {
265	Files: [
266	zoektFile('repo_a',
267	LineMatches: [
268	{
269	Line: b64('a
270	LineNumber: 1,
271	FileName: false,
272	LineFragments: Array.from
273	LineOffset: i,
274	MatchLength: 1,
275	})),
276	},
277	],
278	}),
279	],
280	MatchCount: 100,
281	},
282	});
283
284	const [hit]
285
286	expect(hit.lines
287	});
288
289	it('cuts a two-megabyte line to what is shown, with the matches inside the cut',
290	replyWith({
291	Result: {
292	Files: [
293	zoektFile('repo_a',
294	LineMatches: [
295	{
296	Line: b64('a
297	LineNumber: 1,
298	FileName: false,
299	LineFragments: [
300	{ LineOffset: 10, MatchLength
301	{ LineOffset: 490, MatchLength
302	{ LineOffset: 600, MatchLength
303	],
304	},
305	],
306	}),
307	],
308	MatchCount: 3,
309	},
310	});
311
312	const [hit]
313
314	expect(hit.lines
315	expect(hit.lines
316	{ start: 10, end
317	{ start: 490, end
318	]);
319	});
320	});
321
322	describe.skipIf(!hasZoekt
323	let root:
324	let webserver:
325	let shards:
326	const url =
327
328	const repository =
329	const source =
330	const git =
331	execFileSync('git',
332
333	execFileSync('git',
334	git('config',
335	git('config',
336	writeFileSync(path.join
337	git('add',
338	git('commit',
339	execFileSync('git',
340	'clone',
341	'-q',
342	'--bare',
343	source,
344	path.join(root
345	]);
346	return path.join
347	};
348
349	beforeAll(async ()
350	root = mkdtempSync(
351	const indexDir =
352
353	shards = await
354	indexDir,
355	repoDirectory: repository('repo_public
356	name: 'repo_public',
357	isPublic: true,
358	});
359	await indexRepository({
360	indexDir,
361	repoDirectory: repository('repo_private
362	name: 'repo_private',
363	isPublic: false,
364	});
365
366	webserver = spawn(
367	'-index',
368	indexDir,
369	'-listen',
370	'127.0.0.1:16070',
371	'-rpc',
372	'-html=false',
373	]);
374	await vi.waitFor
375	async () =>
376	{ timeout: 10_000, interval
377	);
378	});
379
380	afterAll(() =>
381	webserver?.kill();
382	rmSync(root, {
383	});
384
385	const searchUntil =
386	vi.waitFor(
387	async () =>
388	const files =
389	expect(files).toHaveLength
390	return searchFiles({
391	},
392	{ timeout: 10_000, interval
393	);
394
395	it('names the shards after the repository',
396	expect(shards).toEqual
397	});
398
399	it('finds a line in every indexed shard',
400	const hits =
401
402	expect(hits.map
403	'repo_private',
404	'repo_public',
405	]);
406	});
407
408	it('narrows to shards flagged public',
409	const [hit]
410
411	expect(hit).toEqual
412	repositoryId: 'repo_public',
413	commit: expect.stringMatching
414	path: 'a.ts',
415	language: 'TypeScript',
416	matchCount: 1,
417	lines: [
418	{
419	lineNumber: 1,
420	line: 'const café = findMe();',
421	ranges: [{ start:
422	},
423	],
424	});
425	});
426	});