| 1 | import { execFile } from 'node:child_process'; |
| 2 | import { readdir } |
| 3 | import { promisify } |
| 4 | |
| 5 | import { runGit } |
| 6 | import { |
| 7 | CodeSearchUnavailableError, |
| 8 | InvalidSearchQueryError, |
| 9 | } from './code-search.errors.js'; |
| 10 | |
| 11 | const execFileAsync = promisify |
| 12 | |
| 13 | // Every constant below bounds what one request can cost, whatever the query: zoekt stops early, and nothing here grows with the size of the index or of a file. |
| 14 | const SEARCH_TIMEOUT_MS = 10_000 |
| 15 | const MAX_WALL_TIME_NS = 3_000_000_000 |
| 16 | const TOTAL_MAX_MATCHES = 10_000 |
| 17 | // zoekt has no offset and its order shifts between calls, so every page is cut from one name list of this fixed size, sorted here. |
| 18 | const LISTED_FILES = 2_000 |
| 19 | // Two thousand names with paths up to git's 4 KiB each. |
| 20 | const LIST_RESPONSE_BYTES = 16 |
| 21 | // The indexer skips larger files, so a file's matching lines can never add up to more than this. |
| 22 | const FILE_LIMIT_BYTES = 2 |
| 23 | // Base64 grows a file's lines by a third; the rest is room for the JSON around them. |
| 24 | const FILE_RESPONSE_BYTES = 2 |
| 25 | // Also the match cap on each file's query, so zoekt itself sends at most this many lines and fragments. |
| 26 | const LINES_PER_FILE = 10 |
| 27 | // Cut before decoding, so a megabyte line costs no more than a short one. Four bytes per character at most, so the cut keeps every character shown. |
| 28 | const MAX_LINE_BYTES = 2_000 |
| 29 | const MAX_LINE_LENGTH = 500 |
| 30 | const RANGES_PER_LINE = 20 |
| 31 | |
| 32 | export interface CodeSearchFile |
| 33 | repositoryId: string; |
| 34 | path: string; |
| 35 | /** The commit the shard was built from, which is what the line numbers refer to. */ |
| 36 | commit: string; |
| 37 | language: string; |
| 38 | } |
| 39 | |
| 40 | export interface CodeSearchHit |
| 41 | /** Matches in the file, as far as zoekt counted before its limits; `lines` holds the first few. */ |
| 42 | matchCount: number; |
| 43 | lines: { |
| 44 | lineNumber: number |
| 45 | line: string |
| 46 | /** Half-open [start, end) offsets that slice `line` as a JavaScript string. */ |
| 47 | ranges: { |
| 48 | }[]; |
| 49 | } |
| 50 | |
| 51 | /** The fields of zoekt's `FileMatch` read here. `[]byte` fields arrive base64 encoded. */ |
| 52 | interface ZoektFile { |
| 53 | FileName: string; |
| 54 | Repository: string; |
| 55 | Version: string; |
| 56 | Language: string; |
| 57 | Score: number; |
| 58 | LineMatches?: { |
| 59 | Line: string |
| 60 | LineNumber: number |
| 61 | FileName: boolean |
| 62 | LineFragments: { |
| 63 | }[]; |
| 64 | } |
| 65 | |
| 66 | /** Shards are named `<name>_v16.00000.zoekt`, with further numbered shards for a large repository. Returns their file names. */ |
| 67 | export async function indexRepository |
| 68 | indexDir, |
| 69 | repoDirectory, |
| 70 | name, |
| 71 | isPublic, |
| 72 | }: { |
| 73 | indexDir: string |
| 74 | repoDirectory: string |
| 75 | name: string |
| 76 | isPublic: boolean |
| 77 | }) { |
| 78 | // zoekt copies the `zoekt` config section into the shard, which is what `public:yes` matches, but only for a repository that has `zoekt.name` or an origin remote; the API's cache has no origin. |
| 79 | for (const [key |
| 80 | ['zoekt.name', name |
| 81 | ['zoekt.public', isPublic |
| 82 | ]) { |
| 83 | await runGit({ |
| 84 | } |
| 85 | await execFileAsync('zoekt-git-index |
| 86 | '-index', |
| 87 | indexDir, |
| 88 | '-branches', |
| 89 | 'HEAD', |
| 90 | '-submodules=false', |
| 91 | '-file_limit', |
| 92 | String(FILE_LIMIT_BYTES), |
| 93 | '-disable_ctags', |
| 94 | repoDirectory, |
| 95 | ]); |
| 96 | return (await readdir |
| 97 | } |
| 98 | |
| 99 | /** Every file that matches, best first, without reading its lines. Ties sort by repository and path, so the same index always gives the same order. */ |
| 100 | // ponytail: a query that hits zoekt's match limits, like `.*`, can list a slightly different set from one call to the next; a cached list per query would pin it. |
| 101 | export async function listMatchingFiles |
| 102 | url, |
| 103 | query, |
| 104 | }: { |
| 105 | url: string |
| 106 | query: string |
| 107 | }): Promise<CodeSearchFile |
| 108 | const { files |
| 109 | url, |
| 110 | `type:filename (${query})` |
| 111 | { MaxDocDisplayCount: LISTED_FILES |
| 112 | LIST_RESPONSE_BYTES, |
| 113 | ); |
| 114 | return files |
| 115 | .sort( |
| 116 | (a, b) |
| 117 | b.Score - |
| 118 | a.Repository.localeCompare |
| 119 | a.FileName.localeCompare |
| 120 | ) |
| 121 | .map((file) |
| 122 | repositoryId: file.Repository |
| 123 | path: file.FileName |
| 124 | commit: file.Version |
| 125 | language: file.Language |
| 126 | })); |
| 127 | } |
| 128 | |
| 129 | /** Exactly these files, in their order, with their first matching lines; a file that matched by its path alone has none. One query per file, so zoekt caps every file's matches on its own and a page of minified bundles stays within a few megabytes per file. */ |
| 130 | export async function searchFiles |
| 131 | url, |
| 132 | query, |
| 133 | files, |
| 134 | }: { |
| 135 | url: string |
| 136 | query: string |
| 137 | files: CodeSearchFile |
| 138 | }): Promise<CodeSearchHit |
| 139 | return Promise.all |
| 140 | files.map(async |
| 141 | const found = |
| 142 | url, |
| 143 | `(${query}) r:^${ |
| 144 | { MaxDocDisplayCount: 1, MaxMatchDisplayCount |
| 145 | FILE_RESPONSE_BYTES, |
| 146 | ); |
| 147 | // the scope narrows the scan; this match on the exact file is what keeps an OR in the query from answering for another |
| 148 | const matches = |
| 149 | found.files.find |
| 150 | (candidate) => |
| 151 | candidate.Repository === |
| 152 | candidate.FileName === |
| 153 | )?.LineMatches ?? [] |
| 154 | ) |
| 155 | .filter((match) |
| 156 | .sort((a, |
| 157 | |
| 158 | return { |
| 159 | ...file, |
| 160 | matchCount: matches.length |
| 161 | lines: matches.map |
| 162 | const line = |
| 163 | match.Line.slice |
| 164 | 'base64', |
| 165 | ).subarray(0, |
| 166 | return { |
| 167 | lineNumber: match.LineNumber |
| 168 | line: line |
| 169 | .toString('utf8') |
| 170 | .replace(/\r?\n |
| 171 | .slice(0, MAX_LINE_LENGTH |
| 172 | ranges: match.LineFragments |
| 173 | (fragment) => |
| 174 | const start = |
| 175 | if (start >= |
| 176 | const end = |
| 177 | line, |
| 178 | fragment.LineOffset + |
| 179 | ); |
| 180 | return [{ start |
| 181 | }, |
| 182 | ), |
| 183 | }; |
| 184 | }), |
| 185 | }; |
| 186 | }), |
| 187 | ); |
| 188 | } |
| 189 | |
| 190 | async function search( |
| 191 | url: string, |
| 192 | query: string, |
| 193 | options: { |
| 194 | maxResponseBytes: number, |
| 195 | ): Promise<{ |
| 196 | const response = |
| 197 | method: 'POST', |
| 198 | headers: { 'Content-Type': |
| 199 | body: JSON.stringify |
| 200 | Q: query, |
| 201 | Opts: { |
| 202 | ...options, |
| 203 | TotalMaxMatchCount: TOTAL_MAX_MATCHES, |
| 204 | MaxWallTime: MAX_WALL_TIME_NS, |
| 205 | }, |
| 206 | }), |
| 207 | signal: AbortSignal.timeout |
| 208 | }).catch((cause |
| 209 | throw new CodeSearchUnavailableError( |
| 210 | }); |
| 211 | |
| 212 | if (response.status |
| 213 | const { Error: |
| 214 | throw new InvalidSearchQueryError( |
| 215 | } |
| 216 | if (!response.ok |
| 217 | throw new CodeSearchUnavailableError( |
| 218 | cause: new Error |
| 219 | `zoekt answered ${response.status |
| 220 | ), |
| 221 | }); |
| 222 | } |
| 223 | |
| 224 | const { Result |
| 225 | await readAtMost(response |
| 226 | ) as { Result |
| 227 | return { files: |
| 228 | } |
| 229 | |
| 230 | /** The budgets are sized so an index built here never reaches them; one that does was built with other limits, and is refused rather than read into memory. */ |
| 231 | async function readAtMost( |
| 232 | const reader = |
| 233 | const chunks: |
| 234 | let size = |
| 235 | for (let read |
| 236 | size += read. |
| 237 | if (size > |
| 238 | await reader.cancel |
| 239 | throw new CodeSearchUnavailableError( |
| 240 | cause: new Error |
| 241 | }); |
| 242 | } |
| 243 | chunks.push(read |
| 244 | } |
| 245 | return Buffer.concat |
| 246 | } |
| 247 | |
| 248 | // zoekt reports byte offsets; a browser slices UTF-16 strings. |
| 249 | function charOffset(line: |
| 250 | return line.subarray |
| 251 | } |
| 252 | |
| 253 | /** A zoekt `r:` clause matching exactly these repositories, or nothing to add when unscoped. */ |
| 254 | export function repositoryScope(repositoryIds |
| 255 | return repositoryIds |
| 256 | ? ` r:^(${repositoryIds. |
| 257 | : ''; |
| 258 | } |
| 259 | |
| 260 | // Every character outside a safe set becomes `\x{..}`: a space, quote or parenthesis would otherwise end the zoekt field it sits in. |
| 261 | function literalRegExp(value: |
| 262 | return [...value |
| 263 | .map((char) |
| 264 | /[A-Za-z0-9_/-]/.test |
| 265 | ? char |
| 266 | : `\\x{${char |
| 267 | ) |
| 268 | .join(''); |
| 269 | } |