| 1 | import { createHash } from 'node:crypto'; |
| 2 | import { createReadStream } |
| 3 | import { mkdtemp, rm |
| 4 | import { tmpdir } |
| 5 | import path from 'node:path |
| 6 | import { pipeline } |
| 7 | |
| 8 | import { Injectable } |
| 9 | |
| 10 | import { GitCommandFailedError } |
| 11 | import { objectTypes } |
| 12 | import { runGit } |
| 13 | import { |
| 14 | bufferBody, |
| 15 | fileBody, |
| 16 | } from '../../../lib/git/protocol/git-request-body.js'; |
| 17 | import { |
| 18 | directoryConflict, |
| 19 | isWellFormedRef, |
| 20 | } from '../../../lib/git/refs/is-valid-ref-name.js'; |
| 21 | import { createUlid } |
| 22 | import { |
| 23 | NonFastForwardError, |
| 24 | RepositoryDeletedError, |
| 25 | } from '../../../lib/git/wal/wal.errors.js'; |
| 26 | import { |
| 27 | type WalIndex, |
| 28 | type WalLayer, |
| 29 | ZERO_OID, |
| 30 | } from '../../../lib/git/wal/wal.types.js'; |
| 31 | import { encodeEntryHeader } |
| 32 | import { isNotFound } |
| 33 | import { RepositoryStorageService } |
| 34 | import { PushTransactionService } |
| 35 | import { WalStoreService } |
| 36 | |
| 37 | // A repair that loses the index to a push starts over; a repository still being pushed to this often was never broken. |
| 38 | const MAX_ATTEMPTS = 3 |
| 39 | const OID_HEX_LENGTH = 40 |
| 40 | |
| 41 | export interface RepairReport |
| 42 | /** Entries the log alone could not index. `rebuilt` borrowed its delta bases from the local cache and now replays by itself; `dropped` is skipped by every replay from now on. */ |
| 43 | layers: { |
| 44 | /** Refs whose missing objects the local cache still held, written back into the log. */ |
| 45 | rescued: string[] |
| 46 | /** Refs removed because no node could ever replay them. */ |
| 47 | dropped: { |
| 48 | } |
| 49 | |
| 50 | interface Diagnosis extends |
| 51 | scratch: string; |
| 52 | rebuiltPacks: Map |
| 53 | rescuePack: string |
| 54 | } |
| 55 | |
| 56 | /** Finds repositories whose log no node can replay, which makes every clone and push of them fail, and repairs them (0034). The log is replayed into an empty scratch repository, not checked against the cache: the cache can hold objects the log lacks and so hide the damage. The cache is only a source of objects to put back. */ |
| 57 | @Injectable() |
| 58 | export class RepositoryRepairService |
| 59 | constructor( |
| 60 | private readonly store |
| 61 | private readonly pushTransaction |
| 62 | private readonly storage |
| 63 | ) {} |
| 64 | |
| 65 | /** What a repair would change, changing nothing. Null when the log replays cleanly or holds nothing. */ |
| 66 | async check(repositoryId |
| 67 | const stored = |
| 68 | if (!stored || |
| 69 | const diagnosis = |
| 70 | try { |
| 71 | return diagnosis.layers |
| 72 | diagnosis.rescued.length |
| 73 | diagnosis.dropped.length |
| 74 | ? null |
| 75 | : report(diagnosis) |
| 76 | } finally { |
| 77 | await rm(diagnosis |
| 78 | } |
| 79 | } |
| 80 | |
| 81 | /** Repairs the log so a node holding nothing else can replay it, and says what changed. Null when nothing needed changing. */ |
| 82 | async repair(repositoryId |
| 83 | for (let attempt |
| 84 | const stored = |
| 85 | if (!stored || |
| 86 | const diagnosis = |
| 87 | try { |
| 88 | if ( |
| 89 | diagnosis.layers.length |
| 90 | diagnosis.rescued.length |
| 91 | diagnosis.dropped.length |
| 92 | ) |
| 93 | return null; |
| 94 | if (await this |
| 95 | return report(diagnosis |
| 96 | } |
| 97 | } catch (error) |
| 98 | if (!(error |
| 99 | } finally { |
| 100 | await rm(diagnosis |
| 101 | } |
| 102 | if (attempt === |
| 103 | throw new Error( |
| 104 | } |
| 105 | } |
| 106 | } |
| 107 | |
| 108 | private async readIndex |
| 109 | try { |
| 110 | return await this |
| 111 | } catch (error) |
| 112 | if (error instanceof |
| 113 | throw error; |
| 114 | } |
| 115 | } |
| 116 | |
| 117 | private async diagnose |
| 118 | repositoryId: string, |
| 119 | index: WalIndex |
| 120 | ): Promise<Diagnosis |
| 121 | const scratch = |
| 122 | try { |
| 123 | await runGit({ |
| 124 | const cache = |
| 125 | const diagnosis: |
| 126 | scratch, |
| 127 | layers: [], |
| 128 | rescued: [], |
| 129 | dropped: [], |
| 130 | rebuiltPacks: new Map |
| 131 | rescuePack: null, |
| 132 | }; |
| 133 | |
| 134 | for (const layer |
| 135 | if (layer.size |
| 136 | const replayed = |
| 137 | if ('pack' in |
| 138 | // The cache can lend the delta bases a thin entry leaned on; the pack index-pack writes then carries them itself. |
| 139 | const rebuilt = |
| 140 | if ('pack' in |
| 141 | diagnosis.rebuiltPacks.set |
| 142 | diagnosis.layers.push |
| 143 | ulid: layer.ulid |
| 144 | outcome: 'rebuilt', |
| 145 | reason: replayed.failure |
| 146 | }); |
| 147 | } else { |
| 148 | diagnosis.layers.push |
| 149 | ulid: layer.ulid |
| 150 | outcome: 'dropped', |
| 151 | reason: rebuilt.failure |
| 152 | }); |
| 153 | } |
| 154 | } |
| 155 | |
| 156 | await this.checkRefs |
| 157 | return diagnosis; |
| 158 | } catch (error) |
| 159 | await rm(scratch |
| 160 | throw error; |
| 161 | } |
| 162 | } |
| 163 | |
| 164 | private async replay |
| 165 | repositoryId: string, |
| 166 | layer: WalLayer |
| 167 | scratch: string, |
| 168 | lender?: string |
| 169 | ): Promise<{ |
| 170 | let input; |
| 171 | try { |
| 172 | input = await |
| 173 | } catch (error) |
| 174 | if (isNotFound(error |
| 175 | return { failure: |
| 176 | } |
| 177 | throw error; |
| 178 | } |
| 179 | return runGit({ |
| 180 | args: ['index-pack', |
| 181 | gitDir: scratch, |
| 182 | env: lender |
| 183 | ? { GIT_ALTERNATE_OBJECT_DIRECTORIES: path |
| 184 | : undefined, |
| 185 | input, |
| 186 | }).then( |
| 187 | // index-pack names the pack it wrote: `pack\t<hash>` |
| 188 | (out) => |
| 189 | pack: path.join |
| 190 | scratch, |
| 191 | 'objects', |
| 192 | 'pack', |
| 193 | `pack-${out.trim |
| 194 | ), |
| 195 | }), |
| 196 | (error: unknown |
| 197 | if (!(error |
| 198 | return { |
| 199 | failure: error.stderr |
| 200 | }; |
| 201 | }, |
| 202 | ); |
| 203 | } |
| 204 | |
| 205 | private async checkRefs |
| 206 | repositoryId: string, |
| 207 | index: WalIndex |
| 208 | diagnosis: Diagnosis |
| 209 | cache: string, |
| 210 | ) { |
| 211 | const { scratch |
| 212 | const live = |
| 213 | const drop = |
| 214 | diagnosis.dropped.push |
| 215 | ref, |
| 216 | oid: live.get |
| 217 | reason, |
| 218 | }); |
| 219 | live.delete(ref |
| 220 | }; |
| 221 | |
| 222 | for (const ref |
| 223 | if (!isWellFormedRef(ref |
| 224 | } |
| 225 | |
| 226 | const unreachable = |
| 227 | if (unreachable.length |
| 228 | diagnosis.rescuePack = |
| 229 | scratch, |
| 230 | cache, |
| 231 | unreachable.map((ref |
| 232 | ); |
| 233 | for (const ref |
| 234 | drop(ref, ' |
| 235 | } |
| 236 | diagnosis.rescued = |
| 237 | } |
| 238 | |
| 239 | if (live.size |
| 240 | const refs = |
| 241 | const types = |
| 242 | gitDir: scratch, |
| 243 | oids: refs.map |
| 244 | }); |
| 245 | for (const [i |
| 246 | if (refs[i |
| 247 | drop(refs[i |
| 248 | } |
| 249 | } |
| 250 | } |
| 251 | |
| 252 | for (const ref |
| 253 | if (!live.has |
| 254 | const conflict = |
| 255 | if (!conflict) |
| 256 | const later = |
| 257 | drop(later, ` |
| 258 | } |
| 259 | } |
| 260 | |
| 261 | /** Of two refs that cannot coexist, the one whose entry came later: that push is the one that should never have landed. The deeper name when no entry says. */ |
| 262 | private async writtenLater |
| 263 | repositoryId: string, |
| 264 | index: WalIndex |
| 265 | a: string, |
| 266 | b: string, |
| 267 | ) { |
| 268 | for (const layer |
| 269 | const header = |
| 270 | const set = |
| 271 | ({ ref, newOid |
| 272 | (ref === a |
| 273 | ); |
| 274 | if (set) return |
| 275 | } |
| 276 | return a.length |
| 277 | } |
| 278 | |
| 279 | /** The refs whose history the scratch repository cannot walk in full. */ |
| 280 | private async unreachable |
| 281 | const walks = |
| 282 | // on stdin: thousands of refs overflow argv, and that failure would read as a broken walk |
| 283 | runGit({ |
| 284 | args: ['rev-list', |
| 285 | gitDir: scratch, |
| 286 | input: Buffer.from |
| 287 | }).then( |
| 288 | () => true, |
| 289 | () => false, |
| 290 | ); |
| 291 | const entries = |
| 292 | ([ref, oid |
| 293 | ); |
| 294 | // ponytail: one walk of every ref, then one per ref only when that fails. A damaged repository with thousands of refs walks its history thousands of times. |
| 295 | if (entries.length |
| 296 | return []; |
| 297 | } |
| 298 | const broken: |
| 299 | for (const [ref |
| 300 | if (!(await |
| 301 | } |
| 302 | return broken; |
| 303 | } |
| 304 | |
| 305 | /** Packs whatever the cache holds of `oids`' history that the scratch repository lacks, indexes it there too, and returns the pack; null when the cache has none of it. */ |
| 306 | private async rescue |
| 307 | // Walked in the scratch repository with the cache lent: the tips may exist only in the log, their history only in the cache. |
| 308 | const reachable = |
| 309 | args: [ |
| 310 | 'rev-list', |
| 311 | '--objects', |
| 312 | '--missing=allow-any', |
| 313 | '--ignore-missing', |
| 314 | '--stdin', |
| 315 | ], |
| 316 | gitDir: scratch, |
| 317 | input: Buffer.from |
| 318 | env: { GIT_ALTERNATE_OBJECT_DIRECTORIES: path |
| 319 | }); |
| 320 | const candidates = |
| 321 | .split('\n') |
| 322 | .filter(Boolean) |
| 323 | .map((line) |
| 324 | if (candidates.length |
| 325 | |
| 326 | const present = |
| 327 | args: ['cat-file', |
| 328 | gitDir: scratch, |
| 329 | input: Buffer.from |
| 330 | }); |
| 331 | const absent = |
| 332 | .trim() |
| 333 | .split('\n') |
| 334 | .filter((line) |
| 335 | .map((line) |
| 336 | if (absent.length |
| 337 | |
| 338 | const prefix = |
| 339 | const name = |
| 340 | args: ['pack-objects', |
| 341 | gitDir: cache, |
| 342 | input: Buffer.from |
| 343 | }); |
| 344 | const pack = |
| 345 | await runGit({ |
| 346 | args: ['index-pack', |
| 347 | gitDir: scratch, |
| 348 | input: createReadStream(pack |
| 349 | }); |
| 350 | return pack; |
| 351 | } |
| 352 | |
| 353 | /** Swaps rebuilt and dropped entries into the index in place, so every sequence number stays where caches expect it, then removes dropped refs and adds rescued objects as one ordinary entry. False when a push moved the index first. */ |
| 354 | private async apply |
| 355 | repositoryId: string, |
| 356 | stored: { |
| 357 | diagnosis: Diagnosis |
| 358 | ) { |
| 359 | if (diagnosis.layers |
| 360 | const layers = |
| 361 | stored.index.layers |
| 362 | const fix = |
| 363 | if (!fix) |
| 364 | if (fix.outcome |
| 365 | return this.rewrite |
| 366 | repositoryId, |
| 367 | layer.ulid, |
| 368 | diagnosis.rebuiltPacks.get |
| 369 | ); |
| 370 | }), |
| 371 | ); |
| 372 | const swapped = |
| 373 | repositoryId, |
| 374 | { ...stored.index |
| 375 | stored.etag, |
| 376 | ); |
| 377 | if (!swapped) |
| 378 | } |
| 379 | |
| 380 | if (diagnosis.dropped |
| 381 | const pack = |
| 382 | await this.pushTransaction |
| 383 | repoId: repositoryId, |
| 384 | transitions: diagnosis.dropped |
| 385 | ref, |
| 386 | oldOid: Buffer.from |
| 387 | newOid: ZERO_OID, |
| 388 | })), |
| 389 | body: pack |
| 390 | ? fileBody(pack, |
| 391 | : bufferBody(Buffer. |
| 392 | packOffset: 0, |
| 393 | }); |
| 394 | } |
| 395 | return true; |
| 396 | } |
| 397 | |
| 398 | /** A new entry holding the rebuilt pack under the old entry's header, so the log still says who pushed it and what it moved. */ |
| 399 | private async rewrite |
| 400 | repositoryId: string, |
| 401 | ulid: string, |
| 402 | pack: string, |
| 403 | ): Promise<WalLayer |
| 404 | const header = |
| 405 | const next = |
| 406 | const { size |
| 407 | await this.store |
| 408 | repositoryId, |
| 409 | next, |
| 410 | encodeEntryHeader({ |
| 411 | ulid: next, |
| 412 | createdAt: header?.createdAt |
| 413 | pushedBy: header?.pushedBy |
| 414 | transitions: header?.transitions |
| 415 | }), |
| 416 | fileBody(pack, size |
| 417 | 0, |
| 418 | ); |
| 419 | const hash = |
| 420 | await pipeline(createReadStream |
| 421 | return { ulid: |
| 422 | } |
| 423 | } |
| 424 | |
| 425 | function report({ layers |
| 426 | return { layers, |
| 427 | } |