| 1 | import { execFile, execFileSync, spawnSync } from 'node:child_process'; |
| 2 | import { createHash, randomBytes |
| 3 | import { mkdtempSync, readFileSync |
| 4 | import { createServer, type |
| 5 | import { tmpdir } |
| 6 | import path from 'node:path |
| 7 | import { promisify } |
| 8 | import type { INestApplication |
| 9 | import request from 'supertest |
| 10 | import { afterAll, beforeAll |
| 11 | |
| 12 | import { type Database, |
| 13 | import { eq } |
| 14 | |
| 15 | import { DATABASE } |
| 16 | import { lfsObjectKey } |
| 17 | import { S3Service } |
| 18 | import { hasBackends, signUp |
| 19 | |
| 20 | const LFS = ' |
| 21 | const hasGitLfs = spawnSync |
| 22 | |
| 23 | /** A port nothing listens on, so the app's own URL, which the batch API's hrefs start with, can be known before it starts. */ |
| 24 | async function freePort() |
| 25 | const server = |
| 26 | await new Promise |
| 27 | const { port |
| 28 | await new Promise |
| 29 | return port; |
| 30 | } |
| 31 | |
| 32 | /** The protocol as git-lfs speaks it: a batch request, then the `basic` transfer's PUT and GET on the hrefs it hands back. */ |
| 33 | describe.skipIf(!hasBackends |
| 34 | let app: |
| 35 | let origin: |
| 36 | let work: |
| 37 | let keys: |
| 38 | let sshKey: |
| 39 | let sshPort: |
| 40 | let owner: |
| 41 | let forker: |
| 42 | const stamp = |
| 43 | const ownerName = |
| 44 | const forkerName = |
| 45 | let repo: |
| 46 | |
| 47 | const file = |
| 48 | const oid = |
| 49 | |
| 50 | const api = |
| 51 | const basic = |
| 52 | `Basic ${Buffer.from |
| 53 | const batch = |
| 54 | path: string, |
| 55 | operation: 'download |
| 56 | objects: { |
| 57 | authorization?: string |
| 58 | ) => { |
| 59 | const call = |
| 60 | .post(`/${path |
| 61 | .set('accept', |
| 62 | .set('content-type', |
| 63 | if (authorization) call |
| 64 | return call.send |
| 65 | JSON.stringify({ |
| 66 | ); |
| 67 | }; |
| 68 | const put = |
| 69 | path: string, |
| 70 | objectId: string, |
| 71 | bytes: Buffer |
| 72 | authorization = basic( |
| 73 | ) => |
| 74 | api() |
| 75 | .put(`/${path |
| 76 | .set('authorization', |
| 77 | .set('content-type', |
| 78 | .send(bytes); |
| 79 | const bytesOf = |
| 80 | call |
| 81 | .buffer(true) |
| 82 | .parse((response, |
| 83 | const chunks: |
| 84 | response.on('data |
| 85 | response.on('end |
| 86 | }) |
| 87 | .expect(200) |
| 88 | .then((response) |
| 89 | const sha256 = |
| 90 | createHash('sha256'). |
| 91 | const pointer = |
| 92 | `version https://git-lfs.github.com/spec/v1\noid sha256:${objectId} |
| 93 | const remote = |
| 94 | `${origin.replace( |
| 95 | const sshCommand = |
| 96 | `ssh -i ${sshKey} -p ${ |
| 97 | // The machine's own git config, such as a global or system-wide `git lfs install` (GitHub's runners have one), must not change what a test pushes. |
| 98 | const gitEnv = |
| 99 | ...process.env, |
| 100 | GIT_CONFIG_GLOBAL: '/dev/null', |
| 101 | GIT_CONFIG_NOSYSTEM: '1', |
| 102 | GIT_TERMINAL_PROMPT: '0', |
| 103 | GIT_SSH_COMMAND: sshCommand(), |
| 104 | }); |
| 105 | const git = |
| 106 | execFileSync( |
| 107 | 'git', |
| 108 | ['-c', 'user.name=E2E' |
| 109 | { cwd, encoding: |
| 110 | ); |
| 111 | // The server runs in this process, so anything that talks to it must not block the event loop. |
| 112 | const run = |
| 113 | promisify(execFile)('git |
| 114 | const usage = |
| 115 | api() |
| 116 | .get(`/api/storage/${account |
| 117 | .set('cookie', |
| 118 | .expect(200) |
| 119 | .then((response) |
| 120 | |
| 121 | beforeAll(async () |
| 122 | keys = mkdtempSync( |
| 123 | const hostKey = |
| 124 | sshKey = path. |
| 125 | for (const file |
| 126 | execFileSync('ssh-keygen', |
| 127 | } |
| 128 | const port = |
| 129 | sshPort = await |
| 130 | ({ app, origin |
| 131 | { |
| 132 | LFS_STORAGE_QUOTA_BYTES: '1kb', |
| 133 | BETTER_AUTH_URL: `http://127.0.0.1:${port |
| 134 | GIT_SSH_HOST_KEY: readFileSync(hostKey |
| 135 | GIT_SSH_PORT: String(sshPort |
| 136 | }, |
| 137 | port, |
| 138 | )); |
| 139 | work = mkdtempSync( |
| 140 | owner = await |
| 141 | forker = await |
| 142 | await api() |
| 143 | .post('/api/ssh-keys') |
| 144 | .set('cookie', |
| 145 | .send({ title: |
| 146 | .expect(201); |
| 147 | repo = ( |
| 148 | await api() |
| 149 | .post('/api/repositories') |
| 150 | .set('cookie', |
| 151 | .send({ name: |
| 152 | .expect(201) |
| 153 | ).body.slug; |
| 154 | }, 120_000); |
| 155 | |
| 156 | afterAll(async () |
| 157 | await app?.close |
| 158 | for (const directory |
| 159 | if (directory) rmSync |
| 160 | } |
| 161 | }); |
| 162 | |
| 163 | it('challenges an anonymous upload, so git-lfs asks for credentials', |
| 164 | await batch(`${ |
| 165 | 401, |
| 166 | ); |
| 167 | }); |
| 168 | |
| 169 | it('hands out an upload action carrying the request’s credentials', |
| 170 | const authorization = |
| 171 | const { body, |
| 172 | `${ownerName}/${repo |
| 173 | 'upload', |
| 174 | [{ oid, size |
| 175 | authorization, |
| 176 | ).expect(200); |
| 177 | |
| 178 | expect(headers['content-type |
| 179 | expect(body.transfer |
| 180 | expect(body.objects |
| 181 | href: `${origin} |
| 182 | header: { Authorization: authorization |
| 183 | }); |
| 184 | }); |
| 185 | |
| 186 | it('refuses bytes that do not hash to the oid, and keeps nothing of them', |
| 187 | const { body |
| 188 | `${ownerName}/${repo |
| 189 | oid, |
| 190 | randomBytes(512), |
| 191 | ).expect(422); |
| 192 | expect(body.message |
| 193 | |
| 194 | const download = |
| 195 | { oid, size: |
| 196 | ]).expect(200); |
| 197 | expect(download.body |
| 198 | }); |
| 199 | |
| 200 | it('stores an upload, then offers it for download and asks for it no more', |
| 201 | await put(`${ |
| 202 | |
| 203 | const upload = |
| 204 | `${ownerName}/${repo |
| 205 | 'upload', |
| 206 | [{ oid, size |
| 207 | basic(ownerName, owner |
| 208 | ).expect(200); |
| 209 | expect(upload.body |
| 210 | |
| 211 | const download = |
| 212 | { oid, size: |
| 213 | ]).expect(200); |
| 214 | const { href |
| 215 | const body = |
| 216 | expect(Buffer.compare |
| 217 | }); |
| 218 | |
| 219 | it('bills uploads against the LFS quota and refuses one past it', |
| 220 | const usageBefore = |
| 221 | expect(usageBefore.lfs |
| 222 | expect(usageBefore.usedBytes |
| 223 | |
| 224 | const large = |
| 225 | const { body |
| 226 | `${ownerName}/${repo |
| 227 | createHash('sha256'). |
| 228 | large, |
| 229 | ).expect(413); |
| 230 | expect(body.message |
| 231 | }); |
| 232 | |
| 233 | it('copies objects into a fork, billed to the forker’s fork limit, and removes them with it', |
| 234 | const { body: |
| 235 | .post(`/api/repositories/${ownerName |
| 236 | .set('cookie', |
| 237 | .send({ name: |
| 238 | .expect(201); |
| 239 | |
| 240 | const { lfs, |
| 241 | expect(lfs.usedBytes |
| 242 | expect(forkUsage.usedBytes |
| 243 | await api() |
| 244 | .get(`/${forkerName |
| 245 | .expect(200); |
| 246 | |
| 247 | await api() |
| 248 | .delete(`/api/repositories/${forkerName |
| 249 | .set('cookie', |
| 250 | .expect((response) |
| 251 | const s3 = |
| 252 | const { KeyCount |
| 253 | Bucket: s3.bucket |
| 254 | Prefix: lfsObjectKey(fork |
| 255 | }); |
| 256 | expect(KeyCount).toBe |
| 257 | }); |
| 258 | |
| 259 | it.skipIf(!hasGitLfs |
| 260 | 'round-trips a file through the git-lfs client', |
| 261 | async () => |
| 262 | const { body: |
| 263 | .post('/api/repositories') |
| 264 | .set('cookie', |
| 265 | .send({ name: |
| 266 | .expect(201); |
| 267 | const target = |
| 268 | const pushed = |
| 269 | |
| 270 | git(work, ' |
| 271 | git(pushed, ' |
| 272 | git(pushed, ' |
| 273 | const contents = |
| 274 | writeFileSync(path.join |
| 275 | git(pushed, ' |
| 276 | git(pushed, ' |
| 277 | await run(pushed |
| 278 | |
| 279 | // `-c` rather than `git lfs install`, which would write to the global config. |
| 280 | await run( |
| 281 | work, |
| 282 | '-c', |
| 283 | 'filter.lfs.process=git-lfs filter-process', |
| 284 | '-c', |
| 285 | 'filter.lfs.required=true', |
| 286 | 'clone', |
| 287 | '-q', |
| 288 | target, |
| 289 | 'cloned', |
| 290 | ); |
| 291 | const cloned = |
| 292 | expect(Buffer.compare |
| 293 | expect((await usage |
| 294 | }, |
| 295 | 60_000, |
| 296 | ); |
| 297 | |
| 298 | it('hands SSH users an HTTPS endpoint and a token scoped to the repository', |
| 299 | const { stdout |
| 300 | ...sshCommand().split |
| 301 | 'git@127.0.0.1', |
| 302 | `git-lfs-authenticate ${ownerName}/${ |
| 303 | ]); |
| 304 | const { href, |
| 305 | expect(href).toBe |
| 306 | expect(expires_in).toBe |
| 307 | |
| 308 | const { body |
| 309 | `${ownerName}/${repo |
| 310 | 'upload', |
| 311 | [{ oid: sha256 |
| 312 | header.Authorization, |
| 313 | ).expect(200); |
| 314 | expect(body.objects |
| 315 | }); |
| 316 | |
| 317 | it('copies the objects a merged pull request from a fork points at, and serves files as their objects', |
| 318 | const main = |
| 319 | git(work, ' |
| 320 | writeFileSync(path.join |
| 321 | git(main, ' |
| 322 | git(main, ' |
| 323 | await run(main |
| 324 | |
| 325 | const { body: |
| 326 | .post(`/api/repositories/${ownerName |
| 327 | .set('cookie', |
| 328 | .send({ name: |
| 329 | .expect(201); |
| 330 | const asset = |
| 331 | await put( |
| 332 | `${forkerName}/${fork |
| 333 | sha256(asset), |
| 334 | asset, |
| 335 | basic(forkerName, forker |
| 336 | ).expect(200); |
| 337 | |
| 338 | git(main, ' |
| 339 | writeFileSync(path.join |
| 340 | writeFileSync( |
| 341 | path.join(main |
| 342 | '*.bin filter=lfs diff=lfs merge=lfs -text\n', |
| 343 | ); |
| 344 | // A pointer to an object nobody uploaded, outside what .gitattributes tracks so a clone can still check it out. |
| 345 | writeFileSync(path.join |
| 346 | git(main, ' |
| 347 | git(main, ' |
| 348 | await run( |
| 349 | main, |
| 350 | 'push', |
| 351 | '-q', |
| 352 | remote(forkerName, forker |
| 353 | 'feature', |
| 354 | ); |
| 355 | |
| 356 | const { body: |
| 357 | .post(`/api/repositories/${ownerName |
| 358 | .set('cookie', |
| 359 | .send({ title: |
| 360 | .expect(201); |
| 361 | await api() |
| 362 | .post(`/api/repositories/${ownerName |
| 363 | .set('cookie', |
| 364 | .send({ method: |
| 365 | .expect(201); |
| 366 | |
| 367 | const download = |
| 368 | { oid: sha256( |
| 369 | ]).expect(200); |
| 370 | expect(download.body |
| 371 | expect((await usage |
| 372 | |
| 373 | const blob = |
| 374 | api() |
| 375 | .get(`/api/repositories/${ownerName |
| 376 | .query({ path |
| 377 | .expect(200) |
| 378 | .then((response) |
| 379 | const stored = |
| 380 | expect(stored).toMatchObject |
| 381 | lfs: 'stored', |
| 382 | size: 300, |
| 383 | encoding: 'base64', |
| 384 | }); |
| 385 | expect(Buffer.from |
| 386 | expect(await blob |
| 387 | lfs: 'missing', |
| 388 | encoding: 'utf-8', |
| 389 | }); |
| 390 | expect((await blob |
| 391 | |
| 392 | const raw = |
| 393 | api() |
| 394 | .get(`/api/repositories/${ownerName |
| 395 | .query({ path: |
| 396 | ); |
| 397 | expect(raw.equals |
| 398 | }); |
| 399 | |
| 400 | it.skipIf(!hasGitLfs |
| 401 | 'clones over SSH, with git-lfs falling back to git-lfs-authenticate', |
| 402 | async () => |
| 403 | await run( |
| 404 | work, |
| 405 | '-c', |
| 406 | 'filter.lfs.process=git-lfs filter-process', |
| 407 | '-c', |
| 408 | 'filter.lfs.required=true', |
| 409 | 'clone', |
| 410 | '-q', |
| 411 | `ssh://git@127.0.0.1:${sshPort}/${ |
| 412 | 'ssh-cloned', |
| 413 | ); |
| 414 | const contents = |
| 415 | expect(contents.length) |
| 416 | }, |
| 417 | 60_000, |
| 418 | ); |
| 419 | |
| 420 | it('locks a path for one person, which others see and cannot undo', |
| 421 | await app |
| 422 | .get<Database |
| 423 | .insert(schema.repositoryCollaborator |
| 424 | .values({ |
| 425 | repositoryId: ( |
| 426 | await app |
| 427 | .get<Database |
| 428 | .select({ id: |
| 429 | .from(schema.repository |
| 430 | .where(eq(schema |
| 431 | )[0].id, |
| 432 | userId: forker.userId |
| 433 | role: 'write', |
| 434 | acceptedAt: new Date |
| 435 | }); |
| 436 | const locks = |
| 437 | api() |
| 438 | .post(`/${ownerName |
| 439 | .set('authorization', |
| 440 | .set('content-type', |
| 441 | .send(JSON.stringify |
| 442 | const asOwner = |
| 443 | const asForker = |
| 444 | |
| 445 | const { body: |
| 446 | path: 'asset.bin', |
| 447 | }).expect(201) |
| 448 | expect(created.lock |
| 449 | path: 'asset.bin', |
| 450 | owner: { name: ownerName |
| 451 | }); |
| 452 | const { body: |
| 453 | path: 'asset.bin', |
| 454 | }).expect(409) |
| 455 | expect(conflict.lock |
| 456 | |
| 457 | const listed = |
| 458 | .get(`/${ownerName |
| 459 | .query({ path: |
| 460 | .expect(200); |
| 461 | expect(listed.body |
| 462 | |
| 463 | const ours = |
| 464 | expect(ours.body |
| 465 | const theirs = |
| 466 | expect(theirs.body |
| 467 | |
| 468 | await locks(`/ |
| 469 | 403, |
| 470 | ); |
| 471 | await locks(`/ |
| 472 | await api() |
| 473 | .get(`/${ownerName |
| 474 | .expect(200, { |
| 475 | }); |
| 476 | }); |