| 1 | /** |
| 2 | * Moves every write-ahead log from the old `repos/<username>/<slug>` keyspace to `repos/<repositoryId>`, which is what the transport reads since repository ids replaced name pairs. |
| 3 | * |
| 4 | * A log left behind is invisible to the API: `readIndex` returns null, the cache replays an empty index, and the repository serves as freshly initialised. |
| 5 | * |
| 6 | * Entries are copied before the index that names them, so an interrupted run leaves the new prefix either absent or complete, never pointing at packs that have not landed. Old keys are kept unless `--prune` is passed. |
| 7 | * |
| 8 | * bun scripts/migrate-wal-keys.ts [--apply] [--prune] |
| 9 | */ |
| 10 | import { S3 } |
| 11 | import { createDatabase, schema |
| 12 | import { eq, isNotNull |
| 13 | |
| 14 | const apply = process |
| 15 | const prune = process |
| 16 | |
| 17 | const bucket = required |
| 18 | const s3 = new |
| 19 | forcePathStyle: true, |
| 20 | endpoint: required('S3_ENDPOINT |
| 21 | region: 'auto', |
| 22 | credentials: { |
| 23 | accessKeyId: required('S3_ACCESS_KEY_ID |
| 24 | secretAccessKey: required('S3_SECRET_ACCESS_KEY |
| 25 | }, |
| 26 | }); |
| 27 | |
| 28 | const { db, |
| 29 | connectionString: required('DATABASE_URL |
| 30 | }); |
| 31 | |
| 32 | const rows = await |
| 33 | .select({ |
| 34 | id: schema.repository |
| 35 | username: schema.user |
| 36 | slug: schema.repository |
| 37 | }) |
| 38 | .from(schema.repository |
| 39 | .innerJoin(schema.user |
| 40 | .where(isNotNull(schema |
| 41 | |
| 42 | let moved = 0 |
| 43 | let skipped = 0 |
| 44 | |
| 45 | for (const { id |
| 46 | const from = |
| 47 | const label = |
| 48 | |
| 49 | if (await exists |
| 50 | skipped++; |
| 51 | continue; |
| 52 | } |
| 53 | if (!(await |
| 54 | console.log(`no log |
| 55 | skipped++; |
| 56 | continue; |
| 57 | } |
| 58 | |
| 59 | const keys = |
| 60 | const entries = |
| 61 | |
| 62 | console.log( |
| 63 | `${apply ? ' |
| 64 | ); |
| 65 | if (!apply) |
| 66 | moved++; |
| 67 | continue; |
| 68 | } |
| 69 | |
| 70 | for (const key |
| 71 | await s3.copyObject |
| 72 | Bucket: bucket, |
| 73 | Key: key.replace |
| 74 | CopySource: `${bucket} |
| 75 | }); |
| 76 | } |
| 77 | |
| 78 | await s3.copyObject |
| 79 | Bucket: bucket, |
| 80 | Key: `repos/${id |
| 81 | CopySource: `${bucket} |
| 82 | }); |
| 83 | |
| 84 | if (prune) { |
| 85 | for (const key |
| 86 | await s3.deleteObject |
| 87 | } |
| 88 | } |
| 89 | |
| 90 | moved++; |
| 91 | } |
| 92 | |
| 93 | console.log( |
| 94 | `\n${apply ? |
| 95 | (apply |
| 96 | ? '' |
| 97 | : '\nre-run with --apply to write, and --prune to drop old keys'), |
| 98 | ); |
| 99 | |
| 100 | await pool.end() |
| 101 | |
| 102 | async function exists( |
| 103 | try { |
| 104 | await s3.headObject |
| 105 | return true; |
| 106 | } catch { |
| 107 | return false; |
| 108 | } |
| 109 | } |
| 110 | |
| 111 | async function listPrefix( |
| 112 | const keys: |
| 113 | let token: |
| 114 | |
| 115 | do { |
| 116 | const page = |
| 117 | Bucket: bucket, |
| 118 | Prefix: prefix, |
| 119 | ContinuationToken: token, |
| 120 | }); |
| 121 | for (const object |
| 122 | if (object.Key |
| 123 | } |
| 124 | token = page. |
| 125 | } while (token) |
| 126 | |
| 127 | return keys; |
| 128 | } |
| 129 | |
| 130 | function required(name: |
| 131 | const value = |
| 132 | if (!value) |
| 133 | return value; |
| 134 | } |