| 1 | import { type Database, schema } from '@ghost/db'; |
| 2 | import { Inject, Injectable |
| 3 | import { |
| 4 | and, |
| 5 | countDistinct, |
| 6 | desc, |
| 7 | eq, |
| 8 | inArray, |
| 9 | isNull, |
| 10 | max, |
| 11 | sql, |
| 12 | sum, |
| 13 | } from 'drizzle-orm'; |
| 14 | |
| 15 | import { DATABASE } |
| 16 | import { runGit, runGitStream |
| 17 | import { resolveCommit } |
| 18 | import { resolveDefaultRef } |
| 19 | import { excluded } |
| 20 | |
| 21 | /** Rows buffered before a flush. Keeps a full rebuild's memory bounded. */ |
| 22 | const FLUSH_THRESHOLD = 5_000 |
| 23 | /** Postgres caps a statement at 65535 bind parameters; this table binds five. */ |
| 24 | const INSERT_CHUNK = 1_000 |
| 25 | |
| 26 | const RECORD = ' |
| 27 | const FIELD = ' |
| 28 | const FORMAT = ` |
| 29 | |
| 30 | interface PendingDay { |
| 31 | authorEmail: string; |
| 32 | authorName: string; |
| 33 | day: string; |
| 34 | commits: number; |
| 35 | } |
| 36 | |
| 37 | /** |
| 38 | * Keeps `repository_contribution` in step with a repository's default branch, so the profile contribution graph costs one indexed query instead of materializing and walking every repository the user owns. |
| 39 | * |
| 40 | * A cache of git, rebuilt from the object database whenever the stored position stops making sense. Only the default branch is indexed. |
| 41 | */ |
| 42 | export interface IndexedContributor |
| 43 | authorEmail: string; |
| 44 | /** Name from the author's newest indexed commit, preferring the account's. */ |
| 45 | authorName: string; |
| 46 | username: string |
| 47 | image: string |
| 48 | commits: number; |
| 49 | /** Newest indexed day, UTC midnight. Day-granular: the index has no times. */ |
| 50 | lastCommittedAt: Date |
| 51 | } |
| 52 | |
| 53 | @Injectable() |
| 54 | export class RepositoryContributionService |
| 55 | private readonly logger |
| 56 | private readonly inFlight |
| 57 | |
| 58 | constructor(@Inject( |
| 59 | |
| 60 | /** Brings the index up to the default tip, returning that tip (null when nothing has been pushed yet). Concurrent callers share one walk. */ |
| 61 | async sync({ |
| 62 | repositoryId, |
| 63 | repoDirectory, |
| 64 | }: { |
| 65 | repositoryId: string |
| 66 | repoDirectory: string |
| 67 | }): Promise |
| 68 | const pending = |
| 69 | if (pending) return |
| 70 | |
| 71 | const run = |
| 72 | this.inFlight.delete |
| 73 | ); |
| 74 | this.inFlight.set |
| 75 | return run; |
| 76 | } |
| 77 | |
| 78 | /** The contributors list, straight from the index: per-author totals with the linked account resolved over the `author_id` foreign key. No git, no materialization - callers only need read access to the repository row. */ |
| 79 | async listContributors({ |
| 80 | repositoryId, |
| 81 | limit = 100, |
| 82 | }: { |
| 83 | repositoryId: string |
| 84 | limit?: number |
| 85 | }): Promise |
| 86 | contributors: IndexedContributor |
| 87 | totalCommits: number |
| 88 | totalContributors: number |
| 89 | }> { |
| 90 | const pageSize = |
| 91 | const contribution = |
| 92 | // One person can commit from several addresses, so the grain of the list is the account where there is one, and the address where there is not. |
| 93 | const identity = |
| 94 | |
| 95 | const [rows, |
| 96 | this.db |
| 97 | .select({ |
| 98 | // Newest address wins, for an author with no account to name. |
| 99 | authorEmail: sql<string |
| 100 | // Newest name wins: the name on the author's latest indexed day, preferring the linked account's below. |
| 101 | gitName: sql<string |
| 102 | username: schema.user |
| 103 | name: schema.user |
| 104 | image: schema.user |
| 105 | commits: sum(contribution |
| 106 | lastDay: max(contribution |
| 107 | }) |
| 108 | .from(contribution) |
| 109 | .leftJoin(schema.user |
| 110 | .where(eq(contribution |
| 111 | .groupBy( |
| 112 | identity, |
| 113 | schema.user.username |
| 114 | schema.user.name |
| 115 | schema.user.image |
| 116 | ) |
| 117 | .orderBy(desc(sum |
| 118 | .limit(pageSize), |
| 119 | this.db |
| 120 | .select({ |
| 121 | commits: sum(contribution |
| 122 | authors: countDistinct(identity |
| 123 | }) |
| 124 | .from(contribution) |
| 125 | .where(eq(contribution |
| 126 | ]); |
| 127 | |
| 128 | return { |
| 129 | contributors: rows.flatMap |
| 130 | if (!row.lastDay |
| 131 | return [ |
| 132 | { |
| 133 | authorEmail: row.authorEmail |
| 134 | authorName: row.name |
| 135 | username: row.username |
| 136 | image: row.image |
| 137 | commits: Number(row |
| 138 | lastCommittedAt: new Date |
| 139 | }, |
| 140 | ]; |
| 141 | }), |
| 142 | totalCommits: Number(totals |
| 143 | totalContributors: Number(totals |
| 144 | }; |
| 145 | } |
| 146 | |
| 147 | private async reindex |
| 148 | repositoryId, |
| 149 | repoDirectory, |
| 150 | }: { |
| 151 | repositoryId: string |
| 152 | repoDirectory: string |
| 153 | }): Promise |
| 154 | const ref = |
| 155 | const tip = |
| 156 | if (!tip) |
| 157 | await this.forget |
| 158 | return null; |
| 159 | } |
| 160 | |
| 161 | const [state] |
| 162 | .select() |
| 163 | .from(schema.repositoryContributionIndex |
| 164 | .where(eq(schema |
| 165 | |
| 166 | if (state?.indexedCommitSha |
| 167 | // No new commits, but an author may have registered since the last walk: link rows that are still unattributed. |
| 168 | if (await this |
| 169 | await this.relink |
| 170 | } |
| 171 | return tip; |
| 172 | } |
| 173 | |
| 174 | // Only a fast-forward can be topped up. A force push or a pruned object makes the stored rows unrelated to the ref, so start over. |
| 175 | const incremental = |
| 176 | state !== undefined && |
| 177 | (await this.isAncestor |
| 178 | |
| 179 | if (!incremental) |
| 180 | |
| 181 | const range = |
| 182 | const written = |
| 183 | |
| 184 | await this.db |
| 185 | .insert(schema.repositoryContributionIndex |
| 186 | .values({ repositoryId |
| 187 | .onConflictDoUpdate({ |
| 188 | target: [schema.repositoryContributionIndex |
| 189 | set: { |
| 190 | indexedCommitSha: tip, |
| 191 | updatedAt: new Date |
| 192 | }, |
| 193 | }); |
| 194 | |
| 195 | this.logger.log |
| 196 | `Indexed ${written} contribution days for ${ |
| 197 | ); |
| 198 | |
| 199 | return tip; |
| 200 | } |
| 201 | |
| 202 | /** Streams the range newest-first, bucketing non-merge commits per author and UTC day. The first name seen for an email is the newest one, so it wins. */ |
| 203 | private async walk |
| 204 | repositoryId, |
| 205 | repoDirectory, |
| 206 | range, |
| 207 | }: { |
| 208 | repositoryId: string |
| 209 | repoDirectory: string |
| 210 | range: string |
| 211 | }) { |
| 212 | const pending = |
| 213 | let written = |
| 214 | |
| 215 | const stream = |
| 216 | args: [ |
| 217 | 'log', |
| 218 | FORMAT, |
| 219 | '--no-merges', |
| 220 | // ranges are built from refs, and one starting with "-" would parse as an option rather than as a revision |
| 221 | '--end-of-options', |
| 222 | range, |
| 223 | ], |
| 224 | gitDir: repoDirectory, |
| 225 | }); |
| 226 | |
| 227 | let carry = |
| 228 | for await (const |
| 229 | carry += chunk; |
| 230 | const records = |
| 231 | carry = records. |
| 232 | for (const record |
| 233 | this.accumulate(pending |
| 234 | if (pending.size |
| 235 | written += await |
| 236 | } |
| 237 | } |
| 238 | } |
| 239 | const trailing = |
| 240 | if (trailing && |
| 241 | written += await |
| 242 | } |
| 243 | |
| 244 | written += await |
| 245 | return written; |
| 246 | } |
| 247 | |
| 248 | /** Returns false for blank or malformed records, which are skipped. */ |
| 249 | private accumulate(pending |
| 250 | const [email, |
| 251 | if (!email || |
| 252 | const seconds = |
| 253 | if (!Number.isFinite |
| 254 | |
| 255 | const authorEmail = |
| 256 | const day = |
| 257 | const key = |
| 258 | const existing = |
| 259 | if (existing) { |
| 260 | existing.commits += |
| 261 | } else { |
| 262 | pending.set(key |
| 263 | authorEmail, |
| 264 | authorName: name || |
| 265 | day, |
| 266 | commits: 1, |
| 267 | }); |
| 268 | } |
| 269 | return true; |
| 270 | } |
| 271 | |
| 272 | private async flush |
| 273 | const rows = |
| 274 | repositoryId, |
| 275 | ...row, |
| 276 | })); |
| 277 | pending.clear(); |
| 278 | |
| 279 | const authorIds = |
| 280 | rows.map((row |
| 281 | ); |
| 282 | |
| 283 | for (let i |
| 284 | await this.db |
| 285 | .insert(schema.repositoryContribution |
| 286 | .values( |
| 287 | rows.slice(i |
| 288 | ...row, |
| 289 | authorId: authorIds.get |
| 290 | })), |
| 291 | ) |
| 292 | .onConflictDoUpdate({ |
| 293 | target: [ |
| 294 | schema.repositoryContribution.repositoryId |
| 295 | schema.repositoryContribution.authorEmail |
| 296 | schema.repositoryContribution.day |
| 297 | ], |
| 298 | set: { |
| 299 | commits: sql`${schema |
| 300 | authorName: excluded(schema |
| 301 | // Never unlink on a top-up: a missing match means "unknown", not "no longer theirs". |
| 302 | authorId: sql`coalesce(${ |
| 303 | }, |
| 304 | }); |
| 305 | } |
| 306 | |
| 307 | return rows.length; |
| 308 | } |
| 309 | |
| 310 | /** Maps lowercased author emails to account ids. An account is reachable under its primary address and under every verified extra, so a person who commits from two addresses attributes to one account. */ |
| 311 | private async resolveAuthorIds |
| 312 | const distinct = |
| 313 | if (distinct.length |
| 314 | |
| 315 | const [primaries, |
| 316 | this.db |
| 317 | .select({ id: |
| 318 | .from(schema.user |
| 319 | .where(inArray(sql |
| 320 | this.db |
| 321 | .select({ id: |
| 322 | .from(schema.userEmail |
| 323 | .where( |
| 324 | and( |
| 325 | inArray(schema.userEmail |
| 326 | eq(schema.userEmail |
| 327 | ), |
| 328 | ), |
| 329 | ]); |
| 330 | |
| 331 | return new Map |
| 332 | [...extras, ... |
| 333 | ); |
| 334 | } |
| 335 | |
| 336 | private async hasUnlinked |
| 337 | const [row] |
| 338 | .select({ one: |
| 339 | .from(schema.repositoryContribution |
| 340 | .where( |
| 341 | and( |
| 342 | eq(schema.repositoryContribution |
| 343 | isNull(schema.repositoryContribution |
| 344 | ), |
| 345 | ) |
| 346 | .limit(1); |
| 347 | return row !== |
| 348 | } |
| 349 | |
| 350 | /** Links unattributed rows to accounts that appeared after the commits were indexed - a registration, or an email added to an account later. The next sync then finds nothing left to link. */ |
| 351 | private async relink |
| 352 | await this.db |
| 353 | UPDATE "repository_contribution" AS c |
| 354 | SET "author_id" = u."id" |
| 355 | FROM "user" AS u |
| 356 | WHERE c."repository_id" = ${repositoryId} |
| 357 | AND c."author_id" IS NULL |
| 358 | AND lower(u."email") = c."author_email" |
| 359 | `); |
| 360 | await this.db |
| 361 | UPDATE "repository_contribution" AS c |
| 362 | SET "author_id" = e."user_id" |
| 363 | FROM "user_email" AS e |
| 364 | WHERE c."repository_id" = ${repositoryId} |
| 365 | AND c."author_id" IS NULL |
| 366 | AND e."verified" = true |
| 367 | AND e."email" = c."author_email" |
| 368 | `); |
| 369 | } |
| 370 | |
| 371 | private async forget |
| 372 | await this.db |
| 373 | .delete(schema.repositoryContribution |
| 374 | .where(eq(schema |
| 375 | await this.db |
| 376 | .delete(schema.repositoryContributionIndex |
| 377 | .where(eq(schema |
| 378 | } |
| 379 | |
| 380 | private async isAncestor |
| 381 | repoDirectory: string, |
| 382 | ancestor: string, |
| 383 | descendant: string, |
| 384 | ) { |
| 385 | // Exits non-zero both for "not an ancestor" and for an object that is gone; either way the stored position is unusable. |
| 386 | return runGit({ |
| 387 | args: ['merge-base', |
| 388 | gitDir: repoDirectory, |
| 389 | }).then( |
| 390 | () => true, |
| 391 | () => false, |
| 392 | ); |
| 393 | } |
| 394 | } |