| 1 | import { mkdir, rm } from "node:fs/promises"; |
| 2 | import path from "node:path |
| 3 | import { z } |
| 4 | |
| 5 | import { chunkBySize } |
| 6 | import type { Config |
| 7 | import { PermanentImportError, StaleAttemptError |
| 8 | import { |
| 9 | fetchFromGitHub, |
| 10 | listRefs, |
| 11 | presentCommits, |
| 12 | pushToGhost, |
| 13 | } from "./git.ts"; |
| 14 | import { |
| 15 | GhostCallbacks, |
| 16 | type ImportedIssue, |
| 17 | type ImportedPullRequest, |
| 18 | } from "./ghost.ts"; |
| 19 | import { |
| 20 | GitHubClient, |
| 21 | type GitHubComment, |
| 22 | type GitHubIssue, |
| 23 | type GitHubPull, |
| 24 | type GitHubRelease, |
| 25 | type GitHubRepository, |
| 26 | issueNumberOf, |
| 27 | loginOf, |
| 28 | } from "./github.ts"; |
| 29 | |
| 30 | const HEARTBEAT_MS = 30_000 |
| 31 | |
| 32 | export const jobSchema = |
| 33 | // Names the import's working directory, so it must not be able to climb out of it. |
| 34 | importId: z.string |
| 35 | attempt: z.uuid |
| 36 | source: z.string |
| 37 | destination: z.string |
| 38 | githubToken: z.string |
| 39 | ghostToken: z.string |
| 40 | lastAttempt: z.boolean |
| 41 | }); |
| 42 | |
| 43 | export type Job |
| 44 | |
| 45 | /** Runs one attempt to the end and reports how it went. Never throws: whatever goes wrong becomes the outcome the API decides to retry or not. */ |
| 46 | export async function runImport |
| 47 | const ghost = |
| 48 | config.GHOST_API_URL, |
| 49 | config.IMPORTER_SECRET, |
| 50 | job.importId, |
| 51 | job.attempt, |
| 52 | ); |
| 53 | const attempt = |
| 54 | const stop = |
| 55 | |
| 56 | const heartbeat = |
| 57 | ghost.heartbeat().catch |
| 58 | if (error instanceof |
| 59 | }); |
| 60 | }, HEARTBEAT_MS); |
| 61 | |
| 62 | // One directory per import, not per attempt: a retry resumes from the clone the last attempt fetched instead of downloading the repository again. Kept only while a retry can still use it. |
| 63 | const dir = |
| 64 | let keep = |
| 65 | try { |
| 66 | await mkdir(dir |
| 67 | const started = |
| 68 | const defaultBranch = |
| 69 | await ghost.finish |
| 70 | console.log(`import |
| 71 | } catch (error) |
| 72 | const reason = |
| 73 | if (reason instanceof |
| 74 | // the attempt that superseded this one owns the directory now |
| 75 | keep = true; |
| 76 | console.log(`import |
| 77 | return; |
| 78 | } |
| 79 | const message = |
| 80 | const retryable = |
| 81 | keep = retryable |
| 82 | console.error(`import |
| 83 | await ghost |
| 84 | .finish({ succeeded: |
| 85 | .catch((finishError: |
| 86 | console.error( |
| 87 | `import ${job.importId |
| 88 | ), |
| 89 | ); |
| 90 | } finally { |
| 91 | clearInterval(heartbeat); |
| 92 | if (!keep) |
| 93 | } |
| 94 | } |
| 95 | |
| 96 | async function importAll( |
| 97 | job: Job |
| 98 | config: Config |
| 99 | ghost: GhostCallbacks |
| 100 | dir: string, |
| 101 | attempt: AbortSignal |
| 102 | ) { |
| 103 | const log = |
| 104 | console.log(`import |
| 105 | // The history and the GitHub listings run side by side; whichever fails first stops the other. |
| 106 | const sibling = |
| 107 | const signal = |
| 108 | const failFast = |
| 109 | work.catch((error |
| 110 | sibling.abort(error |
| 111 | throw error; |
| 112 | }); |
| 113 | |
| 114 | const github = |
| 115 | const repository = |
| 116 | |
| 117 | const [refs, |
| 118 | failFast(copyHistory(job |
| 119 | failFast( |
| 120 | (async () => |
| 121 | await drain( |
| 122 | log, |
| 123 | "releases", |
| 124 | github.pages<GitHubRelease |
| 125 | async (page) |
| 126 | for (const chunk |
| 127 | await ghost.releases |
| 128 | }, |
| 129 | ); |
| 130 | // Only the fields `pullRequestFrom` reads: GitHub's full payload carries both repositories and runs to tens of KB per pull request. |
| 131 | const pulls = |
| 132 | await drain( |
| 133 | log, |
| 134 | "pulls", |
| 135 | github.pages<GitHubPull |
| 136 | async (page) |
| 137 | for (const pull |
| 138 | }, |
| 139 | ); |
| 140 | return pulls; |
| 141 | })(), |
| 142 | ), |
| 143 | ]); |
| 144 | |
| 145 | const started = |
| 146 | const mergeCommits = |
| 147 | pull.merge_commit_sha ? |
| 148 | ); |
| 149 | const present = |
| 150 | log( |
| 151 | `merge commits: ${present.size |
| 152 | ); |
| 153 | |
| 154 | // After the push: a pull request written before its refs exist would be synced against a history that is not there yet. |
| 155 | await drain( |
| 156 | log, |
| 157 | "issues", |
| 158 | github.pages<GitHubIssue |
| 159 | `/repos/${job.source |
| 160 | ), |
| 161 | async (page) |
| 162 | const issues = |
| 163 | issueFrom(issue, pulls |
| 164 | source: job.source |
| 165 | refs, |
| 166 | present, |
| 167 | }), |
| 168 | ); |
| 169 | for (const chunk |
| 170 | }, |
| 171 | ); |
| 172 | |
| 173 | // Sent after every issue, since writing an issue clears the comments a previous attempt left on it. |
| 174 | await drain( |
| 175 | log, |
| 176 | "comments", |
| 177 | commentPages(github, job |
| 178 | async (page) |
| 179 | const comments = |
| 180 | githubId: comment.id |
| 181 | issueNumber: issueNumberOf(comment |
| 182 | authorLogin: loginOf(comment |
| 183 | body: comment.body |
| 184 | createdAt: comment.created_at |
| 185 | updatedAt: comment.updated_at |
| 186 | })); |
| 187 | for (const chunk |
| 188 | }, |
| 189 | ); |
| 190 | |
| 191 | return repository.default_branch |
| 192 | } |
| 193 | |
| 194 | /** Clones from GitHub and pushes everything to Ghost in one push, returning the refs it carried. */ |
| 195 | async function copyHistory( |
| 196 | job: Job |
| 197 | config: Config |
| 198 | dir: string, |
| 199 | github: GitHubClient |
| 200 | signal: AbortSignal |
| 201 | log: (message |
| 202 | ) { |
| 203 | let started = |
| 204 | await fetchFromGitHub({ |
| 205 | dir, |
| 206 | source: job.source |
| 207 | token: job.githubToken |
| 208 | tokenWorked: github.authenticated |
| 209 | signal, |
| 210 | }); |
| 211 | const refs = |
| 212 | log(`fetch: ${refs |
| 213 | |
| 214 | started = performance. |
| 215 | await pushToGhost({ |
| 216 | dir, |
| 217 | apiUrl: config.GHOST_API_URL |
| 218 | destination: job.destination |
| 219 | token: job.ghostToken |
| 220 | signal, |
| 221 | }); |
| 222 | log(`push: ${secondsSince |
| 223 | return refs; |
| 224 | } |
| 225 | |
| 226 | /** |
| 227 | * Every comment on the repository's issues and pull requests. |
| 228 | * |
| 229 | * GitHub ends this listing at page 300 without saying so, so it is read in windows ordered by update time, each starting where the last one reached. Windows overlap on that instant; the API drops the comments it already has. |
| 230 | */ |
| 231 | export async function* |
| 232 | github: Pick |
| 233 | source: string, |
| 234 | ): AsyncGenerator<GitHubComment |
| 235 | let since: |
| 236 | for (;;) { |
| 237 | let reached = |
| 238 | const window = |
| 239 | for await (const |
| 240 | `/repos/${source}/issues/comments?sort=updated&direction=asc${ |
| 241 | )) { |
| 242 | if (page.length) |
| 243 | yield page; |
| 244 | } |
| 245 | if (reached === |
| 246 | since = reached; |
| 247 | } |
| 248 | } |
| 249 | |
| 250 | export function slimPull(pull |
| 251 | return { |
| 252 | number: pull.number |
| 253 | state: pull.state |
| 254 | draft: pull.draft |
| 255 | merged_at: pull.merged_at |
| 256 | merge_commit_sha: pull.merge_commit_sha |
| 257 | base: { ref: pull |
| 258 | head: { |
| 259 | ref: pull.head |
| 260 | sha: pull.head |
| 261 | repo: pull.head |
| 262 | }, |
| 263 | }; |
| 264 | } |
| 265 | |
| 266 | /** Feeds every page to `handle`, then logs the time spent waiting on GitHub apart from the time spent in `handle`, so a slow phase shows which side to fix. */ |
| 267 | async function drain< |
| 268 | log: (message |
| 269 | phase: string, |
| 270 | pages: AsyncIterable |
| 271 | handle: (page |
| 272 | ) { |
| 273 | const started = |
| 274 | let handling = |
| 275 | let items = |
| 276 | let pageCount = |
| 277 | for await (const |
| 278 | pageCount++; |
| 279 | items += page. |
| 280 | const handleStarted = |
| 281 | await handle(page |
| 282 | handling += performance. |
| 283 | } |
| 284 | const total = |
| 285 | log( |
| 286 | `${phase}: ${items |
| 287 | ); |
| 288 | } |
| 289 | |
| 290 | const seconds = ( |
| 291 | const secondsSince = ( |
| 292 | |
| 293 | export function releaseFrom(release |
| 294 | return { |
| 295 | tagName: release.tag_name |
| 296 | name: release.name |
| 297 | body: release.body |
| 298 | isDraft: release.draft |
| 299 | isPrerelease: release.prerelease |
| 300 | createdAt: release.created_at |
| 301 | publishedAt: release.published_at |
| 302 | }; |
| 303 | } |
| 304 | |
| 305 | export function issueFrom( |
| 306 | issue: GitHubIssue |
| 307 | pull: GitHubPull |
| 308 | context: { |
| 309 | ): ImportedIssue { |
| 310 | return { |
| 311 | number: issue.number |
| 312 | title: issue.title |
| 313 | body: issue.body |
| 314 | authorLogin: loginOf(issue |
| 315 | state: issue.state |
| 316 | createdAt: issue.created_at |
| 317 | updatedAt: issue.updated_at |
| 318 | closedAt: issue.closed_at |
| 319 | labels: issue.labels |
| 320 | name: label.name |
| 321 | color: label.color |
| 322 | description: label.description |
| 323 | })), |
| 324 | pullRequest: pull && |
| 325 | }; |
| 326 | } |
| 327 | |
| 328 | function pullRequestFrom( |
| 329 | pull: GitHubPull |
| 330 | { |
| 331 | source, |
| 332 | refs, |
| 333 | present, |
| 334 | }: { source |
| 335 | ): ImportedPullRequest { |
| 336 | const sameRepository = |
| 337 | pull.head.repo |
| 338 | const mergeCommitSha = |
| 339 | pull.merged_at && |
| 340 | pull.merge_commit_sha && |
| 341 | present.has(pull |
| 342 | ? pull.merge_commit_sha |
| 343 | : null; |
| 344 | return { |
| 345 | state: pull.merged_at |
| 346 | draft: pull.draft |
| 347 | baseRef: pull.base |
| 348 | headRef: pull.head |
| 349 | headSha: pull.head |
| 350 | headInRepository: sameRepository && |
| 351 | mergeCommitSha, |
| 352 | mergedAt: pull.merged_at |
| 353 | }; |
| 354 | } |