| 1 | import { type Database, schema } from '@ghost/db'; |
| 2 | import { Inject, Injectable |
| 3 | import { ConfigService } |
| 4 | import { and, eq |
| 5 | |
| 6 | import { DATABASE } |
| 7 | import type { Executor |
| 8 | import { DEFAULT_RELEASE_ASSET_MAX_BYTES } |
| 9 | import { PUT_OBJECT_MAX_BYTES } |
| 10 | import { parseByteSize } |
| 11 | import { RESERVATION_TTL } |
| 12 | import { |
| 13 | MergeStorageQuotaExceededError, |
| 14 | PullRefWriteTooLargeError, |
| 15 | StorageQuotaExceededError, |
| 16 | UnmergedPullRefQuotaExceededError, |
| 17 | } from '../../lib/storage/storage.errors.js'; |
| 18 | import { |
| 19 | billedTo, |
| 20 | type StorageAccount, |
| 21 | storageAccountKey, |
| 22 | type StorageKind, |
| 23 | } from '../../lib/storage/storage-account.js'; |
| 24 | |
| 25 | type PullRequestState |
| 26 | |
| 27 | /** Bytes an account may keep in object storage, per {@ |
| 28 | @Injectable() |
| 29 | export class StorageQuotaService |
| 30 | private readonly defaults |
| 31 | /** Largest single release asset, from `RELEASE_ASSET_MAX_BYTES`; 2 GB unless set, and never past what one upload can carry. */ |
| 32 | readonly maxAssetBytes: |
| 33 | /** Most one update of a request's `refs/pull/*` may add to its base log, from `PULL_REF_MAX_BYTES`; unset, nothing is limited. */ |
| 34 | private readonly maxPullRefWriteBytes |
| 35 | /** Most an author's unmerged requests may hold in base logs altogether, from `PULL_REF_UNMERGED_MAX_BYTES`; unset, nothing is limited. */ |
| 36 | private readonly maxUnmergedPullRefBytes |
| 37 | |
| 38 | constructor( |
| 39 | @Inject(DATABASE) |
| 40 | config: ConfigService |
| 41 | ) { |
| 42 | this.defaults = |
| 43 | repository: parseByteSize( |
| 44 | config.get<string |
| 45 | 'STORAGE_QUOTA_BYTES', |
| 46 | ), |
| 47 | fork: parseByteSize( |
| 48 | config.get<string |
| 49 | 'FORK_STORAGE_QUOTA_BYTES', |
| 50 | ), |
| 51 | lfs: parseByteSize( |
| 52 | config.get<string |
| 53 | 'LFS_STORAGE_QUOTA_BYTES', |
| 54 | ), |
| 55 | asset: parseByteSize( |
| 56 | config.get<string |
| 57 | 'ASSET_STORAGE_QUOTA_BYTES', |
| 58 | ), |
| 59 | }; |
| 60 | this.maxAssetBytes = |
| 61 | parseByteSize( |
| 62 | config.get<string |
| 63 | 'RELEASE_ASSET_MAX_BYTES', |
| 64 | ) ?? DEFAULT_RELEASE_ASSET_MAX_BYTES, |
| 65 | // assets are uploaded in one PutObject |
| 66 | PUT_OBJECT_MAX_BYTES, |
| 67 | ); |
| 68 | this.maxPullRefWriteBytes = |
| 69 | config.get<string |
| 70 | 'PULL_REF_MAX_BYTES', |
| 71 | ); |
| 72 | this.maxUnmergedPullRefBytes = |
| 73 | config.get<string |
| 74 | 'PULL_REF_UNMERGED_MAX_BYTES', |
| 75 | ); |
| 76 | } |
| 77 | |
| 78 | async quotaOf( |
| 79 | account: StorageAccount |
| 80 | kind: StorageKind |
| 81 | executor: Executor |
| 82 | ) { |
| 83 | const [limit] |
| 84 | .select({ |
| 85 | repository: schema.storageLimit |
| 86 | fork: schema.storageLimit |
| 87 | lfs: schema.storageLimit |
| 88 | asset: schema.storageLimit |
| 89 | }) |
| 90 | .from(schema.storageLimit |
| 91 | .where( |
| 92 | 'organizationId' in account |
| 93 | ? eq(schema. |
| 94 | : eq(schema. |
| 95 | ); |
| 96 | return limit?.[kind |
| 97 | } |
| 98 | |
| 99 | async usageOf( |
| 100 | account: StorageAccount |
| 101 | kind: StorageKind |
| 102 | executor: Executor |
| 103 | ) { |
| 104 | // A fork's files all count against the fork limit; elsewhere each kind of file has a limit of its own. |
| 105 | const repositories = |
| 106 | account, |
| 107 | kind === 'fork' |
| 108 | ); |
| 109 | const counts = |
| 110 | let used = |
| 111 | |
| 112 | if (counts('lfs |
| 113 | const [lfs] |
| 114 | .select({ |
| 115 | used: sql<string |
| 116 | }) |
| 117 | .from(schema.lfsObject |
| 118 | .innerJoin( |
| 119 | schema.repository, |
| 120 | eq(schema.repository |
| 121 | ) |
| 122 | .where( |
| 123 | and( |
| 124 | repositories, |
| 125 | or( |
| 126 | isNotNull(schema.lfsObject |
| 127 | gt(schema.lfsObject |
| 128 | ), |
| 129 | ), |
| 130 | ); |
| 131 | used += Number( |
| 132 | } |
| 133 | |
| 134 | if (counts('asset |
| 135 | const [assets] |
| 136 | .select({ |
| 137 | used: sql<string |
| 138 | }) |
| 139 | .from(schema.releaseAsset |
| 140 | .innerJoin( |
| 141 | schema.repository, |
| 142 | eq(schema.repository |
| 143 | ) |
| 144 | .where( |
| 145 | and( |
| 146 | repositories, |
| 147 | or( |
| 148 | eq(schema.releaseAsset |
| 149 | gt( |
| 150 | schema.releaseAsset.createdAt |
| 151 | sql`now() - ${RESERVATION_TTL} |
| 152 | ), |
| 153 | ), |
| 154 | ), |
| 155 | ); |
| 156 | used += Number( |
| 157 | } |
| 158 | |
| 159 | // Attachments bill whoever uploaded them, wherever they are posted, so commenting on someone else's repository never fills the owner's quota. |
| 160 | if (kind === |
| 161 | const [attachments] |
| 162 | .select({ |
| 163 | used: sql<string |
| 164 | }) |
| 165 | .from(schema.attachment |
| 166 | .where( |
| 167 | and( |
| 168 | eq(schema.attachment |
| 169 | or( |
| 170 | isNotNull(schema.attachment |
| 171 | gt(schema.attachment |
| 172 | ), |
| 173 | ), |
| 174 | ); |
| 175 | used += Number( |
| 176 | } |
| 177 | |
| 178 | if (counts('repository |
| 179 | const [logs] |
| 180 | .select({ |
| 181 | used: sql<string |
| 182 | }) |
| 183 | .from(schema.repositoryLogEntry |
| 184 | .innerJoin( |
| 185 | schema.repository, |
| 186 | eq(schema.repository |
| 187 | ) |
| 188 | .where(repositories); |
| 189 | // A request's head starts counting against the base repository once it merges, never before. |
| 190 | const [pullRefs] |
| 191 | .select({ |
| 192 | used: sql<string |
| 193 | }) |
| 194 | .from(schema.pullRequestRefWrite |
| 195 | .innerJoin( |
| 196 | schema.pullRequest, |
| 197 | eq(schema.pullRequest |
| 198 | ) |
| 199 | .innerJoin( |
| 200 | schema.repository, |
| 201 | eq(schema.repository |
| 202 | ) |
| 203 | .where(and(repositories |
| 204 | used += Number( |
| 205 | } |
| 206 | |
| 207 | return used; |
| 208 | } |
| 209 | |
| 210 | /** Bytes Ghost has written into base logs for `authorId`'s requests that never merged, open or closed. */ |
| 211 | async unmergedPullRefBytesOf(authorId |
| 212 | const [row] |
| 213 | .select({ |
| 214 | used: sql<string |
| 215 | }) |
| 216 | .from(schema.pullRequestRefWrite |
| 217 | .innerJoin( |
| 218 | schema.pullRequest, |
| 219 | eq(schema.pullRequest |
| 220 | ) |
| 221 | .innerJoin(schema.issue |
| 222 | .where( |
| 223 | and( |
| 224 | eq(schema.issue |
| 225 | ne(schema.pullRequest |
| 226 | ), |
| 227 | ); |
| 228 | return Number(row |
| 229 | } |
| 230 | |
| 231 | /** Runs `write`, which must commit the entry and record its row, only if neither pull ref limit refuses `bytes`. A merged request's bytes bill its base account, so only the per-write limit applies to it. Writes for one author are serialized, so two syncs cannot both fit into the last free space. */ |
| 232 | async reservePullRefWrite<T |
| 233 | { authorId, state |
| 234 | bytes: number, |
| 235 | write: (tx |
| 236 | ) { |
| 237 | if ( |
| 238 | this.maxPullRefWriteBytes !== |
| 239 | bytes > this. |
| 240 | ) { |
| 241 | throw new PullRefWriteTooLargeError( |
| 242 | } |
| 243 | return this.db |
| 244 | const limit = |
| 245 | if (limit !== |
| 246 | await tx.execute |
| 247 | sql`select pg_advisory_xact_lock(hashtext(${`pull-refs:${ |
| 248 | ); |
| 249 | const used = |
| 250 | if (used + |
| 251 | throw new UnmergedPullRefQuotaExceededError( |
| 252 | } |
| 253 | } |
| 254 | return write(tx |
| 255 | }); |
| 256 | } |
| 257 | |
| 258 | /** Runs `insert`, which must write the row that holds `bytes`, only if the account has room for them. Reservations for one account are serialized, so two uploads cannot both fit into the last free space. */ |
| 259 | reserve<T> |
| 260 | account: StorageAccount |
| 261 | kind: StorageKind |
| 262 | bytes: number, |
| 263 | insert: (tx |
| 264 | ) { |
| 265 | return this.db |
| 266 | const quota = |
| 267 | // An account past its quota may still write nothing, such as a push that only deletes branches. |
| 268 | if (quota !== |
| 269 | await this.lockAccount |
| 270 | const used = |
| 271 | if (used + |
| 272 | throw new StorageQuotaExceededError( |
| 273 | } |
| 274 | } |
| 275 | return insert(tx |
| 276 | }); |
| 277 | } |
| 278 | |
| 279 | /** Refuses a merge once `account` is at or past its quota. A merge needs room left, not room enough: the pull refs it starts billing may carry the account past its quota, and the next merge is the one refused. Run inside the merge's transaction, so merges into one account see each other. */ |
| 280 | async assertRoomToMerge( |
| 281 | account: StorageAccount |
| 282 | kind: StorageKind |
| 283 | tx: Executor |
| 284 | ) { |
| 285 | const quota = |
| 286 | if (quota === |
| 287 | await this.lockAccount |
| 288 | const used = |
| 289 | if (used >= |
| 290 | } |
| 291 | |
| 292 | private lockAccount( |
| 293 | account: StorageAccount |
| 294 | kind: StorageKind |
| 295 | tx: Executor |
| 296 | ) { |
| 297 | return tx.execute |
| 298 | sql`select pg_advisory_xact_lock(hashtext(${`${storageAccountKey |
| 299 | ); |
| 300 | } |
| 301 | } |