flaxo/ghost

public

Source code of Ghost - On Ghost

removed slop09daf9f22 days ago134 lines · 3.4 KB

1	/**
2	* Moves every write-ahead log from the old `repos/<username>/<slug>` keyspace to `repos/<repositoryId>`, which is what the transport reads since repository ids replaced name pairs.
3	*
4	* A log left behind is invisible to the API: `readIndex` returns null, the cache replays an empty index, and the repository serves as freshly initialised.
5	*
6	* Entries are copied before the index that names them, so an interrupted run leaves the new prefix either absent or complete, never pointing at packs that have not landed. Old keys are kept unless `--prune` is passed.
7	*
8	* bun scripts/migrate-wal-keys.ts [--apply] [--prune]
9	*/
10	import { S3 } from '@aws-sdk/client-s3';
11	import { createDatabase, schema } from '@ghost/db';
12	import { eq, isNotNull } from 'drizzle-orm';
13
14	const apply = process.argv.includes('--apply');
15	const prune = process.argv.includes('--prune');
16
17	const bucket = required('S3_BUCKET');
18	const s3 = new S3({
19	forcePathStyle: true,
20	endpoint: required('S3_ENDPOINT'),
21	region: 'auto',
22	credentials: {
23	accessKeyId: required('S3_ACCESS_KEY_ID'),
24	secretAccessKey: required('S3_SECRET_ACCESS_KEY'),
25	},
26	});
27
28	const { db, pool } = createDatabase({
29	connectionString: required('DATABASE_URL'),
30	});
31
32	const rows = await db
33	.select({
34	id: schema.repository.id,
35	username: schema.user.username,
36	slug: schema.repository.slug,
37	})
38	.from(schema.repository)
39	.innerJoin(schema.user, eq(schema.user.id, schema.repository.ownerId))
40	.where(isNotNull(schema.user.username));
41
42	let moved = 0;
43	let skipped = 0;
44
45	for (const { id, username, slug } of rows) {
46	const from = `${username}/${slug}`;
47	const label = `${from} -> ${id}`;
48
49	if (await exists(`repos/${id}/index`)) {
50	skipped++;
51	continue;
52	}
53	if (!(await exists(`repos/${from}/index`))) {
54	console.log(`no log ${label}`);
55	skipped++;
56	continue;
57	}
58
59	const keys = await listPrefix(`repos/${from}/`);
60	const entries = keys.filter((key) => key.includes('/entries/'));
61
62	console.log(
63	`${apply ? 'migrating' : 'would move'} ${label} (${keys.length} objects)`,
64	);
65	if (!apply) {
66	moved++;
67	continue;
68	}
69
70	for (const key of entries) {
71	await s3.copyObject({
72	Bucket: bucket,
73	Key: key.replace(`repos/${from}/`, `repos/${id}/`),
74	CopySource: `${bucket}/${key}`,
75	});
76	}
77
78	await s3.copyObject({
79	Bucket: bucket,
80	Key: `repos/${id}/index`,
81	CopySource: `${bucket}/repos/${from}/index`,
82	});
83
84	if (prune) {
85	for (const key of keys) {
86	await s3.deleteObject({ Bucket: bucket, Key: key });
87	}
88	}
89
90	moved++;
91	}
92
93	console.log(
94	`\n${apply ? 'migrated' : 'to migrate'}: ${moved}, skipped: ${skipped}` +
95	(apply
96	? ''
97	: '\nre-run with --apply to write, and --prune to drop old keys'),
98	);
99
100	await pool.end();
101
102	async function exists(key: string) {
103	try {
104	await s3.headObject({ Bucket: bucket, Key: key });
105	return true;
106	} catch {
107	return false;
108	}
109	}
110
111	async function listPrefix(prefix: string) {
112	const keys: string[] = [];
113	let token: string \| undefined;
114
115	do {
116	const page = await s3.listObjectsV2({
117	Bucket: bucket,
118	Prefix: prefix,
119	ContinuationToken: token,
120	});
121	for (const object of page.Contents ?? []) {
122	if (object.Key) keys.push(object.Key);
123	}
124	token = page.NextContinuationToken;
125	} while (token);
126
127	return keys;
128	}
129
130	function required(name: string) {
131	const value = process.env[name];
132	if (!value) throw new Error(`${name} is not set`);
133	return value;
134	}

removed slop09daf9f22 days ago134 lines · 3.4 KB

1	/**
2	* Moves every write-ahead log from the old `repos/<username>/<slug>` keyspace to `repos/<repositoryId>`, which is what the transport reads since repository ids replaced name pairs.
3	*
4	* A log left behind is invisible to the API: `readIndex` returns null, the cache replays an empty index, and the repository serves as freshly initialised.
5	*
6	* Entries are copied before the index that names them, so an interrupted run leaves the new prefix either absent or complete, never pointing at packs that have not landed. Old keys are kept unless `--prune` is passed.
7	*
8	* bun scripts/migrate-wal-keys.ts [--apply] [--prune]
9	*/
10	import { S3 } from '@aws-sdk/client-s3';
11	import { createDatabase, schema } from '@ghost/db';
12	import { eq, isNotNull } from 'drizzle-orm';
13
14	const apply = process.argv.includes('--apply');
15	const prune = process.argv.includes('--prune');
16
17	const bucket = required('S3_BUCKET');
18	const s3 = new S3({
19	forcePathStyle: true,
20	endpoint: required('S3_ENDPOINT'),
21	region: 'auto',
22	credentials: {
23	accessKeyId: required('S3_ACCESS_KEY_ID'),
24	secretAccessKey: required('S3_SECRET_ACCESS_KEY'),
25	},
26	});
27
28	const { db, pool } = createDatabase({
29	connectionString: required('DATABASE_URL'),
30	});
31
32	const rows = await db
33	.select({
34	id: schema.repository.id,
35	username: schema.user.username,
36	slug: schema.repository.slug,
37	})
38	.from(schema.repository)
39	.innerJoin(schema.user, eq(schema.user.id, schema.repository.ownerId))
40	.where(isNotNull(schema.user.username));
41
42	let moved = 0;
43	let skipped = 0;
44
45	for (const { id, username, slug } of rows) {
46	const from = `${username}/${slug}`;
47	const label = `${from} -> ${id}`;
48
49	if (await exists(`repos/${id}/index`)) {
50	skipped++;
51	continue;
52	}
53	if (!(await exists(`repos/${from}/index`))) {
54	console.log(`no log ${label}`);
55	skipped++;
56	continue;
57	}
58
59	const keys = await listPrefix(`repos/${from}/`);
60	const entries = keys.filter((key) => key.includes('/entries/'));
61
62	console.log(
63	`${apply ? 'migrating' : 'would move'} ${label} (${keys.length} objects)`,
64	);
65	if (!apply) {
66	moved++;
67	continue;
68	}
69
70	for (const key of entries) {
71	await s3.copyObject({
72	Bucket: bucket,
73	Key: key.replace(`repos/${from}/`, `repos/${id}/`),
74	CopySource: `${bucket}/${key}`,
75	});
76	}
77
78	await s3.copyObject({
79	Bucket: bucket,
80	Key: `repos/${id}/index`,
81	CopySource: `${bucket}/repos/${from}/index`,
82	});
83
84	if (prune) {
85	for (const key of keys) {
86	await s3.deleteObject({ Bucket: bucket, Key: key });
87	}
88	}
89
90	moved++;
91	}
92
93	console.log(
94	`\n${apply ? 'migrated' : 'to migrate'}: ${moved}, skipped: ${skipped}` +
95	(apply
96	? ''
97	: '\nre-run with --apply to write, and --prune to drop old keys'),
98	);
99
100	await pool.end();
101
102	async function exists(key: string) {
103	try {
104	await s3.headObject({ Bucket: bucket, Key: key });
105	return true;
106	} catch {
107	return false;
108	}
109	}
110
111	async function listPrefix(prefix: string) {
112	const keys: string[] = [];
113	let token: string \| undefined;
114
115	do {
116	const page = await s3.listObjectsV2({
117	Bucket: bucket,
118	Prefix: prefix,
119	ContinuationToken: token,
120	});
121	for (const object of page.Contents ?? []) {
122	if (object.Key) keys.push(object.Key);
123	}
124	token = page.NextContinuationToken;
125	} while (token);
126
127	return keys;
128	}
129
130	function required(name: string) {
131	const value = process.env[name];
132	if (!value) throw new Error(`${name} is not set`);
133	return value;
134	}