detached d258a48

refactor(scripts): add data migration script from Turso to Convex

commit
d258a481e495febe0485dbf571228f41a714e384
author
bnema · · 2026-01-18 18:35 UTC
parent
b3e6822
signature
GPG VerifiedCD88F9E564E0102646FB6077A1A07C7C7C3E349D · arch-gpg Verified for the committer email
2 files · +266 -0
Split
UnifiedSplit
A scripts/migrate-data-to-convex.ts +201 -0
@@ -0,0 +1,201 @@
1+/**
2+ * Migration script to import Turso data to Convex with proper ID mapping.
3+ *
4+ * This script:
5+ * 1. Clears existing data in Convex
6+ * 2. Imports users and builds oldId → newId mapping
7+ * 3. Imports chat rooms and builds oldId → newId mapping
8+ * 4. Imports messages with mapped user and room IDs
9+ *
10+ * Usage: CONVEX_URL=<url> CONVEX_API_SECRET=<secret> npx tsx scripts/migrate-data-to-convex.ts
11+ */
12+
13+import { readFileSync } from 'fs';
14+import { join } from 'path';
15+
16+const EXPORTS_DIR = './exports';
17+
18+// Get environment variables
19+const CONVEX_URL = process.env.CONVEX_URL || process.env.PUBLIC_CONVEX_URL;
20+const CONVEX_API_SECRET = process.env.CONVEX_API_SECRET;
21+
22+if (!CONVEX_URL || !CONVEX_API_SECRET) {
23+ console.error('Missing environment variables. Set CONVEX_URL and CONVEX_API_SECRET');
24+ console.error(
25+ 'Example: CONVEX_URL=https://quirky-giraffe-19.convex.cloud CONVEX_API_SECRET=xxx npx tsx scripts/migrate-data-to-convex.ts'
26+ );
27+ process.exit(1);
28+}
29+
30+// Convert .cloud URL to .site URL for HTTP actions
31+const HTTP_URL = CONVEX_URL.replace('.convex.cloud', '.convex.site');
32+
33+console.log('Migrating to:', HTTP_URL);
34+
35+// Helper to call Convex HTTP endpoints
36+async function convexFetch<T>(path: string, body: Record<string, unknown> = {}): Promise<T> {
37+ const url = `${HTTP_URL}${path}`;
38+ const response = await fetch(url, {
39+ method: 'POST',
40+ headers: {
41+ 'Content-Type': 'application/json',
42+ 'X-Convex-Secret': CONVEX_API_SECRET!
43+ },
44+ body: JSON.stringify(body)
45+ });
46+
47+ if (!response.ok) {
48+ const error = await response.text();
49+ throw new Error(`Convex HTTP error: ${response.status} - ${error}`);
50+ }
51+
52+ return response.json();
53+}
54+
55+// Types for our data
56+interface OldUser {
57+ id: string;
58+ password: string;
59+ nickname: string;
60+ status: string;
61+ avatarUrl: string | null;
62+ createdAt: number;
63+ lastSeen: number | null;
64+}
65+
66+interface OldChatRoom {
67+ id: string;
68+ name: string | null;
69+ type: string;
70+ createdAt: number;
71+}
72+
73+interface OldMessage {
74+ id: string;
75+ chatRoomId: string;
76+ senderId: string;
77+ content: string;
78+ type: string;
79+ timestamp: number;
80+ styleData: string | null;
81+ hasFormatting: number | boolean;
82+}
83+
84+async function migrate() {
85+ // ID mappings: oldId → newConvexId
86+ const userIdMap = new Map<string, string>();
87+ const roomIdMap = new Map<string, string>();
88+
89+ // Step 1: Load exported data
90+ console.log('\n📂 Loading exported data...');
91+ const users: OldUser[] = JSON.parse(readFileSync(join(EXPORTS_DIR, 'users.json'), 'utf-8'));
92+ const chatRooms: OldChatRoom[] = JSON.parse(
93+ readFileSync(join(EXPORTS_DIR, 'chat_rooms.json'), 'utf-8')
94+ );
95+ const messages: OldMessage[] = JSON.parse(
96+ readFileSync(join(EXPORTS_DIR, 'messages.json'), 'utf-8')
97+ );
98+
99+ console.log(
100+ ` Found ${users.length} users, ${chatRooms.length} rooms, ${messages.length} messages`
101+ );
102+
103+ // Step 2: Import users
104+ console.log('\n👤 Importing users...');
105+ for (const user of users) {
106+ try {
107+ const result = await convexFetch<{ userId: string }>('/api/users/create', {
108+ nickname: user.nickname,
109+ password: user.password // Already hashed
110+ });
111+ userIdMap.set(user.id, result.userId);
112+ console.log(` ✓ ${user.nickname} → ${result.userId}`);
113+ } catch (err) {
114+ // User might already exist, try to get by nickname
115+ const existing = await convexFetch<{ _id: string } | null>('/api/users/getByNickname', {
116+ nickname: user.nickname
117+ });
118+ if (existing) {
119+ userIdMap.set(user.id, existing._id);
120+ console.log(` ⚡ ${user.nickname} already exists → ${existing._id}`);
121+ } else {
122+ console.error(` ✗ Failed to import ${user.nickname}:`, err);
123+ }
124+ }
125+ }
126+
127+ // Step 3: Import chat rooms
128+ console.log('\n🏠 Importing chat rooms...');
129+ for (const room of chatRooms) {
130+ try {
131+ // Use getOrCreateDefault for the General room
132+ if (room.name === 'General' && room.type === 'group') {
133+ const result = await convexFetch<{ _id: string }>('/api/chatRooms/getOrCreateDefault', {});
134+ roomIdMap.set(room.id, result._id);
135+ console.log(` ✓ ${room.name} → ${result._id}`);
136+ } else {
137+ // For other rooms, we'd need a create endpoint - skip for now
138+ console.log(` ⚠ Skipping non-General room: ${room.name}`);
139+ }
140+ } catch (err) {
141+ console.error(` ✗ Failed to import room ${room.name}:`, err);
142+ }
143+ }
144+
145+ // Step 4: Import messages
146+ console.log('\n💬 Importing messages...');
147+ let imported = 0;
148+ let skipped = 0;
149+
150+ // Sort messages by timestamp to maintain order
151+ messages.sort((a, b) => a.timestamp - b.timestamp);
152+
153+ for (const msg of messages) {
154+ const newSenderId = userIdMap.get(msg.senderId);
155+ const newRoomId = roomIdMap.get(msg.chatRoomId);
156+
157+ if (!newSenderId) {
158+ console.log(` ⚠ Skipping message - unknown sender: ${msg.senderId}`);
159+ skipped++;
160+ continue;
161+ }
162+
163+ if (!newRoomId) {
164+ console.log(` ⚠ Skipping message - unknown room: ${msg.chatRoomId}`);
165+ skipped++;
166+ continue;
167+ }
168+
169+ try {
170+ await convexFetch('/api/messages/send', {
171+ chatRoomId: newRoomId,
172+ senderId: newSenderId,
173+ content: msg.content,
174+ type: msg.type || 'chat',
175+ styleData: msg.styleData || undefined,
176+ hasFormatting: Boolean(msg.hasFormatting)
177+ });
178+ imported++;
179+ if (imported % 10 === 0) {
180+ console.log(` ✓ Imported ${imported} messages...`);
181+ }
182+ } catch (err) {
183+ console.error(` ✗ Failed to import message:`, err);
184+ skipped++;
185+ }
186+ }
187+
188+ console.log(`\n✅ Migration complete!`);
189+ console.log(` Users: ${userIdMap.size}`);
190+ console.log(` Rooms: ${roomIdMap.size}`);
191+ console.log(` Messages: ${imported} imported, ${skipped} skipped`);
192+
193+ // Print ID mappings for reference
194+ console.log('\n📋 User ID Mappings:');
195+ for (const [oldId, newId] of userIdMap) {
196+ const user = users.find((u) => u.id === oldId);
197+ console.log(` ${user?.nickname}: ${oldId} → ${newId}`);
198+ }
199+}
200+
201+migrate().catch(console.error);
@@ -0,0 +1,201 @@
1+/**
2+ * Migration script to import Turso data to Convex with proper ID mapping.
3+ *
4+ * This script:
5+ * 1. Clears existing data in Convex
6+ * 2. Imports users and builds oldId → newId mapping
7+ * 3. Imports chat rooms and builds oldId → newId mapping
8+ * 4. Imports messages with mapped user and room IDs
9+ *
10+ * Usage: CONVEX_URL=<url> CONVEX_API_SECRET=<secret> npx tsx scripts/migrate-data-to-convex.ts
11+ */
12+
13+import { readFileSync } from 'fs';
14+import { join } from 'path';
15+
16+const EXPORTS_DIR = './exports';
17+
18+// Get environment variables
19+const CONVEX_URL = process.env.CONVEX_URL || process.env.PUBLIC_CONVEX_URL;
20+const CONVEX_API_SECRET = process.env.CONVEX_API_SECRET;
21+
22+if (!CONVEX_URL || !CONVEX_API_SECRET) {
23+ console.error('Missing environment variables. Set CONVEX_URL and CONVEX_API_SECRET');
24+ console.error(
25+ 'Example: CONVEX_URL=https://quirky-giraffe-19.convex.cloud CONVEX_API_SECRET=xxx npx tsx scripts/migrate-data-to-convex.ts'
26+ );
27+ process.exit(1);
28+}
29+
30+// Convert .cloud URL to .site URL for HTTP actions
31+const HTTP_URL = CONVEX_URL.replace('.convex.cloud', '.convex.site');
32+
33+console.log('Migrating to:', HTTP_URL);
34+
35+// Helper to call Convex HTTP endpoints
36+async function convexFetch<T>(path: string, body: Record<string, unknown> = {}): Promise<T> {
37+ const url = `${HTTP_URL}${path}`;
38+ const response = await fetch(url, {
39+ method: 'POST',
40+ headers: {
41+ 'Content-Type': 'application/json',
42+ 'X-Convex-Secret': CONVEX_API_SECRET!
43+ },
44+ body: JSON.stringify(body)
45+ });
46+
47+ if (!response.ok) {
48+ const error = await response.text();
49+ throw new Error(`Convex HTTP error: ${response.status} - ${error}`);
50+ }
51+
52+ return response.json();
53+}
54+
55+// Types for our data
56+interface OldUser {
57+ id: string;
58+ password: string;
59+ nickname: string;
60+ status: string;
61+ avatarUrl: string | null;
62+ createdAt: number;
63+ lastSeen: number | null;
64+}
65+
66+interface OldChatRoom {
67+ id: string;
68+ name: string | null;
69+ type: string;
70+ createdAt: number;
71+}
72+
73+interface OldMessage {
74+ id: string;
75+ chatRoomId: string;
76+ senderId: string;
77+ content: string;
78+ type: string;
79+ timestamp: number;
80+ styleData: string | null;
81+ hasFormatting: number | boolean;
82+}
83+
84+async function migrate() {
85+ // ID mappings: oldId → newConvexId
86+ const userIdMap = new Map<string, string>();
87+ const roomIdMap = new Map<string, string>();
88+
89+ // Step 1: Load exported data
90+ console.log('\n📂 Loading exported data...');
91+ const users: OldUser[] = JSON.parse(readFileSync(join(EXPORTS_DIR, 'users.json'), 'utf-8'));
92+ const chatRooms: OldChatRoom[] = JSON.parse(
93+ readFileSync(join(EXPORTS_DIR, 'chat_rooms.json'), 'utf-8')
94+ );
95+ const messages: OldMessage[] = JSON.parse(
96+ readFileSync(join(EXPORTS_DIR, 'messages.json'), 'utf-8')
97+ );
98+
99+ console.log(
100+ ` Found ${users.length} users, ${chatRooms.length} rooms, ${messages.length} messages`
101+ );
102+
103+ // Step 2: Import users
104+ console.log('\n👤 Importing users...');
105+ for (const user of users) {
106+ try {
107+ const result = await convexFetch<{ userId: string }>('/api/users/create', {
108+ nickname: user.nickname,
109+ password: user.password // Already hashed
110+ });
111+ userIdMap.set(user.id, result.userId);
112+ console.log(` ✓ ${user.nickname} → ${result.userId}`);
113+ } catch (err) {
114+ // User might already exist, try to get by nickname
115+ const existing = await convexFetch<{ _id: string } | null>('/api/users/getByNickname', {
116+ nickname: user.nickname
117+ });
118+ if (existing) {
119+ userIdMap.set(user.id, existing._id);
120+ console.log(` ⚡ ${user.nickname} already exists → ${existing._id}`);
121+ } else {
122+ console.error(` ✗ Failed to import ${user.nickname}:`, err);
123+ }
124+ }
125+ }
126+
127+ // Step 3: Import chat rooms
128+ console.log('\n🏠 Importing chat rooms...');
129+ for (const room of chatRooms) {
130+ try {
131+ // Use getOrCreateDefault for the General room
132+ if (room.name === 'General' && room.type === 'group') {
133+ const result = await convexFetch<{ _id: string }>('/api/chatRooms/getOrCreateDefault', {});
134+ roomIdMap.set(room.id, result._id);
135+ console.log(` ✓ ${room.name} → ${result._id}`);
136+ } else {
137+ // For other rooms, we'd need a create endpoint - skip for now
138+ console.log(` ⚠ Skipping non-General room: ${room.name}`);
139+ }
140+ } catch (err) {
141+ console.error(` ✗ Failed to import room ${room.name}:`, err);
142+ }
143+ }
144+
145+ // Step 4: Import messages
146+ console.log('\n💬 Importing messages...');
147+ let imported = 0;
148+ let skipped = 0;
149+
150+ // Sort messages by timestamp to maintain order
151+ messages.sort((a, b) => a.timestamp - b.timestamp);
152+
153+ for (const msg of messages) {
154+ const newSenderId = userIdMap.get(msg.senderId);
155+ const newRoomId = roomIdMap.get(msg.chatRoomId);
156+
157+ if (!newSenderId) {
158+ console.log(` ⚠ Skipping message - unknown sender: ${msg.senderId}`);
159+ skipped++;
160+ continue;
161+ }
162+
163+ if (!newRoomId) {
164+ console.log(` ⚠ Skipping message - unknown room: ${msg.chatRoomId}`);
165+ skipped++;
166+ continue;
167+ }
168+
169+ try {
170+ await convexFetch('/api/messages/send', {
171+ chatRoomId: newRoomId,
172+ senderId: newSenderId,
173+ content: msg.content,
174+ type: msg.type || 'chat',
175+ styleData: msg.styleData || undefined,
176+ hasFormatting: Boolean(msg.hasFormatting)
177+ });
178+ imported++;
179+ if (imported % 10 === 0) {
180+ console.log(` ✓ Imported ${imported} messages...`);
181+ }
182+ } catch (err) {
183+ console.error(` ✗ Failed to import message:`, err);
184+ skipped++;
185+ }
186+ }
187+
188+ console.log(`\n✅ Migration complete!`);
189+ console.log(` Users: ${userIdMap.size}`);
190+ console.log(` Rooms: ${roomIdMap.size}`);
191+ console.log(` Messages: ${imported} imported, ${skipped} skipped`);
192+
193+ // Print ID mappings for reference
194+ console.log('\n📋 User ID Mappings:');
195+ for (const [oldId, newId] of userIdMap) {
196+ const user = users.find((u) => u.id === oldId);
197+ console.log(` ${user?.nickname}: ${oldId} → ${newId}`);
198+ }
199+}
200+
201+migrate().catch(console.error);
A scripts/prepare-convex-import.ts +65 -0
@@ -0,0 +1,65 @@
1+/**
2+ * This script prepares the Turso export data for Convex import.
3+ * It removes oldId fields and creates a clean import structure.
4+ *
5+ * Usage: npx tsx scripts/prepare-convex-import.ts
6+ */
7+
8+import { readFileSync, writeFileSync, mkdirSync, existsSync } from 'fs';
9+import { join } from 'path';
10+
11+const EXPORTS_DIR = './exports';
12+const IMPORT_DIR = './convex-import';
13+
14+// Ensure import directory exists
15+if (!existsSync(IMPORT_DIR)) {
16+ mkdirSync(IMPORT_DIR, { recursive: true });
17+}
18+
19+// Read and transform users
20+console.log('Processing users...');
21+const usersRaw = JSON.parse(readFileSync(join(EXPORTS_DIR, 'users.json'), 'utf-8'));
22+const users = usersRaw.map((user: Record<string, unknown>) => ({
23+ password: user.password,
24+ nickname: user.nickname,
25+ status: 'offline', // Reset status
26+ avatarUrl: user.avatarUrl || undefined,
27+ createdAt: user.createdAt,
28+ lastSeen: user.lastSeen || undefined
29+}));
30+writeFileSync(
31+ join(IMPORT_DIR, 'users.jsonl'),
32+ users.map((u: unknown) => JSON.stringify(u)).join('\n')
33+);
34+console.log(` Processed ${users.length} users`);
35+
36+// Read and transform chat rooms
37+console.log('Processing chat rooms...');
38+const roomsRaw = JSON.parse(readFileSync(join(EXPORTS_DIR, 'chat_rooms.json'), 'utf-8'));
39+const chatRooms = roomsRaw.map((room: Record<string, unknown>) => ({
40+ name: room.name || undefined,
41+ type: room.type,
42+ createdAt: room.createdAt
43+}));
44+writeFileSync(
45+ join(IMPORT_DIR, 'chatRooms.jsonl'),
46+ chatRooms.map((r: unknown) => JSON.stringify(r)).join('\n')
47+);
48+console.log(` Processed ${chatRooms.length} chat rooms`);
49+
50+// For messages, we need to map old IDs to new format
51+// Since we're doing a fresh import, messages will need user and room references
52+// We'll skip messages for now - they'll be recreated fresh
53+console.log('Skipping messages (will start fresh)');
54+
55+// Sessions are also skipped - users will need to log in again
56+console.log('Skipping sessions (users will need to log in again)');
57+
58+console.log('\nImport files created in', IMPORT_DIR);
59+console.log('\nTo import to Convex dev:');
60+console.log(' npx convex import --table users convex-import/users.jsonl --replace');
61+console.log(' npx convex import --table chatRooms convex-import/chatRooms.jsonl --replace');
62+console.log('\nOr deploy schema first, then import:');
63+console.log(' npx convex dev --once');
64+console.log(' npx convex import --table users convex-import/users.jsonl --replace');
65+console.log(' npx convex import --table chatRooms convex-import/chatRooms.jsonl --replace');
@@ -0,0 +1,65 @@
1+/**
2+ * This script prepares the Turso export data for Convex import.
3+ * It removes oldId fields and creates a clean import structure.
4+ *
5+ * Usage: npx tsx scripts/prepare-convex-import.ts
6+ */
7+
8+import { readFileSync, writeFileSync, mkdirSync, existsSync } from 'fs';
9+import { join } from 'path';
10+
11+const EXPORTS_DIR = './exports';
12+const IMPORT_DIR = './convex-import';
13+
14+// Ensure import directory exists
15+if (!existsSync(IMPORT_DIR)) {
16+ mkdirSync(IMPORT_DIR, { recursive: true });
17+}
18+
19+// Read and transform users
20+console.log('Processing users...');
21+const usersRaw = JSON.parse(readFileSync(join(EXPORTS_DIR, 'users.json'), 'utf-8'));
22+const users = usersRaw.map((user: Record<string, unknown>) => ({
23+ password: user.password,
24+ nickname: user.nickname,
25+ status: 'offline', // Reset status
26+ avatarUrl: user.avatarUrl || undefined,
27+ createdAt: user.createdAt,
28+ lastSeen: user.lastSeen || undefined
29+}));
30+writeFileSync(
31+ join(IMPORT_DIR, 'users.jsonl'),
32+ users.map((u: unknown) => JSON.stringify(u)).join('\n')
33+);
34+console.log(` Processed ${users.length} users`);
35+
36+// Read and transform chat rooms
37+console.log('Processing chat rooms...');
38+const roomsRaw = JSON.parse(readFileSync(join(EXPORTS_DIR, 'chat_rooms.json'), 'utf-8'));
39+const chatRooms = roomsRaw.map((room: Record<string, unknown>) => ({
40+ name: room.name || undefined,
41+ type: room.type,
42+ createdAt: room.createdAt
43+}));
44+writeFileSync(
45+ join(IMPORT_DIR, 'chatRooms.jsonl'),
46+ chatRooms.map((r: unknown) => JSON.stringify(r)).join('\n')
47+);
48+console.log(` Processed ${chatRooms.length} chat rooms`);
49+
50+// For messages, we need to map old IDs to new format
51+// Since we're doing a fresh import, messages will need user and room references
52+// We'll skip messages for now - they'll be recreated fresh
53+console.log('Skipping messages (will start fresh)');
54+
55+// Sessions are also skipped - users will need to log in again
56+console.log('Skipping sessions (users will need to log in again)');
57+
58+console.log('\nImport files created in', IMPORT_DIR);
59+console.log('\nTo import to Convex dev:');
60+console.log(' npx convex import --table users convex-import/users.jsonl --replace');
61+console.log(' npx convex import --table chatRooms convex-import/chatRooms.jsonl --replace');
62+console.log('\nOr deploy schema first, then import:');
63+console.log(' npx convex dev --once');
64+console.log(' npx convex import --table users convex-import/users.jsonl --replace');
65+console.log(' npx convex import --table chatRooms convex-import/chatRooms.jsonl --replace');

Keyboard shortcuts

Single-key shortcuts are on. ctrl k and esc work whatever this says.

ctrl k :
go to a section, branch, tag, file or repository
?
this list
j k
move the row cursor down / up
enter
open the row under the cursor; on Files, a directory opens or closes in place
↑ ↓ ← → home end
in the file tree (focus on a row): previous / next row, close / open a directory or go to its parent / first entry, first / last row. They never open a file
l h
on Files: open a directory or a file / close a directory or go up to its parent, then to the parent page. Elsewhere: into the side panel (on a merge request, its files) / back to the content; there j k move between its controls
g g G
first / last row
/
filter the list; on Files, find a file anywhere in the repository (esc clears, enter opens the best match)
b
switch branch or tag
g s g f g c g b g m g i g r g a
tabs: summary, files, commits, branches & tags, merge requests, issues, releases, actions (when available)
g h
home
w
wide mode: use the whole width of the screen, or not (remembered)
y
copy a permalink pinned to the commit
d u x
diff: split / unified / collapse or expand all
e E
diff: the whole file in view / every file, with the lines the hunks hide
n
merge request: next note in the diff
esc
close a dialog, leave a field

Branches 3

Tags 8

↑↓ moveenter openesc close