Skip to content

Commit a2dae96

Browse files
gregnrgregnr
committed
feat: export/import dbs
1 parent 5be4cc1 commit a2dae96

21 files changed

+1171
-109
lines changed

‎.npmrc

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
@jsr:registry=https://npm.jsr.io

‎apps/postgres-new/.env.example

+3
Original file line numberDiff line numberDiff line change
@@ -11,3 +11,6 @@ OPENAI_API_KEY="<openai-api-key>"
1111
# Vercel KV (local Docker available)
1212
KV_REST_API_URL="http://localhost:8080"
1313
KV_REST_API_TOKEN="local_token"
14+
15+
LEGACY_DOMAIN=postgres.new
16+
CURRENT_DOMAIN=database.build

‎apps/postgres-new/app/db/[id]/page.tsx renamed to ‎apps/postgres-new/app/(main)/db/[id]/page.tsx

File renamed without changes.

‎apps/postgres-new/app/(main)/layout.tsx

+9
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
import Layout from '~/components/layout'
2+
3+
export default function MainLayout({
4+
children,
5+
}: Readonly<{
6+
children: React.ReactNode
7+
}>) {
8+
return <Layout>{children}</Layout>
9+
}

‎apps/postgres-new/app/page.tsx renamed to ‎apps/postgres-new/app/(main)/page.tsx

File renamed without changes.

‎apps/postgres-new/app/export/page.tsx

+210
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,210 @@
1+
'use client'
2+
3+
import { TarStream, TarStreamInput } from '@std/tar/tar-stream'
4+
import { chunk } from 'lodash'
5+
import { useState } from 'react'
6+
import { useApp } from '~/components/app-provider'
7+
import {
8+
Accordion,
9+
AccordionContent,
10+
AccordionItem,
11+
AccordionTrigger,
12+
} from '~/components/ui/accordion'
13+
import { Button } from '~/components/ui/button'
14+
import { Dialog, DialogContent, DialogHeader, DialogTitle } from '~/components/ui/dialog'
15+
import { Progress } from '~/components/ui/progress'
16+
import { DbManager } from '~/lib/db'
17+
import { countFiles, listFiles } from '~/lib/files'
18+
import {
19+
fileFromStream,
20+
fileToTarStreamFile,
21+
mergeIterables,
22+
readableStreamFromIterable,
23+
transformStreamFromFn,
24+
} from '~/lib/streams'
25+
import { downloadFile } from '~/lib/util'
26+
27+
export default function Page() {
28+
const { dbManager } = useApp()
29+
const [progress, setProgress] = useState<number>()
30+
31+
return (
32+
<>
33+
<Dialog open>
34+
<DialogContent className="max-w-2xl">
35+
<DialogHeader>
36+
<DialogTitle>Export your databases</DialogTitle>
37+
<div className="py-2 border-b" />
38+
</DialogHeader>
39+
<p>
40+
postgres.new is renaming to database.build, which means you need to transfer your
41+
databases if you wish to continue using them.
42+
</p>
43+
44+
<Accordion type="single" collapsible>
45+
<AccordionItem value="item-1" className="border rounded-md">
46+
<AccordionTrigger className="p-0 gap-2 px-3 py-2">
47+
<div className="flex gap-2 items-center font-normal text-lighter text-sm">
48+
<span>Why is postgres.new renaming to database.build?</span>
49+
</div>
50+
</AccordionTrigger>
51+
<AccordionContent className="p-3 prose prose-sm">
52+
We are renaming postgres.new due to a trademark conflict on the name
53+
&quot;Postgres&quot;. To respect intellectual property rights, we are transitioning
54+
to our new name,{' '}
55+
<a href="https://database.build" className="underline">
56+
database.build
57+
</a>
58+
.
59+
</AccordionContent>
60+
</AccordionItem>
61+
</Accordion>
62+
<Accordion type="single" collapsible>
63+
<AccordionItem value="item-1" className="border rounded-md">
64+
<AccordionTrigger className="p-0 gap-2 px-3 py-2">
65+
<div className="flex gap-2 items-center font-normal text-lighter text-sm">
66+
<span>Why do I need to export my databases?</span>
67+
</div>
68+
</AccordionTrigger>
69+
<AccordionContent className="p-3 prose prose-sm">
70+
<p>
71+
Since PGlite databases are stored in your browser&apos;s IndexedDB storage, other
72+
domains like{' '}
73+
<a href="https://database.build" className="underline">
74+
database.build
75+
</a>{' '}
76+
cannot access them directly (this is a security restriction built into every
77+
browser).
78+
</p>
79+
<p>
80+
If you&apos;d like to continue using your previous databases and conversations:
81+
<ol>
82+
<li>Export them from postgres.new</li>
83+
<li>Import them to database.build</li>
84+
</ol>
85+
</p>
86+
</AccordionContent>
87+
</AccordionItem>
88+
</Accordion>
89+
<div className="my-2 border-b" />
90+
<div className="prose">
91+
<h4 className="mb-4">How to transfer your databases to database.build</h4>
92+
<ol>
93+
<li>
94+
Click <strong>Export</strong> to download all of your databases into a single
95+
tarball.
96+
<br />
97+
{progress === undefined ? (
98+
<Button
99+
className="my-2"
100+
onClick={async () => {
101+
if (!dbManager) {
102+
throw new Error('dbManager is not available')
103+
}
104+
105+
setProgress(0)
106+
107+
const dbCount = await dbManager.countDatabases()
108+
const fileCount = await countFiles()
109+
110+
// Plus 1 for the meta DB
111+
const totalFiles = 1 + dbCount + fileCount
112+
113+
// Passthrough stream to increment progress bar
114+
const progressPassthrough = transformStreamFromFn<
115+
TarStreamInput,
116+
TarStreamInput
117+
>((chunk) => {
118+
if (chunk.type === 'file') {
119+
setProgress((progress) => (progress ?? 0) + 100 / totalFiles)
120+
}
121+
return chunk
122+
})
123+
124+
const fileStream = mergeIterables([
125+
createDumpStream(dbManager),
126+
createStorageStream(),
127+
])
128+
129+
const tarGzStream = readableStreamFromIterable(fileStream)
130+
.pipeThrough(progressPassthrough)
131+
.pipeThrough(new TarStream())
132+
.pipeThrough<Uint8Array>(new CompressionStream('gzip'))
133+
134+
const file = await fileFromStream(
135+
tarGzStream,
136+
`${location.hostname}.tar.gz`,
137+
{ type: 'application/x-gzip' }
138+
)
139+
140+
downloadFile(file)
141+
}}
142+
>
143+
Export
144+
</Button>
145+
) : (
146+
<div className="flex gap-2 text-xs items-center">
147+
<Progress className="my-2 w-[60%]" value={Math.round(progress)} />
148+
{Math.round(progress)}%
149+
</div>
150+
)}
151+
<br />
152+
This tarball will contain every PGlite database&apos;s <code>pgdata</code> dump.
153+
</li>
154+
<li>
155+
Navigate to <a href="https://database.build/import">database.build/import</a> and
156+
click <strong>Import</strong>.
157+
</li>
158+
</ol>
159+
</div>
160+
</DialogContent>
161+
</Dialog>
162+
</>
163+
)
164+
}
165+
166+
/**
167+
* Generates a stream of PGlite dump files for all the databases.
168+
*/
169+
async function* createDumpStream(
170+
dbManager: DbManager,
171+
batchSize = 5
172+
): AsyncIterable<TarStreamInput> {
173+
const databases = await dbManager.exportDatabases()
174+
const batches = chunk(databases, batchSize)
175+
176+
// Meta DB has to be dumped separately
177+
// We intentionally yield this first so that it is
178+
// first in the archive
179+
const metaDb = await dbManager.getMetaDb()
180+
const metaDump = await metaDb.dumpDataDir('gzip')
181+
yield fileToTarStreamFile(new File([metaDump], 'meta.tar.gz', { type: metaDump.type }))
182+
183+
yield { type: 'directory', path: '/dbs' }
184+
185+
// Dump in batches to avoid excessive RAM use
186+
for (const batch of batches) {
187+
// All PGlite instances within a batch are loaded in parallel
188+
yield* await Promise.all(
189+
batch.map(async ({ id }) => {
190+
const db = await dbManager.getDbInstance(id)
191+
const dump = await db.dumpDataDir('gzip')
192+
const file = new File([dump], `${id}.tar.gz`, { type: dump.type })
193+
await dbManager.closeDbInstance(id)
194+
return fileToTarStreamFile(file, '/dbs')
195+
})
196+
)
197+
await new Promise((r) => setTimeout(r, 500))
198+
}
199+
}
200+
201+
async function* createStorageStream(): AsyncIterable<TarStreamInput> {
202+
yield { type: 'directory', path: '/files' }
203+
204+
for await (const { id, file } of listFiles()) {
205+
// Capture the ID by storing each file in a sub-dir
206+
// named after the ID
207+
yield { type: 'directory', path: `/files/${id}` }
208+
yield fileToTarStreamFile(file, `/files/${id}`)
209+
}
210+
}

0 commit comments

Comments
 (0)