From 08b9067a2b5436361ea55fffa6a479dcec3157ab Mon Sep 17 00:00:00 2001 From: WarningImHack3r <43064022+WarningImHack3r@users.noreply.github.com> Date: Sat, 10 May 2025 23:53:24 +0200 Subject: [PATCH 1/4] feat: implement backend --- src/lib/server/github-cache.ts | 82 +++++++++++++++++++++++++--- src/lib/server/package-discoverer.ts | 30 +++++----- 2 files changed, 92 insertions(+), 20 deletions(-) diff --git a/src/lib/server/github-cache.ts b/src/lib/server/github-cache.ts index d38ec580..6632a099 100644 --- a/src/lib/server/github-cache.ts +++ b/src/lib/server/github-cache.ts @@ -18,6 +18,7 @@ export type GitHubRelease = Awaited< >["data"][number]; type KeyType = "releases" | "descriptions" | "issue" | "pr"; +type CachedValue = { value: T; cache: boolean }; export type ItemDetails = { comments: Awaited>["data"]; @@ -70,6 +71,10 @@ const FULL_DETAILS_TTL = 60 * 60 * 2; // 2 hours * The TTL of the cached descriptions, in seconds. */ const DESCRIPTIONS_TTL = 60 * 60 * 24 * 10; // 10 days +/** + * The TTL for non-deprecated packages, in seconds + */ +const DEPRECATIONS_TTL = 60 * 60 * 24 * 2; // 2 days /** * A fetch layer to reach the GitHub API @@ -115,6 +120,21 @@ export class GitHubCache { return `repo:${owner}/${repo}:${type}${strArgs}`; } + /** + * Generates a Redis key from the passed info. + * + * @param packageName the package name + * @param args the optional additional values to append + * at the end of the key; every element will be interpolated + * in a string + * @returns the pure computed key + * @private + */ + #getPackageKey(packageName: string, ...args: unknown[]) { + const strArgs = args.map(a => `:${a}`); + return `package:${packageName}${strArgs}`; + } + /** * An abstraction over general processing that: * 1. tries getting stuff from Redis cache @@ -124,6 +144,10 @@ export class GitHubCache { * @private */ #processCached["json"]["set"]>[2]>() { + function isCachedValue(item: T | CachedValue): item is CachedValue { + return typeof item === "object" && item !== null && "value" in item && "cache" in item; + } + /** * Inner currying function to circumvent unsupported partial inference * @@ -137,8 +161,10 @@ export class GitHubCache { return async ( cacheKey: string, promise: () => Promise, - transformer: (from: Awaited) => RType | Promise, - ttl: number | undefined = undefined + transformer: ( + from: Awaited + ) => RType | CachedValue | Promise | Promise>, + ttl: number | ((value: RType) => number | undefined) | undefined = undefined ): Promise => { const cachedValue = await this.#redis.json.get(cacheKey); if (cachedValue) { @@ -148,11 +174,25 @@ export class GitHubCache { console.log(`Cache miss for ${cacheKey}`); - const newValue = await transformer(await promise()); + let newValue = await transformer(await promise()); + let wantsCache = true; + if (isCachedValue(newValue)) { + wantsCache = newValue.cache; + newValue = newValue.value; + } - await this.#redis.json.set(cacheKey, "$", newValue); - if (ttl !== undefined) { - await this.#redis.expire(cacheKey, ttl); + if (wantsCache) { + await this.#redis.json.set(cacheKey, "$", newValue); + if (ttl !== undefined) { + if (typeof ttl === "function") { + const ttlResult = ttl(newValue); + if (ttlResult !== undefined) { + await this.#redis.expire(cacheKey, ttlResult); + } + } else { + await this.#redis.expire(cacheKey, ttl); + } + } } return newValue; @@ -537,7 +577,6 @@ export class GitHubCache { * @param repo the GitHub repository name to fetch the * descriptions in * @returns a map of paths to descriptions. - * @private */ async getDescriptions(owner: string, repo: string) { return await this.#processCached<{ [key: string]: string }>()( @@ -586,6 +625,35 @@ export class GitHubCache { DESCRIPTIONS_TTL ); } + + /** + * Get the deprecation state of a package from its name. + * + * @param packageName the name of the package to search + * @returns the deprecation status message if any, `false` otherwise + */ + async getPackageDeprecation(packageName: string) { + return await this.#processCached()( + this.#getPackageKey(packageName, "deprecation"), + async () => { + try { + const res = await fetch(`https://registry.npmjs.org/${packageName}/latest`); + if (res.status !== 200) return {}; + return (await res.json()) as { deprecated?: boolean | string }; + } catch (error) { + console.error(`Error fetching npmjs.org for package ${packageName}:`, error); + return {}; + } + }, + ({ deprecated }) => { + if (deprecated === undefined) return false; + if (typeof deprecated === "boolean") + return { value: "This package is deprecated", cache: false }; + return { value: deprecated || "This package is deprecated", cache: false }; + }, + item => (item === false ? DEPRECATIONS_TTL : undefined) + ); + } } export const gitHubCache = new GitHubCache(KV_REST_API_URL, KV_REST_API_TOKEN, GITHUB_TOKEN); diff --git a/src/lib/server/package-discoverer.ts b/src/lib/server/package-discoverer.ts index b6ee9e52..1270a719 100644 --- a/src/lib/server/package-discoverer.ts +++ b/src/lib/server/package-discoverer.ts @@ -5,6 +5,7 @@ import { GitHubCache, gitHubCache } from "./github-cache"; type Package = { name: string; description: string; + deprecated?: string; }; export type DiscoveredPackage = Prettify< @@ -54,19 +55,22 @@ export class PackageDiscoverer { ); return { ...repo, - packages: packages.map(pkg => { - const ghName = this.#gitHubDirectoryFromName(pkg); - return { - name: pkg, - description: - descriptions[`packages/${ghName}/package.json`] ?? - descriptions[ - `packages/${ghName.substring(ghName.lastIndexOf("/") + 1)}/package.json` - ] ?? - descriptions["package.json"] ?? - "" - }; - }) + packages: await Promise.all( + packages.map(async (pkg): Promise => { + const ghName = this.#gitHubDirectoryFromName(pkg); + return { + name: pkg, + description: + descriptions[`packages/${ghName}/package.json`] ?? + descriptions[ + `packages/${ghName.substring(ghName.lastIndexOf("/") + 1)}/package.json` + ] ?? + descriptions["package.json"] ?? + "", + deprecated: (await this.#cache.getPackageDeprecation(pkg)) || undefined + }; + }) + ) }; }) ); From 7a3e0418d25c1d42b14e3aed6434a3dce2ebbbd0 Mon Sep 17 00:00:00 2001 From: WarningImHack3r <43064022+WarningImHack3r@users.noreply.github.com> Date: Sun, 11 May 2025 00:18:10 +0200 Subject: [PATCH 2/4] fix: fix caching logic --- src/lib/server/github-cache.ts | 46 ++++++++++------------------ src/lib/server/package-discoverer.ts | 2 +- 2 files changed, 18 insertions(+), 30 deletions(-) diff --git a/src/lib/server/github-cache.ts b/src/lib/server/github-cache.ts index 6632a099..b09f8c09 100644 --- a/src/lib/server/github-cache.ts +++ b/src/lib/server/github-cache.ts @@ -18,7 +18,6 @@ export type GitHubRelease = Awaited< >["data"][number]; type KeyType = "releases" | "descriptions" | "issue" | "pr"; -type CachedValue = { value: T; cache: boolean }; export type ItemDetails = { comments: Awaited>["data"]; @@ -144,10 +143,6 @@ export class GitHubCache { * @private */ #processCached["json"]["set"]>[2]>() { - function isCachedValue(item: T | CachedValue): item is CachedValue { - return typeof item === "object" && item !== null && "value" in item && "cache" in item; - } - /** * Inner currying function to circumvent unsupported partial inference * @@ -161,9 +156,7 @@ export class GitHubCache { return async ( cacheKey: string, promise: () => Promise, - transformer: ( - from: Awaited - ) => RType | CachedValue | Promise | Promise>, + transformer: (from: Awaited) => RType | Promise, ttl: number | ((value: RType) => number | undefined) | undefined = undefined ): Promise => { const cachedValue = await this.#redis.json.get(cacheKey); @@ -174,24 +167,17 @@ export class GitHubCache { console.log(`Cache miss for ${cacheKey}`); - let newValue = await transformer(await promise()); - let wantsCache = true; - if (isCachedValue(newValue)) { - wantsCache = newValue.cache; - newValue = newValue.value; - } + const newValue = await transformer(await promise()); - if (wantsCache) { - await this.#redis.json.set(cacheKey, "$", newValue); - if (ttl !== undefined) { - if (typeof ttl === "function") { - const ttlResult = ttl(newValue); - if (ttlResult !== undefined) { - await this.#redis.expire(cacheKey, ttlResult); - } - } else { - await this.#redis.expire(cacheKey, ttl); + await this.#redis.json.set(cacheKey, "$", newValue); + if (ttl !== undefined) { + if (typeof ttl === "function") { + const ttlResult = ttl(newValue); + if (ttlResult !== undefined) { + await this.#redis.expire(cacheKey, ttlResult); } + } else { + await this.#redis.expire(cacheKey, ttl); } } @@ -633,10 +619,12 @@ export class GitHubCache { * @returns the deprecation status message if any, `false` otherwise */ async getPackageDeprecation(packageName: string) { - return await this.#processCached()( + return await this.#processCached<{ value: string | false }>()( this.#getPackageKey(packageName, "deprecation"), async () => { try { + // npmjs.org in a GitHub cache, I know, but hey, let's put that under the fact that + // GitHub owns npmjs.org okay?? const res = await fetch(`https://registry.npmjs.org/${packageName}/latest`); if (res.status !== 200) return {}; return (await res.json()) as { deprecated?: boolean | string }; @@ -646,12 +634,12 @@ export class GitHubCache { } }, ({ deprecated }) => { - if (deprecated === undefined) return false; + if (deprecated === undefined) return { value: false }; if (typeof deprecated === "boolean") - return { value: "This package is deprecated", cache: false }; - return { value: deprecated || "This package is deprecated", cache: false }; + return { value: deprecated && "This package is deprecated" }; + return { value: deprecated || "This package is deprecated" }; }, - item => (item === false ? DEPRECATIONS_TTL : undefined) + item => (item.value === false ? DEPRECATIONS_TTL : undefined) ); } } diff --git a/src/lib/server/package-discoverer.ts b/src/lib/server/package-discoverer.ts index 1270a719..88938684 100644 --- a/src/lib/server/package-discoverer.ts +++ b/src/lib/server/package-discoverer.ts @@ -67,7 +67,7 @@ export class PackageDiscoverer { ] ?? descriptions["package.json"] ?? "", - deprecated: (await this.#cache.getPackageDeprecation(pkg)) || undefined + deprecated: (await this.#cache.getPackageDeprecation(pkg)).value || undefined }; }) ) From 9a60bde6beff0276d895f448a17804c1c32cdcb8 Mon Sep 17 00:00:00 2001 From: WarningImHack3r <43064022+WarningImHack3r@users.noreply.github.com> Date: Sun, 11 May 2025 11:44:56 +0200 Subject: [PATCH 3/4] feat: make the UI (hints), remove description for deprecated packages --- src/lib/server/package-discoverer.ts | 18 +++++++------ src/routes/package/SidePanel.svelte | 10 +++++++- src/routes/package/[...package]/+page.svelte | 27 +++++++++++++++++++- 3 files changed, 45 insertions(+), 10 deletions(-) diff --git a/src/lib/server/package-discoverer.ts b/src/lib/server/package-discoverer.ts index 88938684..028684eb 100644 --- a/src/lib/server/package-discoverer.ts +++ b/src/lib/server/package-discoverer.ts @@ -58,16 +58,18 @@ export class PackageDiscoverer { packages: await Promise.all( packages.map(async (pkg): Promise => { const ghName = this.#gitHubDirectoryFromName(pkg); + const deprecated = (await this.#cache.getPackageDeprecation(pkg)).value || undefined; return { name: pkg, - description: - descriptions[`packages/${ghName}/package.json`] ?? - descriptions[ - `packages/${ghName.substring(ghName.lastIndexOf("/") + 1)}/package.json` - ] ?? - descriptions["package.json"] ?? - "", - deprecated: (await this.#cache.getPackageDeprecation(pkg)).value || undefined + description: deprecated + ? "" + : (descriptions[`packages/${ghName}/package.json`] ?? + descriptions[ + `packages/${ghName.substring(ghName.lastIndexOf("/") + 1)}/package.json` + ] ?? + descriptions["package.json"] ?? + ""), + deprecated }; }) ) diff --git a/src/routes/package/SidePanel.svelte b/src/routes/package/SidePanel.svelte index 546fe871..6f24dee3 100644 --- a/src/routes/package/SidePanel.svelte +++ b/src/routes/package/SidePanel.svelte @@ -157,7 +157,15 @@ href="/package/{pkg.name}" class="group inline-flex w-full items-center gap-1" > - {pkg.name} + + {pkg.name} + {#if linkedBadgeData} {#await linkedBadgeData then data} diff --git a/src/routes/package/[...package]/+page.svelte b/src/routes/package/[...package]/+page.svelte index 9938b0d0..61b294c8 100644 --- a/src/routes/package/[...package]/+page.svelte +++ b/src/routes/package/[...package]/+page.svelte @@ -1,13 +1,15 @@