diff --git a/env.d.ts b/env.d.ts index 75daf8d..3e7eed2 100644 --- a/env.d.ts +++ b/env.d.ts @@ -11,6 +11,10 @@ declare module "*.txt" { declare namespace Cloudflare { interface Env { DPRINT_PLUGINS_GH_TOKEN?: string; + // account id and a token with Account Analytics read access, used to query + // download counts back out of the analytics engine dataset + CLOUDFLARE_ACCOUNT_ID?: string; + DPRINT_PLUGINS_ANALYTICS_TOKEN?: string; PLUGIN_CACHE: R2Bucket; DPRINT_PLUGIN_DOWNLOAD_ANALYTICS: AnalyticsEngineDataset; } diff --git a/home.tsx b/home.tsx index 349b401..11e10f3 100644 --- a/home.tsx +++ b/home.tsx @@ -71,7 +71,7 @@ function renderPlugins(data: PluginsData) {
Name
Latest URL
-
Downloads
+
Downloads (30d)
{data.latest.map((plugin) => renderPlugin(plugin))} diff --git a/plugins.ts b/plugins.ts index 1d28232..06a1eb8 100644 --- a/plugins.ts +++ b/plugins.ts @@ -149,7 +149,10 @@ export async function getLatestInfo(username: string, repoName: string, origin: : `${origin}/${username}/${displayRepoName}-${releaseInfo.tagName}.${extension}`, version: releaseInfo.tagName.replace(/^v/, ""), checksum: releaseInfo.checksum, - downloadCount: releaseInfo.downloadCount, + // identifies this plugin's download analytics: the `username/repo` key that + // downloads are recorded under and the tag of the latest release + downloadKey: `${username}/${repoName}`, + tag: releaseInfo.tagName, }; } diff --git a/readInfoFile.ts b/readInfoFile.ts index 3125593..d623ae2 100644 --- a/readInfoFile.ts +++ b/readInfoFile.ts @@ -1,7 +1,7 @@ import { env } from "cloudflare:workers"; import infoJson from "./info.json" with { type: "json" }; import { getLatestInfo } from "./plugins.js"; -import { getAllDownloadCount } from "./utils/github.js"; +import { getDownloadCounts, type PluginDownloadCounts } from "./utils/analytics.js"; // only typing what's used on the server export interface PluginsData { @@ -125,6 +125,7 @@ async function buildInfoFile(origin: string): Promise> { }; async function getLatest(latest: typeof infoJson.latest) { + const downloadCounts = await getDownloadCounts(); const results = []; for (const plugin of latest) { const [username, pluginName] = plugin.name.split("/"); @@ -132,15 +133,14 @@ async function buildInfoFile(origin: string): Promise> { ? await getLatestInfo(username, pluginName, origin) : await getLatestInfo("dprint", plugin.name, origin); if (info != null) { + const counts = downloadCounts.get(info.downloadKey); results.push({ ...plugin, version: info.version, url: info.url, downloadCount: { - currentVersion: info.downloadCount, - allVersions: pluginName - ? await getAllDownloadCount(username, pluginName) - : await getAllDownloadCount("dprint", plugin.name), + currentVersion: currentVersionDownloads(counts, info.tag), + allVersions: counts?.allVersions ?? 0, }, }); } @@ -148,3 +148,12 @@ async function buildInfoFile(origin: string): Promise> { return results; } } + +// downloads of the latest release over the last 30 days, counting both the exact +// version tag and the "latest" alias (which always resolves to the current release) +function currentVersionDownloads(counts: PluginDownloadCounts | undefined, tag: string) { + if (counts == null) { + return 0; + } + return (counts.byTag.get(tag) ?? 0) + (counts.byTag.get("latest") ?? 0); +} diff --git a/utils/analytics.ts b/utils/analytics.ts new file mode 100644 index 0000000..da747f2 --- /dev/null +++ b/utils/analytics.ts @@ -0,0 +1,86 @@ +import { env } from "cloudflare:workers"; +import { fetchWithRetries } from "./fetchWithRetries.js"; +import { LazyExpirableValue } from "./LazyExpirableValue.js"; + +// download counts come from the Analytics Engine dataset that each plugin +// download writes a data point to (see trackPluginDownload in handleRequest.ts). +// the dataset only retains roughly the last 90 days, so these are downloads over +// the trailing 30 days ("monthly") rather than all-time totals. + +export interface PluginDownloadCounts { + // downloads over the last 30 days across every version + allVersions: number; + // downloads over the last 30 days keyed by the tag that appeared in the request + // url (a version like "0.1.0" or the "latest" alias) + byTag: Map; +} + +const DATASET = "dprint-plugin-downloads"; +const WINDOW_DAYS = 30; + +const downloadCounts = new LazyExpirableValue>({ + expiryMs: 5 * 60 * 1_000, // keep for 5 minutes + createValue: queryDownloadCounts, +}); + +/** Gets the download counts per plugin keyed by `username/repo`. */ +export async function getDownloadCounts(): Promise> { + try { + return await downloadCounts.getValue(); + } catch (err) { + // don't let analytics being unavailable break the info file build + console.error("Failed to get download counts from analytics.", err); + return new Map(); + } +} + +async function queryDownloadCounts(): Promise> { + const accountId = env.CLOUDFLARE_ACCOUNT_ID; + const token = env.DPRINT_PLUGINS_ANALYTICS_TOKEN; + const result = new Map(); + if (accountId == null || token == null) { + console.warn("Analytics credentials not configured. Skipping download counts."); + return result; + } + + // index1 is `username/repo`, blob1 is the tag, double1 is 1 per download + const query = `SELECT index1 AS plugin, blob1 AS tag, sum(_sample_interval * double1) AS downloads` + + ` FROM "${DATASET}"` + + ` WHERE timestamp > NOW() - INTERVAL '${WINDOW_DAYS}' DAY` + + ` GROUP BY plugin, tag LIMIT 10000`; + const response = await fetchWithRetries( + `https://api.cloudflare.com/client/v4/accounts/${accountId}/analytics_engine/sql`, + { + method: "POST", + headers: { + "authorization": `Bearer ${token}`, + "content-type": "text/plain", + }, + body: query, + }, + ); + if (!response.ok) { + const text = await response.text(); + throw new Error(`Analytics query failed: ${response.status}\n\n${text}`); + } + + const body = await response.json() as { + data: { plugin: string; tag: string; downloads: number | string }[]; + }; + for (const row of body.data) { + const downloads = Number(row.downloads) || 0; + const counts = getOrCreateCounts(result, row.plugin); + counts.allVersions += downloads; + counts.byTag.set(row.tag, (counts.byTag.get(row.tag) ?? 0) + downloads); + } + return result; +} + +function getOrCreateCounts(map: Map, plugin: string) { + let counts = map.get(plugin); + if (counts == null) { + counts = { allVersions: 0, byTag: new Map() }; + map.set(plugin, counts); + } + return counts; +} diff --git a/utils/github.ts b/utils/github.ts index d22b837..fecc9a7 100644 --- a/utils/github.ts +++ b/utils/github.ts @@ -37,7 +37,6 @@ export interface ReleaseInfo { tagName: string; checksum: string | undefined; kind: "wasm" | "process"; - downloadCount: number; } export async function getLatestReleaseInfo(username: string, repoName: string) { @@ -54,7 +53,6 @@ function getReleaseInfo(data: GitHubRelease): ReleaseInfo { tagName: data.tag_name, checksum: getChecksum(), kind: getPluginKind(), - downloadCount: getDownloadCount(data.assets), }; function getChecksum() { @@ -93,21 +91,11 @@ function getReleaseInfo(data: GitHubRelease): ReleaseInfo { } } -function getDownloadCount(assets: ReleaseAsset[]) { - return assets.find(({ name }) => name === "plugin.wasm" || name === "plugin.json")?.download_count ?? 0; -} - interface ReleaseAsset { name: string; - download_count: number; digest: string | null; } -export async function getAllDownloadCount(username: string, repoName: string) { - const releases = await getReleasesData(username, repoName); - return releases?.reduce((total, current) => total + getDownloadCount(current.assets), 0) ?? 0; -} - interface GitHubRelease { tag_name: string; body: string; diff --git a/utils/mod.ts b/utils/mod.ts index e48b202..95c4b1a 100644 --- a/utils/mod.ts +++ b/utils/mod.ts @@ -1,3 +1,4 @@ +export * from "./analytics.js"; export * from "./asyncLazy.js"; export * from "./github.js"; export * from "./version.js";