diff --git a/packages/objectql/src/build-probes.test.ts b/packages/objectql/src/build-probes.test.ts new file mode 100644 index 000000000..57d147a8f --- /dev/null +++ b/packages/objectql/src/build-probes.test.ts @@ -0,0 +1,206 @@ +// Copyright (c) 2025 ObjectStack. Licensed under the Apache-2.0 license. + +import { describe, it, expect, vi } from 'vitest'; +import { runBuildProbes, type ProbeEngine } from './build-probes.js'; +import { ObjectStackProtocolImplementation } from './protocol.js'; + +/** + * ADR-0038 L3 — runtime probes. Each probe is one real read; findings are + * BuildIssue-shaped (layer 'runtime'); a probe must report, never throw. + */ + +const ITEMS: Record = { + 'seed expense_sample': { object: 'expense', records: [{ name: 'a' }] }, + 'view expense.all': { name: 'expense.all', object: 'expense', viewKind: 'list', config: {} }, + 'dashboard spending': { + name: 'spending', + widgets: [{ id: 'w1', dataset: 'expense_ds', values: ['amount'] }], + }, + 'dataset expense_ds': { + name: 'expense_ds', + object: 'expense', + measures: [{ name: 'count', aggregate: 'count' }, { name: 'amount', aggregate: 'sum', field: 'amount' }], + dimensions: [], + }, +}; + +const getItem = async (type: string, name: string) => ITEMS[`${type} ${name}`]; + +function engineWithRows(rowsByObject: Record): ProbeEngine { + return { + find: async (object: string) => + Array.from({ length: Math.min(rowsByObject[object] ?? 0, 1) }, (_, i) => ({ id: String(i) })), + }; +} + +describe('runBuildProbes — seeds', () => { + it('flags seed_not_applied when the seeded object has no rows', async () => { + const report = await runBuildProbes({ + engine: engineWithRows({ expense: 0 }), + getItem, + published: [{ type: 'seed', name: 'expense_sample' }], + }); + expect(report.checked.seeds).toBe(1); + expect(report.issues).toHaveLength(1); + expect(report.issues[0]).toMatchObject({ + layer: 'runtime', + severity: 'error', + code: 'seed_not_applied', + artifact: { type: 'seed', name: 'expense_sample' }, + ref: { type: 'object', name: 'expense' }, + }); + }); + + it('passes when rows exist', async () => { + const report = await runBuildProbes({ + engine: engineWithRows({ expense: 3 }), + getItem, + published: [{ type: 'seed', name: 'expense_sample' }], + }); + expect(report.issues).toHaveLength(0); + expect(report.checked.seeds).toBe(1); + }); +}); + +describe('runBuildProbes — views', () => { + it('flags view_read_failed when the read throws; empty tables are fine', async () => { + const engine: ProbeEngine = { + find: async (object: string) => { + if (object === 'expense') throw new Error('no such table: expense'); + return []; + }, + }; + const report = await runBuildProbes({ + engine, + getItem, + published: [{ type: 'view', name: 'expense.all' }], + }); + expect(report.checked.views).toBe(1); + expect(report.issues).toHaveLength(1); + expect(report.issues[0]).toMatchObject({ + code: 'view_read_failed', + severity: 'error', + artifact: { type: 'view', name: 'expense.all' }, + }); + expect(report.issues[0].message).toContain('no such table'); + + const ok = await runBuildProbes({ + engine: engineWithRows({}), // empty result, no throw + getItem, + published: [{ type: 'view', name: 'expense.all' }], + }); + expect(ok.issues).toHaveLength(0); + }); +}); + +describe('runBuildProbes — dashboard widgets', () => { + it('flags empty_query when the dataset returns nothing on a populated object (incident #4)', async () => { + const analytics = { queryDataset: vi.fn(async () => ({ rows: [] })) }; + const report = await runBuildProbes({ + engine: engineWithRows({ expense: 5 }), + getItem, + analytics, + published: [{ type: 'dashboard', name: 'spending' }], + }); + expect(analytics.queryDataset).toHaveBeenCalledOnce(); + // The widget's own values are used as the probe selection. + expect((analytics.queryDataset.mock.calls[0][1] as any).measures).toEqual(['amount']); + expect(report.checked.widgets).toBe(1); + expect(report.issues).toHaveLength(1); + expect(report.issues[0]).toMatchObject({ + code: 'empty_query', + severity: 'error', + artifact: { type: 'dashboard', name: 'spending' }, + ref: { type: 'dataset', name: 'expense_ds', member: 'w1' }, + }); + }); + + it('passes when the query returns data; empty-on-empty-object is fine', async () => { + const withData = await runBuildProbes({ + engine: engineWithRows({ expense: 5 }), + getItem, + analytics: { queryDataset: async () => ({ rows: [{ amount: 42 }] }) }, + published: [{ type: 'dashboard', name: 'spending' }], + }); + expect(withData.issues).toHaveLength(0); + + const emptyObject = await runBuildProbes({ + engine: engineWithRows({ expense: 0 }), + getItem, + analytics: { queryDataset: async () => ({ rows: [] }) }, + published: [{ type: 'dashboard', name: 'spending' }], + }); + expect(emptyObject.issues).toHaveLength(0); // no rows promised, none missing + }); + + it('flags widget_query_failed when the query throws', async () => { + const report = await runBuildProbes({ + engine: engineWithRows({ expense: 5 }), + getItem, + analytics: { queryDataset: async () => { throw new Error('RAW_SQL_UNSUPPORTED'); } }, + published: [{ type: 'dashboard', name: 'spending' }], + }); + expect(report.issues).toHaveLength(1); + expect(report.issues[0].code).toBe('widget_query_failed'); + expect(report.issues[0].message).toContain('RAW_SQL_UNSUPPORTED'); + }); + + it('emits ONE probes_unavailable warning when widgets exist but no analytics service does', async () => { + const report = await runBuildProbes({ + engine: engineWithRows({ expense: 5 }), + getItem, + published: [{ type: 'dashboard', name: 'spending' }], + }); + expect(report.checked.widgets).toBe(0); + expect(report.issues).toHaveLength(1); + expect(report.issues[0]).toMatchObject({ code: 'probes_unavailable', severity: 'warning' }); + }); + + it('never throws — unreadable items and engine crashes degrade to findings/skips', async () => { + const report = await runBuildProbes({ + engine: { find: async () => { throw new Error('engine down'); } }, + getItem: async () => { throw new Error('metadata down'); }, + published: [ + { type: 'seed', name: 'expense_sample' }, + { type: 'view', name: 'expense.all' }, + { type: 'dashboard', name: 'spending' }, + ], + }); + // getItem failures mean no object bindings resolve — nothing probed, nothing thrown. + expect(report.checked).toEqual({ seeds: 0, views: 0, widgets: 0 }); + }); +}); + +describe('publishPackageDrafts — probes ride the response (ADR-0038 L3)', () => { + it('runs probes over the published set and reports seed_not_applied', async () => { + const protocol = new ObjectStackProtocolImplementation({} as never); + (protocol as any).ensureOverlayIndex = async () => {}; + (protocol as any).getOverlayRepo = () => ({ + listDrafts: async () => [ + { type: 'object', name: 'expense' }, + { type: 'seed', name: 'expense_sample' }, + ], + get: async (_ref: any, opts: any) => + opts?.state === 'draft' ? { body: ITEMS['seed expense_sample'], hash: 'h' } : null, + }); + vi.spyOn(protocol, 'publishMetaItem' as never).mockResolvedValue({ success: true, version: 'h', seq: 1 } as never); + vi.spyOn(protocol as any, 'applySeedBodies').mockResolvedValue({ success: false, inserted: 0, updated: 0, error: 'boom' }); + // Probe reads: active items + an engine whose table stayed empty. + (protocol as any).getMetaItem = async ({ type, name }: any) => ({ item: ITEMS[`${type} ${name}`] }); + (protocol as any).engine = { find: async () => [] }; + + const res = await protocol.publishPackageDrafts({ packageId: 'app.exp' }); + + expect(res.probes).toBeDefined(); + expect(res.probes!.checked.seeds).toBe(1); + expect(res.probes!.issues.map((i) => i.code)).toEqual(['seed_not_applied']); + }); + + it('omits probes when nothing was published', async () => { + const protocol = new ObjectStackProtocolImplementation({} as never); + (protocol as any).ensureOverlayIndex = async () => {}; + (protocol as any).getOverlayRepo = () => ({ listDrafts: async () => [] }); + const res = await protocol.publishPackageDrafts({ packageId: 'app.empty' }); + expect(res.probes).toBeUndefined(); + }); +}); diff --git a/packages/objectql/src/build-probes.ts b/packages/objectql/src/build-probes.ts new file mode 100644 index 000000000..8fddaedc5 --- /dev/null +++ b/packages/objectql/src/build-probes.ts @@ -0,0 +1,268 @@ +// Copyright (c) 2025 ObjectStack. Licensed under the Apache-2.0 license. +// +// ADR-0038 L3 — runtime probes: after a publish, exercise the published app +// the way a user would (one real read per artifact) and report what is +// actually broken AT RUNTIME. Schema-valid ≠ renders ≠ returns data: the +// 2026-06-10/11 incident set shipped seeds that never materialized, dataset +// queries that returned 0 on populated objects, and "Published!" states the +// runtime couldn't read back — all invisible until something actually +// queried. These probes are that something, run by the build pipeline (not +// the user), generalizing the `seedApplied` pattern to the whole artifact +// graph. +// +// Every finding is a BuildIssue (the ADR-0038 verification contract — same +// shape the cloud L1 graph lint emits) with `layer: 'runtime'`, so agents, +// chat surfaces, and eval harnesses consume one stream regardless of which +// verification plane found the problem. + +/** One runtime-verification finding (ADR-0038 BuildIssue, layer 'runtime'). */ +export interface RuntimeBuildIssue { + layer: 'runtime'; + severity: 'error' | 'warning'; + /** The artifact whose runtime behaviour is broken. */ + artifact: { type: string; name: string }; + /** What it exercised, when narrower than the artifact (e.g. a widget). */ + ref?: { type: string; name: string; member?: string }; + /** 'seed_not_applied' | 'view_read_failed' | 'empty_query' | 'widget_query_failed' | 'probes_unavailable' */ + code: string; + message: string; + fix?: string; +} + +/** Aggregate result of one post-publish probe pass. */ +export interface BuildProbeReport { + /** Findings, empty when every probe passed. */ + issues: RuntimeBuildIssue[]; + /** How many probes actually ran, per plane (0s mean nothing to probe). */ + checked: { seeds: number; views: number; widgets: number }; +} + +/** The single read the probes need from the data engine. */ +export interface ProbeEngine { + find(objectName: string, query: unknown): Promise>>; +} + +/** Optional analytics surface — when absent, widget probes degrade to a warning. */ +export interface ProbeAnalytics { + queryDataset(dataset: unknown, selection: unknown, context?: unknown): Promise; +} + +export interface RunBuildProbesOptions { + engine: ProbeEngine; + /** Read an ACTIVE (published) item body by type+name; undefined when absent. */ + getItem: (type: string, name: string) => Promise; + /** The just-published artifact set (publishPackageDrafts' `published`). */ + published: Array<{ type: string; name: string }>; + /** + * The kernel's analytics service, when one is mounted. Widget probes run + * the SAME `queryDataset` path the dashboard renderer hits — absent + * service means widgets can't be probed (one aggregate warning, not + * per-widget noise). + */ + analytics?: ProbeAnalytics; + /** Threaded into engine/analytics reads (tenant scoping). */ + organizationId?: string | null; +} + +type Rec = Record; + +function asRec(v: unknown): Rec | undefined { + return v && typeof v === 'object' && !Array.isArray(v) ? (v as Rec) : undefined; +} + +function asArr(v: unknown): unknown[] { + return Array.isArray(v) ? v : []; +} + +/** Result rows of a queryDataset call, tolerant of {rows}/{data}/array shapes. */ +function resultRows(result: unknown): unknown[] { + if (Array.isArray(result)) return result; + const r = asRec(result); + if (!r) return []; + if (Array.isArray(r.rows)) return r.rows; + if (Array.isArray(r.data)) return r.data; + return []; +} + +/** True when the object has at least one row (probe read, isSystem). */ +async function hasRows( + engine: ProbeEngine, + objectName: string, + organizationId?: string | null, +): Promise { + const rows = await engine.find(objectName, { + fields: ['id'], + limit: 1, + ...(organizationId ? { where: { organization_id: organizationId } } : {}), + context: { isSystem: true }, + }); + return Array.isArray(rows) && rows.length > 0; +} + +/** + * Run the L3 runtime probes over a just-published artifact set: + * + * • per published `seed` — its target object must have rows now + * (`seed_not_applied`: the rows were promised but never materialized); + * • per published `view` — a limit-1 read through the same engine the + * renderer uses must not throw (`view_read_failed`); + * • per published `dashboard` widget — its real dataset selection must + * execute (`widget_query_failed`) and must not return empty on an object + * that HAS rows (`empty_query` — the four-layer staging incident class). + * + * All probes are reads (limit-1 / single aggregate); a probe crash is + * reported, never thrown — verification must not break the publish it + * verifies. + */ +export async function runBuildProbes(opts: RunBuildProbesOptions): Promise { + const issues: RuntimeBuildIssue[] = []; + const checked = { seeds: 0, views: 0, widgets: 0 }; + const { engine, getItem, published, analytics, organizationId } = opts; + + // Memoized active-item reads (a dashboard and its widgets share datasets). + const itemCache = new Map(); + const readItem = async (type: string, name: string): Promise => { + const key = `${type} ${name}`; + if (itemCache.has(key)) return itemCache.get(key); + let item: unknown | undefined; + try { + item = await getItem(type, name); + } catch { + item = undefined; + } + itemCache.set(key, item); + return item; + }; + + // ── Seeds: rows must exist after publish ──────────────────────────────── + for (const p of published.filter((x) => x.type === 'seed')) { + const body = asRec(await readItem('seed', p.name)); + const objectName = typeof body?.object === 'string' ? body.object : undefined; + if (!objectName) continue; + checked.seeds += 1; + try { + if (!(await hasRows(engine, objectName, organizationId))) { + issues.push({ + layer: 'runtime', + severity: 'error', + artifact: { type: 'seed', name: p.name }, + ref: { type: 'object', name: objectName }, + code: 'seed_not_applied', + message: `Seed "${p.name}" was published but object "${objectName}" has no rows — the sample data never materialized.`, + fix: `Check the publish response's seedApplied for the load error, fix the seed rows (field names/types), and republish the seed.`, + }); + } + } catch (e) { + issues.push({ + layer: 'runtime', + severity: 'error', + artifact: { type: 'seed', name: p.name }, + ref: { type: 'object', name: objectName }, + code: 'seed_not_applied', + message: `Seed "${p.name}" probe could not read object "${objectName}": ${String((e as Error)?.message ?? e)}`, + }); + } + } + + // ── Views: the renderer's read path must not throw ────────────────────── + for (const p of published.filter((x) => x.type === 'view')) { + const body = asRec(await readItem('view', p.name)); + const config = asRec(body?.config); + const dataObj = asRec(config?.data)?.object; + const objectName = + typeof body?.object === 'string' ? body.object + : typeof dataObj === 'string' ? dataObj + : undefined; + if (!objectName) continue; + checked.views += 1; + try { + await engine.find(objectName, { + fields: ['id'], + limit: 1, + ...(organizationId ? { where: { organization_id: organizationId } } : {}), + context: { isSystem: true }, + }); + } catch (e) { + issues.push({ + layer: 'runtime', + severity: 'error', + artifact: { type: 'view', name: p.name }, + ref: { type: 'object', name: objectName }, + code: 'view_read_failed', + message: `View "${p.name}" cannot read object "${objectName}": ${String((e as Error)?.message ?? e)} — it will render as an error for every user.`, + fix: `Verify object "${objectName}" published successfully (its table must exist) and that the view's binding is correct.`, + }); + } + } + + // ── Dashboard widgets: the real dataset selection must return data ────── + const dashboards = published.filter((x) => x.type === 'dashboard'); + let widgetsToProbe = 0; + for (const p of dashboards) { + const body = asRec(await readItem('dashboard', p.name)); + const widgets = asArr(body?.widgets).map(asRec).filter((w): w is Rec => !!w); + const datasetBound = widgets.filter((w) => typeof w.dataset === 'string' && w.dataset); + widgetsToProbe += datasetBound.length; + if (!analytics || typeof analytics.queryDataset !== 'function') continue; + + for (const w of datasetBound) { + const widgetId = String(w.id ?? w.title ?? '?'); + const dsName = w.dataset as string; + const dataset = asRec(await readItem('dataset', dsName)); + if (!dataset) continue; // dangling dataset is an L1 (graph) finding, not a runtime one + checked.widgets += 1; + + // The widget's own selection when present, else the dataset's + // first measure — the same default the renderer falls back to. + const measures = asArr(w.values).filter((v): v is string => typeof v === 'string' && v.length > 0); + const firstMeasure = asRec(asArr(dataset.measures)[0])?.name; + const selection = { + measures: measures.length ? measures : typeof firstMeasure === 'string' ? [firstMeasure] : [], + dimensions: [], + limit: 1, + }; + if (selection.measures.length === 0) continue; // nothing selectable — schema/graph problem + + const objectName = typeof dataset.object === 'string' ? dataset.object : undefined; + try { + const result = await analytics.queryDataset(dataset, selection, undefined); + const rows = resultRows(result); + if (rows.length === 0 && objectName && (await hasRows(engine, objectName, organizationId))) { + issues.push({ + layer: 'runtime', + severity: 'error', + artifact: { type: 'dashboard', name: p.name }, + ref: { type: 'dataset', name: dsName, member: widgetId }, + code: 'empty_query', + message: `Dashboard "${p.name}" widget "${widgetId}" returns NO data from dataset "${dsName}" although object "${objectName}" has rows — the widget will render empty for every user.`, + fix: `Run the dataset query directly to see the compiled strategy/SQL; check the dataset's measure/dimension field bindings against object "${objectName}".`, + }); + } + } catch (e) { + issues.push({ + layer: 'runtime', + severity: 'error', + artifact: { type: 'dashboard', name: p.name }, + ref: { type: 'dataset', name: dsName, member: widgetId }, + code: 'widget_query_failed', + message: `Dashboard "${p.name}" widget "${widgetId}" query against dataset "${dsName}" failed: ${String((e as Error)?.message ?? e)}`, + fix: `Fix the dataset definition (or the widget's values/dimensions) so the query compiles, then republish.`, + }); + } + } + } + + // Widgets existed but no analytics service was mounted — say so ONCE + // (silence would read as "probed and passed", which it was not). + if (widgetsToProbe > 0 && (!analytics || typeof analytics.queryDataset !== 'function')) { + issues.push({ + layer: 'runtime', + severity: 'warning', + artifact: { type: 'dashboard', name: dashboards.map((d) => d.name).join(', ') }, + code: 'probes_unavailable', + message: `${widgetsToProbe} dashboard widget(s) could not be probed: no analytics service is mounted on this kernel.`, + }); + } + + return { issues, checked }; +} diff --git a/packages/objectql/src/index.ts b/packages/objectql/src/index.ts index eb730d2b3..4fc260e68 100644 --- a/packages/objectql/src/index.ts +++ b/packages/objectql/src/index.ts @@ -84,3 +84,8 @@ export type { // Seed loader — materializes `seed` metadata into rows (used by publishMetaItem // and the runtime dispatcher/app plugins). export { SeedLoaderService } from './seed-loader.js'; + +// ADR-0038 L3 — post-publish runtime probes (one real read per published +// artifact); findings are BuildIssue-shaped with layer 'runtime'. +export { runBuildProbes } from './build-probes.js'; +export type { RuntimeBuildIssue, BuildProbeReport, RunBuildProbesOptions } from './build-probes.js'; diff --git a/packages/objectql/src/protocol.ts b/packages/objectql/src/protocol.ts index 519ac8f55..f263852f1 100644 --- a/packages/objectql/src/protocol.ts +++ b/packages/objectql/src/protocol.ts @@ -4006,6 +4006,15 @@ export class ObjectStackProtocolImplementation implements ObjectStackProtocol { failed: Array<{ type: string; name: string; error: string; code?: string }>; /** Aggregate result of materializing every published `seed` (absent when no seeds). */ seedApplied?: { success: boolean; inserted: number; updated: number; error?: string; errors?: unknown[] }; + /** + * ADR-0038 L3 — post-publish runtime probe report (absent when nothing + * was publishable). One real read per published artifact: seeded + * objects must have rows, views must be readable, dashboard widgets' + * dataset selections must execute and return data. `issues` carries + * BuildIssue-shaped findings (layer 'runtime') for the agent / chat + * health surfaces; probes never fail the publish itself. + */ + probes?: import('./build-probes.js').BuildProbeReport; }> { await this.ensureOverlayIndex(); const orgId = request.organizationId ?? null; @@ -4054,15 +4063,45 @@ export class ObjectStackProtocolImplementation implements ObjectStackProtocol { } } + const seedApplied = seedBodies.length > 0 ? await this.applySeedBodies(seedBodies, orgId) : undefined; + + // ADR-0038 L3: exercise what was just published — one real read per + // artifact — so "Published!" can never again mean "and silently + // broken". Best-effort by design: a probe crash is swallowed (the + // publish already happened and must report as such), and findings ride + // the response for the agent / chat health card to act on. + let probes: import('./build-probes.js').BuildProbeReport | undefined; + if (published.length > 0) { + try { + const { runBuildProbes } = await import('./build-probes.js'); + const analytics = this.getServicesRegistry?.().get('analytics'); + probes = await runBuildProbes({ + engine: this.engine as any, + getItem: async (type, name) => { + const wrapper: any = await (this as any).getMetaItem({ + type, + name, + ...(orgId ? { organizationId: orgId } : {}), + }); + return wrapper?.item ?? wrapper ?? undefined; + }, + published, + ...(analytics && typeof analytics.queryDataset === 'function' ? { analytics } : {}), + organizationId: orgId, + }); + } catch { + probes = undefined; + } + } + return { success: failed.length === 0 && published.length > 0, publishedCount: published.length, failedCount: failed.length, published, failed, - ...(seedBodies.length > 0 - ? { seedApplied: await this.applySeedBodies(seedBodies, orgId) } - : {}), + ...(seedApplied ? { seedApplied } : {}), + ...(probes ? { probes } : {}), }; }