From cebdffa9323cff21c51ae678cfe2efa6ca1d8209 Mon Sep 17 00:00:00 2001 From: nikitareshetnik Date: Wed, 1 Jul 2026 14:31:58 +0300 Subject: [PATCH 01/19] Redesign AI Evaluation Report: Part 1 Signed-off-by: nikitareshetnik --- .../azure-devops-report/src/main.tsx | 16 +- .../TypeScript/components/App.tsx | 156 +- .../TypeScript/components/AppShell.tsx | 507 +++ .../TypeScript/components/CasesView.tsx | 530 +++ .../components/ChatDetailsSection.tsx | 111 - .../TypeScript/components/ComparisonView.tsx | 526 +++ .../TypeScript/components/HistoryView.tsx | 497 +++ .../components/MetricDetailsSection.tsx | 76 - .../TypeScript/components/MetricPanel.tsx | 424 ++ .../TypeScript/components/OverviewView.tsx | 447 +++ .../TypeScript/components/PassFailBar.tsx | 143 - .../TypeScript/components/ReportContext.tsx | 180 +- .../components/ScenarioRunHistory.tsx | 80 - .../TypeScript/components/ScenarioTree.tsx | 135 - .../TypeScript/components/ScoreDetail.tsx | 40 - .../components/ScoreNodeHistory.tsx | 175 - .../TypeScript/components/SidebarTree.tsx | 256 ++ .../TypeScript/components/Summary.ts | 50 +- .../TypeScript/components/TranscriptBlock.tsx | 377 ++ .../TypeScript/components/TrendChart.tsx | 258 ++ .../TypeScript/components/ViewRouter.tsx | 26 + .../TypeScript/components/reportStyles.ts | 360 ++ .../TypeScript/components/theme.css | 631 +++ .../TypeScript/components/theme.ts | 65 + .../TypeScript/components/useAdoResize.ts | 30 + .../TypeScript/components/viewModels.ts | 276 ++ .../TypeScript/html-report/src/main.tsx | 13 +- .../TypeScript/package-lock.json | 3481 +++++++++++++++-- .../TypeScript/package.json | 11 +- .../TypeScript/test/ado.p4.test.tsx | 147 + .../TypeScript/test/casesView.p2.test.tsx | 57 + .../TypeScript/test/exec-rescope.test.tsx | 120 + .../test/fixtures.self-check.test.ts | 200 + .../TypeScript/test/fixtures/richDataset.ts | 465 +++ .../TypeScript/test/setup.ts | 4 + .../TypeScript/test/summary.smoke.test.ts | 64 + .../TypeScript/test/transcript.p2.test.tsx | 70 + .../TypeScript/test/viewModels.test.ts | 144 + .../TypeScript/test/views.p3.test.tsx | 86 + .../TypeScript/vitest.config.ts | 14 + 40 files changed, 9928 insertions(+), 1320 deletions(-) create mode 100644 src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/TypeScript/components/AppShell.tsx create mode 100644 src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/TypeScript/components/CasesView.tsx delete mode 100644 src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/TypeScript/components/ChatDetailsSection.tsx create mode 100644 src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/TypeScript/components/ComparisonView.tsx create mode 100644 src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/TypeScript/components/HistoryView.tsx delete mode 100644 src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/TypeScript/components/MetricDetailsSection.tsx create mode 100644 src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/TypeScript/components/MetricPanel.tsx create mode 100644 src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/TypeScript/components/OverviewView.tsx delete mode 100644 src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/TypeScript/components/PassFailBar.tsx delete mode 100644 src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/TypeScript/components/ScenarioRunHistory.tsx delete mode 100644 src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/TypeScript/components/ScenarioTree.tsx delete mode 100644 src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/TypeScript/components/ScoreDetail.tsx delete mode 100644 src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/TypeScript/components/ScoreNodeHistory.tsx create mode 100644 src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/TypeScript/components/SidebarTree.tsx create mode 100644 src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/TypeScript/components/TranscriptBlock.tsx create mode 100644 src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/TypeScript/components/TrendChart.tsx create mode 100644 src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/TypeScript/components/ViewRouter.tsx create mode 100644 src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/TypeScript/components/reportStyles.ts create mode 100644 src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/TypeScript/components/theme.css create mode 100644 src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/TypeScript/components/theme.ts create mode 100644 src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/TypeScript/components/useAdoResize.ts create mode 100644 src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/TypeScript/components/viewModels.ts create mode 100644 src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/TypeScript/test/ado.p4.test.tsx create mode 100644 src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/TypeScript/test/casesView.p2.test.tsx create mode 100644 src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/TypeScript/test/exec-rescope.test.tsx create mode 100644 src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/TypeScript/test/fixtures.self-check.test.ts create mode 100644 src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/TypeScript/test/fixtures/richDataset.ts create mode 100644 src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/TypeScript/test/setup.ts create mode 100644 src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/TypeScript/test/summary.smoke.test.ts create mode 100644 src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/TypeScript/test/transcript.p2.test.tsx create mode 100644 src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/TypeScript/test/viewModels.test.ts create mode 100644 src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/TypeScript/test/views.p3.test.tsx create mode 100644 src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/TypeScript/vitest.config.ts diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/TypeScript/azure-devops-report/src/main.tsx b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/TypeScript/azure-devops-report/src/main.tsx index fc5ef8082b2..81951d81f59 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/TypeScript/azure-devops-report/src/main.tsx +++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/TypeScript/azure-devops-report/src/main.tsx @@ -53,7 +53,7 @@ const getReportData = async (client: BuildRestClient, project: string, buildId: const run = async () => { - await init(); + await init({ applyTheme: true }); await ready(); const config = getConfiguration(); @@ -69,13 +69,11 @@ const run = async () => { const scoreSummary = createScoreSummary(dataset); createRoot(document.getElementById('root')!).render( - - - - - - - + + + + + ); } catch (e) { @@ -95,4 +93,4 @@ const run = async () => { }); }; -run(); \ No newline at end of file +run(); diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/TypeScript/components/App.tsx b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/TypeScript/components/App.tsx index 6d73c8220e1..396e9528e11 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/TypeScript/components/App.tsx +++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/TypeScript/components/App.tsx @@ -1,139 +1,25 @@ -// Licensed to the .NET Foundation under one or more agreements. +// Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. -import { useState } from 'react'; -import { Settings28Regular, FilterDismissRegular, DismissRegular, ArrowDownloadRegular } from '@fluentui/react-icons'; -import { Button, Drawer, DrawerBody, DrawerHeader, DrawerHeaderTitle, SearchBox, Switch, Tooltip } from '@fluentui/react-components'; -import { makeStyles } from '@fluentui/react-components'; import './App.css'; -import { ScenarioGroup } from './ScenarioTree'; -import { GlobalTagsDisplay, FilterableTagsDisplay, categorizeAndSortTags } from './TagsDisplay'; -import { tokens } from '@fluentui/react-components'; -import { ScoreNodeHistory } from './ScoreNodeHistory'; -import { useReportContext } from './ReportContext'; - -const useStyles = makeStyles({ - header: { - display: 'flex', - flexDirection: 'column', - gap: '8px', - position: 'sticky', - top: 0, - zIndex: 1, - padding: '0rem 2rem 1rem 2rem', - backgroundColor: tokens.colorNeutralBackground1, - borderBottom: `1px solid ${tokens.colorNeutralStroke2}`, - marginBottom: '1rem', - }, - headerTop: { - display: 'flex', - justifyContent: 'space-between', - alignItems: 'center', - }, - body: { - padding: '0rem 2rem', - }, - headerActions: { - display: 'flex', - alignItems: 'center', - gap: '12px', - }, - footerText: { fontSize: '0.8rem', marginTop: '2rem' }, - closeButton: { - position: 'absolute', - top: '1.5rem', - right: '1rem', - }, - switchLabel: { fontSize: '1rem', paddingTop: '1rem' }, - drawerBody: { paddingTop: '1rem' }, -}); - -export const App = () => { - const classes = useStyles(); - const { dataset, scoreSummary, selectedTags, clearFilters, searchValue, setSearchValue } = useReportContext(); - const [isSettingsOpen, setIsSettingsOpen] = useState(false); - const { renderMarkdown, setRenderMarkdown, prettifyJson, setPrettifyJson } = useReportContext(); - const { globalTags, filterableTags } = categorizeAndSortTags(dataset, scoreSummary.primaryResult.executionName); - - const toggleSettings = () => setIsSettingsOpen(!isSettingsOpen); - const closeSettings = () => setIsSettingsOpen(false); - - const downloadDataset = () => { - // create a stringified JSON of the dataset - const dataStr = JSON.stringify(dataset, null, 2); - - // create a link to download the JSON file in the page and click it - const blob = new Blob([dataStr], { type: 'application/json' }); - const url = URL.createObjectURL(blob); - const a = document.createElement('a'); - a.href = url; - a.download = `${scoreSummary.primaryResult.executionName}.json`; - document.body.appendChild(a); - a.click(); - document.body.removeChild(a); - URL.revokeObjectURL(url); - }; - - return ( - <> -
-
-

AI Evaluation Report

-
- {(selectedTags.length > 0 || !!searchValue) && ( - -
-
- - - - - -
- -
- -
- -

- Generated at {dataset.createdAt} by Microsoft.Extensions.AI.Evaluation.Reporting version {dataset.generatorVersion} -

- - - - Settings - + ); + })} + + + ); +}; + +const Sidebar = () => { + const classes = useStyles(); + const { scoreSummary, setExec, activeExecution } = useReportContext(); + + const executions = useMemo( + () => [...scoreSummary.executionHistory.keys()], + [scoreSummary], + ); + const selectedExec = activeExecution; + + return ( + + ); +}; + +const SettingsDrawer = () => { + const classes = useStyles(); + const { + dataset, scoreSummary, + isSettingsOpen, setIsSettingsOpen, + renderMarkdown, setRenderMarkdown, + prettifyJson, setPrettifyJson, + } = useReportContext(); + + const downloadDataset = () => { + const dataStr = JSON.stringify(dataset, null, 2); + const blob = new Blob([dataStr], { type: 'application/json' }); + const url = URL.createObjectURL(blob); + const a = document.createElement('a'); + a.href = url; + a.download = `${scoreSummary.primaryResult.executionName}.json`; + document.body.appendChild(a); + a.click(); + document.body.removeChild(a); + URL.revokeObjectURL(url); + }; + + return ( + setIsSettingsOpen(data.open)} position="end"> + + Settings + + + + + ); +}; + +const useHostTheme = (themeSource: ThemeSource, setDarkMode: (v: boolean) => void): void => { + useEffect(() => { + if (themeSource !== 'host') return; + + const sync = () => { + setDarkMode(detectHostDarkMode()); + }; + + sync(); + + window.addEventListener('themeChanged', sync); + return () => window.removeEventListener('themeChanged', sync); + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [themeSource, setDarkMode]); +}; + +export const AppShell = ({ + heightStrategy, + themeSource, + children, +}: { + heightStrategy: HeightStrategy; + themeSource: ThemeSource; + children: React.ReactNode; +}) => { + const classes = useStyles(); + const { dataset, scoreSummary, setIsSettingsOpen } = useReportContext(); + const { darkMode, setDarkMode } = useReportContext(); + const { fluentTheme, rootClass } = resolveTheme(darkMode); + + const casesCount = + scoreSummary.primaryResult.numPassingIterations + + scoreSummary.primaryResult.numFailingIterations; + + useHostTheme(themeSource, setDarkMode); + + useAdoResize(themeSource === 'host'); + + const rootClassName = heightStrategy === 'auto-grow' ? classes.rootAutoGrow : classes.rootFill; + + return ( + // FluentProvider carries only the theme class: Fluent copies a provider's + // className onto the FluentProviders it renders inside portals. + +
+
+
+ + AI Evaluation Report +
+
+ {themeSource === 'toggle' && } + +
+
+ +
+ +
+ +
+ {children} +

+ Generated at {dataset.createdAt} by Microsoft.Extensions.AI.Evaluation.Reporting version {dataset.generatorVersion} +

+
+
+
+ + +
+ + ); +}; diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/TypeScript/components/CasesView.tsx b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/TypeScript/components/CasesView.tsx new file mode 100644 index 00000000000..197a23ef6a2 --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/TypeScript/components/CasesView.tsx @@ -0,0 +1,530 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +import { useEffect, useMemo, useRef, useState } from 'react'; +import { + makeStyles, + mergeClasses, + Switch, + SearchBox, + Menu, + MenuTrigger, + MenuButton, + MenuPopover, + MenuList, + MenuItemCheckbox, + MenuDivider, + Button, +} from '@fluentui/react-components'; +import { ChevronRight16Regular, TagMultipleRegular } from '@fluentui/react-icons'; +import { MoverDirections, getTabsterAttribute } from 'tabster'; +import { useReportStyles, statusSolidVar } from './reportStyles'; +import { useReportContext } from './ReportContext'; +import { ScoreNode, getConversationDisplay } from './Summary'; +import { isLeafFailed } from './viewModels'; +import { categorizeAndSortTags } from './TagsDisplay'; +import { TranscriptBlock } from './TranscriptBlock'; +import { MetricPanel } from './MetricPanel'; + +const PAGE_SIZE = 25; + +const useStyles = makeStyles({ + root: { display: 'flex', flexDirection: 'column', gap: 'var(--spacing-l)' }, + + controls: { + display: 'flex', + alignItems: 'center', + gap: 'var(--spacing-s)', + flexWrap: 'wrap', + justifyContent: 'flex-end', + }, + searchWrap: { + flex: '1 1 auto', + minWidth: '180px', + display: 'flex', + }, + search: { width: '100%' }, + tagMenuHead: { + display: 'flex', + justifyContent: 'space-between', + alignItems: 'center', + gap: 'var(--spacing-m)', + padding: 'var(--spacing-xs) var(--spacing-m) var(--spacing-s)', + }, + tagMenuTitle: { + fontSize: 'var(--font-size-100)', + fontWeight: 'var(--font-weight-bold)', + textTransform: 'uppercase', + letterSpacing: '0.5px', + color: 'var(--neutral-foreground-3)', + }, + tagMenuList: { maxHeight: '320px', minWidth: '248px' }, + collapseBtn: { + appearance: 'none', + border: 'none', + cursor: 'pointer', + fontFamily: 'inherit', + fontSize: 'var(--font-size-200)', + fontWeight: 'var(--font-weight-semibold)', + lineHeight: 'var(--line-height-200)', + padding: 'var(--spacing-xs) var(--spacing-s)', + borderRadius: 'var(--radius-medium)', + backgroundColor: 'transparent', + color: 'var(--neutral-foreground-2)', + transition: 'background-color var(--duration-faster) var(--curve-easy-ease)', + '&:hover:not(:disabled)': { backgroundColor: 'var(--subtle-background-hover)', color: 'var(--neutral-foreground-1)' }, + '&:disabled': { color: 'var(--neutral-foreground-disabled)', cursor: 'default' }, + }, + + rowlist: { overflow: 'hidden' }, + rowWrap: { borderTop: '1px solid var(--neutral-stroke-3)', '&:first-child': { borderTop: 'none' } }, + row: { + appearance: 'none', + border: 'none', + margin: 0, + width: '100%', + font: 'inherit', + color: 'inherit', + textAlign: 'left', + display: 'flex', + alignItems: 'center', + gap: 'var(--spacing-m-nudge)', + padding: 'var(--spacing-m) var(--spacing-l)', + cursor: 'pointer', + userSelect: 'none', + backgroundColor: 'transparent', + }, + caret: { + flexShrink: 0, + color: 'var(--neutral-foreground-3)', + transition: 'transform var(--duration-fast) var(--curve-easy-ease)', + }, + caretOpen: { transform: 'rotate(90deg)' }, + dotWrap: { display: 'inline-flex', flex: 'none' }, + statusDot: { + width: '8px', + height: '8px', + borderRadius: 'var(--radius-circular)', + flex: 'none', + boxSizing: 'border-box', + }, + label: { + flex: '1 1 auto', + minWidth: 0, + marginRight: 'auto', + fontFamily: 'var(--font-family-monospace)', + fontSize: 'var(--font-size-200)', + color: 'var(--neutral-foreground-2)', + whiteSpace: 'nowrap', + overflow: 'hidden', + textOverflow: 'ellipsis', + }, + + detail: { + padding: '0 var(--spacing-xxl) var(--spacing-xxl) var(--spacing-xxxl)', + outline: 'none', + backgroundColor: 'transparent', + }, + metaLine: { + padding: 'var(--spacing-l) 0', + maxWidth: '75rem', + }, + metaText: { + fontSize: 'var(--font-size-200)', + color: 'var(--neutral-foreground-3)', + letterSpacing: '0.2px', + }, + twoPane: { + display: 'grid', + gridTemplateColumns: '1.12fr 1fr', + gap: 'var(--spacing-l)', + alignItems: 'start', + maxWidth: '75rem', + }, + + empty: { + display: 'flex', + flexDirection: 'column', + alignItems: 'center', + textAlign: 'center', + gap: 'var(--spacing-s)', + padding: 'var(--spacing-xxxl) var(--spacing-xl)', + }, + emptyTitle: { + fontSize: 'var(--font-size-400)', + fontWeight: 'var(--font-weight-semibold)', + color: 'var(--neutral-foreground-1)', + }, + emptyReason: { + fontSize: 'var(--font-size-300)', + color: 'var(--neutral-foreground-3)', + maxWidth: '400px', + lineHeight: 1.5, + }, + clearLink: { + appearance: 'none', + border: 'none', + cursor: 'pointer', + fontFamily: 'inherit', + fontSize: 'var(--font-size-300)', + fontWeight: 'var(--font-weight-semibold)', + color: 'var(--brand-foreground-1)', + backgroundColor: 'transparent', + padding: 'var(--spacing-xs) var(--spacing-s)', + borderRadius: 'var(--radius-medium)', + marginTop: 'var(--spacing-xs)', + '&:hover': { backgroundColor: 'var(--brand-background-2)', textDecoration: 'underline' }, + }, + + pager: { + display: 'flex', + justifyContent: 'center', + alignItems: 'center', + gap: 'var(--spacing-m)', + marginTop: 'var(--spacing-xs)', + }, + pagerBtn: { + appearance: 'none', + cursor: 'pointer', + fontFamily: 'inherit', + fontSize: 'var(--font-size-300)', + lineHeight: 1, + padding: 'var(--spacing-s) var(--spacing-m)', + borderRadius: 'var(--radius-medium)', + border: '1px solid var(--neutral-stroke-2)', + backgroundColor: 'var(--neutral-background-1)', + color: 'var(--neutral-foreground-2)', + transition: 'background-color var(--duration-faster) var(--curve-easy-ease)', + '&:hover:not(:disabled)': { backgroundColor: 'var(--subtle-background-hover)', color: 'var(--neutral-foreground-1)' }, + '&:disabled': { color: 'var(--neutral-foreground-disabled)', cursor: 'default', opacity: 0.6 }, + }, + pagerLabel: { + fontSize: 'var(--font-size-300)', + color: 'var(--neutral-foreground-3)', + fontVariantNumeric: 'tabular-nums', + padding: '0 var(--spacing-s)', + }, +}); + +type CaseRowVM = { + key: string; + label: string; + group?: string; + failed: boolean; + scenario: ScenarioRunResult; +}; + +const buildRows = (root: ScoreNode): CaseRowVM[] => { + const rows: CaseRowVM[] = []; + for (const node of root.flattenedNodes) { + if (!node.isLeafNode || !node.scenario) { + continue; + } + const segments = node.name.split(' / '); + const label = segments[segments.length - 1]; + const group = segments.length > 1 ? segments.slice(0, -1).join(' · ') : undefined; + rows.push({ + key: node.nodeKey, + label, + group, + failed: isLeafFailed(node.scenario), + scenario: node.scenario, + }); + } + return rows; +}; + +const metaLineFor = (scenario: ScenarioRunResult): string | undefined => { + const tags = scenario.tags ?? []; + if (tags.length === 0) { + return undefined; + } + return tags + .map((t) => { + const i = t.indexOf(':'); + return i > 0 ? t.slice(i + 1).trim() : t; + }) + .join(' · '); +}; + +const CaseRow = ({ + vm, + open, + onToggle, + registerRowRef, +}: { + vm: CaseRowVM; + open: boolean; + onToggle: () => void; + registerRowRef: (key: string, el: HTMLButtonElement | null) => void; +}) => { + const classes = useStyles(); + const s = useReportStyles(); + const detailRef = useRef(null); + + useEffect(() => { + if (open && detailRef.current) { + detailRef.current.focus(); + } + }, [open]); + + const dotSolid = vm.failed ? statusSolidVar('danger') : statusSolidVar('success'); + const conversation = open ? getConversationDisplay(vm.scenario.messages, vm.scenario.modelResponse) : null; + const metaLine = open ? metaLineFor(vm.scenario) : undefined; + + return ( +
+ + + {open && conversation && ( +
+ {metaLine && ( +
+ {metaLine} +
+ )} +
+ + +
+
+ )} +
+ ); +}; + +export const CasesView = () => { + const classes = useStyles(); + const s = useReportStyles(); + const { + dataset, + activeExecution, + activeNode, + filterTree, + failedOnly, + setFailedOnly, + scenSort, + setCasePage, + casePage, + selectedTags, + handleTagClick, + searchValue, + setSearchValue, + clearFilters, + } = useReportContext(); + + const { filterableTags } = categorizeAndSortTags(dataset, activeExecution); + + const [openKey, setOpenKey] = useState(null); + const rowRefs = useRef(new Map()); + const registerRowRef = (key: string, el: HTMLButtonElement | null) => { + if (el) { + rowRefs.current.set(key, el); + } else { + rowRefs.current.delete(key); + } + }; + + const closeOpen = (returnFocus: boolean) => { + const key = openKey; + setOpenKey(null); + if (returnFocus && key) { + requestAnimationFrame(() => rowRefs.current.get(key)?.focus()); + } + }; + + const allRows = useMemo(() => { + const filtered = filterTree(activeNode); + return filtered ? buildRows(filtered) : []; + }, [filterTree, activeNode]); + + const rows = useMemo(() => { + const filtered = failedOnly ? allRows.filter((r) => r.failed) : allRows; + const sorted = [...filtered]; + if (scenSort === 'passRate') { + sorted.sort((a, b) => Number(b.failed) - Number(a.failed) || a.label.localeCompare(b.label)); + } else { + sorted.sort((a, b) => a.label.localeCompare(b.label)); + } + return sorted; + }, [allRows, failedOnly, scenSort]); + + const pageCount = Math.max(1, Math.ceil(rows.length / PAGE_SIZE)); + useEffect(() => { + if (casePage > pageCount) { + setCasePage(pageCount); + } + }, [casePage, pageCount, setCasePage]); + const page = Math.min(casePage, pageCount); + const pageRows = rows.slice((page - 1) * PAGE_SIZE, page * PAGE_SIZE); + + useEffect(() => { + if (openKey && !pageRows.some((r) => r.key === openKey)) { + setOpenKey(null); + } + }, [openKey, pageRows]); + + const hasActiveFilter = selectedTags.length > 0 || !!searchValue || failedOnly; + const searchCount = allRows.length; + const searchPlaceholder = `Search across ${searchCount} ${searchCount === 1 ? 'case' : 'cases'}`; + const tagLabel = selectedTags.length > 0 ? `${selectedTags.length} selected` : 'All tags'; + + return ( +
+
+
+ setSearchValue(data.value)} + /> +
+ + {filterableTags.length > 0 && ( + + + } + className="eval-fitbtn" + > + {tagLabel} + + + +
+ Filter by tag + {selectedTags.length > 0 && ( + + )} +
+ + + {filterableTags.map(({ tag, count }) => ( + { + handleTagClick(tag); + setCasePage(1); + }} + secondaryContent={String(count)} + > + {tag} + + ))} + + + + )} + + + + { + setFailedOnly(data.checked); + setCasePage(1); + }} + label="Failing only" + /> +
+ + {pageRows.length === 0 ? ( +
+
+ No matching cases + + {hasActiveFilter + ? 'No scenarios match the current search, tags, or failing-only filter.' + : 'There are no scenario results to display.'} + + {hasActiveFilter && ( + + )} +
+
+ ) : ( +
+
+ {pageRows.map((vm) => ( + (openKey === vm.key ? closeOpen(true) : setOpenKey(vm.key))} + registerRowRef={registerRowRef} + /> + ))} +
+
+ )} + + {pageCount > 1 && ( +
+ + Page {page} of {pageCount} + +
+ )} +
+ ); +}; diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/TypeScript/components/ChatDetailsSection.tsx b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/TypeScript/components/ChatDetailsSection.tsx deleted file mode 100644 index 545f181220d..00000000000 --- a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/TypeScript/components/ChatDetailsSection.tsx +++ /dev/null @@ -1,111 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -import { Table, TableHeader, TableRow, TableHeaderCell, TableBody, TableCell } from "@fluentui/react-components"; -import { ChevronDown12Regular, ChevronRight12Regular, Warning16Regular, CheckmarkCircle16Regular, Copy16Regular } from "@fluentui/react-icons"; -import { useState } from "react"; -import { useStyles } from "./Styles"; - -export const ChatDetailsSection = ({ chatDetails }: { chatDetails: ChatDetails; }) => { - const classes = useStyles(); - const [isExpanded, setIsExpanded] = useState(false); - - const totalTurns = chatDetails.turnDetails.length; - const cachedTurns = chatDetails.turnDetails.filter(turn => turn.cacheHit === true).length; - - const hasCacheKey = chatDetails.turnDetails.some(turn => turn.cacheKey !== undefined); - const hasCacheStatus = chatDetails.turnDetails.some(turn => turn.cacheHit !== undefined); - const hasModel = chatDetails.turnDetails.some(turn => turn.model !== undefined); - const hasModelProvider = chatDetails.turnDetails.some(turn => turn.modelProvider !== undefined); - const hasInputTokens = chatDetails.turnDetails.some(turn => turn.usage?.inputTokenCount !== undefined); - const hasOutputTokens = chatDetails.turnDetails.some(turn => turn.usage?.outputTokenCount !== undefined); - const hasTotalTokens = chatDetails.turnDetails.some(turn => turn.usage?.totalTokenCount !== undefined); - - const copyToClipboard = (text: string) => { - navigator.clipboard.writeText(text); - }; - return ( -
e.key === 'Enter' && setIsExpanded(!isExpanded)}> -
setIsExpanded(!isExpanded)}> - {isExpanded ? : } -

Diagnostic Data

- {hasCacheStatus && ( -
- {cachedTurns != totalTurns ? - : - } - {cachedTurns}/{totalTurns} chat responses for this evaluation were fulfiled from cache -
- )} -
- - {isExpanded && ( -
-
- - - - {hasCacheKey && Cache Key} - {hasCacheStatus && Cache Status} - Latency (s) - {hasModel && Model} - {hasModelProvider && Model Provider} - {hasInputTokens && Input Tokens} - {hasOutputTokens && Output Tokens} - {hasTotalTokens && Total Tokens} - - - - {chatDetails.turnDetails.map((turn, index) => ( - - {hasCacheKey && ( - - {turn.cacheKey ? ( -
- - {turn.cacheKey.substring(0, 8)}... - - -
- ) : ( - N/A - )} - - )} - {hasCacheStatus && ( - - {turn.cacheHit === true ? - - Hit - : - - Miss - } - - )} - {turn.latency.toFixed(2)} - {hasModel && {turn.model || '-'}} - {hasModelProvider && {turn.modelProvider || '-'}} - {hasInputTokens && {turn.usage?.inputTokenCount || '-'}} - {hasOutputTokens && {turn.usage?.outputTokenCount || '-'}} - {hasTotalTokens && {turn.usage?.totalTokenCount || '-'}} - - ))} - -
-
-
- )} -
- ); -}; diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/TypeScript/components/ComparisonView.tsx b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/TypeScript/components/ComparisonView.tsx new file mode 100644 index 00000000000..458ad3e9d83 --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/TypeScript/components/ComparisonView.tsx @@ -0,0 +1,526 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +import { useMemo, useEffect } from 'react'; +import { + makeStyles, + tokens, + Dropdown, + Option, + Text, +} from '@fluentui/react-components'; +import { useReportContext } from './ReportContext'; +import type { MetricDelta } from './viewModels'; + +const useStyles = makeStyles({ + root: { + display: 'flex', + flexDirection: 'column', + gap: '1.5rem', + }, + emptyState: { + display: 'flex', + flexDirection: 'column', + alignItems: 'center', + justifyContent: 'center', + padding: '3rem 2rem', + gap: '0.75rem', + color: tokens.colorNeutralForeground3, + textAlign: 'center', + border: `1px dashed ${tokens.colorNeutralStroke2}`, + borderRadius: tokens.borderRadiusMedium, + }, + emptyTitle: { + fontWeight: tokens.fontWeightSemibold, + fontSize: tokens.fontSizeBase400, + color: tokens.colorNeutralForeground2, + }, + selectorRow: { + display: 'flex', + alignItems: 'center', + gap: '0.75rem', + flexWrap: 'wrap', + }, + selectorGroup: { + display: 'flex', + flexDirection: 'column', + gap: '0.25rem', + minWidth: '200px', + flex: '1 1 200px', + maxWidth: '340px', + }, + selectorLabel: { + display: 'flex', + alignItems: 'center', + gap: '0.375rem', + fontSize: tokens.fontSizeBase200, + fontWeight: tokens.fontWeightSemibold, + color: tokens.colorNeutralForeground3, + textTransform: 'uppercase', + letterSpacing: '0.5px', + }, + selectorDot: { + width: '8px', + height: '8px', + borderRadius: '50%', + flex: 'none', + }, + selectorArrow: { + alignSelf: 'flex-end', + paddingBottom: '0.5rem', + color: tokens.colorNeutralForeground3, + fontSize: tokens.fontSizeBase400, + }, + kpiStrip: { + display: 'flex', + flexWrap: 'wrap', + gap: '0.75rem', + }, + kpiCard: { + display: 'flex', + flexDirection: 'column', + gap: '0.25rem', + padding: '0.75rem 1rem', + border: `1px solid ${tokens.colorNeutralStroke2}`, + borderRadius: tokens.borderRadiusMedium, + backgroundColor: tokens.colorNeutralBackground2, + minWidth: '140px', + flex: '1 1 140px', + }, + kpiLabel: { + fontSize: tokens.fontSizeBase100, + color: tokens.colorNeutralForeground3, + fontWeight: tokens.fontWeightSemibold, + textTransform: 'uppercase', + letterSpacing: '0.4px', + }, + kpiValue: { + fontSize: tokens.fontSizeBase600, + fontWeight: tokens.fontWeightSemibold, + lineHeight: '1.15', + }, + kpiSub: { + fontSize: tokens.fontSizeBase200, + color: tokens.colorNeutralForeground3, + }, + kpiPositive: { color: tokens.colorStatusSuccessForeground1 }, + kpiNegative: { color: tokens.colorStatusDangerForeground1 }, + kpiNeutral: { color: tokens.colorNeutralForeground1 }, + tableSection: { + display: 'flex', + flexDirection: 'column', + gap: '0.5rem', + }, + tableSectionHeader: { + display: 'flex', + alignItems: 'center', + justifyContent: 'space-between', + flexWrap: 'wrap', + gap: '0.5rem', + }, + tableSectionTitle: { + fontWeight: tokens.fontWeightSemibold, + fontSize: tokens.fontSizeBase300, + }, + legendRow: { + display: 'flex', + alignItems: 'center', + gap: '0.5rem', + fontSize: tokens.fontSizeBase200, + color: tokens.colorNeutralForeground3, + }, + legendSwatch: { + width: '8px', + height: '8px', + borderRadius: '50%', + flex: 'none', + }, + deltaTable: { + width: '100%', + borderCollapse: 'collapse' as const, + fontSize: tokens.fontSizeBase200, + }, + deltaTableHead: { + borderBottom: `1px solid ${tokens.colorNeutralStroke2}`, + }, + deltaTableHeadCell: { + fontSize: tokens.fontSizeBase100, + fontWeight: tokens.fontWeightSemibold, + color: tokens.colorNeutralForeground3, + textTransform: 'uppercase' as const, + letterSpacing: '0.4px', + padding: '0.25rem 0.5rem', + textAlign: 'left' as const, + }, + deltaTableRow: { + borderBottom: `1px solid ${tokens.colorNeutralStroke2}`, + '&:last-child': { borderBottom: 'none' }, + }, + deltaTableCell: { + padding: '0.5rem 0.5rem', + color: tokens.colorNeutralForeground1, + verticalAlign: 'middle' as const, + }, + fromTo: { + display: 'flex', + alignItems: 'center', + gap: '0.375rem', + fontSize: tokens.fontSizeBase200, + color: tokens.colorNeutralForeground2, + }, + fromValue: { + color: tokens.colorNeutralForeground3, + }, + toValue: { + fontWeight: tokens.fontWeightSemibold, + color: tokens.colorNeutralForeground1, + }, + arrow: { + color: tokens.colorNeutralForeground3, + fontSize: tokens.fontSizeBase100, + }, + sparkBar: { + display: 'flex', + alignItems: 'center', + gap: '0.375rem', + minWidth: '100px', + }, + deltaPositive: { + color: tokens.colorStatusSuccessForeground1, + fontWeight: tokens.fontWeightSemibold, + }, + deltaNegative: { + color: tokens.colorStatusDangerForeground1, + fontWeight: tokens.fontWeightSemibold, + }, + deltaNeutral: { + color: tokens.colorNeutralForeground3, + }, + metricName: { + fontWeight: tokens.fontWeightRegular, + color: tokens.colorNeutralForeground1, + }, + scenarioName: { + fontSize: tokens.fontSizeBase100, + color: tokens.colorNeutralForeground3, + marginTop: '0.1rem', + }, +}); + +const formatVal = (v: number): string => (Number.isInteger(v) ? v.toFixed(0) : v.toFixed(2)); +const formatDelta = (d: number): string => { + const s = d > 0 ? '+' : ''; + return `${s}${Number.isInteger(d) ? d.toFixed(0) : d.toFixed(2)}`; +}; + +const normalise = (delta: number, maxAbs: number) => + maxAbs > 0 ? Math.max(-1, Math.min(1, delta / maxAbs)) : 0; + +type SparkBarProps = { normalised: number }; +const SparkBar = ({ normalised }: SparkBarProps) => { + const BAR_W = 80; + const BAR_H = 8; + const mid = BAR_W / 2; + const fillW = Math.abs(normalised) * (BAR_W / 2); + const fillX = normalised >= 0 ? mid : mid - fillW; + const fillColor = + normalised > 0 ? '#16a34a' : normalised < 0 ? '#dc2626' : '#94a3b8'; + + return ( + + ); +}; + +export const ComparisonView = () => { + const classes = useStyles(); + const { scoreSummary, cmpA, setCmpA, cmpB, setCmpB } = useReportContext(); + + const executions = useMemo( + () => [...scoreSummary.executionHistory.keys()], + [scoreSummary], + ); + + useEffect(() => { + if (executions.length >= 2) { + if (!cmpA) setCmpA(executions[executions.length - 2]); + if (!cmpB) setCmpB(executions[executions.length - 1]); + } + }, [executions, cmpA, cmpB, setCmpA, setCmpB]); + + const effectiveA = cmpA ?? (executions.length >= 2 ? executions[executions.length - 2] : undefined); + const effectiveB = cmpB ?? (executions.length >= 1 ? executions[executions.length - 1] : undefined); + + const hasTwoExecs = executions.length >= 2; + + const allDeltas: MetricDelta[] = useMemo(() => { + if (!hasTwoExecs || !effectiveA || !effectiveB || effectiveA === effectiveB) return []; + + const execARoot = scoreSummary.executionHistory.get(effectiveA); + if (!execARoot) return []; + + const deltas: MetricDelta[] = []; + for (const node of execARoot.flattenedNodes) { + if (!node.isLeafNode || !node.scenario) continue; + const scenA = node.scenario; + + const bRoot = scoreSummary.executionHistory.get(effectiveB); + if (!bRoot) continue; + const bLeaf = bRoot.flattenedNodes.find( + (n) => + n.isLeafNode && + n.scenario?.scenarioName === scenA.scenarioName && + n.scenario?.iterationName === scenA.iterationName, + ); + if (!bLeaf?.scenario) continue; + const scenB = bLeaf.scenario; + + for (const [metricName, metricA] of Object.entries(scenA.evaluationResult?.metrics ?? {})) { + if (!metricA || metricA.$type !== 'numeric') continue; + const vA = (metricA as NumericMetric).value; + if (typeof vA !== 'number') continue; + + const metricB = scenB.evaluationResult?.metrics?.[metricName]; + if (!metricB || metricB.$type !== 'numeric') continue; + const vB = (metricB as NumericMetric).value; + if (typeof vB !== 'number') continue; + + deltas.push({ + scenarioName: scenA.scenarioName, + iterationName: scenA.iterationName, + metricName, + fromExecution: effectiveA, + toExecution: effectiveB, + fromValue: vA, + toValue: vB, + delta: vB - vA, + }); + } + } + return deltas; + }, [scoreSummary, effectiveA, effectiveB, hasTwoExecs]); + + const sortedDeltas = useMemo( + () => [...allDeltas].sort((a, b) => a.metricName.localeCompare(b.metricName) || a.scenarioName.localeCompare(b.scenarioName)), + [allDeltas], + ); + + const improvedCount = allDeltas.filter((d) => d.delta > 0).length; + const regressedCount = allDeltas.filter((d) => d.delta < 0).length; + const totalCount = allDeltas.length; + + const biggestMover = useMemo( + () => + allDeltas.length > 0 + ? allDeltas.reduce((a, b) => (Math.abs(b.delta) > Math.abs(a.delta) ? b : a)) + : undefined, + [allDeltas], + ); + + const maxAbsDelta = useMemo( + () => (allDeltas.length > 0 ? Math.max(...allDeltas.map((d) => Math.abs(d.delta))) : 1), + [allDeltas], + ); + + if (!hasTwoExecs) { + return ( +
+ Needs at least 2 executions + + Run the evaluation suite across multiple executions to compare results side by side. + +
+ ); + } + + return ( +
+
+
+ + + setCmpA(data.optionValue)} + > + {executions.map((name) => ( + + ))} + +
+ + + +
+ + + setCmpB(data.optionValue)} + > + {executions.map((name) => ( + + ))} + +
+
+ + {effectiveA === effectiveB && ( +
+ Select two different executions + + The baseline and current executions are the same. Choose different executions to see the delta. + +
+ )} + + {effectiveA !== effectiveB && allDeltas.length > 0 && ( +
+
+ Metrics improved + {improvedCount} + of {totalCount} metrics +
+
+ Metrics regressed + {regressedCount} + of {totalCount} metrics +
+ {biggestMover && ( +
+ Biggest mover + 0 ? classes.kpiPositive : classes.kpiNegative + }`} + > + {biggestMover.delta > 0 ? '▲' : '▼'} {Math.abs(biggestMover.delta).toFixed(3)} + + {biggestMover.metricName} +
+ )} +
+ )} + + {effectiveA !== effectiveB && sortedDeltas.length > 0 && ( +
+
+ Per-metric change +
+
+
+ + + + + + + + + + + {sortedDeltas.map((d, i) => { + const norm = normalise(d.delta, maxAbsDelta); + const isPos = d.delta > 0; + const isNeg = d.delta < 0; + return ( + + + + + + ); + })} + +
Metric ▲ + Baseline → Current + Change
+
{d.metricName}
+
{d.scenarioName}
+ 		+
+ {formatVal(d.fromValue)} + → + + {formatVal(d.toValue)} + +
+ 		+
+ + + {isPos ? '▲' : isNeg ? '▼' : ''}{' '} + {formatDelta(d.delta)} + +
+
+
+ )} + + {effectiveA !== effectiveB && sortedDeltas.length === 0 && allDeltas.length === 0 && ( +
+ No comparable numeric metrics + + The selected executions share no numeric metrics that can be compared. + +
+ )} +
+ ); +}; diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/TypeScript/components/HistoryView.tsx b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/TypeScript/components/HistoryView.tsx new file mode 100644 index 00000000000..d39793d0450 --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/TypeScript/components/HistoryView.tsx @@ -0,0 +1,497 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +import { useMemo, useState } from 'react'; +import { + makeStyles, + tokens, + Tab, + TabList, + Text, + Badge, + type SelectTabEventHandler, +} from '@fluentui/react-components'; +import { useReportContext } from './ReportContext'; +import { metricHistoryForScenario } from './viewModels'; +import { TrendChart } from './TrendChart'; + +const useStyles = makeStyles({ + root: { + display: 'flex', + flexDirection: 'column', + gap: '1.5rem', + }, + emptyState: { + display: 'flex', + flexDirection: 'column', + alignItems: 'center', + justifyContent: 'center', + padding: '3rem 2rem', + gap: '0.75rem', + color: tokens.colorNeutralForeground3, + textAlign: 'center', + border: `1px dashed ${tokens.colorNeutralStroke2}`, + borderRadius: tokens.borderRadiusMedium, + }, + emptyTitle: { + fontWeight: tokens.fontWeightSemibold, + fontSize: tokens.fontSizeBase400, + color: tokens.colorNeutralForeground2, + }, + metricTabsLabel: { + fontSize: tokens.fontSizeBase200, + fontWeight: tokens.fontWeightSemibold, + color: tokens.colorNeutralForeground3, + textTransform: 'uppercase', + letterSpacing: '0.5px', + marginBottom: '0.25rem', + }, + chartCard: { + border: `1px solid ${tokens.colorNeutralStroke2}`, + borderRadius: tokens.borderRadiusMedium, + padding: '1.25rem 1.25rem 1rem', + backgroundColor: tokens.colorNeutralBackground2, + display: 'flex', + flexDirection: 'column', + gap: '1rem', + }, + chartCardHeader: { + display: 'flex', + alignItems: 'baseline', + gap: '0.5rem', + flexWrap: 'wrap', + }, + chartCardTitle: { + fontWeight: tokens.fontWeightSemibold, + fontSize: tokens.fontSizeBase400, + }, + chartCardSubtitle: { + fontSize: tokens.fontSizeBase200, + color: tokens.colorNeutralForeground3, + }, + statsRow: { + display: 'flex', + flexWrap: 'wrap', + gap: '0.75rem', + }, + statCard: { + display: 'flex', + flexDirection: 'column', + gap: '0.125rem', + padding: '0.625rem 0.875rem', + border: `1px solid ${tokens.colorNeutralStroke2}`, + borderRadius: tokens.borderRadiusMedium, + backgroundColor: tokens.colorNeutralBackground1, + minWidth: '110px', + }, + statLabel: { + fontSize: tokens.fontSizeBase100, + color: tokens.colorNeutralForeground3, + fontWeight: tokens.fontWeightSemibold, + textTransform: 'uppercase', + letterSpacing: '0.4px', + }, + statValue: { + fontSize: tokens.fontSizeBase500, + fontWeight: tokens.fontWeightSemibold, + lineHeight: '1.2', + }, + statValuePositive: { + color: tokens.colorStatusSuccessForeground1, + }, + statValueNegative: { + color: tokens.colorStatusDangerForeground1, + }, + statValueNeutral: { + color: tokens.colorNeutralForeground1, + }, + runHistorySection: { + display: 'flex', + flexDirection: 'column', + gap: '0.5rem', + }, + runHistoryTitle: { + fontWeight: tokens.fontWeightSemibold, + fontSize: tokens.fontSizeBase300, + }, + runTable: { + width: '100%', + borderCollapse: 'collapse' as const, + fontSize: tokens.fontSizeBase200, + }, + runTableHead: { + borderBottom: `1px solid ${tokens.colorNeutralStroke2}`, + }, + runTableHeadCell: { + fontSize: tokens.fontSizeBase100, + fontWeight: tokens.fontWeightSemibold, + color: tokens.colorNeutralForeground3, + textTransform: 'uppercase' as const, + letterSpacing: '0.4px', + padding: '0.25rem 0.5rem', + textAlign: 'left' as const, + }, + runTableHeadCellRight: { + textAlign: 'right' as const, + }, + runTableRow: { + borderBottom: `1px solid ${tokens.colorNeutralStroke2}`, + }, + runTableCell: { + padding: '0.5rem 0.5rem', + color: tokens.colorNeutralForeground2, + }, + runTableCellRight: { + textAlign: 'right' as const, + padding: '0.5rem 0.5rem', + }, + deltaPositive: { + color: tokens.colorStatusSuccessForeground1, + fontWeight: tokens.fontWeightSemibold, + }, + deltaNegative: { + color: tokens.colorStatusDangerForeground1, + fontWeight: tokens.fontWeightSemibold, + }, + deltaBaseline: { + color: tokens.colorNeutralForeground3, + fontStyle: 'italic', + }, + scenarioPickerLabel: { + fontSize: tokens.fontSizeBase200, + fontWeight: tokens.fontWeightSemibold, + color: tokens.colorNeutralForeground3, + textTransform: 'uppercase', + letterSpacing: '0.5px', + }, + scenarioPicker: { + display: 'flex', + flexWrap: 'wrap', + gap: '0.375rem', + }, + scenarioChip: { + padding: '0.2rem 0.625rem', + borderRadius: '9999px', + border: `1px solid ${tokens.colorNeutralStroke2}`, + cursor: 'pointer', + fontSize: tokens.fontSizeBase200, + background: 'transparent', + color: tokens.colorNeutralForeground2, + fontFamily: 'inherit', + '&:hover': { + backgroundColor: tokens.colorSubtleBackgroundHover, + }, + }, + scenarioChipSelected: { + backgroundColor: tokens.colorBrandBackground, + color: tokens.colorNeutralForegroundOnBrand, + border: `1px solid ${tokens.colorBrandBackground}`, + '&:hover': { + backgroundColor: tokens.colorBrandBackgroundHover, + }, + }, +}); + +const formatValue = (v: number): string => + Number.isInteger(v) ? v.toFixed(0) : v.toFixed(1); + +const formatDelta = (d: number): string => { + const sign = d > 0 ? '+' : ''; + return `${sign}${Number.isInteger(d) ? d.toFixed(0) : d.toFixed(2)}`; +}; + +type StatCardProps = { + label: string; + value: string; + sentiment?: 'positive' | 'negative' | 'neutral'; +}; + +const StatCard = ({ label, value, sentiment = 'neutral' }: StatCardProps) => { + const classes = useStyles(); + const valueClass = + sentiment === 'positive' + ? classes.statValuePositive + : sentiment === 'negative' + ? classes.statValueNegative + : classes.statValueNeutral; + + return ( +
+ {label} + {value} +
+ ); +}; + +export const HistoryView = () => { + const classes = useStyles(); + const { scoreSummary, dataset } = useReportContext(); + + const leafScenarios = useMemo(() => { + const primaryRoot = [...scoreSummary.executionHistory.values()][0]; + return primaryRoot + ? primaryRoot.flattenedNodes + .filter((n) => n.isLeafNode && n.scenario != null) + .map((n) => n.scenario!) + : []; + }, [scoreSummary]); + + const [selectedScenarioKey, setSelectedScenarioKey] = useState(undefined); + + const selectedScenario = useMemo(() => { + if (!selectedScenarioKey) return leafScenarios[0] ?? undefined; + return ( + leafScenarios.find( + (s) => + `${s.scenarioName}::${s.iterationName}::${s.executionName}` === + selectedScenarioKey, + ) ?? leafScenarios[0] ?? undefined + ); + }, [leafScenarios, selectedScenarioKey]); + + const allSeries = useMemo( + () => (selectedScenario ? metricHistoryForScenario(scoreSummary, selectedScenario) : []), + [scoreSummary, selectedScenario], + ); + + const [selectedMetric, setSelectedMetric] = useState(undefined); + const activeMetric = selectedMetric ?? allSeries[0]?.metricName; + + const onTabSelect: SelectTabEventHandler = (_ev, data) => { + setSelectedMetric(data.value as string); + }; + + const hasTrend = allSeries.length > 0; + + const scenarioPicker = useMemo(() => { + const seen = new Map(); + for (const s of leafScenarios) { + const k = `${s.scenarioName}::${s.iterationName}::${s.executionName}`; + const label = + s.iterationName && s.iterationName !== 'default' + ? `${s.scenarioName} · ${s.iterationName}` + : s.scenarioName; + if (!seen.has(k)) seen.set(k, { key: k, label }); + } + return [...seen.values()]; + }, [leafScenarios]); + + if (leafScenarios.length === 0) { + return ( +
+ No scenario data + + No scenarios are available in this report. + +
+ ); + } + + const chartSeries = hasTrend + ? allSeries.filter((s) => s.metricName === activeMetric) + : []; + + const activeSeriesPoints = hasTrend + ? allSeries.find((s) => s.metricName === activeMetric)?.points ?? [] + : []; + + const spreadByExec = useMemo(() => { + if (!hasTrend || !activeMetric || activeSeriesPoints.length === 0) return new Map(); + const map = new Map(); + for (const r of dataset.scenarioRunResults ?? []) { + const m = r.evaluationResult?.metrics?.[activeMetric]; + if (!m || m.$type !== 'numeric' || typeof (m as NumericMetric).value !== 'number') continue; + const v = (m as NumericMetric).value!; + const existing = map.get(r.executionName); + if (existing) { + if (v < existing.min) existing.min = v; + if (v > existing.max) existing.max = v; + } else { + map.set(r.executionName, { min: v, max: v }); + } + } + return map; + }, [dataset, activeMetric, activeSeriesPoints, hasTrend]); + + const firstPoint = activeSeriesPoints[0]; + const lastPoint = activeSeriesPoints[activeSeriesPoints.length - 1]; + const netDelta = hasTrend && firstPoint && lastPoint ? lastPoint.value - firstPoint.value : 0; + const peakValue = hasTrend + ? Math.max(...activeSeriesPoints.map((p) => p.value)) + : undefined; + + const scaleMax = peakValue !== undefined ? (peakValue <= 5 ? 5 : peakValue <= 10 ? 10 : undefined) : undefined; + const scaleLabel = scaleMax != null ? `SCORE · 1–${scaleMax}` : 'METRIC VALUE'; + + const metricNames = allSeries.map((s) => s.metricName); + + return ( +
+ {scenarioPicker.length > 1 && ( +
+ Scenario +
+ {scenarioPicker.map(({ key, label }) => { + const isActive = + key === + (selectedScenarioKey ?? + (leafScenarios[0] + ? `${leafScenarios[0].scenarioName}::${leafScenarios[0].iterationName}::${leafScenarios[0].executionName}` + : undefined)); + return ( + + ); + })} +
+
+ )} + + {!hasTrend && ( +
+ Needs at least 2 executions + + Run this scenario across multiple executions to see metric trends over time. + +
+ )} + + {hasTrend && metricNames.length > 0 && ( +
+ Metric + + {metricNames.map((name) => ( + {name} + ))} + +
+ )} + + {hasTrend && activeMetric && chartSeries.length > 0 && ( +
+
+ {activeMetric} + {scaleLabel} +
+ + {firstPoint && lastPoint && ( +
+ + + 0 ? 'positive' : netDelta < 0 ? 'negative' : 'neutral'} + /> + {peakValue !== undefined && ( + + )} +
+ )} + + + + {activeSeriesPoints.length > 0 && ( +
+ Run history + + + + + + + + + + + {activeSeriesPoints.map((pt, i) => { + const prev = i > 0 ? activeSeriesPoints[i - 1] : undefined; + const delta = prev != null ? pt.value - prev.value : undefined; + const spread = spreadByExec.get(pt.executionName); + return ( + + + + + + + ); + })} + +
ExecutionMetric scoreSpread + Change vs previous +
{pt.executionName} + {scaleMax != null + ? `${formatValue(pt.value)}/${scaleMax}` + : formatValue(pt.value)} + + {spread && spread.min !== spread.max + ? `${formatValue(spread.min)}–${formatValue(spread.max)}${scaleMax != null ? `/${scaleMax}` : ''}` + : '—'} + + {delta == null ? ( + baseline + ) : delta === 0 ? ( + — + ) : ( + 0 + ? classes.deltaPositive + : classes.deltaNegative + } + > + {delta > 0 ? '▲' : '▼'} {Math.abs(delta).toFixed(1)} + + )} +
+
+ )} +
+ )} + + {hasTrend && allSeries.length > 1 && ( +
+ All metrics overview + +
+ )} +
+ ); +}; diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/TypeScript/components/MetricDetailsSection.tsx b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/TypeScript/components/MetricDetailsSection.tsx deleted file mode 100644 index 11bcd70dd17..00000000000 --- a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/TypeScript/components/MetricDetailsSection.tsx +++ /dev/null @@ -1,76 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -import { ChevronDown12Regular, ChevronRight12Regular, DismissCircle16Regular } from "@fluentui/react-icons"; -import { useState } from "react"; -import type { MetricType } from "./MetricCard"; -import { DiagnosticsContent } from "./DiagnosticsContent"; -import { MetadataContent } from "./MetadataContent"; -import { useStyles } from "./Styles"; - - -export const MetricDetailsSection = ({ metric }: { metric: MetricType; }) => { - const classes = useStyles(); - const [isExpanded, setIsExpanded] = useState(true); - - const reason = metric.reason; - const hasReason = reason != null; - const interpretationReason = metric.interpretation?.reason; - const hasInterpretationReason = interpretationReason != null; - const diagnostics = metric.diagnostics || []; - const hasDiagnostics = diagnostics.length > 0; - const metadata = metric.metadata || {}; - const hasMetadata = Object.keys(metadata).length > 0; - - if (!hasReason && !hasInterpretationReason && !hasDiagnostics && !hasMetadata) return null; - - return ( -
e.key === 'Enter' && setIsExpanded(!isExpanded)}> -
setIsExpanded(!isExpanded)}> - {isExpanded ? : } -

Metric Details: {metric.name}

-
- - {isExpanded && ( -
- {hasReason && ( -
-
Evaluation Reason
-
- {reason} -
-
- )} - - {hasInterpretationReason && ( -
- {metric.interpretation?.failed ? -
Failure Reason
: -
Interpretation Reason
} -
- {metric.interpretation?.failed ? - {interpretationReason} : - {interpretationReason}} -
-
- )} - - {hasDiagnostics && ( -
-
Diagnostics
- -
- )} - - {hasMetadata && ( -
-
Metadata
- -
- )} -
- )} -
- ); -}; diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/TypeScript/components/MetricPanel.tsx b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/TypeScript/components/MetricPanel.tsx new file mode 100644 index 00000000000..822b44ff2ef --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/TypeScript/components/MetricPanel.tsx @@ -0,0 +1,424 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +import { useId, useState } from 'react'; +import { makeStyles, mergeClasses } from '@fluentui/react-components'; +import { ChevronRight16Regular } from '@fluentui/react-icons'; +import { useReportStyles, type ReportStatus } from './reportStyles'; +import { DiagnosticsContent } from './DiagnosticsContent'; +import { MetadataContent } from './MetadataContent'; +import { type MetricType } from './MetricCard'; + +const statusKeyOf = (rating: EvaluationRating | undefined): ReportStatus => { + switch (rating) { + case 'exceptional': + case 'good': + return 'success'; + case 'average': + return 'warning'; + case 'poor': + case 'unacceptable': + return 'danger'; + default: + return 'neutral'; + } +}; + +const ratingWord = (rating: EvaluationRating | undefined): string => { + switch (rating) { + case 'exceptional': + return 'Exceptional'; + case 'good': + return 'Good'; + case 'average': + return 'Fair'; + case 'poor': + return 'Poor'; + case 'unacceptable': + return 'Weak'; + case 'inconclusive': + return 'Inconclusive'; + default: + return 'Unknown'; + } +}; + +const solidVarOf = (sk: ReportStatus): string => + sk === 'success' ? 'var(--status-success-background-3)' + : sk === 'warning' ? 'var(--status-warning-background-3)' + : sk === 'danger' ? 'var(--status-danger-background-3)' + : 'var(--neutral-foreground-4)'; + +const textVarOf = (sk: ReportStatus): string => + sk === 'success' ? 'var(--status-success-foreground-1)' + : sk === 'warning' ? 'var(--status-warning-foreground-1)' + : sk === 'danger' ? 'var(--status-danger-foreground-1)' + : 'var(--neutral-foreground-3)'; + +const metricFailed = (metric: MetricType): boolean => + metric.interpretation?.failed === true || + (metric.diagnostics?.some((d) => d.severity === 'error') ?? false); + +type MetricKind = 'score' | 'fraction' | 'severity' | 'boolean' | 'none' | 'string'; +const metricKind = (metric: MetricType): MetricKind => { + if (metric.$type === 'boolean') return 'boolean'; + if (metric.$type === 'string') return 'string'; + if (metric.$type === 'none') return 'none'; + const declared = metric.metadata?.kind as MetricKind | undefined; + if (declared) return declared; + const v = metric.value; + if (typeof v === 'number') { + if (v >= 0 && v <= 1) return 'fraction'; + if (Number.isInteger(v) && v >= 1 && v <= 5) return 'score'; + } + return 'score'; +}; +const betterLow = (metric: MetricType): boolean => + metric.metadata?.better === 'low' || metricKind(metric) === 'severity'; + +const goodness = (metric: MetricType, kind: MetricKind): number => { + const v = typeof metric.value === 'number' ? metric.value : 0; + if (kind === 'score') return v / 5; + if (kind === 'severity') return (7 - v) / 7; + return betterLow(metric) ? 1 - v : v; +}; + +const scaleMaxOf = (kind: MetricKind): number | null => + kind === 'score' ? 5 : kind === 'severity' ? 7 : kind === 'fraction' ? 1 : null; + +const displayValue = (metric: MetricType): string | undefined => { + switch (metric.$type) { + case 'string': + return metric.value ?? undefined; + case 'boolean': + return metric.value === undefined || metric.value === null ? undefined : metric.value ? 'Yes' : 'No'; + case 'numeric': + return metric.value === undefined || metric.value === null ? undefined : String(metric.value); + case 'none': + default: + return undefined; + } +}; + +const useStyles = makeStyles({ + headerRow: { + display: 'flex', + alignItems: 'center', + justifyContent: 'space-between', + gap: 'var(--spacing-s)', + padding: 'var(--spacing-m) var(--spacing-l)', + borderBottom: '1px solid var(--neutral-stroke-3)', + }, + headerCount: { + fontSize: 'var(--font-size-100)', + color: 'var(--neutral-foreground-4)', + whiteSpace: 'nowrap', + }, + + rowWrap: { borderTop: '1px solid var(--neutral-stroke-3)' }, + row: { + appearance: 'none', + border: 'none', + margin: 0, + width: '100%', + font: 'inherit', + color: 'inherit', + textAlign: 'left', + display: 'flex', + alignItems: 'center', + gap: 'var(--spacing-m-nudge)', + minHeight: '44px', + padding: '12px var(--spacing-l)', + lineHeight: '20px', + cursor: 'pointer', + userSelect: 'none', + backgroundColor: 'transparent', + }, + caret: { + flexShrink: 0, + color: 'var(--neutral-foreground-3)', + transition: 'transform var(--duration-fast) var(--curve-easy-ease)', + }, + caretOpen: { transform: 'rotate(90deg)' }, + dotWrap: { flex: 'none', display: 'inline-flex', alignItems: 'center' }, + dot: { + width: '8px', + height: '8px', + borderRadius: 'var(--radius-circular)', + flex: 'none', + boxSizing: 'border-box', + }, + rowName: { + flex: '1 1 auto', + minWidth: 0, + whiteSpace: 'nowrap', + overflow: 'hidden', + textOverflow: 'ellipsis', + fontSize: 'var(--font-size-300)', + color: 'var(--neutral-foreground-1)', + }, + track: { + flex: 'none', + display: 'flex', + alignItems: 'stretch', + width: '96px', + height: '16px', + gap: '4px', + }, + trackBar: { + flex: 'none', + position: 'relative', + width: '96px', + height: '16px', + borderRadius: '2px', + backgroundColor: 'var(--eval-seg-empty)', + }, + seg: { flex: '1 1 0', minWidth: 0, borderRadius: '2px' }, + segCenter: { display: 'flex', alignItems: 'center', justifyContent: 'center' }, + segIcon: { fontSize: '11px', lineHeight: 1, fontWeight: 700 }, + + panel: { + padding: '0 var(--spacing-l) var(--spacing-l) var(--spacing-xxxl)', + display: 'flex', + flexDirection: 'column', + }, + hero: { + display: 'flex', + flexDirection: 'column', + gap: 'var(--spacing-s)', + padding: 'var(--spacing-m) 0 var(--spacing-l)', + }, + heroLine: { display: 'flex', alignItems: 'baseline', gap: 'var(--spacing-s-nudge)' }, + heroWord: { + fontSize: 'var(--font-size-500)', + fontWeight: 'var(--font-weight-semibold)', + lineHeight: 1.2, + }, + heroNum: { + fontSize: 'var(--font-size-400)', + color: 'var(--neutral-foreground-3)', + fontVariantNumeric: 'tabular-nums', + }, + heroIcon: { fontSize: '28px', lineHeight: 1, fontWeight: 700 }, + heroTrack: { + height: '6px', + width: '100%', + borderRadius: 'var(--radius-circular)', + backgroundColor: 'var(--eval-seg-empty)', + overflow: 'hidden', + }, + heroFill: { height: '100%', borderRadius: 'var(--radius-circular)' }, + + subSection: { + borderTop: '1px solid var(--neutral-stroke-2)', + padding: 'var(--spacing-l) 0', + }, + subHeader: { + fontSize: 'var(--font-size-200)', + fontWeight: 'var(--font-weight-semibold)', + color: 'var(--neutral-foreground-2)', + marginBottom: 'var(--spacing-s)', + }, + subBody: { + fontSize: 'var(--font-size-300)', + lineHeight: 1.55, + whiteSpace: 'pre-wrap', + wordBreak: 'break-word', + }, + empty: { + fontSize: 'var(--font-size-200)', + color: 'var(--neutral-foreground-3)', + fontStyle: 'italic', + padding: 'var(--spacing-l)', + }, +}); + +const SegmentTrack = ({ metric, kind, sk }: { metric: MetricType; kind: MetricKind; sk: ReportStatus }) => { + const classes = useStyles(); + const solid = solidVarOf(sk); + const aura = `0 0 0 3px color-mix(in srgb, ${solid} 18%, transparent)`; + + if (sk === 'neutral') { + return ( + + ); + } + + if (kind === 'boolean') { + const good = sk === 'success'; + return ( + + ); + } + + const scaleMax = kind === 'score' ? 5 : kind === 'severity' ? 7 : 0; + if (scaleMax) { + const segCount = Math.min(scaleMax, 10); + const filled = Math.round(goodness(metric, kind) * segCount); + return ( +