Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add 'compare performance' view #3843

Open
wants to merge 37 commits into
base: main
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
37 commits
Select commit Hold shift + click to select a range
3b06977
Add Compare Performance command (WIP)
asgerf Jun 26, 2024
96aa770
Show evaluation and iteration counts in table
asgerf Nov 13, 2024
aaf23ea
compare-perf: Add support for sorting options
tausbn Nov 13, 2024
70ec570
Make RAPrettyPrinter generate JSX fragments
asgerf Nov 13, 2024
8268d68
Apply styling to RA predicate names
asgerf Nov 13, 2024
58afeba
Apply a background color to the pipeline header rows
asgerf Nov 14, 2024
03ca407
Make "..." clickable to reveal abbreviated name
asgerf Nov 14, 2024
317e52c
Also abbreviate RA names in predicate overview
asgerf Nov 14, 2024
876c5b6
Colorize positive/negative deltas
tausbn Nov 14, 2024
260bf0e
Add option to choose metric
asgerf Nov 14, 2024
453aa83
Check for nullness of 'data' in a separate component
asgerf Nov 14, 2024
ccf2dc6
Simplify datasets assignment
asgerf Nov 14, 2024
412338c
feat: parallel log scanning
esbena Nov 14, 2024
6d4427e
compare-perf: Add support for selecting a single run as input
tausbn Nov 14, 2024
e039f6b
Simplify view when not in comparison mode
tausbn Nov 15, 2024
1d2c2cf
Allow word wrap to break anywhere
asgerf Nov 15, 2024
b05ec33
Use useMemo for 'nameSet'
asgerf Nov 22, 2024
20f6e3d
Use useMemo a few places to speed up UI interactions
asgerf Nov 22, 2024
558d957
Reformat code
asgerf Nov 22, 2024
6568b56
Also useMemo the 'total' row computation
asgerf Nov 22, 2024
6f7eb74
Reset hasCacheMismatch when rebuilding 'rows'
asgerf Nov 22, 2024
62f3b4f
Reformat code again
asgerf Nov 22, 2024
9800fa1
Use interface instead of type alias for TRow
asgerf Nov 22, 2024
d008963
Refactor OptionalValue
asgerf Nov 22, 2024
eec42c5
Split renderAbsoluteValue into renderOptionalValue and renderPredicat…
asgerf Nov 22, 2024
9a0699f
Refactor predicate row into a separate component
asgerf Nov 25, 2024
48954c7
Rename TRow -> Row
asgerf Nov 25, 2024
ab00152
Group together rows by fingerprinting
asgerf Nov 25, 2024
4a835b8
Factor out rendering of table body to a memoized component
asgerf Nov 25, 2024
568f082
Fix some crashes when pretty-printing an empty name
asgerf Nov 25, 2024
2cae71c
Add predicate-renaming support
asgerf Nov 25, 2024
d37469f
Add 'per evaluation' option next to metric
asgerf Nov 26, 2024
8a58279
Move "total" row to the top and render the metric
asgerf Nov 27, 2024
b9d1551
Hide "sort by" dropdown when there is no delta to sort by
asgerf Nov 27, 2024
afa3d55
Factor header rows into a component
asgerf Nov 27, 2024
c99bf5b
Only warn about cache hits in comparison mode
asgerf Nov 27, 2024
aa528c6
Remove unused export
asgerf Nov 27, 2024
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 13 additions & 0 deletions extensions/ql-vscode/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -959,6 +959,10 @@
"command": "codeQLQueryHistory.compareWith",
"title": "Compare Results"
},
{
"command": "codeQLQueryHistory.comparePerformanceWith",
"title": "Compare Performance"
},
{
"command": "codeQLQueryHistory.openOnGithub",
"title": "View Logs"
Expand Down Expand Up @@ -1230,6 +1234,11 @@
"group": "3_queryHistory@0",
"when": "viewItem == rawResultsItem || viewItem == interpretedResultsItem"
},
{
"command": "codeQLQueryHistory.comparePerformanceWith",
"group": "3_queryHistory@1",
"when": "viewItem == rawResultsItem || viewItem == interpretedResultsItem"
},
{
"command": "codeQLQueryHistory.showQueryLog",
"group": "4_queryHistory@4",
Expand Down Expand Up @@ -1733,6 +1742,10 @@
"command": "codeQLQueryHistory.compareWith",
"when": "false"
},
{
"command": "codeQLQueryHistory.comparePerformanceWith",
"when": "false"
},
{
"command": "codeQLQueryHistory.sortByName",
"when": "false"
Expand Down
1 change: 1 addition & 0 deletions extensions/ql-vscode/src/common/commands.ts
Original file line number Diff line number Diff line change
Expand Up @@ -180,6 +180,7 @@ export type QueryHistoryCommands = {
"codeQLQueryHistory.removeHistoryItemContextInline": TreeViewContextMultiSelectionCommandFunction<QueryHistoryInfo>;
"codeQLQueryHistory.renameItem": TreeViewContextMultiSelectionCommandFunction<QueryHistoryInfo>;
"codeQLQueryHistory.compareWith": TreeViewContextMultiSelectionCommandFunction<QueryHistoryInfo>;
"codeQLQueryHistory.comparePerformanceWith": TreeViewContextMultiSelectionCommandFunction<QueryHistoryInfo>;
"codeQLQueryHistory.showEvalLog": TreeViewContextMultiSelectionCommandFunction<QueryHistoryInfo>;
"codeQLQueryHistory.showEvalLogSummary": TreeViewContextMultiSelectionCommandFunction<QueryHistoryInfo>;
"codeQLQueryHistory.showEvalLogViewer": TreeViewContextMultiSelectionCommandFunction<QueryHistoryInfo>;
Expand Down
12 changes: 12 additions & 0 deletions extensions/ql-vscode/src/common/interface-types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ import type {
} from "./raw-result-types";
import type { AccessPathSuggestionOptions } from "../model-editor/suggestions";
import type { ModelEvaluationRunState } from "../model-editor/shared/model-evaluation-run-state";
import type { PerformanceComparisonDataFromLog } from "../log-insights/performance-comparison";

/**
* This module contains types and code that are shared between
Expand Down Expand Up @@ -396,6 +397,17 @@ export interface SetComparisonsMessage {
readonly message: string | undefined;
}

export type ToComparePerformanceViewMessage = SetPerformanceComparisonQueries;

export interface SetPerformanceComparisonQueries {
readonly t: "setPerformanceComparison";
readonly from: PerformanceComparisonDataFromLog;
readonly to: PerformanceComparisonDataFromLog;
readonly comparison: boolean;
}

export type FromComparePerformanceViewMessage = CommonFromViewMessages;

export type QueryCompareResult =
| RawQueryCompareResult
| InterpretedQueryCompareResult;
Expand Down
7 changes: 7 additions & 0 deletions extensions/ql-vscode/src/common/vscode/abstract-webview.ts
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,13 @@ export abstract class AbstractWebview<

constructor(protected readonly app: App) {}

public hidePanel() {
if (this.panel !== undefined) {
this.panel.dispose();
this.panel = undefined;
}
}

public async restoreView(panel: WebviewPanel): Promise<void> {
this.panel = panel;
const config = await this.getPanelConfig();
Expand Down
1 change: 1 addition & 0 deletions extensions/ql-vscode/src/common/vscode/webview-html.ts
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import type { App } from "../app";
export type WebviewKind =
| "results"
| "compare"
| "compare-performance"
| "variant-analysis"
| "data-flow-paths"
| "model-editor"
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,108 @@
import { statSync } from "fs";
import { ViewColumn } from "vscode";

import type { App } from "../common/app";
import { redactableError } from "../common/errors";
import type {
FromComparePerformanceViewMessage,
ToComparePerformanceViewMessage,
} from "../common/interface-types";
import type { Logger } from "../common/logging";
import { showAndLogExceptionWithTelemetry } from "../common/logging";
import { extLogger } from "../common/logging/vscode";
import type { WebviewPanelConfig } from "../common/vscode/abstract-webview";
import { AbstractWebview } from "../common/vscode/abstract-webview";
import { withProgress } from "../common/vscode/progress";
import { telemetryListener } from "../common/vscode/telemetry";
import type { HistoryItemLabelProvider } from "../query-history/history-item-label-provider";
import { PerformanceOverviewScanner } from "../log-insights/performance-comparison";
import { scanLog } from "../log-insights/log-scanner";
import type { ResultsView } from "../local-queries";

export class ComparePerformanceView extends AbstractWebview<
ToComparePerformanceViewMessage,
FromComparePerformanceViewMessage
> {
constructor(
app: App,
public logger: Logger,
public labelProvider: HistoryItemLabelProvider,
private resultsView: ResultsView,
) {
super(app);
}

async showResults(fromJsonLog: string, toJsonLog: string) {
const panel = await this.getPanel();
panel.reveal(undefined, false);

// Close the results viewer as it will have opened when the user clicked the query in the history view
// (which they must do as part of the UI interaction for opening the performance view).
// The performance view generally needs a lot of width so it's annoying to have the result viewer open.
this.resultsView.hidePanel();

await this.waitForPanelLoaded();

function scanLogWithProgress(log: string, logDescription: string) {
const bytes = statSync(log).size;
return withProgress(
async (progress) =>
scanLog(log, new PerformanceOverviewScanner(), progress),

{
title: `Scanning evaluator log ${logDescription} (${(bytes / 1024 / 1024).toFixed(1)} MB)`,
},
);
}

const [fromPerf, toPerf] = await Promise.all([
fromJsonLog === ""
? new PerformanceOverviewScanner()
: scanLogWithProgress(fromJsonLog, "1/2"),
scanLogWithProgress(toJsonLog, fromJsonLog === "" ? "1/1" : "2/2"),
]);

await this.postMessage({
t: "setPerformanceComparison",
from: fromPerf.getData(),
to: toPerf.getData(),
comparison: fromJsonLog !== "",
});
}

protected getPanelConfig(): WebviewPanelConfig {
return {
viewId: "comparePerformanceView",
title: "Compare CodeQL Performance",
viewColumn: ViewColumn.Active,
preserveFocus: true,
view: "compare-performance",
};
}

protected onPanelDispose(): void {}

protected async onMessage(
msg: FromComparePerformanceViewMessage,
): Promise<void> {
switch (msg.t) {
case "viewLoaded":
this.onWebViewLoaded();
break;

case "telemetry":
telemetryListener?.sendUIInteraction(msg.action);
break;

case "unhandledError":
void showAndLogExceptionWithTelemetry(
extLogger,
telemetryListener,
redactableError(
msg.error,
)`Unhandled error in performance comparison view: ${msg.error.message}`,
);
break;
}
}
}
39 changes: 39 additions & 0 deletions extensions/ql-vscode/src/extension.ts
Original file line number Diff line number Diff line change
Expand Up @@ -135,6 +135,7 @@ import { LanguageContextStore } from "./language-context-store";
import { LanguageSelectionPanel } from "./language-selection-panel/language-selection-panel";
import { GitHubDatabasesModule } from "./databases/github-databases";
import { DatabaseFetcher } from "./databases/database-fetcher";
import { ComparePerformanceView } from "./compare-performance/compare-performance-view";

/**
* extension.ts
Expand Down Expand Up @@ -924,6 +925,11 @@ async function activateWithInstalledDistribution(
from: CompletedLocalQueryInfo,
to: CompletedLocalQueryInfo,
): Promise<void> => showResultsForComparison(compareView, from, to),
async (
from: CompletedLocalQueryInfo,
to: CompletedLocalQueryInfo | undefined,
): Promise<void> =>
showPerformanceComparison(comparePerformanceView, from, to),
);

ctx.subscriptions.push(qhm);
Expand All @@ -949,6 +955,15 @@ async function activateWithInstalledDistribution(
);
ctx.subscriptions.push(compareView);

void extLogger.log("Initializing performance comparison view.");
const comparePerformanceView = new ComparePerformanceView(
app,
queryServerLogger,
labelProvider,
localQueryResultsView,
);
ctx.subscriptions.push(comparePerformanceView);

void extLogger.log("Initializing source archive filesystem provider.");
archiveFilesystemProvider_activate(ctx, dbm);

Expand Down Expand Up @@ -1190,6 +1205,30 @@ async function showResultsForComparison(
}
}

async function showPerformanceComparison(
view: ComparePerformanceView,
from: CompletedLocalQueryInfo,
to: CompletedLocalQueryInfo | undefined,
): Promise<void> {
let fromLog = from.evaluatorLogPaths?.jsonSummary;
let toLog = to?.evaluatorLogPaths?.jsonSummary;

if (to === undefined) {
toLog = fromLog;
fromLog = "";
}
if (fromLog === undefined || toLog === undefined) {
return extLogger.showWarningMessage(
`Cannot compare performance as the structured logs are missing. Did they queries complete normally?`,
);
}
await extLogger.log(
`Comparing performance of ${from.getQueryName()} and ${to?.getQueryName() ?? "baseline"}`,
);

await view.showResults(fromLog, toLog);
}

function addUnhandledRejectionListener() {
const handler = (error: unknown) => {
// This listener will be triggered for errors from other extensions as
Expand Down
29 changes: 27 additions & 2 deletions extensions/ql-vscode/src/log-insights/log-scanner.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
import type { SummaryEvent } from "./log-summary";
import { readJsonlFile } from "../common/jsonl-reader";
import type { Disposable } from "../common/disposable-object";
import { readJsonlFile } from "../common/jsonl-reader";
import type { ProgressCallback } from "../common/vscode/progress";
import type { SummaryEvent } from "./log-summary";

/**
* Callback interface used to report diagnostics from a log scanner.
Expand Down Expand Up @@ -112,3 +113,27 @@ export class EvaluationLogScannerSet {
scanners.forEach((scanner) => scanner.onDone());
}
}

/**
* Scan the evaluator summary log using the given scanner. For convenience, returns the scanner.
*
* @param jsonSummaryLocation The file path of the JSON summary log.
* @param scanner The scanner to process events from the log
*/
export async function scanLog<T extends EvaluationLogScanner>(
jsonSummaryLocation: string,
scanner: T,
progress?: ProgressCallback,
): Promise<T> {
progress?.({
// XXX all scans have step 1 - the backing progress tracker allows increments instead of steps - but for now we are happy with a tiny UI that says what is happening
message: `Scanning ...`,
step: 1,
maxStep: 2,
});
await readJsonlFile<SummaryEvent>(jsonSummaryLocation, async (obj) => {
scanner.onEvent(obj);
});
scanner.onDone();
return scanner;
}
2 changes: 2 additions & 0 deletions extensions/ql-vscode/src/log-insights/log-summary.ts
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@ interface ResultEventBase extends SummaryEventBase {
export interface ComputeSimple extends ResultEventBase {
evaluationStrategy: "COMPUTE_SIMPLE";
ra: Ra;
millis: number;
pipelineRuns?: [PipelineRun];
queryCausingWork?: string;
dependencies: { [key: string]: string };
Expand All @@ -42,6 +43,7 @@ export interface ComputeRecursive extends ResultEventBase {
evaluationStrategy: "COMPUTE_RECURSIVE";
deltaSizes: number[];
ra: Ra;
millis: number;
pipelineRuns: PipelineRun[];
queryCausingWork?: string;
dependencies: { [key: string]: string };
Expand Down
Loading
Loading