Merge branch 'main' into aeisenberg/telemetry

.vscode/launch.json

@@ -89,6 +89,9 @@
         "--extensionDevelopmentPath=${workspaceRoot}/extensions/ql-vscode",
         "--extensionTestsPath=${workspaceRoot}/extensions/ql-vscode/out/vscode-tests/cli-integration/index",
         "${workspaceRoot}/extensions/ql-vscode/src/vscode-tests/cli-integration/data",
+        // Add a path to a checked out instance of the codeql repository so the libraries are
+        // available in the workspace for the tests.
+        // "${workspaceRoot}/../codeql"
       ],
       "stopOnEntry": false,
       "sourceMaps": true,

.github/TELEMETRY.md → TELEMETRY.md

@@ -17,11 +17,11 @@ GitHub collects anonymized information related to the usage of the extensions. T
 
 ## How long will data be retained?
 
-Data will be retained for a maximum of 30 days.
+IP address and GUIDs will be retained for a maximum of 30 days. Aggregated data of command identifiers and run times will be retained for a maximum of 180 days.
 
 ## Who will have access to this data?
 
-IP address and GUIDs will be available to the core developers of CodeQL. Aggregated data will be available to GitHub employees.
+IP address and GUIDs will only be available to the core developers of CodeQL. Aggregated data will be available to GitHub employees.
 
 ## What data is **NOT** collected?
 

extensions/ql-vscode/CHANGELOG.md

@@ -2,6 +2,14 @@
 
 ## [UNRELEASED]
 
+## 1.3.10 - 20 January 2021
+
+- Include the full stack in error log messages to help with debugging. [#726](https://github.com/github/vscode-codeql/pull/726)
+
+## 1.3.9 - 12 January 2021
+
+- No changes visible to end users.
+
 ## 1.3.8 - 17 December 2020
 
 - Ensure databases are unlocked when removing them from the workspace. This will ensure that after a database is removed from VS Code, queries can be run on it from the command line without restarting the IDE. Requires CodeQL CLI 2.4.1 or later. [#681](https://github.com/github/vscode-codeql/pull/681)

extensions/ql-vscode/package.json

@@ -4,7 +4,7 @@
   "description": "CodeQL for Visual Studio Code",
   "author": "GitHub",
   "private": true,
-  "version": "1.3.9",
+  "version": "1.3.11",
   "publisher": "GitHub",
   "license": "MIT",
   "icon": "media/VS-marketplace-CodeQL-icon.png",
@@ -708,15 +708,15 @@
     "viewsWelcome": [
       {
         "view": "codeQLAstViewer",
-        "contents": "Run the 'CodeQL: View AST' command on an open source file from a Code QL database.\n[View AST](command:codeQL.viewAst)"
+        "contents": "Run the 'CodeQL: View AST' command on an open source file from a CodeQL database.\n[View AST](command:codeQL.viewAst)"
       },
       {
         "view": "codeQLQueryHistory",
         "contents": "Run the 'CodeQL: Run Query' command on a QL query.\n[Run Query](command:codeQL.runQuery)"
       },
       {
         "view": "codeQLDatabases",
-        "contents": "Add a Code QL database:\n[From a folder](command:codeQLDatabases.chooseDatabaseFolder)\n[From an archive](command:codeQLDatabases.chooseDatabaseArchive)\n[From a URL (as a zip file)](command:codeQLDatabases.chooseDatabaseInternet)\n[From LGTM](command:codeQLDatabases.chooseDatabaseLgtm)"
+        "contents": "Add a CodeQL database:\n[From a folder](command:codeQLDatabases.chooseDatabaseFolder)\n[From an archive](command:codeQLDatabases.chooseDatabaseArchive)\n[From a URL (as a zip file)](command:codeQLDatabases.chooseDatabaseInternet)\n[From LGTM](command:codeQLDatabases.chooseDatabaseLgtm)"
       }
     ]
   },

extensions/ql-vscode/src/cli.ts

@@ -12,6 +12,7 @@ import { promisify } from 'util';
 import { CancellationToken, Disposable } from 'vscode';
 
 import { BQRSInfo, DecodedBqrsChunk } from './pure/bqrs-cli-types';
+import * as config from './config';
 import { CliConfig } from './config';
 import { DistributionProvider, FindDistributionResultKind } from './distribution';
 import { assertNever } from './pure/helpers-pure';
@@ -267,7 +268,7 @@ export class CodeQLCliServer implements Disposable {
       const argsString = args.join(' ');
       this.logger.log(`${description} using CodeQL CLI: ${argsString}...`);
       try {
-        await new Promise((resolve, reject) => {
+        await new Promise<void>((resolve, reject) => {
           // Start listening to stdout
           process.stdout.addListener('data', (newData: Buffer) => {
             stdoutBuffers.push(newData);
@@ -573,7 +574,7 @@ export class CodeQLCliServer implements Disposable {
     return await this.runJsonCodeQlCliCommand<DecodedBqrsChunk>(['bqrs', 'decode'], subcommandArgs, 'Reading bqrs data');
   }
 
-  async interpretBqrs(metadata: { kind: string; id: string }, resultsPath: string, interpretedResultsPath: string, sourceInfo?: SourceInfo): Promise<sarif.Log> {
+  async interpretBqrs(metadata: { kind: string; id: string; scored?: string }, resultsPath: string, interpretedResultsPath: string, sourceInfo?: SourceInfo): Promise<sarif.Log> {
     const args = [
       `-t=kind=${metadata.kind}`,
       `-t=id=${metadata.id}`,
@@ -585,6 +586,9 @@ export class CodeQLCliServer implements Disposable {
       // grouping client-side.
       '--no-group-results',
     ];
+    if (config.isCanary() && metadata.scored !== undefined) {
+      args.push(`-t=scored=${metadata.scored}`);
+    }
     if (sourceInfo !== undefined) {
       args.push(
         '--source-archive', sourceInfo.sourceArchive,

extensions/ql-vscode/src/commandRunner.ts

@@ -131,7 +131,13 @@ export function commandRunner(
           showAndLogWarningMessage(errorMessage);
         }
       } else {
-        showAndLogErrorMessage(errorMessage);
+        // Include the full stack in the error log only.
+        const fullMessage = e.stack
+          ? `${errorMessage}\n${e.stack}`
+          : errorMessage;
+        showAndLogErrorMessage(errorMessage, {
+          fullMessage
+        });
       }
       return undefined;
     } finally {
@@ -176,7 +182,13 @@ export function commandRunnerWithProgress<R>(
           showAndLogWarningMessage(errorMessage);
         }
       } else {
-        showAndLogErrorMessage(errorMessage);
+        // Include the full stack in the error log only.
+        const fullMessage = e.stack
+          ? `${errorMessage}\n${e.stack}`
+          : errorMessage;
+        showAndLogErrorMessage(errorMessage, {
+          fullMessage
+        });
       }
       return undefined;
     } finally {

extensions/ql-vscode/src/config.ts

@@ -253,3 +253,7 @@ export class CliConfigListener extends ConfigListener implements CliConfig {
  * Enables canary features of this extension. Recommended for all internal users.
  */
 export const CANARY_FEATURES = new Setting('canary', ROOT_SETTING);
+
+export function isCanary() {
+  return !!CANARY_FEATURES.getValue<boolean>();
+}

extensions/ql-vscode/src/helpers.ts

@@ -16,14 +16,18 @@ import { logger } from './logging';
  * @param message The message to show.
  * @param options.outputLogger The output logger that will receive the message
  * @param options.items A set of items that will be rendered as actions in the message.
+ * @param options.fullMessage An alternate message that is added to the log, but not displayed
+ *                           in the popup. This is useful for adding extra detail to the logs
+ *                           that would be too noisy for the popup.
  *
  * @return A promise that resolves to the selected item or undefined when being dismissed.
  */
 export async function showAndLogErrorMessage(message: string, {
   outputLogger = logger,
-  items = [] as string[]
+  items = [] as string[],
+  fullMessage = undefined as (string | undefined)
 } = {}): Promise<string | undefined> {
-  return internalShowAndLog(message, items, outputLogger, Window.showErrorMessage);
+  return internalShowAndLog(message, items, outputLogger, Window.showErrorMessage, fullMessage);
 }
 /**
  * Show a warning message and log it to the console
@@ -58,10 +62,15 @@ export async function showAndLogInformationMessage(message: string, {
 
 type ShowMessageFn = (message: string, ...items: string[]) => Thenable<string | undefined>;
 
-async function internalShowAndLog(message: string, items: string[], outputLogger = logger,
-  fn: ShowMessageFn): Promise<string | undefined> {
+async function internalShowAndLog(
+  message: string,
+  items: string[],
+  outputLogger = logger,
+  fn: ShowMessageFn,
+  fullMessage?: string
+): Promise<string | undefined> {
   const label = 'Show Log';
-  outputLogger.log(message);
+  outputLogger.log(fullMessage || message);
   const result = await fn(message, label, ...items);
   if (result === label) {
     outputLogger.show();

extensions/ql-vscode/src/pure/interface-types.ts

@@ -34,6 +34,7 @@ export interface QueryMetadata {
   description?: string;
   id?: string;
   kind?: string;
+  scored?: string;
 }
 
 export interface PreviousExecution {

extensions/ql-vscode/src/query-results.ts

@@ -173,7 +173,7 @@ export async function interpretResults(
   if (metadata === undefined) {
     throw new Error('Can\'t interpret results without query metadata');
   }
-  let { kind, id } = metadata;
+  let { kind, id, scored } = metadata;
   if (kind === undefined) {
     throw new Error('Can\'t interpret results without query metadata including kind');
   }
@@ -182,5 +182,5 @@ export async function interpretResults(
     // SARIF format does, so in the absence of one, we use a dummy id.
     id = 'dummy-id';
   }
-  return await server.interpretBqrs({ kind, id }, resultsPath, interpretedResultsPath, sourceInfo);
+  return await server.interpretBqrs({ kind, id, scored }, resultsPath, interpretedResultsPath, sourceInfo);
 }

extensions/ql-vscode/src/quick-query.ts

@@ -4,7 +4,6 @@ import * as path from 'path';
 import { CancellationToken, ExtensionContext, window as Window, workspace, Uri } from 'vscode';
 import { ErrorCodes, ResponseError } from 'vscode-languageclient';
 import { CodeQLCliServer } from './cli';
-import { ProgressCallback, UserCancellationException } from './commandRunner';
 import { DatabaseUI } from './databases-ui';
 import { logger } from './logging';
 import {
@@ -14,6 +13,10 @@ import {
   showAndLogErrorMessage,
   showBinaryChoiceDialog,
 } from './helpers';
+import {
+  ProgressCallback,
+  UserCancellationException
+} from './commandRunner';
 
 const QUICK_QUERIES_DIR_NAME = 'quick-queries';
 const QUICK_QUERY_QUERY_NAME = 'quick-query.ql';

extensions/ql-vscode/src/telemetry.ts

@@ -87,15 +87,15 @@ export class TelemetryListener extends ConfigListener {
     const client = (this.reporter as any).appInsightsClient as appInsights.TelemetryClient;
     if (client) {
       // add a telemetry processor to delete unwanted properties
-      client.addTelemetryProcessor((envelope) => {
+      client.addTelemetryProcessor((envelope: any) => {
         delete envelope.tags['ai.cloud.roleInstance'];
         delete (envelope.data as any)?.baseData?.properties?.['common.remotename'];
 
         return true;
       });
 
       // add a telemetry processor to log if requested
-      client.addTelemetryProcessor((envelope) => {
+      client.addTelemetryProcessor((envelope: any) => {
         if (LOG_TELEMETRY.getValue<boolean>()) {
           logger.log(`Telemetry: ${JSON.stringify(envelope)}`);
         }

extensions/ql-vscode/src/upgrades.ts

@@ -152,7 +152,7 @@ export async function upgradeDatabase(
 
   if (compileUpgradeResult.compiledUpgrades === undefined) {
     const error = compileUpgradeResult.error || '[no error message available]';
-    (`Compilation of database upgrades failed: ${error}`);
+    showAndLogErrorMessage(`Compilation of database upgrades failed: ${error}`);
     return;
   }
 

extensions/ql-vscode/src/vscode-tests/cli-integration/run-cli.test.ts

@@ -40,7 +40,7 @@ describe('Use cli', function() {
     ]);
   });
 
-  it.only('should resolve query packs', async function() {
+  it('should resolve query packs', async function() {
     skipIfNoCodeQL(this);
     const qlpacks = await cli.resolveQlpacks(getOnDiskWorkspaceFolders());
     // should have a bunch of qlpacks. just check that a few known ones exist

extensions/ql-vscode/src/vscode-tests/ensureCli.ts

@@ -72,7 +72,7 @@ export async function ensureCli(useCli: boolean) {
       console.log('Total content size', Math.round(contentLength / _1MB), 'MB');
       const archiveFile = fs.createWriteStream(downloadedFilePath);
       const body = assetStream.body;
-      await new Promise((resolve, reject) => {
+      await new Promise<void>((resolve, reject) => {
         let numBytesDownloaded = 0;
         let lastMessage = 0;
         body.on('data', (data) => {
@@ -117,7 +117,11 @@ function hasCodeQL() {
 
 export function skipIfNoCodeQL(context: Mocha.Context) {
   if (!hasCodeQL()) {
-    console.log('The CodeQL libraries are not available as a folder in this workspace. To fix: checkout the github/codeql repository and set the TEST_CODEQL_PATH environment variable to the checked out directory.');
+    console.log([
+      'The CodeQL libraries are not available as a folder in this workspace.',
+      'To fix in CI: checkout the github/codeql repository and set the \'TEST_CODEQL_PATH\' environment variable to the checked out directory.',
+      'To fix when running from vs code, see the comment in the launch.json file in the \'Launch Integration Tests - With CLI\' section.'
+    ].join('\n\n'));
     context.skip();
   }
 }

extensions/ql-vscode/src/vscode-tests/no-workspace/query-results.test.ts

@@ -152,7 +152,8 @@ describe('CompletedQuery', () => {
     const sourceInfo = {};
     const metadata = {
       kind: 'my-kind',
-      id: 'my-id' as string | undefined
+      id: 'my-id' as string | undefined,
+      scored: undefined
     };
     const results1 = await interpretResults(
       mockServer,
@@ -183,7 +184,7 @@ describe('CompletedQuery', () => {
     );
     expect(results2).to.eq('1234');
     expect(spy).to.have.been.calledWith(
-      { kind: 'my-kind', id: 'dummy-id' },
+      { kind: 'my-kind', id: 'dummy-id', scored: undefined },
       resultsPath, interpretedResultsPath, sourceInfo
     );