add error event to telemetry (#3750)

This commit is contained in:
Fred K. Schott 2022-06-29 14:54:33 -07:00 committed by GitHub
parent 1eab496e9d
commit dd176ca58d
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
16 changed files with 270 additions and 85 deletions

View file

@ -0,0 +1,6 @@
---
'astro': patch
'@astrojs/telemetry': patch
---
Add basic error reporting to astro telemetry

View file

@ -1,13 +1,11 @@
/* eslint-disable no-console */
import { LogOptions } from '../core/logger/core.js';
import { AstroTelemetry } from '@astrojs/telemetry';
import * as colors from 'kleur/colors';
import yargs from 'yargs-parser';
import { z } from 'zod';
import { telemetry } from '../events/index.js';
import * as event from '../events/index.js';
import add from '../core/add/index.js';
import build from '../core/build/index.js';
import { openConfig } from '../core/config.js';
@ -15,10 +13,12 @@ import devServer from '../core/dev/index.js';
import { enableVerboseLogging, nodeLogDestination } from '../core/logger/node.js';
import { formatConfigErrorMessage, formatErrorMessage, printHelp } from '../core/messages.js';
import preview from '../core/preview/index.js';
import { createSafeError } from '../core/util.js';
import { createSafeError, ASTRO_VERSION } from '../core/util.js';
import { check } from './check.js';
import { openInBrowser } from './open.js';
import * as telemetryHandler from './telemetry.js';
import { collectErrorMetadata } from '../core/errors.js';
import { eventError, eventConfigError } from '../events/index.js';
type Arguments = yargs.Arguments;
type CLICommand =
@ -61,9 +61,6 @@ function printAstroHelp() {
});
}
// PACKAGE_VERSION is injected when we build and publish the astro package.
const ASTRO_VERSION = process.env.PACKAGE_VERSION ?? 'development';
/** Display --version flag */
async function printVersion() {
console.log();
@ -111,7 +108,6 @@ export async function cli(args: string[]) {
} else if (flags.silent) {
logging.level = 'silent';
}
const telemetry = new AstroTelemetry({ version: ASTRO_VERSION });
// Special CLI Commands: "add", "docs", "telemetry"
// These commands run before the user's config is parsed, and may have other special
@ -120,19 +116,19 @@ export async function cli(args: string[]) {
switch (cmd) {
case 'add': {
try {
telemetry.record(event.eventCliSession({ cliCommand: cmd }));
telemetry.record(event.eventCliSession(cmd));
const packages = flags._.slice(3) as string[];
return await add(packages, { cwd: root, flags, logging, telemetry });
} catch (err) {
return throwAndExit(err);
return throwAndExit(cmd, err);
}
}
case 'docs': {
try {
telemetry.record(event.eventCliSession({ cliCommand: cmd }));
telemetry.record(event.eventCliSession(cmd));
return await openInBrowser('https://docs.astro.build/');
} catch (err) {
return throwAndExit(err);
return throwAndExit(cmd, err);
}
}
case 'telemetry': {
@ -142,13 +138,13 @@ export async function cli(args: string[]) {
const subcommand = flags._[3]?.toString();
return await telemetryHandler.update(subcommand, { flags, telemetry });
} catch (err) {
return throwAndExit(err);
return throwAndExit(cmd, err);
}
}
}
const { astroConfig, userConfig } = await openConfig({ cwd: root, flags, cmd });
telemetry.record(event.eventCliSession({ cliCommand: cmd }, userConfig, flags));
telemetry.record(event.eventCliSession(cmd, userConfig, flags));
// Common CLI Commands:
// These commands run normally. All commands are assumed to have been handled
@ -159,7 +155,7 @@ export async function cli(args: string[]) {
await devServer(astroConfig, { logging, telemetry });
return await new Promise(() => {}); // lives forever
} catch (err) {
return throwAndExit(err);
return throwAndExit(cmd, err);
}
}
@ -167,7 +163,7 @@ export async function cli(args: string[]) {
try {
return await build(astroConfig, { logging, telemetry });
} catch (err) {
return throwAndExit(err);
return throwAndExit(cmd, err);
}
}
@ -181,21 +177,29 @@ export async function cli(args: string[]) {
const server = await preview(astroConfig, { logging, telemetry });
return await server.closed(); // keep alive until the server is closed
} catch (err) {
return throwAndExit(err);
return throwAndExit(cmd, err);
}
}
}
// No command handler matched! This is unexpected.
throwAndExit(new Error(`Error running ${cmd} -- no command found.`));
throwAndExit(cmd, new Error(`Error running ${cmd} -- no command found.`));
}
/** Display error and exit */
function throwAndExit(err: unknown) {
function throwAndExit(cmd: string, err: unknown) {
let telemetryPromise: Promise<any>;
if (err instanceof z.ZodError) {
console.error(formatConfigErrorMessage(err));
telemetryPromise = telemetry.record(eventConfigError({ cmd, err, isFatal: true }));
} else {
console.error(formatErrorMessage(createSafeError(err)));
const errorWithMetadata = collectErrorMetadata(createSafeError(err));
console.error(formatErrorMessage(errorWithMetadata));
telemetryPromise = telemetry.record(eventError({ cmd, err: errorWithMetadata, isFatal: true }));
}
process.exit(1);
// Wait for the telemetry event to send, then exit. Ignore an error.
telemetryPromise.catch(() => undefined).then(() => process.exit(1));
// Don't wait too long. Timeout the request faster than usual because the user is waiting.
// TODO: Investigate using an AbortController once we drop Node v14 support.
setTimeout(() => process.exit(1), 300);
}

View file

@ -5,10 +5,19 @@ import type { ViteDevServer } from 'vite';
import type { SSRError } from '../@types/astro';
import { codeFrame, createSafeError } from './util.js';
export enum AstroErrorCodes {
// 1xxx: Astro Runtime Errors
UnknownError = 1000,
ConfigError = 1001,
// 2xxx: Astro Compiler Errors
UnknownCompilerError = 2000,
UnknownCompilerCSSError = 2001,
}
export interface ErrorWithMetadata {
[name: string]: any;
message: string;
stack: string;
code?: number;
hint?: string;
id?: string;
frame?: string;

View file

@ -18,7 +18,7 @@ import type { AddressInfo } from 'net';
import os from 'os';
import { ZodError } from 'zod';
import type { AstroConfig } from '../@types/astro';
import { cleanErrorStack, collectErrorMetadata } from './errors.js';
import { cleanErrorStack, collectErrorMetadata, ErrorWithMetadata } from './errors.js';
import { emoji, getLocalAddress, padMultilineString } from './util.js';
const PREFIX_PADDING = 6;
@ -219,8 +219,7 @@ export function formatConfigErrorMessage(err: ZodError) {
)}`;
}
export function formatErrorMessage(_err: Error, args: string[] = []): string {
const err = collectErrorMetadata(_err);
export function formatErrorMessage(err: ErrorWithMetadata, args: string[] = []): string {
args.push(`${bgRed(black(` error `))}${red(bold(padMultilineString(err.message)))}`);
if (err.hint) {
args.push(` ${bold('Hint:')}`);

View file

@ -8,6 +8,9 @@ import type { ErrorPayload } from 'vite';
import type { AstroConfig } from '../@types/astro';
import { removeTrailingForwardSlash } from './path.js';
// process.env.PACKAGE_VERSION is injected when we build and publish the astro package.
export const ASTRO_VERSION = process.env.PACKAGE_VERSION ?? 'development';
/** Returns true if argument is an object of any prototype/class (but not null). */
export function isObject(value: unknown): value is Record<string, any> {
return typeof value === 'object' && value != null;

View file

@ -0,0 +1,75 @@
import { ZodError } from 'zod';
import { AstroErrorCodes, ErrorWithMetadata } from '../core/errors.js';
const EVENT_ERROR = 'ASTRO_CLI_ERROR';
interface ErrorEventPayload {
code: number | undefined;
isFatal: boolean;
plugin?: string | undefined;
cliCommand: string;
anonymousMessageHint?: string | undefined;
}
interface ConfigErrorEventPayload extends ErrorEventPayload {
isConfig: true;
configErrorPaths: string[];
}
/**
* This regex will grab a small snippet at the start of an error message.
* This was designed to stop capturing at the first sign of some non-message
* content like a filename, filepath, or any other code-specific value.
* We also trim this value even further to just a few words.
*
* Our goal is to remove this entirely before v1.0.0 is released, as we work
* to add a proper error code system (see AstroErrorCodes for examples).
*
* TODO(fks): Remove around v1.0.0 release.
*/
const ANONYMIZE_MESSAGE_REGEX = /^(\w| )+/;
function anonymizeErrorMessage(msg: string): string | undefined {
const matchedMessage = msg.match(ANONYMIZE_MESSAGE_REGEX);
if (!matchedMessage || !matchedMessage[0]) {
return undefined;
}
return matchedMessage[0].trim().substring(0, 20);
}
export function eventConfigError({
err,
cmd,
isFatal,
}: {
err: ZodError;
cmd: string;
isFatal: boolean;
}): { eventName: string; payload: ConfigErrorEventPayload }[] {
const payload: ConfigErrorEventPayload = {
code: AstroErrorCodes.ConfigError,
isFatal,
isConfig: true,
cliCommand: cmd,
configErrorPaths: err.issues.map((issue) => issue.path.join('.')),
};
return [{ eventName: EVENT_ERROR, payload }];
}
export function eventError({
cmd,
err,
isFatal,
}: {
err: ErrorWithMetadata;
cmd: string;
isFatal: boolean;
}): { eventName: string; payload: ErrorEventPayload }[] {
const payload: ErrorEventPayload = {
code: err.code || AstroErrorCodes.UnknownError,
plugin: err.plugin,
cliCommand: cmd,
isFatal: isFatal,
anonymousMessageHint: anonymizeErrorMessage(err.message),
};
return [{ eventName: EVENT_ERROR, payload }];
}

View file

@ -1 +1,18 @@
import { AstroTelemetry } from '@astrojs/telemetry';
import { ASTRO_VERSION } from '../core/util.js';
import { createRequire } from 'module';
const require = createRequire(import.meta.url);
function getViteVersion() {
try {
const { version } = require('vite/package.json');
return version;
} catch (e) {}
return undefined;
}
export const telemetry = new AstroTelemetry({ astroVersion: ASTRO_VERSION, viteVersion: getViteVersion() });
export * from './error.js';
export * from './session.js';

View file

@ -4,10 +4,6 @@ const require = createRequire(import.meta.url);
const EVENT_SESSION = 'ASTRO_CLI_SESSION_STARTED';
interface EventCliSession {
cliCommand: string;
}
interface ConfigInfo {
markdownPlugins: string[];
adapter: string | null;
@ -26,23 +22,14 @@ interface ConfigInfo {
};
}
interface EventCliSessionInternal extends EventCliSession {
nodeVersion: string;
viteVersion: string;
interface EventPayload {
cliCommand: string;
config?: ConfigInfo;
configKeys?: string[];
flags?: string[];
optionalIntegrations?: number;
}
function getViteVersion() {
try {
const { version } = require('vite/package.json');
return version;
} catch (e) {}
return undefined;
}
const multiLevelKeys = new Set([
'build',
'markdown',
@ -82,10 +69,10 @@ function configKeys(obj: Record<string, any> | undefined, parentKey: string): st
}
export function eventCliSession(
event: EventCliSession,
cliCommand: string,
userConfig?: AstroUserConfig,
flags?: Record<string, any>
): { eventName: string; payload: EventCliSessionInternal }[] {
): { eventName: string; payload: EventPayload }[] {
// Filter out falsy integrations
const configValues = userConfig
? {
@ -117,13 +104,9 @@ export function eventCliSession(
// Filter out yargs default `_` flag which is the cli command
const cliFlags = flags ? Object.keys(flags).filter((name) => name != '_') : undefined;
const payload: EventCliSessionInternal = {
cliCommand: event.cliCommand,
// Versions
viteVersion: getViteVersion(),
nodeVersion: process.version.replace(/^v?/, ''),
const payload: EventPayload = {
cliCommand,
configKeys: userConfig ? configKeys(userConfig, '') : undefined,
// Config Values
config: configValues,
flags: cliFlags,
};

View file

@ -6,7 +6,7 @@ import type { SSROptions } from '../core/render/dev/index';
import { Readable } from 'stream';
import stripAnsi from 'strip-ansi';
import { call as callEndpoint } from '../core/endpoint/dev/index.js';
import { fixViteErrorMessage } from '../core/errors.js';
import { collectErrorMetadata, fixViteErrorMessage } from '../core/errors.js';
import { error, info, LogOptions, warn } from '../core/logger/core.js';
import * as msg from '../core/messages.js';
import { appendForwardSlash } from '../core/path.js';
@ -320,7 +320,8 @@ async function handleRequest(
}
} catch (_err) {
const err = fixViteErrorMessage(createSafeError(_err), viteServer);
error(logging, null, msg.formatErrorMessage(err));
const errorWithMetadata = collectErrorMetadata(_err);
error(logging, null, msg.formatErrorMessage(errorWithMetadata));
handle500Response(viteServer, origin, req, res, err);
}
}

View file

@ -8,6 +8,7 @@ import { fileURLToPath } from 'url';
import { prependForwardSlash } from '../core/path.js';
import { viteID } from '../core/util.js';
import { transformWithVite } from './styles.js';
import { AstroErrorCodes } from '../core/errors.js';
type CompilationCache = Map<string, CompileResult>;
type CompileResult = TransformResult & { rawCSSDeps: Set<string> };
@ -120,11 +121,19 @@ async function compile({
return null;
}
},
}).catch((err) => {
// throw compiler errors here if encountered
err.code = err.code || AstroErrorCodes.UnknownCompilerError;
throw err;
}).then((result) => {
// throw CSS transform errors here if encountered
if (cssTransformError) {
(cssTransformError as any).code = (cssTransformError as any).code || AstroErrorCodes.UnknownCompilerCSSError;
throw cssTransformError;
}
return result;
});
// throw CSS transform errors here if encountered
if (cssTransformError) throw cssTransformError;
const compileResult: CompileResult = Object.create(transformResult, {
rawCSSDeps: {
value: rawCSSDeps,

View file

@ -97,4 +97,5 @@ describe('Error display', () => {
expect($('h1').text()).to.equal('No mismatch');
});
});
});

View file

@ -1,8 +1,10 @@
import { expect } from 'chai';
import { AstroErrorCodes } from '../dist/core/errors.js';
import * as events from '../dist/events/index.js';
describe('Session event', () => {
describe('top-level', () => {
describe('Events', () => {
describe('eventCliSession()', () => {
it('All top-level keys added', () => {
const config = {
root: 1,
@ -23,9 +25,7 @@ describe('Session event', () => {
);
expect(payload.configKeys).to.deep.equal(expected);
});
});
describe('config.build', () => {
it('configKeys includes format', () => {
const config = {
srcDir: 1,
@ -57,9 +57,7 @@ describe('Session event', () => {
);
expect(payload.config.build.format).to.equal('file');
});
});
describe('config.server', () => {
it('configKeys includes server props', () => {
const config = {
srcDir: 1,
@ -76,9 +74,7 @@ describe('Session event', () => {
);
expect(payload.configKeys).to.deep.equal(['srcDir', 'server', 'server.host', 'server.port']);
});
});
describe('config.markdown', () => {
it('configKeys is deep', () => {
const config = {
publicDir: 1,
@ -128,10 +124,8 @@ describe('Session event', () => {
);
expect(payload.config.markdown.syntaxHighlight).to.equal('shiki');
});
});
describe('config.vite', () => {
it('top-level keys are captured', async () => {
it('top-level vite keys are captured', async () => {
const config = {
root: 'some/thing',
vite: {
@ -376,23 +370,21 @@ describe('Session event', () => {
'vite.worker.plugins',
]);
});
});
it('falsy integrations', () => {
const config = {
srcDir: 1,
integrations: [null, undefined, false],
};
const [{ payload }] = events.eventCliSession(
{
cliCommand: 'dev',
},
config
);
expect(payload.config.integrations.length).to.equal(0);
});
it('falsy integrations', () => {
const config = {
srcDir: 1,
integrations: [null, undefined, false],
};
const [{ payload }] = events.eventCliSession(
{
cliCommand: 'dev',
},
config
);
expect(payload.config.integrations.length).to.equal(0);
});
describe('flags', () => {
it('includes cli flags in payload', () => {
const config = {};
const flags = {
@ -424,4 +416,75 @@ describe('Session event', () => {
]);
});
});
describe('eventConfigError()', () => {
it('returns the expected event and payload', () => {
const [event] = events.eventConfigError({
err: { issues: [{ path: ['a', 'b', 'c'] }, { path: ['d', 'e', 'f'] }] },
cmd: 'COMMAND_NAME',
isFatal: true
});
expect(event).to.deep.equal({
eventName: 'ASTRO_CLI_ERROR',
payload: {
code: AstroErrorCodes.ConfigError,
isFatal: true,
isConfig: true,
cliCommand: 'COMMAND_NAME',
configErrorPaths: ['a.b.c', 'd.e.f'],
}
});
});
});
describe('eventError()', () => {
it('returns the expected event payload with a detailed error object', () => {
const errorWithFullMetadata = new Error('TEST ERROR MESSAGE');
errorWithFullMetadata.code = 1234;
errorWithFullMetadata.plugin = 'TEST PLUGIN';
const [event] = events.eventError({
err: errorWithFullMetadata,
cmd: 'COMMAND_NAME',
isFatal: true
});
expect(event).to.deep.equal({
eventName: 'ASTRO_CLI_ERROR',
payload: {
code: 1234,
plugin: 'TEST PLUGIN',
isFatal: true,
cliCommand: 'COMMAND_NAME',
anonymousMessageHint: 'TEST ERROR MESSAGE',
}
});
});
it('returns the expected event payload with a generic error', () => {
const [event] = events.eventError({
err: new Error('TEST ERROR MESSAGE'),
cmd: 'COMMAND_NAME',
isFatal: false
});
expect(event).to.deep.equal({
eventName: 'ASTRO_CLI_ERROR',
payload: {
code: AstroErrorCodes.UnknownError,
plugin: undefined,
isFatal: false,
cliCommand: 'COMMAND_NAME',
anonymousMessageHint: 'TEST ERROR MESSAGE',
}
});
});
it('properly creates anonymousMessageHint from a basic error message', () => {
const [event] = events.eventError({
err: new Error('TEST ERROR MESSAGE: Sensitive data is "/Users/MYNAME/foo.astro"'),
cmd: 'COMMAND_NAME',
isFatal: true
});
expect(event.payload.anonymousMessageHint).to.equal('TEST ERROR MESSAGE');
});
});
});

View file

@ -14,6 +14,7 @@ import { TEMPLATES } from './templates.js';
function wait(ms: number) {
return new Promise((resolve) => setTimeout(resolve, ms));
}
function logAndWait(message: string, ms = 100) {
console.log(message);
return wait(ms);

View file

@ -7,7 +7,7 @@ import { post } from './post.js';
import { getProjectInfo, ProjectInfo } from './project-info.js';
import { getSystemInfo, SystemInfo } from './system-info.js';
export type AstroTelemetryOptions = { version: string };
export type AstroTelemetryOptions = { astroVersion: string; viteVersion: string };
export type TelemetryEvent = { eventName: string; payload: Record<string, any> };
interface EventContext {
anonymousId: string;
@ -25,7 +25,10 @@ export class AstroTelemetry {
private debug = debug('astro:telemetry');
private get astroVersion() {
return this.opts.version;
return this.opts.astroVersion;
}
private get viteVersion() {
return this.opts.viteVersion;
}
private get ASTRO_TELEMETRY_DISABLED() {
return process.env.ASTRO_TELEMETRY_DISABLED;
@ -131,7 +134,7 @@ export class AstroTelemetry {
}
const meta: EventMeta = {
...getSystemInfo(this.astroVersion),
...getSystemInfo({ astroVersion: this.astroVersion, viteVersion: this.viteVersion }),
isGit: this.anonymousProjectInfo.isGit,
};
@ -141,6 +144,12 @@ export class AstroTelemetry {
anonymousSessionId: this.anonymousSessionId,
};
// Every CI session also creates a new user, which blows up telemetry.
// To solve this, we track all CI runs under a single "CI" anonymousId.
if (meta.isCI) {
context.anonymousId = `CI.${meta.ciName || 'UNKNOWN'}`;
}
return post({
context,
meta,

View file

@ -31,6 +31,9 @@ export type SystemInfo = {
systemPlatform: NodeJS.Platform;
systemRelease: string;
systemArchitecture: string;
astroVersion: string;
nodeVersion: string;
viteVersion: string;
cpuCount: number;
cpuModel: string | null;
cpuSpeed: number | null;
@ -39,18 +42,21 @@ export type SystemInfo = {
isWSL: boolean;
isCI: boolean;
ciName: string | null;
astroVersion: string;
};
let meta: SystemInfo | undefined;
export function getSystemInfo(astroVersion: string): SystemInfo {
export function getSystemInfo(versions: {viteVersion: string, astroVersion: string}): SystemInfo {
if (meta) {
return meta;
}
const cpus = os.cpus() || [];
meta = {
// Version information
nodeVersion: process.version.replace(/^v?/, ''),
viteVersion: versions.viteVersion,
astroVersion: versions.astroVersion,
// Software information
systemPlatform: os.platform(),
systemRelease: os.release(),
@ -65,7 +71,6 @@ export function getSystemInfo(astroVersion: string): SystemInfo {
isWSL,
isCI,
ciName,
astroVersion,
};
return meta!;

View file

@ -1,5 +1,5 @@
export { pathToPosix } from './lib/utils';
export { AbortController, AbortSignal, alert, atob, Blob, btoa, ByteLengthQueuingStrategy, cancelAnimationFrame, cancelIdleCallback, CanvasRenderingContext2D, CharacterData, clearTimeout, Comment, CountQueuingStrategy, CSSStyleSheet, CustomElementRegistry, CustomEvent, Document, DocumentFragment, DOMException, Element, Event, EventTarget, fetch, File, FormData, Headers, HTMLBodyElement, HTMLCanvasElement, HTMLDivElement, HTMLDocument, HTMLElement, HTMLHeadElement, HTMLHtmlElement, HTMLImageElement, HTMLSpanElement, HTMLStyleElement, HTMLTemplateElement, HTMLUnknownElement, Image, ImageData, IntersectionObserver, MediaQueryList, MutationObserver, Node, NodeFilter, NodeIterator, OffscreenCanvas, ReadableByteStreamController, ReadableStream, ReadableStreamBYOBReader, ReadableStreamBYOBRequest, ReadableStreamDefaultController, ReadableStreamDefaultReader, Request, requestAnimationFrame, requestIdleCallback, ResizeObserver, Response, setTimeout, ShadowRoot, structuredClone, StyleSheet, Text, TransformStream, TreeWalker, URLPattern, Window, WritableStream, WritableStreamDefaultController, WritableStreamDefaultWriter } from './mod.js';
export { AbortController, AbortSignal, alert, atob, Blob, btoa, ByteLengthQueuingStrategy, cancelAnimationFrame, cancelIdleCallback, CanvasRenderingContext2D, CharacterData, clearTimeout, Comment, CountQueuingStrategy, CSSStyleSheet, CustomElementRegistry, CustomEvent, Document, DocumentFragment, DOMException, Element, Event, EventTarget, fetch, File, FormData, Headers, HTMLBodyElement, HTMLCanvasElement, HTMLDivElement, HTMLDocument, HTMLElement, HTMLHeadElement, HTMLHtmlElement, HTMLImageElement, HTMLSpanElement, HTMLStyleElement, HTMLTemplateElement, HTMLUnknownElement, Image, ImageData, IntersectionObserver, MediaQueryList, MutationObserver, Node, NodeFilter, NodeIterator, OffscreenCanvas, ReadableByteStreamController, ReadableStream, ReadableStreamBYOBReader, ReadableStreamBYOBRequest, ReadableStreamDefaultController, ReadableStreamDefaultReader, Request, requestAnimationFrame, requestIdleCallback, ResizeObserver, Response, setTimeout, ShadowRoot, structuredClone, StyleSheet, Text, TransformStream, TreeWalker, URLPattern, Window, WritableStream, WritableStreamDefaultController, WritableStreamDefaultWriter, } from './mod.js';
export declare const polyfill: {
(target: any, options?: PolyfillOptions): any;
internals(target: any, name: string): any;