Fix e2e flaky tests again (#7129)

This commit is contained in:
Bjorn Lu 2023-05-19 22:46:31 +08:00 committed by GitHub
parent 7341d9a288
commit ed4aff0cb9
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
18 changed files with 98 additions and 72 deletions

View file

@ -15,7 +15,7 @@ test.afterAll(async () => {
test.describe('Client only', () => {
test('React counter', async ({ astro, page }) => {
await page.goto('/');
await page.goto(astro.resolveUrl('/'));
const counter = await page.locator('#react-counter');
await expect(counter, 'component is visible').toBeVisible();
@ -33,7 +33,7 @@ test.describe('Client only', () => {
});
test('Preact counter', async ({ astro, page }) => {
await page.goto('/');
await page.goto(astro.resolveUrl('/'));
const counter = await page.locator('#preact-counter');
await expect(counter, 'component is visible').toBeVisible();
@ -51,7 +51,7 @@ test.describe('Client only', () => {
});
test('Solid counter', async ({ astro, page }) => {
await page.goto('/');
await page.goto(astro.resolveUrl('/'));
const counter = await page.locator('#solid-counter');
await expect(counter, 'component is visible').toBeVisible();
@ -69,7 +69,7 @@ test.describe('Client only', () => {
});
test('Vue counter', async ({ astro, page }) => {
await page.goto('/');
await page.goto(astro.resolveUrl('/'));
const counter = await page.locator('#vue-counter');
await expect(counter, 'component is visible').toBeVisible();
@ -87,7 +87,7 @@ test.describe('Client only', () => {
});
test('Svelte counter', async ({ astro, page }) => {
await page.goto('/');
await page.goto(astro.resolveUrl('/'));
const counter = await page.locator('#svelte-counter');
await expect(counter, 'component is visible').toBeVisible();

View file

@ -18,7 +18,7 @@ test.afterAll(async ({ astro }) => {
test.describe('Error display', () => {
test('detect syntax errors in template', async ({ page, astro }) => {
await page.goto(astro.resolveUrl('/astro-syntax-error'));
await page.goto(astro.resolveUrl('/astro-syntax-error'), { waitUntil: 'networkidle' });
const message = (await getErrorOverlayContent(page)).message;
expect(message).toMatch('Unexpected "}"');
@ -37,7 +37,7 @@ test.describe('Error display', () => {
});
test('shows useful error when frontmatter import is not found', async ({ page, astro }) => {
await page.goto(astro.resolveUrl('/import-not-found'));
await page.goto(astro.resolveUrl('/import-not-found'), { waitUntil: 'networkidle' });
const message = (await getErrorOverlayContent(page)).message;
expect(message).toMatch('Could not import ../abc.astro');
@ -53,7 +53,7 @@ test.describe('Error display', () => {
});
test('shows correct file path when a page has an error', async ({ page, astro }) => {
await page.goto(astro.resolveUrl('/import-not-found'));
await page.goto(astro.resolveUrl('/import-not-found'), { waitUntil: 'networkidle' });
const { fileLocation, absoluteFileLocation } = await getErrorOverlayContent(page);
const absoluteFileUrl = 'file://' + absoluteFileLocation.replace(/:\d+:\d+$/, '');
@ -64,7 +64,7 @@ test.describe('Error display', () => {
});
test('shows correct file path when a component has an error', async ({ page, astro }) => {
await page.goto(astro.resolveUrl('/preact-runtime-error'));
await page.goto(astro.resolveUrl('/preact-runtime-error'), { waitUntil: 'networkidle' });
const { fileLocation, absoluteFileLocation } = await getErrorOverlayContent(page);
const absoluteFileUrl = 'file://' + absoluteFileLocation.replace(/:\d+:\d+$/, '');
@ -75,7 +75,7 @@ test.describe('Error display', () => {
});
test('framework errors recover when fixed', async ({ page, astro }) => {
await page.goto(astro.resolveUrl('/svelte-syntax-error'));
await page.goto(astro.resolveUrl('/svelte-syntax-error'), { waitUntil: 'networkidle' });
const message = (await getErrorOverlayContent(page)).message;
expect(message).toMatch('</div> attempted to close an element that was not open');

View file

@ -17,7 +17,7 @@ test.afterEach(async () => {
test.describe('Hydration race', () => {
test('Islands inside of slots hydrate', async ({ page, astro }) => {
await page.goto('/slot');
await page.goto(astro.resolveUrl('/slot'));
const html = await page.content();

View file

@ -15,7 +15,7 @@ test.afterAll(async () => {
test.skip('Multiple frameworks', () => {
test.skip('React counter', async ({ page }) => {
await page.goto('/');
await page.goto(astro.resolveUrl('/'));
const counter = page.locator('#react-counter');
await expect(counter, 'component is visible').toBeVisible();
@ -30,7 +30,7 @@ test.skip('Multiple frameworks', () => {
});
test('Preact counter', async ({ page }) => {
await page.goto('/');
await page.goto(astro.resolveUrl('/'));
const counter = page.locator('#preact-counter');
await expect(counter, 'component is visible').toBeVisible();
@ -45,7 +45,7 @@ test.skip('Multiple frameworks', () => {
});
test.skip('Solid counter', async ({ page }) => {
await page.goto('/');
await page.goto(astro.resolveUrl('/'));
const counter = page.locator('#solid-counter');
await expect(counter, 'component is visible').toBeVisible();
@ -60,7 +60,7 @@ test.skip('Multiple frameworks', () => {
});
test('Vue counter', async ({ page }) => {
await page.goto('/');
await page.goto(astro.resolveUrl('/'));
const counter = page.locator('#vue-counter');
await expect(counter, 'component is visible').toBeVisible();
@ -75,7 +75,7 @@ test.skip('Multiple frameworks', () => {
});
test('Svelte counter', async ({ page }) => {
await page.goto('/');
await page.goto(astro.resolveUrl('/'));
const counter = page.locator('#svelte-counter');
await expect(counter, 'component is visible').toBeVisible();
@ -90,7 +90,7 @@ test.skip('Multiple frameworks', () => {
});
test('Astro components', async ({ page }) => {
await page.goto('/');
await page.goto(astro.resolveUrl('/'));
const aComponent = page.locator('#astro-a');
await expect(aComponent, 'component is visible').toBeVisible();
@ -103,7 +103,7 @@ test.skip('Multiple frameworks', () => {
test.describe('HMR', () => {
test('Page template', async ({ astro, page }) => {
await page.goto('/');
await page.goto(astro.resolveUrl('/'));
const slot = page.locator('#preact-counter + .counter-message');
await expect(slot, 'initial slot content').toHaveText('Hello Preact!');
@ -116,7 +116,7 @@ test.skip('Multiple frameworks', () => {
});
test('React component', async ({ astro, page }) => {
await page.goto('/');
await page.goto(astro.resolveUrl('/'));
const count = page.locator('#react-counter pre');
await expect(count, 'initial count updated to 0').toHaveText('0');
@ -129,7 +129,7 @@ test.skip('Multiple frameworks', () => {
});
test('Preact component', async ({ astro, page }) => {
await page.goto('/');
await page.goto(astro.resolveUrl('/'));
const count = page.locator('#preact-counter pre');
await expect(count, 'initial count updated to 0').toHaveText('0');
@ -142,7 +142,7 @@ test.skip('Multiple frameworks', () => {
});
test('Solid component', async ({ astro, page }) => {
await page.goto('/');
await page.goto(astro.resolveUrl('/'));
const count = page.locator('#solid-counter pre');
await expect(count, 'initial count updated to 0').toHaveText('0');
@ -157,7 +157,7 @@ test.skip('Multiple frameworks', () => {
// TODO: re-enable this test when #3559 is fixed
// https://github.com/withastro/astro/issues/3559
test.skip('Vue component', async ({ astro, page }) => {
await page.goto('/');
await page.goto(astro.resolveUrl('/'));
const count = page.locator('#vue-counter pre');
await expect(count, 'initial count updated to 0').toHaveText('0');
@ -172,7 +172,7 @@ test.skip('Multiple frameworks', () => {
// TODO: track down a reliability issue in this test
// It seems to lost connection to the vite server in CI
test.skip('Svelte component', async ({ astro, page }) => {
await page.goto('/');
await page.goto(astro.resolveUrl('/'));
const count = page.locator('#svelte-counter pre');
await expect(count, 'initial count is 0').toHaveText('0');

View file

@ -16,8 +16,8 @@ test.afterAll(async () => {
});
test.describe('Hydrating namespaced components', () => {
test('Preact Component', async ({ page }) => {
await page.goto('/');
test('Preact Component', async ({ astro, page }) => {
await page.goto(astro.resolveUrl('/'));
// Counter declared with: <ns.components.PreactCounter id="preact-counter-namespace" client:load>
const namespacedCounter = page.locator('#preact-counter-namespace');
@ -54,8 +54,8 @@ test.describe('Hydrating namespaced components', () => {
await expect(namedCount, 'count incremented by 1').toHaveText('1');
});
test('MDX', async ({ page }) => {
await page.goto('/mdx');
test('MDX', async ({ astro, page }) => {
await page.goto(astro.resolveUrl('/mdx'));
// Counter declared with: <ns.components.PreactCounter id="preact-counter-namespace" client:load>
const namespacedCounter = page.locator('#preact-counter-namespace');

View file

@ -15,7 +15,7 @@ test.afterAll(async () => {
test.describe('Nested Frameworks in Preact', () => {
test('React counter', async ({ astro, page }) => {
await page.goto('/');
await page.goto(astro.resolveUrl('/'));
const counter = page.locator('#react-counter');
await expect(counter, 'component is visible').toBeVisible();
@ -32,7 +32,7 @@ test.describe('Nested Frameworks in Preact', () => {
});
test('Preact counter', async ({ astro, page }) => {
await page.goto('/');
await page.goto(astro.resolveUrl('/'));
const counter = page.locator('#preact-counter');
await expect(counter, 'component is visible').toBeVisible();
@ -49,7 +49,7 @@ test.describe('Nested Frameworks in Preact', () => {
});
test('Solid counter', async ({ astro, page }) => {
await page.goto('/');
await page.goto(astro.resolveUrl('/'));
const counter = page.locator('#solid-counter');
await expect(counter, 'component is visible').toBeVisible();
@ -66,7 +66,7 @@ test.describe('Nested Frameworks in Preact', () => {
});
test('Vue counter', async ({ astro, page }) => {
await page.goto('/');
await page.goto(astro.resolveUrl('/'));
const counter = page.locator('#vue-counter');
await expect(counter, 'component is visible').toBeVisible();
@ -83,7 +83,7 @@ test.describe('Nested Frameworks in Preact', () => {
});
test('Svelte counter', async ({ astro, page }) => {
await page.goto('/');
await page.goto(astro.resolveUrl('/'));
const counter = page.locator('#svelte-counter');
await expect(counter, 'component is visible').toBeVisible();

View file

@ -15,7 +15,7 @@ test.afterAll(async () => {
test.describe('Nested Frameworks in React', () => {
test('React counter', async ({ astro, page }) => {
await page.goto('/');
await page.goto(astro.resolveUrl('/'));
const counter = page.locator('#react-counter');
await expect(counter, 'component is visible').toBeVisible();
@ -32,7 +32,7 @@ test.describe('Nested Frameworks in React', () => {
});
test('Preact counter', async ({ astro, page }) => {
await page.goto('/');
await page.goto(astro.resolveUrl('/'));
const counter = page.locator('#preact-counter');
await expect(counter, 'component is visible').toBeVisible();
@ -49,7 +49,7 @@ test.describe('Nested Frameworks in React', () => {
});
test('Solid counter', async ({ astro, page }) => {
await page.goto('/');
await page.goto(astro.resolveUrl('/'));
const counter = page.locator('#solid-counter');
await expect(counter, 'component is visible').toBeVisible();
@ -66,7 +66,7 @@ test.describe('Nested Frameworks in React', () => {
});
test('Vue counter', async ({ astro, page }) => {
await page.goto('/');
await page.goto(astro.resolveUrl('/'));
const counter = page.locator('#vue-counter');
await expect(counter, 'component is visible').toBeVisible();
@ -83,7 +83,7 @@ test.describe('Nested Frameworks in React', () => {
});
test('Svelte counter', async ({ astro, page }) => {
await page.goto('/');
await page.goto(astro.resolveUrl('/'));
const counter = page.locator('#svelte-counter');
await expect(counter, 'component is visible').toBeVisible();

View file

@ -15,7 +15,7 @@ test.afterAll(async () => {
test.describe('Nested Frameworks in Solid', () => {
test('React counter', async ({ astro, page }) => {
await page.goto('/');
await page.goto(astro.resolveUrl('/'));
const counter = page.locator('#react-counter');
await expect(counter, 'component is visible').toBeVisible();
@ -32,7 +32,7 @@ test.describe('Nested Frameworks in Solid', () => {
});
test('Preact counter', async ({ astro, page }) => {
await page.goto('/');
await page.goto(astro.resolveUrl('/'));
const counter = page.locator('#preact-counter');
await expect(counter, 'component is visible').toBeVisible();
@ -49,7 +49,7 @@ test.describe('Nested Frameworks in Solid', () => {
});
test('Solid counter', async ({ astro, page }) => {
await page.goto('/');
await page.goto(astro.resolveUrl('/'));
const counter = page.locator('#solid-counter');
await expect(counter, 'component is visible').toBeVisible();
@ -66,7 +66,7 @@ test.describe('Nested Frameworks in Solid', () => {
});
test('Vue counter', async ({ astro, page }) => {
await page.goto('/');
await page.goto(astro.resolveUrl('/'));
const counter = page.locator('#vue-counter');
await expect(counter, 'component is visible').toBeVisible();
@ -83,7 +83,7 @@ test.describe('Nested Frameworks in Solid', () => {
});
test('Svelte counter', async ({ astro, page }) => {
await page.goto('/');
await page.goto(astro.resolveUrl('/'));
const counter = page.locator('#svelte-counter');
await expect(counter, 'component is visible').toBeVisible();

View file

@ -15,7 +15,7 @@ test.afterAll(async () => {
test.describe('Nested Frameworks in Svelte', () => {
test('React counter', async ({ astro, page }) => {
await page.goto('/');
await page.goto(astro.resolveUrl('/'));
const counter = page.locator('#react-counter');
await expect(counter, 'component is visible').toBeVisible();
@ -32,7 +32,7 @@ test.describe('Nested Frameworks in Svelte', () => {
});
test('Preact counter', async ({ astro, page }) => {
await page.goto('/');
await page.goto(astro.resolveUrl('/'));
const counter = page.locator('#preact-counter');
await expect(counter, 'component is visible').toBeVisible();
@ -49,7 +49,7 @@ test.describe('Nested Frameworks in Svelte', () => {
});
test('Solid counter', async ({ astro, page }) => {
await page.goto('/');
await page.goto(astro.resolveUrl('/'));
const counter = page.locator('#solid-counter');
await expect(counter, 'component is visible').toBeVisible();
@ -66,7 +66,7 @@ test.describe('Nested Frameworks in Svelte', () => {
});
test('Vue counter', async ({ astro, page }) => {
await page.goto('/');
await page.goto(astro.resolveUrl('/'));
const counter = page.locator('#vue-counter');
await expect(counter, 'component is visible').toBeVisible();
@ -83,7 +83,7 @@ test.describe('Nested Frameworks in Svelte', () => {
});
test('Svelte counter', async ({ astro, page }) => {
await page.goto('/');
await page.goto(astro.resolveUrl('/'));
const counter = page.locator('#svelte-counter');
await expect(counter, 'component is visible').toBeVisible();

View file

@ -15,7 +15,7 @@ test.afterAll(async () => {
test.describe('Nested Frameworks in Vue', () => {
test('React counter', async ({ astro, page }) => {
await page.goto('/');
await page.goto(astro.resolveUrl('/'));
const counter = page.locator('#react-counter');
await expect(counter, 'component is visible').toBeVisible();
@ -32,7 +32,7 @@ test.describe('Nested Frameworks in Vue', () => {
});
test('Preact counter', async ({ astro, page }) => {
await page.goto('/');
await page.goto(astro.resolveUrl('/'));
const counter = page.locator('#preact-counter');
await expect(counter, 'component is visible').toBeVisible();
@ -49,7 +49,7 @@ test.describe('Nested Frameworks in Vue', () => {
});
test('Solid counter', async ({ astro, page }) => {
await page.goto('/');
await page.goto(astro.resolveUrl('/'));
const counter = page.locator('#solid-counter');
await expect(counter, 'component is visible').toBeVisible();
@ -66,7 +66,7 @@ test.describe('Nested Frameworks in Vue', () => {
});
test('Vue counter', async ({ astro, page }) => {
await page.goto('/');
await page.goto(astro.resolveUrl('/'));
const counter = page.locator('#vue-counter');
await expect(counter, 'component is visible').toBeVisible();
@ -83,7 +83,7 @@ test.describe('Nested Frameworks in Vue', () => {
});
test('Svelte counter', async ({ astro, page }) => {
await page.goto('/');
await page.goto(astro.resolveUrl('/'));
const counter = page.locator('#svelte-counter');
await expect(counter, 'component is visible').toBeVisible();

View file

@ -20,7 +20,7 @@ test.afterAll(async () => {
test.describe('Recursive Nested Frameworks', () => {
test('React counter', async ({ astro, page }) => {
await page.goto('/');
await page.goto(astro.resolveUrl('/'));
const counter = page.locator('#react-counter');
await expect(counter, 'component is visible').toBeVisible();
@ -37,7 +37,7 @@ test.describe('Recursive Nested Frameworks', () => {
});
test('Preact counter', async ({ astro, page }) => {
await page.goto('/');
await page.goto(astro.resolveUrl('/'));
const counter = page.locator('#preact-counter');
await expect(counter, 'component is visible').toBeVisible();
@ -54,7 +54,7 @@ test.describe('Recursive Nested Frameworks', () => {
});
test('Solid counter', async ({ astro, page }) => {
await page.goto('/');
await page.goto(astro.resolveUrl('/'));
const counter = page.locator('#solid-counter');
await expect(counter, 'component is visible').toBeVisible();
@ -71,7 +71,7 @@ test.describe('Recursive Nested Frameworks', () => {
});
test('Vue counter', async ({ astro, page }) => {
await page.goto('/');
await page.goto(astro.resolveUrl('/'));
const counter = page.locator('#vue-counter');
await expect(counter, 'component is visible').toBeVisible();
@ -88,7 +88,7 @@ test.describe('Recursive Nested Frameworks', () => {
});
test('Svelte counter', async ({ astro, page }) => {
await page.goto('/');
await page.goto(astro.resolveUrl('/'));
const counter = page.locator('#svelte-counter');
await expect(counter, 'component is visible').toBeVisible();

View file

@ -17,7 +17,7 @@ test.afterAll(async () => {
test.describe('Passing JS into client components', () => {
test('Complex nested objects', async ({ astro, page }) => {
await page.goto('/');
await page.goto(astro.resolveUrl('/'));
const nestedDate = await page.locator('#nested-date');
await expect(nestedDate, 'component is visible').toBeVisible();
@ -32,8 +32,8 @@ test.describe('Passing JS into client components', () => {
await expect(regExpValue).toHaveText('ok');
});
test('BigInts', async ({ page }) => {
await page.goto('/');
test('BigInts', async ({ astro, page }) => {
await page.goto(astro.resolveUrl('/'));
const bigIntType = await page.locator('#bigint-type');
await expect(bigIntType, 'is visible').toBeVisible();
@ -44,8 +44,8 @@ test.describe('Passing JS into client components', () => {
await expect(bigIntValue).toHaveText('11');
});
test('Arrays that look like the serialization format', async ({ page }) => {
await page.goto('/');
test('Arrays that look like the serialization format', async ({ astro, page }) => {
await page.goto(astro.resolveUrl('/'));
const arrType = await page.locator('#arr-type');
await expect(arrType, 'is visible').toBeVisible();
@ -56,8 +56,8 @@ test.describe('Passing JS into client components', () => {
await expect(arrValue).toHaveText('0,foo');
});
test('Maps and Sets', async ({ page }) => {
await page.goto('/');
test('Maps and Sets', async ({ astro, page }) => {
await page.goto(astro.resolveUrl('/'));
const mapType = page.locator('#map-type');
await expect(mapType, 'is visible').toBeVisible();

View file

@ -1,5 +1,5 @@
import { expect } from '@playwright/test';
import { testFactory, waitForHydrate } from './test-utils.js';
import { scrollToElement, testFactory, waitForHydrate } from './test-utils.js';
export function prepareTestFactory(opts) {
const test = testFactory(opts);
@ -69,7 +69,11 @@ export function prepareTestFactory(opts) {
// Make sure the component is on screen to trigger hydration
const counter = page.locator('#client-visible');
await counter.scrollIntoViewIfNeeded();
// NOTE: Use custom implementation instead of `counter.scrollIntoViewIfNeeded`
// as Playwright's function doesn't take into account of `counter` being hydrated
// and losing the original DOM reference.
await scrollToElement(counter);
await expect(counter, 'component is visible').toBeVisible();
const count = counter.locator('pre');

View file

@ -15,7 +15,7 @@ test.afterAll(async () => {
test.describe('Circular imports with Solid', () => {
test('Context', async ({ astro, page }) => {
await page.goto('/');
await page.goto(astro.resolveUrl('/'));
const wrapper = page.locator('#context');
await expect(wrapper, 'context should not be duplicated').toHaveText('fr');

View file

@ -15,7 +15,7 @@ test.afterAll(async () => {
test.describe('Recursive elements with Solid', () => {
test('Counter', async ({ astro, page }) => {
await page.goto('/');
await page.goto(astro.resolveUrl('/'));
const wrapper = page.locator('#case1');
await expect(wrapper, 'component is visible').toBeVisible();

View file

@ -1,8 +1,20 @@
import fs from 'fs/promises';
import path from 'path';
import { test as testBase, expect } from '@playwright/test';
import { loadFixture as baseLoadFixture } from '../test/test-utils.js';
export const isWindows = process.platform === 'win32';
// Get all test files in directory, assign unique port for each of them so they don't conflict
const testFiles = await fs.readdir(new URL('.', import.meta.url));
const testFileToPort = new Map();
for (let i = 0; i < testFiles.length; i++) {
const file = testFiles[i];
if (file.endsWith('.test.js')) {
testFileToPort.set(file.slice(0, -8), 4000 + i);
}
}
export function loadFixture(inlineConfig) {
if (!inlineConfig || !inlineConfig.root)
throw new Error("Must provide { root: './fixtures/...' }");
@ -12,6 +24,9 @@ export function loadFixture(inlineConfig) {
return baseLoadFixture({
...inlineConfig,
root: new URL(inlineConfig.root, import.meta.url).toString(),
server: {
port: testFileToPort.get(path.basename(inlineConfig.root)),
},
});
}
@ -74,3 +89,13 @@ export async function waitForHydrate(page, el) {
`astro-island[uid="${astroIslandId}"]`
);
}
/**
* Scroll to element manually without making sure the `el` is stable
* @param {import('@playwright/test').Locator} el
*/
export async function scrollToElement(el) {
await el.evaluate((node) => {
node.scrollIntoView({ behavior: 'auto' });
});
}

View file

@ -24,9 +24,6 @@ const config = {
use: {
/* Maximum time each action such as `click()` can take. Defaults to 0 (no limit). */
actionTimeout: 0,
/* Base URL to use in actions like `await page.goto('/')`. */
baseURL: process.env.PLAYWRIGHT_TEST_BASE_URL || 'http://localhost:3000',
/* Collect trace when retrying the failed test. See https://playwright.dev/docs/trace-viewer */
trace: 'on-first-retry',
},

View file

@ -151,7 +151,7 @@ export async function loadFixture(inlineConfig) {
};
const resolveUrl = (url) =>
`http://${config.server.host}:${config.server.port}${url.replace(/^\/?/, '/')}`;
`http://${config.server.host || 'localhost'}:${config.server.port}${url.replace(/^\/?/, '/')}`;
// A map of files that have been edited.
let fileEdits = new Map();