Adding E2E testing with Playwright (#3349)

* adding Tailwind E2E tests with Playwright

* package.json updates

* adding e2e tests to CI workflow

* using e2e for dev tests, mocha for build tests

* refactor: sharing test-utils helpers

* chore: update lockfile

* Adding contributing docs

* Revert "refactor: sharing test-utils helpers"

This reverts commit 48496f43bc99eab30747baf6e83041ba4932e786.

* refactor: simpler solution to resolving e2e test fixtures

* chore: updating lockfile

* refactor: cleaning up how URLs are resolved in e2e tests

* install playwright deps in CI

* trying pnpm playwright install to fix version mismatch

.github/workflows/ci.yml

@@ -153,6 +153,48 @@ jobs:
       - name: Test
         run: pnpm run test
 
+  e2e:
+    name: 'E2E: ${{ matrix.os }} (node@${{ matrix.node_version }})'
+    runs-on: ${{ matrix.os }}
+    env:
+      ASTRO_TELEMETRY_DISABLED: true
+    strategy:
+      matrix:
+        os: [ubuntu-latest]
+        node_version: [14, 16]
+        include:
+          - os: windows-latest
+            node_version: 16
+          - os: macos-latest
+            node_version: 16
+      fail-fast: false
+    needs:
+      - build
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: Setup PNPM
+        uses: pnpm/action-setup@v2.2.1
+
+      - name: Setup node@${{ matrix.node_version }}
+        uses: actions/setup-node@v3
+        with:
+          node-version: ${{ matrix.node_version }}
+          cache: 'pnpm'
+
+      - name: Download Build Artifacts
+        uses: actions/download-artifact@v3
+
+      - name: Extract Artifacts
+        run: ./.github/extract-artifacts.sh
+
+      - name: Install dependencies
+        run: pnpm install
+
+      - name: Test
+        run: pnpm run test:e2e
+
   smoke:
     name: 'Test (Smoke) ${{ matrix.os }}'
     runs-on: ${{ matrix.os }}

CONTRIBUTING.md

@@ -59,6 +59,24 @@ pnpm run test
 pnpm run test:match "$STRING_MATCH"
 ```
 
+#### E2E tests
+
+Certain features, like HMR and client hydration, need end-to-end tests to verify functionality in the dev server. [Playwright](https://playwright.dev/) is used to test against the dev server.
+
+```shell
+# run this in the top-level project root to run all E2E tests
+pnpm run test:e2e
+# run only a few tests, great for working on a single feature
+# (example - `pnpm run test:e2e:match "Tailwind CSS" runs `tailwindcss.test.js`)
+pnpm run test:e2e:match "$STRING_MATCH"
+```
+
+**When should you add E2E tests?**
+
+Any tests for `astro build` output should use the main `mocha` tests rather than E2E - these tests will run faster than having Playwright start the `astro preview` server.
+
+If a test needs to validate what happens on the page after it's loading in the browser, that's a perfect use for E2E dev server tests, i.e. to verify that hot-module reloading works in `astro dev` or that components were client hydrated and are interactive.
+
 ### Other useful commands
 
 ```shell

package.json

@@ -18,6 +18,7 @@
     "test:templates": "turbo run test --filter=create-astro --concurrency=1",
     "test:smoke": "node scripts/smoke/index.js",
     "test:vite-ci": "turbo run test --no-deps --scope=astro --concurrency=1",
+    "test:e2e": "cd packages/astro && pnpm playwright install && pnpm run test:e2e",
     "benchmark": "turbo run benchmark --scope=astro",
     "lint": "eslint \"packages/**/*.ts\"",
     "format": "prettier -w .",

packages/astro/e2e/fixtures/tailwindcss/astro.config.mjs

@@ -0,0 +1,12 @@
+import { defineConfig } from 'astro/config';
+import tailwind from '@astrojs/tailwind';
+
+// https://astro.build/config
+export default defineConfig({
+	integrations: [tailwind()],
+	vite: {
+		build: {
+			assetsInlineLimit: 0,
+		},
+	},
+});

packages/astro/e2e/fixtures/tailwindcss/package.json

@@ -0,0 +1,9 @@
+{
+  "name": "@test/e2e-tailwindcss",
+  "version": "0.0.0",
+  "private": true,
+  "dependencies": {
+    "astro": "workspace:*",
+    "@astrojs/tailwind": "workspace:*"
+  }
+}

packages/astro/e2e/fixtures/tailwindcss/postcss.config.js

@@ -0,0 +1,9 @@
+const path = require('path');
+module.exports = {
+	plugins: {
+		tailwindcss: {
+			config: path.join(__dirname, 'tailwind.config.js'), // update this if your path differs!
+		},
+		autoprefixer: {}
+	},
+};

packages/astro/e2e/fixtures/tailwindcss/src/components/Button.astro

@@ -0,0 +1,10 @@
+---
+let { type = 'button' } = Astro.props;
+---
+
+<button
+	class="py-2 px-4 lg:py-3 lg:px-5 bg-purple-600 text-white font-[900] rounded-lg shadow-md hover:bg-purple-700 focus:outline-none focus:ring-2 focus:ring-purple-400 focus:ring-opacity-75"
+	{type}
+>
+	<slot />
+</button>

packages/astro/e2e/fixtures/tailwindcss/src/components/Complex.astro

@@ -0,0 +1 @@
+<div id="complex" class="w-10/12 2xl:w-[80%]"></div>

packages/astro/e2e/fixtures/tailwindcss/src/pages/index.astro

@@ -0,0 +1,18 @@
+---
+// Component Imports
+import Button from '../components/Button.astro';
+import Complex from '../components/Complex.astro';
+---
+
+<html lang="en">
+	<head>
+		<meta charset="UTF-8" />
+		<link rel="icon" type="image/x-icon" href="/favicon.ico" />
+		<title>Astro + TailwindCSS</title>
+	</head>
+
+	<body class="bg-dawn text-midnight">
+		<Button>I’m a Tailwind Button!</Button>
+		<Complex />
+	</body>
+</html>

packages/astro/e2e/fixtures/tailwindcss/src/pages/markdown-page.md

@@ -0,0 +1,11 @@
+---
+title: "Markdown + Tailwind"
+setup: |
+    import Button from '../components/Button.astro';
+    import Complex from '../components/Complex.astro';
+---
+
+<div class="grid place-items-center h-screen content-center">
+    <Button>Tailwind Button in Markdown!</Button>
+    <Complex />
+</div>

packages/astro/e2e/fixtures/tailwindcss/tailwind.config.js

@@ -0,0 +1,14 @@
+const path = require('path');
+
+module.exports = {
+	content: [path.join(__dirname, 'src/**/*.{astro,html,js,jsx,md,svelte,ts,tsx,vue}')],
+	theme: {
+		extend: {
+			colors: {
+				dawn: '#f3e9fa',
+				dusk: '#514375',
+				midnight: '#31274a',
+			}
+		}
+	}
+};

packages/astro/e2e/tailwindcss.test.js

@@ -0,0 +1,51 @@
+import { test as base, expect } from '@playwright/test';
+import { loadFixture } from './test-utils.js';
+
+const test = base.extend({
+	astro: async ({}, use) => {
+		const fixture = await loadFixture({ root: './fixtures/tailwindcss/' });
+		await use(fixture);
+	},
+});
+
+let devServer;
+
+test.beforeAll(async ({ astro }) => {
+	devServer = await astro.startDevServer();
+});
+
+test.afterAll(async ({ astro }) => {
+	await devServer.stop();
+});
+
+test('Tailwind CSS', async ({ page, astro }) => {
+	await page.goto(astro.resolveUrl('/'));
+
+	await test.step('body', async () => {
+		const body = page.locator('body');
+
+		await expect(body, 'should have classes').toHaveClass('bg-dawn text-midnight');
+		await expect(body, 'should have background color').toHaveCSS(
+			'background-color',
+			'rgb(243, 233, 250)'
+		);
+		await expect(body, 'should have color').toHaveCSS('color', 'rgb(49, 39, 74)');
+	});
+
+	await test.step('button', async () => {
+		const button = page.locator('button');
+
+		await expect(button, 'should have bg-purple-600').toHaveClass(/bg-purple-600/);
+		await expect(button, 'should have background color').toHaveCSS(
+			'background-color',
+			'rgb(147, 51, 234)'
+		);
+
+		await expect(button, 'should have lg:py-3').toHaveClass(/lg:py-3/);
+		await expect(button, 'should have padding bottom').toHaveCSS('padding-bottom', '12px');
+		await expect(button, 'should have padding top').toHaveCSS('padding-top', '12px');
+
+		await expect(button, 'should have font-[900]').toHaveClass(/font-\[900\]/);
+		await expect(button, 'should have font weight').toHaveCSS('font-weight', '900');
+	});
+});

packages/astro/e2e/test-utils.js

@@ -0,0 +1,13 @@
+import { loadFixture as baseLoadFixture } from '../test/test-utils.js';
+
+export function loadFixture(inlineConfig) {
+	if (!inlineConfig || !inlineConfig.root)
+		throw new Error("Must provide { root: './fixtures/...' }");
+
+	// resolve the relative root (i.e. "./fixtures/tailwindcss") to a full filepath
+	// without this, the main `loadFixture` helper will resolve relative to `packages/astro/test`
+	return baseLoadFixture({
+		...inlineConfig,
+		root: new URL(inlineConfig.root, import.meta.url).toString()
+	})
+}

packages/astro/package.json

@@ -72,7 +72,8 @@
     "postbuild": "astro-scripts copy \"src/**/*.astro\"",
     "benchmark": "node test/benchmark/dev.bench.js && node test/benchmark/build.bench.js",
     "test": "mocha --exit --timeout 20000 --ignore **/lit-element.test.js && mocha --timeout 20000 **/lit-element.test.js",
-    "test:match": "mocha --timeout 20000 -g"
+    "test:match": "mocha --timeout 20000 -g",
+    "test:e2e": "playwright test e2e"
   },
   "dependencies": {
     "@astrojs/compiler": "^0.14.2",
@@ -134,7 +135,8 @@
     "zod": "^3.16.0"
   },
   "devDependencies": {
-    "@babel/types": "^7.17.10",
+    "@babel/types": "^7.17.0",
+    "@playwright/test": "^1.21.1",
     "@types/babel__core": "^7.1.19",
     "@types/babel__generator": "^7.6.4",
     "@types/babel__traverse": "^7.17.1",

packages/astro/test/tailwindcss.test.js

@@ -70,50 +70,4 @@ describe('Tailwind', () => {
 			expect(bundledCSS, 'includes used component classes').to.match(/\.bg-purple-600{/);
 		});
 	});
-
-	// with "build" handling CSS checking, the dev tests are mostly testing the paths resolve in dev
-	describe('dev', () => {
-		let devServer;
-		let $;
-
-		before(async () => {
-			devServer = await fixture.startDevServer();
-			const html = await fixture.fetch('/').then((res) => res.text());
-			$ = cheerio.load(html);
-		});
-
-		after(async () => {
-			devServer && (await devServer.stop());
-		});
-
-		it('resolves CSS in src/styles', async () => {
-			const bundledCSSHREF = $('link[rel=stylesheet]').attr('href');
-			const res = await fixture.fetch(bundledCSSHREF);
-			expect(res.status).to.equal(200);
-
-			const text = await res.text();
-			expect(text, 'includes used component classes').to.match(/\.bg-purple-600/);
-
-			// tests a random tailwind class that isn't used on the page
-			expect(text, 'purges unused classes').not.to.match(/\.bg-blue-600/);
-
-			// tailwind escapes colons, `lg:py-3` compiles to `lg\:py-3`
-			expect(text, 'includes responsive classes').to.match(/\.lg\\\\:py-3/);
-
-			// tailwind escapes brackets, `font-[900]` compiles to `font-\[900\]`
-			expect(text, 'supports arbitrary value classes').to.match(/.font-\\[900\\]/);
-
-			// custom theme colors were included
-			expect(text, 'includes custom theme colors').to.match(/\.text-midnight/);
-			expect(text, 'includes custom theme colors').to.match(/\.bg-dawn/);
-		});
-
-		it('maintains classes in HTML', async () => {
-			const button = $('button');
-
-			expect(button.hasClass('text-white'), 'basic class').to.be.true;
-			expect(button.hasClass('lg:py-3'), 'responsive class').to.be.true;
-			expect(button.hasClass('font-[900]', 'arbitrary value')).to.be.true;
-		});
-	});
 });

packages/astro/test/test-utils.js

@@ -25,6 +25,7 @@ polyfill(globalThis, {
  *
  * @typedef {Object} Fixture
  * @property {typeof build} build
+ * @property {(url: string) => string} resolveUrl
  * @property {(url: string, opts: any) => Promise<Response>} fetch
  * @property {(path: string) => Promise<string>} readFile
  * @property {(path: string) => Promise<string[]>} readdir
@@ -93,6 +94,8 @@ export async function loadFixture(inlineConfig) {
 		},
 	};
 
+	const resolveUrl = (url) => `http://${'127.0.0.1'}:${config.server.port}${url.replace(/^\/?/, '/')}`;
+
 	return {
 		build: (opts = {}) => build(config, { mode: 'development', logging, telemetry, ...opts }),
 		startDevServer: async (opts = {}) => {
@@ -101,8 +104,9 @@ export async function loadFixture(inlineConfig) {
 			return devResult;
 		},
 		config,
+		resolveUrl,
 		fetch: (url, init) =>
-			fetch(`http://${'127.0.0.1'}:${config.server.port}${url.replace(/^\/?/, '/')}`, init),
+			fetch(resolveUrl(url), init),
 		preview: async (opts = {}) => {
 			const previewServer = await preview(config, { logging, telemetry, ...opts });
 			return previewServer;