Refactor @astrojs/prism, fix Prism component import not working (#4114)

* Upgrade @astrojs/prism to a real package, fix component import not working

* Remove `@astrojs/prism` as a dependency of `astro`

* Update lock file

* Refactor to multiple files

* Oops, can't have astro imports run inside node

* Follow Nate's suggestion on being minors instead of patchs

* Update lockfile

.changeset/slow-spiders-fly.md

@@ -0,0 +1,8 @@
+---
+'astro': patch
+'@astrojs/prism': minor
+'@astrojs/mdx': minor
+'@astrojs/markdown-remark': minor
+---
+
+Refactor `@astrojs/mdx` and `@astrojs/markdown-remark` to use `@astrojs/prism` instead of duplicating the code

packages/astro-prism/Prism.astro

@@ -1,7 +1,5 @@
 ---
-import Prism from 'prismjs';
+import { runHighlighterWithAstro } from './dist/highlighter';
-import { addAstro } from './internal.mjs';
-import loadLanguages from 'prismjs/components/index.js';
 
 export interface Props {
 	class?: string;
@@ -10,40 +8,9 @@ export interface Props {
 }
 
 const { class: className, lang, code } = Astro.props as Props;
-
+const { classLanguage, html } = runHighlighterWithAstro(lang, code)
-let classLanguage = `language-${lang}`;
-
-const languageMap = new Map([['ts', 'typescript']]);
-
-if (lang == null) {
-	console.warn('Prism.astro: No language provided.');
-}
-
-const ensureLoaded = (lang) => {
-	if (lang && !Prism.languages[lang]) {
-		loadLanguages([lang]);
-	}
-};
-
-if (languageMap.has(lang)) {
-	ensureLoaded(languageMap.get(lang));
-} else if (lang === 'astro') {
-	ensureLoaded('typescript');
-	addAstro(Prism);
-} else {
-	ensureLoaded('markup-templating'); // Prism expects this to exist for a number of other langs
-	ensureLoaded(lang);
-}
-
-if (lang && !Prism.languages[lang]) {
-	console.warn(`Unable to load the language: ${lang}`);
-}
-
-const grammar = Prism.languages[lang];
-let html = code;
-if (grammar) {
-	html = Prism.highlight(code, grammar, lang);
-}
 ---
 
-<pre class={[className, classLanguage].join(' ')}><code class={classLanguage}><Fragment set:html={html} /></code></pre>
+<pre class={[className, classLanguage].join(' ')}>
+	<code class={classLanguage}><Fragment set:html={html} /></code>
+</pre>

packages/astro-prism/README.md

@@ -0,0 +1,31 @@
+# @astrojs/prism
+
+Supports Prism highlighting in Astro projects
+
+## Component
+
+This package exports a component to support highlighting inside an Astro file. Example:
+
+```astro
+---
+import { Prism } from "@astrojs/prism"
+---
+
+<Prism lang="js" code={`const foo = 'bar';`} />
+```
+
+## Internal
+
+This package exports a `runHighlighterWithAstro` function to highlight while making sure the Astro language is loaded beforehand
+
+```typescript
+import { runHighlighterWithAstro } from '@astrojs/prism';
+
+runHighlighterWithAstro(`
+  ---
+    const helloAstro = 'Hello, Astro!';
+  ---
+
+  <div>{helloAstro}</div>
+`, 'astro');
+```

packages/astro-prism/index.js

@@ -1 +0,0 @@
-export { default as Prism } from './Prism.astro';

packages/astro-prism/internal.d.ts

@@ -1 +0,0 @@
-export function addAstro(Prism: any): void;

packages/astro-prism/package.json

@@ -3,6 +3,7 @@
   "version": "0.6.1",
   "description": "Supports Prism highlighting in Astro projects",
   "author": "withastro",
+  "type": "module",
   "license": "MIT",
   "bugs": "https://github.com/withastro/astro/issues",
   "repository": {
@@ -11,17 +12,28 @@
     "directory": "packages/astro-prism"
   },
   "homepage": "https://astro.build",
-  "main": "index.js",
+  "main": "dist/index.js",
-  "scripts": {},
+  "scripts": {
-  "exports": {
+    "build": "astro-scripts build \"src/**/*.ts\" && tsc",
-    ".": "./index.js",
+    "build:ci": "astro-scripts build \"src/**/*.ts\"",
-    "./internal": "./internal.mjs"
+    "dev": "astro-scripts dev \"src/**/*.ts\""
   },
-  "types": "./internal.d.ts",
+  "exports": {
-  "keywords": [],
+    ".": "./dist/index.js",
-  "devDependencies": {
+    "./Prism.astro": "./Prism.astro",
+    "./dist/highlighter": "./dist/highlighter.js"
+  },
+  "keywords": [
+    "astro",
+    "astro-component"
+  ],
+  "dependencies": {
     "prismjs": "^1.28.0"
   },
+  "devDependencies": {
+    "astro-scripts": "workspace:*",
+    "@types/prismjs": "1.26.0"
+  },
   "engines": {
     "node": "^14.18.0 || >=16.12.0"
   }

packages/astro-prism/src/highlighter.ts

@@ -0,0 +1,42 @@
+import Prism from 'prismjs';
+import loadLanguages from 'prismjs/components/index.js';
+import { addAstro } from './plugin.js';
+
+const languageMap = new Map([['ts', 'typescript']]);
+
+export function runHighlighterWithAstro(lang: string | undefined, code: string) {
+	let classLanguage = `language-${lang}`;
+
+	if (!lang) {
+		lang = 'plaintext';
+	}
+
+	const ensureLoaded = (language: string) => {
+		if (language && !Prism.languages[language]) {
+			loadLanguages([language]);
+		}
+	};
+
+	if (languageMap.has(lang)) {
+		ensureLoaded(languageMap.get(lang)!);
+	} else if (lang === 'astro') {
+		ensureLoaded('typescript');
+		addAstro(Prism);
+	} else {
+		ensureLoaded('markup-templating'); // Prism expects this to exist for a number of other langs
+		ensureLoaded(lang);
+	}
+
+	if (lang && !Prism.languages[lang]) {
+		// eslint-disable-next-line no-console
+		console.warn(`Unable to load the language: ${lang}`);
+	}
+
+	const grammar = Prism.languages[lang];
+	let html = code;
+	if (grammar) {
+		html = Prism.highlight(code, grammar, lang);
+	}
+
+	return { classLanguage, html };
+}

packages/astro-prism/src/index.ts

@@ -0,0 +1,2 @@
+// @ts-expect-error
+export { default as Prism } from '../Prism.astro';

packages/astro-prism/src/plugin.ts

@@ -1,13 +1,14 @@
-export function addAstro(Prism) {
+export function addAstro(Prism: typeof import('prismjs')) {
 	if (Prism.languages.astro) {
 		return;
 	}
 
-	let scriptLang;
+	let scriptLang: string;
 	if (Prism.languages.typescript) {
 		scriptLang = 'typescript';
 	} else {
 		scriptLang = 'javascript';
+		// eslint-disable-next-line no-console
 		console.warn(
 			'Prism TypeScript language not loaded, Astro scripts will be treated as JavaScript.'
 		);
@@ -19,11 +20,7 @@ export function addAstro(Prism) {
 	let braces = /(?:\{(?:\{(?:\{[^{}]*\}|[^{}])*\}|[^{}])*\})/.source;
 	let spread = /(?:\{<S>*\.{3}(?:[^{}]|<BRACES>)*\})/.source;
 
-	/**
+	function re(source: string, flags?: string) {
-	 * @param {string} source
-	 * @param {string} [flags]
-	 */
-	function re(source, flags) {
 		source = source
 			.replace(/<S>/g, function () {
 				return space;
@@ -40,16 +37,18 @@ export function addAstro(Prism) {
 	spread = re(spread).source;
 
 	Prism.languages.astro = Prism.languages.extend('markup', script);
-	Prism.languages.astro.tag.pattern = re(
+
+	(Prism.languages.astro as any).tag.pattern = re(
 		/<\/?(?:[\w.:-]+(?:<S>+(?:[\w.:$-]+(?:=(?:"(?:\\[^]|[^\\"])*"|'(?:\\[^]|[^\\'])*'|[^\s{'"/>=]+|<BRACES>))?|<SPREAD>))*<S>*\/?)?>/
 			.source
 	);
 
-	Prism.languages.astro.tag.inside['tag'].pattern = /^<\/?[^\s>\/]*/i;
+	(Prism.languages.astro as any).tag.inside['tag'].pattern = /^<\/?[^\s>\/]*/i;
-	Prism.languages.astro.tag.inside['attr-value'].pattern =
+	(Prism.languages.astro as any).tag.inside['attr-value'].pattern =
 		/=(?!\{)(?:"(?:\\[^]|[^\\"])*"|'(?:\\[^]|[^\\'])*'|[^\s'">]+)/i;
-	Prism.languages.astro.tag.inside['tag'].inside['class-name'] = /^[A-Z]\w*(?:\.[A-Z]\w*)*$/;
+	(Prism.languages.astro as any).tag.inside['tag'].inside['class-name'] =
-	Prism.languages.astro.tag.inside['comment'] = script['comment'];
+		/^[A-Z]\w*(?:\.[A-Z]\w*)*$/;
+	(Prism.languages.astro as any).tag.inside['comment'] = script['comment'];
 
 	Prism.languages.insertBefore(
 		'inside',
@@ -60,7 +59,7 @@ export function addAstro(Prism) {
 				inside: Prism.languages.astro,
 			},
 		},
-		Prism.languages.astro.tag
+		(Prism.languages.astro as any).tag
 	);
 
 	Prism.languages.insertBefore(
@@ -80,11 +79,11 @@ export function addAstro(Prism) {
 				alias: `language-${scriptLang}`,
 			},
 		},
-		Prism.languages.astro.tag
+		(Prism.languages.astro as any).tag
 	);
 
 	// The following will handle plain text inside tags
-	let stringifyToken = function (token) {
+	let stringifyToken = function (token: any) {
 		if (!token) {
 			return '';
 		}
@@ -97,8 +96,8 @@ export function addAstro(Prism) {
 		return token.content.map(stringifyToken).join('');
 	};
 
-	let walkTokens = function (tokens) {
+	let walkTokens = function (tokens: any) {
-		let openedTags = [];
+		let openedTags: any[] = [];
 		for (let i = 0; i < tokens.length; i++) {
 			let token = tokens[i];
 
@@ -169,7 +168,7 @@ export function addAstro(Prism) {
 						i--;
 					}
 
-					tokens[i] = new Prism.Token('plain-text', plainText, null, plainText);
+					tokens[i] = new Prism.Token('plain-text', plainText, undefined, plainText);
 				}
 			}
 
@@ -179,7 +178,7 @@ export function addAstro(Prism) {
 		}
 	};
 
-	Prism.hooks.add('after-tokenize', function (env) {
+	Prism.hooks.add('after-tokenize', function (env: any) {
 		if (env.language !== 'astro') {
 			return;
 		}

packages/astro-prism/tsconfig.json

@@ -0,0 +1,10 @@
+{
+  "extends": "../../tsconfig.base.json",
+  "include": ["src"],
+  "compilerOptions": {
+    "allowJs": true,
+    "target": "ES2020",
+    "module": "ES2020",
+    "outDir": "./dist"
+  }
+}

packages/astro/package.json

@@ -87,7 +87,6 @@
     "@astrojs/compiler": "^0.22.1",
     "@astrojs/language-server": "^0.20.0",
     "@astrojs/markdown-remark": "^0.13.0",
-    "@astrojs/prism": "0.6.1",
     "@astrojs/telemetry": "^0.4.1",
     "@astrojs/webapi": "^0.12.0",
     "@babel/core": "^7.18.2",
@@ -124,7 +123,6 @@
     "postcss": "^8.4.14",
     "postcss-load-config": "^3.1.4",
     "preferred-pm": "^3.0.3",
-    "prismjs": "^1.28.0",
     "prompts": "^2.4.2",
     "recast": "^0.20.5",
     "rehype": "^12.0.1",

packages/integrations/mdx/package.json

@@ -37,8 +37,6 @@
     "es-module-lexer": "^0.10.5",
     "github-slugger": "^1.4.0",
     "gray-matter": "^4.0.3",
-    "mdast-util-mdx": "^2.0.0",
-    "prismjs": "^1.28.0",
     "rehype-raw": "^6.1.1",
     "remark-frontmatter": "^4.0.1",
     "remark-gfm": "^3.0.1",

packages/integrations/mdx/src/remark-prism.ts

@@ -1,48 +1,6 @@
-// TODO: discuss extracting this file to @astrojs/prism
+import { runHighlighterWithAstro } from '@astrojs/prism/dist/highlighter';
-import { addAstro } from '@astrojs/prism/internal';
-import Prism from 'prismjs';
-import loadLanguages from 'prismjs/components/index.js';
 import { visit } from 'unist-util-visit';
 
-const languageMap = new Map([['ts', 'typescript']]);
-
-function runHighlighter(lang: string, code: string) {
-	let classLanguage = `language-${lang}`;
-
-	if (lang == null) {
-		lang = 'plaintext';
-	}
-
-	const ensureLoaded = (language: string) => {
-		if (language && !Prism.languages[language]) {
-			loadLanguages([language]);
-		}
-	};
-
-	if (languageMap.has(lang)) {
-		ensureLoaded(languageMap.get(lang)!);
-	} else if (lang === 'astro') {
-		ensureLoaded('typescript');
-		addAstro(Prism);
-	} else {
-		ensureLoaded('markup-templating'); // Prism expects this to exist for a number of other langs
-		ensureLoaded(lang);
-	}
-
-	if (lang && !Prism.languages[lang]) {
-		// eslint-disable-next-line no-console
-		console.warn(`Unable to load the language: ${lang}`);
-	}
-
-	const grammar = Prism.languages[lang];
-	let html = code;
-	if (grammar) {
-		html = Prism.highlight(code, grammar, lang);
-	}
-
-	return { classLanguage, html };
-}
-
 /**  */
 export default function remarkPrism() {
 	return (tree: any) =>
@@ -50,7 +8,7 @@ export default function remarkPrism() {
 			let { lang, value } = node;
 			node.type = 'html';
 
-			let { html, classLanguage } = runHighlighter(lang, value);
+			let { html, classLanguage } = runHighlighterWithAstro(lang, value);
 			let classes = [classLanguage];
 			node.value = `<pre class="${classes.join(
 				' '

packages/markdown/remark/package.json

@@ -36,7 +36,6 @@
     "micromark-extension-mdx-expression": "^1.0.3",
     "micromark-extension-mdx-md": "^1.0.0",
     "micromark-util-combine-extensions": "^1.0.0",
-    "prismjs": "^1.28.0",
     "rehype-raw": "^6.1.1",
     "rehype-stringify": "^9.0.3",
     "remark-gfm": "^3.0.1",
@@ -55,7 +54,6 @@
     "@types/hast": "^2.3.4",
     "@types/mdast": "^3.0.10",
     "@types/mocha": "^9.1.1",
-    "@types/prismjs": "^1.26.0",
     "@types/unist": "^2.0.6",
     "astro-scripts": "workspace:*",
     "chai": "^4.3.6",

packages/markdown/remark/src/remark-prism.ts

@@ -1,48 +1,7 @@
-import { addAstro } from '@astrojs/prism/internal';
+import { runHighlighterWithAstro } from '@astrojs/prism/dist/highlighter';
-import Prism from 'prismjs';
-import loadLanguages from 'prismjs/components/index.js';
 import { visit } from 'unist-util-visit';
 const noVisit = new Set(['root', 'html', 'text']);
 
-const languageMap = new Map([['ts', 'typescript']]);
-
-function runHighlighter(lang: string, code: string) {
-	let classLanguage = `language-${lang}`;
-
-	if (lang == null) {
-		lang = 'plaintext';
-	}
-
-	const ensureLoaded = (language: string) => {
-		if (language && !Prism.languages[language]) {
-			loadLanguages([language]);
-		}
-	};
-
-	if (languageMap.has(lang)) {
-		ensureLoaded(languageMap.get(lang)!);
-	} else if (lang === 'astro') {
-		ensureLoaded('typescript');
-		addAstro(Prism);
-	} else {
-		ensureLoaded('markup-templating'); // Prism expects this to exist for a number of other langs
-		ensureLoaded(lang);
-	}
-
-	if (lang && !Prism.languages[lang]) {
-		// eslint-disable-next-line no-console
-		console.warn(`Unable to load the language: ${lang}`);
-	}
-
-	const grammar = Prism.languages[lang];
-	let html = code;
-	if (grammar) {
-		html = Prism.highlight(code, grammar, lang);
-	}
-
-	return { classLanguage, html };
-}
-
 type MaybeString = string | null | undefined;
 
 /**  */
@@ -52,7 +11,7 @@ function transformer(className: MaybeString) {
 			let { lang, value } = node;
 			node.type = 'html';
 
-			let { html, classLanguage } = runHighlighter(lang, value);
+			let { html, classLanguage } = runHighlighterWithAstro(lang, value);
 			let classes = [classLanguage];
 			if (className) {
 				classes.push(className);

packages/webapi/mod.d.ts

@@ -1,5 +1,5 @@
 export { pathToPosix } from './lib/utils';
-export { AbortController, AbortSignal, alert, atob, Blob, btoa, ByteLengthQueuingStrategy, cancelAnimationFrame, cancelIdleCallback, CanvasRenderingContext2D, CharacterData, clearTimeout, Comment, CountQueuingStrategy, CSSStyleSheet, CustomElementRegistry, CustomEvent, Document, DocumentFragment, DOMException, Element, Event, EventTarget, fetch, File, FormData, Headers, HTMLBodyElement, HTMLCanvasElement, HTMLDivElement, HTMLDocument, HTMLElement, HTMLHeadElement, HTMLHtmlElement, HTMLImageElement, HTMLSpanElement, HTMLStyleElement, HTMLTemplateElement, HTMLUnknownElement, Image, ImageData, IntersectionObserver, MediaQueryList, MutationObserver, Node, NodeFilter, NodeIterator, OffscreenCanvas, ReadableByteStreamController, ReadableStream, ReadableStreamBYOBReader, ReadableStreamBYOBRequest, ReadableStreamDefaultController, ReadableStreamDefaultReader, Request, requestAnimationFrame, requestIdleCallback, ResizeObserver, Response, setTimeout, ShadowRoot, structuredClone, StyleSheet, Text, TransformStream, TreeWalker, URLPattern, Window, WritableStream, WritableStreamDefaultController, WritableStreamDefaultWriter } from './mod.js';
+export { AbortController, AbortSignal, alert, atob, Blob, btoa, ByteLengthQueuingStrategy, cancelAnimationFrame, cancelIdleCallback, CanvasRenderingContext2D, CharacterData, clearTimeout, Comment, CountQueuingStrategy, CSSStyleSheet, CustomElementRegistry, CustomEvent, Document, DocumentFragment, DOMException, Element, Event, EventTarget, fetch, File, FormData, Headers, HTMLBodyElement, HTMLCanvasElement, HTMLDivElement, HTMLDocument, HTMLElement, HTMLHeadElement, HTMLHtmlElement, HTMLImageElement, HTMLSpanElement, HTMLStyleElement, HTMLTemplateElement, HTMLUnknownElement, Image, ImageData, IntersectionObserver, MediaQueryList, MutationObserver, Node, NodeFilter, NodeIterator, OffscreenCanvas, ReadableByteStreamController, ReadableStream, ReadableStreamBYOBReader, ReadableStreamBYOBRequest, ReadableStreamDefaultController, ReadableStreamDefaultReader, Request, requestAnimationFrame, requestIdleCallback, ResizeObserver, Response, setTimeout, ShadowRoot, structuredClone, StyleSheet, Text, TransformStream, TreeWalker, URLPattern, Window, WritableStream, WritableStreamDefaultController, WritableStreamDefaultWriter, } from './mod.js';
 export declare const polyfill: {
     (target: any, options?: PolyfillOptions): any;
     internals(target: any, name: string): any;

pnpm-lock.yaml

@@ -441,7 +441,6 @@ importers:
       '@astrojs/compiler': ^0.22.1
       '@astrojs/language-server': ^0.20.0
       '@astrojs/markdown-remark': ^0.13.0
-      '@astrojs/prism': 0.6.1
       '@astrojs/telemetry': ^0.4.1
       '@astrojs/webapi': ^0.12.0
       '@babel/core': ^7.18.2
@@ -503,7 +502,6 @@ importers:
       postcss: ^8.4.14
       postcss-load-config: ^3.1.4
       preferred-pm: ^3.0.3
-      prismjs: ^1.28.0
       prompts: ^2.4.2
       recast: ^0.20.5
       rehype: ^12.0.1
@@ -529,7 +527,6 @@ importers:
       '@astrojs/compiler': 0.22.1
       '@astrojs/language-server': 0.20.3
       '@astrojs/markdown-remark': link:../markdown/remark
-      '@astrojs/prism': link:../astro-prism
       '@astrojs/telemetry': link:../telemetry
       '@astrojs/webapi': link:../webapi
       '@babel/core': 7.18.9
@@ -566,7 +563,6 @@ importers:
       postcss: 8.4.14
       postcss-load-config: 3.1.4_postcss@8.4.14
       preferred-pm: 3.0.3
-      prismjs: 1.28.0
       prompts: 2.4.2
       recast: 0.20.5
       rehype: 12.0.1
@@ -617,9 +613,14 @@ importers:
 
   packages/astro-prism:
     specifiers:
+      '@types/prismjs': 1.26.0
+      astro-scripts: workspace:*
       prismjs: ^1.28.0
-    devDependencies:
+    dependencies:
       prismjs: 1.28.0
+    devDependencies:
+      '@types/prismjs': 1.26.0
+      astro-scripts: link:../../scripts
 
   packages/astro-rss:
     specifiers:
@@ -2186,10 +2187,8 @@ importers:
       github-slugger: ^1.4.0
       gray-matter: ^4.0.3
       linkedom: ^0.14.12
-      mdast-util-mdx: ^2.0.0
       mdast-util-to-string: ^3.1.0
       mocha: ^9.2.2
-      prismjs: ^1.28.0
       reading-time: ^1.5.0
       rehype-raw: ^6.1.1
       remark-frontmatter: ^4.0.1
@@ -2208,8 +2207,6 @@ importers:
       es-module-lexer: 0.10.5
       github-slugger: 1.4.0
       gray-matter: 4.0.3
-      mdast-util-mdx: 2.0.0
-      prismjs: 1.28.0
       rehype-raw: 6.1.1
       remark-frontmatter: 4.0.1
       remark-gfm: 3.0.1
@@ -2613,7 +2610,6 @@ importers:
       '@types/hast': ^2.3.4
       '@types/mdast': ^3.0.10
       '@types/mocha': ^9.1.1
-      '@types/prismjs': ^1.26.0
       '@types/unist': ^2.0.6
       acorn: ^8.7.1
       acorn-jsx: ^5.3.2
@@ -2629,7 +2625,6 @@ importers:
       micromark-util-combine-extensions: ^1.0.0
       micromark-util-types: ^1.0.2
       mocha: ^9.2.2
-      prismjs: ^1.28.0
       rehype-raw: ^6.1.1
       rehype-stringify: ^9.0.3
       remark-gfm: ^3.0.1
@@ -2654,7 +2649,6 @@ importers:
       micromark-extension-mdx-expression: 1.0.3
       micromark-extension-mdx-md: 1.0.0
       micromark-util-combine-extensions: 1.0.0
-      prismjs: 1.28.0
       rehype-raw: 6.1.1
       rehype-stringify: 9.0.3
       remark-gfm: 3.0.1
@@ -2672,7 +2666,6 @@ importers:
       '@types/hast': 2.3.4
       '@types/mdast': 3.0.10
       '@types/mocha': 9.1.1
-      '@types/prismjs': 1.26.0
       '@types/unist': 2.0.6
       astro-scripts: link:../../../scripts
       chai: 4.3.6
@@ -14525,6 +14518,7 @@ packages:
   /prismjs/1.28.0:
     resolution: {integrity: sha512-8aaXdYvl1F7iC7Xm1spqSaY/OJBpYW3v+KJ+F17iYxvdc8sfjW194COK5wVhMZX45tGteiBQgdvD/nhxcRwylw==}
     engines: {node: '>=6'}
+    dev: false
 
   /process-nextick-args/2.0.1:
     resolution: {integrity: sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==}