Use shikiji

This commit is contained in:
bluwy 2023-09-11 19:55:25 +08:00
parent a6a516d944
commit d5d2d3414a
22 changed files with 333 additions and 573 deletions

View file

@ -1,7 +1,13 @@
---
import type * as shiki from 'shiki';
import { renderToHtml } from 'shiki';
import { getHighlighter } from './Shiki.js';
import type {
BuiltinLanguage,
BuiltinTheme,
LanguageRegistration,
SpecialLanguage,
ThemeRegistration,
ThemeRegistrationRaw,
} from 'shikiji';
import { getCachedHighlighter } from './shiki.js';
interface Props {
/** The code to highlight. Required. */
@ -13,7 +19,7 @@ interface Props {
*
* @default "plaintext"
*/
lang?: shiki.Lang | shiki.ILanguageRegistration;
lang?: BuiltinLanguage | SpecialLanguage | LanguageRegistration;
/**
* The styling theme.
* Supports all themes listed here: https://github.com/shikijs/shiki/blob/main/docs/themes.md#all-themes
@ -21,7 +27,7 @@ interface Props {
*
* @default "github-dark"
*/
theme?: shiki.IThemeRegistration;
theme?: BuiltinTheme | ThemeRegistration | ThemeRegistrationRaw;
/**
* Enable word wrapping.
* - true: enabled.
@ -47,41 +53,52 @@ const {
inline = false,
} = Astro.props;
// 1. Get the shiki syntax highlighter
const highlighter = await getHighlighter({
theme,
// Load custom lang if passed an object, otherwise load the default
langs: typeof lang !== 'string' ? [lang] : undefined,
// shiki -> shikiji compat
if (typeof lang === 'object') {
// `id` renamed to `name
if ((lang as any).id && !lang.name) {
lang.name = (lang as any).id;
}
// `grammar` flattened to lang itself
if ((lang as any).grammar) {
Object.assign(lang, (lang as any).grammar);
}
}
const highlighter = await getCachedHighlighter({
langs: [lang],
themes: [theme],
});
// 2. Turn code into shiki theme tokens
const tokens = highlighter.codeToThemedTokens(code, typeof lang === 'string' ? lang : lang.id);
// 3. Get shiki theme object
const _theme = highlighter.getTheme();
// 4. Render the theme tokens as html
const html = renderToHtml(tokens, {
themeName: _theme.name,
fg: _theme.fg,
bg: _theme.bg,
elements: {
pre({ className, style, children }) {
const html = highlighter.codeToHtml(code, {
lang: typeof lang === 'string' ? lang : lang.name,
theme,
transforms: {
pre(node) {
// Swap to `code` tag if inline
const tag = inline ? 'code' : 'pre';
if (inline) {
node.tagName = 'code';
}
const classValue = (node.properties.class as string) ?? '';
const styleValue = (node.properties.style as string) ?? '';
// Replace "shiki" class naming with "astro-code"
className = className.replace(/shiki/g, 'astro-code');
node.properties.class = classValue.replace(/shiki/g, 'astro-code');
// Handle code wrapping
// if wrap=null, do nothing.
if (wrap === false) {
style += '; overflow-x: auto;';
node.properties.style = styleValue + '; overflow-x: auto;';
} else if (wrap === true) {
style += '; overflow-x: auto; white-space: pre-wrap; word-wrap: break-word;';
node.properties.style =
styleValue + '; overflow-x: auto; white-space: pre-wrap; word-wrap: break-word;';
}
return `<${tag} class="${className}" style="${style}" tabindex="0">${children}</${tag}>`;
},
code({ children }) {
return inline ? children : `<code>${children}</code>`;
code(node) {
if (inline) {
return node.children[0] as typeof node;
}
},
},
});

View file

@ -1,97 +0,0 @@
import { getHighlighter as getShikiHighlighter } from 'shiki';
import { themes } from './shiki-themes.js';
import { languages } from './shiki-languages.js';
// Caches Promise<Highlighter> for reuse when the same theme and langs are provided
const _resolvedHighlighters = new Map();
/** @type {Promise<any>} */
let _allLanguages;
function stringify(opts) {
// Always sort keys before stringifying to make sure objects match regardless of parameter ordering
return JSON.stringify(opts, Object.keys(opts).sort());
}
/**
* @param {import('shiki').HighlighterOptions} opts
* @returns {Promise<import('shiki').HighlighterOptions>}
*/
export async function resolveHighlighterOptions(opts) {
const resolvedThemes = [];
if (opts.theme && opts.theme in themes) {
resolvedThemes.push(await themes[opts.theme]());
} else if (Object.keys(opts.theme).length) {
resolvedThemes.push(opts.theme);
}
let resolvedLanguages;
if (opts.langs) {
resolvedLanguages = opts.langs;
} else {
if (!_allLanguages) {
_allLanguages = (await Promise.all(Object.values(languages).map((fn) => fn()))).filter(
Boolean
);
}
resolvedLanguages = await _allLanguages;
}
/** @type {import('shiki').HighlighterOptions} */
const highlighterOptions = {
...opts,
themes: resolvedThemes,
langs: resolvedLanguages,
};
// Do not pass through the theme as that will attempt to load it, even if it's included in themes
delete highlighterOptions['theme'];
return highlighterOptions;
}
/**
* @param {import('shiki').HighlighterOptions} opts
* @returns {Promise<import('shiki').Highlighter>}
*/
async function resolveHighlighter(opts) {
const highlighterOptions = await resolveHighlighterOptions(opts);
// Start the async getHighlighter call and cache the Promise
const highlighter = getShikiHighlighter(highlighterOptions).then((hl) => {
hl.setColorReplacements({
'#000001': 'var(--astro-code-color-text)',
'#000002': 'var(--astro-code-color-background)',
'#000004': 'var(--astro-code-token-constant)',
'#000005': 'var(--astro-code-token-string)',
'#000006': 'var(--astro-code-token-comment)',
'#000007': 'var(--astro-code-token-keyword)',
'#000008': 'var(--astro-code-token-parameter)',
'#000009': 'var(--astro-code-token-function)',
'#000010': 'var(--astro-code-token-string-expression)',
'#000011': 'var(--astro-code-token-punctuation)',
'#000012': 'var(--astro-code-token-link)',
});
return hl;
});
return highlighter;
}
/**
* @param {import('shiki').HighlighterOptions} opts
* @returns {Promise<import('shiki').Highlighter>}
*/
export function getHighlighter(opts) {
const key = stringify(opts);
// Highlighter has already been requested, reuse the same instance
if (_resolvedHighlighters.has(key)) {
return _resolvedHighlighters.get(key);
}
const highlighter = resolveHighlighter(opts);
_resolvedHighlighters.set(key, highlighter);
return highlighter;
}

View file

@ -1,176 +0,0 @@
/**
* This file is prebuilt from packages/astro/scripts/shiki-gen-languages.mjs
* Do not edit this directly, but instead edit that file and rerun it to generate this file.
*/
import { BUNDLED_LANGUAGES } from 'shiki';
function handleLang(grammar, language) {
const lang = BUNDLED_LANGUAGES.find((l) => l.id === language);
if (lang) {
return {
...lang,
grammar,
};
} else {
return undefined;
}
}
// prettier-ignore
export const languages = {
'abap': () => import('shiki/languages/abap.tmLanguage.json').then((mod) => handleLang(mod.default, 'abap')),
'actionscript-3': () => import('shiki/languages/actionscript-3.tmLanguage.json').then((mod) => handleLang(mod.default, 'actionscript-3')),
'ada': () => import('shiki/languages/ada.tmLanguage.json').then((mod) => handleLang(mod.default, 'ada')),
'apache': () => import('shiki/languages/apache.tmLanguage.json').then((mod) => handleLang(mod.default, 'apache')),
'apex': () => import('shiki/languages/apex.tmLanguage.json').then((mod) => handleLang(mod.default, 'apex')),
'apl': () => import('shiki/languages/apl.tmLanguage.json').then((mod) => handleLang(mod.default, 'apl')),
'applescript': () => import('shiki/languages/applescript.tmLanguage.json').then((mod) => handleLang(mod.default, 'applescript')),
'ara': () => import('shiki/languages/ara.tmLanguage.json').then((mod) => handleLang(mod.default, 'ara')),
'asm': () => import('shiki/languages/asm.tmLanguage.json').then((mod) => handleLang(mod.default, 'asm')),
'astro': () => import('shiki/languages/astro.tmLanguage.json').then((mod) => handleLang(mod.default, 'astro')),
'awk': () => import('shiki/languages/awk.tmLanguage.json').then((mod) => handleLang(mod.default, 'awk')),
'ballerina': () => import('shiki/languages/ballerina.tmLanguage.json').then((mod) => handleLang(mod.default, 'ballerina')),
'bat': () => import('shiki/languages/bat.tmLanguage.json').then((mod) => handleLang(mod.default, 'bat')),
'berry': () => import('shiki/languages/berry.tmLanguage.json').then((mod) => handleLang(mod.default, 'berry')),
'bibtex': () => import('shiki/languages/bibtex.tmLanguage.json').then((mod) => handleLang(mod.default, 'bibtex')),
'bicep': () => import('shiki/languages/bicep.tmLanguage.json').then((mod) => handleLang(mod.default, 'bicep')),
'blade': () => import('shiki/languages/blade.tmLanguage.json').then((mod) => handleLang(mod.default, 'blade')),
'c': () => import('shiki/languages/c.tmLanguage.json').then((mod) => handleLang(mod.default, 'c')),
'cadence': () => import('shiki/languages/cadence.tmLanguage.json').then((mod) => handleLang(mod.default, 'cadence')),
'clarity': () => import('shiki/languages/clarity.tmLanguage.json').then((mod) => handleLang(mod.default, 'clarity')),
'clojure': () => import('shiki/languages/clojure.tmLanguage.json').then((mod) => handleLang(mod.default, 'clojure')),
'cmake': () => import('shiki/languages/cmake.tmLanguage.json').then((mod) => handleLang(mod.default, 'cmake')),
'cobol': () => import('shiki/languages/cobol.tmLanguage.json').then((mod) => handleLang(mod.default, 'cobol')),
'codeql': () => import('shiki/languages/codeql.tmLanguage.json').then((mod) => handleLang(mod.default, 'codeql')),
'coffee': () => import('shiki/languages/coffee.tmLanguage.json').then((mod) => handleLang(mod.default, 'coffee')),
'cpp-macro': () => import('shiki/languages/cpp-macro.tmLanguage.json').then((mod) => handleLang(mod.default, 'cpp-macro')),
'cpp': () => import('shiki/languages/cpp.tmLanguage.json').then((mod) => handleLang(mod.default, 'cpp')),
'crystal': () => import('shiki/languages/crystal.tmLanguage.json').then((mod) => handleLang(mod.default, 'crystal')),
'csharp': () => import('shiki/languages/csharp.tmLanguage.json').then((mod) => handleLang(mod.default, 'csharp')),
'css': () => import('shiki/languages/css.tmLanguage.json').then((mod) => handleLang(mod.default, 'css')),
'cue': () => import('shiki/languages/cue.tmLanguage.json').then((mod) => handleLang(mod.default, 'cue')),
'd': () => import('shiki/languages/d.tmLanguage.json').then((mod) => handleLang(mod.default, 'd')),
'dart': () => import('shiki/languages/dart.tmLanguage.json').then((mod) => handleLang(mod.default, 'dart')),
'dax': () => import('shiki/languages/dax.tmLanguage.json').then((mod) => handleLang(mod.default, 'dax')),
'diff': () => import('shiki/languages/diff.tmLanguage.json').then((mod) => handleLang(mod.default, 'diff')),
'docker': () => import('shiki/languages/docker.tmLanguage.json').then((mod) => handleLang(mod.default, 'docker')),
'dream-maker': () => import('shiki/languages/dream-maker.tmLanguage.json').then((mod) => handleLang(mod.default, 'dream-maker')),
'elixir': () => import('shiki/languages/elixir.tmLanguage.json').then((mod) => handleLang(mod.default, 'elixir')),
'elm': () => import('shiki/languages/elm.tmLanguage.json').then((mod) => handleLang(mod.default, 'elm')),
'erb': () => import('shiki/languages/erb.tmLanguage.json').then((mod) => handleLang(mod.default, 'erb')),
'erlang': () => import('shiki/languages/erlang.tmLanguage.json').then((mod) => handleLang(mod.default, 'erlang')),
'fish': () => import('shiki/languages/fish.tmLanguage.json').then((mod) => handleLang(mod.default, 'fish')),
'fsharp': () => import('shiki/languages/fsharp.tmLanguage.json').then((mod) => handleLang(mod.default, 'fsharp')),
'gherkin': () => import('shiki/languages/gherkin.tmLanguage.json').then((mod) => handleLang(mod.default, 'gherkin')),
'git-commit': () => import('shiki/languages/git-commit.tmLanguage.json').then((mod) => handleLang(mod.default, 'git-commit')),
'git-rebase': () => import('shiki/languages/git-rebase.tmLanguage.json').then((mod) => handleLang(mod.default, 'git-rebase')),
'glsl': () => import('shiki/languages/glsl.tmLanguage.json').then((mod) => handleLang(mod.default, 'glsl')),
'gnuplot': () => import('shiki/languages/gnuplot.tmLanguage.json').then((mod) => handleLang(mod.default, 'gnuplot')),
'go': () => import('shiki/languages/go.tmLanguage.json').then((mod) => handleLang(mod.default, 'go')),
'graphql': () => import('shiki/languages/graphql.tmLanguage.json').then((mod) => handleLang(mod.default, 'graphql')),
'groovy': () => import('shiki/languages/groovy.tmLanguage.json').then((mod) => handleLang(mod.default, 'groovy')),
'hack': () => import('shiki/languages/hack.tmLanguage.json').then((mod) => handleLang(mod.default, 'hack')),
'haml': () => import('shiki/languages/haml.tmLanguage.json').then((mod) => handleLang(mod.default, 'haml')),
'handlebars': () => import('shiki/languages/handlebars.tmLanguage.json').then((mod) => handleLang(mod.default, 'handlebars')),
'haskell': () => import('shiki/languages/haskell.tmLanguage.json').then((mod) => handleLang(mod.default, 'haskell')),
'hcl': () => import('shiki/languages/hcl.tmLanguage.json').then((mod) => handleLang(mod.default, 'hcl')),
'hlsl': () => import('shiki/languages/hlsl.tmLanguage.json').then((mod) => handleLang(mod.default, 'hlsl')),
'html': () => import('shiki/languages/html.tmLanguage.json').then((mod) => handleLang(mod.default, 'html')),
'http': () => import('shiki/languages/http.tmLanguage.json').then((mod) => handleLang(mod.default, 'http')),
'imba': () => import('shiki/languages/imba.tmLanguage.json').then((mod) => handleLang(mod.default, 'imba')),
'ini': () => import('shiki/languages/ini.tmLanguage.json').then((mod) => handleLang(mod.default, 'ini')),
'java': () => import('shiki/languages/java.tmLanguage.json').then((mod) => handleLang(mod.default, 'java')),
'javascript': () => import('shiki/languages/javascript.tmLanguage.json').then((mod) => handleLang(mod.default, 'javascript')),
'jinja-html': () => import('shiki/languages/jinja-html.tmLanguage.json').then((mod) => handleLang(mod.default, 'jinja-html')),
'jinja': () => import('shiki/languages/jinja.tmLanguage.json').then((mod) => handleLang(mod.default, 'jinja')),
'jison': () => import('shiki/languages/jison.tmLanguage.json').then((mod) => handleLang(mod.default, 'jison')),
'json': () => import('shiki/languages/json.tmLanguage.json').then((mod) => handleLang(mod.default, 'json')),
'json5': () => import('shiki/languages/json5.tmLanguage.json').then((mod) => handleLang(mod.default, 'json5')),
'jsonc': () => import('shiki/languages/jsonc.tmLanguage.json').then((mod) => handleLang(mod.default, 'jsonc')),
'jsonnet': () => import('shiki/languages/jsonnet.tmLanguage.json').then((mod) => handleLang(mod.default, 'jsonnet')),
'jssm': () => import('shiki/languages/jssm.tmLanguage.json').then((mod) => handleLang(mod.default, 'jssm')),
'jsx': () => import('shiki/languages/jsx.tmLanguage.json').then((mod) => handleLang(mod.default, 'jsx')),
'julia': () => import('shiki/languages/julia.tmLanguage.json').then((mod) => handleLang(mod.default, 'julia')),
'kotlin': () => import('shiki/languages/kotlin.tmLanguage.json').then((mod) => handleLang(mod.default, 'kotlin')),
'latex': () => import('shiki/languages/latex.tmLanguage.json').then((mod) => handleLang(mod.default, 'latex')),
'less': () => import('shiki/languages/less.tmLanguage.json').then((mod) => handleLang(mod.default, 'less')),
'liquid': () => import('shiki/languages/liquid.tmLanguage.json').then((mod) => handleLang(mod.default, 'liquid')),
'lisp': () => import('shiki/languages/lisp.tmLanguage.json').then((mod) => handleLang(mod.default, 'lisp')),
'logo': () => import('shiki/languages/logo.tmLanguage.json').then((mod) => handleLang(mod.default, 'logo')),
'lua': () => import('shiki/languages/lua.tmLanguage.json').then((mod) => handleLang(mod.default, 'lua')),
'make': () => import('shiki/languages/make.tmLanguage.json').then((mod) => handleLang(mod.default, 'make')),
'markdown': () => import('shiki/languages/markdown.tmLanguage.json').then((mod) => handleLang(mod.default, 'markdown')),
'marko': () => import('shiki/languages/marko.tmLanguage.json').then((mod) => handleLang(mod.default, 'marko')),
'matlab': () => import('shiki/languages/matlab.tmLanguage.json').then((mod) => handleLang(mod.default, 'matlab')),
'mdx': () => import('shiki/languages/mdx.tmLanguage.json').then((mod) => handleLang(mod.default, 'mdx')),
'mermaid': () => import('shiki/languages/mermaid.tmLanguage.json').then((mod) => handleLang(mod.default, 'mermaid')),
'nginx': () => import('shiki/languages/nginx.tmLanguage.json').then((mod) => handleLang(mod.default, 'nginx')),
'nim': () => import('shiki/languages/nim.tmLanguage.json').then((mod) => handleLang(mod.default, 'nim')),
'nix': () => import('shiki/languages/nix.tmLanguage.json').then((mod) => handleLang(mod.default, 'nix')),
'objective-c': () => import('shiki/languages/objective-c.tmLanguage.json').then((mod) => handleLang(mod.default, 'objective-c')),
'objective-cpp': () => import('shiki/languages/objective-cpp.tmLanguage.json').then((mod) => handleLang(mod.default, 'objective-cpp')),
'ocaml': () => import('shiki/languages/ocaml.tmLanguage.json').then((mod) => handleLang(mod.default, 'ocaml')),
'pascal': () => import('shiki/languages/pascal.tmLanguage.json').then((mod) => handleLang(mod.default, 'pascal')),
'perl': () => import('shiki/languages/perl.tmLanguage.json').then((mod) => handleLang(mod.default, 'perl')),
'php-html': () => import('shiki/languages/php-html.tmLanguage.json').then((mod) => handleLang(mod.default, 'php-html')),
'php': () => import('shiki/languages/php.tmLanguage.json').then((mod) => handleLang(mod.default, 'php')),
'plsql': () => import('shiki/languages/plsql.tmLanguage.json').then((mod) => handleLang(mod.default, 'plsql')),
'postcss': () => import('shiki/languages/postcss.tmLanguage.json').then((mod) => handleLang(mod.default, 'postcss')),
'powerquery': () => import('shiki/languages/powerquery.tmLanguage.json').then((mod) => handleLang(mod.default, 'powerquery')),
'powershell': () => import('shiki/languages/powershell.tmLanguage.json').then((mod) => handleLang(mod.default, 'powershell')),
'prisma': () => import('shiki/languages/prisma.tmLanguage.json').then((mod) => handleLang(mod.default, 'prisma')),
'prolog': () => import('shiki/languages/prolog.tmLanguage.json').then((mod) => handleLang(mod.default, 'prolog')),
'proto': () => import('shiki/languages/proto.tmLanguage.json').then((mod) => handleLang(mod.default, 'proto')),
'pug': () => import('shiki/languages/pug.tmLanguage.json').then((mod) => handleLang(mod.default, 'pug')),
'puppet': () => import('shiki/languages/puppet.tmLanguage.json').then((mod) => handleLang(mod.default, 'puppet')),
'purescript': () => import('shiki/languages/purescript.tmLanguage.json').then((mod) => handleLang(mod.default, 'purescript')),
'python': () => import('shiki/languages/python.tmLanguage.json').then((mod) => handleLang(mod.default, 'python')),
'r': () => import('shiki/languages/r.tmLanguage.json').then((mod) => handleLang(mod.default, 'r')),
'raku': () => import('shiki/languages/raku.tmLanguage.json').then((mod) => handleLang(mod.default, 'raku')),
'razor': () => import('shiki/languages/razor.tmLanguage.json').then((mod) => handleLang(mod.default, 'razor')),
'rel': () => import('shiki/languages/rel.tmLanguage.json').then((mod) => handleLang(mod.default, 'rel')),
'riscv': () => import('shiki/languages/riscv.tmLanguage.json').then((mod) => handleLang(mod.default, 'riscv')),
'rst': () => import('shiki/languages/rst.tmLanguage.json').then((mod) => handleLang(mod.default, 'rst')),
'ruby': () => import('shiki/languages/ruby.tmLanguage.json').then((mod) => handleLang(mod.default, 'ruby')),
'rust': () => import('shiki/languages/rust.tmLanguage.json').then((mod) => handleLang(mod.default, 'rust')),
'sas': () => import('shiki/languages/sas.tmLanguage.json').then((mod) => handleLang(mod.default, 'sas')),
'sass': () => import('shiki/languages/sass.tmLanguage.json').then((mod) => handleLang(mod.default, 'sass')),
'scala': () => import('shiki/languages/scala.tmLanguage.json').then((mod) => handleLang(mod.default, 'scala')),
'scheme': () => import('shiki/languages/scheme.tmLanguage.json').then((mod) => handleLang(mod.default, 'scheme')),
'scss': () => import('shiki/languages/scss.tmLanguage.json').then((mod) => handleLang(mod.default, 'scss')),
'shaderlab': () => import('shiki/languages/shaderlab.tmLanguage.json').then((mod) => handleLang(mod.default, 'shaderlab')),
'shellscript': () => import('shiki/languages/shellscript.tmLanguage.json').then((mod) => handleLang(mod.default, 'shellscript')),
'smalltalk': () => import('shiki/languages/smalltalk.tmLanguage.json').then((mod) => handleLang(mod.default, 'smalltalk')),
'solidity': () => import('shiki/languages/solidity.tmLanguage.json').then((mod) => handleLang(mod.default, 'solidity')),
'sparql': () => import('shiki/languages/sparql.tmLanguage.json').then((mod) => handleLang(mod.default, 'sparql')),
'sql': () => import('shiki/languages/sql.tmLanguage.json').then((mod) => handleLang(mod.default, 'sql')),
'ssh-config': () => import('shiki/languages/ssh-config.tmLanguage.json').then((mod) => handleLang(mod.default, 'ssh-config')),
'stata': () => import('shiki/languages/stata.tmLanguage.json').then((mod) => handleLang(mod.default, 'stata')),
'stylus': () => import('shiki/languages/stylus.tmLanguage.json').then((mod) => handleLang(mod.default, 'stylus')),
'svelte': () => import('shiki/languages/svelte.tmLanguage.json').then((mod) => handleLang(mod.default, 'svelte')),
'swift': () => import('shiki/languages/swift.tmLanguage.json').then((mod) => handleLang(mod.default, 'swift')),
'system-verilog': () => import('shiki/languages/system-verilog.tmLanguage.json').then((mod) => handleLang(mod.default, 'system-verilog')),
'tasl': () => import('shiki/languages/tasl.tmLanguage.json').then((mod) => handleLang(mod.default, 'tasl')),
'tcl': () => import('shiki/languages/tcl.tmLanguage.json').then((mod) => handleLang(mod.default, 'tcl')),
'tex': () => import('shiki/languages/tex.tmLanguage.json').then((mod) => handleLang(mod.default, 'tex')),
'toml': () => import('shiki/languages/toml.tmLanguage.json').then((mod) => handleLang(mod.default, 'toml')),
'tsx': () => import('shiki/languages/tsx.tmLanguage.json').then((mod) => handleLang(mod.default, 'tsx')),
'turtle': () => import('shiki/languages/turtle.tmLanguage.json').then((mod) => handleLang(mod.default, 'turtle')),
'twig': () => import('shiki/languages/twig.tmLanguage.json').then((mod) => handleLang(mod.default, 'twig')),
'typescript': () => import('shiki/languages/typescript.tmLanguage.json').then((mod) => handleLang(mod.default, 'typescript')),
'v': () => import('shiki/languages/v.tmLanguage.json').then((mod) => handleLang(mod.default, 'v')),
'vb': () => import('shiki/languages/vb.tmLanguage.json').then((mod) => handleLang(mod.default, 'vb')),
'verilog': () => import('shiki/languages/verilog.tmLanguage.json').then((mod) => handleLang(mod.default, 'verilog')),
'vhdl': () => import('shiki/languages/vhdl.tmLanguage.json').then((mod) => handleLang(mod.default, 'vhdl')),
'viml': () => import('shiki/languages/viml.tmLanguage.json').then((mod) => handleLang(mod.default, 'viml')),
'vue-html': () => import('shiki/languages/vue-html.tmLanguage.json').then((mod) => handleLang(mod.default, 'vue-html')),
'vue': () => import('shiki/languages/vue.tmLanguage.json').then((mod) => handleLang(mod.default, 'vue')),
'wasm': () => import('shiki/languages/wasm.tmLanguage.json').then((mod) => handleLang(mod.default, 'wasm')),
'wenyan': () => import('shiki/languages/wenyan.tmLanguage.json').then((mod) => handleLang(mod.default, 'wenyan')),
'wgsl': () => import('shiki/languages/wgsl.tmLanguage.json').then((mod) => handleLang(mod.default, 'wgsl')),
'xml': () => import('shiki/languages/xml.tmLanguage.json').then((mod) => handleLang(mod.default, 'xml')),
'xsl': () => import('shiki/languages/xsl.tmLanguage.json').then((mod) => handleLang(mod.default, 'xsl')),
'yaml': () => import('shiki/languages/yaml.tmLanguage.json').then((mod) => handleLang(mod.default, 'yaml')),
'zenscript': () => import('shiki/languages/zenscript.tmLanguage.json').then((mod) => handleLang(mod.default, 'zenscript')),
};

View file

@ -1,37 +0,0 @@
/**
* This file is prebuilt from packages/astro/scripts/shiki-gen-themes.mjs
* Do not edit this directly, but instead edit that file and rerun it to generate this file.
*/
// prettier-ignore
export const themes = {
'css-variables': () => import('shiki/themes/css-variables.json').then(mod => mod.default),
'dark-plus': () => import('shiki/themes/dark-plus.json').then(mod => mod.default),
'dracula-soft': () => import('shiki/themes/dracula-soft.json').then(mod => mod.default),
'dracula': () => import('shiki/themes/dracula.json').then(mod => mod.default),
'github-dark-dimmed': () => import('shiki/themes/github-dark-dimmed.json').then(mod => mod.default),
'github-dark': () => import('shiki/themes/github-dark.json').then(mod => mod.default),
'github-light': () => import('shiki/themes/github-light.json').then(mod => mod.default),
'hc_light': () => import('shiki/themes/hc_light.json').then(mod => mod.default),
'light-plus': () => import('shiki/themes/light-plus.json').then(mod => mod.default),
'material-theme-darker': () => import('shiki/themes/material-theme-darker.json').then(mod => mod.default),
'material-theme-lighter': () => import('shiki/themes/material-theme-lighter.json').then(mod => mod.default),
'material-theme-ocean': () => import('shiki/themes/material-theme-ocean.json').then(mod => mod.default),
'material-theme-palenight': () => import('shiki/themes/material-theme-palenight.json').then(mod => mod.default),
'material-theme': () => import('shiki/themes/material-theme.json').then(mod => mod.default),
'min-dark': () => import('shiki/themes/min-dark.json').then(mod => mod.default),
'min-light': () => import('shiki/themes/min-light.json').then(mod => mod.default),
'monokai': () => import('shiki/themes/monokai.json').then(mod => mod.default),
'nord': () => import('shiki/themes/nord.json').then(mod => mod.default),
'one-dark-pro': () => import('shiki/themes/one-dark-pro.json').then(mod => mod.default),
'poimandres': () => import('shiki/themes/poimandres.json').then(mod => mod.default),
'rose-pine-dawn': () => import('shiki/themes/rose-pine-dawn.json').then(mod => mod.default),
'rose-pine-moon': () => import('shiki/themes/rose-pine-moon.json').then(mod => mod.default),
'rose-pine': () => import('shiki/themes/rose-pine.json').then(mod => mod.default),
'slack-dark': () => import('shiki/themes/slack-dark.json').then(mod => mod.default),
'slack-ochin': () => import('shiki/themes/slack-ochin.json').then(mod => mod.default),
'solarized-dark': () => import('shiki/themes/solarized-dark.json').then(mod => mod.default),
'solarized-light': () => import('shiki/themes/solarized-light.json').then(mod => mod.default),
'vitesse-dark': () => import('shiki/themes/vitesse-dark.json').then(mod => mod.default),
'vitesse-light': () => import('shiki/themes/vitesse-light.json').then(mod => mod.default),
};

View file

@ -0,0 +1,21 @@
import { type Highlighter, getHighlighter } from 'shikiji';
type HighlighterOptions = NonNullable<Parameters<typeof getHighlighter>[0]>;
// Caches Promise<Highlighter> for reuse when the same theme and langs are provided
const cachedHighlighters = new Map();
export function getCachedHighlighter(opts: HighlighterOptions): Promise<Highlighter> {
// Always sort keys before stringifying to make sure objects match regardless of parameter ordering
const key = JSON.stringify(opts, Object.keys(opts).sort());
// Highlighter has already been requested, reuse the same instance
if (cachedHighlighters.has(key)) {
return cachedHighlighters.get(key);
}
const highlighter = getHighlighter(opts);
cachedHighlighters.set(key, highlighter);
return highlighter;
}

View file

@ -162,7 +162,7 @@
"resolve": "^1.22.4",
"semver": "^7.5.4",
"server-destroy": "^1.0.1",
"shiki": "^0.14.3",
"shikiji": "^0.6.8",
"string-width": "^6.1.0",
"strip-ansi": "^7.1.0",
"tsconfig-resolver": "^3.0.1",

View file

@ -1,44 +0,0 @@
import fs from 'node:fs';
const dir = await fs.promises.readdir('packages/astro/node_modules/shiki/languages/');
const langImports = dir.map((f) => {
const key = f.slice(0, f.indexOf('.tmLanguage.json'));
return [key, `import('shiki/languages/${f}').then((mod) => handleLang(mod.default, '${key}'))`];
});
let code = `\
/**
* This file is prebuilt from packages/astro/scripts/shiki-gen-languages.mjs
* Do not edit this directly, but instead edit that file and rerun it to generate this file.
*/
import { BUNDLED_LANGUAGES } from 'shiki';
function handleLang(grammar, language) {
const lang = BUNDLED_LANGUAGES.find((l) => l.id === language);
if (lang) {
return {
...lang,
grammar,
};
} else {
return undefined;
}
}
// prettier-ignore
export const languages = {`;
for (const [key, imp] of langImports) {
code += `\n\t'${key}': () => ${imp},`;
}
code += '\n};';
// eslint-disable-next-line no-console
console.log(code);
/**
* Run this script and pipe it into the output file, for ex.
* node packages/astro/scripts/shiki-gen-languages.mjs > packages/astro/components/shiki-languages.js
*/

View file

@ -1,31 +0,0 @@
import fs from 'node:fs';
const dir = await fs.promises.readdir('packages/astro/node_modules/shiki/themes/');
const toThemeImport = (theme) => `import('shiki/themes/${theme}').then(mod => mod.default)`;
const themeImports = dir.map((f) => {
return [f.slice(0, f.indexOf('.json')), toThemeImport(f)];
});
let code = `\
/**
* This file is prebuilt from packages/astro/scripts/shiki-gen-themes.mjs
* Do not edit this directly, but instead edit that file and rerun it to generate this file.
*/
// prettier-ignore
export const themes = {`;
for (const [key, imp] of themeImports) {
code += `\n\t'${key}': () => ${imp},`;
}
code += '\n};';
// eslint-disable-next-line no-console
console.log(code);
/**
* Run this script and pipe it into the output file, for ex.
* node packages/astro/scripts/shiki-gen-themes.mjs > packages/astro/components/shiki-themes.js
*/

View file

@ -16,7 +16,6 @@ export function vitePluginInternals(input: Set<string>, internals: BuildInternal
// Except for these packages as they're not bundle-friendly. Users with strict package installations
// need to manually install these themselves if they use the related features.
external: [
'shiki', // For syntax highlighting
'sharp', // For sharp image service
],
},

View file

@ -1,15 +1,22 @@
import type { RehypePlugin, RemarkPlugin, RemarkRehype } from '@astrojs/markdown-remark';
import type {
RehypePlugin,
RemarkPlugin,
RemarkRehype,
ShikiConfig,
} from '@astrojs/markdown-remark';
import { markdownConfigDefaults } from '@astrojs/markdown-remark';
import type { ILanguageRegistration, IShikiTheme, Theme } from 'shiki';
import { bundledThemes, type BuiltinTheme } from 'shikiji';
import type { AstroUserConfig, ViteUserConfig } from '../../@types/astro';
import type { OutgoingHttpHeaders } from 'node:http';
import path from 'node:path';
import { pathToFileURL } from 'node:url';
import { BUNDLED_THEMES } from 'shiki';
import { z } from 'zod';
import { appendForwardSlash, prependForwardSlash, removeTrailingForwardSlash } from '../path.js';
type ShikiLangs = NonNullable<ShikiConfig['langs']>;
type ShikiTheme = NonNullable<ShikiConfig['theme']>;
const ASTRO_CONFIG_DEFAULTS = {
root: '.',
srcDir: './src',
@ -229,11 +236,30 @@ export const AstroConfigSchema = z.object({
.default(ASTRO_CONFIG_DEFAULTS.markdown.syntaxHighlight),
shikiConfig: z
.object({
langs: z.custom<ILanguageRegistration>().array().default([]),
langs: z
.custom<ShikiLangs[number]>()
.array()
.transform((langs) => {
for (const lang of langs) {
// shiki -> shikiji compat
if (typeof lang === 'object') {
// `id` renamed to `name
if ((lang as any).id && !lang.name) {
lang.name = (lang as any).id;
}
// `grammar` flattened to lang itself
if ((lang as any).grammar) {
Object.assign(lang, (lang as any).grammar);
}
}
}
return langs;
})
.default([]),
theme: z
.enum(BUNDLED_THEMES as [Theme, ...Theme[]])
.or(z.custom<IShikiTheme>())
.default(ASTRO_CONFIG_DEFAULTS.markdown.shikiConfig.theme! as Theme),
.enum(Object.keys(bundledThemes) as [BuiltinTheme, ...BuiltinTheme[]])
.or(z.custom<ShikiTheme>())
.default(ASTRO_CONFIG_DEFAULTS.markdown.shikiConfig.theme as BuiltinTheme),
wrap: z.boolean().or(z.null()).default(ASTRO_CONFIG_DEFAULTS.markdown.shikiConfig.wrap!),
})
.default({}),

View file

@ -55,8 +55,6 @@ const ALWAYS_NOEXTERNAL = [
// pipeline, which Vite doesn't support in development time. This hardcoded list temporarily
// fixes things until Vite can properly handle them, or when they support ESM.
const ONLY_DEV_EXTERNAL = [
// Imported by `<Code/>` which is processed by Vite
'shiki',
// Imported by `@astrojs/prism` which exposes `<Prism/>` that is processed by Vite
'prismjs/components/index.js',
// Imported by `astro/assets` -> `packages/astro/src/core/logger/core.ts`

View file

@ -1,6 +1,6 @@
import * as fs from 'node:fs';
import { fileURLToPath } from 'node:url';
import { getHighlighter } from 'shiki';
import { codeToHtml } from 'shikiji';
import type { ErrorPayload } from 'vite';
import type { ModuleLoader } from '../../module-loader/index.js';
import { FailedToLoadModuleSSR, InvalidGlob, MdxIntegrationMissingError } from '../errors-data.js';
@ -139,7 +139,6 @@ export async function getViteErrorPayload(err: ErrorWithMetadata): Promise<Astro
const docslink = getDocsForError(err);
const highlighter = await getHighlighter({ theme: 'css-variables' });
let highlighterLang = err.loc?.file?.split('.').pop();
if (ALTERNATIVE_JS_EXTS.includes(highlighterLang ?? '')) {
highlighterLang = 'js';
@ -148,8 +147,10 @@ export async function getViteErrorPayload(err: ErrorWithMetadata): Promise<Astro
highlighterLang = 'md';
}
const highlightedCode = err.fullCode
? highlighter.codeToHtml(err.fullCode, {
? await codeToHtml(err.fullCode, {
// @ts-expect-error always assume that shiki can accept the lang string
lang: highlighterLang,
theme: 'css-variables',
lineOptions: err.loc?.line ? [{ line: err.loc.line, classes: ['error-line'] }] : undefined,
})
: undefined;

View file

@ -15,7 +15,7 @@ describe('<Code>', () => {
const $ = cheerio.load(html);
expect($('pre')).to.have.lengthOf(1);
expect($('pre').attr('style')).to.equal(
'background-color: #24292e; overflow-x: auto;',
'-background-color:#24292e;color:#e1e4e8; overflow-x: auto;',
'applies default and overflow'
);
expect($('pre > code')).to.have.lengthOf(1);
@ -40,7 +40,7 @@ describe('<Code>', () => {
expect($('pre')).to.have.lengthOf(1);
expect($('pre').attr('class')).to.equal('astro-code nord');
expect($('pre').attr('style')).to.equal(
'background-color: #2e3440ff; overflow-x: auto;',
'background-color:#2e3440ff;color:#d8dee9ff; overflow-x: auto;',
'applies custom theme'
);
});
@ -52,7 +52,7 @@ describe('<Code>', () => {
expect($('pre')).to.have.lengthOf(1);
// test: applies wrap overflow
expect($('pre').attr('style')).to.equal(
'background-color: #24292e; overflow-x: auto; white-space: pre-wrap; word-wrap: break-word;'
'background-color:#24292e;color:#e1e4e8; overflow-x: auto; white-space: pre-wrap; word-wrap: break-word;'
);
}
{
@ -76,6 +76,7 @@ describe('<Code>', () => {
const $ = cheerio.load(html);
expect($('pre')).to.have.lengthOf(1);
expect($('pre').attr('class')).to.equal('astro-code css-variables');
// TODO: We can't specify CSS vars with shikiji
expect(
$('pre, pre span')
.map((i, f) => (f.attribs ? f.attribs.style : 'no style found'))

View file

@ -20,7 +20,9 @@ describe('Astro Markdown Shiki', () => {
expect($('pre')).to.have.lengthOf(2);
expect($('pre').hasClass('astro-code')).to.equal(true);
expect($('pre').attr().style).to.equal('background-color: #24292e; overflow-x: auto;');
expect($('pre').attr().style).to.equal(
'background-color:#24292e;color:#e1e4e8; overflow-x: auto;'
);
});
it('Can render diff syntax with "user-select: none"', async () => {
@ -47,7 +49,9 @@ describe('Astro Markdown Shiki', () => {
expect($('pre')).to.have.lengthOf(1);
expect($('pre').hasClass('astro-code')).to.equal(true);
expect($('pre').attr().style).to.equal('background-color: #fff; overflow-x: auto;');
expect($('pre').attr().style).to.equal(
'background-color:#fff;color:#24292e; overflow-x: auto;'
);
});
});
@ -65,7 +69,9 @@ describe('Astro Markdown Shiki', () => {
expect($('pre')).to.have.lengthOf(1);
expect($('pre').hasClass('astro-code')).to.equal(true);
expect($('pre').attr().style).to.equal('background-color: #FDFDFE; overflow-x: auto;');
expect($('pre').attr().style).to.equal(
'background-color:#FDFDFE;color:#4E5377; overflow-x: auto;'
);
});
});
});
@ -83,12 +89,12 @@ describe('Astro Markdown Shiki', () => {
const $ = cheerio.load(html);
const segments = $('.line').get(6).children;
expect(segments).to.have.lengthOf(3);
expect(segments[0].attribs.style).to.be.equal('color: #E1E4E8');
expect(segments[1].attribs.style).to.be.equal('color: #79B8FF');
expect(segments[2].attribs.style).to.be.equal('color: #E1E4E8');
expect(segments).to.have.lengthOf(2);
expect(segments[0].attribs.style).to.be.equal('color:#79B8FF');
expect(segments[1].attribs.style).to.be.equal('color:#E1E4E8');
const unknownLang = $('.line').last().html();
// TODO: There's a shikiji bug where it's style=""
expect(unknownLang).to.be.equal(
'<span style="color: #e1e4e8">This language does not exist</span>'
);
@ -98,7 +104,7 @@ describe('Astro Markdown Shiki', () => {
describe('Wrap', () => {
describe('wrap = true', () => {
const style =
'background-color: #24292e; overflow-x: auto; white-space: pre-wrap; word-wrap: break-word;';
'background-color:#24292e;color:#e1e4e8; overflow-x: auto; white-space: pre-wrap; word-wrap: break-word;';
let fixture;
before(async () => {
@ -117,7 +123,7 @@ describe('Astro Markdown Shiki', () => {
});
describe('wrap = false', () => {
const style = 'background-color: #24292e; overflow-x: auto;';
const style = 'background-color:#24292e;color:#e1e4e8; overflow-x: auto;';
let fixture;
before(async () => {
@ -135,7 +141,7 @@ describe('Astro Markdown Shiki', () => {
});
describe('wrap = null', () => {
const style = 'background-color: #24292e';
const style = 'background-color:#24292e;color:#e1e4e8';
let fixture;
before(async () => {

View file

@ -1,45 +0,0 @@
import { expect } from 'chai';
import { fileURLToPath } from 'node:url';
import { createContainer } from '../../../dist/core/dev/index.js';
import { createViteLoader } from '../../../dist/core/module-loader/index.js';
import { createBasicSettings, defaultLogger } from '../test-utils.js';
const root = new URL('../../fixtures/alias/', import.meta.url);
describe('<Code />', () => {
describe('Shiki - getHighlighterOptions', () => {
let container;
let mod;
before(async () => {
const settings = await createBasicSettings({ root: fileURLToPath(root) });
container = await createContainer({ settings, logger: defaultLogger });
const loader = createViteLoader(container.viteServer);
mod = await loader.import('astro/components/Shiki.js');
});
after(async () => {
await container.close();
});
it('uses the bundles themes for built-in themes', async () => {
const { resolveHighlighterOptions } = mod;
// NOTE: pass empty `langs` to prevent Shiki from loading all langs by default, which slows down the test
const opts = await resolveHighlighterOptions({ theme: 'css-variables', langs: [] });
const themes = opts.themes;
expect(themes).to.have.a.lengthOf(1);
expect(themes[0]).to.be.an('object');
});
it('uses the string theme name for custom themes', async () => {
const { resolveHighlighterOptions } = mod;
// NOTE: pass empty `langs` to prevent Shiki from loading all langs by default, which slows down the test
const opts = await resolveHighlighterOptions({ theme: 'some-custom-theme', langs: [] });
const themes = opts.themes;
expect(themes).to.have.a.lengthOf(1);
expect(themes[0]).to.be.an('string');
expect(themes[0]).to.equal('some-custom-theme');
});
});
});

View file

@ -71,7 +71,7 @@
"gray-matter": "^4.0.3",
"htmlparser2": "^9.0.0",
"kleur": "^4.1.5",
"shiki": "^0.14.3",
"shikiji": "^0.6.8",
"zod": "3.21.1"
},
"peerDependencies": {

View file

@ -1,8 +1,7 @@
import Markdoc from '@markdoc/markdoc';
import type { ShikiConfig } from 'astro';
import { unescapeHTML } from 'astro/runtime/server/index.js';
import type * as shikiTypes from 'shiki';
import { getHighlighter } from 'shiki';
import { bundledLanguages, getHighlighter, type Highlighter } from 'shikiji';
import type { AstroMarkdocConfig } from '../config.js';
const ASTRO_COLOR_REPLACEMENTS = {
@ -17,7 +16,7 @@ const ASTRO_COLOR_REPLACEMENTS = {
'#000010': 'var(--astro-code-token-string-expression)',
'#000011': 'var(--astro-code-token-punctuation)',
'#000012': 'var(--astro-code-token-link)',
};
} as const;
const PRE_SELECTOR = /<pre class="(.*?)shiki(.*?)"/;
const LINE_SELECTOR = /<span class="line"><span style="(.*?)">([\+|\-])/g;
@ -27,28 +26,23 @@ const INLINE_STYLE_SELECTOR = /style="(.*?)"/;
* Note: cache only needed for dev server reloads, internal test suites, and manual calls to `Markdoc.transform` by the user.
* Otherwise, `shiki()` is only called once per build, NOT once per page, so a cache isn't needed!
*/
const highlighterCache = new Map<string, shikiTypes.Highlighter>();
const highlighterCache = new Map<string, Highlighter>();
export default async function shiki({
langs = [],
theme = 'github-dark',
wrap = false,
}: ShikiConfig = {}): Promise<AstroMarkdocConfig> {
const cacheID: string = typeof theme === 'string' ? theme : theme.name;
if (!highlighterCache.has(cacheID)) {
highlighterCache.set(
cacheID,
await getHighlighter({ theme }).then((hl) => {
hl.setColorReplacements(ASTRO_COLOR_REPLACEMENTS);
return hl;
})
);
const cacheId = typeof theme === 'string' ? theme : theme.name || '';
let highlighter = highlighterCache.get(cacheId)!;
if (!highlighter) {
highlighter = await getHighlighter({
langs: langs.length ? langs : Object.keys(bundledLanguages),
themes: [theme],
});
highlighterCache.set(cacheId, highlighter);
}
const highlighter = highlighterCache.get(cacheID)!;
for (const lang of langs) {
await highlighter.loadLanguage(lang);
}
return {
nodes: {
fence: {
@ -72,7 +66,7 @@ export default async function shiki({
lang = 'plaintext';
}
let html = highlighter.codeToHtml(attributes.content, { lang });
let html = highlighter.codeToHtml(attributes.content, { lang, theme });
// Q: Could these regexes match on a user's inputted code blocks?
// A: Nope! All rendered HTML is properly escaped.

View file

@ -5,8 +5,5 @@
"dependencies": {
"@astrojs/markdoc": "workspace:*",
"astro": "workspace:*"
},
"devDependencies": {
"shiki": "^0.14.3"
}
}

View file

@ -40,7 +40,7 @@
"remark-parse": "^10.0.2",
"remark-rehype": "^10.1.0",
"remark-smartypants": "^2.0.0",
"shiki": "^0.14.3",
"shikiji": "^0.6.8",
"unified": "^10.1.2",
"unist-util-visit": "^4.1.2",
"vfile": "^5.3.7"

View file

@ -1,5 +1,4 @@
import type * as shiki from 'shiki';
import { getHighlighter } from 'shiki';
import { bundledLanguages, getHighlighter, type Highlighter } from 'shikiji';
import { visit } from 'unist-util-visit';
import type { RemarkPlugin, ShikiConfig } from './types.js';
@ -8,49 +7,28 @@ import type { RemarkPlugin, ShikiConfig } from './types.js';
* cache it here as much as possible. Make sure that your highlighters can be cached, state-free.
* We make this async, so that multiple calls to parse markdown still share the same highlighter.
*/
const highlighterCacheAsync = new Map<string, Promise<shiki.Highlighter>>();
const highlighterCacheAsync = new Map<string, Promise<Highlighter>>();
export function remarkShiki({
langs = [],
theme = 'github-dark',
wrap = false,
}: ShikiConfig = {}): ReturnType<RemarkPlugin> {
const cacheID: string = typeof theme === 'string' ? theme : theme.name;
let highlighterAsync = highlighterCacheAsync.get(cacheID);
const cacheId =
(typeof theme === 'string' ? theme : theme.name ?? '') +
langs.map((l) => l.name ?? (l as any).id).join(',');
let highlighterAsync = highlighterCacheAsync.get(cacheId);
if (!highlighterAsync) {
highlighterAsync = getHighlighter({ theme }).then((hl) => {
hl.setColorReplacements({
'#000001': 'var(--astro-code-color-text)',
'#000002': 'var(--astro-code-color-background)',
'#000004': 'var(--astro-code-token-constant)',
'#000005': 'var(--astro-code-token-string)',
'#000006': 'var(--astro-code-token-comment)',
'#000007': 'var(--astro-code-token-keyword)',
'#000008': 'var(--astro-code-token-parameter)',
'#000009': 'var(--astro-code-token-function)',
'#000010': 'var(--astro-code-token-string-expression)',
'#000011': 'var(--astro-code-token-punctuation)',
'#000012': 'var(--astro-code-token-link)',
});
return hl;
highlighterAsync = getHighlighter({
langs: langs.length ? langs : Object.keys(bundledLanguages),
themes: [theme],
});
highlighterCacheAsync.set(cacheID, highlighterAsync);
highlighterCacheAsync.set(cacheId, highlighterAsync);
}
let highlighter: shiki.Highlighter;
return async (tree: any) => {
// Lazily assign the highlighter as async can only happen within this function,
// and not on `remarkShiki` directly.
if (!highlighter) {
highlighter = await highlighterAsync!;
// NOTE: There may be a performance issue here for large sites that use `lang`.
// Since this will be called on every page load. Unclear how to fix this.
for (const lang of langs) {
await highlighter.loadLanguage(lang);
}
}
const highlighter = await highlighterAsync!;
visit(tree, 'code', (node) => {
let lang: string;
@ -68,7 +46,7 @@ export function remarkShiki({
lang = 'plaintext';
}
let html = highlighter.codeToHtml(node.value, { lang });
let html = highlighter.codeToHtml(node.value, { lang, theme });
// Q: Couldn't these regexes match on a user's inputted code blocks?
// A: Nope! All rendered HTML is properly escaped.

View file

@ -5,7 +5,12 @@ import type {
all as Handlers,
Options as RemarkRehypeOptions,
} from 'remark-rehype';
import type { ILanguageRegistration, IThemeRegistration, Theme } from 'shiki';
import type {
BuiltinTheme,
LanguageRegistration,
ThemeRegistration,
ThemeRegistrationRaw,
} from 'shikiji';
import type * as unified from 'unified';
import type { VFile } from 'vfile';
@ -35,8 +40,8 @@ export type RemarkRehype = Omit<RemarkRehypeOptions, 'handlers' | 'unknownHandle
};
export interface ShikiConfig {
langs?: ILanguageRegistration[];
theme?: Theme | IThemeRegistration;
langs?: LanguageRegistration[];
theme?: BuiltinTheme | ThemeRegistration | ThemeRegistrationRaw;
wrap?: boolean | null;
}

View file

@ -611,9 +611,9 @@ importers:
server-destroy:
specifier: ^1.0.1
version: 1.0.1
shiki:
specifier: ^0.14.3
version: 0.14.3
shikiji:
specifier: ^0.6.8
version: 0.6.8
string-width:
specifier: ^6.1.0
version: 6.1.0
@ -3829,9 +3829,9 @@ importers:
kleur:
specifier: ^4.1.5
version: 4.1.5
shiki:
specifier: ^0.14.3
version: 0.14.3
shikiji:
specifier: ^0.6.8
version: 0.6.8
zod:
specifier: 3.21.1
version: 3.21.1
@ -3956,10 +3956,6 @@ importers:
astro:
specifier: workspace:*
version: link:../../../../../astro
devDependencies:
shiki:
specifier: ^0.14.3
version: 0.14.3
packages/integrations/markdoc/test/fixtures/render-with-config:
dependencies:
@ -4915,9 +4911,9 @@ importers:
remark-smartypants:
specifier: ^2.0.0
version: 2.0.0
shiki:
specifier: ^0.14.3
version: 0.14.3
shikiji:
specifier: ^0.6.8
version: 0.6.8
unified:
specifier: ^10.1.2
version: 10.1.2
@ -9086,6 +9082,10 @@ packages:
- supports-color
dev: true
/@ungap/structured-clone@1.2.0:
resolution: {integrity: sha512-zuVdFrMJiuCDQUMCzQaD6KL28MjnqqN8XnAqiEq9PNm/hCPTSGfrXCOfwj1ow4LFb/tNymJPwsNbVePc1xFqrQ==}
dev: false
/@vercel/analytics@1.0.2:
resolution: {integrity: sha512-BZFxVrv24VbNNl5xMxqUojQIegEeXMI6rX3rg1uVLYUEXsuKNBSAEQf4BWEcjQDp/8aYJOj6m8V4PUA3x/cxgg==}
dev: false
@ -9514,9 +9514,6 @@ packages:
resolution: {integrity: sha512-n5M855fKb2SsfMIiFFoVrABHJC8QtHwVx+mHWP3QcEqBHYienj5dHSgjbxtC0WEZXYt4wcD6zrQElDPhFuZgfA==}
engines: {node: '>=12'}
/ansi-sequence-parser@1.1.0:
resolution: {integrity: sha512-lEm8mt52to2fT8GhciPCGeCXACSz2UwIN4X2e2LJSnZ5uAbn2/dsYdOmUXq0AtWS5cpAupysIneExOgH0Vd2TQ==}
/ansi-styles@3.2.1:
resolution: {integrity: sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==}
engines: {node: '>=4'}
@ -12071,6 +12068,19 @@ packages:
web-namespaces: 2.0.1
dev: false
/hast-util-from-parse5@8.0.1:
resolution: {integrity: sha512-Er/Iixbc7IEa7r/XLtuG52zoqn/b3Xng/w6aZQ0xGVxzhw5xUFxcRqdPzP6yFi/4HBYRaifaI5fQ1RH8n0ZeOQ==}
dependencies:
'@types/hast': 3.0.0
'@types/unist': 3.0.0
devlop: 1.1.0
hastscript: 8.0.0
property-information: 6.2.0
vfile: 6.0.1
vfile-location: 5.0.2
web-namespaces: 2.0.1
dev: false
/hast-util-has-property@2.0.1:
resolution: {integrity: sha512-X2+RwZIMTMKpXUzlotatPzWj8bspCymtXH3cfG3iQKV+wPF53Vgaqxi/eLqGck0wKq1kS9nvoB1wchbCPEL8sg==}
@ -12096,6 +12106,12 @@ packages:
dependencies:
'@types/hast': 2.3.5
/hast-util-parse-selector@4.0.0:
resolution: {integrity: sha512-wkQCkSYoOGCRKERFWcxMVMOcYE2K1AaNLU8DXS9arxnLOUEWbOXKXiJUNzEpqZ3JOKpnha3jkFrumEjVliDe7A==}
dependencies:
'@types/hast': 3.0.0
dev: false
/hast-util-raw@7.2.3:
resolution: {integrity: sha512-RujVQfVsOrxzPOPSzZFiwofMArbQke6DJjnFfceiEbFh7S05CbPt0cYN+A5YeD3pso0JQk6O1aHBnx9+Pm2uqg==}
dependencies:
@ -12112,6 +12128,24 @@ packages:
zwitch: 2.0.4
dev: false
/hast-util-raw@9.0.1:
resolution: {integrity: sha512-5m1gmba658Q+lO5uqL5YNGQWeh1MYWZbZmWrM5lncdcuiXuo5E2HT/CIOp0rLF8ksfSwiCVJ3twlgVRyTGThGA==}
dependencies:
'@types/hast': 3.0.0
'@types/unist': 3.0.0
'@ungap/structured-clone': 1.2.0
hast-util-from-parse5: 8.0.1
hast-util-to-parse5: 8.0.0
html-void-elements: 3.0.0
mdast-util-to-hast: 13.0.2
parse5: 7.1.2
unist-util-position: 5.0.0
unist-util-visit: 5.0.0
vfile: 6.0.1
web-namespaces: 2.0.1
zwitch: 2.0.4
dev: false
/hast-util-select@5.0.5:
resolution: {integrity: sha512-QQhWMhgTFRhCaQdgTKzZ5g31GLQ9qRb1hZtDPMqQaOhpLBziWcshUS0uCR5IJ0U1jrK/mxg35fmcq+Dp/Cy2Aw==}
dependencies:
@ -12191,6 +12225,23 @@ packages:
zwitch: 2.0.4
dev: false
/hast-util-to-html@9.0.0:
resolution: {integrity: sha512-IVGhNgg7vANuUA2XKrT6sOIIPgaYZnmLx3l/CCOAK0PtgfoHrZwX7jCSYyFxHTrGmC6S9q8aQQekjp4JPZF+cw==}
dependencies:
'@types/hast': 3.0.0
'@types/unist': 3.0.0
ccount: 2.0.1
comma-separated-tokens: 2.0.3
hast-util-raw: 9.0.1
hast-util-whitespace: 3.0.0
html-void-elements: 3.0.0
mdast-util-to-hast: 13.0.2
property-information: 6.2.0
space-separated-tokens: 2.0.2
stringify-entities: 4.0.3
zwitch: 2.0.4
dev: false
/hast-util-to-parse5@7.1.0:
resolution: {integrity: sha512-YNRgAJkH2Jky5ySkIqFXTQiaqcAtJyVE+D5lkN6CdtOqrnkLfGYYrEcKuHOJZlp+MwjSwuD3fZuawI+sic/RBw==}
dependencies:
@ -12202,6 +12253,18 @@ packages:
zwitch: 2.0.4
dev: false
/hast-util-to-parse5@8.0.0:
resolution: {integrity: sha512-3KKrV5ZVI8if87DVSi1vDeByYrkGzg4mEfeu4alwgmmIeARiBLKCZS2uw5Gb6nU9x9Yufyj3iudm6i7nl52PFw==}
dependencies:
'@types/hast': 3.0.0
comma-separated-tokens: 2.0.3
devlop: 1.1.0
property-information: 6.2.0
space-separated-tokens: 2.0.2
web-namespaces: 2.0.1
zwitch: 2.0.4
dev: false
/hast-util-to-string@2.0.0:
resolution: {integrity: sha512-02AQ3vLhuH3FisaMM+i/9sm4OXGSq1UhOOCpTLLQtHdL3tZt7qil69r8M8iDkZYyC0HCFylcYoP+8IO7ddta1A==}
dependencies:
@ -12235,6 +12298,16 @@ packages:
property-information: 6.2.0
space-separated-tokens: 2.0.2
/hastscript@8.0.0:
resolution: {integrity: sha512-dMOtzCEd3ABUeSIISmrETiKuyydk1w0pa+gE/uormcTpSYuaNJPbX1NU3JLyscSLjwAQM8bWMhhIlnCqnRvDTw==}
dependencies:
'@types/hast': 3.0.0
comma-separated-tokens: 2.0.3
hast-util-parse-selector: 4.0.0
property-information: 6.2.0
space-separated-tokens: 2.0.2
dev: false
/hdr-histogram-js@3.0.0:
resolution: {integrity: sha512-/EpvQI2/Z98mNFYEnlqJ8Ogful8OpArLG/6Tf2bPnkutBVLIeMVNHjk1ZDfshF2BUweipzbk+dB1hgSB7SIakw==}
engines: {node: '>=14'}
@ -12293,6 +12366,10 @@ packages:
resolution: {integrity: sha512-0quDb7s97CfemeJAnW9wC0hw78MtW7NU3hqtCD75g2vFlDLt36llsYD7uB7SUzojLMP24N5IatXf7ylGXiGG9A==}
dev: false
/html-void-elements@3.0.0:
resolution: {integrity: sha512-bEqo66MRXsUGxWHV5IP0PUiAWwoEjba4VCzg0LjFJBpchPaTfyfCKTG6bc5F8ucKec3q5y6qOdGyYTSBEvhCrg==}
dev: false
/htmlparser2@8.0.2:
resolution: {integrity: sha512-GYdjWKDkbRLkZ5geuHs5NY1puJ+PXwP7+fHPRz06Eirsb9ugf6d8kkXav6ADhcODhFFPMIXyxkxSuMf3D6NCFA==}
dependencies:
@ -13442,6 +13519,19 @@ packages:
unist-util-position: 4.0.4
unist-util-visit: 4.1.2
/mdast-util-to-hast@13.0.2:
resolution: {integrity: sha512-U5I+500EOOw9e3ZrclN3Is3fRpw8c19SMyNZlZ2IS+7vLsNzb2Om11VpIVOR+/0137GhZsFEF6YiKD5+0Hr2Og==}
dependencies:
'@types/hast': 3.0.0
'@types/mdast': 4.0.0
'@ungap/structured-clone': 1.2.0
devlop: 1.1.0
micromark-util-sanitize-uri: 2.0.0
trim-lines: 3.0.1
unist-util-position: 5.0.0
unist-util-visit: 5.0.0
dev: false
/mdast-util-to-markdown@1.5.0:
resolution: {integrity: sha512-bbv7TPv/WC49thZPg3jXuqzuvI45IL2EVAr/KxF0BSdHsU0ceFHOmwQn6evxAh1GaoK/6GQ1wp4R4oW2+LFL/A==}
dependencies:
@ -13776,6 +13866,13 @@ packages:
micromark-util-symbol: 1.1.0
micromark-util-types: 1.1.0
/micromark-util-character@2.0.1:
resolution: {integrity: sha512-3wgnrmEAJ4T+mGXAUfMvMAbxU9RDG43XmGce4j6CwPtVxB3vfwXSZ6KhFwDzZ3mZHhmPimMAXg71veiBGzeAZw==}
dependencies:
micromark-util-symbol: 2.0.0
micromark-util-types: 2.0.0
dev: false
/micromark-util-chunked@1.0.0:
resolution: {integrity: sha512-5e8xTis5tEZKgesfbQMKRCyzvffRRUX+lK/y+DvsMFdabAicPkkZV6gO+FEWi9RfuKKoxxPwNL+dFF0SMImc1g==}
dependencies:
@ -13825,6 +13922,10 @@ packages:
/micromark-util-encode@1.0.1:
resolution: {integrity: sha512-U2s5YdnAYexjKDel31SVMPbfi+eF8y1U4pfiRW/Y8EFVCy/vgxk/2wWTxzcqE71LHtCuCzlBDRU2a5CQ5j+mQA==}
/micromark-util-encode@2.0.0:
resolution: {integrity: sha512-pS+ROfCXAGLWCOc8egcBvT0kf27GoWMqtdarNfDcjb6YLuV5cM3ioG45Ys2qOVqeqSbjaKg72vU+Wby3eddPsA==}
dev: false
/micromark-util-events-to-acorn@1.2.3:
resolution: {integrity: sha512-ij4X7Wuc4fED6UoLWkmo0xJQhsktfNh1J0m8g4PbIMPlx+ek/4YdW5mvbye8z/aZvAPUoxgXHrwVlXAPKMRp1w==}
dependencies:
@ -13870,6 +13971,14 @@ packages:
micromark-util-encode: 1.0.1
micromark-util-symbol: 1.1.0
/micromark-util-sanitize-uri@2.0.0:
resolution: {integrity: sha512-WhYv5UEcZrbAtlsnPuChHUAsu/iBPOVaEVsntLBIdpibO0ddy8OzavZz3iL2xVvBZOpolujSliP65Kq0/7KIYw==}
dependencies:
micromark-util-character: 2.0.1
micromark-util-encode: 2.0.0
micromark-util-symbol: 2.0.0
dev: false
/micromark-util-subtokenize@1.0.2:
resolution: {integrity: sha512-d90uqCnXp/cy4G881Ub4psE57Sf8YD0pim9QdjCRNjfas2M1u6Lbt+XZK9gnHL2XFhnozZiEdCa9CNfXSfQ6xA==}
dependencies:
@ -13884,12 +13993,20 @@ packages:
/micromark-util-symbol@1.1.0:
resolution: {integrity: sha512-uEjpEYY6KMs1g7QfJ2eX1SQEV+ZT4rUD3UcF6l57acZvLNK7PBZL+ty82Z1qhK1/yXIY4bdx04FKMgR0g4IAag==}
/micromark-util-symbol@2.0.0:
resolution: {integrity: sha512-8JZt9ElZ5kyTnO94muPxIGS8oyElRJaiJO8EzV6ZSyGQ1Is8xwl4Q45qU5UOg+bGH4AikWziz0iN4sFLWs8PGw==}
dev: false
/micromark-util-types@1.0.2:
resolution: {integrity: sha512-DCfg/T8fcrhrRKTPjRrw/5LLvdGV7BHySf/1LOZx7TzWZdYRjogNtyNq885z3nNallwr3QUKARjqvHqX1/7t+w==}
/micromark-util-types@1.1.0:
resolution: {integrity: sha512-ukRBgie8TIAcacscVHSiddHjO4k/q3pnedmzMQ4iwDcK0FtFCohKOlFbaOL/mPgfnPsL3C1ZyxJa4sbWrBl3jg==}
/micromark-util-types@2.0.0:
resolution: {integrity: sha512-oNh6S2WMHWRZrmutsRmDDfkzKtxF+bc2VxLC9dvtrDIRFln627VsFP6fLMgTryGDljgLPjkrzQSDcPrjPyDJ5w==}
dev: false
/micromark@3.1.0:
resolution: {integrity: sha512-6Mj0yHLdUZjHnOPgr5xfWIMqMWS12zDN6iws9SLuSz76W8jTtAv24MN4/CL7gJrl5vtxGInkkqDv/JIoRsQOvA==}
dependencies:
@ -16092,13 +16209,11 @@ packages:
vscode-textmate: 5.2.0
dev: true
/shiki@0.14.3:
resolution: {integrity: sha512-U3S/a+b0KS+UkTyMjoNojvTgrBHjgp7L6ovhFVZsXmBGnVdQ4K4U9oK0z63w538S91ATngv1vXigHCSWOwnr+g==}
/shikiji@0.6.8:
resolution: {integrity: sha512-K0axxNAdB9KvLUflU7QoLC7p6i2p1R2MFG0eP+iclbjtuEZqng99jHcg3VJL0GWRO67yozTICnykjo1HjOzdkg==}
dependencies:
ansi-sequence-parser: 1.1.0
jsonc-parser: 3.2.0
vscode-oniguruma: 1.7.0
vscode-textmate: 8.0.0
hast-util-to-html: 9.0.0
dev: false
/side-channel@1.0.4:
resolution: {integrity: sha512-q5XPytqFEIKHkGdiMIrY10mvLRvnQh42/+GoBlFW3b2LXLE2xxJpZFdm94we0BaoV3RwJyGqg5wS7epxTv0Zvw==}
@ -17222,6 +17337,12 @@ packages:
dependencies:
'@types/unist': 2.0.7
/unist-util-position@5.0.0:
resolution: {integrity: sha512-fucsC7HjXvkB5R3kTCO7kUjRdrS0BJt3M/FPxmHMBOm8JQi2BsHAHFsy27E0EolP8rp0NzXsJ+jNPyDWvOJZPA==}
dependencies:
'@types/unist': 3.0.0
dev: false
/unist-util-remove-position@4.0.2:
resolution: {integrity: sha512-TkBb0HABNmxzAcfLf4qsIbFbaPDvMO6wa3b3j4VcEzFVaw1LBKwnW4/sRJ/atSLSzoIg41JWEdnE7N6DIhGDGQ==}
dependencies:
@ -17242,6 +17363,12 @@ packages:
dependencies:
'@types/unist': 2.0.7
/unist-util-stringify-position@4.0.0:
resolution: {integrity: sha512-0ASV06AAoKCDkS2+xw5RXJywruurpbC4JZSm7nr7MOt1ojAzvyyaO+UxZf18j8FCF6kmzCZKcAgN/yu2gm2XgQ==}
dependencies:
'@types/unist': 3.0.0
dev: false
/unist-util-visit-children@2.0.2:
resolution: {integrity: sha512-+LWpMFqyUwLGpsQxpumsQ9o9DG2VGLFrpz+rpVXYIEdPy57GSy5HioC0g3bg/8WP9oCLlapQtklOzQ8uLS496Q==}
dependencies:
@ -17421,12 +17548,26 @@ packages:
vfile: 5.3.7
dev: false
/vfile-location@5.0.2:
resolution: {integrity: sha512-NXPYyxyBSH7zB5U6+3uDdd6Nybz6o6/od9rk8bp9H8GR3L+cm/fC0uUTbqBmUTnMCUDslAGBOIKNfvvb+gGlDg==}
dependencies:
'@types/unist': 3.0.0
vfile: 6.0.1
dev: false
/vfile-message@3.1.4:
resolution: {integrity: sha512-fa0Z6P8HUrQN4BZaX05SIVXic+7kE3b05PWAtPuYP9QLHsLKYR7/AlLW3NtOrpXRLeawpDLMsVkmk5DG0NXgWw==}
dependencies:
'@types/unist': 2.0.7
unist-util-stringify-position: 3.0.3
/vfile-message@4.0.2:
resolution: {integrity: sha512-jRDZ1IMLttGj41KcZvlrYAaI3CfqpLpfpf+Mfig13viT6NKvRzWZ+lXz0Y5D60w6uJIBAOGq9mSHf0gktF0duw==}
dependencies:
'@types/unist': 3.0.0
unist-util-stringify-position: 4.0.0
dev: false
/vfile@5.3.7:
resolution: {integrity: sha512-r7qlzkgErKjobAmyNIkkSpizsFPYiUPuJb5pNW1RB4JcYVZhs4lIbVqk8XPk033CV/1z8ss5pkax8SuhGpcG8g==}
dependencies:
@ -17435,6 +17576,14 @@ packages:
unist-util-stringify-position: 3.0.3
vfile-message: 3.1.4
/vfile@6.0.1:
resolution: {integrity: sha512-1bYqc7pt6NIADBJ98UiG0Bn/CHIVOoZ/IyEkqIruLg0mE1BKzkOXY2D6CSqQIcKqgadppE5lrxgWXJmXd7zZJw==}
dependencies:
'@types/unist': 3.0.0
unist-util-stringify-position: 4.0.0
vfile-message: 4.0.2
dev: false
/vite-node@0.34.2(@types/node@20.5.3):
resolution: {integrity: sha512-JtW249Zm3FB+F7pQfH56uWSdlltCo1IOkZW5oHBzeQo0iX4jtC7o1t9aILMGd9kVekXBP2lfJBEQt9rBh07ebA==}
engines: {node: '>=v14.18.0'}
@ -17787,14 +17936,12 @@ packages:
/vscode-oniguruma@1.7.0:
resolution: {integrity: sha512-L9WMGRfrjOhgHSdOYgCt/yRMsXzLDJSL7BPrOZt73gU0iWO4mpqzqQzOz5srxqTvMBaR0XZTSrVWo4j55Rc6cA==}
dev: true
/vscode-textmate@5.2.0:
resolution: {integrity: sha512-Uw5ooOQxRASHgu6C7GVvUxisKXfSgW4oFlO+aa+PAkgmH89O3CXxEEzNRNtHSqtXFTl0nAC1uYj0GMSH27uwtQ==}
dev: true
/vscode-textmate@8.0.0:
resolution: {integrity: sha512-AFbieoL7a5LMqcnOF04ji+rpXadgOXnZsxQr//r83kLPr7biP7am3g9zbaZIaBGwBRWeSvoMD4mgPdX3e4NWBg==}
/vscode-uri@2.1.2:
resolution: {integrity: sha512-8TEXQxlldWAuIODdukIb+TR5s+9Ds40eSJrw+1iDDA9IFORPjMELarNQE3myz5XIkWWpdprmJjm1/SxMlWOC8A==}
dev: true