Start of converting to use the old build

This commit is contained in:
Matthew Phillips 2021-11-03 09:08:33 -04:00
parent 9344500376
commit d2183ba428
13 changed files with 1259 additions and 333 deletions

View file

@ -88,6 +88,7 @@
"shorthash": "^0.0.2",
"slash": "^4.0.0",
"sourcemap-codec": "^1.4.8",
"srcset-parse": "^1.1.0",
"string-width": "^5.0.0",
"strip-ansi": "^7.0.1",
"strip-indent": "^4.0.0",

View file

@ -0,0 +1,35 @@
import type { ScriptInfo } from './astro-core';
/** Entire output of `astro build`, stored in memory */
export interface BuildOutput {
[dist: string]: BuildFile;
}
export interface BuildFile {
/** The original location. Needed for code frame errors. */
srcPath: URL;
/** File contents */
contents: string | Buffer;
/** File content type (to determine encoding, etc) */
contentType: string;
/** Encoding */
encoding?: 'utf8';
/** Extracted scripts */
hoistedScripts?: ScriptInfo[];
}
/** Mapping of every URL and its required assets. All URLs are absolute relative to the project. */
export type BundleMap = {
[pageUrl: string]: PageDependencies;
};
export interface PageDependencies {
/** JavaScript files needed for page. No distinction between blocking/non-blocking or sync/async. */
js: Set<string>;
/** CSS needed for page, whether imported via <link>, JS, or Astro component. */
css: Set<string>;
/** Images needed for page. Can be loaded via CSS, <link>, or otherwise. */
images: Set<string>;
/** Async hoisted Javascript */
hoistedJS: Map<string, ScriptInfo>;
}

View file

@ -0,0 +1,157 @@
import type { AstroConfig } from '../../../@types/astro-core';
import type { BuildOutput, BundleMap } from '../../../@types/astro-build';
import type { LogOptions } from '../../logger.js';
import { performance } from 'perf_hooks';
import shorthash from 'shorthash';
import cheerio from 'cheerio';
import esbuild from 'esbuild';
import { getDistPath, getSrcPath, IS_ASTRO_FILE_URL, stopTimer } from '../util.js';
import { debug } from '../../logger.js';
// config
const COMMON_URL = `/_astro/common-[HASH].css`; // [HASH] will be replaced
/**
* Bundle CSS
* For files within dep tree, find ways to combine them.
* Current logic:
* - If CSS appears across multiple pages, combine into `/_astro/common.css` bundle
* - Otherwise, combine page CSS into one request as `/_astro/[page].css` bundle
*
* This operation _should_ be relatively-safe to do in parallel with other bundling,
* assuming other bundling steps dont touch CSS. While this step does modify HTML,
* it doesnt keep anything in local memory so other processes may modify HTML too.
*
* This operation mutates the original references of the buildOutput not only for
* safety (prevents possible conflicts), but for efficiency.
*/
export async function bundleCSS({
astroConfig,
buildState,
logging,
depTree,
}: {
astroConfig: AstroConfig;
buildState: BuildOutput;
logging: LogOptions;
depTree: BundleMap;
}): Promise<void> {
const timer: Record<string, number> = {};
const cssMap = new Map<string, string>();
// 1. organize CSS into common or page-specific CSS
timer.bundle = performance.now();
const sortedPages = Object.keys(depTree); // these were scanned in parallel; sort to create somewhat deterministic order
sortedPages.sort((a, b) => a.localeCompare(b, 'en', { numeric: true }));
for (const pageUrl of sortedPages) {
const { css } = depTree[pageUrl];
for (const cssUrl of css.keys()) {
if (!IS_ASTRO_FILE_URL.test(cssUrl)) {
// do not add to cssMap, leave as-is.
} else if (cssMap.has(cssUrl)) {
// scenario 1: if multiple URLs require this CSS, upgrade to common chunk
cssMap.set(cssUrl, COMMON_URL);
} else {
// scenario 2: otherwise, assume this CSS is page-specific
cssMap.set(cssUrl, '/_astro' + pageUrl.replace(/.html$/, '').replace(/^\./, '') + '-[HASH].css');
}
}
}
// 2. bundle (note: assume cssMap keys are in specific, correct order; assume buildState[] keys are in different order each time)
timer.bundle = performance.now();
// note: dont parallelize here otherwise CSS may end up in random order
for (const id of cssMap.keys()) {
const newUrl = cssMap.get(id) as string;
// if new bundle, create
if (!buildState[newUrl]) {
buildState[newUrl] = {
srcPath: getSrcPath(id, { astroConfig }), // this isnt accurate, but we can at least reference a file in the bundle
contents: '',
contentType: 'text/css',
encoding: 'utf8',
};
}
// append to bundle, delete old file
(buildState[newUrl] as any).contents += Buffer.isBuffer(buildState[id].contents) ? buildState[id].contents.toString('utf8') : buildState[id].contents;
delete buildState[id];
}
debug(logging, 'css', `bundled [${stopTimer(timer.bundle)}]`);
// 3. minify
timer.minify = performance.now();
await Promise.all(
Object.keys(buildState).map(async (id) => {
if (buildState[id].contentType !== 'text/css') return;
const { code } = await esbuild.transform(buildState[id].contents.toString(), {
loader: 'css',
minify: true,
});
buildState[id].contents = code;
})
);
debug(logging, 'css', `minified [${stopTimer(timer.minify)}]`);
// 4. determine hashes based on CSS content (deterministic), and update HTML <link> tags with final hashed URLs
timer.hashes = performance.now();
const cssHashes = new Map<string, string>();
for (const id of Object.keys(buildState)) {
if (!id.includes('[HASH].css')) continue; // iterate through buildState, looking to replace [HASH]
const hash = shorthash.unique(buildState[id].contents as string);
const newID = id.replace(/\[HASH\]/, hash);
cssHashes.set(id, newID);
buildState[newID] = buildState[id]; // copy ref without cloning to save memory
delete buildState[id]; // delete old ref
}
debug(logging, 'css', `built hashes [${stopTimer(timer.hashes)}]`);
// 5. update HTML <link> tags with final hashed URLs
timer.html = performance.now();
await Promise.all(
Object.keys(buildState).map(async (id) => {
if (buildState[id].contentType !== 'text/html') return;
const $ = cheerio.load(buildState[id].contents);
const stylesheets = new Set<string>(); // keep track of page-specific CSS so we remove dupes
const preloads = new Set<string>(); // list of stylesheets preloads, to remove dupes
$('link[href]').each((i, el) => {
const srcPath = getSrcPath(id, { astroConfig });
const oldHref = getDistPath($(el).attr('href') || '', { astroConfig, srcPath }); // note: this may be a relative URL; transform to absolute to find a buildOutput match
const newHref = cssMap.get(oldHref);
if (!newHref) {
return;
}
if (el.attribs?.rel === 'preload') {
if (preloads.has(newHref)) {
$(el).remove();
} else {
$(el).attr('href', cssHashes.get(newHref) || '');
preloads.add(newHref);
}
return;
}
if (stylesheets.has(newHref)) {
$(el).remove(); // this is a dupe; remove
} else {
$(el).attr('href', cssHashes.get(newHref) || ''); // new CSS; update href (important! use cssHashes, not cssMap)
// bonus: add [rel] and [type]. not necessary, but why not?
$(el).attr('rel', 'stylesheet');
$(el).attr('type', 'text/css');
stylesheets.add(newHref);
}
});
(buildState[id] as any).contents = $.html(); // save updated HTML in global buildState
})
);
debug(logging, 'css', `parsed html [${stopTimer(timer.html)}]`);
}

View file

@ -0,0 +1,256 @@
import type { InputOptions, OutputOptions, OutputChunk } from 'rollup';
import type { AstroConfig, ScriptInfo } from '../../../@types/astro-core';
//import type { AstroConfig, BundleMap, BuildOutput, ScriptInfo, InlineScriptInfo } from '../../../@types/astro-build';
import type { AstroRuntime } from '../../runtime';
import type { LogOptions } from '../../logger.js';
import { fileURLToPath } from 'url';
import { rollup } from 'rollup';
import { terser } from 'rollup-plugin-terser';
import { createBundleStats, addBundleStats, BundleStatsMap } from '../stats.js';
import { IS_ASTRO_FILE_URL } from '../util.js';
import cheerio from 'cheerio';
import path from 'path';
interface BundleOptions {
dist: URL;
astroRuntime: AstroRuntime;
}
/** Collect JS imports from build output */
export function collectJSImports(buildState: BuildOutput): Set<string> {
const imports = new Set<string>();
for (const id of Object.keys(buildState)) {
if (buildState[id].contentType === 'application/javascript') imports.add(id);
}
return imports;
}
function pageUrlToVirtualJSEntry(pageUrl: string) {
return 'astro-virtual:' + pageUrl.replace(/.html$/, '').replace(/^\./, '') + '.js';
}
export async function bundleHoistedJS({
buildState,
astroConfig,
logging,
depTree,
dist,
runtime,
}: {
astroConfig: AstroConfig;
buildState: BuildOutput;
logging: LogOptions;
depTree: BundleMap;
dist: URL;
runtime: AstroRuntime;
}) {
const sortedPages = Object.keys(depTree); // these were scanned in parallel; sort to create somewhat deterministic order
sortedPages.sort((a, b) => a.localeCompare(b, 'en', { numeric: true }));
/**
* 1. Go over sorted pages and create a virtual module for all of its dependencies
*/
const entryImports: string[] = [];
const virtualScripts = new Map<string, ScriptInfo>();
const pageToEntryMap = new Map<string, string>();
for (let pageUrl of sortedPages) {
const hoistedJS = depTree[pageUrl].hoistedJS;
if (hoistedJS.size) {
for (let [url, scriptInfo] of hoistedJS) {
if (virtualScripts.has(url) || !url.startsWith('astro-virtual:')) continue;
virtualScripts.set(url, scriptInfo);
}
const entryURL = pageUrlToVirtualJSEntry(pageUrl);
const entryJS = Array.from(hoistedJS.keys())
.map((url) => `import '${url}';`)
.join('\n');
virtualScripts.set(entryURL, {
content: entryJS,
});
entryImports.push(entryURL);
pageToEntryMap.set(pageUrl, entryURL);
}
}
if (!entryImports.length) {
// There are no hoisted scripts, bail
return;
}
/**
* 2. Run the bundle to bundle each pages JS into a single bundle (with shared content)
*/
const inputOptions: InputOptions = {
input: entryImports,
plugins: [
{
name: 'astro:build',
resolveId(source: string, imported?: string) {
if (virtualScripts.has(source)) {
return source;
}
if (source.startsWith('/')) {
return source;
}
if (imported) {
const outUrl = new URL(source, 'http://example.com' + imported);
return outUrl.pathname;
}
return null;
},
async load(id: string) {
if (virtualScripts.has(id)) {
let info = virtualScripts.get(id) as InlineScriptInfo;
return info.content;
}
const result = await runtime.load(id);
if (result.statusCode !== 200) {
return null;
}
return result.contents.toString('utf-8');
},
},
],
};
const build = await rollup(inputOptions);
const outputOptions: OutputOptions = {
dir: fileURLToPath(dist),
format: 'esm',
exports: 'named',
entryFileNames(chunk) {
const { facadeModuleId } = chunk;
if (!facadeModuleId) throw new Error(`facadeModuleId missing: ${chunk.name}`);
return facadeModuleId.substr('astro-virtual:/'.length, facadeModuleId.length - 'astro-virtual:/'.length - 3 /* .js */) + '-[hash].js';
},
plugins: [
// We are using terser for the demo, but might switch to something else long term
// Look into that rather than adding options here.
terser(),
],
};
const { output } = await build.write(outputOptions);
/**
* 3. Get a mapping of the virtual filename to the chunk file name
*/
const entryToChunkFileName = new Map<string, string>();
output.forEach((chunk) => {
const { fileName, facadeModuleId, isEntry } = chunk as OutputChunk;
if (!facadeModuleId || !isEntry) return;
entryToChunkFileName.set(facadeModuleId, fileName);
});
/**
* 4. Update the original HTML with the new chunk scripts
*/
Object.keys(buildState).forEach((id) => {
if (buildState[id].contentType !== 'text/html') return;
const entryVirtualURL = pageUrlToVirtualJSEntry(id);
let hasHoisted = false;
const $ = cheerio.load(buildState[id].contents);
$('script[data-astro="hoist"]').each((i, el) => {
hasHoisted = true;
if (i === 0) {
let chunkName = entryToChunkFileName.get(entryVirtualURL);
if (!chunkName) return;
let chunkPathname = '/' + chunkName;
let relLink = path.relative(path.dirname(id), chunkPathname);
$(el).attr('src', relLink.startsWith('.') ? relLink : './' + relLink);
$(el).removeAttr('data-astro');
$(el).html('');
} else {
$(el).remove();
}
});
if (hasHoisted) {
(buildState[id] as any).contents = $.html(); // save updated HTML in global buildState
}
});
}
/** Bundle JS action */
export async function bundleJS(imports: Set<string>, { astroRuntime, dist }: BundleOptions): Promise<BundleStatsMap> {
const ROOT = 'astro:root';
const validImports = [...imports].filter((url) => IS_ASTRO_FILE_URL.test(url));
const root = `
${validImports.map((url) => `import '${url}';`).join('\n')}
`;
const inputOptions: InputOptions = {
input: validImports,
plugins: [
{
name: 'astro:build',
resolveId(source: string, imported?: string) {
if (source === ROOT) {
return source;
}
if (source.startsWith('/')) {
return source;
}
if (imported) {
const outUrl = new URL(source, 'http://example.com' + imported);
return outUrl.pathname;
}
return null;
},
async load(id: string) {
if (id === ROOT) {
return root;
}
const result = await astroRuntime.load(id);
if (result.statusCode !== 200) {
return null;
}
return result.contents.toString('utf-8');
},
},
],
};
const build = await rollup(inputOptions);
const outputOptions: OutputOptions = {
dir: fileURLToPath(dist),
format: 'esm',
exports: 'named',
entryFileNames(chunk) {
const { facadeModuleId } = chunk;
if (!facadeModuleId) throw new Error(`facadeModuleId missing: ${chunk.name}`);
return facadeModuleId.substr(1);
},
plugins: [
// We are using terser for the demo, but might switch to something else long term
// Look into that rather than adding options here.
terser(),
],
};
const stats = createBundleStats();
const { output } = await build.write(outputOptions);
await Promise.all(
output.map(async (chunk) => {
const code = (chunk as OutputChunk).code || '';
await addBundleStats(stats, code, chunk.fileName);
})
);
return stats;
}

View file

@ -1,221 +1,395 @@
import type { InputHTMLOptions } from '@web/rollup-plugin-html';
import type { AstroConfig, ComponentInstance, GetStaticPathsResult, ManifestData, RouteCache, RouteData, RSSResult } from '../../@types/astro-core';
import type { LogOptions } from '../logger';
import type { AstroConfig, RouteData, RuntimeMode, ScriptInfo } from '../../@types/astro-core';
import type { BuildOutput, BundleMap, PageDependencies } from '../../@types/astro-build';
import { rollupPluginHTML } from '@web/rollup-plugin-html';
import cheerio from 'cheerio';
import del from 'del';
import eslexer from 'es-module-lexer';
import fs from 'fs';
import { bold, cyan, green, dim } from 'kleur/colors';
import { bold, green, red, underline, yellow } from 'kleur/colors';
import mime from 'mime';
import path from 'path';
import { performance } from 'perf_hooks';
import vite, { ViteDevServer } from '../vite.js';
import glob from 'tiny-glob';
import hash from 'shorthash';
import srcsetParse from 'srcset-parse';
import { fileURLToPath } from 'url';
import { bundleCSS } from './bundle/css.js';
import { bundleJS, bundleHoistedJS, collectJSImports } from './bundle/js.js';
import { buildStaticPage, getStaticPathsForPage } from './page.js';
import { generateSitemap } from './sitemap.js';
import { collectBundleStats, logURLStats, mapBundleStatsToURLStats } from './stats.js';
import { getDistPath, stopTimer } from './util.js';
import type { LogOptions } from '../logger';
import { debug, defaultLogDestination, defaultLogLevel, error, info, warn } from '../logger.js';
import { createVite } from '../create-vite.js';
import { pad } from '../dev/util.js';
import { debug, defaultLogOptions, levels, timerMessage, warn } from '../logger.js';
import { ssr } from '../ssr/index.js';
import { generatePaginateFunction } from '../ssr/paginate.js';
import vite, { ViteDevServer } from '../vite.js';
import { createRouteManifest, validateGetStaticPathsModule, validateGetStaticPathsResult } from '../ssr/routing.js';
import { generateRssFunction } from '../ssr/rss.js';
import { generateSitemap } from '../ssr/sitemap.js';
import { kb, profileHTML, profileJS } from './stats.js';
export interface BuildOptions {
mode?: string;
logging: LogOptions;
// This package isn't real ESM, so have to coerce it
const matchSrcset: typeof srcsetParse = (srcsetParse as any).default;
const defaultLogging: LogOptions = {
level: defaultLogLevel,
dest: defaultLogDestination,
};
/** Is this URL remote or embedded? */
function isRemoteOrEmbedded(url: string) {
return url.startsWith('http://') || url.startsWith('https://') || url.startsWith('//') || url.startsWith('data:');
}
/** `astro build` */
export default async function build(config: AstroConfig, options: BuildOptions = { logging: defaultLogOptions }): Promise<void> {
const builder = new AstroBuilder(config, options);
await builder.build();
}
/** The primary build action */
export async function build(astroConfig: AstroConfig, logging: LogOptions = defaultLogging): Promise<0 | 1> {
const { projectRoot } = astroConfig;
const buildState: BuildOutput = {};
const depTree: BundleMap = {};
const timer: Record<string, number> = {};
class AstroBuilder {
private config: AstroConfig;
private logging: LogOptions;
private mode = 'production';
private origin: string;
private routeCache: RouteCache = {};
private manifest: ManifestData;
private viteServer?: ViteDevServer;
const runtimeLogging: LogOptions = {
level: 'error',
dest: defaultLogDestination,
};
constructor(config: AstroConfig, options: BuildOptions) {
if (!config.buildOptions.site && config.buildOptions.sitemap !== false) {
warn(options.logging, 'config', `Set "buildOptions.site" to generate correct canonical URLs and sitemap`);
// warn users if missing config item in build that may result in broken SEO (cant disable, as they should provide this)
if (!astroConfig.buildOptions.site) {
warn(logging, 'config', `Set "buildOptions.site" to generate correct canonical URLs and sitemap`);
}
if (options.mode) this.mode = options.mode;
this.config = config;
const port = config.devOptions.port; // no need to save this (dont rely on port in builder)
this.logging = options.logging;
this.origin = config.buildOptions.site ? new URL(config.buildOptions.site).origin : `http://localhost:${port}`;
this.manifest = createRouteManifest({ config });
}
const mode: RuntimeMode = 'production';
async build() {
const { logging, origin } = this;
const timer: Record<string, number> = { viteStart: performance.now() };
const viteConfig = await createVite(
{
mode: this.mode,
mode,
server: {
hmr: { overlay: false },
middlewareMode: 'ssr',
},
...(this.config.vite || {}),
...(astroConfig.vite || {}),
},
{ astroConfig: this.config, logging }
{ astroConfig, logging }
);
const viteServer = await vite.createServer(viteConfig);
this.viteServer = viteServer;
debug(logging, 'build', timerMessage('Vite started', timer.viteStart));
timer.renderStart = performance.now();
const assets: Record<string, string> = {};
const allPages: Record<string, RouteData & { paths: string[] }> = {};
// Collect all routes ahead-of-time, before we start the build.
// NOTE: This enforces that `getStaticPaths()` is only called once per route,
// and is then cached across all future SSR builds. In the past, we've had trouble
// with parallelized builds without guaranteeing that this is called first.
await Promise.all(
this.manifest.routes.map(async (route) => {
// static route:
const manifest = createRouteManifest({ config: astroConfig });
//const astroRuntime = await createRuntime(astroConfig, { mode, logging: runtimeLogging });
//const { runtimeConfig } = astroRuntime;
//const { snowpackRuntime } = runtimeConfig;
try {
// 0. erase build directory
await del(fileURLToPath(astroConfig.dist));
/**
* 1. Build Pages
* Source files are built in parallel and stored in memory. Most assets are also gathered here, too.
*/
timer.build = performance.now();
info(logging, 'build', yellow('! building pages...'));
const allRoutesAndPaths = await Promise.all(
manifest.routes.map(async (route): Promise<[RouteData, string[]]> => {
if (route.pathname) {
allPages[route.component] = { ...route, paths: [route.pathname] };
return;
}
// dynamic route:
const result = await this.getStaticPathsForRoute(route);
if (result.rss?.xml) {
const rssFile = new URL(result.rss.url.replace(/^\/?/, './'), this.config.dist);
if (assets[fileURLToPath(rssFile)]) {
return [route, [route.pathname]];
} else {
const result = await getStaticPathsForPage({
astroConfig,
route,
logging,
viteServer
});
if (result.rss.xml) {
if (buildState[result.rss.url]) {
throw new Error(`[getStaticPaths] RSS feed ${result.rss.url} already exists.\nUse \`rss(data, {url: '...'})\` to choose a unique, custom URL. (${route.component})`);
}
assets[fileURLToPath(rssFile)] = result.rss.xml;
}
allPages[route.component] = { ...route, paths: result.paths };
})
);
// After all routes have been collected, start building them.
// TODO: test parallel vs. serial performance. Promise.all() may be
// making debugging harder without any perf gain. If parallel is best,
// then we should set a max number of parallel builds.
const input: InputHTMLOptions[] = [];
await Promise.all(
Object.entries(allPages).map(([component, route]) =>
Promise.all(
route.paths.map(async (pathname) => {
input.push({
html: await ssr({
astroConfig: this.config,
filePath: new URL(`./${component}`, this.config.projectRoot),
logging,
mode: 'production',
origin,
pathname,
route,
routeCache: this.routeCache,
viteServer,
}),
name: pathname.replace(/\/?$/, '/index.html').replace(/^\//, ''),
});
})
)
)
);
debug(logging, 'build', timerMessage('All pages rendered', timer.renderStart));
// Bundle the assets in your final build: This currently takes the HTML output
// of every page (stored in memory) and bundles the assets pointed to on those pages.
timer.buildStart = performance.now();
await vite.build({
logLevel: 'error',
mode: 'production',
build: {
emptyOutDir: true,
minify: 'esbuild', // significantly faster than "terser" but may produce slightly-bigger bundles
outDir: fileURLToPath(this.config.dist),
rollupOptions: {
input: [],
output: { format: 'esm' },
},
target: 'es2020', // must match an esbuild target
},
plugins: [
rollupPluginHTML({
rootDir: viteConfig.root,
input,
extractAssets: false,
}) as any, // "any" needed for CI; also we dont need typedefs for this anyway
...(viteConfig.plugins || []),
],
publicDir: viteConfig.publicDir,
root: viteConfig.root,
server: viteConfig.server,
});
debug(logging, 'build', timerMessage('Vite build finished', timer.buildStart));
// Write any additionally generated assets to disk.
timer.assetsStart = performance.now();
Object.keys(assets).map((k) => {
if (!assets[k]) return;
const filePath = new URL(`file://${k}`);
fs.mkdirSync(new URL('./', filePath), { recursive: true });
fs.writeFileSync(filePath, assets[k], 'utf8');
delete assets[k]; // free up memory
});
debug(logging, 'build', timerMessage('Additional assets copied', timer.assetsStart));
// Build your final sitemap.
timer.sitemapStart = performance.now();
if (this.config.buildOptions.sitemap && this.config.buildOptions.site) {
const sitemapStart = performance.now();
const sitemap = generateSitemap(input.map(({ name }) => new URL(`/${name}`, this.config.buildOptions.site).href));
const sitemapPath = new URL('./sitemap.xml', this.config.dist);
await fs.promises.mkdir(new URL('./', sitemapPath), { recursive: true });
await fs.promises.writeFile(sitemapPath, sitemap, 'utf8');
}
debug(logging, 'build', timerMessage('Sitemap built', timer.sitemapStart));
// You're done! Time to clean up.
await viteServer.close();
if (logging.level && levels[logging.level] <= levels['info']) {
await this.printStats({ cwd: this.config.dist, pageCount: input.length });
}
}
/** Extract all static paths from a dynamic route */
private async getStaticPathsForRoute(route: RouteData): Promise<{ paths: string[]; rss?: RSSResult }> {
if (!this.viteServer) throw new Error(`vite.createServer() not called!`);
const filePath = new URL(`./${route.component}`, this.config.projectRoot);
const mod = (await this.viteServer.ssrLoadModule(fileURLToPath(filePath))) as ComponentInstance;
validateGetStaticPathsModule(mod);
const rss = generateRssFunction(this.config.buildOptions.site, route);
const staticPaths: GetStaticPathsResult = (await mod.getStaticPaths!({ paginate: generatePaginateFunction(route), rss: rss.generator })).flat();
this.routeCache[route.component] = staticPaths;
validateGetStaticPathsResult(staticPaths, this.logging);
return {
paths: staticPaths.map((staticPath) => staticPath.params && route.generate(staticPath.params)).filter(Boolean),
rss: rss.rss,
buildState[result.rss.url] = {
srcPath: new URL(result.rss.url, projectRoot),
contents: result.rss.xml,
contentType: 'text/xml',
encoding: 'utf8',
};
}
return [route, result.paths];
}
})
);
try {
await Promise.all(
allRoutesAndPaths.map(async ([route, paths]: [RouteData, string[]]) => {
for (const p of paths) {
await buildStaticPage({
astroConfig,
buildState,
route,
path: p,
astroRuntime,
});
}
})
);
} catch (e: any) {
if (e.filename) {
let stack = e.stack
.replace(/Object\.__render \(/gm, '')
.replace(/\/_astro\/(.+)\.astro\.js\:\d+\:\d+\)/gm, (_: string, $1: string) => 'file://' + fileURLToPath(projectRoot) + $1 + '.astro')
.split('\n');
stack.splice(1, 0, ` at file://${e.filename}`);
stack = stack.join('\n');
error(
logging,
'build',
`${red(`Unable to render ${underline(e.filename.replace(fileURLToPath(projectRoot), ''))}`)}
/** Stats */
private async printStats({ cwd, pageCount }: { cwd: URL; pageCount: number }) {
const [js, html] = await Promise.all([profileJS({ cwd, entryHTML: new URL('./index.html', cwd) }), profileHTML({ cwd })]);
${stack}
`
);
} else {
error(logging, 'build', e.message);
}
error(logging, 'build', red('✕ building pages failed!'));
/* eslint-disable no-console */
console.log(`${bold(cyan('Done'))}
Pages (${pageCount} total)
${green(`✔ All pages under ${kb(html.maxSize)}`)}
JS
${pad('initial load', 50)}${pad(kb(js.entryHTML || 0), 8, 'left')}
${pad('total size', 50)}${pad(kb(js.total), 8, 'left')}
CSS
${pad('initial load', 50)}${pad('0 kB', 8, 'left')}
${pad('total size', 50)}${pad('0 kB', 8, 'left')}
Images
${green(`✔ All images under 50 kB`)}
`);
await viteServer.close();
return 1;
}
info(logging, 'build', green('✔'), 'pages built.');
debug(logging, 'build', `built pages [${stopTimer(timer.build)}]`);
// after pages are built, build depTree
timer.deps = performance.now();
const scanPromises: Promise<void>[] = [];
await eslexer.init;
for (const id of Object.keys(buildState)) {
if (buildState[id].contentType !== 'text/html') continue; // only scan HTML files
const pageDeps = findDeps(buildState[id].contents as string, {
astroConfig,
srcPath: buildState[id].srcPath,
id,
});
depTree[id] = pageDeps;
// while scanning we will find some unbuilt files; make sure those are all built while scanning
for (const url of [...pageDeps.js, ...pageDeps.css, ...pageDeps.images]) {
if (!buildState[url])
scanPromises.push(
astroRuntime.load(url).then((result: LoadResult) => {
if (result.statusCode === 404) {
if (url.startsWith('/_astro/')) {
throw new Error(`${buildState[id].srcPath.href}: could not find file "${url}".`);
}
warn(logging, 'build', `${buildState[id].srcPath.href}: could not find file "${url}". Marked as external.`);
return;
}
if (result.statusCode !== 200) {
// there shouldnt be a build error here
throw (result as any).error || new Error(`unexpected ${result.statusCode} response from "${url}".`);
}
buildState[url] = {
srcPath: new URL(url, projectRoot),
contents: result.contents,
contentType: result.contentType || mime.getType(url) || '',
};
})
);
}
}
await Promise.all(scanPromises);
debug(logging, 'build', `scanned deps [${stopTimer(timer.deps)}]`);
/**
* 2. Bundling 1st Pass: In-memory
* Bundle CSS, and anything else that can happen in memory (for now, JS bundling happens after writing to disk)
*/
info(logging, 'build', yellow('! optimizing css...'));
timer.prebundleCSS = performance.now();
await Promise.all([
bundleCSS({ buildState, astroConfig, logging, depTree }).then(() => {
debug(logging, 'build', `bundled CSS [${stopTimer(timer.prebundleCSS)}]`);
}),
bundleHoistedJS({ buildState, astroConfig, logging, depTree, runtime: astroRuntime, dist: astroConfig.dist }),
// TODO: optimize images?
]);
// TODO: minify HTML?
info(logging, 'build', green('✔'), 'css optimized.');
/**
* 3. Write to disk
* Also clear in-memory bundle
*/
// collect stats output
const urlStats = await collectBundleStats(buildState, depTree);
// collect JS imports for bundling
const jsImports = await collectJSImports(buildState);
// write sitemap
if (astroConfig.buildOptions.sitemap && astroConfig.buildOptions.site) {
timer.sitemap = performance.now();
info(logging, 'build', yellow('! creating sitemap...'));
const sitemap = generateSitemap(buildState, astroConfig.buildOptions.site);
const sitemapPath = new URL('sitemap.xml', astroConfig.dist);
await fs.promises.mkdir(path.dirname(fileURLToPath(sitemapPath)), { recursive: true });
await fs.promises.writeFile(sitemapPath, sitemap, 'utf8');
info(logging, 'build', green('✔'), 'sitemap built.');
debug(logging, 'build', `built sitemap [${stopTimer(timer.sitemap)}]`);
}
// write to disk and free up memory
timer.write = performance.now();
for (const id of Object.keys(buildState)) {
const outPath = new URL(`.${id}`, astroConfig.dist);
const parentDir = path.dirname(fileURLToPath(outPath));
await fs.promises.mkdir(parentDir, { recursive: true });
const handle = await fs.promises.open(outPath, 'w');
await fs.promises.writeFile(handle, buildState[id].contents, buildState[id].encoding);
// Ensure the file handle is not left hanging which will
// result in the garbage collector loggin errors in the console
// when it eventually has to close them.
await handle.close();
delete buildState[id];
delete depTree[id];
}
debug(logging, 'build', `wrote files to disk [${stopTimer(timer.write)}]`);
/**
* 4. Copy Public Assets
*/
if (fs.existsSync(astroConfig.public)) {
info(logging, 'build', yellow(`! copying public folder...`));
timer.public = performance.now();
const cwd = fileURLToPath(astroConfig.public);
const publicFiles = await glob('**/*', { cwd, filesOnly: true });
await Promise.all(
publicFiles.map(async (filepath) => {
const srcPath = new URL(filepath, astroConfig.public);
const distPath = new URL(filepath, astroConfig.dist);
await fs.promises.mkdir(path.dirname(fileURLToPath(distPath)), { recursive: true });
await fs.promises.copyFile(srcPath, distPath);
})
);
debug(logging, 'build', `copied public folder [${stopTimer(timer.public)}]`);
info(logging, 'build', green('✔'), 'public folder copied.');
} else {
if (path.basename(astroConfig.public.toString()) !== 'public') {
info(logging, 'tip', yellow(`! no public folder ${astroConfig.public} found...`));
}
}
/**
* 5. Bundling 2nd Pass: On disk
* Bundle JS, which requires hard files to optimize
*/
info(logging, 'build', yellow(`! bundling...`));
if (jsImports.size > 0) {
timer.bundleJS = performance.now();
const jsStats = await bundleJS(jsImports, { dist: astroConfig.dist, astroRuntime });
mapBundleStatsToURLStats({ urlStats, depTree, bundleStats: jsStats });
debug(logging, 'build', `bundled JS [${stopTimer(timer.bundleJS)}]`);
info(logging, 'build', green(``), 'bundling complete.');
}
/**
* 6. Print stats
*/
logURLStats(logging, urlStats);
await astroRuntime.shutdown();
info(logging, 'build', bold(green('▶ Build Complete!')));
return 0;
} catch (err) {
error(logging, 'build', err.message);
await astroRuntime.shutdown();
return 1;
}
}
/** Given an HTML string, collect <link> and <img> tags */
export function findDeps(html: string, { astroConfig, srcPath }: { astroConfig: AstroConfig; srcPath: URL; id: string }): PageDependencies {
const pageDeps: PageDependencies = {
js: new Set<string>(),
css: new Set<string>(),
images: new Set<string>(),
hoistedJS: new Map<string, ScriptInfo>(),
};
const $ = cheerio.load(html);
$('script').each((_i, el) => {
const src = $(el).attr('src');
const hoist = $(el).attr('data-astro') === 'hoist';
if (hoist) {
if (src) {
pageDeps.hoistedJS.set(src, {
src,
});
} else {
let content = $(el).html() || '';
pageDeps.hoistedJS.set(`astro-virtual:${hash.unique(content)}`, {
content,
});
}
} else if (src) {
if (isRemoteOrEmbedded(src)) return;
pageDeps.js.add(getDistPath(src, { astroConfig, srcPath }));
} else {
const text = $(el).html();
if (!text) return;
const [imports] = eslexer.parse(text);
for (const spec of imports) {
const importSrc = spec.n;
if (importSrc && !isRemoteOrEmbedded(importSrc)) {
pageDeps.js.add(getDistPath(importSrc, { astroConfig, srcPath }));
}
}
}
});
$('link[href]').each((_i, el) => {
const href = $(el).attr('href');
if (href && !isRemoteOrEmbedded(href) && ($(el).attr('rel') === 'stylesheet' || $(el).attr('type') === 'text/css' || href.endsWith('.css'))) {
const dist = getDistPath(href, { astroConfig, srcPath });
pageDeps.css.add(dist);
}
});
$('img[src]').each((_i, el) => {
const src = $(el).attr('src');
if (src && !isRemoteOrEmbedded(src)) {
pageDeps.images.add(getDistPath(src, { astroConfig, srcPath }));
}
});
$('img[srcset]').each((_i, el) => {
const srcset = $(el).attr('srcset') || '';
for (const src of matchSrcset(srcset)) {
if (!isRemoteOrEmbedded(src.url)) {
pageDeps.images.add(getDistPath(src.url, { astroConfig, srcPath }));
}
}
});
// Add in srcset check for <source>
$('source[srcset]').each((_i, el) => {
const srcset = $(el).attr('srcset') || '';
for (const src of matchSrcset(srcset)) {
if (!isRemoteOrEmbedded(src.url)) {
pageDeps.images.add(getDistPath(src.url, { astroConfig, srcPath }));
}
}
});
// important: preserve the scan order of deps! order matters on pages
return pageDeps;
}
/** `astro build` */
/*export default async function build(config: AstroConfig, options: BuildOptions = { logging: defaultLogOptions }): Promise<void> {
const builder = new AstroBuilder(config, options);
await builder.build();
}
*/

View file

@ -0,0 +1,89 @@
import type { AstroConfig, RouteData } from '../../@types/astro-core';
import type { BuildOutput } from '../../@types/astro-build';
import type { ViteDevServer } from '../vite';
import _path from 'path';
import { fileURLToPath } from 'url';
import { LogOptions } from '../logger';
import { loadModule } from '../ssr/index.js';
import { validateGetStaticPathsModule, validateGetStaticPathsResult } from '../ssr/routing.js';
import { generatePaginateFunction } from './paginate.js';
import { generateRssFunction } from './rss.js';
interface PageLocation {
fileURL: URL;
snowpackURL: string;
}
function convertMatchToLocation(routeMatch: RouteData, astroConfig: AstroConfig): PageLocation {
const url = new URL(`./${routeMatch.component}`, astroConfig.projectRoot);
return {
fileURL: url,
snowpackURL: `/_astro/${routeMatch.component}.js`,
};
}
interface PageBuildOptions {
astroConfig: AstroConfig;
buildState: BuildOutput;
path: string;
route: RouteData;
}
/** Build dynamic page */
export async function getStaticPathsForPage({
astroConfig,
route,
logging,
viteServer,
}: {
astroConfig: AstroConfig;
route: RouteData;
logging: LogOptions;
viteServer: ViteDevServer;
}): Promise<{ paths: string[]; rss: any }> {
const location = convertMatchToLocation(route, astroConfig);
//const mod = await snowpackRuntime.importModule(location.snowpackURL);
const mod = await loadModule(location.fileURL, viteServer);
validateGetStaticPathsModule(mod);
const [rssFunction, rssResult] = generateRssFunction(astroConfig.buildOptions.site, route);
const staticPaths = mod.getStaticPaths!({
paginate: generatePaginateFunction(route),
rss: rssFunction,
});
validateGetStaticPathsResult(staticPaths, logging);
return {
paths: staticPaths.map((staticPath) => staticPath.params && route.generate(staticPath.params)).filter(Boolean),
rss: rssResult,
};
}
function formatOutFile(path: string, pageUrlFormat: AstroConfig['buildOptions']['pageUrlFormat']) {
if (path === '/404') {
return '/404.html';
}
if (path === '/') {
return '/index.html';
}
if (pageUrlFormat === 'directory') {
return _path.posix.join(path, '/index.html');
}
return `${path}.html`;
}
/** Build static page */
export async function buildStaticPage({ astroConfig, buildState, path, route, astroRuntime }: PageBuildOptions): Promise<void> {
const location = convertMatchToLocation(route, astroConfig);
const normalizedPath = astroConfig.devOptions.trailingSlash === 'never' ? path : path.endsWith('/') ? path : `${path}/`;
const result = await astroRuntime.load(normalizedPath);
if (result.statusCode !== 200) {
let err = (result as any).error;
if (!(err instanceof Error)) err = new Error(err);
err.filename = fileURLToPath(location.fileURL);
throw err;
}
buildState[formatOutFile(path, astroConfig.buildOptions.pageUrlFormat)] = {
srcPath: location.fileURL,
contents: result.contents,
contentType: 'text/html',
encoding: 'utf8',
};
}

View file

@ -0,0 +1,63 @@
import { GetStaticPathsResult, PaginatedCollectionProp, PaginateFunction, Params, Props, RouteData } from '../@types/astro';
// return filters.map((filter) => {
// const filteredRecipes = allRecipes.filter((recipe) =>
// filterKeys.some((key) => recipe[key] === filter)
// );
// return paginate(filteredRecipes, {
// params: { slug: slugify(filter) },
// props: { filter },
// });
// });
export function generatePaginateFunction(routeMatch: RouteData): PaginateFunction {
return function paginateUtility(data: any[], args: { pageSize?: number; params?: Params; props?: Props } = {}) {
let { pageSize: _pageSize, params: _params, props: _props } = args;
const pageSize = _pageSize || 10;
const paramName = 'page';
const additoonalParams = _params || {};
const additoonalProps = _props || {};
let includesFirstPageNumber: boolean;
if (routeMatch.params.includes(`...${paramName}`)) {
includesFirstPageNumber = false;
} else if (routeMatch.params.includes(`${paramName}`)) {
includesFirstPageNumber = true;
} else {
throw new Error(
`[paginate()] page number param \`${paramName}\` not found in your filepath.\nRename your file to \`[...page].astro\` or customize the param name via the \`paginate([], {param: '...'}\` option.`
);
}
const lastPage = Math.max(1, Math.ceil(data.length / pageSize));
const result: GetStaticPathsResult = [...Array(lastPage).keys()].map((num) => {
const pageNum = num + 1;
const start = pageSize === Infinity ? 0 : (pageNum - 1) * pageSize; // currentPage is 1-indexed
const end = Math.min(start + pageSize, data.length);
const params = {
...additoonalParams,
[paramName]: includesFirstPageNumber || pageNum > 1 ? String(pageNum) : undefined,
};
return {
params,
props: {
...additoonalProps,
page: {
data: data.slice(start, end),
start,
end: end - 1,
size: pageSize,
total: data.length,
currentPage: pageNum,
lastPage: lastPage,
url: {
current: routeMatch.generate({ ...params }),
next: pageNum === lastPage ? undefined : routeMatch.generate({ ...params, page: String(pageNum + 1) }),
prev: pageNum === 1 ? undefined : routeMatch.generate({ ...params, page: !includesFirstPageNumber && pageNum - 1 === 1 ? undefined : String(pageNum - 1) }),
},
} as PaginatedCollectionProp,
},
};
});
return result;
};
}

View file

@ -0,0 +1,85 @@
import type { RSSFunctionArgs, RouteData } from '../@types/astro';
import parser from 'fast-xml-parser';
import { canonicalURL } from './util.js';
/** Validates getStaticPaths.rss */
export function validateRSS(args: GenerateRSSArgs): void {
const { rssData, srcFile } = args;
if (!rssData.title) throw new Error(`[${srcFile}] rss.title required`);
if (!rssData.description) throw new Error(`[${srcFile}] rss.description required`);
if ((rssData as any).item) throw new Error(`[${srcFile}] \`item: Function\` should be \`items: Item[]\``);
if (!Array.isArray(rssData.items)) throw new Error(`[${srcFile}] rss.items should be an array of items`);
}
type GenerateRSSArgs = { site: string; rssData: RSSFunctionArgs; srcFile: string; feedURL: string };
/** Generate RSS 2.0 feed */
export function generateRSS(args: GenerateRSSArgs): string {
validateRSS(args);
const { srcFile, feedURL, rssData, site } = args;
if ((rssData as any).item) throw new Error(`[${srcFile}] rss() \`item()\` function was deprecated, and is now \`items: object[]\`.`);
let xml = `<?xml version="1.0" encoding="UTF-8"?><rss version="2.0"`;
// xmlns
if (rssData.xmlns) {
for (const [k, v] of Object.entries(rssData.xmlns)) {
xml += ` xmlns:${k}="${v}"`;
}
}
xml += `>`;
xml += `<channel>`;
// title, description, customData
xml += `<title><![CDATA[${rssData.title}]]></title>`;
xml += `<description><![CDATA[${rssData.description}]]></description>`;
xml += `<link>${canonicalURL(feedURL, site).href}</link>`;
if (typeof rssData.customData === 'string') xml += rssData.customData;
// items
for (const result of rssData.items) {
xml += `<item>`;
// validate
if (typeof result !== 'object') throw new Error(`[${srcFile}] rss.items expected an object. got: "${JSON.stringify(result)}"`);
if (!result.title) throw new Error(`[${srcFile}] rss.items required "title" property is missing. got: "${JSON.stringify(result)}"`);
if (!result.link) throw new Error(`[${srcFile}] rss.items required "link" property is missing. got: "${JSON.stringify(result)}"`);
xml += `<title><![CDATA[${result.title}]]></title>`;
xml += `<link>${canonicalURL(result.link, site).href}</link>`;
if (result.description) xml += `<description><![CDATA[${result.description}]]></description>`;
if (result.pubDate) {
// note: this should be a Date, but if user provided a string or number, we can work with that, too.
if (typeof result.pubDate === 'number' || typeof result.pubDate === 'string') {
result.pubDate = new Date(result.pubDate);
} else if (result.pubDate instanceof Date === false) {
throw new Error('[${filename}] rss.item().pubDate must be a Date');
}
xml += `<pubDate>${result.pubDate.toUTCString()}</pubDate>`;
}
if (typeof result.customData === 'string') xml += result.customData;
xml += `</item>`;
}
xml += `</channel></rss>`;
// validate users inputs to see if its valid XML
const isValid = parser.validate(xml);
if (isValid !== true) {
// If valid XML, isValid will be `true`. Otherwise, this will be an error object. Throw.
throw new Error(isValid as any);
}
return xml;
}
export function generateRssFunction(site: string | undefined, routeMatch: RouteData): [(args: any) => void, { url?: string; xml?: string }] {
let result: { url?: string; xml?: string } = {};
function rssUtility(args: any) {
if (!site) {
throw new Error(`[${routeMatch.component}] rss() tried to generate RSS but "buildOptions.site" missing in astro.config.mjs`);
}
const { dest, ...rssData } = args;
const feedURL = dest || '/rss.xml';
result.url = feedURL;
result.xml = generateRSS({ rssData, site, srcFile: routeMatch.component, feedURL });
}
return [rssUtility, result];
}

View file

@ -0,0 +1,27 @@
import type { BuildOutput } from '../@types/astro';
import { canonicalURL } from './util.js';
/** Construct sitemap.xml given a set of URLs */
export function generateSitemap(buildState: BuildOutput, site: string): string {
const uniqueURLs = new Set<string>();
// TODO: find way to respect <link rel="canonical"> URLs here
// TODO: find way to exclude pages from sitemap (currently only skips 404 pages)
// look through built pages, only add HTML
for (const id of Object.keys(buildState)) {
if (buildState[id].contentType !== 'text/html') continue;
if (id === '/404.html') continue;
uniqueURLs.add(canonicalURL(id, site).href);
}
const pages = [...uniqueURLs];
pages.sort((a, b) => a.localeCompare(b, 'en', { numeric: true })); // sort alphabetically so sitemap is same each time
let sitemap = `<?xml version="1.0" encoding="UTF-8"?><urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">`;
for (const page of pages) {
sitemap += `<url><loc>${page}</loc></url>`;
}
sitemap += `</urlset>\n`;
return sitemap;
}

View file

@ -1,144 +1,92 @@
import * as eslexer from 'es-module-lexer';
import fetch from 'node-fetch';
import fs from 'fs';
import slash from 'slash';
import glob from 'tiny-glob';
import { fileURLToPath } from 'url';
import type { BuildOutput, BundleMap } from '../@types/astro';
import type { LogOptions } from '../logger';
type FileSizes = { [file: string]: number };
import { info, table } from '../logger.js';
import { underline, bold } from 'kleur/colors';
import gzipSize from 'gzip-size';
import prettyBytes from 'pretty-bytes';
// Feel free to modify output to whatevers needed in display. If its not needed, kill it and improve stat speeds!
/** JS: prioritize entry HTML, but also show total */
interface JSOutput {
/** breakdown of JS per-file */
js: FileSizes;
/** weight of index.html */
entryHTML?: number;
/** total bytes of [js], added for convenience */
total: number;
interface BundleStats {
size: number;
gzipSize: number;
}
/** HTML: total isnt important, because those are broken up requests. However, surface any anomalies / bloated HTML */
interface HTMLOutput {
/** breakdown of HTML per-file */
html: FileSizes;
/** biggest HTML file */
maxSize: number;
interface URLStats {
dynamicImports: Set<string>;
stats: BundleStats[];
}
/** Scan any directory */
async function scan(cwd: URL, pattern: string): Promise<URL[]> {
const results = await glob(pattern, { cwd: fileURLToPath(cwd) });
return results.map((filepath) => new URL(slash(filepath), cwd));
export type BundleStatsMap = Map<string, BundleStats>;
export type URLStatsMap = Map<string, URLStats>;
export function createURLStats(): URLStatsMap {
return new Map<string, URLStats>();
}
/** get total HTML size */
export async function profileHTML({ cwd }: { cwd: URL }): Promise<HTMLOutput> {
const sizes: FileSizes = {};
const html = await scan(cwd, '**/*.html');
let maxSize = 0;
await Promise.all(
html.map(async (file) => {
const relPath = file.pathname.replace(cwd.pathname, '');
const size = (await fs.promises.stat(file)).size;
sizes[relPath] = size;
if (size > maxSize) maxSize = size;
})
);
return {
html: sizes,
maxSize,
};
export function createBundleStats(): BundleStatsMap {
return new Map<string, BundleStats>();
}
/** get total JS size (note: .wasm counts as JS!) */
export async function profileJS({ cwd, entryHTML }: { cwd: URL; entryHTML?: URL }): Promise<JSOutput> {
const sizes: FileSizes = {};
let htmlSize = 0;
export async function addBundleStats(bundleStatsMap: BundleStatsMap, code: string, filename: string) {
const gzsize = await gzipSize(code);
// profile HTML entry (do this first, before all JS in a project is scanned)
if (entryHTML) {
let entryScripts: URL[] = [];
let visitedEntry = false; // note: a quirk of Vite is that the entry file is async-loaded. Count that, but dont count subsequent async loads
// Note: this function used cheerio to scan HTML, read deps, and build
// an accurate, “production-ready” benchmark for how much HTML, JS, and CSS
// you shipped. Disabled for now, because we have a post-merge cleanup item
// to revisit these build stats.
//
// let $ = cheerio.load(await fs.promises.readFile(entryHTML));
// scan <script> files, keep adding to total until done
// $('script').each((n, el) => {
// const src = $(el).attr('src');
// const innerHTML = $(el).html();
// // if inline script, add to overall JS weight
// if (innerHTML) {
// htmlSize += Buffer.byteLength(innerHTML);
// }
// // otherwise if external script, load & scan it
// if (src) {
// entryScripts.push(new URL(src, entryHTML));
// }
// });
let scanPromises: Promise<void>[] = [];
await Promise.all(entryScripts.map(parseJS));
/** parse JS for imports, and add to total size */
async function parseJS(url: URL): Promise<void> {
const relPath = url.pathname.replace(cwd.pathname, '');
if (sizes[relPath]) return;
try {
let code = url.protocol === 'file:' ? await fs.promises.readFile(url, 'utf8') : await fetch(url.href).then((body) => body.text());
sizes[relPath] = Buffer.byteLength(code);
const staticImports = eslexer.parse(code)[0].filter(({ d }) => {
if (!visitedEntry) return true; // if were on the entry file, count async imports, too
return d === -1; // subsequent runs: dont count deferred code toward total
bundleStatsMap.set(filename, {
size: Buffer.byteLength(code),
gzipSize: gzsize,
});
for (const { n } of staticImports) {
if (!n) continue;
let nextURL: URL | undefined;
// external import
if (n.startsWith('http://') || n.startsWith('https://') || n.startsWith('//')) nextURL = new URL(n);
// relative import
else if (n[0] === '.') nextURL = new URL(n, url);
// absolute import (note: make sure "//" is already handled!)
else if (n[0] === '/') nextURL = new URL(`.${n}`, cwd);
if (!nextURL) continue; // unknown format: skip
if (sizes[nextURL.pathname.replace(cwd.pathname, '')]) continue; // already scanned: skip
scanPromises.push(parseJS(nextURL));
}
} catch (err) {
console.warn(`Could not access ${url.href} to include in bundle size`); // eslint-disable-line no-console
}
visitedEntry = true; // after first run, stop counting async imports toward total
}
}
await Promise.all(scanPromises);
htmlSize = Object.values(sizes).reduce((sum, next) => sum + next, 0);
export function mapBundleStatsToURLStats({ urlStats, depTree, bundleStats }: { urlStats: URLStatsMap; depTree: BundleMap; bundleStats: BundleStatsMap }) {
for (let [srcPath, stats] of bundleStats) {
for (let url of urlStats.keys()) {
if (depTree[url] && depTree[url].js.has('/' + srcPath)) {
urlStats.get(url)?.stats.push(stats);
}
}
}
}
export async function collectBundleStats(buildState: BuildOutput, depTree: BundleMap): Promise<URLStatsMap> {
const urlStats = createURLStats();
// collect size of all JS in project (note: some may have already been scanned; skip when possible)
const js = await scan(cwd, '**/*.(js|mjs|wasm)');
await Promise.all(
js.map(async (file) => {
const relPath = file.pathname.replace(cwd.pathname, '');
if (!sizes[relPath]) sizes[relPath] = (await fs.promises.stat(file)).size; // only scan if new
Object.keys(buildState).map(async (id) => {
if (!depTree[id]) return;
const stats = await Promise.all(
[...depTree[id].js, ...depTree[id].css, ...depTree[id].images].map(async (url) => {
if (!buildState[url]) return undefined;
const stat = {
size: Buffer.byteLength(buildState[url].contents),
gzipSize: await gzipSize(buildState[url].contents),
};
return stat;
})
);
urlStats.set(id, {
dynamicImports: new Set<string>(),
stats: stats.filter((s) => !!s) as any,
});
})
);
return {
js: sizes,
entryHTML: htmlSize || undefined,
total: Object.values(sizes).reduce((sum, acc) => sum + acc, 0),
};
return urlStats;
}
/** b -> kB */
export function kb(bytes: number): string {
if (bytes === 0) return `0 kB`;
return (Math.round(bytes / 1000) || 1) + ' kB'; // if this is between 0.10.4, round up to 1
export function logURLStats(logging: LogOptions, urlStats: URLStatsMap) {
const builtURLs = [...urlStats.keys()].sort((a, b) => a.localeCompare(b, 'en', { numeric: true }));
info(logging, null, '');
const log = table(logging, [60, 20]);
log(info, ' ' + bold(underline('Pages')), bold(underline('Page Weight (GZip)')));
const lastIndex = builtURLs.length - 1;
builtURLs.forEach((url, index) => {
const sep = index === 0 ? '┌' : index === lastIndex ? '└' : '├';
const urlPart = ' ' + sep + ' ' + url;
const bytes =
urlStats
.get(url)
?.stats.map((s) => s.gzipSize)
.reduce((a, b) => a + b, 0) || 0;
const sizePart = prettyBytes(bytes);
log(info, urlPart, sizePart);
});
}

View file

@ -0,0 +1,81 @@
import type { AstroConfig } from '../@types/astro';
import { performance } from 'perf_hooks';
import fs from 'fs';
import path from 'path';
import { URL } from 'url';
/**
* Only Astro-handled imports need bundling. Any other imports are considered
* a part of `public/`, and should not be touched.
*/
export const IS_ASTRO_FILE_URL = /^\/(_astro|_astro_frontend|_snowpack)\//;
/** Normalize URL to its canonical form */
export function canonicalURL(url: string, base?: string): URL {
let pathname = url.replace(/\/index.html$/, ''); // index.html is not canonical
pathname = pathname.replace(/\/1\/?$/, ''); // neither is a trailing /1/ (impl. detail of collections)
if (!path.extname(pathname)) pathname = pathname.replace(/(\/+)?$/, '/'); // add trailing slash if theres no extension
pathname = pathname.replace(/\/+/g, '/'); // remove duplicate slashes (URL() wont)
if (base) {
return new URL('.' + pathname, base);
} else {
return new URL(pathname, base);
}
}
/** Resolve final output URL */
export function getDistPath(specifier: string, { astroConfig, srcPath }: { astroConfig: AstroConfig; srcPath: URL }): string {
if (specifier[0] === '/') return specifier; // assume absolute URLs are correct
const { pages: pagesRoot, projectRoot } = astroConfig;
const fileLoc = new URL(specifier, srcPath);
const projectLoc = fileLoc.pathname.replace(projectRoot.pathname, '');
const ext = path.extname(fileLoc.pathname);
const isPage = fileLoc.pathname.includes(pagesRoot.pathname) && (ext === '.astro' || ext === '.md');
// if this lives in src/pages, return that URL
if (isPage) {
const [, publicURL] = projectLoc.split(pagesRoot.pathname);
return publicURL || '/index.html'; // if this is missing, this is the root
}
// if this is in public/, use that as final URL
const isPublicAsset = fileLoc.pathname.includes(astroConfig.public.pathname);
if (isPublicAsset) {
return fileLoc.pathname.replace(astroConfig.public.pathname, '/');
}
// otherwise, return /_astro/* url
return '/_astro/' + projectLoc;
}
/** Given a final output URL, guess at src path (may be inaccurate; only for non-pages) */
export function getSrcPath(distURL: string, { astroConfig }: { astroConfig: AstroConfig }): URL {
if (distURL.startsWith('/_astro/')) {
return new URL('.' + distURL.replace(/^\/_astro\//, ''), astroConfig.projectRoot);
} else if (distURL === '/index.html') {
return new URL('./index.astro', astroConfig.pages);
}
const possibleURLs = [
new URL('.' + distURL, astroConfig.public), // public asset
new URL('.' + distURL.replace(/([^\/])+\/d+\/index.html/, '$$1.astro'), astroConfig.pages), // collection page
new URL('.' + distURL.replace(/\/index\.html$/, '.astro'), astroConfig.pages), // page
// TODO: Astro pages (this isnt currently used for that lookup)
];
// if this is in public/ or pages/, return that
for (const possibleURL of possibleURLs) {
if (fs.existsSync(possibleURL)) return possibleURL;
}
// otherwise resolve relative to project
return new URL('.' + distURL, astroConfig.projectRoot);
}
/** Stop timer & format time for profiling */
export function stopTimer(start: number): string {
const diff = performance.now() - start;
return diff < 750 ? `${Math.round(diff)}ms` : `${(diff / 1000).toFixed(1)}s`;
}

View file

@ -73,14 +73,19 @@ async function resolveRenderers(viteServer: vite.ViteDevServer, astroConfig: Ast
return renderers;
}
export async function loadModule(filePath: URL, viteServer: vite.ViteDevServer): Promise<ComponentInstance> {
const viteFriendlyURL = `/@fs${filePath.pathname}`;
const mod = (await viteServer.ssrLoadModule(viteFriendlyURL)) as ComponentInstance;
return mod;
}
/** use Vite to SSR */
export async function ssr({ astroConfig, filePath, logging, mode, origin, pathname, route, routeCache, viteServer }: SSROptions): Promise<string> {
try {
// Important: This needs to happen first, in case a renderer provides polyfills.
const renderers = await resolveRenderers(viteServer, astroConfig);
// Load the module from the Vite SSR Runtime.
const viteFriendlyURL = `/@fs${filePath.pathname}`;
const mod = (await viteServer.ssrLoadModule(viteFriendlyURL)) as ComponentInstance;
const mod = await loadModule(filePath, viteServer);
// Handle dynamic routes
let params: Params = {};
let pageProps: Props = {};

View file

@ -9910,6 +9910,11 @@ sprintf-js@~1.0.2:
resolved "https://registry.yarnpkg.com/sprintf-js/-/sprintf-js-1.0.3.tgz#04e6926f662895354f3dd015203633b857297e2c"
integrity sha1-BOaSb2YolTVPPdAVIDYzuFcpfiw=
srcset-parse@^1.1.0:
version "1.1.0"
resolved "https://registry.yarnpkg.com/srcset-parse/-/srcset-parse-1.1.0.tgz#73f787f38b73ede2c5af775e0a3465579488122b"
integrity sha512-JWp4cG2eybkvKA1QUHGoNK6JDEYcOnSuhzNGjZuYUPqXreDl/VkkvP2sZW7Rmh+icuCttrR9ccb2WPIazyM/Cw==
sshpk@^1.7.0:
version "1.16.1"
resolved "https://registry.yarnpkg.com/sshpk/-/sshpk-1.16.1.tgz#fb661c0bef29b39db40769ee39fa70093d6f6877"