michael/astro
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Current progress

Browse source
This commit is contained in:
Matthew Phillips 2021-11-05 08:49:10 -04:00
parent 1ce6b2ac85
commit b82e42500f
12 changed files with 133 additions and 66 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
packages/astro/package.json
View file

@ -71,6 +71,7 @@
"esbuild": "^0.13.6",
"estree-util-value-to-estree": "^1.2.0",
"fast-xml-parser": "^3.19.0",
"gzip-size": "^6.0.0",
"html-entities": "^2.3.2",
"htmlparser2": "^7.1.2",
"kleur": "^4.1.4",
@ -78,9 +79,11 @@
"morphdom": "^2.6.1",
"node-fetch": "^2.6.5",
"path-to-regexp": "^6.2.0",
"pretty-bytes": "^5.6.0",
"prismjs": "^1.25.0",
"remark-slug": "^7.0.0",
"resolve": "^1.20.0",
"rollup-plugin-terser": "^7.0.2",
"sass": "^1.43.3",
"semver": "^7.3.5",
"send": "^0.17.1",

5
packages/astro/src/@types/astro-build.ts
View file

@ -1,4 +1,5 @@
import type { ScriptInfo } from './astro-core';
import type { Response as FetchResponse } from 'node-fetch';
/** Entire output of `astro build`, stored in memory */
export interface BuildOutput {
@ -32,4 +33,6 @@ export interface PageDependencies {
images: Set<string>;
/** Async hoisted Javascript */
hoistedJS: Map<string, ScriptInfo>;
}
}
export type ServerFetch = (pathName: string) => Promise<FetchResponse>;

36
packages/astro/src/core/build/bundle/js.ts
View file

@ -1,8 +1,8 @@
import type { InputOptions, OutputOptions, OutputChunk } from 'rollup';
import type { AstroConfig, ScriptInfo } from '../../../@types/astro-core';
//import type { AstroConfig, BundleMap, BuildOutput, ScriptInfo, InlineScriptInfo } from '../../../@types/astro-build';
import type { AstroRuntime } from '../../runtime';
import type { AstroConfig, ScriptInfo, ScriptInfoInline } from '../../../@types/astro-core';
import type { BundleMap, BuildOutput, ServerFetch } from '../../../@types/astro-build';
import type { LogOptions } from '../../logger.js';
import type { ViteDevServer } from 'vite';
import { fileURLToPath } from 'url';
import { rollup } from 'rollup';
@ -14,7 +14,8 @@ import path from 'path';
interface BundleOptions {
dist: URL;
astroRuntime: AstroRuntime;
fetchPath: ServerFetch;
viteServer: ViteDevServer;
}
/** Collect JS imports from build output */
@ -32,18 +33,14 @@ function pageUrlToVirtualJSEntry(pageUrl: string) {
export async function bundleHoistedJS({
buildState,
astroConfig,
logging,
fetchPath,
depTree,
dist,
runtime,
dist
}: {
astroConfig: AstroConfig;
buildState: BuildOutput;
logging: LogOptions;
depTree: BundleMap;
dist: URL;
runtime: AstroRuntime;
fetchPath: ServerFetch;
}) {
const sortedPages = Object.keys(depTree); // these were scanned in parallel; sort to create somewhat deterministic order
sortedPages.sort((a, b) => a.localeCompare(b, 'en', { numeric: true }));
@ -104,17 +101,18 @@ export async function bundleHoistedJS({
},
async load(id: string) {
if (virtualScripts.has(id)) {
let info = virtualScripts.get(id) as InlineScriptInfo;
let info = virtualScripts.get(id) as ScriptInfoInline;
return info.content;
}
const result = await runtime.load(id);
// TODO replace with fetch
const result = await fetchPath(id);
if (result.statusCode !== 200) {
if (!result.ok) {
return null;
}
return result.contents.toString('utf-8');
return await result.text();
},
},
],
@ -181,7 +179,7 @@ export async function bundleHoistedJS({
}
/** Bundle JS action */
export async function bundleJS(imports: Set<string>, { astroRuntime, dist }: BundleOptions): Promise<BundleStatsMap> {
export async function bundleJS(imports: Set<string>, { fetchPath, dist }: BundleOptions): Promise<BundleStatsMap> {
const ROOT = 'astro:root';
const validImports = [...imports].filter((url) => IS_ASTRO_FILE_URL.test(url));
const root = `
@ -213,13 +211,13 @@ export async function bundleJS(imports: Set<string>, { astroRuntime, dist }: Bun
return root;
}
const result = await astroRuntime.load(id);
const result = await fetchPath(id);
if (result.statusCode !== 200) {
if (!result.ok) {
return null;
}
return result.contents.toString('utf-8');
return await result.text();
},
},
],

63
packages/astro/src/core/build/index.ts
View file

@ -1,9 +1,11 @@
import type { AstroConfig, RouteData, RuntimeMode, ScriptInfo } from '../../@types/astro-core';
import type { BuildOutput, BundleMap, PageDependencies } from '../../@types/astro-build';
import type { AstroConfig, RouteCache, RouteData, RuntimeMode, ScriptInfo } from '../../@types/astro-core';
import type { BuildOutput, BundleMap, PageDependencies, ServerFetch } from '../../@types/astro-build';
import type { LogOptions } from '../logger';
import cheerio from 'cheerio';
import connect from 'connect';
import del from 'del';
import eslexer from 'es-module-lexer';
import * as eslexer from 'es-module-lexer';
import fs from 'fs';
import { bold, green, red, underline, yellow } from 'kleur/colors';
import mime from 'mime';
@ -13,6 +15,7 @@ import glob from 'tiny-glob';
import hash from 'shorthash';
import srcsetParse from 'srcset-parse';
import { fileURLToPath } from 'url';
import fetch from 'node-fetch';
import { bundleCSS } from './bundle/css.js';
import { bundleJS, bundleHoistedJS, collectJSImports } from './bundle/js.js';
@ -20,7 +23,6 @@ import { buildStaticPage, getStaticPathsForPage } from './page.js';
import { generateSitemap } from './sitemap.js';
import { collectBundleStats, logURLStats, mapBundleStatsToURLStats } from './stats.js';
import { getDistPath, stopTimer } from './util.js';
import type { LogOptions } from '../logger';
import { debug, defaultLogDestination, defaultLogLevel, error, info, warn } from '../logger.js';
import { createVite } from '../create-vite.js';
import vite, { ViteDevServer } from '../vite.js';
@ -39,17 +41,18 @@ function isRemoteOrEmbedded(url: string) {
return url.startsWith('http://') || url.startsWith('https://') || url.startsWith('//') || url.startsWith('data:');
}
interface BuildOptions {
logging: LogOptions;
}
/** The primary build action */
export async function build(astroConfig: AstroConfig, logging: LogOptions = defaultLogging): Promise<0 | 1> {
export default async function build(astroConfig: AstroConfig, options: BuildOptions = { logging: defaultLogging }): Promise<0 | 1> {
const { projectRoot } = astroConfig;
const { logging } = options;
const buildState: BuildOutput = {};
const depTree: BundleMap = {};
const timer: Record<string, number> = {};
const runtimeLogging: LogOptions = {
level: 'error',
dest: defaultLogDestination,
};
const routeCache: RouteCache = {};
// warn users if missing config item in build that may result in broken SEO (can’t disable, as they should provide this)
if (!astroConfig.buildOptions.site) {
@ -69,7 +72,17 @@ export async function build(astroConfig: AstroConfig, logging: LogOptions = defa
},
{ astroConfig, logging }
);
const app = connect();
const viteServer = await vite.createServer(viteConfig);
app.use(viteServer.middlewares);
const port = astroConfig.devOptions.port;
const origin = astroConfig.buildOptions.site ? new URL(astroConfig.buildOptions.site).origin : `http://localhost:${port}`;
await new Promise<void>(resolve => {
app.listen(port, () => {
resolve();
});
});
const fetchPath: ServerFetch = (pathName: string) => fetch(new URL(pathName, `http://localhost:${port}`).toString());
const manifest = createRouteManifest({ config: astroConfig });
@ -120,9 +133,12 @@ export async function build(astroConfig: AstroConfig, logging: LogOptions = defa
await buildStaticPage({
astroConfig,
buildState,
logging,
origin,
route,
path: p,
astroRuntime,
routeCache,
viteServer
});
}
})
@ -146,6 +162,7 @@ ${stack}
} else {
error(logging, 'build', e.message);
}
console.error(e); // TODO REMOVE
error(logging, 'build', red('✕ building pages failed!'));
await viteServer.close();
@ -172,22 +189,24 @@ ${stack}
for (const url of [...pageDeps.js, ...pageDeps.css, ...pageDeps.images]) {
if (!buildState[url])
scanPromises.push(
astroRuntime.load(url).then((result: LoadResult) => {
if (result.statusCode === 404) {
fetchPath(url).then(async (result) => {
if (result.status === 404) {
if (url.startsWith('/_astro/')) {
throw new Error(`${buildState[id].srcPath.href}: could not find file "${url}".`);
}
warn(logging, 'build', `${buildState[id].srcPath.href}: could not find file "${url}". Marked as external.`);
return;
}
if (result.statusCode !== 200) {
if (result.status !== 200) {
// there shouldn’t be a build error here
throw (result as any).error || new Error(`unexpected ${result.statusCode} response from "${url}".`);
throw (result as any).error || new Error(`unexpected ${result.status} response from "${url}".`);
}
let contentType = result.headers.get('content-type');
let buffer = Buffer.from(await result.arrayBuffer());
buildState[url] = {
srcPath: new URL(url, projectRoot),
contents: result.contents,
contentType: result.contentType || mime.getType(url) || '',
contents: buffer,
contentType: contentType || mime.getType(url) || '',
};
})
);
@ -206,7 +225,7 @@ ${stack}
bundleCSS({ buildState, astroConfig, logging, depTree }).then(() => {
debug(logging, 'build', `bundled CSS [${stopTimer(timer.prebundleCSS)}]`);
}),
bundleHoistedJS({ buildState, astroConfig, logging, depTree, runtime: astroRuntime, dist: astroConfig.dist }),
bundleHoistedJS({ buildState, depTree, fetchPath, dist: astroConfig.dist }),
// TODO: optimize images?
]);
// TODO: minify HTML?
@ -284,7 +303,7 @@ ${stack}
info(logging, 'build', yellow(`! bundling...`));
if (jsImports.size > 0) {
timer.bundleJS = performance.now();
const jsStats = await bundleJS(jsImports, { dist: astroConfig.dist, astroRuntime });
const jsStats = await bundleJS(jsImports, { dist: astroConfig.dist, fetchPath, viteServer });
mapBundleStatsToURLStats({ urlStats, depTree, bundleStats: jsStats });
debug(logging, 'build', `bundled JS [${stopTimer(timer.bundleJS)}]`);
info(logging, 'build', green(`✔`), 'bundling complete.');
@ -294,12 +313,12 @@ ${stack}
* 6. Print stats
*/
logURLStats(logging, urlStats);
await astroRuntime.shutdown();
await viteServer.close();
info(logging, 'build', bold(green('▶ Build Complete!')));
return 0;
} catch (err) {
} catch (err: any) {
error(logging, 'build', err.message);
await astroRuntime.shutdown();
await viteServer.close();
return 1;
}
}

27
packages/astro/src/core/build/page.ts
View file

@ -1,9 +1,9 @@
import type { AstroConfig, RouteData } from '../../@types/astro-core';
import type { AstroConfig, RouteCache, RouteData } from '../../@types/astro-core';
import type { BuildOutput } from '../../@types/astro-build';
import type { ViteDevServer } from '../vite';
import type { LogOptions } from '../logger';
import _path from 'path';
import { fileURLToPath } from 'url';
import { LogOptions } from '../logger';
import { loadModule, ssr } from '../ssr/index.js';
import { validateGetStaticPathsModule, validateGetStaticPathsResult } from '../ssr/routing.js';
import { generatePaginateFunction } from './paginate.js';
@ -18,15 +18,19 @@ function convertMatchToLocation(routeMatch: RouteData, astroConfig: AstroConfig)
const url = new URL(`./${routeMatch.component}`, astroConfig.projectRoot);
return {
fileURL: url,
snowpackURL: `/_astro/${routeMatch.component}.js`,
snowpackURL: `/_astro/${routeMatch.component}.js`, // TODO remove
};
}
interface PageBuildOptions {
astroConfig: AstroConfig;
buildState: BuildOutput;
logging: LogOptions;
origin: string;
path: string;
route: RouteData;
routeCache: RouteCache;
viteServer: ViteDevServer;
}
/** Build dynamic page */
@ -70,11 +74,11 @@ function formatOutFile(path: string, pageUrlFormat: AstroConfig['buildOptions'][
return `${path}.html`;
}
/** Build static page */
export async function buildStaticPage({ astroConfig, buildState, path, route }: PageBuildOptions): Promise<void> {
export async function buildStaticPage({ astroConfig, buildState, logging, origin, path, route, routeCache, viteServer }: PageBuildOptions): Promise<void> {
const location = convertMatchToLocation(route, astroConfig);
const normalizedPath = astroConfig.devOptions.trailingSlash === 'never' ? path : path.endsWith('/') ? path : `${path}/`;
//const normalizedPath = astroConfig.devOptions.trailingSlash === 'never' ? path : path.endsWith('/') ? path : `${path}/`;
const r = ssr({
const html = await ssr({
astroConfig,
filePath: location.fileURL,
logging,
@ -84,18 +88,11 @@ export async function buildStaticPage({ astroConfig, buildState, path, route }:
route,
routeCache,
viteServer
})
});
const result = await astroRuntime.load(normalizedPath);
if (result.statusCode !== 200) {
let err = (result as any).error;
if (!(err instanceof Error)) err = new Error(err);
err.filename = fileURLToPath(location.fileURL);
throw err;
}
buildState[formatOutFile(path, astroConfig.buildOptions.pageUrlFormat)] = {
srcPath: location.fileURL,
contents: result.contents,
contents: html,
contentType: 'text/html',
encoding: 'utf8',
};

2
packages/astro/src/core/build/paginate.ts
View file

@ -1,4 +1,4 @@
import { GetStaticPathsResult, PaginatedCollectionProp, PaginateFunction, Params, Props, RouteData } from '../@types/astro';
import { GetStaticPathsResult, PaginatedCollectionProp, PaginateFunction, Params, Props, RouteData } from '../../@types/astro-core';
// return filters.map((filter) => {
// const filteredRecipes = allRecipes.filter((recipe) =>

2
packages/astro/src/core/build/rss.ts
View file

@ -1,4 +1,4 @@
import type { RSSFunctionArgs, RouteData } from '../@types/astro';
import type { RSSFunctionArgs, RouteData } from '../../@types/astro-core';
import parser from 'fast-xml-parser';
import { canonicalURL } from './util.js';

2
packages/astro/src/core/build/sitemap.ts
View file

@ -1,4 +1,4 @@
import type { BuildOutput } from '../@types/astro';
import type { BuildOutput } from '../../@types/astro-build';
import { canonicalURL } from './util.js';
/** Construct sitemap.xml given a set of URLs */

2
packages/astro/src/core/build/stats.ts
View file

@ -1,4 +1,4 @@
import type { BuildOutput, BundleMap } from '../@types/astro';
import type { BuildOutput, BundleMap } from '../../@types/astro-build';
import type { LogOptions } from '../logger';
import { info, table } from '../logger.js';

2
packages/astro/src/core/build/util.ts
View file

@ -1,4 +1,4 @@
import type { AstroConfig } from '../@types/astro';
import type { AstroConfig } from '../../@types/astro-core';
import { performance } from 'perf_hooks';
import fs from 'fs';

4
packages/astro/src/core/ssr/index.ts
View file

@ -192,9 +192,9 @@ export async function ssr({ astroConfig, filePath, logging, mode, origin, pathna
html = injectTags(html, tags);
// run transformIndexHtml() in dev to run Vite dev transformations
if (mode === 'development') {
//if (mode === 'development') {
html = await viteServer.transformIndexHtml(filePath.pathname, html, pathname);
}
//}
return html;
} catch (e: any) {

51
yarn.lock
View file

@ -3929,7 +3929,7 @@ dotenv@^8.2.0:
resolved "https://registry.yarnpkg.com/dotenv/-/dotenv-8.6.0.tgz#061af664d19f7f4d8fc6e4ff9b584ce237adcb8b"
integrity sha512-IrPdXQsk2BbzvCBGBOTmmSH5SodmqZNt4ERAZDmW4CT+tL8VtvinqywuANaFu4bOMWki16nqf0e4oC0QIaDr/g==
duplexer@^0.1.1, duplexer@~0.1.1:
duplexer@^0.1.1, duplexer@^0.1.2, duplexer@~0.1.1:
version "0.1.2"
resolved "https://registry.yarnpkg.com/duplexer/-/duplexer-0.1.2.tgz#3abe43aef3835f8ae077d136ddce0f276b0400e6"
integrity sha512-jtD6YG370ZCIi/9GTaJKQxWTZD045+4R4hTk/x1UyoqadyJ9x9CgSi1RlVDQF8U2sxLLSnFkCaMihqljHIWgMg==
@ -5285,6 +5285,13 @@ growl@1.10.5:
resolved "https://registry.yarnpkg.com/growl/-/growl-1.10.5.tgz#f2735dc2283674fa67478b10181059355c369e5e"
integrity sha512-qBr4OuELkhPenW6goKVXiv47US3clb3/IbuWF9KNKEijAy9oeHxU9IgzjvJhHkUzhaj7rOUD7+YGWqUjLp5oSA==
gzip-size@^6.0.0:
version "6.0.0"
resolved "https://registry.yarnpkg.com/gzip-size/-/gzip-size-6.0.0.tgz#065367fd50c239c0671cbcbad5be3e2eeb10e462"
integrity sha512-ax7ZYomf6jqPTQ4+XCpUGyXKHk5WweS+e05MBO4/y3WJ5RkmPXNKvX+bx1behVILVwr6JSQvZAku021CHPXG3Q==
dependencies:
duplexer "^0.1.2"
handlebars@^4.7.6:
version "4.7.7"
resolved "https://registry.yarnpkg.com/handlebars/-/handlebars-4.7.7.tgz#9ce33416aad02dbd6c8fafa8240d5d98004945a1"
@ -6254,6 +6261,15 @@ isstream@~0.1.2:
resolved "https://registry.yarnpkg.com/isstream/-/isstream-0.1.2.tgz#47e63f7af55afa6f92e1500e690eb8b8529c099a"
integrity sha1-R+Y/evVa+m+S4VAOaQ64uFKcCZo=
jest-worker@^26.2.1:
version "26.6.2"
resolved "https://registry.yarnpkg.com/jest-worker/-/jest-worker-26.6.2.tgz#7f72cbc4d643c365e27b9fd775f9d0eaa9c7a8ed"
integrity sha512-KWYVV1c4i+jbMpaBC+U++4Va0cp8OisU185o73T1vo99hqi7w8tSJfUXYswwqqrjzwxa6KpRK54WhPvwf5w6PQ==
dependencies:
"@types/node" "*"
merge-stream "^2.0.0"
supports-color "^7.0.0"
joi@^17.4.0:
version "17.4.2"
resolved "https://registry.yarnpkg.com/joi/-/joi-17.4.2.tgz#02f4eb5cf88e515e614830239379dcbbe28ce7f7"
@ -8740,6 +8756,11 @@ prettier@^2.4.1:
resolved "https://registry.yarnpkg.com/prettier/-/prettier-2.4.1.tgz#671e11c89c14a4cfc876ce564106c4a6726c9f5c"
integrity sha512-9fbDAXSBcc6Bs1mZrDYb3XKzDLm4EXXL9sC1LqKP5rZkT6KRr/rf9amVUcODVXgguK/isJz0d0hP72WeaKWsvA==
pretty-bytes@^5.6.0:
version "5.6.0"
resolved "https://registry.yarnpkg.com/pretty-bytes/-/pretty-bytes-5.6.0.tgz#356256f643804773c82f64723fe78c92c62beaeb"
integrity sha512-FFw039TmrBqFK8ma/7OL3sDz/VytdtJr044/QUJtH0wK9lb9jLq9tJyIxUwtQJHwar2BqtiA4iCWSwo9JLkzFg==
pretty-format@^3.8.0:
version "3.8.0"
resolved "https://registry.yarnpkg.com/pretty-format/-/pretty-format-3.8.0.tgz#bfbed56d5e9a776645f4b1ff7aa1a3ac4fa3c385"
@ -9504,6 +9525,16 @@ robots-txt-parse@~0.0.4:
stream-combiner "^0.2.1"
through "^2.3.4"
rollup-plugin-terser@^7.0.2:
version "7.0.2"
resolved "https://registry.yarnpkg.com/rollup-plugin-terser/-/rollup-plugin-terser-7.0.2.tgz#e8fbba4869981b2dc35ae7e8a502d5c6c04d324d"
integrity sha512-w3iIaU4OxcF52UUXiZNsNeuXIMDvFrr+ZXK6bFZ0Q60qyVfq4uLptoS4bbq3paG3x216eQllFZX7zt6TIImguQ==
dependencies:
"@babel/code-frame" "^7.10.4"
jest-worker "^26.2.1"
serialize-javascript "^4.0.0"
terser "^5.0.0"
rollup@^2.57.0:
version "2.58.0"
resolved "https://registry.yarnpkg.com/rollup/-/rollup-2.58.0.tgz#a643983365e7bf7f5b7c62a8331b983b7c4c67fb"
@ -9625,6 +9656,13 @@ serialize-javascript@6.0.0:
dependencies:
randombytes "^2.1.0"
serialize-javascript@^4.0.0:
version "4.0.0"
resolved "https://registry.yarnpkg.com/serialize-javascript/-/serialize-javascript-4.0.0.tgz#b525e1238489a5ecfc42afacc3fe99e666f4b1aa"
integrity sha512-GaNA54380uFefWghODBWEGisLZFj00nS5ACs6yHa9nLqlLpVLO8ChDGeKRjZnV4Nh4n0Qi7nhYZD/9fCPzEqkw==
dependencies:
randombytes "^2.1.0"
set-blocking@^2.0.0, set-blocking@~2.0.0:
version "2.0.0"
resolved "https://registry.yarnpkg.com/set-blocking/-/set-blocking-2.0.0.tgz#045f9782d011ae9a6803ddd382b24392b3d890f7"
@ -10205,7 +10243,7 @@ supports-color@^5.3.0:
dependencies:
has-flag "^3.0.0"
supports-color@^7.1.0:
supports-color@^7.0.0, supports-color@^7.1.0:
version "7.2.0"
resolved "https://registry.yarnpkg.com/supports-color/-/supports-color-7.2.0.tgz#1b7dcdcb32b8138801b3e478ba6a51caa89648da"
integrity sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==
@ -10320,6 +10358,15 @@ term-size@^2.1.0:
resolved "https://registry.yarnpkg.com/term-size/-/term-size-2.2.1.tgz#2a6a54840432c2fb6320fea0f415531e90189f54"
integrity sha512-wK0Ri4fOGjv/XPy8SBHZChl8CM7uMc5VML7SqiQ0zG7+J5Vr+RMQDoHa2CNT6KHUnTGIXH34UDMkPzAUyapBZg==
terser@^5.0.0:
version "5.9.0"
resolved "https://registry.yarnpkg.com/terser/-/terser-5.9.0.tgz#47d6e629a522963240f2b55fcaa3c99083d2c351"
integrity sha512-h5hxa23sCdpzcye/7b8YqbE5OwKca/ni0RQz1uRX3tGh8haaGHqcuSqbGRybuAKNdntZ0mDgFNXPJ48xQ2RXKQ==
dependencies:
commander "^2.20.0"
source-map "~0.7.2"
source-map-support "~0.5.20"
terser@^5.7.2:
version "5.8.0"
resolved "https://registry.yarnpkg.com/terser/-/terser-5.8.0.tgz#c6d352f91aed85cc6171ccb5e84655b77521d947"