michael/astro
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Make vite-plugin-content-virtual-mod run getEntrySlug 10 at a time (#7125)

Browse source
This commit is contained in:
Bjorn Lu 2023-05-23 21:54:59 +08:00 committed by GitHub
parent 2ca94269ed
commit 4ce8bf7c62
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
4 changed files with 68 additions and 49 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
.changeset/dull-teachers-work.md Normal file
View file

@ -0,0 +1,5 @@
---
'astro': patch
---
Make vite-plugin-content-virtual-mod run `getEntrySlug` 10 at a time to prevent `EMFILE: too many open files` error

1
packages/astro/package.json
View file

@ -148,6 +148,7 @@
"magic-string": "^0.27.0",
"mime": "^3.0.0",
"ora": "^6.1.0",
"p-limit": "^4.0.0",
"path-to-regexp": "^6.2.1",
"preferred-pm": "^3.0.3",
"prompts": "^2.4.2",

108
packages/astro/src/content/vite-plugin-content-virtual-mod.ts
View file

@ -1,4 +1,5 @@
import glob from 'fast-glob';
import pLimit from 'p-limit';
import fsMod from 'node:fs';
import { extname } from 'node:path';
import { fileURLToPath, pathToFileURL } from 'node:url';
@ -114,59 +115,68 @@ export async function getStringifiedLookupMap({
}
);
await Promise.all(
contentGlob.map(async (filePath) => {
const entryType = getEntryType(filePath, contentPaths, contentEntryExts, dataEntryExts);
// Globbed ignored or unsupported entry.
// Logs warning during type generation, should ignore in lookup map.
if (entryType !== 'content' && entryType !== 'data') return;
// Run 10 at a time to prevent `await getEntrySlug` from accessing the filesystem all at once.
// Each await shouldn't take too long for the work to be noticably slow too.
const limit = pLimit(10);
const promises: Promise<void>[] = [];
const collection = getEntryCollectionName({ contentDir, entry: pathToFileURL(filePath) });
if (!collection) throw UnexpectedLookupMapError;
for (const filePath of contentGlob) {
promises.push(
limit(async () => {
const entryType = getEntryType(filePath, contentPaths, contentEntryExts, dataEntryExts);
// Globbed ignored or unsupported entry.
// Logs warning during type generation, should ignore in lookup map.
if (entryType !== 'content' && entryType !== 'data') return;
if (lookupMap[collection]?.type && lookupMap[collection].type !== entryType) {
throw new AstroError({
...AstroErrorData.MixedContentDataCollectionError,
message: AstroErrorData.MixedContentDataCollectionError.message(collection),
});
}
const collection = getEntryCollectionName({ contentDir, entry: pathToFileURL(filePath) });
if (!collection) throw UnexpectedLookupMapError;
if (entryType === 'content') {
const contentEntryType = contentEntryConfigByExt.get(extname(filePath));
if (!contentEntryType) throw UnexpectedLookupMapError;
if (lookupMap[collection]?.type && lookupMap[collection].type !== entryType) {
throw new AstroError({
...AstroErrorData.MixedContentDataCollectionError,
message: AstroErrorData.MixedContentDataCollectionError.message(collection),
});
}
const { id, slug: generatedSlug } = await getContentEntryIdAndSlug({
entry: pathToFileURL(filePath),
contentDir,
collection,
});
const slug = await getEntrySlug({
id,
collection,
generatedSlug,
fs,
fileUrl: pathToFileURL(filePath),
contentEntryType,
});
lookupMap[collection] = {
type: 'content',
entries: {
...lookupMap[collection]?.entries,
[slug]: rootRelativePath(root, filePath),
},
};
} else {
const id = getDataEntryId({ entry: pathToFileURL(filePath), contentDir, collection });
lookupMap[collection] = {
type: 'data',
entries: {
...lookupMap[collection]?.entries,
[id]: rootRelativePath(root, filePath),
},
};
}
})
);
if (entryType === 'content') {
const contentEntryType = contentEntryConfigByExt.get(extname(filePath));
if (!contentEntryType) throw UnexpectedLookupMapError;
const { id, slug: generatedSlug } = await getContentEntryIdAndSlug({
entry: pathToFileURL(filePath),
contentDir,
collection,
});
const slug = await getEntrySlug({
id,
collection,
generatedSlug,
fs,
fileUrl: pathToFileURL(filePath),
contentEntryType,
});
lookupMap[collection] = {
type: 'content',
entries: {
...lookupMap[collection]?.entries,
[slug]: rootRelativePath(root, filePath),
},
};
} else {
const id = getDataEntryId({ entry: pathToFileURL(filePath), contentDir, collection });
lookupMap[collection] = {
type: 'data',
entries: {
...lookupMap[collection]?.entries,
[id]: rootRelativePath(root, filePath),
},
};
}
})
);
}
await Promise.all(promises);
return JSON.stringify(lookupMap);
}

3
pnpm-lock.yaml
View file

@ -639,6 +639,9 @@ importers:
ora:
specifier: ^6.1.0
version: 6.1.0
p-limit:
specifier: ^4.0.0
version: 4.0.0
path-to-regexp:
specifier: ^6.2.1
version: 6.2.1