michael/fastdoc
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Walk

Browse source
This commit is contained in:
Michael Zhang 2023-05-18 13:54:00 -05:00
parent 8371167ee3
commit b5760b09a5
9 changed files with 251 additions and 53 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
.envrc Normal file
View file

@ -0,0 +1 @@
use flake

1
.gitignore vendored
View file

@ -1 +1,2 @@
/target
.direnv

3
Cargo.lock generated
View file

@ -359,7 +359,10 @@ dependencies = [
"console",
"futures",
"indicatif",
"lazy_static",
"regex",
"swc_common",
"swc_ecma_ast",
"swc_ecma_parser",
"tokio",
]

3
Cargo.toml
View file

@ -12,6 +12,9 @@ clap = { version = "4.2.7", features = ["derive"] }
console = "0.15.6"
futures = "0.3.28"
indicatif = { version = "0.17.3", features = ["tokio", "improved_unicode", "vt100"] }
lazy_static = "1.4.0"
regex = "1.8.1"
swc_common = { version = "0.31.10", features = ["tty-emitter"] }
swc_ecma_ast = "0.104.3"
swc_ecma_parser = "0.134.6"
tokio = { version = "1.28.1", features = ["full"] }

112
flake.lock Normal file
View file

@ -0,0 +1,112 @@
{
"nodes": {
"fenix": {
"inputs": {
"nixpkgs": "nixpkgs",
"rust-analyzer-src": "rust-analyzer-src"
},
"locked": {
"lastModified": 1684390923,
"narHash": "sha256-8LSe1K8ua8uyqc0B8wsQEY9qNCTX2h78mzUt2A7Uqhg=",
"owner": "nix-community",
"repo": "fenix",
"rev": "8a69206e50ca6a2cc4ae7a22eef4bfcbe6dbc9f6",
"type": "github"
},
"original": {
"owner": "nix-community",
"repo": "fenix",
"type": "github"
}
},
"flake-utils": {
"inputs": {
"systems": "systems"
},
"locked": {
"lastModified": 1681202837,
"narHash": "sha256-H+Rh19JDwRtpVPAWp64F+rlEtxUWBAQW28eAi3SRSzg=",
"owner": "numtide",
"repo": "flake-utils",
"rev": "cfacdce06f30d2b68473a46042957675eebb3401",
"type": "github"
},
"original": {
"id": "flake-utils",
"type": "indirect"
}
},
"nixpkgs": {
"locked": {
"lastModified": 1684305980,
"narHash": "sha256-vd4SKXX1KZfSX6n3eoguJw/vQ+sBL8XGdgfxjEgLpKc=",
"owner": "nixos",
"repo": "nixpkgs",
"rev": "e6e389917a8c778be636e67a67ec958f511cc55d",
"type": "github"
},
"original": {
"owner": "nixos",
"ref": "nixos-unstable",
"repo": "nixpkgs",
"type": "github"
}
},
"nixpkgs_2": {
"locked": {
"lastModified": 1684432087,
"narHash": "sha256-3zFTOY/3+kN9x9Zdq6ixLmgV4ZcEd1aafq41v/OVUek=",
"owner": "nixos",
"repo": "nixpkgs",
"rev": "7721e0d2c1845c24eafd5a016b9d349187c48097",
"type": "github"
},
"original": {
"owner": "nixos",
"repo": "nixpkgs",
"type": "github"
}
},
"root": {
"inputs": {
"fenix": "fenix",
"flake-utils": "flake-utils",
"nixpkgs": "nixpkgs_2"
}
},
"rust-analyzer-src": {
"flake": false,
"locked": {
"lastModified": 1684145961,
"narHash": "sha256-Ms99ML1P53EC50TnznmV55QwhOJtql75BbXfyiGuFvU=",
"owner": "rust-lang",
"repo": "rust-analyzer",
"rev": "2f8cd66fb4c98026d2bdbdf17270e3472e1ca42a",
"type": "github"
},
"original": {
"owner": "rust-lang",
"ref": "nightly",
"repo": "rust-analyzer",
"type": "github"
}
},
"systems": {
"locked": {
"lastModified": 1681028828,
"narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
"owner": "nix-systems",
"repo": "default",
"rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
"type": "github"
},
"original": {
"owner": "nix-systems",
"repo": "default",
"type": "github"
}
}
},
"root": "root",
"version": 7
}

42
flake.nix Normal file
View file

@ -0,0 +1,42 @@
{
inputs = {
nixpkgs.url = "github:nixos/nixpkgs";
fenix.url = "github:nix-community/fenix";
};
outputs = { self, nixpkgs, flake-utils, fenix }:
flake-utils.lib.eachDefaultSystem (system:
let
pkgs = import nixpkgs {
inherit system;
overlays = [ fenix.overlays.default ];
};
toolchain = pkgs.fenix.stable;
flakePkgs = rec { };
in rec {
packages = flake-utils.lib.flattenTree flakePkgs;
devShell = pkgs.mkShell {
inputsFrom = with flakePkgs; [ ];
packages = (with pkgs; [
cargo-deny
cargo-edit
cargo-expand
cargo-flamegraph
cargo-watch
]) ++ (with toolchain; [
cargo
rustc
clippy
# Get the nightly version of rustfmt so we can wrap comments
pkgs.fenix.default.rustfmt
]);
CARGO_UNSTABLE_SPARSE_REGISTRY = "true";
};
});
}

99
src/main.rs
View file

@ -1,17 +1,21 @@
pub mod progress_bar;
pub mod walkdir;
use std::path::{Path, PathBuf};
use std::sync::mpsc;
use std::sync::mpsc::{self, Sender};
use std::thread;
use anyhow::{Error, Result};
use async_walkdir::{Filtering, WalkDir};
use clap::Parser;
use console::Term;
use futures::{stream, StreamExt, TryStreamExt};
use futures::{StreamExt, TryStreamExt};
use indicatif::{ProgressBar, ProgressStyle};
use swc_common::errors::ColorConfig;
use swc_common::{errors::Handler, sync::Lrc};
use swc_common::{FileName, SourceMap};
use swc_ecma_parser::EsConfig;
use swc_ecma_ast::ModuleDecl;
use swc_ecma_parser::TsConfig;
use swc_ecma_parser::{
lexer::Lexer as EcmaLexer, Parser as EcmaParser, StringInput, Syntax,
};
@ -19,10 +23,13 @@ use tokio::fs;
use tokio::sync::Semaphore;
use tokio::{fs::File, io::AsyncReadExt};
use crate::progress_bar::BarAction;
use crate::walkdir::create_walkdir;
#[derive(Parser, Debug)]
struct Opt {
/// Output directory
#[clap(default_value = "generated/doc")]
#[clap(long = "out-dir", default_value = "generated/doc")]
out_dir: PathBuf,
/// Paths to process
@ -59,33 +66,29 @@ async fn main() -> Result<()> {
);
bar.set_prefix("Documenting");
enum BarAction {
Inc,
Finish,
SetLen(u64),
}
let (tx, rx) = mpsc::channel::<BarAction>();
thread::spawn(move || {
for msg in rx.into_iter() {
match msg {
BarAction::Inc => bar.inc(1),
BarAction::Finish => bar.finish(),
BarAction::Finish => bar.finish_and_clear(),
BarAction::SetLen(n) => bar.set_length(n),
BarAction::Print(s) => bar.println(s),
}
}
});
for base_path in paths.into_iter() {
let num_files = create_walkdir(&base_path).count().await;
tx.send(BarAction::SetLen(num_files as u64));
println!("Counted {num_files} files.");
tx.send(BarAction::SetLen(num_files as u64))?;
tx.send(BarAction::Print(format!("Counted {num_files} files.")))?;
let entries_stream = create_walkdir(&base_path);
let semaphore = Semaphore::new(10);
let base_path2 = base_path.clone();
let processed_stream = entries_stream
let _processed_stream = entries_stream
.map_err(|e| Error::from(e))
.and_then(|dir_ent| {
let tx = tx.clone();
@ -107,9 +110,9 @@ async fn main() -> Result<()> {
contents
};
handle_data(relative_path, data).await?;
handle_data(tx.clone(), relative_path, data).await?;
tx.send(BarAction::Inc);
tx.send(BarAction::Inc)?;
Ok::<_, Error>(false)
}
@ -118,24 +121,27 @@ async fn main() -> Result<()> {
.await;
}
tx.send(BarAction::Finish);
tx.send(BarAction::Finish)?;
Ok(())
}
async fn handle_data(path: impl AsRef<Path>, data: String) -> Result<()> {
async fn handle_data(
tx: Sender<BarAction>,
path: impl AsRef<Path>,
data: String,
) -> Result<()> {
let cm: Lrc<SourceMap> = Default::default();
let handler =
Handler::with_tty_emitter(ColorConfig::Auto, true, false, Some(cm.clone()));
let fm = cm.new_source_file(
FileName::Custom(path.as_ref().display().to_string()),
data,
);
let path_display = path.as_ref().display();
let syntax = Syntax::Es(EsConfig {
jsx: true,
let fm = cm.new_source_file(FileName::Custom(path_display.to_string()), data);
let syntax = Syntax::Typescript(TsConfig {
tsx: true,
..Default::default()
});
@ -162,35 +168,22 @@ async fn handle_data(path: impl AsRef<Path>, data: String) -> Result<()> {
})
.expect("failed to parser module");
// println!("Module: {module:?}");
tx.send(BarAction::Print(format!("Parsed module. {}", path_display)))?;
for item in module.body.iter() {
tx.send(BarAction::Print(format!("SHIET {item:?}")))?;
let decl = match item.as_module_decl() {
Some(v) => v,
None => continue,
};
match decl {
ModuleDecl::ExportDecl(decl) => {
tx.send(BarAction::Print(format!("SHIET {decl:?}")))?;
}
_ => {}
}
}
Ok(())
}
fn create_walkdir(path: impl AsRef<Path>) -> WalkDir {
WalkDir::new(path.as_ref()).filter(|entry| async move {
let path = entry.path();
if let Some(true) = path
.file_name()
.map(|f| f == "node_modules" || f.to_string_lossy().starts_with('.'))
{
return Filtering::IgnoreDir;
}
if path.is_file()
&& !path
.file_name()
.and_then(|f| f.to_str())
.map_or_else(|| false, |f| f.ends_with(".js"))
{
return Filtering::Ignore;
}
if path.is_dir() {
return Filtering::Ignore;
}
Filtering::Continue
})
}

6
src/progress_bar.rs Normal file
View file

@ -0,0 +1,6 @@
pub enum BarAction {
Inc,
Finish,
SetLen(u64),
Print(String),
}

37
src/walkdir.rs Normal file
View file

@ -0,0 +1,37 @@
use std::path::Path;
use async_walkdir::{WalkDir, Filtering};
use regex::Regex;
use lazy_static::lazy_static;
lazy_static! {
static ref MATCH_PATTERN: Regex = Regex::new(r".*\.[jt]sx?$").unwrap();
}
pub fn create_walkdir(path: impl AsRef<Path>) -> WalkDir {
WalkDir::new(path.as_ref()).filter(|entry| async move {
let path = entry.path();
if let Some(true) = path
.file_name()
.map(|f| f == "node_modules" || f.to_string_lossy().starts_with('.'))
{
return Filtering::IgnoreDir;
}
if path.is_file()
&& path
.file_name()
.and_then(|f| f.to_str())
.map_or_else(|| true, |f| !MATCH_PATTERN.is_match(f))
{
return Filtering::Ignore;
}
if path.is_dir() {
return Filtering::Ignore;
}
Filtering::Continue
})
}