michael/rget
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

initial

Browse source
This commit is contained in:
Michael Zhang 2023-09-18 03:30:20 -05:00
commit 12c2016003
11 changed files with 2646 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
.gitignore vendored Normal file
View file

@ -0,0 +1,2 @@
/target
node_modules

2121
Cargo.lock generated Normal file
View file

File diff suppressed because it is too large Load diff

28
Cargo.toml Normal file
View file

@ -0,0 +1,28 @@
[package]
name = "rget"
version = "0.1.0"
edition = "2021"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[dependencies]
anyhow = "1.0.75"
async-compression = { version = "0.4.3", features = ["gzip", "tokio"] }
async-tar = "0.4.2"
async-trait = "0.1.73"
bb8 = "0.8.1"
clap = { version = "4.4.3", features = ["derive"] }
deadqueue = "0.2.4"
flate2 = "1.0.27"
futures = "0.3.28"
hex-literal = "0.4.1"
hyper = { version = "0.14.27", features = ["full"] }
nom = "7.1.3"
reqwest = { version = "0.11.20", features = ["rustls-tls-webpki-roots", "json", "stream"] }
semver = "1.0.18"
serde = { version = "1.0.188", features = ["derive"] }
sha1 = "0.10.5"
simd-json = "0.10.7"
tempfile = "3.8.0"
tokio = { version = "1.32.0", features = ["full"] }
tokio-util = { version = "0.7.8", features = ["full"] }

2
Makefile Normal file
View file

@ -0,0 +1,2 @@
run:
cargo run -- install --dir examples/alotta-files

118
examples/alotta-files/package.json Normal file
View file

@ -0,0 +1,118 @@
{
"name": "alotta-modules",
"version": "0.0.1",
"dependencies": {
"animate.less": "^2.2.0",
"autoprefixer": "^10.4.8",
"babel-core": "^6.26.3",
"babel-eslint": "^10.1.0",
"babel-loader": "^8.2.5",
"babel-plugin-lodash": "^3.3.4",
"babel-plugin-module-resolver": "^4.1.0",
"babel-plugin-transform-decorators-legacy": "^1.3.5",
"babel-plugin-transform-runtime": "^6.23.0",
"babel-polyfill": "^6.26.0",
"babel-preset-es2015": "^6.24.1",
"babel-preset-react": "^6.24.1",
"babel-preset-react-hmre": "^1.1.1",
"babel-preset-stage-1": "^6.24.1",
"babel-runtime": "^6.26.0",
"clean-webpack-plugin": "^4.0.0",
"core-decorators": "^0.20.0",
"css-loader": "^6.7.1",
"css-mqpacker": "^7.0.0",
"cssnano": "^5.1.13",
"custom-event-polyfill": "^1.0.7",
"draft-js": "^0.11.7",
"ejs": "^3.1.8",
"eslint": "^8.23.0",
"eslint-config-airbnb": "^19.0.4",
"eslint-import-resolver-webpack": "^0.13.2",
"eslint-plugin-import": "^2.26.0",
"eslint-plugin-jsx-a11y": "^6.6.1",
"eslint-plugin-react": "^7.31.6",
"express": "^4.18.1",
"express-http-proxy": "^1.6.3",
"font-awesome": "^4.7.0",
"fready": "^1.0.0",
"glob": "^8.0.3",
"gulp": "^4.0.2",
"gulp-concat": "^2.6.1",
"gulp-csslint": "^1.0.1",
"gulp-cssnano": "^2.1.3",
"gulp-eol": "^0.2.0",
"gulp-less": "^5.0.0",
"gulp-livereload": "^4.0.2",
"gulp-minify-css": "^1.2.4",
"gulp-postcss": "^9.0.1",
"gulp-rename": "^2.0.0",
"gulp-util": "^3.0.8",
"happypack": "^5.0.1",
"highcharts": "^10.2.1",
"highcharts-solid-gauge": "^0.1.7",
"history": "^5.3.0",
"howler": "^2.2.3",
"imports-loader": "^4.0.1",
"jquery": "^3.6.1",
"jquery-ui": "1.13.2",
"js-cookie": "^3.0.1",
"json-loader": "^0.5.7",
"leftpad": "^0.0.1",
"less": "^4.1.3",
"lesshat": "^4.1.0",
"lodash": "^4.17.21",
"medium-draft": "^0.5.18",
"mobx": "^6.6.1",
"mobx-react": "^7.5.2",
"moment": "^2.29.4",
"moment-range": "^4.0.2",
"moment-timezone": "^0.5.37",
"password-policy": "0.0.3",
"postcss-reporter": "^7.0.5",
"progress": "^2.0.3",
"qs": "^6.11.0",
"raw-loader": "^4.0.2",
"rc-slider": "^10.0.1",
"react": "^18.2.0",
"react-addons-css-transition-group": "^15.6.2",
"react-addons-shallow-compare": "^15.6.3",
"react-dnd": "^16.0.1",
"react-dnd-html5-backend": "^16.0.1",
"react-dom": "^18.2.0",
"react-draft-wysiwyg": "^1.15.0",
"react-dropzone": "^14.2.2",
"react-grid-layout": "^1.3.4",
"react-highcharts": "^16.1.0",
"react-hot-loader": "4.13.0",
"react-input-calendar": "^0.5.4",
"react-lazyload": "^3.2.0",
"react-measure": "^2.5.2",
"react-mixin": "^5.0.0",
"react-responsive": "9.0.0-beta.10",
"react-responsive-tabs": "^4.4.3",
"react-router": "^6.3.0",
"react-router-dom": "^6.3.0",
"react-select-plus": "1.0.0-rc.3.patch12",
"react-skylight": "^0.5.1",
"react-sortablejs": "^6.1.4",
"react-tappable": "^1.0.4",
"react-tooltip": "4.2.21",
"react-virtualized": "^9.22.3",
"react-waypoint": "^10.3.0",
"sortablejs": "^1.15.0",
"style-loader": "^3.3.1",
"stylelint": "^14.11.0",
"superagent": "^8.0.0",
"uglify-js": "^3.17.0",
"uuid": "^8.3.2",
"verge": "^1.10.2",
"webpack-bundle-analyzer": "^4.6.1",
"webpack-hot-middleware": "^2.25.2",
"webpack-notifier": "^1.15.0",
"webpack-split-by-path": "^2.0.0",
"whatwg-fetch": "^3.6.2"
},
"devDependencies": {
"nan-as": "^1.6.1"
}
}

4
rustfmt.toml Normal file
View file

@ -0,0 +1,4 @@
max_width = 80
tab_spaces = 2
wrap_comments = true
fn_single_line = true

194
src/install_package.rs Normal file
View file

@ -0,0 +1,194 @@
use std::{path::PathBuf, sync::Arc};
use anyhow::{ensure, Context, Result};
use async_compression::tokio::bufread::GzipDecoder;
use async_tar::Archive;
use futures::TryStreamExt;
use reqwest::Client;
use semver::{Version, VersionReq};
use sha1::{Digest, Sha1};
use tempfile::NamedTempFile;
use tokio::{
fs::File,
io::{AsyncReadExt, AsyncWriteExt, BufReader},
};
use tokio_util::{compat::TokioAsyncReadCompatExt, io::StreamReader};
use crate::{
parse_package::{parse_package_json, Package, PackageMetadata},
queue_utils::add_reqs_to_queue,
TaskQueue,
};
#[derive(Debug)]
pub enum PackageDef {
// Look up from registry
Name(String, VersionReq),
// Local file
Local(PathBuf),
}
pub async fn install_package(
queue: Arc<TaskQueue>,
package_def: PackageDef,
client: Client,
rget_path: PathBuf,
) -> Result<()> {
match package_def {
PackageDef::Name(name, _version_req) => {
let uri = format!("https://registry.npmjs.org/{}", name);
let res = client.get(uri).send().await?;
let mut body = res.text().await?;
let res: PackageMetadata =
simd_json::from_slice(unsafe { body.as_bytes_mut() })?;
// println!("body {res:?}");
let matching_versions = res
.versions
.iter()
.map(|(version, object)| {
(Version::parse(version).expect("wtf"), object)
})
// .filter(|(version, _)| version_req.matches(&version))
.collect::<Vec<_>>();
// println!("Matching versions: {matching_versions:?}");
// TODO: Take the highest version lmao
let (used_version, version_object) = matching_versions.first().unwrap();
let version = used_version.to_string();
let base_path = rget_path.join(format!("{name}@{version}"));
let tarball_fut = download_tarball(
&client,
base_path,
&version_object.dist.tarball,
&version_object.dist.shasum,
);
let deps_fut = add_deps(&queue, &client, &name, used_version);
let (tarball_res, deps_res) = futures::join!(tarball_fut, deps_fut,);
tarball_res?;
deps_res?;
Ok::<_, anyhow::Error>(())
}
PackageDef::Local(path) => {
let package = parse_package_json(path)?;
println!("parsed :3");
add_reqs_to_queue(&queue, &package).await?;
println!("added :3");
Ok(())
}
}
}
async fn add_deps(
queue: &TaskQueue,
client: &Client,
name: &str,
used_version: &Version,
) -> Result<()> {
let uri = format!(
"https://registry.npmjs.org/{}/{}",
name,
used_version.to_string()
);
let res = client.get(uri).send().await?;
let mut body = res.text().await?;
let package: Package = simd_json::from_slice(unsafe { body.as_bytes_mut() })?;
add_reqs_to_queue(&queue, &package).await?;
Ok(())
}
async fn download_tarball(
client: &Client,
base_path: PathBuf,
url: &str,
sum: &str,
) -> Result<()> {
let dest = NamedTempFile::new()?;
let (file, path) = dest.keep()?;
let mut dest_file = File::from_std(file);
let res = client.get(url).send().await?;
let stream = res.bytes_stream();
let mut stream_reader = StreamReader::new(stream.map_err(convert_error));
// Hashing copy
let mut buf = [0u8; 4096];
let mut hasher = Sha1::new();
loop {
let bytes = stream_reader.read(&mut buf).await?;
if bytes == 0 {
break;
}
dest_file.write(&buf[..bytes]).await?;
hasher.update(&buf[..bytes]);
}
// tokio::io::copy(&mut stream_reader, &mut dest_file).await?;
let actual_hash = hasher.finalize();
let actual_hash_str = format!("{:x}", actual_hash);
ensure!(sum == actual_hash_str, "SHIT the hash is wrong.");
std::mem::drop(dest_file);
tokio::fs::remove_dir_all(&base_path).await;
tokio::fs::create_dir_all(&base_path).await?;
fn convert_error(err: reqwest::Error) -> std::io::Error {
eprintln!("WTF? {err}");
todo!()
}
// let stream = res.bytes_stream();
// let stream_reader = StreamReader::new(stream.map_err(convert_error));
let file = File::open(path).await?;
let file_reader = BufReader::new(file);
let decoder = GzipDecoder::new(file_reader);
let decoder = TokioAsyncReadCompatExt::compat(decoder);
let archive = Archive::new(decoder);
let temp_dir = tempfile::tempdir()?;
let temp_dir = temp_dir.into_path();
archive.unpack(&temp_dir).await?;
let package_dir = temp_dir.join("package");
tokio::fs::rename(package_dir, &base_path)
.await
.context("could not rename")?;
base_path.read_dir()?.try_for_each(|x| {
let y = x?;
// println!("wtf? {}", y.path().display());
Ok::<_, anyhow::Error>(())
})?;
// println!("Unpacked {}", base_path.display());
// let files = archive.entries()?;
// let files = files.try_collect::<Vec<_>>().await?;
// for mut file in files {
// let tar_path = file.path()?;
// let without_prefix = match tar_path.strip_prefix("package/") {
// Ok(v) => v,
// Err(_) => continue,
// };
// let mut new_path = base_path.clone();
// new_path.extend(without_prefix);
// let parent = new_path.parent().unwrap();
// tokio::fs::create_dir_all(parent).await?;
// file.unpack(&new_path).await?;
// println!("- Unpacked {}", new_path.display());
// // println!("new path: {new_path:?}");
// }
// println!("Installed to {}", base_path.display());
// // println!("files: {:?}", files);
// // let mut content = Cursor::new(res.bytes().await?);
// // println!("dest: {dest:?}");
// // io::copy(&mut content, &mut dest).await?;
// // dest.seek(SeekFrom::Start(0)).await?;
Ok(())
}

99
src/main.rs Normal file
View file

@ -0,0 +1,99 @@
#[macro_use]
extern crate async_trait;
#[macro_use]
extern crate hex_literal;
#[macro_use]
extern crate serde;
mod install_package;
mod parse_package;
mod queue_utils;
mod semver;
use std::{path::PathBuf, sync::Arc, time::Duration};
use anyhow::Result;
use clap::{Parser, Subcommand};
use reqwest::Client;
use tokio::time::sleep;
use crate::install_package::{install_package, PackageDef};
pub type TaskQueue = deadqueue::limited::Queue<PackageDef>;
#[derive(Debug, Parser)]
struct Opt {
#[clap(subcommand)]
command: Command,
}
#[derive(Debug, Subcommand)]
enum Command {
Install {
#[clap(long = "dir")]
dir: PathBuf,
},
}
#[tokio::main]
async fn main() -> Result<()> {
let opt = Opt::parse();
println!("Hello, world! {opt:?}");
let client = Client::builder()
.use_rustls_tls()
.pool_idle_timeout(Duration::from_secs(30))
.build()?;
match opt.command {
Command::Install { dir } => {
// Parse package.json
let package_json_path = dir.join("package.json");
let node_modules_path = dir.join("node_modules");
let rget_path = node_modules_path.join(".rget");
tokio::fs::create_dir_all(&rget_path).await?;
// let parsed = parse_package_json(package_json_path)?;
let queue = Arc::new(TaskQueue::new(1_000));
// Handle the first one specifically
let task = PackageDef::Local(package_json_path);
// queue.push(task).await;
install_package(queue.clone(), task, client.clone(), rget_path.clone())
.await?;
for _worker in 0..10 {
let queue = queue.clone();
let client = client.clone();
let rget_path = rget_path.clone();
tokio::spawn(async move {
loop {
let queue = queue.clone();
let client = client.clone();
let task = queue.pop().await;
// println!("Handling task {task:?}");
match install_package(queue, task, client, rget_path.clone()).await
{
Ok(_) => {}
Err(err) => {
eprintln!("WTF? {err} {err:?}");
}
}
// println!("worker[{}] processing task[{}] ...", worker, task);
}
});
}
while queue.len() > 0 {
// println!("Waiting for workers to finish...");
sleep(Duration::from_millis(1_000)).await;
}
println!("Done!");
}
}
Ok(())
}

44
src/parse_package.rs Normal file
View file

@ -0,0 +1,44 @@
// https://github.com/npm/registry/blob/master/docs/REGISTRY-API.md
use std::{collections::HashMap, fs::File, path::Path};
use anyhow::Result;
// TODO: Optimize using owning ref?
#[derive(Debug, Deserialize)]
pub struct Package {
pub name: String,
pub version: String,
#[serde(default)]
pub dependencies: HashMap<String, String>,
#[serde(rename = "devDependencies", default)]
pub dev_dependencies: HashMap<String, String>,
}
#[derive(Debug, Deserialize)]
pub struct PackageMetadata {
pub name: String,
pub versions: HashMap<String, VersionObject>,
}
#[derive(Debug, Deserialize)]
pub struct VersionObject {
pub dist: Dist,
}
#[derive(Debug, Deserialize)]
pub struct Dist {
pub shasum: String,
pub tarball: String,
}
pub fn parse_package_json(path: impl AsRef<Path>) -> Result<Package> {
let parsed: Package = {
let file = File::open(path)?;
simd_json::from_reader(file)?
};
Ok(parsed)
}

18
src/queue_utils.rs Normal file
View file

@ -0,0 +1,18 @@
use anyhow::{Context, Result};
use semver::VersionReq;
use crate::{install_package::PackageDef, parse_package::Package, TaskQueue};
pub async fn add_reqs_to_queue(
queue: &TaskQueue,
package: &Package,
) -> Result<()> {
for (package_name, version_req) in package.dependencies.iter() {
let version_req = VersionReq::parse(version_req)
.with_context(|| format!("could not parse version {version_req:?}"))?;
queue
.push(PackageDef::Name(package_name.to_owned(), version_req))
.await;
}
Ok(())
}

16
src/semver.rs Normal file
View file

@ -0,0 +1,16 @@
// npm has a kinda fucked version of semver?
// https://github.com/npm/node-semver
use nom::{
branch::alt, bytes::complete::take_while1, character::is_alphanumeric,
};
pub fn parse_semver(wtf: &str) {}
fn is_part(c: u8) -> bool {
c == b'-' || is_alphanumeric(c)
}
// fn part() {
// alt((nr, take_while1(is_part)))
// }