From 7964bb84fd7a6801f356fdfbc730b630a5038ecc Mon Sep 17 00:00:00 2001 From: Didier Date: Thu, 13 Jul 2023 00:11:44 +0200 Subject: [PATCH 001/123] init: initial commit of packager --- .gitignore | 2 + .idea/.gitignore | 8 +++ .idea/discord.xml | 7 +++ .idea/modules.xml | 8 +++ .idea/pkgr.iml | 17 +++++++ .idea/vcs.xml | 6 +++ README.md | 8 +++ bootpkg/Cargo.toml | 12 +++++ bootpkg/src/args.rs | 29 +++++++++++ bootpkg/src/main.rs | 51 +++++++++++++++++++ bootpkg/src/prelude.rs | 22 ++++++++ build-pkg.toml | 14 +++++ manifest/Cargo.toml | 10 ++++ manifest/src/bin.rs | 7 +++ manifest/src/build.rs | 10 ++++ manifest/src/fs.rs | 16 ++++++ manifest/src/lib.rs | 32 ++++++++++++ manifest/src/package.rs | 45 ++++++++++++++++ manifest/src/pkgr.rs | 4 ++ package.toml | 110 ++++++++++++++++++++++++++++++++++++++++ pkgfile/Cargo.toml | 10 ++++ pkgfile/pkgfile.txt | 3 ++ pkgfile/src/lib.rs | 36 +++++++++++++ pkgr/Cargo.toml | 9 ++++ pkgr/src/main.rs | 2 + 25 files changed, 478 insertions(+) create mode 100644 .gitignore create mode 100644 .idea/.gitignore create mode 100644 .idea/discord.xml create mode 100644 .idea/modules.xml create mode 100644 .idea/pkgr.iml create mode 100644 .idea/vcs.xml create mode 100644 README.md create mode 100644 bootpkg/Cargo.toml create mode 100644 bootpkg/src/args.rs create mode 100644 bootpkg/src/main.rs create mode 100644 bootpkg/src/prelude.rs create mode 100644 build-pkg.toml create mode 100644 manifest/Cargo.toml create mode 100644 manifest/src/bin.rs create mode 100644 manifest/src/build.rs create mode 100644 manifest/src/fs.rs create mode 100644 manifest/src/lib.rs create mode 100644 manifest/src/package.rs create mode 100644 manifest/src/pkgr.rs create mode 100644 package.toml create mode 100644 pkgfile/Cargo.toml create mode 100644 pkgfile/pkgfile.txt create mode 100644 pkgfile/src/lib.rs create mode 100644 pkgr/Cargo.toml create mode 100644 pkgr/src/main.rs diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..972b0c4 --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +**/target +**/Cargo.lock diff --git a/.idea/.gitignore b/.idea/.gitignore new file mode 100644 index 0000000..13566b8 --- /dev/null +++ b/.idea/.gitignore @@ -0,0 +1,8 @@ +# Default ignored files +/shelf/ +/workspace.xml +# Editor-based HTTP Client requests +/httpRequests/ +# Datasource local storage ignored files +/dataSources/ +/dataSources.local.xml diff --git a/.idea/discord.xml b/.idea/discord.xml new file mode 100644 index 0000000..d8e9561 --- /dev/null +++ b/.idea/discord.xml @@ -0,0 +1,7 @@ + + + + + \ No newline at end of file diff --git a/.idea/modules.xml b/.idea/modules.xml new file mode 100644 index 0000000..9fe00b2 --- /dev/null +++ b/.idea/modules.xml @@ -0,0 +1,8 @@ + + + + + + + + \ No newline at end of file diff --git a/.idea/pkgr.iml b/.idea/pkgr.iml new file mode 100644 index 0000000..c67ef27 --- /dev/null +++ b/.idea/pkgr.iml @@ -0,0 +1,17 @@ + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/vcs.xml b/.idea/vcs.xml new file mode 100644 index 0000000..35eb1dd --- /dev/null +++ b/.idea/vcs.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..7e24755 --- /dev/null +++ b/README.md @@ -0,0 +1,8 @@ +# Packager + +> "A package manager and builder but like rust." +> -- Gandhi (2050) + +> ***not even close to done*** :) + +Packager is a simple yet powerful package manager \ No newline at end of file diff --git a/bootpkg/Cargo.toml b/bootpkg/Cargo.toml new file mode 100644 index 0000000..c48277f --- /dev/null +++ b/bootpkg/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "bootpkg" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +manifest = { path = "../manifest" } +pkgfile = { path = "../pkgfile" } +regex = "1.9.1" +reqwest = { version = "0.11.18", features = ["blocking"] } diff --git a/bootpkg/src/args.rs b/bootpkg/src/args.rs new file mode 100644 index 0000000..2f51141 --- /dev/null +++ b/bootpkg/src/args.rs @@ -0,0 +1,29 @@ +pub enum Command { + Strap, + Unpack, + None +} + +impl From for Command { + fn from(value: String) -> Self { + match value.to_lowercase().as_str() { + "strap" => Command::Strap, + "unpack" => Command::Unpack, + _ => Command::None + } + } +} + +pub struct Args { + pub command: Command, + pub args: Vec, +} + +impl From> for Args { + fn from(value: Vec) -> Self { + Args { + command: Command::from(value[0].to_owned()), + args: value[1..].to_owned() + } + } +} \ No newline at end of file diff --git a/bootpkg/src/main.rs b/bootpkg/src/main.rs new file mode 100644 index 0000000..8339041 --- /dev/null +++ b/bootpkg/src/main.rs @@ -0,0 +1,51 @@ +use std::env; +use std::process::exit; +use pkgfile::PKGFile; +use crate::args::{Args, Command}; + +mod prelude; +mod args; + +fn main() { + let args = Args::from(env::args().collect::>()[1..].to_owned()); + match args.command { + Command::Strap => { + if args.args.len() == 0 { + println!("no path/uri"); + exit(0); + } + let uri = &args.args[0]; + println!("!! Got package: \"{}\"", uri); + let re = r"[A-z]+://.+"; + let data = if regex::Regex::new(re).unwrap().is_match(uri) { + println!("** detected uri."); + reqwest::blocking::get(uri) + .expect("Could not request file.") + .bytes() + .expect("Could not turn file into bytes.") + .to_vec() + } else { + if let Ok(d) = std::fs::read(uri) { + d + } else { + println!("Could not read file!"); + exit(1); + } + }; + + let p = match PKGFile::try_from(data) { + Ok(p) => p, + Err(_) => { + println!("!! Could not interpret PKGFile..."); + exit(1); + } + }; + + }, + Command::Unpack => todo!(), + _ => { + println!("Unsupported command, allowed commands: strap/unpack."); + exit(1); + } + } +} diff --git a/bootpkg/src/prelude.rs b/bootpkg/src/prelude.rs new file mode 100644 index 0000000..4462619 --- /dev/null +++ b/bootpkg/src/prelude.rs @@ -0,0 +1,22 @@ +use std::collections::HashMap; +use manifest::Manifest; + +pub struct PKGR { + pub bootstrap: Option, +} + +pub struct Bootstrap {} + +pub fn def_manifest() { + Manifest::> { + package: manifest::package::Package::default(), + bin: None, + build: None, + dependencies: HashMap::default(), + fs: manifest::fs::FS { + config: None, + data: None, + }, + pkgr: Some(PKGR { bootstrap: None }), + }; +} \ No newline at end of file diff --git a/build-pkg.toml b/build-pkg.toml new file mode 100644 index 0000000..8c994d9 --- /dev/null +++ b/build-pkg.toml @@ -0,0 +1,14 @@ +[project] +package_file = "./package.toml" +package_dir = "./pkg" +increment_version = false + +[mappings.source] +ignore = ["target/"] + +[mappings.source.map] +"." = "/src" + +[mappings.bin.map] +"target/release/bin/pkgr" = "/bin/pkgr" + diff --git a/manifest/Cargo.toml b/manifest/Cargo.toml new file mode 100644 index 0000000..139b38f --- /dev/null +++ b/manifest/Cargo.toml @@ -0,0 +1,10 @@ +[package] +name = "manifest" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +toml = "0.7.6" +serde = { version = "1.0.171", features = ["derive"] } \ No newline at end of file diff --git a/manifest/src/bin.rs b/manifest/src/bin.rs new file mode 100644 index 0000000..10d1ea1 --- /dev/null +++ b/manifest/src/bin.rs @@ -0,0 +1,7 @@ +use serde::{Serialize, Deserialize}; + +#[derive(Serialize, Deserialize)] +pub struct Bin { + pub root: String, + +} \ No newline at end of file diff --git a/manifest/src/build.rs b/manifest/src/build.rs new file mode 100644 index 0000000..0ad59ec --- /dev/null +++ b/manifest/src/build.rs @@ -0,0 +1,10 @@ +use std::collections::HashMap; + +use serde::{Serialize, Deserialize}; + +#[derive(Serialize, Deserialize)] +pub struct Build { + build_script: String, + install_script: String, + dependencies: HashMap +} \ No newline at end of file diff --git a/manifest/src/fs.rs b/manifest/src/fs.rs new file mode 100644 index 0000000..e02312f --- /dev/null +++ b/manifest/src/fs.rs @@ -0,0 +1,16 @@ +use serde::{Deserialize, Serialize}; + +#[derive(Serialize, Deserialize, Debug)] +pub struct FS { + pub config: Option, + pub data: Option, +} + +impl Default for FS { + fn default() -> Self { + FS { + config: None, + data: None, + } + } +} diff --git a/manifest/src/lib.rs b/manifest/src/lib.rs new file mode 100644 index 0000000..fdc8cbc --- /dev/null +++ b/manifest/src/lib.rs @@ -0,0 +1,32 @@ +use serde::{Deserialize, Serialize}; +use std::collections::HashMap; + +pub mod package; +pub mod fs; +pub mod bin; +pub mod build; +pub mod pkgr; + +#[derive(Serialize, Deserialize)] +#[serde(default)] +pub struct Manifest

> { + pub package: package::Package, + pub dependencies: HashMap, + pub fs: fs::FS, + pub bin: Option, + pub build: Option, + pub pkgr: P +} + +impl Default for Manifest { + fn default() -> Self { + Manifest { + package: package::Package::default(), + dependencies: HashMap::default(), + fs: fs::FS::default(), + bin: None, + build: None, + pkgr: None + } + } +} diff --git a/manifest/src/package.rs b/manifest/src/package.rs new file mode 100644 index 0000000..3fa1626 --- /dev/null +++ b/manifest/src/package.rs @@ -0,0 +1,45 @@ +use std::str::FromStr; + +use serde::{Serialize, Deserialize}; + +#[derive(Serialize, Deserialize, Debug)] +pub enum PackageType { + #[serde(rename = "application")] + Application, + #[serde(rename = "library")] + Library, + #[serde(rename = "meta")] + Meta +} + +#[derive(Serialize, Deserialize, Debug)] +#[serde(default)] +pub struct Package { + pub name: String, + pub description: String, + pub package_type: PackageType, + pub version: u64, + pub tags: Vec, + pub arch: String +} + +impl Default for Package { + fn default() -> Self { + Package { + name: String::default(), + description: String::default(), + package_type: PackageType::Application, + version: 0, + tags: Vec::default(), + arch: std::env::consts::ARCH.to_string() + } + } +} + +impl FromStr for Package { + type Err = toml::de::Error; + + fn from_str(s: &str) -> Result { + toml::from_str(s) + } +} \ No newline at end of file diff --git a/manifest/src/pkgr.rs b/manifest/src/pkgr.rs new file mode 100644 index 0000000..2ca5fd4 --- /dev/null +++ b/manifest/src/pkgr.rs @@ -0,0 +1,4 @@ +use serde::{Serialize, Deserialize}; + +#[derive(Serialize, Deserialize)] +pub struct PKGR {} \ No newline at end of file diff --git a/package.toml b/package.toml new file mode 100644 index 0000000..fbc4070 --- /dev/null +++ b/package.toml @@ -0,0 +1,110 @@ +#* = required. + +[package] +name = "packager" #* +description = "A package installation tool" #* + +## package.version +# needs to be an int and be incremented every time +# if you use a custom version system and want to use that, you can use the tag system! +# the highest package.version on the server will be given the tag "latest" +version = 1 # this can automatically be incremented when publishing by running `pkgr publish -i ...` + +## package.tags +# you can add tags to a package to specify kinds +# there are some special tags: +# - latest +# automatically set on the last published version +# previous version will lose this tag +# - oldest +# assigned to the first version. +tags = [ + "prod", + "pkgr-spec-1" +] + +## package.type +# Supported types: +# - application +# this type specifies that this package is an application. +# - library +# this type specifies that this package is a library. +# - meta +# this type specifies that this package does not install anything upon the system except for dependencies. +type = "application" + +arch = "x86_64" # this is automatically filled by `pkgr publish ...` + +## dependencies +# you may use the following syntax +# "" or ",,..." +[dependencies] +example = { arch = "x86_64", tags = [ "stable" ], version = "v1" } +exnmbr2 = "v1,stable" + +## fs +# specify some directories for ease of use +[fs] +## fs.config +# specifiy the config path +config = "/etc/packager" +## fs.data +# specify the data path +data = "/var/lib/packager" +## replace "pacakger" with your package name to find the default value. + +## bin +# Used for systems that don't want to build pkgs. +[bin] # binary files root + +## bin.root +# ** RELATIVE TO PACKAGE ROOT ** +# bin.root specifies the root of the installed tree. +# anything in here will be overlayed on top of the system. +root = "/root" #* + +## bin.checksums +# ** KEYS are relative to BIN.ROOT ** +# ** VALUES are relative to PKG ROOT ** +# checksums is used to perform checksums before install +# values may be paths or a uri. You may include variables. +# supported variables: +# - @name +# - @version +# - @display_version +[bin.checksums] +"/usr/bin/pkgr" = "https://ixvd.net/checksums/packager/@version" + +## build +# Scripts will be copied to a custom root. +# After the pacakge is built, the install script is ran. +# After the install script the build directory will be deleted and an CTREE (changed tree) will be generated. +# Then the CTREE will be used to copy the files over. +[build] +build_script = "/scripts/build" # relative to pkg +install_script = "/scripts/install" # relative to pkg + +[build.dependencies] +base = "latest,stable" # selected by default + +## pkgr.* +# packager is the official client but you may use other clients supporting the "pkgr v1 spec". +# other clients may offer extra functionality this must be put under "pkgr.*" +[pkgr] + +## pkgr.bootstrap +# This section is used for bootpkg. An edition of packager that bootstraps the full version. +# This exists so that packager is easy to install on anything! +# and only 1 release channel for pkgr +[pkgr.bootstrap] +# any non-zero = installed +check_installed_commands = [ + "/scripts/check_installed" +] + +# any non-zero = fail +commands = [ + "/scripts/bootstrap/download_latest @version /tmp/pkgr.pkg", + "/scripts/bootstrap/dirty_install /tmp/pkgr.pkg", + "/scripts/bootstrap/check_install" +] diff --git a/pkgfile/Cargo.toml b/pkgfile/Cargo.toml new file mode 100644 index 0000000..5fb57fe --- /dev/null +++ b/pkgfile/Cargo.toml @@ -0,0 +1,10 @@ +[package] +name = "pkgfile" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +manifest = { path = "../manifest" } +tar = "0.4.38" diff --git a/pkgfile/pkgfile.txt b/pkgfile/pkgfile.txt new file mode 100644 index 0000000..1592aaa --- /dev/null +++ b/pkgfile/pkgfile.txt @@ -0,0 +1,3 @@ +[pkgfile version (1 byte)][size manifest x256 (1 byte)][size manifest x1 (1 byte)] +[manifest] +[tar file] \ No newline at end of file diff --git a/pkgfile/src/lib.rs b/pkgfile/src/lib.rs new file mode 100644 index 0000000..a6b1d5e --- /dev/null +++ b/pkgfile/src/lib.rs @@ -0,0 +1,36 @@ +pub struct PKGFile { + pub manifest: String, + pub data: Vec, +} + +impl TryFrom> for PKGFile { + type Error = (); + + fn try_from(value: Vec) -> Result { + match value[0] { + 1 => { + let header: Vec = value[..3] + .iter() + .map(|v| u32::from(*v)) + .collect(); + let manifest_size: u32 = ((header[1] << 8) | header[2]); + if manifest_size > value.len() as u32 { + return Err(()); + } + Ok(PKGFile { + manifest: match String::from_utf8( + value[ + 3..(manifest_size as usize) + ].to_vec() + ) + { + Ok(s) => s, + _ => return Err(()) + }, + data: value[(manifest_size as usize)..].to_vec(), + }) + } + _ => Err(()) + } + } +} diff --git a/pkgr/Cargo.toml b/pkgr/Cargo.toml new file mode 100644 index 0000000..df095d2 --- /dev/null +++ b/pkgr/Cargo.toml @@ -0,0 +1,9 @@ +[package] +name = "pkgr" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +manifest = { path = "../manifest" } \ No newline at end of file diff --git a/pkgr/src/main.rs b/pkgr/src/main.rs new file mode 100644 index 0000000..f79c691 --- /dev/null +++ b/pkgr/src/main.rs @@ -0,0 +1,2 @@ +fn main() { +} From 68bb6822ed119157b46f5ff906ad3c5f250dd1c2 Mon Sep 17 00:00:00 2001 From: Didier Date: Thu, 13 Jul 2023 16:50:49 +0200 Subject: [PATCH 002/123] feat: fixed file management --- .idea/pkgr.iml | 6 ++++ bootpkg/Cargo.toml | 3 ++ bootpkg/src/args.rs | 9 +++--- bootpkg/src/main.rs | 44 ++++++++++++++++++++++----- bootpkg/src/prelude.rs | 41 ++++++++++++++++--------- docs/pkgr/flow.md | 3 ++ manifest/src/bin.rs | 3 +- manifest/src/lib.rs | 18 ++++++++++- mock-pkg.py | 22 ++++++++++++++ package.toml | 19 +++++------- pkg/scripts/bootstrap/download_latest | 7 +++++ pkgfile/src/lib.rs | 16 ++++++++-- 12 files changed, 148 insertions(+), 43 deletions(-) create mode 100644 docs/pkgr/flow.md create mode 100644 mock-pkg.py create mode 100644 pkg/scripts/bootstrap/download_latest diff --git a/.idea/pkgr.iml b/.idea/pkgr.iml index c67ef27..c93234c 100644 --- a/.idea/pkgr.iml +++ b/.idea/pkgr.iml @@ -1,5 +1,10 @@ + + + + + @@ -13,5 +18,6 @@ + \ No newline at end of file diff --git a/bootpkg/Cargo.toml b/bootpkg/Cargo.toml index c48277f..8292150 100644 --- a/bootpkg/Cargo.toml +++ b/bootpkg/Cargo.toml @@ -8,5 +8,8 @@ edition = "2021" [dependencies] manifest = { path = "../manifest" } pkgfile = { path = "../pkgfile" } +toml = { version = "0.7.6", features = [ "parse" ] } +serde = { version = "1.0.171", features = ["derive"] } regex = "1.9.1" reqwest = { version = "0.11.18", features = ["blocking"] } +uuid = { version = "1.4.0", features = ["serde", "v4"] } diff --git a/bootpkg/src/args.rs b/bootpkg/src/args.rs index 2f51141..cf061e3 100644 --- a/bootpkg/src/args.rs +++ b/bootpkg/src/args.rs @@ -1,7 +1,7 @@ pub enum Command { Strap, Unpack, - None + None, } impl From for Command { @@ -9,11 +9,10 @@ impl From for Command { match value.to_lowercase().as_str() { "strap" => Command::Strap, "unpack" => Command::Unpack, - _ => Command::None + _ => Command::None, } } } - pub struct Args { pub command: Command, pub args: Vec, @@ -23,7 +22,7 @@ impl From> for Args { fn from(value: Vec) -> Self { Args { command: Command::from(value[0].to_owned()), - args: value[1..].to_owned() + args: value[1..].to_owned(), } } -} \ No newline at end of file +} diff --git a/bootpkg/src/main.rs b/bootpkg/src/main.rs index 8339041..f59cf01 100644 --- a/bootpkg/src/main.rs +++ b/bootpkg/src/main.rs @@ -1,10 +1,15 @@ -use std::env; -use std::process::exit; -use pkgfile::PKGFile; use crate::args::{Args, Command}; +use pkgfile::PKGFile; +use std::env; +use std::env::temp_dir; +use std::fs::{create_dir, create_dir_all, File}; +use std::io::Write; +use std::path::PathBuf; +use std::process::exit; +use uuid::Uuid; -mod prelude; mod args; +mod prelude; fn main() { let args = Args::from(env::args().collect::>()[1..].to_owned()); @@ -33,7 +38,7 @@ fn main() { } }; - let p = match PKGFile::try_from(data) { + let pkg = match PKGFile::try_from(data) { Ok(p) => p, Err(_) => { println!("!! Could not interpret PKGFile..."); @@ -41,8 +46,33 @@ fn main() { } }; - }, - Command::Unpack => todo!(), + let mani = prelude::mani_from_str(&pkg.manifest); + + println!("!! Package: {}", &mani.package.name); + + if !mani.valid() { + println!("!!! Manifest is not valid."); + exit(1); + } + + let mut tmp_dir = temp_dir(); + tmp_dir.push(format!("{}", Uuid::new_v4())); + create_dir_all(&tmp_dir) + .expect("Could not create tmp dir."); + + { + let mut file = File::create(format!("{}/pkgtar", tmp_dir.to_str().unwrap())) + .expect("Could not create tmp pkgtar file"); + + file.write(&pkg.data) + .expect("Could not write pkgtar to tmp file"); + } + + // TODO: untar + // TODO: cd into pkg + // TODO: run bootstrap commands + } + Command::Unpack => {} _ => { println!("Unsupported command, allowed commands: strap/unpack."); exit(1); diff --git a/bootpkg/src/prelude.rs b/bootpkg/src/prelude.rs index 4462619..b62eda1 100644 --- a/bootpkg/src/prelude.rs +++ b/bootpkg/src/prelude.rs @@ -1,22 +1,33 @@ -use std::collections::HashMap; -use manifest::Manifest; +use manifest::{self, Manifest}; +use serde::{Deserialize, Serialize}; +use std::{collections::HashMap, str::FromStr}; +use toml; +#[derive(Serialize, Deserialize)] pub struct PKGR { pub bootstrap: Option, } -pub struct Bootstrap {} +#[derive(Serialize, Deserialize)] +pub struct Bootstrap { + check_installed_commands: Vec, + commands: Vec +} -pub fn def_manifest() { +#[derive(Serialize, Deserialize)] +pub struct BadManifest { + pkgr: Option, +} + +pub fn mani_from_str(s: &str) -> Manifest> { + let mani = Manifest::from_str(s).unwrap(); + let bmani = toml::from_str::(s).unwrap(); Manifest::> { - package: manifest::package::Package::default(), - bin: None, - build: None, - dependencies: HashMap::default(), - fs: manifest::fs::FS { - config: None, - data: None, - }, - pkgr: Some(PKGR { bootstrap: None }), - }; -} \ No newline at end of file + package: mani.package, + dependencies: mani.dependencies, + fs: mani.fs, + bin: mani.bin, + build: mani.build, + pkgr: bmani.pkgr + } +} diff --git a/docs/pkgr/flow.md b/docs/pkgr/flow.md new file mode 100644 index 0000000..d6a9a40 --- /dev/null +++ b/docs/pkgr/flow.md @@ -0,0 +1,3 @@ +# Install flow + +pkgr keeps track of installed files and therefore we have a CTREE. diff --git a/manifest/src/bin.rs b/manifest/src/bin.rs index 10d1ea1..e8b48a3 100644 --- a/manifest/src/bin.rs +++ b/manifest/src/bin.rs @@ -1,7 +1,8 @@ +use std::collections::HashMap; use serde::{Serialize, Deserialize}; #[derive(Serialize, Deserialize)] pub struct Bin { pub root: String, - + pub checksums: HashMap } \ No newline at end of file diff --git a/manifest/src/lib.rs b/manifest/src/lib.rs index fdc8cbc..71efe31 100644 --- a/manifest/src/lib.rs +++ b/manifest/src/lib.rs @@ -1,5 +1,5 @@ use serde::{Deserialize, Serialize}; -use std::collections::HashMap; +use std::{collections::HashMap, str::FromStr}; pub mod package; pub mod fs; @@ -18,6 +18,22 @@ pub struct Manifest

> { pub pkgr: P } +impl Manifest { + pub fn valid(&self) -> bool { + if self.bin.is_none() && self.build.is_none() { + return false + } + true + } +} + +impl FromStr for Manifest { + type Err = toml::de::Error; + fn from_str(s: &str) -> Result { + toml::from_str(s) + } +} + impl Default for Manifest { fn default() -> Self { Manifest { diff --git a/mock-pkg.py b/mock-pkg.py new file mode 100644 index 0000000..57ee008 --- /dev/null +++ b/mock-pkg.py @@ -0,0 +1,22 @@ +import tarfile, os + +with open("./package.toml", mode='r') as mani: + data = mani.read() + with open("./mock.pkg", mode='wb') as pkg: + print("building header...") + pkg.write(bytes([0x01, (len(data) >> 8) & 0xFF, len(data) & 0xFF])) + print("writing manifest into pkg...") + pkg.write(data.encode("utf-8")) + with tarfile.TarFile("/tmp/pkgtar", 'w') as pkgtar: + for root, dirs, files in os.walk("."): + for file in files: + print(f"\33[2Kadd: {file}", end="\r", flush=True) + pkgtar.add(os.path.join(root, file)) + with open("/tmp/pkgtar", 'rb') as pkgtar: + print("appending pkgtar to pkg...") + pkg.write(pkgtar.read()) + print("deleting /tmp/pkgtar...") + os.unlink("/tmp/pkgtar") + print("closing write stream") + pkg.close() + \ No newline at end of file diff --git a/package.toml b/package.toml index fbc4070..54d55b5 100644 --- a/package.toml +++ b/package.toml @@ -18,10 +18,7 @@ version = 1 # this can automatically be incremented when publishing by running ` # previous version will lose this tag # - oldest # assigned to the first version. -tags = [ - "prod", - "pkgr-spec-1" -] +tags = [ "prod", "pkgr-spec-1" ] ## package.type # Supported types: @@ -39,7 +36,7 @@ arch = "x86_64" # this is automatically filled by `pkgr publish ...` # you may use the following syntax # "" or ",,..." [dependencies] -example = { arch = "x86_64", tags = [ "stable" ], version = "v1" } +#example = { arch = "x86_64", tags = [ "stable" ], version = "v1" } exnmbr2 = "v1,stable" ## fs @@ -97,14 +94,12 @@ base = "latest,stable" # selected by default # This exists so that packager is easy to install on anything! # and only 1 release channel for pkgr [pkgr.bootstrap] -# any non-zero = installed -check_installed_commands = [ - "/scripts/check_installed" -] +## any non-zero = installed +check_installed_commands = [ "/scripts/check_installed" ] # any non-zero = fail commands = [ - "/scripts/bootstrap/download_latest @version /tmp/pkgr.pkg", - "/scripts/bootstrap/dirty_install /tmp/pkgr.pkg", - "/scripts/bootstrap/check_install" + "/scripts/bootstrap/download_latest @version /tmp/pkgr.pkg", + "/scripts/bootstrap/dirty_install /tmp/pkgr.pkg", + "/scripts/bootstrap/check_install" ] diff --git a/pkg/scripts/bootstrap/download_latest b/pkg/scripts/bootstrap/download_latest new file mode 100644 index 0000000..229aaa2 --- /dev/null +++ b/pkg/scripts/bootstrap/download_latest @@ -0,0 +1,7 @@ +#!/bin/sh + +fetch_latest_version() { +} + +download_file() { +} \ No newline at end of file diff --git a/pkgfile/src/lib.rs b/pkgfile/src/lib.rs index a6b1d5e..6025cd5 100644 --- a/pkgfile/src/lib.rs +++ b/pkgfile/src/lib.rs @@ -20,17 +20,29 @@ impl TryFrom> for PKGFile { Ok(PKGFile { manifest: match String::from_utf8( value[ - 3..(manifest_size as usize) + 3..(manifest_size as usize + 3) ].to_vec() ) { Ok(s) => s, _ => return Err(()) }, - data: value[(manifest_size as usize)..].to_vec(), + data: value[(manifest_size as usize + 3)..].to_vec(), }) } _ => Err(()) } } } + +impl Into> for PKGFile { + fn into(self) -> Vec { + let mut bytes = vec![0x01]; + let manifest_bytes = self.manifest.into_bytes(); + bytes.push((manifest_bytes.len() >> 8) as u8); + bytes.push((manifest_bytes.len() | 0xFF) as u8); + bytes.extend(manifest_bytes); + bytes.extend(self.data); + bytes + } +} \ No newline at end of file From 618203279920d4fc963918aec283c3318c645b5a Mon Sep 17 00:00:00 2001 From: Didier Date: Thu, 13 Jul 2023 17:25:51 +0200 Subject: [PATCH 003/123] feat: file management and tempdir management --- bootpkg/src/main.rs | 25 +++++++++++++++++++------ bootpkg/src/prelude.rs | 6 ++++-- manifest/src/bin.rs | 3 ++- manifest/src/build.rs | 2 +- manifest/src/lib.rs | 3 ++- manifest/src/package.rs | 2 +- manifest/src/pkgr.rs | 2 +- mock-pkg.py | 3 ++- 8 files changed, 32 insertions(+), 14 deletions(-) diff --git a/bootpkg/src/main.rs b/bootpkg/src/main.rs index f59cf01..8853c9b 100644 --- a/bootpkg/src/main.rs +++ b/bootpkg/src/main.rs @@ -1,13 +1,15 @@ -use crate::args::{Args, Command}; -use pkgfile::PKGFile; use std::env; use std::env::temp_dir; -use std::fs::{create_dir, create_dir_all, File}; +use std::fs::{create_dir_all, File, remove_dir_all}; use std::io::Write; -use std::path::PathBuf; use std::process::exit; + use uuid::Uuid; +use pkgfile::PKGFile; + +use crate::args::{Args, Command}; + mod args; mod prelude; @@ -30,6 +32,7 @@ fn main() { .expect("Could not turn file into bytes.") .to_vec() } else { + println!("** reading data..."); if let Ok(d) = std::fs::read(uri) { d } else { @@ -38,6 +41,7 @@ fn main() { } }; + println!("** parsing pkgfile..."); let pkg = match PKGFile::try_from(data) { Ok(p) => p, Err(_) => { @@ -48,7 +52,10 @@ fn main() { let mani = prelude::mani_from_str(&pkg.manifest); - println!("!! Package: {}", &mani.package.name); + println!("-- Package: {}", &mani.package.name); + println!("-- Description: {}", &mani.package.description); + println!("-- Version: {}", &mani.package.version); + println!("-- Tags: {}", &mani.package.tags.join(", ")); if !mani.valid() { println!("!!! Manifest is not valid."); @@ -61,16 +68,22 @@ fn main() { .expect("Could not create tmp dir."); { - let mut file = File::create(format!("{}/pkgtar", tmp_dir.to_str().unwrap())) + println!("** writing tar to tmp file..."); + let mut file = File::create(format!("{}/pkgtar", tmp_dir.to_str().unwrap_or("/tmp"))) .expect("Could not create tmp pkgtar file"); file.write(&pkg.data) .expect("Could not write pkgtar to tmp file"); } + // TODO: untar // TODO: cd into pkg // TODO: run bootstrap commands + + println!("** removing temporary directory..."); + remove_dir_all(&tmp_dir) + .expect(&*format!("Could not remove tmp dir: {}", &tmp_dir.to_str().unwrap())); } Command::Unpack => {} _ => { diff --git a/bootpkg/src/prelude.rs b/bootpkg/src/prelude.rs index b62eda1..13a9936 100644 --- a/bootpkg/src/prelude.rs +++ b/bootpkg/src/prelude.rs @@ -1,8 +1,10 @@ -use manifest::{self, Manifest}; +use std::str::FromStr; + use serde::{Deserialize, Serialize}; -use std::{collections::HashMap, str::FromStr}; use toml; +use manifest::{self, Manifest}; + #[derive(Serialize, Deserialize)] pub struct PKGR { pub bootstrap: Option, diff --git a/manifest/src/bin.rs b/manifest/src/bin.rs index e8b48a3..a7af0e7 100644 --- a/manifest/src/bin.rs +++ b/manifest/src/bin.rs @@ -1,5 +1,6 @@ use std::collections::HashMap; -use serde::{Serialize, Deserialize}; + +use serde::{Deserialize, Serialize}; #[derive(Serialize, Deserialize)] pub struct Bin { diff --git a/manifest/src/build.rs b/manifest/src/build.rs index 0ad59ec..a727d49 100644 --- a/manifest/src/build.rs +++ b/manifest/src/build.rs @@ -1,6 +1,6 @@ use std::collections::HashMap; -use serde::{Serialize, Deserialize}; +use serde::{Deserialize, Serialize}; #[derive(Serialize, Deserialize)] pub struct Build { diff --git a/manifest/src/lib.rs b/manifest/src/lib.rs index 71efe31..c894ec5 100644 --- a/manifest/src/lib.rs +++ b/manifest/src/lib.rs @@ -1,6 +1,7 @@ -use serde::{Deserialize, Serialize}; use std::{collections::HashMap, str::FromStr}; +use serde::{Deserialize, Serialize}; + pub mod package; pub mod fs; pub mod bin; diff --git a/manifest/src/package.rs b/manifest/src/package.rs index 3fa1626..528aa29 100644 --- a/manifest/src/package.rs +++ b/manifest/src/package.rs @@ -1,6 +1,6 @@ use std::str::FromStr; -use serde::{Serialize, Deserialize}; +use serde::{Deserialize, Serialize}; #[derive(Serialize, Deserialize, Debug)] pub enum PackageType { diff --git a/manifest/src/pkgr.rs b/manifest/src/pkgr.rs index 2ca5fd4..8c4ce55 100644 --- a/manifest/src/pkgr.rs +++ b/manifest/src/pkgr.rs @@ -1,4 +1,4 @@ -use serde::{Serialize, Deserialize}; +use serde::{Deserialize, Serialize}; #[derive(Serialize, Deserialize)] pub struct PKGR {} \ No newline at end of file diff --git a/mock-pkg.py b/mock-pkg.py index 57ee008..70ba501 100644 --- a/mock-pkg.py +++ b/mock-pkg.py @@ -1,4 +1,5 @@ -import tarfile, os +import os +import tarfile with open("./package.toml", mode='r') as mani: data = mani.read() From 4bd99c44ec435bb33959dbc477fe9343e17899c6 Mon Sep 17 00:00:00 2001 From: Didier Date: Thu, 13 Jul 2023 17:27:10 +0200 Subject: [PATCH 004/123] build: bodge package. --- .gitignore | 1 + mock-pkg.py => bodge-pkg.py | 0 2 files changed, 1 insertion(+) rename mock-pkg.py => bodge-pkg.py (100%) diff --git a/.gitignore b/.gitignore index 972b0c4..be024c4 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ **/target **/Cargo.lock +*.pkg \ No newline at end of file diff --git a/mock-pkg.py b/bodge-pkg.py similarity index 100% rename from mock-pkg.py rename to bodge-pkg.py From e5cbd9f9e5e0797d527ef5ec80b3caa00462864e Mon Sep 17 00:00:00 2001 From: Didier Date: Thu, 13 Jul 2023 22:47:09 +0200 Subject: [PATCH 005/123] feat: implemented the whole process --- bootpkg/Cargo.toml | 1 + bootpkg/src/args.rs | 2 -- bootpkg/src/main.rs | 27 ++++++++++----------- bootpkg/src/prelude.rs | 35 +++++++++++++++++++++++++-- package.toml | 14 ++++++----- pkg/scripts/bootstrap/dirty_install | 3 +++ pkg/scripts/bootstrap/download_latest | 2 ++ pkg/scripts/check_installed | 3 +++ pkgfile/src/lib.rs | 2 +- 9 files changed, 64 insertions(+), 25 deletions(-) create mode 100644 pkg/scripts/bootstrap/dirty_install create mode 100644 pkg/scripts/check_installed diff --git a/bootpkg/Cargo.toml b/bootpkg/Cargo.toml index 8292150..d6c3186 100644 --- a/bootpkg/Cargo.toml +++ b/bootpkg/Cargo.toml @@ -13,3 +13,4 @@ serde = { version = "1.0.171", features = ["derive"] } regex = "1.9.1" reqwest = { version = "0.11.18", features = ["blocking"] } uuid = { version = "1.4.0", features = ["serde", "v4"] } +tar = "0.4.39" diff --git a/bootpkg/src/args.rs b/bootpkg/src/args.rs index cf061e3..0df76c9 100644 --- a/bootpkg/src/args.rs +++ b/bootpkg/src/args.rs @@ -1,6 +1,5 @@ pub enum Command { Strap, - Unpack, None, } @@ -8,7 +7,6 @@ impl From for Command { fn from(value: String) -> Self { match value.to_lowercase().as_str() { "strap" => Command::Strap, - "unpack" => Command::Unpack, _ => Command::None, } } diff --git a/bootpkg/src/main.rs b/bootpkg/src/main.rs index 8853c9b..6bc16a8 100644 --- a/bootpkg/src/main.rs +++ b/bootpkg/src/main.rs @@ -1,7 +1,7 @@ use std::env; use std::env::temp_dir; -use std::fs::{create_dir_all, File, remove_dir_all}; -use std::io::Write; +use std::fs::{create_dir_all, remove_dir_all}; +use std::io::Cursor; use std::process::exit; use uuid::Uuid; @@ -9,6 +9,7 @@ use uuid::Uuid; use pkgfile::PKGFile; use crate::args::{Args, Command}; +use crate::prelude::run_bootstrap; mod args; mod prelude; @@ -67,27 +68,25 @@ fn main() { create_dir_all(&tmp_dir) .expect("Could not create tmp dir."); + println!("** extracting pkgtar..."); { - println!("** writing tar to tmp file..."); - let mut file = File::create(format!("{}/pkgtar", tmp_dir.to_str().unwrap_or("/tmp"))) - .expect("Could not create tmp pkgtar file"); - - file.write(&pkg.data) - .expect("Could not write pkgtar to tmp file"); + let mut archive = tar::Archive::new(Cursor::new(pkg.data)); + archive.unpack(format!("{}/contents", &tmp_dir.to_str().unwrap())) + .expect("Could not extract archive"); } + env::set_current_dir(format!("{}/contents", &tmp_dir.to_str().unwrap())) + .expect("could not enter tmp dir"); - // TODO: untar - // TODO: cd into pkg - // TODO: run bootstrap commands - + println!("** running bootstrap commands..."); + let res_bootstrap = run_bootstrap(mani); println!("** removing temporary directory..."); remove_dir_all(&tmp_dir) .expect(&*format!("Could not remove tmp dir: {}", &tmp_dir.to_str().unwrap())); + exit(if res_bootstrap { println!("!! bootstrap success"); 0 } else { 1 }) } - Command::Unpack => {} _ => { - println!("Unsupported command, allowed commands: strap/unpack."); + println!("Unsupported command, allowed commands: strap."); exit(1); } } diff --git a/bootpkg/src/prelude.rs b/bootpkg/src/prelude.rs index 13a9936..aff3164 100644 --- a/bootpkg/src/prelude.rs +++ b/bootpkg/src/prelude.rs @@ -12,8 +12,8 @@ pub struct PKGR { #[derive(Serialize, Deserialize)] pub struct Bootstrap { - check_installed_commands: Vec, - commands: Vec + pub check_installed_commands: Vec, + pub commands: Vec } #[derive(Serialize, Deserialize)] @@ -33,3 +33,34 @@ pub fn mani_from_str(s: &str) -> Manifest> { pkgr: bmani.pkgr } } + +pub fn run_bootstrap(mani: Manifest>) -> bool { + if let Some(pkgr) = mani.pkgr { + if let Some(bootstrap) = pkgr.bootstrap { + fn run_command>(s: S) -> i32 { + std::process::Command::new("sh") + .arg("-c") + .arg(s.into()) + .spawn().expect("Could not spawn process.") + .wait().expect("Could not wait for process.") + .code().expect("Could not fetch exit code.") + } + + for command in &bootstrap.check_installed_commands { + if run_command(command) != 0 { + println!("!! Command failed: {}", command); + println!("!! Already installed."); + return false + } + } + for command in &bootstrap.commands { + if run_command(command) != 0 { + println!("!! Command failed: {}", command); + println!("!! Bootstrap failed!!!"); + return false + } + } + } + } + return true +} diff --git a/package.toml b/package.toml index 54d55b5..c99108d 100644 --- a/package.toml +++ b/package.toml @@ -78,8 +78,8 @@ root = "/root" #* # After the install script the build directory will be deleted and an CTREE (changed tree) will be generated. # Then the CTREE will be used to copy the files over. [build] -build_script = "/scripts/build" # relative to pkg -install_script = "/scripts/install" # relative to pkg +build_script = "scripts/build" # relative to pkg +install_script = "scripts/install" # relative to pkg [build.dependencies] base = "latest,stable" # selected by default @@ -95,11 +95,13 @@ base = "latest,stable" # selected by default # and only 1 release channel for pkgr [pkgr.bootstrap] ## any non-zero = installed -check_installed_commands = [ "/scripts/check_installed" ] +check_installed_commands = [ + "sh scripts/check_installed" +] # any non-zero = fail commands = [ - "/scripts/bootstrap/download_latest @version /tmp/pkgr.pkg", - "/scripts/bootstrap/dirty_install /tmp/pkgr.pkg", - "/scripts/bootstrap/check_install" + "sh scripts/bootstrap/download_latest @version /tmp/pkgr.pkg", + "sh scripts/bootstrap/dirty_install /tmp/pkgr.pkg", + "sh scripts/check_installed" ] diff --git a/pkg/scripts/bootstrap/dirty_install b/pkg/scripts/bootstrap/dirty_install new file mode 100644 index 0000000..41d3f5d --- /dev/null +++ b/pkg/scripts/bootstrap/dirty_install @@ -0,0 +1,3 @@ +#!/bin/sh + +printf "" \ No newline at end of file diff --git a/pkg/scripts/bootstrap/download_latest b/pkg/scripts/bootstrap/download_latest index 229aaa2..178bd4e 100644 --- a/pkg/scripts/bootstrap/download_latest +++ b/pkg/scripts/bootstrap/download_latest @@ -1,7 +1,9 @@ #!/bin/sh fetch_latest_version() { + printf "" } download_file() { + printf "" } \ No newline at end of file diff --git a/pkg/scripts/check_installed b/pkg/scripts/check_installed new file mode 100644 index 0000000..41d3f5d --- /dev/null +++ b/pkg/scripts/check_installed @@ -0,0 +1,3 @@ +#!/bin/sh + +printf "" \ No newline at end of file diff --git a/pkgfile/src/lib.rs b/pkgfile/src/lib.rs index 6025cd5..c3c056b 100644 --- a/pkgfile/src/lib.rs +++ b/pkgfile/src/lib.rs @@ -13,7 +13,7 @@ impl TryFrom> for PKGFile { .iter() .map(|v| u32::from(*v)) .collect(); - let manifest_size: u32 = ((header[1] << 8) | header[2]); + let manifest_size: u32 = (header[1] << 8) | header[2]; if manifest_size > value.len() as u32 { return Err(()); } From 14b8d6ea5424f023d60240d09195a1ff50a93cb3 Mon Sep 17 00:00:00 2001 From: Didier Date: Thu, 13 Jul 2023 22:47:19 +0200 Subject: [PATCH 006/123] build: fixed some bodges --- bodge-pkg.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/bodge-pkg.py b/bodge-pkg.py index 70ba501..cca3307 100644 --- a/bodge-pkg.py +++ b/bodge-pkg.py @@ -3,16 +3,24 @@ import tarfile with open("./package.toml", mode='r') as mani: data = mani.read() - with open("./mock.pkg", mode='wb') as pkg: + with open("./sample.pkg", mode='wb') as pkg: print("building header...") pkg.write(bytes([0x01, (len(data) >> 8) & 0xFF, len(data) & 0xFF])) print("writing manifest into pkg...") pkg.write(data.encode("utf-8")) with tarfile.TarFile("/tmp/pkgtar", 'w') as pkgtar: + print("tarring ./pkg...") + os.chdir("pkg") for root, dirs, files in os.walk("."): for file in files: - print(f"\33[2Kadd: {file}", end="\r", flush=True) + print(f"\33[2Kadd: {os.path.join(root, file)}", end="\r", flush=True) pkgtar.add(os.path.join(root, file)) + os.chdir("..") + for root, dirs, files in os.walk("pkgr"): + for file in files: + print(f"\33[2Kadd: {os.path.join(root, file)}", end="\r", flush=True) + pkgtar.add(os.path.join(root, file)) + print("\33[2K", end="\r", flush=True) with open("/tmp/pkgtar", 'rb') as pkgtar: print("appending pkgtar to pkg...") pkg.write(pkgtar.read()) From 94f0e7070b8d27c1585f596b198ea1b893c1427a Mon Sep 17 00:00:00 2001 From: Didier Date: Sun, 16 Jul 2023 19:00:09 +0200 Subject: [PATCH 007/123] feat: add package identifiers --- pkgr/Cargo.toml | 6 +- pkgr/src/commands/mod.rs | 8 +++ pkgr/src/main.rs | 57 +++++++++++++++++ pkgr/src/package/identifier.rs | 114 +++++++++++++++++++++++++++++++++ pkgr/src/package/mod.rs | 1 + 5 files changed, 185 insertions(+), 1 deletion(-) create mode 100644 pkgr/src/commands/mod.rs create mode 100644 pkgr/src/package/identifier.rs create mode 100644 pkgr/src/package/mod.rs diff --git a/pkgr/Cargo.toml b/pkgr/Cargo.toml index df095d2..92a0e91 100644 --- a/pkgr/Cargo.toml +++ b/pkgr/Cargo.toml @@ -6,4 +6,8 @@ edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -manifest = { path = "../manifest" } \ No newline at end of file +fern = "0.6.2" +getopts = "0.2.21" +log = "0.4.19" +regex = "1.9.1" +manifest = { path = "../manifest" } diff --git a/pkgr/src/commands/mod.rs b/pkgr/src/commands/mod.rs new file mode 100644 index 0000000..61e3fe8 --- /dev/null +++ b/pkgr/src/commands/mod.rs @@ -0,0 +1,8 @@ +use crate::package::identifier::PackageIdentifier; + +pub enum Command { + Grab(PackageIdentifier), // grab a package from a remote source + Remove(PackageIdentifier), // remove a package from the local source + List, // list all packages in the local source + Update, +} \ No newline at end of file diff --git a/pkgr/src/main.rs b/pkgr/src/main.rs index f79c691..8d75350 100644 --- a/pkgr/src/main.rs +++ b/pkgr/src/main.rs @@ -1,2 +1,59 @@ +use std::env; +use std::process::exit; +use std::str::FromStr; + +use getopts::Options; +use log::{info, SetLoggerError}; + +mod commands; +mod package; + fn main() { + setup_logger() + .expect("Unable to setup logger."); + + let args: Vec = env::args().collect(); + let program = args[0].clone(); + + let mut opts = Options::new(); + opts.optflag("h", "help", "Show help message."); + let matches = opts.parse(&args[1..]).unwrap_or_else(|e| panic!("{}", e.to_string())); + + if matches.opt_present("h") { + print_usage(&program, opts); + exit(1); + } + + let command = if !matches.free.is_empty() { + matches.free[0].clone() + } else { + print_usage(&program, opts); + exit(1); + }; + + info!("Identifier: {}", package::identifier::PackageIdentifier::from_str(&command).unwrap()); } + +fn print_usage(program: &str, opts: Options) { + let mut brief = format!("Usage: {} [options]\n\n", program); + brief += &*format!("Commands: grab \n"); + brief += &*format!(" remove\n"); + print!("{}", opts.usage(&brief.trim_end())); +} + +fn setup_logger() -> Result<(), SetLoggerError> { + fern::Dispatch::new() + .format(|out, message, record| { + out.finish(format_args!( + "{} {}", + match record.level().to_string().chars().nth(0).unwrap_or('I') { + 'E' | 'W' => "!!", + _ => "**", + }, + message + )) + }) + .level(log::LevelFilter::Info) + .chain(std::io::stdout()) + .apply() +} \ No newline at end of file diff --git a/pkgr/src/package/identifier.rs b/pkgr/src/package/identifier.rs new file mode 100644 index 0000000..01f9f6b --- /dev/null +++ b/pkgr/src/package/identifier.rs @@ -0,0 +1,114 @@ +use std::str::FromStr; + +use regex::Regex; + +#[derive(Debug)] +pub enum PackageIdentifierError { + InvalidPackageLocator(String), + InvalidURI(String), +} + +impl std::fmt::Display for PackageIdentifierError { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + match self { + PackageIdentifierError::InvalidPackageLocator(s) => write!(f, "Invalid package locator: {}", s), + PackageIdentifierError::InvalidURI(s) => write!(f, "Invalid URI: {}", s), + } + } +} + +#[derive(Debug)] +pub enum PackageIdentifier { + PackageLocator(PackageLocator), + URI(String), +} + +impl std::fmt::Display for PackageIdentifier { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + match self { + PackageIdentifier::PackageLocator(pl) => write!(f, "{}", pl), + PackageIdentifier::URI(uri) => write!(f, "{}", uri), + } + } +} + +#[derive(Debug)] +pub struct PackageLocator { + pub name: String, + pub version: Option, + pub tags: Option>, +} + +impl std::fmt::Display for PackageLocator { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + let mut s = self.name.clone(); + if let Some(version) = self.version { + s += &*format!("@{}", version); + } + if let Some(tags) = &self.tags { + s += &*format!(":{}", tags.join(",")); + } + write!(f, "{}", s) + } +} + +impl FromStr for PackageIdentifier { + type Err = PackageIdentifierError; + + fn from_str(s: &str) -> Result { + let uri_re = regex::Regex::new(r"^[a-zA-Z0-9]+://").unwrap(); + if uri_re.is_match(s) { + // there needs to be stuff after the protocol + let split = s.split("://").collect::>(); + if split.len() != 2 || split[1].is_empty() { + return Err(PackageIdentifierError::InvalidURI(s.to_string())); + } + Ok(PackageIdentifier::URI(s.to_string())) + } else { + let pl = match PackageLocator::from_str(s) { + Ok(pl) => pl, + Err(e) => return Err(PackageIdentifierError::InvalidPackageLocator(e.to_string())), + }; + Ok(PackageIdentifier::PackageLocator(pl)) + } + } +} + +impl FromStr for PackageLocator { + type Err = PackageIdentifierError; + + fn from_str(s: &str) -> Result { + #[allow(unused_assignments)] // false positive + let mut name = None; + let mut version = None; + let mut tags = None; + + if s.is_empty() { + return Err(PackageIdentifierError::InvalidPackageLocator(s.to_string())); + } + + let name_re = Regex::new("^([A-Za-z][A-Za-z0-9_.]+)").unwrap(); + let version_re = Regex::new("@([0-9]+)").unwrap(); + let tags_re = Regex::new(":([a-zA-Z0-9,._]+)").unwrap(); + + if let Some(caps) = name_re.captures(s) { + name = Some(caps.get(1).unwrap().as_str().to_string()); + } else { + return Err(PackageIdentifierError::InvalidPackageLocator(s.to_string())); + } + + if let Some(caps) = version_re.captures(s) { + version = Some(caps.get(1).unwrap().as_str().parse::().unwrap()); + } + + if let Some(caps) = tags_re.captures(s) { + tags = Some(caps.get(1).unwrap().as_str().split(",").map(|s| s.to_string()).collect()); + } + + Ok(PackageLocator { + name: name.unwrap(), + version, + tags, + }) + } +} \ No newline at end of file diff --git a/pkgr/src/package/mod.rs b/pkgr/src/package/mod.rs new file mode 100644 index 0000000..8f83cb4 --- /dev/null +++ b/pkgr/src/package/mod.rs @@ -0,0 +1 @@ +pub mod identifier; \ No newline at end of file From 70a22810b25291073894aa6262adda67513323f7 Mon Sep 17 00:00:00 2001 From: Didier Date: Sun, 16 Jul 2023 19:00:42 +0200 Subject: [PATCH 008/123] feat: add some functionality to bootpkg --- bootpkg/src/main.rs | 19 +++++++++++++------ bootpkg/src/prelude.rs | 19 +++++++++++-------- 2 files changed, 24 insertions(+), 14 deletions(-) diff --git a/bootpkg/src/main.rs b/bootpkg/src/main.rs index 6bc16a8..ffd4338 100644 --- a/bootpkg/src/main.rs +++ b/bootpkg/src/main.rs @@ -65,13 +65,13 @@ fn main() { let mut tmp_dir = temp_dir(); tmp_dir.push(format!("{}", Uuid::new_v4())); - create_dir_all(&tmp_dir) - .expect("Could not create tmp dir."); + create_dir_all(&tmp_dir).expect("Could not create tmp dir."); println!("** extracting pkgtar..."); { let mut archive = tar::Archive::new(Cursor::new(pkg.data)); - archive.unpack(format!("{}/contents", &tmp_dir.to_str().unwrap())) + archive + .unpack(format!("{}/contents", &tmp_dir.to_str().unwrap())) .expect("Could not extract archive"); } @@ -81,9 +81,16 @@ fn main() { println!("** running bootstrap commands..."); let res_bootstrap = run_bootstrap(mani); println!("** removing temporary directory..."); - remove_dir_all(&tmp_dir) - .expect(&*format!("Could not remove tmp dir: {}", &tmp_dir.to_str().unwrap())); - exit(if res_bootstrap { println!("!! bootstrap success"); 0 } else { 1 }) + remove_dir_all(&tmp_dir).expect(&*format!( + "Could not remove tmp dir: {}", + &tmp_dir.to_str().unwrap() + )); + exit(if res_bootstrap { + println!("!! bootstrap success"); + 0 + } else { + 1 + }) } _ => { println!("Unsupported command, allowed commands: strap."); diff --git a/bootpkg/src/prelude.rs b/bootpkg/src/prelude.rs index aff3164..b82b62a 100644 --- a/bootpkg/src/prelude.rs +++ b/bootpkg/src/prelude.rs @@ -13,7 +13,7 @@ pub struct PKGR { #[derive(Serialize, Deserialize)] pub struct Bootstrap { pub check_installed_commands: Vec, - pub commands: Vec + pub commands: Vec, } #[derive(Serialize, Deserialize)] @@ -30,7 +30,7 @@ pub fn mani_from_str(s: &str) -> Manifest> { fs: mani.fs, bin: mani.bin, build: mani.build, - pkgr: bmani.pkgr + pkgr: bmani.pkgr, } } @@ -41,26 +41,29 @@ pub fn run_bootstrap(mani: Manifest>) -> bool { std::process::Command::new("sh") .arg("-c") .arg(s.into()) - .spawn().expect("Could not spawn process.") - .wait().expect("Could not wait for process.") - .code().expect("Could not fetch exit code.") + .spawn() + .expect("Could not spawn process.") + .wait() + .expect("Could not wait for process.") + .code() + .expect("Could not fetch exit code.") } for command in &bootstrap.check_installed_commands { if run_command(command) != 0 { println!("!! Command failed: {}", command); println!("!! Already installed."); - return false + return false; } } for command in &bootstrap.commands { if run_command(command) != 0 { println!("!! Command failed: {}", command); println!("!! Bootstrap failed!!!"); - return false + return false; } } } } - return true + return true; } From 1ec1d5c9b2a2c08fb3ae9624e6c85d41b5f70de8 Mon Sep 17 00:00:00 2001 From: Didier Date: Sun, 16 Jul 2023 19:00:56 +0200 Subject: [PATCH 009/123] fix: pkg interpreting --- pkgfile/src/lib.rs | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/pkgfile/src/lib.rs b/pkgfile/src/lib.rs index c3c056b..bc26f99 100644 --- a/pkgfile/src/lib.rs +++ b/pkgfile/src/lib.rs @@ -9,28 +9,22 @@ impl TryFrom> for PKGFile { fn try_from(value: Vec) -> Result { match value[0] { 1 => { - let header: Vec = value[..3] - .iter() - .map(|v| u32::from(*v)) - .collect(); + let header: Vec = value[..3].iter().map(|v| u32::from(*v)).collect(); let manifest_size: u32 = (header[1] << 8) | header[2]; if manifest_size > value.len() as u32 { return Err(()); } Ok(PKGFile { manifest: match String::from_utf8( - value[ - 3..(manifest_size as usize + 3) - ].to_vec() - ) - { + value[3..(manifest_size as usize + 3)].to_vec(), + ) { Ok(s) => s, - _ => return Err(()) + _ => return Err(()), }, data: value[(manifest_size as usize + 3)..].to_vec(), }) } - _ => Err(()) + _ => Err(()), } } } @@ -45,4 +39,4 @@ impl Into> for PKGFile { bytes.extend(self.data); bytes } -} \ No newline at end of file +} From 650499736caeb618124a590e383d9f9852a8fd0a Mon Sep 17 00:00:00 2001 From: Didier Date: Sun, 16 Jul 2023 19:01:20 +0200 Subject: [PATCH 010/123] style: formatting... --- manifest/src/bin.rs | 4 ++-- manifest/src/build.rs | 4 ++-- manifest/src/lib.rs | 10 +++++----- manifest/src/package.rs | 8 ++++---- 4 files changed, 13 insertions(+), 13 deletions(-) diff --git a/manifest/src/bin.rs b/manifest/src/bin.rs index a7af0e7..e30b528 100644 --- a/manifest/src/bin.rs +++ b/manifest/src/bin.rs @@ -5,5 +5,5 @@ use serde::{Deserialize, Serialize}; #[derive(Serialize, Deserialize)] pub struct Bin { pub root: String, - pub checksums: HashMap -} \ No newline at end of file + pub checksums: HashMap, +} diff --git a/manifest/src/build.rs b/manifest/src/build.rs index a727d49..ac223d2 100644 --- a/manifest/src/build.rs +++ b/manifest/src/build.rs @@ -6,5 +6,5 @@ use serde::{Deserialize, Serialize}; pub struct Build { build_script: String, install_script: String, - dependencies: HashMap -} \ No newline at end of file + dependencies: HashMap, +} diff --git a/manifest/src/lib.rs b/manifest/src/lib.rs index c894ec5..8a943fa 100644 --- a/manifest/src/lib.rs +++ b/manifest/src/lib.rs @@ -2,10 +2,10 @@ use std::{collections::HashMap, str::FromStr}; use serde::{Deserialize, Serialize}; -pub mod package; -pub mod fs; pub mod bin; pub mod build; +pub mod fs; +pub mod package; pub mod pkgr; #[derive(Serialize, Deserialize)] @@ -16,13 +16,13 @@ pub struct Manifest

> { pub fs: fs::FS, pub bin: Option, pub build: Option, - pub pkgr: P + pub pkgr: P, } impl Manifest { pub fn valid(&self) -> bool { if self.bin.is_none() && self.build.is_none() { - return false + return false; } true } @@ -43,7 +43,7 @@ impl Default for Manifest { fs: fs::FS::default(), bin: None, build: None, - pkgr: None + pkgr: None, } } } diff --git a/manifest/src/package.rs b/manifest/src/package.rs index 528aa29..017271d 100644 --- a/manifest/src/package.rs +++ b/manifest/src/package.rs @@ -9,7 +9,7 @@ pub enum PackageType { #[serde(rename = "library")] Library, #[serde(rename = "meta")] - Meta + Meta, } #[derive(Serialize, Deserialize, Debug)] @@ -20,7 +20,7 @@ pub struct Package { pub package_type: PackageType, pub version: u64, pub tags: Vec, - pub arch: String + pub arch: String, } impl Default for Package { @@ -31,7 +31,7 @@ impl Default for Package { package_type: PackageType::Application, version: 0, tags: Vec::default(), - arch: std::env::consts::ARCH.to_string() + arch: std::env::consts::ARCH.to_string(), } } } @@ -42,4 +42,4 @@ impl FromStr for Package { fn from_str(s: &str) -> Result { toml::from_str(s) } -} \ No newline at end of file +} From 11bcaccffc674be43774b6de8fc43cec4e9d2a54 Mon Sep 17 00:00:00 2001 From: Didier Date: Sun, 16 Jul 2023 19:06:14 +0200 Subject: [PATCH 011/123] ci: add ci config --- .github/workflows/bootpkg.yaml | 24 ++++++++++++++++++++++++ .github/workflows/pkgr.yaml | 24 ++++++++++++++++++++++++ 2 files changed, 48 insertions(+) create mode 100644 .github/workflows/bootpkg.yaml create mode 100644 .github/workflows/pkgr.yaml diff --git a/.github/workflows/bootpkg.yaml b/.github/workflows/bootpkg.yaml new file mode 100644 index 0000000..876a53c --- /dev/null +++ b/.github/workflows/bootpkg.yaml @@ -0,0 +1,24 @@ +on: + push: + branches: + - main + +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: install dependencies + run: | + apt update -y + apt install -y curl gcc g++ make + - uses: https://github.com/actions-rs/toolchain@v1 + with: + toolchain: stable + override: true + - run: | + cd bootpkg + - uses: https://github.com/actions-rs/cargo@v1 + with: + command: build + args: --verbose --all diff --git a/.github/workflows/pkgr.yaml b/.github/workflows/pkgr.yaml new file mode 100644 index 0000000..ff8e6a4 --- /dev/null +++ b/.github/workflows/pkgr.yaml @@ -0,0 +1,24 @@ +on: + push: + branches: + - main + +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: install dependencies + run: | + apt update -y + apt install -y curl gcc g++ make + - uses: https://github.com/actions-rs/toolchain@v1 + with: + toolchain: stable + override: true + - run: | + cd pkgr + - uses: https://github.com/actions-rs/cargo@v1 + with: + command: build + args: --verbose --all From 00e4d244aed2f74ba30178c34f3c924f653cb1f1 Mon Sep 17 00:00:00 2001 From: Didier Date: Sun, 16 Jul 2023 19:08:34 +0200 Subject: [PATCH 012/123] ci: set workdir --- .github/workflows/bootpkg.yaml | 3 +-- .github/workflows/pkgr.yaml | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/bootpkg.yaml b/.github/workflows/bootpkg.yaml index 876a53c..9a291d4 100644 --- a/.github/workflows/bootpkg.yaml +++ b/.github/workflows/bootpkg.yaml @@ -16,9 +16,8 @@ jobs: with: toolchain: stable override: true - - run: | - cd bootpkg - uses: https://github.com/actions-rs/cargo@v1 with: + cwd: bootpkg command: build args: --verbose --all diff --git a/.github/workflows/pkgr.yaml b/.github/workflows/pkgr.yaml index ff8e6a4..f1a7971 100644 --- a/.github/workflows/pkgr.yaml +++ b/.github/workflows/pkgr.yaml @@ -16,9 +16,8 @@ jobs: with: toolchain: stable override: true - - run: | - cd pkgr - uses: https://github.com/actions-rs/cargo@v1 with: + cwd: pkgr command: build args: --verbose --all From a07a53dabf94dc91123548e2e8afde0899a580cf Mon Sep 17 00:00:00 2001 From: Didier Date: Sun, 16 Jul 2023 19:15:12 +0200 Subject: [PATCH 013/123] ci: set manifest path --- .github/workflows/bootpkg.yaml | 5 ++--- .github/workflows/pkgr.yaml | 5 ++--- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/.github/workflows/bootpkg.yaml b/.github/workflows/bootpkg.yaml index 9a291d4..baf6377 100644 --- a/.github/workflows/bootpkg.yaml +++ b/.github/workflows/bootpkg.yaml @@ -18,6 +18,5 @@ jobs: override: true - uses: https://github.com/actions-rs/cargo@v1 with: - cwd: bootpkg - command: build - args: --verbose --all + command: check + args: --manifest-path bootpkg/Cargo.toml --verbose --all diff --git a/.github/workflows/pkgr.yaml b/.github/workflows/pkgr.yaml index f1a7971..6084d80 100644 --- a/.github/workflows/pkgr.yaml +++ b/.github/workflows/pkgr.yaml @@ -18,6 +18,5 @@ jobs: override: true - uses: https://github.com/actions-rs/cargo@v1 with: - cwd: pkgr - command: build - args: --verbose --all + command: check + args: --manifest-path pkgr/Cargo.toml --verbose --all \ No newline at end of file From 69e5fa2c0dafc832e77a91c7fef2c9f56b8d8c57 Mon Sep 17 00:00:00 2001 From: Didier Date: Sun, 16 Jul 2023 19:28:42 +0200 Subject: [PATCH 014/123] ci: merge and add step --- .github/workflows/bootpkg.yaml | 22 ------------- .github/workflows/build.yaml | 56 ++++++++++++++++++++++++++++++++++ .github/workflows/pkgr.yaml | 22 ------------- 3 files changed, 56 insertions(+), 44 deletions(-) delete mode 100644 .github/workflows/bootpkg.yaml create mode 100644 .github/workflows/build.yaml delete mode 100644 .github/workflows/pkgr.yaml diff --git a/.github/workflows/bootpkg.yaml b/.github/workflows/bootpkg.yaml deleted file mode 100644 index baf6377..0000000 --- a/.github/workflows/bootpkg.yaml +++ /dev/null @@ -1,22 +0,0 @@ -on: - push: - branches: - - main - -jobs: - build: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - name: install dependencies - run: | - apt update -y - apt install -y curl gcc g++ make - - uses: https://github.com/actions-rs/toolchain@v1 - with: - toolchain: stable - override: true - - uses: https://github.com/actions-rs/cargo@v1 - with: - command: check - args: --manifest-path bootpkg/Cargo.toml --verbose --all diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml new file mode 100644 index 0000000..52bb97b --- /dev/null +++ b/.github/workflows/build.yaml @@ -0,0 +1,56 @@ +on: + push: + branches: + - main + +jobs: + check: # check if the code compiles + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: install dependencies + run: | + apt update -y + apt install -y curl gcc g++ make + - uses: https://github.com/actions-rs/toolchain@v1 + with: + toolchain: stable + override: true + - uses: https://github.com/actions-rs/cargo@v1 + with: + command: check + args: --manifest-path bootpkg/Cargo.toml --verbose --all + - uses: https://github.com/actions-rs/cargo@v1 + with: + command: check + args: --manifest-path pkgr/Cargo.toml --verbose --all + build: # create artifacts and upload them + needs: check + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: install dependencies + run: | + apt update -y + apt install -y curl gcc g++ make + - uses: https://github.com/actions-rs/toolchain@v1 + with: + toolchain: stable + override: true + - uses: https://github.com/actions-rs/cargo@v1 + with: + command: build + args: --release --manifest-path bootpkg/Cargo.toml --verbose --all + continue-on-error: true + - uses: https://github.com/actions-rs/cargo@v1 + with: + command: build + args: --release --manifest-path pkgr/Cargo.toml --verbose --all + - uses: actions/upload-artifact@v2 + with: + name: bootpkg + path: bootpkg/target/release/bootpkg + - uses: actions/upload-artifact@v2 + with: + name: pkgr + path: pkgr/target/release/pkgr diff --git a/.github/workflows/pkgr.yaml b/.github/workflows/pkgr.yaml deleted file mode 100644 index 6084d80..0000000 --- a/.github/workflows/pkgr.yaml +++ /dev/null @@ -1,22 +0,0 @@ -on: - push: - branches: - - main - -jobs: - build: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - name: install dependencies - run: | - apt update -y - apt install -y curl gcc g++ make - - uses: https://github.com/actions-rs/toolchain@v1 - with: - toolchain: stable - override: true - - uses: https://github.com/actions-rs/cargo@v1 - with: - command: check - args: --manifest-path pkgr/Cargo.toml --verbose --all \ No newline at end of file From 93fdeb561af291c4626aac3994cb64e8816c519c Mon Sep 17 00:00:00 2001 From: Didier Date: Sun, 16 Jul 2023 19:34:50 +0200 Subject: [PATCH 015/123] ci: artifacts not supported as of right now. --- .github/workflows/build.yaml | 60 ++++++++++++++++++------------------ 1 file changed, 30 insertions(+), 30 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 52bb97b..2f75ff4 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -24,33 +24,33 @@ jobs: with: command: check args: --manifest-path pkgr/Cargo.toml --verbose --all - build: # create artifacts and upload them - needs: check - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - name: install dependencies - run: | - apt update -y - apt install -y curl gcc g++ make - - uses: https://github.com/actions-rs/toolchain@v1 - with: - toolchain: stable - override: true - - uses: https://github.com/actions-rs/cargo@v1 - with: - command: build - args: --release --manifest-path bootpkg/Cargo.toml --verbose --all - continue-on-error: true - - uses: https://github.com/actions-rs/cargo@v1 - with: - command: build - args: --release --manifest-path pkgr/Cargo.toml --verbose --all - - uses: actions/upload-artifact@v2 - with: - name: bootpkg - path: bootpkg/target/release/bootpkg - - uses: actions/upload-artifact@v2 - with: - name: pkgr - path: pkgr/target/release/pkgr +# build: # create artifacts and upload them +# needs: check +# runs-on: ubuntu-latest +# steps: +# - uses: actions/checkout@v2 +# - name: install dependencies +# run: | +# apt update -y +# apt install -y curl gcc g++ make +# - uses: https://github.com/actions-rs/toolchain@v1 +# with: +# toolchain: stable +# override: true +# - uses: https://github.com/actions-rs/cargo@v1 +# with: +# command: build +# args: --release --manifest-path bootpkg/Cargo.toml --verbose --all +# continue-on-error: true +# - uses: https://github.com/actions-rs/cargo@v1 +# with: +# command: build +# args: --release --manifest-path pkgr/Cargo.toml --verbose --all +# - uses: actions/upload-artifact@v2 +# with: +# name: bootpkg +# path: bootpkg/target/release/bootpkg +# - uses: actions/upload-artifact@v2 +# with: +# name: pkgr +# path: pkgr/target/release/pkgr From 0a002ab4172ef9d1c86271126afb3a24db05b838 Mon Sep 17 00:00:00 2001 From: Didier Date: Sun, 16 Jul 2023 19:36:40 +0200 Subject: [PATCH 016/123] ci: we don't have to compile for docs updates --- .github/workflows/build.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 2f75ff4..13d43f3 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -2,6 +2,11 @@ on: push: branches: - main + paths: + - 'pkgr/**' + - 'manifest/**' + - 'bootpkg/**' + - 'pkgfile/**' jobs: check: # check if the code compiles From 0bbdb9219fa58b92bb920c20d7f370319cf88d5e Mon Sep 17 00:00:00 2001 From: Didier Date: Sun, 16 Jul 2023 20:06:23 +0200 Subject: [PATCH 017/123] docs: added more specification --- docs/README.md | 20 ++++++++++++++++++++ docs/pkgfile.md | 24 ++++++++++++++++++++++++ docs/pkgr/README.md | 28 ++++++++++++++++++++++++++++ docs/pkgr/installedmanifest.md | 19 +++++++++++++++++++ 4 files changed, 91 insertions(+) create mode 100644 docs/README.md create mode 100644 docs/pkgfile.md create mode 100644 docs/pkgr/README.md create mode 100644 docs/pkgr/installedmanifest.md diff --git a/docs/README.md b/docs/README.md new file mode 100644 index 0000000..0e9b295 --- /dev/null +++ b/docs/README.md @@ -0,0 +1,20 @@ +# Packager + +Packager is a tool for package management. +It was built to make it easier to download and distribute packages. + +## pkgr +pkgr is the main tool for the packager. + +Because this tool is so feature rich, it has its own [README](./pkgr/README.md). + +## bootpkg +bootpkg is a tool primariliy used to bootstrap the packager. + +### Usage + +```bash +bootpkg strap ./pkgr.pkg +``` + +This will extract the pkgfile and read the `[pkgr.bootstrap]` section to bootstrap the packager or any other package. \ No newline at end of file diff --git a/docs/pkgfile.md b/docs/pkgfile.md new file mode 100644 index 0000000..4b2b81a --- /dev/null +++ b/docs/pkgfile.md @@ -0,0 +1,24 @@ +# PKGFILE +This file is essentially a tar with the manifest in the header. + +## Format +The format is as follows: +``` +[PKGFILE version (1 byte)][manifest length x 256 (1 byte)][manifest length (1 byte)] +[manifest (manifest length bytes)] +[archive] +``` + +The file is Big Endian. + +## Unpacking +To unpack a PKGFILE, you can use the following command: +```bash +pkgr unpack +``` + +## Packing +You can write your own packer, or use the following command: +```bash +pkgr pack +``` diff --git a/docs/pkgr/README.md b/docs/pkgr/README.md new file mode 100644 index 0000000..042206a --- /dev/null +++ b/docs/pkgr/README.md @@ -0,0 +1,28 @@ +# pkgr +PKGR is the main tool for the packager. + +## Commands +### `pkgr install [--bin/--build] ` +This command will download a package from the repository or the uri provided and install it. +This command can also be used to update a package. + +### `pkgr remove ` +This command will remove a package from the system. + +### `pkgr revert ` +This command will revert a package to the previous version. + +### `pkgr download ` +This command will download a package from the repository or the uri provided into the specified path. + +### `pkgr unpack ` +This command will unpack a pkgfile into the specified output directory. + +### `pkgr pack ` +This command will pack a manifest and archive into a pkgfile. + +### `pkgr list [-i]` +This command will list all packages known or installed packages if `-i` is specified. + +### `pkgr update [package_name]` +This command will update all packages or a specific package. diff --git a/docs/pkgr/installedmanifest.md b/docs/pkgr/installedmanifest.md new file mode 100644 index 0000000..48f7b7b --- /dev/null +++ b/docs/pkgr/installedmanifest.md @@ -0,0 +1,19 @@ +# Installed manifest + +## Where? +The package manager will store the install manifest in: +`/var/lib/pkgr/installed///installed.toml` + +To find the latest we'll use the `latest` symlink: +`/var/lib/pkgr/installed//latest/installed.toml` + +```toml +install_start = "2021-01-01T00:00:00Z" # install_start is the time the pkgr started installing the package +install_end = "2021-01-01T00:00:00Z" # install_end is the time the pkgr finished installing the package +install_by = "pkgr" # install_by is the name of the package manager that installed the package + +[old_versions] # This logs old versions of the package (that were installed before) +9 = "../9/pkgr.pkg" + +# pkgr.* is reserved for the manifest of the package +``` \ No newline at end of file From c603c403b6e7b2f7c5d84d8afde6fe3ac385c50f Mon Sep 17 00:00:00 2001 From: Didier Date: Sun, 16 Jul 2023 20:06:38 +0200 Subject: [PATCH 018/123] fix: update to spec --- pkgr/src/commands/mod.rs | 3 ++- pkgr/src/main.rs | 1 + pkgr/src/prelude.rs | 1 + 3 files changed, 4 insertions(+), 1 deletion(-) create mode 100644 pkgr/src/prelude.rs diff --git a/pkgr/src/commands/mod.rs b/pkgr/src/commands/mod.rs index 61e3fe8..7f18ebd 100644 --- a/pkgr/src/commands/mod.rs +++ b/pkgr/src/commands/mod.rs @@ -1,7 +1,8 @@ use crate::package::identifier::PackageIdentifier; +use crate::prelude::Arguments; pub enum Command { - Grab(PackageIdentifier), // grab a package from a remote source + Install(Arguments, PackageIdentifier), // install a package from a remote source Remove(PackageIdentifier), // remove a package from the local source List, // list all packages in the local source Update, diff --git a/pkgr/src/main.rs b/pkgr/src/main.rs index 8d75350..9029074 100644 --- a/pkgr/src/main.rs +++ b/pkgr/src/main.rs @@ -5,6 +5,7 @@ use std::str::FromStr; use getopts::Options; use log::{info, SetLoggerError}; +mod prelude; mod commands; mod package; diff --git a/pkgr/src/prelude.rs b/pkgr/src/prelude.rs new file mode 100644 index 0000000..d7ac579 --- /dev/null +++ b/pkgr/src/prelude.rs @@ -0,0 +1 @@ +pub struct Arguments(Vec); \ No newline at end of file From 1c758fa0dd56f6d4098674c10be17aceb9f685f4 Mon Sep 17 00:00:00 2001 From: Didier Date: Sun, 16 Jul 2023 20:11:28 +0200 Subject: [PATCH 019/123] ci: remove branch spec --- .github/workflows/build.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 13d43f3..d9bf0bc 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -1,7 +1,5 @@ on: push: - branches: - - main paths: - 'pkgr/**' - 'manifest/**' From 867d91ffab7c6aab65536249eb4a626a39b1ba79 Mon Sep 17 00:00:00 2001 From: Didier Date: Mon, 17 Jul 2023 11:50:19 +0200 Subject: [PATCH 020/123] feat: arguments with clip --- pkgr/Cargo.toml | 2 +- pkgr/src/commands/mod.rs | 64 +++++++++++++++++++++++++++++++--- pkgr/src/main.rs | 38 ++++---------------- pkgr/src/package/identifier.rs | 24 +++++++++---- pkgr/src/package/mod.rs | 2 +- pkgr/src/prelude.rs | 1 - 6 files changed, 85 insertions(+), 46 deletions(-) delete mode 100644 pkgr/src/prelude.rs diff --git a/pkgr/Cargo.toml b/pkgr/Cargo.toml index 92a0e91..9f719ea 100644 --- a/pkgr/Cargo.toml +++ b/pkgr/Cargo.toml @@ -7,7 +7,7 @@ edition = "2021" [dependencies] fern = "0.6.2" -getopts = "0.2.21" log = "0.4.19" regex = "1.9.1" manifest = { path = "../manifest" } +clap = { version = "4.3.12", features = ["derive"] } diff --git a/pkgr/src/commands/mod.rs b/pkgr/src/commands/mod.rs index 7f18ebd..b5d698c 100644 --- a/pkgr/src/commands/mod.rs +++ b/pkgr/src/commands/mod.rs @@ -1,9 +1,63 @@ use crate::package::identifier::PackageIdentifier; -use crate::prelude::Arguments; +use clap::{Parser, Subcommand}; +use log::error; +#[derive(Parser, Debug)] +#[clap(name = "pkgr", version = "0.1.0", author = "")] +pub struct Cli { + #[command(subcommand)] + pub command: Command, +} + +#[derive(Subcommand, Debug)] pub enum Command { - Install(Arguments, PackageIdentifier), // install a package from a remote source - Remove(PackageIdentifier), // remove a package from the local source - List, // list all packages in the local source + /// Install a package + Install { + #[arg(short, long, default_value_t = false)] + build: bool, + package_identifier: PackageIdentifier, + }, + /// Remove a package from the system + Remove { + package_identifier: PackageIdentifier, + }, + /// List packages + List { + /// List installed packages + #[arg(short, long, default_value_t = true)] + installed: bool, + }, + /// Update packages on the system Update, -} \ No newline at end of file + #[command(hide = true)] + None, +} + +impl Command { + pub fn execute(&self) { + match self { + Command::Install { + build, + package_identifier, + } => { + if *build { + error!("Build is not yet implemented."); + } else { + error!("Install is not yet implemented."); + } + } + Command::Remove { package_identifier } => { + error!("Remove is not yet implemented."); + } + Command::List { installed } => { + error!("List is not yet implemented."); + } + Command::Update => { + error!("Update is not yet implemented."); + } + Command::None => { + error!("No command was specified."); + } + } + } +} diff --git a/pkgr/src/main.rs b/pkgr/src/main.rs index 9029074..50b9007 100644 --- a/pkgr/src/main.rs +++ b/pkgr/src/main.rs @@ -1,45 +1,19 @@ +use clap::Parser; use std::env; use std::process::exit; use std::str::FromStr; -use getopts::Options; +use crate::commands::{Cli, Command}; use log::{info, SetLoggerError}; -mod prelude; mod commands; mod package; fn main() { - setup_logger() - .expect("Unable to setup logger."); + setup_logger().expect("Unable to setup logger."); - let args: Vec = env::args().collect(); - let program = args[0].clone(); - - let mut opts = Options::new(); - opts.optflag("h", "help", "Show help message."); - let matches = opts.parse(&args[1..]).unwrap_or_else(|e| panic!("{}", e.to_string())); - - if matches.opt_present("h") { - print_usage(&program, opts); - exit(1); - } - - let command = if !matches.free.is_empty() { - matches.free[0].clone() - } else { - print_usage(&program, opts); - exit(1); - }; - - info!("Identifier: {}", package::identifier::PackageIdentifier::from_str(&command).unwrap()); -} - -fn print_usage(program: &str, opts: Options) { - let mut brief = format!("Usage: {} [options]\n\n", program); - brief += &*format!("Commands: grab \n"); - brief += &*format!(" remove\n"); - print!("{}", opts.usage(&brief.trim_end())); + let c = Cli::parse(); + info!("Command: {:?}", c); } fn setup_logger() -> Result<(), SetLoggerError> { @@ -57,4 +31,4 @@ fn setup_logger() -> Result<(), SetLoggerError> { .level(log::LevelFilter::Info) .chain(std::io::stdout()) .apply() -} \ No newline at end of file +} diff --git a/pkgr/src/package/identifier.rs b/pkgr/src/package/identifier.rs index 01f9f6b..07b2bcc 100644 --- a/pkgr/src/package/identifier.rs +++ b/pkgr/src/package/identifier.rs @@ -1,8 +1,9 @@ +use std::error::Error; use std::str::FromStr; use regex::Regex; -#[derive(Debug)] +#[derive(Debug, Clone)] pub enum PackageIdentifierError { InvalidPackageLocator(String), InvalidURI(String), @@ -11,13 +12,17 @@ pub enum PackageIdentifierError { impl std::fmt::Display for PackageIdentifierError { fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { match self { - PackageIdentifierError::InvalidPackageLocator(s) => write!(f, "Invalid package locator: {}", s), + PackageIdentifierError::InvalidPackageLocator(s) => { + write!(f, "Invalid package locator: {}", s) + } PackageIdentifierError::InvalidURI(s) => write!(f, "Invalid URI: {}", s), } } } -#[derive(Debug)] +impl Error for PackageIdentifierError {} + +#[derive(Debug, Clone)] pub enum PackageIdentifier { PackageLocator(PackageLocator), URI(String), @@ -32,7 +37,7 @@ impl std::fmt::Display for PackageIdentifier { } } -#[derive(Debug)] +#[derive(Debug, Clone)] pub struct PackageLocator { pub name: String, pub version: Option, @@ -102,7 +107,14 @@ impl FromStr for PackageLocator { } if let Some(caps) = tags_re.captures(s) { - tags = Some(caps.get(1).unwrap().as_str().split(",").map(|s| s.to_string()).collect()); + tags = Some( + caps.get(1) + .unwrap() + .as_str() + .split(",") + .map(|s| s.to_string()) + .collect(), + ); } Ok(PackageLocator { @@ -111,4 +123,4 @@ impl FromStr for PackageLocator { tags, }) } -} \ No newline at end of file +} diff --git a/pkgr/src/package/mod.rs b/pkgr/src/package/mod.rs index 8f83cb4..34cdf0a 100644 --- a/pkgr/src/package/mod.rs +++ b/pkgr/src/package/mod.rs @@ -1 +1 @@ -pub mod identifier; \ No newline at end of file +pub mod identifier; diff --git a/pkgr/src/prelude.rs b/pkgr/src/prelude.rs deleted file mode 100644 index d7ac579..0000000 --- a/pkgr/src/prelude.rs +++ /dev/null @@ -1 +0,0 @@ -pub struct Arguments(Vec); \ No newline at end of file From 12f39507e1a59e662ed060766dd87c74a93368ad Mon Sep 17 00:00:00 2001 From: Didier Date: Mon, 17 Jul 2023 11:50:37 +0200 Subject: [PATCH 021/123] docs: change pkgr cli spec --- docs/pkgr/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/pkgr/README.md b/docs/pkgr/README.md index 042206a..0b713bd 100644 --- a/docs/pkgr/README.md +++ b/docs/pkgr/README.md @@ -24,5 +24,5 @@ This command will pack a manifest and archive into a pkgfile. ### `pkgr list [-i]` This command will list all packages known or installed packages if `-i` is specified. -### `pkgr update [package_name]` -This command will update all packages or a specific package. +### `pkgr update` +This command will update all packages. From 6fd8d5e45e58d23e4ff49b34471f3482bfb7f9ea Mon Sep 17 00:00:00 2001 From: Didier Date: Mon, 17 Jul 2023 11:50:51 +0200 Subject: [PATCH 022/123] fix: cleanup code --- pkgr/src/commands/mod.rs | 6 +++--- pkgr/src/main.rs | 8 ++++---- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/pkgr/src/commands/mod.rs b/pkgr/src/commands/mod.rs index b5d698c..3c52aca 100644 --- a/pkgr/src/commands/mod.rs +++ b/pkgr/src/commands/mod.rs @@ -38,7 +38,7 @@ impl Command { match self { Command::Install { build, - package_identifier, + package_identifier: _, } => { if *build { error!("Build is not yet implemented."); @@ -46,10 +46,10 @@ impl Command { error!("Install is not yet implemented."); } } - Command::Remove { package_identifier } => { + Command::Remove { package_identifier: _ } => { error!("Remove is not yet implemented."); } - Command::List { installed } => { + Command::List { installed: _ } => { error!("List is not yet implemented."); } Command::Update => { diff --git a/pkgr/src/main.rs b/pkgr/src/main.rs index 50b9007..17e200f 100644 --- a/pkgr/src/main.rs +++ b/pkgr/src/main.rs @@ -1,9 +1,9 @@ use clap::Parser; -use std::env; -use std::process::exit; -use std::str::FromStr; -use crate::commands::{Cli, Command}; + + + +use crate::commands::{Cli}; use log::{info, SetLoggerError}; mod commands; From 1a95046b876c9346f03205ce3cf1c0ddfd45dcc1 Mon Sep 17 00:00:00 2001 From: Didier Date: Mon, 17 Jul 2023 12:13:14 +0200 Subject: [PATCH 023/123] feat: some changes to logging --- pkgr/Cargo.toml | 1 + pkgr/src/commands/mod.rs | 13 ++++++++++--- pkgr/src/main.rs | 30 +++++++++++++++++++----------- 3 files changed, 30 insertions(+), 14 deletions(-) diff --git a/pkgr/Cargo.toml b/pkgr/Cargo.toml index 9f719ea..6e94865 100644 --- a/pkgr/Cargo.toml +++ b/pkgr/Cargo.toml @@ -11,3 +11,4 @@ log = "0.4.19" regex = "1.9.1" manifest = { path = "../manifest" } clap = { version = "4.3.12", features = ["derive"] } +colored = "2.0.4" diff --git a/pkgr/src/commands/mod.rs b/pkgr/src/commands/mod.rs index 3c52aca..8697d80 100644 --- a/pkgr/src/commands/mod.rs +++ b/pkgr/src/commands/mod.rs @@ -3,7 +3,7 @@ use clap::{Parser, Subcommand}; use log::error; #[derive(Parser, Debug)] -#[clap(name = "pkgr", version = "0.1.0", author = "")] +#[clap(name = "pkgr", version)] pub struct Cli { #[command(subcommand)] pub command: Command, @@ -19,6 +19,9 @@ pub enum Command { }, /// Remove a package from the system Remove { + /// Remove the package from index too + #[arg(short, long, default_value_t = false)] + index: bool, package_identifier: PackageIdentifier, }, /// List packages @@ -46,8 +49,12 @@ impl Command { error!("Install is not yet implemented."); } } - Command::Remove { package_identifier: _ } => { - error!("Remove is not yet implemented."); + Command::Remove { package_identifier: _, index } => { + if *index { + error!("Index removal is not yet implemented."); + } else { + error!("Remove is not yet implemented."); + } } Command::List { installed: _ } => { error!("List is not yet implemented."); diff --git a/pkgr/src/main.rs b/pkgr/src/main.rs index 17e200f..e8c878a 100644 --- a/pkgr/src/main.rs +++ b/pkgr/src/main.rs @@ -1,10 +1,8 @@ use clap::Parser; - - - - -use crate::commands::{Cli}; -use log::{info, SetLoggerError}; +use colored::Colorize; +use crate::commands::Cli; +use log::{debug, info, SetLoggerError, trace}; +use std::env; mod commands; mod package; @@ -12,8 +10,16 @@ mod package; fn main() { setup_logger().expect("Unable to setup logger."); + trace!("Parsing command line arguments..."); let c = Cli::parse(); - info!("Command: {:?}", c); + debug!("Command line arguments parsed."); + + trace!("Executing command..."); + c.command.execute(); + debug!("Command executed."); + + trace!("Exiting..."); + info!("Done."); } fn setup_logger() -> Result<(), SetLoggerError> { @@ -22,13 +28,15 @@ fn setup_logger() -> Result<(), SetLoggerError> { out.finish(format_args!( "{} {}", match record.level().to_string().chars().nth(0).unwrap_or('I') { - 'E' | 'W' => "!!", - _ => "**", + 'T' => "##".cyan(), + 'D' => "::".yellow(), + 'E' | 'W' => "!!".red(), + _ => "**".blue(), }, - message + message.to_string().bright_white() )) }) - .level(log::LevelFilter::Info) + .level(env::var("PKGR_LOG_LEVEL").unwrap_or_else(|_| "info".to_string()).parse().unwrap_or(log::LevelFilter::Info)) .chain(std::io::stdout()) .apply() } From d9d4728dd68a06ff7b2d53398777d457bc05507c Mon Sep 17 00:00:00 2001 From: Didier Date: Mon, 17 Jul 2023 17:47:42 +0200 Subject: [PATCH 024/123] feat: added command functionality to install --- LICENSE | 21 +++++++ bootpkg/src/prelude.rs | 4 +- docs/pkgr/installedmanifest.md | 3 + manifest/src/bin.rs | 2 +- manifest/src/build.rs | 2 +- manifest/src/fs.rs | 2 +- manifest/src/lib.rs | 15 +++-- manifest/src/package.rs | 4 +- manifest/src/pkgr.rs | 11 +++- package.example.toml | 107 ++++++++++++++++++++++++++++++++ package.toml | 38 ------------ pkgfile/src/lib.rs | 16 +++++ pkgr/Cargo.toml | 13 +++- pkgr/src/commands/mod.rs | 68 ++++++++++++++++---- pkgr/src/main.rs | 46 ++++++++++---- pkgr/src/package/builder/mod.rs | 71 +++++++++++++++++++++ pkgr/src/package/fetch.rs | 59 ++++++++++++++++++ pkgr/src/package/identifier.rs | 8 ++- pkgr/src/package/mod.rs | 2 + pkgr/src/tmp.rs | 48 ++++++++++++++ 20 files changed, 464 insertions(+), 76 deletions(-) create mode 100644 LICENSE create mode 100644 package.example.toml create mode 100644 pkgr/src/package/builder/mod.rs create mode 100644 pkgr/src/package/fetch.rs create mode 100644 pkgr/src/tmp.rs diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..5c3335d --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2023 didier + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/bootpkg/src/prelude.rs b/bootpkg/src/prelude.rs index b82b62a..9406cdb 100644 --- a/bootpkg/src/prelude.rs +++ b/bootpkg/src/prelude.rs @@ -5,12 +5,12 @@ use toml; use manifest::{self, Manifest}; -#[derive(Serialize, Deserialize)] +#[derive(Debug, Clone, Serialize, Deserialize)] pub struct PKGR { pub bootstrap: Option, } -#[derive(Serialize, Deserialize)] +#[derive(Debug, Clone, Serialize, Deserialize)] pub struct Bootstrap { pub check_installed_commands: Vec, pub commands: Vec, diff --git a/docs/pkgr/installedmanifest.md b/docs/pkgr/installedmanifest.md index 48f7b7b..04a28ad 100644 --- a/docs/pkgr/installedmanifest.md +++ b/docs/pkgr/installedmanifest.md @@ -11,6 +11,9 @@ To find the latest we'll use the `latest` symlink: install_start = "2021-01-01T00:00:00Z" # install_start is the time the pkgr started installing the package install_end = "2021-01-01T00:00:00Z" # install_end is the time the pkgr finished installing the package install_by = "pkgr" # install_by is the name of the package manager that installed the package +package_identifier = "uri" +package_id = "https://..." + [old_versions] # This logs old versions of the package (that were installed before) 9 = "../9/pkgr.pkg" diff --git a/manifest/src/bin.rs b/manifest/src/bin.rs index e30b528..f55a8e5 100644 --- a/manifest/src/bin.rs +++ b/manifest/src/bin.rs @@ -2,7 +2,7 @@ use std::collections::HashMap; use serde::{Deserialize, Serialize}; -#[derive(Serialize, Deserialize)] +#[derive(Debug, Clone, Serialize, Deserialize)] pub struct Bin { pub root: String, pub checksums: HashMap, diff --git a/manifest/src/build.rs b/manifest/src/build.rs index ac223d2..161b785 100644 --- a/manifest/src/build.rs +++ b/manifest/src/build.rs @@ -2,7 +2,7 @@ use std::collections::HashMap; use serde::{Deserialize, Serialize}; -#[derive(Serialize, Deserialize)] +#[derive(Debug, Clone, Serialize, Deserialize)] pub struct Build { build_script: String, install_script: String, diff --git a/manifest/src/fs.rs b/manifest/src/fs.rs index e02312f..743bc88 100644 --- a/manifest/src/fs.rs +++ b/manifest/src/fs.rs @@ -1,6 +1,6 @@ use serde::{Deserialize, Serialize}; -#[derive(Serialize, Deserialize, Debug)] +#[derive(Debug, Clone, Serialize, Deserialize)] pub struct FS { pub config: Option, pub data: Option, diff --git a/manifest/src/lib.rs b/manifest/src/lib.rs index 8a943fa..0d52840 100644 --- a/manifest/src/lib.rs +++ b/manifest/src/lib.rs @@ -8,9 +8,9 @@ pub mod fs; pub mod package; pub mod pkgr; -#[derive(Serialize, Deserialize)] +#[derive(Debug, Clone, Serialize, Deserialize)] #[serde(default)] -pub struct Manifest

> { +pub struct Manifest> { pub package: package::Package, pub dependencies: HashMap, pub fs: fs::FS, @@ -19,7 +19,7 @@ pub struct Manifest

> { pub pkgr: P, } -impl Manifest { +impl Manifest

{ pub fn valid(&self) -> bool { if self.bin.is_none() && self.build.is_none() { return false; @@ -28,6 +28,13 @@ impl Manifest { } } +impl TryFrom for Manifest { + type Error = toml::de::Error; + fn try_from(s: String) -> Result { + toml::from_str(&s) + } +} + impl FromStr for Manifest { type Err = toml::de::Error; fn from_str(s: &str) -> Result { @@ -36,7 +43,7 @@ impl FromStr for Manifest { } impl Default for Manifest { - fn default() -> Self { + fn default() -> Manifest { Manifest { package: package::Package::default(), dependencies: HashMap::default(), diff --git a/manifest/src/package.rs b/manifest/src/package.rs index 017271d..233206d 100644 --- a/manifest/src/package.rs +++ b/manifest/src/package.rs @@ -2,7 +2,7 @@ use std::str::FromStr; use serde::{Deserialize, Serialize}; -#[derive(Serialize, Deserialize, Debug)] +#[derive(Debug, Clone, Serialize, Deserialize)] pub enum PackageType { #[serde(rename = "application")] Application, @@ -12,7 +12,7 @@ pub enum PackageType { Meta, } -#[derive(Serialize, Deserialize, Debug)] +#[derive(Debug, Clone, Serialize, Deserialize)] #[serde(default)] pub struct Package { pub name: String, diff --git a/manifest/src/pkgr.rs b/manifest/src/pkgr.rs index 8c4ce55..700c6cb 100644 --- a/manifest/src/pkgr.rs +++ b/manifest/src/pkgr.rs @@ -1,4 +1,11 @@ +use std::fmt::{Display, Formatter}; use serde::{Deserialize, Serialize}; -#[derive(Serialize, Deserialize)] -pub struct PKGR {} \ No newline at end of file +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct PKGR {} + +impl Display for PKGR { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + write!(f, "") + } +} \ No newline at end of file diff --git a/package.example.toml b/package.example.toml new file mode 100644 index 0000000..c99108d --- /dev/null +++ b/package.example.toml @@ -0,0 +1,107 @@ +#* = required. + +[package] +name = "packager" #* +description = "A package installation tool" #* + +## package.version +# needs to be an int and be incremented every time +# if you use a custom version system and want to use that, you can use the tag system! +# the highest package.version on the server will be given the tag "latest" +version = 1 # this can automatically be incremented when publishing by running `pkgr publish -i ...` + +## package.tags +# you can add tags to a package to specify kinds +# there are some special tags: +# - latest +# automatically set on the last published version +# previous version will lose this tag +# - oldest +# assigned to the first version. +tags = [ "prod", "pkgr-spec-1" ] + +## package.type +# Supported types: +# - application +# this type specifies that this package is an application. +# - library +# this type specifies that this package is a library. +# - meta +# this type specifies that this package does not install anything upon the system except for dependencies. +type = "application" + +arch = "x86_64" # this is automatically filled by `pkgr publish ...` + +## dependencies +# you may use the following syntax +# "" or ",,..." +[dependencies] +#example = { arch = "x86_64", tags = [ "stable" ], version = "v1" } +exnmbr2 = "v1,stable" + +## fs +# specify some directories for ease of use +[fs] +## fs.config +# specifiy the config path +config = "/etc/packager" +## fs.data +# specify the data path +data = "/var/lib/packager" +## replace "pacakger" with your package name to find the default value. + +## bin +# Used for systems that don't want to build pkgs. +[bin] # binary files root + +## bin.root +# ** RELATIVE TO PACKAGE ROOT ** +# bin.root specifies the root of the installed tree. +# anything in here will be overlayed on top of the system. +root = "/root" #* + +## bin.checksums +# ** KEYS are relative to BIN.ROOT ** +# ** VALUES are relative to PKG ROOT ** +# checksums is used to perform checksums before install +# values may be paths or a uri. You may include variables. +# supported variables: +# - @name +# - @version +# - @display_version +[bin.checksums] +"/usr/bin/pkgr" = "https://ixvd.net/checksums/packager/@version" + +## build +# Scripts will be copied to a custom root. +# After the pacakge is built, the install script is ran. +# After the install script the build directory will be deleted and an CTREE (changed tree) will be generated. +# Then the CTREE will be used to copy the files over. +[build] +build_script = "scripts/build" # relative to pkg +install_script = "scripts/install" # relative to pkg + +[build.dependencies] +base = "latest,stable" # selected by default + +## pkgr.* +# packager is the official client but you may use other clients supporting the "pkgr v1 spec". +# other clients may offer extra functionality this must be put under "pkgr.*" +[pkgr] + +## pkgr.bootstrap +# This section is used for bootpkg. An edition of packager that bootstraps the full version. +# This exists so that packager is easy to install on anything! +# and only 1 release channel for pkgr +[pkgr.bootstrap] +## any non-zero = installed +check_installed_commands = [ + "sh scripts/check_installed" +] + +# any non-zero = fail +commands = [ + "sh scripts/bootstrap/download_latest @version /tmp/pkgr.pkg", + "sh scripts/bootstrap/dirty_install /tmp/pkgr.pkg", + "sh scripts/check_installed" +] diff --git a/package.toml b/package.toml index c99108d..81240ab 100644 --- a/package.toml +++ b/package.toml @@ -1,33 +1,8 @@ -#* = required. - [package] name = "packager" #* description = "A package installation tool" #* - -## package.version -# needs to be an int and be incremented every time -# if you use a custom version system and want to use that, you can use the tag system! -# the highest package.version on the server will be given the tag "latest" version = 1 # this can automatically be incremented when publishing by running `pkgr publish -i ...` - -## package.tags -# you can add tags to a package to specify kinds -# there are some special tags: -# - latest -# automatically set on the last published version -# previous version will lose this tag -# - oldest -# assigned to the first version. tags = [ "prod", "pkgr-spec-1" ] - -## package.type -# Supported types: -# - application -# this type specifies that this package is an application. -# - library -# this type specifies that this package is a library. -# - meta -# this type specifies that this package does not install anything upon the system except for dependencies. type = "application" arch = "x86_64" # this is automatically filled by `pkgr publish ...` @@ -36,19 +11,6 @@ arch = "x86_64" # this is automatically filled by `pkgr publish ...` # you may use the following syntax # "" or ",,..." [dependencies] -#example = { arch = "x86_64", tags = [ "stable" ], version = "v1" } -exnmbr2 = "v1,stable" - -## fs -# specify some directories for ease of use -[fs] -## fs.config -# specifiy the config path -config = "/etc/packager" -## fs.data -# specify the data path -data = "/var/lib/packager" -## replace "pacakger" with your package name to find the default value. ## bin # Used for systems that don't want to build pkgs. diff --git a/pkgfile/src/lib.rs b/pkgfile/src/lib.rs index bc26f99..15249e7 100644 --- a/pkgfile/src/lib.rs +++ b/pkgfile/src/lib.rs @@ -1,8 +1,24 @@ +#[derive(Debug)] pub struct PKGFile { pub manifest: String, pub data: Vec, } +impl PKGFile { + pub fn new(manifest: String, data: Vec) -> PKGFile { + PKGFile { manifest, data } + } +} + +impl Default for PKGFile { + fn default() -> PKGFile { + PKGFile { + manifest: String::new(), + data: Vec::new(), + } + } +} + impl TryFrom> for PKGFile { type Error = (); diff --git a/pkgr/Cargo.toml b/pkgr/Cargo.toml index 6e94865..cc6e55b 100644 --- a/pkgr/Cargo.toml +++ b/pkgr/Cargo.toml @@ -1,14 +1,25 @@ [package] name = "pkgr" +description = "A package manager and build tool." version = "0.1.0" edition = "2021" +authors = [ + "Didier " +] +license-file = "../LICENSE" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] +manifest = { path = "../manifest" } +pkgfile = { path = "../pkgfile" } fern = "0.6.2" log = "0.4.19" regex = "1.9.1" -manifest = { path = "../manifest" } clap = { version = "4.3.12", features = ["derive"] } colored = "2.0.4" +toml = "0.7.6" +serde = { version = "1.0.171", features = ["derive"] } +libc = "0.2.80" +reqwest = { version = "0.11.18", features = ["blocking"] } +tar = "0.4.39" diff --git a/pkgr/src/commands/mod.rs b/pkgr/src/commands/mod.rs index 8697d80..bcbef89 100644 --- a/pkgr/src/commands/mod.rs +++ b/pkgr/src/commands/mod.rs @@ -1,6 +1,10 @@ +use std::process::exit; use crate::package::identifier::PackageIdentifier; use clap::{Parser, Subcommand}; -use log::error; +use log::{debug, error, info, trace, warn}; +use manifest::Manifest; +use crate::package::builder::{InstallType, PackageInstaller}; +use crate::package::fetch::fetch_package; #[derive(Parser, Debug)] #[clap(name = "pkgr", version)] @@ -33,6 +37,8 @@ pub enum Command { /// Update packages on the system Update, #[command(hide = true)] + Debug, + #[command(hide = true)] None, } @@ -41,20 +47,47 @@ impl Command { match self { Command::Install { build, - package_identifier: _, + package_identifier, } => { - if *build { - error!("Build is not yet implemented."); - } else { - error!("Install is not yet implemented."); + warn!("Installer does not run in isolation."); + + let start = std::time::Instant::now(); + let unix_start = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap(); + + trace!("Fetching package: {}", package_identifier); + let pkgfile = fetch_package(package_identifier.clone()).unwrap(); + debug!("size: manifest({}kb) data({}kb)", pkgfile.manifest.len() / 1024, pkgfile.data.len() / 1024); + trace!("parsing manifest"); + let manifest = Manifest::try_from(pkgfile.manifest.clone()).unwrap(); + debug!("manifest pkg name: {}", manifest.package.name); + trace!("creating installer"); + let installer = PackageInstaller::new(manifest.clone(), pkgfile, if *build { InstallType::Build } else { InstallType::Bin }); + + trace!("starting install"); + match installer.install() { + Ok(_) => { + info!("Sucessfully installed: {}", &manifest.package.name); + () + }, + Err(e) => { + error!("{}", e.to_string()) + } } + let unix_end = std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).unwrap(); + let end = std::time::Instant::now(); + // float ms + let duration = (end - start).as_nanos() as f64; + info!("Install took: {}ms", (duration / 1_000_000.0)); } - Command::Remove { package_identifier: _, index } => { - if *index { - error!("Index removal is not yet implemented."); - } else { - error!("Remove is not yet implemented."); + Command::Remove { package_identifier, index } => { + if let PackageIdentifier::URI(_) = package_identifier { + error!("URI is unsupported when removing applications."); + exit(1); } + info!("Index: {}", index); + info!("Package identifier: {}", package_identifier); } Command::List { installed: _ } => { error!("List is not yet implemented."); @@ -62,6 +95,19 @@ impl Command { Command::Update => { error!("Update is not yet implemented."); } + Command::Debug => { + trace!("Trace message."); + debug!("Debug message."); + info!("Info message."); + warn!("Warning message."); + error!("Error message."); + info!(""); + info!("PKGR VERSION: {}", env!("CARGO_PKG_VERSION")); + info!("PKGR AUTHORS: {}", env!("CARGO_PKG_AUTHORS")); + info!("PKGR DESCRIPTION: {}", env!("CARGO_PKG_DESCRIPTION")); + info!(""); + info!("PKGR_LOG_LEVEL: {}", std::env::var("PKGR_LOG_LEVEL").unwrap_or_else(|_| "info".to_string())); + } Command::None => { error!("No command was specified."); } diff --git a/pkgr/src/main.rs b/pkgr/src/main.rs index e8c878a..ba5f97d 100644 --- a/pkgr/src/main.rs +++ b/pkgr/src/main.rs @@ -1,25 +1,35 @@ use clap::Parser; use colored::Colorize; use crate::commands::Cli; -use log::{debug, info, SetLoggerError, trace}; +use log::{SetLoggerError, trace}; use std::env; mod commands; mod package; +mod tmp; fn main() { setup_logger().expect("Unable to setup logger."); + #[cfg(not(debug_assertions))] + { + #[cfg(target_family = "unix")] + if unsafe { libc::getuid() } != 0 { + use log::error; + error!("pkgr must be run as root."); + std::process::exit(1); + } + } + trace!("Parsing command line arguments..."); let c = Cli::parse(); - debug!("Command line arguments parsed."); + trace!("Command line arguments: {:?}", c); trace!("Executing command..."); c.command.execute(); - debug!("Command executed."); + trace!("Command executed."); trace!("Exiting..."); - info!("Done."); } fn setup_logger() -> Result<(), SetLoggerError> { @@ -27,16 +37,30 @@ fn setup_logger() -> Result<(), SetLoggerError> { .format(|out, message, record| { out.finish(format_args!( "{} {}", - match record.level().to_string().chars().nth(0).unwrap_or('I') { - 'T' => "##".cyan(), - 'D' => "::".yellow(), - 'E' | 'W' => "!!".red(), - _ => "**".blue(), - }, + // Some logic so messages look nice + if message.to_string().len() > 0 { + match record + .level() + .to_string() + .chars() + .nth(0) + .unwrap_or('T') + { + 'T' => "[TRACE]".cyan(), + 'D' => "??".green(), + 'I' => "=>".blue(), + 'W' => "##".yellow(), + 'E' => "!!".red(), + _ => "**".blue(), + }.to_string() + } else { "".to_string() }, message.to_string().bright_white() )) }) - .level(env::var("PKGR_LOG_LEVEL").unwrap_or_else(|_| "info".to_string()).parse().unwrap_or(log::LevelFilter::Info)) + .level(env::var("PKGR_LOG_LEVEL") + .unwrap_or_else(|_| "info".to_string()) + .parse() + .unwrap_or(log::LevelFilter::Info)) .chain(std::io::stdout()) .apply() } diff --git a/pkgr/src/package/builder/mod.rs b/pkgr/src/package/builder/mod.rs new file mode 100644 index 0000000..50166e1 --- /dev/null +++ b/pkgr/src/package/builder/mod.rs @@ -0,0 +1,71 @@ +use libc::fork; +use log::{debug, trace}; +use manifest::Manifest; +use pkgfile::PKGFile; +use crate::tmp::TempDir; + +#[derive(Debug)] +pub enum InstallType { + Build, + Bin +} + +#[derive(Debug)] +pub enum InstallError { + BuildError(String), + BinError(String), + InstallError +} + +impl ToString for InstallError { + fn to_string(&self) -> String { + match self { + InstallError::BuildError(e) => format!("Build error: {}", e), + InstallError::BinError(e) => format!("Bin error: {}", e), + InstallError::InstallError => "Install error".to_string(), + } + } +} + +#[derive(Debug)] +pub struct PackageInstaller { + manifest: Manifest, + pkgfile: PKGFile, + install_type: InstallType +} + +impl PackageInstaller { + pub fn new(m: Manifest, p: PKGFile, i: InstallType) -> PackageInstaller { + PackageInstaller { + manifest: m, + pkgfile: p, + install_type: i + } + } + + fn extract_to>(&self, path: S) -> Result<(), String> { + tar::Archive::new(self.pkgfile.data.as_slice()) + .unpack(path.into()) + .map_err(|e| e.to_string()) + } + + fn bin(&self) -> Result<(), String> { + let mut tmpdir = TempDir::default(); + tmpdir.push(&self.manifest.package.name); + trace!("extracting package into: {}", tmpdir.to_string()); + self.extract_to(tmpdir.to_string())?; + debug!("extracted package in: {}", tmpdir.to_string()); + Ok(()) + } + + fn build(&self) -> Result<(), String> { + Ok(()) + } + + pub fn install(&self) -> Result<(), InstallError> { + match self.install_type { + InstallType::Bin => self.bin().map_err(|e| InstallError::BinError(e)), + InstallType::Build => self.build().map_err(|e| InstallError::BuildError(e)) + } + } +} \ No newline at end of file diff --git a/pkgr/src/package/fetch.rs b/pkgr/src/package/fetch.rs new file mode 100644 index 0000000..0e68045 --- /dev/null +++ b/pkgr/src/package/fetch.rs @@ -0,0 +1,59 @@ +use std::error::Error; +use std::fmt::Display; +use std::fs::File; +use std::io::Read; +use log::warn; +use pkgfile::PKGFile; +use crate::package::identifier::PackageIdentifier; +use reqwest::blocking::get; + +#[derive(Debug)] +pub enum FetchError { + HTTPError(reqwest::Error), + IOError(std::io::Error), + ParseError, +} + +impl Display for FetchError { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + match self { + FetchError::HTTPError(e) => write!(f, "HTTP Error: {}", e), + FetchError::IOError(e) => write!(f, "IO Error: {}", e), + FetchError::ParseError => write!(f, "Parse Error"), + } + } +} + +impl Error for FetchError {} + +pub fn fetch_package(package_identifier: PackageIdentifier) -> Result { + match package_identifier { + PackageIdentifier::Path(path) => { + std::fs::read(path) + .map_err(|e| FetchError::IOError(e)).and_then(|bytes| { + PKGFile::try_from(bytes).map_err(|_| FetchError::ParseError) + }) + } + PackageIdentifier::URI(uri) => { + // get file contents as bytes + let mut bytes = Vec::new(); + match get(&uri) { + Ok(mut response) => { + match response.take(1024 * 1024).read_to_end(&mut bytes) { + Ok(_) => (), + Err(e) => return Err(FetchError::IOError(e)), + }; + } + Err(e) => return Err(FetchError::HTTPError(e)), + }; + // parse bytes as PKGFile + match PKGFile::try_from(bytes) { + Ok(pkgfile) => Ok(pkgfile), + Err(e) => Err(FetchError::ParseError), + } + } + PackageIdentifier::PackageLocator(package_locator) => { + Ok(PKGFile::default()) + } + } +} \ No newline at end of file diff --git a/pkgr/src/package/identifier.rs b/pkgr/src/package/identifier.rs index 07b2bcc..fa457d8 100644 --- a/pkgr/src/package/identifier.rs +++ b/pkgr/src/package/identifier.rs @@ -26,6 +26,7 @@ impl Error for PackageIdentifierError {} pub enum PackageIdentifier { PackageLocator(PackageLocator), URI(String), + Path(String), } impl std::fmt::Display for PackageIdentifier { @@ -33,6 +34,7 @@ impl std::fmt::Display for PackageIdentifier { match self { PackageIdentifier::PackageLocator(pl) => write!(f, "{}", pl), PackageIdentifier::URI(uri) => write!(f, "{}", uri), + PackageIdentifier::Path(path) => write!(f, "{}", path), } } } @@ -61,7 +63,7 @@ impl FromStr for PackageIdentifier { type Err = PackageIdentifierError; fn from_str(s: &str) -> Result { - let uri_re = regex::Regex::new(r"^[a-zA-Z0-9]+://").unwrap(); + let uri_re = Regex::new(r"^[a-zA-Z0-9]+://").unwrap(); if uri_re.is_match(s) { // there needs to be stuff after the protocol let split = s.split("://").collect::>(); @@ -69,6 +71,8 @@ impl FromStr for PackageIdentifier { return Err(PackageIdentifierError::InvalidURI(s.to_string())); } Ok(PackageIdentifier::URI(s.to_string())) + } else if std::path::Path::new(s).exists() { + return Ok(PackageIdentifier::Path(s.to_string())); } else { let pl = match PackageLocator::from_str(s) { Ok(pl) => pl, @@ -84,7 +88,7 @@ impl FromStr for PackageLocator { fn from_str(s: &str) -> Result { #[allow(unused_assignments)] // false positive - let mut name = None; + let mut name = None; let mut version = None; let mut tags = None; diff --git a/pkgr/src/package/mod.rs b/pkgr/src/package/mod.rs index 34cdf0a..5c559fc 100644 --- a/pkgr/src/package/mod.rs +++ b/pkgr/src/package/mod.rs @@ -1 +1,3 @@ pub mod identifier; +pub mod builder; +pub mod fetch; \ No newline at end of file diff --git a/pkgr/src/tmp.rs b/pkgr/src/tmp.rs new file mode 100644 index 0000000..509d3f7 --- /dev/null +++ b/pkgr/src/tmp.rs @@ -0,0 +1,48 @@ +use std::path::PathBuf; + +pub struct TempDir { + path: PathBuf, +} + +impl TempDir { + pub fn new(path: PathBuf) -> TempDir { + if !path.exists() { + std::fs::create_dir_all(&path).unwrap(); + } + TempDir { + path + } + } + + pub fn push>(&mut self, path: S) { + self.path.push(path.into()); + } +} + +impl Default for TempDir { + fn default() -> TempDir { + TempDir::new({ + let mut t = std::env::temp_dir(); + t.push("pkgr"); + t + }) + } +} + +impl Into for TempDir { + fn into(self) -> PathBuf { + self.path + } +} + +impl Into for TempDir { + fn into(self) -> String { + self.path.to_str().unwrap().to_string() + } +} + +impl ToString for TempDir { + fn to_string(&self) -> String { + self.path.to_str().unwrap().to_string() + } +} \ No newline at end of file From c3db07dc64a02a571793cd17a1583a4ed0bafdf0 Mon Sep 17 00:00:00 2001 From: Didier Date: Mon, 17 Jul 2023 18:02:10 +0200 Subject: [PATCH 025/123] feat: start on install section --- pkgfile/src/lib.rs | 4 ++ pkgr/src/{commands/mod.rs => commands.rs} | 3 +- pkgr/src/package/builder/errors.rs | 42 +++++++++++++++++++++ pkgr/src/package/builder/mod.rs | 45 ++++++++++------------- 4 files changed, 68 insertions(+), 26 deletions(-) rename pkgr/src/{commands/mod.rs => commands.rs} (98%) create mode 100644 pkgr/src/package/builder/errors.rs diff --git a/pkgfile/src/lib.rs b/pkgfile/src/lib.rs index 15249e7..f7768b0 100644 --- a/pkgfile/src/lib.rs +++ b/pkgfile/src/lib.rs @@ -8,6 +8,10 @@ impl PKGFile { pub fn new(manifest: String, data: Vec) -> PKGFile { PKGFile { manifest, data } } + + pub fn has_data(&self) -> bool { + !self.data.is_empty() + } } impl Default for PKGFile { diff --git a/pkgr/src/commands/mod.rs b/pkgr/src/commands.rs similarity index 98% rename from pkgr/src/commands/mod.rs rename to pkgr/src/commands.rs index bcbef89..e0f7ec0 100644 --- a/pkgr/src/commands/mod.rs +++ b/pkgr/src/commands.rs @@ -72,7 +72,8 @@ impl Command { () }, Err(e) => { - error!("{}", e.to_string()) + error!("{}", e.to_string()); + exit(1); } } let unix_end = std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).unwrap(); diff --git a/pkgr/src/package/builder/errors.rs b/pkgr/src/package/builder/errors.rs new file mode 100644 index 0000000..e5f62d6 --- /dev/null +++ b/pkgr/src/package/builder/errors.rs @@ -0,0 +1,42 @@ +use std::fmt::Display; + +#[derive(Debug)] +pub enum BinError { + UnpackError(String), +} + +impl Display for BinError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + BinError::UnpackError(e) => write!(f, "Unpack error: {}", e), + } + } +} + +#[derive(Debug)] +pub enum BuildError {} + +impl Display for BuildError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + _ => write!(f, "Build error"), + } + } +} + +#[derive(Debug)] +pub enum InstallError { + BuildError(BuildError), + BinError(BinError), + InstallError +} + +impl ToString for InstallError { + fn to_string(&self) -> String { + match self { + InstallError::BuildError(e) => format!("Build error: {}", e), + InstallError::BinError(e) => format!("Bin error: {}", e), + InstallError::InstallError => "Install error".to_string(), + } + } +} diff --git a/pkgr/src/package/builder/mod.rs b/pkgr/src/package/builder/mod.rs index 50166e1..b841a78 100644 --- a/pkgr/src/package/builder/mod.rs +++ b/pkgr/src/package/builder/mod.rs @@ -1,32 +1,18 @@ -use libc::fork; +use std::fmt::Display; use log::{debug, trace}; +use errors::{BinError, BuildError, InstallError}; use manifest::Manifest; use pkgfile::PKGFile; use crate::tmp::TempDir; +mod errors; + #[derive(Debug)] pub enum InstallType { Build, Bin } -#[derive(Debug)] -pub enum InstallError { - BuildError(String), - BinError(String), - InstallError -} - -impl ToString for InstallError { - fn to_string(&self) -> String { - match self { - InstallError::BuildError(e) => format!("Build error: {}", e), - InstallError::BinError(e) => format!("Bin error: {}", e), - InstallError::InstallError => "Install error".to_string(), - } - } -} - #[derive(Debug)] pub struct PackageInstaller { manifest: Manifest, @@ -43,22 +29,31 @@ impl PackageInstaller { } } - fn extract_to>(&self, path: S) -> Result<(), String> { + fn extract_to(&self, path: String) -> Result<(), BinError> { + if !self.pkgfile.has_data() { + return Err(BinError::UnpackError("package has no data".to_string())); + } + if std::path::Path::new(&path).exists() { + return Err(BinError::UnpackError("path already exists".to_string())); + } tar::Archive::new(self.pkgfile.data.as_slice()) - .unpack(path.into()) - .map_err(|e| e.to_string()) + .unpack(&path) + .map_err(|e| BinError::UnpackError(e.to_string())) } - fn bin(&self) -> Result<(), String> { + fn bin(&self) -> Result<(), BinError> { let mut tmpdir = TempDir::default(); tmpdir.push(&self.manifest.package.name); trace!("extracting package into: {}", tmpdir.to_string()); - self.extract_to(tmpdir.to_string())?; + match self.extract_to(tmpdir.to_string()) { + Ok(_) => {}, + Err(e) => return Err(e) + } debug!("extracted package in: {}", tmpdir.to_string()); Ok(()) } - fn build(&self) -> Result<(), String> { + fn build(&self) -> Result<(), BuildError> { Ok(()) } @@ -68,4 +63,4 @@ impl PackageInstaller { InstallType::Build => self.build().map_err(|e| InstallError::BuildError(e)) } } -} \ No newline at end of file +} From 7a5681647204a91d7616164a3790342631d2a575 Mon Sep 17 00:00:00 2001 From: Didier Date: Mon, 17 Jul 2023 20:40:01 +0200 Subject: [PATCH 026/123] refactor: change name from builder to installer --- pkgr/src/commands.rs | 4 +- pkgr/src/logging.rs | 57 +++++++++++++++++++ pkgr/src/main.rs | 44 +++----------- .../package/{builder => installer}/errors.rs | 0 .../src/package/{builder => installer}/mod.rs | 4 +- pkgr/src/package/mod.rs | 2 +- pkgr/src/{tmp.rs => tmpfs.rs} | 0 7 files changed, 70 insertions(+), 41 deletions(-) create mode 100644 pkgr/src/logging.rs rename pkgr/src/package/{builder => installer}/errors.rs (100%) rename pkgr/src/package/{builder => installer}/mod.rs (97%) rename pkgr/src/{tmp.rs => tmpfs.rs} (100%) diff --git a/pkgr/src/commands.rs b/pkgr/src/commands.rs index e0f7ec0..cc3dfcf 100644 --- a/pkgr/src/commands.rs +++ b/pkgr/src/commands.rs @@ -3,7 +3,7 @@ use crate::package::identifier::PackageIdentifier; use clap::{Parser, Subcommand}; use log::{debug, error, info, trace, warn}; use manifest::Manifest; -use crate::package::builder::{InstallType, PackageInstaller}; +use crate::package::installer::{InstallType, PackageInstaller}; use crate::package::fetch::fetch_package; #[derive(Parser, Debug)] @@ -56,6 +56,7 @@ impl Command { .duration_since(std::time::UNIX_EPOCH) .unwrap(); + info!("Parsing package..."); trace!("Fetching package: {}", package_identifier); let pkgfile = fetch_package(package_identifier.clone()).unwrap(); debug!("size: manifest({}kb) data({}kb)", pkgfile.manifest.len() / 1024, pkgfile.data.len() / 1024); @@ -64,6 +65,7 @@ impl Command { debug!("manifest pkg name: {}", manifest.package.name); trace!("creating installer"); let installer = PackageInstaller::new(manifest.clone(), pkgfile, if *build { InstallType::Build } else { InstallType::Bin }); + info!("Installing: {}...", &manifest.package.name); trace!("starting install"); match installer.install() { diff --git a/pkgr/src/logging.rs b/pkgr/src/logging.rs new file mode 100644 index 0000000..6ee98a5 --- /dev/null +++ b/pkgr/src/logging.rs @@ -0,0 +1,57 @@ +use log::SetLoggerError; +use fern::Dispatch; +use std::env; +use colored::Colorize; + +pub fn setup_logger() -> Result<(), SetLoggerError> { + Dispatch::new() + .format(|out, message, record| { + match record.metadata().target() { + "command:stdout" => { + out.finish(format_args!( + "{} {}", + "::".cyan(), + message.to_string() + )); + return; + } + "command:stderr" => { + out.finish(format_args!( + "{} {}", + "!!".red(), + message.to_string() + )); + return; + } + _ => { + out.finish(format_args!( + "{} {}", + // Some logic so messages look nice + if message.to_string().len() > 0 { + match record + .level() + .to_string() + .chars() + .nth(0) + .unwrap_or('T') + { + 'T' => "[TRACE]".bright_blue(), + 'D' => "??".green(), + 'I' => "=>".blue(), + 'W' => "##".yellow(), + 'E' => "!!".red(), + _ => "**".blue(), + }.to_string() + } else { "".to_string() }, + message.to_string().bright_white() + )) + } + } + }) + .level(env::var("PKGR_LOG_LEVEL") + .unwrap_or_else(|_| "info".to_string()) + .parse() + .unwrap_or(log::LevelFilter::Info)) + .chain(std::io::stdout()) + .apply() +} diff --git a/pkgr/src/main.rs b/pkgr/src/main.rs index ba5f97d..aaa5387 100644 --- a/pkgr/src/main.rs +++ b/pkgr/src/main.rs @@ -1,15 +1,18 @@ +use std::process::Command; use clap::Parser; use colored::Colorize; use crate::commands::Cli; -use log::{SetLoggerError, trace}; -use std::env; +use log::trace; +use crate::process::Process; +mod process; mod commands; mod package; -mod tmp; +mod tmpfs; +mod logging; fn main() { - setup_logger().expect("Unable to setup logger."); + logging::setup_logger().expect("Unable to setup logger."); #[cfg(not(debug_assertions))] { @@ -31,36 +34,3 @@ fn main() { trace!("Exiting..."); } - -fn setup_logger() -> Result<(), SetLoggerError> { - fern::Dispatch::new() - .format(|out, message, record| { - out.finish(format_args!( - "{} {}", - // Some logic so messages look nice - if message.to_string().len() > 0 { - match record - .level() - .to_string() - .chars() - .nth(0) - .unwrap_or('T') - { - 'T' => "[TRACE]".cyan(), - 'D' => "??".green(), - 'I' => "=>".blue(), - 'W' => "##".yellow(), - 'E' => "!!".red(), - _ => "**".blue(), - }.to_string() - } else { "".to_string() }, - message.to_string().bright_white() - )) - }) - .level(env::var("PKGR_LOG_LEVEL") - .unwrap_or_else(|_| "info".to_string()) - .parse() - .unwrap_or(log::LevelFilter::Info)) - .chain(std::io::stdout()) - .apply() -} diff --git a/pkgr/src/package/builder/errors.rs b/pkgr/src/package/installer/errors.rs similarity index 100% rename from pkgr/src/package/builder/errors.rs rename to pkgr/src/package/installer/errors.rs diff --git a/pkgr/src/package/builder/mod.rs b/pkgr/src/package/installer/mod.rs similarity index 97% rename from pkgr/src/package/builder/mod.rs rename to pkgr/src/package/installer/mod.rs index b841a78..53602c6 100644 --- a/pkgr/src/package/builder/mod.rs +++ b/pkgr/src/package/installer/mod.rs @@ -3,9 +3,9 @@ use log::{debug, trace}; use errors::{BinError, BuildError, InstallError}; use manifest::Manifest; use pkgfile::PKGFile; -use crate::tmp::TempDir; +use crate::tmpfs::TempDir; -mod errors; +pub mod errors; #[derive(Debug)] pub enum InstallType { diff --git a/pkgr/src/package/mod.rs b/pkgr/src/package/mod.rs index 5c559fc..f838f7e 100644 --- a/pkgr/src/package/mod.rs +++ b/pkgr/src/package/mod.rs @@ -1,3 +1,3 @@ pub mod identifier; -pub mod builder; +pub mod installer; pub mod fetch; \ No newline at end of file diff --git a/pkgr/src/tmp.rs b/pkgr/src/tmpfs.rs similarity index 100% rename from pkgr/src/tmp.rs rename to pkgr/src/tmpfs.rs From b83d44768d4e1673371ecf9a9b14bd76b91b6b2c Mon Sep 17 00:00:00 2001 From: Didier Date: Mon, 17 Jul 2023 20:40:22 +0200 Subject: [PATCH 027/123] feat: Process with multithreaded log pipes --- pkgr/src/process/mod.rs | 44 ++++++++++++++++++++++++++++++++++++++ pkgr/src/process/output.rs | 26 ++++++++++++++++++++++ 2 files changed, 70 insertions(+) create mode 100644 pkgr/src/process/mod.rs create mode 100644 pkgr/src/process/output.rs diff --git a/pkgr/src/process/mod.rs b/pkgr/src/process/mod.rs new file mode 100644 index 0000000..70ab93b --- /dev/null +++ b/pkgr/src/process/mod.rs @@ -0,0 +1,44 @@ +use std::process::Command; + +pub mod output; + +pub struct Process { + pub command: Vec, + pub cwd: Option, + pub env: Vec, +} + +impl Process { + pub fn new(command: Vec, cwd: Option, env: Vec) -> Process { + Process { + command, + cwd, + env, + } + } + + pub fn command>(cmd: Vec) -> Process { + Process::new(cmd.into_iter().map(|v| v.into()).collect(), None, vec![]) + } + + pub fn spawn(&self) -> std::io::Result { + let mut child = Command::new(&self.command[0]) + .args(&self.command[1..]) + .current_dir(self.cwd.clone().unwrap_or(".".to_string())) + .envs(self.env.iter().map(|v| { + let mut split = v.split('='); + (split.next().unwrap().to_string(), split.next().unwrap().to_string()) + })) + .stdout(std::process::Stdio::piped()) + .stderr(std::process::Stdio::piped()) + .spawn()?; + output::spawn_output_handlers(&mut child); + Ok(child) + } +} + +impl Default for Process { + fn default() -> Process { + Process::new(vec![], None, vec![]) + } +} \ No newline at end of file diff --git a/pkgr/src/process/output.rs b/pkgr/src/process/output.rs new file mode 100644 index 0000000..09e788e --- /dev/null +++ b/pkgr/src/process/output.rs @@ -0,0 +1,26 @@ +// create functions that spawn threads for Stdout and Stderr +// that print the output to info!(target: "command:stdout", ...) and info!(target: "command:stderr", ...) respectively + +use std::process::Child; +use std::io::{BufRead, BufReader}; + +use log::{info, error}; + +pub fn spawn_output_handlers(child: &mut Child) { + let stdout = child.stdout.take().unwrap(); + let stderr = child.stderr.take().unwrap(); + + std::thread::spawn(move || { + let reader = BufReader::new(stdout); + for line in reader.lines() { + info!(target: "command:stdout", "{}", line.unwrap()); + } + }); + + std::thread::spawn(move || { + let reader = BufReader::new(stderr); + for line in reader.lines() { + error!(target: "command:stderr", "{}", line.unwrap()); + } + }); +} \ No newline at end of file From 5fea156cfa38b3f20fab7c2a2e9576fa0b4ed706 Mon Sep 17 00:00:00 2001 From: Didier Date: Mon, 17 Jul 2023 21:15:30 +0200 Subject: [PATCH 028/123] refactor: changed how packages work --- manifest/src/build.rs | 6 +-- pkgfile/src/lib.rs | 2 +- pkgr/src/commands.rs | 44 +++++++++--------- pkgr/src/package/fetch.rs | 59 ++++++++++++------------ pkgr/src/package/installer/errors.rs | 8 ++-- pkgr/src/package/installer/mod.rs | 7 +++ pkgr/src/package/mod.rs | 67 +++++++++++++++++++++++++++- 7 files changed, 134 insertions(+), 59 deletions(-) diff --git a/manifest/src/build.rs b/manifest/src/build.rs index 161b785..1879710 100644 --- a/manifest/src/build.rs +++ b/manifest/src/build.rs @@ -4,7 +4,7 @@ use serde::{Deserialize, Serialize}; #[derive(Debug, Clone, Serialize, Deserialize)] pub struct Build { - build_script: String, - install_script: String, - dependencies: HashMap, + pub build_script: String, + pub install_script: String, + pub dependencies: HashMap, } diff --git a/pkgfile/src/lib.rs b/pkgfile/src/lib.rs index f7768b0..407055c 100644 --- a/pkgfile/src/lib.rs +++ b/pkgfile/src/lib.rs @@ -1,4 +1,4 @@ -#[derive(Debug)] +#[derive(Debug, Clone)] pub struct PKGFile { pub manifest: String, pub data: Vec, diff --git a/pkgr/src/commands.rs b/pkgr/src/commands.rs index cc3dfcf..2265330 100644 --- a/pkgr/src/commands.rs +++ b/pkgr/src/commands.rs @@ -4,7 +4,7 @@ use clap::{Parser, Subcommand}; use log::{debug, error, info, trace, warn}; use manifest::Manifest; use crate::package::installer::{InstallType, PackageInstaller}; -use crate::package::fetch::fetch_package; +use crate::package::Package; #[derive(Parser, Debug)] #[clap(name = "pkgr", version)] @@ -58,31 +58,35 @@ impl Command { info!("Parsing package..."); trace!("Fetching package: {}", package_identifier); - let pkgfile = fetch_package(package_identifier.clone()).unwrap(); - debug!("size: manifest({}kb) data({}kb)", pkgfile.manifest.len() / 1024, pkgfile.data.len() / 1024); - trace!("parsing manifest"); - let manifest = Manifest::try_from(pkgfile.manifest.clone()).unwrap(); - debug!("manifest pkg name: {}", manifest.package.name); - trace!("creating installer"); - let installer = PackageInstaller::new(manifest.clone(), pkgfile, if *build { InstallType::Build } else { InstallType::Bin }); - info!("Installing: {}...", &manifest.package.name); + let mut pkg = Package::fetch(package_identifier.clone()).unwrap(); + debug!("manifest size: {}kb", pkg.pkgfile.manifest.len() / 1024); + debug!("files size: {}kb", pkg.pkgfile.data.len() / 1024); - trace!("starting install"); - match installer.install() { - Ok(_) => { - info!("Sucessfully installed: {}", &manifest.package.name); - () - }, + trace!("Checking if package is installed..."); + if pkg.is_installed() { + error!("Package is already installed."); + exit(1); + } + + trace!("Starting install..."); + match pkg.install() { + Ok(_) => (), Err(e) => { - error!("{}", e.to_string()); + error!("Install failed: {}", e.to_string()); exit(1); } } - let unix_end = std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).unwrap(); + let end = std::time::Instant::now(); - // float ms - let duration = (end - start).as_nanos() as f64; - info!("Install took: {}ms", (duration / 1_000_000.0)); + let unix_end = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap(); + + let duration = end.duration_since(start); + + info!("Install complete."); + info!("Install took {}ms.", duration.as_nanos() as f64 / 1000000.0); + } Command::Remove { package_identifier, index } => { if let PackageIdentifier::URI(_) = package_identifier { diff --git a/pkgr/src/package/fetch.rs b/pkgr/src/package/fetch.rs index 0e68045..099c5b2 100644 --- a/pkgr/src/package/fetch.rs +++ b/pkgr/src/package/fetch.rs @@ -1,11 +1,9 @@ use std::error::Error; use std::fmt::Display; -use std::fs::File; use std::io::Read; -use log::warn; use pkgfile::PKGFile; -use crate::package::identifier::PackageIdentifier; use reqwest::blocking::get; +use crate::package::identifier::PackageLocator; #[derive(Debug)] pub enum FetchError { @@ -26,34 +24,33 @@ impl Display for FetchError { impl Error for FetchError {} -pub fn fetch_package(package_identifier: PackageIdentifier) -> Result { - match package_identifier { - PackageIdentifier::Path(path) => { - std::fs::read(path) - .map_err(|e| FetchError::IOError(e)).and_then(|bytes| { - PKGFile::try_from(bytes).map_err(|_| FetchError::ParseError) - }) - } - PackageIdentifier::URI(uri) => { - // get file contents as bytes - let mut bytes = Vec::new(); - match get(&uri) { - Ok(mut response) => { - match response.take(1024 * 1024).read_to_end(&mut bytes) { - Ok(_) => (), - Err(e) => return Err(FetchError::IOError(e)), - }; - } - Err(e) => return Err(FetchError::HTTPError(e)), +pub fn fetch_by_path>(path: S) -> Result { + std::fs::read(path.into()) + .map_err(|e| FetchError::IOError(e)).and_then(|bytes| { + PKGFile::try_from(bytes).map_err(|_| FetchError::ParseError) + }) +} + +pub fn fetch_by_uri>(uri: S) -> Result { + // get file contents as bytes + let mut bytes = Vec::new(); + match get(uri.into()) { + Ok(mut response) => { + match response.take(1024 * 1024).read_to_end(&mut bytes) { + Ok(_) => (), + Err(e) => return Err(FetchError::IOError(e)), }; - // parse bytes as PKGFile - match PKGFile::try_from(bytes) { - Ok(pkgfile) => Ok(pkgfile), - Err(e) => Err(FetchError::ParseError), - } - } - PackageIdentifier::PackageLocator(package_locator) => { - Ok(PKGFile::default()) } + Err(e) => return Err(FetchError::HTTPError(e)), + }; + // parse bytes as PKGFile + match PKGFile::try_from(bytes) { + Ok(pkgfile) => Ok(pkgfile), + Err(e) => Err(FetchError::ParseError), } -} \ No newline at end of file +} + +pub fn fetch_by_package_locator(package_locator: PackageLocator) -> Result { + // TODO: search index for package locator + Ok(PKGFile::default()) +} diff --git a/pkgr/src/package/installer/errors.rs b/pkgr/src/package/installer/errors.rs index e5f62d6..a6801c0 100644 --- a/pkgr/src/package/installer/errors.rs +++ b/pkgr/src/package/installer/errors.rs @@ -14,7 +14,9 @@ impl Display for BinError { } #[derive(Debug)] -pub enum BuildError {} +pub enum BuildError { + InvalidManifest +} impl Display for BuildError { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { @@ -28,7 +30,7 @@ impl Display for BuildError { pub enum InstallError { BuildError(BuildError), BinError(BinError), - InstallError + Generic } impl ToString for InstallError { @@ -36,7 +38,7 @@ impl ToString for InstallError { match self { InstallError::BuildError(e) => format!("Build error: {}", e), InstallError::BinError(e) => format!("Bin error: {}", e), - InstallError::InstallError => "Install error".to_string(), + InstallError::Generic => "Install error".to_string(), } } } diff --git a/pkgr/src/package/installer/mod.rs b/pkgr/src/package/installer/mod.rs index 53602c6..8f75941 100644 --- a/pkgr/src/package/installer/mod.rs +++ b/pkgr/src/package/installer/mod.rs @@ -54,6 +54,13 @@ impl PackageInstaller { } fn build(&self) -> Result<(), BuildError> { + if let None = self.manifest.build.clone() { + return Err(BuildError::InvalidManifest); + } + let build_manifest = self.manifest.build.clone().unwrap(); + // TODO: Check dependencies + + Ok(()) } diff --git a/pkgr/src/package/mod.rs b/pkgr/src/package/mod.rs index f838f7e..349c360 100644 --- a/pkgr/src/package/mod.rs +++ b/pkgr/src/package/mod.rs @@ -1,3 +1,68 @@ +use log::trace; + pub mod identifier; pub mod installer; -pub mod fetch; \ No newline at end of file +pub mod fetch; + +pub struct Package { + pub identifier: identifier::PackageIdentifier, + pub pkgfile: pkgfile::PKGFile, + is_installed: bool, + is_indexed: bool, +} + +impl Package { + pub fn new(identifier: identifier::PackageIdentifier, pkgfile: pkgfile::PKGFile) -> Package { + Package { + identifier, + pkgfile, + is_installed: false, + is_indexed: false, + } + } + + pub fn manifest(&self) -> manifest::Manifest { + manifest::Manifest::try_from(self.pkgfile.manifest.clone()) + .unwrap() + } + + pub fn fetch(package_identifier: identifier::PackageIdentifier) -> Result { + match &package_identifier { + identifier::PackageIdentifier::Path(path) => { + trace!("fetching package from path: {}", path); + let pkgfile = fetch::fetch_by_path(path).unwrap(); + Ok(Package::new(package_identifier, pkgfile)) + }, + identifier::PackageIdentifier::URI(url) => { + trace!("fetching package from uri: {}", url); + let pkgfile = fetch::fetch_by_uri(url).unwrap(); + Ok(Package::new(package_identifier, pkgfile)) + }, + identifier::PackageIdentifier::PackageLocator(locator) => { + trace!("fetching package from locator: {}", locator); + let pkgfile = fetch::fetch_by_package_locator(locator.clone()).unwrap(); + Ok(Package::new(package_identifier, pkgfile)) + }, + } + } + + pub fn install(&mut self) -> Result<(), installer::errors::InstallError> { + let manifest = self.manifest(); + let installer = installer::PackageInstaller::new(manifest.clone(), self.pkgfile.clone(), installer::InstallType::Bin); + match installer.install() { + Ok(_) => { + self.is_installed = true; + Ok(()) + }, + Err(e) => Err(e) + } + } + + pub fn is_installed(&self) -> bool { + self.is_installed + } + + pub fn is_indexed(&self) -> bool { + self.is_indexed + } +} \ No newline at end of file From f51c5dc9d53c08ddb4181ea23013e2996edf790c Mon Sep 17 00:00:00 2001 From: Didier Date: Mon, 17 Jul 2023 21:21:29 +0200 Subject: [PATCH 029/123] doc: rustdocs --- pkgr/src/package/mod.rs | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/pkgr/src/package/mod.rs b/pkgr/src/package/mod.rs index 349c360..7a7d6b2 100644 --- a/pkgr/src/package/mod.rs +++ b/pkgr/src/package/mod.rs @@ -12,6 +12,7 @@ pub struct Package { } impl Package { + /// Create a new package from a package identifier and a package file. pub fn new(identifier: identifier::PackageIdentifier, pkgfile: pkgfile::PKGFile) -> Package { Package { identifier, @@ -21,11 +22,7 @@ impl Package { } } - pub fn manifest(&self) -> manifest::Manifest { - manifest::Manifest::try_from(self.pkgfile.manifest.clone()) - .unwrap() - } - + /// Fetch a package from a package identifier. pub fn fetch(package_identifier: identifier::PackageIdentifier) -> Result { match &package_identifier { identifier::PackageIdentifier::Path(path) => { @@ -46,6 +43,14 @@ impl Package { } } + /// Get the package manifest. + pub fn manifest(&self) -> manifest::Manifest { + manifest::Manifest::try_from(self.pkgfile.manifest.clone()) + .unwrap() + } + + + /// Install the package. pub fn install(&mut self) -> Result<(), installer::errors::InstallError> { let manifest = self.manifest(); let installer = installer::PackageInstaller::new(manifest.clone(), self.pkgfile.clone(), installer::InstallType::Bin); @@ -58,11 +63,18 @@ impl Package { } } + /// Check if the package is installed. pub fn is_installed(&self) -> bool { self.is_installed } + /// Check if the package is indexed. pub fn is_indexed(&self) -> bool { self.is_indexed } + + /// Remove the package from the system. + pub fn remove(&mut self) -> Result<(), ()> { + unimplemented!(); + } } \ No newline at end of file From f050daf035fdfc23a403716b53dea3878af05672 Mon Sep 17 00:00:00 2001 From: Didier Date: Mon, 17 Jul 2023 21:23:59 +0200 Subject: [PATCH 030/123] format: fix formatting --- manifest/src/pkgr.rs | 4 +-- pkgr/src/commands.rs | 17 +++++++---- pkgr/src/logging.rs | 43 +++++++++++----------------- pkgr/src/main.rs | 12 ++++---- pkgr/src/package/fetch.rs | 11 ++++--- pkgr/src/package/identifier.rs | 2 +- pkgr/src/package/installer/errors.rs | 4 +-- pkgr/src/package/installer/mod.rs | 19 ++++++------ pkgr/src/package/mod.rs | 28 ++++++++++-------- pkgr/src/process/mod.rs | 13 ++++----- pkgr/src/process/output.rs | 6 ++-- pkgr/src/tmpfs.rs | 6 ++-- 12 files changed, 80 insertions(+), 85 deletions(-) diff --git a/manifest/src/pkgr.rs b/manifest/src/pkgr.rs index 700c6cb..530a37c 100644 --- a/manifest/src/pkgr.rs +++ b/manifest/src/pkgr.rs @@ -1,5 +1,5 @@ -use std::fmt::{Display, Formatter}; use serde::{Deserialize, Serialize}; +use std::fmt::{Display, Formatter}; #[derive(Debug, Clone, Serialize, Deserialize)] pub struct PKGR {} @@ -8,4 +8,4 @@ impl Display for PKGR { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { write!(f, "") } -} \ No newline at end of file +} diff --git a/pkgr/src/commands.rs b/pkgr/src/commands.rs index 2265330..8c7996e 100644 --- a/pkgr/src/commands.rs +++ b/pkgr/src/commands.rs @@ -1,10 +1,10 @@ -use std::process::exit; use crate::package::identifier::PackageIdentifier; +use crate::package::installer::{InstallType, PackageInstaller}; +use crate::package::Package; use clap::{Parser, Subcommand}; use log::{debug, error, info, trace, warn}; use manifest::Manifest; -use crate::package::installer::{InstallType, PackageInstaller}; -use crate::package::Package; +use std::process::exit; #[derive(Parser, Debug)] #[clap(name = "pkgr", version)] @@ -86,9 +86,11 @@ impl Command { info!("Install complete."); info!("Install took {}ms.", duration.as_nanos() as f64 / 1000000.0); - } - Command::Remove { package_identifier, index } => { + Command::Remove { + package_identifier, + index, + } => { if let PackageIdentifier::URI(_) = package_identifier { error!("URI is unsupported when removing applications."); exit(1); @@ -113,7 +115,10 @@ impl Command { info!("PKGR AUTHORS: {}", env!("CARGO_PKG_AUTHORS")); info!("PKGR DESCRIPTION: {}", env!("CARGO_PKG_DESCRIPTION")); info!(""); - info!("PKGR_LOG_LEVEL: {}", std::env::var("PKGR_LOG_LEVEL").unwrap_or_else(|_| "info".to_string())); + info!( + "PKGR_LOG_LEVEL: {}", + std::env::var("PKGR_LOG_LEVEL").unwrap_or_else(|_| "info".to_string()) + ); } Command::None => { error!("No command was specified."); diff --git a/pkgr/src/logging.rs b/pkgr/src/logging.rs index 6ee98a5..225ef27 100644 --- a/pkgr/src/logging.rs +++ b/pkgr/src/logging.rs @@ -1,26 +1,18 @@ -use log::SetLoggerError; -use fern::Dispatch; -use std::env; use colored::Colorize; +use fern::Dispatch; +use log::SetLoggerError; +use std::env; pub fn setup_logger() -> Result<(), SetLoggerError> { Dispatch::new() .format(|out, message, record| { match record.metadata().target() { "command:stdout" => { - out.finish(format_args!( - "{} {}", - "::".cyan(), - message.to_string() - )); + out.finish(format_args!("{} {}", "::".cyan(), message.to_string())); return; } "command:stderr" => { - out.finish(format_args!( - "{} {}", - "!!".red(), - message.to_string() - )); + out.finish(format_args!("{} {}", "!!".red(), message.to_string())); return; } _ => { @@ -28,30 +20,29 @@ pub fn setup_logger() -> Result<(), SetLoggerError> { "{} {}", // Some logic so messages look nice if message.to_string().len() > 0 { - match record - .level() - .to_string() - .chars() - .nth(0) - .unwrap_or('T') - { + match record.level().to_string().chars().nth(0).unwrap_or('T') { 'T' => "[TRACE]".bright_blue(), 'D' => "??".green(), 'I' => "=>".blue(), 'W' => "##".yellow(), 'E' => "!!".red(), _ => "**".blue(), - }.to_string() - } else { "".to_string() }, + } + .to_string() + } else { + "".to_string() + }, message.to_string().bright_white() )) } } }) - .level(env::var("PKGR_LOG_LEVEL") - .unwrap_or_else(|_| "info".to_string()) - .parse() - .unwrap_or(log::LevelFilter::Info)) + .level( + env::var("PKGR_LOG_LEVEL") + .unwrap_or_else(|_| "info".to_string()) + .parse() + .unwrap_or(log::LevelFilter::Info), + ) .chain(std::io::stdout()) .apply() } diff --git a/pkgr/src/main.rs b/pkgr/src/main.rs index aaa5387..cb73e53 100644 --- a/pkgr/src/main.rs +++ b/pkgr/src/main.rs @@ -1,15 +1,15 @@ -use std::process::Command; +use crate::commands::Cli; +use crate::process::Process; use clap::Parser; use colored::Colorize; -use crate::commands::Cli; use log::trace; -use crate::process::Process; +use std::process::Command; -mod process; mod commands; -mod package; -mod tmpfs; mod logging; +mod package; +mod process; +mod tmpfs; fn main() { logging::setup_logger().expect("Unable to setup logger."); diff --git a/pkgr/src/package/fetch.rs b/pkgr/src/package/fetch.rs index 099c5b2..a97dfb6 100644 --- a/pkgr/src/package/fetch.rs +++ b/pkgr/src/package/fetch.rs @@ -1,9 +1,9 @@ +use crate::package::identifier::PackageLocator; +use pkgfile::PKGFile; +use reqwest::blocking::get; use std::error::Error; use std::fmt::Display; use std::io::Read; -use pkgfile::PKGFile; -use reqwest::blocking::get; -use crate::package::identifier::PackageLocator; #[derive(Debug)] pub enum FetchError { @@ -26,9 +26,8 @@ impl Error for FetchError {} pub fn fetch_by_path>(path: S) -> Result { std::fs::read(path.into()) - .map_err(|e| FetchError::IOError(e)).and_then(|bytes| { - PKGFile::try_from(bytes).map_err(|_| FetchError::ParseError) - }) + .map_err(|e| FetchError::IOError(e)) + .and_then(|bytes| PKGFile::try_from(bytes).map_err(|_| FetchError::ParseError)) } pub fn fetch_by_uri>(uri: S) -> Result { diff --git a/pkgr/src/package/identifier.rs b/pkgr/src/package/identifier.rs index fa457d8..17064e4 100644 --- a/pkgr/src/package/identifier.rs +++ b/pkgr/src/package/identifier.rs @@ -88,7 +88,7 @@ impl FromStr for PackageLocator { fn from_str(s: &str) -> Result { #[allow(unused_assignments)] // false positive - let mut name = None; + let mut name = None; let mut version = None; let mut tags = None; diff --git a/pkgr/src/package/installer/errors.rs b/pkgr/src/package/installer/errors.rs index a6801c0..46cd402 100644 --- a/pkgr/src/package/installer/errors.rs +++ b/pkgr/src/package/installer/errors.rs @@ -15,7 +15,7 @@ impl Display for BinError { #[derive(Debug)] pub enum BuildError { - InvalidManifest + InvalidManifest, } impl Display for BuildError { @@ -30,7 +30,7 @@ impl Display for BuildError { pub enum InstallError { BuildError(BuildError), BinError(BinError), - Generic + Generic, } impl ToString for InstallError { diff --git a/pkgr/src/package/installer/mod.rs b/pkgr/src/package/installer/mod.rs index 8f75941..0d474aa 100644 --- a/pkgr/src/package/installer/mod.rs +++ b/pkgr/src/package/installer/mod.rs @@ -1,23 +1,23 @@ -use std::fmt::Display; -use log::{debug, trace}; +use crate::tmpfs::TempDir; use errors::{BinError, BuildError, InstallError}; +use log::{debug, trace}; use manifest::Manifest; use pkgfile::PKGFile; -use crate::tmpfs::TempDir; +use std::fmt::Display; pub mod errors; #[derive(Debug)] pub enum InstallType { Build, - Bin + Bin, } #[derive(Debug)] pub struct PackageInstaller { manifest: Manifest, pkgfile: PKGFile, - install_type: InstallType + install_type: InstallType, } impl PackageInstaller { @@ -25,7 +25,7 @@ impl PackageInstaller { PackageInstaller { manifest: m, pkgfile: p, - install_type: i + install_type: i, } } @@ -46,8 +46,8 @@ impl PackageInstaller { tmpdir.push(&self.manifest.package.name); trace!("extracting package into: {}", tmpdir.to_string()); match self.extract_to(tmpdir.to_string()) { - Ok(_) => {}, - Err(e) => return Err(e) + Ok(_) => {} + Err(e) => return Err(e), } debug!("extracted package in: {}", tmpdir.to_string()); Ok(()) @@ -60,14 +60,13 @@ impl PackageInstaller { let build_manifest = self.manifest.build.clone().unwrap(); // TODO: Check dependencies - Ok(()) } pub fn install(&self) -> Result<(), InstallError> { match self.install_type { InstallType::Bin => self.bin().map_err(|e| InstallError::BinError(e)), - InstallType::Build => self.build().map_err(|e| InstallError::BuildError(e)) + InstallType::Build => self.build().map_err(|e| InstallError::BuildError(e)), } } } diff --git a/pkgr/src/package/mod.rs b/pkgr/src/package/mod.rs index 7a7d6b2..e23a191 100644 --- a/pkgr/src/package/mod.rs +++ b/pkgr/src/package/mod.rs @@ -1,8 +1,8 @@ use log::trace; +pub mod fetch; pub mod identifier; pub mod installer; -pub mod fetch; pub struct Package { pub identifier: identifier::PackageIdentifier, @@ -23,43 +23,47 @@ impl Package { } /// Fetch a package from a package identifier. - pub fn fetch(package_identifier: identifier::PackageIdentifier) -> Result { + pub fn fetch( + package_identifier: identifier::PackageIdentifier, + ) -> Result { match &package_identifier { identifier::PackageIdentifier::Path(path) => { trace!("fetching package from path: {}", path); let pkgfile = fetch::fetch_by_path(path).unwrap(); Ok(Package::new(package_identifier, pkgfile)) - }, + } identifier::PackageIdentifier::URI(url) => { trace!("fetching package from uri: {}", url); let pkgfile = fetch::fetch_by_uri(url).unwrap(); Ok(Package::new(package_identifier, pkgfile)) - }, + } identifier::PackageIdentifier::PackageLocator(locator) => { trace!("fetching package from locator: {}", locator); let pkgfile = fetch::fetch_by_package_locator(locator.clone()).unwrap(); Ok(Package::new(package_identifier, pkgfile)) - }, + } } } /// Get the package manifest. pub fn manifest(&self) -> manifest::Manifest { - manifest::Manifest::try_from(self.pkgfile.manifest.clone()) - .unwrap() + manifest::Manifest::try_from(self.pkgfile.manifest.clone()).unwrap() } - /// Install the package. pub fn install(&mut self) -> Result<(), installer::errors::InstallError> { let manifest = self.manifest(); - let installer = installer::PackageInstaller::new(manifest.clone(), self.pkgfile.clone(), installer::InstallType::Bin); + let installer = installer::PackageInstaller::new( + manifest.clone(), + self.pkgfile.clone(), + installer::InstallType::Bin, + ); match installer.install() { Ok(_) => { self.is_installed = true; Ok(()) - }, - Err(e) => Err(e) + } + Err(e) => Err(e), } } @@ -77,4 +81,4 @@ impl Package { pub fn remove(&mut self) -> Result<(), ()> { unimplemented!(); } -} \ No newline at end of file +} diff --git a/pkgr/src/process/mod.rs b/pkgr/src/process/mod.rs index 70ab93b..21586c0 100644 --- a/pkgr/src/process/mod.rs +++ b/pkgr/src/process/mod.rs @@ -10,11 +10,7 @@ pub struct Process { impl Process { pub fn new(command: Vec, cwd: Option, env: Vec) -> Process { - Process { - command, - cwd, - env, - } + Process { command, cwd, env } } pub fn command>(cmd: Vec) -> Process { @@ -27,7 +23,10 @@ impl Process { .current_dir(self.cwd.clone().unwrap_or(".".to_string())) .envs(self.env.iter().map(|v| { let mut split = v.split('='); - (split.next().unwrap().to_string(), split.next().unwrap().to_string()) + ( + split.next().unwrap().to_string(), + split.next().unwrap().to_string(), + ) })) .stdout(std::process::Stdio::piped()) .stderr(std::process::Stdio::piped()) @@ -41,4 +40,4 @@ impl Default for Process { fn default() -> Process { Process::new(vec![], None, vec![]) } -} \ No newline at end of file +} diff --git a/pkgr/src/process/output.rs b/pkgr/src/process/output.rs index 09e788e..68d91fd 100644 --- a/pkgr/src/process/output.rs +++ b/pkgr/src/process/output.rs @@ -1,10 +1,10 @@ // create functions that spawn threads for Stdout and Stderr // that print the output to info!(target: "command:stdout", ...) and info!(target: "command:stderr", ...) respectively -use std::process::Child; use std::io::{BufRead, BufReader}; +use std::process::Child; -use log::{info, error}; +use log::{error, info}; pub fn spawn_output_handlers(child: &mut Child) { let stdout = child.stdout.take().unwrap(); @@ -23,4 +23,4 @@ pub fn spawn_output_handlers(child: &mut Child) { error!(target: "command:stderr", "{}", line.unwrap()); } }); -} \ No newline at end of file +} diff --git a/pkgr/src/tmpfs.rs b/pkgr/src/tmpfs.rs index 509d3f7..c9ea6f8 100644 --- a/pkgr/src/tmpfs.rs +++ b/pkgr/src/tmpfs.rs @@ -9,9 +9,7 @@ impl TempDir { if !path.exists() { std::fs::create_dir_all(&path).unwrap(); } - TempDir { - path - } + TempDir { path } } pub fn push>(&mut self, path: S) { @@ -45,4 +43,4 @@ impl ToString for TempDir { fn to_string(&self) -> String { self.path.to_str().unwrap().to_string() } -} \ No newline at end of file +} From a689abbc5d1d79eaf529d5057f5dd01ddf5234cf Mon Sep 17 00:00:00 2001 From: Didier Date: Mon, 17 Jul 2023 21:26:18 +0200 Subject: [PATCH 031/123] fix: removal of file used in old spec --- build-pkg.toml | 14 -------------- 1 file changed, 14 deletions(-) delete mode 100644 build-pkg.toml diff --git a/build-pkg.toml b/build-pkg.toml deleted file mode 100644 index 8c994d9..0000000 --- a/build-pkg.toml +++ /dev/null @@ -1,14 +0,0 @@ -[project] -package_file = "./package.toml" -package_dir = "./pkg" -increment_version = false - -[mappings.source] -ignore = ["target/"] - -[mappings.source.map] -"." = "/src" - -[mappings.bin.map] -"target/release/bin/pkgr" = "/bin/pkgr" - From c12b78b3c148d37b8e7bb5932a367ca627490a35 Mon Sep 17 00:00:00 2001 From: Didier Date: Tue, 18 Jul 2023 17:51:55 +0200 Subject: [PATCH 032/123] fix: remove useless threading --- pkgr/src/commands.rs | 10 ++++- pkgr/src/config/mod.rs | 27 +++++++++++++ pkgr/src/main.rs | 7 +++- pkgr/src/package/identifier.rs | 33 ++++++++++++++++ pkgr/src/package/installer/errors.rs | 2 + pkgr/src/package/installer/mod.rs | 52 +++++++++++++++++++++++-- pkgr/src/package/mod.rs | 22 +++++++++-- pkgr/src/{process/mod.rs => process.rs} | 16 +++++++- pkgr/src/process/output.rs | 26 ------------- 9 files changed, 157 insertions(+), 38 deletions(-) create mode 100644 pkgr/src/config/mod.rs rename pkgr/src/{process/mod.rs => process.rs} (73%) delete mode 100644 pkgr/src/process/output.rs diff --git a/pkgr/src/commands.rs b/pkgr/src/commands.rs index 8c7996e..6d8e139 100644 --- a/pkgr/src/commands.rs +++ b/pkgr/src/commands.rs @@ -5,6 +5,8 @@ use clap::{Parser, Subcommand}; use log::{debug, error, info, trace, warn}; use manifest::Manifest; use std::process::exit; +use crate::CONFIG; +use crate::process::Process; #[derive(Parser, Debug)] #[clap(name = "pkgr", version)] @@ -69,7 +71,7 @@ impl Command { } trace!("Starting install..."); - match pkg.install() { + match pkg.install(CONFIG.with(|c| if !*build { c.build_by_default } else { *build })) { Ok(_) => (), Err(e) => { error!("Install failed: {}", e.to_string()); @@ -110,6 +112,12 @@ impl Command { info!("Info message."); warn!("Warning message."); error!("Error message."); + + info!(""); + Process::command(vec!["sh", "-c", "echo whoami: $(whoami)"].iter_mut().map(|s| s.to_string()).collect()) + .spawn() + .unwrap(); + info!(""); info!("PKGR VERSION: {}", env!("CARGO_PKG_VERSION")); info!("PKGR AUTHORS: {}", env!("CARGO_PKG_AUTHORS")); diff --git a/pkgr/src/config/mod.rs b/pkgr/src/config/mod.rs new file mode 100644 index 0000000..29f4936 --- /dev/null +++ b/pkgr/src/config/mod.rs @@ -0,0 +1,27 @@ +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Serialize, Deserialize)] +pub struct Config { + #[serde(default)] + pub build_by_default: bool, +} + +impl Config { + pub fn from_path(path: &str) -> Result { + match std::fs::read_to_string(path) { + Ok(s) => match toml::from_str(&s) { + Ok(c) => Ok(c), + Err(e) => Err(format!("failed to parse config: {}", e)), + }, + Err(e) => Err(format!("failed to read config: {}", e)), + } + } +} + +impl Default for Config { + fn default() -> Config { + Config { + build_by_default: false, + } + } +} \ No newline at end of file diff --git a/pkgr/src/main.rs b/pkgr/src/main.rs index cb73e53..97f721f 100644 --- a/pkgr/src/main.rs +++ b/pkgr/src/main.rs @@ -10,13 +10,18 @@ mod logging; mod package; mod process; mod tmpfs; +mod config; + +thread_local! { + static CONFIG: config::Config = config::Config::from_path("/etc/pkgr.toml") + .unwrap_or(config::Config::default()); +} fn main() { logging::setup_logger().expect("Unable to setup logger."); #[cfg(not(debug_assertions))] { - #[cfg(target_family = "unix")] if unsafe { libc::getuid() } != 0 { use log::error; error!("pkgr must be run as root."); diff --git a/pkgr/src/package/identifier.rs b/pkgr/src/package/identifier.rs index 17064e4..427b775 100644 --- a/pkgr/src/package/identifier.rs +++ b/pkgr/src/package/identifier.rs @@ -128,3 +128,36 @@ impl FromStr for PackageLocator { }) } } + +impl From<(String, String)> for PackageLocator { + fn from((name, locate_str): (String, String)) -> Self { + // name = "pkg" + // locate_str = "1.0.0:tag1,tag2" or "1.0.0" or "tag1,tag2" + let mut version = None; + let mut tags = None; + + let version_re = Regex::new("^([0-9]+)").unwrap(); + let tags_re = Regex::new("^:([a-zA-Z0-9,._]+)").unwrap(); + + if let Some(caps) = version_re.captures(locate_str.as_str()) { + version = Some(caps.get(1).unwrap().as_str().parse::().unwrap()); + } + + if let Some(caps) = tags_re.captures(locate_str.as_str()) { + tags = Some( + caps.get(1) + .unwrap() + .as_str() + .split(",") + .map(|s| s.to_string()) + .collect(), + ); + } + + PackageLocator { + name, + version, + tags, + } + } +} diff --git a/pkgr/src/package/installer/errors.rs b/pkgr/src/package/installer/errors.rs index 46cd402..637e891 100644 --- a/pkgr/src/package/installer/errors.rs +++ b/pkgr/src/package/installer/errors.rs @@ -30,6 +30,7 @@ impl Display for BuildError { pub enum InstallError { BuildError(BuildError), BinError(BinError), + InvalidManifest, Generic, } @@ -38,6 +39,7 @@ impl ToString for InstallError { match self { InstallError::BuildError(e) => format!("Build error: {}", e), InstallError::BinError(e) => format!("Bin error: {}", e), + InstallError::InvalidManifest => "Invalid manifest".to_string(), InstallError::Generic => "Install error".to_string(), } } diff --git a/pkgr/src/package/installer/mod.rs b/pkgr/src/package/installer/mod.rs index 0d474aa..8181ea8 100644 --- a/pkgr/src/package/installer/mod.rs +++ b/pkgr/src/package/installer/mod.rs @@ -1,9 +1,13 @@ use crate::tmpfs::TempDir; use errors::{BinError, BuildError, InstallError}; -use log::{debug, trace}; +use log::{debug, error, info, trace}; use manifest::Manifest; use pkgfile::PKGFile; use std::fmt::Display; +use std::process::exit; +use crate::CONFIG; +use crate::package::identifier::{PackageIdentifier, PackageLocator}; +use crate::package::Package; pub mod errors; @@ -59,14 +63,54 @@ impl PackageInstaller { } let build_manifest = self.manifest.build.clone().unwrap(); // TODO: Check dependencies + for pkg_tuple in build_manifest.dependencies { + let mut pkg = Package::fetch( + PackageIdentifier::PackageLocator( + PackageLocator::from(pkg_tuple) + ) + ).expect("no pkg"); + + if !pkg.is_installed() { + match pkg.install(CONFIG.with(|c| { + c.build_by_default + })) { + Ok(_) => { info!("Installed dependency: \"{}\"", pkg.manifest().package.name) }, + Err(_) => { + error!("Could not install dependency: \"{}\"", pkg.identifier); + exit(1); + } + } + } + } Ok(()) } - pub fn install(&self) -> Result<(), InstallError> { + pub fn install(&mut self) -> Result<(), InstallError> { match self.install_type { - InstallType::Bin => self.bin().map_err(|e| InstallError::BinError(e)), - InstallType::Build => self.build().map_err(|e| InstallError::BuildError(e)), + InstallType::Bin => { + if let None = self.manifest.bin { + self.install_type = InstallType::Build; + if let None = self.manifest.build { + return Err(InstallError::InvalidManifest); + } + } + } + InstallType::Build => { + if let None = self.manifest.build { + self.install_type = InstallType::Bin; + if let None = self.manifest.bin { + return Err(InstallError::InvalidManifest); + } + } + } + } + + match self.install_type { + InstallType::Bin => self.bin() + .map_err(|e| InstallError::BinError(e)), + InstallType::Build => self.build() + .map_err(|e| InstallError::BuildError(e)), } } } diff --git a/pkgr/src/package/mod.rs b/pkgr/src/package/mod.rs index e23a191..79de18d 100644 --- a/pkgr/src/package/mod.rs +++ b/pkgr/src/package/mod.rs @@ -1,4 +1,4 @@ -use log::trace; +use log::{info, trace}; pub mod fetch; pub mod identifier; @@ -51,12 +51,26 @@ impl Package { } /// Install the package. - pub fn install(&mut self) -> Result<(), installer::errors::InstallError> { + pub fn install(&mut self, build: bool) -> Result<(), installer::errors::InstallError> { let manifest = self.manifest(); - let installer = installer::PackageInstaller::new( + let mut installer = installer::PackageInstaller::new( manifest.clone(), self.pkgfile.clone(), - installer::InstallType::Bin, + if build { + info!("building package: {}...", if self.manifest().package.name.is_empty() { + self.identifier.to_string() + } else { + self.manifest().package.name + }); + installer::InstallType::Build + } else { + info!("using binary install for package: {}...", if self.manifest().package.name.is_empty() { + self.identifier.to_string() + } else { + self.manifest().package.name + }); + installer::InstallType::Bin + }, ); match installer.install() { Ok(_) => { diff --git a/pkgr/src/process/mod.rs b/pkgr/src/process.rs similarity index 73% rename from pkgr/src/process/mod.rs rename to pkgr/src/process.rs index 21586c0..727529f 100644 --- a/pkgr/src/process/mod.rs +++ b/pkgr/src/process.rs @@ -1,6 +1,7 @@ +use std::io::{BufRead, BufReader}; use std::process::Command; +use log::info; -pub mod output; pub struct Process { pub command: Vec, @@ -31,7 +32,18 @@ impl Process { .stdout(std::process::Stdio::piped()) .stderr(std::process::Stdio::piped()) .spawn()?; - output::spawn_output_handlers(&mut child); + + let stdout = child.stdout.take().unwrap(); + let stderr = child.stderr.take().unwrap(); + + for line in BufReader::new(stdout).lines() { + info!(target: "command:stdout", "{}", line.unwrap()); + } + + for line in BufReader::new(stderr).lines() { + info!(target: "command:stderr", "{}", line.unwrap()); + } + Ok(child) } } diff --git a/pkgr/src/process/output.rs b/pkgr/src/process/output.rs deleted file mode 100644 index 68d91fd..0000000 --- a/pkgr/src/process/output.rs +++ /dev/null @@ -1,26 +0,0 @@ -// create functions that spawn threads for Stdout and Stderr -// that print the output to info!(target: "command:stdout", ...) and info!(target: "command:stderr", ...) respectively - -use std::io::{BufRead, BufReader}; -use std::process::Child; - -use log::{error, info}; - -pub fn spawn_output_handlers(child: &mut Child) { - let stdout = child.stdout.take().unwrap(); - let stderr = child.stderr.take().unwrap(); - - std::thread::spawn(move || { - let reader = BufReader::new(stdout); - for line in reader.lines() { - info!(target: "command:stdout", "{}", line.unwrap()); - } - }); - - std::thread::spawn(move || { - let reader = BufReader::new(stderr); - for line in reader.lines() { - error!(target: "command:stderr", "{}", line.unwrap()); - } - }); -} From b115b91ecb5747cf18df2570c97f98f950a82409 Mon Sep 17 00:00:00 2001 From: Didier Date: Wed, 19 Jul 2023 15:35:51 +0200 Subject: [PATCH 033/123] feat: add pkgr global config --- pkgr/src/commands.rs | 7 ++++++- pkgr/src/config/mod.rs | 16 ++++++++-------- pkgr/src/process.rs | 8 ++++++-- 3 files changed, 20 insertions(+), 11 deletions(-) diff --git a/pkgr/src/commands.rs b/pkgr/src/commands.rs index 6d8e139..d63e2e3 100644 --- a/pkgr/src/commands.rs +++ b/pkgr/src/commands.rs @@ -114,9 +114,14 @@ impl Command { error!("Error message."); info!(""); - Process::command(vec!["sh", "-c", "echo whoami: $(whoami)"].iter_mut().map(|s| s.to_string()).collect()) + Process::command(vec!["sh", "-c", "echo whoami: $(whoami)"]) .spawn() .unwrap(); + if let Ok(user) = std::env::var("SUDO_USER") { + Process::command(vec!["sh", "-c", "echo sudo_user: $SUDO_USER"]) + .spawn() + .unwrap(); + } info!(""); info!("PKGR VERSION: {}", env!("CARGO_PKG_VERSION")); diff --git a/pkgr/src/config/mod.rs b/pkgr/src/config/mod.rs index 29f4936..4bb3397 100644 --- a/pkgr/src/config/mod.rs +++ b/pkgr/src/config/mod.rs @@ -6,6 +6,14 @@ pub struct Config { pub build_by_default: bool, } +impl Default for Config { + fn default() -> Config { + Config { + build_by_default: false, + } + } +} + impl Config { pub fn from_path(path: &str) -> Result { match std::fs::read_to_string(path) { @@ -17,11 +25,3 @@ impl Config { } } } - -impl Default for Config { - fn default() -> Config { - Config { - build_by_default: false, - } - } -} \ No newline at end of file diff --git a/pkgr/src/process.rs b/pkgr/src/process.rs index 727529f..f6f4e21 100644 --- a/pkgr/src/process.rs +++ b/pkgr/src/process.rs @@ -1,6 +1,6 @@ use std::io::{BufRead, BufReader}; use std::process::Command; -use log::info; +use log::{info, trace}; pub struct Process { @@ -19,6 +19,7 @@ impl Process { } pub fn spawn(&self) -> std::io::Result { + trace!("Spawning process: {:?}", self.command); let mut child = Command::new(&self.command[0]) .args(&self.command[1..]) .current_dir(self.cwd.clone().unwrap_or(".".to_string())) @@ -32,10 +33,10 @@ impl Process { .stdout(std::process::Stdio::piped()) .stderr(std::process::Stdio::piped()) .spawn()?; - let stdout = child.stdout.take().unwrap(); let stderr = child.stderr.take().unwrap(); + trace!("Processing lines from stdout and stderr"); for line in BufReader::new(stdout).lines() { info!(target: "command:stdout", "{}", line.unwrap()); } @@ -44,6 +45,9 @@ impl Process { info!(target: "command:stderr", "{}", line.unwrap()); } + trace!("End of process"); + + trace!("Returning child"); Ok(child) } } From e4eb85f535d6b7b39ce518ef32187d3f702c48cb Mon Sep 17 00:00:00 2001 From: Didier Date: Wed, 19 Jul 2023 16:15:26 +0200 Subject: [PATCH 034/123] feat: multiline logging --- pkgr/src/commands.rs | 10 ++--- pkgr/src/config/mod.rs | 3 ++ pkgr/src/logging.rs | 63 ++++++++++++++++++---------- pkgr/src/package/identifier.rs | 19 +++++++-- pkgr/src/package/installer/errors.rs | 4 +- pkgr/src/package/installer/mod.rs | 5 ++- pkgr/src/tmpfs.rs | 9 ++-- 7 files changed, 75 insertions(+), 38 deletions(-) diff --git a/pkgr/src/commands.rs b/pkgr/src/commands.rs index d63e2e3..4393869 100644 --- a/pkgr/src/commands.rs +++ b/pkgr/src/commands.rs @@ -107,11 +107,11 @@ impl Command { error!("Update is not yet implemented."); } Command::Debug => { - trace!("Trace message."); - debug!("Debug message."); - info!("Info message."); - warn!("Warning message."); - error!("Error message."); + trace!("Trace message.\nWith newline."); + debug!("Debug message.\nWith newline."); + info!("Info message.\nWith newline."); + warn!("Warning message.\nWith newline."); + error!("Error message.\nWith newline."); info!(""); Process::command(vec!["sh", "-c", "echo whoami: $(whoami)"]) diff --git a/pkgr/src/config/mod.rs b/pkgr/src/config/mod.rs index 4bb3397..9a932d2 100644 --- a/pkgr/src/config/mod.rs +++ b/pkgr/src/config/mod.rs @@ -4,12 +4,15 @@ use serde::{Deserialize, Serialize}; pub struct Config { #[serde(default)] pub build_by_default: bool, + #[serde(default)] + pub tmp_dir: Option, } impl Default for Config { fn default() -> Config { Config { build_by_default: false, + tmp_dir: Some(String::from("/tmp/pkgr")) } } } diff --git a/pkgr/src/logging.rs b/pkgr/src/logging.rs index 225ef27..5e32d9d 100644 --- a/pkgr/src/logging.rs +++ b/pkgr/src/logging.rs @@ -1,40 +1,57 @@ use colored::Colorize; use fern::Dispatch; -use log::SetLoggerError; +use log::{Record, SetLoggerError}; use std::env; +fn format_regular>(log: S, record: &Record) -> String { + let log = log.into(); + let line_prefix = |line: String, extend: bool| { + let prefix = if extend { + match record.level() { + log::Level::Trace => " :".bright_blue(), + log::Level::Debug => " :".green(), + log::Level::Info => " :".blue(), + log::Level::Warn => " :".yellow(), + log::Level::Error => " :".red(), + }.to_string() + } else { + match record.level() { + log::Level::Trace => "[TRACE]".bright_blue(), + log::Level::Debug => "??".green(), + log::Level::Info => "=>".blue(), + log::Level::Warn => "##".yellow(), + log::Level::Error => "!!".red() + }.to_string() + }; + return format!("{} {}", prefix, line); + }; + + let mut lines = log.lines().peekable(); + let mut output = match lines.peek() { + Some(line) => line_prefix(lines.next().unwrap().to_string(), false), + None => return "".to_string(), + }; + + for line in lines { + output.push_str(&*format!("\n{}", line_prefix(line.to_string(), true))); + } + + output +} + pub fn setup_logger() -> Result<(), SetLoggerError> { Dispatch::new() .format(|out, message, record| { match record.metadata().target() { "command:stdout" => { - out.finish(format_args!("{} {}", "::".cyan(), message.to_string())); + out.finish(format_args!("{} {}", ">>".cyan(), message.to_string())); return; } "command:stderr" => { - out.finish(format_args!("{} {}", "!!".red(), message.to_string())); + out.finish(format_args!("{} {}", ">>".red(), message.to_string())); return; } - _ => { - out.finish(format_args!( - "{} {}", - // Some logic so messages look nice - if message.to_string().len() > 0 { - match record.level().to_string().chars().nth(0).unwrap_or('T') { - 'T' => "[TRACE]".bright_blue(), - 'D' => "??".green(), - 'I' => "=>".blue(), - 'W' => "##".yellow(), - 'E' => "!!".red(), - _ => "**".blue(), - } - .to_string() - } else { - "".to_string() - }, - message.to_string().bright_white() - )) - } + _ => out.finish(format_args!("{}", format_regular(message.to_string(), record))), } }) .level( diff --git a/pkgr/src/package/identifier.rs b/pkgr/src/package/identifier.rs index 427b775..3fde5e7 100644 --- a/pkgr/src/package/identifier.rs +++ b/pkgr/src/package/identifier.rs @@ -7,15 +7,15 @@ use regex::Regex; pub enum PackageIdentifierError { InvalidPackageLocator(String), InvalidURI(String), + InvalidPath(String), } impl std::fmt::Display for PackageIdentifierError { fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { match self { - PackageIdentifierError::InvalidPackageLocator(s) => { - write!(f, "Invalid package locator: {}", s) - } + PackageIdentifierError::InvalidPackageLocator(s) => write!(f, "Invalid package locator: {}", s), PackageIdentifierError::InvalidURI(s) => write!(f, "Invalid URI: {}", s), + PackageIdentifierError::InvalidPath(s) => write!(f, "Invalid path: {}", s), } } } @@ -71,7 +71,18 @@ impl FromStr for PackageIdentifier { return Err(PackageIdentifierError::InvalidURI(s.to_string())); } Ok(PackageIdentifier::URI(s.to_string())) - } else if std::path::Path::new(s).exists() { + } else if s.starts_with("/") + || s.starts_with("./") + || s.starts_with("../") + || s.starts_with("~/") + { + let path = std::path::Path::new(s); + if s.ends_with("/") + || !path.exists() + || path.is_dir() + { + return Err(PackageIdentifierError::InvalidPath(s.to_string())); + } return Ok(PackageIdentifier::Path(s.to_string())); } else { let pl = match PackageLocator::from_str(s) { diff --git a/pkgr/src/package/installer/errors.rs b/pkgr/src/package/installer/errors.rs index 637e891..9e86f1b 100644 --- a/pkgr/src/package/installer/errors.rs +++ b/pkgr/src/package/installer/errors.rs @@ -37,8 +37,8 @@ pub enum InstallError { impl ToString for InstallError { fn to_string(&self) -> String { match self { - InstallError::BuildError(e) => format!("Build error: {}", e), - InstallError::BinError(e) => format!("Bin error: {}", e), + InstallError::BuildError(e) => format!("Build error: \n{}", e), + InstallError::BinError(e) => format!("Bin error: \n{}", e), InstallError::InvalidManifest => "Invalid manifest".to_string(), InstallError::Generic => "Install error".to_string(), } diff --git a/pkgr/src/package/installer/mod.rs b/pkgr/src/package/installer/mod.rs index 8181ea8..5da81f9 100644 --- a/pkgr/src/package/installer/mod.rs +++ b/pkgr/src/package/installer/mod.rs @@ -38,7 +38,10 @@ impl PackageInstaller { return Err(BinError::UnpackError("package has no data".to_string())); } if std::path::Path::new(&path).exists() { - return Err(BinError::UnpackError("path already exists".to_string())); + return Err(BinError::UnpackError(format!( + "path already exists: {}", + path + ))); } tar::Archive::new(self.pkgfile.data.as_slice()) .unpack(&path) diff --git a/pkgr/src/tmpfs.rs b/pkgr/src/tmpfs.rs index c9ea6f8..d21aff7 100644 --- a/pkgr/src/tmpfs.rs +++ b/pkgr/src/tmpfs.rs @@ -1,4 +1,5 @@ use std::path::PathBuf; +use crate::CONFIG; pub struct TempDir { path: PathBuf, @@ -20,9 +21,11 @@ impl TempDir { impl Default for TempDir { fn default() -> TempDir { TempDir::new({ - let mut t = std::env::temp_dir(); - t.push("pkgr"); - t + if let Some(d) = CONFIG.with(|c| c.tmp_dir.clone()) { + PathBuf::from(d) + } else { + PathBuf::from("/tmp/pkgr") + } }) } } From 04a160d8d3d7fe41d707c96f6e88cf035177f6a8 Mon Sep 17 00:00:00 2001 From: Didier Date: Tue, 1 Aug 2023 20:58:09 +0200 Subject: [PATCH 035/123] ci: woodpecker ci --- .github/workflows/build.yaml | 59 ------------------------------------ .woodpecker.yml | 24 +++++++++++++++ 2 files changed, 24 insertions(+), 59 deletions(-) delete mode 100644 .github/workflows/build.yaml create mode 100644 .woodpecker.yml diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml deleted file mode 100644 index d9bf0bc..0000000 --- a/.github/workflows/build.yaml +++ /dev/null @@ -1,59 +0,0 @@ -on: - push: - paths: - - 'pkgr/**' - - 'manifest/**' - - 'bootpkg/**' - - 'pkgfile/**' - -jobs: - check: # check if the code compiles - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - name: install dependencies - run: | - apt update -y - apt install -y curl gcc g++ make - - uses: https://github.com/actions-rs/toolchain@v1 - with: - toolchain: stable - override: true - - uses: https://github.com/actions-rs/cargo@v1 - with: - command: check - args: --manifest-path bootpkg/Cargo.toml --verbose --all - - uses: https://github.com/actions-rs/cargo@v1 - with: - command: check - args: --manifest-path pkgr/Cargo.toml --verbose --all -# build: # create artifacts and upload them -# needs: check -# runs-on: ubuntu-latest -# steps: -# - uses: actions/checkout@v2 -# - name: install dependencies -# run: | -# apt update -y -# apt install -y curl gcc g++ make -# - uses: https://github.com/actions-rs/toolchain@v1 -# with: -# toolchain: stable -# override: true -# - uses: https://github.com/actions-rs/cargo@v1 -# with: -# command: build -# args: --release --manifest-path bootpkg/Cargo.toml --verbose --all -# continue-on-error: true -# - uses: https://github.com/actions-rs/cargo@v1 -# with: -# command: build -# args: --release --manifest-path pkgr/Cargo.toml --verbose --all -# - uses: actions/upload-artifact@v2 -# with: -# name: bootpkg -# path: bootpkg/target/release/bootpkg -# - uses: actions/upload-artifact@v2 -# with: -# name: pkgr -# path: pkgr/target/release/pkgr diff --git a/.woodpecker.yml b/.woodpecker.yml new file mode 100644 index 0000000..ee81047 --- /dev/null +++ b/.woodpecker.yml @@ -0,0 +1,24 @@ +when: + event: push + +steps: + check: # check if the code compiles + image: rust + commands: + - cargo check --manifest-path bootpkg/Cargo.toml --verbose --all + - cargo check --manifest-path pkgr/Cargo.toml --verbose --all + build: + image: rust + commands: + - cargo build --release --manifest-path bootpkg/Cargo.toml --verbose --all + - cargo build --release --manifest-path pkgr/Cargo.toml --verbose --all + publish: + image: woodpeckerci/plugin-gitea-release + settings: + base_url: https://git.ixvd.net + files: + - "bootpkg/target/release/bootpkg" + - "pkgr/target/release/pkgr" + api_key: + from_secret: GITEA_ACCESS_TOKEN + target: main \ No newline at end of file From 384843bdc36b75008a1268d7f472fc93297e1f12 Mon Sep 17 00:00:00 2001 From: Didier Date: Tue, 1 Aug 2023 21:02:44 +0200 Subject: [PATCH 036/123] ci: change conditions --- .woodpecker.yml | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/.woodpecker.yml b/.woodpecker.yml index ee81047..5fb6212 100644 --- a/.woodpecker.yml +++ b/.woodpecker.yml @@ -1,5 +1,12 @@ when: event: push + path: + include: + - "pkgr/**" + - "bootpkg/**" + - "pkgfile/**" + - "pkg/**" + - "manifest/**" steps: check: # check if the code compiles @@ -21,4 +28,6 @@ steps: - "pkgr/target/release/pkgr" api_key: from_secret: GITEA_ACCESS_TOKEN - target: main \ No newline at end of file + target: main + when: + event: tag \ No newline at end of file From e18f9926061684572eeb0d51d79f5e98b92f4cdb Mon Sep 17 00:00:00 2001 From: Didier Date: Tue, 1 Aug 2023 21:03:24 +0200 Subject: [PATCH 037/123] ci: allow manual run --- .woodpecker.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.woodpecker.yml b/.woodpecker.yml index 5fb6212..c706c2d 100644 --- a/.woodpecker.yml +++ b/.woodpecker.yml @@ -1,5 +1,7 @@ when: - event: push + event: + - push + - manual path: include: - "pkgr/**" From a3cddf7946114f76e2ad4e0ef0b2d11603eeaf89 Mon Sep 17 00:00:00 2001 From: Didier Date: Tue, 1 Aug 2023 21:04:40 +0200 Subject: [PATCH 038/123] ci: always release and less verbose --- .woodpecker.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.woodpecker.yml b/.woodpecker.yml index c706c2d..5a98af8 100644 --- a/.woodpecker.yml +++ b/.woodpecker.yml @@ -14,13 +14,13 @@ steps: check: # check if the code compiles image: rust commands: - - cargo check --manifest-path bootpkg/Cargo.toml --verbose --all - - cargo check --manifest-path pkgr/Cargo.toml --verbose --all + - cargo check --release --manifest-path bootpkg/Cargo.toml --all + - cargo check --release --manifest-path pkgr/Cargo.toml --all build: image: rust commands: - - cargo build --release --manifest-path bootpkg/Cargo.toml --verbose --all - - cargo build --release --manifest-path pkgr/Cargo.toml --verbose --all + - cargo build --release --manifest-path bootpkg/Cargo.toml --all + - cargo build --release --manifest-path pkgr/Cargo.toml --all publish: image: woodpeckerci/plugin-gitea-release settings: From 0c94c921c2972d6556756afae9b5fcb02079428b Mon Sep 17 00:00:00 2001 From: Didier Date: Tue, 1 Aug 2023 21:07:56 +0200 Subject: [PATCH 039/123] =?UTF-8?q?ci:=20=E3=81=9Fg=20condition?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .woodpecker.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.woodpecker.yml b/.woodpecker.yml index 5a98af8..f4640ea 100644 --- a/.woodpecker.yml +++ b/.woodpecker.yml @@ -2,6 +2,7 @@ when: event: - push - manual + - tag path: include: - "pkgr/**" From a809689a2061cc708088f15c8e51ba1e2697a4bc Mon Sep 17 00:00:00 2001 From: Didier Date: Tue, 1 Aug 2023 21:15:47 +0200 Subject: [PATCH 040/123] ci: token name fix --- .woodpecker.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.woodpecker.yml b/.woodpecker.yml index f4640ea..620c2bd 100644 --- a/.woodpecker.yml +++ b/.woodpecker.yml @@ -30,7 +30,7 @@ steps: - "bootpkg/target/release/bootpkg" - "pkgr/target/release/pkgr" api_key: - from_secret: GITEA_ACCESS_TOKEN + from_secret: gitea_access_token target: main when: event: tag \ No newline at end of file From 36c782a564e532391c58d7cd23b053791d103248 Mon Sep 17 00:00:00 2001 From: Raine Date: Sat, 14 Oct 2023 22:39:39 +0200 Subject: [PATCH 041/123] init: initial commit of packager --- .gitignore | 2 + .idea/.gitignore | 8 +++ .idea/discord.xml | 7 +++ .idea/modules.xml | 8 +++ .idea/pkgr.iml | 17 +++++++ .idea/vcs.xml | 6 +++ README.md | 8 +++ bootpkg/Cargo.toml | 12 +++++ bootpkg/src/args.rs | 29 +++++++++++ bootpkg/src/main.rs | 51 +++++++++++++++++++ bootpkg/src/prelude.rs | 22 ++++++++ build-pkg.toml | 14 +++++ manifest/Cargo.toml | 10 ++++ manifest/src/bin.rs | 7 +++ manifest/src/build.rs | 10 ++++ manifest/src/fs.rs | 16 ++++++ manifest/src/lib.rs | 32 ++++++++++++ manifest/src/package.rs | 45 ++++++++++++++++ manifest/src/pkgr.rs | 4 ++ package.toml | 110 ++++++++++++++++++++++++++++++++++++++++ pkgfile/Cargo.toml | 10 ++++ pkgfile/pkgfile.txt | 3 ++ pkgfile/src/lib.rs | 36 +++++++++++++ pkgr/Cargo.toml | 9 ++++ pkgr/src/main.rs | 2 + 25 files changed, 478 insertions(+) create mode 100644 .gitignore create mode 100644 .idea/.gitignore create mode 100644 .idea/discord.xml create mode 100644 .idea/modules.xml create mode 100644 .idea/pkgr.iml create mode 100644 .idea/vcs.xml create mode 100644 README.md create mode 100644 bootpkg/Cargo.toml create mode 100644 bootpkg/src/args.rs create mode 100644 bootpkg/src/main.rs create mode 100644 bootpkg/src/prelude.rs create mode 100644 build-pkg.toml create mode 100644 manifest/Cargo.toml create mode 100644 manifest/src/bin.rs create mode 100644 manifest/src/build.rs create mode 100644 manifest/src/fs.rs create mode 100644 manifest/src/lib.rs create mode 100644 manifest/src/package.rs create mode 100644 manifest/src/pkgr.rs create mode 100644 package.toml create mode 100644 pkgfile/Cargo.toml create mode 100644 pkgfile/pkgfile.txt create mode 100644 pkgfile/src/lib.rs create mode 100644 pkgr/Cargo.toml create mode 100644 pkgr/src/main.rs diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..972b0c4 --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +**/target +**/Cargo.lock diff --git a/.idea/.gitignore b/.idea/.gitignore new file mode 100644 index 0000000..13566b8 --- /dev/null +++ b/.idea/.gitignore @@ -0,0 +1,8 @@ +# Default ignored files +/shelf/ +/workspace.xml +# Editor-based HTTP Client requests +/httpRequests/ +# Datasource local storage ignored files +/dataSources/ +/dataSources.local.xml diff --git a/.idea/discord.xml b/.idea/discord.xml new file mode 100644 index 0000000..d8e9561 --- /dev/null +++ b/.idea/discord.xml @@ -0,0 +1,7 @@ + + + + + \ No newline at end of file diff --git a/.idea/modules.xml b/.idea/modules.xml new file mode 100644 index 0000000..9fe00b2 --- /dev/null +++ b/.idea/modules.xml @@ -0,0 +1,8 @@ + + + + + + + + \ No newline at end of file diff --git a/.idea/pkgr.iml b/.idea/pkgr.iml new file mode 100644 index 0000000..c67ef27 --- /dev/null +++ b/.idea/pkgr.iml @@ -0,0 +1,17 @@ + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/vcs.xml b/.idea/vcs.xml new file mode 100644 index 0000000..35eb1dd --- /dev/null +++ b/.idea/vcs.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..7e24755 --- /dev/null +++ b/README.md @@ -0,0 +1,8 @@ +# Packager + +> "A package manager and builder but like rust." +> -- Gandhi (2050) + +> ***not even close to done*** :) + +Packager is a simple yet powerful package manager \ No newline at end of file diff --git a/bootpkg/Cargo.toml b/bootpkg/Cargo.toml new file mode 100644 index 0000000..c48277f --- /dev/null +++ b/bootpkg/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "bootpkg" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +manifest = { path = "../manifest" } +pkgfile = { path = "../pkgfile" } +regex = "1.9.1" +reqwest = { version = "0.11.18", features = ["blocking"] } diff --git a/bootpkg/src/args.rs b/bootpkg/src/args.rs new file mode 100644 index 0000000..2f51141 --- /dev/null +++ b/bootpkg/src/args.rs @@ -0,0 +1,29 @@ +pub enum Command { + Strap, + Unpack, + None +} + +impl From for Command { + fn from(value: String) -> Self { + match value.to_lowercase().as_str() { + "strap" => Command::Strap, + "unpack" => Command::Unpack, + _ => Command::None + } + } +} + +pub struct Args { + pub command: Command, + pub args: Vec, +} + +impl From> for Args { + fn from(value: Vec) -> Self { + Args { + command: Command::from(value[0].to_owned()), + args: value[1..].to_owned() + } + } +} \ No newline at end of file diff --git a/bootpkg/src/main.rs b/bootpkg/src/main.rs new file mode 100644 index 0000000..8339041 --- /dev/null +++ b/bootpkg/src/main.rs @@ -0,0 +1,51 @@ +use std::env; +use std::process::exit; +use pkgfile::PKGFile; +use crate::args::{Args, Command}; + +mod prelude; +mod args; + +fn main() { + let args = Args::from(env::args().collect::>()[1..].to_owned()); + match args.command { + Command::Strap => { + if args.args.len() == 0 { + println!("no path/uri"); + exit(0); + } + let uri = &args.args[0]; + println!("!! Got package: \"{}\"", uri); + let re = r"[A-z]+://.+"; + let data = if regex::Regex::new(re).unwrap().is_match(uri) { + println!("** detected uri."); + reqwest::blocking::get(uri) + .expect("Could not request file.") + .bytes() + .expect("Could not turn file into bytes.") + .to_vec() + } else { + if let Ok(d) = std::fs::read(uri) { + d + } else { + println!("Could not read file!"); + exit(1); + } + }; + + let p = match PKGFile::try_from(data) { + Ok(p) => p, + Err(_) => { + println!("!! Could not interpret PKGFile..."); + exit(1); + } + }; + + }, + Command::Unpack => todo!(), + _ => { + println!("Unsupported command, allowed commands: strap/unpack."); + exit(1); + } + } +} diff --git a/bootpkg/src/prelude.rs b/bootpkg/src/prelude.rs new file mode 100644 index 0000000..4462619 --- /dev/null +++ b/bootpkg/src/prelude.rs @@ -0,0 +1,22 @@ +use std::collections::HashMap; +use manifest::Manifest; + +pub struct PKGR { + pub bootstrap: Option, +} + +pub struct Bootstrap {} + +pub fn def_manifest() { + Manifest::> { + package: manifest::package::Package::default(), + bin: None, + build: None, + dependencies: HashMap::default(), + fs: manifest::fs::FS { + config: None, + data: None, + }, + pkgr: Some(PKGR { bootstrap: None }), + }; +} \ No newline at end of file diff --git a/build-pkg.toml b/build-pkg.toml new file mode 100644 index 0000000..8c994d9 --- /dev/null +++ b/build-pkg.toml @@ -0,0 +1,14 @@ +[project] +package_file = "./package.toml" +package_dir = "./pkg" +increment_version = false + +[mappings.source] +ignore = ["target/"] + +[mappings.source.map] +"." = "/src" + +[mappings.bin.map] +"target/release/bin/pkgr" = "/bin/pkgr" + diff --git a/manifest/Cargo.toml b/manifest/Cargo.toml new file mode 100644 index 0000000..139b38f --- /dev/null +++ b/manifest/Cargo.toml @@ -0,0 +1,10 @@ +[package] +name = "manifest" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +toml = "0.7.6" +serde = { version = "1.0.171", features = ["derive"] } \ No newline at end of file diff --git a/manifest/src/bin.rs b/manifest/src/bin.rs new file mode 100644 index 0000000..10d1ea1 --- /dev/null +++ b/manifest/src/bin.rs @@ -0,0 +1,7 @@ +use serde::{Serialize, Deserialize}; + +#[derive(Serialize, Deserialize)] +pub struct Bin { + pub root: String, + +} \ No newline at end of file diff --git a/manifest/src/build.rs b/manifest/src/build.rs new file mode 100644 index 0000000..0ad59ec --- /dev/null +++ b/manifest/src/build.rs @@ -0,0 +1,10 @@ +use std::collections::HashMap; + +use serde::{Serialize, Deserialize}; + +#[derive(Serialize, Deserialize)] +pub struct Build { + build_script: String, + install_script: String, + dependencies: HashMap +} \ No newline at end of file diff --git a/manifest/src/fs.rs b/manifest/src/fs.rs new file mode 100644 index 0000000..e02312f --- /dev/null +++ b/manifest/src/fs.rs @@ -0,0 +1,16 @@ +use serde::{Deserialize, Serialize}; + +#[derive(Serialize, Deserialize, Debug)] +pub struct FS { + pub config: Option, + pub data: Option, +} + +impl Default for FS { + fn default() -> Self { + FS { + config: None, + data: None, + } + } +} diff --git a/manifest/src/lib.rs b/manifest/src/lib.rs new file mode 100644 index 0000000..fdc8cbc --- /dev/null +++ b/manifest/src/lib.rs @@ -0,0 +1,32 @@ +use serde::{Deserialize, Serialize}; +use std::collections::HashMap; + +pub mod package; +pub mod fs; +pub mod bin; +pub mod build; +pub mod pkgr; + +#[derive(Serialize, Deserialize)] +#[serde(default)] +pub struct Manifest

> { + pub package: package::Package, + pub dependencies: HashMap, + pub fs: fs::FS, + pub bin: Option, + pub build: Option, + pub pkgr: P +} + +impl Default for Manifest { + fn default() -> Self { + Manifest { + package: package::Package::default(), + dependencies: HashMap::default(), + fs: fs::FS::default(), + bin: None, + build: None, + pkgr: None + } + } +} diff --git a/manifest/src/package.rs b/manifest/src/package.rs new file mode 100644 index 0000000..3fa1626 --- /dev/null +++ b/manifest/src/package.rs @@ -0,0 +1,45 @@ +use std::str::FromStr; + +use serde::{Serialize, Deserialize}; + +#[derive(Serialize, Deserialize, Debug)] +pub enum PackageType { + #[serde(rename = "application")] + Application, + #[serde(rename = "library")] + Library, + #[serde(rename = "meta")] + Meta +} + +#[derive(Serialize, Deserialize, Debug)] +#[serde(default)] +pub struct Package { + pub name: String, + pub description: String, + pub package_type: PackageType, + pub version: u64, + pub tags: Vec, + pub arch: String +} + +impl Default for Package { + fn default() -> Self { + Package { + name: String::default(), + description: String::default(), + package_type: PackageType::Application, + version: 0, + tags: Vec::default(), + arch: std::env::consts::ARCH.to_string() + } + } +} + +impl FromStr for Package { + type Err = toml::de::Error; + + fn from_str(s: &str) -> Result { + toml::from_str(s) + } +} \ No newline at end of file diff --git a/manifest/src/pkgr.rs b/manifest/src/pkgr.rs new file mode 100644 index 0000000..2ca5fd4 --- /dev/null +++ b/manifest/src/pkgr.rs @@ -0,0 +1,4 @@ +use serde::{Serialize, Deserialize}; + +#[derive(Serialize, Deserialize)] +pub struct PKGR {} \ No newline at end of file diff --git a/package.toml b/package.toml new file mode 100644 index 0000000..fbc4070 --- /dev/null +++ b/package.toml @@ -0,0 +1,110 @@ +#* = required. + +[package] +name = "packager" #* +description = "A package installation tool" #* + +## package.version +# needs to be an int and be incremented every time +# if you use a custom version system and want to use that, you can use the tag system! +# the highest package.version on the server will be given the tag "latest" +version = 1 # this can automatically be incremented when publishing by running `pkgr publish -i ...` + +## package.tags +# you can add tags to a package to specify kinds +# there are some special tags: +# - latest +# automatically set on the last published version +# previous version will lose this tag +# - oldest +# assigned to the first version. +tags = [ + "prod", + "pkgr-spec-1" +] + +## package.type +# Supported types: +# - application +# this type specifies that this package is an application. +# - library +# this type specifies that this package is a library. +# - meta +# this type specifies that this package does not install anything upon the system except for dependencies. +type = "application" + +arch = "x86_64" # this is automatically filled by `pkgr publish ...` + +## dependencies +# you may use the following syntax +# "" or ",,..." +[dependencies] +example = { arch = "x86_64", tags = [ "stable" ], version = "v1" } +exnmbr2 = "v1,stable" + +## fs +# specify some directories for ease of use +[fs] +## fs.config +# specifiy the config path +config = "/etc/packager" +## fs.data +# specify the data path +data = "/var/lib/packager" +## replace "pacakger" with your package name to find the default value. + +## bin +# Used for systems that don't want to build pkgs. +[bin] # binary files root + +## bin.root +# ** RELATIVE TO PACKAGE ROOT ** +# bin.root specifies the root of the installed tree. +# anything in here will be overlayed on top of the system. +root = "/root" #* + +## bin.checksums +# ** KEYS are relative to BIN.ROOT ** +# ** VALUES are relative to PKG ROOT ** +# checksums is used to perform checksums before install +# values may be paths or a uri. You may include variables. +# supported variables: +# - @name +# - @version +# - @display_version +[bin.checksums] +"/usr/bin/pkgr" = "https://ixvd.net/checksums/packager/@version" + +## build +# Scripts will be copied to a custom root. +# After the pacakge is built, the install script is ran. +# After the install script the build directory will be deleted and an CTREE (changed tree) will be generated. +# Then the CTREE will be used to copy the files over. +[build] +build_script = "/scripts/build" # relative to pkg +install_script = "/scripts/install" # relative to pkg + +[build.dependencies] +base = "latest,stable" # selected by default + +## pkgr.* +# packager is the official client but you may use other clients supporting the "pkgr v1 spec". +# other clients may offer extra functionality this must be put under "pkgr.*" +[pkgr] + +## pkgr.bootstrap +# This section is used for bootpkg. An edition of packager that bootstraps the full version. +# This exists so that packager is easy to install on anything! +# and only 1 release channel for pkgr +[pkgr.bootstrap] +# any non-zero = installed +check_installed_commands = [ + "/scripts/check_installed" +] + +# any non-zero = fail +commands = [ + "/scripts/bootstrap/download_latest @version /tmp/pkgr.pkg", + "/scripts/bootstrap/dirty_install /tmp/pkgr.pkg", + "/scripts/bootstrap/check_install" +] diff --git a/pkgfile/Cargo.toml b/pkgfile/Cargo.toml new file mode 100644 index 0000000..5fb57fe --- /dev/null +++ b/pkgfile/Cargo.toml @@ -0,0 +1,10 @@ +[package] +name = "pkgfile" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +manifest = { path = "../manifest" } +tar = "0.4.38" diff --git a/pkgfile/pkgfile.txt b/pkgfile/pkgfile.txt new file mode 100644 index 0000000..1592aaa --- /dev/null +++ b/pkgfile/pkgfile.txt @@ -0,0 +1,3 @@ +[pkgfile version (1 byte)][size manifest x256 (1 byte)][size manifest x1 (1 byte)] +[manifest] +[tar file] \ No newline at end of file diff --git a/pkgfile/src/lib.rs b/pkgfile/src/lib.rs new file mode 100644 index 0000000..a6b1d5e --- /dev/null +++ b/pkgfile/src/lib.rs @@ -0,0 +1,36 @@ +pub struct PKGFile { + pub manifest: String, + pub data: Vec, +} + +impl TryFrom> for PKGFile { + type Error = (); + + fn try_from(value: Vec) -> Result { + match value[0] { + 1 => { + let header: Vec = value[..3] + .iter() + .map(|v| u32::from(*v)) + .collect(); + let manifest_size: u32 = ((header[1] << 8) | header[2]); + if manifest_size > value.len() as u32 { + return Err(()); + } + Ok(PKGFile { + manifest: match String::from_utf8( + value[ + 3..(manifest_size as usize) + ].to_vec() + ) + { + Ok(s) => s, + _ => return Err(()) + }, + data: value[(manifest_size as usize)..].to_vec(), + }) + } + _ => Err(()) + } + } +} diff --git a/pkgr/Cargo.toml b/pkgr/Cargo.toml new file mode 100644 index 0000000..df095d2 --- /dev/null +++ b/pkgr/Cargo.toml @@ -0,0 +1,9 @@ +[package] +name = "pkgr" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +manifest = { path = "../manifest" } \ No newline at end of file diff --git a/pkgr/src/main.rs b/pkgr/src/main.rs new file mode 100644 index 0000000..f79c691 --- /dev/null +++ b/pkgr/src/main.rs @@ -0,0 +1,2 @@ +fn main() { +} From 5083e2ba9b5ca0194772b135c5a32657ff7e478b Mon Sep 17 00:00:00 2001 From: Raine Date: Sat, 14 Oct 2023 22:39:39 +0200 Subject: [PATCH 042/123] feat: fixed file management --- .idea/pkgr.iml | 6 ++++ bootpkg/Cargo.toml | 3 ++ bootpkg/src/args.rs | 9 +++--- bootpkg/src/main.rs | 44 ++++++++++++++++++++++----- bootpkg/src/prelude.rs | 41 ++++++++++++++++--------- docs/pkgr/flow.md | 3 ++ manifest/src/bin.rs | 3 +- manifest/src/lib.rs | 18 ++++++++++- mock-pkg.py | 22 ++++++++++++++ package.toml | 19 +++++------- pkg/scripts/bootstrap/download_latest | 7 +++++ pkgfile/src/lib.rs | 16 ++++++++-- 12 files changed, 148 insertions(+), 43 deletions(-) create mode 100644 docs/pkgr/flow.md create mode 100644 mock-pkg.py create mode 100644 pkg/scripts/bootstrap/download_latest diff --git a/.idea/pkgr.iml b/.idea/pkgr.iml index c67ef27..c93234c 100644 --- a/.idea/pkgr.iml +++ b/.idea/pkgr.iml @@ -1,5 +1,10 @@ + + + + + @@ -13,5 +18,6 @@ + \ No newline at end of file diff --git a/bootpkg/Cargo.toml b/bootpkg/Cargo.toml index c48277f..8292150 100644 --- a/bootpkg/Cargo.toml +++ b/bootpkg/Cargo.toml @@ -8,5 +8,8 @@ edition = "2021" [dependencies] manifest = { path = "../manifest" } pkgfile = { path = "../pkgfile" } +toml = { version = "0.7.6", features = [ "parse" ] } +serde = { version = "1.0.171", features = ["derive"] } regex = "1.9.1" reqwest = { version = "0.11.18", features = ["blocking"] } +uuid = { version = "1.4.0", features = ["serde", "v4"] } diff --git a/bootpkg/src/args.rs b/bootpkg/src/args.rs index 2f51141..cf061e3 100644 --- a/bootpkg/src/args.rs +++ b/bootpkg/src/args.rs @@ -1,7 +1,7 @@ pub enum Command { Strap, Unpack, - None + None, } impl From for Command { @@ -9,11 +9,10 @@ impl From for Command { match value.to_lowercase().as_str() { "strap" => Command::Strap, "unpack" => Command::Unpack, - _ => Command::None + _ => Command::None, } } } - pub struct Args { pub command: Command, pub args: Vec, @@ -23,7 +22,7 @@ impl From> for Args { fn from(value: Vec) -> Self { Args { command: Command::from(value[0].to_owned()), - args: value[1..].to_owned() + args: value[1..].to_owned(), } } -} \ No newline at end of file +} diff --git a/bootpkg/src/main.rs b/bootpkg/src/main.rs index 8339041..f59cf01 100644 --- a/bootpkg/src/main.rs +++ b/bootpkg/src/main.rs @@ -1,10 +1,15 @@ -use std::env; -use std::process::exit; -use pkgfile::PKGFile; use crate::args::{Args, Command}; +use pkgfile::PKGFile; +use std::env; +use std::env::temp_dir; +use std::fs::{create_dir, create_dir_all, File}; +use std::io::Write; +use std::path::PathBuf; +use std::process::exit; +use uuid::Uuid; -mod prelude; mod args; +mod prelude; fn main() { let args = Args::from(env::args().collect::>()[1..].to_owned()); @@ -33,7 +38,7 @@ fn main() { } }; - let p = match PKGFile::try_from(data) { + let pkg = match PKGFile::try_from(data) { Ok(p) => p, Err(_) => { println!("!! Could not interpret PKGFile..."); @@ -41,8 +46,33 @@ fn main() { } }; - }, - Command::Unpack => todo!(), + let mani = prelude::mani_from_str(&pkg.manifest); + + println!("!! Package: {}", &mani.package.name); + + if !mani.valid() { + println!("!!! Manifest is not valid."); + exit(1); + } + + let mut tmp_dir = temp_dir(); + tmp_dir.push(format!("{}", Uuid::new_v4())); + create_dir_all(&tmp_dir) + .expect("Could not create tmp dir."); + + { + let mut file = File::create(format!("{}/pkgtar", tmp_dir.to_str().unwrap())) + .expect("Could not create tmp pkgtar file"); + + file.write(&pkg.data) + .expect("Could not write pkgtar to tmp file"); + } + + // TODO: untar + // TODO: cd into pkg + // TODO: run bootstrap commands + } + Command::Unpack => {} _ => { println!("Unsupported command, allowed commands: strap/unpack."); exit(1); diff --git a/bootpkg/src/prelude.rs b/bootpkg/src/prelude.rs index 4462619..b62eda1 100644 --- a/bootpkg/src/prelude.rs +++ b/bootpkg/src/prelude.rs @@ -1,22 +1,33 @@ -use std::collections::HashMap; -use manifest::Manifest; +use manifest::{self, Manifest}; +use serde::{Deserialize, Serialize}; +use std::{collections::HashMap, str::FromStr}; +use toml; +#[derive(Serialize, Deserialize)] pub struct PKGR { pub bootstrap: Option, } -pub struct Bootstrap {} +#[derive(Serialize, Deserialize)] +pub struct Bootstrap { + check_installed_commands: Vec, + commands: Vec +} -pub fn def_manifest() { +#[derive(Serialize, Deserialize)] +pub struct BadManifest { + pkgr: Option, +} + +pub fn mani_from_str(s: &str) -> Manifest> { + let mani = Manifest::from_str(s).unwrap(); + let bmani = toml::from_str::(s).unwrap(); Manifest::> { - package: manifest::package::Package::default(), - bin: None, - build: None, - dependencies: HashMap::default(), - fs: manifest::fs::FS { - config: None, - data: None, - }, - pkgr: Some(PKGR { bootstrap: None }), - }; -} \ No newline at end of file + package: mani.package, + dependencies: mani.dependencies, + fs: mani.fs, + bin: mani.bin, + build: mani.build, + pkgr: bmani.pkgr + } +} diff --git a/docs/pkgr/flow.md b/docs/pkgr/flow.md new file mode 100644 index 0000000..d6a9a40 --- /dev/null +++ b/docs/pkgr/flow.md @@ -0,0 +1,3 @@ +# Install flow + +pkgr keeps track of installed files and therefore we have a CTREE. diff --git a/manifest/src/bin.rs b/manifest/src/bin.rs index 10d1ea1..e8b48a3 100644 --- a/manifest/src/bin.rs +++ b/manifest/src/bin.rs @@ -1,7 +1,8 @@ +use std::collections::HashMap; use serde::{Serialize, Deserialize}; #[derive(Serialize, Deserialize)] pub struct Bin { pub root: String, - + pub checksums: HashMap } \ No newline at end of file diff --git a/manifest/src/lib.rs b/manifest/src/lib.rs index fdc8cbc..71efe31 100644 --- a/manifest/src/lib.rs +++ b/manifest/src/lib.rs @@ -1,5 +1,5 @@ use serde::{Deserialize, Serialize}; -use std::collections::HashMap; +use std::{collections::HashMap, str::FromStr}; pub mod package; pub mod fs; @@ -18,6 +18,22 @@ pub struct Manifest

> { pub pkgr: P } +impl Manifest { + pub fn valid(&self) -> bool { + if self.bin.is_none() && self.build.is_none() { + return false + } + true + } +} + +impl FromStr for Manifest { + type Err = toml::de::Error; + fn from_str(s: &str) -> Result { + toml::from_str(s) + } +} + impl Default for Manifest { fn default() -> Self { Manifest { diff --git a/mock-pkg.py b/mock-pkg.py new file mode 100644 index 0000000..57ee008 --- /dev/null +++ b/mock-pkg.py @@ -0,0 +1,22 @@ +import tarfile, os + +with open("./package.toml", mode='r') as mani: + data = mani.read() + with open("./mock.pkg", mode='wb') as pkg: + print("building header...") + pkg.write(bytes([0x01, (len(data) >> 8) & 0xFF, len(data) & 0xFF])) + print("writing manifest into pkg...") + pkg.write(data.encode("utf-8")) + with tarfile.TarFile("/tmp/pkgtar", 'w') as pkgtar: + for root, dirs, files in os.walk("."): + for file in files: + print(f"\33[2Kadd: {file}", end="\r", flush=True) + pkgtar.add(os.path.join(root, file)) + with open("/tmp/pkgtar", 'rb') as pkgtar: + print("appending pkgtar to pkg...") + pkg.write(pkgtar.read()) + print("deleting /tmp/pkgtar...") + os.unlink("/tmp/pkgtar") + print("closing write stream") + pkg.close() + \ No newline at end of file diff --git a/package.toml b/package.toml index fbc4070..54d55b5 100644 --- a/package.toml +++ b/package.toml @@ -18,10 +18,7 @@ version = 1 # this can automatically be incremented when publishing by running ` # previous version will lose this tag # - oldest # assigned to the first version. -tags = [ - "prod", - "pkgr-spec-1" -] +tags = [ "prod", "pkgr-spec-1" ] ## package.type # Supported types: @@ -39,7 +36,7 @@ arch = "x86_64" # this is automatically filled by `pkgr publish ...` # you may use the following syntax # "" or ",,..." [dependencies] -example = { arch = "x86_64", tags = [ "stable" ], version = "v1" } +#example = { arch = "x86_64", tags = [ "stable" ], version = "v1" } exnmbr2 = "v1,stable" ## fs @@ -97,14 +94,12 @@ base = "latest,stable" # selected by default # This exists so that packager is easy to install on anything! # and only 1 release channel for pkgr [pkgr.bootstrap] -# any non-zero = installed -check_installed_commands = [ - "/scripts/check_installed" -] +## any non-zero = installed +check_installed_commands = [ "/scripts/check_installed" ] # any non-zero = fail commands = [ - "/scripts/bootstrap/download_latest @version /tmp/pkgr.pkg", - "/scripts/bootstrap/dirty_install /tmp/pkgr.pkg", - "/scripts/bootstrap/check_install" + "/scripts/bootstrap/download_latest @version /tmp/pkgr.pkg", + "/scripts/bootstrap/dirty_install /tmp/pkgr.pkg", + "/scripts/bootstrap/check_install" ] diff --git a/pkg/scripts/bootstrap/download_latest b/pkg/scripts/bootstrap/download_latest new file mode 100644 index 0000000..229aaa2 --- /dev/null +++ b/pkg/scripts/bootstrap/download_latest @@ -0,0 +1,7 @@ +#!/bin/sh + +fetch_latest_version() { +} + +download_file() { +} \ No newline at end of file diff --git a/pkgfile/src/lib.rs b/pkgfile/src/lib.rs index a6b1d5e..6025cd5 100644 --- a/pkgfile/src/lib.rs +++ b/pkgfile/src/lib.rs @@ -20,17 +20,29 @@ impl TryFrom> for PKGFile { Ok(PKGFile { manifest: match String::from_utf8( value[ - 3..(manifest_size as usize) + 3..(manifest_size as usize + 3) ].to_vec() ) { Ok(s) => s, _ => return Err(()) }, - data: value[(manifest_size as usize)..].to_vec(), + data: value[(manifest_size as usize + 3)..].to_vec(), }) } _ => Err(()) } } } + +impl Into> for PKGFile { + fn into(self) -> Vec { + let mut bytes = vec![0x01]; + let manifest_bytes = self.manifest.into_bytes(); + bytes.push((manifest_bytes.len() >> 8) as u8); + bytes.push((manifest_bytes.len() | 0xFF) as u8); + bytes.extend(manifest_bytes); + bytes.extend(self.data); + bytes + } +} \ No newline at end of file From dffa7f536ac90b1483ff7437b0289bd9370e46c0 Mon Sep 17 00:00:00 2001 From: Raine Date: Sat, 14 Oct 2023 22:39:39 +0200 Subject: [PATCH 043/123] feat: file management and tempdir management --- bootpkg/src/main.rs | 25 +++++++++++++++++++------ bootpkg/src/prelude.rs | 6 ++++-- manifest/src/bin.rs | 3 ++- manifest/src/build.rs | 2 +- manifest/src/lib.rs | 3 ++- manifest/src/package.rs | 2 +- manifest/src/pkgr.rs | 2 +- mock-pkg.py | 3 ++- 8 files changed, 32 insertions(+), 14 deletions(-) diff --git a/bootpkg/src/main.rs b/bootpkg/src/main.rs index f59cf01..8853c9b 100644 --- a/bootpkg/src/main.rs +++ b/bootpkg/src/main.rs @@ -1,13 +1,15 @@ -use crate::args::{Args, Command}; -use pkgfile::PKGFile; use std::env; use std::env::temp_dir; -use std::fs::{create_dir, create_dir_all, File}; +use std::fs::{create_dir_all, File, remove_dir_all}; use std::io::Write; -use std::path::PathBuf; use std::process::exit; + use uuid::Uuid; +use pkgfile::PKGFile; + +use crate::args::{Args, Command}; + mod args; mod prelude; @@ -30,6 +32,7 @@ fn main() { .expect("Could not turn file into bytes.") .to_vec() } else { + println!("** reading data..."); if let Ok(d) = std::fs::read(uri) { d } else { @@ -38,6 +41,7 @@ fn main() { } }; + println!("** parsing pkgfile..."); let pkg = match PKGFile::try_from(data) { Ok(p) => p, Err(_) => { @@ -48,7 +52,10 @@ fn main() { let mani = prelude::mani_from_str(&pkg.manifest); - println!("!! Package: {}", &mani.package.name); + println!("-- Package: {}", &mani.package.name); + println!("-- Description: {}", &mani.package.description); + println!("-- Version: {}", &mani.package.version); + println!("-- Tags: {}", &mani.package.tags.join(", ")); if !mani.valid() { println!("!!! Manifest is not valid."); @@ -61,16 +68,22 @@ fn main() { .expect("Could not create tmp dir."); { - let mut file = File::create(format!("{}/pkgtar", tmp_dir.to_str().unwrap())) + println!("** writing tar to tmp file..."); + let mut file = File::create(format!("{}/pkgtar", tmp_dir.to_str().unwrap_or("/tmp"))) .expect("Could not create tmp pkgtar file"); file.write(&pkg.data) .expect("Could not write pkgtar to tmp file"); } + // TODO: untar // TODO: cd into pkg // TODO: run bootstrap commands + + println!("** removing temporary directory..."); + remove_dir_all(&tmp_dir) + .expect(&*format!("Could not remove tmp dir: {}", &tmp_dir.to_str().unwrap())); } Command::Unpack => {} _ => { diff --git a/bootpkg/src/prelude.rs b/bootpkg/src/prelude.rs index b62eda1..13a9936 100644 --- a/bootpkg/src/prelude.rs +++ b/bootpkg/src/prelude.rs @@ -1,8 +1,10 @@ -use manifest::{self, Manifest}; +use std::str::FromStr; + use serde::{Deserialize, Serialize}; -use std::{collections::HashMap, str::FromStr}; use toml; +use manifest::{self, Manifest}; + #[derive(Serialize, Deserialize)] pub struct PKGR { pub bootstrap: Option, diff --git a/manifest/src/bin.rs b/manifest/src/bin.rs index e8b48a3..a7af0e7 100644 --- a/manifest/src/bin.rs +++ b/manifest/src/bin.rs @@ -1,5 +1,6 @@ use std::collections::HashMap; -use serde::{Serialize, Deserialize}; + +use serde::{Deserialize, Serialize}; #[derive(Serialize, Deserialize)] pub struct Bin { diff --git a/manifest/src/build.rs b/manifest/src/build.rs index 0ad59ec..a727d49 100644 --- a/manifest/src/build.rs +++ b/manifest/src/build.rs @@ -1,6 +1,6 @@ use std::collections::HashMap; -use serde::{Serialize, Deserialize}; +use serde::{Deserialize, Serialize}; #[derive(Serialize, Deserialize)] pub struct Build { diff --git a/manifest/src/lib.rs b/manifest/src/lib.rs index 71efe31..c894ec5 100644 --- a/manifest/src/lib.rs +++ b/manifest/src/lib.rs @@ -1,6 +1,7 @@ -use serde::{Deserialize, Serialize}; use std::{collections::HashMap, str::FromStr}; +use serde::{Deserialize, Serialize}; + pub mod package; pub mod fs; pub mod bin; diff --git a/manifest/src/package.rs b/manifest/src/package.rs index 3fa1626..528aa29 100644 --- a/manifest/src/package.rs +++ b/manifest/src/package.rs @@ -1,6 +1,6 @@ use std::str::FromStr; -use serde::{Serialize, Deserialize}; +use serde::{Deserialize, Serialize}; #[derive(Serialize, Deserialize, Debug)] pub enum PackageType { diff --git a/manifest/src/pkgr.rs b/manifest/src/pkgr.rs index 2ca5fd4..8c4ce55 100644 --- a/manifest/src/pkgr.rs +++ b/manifest/src/pkgr.rs @@ -1,4 +1,4 @@ -use serde::{Serialize, Deserialize}; +use serde::{Deserialize, Serialize}; #[derive(Serialize, Deserialize)] pub struct PKGR {} \ No newline at end of file diff --git a/mock-pkg.py b/mock-pkg.py index 57ee008..70ba501 100644 --- a/mock-pkg.py +++ b/mock-pkg.py @@ -1,4 +1,5 @@ -import tarfile, os +import os +import tarfile with open("./package.toml", mode='r') as mani: data = mani.read() From 59b77fd07f4c98c5f59333d1f88d8c43fba1781c Mon Sep 17 00:00:00 2001 From: Raine Date: Sat, 14 Oct 2023 22:39:40 +0200 Subject: [PATCH 044/123] build: bodge package. --- .gitignore | 1 + mock-pkg.py => bodge-pkg.py | 0 2 files changed, 1 insertion(+) rename mock-pkg.py => bodge-pkg.py (100%) diff --git a/.gitignore b/.gitignore index 972b0c4..be024c4 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ **/target **/Cargo.lock +*.pkg \ No newline at end of file diff --git a/mock-pkg.py b/bodge-pkg.py similarity index 100% rename from mock-pkg.py rename to bodge-pkg.py From 61e3c450b735a085250911bcad14dc290804aabb Mon Sep 17 00:00:00 2001 From: Raine Date: Sat, 14 Oct 2023 22:39:40 +0200 Subject: [PATCH 045/123] feat: implemented the whole process --- bootpkg/Cargo.toml | 1 + bootpkg/src/args.rs | 2 -- bootpkg/src/main.rs | 27 ++++++++++----------- bootpkg/src/prelude.rs | 35 +++++++++++++++++++++++++-- package.toml | 14 ++++++----- pkg/scripts/bootstrap/dirty_install | 3 +++ pkg/scripts/bootstrap/download_latest | 2 ++ pkg/scripts/check_installed | 3 +++ pkgfile/src/lib.rs | 2 +- 9 files changed, 64 insertions(+), 25 deletions(-) create mode 100644 pkg/scripts/bootstrap/dirty_install create mode 100644 pkg/scripts/check_installed diff --git a/bootpkg/Cargo.toml b/bootpkg/Cargo.toml index 8292150..d6c3186 100644 --- a/bootpkg/Cargo.toml +++ b/bootpkg/Cargo.toml @@ -13,3 +13,4 @@ serde = { version = "1.0.171", features = ["derive"] } regex = "1.9.1" reqwest = { version = "0.11.18", features = ["blocking"] } uuid = { version = "1.4.0", features = ["serde", "v4"] } +tar = "0.4.39" diff --git a/bootpkg/src/args.rs b/bootpkg/src/args.rs index cf061e3..0df76c9 100644 --- a/bootpkg/src/args.rs +++ b/bootpkg/src/args.rs @@ -1,6 +1,5 @@ pub enum Command { Strap, - Unpack, None, } @@ -8,7 +7,6 @@ impl From for Command { fn from(value: String) -> Self { match value.to_lowercase().as_str() { "strap" => Command::Strap, - "unpack" => Command::Unpack, _ => Command::None, } } diff --git a/bootpkg/src/main.rs b/bootpkg/src/main.rs index 8853c9b..6bc16a8 100644 --- a/bootpkg/src/main.rs +++ b/bootpkg/src/main.rs @@ -1,7 +1,7 @@ use std::env; use std::env::temp_dir; -use std::fs::{create_dir_all, File, remove_dir_all}; -use std::io::Write; +use std::fs::{create_dir_all, remove_dir_all}; +use std::io::Cursor; use std::process::exit; use uuid::Uuid; @@ -9,6 +9,7 @@ use uuid::Uuid; use pkgfile::PKGFile; use crate::args::{Args, Command}; +use crate::prelude::run_bootstrap; mod args; mod prelude; @@ -67,27 +68,25 @@ fn main() { create_dir_all(&tmp_dir) .expect("Could not create tmp dir."); + println!("** extracting pkgtar..."); { - println!("** writing tar to tmp file..."); - let mut file = File::create(format!("{}/pkgtar", tmp_dir.to_str().unwrap_or("/tmp"))) - .expect("Could not create tmp pkgtar file"); - - file.write(&pkg.data) - .expect("Could not write pkgtar to tmp file"); + let mut archive = tar::Archive::new(Cursor::new(pkg.data)); + archive.unpack(format!("{}/contents", &tmp_dir.to_str().unwrap())) + .expect("Could not extract archive"); } + env::set_current_dir(format!("{}/contents", &tmp_dir.to_str().unwrap())) + .expect("could not enter tmp dir"); - // TODO: untar - // TODO: cd into pkg - // TODO: run bootstrap commands - + println!("** running bootstrap commands..."); + let res_bootstrap = run_bootstrap(mani); println!("** removing temporary directory..."); remove_dir_all(&tmp_dir) .expect(&*format!("Could not remove tmp dir: {}", &tmp_dir.to_str().unwrap())); + exit(if res_bootstrap { println!("!! bootstrap success"); 0 } else { 1 }) } - Command::Unpack => {} _ => { - println!("Unsupported command, allowed commands: strap/unpack."); + println!("Unsupported command, allowed commands: strap."); exit(1); } } diff --git a/bootpkg/src/prelude.rs b/bootpkg/src/prelude.rs index 13a9936..aff3164 100644 --- a/bootpkg/src/prelude.rs +++ b/bootpkg/src/prelude.rs @@ -12,8 +12,8 @@ pub struct PKGR { #[derive(Serialize, Deserialize)] pub struct Bootstrap { - check_installed_commands: Vec, - commands: Vec + pub check_installed_commands: Vec, + pub commands: Vec } #[derive(Serialize, Deserialize)] @@ -33,3 +33,34 @@ pub fn mani_from_str(s: &str) -> Manifest> { pkgr: bmani.pkgr } } + +pub fn run_bootstrap(mani: Manifest>) -> bool { + if let Some(pkgr) = mani.pkgr { + if let Some(bootstrap) = pkgr.bootstrap { + fn run_command>(s: S) -> i32 { + std::process::Command::new("sh") + .arg("-c") + .arg(s.into()) + .spawn().expect("Could not spawn process.") + .wait().expect("Could not wait for process.") + .code().expect("Could not fetch exit code.") + } + + for command in &bootstrap.check_installed_commands { + if run_command(command) != 0 { + println!("!! Command failed: {}", command); + println!("!! Already installed."); + return false + } + } + for command in &bootstrap.commands { + if run_command(command) != 0 { + println!("!! Command failed: {}", command); + println!("!! Bootstrap failed!!!"); + return false + } + } + } + } + return true +} diff --git a/package.toml b/package.toml index 54d55b5..c99108d 100644 --- a/package.toml +++ b/package.toml @@ -78,8 +78,8 @@ root = "/root" #* # After the install script the build directory will be deleted and an CTREE (changed tree) will be generated. # Then the CTREE will be used to copy the files over. [build] -build_script = "/scripts/build" # relative to pkg -install_script = "/scripts/install" # relative to pkg +build_script = "scripts/build" # relative to pkg +install_script = "scripts/install" # relative to pkg [build.dependencies] base = "latest,stable" # selected by default @@ -95,11 +95,13 @@ base = "latest,stable" # selected by default # and only 1 release channel for pkgr [pkgr.bootstrap] ## any non-zero = installed -check_installed_commands = [ "/scripts/check_installed" ] +check_installed_commands = [ + "sh scripts/check_installed" +] # any non-zero = fail commands = [ - "/scripts/bootstrap/download_latest @version /tmp/pkgr.pkg", - "/scripts/bootstrap/dirty_install /tmp/pkgr.pkg", - "/scripts/bootstrap/check_install" + "sh scripts/bootstrap/download_latest @version /tmp/pkgr.pkg", + "sh scripts/bootstrap/dirty_install /tmp/pkgr.pkg", + "sh scripts/check_installed" ] diff --git a/pkg/scripts/bootstrap/dirty_install b/pkg/scripts/bootstrap/dirty_install new file mode 100644 index 0000000..41d3f5d --- /dev/null +++ b/pkg/scripts/bootstrap/dirty_install @@ -0,0 +1,3 @@ +#!/bin/sh + +printf "" \ No newline at end of file diff --git a/pkg/scripts/bootstrap/download_latest b/pkg/scripts/bootstrap/download_latest index 229aaa2..178bd4e 100644 --- a/pkg/scripts/bootstrap/download_latest +++ b/pkg/scripts/bootstrap/download_latest @@ -1,7 +1,9 @@ #!/bin/sh fetch_latest_version() { + printf "" } download_file() { + printf "" } \ No newline at end of file diff --git a/pkg/scripts/check_installed b/pkg/scripts/check_installed new file mode 100644 index 0000000..41d3f5d --- /dev/null +++ b/pkg/scripts/check_installed @@ -0,0 +1,3 @@ +#!/bin/sh + +printf "" \ No newline at end of file diff --git a/pkgfile/src/lib.rs b/pkgfile/src/lib.rs index 6025cd5..c3c056b 100644 --- a/pkgfile/src/lib.rs +++ b/pkgfile/src/lib.rs @@ -13,7 +13,7 @@ impl TryFrom> for PKGFile { .iter() .map(|v| u32::from(*v)) .collect(); - let manifest_size: u32 = ((header[1] << 8) | header[2]); + let manifest_size: u32 = (header[1] << 8) | header[2]; if manifest_size > value.len() as u32 { return Err(()); } From 6e30a8bed71f4234ba14cc19c212d71df4f9af3b Mon Sep 17 00:00:00 2001 From: Raine Date: Sat, 14 Oct 2023 22:39:40 +0200 Subject: [PATCH 046/123] build: fixed some bodges --- bodge-pkg.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/bodge-pkg.py b/bodge-pkg.py index 70ba501..cca3307 100644 --- a/bodge-pkg.py +++ b/bodge-pkg.py @@ -3,16 +3,24 @@ import tarfile with open("./package.toml", mode='r') as mani: data = mani.read() - with open("./mock.pkg", mode='wb') as pkg: + with open("./sample.pkg", mode='wb') as pkg: print("building header...") pkg.write(bytes([0x01, (len(data) >> 8) & 0xFF, len(data) & 0xFF])) print("writing manifest into pkg...") pkg.write(data.encode("utf-8")) with tarfile.TarFile("/tmp/pkgtar", 'w') as pkgtar: + print("tarring ./pkg...") + os.chdir("pkg") for root, dirs, files in os.walk("."): for file in files: - print(f"\33[2Kadd: {file}", end="\r", flush=True) + print(f"\33[2Kadd: {os.path.join(root, file)}", end="\r", flush=True) pkgtar.add(os.path.join(root, file)) + os.chdir("..") + for root, dirs, files in os.walk("pkgr"): + for file in files: + print(f"\33[2Kadd: {os.path.join(root, file)}", end="\r", flush=True) + pkgtar.add(os.path.join(root, file)) + print("\33[2K", end="\r", flush=True) with open("/tmp/pkgtar", 'rb') as pkgtar: print("appending pkgtar to pkg...") pkg.write(pkgtar.read()) From a0416cf6a91d6b22260e7101aef1289cd235c2b7 Mon Sep 17 00:00:00 2001 From: Raine Date: Sat, 14 Oct 2023 22:39:40 +0200 Subject: [PATCH 047/123] feat: add package identifiers --- pkgr/Cargo.toml | 6 +- pkgr/src/commands/mod.rs | 8 +++ pkgr/src/main.rs | 57 +++++++++++++++++ pkgr/src/package/identifier.rs | 114 +++++++++++++++++++++++++++++++++ pkgr/src/package/mod.rs | 1 + 5 files changed, 185 insertions(+), 1 deletion(-) create mode 100644 pkgr/src/commands/mod.rs create mode 100644 pkgr/src/package/identifier.rs create mode 100644 pkgr/src/package/mod.rs diff --git a/pkgr/Cargo.toml b/pkgr/Cargo.toml index df095d2..92a0e91 100644 --- a/pkgr/Cargo.toml +++ b/pkgr/Cargo.toml @@ -6,4 +6,8 @@ edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -manifest = { path = "../manifest" } \ No newline at end of file +fern = "0.6.2" +getopts = "0.2.21" +log = "0.4.19" +regex = "1.9.1" +manifest = { path = "../manifest" } diff --git a/pkgr/src/commands/mod.rs b/pkgr/src/commands/mod.rs new file mode 100644 index 0000000..61e3fe8 --- /dev/null +++ b/pkgr/src/commands/mod.rs @@ -0,0 +1,8 @@ +use crate::package::identifier::PackageIdentifier; + +pub enum Command { + Grab(PackageIdentifier), // grab a package from a remote source + Remove(PackageIdentifier), // remove a package from the local source + List, // list all packages in the local source + Update, +} \ No newline at end of file diff --git a/pkgr/src/main.rs b/pkgr/src/main.rs index f79c691..8d75350 100644 --- a/pkgr/src/main.rs +++ b/pkgr/src/main.rs @@ -1,2 +1,59 @@ +use std::env; +use std::process::exit; +use std::str::FromStr; + +use getopts::Options; +use log::{info, SetLoggerError}; + +mod commands; +mod package; + fn main() { + setup_logger() + .expect("Unable to setup logger."); + + let args: Vec = env::args().collect(); + let program = args[0].clone(); + + let mut opts = Options::new(); + opts.optflag("h", "help", "Show help message."); + let matches = opts.parse(&args[1..]).unwrap_or_else(|e| panic!("{}", e.to_string())); + + if matches.opt_present("h") { + print_usage(&program, opts); + exit(1); + } + + let command = if !matches.free.is_empty() { + matches.free[0].clone() + } else { + print_usage(&program, opts); + exit(1); + }; + + info!("Identifier: {}", package::identifier::PackageIdentifier::from_str(&command).unwrap()); } + +fn print_usage(program: &str, opts: Options) { + let mut brief = format!("Usage: {} [options]\n\n", program); + brief += &*format!("Commands: grab \n"); + brief += &*format!(" remove\n"); + print!("{}", opts.usage(&brief.trim_end())); +} + +fn setup_logger() -> Result<(), SetLoggerError> { + fern::Dispatch::new() + .format(|out, message, record| { + out.finish(format_args!( + "{} {}", + match record.level().to_string().chars().nth(0).unwrap_or('I') { + 'E' | 'W' => "!!", + _ => "**", + }, + message + )) + }) + .level(log::LevelFilter::Info) + .chain(std::io::stdout()) + .apply() +} \ No newline at end of file diff --git a/pkgr/src/package/identifier.rs b/pkgr/src/package/identifier.rs new file mode 100644 index 0000000..01f9f6b --- /dev/null +++ b/pkgr/src/package/identifier.rs @@ -0,0 +1,114 @@ +use std::str::FromStr; + +use regex::Regex; + +#[derive(Debug)] +pub enum PackageIdentifierError { + InvalidPackageLocator(String), + InvalidURI(String), +} + +impl std::fmt::Display for PackageIdentifierError { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + match self { + PackageIdentifierError::InvalidPackageLocator(s) => write!(f, "Invalid package locator: {}", s), + PackageIdentifierError::InvalidURI(s) => write!(f, "Invalid URI: {}", s), + } + } +} + +#[derive(Debug)] +pub enum PackageIdentifier { + PackageLocator(PackageLocator), + URI(String), +} + +impl std::fmt::Display for PackageIdentifier { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + match self { + PackageIdentifier::PackageLocator(pl) => write!(f, "{}", pl), + PackageIdentifier::URI(uri) => write!(f, "{}", uri), + } + } +} + +#[derive(Debug)] +pub struct PackageLocator { + pub name: String, + pub version: Option, + pub tags: Option>, +} + +impl std::fmt::Display for PackageLocator { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + let mut s = self.name.clone(); + if let Some(version) = self.version { + s += &*format!("@{}", version); + } + if let Some(tags) = &self.tags { + s += &*format!(":{}", tags.join(",")); + } + write!(f, "{}", s) + } +} + +impl FromStr for PackageIdentifier { + type Err = PackageIdentifierError; + + fn from_str(s: &str) -> Result { + let uri_re = regex::Regex::new(r"^[a-zA-Z0-9]+://").unwrap(); + if uri_re.is_match(s) { + // there needs to be stuff after the protocol + let split = s.split("://").collect::>(); + if split.len() != 2 || split[1].is_empty() { + return Err(PackageIdentifierError::InvalidURI(s.to_string())); + } + Ok(PackageIdentifier::URI(s.to_string())) + } else { + let pl = match PackageLocator::from_str(s) { + Ok(pl) => pl, + Err(e) => return Err(PackageIdentifierError::InvalidPackageLocator(e.to_string())), + }; + Ok(PackageIdentifier::PackageLocator(pl)) + } + } +} + +impl FromStr for PackageLocator { + type Err = PackageIdentifierError; + + fn from_str(s: &str) -> Result { + #[allow(unused_assignments)] // false positive + let mut name = None; + let mut version = None; + let mut tags = None; + + if s.is_empty() { + return Err(PackageIdentifierError::InvalidPackageLocator(s.to_string())); + } + + let name_re = Regex::new("^([A-Za-z][A-Za-z0-9_.]+)").unwrap(); + let version_re = Regex::new("@([0-9]+)").unwrap(); + let tags_re = Regex::new(":([a-zA-Z0-9,._]+)").unwrap(); + + if let Some(caps) = name_re.captures(s) { + name = Some(caps.get(1).unwrap().as_str().to_string()); + } else { + return Err(PackageIdentifierError::InvalidPackageLocator(s.to_string())); + } + + if let Some(caps) = version_re.captures(s) { + version = Some(caps.get(1).unwrap().as_str().parse::().unwrap()); + } + + if let Some(caps) = tags_re.captures(s) { + tags = Some(caps.get(1).unwrap().as_str().split(",").map(|s| s.to_string()).collect()); + } + + Ok(PackageLocator { + name: name.unwrap(), + version, + tags, + }) + } +} \ No newline at end of file diff --git a/pkgr/src/package/mod.rs b/pkgr/src/package/mod.rs new file mode 100644 index 0000000..8f83cb4 --- /dev/null +++ b/pkgr/src/package/mod.rs @@ -0,0 +1 @@ +pub mod identifier; \ No newline at end of file From 174bbe93ecc7ed32c6187c3469b117a769aae004 Mon Sep 17 00:00:00 2001 From: Raine Date: Sat, 14 Oct 2023 22:39:40 +0200 Subject: [PATCH 048/123] feat: add some functionality to bootpkg --- bootpkg/src/main.rs | 19 +++++++++++++------ bootpkg/src/prelude.rs | 19 +++++++++++-------- 2 files changed, 24 insertions(+), 14 deletions(-) diff --git a/bootpkg/src/main.rs b/bootpkg/src/main.rs index 6bc16a8..ffd4338 100644 --- a/bootpkg/src/main.rs +++ b/bootpkg/src/main.rs @@ -65,13 +65,13 @@ fn main() { let mut tmp_dir = temp_dir(); tmp_dir.push(format!("{}", Uuid::new_v4())); - create_dir_all(&tmp_dir) - .expect("Could not create tmp dir."); + create_dir_all(&tmp_dir).expect("Could not create tmp dir."); println!("** extracting pkgtar..."); { let mut archive = tar::Archive::new(Cursor::new(pkg.data)); - archive.unpack(format!("{}/contents", &tmp_dir.to_str().unwrap())) + archive + .unpack(format!("{}/contents", &tmp_dir.to_str().unwrap())) .expect("Could not extract archive"); } @@ -81,9 +81,16 @@ fn main() { println!("** running bootstrap commands..."); let res_bootstrap = run_bootstrap(mani); println!("** removing temporary directory..."); - remove_dir_all(&tmp_dir) - .expect(&*format!("Could not remove tmp dir: {}", &tmp_dir.to_str().unwrap())); - exit(if res_bootstrap { println!("!! bootstrap success"); 0 } else { 1 }) + remove_dir_all(&tmp_dir).expect(&*format!( + "Could not remove tmp dir: {}", + &tmp_dir.to_str().unwrap() + )); + exit(if res_bootstrap { + println!("!! bootstrap success"); + 0 + } else { + 1 + }) } _ => { println!("Unsupported command, allowed commands: strap."); diff --git a/bootpkg/src/prelude.rs b/bootpkg/src/prelude.rs index aff3164..b82b62a 100644 --- a/bootpkg/src/prelude.rs +++ b/bootpkg/src/prelude.rs @@ -13,7 +13,7 @@ pub struct PKGR { #[derive(Serialize, Deserialize)] pub struct Bootstrap { pub check_installed_commands: Vec, - pub commands: Vec + pub commands: Vec, } #[derive(Serialize, Deserialize)] @@ -30,7 +30,7 @@ pub fn mani_from_str(s: &str) -> Manifest> { fs: mani.fs, bin: mani.bin, build: mani.build, - pkgr: bmani.pkgr + pkgr: bmani.pkgr, } } @@ -41,26 +41,29 @@ pub fn run_bootstrap(mani: Manifest>) -> bool { std::process::Command::new("sh") .arg("-c") .arg(s.into()) - .spawn().expect("Could not spawn process.") - .wait().expect("Could not wait for process.") - .code().expect("Could not fetch exit code.") + .spawn() + .expect("Could not spawn process.") + .wait() + .expect("Could not wait for process.") + .code() + .expect("Could not fetch exit code.") } for command in &bootstrap.check_installed_commands { if run_command(command) != 0 { println!("!! Command failed: {}", command); println!("!! Already installed."); - return false + return false; } } for command in &bootstrap.commands { if run_command(command) != 0 { println!("!! Command failed: {}", command); println!("!! Bootstrap failed!!!"); - return false + return false; } } } } - return true + return true; } From ba3e75c3fb374599fd47fed559a82d52d1b649a6 Mon Sep 17 00:00:00 2001 From: Raine Date: Sat, 14 Oct 2023 22:39:40 +0200 Subject: [PATCH 049/123] fix: pkg interpreting --- pkgfile/src/lib.rs | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/pkgfile/src/lib.rs b/pkgfile/src/lib.rs index c3c056b..bc26f99 100644 --- a/pkgfile/src/lib.rs +++ b/pkgfile/src/lib.rs @@ -9,28 +9,22 @@ impl TryFrom> for PKGFile { fn try_from(value: Vec) -> Result { match value[0] { 1 => { - let header: Vec = value[..3] - .iter() - .map(|v| u32::from(*v)) - .collect(); + let header: Vec = value[..3].iter().map(|v| u32::from(*v)).collect(); let manifest_size: u32 = (header[1] << 8) | header[2]; if manifest_size > value.len() as u32 { return Err(()); } Ok(PKGFile { manifest: match String::from_utf8( - value[ - 3..(manifest_size as usize + 3) - ].to_vec() - ) - { + value[3..(manifest_size as usize + 3)].to_vec(), + ) { Ok(s) => s, - _ => return Err(()) + _ => return Err(()), }, data: value[(manifest_size as usize + 3)..].to_vec(), }) } - _ => Err(()) + _ => Err(()), } } } @@ -45,4 +39,4 @@ impl Into> for PKGFile { bytes.extend(self.data); bytes } -} \ No newline at end of file +} From d3469e76186fefef49ff2e00fbb4abd85a28883e Mon Sep 17 00:00:00 2001 From: Raine Date: Sat, 14 Oct 2023 22:39:41 +0200 Subject: [PATCH 050/123] style: formatting... --- manifest/src/bin.rs | 4 ++-- manifest/src/build.rs | 4 ++-- manifest/src/lib.rs | 10 +++++----- manifest/src/package.rs | 8 ++++---- 4 files changed, 13 insertions(+), 13 deletions(-) diff --git a/manifest/src/bin.rs b/manifest/src/bin.rs index a7af0e7..e30b528 100644 --- a/manifest/src/bin.rs +++ b/manifest/src/bin.rs @@ -5,5 +5,5 @@ use serde::{Deserialize, Serialize}; #[derive(Serialize, Deserialize)] pub struct Bin { pub root: String, - pub checksums: HashMap -} \ No newline at end of file + pub checksums: HashMap, +} diff --git a/manifest/src/build.rs b/manifest/src/build.rs index a727d49..ac223d2 100644 --- a/manifest/src/build.rs +++ b/manifest/src/build.rs @@ -6,5 +6,5 @@ use serde::{Deserialize, Serialize}; pub struct Build { build_script: String, install_script: String, - dependencies: HashMap -} \ No newline at end of file + dependencies: HashMap, +} diff --git a/manifest/src/lib.rs b/manifest/src/lib.rs index c894ec5..8a943fa 100644 --- a/manifest/src/lib.rs +++ b/manifest/src/lib.rs @@ -2,10 +2,10 @@ use std::{collections::HashMap, str::FromStr}; use serde::{Deserialize, Serialize}; -pub mod package; -pub mod fs; pub mod bin; pub mod build; +pub mod fs; +pub mod package; pub mod pkgr; #[derive(Serialize, Deserialize)] @@ -16,13 +16,13 @@ pub struct Manifest

> { pub fs: fs::FS, pub bin: Option, pub build: Option, - pub pkgr: P + pub pkgr: P, } impl Manifest { pub fn valid(&self) -> bool { if self.bin.is_none() && self.build.is_none() { - return false + return false; } true } @@ -43,7 +43,7 @@ impl Default for Manifest { fs: fs::FS::default(), bin: None, build: None, - pkgr: None + pkgr: None, } } } diff --git a/manifest/src/package.rs b/manifest/src/package.rs index 528aa29..017271d 100644 --- a/manifest/src/package.rs +++ b/manifest/src/package.rs @@ -9,7 +9,7 @@ pub enum PackageType { #[serde(rename = "library")] Library, #[serde(rename = "meta")] - Meta + Meta, } #[derive(Serialize, Deserialize, Debug)] @@ -20,7 +20,7 @@ pub struct Package { pub package_type: PackageType, pub version: u64, pub tags: Vec, - pub arch: String + pub arch: String, } impl Default for Package { @@ -31,7 +31,7 @@ impl Default for Package { package_type: PackageType::Application, version: 0, tags: Vec::default(), - arch: std::env::consts::ARCH.to_string() + arch: std::env::consts::ARCH.to_string(), } } } @@ -42,4 +42,4 @@ impl FromStr for Package { fn from_str(s: &str) -> Result { toml::from_str(s) } -} \ No newline at end of file +} From 548caa2f47a4b72bb84ea788026d7249dd26390d Mon Sep 17 00:00:00 2001 From: Raine Date: Sat, 14 Oct 2023 22:39:41 +0200 Subject: [PATCH 051/123] ci: add ci config --- .github/workflows/bootpkg.yaml | 24 ++++++++++++++++++++++++ .github/workflows/pkgr.yaml | 24 ++++++++++++++++++++++++ 2 files changed, 48 insertions(+) create mode 100644 .github/workflows/bootpkg.yaml create mode 100644 .github/workflows/pkgr.yaml diff --git a/.github/workflows/bootpkg.yaml b/.github/workflows/bootpkg.yaml new file mode 100644 index 0000000..876a53c --- /dev/null +++ b/.github/workflows/bootpkg.yaml @@ -0,0 +1,24 @@ +on: + push: + branches: + - main + +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: install dependencies + run: | + apt update -y + apt install -y curl gcc g++ make + - uses: https://github.com/actions-rs/toolchain@v1 + with: + toolchain: stable + override: true + - run: | + cd bootpkg + - uses: https://github.com/actions-rs/cargo@v1 + with: + command: build + args: --verbose --all diff --git a/.github/workflows/pkgr.yaml b/.github/workflows/pkgr.yaml new file mode 100644 index 0000000..ff8e6a4 --- /dev/null +++ b/.github/workflows/pkgr.yaml @@ -0,0 +1,24 @@ +on: + push: + branches: + - main + +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: install dependencies + run: | + apt update -y + apt install -y curl gcc g++ make + - uses: https://github.com/actions-rs/toolchain@v1 + with: + toolchain: stable + override: true + - run: | + cd pkgr + - uses: https://github.com/actions-rs/cargo@v1 + with: + command: build + args: --verbose --all From 8c9771e37275b25b2edce5404db10c3522fd5399 Mon Sep 17 00:00:00 2001 From: Raine Date: Sat, 14 Oct 2023 22:39:41 +0200 Subject: [PATCH 052/123] ci: set workdir --- .github/workflows/bootpkg.yaml | 3 +-- .github/workflows/pkgr.yaml | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/bootpkg.yaml b/.github/workflows/bootpkg.yaml index 876a53c..9a291d4 100644 --- a/.github/workflows/bootpkg.yaml +++ b/.github/workflows/bootpkg.yaml @@ -16,9 +16,8 @@ jobs: with: toolchain: stable override: true - - run: | - cd bootpkg - uses: https://github.com/actions-rs/cargo@v1 with: + cwd: bootpkg command: build args: --verbose --all diff --git a/.github/workflows/pkgr.yaml b/.github/workflows/pkgr.yaml index ff8e6a4..f1a7971 100644 --- a/.github/workflows/pkgr.yaml +++ b/.github/workflows/pkgr.yaml @@ -16,9 +16,8 @@ jobs: with: toolchain: stable override: true - - run: | - cd pkgr - uses: https://github.com/actions-rs/cargo@v1 with: + cwd: pkgr command: build args: --verbose --all From 51a3456e3468eeebf2b41cc6639711ff1f62a885 Mon Sep 17 00:00:00 2001 From: Raine Date: Sat, 14 Oct 2023 22:39:41 +0200 Subject: [PATCH 053/123] ci: set manifest path --- .github/workflows/bootpkg.yaml | 5 ++--- .github/workflows/pkgr.yaml | 5 ++--- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/.github/workflows/bootpkg.yaml b/.github/workflows/bootpkg.yaml index 9a291d4..baf6377 100644 --- a/.github/workflows/bootpkg.yaml +++ b/.github/workflows/bootpkg.yaml @@ -18,6 +18,5 @@ jobs: override: true - uses: https://github.com/actions-rs/cargo@v1 with: - cwd: bootpkg - command: build - args: --verbose --all + command: check + args: --manifest-path bootpkg/Cargo.toml --verbose --all diff --git a/.github/workflows/pkgr.yaml b/.github/workflows/pkgr.yaml index f1a7971..6084d80 100644 --- a/.github/workflows/pkgr.yaml +++ b/.github/workflows/pkgr.yaml @@ -18,6 +18,5 @@ jobs: override: true - uses: https://github.com/actions-rs/cargo@v1 with: - cwd: pkgr - command: build - args: --verbose --all + command: check + args: --manifest-path pkgr/Cargo.toml --verbose --all \ No newline at end of file From ffe6e0197d587f94718a436696a34a13173c57bd Mon Sep 17 00:00:00 2001 From: Raine Date: Sat, 14 Oct 2023 22:39:41 +0200 Subject: [PATCH 054/123] ci: merge and add step --- .github/workflows/bootpkg.yaml | 22 ------------- .github/workflows/build.yaml | 56 ++++++++++++++++++++++++++++++++++ .github/workflows/pkgr.yaml | 22 ------------- 3 files changed, 56 insertions(+), 44 deletions(-) delete mode 100644 .github/workflows/bootpkg.yaml create mode 100644 .github/workflows/build.yaml delete mode 100644 .github/workflows/pkgr.yaml diff --git a/.github/workflows/bootpkg.yaml b/.github/workflows/bootpkg.yaml deleted file mode 100644 index baf6377..0000000 --- a/.github/workflows/bootpkg.yaml +++ /dev/null @@ -1,22 +0,0 @@ -on: - push: - branches: - - main - -jobs: - build: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - name: install dependencies - run: | - apt update -y - apt install -y curl gcc g++ make - - uses: https://github.com/actions-rs/toolchain@v1 - with: - toolchain: stable - override: true - - uses: https://github.com/actions-rs/cargo@v1 - with: - command: check - args: --manifest-path bootpkg/Cargo.toml --verbose --all diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml new file mode 100644 index 0000000..52bb97b --- /dev/null +++ b/.github/workflows/build.yaml @@ -0,0 +1,56 @@ +on: + push: + branches: + - main + +jobs: + check: # check if the code compiles + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: install dependencies + run: | + apt update -y + apt install -y curl gcc g++ make + - uses: https://github.com/actions-rs/toolchain@v1 + with: + toolchain: stable + override: true + - uses: https://github.com/actions-rs/cargo@v1 + with: + command: check + args: --manifest-path bootpkg/Cargo.toml --verbose --all + - uses: https://github.com/actions-rs/cargo@v1 + with: + command: check + args: --manifest-path pkgr/Cargo.toml --verbose --all + build: # create artifacts and upload them + needs: check + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: install dependencies + run: | + apt update -y + apt install -y curl gcc g++ make + - uses: https://github.com/actions-rs/toolchain@v1 + with: + toolchain: stable + override: true + - uses: https://github.com/actions-rs/cargo@v1 + with: + command: build + args: --release --manifest-path bootpkg/Cargo.toml --verbose --all + continue-on-error: true + - uses: https://github.com/actions-rs/cargo@v1 + with: + command: build + args: --release --manifest-path pkgr/Cargo.toml --verbose --all + - uses: actions/upload-artifact@v2 + with: + name: bootpkg + path: bootpkg/target/release/bootpkg + - uses: actions/upload-artifact@v2 + with: + name: pkgr + path: pkgr/target/release/pkgr diff --git a/.github/workflows/pkgr.yaml b/.github/workflows/pkgr.yaml deleted file mode 100644 index 6084d80..0000000 --- a/.github/workflows/pkgr.yaml +++ /dev/null @@ -1,22 +0,0 @@ -on: - push: - branches: - - main - -jobs: - build: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - name: install dependencies - run: | - apt update -y - apt install -y curl gcc g++ make - - uses: https://github.com/actions-rs/toolchain@v1 - with: - toolchain: stable - override: true - - uses: https://github.com/actions-rs/cargo@v1 - with: - command: check - args: --manifest-path pkgr/Cargo.toml --verbose --all \ No newline at end of file From c61781187959487c3d5a7e90d8e8b51995421e14 Mon Sep 17 00:00:00 2001 From: Raine Date: Sat, 14 Oct 2023 22:39:41 +0200 Subject: [PATCH 055/123] ci: artifacts not supported as of right now. --- .github/workflows/build.yaml | 60 ++++++++++++++++++------------------ 1 file changed, 30 insertions(+), 30 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 52bb97b..2f75ff4 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -24,33 +24,33 @@ jobs: with: command: check args: --manifest-path pkgr/Cargo.toml --verbose --all - build: # create artifacts and upload them - needs: check - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - name: install dependencies - run: | - apt update -y - apt install -y curl gcc g++ make - - uses: https://github.com/actions-rs/toolchain@v1 - with: - toolchain: stable - override: true - - uses: https://github.com/actions-rs/cargo@v1 - with: - command: build - args: --release --manifest-path bootpkg/Cargo.toml --verbose --all - continue-on-error: true - - uses: https://github.com/actions-rs/cargo@v1 - with: - command: build - args: --release --manifest-path pkgr/Cargo.toml --verbose --all - - uses: actions/upload-artifact@v2 - with: - name: bootpkg - path: bootpkg/target/release/bootpkg - - uses: actions/upload-artifact@v2 - with: - name: pkgr - path: pkgr/target/release/pkgr +# build: # create artifacts and upload them +# needs: check +# runs-on: ubuntu-latest +# steps: +# - uses: actions/checkout@v2 +# - name: install dependencies +# run: | +# apt update -y +# apt install -y curl gcc g++ make +# - uses: https://github.com/actions-rs/toolchain@v1 +# with: +# toolchain: stable +# override: true +# - uses: https://github.com/actions-rs/cargo@v1 +# with: +# command: build +# args: --release --manifest-path bootpkg/Cargo.toml --verbose --all +# continue-on-error: true +# - uses: https://github.com/actions-rs/cargo@v1 +# with: +# command: build +# args: --release --manifest-path pkgr/Cargo.toml --verbose --all +# - uses: actions/upload-artifact@v2 +# with: +# name: bootpkg +# path: bootpkg/target/release/bootpkg +# - uses: actions/upload-artifact@v2 +# with: +# name: pkgr +# path: pkgr/target/release/pkgr From 9dfec956dec5ecea7d2fdf46f183ef460e965d34 Mon Sep 17 00:00:00 2001 From: Raine Date: Sat, 14 Oct 2023 22:39:42 +0200 Subject: [PATCH 056/123] ci: we don't have to compile for docs updates --- .github/workflows/build.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 2f75ff4..13d43f3 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -2,6 +2,11 @@ on: push: branches: - main + paths: + - 'pkgr/**' + - 'manifest/**' + - 'bootpkg/**' + - 'pkgfile/**' jobs: check: # check if the code compiles From 2898d272fb99f1b167900c14beec09dad953d23f Mon Sep 17 00:00:00 2001 From: Raine Date: Sat, 14 Oct 2023 22:39:42 +0200 Subject: [PATCH 057/123] docs: added more specification --- docs/README.md | 20 ++++++++++++++++++++ docs/pkgfile.md | 24 ++++++++++++++++++++++++ docs/pkgr/README.md | 28 ++++++++++++++++++++++++++++ docs/pkgr/installedmanifest.md | 19 +++++++++++++++++++ 4 files changed, 91 insertions(+) create mode 100644 docs/README.md create mode 100644 docs/pkgfile.md create mode 100644 docs/pkgr/README.md create mode 100644 docs/pkgr/installedmanifest.md diff --git a/docs/README.md b/docs/README.md new file mode 100644 index 0000000..0e9b295 --- /dev/null +++ b/docs/README.md @@ -0,0 +1,20 @@ +# Packager + +Packager is a tool for package management. +It was built to make it easier to download and distribute packages. + +## pkgr +pkgr is the main tool for the packager. + +Because this tool is so feature rich, it has its own [README](./pkgr/README.md). + +## bootpkg +bootpkg is a tool primariliy used to bootstrap the packager. + +### Usage + +```bash +bootpkg strap ./pkgr.pkg +``` + +This will extract the pkgfile and read the `[pkgr.bootstrap]` section to bootstrap the packager or any other package. \ No newline at end of file diff --git a/docs/pkgfile.md b/docs/pkgfile.md new file mode 100644 index 0000000..4b2b81a --- /dev/null +++ b/docs/pkgfile.md @@ -0,0 +1,24 @@ +# PKGFILE +This file is essentially a tar with the manifest in the header. + +## Format +The format is as follows: +``` +[PKGFILE version (1 byte)][manifest length x 256 (1 byte)][manifest length (1 byte)] +[manifest (manifest length bytes)] +[archive] +``` + +The file is Big Endian. + +## Unpacking +To unpack a PKGFILE, you can use the following command: +```bash +pkgr unpack +``` + +## Packing +You can write your own packer, or use the following command: +```bash +pkgr pack +``` diff --git a/docs/pkgr/README.md b/docs/pkgr/README.md new file mode 100644 index 0000000..042206a --- /dev/null +++ b/docs/pkgr/README.md @@ -0,0 +1,28 @@ +# pkgr +PKGR is the main tool for the packager. + +## Commands +### `pkgr install [--bin/--build] ` +This command will download a package from the repository or the uri provided and install it. +This command can also be used to update a package. + +### `pkgr remove ` +This command will remove a package from the system. + +### `pkgr revert ` +This command will revert a package to the previous version. + +### `pkgr download ` +This command will download a package from the repository or the uri provided into the specified path. + +### `pkgr unpack ` +This command will unpack a pkgfile into the specified output directory. + +### `pkgr pack ` +This command will pack a manifest and archive into a pkgfile. + +### `pkgr list [-i]` +This command will list all packages known or installed packages if `-i` is specified. + +### `pkgr update [package_name]` +This command will update all packages or a specific package. diff --git a/docs/pkgr/installedmanifest.md b/docs/pkgr/installedmanifest.md new file mode 100644 index 0000000..48f7b7b --- /dev/null +++ b/docs/pkgr/installedmanifest.md @@ -0,0 +1,19 @@ +# Installed manifest + +## Where? +The package manager will store the install manifest in: +`/var/lib/pkgr/installed///installed.toml` + +To find the latest we'll use the `latest` symlink: +`/var/lib/pkgr/installed//latest/installed.toml` + +```toml +install_start = "2021-01-01T00:00:00Z" # install_start is the time the pkgr started installing the package +install_end = "2021-01-01T00:00:00Z" # install_end is the time the pkgr finished installing the package +install_by = "pkgr" # install_by is the name of the package manager that installed the package + +[old_versions] # This logs old versions of the package (that were installed before) +9 = "../9/pkgr.pkg" + +# pkgr.* is reserved for the manifest of the package +``` \ No newline at end of file From 7c08579eccc3c53302fbc28ab8c9d41c04b05e68 Mon Sep 17 00:00:00 2001 From: Raine Date: Sat, 14 Oct 2023 22:39:42 +0200 Subject: [PATCH 058/123] fix: update to spec --- pkgr/src/commands/mod.rs | 3 ++- pkgr/src/main.rs | 1 + pkgr/src/prelude.rs | 1 + 3 files changed, 4 insertions(+), 1 deletion(-) create mode 100644 pkgr/src/prelude.rs diff --git a/pkgr/src/commands/mod.rs b/pkgr/src/commands/mod.rs index 61e3fe8..7f18ebd 100644 --- a/pkgr/src/commands/mod.rs +++ b/pkgr/src/commands/mod.rs @@ -1,7 +1,8 @@ use crate::package::identifier::PackageIdentifier; +use crate::prelude::Arguments; pub enum Command { - Grab(PackageIdentifier), // grab a package from a remote source + Install(Arguments, PackageIdentifier), // install a package from a remote source Remove(PackageIdentifier), // remove a package from the local source List, // list all packages in the local source Update, diff --git a/pkgr/src/main.rs b/pkgr/src/main.rs index 8d75350..9029074 100644 --- a/pkgr/src/main.rs +++ b/pkgr/src/main.rs @@ -5,6 +5,7 @@ use std::str::FromStr; use getopts::Options; use log::{info, SetLoggerError}; +mod prelude; mod commands; mod package; diff --git a/pkgr/src/prelude.rs b/pkgr/src/prelude.rs new file mode 100644 index 0000000..d7ac579 --- /dev/null +++ b/pkgr/src/prelude.rs @@ -0,0 +1 @@ +pub struct Arguments(Vec); \ No newline at end of file From 09250c59cb6615cb188827062239f0c83efeb5d0 Mon Sep 17 00:00:00 2001 From: Raine Date: Sat, 14 Oct 2023 22:39:42 +0200 Subject: [PATCH 059/123] ci: remove branch spec --- .github/workflows/build.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 13d43f3..d9bf0bc 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -1,7 +1,5 @@ on: push: - branches: - - main paths: - 'pkgr/**' - 'manifest/**' From 1d3895bd4325fec63eba0e0dc665a3a6441a7efa Mon Sep 17 00:00:00 2001 From: Raine Date: Sat, 14 Oct 2023 22:39:42 +0200 Subject: [PATCH 060/123] feat: arguments with clip --- pkgr/Cargo.toml | 2 +- pkgr/src/commands/mod.rs | 64 +++++++++++++++++++++++++++++++--- pkgr/src/main.rs | 38 ++++---------------- pkgr/src/package/identifier.rs | 24 +++++++++---- pkgr/src/package/mod.rs | 2 +- pkgr/src/prelude.rs | 1 - 6 files changed, 85 insertions(+), 46 deletions(-) delete mode 100644 pkgr/src/prelude.rs diff --git a/pkgr/Cargo.toml b/pkgr/Cargo.toml index 92a0e91..9f719ea 100644 --- a/pkgr/Cargo.toml +++ b/pkgr/Cargo.toml @@ -7,7 +7,7 @@ edition = "2021" [dependencies] fern = "0.6.2" -getopts = "0.2.21" log = "0.4.19" regex = "1.9.1" manifest = { path = "../manifest" } +clap = { version = "4.3.12", features = ["derive"] } diff --git a/pkgr/src/commands/mod.rs b/pkgr/src/commands/mod.rs index 7f18ebd..b5d698c 100644 --- a/pkgr/src/commands/mod.rs +++ b/pkgr/src/commands/mod.rs @@ -1,9 +1,63 @@ use crate::package::identifier::PackageIdentifier; -use crate::prelude::Arguments; +use clap::{Parser, Subcommand}; +use log::error; +#[derive(Parser, Debug)] +#[clap(name = "pkgr", version = "0.1.0", author = "")] +pub struct Cli { + #[command(subcommand)] + pub command: Command, +} + +#[derive(Subcommand, Debug)] pub enum Command { - Install(Arguments, PackageIdentifier), // install a package from a remote source - Remove(PackageIdentifier), // remove a package from the local source - List, // list all packages in the local source + /// Install a package + Install { + #[arg(short, long, default_value_t = false)] + build: bool, + package_identifier: PackageIdentifier, + }, + /// Remove a package from the system + Remove { + package_identifier: PackageIdentifier, + }, + /// List packages + List { + /// List installed packages + #[arg(short, long, default_value_t = true)] + installed: bool, + }, + /// Update packages on the system Update, -} \ No newline at end of file + #[command(hide = true)] + None, +} + +impl Command { + pub fn execute(&self) { + match self { + Command::Install { + build, + package_identifier, + } => { + if *build { + error!("Build is not yet implemented."); + } else { + error!("Install is not yet implemented."); + } + } + Command::Remove { package_identifier } => { + error!("Remove is not yet implemented."); + } + Command::List { installed } => { + error!("List is not yet implemented."); + } + Command::Update => { + error!("Update is not yet implemented."); + } + Command::None => { + error!("No command was specified."); + } + } + } +} diff --git a/pkgr/src/main.rs b/pkgr/src/main.rs index 9029074..50b9007 100644 --- a/pkgr/src/main.rs +++ b/pkgr/src/main.rs @@ -1,45 +1,19 @@ +use clap::Parser; use std::env; use std::process::exit; use std::str::FromStr; -use getopts::Options; +use crate::commands::{Cli, Command}; use log::{info, SetLoggerError}; -mod prelude; mod commands; mod package; fn main() { - setup_logger() - .expect("Unable to setup logger."); + setup_logger().expect("Unable to setup logger."); - let args: Vec = env::args().collect(); - let program = args[0].clone(); - - let mut opts = Options::new(); - opts.optflag("h", "help", "Show help message."); - let matches = opts.parse(&args[1..]).unwrap_or_else(|e| panic!("{}", e.to_string())); - - if matches.opt_present("h") { - print_usage(&program, opts); - exit(1); - } - - let command = if !matches.free.is_empty() { - matches.free[0].clone() - } else { - print_usage(&program, opts); - exit(1); - }; - - info!("Identifier: {}", package::identifier::PackageIdentifier::from_str(&command).unwrap()); -} - -fn print_usage(program: &str, opts: Options) { - let mut brief = format!("Usage: {} [options]\n\n", program); - brief += &*format!("Commands: grab \n"); - brief += &*format!(" remove\n"); - print!("{}", opts.usage(&brief.trim_end())); + let c = Cli::parse(); + info!("Command: {:?}", c); } fn setup_logger() -> Result<(), SetLoggerError> { @@ -57,4 +31,4 @@ fn setup_logger() -> Result<(), SetLoggerError> { .level(log::LevelFilter::Info) .chain(std::io::stdout()) .apply() -} \ No newline at end of file +} diff --git a/pkgr/src/package/identifier.rs b/pkgr/src/package/identifier.rs index 01f9f6b..07b2bcc 100644 --- a/pkgr/src/package/identifier.rs +++ b/pkgr/src/package/identifier.rs @@ -1,8 +1,9 @@ +use std::error::Error; use std::str::FromStr; use regex::Regex; -#[derive(Debug)] +#[derive(Debug, Clone)] pub enum PackageIdentifierError { InvalidPackageLocator(String), InvalidURI(String), @@ -11,13 +12,17 @@ pub enum PackageIdentifierError { impl std::fmt::Display for PackageIdentifierError { fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { match self { - PackageIdentifierError::InvalidPackageLocator(s) => write!(f, "Invalid package locator: {}", s), + PackageIdentifierError::InvalidPackageLocator(s) => { + write!(f, "Invalid package locator: {}", s) + } PackageIdentifierError::InvalidURI(s) => write!(f, "Invalid URI: {}", s), } } } -#[derive(Debug)] +impl Error for PackageIdentifierError {} + +#[derive(Debug, Clone)] pub enum PackageIdentifier { PackageLocator(PackageLocator), URI(String), @@ -32,7 +37,7 @@ impl std::fmt::Display for PackageIdentifier { } } -#[derive(Debug)] +#[derive(Debug, Clone)] pub struct PackageLocator { pub name: String, pub version: Option, @@ -102,7 +107,14 @@ impl FromStr for PackageLocator { } if let Some(caps) = tags_re.captures(s) { - tags = Some(caps.get(1).unwrap().as_str().split(",").map(|s| s.to_string()).collect()); + tags = Some( + caps.get(1) + .unwrap() + .as_str() + .split(",") + .map(|s| s.to_string()) + .collect(), + ); } Ok(PackageLocator { @@ -111,4 +123,4 @@ impl FromStr for PackageLocator { tags, }) } -} \ No newline at end of file +} diff --git a/pkgr/src/package/mod.rs b/pkgr/src/package/mod.rs index 8f83cb4..34cdf0a 100644 --- a/pkgr/src/package/mod.rs +++ b/pkgr/src/package/mod.rs @@ -1 +1 @@ -pub mod identifier; \ No newline at end of file +pub mod identifier; diff --git a/pkgr/src/prelude.rs b/pkgr/src/prelude.rs deleted file mode 100644 index d7ac579..0000000 --- a/pkgr/src/prelude.rs +++ /dev/null @@ -1 +0,0 @@ -pub struct Arguments(Vec); \ No newline at end of file From b7c6e7aa6f348cd7b584df7046a3fdbfbe90e2cb Mon Sep 17 00:00:00 2001 From: Raine Date: Sat, 14 Oct 2023 22:39:42 +0200 Subject: [PATCH 061/123] docs: change pkgr cli spec --- docs/pkgr/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/pkgr/README.md b/docs/pkgr/README.md index 042206a..0b713bd 100644 --- a/docs/pkgr/README.md +++ b/docs/pkgr/README.md @@ -24,5 +24,5 @@ This command will pack a manifest and archive into a pkgfile. ### `pkgr list [-i]` This command will list all packages known or installed packages if `-i` is specified. -### `pkgr update [package_name]` -This command will update all packages or a specific package. +### `pkgr update` +This command will update all packages. From b34cdb4aa4597723def3795c748767cc5fa1bf9d Mon Sep 17 00:00:00 2001 From: Raine Date: Sat, 14 Oct 2023 22:39:43 +0200 Subject: [PATCH 062/123] fix: cleanup code --- pkgr/src/commands/mod.rs | 6 +++--- pkgr/src/main.rs | 8 ++++---- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/pkgr/src/commands/mod.rs b/pkgr/src/commands/mod.rs index b5d698c..3c52aca 100644 --- a/pkgr/src/commands/mod.rs +++ b/pkgr/src/commands/mod.rs @@ -38,7 +38,7 @@ impl Command { match self { Command::Install { build, - package_identifier, + package_identifier: _, } => { if *build { error!("Build is not yet implemented."); @@ -46,10 +46,10 @@ impl Command { error!("Install is not yet implemented."); } } - Command::Remove { package_identifier } => { + Command::Remove { package_identifier: _ } => { error!("Remove is not yet implemented."); } - Command::List { installed } => { + Command::List { installed: _ } => { error!("List is not yet implemented."); } Command::Update => { diff --git a/pkgr/src/main.rs b/pkgr/src/main.rs index 50b9007..17e200f 100644 --- a/pkgr/src/main.rs +++ b/pkgr/src/main.rs @@ -1,9 +1,9 @@ use clap::Parser; -use std::env; -use std::process::exit; -use std::str::FromStr; -use crate::commands::{Cli, Command}; + + + +use crate::commands::{Cli}; use log::{info, SetLoggerError}; mod commands; From 3a21d12c071273a9c080135414817474721beaa3 Mon Sep 17 00:00:00 2001 From: Raine Date: Sat, 14 Oct 2023 22:39:43 +0200 Subject: [PATCH 063/123] feat: some changes to logging --- pkgr/Cargo.toml | 1 + pkgr/src/commands/mod.rs | 13 ++++++++++--- pkgr/src/main.rs | 30 +++++++++++++++++++----------- 3 files changed, 30 insertions(+), 14 deletions(-) diff --git a/pkgr/Cargo.toml b/pkgr/Cargo.toml index 9f719ea..6e94865 100644 --- a/pkgr/Cargo.toml +++ b/pkgr/Cargo.toml @@ -11,3 +11,4 @@ log = "0.4.19" regex = "1.9.1" manifest = { path = "../manifest" } clap = { version = "4.3.12", features = ["derive"] } +colored = "2.0.4" diff --git a/pkgr/src/commands/mod.rs b/pkgr/src/commands/mod.rs index 3c52aca..8697d80 100644 --- a/pkgr/src/commands/mod.rs +++ b/pkgr/src/commands/mod.rs @@ -3,7 +3,7 @@ use clap::{Parser, Subcommand}; use log::error; #[derive(Parser, Debug)] -#[clap(name = "pkgr", version = "0.1.0", author = "")] +#[clap(name = "pkgr", version)] pub struct Cli { #[command(subcommand)] pub command: Command, @@ -19,6 +19,9 @@ pub enum Command { }, /// Remove a package from the system Remove { + /// Remove the package from index too + #[arg(short, long, default_value_t = false)] + index: bool, package_identifier: PackageIdentifier, }, /// List packages @@ -46,8 +49,12 @@ impl Command { error!("Install is not yet implemented."); } } - Command::Remove { package_identifier: _ } => { - error!("Remove is not yet implemented."); + Command::Remove { package_identifier: _, index } => { + if *index { + error!("Index removal is not yet implemented."); + } else { + error!("Remove is not yet implemented."); + } } Command::List { installed: _ } => { error!("List is not yet implemented."); diff --git a/pkgr/src/main.rs b/pkgr/src/main.rs index 17e200f..e8c878a 100644 --- a/pkgr/src/main.rs +++ b/pkgr/src/main.rs @@ -1,10 +1,8 @@ use clap::Parser; - - - - -use crate::commands::{Cli}; -use log::{info, SetLoggerError}; +use colored::Colorize; +use crate::commands::Cli; +use log::{debug, info, SetLoggerError, trace}; +use std::env; mod commands; mod package; @@ -12,8 +10,16 @@ mod package; fn main() { setup_logger().expect("Unable to setup logger."); + trace!("Parsing command line arguments..."); let c = Cli::parse(); - info!("Command: {:?}", c); + debug!("Command line arguments parsed."); + + trace!("Executing command..."); + c.command.execute(); + debug!("Command executed."); + + trace!("Exiting..."); + info!("Done."); } fn setup_logger() -> Result<(), SetLoggerError> { @@ -22,13 +28,15 @@ fn setup_logger() -> Result<(), SetLoggerError> { out.finish(format_args!( "{} {}", match record.level().to_string().chars().nth(0).unwrap_or('I') { - 'E' | 'W' => "!!", - _ => "**", + 'T' => "##".cyan(), + 'D' => "::".yellow(), + 'E' | 'W' => "!!".red(), + _ => "**".blue(), }, - message + message.to_string().bright_white() )) }) - .level(log::LevelFilter::Info) + .level(env::var("PKGR_LOG_LEVEL").unwrap_or_else(|_| "info".to_string()).parse().unwrap_or(log::LevelFilter::Info)) .chain(std::io::stdout()) .apply() } From 092c616ca4416596994cf6d87e4290a98f79a65c Mon Sep 17 00:00:00 2001 From: Raine Date: Sat, 14 Oct 2023 22:39:43 +0200 Subject: [PATCH 064/123] feat: added command functionality to install --- LICENSE | 21 +++++++ bootpkg/src/prelude.rs | 4 +- docs/pkgr/installedmanifest.md | 3 + manifest/src/bin.rs | 2 +- manifest/src/build.rs | 2 +- manifest/src/fs.rs | 2 +- manifest/src/lib.rs | 15 +++-- manifest/src/package.rs | 4 +- manifest/src/pkgr.rs | 11 +++- package.example.toml | 107 ++++++++++++++++++++++++++++++++ package.toml | 38 ------------ pkgfile/src/lib.rs | 16 +++++ pkgr/Cargo.toml | 13 +++- pkgr/src/commands/mod.rs | 68 ++++++++++++++++---- pkgr/src/main.rs | 46 ++++++++++---- pkgr/src/package/builder/mod.rs | 71 +++++++++++++++++++++ pkgr/src/package/fetch.rs | 59 ++++++++++++++++++ pkgr/src/package/identifier.rs | 8 ++- pkgr/src/package/mod.rs | 2 + pkgr/src/tmp.rs | 48 ++++++++++++++ 20 files changed, 464 insertions(+), 76 deletions(-) create mode 100644 LICENSE create mode 100644 package.example.toml create mode 100644 pkgr/src/package/builder/mod.rs create mode 100644 pkgr/src/package/fetch.rs create mode 100644 pkgr/src/tmp.rs diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..5c3335d --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2023 didier + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/bootpkg/src/prelude.rs b/bootpkg/src/prelude.rs index b82b62a..9406cdb 100644 --- a/bootpkg/src/prelude.rs +++ b/bootpkg/src/prelude.rs @@ -5,12 +5,12 @@ use toml; use manifest::{self, Manifest}; -#[derive(Serialize, Deserialize)] +#[derive(Debug, Clone, Serialize, Deserialize)] pub struct PKGR { pub bootstrap: Option, } -#[derive(Serialize, Deserialize)] +#[derive(Debug, Clone, Serialize, Deserialize)] pub struct Bootstrap { pub check_installed_commands: Vec, pub commands: Vec, diff --git a/docs/pkgr/installedmanifest.md b/docs/pkgr/installedmanifest.md index 48f7b7b..04a28ad 100644 --- a/docs/pkgr/installedmanifest.md +++ b/docs/pkgr/installedmanifest.md @@ -11,6 +11,9 @@ To find the latest we'll use the `latest` symlink: install_start = "2021-01-01T00:00:00Z" # install_start is the time the pkgr started installing the package install_end = "2021-01-01T00:00:00Z" # install_end is the time the pkgr finished installing the package install_by = "pkgr" # install_by is the name of the package manager that installed the package +package_identifier = "uri" +package_id = "https://..." + [old_versions] # This logs old versions of the package (that were installed before) 9 = "../9/pkgr.pkg" diff --git a/manifest/src/bin.rs b/manifest/src/bin.rs index e30b528..f55a8e5 100644 --- a/manifest/src/bin.rs +++ b/manifest/src/bin.rs @@ -2,7 +2,7 @@ use std::collections::HashMap; use serde::{Deserialize, Serialize}; -#[derive(Serialize, Deserialize)] +#[derive(Debug, Clone, Serialize, Deserialize)] pub struct Bin { pub root: String, pub checksums: HashMap, diff --git a/manifest/src/build.rs b/manifest/src/build.rs index ac223d2..161b785 100644 --- a/manifest/src/build.rs +++ b/manifest/src/build.rs @@ -2,7 +2,7 @@ use std::collections::HashMap; use serde::{Deserialize, Serialize}; -#[derive(Serialize, Deserialize)] +#[derive(Debug, Clone, Serialize, Deserialize)] pub struct Build { build_script: String, install_script: String, diff --git a/manifest/src/fs.rs b/manifest/src/fs.rs index e02312f..743bc88 100644 --- a/manifest/src/fs.rs +++ b/manifest/src/fs.rs @@ -1,6 +1,6 @@ use serde::{Deserialize, Serialize}; -#[derive(Serialize, Deserialize, Debug)] +#[derive(Debug, Clone, Serialize, Deserialize)] pub struct FS { pub config: Option, pub data: Option, diff --git a/manifest/src/lib.rs b/manifest/src/lib.rs index 8a943fa..0d52840 100644 --- a/manifest/src/lib.rs +++ b/manifest/src/lib.rs @@ -8,9 +8,9 @@ pub mod fs; pub mod package; pub mod pkgr; -#[derive(Serialize, Deserialize)] +#[derive(Debug, Clone, Serialize, Deserialize)] #[serde(default)] -pub struct Manifest

> { +pub struct Manifest> { pub package: package::Package, pub dependencies: HashMap, pub fs: fs::FS, @@ -19,7 +19,7 @@ pub struct Manifest

> { pub pkgr: P, } -impl Manifest { +impl Manifest

{ pub fn valid(&self) -> bool { if self.bin.is_none() && self.build.is_none() { return false; @@ -28,6 +28,13 @@ impl Manifest { } } +impl TryFrom for Manifest { + type Error = toml::de::Error; + fn try_from(s: String) -> Result { + toml::from_str(&s) + } +} + impl FromStr for Manifest { type Err = toml::de::Error; fn from_str(s: &str) -> Result { @@ -36,7 +43,7 @@ impl FromStr for Manifest { } impl Default for Manifest { - fn default() -> Self { + fn default() -> Manifest { Manifest { package: package::Package::default(), dependencies: HashMap::default(), diff --git a/manifest/src/package.rs b/manifest/src/package.rs index 017271d..233206d 100644 --- a/manifest/src/package.rs +++ b/manifest/src/package.rs @@ -2,7 +2,7 @@ use std::str::FromStr; use serde::{Deserialize, Serialize}; -#[derive(Serialize, Deserialize, Debug)] +#[derive(Debug, Clone, Serialize, Deserialize)] pub enum PackageType { #[serde(rename = "application")] Application, @@ -12,7 +12,7 @@ pub enum PackageType { Meta, } -#[derive(Serialize, Deserialize, Debug)] +#[derive(Debug, Clone, Serialize, Deserialize)] #[serde(default)] pub struct Package { pub name: String, diff --git a/manifest/src/pkgr.rs b/manifest/src/pkgr.rs index 8c4ce55..700c6cb 100644 --- a/manifest/src/pkgr.rs +++ b/manifest/src/pkgr.rs @@ -1,4 +1,11 @@ +use std::fmt::{Display, Formatter}; use serde::{Deserialize, Serialize}; -#[derive(Serialize, Deserialize)] -pub struct PKGR {} \ No newline at end of file +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct PKGR {} + +impl Display for PKGR { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + write!(f, "") + } +} \ No newline at end of file diff --git a/package.example.toml b/package.example.toml new file mode 100644 index 0000000..c99108d --- /dev/null +++ b/package.example.toml @@ -0,0 +1,107 @@ +#* = required. + +[package] +name = "packager" #* +description = "A package installation tool" #* + +## package.version +# needs to be an int and be incremented every time +# if you use a custom version system and want to use that, you can use the tag system! +# the highest package.version on the server will be given the tag "latest" +version = 1 # this can automatically be incremented when publishing by running `pkgr publish -i ...` + +## package.tags +# you can add tags to a package to specify kinds +# there are some special tags: +# - latest +# automatically set on the last published version +# previous version will lose this tag +# - oldest +# assigned to the first version. +tags = [ "prod", "pkgr-spec-1" ] + +## package.type +# Supported types: +# - application +# this type specifies that this package is an application. +# - library +# this type specifies that this package is a library. +# - meta +# this type specifies that this package does not install anything upon the system except for dependencies. +type = "application" + +arch = "x86_64" # this is automatically filled by `pkgr publish ...` + +## dependencies +# you may use the following syntax +# "" or ",,..." +[dependencies] +#example = { arch = "x86_64", tags = [ "stable" ], version = "v1" } +exnmbr2 = "v1,stable" + +## fs +# specify some directories for ease of use +[fs] +## fs.config +# specifiy the config path +config = "/etc/packager" +## fs.data +# specify the data path +data = "/var/lib/packager" +## replace "pacakger" with your package name to find the default value. + +## bin +# Used for systems that don't want to build pkgs. +[bin] # binary files root + +## bin.root +# ** RELATIVE TO PACKAGE ROOT ** +# bin.root specifies the root of the installed tree. +# anything in here will be overlayed on top of the system. +root = "/root" #* + +## bin.checksums +# ** KEYS are relative to BIN.ROOT ** +# ** VALUES are relative to PKG ROOT ** +# checksums is used to perform checksums before install +# values may be paths or a uri. You may include variables. +# supported variables: +# - @name +# - @version +# - @display_version +[bin.checksums] +"/usr/bin/pkgr" = "https://ixvd.net/checksums/packager/@version" + +## build +# Scripts will be copied to a custom root. +# After the pacakge is built, the install script is ran. +# After the install script the build directory will be deleted and an CTREE (changed tree) will be generated. +# Then the CTREE will be used to copy the files over. +[build] +build_script = "scripts/build" # relative to pkg +install_script = "scripts/install" # relative to pkg + +[build.dependencies] +base = "latest,stable" # selected by default + +## pkgr.* +# packager is the official client but you may use other clients supporting the "pkgr v1 spec". +# other clients may offer extra functionality this must be put under "pkgr.*" +[pkgr] + +## pkgr.bootstrap +# This section is used for bootpkg. An edition of packager that bootstraps the full version. +# This exists so that packager is easy to install on anything! +# and only 1 release channel for pkgr +[pkgr.bootstrap] +## any non-zero = installed +check_installed_commands = [ + "sh scripts/check_installed" +] + +# any non-zero = fail +commands = [ + "sh scripts/bootstrap/download_latest @version /tmp/pkgr.pkg", + "sh scripts/bootstrap/dirty_install /tmp/pkgr.pkg", + "sh scripts/check_installed" +] diff --git a/package.toml b/package.toml index c99108d..81240ab 100644 --- a/package.toml +++ b/package.toml @@ -1,33 +1,8 @@ -#* = required. - [package] name = "packager" #* description = "A package installation tool" #* - -## package.version -# needs to be an int and be incremented every time -# if you use a custom version system and want to use that, you can use the tag system! -# the highest package.version on the server will be given the tag "latest" version = 1 # this can automatically be incremented when publishing by running `pkgr publish -i ...` - -## package.tags -# you can add tags to a package to specify kinds -# there are some special tags: -# - latest -# automatically set on the last published version -# previous version will lose this tag -# - oldest -# assigned to the first version. tags = [ "prod", "pkgr-spec-1" ] - -## package.type -# Supported types: -# - application -# this type specifies that this package is an application. -# - library -# this type specifies that this package is a library. -# - meta -# this type specifies that this package does not install anything upon the system except for dependencies. type = "application" arch = "x86_64" # this is automatically filled by `pkgr publish ...` @@ -36,19 +11,6 @@ arch = "x86_64" # this is automatically filled by `pkgr publish ...` # you may use the following syntax # "" or ",,..." [dependencies] -#example = { arch = "x86_64", tags = [ "stable" ], version = "v1" } -exnmbr2 = "v1,stable" - -## fs -# specify some directories for ease of use -[fs] -## fs.config -# specifiy the config path -config = "/etc/packager" -## fs.data -# specify the data path -data = "/var/lib/packager" -## replace "pacakger" with your package name to find the default value. ## bin # Used for systems that don't want to build pkgs. diff --git a/pkgfile/src/lib.rs b/pkgfile/src/lib.rs index bc26f99..15249e7 100644 --- a/pkgfile/src/lib.rs +++ b/pkgfile/src/lib.rs @@ -1,8 +1,24 @@ +#[derive(Debug)] pub struct PKGFile { pub manifest: String, pub data: Vec, } +impl PKGFile { + pub fn new(manifest: String, data: Vec) -> PKGFile { + PKGFile { manifest, data } + } +} + +impl Default for PKGFile { + fn default() -> PKGFile { + PKGFile { + manifest: String::new(), + data: Vec::new(), + } + } +} + impl TryFrom> for PKGFile { type Error = (); diff --git a/pkgr/Cargo.toml b/pkgr/Cargo.toml index 6e94865..cc6e55b 100644 --- a/pkgr/Cargo.toml +++ b/pkgr/Cargo.toml @@ -1,14 +1,25 @@ [package] name = "pkgr" +description = "A package manager and build tool." version = "0.1.0" edition = "2021" +authors = [ + "Didier " +] +license-file = "../LICENSE" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] +manifest = { path = "../manifest" } +pkgfile = { path = "../pkgfile" } fern = "0.6.2" log = "0.4.19" regex = "1.9.1" -manifest = { path = "../manifest" } clap = { version = "4.3.12", features = ["derive"] } colored = "2.0.4" +toml = "0.7.6" +serde = { version = "1.0.171", features = ["derive"] } +libc = "0.2.80" +reqwest = { version = "0.11.18", features = ["blocking"] } +tar = "0.4.39" diff --git a/pkgr/src/commands/mod.rs b/pkgr/src/commands/mod.rs index 8697d80..bcbef89 100644 --- a/pkgr/src/commands/mod.rs +++ b/pkgr/src/commands/mod.rs @@ -1,6 +1,10 @@ +use std::process::exit; use crate::package::identifier::PackageIdentifier; use clap::{Parser, Subcommand}; -use log::error; +use log::{debug, error, info, trace, warn}; +use manifest::Manifest; +use crate::package::builder::{InstallType, PackageInstaller}; +use crate::package::fetch::fetch_package; #[derive(Parser, Debug)] #[clap(name = "pkgr", version)] @@ -33,6 +37,8 @@ pub enum Command { /// Update packages on the system Update, #[command(hide = true)] + Debug, + #[command(hide = true)] None, } @@ -41,20 +47,47 @@ impl Command { match self { Command::Install { build, - package_identifier: _, + package_identifier, } => { - if *build { - error!("Build is not yet implemented."); - } else { - error!("Install is not yet implemented."); + warn!("Installer does not run in isolation."); + + let start = std::time::Instant::now(); + let unix_start = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap(); + + trace!("Fetching package: {}", package_identifier); + let pkgfile = fetch_package(package_identifier.clone()).unwrap(); + debug!("size: manifest({}kb) data({}kb)", pkgfile.manifest.len() / 1024, pkgfile.data.len() / 1024); + trace!("parsing manifest"); + let manifest = Manifest::try_from(pkgfile.manifest.clone()).unwrap(); + debug!("manifest pkg name: {}", manifest.package.name); + trace!("creating installer"); + let installer = PackageInstaller::new(manifest.clone(), pkgfile, if *build { InstallType::Build } else { InstallType::Bin }); + + trace!("starting install"); + match installer.install() { + Ok(_) => { + info!("Sucessfully installed: {}", &manifest.package.name); + () + }, + Err(e) => { + error!("{}", e.to_string()) + } } + let unix_end = std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).unwrap(); + let end = std::time::Instant::now(); + // float ms + let duration = (end - start).as_nanos() as f64; + info!("Install took: {}ms", (duration / 1_000_000.0)); } - Command::Remove { package_identifier: _, index } => { - if *index { - error!("Index removal is not yet implemented."); - } else { - error!("Remove is not yet implemented."); + Command::Remove { package_identifier, index } => { + if let PackageIdentifier::URI(_) = package_identifier { + error!("URI is unsupported when removing applications."); + exit(1); } + info!("Index: {}", index); + info!("Package identifier: {}", package_identifier); } Command::List { installed: _ } => { error!("List is not yet implemented."); @@ -62,6 +95,19 @@ impl Command { Command::Update => { error!("Update is not yet implemented."); } + Command::Debug => { + trace!("Trace message."); + debug!("Debug message."); + info!("Info message."); + warn!("Warning message."); + error!("Error message."); + info!(""); + info!("PKGR VERSION: {}", env!("CARGO_PKG_VERSION")); + info!("PKGR AUTHORS: {}", env!("CARGO_PKG_AUTHORS")); + info!("PKGR DESCRIPTION: {}", env!("CARGO_PKG_DESCRIPTION")); + info!(""); + info!("PKGR_LOG_LEVEL: {}", std::env::var("PKGR_LOG_LEVEL").unwrap_or_else(|_| "info".to_string())); + } Command::None => { error!("No command was specified."); } diff --git a/pkgr/src/main.rs b/pkgr/src/main.rs index e8c878a..ba5f97d 100644 --- a/pkgr/src/main.rs +++ b/pkgr/src/main.rs @@ -1,25 +1,35 @@ use clap::Parser; use colored::Colorize; use crate::commands::Cli; -use log::{debug, info, SetLoggerError, trace}; +use log::{SetLoggerError, trace}; use std::env; mod commands; mod package; +mod tmp; fn main() { setup_logger().expect("Unable to setup logger."); + #[cfg(not(debug_assertions))] + { + #[cfg(target_family = "unix")] + if unsafe { libc::getuid() } != 0 { + use log::error; + error!("pkgr must be run as root."); + std::process::exit(1); + } + } + trace!("Parsing command line arguments..."); let c = Cli::parse(); - debug!("Command line arguments parsed."); + trace!("Command line arguments: {:?}", c); trace!("Executing command..."); c.command.execute(); - debug!("Command executed."); + trace!("Command executed."); trace!("Exiting..."); - info!("Done."); } fn setup_logger() -> Result<(), SetLoggerError> { @@ -27,16 +37,30 @@ fn setup_logger() -> Result<(), SetLoggerError> { .format(|out, message, record| { out.finish(format_args!( "{} {}", - match record.level().to_string().chars().nth(0).unwrap_or('I') { - 'T' => "##".cyan(), - 'D' => "::".yellow(), - 'E' | 'W' => "!!".red(), - _ => "**".blue(), - }, + // Some logic so messages look nice + if message.to_string().len() > 0 { + match record + .level() + .to_string() + .chars() + .nth(0) + .unwrap_or('T') + { + 'T' => "[TRACE]".cyan(), + 'D' => "??".green(), + 'I' => "=>".blue(), + 'W' => "##".yellow(), + 'E' => "!!".red(), + _ => "**".blue(), + }.to_string() + } else { "".to_string() }, message.to_string().bright_white() )) }) - .level(env::var("PKGR_LOG_LEVEL").unwrap_or_else(|_| "info".to_string()).parse().unwrap_or(log::LevelFilter::Info)) + .level(env::var("PKGR_LOG_LEVEL") + .unwrap_or_else(|_| "info".to_string()) + .parse() + .unwrap_or(log::LevelFilter::Info)) .chain(std::io::stdout()) .apply() } diff --git a/pkgr/src/package/builder/mod.rs b/pkgr/src/package/builder/mod.rs new file mode 100644 index 0000000..50166e1 --- /dev/null +++ b/pkgr/src/package/builder/mod.rs @@ -0,0 +1,71 @@ +use libc::fork; +use log::{debug, trace}; +use manifest::Manifest; +use pkgfile::PKGFile; +use crate::tmp::TempDir; + +#[derive(Debug)] +pub enum InstallType { + Build, + Bin +} + +#[derive(Debug)] +pub enum InstallError { + BuildError(String), + BinError(String), + InstallError +} + +impl ToString for InstallError { + fn to_string(&self) -> String { + match self { + InstallError::BuildError(e) => format!("Build error: {}", e), + InstallError::BinError(e) => format!("Bin error: {}", e), + InstallError::InstallError => "Install error".to_string(), + } + } +} + +#[derive(Debug)] +pub struct PackageInstaller { + manifest: Manifest, + pkgfile: PKGFile, + install_type: InstallType +} + +impl PackageInstaller { + pub fn new(m: Manifest, p: PKGFile, i: InstallType) -> PackageInstaller { + PackageInstaller { + manifest: m, + pkgfile: p, + install_type: i + } + } + + fn extract_to>(&self, path: S) -> Result<(), String> { + tar::Archive::new(self.pkgfile.data.as_slice()) + .unpack(path.into()) + .map_err(|e| e.to_string()) + } + + fn bin(&self) -> Result<(), String> { + let mut tmpdir = TempDir::default(); + tmpdir.push(&self.manifest.package.name); + trace!("extracting package into: {}", tmpdir.to_string()); + self.extract_to(tmpdir.to_string())?; + debug!("extracted package in: {}", tmpdir.to_string()); + Ok(()) + } + + fn build(&self) -> Result<(), String> { + Ok(()) + } + + pub fn install(&self) -> Result<(), InstallError> { + match self.install_type { + InstallType::Bin => self.bin().map_err(|e| InstallError::BinError(e)), + InstallType::Build => self.build().map_err(|e| InstallError::BuildError(e)) + } + } +} \ No newline at end of file diff --git a/pkgr/src/package/fetch.rs b/pkgr/src/package/fetch.rs new file mode 100644 index 0000000..0e68045 --- /dev/null +++ b/pkgr/src/package/fetch.rs @@ -0,0 +1,59 @@ +use std::error::Error; +use std::fmt::Display; +use std::fs::File; +use std::io::Read; +use log::warn; +use pkgfile::PKGFile; +use crate::package::identifier::PackageIdentifier; +use reqwest::blocking::get; + +#[derive(Debug)] +pub enum FetchError { + HTTPError(reqwest::Error), + IOError(std::io::Error), + ParseError, +} + +impl Display for FetchError { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + match self { + FetchError::HTTPError(e) => write!(f, "HTTP Error: {}", e), + FetchError::IOError(e) => write!(f, "IO Error: {}", e), + FetchError::ParseError => write!(f, "Parse Error"), + } + } +} + +impl Error for FetchError {} + +pub fn fetch_package(package_identifier: PackageIdentifier) -> Result { + match package_identifier { + PackageIdentifier::Path(path) => { + std::fs::read(path) + .map_err(|e| FetchError::IOError(e)).and_then(|bytes| { + PKGFile::try_from(bytes).map_err(|_| FetchError::ParseError) + }) + } + PackageIdentifier::URI(uri) => { + // get file contents as bytes + let mut bytes = Vec::new(); + match get(&uri) { + Ok(mut response) => { + match response.take(1024 * 1024).read_to_end(&mut bytes) { + Ok(_) => (), + Err(e) => return Err(FetchError::IOError(e)), + }; + } + Err(e) => return Err(FetchError::HTTPError(e)), + }; + // parse bytes as PKGFile + match PKGFile::try_from(bytes) { + Ok(pkgfile) => Ok(pkgfile), + Err(e) => Err(FetchError::ParseError), + } + } + PackageIdentifier::PackageLocator(package_locator) => { + Ok(PKGFile::default()) + } + } +} \ No newline at end of file diff --git a/pkgr/src/package/identifier.rs b/pkgr/src/package/identifier.rs index 07b2bcc..fa457d8 100644 --- a/pkgr/src/package/identifier.rs +++ b/pkgr/src/package/identifier.rs @@ -26,6 +26,7 @@ impl Error for PackageIdentifierError {} pub enum PackageIdentifier { PackageLocator(PackageLocator), URI(String), + Path(String), } impl std::fmt::Display for PackageIdentifier { @@ -33,6 +34,7 @@ impl std::fmt::Display for PackageIdentifier { match self { PackageIdentifier::PackageLocator(pl) => write!(f, "{}", pl), PackageIdentifier::URI(uri) => write!(f, "{}", uri), + PackageIdentifier::Path(path) => write!(f, "{}", path), } } } @@ -61,7 +63,7 @@ impl FromStr for PackageIdentifier { type Err = PackageIdentifierError; fn from_str(s: &str) -> Result { - let uri_re = regex::Regex::new(r"^[a-zA-Z0-9]+://").unwrap(); + let uri_re = Regex::new(r"^[a-zA-Z0-9]+://").unwrap(); if uri_re.is_match(s) { // there needs to be stuff after the protocol let split = s.split("://").collect::>(); @@ -69,6 +71,8 @@ impl FromStr for PackageIdentifier { return Err(PackageIdentifierError::InvalidURI(s.to_string())); } Ok(PackageIdentifier::URI(s.to_string())) + } else if std::path::Path::new(s).exists() { + return Ok(PackageIdentifier::Path(s.to_string())); } else { let pl = match PackageLocator::from_str(s) { Ok(pl) => pl, @@ -84,7 +88,7 @@ impl FromStr for PackageLocator { fn from_str(s: &str) -> Result { #[allow(unused_assignments)] // false positive - let mut name = None; + let mut name = None; let mut version = None; let mut tags = None; diff --git a/pkgr/src/package/mod.rs b/pkgr/src/package/mod.rs index 34cdf0a..5c559fc 100644 --- a/pkgr/src/package/mod.rs +++ b/pkgr/src/package/mod.rs @@ -1 +1,3 @@ pub mod identifier; +pub mod builder; +pub mod fetch; \ No newline at end of file diff --git a/pkgr/src/tmp.rs b/pkgr/src/tmp.rs new file mode 100644 index 0000000..509d3f7 --- /dev/null +++ b/pkgr/src/tmp.rs @@ -0,0 +1,48 @@ +use std::path::PathBuf; + +pub struct TempDir { + path: PathBuf, +} + +impl TempDir { + pub fn new(path: PathBuf) -> TempDir { + if !path.exists() { + std::fs::create_dir_all(&path).unwrap(); + } + TempDir { + path + } + } + + pub fn push>(&mut self, path: S) { + self.path.push(path.into()); + } +} + +impl Default for TempDir { + fn default() -> TempDir { + TempDir::new({ + let mut t = std::env::temp_dir(); + t.push("pkgr"); + t + }) + } +} + +impl Into for TempDir { + fn into(self) -> PathBuf { + self.path + } +} + +impl Into for TempDir { + fn into(self) -> String { + self.path.to_str().unwrap().to_string() + } +} + +impl ToString for TempDir { + fn to_string(&self) -> String { + self.path.to_str().unwrap().to_string() + } +} \ No newline at end of file From 4ecc1eb1e867e923116e677610ab591ccb424aed Mon Sep 17 00:00:00 2001 From: Raine Date: Sat, 14 Oct 2023 22:39:43 +0200 Subject: [PATCH 065/123] feat: start on install section --- pkgfile/src/lib.rs | 4 ++ pkgr/src/{commands/mod.rs => commands.rs} | 3 +- pkgr/src/package/builder/errors.rs | 42 +++++++++++++++++++++ pkgr/src/package/builder/mod.rs | 45 ++++++++++------------- 4 files changed, 68 insertions(+), 26 deletions(-) rename pkgr/src/{commands/mod.rs => commands.rs} (98%) create mode 100644 pkgr/src/package/builder/errors.rs diff --git a/pkgfile/src/lib.rs b/pkgfile/src/lib.rs index 15249e7..f7768b0 100644 --- a/pkgfile/src/lib.rs +++ b/pkgfile/src/lib.rs @@ -8,6 +8,10 @@ impl PKGFile { pub fn new(manifest: String, data: Vec) -> PKGFile { PKGFile { manifest, data } } + + pub fn has_data(&self) -> bool { + !self.data.is_empty() + } } impl Default for PKGFile { diff --git a/pkgr/src/commands/mod.rs b/pkgr/src/commands.rs similarity index 98% rename from pkgr/src/commands/mod.rs rename to pkgr/src/commands.rs index bcbef89..e0f7ec0 100644 --- a/pkgr/src/commands/mod.rs +++ b/pkgr/src/commands.rs @@ -72,7 +72,8 @@ impl Command { () }, Err(e) => { - error!("{}", e.to_string()) + error!("{}", e.to_string()); + exit(1); } } let unix_end = std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).unwrap(); diff --git a/pkgr/src/package/builder/errors.rs b/pkgr/src/package/builder/errors.rs new file mode 100644 index 0000000..e5f62d6 --- /dev/null +++ b/pkgr/src/package/builder/errors.rs @@ -0,0 +1,42 @@ +use std::fmt::Display; + +#[derive(Debug)] +pub enum BinError { + UnpackError(String), +} + +impl Display for BinError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + BinError::UnpackError(e) => write!(f, "Unpack error: {}", e), + } + } +} + +#[derive(Debug)] +pub enum BuildError {} + +impl Display for BuildError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + _ => write!(f, "Build error"), + } + } +} + +#[derive(Debug)] +pub enum InstallError { + BuildError(BuildError), + BinError(BinError), + InstallError +} + +impl ToString for InstallError { + fn to_string(&self) -> String { + match self { + InstallError::BuildError(e) => format!("Build error: {}", e), + InstallError::BinError(e) => format!("Bin error: {}", e), + InstallError::InstallError => "Install error".to_string(), + } + } +} diff --git a/pkgr/src/package/builder/mod.rs b/pkgr/src/package/builder/mod.rs index 50166e1..b841a78 100644 --- a/pkgr/src/package/builder/mod.rs +++ b/pkgr/src/package/builder/mod.rs @@ -1,32 +1,18 @@ -use libc::fork; +use std::fmt::Display; use log::{debug, trace}; +use errors::{BinError, BuildError, InstallError}; use manifest::Manifest; use pkgfile::PKGFile; use crate::tmp::TempDir; +mod errors; + #[derive(Debug)] pub enum InstallType { Build, Bin } -#[derive(Debug)] -pub enum InstallError { - BuildError(String), - BinError(String), - InstallError -} - -impl ToString for InstallError { - fn to_string(&self) -> String { - match self { - InstallError::BuildError(e) => format!("Build error: {}", e), - InstallError::BinError(e) => format!("Bin error: {}", e), - InstallError::InstallError => "Install error".to_string(), - } - } -} - #[derive(Debug)] pub struct PackageInstaller { manifest: Manifest, @@ -43,22 +29,31 @@ impl PackageInstaller { } } - fn extract_to>(&self, path: S) -> Result<(), String> { + fn extract_to(&self, path: String) -> Result<(), BinError> { + if !self.pkgfile.has_data() { + return Err(BinError::UnpackError("package has no data".to_string())); + } + if std::path::Path::new(&path).exists() { + return Err(BinError::UnpackError("path already exists".to_string())); + } tar::Archive::new(self.pkgfile.data.as_slice()) - .unpack(path.into()) - .map_err(|e| e.to_string()) + .unpack(&path) + .map_err(|e| BinError::UnpackError(e.to_string())) } - fn bin(&self) -> Result<(), String> { + fn bin(&self) -> Result<(), BinError> { let mut tmpdir = TempDir::default(); tmpdir.push(&self.manifest.package.name); trace!("extracting package into: {}", tmpdir.to_string()); - self.extract_to(tmpdir.to_string())?; + match self.extract_to(tmpdir.to_string()) { + Ok(_) => {}, + Err(e) => return Err(e) + } debug!("extracted package in: {}", tmpdir.to_string()); Ok(()) } - fn build(&self) -> Result<(), String> { + fn build(&self) -> Result<(), BuildError> { Ok(()) } @@ -68,4 +63,4 @@ impl PackageInstaller { InstallType::Build => self.build().map_err(|e| InstallError::BuildError(e)) } } -} \ No newline at end of file +} From 8cf2876ff00bf86c45acb7588a44ae8bca15d108 Mon Sep 17 00:00:00 2001 From: Raine Date: Sat, 14 Oct 2023 22:39:43 +0200 Subject: [PATCH 066/123] refactor: change name from builder to installer --- pkgr/src/commands.rs | 4 +- pkgr/src/logging.rs | 57 +++++++++++++++++++ pkgr/src/main.rs | 44 +++----------- .../package/{builder => installer}/errors.rs | 0 .../src/package/{builder => installer}/mod.rs | 4 +- pkgr/src/package/mod.rs | 2 +- pkgr/src/{tmp.rs => tmpfs.rs} | 0 7 files changed, 70 insertions(+), 41 deletions(-) create mode 100644 pkgr/src/logging.rs rename pkgr/src/package/{builder => installer}/errors.rs (100%) rename pkgr/src/package/{builder => installer}/mod.rs (97%) rename pkgr/src/{tmp.rs => tmpfs.rs} (100%) diff --git a/pkgr/src/commands.rs b/pkgr/src/commands.rs index e0f7ec0..cc3dfcf 100644 --- a/pkgr/src/commands.rs +++ b/pkgr/src/commands.rs @@ -3,7 +3,7 @@ use crate::package::identifier::PackageIdentifier; use clap::{Parser, Subcommand}; use log::{debug, error, info, trace, warn}; use manifest::Manifest; -use crate::package::builder::{InstallType, PackageInstaller}; +use crate::package::installer::{InstallType, PackageInstaller}; use crate::package::fetch::fetch_package; #[derive(Parser, Debug)] @@ -56,6 +56,7 @@ impl Command { .duration_since(std::time::UNIX_EPOCH) .unwrap(); + info!("Parsing package..."); trace!("Fetching package: {}", package_identifier); let pkgfile = fetch_package(package_identifier.clone()).unwrap(); debug!("size: manifest({}kb) data({}kb)", pkgfile.manifest.len() / 1024, pkgfile.data.len() / 1024); @@ -64,6 +65,7 @@ impl Command { debug!("manifest pkg name: {}", manifest.package.name); trace!("creating installer"); let installer = PackageInstaller::new(manifest.clone(), pkgfile, if *build { InstallType::Build } else { InstallType::Bin }); + info!("Installing: {}...", &manifest.package.name); trace!("starting install"); match installer.install() { diff --git a/pkgr/src/logging.rs b/pkgr/src/logging.rs new file mode 100644 index 0000000..6ee98a5 --- /dev/null +++ b/pkgr/src/logging.rs @@ -0,0 +1,57 @@ +use log::SetLoggerError; +use fern::Dispatch; +use std::env; +use colored::Colorize; + +pub fn setup_logger() -> Result<(), SetLoggerError> { + Dispatch::new() + .format(|out, message, record| { + match record.metadata().target() { + "command:stdout" => { + out.finish(format_args!( + "{} {}", + "::".cyan(), + message.to_string() + )); + return; + } + "command:stderr" => { + out.finish(format_args!( + "{} {}", + "!!".red(), + message.to_string() + )); + return; + } + _ => { + out.finish(format_args!( + "{} {}", + // Some logic so messages look nice + if message.to_string().len() > 0 { + match record + .level() + .to_string() + .chars() + .nth(0) + .unwrap_or('T') + { + 'T' => "[TRACE]".bright_blue(), + 'D' => "??".green(), + 'I' => "=>".blue(), + 'W' => "##".yellow(), + 'E' => "!!".red(), + _ => "**".blue(), + }.to_string() + } else { "".to_string() }, + message.to_string().bright_white() + )) + } + } + }) + .level(env::var("PKGR_LOG_LEVEL") + .unwrap_or_else(|_| "info".to_string()) + .parse() + .unwrap_or(log::LevelFilter::Info)) + .chain(std::io::stdout()) + .apply() +} diff --git a/pkgr/src/main.rs b/pkgr/src/main.rs index ba5f97d..aaa5387 100644 --- a/pkgr/src/main.rs +++ b/pkgr/src/main.rs @@ -1,15 +1,18 @@ +use std::process::Command; use clap::Parser; use colored::Colorize; use crate::commands::Cli; -use log::{SetLoggerError, trace}; -use std::env; +use log::trace; +use crate::process::Process; +mod process; mod commands; mod package; -mod tmp; +mod tmpfs; +mod logging; fn main() { - setup_logger().expect("Unable to setup logger."); + logging::setup_logger().expect("Unable to setup logger."); #[cfg(not(debug_assertions))] { @@ -31,36 +34,3 @@ fn main() { trace!("Exiting..."); } - -fn setup_logger() -> Result<(), SetLoggerError> { - fern::Dispatch::new() - .format(|out, message, record| { - out.finish(format_args!( - "{} {}", - // Some logic so messages look nice - if message.to_string().len() > 0 { - match record - .level() - .to_string() - .chars() - .nth(0) - .unwrap_or('T') - { - 'T' => "[TRACE]".cyan(), - 'D' => "??".green(), - 'I' => "=>".blue(), - 'W' => "##".yellow(), - 'E' => "!!".red(), - _ => "**".blue(), - }.to_string() - } else { "".to_string() }, - message.to_string().bright_white() - )) - }) - .level(env::var("PKGR_LOG_LEVEL") - .unwrap_or_else(|_| "info".to_string()) - .parse() - .unwrap_or(log::LevelFilter::Info)) - .chain(std::io::stdout()) - .apply() -} diff --git a/pkgr/src/package/builder/errors.rs b/pkgr/src/package/installer/errors.rs similarity index 100% rename from pkgr/src/package/builder/errors.rs rename to pkgr/src/package/installer/errors.rs diff --git a/pkgr/src/package/builder/mod.rs b/pkgr/src/package/installer/mod.rs similarity index 97% rename from pkgr/src/package/builder/mod.rs rename to pkgr/src/package/installer/mod.rs index b841a78..53602c6 100644 --- a/pkgr/src/package/builder/mod.rs +++ b/pkgr/src/package/installer/mod.rs @@ -3,9 +3,9 @@ use log::{debug, trace}; use errors::{BinError, BuildError, InstallError}; use manifest::Manifest; use pkgfile::PKGFile; -use crate::tmp::TempDir; +use crate::tmpfs::TempDir; -mod errors; +pub mod errors; #[derive(Debug)] pub enum InstallType { diff --git a/pkgr/src/package/mod.rs b/pkgr/src/package/mod.rs index 5c559fc..f838f7e 100644 --- a/pkgr/src/package/mod.rs +++ b/pkgr/src/package/mod.rs @@ -1,3 +1,3 @@ pub mod identifier; -pub mod builder; +pub mod installer; pub mod fetch; \ No newline at end of file diff --git a/pkgr/src/tmp.rs b/pkgr/src/tmpfs.rs similarity index 100% rename from pkgr/src/tmp.rs rename to pkgr/src/tmpfs.rs From 941679410180024b90bef9ea28b44c7a267d5452 Mon Sep 17 00:00:00 2001 From: Raine Date: Sat, 14 Oct 2023 22:39:43 +0200 Subject: [PATCH 067/123] feat: Process with multithreaded log pipes --- pkgr/src/process/mod.rs | 44 ++++++++++++++++++++++++++++++++++++++ pkgr/src/process/output.rs | 26 ++++++++++++++++++++++ 2 files changed, 70 insertions(+) create mode 100644 pkgr/src/process/mod.rs create mode 100644 pkgr/src/process/output.rs diff --git a/pkgr/src/process/mod.rs b/pkgr/src/process/mod.rs new file mode 100644 index 0000000..70ab93b --- /dev/null +++ b/pkgr/src/process/mod.rs @@ -0,0 +1,44 @@ +use std::process::Command; + +pub mod output; + +pub struct Process { + pub command: Vec, + pub cwd: Option, + pub env: Vec, +} + +impl Process { + pub fn new(command: Vec, cwd: Option, env: Vec) -> Process { + Process { + command, + cwd, + env, + } + } + + pub fn command>(cmd: Vec) -> Process { + Process::new(cmd.into_iter().map(|v| v.into()).collect(), None, vec![]) + } + + pub fn spawn(&self) -> std::io::Result { + let mut child = Command::new(&self.command[0]) + .args(&self.command[1..]) + .current_dir(self.cwd.clone().unwrap_or(".".to_string())) + .envs(self.env.iter().map(|v| { + let mut split = v.split('='); + (split.next().unwrap().to_string(), split.next().unwrap().to_string()) + })) + .stdout(std::process::Stdio::piped()) + .stderr(std::process::Stdio::piped()) + .spawn()?; + output::spawn_output_handlers(&mut child); + Ok(child) + } +} + +impl Default for Process { + fn default() -> Process { + Process::new(vec![], None, vec![]) + } +} \ No newline at end of file diff --git a/pkgr/src/process/output.rs b/pkgr/src/process/output.rs new file mode 100644 index 0000000..09e788e --- /dev/null +++ b/pkgr/src/process/output.rs @@ -0,0 +1,26 @@ +// create functions that spawn threads for Stdout and Stderr +// that print the output to info!(target: "command:stdout", ...) and info!(target: "command:stderr", ...) respectively + +use std::process::Child; +use std::io::{BufRead, BufReader}; + +use log::{info, error}; + +pub fn spawn_output_handlers(child: &mut Child) { + let stdout = child.stdout.take().unwrap(); + let stderr = child.stderr.take().unwrap(); + + std::thread::spawn(move || { + let reader = BufReader::new(stdout); + for line in reader.lines() { + info!(target: "command:stdout", "{}", line.unwrap()); + } + }); + + std::thread::spawn(move || { + let reader = BufReader::new(stderr); + for line in reader.lines() { + error!(target: "command:stderr", "{}", line.unwrap()); + } + }); +} \ No newline at end of file From 3293e407aedaeb6f11b426cf2124623b60e9da85 Mon Sep 17 00:00:00 2001 From: Raine Date: Sat, 14 Oct 2023 22:39:44 +0200 Subject: [PATCH 068/123] refactor: changed how packages work --- manifest/src/build.rs | 6 +-- pkgfile/src/lib.rs | 2 +- pkgr/src/commands.rs | 44 +++++++++--------- pkgr/src/package/fetch.rs | 59 ++++++++++++------------ pkgr/src/package/installer/errors.rs | 8 ++-- pkgr/src/package/installer/mod.rs | 7 +++ pkgr/src/package/mod.rs | 67 +++++++++++++++++++++++++++- 7 files changed, 134 insertions(+), 59 deletions(-) diff --git a/manifest/src/build.rs b/manifest/src/build.rs index 161b785..1879710 100644 --- a/manifest/src/build.rs +++ b/manifest/src/build.rs @@ -4,7 +4,7 @@ use serde::{Deserialize, Serialize}; #[derive(Debug, Clone, Serialize, Deserialize)] pub struct Build { - build_script: String, - install_script: String, - dependencies: HashMap, + pub build_script: String, + pub install_script: String, + pub dependencies: HashMap, } diff --git a/pkgfile/src/lib.rs b/pkgfile/src/lib.rs index f7768b0..407055c 100644 --- a/pkgfile/src/lib.rs +++ b/pkgfile/src/lib.rs @@ -1,4 +1,4 @@ -#[derive(Debug)] +#[derive(Debug, Clone)] pub struct PKGFile { pub manifest: String, pub data: Vec, diff --git a/pkgr/src/commands.rs b/pkgr/src/commands.rs index cc3dfcf..2265330 100644 --- a/pkgr/src/commands.rs +++ b/pkgr/src/commands.rs @@ -4,7 +4,7 @@ use clap::{Parser, Subcommand}; use log::{debug, error, info, trace, warn}; use manifest::Manifest; use crate::package::installer::{InstallType, PackageInstaller}; -use crate::package::fetch::fetch_package; +use crate::package::Package; #[derive(Parser, Debug)] #[clap(name = "pkgr", version)] @@ -58,31 +58,35 @@ impl Command { info!("Parsing package..."); trace!("Fetching package: {}", package_identifier); - let pkgfile = fetch_package(package_identifier.clone()).unwrap(); - debug!("size: manifest({}kb) data({}kb)", pkgfile.manifest.len() / 1024, pkgfile.data.len() / 1024); - trace!("parsing manifest"); - let manifest = Manifest::try_from(pkgfile.manifest.clone()).unwrap(); - debug!("manifest pkg name: {}", manifest.package.name); - trace!("creating installer"); - let installer = PackageInstaller::new(manifest.clone(), pkgfile, if *build { InstallType::Build } else { InstallType::Bin }); - info!("Installing: {}...", &manifest.package.name); + let mut pkg = Package::fetch(package_identifier.clone()).unwrap(); + debug!("manifest size: {}kb", pkg.pkgfile.manifest.len() / 1024); + debug!("files size: {}kb", pkg.pkgfile.data.len() / 1024); - trace!("starting install"); - match installer.install() { - Ok(_) => { - info!("Sucessfully installed: {}", &manifest.package.name); - () - }, + trace!("Checking if package is installed..."); + if pkg.is_installed() { + error!("Package is already installed."); + exit(1); + } + + trace!("Starting install..."); + match pkg.install() { + Ok(_) => (), Err(e) => { - error!("{}", e.to_string()); + error!("Install failed: {}", e.to_string()); exit(1); } } - let unix_end = std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).unwrap(); + let end = std::time::Instant::now(); - // float ms - let duration = (end - start).as_nanos() as f64; - info!("Install took: {}ms", (duration / 1_000_000.0)); + let unix_end = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap(); + + let duration = end.duration_since(start); + + info!("Install complete."); + info!("Install took {}ms.", duration.as_nanos() as f64 / 1000000.0); + } Command::Remove { package_identifier, index } => { if let PackageIdentifier::URI(_) = package_identifier { diff --git a/pkgr/src/package/fetch.rs b/pkgr/src/package/fetch.rs index 0e68045..099c5b2 100644 --- a/pkgr/src/package/fetch.rs +++ b/pkgr/src/package/fetch.rs @@ -1,11 +1,9 @@ use std::error::Error; use std::fmt::Display; -use std::fs::File; use std::io::Read; -use log::warn; use pkgfile::PKGFile; -use crate::package::identifier::PackageIdentifier; use reqwest::blocking::get; +use crate::package::identifier::PackageLocator; #[derive(Debug)] pub enum FetchError { @@ -26,34 +24,33 @@ impl Display for FetchError { impl Error for FetchError {} -pub fn fetch_package(package_identifier: PackageIdentifier) -> Result { - match package_identifier { - PackageIdentifier::Path(path) => { - std::fs::read(path) - .map_err(|e| FetchError::IOError(e)).and_then(|bytes| { - PKGFile::try_from(bytes).map_err(|_| FetchError::ParseError) - }) - } - PackageIdentifier::URI(uri) => { - // get file contents as bytes - let mut bytes = Vec::new(); - match get(&uri) { - Ok(mut response) => { - match response.take(1024 * 1024).read_to_end(&mut bytes) { - Ok(_) => (), - Err(e) => return Err(FetchError::IOError(e)), - }; - } - Err(e) => return Err(FetchError::HTTPError(e)), +pub fn fetch_by_path>(path: S) -> Result { + std::fs::read(path.into()) + .map_err(|e| FetchError::IOError(e)).and_then(|bytes| { + PKGFile::try_from(bytes).map_err(|_| FetchError::ParseError) + }) +} + +pub fn fetch_by_uri>(uri: S) -> Result { + // get file contents as bytes + let mut bytes = Vec::new(); + match get(uri.into()) { + Ok(mut response) => { + match response.take(1024 * 1024).read_to_end(&mut bytes) { + Ok(_) => (), + Err(e) => return Err(FetchError::IOError(e)), }; - // parse bytes as PKGFile - match PKGFile::try_from(bytes) { - Ok(pkgfile) => Ok(pkgfile), - Err(e) => Err(FetchError::ParseError), - } - } - PackageIdentifier::PackageLocator(package_locator) => { - Ok(PKGFile::default()) } + Err(e) => return Err(FetchError::HTTPError(e)), + }; + // parse bytes as PKGFile + match PKGFile::try_from(bytes) { + Ok(pkgfile) => Ok(pkgfile), + Err(e) => Err(FetchError::ParseError), } -} \ No newline at end of file +} + +pub fn fetch_by_package_locator(package_locator: PackageLocator) -> Result { + // TODO: search index for package locator + Ok(PKGFile::default()) +} diff --git a/pkgr/src/package/installer/errors.rs b/pkgr/src/package/installer/errors.rs index e5f62d6..a6801c0 100644 --- a/pkgr/src/package/installer/errors.rs +++ b/pkgr/src/package/installer/errors.rs @@ -14,7 +14,9 @@ impl Display for BinError { } #[derive(Debug)] -pub enum BuildError {} +pub enum BuildError { + InvalidManifest +} impl Display for BuildError { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { @@ -28,7 +30,7 @@ impl Display for BuildError { pub enum InstallError { BuildError(BuildError), BinError(BinError), - InstallError + Generic } impl ToString for InstallError { @@ -36,7 +38,7 @@ impl ToString for InstallError { match self { InstallError::BuildError(e) => format!("Build error: {}", e), InstallError::BinError(e) => format!("Bin error: {}", e), - InstallError::InstallError => "Install error".to_string(), + InstallError::Generic => "Install error".to_string(), } } } diff --git a/pkgr/src/package/installer/mod.rs b/pkgr/src/package/installer/mod.rs index 53602c6..8f75941 100644 --- a/pkgr/src/package/installer/mod.rs +++ b/pkgr/src/package/installer/mod.rs @@ -54,6 +54,13 @@ impl PackageInstaller { } fn build(&self) -> Result<(), BuildError> { + if let None = self.manifest.build.clone() { + return Err(BuildError::InvalidManifest); + } + let build_manifest = self.manifest.build.clone().unwrap(); + // TODO: Check dependencies + + Ok(()) } diff --git a/pkgr/src/package/mod.rs b/pkgr/src/package/mod.rs index f838f7e..349c360 100644 --- a/pkgr/src/package/mod.rs +++ b/pkgr/src/package/mod.rs @@ -1,3 +1,68 @@ +use log::trace; + pub mod identifier; pub mod installer; -pub mod fetch; \ No newline at end of file +pub mod fetch; + +pub struct Package { + pub identifier: identifier::PackageIdentifier, + pub pkgfile: pkgfile::PKGFile, + is_installed: bool, + is_indexed: bool, +} + +impl Package { + pub fn new(identifier: identifier::PackageIdentifier, pkgfile: pkgfile::PKGFile) -> Package { + Package { + identifier, + pkgfile, + is_installed: false, + is_indexed: false, + } + } + + pub fn manifest(&self) -> manifest::Manifest { + manifest::Manifest::try_from(self.pkgfile.manifest.clone()) + .unwrap() + } + + pub fn fetch(package_identifier: identifier::PackageIdentifier) -> Result { + match &package_identifier { + identifier::PackageIdentifier::Path(path) => { + trace!("fetching package from path: {}", path); + let pkgfile = fetch::fetch_by_path(path).unwrap(); + Ok(Package::new(package_identifier, pkgfile)) + }, + identifier::PackageIdentifier::URI(url) => { + trace!("fetching package from uri: {}", url); + let pkgfile = fetch::fetch_by_uri(url).unwrap(); + Ok(Package::new(package_identifier, pkgfile)) + }, + identifier::PackageIdentifier::PackageLocator(locator) => { + trace!("fetching package from locator: {}", locator); + let pkgfile = fetch::fetch_by_package_locator(locator.clone()).unwrap(); + Ok(Package::new(package_identifier, pkgfile)) + }, + } + } + + pub fn install(&mut self) -> Result<(), installer::errors::InstallError> { + let manifest = self.manifest(); + let installer = installer::PackageInstaller::new(manifest.clone(), self.pkgfile.clone(), installer::InstallType::Bin); + match installer.install() { + Ok(_) => { + self.is_installed = true; + Ok(()) + }, + Err(e) => Err(e) + } + } + + pub fn is_installed(&self) -> bool { + self.is_installed + } + + pub fn is_indexed(&self) -> bool { + self.is_indexed + } +} \ No newline at end of file From e3552e9208d7e9746e7155f7d1e0bc7130815793 Mon Sep 17 00:00:00 2001 From: Raine Date: Sat, 14 Oct 2023 22:39:44 +0200 Subject: [PATCH 069/123] doc: rustdocs --- pkgr/src/package/mod.rs | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/pkgr/src/package/mod.rs b/pkgr/src/package/mod.rs index 349c360..7a7d6b2 100644 --- a/pkgr/src/package/mod.rs +++ b/pkgr/src/package/mod.rs @@ -12,6 +12,7 @@ pub struct Package { } impl Package { + /// Create a new package from a package identifier and a package file. pub fn new(identifier: identifier::PackageIdentifier, pkgfile: pkgfile::PKGFile) -> Package { Package { identifier, @@ -21,11 +22,7 @@ impl Package { } } - pub fn manifest(&self) -> manifest::Manifest { - manifest::Manifest::try_from(self.pkgfile.manifest.clone()) - .unwrap() - } - + /// Fetch a package from a package identifier. pub fn fetch(package_identifier: identifier::PackageIdentifier) -> Result { match &package_identifier { identifier::PackageIdentifier::Path(path) => { @@ -46,6 +43,14 @@ impl Package { } } + /// Get the package manifest. + pub fn manifest(&self) -> manifest::Manifest { + manifest::Manifest::try_from(self.pkgfile.manifest.clone()) + .unwrap() + } + + + /// Install the package. pub fn install(&mut self) -> Result<(), installer::errors::InstallError> { let manifest = self.manifest(); let installer = installer::PackageInstaller::new(manifest.clone(), self.pkgfile.clone(), installer::InstallType::Bin); @@ -58,11 +63,18 @@ impl Package { } } + /// Check if the package is installed. pub fn is_installed(&self) -> bool { self.is_installed } + /// Check if the package is indexed. pub fn is_indexed(&self) -> bool { self.is_indexed } + + /// Remove the package from the system. + pub fn remove(&mut self) -> Result<(), ()> { + unimplemented!(); + } } \ No newline at end of file From 9e102ffb52c5ef624d3d10c3eb6f492f1f1bc6e0 Mon Sep 17 00:00:00 2001 From: Raine Date: Sat, 14 Oct 2023 22:39:44 +0200 Subject: [PATCH 070/123] format: fix formatting --- manifest/src/pkgr.rs | 4 +-- pkgr/src/commands.rs | 17 +++++++---- pkgr/src/logging.rs | 43 +++++++++++----------------- pkgr/src/main.rs | 12 ++++---- pkgr/src/package/fetch.rs | 11 ++++--- pkgr/src/package/identifier.rs | 2 +- pkgr/src/package/installer/errors.rs | 4 +-- pkgr/src/package/installer/mod.rs | 19 ++++++------ pkgr/src/package/mod.rs | 28 ++++++++++-------- pkgr/src/process/mod.rs | 13 ++++----- pkgr/src/process/output.rs | 6 ++-- pkgr/src/tmpfs.rs | 6 ++-- 12 files changed, 80 insertions(+), 85 deletions(-) diff --git a/manifest/src/pkgr.rs b/manifest/src/pkgr.rs index 700c6cb..530a37c 100644 --- a/manifest/src/pkgr.rs +++ b/manifest/src/pkgr.rs @@ -1,5 +1,5 @@ -use std::fmt::{Display, Formatter}; use serde::{Deserialize, Serialize}; +use std::fmt::{Display, Formatter}; #[derive(Debug, Clone, Serialize, Deserialize)] pub struct PKGR {} @@ -8,4 +8,4 @@ impl Display for PKGR { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { write!(f, "") } -} \ No newline at end of file +} diff --git a/pkgr/src/commands.rs b/pkgr/src/commands.rs index 2265330..8c7996e 100644 --- a/pkgr/src/commands.rs +++ b/pkgr/src/commands.rs @@ -1,10 +1,10 @@ -use std::process::exit; use crate::package::identifier::PackageIdentifier; +use crate::package::installer::{InstallType, PackageInstaller}; +use crate::package::Package; use clap::{Parser, Subcommand}; use log::{debug, error, info, trace, warn}; use manifest::Manifest; -use crate::package::installer::{InstallType, PackageInstaller}; -use crate::package::Package; +use std::process::exit; #[derive(Parser, Debug)] #[clap(name = "pkgr", version)] @@ -86,9 +86,11 @@ impl Command { info!("Install complete."); info!("Install took {}ms.", duration.as_nanos() as f64 / 1000000.0); - } - Command::Remove { package_identifier, index } => { + Command::Remove { + package_identifier, + index, + } => { if let PackageIdentifier::URI(_) = package_identifier { error!("URI is unsupported when removing applications."); exit(1); @@ -113,7 +115,10 @@ impl Command { info!("PKGR AUTHORS: {}", env!("CARGO_PKG_AUTHORS")); info!("PKGR DESCRIPTION: {}", env!("CARGO_PKG_DESCRIPTION")); info!(""); - info!("PKGR_LOG_LEVEL: {}", std::env::var("PKGR_LOG_LEVEL").unwrap_or_else(|_| "info".to_string())); + info!( + "PKGR_LOG_LEVEL: {}", + std::env::var("PKGR_LOG_LEVEL").unwrap_or_else(|_| "info".to_string()) + ); } Command::None => { error!("No command was specified."); diff --git a/pkgr/src/logging.rs b/pkgr/src/logging.rs index 6ee98a5..225ef27 100644 --- a/pkgr/src/logging.rs +++ b/pkgr/src/logging.rs @@ -1,26 +1,18 @@ -use log::SetLoggerError; -use fern::Dispatch; -use std::env; use colored::Colorize; +use fern::Dispatch; +use log::SetLoggerError; +use std::env; pub fn setup_logger() -> Result<(), SetLoggerError> { Dispatch::new() .format(|out, message, record| { match record.metadata().target() { "command:stdout" => { - out.finish(format_args!( - "{} {}", - "::".cyan(), - message.to_string() - )); + out.finish(format_args!("{} {}", "::".cyan(), message.to_string())); return; } "command:stderr" => { - out.finish(format_args!( - "{} {}", - "!!".red(), - message.to_string() - )); + out.finish(format_args!("{} {}", "!!".red(), message.to_string())); return; } _ => { @@ -28,30 +20,29 @@ pub fn setup_logger() -> Result<(), SetLoggerError> { "{} {}", // Some logic so messages look nice if message.to_string().len() > 0 { - match record - .level() - .to_string() - .chars() - .nth(0) - .unwrap_or('T') - { + match record.level().to_string().chars().nth(0).unwrap_or('T') { 'T' => "[TRACE]".bright_blue(), 'D' => "??".green(), 'I' => "=>".blue(), 'W' => "##".yellow(), 'E' => "!!".red(), _ => "**".blue(), - }.to_string() - } else { "".to_string() }, + } + .to_string() + } else { + "".to_string() + }, message.to_string().bright_white() )) } } }) - .level(env::var("PKGR_LOG_LEVEL") - .unwrap_or_else(|_| "info".to_string()) - .parse() - .unwrap_or(log::LevelFilter::Info)) + .level( + env::var("PKGR_LOG_LEVEL") + .unwrap_or_else(|_| "info".to_string()) + .parse() + .unwrap_or(log::LevelFilter::Info), + ) .chain(std::io::stdout()) .apply() } diff --git a/pkgr/src/main.rs b/pkgr/src/main.rs index aaa5387..cb73e53 100644 --- a/pkgr/src/main.rs +++ b/pkgr/src/main.rs @@ -1,15 +1,15 @@ -use std::process::Command; +use crate::commands::Cli; +use crate::process::Process; use clap::Parser; use colored::Colorize; -use crate::commands::Cli; use log::trace; -use crate::process::Process; +use std::process::Command; -mod process; mod commands; -mod package; -mod tmpfs; mod logging; +mod package; +mod process; +mod tmpfs; fn main() { logging::setup_logger().expect("Unable to setup logger."); diff --git a/pkgr/src/package/fetch.rs b/pkgr/src/package/fetch.rs index 099c5b2..a97dfb6 100644 --- a/pkgr/src/package/fetch.rs +++ b/pkgr/src/package/fetch.rs @@ -1,9 +1,9 @@ +use crate::package::identifier::PackageLocator; +use pkgfile::PKGFile; +use reqwest::blocking::get; use std::error::Error; use std::fmt::Display; use std::io::Read; -use pkgfile::PKGFile; -use reqwest::blocking::get; -use crate::package::identifier::PackageLocator; #[derive(Debug)] pub enum FetchError { @@ -26,9 +26,8 @@ impl Error for FetchError {} pub fn fetch_by_path>(path: S) -> Result { std::fs::read(path.into()) - .map_err(|e| FetchError::IOError(e)).and_then(|bytes| { - PKGFile::try_from(bytes).map_err(|_| FetchError::ParseError) - }) + .map_err(|e| FetchError::IOError(e)) + .and_then(|bytes| PKGFile::try_from(bytes).map_err(|_| FetchError::ParseError)) } pub fn fetch_by_uri>(uri: S) -> Result { diff --git a/pkgr/src/package/identifier.rs b/pkgr/src/package/identifier.rs index fa457d8..17064e4 100644 --- a/pkgr/src/package/identifier.rs +++ b/pkgr/src/package/identifier.rs @@ -88,7 +88,7 @@ impl FromStr for PackageLocator { fn from_str(s: &str) -> Result { #[allow(unused_assignments)] // false positive - let mut name = None; + let mut name = None; let mut version = None; let mut tags = None; diff --git a/pkgr/src/package/installer/errors.rs b/pkgr/src/package/installer/errors.rs index a6801c0..46cd402 100644 --- a/pkgr/src/package/installer/errors.rs +++ b/pkgr/src/package/installer/errors.rs @@ -15,7 +15,7 @@ impl Display for BinError { #[derive(Debug)] pub enum BuildError { - InvalidManifest + InvalidManifest, } impl Display for BuildError { @@ -30,7 +30,7 @@ impl Display for BuildError { pub enum InstallError { BuildError(BuildError), BinError(BinError), - Generic + Generic, } impl ToString for InstallError { diff --git a/pkgr/src/package/installer/mod.rs b/pkgr/src/package/installer/mod.rs index 8f75941..0d474aa 100644 --- a/pkgr/src/package/installer/mod.rs +++ b/pkgr/src/package/installer/mod.rs @@ -1,23 +1,23 @@ -use std::fmt::Display; -use log::{debug, trace}; +use crate::tmpfs::TempDir; use errors::{BinError, BuildError, InstallError}; +use log::{debug, trace}; use manifest::Manifest; use pkgfile::PKGFile; -use crate::tmpfs::TempDir; +use std::fmt::Display; pub mod errors; #[derive(Debug)] pub enum InstallType { Build, - Bin + Bin, } #[derive(Debug)] pub struct PackageInstaller { manifest: Manifest, pkgfile: PKGFile, - install_type: InstallType + install_type: InstallType, } impl PackageInstaller { @@ -25,7 +25,7 @@ impl PackageInstaller { PackageInstaller { manifest: m, pkgfile: p, - install_type: i + install_type: i, } } @@ -46,8 +46,8 @@ impl PackageInstaller { tmpdir.push(&self.manifest.package.name); trace!("extracting package into: {}", tmpdir.to_string()); match self.extract_to(tmpdir.to_string()) { - Ok(_) => {}, - Err(e) => return Err(e) + Ok(_) => {} + Err(e) => return Err(e), } debug!("extracted package in: {}", tmpdir.to_string()); Ok(()) @@ -60,14 +60,13 @@ impl PackageInstaller { let build_manifest = self.manifest.build.clone().unwrap(); // TODO: Check dependencies - Ok(()) } pub fn install(&self) -> Result<(), InstallError> { match self.install_type { InstallType::Bin => self.bin().map_err(|e| InstallError::BinError(e)), - InstallType::Build => self.build().map_err(|e| InstallError::BuildError(e)) + InstallType::Build => self.build().map_err(|e| InstallError::BuildError(e)), } } } diff --git a/pkgr/src/package/mod.rs b/pkgr/src/package/mod.rs index 7a7d6b2..e23a191 100644 --- a/pkgr/src/package/mod.rs +++ b/pkgr/src/package/mod.rs @@ -1,8 +1,8 @@ use log::trace; +pub mod fetch; pub mod identifier; pub mod installer; -pub mod fetch; pub struct Package { pub identifier: identifier::PackageIdentifier, @@ -23,43 +23,47 @@ impl Package { } /// Fetch a package from a package identifier. - pub fn fetch(package_identifier: identifier::PackageIdentifier) -> Result { + pub fn fetch( + package_identifier: identifier::PackageIdentifier, + ) -> Result { match &package_identifier { identifier::PackageIdentifier::Path(path) => { trace!("fetching package from path: {}", path); let pkgfile = fetch::fetch_by_path(path).unwrap(); Ok(Package::new(package_identifier, pkgfile)) - }, + } identifier::PackageIdentifier::URI(url) => { trace!("fetching package from uri: {}", url); let pkgfile = fetch::fetch_by_uri(url).unwrap(); Ok(Package::new(package_identifier, pkgfile)) - }, + } identifier::PackageIdentifier::PackageLocator(locator) => { trace!("fetching package from locator: {}", locator); let pkgfile = fetch::fetch_by_package_locator(locator.clone()).unwrap(); Ok(Package::new(package_identifier, pkgfile)) - }, + } } } /// Get the package manifest. pub fn manifest(&self) -> manifest::Manifest { - manifest::Manifest::try_from(self.pkgfile.manifest.clone()) - .unwrap() + manifest::Manifest::try_from(self.pkgfile.manifest.clone()).unwrap() } - /// Install the package. pub fn install(&mut self) -> Result<(), installer::errors::InstallError> { let manifest = self.manifest(); - let installer = installer::PackageInstaller::new(manifest.clone(), self.pkgfile.clone(), installer::InstallType::Bin); + let installer = installer::PackageInstaller::new( + manifest.clone(), + self.pkgfile.clone(), + installer::InstallType::Bin, + ); match installer.install() { Ok(_) => { self.is_installed = true; Ok(()) - }, - Err(e) => Err(e) + } + Err(e) => Err(e), } } @@ -77,4 +81,4 @@ impl Package { pub fn remove(&mut self) -> Result<(), ()> { unimplemented!(); } -} \ No newline at end of file +} diff --git a/pkgr/src/process/mod.rs b/pkgr/src/process/mod.rs index 70ab93b..21586c0 100644 --- a/pkgr/src/process/mod.rs +++ b/pkgr/src/process/mod.rs @@ -10,11 +10,7 @@ pub struct Process { impl Process { pub fn new(command: Vec, cwd: Option, env: Vec) -> Process { - Process { - command, - cwd, - env, - } + Process { command, cwd, env } } pub fn command>(cmd: Vec) -> Process { @@ -27,7 +23,10 @@ impl Process { .current_dir(self.cwd.clone().unwrap_or(".".to_string())) .envs(self.env.iter().map(|v| { let mut split = v.split('='); - (split.next().unwrap().to_string(), split.next().unwrap().to_string()) + ( + split.next().unwrap().to_string(), + split.next().unwrap().to_string(), + ) })) .stdout(std::process::Stdio::piped()) .stderr(std::process::Stdio::piped()) @@ -41,4 +40,4 @@ impl Default for Process { fn default() -> Process { Process::new(vec![], None, vec![]) } -} \ No newline at end of file +} diff --git a/pkgr/src/process/output.rs b/pkgr/src/process/output.rs index 09e788e..68d91fd 100644 --- a/pkgr/src/process/output.rs +++ b/pkgr/src/process/output.rs @@ -1,10 +1,10 @@ // create functions that spawn threads for Stdout and Stderr // that print the output to info!(target: "command:stdout", ...) and info!(target: "command:stderr", ...) respectively -use std::process::Child; use std::io::{BufRead, BufReader}; +use std::process::Child; -use log::{info, error}; +use log::{error, info}; pub fn spawn_output_handlers(child: &mut Child) { let stdout = child.stdout.take().unwrap(); @@ -23,4 +23,4 @@ pub fn spawn_output_handlers(child: &mut Child) { error!(target: "command:stderr", "{}", line.unwrap()); } }); -} \ No newline at end of file +} diff --git a/pkgr/src/tmpfs.rs b/pkgr/src/tmpfs.rs index 509d3f7..c9ea6f8 100644 --- a/pkgr/src/tmpfs.rs +++ b/pkgr/src/tmpfs.rs @@ -9,9 +9,7 @@ impl TempDir { if !path.exists() { std::fs::create_dir_all(&path).unwrap(); } - TempDir { - path - } + TempDir { path } } pub fn push>(&mut self, path: S) { @@ -45,4 +43,4 @@ impl ToString for TempDir { fn to_string(&self) -> String { self.path.to_str().unwrap().to_string() } -} \ No newline at end of file +} From 6cd0af304fb1f0e77de4731fd6493d70635e7a14 Mon Sep 17 00:00:00 2001 From: Raine Date: Sat, 14 Oct 2023 22:39:44 +0200 Subject: [PATCH 071/123] fix: removal of file used in old spec --- build-pkg.toml | 14 -------------- 1 file changed, 14 deletions(-) delete mode 100644 build-pkg.toml diff --git a/build-pkg.toml b/build-pkg.toml deleted file mode 100644 index 8c994d9..0000000 --- a/build-pkg.toml +++ /dev/null @@ -1,14 +0,0 @@ -[project] -package_file = "./package.toml" -package_dir = "./pkg" -increment_version = false - -[mappings.source] -ignore = ["target/"] - -[mappings.source.map] -"." = "/src" - -[mappings.bin.map] -"target/release/bin/pkgr" = "/bin/pkgr" - From 6981fe8f162c83528eb5a3688583481f9759e3f7 Mon Sep 17 00:00:00 2001 From: Raine Date: Sat, 14 Oct 2023 22:39:44 +0200 Subject: [PATCH 072/123] fix: remove useless threading --- pkgr/src/commands.rs | 10 ++++- pkgr/src/config/mod.rs | 27 +++++++++++++ pkgr/src/main.rs | 7 +++- pkgr/src/package/identifier.rs | 33 ++++++++++++++++ pkgr/src/package/installer/errors.rs | 2 + pkgr/src/package/installer/mod.rs | 52 +++++++++++++++++++++++-- pkgr/src/package/mod.rs | 22 +++++++++-- pkgr/src/{process/mod.rs => process.rs} | 16 +++++++- pkgr/src/process/output.rs | 26 ------------- 9 files changed, 157 insertions(+), 38 deletions(-) create mode 100644 pkgr/src/config/mod.rs rename pkgr/src/{process/mod.rs => process.rs} (73%) delete mode 100644 pkgr/src/process/output.rs diff --git a/pkgr/src/commands.rs b/pkgr/src/commands.rs index 8c7996e..6d8e139 100644 --- a/pkgr/src/commands.rs +++ b/pkgr/src/commands.rs @@ -5,6 +5,8 @@ use clap::{Parser, Subcommand}; use log::{debug, error, info, trace, warn}; use manifest::Manifest; use std::process::exit; +use crate::CONFIG; +use crate::process::Process; #[derive(Parser, Debug)] #[clap(name = "pkgr", version)] @@ -69,7 +71,7 @@ impl Command { } trace!("Starting install..."); - match pkg.install() { + match pkg.install(CONFIG.with(|c| if !*build { c.build_by_default } else { *build })) { Ok(_) => (), Err(e) => { error!("Install failed: {}", e.to_string()); @@ -110,6 +112,12 @@ impl Command { info!("Info message."); warn!("Warning message."); error!("Error message."); + + info!(""); + Process::command(vec!["sh", "-c", "echo whoami: $(whoami)"].iter_mut().map(|s| s.to_string()).collect()) + .spawn() + .unwrap(); + info!(""); info!("PKGR VERSION: {}", env!("CARGO_PKG_VERSION")); info!("PKGR AUTHORS: {}", env!("CARGO_PKG_AUTHORS")); diff --git a/pkgr/src/config/mod.rs b/pkgr/src/config/mod.rs new file mode 100644 index 0000000..29f4936 --- /dev/null +++ b/pkgr/src/config/mod.rs @@ -0,0 +1,27 @@ +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Serialize, Deserialize)] +pub struct Config { + #[serde(default)] + pub build_by_default: bool, +} + +impl Config { + pub fn from_path(path: &str) -> Result { + match std::fs::read_to_string(path) { + Ok(s) => match toml::from_str(&s) { + Ok(c) => Ok(c), + Err(e) => Err(format!("failed to parse config: {}", e)), + }, + Err(e) => Err(format!("failed to read config: {}", e)), + } + } +} + +impl Default for Config { + fn default() -> Config { + Config { + build_by_default: false, + } + } +} \ No newline at end of file diff --git a/pkgr/src/main.rs b/pkgr/src/main.rs index cb73e53..97f721f 100644 --- a/pkgr/src/main.rs +++ b/pkgr/src/main.rs @@ -10,13 +10,18 @@ mod logging; mod package; mod process; mod tmpfs; +mod config; + +thread_local! { + static CONFIG: config::Config = config::Config::from_path("/etc/pkgr.toml") + .unwrap_or(config::Config::default()); +} fn main() { logging::setup_logger().expect("Unable to setup logger."); #[cfg(not(debug_assertions))] { - #[cfg(target_family = "unix")] if unsafe { libc::getuid() } != 0 { use log::error; error!("pkgr must be run as root."); diff --git a/pkgr/src/package/identifier.rs b/pkgr/src/package/identifier.rs index 17064e4..427b775 100644 --- a/pkgr/src/package/identifier.rs +++ b/pkgr/src/package/identifier.rs @@ -128,3 +128,36 @@ impl FromStr for PackageLocator { }) } } + +impl From<(String, String)> for PackageLocator { + fn from((name, locate_str): (String, String)) -> Self { + // name = "pkg" + // locate_str = "1.0.0:tag1,tag2" or "1.0.0" or "tag1,tag2" + let mut version = None; + let mut tags = None; + + let version_re = Regex::new("^([0-9]+)").unwrap(); + let tags_re = Regex::new("^:([a-zA-Z0-9,._]+)").unwrap(); + + if let Some(caps) = version_re.captures(locate_str.as_str()) { + version = Some(caps.get(1).unwrap().as_str().parse::().unwrap()); + } + + if let Some(caps) = tags_re.captures(locate_str.as_str()) { + tags = Some( + caps.get(1) + .unwrap() + .as_str() + .split(",") + .map(|s| s.to_string()) + .collect(), + ); + } + + PackageLocator { + name, + version, + tags, + } + } +} diff --git a/pkgr/src/package/installer/errors.rs b/pkgr/src/package/installer/errors.rs index 46cd402..637e891 100644 --- a/pkgr/src/package/installer/errors.rs +++ b/pkgr/src/package/installer/errors.rs @@ -30,6 +30,7 @@ impl Display for BuildError { pub enum InstallError { BuildError(BuildError), BinError(BinError), + InvalidManifest, Generic, } @@ -38,6 +39,7 @@ impl ToString for InstallError { match self { InstallError::BuildError(e) => format!("Build error: {}", e), InstallError::BinError(e) => format!("Bin error: {}", e), + InstallError::InvalidManifest => "Invalid manifest".to_string(), InstallError::Generic => "Install error".to_string(), } } diff --git a/pkgr/src/package/installer/mod.rs b/pkgr/src/package/installer/mod.rs index 0d474aa..8181ea8 100644 --- a/pkgr/src/package/installer/mod.rs +++ b/pkgr/src/package/installer/mod.rs @@ -1,9 +1,13 @@ use crate::tmpfs::TempDir; use errors::{BinError, BuildError, InstallError}; -use log::{debug, trace}; +use log::{debug, error, info, trace}; use manifest::Manifest; use pkgfile::PKGFile; use std::fmt::Display; +use std::process::exit; +use crate::CONFIG; +use crate::package::identifier::{PackageIdentifier, PackageLocator}; +use crate::package::Package; pub mod errors; @@ -59,14 +63,54 @@ impl PackageInstaller { } let build_manifest = self.manifest.build.clone().unwrap(); // TODO: Check dependencies + for pkg_tuple in build_manifest.dependencies { + let mut pkg = Package::fetch( + PackageIdentifier::PackageLocator( + PackageLocator::from(pkg_tuple) + ) + ).expect("no pkg"); + + if !pkg.is_installed() { + match pkg.install(CONFIG.with(|c| { + c.build_by_default + })) { + Ok(_) => { info!("Installed dependency: \"{}\"", pkg.manifest().package.name) }, + Err(_) => { + error!("Could not install dependency: \"{}\"", pkg.identifier); + exit(1); + } + } + } + } Ok(()) } - pub fn install(&self) -> Result<(), InstallError> { + pub fn install(&mut self) -> Result<(), InstallError> { match self.install_type { - InstallType::Bin => self.bin().map_err(|e| InstallError::BinError(e)), - InstallType::Build => self.build().map_err(|e| InstallError::BuildError(e)), + InstallType::Bin => { + if let None = self.manifest.bin { + self.install_type = InstallType::Build; + if let None = self.manifest.build { + return Err(InstallError::InvalidManifest); + } + } + } + InstallType::Build => { + if let None = self.manifest.build { + self.install_type = InstallType::Bin; + if let None = self.manifest.bin { + return Err(InstallError::InvalidManifest); + } + } + } + } + + match self.install_type { + InstallType::Bin => self.bin() + .map_err(|e| InstallError::BinError(e)), + InstallType::Build => self.build() + .map_err(|e| InstallError::BuildError(e)), } } } diff --git a/pkgr/src/package/mod.rs b/pkgr/src/package/mod.rs index e23a191..79de18d 100644 --- a/pkgr/src/package/mod.rs +++ b/pkgr/src/package/mod.rs @@ -1,4 +1,4 @@ -use log::trace; +use log::{info, trace}; pub mod fetch; pub mod identifier; @@ -51,12 +51,26 @@ impl Package { } /// Install the package. - pub fn install(&mut self) -> Result<(), installer::errors::InstallError> { + pub fn install(&mut self, build: bool) -> Result<(), installer::errors::InstallError> { let manifest = self.manifest(); - let installer = installer::PackageInstaller::new( + let mut installer = installer::PackageInstaller::new( manifest.clone(), self.pkgfile.clone(), - installer::InstallType::Bin, + if build { + info!("building package: {}...", if self.manifest().package.name.is_empty() { + self.identifier.to_string() + } else { + self.manifest().package.name + }); + installer::InstallType::Build + } else { + info!("using binary install for package: {}...", if self.manifest().package.name.is_empty() { + self.identifier.to_string() + } else { + self.manifest().package.name + }); + installer::InstallType::Bin + }, ); match installer.install() { Ok(_) => { diff --git a/pkgr/src/process/mod.rs b/pkgr/src/process.rs similarity index 73% rename from pkgr/src/process/mod.rs rename to pkgr/src/process.rs index 21586c0..727529f 100644 --- a/pkgr/src/process/mod.rs +++ b/pkgr/src/process.rs @@ -1,6 +1,7 @@ +use std::io::{BufRead, BufReader}; use std::process::Command; +use log::info; -pub mod output; pub struct Process { pub command: Vec, @@ -31,7 +32,18 @@ impl Process { .stdout(std::process::Stdio::piped()) .stderr(std::process::Stdio::piped()) .spawn()?; - output::spawn_output_handlers(&mut child); + + let stdout = child.stdout.take().unwrap(); + let stderr = child.stderr.take().unwrap(); + + for line in BufReader::new(stdout).lines() { + info!(target: "command:stdout", "{}", line.unwrap()); + } + + for line in BufReader::new(stderr).lines() { + info!(target: "command:stderr", "{}", line.unwrap()); + } + Ok(child) } } diff --git a/pkgr/src/process/output.rs b/pkgr/src/process/output.rs deleted file mode 100644 index 68d91fd..0000000 --- a/pkgr/src/process/output.rs +++ /dev/null @@ -1,26 +0,0 @@ -// create functions that spawn threads for Stdout and Stderr -// that print the output to info!(target: "command:stdout", ...) and info!(target: "command:stderr", ...) respectively - -use std::io::{BufRead, BufReader}; -use std::process::Child; - -use log::{error, info}; - -pub fn spawn_output_handlers(child: &mut Child) { - let stdout = child.stdout.take().unwrap(); - let stderr = child.stderr.take().unwrap(); - - std::thread::spawn(move || { - let reader = BufReader::new(stdout); - for line in reader.lines() { - info!(target: "command:stdout", "{}", line.unwrap()); - } - }); - - std::thread::spawn(move || { - let reader = BufReader::new(stderr); - for line in reader.lines() { - error!(target: "command:stderr", "{}", line.unwrap()); - } - }); -} From 401efe38623d6477c8ab574daafcec447aefd60b Mon Sep 17 00:00:00 2001 From: Raine Date: Sat, 14 Oct 2023 22:39:44 +0200 Subject: [PATCH 073/123] feat: add pkgr global config --- pkgr/src/commands.rs | 7 ++++++- pkgr/src/config/mod.rs | 16 ++++++++-------- pkgr/src/process.rs | 8 ++++++-- 3 files changed, 20 insertions(+), 11 deletions(-) diff --git a/pkgr/src/commands.rs b/pkgr/src/commands.rs index 6d8e139..d63e2e3 100644 --- a/pkgr/src/commands.rs +++ b/pkgr/src/commands.rs @@ -114,9 +114,14 @@ impl Command { error!("Error message."); info!(""); - Process::command(vec!["sh", "-c", "echo whoami: $(whoami)"].iter_mut().map(|s| s.to_string()).collect()) + Process::command(vec!["sh", "-c", "echo whoami: $(whoami)"]) .spawn() .unwrap(); + if let Ok(user) = std::env::var("SUDO_USER") { + Process::command(vec!["sh", "-c", "echo sudo_user: $SUDO_USER"]) + .spawn() + .unwrap(); + } info!(""); info!("PKGR VERSION: {}", env!("CARGO_PKG_VERSION")); diff --git a/pkgr/src/config/mod.rs b/pkgr/src/config/mod.rs index 29f4936..4bb3397 100644 --- a/pkgr/src/config/mod.rs +++ b/pkgr/src/config/mod.rs @@ -6,6 +6,14 @@ pub struct Config { pub build_by_default: bool, } +impl Default for Config { + fn default() -> Config { + Config { + build_by_default: false, + } + } +} + impl Config { pub fn from_path(path: &str) -> Result { match std::fs::read_to_string(path) { @@ -17,11 +25,3 @@ impl Config { } } } - -impl Default for Config { - fn default() -> Config { - Config { - build_by_default: false, - } - } -} \ No newline at end of file diff --git a/pkgr/src/process.rs b/pkgr/src/process.rs index 727529f..f6f4e21 100644 --- a/pkgr/src/process.rs +++ b/pkgr/src/process.rs @@ -1,6 +1,6 @@ use std::io::{BufRead, BufReader}; use std::process::Command; -use log::info; +use log::{info, trace}; pub struct Process { @@ -19,6 +19,7 @@ impl Process { } pub fn spawn(&self) -> std::io::Result { + trace!("Spawning process: {:?}", self.command); let mut child = Command::new(&self.command[0]) .args(&self.command[1..]) .current_dir(self.cwd.clone().unwrap_or(".".to_string())) @@ -32,10 +33,10 @@ impl Process { .stdout(std::process::Stdio::piped()) .stderr(std::process::Stdio::piped()) .spawn()?; - let stdout = child.stdout.take().unwrap(); let stderr = child.stderr.take().unwrap(); + trace!("Processing lines from stdout and stderr"); for line in BufReader::new(stdout).lines() { info!(target: "command:stdout", "{}", line.unwrap()); } @@ -44,6 +45,9 @@ impl Process { info!(target: "command:stderr", "{}", line.unwrap()); } + trace!("End of process"); + + trace!("Returning child"); Ok(child) } } From 9a852ee88ee775eca3d38bff875448a4474c034b Mon Sep 17 00:00:00 2001 From: Raine Date: Sat, 14 Oct 2023 22:39:44 +0200 Subject: [PATCH 074/123] feat: multiline logging --- pkgr/src/commands.rs | 10 ++--- pkgr/src/config/mod.rs | 3 ++ pkgr/src/logging.rs | 63 ++++++++++++++++++---------- pkgr/src/package/identifier.rs | 19 +++++++-- pkgr/src/package/installer/errors.rs | 4 +- pkgr/src/package/installer/mod.rs | 5 ++- pkgr/src/tmpfs.rs | 9 ++-- 7 files changed, 75 insertions(+), 38 deletions(-) diff --git a/pkgr/src/commands.rs b/pkgr/src/commands.rs index d63e2e3..4393869 100644 --- a/pkgr/src/commands.rs +++ b/pkgr/src/commands.rs @@ -107,11 +107,11 @@ impl Command { error!("Update is not yet implemented."); } Command::Debug => { - trace!("Trace message."); - debug!("Debug message."); - info!("Info message."); - warn!("Warning message."); - error!("Error message."); + trace!("Trace message.\nWith newline."); + debug!("Debug message.\nWith newline."); + info!("Info message.\nWith newline."); + warn!("Warning message.\nWith newline."); + error!("Error message.\nWith newline."); info!(""); Process::command(vec!["sh", "-c", "echo whoami: $(whoami)"]) diff --git a/pkgr/src/config/mod.rs b/pkgr/src/config/mod.rs index 4bb3397..9a932d2 100644 --- a/pkgr/src/config/mod.rs +++ b/pkgr/src/config/mod.rs @@ -4,12 +4,15 @@ use serde::{Deserialize, Serialize}; pub struct Config { #[serde(default)] pub build_by_default: bool, + #[serde(default)] + pub tmp_dir: Option, } impl Default for Config { fn default() -> Config { Config { build_by_default: false, + tmp_dir: Some(String::from("/tmp/pkgr")) } } } diff --git a/pkgr/src/logging.rs b/pkgr/src/logging.rs index 225ef27..5e32d9d 100644 --- a/pkgr/src/logging.rs +++ b/pkgr/src/logging.rs @@ -1,40 +1,57 @@ use colored::Colorize; use fern::Dispatch; -use log::SetLoggerError; +use log::{Record, SetLoggerError}; use std::env; +fn format_regular>(log: S, record: &Record) -> String { + let log = log.into(); + let line_prefix = |line: String, extend: bool| { + let prefix = if extend { + match record.level() { + log::Level::Trace => " :".bright_blue(), + log::Level::Debug => " :".green(), + log::Level::Info => " :".blue(), + log::Level::Warn => " :".yellow(), + log::Level::Error => " :".red(), + }.to_string() + } else { + match record.level() { + log::Level::Trace => "[TRACE]".bright_blue(), + log::Level::Debug => "??".green(), + log::Level::Info => "=>".blue(), + log::Level::Warn => "##".yellow(), + log::Level::Error => "!!".red() + }.to_string() + }; + return format!("{} {}", prefix, line); + }; + + let mut lines = log.lines().peekable(); + let mut output = match lines.peek() { + Some(line) => line_prefix(lines.next().unwrap().to_string(), false), + None => return "".to_string(), + }; + + for line in lines { + output.push_str(&*format!("\n{}", line_prefix(line.to_string(), true))); + } + + output +} + pub fn setup_logger() -> Result<(), SetLoggerError> { Dispatch::new() .format(|out, message, record| { match record.metadata().target() { "command:stdout" => { - out.finish(format_args!("{} {}", "::".cyan(), message.to_string())); + out.finish(format_args!("{} {}", ">>".cyan(), message.to_string())); return; } "command:stderr" => { - out.finish(format_args!("{} {}", "!!".red(), message.to_string())); + out.finish(format_args!("{} {}", ">>".red(), message.to_string())); return; } - _ => { - out.finish(format_args!( - "{} {}", - // Some logic so messages look nice - if message.to_string().len() > 0 { - match record.level().to_string().chars().nth(0).unwrap_or('T') { - 'T' => "[TRACE]".bright_blue(), - 'D' => "??".green(), - 'I' => "=>".blue(), - 'W' => "##".yellow(), - 'E' => "!!".red(), - _ => "**".blue(), - } - .to_string() - } else { - "".to_string() - }, - message.to_string().bright_white() - )) - } + _ => out.finish(format_args!("{}", format_regular(message.to_string(), record))), } }) .level( diff --git a/pkgr/src/package/identifier.rs b/pkgr/src/package/identifier.rs index 427b775..3fde5e7 100644 --- a/pkgr/src/package/identifier.rs +++ b/pkgr/src/package/identifier.rs @@ -7,15 +7,15 @@ use regex::Regex; pub enum PackageIdentifierError { InvalidPackageLocator(String), InvalidURI(String), + InvalidPath(String), } impl std::fmt::Display for PackageIdentifierError { fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { match self { - PackageIdentifierError::InvalidPackageLocator(s) => { - write!(f, "Invalid package locator: {}", s) - } + PackageIdentifierError::InvalidPackageLocator(s) => write!(f, "Invalid package locator: {}", s), PackageIdentifierError::InvalidURI(s) => write!(f, "Invalid URI: {}", s), + PackageIdentifierError::InvalidPath(s) => write!(f, "Invalid path: {}", s), } } } @@ -71,7 +71,18 @@ impl FromStr for PackageIdentifier { return Err(PackageIdentifierError::InvalidURI(s.to_string())); } Ok(PackageIdentifier::URI(s.to_string())) - } else if std::path::Path::new(s).exists() { + } else if s.starts_with("/") + || s.starts_with("./") + || s.starts_with("../") + || s.starts_with("~/") + { + let path = std::path::Path::new(s); + if s.ends_with("/") + || !path.exists() + || path.is_dir() + { + return Err(PackageIdentifierError::InvalidPath(s.to_string())); + } return Ok(PackageIdentifier::Path(s.to_string())); } else { let pl = match PackageLocator::from_str(s) { diff --git a/pkgr/src/package/installer/errors.rs b/pkgr/src/package/installer/errors.rs index 637e891..9e86f1b 100644 --- a/pkgr/src/package/installer/errors.rs +++ b/pkgr/src/package/installer/errors.rs @@ -37,8 +37,8 @@ pub enum InstallError { impl ToString for InstallError { fn to_string(&self) -> String { match self { - InstallError::BuildError(e) => format!("Build error: {}", e), - InstallError::BinError(e) => format!("Bin error: {}", e), + InstallError::BuildError(e) => format!("Build error: \n{}", e), + InstallError::BinError(e) => format!("Bin error: \n{}", e), InstallError::InvalidManifest => "Invalid manifest".to_string(), InstallError::Generic => "Install error".to_string(), } diff --git a/pkgr/src/package/installer/mod.rs b/pkgr/src/package/installer/mod.rs index 8181ea8..5da81f9 100644 --- a/pkgr/src/package/installer/mod.rs +++ b/pkgr/src/package/installer/mod.rs @@ -38,7 +38,10 @@ impl PackageInstaller { return Err(BinError::UnpackError("package has no data".to_string())); } if std::path::Path::new(&path).exists() { - return Err(BinError::UnpackError("path already exists".to_string())); + return Err(BinError::UnpackError(format!( + "path already exists: {}", + path + ))); } tar::Archive::new(self.pkgfile.data.as_slice()) .unpack(&path) diff --git a/pkgr/src/tmpfs.rs b/pkgr/src/tmpfs.rs index c9ea6f8..d21aff7 100644 --- a/pkgr/src/tmpfs.rs +++ b/pkgr/src/tmpfs.rs @@ -1,4 +1,5 @@ use std::path::PathBuf; +use crate::CONFIG; pub struct TempDir { path: PathBuf, @@ -20,9 +21,11 @@ impl TempDir { impl Default for TempDir { fn default() -> TempDir { TempDir::new({ - let mut t = std::env::temp_dir(); - t.push("pkgr"); - t + if let Some(d) = CONFIG.with(|c| c.tmp_dir.clone()) { + PathBuf::from(d) + } else { + PathBuf::from("/tmp/pkgr") + } }) } } From 895909544d2c6fdc7a9ec3cf2787dfa19dd8520d Mon Sep 17 00:00:00 2001 From: Raine Date: Sat, 14 Oct 2023 22:39:45 +0200 Subject: [PATCH 075/123] ci: woodpecker ci --- .github/workflows/build.yaml | 59 ------------------------------------ .woodpecker.yml | 24 +++++++++++++++ 2 files changed, 24 insertions(+), 59 deletions(-) delete mode 100644 .github/workflows/build.yaml create mode 100644 .woodpecker.yml diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml deleted file mode 100644 index d9bf0bc..0000000 --- a/.github/workflows/build.yaml +++ /dev/null @@ -1,59 +0,0 @@ -on: - push: - paths: - - 'pkgr/**' - - 'manifest/**' - - 'bootpkg/**' - - 'pkgfile/**' - -jobs: - check: # check if the code compiles - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - name: install dependencies - run: | - apt update -y - apt install -y curl gcc g++ make - - uses: https://github.com/actions-rs/toolchain@v1 - with: - toolchain: stable - override: true - - uses: https://github.com/actions-rs/cargo@v1 - with: - command: check - args: --manifest-path bootpkg/Cargo.toml --verbose --all - - uses: https://github.com/actions-rs/cargo@v1 - with: - command: check - args: --manifest-path pkgr/Cargo.toml --verbose --all -# build: # create artifacts and upload them -# needs: check -# runs-on: ubuntu-latest -# steps: -# - uses: actions/checkout@v2 -# - name: install dependencies -# run: | -# apt update -y -# apt install -y curl gcc g++ make -# - uses: https://github.com/actions-rs/toolchain@v1 -# with: -# toolchain: stable -# override: true -# - uses: https://github.com/actions-rs/cargo@v1 -# with: -# command: build -# args: --release --manifest-path bootpkg/Cargo.toml --verbose --all -# continue-on-error: true -# - uses: https://github.com/actions-rs/cargo@v1 -# with: -# command: build -# args: --release --manifest-path pkgr/Cargo.toml --verbose --all -# - uses: actions/upload-artifact@v2 -# with: -# name: bootpkg -# path: bootpkg/target/release/bootpkg -# - uses: actions/upload-artifact@v2 -# with: -# name: pkgr -# path: pkgr/target/release/pkgr diff --git a/.woodpecker.yml b/.woodpecker.yml new file mode 100644 index 0000000..ee81047 --- /dev/null +++ b/.woodpecker.yml @@ -0,0 +1,24 @@ +when: + event: push + +steps: + check: # check if the code compiles + image: rust + commands: + - cargo check --manifest-path bootpkg/Cargo.toml --verbose --all + - cargo check --manifest-path pkgr/Cargo.toml --verbose --all + build: + image: rust + commands: + - cargo build --release --manifest-path bootpkg/Cargo.toml --verbose --all + - cargo build --release --manifest-path pkgr/Cargo.toml --verbose --all + publish: + image: woodpeckerci/plugin-gitea-release + settings: + base_url: https://git.ixvd.net + files: + - "bootpkg/target/release/bootpkg" + - "pkgr/target/release/pkgr" + api_key: + from_secret: GITEA_ACCESS_TOKEN + target: main \ No newline at end of file From 363659779c7911090e5cf9e39e097e30908fc380 Mon Sep 17 00:00:00 2001 From: Raine Date: Sat, 14 Oct 2023 22:39:45 +0200 Subject: [PATCH 076/123] ci: change conditions --- .woodpecker.yml | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/.woodpecker.yml b/.woodpecker.yml index ee81047..5fb6212 100644 --- a/.woodpecker.yml +++ b/.woodpecker.yml @@ -1,5 +1,12 @@ when: event: push + path: + include: + - "pkgr/**" + - "bootpkg/**" + - "pkgfile/**" + - "pkg/**" + - "manifest/**" steps: check: # check if the code compiles @@ -21,4 +28,6 @@ steps: - "pkgr/target/release/pkgr" api_key: from_secret: GITEA_ACCESS_TOKEN - target: main \ No newline at end of file + target: main + when: + event: tag \ No newline at end of file From b241856f7829dcab7cc5a37975dc698a7ca36225 Mon Sep 17 00:00:00 2001 From: Raine Date: Sat, 14 Oct 2023 22:39:45 +0200 Subject: [PATCH 077/123] ci: allow manual run --- .woodpecker.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.woodpecker.yml b/.woodpecker.yml index 5fb6212..c706c2d 100644 --- a/.woodpecker.yml +++ b/.woodpecker.yml @@ -1,5 +1,7 @@ when: - event: push + event: + - push + - manual path: include: - "pkgr/**" From 1c1116eb29672e7ecfea8f7df48e4065d046e1bb Mon Sep 17 00:00:00 2001 From: Raine Date: Sat, 14 Oct 2023 22:39:45 +0200 Subject: [PATCH 078/123] ci: always release and less verbose --- .woodpecker.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.woodpecker.yml b/.woodpecker.yml index c706c2d..5a98af8 100644 --- a/.woodpecker.yml +++ b/.woodpecker.yml @@ -14,13 +14,13 @@ steps: check: # check if the code compiles image: rust commands: - - cargo check --manifest-path bootpkg/Cargo.toml --verbose --all - - cargo check --manifest-path pkgr/Cargo.toml --verbose --all + - cargo check --release --manifest-path bootpkg/Cargo.toml --all + - cargo check --release --manifest-path pkgr/Cargo.toml --all build: image: rust commands: - - cargo build --release --manifest-path bootpkg/Cargo.toml --verbose --all - - cargo build --release --manifest-path pkgr/Cargo.toml --verbose --all + - cargo build --release --manifest-path bootpkg/Cargo.toml --all + - cargo build --release --manifest-path pkgr/Cargo.toml --all publish: image: woodpeckerci/plugin-gitea-release settings: From f8290e8160c57377c630176c4274619f38a7e446 Mon Sep 17 00:00:00 2001 From: Raine Date: Sat, 14 Oct 2023 22:39:45 +0200 Subject: [PATCH 079/123] =?UTF-8?q?ci:=20=E3=81=9Fg=20condition?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .woodpecker.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.woodpecker.yml b/.woodpecker.yml index 5a98af8..f4640ea 100644 --- a/.woodpecker.yml +++ b/.woodpecker.yml @@ -2,6 +2,7 @@ when: event: - push - manual + - tag path: include: - "pkgr/**" From 1894ca91d760f5bebe2fa23f16fb820b13c9cb23 Mon Sep 17 00:00:00 2001 From: Raine Date: Sat, 14 Oct 2023 22:39:45 +0200 Subject: [PATCH 080/123] ci: token name fix --- .woodpecker.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.woodpecker.yml b/.woodpecker.yml index f4640ea..620c2bd 100644 --- a/.woodpecker.yml +++ b/.woodpecker.yml @@ -30,7 +30,7 @@ steps: - "bootpkg/target/release/bootpkg" - "pkgr/target/release/pkgr" api_key: - from_secret: GITEA_ACCESS_TOKEN + from_secret: gitea_access_token target: main when: event: tag \ No newline at end of file From dc20ed95483180920f852bab1bf6940da71d738e Mon Sep 17 00:00:00 2001 From: Raine Date: Sat, 14 Oct 2023 22:39:46 +0200 Subject: [PATCH 081/123] ci: change steps --- .woodpecker.yml | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/.woodpecker.yml b/.woodpecker.yml index 620c2bd..8612f8e 100644 --- a/.woodpecker.yml +++ b/.woodpecker.yml @@ -12,16 +12,20 @@ when: - "manifest/**" steps: - check: # check if the code compiles + clippy: # check if the code compiles image: rust commands: - - cargo check --release --manifest-path bootpkg/Cargo.toml --all - - cargo check --release --manifest-path pkgr/Cargo.toml --all + - cargo clippy --release --no-deps --manifest-path bootpkg/Cargo.toml + - cargo clippy --release --no-deps --manifest-path pkgr/Cargo.toml + pull: true + failure: ignore build: image: rust commands: - cargo build --release --manifest-path bootpkg/Cargo.toml --all - cargo build --release --manifest-path pkgr/Cargo.toml --all + pull: true + failure: fail publish: image: woodpeckerci/plugin-gitea-release settings: @@ -32,5 +36,6 @@ steps: api_key: from_secret: gitea_access_token target: main + failure: ignore when: event: tag \ No newline at end of file From f4f83f9fa7b8cbc77669f6f7633eceb6175e1a1f Mon Sep 17 00:00:00 2001 From: Raine Date: Sat, 14 Oct 2023 22:39:46 +0200 Subject: [PATCH 082/123] ci: change condition --- .woodpecker.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.woodpecker.yml b/.woodpecker.yml index 8612f8e..7e9f161 100644 --- a/.woodpecker.yml +++ b/.woodpecker.yml @@ -8,7 +8,6 @@ when: - "pkgr/**" - "bootpkg/**" - "pkgfile/**" - - "pkg/**" - "manifest/**" steps: From 4626533269aa961bf08d0b3f9bd2e7439311ad9b Mon Sep 17 00:00:00 2001 From: Raine Date: Sat, 14 Oct 2023 22:39:46 +0200 Subject: [PATCH 083/123] fix: cargo fix --- pkgr/src/commands.rs | 10 +++++----- pkgr/src/logging.rs | 2 +- pkgr/src/main.rs | 6 +++--- pkgr/src/package/fetch.rs | 6 +++--- pkgr/src/package/installer/mod.rs | 2 +- 5 files changed, 13 insertions(+), 13 deletions(-) diff --git a/pkgr/src/commands.rs b/pkgr/src/commands.rs index 4393869..a1904a7 100644 --- a/pkgr/src/commands.rs +++ b/pkgr/src/commands.rs @@ -1,9 +1,9 @@ use crate::package::identifier::PackageIdentifier; -use crate::package::installer::{InstallType, PackageInstaller}; + use crate::package::Package; use clap::{Parser, Subcommand}; use log::{debug, error, info, trace, warn}; -use manifest::Manifest; + use std::process::exit; use crate::CONFIG; use crate::process::Process; @@ -54,7 +54,7 @@ impl Command { warn!("Installer does not run in isolation."); let start = std::time::Instant::now(); - let unix_start = std::time::SystemTime::now() + let _unix_start = std::time::SystemTime::now() .duration_since(std::time::UNIX_EPOCH) .unwrap(); @@ -80,7 +80,7 @@ impl Command { } let end = std::time::Instant::now(); - let unix_end = std::time::SystemTime::now() + let _unix_end = std::time::SystemTime::now() .duration_since(std::time::UNIX_EPOCH) .unwrap(); @@ -117,7 +117,7 @@ impl Command { Process::command(vec!["sh", "-c", "echo whoami: $(whoami)"]) .spawn() .unwrap(); - if let Ok(user) = std::env::var("SUDO_USER") { + if let Ok(_user) = std::env::var("SUDO_USER") { Process::command(vec!["sh", "-c", "echo sudo_user: $SUDO_USER"]) .spawn() .unwrap(); diff --git a/pkgr/src/logging.rs b/pkgr/src/logging.rs index 5e32d9d..47a66bc 100644 --- a/pkgr/src/logging.rs +++ b/pkgr/src/logging.rs @@ -28,7 +28,7 @@ fn format_regular>(log: S, record: &Record) -> String { let mut lines = log.lines().peekable(); let mut output = match lines.peek() { - Some(line) => line_prefix(lines.next().unwrap().to_string(), false), + Some(_line) => line_prefix(lines.next().unwrap().to_string(), false), None => return "".to_string(), }; diff --git a/pkgr/src/main.rs b/pkgr/src/main.rs index 97f721f..0191923 100644 --- a/pkgr/src/main.rs +++ b/pkgr/src/main.rs @@ -1,9 +1,9 @@ use crate::commands::Cli; -use crate::process::Process; + use clap::Parser; -use colored::Colorize; + use log::trace; -use std::process::Command; + mod commands; mod logging; diff --git a/pkgr/src/package/fetch.rs b/pkgr/src/package/fetch.rs index a97dfb6..bc385ea 100644 --- a/pkgr/src/package/fetch.rs +++ b/pkgr/src/package/fetch.rs @@ -34,7 +34,7 @@ pub fn fetch_by_uri>(uri: S) -> Result { // get file contents as bytes let mut bytes = Vec::new(); match get(uri.into()) { - Ok(mut response) => { + Ok(response) => { match response.take(1024 * 1024).read_to_end(&mut bytes) { Ok(_) => (), Err(e) => return Err(FetchError::IOError(e)), @@ -45,11 +45,11 @@ pub fn fetch_by_uri>(uri: S) -> Result { // parse bytes as PKGFile match PKGFile::try_from(bytes) { Ok(pkgfile) => Ok(pkgfile), - Err(e) => Err(FetchError::ParseError), + Err(_e) => Err(FetchError::ParseError), } } -pub fn fetch_by_package_locator(package_locator: PackageLocator) -> Result { +pub fn fetch_by_package_locator(_package_locator: PackageLocator) -> Result { // TODO: search index for package locator Ok(PKGFile::default()) } diff --git a/pkgr/src/package/installer/mod.rs b/pkgr/src/package/installer/mod.rs index 5da81f9..a28f849 100644 --- a/pkgr/src/package/installer/mod.rs +++ b/pkgr/src/package/installer/mod.rs @@ -3,7 +3,7 @@ use errors::{BinError, BuildError, InstallError}; use log::{debug, error, info, trace}; use manifest::Manifest; use pkgfile::PKGFile; -use std::fmt::Display; + use std::process::exit; use crate::CONFIG; use crate::package::identifier::{PackageIdentifier, PackageLocator}; From 6098dce4aae68a1f63d1398f097459460a61eb6d Mon Sep 17 00:00:00 2001 From: Raine Date: Sat, 14 Oct 2023 22:39:46 +0200 Subject: [PATCH 084/123] ci: add rustup clippy --- .woodpecker.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.woodpecker.yml b/.woodpecker.yml index 7e9f161..068cb06 100644 --- a/.woodpecker.yml +++ b/.woodpecker.yml @@ -14,6 +14,7 @@ steps: clippy: # check if the code compiles image: rust commands: + - rustup component add clippy - cargo clippy --release --no-deps --manifest-path bootpkg/Cargo.toml - cargo clippy --release --no-deps --manifest-path pkgr/Cargo.toml pull: true From 4e64d3a73c8fdbe3e6082332f25fe1a75ab49b10 Mon Sep 17 00:00:00 2001 From: Raine Date: Sat, 14 Oct 2023 22:39:46 +0200 Subject: [PATCH 085/123] refactor: changed structure. --- pkgfile/src/lib.rs | 45 ++++++++++++++--- pkgr/src/commands.rs | 3 +- pkgr/src/main.rs | 2 +- pkgr/src/package/fetch.rs | 55 -------------------- pkgr/src/package/installer/mod.rs | 3 +- pkgr/src/package/mod.rs | 83 +++++++++++++++++++++---------- pkgr/src/types/fetch.rs | 10 ++++ pkgr/src/types/mod.rs | 1 + 8 files changed, 113 insertions(+), 89 deletions(-) delete mode 100644 pkgr/src/package/fetch.rs create mode 100644 pkgr/src/types/fetch.rs create mode 100644 pkgr/src/types/mod.rs diff --git a/pkgfile/src/lib.rs b/pkgfile/src/lib.rs index 407055c..5db8e6f 100644 --- a/pkgfile/src/lib.rs +++ b/pkgfile/src/lib.rs @@ -1,3 +1,8 @@ +use std::error::Error; +use std::fmt::{Display, Formatter}; +use std::io; +use std::path::Path; + #[derive(Debug, Clone)] pub struct PKGFile { pub manifest: String, @@ -23,28 +28,56 @@ impl Default for PKGFile { } } -impl TryFrom> for PKGFile { - type Error = (); +#[derive(Debug)] +pub enum PKGFileError { + IOError(io::Error), + ParsingError(String) +} - fn try_from(value: Vec) -> Result { +impl Display for PKGFileError { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + match self { + PKGFileError::IOError(e) => write!(f, "IOError: {}", e), + PKGFileError::ParsingError(s) => write!(f, "ParsingError: {}", s) + } + } +} + +impl Error for PKGFileError {} + +impl<'a> TryFrom<&'a Path> for PKGFile { + type Error = PKGFileError; + fn try_from(path: &'a Path) -> Result { + let d = match std::fs::read(path) { + Ok(d) => d, + Err(e) => return Err(PKGFileError::IOError(e)) + }; + PKGFile::try_from(d) + } +} + +impl TryFrom> for PKGFile { + type Error = PKGFileError; + + fn try_from(value: Vec) -> Result { match value[0] { 1 => { let header: Vec = value[..3].iter().map(|v| u32::from(*v)).collect(); let manifest_size: u32 = (header[1] << 8) | header[2]; if manifest_size > value.len() as u32 { - return Err(()); + return Err(PKGFileError::ParsingError("Invalid header length".into())); } Ok(PKGFile { manifest: match String::from_utf8( value[3..(manifest_size as usize + 3)].to_vec(), ) { Ok(s) => s, - _ => return Err(()), + _ => return Err(PKGFileError::ParsingError("Could not parse manifest".into())), }, data: value[(manifest_size as usize + 3)..].to_vec(), }) } - _ => Err(()), + _ => Err(PKGFileError::ParsingError("Unknown pkgfile version".into())), } } } diff --git a/pkgr/src/commands.rs b/pkgr/src/commands.rs index a1904a7..01a1619 100644 --- a/pkgr/src/commands.rs +++ b/pkgr/src/commands.rs @@ -7,6 +7,7 @@ use log::{debug, error, info, trace, warn}; use std::process::exit; use crate::CONFIG; use crate::process::Process; +use crate::types::fetch::TryFetch; #[derive(Parser, Debug)] #[clap(name = "pkgr", version)] @@ -60,7 +61,7 @@ impl Command { info!("Parsing package..."); trace!("Fetching package: {}", package_identifier); - let mut pkg = Package::fetch(package_identifier.clone()).unwrap(); + let mut pkg = Package::try_fetch(package_identifier.clone()).unwrap(); debug!("manifest size: {}kb", pkg.pkgfile.manifest.len() / 1024); debug!("files size: {}kb", pkg.pkgfile.data.len() / 1024); diff --git a/pkgr/src/main.rs b/pkgr/src/main.rs index 0191923..7ce334c 100644 --- a/pkgr/src/main.rs +++ b/pkgr/src/main.rs @@ -4,13 +4,13 @@ use clap::Parser; use log::trace; - mod commands; mod logging; mod package; mod process; mod tmpfs; mod config; +mod types; thread_local! { static CONFIG: config::Config = config::Config::from_path("/etc/pkgr.toml") diff --git a/pkgr/src/package/fetch.rs b/pkgr/src/package/fetch.rs deleted file mode 100644 index bc385ea..0000000 --- a/pkgr/src/package/fetch.rs +++ /dev/null @@ -1,55 +0,0 @@ -use crate::package::identifier::PackageLocator; -use pkgfile::PKGFile; -use reqwest::blocking::get; -use std::error::Error; -use std::fmt::Display; -use std::io::Read; - -#[derive(Debug)] -pub enum FetchError { - HTTPError(reqwest::Error), - IOError(std::io::Error), - ParseError, -} - -impl Display for FetchError { - fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { - match self { - FetchError::HTTPError(e) => write!(f, "HTTP Error: {}", e), - FetchError::IOError(e) => write!(f, "IO Error: {}", e), - FetchError::ParseError => write!(f, "Parse Error"), - } - } -} - -impl Error for FetchError {} - -pub fn fetch_by_path>(path: S) -> Result { - std::fs::read(path.into()) - .map_err(|e| FetchError::IOError(e)) - .and_then(|bytes| PKGFile::try_from(bytes).map_err(|_| FetchError::ParseError)) -} - -pub fn fetch_by_uri>(uri: S) -> Result { - // get file contents as bytes - let mut bytes = Vec::new(); - match get(uri.into()) { - Ok(response) => { - match response.take(1024 * 1024).read_to_end(&mut bytes) { - Ok(_) => (), - Err(e) => return Err(FetchError::IOError(e)), - }; - } - Err(e) => return Err(FetchError::HTTPError(e)), - }; - // parse bytes as PKGFile - match PKGFile::try_from(bytes) { - Ok(pkgfile) => Ok(pkgfile), - Err(_e) => Err(FetchError::ParseError), - } -} - -pub fn fetch_by_package_locator(_package_locator: PackageLocator) -> Result { - // TODO: search index for package locator - Ok(PKGFile::default()) -} diff --git a/pkgr/src/package/installer/mod.rs b/pkgr/src/package/installer/mod.rs index a28f849..3300ddf 100644 --- a/pkgr/src/package/installer/mod.rs +++ b/pkgr/src/package/installer/mod.rs @@ -8,6 +8,7 @@ use std::process::exit; use crate::CONFIG; use crate::package::identifier::{PackageIdentifier, PackageLocator}; use crate::package::Package; +use crate::types::fetch::TryFetch; pub mod errors; @@ -67,7 +68,7 @@ impl PackageInstaller { let build_manifest = self.manifest.build.clone().unwrap(); // TODO: Check dependencies for pkg_tuple in build_manifest.dependencies { - let mut pkg = Package::fetch( + let mut pkg = Package::try_fetch( PackageIdentifier::PackageLocator( PackageLocator::from(pkg_tuple) ) diff --git a/pkgr/src/package/mod.rs b/pkgr/src/package/mod.rs index 79de18d..9ade687 100644 --- a/pkgr/src/package/mod.rs +++ b/pkgr/src/package/mod.rs @@ -1,6 +1,11 @@ +use std::fmt::Display; +use std::path::Path; use log::{info, trace}; +use reqwest::blocking::get; +use pkgfile::PKGFile; +use crate::package::identifier::PackageIdentifier; +use crate::types::fetch::TryFetch; -pub mod fetch; pub mod identifier; pub mod installer; @@ -13,7 +18,7 @@ pub struct Package { impl Package { /// Create a new package from a package identifier and a package file. - pub fn new(identifier: identifier::PackageIdentifier, pkgfile: pkgfile::PKGFile) -> Package { + pub fn new(identifier: PackageIdentifier, pkgfile: PKGFile) -> Package { Package { identifier, pkgfile, @@ -22,29 +27,6 @@ impl Package { } } - /// Fetch a package from a package identifier. - pub fn fetch( - package_identifier: identifier::PackageIdentifier, - ) -> Result { - match &package_identifier { - identifier::PackageIdentifier::Path(path) => { - trace!("fetching package from path: {}", path); - let pkgfile = fetch::fetch_by_path(path).unwrap(); - Ok(Package::new(package_identifier, pkgfile)) - } - identifier::PackageIdentifier::URI(url) => { - trace!("fetching package from uri: {}", url); - let pkgfile = fetch::fetch_by_uri(url).unwrap(); - Ok(Package::new(package_identifier, pkgfile)) - } - identifier::PackageIdentifier::PackageLocator(locator) => { - trace!("fetching package from locator: {}", locator); - let pkgfile = fetch::fetch_by_package_locator(locator.clone()).unwrap(); - Ok(Package::new(package_identifier, pkgfile)) - } - } - } - /// Get the package manifest. pub fn manifest(&self) -> manifest::Manifest { manifest::Manifest::try_from(self.pkgfile.manifest.clone()).unwrap() @@ -96,3 +78,54 @@ impl Package { unimplemented!(); } } + +#[derive(Debug)] +pub enum FetchError { + HTTPError(reqwest::Error), + IOError(std::io::Error), + ParseError, +} + +impl Display for FetchError { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + match self { + FetchError::HTTPError(e) => write!(f, "HTTP Error: {}", e), + FetchError::IOError(e) => write!(f, "IO Error: {}", e), + FetchError::ParseError => write!(f, "Parse Error"), + } + } +} + +impl TryFetch for Package { + type Error = FetchError; + + /// Fetch a package from a package identifier. + fn try_fetch(query: PackageIdentifier) -> Result { + let pkgfile = match &query { + PackageIdentifier::Path(s) => match PKGFile::try_from(Path::new(&s)) { + Ok(p) => Ok(p), + Err(e) => Err(FetchError::ParseError) + }, + PackageIdentifier::URI(s) => { + let mut bytes = Vec::new(); + match get::(s.into()) { + Ok(response) => { + if let Ok(b) = response.bytes() { + bytes.extend(b); + } + }, + Err(e) => return Err(FetchError::HTTPError(e)), + }; + // parse bytes as PKGFile + match PKGFile::try_from(bytes) { + Ok(pkgfile) => Ok(pkgfile), + Err(_e) => Err(FetchError::ParseError), + } + } + PackageIdentifier::PackageLocator(l) => Ok(PKGFile::default()) + }; + + pkgfile + .map(|p| Package::new(query, p)) + } +} \ No newline at end of file diff --git a/pkgr/src/types/fetch.rs b/pkgr/src/types/fetch.rs new file mode 100644 index 0000000..d3b7eac --- /dev/null +++ b/pkgr/src/types/fetch.rs @@ -0,0 +1,10 @@ +/// Get a result from an external source +pub trait Fetch { + fn fetch(query: Q) -> R; +} + +/// Try to get a result from an external source +pub trait TryFetch { + type Error; + fn try_fetch(query: Q) -> Result; +} \ No newline at end of file diff --git a/pkgr/src/types/mod.rs b/pkgr/src/types/mod.rs new file mode 100644 index 0000000..c798af5 --- /dev/null +++ b/pkgr/src/types/mod.rs @@ -0,0 +1 @@ +pub mod fetch; \ No newline at end of file From 9dbfdeb82613fbcba2a242f4ae2fd78d622ac4e1 Mon Sep 17 00:00:00 2001 From: Raine Date: Sat, 14 Oct 2023 22:39:46 +0200 Subject: [PATCH 086/123] docs: add pipeline badge [SKIP CI] --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 7e24755..17326aa 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,7 @@ # Packager +[![CI Status](https://ci.ixvd.net/api/badges/2/status.svg)](https://ci.ixvd.net/repos/2) + > "A package manager and builder but like rust." > -- Gandhi (2050) From bac6f80ff7b1e3d41c2c09da71420f626efd8710 Mon Sep 17 00:00:00 2001 From: Raine Date: Sat, 14 Oct 2023 22:39:47 +0200 Subject: [PATCH 087/123] docs: spec changes --- docs/manifest.md | 5 +++++ docs/pkgfile.md | 24 ------------------------ docs/pkgfiles/current.md | 25 +++++++++++++++++++++++++ docs/pkgfiles/planned.md | 22 ++++++++++++++++++++++ 4 files changed, 52 insertions(+), 24 deletions(-) create mode 100644 docs/manifest.md delete mode 100644 docs/pkgfile.md create mode 100644 docs/pkgfiles/current.md create mode 100644 docs/pkgfiles/planned.md diff --git a/docs/manifest.md b/docs/manifest.md new file mode 100644 index 0000000..ef6febe --- /dev/null +++ b/docs/manifest.md @@ -0,0 +1,5 @@ +# Manifest + +--- + +The manifest spec is located in `/package.toml` as an example. \ No newline at end of file diff --git a/docs/pkgfile.md b/docs/pkgfile.md deleted file mode 100644 index 4b2b81a..0000000 --- a/docs/pkgfile.md +++ /dev/null @@ -1,24 +0,0 @@ -# PKGFILE -This file is essentially a tar with the manifest in the header. - -## Format -The format is as follows: -``` -[PKGFILE version (1 byte)][manifest length x 256 (1 byte)][manifest length (1 byte)] -[manifest (manifest length bytes)] -[archive] -``` - -The file is Big Endian. - -## Unpacking -To unpack a PKGFILE, you can use the following command: -```bash -pkgr unpack -``` - -## Packing -You can write your own packer, or use the following command: -```bash -pkgr pack -``` diff --git a/docs/pkgfiles/current.md b/docs/pkgfiles/current.md new file mode 100644 index 0000000..bd0eb22 --- /dev/null +++ b/docs/pkgfiles/current.md @@ -0,0 +1,25 @@ +# PKGFILE + +version 1 + +--- +PKGFiles are an all-in-one solution to install a package. + +## Overhead + +Because pkgfiles include everything, many times you can download more than what is needed. +This may seem stupid but pkgfiles are *not* meant to stay on your system. + +## Format + +The format is as follows: + +``` +[PKGFILE version (1 byte)][manifest length x 256 (1 byte)][manifest length (1 byte)] +[manifest (manifest length bytes)] +[archive] +``` + +### Endianness + +The file is Big Endian. diff --git a/docs/pkgfiles/planned.md b/docs/pkgfiles/planned.md new file mode 100644 index 0000000..398f8da --- /dev/null +++ b/docs/pkgfiles/planned.md @@ -0,0 +1,22 @@ +# Planned PKGFILE features. + +--- + +### Compression + +> Planned for next PKGFILE version. + +Currently, compression is not supported, since PKGFILE is an all-in-one solution this is very unpleasant. +When compression will be implemented, there is a big possibility that the manifest won't be compressed. +This is done to allow servers to easily parse and inspect manifests. +The compression will likely be versatile in the way that the format will be stated in the header and the pkgr +implementation can work it out by itself. +There will be a default compression format, likely gzip. + +### Compact Manifest + +> Not planned yet. + +Right now, the manifest is an utf-8 blob sandwiched between the header and the archive. +This also requires a fixed max size for the manifest which is unfavourable. +In the future, the manifest may turn into a versatile binary blob with no size limitations. \ No newline at end of file From aaf9b540b14179117cab274a4fceb7753808bd20 Mon Sep 17 00:00:00 2001 From: Raine Date: Sat, 14 Oct 2023 22:39:47 +0200 Subject: [PATCH 088/123] docs: name to proper file --- docs/manifest.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/manifest.md b/docs/manifest.md index ef6febe..d6f7f14 100644 --- a/docs/manifest.md +++ b/docs/manifest.md @@ -2,4 +2,4 @@ --- -The manifest spec is located in `/package.toml` as an example. \ No newline at end of file +The manifest spec is located in `/package.example.toml` as an example. \ No newline at end of file From e96d357e6f32f48024d4c1d87ffd5b59cf3dd2e3 Mon Sep 17 00:00:00 2001 From: Raine Date: Sat, 14 Oct 2023 22:39:47 +0200 Subject: [PATCH 089/123] fix: change logging newlines --- pkgr/src/logging.rs | 10 +++++----- pkgr/src/main.rs | 3 ++- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/pkgr/src/logging.rs b/pkgr/src/logging.rs index 47a66bc..5cc511a 100644 --- a/pkgr/src/logging.rs +++ b/pkgr/src/logging.rs @@ -8,11 +8,11 @@ fn format_regular>(log: S, record: &Record) -> String { let line_prefix = |line: String, extend: bool| { let prefix = if extend { match record.level() { - log::Level::Trace => " :".bright_blue(), - log::Level::Debug => " :".green(), - log::Level::Info => " :".blue(), - log::Level::Warn => " :".yellow(), - log::Level::Error => " :".red(), + log::Level::Trace => " ]".bright_blue(), + log::Level::Debug => " ?".green(), + log::Level::Info => " >".blue(), + log::Level::Warn => " #".yellow(), + log::Level::Error => " !".red(), }.to_string() } else { match record.level() { diff --git a/pkgr/src/main.rs b/pkgr/src/main.rs index 7ce334c..3320e71 100644 --- a/pkgr/src/main.rs +++ b/pkgr/src/main.rs @@ -18,7 +18,8 @@ thread_local! { } fn main() { - logging::setup_logger().expect("Unable to setup logger."); + logging::setup_logger() + .expect("unable to set logger."); #[cfg(not(debug_assertions))] { From 63187649d8526658222d611f137633259712cc57 Mon Sep 17 00:00:00 2001 From: Raine Date: Sat, 14 Oct 2023 22:39:47 +0200 Subject: [PATCH 090/123] feat: custom debug impl for identifier --- pkgr/src/package/identifier.rs | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/pkgr/src/package/identifier.rs b/pkgr/src/package/identifier.rs index 3fde5e7..523b74b 100644 --- a/pkgr/src/package/identifier.rs +++ b/pkgr/src/package/identifier.rs @@ -1,7 +1,9 @@ use std::error::Error; +use std::fmt::Formatter; use std::str::FromStr; use regex::Regex; +use serde::Serializer; #[derive(Debug, Clone)] pub enum PackageIdentifierError { @@ -22,13 +24,31 @@ impl std::fmt::Display for PackageIdentifierError { impl Error for PackageIdentifierError {} -#[derive(Debug, Clone)] +#[derive(Clone)] pub enum PackageIdentifier { PackageLocator(PackageLocator), URI(String), Path(String), } +impl std::fmt::Debug for PackageIdentifier { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + if f.alternate() { + match self { + PackageIdentifier::PackageLocator(pl) => write!(f, "PackageLocator({:#?})", pl), + PackageIdentifier::URI(uri) => write!(f, "URI({:?})", uri), + PackageIdentifier::Path(path) => write!(f, "Path({:?})", path), + } + } else { + match self { + PackageIdentifier::PackageLocator(pl) => write!(f, "PL: {:?}", pl), + PackageIdentifier::URI(uri) => write!(f, "URI: {:?}", uri), + PackageIdentifier::Path(path) => write!(f, "Path: {:?}", path), + } + } + } +} + impl std::fmt::Display for PackageIdentifier { fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { match self { From 4e0bb7dfa37b3684b24f1481906ae3e373e3dc1f Mon Sep 17 00:00:00 2001 From: Raine Date: Sat, 14 Oct 2023 22:39:47 +0200 Subject: [PATCH 091/123] feat: debug command only in debug mode --- pkgr/src/commands.rs | 30 ++++++++++++++++++++++++------ 1 file changed, 24 insertions(+), 6 deletions(-) diff --git a/pkgr/src/commands.rs b/pkgr/src/commands.rs index 01a1619..9440645 100644 --- a/pkgr/src/commands.rs +++ b/pkgr/src/commands.rs @@ -39,7 +39,12 @@ pub enum Command { }, /// Update packages on the system Update, - #[command(hide = true)] + /// Get info about pkgr + Info { + #[arg(short, long, default_value_t = false)] + detailed: bool + }, + #[cfg(debug_assertions)] Debug, #[command(hide = true)] None, @@ -60,7 +65,7 @@ impl Command { .unwrap(); info!("Parsing package..."); - trace!("Fetching package: {}", package_identifier); + trace!("Fetching package: {:?}", package_identifier); let mut pkg = Package::try_fetch(package_identifier.clone()).unwrap(); debug!("manifest size: {}kb", pkg.pkgfile.manifest.len() / 1024); debug!("files size: {}kb", pkg.pkgfile.data.len() / 1024); @@ -107,6 +112,23 @@ impl Command { Command::Update => { error!("Update is not yet implemented."); } + Command::Info { + detailed + } => { + info!("Welcome to pkgr!\n\ + {}\n\ + To get help please run \"{} -h\"", env!("CARGO_PKG_DESCRIPTION"), std::env::args().nth(0).unwrap()); + if *detailed { + info!(""); + info!("version: {}", env!("CARGO_PKG_VERSION")); + info!("authors: {}", env!("CARGO_PKG_AUTHORS")); + + info!(""); + info!("If you can't seem to figure something out, use debug!"); + trace!("Trace should really only be used by PKGR devs."); + } + } + #[cfg(debug_assertions)] Command::Debug => { trace!("Trace message.\nWith newline."); debug!("Debug message.\nWith newline."); @@ -124,10 +146,6 @@ impl Command { .unwrap(); } - info!(""); - info!("PKGR VERSION: {}", env!("CARGO_PKG_VERSION")); - info!("PKGR AUTHORS: {}", env!("CARGO_PKG_AUTHORS")); - info!("PKGR DESCRIPTION: {}", env!("CARGO_PKG_DESCRIPTION")); info!(""); info!( "PKGR_LOG_LEVEL: {}", From 11317787f607002c7325aeb605a9b1816036b7b9 Mon Sep 17 00:00:00 2001 From: Raine Date: Sat, 14 Oct 2023 22:39:47 +0200 Subject: [PATCH 092/123] fix: config to file and import optimize does not need to be module --- pkgr/src/{config/mod.rs => config.rs} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename pkgr/src/{config/mod.rs => config.rs} (100%) diff --git a/pkgr/src/config/mod.rs b/pkgr/src/config.rs similarity index 100% rename from pkgr/src/config/mod.rs rename to pkgr/src/config.rs From 7c2f6371843233ce5e6d3599d9ded79ed5b8db86 Mon Sep 17 00:00:00 2001 From: Raine Date: Sat, 14 Oct 2023 22:39:48 +0200 Subject: [PATCH 093/123] fix: optimize imports --- pkgr/src/commands.rs | 6 +++--- pkgr/src/logging.rs | 3 ++- pkgr/src/main.rs | 5 ++--- pkgr/src/package/installer/mod.rs | 8 +++++--- pkgr/src/package/mod.rs | 5 ++++- pkgr/src/process.rs | 2 +- pkgr/src/tmpfs.rs | 1 + 7 files changed, 18 insertions(+), 12 deletions(-) diff --git a/pkgr/src/commands.rs b/pkgr/src/commands.rs index 9440645..52e6fb4 100644 --- a/pkgr/src/commands.rs +++ b/pkgr/src/commands.rs @@ -1,11 +1,11 @@ -use crate::package::identifier::PackageIdentifier; +use std::process::exit; -use crate::package::Package; use clap::{Parser, Subcommand}; use log::{debug, error, info, trace, warn}; -use std::process::exit; use crate::CONFIG; +use crate::package::identifier::PackageIdentifier; +use crate::package::Package; use crate::process::Process; use crate::types::fetch::TryFetch; diff --git a/pkgr/src/logging.rs b/pkgr/src/logging.rs index 5cc511a..f16df2d 100644 --- a/pkgr/src/logging.rs +++ b/pkgr/src/logging.rs @@ -1,7 +1,8 @@ +use std::env; + use colored::Colorize; use fern::Dispatch; use log::{Record, SetLoggerError}; -use std::env; fn format_regular>(log: S, record: &Record) -> String { let log = log.into(); diff --git a/pkgr/src/main.rs b/pkgr/src/main.rs index 3320e71..04a8ec4 100644 --- a/pkgr/src/main.rs +++ b/pkgr/src/main.rs @@ -1,9 +1,8 @@ -use crate::commands::Cli; - use clap::Parser; - use log::trace; +use crate::commands::Cli; + mod commands; mod logging; mod package; diff --git a/pkgr/src/package/installer/mod.rs b/pkgr/src/package/installer/mod.rs index 3300ddf..52852f5 100644 --- a/pkgr/src/package/installer/mod.rs +++ b/pkgr/src/package/installer/mod.rs @@ -1,13 +1,15 @@ -use crate::tmpfs::TempDir; -use errors::{BinError, BuildError, InstallError}; +use std::process::exit; + use log::{debug, error, info, trace}; + +use errors::{BinError, BuildError, InstallError}; use manifest::Manifest; use pkgfile::PKGFile; -use std::process::exit; use crate::CONFIG; use crate::package::identifier::{PackageIdentifier, PackageLocator}; use crate::package::Package; +use crate::tmpfs::TempDir; use crate::types::fetch::TryFetch; pub mod errors; diff --git a/pkgr/src/package/mod.rs b/pkgr/src/package/mod.rs index 9ade687..a0c0fd0 100644 --- a/pkgr/src/package/mod.rs +++ b/pkgr/src/package/mod.rs @@ -1,8 +1,11 @@ use std::fmt::Display; use std::path::Path; -use log::{info, trace}; + +use log::info; use reqwest::blocking::get; + use pkgfile::PKGFile; + use crate::package::identifier::PackageIdentifier; use crate::types::fetch::TryFetch; diff --git a/pkgr/src/process.rs b/pkgr/src/process.rs index f6f4e21..fbaf567 100644 --- a/pkgr/src/process.rs +++ b/pkgr/src/process.rs @@ -1,7 +1,7 @@ use std::io::{BufRead, BufReader}; use std::process::Command; -use log::{info, trace}; +use log::{info, trace}; pub struct Process { pub command: Vec, diff --git a/pkgr/src/tmpfs.rs b/pkgr/src/tmpfs.rs index d21aff7..74c21b8 100644 --- a/pkgr/src/tmpfs.rs +++ b/pkgr/src/tmpfs.rs @@ -1,4 +1,5 @@ use std::path::PathBuf; + use crate::CONFIG; pub struct TempDir { From 3bd9051a67ad71580cd1b871946cad9aa453041c Mon Sep 17 00:00:00 2001 From: Raine Date: Sat, 14 Oct 2023 22:39:48 +0200 Subject: [PATCH 094/123] fix: be more clear in info command --- pkgr/src/commands.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pkgr/src/commands.rs b/pkgr/src/commands.rs index 52e6fb4..1df594b 100644 --- a/pkgr/src/commands.rs +++ b/pkgr/src/commands.rs @@ -3,6 +3,7 @@ use std::process::exit; use clap::{Parser, Subcommand}; use log::{debug, error, info, trace, warn}; +use colored::Colorize; use crate::CONFIG; use crate::package::identifier::PackageIdentifier; use crate::package::Package; @@ -124,8 +125,8 @@ impl Command { info!("authors: {}", env!("CARGO_PKG_AUTHORS")); info!(""); - info!("If you can't seem to figure something out, use debug!"); - trace!("Trace should really only be used by PKGR devs."); + info!("If you can't seem to figure something out, use environment variable: PKGR_LOG_LEVEL=debug"); + trace!("{}", "The trace log level should really only be used by PKGR devs.".red()); } } #[cfg(debug_assertions)] From 5c15c3e07566da6286ac0659bc6ca1f7d47d0bf1 Mon Sep 17 00:00:00 2001 From: Raine Date: Sat, 14 Oct 2023 22:39:48 +0200 Subject: [PATCH 095/123] fix: more comments --- pkgr/src/main.rs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/pkgr/src/main.rs b/pkgr/src/main.rs index 04a8ec4..41f4689 100644 --- a/pkgr/src/main.rs +++ b/pkgr/src/main.rs @@ -3,12 +3,19 @@ use log::trace; use crate::commands::Cli; +/// pkgr's commands. mod commands; +/// Logging implementations for pkgr. mod logging; +/// Package and helpers. mod package; +/// Process wrapper with logging wrapper. mod process; +/// tmpfs wrapper. mod tmpfs; +/// pkgr's optional config. mod config; +/// custom types used by pkgr mod types; thread_local! { From fc3ae1e71a540073e6815bb22caee4660fa75028 Mon Sep 17 00:00:00 2001 From: Raine Date: Sat, 14 Oct 2023 22:39:48 +0200 Subject: [PATCH 096/123] feat: better time display and tmpdir expands homedir --- pkgr/Cargo.toml | 2 ++ pkgr/src/commands.rs | 2 +- pkgr/src/tmpfs.rs | 2 ++ 3 files changed, 5 insertions(+), 1 deletion(-) diff --git a/pkgr/Cargo.toml b/pkgr/Cargo.toml index cc6e55b..d6b098f 100644 --- a/pkgr/Cargo.toml +++ b/pkgr/Cargo.toml @@ -23,3 +23,5 @@ serde = { version = "1.0.171", features = ["derive"] } libc = "0.2.80" reqwest = { version = "0.11.18", features = ["blocking"] } tar = "0.4.39" +humantime = "2.1.0" +expanduser = "1.2.2" diff --git a/pkgr/src/commands.rs b/pkgr/src/commands.rs index 1df594b..87da700 100644 --- a/pkgr/src/commands.rs +++ b/pkgr/src/commands.rs @@ -94,7 +94,7 @@ impl Command { let duration = end.duration_since(start); info!("Install complete."); - info!("Install took {}ms.", duration.as_nanos() as f64 / 1000000.0); + info!("Install took {}.", humantime::format_duration(duration)); } Command::Remove { package_identifier, diff --git a/pkgr/src/tmpfs.rs b/pkgr/src/tmpfs.rs index 74c21b8..a9b756d 100644 --- a/pkgr/src/tmpfs.rs +++ b/pkgr/src/tmpfs.rs @@ -8,6 +8,8 @@ pub struct TempDir { impl TempDir { pub fn new(path: PathBuf) -> TempDir { + let pbs: String = path.to_str().unwrap().into(); + let path = expanduser::expanduser(&pbs).unwrap(); if !path.exists() { std::fs::create_dir_all(&path).unwrap(); } From 89e62ad9c70d7ead401ff8e386583bcf73aea36f Mon Sep 17 00:00:00 2001 From: Raine Date: Sat, 14 Oct 2023 22:39:48 +0200 Subject: [PATCH 097/123] feat: prompts, dir structure, etc. --- pkgr/src/logging.rs | 14 +++++------ pkgr/src/main.rs | 2 ++ pkgr/src/package/installer/errors.rs | 4 +-- pkgr/src/package/installer/mod.rs | 37 +++++++++++++++++++++------- pkgr/src/tmpfs.rs | 6 ++++- pkgr/src/util/mod.rs | 19 ++++++++++++++ pkgr/src/util/prompts.rs | 32 ++++++++++++++++++++++++ 7 files changed, 94 insertions(+), 20 deletions(-) create mode 100644 pkgr/src/util/mod.rs create mode 100644 pkgr/src/util/prompts.rs diff --git a/pkgr/src/logging.rs b/pkgr/src/logging.rs index f16df2d..daef3eb 100644 --- a/pkgr/src/logging.rs +++ b/pkgr/src/logging.rs @@ -44,14 +44,12 @@ pub fn setup_logger() -> Result<(), SetLoggerError> { Dispatch::new() .format(|out, message, record| { match record.metadata().target() { - "command:stdout" => { - out.finish(format_args!("{} {}", ">>".cyan(), message.to_string())); - return; - } - "command:stderr" => { - out.finish(format_args!("{} {}", ">>".red(), message.to_string())); - return; - } + // command output logging + "command:stdout" => out.finish(format_args!("{} {}", ">>".cyan(), message.to_string())), + "command:stderr" => out.finish(format_args!("{} {}", ">>".red(), message.to_string())), + // this target means, it's an item and not a log. + "item" => out.finish(format_args!("{} {}", "*".blue(), message.to_string())), + // default logging _ => out.finish(format_args!("{}", format_regular(message.to_string(), record))), } }) diff --git a/pkgr/src/main.rs b/pkgr/src/main.rs index 41f4689..cdad0a0 100644 --- a/pkgr/src/main.rs +++ b/pkgr/src/main.rs @@ -17,6 +17,8 @@ mod tmpfs; mod config; /// custom types used by pkgr mod types; +/// utils +mod util; thread_local! { static CONFIG: config::Config = config::Config::from_path("/etc/pkgr.toml") diff --git a/pkgr/src/package/installer/errors.rs b/pkgr/src/package/installer/errors.rs index 9e86f1b..ad6d22f 100644 --- a/pkgr/src/package/installer/errors.rs +++ b/pkgr/src/package/installer/errors.rs @@ -37,8 +37,8 @@ pub enum InstallError { impl ToString for InstallError { fn to_string(&self) -> String { match self { - InstallError::BuildError(e) => format!("Build error: \n{}", e), - InstallError::BinError(e) => format!("Bin error: \n{}", e), + InstallError::BuildError(e) => format!("{}", e), + InstallError::BinError(e) => format!("{}", e), InstallError::InvalidManifest => "Invalid manifest".to_string(), InstallError::Generic => "Install error".to_string(), } diff --git a/pkgr/src/package/installer/mod.rs b/pkgr/src/package/installer/mod.rs index 52852f5..73d40fa 100644 --- a/pkgr/src/package/installer/mod.rs +++ b/pkgr/src/package/installer/mod.rs @@ -1,3 +1,4 @@ +use std::fs::{remove_dir, remove_dir_all}; use std::process::exit; use log::{debug, error, info, trace}; @@ -11,6 +12,8 @@ use crate::package::identifier::{PackageIdentifier, PackageLocator}; use crate::package::Package; use crate::tmpfs::TempDir; use crate::types::fetch::TryFetch; +use crate::util::{visit_dirs}; +use crate::util::prompts::prompt_bool; pub mod errors; @@ -40,11 +43,14 @@ impl PackageInstaller { if !self.pkgfile.has_data() { return Err(BinError::UnpackError("package has no data".to_string())); } - if std::path::Path::new(&path).exists() { - return Err(BinError::UnpackError(format!( - "path already exists: {}", - path - ))); + let path = std::path::Path::new(&path); + if path.exists() { + trace!("cache already exists.."); + debug!("removing cache dir..."); + match remove_dir_all(path) { + Ok(_) => debug!("removed cache directory"), + Err(e) => return Err(BinError::UnpackError(format!("unable to remove directory ({}): {}", path.to_str().unwrap_or("unknown"), e.to_string()))) + } } tar::Archive::new(self.pkgfile.data.as_slice()) .unpack(&path) @@ -55,14 +61,27 @@ impl PackageInstaller { let mut tmpdir = TempDir::default(); tmpdir.push(&self.manifest.package.name); trace!("extracting package into: {}", tmpdir.to_string()); - match self.extract_to(tmpdir.to_string()) { - Ok(_) => {} - Err(e) => return Err(e), + if let Err(e) = self.extract_to(tmpdir.to_string()) { + return Err(e) } debug!("extracted package in: {}", tmpdir.to_string()); + if prompt_bool("See all files?", false) { + if let Err(e) = visit_dirs(tmpdir.path().as_path(), &|d| info!(target: "item", "{}", d.path().to_str().unwrap())) { + error!("Could not show files: {}", e); + } + if prompt_bool("Continue?", false) { + self.apply_overlay(); + } + } else { + self.apply_overlay(); + } Ok(()) } + fn apply_overlay(&self) { + + } + fn build(&self) -> Result<(), BuildError> { if let None = self.manifest.build.clone() { return Err(BuildError::InvalidManifest); @@ -80,7 +99,7 @@ impl PackageInstaller { match pkg.install(CONFIG.with(|c| { c.build_by_default })) { - Ok(_) => { info!("Installed dependency: \"{}\"", pkg.manifest().package.name) }, + Ok(_) => { info!("Installed dependency: \"{}\"", pkg.manifest().package.name) } Err(_) => { error!("Could not install dependency: \"{}\"", pkg.identifier); exit(1); diff --git a/pkgr/src/tmpfs.rs b/pkgr/src/tmpfs.rs index a9b756d..f1d0599 100644 --- a/pkgr/src/tmpfs.rs +++ b/pkgr/src/tmpfs.rs @@ -1,4 +1,4 @@ -use std::path::PathBuf; +use std::path::{Path, PathBuf}; use crate::CONFIG; @@ -16,6 +16,10 @@ impl TempDir { TempDir { path } } + pub fn path(&self) -> PathBuf { + self.path.clone() + } + pub fn push>(&mut self, path: S) { self.path.push(path.into()); } diff --git a/pkgr/src/util/mod.rs b/pkgr/src/util/mod.rs new file mode 100644 index 0000000..dc3d868 --- /dev/null +++ b/pkgr/src/util/mod.rs @@ -0,0 +1,19 @@ +use std::fs::DirEntry; +use std::path::Path; + +pub mod prompts; + +pub fn visit_dirs(dir: &Path, cb: &dyn Fn(&DirEntry)) -> std::io::Result<()> { + if dir.is_dir() { + for entry in std::fs::read_dir(dir)? { + let entry = entry?; + let path = entry.path(); + if path.is_dir() { + visit_dirs(&path, cb)?; + } else { + cb(&entry); + } + } + } + Ok(()) +} \ No newline at end of file diff --git a/pkgr/src/util/prompts.rs b/pkgr/src/util/prompts.rs new file mode 100644 index 0000000..6c3e11a --- /dev/null +++ b/pkgr/src/util/prompts.rs @@ -0,0 +1,32 @@ +use std::io::Write; +use log::trace; + +pub fn is_noninteractive() -> bool { + if let Ok(v) = std::env::var("PKGR_NON_INTERACTIVE") { + trace!("PKGR_NON_INTERACTIVE={}", v); + match v.as_str() { + "1" => true, + "true" => true, + _ => false + } + } else { false } +} + +pub fn prompt_bool>(prompt: S, default: bool) -> bool { + if is_noninteractive() { return default; } + print!("{} [{}]: ", prompt.into(), if default { "Y/n" } else { "y/N" }); + match std::io::stdout().flush() { + Ok(_) => (), + _ => println!() + }; + + let mut input = String::new(); + std::io::stdin().read_line(&mut input).expect("Failed to read input."); + let answer = input.trim().to_lowercase(); + + if answer.is_empty() { + default + } else { + answer == "y" || answer == "yes" + } +} \ No newline at end of file From d436a49c8f8b36db0d3b6452c88538cca2a3e6e2 Mon Sep 17 00:00:00 2001 From: Raine Date: Sat, 14 Oct 2023 22:39:48 +0200 Subject: [PATCH 098/123] feat: pkg.py is now usable as production builder --- bodge-pkg.py | 31 ----------- pkg.py | 77 +++++++++++++++++++++++++++ pkg/scripts/bootstrap/dirty_install | 3 -- pkg/scripts/bootstrap/download_latest | 9 ---- pkg/scripts/check_installed | 3 -- 5 files changed, 77 insertions(+), 46 deletions(-) delete mode 100644 bodge-pkg.py create mode 100644 pkg.py delete mode 100644 pkg/scripts/bootstrap/dirty_install delete mode 100644 pkg/scripts/bootstrap/download_latest delete mode 100644 pkg/scripts/check_installed diff --git a/bodge-pkg.py b/bodge-pkg.py deleted file mode 100644 index cca3307..0000000 --- a/bodge-pkg.py +++ /dev/null @@ -1,31 +0,0 @@ -import os -import tarfile - -with open("./package.toml", mode='r') as mani: - data = mani.read() - with open("./sample.pkg", mode='wb') as pkg: - print("building header...") - pkg.write(bytes([0x01, (len(data) >> 8) & 0xFF, len(data) & 0xFF])) - print("writing manifest into pkg...") - pkg.write(data.encode("utf-8")) - with tarfile.TarFile("/tmp/pkgtar", 'w') as pkgtar: - print("tarring ./pkg...") - os.chdir("pkg") - for root, dirs, files in os.walk("."): - for file in files: - print(f"\33[2Kadd: {os.path.join(root, file)}", end="\r", flush=True) - pkgtar.add(os.path.join(root, file)) - os.chdir("..") - for root, dirs, files in os.walk("pkgr"): - for file in files: - print(f"\33[2Kadd: {os.path.join(root, file)}", end="\r", flush=True) - pkgtar.add(os.path.join(root, file)) - print("\33[2K", end="\r", flush=True) - with open("/tmp/pkgtar", 'rb') as pkgtar: - print("appending pkgtar to pkg...") - pkg.write(pkgtar.read()) - print("deleting /tmp/pkgtar...") - os.unlink("/tmp/pkgtar") - print("closing write stream") - pkg.close() - \ No newline at end of file diff --git a/pkg.py b/pkg.py new file mode 100644 index 0000000..c0a8b85 --- /dev/null +++ b/pkg.py @@ -0,0 +1,77 @@ +import os +import sys +import tarfile + + +def build_package(package_toml_path, output_path, directories_to_include): + data = read_package_toml(package_toml_path) + header = build_header(data) + pkg_path = output_path + ".pkg" + + with open(pkg_path, mode='wb') as pkg: + pkg.write(header) + write_manifest(pkg, data) + build_tarball(pkg, directories_to_include) + + return pkg_path + + +def read_package_toml(package_toml_path): + with open(package_toml_path, mode='r') as mani: + return mani.read() + + +def build_header(data): + header = bytes([0x01, (len(data) >> 8) & 0xFF, len(data) & 0xFF]) + return header + + +def write_manifest(pkg, data): + pkg.write(data.encode("utf-8")) + + +def build_tarball(pkg, directories_to_include): + with tarfile.TarFile("/tmp/pkgtar", 'w') as pkgtar: + for directory in directories_to_include: + add_files_to_tar(pkgtar, directory, True) + + append_tar_to_pkg(pkg) + cleanup_tmp_files() + + +def add_files_to_tar(pkgtar, directory, skip_target=False): + for root, dirs, files in os.walk(directory): + for file in files: + if skip_target and "target" in root: + continue + print(f"\33[2Kadd: {os.path.join(root, file)}", end="\r", flush=True) + pkgtar.add(os.path.join(root, file)) + print("\33[2K", end="\r", flush=True) + + +def append_tar_to_pkg(pkg): + with open("/tmp/pkgtar", 'rb') as pkgtar: + pkg.write(pkgtar.read()) + + +def cleanup_tmp_files(): + print("deleting /tmp/pkgtar...") + os.unlink("/tmp/pkgtar") + + +if __name__ == '__main__': + if len(sys.argv) < 3: + print("Usage: pkg.py [ ...]") + sys.exit(1) + + package_toml_path = sys.argv[1] + output_path = sys.argv[2] + directories_to_include = sys.argv[3:] if len(sys.argv) > 3 else None + + try: + pkg_path = build_package(package_toml_path, output_path, directories_to_include) + print(f"Package created: {pkg_path}") + except FileNotFoundError: + print("Error: File not found.") + except Exception as e: + print(f"Error occurred: {e}") diff --git a/pkg/scripts/bootstrap/dirty_install b/pkg/scripts/bootstrap/dirty_install deleted file mode 100644 index 41d3f5d..0000000 --- a/pkg/scripts/bootstrap/dirty_install +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/sh - -printf "" \ No newline at end of file diff --git a/pkg/scripts/bootstrap/download_latest b/pkg/scripts/bootstrap/download_latest deleted file mode 100644 index 178bd4e..0000000 --- a/pkg/scripts/bootstrap/download_latest +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/sh - -fetch_latest_version() { - printf "" -} - -download_file() { - printf "" -} \ No newline at end of file diff --git a/pkg/scripts/check_installed b/pkg/scripts/check_installed deleted file mode 100644 index 41d3f5d..0000000 --- a/pkg/scripts/check_installed +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/sh - -printf "" \ No newline at end of file From d6cfab4eb797e579b51cf5e27bce39a8e69c4715 Mon Sep 17 00:00:00 2001 From: Raine Date: Sat, 14 Oct 2023 22:39:49 +0200 Subject: [PATCH 099/123] fix: binary install is now working --- .gitignore | 1 + pkgr/src/package/installer/errors.rs | 13 +++--- pkgr/src/package/installer/mod.rs | 59 +++++++++++++++++++--------- pkgr/src/tmpfs.rs | 7 +++- pkgr/src/util/fs.rs | 40 +++++++++++++++++++ pkgr/src/util/mod.rs | 19 +-------- 6 files changed, 96 insertions(+), 43 deletions(-) create mode 100644 pkgr/src/util/fs.rs diff --git a/.gitignore b/.gitignore index be024c4..bfa4a4d 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ +dist/ **/target **/Cargo.lock *.pkg \ No newline at end of file diff --git a/pkgr/src/package/installer/errors.rs b/pkgr/src/package/installer/errors.rs index ad6d22f..332a06f 100644 --- a/pkgr/src/package/installer/errors.rs +++ b/pkgr/src/package/installer/errors.rs @@ -1,21 +1,26 @@ use std::fmt::Display; +use std::io; #[derive(Debug)] pub enum BinError { UnpackError(String), + IOError(io::Error), + Cancelled, } impl Display for BinError { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { BinError::UnpackError(e) => write!(f, "Unpack error: {}", e), + BinError::IOError(e) => write!(f, "IO error: {}", e), + BinError::Cancelled => write!(f, "Cancelled by user"), } } } #[derive(Debug)] pub enum BuildError { - InvalidManifest, + InvalidManifest(String), } impl Display for BuildError { @@ -30,8 +35,7 @@ impl Display for BuildError { pub enum InstallError { BuildError(BuildError), BinError(BinError), - InvalidManifest, - Generic, + InvalidManifest(String), } impl ToString for InstallError { @@ -39,8 +43,7 @@ impl ToString for InstallError { match self { InstallError::BuildError(e) => format!("{}", e), InstallError::BinError(e) => format!("{}", e), - InstallError::InvalidManifest => "Invalid manifest".to_string(), - InstallError::Generic => "Install error".to_string(), + InstallError::InvalidManifest(s) => format!("{}", s), } } } diff --git a/pkgr/src/package/installer/mod.rs b/pkgr/src/package/installer/mod.rs index 73d40fa..bea639d 100644 --- a/pkgr/src/package/installer/mod.rs +++ b/pkgr/src/package/installer/mod.rs @@ -1,5 +1,8 @@ -use std::fs::{remove_dir, remove_dir_all}; +use std::fs::remove_dir_all; +use std::path::Path; use std::process::exit; +use std::{io, thread}; +use libc::SIGINT; use log::{debug, error, info, trace}; @@ -12,8 +15,8 @@ use crate::package::identifier::{PackageIdentifier, PackageLocator}; use crate::package::Package; use crate::tmpfs::TempDir; use crate::types::fetch::TryFetch; -use crate::util::{visit_dirs}; -use crate::util::prompts::prompt_bool; +use crate::util::fs::{copy_recursively, visit_dirs}; +use crate::util::prompts::{is_noninteractive, prompt_bool}; pub mod errors; @@ -62,29 +65,47 @@ impl PackageInstaller { tmpdir.push(&self.manifest.package.name); trace!("extracting package into: {}", tmpdir.to_string()); if let Err(e) = self.extract_to(tmpdir.to_string()) { - return Err(e) + return Err(e); } debug!("extracted package in: {}", tmpdir.to_string()); - if prompt_bool("See all files?", false) { - if let Err(e) = visit_dirs(tmpdir.path().as_path(), &|d| info!(target: "item", "{}", d.path().to_str().unwrap())) { - error!("Could not show files: {}", e); + { + let mut tmpdir = tmpdir.clone(); + tmpdir.push(&self.manifest.bin.clone().unwrap().root); + if prompt_bool("See all pending installation files?", true) { + if let Err(e) = visit_dirs(tmpdir.path().as_path(), + &|d| { + info!(target: "item", "{}", d.path() + .to_str() + .unwrap() + .replace(&tmpdir.to_string(), "")) + }) + { + error!("Could not show files: {}", e); + } } - if prompt_bool("Continue?", false) { - self.apply_overlay(); - } - } else { - self.apply_overlay(); + } + if !prompt_bool("Continue?", is_noninteractive()) { + return Err(BinError::Cancelled); + } + match self.apply_overlay() { + Ok(_) => Ok(()), + Err(e) => Err(BinError::IOError(e)) + } + } + + fn apply_overlay(&self) -> Result<(), io::Error> { + let mut tmpdir = TempDir::default(); + tmpdir.push(&self.manifest.package.name); + tmpdir.push(&self.manifest.bin.clone().unwrap().root); + if let Err(e) = copy_recursively(&tmpdir.path(), Path::new("/")) { + return Err(e); } Ok(()) } - fn apply_overlay(&self) { - - } - fn build(&self) -> Result<(), BuildError> { if let None = self.manifest.build.clone() { - return Err(BuildError::InvalidManifest); + return Err(BuildError::InvalidManifest(String::from("No build manifest"))); } let build_manifest = self.manifest.build.clone().unwrap(); // TODO: Check dependencies @@ -117,7 +138,7 @@ impl PackageInstaller { if let None = self.manifest.bin { self.install_type = InstallType::Build; if let None = self.manifest.build { - return Err(InstallError::InvalidManifest); + return Err(InstallError::InvalidManifest(String::from("no bin or build manifest"))); } } } @@ -125,7 +146,7 @@ impl PackageInstaller { if let None = self.manifest.build { self.install_type = InstallType::Bin; if let None = self.manifest.bin { - return Err(InstallError::InvalidManifest); + return Err(InstallError::InvalidManifest(String::from("no build or bin manifest"))); } } } diff --git a/pkgr/src/tmpfs.rs b/pkgr/src/tmpfs.rs index f1d0599..e83b860 100644 --- a/pkgr/src/tmpfs.rs +++ b/pkgr/src/tmpfs.rs @@ -2,6 +2,7 @@ use std::path::{Path, PathBuf}; use crate::CONFIG; +#[derive(Clone)] pub struct TempDir { path: PathBuf, } @@ -21,7 +22,11 @@ impl TempDir { } pub fn push>(&mut self, path: S) { - self.path.push(path.into()); + let mut path_str = path.into(); + if path_str.starts_with('/') { + path_str = path_str[1..].to_string(); + } + self.path.push(path_str); } } diff --git a/pkgr/src/util/fs.rs b/pkgr/src/util/fs.rs new file mode 100644 index 0000000..03b5cb2 --- /dev/null +++ b/pkgr/src/util/fs.rs @@ -0,0 +1,40 @@ +use std::path::Path; +use std::fs::DirEntry; +use std::{fs, io}; +use log::trace; + +pub fn visit_dirs(dir: &Path, cb: &dyn Fn(&DirEntry)) -> std::io::Result<()> { + if dir.is_dir() { + for entry in std::fs::read_dir(dir)? { + let entry = entry?; + let path = entry.path(); + if path.is_dir() { + visit_dirs(&path, cb)?; + } else { + cb(&entry); + } + } + } + Ok(()) +} + +pub fn copy_recursively(source: &Path, target: &Path) -> io::Result<()> { + if source.is_file() { + trace!("source: {:?}, target: {:?}", source, target); + fs::copy(source, target)?; + } else if source.is_dir() { + if !target.exists() { + fs::create_dir(target)?; + } + + for entry in fs::read_dir(source)? { + let entry = entry?; + let file_name = entry.file_name(); + let source_path = entry.path(); + let target_path = target.join(&file_name); + + copy_recursively(&source_path, &target_path)?; + } + } + Ok(()) +} \ No newline at end of file diff --git a/pkgr/src/util/mod.rs b/pkgr/src/util/mod.rs index dc3d868..c239775 100644 --- a/pkgr/src/util/mod.rs +++ b/pkgr/src/util/mod.rs @@ -1,19 +1,2 @@ -use std::fs::DirEntry; -use std::path::Path; - pub mod prompts; - -pub fn visit_dirs(dir: &Path, cb: &dyn Fn(&DirEntry)) -> std::io::Result<()> { - if dir.is_dir() { - for entry in std::fs::read_dir(dir)? { - let entry = entry?; - let path = entry.path(); - if path.is_dir() { - visit_dirs(&path, cb)?; - } else { - cb(&entry); - } - } - } - Ok(()) -} \ No newline at end of file +pub mod fs; \ No newline at end of file From d1f7f3c7ffc96de85cc4a71e96e562d7fba229ef Mon Sep 17 00:00:00 2001 From: Raine Date: Sat, 14 Oct 2023 22:39:49 +0200 Subject: [PATCH 100/123] feat: build_pkg script --- build_pkg.sh | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 build_pkg.sh diff --git a/build_pkg.sh b/build_pkg.sh new file mode 100644 index 0000000..2830ede --- /dev/null +++ b/build_pkg.sh @@ -0,0 +1,9 @@ +#!/bin/sh + +mkdir -p dist/root/usr/bin/ dist/root/etc/pkgr +echo "\x00" > dist/root/etc/pkgr/YOU.CAN.NOT.UPDATE.PKGR.WITH.PKGR +cd pkgr +cargo build -r +cp target/release/pkgr ../dist/root/usr/bin/pkgr +cd ../dist +python ../pkg.py ../package.toml pkgr root \ No newline at end of file From 497b53eb91a7d17f842bd02f3665cae69a48e3e3 Mon Sep 17 00:00:00 2001 From: Raine Date: Sat, 14 Oct 2023 22:39:49 +0200 Subject: [PATCH 101/123] fix: bootpkg only as root --- bootpkg/src/main.rs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/bootpkg/src/main.rs b/bootpkg/src/main.rs index ffd4338..65037f3 100644 --- a/bootpkg/src/main.rs +++ b/bootpkg/src/main.rs @@ -15,6 +15,14 @@ mod args; mod prelude; fn main() { + #[cfg(not(debug_assertions))] + { + if unsafe { libc::getuid() } != 0 { + println!("bootpkg must be run as root."); + std::process::exit(1); + } + } + let args = Args::from(env::args().collect::>()[1..].to_owned()); match args.command { Command::Strap => { From b434d086a170138bb8805821c26cadbf64124e62 Mon Sep 17 00:00:00 2001 From: Raine Date: Sat, 14 Oct 2023 22:39:49 +0200 Subject: [PATCH 102/123] fix: no args = panic --- bootpkg/src/args.rs | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/bootpkg/src/args.rs b/bootpkg/src/args.rs index 0df76c9..373830b 100644 --- a/bootpkg/src/args.rs +++ b/bootpkg/src/args.rs @@ -18,9 +18,20 @@ pub struct Args { impl From> for Args { fn from(value: Vec) -> Self { + if value.len() == 0 { + return Args { + command: Command::from(String::default()), + args: vec![], + } + } + Args { command: Command::from(value[0].to_owned()), - args: value[1..].to_owned(), + args: if value.len() > 1 { + value[1..].to_owned() + } else { + vec![] + }, } } } From 163362d95437df4de30fdd9d5e449af483be5227 Mon Sep 17 00:00:00 2001 From: Raine Date: Sat, 14 Oct 2023 22:39:49 +0200 Subject: [PATCH 103/123] fix(manifest): package supports bootpkg --- package.toml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/package.toml b/package.toml index 81240ab..91ff425 100644 --- a/package.toml +++ b/package.toml @@ -58,12 +58,10 @@ base = "latest,stable" # selected by default [pkgr.bootstrap] ## any non-zero = installed check_installed_commands = [ - "sh scripts/check_installed" + "file /usr/bin/pkgr" ] # any non-zero = fail commands = [ - "sh scripts/bootstrap/download_latest @version /tmp/pkgr.pkg", - "sh scripts/bootstrap/dirty_install /tmp/pkgr.pkg", - "sh scripts/check_installed" + "cp root/usr/bin/pkgr /usr/bin/pkgr" ] From 24993f2f229ee4d9026ee7980755b0f60f7af0cc Mon Sep 17 00:00:00 2001 From: Raine Date: Sat, 14 Oct 2023 22:39:49 +0200 Subject: [PATCH 104/123] fix(dep): add libc dependency --- bootpkg/Cargo.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/bootpkg/Cargo.toml b/bootpkg/Cargo.toml index d6c3186..5530ce3 100644 --- a/bootpkg/Cargo.toml +++ b/bootpkg/Cargo.toml @@ -14,3 +14,4 @@ regex = "1.9.1" reqwest = { version = "0.11.18", features = ["blocking"] } uuid = { version = "1.4.0", features = ["serde", "v4"] } tar = "0.4.39" +libc = "0.2.80" From 671590a2fa382339779482ba8d053f3b13d687c0 Mon Sep 17 00:00:00 2001 From: Raine Date: Sat, 14 Oct 2023 22:39:50 +0200 Subject: [PATCH 105/123] feat: added default for Bin so hashmap doesn't have to be specified --- manifest/src/bin.rs | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/manifest/src/bin.rs b/manifest/src/bin.rs index f55a8e5..5142d99 100644 --- a/manifest/src/bin.rs +++ b/manifest/src/bin.rs @@ -5,5 +5,15 @@ use serde::{Deserialize, Serialize}; #[derive(Debug, Clone, Serialize, Deserialize)] pub struct Bin { pub root: String, + #[serde(default)] pub checksums: HashMap, } + +impl Default for Bin { + fn default() -> Self { + Bin { + root: String::default(), + checksums: HashMap::default(), + } + } +} From 656d10ab8c9b8e9ed420110fc4a22215a26b91fd Mon Sep 17 00:00:00 2001 From: Raine Date: Sat, 14 Oct 2023 22:39:50 +0200 Subject: [PATCH 106/123] feat: added packages for pkgr and bootpkg --- .gitignore | 2 +- bootpkg/package.toml | 10 ++++++++++ package.toml => pkgr/package.toml | 0 3 files changed, 11 insertions(+), 1 deletion(-) create mode 100644 bootpkg/package.toml rename package.toml => pkgr/package.toml (100%) diff --git a/.gitignore b/.gitignore index bfa4a4d..7c9f50c 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,4 @@ -dist/ +**/dist/ **/target **/Cargo.lock *.pkg \ No newline at end of file diff --git a/bootpkg/package.toml b/bootpkg/package.toml new file mode 100644 index 0000000..c272a08 --- /dev/null +++ b/bootpkg/package.toml @@ -0,0 +1,10 @@ +[package] +name = "bootpkg" #* +description = "A tool to strap pkgs" #* +version = 1 +tags = [] +type = "application" +arch = "x86_64" + +[bin] +root = "/root" \ No newline at end of file diff --git a/package.toml b/pkgr/package.toml similarity index 100% rename from package.toml rename to pkgr/package.toml From 8d6ed71dfc6870b52b518962dcb283b0d2f1f756 Mon Sep 17 00:00:00 2001 From: Raine Date: Sat, 14 Oct 2023 22:39:50 +0200 Subject: [PATCH 107/123] build: changed build scripts --- build | 39 +++++++++++++++++++++++++++++++++++++++ build_pkg.sh | 9 --------- pkg.py | 6 ++++-- 3 files changed, 43 insertions(+), 11 deletions(-) create mode 100755 build delete mode 100644 build_pkg.sh diff --git a/build b/build new file mode 100755 index 0000000..2f69899 --- /dev/null +++ b/build @@ -0,0 +1,39 @@ +#!/bin/bash + +build_pkgr() { + cd pkgr + mkdir -p dist/root/{usr/bin,etc/pkgr} + echo -e "You can't use pkgr to update pkgr because the file will be in use while updating.\nuse bootpkg" > dist/root/etc/pkgr/YOU-CAN-NOT-USE-PKGR-TO-UPDATE-PKGR.txt + + # for bin + cargo build -r + cp target/release/pkgr dist/root/usr/bin/pkgr + + # for build + mkdir -p dist/pkgr + cp -r src/ Cargo.toml dist/pkgr + cp -r ../manifest dist/manifest + cp -r ../pkgfile dist/pkgfile + + cd dist + python ../../pkg.py ../package.toml pkgr.pkg "*" + cd ../.. +} + +build_bootpkg() { + cd bootpkg + mkdir -p dist/root/usr/bin + + cargo build -r + cp target/release/bootpkg dist/root/usr/bin/bootpkg + + + cd dist + python ../../pkg.py ../package.toml bootpkg.pkg "*" + cd ../.. +} + +set -e + +build_bootpkg +build_pkgr diff --git a/build_pkg.sh b/build_pkg.sh deleted file mode 100644 index 2830ede..0000000 --- a/build_pkg.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/sh - -mkdir -p dist/root/usr/bin/ dist/root/etc/pkgr -echo "\x00" > dist/root/etc/pkgr/YOU.CAN.NOT.UPDATE.PKGR.WITH.PKGR -cd pkgr -cargo build -r -cp target/release/pkgr ../dist/root/usr/bin/pkgr -cd ../dist -python ../pkg.py ../package.toml pkgr root \ No newline at end of file diff --git a/pkg.py b/pkg.py index c0a8b85..03e20ab 100644 --- a/pkg.py +++ b/pkg.py @@ -1,3 +1,4 @@ +import glob import os import sys import tarfile @@ -6,7 +7,7 @@ import tarfile def build_package(package_toml_path, output_path, directories_to_include): data = read_package_toml(package_toml_path) header = build_header(data) - pkg_path = output_path + ".pkg" + pkg_path = output_path with open(pkg_path, mode='wb') as pkg: pkg.write(header) @@ -45,6 +46,7 @@ def add_files_to_tar(pkgtar, directory, skip_target=False): if skip_target and "target" in root: continue print(f"\33[2Kadd: {os.path.join(root, file)}", end="\r", flush=True) + # print() pkgtar.add(os.path.join(root, file)) print("\33[2K", end="\r", flush=True) @@ -66,7 +68,7 @@ if __name__ == '__main__': package_toml_path = sys.argv[1] output_path = sys.argv[2] - directories_to_include = sys.argv[3:] if len(sys.argv) > 3 else None + directories_to_include = glob.glob(sys.argv[3]) if len(sys.argv) == 4 else sys.argv[3:] try: pkg_path = build_package(package_toml_path, output_path, directories_to_include) From 19b834e65dd73b3c65fb57a7731b9b447c2b9d60 Mon Sep 17 00:00:00 2001 From: Raine Date: Sat, 14 Oct 2023 22:39:50 +0200 Subject: [PATCH 108/123] feat: info is now an info command and fixed logging --- manifest/src/package.rs | 10 ++++++++++ pkgr/src/commands.rs | 38 ++++++++++++++++++++++++++++++-------- pkgr/src/package/mod.rs | 7 +++++-- 3 files changed, 45 insertions(+), 10 deletions(-) diff --git a/manifest/src/package.rs b/manifest/src/package.rs index 233206d..d7c126c 100644 --- a/manifest/src/package.rs +++ b/manifest/src/package.rs @@ -12,6 +12,16 @@ pub enum PackageType { Meta, } +impl ToString for PackageType { + fn to_string(&self) -> String { + match &self { + PackageType::Application => "application", + PackageType::Library => "library", + PackageType::Meta => "meta" + }.to_string() + } +} + #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(default)] pub struct Package { diff --git a/pkgr/src/commands.rs b/pkgr/src/commands.rs index 87da700..31ae49c 100644 --- a/pkgr/src/commands.rs +++ b/pkgr/src/commands.rs @@ -4,6 +4,7 @@ use clap::{Parser, Subcommand}; use log::{debug, error, info, trace, warn}; use colored::Colorize; +use manifest::package::PackageType; use crate::CONFIG; use crate::package::identifier::PackageIdentifier; use crate::package::Package; @@ -40,10 +41,9 @@ pub enum Command { }, /// Update packages on the system Update, - /// Get info about pkgr + /// Get info about a package Info { - #[arg(short, long, default_value_t = false)] - detailed: bool + package_identifier: Option }, #[cfg(debug_assertions)] Debug, @@ -114,12 +114,34 @@ impl Command { error!("Update is not yet implemented."); } Command::Info { - detailed + package_identifier } => { - info!("Welcome to pkgr!\n\ - {}\n\ - To get help please run \"{} -h\"", env!("CARGO_PKG_DESCRIPTION"), std::env::args().nth(0).unwrap()); - if *detailed { + if let Some(p) = package_identifier { + if let Ok(mut pkg) = Package::try_fetch(p.clone()) { + info!(target: "item", "Identifier: {:?}", pkg.identifier); + info!(target: "item", ""); + let mani = pkg.manifest(); + info!(target: "item", "Package name: {}", mani.package.name); + info!(target: "item", "Package description: {}", mani.package.description); + info!(target: "item", "Package version: {}", mani.package.version); + info!(target: "item", "Package tags: {}", mani.package.tags.join(", ")); + info!(target: "item", "Package type: {}", mani.package.package_type.to_string()); + info!(target: "item", ""); + info!(target: "item", "Supported install types: {}", { + let mut types = vec![]; + if let Some(_) = mani.bin { types.push("bin") } + if let Some(_) = mani.build { types.push("build") } + if let PackageType::Meta = mani.package.package_type { types.push("meta") } + types.join(", ") + }); + info!(target: "item", ""); + } else { + error!("Could not find {p}"); + } + } else { + info!("Welcome to pkgr!\n\ + {}\n\ + To get help please run \"{} -h\"", env!("CARGO_PKG_DESCRIPTION"), std::env::args().nth(0).unwrap()); info!(""); info!("version: {}", env!("CARGO_PKG_VERSION")); info!("authors: {}", env!("CARGO_PKG_AUTHORS")); diff --git a/pkgr/src/package/mod.rs b/pkgr/src/package/mod.rs index a0c0fd0..96328f4 100644 --- a/pkgr/src/package/mod.rs +++ b/pkgr/src/package/mod.rs @@ -1,7 +1,7 @@ use std::fmt::Display; use std::path::Path; -use log::info; +use log::{debug, info, trace}; use reqwest::blocking::get; use pkgfile::PKGFile; @@ -104,6 +104,7 @@ impl TryFetch for Package { /// Fetch a package from a package identifier. fn try_fetch(query: PackageIdentifier) -> Result { + trace!("Fetching: {query:#?}"); let pkgfile = match &query { PackageIdentifier::Path(s) => match PKGFile::try_from(Path::new(&s)) { Ok(p) => Ok(p), @@ -111,8 +112,10 @@ impl TryFetch for Package { }, PackageIdentifier::URI(s) => { let mut bytes = Vec::new(); + debug!("sending GET request..."); match get::(s.into()) { Ok(response) => { + debug!("Got response!"); if let Ok(b) = response.bytes() { bytes.extend(b); } @@ -125,7 +128,7 @@ impl TryFetch for Package { Err(_e) => Err(FetchError::ParseError), } } - PackageIdentifier::PackageLocator(l) => Ok(PKGFile::default()) + PackageIdentifier::PackageLocator(l) => unimplemented!() }; pkgfile From 825114136a0af63a9c2f991001f7ccfec4db0e87 Mon Sep 17 00:00:00 2001 From: Raine Date: Sat, 14 Oct 2023 22:39:50 +0200 Subject: [PATCH 109/123] fix: removed double log --- pkgr/src/commands.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/pkgr/src/commands.rs b/pkgr/src/commands.rs index 31ae49c..8e35d49 100644 --- a/pkgr/src/commands.rs +++ b/pkgr/src/commands.rs @@ -66,7 +66,6 @@ impl Command { .unwrap(); info!("Parsing package..."); - trace!("Fetching package: {:?}", package_identifier); let mut pkg = Package::try_fetch(package_identifier.clone()).unwrap(); debug!("manifest size: {}kb", pkg.pkgfile.manifest.len() / 1024); debug!("files size: {}kb", pkg.pkgfile.data.len() / 1024); From f29f944f99ff97a1b697d3d50dc250e288bfd6d1 Mon Sep 17 00:00:00 2001 From: Raine Date: Sat, 14 Oct 2023 22:39:50 +0200 Subject: [PATCH 110/123] feat: implement packagequeue --- manifest/src/build.rs | 10 +++++ pkgfile/src/lib.rs | 2 +- pkgr/package.toml | 1 - pkgr/src/api/mod.rs | 27 +++++++++++++ pkgr/src/commands.rs | 42 +++++++++++++++++---- pkgr/src/main.rs | 1 + pkgr/src/package/identifier.rs | 15 ++++---- pkgr/src/package/installer/mod.rs | 38 ------------------- pkgr/src/package/mod.rs | 63 ++++++++++++++++++++++++++++--- pkgr/src/package/queue.rs | 37 ++++++++++++++++++ 10 files changed, 176 insertions(+), 60 deletions(-) create mode 100644 pkgr/src/api/mod.rs create mode 100644 pkgr/src/package/queue.rs diff --git a/manifest/src/build.rs b/manifest/src/build.rs index 1879710..6d406b1 100644 --- a/manifest/src/build.rs +++ b/manifest/src/build.rs @@ -8,3 +8,13 @@ pub struct Build { pub install_script: String, pub dependencies: HashMap, } + +impl Default for Build { + fn default() -> Self { + Build { + dependencies: HashMap::default(), + build_script: String::default(), + install_script: String::default() + } + } +} \ No newline at end of file diff --git a/pkgfile/src/lib.rs b/pkgfile/src/lib.rs index 5db8e6f..6a9457d 100644 --- a/pkgfile/src/lib.rs +++ b/pkgfile/src/lib.rs @@ -3,7 +3,7 @@ use std::fmt::{Display, Formatter}; use std::io; use std::path::Path; -#[derive(Debug, Clone)] +#[derive(Debug, Clone, Eq, PartialEq)] pub struct PKGFile { pub manifest: String, pub data: Vec, diff --git a/pkgr/package.toml b/pkgr/package.toml index 91ff425..723ebed 100644 --- a/pkgr/package.toml +++ b/pkgr/package.toml @@ -44,7 +44,6 @@ build_script = "scripts/build" # relative to pkg install_script = "scripts/install" # relative to pkg [build.dependencies] -base = "latest,stable" # selected by default ## pkgr.* # packager is the official client but you may use other clients supporting the "pkgr v1 spec". diff --git a/pkgr/src/api/mod.rs b/pkgr/src/api/mod.rs new file mode 100644 index 0000000..3cec172 --- /dev/null +++ b/pkgr/src/api/mod.rs @@ -0,0 +1,27 @@ +use reqwest::blocking::Response; + +pub struct APITransaction { + /// Query made to API. + query: String, //TODO: enum + /// Response struct from abstracted library + _response: Option +} + +impl APITransaction { + pub fn new(_response: Option) -> Self { + APITransaction { + query: String::default(), + _response + } + } +} + +pub struct API { + base_url: String, + token: Option, + history: Vec +} + +impl API { + +} \ No newline at end of file diff --git a/pkgr/src/commands.rs b/pkgr/src/commands.rs index 8e35d49..869e961 100644 --- a/pkgr/src/commands.rs +++ b/pkgr/src/commands.rs @@ -8,6 +8,7 @@ use manifest::package::PackageType; use crate::CONFIG; use crate::package::identifier::PackageIdentifier; use crate::package::Package; +use crate::package::queue::PackageQueue; use crate::process::Process; use crate::types::fetch::TryFetch; @@ -76,14 +77,10 @@ impl Command { exit(1); } - trace!("Starting install..."); - match pkg.install(CONFIG.with(|c| if !*build { c.build_by_default } else { *build })) { - Ok(_) => (), - Err(e) => { - error!("Install failed: {}", e.to_string()); - exit(1); - } - } + let mut queue = PackageQueue::new(); + queue.add_package(pkg, *build); + trace!("Installing queue..."); + queue.install(*build); let end = std::time::Instant::now(); let _unix_end = std::time::SystemTime::now() @@ -117,6 +114,7 @@ impl Command { } => { if let Some(p) = package_identifier { if let Ok(mut pkg) = Package::try_fetch(p.clone()) { + trace!("{}", pkg.pkgfile.manifest); info!(target: "item", "Identifier: {:?}", pkg.identifier); info!(target: "item", ""); let mani = pkg.manifest(); @@ -134,6 +132,34 @@ impl Command { types.join(", ") }); info!(target: "item", ""); + info!(target: "item", "Dependencies: {}", { + let deps = pkg.dependencies(); + if deps.len() == 0 { + String::from("None") + } else { + deps + .iter() + .map(|p| { + p.identifier.to_string() + }) + .collect::>() + .join(", ") + } + }); + info!(target: "item", "Build Dependencies: {}", { + let deps = pkg.build_dependencies(); + if deps.len() == 0 { + String::from("None") + } else { + deps + .iter() + .map(|p| { + p.identifier.to_string() + }) + .collect::>() + .join(", ") + } + }); } else { error!("Could not find {p}"); } diff --git a/pkgr/src/main.rs b/pkgr/src/main.rs index cdad0a0..798eb1d 100644 --- a/pkgr/src/main.rs +++ b/pkgr/src/main.rs @@ -19,6 +19,7 @@ mod config; mod types; /// utils mod util; +mod api; thread_local! { static CONFIG: config::Config = config::Config::from_path("/etc/pkgr.toml") diff --git a/pkgr/src/package/identifier.rs b/pkgr/src/package/identifier.rs index 523b74b..e39ac87 100644 --- a/pkgr/src/package/identifier.rs +++ b/pkgr/src/package/identifier.rs @@ -24,11 +24,11 @@ impl std::fmt::Display for PackageIdentifierError { impl Error for PackageIdentifierError {} -#[derive(Clone)] +#[derive(Clone, Eq, PartialEq)] pub enum PackageIdentifier { PackageLocator(PackageLocator), URI(String), - Path(String), + Path(String) } impl std::fmt::Debug for PackageIdentifier { @@ -59,7 +59,7 @@ impl std::fmt::Display for PackageIdentifier { } } -#[derive(Debug, Clone)] +#[derive(Debug, Clone, Eq, PartialEq)] pub struct PackageLocator { pub name: String, pub version: Option, @@ -160,8 +160,9 @@ impl FromStr for PackageLocator { } } -impl From<(String, String)> for PackageLocator { - fn from((name, locate_str): (String, String)) -> Self { +impl TryFrom<(String, String)> for PackageLocator { + type Error = PackageIdentifierError; + fn try_from((name, locate_str): (String, String)) -> Result { // name = "pkg" // locate_str = "1.0.0:tag1,tag2" or "1.0.0" or "tag1,tag2" let mut version = None; @@ -185,10 +186,10 @@ impl From<(String, String)> for PackageLocator { ); } - PackageLocator { + Ok(PackageLocator { name, version, tags, - } + }) } } diff --git a/pkgr/src/package/installer/mod.rs b/pkgr/src/package/installer/mod.rs index bea639d..da37f1b 100644 --- a/pkgr/src/package/installer/mod.rs +++ b/pkgr/src/package/installer/mod.rs @@ -68,25 +68,6 @@ impl PackageInstaller { return Err(e); } debug!("extracted package in: {}", tmpdir.to_string()); - { - let mut tmpdir = tmpdir.clone(); - tmpdir.push(&self.manifest.bin.clone().unwrap().root); - if prompt_bool("See all pending installation files?", true) { - if let Err(e) = visit_dirs(tmpdir.path().as_path(), - &|d| { - info!(target: "item", "{}", d.path() - .to_str() - .unwrap() - .replace(&tmpdir.to_string(), "")) - }) - { - error!("Could not show files: {}", e); - } - } - } - if !prompt_bool("Continue?", is_noninteractive()) { - return Err(BinError::Cancelled); - } match self.apply_overlay() { Ok(_) => Ok(()), Err(e) => Err(BinError::IOError(e)) @@ -109,25 +90,6 @@ impl PackageInstaller { } let build_manifest = self.manifest.build.clone().unwrap(); // TODO: Check dependencies - for pkg_tuple in build_manifest.dependencies { - let mut pkg = Package::try_fetch( - PackageIdentifier::PackageLocator( - PackageLocator::from(pkg_tuple) - ) - ).expect("no pkg"); - - if !pkg.is_installed() { - match pkg.install(CONFIG.with(|c| { - c.build_by_default - })) { - Ok(_) => { info!("Installed dependency: \"{}\"", pkg.manifest().package.name) } - Err(_) => { - error!("Could not install dependency: \"{}\"", pkg.identifier); - exit(1); - } - } - } - } Ok(()) } diff --git a/pkgr/src/package/mod.rs b/pkgr/src/package/mod.rs index 96328f4..6378cef 100644 --- a/pkgr/src/package/mod.rs +++ b/pkgr/src/package/mod.rs @@ -1,20 +1,24 @@ use std::fmt::Display; use std::path::Path; +use std::process::exit; -use log::{debug, info, trace}; +use log::{debug, error, info, trace}; use reqwest::blocking::get; +use manifest::build::Build; use pkgfile::PKGFile; -use crate::package::identifier::PackageIdentifier; +use crate::package::identifier::{PackageIdentifier, PackageLocator}; use crate::types::fetch::TryFetch; pub mod identifier; pub mod installer; +pub mod queue; +#[derive(Eq, PartialEq)] pub struct Package { - pub identifier: identifier::PackageIdentifier, - pub pkgfile: pkgfile::PKGFile, + pub identifier: PackageIdentifier, + pub pkgfile: PKGFile, is_installed: bool, is_indexed: bool, } @@ -35,6 +39,55 @@ impl Package { manifest::Manifest::try_from(self.pkgfile.manifest.clone()).unwrap() } + /// Get package dependencies + pub fn dependencies(&self) -> Vec { + let mut dependencies = vec![]; + for dependency in self.manifest().dependencies { + let pkglocate = if let Ok(pl) = PackageLocator::try_from(dependency.clone()) { + trace!("parsed pl successfully..."); + pl + } else { + error!("Could not parse package locator: {:?} in dependencies.", &dependency); + exit(1); + }; + let pkg = match Package::try_fetch(PackageIdentifier::PackageLocator(pkglocate)) { + Ok(p) => p, + Err(e) => { + error!("Could not fetch dependency: {}", e); + exit(1); + } + }; + dependencies.push(pkg); + } + dependencies + } + + pub fn build_dependencies(&self) -> Vec { + let mut dependencies = vec![]; + for dependency in self + .manifest() + .build + .unwrap_or(Build::default()) + .dependencies { + let pkglocate = if let Ok(pl) = PackageLocator::try_from(dependency.clone()) { + trace!("parsed pl successfully..."); + pl + } else { + error!("Could not parse package locator: {:?} in dependencies.", &dependency); + exit(1); + }; + let pkg = match Package::try_fetch(PackageIdentifier::PackageLocator(pkglocate)) { + Ok(p) => p, + Err(e) => { + error!("Could not fetch dependency: {}", e); + exit(1); + } + }; + dependencies.push(pkg); + } + dependencies + } + /// Install the package. pub fn install(&mut self, build: bool) -> Result<(), installer::errors::InstallError> { let manifest = self.manifest(); @@ -119,7 +172,7 @@ impl TryFetch for Package { if let Ok(b) = response.bytes() { bytes.extend(b); } - }, + } Err(e) => return Err(FetchError::HTTPError(e)), }; // parse bytes as PKGFile diff --git a/pkgr/src/package/queue.rs b/pkgr/src/package/queue.rs new file mode 100644 index 0000000..16a7b95 --- /dev/null +++ b/pkgr/src/package/queue.rs @@ -0,0 +1,37 @@ +use log::trace; +use crate::package::Package; + +pub struct PackageQueue { + packages: Vec +} + +impl PackageQueue { + pub fn new() -> Self { + PackageQueue { + packages: vec![] + } + } + + pub fn add_package(&mut self, package: Package, build: bool) { + let dependencies = package.dependencies(); + for dependency in dependencies { + trace!("Checking package: {}", &dependency.identifier); + if self.packages.contains(&dependency) { + continue + } + trace!("Adding package: {}", &dependency.identifier); + self.packages.push(dependency); + } + if !self.packages.contains(&package) { + self.packages.push(package); + } + } + + pub fn install(&mut self, build: bool) { + self.packages + .iter_mut() + .for_each(|pkg| { + pkg.install(build); + }); + } +} From 247e653ba3a014384cfe0275f027ad37a101688e Mon Sep 17 00:00:00 2001 From: Raine Date: Sat, 14 Oct 2023 22:39:51 +0200 Subject: [PATCH 111/123] docs: i love rust --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 17326aa..f4e975c 100644 --- a/README.md +++ b/README.md @@ -5,6 +5,6 @@ > "A package manager and builder but like rust." > -- Gandhi (2050) -> ***not even close to done*** :) +> ***Almost to a point you can use it*** :) Packager is a simple yet powerful package manager \ No newline at end of file From a28e7b57a202d14ce8df04c07cb257d3a7a05609 Mon Sep 17 00:00:00 2001 From: Raine Date: Sat, 14 Oct 2023 22:39:51 +0200 Subject: [PATCH 112/123] docs: some details --- README.md | 39 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 38 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index f4e975c..42323c9 100644 --- a/README.md +++ b/README.md @@ -7,4 +7,41 @@ > ***Almost to a point you can use it*** :) -Packager is a simple yet powerful package manager \ No newline at end of file +Packager is a simple yet powerful package manager + +## Install a package +pkgr supports fetching packages with a: +- path +- url (http/https) +- package locator + +```shell +# example with path +pkgr install ./foxy/snek.pkg +# example with url +pkgr install https://example.com/doggo.pkg +# example with package locator +pkgr install foo:stable,bar +``` + +## Composing a package +Right now, pkgr does not have a compose/package command. (it's in the works dw!) +You can create a package with `pkg.py`! + +```shell +# Usage: pkg.py [ ...] +# example: +python pkg.py manifest.toml example.pkg root/ source/ scripts/ +# this will create a package with an archive that look like this: +# [header+manifest] +# ... +# [archive: +# root/ +# source/ +# scripts/ +# ] +``` + +It's not perfect, but it'll get the job done :) + + From e4fe3c5779a821ab46c0b9bbc117bcae9145e71c Mon Sep 17 00:00:00 2001 From: Raine Date: Sat, 14 Oct 2023 22:39:51 +0200 Subject: [PATCH 113/123] feat: update according to spec --- pkgr/src/api/mod.rs | 54 ++++++++++++++++++++++++++++++--------------- 1 file changed, 36 insertions(+), 18 deletions(-) diff --git a/pkgr/src/api/mod.rs b/pkgr/src/api/mod.rs index 3cec172..a46a4d0 100644 --- a/pkgr/src/api/mod.rs +++ b/pkgr/src/api/mod.rs @@ -1,27 +1,45 @@ -use reqwest::blocking::Response; +use std::collections::HashMap; -pub struct APITransaction { - /// Query made to API. - query: String, //TODO: enum - /// Response struct from abstracted library - _response: Option -} - -impl APITransaction { - pub fn new(_response: Option) -> Self { - APITransaction { - query: String::default(), - _response - } +enum Query { + Pull { + name: String, + version: Option, + tags: Vec + }, + Push { + _data: Vec } } -pub struct API { - base_url: String, +struct Request { + version: u32, + id: String, token: Option, - history: Vec + query: HashMap } -impl API { +struct DataErrorDetails { + actor: String, + detailed_cause: String, + recovery_options: Vec, +} +struct DataError { + name: String, + cause: Option, + details: Option +} + +enum Data { + Pull { + _data: Option>, + } +} + +struct Response { + version: u32, + id: String, + reply_to: String, + errors: HashMap, + data: HashMap } \ No newline at end of file From 2a67c6f6330c9cb510a63eb5896ca06e053d611b Mon Sep 17 00:00:00 2001 From: Raine Date: Sat, 14 Oct 2023 22:39:51 +0200 Subject: [PATCH 114/123] feat: add prompt to command --- pkgr/src/commands.rs | 16 ++++++++++++++-- pkgr/src/package/installer/mod.rs | 12 +++--------- pkgr/src/package/queue.rs | 20 +++++++++++++++++--- pkgr/src/tmpfs.rs | 2 +- pkgr/src/util/fs.rs | 5 +++-- pkgr/src/util/prompts.rs | 1 + 6 files changed, 39 insertions(+), 17 deletions(-) diff --git a/pkgr/src/commands.rs b/pkgr/src/commands.rs index 869e961..b996408 100644 --- a/pkgr/src/commands.rs +++ b/pkgr/src/commands.rs @@ -1,16 +1,17 @@ use std::process::exit; use clap::{Parser, Subcommand}; +use colored::Colorize; use log::{debug, error, info, trace, warn}; -use colored::Colorize; use manifest::package::PackageType; -use crate::CONFIG; + use crate::package::identifier::PackageIdentifier; use crate::package::Package; use crate::package::queue::PackageQueue; use crate::process::Process; use crate::types::fetch::TryFetch; +use crate::util::prompts::prompt_bool; #[derive(Parser, Debug)] #[clap(name = "pkgr", version)] @@ -25,6 +26,8 @@ pub enum Command { Install { #[arg(short, long, default_value_t = false)] build: bool, + #[arg(short, long, default_value_t = false)] + ask: bool, package_identifier: PackageIdentifier, }, /// Remove a package from the system @@ -57,6 +60,7 @@ impl Command { match self { Command::Install { build, + ask, package_identifier, } => { warn!("Installer does not run in isolation."); @@ -80,6 +84,14 @@ impl Command { let mut queue = PackageQueue::new(); queue.add_package(pkg, *build); trace!("Installing queue..."); + { + if *ask { + info!(target: "item", "{}", queue); + if prompt_bool("Install queue?", false) { + + } + } + } queue.install(*build); let end = std::time::Instant::now(); diff --git a/pkgr/src/package/installer/mod.rs b/pkgr/src/package/installer/mod.rs index da37f1b..30343cb 100644 --- a/pkgr/src/package/installer/mod.rs +++ b/pkgr/src/package/installer/mod.rs @@ -1,22 +1,16 @@ +use std::io; use std::fs::remove_dir_all; use std::path::Path; -use std::process::exit; -use std::{io, thread}; -use libc::SIGINT; -use log::{debug, error, info, trace}; +use log::{debug, trace}; use errors::{BinError, BuildError, InstallError}; use manifest::Manifest; use pkgfile::PKGFile; -use crate::CONFIG; -use crate::package::identifier::{PackageIdentifier, PackageLocator}; -use crate::package::Package; use crate::tmpfs::TempDir; use crate::types::fetch::TryFetch; -use crate::util::fs::{copy_recursively, visit_dirs}; -use crate::util::prompts::{is_noninteractive, prompt_bool}; +use crate::util::fs::copy_recursively; pub mod errors; diff --git a/pkgr/src/package/queue.rs b/pkgr/src/package/queue.rs index 16a7b95..601d8da 100644 --- a/pkgr/src/package/queue.rs +++ b/pkgr/src/package/queue.rs @@ -1,8 +1,11 @@ +use std::fmt::{Display, Formatter}; + use log::trace; + use crate::package::Package; pub struct PackageQueue { - packages: Vec + packages: Vec, } impl PackageQueue { @@ -17,7 +20,7 @@ impl PackageQueue { for dependency in dependencies { trace!("Checking package: {}", &dependency.identifier); if self.packages.contains(&dependency) { - continue + continue; } trace!("Adding package: {}", &dependency.identifier); self.packages.push(dependency); @@ -31,7 +34,18 @@ impl PackageQueue { self.packages .iter_mut() .for_each(|pkg| { - pkg.install(build); + pkg.install(build).expect("TODO: panic message"); }); } } + +impl Display for PackageQueue { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + for pkg in &self.packages { + if let Err(e) = write!(f, "{}", pkg.identifier) { + return Err(e); + } + } + write!(f, "") + } +} \ No newline at end of file diff --git a/pkgr/src/tmpfs.rs b/pkgr/src/tmpfs.rs index e83b860..7d68892 100644 --- a/pkgr/src/tmpfs.rs +++ b/pkgr/src/tmpfs.rs @@ -1,4 +1,4 @@ -use std::path::{Path, PathBuf}; +use std::path::PathBuf; use crate::CONFIG; diff --git a/pkgr/src/util/fs.rs b/pkgr/src/util/fs.rs index 03b5cb2..815b950 100644 --- a/pkgr/src/util/fs.rs +++ b/pkgr/src/util/fs.rs @@ -1,6 +1,7 @@ -use std::path::Path; -use std::fs::DirEntry; use std::{fs, io}; +use std::fs::DirEntry; +use std::path::Path; + use log::trace; pub fn visit_dirs(dir: &Path, cb: &dyn Fn(&DirEntry)) -> std::io::Result<()> { diff --git a/pkgr/src/util/prompts.rs b/pkgr/src/util/prompts.rs index 6c3e11a..9dcae5d 100644 --- a/pkgr/src/util/prompts.rs +++ b/pkgr/src/util/prompts.rs @@ -1,4 +1,5 @@ use std::io::Write; + use log::trace; pub fn is_noninteractive() -> bool { From 1af7fade55a83c8cd99fd63b55005fe28cf7887d Mon Sep 17 00:00:00 2001 From: Raine Date: Sat, 14 Oct 2023 22:39:51 +0200 Subject: [PATCH 115/123] fix: add actual cancel code --- pkgr/src/commands.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkgr/src/commands.rs b/pkgr/src/commands.rs index b996408..ab669cd 100644 --- a/pkgr/src/commands.rs +++ b/pkgr/src/commands.rs @@ -87,8 +87,8 @@ impl Command { { if *ask { info!(target: "item", "{}", queue); - if prompt_bool("Install queue?", false) { - + if !prompt_bool("Install queue?", false) { + return; } } } From 595cc4b6518fc28cf6cbe8ddd35a630c788ad0af Mon Sep 17 00:00:00 2001 From: Raine Date: Sat, 14 Oct 2023 22:39:51 +0200 Subject: [PATCH 116/123] fix: perfection okay im eepy, im goin to eep --- pkgr/src/commands.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pkgr/src/commands.rs b/pkgr/src/commands.rs index ab669cd..80e9448 100644 --- a/pkgr/src/commands.rs +++ b/pkgr/src/commands.rs @@ -86,8 +86,9 @@ impl Command { trace!("Installing queue..."); { if *ask { + info!("Install following packages?"); info!(target: "item", "{}", queue); - if !prompt_bool("Install queue?", false) { + if !prompt_bool("Continue?", false) { return; } } From f54c72c8ecb52138a94c6588cac881954ec78d34 Mon Sep 17 00:00:00 2001 From: Raine Date: Sat, 14 Oct 2023 22:39:51 +0200 Subject: [PATCH 117/123] fix: rename pkgr to extension better name --- manifest/src/{pkgr.rs => ext.rs} | 4 ++-- manifest/src/lib.rs | 6 +++--- package.example.toml | 8 ++++---- 3 files changed, 9 insertions(+), 9 deletions(-) rename manifest/src/{pkgr.rs => ext.rs} (80%) diff --git a/manifest/src/pkgr.rs b/manifest/src/ext.rs similarity index 80% rename from manifest/src/pkgr.rs rename to manifest/src/ext.rs index 530a37c..b47a60c 100644 --- a/manifest/src/pkgr.rs +++ b/manifest/src/ext.rs @@ -2,9 +2,9 @@ use serde::{Deserialize, Serialize}; use std::fmt::{Display, Formatter}; #[derive(Debug, Clone, Serialize, Deserialize)] -pub struct PKGR {} +pub struct Extension {} -impl Display for PKGR { +impl Display for Extension { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { write!(f, "") } diff --git a/manifest/src/lib.rs b/manifest/src/lib.rs index 0d52840..af685e7 100644 --- a/manifest/src/lib.rs +++ b/manifest/src/lib.rs @@ -6,17 +6,17 @@ pub mod bin; pub mod build; pub mod fs; pub mod package; -pub mod pkgr; +pub mod ext; #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(default)] -pub struct Manifest> { +pub struct Manifest> { pub package: package::Package, pub dependencies: HashMap, pub fs: fs::FS, pub bin: Option, pub build: Option, - pub pkgr: P, + pub pkgr: E, } impl Manifest

{ diff --git a/package.example.toml b/package.example.toml index c99108d..eb5618d 100644 --- a/package.example.toml +++ b/package.example.toml @@ -84,16 +84,16 @@ install_script = "scripts/install" # relative to pkg [build.dependencies] base = "latest,stable" # selected by default -## pkgr.* +## ext.* # packager is the official client but you may use other clients supporting the "pkgr v1 spec". -# other clients may offer extra functionality this must be put under "pkgr.*" -[pkgr] +# other clients may offer extra functionality this must be put under "ext.*" +[ext] ## pkgr.bootstrap # This section is used for bootpkg. An edition of packager that bootstraps the full version. # This exists so that packager is easy to install on anything! # and only 1 release channel for pkgr -[pkgr.bootstrap] +[ext.bootstrap] ## any non-zero = installed check_installed_commands = [ "sh scripts/check_installed" From ee81acb95351e25f923c0be9d25bf1dd05c141cc Mon Sep 17 00:00:00 2001 From: Raine Date: Sat, 14 Oct 2023 22:39:52 +0200 Subject: [PATCH 118/123] fix: pkgr->ext pkgrfile --- bootpkg/src/prelude.rs | 4 ++-- manifest/src/lib.rs | 4 ++-- pkgr/package.toml | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/bootpkg/src/prelude.rs b/bootpkg/src/prelude.rs index 9406cdb..b5ec6fe 100644 --- a/bootpkg/src/prelude.rs +++ b/bootpkg/src/prelude.rs @@ -30,12 +30,12 @@ pub fn mani_from_str(s: &str) -> Manifest> { fs: mani.fs, bin: mani.bin, build: mani.build, - pkgr: bmani.pkgr, + ext: bmani.pkgr, } } pub fn run_bootstrap(mani: Manifest>) -> bool { - if let Some(pkgr) = mani.pkgr { + if let Some(pkgr) = mani.ext { if let Some(bootstrap) = pkgr.bootstrap { fn run_command>(s: S) -> i32 { std::process::Command::new("sh") diff --git a/manifest/src/lib.rs b/manifest/src/lib.rs index af685e7..89526f2 100644 --- a/manifest/src/lib.rs +++ b/manifest/src/lib.rs @@ -16,7 +16,7 @@ pub struct Manifest> { pub fs: fs::FS, pub bin: Option, pub build: Option, - pub pkgr: E, + pub ext: E, } impl Manifest

{ @@ -50,7 +50,7 @@ impl Default for Manifest { fs: fs::FS::default(), bin: None, build: None, - pkgr: None, + ext: None, } } } diff --git a/pkgr/package.toml b/pkgr/package.toml index 723ebed..983ea03 100644 --- a/pkgr/package.toml +++ b/pkgr/package.toml @@ -48,13 +48,13 @@ install_script = "scripts/install" # relative to pkg ## pkgr.* # packager is the official client but you may use other clients supporting the "pkgr v1 spec". # other clients may offer extra functionality this must be put under "pkgr.*" -[pkgr] +[ext] ## pkgr.bootstrap # This section is used for bootpkg. An edition of packager that bootstraps the full version. # This exists so that packager is easy to install on anything! # and only 1 release channel for pkgr -[pkgr.bootstrap] +[ext.bootstrap] ## any non-zero = installed check_installed_commands = [ "file /usr/bin/pkgr" From 3902a6beafce797c34c8ce52cb944f0df110d2ef Mon Sep 17 00:00:00 2001 From: Raine Date: Sat, 14 Oct 2023 22:39:52 +0200 Subject: [PATCH 119/123] docs: change names SKIP-CI --- package.example.toml | 2 +- pkgr/package.toml | 50 +++++++------------------------------------- 2 files changed, 9 insertions(+), 43 deletions(-) diff --git a/package.example.toml b/package.example.toml index eb5618d..c20aabb 100644 --- a/package.example.toml +++ b/package.example.toml @@ -89,7 +89,7 @@ base = "latest,stable" # selected by default # other clients may offer extra functionality this must be put under "ext.*" [ext] -## pkgr.bootstrap +## ext.bootstrap # This section is used for bootpkg. An edition of packager that bootstraps the full version. # This exists so that packager is easy to install on anything! # and only 1 release channel for pkgr diff --git a/pkgr/package.toml b/pkgr/package.toml index 983ea03..fa95cd9 100644 --- a/pkgr/package.toml +++ b/pkgr/package.toml @@ -1,66 +1,32 @@ [package] -name = "packager" #* -description = "A package installation tool" #* -version = 1 # this can automatically be incremented when publishing by running `pkgr publish -i ...` +name = "packager" +description = "A package installation tool" +version = 1 tags = [ "prod", "pkgr-spec-1" ] type = "application" +arch = "x86_64" -arch = "x86_64" # this is automatically filled by `pkgr publish ...` - -## dependencies -# you may use the following syntax -# "" or ",,..." [dependencies] -## bin -# Used for systems that don't want to build pkgs. -[bin] # binary files root +[bin] +root = "/root" -## bin.root -# ** RELATIVE TO PACKAGE ROOT ** -# bin.root specifies the root of the installed tree. -# anything in here will be overlayed on top of the system. -root = "/root" #* - -## bin.checksums -# ** KEYS are relative to BIN.ROOT ** -# ** VALUES are relative to PKG ROOT ** -# checksums is used to perform checksums before install -# values may be paths or a uri. You may include variables. -# supported variables: -# - @name -# - @version -# - @display_version [bin.checksums] "/usr/bin/pkgr" = "https://ixvd.net/checksums/packager/@version" -## build -# Scripts will be copied to a custom root. -# After the pacakge is built, the install script is ran. -# After the install script the build directory will be deleted and an CTREE (changed tree) will be generated. -# Then the CTREE will be used to copy the files over. [build] -build_script = "scripts/build" # relative to pkg -install_script = "scripts/install" # relative to pkg +build_script = "scripts/build" +install_script = "scripts/install" [build.dependencies] -## pkgr.* -# packager is the official client but you may use other clients supporting the "pkgr v1 spec". -# other clients may offer extra functionality this must be put under "pkgr.*" [ext] -## pkgr.bootstrap -# This section is used for bootpkg. An edition of packager that bootstraps the full version. -# This exists so that packager is easy to install on anything! -# and only 1 release channel for pkgr [ext.bootstrap] -## any non-zero = installed check_installed_commands = [ "file /usr/bin/pkgr" ] -# any non-zero = fail commands = [ "cp root/usr/bin/pkgr /usr/bin/pkgr" ] From d1882114c74e8cdf8737679f6014f975d71218ab Mon Sep 17 00:00:00 2001 From: Raine Date: Sat, 14 Oct 2023 22:39:52 +0200 Subject: [PATCH 120/123] feat: new config files --- pkgr/Cargo.toml | 1 + pkgr/skel/etc/pkgr.d/repos.toml | 3 +++ pkgr/skel/etc/pkgr.toml | 6 ++++++ pkgr/src/{config.rs => config/mod.rs} | 9 ++++++++- pkgr/src/config/repos.rs | 25 +++++++++++++++++++++++++ pkgr/src/config/storage.rs | 20 ++++++++++++++++++++ pkgr/src/package/installer/mod.rs | 9 ++++++++- 7 files changed, 71 insertions(+), 2 deletions(-) create mode 100644 pkgr/skel/etc/pkgr.d/repos.toml create mode 100644 pkgr/skel/etc/pkgr.toml rename pkgr/src/{config.rs => config/mod.rs} (78%) create mode 100644 pkgr/src/config/repos.rs create mode 100644 pkgr/src/config/storage.rs diff --git a/pkgr/Cargo.toml b/pkgr/Cargo.toml index d6b098f..cb52813 100644 --- a/pkgr/Cargo.toml +++ b/pkgr/Cargo.toml @@ -25,3 +25,4 @@ reqwest = { version = "0.11.18", features = ["blocking"] } tar = "0.4.39" humantime = "2.1.0" expanduser = "1.2.2" +url = { version = "2.4.0", features = ["serde"] } \ No newline at end of file diff --git a/pkgr/skel/etc/pkgr.d/repos.toml b/pkgr/skel/etc/pkgr.d/repos.toml new file mode 100644 index 0000000..cf1316a --- /dev/null +++ b/pkgr/skel/etc/pkgr.d/repos.toml @@ -0,0 +1,3 @@ +[repo.main] +name = "Main" +url = "tcp://pkgs.ixvd.net:1050" \ No newline at end of file diff --git a/pkgr/skel/etc/pkgr.toml b/pkgr/skel/etc/pkgr.toml new file mode 100644 index 0000000..41f8e2c --- /dev/null +++ b/pkgr/skel/etc/pkgr.toml @@ -0,0 +1,6 @@ +build_by_default = false +tmp_dir = "/tmp/pkgr" + +[storage] +repo_file = "/etc/pkgr.d/repos.toml" +data_dir = "/var/lib/pkgr/packages" \ No newline at end of file diff --git a/pkgr/src/config.rs b/pkgr/src/config/mod.rs similarity index 78% rename from pkgr/src/config.rs rename to pkgr/src/config/mod.rs index 9a932d2..4c6876e 100644 --- a/pkgr/src/config.rs +++ b/pkgr/src/config/mod.rs @@ -1,4 +1,8 @@ use serde::{Deserialize, Serialize}; +use storage::Storage; + +mod storage; +pub mod repos; #[derive(Debug, Serialize, Deserialize)] pub struct Config { @@ -6,13 +10,16 @@ pub struct Config { pub build_by_default: bool, #[serde(default)] pub tmp_dir: Option, + #[serde(default)] + pub storage: Storage } impl Default for Config { fn default() -> Config { Config { build_by_default: false, - tmp_dir: Some(String::from("/tmp/pkgr")) + tmp_dir: Some(String::from("/tmp/pkgr")), + storage: Storage::default() } } } diff --git a/pkgr/src/config/repos.rs b/pkgr/src/config/repos.rs new file mode 100644 index 0000000..d689a92 --- /dev/null +++ b/pkgr/src/config/repos.rs @@ -0,0 +1,25 @@ +use std::collections::HashMap; +use std::str::FromStr; +use serde::{Deserialize, Serialize}; +use url::Url; + +#[derive(Clone, Serialize, Deserialize)] +pub struct Repo { + #[serde(default)] + pub name: String, + pub uri: Url +} + +impl Default for Repo { + fn default() -> Self { + Repo { + name: String::from("Repo"), + uri: Url::parse("tcp://0.0.0.0:0000").unwrap() + } + } +} + +#[derive(Clone, Serialize, Deserialize)] +pub struct RepoFile { + pub repos: HashMap +} \ No newline at end of file diff --git a/pkgr/src/config/storage.rs b/pkgr/src/config/storage.rs new file mode 100644 index 0000000..ba18e5f --- /dev/null +++ b/pkgr/src/config/storage.rs @@ -0,0 +1,20 @@ +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Serialize, Deserialize)] +pub struct Storage { + /// Where the repositories are defined. + #[serde(default)] + pub repo_file: String, + /// Where to store pkgs data + #[serde(default)] + pub data_dir: String +} + +impl Default for Storage { + fn default() -> Self { + Storage { + repo_file: String::from("/etc/pkgr.d/repos.toml"), + data_dir: String::from("/var/lib/pkgr/packages"), + } + } +} diff --git a/pkgr/src/package/installer/mod.rs b/pkgr/src/package/installer/mod.rs index 30343cb..4dbeab0 100644 --- a/pkgr/src/package/installer/mod.rs +++ b/pkgr/src/package/installer/mod.rs @@ -7,6 +7,7 @@ use log::{debug, trace}; use errors::{BinError, BuildError, InstallError}; use manifest::Manifest; use pkgfile::PKGFile; +use crate::CONFIG; use crate::tmpfs::TempDir; use crate::types::fetch::TryFetch; @@ -108,11 +109,17 @@ impl PackageInstaller { } } - match self.install_type { + let r = match self.install_type { InstallType::Bin => self.bin() .map_err(|e| InstallError::BinError(e)), InstallType::Build => self.build() .map_err(|e| InstallError::BuildError(e)), + }; + + if let Err(e) = r { + return Err(e); } + + Ok(()) } } From bd589d8b455a8efd299e1d4a996fc6623d116860 Mon Sep 17 00:00:00 2001 From: Raine Date: Sat, 14 Oct 2023 22:39:52 +0200 Subject: [PATCH 121/123] feat: new stuff --- .idea/jsonSchemas.xml | 25 ++++++++++ build | 2 +- pkgr/Cargo.toml | 3 +- pkgr/skel/etc/pkgr.d/repos.toml | 3 +- pkgr/skel/etc/pkgr.toml | 3 +- pkgr/skel/var/lib/pkgr/indexes/repo.toml | 20 ++++++++ pkgr/src/api/client.rs | 0 pkgr/src/api/mod.rs | 30 +++++++++--- pkgr/src/config/repos.rs | 12 +++++ pkgr/src/config/storage.rs | 6 ++- pkgr/src/main.rs | 2 + pkgr/src/repo/index/index_package.rs | 18 ++++++++ pkgr/src/repo/index/mod.rs | 58 ++++++++++++++++++++++++ pkgr/src/repo/mod.rs | 42 +++++++++++++++++ pkgr/src/util/mod.rs | 9 +++- 15 files changed, 221 insertions(+), 12 deletions(-) create mode 100644 .idea/jsonSchemas.xml create mode 100644 pkgr/skel/var/lib/pkgr/indexes/repo.toml create mode 100644 pkgr/src/api/client.rs create mode 100644 pkgr/src/repo/index/index_package.rs create mode 100644 pkgr/src/repo/index/mod.rs create mode 100644 pkgr/src/repo/mod.rs diff --git a/.idea/jsonSchemas.xml b/.idea/jsonSchemas.xml new file mode 100644 index 0000000..f5ad76c --- /dev/null +++ b/.idea/jsonSchemas.xml @@ -0,0 +1,25 @@ + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/build b/build index 2f69899..c245f7a 100755 --- a/build +++ b/build @@ -3,7 +3,7 @@ build_pkgr() { cd pkgr mkdir -p dist/root/{usr/bin,etc/pkgr} - echo -e "You can't use pkgr to update pkgr because the file will be in use while updating.\nuse bootpkg" > dist/root/etc/pkgr/YOU-CAN-NOT-USE-PKGR-TO-UPDATE-PKGR.txt + echo -e "You can't use pkgr to update pkgr because the file will be in use while updating.\nuse bootpkg" > dist/root/etc/pkgr.d/YOU-CAN-NOT-USE-PKGR-TO-UPDATE-PKGR.txt # for bin cargo build -r diff --git a/pkgr/Cargo.toml b/pkgr/Cargo.toml index cb52813..e43b436 100644 --- a/pkgr/Cargo.toml +++ b/pkgr/Cargo.toml @@ -25,4 +25,5 @@ reqwest = { version = "0.11.18", features = ["blocking"] } tar = "0.4.39" humantime = "2.1.0" expanduser = "1.2.2" -url = { version = "2.4.0", features = ["serde"] } \ No newline at end of file +url = { version = "2.4.0", features = ["serde"] } +dns-lookup = "2.0.3" diff --git a/pkgr/skel/etc/pkgr.d/repos.toml b/pkgr/skel/etc/pkgr.d/repos.toml index cf1316a..2723e7b 100644 --- a/pkgr/skel/etc/pkgr.d/repos.toml +++ b/pkgr/skel/etc/pkgr.d/repos.toml @@ -1,3 +1,4 @@ [repo.main] name = "Main" -url = "tcp://pkgs.ixvd.net:1050" \ No newline at end of file +url = "tcp://pkgs.ixvd.net:1050" + diff --git a/pkgr/skel/etc/pkgr.toml b/pkgr/skel/etc/pkgr.toml index 41f8e2c..2ee5004 100644 --- a/pkgr/skel/etc/pkgr.toml +++ b/pkgr/skel/etc/pkgr.toml @@ -3,4 +3,5 @@ tmp_dir = "/tmp/pkgr" [storage] repo_file = "/etc/pkgr.d/repos.toml" -data_dir = "/var/lib/pkgr/packages" \ No newline at end of file +data_dir = "/var/lib/pkgr/packages" +index_dir = "/var/lib/pkgr/indexes" \ No newline at end of file diff --git a/pkgr/skel/var/lib/pkgr/indexes/repo.toml b/pkgr/skel/var/lib/pkgr/indexes/repo.toml new file mode 100644 index 0000000..47fa234 --- /dev/null +++ b/pkgr/skel/var/lib/pkgr/indexes/repo.toml @@ -0,0 +1,20 @@ +[index.main] +last_update = "1610219288" +[index.main.packager] +versions = [1, 2, 3] +tags = ["tag"] + +## rust +# IndexedPackage +# - name -> packager +# - origin_repo -> main +# - versions -> [1,2,3] +# - tags -> ["tag"] +# - get_package() -> Package + +## rust +# Repo +# - name -> main +# - last_update -> 1610219288 +# - update_repo() -> io::Result<()> +# - searchIndex(p: PackageIdentifier) -> IndexedPackage \ No newline at end of file diff --git a/pkgr/src/api/client.rs b/pkgr/src/api/client.rs new file mode 100644 index 0000000..e69de29 diff --git a/pkgr/src/api/mod.rs b/pkgr/src/api/mod.rs index a46a4d0..1ed3d70 100644 --- a/pkgr/src/api/mod.rs +++ b/pkgr/src/api/mod.rs @@ -1,42 +1,60 @@ use std::collections::HashMap; +use serde; +use serde::{Deserialize, Serialize}; -enum Query { +pub mod client; + +#[derive(Serialize, Deserialize)] +pub enum Query { + #[serde(rename = "pull")] Pull { name: String, version: Option, tags: Vec }, + #[serde(rename = "push")] Push { + // todo: review me pls _data: Vec + }, + #[serde(rename = "index")] + Index { + request_update: bool, + fetch: bool } } -struct Request { +#[derive(Serialize, Deserialize)] +pub struct Request { version: u32, id: String, token: Option, query: HashMap } -struct DataErrorDetails { +#[derive(Serialize, Deserialize)] +pub struct DataErrorDetails { actor: String, detailed_cause: String, recovery_options: Vec, } -struct DataError { +#[derive(Serialize, Deserialize)] +pub struct DataError { name: String, cause: Option, details: Option } -enum Data { +#[derive(Serialize, Deserialize)] +pub enum Data { Pull { _data: Option>, } } -struct Response { +#[derive(Serialize, Deserialize)] +pub struct Response { version: u32, id: String, reply_to: String, diff --git a/pkgr/src/config/repos.rs b/pkgr/src/config/repos.rs index d689a92..8594a25 100644 --- a/pkgr/src/config/repos.rs +++ b/pkgr/src/config/repos.rs @@ -22,4 +22,16 @@ impl Default for Repo { #[derive(Clone, Serialize, Deserialize)] pub struct RepoFile { pub repos: HashMap +} + +impl RepoFile { + pub fn from_path(path: &str) -> Result { + match std::fs::read_to_string(path) { + Ok(s) => match toml::from_str(&s) { + Ok(c) => Ok(c), + Err(e) => Err(format!("failed to parse config: {}", e)), + }, + Err(e) => Err(format!("failed to read config: {}", e)), + } + } } \ No newline at end of file diff --git a/pkgr/src/config/storage.rs b/pkgr/src/config/storage.rs index ba18e5f..5fa8fa1 100644 --- a/pkgr/src/config/storage.rs +++ b/pkgr/src/config/storage.rs @@ -7,7 +7,10 @@ pub struct Storage { pub repo_file: String, /// Where to store pkgs data #[serde(default)] - pub data_dir: String + pub data_dir: String, + /// Where to store repo indexes + #[serde(default)] + pub index_dir: String, } impl Default for Storage { @@ -15,6 +18,7 @@ impl Default for Storage { Storage { repo_file: String::from("/etc/pkgr.d/repos.toml"), data_dir: String::from("/var/lib/pkgr/packages"), + index_dir: String::from("/var/lib/pkgr/indexes"), } } } diff --git a/pkgr/src/main.rs b/pkgr/src/main.rs index 798eb1d..3aff896 100644 --- a/pkgr/src/main.rs +++ b/pkgr/src/main.rs @@ -9,6 +9,8 @@ mod commands; mod logging; /// Package and helpers. mod package; +/// Repo and helpers +mod repo; /// Process wrapper with logging wrapper. mod process; /// tmpfs wrapper. diff --git a/pkgr/src/repo/index/index_package.rs b/pkgr/src/repo/index/index_package.rs new file mode 100644 index 0000000..ca60160 --- /dev/null +++ b/pkgr/src/repo/index/index_package.rs @@ -0,0 +1,18 @@ +use std::io; +use std::path::Path; +use serde::{Deserialize, Serialize}; +use url::Url; +use pkgfile::PKGFile; +use crate::api::Query; +use crate::CONFIG; +use crate::package::Package; +use crate::repo::index::RepoIndex; + +/// This struct solely exists for indexing and has no real functionality. +#[derive(Serialize, Deserialize)] +pub struct IndexPackage { + name: String, + versions: Vec, + tags: Vec, + uri: Url +} \ No newline at end of file diff --git a/pkgr/src/repo/index/mod.rs b/pkgr/src/repo/index/mod.rs new file mode 100644 index 0000000..fc516b6 --- /dev/null +++ b/pkgr/src/repo/index/mod.rs @@ -0,0 +1,58 @@ +use std::collections::HashMap; +use std::io; +use std::path::Path; +use serde::{Deserialize, Serialize}; +use crate::api::Request; +use crate::CONFIG; +use crate::repo::index::index_package::IndexPackage; +use crate::repo::Repo; +use crate::types::fetch::{Fetch, TryFetch}; +use crate::util::create_uuid; + +pub mod index_package; + +#[derive(Serialize, Deserialize)] +pub struct RepoIndex { + origin_repo: String, + packages: HashMap +} + +impl RepoIndex { + // /// Fetch existing index or create a new one. + // pub fn from_repo(repo: Repo) -> Self { + // + // } + // + // /// Create new index. + // pub fn create_with_repo(repo: Repo) -> Self { + // } + + /// Get repo + pub fn get_repo(&self) -> io::Result { + Ok(Repo::from_name(&self.origin_repo)?) + } + + pub fn from_path(path: &str) -> Result { + match std::fs::read_to_string(path) { + Ok(s) => match toml::from_str(&s) { + Ok(c) => Ok(c), + Err(e) => Err(format!("failed to parse config: {}", e)), + }, + Err(e) => Err(format!("failed to read config: {}", e)), + } + } +} + +impl TryFetch for RepoIndex { + type Error = (); + /// Fetch + fn try_fetch(query: Repo) -> Result { + let path = CONFIG.with(|c| c.storage.index_dir.clone()) + query.uri.as_str() + ".toml"; + if Path::new(path.as_str()).exists() { + RepoIndex::from_path(path.as_str()) + .map_err(|_| ()) + } else { + Err(()) + } + } +} \ No newline at end of file diff --git a/pkgr/src/repo/mod.rs b/pkgr/src/repo/mod.rs new file mode 100644 index 0000000..d0c3e88 --- /dev/null +++ b/pkgr/src/repo/mod.rs @@ -0,0 +1,42 @@ +use std::io; +use std::io::{Error, ErrorKind}; +use std::path::Path; +use url::Url; +use crate::CONFIG; +use crate::config::repos::RepoFile; + +/// Indexed repos +pub mod index; + +pub struct Repo { + name: String, + uri: Url, +} + +impl Repo { + pub fn from_name(name: &String) -> io::Result { + let r = RepoFile::from_path(CONFIG.with(|c| c.storage.repo_file.clone()).as_str()) + .map_err(|e| Error::new(ErrorKind::Other, e))?; + let r = r + .repos + .get(name) + .ok_or(Error::new(ErrorKind::InvalidData, "Could not get repo"))?; + Ok(Repo { + name: r.name.clone(), + uri: r.uri.clone() + }) + } + + pub fn get_name(&self) -> String { + self.name.clone() + } + + pub fn get_uri(&self) -> Url { + self.uri.clone() + } + + /// Fetch indexed repo + pub fn get_index(&self) { + + } +} \ No newline at end of file diff --git a/pkgr/src/util/mod.rs b/pkgr/src/util/mod.rs index c239775..25c313c 100644 --- a/pkgr/src/util/mod.rs +++ b/pkgr/src/util/mod.rs @@ -1,2 +1,9 @@ pub mod prompts; -pub mod fs; \ No newline at end of file +/// Helpers for fs +pub mod fs; + +/// Create a UUID +pub fn create_uuid() -> String { + // TODO + String::from("rand") +} \ No newline at end of file From 971ad81fa4881cf68e0123007ff02c1b1e46a033 Mon Sep 17 00:00:00 2001 From: Strix Date: Thu, 21 Nov 2024 15:24:27 +0100 Subject: [PATCH 122/123] Update docs/pkgfiles/current.md --- docs/pkgfiles/current.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/pkgfiles/current.md b/docs/pkgfiles/current.md index bd0eb22..e5cd8cc 100644 --- a/docs/pkgfiles/current.md +++ b/docs/pkgfiles/current.md @@ -7,7 +7,7 @@ PKGFiles are an all-in-one solution to install a package. ## Overhead -Because pkgfiles include everything, many times you can download more than what is needed. +Because pkgfiles include everything, many times you're downloading more than what is needed. This may seem stupid but pkgfiles are *not* meant to stay on your system. ## Format From e7c6798ad30774292e44709ca404390eb0b996b2 Mon Sep 17 00:00:00 2001 From: Strix Date: Thu, 21 Nov 2024 15:29:54 +0100 Subject: [PATCH 123/123] Update docs/README.md --- docs/README.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/README.md b/docs/README.md index 0e9b295..5b84964 100644 --- a/docs/README.md +++ b/docs/README.md @@ -9,7 +9,9 @@ pkgr is the main tool for the packager. Because this tool is so feature rich, it has its own [README](./pkgr/README.md). ## bootpkg -bootpkg is a tool primariliy used to bootstrap the packager. +`bootpkg` is a simple pkgr variant. +It doesn't contain anything fancy and is only used to install and update pkgr. +Using it for anything other than pkgr is not recommended. ### Usage @@ -17,4 +19,4 @@ bootpkg is a tool primariliy used to bootstrap the packager. bootpkg strap ./pkgr.pkg ``` -This will extract the pkgfile and read the `[pkgr.bootstrap]` section to bootstrap the packager or any other package. \ No newline at end of file +This will extract the pkgfile and read the `[pkgr.bootstrap]` section to bootstrap the packager or any other package.