From db3cf3e42d93112278f5e86cca2c886627ef48b2 Mon Sep 17 00:00:00 2001 From: Ava Hahn Date: Mon, 22 Apr 2024 13:26:34 +0100 Subject: tools: Add unitctl CLI * Pull in entire unit-rust-sdk project * not included: CLA, COC, License * not included: duplicate openapi spec * not included: CI workflows * not included: changelog tooling * not included: commitsar tooling * not included: OpenAPI Web UI feature * update links in unitctl manpage * remove IDE configuration from .gitignore * rename Containerfile.debian to Dockerfile * simplify call to uname * keep Readmes and Makefiles to 80 character lines * outline specifically how to build unitctl for any desired target, and where to then find the binary for use * remove a section on the vision of the CLI which was superfluous given the state of completeness of the code and its use in unit * remove out of date feature proposals from readme * makefile: do not run when Rustup is not present * bump mio version to latest * generate openapi client library on demand * generate-openapi only runs when not present * generate-openapi now a dependency of binary build targets * deleted autogenerated code * reverted readme and Cargo document to autogenerated state * add additional build requirement to Readme Co-developed-by: Elijah Zupancic Signed-off-by: Elijah Zupancic Signed-off-by: Ava Hahn Reviewed-by: Andrew Clayton # non rust stuff [ tools/cli => tools/unitctl and subject tweak - Andrew ] Signed-off-by: Andrew Clayton --- tools/unitctl/unit-client-rs/src/unitd_instance.rs | 360 +++++++++++++++++++++ 1 file changed, 360 insertions(+) create mode 100644 tools/unitctl/unit-client-rs/src/unitd_instance.rs (limited to 'tools/unitctl/unit-client-rs/src/unitd_instance.rs') diff --git a/tools/unitctl/unit-client-rs/src/unitd_instance.rs b/tools/unitctl/unit-client-rs/src/unitd_instance.rs new file mode 100644 index 00000000..9467fcb7 --- /dev/null +++ b/tools/unitctl/unit-client-rs/src/unitd_instance.rs @@ -0,0 +1,360 @@ +use crate::unit_client::UnitClientError; +use serde::ser::SerializeMap; +use serde::{Serialize, Serializer}; +use std::error::Error as StdError; +use std::path::{Path, PathBuf}; +use std::{fmt, io}; +use which::which; + +use crate::runtime_flags::RuntimeFlags; +use crate::unitd_configure_options::UnitdConfigureOptions; +use crate::unitd_process::UnitdProcess; + +pub const UNITD_PATH_ENV_KEY: &str = "UNITD_PATH"; +pub const UNITD_BINARY_NAMES: [&str; 2] = ["unitd", "unitd-debug"]; + +#[derive(Debug)] +pub struct UnitdInstance { + pub process: UnitdProcess, + pub configure_options: Option, + pub errors: Vec, +} + +impl Serialize for UnitdInstance { + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + let mut state = serializer.serialize_map(Some(15))?; + let runtime_flags = self + .process + .cmd() + .and_then(|cmd| cmd.flags) + .map(|flags| flags.to_string()); + + let configure_flags = self.configure_options.as_ref().map(|opts| opts.all_flags.clone()); + + state.serialize_entry("pid", &self.process.process_id)?; + state.serialize_entry("version", &self.version())?; + state.serialize_entry("user", &self.process.user)?; + state.serialize_entry("effective_user", &self.process.effective_user)?; + state.serialize_entry("executable", &self.process.executable_path())?; + state.serialize_entry("control_socket", &self.control_api_socket_address())?; + state.serialize_entry("child_pids", &self.process.child_pids)?; + state.serialize_entry("log_path", &self.log_path())?; + state.serialize_entry("pid_path", &self.pid_path())?; + state.serialize_entry("modules_directory", &self.modules_directory())?; + state.serialize_entry("state_directory", &self.state_directory())?; + state.serialize_entry("tmp_directory", &self.tmp_directory())?; + state.serialize_entry("runtime_flags", &runtime_flags)?; + state.serialize_entry("configure_flags", &configure_flags)?; + let string_errors = &self.errors.iter().map(|e| e.to_string()).collect::>(); + state.serialize_entry("errors", string_errors)?; + + state.end() + } +} + +impl UnitdInstance { + pub fn running_unitd_instances() -> Vec { + Self::collect_unitd_processes(UnitdProcess::find_unitd_processes()) + } + + /// Find all running unitd processes and convert them into UnitdInstances and filter + /// out all errors by printing them to stderr and leaving errored instances out of + /// the returned vector. + fn collect_unitd_processes(processes: Vec) -> Vec { + Self::map_processes_to_instances(processes).into_iter().collect() + } + + fn map_processes_to_instances(processes: Vec) -> Vec { + fn unitd_path_from_process(process: &UnitdProcess) -> Result, UnitClientError> { + match process.executable_path() { + Some(executable_path) => { + let is_absolute_working_dir = process + .working_dir + .as_ref() + .map(|p| p.is_absolute()) + .unwrap_or_default(); + if executable_path.is_absolute() { + Ok(executable_path.to_owned()) + } else if executable_path.is_relative() && is_absolute_working_dir { + let new_path = process + .working_dir + .as_ref() + .unwrap() + .join(executable_path) + .canonicalize() + .map(|path| path.into_boxed_path()) + .map_err(|error| UnitClientError::UnitdProcessParseError { + message: format!("Error canonicalizing unitd executable path: {}", error), + pid: process.process_id, + })?; + Ok(new_path) + } else { + Err(UnitClientError::UnitdProcessParseError { + message: "Unable to get absolute unitd executable path from process".to_string(), + pid: process.process_id, + }) + } + } + None => Err(UnitClientError::UnitdProcessParseError { + message: "Unable to get unitd executable path from process".to_string(), + pid: process.process_id, + }), + } + } + + fn map_process_to_unitd_instance(process: &UnitdProcess) -> UnitdInstance { + match unitd_path_from_process(process) { + Ok(unitd_path) => match UnitdConfigureOptions::new(&unitd_path.clone().into_path_buf()) { + Ok(configure_options) => UnitdInstance { + process: process.to_owned(), + configure_options: Some(configure_options), + errors: vec![], + }, + Err(error) => { + let error = UnitClientError::UnitdProcessExecError { + source: error, + executable_path: unitd_path.to_string_lossy().parse().unwrap_or_default(), + message: "Error running unitd binary to get configure options".to_string(), + pid: process.process_id, + }; + UnitdInstance { + process: process.to_owned(), + configure_options: None, + errors: vec![error], + } + } + }, + Err(err) => UnitdInstance { + process: process.to_owned(), + configure_options: None, + errors: vec![err], + }, + } + } + + processes + .iter() + // This converts processes into a UnitdInstance + .map(map_process_to_unitd_instance) + .collect() + } + + fn version(&self) -> Option { + match self.process.cmd()?.version { + Some(version) => Some(version), + None => self.configure_options.as_ref().map(|opts| opts.version.to_string()), + } + } + + fn flag_or_default_option( + &self, + read_flag: fn(RuntimeFlags) -> Option, + read_opts: fn(UnitdConfigureOptions) -> Option, + ) -> Option { + self.process + .cmd()? + .flags + .and_then(read_flag) + .or_else(|| self.configure_options.to_owned().and_then(read_opts)) + } + + pub fn control_api_socket_address(&self) -> Option { + self.flag_or_default_option( + |flags| flags.control_api_socket_address(), + |opts| opts.default_control_api_socket_address(), + ) + } + + pub fn pid_path(&self) -> Option> { + self.flag_or_default_option(|flags| flags.pid_path(), |opts| opts.default_pid_path()) + } + + pub fn log_path(&self) -> Option> { + self.flag_or_default_option(|flags| flags.log_path(), |opts| opts.default_log_path()) + } + + pub fn modules_directory(&self) -> Option> { + self.flag_or_default_option( + |flags| flags.modules_directory(), + |opts| opts.default_modules_directory(), + ) + } + + pub fn state_directory(&self) -> Option> { + self.flag_or_default_option(|flags| flags.state_directory(), |opts| opts.default_state_directory()) + } + + pub fn tmp_directory(&self) -> Option> { + self.flag_or_default_option(|flags| flags.tmp_directory(), |opts| opts.default_tmp_directory()) + } +} + +impl fmt::Display for UnitdInstance { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + const UNKNOWN: &str = "[unknown]"; + let version = self.version().unwrap_or_else(|| String::from("[unknown]")); + let runtime_flags = self + .process + .cmd() + .and_then(|cmd| cmd.flags) + .map(|flags| flags.to_string()) + .unwrap_or_else(|| UNKNOWN.into()); + let configure_flags = self + .configure_options + .as_ref() + .map(|opts| opts.all_flags.clone()) + .unwrap_or_else(|| UNKNOWN.into()); + let unitd_path: String = self + .process + .executable_path() + .map(|p| p.to_string_lossy().into()) + .unwrap_or_else(|| UNKNOWN.into()); + let working_dir: String = self + .process + .working_dir + .as_ref() + .map(|p| p.to_string_lossy().into()) + .unwrap_or_else(|| UNKNOWN.into()); + let socket_address = self.control_api_socket_address().unwrap_or_else(|| UNKNOWN.to_string()); + let child_pids = self + .process + .child_pids + .iter() + .map(u64::to_string) + .collect::>() + .join(", "); + + writeln!( + f, + "{} instance [pid: {}, version: {}]:", + self.process.binary_name, self.process.process_id, version + )?; + writeln!(f, " Executable: {}", unitd_path)?; + writeln!(f, " Process working directory: {}", working_dir)?; + write!(f, " Process ownership: ")?; + if let Some(user) = &self.process.user { + writeln!(f, "name: {}, uid: {}, gid: {}", user.name, user.uid, user.gid)?; + } else { + writeln!(f, "{}", UNKNOWN)?; + } + write!(f, " Process effective ownership: ")?; + if let Some(user) = &self.process.effective_user { + writeln!(f, "name: {}, uid: {}, gid: {}", user.name, user.uid, user.gid)?; + } else { + writeln!(f, "{}", UNKNOWN)?; + } + + writeln!(f, " API control unix socket: {}", socket_address)?; + writeln!(f, " Child processes ids: {}", child_pids)?; + writeln!(f, " Runtime flags: {}", runtime_flags)?; + write!(f, " Configure options: {}", configure_flags)?; + + if !self.errors.is_empty() { + write!(f, "\n Errors:")?; + for error in &self.errors { + write!(f, "\n {}", error)?; + } + } + + Ok(()) + } +} + +pub fn find_executable_path(specific_path: Result>) -> Result> { + fn find_unitd_in_system_path() -> Vec { + UNITD_BINARY_NAMES + .iter() + .map(which) + .filter_map(Result::ok) + .collect::>() + } + + match specific_path { + Ok(path) => Ok(PathBuf::from(path)), + Err(_) => { + let unitd_paths = find_unitd_in_system_path(); + if unitd_paths.is_empty() { + let err_msg = format!( + "Could not find unitd in system path or in UNITD_PATH environment variable. Searched for: {:?}", + UNITD_BINARY_NAMES + ); + let err = io::Error::new(io::ErrorKind::NotFound, err_msg); + Err(Box::from(err)) + } else { + Ok(unitd_paths[0].clone()) + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use rand::rngs::StdRng; + use rand::{RngCore, SeedableRng}; + + // We don't need a secure seed for testing, in fact it is better that we have a + // predictable value + const SEED: [u8; 32] = [ + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, + 30, 31, + ]; + #[test] + fn can_find_unitd_instances() { + UnitdInstance::running_unitd_instances().iter().for_each(|p| { + println!("{:?}", p); + println!("Runtime Flags: {:?}", p.process.cmd().map(|c| c.flags)); + println!("Temp directory: {:?}", p.tmp_directory()); + }) + } + + fn mock_process>( + rng: &mut StdRng, + binary_name: S, + executable_path: Option, + ) -> UnitdProcess { + UnitdProcess { + process_id: rng.next_u32() as u64, + binary_name: binary_name.into(), + executable_path: executable_path.map(|p| Box::from(Path::new(&p))), + environ: vec![], + all_cmds: vec![], + working_dir: Some(Box::from(Path::new("/opt/unit"))), + child_pids: vec![], + user: None, + effective_user: None, + } + } + + #[test] + fn will_list_without_errors_valid_processes() { + let specific_path = std::env::var(UNITD_PATH_ENV_KEY).map_err(|error| Box::new(error) as Box); + let binding = match find_executable_path(specific_path) { + Ok(path) => path, + Err(error) => { + eprintln!("Could not find unitd executable path: {} - skipping test", error); + return; + } + }; + let binary_name = binding + .file_name() + .expect("Could not get binary name") + .to_string_lossy() + .to_string(); + let unitd_path = binding.to_string_lossy(); + let mut rng: StdRng = SeedableRng::from_seed(SEED); + + let processes = vec![ + mock_process(&mut rng, &binary_name, Some(unitd_path.to_string())), + mock_process(&mut rng, &binary_name, Some(unitd_path.to_string())), + ]; + let instances = UnitdInstance::collect_unitd_processes(processes); + // assert_eq!(instances.len(), 3); + instances.iter().for_each(|p| { + assert_eq!(p.errors.len(), 0, "Expected no errors, got: {:?}", p.errors); + }) + } +} -- cgit From 6e8f7bbb91e7069d82abd22fbe8d0fcaa1bb2f8c Mon Sep 17 00:00:00 2001 From: Ava Hahn Date: Thu, 25 Apr 2024 20:02:26 -0700 Subject: tools/unitctl: Initial Docker Procedures * move UnitdProcess serialization logic into UnitdProcess * filter out docker processes from process output on Linux * initial implementation of a UnitdContainer type * initial implementation of a docker container search for unitd * pull out custom openapi future executor and use same tokio runtime as docker client * refactor openapi client to not manage its own tokio runtime * process mount points per docker container * correctly output docker container info in relevant unitd instances * create UnitdProcess from UnitdContainer * UnitdProcess now owns UnitdContainer * get and parse container details from docker API * introduce procedure to rewrite file paths based on docker container mounts * test path rewrite facilities * apply path rewrite to unix socket Signed-off-by: Ava Hahn --- tools/unitctl/unit-client-rs/src/unitd_instance.rs | 77 +++++++++++++++++----- 1 file changed, 62 insertions(+), 15 deletions(-) (limited to 'tools/unitctl/unit-client-rs/src/unitd_instance.rs') diff --git a/tools/unitctl/unit-client-rs/src/unitd_instance.rs b/tools/unitctl/unit-client-rs/src/unitd_instance.rs index 9467fcb7..86f8e73d 100644 --- a/tools/unitctl/unit-client-rs/src/unitd_instance.rs +++ b/tools/unitctl/unit-client-rs/src/unitd_instance.rs @@ -1,4 +1,5 @@ use crate::unit_client::UnitClientError; +use crate::unitd_docker::UnitdContainer; use serde::ser::SerializeMap; use serde::{Serialize, Serializer}; use std::error::Error as StdError; @@ -25,7 +26,7 @@ impl Serialize for UnitdInstance { where S: Serializer, { - let mut state = serializer.serialize_map(Some(15))?; + let mut state = serializer.serialize_map(Some(11))?; let runtime_flags = self .process .cmd() @@ -34,13 +35,9 @@ impl Serialize for UnitdInstance { let configure_flags = self.configure_options.as_ref().map(|opts| opts.all_flags.clone()); - state.serialize_entry("pid", &self.process.process_id)?; + state.serialize_entry("process", &self.process)?; state.serialize_entry("version", &self.version())?; - state.serialize_entry("user", &self.process.user)?; - state.serialize_entry("effective_user", &self.process.effective_user)?; - state.serialize_entry("executable", &self.process.executable_path())?; state.serialize_entry("control_socket", &self.control_api_socket_address())?; - state.serialize_entry("child_pids", &self.process.child_pids)?; state.serialize_entry("log_path", &self.log_path())?; state.serialize_entry("pid_path", &self.pid_path())?; state.serialize_entry("modules_directory", &self.modules_directory())?; @@ -56,8 +53,19 @@ impl Serialize for UnitdInstance { } impl UnitdInstance { - pub fn running_unitd_instances() -> Vec { - Self::collect_unitd_processes(UnitdProcess::find_unitd_processes()) + pub async fn running_unitd_instances() -> Vec { + Self::collect_unitd_processes( + UnitdProcess::find_unitd_processes() + .into_iter() + .chain( + UnitdContainer::find_unitd_containers() + .await + .into_iter() + .map(|x| UnitdProcess::from(&x)) + .collect::>(), + ) + .collect(), + ) } /// Find all running unitd processes and convert them into UnitdInstances and filter @@ -91,11 +99,14 @@ impl UnitdInstance { pid: process.process_id, })?; Ok(new_path) - } else { + } else if process.container.is_none() { Err(UnitClientError::UnitdProcessParseError { message: "Unable to get absolute unitd executable path from process".to_string(), pid: process.process_id, }) + } else { + // container case + Ok(PathBuf::from("/").into_boxed_path()) } } None => Err(UnitClientError::UnitdProcessParseError { @@ -107,7 +118,30 @@ impl UnitdInstance { fn map_process_to_unitd_instance(process: &UnitdProcess) -> UnitdInstance { match unitd_path_from_process(process) { - Ok(unitd_path) => match UnitdConfigureOptions::new(&unitd_path.clone().into_path_buf()) { + Ok(_) if process.container.is_some() => { + let mut err = vec![]; + // double check that it is running + let running = process.container + .as_ref() + .unwrap() + .container_is_running(); + + if running.is_none() || !running.unwrap() { + err.push(UnitClientError::UnitdProcessParseError{ + message: "process container is not running".to_string(), + pid: process.process_id, + }); + } + + UnitdInstance { + process: process.to_owned(), + configure_options: None, + errors: err, + } + }, + Ok(unitd_path) => match UnitdConfigureOptions::new( + &unitd_path.clone() + .into_path_buf()) { Ok(configure_options) => UnitdInstance { process: process.to_owned(), configure_options: Some(configure_options), @@ -250,10 +284,22 @@ impl fmt::Display for UnitdInstance { writeln!(f, " API control unix socket: {}", socket_address)?; writeln!(f, " Child processes ids: {}", child_pids)?; writeln!(f, " Runtime flags: {}", runtime_flags)?; - write!(f, " Configure options: {}", configure_flags)?; + writeln!(f, " Configure options: {}", configure_flags)?; + + if let Some(ctr) = &self.process.container { + writeln!(f, " Container:")?; + writeln!(f, " Platform: {}", ctr.platform)?; + if let Some(id) = ctr.container_id.clone() { + writeln!(f, " Container ID: {}", id)?; + } + writeln!(f, " Mounts:")?; + for (k, v) in &ctr.mounts { + writeln!(f, " {} => {}", k.to_string_lossy(), v.to_string_lossy())?; + } + } if !self.errors.is_empty() { - write!(f, "\n Errors:")?; + write!(f, " Errors:")?; for error in &self.errors { write!(f, "\n {}", error)?; } @@ -302,9 +348,9 @@ mod tests { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, ]; - #[test] - fn can_find_unitd_instances() { - UnitdInstance::running_unitd_instances().iter().for_each(|p| { + #[tokio::test] + async fn can_find_unitd_instances() { + UnitdInstance::running_unitd_instances().await.iter().for_each(|p| { println!("{:?}", p); println!("Runtime Flags: {:?}", p.process.cmd().map(|c| c.flags)); println!("Temp directory: {:?}", p.tmp_directory()); @@ -326,6 +372,7 @@ mod tests { child_pids: vec![], user: None, effective_user: None, + container: None, } } -- cgit From 1d237990c589a90d7ec85f211c01b709b1e5dc65 Mon Sep 17 00:00:00 2001 From: Ava Hahn Date: Wed, 1 May 2024 18:21:55 -0700 Subject: tools/unitctl: Add new functionality to README.md and fmt code Signed-off-by: Ava Hahn --- tools/unitctl/unit-client-rs/src/unitd_instance.rs | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) (limited to 'tools/unitctl/unit-client-rs/src/unitd_instance.rs') diff --git a/tools/unitctl/unit-client-rs/src/unitd_instance.rs b/tools/unitctl/unit-client-rs/src/unitd_instance.rs index 86f8e73d..a7fb1bdc 100644 --- a/tools/unitctl/unit-client-rs/src/unitd_instance.rs +++ b/tools/unitctl/unit-client-rs/src/unitd_instance.rs @@ -121,13 +121,10 @@ impl UnitdInstance { Ok(_) if process.container.is_some() => { let mut err = vec![]; // double check that it is running - let running = process.container - .as_ref() - .unwrap() - .container_is_running(); + let running = process.container.as_ref().unwrap().container_is_running(); if running.is_none() || !running.unwrap() { - err.push(UnitClientError::UnitdProcessParseError{ + err.push(UnitClientError::UnitdProcessParseError { message: "process container is not running".to_string(), pid: process.process_id, }); @@ -138,10 +135,8 @@ impl UnitdInstance { configure_options: None, errors: err, } - }, - Ok(unitd_path) => match UnitdConfigureOptions::new( - &unitd_path.clone() - .into_path_buf()) { + } + Ok(unitd_path) => match UnitdConfigureOptions::new(&unitd_path.clone().into_path_buf()) { Ok(configure_options) => UnitdInstance { process: process.to_owned(), configure_options: Some(configure_options), -- cgit From cc9eb8e756e84cd3fd59baf5b80efab0ffd5757d Mon Sep 17 00:00:00 2001 From: Ava Hahn Date: Mon, 6 May 2024 12:28:40 -0700 Subject: tools/unitctl: enable passing IP addresses to the 'instances new' command * use path seperator constant from rust std package * pass a ControlSocket into deploy_new_container instead of a string * parse and validate a ControlSocket from argument to instances new * conditionally mount control socket only if its a unix socket * use create_image in a way that actually pulls nonpresent images * possibly override container command if TCP socket passed in * handle more weird error cases * add a ton of validation cases in the CLI command handler * add a nice little progress bar :) Signed-off-by: Ava Hahn --- tools/unitctl/unit-client-rs/src/unitd_instance.rs | 1 + 1 file changed, 1 insertion(+) (limited to 'tools/unitctl/unit-client-rs/src/unitd_instance.rs') diff --git a/tools/unitctl/unit-client-rs/src/unitd_instance.rs b/tools/unitctl/unit-client-rs/src/unitd_instance.rs index a7fb1bdc..ace8e858 100644 --- a/tools/unitctl/unit-client-rs/src/unitd_instance.rs +++ b/tools/unitctl/unit-client-rs/src/unitd_instance.rs @@ -26,6 +26,7 @@ impl Serialize for UnitdInstance { where S: Serializer, { + // 11 = fields to serialize let mut state = serializer.serialize_map(Some(11))?; let runtime_flags = self .process -- cgit