From 6e8f7bbb91e7069d82abd22fbe8d0fcaa1bb2f8c Mon Sep 17 00:00:00 2001 From: Ava Hahn Date: Thu, 25 Apr 2024 20:02:26 -0700 Subject: tools/unitctl: Initial Docker Procedures * move UnitdProcess serialization logic into UnitdProcess * filter out docker processes from process output on Linux * initial implementation of a UnitdContainer type * initial implementation of a docker container search for unitd * pull out custom openapi future executor and use same tokio runtime as docker client * refactor openapi client to not manage its own tokio runtime * process mount points per docker container * correctly output docker container info in relevant unitd instances * create UnitdProcess from UnitdContainer * UnitdProcess now owns UnitdContainer * get and parse container details from docker API * introduce procedure to rewrite file paths based on docker container mounts * test path rewrite facilities * apply path rewrite to unix socket Signed-off-by: Ava Hahn --- tools/unitctl/unit-client-rs/src/unitd_docker.rs | 282 +++++++++++++++++++++++ 1 file changed, 282 insertions(+) create mode 100644 tools/unitctl/unit-client-rs/src/unitd_docker.rs (limited to 'tools/unitctl/unit-client-rs/src/unitd_docker.rs') diff --git a/tools/unitctl/unit-client-rs/src/unitd_docker.rs b/tools/unitctl/unit-client-rs/src/unitd_docker.rs new file mode 100644 index 00000000..d5028afc --- /dev/null +++ b/tools/unitctl/unit-client-rs/src/unitd_docker.rs @@ -0,0 +1,282 @@ +use std::collections::HashMap; +use std::fs::read_to_string; +use std::path::PathBuf; + +use crate::unitd_process::UnitdProcess; + +use bollard::secret::ContainerInspectResponse; +use regex::Regex; +use serde::ser::SerializeMap; +use serde::{Serialize, Serializer}; + +use bollard::{models::ContainerSummary, Docker}; + +#[derive(Clone, Debug)] +pub struct UnitdContainer { + pub container_id: Option, + pub container_image: String, + pub command: Option, + pub mounts: HashMap, + pub platform: String, + details: Option, +} + +impl From<&ContainerSummary> for UnitdContainer { + fn from(ctr: &ContainerSummary) -> Self { + // we assume paths from the docker api are absolute + // they certainly have to be later... + let mut mounts = HashMap::new(); + if let Some(mts) = &ctr.mounts { + for i in mts { + if let Some(ref src) = i.source { + if let Some(ref dest) = i.destination { + mounts.insert(PathBuf::from(dest.clone()), PathBuf::from(src.clone())); + } + } + } + } + + UnitdContainer { + container_id: ctr.id.clone(), + container_image: format!( + "{} (docker)", + ctr.image.clone().unwrap_or(String::from("unknown container")), + ), + command: ctr.command.clone(), + mounts: mounts, + platform: String::from("Docker"), + details: None, + } + } +} + +impl From<&UnitdContainer> for UnitdProcess { + fn from(ctr: &UnitdContainer) -> Self { + let version = ctr.details.as_ref().and_then(|details| { + details.config.as_ref().and_then(|conf| { + conf.labels.as_ref().and_then(|labels| { + labels + .get("org.opencontainers.image.version") + .and_then(|version| Some(version.clone())) + }) + }) + }); + let command = ctr.command.clone().and_then(|cmd| { + Some(format!( + "{}{} [{}{}]", + "unit: main v", + version.or(Some(String::from(""))).unwrap(), + ctr.container_image, + ctr.rewrite_socket( + cmd.strip_prefix("/usr/local/bin/docker-entrypoint.sh") + .or_else(|| Some("")) + .unwrap() + .to_string()) + )) + }); + let mut cmds = vec![]; + let _ = command.map_or((), |cmd| cmds.push(cmd)); + UnitdProcess { + all_cmds: cmds, + binary_name: ctr.container_image.clone(), + process_id: ctr + .details + .as_ref() + .and_then(|details| { + details + .state + .as_ref() + .and_then(|state| state.pid.and_then(|pid| Some(pid.clone() as u64))) + }) + .or(Some(0 as u64)) + .unwrap(), + executable_path: None, + environ: vec![], + working_dir: ctr.details.as_ref().and_then(|details| { + details.config.as_ref().and_then(|conf| { + Some( + PathBuf::from( + conf.working_dir + .as_ref() + .map_or(String::new(), |dir| ctr.host_path(dir.clone())), + ) + .into_boxed_path(), + ) + }) + }), + child_pids: vec![], + user: None, + effective_user: None, + container: Some(ctr.clone()), + } + } +} + +impl Serialize for UnitdContainer { + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + let mut state = serializer.serialize_map(Some(5))?; + state.serialize_entry("container_id", &self.container_id)?; + state.serialize_entry("container_image", &self.container_image)?; + state.serialize_entry("command", &self.command)?; + state.serialize_entry("mounts", &self.mounts)?; + state.serialize_entry("platform", &self.platform)?; + state.end() + } +} + +impl UnitdContainer { + pub async fn find_unitd_containers() -> Vec { + if let Ok(docker) = Docker::connect_with_local_defaults() { + match docker.list_containers::(None).await { + Err(e) => { + eprintln!("{}", e); + vec![] + } + Ok(summary) => { + // cant do this functionally because of the async call + let mut mapped = vec![]; + for ctr in summary { + if ctr.clone().image.or(Some(String::new())).unwrap().contains("unit") { + let mut c = UnitdContainer::from(&ctr); + if let Some(names) = ctr.names { + if names.len() > 0 { + let name = names[0].strip_prefix("/").or(Some(names[0].as_str())).unwrap(); + if let Ok(cir) = docker.inspect_container(name, None).await { + c.details = Some(cir); + } + } + } + mapped.push(c); + } + } + mapped + } + } + } else { + vec![] + } + } + + pub fn host_path(&self, container_path: String) -> String { + let cp = PathBuf::from(container_path); + + // get only possible mount points + // sort to deepest mountpoint first + // assumed deepest possible mount point takes precedence + let mut keys = self + .mounts + .clone() + .into_keys() + .filter(|mp| cp.as_path().starts_with(mp)) + .collect::>(); + keys.sort_by_key(|a| 0 as isize - a.ancestors().count() as isize); + + // either return translated path or original prefixed with "container" + if keys.len() > 0 { + self.mounts[&keys[0]] + .clone() + .join( + cp.as_path() + .strip_prefix(keys[0].clone()) + .expect("error checking path prefix"), + ) + .to_string_lossy() + .to_string() + } else { + format!(":{}", cp.display()) + } + } + + pub fn rewrite_socket(&self, command: String) -> String { + command + .split(" ") + .map(|tok| if tok.starts_with("unix:") { + format!("unix:{}", self.host_path( + tok.strip_prefix("unix:") + .unwrap() + .to_string())) + } else { + tok.to_string() + }) + .collect::>() + .join(" ") + } + + pub fn container_is_running(&self) -> Option { + self.details + .as_ref() + .and_then(|details| details.state.as_ref().and_then(|state| state.running)) + } +} + +/* Returns either 64 char docker container ID or None */ +pub fn pid_is_dockerized(pid: u64) -> bool { + let cg_filepath = format!("/proc/{}/cgroup", pid); + match read_to_string(cg_filepath) { + Err(e) => { + eprintln!("{}", e); + false + } + Ok(contents) => { + let docker_re = Regex::new(r"docker-([a-zA-Z0-9]{64})").unwrap(); + docker_re.is_match(contents.as_str()) + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_path_translation() { + let mut mounts = HashMap::new(); + mounts.insert("/1/2/3/4/5/6/7".into(), "/0".into()); + mounts.insert("/root".into(), "/1".into()); + mounts.insert("/root/mid".into(), "/2".into()); + mounts.insert("/root/mid/child".into(), "/3".into()); + mounts.insert("/mid/child".into(), "/4".into()); + mounts.insert("/child".into(), "/5".into()); + + let ctr = UnitdContainer { + container_id: None, + container_image: String::from(""), + command: None, + platform: "test".to_string(), + details: None, + mounts: mounts, + }; + + assert_eq!( + "/3/c2/test".to_string(), + ctr.host_path("/root/mid/child/c2/test".to_string()) + ); + assert_eq!( + ":/path/to/conf".to_string(), + ctr.host_path("/path/to/conf".to_string()) + ); + } + + #[test] + fn test_unix_sock_path_translate() { + let mut mounts = HashMap::new(); + mounts.insert("/var/run".into(), "/tmp".into()); + + let ctr = UnitdContainer { + container_id: None, + container_image: String::from(""), + command: None, + platform: "test".to_string(), + details: None, + mounts: mounts, + }; + + assert_eq!( + ctr.rewrite_socket("unitd --no-daemon --control unix:/var/run/control.unit.sock".to_string()), + "unitd --no-daemon --control unix:/tmp/control.unit.sock".to_string()); + + } +} -- cgit From 818d4ad76592c87a5c0c7bbd728636023c07daa0 Mon Sep 17 00:00:00 2001 From: Ava Hahn Date: Wed, 1 May 2024 13:59:33 -0700 Subject: tools/unitctl: API Plumbing for docker deployments * refactored "instance" command out of enum * plumbed through function stub from client library * error handling Signed-off-by: Ava Hahn --- tools/unitctl/unit-client-rs/src/unitd_docker.rs | 12 ++++++++++++ 1 file changed, 12 insertions(+) (limited to 'tools/unitctl/unit-client-rs/src/unitd_docker.rs') diff --git a/tools/unitctl/unit-client-rs/src/unitd_docker.rs b/tools/unitctl/unit-client-rs/src/unitd_docker.rs index d5028afc..0f30ae8a 100644 --- a/tools/unitctl/unit-client-rs/src/unitd_docker.rs +++ b/tools/unitctl/unit-client-rs/src/unitd_docker.rs @@ -3,6 +3,7 @@ use std::fs::read_to_string; use std::path::PathBuf; use crate::unitd_process::UnitdProcess; +use crate::unit_client::UnitClientError; use bollard::secret::ContainerInspectResponse; use regex::Regex; @@ -212,6 +213,17 @@ impl UnitdContainer { } } +/* deploys a new docker image of tag $image_tag. + * mounts $socket to /var/run in the new container. + * mounts $application to /www in the new container. */ +pub fn deploy_new_container( + _socket: &String, + _application: &String, + _image: &String +) -> Result<(), UnitClientError> { + todo!() +} + /* Returns either 64 char docker container ID or None */ pub fn pid_is_dockerized(pid: u64) -> bool { let cg_filepath = format!("/proc/{}/cgroup", pid); -- cgit From f6989dd67965c7489f8c68ecd0e25f0358b5993f Mon Sep 17 00:00:00 2001 From: Ava Hahn Date: Wed, 1 May 2024 17:08:56 -0700 Subject: tools/unitctl: Add Docker deployment functionality * add UnitdDockerError type * write complete procedure to deploy unit via docker * additional tweaks verifying it fails peacefully * print important information in client Signed-off-by: Ava Hahn --- tools/unitctl/unit-client-rs/src/unitd_docker.rs | 102 +++++++++++++++++++++-- 1 file changed, 93 insertions(+), 9 deletions(-) (limited to 'tools/unitctl/unit-client-rs/src/unitd_docker.rs') diff --git a/tools/unitctl/unit-client-rs/src/unitd_docker.rs b/tools/unitctl/unit-client-rs/src/unitd_docker.rs index 0f30ae8a..47f017a9 100644 --- a/tools/unitctl/unit-client-rs/src/unitd_docker.rs +++ b/tools/unitctl/unit-client-rs/src/unitd_docker.rs @@ -4,13 +4,16 @@ use std::path::PathBuf; use crate::unitd_process::UnitdProcess; use crate::unit_client::UnitClientError; - +use bollard::{models::ContainerSummary, Docker}; +use bollard::container::{Config, StartContainerOptions, ListContainersOptions}; +use bollard::image::CreateImageOptions; use bollard::secret::ContainerInspectResponse; +use bollard::models::{HostConfig, MountTypeEnum, Mount, ContainerCreateResponse}; use regex::Regex; use serde::ser::SerializeMap; use serde::{Serialize, Serializer}; +use crate::futures::StreamExt; -use bollard::{models::ContainerSummary, Docker}; #[derive(Clone, Debug)] pub struct UnitdContainer { @@ -215,13 +218,94 @@ impl UnitdContainer { /* deploys a new docker image of tag $image_tag. * mounts $socket to /var/run in the new container. - * mounts $application to /www in the new container. */ -pub fn deploy_new_container( - _socket: &String, - _application: &String, - _image: &String -) -> Result<(), UnitClientError> { - todo!() + * mounts $application read only to /www. + * new container is on host network. + * + * ON SUCCESS returns vector of warnings from Docker API + * ON FAILURE returns wrapped error from Docker API + */ +pub async fn deploy_new_container( + socket: &String, + application: &String, + image: &String +) -> Result, UnitClientError> { + match Docker::connect_with_local_defaults() { + Ok(docker) => { + let mut mounts = vec![]; + mounts.push(Mount{ + typ: Some(MountTypeEnum::BIND), + source: Some(socket.clone()), + target: Some("/var/run".to_string()), + ..Default::default() + }); + mounts.push(Mount{ + typ: Some(MountTypeEnum::BIND), + source: Some(application.clone()), + target: Some("/www".to_string()), + read_only: Some(true), + ..Default::default() + }); + + let _ = docker.create_image( + Some(CreateImageOptions { + from_image: image.as_str(), + ..Default::default() + }), None, None) + .next() + .await + .unwrap() + .or_else(|err| Err(UnitClientError::UnitdDockerError{message: err.to_string()})); + + let resp: ContainerCreateResponse; + match docker.create_container::( + None, Config { + image: Some(image.clone()), + host_config: Some(HostConfig { + network_mode: Some("host".to_string()), + mounts: Some(mounts), + ..Default::default() + }), ..Default::default()}) + .await { + Err(err) => return Err(UnitClientError::UnitdDockerError{message: err.to_string()}), + Ok(response) => resp = response, + } + + let mut list_container_filters = HashMap::new(); + list_container_filters.insert("id".to_string(), vec![resp.id]); + match docker.list_containers::( + Some(ListContainersOptions{ + all: true, + limit: None, + size: false, + filters: list_container_filters, + })) + .await { + Err(e) => Err(UnitClientError::UnitdDockerError{message: e.to_string()}), + Ok(info) => { + if info.len() < 1 { + return Err(UnitClientError::UnitdDockerError{message: "couldnt find new container".to_string()}); + } + if info[0].names.is_none() || info[0].names.clone().unwrap().len() < 1 { + return Err(UnitClientError::UnitdDockerError{message: "new container has no name".to_string()}); + } + + match docker.start_container( + info[0] + .names + .clone() + .unwrap()[0] + .strip_prefix("/") + .unwrap(), + None::> + ).await { + Err(err) => Err(UnitClientError::UnitdDockerError{message: err.to_string()}), + Ok(_) => Ok(resp.warnings) + } + } + } + }, + Err(e) => Err(UnitClientError::UnitdDockerError{message: e.to_string()}) + } } /* Returns either 64 char docker container ID or None */ -- cgit From 1d237990c589a90d7ec85f211c01b709b1e5dc65 Mon Sep 17 00:00:00 2001 From: Ava Hahn Date: Wed, 1 May 2024 18:21:55 -0700 Subject: tools/unitctl: Add new functionality to README.md and fmt code Signed-off-by: Ava Hahn --- tools/unitctl/unit-client-rs/src/unitd_docker.rs | 145 +++++++++++++---------- 1 file changed, 85 insertions(+), 60 deletions(-) (limited to 'tools/unitctl/unit-client-rs/src/unitd_docker.rs') diff --git a/tools/unitctl/unit-client-rs/src/unitd_docker.rs b/tools/unitctl/unit-client-rs/src/unitd_docker.rs index 47f017a9..2b51ddd3 100644 --- a/tools/unitctl/unit-client-rs/src/unitd_docker.rs +++ b/tools/unitctl/unit-client-rs/src/unitd_docker.rs @@ -2,18 +2,17 @@ use std::collections::HashMap; use std::fs::read_to_string; use std::path::PathBuf; -use crate::unitd_process::UnitdProcess; +use crate::futures::StreamExt; use crate::unit_client::UnitClientError; -use bollard::{models::ContainerSummary, Docker}; -use bollard::container::{Config, StartContainerOptions, ListContainersOptions}; +use crate::unitd_process::UnitdProcess; +use bollard::container::{Config, ListContainersOptions, StartContainerOptions}; use bollard::image::CreateImageOptions; +use bollard::models::{ContainerCreateResponse, HostConfig, Mount, MountTypeEnum}; use bollard::secret::ContainerInspectResponse; -use bollard::models::{HostConfig, MountTypeEnum, Mount, ContainerCreateResponse}; +use bollard::{models::ContainerSummary, Docker}; use regex::Regex; use serde::ser::SerializeMap; use serde::{Serialize, Serializer}; -use crate::futures::StreamExt; - #[derive(Clone, Debug)] pub struct UnitdContainer { @@ -75,7 +74,8 @@ impl From<&UnitdContainer> for UnitdProcess { cmd.strip_prefix("/usr/local/bin/docker-entrypoint.sh") .or_else(|| Some("")) .unwrap() - .to_string()) + .to_string() + ) )) }); let mut cmds = vec![]; @@ -197,13 +197,15 @@ impl UnitdContainer { pub fn rewrite_socket(&self, command: String) -> String { command .split(" ") - .map(|tok| if tok.starts_with("unix:") { - format!("unix:{}", self.host_path( - tok.strip_prefix("unix:") - .unwrap() - .to_string())) - } else { - tok.to_string() + .map(|tok| { + if tok.starts_with("unix:") { + format!( + "unix:{}", + self.host_path(tok.strip_prefix("unix:").unwrap().to_string()) + ) + } else { + tok.to_string() + } }) .collect::>() .join(" ") @@ -227,18 +229,18 @@ impl UnitdContainer { pub async fn deploy_new_container( socket: &String, application: &String, - image: &String + image: &String, ) -> Result, UnitClientError> { match Docker::connect_with_local_defaults() { Ok(docker) => { let mut mounts = vec![]; - mounts.push(Mount{ + mounts.push(Mount { typ: Some(MountTypeEnum::BIND), source: Some(socket.clone()), target: Some("/var/run".to_string()), ..Default::default() }); - mounts.push(Mount{ + mounts.push(Mount { typ: Some(MountTypeEnum::BIND), source: Some(application.clone()), target: Some("/www".to_string()), @@ -246,65 +248,88 @@ pub async fn deploy_new_container( ..Default::default() }); - let _ = docker.create_image( - Some(CreateImageOptions { - from_image: image.as_str(), - ..Default::default() - }), None, None) + let _ = docker + .create_image( + Some(CreateImageOptions { + from_image: image.as_str(), + ..Default::default() + }), + None, + None, + ) .next() .await .unwrap() - .or_else(|err| Err(UnitClientError::UnitdDockerError{message: err.to_string()})); + .or_else(|err| { + Err(UnitClientError::UnitdDockerError { + message: err.to_string(), + }) + }); let resp: ContainerCreateResponse; - match docker.create_container::( - None, Config { - image: Some(image.clone()), - host_config: Some(HostConfig { - network_mode: Some("host".to_string()), - mounts: Some(mounts), + match docker + .create_container::( + None, + Config { + image: Some(image.clone()), + host_config: Some(HostConfig { + network_mode: Some("host".to_string()), + mounts: Some(mounts), + ..Default::default() + }), ..Default::default() - }), ..Default::default()}) - .await { - Err(err) => return Err(UnitClientError::UnitdDockerError{message: err.to_string()}), - Ok(response) => resp = response, + }, + ) + .await + { + Err(err) => { + return Err(UnitClientError::UnitdDockerError { + message: err.to_string(), + }) } + Ok(response) => resp = response, + } let mut list_container_filters = HashMap::new(); list_container_filters.insert("id".to_string(), vec![resp.id]); - match docker.list_containers::( - Some(ListContainersOptions{ + match docker + .list_containers::(Some(ListContainersOptions { all: true, limit: None, size: false, filters: list_container_filters, })) - .await { - Err(e) => Err(UnitClientError::UnitdDockerError{message: e.to_string()}), - Ok(info) => { - if info.len() < 1 { - return Err(UnitClientError::UnitdDockerError{message: "couldnt find new container".to_string()}); - } - if info[0].names.is_none() || info[0].names.clone().unwrap().len() < 1 { - return Err(UnitClientError::UnitdDockerError{message: "new container has no name".to_string()}); - } + .await + { + Err(e) => Err(UnitClientError::UnitdDockerError { message: e.to_string() }), + Ok(info) => { + if info.len() < 1 { + return Err(UnitClientError::UnitdDockerError { + message: "couldnt find new container".to_string(), + }); + } + if info[0].names.is_none() || info[0].names.clone().unwrap().len() < 1 { + return Err(UnitClientError::UnitdDockerError { + message: "new container has no name".to_string(), + }); + } - match docker.start_container( - info[0] - .names - .clone() - .unwrap()[0] - .strip_prefix("/") - .unwrap(), - None::> - ).await { - Err(err) => Err(UnitClientError::UnitdDockerError{message: err.to_string()}), - Ok(_) => Ok(resp.warnings) - } + match docker + .start_container( + info[0].names.clone().unwrap()[0].strip_prefix("/").unwrap(), + None::>, + ) + .await + { + Err(err) => Err(UnitClientError::UnitdDockerError { + message: err.to_string(), + }), + Ok(_) => Ok(resp.warnings), } } - }, - Err(e) => Err(UnitClientError::UnitdDockerError{message: e.to_string()}) + } + } + Err(e) => Err(UnitClientError::UnitdDockerError { message: e.to_string() }), } } @@ -372,7 +397,7 @@ mod tests { assert_eq!( ctr.rewrite_socket("unitd --no-daemon --control unix:/var/run/control.unit.sock".to_string()), - "unitd --no-daemon --control unix:/tmp/control.unit.sock".to_string()); - + "unitd --no-daemon --control unix:/tmp/control.unit.sock".to_string() + ); } } -- cgit From cb03d31e02741b87fc3a3aa379bf8f1849442ae2 Mon Sep 17 00:00:00 2001 From: Ava Hahn Date: Fri, 3 May 2024 17:52:47 -0700 Subject: tools/unitctl: Update host_path() to account for OSX special behaviour Signed-off-by: Ava Hahn --- tools/unitctl/unit-client-rs/src/unitd_docker.rs | 39 +++++++++++++++++++----- 1 file changed, 31 insertions(+), 8 deletions(-) (limited to 'tools/unitctl/unit-client-rs/src/unitd_docker.rs') diff --git a/tools/unitctl/unit-client-rs/src/unitd_docker.rs b/tools/unitctl/unit-client-rs/src/unitd_docker.rs index 2b51ddd3..4c86c870 100644 --- a/tools/unitctl/unit-client-rs/src/unitd_docker.rs +++ b/tools/unitctl/unit-client-rs/src/unitd_docker.rs @@ -166,7 +166,6 @@ impl UnitdContainer { pub fn host_path(&self, container_path: String) -> String { let cp = PathBuf::from(container_path); - // get only possible mount points // sort to deepest mountpoint first // assumed deepest possible mount point takes precedence @@ -180,14 +179,26 @@ impl UnitdContainer { // either return translated path or original prefixed with "container" if keys.len() > 0 { - self.mounts[&keys[0]] + let mut matches = self.mounts[&keys[0]] .clone() - .join( - cp.as_path() - .strip_prefix(keys[0].clone()) - .expect("error checking path prefix"), - ) - .to_string_lossy() + .join(cp.as_path() + .strip_prefix(keys[0].clone()) + .expect("error checking path prefix")); + /* Observed on M1 Mac that Docker on OSX + * adds a bunch of garbage to the mount path + * converting it into a useless directory + * that doesnt actually exist + */ + if cfg!(target_os = "macos") { + let mut abs = PathBuf::from("/"); + let m = matches.strip_prefix("/host_mnt/private") + .unwrap_or(matches.strip_prefix("/host_mnt") + .unwrap_or(matches.as_path())); + // make it absolute again + abs.push(m); + matches = abs; + } + matches.to_string_lossy() .to_string() } else { format!(":{}", cp.display()) @@ -361,6 +372,8 @@ mod tests { mounts.insert("/root/mid/child".into(), "/3".into()); mounts.insert("/mid/child".into(), "/4".into()); mounts.insert("/child".into(), "/5".into()); + mounts.insert("/var".into(), "/host_mnt/private/6".into()); + mounts.insert("/var/var".into(), "/host_mnt/7".into()); let ctr = UnitdContainer { container_id: None, @@ -379,6 +392,16 @@ mod tests { ":/path/to/conf".to_string(), ctr.host_path("/path/to/conf".to_string()) ); + if cfg!(target_os = "macos") { + assert_eq!( + "/6/test".to_string(), + ctr.host_path("/var/test".to_string()) + ); + assert_eq!( + "/7/test".to_string(), + ctr.host_path("/var/var/test".to_string()) + ); + } } #[test] -- cgit From cc9eb8e756e84cd3fd59baf5b80efab0ffd5757d Mon Sep 17 00:00:00 2001 From: Ava Hahn Date: Mon, 6 May 2024 12:28:40 -0700 Subject: tools/unitctl: enable passing IP addresses to the 'instances new' command * use path seperator constant from rust std package * pass a ControlSocket into deploy_new_container instead of a string * parse and validate a ControlSocket from argument to instances new * conditionally mount control socket only if its a unix socket * use create_image in a way that actually pulls nonpresent images * possibly override container command if TCP socket passed in * handle more weird error cases * add a ton of validation cases in the CLI command handler * add a nice little progress bar :) Signed-off-by: Ava Hahn --- tools/unitctl/unit-client-rs/src/unitd_docker.rs | 177 ++++++++++++++--------- 1 file changed, 111 insertions(+), 66 deletions(-) (limited to 'tools/unitctl/unit-client-rs/src/unitd_docker.rs') diff --git a/tools/unitctl/unit-client-rs/src/unitd_docker.rs b/tools/unitctl/unit-client-rs/src/unitd_docker.rs index 4c86c870..b9199e40 100644 --- a/tools/unitctl/unit-client-rs/src/unitd_docker.rs +++ b/tools/unitctl/unit-client-rs/src/unitd_docker.rs @@ -1,19 +1,29 @@ use std::collections::HashMap; use std::fs::read_to_string; -use std::path::PathBuf; +use std::path::{PathBuf, MAIN_SEPARATOR}; +use std::io::stderr; use crate::futures::StreamExt; use crate::unit_client::UnitClientError; use crate::unitd_process::UnitdProcess; +use crate::control_socket_address::ControlSocket; + use bollard::container::{Config, ListContainersOptions, StartContainerOptions}; use bollard::image::CreateImageOptions; -use bollard::models::{ContainerCreateResponse, HostConfig, Mount, MountTypeEnum}; +use bollard::models::{ + ContainerCreateResponse, HostConfig, Mount, + MountTypeEnum, ContainerSummary, +}; use bollard::secret::ContainerInspectResponse; -use bollard::{models::ContainerSummary, Docker}; +use bollard::Docker; + use regex::Regex; + use serde::ser::SerializeMap; use serde::{Serialize, Serializer}; +use pbr::ProgressBar; + #[derive(Clone, Debug)] pub struct UnitdContainer { pub container_id: Option, @@ -121,6 +131,7 @@ impl Serialize for UnitdContainer { where S: Serializer, { + // 5 = fields to serialize let mut state = serializer.serialize_map(Some(5))?; state.serialize_entry("container_id", &self.container_id)?; state.serialize_entry("container_image", &self.container_image)?; @@ -143,18 +154,23 @@ impl UnitdContainer { // cant do this functionally because of the async call let mut mapped = vec![]; for ctr in summary { - if ctr.clone().image.or(Some(String::new())).unwrap().contains("unit") { - let mut c = UnitdContainer::from(&ctr); - if let Some(names) = ctr.names { - if names.len() > 0 { - let name = names[0].strip_prefix("/").or(Some(names[0].as_str())).unwrap(); - if let Ok(cir) = docker.inspect_container(name, None).await { - c.details = Some(cir); + if ctr.clone().image + .or(Some(String::new())) + .unwrap() + .contains("unit") { + let mut c = UnitdContainer::from(&ctr); + if let Some(names) = ctr.names { + if names.len() > 0 { + let name = names[0].strip_prefix("/") + .or(Some(names[0].as_str())).unwrap(); + if let Ok(cir) = docker + .inspect_container(name, None).await { + c.details = Some(cir); + } } } + mapped.push(c); } - mapped.push(c); - } } mapped } @@ -190,7 +206,7 @@ impl UnitdContainer { * that doesnt actually exist */ if cfg!(target_os = "macos") { - let mut abs = PathBuf::from("/"); + let mut abs = PathBuf::from(String::from(MAIN_SEPARATOR)); let m = matches.strip_prefix("/host_mnt/private") .unwrap_or(matches.strip_prefix("/host_mnt") .unwrap_or(matches.as_path())); @@ -238,19 +254,27 @@ impl UnitdContainer { * ON FAILURE returns wrapped error from Docker API */ pub async fn deploy_new_container( - socket: &String, + socket: ControlSocket, application: &String, image: &String, ) -> Result, UnitClientError> { match Docker::connect_with_local_defaults() { Ok(docker) => { let mut mounts = vec![]; - mounts.push(Mount { - typ: Some(MountTypeEnum::BIND), - source: Some(socket.clone()), - target: Some("/var/run".to_string()), - ..Default::default() - }); + // if a unix socket is specified, mounts its directory + if socket.is_local_socket() { + let mount_path = PathBuf::from(socket.clone()) + .as_path() + .to_string_lossy() + .to_string(); + mounts.push(Mount { + typ: Some(MountTypeEnum::BIND), + source: Some(mount_path), + target: Some("/var/run".to_string()), + ..Default::default() + }); + } + // mount application dir mounts.push(Mount { typ: Some(MountTypeEnum::BIND), source: Some(application.clone()), @@ -259,40 +283,57 @@ pub async fn deploy_new_container( ..Default::default() }); - let _ = docker - .create_image( - Some(CreateImageOptions { - from_image: image.as_str(), - ..Default::default() - }), - None, - None, - ) - .next() - .await - .unwrap() - .or_else(|err| { - Err(UnitClientError::UnitdDockerError { - message: err.to_string(), - }) - }); + let mut pb = ProgressBar::on(stderr(), 10); + let mut totals = HashMap::new(); + let mut stream = docker.create_image( + Some(CreateImageOptions { + from_image: image.as_str(), + ..Default::default() + }), None, None + ); + while let Some(res) = stream.next().await { + if let Ok(info) = res { + if let Some(id) = info.id { + if let Some(_) = totals.get_mut(&id) { + if let Some(delta) = info.progress_detail + .and_then(|detail| detail.current) { + pb.add(delta as u64); + } + } else { + if let Some(total) = info.progress_detail + .and_then(|detail| detail.total) { + totals.insert(id, total); + pb.total += total as u64; + } + } + } + } + } + pb.finish(); + // create the new unit container let resp: ContainerCreateResponse; - match docker - .create_container::( - None, - Config { - image: Some(image.clone()), - host_config: Some(HostConfig { - network_mode: Some("host".to_string()), - mounts: Some(mounts), - ..Default::default() - }), - ..Default::default() - }, - ) - .await - { + let host_conf = HostConfig { + mounts: Some(mounts), + network_mode: Some("host".to_string()), + ..Default::default() + }; + let mut container_conf = Config { + image: Some(image.clone()), + ..Default::default() + }; + if let ControlSocket::TcpSocket(ref uri) = socket { + let port = uri.port_u16().or(Some(80)).unwrap(); + // override port + container_conf.cmd = Some(vec![ + "unitd".to_string(), + "--no-daemon".to_string(), + "--control".to_string(), + format!("{}:{}", uri.host().unwrap(), port), + ]); + } + container_conf.host_config = Some(host_conf); + match docker.create_container::(None, container_conf).await { Err(err) => { return Err(UnitClientError::UnitdDockerError { message: err.to_string(), @@ -301,6 +342,8 @@ pub async fn deploy_new_container( Ok(response) => resp = response, } + // create container gives us an ID + // but start container requires a name let mut list_container_filters = HashMap::new(); list_container_filters.insert("id".to_string(), vec![resp.id]); match docker @@ -312,26 +355,28 @@ pub async fn deploy_new_container( })) .await { - Err(e) => Err(UnitClientError::UnitdDockerError { message: e.to_string() }), + // somehow our container doesnt exist + Err(e) => Err(UnitClientError::UnitdDockerError{ + message: e.to_string() + }), + // here it is! Ok(info) => { if info.len() < 1 { return Err(UnitClientError::UnitdDockerError { message: "couldnt find new container".to_string(), }); - } - if info[0].names.is_none() || info[0].names.clone().unwrap().len() < 1 { - return Err(UnitClientError::UnitdDockerError { - message: "new container has no name".to_string(), - }); - } + } else if info[0].names.is_none() || + info[0].names.clone().unwrap().len() < 1 { + return Err(UnitClientError::UnitdDockerError { + message: "new container has no name".to_string(), + }); + } - match docker - .start_container( - info[0].names.clone().unwrap()[0].strip_prefix("/").unwrap(), - None::>, - ) - .await - { + // start our container + match docker.start_container( + info[0].names.clone().unwrap()[0].strip_prefix(MAIN_SEPARATOR).unwrap(), + None::>, + ).await { Err(err) => Err(UnitClientError::UnitdDockerError { message: err.to_string(), }), -- cgit From 4e884d9ecce90e262ea2a8dd6fa53f6d5dce011e Mon Sep 17 00:00:00 2001 From: Gabor Javorszky Date: Fri, 17 May 2024 11:23:47 +0100 Subject: tools/unitctl: Replace matching image name to matching command Closes #1254 Matching to the `unitd` command is a far more reliable way to filtering docker instances that are running Unit. --- tools/unitctl/unit-client-rs/src/unitd_docker.rs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) (limited to 'tools/unitctl/unit-client-rs/src/unitd_docker.rs') diff --git a/tools/unitctl/unit-client-rs/src/unitd_docker.rs b/tools/unitctl/unit-client-rs/src/unitd_docker.rs index b9199e40..6881893d 100644 --- a/tools/unitctl/unit-client-rs/src/unitd_docker.rs +++ b/tools/unitctl/unit-client-rs/src/unitd_docker.rs @@ -151,13 +151,14 @@ impl UnitdContainer { vec![] } Ok(summary) => { + let unitd_command_re = Regex::new(r"^(.* )?unitd( .*)?$").unwrap(); + // cant do this functionally because of the async call let mut mapped = vec![]; for ctr in summary { - if ctr.clone().image + if unitd_command_re.is_match(&ctr.clone().command .or(Some(String::new())) - .unwrap() - .contains("unit") { + .unwrap()) { let mut c = UnitdContainer::from(&ctr); if let Some(names) = ctr.names { if names.len() > 0 { -- cgit From e0c15ae4575335fb079e2d33fc853a547b2380c9 Mon Sep 17 00:00:00 2001 From: Ava Hahn Date: Fri, 14 Jun 2024 21:04:15 -0700 Subject: tools/unitctl: implement application subcommand * application subcommand UI schema * application subcommand handler * additions to unit-client-rs to expose application API * elaborate on OpenAPI error handling * adds wasm and wasi app schemas to OpenAPI Schema * updates tools/unitctl OpenAPI library * many linter fixes * README.md updates Signed-off-by: Ava Hahn --- tools/unitctl/unit-client-rs/src/unitd_docker.rs | 111 ++++++++++------------- 1 file changed, 47 insertions(+), 64 deletions(-) (limited to 'tools/unitctl/unit-client-rs/src/unitd_docker.rs') diff --git a/tools/unitctl/unit-client-rs/src/unitd_docker.rs b/tools/unitctl/unit-client-rs/src/unitd_docker.rs index 6881893d..0d318096 100644 --- a/tools/unitctl/unit-client-rs/src/unitd_docker.rs +++ b/tools/unitctl/unit-client-rs/src/unitd_docker.rs @@ -1,19 +1,16 @@ use std::collections::HashMap; use std::fs::read_to_string; -use std::path::{PathBuf, MAIN_SEPARATOR}; use std::io::stderr; +use std::path::{PathBuf, MAIN_SEPARATOR}; +use crate::control_socket_address::ControlSocket; use crate::futures::StreamExt; use crate::unit_client::UnitClientError; use crate::unitd_process::UnitdProcess; -use crate::control_socket_address::ControlSocket; use bollard::container::{Config, ListContainersOptions, StartContainerOptions}; use bollard::image::CreateImageOptions; -use bollard::models::{ - ContainerCreateResponse, HostConfig, Mount, - MountTypeEnum, ContainerSummary, -}; +use bollard::models::{ContainerCreateResponse, ContainerSummary, HostConfig, Mount, MountTypeEnum}; use bollard::secret::ContainerInspectResponse; use bollard::Docker; @@ -156,22 +153,18 @@ impl UnitdContainer { // cant do this functionally because of the async call let mut mapped = vec![]; for ctr in summary { - if unitd_command_re.is_match(&ctr.clone().command - .or(Some(String::new())) - .unwrap()) { - let mut c = UnitdContainer::from(&ctr); - if let Some(names) = ctr.names { - if names.len() > 0 { - let name = names[0].strip_prefix("/") - .or(Some(names[0].as_str())).unwrap(); - if let Ok(cir) = docker - .inspect_container(name, None).await { - c.details = Some(cir); - } + if unitd_command_re.is_match(&ctr.clone().command.or(Some(String::new())).unwrap()) { + let mut c = UnitdContainer::from(&ctr); + if let Some(names) = ctr.names { + if names.len() > 0 { + let name = names[0].strip_prefix("/").or(Some(names[0].as_str())).unwrap(); + if let Ok(cir) = docker.inspect_container(name, None).await { + c.details = Some(cir); } } - mapped.push(c); } + mapped.push(c); + } } mapped } @@ -196,11 +189,11 @@ impl UnitdContainer { // either return translated path or original prefixed with "container" if keys.len() > 0 { - let mut matches = self.mounts[&keys[0]] - .clone() - .join(cp.as_path() - .strip_prefix(keys[0].clone()) - .expect("error checking path prefix")); + let mut matches = self.mounts[&keys[0]].clone().join( + cp.as_path() + .strip_prefix(keys[0].clone()) + .expect("error checking path prefix"), + ); /* Observed on M1 Mac that Docker on OSX * adds a bunch of garbage to the mount path * converting it into a useless directory @@ -208,15 +201,14 @@ impl UnitdContainer { */ if cfg!(target_os = "macos") { let mut abs = PathBuf::from(String::from(MAIN_SEPARATOR)); - let m = matches.strip_prefix("/host_mnt/private") - .unwrap_or(matches.strip_prefix("/host_mnt") - .unwrap_or(matches.as_path())); + let m = matches + .strip_prefix("/host_mnt/private") + .unwrap_or(matches.strip_prefix("/host_mnt").unwrap_or(matches.as_path())); // make it absolute again abs.push(m); matches = abs; } - matches.to_string_lossy() - .to_string() + matches.to_string_lossy().to_string() } else { format!(":{}", cp.display()) } @@ -264,10 +256,7 @@ pub async fn deploy_new_container( let mut mounts = vec![]; // if a unix socket is specified, mounts its directory if socket.is_local_socket() { - let mount_path = PathBuf::from(socket.clone()) - .as_path() - .to_string_lossy() - .to_string(); + let mount_path = PathBuf::from(socket.clone()).as_path().to_string_lossy().to_string(); mounts.push(Mount { typ: Some(MountTypeEnum::BIND), source: Some(mount_path), @@ -290,22 +279,22 @@ pub async fn deploy_new_container( Some(CreateImageOptions { from_image: image.as_str(), ..Default::default() - }), None, None + }), + None, + None, ); while let Some(res) = stream.next().await { if let Ok(info) = res { if let Some(id) = info.id { if let Some(_) = totals.get_mut(&id) { - if let Some(delta) = info.progress_detail - .and_then(|detail| detail.current) { - pb.add(delta as u64); - } + if let Some(delta) = info.progress_detail.and_then(|detail| detail.current) { + pb.add(delta as u64); + } } else { - if let Some(total) = info.progress_detail - .and_then(|detail| detail.total) { - totals.insert(id, total); - pb.total += total as u64; - } + if let Some(total) = info.progress_detail.and_then(|detail| detail.total) { + totals.insert(id, total); + pb.total += total as u64; + } } } } @@ -357,27 +346,27 @@ pub async fn deploy_new_container( .await { // somehow our container doesnt exist - Err(e) => Err(UnitClientError::UnitdDockerError{ - message: e.to_string() - }), + Err(e) => Err(UnitClientError::UnitdDockerError { message: e.to_string() }), // here it is! Ok(info) => { if info.len() < 1 { return Err(UnitClientError::UnitdDockerError { message: "couldnt find new container".to_string(), }); - } else if info[0].names.is_none() || - info[0].names.clone().unwrap().len() < 1 { - return Err(UnitClientError::UnitdDockerError { - message: "new container has no name".to_string(), - }); - } + } else if info[0].names.is_none() || info[0].names.clone().unwrap().len() < 1 { + return Err(UnitClientError::UnitdDockerError { + message: "new container has no name".to_string(), + }); + } // start our container - match docker.start_container( - info[0].names.clone().unwrap()[0].strip_prefix(MAIN_SEPARATOR).unwrap(), - None::>, - ).await { + match docker + .start_container( + info[0].names.clone().unwrap()[0].strip_prefix(MAIN_SEPARATOR).unwrap(), + None::>, + ) + .await + { Err(err) => Err(UnitClientError::UnitdDockerError { message: err.to_string(), }), @@ -439,14 +428,8 @@ mod tests { ctr.host_path("/path/to/conf".to_string()) ); if cfg!(target_os = "macos") { - assert_eq!( - "/6/test".to_string(), - ctr.host_path("/var/test".to_string()) - ); - assert_eq!( - "/7/test".to_string(), - ctr.host_path("/var/var/test".to_string()) - ); + assert_eq!("/6/test".to_string(), ctr.host_path("/var/test".to_string())); + assert_eq!("/7/test".to_string(), ctr.host_path("/var/var/test".to_string())); } } -- cgit From a91b961d620dc17c98b998a9142050defe46b56e Mon Sep 17 00:00:00 2001 From: Ava Hahn Date: Wed, 31 Jul 2024 13:39:37 -0700 Subject: tools/unitctl: make application directory configurable * default behavior is now a read write application mount * use can specify a flag (-r) to mount app dir as read only Signed-off-by: Ava Hahn --- tools/unitctl/unit-client-rs/src/unitd_docker.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'tools/unitctl/unit-client-rs/src/unitd_docker.rs') diff --git a/tools/unitctl/unit-client-rs/src/unitd_docker.rs b/tools/unitctl/unit-client-rs/src/unitd_docker.rs index 0d318096..2b9e0c7d 100644 --- a/tools/unitctl/unit-client-rs/src/unitd_docker.rs +++ b/tools/unitctl/unit-client-rs/src/unitd_docker.rs @@ -249,6 +249,7 @@ impl UnitdContainer { pub async fn deploy_new_container( socket: ControlSocket, application: &String, + application_read_only: bool, image: &String, ) -> Result, UnitClientError> { match Docker::connect_with_local_defaults() { @@ -269,7 +270,7 @@ pub async fn deploy_new_container( typ: Some(MountTypeEnum::BIND), source: Some(application.clone()), target: Some("/www".to_string()), - read_only: Some(true), + read_only: Some(application_read_only), ..Default::default() }); -- cgit