diff options
author | mdecimus <mauro@stalw.art> | 2023-12-18 22:25:42 +0100 |
---|---|---|
committer | mdecimus <mauro@stalw.art> | 2023-12-18 22:25:42 +0100 |
commit | ea94de6d7705ca32949447a80b90633dcd4daf95 (patch) | |
tree | 8741fa310b11d118e13f061e10b2ca7a62b2203a /crates | |
parent | 566a2a0ab8f06bde4eb8086942be2b630086f7f2 (diff) |
CLI account management + Directory refactoring
Diffstat (limited to 'crates')
56 files changed, 3065 insertions, 1614 deletions
diff --git a/crates/cli/Cargo.toml b/crates/cli/Cargo.toml index 48aa53d1..3f19fdc6 100644 --- a/crates/cli/Cargo.toml +++ b/crates/cli/Cargo.toml @@ -27,3 +27,4 @@ csv = "1.1" form_urlencoded = "1.1.0" human-size = "0.4.2" futures = "0.3.28" +pwhash = "1.0.0" diff --git a/crates/cli/src/main.rs b/crates/cli/src/main.rs index 4e33500c..44fc6a09 100644 --- a/crates/cli/src/main.rs +++ b/crates/cli/src/main.rs @@ -29,17 +29,13 @@ use std::{ use clap::Parser; use console::style; -use jmap_client::client::{Client, Credentials}; +use jmap_client::client::Credentials; use modules::{ - cli::{Cli, Commands}, - database::cmd_database, - export::cmd_export, - get, - import::cmd_import, - is_localhost, post, - queue::cmd_queue, - report::cmd_report, + cli::{Cli, Client, Commands}, + is_localhost, UnwrapResult, }; +use reqwest::{header::AUTHORIZATION, Method, StatusCode}; +use serde::{de::DeserializeOwned, Deserialize, Serialize}; use crate::modules::OAuthResponse; @@ -48,56 +44,43 @@ pub mod modules; #[tokio::main] async fn main() -> std::io::Result<()> { let args = Cli::parse(); - let is_jmap = args.command.is_jmap(); - let credentials = if let Some(credentials) = args.credentials { - parse_credentials(&credentials) - } else { - let credentials = rpassword::prompt_password( - "\nEnter administrator credentials or press [ENTER] to use OAuth: ", - ) - .unwrap(); - if !credentials.is_empty() { + let client = Client { + credentials: if let Some(credentials) = args.credentials { parse_credentials(&credentials) } else { - oauth(&args.url).await - } + let credentials = rpassword::prompt_password( + "\nEnter administrator credentials or press [ENTER] to use OAuth: ", + ) + .unwrap(); + if !credentials.is_empty() { + parse_credentials(&credentials) + } else { + oauth(&args.url).await + } + }, + timeout: args.timeout, + url: args.url, }; - if is_jmap { - match args.command { - Commands::Import(command) => { - cmd_import(build_client(&args.url, credentials).await, command).await - } - Commands::Export(command) => { - cmd_export(build_client(&args.url, credentials).await, command).await - } - Commands::Database(command) => cmd_database(&args.url, credentials, command).await, - Commands::Queue(_) | Commands::Report(_) => unreachable!(), + match args.command { + Commands::Import(command) => { + command.exec(client).await; } - } else { - match args.command { - Commands::Queue(command) => cmd_queue(&args.url, credentials, command).await, - Commands::Report(command) => cmd_report(&args.url, credentials, command).await, - _ => unreachable!(), + Commands::Export(command) => { + command.exec(client).await; } + Commands::Database(command) => command.exec(client).await, + Commands::Account(_) => todo!(), + Commands::Domain(_) => todo!(), + Commands::List(_) => todo!(), + Commands::Group(_) => todo!(), + Commands::Queue(command) => command.exec(client).await, + Commands::Report(command) => command.exec(client).await, } Ok(()) } -async fn build_client(url: &str, credentials: Credentials) -> Client { - Client::new() - .credentials(credentials) - .accept_invalid_certs(is_localhost(url)) - .timeout(Duration::from_secs(60)) - .connect(url) - .await - .unwrap_or_else(|err| { - eprintln!("Failed to connect to JMAP server {}: {}.", url, err); - std::process::exit(1); - }) -} - fn parse_credentials(credentials: &str) -> Credentials { if let Some((account, secret)) = credentials.split_once(':') { Credentials::basic(account, secret) @@ -107,10 +90,39 @@ fn parse_credentials(credentials: &str) -> Credentials { } async fn oauth(url: &str) -> Credentials { - let metadata = get(&format!("{}/.well-known/oauth-authorization-server", url)).await; + let metadata: HashMap<String, serde_json::Value> = serde_json::from_slice( + &reqwest::Client::builder() + .danger_accept_invalid_certs(is_localhost(url)) + .build() + .unwrap_or_default() + .get(&format!("{}/.well-known/oauth-authorization-server", url)) + .send() + .await + .unwrap_result("send OAuth GET request") + .bytes() + .await + .unwrap_result("fetch bytes"), + ) + .unwrap_result("deserialize OAuth GET response"); + let token_endpoint = metadata.property("token_endpoint"); - let mut params = HashMap::from_iter([("client_id".to_string(), "Stalwart_CLI".to_string())]); - let response = post(metadata.property("device_authorization_endpoint"), ¶ms).await; + let mut params: HashMap<String, String> = + HashMap::from_iter([("client_id".to_string(), "Stalwart_CLI".to_string())]); + let response: HashMap<String, serde_json::Value> = serde_json::from_slice( + &reqwest::Client::builder() + .danger_accept_invalid_certs(is_localhost(url)) + .build() + .unwrap_or_default() + .post(metadata.property("device_authorization_endpoint")) + .form(¶ms) + .send() + .await + .unwrap_result("send OAuth POST request") + .bytes() + .await + .unwrap_result("fetch bytes"), + ) + .unwrap_result("deserialize OAuth POST response"); params.insert( "grant_type".to_string(), @@ -130,7 +142,22 @@ async fn oauth(url: &str) -> Credentials { std::io::stdout().flush().unwrap(); std::io::stdin().lock().lines().next(); - let mut response = post(token_endpoint, ¶ms).await; + let mut response: HashMap<String, serde_json::Value> = serde_json::from_slice( + &reqwest::Client::builder() + .danger_accept_invalid_certs(is_localhost(url)) + .build() + .unwrap_or_default() + .post(token_endpoint) + .form(¶ms) + .send() + .await + .unwrap_result("send OAuth POST request") + .bytes() + .await + .unwrap_result("fetch bytes"), + ) + .unwrap_result("deserialize OAuth POST response"); + if let Some(serde_json::Value::String(access_token)) = response.remove("access_token") { Credentials::Bearer(access_token) } else { @@ -144,3 +171,89 @@ async fn oauth(url: &str) -> Credentials { std::process::exit(1); } } + +#[derive(Deserialize)] +#[serde(untagged)] +pub enum Response<T> { + Data { data: T }, + Error { error: String, details: String }, +} + +impl Client { + pub async fn into_jmap_client(self) -> jmap_client::client::Client { + jmap_client::client::Client::new() + .credentials(self.credentials) + .accept_invalid_certs(is_localhost(&self.url)) + .timeout(Duration::from_secs(self.timeout.unwrap_or(60))) + .connect(&self.url) + .await + .unwrap_or_else(|err| { + eprintln!("Failed to connect to JMAP server {}: {}.", &self.url, err); + std::process::exit(1); + }) + } + + pub async fn http_request<R: DeserializeOwned, B: Serialize>( + &self, + method: Method, + url: &str, + body: Option<B>, + ) -> R { + let url = format!( + "{}{}{}", + self.url, + if !self.url.ends_with('/') && !url.starts_with('/') { + "/" + } else { + "" + }, + url + ); + let mut request = reqwest::Client::builder() + .danger_accept_invalid_certs(is_localhost(&url)) + .timeout(Duration::from_secs(self.timeout.unwrap_or(60))) + .build() + .unwrap_or_default() + .request(method, url) + .header( + AUTHORIZATION, + match &self.credentials { + Credentials::Basic(s) => format!("Basic {s}"), + Credentials::Bearer(s) => format!("Bearer {s}"), + }, + ); + + if let Some(body) = body { + request = request.body(serde_json::to_string(&body).unwrap_result("serialize body")); + } + + let response = request.send().await.unwrap_result("send HTTP request"); + + match response.status() { + StatusCode::OK => (), + StatusCode::UNAUTHORIZED => { + eprintln!("Authentication failed. Make sure the credentials are correct and that the account has administrator rights."); + std::process::exit(1); + } + _ => { + eprintln!( + "Request failed: {}", + response.text().await.unwrap_result("fetch text") + ); + std::process::exit(1); + } + } + + match serde_json::from_slice::<Response<R>>( + &response.bytes().await.unwrap_result("fetch bytes"), + ) + .unwrap_result("deserialize response") + { + Response::Data { data } => data, + Response::Error { error, details } => { + eprintln!("Request failed: {details} ({error:?})"); + std::process::exit(1); + } + } + } +} diff --git a/crates/cli/src/modules/account.rs b/crates/cli/src/modules/account.rs new file mode 100644 index 00000000..aa487a9f --- /dev/null +++ b/crates/cli/src/modules/account.rs @@ -0,0 +1,360 @@ +/* + * Copyright (c) 2020-2023, Stalwart Labs Ltd. + * + * This file is part of Stalwart Mail Server. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * in the LICENSE file at the top-level directory of this distribution. + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * + * You can be released from the requirements of the AGPLv3 license by + * purchasing a commercial license. Please contact licensing@stalw.art + * for more details. +*/ + +use std::fmt::Display; + +use prettytable::{Attr, Cell, Row, Table}; +use pwhash::sha512_crypt; +use reqwest::Method; +use serde_json::Value; + +use super::{ + cli::{AccountCommands, Client}, + Principal, PrincipalField, PrincipalUpdate, PrincipalValue, Type, +}; + +impl AccountCommands { + pub async fn exec(self, client: Client) { + match self { + AccountCommands::Create { + name, + password, + description, + quota, + is_admin, + addresses, + member_of, + } => { + let principal = Principal { + id: None, + typ: if is_admin.unwrap_or_default() { + Type::Superuser + } else { + Type::Individual + } + .into(), + quota, + used_quota: None, + name: name.clone().into(), + secrets: vec![sha512_crypt::hash(password).unwrap()], + emails: addresses.unwrap_or_default(), + member_of: member_of.unwrap_or_default(), + description, + }; + let account_id = client + .http_request::<u32, _>(Method::POST, "/admin/principal", Some(principal)) + .await; + eprintln!("Successfully created account {name:?} with id {account_id}."); + } + AccountCommands::Update { + name, + new_name, + password, + description, + quota, + is_admin, + addresses, + member_of, + } => { + let mut changes = Vec::new(); + if let Some(new_name) = new_name { + changes.push(PrincipalUpdate::set( + PrincipalField::Name, + PrincipalValue::String(new_name), + )); + } + if let Some(password) = password { + changes.push(PrincipalUpdate::set( + PrincipalField::Secrets, + PrincipalValue::StringList(vec![sha512_crypt::hash(password).unwrap()]), + )); + } + if let Some(description) = description { + changes.push(PrincipalUpdate::set( + PrincipalField::Description, + PrincipalValue::String(description), + )); + } + if let Some(quota) = quota { + changes.push(PrincipalUpdate::set( + PrincipalField::Quota, + PrincipalValue::Integer(quota), + )); + } + if let Some(is_admin) = is_admin { + changes.push(PrincipalUpdate::set( + PrincipalField::Type, + PrincipalValue::Type(if is_admin { + Type::Superuser + } else { + Type::Individual + }), + )); + } + if let Some(addresses) = addresses { + changes.push(PrincipalUpdate::set( + PrincipalField::Emails, + PrincipalValue::StringList(addresses), + )); + } + if let Some(member_of) = member_of { + changes.push(PrincipalUpdate::set( + PrincipalField::MemberOf, + PrincipalValue::StringList(member_of), + )); + } + + if !changes.is_empty() { + client + .http_request::<Value, _>( + Method::PATCH, + &format!("/admin/principal/{name}"), + Some(changes), + ) + .await; + eprintln!("Successfully updated account {name:?}."); + } else { + eprintln!("No changes to apply."); + } + } + AccountCommands::AddEmail { name, addresses } => { + client + .http_request::<Value, _>( + Method::PATCH, + &format!("/admin/principal/{name}"), + Some( + addresses + .into_iter() + .map(|address| { + PrincipalUpdate::add_item( + PrincipalField::Emails, + PrincipalValue::String(address), + ) + }) + .collect::<Vec<_>>(), + ), + ) + .await; + eprintln!("Successfully updated account {name:?}."); + } + AccountCommands::RemoveEmail { name, addresses } => { + client + .http_request::<Value, _>( + Method::PATCH, + &format!("/admin/principal/{name}"), + Some( + addresses + .into_iter() + .map(|address| { + PrincipalUpdate::remove_item( + PrincipalField::Emails, + PrincipalValue::String(address), + ) + }) + .collect::<Vec<_>>(), + ), + ) + .await; + eprintln!("Successfully updated account {name:?}."); + } + AccountCommands::AddToGroup { name, member_of } => { + client + .http_request::<Value, _>( + Method::PATCH, + &format!("/admin/principal/{name}"), + Some( + member_of + .into_iter() + .map(|group| { + PrincipalUpdate::add_item( + PrincipalField::MemberOf, + PrincipalValue::String(group), + ) + }) + .collect::<Vec<_>>(), + ), + ) + .await; + eprintln!("Successfully updated account {name:?}."); + } + AccountCommands::RemoveFromGroup { name, member_of } => { + client + .http_request::<Value, _>( + Method::PATCH, + &format!("/admin/principal/{name}"), + Some( + member_of + .into_iter() + .map(|group| { + PrincipalUpdate::remove_item( + PrincipalField::MemberOf, + PrincipalValue::String(group), + ) + }) + .collect::<Vec<_>>(), + ), + ) + .await; + eprintln!("Successfully updated account {name:?}."); + } + AccountCommands::Delete { name } => { + client + .http_request::<Value, String>( + Method::DELETE, + &format!("/admin/principal/{name}"), + None, + ) + .await; + eprintln!("Successfully deleted account {name:?}."); + } + AccountCommands::Display { name } => { + client.display_principal(&name).await; + } + AccountCommands::List { from, limit } => { + client + .list_principals("individual", "Account", from, limit) + .await; + } + } + } +} + +impl Client { + pub async fn display_principal(&self, name: &str) { + let principal = self + .http_request::<Principal, String>( + Method::GET, + &format!("/admin/principal/{name}"), + None, + ) + .await; + let mut table = Table::new(); + if let Some(name) = principal.name { + table.add_row(Row::new(vec![ + Cell::new("Name").with_style(Attr::Bold), + Cell::new(&name), + ])); + } + let is_list = if let Some(typ) = principal.typ { + table.add_row(Row::new(vec![ + Cell::new("Type").with_style(Attr::Bold), + Cell::new(&typ.to_string()), + ])); + matches!(typ, Type::List) + } else { + false + }; + if let Some(description) = principal.description { + table.add_row(Row::new(vec![ + Cell::new("Description").with_style(Attr::Bold), + Cell::new(&description), + ])); + } + if let Some(quota) = principal.quota { + table.add_row(Row::new(vec![ + Cell::new("Quota").with_style(Attr::Bold), + Cell::new("a.to_string()), + ])); + } + if let Some(used_quota) = principal.used_quota { + table.add_row(Row::new(vec![ + Cell::new("Used Quota").with_style(Attr::Bold), + Cell::new(&used_quota.to_string()), + ])); + } + if !principal.member_of.is_empty() { + table.add_row(Row::new(vec![ + Cell::new(if is_list { "List members" } else { "Member of" }) + .with_style(Attr::Bold), + Cell::new(&principal.member_of.join(", ")), + ])); + } + if !principal.emails.is_empty() { + table.add_row(Row::new(vec![ + Cell::new("E-mail address(es)").with_style(Attr::Bold), + Cell::new(&principal.emails.join(", ")), + ])); + } + eprintln!(); + table.printstd(); + eprintln!(); + } + + pub async fn list_principals( + &self, + record_type: &str, + record_name: &str, + from: Option<String>, + limit: Option<usize>, + ) { + let mut query = form_urlencoded::Serializer::new("/admin/principal?".to_string()); + + query.append_pair("type", record_type); + + if let Some(from) = &from { + query.append_pair("from", from); + } + if let Some(limit) = limit { + query.append_pair("limit", &limit.to_string()); + } + + let results = self + .http_request::<Vec<String>, String>(Method::GET, &query.finish(), None) + .await; + if !results.is_empty() { + let mut table = Table::new(); + table.add_row(Row::new(vec![ + Cell::new(&format!("{record_name} Name")).with_style(Attr::Bold) + ])); + + for domain in &results { + table.add_row(Row::new(vec![Cell::new(domain)])); + } + + eprintln!(); + table.printstd(); + eprintln!(); + } + + eprintln!( + "\n\n{} {}{} found.\n", + results.len(), + record_name.to_ascii_lowercase(), + if results.len() == 1 { "" } else { "s" } + ); + } +} + +impl Display for Type { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Type::Superuser => write!(f, "Superuser"), + Type::Individual => write!(f, "Individual"), + Type::Group => write!(f, "Group"), + Type::List => write!(f, "List"), + Type::Resource => write!(f, "Resource"), + Type::Location => write!(f, "Location"), + Type::Other => write!(f, "Other"), + } + } +} diff --git a/crates/cli/src/modules/cli.rs b/crates/cli/src/modules/cli.rs index 6f66cdbd..67e757d0 100644 --- a/crates/cli/src/modules/cli.rs +++ b/crates/cli/src/modules/cli.rs @@ -22,6 +22,7 @@ */ use clap::{Parser, Subcommand, ValueEnum}; +use jmap_client::client::Credentials; use mail_parser::DateTime; use serde::Deserialize; @@ -37,10 +38,29 @@ pub struct Cli { /// Authentication credentials #[clap(short, long)] pub credentials: Option<String>, + /// Connection timeout in seconds + #[clap(short, long)] + pub timeout: Option<u64>, } #[derive(Subcommand)] pub enum Commands { + /// Manage user accounts + #[clap(subcommand)] + Account(AccountCommands), + + /// Manage domains + #[clap(subcommand)] + Domain(DomainCommands), + + /// Manage mailing lists + #[clap(subcommand)] + List(ListCommands), + + /// Manage groups + #[clap(subcommand)] + Group(GroupCommands), + /// Import JMAP accounts and Maildir/mbox mailboxes #[clap(subcommand)] Import(ImportCommands), @@ -62,6 +82,274 @@ pub enum Commands { Report(ReportCommands), } +pub struct Client { + pub url: String, + pub credentials: Credentials, + pub timeout: Option<u64>, +} + +#[derive(Subcommand)] +pub enum AccountCommands { + /// Create a new user account + Create { + /// Login Name + name: String, + /// Password + password: String, + /// Account description + #[clap(short, long)] + description: Option<String>, + /// Quota in bytes + #[clap(short, long)] + quota: Option<u32>, + /// Whether the account is an administrator + #[clap(short, long)] + is_admin: Option<bool>, + /// E-mail addresses + #[clap(short, long)] + addresses: Option<Vec<String>>, + /// Groups this account is a member of + #[clap(short, long)] + member_of: Option<Vec<String>>, + }, + + /// Update an existing user account + Update { + /// Account login + name: String, + /// Rename account login + #[clap(short, long)] + new_name: Option<String>, + /// Update password + #[clap(short, long)] + password: Option<String>, + /// Update account description + #[clap(short, long)] + description: Option<String>, + /// Update quota in bytes + #[clap(short, long)] + quota: Option<u32>, + /// Whether the account is an administrator + #[clap(short, long)] + is_admin: Option<bool>, + /// Update e-mail addresses + #[clap(short, long)] + addresses: Option<Vec<String>>, + /// Update groups this account is a member of + #[clap(short, long)] + member_of: Option<Vec<String>>, + }, + + /// Add e-mail aliases to a user account + AddEmail { + /// Account login + name: String, + /// E-mail aliases to add + #[clap(required = true)] + addresses: Vec<String>, + }, + + /// Remove e-mail aliases to a user account + RemoveEmail { + /// Account login + name: String, + /// E-mail aliases to remove + #[clap(required = true)] + addresses: Vec<String>, + }, + + /// Add a user account to groups + AddToGroup { + /// Account login + name: String, + /// Groups to add + #[clap(required = true)] + member_of: Vec<String>, + }, + + /// Remove a user account from groups + RemoveFromGroup { + /// Account login + name: String, + /// Groups to remove + #[clap(required = true)] + member_of: Vec<String>, + }, + + /// Delete an existing user account + Delete { + /// Account name to delete + name: String, + }, + + /// Display an existing user account + Display { + /// Account name to display + name: String, + }, + + /// List all user accounts + List { + /// Starting point for listing accounts + from: Option<String>, + /// Maximum number of accounts to list + limit: Option<usize>, + }, +} + +#[derive(Subcommand)] +pub enum ListCommands { + /// Create a new mailing list + Create { + /// List Name + name: String, + /// List email address + email: String, + /// Description + #[clap(short, long)] + description: Option<String>, + /// Mailing list members + #[clap(short, long)] + members: Option<Vec<String>>, + }, + + /// Update an existing mailing list + Update { + /// List Name + name: String, + /// Rename list + new_name: Option<String>, + /// List email address + email: Option<String>, + /// Description + #[clap(short, long)] + description: Option<String>, + /// Mailing list members + #[clap(short, long)] + members: Option<Vec<String>>, + }, + + /// Add members to a mailing list + AddMembers { + /// List Name + name: String, + /// Members to add + #[clap(required = true)] + members: Vec<String>, + }, + + /// Remove members from a mailing list + RemoveMembers { + /// List Name + name: String, + /// Members to remove + #[clap(required = true)] + members: Vec<String>, + }, + + /// Display an existing mailing list + Display { + /// Mailing list to display + name: String, + }, + + /// List all mailing lists + List { + /// Starting point for listing mailing lists + from: Option<String>, + /// Maximum number of mailing lists to list + limit: Option<usize>, + }, +} + +#[derive(Subcommand)] +pub enum GroupCommands { + /// Create a group + Create { + /// Group Name + name: String, + /// Group email address + email: Option<String>, + /// Description + #[clap(short, long)] + description: Option<String>, + /// Groups that this group is a member of + #[clap(short, long)] + member_of: Option<Vec<String>>, + }, + + /// Update an existing group + Update { + /// Group Name + name: String, + /// Rename group + new_name: Option<String>, + /// Group email address + email: Option<String>, + /// Description + #[clap(short, long)] + description: Option<String>, + /// Update groups that this group is a member of + #[clap(short, long)] + member_of: Option<Vec<String>>, + }, + + /// Add a group to other groups + AddToGroup { + /// Group name + name: String, + /// Groups to add + #[clap(required = true)] + member_of: Vec<String>, + }, + + /// Remove a group account from groups + RemoveFromGroup { + /// Group name + name: String, + /// Groups to remove + #[clap(required = true)] + member_of: Vec<String>, + }, + + /// Display an existing group + Display { + /// Group name to display + name: String, + }, + + /// List all groups + List { + /// Starting point for listing groups + from: Option<String>, + /// Maximum number of groups to list + limit: Option<usize>, + }, +} + +#[derive(Subcommand)] +pub enum DomainCommands { + /// Create a new domain + Create { + /// Domain name to create + name: String, + }, + + /// Delete an existing domain + Delete { + /// Domain name to delete + name: String, + }, + + /// List all domains + List { + /// Starting point for listing domains + from: Option<String>, + /// Maximum number of domains to list + limit: Option<usize>, + }, +} + #[derive(Subcommand)] pub enum ImportCommands { /// Import messages and folders @@ -112,22 +400,8 @@ pub enum ExportCommands { #[derive(Subcommand)] pub enum DatabaseCommands { - /// Delete a JMAP account - Delete { - /// Account name to delete - account: String, - }, - /// Rename a JMAP account - Rename { - /// Account name to rename - account: String, - - /// New account name - new_account: String, - }, - - /// Purge expired blobs - Purge {}, + /// Perform database maintenance + Maintenance {}, } #[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Ord, ValueEnum)] @@ -243,12 +517,6 @@ pub enum ReportCommands { }, } -impl Commands { - pub fn is_jmap(&self) -> bool { - !matches!(self, Commands::Queue(_) | Commands::Report(_)) - } -} - #[derive(Debug, Copy, Clone, PartialEq, Eq, PartialOrd, Ord, ValueEnum, Deserialize)] pub enum ReportFormat { /// DMARC report diff --git a/crates/cli/src/modules/database.rs b/crates/cli/src/modules/database.rs index 77d80cfc..42c4c769 100644 --- a/crates/cli/src/modules/database.rs +++ b/crates/cli/src/modules/database.rs @@ -21,42 +21,20 @@ * for more details. */ -use jmap_client::client::Credentials; -use reqwest::header::AUTHORIZATION; +use reqwest::Method; +use serde_json::Value; -use super::{cli::DatabaseCommands, is_localhost, UnwrapResult}; +use super::cli::{Client, DatabaseCommands}; -pub async fn cmd_database(url: &str, credentials: Credentials, command: DatabaseCommands) { - let url = match command { - DatabaseCommands::Delete { account } => format!("{}/admin/account/delete/{}", url, account), - DatabaseCommands::Rename { - account, - new_account, - } => format!("{}/admin/account/rename/{}/{}", url, account, new_account), - DatabaseCommands::Purge {} => format!("{}/admin/blob/purge", url), - }; - - let response = reqwest::Client::builder() - .danger_accept_invalid_certs(is_localhost(&url)) - .build() - .unwrap_or_default() - .get(url) - .header( - AUTHORIZATION, - match credentials { - Credentials::Basic(s) => format!("Basic {s}"), - Credentials::Bearer(s) => format!("Bearer {s}"), - }, - ) - .send() - .await - .unwrap_result("send GET request"); - if response.status().is_success() { - eprintln!("Success."); - } else { - eprintln!( - "Request Failed: {}", - response.text().await.unwrap_result("fetch text") - ); +impl DatabaseCommands { + pub async fn exec(self, client: Client) { + match self { + DatabaseCommands::Maintenance {} => { + client + .http_request::<Value, String>(Method::GET, "/admin/store/maintenance", None) + .await; + eprintln!("Success."); + } + } } } diff --git a/crates/cli/src/modules/domain.rs b/crates/cli/src/modules/domain.rs new file mode 100644 index 00000000..f2fe7332 --- /dev/null +++ b/crates/cli/src/modules/domain.rs @@ -0,0 +1,98 @@ +/* + * Copyright (c) 2020-2023, Stalwart Labs Ltd. + * + * This file is part of Stalwart Mail Server. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * in the LICENSE file at the top-level directory of this distribution. + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * + * You can be released from the requirements of the AGPLv3 license by + * purchasing a commercial license. Please contact licensing@stalw.art + * for more details. +*/ + +use std::borrow::Cow; + +use prettytable::{Attr, Cell, Row, Table}; +use reqwest::Method; +use serde_json::Value; + +use super::cli::{Client, DomainCommands}; + +impl DomainCommands { + pub async fn exec(self, client: Client) { + match self { + DomainCommands::Create { name } => { + client + .http_request::<Value, String>( + Method::POST, + &format!("/admin/domain/{name}"), + None, + ) + .await; + eprintln!("Successfully created domain {name:?}"); + } + DomainCommands::Delete { name } => { + client + .http_request::<Value, String>( + Method::DELETE, + &format!("/admin/domain/{name}"), + None, + ) + .await; + eprintln!("Successfully deleted domain {name:?}"); + } + DomainCommands::List { from, limit } => { + let query = if from.is_none() && limit.is_none() { + Cow::Borrowed("/admin/domain") + } else { + let mut query = "/admin/domain?".to_string(); + if let Some(from) = &from { + query.push_str(&format!("from={from}")); + } + if let Some(limit) = limit { + query.push_str(&format!( + "{}limit={limit}", + if from.is_some() { "&" } else { "" } + )); + } + Cow::Owned(query) + }; + + let domains = client + .http_request::<Vec<String>, String>(Method::GET, query.as_ref(), None) + .await; + if !domains.is_empty() { + let mut table = Table::new(); + table.add_row(Row::new(vec![ + Cell::new("Domain Name").with_style(Attr::Bold) + ])); + + for domain in &domains { + table.add_row(Row::new(vec![Cell::new(domain)])); + } + + eprintln!(); + table.printstd(); + eprintln!(); + } + + eprintln!( + "\n\n{} domain{} found.\n", + domains.len(), + if domains.len() == 1 { "" } else { "s" } + ); + } + } + } +} diff --git a/crates/cli/src/modules/export.rs b/crates/cli/src/modules/export.rs index a59c9ede..4207e3b8 100644 --- a/crates/cli/src/modules/export.rs +++ b/crates/cli/src/modules/export.rs @@ -28,7 +28,6 @@ use std::{ use futures::{stream::FuturesUnordered, StreamExt}; use jmap_client::{ - client::Client, email::{self, Email}, identity::{self, Identity}, mailbox::{self, Mailbox}, @@ -40,102 +39,111 @@ use tokio::io::AsyncWriteExt; use crate::modules::RETRY_ATTEMPTS; -use super::{cli::ExportCommands, name_to_id, UnwrapResult}; +use super::{ + cli::{Client, ExportCommands}, + name_to_id, UnwrapResult, +}; -pub async fn cmd_export(mut client: Client, command: ExportCommands) { - match command { - ExportCommands::Account { - num_concurrent, - account, - path, - } => { - client.set_default_account_id(name_to_id(&client, &account).await); - let max_objects_in_get = client - .session() - .core_capabilities() - .map(|c| c.max_objects_in_get()) - .unwrap_or(500); - - // Create directory - let mut path = PathBuf::from(path); - if !path.is_dir() { - eprintln!("Directory {} does not exist.", path.display()); - std::process::exit(1); - } - path.push(&account); - if !path.is_dir() { - std::fs::create_dir(&path).unwrap_or_else(|_| { - eprintln!("Failed to create directory: {}", path.display()); +impl ExportCommands { + pub async fn exec(self, client: Client) { + let mut client = client.into_jmap_client().await; + match self { + ExportCommands::Account { + num_concurrent, + account, + path, + } => { + client.set_default_account_id(name_to_id(&client, &account).await); + let max_objects_in_get = client + .session() + .core_capabilities() + .map(|c| c.max_objects_in_get()) + .unwrap_or(500); + + // Create directory + let mut path = PathBuf::from(path); + if !path.is_dir() { + eprintln!("Directory {} does not exist.", path.display()); std::process::exit(1); - }); - } + } + path.push(&account); + if !path.is_dir() { + std::fs::create_dir(&path).unwrap_or_else(|_| { + eprintln!("Failed to create directory: {}", path.display()); + std::process::exit(1); + }); + } - // Export metadata - let mut blobs = Vec::new(); - export_mailboxes(&client, max_objects_in_get, &path).await; - export_emails(&client, max_objects_in_get, &mut blobs, &path).await; - export_sieve_scripts(&client, max_objects_in_get, &mut blobs, &path).await; - export_identities(&client, &path).await; - export_vacation_responses(&client, &path).await; - - // Export blobs - path.push("blobs"); - if !path.exists() { - std::fs::create_dir(&path).unwrap_or_else(|_| { - eprintln!("Failed to create directory: {}", path.display()); - std::process::exit(1); - }); - } - let client = Arc::new(client); - let num_concurrent = num_concurrent.unwrap_or_else(num_cpus::get); - let mut futures = FuturesUnordered::new(); - eprintln!("Exporting {} blobs...", blobs.len()); - for blob_id in blobs { - let client = client.clone(); - let mut blob_path = path.clone(); - blob_path.push(&blob_id); - - futures.push(async move { - let mut retry_count = 0; - - let bytes = loop { - match client.download(&blob_id).await { - Ok(bytes) => break bytes, - Err(_) if retry_count < RETRY_ATTEMPTS => { - tokio::time::sleep(std::time::Duration::from_secs(1)).await; - retry_count += 1; - } - result => { - result.unwrap_result("download blob"); - return; + // Export metadata + let mut blobs = Vec::new(); + export_mailboxes(&client, max_objects_in_get, &path).await; + export_emails(&client, max_objects_in_get, &mut blobs, &path).await; + export_sieve_scripts(&client, max_objects_in_get, &mut blobs, &path).await; + export_identities(&client, &path).await; + export_vacation_responses(&client, &path).await; + + // Export blobs + path.push("blobs"); + if !path.exists() { + std::fs::create_dir(&path).unwrap_or_else(|_| { + eprintln!("Failed to create directory: {}", path.display()); + std::process::exit(1); + }); + } + let client = Arc::new(client); + let num_concurrent = num_concurrent.unwrap_or_else(num_cpus::get); + let mut futures = FuturesUnordered::new(); + eprintln!("Exporting {} blobs...", blobs.len()); + for blob_id in blobs { + let client = client.clone(); + let mut blob_path = path.clone(); + blob_path.push(&blob_id); + + futures.push(async move { + let mut retry_count = 0; + + let bytes = loop { + match client.download(&blob_id).await { + Ok(bytes) => break bytes, + Err(_) if retry_count < RETRY_ATTEMPTS => { + tokio::time::sleep(std::time::Duration::from_secs(1)).await; + retry_count += 1; + } + result => { + result.unwrap_result("download blob"); + return; + } } - } - }; - - tokio::fs::OpenOptions::new() - .create(true) - .write(true) - .truncate(true) - .open(&blob_path) - .await - .unwrap_result(&format!("open {}", blob_path.display())) - .write_all(&bytes) - .await - .unwrap_result(&format!("write {}", blob_path.display())); - }); - - if futures.len() == num_concurrent { - futures.next().await.unwrap(); + }; + + tokio::fs::OpenOptions::new() + .create(true) + .write(true) + .truncate(true) + .open(&blob_path) + .await + .unwrap_result(&format!("open {}", blob_path.display())) + .write_all(&bytes) + .await + .unwrap_result(&format!("write {}", blob_path.display())); + }); + + if futures.len() == num_concurrent { + futures.next().await.unwrap(); + } } - } - // Wait for remaining futures - while futures.next().await.is_some() {} + // Wait for remaining futures + while futures.next().await.is_some() {} + } } } } -pub async fn fetch_mailboxes(client: &Client, max_objects_in_get: usize) -> Vec<Mailbox> { +pub async fn fetch_mailboxes( + client: &jmap_client::client::Client, + max_objects_in_get: usize, +) -> Vec<Mailbox> { let mut position = 0; let mut results = Vec::new(); loop { @@ -192,7 +200,11 @@ pub async fn fetch_mailboxes(client: &Client, max_objects_in_get: usize) -> Vec< results } -async fn export_mailboxes(client: &Client, max_objects_in_get: usize, path: &Path) { +async fn export_mailboxes( + client: &jmap_client::client::Client, + max_objects_in_get: usize, + path: &Path, +) { eprintln!( "Exported {} mailboxes.", write_file( @@ -204,7 +216,10 @@ async fn export_mailboxes(client: &Client, max_objects_in_get: usize, path: &Pat ); } -pub async fn fetch_emails(client: &Client, max_objects_in_get: usize) -> Vec<Email> { +pub async fn fetch_emails( + client: &jmap_client::client::Client, + max_objects_in_get: usize, +) -> Vec<Email> { let mut position = 0; let mut results = Vec::new(); @@ -263,7 +278,7 @@ pub async fn fetch_emails(client: &Client, max_objects_in_get: usize) -> Vec<Ema } async fn export_emails( - client: &Client, + client: &jmap_client::client::Client, max_objects_in_get: usize, blobs: &mut Vec<String>, path: &Path, @@ -287,7 +302,10 @@ async fn export_emails( ); } -pub async fn fetch_sieve_scripts(client: &Client, max_objects_in_get: usize) -> Vec<SieveScript> { +pub async fn fetch_sieve_scripts( + client: &jmap_client::client::Client, + max_objects_in_get: usize, +) -> Vec<SieveScript> { let mut position = 0; let mut results = Vec::new(); @@ -347,7 +365,7 @@ pub async fn fetch_sieve_scripts(client: &Client, max_objects_in_get: usize) -> } async fn export_sieve_scripts( - client: &Client, + client: &jmap_client::client::Client, max_objects_in_get: usize, blobs: &mut Vec<String>, path: &Path, @@ -370,7 +388,7 @@ async fn export_sieve_scripts( ); } -pub async fn fetch_identities(client: &Client) -> Vec<Identity> { +pub async fn fetch_identities(client: &jmap_client::client::Client) -> Vec<Identity> { let mut request = client.build(); request.get_identity().properties([ identity::Property::Id, @@ -388,14 +406,16 @@ pub async fn fetch_identities(client: &Client) -> Vec<Identity> { .take_list() } -async fn export_identities(client: &Client, path: &Path) { +async fn export_identities(client: &jmap_client::client::Client, path: &Path) { eprintln!( "Exported {} identities.", write_file(path, "identities.json", fetch_identities(client).await).await ); } -pub async fn fetch_vacation_responses(client: &Client) -> Vec<VacationResponse> { +pub async fn fetch_vacation_responses( + client: &jmap_client::client::Client, +) -> Vec<VacationResponse> { let mut request = client.build(); request.get_vacation_response().properties([ vacation_response::Property::Id, @@ -413,7 +433,7 @@ pub async fn fetch_vacation_responses(client: &Client) -> Vec<VacationResponse> .take_list() } -async fn export_vacation_responses(client: &Client, path: &Path) { +async fn export_vacation_responses(client: &jmap_client::client::Client, path: &Path) { eprintln!( "Exported {} vacation responses.", write_file( diff --git a/crates/cli/src/modules/group.rs b/crates/cli/src/modules/group.rs new file mode 100644 index 00000000..7e61c98a --- /dev/null +++ b/crates/cli/src/modules/group.rs @@ -0,0 +1,153 @@ +/* + * Copyright (c) 2020-2023, Stalwart Labs Ltd. + * + * This file is part of Stalwart Mail Server. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * in the LICENSE file at the top-level directory of this distribution. + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * + * You can be released from the requirements of the AGPLv3 license by + * purchasing a commercial license. Please contact licensing@stalw.art + * for more details. +*/ + +use reqwest::Method; +use serde_json::Value; + +use crate::modules::{Principal, Type}; + +use super::{ + cli::{Client, GroupCommands}, + PrincipalField, PrincipalUpdate, PrincipalValue, +}; + +impl GroupCommands { + pub async fn exec(self, client: Client) { + match self { + GroupCommands::Create { + name, + email, + description, + member_of, + } => { + let principal = Principal { + id: None, + typ: Some(Type::Group), + quota: None, + used_quota: None, + name: name.clone().into(), + secrets: vec![], + emails: email.map(|e| vec![e]).unwrap_or_default(), + member_of: member_of.unwrap_or_default(), + description, + }; + let account_id = client + .http_request::<u32, _>(Method::POST, "/admin/principal", Some(principal)) + .await; + eprintln!("Successfully created group {name:?} with id {account_id}."); + } + GroupCommands::Update { + name, + new_name, + email, + description, + member_of, + } => { + let mut changes = Vec::new(); + if let Some(new_name) = new_name { + changes.push(PrincipalUpdate::set( + PrincipalField::Name, + PrincipalValue::String(new_name), + )); + } + if let Some(email) = email { + changes.push(PrincipalUpdate::set( + PrincipalField::Emails, + PrincipalValue::StringList(vec![email]), + )); + } + if let Some(member_of) = member_of { + changes.push(PrincipalUpdate::set( + PrincipalField::MemberOf, + PrincipalValue::StringList(member_of), + )); + } + if let Some(description) = description { + changes.push(PrincipalUpdate::set( + PrincipalField::Description, + PrincipalValue::String(description), + )); + } + + if !changes.is_empty() { + client + .http_request::<Value, _>( + Method::PATCH, + &format!("/admin/principal/{name}"), + Some(changes), + ) + .await; + eprintln!("Successfully updated group {name:?}."); + } else { + eprintln!("No changes to apply."); + } + } + GroupCommands::AddToGroup { name, member_of } => { + client + .http_request::<Value, _>( + Method::PATCH, + &format!("/admin/principal/{name}"), + Some( + member_of + .into_iter() + .map(|group| { + PrincipalUpdate::add_item( + PrincipalField::MemberOf, + PrincipalValue::String(group), + ) + }) + .collect::<Vec<_>>(), + ), + ) + .await; + eprintln!("Successfully updated group {name:?}."); + } + GroupCommands::RemoveFromGroup { name, member_of } => { + client + .http_request::<Value, _>( + Method::PATCH, + &format!("/admin/principal/{name}"), + Some( + member_of + .into_iter() + .map(|group| { + PrincipalUpdate::remove_item( + PrincipalField::MemberOf, + PrincipalValue::String(group), + ) + }) + .collect::<Vec<_>>(), + ), + ) + .await; + eprintln!("Successfully updated group {name:?}."); + } + GroupCommands::Display { name } => { + client.display_principal(&name).await; + } + GroupCommands::List { from, limit } => { + client.list_principals("group", "Group", from, limit).await; + } + } + } +} diff --git a/crates/cli/src/modules/import.rs b/crates/cli/src/modules/import.rs index 5869d120..af586d48 100644 --- a/crates/cli/src/modules/import.rs +++ b/crates/cli/src/modules/import.rs @@ -35,7 +35,6 @@ use console::style; use futures::{stream::FuturesUnordered, StreamExt}; use indicatif::{MultiProgress, ProgressBar, ProgressStyle}; use jmap_client::{ - client::Client, core::set::SetObject, mailbox::{self, Role}, }; @@ -49,7 +48,7 @@ use tokio::{fs::File, io::AsyncReadExt}; use crate::modules::{name_to_id, UnwrapResult, RETRY_ATTEMPTS}; use super::{ - cli::{ImportCommands, MailboxFormat}, + cli::{Client, ImportCommands, MailboxFormat}, export::{ fetch_emails, fetch_identities, fetch_mailboxes, fetch_sieve_scripts, fetch_vacation_responses, @@ -77,375 +76,381 @@ struct Message { internal_date: u64, contents: Vec<u8>, } +impl ImportCommands { + pub async fn exec(self, client: Client) { + let mut client = client.into_jmap_client().await; -pub async fn cmd_import(mut client: Client, command: ImportCommands) { - match command { - ImportCommands::Messages { - num_concurrent, - format, - account, - path, - } => { - client.set_default_account_id(name_to_id(&client, &account).await); - let mut create_mailboxes = Vec::new(); - let mut create_mailbox_names = Vec::new(); - let mut create_mailbox_ids = Vec::new(); - - eprintln!("{} Parsing mailbox...", style("[1/4]").bold().dim(),); - - match format { - MailboxFormat::Mbox => { - create_mailbox_names.push(Vec::new()); - create_mailboxes.push(Mailbox::Mbox(MessageIterator::new(Cursor::new( - read_file(&path), - )))); - } - MailboxFormat::Maildir | MailboxFormat::MaildirNested => { - let (folder_sep, folder_split) = if format == MailboxFormat::Maildir { - (Some("."), ".") - } else { - (None, "/") - }; - - for folder in maildir::FolderIterator::new(path, folder_sep) - .unwrap_result("read Maildir folder") - { - let folder = folder.unwrap_result("read Maildir folder"); - if let Some(folder_name) = folder.name() { - let mut folder_parts = Vec::new(); - for folder_name in folder_name.split(folder_split) { - let mut folder_name = folder_name.trim(); - if folder_name.is_empty() { - folder_name = "."; - } - folder_parts.push(folder_name.to_string()); - if !create_mailbox_names.contains(&folder_parts) { - create_mailboxes.push(Mailbox::None); - create_mailbox_names.push(folder_parts.clone()); - } - } - - *create_mailboxes.last_mut().unwrap() = Mailbox::Maildir(folder); + match self { + ImportCommands::Messages { + num_concurrent, + format, + account, + path, + } => { + client.set_default_account_id(name_to_id(&client, &account).await); + let mut create_mailboxes = Vec::new(); + let mut create_mailbox_names = Vec::new(); + let mut create_mailbox_ids = Vec::new(); + + eprintln!("{} Parsing mailbox...", style("[1/4]").bold().dim(),); + + match format { + MailboxFormat::Mbox => { + create_mailbox_names.push(Vec::new()); + create_mailboxes.push(Mailbox::Mbox(MessageIterator::new(Cursor::new( + read_file(&path), + )))); + } + MailboxFormat::Maildir | MailboxFormat::MaildirNested => { + let (folder_sep, folder_split) = if format == MailboxFormat::Maildir { + (Some("."), ".") } else { - create_mailboxes.push(Mailbox::Maildir(folder)); - create_mailbox_names.push(Vec::new()); + (None, "/") }; + + for folder in maildir::FolderIterator::new(path, folder_sep) + .unwrap_result("read Maildir folder") + { + let folder = folder.unwrap_result("read Maildir folder"); + if let Some(folder_name) = folder.name() { + let mut folder_parts = Vec::new(); + for folder_name in folder_name.split(folder_split) { + let mut folder_name = folder_name.trim(); + if folder_name.is_empty() { + folder_name = "."; + } + folder_parts.push(folder_name.to_string()); + if !create_mailbox_names.contains(&folder_parts) { + create_mailboxes.push(Mailbox::None); + create_mailbox_names.push(folder_parts.clone()); + } + } + + *create_mailboxes.last_mut().unwrap() = Mailbox::Maildir(folder); + } else { + create_mailboxes.push(Mailbox::Maildir(folder)); + create_mailbox_names.push(Vec::new()); + }; + } } } - } - // Fetch all mailboxes for the account - eprintln!( - "{} Fetching existing mailboxes for account...", - style("[2/4]").bold().dim(), - ); + // Fetch all mailboxes for the account + eprintln!( + "{} Fetching existing mailboxes for account...", + style("[2/4]").bold().dim(), + ); - let mut inbox_id = None; - let mut mailbox_ids = HashMap::new(); - let mut children: HashMap<Option<&str>, Vec<&str>> = - HashMap::from_iter([(None, Vec::new())]); - let mut request = client.build(); - request.get_mailbox().properties([ - mailbox::Property::Name, - mailbox::Property::ParentId, - mailbox::Property::Role, - mailbox::Property::Id, - ]); - let response = request - .send_get_mailbox() - .await - .unwrap_result("fetch mailboxes"); - for mailbox in response.list() { - let mailbox_id = mailbox.id().unwrap(); - if mailbox.role() == Role::Inbox { - inbox_id = mailbox_id.into(); + let mut inbox_id = None; + let mut mailbox_ids = HashMap::new(); + let mut children: HashMap<Option<&str>, Vec<&str>> = + HashMap::from_iter([(None, Vec::new())]); + let mut request = client.build(); + request.get_mailbox().properties([ + mailbox::Property::Name, + mailbox::Property::ParentId, + mailbox::Property::Role, + mailbox::Property::Id, + ]); + let response = request + .send_get_mailbox() + .await + .unwrap_result("fetch mailboxes"); + for mailbox in response.list() { + let mailbox_id = mailbox.id().unwrap(); + if mailbox.role() == Role::Inbox { + inbox_id = mailbox_id.into(); + } + children + .entry(mailbox.parent_id()) + .or_insert_with(Vec::new) + .push(mailbox_id); + mailbox_ids.insert(mailbox_id, mailbox.name().unwrap_or("Untitled")); } - children - .entry(mailbox.parent_id()) - .or_insert_with(Vec::new) - .push(mailbox_id); - mailbox_ids.insert(mailbox_id, mailbox.name().unwrap_or("Untitled")); - } - let inbox_id = - inbox_id.unwrap_result("locate Inbox on account, please check the server logs."); - let mut it = children.get(&None).unwrap().iter(); - let mut it_stack = Vec::new(); - let mut name_stack = Vec::new(); - let mut mailbox_names = HashMap::with_capacity(mailbox_ids.len()); - - // Build mailbox hierarchy on the server - eprintln!( - "{} Creating missing mailboxes...", - style("[3/4]").bold().dim(), - ); + let inbox_id = inbox_id + .unwrap_result("locate Inbox on account, please check the server logs."); + let mut it = children.get(&None).unwrap().iter(); + let mut it_stack = Vec::new(); + let mut name_stack = Vec::new(); + let mut mailbox_names = HashMap::with_capacity(mailbox_ids.len()); + + // Build mailbox hierarchy on the server + eprintln!( + "{} Creating missing mailboxes...", + style("[3/4]").bold().dim(), + ); - loop { - while let Some(mailbox_id) = it.next() { - let name = mailbox_ids[mailbox_id]; - let mut mailbox_name = name_stack.clone(); - mailbox_name.push(name.to_string()); - - mailbox_names.insert(mailbox_name, mailbox_id); - if let Some(next_it) = children.get(&Some(mailbox_id)).map(|c| c.iter()) { - name_stack.push(name.to_string()); - it_stack.push(it); - it = next_it; + loop { + while let Some(mailbox_id) = it.next() { + let name = mailbox_ids[mailbox_id]; + let mut mailbox_name = name_stack.clone(); + mailbox_name.push(name.to_string()); + + mailbox_names.insert(mailbox_name, mailbox_id); + if let Some(next_it) = children.get(&Some(mailbox_id)).map(|c| c.iter()) { + name_stack.push(name.to_string()); + it_stack.push(it); + it = next_it; + } } - } - if let Some(prev_it) = it_stack.pop() { - name_stack.pop(); - it = prev_it; - } else { - break; + if let Some(prev_it) = it_stack.pop() { + name_stack.pop(); + it = prev_it; + } else { + break; + } } - } - // Check whether the mailboxes to be created already exist - let mut has_missing_mailboxes = false; - for mailbox_name in &create_mailbox_names { - create_mailbox_ids.push(if !mailbox_name.is_empty() { - if let Some(mailbox_id) = mailbox_names.get(mailbox_name) { - MailboxId::ExistingId(mailbox_id) + // Check whether the mailboxes to be created already exist + let mut has_missing_mailboxes = false; + for mailbox_name in &create_mailbox_names { + create_mailbox_ids.push(if !mailbox_name.is_empty() { + if let Some(mailbox_id) = mailbox_names.get(mailbox_name) { + MailboxId::ExistingId(mailbox_id) + } else { + has_missing_mailboxes = true; + MailboxId::None + } } else { - has_missing_mailboxes = true; - MailboxId::None - } - } else { - MailboxId::ExistingId(inbox_id) - }); - } + MailboxId::ExistingId(inbox_id) + }); + } - // Create any missing mailboxes - if has_missing_mailboxes { - let mut request = client.build(); - let set_request = request.set_mailbox(); - - for pos in 0..create_mailbox_ids.len() { - if let MailboxId::None = create_mailbox_ids[pos] { - let mailbox_name = &create_mailbox_names[pos]; - let create_request = - set_request.create().name(mailbox_name.last().unwrap()); - - if mailbox_name.len() > 1 { - let parent_mailbox_name = &mailbox_name[..mailbox_name.len() - 1]; - let parent_mailbox_pos = create_mailbox_names - .iter() - .position(|n| n == parent_mailbox_name) - .unwrap(); - match &create_mailbox_ids[parent_mailbox_pos] { - MailboxId::ExistingId(id) => { - create_request.parent_id((*id).into()); - } - MailboxId::CreateId(id_ref) => { - create_request.parent_id_ref(id_ref); + // Create any missing mailboxes + if has_missing_mailboxes { + let mut request = client.build(); + let set_request = request.set_mailbox(); + + for pos in 0..create_mailbox_ids.len() { + if let MailboxId::None = create_mailbox_ids[pos] { + let mailbox_name = &create_mailbox_names[pos]; + let create_request = + set_request.create().name(mailbox_name.last().unwrap()); + + if mailbox_name.len() > 1 { + let parent_mailbox_name = &mailbox_name[..mailbox_name.len() - 1]; + let parent_mailbox_pos = create_mailbox_names + .iter() + .position(|n| n == parent_mailbox_name) + .unwrap(); + match &create_mailbox_ids[parent_mailbox_pos] { + MailboxId::ExistingId(id) => { + create_request.parent_id((*id).into()); + } + MailboxId::CreateId(id_ref) => { + create_request.parent_id_ref(id_ref); + } + MailboxId::None => unreachable!(), } - MailboxId::None => unreachable!(), + } else { + create_request.parent_id(None::<String>); } - } else { - create_request.parent_id(None::<String>); + create_mailbox_ids[pos] = + MailboxId::CreateId(create_request.create_id().unwrap()); } - create_mailbox_ids[pos] = - MailboxId::CreateId(create_request.create_id().unwrap()); } - } - // Create mailboxes - let mut response = request - .send_set_mailbox() - .await - .unwrap_result("create mailboxes"); - for create_mailbox_id in create_mailbox_ids.iter_mut() { - if let MailboxId::CreateId(id) = create_mailbox_id { - *id = response - .created(id) - .unwrap_result("create mailbox") - .take_id(); + // Create mailboxes + let mut response = request + .send_set_mailbox() + .await + .unwrap_result("create mailboxes"); + for create_mailbox_id in create_mailbox_ids.iter_mut() { + if let MailboxId::CreateId(id) = create_mailbox_id { + *id = response + .created(id) + .unwrap_result("create mailbox") + .take_id(); + } } } - } - - // Import messages - eprintln!("{} Importing messages...", style("[4/4]").bold().dim(),); - - let client = Arc::new(client); - let total_imported = Arc::new(AtomicUsize::from(0)); - let m = MultiProgress::new(); - let num_concurrent = num_concurrent.unwrap_or_else(num_cpus::get); - let spinner_style = - ProgressStyle::with_template("{prefix:.bold.dim} {spinner} {wide_msg}") - .unwrap() - .tick_chars("⠁⠂⠄⡀⢀⠠⠐⠈ "); - let pbs = Arc::new(Mutex::new(( - (0..num_concurrent) - .map(|n| { - let pb = m.add(ProgressBar::new(40)); - pb.set_style(spinner_style.clone()); - pb.set_prefix(format!("[{}/?]", n + 1)); - pb - }) - .collect::<Vec<_>>(), - 0usize, - ))); - let failures = Arc::new(Mutex::new(Vec::new())); - let mut message_num = 0; - - for ((mut mailbox, mailbox_id), mailbox_name) in create_mailboxes - .into_iter() - .zip(create_mailbox_ids) - .zip(create_mailbox_names) - { - let mut futures = FuturesUnordered::new(); - let mailbox_id = Arc::new(match mailbox_id { - MailboxId::ExistingId(id) => id.to_string(), - MailboxId::CreateId(id) => id, - MailboxId::None => unreachable!(), - }); - let mailbox_name = Arc::new(if !mailbox_name.is_empty() { - mailbox_name.join("/") - } else { - "Inbox".to_string() - }); - - for result in mailbox.by_ref() { - match result { - Ok(message) => { - message_num += 1; - let client = client.clone(); - let mailbox_id = mailbox_id.clone(); - let mailbox_name = mailbox_name.clone(); - let total_imported = total_imported.clone(); - let pbs = pbs.clone(); - let failures = failures.clone(); - - futures.push(async move { - // Update progress bar - { - let mut pbs = pbs.lock().unwrap(); - let pb = &pbs.0[pbs.1 % pbs.0.len()]; - pb.set_message(format!( - "Importing {}: {}/{}", - message_num, mailbox_name, message.identifier - )); - pb.inc(1); - pbs.1 += 1; - } - let mut retry_count = 0; - loop { - match client - .email_import( - message.contents.clone(), - [mailbox_id.as_ref()], - if !message.flags.is_empty() { - message - .flags - .iter() - .map(|f| match f { - maildir::Flag::Passed => "$passed", - maildir::Flag::Replied => "$answered", - maildir::Flag::Seen => "$seen", - maildir::Flag::Trashed => "$deleted", - maildir::Flag::Draft => "$draft", - maildir::Flag::Flagged => "$flagged", - }) - .into() - } else { - None - }, - if message.internal_date > 0 { - (message.internal_date as i64).into() - } else { - None - }, - ) - .await + // Import messages + eprintln!("{} Importing messages...", style("[4/4]").bold().dim(),); + + let client = Arc::new(client); + let total_imported = Arc::new(AtomicUsize::from(0)); + let m = MultiProgress::new(); + let num_concurrent = num_concurrent.unwrap_or_else(num_cpus::get); + let spinner_style = + ProgressStyle::with_template("{prefix:.bold.dim} {spinner} {wide_msg}") + .unwrap() + .tick_chars("⠁⠂⠄⡀⢀⠠⠐⠈ "); + let pbs = Arc::new(Mutex::new(( + (0..num_concurrent) + .map(|n| { + let pb = m.add(ProgressBar::new(40)); + pb.set_style(spinner_style.clone()); + pb.set_prefix(format!("[{}/?]", n + 1)); + pb + }) + .collect::<Vec<_>>(), + 0usize, + ))); + let failures = Arc::new(Mutex::new(Vec::new())); + let mut message_num = 0; + + for ((mut mailbox, mailbox_id), mailbox_name) in create_mailboxes + .into_iter() + .zip(create_mailbox_ids) + .zip(create_mailbox_names) + { + let mut futures = FuturesUnordered::new(); + let mailbox_id = Arc::new(match mailbox_id { + MailboxId::ExistingId(id) => id.to_string(), + MailboxId::CreateId(id) => id, + MailboxId::None => unreachable!(), + }); + let mailbox_name = Arc::new(if !mailbox_name.is_empty() { + mailbox_name.join("/") + } else { + "Inbox".to_string() + }); + + for result in mailbox.by_ref() { + match result { + Ok(message) => { + message_num += 1; + let client = client.clone(); + let mailbox_id = mailbox_id.clone(); + let mailbox_name = mailbox_name.clone(); + let total_imported = total_imported.clone(); + let pbs = pbs.clone(); + let failures = failures.clone(); + + futures.push(async move { + // Update progress bar { - Ok(_) => { - total_imported.fetch_add(1, Ordering::Relaxed); - } - Err(_) if retry_count < RETRY_ATTEMPTS => { - retry_count += 1; - continue; - } - Err(err) => { - failures.lock().unwrap().push(format!( - concat!( - "Failed to import message {} ", - "with identifier '{}': {}" - ), - message_num, message.identifier, err - )); + let mut pbs = pbs.lock().unwrap(); + let pb = &pbs.0[pbs.1 % pbs.0.len()]; + pb.set_message(format!( + "Importing {}: {}/{}", + message_num, mailbox_name, message.identifier + )); + pb.inc(1); + pbs.1 += 1; + } + + let mut retry_count = 0; + loop { + match client + .email_import( + message.contents.clone(), + [mailbox_id.as_ref()], + if !message.flags.is_empty() { + message + .flags + .iter() + .map(|f| match f { + maildir::Flag::Passed => "$passed", + maildir::Flag::Replied => "$answered", + maildir::Flag::Seen => "$seen", + maildir::Flag::Trashed => "$deleted", + maildir::Flag::Draft => "$draft", + maildir::Flag::Flagged => "$flagged", + }) + .into() + } else { + None + }, + if message.internal_date > 0 { + (message.internal_date as i64).into() + } else { + None + }, + ) + .await + { + Ok(_) => { + total_imported.fetch_add(1, Ordering::Relaxed); + } + Err(_) if retry_count < RETRY_ATTEMPTS => { + retry_count += 1; + continue; + } + Err(err) => { + failures.lock().unwrap().push(format!( + concat!( + "Failed to import message {} ", + "with identifier '{}': {}" + ), + message_num, message.identifier, err + )); + } } + break; } - break; - } - }); + }); - if futures.len() == num_concurrent { - futures.next().await.unwrap(); + if futures.len() == num_concurrent { + futures.next().await.unwrap(); + } + } + Err(e) => { + failures + .lock() + .unwrap() + .push(format!("I/O error reading message: {}", e)); } - } - Err(e) => { - failures - .lock() - .unwrap() - .push(format!("I/O error reading message: {}", e)); } } - } - - // Wait for remaining futures - while futures.next().await.is_some() {} - } - // Done - for pb in pbs.lock().unwrap().0.iter() { - pb.finish_with_message("Done"); - } - let failures = failures.lock().unwrap(); - eprintln!( - "\n\nSuccessfully imported {} messages.\n", - total_imported.load(Ordering::Relaxed) - ); + // Wait for remaining futures + while futures.next().await.is_some() {} + } - if !failures.is_empty() { - eprintln!("There were {} failures:\n", failures.len()); - for failure in failures.iter() { - eprintln!("{}", failure); + // Done + for pb in pbs.lock().unwrap().0.iter() { + pb.finish_with_message("Done"); } - } - } + let failures = failures.lock().unwrap(); + eprintln!( + "\n\nSuccessfully imported {} messages.\n", + total_imported.load(Ordering::Relaxed) + ); - ImportCommands::Account { - num_concurrent, - account, - path, - } => { - client.set_default_account_id(name_to_id(&client, &account).await); - let path = PathBuf::from(path); - if !path.exists() { - eprintln!("Path '{}' does not exist.", path.display()); - return; + if !failures.is_empty() { + eprintln!("There were {} failures:\n", failures.len()); + for failure in failures.iter() { + eprintln!("{}", failure); + } + } } - let num_concurrent = num_concurrent.unwrap_or_else(num_cpus::get); - // Import objects - import_emails( - &client, - &path, - import_mailboxes(&client, &path).await.into(), + ImportCommands::Account { num_concurrent, - ) - .await; - import_sieve_scripts(&client, &path, num_concurrent).await; - import_identities(&client, &path).await; - import_vacation_responses(&client, &path).await; + account, + path, + } => { + client.set_default_account_id(name_to_id(&client, &account).await); + let path = PathBuf::from(path); + if !path.exists() { + eprintln!("Path '{}' does not exist.", path.display()); + return; + } + let num_concurrent = num_concurrent.unwrap_or_else(num_cpus::get); + + // Import objects + import_emails( + &client, + &path, + import_mailboxes(&client, &path).await.into(), + num_concurrent, + ) + .await; + import_sieve_scripts(&client, &path, num_concurrent).await; + import_identities(&client, &path).await; + import_vacation_responses(&client, &path).await; + } } } } -async fn import_mailboxes(client: &Client, path: &Path) -> HashMap<String, String> { +async fn import_mailboxes( + client: &jmap_client::client::Client, + path: &Path, +) -> HashMap<String, String> { // Deserialize mailboxes let mailboxes = read_json::<jmap_client::mailbox::Mailbox>(path, "mailboxes.json").await; if mailboxes.is_empty() { @@ -561,7 +566,7 @@ async fn import_mailboxes(client: &Client, path: &Path) -> HashMap<String, Strin } async fn import_emails( - client: &Client, + client: &jmap_client::client::Client, path: &Path, mailbox_ids: Arc<HashMap<String, String>>, num_concurrent: usize, @@ -706,7 +711,11 @@ async fn import_emails( ); } -async fn import_sieve_scripts(client: &Client, path: &Path, num_concurrent: usize) { +async fn import_sieve_scripts( + client: &jmap_client::client::Client, + path: &Path, + num_concurrent: usize, +) { // Deserialize scripts let scripts = read_json::<jmap_client::sieve::SieveScript>(path, "sieve.json").await; if scripts.is_empty() { @@ -812,7 +821,7 @@ async fn import_sieve_scripts(client: &Client, path: &Path, num_concurrent: usiz ); } -async fn import_identities(client: &Client, path: &Path) { +async fn import_identities(client: &jmap_client::client::Client, path: &Path) { // Deserialize mailboxes let identities = read_json::<jmap_client::identity::Identity>(path, "identities.json").await; if identities.is_empty() { @@ -886,7 +895,7 @@ async fn import_identities(client: &Client, path: &Path) { ); } -async fn import_vacation_responses(client: &Client, path: &Path) { +async fn import_vacation_responses(client: &jmap_client::client::Client, path: &Path) { // Deserialize mailboxes let vacation_responses = read_json::<jmap_client::vacation_response::VacationResponse>(path, "vacation.json").await; diff --git a/crates/cli/src/modules/list.rs b/crates/cli/src/modules/list.rs new file mode 100644 index 00000000..6b2c241b --- /dev/null +++ b/crates/cli/src/modules/list.rs @@ -0,0 +1,155 @@ +/* + * Copyright (c) 2020-2023, Stalwart Labs Ltd. + * + * This file is part of Stalwart Mail Server. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * in the LICENSE file at the top-level directory of this distribution. + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * + * You can be released from the requirements of the AGPLv3 license by + * purchasing a commercial license. Please contact licensing@stalw.art + * for more details. +*/ + +use reqwest::Method; +use serde_json::Value; + +use crate::modules::{Principal, Type}; + +use super::{ + cli::{Client, ListCommands}, + PrincipalField, PrincipalUpdate, PrincipalValue, +}; + +impl ListCommands { + pub async fn exec(self, client: Client) { + match self { + ListCommands::Create { + name, + email, + description, + members, + } => { + let principal = Principal { + id: None, + typ: Some(Type::List), + quota: None, + used_quota: None, + name: name.clone().into(), + secrets: vec![], + emails: vec![email], + member_of: members.unwrap_or_default(), + description, + }; + let account_id = client + .http_request::<u32, _>(Method::POST, "/admin/principal", Some(principal)) + .await; + eprintln!("Successfully created mailing list {name:?} with id {account_id}."); + } + ListCommands::Update { + name, + new_name, + email, + description, + members, + } => { + let mut changes = Vec::new(); + if let Some(new_name) = new_name { + changes.push(PrincipalUpdate::set( + PrincipalField::Name, + PrincipalValue::String(new_name), + )); + } + if let Some(email) = email { + changes.push(PrincipalUpdate::set( + PrincipalField::Emails, + PrincipalValue::StringList(vec![email]), + )); + } + if let Some(members) = members { + changes.push(PrincipalUpdate::set( + PrincipalField::MemberOf, + PrincipalValue::StringList(members), + )); + } + if let Some(description) = description { + changes.push(PrincipalUpdate::set( + PrincipalField::Description, + PrincipalValue::String(description), + )); + } + + if !changes.is_empty() { + client + .http_request::<Value, _>( + Method::PATCH, + &format!("/admin/principal/{name}"), + Some(changes), + ) + .await; + eprintln!("Successfully updated mailing list {name:?}."); + } else { + eprintln!("No changes to apply."); + } + } + ListCommands::AddMembers { name, members } => { + client + .http_request::<Value, _>( + Method::PATCH, + &format!("/admin/principal/{name}"), + Some( + members + .into_iter() + .map(|group| { + PrincipalUpdate::add_item( + PrincipalField::MemberOf, + PrincipalValue::String(group), + ) + }) + .collect::<Vec<_>>(), + ), + ) + .await; + eprintln!("Successfully updated mailing list {name:?}."); + } + ListCommands::RemoveMembers { name, members } => { + client + .http_request::<Value, _>( + Method::PATCH, + &format!("/admin/principal/{name}"), + Some( + members + .into_iter() + .map(|group| { + PrincipalUpdate::remove_item( + PrincipalField::MemberOf, + PrincipalValue::String(group), + ) + }) + .collect::<Vec<_>>(), + ), + ) + .await; + eprintln!("Successfully updated mailing list {name:?}."); + } + ListCommands::Display { name } => { + client.display_principal(&name).await; + } + ListCommands::List { from, limit } => { + client + .list_principals("list", "Mailing List", from, limit) + .await; + } + } + } +} diff --git a/crates/cli/src/modules/mod.rs b/crates/cli/src/modules/mod.rs index 00dbc37d..f892a827 100644 --- a/crates/cli/src/modules/mod.rs +++ b/crates/cli/src/modules/mod.rs @@ -30,16 +30,130 @@ use jmap_client::{ Property, }, }; +use serde::{Deserialize, Serialize}; +pub mod account; pub mod cli; pub mod database; +pub mod domain; pub mod export; +pub mod group; pub mod import; +pub mod list; pub mod queue; pub mod report; const RETRY_ATTEMPTS: usize = 5; +#[derive(Debug, Default, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct Principal { + pub id: Option<u32>, + #[serde(rename = "type")] + pub typ: Option<Type>, + pub quota: Option<u32>, + #[serde(rename = "usedQuota")] + pub used_quota: Option<u32>, + pub name: Option<String>, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub secrets: Vec<String>, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub emails: Vec<String>, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + #[serde(rename = "memberOf")] + pub member_of: Vec<String>, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub description: Option<String>, +} + +#[derive(Debug, Default, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +pub enum Type { + #[serde(rename = "individual")] + #[default] + Individual = 0, + #[serde(rename = "group")] + Group = 1, + #[serde(rename = "resource")] + Resource = 2, + #[serde(rename = "location")] + Location = 3, + #[serde(rename = "superuser")] + Superuser = 4, + #[serde(rename = "list")] + List = 5, + #[serde(rename = "other")] + Other = 6, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +pub enum PrincipalField { + #[serde(rename = "name")] + Name, + #[serde(rename = "type")] + Type, + #[serde(rename = "quota")] + Quota, + #[serde(rename = "description")] + Description, + #[serde(rename = "secrets")] + Secrets, + #[serde(rename = "emails")] + Emails, + #[serde(rename = "memberOf")] + MemberOf, +} + +#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +pub struct PrincipalUpdate { + action: PrincipalAction, + field: PrincipalField, + value: PrincipalValue, +} + +#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +pub enum PrincipalAction { + #[serde(rename = "set")] + Set, + #[serde(rename = "addItem")] + AddItem, + #[serde(rename = "removeItem")] + RemoveItem, +} + +#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +#[serde(untagged)] +pub enum PrincipalValue { + String(String), + StringList(Vec<String>), + Integer(u32), + Type(Type), +} + +impl PrincipalUpdate { + pub fn set(field: PrincipalField, value: PrincipalValue) -> PrincipalUpdate { + PrincipalUpdate { + action: PrincipalAction::Set, + field, + value, + } + } + + pub fn add_item(field: PrincipalField, value: PrincipalValue) -> PrincipalUpdate { + PrincipalUpdate { + action: PrincipalAction::AddItem, + field, + value, + } + } + + pub fn remove_item(field: PrincipalField, value: PrincipalValue) -> PrincipalUpdate { + PrincipalUpdate { + action: PrincipalAction::RemoveItem, + field, + value, + } + } +} + pub trait UnwrapResult<T> { fn unwrap_result(self, action: &str) -> T; } @@ -114,44 +228,6 @@ pub fn read_file(path: &str) -> Vec<u8> { } } -pub async fn get(url: &str) -> HashMap<String, serde_json::Value> { - serde_json::from_slice( - &reqwest::Client::builder() - .danger_accept_invalid_certs(is_localhost(url)) - .build() - .unwrap_or_default() - .get(url) - .send() - .await - .unwrap_result("send OAuth GET request") - .bytes() - .await - .unwrap_result("fetch bytes"), - ) - .unwrap_result("deserialize OAuth GET response") -} - -pub async fn post( - url: &str, - params: &HashMap<String, String>, -) -> HashMap<String, serde_json::Value> { - serde_json::from_slice( - &reqwest::Client::builder() - .danger_accept_invalid_certs(is_localhost(url)) - .build() - .unwrap_or_default() - .post(url) - .form(params) - .send() - .await - .unwrap_result("send OAuth POST request") - .bytes() - .await - .unwrap_result("fetch bytes"), - ) - .unwrap_result("deserialize OAuth POST response") -} - pub async fn name_to_id(client: &Client, name: &str) -> String { let filter = if name.contains('@') { query::Filter::email(name) diff --git a/crates/cli/src/modules/queue.rs b/crates/cli/src/modules/queue.rs index 90fb88c1..7b5dd569 100644 --- a/crates/cli/src/modules/queue.rs +++ b/crates/cli/src/modules/queue.rs @@ -21,14 +21,13 @@ * for more details. */ -use super::{cli::QueueCommands, is_localhost, UnwrapResult}; +use super::cli::{Client, QueueCommands}; use console::Term; use human_size::{Byte, SpecificSize}; -use jmap_client::client::Credentials; use mail_parser::DateTime; use prettytable::{format::Alignment, Attr, Cell, Row, Table}; -use reqwest::header::AUTHORIZATION; -use serde::{de::DeserializeOwned, Deserialize, Deserializer}; +use reqwest::Method; +use serde::{Deserialize, Deserializer}; #[derive(Debug, Deserialize, PartialEq, Eq)] pub struct Message { @@ -76,389 +75,363 @@ pub enum Status { PermanentFailure(String), } -pub async fn cmd_queue(url: &str, credentials: Credentials, command: QueueCommands) { - match command { - QueueCommands::List { - sender, - rcpt, - before, - after, - page_size, - } => { - let stdout = Term::buffered_stdout(); - let ids = query_messages(url, &credentials, &sender, &rcpt, &before, &after).await; - let ids_len = ids.len(); - let page_size = page_size.map(|p| std::cmp::max(p, 1)).unwrap_or(20); - let pages_total = (ids_len as f64 / page_size as f64).ceil() as usize; - for (page_num, chunk) in ids.chunks(page_size).enumerate() { - // Build table - let mut table = Table::new(); - table.add_row(Row::new( - ["ID", "Delivery Due", "Sender", "Recipients", "Size"] - .iter() - .map(|p| Cell::new(p).with_style(Attr::Bold)) - .collect(), - )); - for (message, id) in smtp_manage_request::<Vec<Option<Message>>>( - &build_query(url, "/admin/queue/status?ids=", chunk), - &credentials, - ) - .await - .into_iter() - .zip(chunk) - { - if let Some(message) = message { - let mut rcpts = String::new(); - let mut deliver_at = i64::MAX; - let mut deliver_pos = 0; - for (pos, domain) in message.domains.iter().enumerate() { - if let Some(next_retry) = &domain.next_retry { - let ts = next_retry.to_timestamp(); - if ts < deliver_at { - deliver_at = ts; - deliver_pos = pos; +impl QueueCommands { + pub async fn exec(self, client: Client) { + match self { + QueueCommands::List { + sender, + rcpt, + before, + after, + page_size, + } => { + let stdout = Term::buffered_stdout(); + let ids = client.query_messages(&sender, &rcpt, &before, &after).await; + let ids_len = ids.len(); + let page_size = page_size.map(|p| std::cmp::max(p, 1)).unwrap_or(20); + let pages_total = (ids_len as f64 / page_size as f64).ceil() as usize; + for (page_num, chunk) in ids.chunks(page_size).enumerate() { + // Build table + let mut table = Table::new(); + table.add_row(Row::new( + ["ID", "Delivery Due", "Sender", "Recipients", "Size"] + .iter() + .map(|p| Cell::new(p).with_style(Attr::Bold)) + .collect(), + )); + for (message, id) in client + .http_request::<Vec<Option<Message>>, String>( + Method::GET, + &build_query("/admin/queue/status?ids=", chunk), + None, + ) + .await + .into_iter() + .zip(chunk) + { + if let Some(message) = message { + let mut rcpts = String::new(); + let mut deliver_at = i64::MAX; + let mut deliver_pos = 0; + for (pos, domain) in message.domains.iter().enumerate() { + if let Some(next_retry) = &domain.next_retry { + let ts = next_retry.to_timestamp(); + if ts < deliver_at { + deliver_at = ts; + deliver_pos = pos; + } } - } - for rcpt in &domain.recipients { - if !rcpts.is_empty() { - rcpts.push('\n'); + for rcpt in &domain.recipients { + if !rcpts.is_empty() { + rcpts.push('\n'); + } + rcpts.push_str(&rcpt.address); + rcpts.push_str(" ("); + rcpts.push_str(rcpt.status.status_short()); + rcpts.push(')'); } - rcpts.push_str(&rcpt.address); - rcpts.push_str(" ("); - rcpts.push_str(rcpt.status.status_short()); - rcpts.push(')'); } - } - let mut cells = Vec::new(); - cells.push(Cell::new(&format!("{id:X}"))); - cells.push(if deliver_at != i64::MAX { - Cell::new( - &message.domains[deliver_pos] - .next_retry - .as_ref() + let mut cells = Vec::new(); + cells.push(Cell::new(&format!("{id:X}"))); + cells.push(if deliver_at != i64::MAX { + Cell::new( + &message.domains[deliver_pos] + .next_retry + .as_ref() + .unwrap() + .to_rfc822(), + ) + } else { + Cell::new("None") + }); + cells.push(Cell::new(if !message.return_path.is_empty() { + &message.return_path + } else { + "<>" + })); + cells.push(Cell::new(&rcpts)); + cells.push(Cell::new( + &SpecificSize::new(message.size as u32, Byte) .unwrap() - .to_rfc822(), - ) - } else { - Cell::new("None") - }); - cells.push(Cell::new(if !message.return_path.is_empty() { - &message.return_path - } else { - "<>" - })); - cells.push(Cell::new(&rcpts)); - cells.push(Cell::new( - &SpecificSize::new(message.size as u32, Byte) - .unwrap() - .to_string(), - )); - table.add_row(Row::new(cells)); + .to_string(), + )); + table.add_row(Row::new(cells)); + } } - } - eprintln!(); - table.printstd(); - eprintln!(); - if page_num + 1 != pages_total { - eprintln!("\n--- Press any key to continue or 'q' to exit ---"); - if let Ok('q' | 'Q') = stdout.read_char() { - break; + eprintln!(); + table.printstd(); + eprintln!(); + if page_num + 1 != pages_total { + eprintln!("\n--- Press any key to continue or 'q' to exit ---"); + if let Ok('q' | 'Q') = stdout.read_char() { + break; + } } } + eprintln!("\n{ids_len} queued message(s) found.") } - eprintln!("\n{ids_len} queued message(s) found.") - } - QueueCommands::Status { ids } => { - for (message, id) in smtp_manage_request::<Vec<Option<Message>>>( - &build_query(url, "/admin/queue/status?ids=", &parse_ids(&ids)), - &credentials, - ) - .await - .into_iter() - .zip(&ids) - { - let mut table = Table::new(); - table.add_row(Row::new(vec![ - Cell::new("ID").with_style(Attr::Bold), - Cell::new(id), - ])); - if let Some(message) = message { - table.add_row(Row::new(vec![ - Cell::new("Sender").with_style(Attr::Bold), - Cell::new(if !message.return_path.is_empty() { - &message.return_path - } else { - "<>" - }), - ])); - table.add_row(Row::new(vec![ - Cell::new("Created").with_style(Attr::Bold), - Cell::new(&message.created.to_rfc822()), - ])); + QueueCommands::Status { ids } => { + for (message, id) in client + .http_request::<Vec<Option<Message>>, String>( + Method::GET, + &build_query("/admin/queue/status?ids=", &parse_ids(&ids)), + None, + ) + .await + .into_iter() + .zip(&ids) + { + let mut table = Table::new(); table.add_row(Row::new(vec![ - Cell::new("Size").with_style(Attr::Bold), - Cell::new( - &SpecificSize::new(message.size as u32, Byte) - .unwrap() - .to_string(), - ), + Cell::new("ID").with_style(Attr::Bold), + Cell::new(id), ])); - if let Some(env_id) = &message.env_id { - table.add_row(Row::new(vec![ - Cell::new("Env-Id").with_style(Attr::Bold), - Cell::new(env_id), - ])); - } - if message.priority != 0 { - table.add_row(Row::new(vec![ - Cell::new("Priority").with_style(Attr::Bold), - Cell::new(&message.priority.to_string()), - ])); - } - for domain in &message.domains { - table.add_row(Row::new(vec![Cell::new_align( - &domain.name, - Alignment::RIGHT, - ) - .with_style(Attr::Bold) - .with_style(Attr::Italic(true)) - .with_hspan(2)])); + if let Some(message) = message { table.add_row(Row::new(vec![ - Cell::new("Status").with_style(Attr::Bold), - Cell::new(domain.status.status()), + Cell::new("Sender").with_style(Attr::Bold), + Cell::new(if !message.return_path.is_empty() { + &message.return_path + } else { + "<>" + }), ])); table.add_row(Row::new(vec![ - Cell::new("Details").with_style(Attr::Bold), - Cell::new(domain.status.details()), + Cell::new("Created").with_style(Attr::Bold), + Cell::new(&message.created.to_rfc822()), ])); table.add_row(Row::new(vec![ - Cell::new("Retry #").with_style(Attr::Bold), - Cell::new(&domain.retry_num.to_string()), + Cell::new("Size").with_style(Attr::Bold), + Cell::new( + &SpecificSize::new(message.size as u32, Byte) + .unwrap() + .to_string(), + ), ])); - if let Some(dt) = &domain.next_retry { + if let Some(env_id) = &message.env_id { table.add_row(Row::new(vec![ - Cell::new("Delivery Due").with_style(Attr::Bold), - Cell::new(&dt.to_rfc822()), + Cell::new("Env-Id").with_style(Attr::Bold), + Cell::new(env_id), ])); } - if let Some(dt) = &domain.next_notify { + if message.priority != 0 { table.add_row(Row::new(vec![ - Cell::new("Notify at").with_style(Attr::Bold), - Cell::new(&dt.to_rfc822()), + Cell::new("Priority").with_style(Attr::Bold), + Cell::new(&message.priority.to_string()), ])); } - table.add_row(Row::new(vec![ - Cell::new("Expires").with_style(Attr::Bold), - Cell::new(&domain.expires.to_rfc822()), - ])); + for domain in &message.domains { + table.add_row(Row::new(vec![Cell::new_align( + &domain.name, + Alignment::RIGHT, + ) + .with_style(Attr::Bold) + .with_style(Attr::Italic(true)) + .with_hspan(2)])); + table.add_row(Row::new(vec![ + Cell::new("Status").with_style(Attr::Bold), + Cell::new(domain.status.status()), + ])); + table.add_row(Row::new(vec![ + Cell::new("Details").with_style(Attr::Bold), + Cell::new(domain.status.details()), + ])); + table.add_row(Row::new(vec![ + Cell::new("Retry #").with_style(Attr::Bold), + Cell::new(&domain.retry_num.to_string()), + ])); + if let Some(dt) = &domain.next_retry { + table.add_row(Row::new(vec![ + Cell::new("Delivery Due").with_style(Attr::Bold), + Cell::new(&dt.to_rfc822()), + ])); + } + if let Some(dt) = &domain.next_notify { + table.add_row(Row::new(vec![ + Cell::new("Notify at").with_style(Attr::Bold), + Cell::new(&dt.to_rfc822()), + ])); + } + table.add_row(Row::new(vec![ + Cell::new("Expires").with_style(Attr::Bold), + Cell::new(&domain.expires.to_rfc822()), + ])); - let mut rcpts = Table::new(); - rcpts.add_row(Row::new(vec![ - Cell::new("Address").with_style(Attr::Bold), - Cell::new("Status").with_style(Attr::Bold), - Cell::new("Details").with_style(Attr::Bold), - ])); - for rcpt in &domain.recipients { + let mut rcpts = Table::new(); rcpts.add_row(Row::new(vec![ - Cell::new(&rcpt.address), - Cell::new(rcpt.status.status()), - Cell::new(rcpt.status.details()), + Cell::new("Address").with_style(Attr::Bold), + Cell::new("Status").with_style(Attr::Bold), + Cell::new("Details").with_style(Attr::Bold), + ])); + for rcpt in &domain.recipients { + rcpts.add_row(Row::new(vec![ + Cell::new(&rcpt.address), + Cell::new(rcpt.status.status()), + Cell::new(rcpt.status.details()), + ])); + } + table.add_row(Row::new(vec![ + Cell::new("Recipients").with_style(Attr::Bold), + Cell::from(&rcpts), ])); } - table.add_row(Row::new(vec![ - Cell::new("Recipients").with_style(Attr::Bold), - Cell::from(&rcpts), - ])); + } else { + table.add_row(Row::new(vec![Cell::new_align( + "-- Not found --", + Alignment::CENTER, + ) + .with_hspan(2)])); } - } else { - table.add_row(Row::new(vec![Cell::new_align( - "-- Not found --", - Alignment::CENTER, - ) - .with_hspan(2)])); - } - eprintln!(); - table.printstd(); - eprintln!(); - } - } - QueueCommands::Retry { - sender, - domain, - before, - after, - time, - ids, - } => { - let (parsed_ids, ids) = if ids.is_empty() { - if sender.is_some() || domain.is_some() || before.is_some() || after.is_some() { - let parsed_ids = - query_messages(url, &credentials, &sender, &domain, &before, &after).await; - let ids = parsed_ids.iter().map(|id| format!("{id:X}")).collect(); - (parsed_ids, ids) - } else { - (vec![], vec![]) + eprintln!(); + table.printstd(); + eprintln!(); } - } else { - (parse_ids(&ids), ids) - }; - - if ids.is_empty() { - eprintln!("No messages were found."); - std::process::exit(1); } + QueueCommands::Retry { + sender, + domain, + before, + after, + time, + ids, + } => { + let (parsed_ids, ids) = if ids.is_empty() { + if sender.is_some() || domain.is_some() || before.is_some() || after.is_some() { + let parsed_ids = client + .query_messages(&sender, &domain, &before, &after) + .await; + let ids = parsed_ids.iter().map(|id| format!("{id:X}")).collect(); + (parsed_ids, ids) + } else { + (vec![], vec![]) + } + } else { + (parse_ids(&ids), ids) + }; - let mut query = form_urlencoded::Serializer::new(format!("{url}/admin/queue/retry?")); + if ids.is_empty() { + eprintln!("No messages were found."); + std::process::exit(1); + } - if let Some(filter) = &domain { - query.append_pair("filter", filter); - } - if let Some(at) = time { - query.append_pair("at", &at.to_rfc3339()); - } - query.append_pair("ids", &append_ids(String::new(), &parsed_ids)); + let mut query = form_urlencoded::Serializer::new("/admin/queue/retry?".to_string()); - let mut success_count = 0; - let mut failed_list = vec![]; - for (success, id) in smtp_manage_request::<Vec<bool>>(&query.finish(), &credentials) - .await - .into_iter() - .zip(ids) - { - if success { - success_count += 1; - } else { - failed_list.push(id); + if let Some(filter) = &domain { + query.append_pair("filter", filter); } - } - eprint!("\nSuccessfully rescheduled {success_count} message(s)."); - if !failed_list.is_empty() { - eprint!(" Unable to reschedule id(s): {}.", failed_list.join(", ")); - } - eprintln!(); - } - QueueCommands::Cancel { - sender, - rcpt, - before, - after, - ids, - } => { - let (parsed_ids, ids) = if ids.is_empty() { - if sender.is_some() || rcpt.is_some() || before.is_some() || after.is_some() { - let parsed_ids = - query_messages(url, &credentials, &sender, &rcpt, &before, &after).await; - let ids = parsed_ids.iter().map(|id| format!("{id:X}")).collect(); - (parsed_ids, ids) - } else { - (vec![], vec![]) + if let Some(at) = time { + query.append_pair("at", &at.to_rfc3339()); } - } else { - (parse_ids(&ids), ids) - }; - - if ids.is_empty() { - eprintln!("No messages were found."); - std::process::exit(1); + query.append_pair("ids", &append_ids(String::new(), &parsed_ids)); + + let mut success_count = 0; + let mut failed_list = vec![]; + for (success, id) in client + .http_request::<Vec<bool>, String>(Method::GET, &query.finish(), None) + .await + .into_iter() + .zip(ids) + { + if success { + success_count += 1; + } else { + failed_list.push(id); + } + } + eprint!("\nSuccessfully rescheduled {success_count} message(s)."); + if !failed_list.is_empty() { + eprint!(" Unable to reschedule id(s): {}.", failed_list.join(", ")); + } + eprintln!(); } + QueueCommands::Cancel { + sender, + rcpt, + before, + after, + ids, + } => { + let (parsed_ids, ids) = if ids.is_empty() { + if sender.is_some() || rcpt.is_some() || before.is_some() || after.is_some() { + let parsed_ids = + client.query_messages(&sender, &rcpt, &before, &after).await; + let ids = parsed_ids.iter().map(|id| format!("{id:X}")).collect(); + (parsed_ids, ids) + } else { + (vec![], vec![]) + } + } else { + (parse_ids(&ids), ids) + }; - let mut query = form_urlencoded::Serializer::new(format!("{url}/admin/queue/cancel?")); + if ids.is_empty() { + eprintln!("No messages were found."); + std::process::exit(1); + } - if let Some(filter) = &rcpt { - query.append_pair("filter", filter); - } - query.append_pair("ids", &append_ids(String::new(), &parsed_ids)); + let mut query = + form_urlencoded::Serializer::new("/admin/queue/cancel?".to_string()); - let mut success_count = 0; - let mut failed_list = vec![]; - for (success, id) in smtp_manage_request::<Vec<bool>>(&query.finish(), &credentials) - .await - .into_iter() - .zip(ids) - { - if success { - success_count += 1; - } else { - failed_list.push(id); + if let Some(filter) = &rcpt { + query.append_pair("filter", filter); } + query.append_pair("ids", &append_ids(String::new(), &parsed_ids)); + + let mut success_count = 0; + let mut failed_list = vec![]; + for (success, id) in client + .http_request::<Vec<bool>, String>(Method::GET, &query.finish(), None) + .await + .into_iter() + .zip(ids) + { + if success { + success_count += 1; + } else { + failed_list.push(id); + } + } + eprint!("\nCancelled delivery of {success_count} message(s)."); + if !failed_list.is_empty() { + eprint!( + " Unable to cancel delivery for id(s): {}.", + failed_list.join(", ") + ); + } + eprintln!(); } - eprint!("\nCancelled delivery of {success_count} message(s)."); - if !failed_list.is_empty() { - eprint!( - " Unable to cancel delivery for id(s): {}.", - failed_list.join(", ") - ); - } - eprintln!(); } } } -#[derive(Deserialize)] -#[serde(untagged)] -pub enum Response<T> { - Data { data: T }, - Error { error: String, details: String }, -} - -pub async fn smtp_manage_request<T: DeserializeOwned>(url: &str, credentials: &Credentials) -> T { - match serde_json::from_slice::<Response<T>>( - &reqwest::Client::builder() - .danger_accept_invalid_certs(is_localhost(url)) - .build() - .unwrap_or_default() - .get(url) - .header( - AUTHORIZATION, - match credentials { - Credentials::Basic(s) => format!("Basic {s}"), - Credentials::Bearer(s) => format!("Bearer {s}"), - }, - ) - .send() - .await - .unwrap_result("send GET request") - .bytes() - .await - .unwrap_result("fetch bytes"), - ) - .unwrap_result("deserialize response") - { - Response::Data { data } => data, - Response::Error { error, details } => { - eprintln!("Request failed: {details} ({error:?})"); - std::process::exit(1); +impl Client { + async fn query_messages( + &self, + from: &Option<String>, + rcpt: &Option<String>, + before: &Option<DateTime>, + after: &Option<DateTime>, + ) -> Vec<u64> { + let mut query = form_urlencoded::Serializer::new("/admin/queue/list?".to_string()); + + if let Some(sender) = from { + query.append_pair("from", sender); + } + if let Some(rcpt) = rcpt { + query.append_pair("to", rcpt); + } + if let Some(before) = before { + query.append_pair("before", &before.to_rfc3339()); + } + if let Some(after) = after { + query.append_pair("after", &after.to_rfc3339()); } - } -} - -async fn query_messages( - url: &str, - credentials: &Credentials, - from: &Option<String>, - rcpt: &Option<String>, - before: &Option<DateTime>, - after: &Option<DateTime>, -) -> Vec<u64> { - let mut query = form_urlencoded::Serializer::new(format!("{url}/admin/queue/list?")); - if let Some(sender) = from { - query.append_pair("from", sender); - } - if let Some(rcpt) = rcpt { - query.append_pair("to", rcpt); - } - if let Some(before) = before { - query.append_pair("before", &before.to_rfc3339()); - } - if let Some(after) = after { - query.append_pair("after", &after.to_rfc3339()); + self.http_request::<Vec<u64>, String>(Method::GET, &query.finish(), None) + .await } - - smtp_manage_request::<Vec<u64>>(&query.finish(), credentials).await } fn deserialize_maybe_datetime<'de, D>(deserializer: D) -> Result<Option<DateTime>, D::Error> @@ -507,9 +480,8 @@ fn parse_ids(ids: &[String]) -> Vec<u64> { result } -fn build_query(url: &str, path: &str, ids: &[u64]) -> String { - let mut query = String::with_capacity(url.len() + path.len() + (ids.len() * 10)); - query.push_str(url); +fn build_query(path: &str, ids: &[u64]) -> String { + let mut query = String::with_capacity(path.len() + (ids.len() * 10)); query.push_str(path); append_ids(query, ids) } diff --git a/crates/cli/src/modules/report.rs b/crates/cli/src/modules/report.rs index 42c3198f..696f9054 100644 --- a/crates/cli/src/modules/report.rs +++ b/crates/cli/src/modules/report.rs @@ -21,13 +21,13 @@ * for more details. */ -use super::cli::{ReportCommands, ReportFormat}; -use crate::modules::queue::{deserialize_datetime, smtp_manage_request}; +use super::cli::{Client, ReportCommands, ReportFormat}; +use crate::modules::queue::deserialize_datetime; use console::Term; use human_size::{Byte, SpecificSize}; -use jmap_client::client::Credentials; use mail_parser::DateTime; use prettytable::{format::Alignment, Attr, Cell, Row, Table}; +use reqwest::Method; use serde::Deserialize; #[derive(Debug, Deserialize)] @@ -42,149 +42,159 @@ pub struct Report { pub size: usize, } -pub async fn cmd_report(url: &str, credentials: Credentials, command: ReportCommands) { - match command { - ReportCommands::List { - domain, - format, - page_size, - } => { - let stdout = Term::buffered_stdout(); - let mut query = form_urlencoded::Serializer::new(format!("{url}/admin/report/list?")); +impl ReportCommands { + pub async fn exec(self, client: Client) { + match self { + ReportCommands::List { + domain, + format, + page_size, + } => { + let stdout = Term::buffered_stdout(); + let mut query = form_urlencoded::Serializer::new("/admin/report/list?".to_string()); - if let Some(domain) = &domain { - query.append_pair("domain", domain); - } - if let Some(format) = &format { - query.append_pair("type", format.id()); - } + if let Some(domain) = &domain { + query.append_pair("domain", domain); + } + if let Some(format) = &format { + query.append_pair("type", format.id()); + } + + let ids = client + .http_request::<Vec<String>, String>(Method::GET, &query.finish(), None) + .await; + let ids_len = ids.len(); + let page_size = page_size.map(|p| std::cmp::max(p, 1)).unwrap_or(20); + let pages_total = (ids_len as f64 / page_size as f64).ceil() as usize; + for (page_num, chunk) in ids.chunks(page_size).enumerate() { + // Build table + let mut table = Table::new(); + table.add_row(Row::new( + ["ID", "Domain", "Type", "From Date", "To Date", "Size"] + .iter() + .map(|p| Cell::new(p).with_style(Attr::Bold)) + .collect(), + )); + for (report, id) in client + .http_request::<Vec<Option<Report>>, String>( + Method::GET, + &format!("/admin/report/status?ids={}", chunk.join(",")), + None, + ) + .await + .into_iter() + .zip(chunk) + { + if let Some(report) = report { + table.add_row(Row::new(vec![ + Cell::new(id), + Cell::new(&report.domain), + Cell::new(report.type_.name()), + Cell::new(&report.range_from.to_rfc822()), + Cell::new(&report.range_to.to_rfc822()), + Cell::new( + &SpecificSize::new(report.size as u32, Byte) + .unwrap() + .to_string(), + ), + ])); + } + } - let ids = smtp_manage_request::<Vec<String>>(&query.finish(), &credentials).await; - let ids_len = ids.len(); - let page_size = page_size.map(|p| std::cmp::max(p, 1)).unwrap_or(20); - let pages_total = (ids_len as f64 / page_size as f64).ceil() as usize; - for (page_num, chunk) in ids.chunks(page_size).enumerate() { - // Build table - let mut table = Table::new(); - table.add_row(Row::new( - ["ID", "Domain", "Type", "From Date", "To Date", "Size"] - .iter() - .map(|p| Cell::new(p).with_style(Attr::Bold)) - .collect(), - )); - for (report, id) in smtp_manage_request::<Vec<Option<Report>>>( - &format!("{url}/admin/report/status?ids={}", chunk.join(",")), - &credentials, - ) - .await - .into_iter() - .zip(chunk) + eprintln!(); + table.printstd(); + eprintln!(); + if page_num + 1 != pages_total { + eprintln!("\n--- Press any key to continue or 'q' to exit ---"); + if let Ok('q' | 'Q') = stdout.read_char() { + break; + } + } + } + eprintln!("\n{ids_len} queued message(s) found.") + } + ReportCommands::Status { ids } => { + for (report, id) in client + .http_request::<Vec<Option<Report>>, String>( + Method::GET, + &format!("/admin/report/status?ids={}", ids.join(",")), + None, + ) + .await + .into_iter() + .zip(&ids) { + let mut table = Table::new(); + table.add_row(Row::new(vec![ + Cell::new("ID").with_style(Attr::Bold), + Cell::new(id), + ])); if let Some(report) = report { table.add_row(Row::new(vec![ - Cell::new(id), + Cell::new("Domain Name").with_style(Attr::Bold), Cell::new(&report.domain), + ])); + table.add_row(Row::new(vec![ + Cell::new("Type").with_style(Attr::Bold), Cell::new(report.type_.name()), + ])); + table.add_row(Row::new(vec![ + Cell::new("From Date").with_style(Attr::Bold), Cell::new(&report.range_from.to_rfc822()), + ])); + table.add_row(Row::new(vec![ + Cell::new("To Date").with_style(Attr::Bold), Cell::new(&report.range_to.to_rfc822()), + ])); + table.add_row(Row::new(vec![ + Cell::new("Size").with_style(Attr::Bold), Cell::new( &SpecificSize::new(report.size as u32, Byte) .unwrap() .to_string(), ), ])); + } else { + table.add_row(Row::new(vec![Cell::new_align( + "-- Not found --", + Alignment::CENTER, + ) + .with_hspan(2)])); } - } - eprintln!(); - table.printstd(); - eprintln!(); - if page_num + 1 != pages_total { - eprintln!("\n--- Press any key to continue or 'q' to exit ---"); - if let Ok('q' | 'Q') = stdout.read_char() { - break; - } + eprintln!(); + table.printstd(); + eprintln!(); } } - eprintln!("\n{ids_len} queued message(s) found.") - } - ReportCommands::Status { ids } => { - for (report, id) in smtp_manage_request::<Vec<Option<Report>>>( - &format!("{url}/admin/report/status?ids={}", ids.join(",")), - &credentials, - ) - .await - .into_iter() - .zip(&ids) - { - let mut table = Table::new(); - table.add_row(Row::new(vec![ - Cell::new("ID").with_style(Attr::Bold), - Cell::new(id), - ])); - if let Some(report) = report { - table.add_row(Row::new(vec![ - Cell::new("Domain Name").with_style(Attr::Bold), - Cell::new(&report.domain), - ])); - table.add_row(Row::new(vec![ - Cell::new("Type").with_style(Attr::Bold), - Cell::new(report.type_.name()), - ])); - table.add_row(Row::new(vec![ - Cell::new("From Date").with_style(Attr::Bold), - Cell::new(&report.range_from.to_rfc822()), - ])); - table.add_row(Row::new(vec![ - Cell::new("To Date").with_style(Attr::Bold), - Cell::new(&report.range_to.to_rfc822()), - ])); - table.add_row(Row::new(vec![ - Cell::new("Size").with_style(Attr::Bold), - Cell::new( - &SpecificSize::new(report.size as u32, Byte) - .unwrap() - .to_string(), - ), - ])); - } else { - table.add_row(Row::new(vec![Cell::new_align( - "-- Not found --", - Alignment::CENTER, + ReportCommands::Cancel { ids } => { + let mut success_count = 0; + let mut failed_list = vec![]; + for (success, id) in client + .http_request::<Vec<bool>, String>( + Method::GET, + &format!("/admin/report/cancel?ids={}", ids.join(",")), + None, ) - .with_hspan(2)])); + .await + .into_iter() + .zip(ids) + { + if success { + success_count += 1; + } else { + failed_list.push(id); + } } - - eprintln!(); - table.printstd(); - eprintln!(); - } - } - ReportCommands::Cancel { ids } => { - let mut success_count = 0; - let mut failed_list = vec![]; - for (success, id) in smtp_manage_request::<Vec<bool>>( - &format!("{url}/admin/report/cancel?ids={}", ids.join(",")), - &credentials, - ) - .await - .into_iter() - .zip(ids) - { - if success { - success_count += 1; - } else { - failed_list.push(id); + eprint!("\nRemoved {success_count} report(s)."); + if !failed_list.is_empty() { + eprint!( + " Unable to remove report id(s): {}.", + failed_list.join(", ") + ); } + eprintln!(); } - eprint!("\nRemoved {success_count} report(s)."); - if !failed_list.is_empty() { - eprint!( - " Unable to remove report id(s): {}.", - failed_list.join(", ") - ); - } - eprintln!(); } } } diff --git a/crates/directory/src/backend/imap/config.rs b/crates/directory/src/backend/imap/config.rs index 9859a25d..cd621a92 100644 --- a/crates/directory/src/backend/imap/config.rs +++ b/crates/directory/src/backend/imap/config.rs @@ -21,20 +21,15 @@ * for more details. */ -use std::sync::Arc; - use mail_send::smtp::tls::build_tls_connector; use utils::config::{utils::AsKey, Config}; -use crate::{cache::CachedDirectory, config::build_pool, Directory}; +use crate::core::config::build_pool; use super::{ImapConnectionManager, ImapDirectory}; impl ImapDirectory { - pub fn from_config( - config: &Config, - prefix: impl AsKey, - ) -> utils::config::Result<Arc<dyn Directory>> { + pub fn from_config(config: &Config, prefix: impl AsKey) -> utils::config::Result<Self> { let prefix = prefix.as_key(); let address = config.value_require((&prefix, "address"))?; let tls_implicit: bool = config.property_or_static((&prefix, "tls.implicit"), "false")?; @@ -52,16 +47,12 @@ impl ImapDirectory { mechanisms: 0.into(), }; - CachedDirectory::try_from_config( - config, - &prefix, - ImapDirectory { - pool: build_pool(config, &prefix, manager)?, - domains: config - .values((&prefix, "local-domains")) - .map(|(_, v)| v.to_lowercase()) - .collect(), - }, - ) + Ok(ImapDirectory { + pool: build_pool(config, &prefix, manager)?, + domains: config + .values((&prefix, "lookup.domains")) + .map(|(_, v)| v.to_lowercase()) + .collect(), + }) } } diff --git a/crates/directory/src/backend/imap/lookup.rs b/crates/directory/src/backend/imap/lookup.rs index 53a642f5..d82ec914 100644 --- a/crates/directory/src/backend/imap/lookup.rs +++ b/crates/directory/src/backend/imap/lookup.rs @@ -24,13 +24,12 @@ use mail_send::Credentials; use smtp_proto::{AUTH_CRAM_MD5, AUTH_LOGIN, AUTH_OAUTHBEARER, AUTH_PLAIN, AUTH_XOAUTH2}; -use crate::{Directory, DirectoryError, Principal, QueryBy}; +use crate::{DirectoryError, Principal, QueryBy}; use super::{ImapDirectory, ImapError}; -#[async_trait::async_trait] -impl Directory for ImapDirectory { - async fn query(&self, query: QueryBy<'_>) -> crate::Result<Option<Principal<u32>>> { +impl ImapDirectory { + pub async fn query(&self, query: QueryBy<'_>) -> crate::Result<Option<Principal<u32>>> { if let QueryBy::Credentials(credentials) = query { let mut client = self.pool.get().await?; let mechanism = match credentials { @@ -77,23 +76,23 @@ impl Directory for ImapDirectory { } } - async fn email_to_ids(&self, _address: &str) -> crate::Result<Vec<u32>> { + pub async fn email_to_ids(&self, _address: &str) -> crate::Result<Vec<u32>> { Err(DirectoryError::unsupported("imap", "email_to_ids")) } - async fn rcpt(&self, _address: &str) -> crate::Result<bool> { + pub async fn rcpt(&self, _address: &str) -> crate::Result<bool> { Err(DirectoryError::unsupported("imap", "rcpt")) } - async fn vrfy(&self, _address: &str) -> crate::Result<Vec<String>> { + pub async fn vrfy(&self, _address: &str) -> crate::Result<Vec<String>> { Err(DirectoryError::unsupported("imap", "vrfy")) } - async fn expn(&self, _address: &str) -> crate::Result<Vec<String>> { + pub async fn expn(&self, _address: &str) -> crate::Result<Vec<String>> { Err(DirectoryError::unsupported("imap", "expn")) } - async fn is_local_domain(&self, domain: &str) -> crate::Result<bool> { + pub async fn is_local_domain(&self, domain: &str) -> crate::Result<bool> { Ok(self.domains.contains(domain)) } } diff --git a/crates/directory/src/backend/internal/lookup.rs b/crates/directory/src/backend/internal/lookup.rs index d173b793..d20b4576 100644 --- a/crates/directory/src/backend/internal/lookup.rs +++ b/crates/directory/src/backend/internal/lookup.rs @@ -27,12 +27,23 @@ use store::{ IterateParams, Store, ValueKey, }; -use crate::{Directory, Principal, QueryBy}; +use crate::{Principal, QueryBy}; use super::manage::ManageDirectory; #[async_trait::async_trait] -impl Directory for Store { +pub trait DirectoryStore: Sync + Send { + async fn query(&self, by: QueryBy<'_>) -> crate::Result<Option<Principal<u32>>>; + async fn email_to_ids(&self, email: &str) -> crate::Result<Vec<u32>>; + + async fn is_local_domain(&self, domain: &str) -> crate::Result<bool>; + async fn rcpt(&self, address: &str) -> crate::Result<bool>; + async fn vrfy(&self, address: &str) -> crate::Result<Vec<String>>; + async fn expn(&self, address: &str) -> crate::Result<Vec<String>>; +} + +#[async_trait::async_trait] +impl DirectoryStore for Store { async fn query(&self, by: QueryBy<'_>) -> crate::Result<Option<Principal<u32>>> { let (username, secret) = match by { QueryBy::Name(name) => (name, None), diff --git a/crates/directory/src/backend/internal/manage.rs b/crates/directory/src/backend/internal/manage.rs index 18e1e69b..0f6f19a0 100644 --- a/crates/directory/src/backend/internal/manage.rs +++ b/crates/directory/src/backend/internal/manage.rs @@ -24,12 +24,15 @@ use jmap_proto::types::collection::Collection; use store::{ write::{assert::HashedValue, BatchBuilder, DirectoryClass, ValueClass}, - IterateParams, Serialize, Store, ValueKey, + Deserialize, IterateParams, Serialize, Store, ValueKey, }; -use crate::{Directory, DirectoryError, ManagementError, Principal, QueryBy, Type}; +use crate::{DirectoryError, ManagementError, Principal, QueryBy, Type}; -use super::{PrincipalAction, PrincipalField, PrincipalUpdate, PrincipalValue}; +use super::{ + lookup::DirectoryStore, PrincipalAction, PrincipalField, PrincipalIdType, PrincipalUpdate, + PrincipalValue, +}; #[async_trait::async_trait] pub trait ManageDirectory { @@ -43,11 +46,10 @@ pub trait ManageDirectory { changes: Vec<PrincipalUpdate>, ) -> crate::Result<()>; async fn delete_account(&self, by: QueryBy<'_>) -> crate::Result<()>; - async fn create_domain(&self, domain: &str) -> crate::Result<()>; - async fn delete_domain(&self, domain: &str) -> crate::Result<()>; async fn list_accounts( &self, start_from: Option<&str>, + typ: Option<Type>, limit: usize, ) -> crate::Result<Vec<String>>; async fn map_group_ids(&self, principal: Principal<u32>) -> crate::Result<Principal<String>>; @@ -56,6 +58,13 @@ pub trait ManageDirectory { principal: Principal<String>, create_if_missing: bool, ) -> crate::Result<Principal<u32>>; + async fn create_domain(&self, domain: &str) -> crate::Result<()>; + async fn delete_domain(&self, domain: &str) -> crate::Result<()>; + async fn list_domains( + &self, + start_from: Option<&str>, + limit: usize, + ) -> crate::Result<Vec<String>>; } #[async_trait::async_trait] @@ -83,10 +92,11 @@ impl ManageDirectory for Store { } async fn get_account_id(&self, name: &str) -> crate::Result<Option<u32>> { - self.get_value::<u32>(ValueKey::from(ValueClass::Directory( + self.get_value::<PrincipalIdType>(ValueKey::from(ValueClass::Directory( DirectoryClass::NameToId(name.as_bytes().to_vec()), ))) .await + .map(|v| v.map(|v| v.account_id)) .map_err(Into::into) } @@ -114,7 +124,10 @@ impl ManageDirectory for Store { .with_collection(Collection::Principal) .create_document(account_id) .assert_value(name_key.clone(), ()) - .set(name_key, account_id.serialize()) + .set( + name_key, + PrincipalIdType::new(account_id, Type::Individual).serialize(), + ) .set( ValueClass::Directory(DirectoryClass::Principal(account_id)), Principal { @@ -159,18 +172,20 @@ impl ManageDirectory for Store { // Make sure new name is not taken principal.name = principal.name.to_lowercase(); if self.get_account_id(&principal.name).await?.is_some() { - return Err(DirectoryError::Management(ManagementError::NotUniqueField( - PrincipalField::Name, - ))); + return Err(DirectoryError::Management(ManagementError::AlreadyExists { + field: PrincipalField::Name, + value: principal.name, + })); } // Make sure the e-mail is not taken and validate domain for email in principal.emails.iter_mut() { *email = email.to_lowercase(); if self.rcpt(email).await? { - return Err(DirectoryError::Management(ManagementError::NotUniqueField( - PrincipalField::Emails, - ))); + return Err(DirectoryError::Management(ManagementError::AlreadyExists { + field: PrincipalField::Emails, + value: email.to_string(), + })); } if let Some(domain) = email.split('@').nth(1) { if !self.is_local_domain(domain).await? { @@ -201,7 +216,7 @@ impl ManageDirectory for Store { ) .set( ValueClass::Directory(DirectoryClass::NameToId(principal.name.into_bytes())), - account_id.serialize(), + PrincipalIdType::new(account_id, principal.typ.into_base_type()).serialize(), ); // Write email to id mapping @@ -306,7 +321,10 @@ impl ManageDirectory for Store { if principal.inner.name != new_name { if self.get_account_id(&new_name).await?.is_some() { return Err(DirectoryError::Management( - ManagementError::NotUniqueField(PrincipalField::Name), + ManagementError::AlreadyExists { + field: PrincipalField::Name, + value: new_name, + }, )); } @@ -318,12 +336,14 @@ impl ManageDirectory for Store { batch.set( ValueClass::Directory(DirectoryClass::NameToId(new_name.into_bytes())), - account_id.serialize(), + PrincipalIdType::new(account_id, principal.inner.typ.into_base_type()) + .serialize(), ); } } (PrincipalAction::Set, PrincipalField::Type, PrincipalValue::Type(new_type)) - if principal.inner.typ != Type::List && new_type != Type::List => + if matches!(principal.inner.typ, Type::Individual | Type::Superuser) + && matches!(new_type, Type::Individual | Type::Superuser) => { principal.inner.typ = new_type; } @@ -362,7 +382,10 @@ impl ManageDirectory for Store { if !principal.inner.emails.contains(email) { if self.rcpt(email).await? { return Err(DirectoryError::Management( - ManagementError::NotUniqueField(PrincipalField::Emails), + ManagementError::AlreadyExists { + field: PrincipalField::Emails, + value: email.to_string(), + }, )); } if let Some(domain) = email.split('@').nth(1) { @@ -434,7 +457,10 @@ impl ManageDirectory for Store { if !principal.inner.emails.contains(&email) { if self.rcpt(&email).await? { return Err(DirectoryError::Management( - ManagementError::NotUniqueField(PrincipalField::Emails), + ManagementError::AlreadyExists { + field: PrincipalField::Emails, + value: email, + }, )); } if let Some(domain) = email.split('@').nth(1) { @@ -595,6 +621,7 @@ impl ManageDirectory for Store { async fn list_accounts( &self, start_from: Option<&str>, + typ: Option<Type>, limit: usize, ) -> crate::Result<Vec<String>> { let from_key = ValueKey::from(ValueClass::Directory(DirectoryClass::NameToId( @@ -607,6 +634,42 @@ impl ManageDirectory for Store { let mut results = Vec::with_capacity(limit); self.iterate( + IterateParams::new(from_key, to_key) + .set_values(typ.is_some()) + .ascending(), + |key, value| { + if typ.map_or(true, |t| { + PrincipalIdType::deserialize(value) + .map(|v| v.typ == t) + .unwrap_or(false) + }) { + results.push( + String::from_utf8_lossy(key.get(1..).unwrap_or_default()).into_owned(), + ); + } + Ok(limit == 0 || results.len() < limit) + }, + ) + .await?; + + Ok(results) + } + + async fn list_domains( + &self, + start_from: Option<&str>, + limit: usize, + ) -> crate::Result<Vec<String>> { + let from_key = ValueKey::from(ValueClass::Directory(DirectoryClass::Domain( + start_from.unwrap_or("").as_bytes().to_vec(), + ))); + let to_key = ValueKey::from(ValueClass::Directory(DirectoryClass::Domain(vec![ + u8::MAX; + 10 + ]))); + + let mut results = Vec::with_capacity(limit); + self.iterate( IterateParams::new(from_key, to_key).no_values().ascending(), |key, _| { results diff --git a/crates/directory/src/backend/internal/mod.rs b/crates/directory/src/backend/internal/mod.rs index 5c58b92b..8bc86cc2 100644 --- a/crates/directory/src/backend/internal/mod.rs +++ b/crates/directory/src/backend/internal/mod.rs @@ -24,13 +24,18 @@ pub mod lookup; pub mod manage; -use std::slice::Iter; +use std::{fmt::Display, slice::Iter}; use store::{write::key::KeySerializer, Deserialize, Serialize, U32_LEN}; use utils::codec::leb128::Leb128Iterator; use crate::{Principal, Type}; +pub(super) struct PrincipalIdType { + pub account_id: u32, + pub typ: Type, +} + impl Serialize for Principal<u32> { fn serialize(self) -> Vec<u8> { (&self).serialize() @@ -80,6 +85,35 @@ impl Deserialize for Principal<u32> { } } +impl Serialize for PrincipalIdType { + fn serialize(self) -> Vec<u8> { + KeySerializer::new(U32_LEN + 1) + .write_leb128(self.account_id) + .write(self.typ as u8) + .finalize() + } +} + +impl Deserialize for PrincipalIdType { + fn deserialize(bytes: &[u8]) -> store::Result<Self> { + let mut bytes = bytes.iter(); + Ok(PrincipalIdType { + account_id: bytes.next_leb128().ok_or_else(|| { + store::Error::InternalError("Failed to deserialize principal account id".into()) + })?, + typ: Type::from_u8(*bytes.next().ok_or_else(|| { + store::Error::InternalError("Failed to deserialize principal id type".into()) + })?), + }) + } +} + +impl PrincipalIdType { + pub fn new(account_id: u32, typ: Type) -> Self { + Self { account_id, typ } + } +} + fn deserialize(bytes: &[u8]) -> Option<Principal<u32>> { let mut bytes = bytes.iter(); if bytes.next()? != &1 { @@ -175,6 +209,20 @@ impl PrincipalUpdate { } } +impl Display for PrincipalField { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + PrincipalField::Name => write!(f, "name"), + PrincipalField::Type => write!(f, "type"), + PrincipalField::Quota => write!(f, "quota"), + PrincipalField::Description => write!(f, "description"), + PrincipalField::Secrets => write!(f, "secrets"), + PrincipalField::Emails => write!(f, "emails"), + PrincipalField::MemberOf => write!(f, "memberOf"), + } + } +} + fn deserialize_string(bytes: &mut Iter<'_, u8>) -> Option<String> { let len = bytes.next_leb128()?; let mut string = Vec::with_capacity(len); @@ -203,6 +251,17 @@ fn deserialize_u32_list(bytes: &mut Iter<'_, u8>) -> Option<Vec<u32>> { } impl Type { + pub fn parse(value: &str) -> Option<Self> { + match value { + "individual" | "superuser" => Some(Type::Individual), + "group" => Some(Type::Group), + "resource" => Some(Type::Resource), + "location" => Some(Type::Location), + "list" => Some(Type::List), + _ => None, + } + } + pub fn from_u8(value: u8) -> Self { match value { 0 => Type::Individual, @@ -214,4 +273,11 @@ impl Type { _ => Type::Other, } } + + pub fn into_base_type(self) -> Self { + match self { + Type::Superuser => Type::Individual, + any => any, + } + } } diff --git a/crates/directory/src/backend/ldap/config.rs b/crates/directory/src/backend/ldap/config.rs index ce60679e..c4bf563e 100644 --- a/crates/directory/src/backend/ldap/config.rs +++ b/crates/directory/src/backend/ldap/config.rs @@ -21,13 +21,11 @@ * for more details. */ -use std::sync::Arc; - use ldap3::LdapConnSettings; use store::Store; use utils::config::{utils::AsKey, Config}; -use crate::{cache::CachedDirectory, config::build_pool, Directory, DirectoryOptions}; +use crate::core::config::build_pool; use super::{Bind, LdapConnectionManager, LdapDirectory, LdapFilter, LdapMappings}; @@ -36,7 +34,7 @@ impl LdapDirectory { config: &Config, prefix: impl AsKey, id_store: Option<Store>, - ) -> utils::config::Result<Arc<dyn Directory>> { + ) -> utils::config::Result<Self> { let prefix = prefix.as_key(); let bind_dn = if let Some(dn) = config.value((&prefix, "bind.dn")) { Bind::new( @@ -52,9 +50,9 @@ impl LdapDirectory { config.value_require((&prefix, "address"))?.to_string(), LdapConnSettings::new() .set_conn_timeout(config.property_or_static((&prefix, "timeout"), "30s")?) - .set_starttls(config.property_or_static((&prefix, "tls"), "false")?) + .set_starttls(config.property_or_static((&prefix, "tls.enable"), "false")?) .set_no_tls_verify( - config.property_or_static((&prefix, "allow-invalid-certs"), "false")?, + config.property_or_static((&prefix, "tls.allow-invalid-certs"), "false")?, ), bind_dn, ); @@ -115,23 +113,18 @@ impl LdapDirectory { } let auth_bind = - if config.property_or_static::<bool>((&prefix, "auth-bind.enable"), "false")? { - LdapFilter::from_config(config, (&prefix, "auth-bind.dn"))?.into() + if config.property_or_static::<bool>((&prefix, "bind.auth.enable"), "false")? { + LdapFilter::from_config(config, (&prefix, "bind.auth.dn"))?.into() } else { None }; - CachedDirectory::try_from_config( - config, - &prefix, - LdapDirectory { - mappings, - pool: build_pool(config, &prefix, manager)?, - opt: DirectoryOptions::from_config(config, prefix.as_str())?, - auth_bind, - id_store, - }, - ) + Ok(LdapDirectory { + mappings, + pool: build_pool(config, &prefix, manager)?, + auth_bind, + id_store, + }) } } diff --git a/crates/directory/src/backend/ldap/lookup.rs b/crates/directory/src/backend/ldap/lookup.rs index 61e6146d..05bdcaf9 100644 --- a/crates/directory/src/backend/ldap/lookup.rs +++ b/crates/directory/src/backend/ldap/lookup.rs @@ -25,15 +25,12 @@ use ldap3::{Ldap, LdapConnAsync, LdapError, Scope, SearchEntry}; use mail_send::Credentials; use store::Store; -use crate::{ - backend::internal::manage::ManageDirectory, Directory, DirectoryError, Principal, QueryBy, Type, -}; +use crate::{backend::internal::manage::ManageDirectory, DirectoryError, Principal, QueryBy, Type}; use super::{LdapDirectory, LdapMappings}; -#[async_trait::async_trait] -impl Directory for LdapDirectory { - async fn query(&self, by: QueryBy<'_>) -> crate::Result<Option<Principal<u32>>> { +impl LdapDirectory { + pub async fn query(&self, by: QueryBy<'_>) -> crate::Result<Option<Principal<u32>>> { let mut conn = self.pool.get().await?; let mut account_id = None; let account_name; @@ -172,44 +169,21 @@ impl Directory for LdapDirectory { } } - async fn email_to_ids(&self, address: &str) -> crate::Result<Vec<u32>> { - let mut rs = self + pub async fn email_to_ids(&self, address: &str) -> crate::Result<Vec<u32>> { + let rs = self .pool .get() .await? .search( &self.mappings.base_dn, Scope::Subtree, - &self - .mappings - .filter_email - .build(self.opt.subaddressing.to_subaddress(address).as_ref()), + &self.mappings.filter_email.build(address.as_ref()), &self.mappings.attr_name, ) .await? .success() .map(|(rs, _res)| rs)?; - if rs.is_empty() { - if let Some(address) = self.opt.catch_all.to_catch_all(address) { - rs = self - .pool - .get() - .await? - .search( - &self.mappings.base_dn, - Scope::Subtree, - &self.mappings.filter_email.build(address.as_ref()), - &self.mappings.attr_name, - ) - .await? - .success() - .map(|(rs, _res)| rs)?; - } else { - return Ok(Vec::new()); - } - } - let mut ids = Vec::with_capacity(rs.len()); for entry in rs { let entry = SearchEntry::construct(entry); @@ -230,51 +204,24 @@ impl Directory for LdapDirectory { Ok(ids) } - async fn rcpt(&self, address: &str) -> crate::Result<bool> { - match self - .pool + pub async fn rcpt(&self, address: &str) -> crate::Result<bool> { + self.pool .get() .await? .streaming_search( &self.mappings.base_dn, Scope::Subtree, - &self - .mappings - .filter_email - .build(self.opt.subaddressing.to_subaddress(address).as_ref()), + &self.mappings.filter_email.build(address.as_ref()), &self.mappings.attr_email_address, ) .await? .next() .await - { - Ok(Some(_)) => Ok(true), - Ok(None) => { - if let Some(address) = self.opt.catch_all.to_catch_all(address) { - self.pool - .get() - .await? - .streaming_search( - &self.mappings.base_dn, - Scope::Subtree, - &self.mappings.filter_email.build(address.as_ref()), - &self.mappings.attr_email_address, - ) - .await? - .next() - .await - .map(|entry| entry.is_some()) - .map_err(|e| e.into()) - } else { - Ok(false) - } - } - - Err(e) => Err(e.into()), - } + .map(|entry| entry.is_some()) + .map_err(|e| e.into()) } - async fn vrfy(&self, address: &str) -> crate::Result<Vec<String>> { + pub async fn vrfy(&self, address: &str) -> crate::Result<Vec<String>> { let mut stream = self .pool .get() @@ -282,10 +229,7 @@ impl Directory for LdapDirectory { .streaming_search( &self.mappings.base_dn, Scope::Subtree, - &self - .mappings - .filter_verify - .build(self.opt.subaddressing.to_subaddress(address).as_ref()), + &self.mappings.filter_verify.build(address), &self.mappings.attr_email_address, ) .await?; @@ -307,7 +251,7 @@ impl Directory for LdapDirectory { Ok(emails) } - async fn expn(&self, address: &str) -> crate::Result<Vec<String>> { + pub async fn expn(&self, address: &str) -> crate::Result<Vec<String>> { let mut stream = self .pool .get() @@ -315,10 +259,7 @@ impl Directory for LdapDirectory { .streaming_search( &self.mappings.base_dn, Scope::Subtree, - &self - .mappings - .filter_expand - .build(self.opt.subaddressing.to_subaddress(address).as_ref()), + &self.mappings.filter_expand.build(address), &self.mappings.attr_email_address, ) .await?; @@ -340,7 +281,7 @@ impl Directory for LdapDirectory { Ok(emails) } - async fn is_local_domain(&self, domain: &str) -> crate::Result<bool> { + pub async fn is_local_domain(&self, domain: &str) -> crate::Result<bool> { self.pool .get() .await? diff --git a/crates/directory/src/backend/ldap/mod.rs b/crates/directory/src/backend/ldap/mod.rs index 06e2a667..b1a56721 100644 --- a/crates/directory/src/backend/ldap/mod.rs +++ b/crates/directory/src/backend/ldap/mod.rs @@ -25,8 +25,6 @@ use deadpool::managed::Pool; use ldap3::{ldap_escape, LdapConnSettings}; use store::Store; -use crate::DirectoryOptions; - pub mod config; pub mod lookup; pub mod pool; @@ -34,7 +32,6 @@ pub mod pool; pub struct LdapDirectory { pool: Pool<LdapConnectionManager>, mappings: LdapMappings, - opt: DirectoryOptions, auth_bind: Option<LdapFilter>, id_store: Option<Store>, } diff --git a/crates/directory/src/backend/memory/config.rs b/crates/directory/src/backend/memory/config.rs index cbcd692b..4da876a0 100644 --- a/crates/directory/src/backend/memory/config.rs +++ b/crates/directory/src/backend/memory/config.rs @@ -21,22 +21,23 @@ * for more details. */ -use std::sync::Arc; - +use ahash::AHashMap; +use store::Store; use utils::config::{utils::AsKey, Config}; -use crate::{Directory, DirectoryOptions, Principal, Type}; +use crate::{Principal, Type}; -use super::{EmailType, MemoryDirectory}; +use super::{EmailType, MemoryDirectory, NameToId}; impl MemoryDirectory { pub fn from_config( config: &Config, prefix: impl AsKey, - ) -> utils::config::Result<Arc<dyn Directory>> { + _: Option<Store>, + ) -> utils::config::Result<Self> { let prefix = prefix.as_key(); let mut directory = MemoryDirectory { - opt: DirectoryOptions::from_config(config, prefix.clone())?, + names_to_ids: NameToId::Internal(AHashMap::new()), ..Default::default() }; @@ -45,31 +46,31 @@ impl MemoryDirectory { .value_require((prefix.as_str(), "principals", lookup_id, "name"))? .to_string(); let typ = - match config.value_require((prefix.as_str(), "principals", lookup_id, "name"))? { + match config.value_require((prefix.as_str(), "principals", lookup_id, "type"))? { "individual" => Type::Individual, "admin" => Type::Superuser, "group" => Type::Group, - _ => Type::Other, + _ => Type::Individual, }; // Obtain id - let next_user_id = directory.names_to_ids.len() as u32; - let id = *directory - .names_to_ids - .entry(name.to_string()) - .or_insert(next_user_id); + let id = directory.names_to_ids.get_or_insert(&name).map_err(|err| { + format!( + "Failed to obtain id for principal {} ({}): {:?}", + name, lookup_id, err + ) + })?; // Obtain group ids let mut member_of = Vec::new(); for (_, group) in config.values((prefix.as_str(), "principals", lookup_id, "member-of")) { - let next_group_id = directory.names_to_ids.len() as u32; - member_of.push( - *directory - .names_to_ids - .entry(group.to_string()) - .or_insert(next_group_id), - ); + member_of.push(directory.names_to_ids.get_or_insert(group).map_err(|err| { + format!( + "Failed to obtain id for principal {} ({}): {:?}", + name, lookup_id, err + ) + })?); } // Parse email addresses @@ -128,6 +129,6 @@ impl MemoryDirectory { }); } - Ok(Arc::new(directory)) + Ok(directory) } } diff --git a/crates/directory/src/backend/memory/lookup.rs b/crates/directory/src/backend/memory/lookup.rs index ff80cc89..eb2bd398 100644 --- a/crates/directory/src/backend/memory/lookup.rs +++ b/crates/directory/src/backend/memory/lookup.rs @@ -23,13 +23,12 @@ use mail_send::Credentials; -use crate::{Directory, Principal, QueryBy}; +use crate::{Principal, QueryBy}; use super::{EmailType, MemoryDirectory}; -#[async_trait::async_trait] -impl Directory for MemoryDirectory { - async fn query(&self, by: QueryBy<'_>) -> crate::Result<Option<Principal<u32>>> { +impl MemoryDirectory { + pub async fn query(&self, by: QueryBy<'_>) -> crate::Result<Option<Principal<u32>>> { match by { QueryBy::Name(name) => { for principal in &self.principals { @@ -66,16 +65,10 @@ impl Directory for MemoryDirectory { Ok(None) } - async fn email_to_ids(&self, address: &str) -> crate::Result<Vec<u32>> { + pub async fn email_to_ids(&self, address: &str) -> crate::Result<Vec<u32>> { Ok(self .emails_to_ids - .get(self.opt.subaddressing.to_subaddress(address).as_ref()) - .or_else(|| { - self.opt - .catch_all - .to_catch_all(address) - .and_then(|address| self.emails_to_ids.get(address.as_ref())) - }) + .get(address) .map(|names| { names .iter() @@ -89,37 +82,24 @@ impl Directory for MemoryDirectory { .unwrap_or_default()) } - async fn rcpt(&self, address: &str) -> crate::Result<bool> { - Ok(self - .emails_to_ids - .contains_key(self.opt.subaddressing.to_subaddress(address).as_ref()) - || self - .opt - .catch_all - .to_catch_all(address) - .map_or(false, |address| { - self.emails_to_ids.contains_key(address.as_ref()) - })) + pub async fn rcpt(&self, address: &str) -> crate::Result<bool> { + Ok(self.emails_to_ids.contains_key(address)) } - async fn vrfy(&self, address: &str) -> crate::Result<Vec<String>> { + pub async fn vrfy(&self, address: &str) -> crate::Result<Vec<String>> { let mut result = Vec::new(); - let address = self.opt.subaddressing.to_subaddress(address); for (key, value) in &self.emails_to_ids { - if key.contains(address.as_ref()) - && value.iter().any(|t| matches!(t, EmailType::Primary(_))) - { + if key.contains(address) && value.iter().any(|t| matches!(t, EmailType::Primary(_))) { result.push(key.clone()) } } Ok(result) } - async fn expn(&self, address: &str) -> crate::Result<Vec<String>> { + pub async fn expn(&self, address: &str) -> crate::Result<Vec<String>> { let mut result = Vec::new(); - let address = self.opt.subaddressing.to_subaddress(address); for (key, value) in &self.emails_to_ids { - if key == address.as_ref() { + if key == address { for item in value { if let EmailType::List(uid) = item { for principal in &self.principals { @@ -137,7 +117,7 @@ impl Directory for MemoryDirectory { Ok(result) } - async fn is_local_domain(&self, domain: &str) -> crate::Result<bool> { + pub async fn is_local_domain(&self, domain: &str) -> crate::Result<bool> { Ok(self.domains.contains(domain)) } } diff --git a/crates/directory/src/backend/memory/mod.rs b/crates/directory/src/backend/memory/mod.rs index 973f31ba..4cec3783 100644 --- a/crates/directory/src/backend/memory/mod.rs +++ b/crates/directory/src/backend/memory/mod.rs @@ -22,8 +22,12 @@ */ use ahash::{AHashMap, AHashSet}; +use store::Store; +use tokio::sync::oneshot; -use crate::{DirectoryOptions, Principal}; +use crate::Principal; + +use super::internal::manage::ManageDirectory; pub mod config; pub mod lookup; @@ -32,9 +36,60 @@ pub mod lookup; pub struct MemoryDirectory { principals: Vec<Principal<u32>>, emails_to_ids: AHashMap<String, Vec<EmailType>>, - names_to_ids: AHashMap<String, u32>, + names_to_ids: NameToId, domains: AHashSet<String>, - opt: DirectoryOptions, +} + +pub enum NameToId { + Internal(AHashMap<String, u32>), + Store(Store), +} + +impl Default for NameToId { + fn default() -> Self { + Self::Internal(AHashMap::new()) + } +} + +impl std::fmt::Debug for NameToId { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Internal(arg0) => f.debug_tuple("Internal").field(arg0).finish(), + Self::Store(_) => f.debug_tuple("Store").finish(), + } + } +} + +impl From<Option<Store>> for NameToId { + fn from(store: Option<Store>) -> Self { + match store { + Some(store) => Self::Store(store), + None => Self::Internal(AHashMap::new()), + } + } +} + +impl NameToId { + pub fn get_or_insert(&mut self, name: &str) -> crate::Result<u32> { + match self { + Self::Internal(map) => { + let next_id = map.len() as u32; + Ok(*map.entry(name.to_string()).or_insert(next_id)) + } + Self::Store(store) => { + let (tx, rx) = oneshot::channel(); + let store = store.clone(); + let name = name.to_string(); + tokio::spawn(async move { + let _ = tx.send(store.get_or_create_account_id(&name).await); + }); + match rx.blocking_recv() { + Ok(result) => result, + Err(_) => Err(crate::DirectoryError::Unsupported), + } + } + } + } } #[derive(Debug)] diff --git a/crates/directory/src/backend/smtp/config.rs b/crates/directory/src/backend/smtp/config.rs index 6accb4b1..3b017133 100644 --- a/crates/directory/src/backend/smtp/config.rs +++ b/crates/directory/src/backend/smtp/config.rs @@ -21,12 +21,10 @@ * for more details. */ -use std::sync::Arc; - use mail_send::{smtp::tls::build_tls_connector, SmtpClientBuilder}; use utils::config::{utils::AsKey, Config}; -use crate::{cache::CachedDirectory, config::build_pool, Directory}; +use crate::core::config::build_pool; use super::{SmtpConnectionManager, SmtpDirectory}; @@ -35,7 +33,7 @@ impl SmtpDirectory { config: &Config, prefix: impl AsKey, is_lmtp: bool, - ) -> utils::config::Result<Arc<dyn Directory>> { + ) -> utils::config::Result<Self> { let prefix = prefix.as_key(); let address = config.value_require((&prefix, "address"))?; let tls_implicit: bool = config.property_or_static((&prefix, "tls.implicit"), "false")?; @@ -62,16 +60,12 @@ impl SmtpDirectory { max_auth_errors: config.property_or_static((&prefix, "limits.auth-errors"), "3")?, }; - CachedDirectory::try_from_config( - config, - &prefix, - SmtpDirectory { - pool: build_pool(config, &prefix, manager)?, - domains: config - .values((&prefix, "local-domains")) - .map(|(_, v)| v.to_lowercase()) - .collect(), - }, - ) + Ok(SmtpDirectory { + pool: build_pool(config, &prefix, manager)?, + domains: config + .values((&prefix, "lookup.domains")) + .map(|(_, v)| v.to_lowercase()) + .collect(), + }) } } diff --git a/crates/directory/src/backend/smtp/lookup.rs b/crates/directory/src/backend/smtp/lookup.rs index 5744f3a4..a6c25e17 100644 --- a/crates/directory/src/backend/smtp/lookup.rs +++ b/crates/directory/src/backend/smtp/lookup.rs @@ -24,13 +24,12 @@ use mail_send::{smtp::AssertReply, Credentials}; use smtp_proto::Severity; -use crate::{Directory, DirectoryError, Principal, QueryBy}; +use crate::{DirectoryError, Principal, QueryBy}; use super::{SmtpClient, SmtpDirectory}; -#[async_trait::async_trait] -impl Directory for SmtpDirectory { - async fn query(&self, query: QueryBy<'_>) -> crate::Result<Option<Principal<u32>>> { +impl SmtpDirectory { + pub async fn query(&self, query: QueryBy<'_>) -> crate::Result<Option<Principal<u32>>> { if let QueryBy::Credentials(credentials) = query { self.pool.get().await?.authenticate(credentials).await } else { @@ -38,11 +37,11 @@ impl Directory for SmtpDirectory { } } - async fn email_to_ids(&self, _address: &str) -> crate::Result<Vec<u32>> { + pub async fn email_to_ids(&self, _address: &str) -> crate::Result<Vec<u32>> { Err(DirectoryError::unsupported("smtp", "email_to_ids")) } - async fn rcpt(&self, address: &str) -> crate::Result<bool> { + pub async fn rcpt(&self, address: &str) -> crate::Result<bool> { let mut conn = self.pool.get().await?; if !conn.sent_mail_from { conn.client @@ -70,7 +69,7 @@ impl Directory for SmtpDirectory { } } - async fn vrfy(&self, address: &str) -> crate::Result<Vec<String>> { + pub async fn vrfy(&self, address: &str) -> crate::Result<Vec<String>> { self.pool .get() .await? @@ -78,7 +77,7 @@ impl Directory for SmtpDirectory { .await } - async fn expn(&self, address: &str) -> crate::Result<Vec<String>> { + pub async fn expn(&self, address: &str) -> crate::Result<Vec<String>> { self.pool .get() .await? @@ -86,7 +85,7 @@ impl Directory for SmtpDirectory { .await } - async fn is_local_domain(&self, domain: &str) -> crate::Result<bool> { + pub async fn is_local_domain(&self, domain: &str) -> crate::Result<bool> { Ok(self.domains.contains(domain)) } } diff --git a/crates/directory/src/backend/sql/config.rs b/crates/directory/src/backend/sql/config.rs index 2ed6eda8..709c593c 100644 --- a/crates/directory/src/backend/sql/config.rs +++ b/crates/directory/src/backend/sql/config.rs @@ -21,13 +21,9 @@ * for more details. */ -use std::sync::Arc; - use store::{Store, Stores}; use utils::config::{utils::AsKey, Config}; -use crate::{cache::CachedDirectory, Directory, DirectoryOptions}; - use super::{SqlDirectory, SqlMappings}; impl SqlDirectory { @@ -36,7 +32,7 @@ impl SqlDirectory { prefix: impl AsKey, stores: &Stores, id_store: Option<Store>, - ) -> utils::config::Result<Arc<dyn Directory>> { + ) -> utils::config::Result<Self> { let prefix = prefix.as_key(); let store_id = config.value_require((&prefix, "store"))?; let store = stores @@ -81,15 +77,10 @@ impl SqlDirectory { } } - CachedDirectory::try_from_config( - config, - &prefix, - SqlDirectory { - store, - mappings, - opt: DirectoryOptions::from_config(config, prefix.as_str())?, - id_store, - }, - ) + Ok(SqlDirectory { + store, + mappings, + id_store, + }) } } diff --git a/crates/directory/src/backend/sql/lookup.rs b/crates/directory/src/backend/sql/lookup.rs index 50960113..c57651a7 100644 --- a/crates/directory/src/backend/sql/lookup.rs +++ b/crates/directory/src/backend/sql/lookup.rs @@ -24,13 +24,12 @@ use mail_send::Credentials; use store::{NamedRows, Rows, Store, Value}; -use crate::{backend::internal::manage::ManageDirectory, Directory, Principal, QueryBy, Type}; +use crate::{backend::internal::manage::ManageDirectory, Principal, QueryBy, Type}; use super::{SqlDirectory, SqlMappings}; -#[async_trait::async_trait] -impl Directory for SqlDirectory { - async fn query(&self, by: QueryBy<'_>) -> crate::Result<Option<Principal<u32>>> { +impl SqlDirectory { + pub async fn query(&self, by: QueryBy<'_>) -> crate::Result<Option<Principal<u32>>> { let mut account_id = None; let account_name; let mut secret = None; @@ -143,31 +142,12 @@ impl Directory for SqlDirectory { Ok(Some(principal)) } - async fn email_to_ids(&self, address: &str) -> crate::Result<Vec<u32>> { - let mut names = self + pub async fn email_to_ids(&self, address: &str) -> crate::Result<Vec<u32>> { + let names = self .store - .query::<Rows>( - &self.mappings.query_recipients, - vec![self - .opt - .subaddressing - .to_subaddress(address) - .into_owned() - .into()], - ) + .query::<Rows>(&self.mappings.query_recipients, vec![address.into()]) .await?; - if names.rows.is_empty() { - if let Some(address) = self.opt.catch_all.to_catch_all(address) { - names = self - .store - .query::<Rows>(&self.mappings.query_recipients, vec![address.into()]) - .await?; - } else { - return Ok(vec![]); - } - } - let mut ids = Vec::with_capacity(names.rows.len()); for row in names.rows { @@ -183,67 +163,39 @@ impl Directory for SqlDirectory { Ok(ids) } - async fn rcpt(&self, address: &str) -> crate::Result<bool> { - if self - .store + pub async fn rcpt(&self, address: &str) -> crate::Result<bool> { + self.store .query::<bool>( &self.mappings.query_recipients, - vec![self - .opt - .subaddressing - .to_subaddress(address) - .into_owned() - .into()], + vec![address.to_string().into()], ) - .await? - { - Ok(true) - } else if let Some(address) = self.opt.catch_all.to_catch_all(address) { - self.store - .query::<bool>( - &self.mappings.query_recipients, - vec![address.into_owned().into()], - ) - .await - .map_err(Into::into) - } else { - Ok(false) - } + .await + .map_err(Into::into) } - async fn vrfy(&self, address: &str) -> crate::Result<Vec<String>> { + pub async fn vrfy(&self, address: &str) -> crate::Result<Vec<String>> { self.store .query::<Rows>( &self.mappings.query_verify, - vec![self - .opt - .subaddressing - .to_subaddress(address) - .into_owned() - .into()], + vec![address.to_string().into()], ) .await .map(Into::into) .map_err(Into::into) } - async fn expn(&self, address: &str) -> crate::Result<Vec<String>> { + pub async fn expn(&self, address: &str) -> crate::Result<Vec<String>> { self.store .query::<Rows>( &self.mappings.query_expand, - vec![self - .opt - .subaddressing - .to_subaddress(address) - .into_owned() - .into()], + vec![address.to_string().into()], ) .await .map(Into::into) .map_err(Into::into) } - async fn is_local_domain(&self, domain: &str) -> crate::Result<bool> { + pub async fn is_local_domain(&self, domain: &str) -> crate::Result<bool> { self.store .query::<bool>(&self.mappings.query_domains, vec![domain.into()]) .await diff --git a/crates/directory/src/backend/sql/mod.rs b/crates/directory/src/backend/sql/mod.rs index e081f8bc..46b6a1fb 100644 --- a/crates/directory/src/backend/sql/mod.rs +++ b/crates/directory/src/backend/sql/mod.rs @@ -23,15 +23,12 @@ use store::{LookupStore, Store}; -use crate::DirectoryOptions; - pub mod config; pub mod lookup; pub struct SqlDirectory { store: LookupStore, mappings: SqlMappings, - opt: DirectoryOptions, id_store: Option<Store>, } diff --git a/crates/directory/src/cache/config.rs b/crates/directory/src/cache/config.rs deleted file mode 100644 index 8c553954..00000000 --- a/crates/directory/src/cache/config.rs +++ /dev/null @@ -1,64 +0,0 @@ -/* - * Copyright (c) 2023 Stalwart Labs Ltd. - * - * This file is part of Stalwart Mail Server. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * in the LICENSE file at the top-level directory of this distribution. - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. - * - * You can be released from the requirements of the AGPLv3 license by - * purchasing a commercial license. Please contact licensing@stalw.art - * for more details. -*/ - -use std::{sync::Arc, time::Duration}; - -use parking_lot::lock_api::Mutex; -use utils::config::Config; - -use crate::Directory; - -use super::{lru::LookupCache, CachedDirectory}; - -impl<T: Directory + 'static> CachedDirectory<T> { - pub fn try_from_config( - config: &Config, - prefix: &str, - inner: T, - ) -> utils::config::Result<Arc<dyn Directory>> { - if let Some(cached_entries) = config.property((prefix, "cache.entries"))? { - let cache_ttl_positive = config - .property((prefix, "cache.ttl.positive"))? - .unwrap_or(Duration::from_secs(86400)); - let cache_ttl_negative = config - .property((prefix, "cache.ttl.positive"))? - .unwrap_or_else(|| Duration::from_secs(3600)); - - Ok(Arc::new(CachedDirectory { - inner, - cached_domains: Mutex::new(LookupCache::new( - cached_entries, - cache_ttl_positive, - cache_ttl_negative, - )), - cached_rcpts: Mutex::new(LookupCache::new( - cached_entries, - cache_ttl_positive, - cache_ttl_negative, - )), - })) - } else { - Ok(Arc::new(inner)) - } - } -} diff --git a/crates/directory/src/cache/lookup.rs b/crates/directory/src/cache/lookup.rs deleted file mode 100644 index 6729a769..00000000 --- a/crates/directory/src/cache/lookup.rs +++ /dev/null @@ -1,75 +0,0 @@ -/* - * Copyright (c) 2023 Stalwart Labs Ltd. - * - * This file is part of Stalwart Mail Server. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * in the LICENSE file at the top-level directory of this distribution. - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. - * - * You can be released from the requirements of the AGPLv3 license by - * purchasing a commercial license. Please contact licensing@stalw.art - * for more details. -*/ - -use crate::{Directory, Principal, QueryBy}; - -use super::CachedDirectory; - -#[async_trait::async_trait] -impl<T: Directory> Directory for CachedDirectory<T> { - async fn query(&self, by: QueryBy<'_>) -> crate::Result<Option<Principal<u32>>> { - self.inner.query(by).await - } - - async fn email_to_ids(&self, address: &str) -> crate::Result<Vec<u32>> { - self.inner.email_to_ids(address).await - } - - async fn rcpt(&self, address: &str) -> crate::Result<bool> { - if let Some(result) = { - let result = self.cached_rcpts.lock().get(address); - result - } { - Ok(result) - } else if self.inner.rcpt(address).await? { - self.cached_rcpts.lock().insert_pos(address.to_string()); - Ok(true) - } else { - self.cached_rcpts.lock().insert_neg(address.to_string()); - Ok(false) - } - } - - async fn vrfy(&self, address: &str) -> crate::Result<Vec<String>> { - self.inner.vrfy(address).await - } - - async fn expn(&self, address: &str) -> crate::Result<Vec<String>> { - self.inner.expn(address).await - } - - async fn is_local_domain(&self, domain: &str) -> crate::Result<bool> { - if let Some(result) = { - let result = self.cached_domains.lock().get(domain); - result - } { - Ok(result) - } else if self.inner.is_local_domain(domain).await? { - self.cached_domains.lock().insert_pos(domain.to_string()); - Ok(true) - } else { - self.cached_domains.lock().insert_neg(domain.to_string()); - Ok(false) - } - } -} diff --git a/crates/directory/src/cache/lru.rs b/crates/directory/src/core/cache.rs index 2f3baf58..8c96780d 100644 --- a/crates/directory/src/cache/lru.rs +++ b/crates/directory/src/core/cache.rs @@ -27,6 +27,14 @@ use std::{ time::{Duration, Instant}, }; +use parking_lot::Mutex; +use utils::config::{utils::AsKey, Config}; + +pub struct CachedDirectory { + cached_domains: Mutex<LookupCache<String>>, + cached_rcpts: Mutex<LookupCache<String>>, +} + #[allow(clippy::type_complexity)] #[derive(Debug)] pub struct LookupCache<T: Hash + Eq> { @@ -36,6 +44,62 @@ pub struct LookupCache<T: Hash + Eq> { ttl_neg: Duration, } +impl CachedDirectory { + pub fn try_from_config( + config: &Config, + prefix: impl AsKey, + ) -> utils::config::Result<Option<Self>> { + let prefix = prefix.as_key(); + if let Some(cached_entries) = config.property((&prefix, "cache.entries"))? { + let cache_ttl_positive = config + .property((&prefix, "cache.ttl.positive"))? + .unwrap_or(Duration::from_secs(86400)); + let cache_ttl_negative = config + .property((&prefix, "cache.ttl.positive"))? + .unwrap_or_else(|| Duration::from_secs(3600)); + + Ok(Some(CachedDirectory { + cached_domains: Mutex::new(LookupCache::new( + cached_entries, + cache_ttl_positive, + cache_ttl_negative, + )), + cached_rcpts: Mutex::new(LookupCache::new( + cached_entries, + cache_ttl_positive, + cache_ttl_negative, + )), + })) + } else { + Ok(None) + } + } + + pub fn get_rcpt(&self, address: &str) -> Option<bool> { + self.cached_rcpts.lock().get(address) + } + + pub fn set_rcpt(&self, address: &str, exists: bool) { + if exists { + self.cached_rcpts.lock().insert_pos(address.to_string()); + } else { + self.cached_rcpts.lock().insert_neg(address.to_string()); + } + } + + pub fn get_domain(&self, domain: &str) -> Option<bool> { + self.cached_domains.lock().get(domain) + } + + pub fn set_domain(&self, domain: &str, exists: bool) { + if exists { + self.cached_domains.lock().insert_pos(domain.to_string()); + } else { + self.cached_domains.lock().insert_neg(domain.to_string()); + } + } +} + impl<T: Hash + Eq> LookupCache<T> { pub fn new(capacity: usize, ttl_pos: Duration, ttl_neg: Duration) -> Self { Self { diff --git a/crates/directory/src/config.rs b/crates/directory/src/core/config.rs index d86c633b..74c09ee5 100644 --- a/crates/directory/src/config.rs +++ b/crates/directory/src/core/config.rs @@ -26,7 +26,7 @@ use deadpool::{ Runtime, }; use regex::Regex; -use std::time::Duration; +use std::{sync::Arc, time::Duration}; use store::Stores; use utils::config::{ utils::{AsKey, ParseValue}, @@ -40,9 +40,11 @@ use crate::{ imap::ImapDirectory, ldap::LdapDirectory, memory::MemoryDirectory, smtp::SmtpDirectory, sql::SqlDirectory, }, - AddressMapping, Directories, DirectoryOptions, + AddressMapping, Directories, Directory, DirectoryInner, }; +use super::cache::CachedDirectory; + pub trait ConfigDirectory { fn parse_directory( &self, @@ -66,35 +68,65 @@ impl ConfigDirectory for Config { // Parse directory let protocol = self.value_require(("directory", id, "type"))?; let prefix = ("directory", id); - let directory = match protocol { - "ldap" => LdapDirectory::from_config(self, prefix, id_store.clone())?, - "sql" => SqlDirectory::from_config(self, prefix, stores, id_store.clone())?, - "imap" => ImapDirectory::from_config(self, prefix)?, - "smtp" => SmtpDirectory::from_config(self, prefix, false)?, - "lmtp" => SmtpDirectory::from_config(self, prefix, true)?, - "memory" => MemoryDirectory::from_config(self, prefix)?, + let store = match protocol { + "internal" => DirectoryInner::Internal( + stores + .stores + .get(self.value_require(("directory", id, "store"))?) + .cloned() + .ok_or_else(|| { + format!( + "Failed to find store {:?} for directory {:?}.", + self.value_require(("directory", id, "store")).unwrap(), + id + ) + })?, + ), + "ldap" => DirectoryInner::Ldap(LdapDirectory::from_config( + self, + prefix, + id_store.clone(), + )?), + "sql" => DirectoryInner::Sql(SqlDirectory::from_config( + self, + prefix, + stores, + id_store.clone(), + )?), + "imap" => DirectoryInner::Imap(ImapDirectory::from_config(self, prefix)?), + "smtp" => DirectoryInner::Smtp(SmtpDirectory::from_config(self, prefix, false)?), + "lmtp" => DirectoryInner::Smtp(SmtpDirectory::from_config(self, prefix, true)?), + "memory" => DirectoryInner::Memory(MemoryDirectory::from_config( + self, + prefix, + id_store.clone(), + )?), unknown => { return Err(format!("Unknown directory type: {unknown:?}")); } }; - config.directories.insert(id.to_string(), directory); + config.directories.insert( + id.to_string(), + Arc::new(Directory { + store, + catch_all: AddressMapping::from_config( + self, + ("directory", id, "options.catch-all"), + )?, + subaddressing: AddressMapping::from_config( + self, + ("directory", id, "options.subaddressing"), + )?, + cache: CachedDirectory::try_from_config(self, ("directory", id))?, + }), + ); } Ok(config) } } -impl DirectoryOptions { - pub fn from_config(config: &Config, key: impl AsKey) -> utils::config::Result<Self> { - let key = key.as_key(); - Ok(DirectoryOptions { - catch_all: AddressMapping::from_config(config, (&key, "options.catch-all"))?, - subaddressing: AddressMapping::from_config(config, (&key, "options.subaddressing"))?, - }) - } -} - impl AddressMapping { pub fn from_config(config: &Config, key: impl AsKey) -> utils::config::Result<Self> { let key = key.as_key(); @@ -134,11 +166,11 @@ pub(crate) fn build_pool<M: Manager>( .max_size(config.property_or_static((prefix, "pool.max-connections"), "10")?) .create_timeout( config - .property_or_static::<Duration>((prefix, "pool.create-timeout"), "30s")? + .property_or_static::<Duration>((prefix, "pool.timeout.create"), "30s")? .into(), ) - .wait_timeout(config.property_or_static((prefix, "pool.wait-timeout"), "30s")?) - .recycle_timeout(config.property_or_static((prefix, "pool.recycle-timeout"), "30s")?) + .wait_timeout(config.property_or_static((prefix, "pool.timeout.wait"), "30s")?) + .recycle_timeout(config.property_or_static((prefix, "pool.timeout.recycle"), "30s")?) .build() .map_err(|err| { format!( diff --git a/crates/directory/src/core/dispatch.rs b/crates/directory/src/core/dispatch.rs new file mode 100644 index 00000000..510aea7f --- /dev/null +++ b/crates/directory/src/core/dispatch.rs @@ -0,0 +1,160 @@ +/* + * Copyright (c) 2023 Stalwart Labs Ltd. + * + * This file is part of Stalwart Mail Server. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * in the LICENSE file at the top-level directory of this distribution. + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * + * You can be released from the requirements of the AGPLv3 license by + * purchasing a commercial license. Please contact licensing@stalw.art + * for more details. +*/ + +use crate::{ + backend::internal::lookup::DirectoryStore, Directory, DirectoryInner, Principal, QueryBy, +}; + +impl Directory { + pub async fn query(&self, by: QueryBy<'_>) -> crate::Result<Option<Principal<u32>>> { + match &self.store { + DirectoryInner::Internal(store) => store.query(by).await, + DirectoryInner::Ldap(store) => store.query(by).await, + DirectoryInner::Sql(store) => store.query(by).await, + DirectoryInner::Imap(store) => store.query(by).await, + DirectoryInner::Smtp(store) => store.query(by).await, + DirectoryInner::Memory(store) => store.query(by).await, + } + } + + pub async fn email_to_ids(&self, email: &str) -> crate::Result<Vec<u32>> { + let mut address = self.subaddressing.to_subaddress(email); + for _ in 0..2 { + let result = match &self.store { + DirectoryInner::Internal(store) => store.email_to_ids(address.as_ref()).await, + DirectoryInner::Ldap(store) => store.email_to_ids(address.as_ref()).await, + DirectoryInner::Sql(store) => store.email_to_ids(address.as_ref()).await, + DirectoryInner::Imap(store) => store.email_to_ids(address.as_ref()).await, + DirectoryInner::Smtp(store) => store.email_to_ids(address.as_ref()).await, + DirectoryInner::Memory(store) => store.email_to_ids(address.as_ref()).await, + }?; + + if !result.is_empty() { + return Ok(result); + } else if let Some(catch_all) = self.catch_all.to_catch_all(email) { + address = catch_all; + } else { + break; + } + } + + Ok(vec![]) + } + + pub async fn is_local_domain(&self, domain: &str) -> crate::Result<bool> { + // Check cache + if let Some(cache) = &self.cache { + if let Some(result) = cache.get_domain(domain) { + return Ok(result); + } + } + + let result = match &self.store { + DirectoryInner::Internal(store) => store.is_local_domain(domain).await, + DirectoryInner::Ldap(store) => store.is_local_domain(domain).await, + DirectoryInner::Sql(store) => store.is_local_domain(domain).await, + DirectoryInner::Imap(store) => store.is_local_domain(domain).await, + DirectoryInner::Smtp(store) => store.is_local_domain(domain).await, + DirectoryInner::Memory(store) => store.is_local_domain(domain).await, + }?; + + // Update cache + if let Some(cache) = &self.cache { + cache.set_domain(domain, result); + } + + Ok(result) + } + + pub async fn rcpt(&self, email: &str) -> crate::Result<bool> { + // Expand subaddress + let mut address = self.subaddressing.to_subaddress(email); + + // Check cache + if let Some(cache) = &self.cache { + if let Some(result) = cache.get_rcpt(address.as_ref()) { + return Ok(result); + } + } + + for _ in 0..2 { + let result = match &self.store { + DirectoryInner::Internal(store) => store.rcpt(address.as_ref()).await, + DirectoryInner::Ldap(store) => store.rcpt(address.as_ref()).await, + DirectoryInner::Sql(store) => store.rcpt(address.as_ref()).await, + DirectoryInner::Imap(store) => store.rcpt(address.as_ref()).await, + DirectoryInner::Smtp(store) => store.rcpt(address.as_ref()).await, + DirectoryInner::Memory(store) => store.rcpt(address.as_ref()).await, + }?; + + if result { + // Update cache + if let Some(cache) = &self.cache { + cache.set_rcpt(address.as_ref(), true); + } + return Ok(true); + } else if let Some(catch_all) = self.catch_all.to_catch_all(email) { + // Check cache + if let Some(cache) = &self.cache { + if let Some(result) = cache.get_rcpt(catch_all.as_ref()) { + return Ok(result); + } + } + address = catch_all; + } else { + break; + } + } + + // Update cache + if let Some(cache) = &self.cache { + cache.set_rcpt(address.as_ref(), false); + } + + Ok(false) + } + + pub async fn vrfy(&self, address: &str) -> crate::Result<Vec<String>> { + let address = self.subaddressing.to_subaddress(address); + match &self.store { + DirectoryInner::Internal(store) => store.vrfy(address.as_ref()).await, + DirectoryInner::Ldap(store) => store.vrfy(address.as_ref()).await, + DirectoryInner::Sql(store) => store.vrfy(address.as_ref()).await, + DirectoryInner::Imap(store) => store.vrfy(address.as_ref()).await, + DirectoryInner::Smtp(store) => store.vrfy(address.as_ref()).await, + DirectoryInner::Memory(store) => store.vrfy(address.as_ref()).await, + } + } + + pub async fn expn(&self, address: &str) -> crate::Result<Vec<String>> { + let address = self.subaddressing.to_subaddress(address); + match &self.store { + DirectoryInner::Internal(store) => store.expn(address.as_ref()).await, + DirectoryInner::Ldap(store) => store.expn(address.as_ref()).await, + DirectoryInner::Sql(store) => store.expn(address.as_ref()).await, + DirectoryInner::Imap(store) => store.expn(address.as_ref()).await, + DirectoryInner::Smtp(store) => store.expn(address.as_ref()).await, + DirectoryInner::Memory(store) => store.expn(address.as_ref()).await, + } + } +} diff --git a/crates/directory/src/cache/mod.rs b/crates/directory/src/core/mod.rs index 6c2a39db..8742eb5e 100644 --- a/crates/directory/src/cache/mod.rs +++ b/crates/directory/src/core/mod.rs @@ -21,18 +21,20 @@ * for more details. */ -use parking_lot::Mutex; - -use crate::Directory; - -use self::lru::LookupCache; +use crate::{backend::memory::MemoryDirectory, AddressMapping, Directory, DirectoryInner}; +pub mod cache; pub mod config; -pub mod lookup; -pub mod lru; +pub mod dispatch; +pub mod secret; -pub struct CachedDirectory<T: Directory> { - inner: T, - cached_domains: Mutex<LookupCache<String>>, - cached_rcpts: Mutex<LookupCache<String>>, +impl Default for Directory { + fn default() -> Self { + Directory { + store: DirectoryInner::Memory(MemoryDirectory::default()), + catch_all: AddressMapping::Disable, + subaddressing: AddressMapping::Disable, + cache: None, + } + } } diff --git a/crates/directory/src/secret.rs b/crates/directory/src/core/secret.rs index 4cb0bb7c..4cb0bb7c 100644 --- a/crates/directory/src/secret.rs +++ b/crates/directory/src/core/secret.rs diff --git a/crates/directory/src/lib.rs b/crates/directory/src/lib.rs index 95e8d13c..4608d138 100644 --- a/crates/directory/src/lib.rs +++ b/crates/directory/src/lib.rs @@ -21,19 +21,33 @@ * for more details. */ +use core::cache::CachedDirectory; use std::{borrow::Cow, fmt::Debug, sync::Arc}; use ahash::AHashMap; -use backend::{imap::ImapError, internal::PrincipalField}; +use backend::{ + imap::{ImapDirectory, ImapError}, + internal::PrincipalField, + ldap::LdapDirectory, + memory::MemoryDirectory, + smtp::SmtpDirectory, + sql::SqlDirectory, +}; use deadpool::managed::PoolError; use ldap3::LdapError; use mail_send::Credentials; +use store::Store; use utils::config::DynValue; pub mod backend; -pub mod cache; -pub mod config; -pub mod secret; +pub mod core; + +pub struct Directory { + store: DirectoryInner, + catch_all: AddressMapping, + subaddressing: AddressMapping, + cache: Option<CachedDirectory>, +} #[derive(Debug, Default, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)] pub struct Principal<T> { @@ -44,7 +58,7 @@ pub struct Principal<T> { #[serde(default)] pub quota: u32, pub name: String, - #[serde(default, skip_serializing)] + #[serde(default)] pub secrets: Vec<String>, #[serde(default)] pub emails: Vec<String>, @@ -58,6 +72,7 @@ pub struct Principal<T> { #[derive(Debug, Default, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize)] pub enum Type { #[serde(rename = "individual")] + #[default] Individual = 0, #[serde(rename = "group")] Group = 1, @@ -69,7 +84,6 @@ pub enum Type { Superuser = 4, #[serde(rename = "list")] List = 5, - #[default] #[serde(rename = "other")] Other = 6, } @@ -89,19 +103,20 @@ pub enum DirectoryError { #[derive(Debug, PartialEq, Eq)] pub enum ManagementError { MissingField(PrincipalField), - NotUniqueField(PrincipalField), + AlreadyExists { + field: PrincipalField, + value: String, + }, NotFound(String), } -#[async_trait::async_trait] -pub trait Directory: Sync + Send { - async fn query(&self, by: QueryBy<'_>) -> Result<Option<Principal<u32>>>; - async fn email_to_ids(&self, email: &str) -> Result<Vec<u32>>; - - async fn is_local_domain(&self, domain: &str) -> crate::Result<bool>; - async fn rcpt(&self, address: &str) -> crate::Result<bool>; - async fn vrfy(&self, address: &str) -> Result<Vec<String>>; - async fn expn(&self, address: &str) -> Result<Vec<String>>; +pub enum DirectoryInner { + Internal(Store), + Ldap(LdapDirectory), + Sql(SqlDirectory), + Imap(ImapDirectory), + Smtp(SmtpDirectory), + Memory(MemoryDirectory), } pub enum QueryBy<'x> { @@ -124,7 +139,7 @@ impl<T: serde::Serialize + serde::de::DeserializeOwned> Principal<T> { } } -impl Debug for dyn Directory { +impl Debug for Directory { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.debug_struct("Directory").finish() } @@ -144,12 +159,6 @@ impl Type { } #[derive(Debug, Default)] -struct DirectoryOptions { - catch_all: AddressMapping, - subaddressing: AddressMapping, -} - -#[derive(Debug, Default)] pub enum AddressMapping { Enable, Custom { @@ -162,7 +171,7 @@ pub enum AddressMapping { #[derive(Default, Clone, Debug)] pub struct Directories { - pub directories: AHashMap<String, Arc<dyn Directory>>, + pub directories: AHashMap<String, Arc<Directory>>, } pub type Result<T> = std::result::Result<T, DirectoryError>; diff --git a/crates/jmap/Cargo.toml b/crates/jmap/Cargo.toml index 9412f097..14c989c6 100644 --- a/crates/jmap/Cargo.toml +++ b/crates/jmap/Cargo.toml @@ -37,8 +37,8 @@ hkdf = "0.12.3" sha1 = "0.10" sha2 = "0.10" reqwest = { version = "0.11", default-features = false, features = ["rustls-tls-webpki-roots"]} -tokio-tungstenite = "0.20" -tungstenite = "0.20" +tokio-tungstenite = "0.21" +tungstenite = "0.21" chrono = "0.4" dashmap = "5.4" aes = "0.8.3" diff --git a/crates/jmap/src/api/admin.rs b/crates/jmap/src/api/admin.rs index a832c8c7..c675bdf9 100644 --- a/crates/jmap/src/api/admin.rs +++ b/crates/jmap/src/api/admin.rs @@ -22,8 +22,8 @@ */ use directory::{ - backend::internal::{manage::ManageDirectory, PrincipalUpdate}, - Directory, DirectoryError, ManagementError, Principal, QueryBy, Type, + backend::internal::{lookup::DirectoryStore, manage::ManageDirectory, PrincipalUpdate}, + DirectoryError, ManagementError, Principal, QueryBy, Type, }; use http_body_util::combinators::BoxBody; use hyper::{body::Bytes, Method, StatusCode}; @@ -44,6 +44,7 @@ pub struct PrincipalResponse { pub used_quota: u32, pub name: String, pub emails: Vec<String>, + pub secrets: Vec<String>, #[serde(rename = "memberOf")] pub member_of: Vec<String>, pub description: Option<String>, @@ -67,8 +68,7 @@ impl JMAP { { match self.store.create_account(principal).await { Ok(account_id) => JsonResponse::new(json!({ - "accountId": account_id, - "status": "success", + "data": account_id, })) .into_http_response(), Err(err) => map_directory_error(err), @@ -85,6 +85,7 @@ impl JMAP { ("principal", None, &Method::GET) => { // List principal ids let mut from_key = None; + let mut typ = None; let mut limit: usize = 0; if let Some(query) = req.uri().query() { @@ -93,6 +94,9 @@ impl JMAP { "limit" => { limit = value.parse().unwrap_or_default(); } + "type" => { + typ = Type::parse(value.as_ref()); + } "from" => { from_key = value.into(); } @@ -101,9 +105,12 @@ impl JMAP { } } - match self.store.list_accounts(from_key.as_deref(), limit).await { + match self + .store + .list_accounts(from_key.as_deref(), typ, limit) + .await + { Ok(accounts) => JsonResponse::new(json!({ - "status": "success", "data": accounts, })) .into_http_response(), @@ -151,7 +158,6 @@ impl JMAP { as u32; JsonResponse::new(json!({ - "status": "success", "data": principal, })) .into_http_response() @@ -179,13 +185,13 @@ impl JMAP { // Delete account match self.store.delete_account(QueryBy::Id(account_id)).await { Ok(_) => JsonResponse::new(json!({ - "status": "success", + "data": [], })) .into_http_response(), Err(err) => map_directory_error(err), } } - Method::PUT => { + Method::PATCH => { if let Some(changes) = body.and_then(|body| { serde_json::from_slice::<Vec<PrincipalUpdate>>(&body).ok() }) { @@ -195,8 +201,7 @@ impl JMAP { .await { Ok(account_id) => JsonResponse::new(json!({ - "accountId": account_id, - "status": "success", + "data": account_id, })) .into_http_response(), Err(err) => map_directory_error(err), @@ -213,11 +218,58 @@ impl JMAP { _ => RequestError::not_found().into_http_response(), } } - ("store", Some("purge"), &Method::GET) => { + ("domain", None, &Method::GET) => { + // List principal ids + let mut from_key = None; + let mut limit: usize = 0; + + if let Some(query) = req.uri().query() { + for (key, value) in form_urlencoded::parse(query.as_bytes()) { + match key.as_ref() { + "limit" => { + limit = value.parse().unwrap_or_default(); + } + "from" => { + from_key = value.into(); + } + _ => {} + } + } + } + + match self.store.list_domains(from_key.as_deref(), limit).await { + Ok(domains) => JsonResponse::new(json!({ + "data": domains, + })) + .into_http_response(), + Err(err) => map_directory_error(err), + } + } + ("domain", Some(domain), &Method::POST) => { + // Create domain + match self.store.create_domain(domain).await { + Ok(_) => JsonResponse::new(json!({ + "data": [], + })) + .into_http_response(), + Err(err) => map_directory_error(err), + } + } + ("domain", Some(domain), &Method::DELETE) => { + // Delete domain + match self.store.delete_domain(domain).await { + Ok(_) => JsonResponse::new(json!({ + "data": [], + })) + .into_http_response(), + Err(err) => map_directory_error(err), + } + } + ("store", Some("maintenance"), &Method::GET) => { match self.store.purge_blobs(self.blob_store.clone()).await { Ok(_) => match self.store.purge_bitmaps().await { Ok(_) => JsonResponse::new(json!({ - "status": "success", + "data": [], })) .into_http_response(), Err(err) => RequestError::blank( @@ -249,23 +301,27 @@ fn map_directory_error(err: DirectoryError) -> hyper::Response<BoxBody<Bytes, hy match err { DirectoryError::Management(err) => { let response = match err { - ManagementError::MissingField(details) => json!({ - "status": "missingField", - "details": details, + ManagementError::MissingField(field) => json!({ + "error": "missingField", + "field": field, + "details": format!("Missing required field '{field}'."), }), - ManagementError::NotUniqueField(details) => json!({ - "status": "notUniqueField", - "details": details, + ManagementError::AlreadyExists { field, value } => json!({ + "error": "alreadyExists", + "field": field, + "value": value, + "details": format!("Another record exists containing '{value}' in the '{field}' field."), }), ManagementError::NotFound(details) => json!({ - "status": "notFound", - "details": details, + "error": "notFound", + "item": details, + "details": format!("'{details}' does not exist."), }), }; JsonResponse::new(response).into_http_response() } DirectoryError::Unsupported => JsonResponse::new(json!({ - "status": "unsupported", + "error": "unsupported", "details": "Requested action is unsupported", })) .into_http_response(), @@ -297,6 +353,7 @@ impl From<Principal<String>> for PrincipalResponse { emails: principal.emails, member_of: principal.member_of, description: principal.description, + secrets: principal.secrets, used_quota: 0, } } diff --git a/crates/jmap/src/lib.rs b/crates/jmap/src/lib.rs index e09fca66..90d0e4c5 100644 --- a/crates/jmap/src/lib.rs +++ b/crates/jmap/src/lib.rs @@ -88,7 +88,7 @@ pub struct JMAP { pub blob_store: BlobStore, pub fts_store: FtsStore, pub config: Config, - pub directory: Arc<dyn Directory>, + pub directory: Arc<Directory>, pub sessions: TtlDashMap<String, u32>, pub access_tokens: TtlDashMap<u32, Arc<AccessToken>>, @@ -203,7 +203,7 @@ impl JMAP { )) .clone(), snowflake_id: config - .property::<u64>("global.node-id")? + .property::<u64>("jmap.cluster.node-id")? .map(SnowflakeIdGenerator::with_node_id) .unwrap_or_else(SnowflakeIdGenerator::new), store: stores diff --git a/crates/jmap/src/services/housekeeper.rs b/crates/jmap/src/services/housekeeper.rs index b3e1af85..6776fe22 100644 --- a/crates/jmap/src/services/housekeeper.rs +++ b/crates/jmap/src/services/housekeeper.rs @@ -45,7 +45,7 @@ pub enum Event { pub fn spawn_housekeeper(core: Arc<JMAP>, settings: &Config, mut rx: mpsc::Receiver<Event>) { let purge_cache = settings - .property_or_static::<SimpleCron>("jmap.purge.schedule.sessions", "15 * *") + .property_or_static::<SimpleCron>("jmap.session.purge.frequency", "15 * *") .failed("Initialize housekeeper"); tokio::spawn(async move { diff --git a/crates/main/src/main.rs b/crates/main/src/main.rs index dec0dfc6..0a9f2204 100644 --- a/crates/main/src/main.rs +++ b/crates/main/src/main.rs @@ -23,7 +23,7 @@ use std::time::Duration; -use directory::config::ConfigDirectory; +use directory::core::config::ConfigDirectory; use imap::core::{ImapSessionManager, IMAP}; use jmap::{api::JmapSessionManager, services::IPC_CHANNEL_BUFFER, JMAP}; use managesieve::core::ManageSieveSessionManager; diff --git a/crates/smtp/src/config/mod.rs b/crates/smtp/src/config/mod.rs index c5cff22a..92da2458 100644 --- a/crates/smtp/src/config/mod.rs +++ b/crates/smtp/src/config/mod.rs @@ -223,7 +223,7 @@ pub struct Extensions { } pub struct Auth { - pub directory: IfBlock<Option<MaybeDynValue<dyn Directory>>>, + pub directory: IfBlock<Option<MaybeDynValue<Directory>>>, pub mechanisms: IfBlock<u64>, pub require: IfBlock<bool>, pub allow_plain_text: IfBlock<bool>, @@ -239,7 +239,7 @@ pub struct Mail { pub struct Rcpt { pub script: IfBlock<Option<Arc<Sieve>>>, pub relay: IfBlock<bool>, - pub directory: IfBlock<Option<MaybeDynValue<dyn Directory>>>, + pub directory: IfBlock<Option<MaybeDynValue<Directory>>>, pub rewrite: IfBlock<Option<DynValue<EnvelopeKey>>>, // Errors @@ -347,7 +347,7 @@ pub struct QueueConfig { // Throttle and Quotas pub throttle: QueueThrottle, pub quota: QueueQuotas, - pub management_lookup: Arc<dyn Directory>, + pub management_lookup: Arc<Directory>, } pub struct QueueOutboundSourceIp { diff --git a/crates/smtp/src/config/queue.rs b/crates/smtp/src/config/queue.rs index de950842..93a1c668 100644 --- a/crates/smtp/src/config/queue.rs +++ b/crates/smtp/src/config/queue.rs @@ -23,7 +23,6 @@ use std::time::Duration; -use directory::backend::memory::MemoryDirectory; use mail_send::Credentials; use super::{ @@ -213,7 +212,7 @@ impl ConfigQueue for Config { })? .clone() } else { - Arc::new(MemoryDirectory::default()) + Arc::new(Directory::default()) }, }; diff --git a/crates/smtp/src/core/mod.rs b/crates/smtp/src/core/mod.rs index db50c672..b01eb2c9 100644 --- a/crates/smtp/src/core/mod.rs +++ b/crates/smtp/src/core/mod.rs @@ -117,7 +117,7 @@ pub struct SieveCore { pub from_name: String, pub return_path: String, pub sign: Vec<Arc<DkimSigner>>, - pub directories: AHashMap<String, Arc<dyn Directory>>, + pub directories: AHashMap<String, Arc<Directory>>, pub lookup_stores: AHashMap<String, LookupStore>, pub lookup: AHashMap<String, Lookup>, } @@ -229,7 +229,7 @@ pub struct SessionParameters { pub ehlo_reject_non_fqdn: bool, // Auth parameters - pub auth_directory: Option<Arc<dyn Directory>>, + pub auth_directory: Option<Arc<Directory>>, pub auth_require: bool, pub auth_errors_max: usize, pub auth_errors_wait: Duration, diff --git a/crates/store/src/backend/elastic/mod.rs b/crates/store/src/backend/elastic/mod.rs index b098dead..07945520 100644 --- a/crates/store/src/backend/elastic/mod.rs +++ b/crates/store/src/backend/elastic/mod.rs @@ -66,7 +66,7 @@ impl ElasticSearchStore { if let Some(credentials) = credentials { builder = builder.auth(credentials); } - if config.property_or_static::<bool>((&prefix, "allow-invalid-certs"), "false")? { + if config.property_or_static::<bool>((&prefix, "tls.allow-invalid-certs"), "false")? { builder = builder.cert_validation(CertificateValidation::None); } diff --git a/crates/store/src/backend/foundationdb/main.rs b/crates/store/src/backend/foundationdb/main.rs index 6e6ebf04..e2f77be0 100644 --- a/crates/store/src/backend/foundationdb/main.rs +++ b/crates/store/src/backend/foundationdb/main.rs @@ -21,6 +21,8 @@ * for more details. */ +use std::time::Duration; + use foundationdb::{options::DatabaseOption, Database}; use utils::config::{utils::AsKey, Config}; @@ -32,14 +34,18 @@ impl FdbStore { let guard = unsafe { foundationdb::boot() }; let db = Database::new(config.value((&prefix, "path")))?; - if let Some(value) = config.property((&prefix, "transaction.timeout"))? { - db.set_option(DatabaseOption::TransactionTimeout(value))?; + if let Some(value) = config.property::<Duration>((&prefix, "transaction.timeout"))? { + db.set_option(DatabaseOption::TransactionTimeout(value.as_millis() as i32))?; } if let Some(value) = config.property((&prefix, "transaction.retry-limit"))? { db.set_option(DatabaseOption::TransactionRetryLimit(value))?; } - if let Some(value) = config.property((&prefix, "transaction.max-retry-delay"))? { - db.set_option(DatabaseOption::TransactionMaxRetryDelay(value))?; + if let Some(value) = + config.property::<Duration>((&prefix, "transaction.max-retry-delay"))? + { + db.set_option(DatabaseOption::TransactionMaxRetryDelay( + value.as_millis() as i32 + ))?; } if let Some(value) = config.property((&prefix, "transaction.machine-id"))? { db.set_option(DatabaseOption::MachineId(value))?; diff --git a/crates/store/src/backend/fs/mod.rs b/crates/store/src/backend/fs/mod.rs index aff54c17..a0298b0f 100644 --- a/crates/store/src/backend/fs/mod.rs +++ b/crates/store/src/backend/fs/mod.rs @@ -41,17 +41,19 @@ impl FsStore { pub async fn open(config: &Config, prefix: impl AsKey) -> crate::Result<Self> { let prefix = prefix.as_key(); let path = config.property_require::<PathBuf>((&prefix, "path"))?; - if path.exists() { - Ok(FsStore { - path, - hash_levels: std::cmp::min(config.property_or_static((&prefix, "depth"), "2")?, 5), - }) - } else { - Err(crate::Error::InternalError(format!( - "Blob store path {:?} does not exist", - path - ))) + if !path.exists() { + fs::create_dir_all(&path).await.map_err(|e| { + crate::Error::InternalError(format!( + "Failed to create blob store path {:?}: {}", + path, e + )) + })?; } + + Ok(FsStore { + path, + hash_levels: std::cmp::min(config.property_or_static((&prefix, "depth"), "2")?, 5), + }) } pub(crate) async fn get_blob( diff --git a/crates/store/src/backend/redis/mod.rs b/crates/store/src/backend/redis/mod.rs index 207d7441..c313320f 100644 --- a/crates/store/src/backend/redis/mod.rs +++ b/crates/store/src/backend/redis/mod.rs @@ -91,10 +91,10 @@ impl RedisStore { builder = builder.retries(value); } if let Some(value) = config.property::<Duration>((&prefix, "max-retry-wait"))? { - builder = builder.max_retry_wait(value.as_secs()); + builder = builder.max_retry_wait(value.as_millis() as u64); } if let Some(value) = config.property::<Duration>((&prefix, "min-retry-wait"))? { - builder = builder.min_retry_wait(value.as_secs()); + builder = builder.min_retry_wait(value.as_millis() as u64); } if let Some(true) = config.property::<bool>((&prefix, "read-from-replicas"))? { builder = builder.read_from_replicas(); diff --git a/crates/store/src/backend/rocksdb/main.rs b/crates/store/src/backend/rocksdb/main.rs index fb6eb001..954b6d42 100644 --- a/crates/store/src/backend/rocksdb/main.rs +++ b/crates/store/src/backend/rocksdb/main.rs @@ -99,7 +99,7 @@ impl RocksDbStore { config .property::<usize>((&prefix, "pool.workers"))? .filter(|v| *v > 0) - .unwrap_or_else(num_cpus::get), + .unwrap_or_else(|| num_cpus::get() * 4), ) .build() .map_err(|err| { diff --git a/crates/store/src/backend/sqlite/main.rs b/crates/store/src/backend/sqlite/main.rs index 5424479c..781883b2 100644 --- a/crates/store/src/backend/sqlite/main.rs +++ b/crates/store/src/backend/sqlite/main.rs @@ -40,7 +40,11 @@ impl SqliteStore { let prefix = prefix.as_key(); let db = Self { conn_pool: Pool::builder() - .max_size(config.property_or_static((&prefix, "pool.max-connections"), "10")?) + .max_size( + config + .property((&prefix, "pool.max-connections"))? + .unwrap_or_else(|| (num_cpus::get() * 4) as u32), + ) .build( SqliteConnectionManager::file( config diff --git a/crates/store/src/config.rs b/crates/store/src/config.rs index db23a876..eb3ef11b 100644 --- a/crates/store/src/config.rs +++ b/crates/store/src/config.rs @@ -210,7 +210,7 @@ impl ConfigStore for Config { config.lookup_stores.insert(store_id, lookup_store.clone()); // Run init queries on database - for (_, query) in self.values(("store", id, "init")) { + for (_, query) in self.values(("store", id, "init.execute")) { if let Err(err) = lookup_store.query::<usize>(query, Vec::new()).await { tracing::warn!("Failed to initialize store {id:?}: {err}"); } diff --git a/crates/store/src/dispatch/store.rs b/crates/store/src/dispatch/store.rs index 7487b339..0460d916 100644 --- a/crates/store/src/dispatch/store.rs +++ b/crates/store/src/dispatch/store.rs @@ -306,7 +306,7 @@ impl Store { } #[cfg(feature = "test_mode")] - pub async fn blob_hash_expire_all(&self) { + pub async fn blob_expire_all(&self) { use crate::{ write::{key::DeserializeBigEndian, BatchBuilder, BlobOp, Operation, ValueOp}, BlobHash, BLOB_HASH_LEN, U64_LEN, @@ -367,7 +367,7 @@ impl Store { pub async fn assert_is_empty(&self, blob_store: crate::BlobStore) { use crate::{SUBSPACE_BLOBS, SUBSPACE_COUNTERS}; - self.blob_hash_expire_all().await; + self.blob_expire_all().await; self.purge_blobs(blob_store).await.unwrap(); self.purge_bitmaps().await.unwrap(); @@ -420,7 +420,7 @@ impl Store { ); } SUBSPACE_VALUES - if key[0] >= 6 + if key[0] >= 20 || key.get(1..5).unwrap_or_default() == u32::MAX.to_be_bytes() => { // Ignore lastId counter and ID mappings diff --git a/crates/store/src/fts/index.rs b/crates/store/src/fts/index.rs index d00bbf15..33acc511 100644 --- a/crates/store/src/fts/index.rs +++ b/crates/store/src/fts/index.rs @@ -44,6 +44,7 @@ use crate::{ }; use super::Field; +pub const TERM_INDEX_VERSION: u8 = 1; #[derive(Debug)] pub(crate) struct Text<'x, T: Into<u8> + Display + Clone + std::fmt::Debug> { @@ -234,14 +235,13 @@ impl Store { // Write term index let mut batch = BatchBuilder::new(); + let mut term_index = lz4_flex::compress_prepend_size(&serializer.finalize()); + term_index.insert(0, TERM_INDEX_VERSION); batch .with_account_id(document.account_id) .with_collection(document.collection) .update_document(document.document_id) - .set( - ValueClass::TermIndex, - lz4_flex::compress_prepend_size(&serializer.finalize()), - ); + .set(ValueClass::TermIndex, term_index); self.write(batch.build()).await?; let mut batch = BatchBuilder::new(); batch @@ -339,7 +339,12 @@ struct TermIndex { impl Deserialize for TermIndex { fn deserialize(bytes: &[u8]) -> crate::Result<Self> { - let bytes = lz4_flex::decompress_size_prepended(bytes) + if bytes.first().copied().unwrap_or_default() != TERM_INDEX_VERSION { + return Err(Error::InternalError( + "Unsupported term index version".to_string(), + )); + } + let bytes = lz4_flex::decompress_size_prepended(bytes.get(1..).unwrap_or_default()) .map_err(|_| Error::InternalError("Failed to decompress term index".to_string()))?; let mut ops = Vec::new(); diff --git a/crates/store/src/fts/query.rs b/crates/store/src/fts/query.rs index e549956e..f57153f9 100644 --- a/crates/store/src/fts/query.rs +++ b/crates/store/src/fts/query.rs @@ -39,6 +39,8 @@ use crate::{ BitmapKey, Deserialize, Error, Store, ValueKey, }; +use super::index::TERM_INDEX_VERSION; + struct State<T: Into<u8> + Display + Clone + std::fmt::Debug> { pub op: FtsFilter<T>, pub bm: Option<RoaringBitmap>, @@ -278,7 +280,12 @@ impl Store { impl Deserialize for BigramIndex { fn deserialize(bytes: &[u8]) -> crate::Result<Self> { - let bytes = lz4_flex::decompress_size_prepended(bytes) + if bytes.first().copied().unwrap_or_default() != TERM_INDEX_VERSION { + return Err(Error::InternalError( + "Unsupported term index version".to_string(), + )); + } + let bytes = lz4_flex::decompress_size_prepended(bytes.get(1..).unwrap_or_default()) .map_err(|_| Error::InternalError("Failed to decompress term index".to_string()))?; let (num_items, pos) = bytes.read_leb128::<usize>().ok_or(Error::InternalError( diff --git a/crates/store/src/write/key.rs b/crates/store/src/write/key.rs index 5027294d..0b241def 100644 --- a/crates/store/src/write/key.rs +++ b/crates/store/src/write/key.rs @@ -275,11 +275,11 @@ impl<T: AsRef<ValueClass> + Sync + Send> Key for ValueKey<T> { .write(self.document_id), }, ValueClass::Directory(directory) => match directory { - DirectoryClass::NameToId(name) => serializer.write(8u8).write(name.as_slice()), - DirectoryClass::EmailToId(email) => serializer.write(9u8).write(email.as_slice()), - DirectoryClass::Principal(uid) => serializer.write(10u8).write_leb128(*uid), - DirectoryClass::Domain(name) => serializer.write(11u8).write(name.as_slice()), - DirectoryClass::UsedQuota(uid) => serializer.write(12u8).write_leb128(*uid), + DirectoryClass::NameToId(name) => serializer.write(20u8).write(name.as_slice()), + DirectoryClass::EmailToId(email) => serializer.write(21u8).write(email.as_slice()), + DirectoryClass::Principal(uid) => serializer.write(22u8).write_leb128(*uid), + DirectoryClass::Domain(name) => serializer.write(23u8).write(name.as_slice()), + DirectoryClass::UsedQuota(uid) => serializer.write(24u8).write_leb128(*uid), }, } .finalize() |