diff options
267 files changed, 5861 insertions, 4436 deletions
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 15eeef2e..1a895076 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -2,9 +2,6 @@ name: Test on: workflow_dispatch: - pull_request: - push: - tags: ["v*.*.*"] jobs: style: @@ -352,9 +352,9 @@ dependencies = [ [[package]] name = "async-trait" -version = "0.1.82" +version = "0.1.83" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a27b8a3a6e1a44fa4c8baf1f653e4172e81486d4941f2237e20dc2d0cf4ddff1" +checksum = "721cae7de5c34fbb2acd27e21e6d2cf7b886dce0c27388d46c4e6c47ea4318dd" dependencies = [ "proc-macro2", "quote", @@ -416,9 +416,9 @@ dependencies = [ [[package]] name = "axum" -version = "0.7.5" +version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a6c9af12842a67734c9a2e355436e5d03b22383ed60cf13cd0c18fbfe3dcbcf" +checksum = "8f43644eed690f5374f1af436ecd6aea01cd201f6fbdf0178adaf6907afb2cec" dependencies = [ "async-trait", "axum-core", @@ -436,16 +436,16 @@ dependencies = [ "rustversion", "serde", "sync_wrapper 1.0.1", - "tower", + "tower 0.5.1", "tower-layer", "tower-service", ] [[package]] name = "axum-core" -version = "0.4.3" +version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a15c63fd72d41492dc4f497196f5da1fb04fb7529e631d73630d1b491e47a2e3" +checksum = "5e6b8ba012a258d63c9adfa28b9ddcf66149da6f986c5b5452e629d5ee64bf00" dependencies = [ "async-trait", "bytes", @@ -456,7 +456,7 @@ dependencies = [ "mime", "pin-project-lite", "rustversion", - "sync_wrapper 0.1.2", + "sync_wrapper 1.0.1", "tower-layer", "tower-service", ] @@ -956,9 +956,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.5.17" +version = "4.5.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e5a21b8495e732f1b3c364c9949b201ca7bae518c502c80256c96ad79eaf6ac" +checksum = "b0956a43b323ac1afaffc053ed5c4b7c1f1800bacd1683c353aabbb752515dd3" dependencies = [ "clap_builder", "clap_derive", @@ -966,9 +966,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.17" +version = "4.5.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8cf2dd12af7a047ad9d6da2b6b249759a22a7abc0f474c1dae1777afa4b21a73" +checksum = "4d72166dd41634086d5803a47eb71ae740e61d84709c36f3c34110173db3961b" dependencies = [ "anstream", "anstyle", @@ -978,9 +978,9 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.5.13" +version = "4.5.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "501d359d5f3dcaf6ecdeee48833ae73ec6e42723a1e52419c79abf9507eec0a0" +checksum = "4ac6a0c7b1a9e9a5186361f67dfa1b88213572f427fb9ab038efb2bd8c582dab" dependencies = [ "heck 0.5.0", "proc-macro2", @@ -1043,6 +1043,7 @@ dependencies = [ "base64 0.22.1", "bincode", "chrono", + "dashmap", "decancer", "directory", "dns-update", @@ -1051,6 +1052,7 @@ dependencies = [ "hyper 1.4.1", "idna 1.0.2", "imagesize", + "imap_proto", "infer", "jmap_proto", "libc", @@ -2765,9 +2767,9 @@ dependencies = [ [[package]] name = "hyper-util" -version = "0.1.8" +version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da62f120a8a37763efb0cf8fdf264b884c7b8b9ac8660b900c8661030c00e6ba" +checksum = "41296eb09f183ac68eec06e03cdbea2e759633d4067b2f6552fc2e009bcad08b" dependencies = [ "bytes", "futures-channel", @@ -2778,7 +2780,6 @@ dependencies = [ "pin-project-lite", "socket2", "tokio", - "tower", "tower-service", "tracing", ] @@ -3223,7 +3224,7 @@ dependencies = [ "nlp", "p256", "pkcs8", - "quick-xml 0.36.1", + "quick-xml 0.36.2", "rand", "rasn", "rasn-cms", @@ -3416,9 +3417,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.158" +version = "0.2.159" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d8adc4bb1803a324070e64a98ae98f38934d91957a99cfb3a43dcbc01bc56439" +checksum = "561d97a539a36e26a9a5fad1ea11a3039a67714694aaa379433e580854bc3dc5" [[package]] name = "libloading" @@ -3581,7 +3582,7 @@ dependencies = [ "mail-builder", "mail-parser", "parking_lot", - "quick-xml 0.36.1", + "quick-xml 0.36.2", "rand", "ring 0.17.8", "rsa", @@ -4471,9 +4472,9 @@ dependencies = [ [[package]] name = "pkg-config" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d231b230927b5e4ad203db57bbcbee2802f6bce620b1e4a9024a07d94e2907ec" +checksum = "953ec861398dccce10c670dfeaf3ec4911ca479e9c02154b3a215178c5f566f2" [[package]] name = "polyval" @@ -4508,9 +4509,9 @@ dependencies = [ [[package]] name = "portable-atomic" -version = "1.7.0" +version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da544ee218f0d287a911e9c99a39a8c9bc8fcad3cb8db5959940044ecfc67265" +checksum = "d30538d42559de6b034bc76fd6dd4c38961b1ee5c6c56e3808c50128fdbc22ce" [[package]] name = "postgres-protocol" @@ -4672,9 +4673,9 @@ dependencies = [ [[package]] name = "prost" -version = "0.13.2" +version = "0.13.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b2ecbe40f08db5c006b5764a2645f7f3f141ce756412ac9e1dd6087e6d32995" +checksum = "7b0487d90e047de87f984913713b85c601c05609aad5b0df4b4573fbf69aa13f" dependencies = [ "bytes", "prost-derive", @@ -4682,9 +4683,9 @@ dependencies = [ [[package]] name = "prost-derive" -version = "0.13.2" +version = "0.13.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "acf0c195eebb4af52c752bec4f52f645da98b6e92077a04110c7f349477ae5ac" +checksum = "e9552f850d5f0964a4e4d0bf306459ac29323ddfbae05e35a7c0d35cb0803cc5" dependencies = [ "anyhow", "itertools 0.13.0", @@ -4771,9 +4772,9 @@ dependencies = [ [[package]] name = "quick-xml" -version = "0.36.1" +version = "0.36.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96a05e2e8efddfa51a84ca47cec303fac86c8541b686d37cac5efc0e094417bc" +checksum = "f7649a7b4df05aed9ea7ec6f628c67c9953a43869b8bc50929569b2999d443fe" dependencies = [ "memchr", ] @@ -5028,9 +5029,9 @@ dependencies = [ [[package]] name = "redox_syscall" -version = "0.5.4" +version = "0.5.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0884ad60e090bf1345b93da0a5de8923c93884cd03f40dfcfddd3b4bee661853" +checksum = "355ae415ccd3a04315d3f8246e86d67689ea74d88d915576e1589a351062a13b" dependencies = [ "bitflags 2.6.0", ] @@ -5728,9 +5729,9 @@ dependencies = [ [[package]] name = "security-framework-sys" -version = "2.11.1" +version = "2.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75da29fe9b9b08fe9d6b22b5b4bcbc75d8db3aa31e639aa56bb62e9d46bfceaf" +checksum = "ea4a292869320c0272d7bc55a5a6aafaff59b4f63404a003887b679a2e05b4b6" dependencies = [ "core-foundation-sys", "libc", @@ -6026,9 +6027,9 @@ checksum = "d66dc143e6b11c1eddc06d5c423cfc97062865baf299914ab64caa38182078fe" [[package]] name = "simdutf8" -version = "0.1.4" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f27f6278552951f1f2b8cf9da965d10969b2efdea95a6ec47987ab46edfe263a" +checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e" [[package]] name = "siphasher" @@ -6474,18 +6475,18 @@ dependencies = [ [[package]] name = "thiserror" -version = "1.0.63" +version = "1.0.64" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0342370b38b6a11b6cc11d6a805569958d54cfa061a29969c3b5ce2ea405724" +checksum = "d50af8abc119fb8bb6dbabcfa89656f46f84aa0ac7688088608076ad2b459a84" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.63" +version = "1.0.64" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4558b58466b9ad7ca0f102865eccc95938dca1a74a856f2b57b6629050da261" +checksum = "08904e7672f5eb876eaaf87e0ce17857500934f4981c4a0ab2b4aa98baac7fc3" dependencies = [ "proc-macro2", "quote", @@ -6714,9 +6715,9 @@ checksum = "0dd7358ecb8fc2f8d014bf86f6f638ce72ba252a2c3a2572f2a795f1d23efb41" [[package]] name = "toml_edit" -version = "0.22.21" +version = "0.22.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b072cee73c449a636ffd6f32bd8de3a9f7119139aff882f44943ce2986dc5cf" +checksum = "4ae48d6208a266e853d946088ed816055e556cc6028c5e8e2b84d9fa5dd7c7f5" dependencies = [ "indexmap 2.5.0", "toml_datetime", @@ -6747,7 +6748,7 @@ dependencies = [ "socket2", "tokio", "tokio-stream", - "tower", + "tower 0.4.13", "tower-layer", "tower-service", "tracing", @@ -6789,6 +6790,20 @@ dependencies = [ ] [[package]] +name = "tower" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2873938d487c3cfb9aed7546dc9f2711d867c9f90c46b889989a2cb84eba6b4f" +dependencies = [ + "futures-core", + "futures-util", + "pin-project-lite", + "sync_wrapper 0.1.2", + "tower-layer", + "tower-service", +] + +[[package]] name = "tower-layer" version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -6985,9 +7000,9 @@ checksum = "52ea75f83c0137a9b98608359a5f1af8144876eb67bcb1ce837368e906a9f524" [[package]] name = "unicode-script" -version = "0.5.6" +version = "0.5.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad8d71f5726e5f285a935e9fe8edfd53f0491eb6e9a5774097fdabee7cd8c9cd" +checksum = "9fb421b350c9aff471779e262955939f565ec18b86c15364e6bdf0d662ca7c1f" [[package]] name = "unicode-security" @@ -7001,9 +7016,9 @@ dependencies = [ [[package]] name = "unicode-width" -version = "0.1.13" +version = "0.1.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0336d538f7abc86d282a4189614dfaa90810dfc2c6f6427eaf88e16311dd225d" +checksum = "7dd6e30e90baa6f72411720665d41d89b9a3d039dc45b8faea1ddd07f617f6af" [[package]] name = "unicode-xid" @@ -7552,9 +7567,9 @@ checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" [[package]] name = "winnow" -version = "0.6.18" +version = "0.6.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68a9bda4691f099d435ad181000724da8e5899daa10713c2d432552b9ccd3a6f" +checksum = "36c1fec1a2bb5866f07c25f68c26e565c4c200aebb96d7e55710c19d3e8ac49b" dependencies = [ "memchr", ] diff --git a/crates/common/Cargo.toml b/crates/common/Cargo.toml index 6f30e404..6dac6a47 100644 --- a/crates/common/Cargo.toml +++ b/crates/common/Cargo.toml @@ -11,6 +11,7 @@ store = { path = "../store" } trc = { path = "../trc" } directory = { path = "../directory" } jmap_proto = { path = "../jmap-proto" } +imap_proto = { path = "../imap-proto" } sieve-rs = { version = "0.5" } mail-parser = { version = "0.9", features = ["full_encoding", "ludicrous_mode"] } mail-builder = { version = "0.3", features = ["ludicrous_mode"] } @@ -59,6 +60,7 @@ zip = "2.1" pwhash = "1.0.0" xxhash-rust = { version = "0.8.5", features = ["xxh3"] } psl = "2" +dashmap = "6.0" [target.'cfg(unix)'.dependencies] privdrop = "0.5.3" diff --git a/crates/common/src/addresses.rs b/crates/common/src/addresses.rs index 0303b7fc..d56385b1 100644 --- a/crates/common/src/addresses.rs +++ b/crates/common/src/addresses.rs @@ -14,10 +14,10 @@ use crate::{ expr::{ functions::ResolveVariable, if_block::IfBlock, tokenizer::TokenMap, Variable, V_RECIPIENT, }, - Core, + Server, }; -impl Core { +impl Server { pub async fn email_to_ids( &self, directory: &Directory, @@ -25,6 +25,7 @@ impl Core { session_id: u64, ) -> trc::Result<Vec<u32>> { let mut address = self + .core .smtp .session .rcpt @@ -38,6 +39,7 @@ impl Core { if !result.is_empty() { return Ok(result); } else if let Some(catch_all) = self + .core .smtp .session .rcpt @@ -62,6 +64,7 @@ impl Core { ) -> trc::Result<bool> { // Expand subaddress let mut address = self + .core .smtp .session .rcpt @@ -73,6 +76,7 @@ impl Core { if directory.rcpt(address.as_ref()).await? { return Ok(true); } else if let Some(catch_all) = self + .core .smtp .session .rcpt @@ -97,7 +101,8 @@ impl Core { ) -> trc::Result<Vec<String>> { directory .vrfy( - self.smtp + self.core + .smtp .session .rcpt .subaddressing @@ -116,7 +121,8 @@ impl Core { ) -> trc::Result<Vec<String>> { directory .expn( - self.smtp + self.core + .smtp .session .rcpt .subaddressing @@ -170,7 +176,7 @@ impl ResolveVariable for Address<'_> { impl AddressMapping { pub async fn to_subaddress<'x, 'y: 'x>( &'x self, - core: &Core, + core: &Server, address: &'y str, session_id: u64, ) -> Cow<'x, str> { @@ -198,7 +204,7 @@ impl AddressMapping { pub async fn to_catch_all<'x, 'y: 'x>( &'x self, - core: &Core, + core: &Server, address: &'y str, session_id: u64, ) -> Option<Cow<'x, str>> { diff --git a/crates/common/src/auth/access_token.rs b/crates/common/src/auth/access_token.rs index 7ff882e4..113c33d8 100644 --- a/crates/common/src/auth/access_token.rs +++ b/crates/common/src/auth/access_token.rs @@ -25,11 +25,11 @@ use utils::map::{ vec_map::VecMap, }; -use crate::Core; +use crate::Server; use super::{roles::RolePermissions, AccessToken, ResourceToken, TenantInfo}; -impl Core { +impl Server { pub async fn build_access_token(&self, mut principal: Principal) -> trc::Result<AccessToken> { let mut role_permissions = RolePermissions::default(); @@ -75,8 +75,7 @@ impl Core { tenant = Some(TenantInfo { id: tenant_id, quota: self - .storage - .data + .store() .query(QueryBy::Id(tenant_id), false) .await .caused_by(trc::location!())? @@ -111,12 +110,7 @@ impl Core { } pub async fn get_access_token(&self, account_id: u32) -> trc::Result<AccessToken> { - let err = match self - .storage - .directory - .query(QueryBy::Id(account_id), true) - .await - { + let err = match self.directory().query(QueryBy::Id(account_id), true).await { Ok(Some(principal)) => { return self .update_access_token(self.build_access_token(principal).await?) @@ -129,7 +123,7 @@ impl Core { Err(err) => Err(err), }; - match &self.jmap.fallback_admin { + match &self.core.jmap.fallback_admin { Some((_, secret)) if account_id == u32::MAX => { self.update_access_token( self.build_access_token(Principal::fallback_admin(secret)) @@ -150,8 +144,7 @@ impl Core { .chain(access_token.member_of.iter().copied()) { for acl_item in self - .storage - .data + .store() .acl_query(AclQuery::HasAccess { grant_account_id }) .await .caused_by(trc::location!())? @@ -191,15 +184,15 @@ impl Core { } pub fn cache_access_token(&self, access_token: Arc<AccessToken>) { - self.security.access_tokens.insert_with_ttl( + self.inner.data.access_tokens.insert_with_ttl( access_token.primary_id(), access_token, - Instant::now() + self.jmap.session_cache_ttl, + Instant::now() + self.core.jmap.session_cache_ttl, ); } pub async fn get_cached_access_token(&self, primary_id: u32) -> trc::Result<Arc<AccessToken>> { - if let Some(access_token) = self.security.access_tokens.get_with_ttl(&primary_id) { + if let Some(access_token) = self.inner.data.access_tokens.get_with_ttl(&primary_id) { Ok(access_token) } else { // Refresh ACL token diff --git a/crates/common/src/auth/roles.rs b/crates/common/src/auth/roles.rs index 79a59cfd..1180ed79 100644 --- a/crates/common/src/auth/roles.rs +++ b/crates/common/src/auth/roles.rs @@ -13,7 +13,7 @@ use directory::{ }; use trc::AddContext; -use crate::Core; +use crate::Server; #[derive(Debug, Clone, Default)] pub struct RolePermissions { @@ -26,14 +26,14 @@ static ADMIN_PERMISSIONS: LazyLock<Arc<RolePermissions>> = LazyLock::new(admin_p static TENANT_ADMIN_PERMISSIONS: LazyLock<Arc<RolePermissions>> = LazyLock::new(tenant_admin_permissions); -impl Core { +impl Server { pub async fn get_role_permissions(&self, role_id: u32) -> trc::Result<Arc<RolePermissions>> { match role_id { ROLE_USER => Ok(USER_PERMISSIONS.clone()), ROLE_ADMIN => Ok(ADMIN_PERMISSIONS.clone()), ROLE_TENANT_ADMIN => Ok(TENANT_ADMIN_PERMISSIONS.clone()), role_id => { - if let Some(role_permissions) = self.security.permissions.get(&role_id) { + if let Some(role_permissions) = self.inner.data.permissions.get(&role_id) { Ok(role_permissions.clone()) } else { self.build_role_permissions(role_id).await @@ -81,15 +81,14 @@ impl Core { } role_id => { // Try with the cache - if let Some(role_permissions) = self.security.permissions.get(&role_id) { + if let Some(role_permissions) = self.inner.data.permissions.get(&role_id) { return_permissions.union(role_permissions.as_ref()); } else { let mut role_permissions = RolePermissions::default(); // Obtain principal let mut principal = self - .storage - .data + .store() .query(QueryBy::Id(role_id), true) .await .caused_by(trc::location!())? @@ -133,7 +132,8 @@ impl Core { role_ids = parent_role_ids.into_iter(); } else { // Cache role - self.security + self.inner + .data .permissions .insert(role_id, Arc::new(role_permissions)); } @@ -149,7 +149,8 @@ impl Core { // Cache role let return_permissions = Arc::new(return_permissions); - self.security + self.inner + .data .permissions .insert(role_id, return_permissions.clone()); Ok(return_permissions) diff --git a/crates/common/src/config/inner.rs b/crates/common/src/config/inner.rs new file mode 100644 index 00000000..92528e26 --- /dev/null +++ b/crates/common/src/config/inner.rs @@ -0,0 +1,163 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs Ltd <hello@stalw.art> + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use std::{sync::Arc, time::Duration}; + +use ahash::{AHashMap, AHashSet, RandomState}; +use arc_swap::ArcSwap; +use dashmap::DashMap; +use mail_send::smtp::tls::build_tls_connector; +use nlp::bayes::cache::BayesTokenCache; +use parking_lot::RwLock; +use utils::{ + config::Config, + lru_cache::{LruCache, LruCached}, + map::ttl_dashmap::{TtlDashMap, TtlMap}, + snowflake::SnowflakeIdGenerator, +}; + +use crate::{ + listener::blocked::BlockedIps, manager::webadmin::WebAdminManager, Data, + ThrottleKeyHasherBuilder, TlsConnectors, +}; + +use super::server::tls::{build_self_signed_cert, parse_certificates}; + +impl Data { + pub fn parse(config: &mut Config) -> Self { + // Parse certificates + let mut certificates = AHashMap::new(); + let mut subject_names = AHashSet::new(); + parse_certificates(config, &mut certificates, &mut subject_names); + if subject_names.is_empty() { + subject_names.insert("localhost".to_string()); + } + + // Parse capacities + let shard_amount = config + .property::<u64>("cache.shard") + .unwrap_or(32) + .next_power_of_two() as usize; + let capacity = config.property("cache.capacity").unwrap_or(100); + + // Parse id generator + let id_generator = config + .property::<u64>("cluster.node-id") + .map(SnowflakeIdGenerator::with_node_id) + .unwrap_or_default(); + + Data { + tls_certificates: ArcSwap::from_pointee(certificates), + tls_self_signed_cert: build_self_signed_cert( + subject_names.into_iter().collect::<Vec<_>>(), + ) + .or_else(|err| { + config.new_build_error("certificate.self-signed", err); + build_self_signed_cert(vec!["localhost".to_string()]) + }) + .ok() + .map(Arc::new), + access_tokens: TtlDashMap::with_capacity(capacity, shard_amount), + http_auth_cache: TtlDashMap::with_capacity(capacity, shard_amount), + blocked_ips: RwLock::new(BlockedIps::parse(config).blocked_ip_addresses), + blocked_ips_version: 0.into(), + permissions: Default::default(), + permissions_version: 0.into(), + jmap_id_gen: id_generator.clone(), + queue_id_gen: id_generator.clone(), + span_id_gen: id_generator, + webadmin: WebAdminManager::new(), + config_version: 0.into(), + jmap_limiter: DashMap::with_capacity_and_hasher_and_shard_amount( + capacity, + RandomState::default(), + shard_amount, + ), + imap_limiter: DashMap::with_capacity_and_hasher_and_shard_amount( + capacity, + RandomState::default(), + shard_amount, + ), + account_cache: LruCache::with_capacity( + config.property("cache.account.size").unwrap_or(2048), + ), + mailbox_cache: LruCache::with_capacity( + config.property("cache.mailbox.size").unwrap_or(2048), + ), + threads_cache: LruCache::with_capacity( + config.property("cache.thread.size").unwrap_or(2048), + ), + logos: Default::default(), + smtp_session_throttle: DashMap::with_capacity_and_hasher_and_shard_amount( + capacity, + ThrottleKeyHasherBuilder::default(), + shard_amount, + ), + smtp_queue_throttle: DashMap::with_capacity_and_hasher_and_shard_amount( + capacity, + ThrottleKeyHasherBuilder::default(), + shard_amount, + ), + smtp_connectors: TlsConnectors::default(), + bayes_cache: BayesTokenCache::new( + config + .property_or_default("cache.bayes.capacity", "8192") + .unwrap_or(8192), + config + .property_or_default("cache.bayes.ttl.positive", "1h") + .unwrap_or_else(|| Duration::from_secs(3600)), + config + .property_or_default("cache.bayes.ttl.negative", "1h") + .unwrap_or_else(|| Duration::from_secs(3600)), + ), + remote_lists: Default::default(), + } + } +} + +impl Default for Data { + fn default() -> Self { + Self { + tls_certificates: Default::default(), + tls_self_signed_cert: Default::default(), + access_tokens: Default::default(), + http_auth_cache: Default::default(), + blocked_ips: Default::default(), + blocked_ips_version: 0.into(), + permissions: Default::default(), + permissions_version: 0.into(), + remote_lists: Default::default(), + jmap_id_gen: Default::default(), + queue_id_gen: Default::default(), + span_id_gen: Default::default(), + webadmin: Default::default(), + config_version: Default::default(), + jmap_limiter: Default::default(), + imap_limiter: Default::default(), + account_cache: LruCache::with_capacity(2048), + mailbox_cache: LruCache::with_capacity(2048), + threads_cache: LruCache::with_capacity(2048), + logos: Default::default(), + smtp_session_throttle: Default::default(), + smtp_queue_throttle: Default::default(), + smtp_connectors: Default::default(), + bayes_cache: BayesTokenCache::new( + 8192, + Duration::from_secs(3600), + Duration::from_secs(3600), + ), + } + } +} + +impl Default for TlsConnectors { + fn default() -> Self { + TlsConnectors { + pki_verify: build_tls_connector(false), + dummy_verify: build_tls_connector(true), + } + } +} diff --git a/crates/common/src/config/mod.rs b/crates/common/src/config/mod.rs index edd5ed29..79b7c059 100644 --- a/crates/common/src/config/mod.rs +++ b/crates/common/src/config/mod.rs @@ -10,13 +10,10 @@ use arc_swap::ArcSwap; use directory::{Directories, Directory}; use store::{BlobBackend, BlobStore, FtsStore, LookupStore, Store, Stores}; use telemetry::Metrics; -use utils::{ - config::Config, - map::ttl_dashmap::{ADashMap, TtlDashMap, TtlMap}, -}; +use utils::config::Config; use crate::{ - expr::*, listener::tls::TlsManager, manager::config::ConfigManager, Core, Network, Security, + expr::*, listener::tls::AcmeProviders, manager::config::ConfigManager, Core, Network, Security, }; use self::{ @@ -25,6 +22,7 @@ use self::{ }; pub mod imap; +pub mod inner; pub mod jmap; pub mod network; pub mod scripts; @@ -165,18 +163,8 @@ impl Core { smtp: SmtpConfig::parse(config).await, jmap: JmapConfig::parse(config), imap: ImapConfig::parse(config), - tls: TlsManager::parse(config), + acme: AcmeProviders::parse(config), metrics: Metrics::parse(config), - security: Security { - access_tokens: TtlDashMap::with_capacity(100, 32), - permissions: ADashMap::with_capacity_and_hasher_and_shard_amount( - 100, - ahash::RandomState::new(), - 32, - ), - permissions_version: Default::default(), - logos: Default::default(), - }, storage: Storage { data, blob, @@ -194,7 +182,7 @@ impl Core { } } - pub fn into_shared(self) -> Arc<ArcSwap<Self>> { - Arc::new(ArcSwap::from_pointee(self)) + pub fn into_shared(self) -> ArcSwap<Self> { + ArcSwap::from_pointee(self) } } diff --git a/crates/common/src/config/network.rs b/crates/common/src/config/network.rs index c3829dbf..b7c3faa5 100644 --- a/crates/common/src/config/network.rs +++ b/crates/common/src/config/network.rs @@ -6,7 +6,6 @@ use crate::{ expr::{if_block::IfBlock, tokenizer::TokenMap}, - listener::blocked::{AllowedIps, BlockedIps}, Network, }; use utils::config::Config; @@ -30,8 +29,7 @@ pub(crate) const HTTP_VARS: &[u32; 11] = &[ impl Default for Network { fn default() -> Self { Self { - blocked_ips: Default::default(), - allowed_ips: Default::default(), + security: Default::default(), node_id: 0, http_response_url: IfBlock::new::<()>( "server.http.url", @@ -47,8 +45,7 @@ impl Network { pub fn parse(config: &mut Config) -> Self { let mut network = Network { node_id: config.property("cluster.node-id").unwrap_or_default(), - blocked_ips: BlockedIps::parse(config), - allowed_ips: AllowedIps::parse(config), + security: Security::parse(config), ..Default::default() }; let token_map = &TokenMap::default().with_variables(HTTP_VARS); diff --git a/crates/common/src/config/scripts.rs b/crates/common/src/config/scripts.rs index 5e68e435..aa91a2f7 100644 --- a/crates/common/src/config/scripts.rs +++ b/crates/common/src/config/scripts.rs @@ -11,8 +11,6 @@ use std::{ }; use ahash::AHashMap; -use nlp::bayes::cache::BayesTokenCache; -use parking_lot::RwLock; use sieve::{compiler::grammar::Capability, Compiler, Runtime, Sieve}; use store::Stores; use utils::config::Config; @@ -33,11 +31,6 @@ pub struct Scripting { pub untrusted_scripts: AHashMap<String, Arc<Sieve>>, } -pub struct ScriptCache { - pub bayes_cache: BayesTokenCache, - pub remote_lists: RwLock<AHashMap<String, RemoteList>>, -} - #[derive(Clone)] pub struct RemoteList { pub entries: HashSet<String>, @@ -364,25 +357,6 @@ impl Scripting { } } -impl ScriptCache { - pub fn parse(config: &mut Config) -> Self { - ScriptCache { - bayes_cache: BayesTokenCache::new( - config - .property_or_default("cache.bayes.capacity", "8192") - .unwrap_or(8192), - config - .property_or_default("cache.bayes.ttl.positive", "1h") - .unwrap_or_else(|| Duration::from_secs(3600)), - config - .property_or_default("cache.bayes.ttl.negative", "1h") - .unwrap_or_else(|| Duration::from_secs(3600)), - ), - remote_lists: Default::default(), - } - } -} - impl Default for Scripting { fn default() -> Self { Scripting { @@ -410,19 +384,6 @@ impl Default for Scripting { } } -impl Default for ScriptCache { - fn default() -> Self { - Self { - bayes_cache: BayesTokenCache::new( - 8192, - Duration::from_secs(3600), - Duration::from_secs(3600), - ), - remote_lists: Default::default(), - } - } -} - impl Clone for Scripting { fn clone(&self) -> Self { Self { diff --git a/crates/common/src/config/server/listener.rs b/crates/common/src/config/server/listener.rs index 130a6b91..651b841d 100644 --- a/crates/common/src/config/server/listener.rs +++ b/crates/common/src/config/server/listener.rs @@ -23,18 +23,18 @@ use utils::{ use crate::{ listener::{tls::CertificateResolver, TcpAcceptor}, - SharedCore, + Inner, }; use super::{ tls::{TLS12_VERSION, TLS13_VERSION}, - Listener, Server, ServerProtocol, Servers, + Listener, Listeners, ServerProtocol, TcpListener, }; -impl Servers { +impl Listeners { pub fn parse(config: &mut Config) -> Self { // Parse ACME managers - let mut servers = Servers { + let mut servers = Listeners { span_id_gen: Arc::new( config .property::<u64>("cluster.node-id") @@ -139,7 +139,7 @@ impl Servers { let _ = socket.set_reuseaddr(true); } - listeners.push(Listener { + listeners.push(TcpListener { socket, addr, ttl: config @@ -197,7 +197,7 @@ impl Servers { } let span_id_gen = self.span_id_gen.clone(); - self.servers.push(Server { + self.servers.push(Listener { max_connections: config .property_or_else( ("server.listener", id, "max-connections"), @@ -213,8 +213,8 @@ impl Servers { }); } - pub fn parse_tcp_acceptors(&mut self, config: &mut Config, core: SharedCore) { - let resolver = Arc::new(CertificateResolver::new(core.clone())); + pub fn parse_tcp_acceptors(&mut self, config: &mut Config, inner: Arc<Inner>) { + let resolver = Arc::new(CertificateResolver::new(inner.clone())); for id_ in config .sub_keys("server.listener", ".protocol") diff --git a/crates/common/src/config/server/mod.rs b/crates/common/src/config/server/mod.rs index 67c12c00..9040af39 100644 --- a/crates/common/src/config/server/mod.rs +++ b/crates/common/src/config/server/mod.rs @@ -17,24 +17,24 @@ pub mod listener; pub mod tls; #[derive(Default)] -pub struct Servers { - pub servers: Vec<Server>, +pub struct Listeners { + pub servers: Vec<Listener>, pub tcp_acceptors: AHashMap<String, TcpAcceptor>, pub span_id_gen: Arc<SnowflakeIdGenerator>, } #[derive(Debug, Default)] -pub struct Server { +pub struct Listener { pub id: String, pub protocol: ServerProtocol, - pub listeners: Vec<Listener>, + pub listeners: Vec<TcpListener>, pub proxy_networks: Vec<IpAddrMask>, pub max_connections: u64, pub span_id_gen: Arc<SnowflakeIdGenerator>, } #[derive(Debug)] -pub struct Listener { +pub struct TcpListener { pub socket: TcpSocket, pub addr: SocketAddr, pub backlog: Option<u32>, diff --git a/crates/common/src/config/server/tls.rs b/crates/common/src/config/server/tls.rs index a98d8583..dfe199d0 100644 --- a/crates/common/src/config/server/tls.rs +++ b/crates/common/src/config/server/tls.rs @@ -12,7 +12,6 @@ use std::{ }; use ahash::{AHashMap, AHashSet}; -use arc_swap::ArcSwap; use base64::{engine::general_purpose::STANDARD, Engine}; use dns_update::{providers::rfc2136::DnsAddress, DnsUpdater, TsigAlgorithm}; use rcgen::generate_simple_self_signed; @@ -33,20 +32,15 @@ use x509_parser::{ use crate::listener::{ acme::{directory::LETS_ENCRYPT_PRODUCTION_DIRECTORY, AcmeProvider, ChallengeSettings}, - tls::TlsManager, + tls::AcmeProviders, }; pub static TLS13_VERSION: &[&SupportedProtocolVersion] = &[&TLS13]; pub static TLS12_VERSION: &[&SupportedProtocolVersion] = &[&TLS12]; -impl TlsManager { +impl AcmeProviders { pub fn parse(config: &mut Config) -> Self { - let mut certificates = AHashMap::new(); - let mut acme_providers = AHashMap::new(); - let mut subject_names = AHashSet::new(); - - // Parse certificates - parse_certificates(config, &mut certificates, &mut subject_names); + let mut providers = AHashMap::new(); // Parse ACME providers 'outer: for acme_id in config @@ -140,9 +134,6 @@ impl TlsManager { .property::<bool>(("acme", acme_id, "default")) .unwrap_or_default(); - // Add domains for self-signed certificate - subject_names.extend(domains.iter().cloned()); - if !domains.is_empty() { match AcmeProvider::new( acme_id.to_string(), @@ -154,7 +145,7 @@ impl TlsManager { default, ) { Ok(acme_provider) => { - acme_providers.insert(acme_id.to_string(), acme_provider); + providers.insert(acme_id.to_string(), acme_provider); } Err(err) => { config.new_build_error(format!("acme.{acme_id}"), err.to_string()); @@ -163,21 +154,7 @@ impl TlsManager { } } - if subject_names.is_empty() { - subject_names.insert("localhost".to_string()); - } - - TlsManager { - certificates: ArcSwap::from_pointee(certificates), - acme_providers, - self_signed_cert: build_self_signed_cert(subject_names.into_iter().collect::<Vec<_>>()) - .or_else(|err| { - config.new_build_error("certificate.self-signed", err); - build_self_signed_cert(vec!["localhost".to_string()]) - }) - .ok() - .map(Arc::new), - } + AcmeProviders { providers } } } diff --git a/crates/common/src/config/smtp/resolver.rs b/crates/common/src/config/smtp/resolver.rs index ffe43328..b6e6802f 100644 --- a/crates/common/src/config/smtp/resolver.rs +++ b/crates/common/src/config/smtp/resolver.rs @@ -24,7 +24,7 @@ use mail_auth::{ use parking_lot::Mutex; use utils::config::{utils::ParseValue, Config}; -use crate::Core; +use crate::Server; pub struct Resolvers { pub dns: Resolver, @@ -307,15 +307,27 @@ impl Policy { } } -impl Core { +impl Server { pub fn build_mta_sts_policy(&self) -> Option<Policy> { - self.smtp.session.mta_sts_policy.clone().and_then(|policy| { - policy.try_build(self.tls.certificates.load().keys().filter(|key| { - !key.starts_with("mta-sts.") - && !key.starts_with("autoconfig.") - && !key.starts_with("autodiscover.") - })) - }) + self.core + .smtp + .session + .mta_sts_policy + .clone() + .and_then(|policy| { + policy.try_build( + self.inner + .data + .tls_certificates + .load() + .keys() + .filter(|key| { + !key.starts_with("mta-sts.") + && !key.starts_with("autoconfig.") + && !key.starts_with("autodiscover.") + }), + ) + }) } } diff --git a/crates/common/src/core.rs b/crates/common/src/core.rs new file mode 100644 index 00000000..2ebf66dd --- /dev/null +++ b/crates/common/src/core.rs @@ -0,0 +1,335 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs Ltd <hello@stalw.art> + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use std::{net::IpAddr, sync::Arc}; + +use directory::{ + backend::internal::manage::ManageDirectory, core::secret::verify_secret_hash, Directory, + Principal, QueryBy, Type, +}; +use mail_send::Credentials; +use sieve::Sieve; +use store::{ + write::{QueueClass, ValueClass}, + BlobStore, FtsStore, IterateParams, LookupStore, Store, ValueKey, +}; +use trc::AddContext; + +use crate::{ + config::smtp::{ + auth::{ArcSealer, DkimSigner}, + queue::RelayHost, + }, + ImapId, Inner, MailboxState, Server, +}; + +impl Server { + #[inline(always)] + pub fn store(&self) -> &Store { + &self.core.storage.data + } + + #[inline(always)] + pub fn blob_store(&self) -> &BlobStore { + &self.core.storage.blob + } + + #[inline(always)] + pub fn fts_store(&self) -> &FtsStore { + &self.core.storage.fts + } + + #[inline(always)] + pub fn lookup_store(&self) -> &LookupStore { + &self.core.storage.lookup + } + + #[inline(always)] + pub fn directory(&self) -> &Directory { + &self.core.storage.directory + } + + pub fn get_directory(&self, name: &str) -> Option<&Arc<Directory>> { + self.core.storage.directories.get(name) + } + + pub fn get_directory_or_default(&self, name: &str, session_id: u64) -> &Arc<Directory> { + self.core.storage.directories.get(name).unwrap_or_else(|| { + if !name.is_empty() { + trc::event!( + Eval(trc::EvalEvent::DirectoryNotFound), + Id = name.to_string(), + SpanId = session_id, + ); + } + + &self.core.storage.directory + }) + } + + pub fn get_lookup_store(&self, name: &str, session_id: u64) -> &LookupStore { + self.core.storage.lookups.get(name).unwrap_or_else(|| { + if !name.is_empty() { + trc::event!( + Eval(trc::EvalEvent::StoreNotFound), + Id = name.to_string(), + SpanId = session_id, + ); + } + + &self.core.storage.lookup + }) + } + + pub fn get_arc_sealer(&self, name: &str, session_id: u64) -> Option<&ArcSealer> { + self.core + .smtp + .mail_auth + .sealers + .get(name) + .map(|s| s.as_ref()) + .or_else(|| { + trc::event!( + Arc(trc::ArcEvent::SealerNotFound), + Id = name.to_string(), + SpanId = session_id, + ); + + None + }) + } + + pub fn get_dkim_signer(&self, name: &str, session_id: u64) -> Option<&DkimSigner> { + self.core + .smtp + .mail_auth + .signers + .get(name) + .map(|s| s.as_ref()) + .or_else(|| { + trc::event!( + Dkim(trc::DkimEvent::SignerNotFound), + Id = name.to_string(), + SpanId = session_id, + ); + + None + }) + } + + pub fn get_trusted_sieve_script(&self, name: &str, session_id: u64) -> Option<&Arc<Sieve>> { + self.core.sieve.trusted_scripts.get(name).or_else(|| { + trc::event!( + Sieve(trc::SieveEvent::ScriptNotFound), + Id = name.to_string(), + SpanId = session_id, + ); + + None + }) + } + + pub fn get_untrusted_sieve_script(&self, name: &str, session_id: u64) -> Option<&Arc<Sieve>> { + self.core.sieve.untrusted_scripts.get(name).or_else(|| { + trc::event!( + Sieve(trc::SieveEvent::ScriptNotFound), + Id = name.to_string(), + SpanId = session_id, + ); + + None + }) + } + + pub fn get_relay_host(&self, name: &str, session_id: u64) -> Option<&RelayHost> { + self.core.smtp.queue.relay_hosts.get(name).or_else(|| { + trc::event!( + Smtp(trc::SmtpEvent::RemoteIdNotFound), + Id = name.to_string(), + SpanId = session_id, + ); + + None + }) + } + + pub async fn authenticate( + &self, + directory: &Directory, + session_id: u64, + credentials: &Credentials<String>, + remote_ip: IpAddr, + return_member_of: bool, + ) -> trc::Result<Principal> { + // First try to authenticate the user against the default directory + let result = match directory + .query(QueryBy::Credentials(credentials), return_member_of) + .await + { + Ok(Some(principal)) => { + trc::event!( + Auth(trc::AuthEvent::Success), + AccountName = credentials.login().to_string(), + AccountId = principal.id(), + SpanId = session_id, + Type = principal.typ().as_str(), + ); + + return Ok(principal); + } + Ok(None) => Ok(()), + Err(err) => { + if err.matches(trc::EventType::Auth(trc::AuthEvent::MissingTotp)) { + return Err(err); + } else { + Err(err) + } + } + }; + + // Then check if the credentials match the fallback admin or master user + match ( + &self.core.jmap.fallback_admin, + &self.core.jmap.master_user, + credentials, + ) { + (Some((fallback_admin, fallback_pass)), _, Credentials::Plain { username, secret }) + if username == fallback_admin => + { + if verify_secret_hash(fallback_pass, secret).await? { + trc::event!( + Auth(trc::AuthEvent::Success), + AccountName = username.clone(), + SpanId = session_id, + ); + + return Ok(Principal::fallback_admin(fallback_pass)); + } + } + (_, Some((master_user, master_pass)), Credentials::Plain { username, secret }) + if username.ends_with(master_user) => + { + if verify_secret_hash(master_pass, secret).await? { + let username = username.strip_suffix(master_user).unwrap(); + let username = username.strip_suffix('%').unwrap_or(username); + + if let Some(principal) = directory + .query(QueryBy::Name(username), return_member_of) + .await? + { + trc::event!( + Auth(trc::AuthEvent::Success), + AccountName = username.to_string(), + SpanId = session_id, + AccountId = principal.id(), + Type = principal.typ().as_str(), + ); + + return Ok(principal); + } + } + } + _ => {} + } + + if let Err(err) = result { + Err(err) + } else if self.has_auth_fail2ban() { + let login = credentials.login(); + if self.is_auth_fail2banned(remote_ip, login).await? { + Err(trc::SecurityEvent::AuthenticationBan + .into_err() + .ctx(trc::Key::RemoteIp, remote_ip) + .ctx(trc::Key::AccountName, login.to_string())) + } else { + Err(trc::AuthEvent::Failed + .ctx(trc::Key::RemoteIp, remote_ip) + .ctx(trc::Key::AccountName, login.to_string())) + } + } else { + Err(trc::AuthEvent::Failed + .ctx(trc::Key::RemoteIp, remote_ip) + .ctx(trc::Key::AccountName, credentials.login().to_string())) + } + } + + pub async fn total_queued_messages(&self) -> trc::Result<u64> { + let mut total = 0; + self.store() + .iterate( + IterateParams::new( + ValueKey::from(ValueClass::Queue(QueueClass::Message(0))), + ValueKey::from(ValueClass::Queue(QueueClass::Message(u64::MAX))), + ) + .no_values(), + |_, _| { + total += 1; + + Ok(true) + }, + ) + .await + .map(|_| total) + } + + pub async fn total_accounts(&self) -> trc::Result<u64> { + self.store() + .count_principals(None, Type::Individual.into(), None) + .await + .caused_by(trc::location!()) + } + + pub async fn total_domains(&self) -> trc::Result<u64> { + self.store() + .count_principals(None, Type::Domain.into(), None) + .await + .caused_by(trc::location!()) + } +} + +pub trait BuildServer { + fn build_server(&self) -> Server; +} + +impl BuildServer for Arc<Inner> { + fn build_server(&self) -> Server { + Server { + inner: self.clone(), + core: self.shared_core.load_full(), + } + } +} + +trait CredentialsUsername { + fn login(&self) -> &str; +} + +impl CredentialsUsername for Credentials<String> { + fn login(&self) -> &str { + match self { + Credentials::Plain { username, .. } + | Credentials::XOauth2 { username, .. } + | Credentials::OAuthBearer { token: username } => username, + } + } +} + +impl MailboxState { + pub fn map_result_id(&self, document_id: u32, is_uid: bool) -> Option<(u32, ImapId)> { + if let Some(imap_id) = self.id_to_imap.get(&document_id) { + Some((if is_uid { imap_id.uid } else { imap_id.seqnum }, *imap_id)) + } else if is_uid { + self.next_state.as_ref().and_then(|s| { + s.next_state + .id_to_imap + .get(&document_id) + .map(|imap_id| (imap_id.uid, *imap_id)) + }) + } else { + None + } + } +} diff --git a/crates/common/src/enterprise/alerts.rs b/crates/common/src/enterprise/alerts.rs index a7d26404..6cff45e2 100644 --- a/crates/common/src/enterprise/alerts.rs +++ b/crates/common/src/enterprise/alerts.rs @@ -20,7 +20,7 @@ use trc::{Collector, MetricType, TelemetryEvent, TOTAL_EVENT_COUNT}; use super::{AlertContent, AlertContentToken, AlertMethod}; use crate::{ expr::{functions::ResolveVariable, Variable}, - Core, + Server, }; use std::fmt::Write; @@ -33,9 +33,9 @@ pub struct AlertMessage { struct CollectorResolver; -impl Core { +impl Server { pub async fn process_alerts(&self) -> Option<Vec<AlertMessage>> { - let alerts = &self.enterprise.as_ref()?.metrics_alerts; + let alerts = &self.core.enterprise.as_ref()?.metrics_alerts; if alerts.is_empty() { return None; } diff --git a/crates/common/src/enterprise/mod.rs b/crates/common/src/enterprise/mod.rs index 69608023..9f634564 100644 --- a/crates/common/src/enterprise/mod.rs +++ b/crates/common/src/enterprise/mod.rs @@ -25,7 +25,7 @@ use store::Store; use trc::{AddContext, EventType, MetricType}; use utils::config::cron::SimpleCron; -use crate::{expr::Expression, manager::webadmin::Resource, Core, HttpLimitResponse}; +use crate::{expr::Expression, manager::webadmin::Resource, Core, HttpLimitResponse, Server}; #[derive(Clone)] pub struct Enterprise { @@ -87,6 +87,14 @@ pub enum AlertContentToken { } impl Core { + pub fn is_enterprise_edition(&self) -> bool { + self.enterprise + .as_ref() + .map_or(false, |e| !e.license.is_expired()) + } +} + +impl Server { // WARNING: TAMPERING WITH THIS FUNCTION IS STRICTLY PROHIBITED // Any attempt to modify, bypass, or disable this license validation mechanism // constitutes a severe violation of the Stalwart Enterprise License Agreement. @@ -96,18 +104,20 @@ impl Core { // violators to the fullest extent of the law, including but not limited to claims // for copyright infringement, breach of contract, and fraud. + #[inline] pub fn is_enterprise_edition(&self) -> bool { - self.enterprise - .as_ref() - .map_or(false, |e| !e.license.is_expired()) + self.core.is_enterprise_edition() } pub fn licensed_accounts(&self) -> u32 { - self.enterprise.as_ref().map_or(0, |e| e.license.accounts) + self.core + .enterprise + .as_ref() + .map_or(0, |e| e.license.accounts) } pub fn log_license_details(&self) { - if let Some(enterprise) = &self.enterprise { + if let Some(enterprise) = &self.core.enterprise { trc::event!( Server(trc::ServerEvent::Licensing), Details = "Stalwart Enterprise Edition license key is valid", @@ -125,15 +135,14 @@ impl Core { if self.is_enterprise_edition() { let domain = psl::domain_str(domain).unwrap_or(domain); - let logo = { self.security.logos.lock().get(domain).cloned() }; + let logo = { self.inner.data.logos.lock().get(domain).cloned() }; if let Some(logo) = logo { Ok(logo) } else { // Try fetching the logo for the domain let logo_url = if let Some(mut principal) = self - .storage - .data + .store() .query(QueryBy::Name(domain), false) .await .caused_by(trc::location!())? @@ -146,8 +155,7 @@ impl Core { logo.into() } else if let Some(tenant_id) = principal.get_int(PrincipalField::Tenant) { if let Some(logo) = self - .storage - .data + .store() .query(QueryBy::Id(tenant_id as u32), false) .await .caused_by(trc::location!())? @@ -199,7 +207,8 @@ impl Core { logo = Resource::new(content_type, contents).into(); } - self.security + self.inner + .data .logos .lock() .insert(domain.to_string(), logo.clone()); @@ -212,7 +221,8 @@ impl Core { } fn default_logo_url(&self) -> Option<String> { - self.enterprise + self.core + .enterprise .as_ref() .and_then(|e| e.logo_url.as_ref().map(|l| l.to_string())) } diff --git a/crates/common/src/expr/eval.rs b/crates/common/src/expr/eval.rs index 371a0476..0e87c203 100644 --- a/crates/common/src/expr/eval.rs +++ b/crates/common/src/expr/eval.rs @@ -9,7 +9,7 @@ use std::{borrow::Cow, cmp::Ordering, fmt::Display}; use hyper::StatusCode; use trc::EvalEvent; -use crate::Core; +use crate::Server; use super::{ functions::{ResolveVariable, FUNCTIONS}, @@ -17,7 +17,7 @@ use super::{ BinaryOperator, Constant, Expression, ExpressionItem, UnaryOperator, Variable, }; -impl Core { +impl Server { pub async fn eval_if<'x, R: TryFrom<Variable<'x>>, V: ResolveVariable>( &self, if_block: &'x IfBlock, @@ -123,7 +123,7 @@ impl IfBlock { pub async fn eval<'x, V: ResolveVariable>( &'x self, resolver: &'x V, - core: &Core, + core: &Server, session_id: u64, ) -> trc::Result<Variable<'x>> { let mut captures = Vec::new(); @@ -152,7 +152,7 @@ impl Expression { async fn eval<'x, 'y, V: ResolveVariable>( &'x self, resolver: &'x V, - core: &Core, + core: &Server, captures: &'y mut Vec<String>, session_id: u64, ) -> trc::Result<Variable<'x>> { diff --git a/crates/common/src/expr/functions/asynch.rs b/crates/common/src/expr/functions/asynch.rs index 7e94f7b2..162929cc 100644 --- a/crates/common/src/expr/functions/asynch.rs +++ b/crates/common/src/expr/functions/asynch.rs @@ -4,11 +4,11 @@ use mail_auth::IpLookupStrategy; use store::{Deserialize, Rows, Value}; use trc::AddContext; -use crate::Core; +use crate::Server; use super::*; -impl Core { +impl Server { pub(crate) async fn eval_fnc<'x>( &self, fnc_id: u32, @@ -168,7 +168,8 @@ impl Core { let record_type = arguments.next_as_string(); if record_type.eq_ignore_ascii_case("ip") { - self.smtp + self.core + .smtp .resolvers .dns .ip_lookup(entry.as_ref(), IpLookupStrategy::Ipv4thenIpv6, 10) @@ -182,7 +183,8 @@ impl Core { .into() }) } else if record_type.eq_ignore_ascii_case("mx") { - self.smtp + self.core + .smtp .resolvers .dns .mx_lookup(entry.as_ref()) @@ -205,7 +207,8 @@ impl Core { .into() }) } else if record_type.eq_ignore_ascii_case("txt") { - self.smtp + self.core + .smtp .resolvers .dns .txt_raw_lookup(entry.as_ref()) @@ -213,7 +216,8 @@ impl Core { .map_err(|err| trc::Error::from(err).caused_by(trc::location!())) .map(|result| Variable::from(String::from_utf8(result).unwrap_or_default())) } else if record_type.eq_ignore_ascii_case("ptr") { - self.smtp + self.core + .smtp .resolvers .dns .ptr_lookup(entry.parse::<IpAddr>().map_err(|err| { @@ -232,7 +236,8 @@ impl Core { .into() }) } else if record_type.eq_ignore_ascii_case("ipv4") { - self.smtp + self.core + .smtp .resolvers .dns .ipv4_lookup(entry.as_ref()) @@ -246,7 +251,8 @@ impl Core { .into() }) } else if record_type.eq_ignore_ascii_case("ipv6") { - self.smtp + self.core + .smtp .resolvers .dns .ipv6_lookup(entry.as_ref()) diff --git a/crates/common/src/expr/functions/mod.rs b/crates/common/src/expr/functions/mod.rs index 94646b36..3d6f0c28 100644 --- a/crates/common/src/expr/functions/mod.rs +++ b/crates/common/src/expr/functions/mod.rs @@ -14,7 +14,7 @@ pub mod email; pub mod misc; pub mod text; -pub trait ResolveVariable { +pub trait ResolveVariable: Sync + Send { fn resolve_variable(&self, variable: u32) -> Variable<'_>; } diff --git a/crates/common/src/ipc.rs b/crates/common/src/ipc.rs new file mode 100644 index 00000000..9b64a318 --- /dev/null +++ b/crates/common/src/ipc.rs @@ -0,0 +1,233 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs Ltd <hello@stalw.art> + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use std::{borrow::Cow, sync::Arc, time::Instant}; + +use ahash::RandomState; +use jmap_proto::types::{state::StateChange, type_state::DataType}; +use mail_auth::{ + dmarc::Dmarc, + mta_sts::TlsRpt, + report::{tlsrpt::FailureDetails, Record}, +}; +use store::{BlobStore, LookupStore, Store}; +use tokio::sync::{mpsc, oneshot}; +use utils::{map::bitmap::Bitmap, BlobHash}; + +use crate::{ + config::smtp::{ + report::AggregateFrequency, + resolver::{Policy, Tlsa}, + }, + listener::limiter::ConcurrencyLimiter, +}; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum DeliveryResult { + Success, + TemporaryFailure { + reason: Cow<'static, str>, + }, + PermanentFailure { + code: [u8; 3], + reason: Cow<'static, str>, + }, +} + +#[derive(Debug)] +pub enum DeliveryEvent { + Ingest { + message: IngestMessage, + result_tx: oneshot::Sender<Vec<DeliveryResult>>, + }, + Stop, +} + +#[derive(Debug)] +pub struct IngestMessage { + pub sender_address: String, + pub recipients: Vec<String>, + pub message_blob: BlobHash, + pub message_size: usize, + pub session_id: u64, +} + +pub enum HousekeeperEvent { + AcmeReschedule { + provider_id: String, + renew_at: Instant, + }, + Purge(PurgeType), + ReloadSettings, + Exit, +} + +pub enum PurgeType { + Data(Store), + Blobs { store: Store, blob_store: BlobStore }, + Lookup(LookupStore), + Account(Option<u32>), +} + +#[derive(Debug)] +pub enum StateEvent { + Subscribe { + account_id: u32, + types: Bitmap<DataType>, + tx: mpsc::Sender<StateChange>, + }, + Publish { + state_change: StateChange, + }, + UpdateSharedAccounts { + account_id: u32, + }, + UpdateSubscriptions { + account_id: u32, + subscriptions: Vec<UpdateSubscription>, + }, + Stop, +} + +#[derive(Debug)] +pub enum UpdateSubscription { + Unverified { + id: u32, + url: String, + code: String, + keys: Option<EncryptionKeys>, + }, + Verified(PushSubscription), +} + +#[derive(Debug)] +pub struct PushSubscription { + pub id: u32, + pub url: String, + pub expires: u64, + pub types: Bitmap<DataType>, + pub keys: Option<EncryptionKeys>, +} + +#[derive(Debug, Clone)] +pub struct EncryptionKeys { + pub p256dh: Vec<u8>, + pub auth: Vec<u8>, +} + +#[derive(Debug)] +pub enum QueueEvent { + Reload, + OnHold(OnHold<QueueEventLock>), + Stop, +} + +#[derive(Debug)] +pub struct OnHold<T> { + pub next_due: Option<u64>, + pub limiters: Vec<ConcurrencyLimiter>, + pub message: T, +} + +#[derive(Debug)] +pub struct QueueEventLock { + pub due: u64, + pub queue_id: u64, + pub lock_expiry: u64, +} + +#[derive(Debug)] +pub enum ReportingEvent { + Dmarc(Box<DmarcEvent>), + Tls(Box<TlsEvent>), + Stop, +} + +#[derive(Debug)] +pub struct DmarcEvent { + pub domain: String, + pub report_record: Record, + pub dmarc_record: Arc<Dmarc>, + pub interval: AggregateFrequency, +} + +#[derive(Debug)] +pub struct TlsEvent { + pub domain: String, + pub policy: PolicyType, + pub failure: Option<FailureDetails>, + pub tls_record: Arc<TlsRpt>, + pub interval: AggregateFrequency, +} + +#[derive(Debug, Hash, PartialEq, Eq)] +pub enum PolicyType { + Tlsa(Option<Arc<Tlsa>>), + Sts(Option<Arc<Policy>>), + None, +} + +pub trait ToHash { + fn to_hash(&self) -> u64; +} + +impl ToHash for Dmarc { + fn to_hash(&self) -> u64 { + RandomState::with_seeds(1, 9, 7, 9).hash_one(self) + } +} + +impl ToHash for PolicyType { + fn to_hash(&self) -> u64 { + RandomState::with_seeds(1, 9, 7, 9).hash_one(self) + } +} + +impl From<DmarcEvent> for ReportingEvent { + fn from(value: DmarcEvent) -> Self { + ReportingEvent::Dmarc(Box::new(value)) + } +} + +impl From<TlsEvent> for ReportingEvent { + fn from(value: TlsEvent) -> Self { + ReportingEvent::Tls(Box::new(value)) + } +} + +impl From<Arc<Tlsa>> for PolicyType { + fn from(value: Arc<Tlsa>) -> Self { + PolicyType::Tlsa(Some(value)) + } +} + +impl From<Arc<Policy>> for PolicyType { + fn from(value: Arc<Policy>) -> Self { + PolicyType::Sts(Some(value)) + } +} + +impl From<&Arc<Tlsa>> for PolicyType { + fn from(value: &Arc<Tlsa>) -> Self { + PolicyType::Tlsa(Some(value.clone())) + } +} + +impl From<&Arc<Policy>> for PolicyType { + fn from(value: &Arc<Policy>) -> Self { + PolicyType::Sts(Some(value.clone())) + } +} + +impl From<(&Option<Arc<Policy>>, &Option<Arc<Tlsa>>)> for PolicyType { + fn from(value: (&Option<Arc<Policy>>, &Option<Arc<Tlsa>>)) -> Self { + match value { + (Some(value), _) => PolicyType::Sts(Some(value.clone())), + (_, Some(value)) => PolicyType::Tlsa(Some(value.clone())), + _ => PolicyType::None, + } + } +} diff --git a/crates/common/src/lib.rs b/crates/common/src/lib.rs index e8dbf60e..f983917c 100644 --- a/crates/common/src/lib.rs +++ b/crates/common/src/lib.rs @@ -5,59 +5,52 @@ */ use std::{ - borrow::Cow, + collections::BTreeMap, + hash::{BuildHasher, Hasher}, net::IpAddr, sync::{atomic::AtomicU8, Arc}, }; -use ahash::AHashMap; +use ahash::{AHashMap, AHashSet, RandomState}; use arc_swap::ArcSwap; use auth::{roles::RolePermissions, AccessToken}; use config::{ imap::ImapConfig, jmap::settings::JmapConfig, - scripts::Scripting, - smtp::{ - auth::{ArcSealer, DkimSigner}, - queue::RelayHost, - SmtpConfig, - }, + scripts::{RemoteList, Scripting}, + smtp::SmtpConfig, storage::Storage, telemetry::Metrics, }; -use directory::{ - backend::internal::manage::ManageDirectory, core::secret::verify_secret_hash, Directory, - Principal, QueryBy, Type, -}; +use dashmap::DashMap; + use expr::if_block::IfBlock; use futures::StreamExt; -use listener::{ - blocked::{AllowedIps, BlockedIps}, - tls::TlsManager, -}; -use mail_send::Credentials; +use imap_proto::protocol::list::Attribute; +use ipc::{DeliveryEvent, HousekeeperEvent, QueueEvent, ReportingEvent, StateEvent}; +use listener::{blocked::Security, limiter::ConcurrencyLimiter, tls::AcmeProviders}; -use manager::webadmin::Resource; -use parking_lot::Mutex; +use manager::webadmin::{Resource, WebAdminManager}; +use nlp::bayes::cache::BayesTokenCache; +use parking_lot::{Mutex, RwLock}; use reqwest::Response; -use sieve::Sieve; -use store::{ - write::{QueueClass, ValueClass}, - IterateParams, LookupStore, ValueKey, -}; -use tokio::sync::{mpsc, oneshot}; -use trc::AddContext; +use rustls::sign::CertifiedKey; +use tokio::sync::{mpsc, Notify}; +use tokio_rustls::TlsConnector; use utils::{ + lru_cache::LruCache, map::ttl_dashmap::{ADashMap, TtlDashMap}, - BlobHash, + snowflake::SnowflakeIdGenerator, }; pub mod addresses; pub mod auth; pub mod config; +pub mod core; #[cfg(feature = "enterprise")] pub mod enterprise; pub mod expr; +pub mod ipc; pub mod listener; pub mod manager; pub mod scripts; @@ -70,351 +63,219 @@ pub static DAEMON_NAME: &str = concat!("Stalwart Mail Server v", env!("CARGO_PKG pub const IPC_CHANNEL_BUFFER: usize = 1024; -pub type SharedCore = Arc<ArcSwap<Core>>; - #[derive(Clone, Default)] -pub struct Core { - pub storage: Storage, - pub sieve: Scripting, - pub network: Network, - pub tls: TlsManager, - pub smtp: SmtpConfig, - pub jmap: JmapConfig, - pub imap: ImapConfig, - pub metrics: Metrics, - pub security: Security, - #[cfg(feature = "enterprise")] - pub enterprise: Option<enterprise::Enterprise>, +pub struct Server { + pub inner: Arc<Inner>, + pub core: Arc<Core>, } -//TODO: temporary hack until OIDC is implemented #[derive(Default)] -pub struct Security { - pub logos: Mutex<AHashMap<String, Option<Resource<Vec<u8>>>>>, +pub struct Inner { + pub shared_core: ArcSwap<Core>, + pub data: Data, + pub ipc: Ipc, +} + +pub struct Data { + pub tls_certificates: ArcSwap<AHashMap<String, Arc<CertifiedKey>>>, + pub tls_self_signed_cert: Option<Arc<CertifiedKey>>, + pub access_tokens: TtlDashMap<u32, Arc<AccessToken>>, + pub http_auth_cache: TtlDashMap<String, u32>, + + pub blocked_ips: RwLock<AHashSet<IpAddr>>, + pub blocked_ips_version: AtomicU8, + pub permissions: ADashMap<u32, Arc<RolePermissions>>, pub permissions_version: AtomicU8, -} -#[derive(Clone)] -pub struct Network { - pub node_id: u64, - pub blocked_ips: BlockedIps, - pub allowed_ips: AllowedIps, - pub http_response_url: IfBlock, - pub http_allowed_endpoint: IfBlock, -} + pub bayes_cache: BayesTokenCache, + pub remote_lists: RwLock<AHashMap<String, RemoteList>>, -#[derive(Debug)] -pub enum DeliveryEvent { - Ingest { - message: IngestMessage, - result_tx: oneshot::Sender<Vec<DeliveryResult>>, - }, - Stop, + pub jmap_id_gen: SnowflakeIdGenerator, + pub queue_id_gen: SnowflakeIdGenerator, + pub span_id_gen: SnowflakeIdGenerator, + + pub webadmin: WebAdminManager, + pub config_version: AtomicU8, + + pub jmap_limiter: DashMap<u32, Arc<ConcurrencyLimiters>, RandomState>, + pub imap_limiter: DashMap<u32, Arc<ConcurrencyLimiters>, RandomState>, + + pub account_cache: LruCache<AccountId, Arc<Account>>, + pub mailbox_cache: LruCache<MailboxId, Arc<MailboxState>>, + pub threads_cache: LruCache<u32, Arc<Threads>>, + + pub logos: Mutex<AHashMap<String, Option<Resource<Vec<u8>>>>>, + + pub smtp_session_throttle: DashMap<ThrottleKey, ConcurrencyLimiter, ThrottleKeyHasherBuilder>, + pub smtp_queue_throttle: DashMap<ThrottleKey, ConcurrencyLimiter, ThrottleKeyHasherBuilder>, + pub smtp_connectors: TlsConnectors, } pub struct Ipc { + pub state_tx: mpsc::Sender<StateEvent>, + pub housekeeper_tx: mpsc::Sender<HousekeeperEvent>, pub delivery_tx: mpsc::Sender<DeliveryEvent>, + pub index_tx: Arc<Notify>, + pub queue_tx: mpsc::Sender<QueueEvent>, + pub report_tx: mpsc::Sender<ReportingEvent>, } -#[derive(Debug)] -pub struct IngestMessage { - pub sender_address: String, - pub recipients: Vec<String>, - pub message_blob: BlobHash, - pub message_size: usize, - pub session_id: u64, +pub struct TlsConnectors { + pub pki_verify: TlsConnector, + pub dummy_verify: TlsConnector, } -#[derive(Debug, Clone, PartialEq, Eq)] -pub enum DeliveryResult { - Success, - TemporaryFailure { - reason: Cow<'static, str>, - }, - PermanentFailure { - code: [u8; 3], - reason: Cow<'static, str>, - }, +#[derive(Debug, PartialEq, Eq, Hash, Clone, Copy)] +pub struct AccountId { + pub account_id: u32, + pub primary_id: u32, } -pub trait IntoString: Sized { - fn into_string(self) -> String; +#[derive(Debug, PartialEq, Eq, Hash, Clone, Copy)] +pub struct MailboxId { + pub account_id: u32, + pub mailbox_id: u32, } -impl IntoString for Vec<u8> { - fn into_string(self) -> String { - String::from_utf8(self) - .unwrap_or_else(|err| String::from_utf8_lossy(err.as_bytes()).into_owned()) - } +#[derive(Debug, Clone, Default)] +pub struct Account { + pub account_id: u32, + pub prefix: Option<String>, + pub mailbox_names: BTreeMap<String, u32>, + pub mailbox_state: AHashMap<u32, Mailbox>, + pub state_email: Option<u64>, + pub state_mailbox: Option<u64>, } -impl Core { - pub fn get_directory(&self, name: &str) -> Option<&Arc<Directory>> { - self.storage.directories.get(name) - } - - pub fn get_directory_or_default(&self, name: &str, session_id: u64) -> &Arc<Directory> { - self.storage.directories.get(name).unwrap_or_else(|| { - if !name.is_empty() { - trc::event!( - Eval(trc::EvalEvent::DirectoryNotFound), - Id = name.to_string(), - SpanId = session_id, - ); - } +#[derive(Debug, Default, Clone)] +pub struct Mailbox { + pub has_children: bool, + pub is_subscribed: bool, + pub special_use: Option<Attribute>, + pub total_messages: Option<u32>, + pub total_unseen: Option<u32>, + pub total_deleted: Option<u32>, + pub uid_validity: Option<u32>, + pub uid_next: Option<u32>, + pub size: Option<u32>, +} - &self.storage.directory - }) - } +#[derive(Debug, Clone, Default)] +pub struct MailboxState { + pub uid_next: u32, + pub uid_validity: u32, + pub uid_max: u32, + pub id_to_imap: AHashMap<u32, ImapId>, + pub uid_to_id: AHashMap<u32, u32>, + pub total_messages: usize, + pub modseq: Option<u64>, + pub next_state: Option<Box<NextMailboxState>>, +} - pub fn get_lookup_store(&self, name: &str, session_id: u64) -> &LookupStore { - self.storage.lookups.get(name).unwrap_or_else(|| { - if !name.is_empty() { - trc::event!( - Eval(trc::EvalEvent::StoreNotFound), - Id = name.to_string(), - SpanId = session_id, - ); - } +#[derive(Debug, Clone)] +pub struct NextMailboxState { + pub next_state: MailboxState, + pub deletions: Vec<ImapId>, +} - &self.storage.lookup - }) - } +#[derive(Debug, Clone, Copy, Default)] +pub struct ImapId { + pub uid: u32, + pub seqnum: u32, +} - pub fn get_arc_sealer(&self, name: &str, session_id: u64) -> Option<&ArcSealer> { - self.smtp - .mail_auth - .sealers - .get(name) - .map(|s| s.as_ref()) - .or_else(|| { - trc::event!( - Arc(trc::ArcEvent::SealerNotFound), - Id = name.to_string(), - SpanId = session_id, - ); - - None - }) - } +#[derive(Debug, Default)] +pub struct Threads { + pub threads: AHashMap<u32, u32>, + pub modseq: Option<u64>, +} - pub fn get_dkim_signer(&self, name: &str, session_id: u64) -> Option<&DkimSigner> { - self.smtp - .mail_auth - .signers - .get(name) - .map(|s| s.as_ref()) - .or_else(|| { - trc::event!( - Dkim(trc::DkimEvent::SignerNotFound), - Id = name.to_string(), - SpanId = session_id, - ); - - None - }) - } +pub struct ConcurrencyLimiters { + pub concurrent_requests: ConcurrencyLimiter, + pub concurrent_uploads: ConcurrencyLimiter, +} - pub fn get_trusted_sieve_script(&self, name: &str, session_id: u64) -> Option<&Arc<Sieve>> { - self.sieve.trusted_scripts.get(name).or_else(|| { - trc::event!( - Sieve(trc::SieveEvent::ScriptNotFound), - Id = name.to_string(), - SpanId = session_id, - ); +#[derive(Clone, Default)] +pub struct Core { + pub storage: Storage, + pub sieve: Scripting, + pub network: Network, + pub acme: AcmeProviders, + pub smtp: SmtpConfig, + pub jmap: JmapConfig, + pub imap: ImapConfig, + pub metrics: Metrics, + #[cfg(feature = "enterprise")] + pub enterprise: Option<enterprise::Enterprise>, +} - None - }) - } +#[derive(Clone)] +pub struct Network { + pub node_id: u64, + pub security: Security, + pub http_response_url: IfBlock, + pub http_allowed_endpoint: IfBlock, +} - pub fn get_untrusted_sieve_script(&self, name: &str, session_id: u64) -> Option<&Arc<Sieve>> { - self.sieve.untrusted_scripts.get(name).or_else(|| { - trc::event!( - Sieve(trc::SieveEvent::ScriptNotFound), - Id = name.to_string(), - SpanId = session_id, - ); +pub trait IntoString: Sized { + fn into_string(self) -> String; +} - None - }) +impl IntoString for Vec<u8> { + fn into_string(self) -> String { + String::from_utf8(self) + .unwrap_or_else(|err| String::from_utf8_lossy(err.as_bytes()).into_owned()) } +} - pub fn get_relay_host(&self, name: &str, session_id: u64) -> Option<&RelayHost> { - self.smtp.queue.relay_hosts.get(name).or_else(|| { - trc::event!( - Smtp(trc::SmtpEvent::RemoteIdNotFound), - Id = name.to_string(), - SpanId = session_id, - ); +#[derive(Debug, Clone, Eq)] +pub struct ThrottleKey { + pub hash: [u8; 32], +} - None - }) +impl PartialEq for ThrottleKey { + fn eq(&self, other: &Self) -> bool { + self.hash == other.hash } +} - pub async fn authenticate( - &self, - directory: &Directory, - session_id: u64, - credentials: &Credentials<String>, - remote_ip: IpAddr, - return_member_of: bool, - ) -> trc::Result<Principal> { - // First try to authenticate the user against the default directory - let result = match directory - .query(QueryBy::Credentials(credentials), return_member_of) - .await - { - Ok(Some(principal)) => { - trc::event!( - Auth(trc::AuthEvent::Success), - AccountName = credentials.login().to_string(), - AccountId = principal.id(), - SpanId = session_id, - Type = principal.typ().as_str(), - ); - - return Ok(principal); - } - Ok(None) => Ok(()), - Err(err) => { - if err.matches(trc::EventType::Auth(trc::AuthEvent::MissingTotp)) { - return Err(err); - } else { - Err(err) - } - } - }; - - // Then check if the credentials match the fallback admin or master user - match ( - &self.jmap.fallback_admin, - &self.jmap.master_user, - credentials, - ) { - (Some((fallback_admin, fallback_pass)), _, Credentials::Plain { username, secret }) - if username == fallback_admin => - { - if verify_secret_hash(fallback_pass, secret).await? { - trc::event!( - Auth(trc::AuthEvent::Success), - AccountName = username.clone(), - SpanId = session_id, - ); - - return Ok(Principal::fallback_admin(fallback_pass)); - } - } - (_, Some((master_user, master_pass)), Credentials::Plain { username, secret }) - if username.ends_with(master_user) => - { - if verify_secret_hash(master_pass, secret).await? { - let username = username.strip_suffix(master_user).unwrap(); - let username = username.strip_suffix('%').unwrap_or(username); - - if let Some(principal) = directory - .query(QueryBy::Name(username), return_member_of) - .await? - { - trc::event!( - Auth(trc::AuthEvent::Success), - AccountName = username.to_string(), - SpanId = session_id, - AccountId = principal.id(), - Type = principal.typ().as_str(), - ); - - return Ok(principal); - } - } - } - _ => {} - } - - if let Err(err) = result { - Err(err) - } else if self.has_auth_fail2ban() { - let login = credentials.login(); - if self.is_auth_fail2banned(remote_ip, login).await? { - Err(trc::SecurityEvent::AuthenticationBan - .into_err() - .ctx(trc::Key::RemoteIp, remote_ip) - .ctx(trc::Key::AccountName, login.to_string())) - } else { - Err(trc::AuthEvent::Failed - .ctx(trc::Key::RemoteIp, remote_ip) - .ctx(trc::Key::AccountName, login.to_string())) - } - } else { - Err(trc::AuthEvent::Failed - .ctx(trc::Key::RemoteIp, remote_ip) - .ctx(trc::Key::AccountName, credentials.login().to_string())) - } +impl std::hash::Hash for ThrottleKey { + fn hash<H: Hasher>(&self, state: &mut H) { + self.hash.hash(state); } +} - pub async fn total_queued_messages(&self) -> trc::Result<u64> { - let mut total = 0; - self.storage - .data - .iterate( - IterateParams::new( - ValueKey::from(ValueClass::Queue(QueueClass::Message(0))), - ValueKey::from(ValueClass::Queue(QueueClass::Message(u64::MAX))), - ) - .no_values(), - |_, _| { - total += 1; - - Ok(true) - }, - ) - .await - .map(|_| total) +impl AsRef<[u8]> for ThrottleKey { + fn as_ref(&self) -> &[u8] { + &self.hash } +} - pub async fn total_accounts(&self) -> trc::Result<u64> { - self.storage - .data - .count_principals(None, Type::Individual.into(), None) - .await - .caused_by(trc::location!()) +#[derive(Default)] +pub struct ThrottleKeyHasher { + hash: u64, +} + +impl Hasher for ThrottleKeyHasher { + fn finish(&self) -> u64 { + self.hash } - pub async fn total_domains(&self) -> trc::Result<u64> { - self.storage - .data - .count_principals(None, Type::Domain.into(), None) - .await - .caused_by(trc::location!()) + fn write(&mut self, bytes: &[u8]) { + self.hash = u64::from_ne_bytes((&bytes[..std::mem::size_of::<u64>()]).try_into().unwrap()); } } -trait CredentialsUsername { - fn login(&self) -> &str; -} +#[derive(Clone, Default)] +pub struct ThrottleKeyHasherBuilder {} -impl CredentialsUsername for Credentials<String> { - fn login(&self) -> &str { - match self { - Credentials::Plain { username, .. } - | Credentials::XOauth2 { username, .. } - | Credentials::OAuthBearer { token: username } => username, - } - } -} +impl BuildHasher for ThrottleKeyHasherBuilder { + type Hasher = ThrottleKeyHasher; -impl Clone for Security { - fn clone(&self) -> Self { - Self { - access_tokens: self.access_tokens.clone(), - permissions: self.permissions.clone(), - permissions_version: AtomicU8::new( - self.permissions_version - .load(std::sync::atomic::Ordering::Relaxed), - ), - logos: Mutex::new(self.logos.lock().clone()), - } + fn build_hasher(&self) -> Self::Hasher { + ThrottleKeyHasher::default() } } @@ -448,3 +309,23 @@ impl HttpLimitResponse for Response { Ok(Some(bytes)) } } + +impl ConcurrencyLimiters { + pub fn is_active(&self) -> bool { + self.concurrent_requests.is_active() || self.concurrent_uploads.is_active() + } +} + +#[cfg(feature = "test_mode")] +impl Default for Ipc { + fn default() -> Self { + Self { + state_tx: mpsc::channel(IPC_CHANNEL_BUFFER).0, + housekeeper_tx: mpsc::channel(IPC_CHANNEL_BUFFER).0, + delivery_tx: mpsc::channel(IPC_CHANNEL_BUFFER).0, + index_tx: Default::default(), + queue_tx: mpsc::channel(IPC_CHANNEL_BUFFER).0, + report_tx: mpsc::channel(IPC_CHANNEL_BUFFER).0, + } + } +} diff --git a/crates/common/src/listener/acme/cache.rs b/crates/common/src/listener/acme/cache.rs index 5c30ee43..c7269fc7 100644 --- a/crates/common/src/listener/acme/cache.rs +++ b/crates/common/src/listener/acme/cache.rs @@ -8,11 +8,11 @@ use base64::{engine::general_purpose::URL_SAFE_NO_PAD, Engine}; use trc::AddContext; use utils::config::ConfigKey; -use crate::Core; +use crate::Server; use super::AcmeProvider; -impl Core { +impl Server { pub(crate) async fn load_cert(&self, provider: &AcmeProvider) -> trc::Result<Option<Vec<u8>>> { self.read_if_exists(provider, "cert", provider.domains.as_slice()) .await @@ -68,6 +68,7 @@ impl Core { items: &[String], ) -> trc::Result<Option<Vec<u8>>> { if let Some(content) = self + .core .storage .config .get(self.build_key(provider, class, items)) @@ -94,7 +95,8 @@ impl Core { items: &[String], contents: impl AsRef<[u8]>, ) -> trc::Result<()> { - self.storage + self.core + .storage .config .set([ConfigKey { key: self.build_key(provider, class, items), diff --git a/crates/common/src/listener/acme/mod.rs b/crates/common/src/listener/acme/mod.rs index f429984d..c0faef56 100644 --- a/crates/common/src/listener/acme/mod.rs +++ b/crates/common/src/listener/acme/mod.rs @@ -16,7 +16,7 @@ use arc_swap::ArcSwap; use dns_update::DnsUpdater; use rustls::sign::CertifiedKey; -use crate::Core; +use crate::Server; use self::directory::{Account, ChallengeType}; @@ -80,7 +80,7 @@ impl AcmeProvider { } } -impl Core { +impl Server { pub async fn init_acme(&self, provider: &AcmeProvider) -> trc::Result<Duration> { // Load account key from cache or generate a new one if let Some(account_key) = self.load_account(provider).await? { @@ -100,15 +100,17 @@ impl Core { } pub fn has_acme_tls_providers(&self) -> bool { - self.tls - .acme_providers + self.core + .acme + .providers .values() .any(|p| matches!(p.challenge, ChallengeSettings::TlsAlpn01)) } pub fn has_acme_http_providers(&self) -> bool { - self.tls - .acme_providers + self.core + .acme + .providers .values() .any(|p| matches!(p.challenge, ChallengeSettings::Http01)) } diff --git a/crates/common/src/listener/acme/order.rs b/crates/common/src/listener/acme/order.rs index 983198ea..38fc1290 100644 --- a/crates/common/src/listener/acme/order.rs +++ b/crates/common/src/listener/acme/order.rs @@ -13,12 +13,12 @@ use x509_parser::parse_x509_certificate; use crate::listener::acme::directory::Identifier; use crate::listener::acme::ChallengeSettings; -use crate::Core; +use crate::Server; use super::directory::{Account, AuthStatus, Directory, OrderStatus}; use super::AcmeProvider; -impl Core { +impl Server { pub(crate) async fn process_cert( &self, provider: &AcmeProvider, @@ -210,8 +210,7 @@ impl Core { match &provider.challenge { ChallengeSettings::TlsAlpn01 => { - self.storage - .lookup + self.lookup_store() .key_set( format!("acme:{domain}").into_bytes(), account.tls_alpn_key(challenge, domain.clone())?, @@ -220,8 +219,7 @@ impl Core { .await?; } ChallengeSettings::Http01 => { - self.storage - .lookup + self.lookup_store() .key_set( format!("acme:{}", challenge.token).into_bytes(), account.http_proof(challenge)?, @@ -289,7 +287,7 @@ impl Core { let wait_until = Instant::now() + *propagation_timeout; let mut did_propagate = false; while Instant::now() < wait_until { - match self.smtp.resolvers.dns.txt_raw_lookup(&name).await { + match self.core.smtp.resolvers.dns.txt_raw_lookup(&name).await { Ok(result) => { let result = std::str::from_utf8(&result).unwrap_or_default(); if result.contains(&dns_proof) { diff --git a/crates/common/src/listener/acme/resolver.rs b/crates/common/src/listener/acme/resolver.rs index 26d91cfe..fec948fc 100644 --- a/crates/common/src/listener/acme/resolver.rs +++ b/crates/common/src/listener/acme/resolver.rs @@ -16,14 +16,14 @@ use rustls_pki_types::{CertificateDer, PrivateKeyDer, PrivatePkcs8KeyDer}; use store::write::Bincode; use trc::AcmeEvent; -use crate::{listener::acme::directory::SerializedCert, Core}; +use crate::{listener::acme::directory::SerializedCert, Server}; use super::{directory::ACME_TLS_ALPN_NAME, AcmeProvider, StaticResolver}; -impl Core { +impl Server { pub(crate) fn set_cert(&self, provider: &AcmeProvider, cert: Arc<CertifiedKey>) { // Add certificates - let mut certificates = self.tls.certificates.load().as_ref().clone(); + let mut certificates = self.inner.data.tls_certificates.load().as_ref().clone(); for domain in provider.domains.iter() { certificates.insert( domain @@ -39,29 +39,12 @@ impl Core { certificates.insert("*".to_string(), cert); } - self.tls.certificates.store(certificates.into()); + self.inner.data.tls_certificates.store(certificates.into()); } -} - -impl ResolvesServerCert for StaticResolver { - fn resolve(&self, _: ClientHello) -> Option<Arc<CertifiedKey>> { - self.key.clone() - } -} - -pub(crate) fn build_acme_static_resolver(key: Option<Arc<CertifiedKey>>) -> Arc<ServerConfig> { - let mut challenge = ServerConfig::builder() - .with_no_client_auth() - .with_cert_resolver(Arc::new(StaticResolver { key })); - challenge.alpn_protocols.push(ACME_TLS_ALPN_NAME.to_vec()); - Arc::new(challenge) -} -impl Core { pub(crate) async fn build_acme_certificate(&self, domain: &str) -> Option<Arc<CertifiedKey>> { match self - .storage - .lookup + .lookup_store() .key_get::<Bincode<SerializedCert>>(format!("acme:{domain}").into_bytes()) .await { @@ -100,6 +83,20 @@ impl Core { } } +impl ResolvesServerCert for StaticResolver { + fn resolve(&self, _: ClientHello) -> Option<Arc<CertifiedKey>> { + self.key.clone() + } +} + +pub(crate) fn build_acme_static_resolver(key: Option<Arc<CertifiedKey>>) -> Arc<ServerConfig> { + let mut challenge = ServerConfig::builder() + .with_no_client_auth() + .with_cert_resolver(Arc::new(StaticResolver { key })); + challenge.alpn_protocols.push(ACME_TLS_ALPN_NAME.to_vec()); + Arc::new(challenge) +} + pub trait IsTlsAlpnChallenge { fn is_tls_alpn_challenge(&self) -> bool; } diff --git a/crates/common/src/listener/blocked.rs b/crates/common/src/listener/blocked.rs index a46a3ce1..20de1951 100644 --- a/crates/common/src/listener/blocked.rs +++ b/crates/common/src/listener/blocked.rs @@ -4,85 +4,45 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use std::{fmt::Debug, net::IpAddr, sync::atomic::AtomicU8}; +use std::{fmt::Debug, net::IpAddr}; use ahash::AHashSet; -use parking_lot::RwLock; use utils::config::{ ipmask::{IpAddrMask, IpAddrOrMask}, utils::ParseValue, Config, ConfigKey, Rate, }; -use crate::Core; +use crate::Server; + +#[derive(Debug, Clone)] +pub struct Security { + blocked_ip_networks: Vec<IpAddrMask>, + has_blocked_networks: bool, + + allowed_ip_addresses: AHashSet<IpAddr>, + allowed_ip_networks: Vec<IpAddrMask>, + has_allowed_networks: bool, -pub struct BlockedIps { - pub ip_addresses: RwLock<AHashSet<IpAddr>>, - pub version: AtomicU8, - ip_networks: Vec<IpAddrMask>, - has_networks: bool, auth_fail_rate: Option<Rate>, rcpt_fail_rate: Option<Rate>, loiter_fail_rate: Option<Rate>, } -#[derive(Clone)] -pub struct AllowedIps { - ip_addresses: AHashSet<IpAddr>, - ip_networks: Vec<IpAddrMask>, - has_networks: bool, -} - pub const BLOCKED_IP_KEY: &str = "server.blocked-ip"; pub const BLOCKED_IP_PREFIX: &str = "server.blocked-ip."; pub const ALLOWED_IP_KEY: &str = "server.allowed-ip"; pub const ALLOWED_IP_PREFIX: &str = "server.allowed-ip."; -impl BlockedIps { - pub fn parse(config: &mut Config) -> Self { - let mut ip_addresses = AHashSet::new(); - let mut ip_networks = Vec::new(); - - for ip in config - .set_values(BLOCKED_IP_KEY) - .map(IpAddrOrMask::parse_value) - .collect::<Vec<_>>() - { - match ip { - Ok(IpAddrOrMask::Ip(ip)) => { - ip_addresses.insert(ip); - } - Ok(IpAddrOrMask::Mask(ip)) => { - ip_networks.push(ip); - } - Err(err) => { - config.new_parse_error(BLOCKED_IP_KEY, err); - } - } - } - - BlockedIps { - ip_addresses: RwLock::new(ip_addresses), - has_networks: !ip_networks.is_empty(), - ip_networks, - auth_fail_rate: config - .property_or_default::<Option<Rate>>("server.fail2ban.authentication", "100/1d") - .unwrap_or_default(), - rcpt_fail_rate: config - .property_or_default::<Option<Rate>>("server.fail2ban.invalid-rcpt", "35/1d") - .unwrap_or_default(), - loiter_fail_rate: config - .property_or_default::<Option<Rate>>("server.fail2ban.loitering", "150/1d") - .unwrap_or_default(), - version: 0.into(), - } - } +pub struct BlockedIps { + pub blocked_ip_addresses: AHashSet<IpAddr>, + pub blocked_ip_networks: Vec<IpAddrMask>, } -impl AllowedIps { +impl Security { pub fn parse(config: &mut Config) -> Self { - let mut ip_addresses = AHashSet::new(); - let mut ip_networks = Vec::new(); + let mut allowed_ip_addresses = AHashSet::new(); + let mut allowed_ip_networks = Vec::new(); for ip in config .set_values(ALLOWED_IP_KEY) @@ -91,10 +51,10 @@ impl AllowedIps { { match ip { Ok(IpAddrOrMask::Ip(ip)) => { - ip_addresses.insert(ip); + allowed_ip_addresses.insert(ip); } Ok(IpAddrOrMask::Mask(ip)) => { - ip_networks.push(ip); + allowed_ip_networks.push(ip); } Err(err) => { config.new_parse_error(ALLOWED_IP_KEY, err); @@ -105,25 +65,37 @@ impl AllowedIps { #[cfg(not(feature = "test_mode"))] { // Add loopback addresses - ip_addresses.insert(IpAddr::V4(std::net::Ipv4Addr::LOCALHOST)); - ip_addresses.insert(IpAddr::V6(std::net::Ipv6Addr::LOCALHOST)); + allowed_ip_addresses.insert(IpAddr::V4(std::net::Ipv4Addr::LOCALHOST)); + allowed_ip_addresses.insert(IpAddr::V6(std::net::Ipv6Addr::LOCALHOST)); } - AllowedIps { - ip_addresses, - has_networks: !ip_networks.is_empty(), - ip_networks, + let blocked = BlockedIps::parse(config); + + Security { + has_blocked_networks: !blocked.blocked_ip_networks.is_empty(), + blocked_ip_networks: blocked.blocked_ip_networks, + has_allowed_networks: !allowed_ip_networks.is_empty(), + allowed_ip_addresses, + allowed_ip_networks, + auth_fail_rate: config + .property_or_default::<Option<Rate>>("server.fail2ban.authentication", "100/1d") + .unwrap_or_default(), + rcpt_fail_rate: config + .property_or_default::<Option<Rate>>("server.fail2ban.invalid-rcpt", "35/1d") + .unwrap_or_default(), + loiter_fail_rate: config + .property_or_default::<Option<Rate>>("server.fail2ban.loitering", "150/1d") + .unwrap_or_default(), } } } -impl Core { +impl Server { pub async fn is_rcpt_fail2banned(&self, ip: IpAddr) -> trc::Result<bool> { - if let Some(rate) = &self.network.blocked_ips.rcpt_fail_rate { + if let Some(rate) = &self.core.network.security.rcpt_fail_rate { let is_allowed = self.is_ip_allowed(&ip) || self - .storage - .lookup + .lookup_store() .is_rate_allowed(format!("r:{ip}").as_bytes(), rate, false) .await? .is_none(); @@ -137,11 +109,10 @@ impl Core { } pub async fn is_loiter_fail2banned(&self, ip: IpAddr) -> trc::Result<bool> { - if let Some(rate) = &self.network.blocked_ips.loiter_fail_rate { + if let Some(rate) = &self.core.network.security.loiter_fail_rate { let is_allowed = self.is_ip_allowed(&ip) || self - .storage - .lookup + .lookup_store() .is_rate_allowed(format!("l:{ip}").as_bytes(), rate, false) .await? .is_none(); @@ -155,17 +126,15 @@ impl Core { } pub async fn is_auth_fail2banned(&self, ip: IpAddr, login: &str) -> trc::Result<bool> { - if let Some(rate) = &self.network.blocked_ips.auth_fail_rate { + if let Some(rate) = &self.core.network.security.auth_fail_rate { let is_allowed = self.is_ip_allowed(&ip) || (self - .storage - .lookup + .lookup_store() .is_rate_allowed(format!("b:{ip}").as_bytes(), rate, false) .await? .is_none() && self - .storage - .lookup + .lookup_store() .is_rate_allowed(format!("b:{login}").as_bytes(), rate, false) .await? .is_none()); @@ -179,10 +148,11 @@ impl Core { async fn block_ip(&self, ip: IpAddr) -> trc::Result<()> { // Add IP to blocked list - self.network.blocked_ips.ip_addresses.write().insert(ip); + self.inner.data.blocked_ips.write().insert(ip); // Write blocked IP to config - self.storage + self.core + .storage .config .set([ConfigKey { key: format!("{}.{}", BLOCKED_IP_KEY, ip), @@ -191,104 +161,96 @@ impl Core { .await?; // Increment version - self.network.blocked_ips.increment_version(); + self.increment_blocked_version(); Ok(()) } pub fn has_auth_fail2ban(&self) -> bool { - self.network.blocked_ips.auth_fail_rate.is_some() + self.core.network.security.auth_fail_rate.is_some() } pub fn is_ip_blocked(&self, ip: &IpAddr) -> bool { - self.network.blocked_ips.ip_addresses.read().contains(ip) - || (self.network.blocked_ips.has_networks + self.inner.data.blocked_ips.read().contains(ip) + || (self.core.network.security.has_blocked_networks && self + .core .network - .blocked_ips - .ip_networks + .security + .blocked_ip_networks .iter() .any(|network| network.matches(ip))) } pub fn is_ip_allowed(&self, ip: &IpAddr) -> bool { - self.network.allowed_ips.ip_addresses.contains(ip) - || (self.network.allowed_ips.has_networks + self.core.network.security.allowed_ip_addresses.contains(ip) + || (self.core.network.security.has_allowed_networks && self + .core .network - .allowed_ips - .ip_networks + .security + .allowed_ip_networks .iter() .any(|network| network.matches(ip))) } -} -impl BlockedIps { - pub fn increment_version(&self) { - self.version + pub fn increment_blocked_version(&self) { + self.inner + .data + .blocked_ips_version .fetch_add(1, std::sync::atomic::Ordering::Relaxed); } } -impl Default for BlockedIps { - fn default() -> Self { +impl BlockedIps { + pub fn parse(config: &mut Config) -> Self { + let mut blocked_ip_addresses = AHashSet::new(); + let mut blocked_ip_networks = Vec::new(); + + for ip in config + .set_values(BLOCKED_IP_KEY) + .map(IpAddrOrMask::parse_value) + .collect::<Vec<_>>() + { + match ip { + Ok(IpAddrOrMask::Ip(ip)) => { + blocked_ip_addresses.insert(ip); + } + Ok(IpAddrOrMask::Mask(ip)) => { + blocked_ip_networks.push(ip); + } + Err(err) => { + config.new_parse_error(BLOCKED_IP_KEY, err); + } + } + } + Self { - ip_addresses: RwLock::new(AHashSet::new()), - ip_networks: Default::default(), - has_networks: Default::default(), - version: Default::default(), - auth_fail_rate: Default::default(), - rcpt_fail_rate: Default::default(), - loiter_fail_rate: Default::default(), + blocked_ip_addresses, + blocked_ip_networks, } } } #[allow(clippy::derivable_impls)] -impl Default for AllowedIps { +impl Default for Security { fn default() -> Self { // Add IPv4 and IPv6 loopback addresses Self { #[cfg(not(feature = "test_mode"))] - ip_addresses: AHashSet::from_iter([ + allowed_ip_addresses: AHashSet::from_iter([ IpAddr::V4(std::net::Ipv4Addr::LOCALHOST), IpAddr::V6(std::net::Ipv6Addr::LOCALHOST), ]), #[cfg(feature = "test_mode")] - ip_addresses: Default::default(), - ip_networks: Default::default(), - has_networks: Default::default(), - } - } -} - -impl Clone for BlockedIps { - fn clone(&self) -> Self { - Self { - ip_addresses: RwLock::new(self.ip_addresses.read().clone()), - ip_networks: self.ip_networks.clone(), - has_networks: self.has_networks, - version: self - .version - .load(std::sync::atomic::Ordering::Relaxed) - .into(), - auth_fail_rate: self.auth_fail_rate.clone(), - rcpt_fail_rate: self.rcpt_fail_rate.clone(), - loiter_fail_rate: self.loiter_fail_rate.clone(), + allowed_ip_addresses: Default::default(), + allowed_ip_networks: Default::default(), + has_allowed_networks: Default::default(), + blocked_ip_networks: Default::default(), + has_blocked_networks: Default::default(), + auth_fail_rate: Default::default(), + rcpt_fail_rate: Default::default(), + loiter_fail_rate: Default::default(), } } } - -impl Debug for BlockedIps { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("BlockedIps") - .field("ip_addresses", &self.ip_addresses) - .field("ip_networks", &self.ip_networks) - .field("has_networks", &self.has_networks) - .field("version", &self.version) - .field("auth_fail_rate", &self.auth_fail_rate) - .field("rcpt_fail_rate", &self.rcpt_fail_rate) - .field("loiter_fail_rate", &self.loiter_fail_rate) - .finish() - } -} diff --git a/crates/common/src/listener/listen.rs b/crates/common/src/listener/listen.rs index a2877ce9..fc1cd38f 100644 --- a/crates/common/src/listener/listen.rs +++ b/crates/common/src/listener/listen.rs @@ -10,20 +10,17 @@ use std::{ time::Duration, }; -use arc_swap::ArcSwap; use proxy_header::io::ProxiedStream; use rustls::crypto::ring::cipher_suite::TLS13_AES_128_GCM_SHA256; -use tokio::{ - net::{TcpListener, TcpStream}, - sync::watch, -}; +use tokio::{net::TcpStream, sync::watch}; use tokio_rustls::server::TlsStream; use trc::{EventType, HttpEvent, ImapEvent, ManageSieveEvent, Pop3Event, SmtpEvent}; use utils::{config::Config, UnwrapFailure}; use crate::{ - config::server::{Listener, Server, ServerProtocol, Servers}, - Core, + config::server::{Listener, Listeners, ServerProtocol, TcpListener}, + core::BuildServer, + Inner, Server, }; use super::{ @@ -31,11 +28,11 @@ use super::{ TcpAcceptor, }; -impl Server { +impl Listener { pub fn spawn( self, manager: impl SessionManager, - core: Arc<ArcSwap<Core>>, + inner: Arc<Inner>, acceptor: TcpAcceptor, shutdown_rx: watch::Receiver<bool>, ) { @@ -95,7 +92,7 @@ impl Server { let mut shutdown_rx = instance.shutdown_rx.clone(); let manager = manager.clone(); let instance = instance.clone(); - let core = core.clone(); + let inner = inner.clone(); tokio::spawn(async move { let (span_start, span_end) = match self.protocol { ServerProtocol::Smtp | ServerProtocol::Lmtp => ( @@ -125,8 +122,8 @@ impl Server { stream = listener.accept() => { match stream { Ok((stream, remote_addr)) => { - let core = core.as_ref().load_full(); - let enable_acme = (is_https && core.has_acme_tls_providers()).then_some(core.clone()); + let server = inner.build_server(); + let enable_acme = (is_https && server.has_acme_tls_providers()).then(|| server.clone()); if has_proxies && instance.proxy_networks.iter().any(|network| network.matches(&remote_addr.ip())) { let instance = instance.clone(); @@ -142,7 +139,7 @@ impl Server { .proxied_address() .map(|addr| addr.source) .unwrap_or(remote_addr); - if let Some(session) = instance.build_session(stream, local_addr, remote_addr, &core) { + if let Some(session) = instance.build_session(stream, local_addr, remote_addr, &server) { // Spawn session manager.spawn(session, is_tls, enable_acme, span_start, span_end); } @@ -159,7 +156,7 @@ impl Server { } } }); - } else if let Some(session) = instance.build_session(stream, local_addr, remote_addr, &core) { + } else if let Some(session) = instance.build_session(stream, local_addr, remote_addr, &server) { // Set socket options opts.apply(&session.stream); @@ -205,7 +202,7 @@ trait BuildSession { stream: T, local_addr: SocketAddr, remote_addr: SocketAddr, - core: &Core, + server: &Server, ) -> Option<SessionData<T>>; } @@ -215,7 +212,7 @@ impl BuildSession for Arc<ServerInstance> { stream: T, local_addr: SocketAddr, remote_addr: SocketAddr, - core: &Core, + server: &Server, ) -> Option<SessionData<T>> { // Convert mapped IPv6 addresses to IPv4 let remote_ip = match remote_addr.ip() { @@ -228,7 +225,7 @@ impl BuildSession for Arc<ServerInstance> { let remote_port = remote_addr.port(); // Check if blocked - if core.is_ip_blocked(&remote_ip) { + if server.is_ip_blocked(&remote_ip) { trc::event!( Security(trc::SecurityEvent::IpBlocked), ListenerId = self.id.clone(), @@ -303,7 +300,7 @@ impl SocketOpts { } } -impl Servers { +impl Listeners { pub fn bind_and_drop_priv(&self, config: &mut Config) { // Bind as root for server in &self.servers { @@ -332,7 +329,7 @@ impl Servers { pub fn spawn( mut self, - spawn: impl Fn(Server, TcpAcceptor, watch::Receiver<bool>), + spawn: impl Fn(Listener, TcpAcceptor, watch::Receiver<bool>), ) -> (watch::Sender<bool>, watch::Receiver<bool>) { // Spawn listeners let (shutdown_tx, shutdown_rx) = watch::channel(false); @@ -348,8 +345,8 @@ impl Servers { } } -impl Listener { - pub fn listen(self) -> Result<TcpListener, String> { +impl TcpListener { + pub fn listen(self) -> Result<tokio::net::TcpListener, String> { self.socket .listen(self.backlog.unwrap_or(1024)) .map_err(|err| format!("Failed to listen on {}: {}", self.addr, err)) diff --git a/crates/common/src/listener/mod.rs b/crates/common/src/listener/mod.rs index 790adaf6..6704a228 100644 --- a/crates/common/src/listener/mod.rs +++ b/crates/common/src/listener/mod.rs @@ -19,7 +19,7 @@ use utils::{config::ipmask::IpAddrMask, snowflake::SnowflakeIdGenerator}; use crate::{ config::server::ServerProtocol, expr::{functions::ResolveVariable, *}, - Core, + Server, }; use self::limiter::{ConcurrencyLimiter, InFlight}; @@ -91,7 +91,7 @@ pub trait SessionManager: Sync + Send + 'static + Clone { &self, mut session: SessionData<T>, is_tls: bool, - acme_core: Option<Arc<Core>>, + acme_core: Option<Server>, span_start: EventType, span_end: EventType, ) { diff --git a/crates/common/src/listener/tls.rs b/crates/common/src/listener/tls.rs index fc67521c..15ec20f1 100644 --- a/crates/common/src/listener/tls.rs +++ b/crates/common/src/listener/tls.rs @@ -11,7 +11,6 @@ use std::{ }; use ahash::AHashMap; -use arc_swap::ArcSwap; use rustls::{ server::{ClientHello, ResolvesServerCert}, sign::CertifiedKey, @@ -21,7 +20,7 @@ use rustls::{ use tokio::io::{AsyncRead, AsyncWrite, AsyncWriteExt}; use tokio_rustls::{Accept, LazyConfigAcceptor}; -use crate::{Core, SharedCore}; +use crate::{Inner, Server}; use super::{ acme::{ @@ -34,36 +33,31 @@ use super::{ pub static TLS13_VERSION: &[&SupportedProtocolVersion] = &[&TLS13]; pub static TLS12_VERSION: &[&SupportedProtocolVersion] = &[&TLS12]; -#[derive(Default)] -pub struct TlsManager { - pub certificates: ArcSwap<AHashMap<String, Arc<CertifiedKey>>>, - pub acme_providers: AHashMap<String, AcmeProvider>, - pub self_signed_cert: Option<Arc<CertifiedKey>>, +#[derive(Default, Clone)] +pub struct AcmeProviders { + pub providers: AHashMap<String, AcmeProvider>, } #[derive(Clone)] pub struct CertificateResolver { - pub core: SharedCore, + pub inner: Arc<Inner>, } impl CertificateResolver { - pub fn new(core: SharedCore) -> Self { - Self { core } + pub fn new(inner: Arc<Inner>) -> Self { + Self { inner } } } impl ResolvesServerCert for CertificateResolver { fn resolve(&self, hello: ClientHello<'_>) -> Option<Arc<CertifiedKey>> { - self.core - .as_ref() - .load() - .resolve_certificate(hello.server_name()) + self.resolve_certificate(hello.server_name()) } } -impl Core { +impl CertificateResolver { pub(crate) fn resolve_certificate(&self, name: Option<&str>) -> Option<Arc<CertifiedKey>> { - let certs = self.tls.certificates.load(); + let certs = self.inner.data.tls_certificates.load(); name.map_or_else( || certs.get("*"), @@ -98,7 +92,7 @@ impl Core { Tls(trc::TlsEvent::NoCertificatesAvailable), Total = certs.len(), ); - self.tls.self_signed_cert.as_ref() + self.inner.data.tls_self_signed_cert.as_ref() } }) .cloned() @@ -109,7 +103,7 @@ impl TcpAcceptor { pub async fn accept<IO>( &self, stream: IO, - enable_acme: Option<Arc<Core>>, + enable_acme: Option<Server>, instance: &ServerInstance, ) -> TcpAcceptorResult<IO> where @@ -215,13 +209,3 @@ impl std::fmt::Debug for CertificateResolver { f.debug_struct("CertificateResolver").finish() } } - -impl Clone for TlsManager { - fn clone(&self) -> Self { - Self { - certificates: ArcSwap::from_pointee(self.certificates.load().as_ref().clone()), - acme_providers: self.acme_providers.clone(), - self_signed_cert: self.self_signed_cert.clone(), - } - } -} diff --git a/crates/common/src/manager/boot.rs b/crates/common/src/manager/boot.rs index b11b5988..bf3dd8b5 100644 --- a/crates/common/src/manager/boot.rs +++ b/crates/common/src/manager/boot.rs @@ -4,7 +4,7 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use std::path::PathBuf; +use std::{path::PathBuf, sync::Arc}; use arc_swap::ArcSwap; use pwhash::sha512_crypt; @@ -12,14 +12,16 @@ use store::{ rand::{distributions::Alphanumeric, thread_rng, Rng}, Stores, }; +use tokio::sync::{mpsc, Notify}; use utils::{ config::{Config, ConfigKey}, failed, UnwrapFailure, }; use crate::{ - config::{server::Servers, telemetry::Telemetry}, - Core, SharedCore, + config::{server::Listeners, telemetry::Telemetry}, + ipc::{DeliveryEvent, HousekeeperEvent, QueueEvent, ReportingEvent, StateEvent}, + Core, Data, Inner, Ipc, IPC_CHANNEL_BUFFER, }; use super::{ @@ -30,8 +32,17 @@ use super::{ pub struct BootManager { pub config: Config, - pub core: SharedCore, - pub servers: Servers, + pub inner: Arc<Inner>, + pub servers: Listeners, + pub ipc_rxs: IpcReceivers, +} + +pub struct IpcReceivers { + pub state_rx: Option<mpsc::Receiver<StateEvent>>, + pub housekeeper_rx: Option<mpsc::Receiver<HousekeeperEvent>>, + pub delivery_rx: Option<mpsc::Receiver<DeliveryEvent>>, + pub queue_rx: Option<mpsc::Receiver<QueueEvent>>, + pub report_rx: Option<mpsc::Receiver<ReportingEvent>>, } const HELP: &str = concat!( @@ -135,7 +146,7 @@ impl BootManager { config.resolve_macros(&["env"]).await; // Parser servers - let mut servers = Servers::parse(&mut config); + let mut servers = Listeners::parse(&mut config); // Bind ports and drop privileges servers.bind_and_drop_priv(&mut config); @@ -314,6 +325,9 @@ impl BootManager { // Parse settings let core = Core::parse(&mut config, stores, manager).await; + // Parse data + let data = Data::parse(&mut config); + // Enable telemetry #[cfg(feature = "enterprise")] telemetry.enable(core.is_enterprise_edition()); @@ -325,16 +339,22 @@ impl BootManager { Version = env!("CARGO_PKG_VERSION"), ); - // Build shared core - let core = core.into_shared(); + // Build shared inner + let (ipc, ipc_rxs) = build_ipc(); + let inner = Arc::new(Inner { + shared_core: ArcSwap::from_pointee(core), + data, + ipc, + }); // Parse TCP acceptors - servers.parse_tcp_acceptors(&mut config, core.clone()); + servers.parse_tcp_acceptors(&mut config, inner.clone()); BootManager { - core, + inner, config, servers, + ipc_rxs, } } ImportExport::Export(path) => { @@ -363,6 +383,32 @@ impl BootManager { } } +pub fn build_ipc() -> (Ipc, IpcReceivers) { + // Build ipc receivers + let (delivery_tx, delivery_rx) = mpsc::channel(IPC_CHANNEL_BUFFER); + let (state_tx, state_rx) = mpsc::channel(IPC_CHANNEL_BUFFER); + let (housekeeper_tx, housekeeper_rx) = mpsc::channel(IPC_CHANNEL_BUFFER); + let (queue_tx, queue_rx) = mpsc::channel(IPC_CHANNEL_BUFFER); + let (report_tx, report_rx) = mpsc::channel(IPC_CHANNEL_BUFFER); + ( + Ipc { + state_tx, + housekeeper_tx, + delivery_tx, + queue_tx, + report_tx, + index_tx: Arc::new(Notify::new()), + }, + IpcReceivers { + state_rx: Some(state_rx), + housekeeper_rx: Some(housekeeper_rx), + delivery_rx: Some(delivery_rx), + queue_rx: Some(queue_rx), + report_rx: Some(report_rx), + }, + ) +} + fn quickstart(path: impl Into<PathBuf>) { let path = path.into(); diff --git a/crates/common/src/manager/reload.rs b/crates/common/src/manager/reload.rs index f7532a36..143d4008 100644 --- a/crates/common/src/manager/reload.rs +++ b/crates/common/src/manager/reload.rs @@ -4,18 +4,18 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use ahash::AHashSet; +use ahash::AHashMap; use arc_swap::ArcSwap; use store::Stores; -use utils::config::{ipmask::IpAddrOrMask, utils::ParseValue, Config}; +use utils::config::Config; use crate::{ config::{ - server::{tls::parse_certificates, Servers}, + server::{tls::parse_certificates, Listeners}, telemetry::Telemetry, }, - listener::blocked::BLOCKED_IP_KEY, - Core, + listener::blocked::{BlockedIps, BLOCKED_IP_KEY}, + Core, Server, }; use super::config::{ConfigManager, Patterns}; @@ -26,49 +26,36 @@ pub struct ReloadResult { pub tracers: Option<Telemetry>, } -impl Core { +impl Server { pub async fn reload_blocked_ips(&self) -> trc::Result<ReloadResult> { - let mut ip_addresses = AHashSet::new(); - let mut config = self.storage.config.build_config(BLOCKED_IP_KEY).await?; - - for ip in config - .set_values(BLOCKED_IP_KEY) - .map(IpAddrOrMask::parse_value) - .collect::<Vec<_>>() - { - match ip { - Ok(IpAddrOrMask::Ip(ip)) => { - ip_addresses.insert(ip); - } - Ok(IpAddrOrMask::Mask(_)) => {} - Err(err) => { - config.new_parse_error(BLOCKED_IP_KEY, err); - } - } - } - - *self.network.blocked_ips.ip_addresses.write() = ip_addresses; + let mut config = self + .core + .storage + .config + .build_config(BLOCKED_IP_KEY) + .await?; + *self.inner.data.blocked_ips.write() = BlockedIps::parse(&mut config).blocked_ip_addresses; Ok(config.into()) } pub async fn reload_certificates(&self) -> trc::Result<ReloadResult> { - let mut config = self.storage.config.build_config("certificate").await?; - let mut certificates = self.tls.certificates.load().as_ref().clone(); + let mut config = self.core.storage.config.build_config("certificate").await?; + let mut certificates = self.inner.data.tls_certificates.load().as_ref().clone(); parse_certificates(&mut config, &mut certificates, &mut Default::default()); - self.tls.certificates.store(certificates.into()); + self.inner.data.tls_certificates.store(certificates.into()); Ok(config.into()) } pub async fn reload_lookups(&self) -> trc::Result<ReloadResult> { - let mut config = self.storage.config.build_config("lookup").await?; + let mut config = self.core.storage.config.build_config("lookup").await?; let mut stores = Stores::default(); stores.parse_memory_stores(&mut config); - let mut core = self.clone(); + let mut core = self.core.as_ref().clone(); for (id, store) in stores.lookup_stores { core.storage.lookups.insert(id, store); } @@ -81,14 +68,14 @@ impl Core { } pub async fn reload(&self) -> trc::Result<ReloadResult> { - let mut config = self.storage.config.build_config("").await?; + let mut config = self.core.storage.config.build_config("").await?; // Load stores let mut stores = Stores { - stores: self.storage.stores.clone(), - blob_stores: self.storage.blobs.clone(), - fts_stores: self.storage.ftss.clone(), - lookup_stores: self.storage.lookups.clone(), + stores: self.core.storage.stores.clone(), + blob_stores: self.core.storage.blobs.clone(), + fts_stores: self.core.storage.ftss.clone(), + lookup_stores: self.core.storage.lookups.clone(), purge_schedules: Default::default(), }; stores.parse_stores(&mut config).await; @@ -103,8 +90,10 @@ impl Core { // Build manager let manager = ConfigManager { - cfg_local: ArcSwap::from_pointee(self.storage.config.cfg_local.load().as_ref().clone()), - cfg_local_path: self.storage.config.cfg_local_path.clone(), + cfg_local: ArcSwap::from_pointee( + self.core.storage.config.cfg_local.load().as_ref().clone(), + ), + cfg_local_path: self.core.storage.config.cfg_local_path.clone(), cfg_local_patterns: Patterns::parse(&mut config).into(), cfg_store: config .value("storage.data") @@ -114,26 +103,29 @@ impl Core { }; // Parse settings and build shared core - let mut core = Core::parse(&mut config, stores, manager).await; + let core = Core::parse(&mut config, stores, manager).await; if !config.errors.is_empty() { return Ok(config.into()); } - // Copy ACME certificates - let mut certificates = core.tls.certificates.load().as_ref().clone(); - for (cert_id, cert) in self.tls.certificates.load().iter() { - certificates - .entry(cert_id.to_string()) - .or_insert(cert.clone()); + // Update TLS certificates + let mut new_certificates = AHashMap::new(); + parse_certificates(&mut config, &mut new_certificates, &mut Default::default()); + let mut current_certificates = self.inner.data.tls_certificates.load().as_ref().clone(); + for (cert_id, cert) in new_certificates { + current_certificates.insert(cert_id, cert); } - core.tls.certificates.store(certificates.into()); - core.tls - .self_signed_cert - .clone_from(&self.tls.self_signed_cert); + self.inner + .data + .tls_certificates + .store(current_certificates.into()); + + // Update blocked IPs + *self.inner.data.blocked_ips.write() = BlockedIps::parse(&mut config).blocked_ip_addresses; // Parser servers - let mut servers = Servers::parse(&mut config); - servers.parse_tcp_acceptors(&mut config, core.clone().into_shared()); + let mut servers = Listeners::parse(&mut config); + servers.parse_tcp_acceptors(&mut config, self.inner.clone()); Ok(if config.errors.is_empty() { ReloadResult { diff --git a/crates/common/src/scripts/plugins/bayes.rs b/crates/common/src/scripts/plugins/bayes.rs index 5250cac9..814f5a97 100644 --- a/crates/common/src/scripts/plugins/bayes.rs +++ b/crates/common/src/scripts/plugins/bayes.rs @@ -43,8 +43,8 @@ pub async fn exec_untrain(ctx: PluginContext<'_>) -> trc::Result<Variable> { async fn train(ctx: PluginContext<'_>, is_train: bool) -> trc::Result<Variable> { let store = match &ctx.arguments[0] { - Variable::String(v) if !v.is_empty() => ctx.core.storage.lookups.get(v.as_ref()), - _ => Some(&ctx.core.storage.lookup), + Variable::String(v) if !v.is_empty() => ctx.server.core.storage.lookups.get(v.as_ref()), + _ => Some(&ctx.server.core.storage.lookup), } .ok_or_else(|| { trc::SieveEvent::RuntimeError @@ -80,7 +80,7 @@ async fn train(ctx: PluginContext<'_>, is_train: bool) -> trc::Result<Variable> ); // Update weight and invalidate cache - let bayes_cache = &ctx.cache.bayes_cache; + let bayes_cache = &ctx.server.inner.data.bayes_cache; if is_train { for (hash, weights) in model.weights { store @@ -129,8 +129,8 @@ async fn train(ctx: PluginContext<'_>, is_train: bool) -> trc::Result<Variable> pub async fn exec_classify(ctx: PluginContext<'_>) -> trc::Result<Variable> { let store = match &ctx.arguments[0] { - Variable::String(v) if !v.is_empty() => ctx.core.storage.lookups.get(v.as_ref()), - _ => Some(&ctx.core.storage.lookup), + Variable::String(v) if !v.is_empty() => ctx.server.core.storage.lookups.get(v.as_ref()), + _ => Some(&ctx.server.core.storage.lookup), } .ok_or_else(|| { trc::SieveEvent::RuntimeError @@ -162,7 +162,7 @@ pub async fn exec_classify(ctx: PluginContext<'_>) -> trc::Result<Variable> { } // Obtain training counts - let bayes_cache = &ctx.cache.bayes_cache; + let bayes_cache = &ctx.server.inner.data.bayes_cache; let (spam_learns, ham_learns) = bayes_cache .get_or_update(TokenHash::default(), store) .await @@ -219,8 +219,8 @@ pub async fn exec_is_balanced(ctx: PluginContext<'_>) -> trc::Result<Variable> { } let store = match &ctx.arguments[0] { - Variable::String(v) if !v.is_empty() => ctx.core.storage.lookups.get(v.as_ref()), - _ => Some(&ctx.core.storage.lookup), + Variable::String(v) if !v.is_empty() => ctx.server.core.storage.lookups.get(v.as_ref()), + _ => Some(&ctx.server.core.storage.lookup), } .ok_or_else(|| { trc::SieveEvent::RuntimeError @@ -231,7 +231,7 @@ pub async fn exec_is_balanced(ctx: PluginContext<'_>) -> trc::Result<Variable> { let learn_spam = ctx.arguments[1].to_bool(); // Obtain training counts - let bayes_cache = &ctx.cache.bayes_cache; + let bayes_cache = &ctx.server.inner.data.bayes_cache; let (spam_learns, ham_learns) = bayes_cache .get_or_update(TokenHash::default(), store) .await diff --git a/crates/common/src/scripts/plugins/dns.rs b/crates/common/src/scripts/plugins/dns.rs index cdbacb17..6e145ddd 100644 --- a/crates/common/src/scripts/plugins/dns.rs +++ b/crates/common/src/scripts/plugins/dns.rs @@ -25,6 +25,7 @@ pub async fn exec(ctx: PluginContext<'_>) -> trc::Result<Variable> { Ok(if record_type.eq_ignore_ascii_case("ip") { match ctx + .server .core .smtp .resolvers @@ -40,7 +41,15 @@ pub async fn exec(ctx: PluginContext<'_>) -> trc::Result<Variable> { Err(err) => err.short_error().into(), } } else if record_type.eq_ignore_ascii_case("mx") { - match ctx.core.smtp.resolvers.dns.mx_lookup(entry.as_ref()).await { + match ctx + .server + .core + .smtp + .resolvers + .dns + .mx_lookup(entry.as_ref()) + .await + { Ok(result) => result .iter() .flat_map(|mx| { @@ -61,6 +70,7 @@ pub async fn exec(ctx: PluginContext<'_>) -> trc::Result<Variable> { } match ctx + .server .core .smtp .resolvers @@ -73,7 +83,7 @@ pub async fn exec(ctx: PluginContext<'_>) -> trc::Result<Variable> { } } else if record_type.eq_ignore_ascii_case("ptr") { if let Ok(addr) = entry.parse::<IpAddr>() { - match ctx.core.smtp.resolvers.dns.ptr_lookup(addr).await { + match ctx.server.core.smtp.resolvers.dns.ptr_lookup(addr).await { Ok(result) => result .iter() .map(|host| Variable::from(host.to_string())) @@ -94,6 +104,7 @@ pub async fn exec(ctx: PluginContext<'_>) -> trc::Result<Variable> { } match ctx + .server .core .smtp .resolvers @@ -110,6 +121,7 @@ pub async fn exec(ctx: PluginContext<'_>) -> trc::Result<Variable> { } } else if record_type.eq_ignore_ascii_case("ipv6") { match ctx + .server .core .smtp .resolvers @@ -135,6 +147,7 @@ pub async fn exec_exists(ctx: PluginContext<'_>) -> trc::Result<Variable> { Ok(if record_type.eq_ignore_ascii_case("ip") { match ctx + .server .core .smtp .resolvers @@ -147,14 +160,22 @@ pub async fn exec_exists(ctx: PluginContext<'_>) -> trc::Result<Variable> { Err(_) => -1, } } else if record_type.eq_ignore_ascii_case("mx") { - match ctx.core.smtp.resolvers.dns.mx_lookup(entry.as_ref()).await { + match ctx + .server + .core + .smtp + .resolvers + .dns + .mx_lookup(entry.as_ref()) + .await + { Ok(result) => i64::from(result.iter().any(|mx| !mx.exchanges.is_empty())), Err(Error::DnsRecordNotFound(_)) => 0, Err(_) => -1, } } else if record_type.eq_ignore_ascii_case("ptr") { if let Ok(addr) = entry.parse::<IpAddr>() { - match ctx.core.smtp.resolvers.dns.ptr_lookup(addr).await { + match ctx.server.core.smtp.resolvers.dns.ptr_lookup(addr).await { Ok(result) => i64::from(!result.is_empty()), Err(Error::DnsRecordNotFound(_)) => 0, Err(_) => -1, @@ -171,6 +192,7 @@ pub async fn exec_exists(ctx: PluginContext<'_>) -> trc::Result<Variable> { } match ctx + .server .core .smtp .resolvers @@ -184,6 +206,7 @@ pub async fn exec_exists(ctx: PluginContext<'_>) -> trc::Result<Variable> { } } else if record_type.eq_ignore_ascii_case("ipv6") { match ctx + .server .core .smtp .resolvers diff --git a/crates/common/src/scripts/plugins/lookup.rs b/crates/common/src/scripts/plugins/lookup.rs index f19721c5..1d82ba93 100644 --- a/crates/common/src/scripts/plugins/lookup.rs +++ b/crates/common/src/scripts/plugins/lookup.rs @@ -42,8 +42,8 @@ pub fn register_local_domain(plugin_id: u32, fnc_map: &mut FunctionMap) { pub async fn exec(ctx: PluginContext<'_>) -> trc::Result<Variable> { let store = match &ctx.arguments[0] { - Variable::String(v) if !v.is_empty() => ctx.core.storage.lookups.get(v.as_ref()), - _ => Some(&ctx.core.storage.lookup), + Variable::String(v) if !v.is_empty() => ctx.server.core.storage.lookups.get(v.as_ref()), + _ => Some(&ctx.server.core.storage.lookup), } .ok_or_else(|| { trc::SieveEvent::RuntimeError @@ -76,8 +76,8 @@ pub async fn exec(ctx: PluginContext<'_>) -> trc::Result<Variable> { pub async fn exec_get(ctx: PluginContext<'_>) -> trc::Result<Variable> { match &ctx.arguments[0] { - Variable::String(v) if !v.is_empty() => ctx.core.storage.lookups.get(v.as_ref()), - _ => Some(&ctx.core.storage.lookup), + Variable::String(v) if !v.is_empty() => ctx.server.core.storage.lookups.get(v.as_ref()), + _ => Some(&ctx.server.core.storage.lookup), } .ok_or_else(|| { trc::SieveEvent::RuntimeError @@ -97,8 +97,8 @@ pub async fn exec_set(ctx: PluginContext<'_>) -> trc::Result<Variable> { }; match &ctx.arguments[0] { - Variable::String(v) if !v.is_empty() => ctx.core.storage.lookups.get(v.as_ref()), - _ => Some(&ctx.core.storage.lookup), + Variable::String(v) if !v.is_empty() => ctx.server.core.storage.lookups.get(v.as_ref()), + _ => Some(&ctx.server.core.storage.lookup), } .ok_or_else(|| { trc::SieveEvent::RuntimeError @@ -125,7 +125,7 @@ pub async fn exec_remote(ctx: PluginContext<'_>) -> trc::Result<Variable> { // Something went wrong, try again in one hour const RETRY: Duration = Duration::from_secs(3600); - let mut _lock = ctx.cache.remote_lists.write(); + let mut _lock = ctx.server.inner.data.remote_lists.write(); let list = _lock .entry(ctx.arguments[0].to_string().to_string()) .or_insert_with(|| RemoteList { @@ -169,7 +169,14 @@ async fn exec_remote_(ctx: &PluginContext<'_>) -> trc::Result<Variable> { const MAX_ENTRY_SIZE: usize = 256; const MAX_ENTRIES: usize = 100000; - match ctx.cache.remote_lists.read().get(resource.as_ref()) { + match ctx + .server + .inner + .data + .remote_lists + .read() + .get(resource.as_ref()) + { Some(remote_list) if remote_list.expires < Instant::now() => { return Ok(remote_list.entries.contains(item.as_ref()).into()) } @@ -256,7 +263,7 @@ async fn exec_remote_(ctx: &PluginContext<'_>) -> trc::Result<Variable> { }; // Lock remote list for writing - let mut _lock = ctx.cache.remote_lists.write(); + let mut _lock = ctx.server.inner.data.remote_lists.write(); let list = _lock .entry(resource.to_string()) .or_insert_with(|| RemoteList { @@ -352,8 +359,10 @@ pub async fn exec_local_domain(ctx: PluginContext<'_>) -> trc::Result<Variable> if !domain.is_empty() { return match &ctx.arguments[0] { - Variable::String(v) if !v.is_empty() => ctx.core.storage.directories.get(v.as_ref()), - _ => Some(&ctx.core.storage.directory), + Variable::String(v) if !v.is_empty() => { + ctx.server.core.storage.directories.get(v.as_ref()) + } + _ => Some(&ctx.server.core.storage.directory), } .ok_or_else(|| { trc::SieveEvent::RuntimeError diff --git a/crates/common/src/scripts/plugins/mod.rs b/crates/common/src/scripts/plugins/mod.rs index f2802138..ba681eee 100644 --- a/crates/common/src/scripts/plugins/mod.rs +++ b/crates/common/src/scripts/plugins/mod.rs @@ -17,7 +17,7 @@ pub mod text; use mail_parser::Message; use sieve::{runtime::Variable, FunctionMap, Input}; -use crate::{config::scripts::ScriptCache, Core}; +use crate::{Core, Server}; use super::ScriptModification; @@ -25,8 +25,7 @@ type RegisterPluginFnc = fn(u32, &mut FunctionMap) -> (); pub struct PluginContext<'x> { pub session_id: u64, - pub core: &'x Core, - pub cache: &'x ScriptCache, + pub server: &'x Server, pub message: &'x Message<'x>, pub modifications: &'x mut Vec<ScriptModification>, pub arguments: Vec<Variable>, diff --git a/crates/common/src/scripts/plugins/query.rs b/crates/common/src/scripts/plugins/query.rs index 356808ae..15092caf 100644 --- a/crates/common/src/scripts/plugins/query.rs +++ b/crates/common/src/scripts/plugins/query.rs @@ -19,8 +19,8 @@ pub fn register(plugin_id: u32, fnc_map: &mut FunctionMap) { pub async fn exec(ctx: PluginContext<'_>) -> trc::Result<Variable> { // Obtain store name let store = match &ctx.arguments[0] { - Variable::String(v) if !v.is_empty() => ctx.core.storage.lookups.get(v.as_ref()), - _ => Some(&ctx.core.storage.lookup), + Variable::String(v) if !v.is_empty() => ctx.server.core.storage.lookups.get(v.as_ref()), + _ => Some(&ctx.server.core.storage.lookup), } .ok_or_else(|| { trc::SieveEvent::RuntimeError diff --git a/crates/common/src/telemetry/metrics/otel.rs b/crates/common/src/telemetry/metrics/otel.rs index a4e10c9b..7ce117f8 100644 --- a/crates/common/src/telemetry/metrics/otel.rs +++ b/crates/common/src/telemetry/metrics/otel.rs @@ -4,7 +4,7 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use std::{sync::Arc, time::SystemTime}; +use std::time::SystemTime; use opentelemetry::global::set_error_handler; use opentelemetry_sdk::metrics::data::{ @@ -13,19 +13,13 @@ use opentelemetry_sdk::metrics::data::{ }; use trc::{Collector, TelemetryEvent}; -use crate::{config::telemetry::OtelMetrics, Core}; +use crate::config::telemetry::OtelMetrics; impl OtelMetrics { - pub async fn push_metrics(&self, core: Arc<Core>, start_time: SystemTime) { + pub async fn push_metrics(&self, is_enterprise: bool, start_time: SystemTime) { let mut metrics = Vec::with_capacity(256); let now = SystemTime::now(); - #[cfg(feature = "enterprise")] - let is_enterprise = core.is_enterprise_edition(); - - #[cfg(not(feature = "enterprise"))] - let is_enterprise = false; - // Add counters for counter in Collector::collect_counters(is_enterprise) { metrics.push(Metric { diff --git a/crates/common/src/telemetry/metrics/prometheus.rs b/crates/common/src/telemetry/metrics/prometheus.rs index 35ff3c01..46bcf9c7 100644 --- a/crates/common/src/telemetry/metrics/prometheus.rs +++ b/crates/common/src/telemetry/metrics/prometheus.rs @@ -10,9 +10,9 @@ use prometheus::{ }; use trc::{atomics::histogram::AtomicHistogram, Collector}; -use crate::Core; +use crate::Server; -impl Core { +impl Server { pub async fn export_prometheus_metrics(&self) -> trc::Result<String> { let mut metrics = Vec::new(); diff --git a/crates/directory/src/backend/internal/mod.rs b/crates/directory/src/backend/internal/mod.rs index ee676080..9806d184 100644 --- a/crates/directory/src/backend/internal/mod.rs +++ b/crates/directory/src/backend/internal/mod.rs @@ -274,22 +274,26 @@ impl MigrateDirectory for Store { }, ), |key, value| { - if key[0] == 2 && value[0] == 1 { - principals.push(( - key.get(1..) - .and_then(|b| b.read_leb128::<u32>().map(|(v, _)| v)) - .ok_or_else(|| { - trc::StoreEvent::DataCorruption - .caused_by(trc::location!()) - .ctx(trc::Key::Value, key) - })?, - Principal::deserialize(value)?, - )); - } else if key[0] == 3 { - let domain = std::str::from_utf8(&key[1..]).unwrap_or_default(); - if !domain.is_empty() { - domains.push(domain.to_string()); + match (key.first(), value.first()) { + (Some(2), Some(1)) => { + principals.push(( + key.get(1..) + .and_then(|b| b.read_leb128::<u32>().map(|(v, _)| v)) + .ok_or_else(|| { + trc::StoreEvent::DataCorruption + .caused_by(trc::location!()) + .ctx(trc::Key::Value, key) + })?, + Principal::deserialize(value)?, + )); } + (Some(3), _) => { + let domain = std::str::from_utf8(&key[1..]).unwrap_or_default(); + if !domain.is_empty() { + domains.push(domain.to_string()); + } + } + _ => {} } Ok(true) diff --git a/crates/imap/src/core/client.rs b/crates/imap/src/core/client.rs index 1952da3c..a6bea4e1 100644 --- a/crates/imap/src/core/client.rs +++ b/crates/imap/src/core/client.rs @@ -6,12 +6,14 @@ use std::{iter::Peekable, sync::Arc, vec::IntoIter}; -use common::listener::{limiter::ConcurrencyLimiter, SessionResult, SessionStream}; +use common::{ + listener::{limiter::ConcurrencyLimiter, SessionResult, SessionStream}, + ConcurrencyLimiters, +}; use imap_proto::{ receiver::{self, Request}, Command, ResponseType, StatusResponse, }; -use jmap::auth::rate_limit::ConcurrencyLimiters; use super::{SelectedMailbox, Session, SessionData, State}; @@ -255,9 +257,9 @@ impl<T: SessionStream> Session<T> { let state = &self.state; // Rate limit request if let State::Authenticated { data } | State::Selected { data, .. } = state { - if let Some(rate) = &self.jmap.core.imap.rate_requests { + if let Some(rate) = &self.server.core.imap.rate_requests { if data - .jmap + .server .core .storage .lookup @@ -301,7 +303,7 @@ impl<T: SessionStream> Session<T> { } Command::Login => { if let State::NotAuthenticated { .. } = state { - if self.is_tls || self.jmap.core.imap.allow_plain_auth { + if self.is_tls || self.server.core.imap.allow_plain_auth { Ok(request) } else { Err(trc::ImapEvent::Error @@ -385,9 +387,11 @@ impl<T: SessionStream> Session<T> { } pub fn get_concurrency_limiter(&self, account_id: u32) -> Option<Arc<ConcurrencyLimiters>> { - let rate = self.jmap.core.imap.rate_concurrent?; - self.imap - .rate_limiter + let rate = self.server.core.imap.rate_concurrent?; + self.server + .inner + .data + .imap_limiter .get(&account_id) .map(|limiter| limiter.clone()) .unwrap_or_else(|| { @@ -395,7 +399,11 @@ impl<T: SessionStream> Session<T> { concurrent_requests: ConcurrencyLimiter::new(rate), concurrent_uploads: ConcurrencyLimiter::new(rate), }); - self.imap.rate_limiter.insert(account_id, limiter.clone()); + self.server + .inner + .data + .imap_limiter + .insert(account_id, limiter.clone()); limiter }) .into() diff --git a/crates/imap/src/core/mailbox.rs b/crates/imap/src/core/mailbox.rs index 4a780ee6..5f872df6 100644 --- a/crates/imap/src/core/mailbox.rs +++ b/crates/imap/src/core/mailbox.rs @@ -8,10 +8,16 @@ use common::{ auth::AccessToken, config::jmap::settings::SpecialUse, listener::{limiter::InFlight, SessionStream}, + AccountId, Mailbox, }; use directory::{backend::internal::PrincipalField, QueryBy}; use imap_proto::protocol::list::Attribute; -use jmap::{auth::acl::EffectiveAcl, mailbox::INBOX_ID}; +use jmap::{ + auth::acl::{AclMethods, EffectiveAcl}, + changes::get::ChangesLookup, + mailbox::{get::MailboxGet, set::MailboxSet, INBOX_ID}, + JmapMethods, +}; use jmap_proto::{ object::Object, types::{acl::Acl, collection::Collection, id::Id, property::Property, value::Value}, @@ -21,7 +27,7 @@ use store::query::log::{Change, Query}; use trc::AddContext; use utils::lru_cache::LruCached; -use super::{Account, AccountId, Mailbox, MailboxId, MailboxSync, Session, SessionData}; +use super::{Account, MailboxId, MailboxSync, Session, SessionData}; impl<T: SessionStream> SessionData<T> { pub async fn new( @@ -31,8 +37,7 @@ impl<T: SessionStream> SessionData<T> { ) -> trc::Result<Self> { let mut session = SessionData { stream_tx: session.stream_tx.clone(), - jmap: session.jmap.clone(), - imap: session.imap.clone(), + server: session.server.clone(), account_id: access_token.primary_id(), session_id: session.session_id, mailboxes: Mutex::new(vec![]), @@ -56,9 +61,9 @@ impl<T: SessionStream> SessionData<T> { account_id, format!( "{}/{}", - session.jmap.core.jmap.shared_folder, + session.server.core.jmap.shared_folder, session - .jmap + .server .core .storage .directory @@ -88,7 +93,7 @@ impl<T: SessionStream> SessionData<T> { access_token: &AccessToken, ) -> trc::Result<Account> { let state_mailbox = self - .jmap + .server .core .storage .data @@ -96,7 +101,7 @@ impl<T: SessionStream> SessionData<T> { .await .caused_by(trc::location!())?; let state_email = self - .jmap + .server .core .storage .data @@ -107,19 +112,21 @@ impl<T: SessionStream> SessionData<T> { account_id, primary_id: access_token.primary_id(), }; - if let Some(cached_account) = - self.imap - .cache_account - .get(&cached_account_id) - .and_then(|cached_account| { - if cached_account.state_mailbox == state_mailbox - && cached_account.state_email == state_email - { - Some(cached_account) - } else { - None - } - }) + if let Some(cached_account) = self + .server + .inner + .data + .account_cache + .get(&cached_account_id) + .and_then(|cached_account| { + if cached_account.state_mailbox == state_mailbox + && cached_account.state_email == state_email + { + Some(cached_account) + } else { + None + } + }) { return Ok(cached_account.as_ref().clone()); } @@ -127,12 +134,12 @@ impl<T: SessionStream> SessionData<T> { let mailbox_ids = if access_token.is_primary_id(account_id) || access_token.member_of.contains(&account_id) { - self.jmap + self.server .mailbox_get_or_create(account_id) .await .caused_by(trc::location!())? } else { - self.jmap + self.server .shared_documents(access_token, account_id, Collection::Mailbox, Acl::Read) .await .caused_by(trc::location!())? @@ -142,7 +149,7 @@ impl<T: SessionStream> SessionData<T> { let mut mailboxes = Vec::with_capacity(10); let mut special_uses = AHashMap::new(); for (mailbox_id, values) in self - .jmap + .server .get_properties::<Object<Value>, _, _>( account_id, Collection::Mailbox, @@ -189,7 +196,7 @@ impl<T: SessionStream> SessionData<T> { let mut path = Vec::new(); let mut iter_stack = Vec::new(); let message_ids = self - .jmap + .server .get_document_ids(account_id, Collection::Email) .await .caused_by(trc::location!())?; @@ -246,7 +253,7 @@ impl<T: SessionStream> SessionData<T> { }, ), total_messages: self - .jmap + .server .get_tag( account_id, Collection::Email, @@ -259,7 +266,7 @@ impl<T: SessionStream> SessionData<T> { .unwrap_or(0) .into(), total_unseen: self - .jmap + .server .mailbox_unread_tags(account_id, *mailbox_id, &message_ids) .await .caused_by(trc::location!())? @@ -278,7 +285,7 @@ impl<T: SessionStream> SessionData<T> { // Map special use folder aliases to their internal ids let effective_mailbox_id = self - .jmap + .server .core .jmap .default_folders @@ -313,8 +320,10 @@ impl<T: SessionStream> SessionData<T> { } // Update cache - self.imap - .cache_account + self.server + .inner + .data + .account_cache .insert(cached_account_id, Arc::new(account.clone())); Ok(account) @@ -332,8 +341,7 @@ impl<T: SessionStream> SessionData<T> { // Obtain access token let access_token = self - .jmap - .core + .server .get_cached_access_token(self.account_id) .await .caused_by(trc::location!())?; @@ -383,8 +391,8 @@ impl<T: SessionStream> SessionData<T> { for account_id in added_account_ids { let prefix = format!( "{}/{}", - self.jmap.core.jmap.shared_folder, - self.jmap + self.server.core.jmap.shared_folder, + self.server .core .storage .directory @@ -414,7 +422,7 @@ impl<T: SessionStream> SessionData<T> { .collect::<Vec<_>>(); for (account_id, last_state) in account_states { let changelog = self - .jmap + .server .changes_( account_id, Collection::Mailbox, @@ -437,7 +445,7 @@ impl<T: SessionStream> SessionData<T> { if has_child_changes && !has_changes && changes.is_none() { // Only child changes, no need to re-fetch mailboxes let state_email = self - .jmap + .server .core .storage .data @@ -461,8 +469,13 @@ impl<T: SessionStream> SessionData<T> { } // Update cache - if let Some(cached_account_) = - self.imap.cache_account.lock().get_mut(&AccountId { + if let Some(cached_account_) = self + .server + .inner + .data + .account_cache + .lock() + .get_mut(&AccountId { account_id, primary_id: access_token.primary_id(), }) @@ -488,8 +501,8 @@ impl<T: SessionStream> SessionData<T> { let mailbox_prefix = if !access_token.is_primary_id(account_id) { format!( "{}/{}", - self.jmap.core.jmap.shared_folder, - self.jmap + self.server.core.jmap.shared_folder, + self.server .core .storage .directory @@ -613,7 +626,7 @@ impl<T: SessionStream> SessionData<T> { let access_token = self.get_access_token().await?; Ok(access_token.is_member(account_id) || self - .jmap + .server .get_property::<Object<Value>>( account_id, Collection::Mailbox, diff --git a/crates/imap/src/core/message.rs b/crates/imap/src/core/message.rs index 2a3df644..0706eb89 100644 --- a/crates/imap/src/core/message.rs +++ b/crates/imap/src/core/message.rs @@ -7,9 +7,9 @@ use std::{collections::BTreeMap, sync::Arc}; use ahash::AHashMap; -use common::listener::SessionStream; +use common::{listener::SessionStream, NextMailboxState}; use imap_proto::protocol::{expunge, select::Exists, Sequence}; -use jmap::mailbox::UidMailbox; +use jmap::{mailbox::UidMailbox, JmapMethods}; use jmap_proto::{ object::Object, types::{collection::Collection, property::Property, value::Value}, @@ -20,7 +20,7 @@ use utils::lru_cache::LruCached; use crate::core::ImapId; -use super::{ImapUidToId, MailboxId, MailboxState, NextMailboxState, SelectedMailbox, SessionData}; +use super::{ImapUidToId, MailboxId, MailboxState, SelectedMailbox, SessionData}; pub(crate) const MAX_RETRIES: usize = 10; @@ -28,7 +28,7 @@ impl<T: SessionStream> SessionData<T> { pub async fn fetch_messages(&self, mailbox: &MailboxId) -> trc::Result<MailboxState> { // Obtain message ids let message_ids = self - .jmap + .server .get_tag( mailbox.account_id, Collection::Email, @@ -43,7 +43,7 @@ impl<T: SessionStream> SessionData<T> { // Obtain current state let modseq = self - .jmap + .server .core .storage .data @@ -54,7 +54,7 @@ impl<T: SessionStream> SessionData<T> { // Obtain all message ids let mut uid_map = BTreeMap::new(); for (message_id, uid_mailbox) in self - .jmap + .server .get_properties::<HashedValue<Vec<UidMailbox>>, _, _>( mailbox.account_id, Collection::Email, @@ -148,8 +148,10 @@ impl<T: SessionStream> SessionData<T> { current_state.id_to_imap = id_to_imap; // Update cache - self.imap - .cache_mailbox + self.server + .inner + .data + .mailbox_cache .insert(mailbox.id, Arc::new(new_state.clone())); // Update state @@ -207,7 +209,7 @@ impl<T: SessionStream> SessionData<T> { pub async fn get_modseq(&self, account_id: u32) -> trc::Result<Option<u64>> { // Obtain current modseq - self.jmap + self.server .core .storage .data @@ -221,7 +223,7 @@ impl<T: SessionStream> SessionData<T> { } pub async fn get_uid_validity(&self, mailbox: &MailboxId) -> trc::Result<u32> { - self.jmap + self.server .get_property::<Object<Value>>( mailbox.account_id, Collection::Mailbox, diff --git a/crates/imap/src/core/mod.rs b/crates/imap/src/core/mod.rs index b8175d67..db92118c 100644 --- a/crates/imap/src/core/mod.rs +++ b/crates/imap/src/core/mod.rs @@ -5,29 +5,21 @@ */ use std::{ - collections::BTreeMap, net::IpAddr, sync::{atomic::AtomicU32, Arc}, }; -use ahash::AHashMap; use common::{ auth::AccessToken, listener::{limiter::InFlight, ServerInstance, SessionStream}, + Account, ImapId, Inner, MailboxId, MailboxState, Server, }; -use dashmap::DashMap; -use imap_proto::{ - protocol::{list::Attribute, ProtocolVersion}, - receiver::Receiver, - Command, -}; -use jmap::{auth::rate_limit::ConcurrencyLimiters, JmapInstance, JMAP}; +use imap_proto::{protocol::ProtocolVersion, receiver::Receiver, Command}; use tokio::{ io::{ReadHalf, WriteHalf}, sync::watch, }; use trc::AddContext; -use utils::lru_cache::LruCache; pub mod client; pub mod mailbox; @@ -36,32 +28,17 @@ pub mod session; #[derive(Clone)] pub struct ImapSessionManager { - pub imap: ImapInstance, + pub inner: Arc<Inner>, } impl ImapSessionManager { - pub fn new(imap: ImapInstance) -> Self { - Self { imap } + pub fn new(inner: Arc<Inner>) -> Self { + Self { inner } } } -#[derive(Clone)] -pub struct ImapInstance { - pub jmap_instance: JmapInstance, - pub imap_inner: Arc<Inner>, -} - -pub struct Inner { - pub rate_limiter: DashMap<u32, Arc<ConcurrencyLimiters>>, - pub cache_account: LruCache<AccountId, Arc<Account>>, - pub cache_mailbox: LruCache<MailboxId, Arc<MailboxState>>, -} - -pub struct IMAP {} - pub struct Session<T: SessionStream> { - pub jmap: JMAP, - pub imap: Arc<Inner>, + pub server: Server, pub instance: Arc<ServerInstance>, pub receiver: Receiver<Command>, pub version: ProtocolVersion, @@ -79,8 +56,7 @@ pub struct Session<T: SessionStream> { pub struct SessionData<T: SessionStream> { pub account_id: u32, pub access_token: Arc<AccessToken>, - pub jmap: JMAP, - pub imap: Arc<Inner>, + pub server: Server, pub session_id: u64, pub mailboxes: parking_lot::Mutex<Vec<Account>>, pub stream_tx: Arc<tokio::sync::Mutex<WriteHalf<T>>>, @@ -88,29 +64,6 @@ pub struct SessionData<T: SessionStream> { pub in_flight: Option<InFlight>, } -#[derive(Debug, Default, Clone)] -pub struct Mailbox { - pub has_children: bool, - pub is_subscribed: bool, - pub special_use: Option<Attribute>, - pub total_messages: Option<u32>, - pub total_unseen: Option<u32>, - pub total_deleted: Option<u32>, - pub uid_validity: Option<u32>, - pub uid_next: Option<u32>, - pub size: Option<u32>, -} - -#[derive(Debug, Clone, Default)] -pub struct Account { - pub account_id: u32, - pub prefix: Option<String>, - pub mailbox_names: BTreeMap<String, u32>, - pub mailbox_state: AHashMap<u32, Mailbox>, - pub state_email: Option<u64>, - pub state_mailbox: Option<u64>, -} - pub struct SelectedMailbox { pub id: MailboxId, pub state: parking_lot::Mutex<MailboxState>, @@ -119,36 +72,6 @@ pub struct SelectedMailbox { pub is_condstore: bool, } -#[derive(Debug, PartialEq, Eq, Hash, Clone, Copy)] -pub struct MailboxId { - pub account_id: u32, - pub mailbox_id: u32, -} - -#[derive(Debug, PartialEq, Eq, Hash, Clone, Copy)] -pub struct AccountId { - pub account_id: u32, - pub primary_id: u32, -} - -#[derive(Debug, Clone, Default)] -pub struct MailboxState { - pub uid_next: u32, - pub uid_validity: u32, - pub uid_max: u32, - pub id_to_imap: AHashMap<u32, ImapId>, - pub uid_to_id: AHashMap<u32, u32>, - pub total_messages: usize, - pub modseq: Option<u64>, - pub next_state: Option<Box<NextMailboxState>>, -} - -#[derive(Debug, Clone)] -pub struct NextMailboxState { - pub next_state: MailboxState, - pub deletions: Vec<ImapId>, -} - #[derive(Debug, Default)] pub struct MailboxSync { pub added: Vec<String>, @@ -167,12 +90,6 @@ pub enum SavedSearch { } #[derive(Debug, Clone, Copy, Default)] -pub struct ImapId { - pub uid: u32, - pub seqnum: u32, -} - -#[derive(Debug, Clone, Copy, Default)] pub struct ImapUidToId { pub uid: u32, pub id: u32, @@ -217,8 +134,7 @@ impl<T: SessionStream> State<T> { impl<T: SessionStream> SessionData<T> { pub async fn get_access_token(&self) -> trc::Result<Arc<AccessToken>> { - self.jmap - .core + self.server .get_cached_access_token(self.account_id) .await .caused_by(trc::location!()) @@ -230,8 +146,7 @@ impl<T: SessionStream> SessionData<T> { ) -> SessionData<U> { SessionData { account_id: self.account_id, - jmap: self.jmap, - imap: self.imap, + server: self.server, session_id: self.session_id, mailboxes: self.mailboxes, stream_tx: new_stream, diff --git a/crates/imap/src/core/session.rs b/crates/imap/src/core/session.rs index 8284ed67..9f2bc75e 100644 --- a/crates/imap/src/core/session.rs +++ b/crates/imap/src/core/session.rs @@ -6,12 +6,14 @@ use std::sync::Arc; -use common::listener::{stream::NullIo, SessionData, SessionManager, SessionResult, SessionStream}; +use common::{ + core::BuildServer, + listener::{stream::NullIo, SessionData, SessionManager, SessionResult, SessionStream}, +}; use imap_proto::{ protocol::{ProtocolVersion, SerializeResponse}, receiver::Receiver, }; -use jmap::JMAP; use tokio::io::{AsyncReadExt, AsyncWriteExt}; use tokio_rustls::server::TlsStream; @@ -51,9 +53,9 @@ impl<T: SessionStream> Session<T> { tokio::select! { result = tokio::time::timeout( if !matches!(self.state, State::NotAuthenticated {..}) { - self.jmap.core.imap.timeout_auth + self.server.core.imap.timeout_auth } else { - self.jmap.core.imap.timeout_unauth + self.server.core.imap.timeout_unauth }, self.stream_rx.read(&mut buf)) => { match result { @@ -138,17 +140,16 @@ impl<T: SessionStream> Session<T> { // Split stream into read and write halves let (stream_rx, stream_tx) = tokio::io::split(session.stream); - let jmap = JMAP::from(manager.imap.jmap_instance); + let server = manager.inner.build_server(); Ok(Session { - receiver: Receiver::with_max_request_size(jmap.core.imap.max_request_size), + receiver: Receiver::with_max_request_size(server.core.imap.max_request_size), version: ProtocolVersion::Rev1, state: State::NotAuthenticated { auth_failures: 0 }, is_tls, is_condstore: false, is_qresync: false, - jmap, - imap: manager.imap.imap_inner, + server, instance: session.instance, session_id: session.session_id, in_flight: session.in_flight, @@ -196,8 +197,7 @@ impl<T: SessionStream> Session<T> { let stream_tx = Arc::new(tokio::sync::Mutex::new(stream_tx)); Ok(Session { - jmap: self.jmap, - imap: self.imap, + server: self.server, instance: self.instance, receiver: self.receiver, version: self.version, diff --git a/crates/imap/src/lib.rs b/crates/imap/src/lib.rs index e4a0eccc..f6df61e0 100644 --- a/crates/imap/src/lib.rs +++ b/crates/imap/src/lib.rs @@ -4,19 +4,9 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use core::{ImapInstance, Inner, IMAP}; -use std::{ - collections::hash_map::RandomState, - sync::{Arc, LazyLock}, -}; +use std::sync::LazyLock; -use dashmap::DashMap; use imap_proto::{protocol::capability::Capability, ResponseCode, StatusResponse}; -use jmap::JmapInstance; -use utils::{ - config::Config, - lru_cache::{LruCache, LruCached}, -}; pub mod core; pub mod op; @@ -39,33 +29,4 @@ pub(crate) static GREETING_WITHOUT_TLS: LazyLock<Vec<u8>> = LazyLock::new(|| { .into_bytes() }); -impl IMAP { - pub async fn init(config: &mut Config, jmap_instance: JmapInstance) -> ImapInstance { - let shard_amount = config - .property::<u64>("cache.shard") - .unwrap_or(32) - .next_power_of_two() as usize; - let capacity = config.property("cache.capacity").unwrap_or(100); - - let inner = Inner { - rate_limiter: DashMap::with_capacity_and_hasher_and_shard_amount( - capacity, - RandomState::default(), - shard_amount, - ), - cache_account: LruCache::with_capacity( - config.property("cache.account.size").unwrap_or(2048), - ), - cache_mailbox: LruCache::with_capacity( - config.property("cache.mailbox.size").unwrap_or(2048), - ), - }; - - ImapInstance { - jmap_instance, - imap_inner: Arc::new(inner), - } - } -} - pub struct ImapError; diff --git a/crates/imap/src/op/acl.rs b/crates/imap/src/op/acl.rs index b2bc1585..f52026d3 100644 --- a/crates/imap/src/op/acl.rs +++ b/crates/imap/src/op/acl.rs @@ -6,7 +6,7 @@ use std::{sync::Arc, time::Instant}; -use common::{auth::AccessToken, listener::SessionStream}; +use common::{auth::AccessToken, listener::SessionStream, MailboxId}; use directory::{backend::internal::PrincipalField, Permission, QueryBy}; use imap_proto::{ protocol::acl::{ @@ -16,7 +16,10 @@ use imap_proto::{ Command, ResponseCode, StatusResponse, }; -use jmap::{auth::acl::EffectiveAcl, mailbox::set::SCHEMA}; +use jmap::{ + auth::acl::EffectiveAcl, changes::write::ChangeLog, mailbox::set::SCHEMA, + services::state::StateManager, JmapMethods, +}; use jmap_proto::{ object::{index::ObjectIndexBuilder, Object}, types::{ @@ -33,7 +36,7 @@ use trc::AddContext; use utils::map::bitmap::Bitmap; use crate::{ - core::{MailboxId, Session, SessionData, State}, + core::{Session, SessionData, State}, op::ImapContext, spawn_op, }; @@ -62,7 +65,7 @@ impl<T: SessionStream> Session<T> { { for item in acls { if let Some(account_name) = data - .jmap + .server .core .storage .directory @@ -244,7 +247,7 @@ impl<T: SessionStream> Session<T> { // Obtain principal id let acl_account_id = data - .jmap + .server .core .storage .directory @@ -345,18 +348,18 @@ impl<T: SessionStream> Session<T> { .with_current(values), ); if !batch.is_empty() { - data.jmap + data.server .write_batch(batch) .await .imap_ctx(&arguments.tag, trc::location!())?; let mut changes = ChangeLogBuilder::new(); changes.log_update(Collection::Mailbox, mailbox_id); let change_id = data - .jmap + .server .commit_changes(mailbox.account_id, changes) .await .imap_ctx(&arguments.tag, trc::location!())?; - data.jmap + data.server .broadcast_state_change( StateChange::new(mailbox.account_id) .with_change(DataType::Mailbox, change_id), @@ -365,11 +368,7 @@ impl<T: SessionStream> Session<T> { } // Invalidate ACLs - data.jmap - .core - .security - .access_tokens - .remove(&acl_account_id); + data.server.inner.data.access_tokens.remove(&acl_account_id); trc::event!( Imap(trc::ImapEvent::SetAcl), @@ -447,7 +446,7 @@ impl<T: SessionStream> SessionData<T> { ) -> trc::Result<(MailboxId, HashedValue<Object<Value>>, Arc<AccessToken>)> { if let Some(mailbox) = self.get_mailbox_by_name(&arguments.mailbox_name) { if let Some(values) = self - .jmap + .server .get_property::<HashedValue<Object<Value>>>( mailbox.account_id, Collection::Mailbox, diff --git a/crates/imap/src/op/append.rs b/crates/imap/src/op/append.rs index 02308ef9..284b5c7d 100644 --- a/crates/imap/src/op/append.rs +++ b/crates/imap/src/op/append.rs @@ -14,11 +14,14 @@ use imap_proto::{ }; use crate::{ - core::{ImapUidToId, MailboxId, SelectedMailbox, Session, SessionData}, + core::{ImapUidToId, SelectedMailbox, Session, SessionData}, spawn_op, }; -use common::listener::SessionStream; -use jmap::email::ingest::{IngestEmail, IngestSource}; +use common::{listener::SessionStream, MailboxId}; +use jmap::{ + email::ingest::{EmailIngest, IngestEmail, IngestSource}, + services::state::StateManager, +}; use jmap_proto::types::{acl::Acl, keyword::Keyword, state::StateChange, type_state::DataType}; use mail_parser::MessageParser; @@ -89,8 +92,7 @@ impl<T: SessionStream> SessionData<T> { // Obtain quota let resource_token = self - .jmap - .core + .server .get_cached_access_token(mailbox.account_id) .await .imap_ctx(&arguments.tag, trc::location!())? @@ -102,7 +104,7 @@ impl<T: SessionStream> SessionData<T> { let mut last_change_id = None; for message in arguments.messages { match self - .jmap + .server .email_ingest(IngestEmail { raw_message: &message.message, message: MessageParser::new().parse(&message.message), @@ -111,7 +113,7 @@ impl<T: SessionStream> SessionData<T> { keywords: message.flags.into_iter().map(Keyword::from).collect(), received_at: message.received_at.map(|d| d as u64), source: IngestSource::Imap, - encrypt: self.jmap.core.jmap.encrypt && self.jmap.core.jmap.encrypt_append, + encrypt: self.server.core.jmap.encrypt && self.server.core.jmap.encrypt_append, session_id: self.session_id, }) .await @@ -142,7 +144,7 @@ impl<T: SessionStream> SessionData<T> { // Broadcast changes if let Some(change_id) = last_change_id { - self.jmap + self.server .broadcast_state_change( StateChange::new(account_id) .with_change(DataType::Email, change_id) diff --git a/crates/imap/src/op/authenticate.rs b/crates/imap/src/op/authenticate.rs index 5724c6af..83327535 100644 --- a/crates/imap/src/op/authenticate.rs +++ b/crates/imap/src/op/authenticate.rs @@ -11,6 +11,9 @@ use imap_proto::{ receiver::{self, Request}, Command, ResponseCode, StatusResponse, }; +use jmap::auth::{ + authenticate::Authenticator, oauth::token::TokenHandler, rate_limit::RateLimiter, +}; use mail_parser::decoders::base64::base64_decode; use mail_send::Credentials; use std::sync::Arc; @@ -70,7 +73,7 @@ impl<T: SessionStream> Session<T> { tag: String, ) -> trc::Result<()> { // Throttle authentication requests - self.jmap + self.server .is_auth_allowed_soft(&self.remote_addr) .await .map_err(|err| err.id(tag.clone()))?; @@ -78,17 +81,17 @@ impl<T: SessionStream> Session<T> { // Authenticate let access_token = match credentials { Credentials::Plain { username, secret } | Credentials::XOauth2 { username, secret } => { - self.jmap + self.server .authenticate_plain(&username, &secret, self.remote_addr, self.session_id) .await } Credentials::OAuthBearer { token } => { match self - .jmap + .server .validate_access_token("access_token", &token) .await { - Ok((account_id, _, _)) => self.jmap.core.get_access_token(account_id).await, + Ok((account_id, _, _)) => self.server.get_access_token(account_id).await, Err(err) => Err(err), } } @@ -96,7 +99,7 @@ impl<T: SessionStream> Session<T> { .map_err(|err| { if err.matches(trc::EventType::Auth(trc::AuthEvent::Failed)) { let auth_failures = self.state.auth_failures(); - if auth_failures < self.jmap.core.imap.max_auth_failures { + if auth_failures < self.server.core.imap.max_auth_failures { self.state = State::NotAuthenticated { auth_failures: auth_failures + 1, }; @@ -127,7 +130,7 @@ impl<T: SessionStream> Session<T> { // Cache access token let access_token = Arc::new(access_token); - self.jmap.core.cache_access_token(access_token.clone()); + self.server.cache_access_token(access_token.clone()); // Create session self.state = State::Authenticated { diff --git a/crates/imap/src/op/capability.rs b/crates/imap/src/op/capability.rs index a8f3d7f5..deebbb76 100644 --- a/crates/imap/src/op/capability.rs +++ b/crates/imap/src/op/capability.rs @@ -28,7 +28,7 @@ impl<T: SessionStream> Session<T> { Imap(trc::ImapEvent::Capabilities), SpanId = self.session_id, Tls = self.is_tls, - Strict = !self.jmap.core.imap.allow_plain_auth, + Strict = !self.server.core.imap.allow_plain_auth, Elapsed = op_start.elapsed() ); diff --git a/crates/imap/src/op/copy_move.rs b/crates/imap/src/op/copy_move.rs index 43fd6795..697748fa 100644 --- a/crates/imap/src/op/copy_move.rs +++ b/crates/imap/src/op/copy_move.rs @@ -13,11 +13,17 @@ use imap_proto::{ }; use crate::{ - core::{MailboxId, SelectedMailbox, Session, SessionData}, + core::{SelectedMailbox, Session, SessionData}, spawn_op, }; -use common::listener::SessionStream; -use jmap::{email::set::TagManager, mailbox::UidMailbox}; +use common::{listener::SessionStream, MailboxId}; +use jmap::{ + changes::write::ChangeLog, + email::{copy::EmailCopy, ingest::EmailIngest, set::TagManager}, + mailbox::UidMailbox, + services::state::StateManager, + JmapMethods, +}; use jmap_proto::{ error::set::SetErrorType, types::{ @@ -200,7 +206,7 @@ impl<T: SessionStream> SessionData<T> { for uid_mailbox in mailboxes.inner_tags_mut() { if uid_mailbox.uid == 0 { let assigned_uid = self - .jmap + .server .assign_imap_uid(account_id, uid_mailbox.mailbox_id) .await .imap_ctx(&arguments.tag, trc::location!())?; @@ -219,13 +225,13 @@ impl<T: SessionStream> SessionData<T> { mailboxes.update_batch(&mut batch, Property::MailboxIds); if changelog.change_id == u64::MAX { changelog.change_id = self - .jmap + .server .assign_change_id(account_id) .await .imap_ctx(&arguments.tag, trc::location!())?; } batch.value(Property::Cid, changelog.change_id, F_VALUE); - self.jmap + self.server .write_batch(batch) .await .imap_ctx(&arguments.tag, trc::location!())?; @@ -242,8 +248,7 @@ impl<T: SessionStream> SessionData<T> { let mut dest_change_id = None; let dest_account_id = dest_mailbox.account_id; let resource_token = self - .jmap - .core + .server .get_cached_access_token(dest_account_id) .await .imap_ctx(&arguments.tag, trc::location!())? @@ -251,7 +256,7 @@ impl<T: SessionStream> SessionData<T> { let mut destroy_ids = RoaringBitmap::new(); for (id, imap_id) in ids { match self - .jmap + .server .copy_message( src_account_id, id, @@ -303,7 +308,7 @@ impl<T: SessionStream> SessionData<T> { // Broadcast changes on destination account if let Some(change_id) = dest_change_id { - self.jmap + self.server .broadcast_state_change( StateChange::new(dest_account_id) .with_change(DataType::Email, change_id) @@ -317,11 +322,11 @@ impl<T: SessionStream> SessionData<T> { // Write changes on source account if !changelog.is_empty() { let change_id = self - .jmap + .server .commit_changes(src_mailbox.id.account_id, changelog) .await .imap_ctx(&arguments.tag, trc::location!())?; - self.jmap + self.server .broadcast_state_change( StateChange::new(src_mailbox.id.account_id) .with_change(DataType::Email, change_id) @@ -423,7 +428,7 @@ impl<T: SessionStream> SessionData<T> { ) -> trc::Result<Option<(TagManager<UidMailbox>, u32)>> { // Obtain mailbox tags if let (Some(mailboxes), Some(thread_id)) = ( - self.jmap + self.server .get_property::<HashedValue<Vec<UidMailbox>>>( account_id, Collection::Email, @@ -431,7 +436,7 @@ impl<T: SessionStream> SessionData<T> { Property::MailboxIds, ) .await?, - self.jmap + self.server .get_property::<u32>(account_id, Collection::Email, id, Property::ThreadId) .await?, ) { diff --git a/crates/imap/src/op/create.rs b/crates/imap/src/op/create.rs index edd10ea0..2889cb5f 100644 --- a/crates/imap/src/op/create.rs +++ b/crates/imap/src/op/create.rs @@ -7,18 +7,20 @@ use std::time::Instant; use crate::{ - core::{Account, Mailbox, Session, SessionData}, + core::{Session, SessionData}, op::ImapContext, spawn_op, }; -use common::listener::SessionStream; +use common::{listener::SessionStream, Account, Mailbox}; use directory::Permission; use imap_proto::{ protocol::{create::Arguments, list::Attribute}, receiver::Request, Command, ResponseCode, StatusResponse, }; -use jmap::mailbox::set::SCHEMA; +use jmap::{ + changes::write::ChangeLog, mailbox::set::SCHEMA, services::state::StateManager, JmapMethods, +}; use jmap_proto::{ object::{index::ObjectIndexBuilder, Object}, types::{ @@ -75,7 +77,7 @@ impl<T: SessionStream> SessionData<T> { // Build batch let mut changes = self - .jmap + .server .begin_changes(params.account_id) .await .imap_ctx(&arguments.tag, trc::location!())?; @@ -102,7 +104,7 @@ impl<T: SessionStream> SessionData<T> { .create_document() .custom(ObjectIndexBuilder::new(SCHEMA).with_changes(mailbox)); let mailbox_id = self - .jmap + .server .write_batch_expect_id(batch) .await .imap_ctx(&arguments.tag, trc::location!())?; @@ -118,13 +120,13 @@ impl<T: SessionStream> SessionData<T> { .with_account_id(params.account_id) .with_collection(Collection::Mailbox) .custom(changes); - self.jmap + self.server .write_batch(batch) .await .imap_ctx(&arguments.tag, trc::location!())?; // Broadcast changes - self.jmap + self.server .broadcast_state_change( StateChange::new(params.account_id).with_change(DataType::Mailbox, change_id), ) @@ -205,7 +207,7 @@ impl<T: SessionStream> SessionData<T> { }; let effective_id = self - .jmap + .server .core .jmap .default_folders @@ -271,7 +273,7 @@ impl<T: SessionStream> SessionData<T> { return Err(trc::ImapEvent::Error .into_err() .details("Invalid empty path item.")); - } else if path_item.len() > self.jmap.core.jmap.mailbox_name_max_len { + } else if path_item.len() > self.server.core.jmap.mailbox_name_max_len { return Err(trc::ImapEvent::Error .into_err() .details("Mailbox name is too long.")); @@ -279,7 +281,7 @@ impl<T: SessionStream> SessionData<T> { path.push(path_item); } - if path.len() > self.jmap.core.jmap.mailbox_max_depth { + if path.len() > self.server.core.jmap.mailbox_max_depth { return Err(trc::ImapEvent::Error .into_err() .details("Mailbox path is too deep.")); @@ -295,7 +297,7 @@ impl<T: SessionStream> SessionData<T> { let (account_id, path) = { let mailboxes = self.mailboxes.lock(); let first_path_item = path.first().unwrap(); - let account = if first_path_item == &self.jmap.core.jmap.shared_folder { + let account = if first_path_item == &self.server.core.jmap.shared_folder { // Shared Folders/<username>/<folder> if path.len() < 3 { return Err(trc::ImapEvent::Error @@ -391,7 +393,7 @@ impl<T: SessionStream> SessionData<T> { special_use: if let Some(mailbox_role) = mailbox_role { // Make sure role is unique if !self - .jmap + .server .filter( account_id, Collection::Mailbox, diff --git a/crates/imap/src/op/delete.rs b/crates/imap/src/op/delete.rs index d8d028a8..0d9ecec8 100644 --- a/crates/imap/src/op/delete.rs +++ b/crates/imap/src/op/delete.rs @@ -15,6 +15,7 @@ use directory::Permission; use imap_proto::{ protocol::delete::Arguments, receiver::Request, Command, ResponseCode, StatusResponse, }; +use jmap::{changes::write::ChangeLog, mailbox::set::MailboxSet, services::state::StateManager}; use jmap_proto::types::{state::StateChange, type_state::DataType}; use store::write::log::ChangeLogBuilder; @@ -76,7 +77,7 @@ impl<T: SessionStream> SessionData<T> { .imap_ctx(&arguments.tag, trc::location!())?; let mut changelog = ChangeLogBuilder::new(); let did_remove_emails = match self - .jmap + .server .mailbox_destroy(account_id, mailbox_id, &mut changelog, &access_token, true) .await .imap_ctx(&arguments.tag, trc::location!())? @@ -93,13 +94,13 @@ impl<T: SessionStream> SessionData<T> { // Write changes let change_id = self - .jmap + .server .commit_changes(account_id, changelog) .await .imap_ctx(&arguments.tag, trc::location!())?; // Broadcast changes - self.jmap + self.server .broadcast_state_change(if did_remove_emails { StateChange::new(account_id) .with_change(DataType::Mailbox, change_id) diff --git a/crates/imap/src/op/expunge.rs b/crates/imap/src/op/expunge.rs index 36c99003..519c2808 100644 --- a/crates/imap/src/op/expunge.rs +++ b/crates/imap/src/op/expunge.rs @@ -15,9 +15,15 @@ use imap_proto::{ }; use trc::AddContext; -use crate::core::{ImapId, SavedSearch, SelectedMailbox, Session, SessionData}; -use common::listener::SessionStream; -use jmap::{email::set::TagManager, mailbox::UidMailbox}; +use crate::core::{SavedSearch, SelectedMailbox, Session, SessionData}; +use common::{listener::SessionStream, ImapId}; +use jmap::{ + changes::write::ChangeLog, + email::{delete::EmailDeletion, set::TagManager}, + mailbox::UidMailbox, + services::state::StateManager, + JmapMethods, +}; use jmap_proto::types::{ acl::Acl, collection::Collection, id::Id, keyword::Keyword, property::Property, state::StateChange, type_state::DataType, @@ -118,7 +124,7 @@ impl<T: SessionStream> SessionData<T> { // Obtain message ids let account_id = mailbox.id.account_id; let mut deleted_ids = self - .jmap + .server .get_tag( account_id, Collection::Email, @@ -129,7 +135,7 @@ impl<T: SessionStream> SessionData<T> { .caused_by(trc::location!())? .unwrap_or_default() & self - .jmap + .server .get_tag( account_id, Collection::Email, @@ -167,8 +173,8 @@ impl<T: SessionStream> SessionData<T> { // Write changes on source account if !changelog.is_empty() { - let change_id = self.jmap.commit_changes(account_id, changelog).await?; - self.jmap + let change_id = self.server.commit_changes(account_id, changelog).await?; + self.server .broadcast_state_change( StateChange::new(account_id) .with_change(DataType::Email, change_id) @@ -192,7 +198,7 @@ impl<T: SessionStream> SessionData<T> { let mut destroy_ids = RoaringBitmap::new(); for (id, mailbox_ids) in self - .jmap + .server .get_properties::<HashedValue<Vec<UidMailbox>>, _, _>( account_id, Collection::Email, @@ -208,7 +214,7 @@ impl<T: SessionStream> SessionData<T> { if mailboxes.current().len() > 1 { // Remove deleted flag let (mut keywords, thread_id) = if let (Some(keywords), Some(thread_id)) = ( - self.jmap + self.server .get_property::<HashedValue<Vec<Keyword>>>( account_id, Collection::Email, @@ -217,7 +223,7 @@ impl<T: SessionStream> SessionData<T> { ) .await .caused_by(trc::location!())?, - self.jmap + self.server .get_property::<u32>( account_id, Collection::Email, @@ -245,10 +251,10 @@ impl<T: SessionStream> SessionData<T> { mailboxes.update_batch(&mut batch, Property::MailboxIds); keywords.update_batch(&mut batch, Property::Keywords); if changelog.change_id == u64::MAX { - changelog.change_id = self.jmap.assign_change_id(account_id).await? + changelog.change_id = self.server.assign_change_id(account_id).await? } batch.value(Property::Cid, changelog.change_id, F_VALUE); - match self.jmap.write_batch(batch).await { + match self.server.write_batch(batch).await { Ok(_) => { changelog.log_update(Collection::Email, Id::from_parts(thread_id, id)); changelog.log_child_update(Collection::Mailbox, mailbox_id.mailbox_id); @@ -268,7 +274,7 @@ impl<T: SessionStream> SessionData<T> { if !destroy_ids.is_empty() { // Delete message from all mailboxes let (changes, _) = self - .jmap + .server .emails_tombstone(account_id, destroy_ids) .await .caused_by(trc::location!())?; diff --git a/crates/imap/src/op/fetch.rs b/crates/imap/src/op/fetch.rs index 182b8303..be2b1fc8 100644 --- a/crates/imap/src/op/fetch.rs +++ b/crates/imap/src/op/fetch.rs @@ -26,7 +26,13 @@ use imap_proto::{ receiver::Request, Command, ResponseCode, ResponseType, StatusResponse, }; -use jmap::email::metadata::MessageMetadata; +use jmap::{ + blob::download::BlobDownload, + changes::{get::ChangesLookup, write::ChangeLog}, + email::metadata::MessageMetadata, + services::state::StateManager, + JmapMethods, +}; use jmap_proto::types::{ acl::Acl, collection::Collection, id::Id, keyword::Keyword, property::Property, state::StateChange, type_state::DataType, @@ -127,7 +133,7 @@ impl<T: SessionStream> SessionData<T> { if let Some(changed_since) = arguments.changed_since { // Obtain changes since the modseq. let changelog = self - .jmap + .server .changes_( account_id, Collection::Email, @@ -280,7 +286,7 @@ impl<T: SessionStream> SessionData<T> { for (seqnum, uid, id) in ids { // Obtain attributes and keywords let (email, keywords) = if let (Some(email), Some(keywords)) = ( - self.jmap + self.server .get_property::<Bincode<MessageMetadata>>( account_id, Collection::Email, @@ -289,7 +295,7 @@ impl<T: SessionStream> SessionData<T> { ) .await .imap_ctx(&arguments.tag, trc::location!())?, - self.jmap + self.server .get_property::<HashedValue<Vec<Keyword>>>( account_id, Collection::Email, @@ -316,7 +322,7 @@ impl<T: SessionStream> SessionData<T> { let raw_message = if needs_blobs { // Retrieve raw message if needed match self - .jmap + .server .get_blob(&email.blob_hash, 0..usize::MAX) .await .imap_ctx(&arguments.tag, trc::location!())? @@ -347,7 +353,7 @@ impl<T: SessionStream> SessionData<T> { set_seen_flags && !keywords.inner.iter().any(|k| k == &Keyword::Seen); let thread_id = if needs_thread_id || set_seen_flag { if let Some(thread_id) = self - .jmap + .server .get_property::<u32>(account_id, Collection::Email, id, Property::ThreadId) .await .imap_ctx(&arguments.tag, trc::location!())? @@ -479,7 +485,7 @@ impl<T: SessionStream> SessionData<T> { } Attribute::ModSeq => { if let Ok(Some(modseq)) = self - .jmap + .server .get_property::<u64>(account_id, Collection::Email, id, Property::Cid) .await { @@ -524,7 +530,7 @@ impl<T: SessionStream> SessionData<T> { // Set Seen ids if !set_seen_ids.is_empty() { let mut changelog = self - .jmap + .server .begin_changes(account_id) .await .imap_ctx(&arguments.tag, trc::location!())?; @@ -539,7 +545,7 @@ impl<T: SessionStream> SessionData<T> { .value(Property::Keywords, keywords.inner, F_VALUE) .value(Property::Keywords, Keyword::Seen, F_BITMAP) .value(Property::Cid, changelog.change_id, F_VALUE); - match self.jmap.write_batch(batch).await { + match self.server.write_batch(batch).await { Ok(_) => { changelog.log_update(Collection::Email, id); } @@ -553,12 +559,12 @@ impl<T: SessionStream> SessionData<T> { if !changelog.is_empty() { // Write changes let change_id = self - .jmap + .server .commit_changes(account_id, changelog) .await .imap_ctx(&arguments.tag, trc::location!())?; modseq = change_id.into(); - self.jmap + self.server .broadcast_state_change( StateChange::new(account_id).with_change(DataType::Email, change_id), ) diff --git a/crates/imap/src/op/idle.rs b/crates/imap/src/op/idle.rs index 819311b9..c395bf21 100644 --- a/crates/imap/src/op/idle.rs +++ b/crates/imap/src/op/idle.rs @@ -20,6 +20,7 @@ use imap_proto::{ }; use common::listener::SessionStream; +use jmap::{changes::get::ChangesLookup, services::state::StateManager}; use jmap_proto::types::{collection::Collection, type_state::DataType}; use store::query::log::Query; use tokio::io::AsyncReadExt; @@ -53,7 +54,7 @@ impl<T: SessionStream> Session<T> { // Register with state manager let mut change_rx = self - .jmap + .server .subscribe_state_manager(data.account_id, types) .await .imap_ctx(&request.tag, trc::location!())?; @@ -72,7 +73,7 @@ impl<T: SessionStream> Session<T> { let mut buf = vec![0; 4]; loop { tokio::select! { - result = tokio::time::timeout(self.jmap.core.imap.timeout_idle, self.stream_rx.read_exact(&mut buf)) => { + result = tokio::time::timeout(self.server.core.imap.timeout_idle, self.stream_rx.read_exact(&mut buf)) => { match result { Ok(Ok(bytes_read)) => { if bytes_read > 0 { @@ -202,7 +203,7 @@ impl<T: SessionStream> SessionData<T> { // Obtain changed messages let changelog = self - .jmap + .server .changes_( mailbox.id.account_id, Collection::Email, diff --git a/crates/imap/src/op/list.rs b/crates/imap/src/op/list.rs index dce26b75..9b2cc3f3 100644 --- a/crates/imap/src/op/list.rs +++ b/crates/imap/src/op/list.rs @@ -173,10 +173,10 @@ impl<T: SessionStream> SessionData<T> { if let Some(prefix) = &account.prefix { if !added_shared_folder { if !filter_subscribed - && matches_pattern(&patterns, &self.jmap.core.jmap.shared_folder) + && matches_pattern(&patterns, &self.server.core.jmap.shared_folder) { list_items.push(ListItem { - mailbox_name: self.jmap.core.jmap.shared_folder.clone(), + mailbox_name: self.server.core.jmap.shared_folder.clone(), attributes: if include_children { vec![Attribute::HasChildren, Attribute::NoSelect] } else { diff --git a/crates/imap/src/op/namespace.rs b/crates/imap/src/op/namespace.rs index 0f9ed1ac..10f0ea00 100644 --- a/crates/imap/src/op/namespace.rs +++ b/crates/imap/src/op/namespace.rs @@ -30,7 +30,7 @@ impl<T: SessionStream> Session<T> { .serialize( Response { shared_prefix: if self.state.session_data().mailboxes.lock().len() > 1 { - self.jmap.core.jmap.shared_folder.clone().into() + self.server.core.jmap.shared_folder.clone().into() } else { None }, diff --git a/crates/imap/src/op/rename.rs b/crates/imap/src/op/rename.rs index cc8b27fb..9daa7fd7 100644 --- a/crates/imap/src/op/rename.rs +++ b/crates/imap/src/op/rename.rs @@ -15,7 +15,10 @@ use directory::Permission; use imap_proto::{ protocol::rename::Arguments, receiver::Request, Command, ResponseCode, StatusResponse, }; -use jmap::{auth::acl::EffectiveAcl, mailbox::set::SCHEMA}; +use jmap::{ + auth::acl::EffectiveAcl, changes::write::ChangeLog, mailbox::set::SCHEMA, + services::state::StateManager, JmapMethods, +}; use jmap_proto::{ object::{index::ObjectIndexBuilder, Object}, types::{ @@ -92,7 +95,7 @@ impl<T: SessionStream> SessionData<T> { // Obtain mailbox let mailbox = self - .jmap + .server .get_property::<HashedValue<Object<Value>>>( params.account_id, Collection::Mailbox, @@ -133,7 +136,7 @@ impl<T: SessionStream> SessionData<T> { // Build batch let mut changes = self - .jmap + .server .begin_changes(params.account_id) .await .imap_ctx(&arguments.tag, trc::location!())?; @@ -159,7 +162,7 @@ impl<T: SessionStream> SessionData<T> { ); let mailbox_id = self - .jmap + .server .write_batch_expect_id(batch) .await .imap_ctx(&arguments.tag, trc::location!())?; @@ -191,13 +194,13 @@ impl<T: SessionStream> SessionData<T> { let change_id = changes.change_id; batch.custom(changes); - self.jmap + self.server .write_batch(batch) .await .imap_ctx(&arguments.tag, trc::location!())?; // Broadcast changes - self.jmap + self.server .broadcast_state_change( StateChange::new(params.account_id).with_change(DataType::Mailbox, change_id), ) diff --git a/crates/imap/src/op/search.rs b/crates/imap/src/op/search.rs index 86329275..bf8670ac 100644 --- a/crates/imap/src/op/search.rs +++ b/crates/imap/src/op/search.rs @@ -6,7 +6,7 @@ use std::{sync::Arc, time::Instant}; -use common::listener::SessionStream; +use common::{listener::SessionStream, ImapId}; use directory::Permission; use imap_proto::{ protocol::{ @@ -16,6 +16,7 @@ use imap_proto::{ receiver::Request, Command, StatusResponse, }; +use jmap::{changes::get::ChangesLookup, JmapMethods}; use jmap_proto::types::{collection::Collection, id::Id, keyword::Keyword, property::Property}; use mail_parser::HeaderName; use nlp::language::Language; @@ -29,7 +30,7 @@ use tokio::sync::watch; use trc::AddContext; use crate::{ - core::{ImapId, MailboxState, SavedSearch, SelectedMailbox, Session, SessionData}, + core::{SavedSearch, SelectedMailbox, Session, SessionData}, spawn_op, }; @@ -142,7 +143,7 @@ impl<T: SessionStream> SessionData<T> { let mut imap_ids = Vec::with_capacity(results_len); let is_sort = if let Some(sort) = arguments.sort { mailbox.map_search_results( - self.jmap + self.server .core .storage .data @@ -260,7 +261,7 @@ impl<T: SessionStream> SessionData<T> { // Obtain message ids let mut filters = Vec::with_capacity(imap_filter.len() + 1); let message_ids = self - .jmap + .server .get_tag( mailbox.id.account_id, Collection::Email, @@ -290,7 +291,7 @@ impl<T: SessionStream> SessionData<T> { fts_filters.push(FtsFilter::has_text_detect( Field::Body, text, - self.jmap.core.jmap.default_language, + self.server.core.jmap.default_language, )); } search::Filter::Cc(text) => { @@ -350,7 +351,7 @@ impl<T: SessionStream> SessionData<T> { fts_filters.push(FtsFilter::has_text_detect( Field::Header(HeaderName::Subject), text, - self.jmap.core.jmap.default_language, + self.server.core.jmap.default_language, )); } search::Filter::Text(text) => { @@ -378,17 +379,17 @@ impl<T: SessionStream> SessionData<T> { fts_filters.push(FtsFilter::has_text_detect( Field::Header(HeaderName::Subject), &text, - self.jmap.core.jmap.default_language, + self.server.core.jmap.default_language, )); fts_filters.push(FtsFilter::has_text_detect( Field::Body, &text, - self.jmap.core.jmap.default_language, + self.server.core.jmap.default_language, )); fts_filters.push(FtsFilter::has_text_detect( Field::Attachment, text, - self.jmap.core.jmap.default_language, + self.server.core.jmap.default_language, )); fts_filters.push(FtsFilter::End); } @@ -416,7 +417,7 @@ impl<T: SessionStream> SessionData<T> { } filters.push(query::Filter::is_in_set( - self.jmap + self.server .fts_filter(mailbox.id.account_id, Collection::Email, fts_filters) .await?, )); @@ -612,7 +613,7 @@ impl<T: SessionStream> SessionData<T> { search::Filter::ModSeq((modseq, _)) => { let mut set = RoaringBitmap::new(); for change in self - .jmap + .server .changes_( mailbox.id.account_id, Collection::Email, @@ -658,7 +659,7 @@ impl<T: SessionStream> SessionData<T> { } // Run query - self.jmap + self.server .filter(mailbox.id.account_id, Collection::Email, filters) .await .map(|res| (res, include_highest_modseq)) @@ -738,23 +739,6 @@ impl SelectedMailbox { } } -impl MailboxState { - pub fn map_result_id(&self, document_id: u32, is_uid: bool) -> Option<(u32, ImapId)> { - if let Some(imap_id) = self.id_to_imap.get(&document_id) { - Some((if is_uid { imap_id.uid } else { imap_id.seqnum }, *imap_id)) - } else if is_uid { - self.next_state.as_ref().and_then(|s| { - s.next_state - .id_to_imap - .get(&document_id) - .map(|imap_id| (imap_id.uid, *imap_id)) - }) - } else { - None - } - } -} - impl SavedSearch { pub async fn unwrap(&self) -> Option<Arc<Vec<ImapId>>> { match self { diff --git a/crates/imap/src/op/select.rs b/crates/imap/src/op/select.rs index a36125ea..5d6c1853 100644 --- a/crates/imap/src/op/select.rs +++ b/crates/imap/src/op/select.rs @@ -47,35 +47,39 @@ impl<T: SessionStream> Session<T> { if let Some(mailbox) = data.get_mailbox_by_name(&arguments.mailbox_name) { // Try obtaining the mailbox from the cache - let state = { - let modseq = data - .get_modseq(mailbox.account_id) - .await - .imap_ctx(&arguments.tag, trc::location!())?; - - if let Some(cached_state) = - self.imap - .cache_mailbox - .get(&mailbox) - .and_then(|cached_state| { - if cached_state.modseq.unwrap_or(0) >= modseq.unwrap_or(0) { - Some(cached_state) - } else { - None - } - }) + let state = { - cached_state.as_ref().clone() - } else { - let new_state = Arc::new( - data.fetch_messages(&mailbox) - .await - .imap_ctx(&arguments.tag, trc::location!())?, - ); - self.imap.cache_mailbox.insert(mailbox, new_state.clone()); - new_state.as_ref().clone() - } - }; + let modseq = data + .get_modseq(mailbox.account_id) + .await + .imap_ctx(&arguments.tag, trc::location!())?; + + if let Some(cached_state) = + self.server.inner.data.mailbox_cache.get(&mailbox).and_then( + |cached_state| { + if cached_state.modseq.unwrap_or(0) >= modseq.unwrap_or(0) { + Some(cached_state) + } else { + None + } + }, + ) + { + cached_state.as_ref().clone() + } else { + let new_state = Arc::new( + data.fetch_messages(&mailbox) + .await + .imap_ctx(&arguments.tag, trc::location!())?, + ); + self.server + .inner + .data + .mailbox_cache + .insert(mailbox, new_state.clone()); + new_state.as_ref().clone() + } + }; // Synchronize messages let closed_previous = self.state.close_mailbox(); diff --git a/crates/imap/src/op/status.rs b/crates/imap/src/op/status.rs index ce8c6029..d06ae582 100644 --- a/crates/imap/src/op/status.rs +++ b/crates/imap/src/op/status.rs @@ -7,11 +7,11 @@ use std::{sync::Arc, time::Instant}; use crate::{ - core::{Mailbox, Session, SessionData}, + core::{Session, SessionData}, op::ImapContext, spawn_op, }; -use common::listener::SessionStream; +use common::{listener::SessionStream, Mailbox}; use directory::Permission; use imap_proto::{ parser::PushUnique, @@ -19,6 +19,7 @@ use imap_proto::{ receiver::Request, Command, ResponseCode, StatusResponse, }; +use jmap::JmapMethods; use jmap_proto::{ object::Object, types::{collection::Collection, id::Id, keyword::Keyword, property::Property, value::Value}, @@ -86,11 +87,11 @@ impl<T: SessionStream> SessionData<T> { mailbox } else { // Some IMAP clients will try to get the status of a mailbox with the NoSelect flag - return if mailbox_name == self.jmap.core.jmap.shared_folder + return if mailbox_name == self.server.core.jmap.shared_folder || mailbox_name .split_once('/') .map_or(false, |(base_name, path)| { - base_name == self.jmap.core.jmap.shared_folder && !path.contains('/') + base_name == self.server.core.jmap.shared_folder && !path.contains('/') }) { Ok(StatusItem { @@ -211,7 +212,7 @@ impl<T: SessionStream> SessionData<T> { // Retrieve latest values let mut values_update = Vec::with_capacity(items_update.len()); let mailbox_message_ids = self - .jmap + .server .get_tag( mailbox.account_id, Collection::Email, @@ -222,7 +223,7 @@ impl<T: SessionStream> SessionData<T> { .caused_by(trc::location!())? .map(Arc::new); let message_ids = self - .jmap + .server .get_document_ids(mailbox.account_id, Collection::Email) .await .caused_by(trc::location!())?; @@ -232,7 +233,7 @@ impl<T: SessionStream> SessionData<T> { Status::Messages => mailbox_message_ids.as_ref().map(|v| v.len()).unwrap_or(0), Status::UidNext => { (self - .jmap + .server .core .storage .data @@ -247,7 +248,7 @@ impl<T: SessionStream> SessionData<T> { + 1) as u64 } Status::UidValidity => self - .jmap + .server .get_property::<Object<Value>>( mailbox.account_id, Collection::Mailbox, @@ -270,7 +271,7 @@ impl<T: SessionStream> SessionData<T> { (&message_ids, &mailbox_message_ids) { if let Some(mut seen) = self - .jmap + .server .get_tag( mailbox.account_id, Collection::Email, @@ -293,7 +294,7 @@ impl<T: SessionStream> SessionData<T> { Status::Deleted => { if let (Some(mailbox_message_ids), Some(mut deleted)) = ( &mailbox_message_ids, - self.jmap + self.server .get_tag( mailbox.account_id, Collection::Email, @@ -378,7 +379,7 @@ impl<T: SessionStream> SessionData<T> { message_ids: &Arc<RoaringBitmap>, ) -> trc::Result<u32> { let mut total_size = 0u32; - self.jmap + self.server .core .storage .data diff --git a/crates/imap/src/op/store.rs b/crates/imap/src/op/store.rs index 3a799841..14a5626a 100644 --- a/crates/imap/src/op/store.rs +++ b/crates/imap/src/op/store.rs @@ -22,7 +22,13 @@ use imap_proto::{ receiver::Request, Command, ResponseCode, ResponseType, StatusResponse, }; -use jmap::{email::set::TagManager, mailbox::UidMailbox}; +use jmap::{ + changes::{get::ChangesLookup, write::ChangeLog}, + email::set::TagManager, + mailbox::UidMailbox, + services::state::StateManager, + JmapMethods, +}; use jmap_proto::types::{ acl::Acl, collection::Collection, id::Id, keyword::Keyword, property::Property, state::StateChange, type_state::DataType, @@ -110,7 +116,7 @@ impl<T: SessionStream> SessionData<T> { if let Some(unchanged_since) = arguments.unchanged_since { // Obtain changes since the modseq. let changelog = self - .jmap + .server .changes_( account_id, Collection::Email, @@ -192,7 +198,7 @@ impl<T: SessionStream> SessionData<T> { loop { // Obtain current keywords let (mut keywords, thread_id) = if let (Some(keywords), Some(thread_id)) = ( - self.jmap + self.server .get_property::<HashedValue<Vec<Keyword>>>( account_id, Collection::Email, @@ -201,7 +207,7 @@ impl<T: SessionStream> SessionData<T> { ) .await .imap_ctx(response.tag.as_ref().unwrap(), trc::location!())?, - self.jmap + self.server .get_property::<u32>(account_id, Collection::Email, *id, Property::ThreadId) .await .imap_ctx(response.tag.as_ref().unwrap(), trc::location!())?, @@ -253,18 +259,18 @@ impl<T: SessionStream> SessionData<T> { keywords.update_batch(&mut batch, Property::Keywords); if changelog.change_id == u64::MAX { changelog.change_id = self - .jmap + .server .assign_change_id(account_id) .await .imap_ctx(response.tag.as_ref().unwrap(), trc::location!())? } batch.value(Property::Cid, changelog.change_id, F_VALUE); - match self.jmap.write_batch(batch).await { + match self.server.write_batch(batch).await { Ok(_) => { // Set all current mailboxes as changed if the Seen tag changed if seen_changed { if let Some(mailboxes) = self - .jmap + .server .get_property::<Vec<UidMailbox>>( account_id, Collection::Email, @@ -335,11 +341,11 @@ impl<T: SessionStream> SessionData<T> { // Write changes if !changelog.is_empty() { let change_id = self - .jmap + .server .commit_changes(account_id, changelog) .await .imap_ctx(response.tag.as_ref().unwrap(), trc::location!())?; - self.jmap + self.server .broadcast_state_change(if !changed_mailboxes.is_empty() { StateChange::new(account_id) .with_change(DataType::Email, change_id) diff --git a/crates/imap/src/op/subscribe.rs b/crates/imap/src/op/subscribe.rs index 995ed430..79b426d1 100644 --- a/crates/imap/src/op/subscribe.rs +++ b/crates/imap/src/op/subscribe.rs @@ -13,7 +13,12 @@ use crate::{ use common::listener::SessionStream; use directory::Permission; use imap_proto::{receiver::Request, Command, ResponseCode, StatusResponse}; -use jmap::mailbox::set::{MailboxSubscribe, SCHEMA}; +use jmap::{ + changes::write::ChangeLog, + mailbox::set::{MailboxSubscribe, SCHEMA}, + services::state::StateManager, + JmapMethods, +}; use jmap_proto::{ object::{index::ObjectIndexBuilder, Object}, types::{ @@ -100,7 +105,7 @@ impl<T: SessionStream> SessionData<T> { // Obtain mailbox let mailbox = self - .jmap + .server .get_property::<HashedValue<Object<Value>>>( account_id, Collection::Mailbox, @@ -122,7 +127,7 @@ impl<T: SessionStream> SessionData<T> { if let Some(value) = mailbox.inner.mailbox_subscribe(self.account_id, subscribe) { // Build batch let mut changes = self - .jmap + .server .begin_changes(account_id) .await .imap_ctx(&tag, trc::location!())?; @@ -142,13 +147,13 @@ impl<T: SessionStream> SessionData<T> { let change_id = changes.change_id; batch.custom(changes); - self.jmap + self.server .write_batch(batch) .await .imap_ctx(&tag, trc::location!())?; // Broadcast changes - self.jmap + self.server .broadcast_state_change( StateChange::new(account_id).with_change(DataType::Mailbox, change_id), ) diff --git a/crates/imap/src/op/thread.rs b/crates/imap/src/op/thread.rs index a8425652..94d8da12 100644 --- a/crates/imap/src/op/thread.rs +++ b/crates/imap/src/op/thread.rs @@ -21,6 +21,7 @@ use imap_proto::{ receiver::Request, Command, StatusResponse, }; +use jmap::email::cache::ThreadCache; use trc::AddContext; impl<T: SessionStream> Session<T> { @@ -80,7 +81,7 @@ impl<T: SessionStream> SessionData<T> { // Lock the cache let thread_ids = self - .jmap + .server .get_cached_thread_ids(mailbox.id.account_id, result_set.results.iter()) .await .caused_by(trc::location!())?; diff --git a/crates/jmap/src/api/autoconfig.rs b/crates/jmap/src/api/autoconfig.rs index 296dd5aa..27d61ac4 100644 --- a/crates/jmap/src/api/autoconfig.rs +++ b/crates/jmap/src/api/autoconfig.rs @@ -6,18 +6,34 @@ use std::fmt::Write; -use common::manager::webadmin::Resource; +use common::{manager::webadmin::Resource, Server}; use directory::{backend::internal::PrincipalField, QueryBy}; use quick_xml::events::Event; use quick_xml::Reader; use utils::url_params::UrlParams; -use crate::{api::http::ToHttpResponse, JMAP}; +use crate::api::http::ToHttpResponse; use super::{HttpRequest, HttpResponse}; +use std::future::Future; -impl JMAP { - pub async fn handle_autoconfig_request(&self, req: &HttpRequest) -> trc::Result<HttpResponse> { +pub trait Autoconfig: Sync + Send { + fn handle_autoconfig_request( + &self, + req: &HttpRequest, + ) -> impl Future<Output = trc::Result<HttpResponse>> + Send; + fn handle_autodiscover_request( + &self, + body: Option<Vec<u8>>, + ) -> impl Future<Output = trc::Result<HttpResponse>> + Send; + fn autoconfig_parameters<'x>( + &self, + emailaddress: &'x str, + ) -> impl Future<Output = trc::Result<(String, String, &'x str)>> + Send; +} + +impl Autoconfig for Server { + async fn handle_autoconfig_request(&self, req: &HttpRequest) -> trc::Result<HttpResponse> { // Obtain parameters let params = UrlParams::new(req.uri().query()); let emailaddress = params @@ -73,7 +89,7 @@ impl JMAP { ) } - pub async fn handle_autodiscover_request( + async fn handle_autodiscover_request( &self, body: Option<Vec<u8>>, ) -> trc::Result<HttpResponse> { diff --git a/crates/jmap/src/api/event_source.rs b/crates/jmap/src/api/event_source.rs index 362eba1f..0a37544c 100644 --- a/crates/jmap/src/api/event_source.rs +++ b/crates/jmap/src/api/event_source.rs @@ -9,7 +9,7 @@ use std::{ time::{Duration, Instant}, }; -use common::auth::AccessToken; +use common::{auth::AccessToken, Server}; use http_body_util::{combinators::BoxBody, StreamBody}; use hyper::{ body::{Bytes, Frame}, @@ -18,9 +18,10 @@ use hyper::{ use jmap_proto::types::type_state::DataType; use utils::map::bitmap::Bitmap; -use crate::{JMAP, LONG_SLUMBER}; +use crate::{services::state::StateManager, LONG_SLUMBER}; use super::{HttpRequest, HttpResponse, HttpResponseBody, StateChangeResponse}; +use std::future::Future; struct Ping { interval: Duration, @@ -28,8 +29,16 @@ struct Ping { payload: Bytes, } -impl JMAP { - pub async fn handle_event_source( +pub trait EventSourceHandler: Sync + Send { + fn handle_event_source( + &self, + req: HttpRequest, + access_token: Arc<AccessToken>, + ) -> impl Future<Output = trc::Result<HttpResponse>> + Send; +} + +impl EventSourceHandler for Server { + async fn handle_event_source( &self, req: HttpRequest, access_token: Arc<AccessToken>, diff --git a/crates/jmap/src/api/http.rs b/crates/jmap/src/api/http.rs index 86943ffe..4252edcb 100644 --- a/crates/jmap/src/api/http.rs +++ b/crates/jmap/src/api/http.rs @@ -8,10 +8,12 @@ use std::{borrow::Cow, net::IpAddr, sync::Arc}; use common::{ auth::AccessToken, + core::BuildServer, expr::{functions::ResolveVariable, *}, + ipc::StateEvent, listener::{ServerInstance, SessionData, SessionManager, SessionStream}, manager::webadmin::Resource, - Core, + Inner, Server, }; use directory::Permission; use http_body_util::{BodyExt, Full}; @@ -29,17 +31,26 @@ use jmap_proto::{ response::Response, types::{blob::BlobId, id::Id}, }; +use std::future::Future; use crate::{ - auth::{authenticate::HttpHeaders, oauth::OAuthMetadata}, - blob::{DownloadResponse, UploadResponse}, - services::state, - JmapInstance, JMAP, + api::management::enterprise::telemetry::TelemetryApi, + auth::{ + authenticate::{Authenticator, HttpHeaders}, + oauth::{auth::OAuthApiHandler, token::TokenHandler, OAuthMetadata}, + rate_limit::RateLimiter, + }, + blob::{download::BlobDownload, upload::BlobUpload, DownloadResponse, UploadResponse}, + websocket::upgrade::WebSocketUpgrade, }; use super::{ - management::ManagementApiError, HtmlResponse, HttpRequest, HttpResponse, HttpResponseBody, - JmapSessionManager, JsonResponse, + autoconfig::Autoconfig, + event_source::EventSourceHandler, + management::{ManagementApi, ManagementApiError}, + request::RequestHandler, + session::SessionHandler, + HtmlResponse, HttpRequest, HttpResponse, HttpResponseBody, JmapSessionManager, JsonResponse, }; pub struct HttpSessionData { @@ -52,8 +63,16 @@ pub struct HttpSessionData { pub session_id: u64, } -impl JMAP { - pub async fn parse_http_request( +pub trait ParseHttp: Sync + Send { + fn parse_http_request( + &self, + req: HttpRequest, + session: HttpSessionData, + ) -> impl Future<Output = trc::Result<HttpResponse>> + Send; +} + +impl ParseHttp for Server { + async fn parse_http_request( &self, mut req: HttpRequest, session: HttpSessionData, @@ -63,7 +82,7 @@ impl JMAP { // Validate endpoint access let ctx = HttpContext::new(&session, &req); - match ctx.has_endpoint_access(&self.core).await { + match ctx.has_endpoint_access(self).await { StatusCode::OK => (), status => { // Allow loopback address to avoid lockouts @@ -198,10 +217,7 @@ impl JMAP { self.authenticate_headers(&req, &session).await?; return Ok(self - .handle_session_resource( - ctx.resolve_response_url(&self.core).await, - access_token, - ) + .handle_session_resource(ctx.resolve_response_url(self).await, access_token) .await? .into_http_response()); } @@ -210,11 +226,11 @@ impl JMAP { self.is_anonymous_allowed(&session.remote_ip).await?; return Ok(JsonResponse::new(OAuthMetadata::new( - ctx.resolve_response_url(&self.core).await, + ctx.resolve_response_url(self).await, )) .into_http_response()); } - ("acme-challenge", &Method::GET) if self.core.has_acme_http_providers() => { + ("acme-challenge", &Method::GET) if self.has_acme_http_providers() => { if let Some(token) = path.next() { return match self .core @@ -230,7 +246,7 @@ impl JMAP { } } ("mta-sts.txt", &Method::GET) => { - if let Some(policy) = self.core.build_mta_sts_policy() { + if let Some(policy) = self.build_mta_sts_policy() { return Ok(Resource::new("text/plain", policy.to_string().into_bytes()) .into_http_response()); } else { @@ -256,7 +272,7 @@ impl JMAP { ("device", &Method::POST) => { self.is_anonymous_allowed(&session.remote_ip).await?; - let url = ctx.resolve_response_url(&self.core).await; + let url = ctx.resolve_response_url(self).await; return self .handle_device_auth(&mut req, url, session.session_id) .await; @@ -389,7 +405,7 @@ impl JMAP { return Ok(Resource::new( "text/plain; version=0.0.4", - self.core.export_prometheus_metrics().await?.into_bytes(), + self.export_prometheus_metrics().await?.into_bytes(), ) .into_http_response()); } @@ -400,13 +416,12 @@ impl JMAP { _ => (), }, #[cfg(feature = "enterprise")] - "logo.svg" if self.core.is_enterprise_edition() => { + "logo.svg" if self.is_enterprise_edition() => { // SPDX-SnippetBegin // SPDX-FileCopyrightText: 2020 Stalwart Labs Ltd <hello@stalw.art> // SPDX-License-Identifier: LicenseRef-SEL match self - .core .logo_resource( req.headers() .get(header::HOST) @@ -425,7 +440,7 @@ impl JMAP { } } - let resource = self.inner.webadmin.get("logo.svg").await?; + let resource = self.inner.data.webadmin.get("logo.svg").await?; return if !resource.is_empty() { Ok(resource.into_http_response()) @@ -439,6 +454,7 @@ impl JMAP { let path = req.uri().path(); let resource = self .inner + .data .webadmin .get(path.strip_prefix('/').unwrap_or(path)) .await?; @@ -455,166 +471,156 @@ impl JMAP { } } -impl JmapInstance { - async fn handle_session<T: SessionStream>(self, session: SessionData<T>) { - let _in_flight = session.in_flight; - let is_tls = session.stream.is_tls(); - - if let Err(http_err) = http1::Builder::new() - .keep_alive(true) - .serve_connection( - TokioIo::new(session.stream), - service_fn(|req: hyper::Request<body::Incoming>| { - let jmap_instance = self.clone(); - let instance = session.instance.clone(); - - async move { - let jmap = JMAP::from(jmap_instance); - - // Obtain remote IP - let remote_ip = if !jmap.core.jmap.http_use_forwarded { - trc::event!( - Http(trc::HttpEvent::RequestUrl), - SpanId = session.session_id, - Url = req.uri().to_string(), - ); - - session.remote_ip - } else if let Some(forwarded_for) = req - .headers() - .get(header::FORWARDED) - .and_then(|h| h.to_str().ok()) - .and_then(|h| { - let h = h.to_ascii_lowercase(); - h.split_once("for=").and_then(|(_, rest)| { - let mut start_ip = usize::MAX; - let mut end_ip = usize::MAX; - - for (pos, ch) in rest.char_indices() { - match ch { - '0'..='9' | 'a'..='f' | ':' | '.' => { - if start_ip == usize::MAX { - start_ip = pos; - } - end_ip = pos; - } - '"' | '[' | ' ' if start_ip == usize::MAX => {} - _ => { - break; +async fn handle_session<T: SessionStream>(inner: Arc<Inner>, session: SessionData<T>) { + let _in_flight = session.in_flight; + let is_tls = session.stream.is_tls(); + + if let Err(http_err) = http1::Builder::new() + .keep_alive(true) + .serve_connection( + TokioIo::new(session.stream), + service_fn(|req: hyper::Request<body::Incoming>| { + let instance = session.instance.clone(); + let inner = inner.clone(); + + async move { + let server = inner.build_server(); + + // Obtain remote IP + let remote_ip = if !server.core.jmap.http_use_forwarded { + trc::event!( + Http(trc::HttpEvent::RequestUrl), + SpanId = session.session_id, + Url = req.uri().to_string(), + ); + + session.remote_ip + } else if let Some(forwarded_for) = req + .headers() + .get(header::FORWARDED) + .and_then(|h| h.to_str().ok()) + .and_then(|h| { + let h = h.to_ascii_lowercase(); + h.split_once("for=").and_then(|(_, rest)| { + let mut start_ip = usize::MAX; + let mut end_ip = usize::MAX; + + for (pos, ch) in rest.char_indices() { + match ch { + '0'..='9' | 'a'..='f' | ':' | '.' => { + if start_ip == usize::MAX { + start_ip = pos; } + end_ip = pos; + } + '"' | '[' | ' ' if start_ip == usize::MAX => {} + _ => { + break; } } + } - rest.get(start_ip..=end_ip) - .and_then(|h| h.parse::<IpAddr>().ok()) - }) - }) - .or_else(|| { - req.headers() - .get("X-Forwarded-For") - .and_then(|h| h.to_str().ok()) - .map(|h| h.split_once(',').map_or(h, |(ip, _)| ip).trim()) + rest.get(start_ip..=end_ip) .and_then(|h| h.parse::<IpAddr>().ok()) }) - { - trc::event!( - Http(trc::HttpEvent::RequestUrl), - SpanId = session.session_id, - RemoteIp = forwarded_for, - Url = req.uri().to_string(), - ); - - forwarded_for - } else { - trc::event!( - Http(trc::HttpEvent::XForwardedMissing), - SpanId = session.session_id, - ); - session.remote_ip - }; - - // Parse HTTP request - let response = match jmap - .parse_http_request( - req, - HttpSessionData { - instance, - local_ip: session.local_ip, - local_port: session.local_port, - remote_ip, - remote_port: session.remote_port, - is_tls, - session_id: session.session_id, - }, - ) - .await - { - Ok(response) => response, - Err(err) => { - let response = err.into_http_response(); - trc::error!(err.span_id(session.session_id)); - response - } - }; + }) + .or_else(|| { + req.headers() + .get("X-Forwarded-For") + .and_then(|h| h.to_str().ok()) + .map(|h| h.split_once(',').map_or(h, |(ip, _)| ip).trim()) + .and_then(|h| h.parse::<IpAddr>().ok()) + }) + { + trc::event!( + Http(trc::HttpEvent::RequestUrl), + SpanId = session.session_id, + RemoteIp = forwarded_for, + Url = req.uri().to_string(), + ); + forwarded_for + } else { trc::event!( - Http(trc::HttpEvent::ResponseBody), + Http(trc::HttpEvent::XForwardedMissing), SpanId = session.session_id, - Contents = match &response.body { - HttpResponseBody::Text(value) => trc::Value::String(value.clone()), - HttpResponseBody::Binary(_) => trc::Value::Static("[binary data]"), - HttpResponseBody::Stream(_) => trc::Value::Static("[stream]"), - _ => trc::Value::None, - }, - Code = response.status.as_u16(), - Size = response.size(), ); + session.remote_ip + }; + + // Parse HTTP request + let response = match server + .parse_http_request( + req, + HttpSessionData { + instance, + local_ip: session.local_ip, + local_port: session.local_port, + remote_ip, + remote_port: session.remote_port, + is_tls, + session_id: session.session_id, + }, + ) + .await + { + Ok(response) => response, + Err(err) => { + let response = err.into_http_response(); + trc::error!(err.span_id(session.session_id)); + response + } + }; + + trc::event!( + Http(trc::HttpEvent::ResponseBody), + SpanId = session.session_id, + Contents = match &response.body { + HttpResponseBody::Text(value) => trc::Value::String(value.clone()), + HttpResponseBody::Binary(_) => trc::Value::Static("[binary data]"), + HttpResponseBody::Stream(_) => trc::Value::Static("[stream]"), + _ => trc::Value::None, + }, + Code = response.status.as_u16(), + Size = response.size(), + ); - // Build response - let mut response = response.build(); + // Build response + let mut response = response.build(); - // Add custom headers - if !jmap.core.jmap.http_headers.is_empty() { - let headers = response.headers_mut(); + // Add custom headers + if !server.core.jmap.http_headers.is_empty() { + let headers = response.headers_mut(); - for (header, value) in &jmap.core.jmap.http_headers { - headers.insert(header.clone(), value.clone()); - } + for (header, value) in &server.core.jmap.http_headers { + headers.insert(header.clone(), value.clone()); } - - Ok::<_, hyper::Error>(response) } - }), - ) - .with_upgrades() - .await - { - trc::event!( - Http(trc::HttpEvent::Error), - SpanId = session.session_id, - Reason = http_err.to_string(), - ); - } + + Ok::<_, hyper::Error>(response) + } + }), + ) + .with_upgrades() + .await + { + trc::event!( + Http(trc::HttpEvent::Error), + SpanId = session.session_id, + Reason = http_err.to_string(), + ); } } impl SessionManager for JmapSessionManager { - fn handle<T: SessionStream>( - self, - session: SessionData<T>, - ) -> impl std::future::Future<Output = ()> + Send { - self.inner.handle_session(session) + fn handle<T: SessionStream>(self, session: SessionData<T>) -> impl Future<Output = ()> + Send { + handle_session(self.inner, session) } #[allow(clippy::manual_async_fn)] fn shutdown(&self) -> impl std::future::Future<Output = ()> + Send { async { - let _ = self - .inner - .jmap_inner - .state_tx - .send(state::Event::Stop) - .await; + let _ = self.inner.ipc.state_tx.send(StateEvent::Stop).await; } } } @@ -629,31 +635,33 @@ impl<'x> HttpContext<'x> { Self { session, req } } - pub async fn resolve_response_url(&self, core: &Core) -> String { - core.eval_if( - &core.network.http_response_url, - self, - self.session.session_id, - ) - .await - .unwrap_or_else(|| { - format!( - "http{}://{}:{}", - if self.session.is_tls { "s" } else { "" }, - self.session.local_ip, - self.session.local_port + async fn resolve_response_url(&self, server: &Server) -> String { + server + .eval_if( + &server.core.network.http_response_url, + self, + self.session.session_id, ) - }) + .await + .unwrap_or_else(|| { + format!( + "http{}://{}:{}", + if self.session.is_tls { "s" } else { "" }, + self.session.local_ip, + self.session.local_port + ) + }) } - pub async fn has_endpoint_access(&self, core: &Core) -> StatusCode { - core.eval_if( - &core.network.http_allowed_endpoint, - self, - self.session.session_id, - ) - .await - .unwrap_or(StatusCode::OK) + async fn has_endpoint_access(&self, server: &Server) -> StatusCode { + server + .eval_if( + &server.core.network.http_allowed_endpoint, + self, + self.session.session_id, + ) + .await + .unwrap_or(StatusCode::OK) } } diff --git a/crates/jmap/src/api/management/dkim.rs b/crates/jmap/src/api/management/dkim.rs index 09e0c383..5fcbe4eb 100644 --- a/crates/jmap/src/api/management/dkim.rs +++ b/crates/jmap/src/api/management/dkim.rs @@ -6,7 +6,7 @@ use std::str::FromStr; -use common::{auth::AccessToken, config::smtp::auth::simple_pem_parse}; +use common::{auth::AccessToken, config::smtp::auth::simple_pem_parse, Server}; use directory::{backend::internal::manage, Permission}; use hyper::Method; use mail_auth::{ @@ -21,12 +21,10 @@ use serde::{Deserialize, Serialize}; use serde_json::json; use store::write::now; -use crate::{ - api::{http::ToHttpResponse, HttpRequest, HttpResponse, JsonResponse}, - JMAP, -}; +use crate::api::{http::ToHttpResponse, HttpRequest, HttpResponse, JsonResponse}; use super::decode_path_element; +use std::future::Future; #[derive(Debug, Serialize, Deserialize, Copy, Clone, PartialEq, Eq)] pub enum Algorithm { @@ -42,8 +40,36 @@ struct DkimSignature { selector: Option<String>, } -impl JMAP { - pub async fn handle_manage_dkim( +pub trait DkimManagement: Sync + Send { + fn handle_manage_dkim( + &self, + req: &HttpRequest, + path: Vec<&str>, + body: Option<Vec<u8>>, + access_token: &AccessToken, + ) -> impl Future<Output = trc::Result<HttpResponse>> + Send; + + fn handle_get_public_key( + &self, + path: Vec<&str>, + ) -> impl Future<Output = trc::Result<HttpResponse>> + Send; + + fn handle_create_signature( + &self, + body: Option<Vec<u8>>, + ) -> impl Future<Output = trc::Result<HttpResponse>> + Send; + + fn create_dkim_key( + &self, + algo: Algorithm, + id: impl AsRef<str> + Send, + domain: impl Into<String> + Send, + selector: impl Into<String> + Send, + ) -> impl Future<Output = trc::Result<()>> + Send; +} + +impl DkimManagement for Server { + async fn handle_manage_dkim( &self, req: &HttpRequest, path: Vec<&str>, diff --git a/crates/jmap/src/api/management/dns.rs b/crates/jmap/src/api/management/dns.rs index 65e56bb1..72a7a54e 100644 --- a/crates/jmap/src/api/management/dns.rs +++ b/crates/jmap/src/api/management/dns.rs @@ -4,7 +4,7 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use common::auth::AccessToken; +use common::{auth::AccessToken, Server}; use directory::{ backend::internal::manage::{self}, Permission, @@ -17,27 +17,39 @@ use sha1::Digest; use utils::config::Config; use x509_parser::parse_x509_certificate; -use crate::{ - api::{ - http::ToHttpResponse, - management::dkim::{obtain_dkim_public_key, Algorithm}, - HttpRequest, HttpResponse, JsonResponse, - }, - JMAP, +use crate::api::{ + http::ToHttpResponse, + management::dkim::{obtain_dkim_public_key, Algorithm}, + HttpRequest, HttpResponse, JsonResponse, }; use super::decode_path_element; +use std::future::Future; #[derive(Debug, Serialize, Deserialize)] -struct DnsRecord { +pub struct DnsRecord { #[serde(rename = "type")] typ: String, name: String, content: String, } -impl JMAP { - pub async fn handle_manage_dns( +pub trait DnsManagement: Sync + Send { + fn handle_manage_dns( + &self, + req: &HttpRequest, + path: Vec<&str>, + access_token: &AccessToken, + ) -> impl Future<Output = trc::Result<HttpResponse>> + Send; + + fn build_dns_records( + &self, + domain_name: &str, + ) -> impl Future<Output = trc::Result<Vec<DnsRecord>>> + Send; +} + +impl DnsManagement for Server { + async fn handle_manage_dns( &self, req: &HttpRequest, path: Vec<&str>, @@ -210,7 +222,7 @@ impl JMAP { }); // Add MTA-STS records - if let Some(policy) = self.core.build_mta_sts_policy() { + if let Some(policy) = self.build_mta_sts_policy() { records.push(DnsRecord { typ: "CNAME".to_string(), name: format!("mta-sts.{domain_name}."), @@ -239,7 +251,7 @@ impl JMAP { }); // Add TLSA records - for (name, key) in self.core.tls.certificates.load().iter() { + for (name, key) in self.inner.data.tls_certificates.load().iter() { if !name.ends_with(domain_name) || name.starts_with("mta-sts.") || name.starts_with("autoconfig.") diff --git a/crates/jmap/src/api/management/enterprise/telemetry.rs b/crates/jmap/src/api/management/enterprise/telemetry.rs index b5b06e8f..6a1ff7b2 100644 --- a/crates/jmap/src/api/management/enterprise/telemetry.rs +++ b/crates/jmap/src/api/management/enterprise/telemetry.rs @@ -19,6 +19,7 @@ use common::{ metrics::store::{Metric, MetricsStore}, tracers::store::{TracingQuery, TracingStore}, }, + Server, }; use directory::{backend::internal::manage, Permission}; use http_body_util::{combinators::BoxBody, StreamBody}; @@ -28,6 +29,7 @@ use hyper::{ }; use mail_parser::DateTime; use serde_json::json; +use std::future::Future; use store::ahash::{AHashMap, AHashSet}; use trc::{ ipc::{bitset::Bitset, subscriber::SubscriberBuilder}, @@ -41,11 +43,20 @@ use crate::{ http::ToHttpResponse, management::Timestamp, HttpRequest, HttpResponse, HttpResponseBody, JsonResponse, }, - JMAP, + auth::oauth::token::TokenHandler, }; -impl JMAP { - pub async fn handle_telemetry_api_request( +pub trait TelemetryApi: Sync + Send { + fn handle_telemetry_api_request( + &self, + req: &HttpRequest, + path: Vec<&str>, + access_token: &AccessToken, + ) -> impl Future<Output = trc::Result<HttpResponse>> + Send; +} + +impl TelemetryApi for Server { + async fn handle_telemetry_api_request( &self, req: &HttpRequest, path: Vec<&str>, @@ -455,9 +466,9 @@ impl JMAP { ] { if metric_types.contains(&metric_type) { let value = match metric_type { - MetricType::QueueCount => self.core.total_queued_messages().await?, - MetricType::UserCount => self.core.total_accounts().await?, - MetricType::DomainCount => self.core.total_domains().await?, + MetricType::QueueCount => self.total_queued_messages().await?, + MetricType::UserCount => self.total_accounts().await?, + MetricType::DomainCount => self.total_domains().await?, _ => unreachable!(), }; Collector::update_gauge(metric_type, value); diff --git a/crates/jmap/src/api/management/enterprise/undelete.rs b/crates/jmap/src/api/management/enterprise/undelete.rs index e14de9ed..e3a40cef 100644 --- a/crates/jmap/src/api/management/enterprise/undelete.rs +++ b/crates/jmap/src/api/management/enterprise/undelete.rs @@ -11,12 +11,13 @@ use std::str::FromStr; use base64::{engine::general_purpose::URL_SAFE_NO_PAD, Engine}; -use common::{auth::AccessToken, enterprise::undelete::DeletedBlob}; +use common::{auth::AccessToken, enterprise::undelete::DeletedBlob, Server}; use directory::backend::internal::manage::ManageDirectory; use hyper::Method; use jmap_proto::types::collection::Collection; use mail_parser::{DateTime, MessageParser}; use serde_json::json; +use std::future::Future; use store::write::{BatchBuilder, BlobOp, ValueClass}; use trc::AddContext; use utils::{url_params::UrlParams, BlobHash}; @@ -27,9 +28,10 @@ use crate::{ management::decode_path_element, HttpRequest, HttpResponse, JsonResponse, }, - email::ingest::{IngestEmail, IngestSource}, + blob::download::BlobDownload, + email::ingest::{EmailIngest, IngestEmail, IngestSource}, mailbox::INBOX_ID, - JMAP, + JmapMethods, }; #[derive(serde::Deserialize, serde::Serialize)] @@ -52,8 +54,18 @@ pub enum UndeleteResponse { Error { reason: String }, } -impl JMAP { - pub async fn handle_undelete_api_request( +pub trait UndeleteApi: Sync + Send { + fn handle_undelete_api_request( + &self, + req: &HttpRequest, + path: Vec<&str>, + body: Option<Vec<u8>>, + session: &HttpSessionData, + ) -> impl Future<Output = trc::Result<HttpResponse>> + Send; +} + +impl UndeleteApi for Server { + async fn handle_undelete_api_request( &self, req: &HttpRequest, path: Vec<&str>, diff --git a/crates/jmap/src/api/management/log.rs b/crates/jmap/src/api/management/log.rs index 94a87af2..ef4191e9 100644 --- a/crates/jmap/src/api/management/log.rs +++ b/crates/jmap/src/api/management/log.rs @@ -5,18 +5,16 @@ use std::{ }; use chrono::DateTime; -use common::auth::AccessToken; +use common::{auth::AccessToken, Server}; use directory::{backend::internal::manage, Permission}; use rev_lines::RevLines; use serde::Serialize; use serde_json::json; +use std::future::Future; use tokio::sync::oneshot; use utils::url_params::UrlParams; -use crate::{ - api::{http::ToHttpResponse, HttpRequest, HttpResponse, JsonResponse}, - JMAP, -}; +use crate::api::{http::ToHttpResponse, HttpRequest, HttpResponse, JsonResponse}; #[derive(Serialize)] struct LogEntry { @@ -27,8 +25,16 @@ struct LogEntry { details: String, } -impl JMAP { - pub async fn handle_view_logs( +pub trait LogManagement: Sync + Send { + fn handle_view_logs( + &self, + req: &HttpRequest, + access_token: &AccessToken, + ) -> impl Future<Output = trc::Result<HttpResponse>> + Send; +} + +impl LogManagement for Server { + async fn handle_view_logs( &self, req: &HttpRequest, access_token: &AccessToken, diff --git a/crates/jmap/src/api/management/mod.rs b/crates/jmap/src/api/management/mod.rs index 0a3dc0c2..d7a0b704 100644 --- a/crates/jmap/src/api/management/mod.rs +++ b/crates/jmap/src/api/management/mod.rs @@ -19,15 +19,28 @@ pub mod stores; use std::{borrow::Cow, str::FromStr, sync::Arc}; -use common::auth::AccessToken; +use common::{auth::AccessToken, Server}; use directory::{backend::internal::manage, Permission}; +use dkim::DkimManagement; +use dns::DnsManagement; +use enterprise::telemetry::TelemetryApi; use hyper::Method; +use log::LogManagement; use mail_parser::DateTime; +use principal::PrincipalManager; +use queue::QueueManagement; +use reload::ManageReload; +use report::ManageReports; use serde::Serialize; +use settings::ManageSettings; +use sieve::SieveHandler; use store::write::now; +use stores::ManageStore; + +use crate::{auth::oauth::auth::OAuthApiHandler, email::crypto::CryptoHandler}; use super::{http::HttpSessionData, HttpRequest, HttpResponse}; -use crate::JMAP; +use std::future::Future; #[derive(Serialize)] #[serde(tag = "error")] @@ -53,9 +66,19 @@ pub enum ManagementApiError<'x> { }, } -impl JMAP { +pub trait ManagementApi: Sync + Send { + fn handle_api_manage_request( + &self, + req: &HttpRequest, + body: Option<Vec<u8>>, + access_token: Arc<AccessToken>, + session: &HttpSessionData, + ) -> impl Future<Output = trc::Result<HttpResponse>> + Send; +} + +impl ManagementApi for Server { #[allow(unused_variables)] - pub async fn handle_api_manage_request( + async fn handle_api_manage_request( &self, req: &HttpRequest, body: Option<Vec<u8>>, diff --git a/crates/jmap/src/api/management/principal.rs b/crates/jmap/src/api/management/principal.rs index 51994756..764414de 100644 --- a/crates/jmap/src/api/management/principal.rs +++ b/crates/jmap/src/api/management/principal.rs @@ -6,7 +6,7 @@ use std::sync::{atomic::Ordering, Arc}; -use common::auth::AccessToken; +use common::{auth::AccessToken, Server}; use directory::{ backend::internal::{ lookup::DirectoryStore, @@ -21,12 +21,10 @@ use serde_json::json; use trc::AddContext; use utils::url_params::UrlParams; -use crate::{ - api::{http::ToHttpResponse, HttpRequest, HttpResponse, JsonResponse}, - JMAP, -}; +use crate::api::{http::ToHttpResponse, HttpRequest, HttpResponse, JsonResponse}; use super::decode_path_element; +use std::future::Future; #[derive(Debug, serde::Serialize, serde::Deserialize)] #[serde(tag = "type")] @@ -47,8 +45,32 @@ pub struct AccountAuthResponse { pub app_passwords: Vec<String>, } -impl JMAP { - pub async fn handle_manage_principal( +pub trait PrincipalManager: Sync + Send { + fn handle_manage_principal( + &self, + req: &HttpRequest, + path: Vec<&str>, + body: Option<Vec<u8>>, + access_token: &AccessToken, + ) -> impl Future<Output = trc::Result<HttpResponse>> + Send; + + fn handle_account_auth_get( + &self, + access_token: Arc<AccessToken>, + ) -> impl Future<Output = trc::Result<HttpResponse>> + Send; + + fn handle_account_auth_post( + &self, + req: &HttpRequest, + access_token: Arc<AccessToken>, + body: Option<Vec<u8>>, + ) -> impl Future<Output = trc::Result<HttpResponse>> + Send; + + fn assert_supported_directory(&self) -> trc::Result<()>; +} + +impl PrincipalManager for Server { + async fn handle_manage_principal( &self, req: &HttpRequest, path: Vec<&str>, @@ -297,13 +319,16 @@ impl JMAP { } // Remove entries from cache - self.inner.sessions.retain(|_, id| id.item != account_id); + self.inner + .data + .http_auth_cache + .retain(|_, id| id.item != account_id); if matches!(typ, Type::Role | Type::Tenant) { // Update permissions cache - self.core.security.permissions.clear(); - self.core - .security + self.inner.data.permissions.clear(); + self.inner + .data .permissions_version .fetch_add(1, Ordering::Relaxed); } @@ -399,20 +424,23 @@ impl JMAP { if expire_session { // Remove entries from cache - self.inner.sessions.retain(|_, id| id.item != account_id); + self.inner + .data + .http_auth_cache + .retain(|_, id| id.item != account_id); } if is_role_change { // Update permissions cache - self.core.security.permissions.clear(); - self.core - .security + self.inner.data.permissions.clear(); + self.inner + .data .permissions_version .fetch_add(1, Ordering::Relaxed); } if expire_token { - self.core.security.access_tokens.remove(&account_id); + self.inner.data.access_tokens.remove(&account_id); } Ok(JsonResponse::new(json!({ @@ -428,7 +456,7 @@ impl JMAP { } } - pub async fn handle_account_auth_get( + async fn handle_account_auth_get( &self, access_token: Arc<AccessToken>, ) -> trc::Result<HttpResponse> { @@ -463,7 +491,7 @@ impl JMAP { .into_http_response()) } - pub async fn handle_account_auth_post( + async fn handle_account_auth_post( &self, req: &HttpRequest, access_token: Arc<AccessToken>, @@ -513,7 +541,10 @@ impl JMAP { .await?; // Remove entries from cache - self.inner.sessions.retain(|_, id| id.item != u32::MAX); + self.inner + .data + .http_auth_cache + .retain(|_, id| id.item != u32::MAX); return Ok(JsonResponse::new(json!({ "data": (), @@ -578,7 +609,8 @@ impl JMAP { // Remove entries from cache self.inner - .sessions + .data + .http_auth_cache .retain(|_, id| id.item != access_token.primary_id()); Ok(JsonResponse::new(json!({ @@ -587,7 +619,7 @@ impl JMAP { .into_http_response()) } - pub fn assert_supported_directory(&self) -> trc::Result<()> { + fn assert_supported_directory(&self) -> trc::Result<()> { let class = match &self.core.storage.directory.store { DirectoryInner::Internal(_) => return Ok(()), DirectoryInner::Ldap(_) => "LDAP", diff --git a/crates/jmap/src/api/management/queue.rs b/crates/jmap/src/api/management/queue.rs index 596abf65..c19bc707 100644 --- a/crates/jmap/src/api/management/queue.rs +++ b/crates/jmap/src/api/management/queue.rs @@ -4,8 +4,10 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ +use std::future::Future; + use base64::{engine::general_purpose::URL_SAFE_NO_PAD, Engine}; -use common::auth::AccessToken; +use common::{auth::AccessToken, ipc::QueueEvent, Server}; use directory::{ backend::internal::{manage::ManageDirectory, PrincipalField}, Permission, Type, @@ -19,7 +21,10 @@ use mail_auth::{ use mail_parser::DateTime; use serde::{Deserializer, Serializer}; use serde_json::json; -use smtp::queue::{self, ErrorDetails, HostResponse, QueueId, Status}; +use smtp::{ + queue::{self, spool::SmtpSpool, ErrorDetails, HostResponse, QueueId, Status}, + reporting::{dmarc::DmarcReporting, tls::TlsReporting}, +}; use store::{ write::{key::DeserializeBigEndian, now, Bincode, QueueClass, ReportEvent, ValueClass}, Deserialize, IterateParams, ValueKey, @@ -27,10 +32,7 @@ use store::{ use trc::AddContext; use utils::url_params::UrlParams; -use crate::{ - api::{http::ToHttpResponse, HttpRequest, HttpResponse, JsonResponse}, - JMAP, -}; +use crate::api::{http::ToHttpResponse, HttpRequest, HttpResponse, JsonResponse}; use super::{decode_path_element, FutureTimestamp}; @@ -106,8 +108,17 @@ pub enum Report { }, } -impl JMAP { - pub async fn handle_manage_queue( +pub trait QueueManagement: Sync + Send { + fn handle_manage_queue( + &self, + req: &HttpRequest, + path: Vec<&str>, + access_token: &AccessToken, + ) -> impl Future<Output = trc::Result<HttpResponse>> + Send; +} + +impl QueueManagement for Server { + async fn handle_manage_queue( &self, req: &HttpRequest, path: Vec<&str>, @@ -270,7 +281,6 @@ impl JMAP { access_token.assert_has_permission(Permission::MessageQueueGet)?; if let Some(message) = self - .smtp .read_message(queue_id.parse().unwrap_or_default()) .await .filter(|message| { @@ -298,7 +308,6 @@ impl JMAP { let item = params.get("filter"); if let Some(mut message) = self - .smtp .read_message(queue_id.parse().unwrap_or_default()) .await .filter(|message| { @@ -329,9 +338,9 @@ impl JMAP { if found { let next_event = message.next_event().unwrap_or_default(); message - .save_changes(&self.smtp, prev_event.into(), next_event.into()) + .save_changes(self, prev_event.into(), next_event.into()) .await; - let _ = self.smtp.inner.queue_tx.send(queue::Event::Reload).await; + let _ = self.inner.ipc.queue_tx.send(QueueEvent::Reload).await; } Ok(JsonResponse::new(json!({ @@ -347,7 +356,6 @@ impl JMAP { access_token.assert_has_permission(Permission::MessageQueueDelete)?; if let Some(mut message) = self - .smtp .read_message(queue_id.parse().unwrap_or_default()) .await .filter(|message| { @@ -411,14 +419,14 @@ impl JMAP { }) { let next_event = message.next_event().unwrap_or_default(); message - .save_changes(&self.smtp, next_event.into(), prev_event.into()) + .save_changes(self, next_event.into(), prev_event.into()) .await; } else { - message.remove(&self.smtp, prev_event).await; + message.remove(self, prev_event).await; } } } else { - message.remove(&self.smtp, prev_event).await; + message.remove(self, prev_event).await; found = true; } @@ -528,7 +536,6 @@ impl JMAP { { let mut rua = Vec::new(); if let Some(report) = self - .smtp .generate_dmarc_aggregate_report(&event, &mut rua, None, 0) .await? { @@ -542,7 +549,6 @@ impl JMAP { { let mut rua = Vec::new(); if let Some(report) = self - .smtp .generate_tls_aggregate_report(&[event.clone()], &mut rua, None, 0) .await? { @@ -573,7 +579,7 @@ impl JMAP { .as_ref() .map_or(true, |domains| domains.contains(&event.domain)) => { - self.smtp.delete_dmarc_report(event).await; + self.delete_dmarc_report(event).await; true } QueueClass::TlsReportHeader(event) @@ -581,7 +587,7 @@ impl JMAP { .as_ref() .map_or(true, |domains| domains.contains(&event.domain)) => { - self.smtp.delete_tls_report(vec![event]).await; + self.delete_tls_report(vec![event]).await; true } _ => false, diff --git a/crates/jmap/src/api/management/reload.rs b/crates/jmap/src/api/management/reload.rs index f396e1b6..4c4fe110 100644 --- a/crates/jmap/src/api/management/reload.rs +++ b/crates/jmap/src/api/management/reload.rs @@ -4,20 +4,36 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use common::auth::AccessToken; +use common::{auth::AccessToken, ipc::HousekeeperEvent, Server}; use directory::Permission; use hyper::Method; use serde_json::json; +use std::future::Future; use utils::url_params::UrlParams; use crate::{ api::{http::ToHttpResponse, HttpRequest, HttpResponse, JsonResponse}, - services::housekeeper::Event, - JMAP, + JmapMethods, }; -impl JMAP { - pub async fn handle_manage_reload( +pub trait ManageReload: Sync + Send { + fn handle_manage_reload( + &self, + req: &HttpRequest, + path: Vec<&str>, + access_token: &AccessToken, + ) -> impl Future<Output = trc::Result<HttpResponse>> + Send; + + fn handle_manage_update( + &self, + req: &HttpRequest, + path: Vec<&str>, + access_token: &AccessToken, + ) -> impl Future<Output = trc::Result<HttpResponse>> + Send; +} + +impl ManageReload for Server { + async fn handle_manage_reload( &self, req: &HttpRequest, path: Vec<&str>, @@ -28,10 +44,10 @@ impl JMAP { match (path.get(1).copied(), req.method()) { (Some("lookup"), &Method::GET) => { - let result = self.core.reload_lookups().await?; + let result = self.reload_lookups().await?; // Update core if let Some(core) = result.new_core { - self.shared_core.store(core.into()); + self.inner.shared_core.store(core.into()); } Ok(JsonResponse::new(json!({ @@ -40,13 +56,14 @@ impl JMAP { .into_http_response()) } (Some("certificate"), &Method::GET) => Ok(JsonResponse::new(json!({ - "data": self.core.reload_certificates().await?.config, + "data": self.reload_certificates().await?.config, })) .into_http_response()), (Some("server.blocked-ip"), &Method::GET) => { - let result = self.core.reload_blocked_ips().await?; + let result = self.reload_blocked_ips().await?; + // Increment version counter - self.core.network.blocked_ips.increment_version(); + self.increment_blocked_version(); Ok(JsonResponse::new(json!({ "data": result.config, @@ -54,28 +71,29 @@ impl JMAP { .into_http_response()) } (_, &Method::GET) => { - let result = self.core.reload().await?; + let result = self.reload().await?; if !UrlParams::new(req.uri().query()).has_key("dry-run") { if let Some(core) = result.new_core { // Update core - self.shared_core.store(core.into()); + self.inner.shared_core.store(core.into()); // Increment version counter - self.inner.increment_config_version(); + self.increment_config_version(); } if let Some(tracers) = result.tracers { // Update tracers #[cfg(feature = "enterprise")] - tracers.update(self.shared_core.load().is_enterprise_edition()); + tracers.update(self.inner.shared_core.load().is_enterprise_edition()); #[cfg(not(feature = "enterprise"))] tracers.update(false); } // Reload settings self.inner + .ipc .housekeeper_tx - .send(Event::ReloadSettings) + .send(HousekeeperEvent::ReloadSettings) .await .map_err(|err| { trc::EventType::Server(trc::ServerEvent::ThreadError) @@ -94,7 +112,7 @@ impl JMAP { } } - pub async fn handle_manage_update( + async fn handle_manage_update( &self, req: &HttpRequest, path: Vec<&str>, @@ -119,7 +137,11 @@ impl JMAP { // Validate the access token access_token.assert_has_permission(Permission::UpdateWebadmin)?; - self.inner.webadmin.update_and_unpack(&self.core).await?; + self.inner + .data + .webadmin + .update_and_unpack(&self.core) + .await?; Ok(JsonResponse::new(json!({ "data": (), diff --git a/crates/jmap/src/api/management/report.rs b/crates/jmap/src/api/management/report.rs index daa91451..5540d41b 100644 --- a/crates/jmap/src/api/management/report.rs +++ b/crates/jmap/src/api/management/report.rs @@ -4,7 +4,9 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use common::auth::AccessToken; +use std::future::Future; + +use common::{auth::AccessToken, Server}; use directory::{ backend::internal::{manage::ManageDirectory, PrincipalField}, Permission, Type, @@ -23,10 +25,7 @@ use store::{ use trc::AddContext; use utils::url_params::UrlParams; -use crate::{ - api::{http::ToHttpResponse, HttpRequest, HttpResponse, JsonResponse}, - JMAP, -}; +use crate::api::{http::ToHttpResponse, HttpRequest, HttpResponse, JsonResponse}; use super::decode_path_element; @@ -36,8 +35,17 @@ enum ReportType { Arf, } -impl JMAP { - pub async fn handle_manage_reports( +pub trait ManageReports: Sync + Send { + fn handle_manage_reports( + &self, + req: &HttpRequest, + path: Vec<&str>, + access_token: &AccessToken, + ) -> impl Future<Output = trc::Result<HttpResponse>> + Send; +} + +impl ManageReports for Server { + async fn handle_manage_reports( &self, req: &HttpRequest, path: Vec<&str>, diff --git a/crates/jmap/src/api/management/settings.rs b/crates/jmap/src/api/management/settings.rs index 6844c503..ccabef51 100644 --- a/crates/jmap/src/api/management/settings.rs +++ b/crates/jmap/src/api/management/settings.rs @@ -4,19 +4,17 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use common::auth::AccessToken; +use common::{auth::AccessToken, Server}; use directory::Permission; use hyper::Method; use serde_json::json; use store::ahash::AHashMap; use utils::{config::ConfigKey, map::vec_map::VecMap, url_params::UrlParams}; -use crate::{ - api::{http::ToHttpResponse, HttpRequest, HttpResponse, JsonResponse}, - JMAP, -}; +use crate::api::{http::ToHttpResponse, HttpRequest, HttpResponse, JsonResponse}; use super::decode_path_element; +use std::future::Future; #[derive(Debug, serde::Serialize, serde::Deserialize)] #[serde(tag = "type")] @@ -34,8 +32,18 @@ pub enum UpdateSettings { }, } -impl JMAP { - pub async fn handle_manage_settings( +pub trait ManageSettings: Sync + Send { + fn handle_manage_settings( + &self, + req: &HttpRequest, + path: Vec<&str>, + body: Option<Vec<u8>>, + access_token: &AccessToken, + ) -> impl Future<Output = trc::Result<HttpResponse>> + Send; +} + +impl ManageSettings for Server { + async fn handle_manage_settings( &self, req: &HttpRequest, path: Vec<&str>, diff --git a/crates/jmap/src/api/management/sieve.rs b/crates/jmap/src/api/management/sieve.rs index eb918185..85159378 100644 --- a/crates/jmap/src/api/management/sieve.rs +++ b/crates/jmap/src/api/management/sieve.rs @@ -6,18 +6,16 @@ use std::time::SystemTime; -use common::{auth::AccessToken, scripts::ScriptModification, IntoString}; +use common::{auth::AccessToken, scripts::ScriptModification, IntoString, Server}; use directory::Permission; use hyper::Method; use serde_json::json; use sieve::{runtime::Variable, Envelope}; -use smtp::scripts::{ScriptParameters, ScriptResult}; +use smtp::scripts::{event_loop::RunScript, ScriptParameters, ScriptResult}; +use std::future::Future; use utils::url_params::UrlParams; -use crate::{ - api::{http::ToHttpResponse, HttpRequest, HttpResponse, JsonResponse}, - JMAP, -}; +use crate::api::{http::ToHttpResponse, HttpRequest, HttpResponse, JsonResponse}; #[derive(Debug, serde::Serialize)] #[serde(tag = "action")] @@ -36,8 +34,18 @@ pub enum Response { Discard, } -impl JMAP { - pub async fn handle_run_sieve( +pub trait SieveHandler: Sync + Send { + fn handle_run_sieve( + &self, + req: &HttpRequest, + path: Vec<&str>, + body: Option<Vec<u8>>, + access_token: &AccessToken, + ) -> impl Future<Output = trc::Result<HttpResponse>> + Send; +} + +impl SieveHandler for Server { + async fn handle_run_sieve( &self, req: &HttpRequest, path: Vec<&str>, @@ -103,7 +111,7 @@ impl JMAP { } // Run script - let result = match self.smtp.run_script(script_id, script, params, 0).await { + let result = match self.run_script(script_id, script, params, 0).await { ScriptResult::Accept { modifications } => Response::Accept { modifications }, ScriptResult::Replace { message, diff --git a/crates/jmap/src/api/management/stores.rs b/crates/jmap/src/api/management/stores.rs index b86c88be..0b9dea48 100644 --- a/crates/jmap/src/api/management/stores.rs +++ b/crates/jmap/src/api/management/stores.rs @@ -5,7 +5,12 @@ */ use base64::{engine::general_purpose::URL_SAFE_NO_PAD, Engine}; -use common::{auth::AccessToken, manager::webadmin::Resource}; +use common::{ + auth::AccessToken, + ipc::{HousekeeperEvent, PurgeType}, + manager::webadmin::Resource, + Server, +}; use directory::{ backend::internal::manage::{self, ManageDirectory}, Permission, @@ -19,14 +24,30 @@ use crate::{ http::{HttpSessionData, ToHttpResponse}, HttpRequest, HttpResponse, JsonResponse, }, - services::housekeeper::{Event, PurgeType}, - JMAP, + services::index::Indexer, }; -use super::decode_path_element; +use super::{decode_path_element, enterprise::undelete::UndeleteApi}; +use std::future::Future; + +pub trait ManageStore: Sync + Send { + fn handle_manage_store( + &self, + req: &HttpRequest, + path: Vec<&str>, + body: Option<Vec<u8>>, + session: &HttpSessionData, + access_token: &AccessToken, + ) -> impl Future<Output = trc::Result<HttpResponse>> + Send; + + fn housekeeper_request( + &self, + event: HousekeeperEvent, + ) -> impl Future<Output = trc::Result<HttpResponse>> + Send; +} -impl JMAP { - pub async fn handle_manage_store( +impl ManageStore for Server { + async fn handle_manage_store( &self, req: &HttpRequest, path: Vec<&str>, @@ -75,7 +96,7 @@ impl JMAP { // Validate the access token access_token.assert_has_permission(Permission::PurgeBlobStore)?; - self.housekeeper_request(Event::Purge(PurgeType::Blobs { + self.housekeeper_request(HousekeeperEvent::Purge(PurgeType::Blobs { store: self.core.storage.data.clone(), blob_store: self.core.storage.blob.clone(), })) @@ -95,7 +116,7 @@ impl JMAP { self.core.storage.data.clone() }; - self.housekeeper_request(Event::Purge(PurgeType::Data(store))) + self.housekeeper_request(HousekeeperEvent::Purge(PurgeType::Data(store))) .await } (Some("purge"), Some("lookup"), id, &Method::GET) => { @@ -112,7 +133,7 @@ impl JMAP { self.core.storage.lookup.clone() }; - self.housekeeper_request(Event::Purge(PurgeType::Lookup(store))) + self.housekeeper_request(HousekeeperEvent::Purge(PurgeType::Lookup(store))) .await } (Some("purge"), Some("account"), id, &Method::GET) => { @@ -131,7 +152,7 @@ impl JMAP { None }; - self.housekeeper_request(Event::Purge(PurgeType::Account(account_id))) + self.housekeeper_request(HousekeeperEvent::Purge(PurgeType::Account(account_id))) .await } (Some("reindex"), id, None, &Method::GET) => { @@ -192,12 +213,17 @@ impl JMAP { } } - async fn housekeeper_request(&self, event: Event) -> trc::Result<HttpResponse> { - self.inner.housekeeper_tx.send(event).await.map_err(|err| { - trc::EventType::Server(trc::ServerEvent::ThreadError) - .reason(err) - .details("Failed to send housekeeper event") - })?; + async fn housekeeper_request(&self, event: HousekeeperEvent) -> trc::Result<HttpResponse> { + self.inner + .ipc + .housekeeper_tx + .send(event) + .await + .map_err(|err| { + trc::EventType::Server(trc::ServerEvent::ThreadError) + .reason(err) + .details("Failed to send housekeeper event") + })?; Ok(JsonResponse::new(json!({ "data": (), diff --git a/crates/jmap/src/api/mod.rs b/crates/jmap/src/api/mod.rs index 2d9fee0e..0ee7ef52 100644 --- a/crates/jmap/src/api/mod.rs +++ b/crates/jmap/src/api/mod.rs @@ -4,15 +4,14 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use std::borrow::Cow; +use std::{borrow::Cow, sync::Arc}; +use common::Inner; use hyper::StatusCode; use jmap_proto::types::{id::Id, state::State, type_state::DataType}; use serde::Serialize; use utils::map::vec_map::VecMap; -use crate::JmapInstance; - pub mod autoconfig; pub mod event_source; pub mod http; @@ -22,11 +21,11 @@ pub mod session; #[derive(Clone)] pub struct JmapSessionManager { - pub inner: JmapInstance, + pub inner: Arc<Inner>, } impl JmapSessionManager { - pub fn new(inner: JmapInstance) -> Self { + pub fn new(inner: Arc<Inner>) -> Self { Self { inner } } } diff --git a/crates/jmap/src/api/request.rs b/crates/jmap/src/api/request.rs index 6d81b480..4b2a184a 100644 --- a/crates/jmap/src/api/request.rs +++ b/crates/jmap/src/api/request.rs @@ -6,7 +6,7 @@ use std::{sync::Arc, time::Instant}; -use common::auth::AccessToken; +use common::{auth::AccessToken, Server}; use jmap_proto::{ method::{ get, query, @@ -18,12 +18,51 @@ use jmap_proto::{ }; use trc::JmapEvent; -use crate::JMAP; +use crate::{ + blob::{copy::BlobCopy, get::BlobOperations, upload::BlobUpload}, + changes::{get::ChangesLookup, query::QueryChanges}, + email::{ + copy::EmailCopy, get::EmailGet, import::EmailImport, parse::EmailParse, query::EmailQuery, + set::EmailSet, snippet::EmailSearchSnippet, + }, + identity::{get::IdentityGet, set::IdentitySet}, + mailbox::{get::MailboxGet, query::MailboxQuery, set::MailboxSet}, + principal::{get::PrincipalGet, query::PrincipalQuery}, + push::{get::PushSubscriptionFetch, set::PushSubscriptionSet}, + quota::{get::QuotaGet, query::QuotaQuery}, + services::state::StateManager, + sieve::{ + get::SieveScriptGet, query::SieveScriptQuery, set::SieveScriptSet, + validate::SieveScriptValidate, + }, + submission::{get::EmailSubmissionGet, query::EmailSubmissionQuery, set::EmailSubmissionSet}, + thread::get::ThreadGet, + vacation::{get::VacationResponseGet, set::VacationResponseSet}, +}; use super::http::HttpSessionData; +use std::future::Future; + +pub trait RequestHandler: Sync + Send { + fn handle_request( + &self, + request: Request, + access_token: Arc<AccessToken>, + session: &HttpSessionData, + ) -> impl Future<Output = Response> + Send; + + fn handle_method_call( + &self, + method: RequestMethod, + method_name: &'static str, + access_token: &AccessToken, + next_call: &mut Option<Call<RequestMethod>>, + session: &HttpSessionData, + ) -> impl Future<Output = trc::Result<ResponseMethod>> + Send; +} -impl JMAP { - pub async fn handle_request( +impl RequestHandler for Server { + async fn handle_request( &self, request: Request, access_token: Arc<AccessToken>, diff --git a/crates/jmap/src/api/session.rs b/crates/jmap/src/api/session.rs index 1480487f..3c94a334 100644 --- a/crates/jmap/src/api/session.rs +++ b/crates/jmap/src/api/session.rs @@ -6,18 +6,27 @@ use std::sync::Arc; -use common::auth::AccessToken; +use common::{auth::AccessToken, Server}; use directory::{backend::internal::PrincipalField, QueryBy}; use jmap_proto::{ request::capability::{Capability, Session}, types::{acl::Acl, collection::Collection, id::Id}, }; +use std::future::Future; use trc::AddContext; -use crate::JMAP; +use crate::auth::acl::AclMethods; -impl JMAP { - pub async fn handle_session_resource( +pub trait SessionHandler: Sync + Send { + fn handle_session_resource( + &self, + base_url: String, + access_token: Arc<AccessToken>, + ) -> impl Future<Output = trc::Result<Session>> + Send; +} + +impl SessionHandler for Server { + async fn handle_session_resource( &self, base_url: String, access_token: Arc<AccessToken>, diff --git a/crates/jmap/src/auth/acl.rs b/crates/jmap/src/auth/acl.rs index c574da61..e7a40117 100644 --- a/crates/jmap/src/auth/acl.rs +++ b/crates/jmap/src/auth/acl.rs @@ -4,7 +4,9 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use common::auth::AccessToken; +use std::future::Future; + +use common::{auth::AccessToken, Server}; use directory::{backend::internal::PrincipalField, QueryBy}; use jmap_proto::{ error::set::SetError, @@ -25,10 +27,77 @@ use store::{ use trc::AddContext; use utils::map::bitmap::Bitmap; -use crate::JMAP; +use crate::JmapMethods; + +pub trait AclMethods: Sync + Send { + fn shared_documents( + &self, + access_token: &AccessToken, + to_account_id: u32, + to_collection: Collection, + check_acls: impl Into<Bitmap<Acl>> + Send, + ) -> impl Future<Output = trc::Result<RoaringBitmap>> + Send; + + fn shared_messages( + &self, + access_token: &AccessToken, + to_account_id: u32, + check_acls: impl Into<Bitmap<Acl>> + Send, + ) -> impl Future<Output = trc::Result<RoaringBitmap>> + Send; + + fn owned_or_shared_documents( + &self, + access_token: &AccessToken, + account_id: u32, + collection: Collection, + check_acls: impl Into<Bitmap<Acl>> + Send, + ) -> impl Future<Output = trc::Result<RoaringBitmap>> + Send; + + fn owned_or_shared_messages( + &self, + access_token: &AccessToken, + account_id: u32, + check_acls: impl Into<Bitmap<Acl>> + Send, + ) -> impl Future<Output = trc::Result<RoaringBitmap>> + Send; + + fn has_access_to_document( + &self, + access_token: &AccessToken, + to_account_id: u32, + to_collection: impl Into<u8> + Send, + to_document_id: u32, + check_acls: impl Into<Bitmap<Acl>> + Send, + ) -> impl Future<Output = trc::Result<bool>> + Send; + + fn acl_set( + &self, + changes: &mut Object<Value>, + current: Option<&HashedValue<Object<Value>>>, + acl_changes: MaybePatchValue, + ) -> impl Future<Output = Result<(), SetError>> + Send; -impl JMAP { - pub async fn shared_documents( + fn acl_get( + &self, + value: &[AclGrant], + access_token: &AccessToken, + account_id: u32, + ) -> impl Future<Output = Value> + Send; + + fn refresh_acls(&self, changes: &Object<Value>, current: &Option<HashedValue<Object<Value>>>); + + fn map_acl_set( + &self, + acl_set: Vec<Value>, + ) -> impl Future<Output = Result<Vec<AclGrant>, SetError>> + Send; + + fn map_acl_patch( + &self, + acl_patch: Vec<Value>, + ) -> impl Future<Output = Result<(AclGrant, Option<bool>), SetError>> + Send; +} + +impl AclMethods for Server { + async fn shared_documents( &self, access_token: &AccessToken, to_account_id: u32, @@ -66,7 +135,7 @@ impl JMAP { Ok(document_ids) } - pub async fn shared_messages( + async fn shared_messages( &self, access_token: &AccessToken, to_account_id: u32, @@ -97,7 +166,7 @@ impl JMAP { Ok(shared_messages) } - pub async fn owned_or_shared_documents( + async fn owned_or_shared_documents( &self, access_token: &AccessToken, account_id: u32, @@ -117,7 +186,7 @@ impl JMAP { Ok(document_ids) } - pub async fn owned_or_shared_messages( + async fn owned_or_shared_messages( &self, access_token: &AccessToken, account_id: u32, @@ -136,7 +205,7 @@ impl JMAP { Ok(document_ids) } - pub async fn has_access_to_document( + async fn has_access_to_document( &self, access_token: &AccessToken, to_account_id: u32, @@ -179,7 +248,7 @@ impl JMAP { Ok(false) } - pub async fn acl_set( + async fn acl_set( &self, changes: &mut Object<Value>, current: Option<&HashedValue<Object<Value>>>, @@ -251,7 +320,7 @@ impl JMAP { Ok(()) } - pub async fn acl_get( + async fn acl_get( &self, value: &[AclGrant], access_token: &AccessToken, @@ -287,13 +356,9 @@ impl JMAP { } } - pub fn refresh_acls( - &self, - changes: &Object<Value>, - current: &Option<HashedValue<Object<Value>>>, - ) { + fn refresh_acls(&self, changes: &Object<Value>, current: &Option<HashedValue<Object<Value>>>) { if let Value::Acl(acl_changes) = changes.get(&Property::Acl) { - let access_tokens = &self.core.security.access_tokens; + let access_tokens = &self.inner.data.access_tokens; if let Some(Value::Acl(acl_current)) = current .as_ref() .and_then(|current| current.inner.properties.get(&Property::Acl)) diff --git a/crates/jmap/src/auth/authenticate.rs b/crates/jmap/src/auth/authenticate.rs index 9bb5cc2d..8009ab6d 100644 --- a/crates/jmap/src/auth/authenticate.rs +++ b/crates/jmap/src/auth/authenticate.rs @@ -6,82 +6,101 @@ use std::{net::IpAddr, sync::Arc, time::Instant}; -use common::listener::limiter::InFlight; +use common::{listener::limiter::InFlight, Server}; use directory::Permission; use hyper::header; use mail_parser::decoders::base64::base64_decode; use mail_send::Credentials; use utils::map::ttl_dashmap::TtlMap; -use crate::{ - api::{http::HttpSessionData, HttpRequest}, - JMAP, -}; +use crate::api::{http::HttpSessionData, HttpRequest}; use common::auth::AccessToken; +use std::future::Future; -impl JMAP { - pub async fn authenticate_headers( +use super::{oauth::token::TokenHandler, rate_limit::RateLimiter}; + +pub trait Authenticator: Sync + Send { + fn authenticate_headers( + &self, + req: &HttpRequest, + session: &HttpSessionData, + ) -> impl Future<Output = trc::Result<(InFlight, Arc<AccessToken>)>> + Send; + + fn cache_session(&self, session_id: String, access_token: &AccessToken); + + fn authenticate_plain( + &self, + username: &str, + secret: &str, + remote_ip: IpAddr, + session_id: u64, + ) -> impl Future<Output = trc::Result<AccessToken>> + Send; +} + +impl Authenticator for Server { + async fn authenticate_headers( &self, req: &HttpRequest, session: &HttpSessionData, ) -> trc::Result<(InFlight, Arc<AccessToken>)> { if let Some((mechanism, token)) = req.authorization() { - let access_token = if let Some(account_id) = self.inner.sessions.get_with_ttl(token) { - self.core.get_cached_access_token(account_id).await? - } else { - let access_token = if mechanism.eq_ignore_ascii_case("basic") { - // Enforce rate limit for authentication requests - self.is_auth_allowed_soft(&session.remote_ip).await?; - - // Decode the base64 encoded credentials - if let Some((account, secret)) = base64_decode(token.as_bytes()) - .and_then(|token| String::from_utf8(token).ok()) - .and_then(|token| { - token.split_once(':').map(|(login, secret)| { - (login.trim().to_lowercase(), secret.to_string()) + let access_token = + if let Some(account_id) = self.inner.data.http_auth_cache.get_with_ttl(token) { + self.get_cached_access_token(account_id).await? + } else { + let access_token = if mechanism.eq_ignore_ascii_case("basic") { + // Enforce rate limit for authentication requests + self.is_auth_allowed_soft(&session.remote_ip).await?; + + // Decode the base64 encoded credentials + if let Some((account, secret)) = base64_decode(token.as_bytes()) + .and_then(|token| String::from_utf8(token).ok()) + .and_then(|token| { + token.split_once(':').map(|(login, secret)| { + (login.trim().to_lowercase(), secret.to_string()) + }) }) - }) - { - self.authenticate_plain( - &account, - &secret, - session.remote_ip, - session.session_id, - ) - .await? + { + self.authenticate_plain( + &account, + &secret, + session.remote_ip, + session.session_id, + ) + .await? + } else { + return Err(trc::AuthEvent::Error + .into_err() + .details("Failed to decode Basic auth request.") + .id(token.to_string()) + .caused_by(trc::location!())); + } + } else if mechanism.eq_ignore_ascii_case("bearer") { + // Enforce anonymous rate limit for bearer auth requests + self.is_anonymous_allowed(&session.remote_ip).await?; + + let (account_id, _, _) = + self.validate_access_token("access_token", token).await?; + + self.get_access_token(account_id).await? } else { + // Enforce anonymous rate limit + self.is_anonymous_allowed(&session.remote_ip).await?; return Err(trc::AuthEvent::Error .into_err() - .details("Failed to decode Basic auth request.") - .id(token.to_string()) + .reason("Unsupported authentication mechanism.") + .details(token.to_string()) .caused_by(trc::location!())); - } - } else if mechanism.eq_ignore_ascii_case("bearer") { - // Enforce anonymous rate limit for bearer auth requests - self.is_anonymous_allowed(&session.remote_ip).await?; + }; - let (account_id, _, _) = - self.validate_access_token("access_token", token).await?; - - self.core.get_access_token(account_id).await? - } else { - // Enforce anonymous rate limit - self.is_anonymous_allowed(&session.remote_ip).await?; - return Err(trc::AuthEvent::Error - .into_err() - .reason("Unsupported authentication mechanism.") - .details(token.to_string()) - .caused_by(trc::location!())); + // Cache session + let access_token = Arc::new(access_token); + self.cache_session(token.to_string(), &access_token); + self.cache_access_token(access_token.clone()); + access_token }; - // Cache session - let access_token = Arc::new(access_token); - self.cache_session(token.to_string(), &access_token); - self.core.cache_access_token(access_token.clone()); - access_token - }; - // Enforce authenticated rate limit self.is_account_allowed(&access_token) .await @@ -97,15 +116,15 @@ impl JMAP { } } - pub fn cache_session(&self, session_id: String, access_token: &AccessToken) { - self.inner.sessions.insert_with_ttl( + fn cache_session(&self, session_id: String, access_token: &AccessToken) { + self.inner.data.http_auth_cache.insert_with_ttl( session_id, access_token.primary_id(), Instant::now() + self.core.jmap.session_cache_ttl, ); } - pub async fn authenticate_plain( + async fn authenticate_plain( &self, username: &str, secret: &str, @@ -113,7 +132,6 @@ impl JMAP { session_id: u64, ) -> trc::Result<AccessToken> { match self - .core .authenticate( &self.core.storage.directory, session_id, @@ -126,15 +144,11 @@ impl JMAP { ) .await { - Ok(principal) => self - .core - .build_access_token(principal) - .await - .and_then(|token| { - token - .assert_has_permission(Permission::Authenticate) - .map(|_| token) - }), + Ok(principal) => self.build_access_token(principal).await.and_then(|token| { + token + .assert_has_permission(Permission::Authenticate) + .map(|_| token) + }), Err(err) => { if !err.matches(trc::EventType::Auth(trc::AuthEvent::MissingTotp)) { let _ = self.is_auth_allowed_hard(&remote_ip).await; diff --git a/crates/jmap/src/auth/oauth/auth.rs b/crates/jmap/src/auth/oauth/auth.rs index 78f53454..4274dc17 100644 --- a/crates/jmap/src/auth/oauth/auth.rs +++ b/crates/jmap/src/auth/oauth/auth.rs @@ -6,9 +6,10 @@ use std::sync::Arc; -use common::auth::AccessToken; +use common::{auth::AccessToken, Server}; use rand::distributions::Standard; use serde_json::json; +use std::future::Future; use store::{ rand::{distributions::Alphanumeric, thread_rng, Rng}, write::Bincode, @@ -18,7 +19,6 @@ use store::{ use crate::{ api::{http::ToHttpResponse, HttpRequest, HttpResponse, JsonResponse}, auth::oauth::OAuthStatus, - JMAP, }; use super::{ @@ -26,8 +26,23 @@ use super::{ MAX_POST_LEN, USER_CODE_ALPHABET, USER_CODE_LEN, }; -impl JMAP { - pub async fn handle_oauth_api_request( +pub trait OAuthApiHandler: Sync + Send { + fn handle_oauth_api_request( + &self, + access_token: Arc<AccessToken>, + body: Option<Vec<u8>>, + ) -> impl Future<Output = trc::Result<HttpResponse>> + Send; + + fn handle_device_auth( + &self, + req: &mut HttpRequest, + base_url: impl AsRef<str> + Send, + session_id: u64, + ) -> impl Future<Output = trc::Result<HttpResponse>> + Send; +} + +impl OAuthApiHandler for Server { + async fn handle_oauth_api_request( &self, access_token: Arc<AccessToken>, body: Option<Vec<u8>>, @@ -143,7 +158,7 @@ impl JMAP { Ok(JsonResponse::new(response).into_http_response()) } - pub async fn handle_device_auth( + async fn handle_device_auth( &self, req: &mut HttpRequest, base_url: impl AsRef<str>, diff --git a/crates/jmap/src/auth/oauth/mod.rs b/crates/jmap/src/auth/oauth/mod.rs index 0a83063a..7c9e7337 100644 --- a/crates/jmap/src/auth/oauth/mod.rs +++ b/crates/jmap/src/auth/oauth/mod.rs @@ -202,7 +202,7 @@ pub struct FormData { } impl FormData { - pub async fn from_request( + async fn from_request( req: &mut HttpRequest, max_len: usize, session_id: u64, diff --git a/crates/jmap/src/auth/oauth/token.rs b/crates/jmap/src/auth/oauth/token.rs index 590a9c4d..ec23b3c4 100644 --- a/crates/jmap/src/auth/oauth/token.rs +++ b/crates/jmap/src/auth/oauth/token.rs @@ -6,10 +6,12 @@ use std::time::SystemTime; +use common::Server; use directory::{backend::internal::PrincipalField, QueryBy}; use hyper::StatusCode; use mail_builder::encoders::base64::base64_encode; use mail_parser::decoders::base64::base64_decode; +use std::future::Future; use store::{ blake3, rand::{thread_rng, Rng}, @@ -20,7 +22,6 @@ use utils::codec::leb128::{Leb128Iterator, Leb128Vec}; use crate::{ api::{http::ToHttpResponse, HttpRequest, HttpResponse, JsonResponse}, auth::SymmetricEncrypt, - JMAP, }; use super::{ @@ -28,9 +29,52 @@ use super::{ MAX_POST_LEN, RANDOM_CODE_LEN, }; -impl JMAP { +pub trait TokenHandler: Sync + Send { + fn handle_token_request( + &self, + req: &mut HttpRequest, + session_id: u64, + ) -> impl Future<Output = trc::Result<HttpResponse>> + Send; + + fn password_hash( + &self, + account_id: u32, + ) -> impl Future<Output = Result<String, &'static str>> + Send; + + fn issue_token( + &self, + account_id: u32, + client_id: &str, + with_refresh_token: bool, + ) -> impl Future<Output = Result<OAuthResponse, &'static str>> + Send; + + fn issue_custom_token( + &self, + account_id: u32, + grant_type: &str, + client_id: &str, + expiry_in: u64, + ) -> impl Future<Output = trc::Result<String>> + Send; + + fn encode_access_token( + &self, + grant_type: &str, + account_id: u32, + password_hash: &str, + client_id: &str, + expiry_in: u64, + ) -> Result<String, &'static str>; + + fn validate_access_token( + &self, + grant_type: &str, + token_: &str, + ) -> impl Future<Output = trc::Result<(u32, String, u64)>> + Send; +} + +impl TokenHandler for Server { // Token endpoint - pub async fn handle_token_request( + async fn handle_token_request( &self, req: &mut HttpRequest, session_id: u64, @@ -199,7 +243,7 @@ impl JMAP { } } - pub async fn issue_token( + async fn issue_token( &self, account_id: u32, client_id: &str, @@ -233,7 +277,7 @@ impl JMAP { }) } - pub async fn issue_custom_token( + async fn issue_custom_token( &self, account_id: u32, grant_type: &str, @@ -303,7 +347,7 @@ impl JMAP { Ok(String::from_utf8(base64_encode(&token).unwrap_or_default()).unwrap()) } - pub async fn validate_access_token( + async fn validate_access_token( &self, grant_type: &str, token_: &str, diff --git a/crates/jmap/src/auth/rate_limit.rs b/crates/jmap/src/auth/rate_limit.rs index 25b1ec9b..6d7be622 100644 --- a/crates/jmap/src/auth/rate_limit.rs +++ b/crates/jmap/src/auth/rate_limit.rs @@ -6,23 +6,33 @@ use std::{net::IpAddr, sync::Arc}; -use common::listener::limiter::{ConcurrencyLimiter, InFlight}; +use common::{ + listener::limiter::{ConcurrencyLimiter, InFlight}, + ConcurrencyLimiters, Server, +}; use directory::Permission; use trc::AddContext; -use crate::JMAP; - use common::auth::AccessToken; +use std::future::Future; -pub struct ConcurrencyLimiters { - pub concurrent_requests: ConcurrencyLimiter, - pub concurrent_uploads: ConcurrencyLimiter, +pub trait RateLimiter: Sync + Send { + fn get_concurrency_limiter(&self, account_id: u32) -> Arc<ConcurrencyLimiters>; + fn is_account_allowed( + &self, + access_token: &AccessToken, + ) -> impl Future<Output = trc::Result<InFlight>> + Send; + fn is_anonymous_allowed(&self, addr: &IpAddr) -> impl Future<Output = trc::Result<()>> + Send; + fn is_upload_allowed(&self, access_token: &AccessToken) -> trc::Result<InFlight>; + fn is_auth_allowed_soft(&self, addr: &IpAddr) -> impl Future<Output = trc::Result<()>> + Send; + fn is_auth_allowed_hard(&self, addr: &IpAddr) -> impl Future<Output = trc::Result<()>> + Send; } -impl JMAP { - pub fn get_concurrency_limiter(&self, account_id: u32) -> Arc<ConcurrencyLimiters> { +impl RateLimiter for Server { + fn get_concurrency_limiter(&self, account_id: u32) -> Arc<ConcurrencyLimiters> { self.inner - .concurrency_limiter + .data + .jmap_limiter .get(&account_id) .map(|limiter| limiter.clone()) .unwrap_or_else(|| { @@ -35,13 +45,14 @@ impl JMAP { ), }); self.inner - .concurrency_limiter + .data + .jmap_limiter .insert(account_id, limiter.clone()); limiter }) } - pub async fn is_account_allowed(&self, access_token: &AccessToken) -> trc::Result<InFlight> { + async fn is_account_allowed(&self, access_token: &AccessToken) -> trc::Result<InFlight> { let limiter = self.get_concurrency_limiter(access_token.primary_id()); let is_rate_allowed = if let Some(rate) = &self.core.jmap.rate_authenticated { self.core @@ -74,7 +85,7 @@ impl JMAP { } } - pub async fn is_anonymous_allowed(&self, addr: &IpAddr) -> trc::Result<()> { + async fn is_anonymous_allowed(&self, addr: &IpAddr) -> trc::Result<()> { if let Some(rate) = &self.core.jmap.rate_anonymous { if self .core @@ -91,7 +102,7 @@ impl JMAP { Ok(()) } - pub fn is_upload_allowed(&self, access_token: &AccessToken) -> trc::Result<InFlight> { + fn is_upload_allowed(&self, access_token: &AccessToken) -> trc::Result<InFlight> { if let Some(in_flight_request) = self .get_concurrency_limiter(access_token.primary_id()) .concurrent_uploads @@ -105,7 +116,7 @@ impl JMAP { } } - pub async fn is_auth_allowed_soft(&self, addr: &IpAddr) -> trc::Result<()> { + async fn is_auth_allowed_soft(&self, addr: &IpAddr) -> trc::Result<()> { if let Some(rate) = &self.core.jmap.rate_authenticate_req { if self .core @@ -122,7 +133,7 @@ impl JMAP { Ok(()) } - pub async fn is_auth_allowed_hard(&self, addr: &IpAddr) -> trc::Result<()> { + async fn is_auth_allowed_hard(&self, addr: &IpAddr) -> trc::Result<()> { if let Some(rate) = &self.core.jmap.rate_authenticate_req { if self .core @@ -139,9 +150,3 @@ impl JMAP { Ok(()) } } - -impl ConcurrencyLimiters { - pub fn is_active(&self) -> bool { - self.concurrent_requests.is_active() || self.concurrent_uploads.is_active() - } -} diff --git a/crates/jmap/src/blob/copy.rs b/crates/jmap/src/blob/copy.rs index 635b6bc0..19aa6df0 100644 --- a/crates/jmap/src/blob/copy.rs +++ b/crates/jmap/src/blob/copy.rs @@ -4,23 +4,34 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use common::auth::AccessToken; +use common::{auth::AccessToken, Server}; use jmap_proto::{ error::set::{SetError, SetErrorType}, method::copy::{CopyBlobRequest, CopyBlobResponse}, types::blob::BlobId, }; +use std::future::Future; use store::{ write::{now, BatchBuilder, BlobOp}, BlobClass, Serialize, }; use utils::map::vec_map::VecMap; -use crate::JMAP; +use crate::JmapMethods; -impl JMAP { - pub async fn blob_copy( +use super::download::BlobDownload; + +pub trait BlobCopy: Sync + Send { + fn blob_copy( + &self, + request: CopyBlobRequest, + access_token: &AccessToken, + ) -> impl Future<Output = trc::Result<CopyBlobResponse>> + Send; +} + +impl BlobCopy for Server { + async fn blob_copy( &self, request: CopyBlobRequest, access_token: &AccessToken, diff --git a/crates/jmap/src/blob/download.rs b/crates/jmap/src/blob/download.rs index 8b289e59..19534b07 100644 --- a/crates/jmap/src/blob/download.rs +++ b/crates/jmap/src/blob/download.rs @@ -6,7 +6,7 @@ use std::ops::Range; -use common::auth::AccessToken; +use common::{auth::AccessToken, Server}; use jmap_proto::types::{ acl::Acl, blob::{BlobId, BlobSection}, @@ -16,15 +16,42 @@ use mail_parser::{ decoders::{base64::base64_decode, quoted_printable::quoted_printable_decode}, Encoding, }; +use std::future::Future; use store::BlobClass; use trc::AddContext; use utils::BlobHash; -use crate::JMAP; +use crate::auth::acl::AclMethods; -impl JMAP { +pub trait BlobDownload: Sync + Send { + fn blob_download( + &self, + blob_id: &BlobId, + access_token: &AccessToken, + ) -> impl Future<Output = trc::Result<Option<Vec<u8>>>> + Send; + + fn get_blob_section( + &self, + hash: &BlobHash, + section: &BlobSection, + ) -> impl Future<Output = trc::Result<Option<Vec<u8>>>> + Send; + + fn get_blob( + &self, + hash: &BlobHash, + range: Range<usize>, + ) -> impl Future<Output = trc::Result<Option<Vec<u8>>>> + Send; + + fn has_access_blob( + &self, + blob_id: &BlobId, + access_token: &AccessToken, + ) -> impl Future<Output = trc::Result<bool>> + Send; +} + +impl BlobDownload for Server { #[allow(clippy::blocks_in_conditions)] - pub async fn blob_download( + async fn blob_download( &self, blob_id: &BlobId, access_token: &AccessToken, @@ -84,7 +111,7 @@ impl JMAP { } } - pub async fn get_blob_section( + async fn get_blob_section( &self, hash: &BlobHash, section: &BlobSection, @@ -102,11 +129,7 @@ impl JMAP { })) } - pub async fn get_blob( - &self, - hash: &BlobHash, - range: Range<usize>, - ) -> trc::Result<Option<Vec<u8>>> { + async fn get_blob(&self, hash: &BlobHash, range: Range<usize>) -> trc::Result<Option<Vec<u8>>> { self.core .storage .blob @@ -115,7 +138,7 @@ impl JMAP { .caused_by(trc::location!()) } - pub async fn has_access_blob( + async fn has_access_blob( &self, blob_id: &BlobId, access_token: &AccessToken, diff --git a/crates/jmap/src/blob/get.rs b/crates/jmap/src/blob/get.rs index 3add516d..b73e05c5 100644 --- a/crates/jmap/src/blob/get.rs +++ b/crates/jmap/src/blob/get.rs @@ -4,7 +4,7 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use common::auth::AccessToken; +use common::{auth::AccessToken, Server}; use jmap_proto::{ method::{ get::{GetRequest, GetResponse}, @@ -26,10 +26,26 @@ use sha2::{Sha256, Sha512}; use store::BlobClass; use utils::map::vec_map::VecMap; -use crate::{mailbox::UidMailbox, JMAP}; +use crate::{mailbox::UidMailbox, JmapMethods}; +use std::future::Future; -impl JMAP { - pub async fn blob_get( +use super::download::BlobDownload; + +pub trait BlobOperations: Sync + Send { + fn blob_get( + &self, + request: GetRequest<GetArguments>, + access_token: &AccessToken, + ) -> impl Future<Output = trc::Result<GetResponse>> + Send; + + fn blob_lookup( + &self, + request: BlobLookupRequest, + ) -> impl Future<Output = trc::Result<BlobLookupResponse>> + Send; +} + +impl BlobOperations for Server { + async fn blob_get( &self, mut request: GetRequest<GetArguments>, access_token: &AccessToken, @@ -148,7 +164,7 @@ impl JMAP { Ok(response) } - pub async fn blob_lookup(&self, request: BlobLookupRequest) -> trc::Result<BlobLookupResponse> { + async fn blob_lookup(&self, request: BlobLookupRequest) -> trc::Result<BlobLookupResponse> { let mut include_email = false; let mut include_mailbox = false; let mut include_thread = false; diff --git a/crates/jmap/src/blob/upload.rs b/crates/jmap/src/blob/upload.rs index dfa902dc..49255196 100644 --- a/crates/jmap/src/blob/upload.rs +++ b/crates/jmap/src/blob/upload.rs @@ -6,7 +6,7 @@ use std::sync::Arc; -use common::auth::AccessToken; +use common::{auth::AccessToken, Server}; use directory::Permission; use jmap_proto::{ error::set::SetError, @@ -23,16 +23,40 @@ use store::{ use trc::AddContext; use utils::BlobHash; -use crate::JMAP; +use crate::{auth::rate_limit::RateLimiter, JmapMethods}; -use super::UploadResponse; +use super::{download::BlobDownload, UploadResponse}; +use std::future::Future; #[cfg(feature = "test_mode")] pub static DISABLE_UPLOAD_QUOTA: std::sync::atomic::AtomicBool = std::sync::atomic::AtomicBool::new(true); -impl JMAP { - pub async fn blob_upload_many( +pub trait BlobUpload: Sync + Send { + fn blob_upload_many( + &self, + request: BlobUploadRequest, + access_token: &AccessToken, + ) -> impl Future<Output = trc::Result<BlobUploadResponse>> + Send; + + fn blob_upload( + &self, + account_id: Id, + content_type: &str, + data: &[u8], + access_token: Arc<AccessToken>, + ) -> impl Future<Output = trc::Result<UploadResponse>> + Send; + + fn put_blob( + &self, + account_id: u32, + data: &[u8], + set_quota: bool, + ) -> impl Future<Output = trc::Result<BlobId>> + Send; +} + +impl BlobUpload for Server { + async fn blob_upload_many( &self, request: BlobUploadRequest, access_token: &AccessToken, @@ -178,7 +202,7 @@ impl JMAP { Ok(response) } - pub async fn blob_upload( + async fn blob_upload( &self, account_id: Id, content_type: &str, @@ -239,12 +263,7 @@ impl JMAP { } #[allow(clippy::blocks_in_conditions)] - pub async fn put_blob( - &self, - account_id: u32, - data: &[u8], - set_quota: bool, - ) -> trc::Result<BlobId> { + async fn put_blob(&self, account_id: u32, data: &[u8], set_quota: bool) -> trc::Result<BlobId> { // First reserve the hash let hash = BlobHash::from(data); let mut batch = BatchBuilder::new(); diff --git a/crates/jmap/src/changes/get.rs b/crates/jmap/src/changes/get.rs index cd68d95a..3da6ba15 100644 --- a/crates/jmap/src/changes/get.rs +++ b/crates/jmap/src/changes/get.rs @@ -4,18 +4,32 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use common::auth::AccessToken; +use common::{auth::AccessToken, Server}; use jmap_proto::{ method::changes::{ChangesRequest, ChangesResponse, RequestArguments}, types::{collection::Collection, property::Property, state::State}, }; +use std::future::Future; use store::query::log::{Change, Changes, Query}; use trc::AddContext; -use crate::JMAP; +pub trait ChangesLookup: Sync + Send { + fn changes( + &self, + request: ChangesRequest, + access_token: &AccessToken, + ) -> impl Future<Output = trc::Result<ChangesResponse>> + Send; + + fn changes_( + &self, + account_id: u32, + collection: Collection, + query: Query, + ) -> impl Future<Output = trc::Result<Changes>> + Send; +} -impl JMAP { - pub async fn changes( +impl ChangesLookup for Server { + async fn changes( &self, request: ChangesRequest, access_token: &AccessToken, @@ -160,7 +174,7 @@ impl JMAP { Ok(response) } - pub async fn changes_( + async fn changes_( &self, account_id: u32, collection: Collection, diff --git a/crates/jmap/src/changes/query.rs b/crates/jmap/src/changes/query.rs index a6d31831..bfcb1517 100644 --- a/crates/jmap/src/changes/query.rs +++ b/crates/jmap/src/changes/query.rs @@ -4,17 +4,31 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use common::auth::AccessToken; +use common::{auth::AccessToken, Server}; use jmap_proto::method::{ changes::{self, ChangesRequest}, query::{self, QueryRequest}, query_changes::{AddedItem, QueryChangesRequest, QueryChangesResponse}, }; +use std::future::Future; -use crate::JMAP; +use crate::{ + email::query::EmailQuery, mailbox::query::MailboxQuery, quota::query::QuotaQuery, + submission::query::EmailSubmissionQuery, +}; + +use super::get::ChangesLookup; + +pub trait QueryChanges: Sync + Send { + fn query_changes( + &self, + request: QueryChangesRequest, + access_token: &AccessToken, + ) -> impl Future<Output = trc::Result<QueryChangesResponse>> + Send; +} -impl JMAP { - pub async fn query_changes( +impl QueryChanges for Server { + async fn query_changes( &self, request: QueryChangesRequest, access_token: &AccessToken, diff --git a/crates/jmap/src/changes/state.rs b/crates/jmap/src/changes/state.rs index 96f75c0d..266ef2bc 100644 --- a/crates/jmap/src/changes/state.rs +++ b/crates/jmap/src/changes/state.rs @@ -4,16 +4,31 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ +use common::Server; use jmap_proto::types::{collection::Collection, state::State}; +use std::future::Future; use trc::AddContext; -use crate::JMAP; +pub trait StateManager: Sync + Send { + fn get_state( + &self, + account_id: u32, + collection: impl Into<u8> + Send, + ) -> impl Future<Output = trc::Result<State>> + Send; + + fn assert_state( + &self, + account_id: u32, + collection: Collection, + if_in_state: &Option<State>, + ) -> impl Future<Output = trc::Result<State>> + Send; +} -impl JMAP { - pub async fn get_state( +impl StateManager for Server { + async fn get_state( &self, account_id: u32, - collection: impl Into<u8>, + collection: impl Into<u8> + Send, ) -> trc::Result<State> { let collection = collection.into(); self.core @@ -25,7 +40,7 @@ impl JMAP { .map(State::from) } - pub async fn assert_state( + async fn assert_state( &self, account_id: u32, collection: Collection, diff --git a/crates/jmap/src/changes/write.rs b/crates/jmap/src/changes/write.rs index a765a480..7074cc21 100644 --- a/crates/jmap/src/changes/write.rs +++ b/crates/jmap/src/changes/write.rs @@ -6,28 +6,47 @@ use std::time::Duration; +use common::Server; use jmap_proto::types::collection::Collection; +use std::future::Future; use store::{ write::{log::ChangeLogBuilder, BatchBuilder}, LogKey, }; use trc::AddContext; -use crate::JMAP; +pub trait ChangeLog: Sync + Send { + fn begin_changes( + &self, + account_id: u32, + ) -> impl Future<Output = trc::Result<ChangeLogBuilder>> + Send; + fn assign_change_id(&self, account_id: u32) -> impl Future<Output = trc::Result<u64>> + Send; + fn generate_snowflake_id(&self) -> trc::Result<u64>; + fn commit_changes( + &self, + account_id: u32, + changes: ChangeLogBuilder, + ) -> impl Future<Output = trc::Result<u64>> + Send; + fn delete_changes( + &self, + account_id: u32, + before: Duration, + ) -> impl Future<Output = trc::Result<()>> + Send; +} -impl JMAP { - pub async fn begin_changes(&self, account_id: u32) -> trc::Result<ChangeLogBuilder> { +impl ChangeLog for Server { + async fn begin_changes(&self, account_id: u32) -> trc::Result<ChangeLogBuilder> { self.assign_change_id(account_id) .await .map(ChangeLogBuilder::with_change_id) } - pub async fn assign_change_id(&self, _: u32) -> trc::Result<u64> { + async fn assign_change_id(&self, _: u32) -> trc::Result<u64> { self.generate_snowflake_id() } - pub fn generate_snowflake_id(&self) -> trc::Result<u64> { - self.inner.snowflake_id.generate().ok_or_else(|| { + fn generate_snowflake_id(&self) -> trc::Result<u64> { + self.inner.data.jmap_id_gen.generate().ok_or_else(|| { trc::StoreEvent::UnexpectedError .into_err() .caused_by(trc::location!()) @@ -35,7 +54,7 @@ impl JMAP { }) } - pub async fn commit_changes( + async fn commit_changes( &self, account_id: u32, mut changes: ChangeLogBuilder, @@ -56,8 +75,8 @@ impl JMAP { .map(|_| state) } - pub async fn delete_changes(&self, account_id: u32, before: Duration) -> trc::Result<()> { - let reference_cid = self.inner.snowflake_id.past_id(before).ok_or_else(|| { + async fn delete_changes(&self, account_id: u32, before: Duration) -> trc::Result<()> { + let reference_cid = self.inner.data.jmap_id_gen.past_id(before).ok_or_else(|| { trc::StoreEvent::UnexpectedError .caused_by(trc::location!()) .ctx(trc::Key::Reason, "Failed to generate reference change id.") diff --git a/crates/jmap/src/email/cache.rs b/crates/jmap/src/email/cache.rs index 2d73c6a8..830ee312 100644 --- a/crates/jmap/src/email/cache.rs +++ b/crates/jmap/src/email/cache.rs @@ -4,25 +4,29 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use std::{collections::HashMap, sync::Arc}; +use std::sync::Arc; +use common::{Server, Threads}; use jmap_proto::types::{collection::Collection, property::Property}; +use std::future::Future; use trc::AddContext; use utils::lru_cache::LruCached; -use crate::JMAP; +use crate::JmapMethods; -#[derive(Debug, Default)] -pub struct Threads { - pub threads: HashMap<u32, u32>, - pub modseq: Option<u64>, +pub trait ThreadCache: Sync + Send { + fn get_cached_thread_ids( + &self, + account_id: u32, + message_ids: impl Iterator<Item = u32> + Send, + ) -> impl Future<Output = trc::Result<Vec<(u32, u32)>>> + Send; } -impl JMAP { - pub async fn get_cached_thread_ids( +impl ThreadCache for Server { + async fn get_cached_thread_ids( &self, account_id: u32, - message_ids: impl Iterator<Item = u32>, + message_ids: impl Iterator<Item = u32> + Send, ) -> trc::Result<Vec<(u32, u32)>> { // Obtain current state let modseq = self @@ -34,8 +38,12 @@ impl JMAP { .caused_by(trc::location!())?; // Lock the cache - let thread_cache = if let Some(thread_cache) = - self.inner.cache_threads.get(&account_id).and_then(|t| { + let thread_cache = if let Some(thread_cache) = self + .inner + .data + .threads_cache + .get(&account_id) + .and_then(|t| { if t.modseq.unwrap_or(0) >= modseq.unwrap_or(0) { Some(t) } else { @@ -58,7 +66,8 @@ impl JMAP { modseq, }); self.inner - .cache_threads + .data + .threads_cache .insert(account_id, thread_cache.clone()); thread_cache }; diff --git a/crates/jmap/src/email/copy.rs b/crates/jmap/src/email/copy.rs index 472df5d0..bc7aa75b 100644 --- a/crates/jmap/src/email/copy.rs +++ b/crates/jmap/src/email/copy.rs @@ -4,7 +4,10 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use common::auth::{AccessToken, ResourceToken}; +use common::{ + auth::{AccessToken, ResourceToken}, + Server, +}; use jmap_proto::{ error::set::SetError, method::{ @@ -42,16 +45,46 @@ use store::{ use trc::AddContext; use utils::map::vec_map::VecMap; -use crate::{api::http::HttpSessionData, mailbox::UidMailbox, JMAP}; +use crate::{ + api::http::HttpSessionData, + auth::acl::AclMethods, + changes::{state::StateManager, write::ChangeLog}, + mailbox::{set::MailboxSet, UidMailbox}, + services::index::Indexer, + JmapMethods, +}; +use std::future::Future; use super::{ index::{EmailIndexBuilder, TrimTextValue, VisitValues, MAX_ID_LENGTH, MAX_SORT_FIELD_LENGTH}, - ingest::{IngestedEmail, LogEmailInsert}, + ingest::{EmailIngest, IngestedEmail, LogEmailInsert}, metadata::MessageMetadata, }; -impl JMAP { - pub async fn email_copy( +pub trait EmailCopy: Sync + Send { + fn email_copy( + &self, + request: CopyRequest<RequestArguments>, + access_token: &AccessToken, + next_call: &mut Option<Call<RequestMethod>>, + session: &HttpSessionData, + ) -> impl Future<Output = trc::Result<CopyResponse>> + Send; + + #[allow(clippy::too_many_arguments)] + fn copy_message( + &self, + from_account_id: u32, + from_message_id: u32, + resource_token: &ResourceToken, + mailboxes: Vec<u32>, + keywords: Vec<Keyword>, + received_at: Option<UTCDate>, + session_id: u64, + ) -> impl Future<Output = trc::Result<Result<IngestedEmail, SetError>>> + Send; +} + +impl EmailCopy for Server { + async fn email_copy( &self, request: CopyRequest<RequestArguments>, access_token: &AccessToken, @@ -271,7 +304,7 @@ impl JMAP { } #[allow(clippy::too_many_arguments)] - pub async fn copy_message( + async fn copy_message( &self, from_account_id: u32, from_message_id: u32, @@ -435,7 +468,7 @@ impl JMAP { let document_id = ids.last_document_id().caused_by(trc::location!())?; // Request FTS index - self.inner.request_fts_index(); + self.request_fts_index(); // Update response email.id = Id::from_parts(thread_id, document_id); diff --git a/crates/jmap/src/email/crypto.rs b/crates/jmap/src/email/crypto.rs index 815de86d..758177a5 100644 --- a/crates/jmap/src/email/crypto.rs +++ b/crates/jmap/src/email/crypto.rs @@ -4,14 +4,16 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use std::{borrow::Cow, collections::BTreeSet, fmt::Display, io::Cursor, sync::Arc}; +use std::{ + borrow::Cow, collections::BTreeSet, fmt::Display, future::Future, io::Cursor, sync::Arc, +}; use crate::{ api::{http::ToHttpResponse, HttpResponse, JsonResponse}, - JMAP, + JmapMethods, }; use aes::cipher::{block_padding::Pkcs7, BlockEncryptMut, KeyIvInit}; -use common::auth::AccessToken; +use common::{auth::AccessToken, Server}; use directory::backend::internal::manage; use jmap_proto::types::{collection::Collection, property::Property}; use mail_builder::{encoders::base64::base64_encode_mime, mime::make_boundary}; @@ -628,11 +630,21 @@ impl ToBitmaps for &EncryptionParams { } } -impl JMAP { - pub async fn handle_crypto_get( +pub trait CryptoHandler: Sync + Send { + fn handle_crypto_get( &self, access_token: Arc<AccessToken>, - ) -> trc::Result<HttpResponse> { + ) -> impl Future<Output = trc::Result<HttpResponse>> + Send; + + fn handle_crypto_post( + &self, + access_token: Arc<AccessToken>, + body: Option<Vec<u8>>, + ) -> impl Future<Output = trc::Result<HttpResponse>> + Send; +} + +impl CryptoHandler for Server { + async fn handle_crypto_get(&self, access_token: Arc<AccessToken>) -> trc::Result<HttpResponse> { let params = self .get_property::<EncryptionParams>( access_token.primary_id(), @@ -664,7 +676,7 @@ impl JMAP { .into_http_response()) } - pub async fn handle_crypto_post( + async fn handle_crypto_post( &self, access_token: Arc<AccessToken>, body: Option<Vec<u8>>, diff --git a/crates/jmap/src/email/delete.rs b/crates/jmap/src/email/delete.rs index 5268925e..5d587aa3 100644 --- a/crates/jmap/src/email/delete.rs +++ b/crates/jmap/src/email/delete.rs @@ -6,6 +6,7 @@ use std::time::Duration; +use common::Server; use jmap_proto::types::{ collection::Collection, id::Id, keyword::Keyword, property::Property, state::StateChange, type_state::DataType, @@ -23,15 +24,41 @@ use trc::{AddContext, StoreEvent}; use utils::codec::leb128::Leb128Reader; use crate::{ + changes::write::ChangeLog, mailbox::{UidMailbox, JUNK_ID, TOMBSTONE_ID, TRASH_ID}, - JMAP, + services::state::StateManager, + JmapMethods, }; use super::{index::EmailIndexBuilder, metadata::MessageMetadata}; use rand::prelude::SliceRandom; +use std::future::Future; -impl JMAP { - pub async fn emails_tombstone( +pub trait EmailDeletion: Sync + Send { + fn emails_tombstone( + &self, + account_id: u32, + document_ids: RoaringBitmap, + ) -> impl Future<Output = trc::Result<(ChangeLogBuilder, RoaringBitmap)>> + Send; + + fn purge_accounts(&self) -> impl Future<Output = ()> + Send; + + fn purge_account(&self, account_id: u32) -> impl Future<Output = ()> + Send; + + fn emails_auto_expunge( + &self, + account_id: u32, + period: Duration, + ) -> impl Future<Output = trc::Result<()>> + Send; + + fn emails_purge_tombstoned( + &self, + account_id: u32, + ) -> impl Future<Output = trc::Result<()>> + Send; +} + +impl EmailDeletion for Server { + async fn emails_tombstone( &self, account_id: u32, mut document_ids: RoaringBitmap, @@ -208,7 +235,7 @@ impl JMAP { Ok((changes, document_ids)) } - pub async fn purge_accounts(&self) { + async fn purge_accounts(&self) { if let Ok(Some(account_ids)) = self.get_document_ids(u32::MAX, Collection::Principal).await { let mut account_ids: Vec<u32> = account_ids.into_iter().collect(); @@ -222,7 +249,7 @@ impl JMAP { } } - pub async fn purge_account(&self, account_id: u32) { + async fn purge_account(&self, account_id: u32) { // Lock account match self .core @@ -290,7 +317,7 @@ impl JMAP { } } - pub async fn emails_auto_expunge(&self, account_id: u32, period: Duration) -> trc::Result<()> { + async fn emails_auto_expunge(&self, account_id: u32, period: Duration) -> trc::Result<()> { let deletion_candidates = self .get_tag( account_id, @@ -313,7 +340,7 @@ impl JMAP { if deletion_candidates.is_empty() { return Ok(()); } - let reference_cid = self.inner.snowflake_id.past_id(period).ok_or_else(|| { + let reference_cid = self.inner.data.jmap_id_gen.past_id(period).ok_or_else(|| { trc::StoreEvent::UnexpectedError .into_err() .caused_by(trc::location!()) @@ -364,7 +391,7 @@ impl JMAP { Ok(()) } - pub async fn emails_purge_tombstoned(&self, account_id: u32) -> trc::Result<()> { + async fn emails_purge_tombstoned(&self, account_id: u32) -> trc::Result<()> { // Obtain tombstoned messages let tombstoned_ids = self .core @@ -401,7 +428,6 @@ impl JMAP { // Obtain tenant id let tenant_id = self - .core .get_cached_access_token(account_id) .await .caused_by(trc::location!())? diff --git a/crates/jmap/src/email/get.rs b/crates/jmap/src/email/get.rs index cc891d37..921ecd5f 100644 --- a/crates/jmap/src/email/get.rs +++ b/crates/jmap/src/email/get.rs @@ -4,7 +4,7 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use common::auth::AccessToken; +use common::{auth::AccessToken, Server}; use jmap_proto::{ method::get::{GetRequest, GetResponse}, object::{email::GetArguments, Object}, @@ -23,16 +23,29 @@ use mail_parser::HeaderName; use store::{write::Bincode, BlobClass}; use trc::{AddContext, StoreEvent}; -use crate::{email::headers::HeaderToValue, mailbox::UidMailbox, JMAP}; +use crate::{ + auth::acl::AclMethods, blob::download::BlobDownload, changes::state::StateManager, + email::headers::HeaderToValue, mailbox::UidMailbox, JmapMethods, +}; +use std::future::Future; use super::{ body::{ToBodyPart, TruncateBody}, + cache::ThreadCache, headers::IntoForm, metadata::{MessageMetadata, MetadataPartType}, }; -impl JMAP { - pub async fn email_get( +pub trait EmailGet: Sync + Send { + fn email_get( + &self, + request: GetRequest<GetArguments>, + access_token: &AccessToken, + ) -> impl Future<Output = trc::Result<GetResponse>> + Send; +} + +impl EmailGet for Server { + async fn email_get( &self, mut request: GetRequest<GetArguments>, access_token: &AccessToken, diff --git a/crates/jmap/src/email/import.rs b/crates/jmap/src/email/import.rs index 1abe06cc..1d9b441d 100644 --- a/crates/jmap/src/email/import.rs +++ b/crates/jmap/src/email/import.rs @@ -4,7 +4,7 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use common::auth::AccessToken; +use common::{auth::AccessToken, Server}; use jmap_proto::{ error::set::{SetError, SetErrorType}, method::import::{ImportEmailRequest, ImportEmailResponse}, @@ -20,12 +20,25 @@ use jmap_proto::{ use mail_parser::MessageParser; use utils::map::vec_map::VecMap; -use crate::{api::http::HttpSessionData, JMAP}; +use crate::{ + api::http::HttpSessionData, auth::acl::AclMethods, blob::download::BlobDownload, + changes::state::StateManager, mailbox::set::MailboxSet, JmapMethods, +}; + +use super::ingest::{EmailIngest, IngestEmail, IngestSource}; +use std::future::Future; -use super::ingest::{IngestEmail, IngestSource}; +pub trait EmailImport: Sync + Send { + fn email_import( + &self, + request: ImportEmailRequest, + access_token: &AccessToken, + session: &HttpSessionData, + ) -> impl Future<Output = trc::Result<ImportEmailResponse>> + Send; +} -impl JMAP { - pub async fn email_import( +impl EmailImport for Server { + async fn email_import( &self, request: ImportEmailRequest, access_token: &AccessToken, diff --git a/crates/jmap/src/email/ingest.rs b/crates/jmap/src/email/ingest.rs index 66a4cced..3a608113 100644 --- a/crates/jmap/src/email/ingest.rs +++ b/crates/jmap/src/email/ingest.rs @@ -9,7 +9,7 @@ use std::{ time::{Duration, Instant}, }; -use common::auth::ResourceToken; +use common::{auth::ResourceToken, Server}; use jmap_proto::{ object::Object, types::{ @@ -22,6 +22,7 @@ use mail_parser::{ }; use rand::Rng; +use std::future::Future; use store::{ ahash::AHashSet, query::Filter, @@ -36,12 +37,16 @@ use trc::{AddContext, MessageIngestEvent}; use utils::map::vec_map::VecMap; use crate::{ + blob::upload::BlobUpload, + changes::write::ChangeLog, email::index::{IndexMessage, VisitValues, MAX_ID_LENGTH}, mailbox::{UidMailbox, INBOX_ID, JUNK_ID}, - JMAP, + services::index::Indexer, + JmapMethods, }; use super::{ + cache::ThreadCache, crypto::{EncryptMessage, EncryptMessageError, EncryptionParams}, index::{TrimTextValue, MAX_SORT_FIELD_LENGTH}, }; @@ -76,9 +81,27 @@ pub enum IngestSource { const MAX_RETRIES: u32 = 10; -impl JMAP { +pub trait EmailIngest: Sync + Send { + fn email_ingest( + &self, + params: IngestEmail, + ) -> impl Future<Output = trc::Result<IngestedEmail>> + Send; + fn find_or_merge_thread( + &self, + account_id: u32, + thread_name: &str, + references: &[&str], + ) -> impl Future<Output = trc::Result<Option<u32>>> + Send; + fn assign_imap_uid( + &self, + account_id: u32, + mailbox_id: u32, + ) -> impl Future<Output = trc::Result<u32>> + Send; +} + +impl EmailIngest for Server { #[allow(clippy::blocks_in_conditions)] - pub async fn email_ingest(&self, mut params: IngestEmail<'_>) -> trc::Result<IngestedEmail> { + async fn email_ingest(&self, mut params: IngestEmail<'_>) -> trc::Result<IngestedEmail> { // Check quota let start_time = Instant::now(); let account_id = params.resource.account_id; @@ -338,7 +361,7 @@ impl JMAP { let id = Id::from_parts(thread_id, document_id); // Request FTS index - self.inner.request_fts_index(); + self.request_fts_index(); trc::event!( MessageIngest(match params.source { @@ -379,7 +402,7 @@ impl JMAP { }) } - pub async fn find_or_merge_thread( + async fn find_or_merge_thread( &self, account_id: u32, thread_name: &str, @@ -519,7 +542,7 @@ impl JMAP { } } - pub async fn assign_imap_uid(&self, account_id: u32, mailbox_id: u32) -> trc::Result<u32> { + async fn assign_imap_uid(&self, account_id: u32, mailbox_id: u32) -> trc::Result<u32> { // Increment UID next let mut batch = BatchBuilder::new(); batch diff --git a/crates/jmap/src/email/parse.rs b/crates/jmap/src/email/parse.rs index 83ec8162..d0b33751 100644 --- a/crates/jmap/src/email/parse.rs +++ b/crates/jmap/src/email/parse.rs @@ -4,7 +4,7 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use common::auth::AccessToken; +use common::{auth::AccessToken, Server}; use jmap_proto::{ method::parse::{ParseEmailRequest, ParseEmailResponse}, object::Object, @@ -13,9 +13,10 @@ use jmap_proto::{ use mail_parser::{ decoders::html::html_to_text, parsers::preview::preview_text, MessageParser, PartType, }; +use std::future::Future; use utils::map::vec_map::VecMap; -use crate::JMAP; +use crate::blob::download::BlobDownload; use super::{ body::{ToBodyPart, TruncateBody}, @@ -23,8 +24,16 @@ use super::{ index::PREVIEW_LENGTH, }; -impl JMAP { - pub async fn email_parse( +pub trait EmailParse: Sync + Send { + fn email_parse( + &self, + request: ParseEmailRequest, + access_token: &AccessToken, + ) -> impl Future<Output = trc::Result<ParseEmailResponse>> + Send; +} + +impl EmailParse for Server { + async fn email_parse( &self, request: ParseEmailRequest, access_token: &AccessToken, diff --git a/crates/jmap/src/email/query.rs b/crates/jmap/src/email/query.rs index 56942279..1a56c33f 100644 --- a/crates/jmap/src/email/query.rs +++ b/crates/jmap/src/email/query.rs @@ -4,7 +4,7 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use common::auth::AccessToken; +use common::{auth::AccessToken, Server}; use jmap_proto::{ method::query::{Comparator, Filter, QueryRequest, QueryResponse, SortProperty}, object::email::QueryArguments, @@ -12,6 +12,7 @@ use jmap_proto::{ }; use mail_parser::HeaderName; use nlp::language::Language; +use std::future::Future; use store::{ fts::{Field, FilterGroup, FtsFilter, IntoFilterGroup}, query::{self}, @@ -20,10 +21,27 @@ use store::{ ValueKey, }; -use crate::JMAP; +use crate::{auth::acl::AclMethods, JmapMethods}; -impl JMAP { - pub async fn email_query( +use super::cache::ThreadCache; + +pub trait EmailQuery: Sync + Send { + fn email_query( + &self, + request: QueryRequest<QueryArguments>, + access_token: &AccessToken, + ) -> impl Future<Output = trc::Result<QueryResponse>> + Send; + + fn thread_keywords( + &self, + account_id: u32, + keyword: Keyword, + match_all: bool, + ) -> impl Future<Output = trc::Result<RoaringBitmap>> + Send; +} + +impl EmailQuery for Server { + async fn email_query( &self, mut request: QueryRequest<QueryArguments>, access_token: &AccessToken, diff --git a/crates/jmap/src/email/set.rs b/crates/jmap/src/email/set.rs index 62a7562f..a1a0f719 100644 --- a/crates/jmap/src/email/set.rs +++ b/crates/jmap/src/email/set.rs @@ -6,7 +6,7 @@ use std::{borrow::Cow, collections::HashMap, slice::IterMut}; -use common::auth::AccessToken; +use common::{auth::AccessToken, Server}; use jmap_proto::{ error::set::{SetError, SetErrorType}, method::set::{RequestArguments, SetRequest, SetResponse}, @@ -41,15 +41,33 @@ use store::{ }; use trc::AddContext; -use crate::{api::http::HttpSessionData, mailbox::UidMailbox, JMAP}; +use crate::{ + api::http::HttpSessionData, + auth::acl::AclMethods, + blob::download::BlobDownload, + changes::{state::StateManager, write::ChangeLog}, + mailbox::{set::MailboxSet, UidMailbox}, + JmapMethods, +}; +use std::future::Future; use super::{ + delete::EmailDeletion, headers::{BuildHeader, ValueToHeader}, - ingest::{IngestEmail, IngestSource}, + ingest::{EmailIngest, IngestEmail, IngestSource}, }; -impl JMAP { - pub async fn email_set( +pub trait EmailSet: Sync + Send { + fn email_set( + &self, + request: SetRequest<RequestArguments>, + access_token: &AccessToken, + session: &HttpSessionData, + ) -> impl Future<Output = trc::Result<SetResponse>> + Send; +} + +impl EmailSet for Server { + async fn email_set( &self, mut request: SetRequest<RequestArguments>, access_token: &AccessToken, diff --git a/crates/jmap/src/email/snippet.rs b/crates/jmap/src/email/snippet.rs index dbb6e67c..bace9f17 100644 --- a/crates/jmap/src/email/snippet.rs +++ b/crates/jmap/src/email/snippet.rs @@ -4,7 +4,7 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use common::auth::AccessToken; +use common::{auth::AccessToken, Server}; use jmap_proto::{ method::{ query::Filter, @@ -16,12 +16,21 @@ use mail_parser::{decoders::html::html_to_text, GetHeader, HeaderName, PartType} use nlp::language::{search_snippet::generate_snippet, stemmer::Stemmer, Language}; use store::{backend::MAX_TOKEN_LENGTH, write::Bincode}; -use crate::JMAP; +use crate::{auth::acl::AclMethods, blob::download::BlobDownload, JmapMethods}; use super::metadata::{MessageMetadata, MetadataPartType}; +use std::future::Future; -impl JMAP { - pub async fn email_search_snippet( +pub trait EmailSearchSnippet: Sync + Send { + fn email_search_snippet( + &self, + request: GetSearchSnippetRequest, + access_token: &AccessToken, + ) -> impl Future<Output = trc::Result<GetSearchSnippetResponse>> + Send; +} + +impl EmailSearchSnippet for Server { + async fn email_search_snippet( &self, request: GetSearchSnippetRequest, access_token: &AccessToken, diff --git a/crates/jmap/src/identity/get.rs b/crates/jmap/src/identity/get.rs index f02d8e24..6b934472 100644 --- a/crates/jmap/src/identity/get.rs +++ b/crates/jmap/src/identity/get.rs @@ -4,6 +4,7 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ +use common::Server; use directory::{backend::internal::PrincipalField, QueryBy}; use jmap_proto::{ method::get::{GetRequest, GetResponse, RequestArguments}, @@ -16,12 +17,25 @@ use store::{ }; use trc::AddContext; -use crate::JMAP; +use crate::{changes::state::StateManager, JmapMethods}; use super::set::sanitize_email; +use std::future::Future; -impl JMAP { - pub async fn identity_get( +pub trait IdentityGet: Sync + Send { + fn identity_get( + &self, + request: GetRequest<RequestArguments>, + ) -> impl Future<Output = trc::Result<GetResponse>> + Send; + + fn identity_get_or_create( + &self, + account_id: u32, + ) -> impl Future<Output = trc::Result<RoaringBitmap>> + Send; +} + +impl IdentityGet for Server { + async fn identity_get( &self, mut request: GetRequest<RequestArguments>, ) -> trc::Result<GetResponse> { @@ -107,7 +121,7 @@ impl JMAP { Ok(response) } - pub async fn identity_get_or_create(&self, account_id: u32) -> trc::Result<RoaringBitmap> { + async fn identity_get_or_create(&self, account_id: u32) -> trc::Result<RoaringBitmap> { let mut identity_ids = self .get_document_ids(account_id, Collection::Identity) .await? diff --git a/crates/jmap/src/identity/set.rs b/crates/jmap/src/identity/set.rs index 3ab94e51..9e82dcb8 100644 --- a/crates/jmap/src/identity/set.rs +++ b/crates/jmap/src/identity/set.rs @@ -4,6 +4,7 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ +use common::Server; use directory::{backend::internal::PrincipalField, QueryBy}; use jmap_proto::{ error::set::SetError, @@ -16,12 +17,20 @@ use jmap_proto::{ value::{MaybePatchValue, Value}, }, }; +use std::future::Future; use store::write::{log::ChangeLogBuilder, BatchBuilder, F_CLEAR, F_VALUE}; -use crate::JMAP; +use crate::{changes::write::ChangeLog, JmapMethods}; -impl JMAP { - pub async fn identity_set( +pub trait IdentitySet: Sync + Send { + fn identity_set( + &self, + request: SetRequest<RequestArguments>, + ) -> impl Future<Output = trc::Result<SetResponse>> + Send; +} + +impl IdentitySet for Server { + async fn identity_set( &self, mut request: SetRequest<RequestArguments>, ) -> trc::Result<SetResponse> { diff --git a/crates/jmap/src/lib.rs b/crates/jmap/src/lib.rs index ad83a0bd..bd75a9fe 100644 --- a/crates/jmap/src/lib.rs +++ b/crates/jmap/src/lib.rs @@ -4,22 +4,15 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use std::{ - collections::hash_map::RandomState, - fmt::Display, - sync::{atomic::AtomicU8, Arc}, - time::Duration, -}; +use std::{fmt::Display, future::Future, sync::Arc, time::Duration}; -use auth::rate_limit::ConcurrencyLimiters; +use changes::state::StateManager; use common::{ auth::{AccessToken, ResourceToken, TenantInfo}, - manager::webadmin::WebAdminManager, - Core, DeliveryEvent, SharedCore, + manager::boot::{BootManager, IpcReceivers}, + Inner, Server, }; -use dashmap::DashMap; use directory::QueryBy; -use email::cache::Threads; use jmap_proto::{ method::{ query::{QueryRequest, QueryResponse}, @@ -28,13 +21,10 @@ use jmap_proto::{ types::{collection::Collection, property::Property}, }; use services::{ - delivery::spawn_delivery_manager, - housekeeper::{self, init_housekeeper, spawn_housekeeper}, - index::spawn_index_task, - state::{self, init_state_manager, spawn_state_manager}, + delivery::spawn_delivery_manager, housekeeper::spawn_housekeeper, index::spawn_index_task, + state::spawn_state_manager, }; -use smtp::core::SMTP; use store::{ dispatch::DocumentSet, fts::FtsFilter, @@ -46,14 +36,7 @@ use store::{ }, BitmapKey, Deserialize, IterateParams, ValueKey, U32_LEN, }; -use tokio::sync::{mpsc, Notify}; use trc::AddContext; -use utils::{ - config::Config, - lru_cache::{LruCache, LruCached}, - map::ttl_dashmap::{TtlDashMap, TtlMap}, - snowflake::SnowflakeIdGenerator, -}; pub mod api; pub mod auth; @@ -74,76 +57,24 @@ pub mod websocket; pub const LONG_SLUMBER: Duration = Duration::from_secs(60 * 60 * 24); -#[derive(Clone)] -pub struct JMAP { - pub core: Arc<Core>, - pub shared_core: SharedCore, - pub inner: Arc<Inner>, - pub smtp: SMTP, +pub trait StartServices: Sync + Send { + fn start_services(&mut self) -> impl Future<Output = ()> + Send; } -#[derive(Clone)] -pub struct JmapInstance { - pub core: SharedCore, - pub jmap_inner: Arc<Inner>, - pub smtp_inner: Arc<smtp::core::Inner>, +pub trait SpawnServices { + fn spawn_services(&mut self, inner: Arc<Inner>); } -pub struct Inner { - pub sessions: TtlDashMap<String, u32>, - pub snowflake_id: SnowflakeIdGenerator, - pub webadmin: WebAdminManager, - pub config_version: AtomicU8, - - pub concurrency_limiter: DashMap<u32, Arc<ConcurrencyLimiters>>, - - pub state_tx: mpsc::Sender<state::Event>, - pub housekeeper_tx: mpsc::Sender<housekeeper::Event>, - pub index_tx: Arc<Notify>, - - pub cache_threads: LruCache<u32, Arc<Threads>>, -} - -impl JMAP { - pub async fn init( - config: &mut Config, - delivery_rx: mpsc::Receiver<DeliveryEvent>, - core: SharedCore, - smtp_inner: Arc<smtp::core::Inner>, - ) -> JmapInstance { - // Init state manager and housekeeper - let (state_tx, state_rx) = init_state_manager(); - let (housekeeper_tx, housekeeper_rx) = init_housekeeper(); - let index_tx = Arc::new(Notify::new()); - let shard_amount = config - .property::<u64>("cache.shard") - .unwrap_or(32) - .next_power_of_two() as usize; - let capacity = config.property("cache.capacity").unwrap_or(100); - - let inner = Inner { - webadmin: WebAdminManager::new(), - sessions: TtlDashMap::with_capacity(capacity, shard_amount), - snowflake_id: config - .property::<u64>("cluster.node-id") - .map(SnowflakeIdGenerator::with_node_id) - .unwrap_or_default(), - concurrency_limiter: DashMap::with_capacity_and_hasher_and_shard_amount( - capacity, - RandomState::default(), - shard_amount, - ), - state_tx, - housekeeper_tx, - index_tx: index_tx.clone(), - cache_threads: LruCache::with_capacity( - config.property("cache.thread.size").unwrap_or(2048), - ), - config_version: 0.into(), - }; - +impl StartServices for BootManager { + async fn start_services(&mut self) { // Unpack webadmin - if let Err(err) = inner.webadmin.unpack(&core.load().storage.blob).await { + if let Err(err) = self + .inner + .data + .webadmin + .unpack(&self.inner.shared_core.load().storage.blob) + .await + { trc::event!( Resource(trc::ResourceEvent::Error), Reason = err, @@ -151,33 +82,33 @@ impl JMAP { ); } - let jmap_instance = JmapInstance { - core, - jmap_inner: Arc::new(inner), - smtp_inner, - }; + self.ipc_rxs.spawn_services(self.inner.clone()); + } +} +impl SpawnServices for IpcReceivers { + fn spawn_services(&mut self, inner: Arc<Inner>) { // Spawn delivery manager - spawn_delivery_manager(jmap_instance.clone(), delivery_rx); + spawn_delivery_manager(inner.clone(), self.delivery_rx.take().unwrap()); // Spawn state manager - spawn_state_manager(jmap_instance.clone(), state_rx); + spawn_state_manager(inner.clone(), self.state_rx.take().unwrap()); // Spawn housekeeper - spawn_housekeeper(jmap_instance.clone(), housekeeper_rx); + spawn_housekeeper(inner.clone(), self.housekeeper_rx.take().unwrap()); // Spawn index task - spawn_index_task(jmap_instance.clone(), index_tx); - - jmap_instance + spawn_index_task(inner); } +} - pub async fn get_property<U>( +impl JmapMethods for Server { + async fn get_property<U>( &self, account_id: u32, collection: Collection, document_id: u32, - property: impl AsRef<Property>, + property: impl AsRef<Property> + Sync + Send, ) -> trc::Result<Option<U>> where U: Deserialize + 'static, @@ -203,7 +134,7 @@ impl JMAP { }) } - pub async fn get_properties<U, I, P>( + async fn get_properties<U, I, P>( &self, account_id: u32, collection: Collection, @@ -212,7 +143,7 @@ impl JMAP { ) -> trc::Result<Vec<(u32, U)>> where I: DocumentSet + Send + Sync, - P: AsRef<Property>, + P: AsRef<Property> + Sync + Send, U: Deserialize + 'static, { let property: u8 = property.as_ref().into(); @@ -258,7 +189,7 @@ impl JMAP { .map(|_| results) } - pub async fn get_document_ids( + async fn get_document_ids( &self, account_id: u32, collection: Collection, @@ -275,12 +206,12 @@ impl JMAP { }) } - pub async fn get_tag( + async fn get_tag( &self, account_id: u32, collection: Collection, - property: impl AsRef<Property>, - value: impl Into<TagValue<u32>>, + property: impl AsRef<Property> + Sync + Send, + value: impl Into<TagValue<u32>> + Sync + Send, ) -> trc::Result<Option<RoaringBitmap>> { let property = property.as_ref(); self.core @@ -304,7 +235,7 @@ impl JMAP { }) } - pub async fn prepare_set_response<T>( + async fn prepare_set_response<T: Sync + Send>( &self, request: &SetRequest<T>, collection: Collection, @@ -321,7 +252,7 @@ impl JMAP { ) } - pub async fn get_resource_token( + async fn get_resource_token( &self, access_token: &AccessToken, account_id: u32, @@ -380,7 +311,7 @@ impl JMAP { }) } - pub async fn get_used_quota(&self, account_id: u32) -> trc::Result<i64> { + async fn get_used_quota(&self, account_id: u32) -> trc::Result<i64> { self.core .storage .data @@ -389,11 +320,7 @@ impl JMAP { .add_context(|err| err.caused_by(trc::location!()).account_id(account_id)) } - pub async fn has_available_quota( - &self, - quotas: &ResourceToken, - item_size: u64, - ) -> trc::Result<()> { + async fn has_available_quota(&self, quotas: &ResourceToken, item_size: u64) -> trc::Result<()> { if quotas.quota != 0 { let used_quota = self.get_used_quota(quotas.account_id).await? as u64; @@ -428,7 +355,7 @@ impl JMAP { Ok(()) } - pub async fn filter( + async fn filter( &self, account_id: u32, collection: Collection, @@ -446,7 +373,7 @@ impl JMAP { }) } - pub async fn fts_filter<T: Into<u8> + Display + Clone + std::fmt::Debug>( + async fn fts_filter<T: Into<u8> + Display + Clone + std::fmt::Debug + Sync + Send>( &self, account_id: u32, collection: Collection, @@ -464,7 +391,7 @@ impl JMAP { }) } - pub async fn build_query_response<T>( + async fn build_query_response<T: Sync + Send>( &self, result_set: &ResultSet, request: &QueryRequest<T>, @@ -513,7 +440,7 @@ impl JMAP { )) } - pub async fn sort( + async fn sort( &self, result_set: ResultSet, comparators: Vec<Comparator>, @@ -539,7 +466,7 @@ impl JMAP { Ok(response) } - pub async fn write_batch(&self, batch: BatchBuilder) -> trc::Result<AssignedIds> { + async fn write_batch(&self, batch: BatchBuilder) -> trc::Result<AssignedIds> { self.core .storage .data @@ -548,34 +475,116 @@ impl JMAP { .caused_by(trc::location!()) } - pub async fn write_batch_expect_id(&self, batch: BatchBuilder) -> trc::Result<u32> { + async fn write_batch_expect_id(&self, batch: BatchBuilder) -> trc::Result<u32> { self.write_batch(batch) .await .and_then(|ids| ids.last_document_id().caused_by(trc::location!())) } -} -impl Inner { - pub fn increment_config_version(&self) { - self.config_version + fn increment_config_version(&self) { + self.inner + .data + .config_version .fetch_add(1, std::sync::atomic::Ordering::Relaxed); } } -impl From<JmapInstance> for JMAP { - fn from(value: JmapInstance) -> Self { - let shared_core = value.core.clone(); - let core = value.core.load_full(); - JMAP { - smtp: SMTP { - core: core.clone(), - inner: value.smtp_inner, - }, - core, - shared_core, - inner: value.jmap_inner, - } - } +pub trait JmapMethods: Sync + Send { + fn get_property<U>( + &self, + account_id: u32, + collection: Collection, + document_id: u32, + property: impl AsRef<Property> + Sync + Send, + ) -> impl Future<Output = trc::Result<Option<U>>> + Send + where + U: Deserialize + 'static; + + fn get_properties<U, I, P>( + &self, + account_id: u32, + collection: Collection, + iterate: &I, + property: P, + ) -> impl Future<Output = trc::Result<Vec<(u32, U)>>> + Send + where + I: DocumentSet + Send + Sync, + P: AsRef<Property> + Sync + Send, + U: Deserialize + 'static; + + fn get_document_ids( + &self, + account_id: u32, + collection: Collection, + ) -> impl Future<Output = trc::Result<Option<RoaringBitmap>>> + Send; + + fn get_tag( + &self, + account_id: u32, + collection: Collection, + property: impl AsRef<Property> + Sync + Send, + value: impl Into<TagValue<u32>> + Sync + Send, + ) -> impl Future<Output = trc::Result<Option<RoaringBitmap>>> + Send; + + fn prepare_set_response<T: Sync + Send>( + &self, + request: &SetRequest<T>, + collection: Collection, + ) -> impl Future<Output = trc::Result<SetResponse>> + Send; + + fn get_resource_token( + &self, + access_token: &AccessToken, + account_id: u32, + ) -> impl Future<Output = trc::Result<ResourceToken>> + Send; + + fn get_used_quota(&self, account_id: u32) -> impl Future<Output = trc::Result<i64>> + Send; + + fn has_available_quota( + &self, + quotas: &ResourceToken, + item_size: u64, + ) -> impl Future<Output = trc::Result<()>> + Send; + + fn filter( + &self, + account_id: u32, + collection: Collection, + filters: Vec<Filter>, + ) -> impl Future<Output = trc::Result<ResultSet>> + Send; + + fn fts_filter<T: Into<u8> + Display + Clone + std::fmt::Debug + Sync + Send>( + &self, + account_id: u32, + collection: Collection, + filters: Vec<FtsFilter<T>>, + ) -> impl Future<Output = trc::Result<RoaringBitmap>> + Send; + + fn build_query_response<T: Sync + Send>( + &self, + result_set: &ResultSet, + request: &QueryRequest<T>, + ) -> impl Future<Output = trc::Result<(QueryResponse, Option<Pagination>)>> + Send; + + fn sort( + &self, + result_set: ResultSet, + comparators: Vec<Comparator>, + paginate: Pagination, + response: QueryResponse, + ) -> impl Future<Output = trc::Result<QueryResponse>> + Send; + + fn write_batch( + &self, + batch: BatchBuilder, + ) -> impl Future<Output = trc::Result<AssignedIds>> + Send; + + fn write_batch_expect_id( + &self, + batch: BatchBuilder, + ) -> impl Future<Output = trc::Result<u32>> + Send; + + fn increment_config_version(&self); } trait UpdateResults: Sized { diff --git a/crates/jmap/src/mailbox/get.rs b/crates/jmap/src/mailbox/get.rs index 069b678d..94f93304 100644 --- a/crates/jmap/src/mailbox/get.rs +++ b/crates/jmap/src/mailbox/get.rs @@ -4,7 +4,7 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use common::auth::AccessToken; +use common::{auth::AccessToken, Server}; use jmap_proto::{ method::get::{GetRequest, GetResponse, RequestArguments}, object::Object, @@ -13,12 +13,58 @@ use jmap_proto::{ use store::{ahash::AHashSet, query::Filter, roaring::RoaringBitmap}; use trc::AddContext; -use crate::{auth::acl::EffectiveAcl, JMAP}; +use crate::{ + auth::acl::{AclMethods, EffectiveAcl}, + changes::state::StateManager, + email::cache::ThreadCache, + JmapMethods, +}; -use super::INBOX_ID; +use super::{set::MailboxSet, INBOX_ID}; +use std::future::Future; -impl JMAP { - pub async fn mailbox_get( +pub trait MailboxGet: Sync + Send { + fn mailbox_get( + &self, + request: GetRequest<RequestArguments>, + access_token: &AccessToken, + ) -> impl Future<Output = trc::Result<GetResponse>> + Send; + + fn mailbox_count_threads( + &self, + account_id: u32, + document_ids: Option<RoaringBitmap>, + ) -> impl Future<Output = trc::Result<usize>> + Send; + + fn mailbox_unread_tags( + &self, + account_id: u32, + document_id: u32, + message_ids: &Option<RoaringBitmap>, + ) -> impl Future<Output = trc::Result<Option<RoaringBitmap>>> + Send; + + fn mailbox_expand_path<'x>( + &self, + account_id: u32, + path: &'x str, + exact_match: bool, + ) -> impl Future<Output = trc::Result<Option<ExpandPath<'x>>>> + Send; + + fn mailbox_get_by_name( + &self, + account_id: u32, + path: &str, + ) -> impl Future<Output = trc::Result<Option<u32>>> + Send; + + fn mailbox_get_by_role( + &self, + account_id: u32, + role: &str, + ) -> impl Future<Output = trc::Result<Option<u32>>> + Send; +} + +impl MailboxGet for Server { + async fn mailbox_get( &self, mut request: GetRequest<RequestArguments>, access_token: &AccessToken, @@ -257,7 +303,7 @@ impl JMAP { } } - pub async fn mailbox_unread_tags( + async fn mailbox_unread_tags( &self, account_id: u32, document_id: u32, @@ -297,7 +343,7 @@ impl JMAP { } } - pub async fn mailbox_expand_path<'x>( + async fn mailbox_expand_path<'x>( &self, account_id: u32, path: &'x str, @@ -376,11 +422,7 @@ impl JMAP { Ok(Some(ExpandPath { path, found_names })) } - pub async fn mailbox_get_by_name( - &self, - account_id: u32, - path: &str, - ) -> trc::Result<Option<u32>> { + async fn mailbox_get_by_name(&self, account_id: u32, path: &str) -> trc::Result<Option<u32>> { Ok(self .mailbox_expand_path(account_id, path, true) .await? @@ -403,11 +445,7 @@ impl JMAP { })) } - pub async fn mailbox_get_by_role( - &self, - account_id: u32, - role: &str, - ) -> trc::Result<Option<u32>> { + async fn mailbox_get_by_role(&self, account_id: u32, role: &str) -> trc::Result<Option<u32>> { self.filter( account_id, Collection::Mailbox, diff --git a/crates/jmap/src/mailbox/query.rs b/crates/jmap/src/mailbox/query.rs index 16295772..33263db9 100644 --- a/crates/jmap/src/mailbox/query.rs +++ b/crates/jmap/src/mailbox/query.rs @@ -4,7 +4,7 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use common::auth::AccessToken; +use common::{auth::AccessToken, Server}; use jmap_proto::{ method::query::{Comparator, Filter, QueryRequest, QueryResponse, SortProperty}, object::{mailbox::QueryArguments, Object}, @@ -16,10 +16,21 @@ use store::{ roaring::RoaringBitmap, }; -use crate::{UpdateResults, JMAP}; +use crate::{auth::acl::AclMethods, JmapMethods, UpdateResults}; +use std::future::Future; -impl JMAP { - pub async fn mailbox_query( +use super::set::MailboxSet; + +pub trait MailboxQuery: Sync + Send { + fn mailbox_query( + &self, + request: QueryRequest<QueryArguments>, + access_token: &AccessToken, + ) -> impl Future<Output = trc::Result<QueryResponse>> + Send; +} + +impl MailboxQuery for Server { + async fn mailbox_query( &self, mut request: QueryRequest<QueryArguments>, access_token: &AccessToken, diff --git a/crates/jmap/src/mailbox/set.rs b/crates/jmap/src/mailbox/set.rs index 91ee0216..beaca13d 100644 --- a/crates/jmap/src/mailbox/set.rs +++ b/crates/jmap/src/mailbox/set.rs @@ -4,7 +4,7 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use common::{auth::AccessToken, config::jmap::settings::SpecialUse}; +use common::{auth::AccessToken, config::jmap::settings::SpecialUse, Server}; use directory::Permission; use jmap_proto::{ error::set::{SetError, SetErrorType}, @@ -36,13 +36,19 @@ use store::{ }; use trc::AddContext; -use crate::{auth::acl::EffectiveAcl, JMAP}; +use crate::{ + auth::acl::{AclMethods, EffectiveAcl}, + changes::write::ChangeLog, + email::delete::EmailDeletion, + JmapMethods, +}; +use super::{get::MailboxGet, ARCHIVE_ID, DRAFTS_ID, SENT_ID}; #[allow(unused_imports)] use super::{UidMailbox, INBOX_ID, JUNK_ID, TRASH_ID}; -use super::{ARCHIVE_ID, DRAFTS_ID, SENT_ID}; +use std::future::Future; -struct SetContext<'x> { +pub struct SetContext<'x> { account_id: u32, access_token: &'x AccessToken, is_shared: bool, @@ -69,9 +75,44 @@ pub static SCHEMA: &[IndexProperty] = &[ IndexProperty::new(Property::Acl).index_as(IndexAs::Acl), ]; -impl JMAP { +pub trait MailboxSet: Sync + Send { + fn mailbox_set( + &self, + request: SetRequest<SetArguments>, + access_token: &AccessToken, + ) -> impl Future<Output = trc::Result<SetResponse>> + Send; + + fn mailbox_destroy( + &self, + account_id: u32, + document_id: u32, + changes: &mut ChangeLogBuilder, + access_token: &AccessToken, + remove_emails: bool, + ) -> impl Future<Output = trc::Result<Result<bool, SetError>>> + Send; + + fn mailbox_set_item( + &self, + changes_: Object<SetValue>, + update: Option<(u32, HashedValue<Object<Value>>)>, + ctx: &SetContext, + ) -> impl Future<Output = trc::Result<Result<ObjectIndexBuilder, SetError>>> + Send; + + fn mailbox_get_or_create( + &self, + account_id: u32, + ) -> impl Future<Output = trc::Result<RoaringBitmap>> + Send; + + fn mailbox_create_path( + &self, + account_id: u32, + path: &str, + ) -> impl Future<Output = trc::Result<Option<(u32, Option<u64>)>>> + Send; +} + +impl MailboxSet for Server { #[allow(clippy::blocks_in_conditions)] - pub async fn mailbox_set( + async fn mailbox_set( &self, mut request: SetRequest<SetArguments>, access_token: &AccessToken, @@ -283,7 +324,7 @@ impl JMAP { Ok(ctx.response) } - pub async fn mailbox_destroy( + async fn mailbox_destroy( &self, account_id: u32, document_id: u32, @@ -761,7 +802,7 @@ impl JMAP { .validate()) } - pub async fn mailbox_get_or_create(&self, account_id: u32) -> trc::Result<RoaringBitmap> { + async fn mailbox_get_or_create(&self, account_id: u32) -> trc::Result<RoaringBitmap> { let mut mailbox_ids = self .get_document_ids(account_id, Collection::Mailbox) .await? @@ -828,7 +869,7 @@ impl JMAP { .map(|_| mailbox_ids) } - pub async fn mailbox_create_path( + async fn mailbox_create_path( &self, account_id: u32, path: &str, diff --git a/crates/jmap/src/principal/get.rs b/crates/jmap/src/principal/get.rs index b59abd77..f17d1198 100644 --- a/crates/jmap/src/principal/get.rs +++ b/crates/jmap/src/principal/get.rs @@ -4,17 +4,26 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ +use common::Server; use directory::{backend::internal::PrincipalField, QueryBy}; use jmap_proto::{ method::get::{GetRequest, GetResponse, RequestArguments}, object::Object, types::{collection::Collection, property::Property, state::State, value::Value}, }; +use std::future::Future; -use crate::JMAP; +use crate::JmapMethods; -impl JMAP { - pub async fn principal_get( +pub trait PrincipalGet: Sync + Send { + fn principal_get( + &self, + request: GetRequest<RequestArguments>, + ) -> impl Future<Output = trc::Result<GetResponse>> + Send; +} + +impl PrincipalGet for Server { + async fn principal_get( &self, mut request: GetRequest<RequestArguments>, ) -> trc::Result<GetResponse> { diff --git a/crates/jmap/src/principal/query.rs b/crates/jmap/src/principal/query.rs index 0ed616ea..fa9793ee 100644 --- a/crates/jmap/src/principal/query.rs +++ b/crates/jmap/src/principal/query.rs @@ -4,6 +4,7 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ +use common::Server; use directory::QueryBy; use jmap_proto::{ method::query::{Filter, QueryRequest, QueryResponse, RequestArguments}, @@ -11,10 +12,19 @@ use jmap_proto::{ }; use store::{query::ResultSet, roaring::RoaringBitmap}; -use crate::{api::http::HttpSessionData, JMAP}; +use crate::{api::http::HttpSessionData, JmapMethods}; +use std::future::Future; -impl JMAP { - pub async fn principal_query( +pub trait PrincipalQuery: Sync + Send { + fn principal_query( + &self, + request: QueryRequest<RequestArguments>, + session: &HttpSessionData, + ) -> impl Future<Output = trc::Result<QueryResponse>> + Send; +} + +impl PrincipalQuery for Server { + async fn principal_query( &self, mut request: QueryRequest<RequestArguments>, session: &HttpSessionData, @@ -51,7 +61,6 @@ impl JMAP { Filter::Email(email) => { let mut ids = RoaringBitmap::new(); for id in self - .core .email_to_ids(&self.core.storage.directory, &email, session.session_id) .await? { diff --git a/crates/jmap/src/push/get.rs b/crates/jmap/src/push/get.rs index 175b8be6..41692f9b 100644 --- a/crates/jmap/src/push/get.rs +++ b/crates/jmap/src/push/get.rs @@ -5,7 +5,11 @@ */ use base64::{engine::general_purpose, Engine}; -use common::auth::AccessToken; +use common::{ + auth::AccessToken, + ipc::{StateEvent, UpdateSubscription}, + Server, +}; use jmap_proto::{ method::get::{GetRequest, GetResponse, RequestArguments}, object::Object, @@ -17,12 +21,26 @@ use store::{ }; use utils::map::bitmap::Bitmap; -use crate::{services::state, JMAP}; +use crate::JmapMethods; + +use super::{EncryptionKeys, PushSubscription}; +use std::future::Future; -use super::{EncryptionKeys, PushSubscription, UpdateSubscription}; +pub trait PushSubscriptionFetch: Sync + Send { + fn push_subscription_get( + &self, + request: GetRequest<RequestArguments>, + access_token: &AccessToken, + ) -> impl Future<Output = trc::Result<GetResponse>> + Send; + + fn fetch_push_subscriptions( + &self, + account_id: u32, + ) -> impl Future<Output = trc::Result<StateEvent>> + Send; +} -impl JMAP { - pub async fn push_subscription_get( +impl PushSubscriptionFetch for Server { + async fn push_subscription_get( &self, mut request: GetRequest<RequestArguments>, access_token: &AccessToken, @@ -99,7 +117,7 @@ impl JMAP { Ok(response) } - pub async fn fetch_push_subscriptions(&self, account_id: u32) -> trc::Result<state::Event> { + async fn fetch_push_subscriptions(&self, account_id: u32) -> trc::Result<StateEvent> { let mut subscriptions = Vec::new(); let document_ids = self .core @@ -235,7 +253,7 @@ impl JMAP { } } - Ok(state::Event::UpdateSubscriptions { + Ok(StateEvent::UpdateSubscriptions { account_id, subscriptions, }) diff --git a/crates/jmap/src/push/manager.rs b/crates/jmap/src/push/manager.rs index ba7acbfd..c543f400 100644 --- a/crates/jmap/src/push/manager.rs +++ b/crates/jmap/src/push/manager.rs @@ -5,23 +5,24 @@ */ use base64::{engine::general_purpose, Engine}; -use common::IPC_CHANNEL_BUFFER; +use common::{core::BuildServer, Inner, IPC_CHANNEL_BUFFER}; use jmap_proto::types::id::Id; use store::ahash::{AHashMap, AHashSet}; use tokio::sync::mpsc; use trc::PushSubscriptionEvent; -use crate::{api::StateChangeResponse, JmapInstance, LONG_SLUMBER}; +use crate::{api::StateChangeResponse, LONG_SLUMBER}; use super::{ece::ece_encrypt, EncryptionKeys, Event, PushServer, PushUpdate}; use reqwest::header::{CONTENT_ENCODING, CONTENT_TYPE}; use std::{ collections::hash_map::Entry, + sync::Arc, time::{Duration, Instant}, }; -pub fn spawn_push_manager(core: JmapInstance) -> mpsc::Sender<Event> { +pub fn spawn_push_manager(inner: Arc<Inner>) -> mpsc::Sender<Event> { let (push_tx_, mut push_rx) = mpsc::channel::<Event>(IPC_CHANNEL_BUFFER); let push_tx = push_tx_.clone(); @@ -37,13 +38,13 @@ pub fn spawn_push_manager(core: JmapInstance) -> mpsc::Sender<Event> { let event_or_timeout = tokio::time::timeout(retry_timeout, push_rx.recv()).await; // Load settings - let core_ = core.core.load_full(); - let push_attempt_interval = core_.jmap.push_attempt_interval; - let push_attempts_max = core_.jmap.push_attempts_max; - let push_retry_interval = core_.jmap.push_retry_interval; - let push_timeout = core_.jmap.push_timeout; - let push_verify_timeout = core_.jmap.push_verify_timeout; - let push_throttle = core_.jmap.push_throttle; + let server = inner.build_server(); + let push_attempt_interval = server.core.jmap.push_attempt_interval; + let push_attempts_max = server.core.jmap.push_attempts_max; + let push_retry_interval = server.core.jmap.push_retry_interval; + let push_timeout = server.core.jmap.push_timeout; + let push_verify_timeout = server.core.jmap.push_verify_timeout; + let push_throttle = server.core.jmap.push_throttle; match event_or_timeout { Ok(Some(event)) => match event { diff --git a/crates/jmap/src/push/mod.rs b/crates/jmap/src/push/mod.rs index 9e6dae69..65669632 100644 --- a/crates/jmap/src/push/mod.rs +++ b/crates/jmap/src/push/mod.rs @@ -11,34 +11,8 @@ pub mod set; use std::time::Instant; -use jmap_proto::types::{id::Id, state::StateChange, type_state::DataType}; -use utils::map::bitmap::Bitmap; - -#[derive(Debug)] -pub enum UpdateSubscription { - Unverified { - id: u32, - url: String, - code: String, - keys: Option<EncryptionKeys>, - }, - Verified(PushSubscription), -} - -#[derive(Debug)] -pub struct PushSubscription { - pub id: u32, - pub url: String, - pub expires: u64, - pub types: Bitmap<DataType>, - pub keys: Option<EncryptionKeys>, -} - -#[derive(Debug, Clone)] -pub struct EncryptionKeys { - pub p256dh: Vec<u8>, - pub auth: Vec<u8>, -} +use common::ipc::{EncryptionKeys, PushSubscription}; +use jmap_proto::types::{id::Id, state::StateChange}; #[derive(Debug)] pub enum Event { diff --git a/crates/jmap/src/push/set.rs b/crates/jmap/src/push/set.rs index 22f56f3e..d23b9436 100644 --- a/crates/jmap/src/push/set.rs +++ b/crates/jmap/src/push/set.rs @@ -5,7 +5,7 @@ */ use base64::{engine::general_purpose, Engine}; -use common::auth::AccessToken; +use common::{auth::AccessToken, Server}; use jmap_proto::{ error::set::SetError, method::set::{RequestArguments, SetRequest, SetResponse}, @@ -19,18 +19,27 @@ use jmap_proto::{ value::{MaybePatchValue, Value}, }, }; +use std::future::Future; use store::{ rand::{distributions::Alphanumeric, thread_rng, Rng}, write::{now, BatchBuilder, F_CLEAR, F_VALUE}, }; -use crate::JMAP; +use crate::{services::state::StateManager, JmapMethods}; const EXPIRES_MAX: i64 = 7 * 24 * 3600; // 7 days const VERIFICATION_CODE_LEN: usize = 32; -impl JMAP { - pub async fn push_subscription_set( +pub trait PushSubscriptionSet: Sync + Send { + fn push_subscription_set( + &self, + request: SetRequest<RequestArguments>, + access_token: &AccessToken, + ) -> impl Future<Output = trc::Result<SetResponse>> + Send; +} + +impl PushSubscriptionSet for Server { + async fn push_subscription_set( &self, mut request: SetRequest<RequestArguments>, access_token: &AccessToken, diff --git a/crates/jmap/src/quota/get.rs b/crates/jmap/src/quota/get.rs index 193c77fb..a4edeb52 100644 --- a/crates/jmap/src/quota/get.rs +++ b/crates/jmap/src/quota/get.rs @@ -4,17 +4,26 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use common::auth::AccessToken; +use common::{auth::AccessToken, Server}; use jmap_proto::{ method::get::{GetRequest, GetResponse, RequestArguments}, object::Object, types::{id::Id, property::Property, state::State, type_state::DataType, value::Value}, }; +use std::future::Future; -use crate::JMAP; +use crate::JmapMethods; -impl JMAP { - pub async fn quota_get( +pub trait QuotaGet: Sync + Send { + fn quota_get( + &self, + request: GetRequest<RequestArguments>, + access_token: &AccessToken, + ) -> impl Future<Output = trc::Result<GetResponse>> + Send; +} + +impl QuotaGet for Server { + async fn quota_get( &self, mut request: GetRequest<RequestArguments>, access_token: &AccessToken, diff --git a/crates/jmap/src/quota/query.rs b/crates/jmap/src/quota/query.rs index 40d65a52..dae4ce1e 100644 --- a/crates/jmap/src/quota/query.rs +++ b/crates/jmap/src/quota/query.rs @@ -4,16 +4,23 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use common::auth::AccessToken; +use common::{auth::AccessToken, Server}; use jmap_proto::{ method::query::{QueryRequest, QueryResponse, RequestArguments}, types::{id::Id, state::State}, }; +use std::future::Future; -use crate::JMAP; +pub trait QuotaQuery: Sync + Send { + fn quota_query( + &self, + request: QueryRequest<RequestArguments>, + access_token: &AccessToken, + ) -> impl Future<Output = trc::Result<QueryResponse>> + Send; +} -impl JMAP { - pub async fn quota_query( +impl QuotaQuery for Server { + async fn quota_query( &self, request: QueryRequest<RequestArguments>, access_token: &AccessToken, diff --git a/crates/jmap/src/quota/set.rs b/crates/jmap/src/quota/set.rs index 6aec092a..4a08f11c 100644 --- a/crates/jmap/src/quota/set.rs +++ b/crates/jmap/src/quota/set.rs @@ -8,14 +8,16 @@ use jmap_proto::{ object::index::{IndexAs, IndexProperty}, types::property::Property, }; +use std::future::Future; -use crate::JMAP; - -impl JMAP { - pub async fn quota_set( +pub trait QuotaSet: Sync + Send { + fn quota_set( &self, account_id: u32, quota: &AccessToken, - ) -> trc::Result<SetResponse> { - } + ) -> impl Future<Output = trc::Result<SetResponse>> + Send; +} + +impl QuotaSet for Server { + async fn quota_set(&self, account_id: u32, quota: &AccessToken) -> trc::Result<SetResponse> {} } diff --git a/crates/jmap/src/services/delivery.rs b/crates/jmap/src/services/delivery.rs index fc732125..b3f8b674 100644 --- a/crates/jmap/src/services/delivery.rs +++ b/crates/jmap/src/services/delivery.rs @@ -4,18 +4,20 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use common::DeliveryEvent; +use std::sync::Arc; + +use common::{core::BuildServer, ipc::DeliveryEvent, Inner}; use tokio::sync::mpsc; -use crate::{JmapInstance, JMAP}; +use super::ingest::MailDelivery; -pub fn spawn_delivery_manager(core: JmapInstance, mut delivery_rx: mpsc::Receiver<DeliveryEvent>) { +pub fn spawn_delivery_manager(inner: Arc<Inner>, mut delivery_rx: mpsc::Receiver<DeliveryEvent>) { tokio::spawn(async move { while let Some(event) = delivery_rx.recv().await { match event { DeliveryEvent::Ingest { message, result_tx } => { result_tx - .send(JMAP::from(core.clone()).deliver_message(message).await) + .send(inner.build_server().deliver_message(message).await) .ok(); } DeliveryEvent::Stop => break, diff --git a/crates/jmap/src/services/gossip/mod.rs b/crates/jmap/src/services/gossip/mod.rs index def13548..959e1fe2 100644 --- a/crates/jmap/src/services/gossip/mod.rs +++ b/crates/jmap/src/services/gossip/mod.rs @@ -11,17 +11,16 @@ pub mod ping; pub mod request; pub mod spawn; +use common::Inner; use serde::{Deserialize, Serialize}; use std::{ net::{IpAddr, SocketAddr}, - sync::atomic::Ordering, + sync::{atomic::Ordering, Arc}, time::Instant, }; use tokio::sync::mpsc; use trc::ClusterEvent; -use crate::JmapInstance; - use self::request::Request; const UDP_MAX_PAYLOAD: usize = 65500; @@ -44,7 +43,7 @@ pub struct Gossiper { pub last_peer_pinged: usize, // IPC - pub core: JmapInstance, + pub inner: Arc<Inner>, pub gossip_tx: mpsc::Sender<(SocketAddr, Request)>, } @@ -101,23 +100,26 @@ impl From<&Peer> for PeerStatus { impl From<&Gossiper> for PeerStatus { fn from(cluster: &Gossiper) -> Self { - let core = cluster.core.core.load(); PeerStatus { addr: cluster.addr, epoch: cluster.epoch, - gen_config: cluster - .core - .jmap_inner - .config_version + gen_config: cluster.inner.data.config_version.load(Ordering::Relaxed), + gen_lists: cluster + .inner + .data + .blocked_ips_version + .load(Ordering::Relaxed), + gen_permissions: cluster + .inner + .data + .permissions_version .load(Ordering::Relaxed), - gen_lists: core.network.blocked_ips.version.load(Ordering::Relaxed), - gen_permissions: core.security.permissions_version.load(Ordering::Relaxed), } } } impl Gossiper { - pub async fn send_gossip(&self, dest: IpAddr, request: Request) { + async fn send_gossip(&self, dest: IpAddr, request: Request) { if let Err(err) = self .gossip_tx .send((SocketAddr::new(dest, self.port), request)) diff --git a/crates/jmap/src/services/gossip/ping.rs b/crates/jmap/src/services/gossip/ping.rs index ab30f88c..dc05a7e3 100644 --- a/crates/jmap/src/services/gossip/ping.rs +++ b/crates/jmap/src/services/gossip/ping.rs @@ -4,10 +4,13 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use smtp::queue; +use common::{ + core::BuildServer, + ipc::{HousekeeperEvent, QueueEvent}, +}; use trc::ClusterEvent; -use crate::services::housekeeper; +use crate::services::index::Indexer; use super::{request::Request, Gossiper, PeerStatus}; @@ -66,13 +69,13 @@ impl Gossiper { } pub fn request_reload(&self) { - let core = self.core.clone(); + let server = self.inner.build_server(); tokio::spawn(async move { trc::event!(Cluster(ClusterEvent::OneOrMorePeersOffline)); - core.jmap_inner.request_fts_index(); - let _ = core.smtp_inner.queue_tx.send(queue::Event::Reload).await; + server.request_fts_index(); + let _ = server.inner.ipc.queue_tx.send(QueueEvent::Reload).await; }); } @@ -174,29 +177,30 @@ impl Gossiper { // Reload settings if update_permissions { - self.core.core.load().security.permissions.clear(); + self.inner.data.permissions.clear(); } if update_config || update_lists { - let core = self.core.core.clone(); - let inner = self.core.jmap_inner.clone(); + let server = self.inner.build_server(); tokio::spawn(async move { let result = if update_config { - core.load().reload().await + server.reload().await } else { - core.load().reload_blocked_ips().await + server.reload_blocked_ips().await }; match result { Ok(result) => { if let Some(new_core) = result.new_core { // Update core - core.store(new_core.into()); + server.inner.shared_core.store(new_core.into()); // Reload ACME - if inner + if server + .inner + .ipc .housekeeper_tx - .send(housekeeper::Event::ReloadSettings) + .send(HousekeeperEvent::ReloadSettings) .await .is_err() { diff --git a/crates/jmap/src/services/gossip/spawn.rs b/crates/jmap/src/services/gossip/spawn.rs index 579cf40a..f5d41056 100644 --- a/crates/jmap/src/services/gossip/spawn.rs +++ b/crates/jmap/src/services/gossip/spawn.rs @@ -5,11 +5,10 @@ */ use crate::auth::SymmetricEncrypt; -use crate::JmapInstance; use super::request::Request; use super::{Gossiper, Peer, UDP_MAX_PAYLOAD}; -use common::IPC_CHANNEL_BUFFER; +use common::{Inner, IPC_CHANNEL_BUFFER}; use std::net::IpAddr; use std::time::{Duration, Instant}; use std::{net::SocketAddr, sync::Arc}; @@ -64,7 +63,7 @@ impl GossiperBuilder { builder.into() } - pub async fn spawn(self, core: JmapInstance, mut shutdown_rx: watch::Receiver<bool>) { + pub async fn spawn(self, inner: Arc<Inner>, mut shutdown_rx: watch::Receiver<bool>) { // Bind port let quidnunc = Arc::new(Quidnunc { socket: match UdpSocket::bind(SocketAddr::new(self.bind_addr, self.port)).await { @@ -100,7 +99,7 @@ impl GossiperBuilder { epoch: 0, peers: self.peers, last_peer_pinged: u32::MAX as usize, - core, + inner, gossip_tx, }; let quidnunc_ = quidnunc.clone(); diff --git a/crates/jmap/src/services/housekeeper.rs b/crates/jmap/src/services/housekeeper.rs index ed05f8c6..4387146a 100644 --- a/crates/jmap/src/services/housekeeper.rs +++ b/crates/jmap/src/services/housekeeper.rs @@ -6,10 +6,16 @@ use std::{ collections::BinaryHeap, + sync::{atomic::Ordering, Arc}, time::{Duration, Instant, SystemTime}, }; -use common::{config::telemetry::OtelMetrics, IPC_CHANNEL_BUFFER}; +use common::{ + config::telemetry::OtelMetrics, + core::BuildServer, + ipc::{HousekeeperEvent, PurgeType}, + Inner, +}; #[cfg(feature = "enterprise")] use common::telemetry::{ @@ -17,33 +23,13 @@ use common::telemetry::{ tracers::store::TracingStore, }; -use smtp::core::SMTP; -use store::{ - write::{now, purge::PurgeStore}, - BlobStore, LookupStore, Store, -}; +use smtp::reporting::SmtpReporting; +use store::write::{now, purge::PurgeStore}; use tokio::sync::mpsc; -use trc::{Collector, HousekeeperEvent, MetricType}; +use trc::{Collector, MetricType}; use utils::map::ttl_dashmap::TtlMap; -use crate::{Inner, JmapInstance, JMAP, LONG_SLUMBER}; - -pub enum Event { - AcmeReschedule { - provider_id: String, - renew_at: Instant, - }, - Purge(PurgeType), - ReloadSettings, - Exit, -} - -pub enum PurgeType { - Data(Store), - Blobs { store: Store, blob_store: BlobStore }, - Lookup(LookupStore), - Account(Option<u32>), -} +use crate::{email::delete::EmailDeletion, JmapMethods, LONG_SLUMBER}; #[derive(PartialEq, Eq)] struct Action { @@ -75,30 +61,30 @@ struct Queue { #[cfg(feature = "enterprise")] const METRIC_ALERTS_INTERVAL: Duration = Duration::from_secs(5 * 60); -pub fn spawn_housekeeper(core: JmapInstance, mut rx: mpsc::Receiver<Event>) { +pub fn spawn_housekeeper(inner: Arc<Inner>, mut rx: mpsc::Receiver<HousekeeperEvent>) { tokio::spawn(async move { - trc::event!(Housekeeper(HousekeeperEvent::Start)); + trc::event!(Housekeeper(trc::HousekeeperEvent::Start)); let start_time = SystemTime::now(); // Add all events to queue let mut queue = Queue::default(); { - let core_ = core.core.load_full(); + let server = inner.build_server(); // Session purge queue.schedule( - Instant::now() + core_.jmap.session_purge_frequency.time_to_next(), + Instant::now() + server.core.jmap.session_purge_frequency.time_to_next(), ActionClass::Session, ); // Account purge queue.schedule( - Instant::now() + core_.jmap.account_purge_frequency.time_to_next(), + Instant::now() + server.core.jmap.account_purge_frequency.time_to_next(), ActionClass::Account, ); // Store purges - for (idx, schedule) in core_.storage.purge_schedules.iter().enumerate() { + for (idx, schedule) in server.core.storage.purge_schedules.iter().enumerate() { queue.schedule( Instant::now() + schedule.cron.time_to_next(), ActionClass::Store(idx), @@ -106,7 +92,7 @@ pub fn spawn_housekeeper(core: JmapInstance, mut rx: mpsc::Receiver<Event>) { } // OTEL Push Metrics - if let Some(otel) = &core_.metrics.otel { + if let Some(otel) = &server.core.metrics.otel { OtelMetrics::enable_errors(); queue.schedule(Instant::now() + otel.interval, ActionClass::OtelMetrics); } @@ -115,8 +101,8 @@ pub fn spawn_housekeeper(core: JmapInstance, mut rx: mpsc::Receiver<Event>) { queue.schedule(Instant::now(), ActionClass::CalculateMetrics); // Add all ACME renewals to heap - for provider in core_.tls.acme_providers.values() { - match core_.init_acme(provider).await { + for provider in server.core.acme.providers.values() { + match server.init_acme(provider).await { Ok(renew_at) => { queue.schedule( Instant::now() + renew_at, @@ -135,7 +121,7 @@ pub fn spawn_housekeeper(core: JmapInstance, mut rx: mpsc::Receiver<Event>) { // Enterprise Edition license management #[cfg(feature = "enterprise")] - if let Some(enterprise) = &core_.enterprise { + if let Some(enterprise) = &server.core.enterprise { queue.schedule( Instant::now() + enterprise.license.expires_in(), ActionClass::ValidateLicense, @@ -166,12 +152,11 @@ pub fn spawn_housekeeper(core: JmapInstance, mut rx: mpsc::Receiver<Event>) { loop { match tokio::time::timeout(queue.wake_up_time(), rx.recv()).await { Ok(Some(event)) => match event { - Event::ReloadSettings => { - let core_ = core.core.load_full(); - let inner = core.jmap_inner.clone(); + HousekeeperEvent::ReloadSettings => { + let server = inner.build_server(); // Reload OTEL push metrics - match &core_.metrics.otel { + match &server.core.metrics.otel { Some(otel) if !queue.has_action(&ActionClass::OtelMetrics) => { OtelMetrics::enable_errors(); @@ -187,7 +172,7 @@ pub fn spawn_housekeeper(core: JmapInstance, mut rx: mpsc::Receiver<Event>) { // SPDX-FileCopyrightText: 2020 Stalwart Labs Ltd <hello@stalw.art> // SPDX-License-Identifier: LicenseRef-SEL #[cfg(feature = "enterprise")] - if let Some(enterprise) = &core_.enterprise { + if let Some(enterprise) = &server.core.enterprise { if !queue.has_action(&ActionClass::ValidateLicense) { queue.schedule( Instant::now() + enterprise.license.expires_in(), @@ -214,12 +199,14 @@ pub fn spawn_housekeeper(core: JmapInstance, mut rx: mpsc::Receiver<Event>) { // Reload ACME certificates tokio::spawn(async move { - for provider in core_.tls.acme_providers.values() { - match core_.init_acme(provider).await { + for provider in server.core.acme.providers.values() { + match server.init_acme(provider).await { Ok(renew_at) => { - inner + server + .inner + .ipc .housekeeper_tx - .send(Event::AcmeReschedule { + .send(HousekeeperEvent::AcmeReschedule { provider_id: provider.id.clone(), renew_at: Instant::now() + renew_at, }) @@ -234,7 +221,7 @@ pub fn spawn_housekeeper(core: JmapInstance, mut rx: mpsc::Receiver<Event>) { } }); } - Event::AcmeReschedule { + HousekeeperEvent::AcmeReschedule { provider_id, renew_at, } => { @@ -242,22 +229,22 @@ pub fn spawn_housekeeper(core: JmapInstance, mut rx: mpsc::Receiver<Event>) { queue.remove_action(&action); queue.schedule(renew_at, action); } - Event::Purge(purge) => match purge { + HousekeeperEvent::Purge(purge) => match purge { PurgeType::Data(store) => { // SPDX-SnippetBegin // SPDX-FileCopyrightText: 2020 Stalwart Labs Ltd <hello@stalw.art> // SPDX-License-Identifier: LicenseRef-SEL #[cfg(feature = "enterprise")] - let trace_retention = core - .core + let trace_retention = inner + .shared_core .load() .enterprise .as_ref() .and_then(|e| e.trace_store.as_ref()) .and_then(|t| t.retention); #[cfg(feature = "enterprise")] - let metrics_retention = core - .core + let metrics_retention = inner + .shared_core .load() .enterprise .as_ref() @@ -267,7 +254,7 @@ pub fn spawn_housekeeper(core: JmapInstance, mut rx: mpsc::Receiver<Event>) { tokio::spawn(async move { trc::event!( - Housekeeper(HousekeeperEvent::PurgeStore), + Housekeeper(trc::HousekeeperEvent::PurgeStore), Type = "data" ); if let Err(err) = store.purge_store().await { @@ -294,7 +281,10 @@ pub fn spawn_housekeeper(core: JmapInstance, mut rx: mpsc::Receiver<Event>) { }); } PurgeType::Blobs { store, blob_store } => { - trc::event!(Housekeeper(HousekeeperEvent::PurgeStore), Type = "blob"); + trc::event!( + Housekeeper(trc::HousekeeperEvent::PurgeStore), + Type = "blob" + ); tokio::spawn(async move { if let Err(err) = store.purge_blobs(blob_store).await { @@ -303,7 +293,10 @@ pub fn spawn_housekeeper(core: JmapInstance, mut rx: mpsc::Receiver<Event>) { }); } PurgeType::Lookup(store) => { - trc::event!(Housekeeper(HousekeeperEvent::PurgeStore), Type = "lookup"); + trc::event!( + Housekeeper(trc::HousekeeperEvent::PurgeStore), + Type = "lookup" + ); tokio::spawn(async move { if let Err(err) = store.purge_lookup_store().await { @@ -312,45 +305,44 @@ pub fn spawn_housekeeper(core: JmapInstance, mut rx: mpsc::Receiver<Event>) { }); } PurgeType::Account(account_id) => { - let jmap = JMAP::from(core.clone()); + let server = inner.build_server(); tokio::spawn(async move { - trc::event!(Housekeeper(HousekeeperEvent::PurgeAccounts)); + trc::event!(Housekeeper(trc::HousekeeperEvent::PurgeAccounts)); if let Some(account_id) = account_id { - jmap.purge_account(account_id).await; + server.purge_account(account_id).await; } else { - jmap.purge_accounts().await; + server.purge_accounts().await; } }); } }, - Event::Exit => { - trc::event!(Housekeeper(HousekeeperEvent::Stop)); + HousekeeperEvent::Exit => { + trc::event!(Housekeeper(trc::HousekeeperEvent::Stop)); return; } }, Ok(None) => { - trc::event!(Housekeeper(HousekeeperEvent::Stop)); + trc::event!(Housekeeper(trc::HousekeeperEvent::Stop)); return; } Err(_) => { - let core_ = core.core.load_full(); + let server = inner.build_server(); while let Some(event) = queue.pop() { match event.event { ActionClass::Acme(provider_id) => { - let inner = core.jmap_inner.clone(); - let core = core_.clone(); + let server = server.clone(); tokio::spawn(async move { if let Some(provider) = - core.tls.acme_providers.get(&provider_id) + server.core.acme.providers.get(&provider_id) { trc::event!( Acme(trc::AcmeEvent::OrderStart), Hostname = provider.domains.as_slice() ); - let renew_at = match core.renew(provider).await { + let renew_at = match server.renew(provider).await { Ok(renew_at) => { trc::event!( Acme(trc::AcmeEvent::OrderCompleted), @@ -371,11 +363,13 @@ pub fn spawn_housekeeper(core: JmapInstance, mut rx: mpsc::Receiver<Event>) { } }; - inner.increment_config_version(); + server.increment_config_version(); - inner + server + .inner + .ipc .housekeeper_tx - .send(Event::AcmeReschedule { + .send(HousekeeperEvent::AcmeReschedule { provider_id: provider_id.clone(), renew_at: Instant::now() + renew_at, }) @@ -385,35 +379,48 @@ pub fn spawn_housekeeper(core: JmapInstance, mut rx: mpsc::Receiver<Event>) { }); } ActionClass::Account => { - let jmap = JMAP::from(core.clone()); - tokio::spawn(async move { - trc::event!(Housekeeper(HousekeeperEvent::PurgeAccounts)); - jmap.purge_accounts().await; - }); + let server = server.clone(); queue.schedule( Instant::now() - + core_.jmap.account_purge_frequency.time_to_next(), + + server.core.jmap.account_purge_frequency.time_to_next(), ActionClass::Account, ); - } - ActionClass::Session => { - let inner = core.jmap_inner.clone(); - let core = core_.clone(); - tokio::spawn(async move { - trc::event!(Housekeeper(HousekeeperEvent::PurgeSessions)); - inner.purge(); - core.security.access_tokens.cleanup(); + trc::event!(Housekeeper(trc::HousekeeperEvent::PurgeAccounts)); + server.purge_accounts().await; }); + } + ActionClass::Session => { + let server = server.clone(); queue.schedule( Instant::now() - + core_.jmap.session_purge_frequency.time_to_next(), + + server.core.jmap.session_purge_frequency.time_to_next(), ActionClass::Session, ); + + tokio::spawn(async move { + trc::event!(Housekeeper(trc::HousekeeperEvent::PurgeSessions)); + server.inner.data.http_auth_cache.cleanup(); + server + .inner + .data + .jmap_limiter + .retain(|_, limiter| limiter.is_active()); + server.inner.data.access_tokens.cleanup(); + + for throttle in [ + &server.inner.data.smtp_session_throttle, + &server.inner.data.smtp_queue_throttle, + ] { + throttle.retain(|_, v| { + v.concurrent.load(Ordering::Relaxed) > 0 + }); + } + }); } ActionClass::Store(idx) => { if let Some(schedule) = - core_.storage.purge_schedules.get(idx).cloned() + server.core.storage.purge_schedules.get(idx).cloned() { queue.schedule( Instant::now() + schedule.cron.time_to_next(), @@ -435,7 +442,7 @@ pub fn spawn_housekeeper(core: JmapInstance, mut rx: mpsc::Receiver<Event>) { match result { Ok(_) => { trc::event!( - Housekeeper(HousekeeperEvent::PurgeStore), + Housekeeper(trc::HousekeeperEvent::PurgeStore), Id = schedule.store_id ); } @@ -451,16 +458,22 @@ pub fn spawn_housekeeper(core: JmapInstance, mut rx: mpsc::Receiver<Event>) { } } ActionClass::OtelMetrics => { - if let Some(otel) = &core_.metrics.otel { + if let Some(otel) = &server.core.metrics.otel { queue.schedule( Instant::now() + otel.interval, ActionClass::OtelMetrics, ); let otel = otel.clone(); - let core = core_.clone(); + + #[cfg(feature = "enterprise")] + let is_enterprise = server.is_enterprise_edition(); + + #[cfg(not(feature = "enterprise"))] + let is_enterprise = false; + tokio::spawn(async move { - otel.push_metrics(core, start_time).await; + otel.push_metrics(is_enterprise, start_time).await; }); } } @@ -479,12 +492,12 @@ pub fn spawn_housekeeper(core: JmapInstance, mut rx: mpsc::Receiver<Event>) { false }; - let core = core_.clone(); + let server = server.clone(); tokio::spawn(async move { #[cfg(feature = "enterprise")] - if core.is_enterprise_edition() { + if server.is_enterprise_edition() { // Obtain queue size - match core.total_queued_messages().await { + match server.total_queued_messages().await { Ok(total) => { Collector::update_gauge( MetricType::QueueCount, @@ -500,7 +513,7 @@ pub fn spawn_housekeeper(core: JmapInstance, mut rx: mpsc::Receiver<Event>) { } if update_other_metrics { - match core.total_accounts().await { + match server.total_accounts().await { Ok(total) => { Collector::update_gauge( MetricType::UserCount, @@ -514,7 +527,7 @@ pub fn spawn_housekeeper(core: JmapInstance, mut rx: mpsc::Receiver<Event>) { } } - match core.total_domains().await { + match server.total_domains().await { Ok(total) => { Collector::update_gauge( MetricType::DomainCount, @@ -556,7 +569,8 @@ pub fn spawn_housekeeper(core: JmapInstance, mut rx: mpsc::Receiver<Event>) { // SPDX-License-Identifier: LicenseRef-SEL #[cfg(feature = "enterprise")] ActionClass::InternalMetrics => { - if let Some(metrics_store) = &core_ + if let Some(metrics_store) = &server + .core .enterprise .as_ref() .and_then(|e| e.metrics_store.as_ref()) @@ -568,7 +582,7 @@ pub fn spawn_housekeeper(core: JmapInstance, mut rx: mpsc::Receiver<Event>) { let metrics_store = metrics_store.store.clone(); let metrics_history = metrics_history.clone(); - let core = core_.clone(); + let core = server.core.clone(); tokio::spawn(async move { if let Err(err) = metrics_store .write_metrics(core, now(), metrics_history) @@ -582,22 +596,20 @@ pub fn spawn_housekeeper(core: JmapInstance, mut rx: mpsc::Receiver<Event>) { #[cfg(feature = "enterprise")] ActionClass::AlertMetrics => { - let smtp = SMTP { - core: core_.clone(), - inner: core.smtp_inner.clone(), - }; + let server = server.clone(); tokio::spawn(async move { - if let Some(messages) = smtp.core.process_alerts().await { + if let Some(messages) = server.process_alerts().await { for message in messages { - smtp.send_autogenerated( - message.from, - message.to.into_iter(), - message.body, - None, - 0, - ) - .await; + server + .send_autogenerated( + message.from, + message.to.into_iter(), + message.body, + None, + 0, + ) + .await; } } }); @@ -605,7 +617,7 @@ pub fn spawn_housekeeper(core: JmapInstance, mut rx: mpsc::Receiver<Event>) { #[cfg(feature = "enterprise")] ActionClass::ValidateLicense => { - match core_.reload().await { + match server.reload().await { Ok(result) => { if let Some(new_core) = result.new_core { if let Some(enterprise) = &new_core.enterprise { @@ -617,10 +629,10 @@ pub fn spawn_housekeeper(core: JmapInstance, mut rx: mpsc::Receiver<Event>) { } // Update core - core.core.store(new_core.into()); + server.inner.shared_core.store(new_core.into()); // Increment version counter - core.jmap_inner.increment_config_version(); + server.increment_config_version(); } } Err(err) => { @@ -639,7 +651,7 @@ pub fn spawn_housekeeper(core: JmapInstance, mut rx: mpsc::Receiver<Event>) { impl Queue { pub fn schedule(&mut self, due: Instant, event: ActionClass) { trc::event!( - Housekeeper(HousekeeperEvent::Schedule), + Housekeeper(trc::HousekeeperEvent::Schedule), Due = trc::Value::Timestamp( now() + due.saturating_duration_since(Instant::now()).as_secs() ), @@ -684,15 +696,3 @@ impl PartialOrd for Action { Some(self.cmp(other)) } } - -impl Inner { - pub fn purge(&self) { - self.sessions.cleanup(); - self.concurrency_limiter - .retain(|_, limiter| limiter.is_active()); - } -} - -pub fn init_housekeeper() -> (mpsc::Sender<Event>, mpsc::Receiver<Event>) { - mpsc::channel::<Event>(IPC_CHANNEL_BUFFER) -} diff --git a/crates/jmap/src/services/index.rs b/crates/jmap/src/services/index.rs index 5a17fa3e..7481cbee 100644 --- a/crates/jmap/src/services/index.rs +++ b/crates/jmap/src/services/index.rs @@ -6,6 +6,7 @@ use std::{sync::Arc, time::Instant}; +use common::{core::BuildServer, Inner, Server}; use directory::{ backend::internal::{manage::ManageDirectory, PrincipalField}, Type, @@ -22,17 +23,19 @@ use store::{ Deserialize, IterateParams, Serialize, ValueKey, U32_LEN, U64_LEN, }; -use tokio::sync::Notify; +use std::future::Future; use trc::{AddContext, FtsIndexEvent}; use utils::{BlobHash, BLOB_HASH_LEN}; use crate::{ + blob::download::BlobDownload, + changes::write::ChangeLog, email::{index::IndexMessageText, metadata::MessageMetadata}, - Inner, JmapInstance, JMAP, + JmapMethods, }; #[derive(Debug)] -struct IndexEmail { +pub struct IndexEmail { account_id: u32, document_id: u32, seq: u64, @@ -42,11 +45,12 @@ struct IndexEmail { const INDEX_LOCK_EXPIRY: u64 = 60 * 5; -pub fn spawn_index_task(core: JmapInstance, rx: Arc<Notify>) { +pub fn spawn_index_task(inner: Arc<Inner>) { tokio::spawn(async move { + let rx = inner.ipc.index_tx.clone(); loop { // Index any queued messages - JMAP::from(core.clone()).fts_index_queued().await; + inner.build_server().fts_index_queued().await; // Wait for a signal to index more messages rx.notified().await; @@ -54,8 +58,19 @@ pub fn spawn_index_task(core: JmapInstance, rx: Arc<Notify>) { }); } -impl JMAP { - pub async fn fts_index_queued(&self) { +pub trait Indexer: Sync + Send { + fn fts_index_queued(&self) -> impl Future<Output = ()> + Send; + fn try_lock_index(&self, event: &IndexEmail) -> impl Future<Output = bool> + Send; + fn reindex( + &self, + account_id: Option<u32>, + tenant_id: Option<u32>, + ) -> impl Future<Output = trc::Result<()>> + Send; + fn request_fts_index(&self); +} + +impl Indexer for Server { + async fn fts_index_queued(&self) { let from_key = ValueKey::<ValueClass<u32>> { account_id: 0, collection: 0, @@ -243,15 +258,11 @@ impl JMAP { } } - pub fn request_fts_index(&self) { - self.inner.request_fts_index(); + fn request_fts_index(&self) { + self.inner.ipc.index_tx.notify_one(); } - pub async fn reindex( - &self, - account_id: Option<u32>, - tenant_id: Option<u32>, - ) -> trc::Result<()> { + async fn reindex(&self, account_id: Option<u32>, tenant_id: Option<u32>) -> trc::Result<()> { let accounts = if let Some(account_id) = account_id { RoaringBitmap::from_sorted_iter([account_id]).unwrap() } else { @@ -362,12 +373,6 @@ impl JMAP { } } -impl Inner { - pub fn request_fts_index(&self) { - self.index_tx.notify_one(); - } -} - impl IndexEmail { fn value_class(&self) -> ValueClass<MaybeDynamicId> { ValueClass::FtsQueue(FtsQueueClass { diff --git a/crates/jmap/src/services/ingest.rs b/crates/jmap/src/services/ingest.rs index 07967628..aadeb15b 100644 --- a/crates/jmap/src/services/ingest.rs +++ b/crates/jmap/src/services/ingest.rs @@ -4,20 +4,33 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use common::{DeliveryResult, IngestMessage}; +use common::{ + ipc::{DeliveryResult, IngestMessage}, + Server, +}; use directory::Permission; use jmap_proto::types::{state::StateChange, type_state::DataType}; use mail_parser::MessageParser; +use std::future::Future; use store::ahash::AHashMap; use crate::{ - email::ingest::{IngestEmail, IngestSource}, + email::ingest::{EmailIngest, IngestEmail, IngestSource}, mailbox::INBOX_ID, - JMAP, + sieve::{get::SieveScriptGet, ingest::SieveScriptIngest}, }; -impl JMAP { - pub async fn deliver_message(&self, message: IngestMessage) -> Vec<DeliveryResult> { +use super::state::StateManager; + +pub trait MailDelivery: Sync + Send { + fn deliver_message( + &self, + message: IngestMessage, + ) -> impl Future<Output = Vec<DeliveryResult>> + Send; +} + +impl MailDelivery for Server { + async fn deliver_message(&self, message: IngestMessage) -> Vec<DeliveryResult> { // Read message let raw_message = match self .core @@ -60,7 +73,6 @@ impl JMAP { let mut deliver_names = AHashMap::with_capacity(message.recipients.len()); for rcpt in &message.recipients { match self - .core .email_to_ids(&self.core.storage.directory, rcpt, message.session_id) .await { @@ -84,15 +96,11 @@ impl JMAP { // Deliver to each recipient for (uid, (status, rcpt)) in &mut deliver_names { // Obtain access token - let result = match self - .core - .get_cached_access_token(*uid) - .await - .and_then(|token| { - token - .assert_has_permission(Permission::EmailReceive) - .map(|_| token) - }) { + let result = match self.get_cached_access_token(*uid).await.and_then(|token| { + token + .assert_has_permission(Permission::EmailReceive) + .map(|_| token) + }) { Ok(access_token) => { // Check if there is an active sieve script match self.sieve_script_get_active(*uid).await { diff --git a/crates/jmap/src/services/state.rs b/crates/jmap/src/services/state.rs index 6f448d50..670f5ff9 100644 --- a/crates/jmap/src/services/state.rs +++ b/crates/jmap/src/services/state.rs @@ -4,39 +4,24 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use std::time::{Duration, Instant, SystemTime}; +use std::{ + sync::Arc, + time::{Duration, Instant, SystemTime}, +}; -use common::IPC_CHANNEL_BUFFER; +use common::{ + core::BuildServer, + ipc::{PushSubscription, StateEvent, UpdateSubscription}, + Inner, Server, IPC_CHANNEL_BUFFER, +}; use jmap_proto::types::{id::Id, state::StateChange, type_state::DataType}; +use std::future::Future; use store::ahash::AHashMap; use tokio::sync::mpsc; use trc::ServerEvent; use utils::map::bitmap::Bitmap; -use crate::{ - push::{manager::spawn_push_manager, UpdateSubscription}, - JmapInstance, JMAP, -}; - -#[derive(Debug)] -pub enum Event { - Subscribe { - account_id: u32, - types: Bitmap<DataType>, - tx: mpsc::Sender<StateChange>, - }, - Publish { - state_change: StateChange, - }, - UpdateSharedAccounts { - account_id: u32, - }, - UpdateSubscriptions { - account_id: u32, - subscriptions: Vec<UpdateSubscription>, - }, - Stop, -} +use crate::push::{get::PushSubscriptionFetch, manager::spawn_push_manager}; #[derive(Debug)] struct Subscriber { @@ -62,10 +47,6 @@ impl Subscriber { const PURGE_EVERY: Duration = Duration::from_secs(3600); const SEND_TIMEOUT: Duration = Duration::from_millis(500); -pub fn init_state_manager() -> (mpsc::Sender<Event>, mpsc::Receiver<Event>) { - mpsc::channel::<Event>(IPC_CHANNEL_BUFFER) -} - #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] enum SubscriberId { Ipc(u32), @@ -73,8 +54,8 @@ enum SubscriberId { } #[allow(clippy::unwrap_or_default)] -pub fn spawn_state_manager(core: JmapInstance, mut change_rx: mpsc::Receiver<Event>) { - let push_tx = spawn_push_manager(core.clone()); +pub fn spawn_state_manager(inner: Arc<Inner>, mut change_rx: mpsc::Receiver<StateEvent>) { + let push_tx = spawn_push_manager(inner.clone()); tokio::spawn(async move { let mut subscribers: AHashMap<u32, AHashMap<SubscriberId, Subscriber>> = @@ -89,7 +70,7 @@ pub fn spawn_state_manager(core: JmapInstance, mut change_rx: mpsc::Receiver<Eve let mut purge_needed = last_purge.elapsed() >= PURGE_EVERY; match event { - Event::Stop => { + StateEvent::Stop => { if push_tx.send(crate::push::Event::Reset).await.is_err() { trc::event!( Server(ServerEvent::ThreadError), @@ -99,13 +80,9 @@ pub fn spawn_state_manager(core: JmapInstance, mut change_rx: mpsc::Receiver<Eve } break; } - Event::UpdateSharedAccounts { account_id } => { + StateEvent::UpdateSharedAccounts { account_id } => { // Obtain account membership and shared mailboxes - let acl = match JMAP::from(core.clone()) - .core - .get_access_token(account_id) - .await - { + let acl = match inner.build_server().get_access_token(account_id).await { Ok(result) => result, Err(err) => { trc::error!(err @@ -169,7 +146,7 @@ pub fn spawn_state_manager(core: JmapInstance, mut change_rx: mpsc::Receiver<Eve } shared_accounts.insert(account_id, shared_account_ids); } - Event::Subscribe { + StateEvent::Subscribe { account_id, types, tx, @@ -185,7 +162,7 @@ pub fn spawn_state_manager(core: JmapInstance, mut change_rx: mpsc::Receiver<Eve }, ); } - Event::Publish { state_change } => { + StateEvent::Publish { state_change } => { if let Some(shared_accounts) = shared_accounts_map.get(&state_change.account_id) { let current_time = SystemTime::now() @@ -266,7 +243,7 @@ pub fn spawn_state_manager(core: JmapInstance, mut change_rx: mpsc::Receiver<Eve } } } - Event::UpdateSubscriptions { + StateEvent::UpdateSubscriptions { account_id, subscriptions, } => { @@ -280,7 +257,7 @@ pub fn spawn_state_manager(core: JmapInstance, mut change_rx: mpsc::Receiver<Eve if let SubscriberId::Push(push_id) = subscriber_id { if !subscriptions.iter().any(|s| { matches!(s, UpdateSubscription::Verified( - crate::push::PushSubscription { id, .. } + PushSubscription { id, .. } ) if id == push_id) }) { remove_ids.push(*subscriber_id); @@ -388,18 +365,33 @@ pub fn spawn_state_manager(core: JmapInstance, mut change_rx: mpsc::Receiver<Eve }); } -impl JMAP { - pub async fn subscribe_state_manager( +pub trait StateManager: Sync + Send { + fn subscribe_state_manager( + &self, + account_id: u32, + types: Bitmap<DataType>, + ) -> impl Future<Output = trc::Result<mpsc::Receiver<StateChange>>> + Send; + + fn broadcast_state_change( + &self, + state_change: StateChange, + ) -> impl Future<Output = bool> + Send; + + fn update_push_subscriptions(&self, account_id: u32) -> impl Future<Output = bool> + Send; +} + +impl StateManager for Server { + async fn subscribe_state_manager( &self, account_id: u32, types: Bitmap<DataType>, ) -> trc::Result<mpsc::Receiver<StateChange>> { let (change_tx, change_rx) = mpsc::channel::<StateChange>(IPC_CHANNEL_BUFFER); - let state_tx = self.inner.state_tx.clone(); + let state_tx = self.inner.ipc.state_tx.clone(); for event in [ - Event::UpdateSharedAccounts { account_id }, - Event::Subscribe { + StateEvent::UpdateSharedAccounts { account_id }, + StateEvent::Subscribe { account_id, types, tx: change_tx, @@ -415,12 +407,13 @@ impl JMAP { Ok(change_rx) } - pub async fn broadcast_state_change(&self, state_change: StateChange) -> bool { + async fn broadcast_state_change(&self, state_change: StateChange) -> bool { match self .inner + .ipc .state_tx .clone() - .send(Event::Publish { state_change }) + .send(StateEvent::Publish { state_change }) .await { Ok(_) => true, @@ -436,7 +429,7 @@ impl JMAP { } } - pub async fn update_push_subscriptions(&self, account_id: u32) -> bool { + async fn update_push_subscriptions(&self, account_id: u32) -> bool { let push_subs = match self.fetch_push_subscriptions(account_id).await { Ok(push_subs) => push_subs, Err(err) => { @@ -447,8 +440,8 @@ impl JMAP { } }; - let state_tx = self.inner.state_tx.clone(); - for event in [Event::UpdateSharedAccounts { account_id }, push_subs] { + let state_tx = self.inner.ipc.state_tx.clone(); + for event in [StateEvent::UpdateSharedAccounts { account_id }, push_subs] { if state_tx.send(event).await.is_err() { trc::event!( Server(ServerEvent::ThreadError), diff --git a/crates/jmap/src/sieve/get.rs b/crates/jmap/src/sieve/get.rs index 7bee50a3..d40bea1d 100644 --- a/crates/jmap/src/sieve/get.rs +++ b/crates/jmap/src/sieve/get.rs @@ -6,6 +6,7 @@ use std::sync::Arc; +use common::Server; use jmap_proto::{ method::get::{GetRequest, GetResponse, RequestArguments}, object::Object, @@ -18,12 +19,42 @@ use store::{ BlobClass, Deserialize, Serialize, }; -use crate::{sieve::SeenIds, JMAP}; +use crate::{ + blob::{download::BlobDownload, upload::BlobUpload}, + changes::state::StateManager, + sieve::SeenIds, + JmapMethods, +}; use super::ActiveScript; +use std::future::Future; + +pub trait SieveScriptGet: Sync + Send { + fn sieve_script_get( + &self, + request: GetRequest<RequestArguments>, + ) -> impl Future<Output = trc::Result<GetResponse>> + Send; -impl JMAP { - pub async fn sieve_script_get( + fn sieve_script_get_active( + &self, + account_id: u32, + ) -> impl Future<Output = trc::Result<Option<ActiveScript>>> + Send; + + fn sieve_script_get_by_name( + &self, + account_id: u32, + name: &str, + ) -> impl Future<Output = trc::Result<Option<Sieve>>> + Send; + + fn sieve_script_compile( + &self, + account_id: u32, + document_id: u32, + ) -> impl Future<Output = trc::Result<(Sieve, Object<Value>)>> + Send; +} + +impl SieveScriptGet for Server { + async fn sieve_script_get( &self, mut request: GetRequest<RequestArguments>, ) -> trc::Result<GetResponse> { @@ -115,10 +146,7 @@ impl JMAP { Ok(response) } - pub async fn sieve_script_get_active( - &self, - account_id: u32, - ) -> trc::Result<Option<ActiveScript>> { + async fn sieve_script_get_active(&self, account_id: u32) -> trc::Result<Option<ActiveScript>> { // Find the currently active script if let Some(document_id) = self .filter( @@ -156,7 +184,7 @@ impl JMAP { } } - pub async fn sieve_script_get_by_name( + async fn sieve_script_get_by_name( &self, account_id: u32, name: &str, diff --git a/crates/jmap/src/sieve/ingest.rs b/crates/jmap/src/sieve/ingest.rs index 16ecbe1f..08ba6051 100644 --- a/crates/jmap/src/sieve/ingest.rs +++ b/crates/jmap/src/sieve/ingest.rs @@ -6,7 +6,7 @@ use std::borrow::Cow; -use common::{auth::AccessToken, listener::stream::NullIo}; +use common::{auth::AccessToken, listener::stream::NullIo, Server}; use directory::{backend::internal::PrincipalField, QueryBy}; use jmap_proto::types::{collection::Collection, id::Id, keyword::Keyword, property::Property}; use mail_parser::MessageParser; @@ -19,13 +19,14 @@ use store::{ use trc::{AddContext, SieveEvent}; use crate::{ - email::ingest::{IngestEmail, IngestSource, IngestedEmail}, - mailbox::{INBOX_ID, TRASH_ID}, + email::ingest::{EmailIngest, IngestEmail, IngestSource, IngestedEmail}, + mailbox::{get::MailboxGet, set::MailboxSet, INBOX_ID, TRASH_ID}, sieve::SeenIdHash, - JMAP, + JmapMethods, }; -use super::ActiveScript; +use super::{get::SieveScriptGet, ActiveScript}; +use std::future::Future; struct SieveMessage<'x> { pub raw_message: Cow<'x, [u8]>, @@ -33,9 +34,21 @@ struct SieveMessage<'x> { pub flags: Vec<Keyword>, } -impl JMAP { +pub trait SieveScriptIngest: Sync + Send { + fn sieve_script_ingest( + &self, + access_token: &AccessToken, + raw_message: &[u8], + envelope_from: &str, + envelope_to: &str, + session_id: u64, + active_script: ActiveScript, + ) -> impl Future<Output = trc::Result<IngestedEmail>> + Send; +} + +impl SieveScriptIngest for Server { #[allow(clippy::blocks_in_conditions)] - pub async fn sieve_script_ingest( + async fn sieve_script_ingest( &self, access_token: &AccessToken, raw_message: &[u8], @@ -124,8 +137,7 @@ impl JMAP { } } sieve::Script::Global(name_) => { - if let Some(script) = - self.core.get_untrusted_sieve_script(name_, session_id) + if let Some(script) = self.get_untrusted_sieve_script(name_, session_id) { input = Input::script(name, script.clone()); } else { @@ -353,7 +365,7 @@ impl JMAP { ); Session::<NullIo>::sieve( - self.smtp.clone(), + self.clone(), SessionAddress::new(mail_from.clone()), recipients, message.raw_message.to_vec(), diff --git a/crates/jmap/src/sieve/query.rs b/crates/jmap/src/sieve/query.rs index 144ccd65..0e1c8cc9 100644 --- a/crates/jmap/src/sieve/query.rs +++ b/crates/jmap/src/sieve/query.rs @@ -4,18 +4,27 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ +use common::Server; use jmap_proto::{ method::query::{ Comparator, Filter, QueryRequest, QueryResponse, RequestArguments, SortProperty, }, types::{collection::Collection, property::Property}, }; +use std::future::Future; use store::query::{self}; -use crate::JMAP; +use crate::JmapMethods; -impl JMAP { - pub async fn sieve_script_query( +pub trait SieveScriptQuery: Sync + Send { + fn sieve_script_query( + &self, + request: QueryRequest<RequestArguments>, + ) -> impl Future<Output = trc::Result<QueryResponse>> + Send; +} + +impl SieveScriptQuery for Server { + async fn sieve_script_query( &self, mut request: QueryRequest<RequestArguments>, ) -> trc::Result<QueryResponse> { diff --git a/crates/jmap/src/sieve/set.rs b/crates/jmap/src/sieve/set.rs index 12f38da9..7ba9eec0 100644 --- a/crates/jmap/src/sieve/set.rs +++ b/crates/jmap/src/sieve/set.rs @@ -4,7 +4,10 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use common::auth::{AccessToken, ResourceToken}; +use common::{ + auth::{AccessToken, ResourceToken}, + Server, +}; use jmap_proto::{ error::set::{SetError, SetErrorType}, method::set::{SetRequest, SetResponse}, @@ -34,9 +37,15 @@ use store::{ BlobClass, }; -use crate::{api::http::HttpSessionData, JMAP}; +use crate::{ + api::http::HttpSessionData, + blob::{download::BlobDownload, upload::BlobUpload}, + changes::write::ChangeLog, + JmapMethods, +}; +use std::future::Future; -struct SetContext<'x> { +pub struct SetContext<'x> { resource_token: ResourceToken, access_token: &'x AccessToken, response: SetResponse, @@ -53,8 +62,38 @@ pub static SCHEMA: &[IndexProperty] = &[ IndexProperty::new(Property::IsActive).index_as(IndexAs::Integer), ]; -impl JMAP { - pub async fn sieve_script_set( +pub trait SieveScriptSet: Sync + Send { + fn sieve_script_set( + &self, + request: SetRequest<SetArguments>, + access_token: &AccessToken, + session: &HttpSessionData, + ) -> impl Future<Output = trc::Result<SetResponse>> + Send; + + fn sieve_script_delete( + &self, + resource_token: &ResourceToken, + document_id: u32, + fail_if_active: bool, + ) -> impl Future<Output = trc::Result<bool>> + Send; + + fn sieve_set_item( + &self, + changes_: Object<SetValue>, + update: Option<(u32, HashedValue<Object<Value>>)>, + ctx: &SetContext, + session_id: u64, + ) -> impl Future<Output = trc::Result<Result<(ObjectIndexBuilder, Option<Vec<u8>>), SetError>>> + Send; + + fn sieve_activate_script( + &self, + account_id: u32, + activate_id: Option<u32>, + ) -> impl Future<Output = trc::Result<Vec<(u32, bool)>>> + Send; +} + +impl SieveScriptSet for Server { + async fn sieve_script_set( &self, mut request: SetRequest<SetArguments>, access_token: &AccessToken, @@ -343,7 +382,7 @@ impl JMAP { Ok(ctx.response) } - pub async fn sieve_script_delete( + async fn sieve_script_delete( &self, resource_token: &ResourceToken, document_id: u32, @@ -581,7 +620,7 @@ impl JMAP { .map(|obj| (obj, blob_update))) } - pub async fn sieve_activate_script( + async fn sieve_activate_script( &self, account_id: u32, mut activate_id: Option<u32>, diff --git a/crates/jmap/src/sieve/validate.rs b/crates/jmap/src/sieve/validate.rs index 2413a572..81b19fbb 100644 --- a/crates/jmap/src/sieve/validate.rs +++ b/crates/jmap/src/sieve/validate.rs @@ -4,16 +4,25 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use common::auth::AccessToken; +use common::{auth::AccessToken, Server}; use jmap_proto::{ error::set::{SetError, SetErrorType}, method::validate::{ValidateSieveScriptRequest, ValidateSieveScriptResponse}, }; +use std::future::Future; -use crate::JMAP; +use crate::blob::download::BlobDownload; -impl JMAP { - pub async fn sieve_script_validate( +pub trait SieveScriptValidate: Sync + Send { + fn sieve_script_validate( + &self, + request: ValidateSieveScriptRequest, + access_token: &AccessToken, + ) -> impl Future<Output = trc::Result<ValidateSieveScriptResponse>> + Send; +} + +impl SieveScriptValidate for Server { + async fn sieve_script_validate( &self, request: ValidateSieveScriptRequest, access_token: &AccessToken, diff --git a/crates/jmap/src/submission/get.rs b/crates/jmap/src/submission/get.rs index f08cb166..ef9ca794 100644 --- a/crates/jmap/src/submission/get.rs +++ b/crates/jmap/src/submission/get.rs @@ -4,17 +4,26 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ +use common::Server; use jmap_proto::{ method::get::{GetRequest, GetResponse, RequestArguments}, object::Object, types::{collection::Collection, property::Property, value::Value}, }; -use smtp::queue; +use smtp::queue::{self, spool::SmtpSpool}; +use std::future::Future; -use crate::JMAP; +use crate::{changes::state::StateManager, JmapMethods}; -impl JMAP { - pub async fn email_submission_get( +pub trait EmailSubmissionGet: Sync + Send { + fn email_submission_get( + &self, + request: GetRequest<RequestArguments>, + ) -> impl Future<Output = trc::Result<GetResponse>> + Send; +} + +impl EmailSubmissionGet for Server { + async fn email_submission_get( &self, mut request: GetRequest<RequestArguments>, ) -> trc::Result<GetResponse> { @@ -79,7 +88,6 @@ impl JMAP { // Obtain queueId let queued_message = self - .smtp .read_message(push.get(&Property::MessageId).as_uint().unwrap_or(u64::MAX)) .await; diff --git a/crates/jmap/src/submission/query.rs b/crates/jmap/src/submission/query.rs index 80ce88bc..fb1592e3 100644 --- a/crates/jmap/src/submission/query.rs +++ b/crates/jmap/src/submission/query.rs @@ -4,18 +4,27 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ +use common::Server; use jmap_proto::{ method::query::{ Comparator, Filter, QueryRequest, QueryResponse, RequestArguments, SortProperty, }, types::{collection::Collection, property::Property}, }; +use std::future::Future; use store::query::{self}; -use crate::JMAP; +use crate::JmapMethods; -impl JMAP { - pub async fn email_submission_query( +pub trait EmailSubmissionQuery: Sync + Send { + fn email_submission_query( + &self, + request: QueryRequest<RequestArguments>, + ) -> impl Future<Output = trc::Result<QueryResponse>> + Send; +} + +impl EmailSubmissionQuery for Server { + async fn email_submission_query( &self, mut request: QueryRequest<RequestArguments>, ) -> trc::Result<QueryResponse> { diff --git a/crates/jmap/src/submission/set.rs b/crates/jmap/src/submission/set.rs index 54e64a16..c92ae0da 100644 --- a/crates/jmap/src/submission/set.rs +++ b/crates/jmap/src/submission/set.rs @@ -6,7 +6,10 @@ use std::{collections::HashMap, sync::Arc}; -use common::listener::{stream::NullIo, ServerInstance}; +use common::{ + listener::{stream::NullIo, ServerInstance}, + Server, +}; use jmap_proto::{ error::set::{SetError, SetErrorType}, method::set::{self, SetRequest, SetResponse}, @@ -29,12 +32,19 @@ use jmap_proto::{ }, }; use mail_parser::{HeaderName, HeaderValue}; -use smtp::core::{Session, SessionData, State}; +use smtp::{ + core::{Session, SessionData, State}, + queue::spool::SmtpSpool, +}; use smtp_proto::{request::parser::Rfc5321Parser, MailFrom, RcptTo}; use store::write::{assert::HashedValue, log::ChangeLogBuilder, now, BatchBuilder, Bincode}; use utils::map::vec_map::VecMap; -use crate::{email::metadata::MessageMetadata, identity::set::sanitize_email, JMAP}; +use crate::{ + blob::download::BlobDownload, changes::write::ChangeLog, email::metadata::MessageMetadata, + identity::set::sanitize_email, JmapMethods, +}; +use std::future::Future; pub static SCHEMA: &[IndexProperty] = &[ IndexProperty::new(Property::UndoStatus).index_as(IndexAs::Text { @@ -47,8 +57,25 @@ pub static SCHEMA: &[IndexProperty] = &[ IndexProperty::new(Property::SendAt).index_as(IndexAs::LongInteger), ]; -impl JMAP { - pub async fn email_submission_set( +pub trait EmailSubmissionSet: Sync + Send { + fn email_submission_set( + &self, + request: SetRequest<SetArguments>, + instance: &Arc<ServerInstance>, + next_call: &mut Option<Call<RequestMethod>>, + ) -> impl Future<Output = trc::Result<SetResponse>> + Send; + + fn send_message( + &self, + account_id: u32, + response: &SetResponse, + instance: &Arc<ServerInstance>, + object: Object<SetValue>, + ) -> impl Future<Output = trc::Result<Result<Object<Value>, SetError>>> + Send; +} + +impl EmailSubmissionSet for Server { + async fn email_submission_set( &self, mut request: SetRequest<SetArguments>, instance: &Arc<ServerInstance>, @@ -147,10 +174,10 @@ impl JMAP { match undo_status { Some(undo_status) if undo_status == "canceled" => { - if let Some(queue_message) = self.smtp.read_message(queue_id).await { + if let Some(queue_message) = self.read_message(queue_id).await { // Delete message from queue let message_due = queue_message.next_event().unwrap_or_default(); - queue_message.remove(&self.smtp, message_due).await; + queue_message.remove(self, message_due).await; // Update record let mut batch = BatchBuilder::new(); @@ -553,7 +580,7 @@ impl JMAP { // Begin local SMTP session let mut session = - Session::<NullIo>::local(self.smtp.clone(), instance.clone(), SessionData::default()); + Session::<NullIo>::local(self.clone(), instance.clone(), SessionData::default()); // MAIL FROM let _ = session.handle_mail_from(mail_from).await; diff --git a/crates/jmap/src/thread/get.rs b/crates/jmap/src/thread/get.rs index 39403806..c328da38 100644 --- a/crates/jmap/src/thread/get.rs +++ b/crates/jmap/src/thread/get.rs @@ -4,18 +4,27 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ +use common::Server; use jmap_proto::{ method::get::{GetRequest, GetResponse, RequestArguments}, object::Object, types::{collection::Collection, id::Id, property::Property}, }; +use std::future::Future; use store::query::{sort::Pagination, Comparator, ResultSet}; use trc::AddContext; -use crate::JMAP; +use crate::{changes::state::StateManager, JmapMethods}; -impl JMAP { - pub async fn thread_get( +pub trait ThreadGet: Sync + Send { + fn thread_get( + &self, + request: GetRequest<RequestArguments>, + ) -> impl Future<Output = trc::Result<GetResponse>> + Send; +} + +impl ThreadGet for Server { + async fn thread_get( &self, mut request: GetRequest<RequestArguments>, ) -> trc::Result<GetResponse> { diff --git a/crates/jmap/src/vacation/get.rs b/crates/jmap/src/vacation/get.rs index 087d5561..de491cba 100644 --- a/crates/jmap/src/vacation/get.rs +++ b/crates/jmap/src/vacation/get.rs @@ -4,18 +4,32 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ +use common::Server; use jmap_proto::{ method::get::{GetRequest, GetResponse, RequestArguments}, object::Object, request::reference::MaybeReference, types::{any_id::AnyId, collection::Collection, id::Id, property::Property, value::Value}, }; +use std::future::Future; use store::query::Filter; -use crate::JMAP; +use crate::{changes::state::StateManager, JmapMethods}; -impl JMAP { - pub async fn vacation_response_get( +pub trait VacationResponseGet: Sync + Send { + fn vacation_response_get( + &self, + request: GetRequest<RequestArguments>, + ) -> impl Future<Output = trc::Result<GetResponse>> + Send; + + fn get_vacation_sieve_script_id( + &self, + account_id: u32, + ) -> impl Future<Output = trc::Result<Option<u32>>> + Send; +} + +impl VacationResponseGet for Server { + async fn vacation_response_get( &self, mut request: GetRequest<RequestArguments>, ) -> trc::Result<GetResponse> { @@ -100,7 +114,7 @@ impl JMAP { Ok(response) } - pub async fn get_vacation_sieve_script_id(&self, account_id: u32) -> trc::Result<Option<u32>> { + async fn get_vacation_sieve_script_id(&self, account_id: u32) -> trc::Result<Option<u32>> { self.filter( account_id, Collection::SieveScript, diff --git a/crates/jmap/src/vacation/set.rs b/crates/jmap/src/vacation/set.rs index 604e24f0..f924a782 100644 --- a/crates/jmap/src/vacation/set.rs +++ b/crates/jmap/src/vacation/set.rs @@ -6,7 +6,7 @@ use std::borrow::Cow; -use common::auth::AccessToken; +use common::{auth::AccessToken, Server}; use jmap_proto::{ error::set::{SetError, SetErrorType}, method::set::{RequestArguments, SetRequest, SetResponse}, @@ -22,6 +22,7 @@ use jmap_proto::{ }; use mail_builder::MessageBuilder; use mail_parser::decoders::html::html_to_text; +use std::future::Future; use store::write::{ assert::HashedValue, log::{Changes, LogInsert}, @@ -29,12 +30,26 @@ use store::write::{ }; use crate::{ - sieve::set::{ObjectBlobId, SCHEMA}, - JMAP, + blob::upload::BlobUpload, + changes::write::ChangeLog, + sieve::set::{ObjectBlobId, SieveScriptSet, SCHEMA}, + JmapMethods, }; -impl JMAP { - pub async fn vacation_response_set( +use super::get::VacationResponseGet; + +pub trait VacationResponseSet: Sync + Send { + fn vacation_response_set( + &self, + request: SetRequest<RequestArguments>, + access_token: &AccessToken, + ) -> impl Future<Output = trc::Result<SetResponse>> + Send; + + fn build_script(&self, obj: &mut ObjectIndexBuilder) -> trc::Result<Vec<u8>>; +} + +impl VacationResponseSet for Server { + async fn vacation_response_set( &self, mut request: SetRequest<RequestArguments>, access_token: &AccessToken, diff --git a/crates/jmap/src/websocket/stream.rs b/crates/jmap/src/websocket/stream.rs index 20fbac55..85fa8482 100644 --- a/crates/jmap/src/websocket/stream.rs +++ b/crates/jmap/src/websocket/stream.rs @@ -6,7 +6,7 @@ use std::{sync::Arc, time::Instant}; -use common::auth::AccessToken; +use common::{auth::AccessToken, Server}; use futures_util::{SinkExt, StreamExt}; use hyper::upgrade::Upgraded; use hyper_util::rt::TokioIo; @@ -23,12 +23,25 @@ use tungstenite::Message; use utils::map::bitmap::Bitmap; use crate::{ - api::http::{HttpSessionData, ToRequestError}, - JMAP, + api::{ + http::{HttpSessionData, ToRequestError}, + request::RequestHandler, + }, + services::state::StateManager, }; +use std::future::Future; + +pub trait WebSocketHandler: Sync + Send { + fn handle_websocket_stream( + &self, + stream: WebSocketStream<TokioIo<Upgraded>>, + access_token: Arc<AccessToken>, + session: HttpSessionData, + ) -> impl Future<Output = ()> + Send; +} -impl JMAP { - pub async fn handle_websocket_stream( +impl WebSocketHandler for Server { + async fn handle_websocket_stream( &self, mut stream: WebSocketStream<TokioIo<Upgraded>>, access_token: Arc<AccessToken>, diff --git a/crates/jmap/src/websocket/upgrade.rs b/crates/jmap/src/websocket/upgrade.rs index ee9aef1a..cdafc5ad 100644 --- a/crates/jmap/src/websocket/upgrade.rs +++ b/crates/jmap/src/websocket/upgrade.rs @@ -6,20 +6,29 @@ use std::sync::Arc; -use common::auth::AccessToken; +use common::{auth::AccessToken, Server}; use hyper::StatusCode; use hyper_util::rt::TokioIo; use tokio_tungstenite::WebSocketStream; use trc::JmapEvent; use tungstenite::{handshake::derive_accept_key, protocol::Role}; -use crate::{ - api::{http::HttpSessionData, HttpRequest, HttpResponse, HttpResponseBody}, - JMAP, -}; +use crate::api::{http::HttpSessionData, HttpRequest, HttpResponse, HttpResponseBody}; +use std::future::Future; -impl JMAP { - pub async fn upgrade_websocket_connection( +use super::stream::WebSocketHandler; + +pub trait WebSocketUpgrade: Sync + Send { + fn upgrade_websocket_connection( + &self, + req: HttpRequest, + access_token: Arc<AccessToken>, + session: HttpSessionData, + ) -> impl Future<Output = trc::Result<HttpResponse>> + Send; +} + +impl WebSocketUpgrade for Server { + async fn upgrade_websocket_connection( &self, req: HttpRequest, access_token: Arc<AccessToken>, diff --git a/crates/main/src/main.rs b/crates/main/src/main.rs index 68335de2..c35c24d1 100644 --- a/crates/main/src/main.rs +++ b/crates/main/src/main.rs @@ -6,14 +6,13 @@ use std::time::Duration; -use common::{config::server::ServerProtocol, manager::boot::BootManager, Ipc, IPC_CHANNEL_BUFFER}; +use common::{config::server::ServerProtocol, core::BuildServer, manager::boot::BootManager}; use directory::backend::internal::MigrateDirectory; -use imap::core::{ImapSessionManager, IMAP}; -use jmap::{api::JmapSessionManager, services::gossip::spawn::GossiperBuilder, JMAP}; +use imap::core::ImapSessionManager; +use jmap::{api::JmapSessionManager, services::gossip::spawn::GossiperBuilder, StartServices}; use managesieve::core::ManageSieveSessionManager; use pop3::Pop3SessionManager; -use smtp::core::{SmtpSessionManager, SMTP}; -use tokio::sync::mpsc; +use smtp::{core::SmtpSessionManager, StartQueueManager}; use trc::Collector; use utils::wait_for_shutdown; @@ -27,72 +26,61 @@ static GLOBAL: Jemalloc = Jemalloc; #[tokio::main] async fn main() -> std::io::Result<()> { // Load config and apply macros - let init = BootManager::init().await; - - // Parse core - let mut config = init.config; - let core = init.core; - - // Setup IPC channels - let (delivery_tx, delivery_rx) = mpsc::channel(IPC_CHANNEL_BUFFER); - let ipc = Ipc { delivery_tx }; - - // Init servers - let smtp = SMTP::init( - &mut config, - core.clone(), - ipc, - init.servers.span_id_gen.clone(), - ) - .await; - let jmap = JMAP::init(&mut config, delivery_rx, core.clone(), smtp.inner.clone()).await; - let imap = IMAP::init(&mut config, jmap.clone()).await; - let gossiper = GossiperBuilder::try_parse(&mut config); + let mut init = BootManager::init().await; + + // Init services + init.start_services().await; + init.start_queue_manager(); + let gossiper = GossiperBuilder::try_parse(&mut init.config); // Log configuration errors - config.log_errors(); - config.log_warnings(); + init.config.log_errors(); + init.config.log_warnings(); + + { + let server = init.inner.build_server(); - // Log licensing information - #[cfg(feature = "enterprise")] - core.load().as_ref().log_license_details(); + // Log licensing information + #[cfg(feature = "enterprise")] + server.log_license_details(); - // Migrate directory - if let Err(err) = core.load().storage.data.migrate_directory().await { - trc::error!(err.details("Directory migration failed")); - std::process::exit(1); + // Migrate directory + if let Err(err) = server.store().migrate_directory().await { + trc::error!(err.details("Directory migration failed")); + std::process::exit(1); + } } // Spawn servers let (shutdown_tx, shutdown_rx) = init.servers.spawn(|server, acceptor, shutdown_rx| { match &server.protocol { ServerProtocol::Smtp | ServerProtocol::Lmtp => server.spawn( - SmtpSessionManager::new(smtp.clone()), - core.clone(), + SmtpSessionManager::new(init.inner.clone()), + init.inner.clone(), acceptor, shutdown_rx, ), ServerProtocol::Http => server.spawn( - JmapSessionManager::new(jmap.clone()), - core.clone(), + JmapSessionManager::new(init.inner.clone()), + init.inner.clone(), acceptor, shutdown_rx, ), ServerProtocol::Imap => server.spawn( - ImapSessionManager::new(imap.clone()), - core.clone(), + ImapSessionManager::new(init.inner.clone()), + init.inner.clone(), acceptor, shutdown_rx, ), ServerProtocol::Pop3 => server.spawn( - Pop3SessionManager::new(imap.clone()), - core.clone(), + Pop3SessionManager::new(init.inner.clone()), + init.inner.clone(), acceptor, shutdown_rx, ), ServerProtocol::ManageSieve => server.spawn( - ManageSieveSessionManager::new(imap.clone()), - core.clone(), + ManageSieveSessionManager::new(init.inner.clone()), + init.inner.clone(), acceptor, shutdown_rx, ), @@ -101,7 +89,7 @@ async fn main() -> std::io::Result<()> { // Spawn gossip if let Some(gossiper) = gossiper { - gossiper.spawn(jmap, shutdown_rx.clone()).await; + gossiper.spawn(init.inner, shutdown_rx.clone()).await; } // Wait for shutdown signal diff --git a/crates/managesieve/src/core/client.rs b/crates/managesieve/src/core/client.rs index 2380e8ef..87603b7e 100644 --- a/crates/managesieve/src/core/client.rs +++ b/crates/managesieve/src/core/client.rs @@ -118,7 +118,7 @@ impl<T: SessionStream> Session<T> { Command::Capability | Command::Logout | Command::Noop => Ok(command), Command::Authenticate => { if let State::NotAuthenticated { .. } = &self.state { - if self.stream.is_tls() || self.jmap.core.imap.allow_plain_auth { + if self.stream.is_tls() || self.server.core.imap.allow_plain_auth { Ok(command) } else { Err(trc::ManageSieveEvent::Error @@ -151,9 +151,9 @@ impl<T: SessionStream> Session<T> { | Command::CheckScript | Command::Unauthenticate => { if let State::Authenticated { access_token, .. } = &self.state { - if let Some(rate) = &self.jmap.core.imap.rate_requests { + if let Some(rate) = &self.server.core.imap.rate_requests { if self - .jmap + .server .core .storage .lookup @@ -239,7 +239,7 @@ impl<T: AsyncWrite + AsyncRead + Unpin> Session<T> { impl<T: AsyncWrite + AsyncRead> Session<T> { pub async fn get_script_id(&self, account_id: u32, name: &str) -> trc::Result<u32> { - self.jmap + self.server .core .storage .data diff --git a/crates/managesieve/src/core/mod.rs b/crates/managesieve/src/core/mod.rs index 1f89b895..20f41ab4 100644 --- a/crates/managesieve/src/core/mod.rs +++ b/crates/managesieve/src/core/mod.rs @@ -12,15 +12,13 @@ use std::{borrow::Cow, net::IpAddr, sync::Arc}; use common::{ auth::AccessToken, listener::{limiter::InFlight, ServerInstance}, + Inner, Server, }; -use imap::core::{ImapInstance, Inner}; use imap_proto::receiver::{CommandParser, Receiver}; -use jmap::JMAP; use tokio::io::{AsyncRead, AsyncWrite}; pub struct Session<T: AsyncRead + AsyncWrite> { - pub jmap: JMAP, - pub imap: Arc<Inner>, + pub server: Server, pub instance: Arc<ServerInstance>, pub receiver: Receiver<Command>, pub state: State, @@ -51,12 +49,12 @@ impl State { #[derive(Clone)] pub struct ManageSieveSessionManager { - pub imap: ImapInstance, + pub inner: Arc<Inner>, } impl ManageSieveSessionManager { - pub fn new(imap: ImapInstance) -> Self { - Self { imap } + pub fn new(inner: Arc<Inner>) -> Self { + Self { inner } } } diff --git a/crates/managesieve/src/core/session.rs b/crates/managesieve/src/core/session.rs index 910fe6a3..e56ed5cc 100644 --- a/crates/managesieve/src/core/session.rs +++ b/crates/managesieve/src/core/session.rs @@ -4,9 +4,11 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use common::listener::{SessionData, SessionManager, SessionResult, SessionStream}; +use common::{ + core::BuildServer, + listener::{SessionData, SessionManager, SessionResult, SessionStream}, +}; use imap_proto::receiver::{self, Receiver}; -use jmap::JMAP; use tokio_rustls::server::TlsStream; use crate::SERVER_GREETING; @@ -21,12 +23,11 @@ impl SessionManager for ManageSieveSessionManager { ) -> impl std::future::Future<Output = ()> + Send { async move { // Create session - let jmap = JMAP::from(self.imap.jmap_instance); + let server = self.inner.build_server(); let mut session = Session { - receiver: Receiver::with_max_request_size(jmap.core.imap.max_request_size) + receiver: Receiver::with_max_request_size(server.core.imap.max_request_size) .with_start_state(receiver::State::Command { is_uid: false }), - jmap, - imap: self.imap.imap_inner, + server, instance: session.instance, state: State::NotAuthenticated { auth_failures: 0 }, session_id: session.session_id, @@ -67,9 +68,9 @@ impl<T: SessionStream> Session<T> { tokio::select! { result = tokio::time::timeout( if !matches!(self.state, State::NotAuthenticated {..}) { - self.jmap.core.imap.timeout_auth + self.server.core.imap.timeout_auth } else { - self.jmap.core.imap.timeout_unauth + self.server.core.imap.timeout_unauth }, self.read(&mut buf)) => { match result { @@ -142,8 +143,7 @@ impl<T: SessionStream> Session<T> { instance: self.instance, in_flight: self.in_flight, session_id: self.session_id, - jmap: self.jmap, - imap: self.imap, + server: self.server, receiver: self.receiver, remote_addr: self.remote_addr, }) diff --git a/crates/managesieve/src/op/authenticate.rs b/crates/managesieve/src/op/authenticate.rs index da67f2e5..5157c04a 100644 --- a/crates/managesieve/src/op/authenticate.rs +++ b/crates/managesieve/src/op/authenticate.rs @@ -4,14 +4,19 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use common::listener::{limiter::ConcurrencyLimiter, SessionStream}; +use common::{ + listener::{limiter::ConcurrencyLimiter, SessionStream}, + ConcurrencyLimiters, +}; use directory::Permission; use imap::op::authenticate::{decode_challenge_oauth, decode_challenge_plain}; use imap_proto::{ protocol::authenticate::Mechanism, receiver::{self, Request}, }; -use jmap::auth::rate_limit::ConcurrencyLimiters; +use jmap::auth::{ + authenticate::Authenticator, oauth::token::TokenHandler, rate_limit::RateLimiter, +}; use mail_parser::decoders::base64::base64_decode; use mail_send::Credentials; use std::sync::Arc; @@ -66,22 +71,22 @@ impl<T: SessionStream> Session<T> { }; // Throttle authentication requests - self.jmap.is_auth_allowed_soft(&self.remote_addr).await?; + self.server.is_auth_allowed_soft(&self.remote_addr).await?; // Authenticate let access_token = match credentials { Credentials::Plain { username, secret } | Credentials::XOauth2 { username, secret } => { - self.jmap + self.server .authenticate_plain(&username, &secret, self.remote_addr, self.session_id) .await } Credentials::OAuthBearer { token } => { match self - .jmap + .server .validate_access_token("access_token", &token) .await { - Ok((account_id, _, _)) => self.jmap.core.get_access_token(account_id).await, + Ok((account_id, _, _)) => self.server.get_access_token(account_id).await, Err(err) => Err(err), } } @@ -90,7 +95,7 @@ impl<T: SessionStream> Session<T> { if err.matches(trc::EventType::Auth(trc::AuthEvent::Failed)) { match &self.state { State::NotAuthenticated { auth_failures } - if *auth_failures < self.jmap.core.imap.max_auth_failures => + if *auth_failures < self.server.core.imap.max_auth_failures => { self.state = State::NotAuthenticated { auth_failures: auth_failures + 1, @@ -122,7 +127,7 @@ impl<T: SessionStream> Session<T> { // Cache access token let access_token = Arc::new(access_token); - self.jmap.core.cache_access_token(access_token.clone()); + self.server.cache_access_token(access_token.clone()); // Create session self.state = State::Authenticated { @@ -146,9 +151,11 @@ impl<T: SessionStream> Session<T> { } pub fn get_concurrency_limiter(&self, account_id: u32) -> Option<Arc<ConcurrencyLimiters>> { - let rate = self.jmap.core.imap.rate_concurrent?; - self.imap - .rate_limiter + let rate = self.server.core.imap.rate_concurrent?; + self.server + .inner + .data + .imap_limiter .get(&account_id) .map(|limiter| limiter.clone()) .unwrap_or_else(|| { @@ -156,7 +163,11 @@ impl<T: SessionStream> Session<T> { concurrent_requests: ConcurrencyLimiter::new(rate), concurrent_uploads: ConcurrencyLimiter::new(rate), }); - self.imap.rate_limiter.insert(account_id, limiter.clone()); + self.server + .inner + .data + .imap_limiter + .insert(account_id, limiter.clone()); limiter }) .into() diff --git a/crates/managesieve/src/op/capability.rs b/crates/managesieve/src/op/capability.rs index 13f825db..33c1fb47 100644 --- a/crates/managesieve/src/op/capability.rs +++ b/crates/managesieve/src/op/capability.rs @@ -21,13 +21,13 @@ impl<T: SessionStream> Session<T> { if !self.stream.is_tls() { response.extend_from_slice(b"\"STARTTLS\"\r\n"); } - if self.stream.is_tls() || self.jmap.core.imap.allow_plain_auth { + if self.stream.is_tls() || self.server.core.imap.allow_plain_auth { response.extend_from_slice(b"\"SASL\" \"PLAIN OAUTHBEARER\"\r\n"); } else { response.extend_from_slice(b"\"SASL\" \"OAUTHBEARER\"\r\n"); }; if let Some(sieve) = - self.jmap + self.server .core .jmap .capabilities @@ -62,7 +62,7 @@ impl<T: SessionStream> Session<T> { ManageSieve(trc::ManageSieveEvent::Capabilities), SpanId = self.session_id, Tls = self.stream.is_tls(), - Strict = !self.jmap.core.imap.allow_plain_auth, + Strict = !self.server.core.imap.allow_plain_auth, Elapsed = op_start.elapsed() ); diff --git a/crates/managesieve/src/op/checkscript.rs b/crates/managesieve/src/op/checkscript.rs index 58f79217..8e70b921 100644 --- a/crates/managesieve/src/op/checkscript.rs +++ b/crates/managesieve/src/op/checkscript.rs @@ -26,7 +26,7 @@ impl<T: SessionStream> Session<T> { } let script = request.tokens.into_iter().next().unwrap().unwrap_bytes(); - self.jmap + self.server .core .sieve .untrusted_compiler diff --git a/crates/managesieve/src/op/deletescript.rs b/crates/managesieve/src/op/deletescript.rs index f620769e..1a2b218c 100644 --- a/crates/managesieve/src/op/deletescript.rs +++ b/crates/managesieve/src/op/deletescript.rs @@ -9,6 +9,7 @@ use std::time::Instant; use common::listener::SessionStream; use directory::Permission; use imap_proto::receiver::Request; +use jmap::{changes::write::ChangeLog, sieve::set::SieveScriptSet}; use jmap_proto::types::collection::Collection; use store::write::log::ChangeLogBuilder; use trc::AddContext; @@ -37,7 +38,7 @@ impl<T: SessionStream> Session<T> { let account_id = access_token.primary_id(); let document_id = self.get_script_id(account_id, &name).await?; if self - .jmap + .server .sieve_script_delete(&access_token.as_resource_token(), document_id, true) .await .caused_by(trc::location!())? @@ -45,7 +46,7 @@ impl<T: SessionStream> Session<T> { // Write changes let mut changelog = ChangeLogBuilder::new(); changelog.log_delete(Collection::SieveScript, document_id); - self.jmap + self.server .commit_changes(account_id, changelog) .await .caused_by(trc::location!())?; diff --git a/crates/managesieve/src/op/getscript.rs b/crates/managesieve/src/op/getscript.rs index e21f368d..8837521a 100644 --- a/crates/managesieve/src/op/getscript.rs +++ b/crates/managesieve/src/op/getscript.rs @@ -9,7 +9,7 @@ use std::time::Instant; use common::listener::SessionStream; use directory::Permission; use imap_proto::receiver::Request; -use jmap::sieve::set::ObjectBlobId; +use jmap::{blob::download::BlobDownload, sieve::set::ObjectBlobId, JmapMethods}; use jmap_proto::{ object::Object, types::{collection::Collection, property::Property, value::Value}, @@ -37,7 +37,7 @@ impl<T: SessionStream> Session<T> { let account_id = self.state.access_token().primary_id(); let document_id = self.get_script_id(account_id, &name).await?; let (blob_section, blob_hash) = self - .jmap + .server .get_property::<Object<Value>>( account_id, Collection::SieveScript, @@ -61,7 +61,7 @@ impl<T: SessionStream> Session<T> { .code(ResponseCode::TryLater) })?; let script = self - .jmap + .server .get_blob_section(&blob_hash, &blob_section) .await .caused_by(trc::location!())? diff --git a/crates/managesieve/src/op/havespace.rs b/crates/managesieve/src/op/havespace.rs index 056c6208..a2b1a212 100644 --- a/crates/managesieve/src/op/havespace.rs +++ b/crates/managesieve/src/op/havespace.rs @@ -9,6 +9,7 @@ use std::time::Instant; use common::listener::SessionStream; use directory::Permission; use imap_proto::receiver::Request; +use jmap::JmapMethods; use trc::AddContext; use crate::core::{Command, ResponseCode, Session, StatusResponse}; @@ -52,7 +53,7 @@ impl<T: SessionStream> Session<T> { if access_token.quota == 0 || size as i64 + self - .jmap + .server .get_used_quota(account_id) .await .caused_by(trc::location!())? diff --git a/crates/managesieve/src/op/listscripts.rs b/crates/managesieve/src/op/listscripts.rs index 5659d293..35fd38ca 100644 --- a/crates/managesieve/src/op/listscripts.rs +++ b/crates/managesieve/src/op/listscripts.rs @@ -8,6 +8,7 @@ use std::time::Instant; use common::listener::SessionStream; use directory::Permission; +use jmap::JmapMethods; use jmap_proto::{ object::Object, types::{collection::Collection, property::Property, value::Value}, @@ -24,7 +25,7 @@ impl<T: SessionStream> Session<T> { let op_start = Instant::now(); let account_id = self.state.access_token().primary_id(); let document_ids = self - .jmap + .server .get_document_ids(account_id, Collection::SieveScript) .await .caused_by(trc::location!())? @@ -39,7 +40,7 @@ impl<T: SessionStream> Session<T> { for document_id in document_ids { if let Some(script) = self - .jmap + .server .get_property::<Object<Value>>( account_id, Collection::SieveScript, diff --git a/crates/managesieve/src/op/putscript.rs b/crates/managesieve/src/op/putscript.rs index d980a768..d7a36279 100644 --- a/crates/managesieve/src/op/putscript.rs +++ b/crates/managesieve/src/op/putscript.rs @@ -9,7 +9,11 @@ use std::time::Instant; use common::listener::SessionStream; use directory::Permission; use imap_proto::receiver::Request; -use jmap::sieve::set::{ObjectBlobId, SCHEMA}; +use jmap::{ + blob::upload::BlobUpload, + sieve::set::{ObjectBlobId, SCHEMA}, + JmapMethods, +}; use jmap_proto::{ object::{index::ObjectIndexBuilder, Object}, types::{blob::BlobId, collection::Collection, property::Property, value::Value}, @@ -54,19 +58,19 @@ impl<T: SessionStream> Session<T> { // Check quota let resource_token = self.state.access_token().as_resource_token(); let account_id = resource_token.account_id; - self.jmap + self.server .has_available_quota(&resource_token, script_bytes.len() as u64) .await .caused_by(trc::location!())?; if self - .jmap + .server .get_document_ids(account_id, Collection::SieveScript) .await .caused_by(trc::location!())? .map(|ids| ids.len() as usize) .unwrap_or(0) - > self.jmap.core.jmap.sieve_max_scripts + > self.server.core.jmap.sieve_max_scripts { return Err(trc::ManageSieveEvent::Error .into_err() @@ -76,7 +80,7 @@ impl<T: SessionStream> Session<T> { // Compile script match self - .jmap + .server .core .sieve .untrusted_compiler @@ -103,7 +107,7 @@ impl<T: SessionStream> Session<T> { if let Some(document_id) = self.validate_name(account_id, &name).await? { // Obtain script values let script = self - .jmap + .server .get_property::<HashedValue<Object<Value>>>( account_id, Collection::SieveScript, @@ -127,7 +131,7 @@ impl<T: SessionStream> Session<T> { // Write script blob let blob_id = BlobId::new( - self.jmap + self.server .put_blob(account_id, &script_bytes, false) .await .caused_by(trc::location!())? @@ -168,7 +172,7 @@ impl<T: SessionStream> Session<T> { // Update tenant quota #[cfg(feature = "enterprise")] - if self.jmap.core.is_enterprise_edition() { + if self.server.core.is_enterprise_edition() { if let Some(tenant) = resource_token.tenant { batch.add(DirectoryClass::UsedQuota(tenant.id), update_quota); } @@ -183,7 +187,7 @@ impl<T: SessionStream> Session<T> { .with_property(Property::BlobId, Value::BlobId(blob_id)), ), ); - self.jmap + self.server .write_batch(batch) .await .caused_by(trc::location!())?; @@ -199,7 +203,7 @@ impl<T: SessionStream> Session<T> { } else { // Write script blob let blob_id = BlobId::new( - self.jmap + self.server .put_blob(account_id, &script_bytes, false) .await? .hash, @@ -236,14 +240,14 @@ impl<T: SessionStream> Session<T> { // Update tenant quota #[cfg(feature = "enterprise")] - if self.jmap.core.is_enterprise_edition() { + if self.server.core.is_enterprise_edition() { if let Some(tenant) = resource_token.tenant { batch.add(DirectoryClass::UsedQuota(tenant.id), script_size); } } let assigned_ids = self - .jmap + .server .write_batch(batch) .await .caused_by(trc::location!())?; @@ -265,7 +269,7 @@ impl<T: SessionStream> Session<T> { Err(trc::ManageSieveEvent::Error .into_err() .details("Script name cannot be empty.")) - } else if name.len() > self.jmap.core.jmap.sieve_max_script_name { + } else if name.len() > self.server.core.jmap.sieve_max_script_name { Err(trc::ManageSieveEvent::Error .into_err() .details("Script name is too long.")) @@ -275,7 +279,7 @@ impl<T: SessionStream> Session<T> { .details("The 'vacation' name is reserved, please use a different name.")) } else { Ok(self - .jmap + .server .filter( account_id, Collection::SieveScript, diff --git a/crates/managesieve/src/op/renamescript.rs b/crates/managesieve/src/op/renamescript.rs index 6bd84345..8164768c 100644 --- a/crates/managesieve/src/op/renamescript.rs +++ b/crates/managesieve/src/op/renamescript.rs @@ -9,7 +9,7 @@ use std::time::Instant; use common::listener::SessionStream; use directory::Permission; use imap_proto::receiver::Request; -use jmap::sieve::set::SCHEMA; +use jmap::{changes::write::ChangeLog, sieve::set::SCHEMA, JmapMethods}; use jmap_proto::{ object::{index::ObjectIndexBuilder, Object}, types::{collection::Collection, property::Property, value::Value}, @@ -62,7 +62,7 @@ impl<T: SessionStream> Session<T> { // Obtain script values let script = self - .jmap + .server .get_property::<HashedValue<Object<Value>>>( account_id, Collection::SieveScript, @@ -92,13 +92,13 @@ impl<T: SessionStream> Session<T> { ), ); if !batch.is_empty() { - self.jmap + self.server .write_batch(batch) .await .caused_by(trc::location!())?; let mut changelog = ChangeLogBuilder::new(); changelog.log_update(Collection::SieveScript, document_id); - self.jmap + self.server .commit_changes(account_id, changelog) .await .caused_by(trc::location!())?; diff --git a/crates/managesieve/src/op/setactive.rs b/crates/managesieve/src/op/setactive.rs index fa88b725..4ea6283c 100644 --- a/crates/managesieve/src/op/setactive.rs +++ b/crates/managesieve/src/op/setactive.rs @@ -9,6 +9,7 @@ use std::time::Instant; use common::listener::SessionStream; use directory::Permission; use imap_proto::receiver::Request; +use jmap::{changes::write::ChangeLog, sieve::set::SieveScriptSet}; use jmap_proto::types::collection::Collection; use store::write::log::ChangeLogBuilder; use trc::AddContext; @@ -35,7 +36,7 @@ impl<T: SessionStream> Session<T> { // De/activate script let account_id = self.state.access_token().primary_id(); let changes = self - .jmap + .server .sieve_activate_script( account_id, if !name.is_empty() { @@ -53,7 +54,7 @@ impl<T: SessionStream> Session<T> { for (document_id, _) in changes { changelog.log_update(Collection::SieveScript, document_id); } - self.jmap + self.server .commit_changes(account_id, changelog) .await .caused_by(trc::location!())?; diff --git a/crates/pop3/src/client.rs b/crates/pop3/src/client.rs index 5cea1276..ccc6f88c 100644 --- a/crates/pop3/src/client.rs +++ b/crates/pop3/src/client.rs @@ -163,7 +163,7 @@ impl<T: SessionStream> Session<T> { | Command::Pass { .. } | Command::Apop { .. } => { if let State::NotAuthenticated { username, .. } = &self.state { - if self.stream.is_tls() || self.jmap.core.imap.allow_plain_auth { + if self.stream.is_tls() || self.server.core.imap.allow_plain_auth { if !matches!(command, Command::Pass { .. }) || username.is_some() { Ok(command) } else { @@ -211,9 +211,9 @@ impl<T: SessionStream> Session<T> { | Command::Stat | Command::Rset => { if let State::Authenticated { mailbox, .. } = &self.state { - if let Some(rate) = &self.jmap.core.imap.rate_requests { + if let Some(rate) = &self.server.core.imap.rate_requests { if self - .jmap + .server .core .storage .lookup diff --git a/crates/pop3/src/lib.rs b/crates/pop3/src/lib.rs index 5212dafe..a7429338 100644 --- a/crates/pop3/src/lib.rs +++ b/crates/pop3/src/lib.rs @@ -9,9 +9,8 @@ use std::{net::IpAddr, sync::Arc}; use common::{ auth::AccessToken, listener::{limiter::InFlight, ServerInstance, SessionStream}, + Inner, Server, }; -use imap::core::{ImapInstance, Inner}; -use jmap::JMAP; use mailbox::Mailbox; use protocol::request::Parser; @@ -25,18 +24,17 @@ static SERVER_GREETING: &str = "+OK Stalwart POP3 at your service.\r\n"; #[derive(Clone)] pub struct Pop3SessionManager { - pub pop3: ImapInstance, + pub inner: Arc<Inner>, } impl Pop3SessionManager { - pub fn new(pop3: ImapInstance) -> Self { - Self { pop3 } + pub fn new(inner: Arc<Inner>) -> Self { + Self { inner } } } pub struct Session<T: SessionStream> { - pub jmap: JMAP, - pub imap: Arc<Inner>, + pub server: Server, pub instance: Arc<ServerInstance>, pub receiver: Parser, pub state: State, diff --git a/crates/pop3/src/mailbox.rs b/crates/pop3/src/mailbox.rs index 10accd12..708bd86d 100644 --- a/crates/pop3/src/mailbox.rs +++ b/crates/pop3/src/mailbox.rs @@ -7,7 +7,10 @@ use std::collections::BTreeMap; use common::listener::SessionStream; -use jmap::mailbox::{UidMailbox, INBOX_ID}; +use jmap::{ + mailbox::{set::MailboxSet, UidMailbox, INBOX_ID}, + JmapMethods, +}; use jmap_proto::{ object::Object, types::{collection::Collection, property::Property, value::Value}, @@ -39,7 +42,7 @@ impl<T: SessionStream> Session<T> { pub async fn fetch_mailbox(&self, account_id: u32) -> trc::Result<Mailbox> { // Obtain message ids let message_ids = self - .jmap + .server .get_tag( account_id, Collection::Email, @@ -58,12 +61,12 @@ impl<T: SessionStream> Session<T> { let mut message_sizes = AHashMap::new(); // Obtain UID validity - self.jmap + self.server .mailbox_get_or_create(account_id) .await .caused_by(trc::location!())?; let uid_validity = self - .jmap + .server .get_property::<Object<Value>>( account_id, Collection::Mailbox, @@ -83,7 +86,7 @@ impl<T: SessionStream> Session<T> { .map(|v| v as u32)?; // Obtain message sizes - self.jmap + self.server .core .storage .data @@ -122,7 +125,7 @@ impl<T: SessionStream> Session<T> { // Sort by UID for (message_id, uid_mailbox) in self - .jmap + .server .get_properties::<Vec<UidMailbox>, _, _>( account_id, Collection::Email, diff --git a/crates/pop3/src/op/authenticate.rs b/crates/pop3/src/op/authenticate.rs index fe7ca09d..1924d4f2 100644 --- a/crates/pop3/src/op/authenticate.rs +++ b/crates/pop3/src/op/authenticate.rs @@ -4,10 +4,15 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use common::listener::{limiter::ConcurrencyLimiter, SessionStream}; +use common::{ + listener::{limiter::ConcurrencyLimiter, SessionStream}, + ConcurrencyLimiters, +}; use directory::Permission; use imap::op::authenticate::{decode_challenge_oauth, decode_challenge_plain}; -use jmap::auth::rate_limit::ConcurrencyLimiters; +use jmap::auth::{ + authenticate::Authenticator, oauth::token::TokenHandler, rate_limit::RateLimiter, +}; use mail_parser::decoders::base64::base64_decode; use mail_send::Credentials; use std::sync::Arc; @@ -60,22 +65,22 @@ impl<T: SessionStream> Session<T> { pub async fn handle_auth(&mut self, credentials: Credentials<String>) -> trc::Result<()> { // Throttle authentication requests - self.jmap.is_auth_allowed_soft(&self.remote_addr).await?; + self.server.is_auth_allowed_soft(&self.remote_addr).await?; // Authenticate let access_token = match credentials { Credentials::Plain { username, secret } | Credentials::XOauth2 { username, secret } => { - self.jmap + self.server .authenticate_plain(&username, &secret, self.remote_addr, self.session_id) .await } Credentials::OAuthBearer { token } => { match self - .jmap + .server .validate_access_token("access_token", &token) .await { - Ok((account_id, _, _)) => self.jmap.core.get_access_token(account_id).await, + Ok((account_id, _, _)) => self.server.get_access_token(account_id).await, Err(err) => Err(err), } } @@ -86,7 +91,7 @@ impl<T: SessionStream> Session<T> { State::NotAuthenticated { auth_failures, username, - } if *auth_failures < self.jmap.core.imap.max_auth_failures => { + } if *auth_failures < self.server.core.imap.max_auth_failures => { self.state = State::NotAuthenticated { auth_failures: auth_failures + 1, username: username.clone(), @@ -118,7 +123,7 @@ impl<T: SessionStream> Session<T> { // Cache access token let access_token = Arc::new(access_token); - self.jmap.core.cache_access_token(access_token.clone()); + self.server.cache_access_token(access_token.clone()); // Fetch mailbox let mailbox = self.fetch_mailbox(access_token.primary_id()).await?; @@ -133,9 +138,11 @@ impl<T: SessionStream> Session<T> { } pub fn get_concurrency_limiter(&self, account_id: u32) -> Option<Arc<ConcurrencyLimiters>> { - let rate = self.jmap.core.imap.rate_concurrent?; - self.imap - .rate_limiter + let rate = self.server.core.imap.rate_concurrent?; + self.server + .inner + .data + .imap_limiter .get(&account_id) .map(|limiter| limiter.clone()) .unwrap_or_else(|| { @@ -143,7 +150,11 @@ impl<T: SessionStream> Session<T> { concurrent_requests: ConcurrencyLimiter::new(rate), concurrent_uploads: ConcurrencyLimiter::new(rate), }); - self.imap.rate_limiter.insert(account_id, limiter.clone()); + self.server + .inner + .data + .imap_limiter + .insert(account_id, limiter.clone()); limiter }) .into() diff --git a/crates/pop3/src/op/delete.rs b/crates/pop3/src/op/delete.rs index 4e7799b2..e8b53c49 100644 --- a/crates/pop3/src/op/delete.rs +++ b/crates/pop3/src/op/delete.rs @@ -8,6 +8,9 @@ use std::time::Instant; use common::listener::SessionStream; use directory::Permission; +use jmap::{ + changes::write::ChangeLog, email::delete::EmailDeletion, services::state::StateManager, +}; use jmap_proto::types::{state::StateChange, type_state::DataType}; use store::roaring::RoaringBitmap; use trc::AddContext; @@ -87,16 +90,18 @@ impl<T: SessionStream> Session<T> { if !deleted.is_empty() { let num_deleted = deleted.len(); let (changes, not_deleted) = self - .jmap + .server .emails_tombstone(mailbox.account_id, deleted) .await .caused_by(trc::location!())?; if !changes.is_empty() { - if let Ok(change_id) = - self.jmap.commit_changes(mailbox.account_id, changes).await + if let Ok(change_id) = self + .server + .commit_changes(mailbox.account_id, changes) + .await { - self.jmap + self.server .broadcast_state_change( StateChange::new(mailbox.account_id) .with_change(DataType::Email, change_id) diff --git a/crates/pop3/src/op/fetch.rs b/crates/pop3/src/op/fetch.rs index be322723..9d58cf45 100644 --- a/crates/pop3/src/op/fetch.rs +++ b/crates/pop3/src/op/fetch.rs @@ -8,7 +8,7 @@ use std::time::Instant; use common::listener::SessionStream; use directory::Permission; -use jmap::email::metadata::MessageMetadata; +use jmap::{blob::download::BlobDownload, email::metadata::MessageMetadata, JmapMethods}; use jmap_proto::types::{collection::Collection, property::Property}; use store::write::Bincode; use trc::AddContext; @@ -26,7 +26,7 @@ impl<T: SessionStream> Session<T> { let mailbox = self.state.mailbox(); if let Some(message) = mailbox.messages.get(msg.saturating_sub(1) as usize) { if let Some(metadata) = self - .jmap + .server .get_property::<Bincode<MessageMetadata>>( mailbox.account_id, Collection::Email, @@ -37,7 +37,7 @@ impl<T: SessionStream> Session<T> { .caused_by(trc::location!())? { if let Some(bytes) = self - .jmap + .server .get_blob(&metadata.inner.blob_hash, 0..usize::MAX) .await .caused_by(trc::location!())? diff --git a/crates/pop3/src/op/mod.rs b/crates/pop3/src/op/mod.rs index 0488caa8..bfc1db7d 100644 --- a/crates/pop3/src/op/mod.rs +++ b/crates/pop3/src/op/mod.rs @@ -18,7 +18,7 @@ pub mod list; impl<T: SessionStream> Session<T> { pub async fn handle_capa(&mut self) -> trc::Result<()> { - let mechanisms = if self.stream.is_tls() || self.jmap.core.imap.allow_plain_auth { + let mechanisms = if self.stream.is_tls() || self.server.core.imap.allow_plain_auth { vec![Mechanism::Plain, Mechanism::OAuthBearer] } else { vec![Mechanism::OAuthBearer] @@ -28,7 +28,7 @@ impl<T: SessionStream> Session<T> { Pop3(trc::Pop3Event::Capabilities), SpanId = self.session_id, Tls = self.stream.is_tls(), - Strict = !self.jmap.core.imap.allow_plain_auth, + Strict = !self.server.core.imap.allow_plain_auth, Elapsed = trc::Value::Duration(0) ); diff --git a/crates/pop3/src/session.rs b/crates/pop3/src/session.rs index ca0ae1d9..d016170a 100644 --- a/crates/pop3/src/session.rs +++ b/crates/pop3/src/session.rs @@ -6,8 +6,10 @@ use std::borrow::Cow; -use common::listener::{SessionData, SessionManager, SessionResult, SessionStream}; -use jmap::JMAP; +use common::{ + core::BuildServer, + listener::{SessionData, SessionManager, SessionResult, SessionStream}, +}; use tokio_rustls::server::TlsStream; use crate::{ @@ -28,8 +30,7 @@ impl SessionManager for Pop3SessionManager { ) -> impl std::future::Future<Output = ()> + Send { async move { let mut session = Session { - jmap: JMAP::from(self.pop3.jmap_instance), - imap: self.pop3.imap_inner, + server: self.inner.build_server(), instance: session.instance, receiver: Parser::default(), state: State::NotAuthenticated { @@ -71,9 +72,9 @@ impl<T: SessionStream> Session<T> { tokio::select! { result = tokio::time::timeout( if !matches!(self.state, State::NotAuthenticated {..}) { - self.jmap.core.imap.timeout_auth + self.server.core.imap.timeout_auth } else { - self.jmap.core.imap.timeout_unauth + self.server.core.imap.timeout_unauth }, self.stream.read(&mut buf)) => { match result { @@ -141,8 +142,7 @@ impl<T: SessionStream> Session<T> { .instance .tls_accept(self.stream, self.session_id) .await?, - jmap: self.jmap, - imap: self.imap, + server: self.server, instance: self.instance, receiver: self.receiver, state: self.state, diff --git a/crates/smtp/src/core/mod.rs b/crates/smtp/src/core/mod.rs index d6832d7a..810521a1 100644 --- a/crates/smtp/src/core/mod.rs +++ b/crates/smtp/src/core/mod.rs @@ -12,86 +12,40 @@ use std::{ }; use common::{ - config::{scripts::ScriptCache, smtp::auth::VerifyStrategy}, + config::smtp::auth::VerifyStrategy, listener::{ limiter::{ConcurrencyLimiter, InFlight}, ServerInstance, }, - Core, Ipc, SharedCore, + Inner, Server, }; -use dashmap::DashMap; use directory::Directory; use mail_auth::{IprevOutput, SpfOutput}; use smtp_proto::request::receiver::{ BdatReceiver, DataReceiver, DummyDataReceiver, DummyLineReceiver, LineReceiver, RequestReceiver, }; -use tokio::{ - io::{AsyncRead, AsyncWrite}, - sync::mpsc, -}; -use tokio_rustls::TlsConnector; +use tokio::io::{AsyncRead, AsyncWrite}; use utils::snowflake::SnowflakeIdGenerator; use crate::{ inbound::auth::SaslToken, - queue::{self, DomainPart, QueueId}, - reporting, + queue::{DomainPart, QueueId}, }; -use self::throttle::{ThrottleKey, ThrottleKeyHasherBuilder}; - pub mod params; pub mod throttle; #[derive(Clone)] -pub struct SmtpInstance { - pub inner: Arc<Inner>, - pub core: SharedCore, -} - -impl SmtpInstance { - pub fn new(core: SharedCore, inner: impl Into<Arc<Inner>>) -> Self { - Self { - core, - inner: inner.into(), - } - } -} - -#[derive(Clone)] pub struct SmtpSessionManager { - pub inner: SmtpInstance, + pub inner: Arc<Inner>, } impl SmtpSessionManager { - pub fn new(inner: SmtpInstance) -> Self { + pub fn new(inner: Arc<Inner>) -> Self { Self { inner } } } -#[derive(Clone)] -pub struct SMTP { - pub core: Arc<Core>, - pub inner: Arc<Inner>, -} - -pub struct Inner { - pub session_throttle: DashMap<ThrottleKey, ConcurrencyLimiter, ThrottleKeyHasherBuilder>, - pub queue_throttle: DashMap<ThrottleKey, ConcurrencyLimiter, ThrottleKeyHasherBuilder>, - pub queue_tx: mpsc::Sender<queue::Event>, - pub report_tx: mpsc::Sender<reporting::Event>, - pub queue_id_gen: SnowflakeIdGenerator, - pub span_id_gen: Arc<SnowflakeIdGenerator>, - pub connectors: TlsConnectors, - pub ipc: Ipc, - pub script_cache: ScriptCache, -} - -pub struct TlsConnectors { - pub pki_verify: TlsConnector, - pub dummy_verify: TlsConnector, -} - pub enum State { Request(RequestReceiver), Bdat(BdatReceiver), @@ -107,7 +61,7 @@ pub struct Session<T: AsyncWrite + AsyncRead> { pub hostname: String, pub state: State, pub instance: Arc<ServerInstance>, - pub core: SMTP, + pub server: Server, pub stream: T, pub data: SessionData, pub params: SessionParameters, @@ -260,15 +214,6 @@ impl PartialOrd for SessionAddress { } } -impl From<SmtpInstance> for SMTP { - fn from(value: SmtpInstance) -> Self { - SMTP { - core: value.core.load_full(), - inner: value.inner, - } - } -} - static SIEVE: LazyLock<Arc<ServerInstance>> = LazyLock::new(|| { Arc::new(ServerInstance { id: "sieve".to_string(), @@ -282,12 +227,16 @@ static SIEVE: LazyLock<Arc<ServerInstance>> = LazyLock::new(|| { }); impl Session<common::listener::stream::NullIo> { - pub fn local(core: SMTP, instance: std::sync::Arc<ServerInstance>, data: SessionData) -> Self { + pub fn local( + server: Server, + instance: std::sync::Arc<ServerInstance>, + data: SessionData, + ) -> Self { Session { hostname: "localhost".to_string(), state: State::None, instance, - core, + server, stream: common::listener::stream::NullIo::default(), data, params: SessionParameters { @@ -315,14 +264,14 @@ impl Session<common::listener::stream::NullIo> { } pub fn sieve( - core: SMTP, + server: Server, mail_from: SessionAddress, rcpt_to: Vec<SessionAddress>, message: Vec<u8>, session_id: u64, ) -> Self { Self::local( - core, + server, SIEVE.clone(), SessionData::local(mail_from.into(), rcpt_to, message, session_id), ) @@ -398,25 +347,3 @@ impl SessionAddress { } } } - -#[cfg(feature = "test_mode")] -impl Default for Inner { - fn default() -> Self { - Self { - session_throttle: Default::default(), - queue_throttle: Default::default(), - queue_tx: mpsc::channel(1).0, - report_tx: mpsc::channel(1).0, - queue_id_gen: Default::default(), - span_id_gen: Arc::new(SnowflakeIdGenerator::new()), - connectors: TlsConnectors { - pki_verify: mail_send::smtp::tls::build_tls_connector(false), - dummy_verify: mail_send::smtp::tls::build_tls_connector(true), - }, - ipc: Ipc { - delivery_tx: mpsc::channel(1).0, - }, - script_cache: Default::default(), - } - } -} diff --git a/crates/smtp/src/core/params.rs b/crates/smtp/src/core/params.rs index 4e555a24..5a4e873e 100644 --- a/crates/smtp/src/core/params.rs +++ b/crates/smtp/src/core/params.rs @@ -12,51 +12,45 @@ use super::Session; impl<T: SessionStream> Session<T> { pub async fn eval_session_params(&mut self) { - let c = &self.core.core.smtp.session; + let c = &self.server.core.smtp.session; self.data.bytes_left = self - .core - .core + .server .eval_if(&c.transfer_limit, self, self.data.session_id) .await .unwrap_or(250 * 1024 * 1024); self.data.valid_until += self - .core - .core + .server .eval_if(&c.duration, self, self.data.session_id) .await .unwrap_or_else(|| Duration::from_secs(15 * 60)); self.params.timeout = self - .core - .core + .server .eval_if(&c.timeout, self, self.data.session_id) .await .unwrap_or_else(|| Duration::from_secs(5 * 60)); self.params.spf_ehlo = self - .core - .core + .server .eval_if( - &self.core.core.smtp.mail_auth.spf.verify_ehlo, + &self.server.core.smtp.mail_auth.spf.verify_ehlo, self, self.data.session_id, ) .await .unwrap_or(VerifyStrategy::Relaxed); self.params.spf_mail_from = self - .core - .core + .server .eval_if( - &self.core.core.smtp.mail_auth.spf.verify_mail_from, + &self.server.core.smtp.mail_auth.spf.verify_mail_from, self, self.data.session_id, ) .await .unwrap_or(VerifyStrategy::Relaxed); self.params.iprev = self - .core - .core + .server .eval_if( - &self.core.core.smtp.mail_auth.iprev.verify, + &self.server.core.smtp.mail_auth.iprev.verify, self, self.data.session_id, ) @@ -64,65 +58,56 @@ impl<T: SessionStream> Session<T> { .unwrap_or(VerifyStrategy::Relaxed); // Ehlo parameters - let ec = &self.core.core.smtp.session.ehlo; + let ec = &self.server.core.smtp.session.ehlo; self.params.ehlo_require = self - .core - .core + .server .eval_if(&ec.require, self, self.data.session_id) .await .unwrap_or(true); self.params.ehlo_reject_non_fqdn = self - .core - .core + .server .eval_if(&ec.reject_non_fqdn, self, self.data.session_id) .await .unwrap_or(true); // Auth parameters - let ac = &self.core.core.smtp.session.auth; + let ac = &self.server.core.smtp.session.auth; self.params.auth_directory = self - .core - .core + .server .eval_if::<String, _>(&ac.directory, self, self.data.session_id) .await - .and_then(|name| self.core.core.get_directory(&name)) + .and_then(|name| self.server.get_directory(&name)) .cloned(); self.params.auth_require = self - .core - .core + .server .eval_if(&ac.require, self, self.data.session_id) .await .unwrap_or(false); self.params.auth_errors_max = self - .core - .core + .server .eval_if(&ac.errors_max, self, self.data.session_id) .await .unwrap_or(3); self.params.auth_errors_wait = self - .core - .core + .server .eval_if(&ac.errors_wait, self, self.data.session_id) .await .unwrap_or_else(|| Duration::from_secs(30)); self.params.auth_match_sender = self - .core - .core + .server .eval_if(&ac.must_match_sender, self, self.data.session_id) .await .unwrap_or(true); // VRFY/EXPN parameters - let ec = &self.core.core.smtp.session.extensions; + let ec = &self.server.core.smtp.session.extensions; self.params.can_expn = self - .core - .core + .server .eval_if(&ec.expn, self, self.data.session_id) .await .unwrap_or(false); self.params.can_vrfy = self - .core - .core + .server .eval_if(&ec.vrfy, self, self.data.session_id) .await .unwrap_or(false); @@ -130,24 +115,21 @@ impl<T: SessionStream> Session<T> { pub async fn eval_post_auth_params(&mut self) { // Refresh VRFY/EXPN parameters - let ec = &self.core.core.smtp.session.extensions; + let ec = &self.server.core.smtp.session.extensions; self.params.can_expn = self - .core - .core + .server .eval_if(&ec.expn, self, self.data.session_id) .await .unwrap_or(false); self.params.can_vrfy = self - .core - .core + .server .eval_if(&ec.vrfy, self, self.data.session_id) .await .unwrap_or(false); self.params.auth_match_sender = self - .core - .core + .server .eval_if( - &self.core.core.smtp.session.auth.must_match_sender, + &self.server.core.smtp.session.auth.must_match_sender, self, self.data.session_id, ) @@ -156,30 +138,26 @@ impl<T: SessionStream> Session<T> { } pub async fn eval_rcpt_params(&mut self) { - let rc = &self.core.core.smtp.session.rcpt; + let rc = &self.server.core.smtp.session.rcpt; self.params.rcpt_errors_max = self - .core - .core + .server .eval_if(&rc.errors_max, self, self.data.session_id) .await .unwrap_or(10); self.params.rcpt_errors_wait = self - .core - .core + .server .eval_if(&rc.errors_wait, self, self.data.session_id) .await .unwrap_or_else(|| Duration::from_secs(30)); self.params.rcpt_max = self - .core - .core + .server .eval_if(&rc.max_recipients, self, self.data.session_id) .await .unwrap_or(100); self.params.rcpt_dsn = self - .core - .core + .server .eval_if( - &self.core.core.smtp.session.extensions.dsn, + &self.server.core.smtp.session.extensions.dsn, self, self.data.session_id, ) @@ -187,10 +165,9 @@ impl<T: SessionStream> Session<T> { .unwrap_or(true); self.params.max_message_size = self - .core - .core + .server .eval_if( - &self.core.core.smtp.session.data.max_message_size, + &self.server.core.smtp.session.data.max_message_size, self, self.data.session_id, ) diff --git a/crates/smtp/src/core/throttle.rs b/crates/smtp/src/core/throttle.rs index dc9fadf7..9abed15c 100644 --- a/crates/smtp/src/core/throttle.rs +++ b/crates/smtp/src/core/throttle.rs @@ -8,66 +8,13 @@ use common::{ config::smtp::{queue::QueueQuota, *}, expr::{functions::ResolveVariable, *}, listener::{limiter::ConcurrencyLimiter, SessionStream}, + ThrottleKey, }; use dashmap::mapref::entry::Entry; use trc::SmtpEvent; use utils::config::Rate; -use std::{ - hash::{BuildHasher, Hash, Hasher}, - sync::atomic::Ordering, -}; - -use super::{Session, SMTP}; - -#[derive(Debug, Clone, Eq)] -pub struct ThrottleKey { - hash: [u8; 32], -} - -impl PartialEq for ThrottleKey { - fn eq(&self, other: &Self) -> bool { - self.hash == other.hash - } -} - -impl Hash for ThrottleKey { - fn hash<H: Hasher>(&self, state: &mut H) { - self.hash.hash(state); - } -} - -impl AsRef<[u8]> for ThrottleKey { - fn as_ref(&self) -> &[u8] { - &self.hash - } -} - -#[derive(Default)] -pub struct ThrottleKeyHasher { - hash: u64, -} - -impl Hasher for ThrottleKeyHasher { - fn finish(&self) -> u64 { - self.hash - } - - fn write(&mut self, bytes: &[u8]) { - self.hash = u64::from_ne_bytes((&bytes[..std::mem::size_of::<u64>()]).try_into().unwrap()); - } -} - -#[derive(Clone, Default)] -pub struct ThrottleKeyHasherBuilder {} - -impl BuildHasher for ThrottleKeyHasherBuilder { - type Hasher = ThrottleKeyHasher; - - fn build_hasher(&self) -> Self::Hasher { - ThrottleKeyHasher::default() - } -} +use super::Session; pub trait NewKey: Sized { fn new_key(&self, e: &impl ResolveVariable) -> ThrottleKey; @@ -199,18 +146,17 @@ impl NewKey for Throttle { impl<T: SessionStream> Session<T> { pub async fn is_allowed(&mut self) -> bool { let throttles = if !self.data.rcpt_to.is_empty() { - &self.core.core.smtp.session.throttle.rcpt_to + &self.server.core.smtp.session.throttle.rcpt_to } else if self.data.mail_from.is_some() { - &self.core.core.smtp.session.throttle.mail_from + &self.server.core.smtp.session.throttle.mail_from } else { - &self.core.core.smtp.session.throttle.connect + &self.server.core.smtp.session.throttle.connect }; for t in throttles { if t.expr.is_empty() || self - .core - .core + .server .eval_expr(&t.expr, self, "throttle", self.data.session_id) .await .unwrap_or(false) @@ -233,7 +179,13 @@ impl<T: SessionStream> Session<T> { // Check concurrency if let Some(concurrency) = &t.concurrency { - match self.core.inner.session_throttle.entry(key.clone()) { + match self + .server + .inner + .data + .smtp_session_throttle + .entry(key.clone()) + { Entry::Occupied(mut e) => { let limiter = e.get_mut(); if let Some(inflight) = limiter.is_allowed() { @@ -261,7 +213,7 @@ impl<T: SessionStream> Session<T> { // Check rate if let Some(rate) = &t.rate { if self - .core + .server .core .storage .lookup @@ -296,7 +248,7 @@ impl<T: SessionStream> Session<T> { hasher.update(&rate.period.as_secs().to_ne_bytes()[..]); hasher.update(&rate.requests.to_ne_bytes()[..]); - self.core + self.server .core .storage .lookup @@ -306,11 +258,3 @@ impl<T: SessionStream> Session<T> { .is_none() } } - -impl SMTP { - pub fn cleanup(&self) { - for throttle in [&self.inner.session_throttle, &self.inner.queue_throttle] { - throttle.retain(|_, v| v.concurrent.load(Ordering::Relaxed) > 0); - } - } -} diff --git a/crates/smtp/src/inbound/auth.rs b/crates/smtp/src/inbound/auth.rs index 419b4dab..d1ab3470 100644 --- a/crates/smtp/src/inbound/auth.rs +++ b/crates/smtp/src/inbound/auth.rs @@ -168,8 +168,7 @@ impl<T: SessionStream> Session<T> { // Authenticate let mut result = self - .core - .core + .server .authenticate( directory, self.data.session_id, @@ -182,8 +181,7 @@ impl<T: SessionStream> Session<T> { // Validate permissions if let Ok(principal) = &result { match self - .core - .core + .server .get_cached_access_token(principal.id()) .await .caused_by(trc::location!()) diff --git a/crates/smtp/src/inbound/data.rs b/crates/smtp/src/inbound/data.rs index 822fc734..80477eac 100644 --- a/crates/smtp/src/inbound/data.rs +++ b/crates/smtp/src/inbound/data.rs @@ -34,7 +34,8 @@ use utils::config::Rate; use crate::{ core::{Session, SessionAddress, State}, inbound::milter::Modification, - queue::{self, Message, MessageSource, QueueEnvelope, Schedule}, + queue::{self, quota::HasQueueQuota, Message, MessageSource, QueueEnvelope, Schedule}, + reporting::analysis::AnalyzeReport, scripts::ScriptResult, }; @@ -46,7 +47,7 @@ impl<T: SessionStream> Session<T> { let raw_message = Arc::new(std::mem::take(&mut self.data.message)); let auth_message = if let Some(auth_message) = AuthenticatedMessage::parse_with_opts( &raw_message, - self.core.core.smtp.mail_auth.dkim.strict, + self.server.core.smtp.mail_auth.dkim.strict, ) { auth_message } else { @@ -59,13 +60,12 @@ impl<T: SessionStream> Session<T> { }; // Loop detection - let dc = &self.core.core.smtp.session.data; - let ac = &self.core.core.smtp.mail_auth; - let rc = &self.core.core.smtp.report; + let dc = &self.server.core.smtp.session.data; + let ac = &self.server.core.smtp.mail_auth; + let rc = &self.server.core.smtp.report; if auth_message.received_headers_count() > self - .core - .core + .server .eval_if(&dc.max_received_headers, self, self.data.session_id) .await .unwrap_or(50) @@ -82,21 +82,19 @@ impl<T: SessionStream> Session<T> { // Verify DKIM let dkim = self - .core - .core + .server .eval_if(&ac.dkim.verify, self, self.data.session_id) .await .unwrap_or(VerifyStrategy::Relaxed); let dmarc = self - .core - .core + .server .eval_if(&ac.dmarc.verify, self, self.data.session_id) .await .unwrap_or(VerifyStrategy::Relaxed); let dkim_output = if dkim.verify() || dmarc.verify() { let time = Instant::now(); let dkim_output = self - .core + .server .core .smtp .resolvers @@ -111,8 +109,7 @@ impl<T: SessionStream> Session<T> { // Send reports for failed signatures if let Some(rate) = self - .core - .core + .server .eval_if::<Rate, _>(&rc.dkim.send, self, self.data.session_id) .await { @@ -155,21 +152,19 @@ impl<T: SessionStream> Session<T> { // Verify ARC let arc = self - .core - .core + .server .eval_if(&ac.arc.verify, self, self.data.session_id) .await .unwrap_or(VerifyStrategy::Relaxed); let arc_sealer = self - .core - .core + .server .eval_if::<String, _>(&ac.arc.seal, self, self.data.session_id) .await - .and_then(|name| self.core.core.get_arc_sealer(&name, self.data.session_id)); + .and_then(|name| self.server.get_arc_sealer(&name, self.data.session_id)); let arc_output = if arc.verify() || arc_sealer.is_some() { let time = Instant::now(); let arc_output = self - .core + .server .core .smtp .resolvers @@ -236,7 +231,7 @@ impl<T: SessionStream> Session<T> { Some(spf_output) if dmarc.verify() => { let time = Instant::now(); let dmarc_output = self - .core + .server .core .smtp .resolvers @@ -317,7 +312,7 @@ impl<T: SessionStream> Session<T> { // Analyze reports if is_report { - self.core + self.server .analyze_report(raw_message.clone(), self.data.session_id); if !rc.analysis.forward { self.data.messages_sent += 1; @@ -326,11 +321,16 @@ impl<T: SessionStream> Session<T> { } // Add Received header - let message_id = self.core.inner.queue_id_gen.generate().unwrap_or_else(now); + let message_id = self + .server + .inner + .data + .queue_id_gen + .generate() + .unwrap_or_else(now); let mut headers = Vec::with_capacity(64); if self - .core - .core + .server .eval_if(&dc.add_received, self, self.data.session_id) .await .unwrap_or(true) @@ -340,8 +340,7 @@ impl<T: SessionStream> Session<T> { // Add authentication results header if self - .core - .core + .server .eval_if(&dc.add_auth_results, self, self.data.session_id) .await .unwrap_or(true) @@ -352,8 +351,7 @@ impl<T: SessionStream> Session<T> { // Add Received-SPF header if let Some(spf_output) = &self.data.spf_mail_from { if self - .core - .core + .server .eval_if(&dc.add_received_spf, self, self.data.session_id) .await .unwrap_or(true) @@ -425,23 +423,20 @@ impl<T: SessionStream> Session<T> { // Pipe message for pipe in &dc.pipe_commands { if let Some(command_) = self - .core - .core + .server .eval_if::<String, _>(&pipe.command, self, self.data.session_id) .await { let piped_message = edited_message.as_ref().unwrap_or(&raw_message).clone(); let timeout = self - .core - .core + .server .eval_if(&pipe.timeout, self, self.data.session_id) .await .unwrap_or_else(|| Duration::from_secs(30)); let mut command = Command::new(&command_); for argument in self - .core - .core + .server .eval_if::<Vec<String>, _>(&pipe.arguments, self, self.data.session_id) .await .unwrap_or_default() @@ -538,13 +533,11 @@ impl<T: SessionStream> Session<T> { // Sieve filtering if let Some((script, script_id)) = self - .core - .core + .server .eval_if::<String, _>(&dc.script, self, self.data.session_id) .await .and_then(|name| { - self.core - .core + self.server .get_trusted_sieve_script(&name, self.data.session_id) .map(|s| (s, name)) }) @@ -642,8 +635,7 @@ impl<T: SessionStream> Session<T> { // Add Return-Path if self - .core - .core + .server .eval_if(&dc.add_return_path, self, self.data.session_id) .await .unwrap_or(true) @@ -656,8 +648,7 @@ impl<T: SessionStream> Session<T> { // Add any missing headers if !auth_message.has_date_header() && self - .core - .core + .server .eval_if(&dc.add_date, self, self.data.session_id) .await .unwrap_or(true) @@ -668,8 +659,7 @@ impl<T: SessionStream> Session<T> { } if !auth_message.has_message_id_header() && self - .core - .core + .server .eval_if(&dc.add_message_id, self, self.data.session_id) .await .unwrap_or(true) @@ -684,17 +674,12 @@ impl<T: SessionStream> Session<T> { .as_deref() .unwrap_or_else(|| raw_message.as_slice()); for signer in self - .core - .core + .server .eval_if::<Vec<String>, _>(&ac.dkim.sign, self, self.data.session_id) .await .unwrap_or_default() { - if let Some(signer) = self - .core - .core - .get_dkim_signer(&signer, self.data.session_id) - { + if let Some(signer) = self.server.get_dkim_signer(&signer, self.data.session_id) { match signer.sign_chained(&[headers.as_ref(), raw_message]) { Ok(signature) => { signature.write_header(&mut headers); @@ -712,7 +697,7 @@ impl<T: SessionStream> Session<T> { message.size = raw_message.len() + headers.len(); // Verify queue quota - if self.core.has_quota(&mut message).await { + if self.server.has_quota(&mut message).await { // Prepare webhook event let queue_id = message.queue_id; @@ -727,7 +712,7 @@ impl<T: SessionStream> Session<T> { Some(&headers), raw_message, self.data.session_id, - &self.core, + &self.server, source, ) .await @@ -799,10 +784,9 @@ impl<T: SessionStream> Session<T> { }; // Set expiration and notification times - let config = &self.core.core.smtp.queue; + let config = &self.server.core.smtp.queue; let (num_intervals, next_notify) = self - .core - .core + .server .eval_if::<Vec<Duration>, _>(&config.notify, &envelope, self.data.session_id) .await .and_then(|v| (v.len(), v.into_iter().next()?).into()) @@ -813,8 +797,7 @@ impl<T: SessionStream> Session<T> { now() + future_release.as_secs() + self - .core - .core + .server .eval_if(&config.expire, &envelope, self.data.session_id) .await .unwrap_or_else(|| Duration::from_secs(5 * 86400)) @@ -827,8 +810,7 @@ impl<T: SessionStream> Session<T> { ) } else { let expire = self - .core - .core + .server .eval_if(&config.expire, &envelope, self.data.session_id) .await .unwrap_or_else(|| Duration::from_secs(5 * 86400)); @@ -887,10 +869,9 @@ impl<T: SessionStream> Session<T> { if !self.data.rcpt_to.is_empty() { if self.data.messages_sent < self - .core - .core + .server .eval_if( - &self.core.core.smtp.session.data.max_messages, + &self.server.core.smtp.session.data.max_messages, self, self.data.session_id, ) diff --git a/crates/smtp/src/inbound/ehlo.rs b/crates/smtp/src/inbound/ehlo.rs index 49580945..deeae3d1 100644 --- a/crates/smtp/src/inbound/ehlo.rs +++ b/crates/smtp/src/inbound/ehlo.rs @@ -42,7 +42,7 @@ impl<T: SessionStream> Session<T> { if self.params.spf_ehlo.verify() { let time = Instant::now(); let spf_output = self - .core + .server .core .smtp .resolvers @@ -76,17 +76,15 @@ impl<T: SessionStream> Session<T> { // Sieve filtering if let Some((script, script_id)) = self - .core - .core + .server .eval_if::<String, _>( - &self.core.core.smtp.session.ehlo.script, + &self.server.core.smtp.session.ehlo.script, self, self.data.session_id, ) .await .and_then(|name| { - self.core - .core + self.server .get_trusted_sieve_script(&name, self.data.session_id) .map(|s| (s, name)) }) @@ -140,14 +138,13 @@ impl<T: SessionStream> Session<T> { if !self.stream.is_tls() && self.instance.acceptor.is_tls() { response.capabilities |= EXT_START_TLS; } - let ec = &self.core.core.smtp.session.extensions; - let ac = &self.core.core.smtp.session.auth; - let dc = &self.core.core.smtp.session.data; + let ec = &self.server.core.smtp.session.extensions; + let ac = &self.server.core.smtp.session.auth; + let dc = &self.server.core.smtp.session.data; // Pipelining if self - .core - .core + .server .eval_if(&ec.pipelining, self, self.data.session_id) .await .unwrap_or(true) @@ -157,8 +154,7 @@ impl<T: SessionStream> Session<T> { // Chunking if self - .core - .core + .server .eval_if(&ec.chunking, self, self.data.session_id) .await .unwrap_or(true) @@ -168,8 +164,7 @@ impl<T: SessionStream> Session<T> { // Address Expansion if self - .core - .core + .server .eval_if(&ec.expn, self, self.data.session_id) .await .unwrap_or(false) @@ -179,8 +174,7 @@ impl<T: SessionStream> Session<T> { // Recipient Verification if self - .core - .core + .server .eval_if(&ec.vrfy, self, self.data.session_id) .await .unwrap_or(false) @@ -190,8 +184,7 @@ impl<T: SessionStream> Session<T> { // Require TLS if self - .core - .core + .server .eval_if(&ec.requiretls, self, self.data.session_id) .await .unwrap_or(true) @@ -201,8 +194,7 @@ impl<T: SessionStream> Session<T> { // DSN if self - .core - .core + .server .eval_if(&ec.dsn, self, self.data.session_id) .await .unwrap_or(false) @@ -213,8 +205,7 @@ impl<T: SessionStream> Session<T> { // Authentication if self.data.authenticated_as.is_empty() { response.auth_mechanisms = self - .core - .core + .server .eval_if::<Mechanism, _>(&ac.mechanisms, self, self.data.session_id) .await .unwrap_or_default() @@ -226,8 +217,7 @@ impl<T: SessionStream> Session<T> { // Future release if let Some(value) = self - .core - .core + .server .eval_if::<Duration, _>(&ec.future_release, self, self.data.session_id) .await { @@ -242,8 +232,7 @@ impl<T: SessionStream> Session<T> { // Deliver By if let Some(value) = self - .core - .core + .server .eval_if::<Duration, _>(&ec.deliver_by, self, self.data.session_id) .await { @@ -253,8 +242,7 @@ impl<T: SessionStream> Session<T> { // Priority if let Some(value) = self - .core - .core + .server .eval_if::<MtPriority, _>(&ec.mt_priority, self, self.data.session_id) .await { @@ -264,8 +252,7 @@ impl<T: SessionStream> Session<T> { // Size response.size = self - .core - .core + .server .eval_if(&dc.max_message_size, self, self.data.session_id) .await .unwrap_or(25 * 1024 * 1024); @@ -275,8 +262,7 @@ impl<T: SessionStream> Session<T> { // No soliciting if let Some(value) = self - .core - .core + .server .eval_if::<String, _>(&ec.no_soliciting, self, self.data.session_id) .await { diff --git a/crates/smtp/src/inbound/hooks/message.rs b/crates/smtp/src/inbound/hooks/message.rs index 6ad3b44f..37720509 100644 --- a/crates/smtp/src/inbound/hooks/message.rs +++ b/crates/smtp/src/inbound/hooks/message.rs @@ -36,7 +36,7 @@ impl<T: SessionStream> Session<T> { message: Option<&AuthenticatedMessage<'_>>, queue_id: Option<QueueId>, ) -> Result<Vec<Modification>, FilterResponse> { - let mta_hooks = &self.core.core.smtp.session.hooks; + let mta_hooks = &self.server.core.smtp.session.hooks; if mta_hooks.is_empty() { return Ok(Vec::new()); } @@ -45,8 +45,7 @@ impl<T: SessionStream> Session<T> { for mta_hook in mta_hooks { if !mta_hook.run_on_stage.contains(&stage) || !self - .core - .core + .server .eval_if(&mta_hook.enable, self, self.data.session_id) .await .unwrap_or(false) diff --git a/crates/smtp/src/inbound/mail.rs b/crates/smtp/src/inbound/mail.rs index 45428f9e..033b7d3d 100644 --- a/crates/smtp/src/inbound/mail.rs +++ b/crates/smtp/src/inbound/mail.rs @@ -54,7 +54,7 @@ impl<T: SessionStream> Session<T> { } else if self.data.iprev.is_none() && self.params.iprev.verify() { let time = Instant::now(); let iprev = self - .core + .server .core .smtp .resolvers @@ -122,10 +122,9 @@ impl<T: SessionStream> Session<T> { // Check whether the address is allowed if !self - .core - .core + .server .eval_if::<bool, _>( - &self.core.core.smtp.session.mail.is_allowed, + &self.server.core.smtp.session.mail.is_allowed, self, self.data.session_id, ) @@ -145,17 +144,15 @@ impl<T: SessionStream> Session<T> { // Sieve filtering if let Some((script, script_id)) = self - .core - .core + .server .eval_if::<String, _>( - &self.core.core.smtp.session.mail.script, + &self.server.core.smtp.session.mail.script, self, self.data.session_id, ) .await .and_then(|name| { - self.core - .core + self.server .get_trusted_sieve_script(&name, self.data.session_id) .map(|s| (s, name)) }) @@ -199,10 +196,9 @@ impl<T: SessionStream> Session<T> { // Address rewriting if let Some(new_address) = self - .core - .core + .server .eval_if::<String, _>( - &self.core.core.smtp.session.mail.rewrite, + &self.server.core.smtp.session.mail.rewrite, self, self.data.session_id, ) @@ -258,12 +254,11 @@ impl<T: SessionStream> Session<T> { } // Validate parameters - let config = &self.core.core.smtp.session.extensions; - let config_data = &self.core.core.smtp.session.data; + let config = &self.server.core.smtp.session.extensions; + let config_data = &self.server.core.smtp.session.data; if (from.flags & MAIL_REQUIRETLS) != 0 && !self - .core - .core + .server .eval_if(&config.requiretls, self, self.data.session_id) .await .unwrap_or(false) @@ -279,8 +274,7 @@ impl<T: SessionStream> Session<T> { } if (from.flags & (MAIL_BY_NOTIFY | MAIL_BY_RETURN)) != 0 { if let Some(duration) = self - .core - .core + .server .eval_if::<Duration, _>(&config.deliver_by, self, self.data.session_id) .await { @@ -320,8 +314,7 @@ impl<T: SessionStream> Session<T> { } if from.mt_priority != 0 { if self - .core - .core + .server .eval_if::<MtPriority, _>(&config.mt_priority, self, self.data.session_id) .await .is_some() @@ -351,8 +344,7 @@ impl<T: SessionStream> Session<T> { if from.size > 0 && from.size > self - .core - .core + .server .eval_if(&config_data.max_message_size, self, self.data.session_id) .await .unwrap_or(25 * 1024 * 1024) @@ -370,8 +362,7 @@ impl<T: SessionStream> Session<T> { } if from.hold_for != 0 || from.hold_until != 0 { if let Some(max_hold) = self - .core - .core + .server .eval_if::<Duration, _>(&config.future_release, self, self.data.session_id) .await { @@ -419,8 +410,7 @@ impl<T: SessionStream> Session<T> { } if has_dsn && !self - .core - .core + .server .eval_if(&config.dsn, self, self.data.session_id) .await .unwrap_or(false) @@ -438,7 +428,7 @@ impl<T: SessionStream> Session<T> { let time = Instant::now(); let mail_from = self.data.mail_from.as_ref().unwrap(); let spf_output = if !mail_from.address.is_empty() { - self.core + self.server .core .smtp .resolvers @@ -452,7 +442,7 @@ impl<T: SessionStream> Session<T> { ) .await } else { - self.core + self.server .core .smtp .resolvers @@ -542,10 +532,9 @@ impl<T: SessionStream> Session<T> { // Send report if let (Some(recipient), Some(rate)) = ( spf_output.report_address(), - self.core - .core + self.server .eval_if::<Rate, _>( - &self.core.core.smtp.report.spf.send, + &self.server.core.smtp.report.spf.send, self, self.data.session_id, ) diff --git a/crates/smtp/src/inbound/milter/message.rs b/crates/smtp/src/inbound/milter/message.rs index 1bcbde1b..0b369161 100644 --- a/crates/smtp/src/inbound/milter/message.rs +++ b/crates/smtp/src/inbound/milter/message.rs @@ -35,7 +35,7 @@ impl<T: SessionStream> Session<T> { stage: Stage, message: Option<&AuthenticatedMessage<'_>>, ) -> Result<Vec<Modification>, FilterResponse> { - let milters = &self.core.core.smtp.session.milters; + let milters = &self.server.core.smtp.session.milters; if milters.is_empty() { return Ok(Vec::new()); } @@ -44,8 +44,7 @@ impl<T: SessionStream> Session<T> { for milter in milters { if !milter.run_on_stage.contains(&stage) || !self - .core - .core + .server .eval_if(&milter.enable, self, self.data.session_id) .await .unwrap_or(false) @@ -170,9 +169,9 @@ impl<T: SessionStream> Session<T> { client .into_tls( if !milter.tls_allow_invalid_certs { - &self.core.inner.connectors.pki_verify + &self.server.inner.data.smtp_connectors.pki_verify } else { - &self.core.inner.connectors.dummy_verify + &self.server.inner.data.smtp_connectors.dummy_verify }, &milter.hostname, ) diff --git a/crates/smtp/src/inbound/rcpt.rs b/crates/smtp/src/inbound/rcpt.rs index 89c7b8dc..aab0be1e 100644 --- a/crates/smtp/src/inbound/rcpt.rs +++ b/crates/smtp/src/inbound/rcpt.rs @@ -77,25 +77,23 @@ impl<T: SessionStream> Session<T> { // Address rewriting and Sieve filtering let rcpt_script = self - .core - .core + .server .eval_if::<String, _>( - &self.core.core.smtp.session.rcpt.script, + &self.server.core.smtp.session.rcpt.script, self, self.data.session_id, ) .await .and_then(|name| { - self.core - .core + self.server .get_trusted_sieve_script(&name, self.data.session_id) .map(|s| (s.clone(), name)) }); if rcpt_script.is_some() - || !self.core.core.smtp.session.rcpt.rewrite.is_empty() + || !self.server.core.smtp.session.rcpt.rewrite.is_empty() || self - .core + .server .core .smtp .session @@ -146,10 +144,9 @@ impl<T: SessionStream> Session<T> { // Address rewriting if let Some(new_address) = self - .core - .core + .server .eval_if::<String, _>( - &self.core.core.smtp.session.rcpt.rewrite, + &self.server.core.smtp.session.rcpt.rewrite, self, self.data.session_id, ) @@ -187,22 +184,20 @@ impl<T: SessionStream> Session<T> { // Verify address let rcpt = self.data.rcpt_to.last().unwrap(); if let Some(directory) = self - .core - .core + .server .eval_if::<String, _>( - &self.core.core.smtp.session.rcpt.directory, + &self.server.core.smtp.session.rcpt.directory, self, self.data.session_id, ) .await - .and_then(|name| self.core.core.get_directory(&name)) + .and_then(|name| self.server.get_directory(&name)) { match directory.is_local_domain(&rcpt.domain).await { Ok(is_local_domain) => { if is_local_domain { match self - .core - .core + .server .rcpt(directory, &rcpt.address_lcase, self.data.session_id) .await { @@ -233,10 +228,9 @@ impl<T: SessionStream> Session<T> { } } } else if !self - .core - .core + .server .eval_if( - &self.core.core.smtp.session.rcpt.relay, + &self.server.core.smtp.session.rcpt.relay, self, self.data.session_id, ) @@ -266,10 +260,9 @@ impl<T: SessionStream> Session<T> { } } } else if !self - .core - .core + .server .eval_if( - &self.core.core.smtp.session.rcpt.relay, + &self.server.core.smtp.session.rcpt.relay, self, self.data.session_id, ) @@ -315,12 +308,7 @@ impl<T: SessionStream> Session<T> { if self.data.rcpt_errors < self.params.rcpt_errors_max { Ok(()) } else { - match self - .core - .core - .is_rcpt_fail2banned(self.data.remote_ip) - .await - { + match self.server.is_rcpt_fail2banned(self.data.remote_ip).await { Ok(true) => { trc::event!( Security(SecurityEvent::BruteForceBan), diff --git a/crates/smtp/src/inbound/session.rs b/crates/smtp/src/inbound/session.rs index b0c91672..02648b5f 100644 --- a/crates/smtp/src/inbound/session.rs +++ b/crates/smtp/src/inbound/session.rs @@ -84,10 +84,9 @@ impl<T: SessionStream> Session<T> { initial_response, } => { let auth: u64 = self - .core - .core + .server .eval_if::<Mechanism, _>( - &self.core.core.smtp.session.auth.mechanisms, + &self.server.core.smtp.session.auth.mechanisms, self, self.data.session_id, ) diff --git a/crates/smtp/src/inbound/spawn.rs b/crates/smtp/src/inbound/spawn.rs index e6892fc1..d7e08f8b 100644 --- a/crates/smtp/src/inbound/spawn.rs +++ b/crates/smtp/src/inbound/spawn.rs @@ -8,6 +8,7 @@ use std::time::Instant; use common::{ config::smtp::session::Stage, + core::BuildServer, listener::{self, SessionManager, SessionStream}, }; use tokio_rustls::server::TlsStream; @@ -15,7 +16,6 @@ use trc::{SecurityEvent, SmtpEvent}; use crate::{ core::{Session, SessionData, SessionParameters, SmtpSessionManager, State}, - queue, reporting, scripts::ScriptResult, }; @@ -27,7 +27,7 @@ impl SessionManager for SmtpSessionManager { // Create session let mut session = Session { hostname: String::new(), - core: self.inner.into(), + server: self.inner.build_server(), instance: session.instance, state: State::default(), stream: session.stream, @@ -59,19 +59,23 @@ impl SessionManager for SmtpSessionManager { #[allow(clippy::manual_async_fn)] fn shutdown(&self) -> impl std::future::Future<Output = ()> + Send { async { - let _ = self.inner.inner.queue_tx.send(queue::Event::Stop).await; let _ = self .inner + .ipc + .queue_tx + .send(common::ipc::QueueEvent::Stop) + .await; + let _ = self .inner + .ipc .report_tx - .send(reporting::Event::Stop) + .send(common::ipc::ReportingEvent::Stop) .await; let _ = self .inner - .inner .ipc .delivery_tx - .send(common::DeliveryEvent::Stop) + .send(common::ipc::DeliveryEvent::Stop) .await; } } @@ -81,17 +85,15 @@ impl<T: SessionStream> Session<T> { pub async fn init_conn(&mut self) -> bool { self.eval_session_params().await; - let config = &self.core.core.smtp.session.connect; + let config = &self.server.core.smtp.session.connect; // Sieve filtering if let Some((script, script_id)) = self - .core - .core + .server .eval_if::<String, _>(&config.script, self, self.data.session_id) .await .and_then(|name| { - self.core - .core + self.server .get_trusted_sieve_script(&name, self.data.session_id) .map(|s| (s, name)) }) @@ -123,8 +125,7 @@ impl<T: SessionStream> Session<T> { // Obtain hostname self.hostname = self - .core - .core + .server .eval_if::<String, _>(&config.hostname, self, self.data.session_id) .await .unwrap_or_default(); @@ -138,8 +139,7 @@ impl<T: SessionStream> Session<T> { // Obtain greeting let greeting = self - .core - .core + .server .eval_if::<String, _>(&config.greeting, self, self.data.session_id) .await .filter(|g| !g.is_empty()) @@ -194,10 +194,7 @@ impl<T: SessionStream> Session<T> { .await .ok(); - match self - .core - .core - .is_loiter_fail2banned(self.data.remote_ip) + match self.server.is_loiter_fail2banned(self.data.remote_ip) .await { Ok(true) => { @@ -277,7 +274,7 @@ impl<T: SessionStream> Session<T> { state: self.state, data: self.data, instance: self.instance, - core: self.core, + server: self.server, in_flight: self.in_flight, params: self.params, }) diff --git a/crates/smtp/src/inbound/vrfy.rs b/crates/smtp/src/inbound/vrfy.rs index 0c23687d..a99823d0 100644 --- a/crates/smtp/src/inbound/vrfy.rs +++ b/crates/smtp/src/inbound/vrfy.rs @@ -13,20 +13,18 @@ use std::fmt::Write; impl<T: SessionStream> Session<T> { pub async fn handle_vrfy(&mut self, address: String) -> Result<(), ()> { match self - .core - .core + .server .eval_if::<String, _>( - &self.core.core.smtp.session.rcpt.directory, + &self.server.core.smtp.session.rcpt.directory, self, self.data.session_id, ) .await - .and_then(|name| self.core.core.get_directory(&name)) + .and_then(|name| self.server.get_directory(&name)) { Some(directory) if self.params.can_vrfy => { match self - .core - .core + .server .vrfy(directory, &address.to_lowercase(), self.data.session_id) .await { @@ -88,20 +86,18 @@ impl<T: SessionStream> Session<T> { pub async fn handle_expn(&mut self, address: String) -> Result<(), ()> { match self - .core - .core + .server .eval_if::<String, _>( - &self.core.core.smtp.session.rcpt.directory, + &self.server.core.smtp.session.rcpt.directory, self, self.data.session_id, ) .await - .and_then(|name| self.core.core.get_directory(&name)) + .and_then(|name| self.server.get_directory(&name)) { Some(directory) if self.params.can_expn => { match self - .core - .core + .server .expn(directory, &address.to_lowercase(), self.data.session_id) .await { diff --git a/crates/smtp/src/lib.rs b/crates/smtp/src/lib.rs index eaef7d5d..0713f9bb 100644 --- a/crates/smtp/src/lib.rs +++ b/crates/smtp/src/lib.rs @@ -4,17 +4,14 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use crate::core::{throttle::ThrottleKeyHasherBuilder, TlsConnectors}; -use core::{Inner, SmtpInstance, SMTP}; use std::sync::Arc; -use common::{config::scripts::ScriptCache, Ipc, SharedCore}; -use dashmap::DashMap; -use mail_send::smtp::tls::build_tls_connector; +use common::{ + manager::boot::{BootManager, IpcReceivers}, + Inner, +}; use queue::manager::SpawnQueue; use reporting::scheduler::SpawnReport; -use tokio::sync::mpsc; -use utils::{config::Config, snowflake::SnowflakeIdGenerator}; pub mod core; pub mod inbound; @@ -23,54 +20,26 @@ pub mod queue; pub mod reporting; pub mod scripts; -impl SMTP { - pub async fn init( - config: &mut Config, - core: SharedCore, - ipc: Ipc, - span_id_gen: Arc<SnowflakeIdGenerator>, - ) -> SmtpInstance { - // Build inner - let capacity = config.property("cache.capacity").unwrap_or(2); - let shard = config - .property::<u64>("cache.shard") - .unwrap_or(32) - .next_power_of_two() as usize; - let (queue_tx, queue_rx) = mpsc::channel(1024); - let (report_tx, report_rx) = mpsc::channel(1024); - let inner = Inner { - session_throttle: DashMap::with_capacity_and_hasher_and_shard_amount( - capacity, - ThrottleKeyHasherBuilder::default(), - shard, - ), - queue_throttle: DashMap::with_capacity_and_hasher_and_shard_amount( - capacity, - ThrottleKeyHasherBuilder::default(), - shard, - ), - queue_tx, - report_tx, - queue_id_gen: config - .property::<u64>("cluster.node-id") - .map(SnowflakeIdGenerator::with_node_id) - .unwrap_or_default(), - span_id_gen, - connectors: TlsConnectors { - pki_verify: build_tls_connector(false), - dummy_verify: build_tls_connector(true), - }, - ipc, - script_cache: ScriptCache::parse(config), - }; - let inner = SmtpInstance::new(core, inner); +pub trait StartQueueManager { + fn start_queue_manager(&mut self); +} + +pub trait SpawnQueueManager { + fn spawn_queue_manager(&mut self, inner: Arc<Inner>); +} +impl StartQueueManager for BootManager { + fn start_queue_manager(&mut self) { + self.ipc_rxs.spawn_queue_manager(self.inner.clone()); + } +} + +impl SpawnQueueManager for IpcReceivers { + fn spawn_queue_manager(&mut self, inner: Arc<Inner>) { // Spawn queue manager - queue_rx.spawn(inner.clone()); + self.queue_rx.take().unwrap().spawn(inner.clone()); // Spawn report manager - report_rx.spawn(inner.clone()); - - inner + self.report_rx.take().unwrap().spawn(inner); } } diff --git a/crates/smtp/src/outbound/client.rs b/crates/smtp/src/outbound/client.rs index f064781a..5391cc52 100644 --- a/crates/smtp/src/outbound/client.rs +++ b/crates/smtp/src/outbound/client.rs @@ -177,10 +177,8 @@ impl<T: AsyncRead + AsyncWrite + Unpin> SmtpClient<T> { params: &SessionParams<'_>, ) -> Result<(), Status<(), Error>> { match params - .core - .core - .storage - .blob + .server + .blob_store() .get_blob(message.blob_hash.as_slice(), 0..usize::MAX) .await { diff --git a/crates/smtp/src/outbound/dane/dnssec.rs b/crates/smtp/src/outbound/dane/dnssec.rs index 1c2f11e3..de89c1f3 100644 --- a/crates/smtp/src/outbound/dane/dnssec.rs +++ b/crates/smtp/src/outbound/dane/dnssec.rs @@ -4,7 +4,10 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use common::config::smtp::resolver::{Tlsa, TlsaEntry}; +use common::{ + config::smtp::resolver::{Tlsa, TlsaEntry}, + Server, +}; use mail_auth::{ common::{lru::DnsCache, resolver::IntoFqdn}, hickory_resolver::{ @@ -16,14 +19,27 @@ use mail_auth::{ Name, }, }; -use std::sync::Arc; +use std::{future::Future, sync::Arc}; -use crate::core::SMTP; +pub trait TlsaLookup: Sync + Send { + fn tlsa_lookup<'x>( + &self, + key: impl IntoFqdn<'x> + Sync + Send, + ) -> impl Future<Output = mail_auth::Result<Option<Arc<Tlsa>>>> + Send; -impl SMTP { - pub async fn tlsa_lookup<'x>( + #[cfg(feature = "test_mode")] + fn tlsa_add<'x>( &self, key: impl IntoFqdn<'x>, + value: impl Into<Arc<Tlsa>>, + valid_until: std::time::Instant, + ); +} + +impl TlsaLookup for Server { + async fn tlsa_lookup<'x>( + &self, + key: impl IntoFqdn<'x> + Sync + Send, ) -> mail_auth::Result<Option<Arc<Tlsa>>> { let key = key.into_fqdn(); if let Some(value) = self.core.smtp.resolvers.cache.tlsa.get(key.as_ref()) { @@ -102,7 +118,7 @@ impl SMTP { } #[cfg(feature = "test_mode")] - pub fn tlsa_add<'x>( + fn tlsa_add<'x>( &self, key: impl IntoFqdn<'x>, value: impl Into<Arc<Tlsa>>, diff --git a/crates/smtp/src/outbound/delivery.rs b/crates/smtp/src/outbound/delivery.rs index 8fe5c968..2a8694f2 100644 --- a/crates/smtp/src/outbound/delivery.rs +++ b/crates/smtp/src/outbound/delivery.rs @@ -5,12 +5,21 @@ */ use crate::outbound::client::{from_error_status, from_mail_send_error, SmtpClient}; +use crate::outbound::dane::dnssec::TlsaLookup; +use crate::outbound::lookup::DnsLookup; +use crate::outbound::mta_sts::lookup::MtaStsLookup; use crate::outbound::mta_sts::verify::VerifyPolicy; use crate::outbound::{client::StartTlsResult, dane::verify::TlsaVerify}; +use crate::queue::dsn::SendDsn; +use crate::queue::spool::SmtpSpool; +use crate::queue::throttle::IsAllowed; +use crate::reporting::SmtpReporting; use common::config::{ server::ServerProtocol, smtp::{queue::RequireOptional, report::AggregateFrequency}, }; +use common::ipc::{OnHold, PolicyType, QueueEvent, TlsEvent}; +use common::Server; use mail_auth::{ mta_sts::TlsRpt, report::tlsrpt::{FailureDetails, ResultType}, @@ -20,31 +29,28 @@ use std::{ net::{IpAddr, Ipv4Addr, SocketAddr}, time::{Duration, Instant}, }; -use store::write::{now, BatchBuilder, QueueClass, QueueEvent, ValueClass}; +use store::write::{now, BatchBuilder, QueueClass, ValueClass}; use trc::{DaneEvent, DeliveryEvent, MtaStsEvent, ServerEvent, TlsRptEvent}; use crate::{ - core::SMTP, queue::{ErrorDetails, Message}, - reporting::{tls::TlsRptOptions, PolicyType, TlsEvent}, + reporting::tls::TlsRptOptions, }; use super::{lookup::ToNextHop, mta_sts, session::SessionParams, NextHop, TlsStrategy}; -use crate::queue::{ - throttle, DeliveryAttempt, Domain, Error, Event, OnHold, QueueEnvelope, Status, -}; +use crate::queue::{throttle, DeliveryAttempt, Domain, Error, QueueEnvelope, Status}; impl DeliveryAttempt { - pub async fn try_deliver(mut self, core: SMTP) { + pub async fn try_deliver(mut self, server: Server) { tokio::spawn(async move { // Lock message - if let Some(event) = core.try_lock_event(self.event).await { + if let Some(event) = server.try_lock_event(self.event).await { self.event = event; // Fetch message - if let Some(mut message) = core.read_message(self.event.queue_id).await { + if let Some(mut message) = server.read_message(self.event.queue_id).await { // Generate span id - message.span_id = core.inner.span_id_gen.generate().unwrap_or_else(now); + message.span_id = server.inner.data.span_id_gen.generate().unwrap_or_else(now); let span_id = message.span_id; trc::event!( @@ -76,7 +82,7 @@ impl DeliveryAttempt { // Attempt delivery let start_time = Instant::now(); - self.deliver_task(core, message).await; + self.deliver_task(server, message).await; trc::event!( Delivery(DeliveryEvent::AttemptEnd), @@ -86,12 +92,14 @@ impl DeliveryAttempt { } else { // Message no longer exists, delete queue event. let mut batch = BatchBuilder::new(); - batch.clear(ValueClass::Queue(QueueClass::MessageEvent(QueueEvent { - due: self.event.due, - queue_id: self.event.queue_id, - }))); + batch.clear(ValueClass::Queue(QueueClass::MessageEvent( + store::write::QueueEvent { + due: self.event.due, + queue_id: self.event.queue_id, + }, + ))); - if let Err(err) = core.core.storage.data.write(batch.build()).await { + if let Err(err) = server.store().write(batch.build()).await { trc::error!(err .details("Failed to delete queue event.") .caused_by(trc::location!())); @@ -101,13 +109,13 @@ impl DeliveryAttempt { }); } - async fn deliver_task(mut self, core: SMTP, mut message: Message) { + async fn deliver_task(mut self, server: Server, mut message: Message) { // Check that the message still has recipients to be delivered let has_pending_delivery = message.has_pending_delivery(); let span_id = message.span_id; // Send any due Delivery Status Notifications - core.send_dsn(&mut message).await; + server.send_dsn(&mut message).await; if has_pending_delivery { // Re-queue the message if its not yet due for delivery @@ -115,9 +123,16 @@ impl DeliveryAttempt { if due > now() { // Save changes message - .save_changes(&core, self.event.due.into(), due.into()) + .save_changes(&server, self.event.due.into(), due.into()) .await; - if core.inner.queue_tx.send(Event::Reload).await.is_err() { + if server + .inner + .ipc + .queue_tx + .send(QueueEvent::Reload) + .await + .is_err() + { trc::event!( Server(ServerEvent::ThreadError), Reason = "Channel closed.", @@ -135,8 +150,15 @@ impl DeliveryAttempt { ); // All message recipients expired, do not re-queue. (DSN has been already sent) - message.remove(&core, self.event.due).await; - if core.inner.queue_tx.send(Event::Reload).await.is_err() { + message.remove(&server, self.event.due).await; + if server + .inner + .ipc + .queue_tx + .send(QueueEvent::Reload) + .await + .is_err() + { trc::event!( Server(ServerEvent::ThreadError), Reason = "Channel closed.", @@ -149,8 +171,8 @@ impl DeliveryAttempt { } // Throttle sender - for throttle in &core.core.smtp.queue.throttle.sender { - if let Err(err) = core + for throttle in &server.core.smtp.queue.throttle.sender { + if let Err(err) = server .is_allowed(throttle, &message, &mut self.in_flight, message.span_id) .await { @@ -158,7 +180,7 @@ impl DeliveryAttempt { throttle::Error::Concurrency { limiter } => { // Save changes to disk let next_due = message.next_event_after(now()); - message.save_changes(&core, None, None).await; + message.save_changes(&server, None, None).await; trc::event!( Delivery(DeliveryEvent::ConcurrencyLimitExceeded), @@ -166,7 +188,7 @@ impl DeliveryAttempt { SpanId = span_id, ); - Event::OnHold(OnHold { + QueueEvent::OnHold(OnHold { next_due, limiters: vec![limiter], message: self.event, @@ -187,14 +209,14 @@ impl DeliveryAttempt { ); message - .save_changes(&core, self.event.due.into(), next_event.into()) + .save_changes(&server, self.event.due.into(), next_event.into()) .await; - Event::Reload + QueueEvent::Reload } }; - if core.inner.queue_tx.send(event).await.is_err() { + if server.inner.ipc.queue_tx.send(event).await.is_err() { trc::event!( Server(ServerEvent::ThreadError), Reason = "Channel closed.", @@ -206,7 +228,7 @@ impl DeliveryAttempt { } } - let queue_config = &core.core.smtp.queue; + let queue_config = &server.core.smtp.queue; let mut on_hold = Vec::new(); let no_ip = IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0)); let mut recipients = std::mem::take(&mut message.recipients); @@ -232,7 +254,7 @@ impl DeliveryAttempt { // Throttle recipient domain let mut in_flight = Vec::new(); for throttle in &queue_config.throttle.rcpt { - if let Err(err) = core + if let Err(err) = server .is_allowed(throttle, &envelope, &mut in_flight, message.span_id) .await { @@ -249,24 +271,22 @@ impl DeliveryAttempt { } // Obtain next hop - let (mut remote_hosts, is_smtp) = match core - .core + let (mut remote_hosts, is_smtp) = match server .eval_if::<String, _>(&queue_config.next_hop, &envelope, message.span_id) .await - .and_then(|name| core.core.get_relay_host(&name, message.span_id)) + .and_then(|name| server.get_relay_host(&name, message.span_id)) { Some(next_hop) if next_hop.protocol == ServerProtocol::Http => { // Deliver message locally let delivery_result = message .deliver_local( recipients.iter_mut().filter(|r| r.domain_idx == domain_idx), - &core.inner.ipc.delivery_tx, + &server.inner.ipc.delivery_tx, ) .await; // Update status for the current domain and continue with the next one - let schedule = core - .core + let schedule = server .eval_if::<Vec<Duration>, _>( &queue_config.retry, &envelope, @@ -286,23 +306,24 @@ impl DeliveryAttempt { // Prepare TLS strategy let mut tls_strategy = TlsStrategy { - mta_sts: core - .core + mta_sts: server .eval_if(&queue_config.tls.mta_sts, &envelope, message.span_id) .await .unwrap_or(RequireOptional::Optional), ..Default::default() }; - let allow_invalid_certs = core - .core + let allow_invalid_certs = server .eval_if(&queue_config.tls.invalid_certs, &envelope, message.span_id) .await .unwrap_or(false); // Obtain TLS reporting - let tls_report = match core - .core - .eval_if(&core.core.smtp.report.tls.send, &envelope, message.span_id) + let tls_report = match server + .eval_if( + &server.core.smtp.report.tls.send, + &envelope, + message.span_id, + ) .await .unwrap_or(AggregateFrequency::Never) { @@ -312,7 +333,7 @@ impl DeliveryAttempt { if is_smtp => { let time = Instant::now(); - match core + match server .core .smtp .resolvers @@ -357,10 +378,10 @@ impl DeliveryAttempt { // Obtain MTA-STS policy for domain let mta_sts_policy = if tls_strategy.try_mta_sts() && is_smtp { let time = Instant::now(); - match core + match server .lookup_mta_sts_policy( &domain.domain, - core.core + server .eval_if(&queue_config.timeout.mta_sts, &envelope, message.span_id) .await .unwrap_or_else(|| Duration::from_secs(10 * 60)), @@ -390,7 +411,7 @@ impl DeliveryAttempt { match &err { mta_sts::Error::Dns(mail_auth::Error::DnsRecordNotFound(_)) => { if strict { - core.schedule_report(TlsEvent { + server.schedule_report(TlsEvent { policy: PolicyType::Sts(None), domain: domain.domain.to_string(), failure: FailureDetails::new(ResultType::Other) @@ -406,16 +427,17 @@ impl DeliveryAttempt { } mta_sts::Error::Dns(mail_auth::Error::DnsError(_)) => (), _ => { - core.schedule_report(TlsEvent { - policy: PolicyType::Sts(None), - domain: domain.domain.to_string(), - failure: FailureDetails::new(&err) - .with_failure_reason_code(err.to_string()) - .into(), - tls_record: tls_report.record.clone(), - interval: tls_report.interval, - }) - .await; + server + .schedule_report(TlsEvent { + policy: PolicyType::Sts(None), + domain: domain.domain.to_string(), + failure: FailureDetails::new(&err) + .with_failure_reason_code(err.to_string()) + .into(), + tls_record: tls_report.record.clone(), + interval: tls_report.interval, + }) + .await; } } } @@ -463,8 +485,7 @@ impl DeliveryAttempt { } if strict { - let schedule = core - .core + let schedule = server .eval_if::<Vec<Duration>, _>( &queue_config.retry, &envelope, @@ -488,7 +509,14 @@ impl DeliveryAttempt { if is_smtp && remote_hosts.is_empty() { // Lookup MX let time = Instant::now(); - mx_list = match core.core.smtp.resolvers.dns.mx_lookup(&domain.domain).await { + mx_list = match server + .core + .smtp + .resolvers + .dns + .mx_lookup(&domain.domain) + .await + { Ok(mx) => mx, Err(err) => { trc::event!( @@ -499,8 +527,7 @@ impl DeliveryAttempt { Elapsed = time.elapsed(), ); - let schedule = core - .core + let schedule = server .eval_if::<Vec<Duration>, _>( &queue_config.retry, &envelope, @@ -515,7 +542,7 @@ impl DeliveryAttempt { if let Some(remote_hosts_) = mx_list.to_remote_hosts( &domain.domain, - core.core + server .eval_if(&queue_config.max_mx, &envelope, message.span_id) .await .unwrap_or(5), @@ -539,8 +566,7 @@ impl DeliveryAttempt { Elapsed = time.elapsed(), ); - let schedule = core - .core + let schedule = server .eval_if::<Vec<Duration>, _>( &queue_config.retry, &envelope, @@ -559,8 +585,7 @@ impl DeliveryAttempt { } // Try delivering message - let max_multihomed = core - .core + let max_multihomed = server .eval_if(&queue_config.max_multihomed, &envelope, message.span_id) .await .unwrap_or(2); @@ -573,17 +598,18 @@ impl DeliveryAttempt { if !mta_sts_policy.verify(envelope.mx) { // Report MTA-STS failed verification if let Some(tls_report) = &tls_report { - core.schedule_report(TlsEvent { - policy: mta_sts_policy.into(), - domain: domain.domain.to_string(), - failure: FailureDetails::new(ResultType::ValidationFailure) - .with_receiving_mx_hostname(envelope.mx) - .with_failure_reason_code("MX not authorized by policy.") - .into(), - tls_record: tls_report.record.clone(), - interval: tls_report.interval, - }) - .await; + server + .schedule_report(TlsEvent { + policy: mta_sts_policy.into(), + domain: domain.domain.to_string(), + failure: FailureDetails::new(ResultType::ValidationFailure) + .with_receiving_mx_hostname(envelope.mx) + .with_failure_reason_code("MX not authorized by policy.") + .into(), + tls_record: tls_report.record.clone(), + interval: tls_report.interval, + }) + .await; } trc::event!( @@ -624,7 +650,7 @@ impl DeliveryAttempt { // Obtain source and remote IPs let time = Instant::now(); - let resolve_result = match core + let resolve_result = match server .resolve_host(remote_host, &envelope, max_multihomed, message.span_id) .await { @@ -661,13 +687,11 @@ impl DeliveryAttempt { }; // Update TLS strategy - tls_strategy.dane = core - .core + tls_strategy.dane = server .eval_if(&queue_config.tls.dane, &envelope, message.span_id) .await .unwrap_or(RequireOptional::Optional); - tls_strategy.tls = core - .core + tls_strategy.tls = server .eval_if(&queue_config.tls.start, &envelope, message.span_id) .await .unwrap_or(RequireOptional::Optional); @@ -676,7 +700,10 @@ impl DeliveryAttempt { let dane_policy = if tls_strategy.try_dane() && is_smtp { let time = Instant::now(); let strict = tls_strategy.is_dane_required(); - match core.tlsa_lookup(format!("_25._tcp.{}.", envelope.mx)).await { + match server + .tlsa_lookup(format!("_25._tcp.{}.", envelope.mx)) + .await + { Ok(Some(tlsa)) => { if tlsa.has_end_entities { trc::event!( @@ -703,17 +730,18 @@ impl DeliveryAttempt { // Report invalid TLSA record if let Some(tls_report) = &tls_report { - core.schedule_report(TlsEvent { - policy: tlsa.into(), - domain: domain.domain.to_string(), - failure: FailureDetails::new(ResultType::TlsaInvalid) - .with_receiving_mx_hostname(envelope.mx) - .with_failure_reason_code("Invalid TLSA record.") - .into(), - tls_record: tls_report.record.clone(), - interval: tls_report.interval, - }) - .await; + server + .schedule_report(TlsEvent { + policy: tlsa.into(), + domain: domain.domain.to_string(), + failure: FailureDetails::new(ResultType::TlsaInvalid) + .with_receiving_mx_hostname(envelope.mx) + .with_failure_reason_code("Invalid TLSA record.") + .into(), + tls_record: tls_report.record.clone(), + interval: tls_report.interval, + }) + .await; } if strict { @@ -740,19 +768,20 @@ impl DeliveryAttempt { if strict { // Report DANE required if let Some(tls_report) = &tls_report { - core.schedule_report(TlsEvent { - policy: PolicyType::Tlsa(None), - domain: domain.domain.to_string(), - failure: FailureDetails::new(ResultType::DaneRequired) - .with_receiving_mx_hostname(envelope.mx) - .with_failure_reason_code( - "No TLSA DNSSEC records found.", - ) - .into(), - tls_record: tls_report.record.clone(), - interval: tls_report.interval, - }) - .await; + server + .schedule_report(TlsEvent { + policy: PolicyType::Tlsa(None), + domain: domain.domain.to_string(), + failure: FailureDetails::new(ResultType::DaneRequired) + .with_receiving_mx_hostname(envelope.mx) + .with_failure_reason_code( + "No TLSA DNSSEC records found.", + ) + .into(), + tls_record: tls_report.record.clone(), + interval: tls_report.interval, + }) + .await; } last_status = @@ -792,19 +821,22 @@ impl DeliveryAttempt { last_status = if not_found { // Report DANE required if let Some(tls_report) = &tls_report { - core.schedule_report(TlsEvent { - policy: PolicyType::Tlsa(None), - domain: domain.domain.to_string(), - failure: FailureDetails::new(ResultType::DaneRequired) + server + .schedule_report(TlsEvent { + policy: PolicyType::Tlsa(None), + domain: domain.domain.to_string(), + failure: FailureDetails::new( + ResultType::DaneRequired, + ) .with_receiving_mx_hostname(envelope.mx) .with_failure_reason_code( "No TLSA records found for MX.", ) .into(), - tls_record: tls_report.record.clone(), - interval: tls_report.interval, - }) - .await; + tls_record: tls_report.record.clone(), + interval: tls_report.interval, + }) + .await; } Status::PermanentFailure(Error::DaneError(ErrorDetails { @@ -837,7 +869,7 @@ impl DeliveryAttempt { let mut in_flight_host = Vec::new(); envelope.remote_ip = remote_ip; for throttle in &queue_config.throttle.host { - if let Err(err) = core + if let Err(err) = server .is_allowed(throttle, &envelope, &mut in_flight_host, message.span_id) .await { @@ -854,8 +886,7 @@ impl DeliveryAttempt { // Connect let time = Instant::now(); - let conn_timeout = core - .core + let conn_timeout = server .eval_if(&queue_config.timeout.connect, &envelope, message.span_id) .await .unwrap_or_else(|| Duration::from_secs(5 * 60)); @@ -908,8 +939,7 @@ impl DeliveryAttempt { }; // Obtain session parameters - let local_hostname = core - .core + let local_hostname = server .eval_if::<String, _>(&queue_config.hostname, &envelope, message.span_id) .await .filter(|s| !s.is_empty()) @@ -922,28 +952,24 @@ impl DeliveryAttempt { }); let params = SessionParams { session_id: message.span_id, - core: &core, + server: &server, credentials: remote_host.credentials(), is_smtp: remote_host.is_smtp(), hostname: envelope.mx, local_hostname: &local_hostname, - timeout_ehlo: core - .core + timeout_ehlo: server .eval_if(&queue_config.timeout.ehlo, &envelope, message.span_id) .await .unwrap_or_else(|| Duration::from_secs(5 * 60)), - timeout_mail: core - .core + timeout_mail: server .eval_if(&queue_config.timeout.mail, &envelope, message.span_id) .await .unwrap_or_else(|| Duration::from_secs(5 * 60)), - timeout_rcpt: core - .core + timeout_rcpt: server .eval_if(&queue_config.timeout.rcpt, &envelope, message.span_id) .await .unwrap_or_else(|| Duration::from_secs(5 * 60)), - timeout_data: core - .core + timeout_data: server .eval_if(&queue_config.timeout.data, &envelope, message.span_id) .await .unwrap_or_else(|| Duration::from_secs(5 * 60)), @@ -956,15 +982,14 @@ impl DeliveryAttempt { || dane_policy.is_some(); let tls_connector = if allow_invalid_certs || remote_host.allow_invalid_certs() { - &core.inner.connectors.dummy_verify + &server.inner.data.smtp_connectors.dummy_verify } else { - &core.inner.connectors.pki_verify + &server.inner.data.smtp_connectors.pki_verify }; let delivery_result = if !remote_host.implicit_tls() { // Read greeting - smtp_client.timeout = core - .core + smtp_client.timeout = server .eval_if(&queue_config.timeout.greeting, &envelope, message.span_id) .await .unwrap_or_else(|| Duration::from_secs(5 * 60)); @@ -1014,8 +1039,7 @@ impl DeliveryAttempt { // Try starting TLS if tls_strategy.try_start_tls() { let time = Instant::now(); - smtp_client.timeout = core - .core + smtp_client.timeout = server .eval_if(&queue_config.timeout.tls, &envelope, message.span_id) .await .unwrap_or_else(|| Duration::from_secs(3 * 60)); @@ -1055,22 +1079,23 @@ impl DeliveryAttempt { ) { // Report DANE verification failure if let Some(tls_report) = &tls_report { - core.schedule_report(TlsEvent { - policy: dane_policy.into(), - domain: domain.domain.to_string(), - failure: FailureDetails::new( - ResultType::ValidationFailure, - ) - .with_receiving_mx_hostname(envelope.mx) - .with_receiving_ip(remote_ip) - .with_failure_reason_code( - "No matching certificates found.", - ) - .into(), - tls_record: tls_report.record.clone(), - interval: tls_report.interval, - }) - .await; + server + .schedule_report(TlsEvent { + policy: dane_policy.into(), + domain: domain.domain.to_string(), + failure: FailureDetails::new( + ResultType::ValidationFailure, + ) + .with_receiving_mx_hostname(envelope.mx) + .with_receiving_ip(remote_ip) + .with_failure_reason_code( + "No matching certificates found.", + ) + .into(), + tls_record: tls_report.record.clone(), + interval: tls_report.interval, + }) + .await; } last_status = status; @@ -1080,14 +1105,15 @@ impl DeliveryAttempt { // Report TLS success if let Some(tls_report) = &tls_report { - core.schedule_report(TlsEvent { - policy: (&mta_sts_policy, &dane_policy).into(), - domain: domain.domain.to_string(), - failure: None, - tls_record: tls_report.record.clone(), - interval: tls_report.interval, - }) - .await; + server + .schedule_report(TlsEvent { + policy: (&mta_sts_policy, &dane_policy).into(), + domain: domain.domain.to_string(), + failure: None, + tls_record: tls_report.record.clone(), + interval: tls_report.interval, + }) + .await; } // Deliver message over TLS @@ -1126,20 +1152,21 @@ impl DeliveryAttempt { ); if let Some(tls_report) = &tls_report { - core.schedule_report(TlsEvent { - policy: (&mta_sts_policy, &dane_policy).into(), - domain: domain.domain.to_string(), - failure: FailureDetails::new( - ResultType::StartTlsNotSupported, - ) - .with_receiving_mx_hostname(envelope.mx) - .with_receiving_ip(remote_ip) - .with_failure_reason_code(reason) - .into(), - tls_record: tls_report.record.clone(), - interval: tls_report.interval, - }) - .await; + server + .schedule_report(TlsEvent { + policy: (&mta_sts_policy, &dane_policy).into(), + domain: domain.domain.to_string(), + failure: FailureDetails::new( + ResultType::StartTlsNotSupported, + ) + .with_receiving_mx_hostname(envelope.mx) + .with_receiving_ip(remote_ip) + .with_failure_reason_code(reason) + .into(), + tls_record: tls_report.record.clone(), + interval: tls_report.interval, + }) + .await; } if is_strict_tls { @@ -1173,20 +1200,21 @@ impl DeliveryAttempt { if let (Some(tls_report), mail_send::Error::Tls(error)) = (&tls_report, &error) { - core.schedule_report(TlsEvent { - policy: (&mta_sts_policy, &dane_policy).into(), - domain: domain.domain.to_string(), - failure: FailureDetails::new( - ResultType::CertificateNotTrusted, - ) - .with_receiving_mx_hostname(envelope.mx) - .with_receiving_ip(remote_ip) - .with_failure_reason_code(error.to_string()) - .into(), - tls_record: tls_report.record.clone(), - interval: tls_report.interval, - }) - .await; + server + .schedule_report(TlsEvent { + policy: (&mta_sts_policy, &dane_policy).into(), + domain: domain.domain.to_string(), + failure: FailureDetails::new( + ResultType::CertificateNotTrusted, + ) + .with_receiving_mx_hostname(envelope.mx) + .with_receiving_ip(remote_ip) + .with_failure_reason_code(error.to_string()) + .into(), + tls_record: tls_report.record.clone(), + interval: tls_report.interval, + }) + .await; } last_status = if is_strict_tls { @@ -1216,8 +1244,7 @@ impl DeliveryAttempt { } } else { // Start TLS - smtp_client.timeout = core - .core + smtp_client.timeout = server .eval_if(&queue_config.timeout.tls, &envelope, message.span_id) .await .unwrap_or_else(|| Duration::from_secs(3 * 60)); @@ -1239,8 +1266,7 @@ impl DeliveryAttempt { }; // Read greeting - smtp_client.timeout = core - .core + smtp_client.timeout = server .eval_if(&queue_config.timeout.greeting, &envelope, message.span_id) .await .unwrap_or_else(|| Duration::from_secs(5 * 60)); @@ -1268,8 +1294,7 @@ impl DeliveryAttempt { }; // Update status for the current domain and continue with the next one - let schedule = core - .core + let schedule = server .eval_if::<Vec<Duration>, _>( &queue_config.retry, &envelope, @@ -1283,8 +1308,7 @@ impl DeliveryAttempt { } // Update status - let schedule = core - .core + let schedule = server .eval_if::<Vec<Duration>, _>(&queue_config.retry, &envelope, message.span_id) .await .unwrap_or_else(|| vec![Duration::from_secs(60)]); @@ -1293,20 +1317,20 @@ impl DeliveryAttempt { message.recipients = recipients; // Send Delivery Status Notifications - core.send_dsn(&mut message).await; + server.send_dsn(&mut message).await; // Notify queue manager let result = if !on_hold.is_empty() { // Save changes to disk let next_due = message.next_event_after(now()); - message.save_changes(&core, None, None).await; + message.save_changes(&server, None, None).await; trc::event!( Delivery(DeliveryEvent::ConcurrencyLimitExceeded), SpanId = span_id, ); - Event::OnHold(OnHold { + QueueEvent::OnHold(OnHold { next_due, limiters: on_hold, message: self.event, @@ -1322,10 +1346,10 @@ impl DeliveryAttempt { // Save changes to disk message - .save_changes(&core, self.event.due.into(), due.into()) + .save_changes(&server, self.event.due.into(), due.into()) .await; - Event::Reload + QueueEvent::Reload } else { trc::event!( Delivery(DeliveryEvent::Completed), @@ -1334,11 +1358,11 @@ impl DeliveryAttempt { ); // Delete message from queue - message.remove(&core, self.event.due).await; + message.remove(&server, self.event.due).await; - Event::Reload + QueueEvent::Reload }; - if core.inner.queue_tx.send(result).await.is_err() { + if server.inner.ipc.queue_tx.send(result).await.is_err() { trc::event!( Server(ServerEvent::ThreadError), Reason = "Channel closed.", diff --git a/crates/smtp/src/outbound/local.rs b/crates/smtp/src/outbound/local.rs index c4f71501..4d900967 100644 --- a/crates/smtp/src/outbound/local.rs +++ b/crates/smtp/src/outbound/local.rs @@ -4,7 +4,7 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use common::{DeliveryEvent, DeliveryResult, IngestMessage}; +use common::ipc::{DeliveryEvent, DeliveryResult, IngestMessage}; use smtp_proto::Response; use tokio::sync::{mpsc, oneshot}; use trc::ServerEvent; diff --git a/crates/smtp/src/outbound/lookup.rs b/crates/smtp/src/outbound/lookup.rs index 8b8ba8f5..8b66cc86 100644 --- a/crates/smtp/src/outbound/lookup.rs +++ b/crates/smtp/src/outbound/lookup.rs @@ -5,18 +5,19 @@ */ use std::{ + future::Future, net::{IpAddr, Ipv4Addr, Ipv6Addr}, sync::Arc, }; -use common::expr::{functions::ResolveVariable, V_MX}; +use common::{ + expr::{functions::ResolveVariable, V_MX}, + Server, +}; use mail_auth::{IpLookupStrategy, MX}; use rand::{seq::SliceRandom, Rng}; -use crate::{ - core::SMTP, - queue::{Error, ErrorDetails, Status}, -}; +use crate::queue::{Error, ErrorDetails, Status}; use super::NextHop; @@ -26,8 +27,25 @@ pub struct IpLookupResult { pub remote_ips: Vec<IpAddr>, } -impl SMTP { - pub async fn ip_lookup( +pub trait DnsLookup: Sync + Send { + fn ip_lookup( + &self, + key: &str, + strategy: IpLookupStrategy, + max_results: usize, + ) -> impl Future<Output = mail_auth::Result<Vec<IpAddr>>> + Send; + + fn resolve_host<'x>( + &'x self, + remote_host: &NextHop<'_>, + envelope: &impl ResolveVariable, + max_multihomed: usize, + session_id: u64, + ) -> impl Future<Output = Result<IpLookupResult, Status<(), Error>>> + Send; +} + +impl DnsLookup for Server { + async fn ip_lookup( &self, key: &str, strategy: IpLookupStrategy, @@ -82,7 +100,7 @@ impl SMTP { } } - pub async fn resolve_host<'x>( + async fn resolve_host<'x>( &'x self, remote_host: &NextHop<'_>, envelope: &impl ResolveVariable, @@ -92,8 +110,7 @@ impl SMTP { let remote_ips = self .ip_lookup( remote_host.fqdn_hostname().as_ref(), - self.core - .eval_if(&self.core.smtp.queue.ip_strategy, envelope, session_id) + self.eval_if(&self.core.smtp.queue.ip_strategy, envelope, session_id) .await .unwrap_or(IpLookupStrategy::Ipv4thenIpv6), max_multihomed, @@ -122,7 +139,6 @@ impl SMTP { // Obtain source IPv4 address let source_ips = self - .core .eval_if::<Vec<Ipv4Addr>, _>( &self.core.smtp.queue.source_ip.ipv4, envelope, @@ -144,7 +160,6 @@ impl SMTP { // Obtain source IPv6 address let source_ips = self - .core .eval_if::<Vec<Ipv6Addr>, _>( &self.core.smtp.queue.source_ip.ipv6, envelope, diff --git a/crates/smtp/src/outbound/mod.rs b/crates/smtp/src/outbound/mod.rs index 5f963f65..7bd71e2f 100644 --- a/crates/smtp/src/outbound/mod.rs +++ b/crates/smtp/src/outbound/mod.rs @@ -6,16 +6,17 @@ use std::borrow::Cow; -use common::config::{ - server::ServerProtocol, - smtp::queue::{RelayHost, RequireOptional}, +use common::{ + config::{ + server::ServerProtocol, + smtp::queue::{RelayHost, RequireOptional}, + }, + ipc::QueueEventLock, }; use mail_send::Credentials; use smtp_proto::{Response, Severity}; -use crate::queue::{ - spool::QueueEventLock, DeliveryAttempt, Error, ErrorDetails, HostResponse, Status, -}; +use crate::queue::{DeliveryAttempt, Error, ErrorDetails, HostResponse, Status}; pub mod client; pub mod dane; diff --git a/crates/smtp/src/outbound/mta_sts/lookup.rs b/crates/smtp/src/outbound/mta_sts/lookup.rs index 38fe9164..e6ee8f6b 100644 --- a/crates/smtp/src/outbound/mta_sts/lookup.rs +++ b/crates/smtp/src/outbound/mta_sts/lookup.rs @@ -13,11 +13,9 @@ use std::{ #[cfg(feature = "test_mode")] pub static STS_TEST_POLICY: parking_lot::Mutex<Vec<u8>> = parking_lot::Mutex::new(Vec::new()); -use common::config::smtp::resolver::Policy; +use common::{config::smtp::resolver::Policy, Server}; use mail_auth::{common::lru::DnsCache, mta_sts::MtaSts, report::tlsrpt::ResultType}; -use crate::core::SMTP; - use super::{parse::ParsePolicy, Error}; #[cfg(not(feature = "test_mode"))] @@ -26,9 +24,25 @@ use common::HttpLimitResponse; #[cfg(not(feature = "test_mode"))] const MAX_POLICY_SIZE: usize = 1024 * 1024; +pub trait MtaStsLookup: Sync + Send { + fn lookup_mta_sts_policy<'x>( + &self, + domain: &str, + timeout: Duration, + ) -> impl std::future::Future<Output = Result<Arc<Policy>, Error>> + Send; + + #[cfg(feature = "test_mode")] + fn policy_add<'x>( + &self, + key: impl mail_auth::common::resolver::IntoFqdn<'x>, + value: Policy, + valid_until: std::time::Instant, + ); +} + #[allow(unused_variables)] -impl SMTP { - pub async fn lookup_mta_sts_policy<'x>( +impl MtaStsLookup for Server { + async fn lookup_mta_sts_policy<'x>( &self, domain: &str, timeout: Duration, @@ -96,7 +110,7 @@ impl SMTP { } #[cfg(feature = "test_mode")] - pub fn policy_add<'x>( + fn policy_add<'x>( &self, key: impl mail_auth::common::resolver::IntoFqdn<'x>, value: Policy, diff --git a/crates/smtp/src/outbound/session.rs b/crates/smtp/src/outbound/session.rs index b2e0dfef..41bf98b6 100644 --- a/crates/smtp/src/outbound/session.rs +++ b/crates/smtp/src/outbound/session.rs @@ -5,6 +5,7 @@ */ use common::config::smtp::queue::RequireOptional; +use common::Server; use mail_send::Credentials; use smtp_proto::{ EhloResponse, Severity, EXT_CHUNKING, EXT_DSN, EXT_REQUIRE_TLS, EXT_SIZE, EXT_SMTP_UTF8, @@ -17,17 +18,14 @@ use tokio::io::{AsyncRead, AsyncWrite}; use trc::DeliveryEvent; use crate::outbound::client::{from_error_status, from_mail_send_error}; -use crate::{ - core::SMTP, - queue::{ErrorDetails, HostResponse, RCPT_STATUS_CHANGED}, -}; +use crate::queue::{ErrorDetails, HostResponse, RCPT_STATUS_CHANGED}; use crate::queue::{Error, Message, Recipient, Status}; use super::{client::SmtpClient, TlsStrategy}; pub struct SessionParams<'x> { - pub core: &'x SMTP, + pub server: &'x Server, pub hostname: &'x str, pub credentials: Option<&'x Credentials<String>>, pub is_smtp: bool, diff --git a/crates/smtp/src/queue/dsn.rs b/crates/smtp/src/queue/dsn.rs index ce2e5f34..97fb572e 100644 --- a/crates/smtp/src/queue/dsn.rs +++ b/crates/smtp/src/queue/dsn.rs @@ -4,6 +4,7 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ +use common::Server; use mail_builder::headers::content_type::ContentType; use mail_builder::headers::HeaderType; use mail_builder::mime::{make_boundary, BodyPart, MimePart}; @@ -13,19 +14,26 @@ use smtp_proto::{ Response, RCPT_NOTIFY_DELAY, RCPT_NOTIFY_FAILURE, RCPT_NOTIFY_NEVER, RCPT_NOTIFY_SUCCESS, }; use std::fmt::Write; +use std::future::Future; use std::time::Duration; use store::write::now; -use crate::core::SMTP; use crate::outbound::client::from_error_status; +use crate::reporting::SmtpReporting; +use super::spool::SmtpSpool; use super::{ Domain, Error, ErrorDetails, HostResponse, Message, MessageSource, QueueEnvelope, Recipient, Status, RCPT_DSN_SENT, RCPT_STATUS_CHANGED, }; -impl SMTP { - pub async fn send_dsn(&self, message: &mut Message) { +pub trait SendDsn: Sync + Send { + fn send_dsn(&self, message: &mut Message) -> impl Future<Output = ()> + Send; + fn log_dsn(&self, message: &Message) -> impl Future<Output = ()> + Send; +} + +impl SendDsn for Server { + async fn send_dsn(&self, message: &mut Message) { // Send DSN events self.log_dsn(message).await; @@ -152,8 +160,8 @@ impl SMTP { } impl Message { - pub async fn build_dsn(&mut self, core: &SMTP) -> Option<Vec<u8>> { - let config = &core.core.smtp.queue; + pub async fn build_dsn(&mut self, server: &Server) -> Option<Vec<u8>> { + let config = &server.core.smtp.queue; let now = now(); let mut txt_success = String::new(); @@ -314,8 +322,7 @@ impl Message { { let envelope = QueueEnvelope::new(self, domain_idx); - if let Some(next_notify) = core - .core + if let Some(next_notify) = server .eval_if::<Vec<Duration>, _>(&config.notify, &envelope, self.span_id) .await .and_then(|notify| { @@ -337,19 +344,16 @@ impl Message { } // Obtain hostname and sender addresses - let from_name = core - .core + let from_name = server .eval_if(&config.dsn.name, self, self.span_id) .await .unwrap_or_else(|| String::from("Mail Delivery Subsystem")); - let from_addr = core - .core + let from_addr = server .eval_if(&config.dsn.address, self, self.span_id) .await .unwrap_or_else(|| String::from("MAILER-DAEMON@localhost")); - let reporting_mta = core - .core - .eval_if(&core.core.smtp.report.submitter, self, self.span_id) + let reporting_mta = server + .eval_if(&server.core.smtp.report.submitter, self, self.span_id) .await .unwrap_or_else(|| String::from("localhost")); @@ -359,10 +363,8 @@ impl Message { let dsn = dsn_header + dsn.as_str(); // Fetch up to 1024 bytes of message headers - let headers = match core - .core - .storage - .blob + let headers = match server + .blob_store() .get_blob(self.blob_hash.as_slice(), 0..1024) .await { diff --git a/crates/smtp/src/queue/manager.rs b/crates/smtp/src/queue/manager.rs index aa2ae6ad..6f16d7d9 100644 --- a/crates/smtp/src/queue/manager.rs +++ b/crates/smtp/src/queue/manager.rs @@ -4,33 +4,39 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use std::{sync::atomic::Ordering, time::Duration}; - +use std::{ + sync::{atomic::Ordering, Arc}, + time::Duration, +}; + +use common::{ + core::BuildServer, + ipc::{OnHold, QueueEvent, QueueEventLock}, + Inner, +}; use store::write::now; use tokio::sync::mpsc; -use crate::core::{SmtpInstance, SMTP}; - -use super::{spool::QueueEventLock, DeliveryAttempt, Event, Message, OnHold, Status}; +use super::{spool::SmtpSpool, DeliveryAttempt, Message, Status}; pub(crate) const SHORT_WAIT: Duration = Duration::from_millis(1); pub(crate) const LONG_WAIT: Duration = Duration::from_secs(86400 * 365); pub struct Queue { - pub core: SmtpInstance, + pub core: Arc<Inner>, pub on_hold: Vec<OnHold<QueueEventLock>>, pub next_wake_up: Duration, } -impl SpawnQueue for mpsc::Receiver<Event> { - fn spawn(mut self, core: SmtpInstance) { +impl SpawnQueue for mpsc::Receiver<QueueEvent> { + fn spawn(mut self, core: Arc<Inner>) { tokio::spawn(async move { let mut queue = Queue::new(core); loop { let on_hold = match tokio::time::timeout(queue.next_wake_up, self.recv()).await { - Ok(Some(Event::OnHold(on_hold))) => on_hold.into(), - Ok(Some(Event::Stop)) | Ok(None) => { + Ok(Some(QueueEvent::OnHold(on_hold))) => on_hold.into(), + Ok(Some(QueueEvent::Stop)) | Ok(None) => { break; } _ => None, @@ -48,7 +54,7 @@ impl SpawnQueue for mpsc::Receiver<Event> { } impl Queue { - pub fn new(core: SmtpInstance) -> Self { + pub fn new(core: Arc<Inner>) -> Self { Queue { core, on_hold: Vec::with_capacity(128), @@ -58,20 +64,20 @@ impl Queue { pub async fn process_events(&mut self) { // Deliver any concurrency limited messages - let core = SMTP::from(self.core.clone()); + let server = self.core.build_server(); while let Some(queue_event) = self.next_on_hold() { DeliveryAttempt::new(queue_event) - .try_deliver(core.clone()) + .try_deliver(server.clone()) .await; } // Deliver scheduled messages let now = now(); self.next_wake_up = LONG_WAIT; - for queue_event in core.next_event().await { + for queue_event in server.next_event().await { if queue_event.due <= now { DeliveryAttempt::new(queue_event) - .try_deliver(core.clone()) + .try_deliver(server.clone()) .await; } else { self.next_wake_up = Duration::from_secs(queue_event.due - now); @@ -217,5 +223,5 @@ impl Message { } pub trait SpawnQueue { - fn spawn(self, core: SmtpInstance); + fn spawn(self, core: Arc<Inner>); } diff --git a/crates/smtp/src/queue/mod.rs b/crates/smtp/src/queue/mod.rs index e1e98772..5849054c 100644 --- a/crates/smtp/src/queue/mod.rs +++ b/crates/smtp/src/queue/mod.rs @@ -12,15 +12,14 @@ use std::{ use common::{ expr::{self, functions::ResolveVariable, *}, - listener::limiter::{ConcurrencyLimiter, InFlight}, + ipc::QueueEventLock, + listener::limiter::InFlight, }; use serde::{Deserialize, Serialize}; use smtp_proto::Response; use store::write::now; use utils::BlobHash; -use self::spool::QueueEventLock; - pub mod dsn; pub mod manager; pub mod quota; @@ -29,20 +28,6 @@ pub mod throttle; pub type QueueId = u64; -#[derive(Debug)] -pub enum Event { - Reload, - OnHold(OnHold<QueueEventLock>), - Stop, -} - -#[derive(Debug)] -pub struct OnHold<T> { - pub next_due: Option<u64>, - pub limiters: Vec<ConcurrencyLimiter>, - pub message: T, -} - #[derive(Debug, Clone, Serialize, Deserialize)] pub struct Schedule<T> { pub due: u64, diff --git a/crates/smtp/src/queue/quota.rs b/crates/smtp/src/queue/quota.rs index e7323b63..73f65380 100644 --- a/crates/smtp/src/queue/quota.rs +++ b/crates/smtp/src/queue/quota.rs @@ -4,19 +4,34 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use common::{config::smtp::queue::QueueQuota, expr::functions::ResolveVariable}; +use std::future::Future; + +use common::{config::smtp::queue::QueueQuota, expr::functions::ResolveVariable, Server}; use store::{ write::{BatchBuilder, QueueClass, ValueClass}, ValueKey, }; use trc::QueueEvent; -use crate::core::{throttle::NewKey, SMTP}; +use crate::core::throttle::NewKey; use super::{Message, QueueEnvelope, QuotaKey, Status}; -impl SMTP { - pub async fn has_quota(&self, message: &mut Message) -> bool { +pub trait HasQueueQuota: Sync + Send { + fn has_quota(&self, message: &mut Message) -> impl Future<Output = bool> + Send; + fn check_quota<'x>( + &'x self, + quota: &'x QueueQuota, + envelope: &impl ResolveVariable, + size: usize, + id: u64, + refs: &mut Vec<QuotaKey>, + session_id: u64, + ) -> impl Future<Output = bool> + Send; +} + +impl HasQueueQuota for Server { + async fn has_quota(&self, message: &mut Message) -> bool { let mut quota_keys = Vec::new(); if !self.core.smtp.queue.quota.sender.is_empty() { @@ -110,7 +125,6 @@ impl SMTP { ) -> bool { if !quota.expr.is_empty() && self - .core .eval_expr("a.expr, envelope, "check_quota", session_id) .await .unwrap_or(false) diff --git a/crates/smtp/src/queue/spool.rs b/crates/smtp/src/queue/spool.rs index 54aecc7e..5c33cc12 100644 --- a/crates/smtp/src/queue/spool.rs +++ b/crates/smtp/src/queue/spool.rs @@ -5,32 +5,44 @@ */ use crate::queue::DomainPart; +use common::ipc::{QueueEvent, QueueEventLock}; +use common::Server; use std::borrow::Cow; +use std::future::Future; use std::time::{Duration, SystemTime}; use store::write::key::DeserializeBigEndian; -use store::write::{now, BatchBuilder, Bincode, BlobOp, QueueClass, QueueEvent, ValueClass}; +use store::write::{now, BatchBuilder, Bincode, BlobOp, QueueClass, ValueClass}; use store::{Deserialize, IterateParams, Serialize, ValueKey, U64_LEN}; use trc::ServerEvent; use utils::BlobHash; -use crate::core::SMTP; - use super::{ - Domain, Event, Message, MessageSource, QueueEnvelope, QueueId, QuotaKey, Recipient, Schedule, - Status, + Domain, Message, MessageSource, QueueEnvelope, QueueId, QuotaKey, Recipient, Schedule, Status, }; pub const LOCK_EXPIRY: u64 = 300; -#[derive(Debug)] -pub struct QueueEventLock { - pub due: u64, - pub queue_id: u64, - pub lock_expiry: u64, +pub trait SmtpSpool: Sync + Send { + fn new_message( + &self, + return_path: impl Into<String>, + return_path_lcase: impl Into<String>, + return_path_domain: impl Into<String>, + span_id: u64, + ) -> Message; + + fn next_event(&self) -> impl Future<Output = Vec<QueueEventLock>> + Send; + + fn try_lock_event( + &self, + event: QueueEventLock, + ) -> impl Future<Output = Option<QueueEventLock>> + Send; + + fn read_message(&self, id: QueueId) -> impl Future<Output = Option<Message>> + Send; } -impl SMTP { - pub fn new_message( +impl SmtpSpool for Server { + fn new_message( &self, return_path: impl Into<String>, return_path_lcase: impl Into<String>, @@ -41,7 +53,7 @@ impl SMTP { .duration_since(SystemTime::UNIX_EPOCH) .map_or(0, |d| d.as_secs()); Message { - queue_id: self.inner.queue_id_gen.generate().unwrap_or(created), + queue_id: self.inner.data.queue_id_gen.generate().unwrap_or(created), span_id, created, return_path: return_path.into(), @@ -58,22 +70,24 @@ impl SMTP { } } - pub async fn next_event(&self) -> Vec<QueueEventLock> { - let from_key = ValueKey::from(ValueClass::Queue(QueueClass::MessageEvent(QueueEvent { - due: 0, - queue_id: 0, - }))); - let to_key = ValueKey::from(ValueClass::Queue(QueueClass::MessageEvent(QueueEvent { - due: u64::MAX, - queue_id: u64::MAX, - }))); + async fn next_event(&self) -> Vec<QueueEventLock> { + let from_key = ValueKey::from(ValueClass::Queue(QueueClass::MessageEvent( + store::write::QueueEvent { + due: 0, + queue_id: 0, + }, + ))); + let to_key = ValueKey::from(ValueClass::Queue(QueueClass::MessageEvent( + store::write::QueueEvent { + due: u64::MAX, + queue_id: u64::MAX, + }, + ))); let mut events = Vec::new(); let now = now(); let result = self - .core - .storage - .data + .store() .iterate( IterateParams::new(from_key, to_key).ascending(), |key, value| { @@ -107,10 +121,10 @@ impl SMTP { events } - pub async fn try_lock_event(&self, mut event: QueueEventLock) -> Option<QueueEventLock> { + async fn try_lock_event(&self, mut event: QueueEventLock) -> Option<QueueEventLock> { let mut batch = BatchBuilder::new(); batch.assert_value( - ValueClass::Queue(QueueClass::MessageEvent(QueueEvent { + ValueClass::Queue(QueueClass::MessageEvent(store::write::QueueEvent { due: event.due, queue_id: event.queue_id, })), @@ -118,13 +132,13 @@ impl SMTP { ); event.lock_expiry = now() + LOCK_EXPIRY; batch.set( - ValueClass::Queue(QueueClass::MessageEvent(QueueEvent { + ValueClass::Queue(QueueClass::MessageEvent(store::write::QueueEvent { due: event.due, queue_id: event.queue_id, })), event.lock_expiry.serialize(), ); - match self.core.storage.data.write(batch.build()).await { + match self.store().write(batch.build()).await { Ok(_) => Some(event), Err(err) if err.is_assertion_failure() => { trc::event!( @@ -145,11 +159,9 @@ impl SMTP { } } - pub async fn read_message(&self, id: QueueId) -> Option<Message> { + async fn read_message(&self, id: QueueId) -> Option<Message> { match self - .core - .storage - .data + .store() .get_value::<Bincode<Message>>(ValueKey::from(ValueClass::Queue(QueueClass::Message( id, )))) @@ -174,7 +186,7 @@ impl Message { raw_headers: Option<&[u8]>, raw_message: &[u8], session_id: u64, - core: &SMTP, + server: &Server, source: MessageSource, ) -> bool { // Write blob @@ -203,7 +215,7 @@ impl Message { }, 0u32.serialize(), ); - if let Err(err) = core.core.storage.data.write(batch.build()).await { + if let Err(err) = server.store().write(batch.build()).await { trc::error!(err .details("Failed to write to store.") .span_id(session_id) @@ -211,10 +223,8 @@ impl Message { return false; } - if let Err(err) = core - .core - .storage - .blob + if let Err(err) = server + .blob_store() .put_blob(self.blob_hash.as_slice(), message.as_ref()) .await { @@ -271,7 +281,7 @@ impl Message { } batch .set( - ValueClass::Queue(QueueClass::MessageEvent(QueueEvent { + ValueClass::Queue(QueueClass::MessageEvent(store::write::QueueEvent { due: self.next_event().unwrap_or_default(), queue_id: self.queue_id, })), @@ -299,7 +309,7 @@ impl Message { Bincode::new(self).serialize(), ); - if let Err(err) = core.core.storage.data.write(batch.build()).await { + if let Err(err) = server.store().write(batch.build()).await { trc::error!(err .details("Failed to write to store.") .span_id(session_id) @@ -309,7 +319,14 @@ impl Message { } // Queue the message - if core.inner.queue_tx.send(Event::Reload).await.is_err() { + if server + .inner + .ipc + .queue_tx + .send(QueueEvent::Reload) + .await + .is_err() + { trc::event!( Server(ServerEvent::ThreadError), Reason = "Channel closed.", @@ -326,7 +343,7 @@ impl Message { rcpt: impl Into<String>, rcpt_lcase: impl Into<String>, rcpt_domain: impl Into<String>, - core: &SMTP, + server: &Server, ) { let rcpt_domain = rcpt_domain.into(); let domain_idx = @@ -343,10 +360,9 @@ impl Message { status: Status::Scheduled, }); - let expires = core - .core + let expires = server .eval_if( - &core.core.smtp.queue.expire, + &server.core.smtp.queue.expire, &QueueEnvelope::new(self, idx), self.span_id, ) @@ -370,17 +386,17 @@ impl Message { }); } - pub async fn add_recipient(&mut self, rcpt: impl Into<String>, core: &SMTP) { + pub async fn add_recipient(&mut self, rcpt: impl Into<String>, server: &Server) { let rcpt = rcpt.into(); let rcpt_lcase = rcpt.to_lowercase(); let rcpt_domain = rcpt_lcase.domain_part().to_string(); - self.add_recipient_parts(rcpt, rcpt_lcase, rcpt_domain, core) + self.add_recipient_parts(rcpt, rcpt_lcase, rcpt_domain, server) .await; } pub async fn save_changes( mut self, - core: &SMTP, + server: &Server, prev_event: Option<u64>, next_event: Option<u64>, ) -> bool { @@ -395,12 +411,14 @@ impl Message { let mut batch = BatchBuilder::new(); if let (Some(prev_event), Some(next_event)) = (prev_event, next_event) { batch - .clear(ValueClass::Queue(QueueClass::MessageEvent(QueueEvent { - due: prev_event, - queue_id: self.queue_id, - }))) + .clear(ValueClass::Queue(QueueClass::MessageEvent( + store::write::QueueEvent { + due: prev_event, + queue_id: self.queue_id, + }, + ))) .set( - ValueClass::Queue(QueueClass::MessageEvent(QueueEvent { + ValueClass::Queue(QueueClass::MessageEvent(store::write::QueueEvent { due: next_event, queue_id: self.queue_id, })), @@ -414,7 +432,7 @@ impl Message { Bincode::new(self).serialize(), ); - if let Err(err) = core.core.storage.data.write(batch.build()).await { + if let Err(err) = server.store().write(batch.build()).await { trc::error!(err .details("Failed to save changes.") .span_id(span_id) @@ -425,7 +443,7 @@ impl Message { } } - pub async fn remove(self, core: &SMTP, prev_event: u64) -> bool { + pub async fn remove(self, server: &Server, prev_event: u64) -> bool { let mut batch = BatchBuilder::new(); // Release all quotas @@ -448,13 +466,15 @@ impl Message { hash: self.blob_hash.clone(), id: self.queue_id, }) - .clear(ValueClass::Queue(QueueClass::MessageEvent(QueueEvent { - due: prev_event, - queue_id: self.queue_id, - }))) + .clear(ValueClass::Queue(QueueClass::MessageEvent( + store::write::QueueEvent { + due: prev_event, + queue_id: self.queue_id, + }, + ))) .clear(ValueClass::Queue(QueueClass::Message(self.queue_id))); - if let Err(err) = core.core.storage.data.write(batch.build()).await { + if let Err(err) = server.store().write(batch.build()).await { trc::error!(err .details("Failed to write to update queue.") .span_id(self.span_id) diff --git a/crates/smtp/src/queue/throttle.rs b/crates/smtp/src/queue/throttle.rs index a9de94bc..e80c3a44 100644 --- a/crates/smtp/src/queue/throttle.rs +++ b/crates/smtp/src/queue/throttle.rs @@ -4,15 +4,18 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ +use std::future::Future; + use common::{ config::smtp::Throttle, expr::functions::ResolveVariable, listener::limiter::{ConcurrencyLimiter, InFlight}, + Server, }; use dashmap::mapref::entry::Entry; use store::write::now; -use crate::core::{throttle::NewKey, SMTP}; +use crate::core::throttle::NewKey; use super::{Domain, Status}; @@ -22,8 +25,18 @@ pub enum Error { Rate { retry_at: u64 }, } -impl SMTP { - pub async fn is_allowed<'x>( +pub trait IsAllowed: Sync + Send { + fn is_allowed<'x>( + &'x self, + throttle: &'x Throttle, + envelope: &impl ResolveVariable, + in_flight: &mut Vec<InFlight>, + session_id: u64, + ) -> impl Future<Output = Result<(), Error>> + Send; +} + +impl IsAllowed for Server { + async fn is_allowed<'x>( &'x self, throttle: &'x Throttle, envelope: &impl ResolveVariable, @@ -32,7 +45,6 @@ impl SMTP { ) -> Result<(), Error> { if throttle.expr.is_empty() || self - .core .eval_expr(&throttle.expr, envelope, "throttle", session_id) .await .unwrap_or(false) @@ -64,7 +76,7 @@ impl SMTP { } if let Some(concurrency) = &throttle.concurrency { - match self.inner.queue_throttle.entry(key) { + match self.inner.data.smtp_queue_throttle.entry(key) { Entry::Occupied(mut e) => { let limiter = e.get_mut(); if let Some(inflight) = limiter.is_allowed() { diff --git a/crates/smtp/src/reporting/analysis.rs b/crates/smtp/src/reporting/analysis.rs index 99ded690..ee6b4230 100644 --- a/crates/smtp/src/reporting/analysis.rs +++ b/crates/smtp/src/reporting/analysis.rs @@ -12,6 +12,7 @@ use std::{ }; use ahash::AHashMap; +use common::Server; use mail_auth::{ flate2::read::GzDecoder, report::{tlsrpt::TlsReport, ActionDisposition, DmarcResult, Feedback, Report}, @@ -25,8 +26,6 @@ use store::{ }; use trc::IncomingReportEvent; -use crate::core::SMTP; - enum Compression { None, Gzip, @@ -53,8 +52,12 @@ pub struct IncomingReport<T> { pub report: T, } -impl SMTP { - pub fn analyze_report(&self, message: Arc<Vec<u8>>, session_id: u64) { +pub trait AnalyzeReport: Sync + Send { + fn analyze_report(&self, message: Arc<Vec<u8>>, session_id: u64); +} + +impl AnalyzeReport for Server { + fn analyze_report(&self, message: Arc<Vec<u8>>, session_id: u64) { let core = self.clone(); tokio::spawn(async move { let message = if let Some(message) = MessageParser::default().parse(message.as_ref()) { @@ -282,7 +285,7 @@ impl SMTP { // Store report if let Some(expires_in) = &core.core.smtp.report.analysis.store { let expires = now() + expires_in.as_secs(); - let id = core.inner.queue_id_gen.generate().unwrap_or(expires); + let id = core.inner.data.queue_id_gen.generate().unwrap_or(expires); let mut batch = BatchBuilder::new(); match report { diff --git a/crates/smtp/src/reporting/dkim.rs b/crates/smtp/src/reporting/dkim.rs index 9c3e4b4e..f0b4d682 100644 --- a/crates/smtp/src/reporting/dkim.rs +++ b/crates/smtp/src/reporting/dkim.rs @@ -11,7 +11,7 @@ use mail_auth::{ use trc::OutgoingReportEvent; use utils::config::Rate; -use crate::core::Session; +use crate::{core::Session, reporting::SmtpReporting}; impl<T: SessionStream> Session<T> { pub async fn send_dkim_report( @@ -44,10 +44,9 @@ impl<T: SessionStream> Session<T> { return; } - let config = &self.core.core.smtp.report.dkim; + let config = &self.server.core.smtp.report.dkim; let from_addr = self - .core - .core + .server .eval_if(&config.address, self, self.data.session_id) .await .unwrap_or_else(|| "MAILER-DAEMON@localhost".to_string()); @@ -64,8 +63,7 @@ impl<T: SessionStream> Session<T> { .with_headers(std::str::from_utf8(message.raw_headers()).unwrap_or_default()) .write_rfc5322( ( - self.core - .core + self.server .eval_if(&config.name, self, self.data.session_id) .await .unwrap_or_else(|| "Mail Delivery Subsystem".to_string()) @@ -74,8 +72,7 @@ impl<T: SessionStream> Session<T> { ), rcpt, &self - .core - .core + .server .eval_if(&config.subject, self, self.data.session_id) .await .unwrap_or_else(|| "DKIM Report".to_string()), @@ -91,7 +88,7 @@ impl<T: SessionStream> Session<T> { ); // Send report - self.core + self.server .send_report( &from_addr, [rcpt].into_iter(), diff --git a/crates/smtp/src/reporting/dmarc.rs b/crates/smtp/src/reporting/dmarc.rs index 42ce6b38..5b106fea 100644 --- a/crates/smtp/src/reporting/dmarc.rs +++ b/crates/smtp/src/reporting/dmarc.rs @@ -4,10 +4,15 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use std::collections::hash_map::Entry; +use std::{collections::hash_map::Entry, future::Future}; use ahash::AHashMap; -use common::{config::smtp::report::AggregateFrequency, listener::SessionStream}; +use common::{ + config::smtp::report::AggregateFrequency, + ipc::{DmarcEvent, ToHash}, + listener::SessionStream, + Server, +}; use mail_auth::{ common::verify::VerifySignature, dmarc::{self, URI}, @@ -23,11 +28,12 @@ use trc::OutgoingReportEvent; use utils::config::Rate; use crate::{ - core::{Session, SMTP}, + core::Session, queue::{DomainPart, RecipientDomain}, + reporting::SmtpReporting, }; -use super::{scheduler::ToHash, AggregateTimestamp, DmarcEvent, ReportLock, SerializedSize}; +use super::{AggregateTimestamp, ReportLock, SerializedSize}; #[derive(Debug, PartialEq, Eq, serde::Serialize, serde::Deserialize)] pub struct DmarcFormat { @@ -48,19 +54,18 @@ impl<T: SessionStream> Session<T> { arc_output: &Option<ArcOutput<'_>>, ) { let dmarc_record = dmarc_output.dmarc_record_cloned().unwrap(); - let config = &self.core.core.smtp.report.dmarc; + let config = &self.server.core.smtp.report.dmarc; // Send failure report if let (Some(failure_rate), Some(report_options)) = ( - self.core - .core + self.server .eval_if::<Rate, _>(&config.send, self, self.data.session_id) .await, dmarc_output.failure_report(), ) { // Verify that any external reporting addresses are authorized let rcpts = match self - .core + .server .core .smtp .resolvers @@ -113,8 +118,7 @@ impl<T: SessionStream> Session<T> { if !rcpts.is_empty() { let mut report = Vec::with_capacity(128); let from_addr = self - .core - .core + .server .eval_if(&config.address, self, self.data.session_id) .await .unwrap_or_else(|| "MAILER-DAEMON@localhost".to_string()); @@ -197,8 +201,7 @@ impl<T: SessionStream> Session<T> { }) .write_rfc5322( ( - self.core - .core + self.server .eval_if(&config.name, self, self.data.session_id) .await .unwrap_or_else(|| "Mail Delivery Subsystem".to_string()) @@ -207,8 +210,7 @@ impl<T: SessionStream> Session<T> { ), &rcpts.join(", "), &self - .core - .core + .server .eval_if(&config.subject, self, self.data.session_id) .await .unwrap_or_else(|| "DMARC Report".to_string()), @@ -227,7 +229,7 @@ impl<T: SessionStream> Session<T> { ); // Send report - self.core + self.server .send_report( &from_addr, rcpts.into_iter(), @@ -251,10 +253,9 @@ impl<T: SessionStream> Session<T> { // Send aggregate reports let interval = self - .core - .core + .server .eval_if( - &self.core.core.smtp.report.dmarc_aggregate.send, + &self.server.core.smtp.report.dmarc_aggregate.send, self, self.data.session_id, ) @@ -289,7 +290,7 @@ impl<T: SessionStream> Session<T> { } // Submit DMARC report event - self.core + self.server .schedule_report(DmarcEvent { domain: dmarc_output.into_domain(), report_record, @@ -300,9 +301,22 @@ impl<T: SessionStream> Session<T> { } } -impl SMTP { - pub async fn send_dmarc_aggregate_report(&self, event: ReportEvent) { - let span_id = self.inner.span_id_gen.generate().unwrap_or_else(now); +pub trait DmarcReporting: Sync + Send { + fn send_dmarc_aggregate_report(&self, event: ReportEvent) -> impl Future<Output = ()> + Send; + fn generate_dmarc_aggregate_report( + &self, + event: &ReportEvent, + rua: &mut Vec<URI>, + serialized_size: Option<&mut serde_json::Serializer<SerializedSize>>, + span_id: u64, + ) -> impl Future<Output = trc::Result<Option<Report>>> + Send; + fn delete_dmarc_report(&self, event: ReportEvent) -> impl Future<Output = ()> + Send; + fn schedule_dmarc(&self, event: Box<DmarcEvent>) -> impl Future<Output = ()> + Send; +} + +impl DmarcReporting for Server { + async fn send_dmarc_aggregate_report(&self, event: ReportEvent) { + let span_id = self.inner.data.span_id_gen.generate().unwrap_or_else(now); trc::event!( OutgoingReport(OutgoingReportEvent::DmarcAggregateReport), @@ -315,14 +329,13 @@ impl SMTP { // Generate report let mut serialized_size = serde_json::Serializer::new(SerializedSize::new( - self.core - .eval_if( - &self.core.smtp.report.dmarc_aggregate.max_size, - &RecipientDomain::new(event.domain.as_str()), - span_id, - ) - .await - .unwrap_or(25 * 1024 * 1024), + self.eval_if( + &self.core.smtp.report.dmarc_aggregate.max_size, + &RecipientDomain::new(event.domain.as_str()), + span_id, + ) + .await + .unwrap_or(25 * 1024 * 1024), )); let mut rua = Vec::new(); let report = match self @@ -392,7 +405,6 @@ impl SMTP { // Serialize report let config = &self.core.smtp.report.dmarc_aggregate; let from_addr = self - .core .eval_if( &config.address, &RecipientDomain::new(event.domain.as_str()), @@ -403,7 +415,6 @@ impl SMTP { let mut message = Vec::with_capacity(2048); let _ = report.write_rfc5322( &self - .core .eval_if( &self.core.smtp.report.submitter, &RecipientDomain::new(event.domain.as_str()), @@ -412,15 +423,14 @@ impl SMTP { .await .unwrap_or_else(|| "localhost".to_string()), ( - self.core - .eval_if( - &config.name, - &RecipientDomain::new(event.domain.as_str()), - span_id, - ) - .await - .unwrap_or_else(|| "Mail Delivery Subsystem".to_string()) - .as_str(), + self.eval_if( + &config.name, + &RecipientDomain::new(event.domain.as_str()), + span_id, + ) + .await + .unwrap_or_else(|| "Mail Delivery Subsystem".to_string()) + .as_str(), from_addr.as_str(), ), rua.iter().map(|a| a.as_str()), @@ -441,7 +451,7 @@ impl SMTP { self.delete_dmarc_report(event).await; } - pub async fn generate_dmarc_aggregate_report( + async fn generate_dmarc_aggregate_report( &self, event: &ReportEvent, rua: &mut Vec<URI>, @@ -473,17 +483,15 @@ impl SMTP { .with_date_range_end(event.due) .with_report_id(format!("{}_{}", event.policy_hash, event.seq_id)) .with_email( - self.core - .eval_if( - &config.address, - &RecipientDomain::new(event.domain.as_str()), - span_id, - ) - .await - .unwrap_or_else(|| "MAILER-DAEMON@localhost".to_string()), + self.eval_if( + &config.address, + &RecipientDomain::new(event.domain.as_str()), + span_id, + ) + .await + .unwrap_or_else(|| "MAILER-DAEMON@localhost".to_string()), ); if let Some(org_name) = self - .core .eval_if::<String, _>( &config.org_name, &RecipientDomain::new(event.domain.as_str()), @@ -494,7 +502,6 @@ impl SMTP { report = report.with_org_name(org_name); } if let Some(contact_info) = self - .core .eval_if::<String, _>( &config.contact_info, &RecipientDomain::new(event.domain.as_str()), @@ -561,7 +568,7 @@ impl SMTP { Ok(Some(report)) } - pub async fn delete_dmarc_report(&self, event: ReportEvent) { + async fn delete_dmarc_report(&self, event: ReportEvent) { let from_key = ReportEvent { due: event.due, policy_hash: event.policy_hash, @@ -600,7 +607,7 @@ impl SMTP { } } - pub async fn schedule_dmarc(&self, event: Box<DmarcEvent>) { + async fn schedule_dmarc(&self, event: Box<DmarcEvent>) { let created = event.interval.to_timestamp(); let deliver_at = created + event.interval.as_secs(); let mut report_event = ReportEvent { @@ -647,7 +654,7 @@ impl SMTP { } // Write entry - report_event.seq_id = self.inner.queue_id_gen.generate().unwrap_or_else(now); + report_event.seq_id = self.inner.data.queue_id_gen.generate().unwrap_or_else(now); builder.set( ValueClass::Queue(QueueClass::DmarcReportEvent(report_event)), Bincode::new(event.report_record).serialize(), diff --git a/crates/smtp/src/reporting/mod.rs b/crates/smtp/src/reporting/mod.rs index fedd6689..92c81777 100644 --- a/crates/smtp/src/reporting/mod.rs +++ b/crates/smtp/src/reporting/mod.rs @@ -4,23 +4,17 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use std::{io, sync::Arc, time::SystemTime}; +use std::{future::Future, io, time::SystemTime}; use common::{ - config::smtp::{ - report::{AddressMatch, AggregateFrequency}, - resolver::{Policy, Tlsa}, - }, + config::smtp::report::{AddressMatch, AggregateFrequency}, expr::if_block::IfBlock, - USER_AGENT, + ipc::ReportingEvent, + Server, USER_AGENT, }; use mail_auth::{ common::headers::HeaderWriter, - dmarc::Dmarc, - mta_sts::TlsRpt, - report::{ - tlsrpt::FailureDetails, AuthFailureType, DeliveryResult, Feedback, FeedbackType, Record, - }, + report::{AuthFailureType, DeliveryResult, Feedback, FeedbackType}, }; use mail_parser::DateTime; @@ -28,9 +22,9 @@ use store::write::{QueueClass, ReportEvent}; use tokio::io::{AsyncRead, AsyncWrite}; use crate::{ - core::{Session, SMTP}, + core::Session, inbound::DkimSign, - queue::{DomainPart, Message, MessageSource}, + queue::{spool::SmtpSpool, DomainPart, Message, MessageSource}, }; pub mod analysis; @@ -40,37 +34,6 @@ pub mod scheduler; pub mod spf; pub mod tls; -#[derive(Debug)] -pub enum Event { - Dmarc(Box<DmarcEvent>), - Tls(Box<TlsEvent>), - Stop, -} - -#[derive(Debug)] -pub struct DmarcEvent { - pub domain: String, - pub report_record: Record, - pub dmarc_record: Arc<Dmarc>, - pub interval: AggregateFrequency, -} - -#[derive(Debug)] -pub struct TlsEvent { - pub domain: String, - pub policy: PolicyType, - pub failure: Option<FailureDetails>, - pub tls_record: Arc<TlsRpt>, - pub interval: AggregateFrequency, -} - -#[derive(Debug, Hash, PartialEq, Eq)] -pub enum PolicyType { - Tlsa(Option<Arc<Tlsa>>), - Sts(Option<Arc<Policy>>), - None, -} - impl<T: AsyncWrite + AsyncRead + Unpin> Session<T> { pub fn new_auth_failure(&self, ft: AuthFailureType, rejected: bool) -> Feedback<'_> { Feedback::new(FeedbackType::AuthFailure) @@ -91,7 +54,7 @@ impl<T: AsyncWrite + AsyncRead + Unpin> Session<T> { } pub fn is_report(&self) -> bool { - for addr_match in &self.core.core.smtp.report.analysis.addresses { + for addr_match in &self.server.core.smtp.report.analysis.addresses { for addr in &self.data.rcpt_to { match addr_match { AddressMatch::StartsWith(prefix) if addr.address_lcase.starts_with(prefix) => { @@ -110,11 +73,44 @@ impl<T: AsyncWrite + AsyncRead + Unpin> Session<T> { } } -impl SMTP { - pub async fn send_report( +pub trait SmtpReporting: Sync + Send { + fn send_report( + &self, + from_addr: &str, + rcpts: impl Iterator<Item = impl AsRef<str> + Sync + Send> + Sync + Send, + report: Vec<u8>, + sign_config: &IfBlock, + deliver_now: bool, + parent_session_id: u64, + ) -> impl Future<Output = ()> + Send; + + fn send_autogenerated( + &self, + from_addr: impl Into<String> + Sync + Send, + rcpts: impl Iterator<Item = impl Into<String> + Sync + Send> + Sync + Send, + raw_message: Vec<u8>, + sign_config: Option<&IfBlock>, + parent_session_id: u64, + ) -> impl Future<Output = ()> + Send; + + fn schedule_report( + &self, + report: impl Into<ReportingEvent> + Sync + Send, + ) -> impl Future<Output = ()> + Send; + + fn sign_message( + &self, + message: &mut Message, + config: &IfBlock, + bytes: &[u8], + ) -> impl Future<Output = Option<Vec<u8>>> + Send; +} + +impl SmtpReporting for Server { + async fn send_report( &self, from_addr: &str, - rcpts: impl Iterator<Item = impl AsRef<str>>, + rcpts: impl Iterator<Item = impl AsRef<str> + Sync + Send> + Sync + Send, report: Vec<u8>, sign_config: &IfBlock, deliver_now: bool, @@ -163,10 +159,10 @@ impl SMTP { .await; } - pub async fn send_autogenerated( + async fn send_autogenerated( &self, - from_addr: impl Into<String>, - rcpts: impl Iterator<Item = impl Into<String>>, + from_addr: impl Into<String> + Sync + Send, + rcpts: impl Iterator<Item = impl Into<String> + Sync + Send> + Sync + Send, raw_message: Vec<u8>, sign_config: Option<&IfBlock>, parent_session_id: u64, @@ -205,8 +201,8 @@ impl SMTP { .await; } - pub async fn schedule_report(&self, report: impl Into<Event>) { - if self.inner.report_tx.send(report.into()).await.is_err() { + async fn schedule_report(&self, report: impl Into<ReportingEvent> + Sync + Send) { + if self.inner.ipc.report_tx.send(report.into()).await.is_err() { trc::event!( Server(trc::ServerEvent::ThreadError), CausedBy = trc::location!(), @@ -215,21 +211,20 @@ impl SMTP { } } - pub async fn sign_message( + async fn sign_message( &self, message: &mut Message, config: &IfBlock, bytes: &[u8], ) -> Option<Vec<u8>> { let signers = self - .core .eval_if::<Vec<String>, _>(config, message, message.span_id) .await .unwrap_or_default(); if !signers.is_empty() { let mut headers = Vec::with_capacity(64); for signer in signers.iter() { - if let Some(signer) = self.core.get_dkim_signer(signer, message.span_id) { + if let Some(signer) = self.get_dkim_signer(signer, message.span_id) { match signer.sign(bytes) { Ok(signature) => { signature.write_header(&mut headers); @@ -300,52 +295,6 @@ impl AggregateTimestamp for AggregateFrequency { } } -impl From<DmarcEvent> for Event { - fn from(value: DmarcEvent) -> Self { - Event::Dmarc(Box::new(value)) - } -} - -impl From<TlsEvent> for Event { - fn from(value: TlsEvent) -> Self { - Event::Tls(Box::new(value)) - } -} - -impl From<Arc<Tlsa>> for PolicyType { - fn from(value: Arc<Tlsa>) -> Self { - PolicyType::Tlsa(Some(value)) - } -} - -impl From<Arc<Policy>> for PolicyType { - fn from(value: Arc<Policy>) -> Self { - PolicyType::Sts(Some(value)) - } -} - -impl From<&Arc<Tlsa>> for PolicyType { - fn from(value: &Arc<Tlsa>) -> Self { - PolicyType::Tlsa(Some(value.clone())) - } -} - -impl From<&Arc<Policy>> for PolicyType { - fn from(value: &Arc<Policy>) -> Self { - PolicyType::Sts(Some(value.clone())) - } -} - -impl From<(&Option<Arc<Policy>>, &Option<Arc<Tlsa>>)> for PolicyType { - fn from(value: (&Option<Arc<Policy>>, &Option<Arc<Tlsa>>)) -> Self { - match value { - (Some(value), _) => PolicyType::Sts(Some(value.clone())), - (_, Some(value)) => PolicyType::Tlsa(Some(value.clone())), - _ => PolicyType::None, - } - } -} - pub struct SerializedSize { bytes_left: usize, } diff --git a/crates/smtp/src/reporting/scheduler.rs b/crates/smtp/src/reporting/scheduler.rs index 83a1c354..ba1c9fc3 100644 --- a/crates/smtp/src/reporting/scheduler.rs +++ b/crates/smtp/src/reporting/scheduler.rs @@ -4,34 +4,33 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use ahash::{AHashMap, RandomState}; -use common::Core; -use mail_auth::dmarc::Dmarc; +use ahash::AHashMap; +use common::{core::BuildServer, ipc::ReportingEvent, Inner, Server}; -use std::time::{Duration, Instant, SystemTime}; +use std::{ + future::Future, + sync::Arc, + time::{Duration, SystemTime}, +}; use store::{ write::{now, BatchBuilder, QueueClass, ReportEvent, ValueClass}, - Deserialize, IterateParams, Key, Serialize, ValueKey, + Deserialize, IterateParams, Key, Serialize, Store, ValueKey, }; use tokio::sync::mpsc; -use crate::{ - core::{SmtpInstance, SMTP}, - queue::{manager::LONG_WAIT, spool::LOCK_EXPIRY}, -}; +use crate::queue::{manager::LONG_WAIT, spool::LOCK_EXPIRY}; -use super::{Event, ReportLock}; +use super::{dmarc::DmarcReporting, tls::TlsReporting, ReportLock}; -impl SpawnReport for mpsc::Receiver<Event> { - fn spawn(mut self, core: SmtpInstance) { +impl SpawnReport for mpsc::Receiver<ReportingEvent> { + fn spawn(mut self, inner: Arc<Inner>) { tokio::spawn(async move { - let mut last_cleanup = Instant::now(); let mut next_wake_up; loop { // Read events let now = now(); - let events = next_report_event(&core.core.load_full()).await; + let events = next_report_event(inner.shared_core.load().storage.data.clone()).await; next_wake_up = events .last() .and_then(|e| match e { @@ -44,15 +43,15 @@ impl SpawnReport for mpsc::Receiver<Event> { }) .unwrap_or(LONG_WAIT); - let core = SMTP::from(core.clone()); - let core_ = core.clone(); + let server = inner.build_server(); + let server_ = server.clone(); tokio::spawn(async move { let mut tls_reports = AHashMap::new(); for report_event in events { match report_event { QueueClass::DmarcReportHeader(event) if event.due <= now => { - if core_.try_lock_report(QueueClass::dmarc_lock(&event)).await { - core_.send_dmarc_aggregate_report(event).await; + if server.try_lock_report(QueueClass::dmarc_lock(&event)).await { + server.send_dmarc_aggregate_report(event).await; } } QueueClass::TlsReportHeader(event) if event.due <= now => { @@ -66,40 +65,34 @@ impl SpawnReport for mpsc::Receiver<Event> { } for (_, tls_report) in tls_reports { - if core_ + if server .try_lock_report(QueueClass::tls_lock(tls_report.first().unwrap())) .await { - core_.send_tls_aggregate_report(tls_report).await; + server.send_tls_aggregate_report(tls_report).await; } } }); match tokio::time::timeout(next_wake_up, self.recv()).await { Ok(Some(event)) => match event { - Event::Dmarc(event) => { - core.schedule_dmarc(event).await; + ReportingEvent::Dmarc(event) => { + server_.schedule_dmarc(event).await; } - Event::Tls(event) => { - core.schedule_tls(event).await; + ReportingEvent::Tls(event) => { + server_.schedule_tls(event).await; } - Event::Stop => break, + ReportingEvent::Stop => break, }, Ok(None) => break, - Err(_) => { - // Cleanup expired throttles - if last_cleanup.elapsed().as_secs() >= 86400 { - last_cleanup = Instant::now(); - core.cleanup(); - } - } + Err(_) => {} } } }); } } -async fn next_report_event(core: &Core) -> Vec<QueueClass> { +async fn next_report_event(store: Store) -> Vec<QueueClass> { let from_key = ValueKey::from(ValueClass::Queue(QueueClass::DmarcReportHeader( ReportEvent { due: 0, @@ -119,9 +112,7 @@ async fn next_report_event(core: &Core) -> Vec<QueueClass> { let mut events = Vec::new(); let now = now(); - let result = core - .storage - .data + let result = store .iterate( IterateParams::new(from_key, to_key).ascending().no_values(), |key, _| { @@ -150,13 +141,15 @@ async fn next_report_event(core: &Core) -> Vec<QueueClass> { events } -impl SMTP { - pub async fn try_lock_report(&self, lock: QueueClass) -> bool { +pub trait LockReport: Sync + Send { + fn try_lock_report(&self, lock: QueueClass) -> impl Future<Output = bool> + Send; +} + +impl LockReport for Server { + async fn try_lock_report(&self, lock: QueueClass) -> bool { let now = now(); match self - .core - .storage - .data + .store() .get_value::<u64>(ValueKey::from(ValueClass::Queue(lock.clone()))) .await { @@ -216,22 +209,6 @@ impl SMTP { } } -pub trait ToHash { - fn to_hash(&self) -> u64; -} - -impl ToHash for Dmarc { - fn to_hash(&self) -> u64 { - RandomState::with_seeds(1, 9, 7, 9).hash_one(self) - } -} - -impl ToHash for super::PolicyType { - fn to_hash(&self) -> u64 { - RandomState::with_seeds(1, 9, 7, 9).hash_one(self) - } -} - pub trait ToTimestamp { fn to_timestamp(&self) -> u64; } @@ -246,5 +223,5 @@ impl ToTimestamp for Duration { } pub trait SpawnReport { - fn spawn(self, core: SmtpInstance); + fn spawn(self, core: Arc<Inner>); } diff --git a/crates/smtp/src/reporting/spf.rs b/crates/smtp/src/reporting/spf.rs index 830a2906..8b7c82b7 100644 --- a/crates/smtp/src/reporting/spf.rs +++ b/crates/smtp/src/reporting/spf.rs @@ -9,7 +9,7 @@ use mail_auth::{report::AuthFailureType, AuthenticationResults, SpfOutput}; use trc::OutgoingReportEvent; use utils::config::Rate; -use crate::core::Session; +use crate::{core::Session, reporting::SmtpReporting}; impl<T: SessionStream> Session<T> { pub async fn send_spf_report( @@ -35,10 +35,9 @@ impl<T: SessionStream> Session<T> { } // Generate report - let config = &self.core.core.smtp.report.spf; + let config = &self.server.core.smtp.report.spf; let from_addr = self - .core - .core + .server .eval_if(&config.address, self, self.data.session_id) .await .unwrap_or_else(|| "MAILER-DAEMON@localhost".to_string()); @@ -64,8 +63,7 @@ impl<T: SessionStream> Session<T> { .with_spf_dns(format!("txt : {} : v=SPF1", output.domain())) // TODO use DNS record .write_rfc5322( ( - self.core - .core + self.server .eval_if(&config.name, self, self.data.session_id) .await .unwrap_or_else(|| "Mailer Daemon".to_string()) @@ -74,8 +72,7 @@ impl<T: SessionStream> Session<T> { ), rcpt, &self - .core - .core + .server .eval_if(&config.subject, self, self.data.session_id) .await .unwrap_or_else(|| "SPF Report".to_string()), @@ -91,7 +88,7 @@ impl<T: SessionStream> Session<T> { ); // Send report - self.core + self.server .send_report( &from_addr, [rcpt].into_iter(), diff --git a/crates/smtp/src/reporting/tls.rs b/crates/smtp/src/reporting/tls.rs index 5fa82a75..0430badb 100644 --- a/crates/smtp/src/reporting/tls.rs +++ b/crates/smtp/src/reporting/tls.rs @@ -4,7 +4,7 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use std::{collections::hash_map::Entry, sync::Arc, time::Duration}; +use std::{collections::hash_map::Entry, future::Future, sync::Arc, time::Duration}; use ahash::AHashMap; use common::{ @@ -12,7 +12,8 @@ use common::{ report::AggregateFrequency, resolver::{Mode, MxPattern}, }, - USER_AGENT, + ipc::{TlsEvent, ToHash}, + Server, USER_AGENT, }; use mail_auth::{ flate2::{write::GzEncoder, Compression}, @@ -31,9 +32,9 @@ use store::{ }; use trc::OutgoingReportEvent; -use crate::{core::SMTP, queue::RecipientDomain}; +use crate::{queue::RecipientDomain, reporting::SmtpReporting}; -use super::{scheduler::ToHash, AggregateTimestamp, ReportLock, SerializedSize, TlsEvent}; +use super::{AggregateTimestamp, ReportLock, SerializedSize}; #[derive(Debug, Clone)] pub struct TlsRptOptions { @@ -51,14 +52,30 @@ pub struct TlsFormat { #[cfg(feature = "test_mode")] pub static TLS_HTTP_REPORT: parking_lot::Mutex<Vec<u8>> = parking_lot::Mutex::new(Vec::new()); -impl SMTP { - pub async fn send_tls_aggregate_report(&self, events: Vec<ReportEvent>) { +pub trait TlsReporting: Sync + Send { + fn send_tls_aggregate_report( + &self, + events: Vec<ReportEvent>, + ) -> impl Future<Output = ()> + Send; + fn generate_tls_aggregate_report( + &self, + events: &[ReportEvent], + rua: &mut Vec<ReportUri>, + serialized_size: Option<&mut serde_json::Serializer<SerializedSize>>, + span_id: u64, + ) -> impl Future<Output = trc::Result<Option<TlsReport>>> + Send; + fn schedule_tls(&self, event: Box<TlsEvent>) -> impl Future<Output = ()> + Send; + fn delete_tls_report(&self, events: Vec<ReportEvent>) -> impl Future<Output = ()> + Send; +} + +impl TlsReporting for Server { + async fn send_tls_aggregate_report(&self, events: Vec<ReportEvent>) { let (domain_name, event_from, event_to) = events .first() .map(|e| (e.domain.as_str(), e.seq_id, e.due)) .unwrap(); - let span_id = self.inner.span_id_gen.generate().unwrap_or_else(now); + let span_id = self.inner.data.span_id_gen.generate().unwrap_or_else(now); trc::event!( OutgoingReport(OutgoingReportEvent::TlsAggregate), @@ -72,14 +89,13 @@ impl SMTP { // Generate report let mut rua = Vec::new(); let mut serialized_size = serde_json::Serializer::new(SerializedSize::new( - self.core - .eval_if( - &self.core.smtp.report.tls.max_size, - &RecipientDomain::new(domain_name), - span_id, - ) - .await - .unwrap_or(25 * 1024 * 1024), + self.eval_if( + &self.core.smtp.report.tls.max_size, + &RecipientDomain::new(domain_name), + span_id, + ) + .await + .unwrap_or(25 * 1024 * 1024), )); let report = match self .generate_tls_aggregate_report(&events, &mut rua, Some(&mut serialized_size), span_id) @@ -191,7 +207,6 @@ impl SMTP { if !rcpts.is_empty() { let config = &self.core.smtp.report.tls; let from_addr = self - .core .eval_if(&config.address, &RecipientDomain::new(domain_name), span_id) .await .unwrap_or_else(|| "MAILER-DAEMON@localhost".to_string()); @@ -199,7 +214,6 @@ impl SMTP { let _ = report.write_rfc5322_from_bytes( domain_name, &self - .core .eval_if( &self.core.smtp.report.submitter, &RecipientDomain::new(domain_name), @@ -208,8 +222,7 @@ impl SMTP { .await .unwrap_or_else(|| "localhost".to_string()), ( - self.core - .eval_if(&config.name, &RecipientDomain::new(domain_name), span_id) + self.eval_if(&config.name, &RecipientDomain::new(domain_name), span_id) .await .unwrap_or_else(|| "Mail Delivery Subsystem".to_string()) .as_str(), @@ -239,7 +252,7 @@ impl SMTP { self.delete_tls_report(events).await; } - pub async fn generate_tls_aggregate_report( + async fn generate_tls_aggregate_report( &self, events: &[ReportEvent], rua: &mut Vec<ReportUri>, @@ -253,7 +266,6 @@ impl SMTP { let config = &self.core.smtp.report.tls; let mut report = TlsReport { organization_name: self - .core .eval_if( &config.org_name, &RecipientDomain::new(domain_name), @@ -266,7 +278,6 @@ impl SMTP { end_datetime: DateTime::from_timestamp(event_to as i64), }, contact_info: self - .core .eval_if( &config.contact_info, &RecipientDomain::new(domain_name), @@ -388,7 +399,7 @@ impl SMTP { }) } - pub async fn schedule_tls(&self, event: Box<TlsEvent>) { + async fn schedule_tls(&self, event: Box<TlsEvent>) { let created = event.interval.to_timestamp(); let deliver_at = created + event.interval.as_secs(); let mut report_event = ReportEvent { @@ -420,7 +431,7 @@ impl SMTP { }; match event.policy { - super::PolicyType::Tlsa(tlsa) => { + common::ipc::PolicyType::Tlsa(tlsa) => { policy.policy_type = PolicyType::Tlsa; if let Some(tlsa) = tlsa { for entry in &tlsa.entries { @@ -440,7 +451,7 @@ impl SMTP { } } } - super::PolicyType::Sts(sts) => { + common::ipc::PolicyType::Sts(sts) => { policy.policy_type = PolicyType::Sts; if let Some(sts) = sts { policy.policy_string.push("version: STSv1".to_string()); @@ -489,7 +500,7 @@ impl SMTP { } // Write entry - report_event.seq_id = self.inner.queue_id_gen.generate().unwrap_or_else(now); + report_event.seq_id = self.inner.data.queue_id_gen.generate().unwrap_or_else(now); builder.set( ValueClass::Queue(QueueClass::TlsReportEvent(report_event)), Bincode::new(event.failure).serialize(), @@ -502,7 +513,7 @@ impl SMTP { } } - pub async fn delete_tls_report(&self, events: Vec<ReportEvent>) { + async fn delete_tls_report(&self, events: Vec<ReportEvent>) { let mut batch = BatchBuilder::new(); for (pos, event) in events.into_iter().enumerate() { diff --git a/crates/smtp/src/scripts/event_loop.rs b/crates/smtp/src/scripts/event_loop.rs index 8dcc1d15..da6885cd 100644 --- a/crates/smtp/src/scripts/event_loop.rs +++ b/crates/smtp/src/scripts/event_loop.rs @@ -4,9 +4,9 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use std::{borrow::Cow, sync::Arc, time::Instant}; +use std::{borrow::Cow, future::Future, sync::Arc, time::Instant}; -use common::scripts::plugins::PluginContext; +use common::{scripts::plugins::PluginContext, Server}; use mail_auth::common::headers::HeaderWriter; use sieve::{ compiler::grammar::actions::action_redirect::{ByMode, ByTime, Notify, NotifyItem, Ret}, @@ -19,15 +19,24 @@ use smtp_proto::{ use trc::SieveEvent; use crate::{ - core::SMTP, inbound::DkimSign, - queue::{DomainPart, MessageSource}, + queue::{quota::HasQueueQuota, spool::SmtpSpool, DomainPart, MessageSource}, }; use super::{ScriptModification, ScriptParameters, ScriptResult}; -impl SMTP { - pub async fn run_script( +pub trait RunScript: Sync + Send { + fn run_script( + &self, + script_id: String, + script: Arc<Sieve>, + params: ScriptParameters<'_>, + session_id: u64, + ) -> impl Future<Output = ScriptResult> + Send; +} + +impl RunScript for Server { + async fn run_script( &self, script_id: String, script: Arc<Sieve>, @@ -112,8 +121,7 @@ impl SMTP { id, PluginContext { session_id, - core: &self.core, - cache: &self.inner.script_cache, + server: self, message: instance.message(), modifications: &mut modifications, arguments, @@ -264,8 +272,7 @@ impl SMTP { let mut headers = Vec::new(); for dkim in ¶ms.sign { - if let Some(dkim) = self.core.get_dkim_signer(dkim, session_id) - { + if let Some(dkim) = self.get_dkim_signer(dkim, session_id) { match dkim.sign(raw_message) { Ok(signature) => { signature.write_header(&mut headers); diff --git a/crates/smtp/src/scripts/exec.rs b/crates/smtp/src/scripts/exec.rs index ec85c80b..7b92ccfa 100644 --- a/crates/smtp/src/scripts/exec.rs +++ b/crates/smtp/src/scripts/exec.rs @@ -13,7 +13,7 @@ use smtp_proto::*; use crate::{core::Session, inbound::AuthResult}; -use super::{ScriptParameters, ScriptResult}; +use super::{event_loop::RunScript, ScriptParameters, ScriptResult}; impl<T: SessionStream> Session<T> { pub fn build_script_parameters(&self, stage: &'static str) -> ScriptParameters<'_> { @@ -124,12 +124,12 @@ impl<T: SessionStream> Session<T> { script: Arc<Sieve>, params: ScriptParameters<'_>, ) -> ScriptResult { - self.core + self.server .run_script( script_id, script, params - .with_envelope(&self.core.core, self, self.data.session_id) + .with_envelope(&self.server, self, self.data.session_id) .await, self.data.session_id, ) diff --git a/crates/smtp/src/scripts/mod.rs b/crates/smtp/src/scripts/mod.rs index ee3c3ca7..cd162731 100644 --- a/crates/smtp/src/scripts/mod.rs +++ b/crates/smtp/src/scripts/mod.rs @@ -7,7 +7,7 @@ use std::borrow::Cow; use ahash::AHashMap; -use common::{expr::functions::ResolveVariable, scripts::ScriptModification, Core}; +use common::{expr::functions::ResolveVariable, scripts::ScriptModification, Server}; use sieve::{runtime::Variable, Envelope}; pub mod envelope; @@ -58,20 +58,23 @@ impl<'x> ScriptParameters<'x> { pub async fn with_envelope( mut self, - core: &Core, + server: &Server, vars: &impl ResolveVariable, session_id: u64, ) -> Self { for (variable, expr) in [ - (&mut self.from_addr, &core.sieve.from_addr), - (&mut self.from_name, &core.sieve.from_name), - (&mut self.return_path, &core.sieve.return_path), + (&mut self.from_addr, &server.core.sieve.from_addr), + (&mut self.from_name, &server.core.sieve.from_name), + (&mut self.return_path, &server.core.sieve.return_path), ] { - if let Some(value) = core.eval_if(expr, vars, session_id).await { + if let Some(value) = server.eval_if(expr, vars, session_id).await { *variable = value; } } - if let Some(value) = core.eval_if(&core.sieve.sign, vars, session_id).await { + if let Some(value) = server + .eval_if(&server.core.sieve.sign, vars, session_id) + .await + { self.sign = value; } self diff --git a/crates/utils/src/snowflake.rs b/crates/utils/src/snowflake.rs index a8619b89..99e8ecc7 100644 --- a/crates/utils/src/snowflake.rs +++ b/crates/utils/src/snowflake.rs @@ -87,3 +87,13 @@ impl Default for SnowflakeIdGenerator { Self::new() } } + +impl Clone for SnowflakeIdGenerator { + fn clone(&self) -> Self { + Self { + epoch: self.epoch, + node_id: self.node_id, + sequence: 0.into(), + } + } +} diff --git a/tests/src/directory/ldap.rs b/tests/src/directory/ldap.rs index ed3a6484..4948977d 100644 --- a/tests/src/directory/ldap.rs +++ b/tests/src/directory/ldap.rs @@ -25,7 +25,7 @@ async fn ldap_directory() { let mut config = DirectoryTest::new("sqlite".into()).await; let handle = config.directories.directories.remove("ldap").unwrap(); let base_store = config.stores.stores.get("sqlite").unwrap(); - let core = config.core; + let core = config.server; // Test authentication assert_eq!( diff --git a/tests/src/directory/mod.rs b/tests/src/directory/mod.rs index 7e34b2bc..ea37ca79 100644 --- a/tests/src/directory/mod.rs +++ b/tests/src/directory/mod.rs @@ -10,7 +10,7 @@ pub mod ldap; pub mod smtp; pub mod sql; -use common::{config::smtp::session::AddressMapping, Core}; +use common::{config::smtp::session::AddressMapping, Core, Server}; use directory::{ backend::internal::{manage::ManageDirectory, PrincipalField}, Directories, Principal, Type, @@ -254,7 +254,7 @@ pub struct DirectoryTest { pub directories: Directories, pub stores: Stores, pub temp_dir: TempDir, - pub core: Core, + pub server: Server, } #[derive(Debug, Default, Clone, PartialEq, Eq)] @@ -311,7 +311,10 @@ impl DirectoryTest { directories, stores, temp_dir, - core, + server: Server { + inner: Default::default(), + core: core.into(), + }, } } } @@ -598,7 +601,7 @@ async fn address_mappings() { let mut config = utils::config::Config::new(MAPPINGS).unwrap(); const ADDR: &str = "john.doe+alias@example.org"; const ADDR_NO_MATCH: &str = "jane@example.org"; - let core = Core::default(); + let core = Server::default(); for test in ["enable", "disable", "custom"] { let catch_all = AddressMapping::parse(&mut config, (test, "catch-all")); diff --git a/tests/src/directory/smtp.rs b/tests/src/directory/smtp.rs index 3b8b2440..cb5fdc72 100644 --- a/tests/src/directory/smtp.rs +++ b/tests/src/directory/smtp.rs @@ -30,7 +30,7 @@ async fn lmtp_directory() { // Obtain directory handle let mut config = DirectoryTest::new(None).await; let handle = config.directories.directories.remove("smtp").unwrap(); - let core = config.core; + let core = config.server; // Basic lookup let tests = vec![ diff --git a/tests/src/directory/sql.rs b/tests/src/directory/sql.rs index 20f2c6eb..12da895a 100644 --- a/tests/src/directory/sql.rs +++ b/tests/src/directory/sql.rs @@ -33,7 +33,7 @@ async fn sql_directory() { store: config.stores.lookup_stores.remove(directory_id).unwrap(), }; let base_store = config.stores.stores.get(directory_id).unwrap(); - let core = config.core; + let core = config.server; // Create tables store.create_test_directory().await; diff --git a/tests/src/imap/append.rs b/tests/src/imap/append.rs index 1a4c60d0..4b45511d 100644 --- a/tests/src/imap/append.rs +++ b/tests/src/imap/append.rs @@ -64,7 +64,7 @@ pub async fn test(imap: &mut ImapConnection, _imap_check: &mut ImapConnection, h expected_uid += 1; } - wait_for_index(&handle.jmap).await; + wait_for_index(&handle.server).await; } pub async fn assert_append_message( diff --git a/tests/src/imap/mod.rs b/tests/src/imap/mod.rs index 0dcfe9b3..584228f8 100644 --- a/tests/src/imap/mod.rs +++ b/tests/src/imap/mod.rs @@ -28,23 +28,25 @@ use std::{ use ::managesieve::core::ManageSieveSessionManager; use common::{ config::{ - server::{ServerProtocol, Servers}, + server::{Listeners, ServerProtocol}, telemetry::Telemetry, }, - Core, Ipc, IPC_CHANNEL_BUFFER, + core::BuildServer, + manager::boot::build_ipc, + Core, Data, Inner, Server, }; use ::store::Stores; use ahash::AHashSet; -use imap::core::{ImapSessionManager, Inner, IMAP}; +use imap::core::ImapSessionManager; use imap_proto::ResponseType; -use jmap::{api::JmapSessionManager, JMAP}; +use jmap::{api::JmapSessionManager, SpawnServices}; use pop3::Pop3SessionManager; -use smtp::core::{SmtpSessionManager, SMTP}; +use smtp::{core::SmtpSessionManager, SpawnQueueManager}; use tokio::{ io::{AsyncBufReadExt, AsyncWriteExt, BufReader, Lines, ReadHalf, WriteHalf}, net::TcpStream, - sync::{mpsc, watch}, + sync::watch, }; use utils::config::Config; @@ -279,8 +281,7 @@ disabled-events = ["network.*"] #[allow(dead_code)] pub struct IMAPTest { - jmap: Arc<JMAP>, - imap: Arc<Inner>, + server: Server, temp_dir: TempDir, shutdown_tx: watch::Sender<bool>, } @@ -301,7 +302,7 @@ async fn init_imap_tests(store_id: &str, delete_if_exists: bool) -> IMAPTest { config.resolve_all_macros().await; // Parse servers - let mut servers = Servers::parse(&mut config); + let mut servers = Listeners::parse(&mut config); // Bind ports and drop privileges servers.bind_and_drop_priv(&mut config); @@ -312,67 +313,56 @@ async fn init_imap_tests(store_id: &str, delete_if_exists: bool) -> IMAPTest { // Parse core let tracers = Telemetry::parse(&mut config, &stores); let core = Core::parse(&mut config, stores, Default::default()).await; + let data = Data::parse(&mut config); let store = core.storage.data.clone(); - let shared_core = core.into_shared(); + let (ipc, mut ipc_rxs) = build_ipc(); + let inner = Arc::new(Inner { + shared_core: core.into_shared(), + data, + ipc, + }); // Parse acceptors - servers.parse_tcp_acceptors(&mut config, shared_core.clone()); + servers.parse_tcp_acceptors(&mut config, inner.clone()); // Enable tracing tracers.enable(true); - // Setup IPC channels - let (delivery_tx, delivery_rx) = mpsc::channel(IPC_CHANNEL_BUFFER); - let ipc = Ipc { delivery_tx }; - - // Init servers - let smtp = SMTP::init( - &mut config, - shared_core.clone(), - ipc, - servers.span_id_gen.clone(), - ) - .await; - let jmap = JMAP::init( - &mut config, - delivery_rx, - shared_core.clone(), - smtp.inner.clone(), - ) - .await; - let imap = IMAP::init(&mut config, jmap.clone()).await; + // Start services config.assert_no_errors(); + ipc_rxs.spawn_queue_manager(inner.clone()); + ipc_rxs.spawn_services(inner.clone()); // Spawn servers let (shutdown_tx, _) = servers.spawn(|server, acceptor, shutdown_rx| { match &server.protocol { ServerProtocol::Smtp | ServerProtocol::Lmtp => server.spawn( - SmtpSessionManager::new(smtp.clone()), - shared_core.clone(), + SmtpSessionManager::new(inner.clone()), + inner.clone(), acceptor, shutdown_rx, ), ServerProtocol::Http => server.spawn( - JmapSessionManager::new(jmap.clone()), - shared_core.clone(), + JmapSessionManager::new(inner.clone()), + inner.clone(), acceptor, shutdown_rx, ), ServerProtocol::Imap => server.spawn( - ImapSessionManager::new(imap.clone()), - shared_core.clone(), + ImapSessionManager::new(inner.clone()), + inner.clone(), acceptor, shutdown_rx, ), ServerProtocol::Pop3 => server.spawn( - Pop3SessionManager::new(imap.clone()), - shared_core.clone(), + Pop3SessionManager::new(inner.clone()), + inner.clone(), acceptor, shutdown_rx, ), ServerProtocol::ManageSieve => server.spawn( - ManageSieveSessionManager::new(imap.clone()), - shared_core.clone(), + ManageSieveSessionManager::new(inner.clone()), + inner.clone(), acceptor, shutdown_rx, ), @@ -431,8 +421,7 @@ async fn init_imap_tests(store_id: &str, delete_if_exists: bool) -> IMAPTest { .await; IMAPTest { - jmap: JMAP::from(jmap.clone()).into(), - imap: imap.imap_inner, + server: inner.build_server(), temp_dir, shutdown_tx, } diff --git a/tests/src/imap/store.rs b/tests/src/imap/store.rs index 31e0f73b..9f9c6b4e 100644 --- a/tests/src/imap/store.rs +++ b/tests/src/imap/store.rs @@ -60,7 +60,7 @@ pub async fn test(imap: &mut ImapConnection, _imap_check: &mut ImapConnection, h .assert_contains("UIDNEXT 11"); // Store using saved searches - wait_for_index(&handle.jmap).await; + wait_for_index(&handle.server).await; imap.send("SEARCH RETURN (SAVE) FROM nathaniel").await; imap.assert_read(Type::Tagged, ResponseType::Ok).await; imap.send("UID STORE $ +FLAGS (\\Answered)").await; diff --git a/tests/src/jmap/auth_acl.rs b/tests/src/jmap/auth_acl.rs index 23a7d977..59346287 100644 --- a/tests/src/jmap/auth_acl.rs +++ b/tests/src/jmap/auth_acl.rs @@ -670,7 +670,7 @@ pub async fn test(params: &mut JMAPTest) { .add_to_group(name, "sales@example.com") .await; } - server.core.security.access_tokens.clear(); + server.inner.data.access_tokens.clear(); john_client.refresh_session().await.unwrap(); jane_client.refresh_session().await.unwrap(); bill_client.refresh_session().await.unwrap(); @@ -770,7 +770,7 @@ pub async fn test(params: &mut JMAPTest) { .data .remove_from_group("jdoe@example.com", "sales@example.com") .await; - server.inner.sessions.clear(); + server.inner.data.http_auth_cache.clear(); assert_forbidden( john_client .set_default_account_id(sales_id.to_string()) diff --git a/tests/src/jmap/auth_limits.rs b/tests/src/jmap/auth_limits.rs index a9b5ad3c..a5028b15 100644 --- a/tests/src/jmap/auth_limits.rs +++ b/tests/src/jmap/auth_limits.rs @@ -49,7 +49,7 @@ pub async fn test(params: &mut JMAPTest) { .to_string(); // Reset rate limiters - server.inner.concurrency_limiter.clear(); + server.inner.data.jmap_limiter.clear(); params.webhook.clear(); // Incorrect passwords should be rejected with a 401 error @@ -149,10 +149,9 @@ pub async fn test(params: &mut JMAPTest) { .await .unwrap(); server - .core - .network + .inner + .data .blocked_ips - .ip_addresses .write() .remove(&IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1))); diff --git a/tests/src/jmap/delivery.rs b/tests/src/jmap/delivery.rs index 1b238170..18710268 100644 --- a/tests/src/jmap/delivery.rs +++ b/tests/src/jmap/delivery.rs @@ -6,7 +6,10 @@ use std::time::Duration; -use jmap::mailbox::{INBOX_ID, JUNK_ID}; +use jmap::{ + mailbox::{INBOX_ID, JUNK_ID}, + JmapMethods, +}; use jmap_proto::types::{collection::Collection, id::Id, property::Property}; use tokio::{ diff --git a/tests/src/jmap/email_query.rs b/tests/src/jmap/email_query.rs index 4e982d6c..96092f37 100644 --- a/tests/src/jmap/email_query.rs +++ b/tests/src/jmap/email_query.rs @@ -10,6 +10,7 @@ use crate::{ jmap::{assert_is_empty, mailbox::destroy_all_mailboxes, wait_for_index}, store::{deflate_test_resource, query::FIELDS}, }; +use jmap::JmapMethods; use jmap_client::{ client::Client, core::query::{Comparator, Filter}, diff --git a/tests/src/jmap/email_query_changes.rs b/tests/src/jmap/email_query_changes.rs index ab286237..a40e5b48 100644 --- a/tests/src/jmap/email_query_changes.rs +++ b/tests/src/jmap/email_query_changes.rs @@ -4,6 +4,7 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ +use jmap::{changes::write::ChangeLog, JmapMethods}; use jmap_client::{ core::query::{Comparator, Filter}, email, diff --git a/tests/src/jmap/enterprise.rs b/tests/src/jmap/enterprise.rs index ddf8d809..d81eccd8 100644 --- a/tests/src/jmap/enterprise.rs +++ b/tests/src/jmap/enterprise.rs @@ -12,6 +12,7 @@ use std::{sync::Arc, time::Duration}; use common::{ config::telemetry::{StoreTracer, TelemetrySubscriberType}, + core::BuildServer, enterprise::{ config::parse_metric_alerts, license::LicenseKey, undelete::DeletedBlob, Enterprise, MetricStore, TraceStore, Undelete, @@ -20,7 +21,7 @@ use common::{ metrics::store::{Metric, MetricsStore, SharedMetricHistory}, tracers::store::{TracingQuery, TracingStore}, }, - Core, + Core, Server, }; use imap_proto::ResponseType; use jmap::api::management::enterprise::undelete::{UndeleteRequest, UndeleteResponse}; @@ -79,7 +80,7 @@ test pub async fn test(params: &mut JMAPTest) { // Enable Enterprise - let mut core = params.server.shared_core.load_full().as_ref().clone(); + let mut core = params.server.inner.shared_core.load_full().as_ref().clone(); let mut config = Config::new(METRICS_CONFIG).unwrap(); core.enterprise = Enterprise { license: LicenseKey { @@ -109,12 +110,18 @@ pub async fn test(params: &mut JMAPTest) { .into(); config.assert_no_errors(); assert_ne!(core.enterprise.as_ref().unwrap().metrics_alerts.len(), 0); - params.server.shared_core.store(core.into()); - assert!(params.server.shared_core.load().is_enterprise_edition()); + params.server.inner.shared_core.store(core.into()); + assert!(params + .server + .inner + .shared_core + .load() + .is_enterprise_edition()); // Create test account params .server + .inner .shared_core .load() .storage @@ -127,14 +134,15 @@ pub async fn test(params: &mut JMAPTest) { ) .await; - alerts(¶ms.server.shared_core.load()).await; + alerts(¶ms.server.inner.build_server()).await; undelete(params).await; tracing(params).await; metrics(params).await; - params.server.shared_core.store( + params.server.inner.shared_core.store( params .server + .inner .shared_core .load_full() .as_ref() @@ -168,7 +176,7 @@ impl EnterpriseCore for Core { } } -async fn alerts(core: &Core) { +async fn alerts(server: &Server) { // Make sure the required metrics are set to 0 assert_eq!( Collector::read_event_metric(EventType::Cluster(ClusterEvent::Error).id()), @@ -192,7 +200,7 @@ async fn alerts(core: &Core) { assert_eq!(Collector::read_metric(MetricType::DomainCount), 3.0); // Process alerts - let message = core.process_alerts().await.unwrap().pop().unwrap(); + let message = server.process_alerts().await.unwrap().pop().unwrap(); assert_eq!(message.from, "alert@example.com"); assert_eq!(message.to, vec!["jdoe@example.com".to_string()]); let body = String::from_utf8(message.body).unwrap(); diff --git a/tests/src/jmap/mod.rs b/tests/src/jmap/mod.rs index 76bd961d..e8805993 100644 --- a/tests/src/jmap/mod.rs +++ b/tests/src/jmap/mod.rs @@ -18,30 +18,34 @@ use base64::{ use common::{ auth::AccessToken, config::{ - server::{ServerProtocol, Servers}, + server::{Listeners, ServerProtocol}, telemetry::Telemetry, }, - manager::config::{ConfigManager, Patterns}, - Core, Ipc, IPC_CHANNEL_BUFFER, + core::BuildServer, + manager::{ + boot::build_ipc, + config::{ConfigManager, Patterns}, + }, + Core, Data, Inner, Server, }; use enterprise::{insert_test_metrics, EnterpriseCore}; use hyper::{header::AUTHORIZATION, Method}; -use imap::core::{ImapSessionManager, IMAP}; -use jmap::{api::JmapSessionManager, JMAP}; +use imap::core::ImapSessionManager; +use jmap::{api::JmapSessionManager, email::delete::EmailDeletion, SpawnServices}; use jmap_client::client::{Client, Credentials}; use jmap_proto::{error::request::RequestError, types::id::Id}; use managesieve::core::ManageSieveSessionManager; use pop3::Pop3SessionManager; use reqwest::header; use serde::{de::DeserializeOwned, Deserialize, Serialize}; -use smtp::core::{SmtpSessionManager, SMTP}; +use smtp::{core::SmtpSessionManager, SpawnQueueManager}; use store::{ roaring::RoaringBitmap, write::{key::DeserializeBigEndian, AnyKey, FtsQueueClass, ValueClass}, IterateParams, Stores, ValueKey, SUBSPACE_PROPERTY, }; -use tokio::sync::{mpsc, watch}; +use tokio::sync::watch; use utils::{config::Config, map::ttl_dashmap::TtlMap, BlobHash}; use webhooks::{spawn_mock_webhook_endpoint, MockWebhookEndpoint}; @@ -311,7 +315,7 @@ pub async fn jmap_tests() { ) .await; - webhooks::test(&mut params).await; + /*webhooks::test(&mut params).await; email_query::test(&mut params, delete).await; email_get::test(&mut params).await; email_set::test(&mut params).await; @@ -335,7 +339,7 @@ pub async fn jmap_tests() { websocket::test(&mut params).await; quota::test(&mut params).await; crypto::test(&mut params).await; - blob::test(&mut params).await; + blob::test(&mut params).await;*/ permissions::test(¶ms).await; purge::test(&mut params).await; enterprise::test(&mut params).await; @@ -373,14 +377,14 @@ pub async fn jmap_metric_tests() { #[allow(dead_code)] pub struct JMAPTest { - server: Arc<JMAP>, + server: Server, client: Client, temp_dir: TempDir, webhook: Arc<MockWebhookEndpoint>, shutdown_tx: watch::Sender<bool>, } -pub async fn wait_for_index(server: &JMAP) { +pub async fn wait_for_index(server: &Server) { loop { let mut has_index_tasks = false; server @@ -426,7 +430,7 @@ pub async fn wait_for_index(server: &JMAP) { } } -pub async fn assert_is_empty(server: Arc<JMAP>) { +pub async fn assert_is_empty(server: Server) { // Wait for pending FTS index tasks wait_for_index(&server).await; @@ -442,7 +446,7 @@ pub async fn assert_is_empty(server: Arc<JMAP>) { .await; } -pub async fn emails_purge_tombstoned(server: &JMAP) { +pub async fn emails_purge_tombstoned(server: &Server) { let mut account_ids = RoaringBitmap::new(); server .core @@ -471,14 +475,14 @@ pub async fn emails_purge_tombstoned(server: &JMAP) { for account_id in account_ids { let do_add = server - .core - .security + .inner + .data .access_tokens .get_with_ttl(&account_id) .is_none(); if do_add { - server.core.security.access_tokens.insert_with_ttl( + server.inner.data.access_tokens.insert_with_ttl( account_id, Arc::new(AccessToken::from_id(account_id)), Instant::now() + Duration::from_secs(3600), @@ -486,7 +490,7 @@ pub async fn emails_purge_tombstoned(server: &JMAP) { } server.emails_purge_tombstoned(account_id).await.unwrap(); if do_add { - server.core.security.access_tokens.remove(&account_id); + server.inner.data.access_tokens.remove(&account_id); } } } @@ -507,7 +511,7 @@ async fn init_jmap_tests(store_id: &str, delete_if_exists: bool) -> JMAPTest { config.resolve_all_macros().await; // Parse servers - let mut servers = Servers::parse(&mut config); + let mut servers = Listeners::parse(&mut config); // Bind ports and drop privileges servers.bind_and_drop_priv(&mut config); @@ -530,67 +534,56 @@ async fn init_jmap_tests(store_id: &str, delete_if_exists: bool) -> JMAPTest { let core = Core::parse(&mut config, stores, config_manager) .await .enable_enterprise(); + let data = Data::parse(&mut config); let store = core.storage.data.clone(); - let shared_core = core.into_shared(); + let (ipc, mut ipc_rxs) = build_ipc(); + let inner = Arc::new(Inner { + shared_core: core.into_shared(), + data, + ipc, + }); // Parse acceptors - servers.parse_tcp_acceptors(&mut config, shared_core.clone()); + servers.parse_tcp_acceptors(&mut config, inner.clone()); // Enable tracing tracers.enable(true); - // Setup IPC channels - let (delivery_tx, delivery_rx) = mpsc::channel(IPC_CHANNEL_BUFFER); - let ipc = Ipc { delivery_tx }; - - // Init servers - let smtp = SMTP::init( - &mut config, - shared_core.clone(), - ipc, - servers.span_id_gen.clone(), - ) - .await; - let jmap = JMAP::init( - &mut config, - delivery_rx, - shared_core.clone(), - smtp.inner.clone(), - ) - .await; - let imap = IMAP::init(&mut config, jmap.clone()).await; + // Start services config.assert_no_errors(); + ipc_rxs.spawn_queue_manager(inner.clone()); + ipc_rxs.spawn_services(inner.clone()); // Spawn servers let (shutdown_tx, _) = servers.spawn(|server, acceptor, shutdown_rx| { match &server.protocol { ServerProtocol::Smtp | ServerProtocol::Lmtp => server.spawn( - SmtpSessionManager::new(smtp.clone()), - shared_core.clone(), + SmtpSessionManager::new(inner.clone()), + inner.clone(), acceptor, shutdown_rx, ), ServerProtocol::Http => server.spawn( - JmapSessionManager::new(jmap.clone()), - shared_core.clone(), + JmapSessionManager::new(inner.clone()), + inner.clone(), acceptor, shutdown_rx, ), ServerProtocol::Imap => server.spawn( - ImapSessionManager::new(imap.clone()), - shared_core.clone(), + ImapSessionManager::new(inner.clone()), + inner.clone(), acceptor, shutdown_rx, ), ServerProtocol::Pop3 => server.spawn( - Pop3SessionManager::new(imap.clone()), - shared_core.clone(), + Pop3SessionManager::new(inner.clone()), + inner.clone(), acceptor, shutdown_rx, ), ServerProtocol::ManageSieve => server.spawn( - ManageSieveSessionManager::new(imap.clone()), - shared_core.clone(), + ManageSieveSessionManager::new(inner.clone()), + inner.clone(), acceptor, shutdown_rx, ), @@ -602,7 +595,8 @@ async fn init_jmap_tests(store_id: &str, delete_if_exists: bool) -> JMAPTest { } // Create tables - shared_core + inner + .shared_core .load() .storage .data @@ -620,7 +614,7 @@ async fn init_jmap_tests(store_id: &str, delete_if_exists: bool) -> JMAPTest { client.set_default_account_id(Id::new(1)); JMAPTest { - server: JMAP::from(jmap).into(), + server: inner.build_server(), temp_dir, client, shutdown_tx, diff --git a/tests/src/jmap/permissions.rs b/tests/src/jmap/permissions.rs index 37eb1e64..987901a9 100644 --- a/tests/src/jmap/permissions.rs +++ b/tests/src/jmap/permissions.rs @@ -7,12 +7,13 @@ use ahash::AHashSet; use common::{ auth::{AccessToken, TenantInfo}, - DeliveryResult, IngestMessage, + ipc::{DeliveryResult, IngestMessage}, }; use directory::{ backend::internal::{PrincipalField, PrincipalUpdate, PrincipalValue}, Permission, Principal, Type, }; +use jmap::{services::ingest::MailDelivery, JmapMethods}; use utils::BlobHash; use crate::jmap::assert_is_empty; @@ -20,7 +21,7 @@ use crate::jmap::assert_is_empty; use super::{enterprise::List, JMAPTest, ManagementApi}; pub async fn test(params: &JMAPTest) { - let core = params.server.core.clone(); + println!("Running permissions tests..."); let server = params.server.clone(); // Prepare management API @@ -41,7 +42,8 @@ pub async fn test(params: &JMAPTest) { .await .unwrap() .unwrap_data(); - core.get_access_token(account_id) + server + .get_access_token(account_id) .await .unwrap() .validate_permissions( @@ -122,7 +124,8 @@ pub async fn test(params: &JMAPTest) { .await .unwrap() .unwrap_data(); - core.get_access_token(account_id) + server + .get_access_token(account_id) .await .unwrap() .validate_permissions([ @@ -330,7 +333,8 @@ pub async fn test(params: &JMAPTest) { .unwrap_data(); // Verify permissions - core.get_access_token(tenant_admin_id) + server + .get_access_token(tenant_admin_id) .await .unwrap() .validate_permissions(Permission::all().filter(|p| p.is_tenant_admin_permission())) @@ -425,7 +429,8 @@ pub async fn test(params: &JMAPTest) { .unwrap_data(); // Although super user privileges were used and a different tenant name was provided, this should be ignored - core.get_access_token(tenant_user_id) + server + .get_access_token(tenant_user_id) .await .unwrap() .validate_permissions( @@ -487,7 +492,8 @@ pub async fn test(params: &JMAPTest) { .unwrap_data(); // Check updated permissions - core.get_access_token(tenant_user_id) + server + .get_access_token(tenant_user_id) .await .unwrap() .validate_permissions(Permission::all().filter(|p| { @@ -588,8 +594,8 @@ pub async fn test(params: &JMAPTest) { // John should not be allowed to receive email let message_blob = BlobHash::from(TEST_MESSAGE.as_bytes()); - core.storage - .blob + server + .blob_store() .put_blob(message_blob.as_ref(), TEST_MESSAGE.as_bytes()) .await .unwrap(); @@ -621,7 +627,8 @@ pub async fn test(params: &JMAPTest) { .await .unwrap() .unwrap_data(); - core.get_access_token(tenant_user_id) + server + .get_access_token(tenant_user_id) .await .unwrap() .validate_permissions( diff --git a/tests/src/jmap/purge.rs b/tests/src/jmap/purge.rs index 3849ee73..c7489f64 100644 --- a/tests/src/jmap/purge.rs +++ b/tests/src/jmap/purge.rs @@ -5,11 +5,13 @@ */ use ahash::AHashSet; +use common::Server; use directory::{backend::internal::manage::ManageDirectory, QueryBy}; use imap_proto::ResponseType; use jmap::{ + email::delete::EmailDeletion, mailbox::{INBOX_ID, JUNK_ID, TRASH_ID}, - JMAP, + JmapMethods, }; use jmap_proto::types::{collection::Collection, id::Id, property::Property}; use store::{ @@ -205,7 +207,7 @@ pub async fn test(params: &mut JMAPTest) { assert_is_empty(server).await; } -async fn get_changes(server: &JMAP) -> AHashSet<(u64, u8)> { +async fn get_changes(server: &Server) -> AHashSet<(u64, u8)> { let mut changes = AHashSet::new(); server .core diff --git a/tests/src/jmap/push_subscription.rs b/tests/src/jmap/push_subscription.rs index 442e9172..a846a722 100644 --- a/tests/src/jmap/push_subscription.rs +++ b/tests/src/jmap/push_subscription.rs @@ -13,7 +13,7 @@ use std::{ }; use base64::{engine::general_purpose, Engine}; -use common::{config::server::Servers, listener::SessionData, Core}; +use common::{config::server::Listeners, listener::SessionData, Core, Data, Inner}; use ece::EcKeyComponents; use hyper::{body, header::CONTENT_ENCODING, server::conn::http1, service::service_fn, StatusCode}; use hyper_util::rt::TokioIo; @@ -99,20 +99,28 @@ pub async fn test(params: &mut JMAPTest) { // Start mock push server let mut settings = Config::new(add_test_certs(SERVER)).unwrap(); settings.resolve_all_macros().await; - let mock_core = Core::parse(&mut settings, Default::default(), Default::default()) - .await - .into_shared(); + let mock_inner = Arc::new(Inner { + shared_core: Core::parse(&mut settings, Default::default(), Default::default()) + .await + .into_shared(), + data: Data::parse(&mut settings), + ..Default::default() + }); settings.errors.clear(); settings.warnings.clear(); - let mut servers = Servers::parse(&mut settings); - servers.parse_tcp_acceptors(&mut settings, mock_core.clone()); + let mut servers = Listeners::parse(&mut settings); + servers.parse_tcp_acceptors(&mut settings, mock_inner.clone()); // Start JMAP server - let manager = SessionManager::from(push_server.clone()); servers.bind_and_drop_priv(&mut settings); settings.assert_no_errors(); let _shutdown_tx = servers.spawn(|server, acceptor, shutdown_rx| { - server.spawn(manager.clone(), mock_core.clone(), acceptor, shutdown_rx); + server.spawn( + SessionManager::from(push_server.clone()), + mock_inner.clone(), + acceptor, + shutdown_rx, + ); }); // Register push notification (no encryption) diff --git a/tests/src/jmap/quota.rs b/tests/src/jmap/quota.rs index c186c90f..7e7dffb1 100644 --- a/tests/src/jmap/quota.rs +++ b/tests/src/jmap/quota.rs @@ -11,12 +11,13 @@ use crate::{ mailbox::destroy_all_mailboxes, test_account_login, }, }; -use jmap::{blob::upload::DISABLE_UPLOAD_QUOTA, mailbox::INBOX_ID}; +use jmap::{blob::upload::DISABLE_UPLOAD_QUOTA, mailbox::INBOX_ID, JmapMethods}; use jmap_client::{ core::set::{SetErrorType, SetObject}, email::EmailBodyPart, }; use jmap_proto::types::{collection::Collection, id::Id}; +use smtp::queue::spool::SmtpSpool; use super::JMAPTest; @@ -338,13 +339,12 @@ pub async fn test(params: &mut JMAPTest) { params.client.set_default_account_id(account_id.to_string()); destroy_all_mailboxes(params).await; } - for event in server.smtp.next_event().await { + for event in server.next_event().await { server - .smtp .read_message(event.queue_id) .await .unwrap() - .remove(&server.smtp, event.due) + .remove(&server, event.due) .await; } assert_is_empty(server).await; diff --git a/tests/src/jmap/stress_test.rs b/tests/src/jmap/stress_test.rs index d6e5ef28..466eced5 100644 --- a/tests/src/jmap/stress_test.rs +++ b/tests/src/jmap/stress_test.rs @@ -7,9 +7,10 @@ use std::{sync::Arc, time::Duration}; use crate::jmap::{mailbox::destroy_all_mailboxes_no_wait, wait_for_index}; +use common::Server; use directory::backend::internal::manage::ManageDirectory; use futures::future::join_all; -use jmap::{mailbox::UidMailbox, JMAP}; +use jmap::{mailbox::UidMailbox, JmapMethods}; use jmap_client::{ client::Client, core::set::{SetErrorType, SetObject}, @@ -23,7 +24,7 @@ use super::assert_is_empty; const TEST_USER_ID: u32 = 1; const NUM_PASSES: usize = 1; -pub async fn test(server: Arc<JMAP>, mut client: Client) { +pub async fn test(server: Server, mut client: Client) { println!("Running concurrency stress tests..."); server .core @@ -38,7 +39,7 @@ pub async fn test(server: Arc<JMAP>, mut client: Client) { mailbox_tests(server.clone(), client.clone()).await; } -async fn email_tests(server: Arc<JMAP>, client: Arc<Client>) { +async fn email_tests(server: Server, client: Arc<Client>) { for pass in 0..NUM_PASSES { println!( "----------------- EMAIL STRESS TEST {} -----------------", @@ -270,7 +271,7 @@ async fn email_tests(server: Arc<JMAP>, client: Arc<Client>) { } } -async fn mailbox_tests(server: Arc<JMAP>, client: Arc<Client>) { +async fn mailbox_tests(server: Server, client: Arc<Client>) { let mailboxes = Arc::new(vec![ "test/test1/test2/test3".to_string(), "test1/test2/test3".to_string(), diff --git a/tests/src/jmap/thread_merge.rs b/tests/src/jmap/thread_merge.rs index 5a439c0d..56322061 100644 --- a/tests/src/jmap/thread_merge.rs +++ b/tests/src/jmap/thread_merge.rs @@ -11,7 +11,10 @@ use crate::{ store::deflate_test_resource, }; use common::auth::AccessToken; -use jmap::email::ingest::{IngestEmail, IngestSource}; +use jmap::{ + email::ingest::{EmailIngest, IngestEmail, IngestSource}, + JmapMethods, +}; use jmap_client::{email, mailbox::Role}; use jmap_proto::types::{collection::Collection, id::Id}; use mail_parser::{mailbox::mbox::MessageIterator, MessageParser}; diff --git a/tests/src/smtp/config.rs b/tests/src/smtp/config.rs index bf578be4..7811d6f8 100644 --- a/tests/src/smtp/config.rs +++ b/tests/src/smtp/config.rs @@ -8,11 +8,11 @@ use std::{fs, net::IpAddr, path::PathBuf, sync::Arc, time::Duration}; use common::{ config::{ - server::{Listener, Server, ServerProtocol, Servers}, + server::{Listener, Listeners, ServerProtocol, TcpListener}, smtp::{throttle::parse_throttle, *}, }, expr::{functions::ResolveVariable, if_block::*, tokenizer::TokenMap, *}, - Core, + Server, }; use tokio::net::TcpSocket; @@ -301,13 +301,13 @@ fn parse_servers() { // Parse servers let mut config = Config::new(toml).unwrap(); - let servers = Servers::parse(&mut config).servers; + let servers = Listeners::parse(&mut config).servers; let id_generator = Arc::new(utils::snowflake::SnowflakeIdGenerator::new()); let expected_servers = vec![ - Server { + Listener { id: "smtp".to_string(), protocol: ServerProtocol::Smtp, - listeners: vec![Listener { + listeners: vec![TcpListener { socket: TcpSocket::new_v4().unwrap(), addr: "127.0.0.1:9925".parse().unwrap(), ttl: 3600.into(), @@ -319,11 +319,11 @@ fn parse_servers() { proxy_networks: vec![], span_id_gen: id_generator.clone(), }, - Server { + Listener { id: "smtps".to_string(), protocol: ServerProtocol::Smtp, listeners: vec![ - Listener { + TcpListener { socket: TcpSocket::new_v4().unwrap(), addr: "127.0.0.1:9465".parse().unwrap(), ttl: 4096.into(), @@ -331,7 +331,7 @@ fn parse_servers() { linger: None, nodelay: true, }, - Listener { + TcpListener { socket: TcpSocket::new_v4().unwrap(), addr: "127.0.0.1:9466".parse().unwrap(), ttl: 4096.into(), @@ -344,10 +344,10 @@ fn parse_servers() { proxy_networks: vec![], span_id_gen: id_generator.clone(), }, - Server { + Listener { id: "submission".to_string(), protocol: ServerProtocol::Smtp, - listeners: vec![Listener { + listeners: vec![TcpListener { socket: TcpSocket::new_v4().unwrap(), addr: "127.0.0.1:9991".parse().unwrap(), ttl: 3600.into(), @@ -416,7 +416,7 @@ async fn eval_if() { V_PRIORITY, V_MX, ]); - let core = Core::default(); + let core = Server::default(); for (key, _) in config.keys.clone() { if !key.starts_with("rule.") { @@ -466,7 +466,7 @@ async fn eval_dynvalue() { V_PRIORITY, V_MX, ]); - let core = Core::default(); + let core = Server::default(); for test_name in config .sub_keys("eval", "") diff --git a/tests/src/smtp/inbound/antispam.rs b/tests/src/smtp/inbound/antispam.rs index 1c3e5eb9..524ae137 100644 --- a/tests/src/smtp/inbound/antispam.rs +++ b/tests/src/smtp/inbound/antispam.rs @@ -17,14 +17,14 @@ use common::{ use mail_auth::{dmarc::Policy, DkimResult, DmarcResult, IprevResult, SpfResult, MX}; use sieve::runtime::Variable; use smtp::{ - core::{Inner, Session, SessionAddress}, + core::{Session, SessionAddress}, inbound::AuthResult, - scripts::ScriptResult, + scripts::{event_loop::RunScript, ScriptResult}, }; use store::Stores; use utils::config::Config; -use crate::smtp::{build_smtp, session::TestSession, TempDir}; +use crate::smtp::{session::TestSession, TempDir, TestSMTP}; const CONFIG: &str = r#" [spam.header] @@ -248,7 +248,7 @@ async fn antispam() { ); } - let core = build_smtp(core, Inner::default()); + let server = TestSMTP::from_core(core).server; // Run tests let base_path = PathBuf::from(env!("CARGO_MANIFEST_DIR")) @@ -260,7 +260,7 @@ async fn antispam() { continue; }*/ println!("===== {test_name} ====="); - let script = core + let script = server .core .sieve .trusted_scripts @@ -280,7 +280,7 @@ async fn antispam() { let mut expected_headers = AHashMap::new(); // Build session - let mut session = Session::test(core.clone()); + let mut session = Session::test(server.clone()); for line in lines.by_ref() { if in_params { if line.is_empty() { @@ -413,9 +413,9 @@ async fn antispam() { } // Run script - let core_ = core.clone(); + let server_ = server.clone(); let script = script.clone(); - match core_ + match server_ .run_script("test".to_string(), script, params, 0) .await { diff --git a/tests/src/smtp/inbound/auth.rs b/tests/src/smtp/inbound/auth.rs index 037bbe7a..6a9679cb 100644 --- a/tests/src/smtp/inbound/auth.rs +++ b/tests/src/smtp/inbound/auth.rs @@ -11,13 +11,12 @@ use utils::config::Config; use crate::{ smtp::{ - build_smtp, session::{TestSession, VerifyResponse}, - TempDir, + TempDir, TestSMTP, }, AssertConfig, }; -use smtp::core::{Inner, Session, State}; +use smtp::core::{Session, State}; const CONFIG: &str = r#" [storage] @@ -81,7 +80,7 @@ async fn auth() { config.assert_no_errors(); // EHLO should not advertise plain text auth without TLS - let mut session = Session::test(build_smtp(core, Inner::default())); + let mut session = Session::test(TestSMTP::from_core(core).server); session.data.remote_ip_str = "10.0.0.1".to_string(); session.eval_session_params().await; session.stream.tls = false; diff --git a/tests/src/smtp/inbound/basic.rs b/tests/src/smtp/inbound/basic.rs index 1db73a77..62e0d978 100644 --- a/tests/src/smtp/inbound/basic.rs +++ b/tests/src/smtp/inbound/basic.rs @@ -5,11 +5,11 @@ */ use common::Core; -use smtp::core::{Inner, Session}; +use smtp::core::Session; use crate::smtp::{ - build_smtp, session::{TestSession, VerifyResponse}, + TestSMTP, }; #[tokio::test] @@ -17,7 +17,7 @@ async fn basic_commands() { // Enable logging crate::enable_logging(); - let mut session = Session::test(build_smtp(Core::default(), Inner::default())); + let mut session = Session::test(TestSMTP::from_core(Core::default()).server); // STARTTLS should be available on clear text connections session.stream.tls = false; diff --git a/tests/src/smtp/inbound/data.rs b/tests/src/smtp/inbound/data.rs index 1f6fb195..d4415bb6 100644 --- a/tests/src/smtp/inbound/data.rs +++ b/tests/src/smtp/inbound/data.rs @@ -10,14 +10,13 @@ use utils::config::Config; use crate::{ smtp::{ - build_smtp, inbound::TestMessage, session::{load_test_message, TestSession, VerifyResponse}, TempDir, TestSMTP, }, AssertConfig, }; -use smtp::core::{Inner, Session}; +use smtp::core::Session; const CONFIG: &str = r#" [storage] @@ -105,17 +104,16 @@ async fn data() { crate::enable_logging(); // Create temp dir for queue - let mut inner = Inner::default(); let tmp_dir = TempDir::new("smtp_data_test", true); let mut config = Config::new(tmp_dir.update_config(CONFIG)).unwrap(); let stores = Stores::parse_all(&mut config).await; let core = Core::parse(&mut config, stores, Default::default()).await; config.assert_no_errors(); - let mut qr = inner.init_test_queue(&core); // Test queue message builder - let core = build_smtp(core, inner); - let mut session = Session::test(core.clone()); + let test = TestSMTP::from_core(core); + let mut qr = test.queue_receiver; + let mut session = Session::test(test.server.clone()); session.data.remote_ip_str = "10.0.0.1".to_string(); session.eval_session_params().await; session.test_builder().await; @@ -197,7 +195,7 @@ async fn data() { .await; // Release quota - qr.clear_queue(&core).await; + qr.clear_queue(&test.server).await; // Only 1500 bytes are allowed in the queue to domain foobar.org session @@ -236,10 +234,9 @@ async fn data() { .await; // Make sure store is empty - qr.clear_queue(&core).await; - core.core - .storage - .data - .assert_is_empty(core.core.storage.blob.clone()) + qr.clear_queue(&test.server).await; + test.server + .store() + .assert_is_empty(test.server.blob_store().clone()) .await; } diff --git a/tests/src/smtp/inbound/dmarc.rs b/tests/src/smtp/inbound/dmarc.rs index bd57d035..951c9c66 100644 --- a/tests/src/smtp/inbound/dmarc.rs +++ b/tests/src/smtp/inbound/dmarc.rs @@ -19,12 +19,11 @@ use store::Stores; use utils::config::Config; use crate::smtp::{ - build_smtp, inbound::{sign::SIGNATURES, TestMessage, TestReportingEvent}, session::{TestSession, VerifyResponse}, TempDir, TestSMTP, }; -use smtp::core::{Inner, Session}; +use smtp::core::Session; const CONFIG: &str = r#" [storage] @@ -95,15 +94,11 @@ async fn dmarc() { // Enable logging crate::enable_logging(); - let mut inner = Inner::default(); let tmp_dir = TempDir::new("smtp_dmarc_test", true); let mut config = Config::new(tmp_dir.update_config(CONFIG.to_string() + SIGNATURES)).unwrap(); let stores = Stores::parse_all(&mut config).await; let core = Core::parse(&mut config, stores, Default::default()).await; - // Create temp dir for queue - let mut qr = inner.init_test_queue(&core); - // Add SPF, DKIM and DMARC records core.smtp.resolvers.dns.txt_add( "mx.example.com", @@ -166,12 +161,11 @@ async fn dmarc() { Instant::now() + Duration::from_secs(5), ); - // Create report channels - let mut rr = inner.init_test_report(); - // SPF must pass - let core = build_smtp(core, inner); - let mut session = Session::test(core.clone()); + let test = TestSMTP::from_core(core); + let mut rr = test.report_receiver; + let mut qr = test.queue_receiver; + let mut session = Session::test(test.server.clone()); session.data.remote_ip_str = "10.0.0.2".to_string(); session.data.remote_ip = session.data.remote_ip_str.parse().unwrap(); session.eval_session_params().await; @@ -246,7 +240,7 @@ async fn dmarc() { qr.assert_no_events(); // Unaligned DMARC should be rejected - core.core.smtp.resolvers.dns.txt_add( + test.server.core.smtp.resolvers.dns.txt_add( "test.net", Spf::parse(b"v=spf1 -all").unwrap(), Instant::now() + Duration::from_secs(5), diff --git a/tests/src/smtp/inbound/ehlo.rs b/tests/src/smtp/inbound/ehlo.rs index 48445ddd..907eb75d 100644 --- a/tests/src/smtp/inbound/ehlo.rs +++ b/tests/src/smtp/inbound/ehlo.rs @@ -9,12 +9,12 @@ use std::time::{Duration, Instant}; use common::Core; use mail_auth::{common::parse::TxtRecordParser, spf::Spf, SpfResult}; -use smtp::core::{Inner, Session}; +use smtp::core::Session; use utils::config::Config; use crate::smtp::{ - build_smtp, session::{TestSession, VerifyResponse}, + TestSMTP, }; const CONFIG: &str = r#" @@ -55,7 +55,7 @@ async fn ehlo() { ); // Reject non-FQDN domains - let mut session = Session::test(build_smtp(core, Inner::default())); + let mut session = Session::test(TestSMTP::from_core(core).server); session.data.remote_ip_str = "10.0.0.1".to_string(); session.data.remote_ip = session.data.remote_ip_str.parse().unwrap(); session.stream.tls = false; diff --git a/tests/src/smtp/inbound/limits.rs b/tests/src/smtp/inbound/limits.rs index 8f92d0eb..078df56a 100644 --- a/tests/src/smtp/inbound/limits.rs +++ b/tests/src/smtp/inbound/limits.rs @@ -9,12 +9,12 @@ use std::time::{Duration, Instant}; use common::Core; use tokio::sync::watch; -use smtp::core::{Inner, Session}; +use smtp::core::Session; use utils::config::Config; use crate::smtp::{ - build_smtp, session::{TestSession, VerifyResponse}, + TestSMTP, }; const CONFIG: &str = r#" @@ -38,7 +38,7 @@ async fn limits() { let (_tx, rx) = watch::channel(true); // Exceed max line length - let mut session = Session::test_with_shutdown(build_smtp(core, Inner::default()), rx); + let mut session = Session::test_with_shutdown(TestSMTP::from_core(core).server, rx); session.data.remote_ip_str = "10.0.0.1".to_string(); let mut buf = vec![b'A'; 2049]; session.ingest(&buf).await.unwrap(); diff --git a/tests/src/smtp/inbound/mail.rs b/tests/src/smtp/inbound/mail.rs index 9cdf96d4..485c039f 100644 --- a/tests/src/smtp/inbound/mail.rs +++ b/tests/src/smtp/inbound/mail.rs @@ -13,14 +13,13 @@ use common::Core; use mail_auth::{common::parse::TxtRecordParser, spf::Spf, IprevResult, SpfResult}; use smtp_proto::{MAIL_BY_NOTIFY, MAIL_BY_RETURN, MAIL_REQUIRETLS}; -use smtp::core::{Inner, Session}; +use smtp::core::Session; use store::Stores; use utils::config::Config; use crate::smtp::{ - build_smtp, session::{TestSession, VerifyResponse}, - TempDir, + TempDir, TestSMTP, }; const CONFIG: &str = r#" @@ -108,7 +107,7 @@ async fn mail() { // Be rude and do not say EHLO let core = Arc::new(core); - let mut session = Session::test(build_smtp(core.clone(), Inner::default())); + let mut session = Session::test(TestSMTP::from_core(core.clone()).server); session.data.remote_ip_str = "10.0.0.1".to_string(); session.data.remote_ip = session.data.remote_ip_str.parse().unwrap(); session.eval_session_params().await; diff --git a/tests/src/smtp/inbound/milter.rs b/tests/src/smtp/inbound/milter.rs index ccbac2f5..ed21df7e 100644 --- a/tests/src/smtp/inbound/milter.rs +++ b/tests/src/smtp/inbound/milter.rs @@ -20,7 +20,7 @@ use mail_auth::AuthenticatedMessage; use mail_parser::MessageParser; use serde::Deserialize; use smtp::{ - core::{Inner, Session, SessionData}, + core::{Session, SessionData}, inbound::{ hooks::{self, Request, SmtpResponse}, milter::{ @@ -38,7 +38,6 @@ use tokio::{ use utils::config::Config; use crate::smtp::{ - build_smtp, inbound::TestMessage, session::{load_test_message, TestSession, VerifyResponse}, TempDir, TestSMTP, @@ -108,11 +107,11 @@ async fn milter_session() { let core = Core::parse(&mut config, stores, Default::default()).await; let _rx = spawn_mock_milter_server(); tokio::time::sleep(Duration::from_millis(100)).await; - let mut inner = Inner::default(); - let mut qr = inner.init_test_queue(&core); // Build session - let mut session = Session::test(build_smtp(core, inner)); + let test = TestSMTP::from_core(core); + let mut qr = test.queue_receiver; + let mut session = Session::test(test.server); session.data.remote_ip_str = "10.0.0.1".to_string(); session.eval_session_params().await; session.ehlo("mx.doe.org").await; @@ -241,11 +240,11 @@ async fn mta_hook_session() { let core = Core::parse(&mut config, stores, Default::default()).await; let _rx = spawn_mock_mta_hook_server(); tokio::time::sleep(Duration::from_millis(100)).await; - let mut inner = Inner::default(); - let mut qr = inner.init_test_queue(&core); // Build session - let mut session = Session::test(build_smtp(core, inner)); + let test = TestSMTP::from_core(core); + let mut qr = test.queue_receiver; + let mut session = Session::test(test.server); session.data.remote_ip_str = "10.0.0.1".to_string(); session.eval_session_params().await; session.ehlo("mx.doe.org").await; diff --git a/tests/src/smtp/inbound/mod.rs b/tests/src/smtp/inbound/mod.rs index 26b9e0e0..d55bbbba 100644 --- a/tests/src/smtp/inbound/mod.rs +++ b/tests/src/smtp/inbound/mod.rs @@ -6,17 +6,17 @@ use std::time::Duration; +use common::{ + ipc::{DmarcEvent, OnHold, QueueEvent, QueueEventLock, ReportingEvent, TlsEvent}, + Server, +}; use store::{ - write::{key::DeserializeBigEndian, Bincode, QueueClass, QueueEvent, ReportEvent, ValueClass}, + write::{key::DeserializeBigEndian, Bincode, QueueClass, ReportEvent, ValueClass}, Deserialize, IterateParams, ValueKey, U64_LEN, }; use tokio::sync::mpsc::error::TryRecvError; -use smtp::{ - core::SMTP, - queue::{self, spool::QueueEventLock, DeliveryAttempt, Message, OnHold, QueueId}, - reporting::{self, DmarcEvent, TlsEvent}, -}; +use smtp::queue::{DeliveryAttempt, Message, QueueId}; use super::{QueueReceiver, ReportReceiver}; @@ -37,7 +37,7 @@ pub mod throttle; pub mod vrfy; impl QueueReceiver { - pub async fn read_event(&mut self) -> queue::Event { + pub async fn read_event(&mut self) -> QueueEvent { match tokio::time::timeout(Duration::from_millis(100), self.queue_rx.recv()).await { Ok(Some(event)) => event, Ok(None) => panic!("Channel closed."), @@ -45,7 +45,7 @@ impl QueueReceiver { } } - pub async fn try_read_event(&mut self) -> Option<queue::Event> { + pub async fn try_read_event(&mut self) -> Option<QueueEvent> { match tokio::time::timeout(Duration::from_millis(100), self.queue_rx.recv()).await { Ok(Some(event)) => Some(event), Ok(None) => panic!("Channel closed."), @@ -120,12 +120,12 @@ impl QueueReceiver { self.last_queued_message().await } - pub async fn consume_message(&mut self, core: &SMTP) -> Message { + pub async fn consume_message(&mut self, server: &Server) -> Message { self.read_event().await.assert_reload(); let message = self.last_queued_message().await; message .clone() - .remove(core, self.last_queued_due().await) + .remove(server, self.last_queued_due().await) .await; message } @@ -144,23 +144,27 @@ impl QueueReceiver { }) } - pub async fn read_queued_events(&self) -> Vec<QueueEvent> { + pub async fn read_queued_events(&self) -> Vec<store::write::QueueEvent> { let mut events = Vec::new(); - let from_key = ValueKey::from(ValueClass::Queue(QueueClass::MessageEvent(QueueEvent { - due: 0, - queue_id: 0, - }))); - let to_key = ValueKey::from(ValueClass::Queue(QueueClass::MessageEvent(QueueEvent { - due: u64::MAX, - queue_id: u64::MAX, - }))); + let from_key = ValueKey::from(ValueClass::Queue(QueueClass::MessageEvent( + store::write::QueueEvent { + due: 0, + queue_id: 0, + }, + ))); + let to_key = ValueKey::from(ValueClass::Queue(QueueClass::MessageEvent( + store::write::QueueEvent { + due: u64::MAX, + queue_id: u64::MAX, + }, + ))); self.store .iterate( IterateParams::new(from_key, to_key).ascending().no_values(), |key, _| { - events.push(QueueEvent { + events.push(store::write::QueueEvent { due: key.deserialize_be_u64(0)?, queue_id: key.deserialize_be_u64(U64_LEN)?, }); @@ -261,16 +265,16 @@ impl QueueReceiver { .expect("No event found in queue for message") } - pub async fn clear_queue(&self, core: &SMTP) { + pub async fn clear_queue(&self, server: &Server) { for message in self.read_queued_messages().await { let due = self.message_due(message.queue_id).await; - message.remove(core, due).await; + message.remove(server, due).await; } } } impl ReportReceiver { - pub async fn read_report(&mut self) -> reporting::Event { + pub async fn read_report(&mut self) -> ReportingEvent { match tokio::time::timeout(Duration::from_millis(100), self.report_rx.recv()).await { Ok(Some(event)) => event, Ok(None) => panic!("Channel closed."), @@ -278,7 +282,7 @@ impl ReportReceiver { } } - pub async fn try_read_report(&mut self) -> Option<reporting::Event> { + pub async fn try_read_report(&mut self) -> Option<ReportingEvent> { match tokio::time::timeout(Duration::from_millis(100), self.report_rx.recv()).await { Ok(Some(event)) => Some(event), Ok(None) => panic!("Channel closed."), @@ -299,17 +303,17 @@ pub trait TestQueueEvent { fn unwrap_on_hold(self) -> OnHold<QueueEventLock>; } -impl TestQueueEvent for queue::Event { +impl TestQueueEvent for QueueEvent { fn assert_reload(self) { match self { - queue::Event::Reload => (), + QueueEvent::Reload => (), e => panic!("Unexpected event: {e:?}"), } } fn unwrap_on_hold(self) -> OnHold<QueueEventLock> { match self { - queue::Event::OnHold(value) => value, + QueueEvent::OnHold(value) => value, e => panic!("Unexpected event: {e:?}"), } } @@ -320,17 +324,17 @@ pub trait TestReportingEvent { fn unwrap_tls(self) -> Box<TlsEvent>; } -impl TestReportingEvent for reporting::Event { +impl TestReportingEvent for ReportingEvent { fn unwrap_dmarc(self) -> Box<DmarcEvent> { match self { - reporting::Event::Dmarc(event) => event, + ReportingEvent::Dmarc(event) => event, e => panic!("Unexpected event: {e:?}"), } } fn unwrap_tls(self) -> Box<TlsEvent> { match self { - reporting::Event::Tls(event) => event, + ReportingEvent::Tls(event) => event, e => panic!("Unexpected event: {e:?}"), } } diff --git a/tests/src/smtp/inbound/rcpt.rs b/tests/src/smtp/inbound/rcpt.rs index 6eea2ff1..2f972174 100644 --- a/tests/src/smtp/inbound/rcpt.rs +++ b/tests/src/smtp/inbound/rcpt.rs @@ -12,12 +12,11 @@ use smtp_proto::{RCPT_NOTIFY_DELAY, RCPT_NOTIFY_FAILURE, RCPT_NOTIFY_SUCCESS}; use store::Stores; use utils::config::Config; -use smtp::core::{Inner, Session, State}; +use smtp::core::{Session, State}; use crate::smtp::{ - build_smtp, session::{TestSession, VerifyResponse}, - TempDir, + TempDir, TestSMTP, }; const CONFIG: &str = r#" @@ -94,7 +93,7 @@ async fn rcpt() { let core = Core::parse(&mut config, stores, Default::default()).await; // RCPT without MAIL FROM - let mut session = Session::test(build_smtp(core, Inner::default())); + let mut session = Session::test(TestSMTP::from_core(core).server); session.data.remote_ip_str = "10.0.0.1".to_string(); session.eval_session_params().await; session.ehlo("mx1.foobar.org").await; diff --git a/tests/src/smtp/inbound/rewrite.rs b/tests/src/smtp/inbound/rewrite.rs index 3ea7fe3d..4b17d70e 100644 --- a/tests/src/smtp/inbound/rewrite.rs +++ b/tests/src/smtp/inbound/rewrite.rs @@ -6,10 +6,10 @@ use common::Core; -use smtp::core::{Inner, Session}; +use smtp::core::Session; use utils::config::Config; -use crate::smtp::{build_smtp, session::TestSession}; +use crate::smtp::{session::TestSession, TestSMTP}; const CONFIG: &str = r#" [session.mail] @@ -74,7 +74,7 @@ async fn address_rewrite() { let core = Core::parse(&mut config, Default::default(), Default::default()).await; // Init session - let mut session = Session::test(build_smtp(core, Inner::default())); + let mut session = Session::test(TestSMTP::from_core(core).server); session.data.remote_ip_str = "10.0.0.1".to_string(); session.eval_session_params().await; session.ehlo("mx.doe.org").await; diff --git a/tests/src/smtp/inbound/scripts.rs b/tests/src/smtp/inbound/scripts.rs index 144fc908..4833aad9 100644 --- a/tests/src/smtp/inbound/scripts.rs +++ b/tests/src/smtp/inbound/scripts.rs @@ -10,7 +10,6 @@ use std::{fmt::Write, fs, path::PathBuf}; use crate::{ enable_logging, smtp::{ - build_smtp, inbound::{sign::SIGNATURES, TestMessage, TestQueueEvent}, session::{TestSession, VerifyResponse}, TempDir, TestSMTP, @@ -20,8 +19,8 @@ use crate::{ use common::Core; use smtp::{ - core::{Inner, Session}, - scripts::ScriptResult, + core::Session, + scripts::{event_loop::RunScript, ScriptResult}, }; use store::Stores; use utils::config::Config; @@ -135,7 +134,7 @@ async fn sieve_scripts() { } // Prepare config - let mut inner = Inner::default(); + let tmp_dir = TempDir::new("smtp_sieve_test", true); let mut config = Config::new( tmp_dir.update_config( @@ -155,18 +154,18 @@ async fn sieve_scripts() { config.resolve_all_macros().await; let stores = Stores::parse_all(&mut config).await; let core = Core::parse(&mut config, stores, Default::default()).await; - let mut qr = inner.init_test_queue(&core); config.assert_no_errors(); // Build session - let core = build_smtp(core, inner); - let mut session = Session::test(core.clone()); + let test = TestSMTP::from_core(core); + let mut qr = test.queue_receiver; + let mut session = Session::test(test.server.clone()); session.data.remote_ip_str = "10.0.0.88".parse().unwrap(); session.data.remote_ip = session.data.remote_ip_str.parse().unwrap(); assert!(!session.init_conn().await); // Run tests - for (name, script) in &core.core.sieve.trusted_scripts { + for (name, script) in &test.server.core.sieve.trusted_scripts { if name.starts_with("stage_") || name.ends_with("_include") { continue; } @@ -174,10 +173,13 @@ async fn sieve_scripts() { let params = session .build_script_parameters("data") .set_variable("from", "john.doe@example.org") - .with_envelope(&core.core, &session, 0) + .with_envelope(&test.server, &session, 0) .await; - let core_ = core.clone(); - match core_.run_script(name.to_string(), script, params, 0).await { + match test + .server + .run_script(name.to_string(), script, params, 0) + .await + { ScriptResult::Accept { .. } => (), ScriptResult::Reject(message) => panic!("{}", message), err => { @@ -248,7 +250,7 @@ async fn sieve_scripts() { ) .await; qr.assert_no_events(); - qr.clear_queue(&core).await; + qr.clear_queue(&test.server).await; // Expect message delivery plus a notification session @@ -295,7 +297,7 @@ async fn sieve_scripts() { .assert_not_contains("X-Part-Number: 5") .assert_not_contains("THIS IS A PIECE OF HTML TEXT"); qr.assert_no_events(); - qr.clear_queue(&core).await; + qr.clear_queue(&test.server).await; // Expect a modified message delivery plus a notification session @@ -332,7 +334,7 @@ async fn sieve_scripts() { .assert_contains("X-Part-Number: 5") .assert_contains("THIS IS A PIECE OF HTML TEXT") .assert_not_contains("X-My-Header: true"); - qr.clear_queue(&core).await; + qr.clear_queue(&test.server).await; // Expect a modified redirected message session diff --git a/tests/src/smtp/inbound/sign.rs b/tests/src/smtp/inbound/sign.rs index 6b2a39bb..5ccc2dae 100644 --- a/tests/src/smtp/inbound/sign.rs +++ b/tests/src/smtp/inbound/sign.rs @@ -16,12 +16,11 @@ use store::Stores; use utils::config::Config; use crate::smtp::{ - build_smtp, inbound::TestMessage, session::{TestSession, VerifyResponse}, TempDir, TestSMTP, }; -use smtp::core::{Inner, Session}; +use smtp::core::Session; pub const SIGNATURES: &str = " [signature.rsa] @@ -131,10 +130,6 @@ async fn sign_and_seal() { let mut config = Config::new(tmp_dir.update_config(CONFIG.to_string() + SIGNATURES)).unwrap(); let stores = Stores::parse_all(&mut config).await; let core = Core::parse(&mut config, stores, Default::default()).await; - let mut inner = Inner::default(); - - // Create temp dir for queue - let mut qr = inner.init_test_queue(&core); // Add SPF, DKIM and DMARC records core.smtp.resolvers.dns.txt_add( @@ -176,7 +171,9 @@ async fn sign_and_seal() { ); // Test DKIM signing - let mut session = Session::test(build_smtp(core, inner)); + let test = TestSMTP::from_core(core); + let mut qr = test.queue_receiver; + let mut session = Session::test(test.server); session.data.remote_ip_str = "10.0.0.2".to_string(); session.eval_session_params().await; session.ehlo("mx.example.com").await; diff --git a/tests/src/smtp/inbound/throttle.rs b/tests/src/smtp/inbound/throttle.rs index 7def5f38..414ec302 100644 --- a/tests/src/smtp/inbound/throttle.rs +++ b/tests/src/smtp/inbound/throttle.rs @@ -6,9 +6,9 @@ use std::time::Duration; -use crate::smtp::{build_smtp, session::TestSession, TempDir}; +use crate::smtp::{session::TestSession, TempDir, TestSMTP}; use common::Core; -use smtp::core::{Inner, Session, SessionAddress}; +use smtp::core::{Session, SessionAddress}; use store::Stores; use utils::config::Config; @@ -51,10 +51,9 @@ async fn throttle_inbound() { let mut config = Config::new(tmp_dir.update_config(CONFIG)).unwrap(); let stores = Stores::parse_all(&mut config).await; let core = Core::parse(&mut config, stores, Default::default()).await; - let inner = Inner::default(); // Test connection concurrency limit - let mut session = Session::test(build_smtp(core, inner)); + let mut session = Session::test(TestSMTP::from_core(core).server); session.data.remote_ip_str = "10.0.0.1".to_string(); assert!( session.is_allowed().await, diff --git a/tests/src/smtp/inbound/vrfy.rs b/tests/src/smtp/inbound/vrfy.rs index f431d503..9345dad1 100644 --- a/tests/src/smtp/inbound/vrfy.rs +++ b/tests/src/smtp/inbound/vrfy.rs @@ -9,13 +9,12 @@ use common::Core; use store::Stores; use utils::config::Config; -use smtp::core::{Inner, Session}; +use smtp::core::Session; use crate::{ smtp::{ - build_smtp, session::{TestSession, VerifyResponse}, - TempDir, + TempDir, TestSMTP, }, AssertConfig, }; @@ -79,7 +78,7 @@ async fn vrfy_expn() { config.assert_no_errors(); // EHLO should not advertise VRFY/EXPN to 10.0.0.2 - let mut session = Session::test(build_smtp(core, Inner::default())); + let mut session = Session::test(TestSMTP::from_core(core).server); session.data.remote_ip_str = "10.0.0.2".to_string(); session.eval_session_params().await; session diff --git a/tests/src/smtp/lookup/sql.rs b/tests/src/smtp/lookup/sql.rs index b20ce71a..a6582597 100644 --- a/tests/src/smtp/lookup/sql.rs +++ b/tests/src/smtp/lookup/sql.rs @@ -18,15 +18,11 @@ use utils::config::Config; use crate::{ directory::DirectoryStore, smtp::{ - build_smtp, session::{TestSession, VerifyResponse}, - TempDir, + TempDir, TestSMTP, }, }; -use smtp::{ - core::{Inner, Session}, - queue::RecipientDomain, -}; +use smtp::{core::Session, queue::RecipientDomain}; const CONFIG: &str = r#" [storage] @@ -106,7 +102,6 @@ async fn lookup_sql() { let mut config = Config::new(temp_dir.update_config(CONFIG)).unwrap(); let stores = Stores::parse_all(&mut config).await; - let inner = Inner::default(); let core = Core::parse(&mut config, stores, Default::default()).await; core.smtp.resolvers.dns.mx_add( @@ -123,6 +118,8 @@ async fn lookup_sql() { store: core.storage.lookups.get("sql").unwrap().clone(), }; + let test = TestSMTP::from_core(core); + // Create tables handle.create_test_directory().await; @@ -184,7 +181,8 @@ async fn lookup_sql() { let e = Expression::try_parse(&mut config, ("test", test_name, "expr"), &token_map).unwrap(); assert_eq!( - core.eval_expr::<String, _>(&e, &RecipientDomain::new("test.org"), "text", 0) + test.server + .eval_expr::<String, _>(&e, &RecipientDomain::new("test.org"), "text", 0) .await .unwrap(), config.value(("test", test_name, "expect")).unwrap(), @@ -193,7 +191,7 @@ async fn lookup_sql() { ); } - let mut session = Session::test(build_smtp(core, inner)); + let mut session = Session::test(test.server); session.data.remote_ip_str = "10.0.0.50".parse().unwrap(); session.eval_session_params().await; session.stream.tls = true; diff --git a/tests/src/smtp/lookup/utils.rs b/tests/src/smtp/lookup/utils.rs index 17bf5821..59773fcc 100644 --- a/tests/src/smtp/lookup/utils.rs +++ b/tests/src/smtp/lookup/utils.rs @@ -18,14 +18,16 @@ use mail_auth::MX; use ::smtp::outbound::NextHop; use mail_parser::DateTime; use smtp::{ - core::Inner, - outbound::{lookup::ToNextHop, mta_sts::parse::ParsePolicy}, + outbound::{ + lookup::{DnsLookup, ToNextHop}, + mta_sts::parse::ParsePolicy, + }, queue::RecipientDomain, reporting::AggregateTimestamp, }; use utils::config::Config; -use crate::smtp::build_smtp; +use crate::smtp::TestSMTP; const CONFIG_V4: &str = r#" [queue.outbound.source-ip] @@ -65,11 +67,9 @@ async fn lookup_ip() { "10.0.0.4".parse().unwrap(), ]; let mut config = Config::new(CONFIG_V4).unwrap(); - let core = build_smtp( - Core::parse(&mut config, Default::default(), Default::default()).await, - Inner::default(), - ); - core.core.smtp.resolvers.dns.ipv4_add( + let test = + TestSMTP::from_core(Core::parse(&mut config, Default::default(), Default::default()).await); + test.server.core.smtp.resolvers.dns.ipv4_add( "mx.foobar.org", vec![ "172.168.0.100".parse().unwrap(), @@ -77,14 +77,15 @@ async fn lookup_ip() { ], Instant::now() + Duration::from_secs(10), ); - core.core.smtp.resolvers.dns.ipv6_add( + test.server.core.smtp.resolvers.dns.ipv6_add( "mx.foobar.org", vec!["e:f::a".parse().unwrap(), "e:f::b".parse().unwrap()], Instant::now() + Duration::from_secs(10), ); // Ipv4 strategy - let resolve_result = core + let resolve_result = test + .server .resolve_host( &NextHop::MX("mx.foobar.org"), &RecipientDomain::new("envelope"), @@ -103,11 +104,9 @@ async fn lookup_ip() { // Ipv6 strategy let mut config = Config::new(CONFIG_V6).unwrap(); - let core = build_smtp( - Core::parse(&mut config, Default::default(), Default::default()).await, - Inner::default(), - ); - core.core.smtp.resolvers.dns.ipv4_add( + let test = + TestSMTP::from_core(Core::parse(&mut config, Default::default(), Default::default()).await); + test.server.core.smtp.resolvers.dns.ipv4_add( "mx.foobar.org", vec![ "172.168.0.100".parse().unwrap(), @@ -115,12 +114,13 @@ async fn lookup_ip() { ], Instant::now() + Duration::from_secs(10), ); - core.core.smtp.resolvers.dns.ipv6_add( + test.server.core.smtp.resolvers.dns.ipv6_add( "mx.foobar.org", vec!["e:f::a".parse().unwrap(), "e:f::b".parse().unwrap()], Instant::now() + Duration::from_secs(10), ); - let resolve_result = core + let resolve_result = test + .server .resolve_host( &NextHop::MX("mx.foobar.org"), &RecipientDomain::new("envelope"), diff --git a/tests/src/smtp/management/queue.rs b/tests/src/smtp/management/queue.rs index 181713fb..9c0a4fa8 100644 --- a/tests/src/smtp/management/queue.rs +++ b/tests/src/smtp/management/queue.rs @@ -16,7 +16,7 @@ use reqwest::{header::AUTHORIZATION, Method, StatusCode}; use crate::{ jmap::ManagementApi, - smtp::{outbound::TestServer, session::TestSession}, + smtp::{session::TestSession, TestSMTP}, }; use smtp::queue::{manager::SpawnQueue, QueueId, Status}; @@ -70,12 +70,12 @@ async fn manage_queue() { crate::enable_logging(); // Start remote test server - let mut remote = TestServer::new("smtp_manage_queue_remote", REMOTE, true).await; + let mut remote = TestSMTP::new("smtp_manage_queue_remote", REMOTE).await; let _rx = remote.start(&[ServerProtocol::Smtp]).await; let remote_core = remote.build_smtp(); // Start local management interface - let local = TestServer::new("smtp_manage_queue_local", LOCAL, true).await; + let local = TestSMTP::new("smtp_manage_queue_local", LOCAL).await; // Add mock DNS entries let core = local.build_smtp(); @@ -134,7 +134,10 @@ async fn manage_queue() { ("f", ("", vec!["success@foobar.org", "delay@foobar.org"])), ]); let mut session = local.new_session(); - local.qr.queue_rx.spawn(local.instance.clone()); + local + .queue_receiver + .queue_rx + .spawn(local.server.inner.clone()); session.data.remote_ip_str = "10.0.0.1".to_string(); session.eval_session_params().await; session.ehlo("foobar.net").await; @@ -160,7 +163,7 @@ async fn manage_queue() { tokio::time::sleep(Duration::from_millis(100)).await; assert_eq!( remote - .qr + .queue_receiver .consume_message(&remote_core) .await .recipients @@ -315,7 +318,7 @@ async fn manage_queue() { tokio::time::sleep(Duration::from_millis(100)).await; assert_eq!( remote - .qr + .queue_receiver .consume_message(&remote_core) .await .recipients diff --git a/tests/src/smtp/management/report.rs b/tests/src/smtp/management/report.rs index 76052870..28754963 100644 --- a/tests/src/smtp/management/report.rs +++ b/tests/src/smtp/management/report.rs @@ -7,7 +7,10 @@ use std::sync::Arc; use ahash::{AHashMap, HashSet}; -use common::config::{server::ServerProtocol, smtp::report::AggregateFrequency}; +use common::{ + config::{server::ServerProtocol, smtp::report::AggregateFrequency}, + ipc::{DmarcEvent, PolicyType, TlsEvent}, +}; use jmap::api::management::queue::Report; use mail_auth::{ @@ -23,9 +26,9 @@ use reqwest::Method; use crate::{ jmap::ManagementApi, - smtp::{management::queue::List, outbound::TestServer}, + smtp::{management::queue::List, TestSMTP}, }; -use smtp::reporting::{scheduler::SpawnReport, DmarcEvent, TlsEvent}; +use smtp::reporting::{scheduler::SpawnReport, SmtpReporting}; const CONFIG: &str = r#" [storage] @@ -58,10 +61,13 @@ async fn manage_reports() { crate::enable_logging(); // Start reporting service - let local = TestServer::new("smtp_manage_reports", CONFIG, true).await; + let local = TestSMTP::new("smtp_manage_reports", CONFIG).await; let _rx = local.start(&[ServerProtocol::Http]).await; let core = local.build_smtp(); - local.rr.report_rx.spawn(local.instance.clone()); + local + .report_receiver + .report_rx + .spawn(local.server.inner.clone()); // Send test reporting events core.schedule_report(DmarcEvent { @@ -98,7 +104,7 @@ async fn manage_reports() { .await; core.schedule_report(TlsEvent { domain: "foobar.org".to_string(), - policy: smtp::reporting::PolicyType::None, + policy: PolicyType::None, failure: None, tls_record: Arc::new(TlsRpt::parse(b"v=TLSRPTv1;rua=mailto:reports@foobar.org").unwrap()), interval: AggregateFrequency::Daily, @@ -106,7 +112,7 @@ async fn manage_reports() { .await; core.schedule_report(TlsEvent { domain: "foobar.net".to_string(), - policy: smtp::reporting::PolicyType::Sts(None), + policy: PolicyType::Sts(None), failure: FailureDetails::new(ResultType::StsPolicyInvalid).into(), tls_record: Arc::new(TlsRpt::parse(b"v=TLSRPTv1;rua=mailto:reports@foobar.net").unwrap()), interval: AggregateFrequency::Weekly, diff --git a/tests/src/smtp/mod.rs b/tests/src/smtp/mod.rs index 09d60a33..275c42ae 100644 --- a/tests/src/smtp/mod.rs +++ b/tests/src/smtp/mod.rs @@ -6,11 +6,21 @@ use std::{path::PathBuf, sync::Arc}; -use common::Core; - -use smtp::core::{Inner, SMTP}; -use store::{BlobStore, Store}; -use tokio::sync::mpsc; +use common::{ + config::server::{Listeners, ServerProtocol}, + ipc::{QueueEvent, ReportingEvent}, + manager::boot::build_ipc, + Core, Inner, Server, +}; + +use jmap::api::JmapSessionManager; +use session::{DummyIo, TestSession}; +use smtp::core::{Session, SmtpSessionManager}; +use store::{BlobStore, Store, Stores}; +use tokio::sync::{mpsc, watch}; +use utils::config::Config; + +use crate::AssertConfig; pub mod config; pub mod inbound; @@ -72,40 +82,148 @@ pub fn add_test_certs(config: &str) -> String { pub struct QueueReceiver { store: Store, blob_store: BlobStore, - pub queue_rx: mpsc::Receiver<smtp::queue::Event>, + pub queue_rx: mpsc::Receiver<QueueEvent>, } pub struct ReportReceiver { - pub report_rx: mpsc::Receiver<smtp::reporting::Event>, + pub report_rx: mpsc::Receiver<ReportingEvent>, } -pub trait TestSMTP { - fn init_test_queue(&mut self, core: &Core) -> QueueReceiver; - fn init_test_report(&mut self) -> ReportReceiver; +pub struct TestSMTP { + pub server: Server, + pub temp_dir: Option<TempDir>, + pub queue_receiver: QueueReceiver, + pub report_receiver: ReportReceiver, } -impl TestSMTP for Inner { - fn init_test_queue(&mut self, core: &Core) -> QueueReceiver { - let (queue_tx, queue_rx) = mpsc::channel(128); - self.queue_tx = queue_tx; +const CONFIG: &str = r#" +[session.connect] +hostname = "'mx.example.org'" +greeting = "'Test SMTP instance'" + +[server.listener.smtp-debug] +bind = ['127.0.0.1:9925'] +protocol = 'smtp' + +[server.listener.lmtp-debug] +bind = ['127.0.0.1:9924'] +protocol = 'lmtp' +tls.implicit = true + +[server.listener.management-debug] +bind = ['127.0.0.1:9980'] +protocol = 'http' +tls.implicit = true + +[server.socket] +reuse-addr = true + +[server.tls] +enable = true +implicit = false +certificate = 'default' + +[certificate.default] +cert = '%{file:{CERT}}%' +private-key = '%{file:{PK}}%' + +[storage] +data = "sqlite" +lookup = "sqlite" +blob = "sqlite" +fts = "sqlite" + +[store."sqlite"] +type = "sqlite" +path = "{TMP}/queue.db" - QueueReceiver { - blob_store: core.storage.blob.clone(), - store: core.storage.data.clone(), - queue_rx, +"#; + +impl TestSMTP { + pub fn from_core(core: impl Into<Arc<Core>>) -> Self { + Self::from_core_and_tempdir(core, None) + } + + fn from_core_and_tempdir(core: impl Into<Arc<Core>>, temp_dir: Option<TempDir>) -> Self { + let core = core.into(); + let (ipc, mut ipc_rxs) = build_ipc(); + + TestSMTP { + queue_receiver: QueueReceiver { + store: core.storage.data.clone(), + blob_store: core.storage.blob.clone(), + queue_rx: ipc_rxs.queue_rx.take().unwrap(), + }, + report_receiver: ReportReceiver { + report_rx: ipc_rxs.report_rx.take().unwrap(), + }, + server: Server { + inner: Inner { + shared_core: core.as_ref().clone().into_shared(), + data: Default::default(), + ipc, + } + .into(), + core, + }, + temp_dir, } } - fn init_test_report(&mut self) -> ReportReceiver { - let (report_tx, report_rx) = mpsc::channel(128); - self.report_tx = report_tx; - ReportReceiver { report_rx } + pub async fn new(name: &str, config: impl AsRef<str>) -> TestSMTP { + let temp_dir = TempDir::new(name, true); + let mut config = + Config::new(temp_dir.update_config(add_test_certs(CONFIG) + config.as_ref())).unwrap(); + config.resolve_all_macros().await; + let stores = Stores::parse_all(&mut config).await; + let core = Core::parse(&mut config, stores, Default::default()).await; + + Self::from_core_and_tempdir(core, Some(temp_dir)) + } + + pub async fn start(&self, protocols: &[ServerProtocol]) -> watch::Sender<bool> { + // Spawn listeners + let mut config = Config::new(CONFIG).unwrap(); + let mut servers = Listeners::parse(&mut config); + servers.parse_tcp_acceptors(&mut config, self.server.inner.clone()); + + // Filter out protocols + servers + .servers + .retain(|server| protocols.contains(&server.protocol)); + + // Start servers + servers.bind_and_drop_priv(&mut config); + config.assert_no_errors(); + + servers + .spawn(|server, acceptor, shutdown_rx| { + match &server.protocol { + ServerProtocol::Smtp | ServerProtocol::Lmtp => server.spawn( + SmtpSessionManager::new(self.server.inner.clone()), + self.server.inner.clone(), + acceptor, + shutdown_rx, + ), + ServerProtocol::Http => server.spawn( + JmapSessionManager::new(self.server.inner.clone()), + self.server.inner.clone(), + acceptor, + shutdown_rx, + ), + ServerProtocol::Imap | ServerProtocol::Pop3 | ServerProtocol::ManageSieve => { + unreachable!() + } + }; + }) + .0 + } + + pub fn new_session(&self) -> Session<DummyIo> { + Session::test(self.server.clone()) } -} -fn build_smtp(core: impl Into<Arc<Core>>, inner: impl Into<Arc<Inner>>) -> SMTP { - SMTP { - core: core.into(), - inner: inner.into(), + pub fn build_smtp(&self) -> Server { + self.server.clone() } } diff --git a/tests/src/smtp/outbound/dane.rs b/tests/src/smtp/outbound/dane.rs index 80c5242a..4bc3d853 100644 --- a/tests/src/smtp/outbound/dane.rs +++ b/tests/src/smtp/outbound/dane.rs @@ -19,6 +19,7 @@ use common::{ server::ServerProtocol, smtp::resolver::{DnsRecordCache, DnssecResolver, Resolvers, Tlsa, TlsaEntry}, }, + ipc::PolicyType, Core, }; use mail_auth::{ @@ -38,15 +39,11 @@ use rustls_pki_types::CertificateDer; use crate::smtp::{ inbound::{TestMessage, TestQueueEvent, TestReportingEvent}, - outbound::TestServer, session::{TestSession, VerifyResponse}, + TestSMTP, }; -use smtp::outbound::dane::verify::TlsaVerify; -use smtp::{ - core::SMTP, - queue::{Error, ErrorDetails, Status}, - reporting::PolicyType, -}; +use smtp::outbound::dane::{dnssec::TlsaLookup, verify::TlsaVerify}; +use smtp::queue::{Error, ErrorDetails, Status}; const LOCAL: &str = r#" [session.rcpt] @@ -85,11 +82,11 @@ async fn dane_verify() { crate::enable_logging(); // Start test server - let mut remote = TestServer::new("smtp_dane_remote", REMOTE, true).await; + let mut remote = TestSMTP::new("smtp_dane_remote", REMOTE).await; let _rx = remote.start(&[ServerProtocol::Smtp]).await; // Fail on missing TLSA record - let mut local = TestServer::new("smtp_dane_local", LOCAL, true).await; + let mut local = TestSMTP::new("smtp_dane_local", LOCAL).await; // Add mock DNS entries let core = local.build_smtp(); @@ -120,24 +117,24 @@ async fn dane_verify() { .send_message("john@test.org", &["bill@foobar.org"], "test:no_dkim", "250") .await; local - .qr + .queue_receiver .expect_message_then_deliver() .await .try_deliver(core.clone()) .await; local - .qr + .queue_receiver .expect_message() .await - .read_lines(&local.qr) + .read_lines(&local.queue_receiver) .await .assert_contains("<bill@foobar.org> (DANE failed to authenticate") .assert_contains("No TLSA records found"); - local.qr.read_event().await.assert_reload(); - local.qr.assert_no_events(); + local.queue_receiver.read_event().await.assert_reload(); + local.queue_receiver.assert_no_events(); // Expect TLS failure report - let report = local.rr.read_report().await.unwrap_tls(); + let report = local.report_receiver.read_report().await.unwrap_tls(); assert_eq!(report.domain, "foobar.org"); assert_eq!(report.policy, PolicyType::Tlsa(None)); assert_eq!( @@ -173,30 +170,30 @@ async fn dane_verify() { .send_message("john@test.org", &["bill@foobar.org"], "test:no_dkim", "250") .await; local - .qr + .queue_receiver .expect_message_then_deliver() .await .try_deliver(core.clone()) .await; local - .qr + .queue_receiver .expect_message() .await - .read_lines(&local.qr) + .read_lines(&local.queue_receiver) .await .assert_contains("<bill@foobar.org> (DANE failed to authenticate") .assert_contains("No matching certificates found"); - local.qr.read_event().await.assert_reload(); - local.qr.assert_no_events(); + local.queue_receiver.read_event().await.assert_reload(); + local.queue_receiver.assert_no_events(); // Expect TLS failure report - let report = local.rr.read_report().await.unwrap_tls(); + let report = local.report_receiver.read_report().await.unwrap_tls(); assert_eq!(report.policy, PolicyType::Tlsa(tlsa.into())); assert_eq!( report.failure.as_ref().unwrap().result_type, ResultType::ValidationFailure ); - remote.qr.assert_no_events(); + remote.queue_receiver.assert_no_events(); // DANE successful delivery let tlsa = Arc::new(Tlsa { @@ -221,23 +218,23 @@ async fn dane_verify() { .send_message("john@test.org", &["bill@foobar.org"], "test:no_dkim", "250") .await; local - .qr + .queue_receiver .expect_message_then_deliver() .await .try_deliver(core.clone()) .await; - local.qr.read_event().await.assert_reload(); - local.qr.assert_no_events(); + local.queue_receiver.read_event().await.assert_reload(); + local.queue_receiver.assert_no_events(); remote - .qr + .queue_receiver .expect_message() .await - .read_lines(&remote.qr) + .read_lines(&remote.queue_receiver) .await .assert_contains("using TLSv1.3 with cipher"); // Expect TLS success report - let report = local.rr.read_report().await.unwrap_tls(); + let report = local.report_receiver.read_report().await.unwrap_tls(); assert_eq!(report.policy, PolicyType::Tlsa(tlsa.into())); assert!(report.failure.is_none()); } @@ -260,10 +257,7 @@ async fn dane_test() { mta_sts: LruCache::with_capacity(10), }, }; - let r = SMTP { - core: core.into(), - inner: Default::default(), - }; + let r = TestSMTP::from_core(core).build_smtp(); // Add dns entries let mut path = PathBuf::from(env!("CARGO_MANIFEST_DIR")); diff --git a/tests/src/smtp/outbound/extensions.rs b/tests/src/smtp/outbound/extensions.rs index 57bbb1f2..e434769c 100644 --- a/tests/src/smtp/outbound/extensions.rs +++ b/tests/src/smtp/outbound/extensions.rs @@ -12,8 +12,8 @@ use smtp_proto::{MAIL_REQUIRETLS, MAIL_RET_HDRS, MAIL_SMTPUTF8, RCPT_NOTIFY_NEVE use crate::smtp::{ inbound::{TestMessage, TestQueueEvent}, - outbound::TestServer, session::{TestSession, VerifyResponse}, + TestSMTP, }; const LOCAL: &str = r#" @@ -54,11 +54,11 @@ async fn extensions() { crate::enable_logging(); // Start test server - let mut remote = TestServer::new("smtp_ext_remote", REMOTE, true).await; + let mut remote = TestSMTP::new("smtp_ext_remote", REMOTE).await; let _rx = remote.start(&[ServerProtocol::Smtp]).await; // Successful delivery with DSN - let mut local = TestServer::new("smtp_ext_local", LOCAL, true).await; + let mut local = TestSMTP::new("smtp_ext_local", LOCAL).await; // Add mock DNS entries let core = local.build_smtp(); @@ -89,27 +89,27 @@ async fn extensions() { ) .await; local - .qr + .queue_receiver .expect_message_then_deliver() .await .try_deliver(core.clone()) .await; local - .qr + .queue_receiver .expect_message() .await - .read_lines(&local.qr) + .read_lines(&local.queue_receiver) .await .assert_contains("<bill@foobar.org> (delivered to") .assert_contains("Final-Recipient: rfc822;bill@foobar.org") .assert_contains("Action: delivered"); - local.qr.read_event().await.assert_reload(); + local.queue_receiver.read_event().await.assert_reload(); remote - .qr + .queue_receiver .expect_message() .await - .read_lines(&remote.qr) + .read_lines(&remote.queue_receiver) .await .assert_contains("using TLSv1.3 with cipher"); @@ -118,23 +118,23 @@ async fn extensions() { .send_message("john@test.org", &["bill@foobar.org"], "test:arc", "250") .await; local - .qr + .queue_receiver .expect_message_then_deliver() .await .try_deliver(core.clone()) .await; local - .qr + .queue_receiver .expect_message() .await - .read_lines(&local.qr) + .read_lines(&local.queue_receiver) .await .assert_contains("<bill@foobar.org> (host 'mx.foobar.org' rejected command 'MAIL FROM:") .assert_contains("Action: failed") .assert_contains("Diagnostic-Code: smtp;552") .assert_contains("Status: 5.3.4"); - local.qr.read_event().await.assert_reload(); - remote.qr.assert_no_events(); + local.queue_receiver.read_event().await.assert_reload(); + remote.queue_receiver.assert_no_events(); // Test DSN, SMTPUTF8 and REQUIRETLS extensions session @@ -146,13 +146,13 @@ async fn extensions() { ) .await; local - .qr + .queue_receiver .expect_message_then_deliver() .await .try_deliver(core.clone()) .await; - local.qr.read_event().await.assert_reload(); - let message = remote.qr.expect_message().await; + local.queue_receiver.read_event().await.assert_reload(); + let message = remote.queue_receiver.expect_message().await; assert_eq!(message.env_id, Some("abc123".to_string())); assert!((message.flags & MAIL_RET_HDRS) != 0); assert!((message.flags & MAIL_REQUIRETLS) != 0); diff --git a/tests/src/smtp/outbound/fallback_relay.rs b/tests/src/smtp/outbound/fallback_relay.rs index 76173bda..7c7a8831 100644 --- a/tests/src/smtp/outbound/fallback_relay.rs +++ b/tests/src/smtp/outbound/fallback_relay.rs @@ -10,7 +10,7 @@ use common::config::server::ServerProtocol; use mail_auth::MX; use store::write::now; -use crate::smtp::{outbound::TestServer, session::TestSession}; +use crate::smtp::{session::TestSession, TestSMTP}; const LOCAL: &str = r#" [queue.outbound] @@ -55,9 +55,9 @@ async fn fallback_relay() { crate::enable_logging(); // Start test server - let mut remote = TestServer::new("smtp_fallback_remote", REMOTE, true).await; + let mut remote = TestSMTP::new("smtp_fallback_remote", REMOTE).await; let _rx = remote.start(&[ServerProtocol::Smtp]).await; - let mut local = TestServer::new("smtp_fallback_local", LOCAL, true).await; + let mut local = TestSMTP::new("smtp_fallback_local", LOCAL).await; // Add mock DNS entries let core = local.build_smtp(); @@ -88,12 +88,12 @@ async fn fallback_relay() { .send_message("john@test.org", &["bill@foobar.org"], "test:no_dkim", "250") .await; local - .qr + .queue_receiver .expect_message_then_deliver() .await .try_deliver(core.clone()) .await; - let mut retry = local.qr.expect_message().await; + let mut retry = local.queue_receiver.expect_message().await; let prev_due = retry.domains[0].retry.due; let next_due = now(); let queue_id = retry.queue_id; @@ -102,11 +102,11 @@ async fn fallback_relay() { .save_changes(&core, prev_due.into(), next_due.into()) .await; local - .qr + .queue_receiver .delivery_attempt(queue_id) .await .try_deliver(core.clone()) .await; tokio::time::sleep(Duration::from_millis(100)).await; - remote.qr.expect_message().await; + remote.queue_receiver.expect_message().await; } diff --git a/tests/src/smtp/outbound/ip_lookup.rs b/tests/src/smtp/outbound/ip_lookup.rs index 23e4625c..d19fc2d4 100644 --- a/tests/src/smtp/outbound/ip_lookup.rs +++ b/tests/src/smtp/outbound/ip_lookup.rs @@ -9,7 +9,7 @@ use std::time::{Duration, Instant}; use common::config::server::ServerProtocol; use mail_auth::{IpLookupStrategy, MX}; -use crate::smtp::{outbound::TestServer, session::TestSession}; +use crate::smtp::{session::TestSession, TestSMTP}; const LOCAL: &str = r#" [session.rcpt] @@ -34,13 +34,13 @@ async fn ip_lookup_strategy() { crate::enable_logging(); // Start test server - let mut remote = TestServer::new("smtp_iplookup_remote", REMOTE, true).await; + let mut remote = TestSMTP::new("smtp_iplookup_remote", REMOTE).await; let _rx = remote.start(&[ServerProtocol::Smtp]).await; for strategy in [IpLookupStrategy::Ipv6Only, IpLookupStrategy::Ipv6thenIpv4] { //println!("-> Strategy: {:?}", strategy); // Add mock DNS entries - let mut local = TestServer::new("smtp_iplookup_local", LOCAL, true).await; + let mut local = TestSMTP::new("smtp_iplookup_local", LOCAL).await; let core = local.build_smtp(); core.core.smtp.resolvers.dns.mx_add( "foobar.org", @@ -72,16 +72,16 @@ async fn ip_lookup_strategy() { .send_message("john@test.org", &["bill@foobar.org"], "test:no_dkim", "250") .await; local - .qr + .queue_receiver .expect_message_then_deliver() .await .try_deliver(core.clone()) .await; tokio::time::sleep(Duration::from_millis(100)).await; if matches!(strategy, IpLookupStrategy::Ipv6thenIpv4) { - remote.qr.expect_message().await; + remote.queue_receiver.expect_message().await; } else { - let message = local.qr.last_queued_message().await; + let message = local.queue_receiver.last_queued_message().await; let status = message.domains[0].status.to_string(); assert!( status.contains("Connection refused"), diff --git a/tests/src/smtp/outbound/lmtp.rs b/tests/src/smtp/outbound/lmtp.rs index f27c94fc..f47e40d9 100644 --- a/tests/src/smtp/outbound/lmtp.rs +++ b/tests/src/smtp/outbound/lmtp.rs @@ -8,11 +8,11 @@ use std::time::{Duration, Instant}; use crate::smtp::{ inbound::TestMessage, - outbound::TestServer, session::{TestSession, VerifyResponse}, + TestSMTP, }; -use common::config::server::ServerProtocol; -use smtp::queue::{DeliveryAttempt, Event}; +use common::{config::server::ServerProtocol, ipc::QueueEvent}; +use smtp::queue::{spool::SmtpSpool, DeliveryAttempt}; use store::write::now; const REMOTE: &str = " @@ -66,11 +66,11 @@ async fn lmtp_delivery() { crate::enable_logging(); // Start test server - let mut remote = TestServer::new("lmtp_delivery_remote", REMOTE, true).await; + let mut remote = TestSMTP::new("lmtp_delivery_remote", REMOTE).await; let _rx = remote.start(&[ServerProtocol::Lmtp]).await; // Multiple delivery attempts - let mut local = TestServer::new("lmtp_delivery_local", LOCAL, true).await; + let mut local = TestSMTP::new("lmtp_delivery_local", LOCAL).await; // Add mock DNS entries let core = local.build_smtp(); @@ -100,17 +100,17 @@ async fn lmtp_delivery() { ) .await; local - .qr + .queue_receiver .expect_message_then_deliver() .await .try_deliver(core.clone()) .await; let mut dsn = Vec::new(); loop { - match local.qr.try_read_event().await { - Some(Event::Reload) => {} - Some(Event::OnHold(_)) => unreachable!(), - None | Some(Event::Stop) => break, + match local.queue_receiver.try_read_event().await { + Some(QueueEvent::Reload) => {} + Some(QueueEvent::OnHold(_)) => unreachable!(), + None | Some(QueueEvent::Stop) => break, } let events = core.next_event().await; @@ -133,14 +133,14 @@ async fn lmtp_delivery() { } } } - local.qr.assert_queue_is_empty().await; + local.queue_receiver.assert_queue_is_empty().await; assert_eq!(dsn.len(), 4); let mut dsn = dsn.into_iter(); dsn.next() .unwrap() - .read_lines(&local.qr) + .read_lines(&local.queue_receiver) .await .assert_contains("<bill@foobar.org> (delivered to") .assert_contains("<jane@foobar.org> (delivered to") @@ -150,28 +150,28 @@ async fn lmtp_delivery() { dsn.next() .unwrap() - .read_lines(&local.qr) + .read_lines(&local.queue_receiver) .await .assert_contains("<delay@foobar.org> (host 'lmtp.foobar.org' rejected") .assert_contains("Action: delayed"); dsn.next() .unwrap() - .read_lines(&local.qr) + .read_lines(&local.queue_receiver) .await .assert_contains("<delay@foobar.org> (host 'lmtp.foobar.org' rejected") .assert_contains("Action: delayed"); dsn.next() .unwrap() - .read_lines(&local.qr) + .read_lines(&local.queue_receiver) .await .assert_contains("<delay@foobar.org> (host 'lmtp.foobar.org' rejected") .assert_contains("Action: failed"); assert_eq!( remote - .qr + .queue_receiver .expect_message() .await .recipients @@ -184,5 +184,5 @@ async fn lmtp_delivery() { "john@foobar.org".to_string() ] ); - remote.qr.assert_no_events(); + remote.queue_receiver.assert_no_events(); } diff --git a/tests/src/smtp/outbound/mod.rs b/tests/src/smtp/outbound/mod.rs index 43334dbe..fd5900fb 100644 --- a/tests/src/smtp/outbound/mod.rs +++ b/tests/src/smtp/outbound/mod.rs @@ -4,25 +4,6 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use common::{ - config::server::{ServerProtocol, Servers}, - Core, -}; -use jmap::{api::JmapSessionManager, JMAP}; -use store::{BlobStore, Store, Stores}; -use tokio::sync::{mpsc, watch}; - -use ::smtp::core::{Inner, Session, SmtpInstance, SmtpSessionManager, SMTP}; -use utils::config::Config; - -use crate::AssertConfig; - -use super::{ - add_test_certs, - session::{DummyIo, TestSession}, - QueueReceiver, ReportReceiver, TempDir, TestSMTP, -}; - pub mod dane; pub mod extensions; pub mod fallback_relay; @@ -32,144 +13,3 @@ pub mod mta_sts; pub mod smtp; pub mod throttle; pub mod tls; - -const CONFIG: &str = r#" -[session.connect] -hostname = "'mx.example.org'" -greeting = "'Test SMTP instance'" - -[server.listener.smtp-debug] -bind = ['127.0.0.1:9925'] -protocol = 'smtp' - -[server.listener.lmtp-debug] -bind = ['127.0.0.1:9924'] -protocol = 'lmtp' -tls.implicit = true - -[server.listener.management-debug] -bind = ['127.0.0.1:9980'] -protocol = 'http' -tls.implicit = true - -[server.socket] -reuse-addr = true - -[server.tls] -enable = true -implicit = false -certificate = 'default' - -[certificate.default] -cert = '%{file:{CERT}}%' -private-key = '%{file:{PK}}%' - -[storage] -data = "sqlite" -lookup = "sqlite" -blob = "sqlite" -fts = "sqlite" - -[store."sqlite"] -type = "sqlite" -path = "{TMP}/queue.db" - -"#; - -pub struct TestServer { - pub instance: SmtpInstance, - pub temp_dir: TempDir, - pub qr: QueueReceiver, - pub rr: ReportReceiver, -} - -impl TestServer { - pub async fn new(name: &str, config: impl AsRef<str>, with_receiver: bool) -> TestServer { - let temp_dir = TempDir::new(name, true); - let mut config = - Config::new(temp_dir.update_config(add_test_certs(CONFIG) + config.as_ref())).unwrap(); - config.resolve_all_macros().await; - let stores = Stores::parse_all(&mut config).await; - let core = Core::parse(&mut config, stores, Default::default()).await; - let mut inner = Inner::default(); - let qr = if with_receiver { - inner.init_test_queue(&core) - } else { - QueueReceiver { - store: Store::default(), - blob_store: BlobStore::default(), - queue_rx: mpsc::channel(1).1, - } - }; - let rr = if with_receiver { - inner.init_test_report() - } else { - ReportReceiver { - report_rx: mpsc::channel(1).1, - } - }; - - TestServer { - instance: SmtpInstance::new(core.into_shared(), inner), - temp_dir, - qr, - rr, - } - } - - pub async fn start(&self, protocols: &[ServerProtocol]) -> watch::Sender<bool> { - // Spawn listeners - let mut config = Config::new(CONFIG).unwrap(); - let mut servers = Servers::parse(&mut config); - servers.parse_tcp_acceptors(&mut config, self.instance.core.clone()); - - // Filter out protocols - servers - .servers - .retain(|server| protocols.contains(&server.protocol)); - - // Start servers - servers.bind_and_drop_priv(&mut config); - let instance = self.instance.clone(); - let smtp_manager = SmtpSessionManager::new(instance.clone()); - let jmap = JMAP::init( - &mut config, - mpsc::channel(1).1, - instance.core.clone(), - instance.inner.clone(), - ) - .await; - let jmap_manager = JmapSessionManager::new(jmap); - config.assert_no_errors(); - - servers - .spawn(|server, acceptor, shutdown_rx| { - match &server.protocol { - ServerProtocol::Smtp | ServerProtocol::Lmtp => server.spawn( - smtp_manager.clone(), - instance.core.clone(), - acceptor, - shutdown_rx, - ), - ServerProtocol::Http => server.spawn( - jmap_manager.clone(), - instance.core.clone(), - acceptor, - shutdown_rx, - ), - ServerProtocol::Imap | ServerProtocol::Pop3 | ServerProtocol::ManageSieve => { - unreachable!() - } - }; - }) - .0 - } - - pub fn new_session(&self) -> Session<DummyIo> { - Session::test(self.build_smtp()) - } - - pub fn build_smtp(&self) -> SMTP { - SMTP::from(self.instance.clone()) - } -} diff --git a/tests/src/smtp/outbound/mta_sts.rs b/tests/src/smtp/outbound/mta_sts.rs index bae82bd7..29e8a605 100644 --- a/tests/src/smtp/outbound/mta_sts.rs +++ b/tests/src/smtp/outbound/mta_sts.rs @@ -9,7 +9,10 @@ use std::{ time::{Duration, Instant}, }; -use common::config::{server::ServerProtocol, smtp::resolver::Policy}; +use common::{ + config::{server::ServerProtocol, smtp::resolver::Policy}, + ipc::PolicyType, +}; use mail_auth::{ common::parse::TxtRecordParser, mta_sts::{MtaSts, ReportUri, TlsRpt}, @@ -19,13 +22,10 @@ use mail_auth::{ use crate::smtp::{ inbound::{TestMessage, TestQueueEvent, TestReportingEvent}, - outbound::TestServer, session::{TestSession, VerifyResponse}, + TestSMTP, }; -use smtp::{ - outbound::mta_sts::{lookup::STS_TEST_POLICY, parse::ParsePolicy}, - reporting::PolicyType, -}; +use smtp::outbound::mta_sts::{lookup::STS_TEST_POLICY, parse::ParsePolicy}; const LOCAL: &str = r#" [session.rcpt] @@ -63,11 +63,11 @@ async fn mta_sts_verify() { crate::enable_logging(); // Start test server - let mut remote = TestServer::new("smtp_mta_sts_remote", REMOTE, true).await; + let mut remote = TestSMTP::new("smtp_mta_sts_remote", REMOTE).await; let _rx = remote.start(&[ServerProtocol::Smtp]).await; // Fail on missing MTA-STS record - let mut local = TestServer::new("smtp_mta_sts_local", LOCAL, true).await; + let mut local = TestSMTP::new("smtp_mta_sts_local", LOCAL).await; // Add mock DNS entries let core = local.build_smtp(); @@ -98,23 +98,23 @@ async fn mta_sts_verify() { .send_message("john@test.org", &["bill@foobar.org"], "test:no_dkim", "250") .await; local - .qr + .queue_receiver .expect_message_then_deliver() .await .try_deliver(core.clone()) .await; local - .qr + .queue_receiver .expect_message() .await - .read_lines(&local.qr) + .read_lines(&local.queue_receiver) .await .assert_contains("<bill@foobar.org> (MTA-STS failed to authenticate") .assert_contains("Record not found"); - local.qr.read_event().await.assert_reload(); + local.queue_receiver.read_event().await.assert_reload(); // Expect TLS failure report - let report = local.rr.read_report().await.unwrap_tls(); + let report = local.report_receiver.read_report().await.unwrap_tls(); assert_eq!(report.domain, "foobar.org"); assert_eq!(report.policy, PolicyType::Sts(None)); assert_eq!( @@ -136,23 +136,23 @@ async fn mta_sts_verify() { .send_message("john@test.org", &["bill@foobar.org"], "test:no_dkim", "250") .await; local - .qr + .queue_receiver .expect_message_then_deliver() .await .try_deliver(core.clone()) .await; local - .qr + .queue_receiver .expect_message() .await - .read_lines(&local.qr) + .read_lines(&local.queue_receiver) .await .assert_contains("<bill@foobar.org> (MTA-STS failed to authenticate") .assert_contains("No 'mx' entries found"); - local.qr.read_event().await.assert_reload(); + local.queue_receiver.read_event().await.assert_reload(); // Expect TLS failure report - let report = local.rr.read_report().await.unwrap_tls(); + let report = local.report_receiver.read_report().await.unwrap_tls(); assert_eq!(report.policy, PolicyType::Sts(None)); assert_eq!( report.failure.as_ref().unwrap().result_type, @@ -171,23 +171,23 @@ async fn mta_sts_verify() { .send_message("john@test.org", &["bill@foobar.org"], "test:no_dkim", "250") .await; local - .qr + .queue_receiver .expect_message_then_deliver() .await .try_deliver(core.clone()) .await; local - .qr + .queue_receiver .expect_message() .await - .read_lines(&local.qr) + .read_lines(&local.queue_receiver) .await .assert_contains("<bill@foobar.org> (MTA-STS failed to authenticate") .assert_contains("not authorized by policy"); - local.qr.read_event().await.assert_reload(); + local.queue_receiver.read_event().await.assert_reload(); // Expect TLS failure report - let report = local.rr.read_report().await.unwrap_tls(); + let report = local.report_receiver.read_report().await.unwrap_tls(); assert_eq!( report.policy, PolicyType::Sts( @@ -202,7 +202,7 @@ async fn mta_sts_verify() { report.failure.as_ref().unwrap().result_type, ResultType::ValidationFailure ); - remote.qr.assert_no_events(); + remote.queue_receiver.assert_no_events(); // MTA-STS successful validation core.core.smtp.resolvers.dns.txt_add( @@ -222,22 +222,22 @@ async fn mta_sts_verify() { .send_message("john@test.org", &["bill@foobar.org"], "test:no_dkim", "250") .await; local - .qr + .queue_receiver .expect_message_then_deliver() .await .try_deliver(core.clone()) .await; - local.qr.read_event().await.assert_reload(); + local.queue_receiver.read_event().await.assert_reload(); remote - .qr + .queue_receiver .expect_message() .await - .read_lines(&remote.qr) + .read_lines(&remote.queue_receiver) .await .assert_contains("using TLSv1.3 with cipher"); // Expect TLS success report - let report = local.rr.read_report().await.unwrap_tls(); + let report = local.report_receiver.read_report().await.unwrap_tls(); assert_eq!( report.policy, PolicyType::Sts( diff --git a/tests/src/smtp/outbound/smtp.rs b/tests/src/smtp/outbound/smtp.rs index f24ffb1f..13b2f7ff 100644 --- a/tests/src/smtp/outbound/smtp.rs +++ b/tests/src/smtp/outbound/smtp.rs @@ -6,16 +6,16 @@ use std::time::{Duration, Instant}; -use common::config::server::ServerProtocol; +use common::{config::server::ServerProtocol, ipc::QueueEvent}; use mail_auth::MX; use store::write::now; use crate::smtp::{ inbound::{TestMessage, TestQueueEvent}, - outbound::TestServer, session::{TestSession, VerifyResponse}, + TestSMTP, }; -use smtp::queue::{DeliveryAttempt, Event}; +use smtp::queue::{spool::SmtpSpool, DeliveryAttempt}; const LOCAL: &str = r#" [session.rcpt] @@ -74,12 +74,12 @@ async fn smtp_delivery() { crate::enable_logging(); // Start test server - let mut remote = TestServer::new("smtp_delivery_remote", REMOTE, true).await; + let mut remote = TestSMTP::new("smtp_delivery_remote", REMOTE).await; let _rx = remote.start(&[ServerProtocol::Smtp]).await; let remote_core = remote.build_smtp(); // Multiple delivery attempts - let mut local = TestServer::new("smtp_delivery_local", LOCAL, true).await; + let mut local = TestSMTP::new("smtp_delivery_local", LOCAL).await; // Add mock DNS entries let core = local.build_smtp(); @@ -124,11 +124,11 @@ async fn smtp_delivery() { "250", ) .await; - let message = local.qr.expect_message().await; + let message = local.queue_receiver.expect_message().await; let num_domains = message.domains.len(); assert_eq!(num_domains, 3); local - .qr + .queue_receiver .delivery_attempt(message.queue_id) .await .try_deliver(core.clone()) @@ -136,10 +136,10 @@ async fn smtp_delivery() { let mut dsn = Vec::new(); let mut domain_retries = vec![0; num_domains]; loop { - match local.qr.try_read_event().await { - Some(Event::Reload) => {} - Some(Event::OnHold(_)) => unreachable!(), - None | Some(Event::Stop) => break, + match local.queue_receiver.try_read_event().await { + Some(QueueEvent::Reload) => {} + Some(QueueEvent::OnHold(_)) => unreachable!(), + None | Some(QueueEvent::Stop) => break, } let events = core.next_event().await; @@ -173,14 +173,14 @@ async fn smtp_delivery() { "retries {domain_retries:?}" ); - local.qr.assert_queue_is_empty().await; + local.queue_receiver.assert_queue_is_empty().await; assert_eq!(dsn.len(), 5); let mut dsn = dsn.into_iter(); dsn.next() .unwrap() - .read_lines(&local.qr) + .read_lines(&local.queue_receiver) .await .assert_contains("<ok@foobar.net> (delivered to") .assert_contains("<ok@foobar.org> (delivered to") @@ -190,7 +190,7 @@ async fn smtp_delivery() { dsn.next() .unwrap() - .read_lines(&local.qr) + .read_lines(&local.queue_receiver) .await .assert_contains("<delay@foobar.net> (host ") .assert_contains("<delay@foobar.org> (host ") @@ -198,27 +198,27 @@ async fn smtp_delivery() { dsn.next() .unwrap() - .read_lines(&local.qr) + .read_lines(&local.queue_receiver) .await .assert_contains("<delay@foobar.org> (host ") .assert_contains("Action: delayed"); dsn.next() .unwrap() - .read_lines(&local.qr) + .read_lines(&local.queue_receiver) .await .assert_contains("<delay@foobar.org> (host "); dsn.next() .unwrap() - .read_lines(&local.qr) + .read_lines(&local.queue_receiver) .await .assert_contains("<delay@foobar.net> (host ") .assert_contains("Action: failed"); assert_eq!( remote - .qr + .queue_receiver .consume_message(&remote_core) .await .recipients @@ -229,7 +229,7 @@ async fn smtp_delivery() { ); assert_eq!( remote - .qr + .queue_receiver .consume_message(&remote_core) .await .recipients @@ -239,7 +239,7 @@ async fn smtp_delivery() { vec!["ok@foobar.net".to_string()] ); - remote.qr.assert_no_events(); + remote.queue_receiver.assert_no_events(); // SMTP smuggling for separator in ["\n", "\r"].iter() { @@ -256,18 +256,18 @@ async fn smtp_delivery() { .send_message("john@doe.org", &["bill@foobar.com"], &message, "250") .await; local - .qr + .queue_receiver .expect_message_then_deliver() .await .try_deliver(core.clone()) .await; - local.qr.read_event().await.assert_reload(); + local.queue_receiver.read_event().await.assert_reload(); let message = remote - .qr + .queue_receiver .consume_message(&remote_core) .await - .read_message(&remote.qr) + .read_message(&remote.queue_receiver) .await; assert!( diff --git a/tests/src/smtp/outbound/throttle.rs b/tests/src/smtp/outbound/throttle.rs index 2ea8223a..9c625a51 100644 --- a/tests/src/smtp/outbound/throttle.rs +++ b/tests/src/smtp/outbound/throttle.rs @@ -13,10 +13,9 @@ use mail_auth::MX; use store::write::now; use crate::smtp::{ - inbound::TestQueueEvent, outbound::TestServer, queue::manager::new_message, - session::TestSession, + inbound::TestQueueEvent, queue::manager::new_message, session::TestSession, TestSMTP, }; -use smtp::queue::{Domain, Message, QueueEnvelope, Schedule, Status}; +use smtp::queue::{throttle::IsAllowed, Domain, Message, QueueEnvelope, Schedule, Status}; const CONFIG: &str = r#" [session.rcpt] @@ -73,7 +72,7 @@ async fn throttle_outbound() { let mut test_message = new_message(0); test_message.return_path_domain = "foobar.org".to_string(); - let mut local = TestServer::new("smtp_throttle_outbound", CONFIG, true).await; + let mut local = TestSMTP::new("smtp_throttle_outbound", CONFIG).await; let core = local.build_smtp(); let mut session = local.new_session(); @@ -83,7 +82,10 @@ async fn throttle_outbound() { session .send_message("john@foobar.org", &["bill@test.org"], "test:no_dkim", "250") .await; - assert_eq!(local.qr.last_queued_due().await as i64 - now() as i64, 0); + assert_eq!( + local.queue_receiver.last_queued_due().await as i64 - now() as i64, + 0 + ); // Throttle sender let mut in_flight = vec![]; @@ -102,13 +104,13 @@ async fn throttle_outbound() { // Expect concurrency throttle for sender domain 'foobar.org' local - .qr + .queue_receiver .expect_message_then_deliver() .await .try_deliver(core.clone()) .await; tokio::time::sleep(Duration::from_millis(100)).await; - local.qr.read_event().await.unwrap_on_hold(); + local.queue_receiver.read_event().await.unwrap_on_hold(); in_flight.clear(); // Expect rate limit throttle for sender domain 'foobar.net' @@ -128,14 +130,14 @@ async fn throttle_outbound() { .send_message("john@foobar.net", &["bill@test.org"], "test:no_dkim", "250") .await; local - .qr + .queue_receiver .expect_message_then_deliver() .await .try_deliver(core.clone()) .await; tokio::time::sleep(Duration::from_millis(100)).await; - local.qr.read_event().await.assert_reload(); - let due = local.qr.last_queued_due().await - now(); + local.queue_receiver.read_event().await.assert_reload(); + let due = local.queue_receiver.last_queued_due().await - now(); assert!(due > 0, "Due: {}", due); // Expect concurrency throttle for recipient domain 'example.org' @@ -167,13 +169,13 @@ async fn throttle_outbound() { ) .await; local - .qr + .queue_receiver .expect_message_then_deliver() .await .try_deliver(core.clone()) .await; tokio::time::sleep(Duration::from_millis(100)).await; - local.qr.read_event().await.unwrap_on_hold(); + local.queue_receiver.read_event().await.unwrap_on_hold(); in_flight.clear(); // Expect rate limit throttle for recipient domain 'example.net' @@ -204,14 +206,14 @@ async fn throttle_outbound() { ) .await; local - .qr + .queue_receiver .expect_message_then_deliver() .await .try_deliver(core.clone()) .await; tokio::time::sleep(Duration::from_millis(100)).await; - local.qr.read_event().await.assert_reload(); - let due = local.qr.last_queued_due().await - now(); + local.queue_receiver.read_event().await.assert_reload(); + let due = local.queue_receiver.last_queued_due().await - now(); assert!(due > 0, "Due: {}", due); // Expect concurrency throttle for mx 'mx.test.org' @@ -250,12 +252,12 @@ async fn throttle_outbound() { .send_message("john@test.net", &["jane@test.org"], "test:no_dkim", "250") .await; local - .qr + .queue_receiver .expect_message_then_deliver() .await .try_deliver(core.clone()) .await; - local.qr.read_event().await.unwrap_on_hold(); + local.queue_receiver.read_event().await.unwrap_on_hold(); in_flight.clear(); // Expect rate limit throttle for mx 'mx.test.net' @@ -287,15 +289,15 @@ async fn throttle_outbound() { .send_message("john@test.net", &["jane@test.net"], "test:no_dkim", "250") .await; local - .qr + .queue_receiver .expect_message_then_deliver() .await .try_deliver(core.clone()) .await; tokio::time::sleep(Duration::from_millis(100)).await; - local.qr.read_event().await.assert_reload(); - let due = local.qr.last_queued_due().await - now(); + local.queue_receiver.read_event().await.assert_reload(); + let due = local.queue_receiver.last_queued_due().await - now(); assert!(due > 0, "Due: {}", due); } diff --git a/tests/src/smtp/outbound/tls.rs b/tests/src/smtp/outbound/tls.rs index 32c712ce..105c7f3e 100644 --- a/tests/src/smtp/outbound/tls.rs +++ b/tests/src/smtp/outbound/tls.rs @@ -12,8 +12,8 @@ use store::write::now; use crate::smtp::{ inbound::TestMessage, - outbound::TestServer, session::{TestSession, VerifyResponse}, + TestSMTP, }; const LOCAL: &str = r#" @@ -47,11 +47,11 @@ async fn starttls_optional() { crate::enable_logging(); // Start test server - let mut remote = TestServer::new("smtp_starttls_remote", REMOTE, true).await; + let mut remote = TestSMTP::new("smtp_starttls_remote", REMOTE).await; let _rx = remote.start(&[ServerProtocol::Smtp]).await; // Retry on failed STARTTLS - let mut local = TestServer::new("smtp_starttls_local", LOCAL, true).await; + let mut local = TestSMTP::new("smtp_starttls_local", LOCAL).await; // Add mock DNS entries let core = local.build_smtp(); @@ -77,12 +77,12 @@ async fn starttls_optional() { .send_message("john@test.org", &["bill@foobar.org"], "test:no_dkim", "250") .await; local - .qr + .queue_receiver .expect_message_then_deliver() .await .try_deliver(core.clone()) .await; - let mut retry = local.qr.expect_message().await; + let mut retry = local.queue_receiver.expect_message().await; let prev_due = retry.domains[0].retry.due; let next_due = now(); let queue_id = retry.queue_id; @@ -91,17 +91,17 @@ async fn starttls_optional() { .save_changes(&core, prev_due.into(), next_due.into()) .await; local - .qr + .queue_receiver .delivery_attempt(queue_id) .await .try_deliver(core.clone()) .await; tokio::time::sleep(Duration::from_millis(100)).await; remote - .qr + .queue_receiver .expect_message() .await - .read_lines(&remote.qr) + .read_lines(&remote.queue_receiver) .await .assert_not_contains("using TLSv1.3 with cipher"); } diff --git a/tests/src/smtp/queue/concurrent.rs b/tests/src/smtp/queue/concurrent.rs index 8f40e231..ef66190f 100644 --- a/tests/src/smtp/queue/concurrent.rs +++ b/tests/src/smtp/queue/concurrent.rs @@ -9,7 +9,7 @@ use std::time::{Duration, Instant}; use common::config::server::ServerProtocol; use mail_auth::MX; -use crate::smtp::{outbound::TestServer, session::TestSession}; +use crate::smtp::{session::TestSession, TestSMTP}; use smtp::queue::manager::Queue; const LOCAL: &str = r#" @@ -37,10 +37,10 @@ async fn concurrent_queue() { crate::enable_logging(); // Start test server - let remote = TestServer::new("smtp_concurrent_queue_remote", REMOTE, true).await; + let remote = TestSMTP::new("smtp_concurrent_queue_remote", REMOTE).await; let _rx = remote.start(&[ServerProtocol::Smtp]).await; - let local = TestServer::new("smtp_concurrent_queue_local", LOCAL, true).await; + let local = TestSMTP::new("smtp_concurrent_queue_local", LOCAL).await; // Add mock DNS entries let core = local.build_smtp(); @@ -72,22 +72,22 @@ async fn concurrent_queue() { // Spawn 20 concurrent queues at different times for _ in 0..10 { - let local = local.instance.clone(); + let local = local.server.clone(); tokio::spawn(async move { - Queue::new(local).process_events().await; + Queue::new(local.inner).process_events().await; }); } tokio::time::sleep(Duration::from_millis(500)).await; for _ in 0..10 { - let local = local.instance.clone(); + let local = local.server.clone(); tokio::spawn(async move { - Queue::new(local).process_events().await; + Queue::new(local.inner).process_events().await; }); } tokio::time::sleep(Duration::from_millis(1500)).await; - local.qr.assert_queue_is_empty().await; - let remote_messages = remote.qr.read_queued_messages().await; + local.queue_receiver.assert_queue_is_empty().await; + let remote_messages = remote.queue_receiver.read_queued_messages().await; assert_eq!(remote_messages.len(), 100); // Make sure local store is queue diff --git a/tests/src/smtp/queue/dsn.rs b/tests/src/smtp/queue/dsn.rs index 9397edee..75624ba9 100644 --- a/tests/src/smtp/queue/dsn.rs +++ b/tests/src/smtp/queue/dsn.rs @@ -10,9 +10,9 @@ use smtp_proto::{Response, RCPT_NOTIFY_DELAY, RCPT_NOTIFY_FAILURE, RCPT_NOTIFY_S use store::write::now; use utils::BlobHash; -use crate::smtp::{inbound::sign::SIGNATURES, outbound::TestServer, QueueReceiver}; +use crate::smtp::{inbound::sign::SIGNATURES, QueueReceiver, TestSMTP}; use smtp::queue::{ - Domain, Error, ErrorDetails, HostResponse, Message, Recipient, Schedule, Status, + dsn::SendDsn, Domain, Error, ErrorDetails, HostResponse, Message, Recipient, Schedule, Status, }; const CONFIG: &str = r#" @@ -92,9 +92,9 @@ async fn generate_dsn() { }; // Load config - let mut local = TestServer::new("smtp_dsn_test", CONFIG.to_string() + SIGNATURES, true).await; + let mut local = TestSMTP::new("smtp_dsn_test", CONFIG.to_string() + SIGNATURES).await; let core = local.build_smtp(); - let qr = &mut local.qr; + let qr = &mut local.queue_receiver; // Create temp dir for queue qr.blob_store diff --git a/tests/src/smtp/queue/manager.rs b/tests/src/smtp/queue/manager.rs index 58b749c2..2f9684cf 100644 --- a/tests/src/smtp/queue/manager.rs +++ b/tests/src/smtp/queue/manager.rs @@ -8,10 +8,10 @@ use std::time::Duration; use mail_auth::hickory_resolver::proto::op::ResponseCode; -use smtp::queue::{Domain, Message, Schedule, Status}; +use smtp::queue::{spool::SmtpSpool, Domain, Message, Schedule, Status}; use store::write::now; -use crate::smtp::outbound::TestServer; +use crate::smtp::TestSMTP; const CONFIG: &str = r#" [session.ehlo] @@ -26,9 +26,9 @@ async fn queue_due() { // Enable logging crate::enable_logging(); - let local = TestServer::new("smtp_queue_due_test", CONFIG, true).await; + let local = TestSMTP::new("smtp_queue_due_test", CONFIG).await; let core = local.build_smtp(); - let qr = &local.qr; + let qr = &local.queue_receiver; let mut message = new_message(0); message.domains.push(domain("c", 3, 8, 9)); diff --git a/tests/src/smtp/queue/retry.rs b/tests/src/smtp/queue/retry.rs index aaaaea60..090ca34e 100644 --- a/tests/src/smtp/queue/retry.rs +++ b/tests/src/smtp/queue/retry.rs @@ -8,10 +8,11 @@ use std::time::Duration; use crate::smtp::{ inbound::{TestMessage, TestQueueEvent}, - outbound::TestServer, session::{TestSession, VerifyResponse}, + TestSMTP, }; -use smtp::queue::{DeliveryAttempt, Event}; +use common::ipc::QueueEvent; +use smtp::queue::{spool::SmtpSpool, DeliveryAttempt}; use store::write::now; const CONFIG: &str = r#" @@ -39,12 +40,12 @@ async fn queue_retry() { crate::enable_logging(); // Create temp dir for queue - let mut local = TestServer::new("smtp_queue_retry_test", CONFIG, true).await; + let mut local = TestSMTP::new("smtp_queue_retry_test", CONFIG).await; // Create test message let core = local.build_smtp(); let mut session = local.new_session(); - let qr = &mut local.qr; + let qr = &mut local.queue_receiver; session.data.remote_ip_str = "10.0.0.1".to_string(); session.eval_session_params().await; @@ -85,9 +86,9 @@ async fn queue_retry() { attempt.try_deliver(core.clone()).await; loop { match qr.try_read_event().await { - Some(Event::Reload) => {} - Some(Event::OnHold(_)) => unreachable!(), - None | Some(Event::Stop) => break, + Some(QueueEvent::Reload) => {} + Some(QueueEvent::OnHold(_)) => unreachable!(), + None | Some(QueueEvent::Stop) => break, } let now = now(); diff --git a/tests/src/smtp/reporting/analyze.rs b/tests/src/smtp/reporting/analyze.rs index 9a28ae1d..529f7e58 100644 --- a/tests/src/smtp/reporting/analyze.rs +++ b/tests/src/smtp/reporting/analyze.rs @@ -6,7 +6,7 @@ use std::time::Duration; -use crate::smtp::{inbound::TestQueueEvent, outbound::TestServer, session::TestSession}; +use crate::smtp::{inbound::TestQueueEvent, session::TestSession, TestSMTP}; use store::{ write::{ReportClass, ValueClass}, @@ -32,11 +32,11 @@ async fn report_analyze() { crate::enable_logging(); // Create temp dir for queue - let mut local = TestServer::new("smtp_analyze_report_test", CONFIG, true).await; + let mut local = TestSMTP::new("smtp_analyze_report_test", CONFIG).await; // Create test message let mut session = local.new_session(); - let qr = &mut local.qr; + let qr = &mut local.queue_receiver; session.data.remote_ip_str = "10.0.0.1".to_string(); session.eval_session_params().await; session.ehlo("mx.test.org").await; diff --git a/tests/src/smtp/reporting/dmarc.rs b/tests/src/smtp/reporting/dmarc.rs index 5e4ea4be..363f3007 100644 --- a/tests/src/smtp/reporting/dmarc.rs +++ b/tests/src/smtp/reporting/dmarc.rs @@ -10,20 +10,19 @@ use std::{ time::{Duration, Instant}, }; -use common::config::smtp::report::AggregateFrequency; +use common::{config::smtp::report::AggregateFrequency, ipc::DmarcEvent}; use mail_auth::{ common::parse::TxtRecordParser, dmarc::Dmarc, report::{ActionDisposition, Disposition, DmarcResult, Record, Report}, }; +use smtp::reporting::dmarc::DmarcReporting; use store::write::QueueClass; -use smtp::reporting::DmarcEvent; - use crate::smtp::{ inbound::{sign::SIGNATURES, TestMessage}, - outbound::TestServer, session::VerifyResponse, + TestSMTP, }; const CONFIG: &str = r#" @@ -53,12 +52,7 @@ async fn report_dmarc() { crate::enable_logging(); // Create scheduler - let mut local = TestServer::new( - "smtp_report_dmarc_test", - CONFIG.to_string() + SIGNATURES, - true, - ) - .await; + let mut local = TestSMTP::new("smtp_report_dmarc_test", CONFIG.to_string() + SIGNATURES).await; // Authorize external report for foobar.org let core = local.build_smtp(); @@ -67,7 +61,7 @@ async fn report_dmarc() { Dmarc::parse(b"v=DMARC1;").unwrap(), Instant::now() + Duration::from_secs(10), ); - let qr = &mut local.qr; + let qr = &mut local.queue_receiver; // Schedule two events with a same policy and another one with a different policy let dmarc_record = Arc::new( diff --git a/tests/src/smtp/reporting/scheduler.rs b/tests/src/smtp/reporting/scheduler.rs index 4b9b5d97..306339ef 100644 --- a/tests/src/smtp/reporting/scheduler.rs +++ b/tests/src/smtp/reporting/scheduler.rs @@ -6,7 +6,10 @@ use std::sync::Arc; -use common::config::smtp::report::AggregateFrequency; +use common::{ + config::smtp::report::AggregateFrequency, + ipc::{DmarcEvent, PolicyType, TlsEvent}, +}; use mail_auth::{ common::parse::TxtRecordParser, dmarc::{Dmarc, URI}, @@ -15,8 +18,12 @@ use mail_auth::{ }; use store::write::QueueClass; -use crate::smtp::outbound::TestServer; -use smtp::reporting::{dmarc::DmarcFormat, DmarcEvent, PolicyType, TlsEvent}; +use smtp::reporting::{ + dmarc::{DmarcFormat, DmarcReporting}, + tls::TlsReporting, +}; + +use crate::smtp::TestSMTP; const CONFIG: &str = r#" [session.rcpt] @@ -37,9 +44,9 @@ async fn report_scheduler() { crate::enable_logging(); // Create scheduler - let local = TestServer::new("smtp_report_queue_test", CONFIG, true).await; + let local = TestSMTP::new("smtp_report_queue_test", CONFIG).await; let core = local.build_smtp(); - let qr = &local.qr; + let qr = &local.queue_receiver; // Schedule two events with a same policy and another one with a different policy let dmarc_record = diff --git a/tests/src/smtp/reporting/tls.rs b/tests/src/smtp/reporting/tls.rs index e64d5234..dc2d550e 100644 --- a/tests/src/smtp/reporting/tls.rs +++ b/tests/src/smtp/reporting/tls.rs @@ -6,7 +6,7 @@ use std::{io::Read, sync::Arc, time::Duration}; -use common::config::smtp::report::AggregateFrequency; +use common::{config::smtp::report::AggregateFrequency, ipc::TlsEvent}; use mail_auth::{ common::parse::TxtRecordParser, flate2::read::GzDecoder, @@ -15,12 +15,12 @@ use mail_auth::{ }; use store::write::QueueClass; -use smtp::reporting::{tls::TLS_HTTP_REPORT, TlsEvent}; +use smtp::reporting::tls::{TlsReporting, TLS_HTTP_REPORT}; use crate::smtp::{ inbound::{sign::SIGNATURES, TestMessage}, - outbound::TestServer, session::VerifyResponse, + TestSMTP, }; const CONFIG: &str = r#" @@ -46,14 +46,9 @@ async fn report_tls() { crate::enable_logging(); // Create scheduler - let mut local = TestServer::new( - "smtp_report_tls_test", - CONFIG.to_string() + SIGNATURES, - true, - ) - .await; + let mut local = TestSMTP::new("smtp_report_tls_test", CONFIG.to_string() + SIGNATURES).await; let core = local.build_smtp(); - let qr = &mut local.qr; + let qr = &mut local.queue_receiver; // Schedule TLS reports to be delivered via email let tls_record = Arc::new(TlsRpt::parse(b"v=TLSRPTv1;rua=mailto:reports@foobar.org").unwrap()); @@ -62,7 +57,7 @@ async fn report_tls() { // Add two successful records core.schedule_tls(Box::new(TlsEvent { domain: "foobar.org".to_string(), - policy: smtp::reporting::PolicyType::None, + policy: common::ipc::PolicyType::None, failure: None, tls_record: tls_record.clone(), interval: AggregateFrequency::Daily, @@ -72,23 +67,20 @@ async fn report_tls() { for (policy, rt) in [ ( - smtp::reporting::PolicyType::None, // Quota limited at 1532 bytes, this should not be included in the report. + common::ipc::PolicyType::None, // Quota limited at 1532 bytes, this should not be included in the report. ResultType::CertificateExpired, ), + (common::ipc::PolicyType::Tlsa(None), ResultType::TlsaInvalid), ( - smtp::reporting::PolicyType::Tlsa(None), - ResultType::TlsaInvalid, - ), - ( - smtp::reporting::PolicyType::Sts(None), + common::ipc::PolicyType::Sts(None), ResultType::StsPolicyFetchError, ), ( - smtp::reporting::PolicyType::Sts(None), + common::ipc::PolicyType::Sts(None), ResultType::StsPolicyInvalid, ), ( - smtp::reporting::PolicyType::Sts(None), + common::ipc::PolicyType::Sts(None), ResultType::StsWebpkiInvalid, ), ] { @@ -192,7 +184,7 @@ async fn report_tls() { // Add two successful records core.schedule_tls(Box::new(TlsEvent { domain: "foobar.org".to_string(), - policy: smtp::reporting::PolicyType::None, + policy: common::ipc::PolicyType::None, failure: None, tls_record: tls_record.clone(), interval: AggregateFrequency::Daily, diff --git a/tests/src/smtp/session.rs b/tests/src/smtp/session.rs index fd1173c0..3ba4d837 100644 --- a/tests/src/smtp/session.rs +++ b/tests/src/smtp/session.rs @@ -9,6 +9,7 @@ use std::{borrow::Cow, path::PathBuf, sync::Arc}; use common::{ config::server::ServerProtocol, listener::{limiter::ConcurrencyLimiter, ServerInstance, SessionStream, TcpAcceptor}, + Server, }; use rustls::{server::ResolvesServerCert, ServerConfig}; use tokio::{ @@ -16,7 +17,7 @@ use tokio::{ sync::watch, }; -use smtp::core::{Session, SessionAddress, SessionData, SessionParameters, State, SMTP}; +use smtp::core::{Session, SessionAddress, SessionData, SessionParameters, State}; use tokio_rustls::TlsAcceptor; use utils::snowflake::SnowflakeIdGenerator; @@ -81,8 +82,8 @@ impl Unpin for DummyIo {} #[allow(async_fn_in_trait)] pub trait TestSession { - fn test(core: SMTP) -> Self; - fn test_with_shutdown(core: SMTP, shutdown_rx: watch::Receiver<bool>) -> Self; + fn test(server: Server) -> Self; + fn test_with_shutdown(server: Server, shutdown_rx: watch::Receiver<bool>) -> Self; fn response(&mut self) -> Vec<String>; fn write_rx(&mut self, data: &str); async fn rset(&mut self); @@ -96,11 +97,11 @@ pub trait TestSession { } impl TestSession for Session<DummyIo> { - fn test_with_shutdown(core: SMTP, shutdown_rx: watch::Receiver<bool>) -> Self { + fn test_with_shutdown(server: Server, shutdown_rx: watch::Receiver<bool>) -> Self { Self { state: State::default(), instance: Arc::new(ServerInstance::test_with_shutdown(shutdown_rx)), - core, + server, stream: DummyIo { rx_buf: vec![], tx_buf: vec![], @@ -119,8 +120,8 @@ impl TestSession for Session<DummyIo> { } } - fn test(core: SMTP) -> Self { - Self::test_with_shutdown(core, watch::channel(false).1) + fn test(server: Server) -> Self { + Self::test_with_shutdown(server, watch::channel(false).1) } fn response(&mut self) -> Vec<String> { @@ -258,7 +259,7 @@ impl TestSession for Session<DummyIo> { dsn_info: None, }, ], - self.core.inner.queue_id_gen.generate().unwrap(), + self.server.inner.data.queue_id_gen.generate().unwrap(), 0, ) .await; |