diff options
| -rw-r--r-- | CHANGELOG.md | 18 | ||||
| -rw-r--r-- | README.md | 10 | ||||
| -rw-r--r-- | UPGRADING.md | 5 | ||||
| -rw-r--r-- | crates/install/Cargo.toml | 4 | ||||
| -rw-r--r-- | crates/install/src/main.rs | 374 | ||||
| -rw-r--r-- | crates/smtp/src/scripts/plugins/bayes.rs | 2 | ||||
| -rw-r--r-- | tests/src/smtp/inbound/antispam.rs | 2 |
7 files changed, 240 insertions, 175 deletions
diff --git a/CHANGELOG.md b/CHANGELOG.md index 19514dc7..326bc7eb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,24 @@ All notable changes to this project will be documented in this file. This project adheres to [Semantic Versioning](http://semver.org/). +## [0.4.0] - 2023-10-25 + +This version introduces some breaking changes in the configuration file. Please read the [UPGRADING.md](UPGRADING.md) file for more information. + +## Added +- Built-in Spam and Phishing filter. +- Scheduled queries on some directory types. +- In-memory maps and lists containing glob or regex patterns. +- Remote retrieval of in-memory list/maps with fallback mechanisms. +- Macros and support for including files from TOML config files. + +### Changed +- `config.toml` is now split in multiple TOML files for better organization. +- **BREAKING:** Configuration key prefix `jmap.sieve` (JMAP Sieve Interpreter) has been renamed to `sieve.untrusted`. +- **BREAKING:** Configuration key prefix `sieve` (SMTP Sieve Interpreter) has been renamed to `sieve.trusted`. + +### Fixed + ## [0.3.10] - 2023-10-17 ## Added @@ -26,6 +26,16 @@ Key features: - Inbound throttling and filtering with granular configuration rules, sieve scripting and milter integration. - Virtual queues with delayed delivery, priority delivery, quotas, routing rules and throttling support. - Envelope rewriting and message modification. +- **Spam and Phishing** filter: + - Comprehensive set of filtering **rules** on par with popular solutions. + - Statistical **spam classifier** with automatic training capabilities. + - DNS Blocklists (**DNSBLs**) checking of IP addresses, domains, and hashes. + - Collaborative digest-based spam filtering with **Pyzor**. + - **Phishing** protection against homographic URL attacks, sender spoofing and other techniques. + - Trusted **reply** tracking to recognize and prioritize genuine e-mail replies. + - Sender **reputation** monitoring by IP address, ASN, domain and email address. + - **Greylisting** to temporarily defer unknown senders. + - **Spam traps** to set up decoy email addresses that catch and analyze spam. - **Flexible**: - **LDAP** directory and **SQL** database authentication. - Full-text search available in 17 languages. diff --git a/UPGRADING.md b/UPGRADING.md new file mode 100644 index 00000000..06139d5d --- /dev/null +++ b/UPGRADING.md @@ -0,0 +1,5 @@ +Upgrading from `v0.3.x` to `v0.4.0` +----------------------------------- + +The contents of this file will be made available as soon as the CI compile job finishes. + diff --git a/crates/install/Cargo.toml b/crates/install/Cargo.toml index f70ef6cc..bbb02647 100644 --- a/crates/install/Cargo.toml +++ b/crates/install/Cargo.toml @@ -21,11 +21,9 @@ base64 = "0.21.2" pwhash = "1.0.0" rand = "0.8.5" clap = { version = "4.1.6", features = ["derive"] } +zip-extract = "0.1.2" [target.'cfg(not(target_env = "msvc"))'.dependencies] libc = "0.2.147" flate2 = "1.0.26" tar = "0.4.38" - -[target.'cfg(target_env = "msvc")'.dependencies] -zip-extract = "0.1.2" diff --git a/crates/install/src/main.rs b/crates/install/src/main.rs index 7ba289c6..8c9c02da 100644 --- a/crates/install/src/main.rs +++ b/crates/install/src/main.rs @@ -25,7 +25,7 @@ use std::{ fs, io::Cursor, path::{Path, PathBuf}, - time::SystemTime, + process::exit, }; use base64::{engine::general_purpose, Engine}; @@ -36,11 +36,7 @@ use pwhash::sha512_crypt; use rand::{distributions::Alphanumeric, thread_rng, Rng}; use rusqlite::{Connection, OpenFlags}; -const CFG_COMMON: &str = "../../../resources/config/common.toml"; -const CFG_DIRECTORY: &str = "../../../resources/config/directory_sql.toml"; -const CFG_JMAP: &str = "../../../resources/config/jmap.toml"; -const CFG_IMAP: &str = "../../../resources/config/imap.toml"; -const CFG_SMTP: &str = "../../../resources/config/smtp.toml"; +const CONFIG_URL: &str = "https://get.stalw.art/resources/config.zip"; #[cfg(target_os = "linux")] const SERVICE: &str = include_str!("../../../resources/systemd/stalwart-mail.service"); @@ -163,14 +159,17 @@ fn main() -> std::io::Result<()> { }; create_directories(&base_path)?; + // Download and unpack configuration files + let cfg_path = base_path.join("etc"); + if let Err(err) = zip_extract::extract(Cursor::new(download(CONFIG_URL)), &cfg_path, true) { + eprintln!( + "❌ Failed to unpack configuration bundle {}: {}", + CONFIG_URL, err + ); + return Ok(()); + } + // Build configuration file - let mut cfg_file = match component { - Component::AllInOne | Component::Imap => { - [CFG_COMMON, CFG_DIRECTORY, CFG_JMAP, CFG_IMAP, CFG_SMTP].join("\n") - } - Component::Jmap => [CFG_COMMON, CFG_DIRECTORY, CFG_JMAP, CFG_SMTP].join("\n"), - Component::Smtp => [CFG_COMMON, CFG_DIRECTORY, CFG_SMTP].join("\n"), - }; let mut download_url = None; // Obtain database engine @@ -232,37 +231,28 @@ fn main() -> std::io::Result<()> { ], Directory::None, )?; - cfg_file = cfg_file - .replace( - "__BLOB_STORE__", - match blob { - Blob::Local => "local", - _ => "s3", - }, - ) - .replace("__NEXT_HOP__", "local") - .replace( - "__DIRECTORY__", - match directory { - Directory::Sql | Directory::None => "sql", - Directory::Ldap => "ldap", - }, - ) - .replace( - "__SMTP_DIRECTORY__", - match directory { - Directory::Sql | Directory::None => "sql", - Directory::Ldap => "ldap", - }, - ) - .replace( + + // Update settings + if blob != Blob::Local { + sed( + cfg_path.join("jmap").join("store.toml"), + &[("\"local\"", "\"s3\"")], + ); + } + if directory == Directory::Ldap { + sed(cfg_path.join("config.toml"), &[("/sql.toml", "/ldap.toml")]); + } + sed( + cfg_path.join("jmap").join("oauth.toml"), + &[( "__OAUTH_KEY__", - &thread_rng() + thread_rng() .sample_iter(Alphanumeric) .take(64) .map(char::from) .collect::<String>(), - ); + )], + ); directory } else { @@ -276,25 +266,36 @@ fn main() -> std::io::Result<()> { ], SmtpDirectory::Lmtp, )?; - cfg_file = cfg_file - .replace("__NEXT_HOP__", "lmtp") - .replace( - "__SMTP_DIRECTORY__", - match smtp_directory { - SmtpDirectory::Sql => "sql", - SmtpDirectory::Ldap => "ldap", - SmtpDirectory::Lmtp => "lmtp", - SmtpDirectory::Imap => "imap", - }, - ) - .replace( - "__DIRECTORY__", - match smtp_directory { - SmtpDirectory::Sql | SmtpDirectory::Lmtp | SmtpDirectory::Imap => "sql", - SmtpDirectory::Ldap => "ldap", - }, - ) - .replace("__NEXT_HOP__", "lmtp"); + + if smtp_directory == SmtpDirectory::Ldap { + sed(cfg_path.join("config.toml"), &[("/sql.toml", "/ldap.toml")]); + } + + match smtp_directory { + SmtpDirectory::Ldap => { + sed(cfg_path.join("config.toml"), &[("/sql.toml", "/ldap.toml")]); + } + SmtpDirectory::Lmtp | SmtpDirectory::Imap => { + let d_type = if smtp_directory == SmtpDirectory::Lmtp { + "lmtp" + } else { + "imap" + }; + sed( + cfg_path.join("smtp").join("queue.toml"), + &[("default", d_type)], + ); + sed( + cfg_path.join("smtp").join("session.toml"), + &[("default", d_type)], + ); + sed( + cfg_path.join("config.toml"), + &[("\"%{BASE_PATH}%/etc/directory/", format!("\"%{{BASE_PATH}}%/etc/directory/{d_type}.toml\",\n\t\"%{{BASE_PATH}}%/etc/directory/"))], + ); + } + SmtpDirectory::Sql => (), + } if !skip_download { download_url = format!( @@ -326,48 +327,25 @@ fn main() -> std::io::Result<()> { TARGET, PKG_EXTENSION ), ] { - match reqwest::blocking::get(&url).and_then(|r| { - if r.status().is_success() { - r.bytes().map(Ok) - } else { - Ok(Err(r)) - } - }) { - Ok(Ok(bytes)) => { - let unpack_path = if !args.docker { - base_path.join("bin") - } else { - PathBuf::from("/usr/local/bin") - }; - - #[cfg(not(target_env = "msvc"))] - if let Err(err) = - tar::Archive::new(flate2::bufread::GzDecoder::new(Cursor::new(bytes))) - .unpack(unpack_path) - { - eprintln!("❌ Failed to unpack {}: {}", url, err); - return Ok(()); - } - - #[cfg(target_env = "msvc")] - if let Err(err) = zip_extract::extract(Cursor::new(bytes), &unpack_path, true) { - eprintln!("❌ Failed to unpack {}: {}", url, err); - return Ok(()); - } - } - Ok(Err(response)) => { - eprintln!( - "❌ Failed to download {}, make sure your platform is supported: {}", - url, - response.status() - ); - return Ok(()); - } + let bytes = download(&url); + let unpack_path = if !args.docker { + base_path.join("bin") + } else { + PathBuf::from("/usr/local/bin") + }; + + #[cfg(not(target_env = "msvc"))] + if let Err(err) = tar::Archive::new(flate2::bufread::GzDecoder::new(Cursor::new(bytes))) + .unpack(unpack_path) + { + eprintln!("❌ Failed to unpack {}: {}", url, err); + return Ok(()); + } - Err(err) => { - eprintln!("❌ Failed to download {}: {}", url, err); - return Ok(()); - } + #[cfg(target_env = "msvc")] + if let Err(err) = zip_extract::extract(Cursor::new(bytes), &unpack_path, true) { + eprintln!("❌ Failed to unpack {}: {}", url, err); + return Ok(()); } } } @@ -423,42 +401,43 @@ fn main() -> std::io::Result<()> { // Generate DKIM key and instructions let dkim_instructions = generate_dkim(&base_path, &domain, &hostname)?; - // Create authentication SQLite database - let admin_password = if matches!(directory, Directory::None) { - create_databases(&base_path, &domain)?.into() - } else { - None - }; + // Create authentication and spam filter SQLite databases + let admin_password = create_databases( + &base_path, + if matches!(directory, Directory::None) { + Some(&domain) + } else { + None + }, + )?; - // Write config file - let cfg_path = base_path.join("etc").join("config.toml"); - if cfg_path.exists() { - // Rename existing config file - let backup_path = base_path.join("etc").join(format!( - "config.toml.bak.{}", - SystemTime::now() - .duration_since(SystemTime::UNIX_EPOCH) - .map(|d| d.as_secs()) - .unwrap_or(0) - )); - fs::rename(&cfg_path, backup_path)?; - } + // Update config file if args.docker { - cfg_file = cfg_file - .replace("127.0.0.1:8080", "0.0.0.0:8080") - .replace("[server.run-as]", "#[server.run-as]") - .replace("user = \"stalwart-mail\"", "#user = \"stalwart-mail\"") - .replace("group = \"stalwart-mail\"", "#group = \"stalwart-mail\""); + sed( + cfg_path.join("common").join("server.toml"), + &[ + ("[server.run-as]", "#[server.run-as]"), + ("user = \"stalwart-mail\"", "#user = \"stalwart-mail\""), + ("group = \"stalwart-mail\"", "#group = \"stalwart-mail\""), + ], + ); + sed( + cfg_path.join("smtp").join("listener.toml"), + &[("127.0.0.1:8080", "[::]:8080")], + ); } - fs::write( - cfg_path, - cfg_file - .replace("__PATH__", base_path.to_str().unwrap()) - .replace("__DOMAIN__", &domain) - .replace("__HOST__", &hostname) - .replace("__CERT_PATH__", &cert_path) - .replace("__PK_PATH__", &pk_path), - )?; + sed( + cfg_path.join("config.toml"), + &[ + ("__BASE_PATH__", base_path.to_str().unwrap()), + ("__DOMAIN__", &domain), + ("__HOST__", &hostname), + ], + ); + sed( + cfg_path.join("common").join("tls.toml"), + &[("__CERT_PATH__", &cert_path), ("__PK_PATH__", &pk_path)], + ); // Write service file if !args.docker { @@ -570,6 +549,58 @@ fn main() -> std::io::Result<()> { Ok(()) } +fn sed(path: impl AsRef<Path>, replacements: &[(&str, impl AsRef<str>)]) { + let path = path.as_ref(); + match fs::read_to_string(path) { + Ok(mut contents) => { + for (from, to) in replacements { + contents = contents.replace(from, to.as_ref()); + } + if let Err(err) = fs::write(path, contents) { + eprintln!( + "❌ Failed to write configuration file {}: {}", + path.display(), + err + ); + exit(1); + } + } + Err(err) => { + eprintln!( + "❌ Failed to read configuration file {}: {}", + path.display(), + err + ); + exit(1); + } + } +} + +fn download(url: &str) -> Vec<u8> { + match reqwest::blocking::get(url).and_then(|r| { + if r.status().is_success() { + r.bytes().map(Ok) + } else { + Ok(Err(r)) + } + }) { + Ok(Ok(bytes)) => bytes.to_vec(), + Ok(Err(response)) => { + eprintln!( + "❌ Failed to download {}, make sure your platform is supported: {}", + url, + response.status() + ); + exit(1); + } + + Err(err) => { + eprintln!("❌ Failed to download {}: {}", url, err); + exit(1); + } + } +} + fn select<T: SelectItem>(prompt: &str, items: &[&str], default: T) -> std::io::Result<T> { if let Some(index) = Select::with_theme(&ColorfulTheme::default()) .items(items) @@ -659,43 +690,7 @@ fn create_directories(path: &Path) -> std::io::Result<()> { Ok(()) } -fn create_databases(base_path: &Path, domain: &str) -> std::io::Result<String> { - // Create accounts database - let mut path = PathBuf::from(base_path); - path.push("data"); - if !path.exists() { - fs::create_dir_all(&path)?; - } - path.push("accounts.sqlite3"); - - let conn = Connection::open_with_flags(path, OpenFlags::default()).map_err(|err| { - std::io::Error::new( - std::io::ErrorKind::Other, - format!("Failed to open database: {}", err), - ) - })?; - let secret = thread_rng() - .sample_iter(Alphanumeric) - .take(12) - .map(char::from) - .collect::<String>(); - let hashed_secret = sha512_crypt::hash(&secret).unwrap(); - for query in [ - concat!("CREATE TABLE IF NOT EXISTS accounts (name TEXT PRIMARY KEY, secret TEXT, description TEXT, ","type TEXT NOT NULL, quota INTEGER DEFAULT 0, active BOOLEAN DEFAULT 1)").to_string(), - concat!("CREATE TABLE IF NOT EXISTS group_members (name TEXT NOT NULL, member_of ","TEXT NOT NULL, PRIMARY KEY (name, member_of))").to_string(), - concat!("CREATE TABLE IF NOT EXISTS emails (name TEXT NOT NULL, address TEXT NOT NULL",", type TEXT, PRIMARY KEY (name, address))").to_string(), - format!("INSERT OR REPLACE INTO accounts (name, secret, description, type) VALUES ('admin', '{hashed_secret}', 'Postmaster', 'individual')"), - format!("INSERT OR REPLACE INTO emails (name, address, type) VALUES ('admin', 'postmaster@{domain}', 'primary')"), - "INSERT OR IGNORE INTO group_members (name, member_of) VALUES ('admin', 'superusers')".to_string() - ] { - conn.execute(&query, []).map_err(|err| { - std::io::Error::new( - std::io::ErrorKind::Other, - format!("Failed to create database: {}", err), - ) - })?; - } - +fn create_databases(base_path: &Path, domain: Option<&str>) -> std::io::Result<Option<String>> { // Create Spam database let path = PathBuf::from(base_path) .join("data") @@ -729,7 +724,46 @@ fn create_databases(base_path: &Path, domain: &str) -> std::io::Result<String> { })?; } - Ok(secret) + if let Some(domain) = domain { + // Create accounts database + let mut path = PathBuf::from(base_path); + path.push("data"); + if !path.exists() { + fs::create_dir_all(&path)?; + } + path.push("accounts.sqlite3"); + + let conn = Connection::open_with_flags(path, OpenFlags::default()).map_err(|err| { + std::io::Error::new( + std::io::ErrorKind::Other, + format!("Failed to open database: {}", err), + ) + })?; + let secret = thread_rng() + .sample_iter(Alphanumeric) + .take(12) + .map(char::from) + .collect::<String>(); + let hashed_secret = sha512_crypt::hash(&secret).unwrap(); + for query in [ + concat!("CREATE TABLE IF NOT EXISTS accounts (name TEXT PRIMARY KEY, secret TEXT, description TEXT, ","type TEXT NOT NULL, quota INTEGER DEFAULT 0, active BOOLEAN DEFAULT 1)").to_string(), + concat!("CREATE TABLE IF NOT EXISTS group_members (name TEXT NOT NULL, member_of ","TEXT NOT NULL, PRIMARY KEY (name, member_of))").to_string(), + concat!("CREATE TABLE IF NOT EXISTS emails (name TEXT NOT NULL, address TEXT NOT NULL",", type TEXT, PRIMARY KEY (name, address))").to_string(), + format!("INSERT OR REPLACE INTO accounts (name, secret, description, type) VALUES ('admin', '{hashed_secret}', 'Postmaster', 'individual')"), + format!("INSERT OR REPLACE INTO emails (name, address, type) VALUES ('admin', 'postmaster@{domain}', 'primary')"), + "INSERT OR IGNORE INTO group_members (name, member_of) VALUES ('admin', 'superusers')".to_string() + ] { + conn.execute(&query, []).map_err(|err| { + std::io::Error::new( + std::io::ErrorKind::Other, + format!("Failed to create database: {}", err), + ) + })?; + } + Ok(Some(secret)) + } else { + Ok(None) + } } fn generate_dkim(path: &Path, domain: &str, hostname: &str) -> std::io::Result<String> { diff --git a/crates/smtp/src/scripts/plugins/bayes.rs b/crates/smtp/src/scripts/plugins/bayes.rs index d6bde305..5e923b98 100644 --- a/crates/smtp/src/scripts/plugins/bayes.rs +++ b/crates/smtp/src/scripts/plugins/bayes.rs @@ -96,7 +96,7 @@ fn train(ctx: PluginContext<'_>, is_train: bool) -> Variable { tracing::debug!( parent: span, context = "sieve:bayes_train", - event = "classify", + event = "train", is_spam = is_spam, num_tokens = model.weights.len(), ); diff --git a/tests/src/smtp/inbound/antispam.rs b/tests/src/smtp/inbound/antispam.rs index abcb75d5..71687422 100644 --- a/tests/src/smtp/inbound/antispam.rs +++ b/tests/src/smtp/inbound/antispam.rs @@ -398,7 +398,7 @@ async fn antispam() { .rcpt_to .push(SessionAddress::new(value.to_string())); } - "iprev.ptr" | "dmarc.from" => { + "iprev.ptr" => { variables.insert(param.to_string(), value.to_string().into()); } "dmarc.result" => { |