diff options
| author | mdecimus <mauro@stalw.art> | 2023-10-15 09:07:19 +0200 |
|---|---|---|
| committer | mdecimus <mauro@stalw.art> | 2023-10-15 09:07:19 +0200 |
| commit | 0b67f312207f2f5722236b469836c001e498407f (patch) | |
| tree | e74b4a3fb4a149a84ccce29cbc220a84960121a7 | |
| parent | ace58f74eb662bff2b55e8242ec796e590a35bf1 (diff) | |
Bayes classifier passing tests
54 files changed, 923 insertions, 786 deletions
diff --git a/CHANGELOG.md b/CHANGELOG.md index c20ad3de..bb0eaf2b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,18 @@ All notable changes to this project will be documented in this file. This project adheres to [Semantic Versioning](http://semver.org/). +## [0.x.x] - 2023-xx-xx + +## Added +- Option to allow invalid certificates on outbound SMTP connections. +- Option to disable ansi colors on `stdout`. + +### Changed +- SMTP reject messages are now logged as `info` rather than `debug`. + +### Fixed + + ## [0.3.9] - 2023-10-07 ## Added @@ -282,9 +282,9 @@ dependencies = [ [[package]] name = "async-trait" -version = "0.1.73" +version = "0.1.74" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc00ceb34980c03614e35a3a4e218276a0a824e911d07651cd0d858a51e8c0f0" +checksum = "a66537f1bb974b254c98ed142ff995236e81b9d0fe4db0575f46612cb15eb0f9" dependencies = [ "proc-macro2", "quote", @@ -1158,10 +1158,11 @@ dependencies = [ [[package]] name = "deranged" -version = "0.3.8" +version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2696e8a945f658fd14dc3b87242e6b80cd0f36ff04ea560fa39082368847946" +checksum = "0f32d04922c60427da6f9fef14d042d9edddef64cb9d4ce0d64d0685fbeb1fd3" dependencies = [ + "powerfmt", "serde", ] @@ -1581,9 +1582,9 @@ checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80" [[package]] name = "flate2" -version = "1.0.27" +version = "1.0.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c6c98ee8095e9d1dcbf2fcc6d95acccb90d1c81db1e44725c6a984b1dbdfb010" +checksum = "46303f565772937ffe1d394a4fac6f411c6013172fadde9dcdb1e147a086940e" dependencies = [ "crc32fast", "libz-sys", @@ -3570,6 +3571,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "31114a898e107c51bb1609ffaf55a0e011cf6a4d7f1170d0015a165082c0338b" [[package]] +name = "powerfmt" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" + +[[package]] name = "ppv-lite86" version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -3817,9 +3824,9 @@ dependencies = [ [[package]] name = "rasn-cms" -version = "0.10.2" +version = "0.10.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ecf9f1bb38cbb2a032014f0329d7fd9c2b08f26c4fc882ad642bb95dfefd74f" +checksum = "1b33fab229bc35b33782880040bfc71ecad5dffb040752e7ceb20f6bddbc33c1" dependencies = [ "rasn", "rasn-pkix", @@ -3842,9 +3849,9 @@ dependencies = [ [[package]] name = "rasn-pkix" -version = "0.10.2" +version = "0.10.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b894c903130c4915d79d8d9ce155429b3896b25efa5f81de4d9ab7b1b0f0b7cf" +checksum = "a96a8f3a9c45f9e76eff1b5d72385296cbfe2c2c22ecaccb8f493f4b60be524d" dependencies = [ "rasn", ] @@ -3900,14 +3907,14 @@ dependencies = [ [[package]] name = "regex" -version = "1.10.0" +version = "1.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d119d7c7ca818f8a53c300863d4f87566aac09943aef5b355bb83969dae75d87" +checksum = "aaac441002f822bc9705a681810a4dd2963094b9ca0ddc41cb963a4c189189ea" dependencies = [ "aho-corasick", "memchr", - "regex-automata 0.4.1", - "regex-syntax 0.8.0", + "regex-automata 0.4.2", + "regex-syntax 0.8.2", ] [[package]] @@ -3921,13 +3928,13 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.1" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "465c6fc0621e4abc4187a2bda0937bfd4f722c2730b29562e19689ea796c9a4b" +checksum = "5011c7e263a695dc8ca064cddb722af1be54e517a280b12a5356f98366899e5d" dependencies = [ "aho-corasick", "memchr", - "regex-syntax 0.8.0", + "regex-syntax 0.8.2", ] [[package]] @@ -3938,9 +3945,9 @@ checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1" [[package]] name = "regex-syntax" -version = "0.8.0" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3cbb081b9784b07cceb8824c8583f86db4814d172ab043f3c23f7dc600bf83d" +checksum = "c08c74e62047bb2de4ff487b251e4a92e24f48745648451635cec7d591162d9f" [[package]] name = "reqwest" @@ -4202,9 +4209,9 @@ dependencies = [ [[package]] name = "rustix" -version = "0.38.18" +version = "0.38.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a74ee2d7c2581cd139b42447d7d9389b889bdaad3a73f1ebb16f2a3237bb19c" +checksum = "745ecfa778e66b2b63c88a61cb36e0eea109e803b0b86bf9879fbc77c70e86ed" dependencies = [ "bitflags 2.4.0", "errno", @@ -4438,9 +4445,9 @@ dependencies = [ [[package]] name = "serde" -version = "1.0.188" +version = "1.0.189" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf9e0fcba69a370eed61bcf2b728575f726b50b55cba78064753d708ddc7549e" +checksum = "8e422a44e74ad4001bdc8eede9a4570ab52f71190e9c076d14369f38b9200537" dependencies = [ "serde_derive", ] @@ -4456,9 +4463,9 @@ dependencies = [ [[package]] name = "serde_derive" -version = "1.0.188" +version = "1.0.189" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4eca7ac642d82aa35b60049a6eccb4be6be75e599bd2e9adb5f875a737654af2" +checksum = "1e48d1f918009ce3145511378cf68d613e3b3d9137d67272562080d68a2b32d5" dependencies = [ "proc-macro2", "quote", @@ -4606,7 +4613,7 @@ checksum = "a7cee0529a6d40f580e7a5e6c495c8fbfe21b7b52795ed4bb5e62cdf92bc6380" [[package]] name = "sieve-rs" version = "0.3.1" -source = "git+https://github.com/stalwartlabs/sieve#bbb265765ebe92394e429001e90ba2e9b4201f9a" +source = "git+https://github.com/stalwartlabs/sieve#d6251ec246011d4bbd8ee9919b7f4864d626d915" dependencies = [ "ahash 0.8.3", "bincode", @@ -5304,12 +5311,13 @@ dependencies = [ [[package]] name = "time" -version = "0.3.29" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "426f806f4089c493dcac0d24c29c01e2c38baf8e30f1b716ee37e83d200b18fe" +checksum = "c4a34ab300f2dee6e562c10a046fc05e358b29f9bf92277f30c3c8d82275f6f5" dependencies = [ "deranged", "itoa", + "powerfmt", "serde", "time-core", "time-macros", @@ -5528,11 +5536,10 @@ checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52" [[package]] name = "tracing" -version = "0.1.37" +version = "0.1.39" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ce8c33a8d48bd45d624a6e523445fd21ec13d3653cd51f681abf67418f54eb8" +checksum = "ee2ef2af84856a50c1d430afce2fdded0a4ec7eda868db86409b4543df0797f9" dependencies = [ - "cfg-if", "log", "pin-project-lite", "tracing-attributes", @@ -5552,9 +5559,9 @@ dependencies = [ [[package]] name = "tracing-attributes" -version = "0.1.26" +version = "0.1.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f4f31f56159e98206da9efd823404b79b6ef3143b4a7ab76e67b1751b25a4ab" +checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" dependencies = [ "proc-macro2", "quote", @@ -5563,9 +5570,9 @@ dependencies = [ [[package]] name = "tracing-core" -version = "0.1.31" +version = "0.1.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0955b8137a1df6f1a2e9a37d8a6656291ff0297c1a97c24e0d8425fe2312f79a" +checksum = "c06d3da6113f116aaee68e4d601191614c9053067f9ab7f6edbcb161237daa54" dependencies = [ "once_cell", "valuable", @@ -5629,9 +5636,9 @@ dependencies = [ [[package]] name = "trust-dns-proto" -version = "0.23.0" +version = "0.23.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0dc775440033cb114085f6f2437682b194fa7546466024b1037e82a48a052a69" +checksum = "559ac980345f7f5020883dd3bcacf176355225e01916f8c2efecad7534f682c6" dependencies = [ "async-trait", "cfg-if", @@ -5659,9 +5666,9 @@ dependencies = [ [[package]] name = "trust-dns-resolver" -version = "0.23.0" +version = "0.23.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2dff7aed33ef3e8bf2c9966fccdfed93f93d46f432282ea875cd66faabc6ef2f" +checksum = "c723b0e608b24ad04c73b2607e0241b2c98fd79795a95e98b068b6966138a29d" dependencies = [ "cfg-if", "futures-util", @@ -6427,11 +6434,10 @@ dependencies = [ [[package]] name = "zstd-sys" -version = "2.0.8+zstd.1.5.5" +version = "2.0.9+zstd.1.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5556e6ee25d32df2586c098bbfa278803692a20d0ab9565e049480d52707ec8c" +checksum = "9e16efa8a874a0481a574084d34cc26fdb3b99627480f785888deb6386506656" dependencies = [ "cc", - "libc", "pkg-config", ] diff --git a/crates/cli/src/main.rs b/crates/cli/src/main.rs index 2be60cc7..4e33500c 100644 --- a/crates/cli/src/main.rs +++ b/crates/cli/src/main.rs @@ -24,6 +24,7 @@ use std::{ collections::HashMap, io::{BufRead, Write}, + time::Duration, }; use clap::Parser; @@ -88,6 +89,7 @@ async fn build_client(url: &str, credentials: Credentials) -> Client { Client::new() .credentials(credentials) .accept_invalid_certs(is_localhost(url)) + .timeout(Duration::from_secs(60)) .connect(url) .await .unwrap_or_else(|err| { diff --git a/crates/directory/src/config.rs b/crates/directory/src/config.rs index 1afda89d..efe216e6 100644 --- a/crates/directory/src/config.rs +++ b/crates/directory/src/config.rs @@ -307,7 +307,7 @@ impl InsertLine for LookupList { } } -impl InsertLine for AHashMap<String, Variable<'static>> { +impl InsertLine for AHashMap<String, Variable> { fn insert(&mut self, entry: String, format: &LookupFormat) { let (key, value) = entry .split_once(format.separator.as_deref().unwrap_or(" ")) @@ -334,17 +334,17 @@ impl InsertLine for AHashMap<String, Variable<'static>> { } let value = if has_other || !has_digit { - Variable::String(value.to_string()) + Variable::String(value.to_string().into()) } else if has_dots { value .parse() .map(Variable::Float) - .unwrap_or_else(|_| Variable::String(value.to_string())) + .unwrap_or_else(|_| Variable::String(value.to_string().into())) } else { value .parse() .map(Variable::Integer) - .unwrap_or_else(|_| Variable::String(value.to_string())) + .unwrap_or_else(|_| Variable::String(value.to_string().into())) }; self.insert(key.to_string(), value); diff --git a/crates/directory/src/lib.rs b/crates/directory/src/lib.rs index 191fe9ba..b8249d31 100644 --- a/crates/directory/src/lib.rs +++ b/crates/directory/src/lib.rs @@ -118,7 +118,7 @@ pub enum Lookup { list: LookupList, }, Map { - map: AHashMap<String, Variable<'static>>, + map: AHashMap<String, Variable>, }, } @@ -190,7 +190,7 @@ impl Lookup { } } - pub async fn lookup(&self, items: &[DatabaseColumn<'_>]) -> Option<Variable<'static>> { + pub async fn lookup(&self, items: &[DatabaseColumn<'_>]) -> Option<Variable> { match self { Lookup::Directory { directory, query } => match directory.query(query, items).await { Ok(mut result) => match result.len() { @@ -198,7 +198,13 @@ impl Lookup { result.pop().map(Variable::from).unwrap() } 0 => Variable::default(), - _ => Variable::Array(result.into_iter().map(Variable::from).collect()), + _ => Variable::Array( + result + .into_iter() + .map(Variable::from) + .collect::<Vec<_>>() + .into(), + ), } .into(), Err(_) => None, @@ -222,15 +228,15 @@ impl Lookup { } } -impl<'x> From<DatabaseColumn<'x>> for Variable<'static> { +impl<'x> From<DatabaseColumn<'x>> for Variable { fn from(value: DatabaseColumn) -> Self { match value { DatabaseColumn::Integer(v) => Variable::Integer(v), DatabaseColumn::Bool(v) => Variable::Integer(i64::from(v)), DatabaseColumn::Float(v) => Variable::Float(v), - DatabaseColumn::Text(v) => Variable::String(v.into_owned()), - DatabaseColumn::Blob(v) => Variable::String(v.into_owned().into_string()), - DatabaseColumn::Null => Variable::StringRef(""), + DatabaseColumn::Text(v) => Variable::String(v.into_owned().into()), + DatabaseColumn::Blob(v) => Variable::String(v.into_owned().into_string().into()), + DatabaseColumn::Null => Variable::default(), } } } @@ -554,26 +560,24 @@ impl<'x> From<Vec<u8>> for DatabaseColumn<'x> { } } -impl<'x> From<Variable<'x>> for DatabaseColumn<'x> { - fn from(value: Variable<'x>) -> Self { +impl<'x> From<Variable> for DatabaseColumn<'x> { + fn from(value: Variable) -> Self { match value { - Variable::String(v) => Self::Text(v.into()), - Variable::StringRef(v) => Self::Text(v.into()), + Variable::String(v) => Self::Text(v.to_string().into()), Variable::Integer(v) => Self::Integer(v), Variable::Float(v) => Self::Float(v), - v => Self::Text(v.into_string().into()), + v => Self::Text(v.to_string().into_owned().into()), } } } -impl<'x> From<&'x Variable<'x>> for DatabaseColumn<'x> { - fn from(value: &'x Variable<'x>) -> Self { +impl<'x> From<&'x Variable> for DatabaseColumn<'x> { + fn from(value: &'x Variable) -> Self { match value { - Variable::String(v) => Self::Text(v.into()), - Variable::StringRef(v) => Self::Text((*v).into()), + Variable::String(v) => Self::Text(v.to_string().into()), Variable::Integer(v) => Self::Integer(*v), Variable::Float(v) => Self::Float(*v), - v => Self::Text(v.to_string().into()), + v => Self::Text(v.to_string().into_owned().into()), } } } diff --git a/crates/nlp/src/bayes/cache.rs b/crates/nlp/src/bayes/cache.rs index 6645ec85..52cfcda4 100644 --- a/crates/nlp/src/bayes/cache.rs +++ b/crates/nlp/src/bayes/cache.rs @@ -105,3 +105,14 @@ impl BayesTokenCache { } } } + +impl Default for BayesTokenCache { + fn default() -> Self { + Self { + positive: Mutex::new(LruCache::with_hasher(1024, Default::default())), + negative: Mutex::new(LruCache::with_hasher(1024, Default::default())), + ttl_negative: Default::default(), + ttl_positive: Default::default(), + } + } +} diff --git a/crates/nlp/src/bayes/classify.rs b/crates/nlp/src/bayes/classify.rs index a2b36b2a..63478a85 100644 --- a/crates/nlp/src/bayes/classify.rs +++ b/crates/nlp/src/bayes/classify.rs @@ -68,9 +68,7 @@ impl BayesClassifier { } } - if processed_tokens == 0 - || self.min_tokens > 0 && processed_tokens < (self.min_tokens as f64 * 0.1) as u32 - { + if processed_tokens == 0 || self.min_tokens > 0 && processed_tokens < self.min_tokens { return None; } diff --git a/crates/nlp/src/bayes/mod.rs b/crates/nlp/src/bayes/mod.rs index 99a38bcf..96aefa51 100644 --- a/crates/nlp/src/bayes/mod.rs +++ b/crates/nlp/src/bayes/mod.rs @@ -54,7 +54,7 @@ pub struct TokenHash { pub h2: u64, } -#[derive(Debug, Serialize, Deserialize, Default, Copy, Clone)] +#[derive(Debug, Serialize, Deserialize, Default, Copy, Clone, Hash, PartialEq, Eq)] pub struct Weights { pub spam: u32, pub ham: u32, diff --git a/crates/nlp/src/bayes/tokenize.rs b/crates/nlp/src/bayes/tokenize.rs index ebc44444..4e171d01 100644 --- a/crates/nlp/src/bayes/tokenize.rs +++ b/crates/nlp/src/bayes/tokenize.rs @@ -67,7 +67,7 @@ impl<'x, 'y> Iterator for BayesTokenizer<'x, 'y> { let token = self.tokenizer.next()?; let word: Cow<str> = match token.word { - TokenType::Alphabetic(word) | TokenType::Hexadecimal(word) => { + TokenType::Alphabetic(word) => { let word = word.to_lowercase(); if self .stop_words @@ -133,7 +133,8 @@ impl<'x, 'y> Iterator for BayesTokenizer<'x, 'y> { continue; } } - TokenType::Integer(word) | TokenType::Float(word) => word.into(), + TokenType::Integer(word) => number_to_tag("INTEGER", word).into(), + TokenType::Float(word) => number_to_tag("FLOAT", word).into(), TokenType::Punctuation(_) | TokenType::Space => { continue; } @@ -144,6 +145,19 @@ impl<'x, 'y> Iterator for BayesTokenizer<'x, 'y> { } } +fn number_to_tag(prefix: &str, num: &str) -> String { + format!( + "{}_{}_{}", + prefix, + if prefix.starts_with('-') { + "NEG" + } else { + "POS" + }, + num.len() + ) +} + pub static SYMBOLS: phf::Set<char> = phf::phf_set! { // Currency '\u{0024}', '\u{00A2}', '\u{00A3}', '\u{00A4}', '\u{00A5}', '\u{058F}', '\u{060B}', '\u{07FE}', diff --git a/crates/nlp/src/tokenizers/osb.rs b/crates/nlp/src/tokenizers/osb.rs index c646a3a4..53976140 100644 --- a/crates/nlp/src/tokenizers/osb.rs +++ b/crates/nlp/src/tokenizers/osb.rs @@ -23,7 +23,7 @@ use std::{borrow::Cow, iter::Peekable}; -#[derive(Debug, Clone, PartialEq, Eq)] +#[derive(Debug, Clone, PartialEq, Eq, Hash)] pub struct OsbToken<T> { pub inner: T, pub idx: usize, diff --git a/crates/nlp/src/tokenizers/types.rs b/crates/nlp/src/tokenizers/types.rs index 027cf767..8c36b5f2 100644 --- a/crates/nlp/src/tokenizers/types.rs +++ b/crates/nlp/src/tokenizers/types.rs @@ -45,9 +45,8 @@ pub struct TypesTokenizer<'x, 'y> { #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum TokenType<T> { Alphabetic(T), - Integer(T), Alphanumeric(T), - Hexadecimal(T), + Integer(T), Other(char), Punctuation(char), Space, @@ -74,7 +73,7 @@ impl<'x, 'y> Iterator for TypesTokenizer<'x, 'y> { if self.tokenize_urls && matches!( token.word, - TokenType::Alphabetic(t) | TokenType::Hexadecimal(t) + TokenType::Alphabetic(t) | TokenType::Alphanumeric(t) if t.len() <= 8 && t.chars().all(|c| c.is_ascii())) && self.try_skip_url_scheme() { @@ -169,7 +168,6 @@ impl<'x, 'y> TypesTokenizer<'x, 'y> { fn consume(&mut self) -> bool { let mut has_alpha = false; let mut has_number = false; - let mut has_hex = false; let mut start_pos = usize::MAX; let mut end_pos = usize::MAX; @@ -178,11 +176,7 @@ impl<'x, 'y> TypesTokenizer<'x, 'y> { for (pos, ch) in self.iter.by_ref() { if ch.is_alphabetic() { - if ch.is_ascii_hexdigit() { - has_hex = true; - } else { - has_alpha = true; - } + has_alpha = true; } else if ch.is_ascii_digit() { has_number = true; } else { @@ -222,8 +216,6 @@ impl<'x, 'y> TypesTokenizer<'x, 'y> { TokenType::Alphanumeric(text) } else if has_alpha { TokenType::Alphabetic(text) - } else if has_hex { - TokenType::Hexadecimal(text) } else { TokenType::Integer(text) }, @@ -321,7 +313,6 @@ impl<'x, 'y> TypesTokenizer<'x, 'y> { TokenType::Alphabetic(_) | TokenType::Alphanumeric(_) | TokenType::Integer(_) - | TokenType::Hexadecimal(_) | TokenType::Punctuation( '-' | '.' | '_' | '~' | '!' | '$' | '&' | '\'' | '(' | ')' | '*' | '+' | ',' | ';' | '=' | ':', @@ -351,9 +342,7 @@ impl<'x, 'y> TypesTokenizer<'x, 'y> { while let Some(token) = self.peek() { match token.word { - TokenType::Alphabetic(text) - | TokenType::Alphanumeric(text) - | TokenType::Hexadecimal(text) => { + TokenType::Alphabetic(text) | TokenType::Alphanumeric(text) => { last_label_is_tld = text.len() >= 2 && self.suffixes.contains(&text.to_ascii_lowercase()); text_count += 1; @@ -453,7 +442,6 @@ impl<'x, 'y> TypesTokenizer<'x, 'y> { TokenType::Alphabetic(_) | TokenType::Alphanumeric(_) | TokenType::Integer(_) - | TokenType::Hexadecimal(_) | TokenType::Other(_) => {} TokenType::Punctuation('(') => { p_count += 1; @@ -575,11 +563,7 @@ impl<'x, 'y> TypesTokenizer<'x, 'y> { TokenType::Punctuation('[') if start_pos == usize::MAX => { return self.try_parse_ipv6(token.from); } - TokenType::Alphabetic(text) - | TokenType::Alphanumeric(text) - | TokenType::Hexadecimal(text) - if text.len() <= 63 => - { + TokenType::Alphabetic(text) | TokenType::Alphanumeric(text) if text.len() <= 63 => { last_label_is_tld = text.len() >= 2 && self.suffixes.contains(&text.to_ascii_lowercase()); has_alpha = true; @@ -630,7 +614,7 @@ impl<'x, 'y> TypesTokenizer<'x, 'y> { while let Some(token) = self.peek() { match token.word { - TokenType::Integer(_) | TokenType::Hexadecimal(_) => { + TokenType::Integer(_) | TokenType::Alphanumeric(_) => { last_ch = 0; } TokenType::Punctuation(':') if last_ch != b'.' => { @@ -720,7 +704,7 @@ impl<'x, 'y> TypesTokenizer<'x, 'y> { (TokenType::Punctuation('/'), State::Slash1) => State::Slash2, (TokenType::Punctuation('/'), State::Slash2) => return true, (TokenType::Punctuation('+'), State::None) => State::PlusAlpha, - (TokenType::Alphabetic(t) | TokenType::Hexadecimal(t), State::PlusAlpha) + (TokenType::Alphabetic(t) | TokenType::Alphanumeric(t), State::PlusAlpha) if t.chars().all(|c| c.is_ascii()) => { State::Colon @@ -740,7 +724,6 @@ impl<T> TokenType<T> { TokenType::Alphabetic(_) | TokenType::Integer(_) | TokenType::Alphanumeric(_) - | TokenType::Hexadecimal(_) | TokenType::Other(_) | TokenType::Punctuation( '!' | '#' @@ -771,7 +754,6 @@ impl<T> TokenType<T> { TokenType::Alphabetic(_) | TokenType::Integer(_) | TokenType::Alphanumeric(_) - | TokenType::Hexadecimal(_) | TokenType::Other(_) ) || (!is_start && matches!(self, TokenType::Punctuation('-'))) } @@ -879,7 +861,7 @@ mod test { ( "a-b://foo", vec![ - TokenType::Hexadecimal("a"), + TokenType::Alphabetic("a"), TokenType::Punctuation('-'), TokenType::UrlNoHost("b://foo"), ], @@ -887,7 +869,7 @@ mod test { ( "a.b://foo", vec![ - TokenType::Hexadecimal("a"), + TokenType::Alphabetic("a"), TokenType::Punctuation('.'), TokenType::UrlNoHost("b://foo"), ], @@ -911,7 +893,7 @@ mod test { ( "ab://", vec![ - TokenType::Hexadecimal("ab"), + TokenType::Alphabetic("ab"), TokenType::Punctuation(':'), TokenType::Punctuation('/'), TokenType::Punctuation('/'), @@ -1722,7 +1704,7 @@ mod test { vec![ TokenType::Url("http://example.org/"), TokenType::Punctuation('"'), - TokenType::Hexadecimal("a"), + TokenType::Alphabetic("a"), ], ), ( @@ -1730,7 +1712,7 @@ mod test { vec![ TokenType::Url("http://example.org/"), TokenType::Punctuation('"'), - TokenType::Hexadecimal("a"), + TokenType::Alphabetic("a"), TokenType::Punctuation('"'), ], ), @@ -1739,7 +1721,7 @@ mod test { vec![ TokenType::Url("http://example.org/"), TokenType::Punctuation('`'), - TokenType::Hexadecimal("a"), + TokenType::Alphabetic("a"), ], ), ( @@ -1747,7 +1729,7 @@ mod test { vec![ TokenType::Url("http://example.org/"), TokenType::Punctuation('`'), - TokenType::Hexadecimal("a"), + TokenType::Alphabetic("a"), TokenType::Punctuation('`'), ], ), @@ -1782,7 +1764,7 @@ mod test { vec![ TokenType::UrlNoScheme("example.org/"), TokenType::Punctuation('`'), - TokenType::Hexadecimal("a"), + TokenType::Alphabetic("a"), ], ), ( @@ -1790,7 +1772,7 @@ mod test { vec![ TokenType::UrlNoScheme("example.org/"), TokenType::Punctuation('`'), - TokenType::Hexadecimal("a"), + TokenType::Alphabetic("a"), TokenType::Punctuation('`'), ], ), @@ -1939,7 +1921,7 @@ mod test { TokenType::Alphabetic("div"), TokenType::Punctuation('>'), TokenType::Punctuation('<'), - TokenType::Hexadecimal("a"), + TokenType::Alphabetic("a"), TokenType::Space, TokenType::Alphabetic("href"), TokenType::Punctuation('='), @@ -1949,7 +1931,7 @@ mod test { TokenType::Punctuation('>'), TokenType::Punctuation('<'), TokenType::Punctuation('/'), - TokenType::Hexadecimal("a"), + TokenType::Alphabetic("a"), TokenType::Punctuation('>'), TokenType::Punctuation('<'), TokenType::Punctuation('/'), @@ -1964,7 +1946,7 @@ mod test { TokenType::Alphabetic("div"), TokenType::Punctuation('>'), TokenType::Punctuation('<'), - TokenType::Hexadecimal("a"), + TokenType::Alphabetic("a"), TokenType::Space, TokenType::Alphabetic("href"), TokenType::Punctuation('='), @@ -1975,7 +1957,7 @@ mod test { TokenType::Punctuation('>'), TokenType::Punctuation('<'), TokenType::Punctuation('/'), - TokenType::Hexadecimal("a"), + TokenType::Alphabetic("a"), TokenType::Punctuation('>'), TokenType::Punctuation('<'), TokenType::Punctuation('/'), @@ -2214,7 +2196,7 @@ mod test { vec![ TokenType::Alphabetic("example"), TokenType::Punctuation('.'), - TokenType::Hexadecimal("c"), + TokenType::Alphabetic("c"), ], ), ("example.co", vec![TokenType::UrlNoScheme("example.co")]), @@ -2225,20 +2207,20 @@ mod test { vec![ TokenType::Alphabetic("exampl"), TokenType::Punctuation('.'), - TokenType::Hexadecimal("e"), + TokenType::Alphabetic("e"), TokenType::Punctuation('.'), - TokenType::Hexadecimal("c"), + TokenType::Alphabetic("c"), ], ), ("exampl.e.co", vec![TokenType::UrlNoScheme("exampl.e.co")]), ( "e.xample.c", vec![ - TokenType::Hexadecimal("e"), + TokenType::Alphabetic("e"), TokenType::Punctuation('.'), TokenType::Alphabetic("xample"), TokenType::Punctuation('.'), - TokenType::Hexadecimal("c"), + TokenType::Alphabetic("c"), ], ), ("e.xample.co", vec![TokenType::UrlNoScheme("e.xample.co")]), @@ -2284,7 +2266,7 @@ mod test { TokenType::Space, TokenType::UrlNoScheme("www.foobar.co"), TokenType::Space, - TokenType::Hexadecimal("E"), + TokenType::Alphabetic("E"), TokenType::Punctuation('-'), TokenType::Alphabetic("Mail"), TokenType::Punctuation(':'), @@ -2462,7 +2444,7 @@ mod test { ( "a-.com", vec![ - TokenType::Hexadecimal("a"), + TokenType::Alphabetic("a"), TokenType::Punctuation('-'), TokenType::Punctuation('.'), TokenType::Alphabetic("com"), @@ -2471,9 +2453,9 @@ mod test { ( "a.b-.com", vec![ - TokenType::Hexadecimal("a"), + TokenType::Alphabetic("a"), TokenType::Punctuation('.'), - TokenType::Hexadecimal("b"), + TokenType::Alphabetic("b"), TokenType::Punctuation('-'), TokenType::Punctuation('.'), TokenType::Alphabetic("com"), @@ -2586,11 +2568,11 @@ mod test { ("@", vec![TokenType::Punctuation('@')]), ( "a@", - vec![TokenType::Hexadecimal("a"), TokenType::Punctuation('@')], + vec![TokenType::Alphabetic("a"), TokenType::Punctuation('@')], ), ( "@a", - vec![TokenType::Punctuation('@'), TokenType::Hexadecimal("a")], + vec![TokenType::Punctuation('@'), TokenType::Alphabetic("a")], ), ( "@@@", @@ -2715,7 +2697,7 @@ mod test { ( "a..b@example.com", vec![ - TokenType::Hexadecimal("a"), + TokenType::Alphabetic("a"), TokenType::Punctuation('.'), TokenType::Punctuation('.'), TokenType::Email("b@example.com"), @@ -2731,17 +2713,17 @@ mod test { ( "a@b", vec![ - TokenType::Hexadecimal("a"), + TokenType::Alphabetic("a"), TokenType::Punctuation('@'), - TokenType::Hexadecimal("b"), + TokenType::Alphabetic("b"), ], ), ( "a@b.", vec![ - TokenType::Hexadecimal("a"), + TokenType::Alphabetic("a"), TokenType::Punctuation('@'), - TokenType::Hexadecimal("b"), + TokenType::Alphabetic("b"), TokenType::Punctuation('.'), ], ), @@ -2760,7 +2742,7 @@ mod test { ( "a@-foo.com", vec![ - TokenType::Hexadecimal("a"), + TokenType::Alphabetic("a"), TokenType::Punctuation('@'), TokenType::Punctuation('-'), TokenType::UrlNoScheme("foo.com"), @@ -2769,9 +2751,9 @@ mod test { ( "a@b-.", vec![ - TokenType::Hexadecimal("a"), + TokenType::Alphabetic("a"), TokenType::Punctuation('@'), - TokenType::Hexadecimal("b"), + TokenType::Alphabetic("b"), TokenType::Punctuation('-'), TokenType::Punctuation('.'), ], @@ -2779,17 +2761,17 @@ mod test { ( "a@b", vec![ - TokenType::Hexadecimal("a"), + TokenType::Alphabetic("a"), TokenType::Punctuation('@'), - TokenType::Hexadecimal("b"), + TokenType::Alphabetic("b"), ], ), ( "a@b.", vec![ - TokenType::Hexadecimal("a"), + TokenType::Alphabetic("a"), TokenType::Punctuation('@'), - TokenType::Hexadecimal("b"), + TokenType::Alphabetic("b"), TokenType::Punctuation('.'), ], ), @@ -2833,9 +2815,9 @@ mod test { ( "a@a.xyϸ", vec![ - TokenType::Hexadecimal("a"), + TokenType::Alphabetic("a"), TokenType::Punctuation('@'), - TokenType::Hexadecimal("a"), + TokenType::Alphabetic("a"), TokenType::Punctuation('.'), TokenType::Alphabetic("xyϸ"), ], diff --git a/crates/smtp/src/config/mod.rs b/crates/smtp/src/config/mod.rs index 13c4b1fc..f39881eb 100644 --- a/crates/smtp/src/config/mod.rs +++ b/crates/smtp/src/config/mod.rs @@ -417,6 +417,7 @@ pub struct QueueOutboundTls { pub dane: IfBlock<RequireOptional>, pub mta_sts: IfBlock<RequireOptional>, pub start: IfBlock<RequireOptional>, + pub invalid_certs: IfBlock<bool>, } pub struct QueueOutboundTimeout { diff --git a/crates/smtp/src/config/queue.rs b/crates/smtp/src/config/queue.rs index b666370b..09b0470a 100644 --- a/crates/smtp/src/config/queue.rs +++ b/crates/smtp/src/config/queue.rs @@ -148,6 +148,13 @@ impl ConfigQueue for Config { start: self .parse_if_block("queue.outbound.tls.starttls", ctx, &mx_envelope_keys)? .unwrap_or_else(|| IfBlock::new(RequireOptional::Optional)), + invalid_certs: self + .parse_if_block( + "queue.outbound.tls.allow-invalid-certs", + ctx, + &mx_envelope_keys, + )? + .unwrap_or_else(|| IfBlock::new(false)), }, throttle: self.parse_queue_throttle(ctx)?, quota: self.parse_queue_quota(ctx)?, diff --git a/crates/smtp/src/config/scripts.rs b/crates/smtp/src/config/scripts.rs index c841e2b6..71040842 100644 --- a/crates/smtp/src/config/scripts.rs +++ b/crates/smtp/src/config/scripts.rs @@ -21,9 +21,8 @@ * for more details. */ -use std::{sync::Arc, time::Duration}; +use std::time::Duration; -use directory::Lookup; use nlp::bayes::{cache::BayesTokenCache, BayesClassifier}; use sieve::{compiler::grammar::Capability, Compiler, Runtime}; @@ -42,12 +41,11 @@ pub trait ConfigSieve { fn parse_sieve(&self, ctx: &mut ConfigContext) -> super::Result<SieveCore>; } +#[derive(Default)] pub struct SieveContext { pub psl: PublicSuffix, pub bayes_classify: BayesClassifier, pub bayes_cache: BayesTokenCache, - pub lookup_classify: Arc<Lookup>, - pub lookup_train: Arc<Lookup>, } impl ConfigSieve for Config { @@ -67,18 +65,6 @@ impl ConfigSieve for Config { self.property_or_static("bayes.cache.ttl.positive", "1h")?, self.property_or_static("bayes.cache.ttl.negative", "1h")?, ), - lookup_classify: ctx - .directory - .lookups - .get("bayes.tokens.classify") - .ok_or("No lookup found for key bayes.tokens.classify.".to_string())? - .clone(), - lookup_train: ctx - .directory - .lookups - .get("bayes.tokens.train") - .ok_or("No lookup found for key bayes.tokens.train.".to_string())? - .clone(), }; // Allocate compiler and runtime diff --git a/crates/smtp/src/inbound/data.rs b/crates/smtp/src/inbound/data.rs index a1c87c5b..11afb640 100644 --- a/crates/smtp/src/inbound/data.rs +++ b/crates/smtp/src/inbound/data.rs @@ -440,7 +440,7 @@ impl<T: AsyncWrite + AsyncRead + IsTls + Unpin> Session<T> { .filter_map(|r| { if matches!(r.result(), DkimResult::Pass) { r.signature() - .map(|s| Variable::String(s.domain().to_lowercase())) + .map(|s| Variable::from(s.domain().to_lowercase())) } else { None } @@ -478,7 +478,7 @@ impl<T: AsyncWrite + AsyncRead + IsTls + Unpin> Session<T> { edited_message = Arc::new(message).into(); } ScriptResult::Reject(message) => { - tracing::debug!(parent: &self.span, + tracing::info!(parent: &self.span, context = "sieve", event = "reject", reason = message); diff --git a/crates/smtp/src/inbound/ehlo.rs b/crates/smtp/src/inbound/ehlo.rs index d8521eaa..f36d7d68 100644 --- a/crates/smtp/src/inbound/ehlo.rs +++ b/crates/smtp/src/inbound/ehlo.rs @@ -41,7 +41,7 @@ impl<T: AsyncWrite + AsyncRead + IsTls + Unpin> Session<T> { if domain != self.data.helo_domain { // Reject non-FQDN EHLO domains - simply checks that the hostname has at least one dot if self.params.ehlo_reject_non_fqdn && !domain.as_str().has_labels() { - tracing::debug!(parent: &self.span, + tracing::info!(parent: &self.span, context = "ehlo", event = "reject", reason = "invalid", @@ -101,7 +101,7 @@ impl<T: AsyncWrite + AsyncRead + IsTls + Unpin> Session<T> { .run_script(script.clone(), self.build_script_parameters("ehlo")) .await { - tracing::debug!(parent: &self.span, + tracing::info!(parent: &self.span, context = "sieve", event = "reject", domain = &self.data.helo_domain, @@ -242,7 +242,7 @@ impl<T: AsyncWrite + AsyncRead + IsTls + Unpin> Session<T> { }) .await { - tracing::debug!(parent: &self.span, + tracing::info!(parent: &self.span, context = context, event = "reject", reason = "dnsbl", @@ -268,7 +268,7 @@ impl<T: AsyncWrite + AsyncRead + IsTls + Unpin> Session<T> { .is_dns_blocked(self.data.remote_ip.to_dnsbl(dnsbl)) .await { - tracing::debug!(parent: &self.span, + tracing::info!(parent: &self.span, context = "connect", event = "reject", reason = "dnsbl", diff --git a/crates/smtp/src/inbound/mail.rs b/crates/smtp/src/inbound/mail.rs index 82f78a26..ed52d5c8 100644 --- a/crates/smtp/src/inbound/mail.rs +++ b/crates/smtp/src/inbound/mail.rs @@ -155,7 +155,7 @@ impl<T: AsyncWrite + AsyncRead + Unpin + IsTls> Session<T> { } } ScriptResult::Reject(message) => { - tracing::debug!(parent: &self.span, + tracing::info!(parent: &self.span, context = "sieve", event = "reject", address = &self.data.mail_from.as_ref().unwrap().address, diff --git a/crates/smtp/src/inbound/milter/message.rs b/crates/smtp/src/inbound/milter/message.rs index 2de5a43b..2fef32ae 100644 --- a/crates/smtp/src/inbound/milter/message.rs +++ b/crates/smtp/src/inbound/milter/message.rs @@ -70,7 +70,7 @@ impl<T: AsyncWrite + AsyncRead + IsTls + Unpin> Session<T> { } } Err(Rejection::Action(action)) => { - tracing::debug!( + tracing::info!( parent: &self.span, milter.host = &milter.hostname, milter.port = &milter.port, diff --git a/crates/smtp/src/inbound/rcpt.rs b/crates/smtp/src/inbound/rcpt.rs index b4f5829e..7b627686 100644 --- a/crates/smtp/src/inbound/rcpt.rs +++ b/crates/smtp/src/inbound/rcpt.rs @@ -106,7 +106,7 @@ impl<T: AsyncWrite + AsyncRead + Unpin + IsTls> Session<T> { } } ScriptResult::Reject(message) => { - tracing::debug!(parent: &self.span, + tracing::info!(parent: &self.span, context = "sieve", event = "reject", address = self.data.rcpt_to.last().unwrap().address, diff --git a/crates/smtp/src/outbound/delivery.rs b/crates/smtp/src/outbound/delivery.rs index 02d10ef4..b19f77af 100644 --- a/crates/smtp/src/outbound/delivery.rs +++ b/crates/smtp/src/outbound/delivery.rs @@ -192,6 +192,7 @@ impl DeliveryAttempt { mta_sts: *queue_config.tls.mta_sts.eval(&envelope).await, ..Default::default() }; + let allow_invalid_certs = *queue_config.tls.invalid_certs.eval(&envelope).await; // Obtain TLS reporting let tls_report = match core.report.config.tls.send.eval(&envelope).await { @@ -638,13 +639,12 @@ impl DeliveryAttempt { || (self.message.flags & MAIL_REQUIRETLS) != 0 || mta_sts_policy.is_some() || dane_policy.is_some(); - let tls_connector = if !is_strict_tls || remote_host.allow_invalid_certs() { - // Many mail servers on the internet have invalid certificates, if TLS is set to optional and - // the remote host does not have a DANE or MTA-STS policy, then we allow invalid certificates. - &core.queue.connectors.dummy_verify - } else { - &core.queue.connectors.pki_verify - }; + let tls_connector = + if allow_invalid_certs || remote_host.allow_invalid_certs() { + &core.queue.connectors.dummy_verify + } else { + &core.queue.connectors.pki_verify + }; let delivery_result = if !remote_host.implicit_tls() { // Read greeting diff --git a/crates/smtp/src/scripts/exec.rs b/crates/smtp/src/scripts/exec.rs index b961b65c..f21bd2cd 100644 --- a/crates/smtp/src/scripts/exec.rs +++ b/crates/smtp/src/scripts/exec.rs @@ -106,7 +106,7 @@ impl<T: AsyncWrite + AsyncRead + Unpin + IsTls> Session<T> { // Build recipients list let mut recipients = vec![]; for rcpt in &self.data.rcpt_to { - recipients.push(Variable::String(rcpt.address_lcase.to_string())); + recipients.push(Variable::from(rcpt.address_lcase.to_string())); } params.envelope.push((Envelope::To, recipients.into())); } diff --git a/crates/smtp/src/scripts/functions/array.rs b/crates/smtp/src/scripts/functions/array.rs index e9943db5..450de724 100644 --- a/crates/smtp/src/scripts/functions/array.rs +++ b/crates/smtp/src/scripts/functions/array.rs @@ -21,19 +21,15 @@ * for more details. */ -use std::{ - borrow::Cow, - collections::{HashMap, HashSet}, -}; +use std::collections::{HashMap, HashSet}; use sieve::{runtime::Variable, Context}; use crate::config::scripts::SieveContext; -pub fn fn_count<'x>(_: &'x Context<'x, SieveContext>, v: Vec<Variable<'x>>) -> Variable<'x> { +pub fn fn_count<'x>(_: &'x Context<'x, SieveContext>, v: Vec<Variable>) -> Variable { match &v[0] { Variable::Array(a) => a.len(), - Variable::ArrayRef(a) => a.len(), v => { if !v.is_empty() { 1 @@ -45,9 +41,9 @@ pub fn fn_count<'x>(_: &'x Context<'x, SieveContext>, v: Vec<Variable<'x>>) -> V .into() } -pub fn fn_sort<'x>(_: &'x Context<'x, SieveContext>, v: Vec<Variable<'x>>) -> Variable<'x> { +pub fn fn_sort<'x>(_: &'x Context<'x, SieveContext>, v: Vec<Variable>) -> Variable { let is_asc = v[1].to_bool(); - let mut arr = v.into_iter().next().unwrap().into_array(); + let mut arr = (*v[0].to_array()).clone(); if is_asc { arr.sort_unstable_by(|a, b| b.cmp(a)); } else { @@ -56,39 +52,31 @@ pub fn fn_sort<'x>(_: &'x Context<'x, SieveContext>, v: Vec<Variable<'x>>) -> Va arr.into() } -pub fn fn_dedup<'x>(_: &'x Context<'x, SieveContext>, v: Vec<Variable<'x>>) -> Variable<'x> { - let arr = v.into_iter().next().unwrap().into_array(); +pub fn fn_dedup<'x>(_: &'x Context<'x, SieveContext>, v: Vec<Variable>) -> Variable { + let arr = v[0].to_array(); let mut result = Vec::with_capacity(arr.len()); - for item in arr { - if !result.contains(&item) { - result.push(item); + for item in arr.iter() { + if !result.contains(item) { + result.push(item.clone()); } } result.into() } -pub fn fn_cosine_similarity<'x>( - _: &'x Context<'x, SieveContext>, - v: Vec<Variable<'x>>, -) -> Variable<'x> { - let mut word_freq: HashMap<Cow<str>, [u32; 2]> = HashMap::new(); +pub fn fn_cosine_similarity<'x>(_: &'x Context<'x, SieveContext>, v: Vec<Variable>) -> Variable { + let mut word_freq: HashMap<Variable, [u32; 2]> = HashMap::new(); for (idx, var) in v.into_iter().enumerate() { match var { Variable::Array(l) => { - for item in l { - word_freq.entry(item.into_cow()).or_insert([0, 0])[idx] += 1; - } - } - Variable::ArrayRef(l) => { - for item in l { - word_freq.entry(item.to_cow()).or_insert([0, 0])[idx] += 1; + for item in l.iter() { + word_freq.entry(item.clone()).or_insert([0, 0])[idx] += 1; } } _ => { - for char in var.to_cow().chars() { + for char in var.to_string().chars() { word_freq.entry(char.to_string().into()).or_insert([0, 0])[idx] += 1; } } @@ -113,26 +101,18 @@ pub fn fn_cosine_similarity<'x>( .into() } -pub fn fn_jaccard_similarity<'x>( - _: &'x Context<'x, SieveContext>, - v: Vec<Variable<'x>>, -) -> Variable<'x> { +pub fn fn_jaccard_similarity<'x>(_: &'x Context<'x, SieveContext>, v: Vec<Variable>) -> Variable { let mut word_freq = [HashSet::new(), HashSet::new()]; for (idx, var) in v.into_iter().enumerate() { match var { Variable::Array(l) => { - for item in l { - word_freq[idx].insert(item.into_cow()); - } - } - Variable::ArrayRef(l) => { - for item in l { - word_freq[idx].insert(item.to_cow()); + for item in l.iter() { + word_freq[idx].insert(item.clone()); } } _ => { - for char in var.to_cow().chars() { + for char in var.to_string().chars() { word_freq[idx].insert(char.to_string().into()); } } @@ -150,35 +130,21 @@ pub fn fn_jaccard_similarity<'x>( .into() } -pub fn fn_is_intersect<'x>(_: &'x Context<'x, SieveContext>, v: Vec<Variable<'x>>) -> Variable<'x> { +pub fn fn_is_intersect<'x>(_: &'x Context<'x, SieveContext>, v: Vec<Variable>) -> Variable { match (&v[0], &v[1]) { (Variable::Array(a), Variable::Array(b)) => a.iter().any(|x| b.contains(x)), - (Variable::ArrayRef(a), Variable::ArrayRef(b)) => a.iter().any(|x| b.contains(x)), - (Variable::Array(a), Variable::ArrayRef(b)) - | (Variable::ArrayRef(b), Variable::Array(a)) => a.iter().any(|x| b.contains(x)), (Variable::Array(a), item) | (item, Variable::Array(a)) => a.contains(item), - (Variable::ArrayRef(a), item) | (item, Variable::ArrayRef(a)) => a.contains(item), _ => false, } .into() } -pub fn fn_winnow<'x>(_: &'x Context<'x, SieveContext>, mut v: Vec<Variable<'x>>) -> Variable<'x> { +pub fn fn_winnow<'x>(_: &'x Context<'x, SieveContext>, mut v: Vec<Variable>) -> Variable { match v.remove(0) { Variable::Array(a) => a - .into_iter() - .filter(|i| !i.is_empty()) - .collect::<Vec<_>>() - .into(), - Variable::ArrayRef(a) => a .iter() - .filter_map(|i| { - if !i.is_empty() { - i.clone().into() - } else { - None - } - }) + .filter(|i| !i.is_empty()) + .cloned() .collect::<Vec<_>>() .into(), v => v, diff --git a/crates/smtp/src/scripts/functions/email.rs b/crates/smtp/src/scripts/functions/email.rs index 70424579..93bf4264 100644 --- a/crates/smtp/src/scripts/functions/email.rs +++ b/crates/smtp/src/scripts/functions/email.rs @@ -27,7 +27,7 @@ use crate::config::scripts::SieveContext; use super::ApplyString; -pub fn fn_is_email<'x>(_: &'x Context<'x, SieveContext>, v: Vec<Variable<'x>>) -> Variable<'x> { +pub fn fn_is_email<'x>(_: &'x Context<'x, SieveContext>, v: Vec<Variable>) -> Variable { let mut last_ch = 0; let mut in_quote = false; let mut at_count = 0; @@ -35,7 +35,7 @@ pub fn fn_is_email<'x>(_: &'x Context<'x, SieveContext>, v: Vec<Variable<'x>>) - let mut lp_len = 0; let mut value = 0; - for ch in v[0].to_cow().bytes() { + for ch in v[0].to_string().bytes() { match ch { b'0'..=b'9' | b'a'..=b'z' @@ -97,12 +97,12 @@ pub fn fn_is_email<'x>(_: &'x Context<'x, SieveContext>, v: Vec<Variable<'x>>) - (at_count == 1 && dot_count > 0 && lp_len > 0 && value > 0).into() } -pub fn fn_email_part<'x>(_: &'x Context<'x, SieveContext>, v: Vec<Variable<'x>>) -> Variable<'x> { +pub fn fn_email_part<'x>(_: &'x Context<'x, SieveContext>, v: Vec<Variable>) -> Variable { v[0].transform(|s| { s.rsplit_once('@') - .map(|(u, d)| match v[1].to_cow().as_ref() { - "local" => Variable::StringRef(u.trim()), - "domain" => Variable::StringRef(d.trim()), + .map(|(u, d)| match v[1].to_string().as_ref() { + "local" => Variable::from(u.trim()), + "domain" => Variable::from(d.trim()), _ => Variable::default(), }) .unwrap_or_default() @@ -116,11 +116,8 @@ enum MatchPart { Host, } -pub fn fn_domain_part<'x>( - ctx: &'x Context<'x, SieveContext>, - v: Vec<Variable<'x>>, -) -> Variable<'x> { - let match_part = match v[1].to_cow().as_ref() { +pub fn fn_domain_part<'x>(ctx: &'x Context<'x, SieveContext>, v: Vec<Variable>) -> Variable { + let match_part = match v[1].to_string().as_ref() { "sld" => MatchPart::Sld, "tld" => MatchPart::Tld, "host" => MatchPart::Host, diff --git a/crates/smtp/src/scripts/functions/header.rs b/crates/smtp/src/scripts/functions/header.rs index c54f1495..84180f76 100644 --- a/crates/smtp/src/scripts/functions/header.rs +++ b/crates/smtp/src/scripts/functions/header.rs @@ -28,12 +28,9 @@ use crate::config::scripts::SieveContext; use super::ApplyString; -pub fn fn_received_part<'x>( - ctx: &'x Context<'x, SieveContext>, - v: Vec<Variable<'x>>, -) -> Variable<'x> { +pub fn fn_received_part<'x>(ctx: &'x Context<'x, SieveContext>, v: Vec<Variable>) -> Variable { if let (Ok(part), Some(HeaderValue::Received(rcvd))) = ( - ReceivedPart::try_from(v[1].to_cow().as_ref()), + ReceivedPart::try_from(v[1].to_string().as_ref()), ctx.message() .part(ctx.part()) .and_then(|p| { @@ -52,8 +49,8 @@ pub fn fn_received_part<'x>( pub fn fn_is_encoding_problem<'x>( ctx: &'x Context<'x, SieveContext>, - _: Vec<Variable<'x>>, -) -> Variable<'x> { + _: Vec<Variable>, +) -> Variable { ctx.message() .part(ctx.part()) .map(|p| p.is_encoding_problem) @@ -61,22 +58,16 @@ pub fn fn_is_encoding_problem<'x>( .into() } -pub fn fn_is_attachment<'x>( - ctx: &'x Context<'x, SieveContext>, - _: Vec<Variable<'x>>, -) -> Variable<'x> { +pub fn fn_is_attachment<'x>(ctx: &'x Context<'x, SieveContext>, _: Vec<Variable>) -> Variable { ctx.message().attachments.contains(&ctx.part()).into() } -pub fn fn_is_body<'x>(ctx: &'x Context<'x, SieveContext>, _: Vec<Variable<'x>>) -> Variable<'x> { +pub fn fn_is_body<'x>(ctx: &'x Context<'x, SieveContext>, _: Vec<Variable>) -> Variable { (ctx.message().text_body.contains(&ctx.part()) || ctx.message().html_body.contains(&ctx.part())) .into() } -pub fn fn_attachment_name<'x>( - ctx: &'x Context<'x, SieveContext>, - _: Vec<Variable<'x>>, -) -> Variable<'x> { +pub fn fn_attachment_name<'x>(ctx: &'x Context<'x, SieveContext>, _: Vec<Variable>) -> Variable { ctx.message() .part(ctx.part()) .and_then(|p| p.attachment_name()) @@ -84,20 +75,20 @@ pub fn fn_attachment_name<'x>( .into() } -pub fn fn_thread_name<'x>(_: &'x Context<'x, SieveContext>, v: Vec<Variable<'x>>) -> Variable<'x> { +pub fn fn_thread_name<'x>(_: &'x Context<'x, SieveContext>, v: Vec<Variable>) -> Variable { v[0].transform(|s| thread_name(s).into()) } pub fn fn_is_header_utf8_valid<'x>( ctx: &'x Context<'x, SieveContext>, - v: Vec<Variable<'x>>, -) -> Variable<'x> { + v: Vec<Variable>, +) -> Variable { ctx.message() .part(ctx.part()) .map(|p| { let raw = ctx.message().raw_message(); let mut is_valid = true; - if let Some(header_name) = HeaderName::parse(v[0].to_cow().as_ref()) { + if let Some(header_name) = HeaderName::parse(v[0].to_string().as_ref()) { for header in &p.headers { if header.name == header_name && raw diff --git a/crates/smtp/src/scripts/functions/html.rs b/crates/smtp/src/scripts/functions/html.rs index 874b5294..9879075f 100644 --- a/crates/smtp/src/scripts/functions/html.rs +++ b/crates/smtp/src/scripts/functions/html.rs @@ -28,16 +28,16 @@ use sieve::{runtime::Variable, Context}; use crate::config::scripts::SieveContext; -pub fn fn_html_to_text<'x>(_: &'x Context<'x, SieveContext>, v: Vec<Variable<'x>>) -> Variable<'x> { - html_to_text(v[0].to_cow().as_ref()).into() +pub fn fn_html_to_text<'x>(_: &'x Context<'x, SieveContext>, v: Vec<Variable>) -> Variable { + html_to_text(v[0].to_string().as_ref()).into() } -pub fn fn_html_has_tag<'x>(_: &'x Context<'x, SieveContext>, v: Vec<Variable<'x>>) -> Variable<'x> { +pub fn fn_html_has_tag<'x>(_: &'x Context<'x, SieveContext>, v: Vec<Variable>) -> Variable { v[0].as_array() .map(|arr| { - let token = v[1].to_cow(); + let token = v[1].to_string(); arr.iter().any(|v| { - v.to_cow() + v.to_string() .as_ref() .strip_prefix('<') .map_or(false, |tag| tag.starts_with(token.as_ref())) @@ -47,14 +47,11 @@ pub fn fn_html_has_tag<'x>(_: &'x Context<'x, SieveContext>, v: Vec<Variable<'x> .into() } -pub fn fn_html_attr_size<'x>( - _: &'x Context<'x, SieveContext>, - v: Vec<Variable<'x>>, -) -> Variable<'x> { - let t = v[0].to_cow(); +pub fn fn_html_attr_size<'x>(_: &'x Context<'x, SieveContext>, v: Vec<Variable>) -> Variable { + let t = v[0].to_string(); let mut dimension = None; - if let Some(value) = get_attribute(t.as_ref(), v[1].to_cow().as_ref()) { + if let Some(value) = get_attribute(t.as_ref(), v[1].to_string().as_ref()) { let value = value.trim(); if let Some(pct) = value.strip_suffix('%') { if let Ok(pct) = pct.trim().parse::<u32>() { @@ -68,22 +65,22 @@ pub fn fn_html_attr_size<'x>( dimension.map(Variable::Integer).unwrap_or_default() } -pub fn fn_html_attrs<'x>(_: &'x Context<'x, SieveContext>, v: Vec<Variable<'x>>) -> Variable<'x> { +pub fn fn_html_attrs<'x>(_: &'x Context<'x, SieveContext>, v: Vec<Variable>) -> Variable { html_attr_tokens( - v[0].to_cow().as_ref(), - v[1].to_cow().as_ref(), + v[0].to_string().as_ref(), + v[1].to_string().as_ref(), v[2].to_string_array(), ) .into() } -pub fn fn_html_attr<'x>(_: &'x Context<'x, SieveContext>, v: Vec<Variable<'x>>) -> Variable<'x> { - get_attribute(v[0].to_cow().as_ref(), v[1].to_cow().as_ref()) - .map(|s| Variable::String(s.to_string())) +pub fn fn_html_attr<'x>(_: &'x Context<'x, SieveContext>, v: Vec<Variable>) -> Variable { + get_attribute(v[0].to_string().as_ref(), v[1].to_string().as_ref()) + .map(Variable::from) .unwrap_or_default() } -pub fn html_to_tokens(input: &str) -> Vec<Variable<'static>> { +pub fn html_to_tokens(input: &str) -> Vec<Variable> { let input = input.as_bytes(); let mut iter = input.iter().enumerate(); let mut tags = vec![]; @@ -110,7 +107,7 @@ pub fn html_to_tokens(input: &str) -> Vec<Variable<'static>> { is_token_start = true; } if text.len() > 1 { - tags.push(Variable::String(text)); + tags.push(Variable::String(text.into())); text = String::from("_"); } @@ -174,7 +171,9 @@ pub fn html_to_tokens(input: &str) -> Vec<Variable<'static>> { last_ch = ch; } } - tags.push(Variable::String(String::from_utf8(tag).unwrap_or_default())); + tags.push(Variable::String( + String::from_utf8(tag).unwrap_or_default().into(), + )); continue; } b' ' | b'\t' | b'\r' | b'\n' => { @@ -229,13 +228,13 @@ pub fn html_to_tokens(input: &str) -> Vec<Variable<'static>> { ); } if text.len() > 1 { - tags.push(Variable::String(text)); + tags.push(Variable::String(text.into())); } tags } -pub fn html_attr_tokens(input: &str, tag: &str, attrs: Vec<Cow<str>>) -> Vec<Variable<'static>> { +pub fn html_attr_tokens(input: &str, tag: &str, attrs: Vec<Cow<str>>) -> Vec<Variable> { let input = input.as_bytes(); let mut iter = input.iter().enumerate().peekable(); let mut tags = vec![]; @@ -279,7 +278,7 @@ pub fn html_attr_tokens(input: &str, tag: &str, attrs: Vec<Cow<str>>) -> Vec<Var b'>' if !in_quote => { if !tag.is_empty() { tags.push(Variable::String( - String::from_utf8(tag).unwrap_or_default(), + String::from_utf8(tag).unwrap_or_default().into(), )); } break 'outer; @@ -303,7 +302,7 @@ pub fn html_attr_tokens(input: &str, tag: &str, attrs: Vec<Cow<str>>) -> Vec<Var if !tag.is_empty() { tags.push(Variable::String( - String::from_utf8(tag).unwrap_or_default(), + String::from_utf8(tag).unwrap_or_default().into(), )); } } @@ -331,10 +330,10 @@ pub fn html_attr_tokens(input: &str, tag: &str, attrs: Vec<Cow<str>>) -> Vec<Var tags } -pub fn html_img_area(arr: &[Variable<'_>]) -> u32 { +pub fn html_img_area(arr: &[Variable]) -> u32 { arr.iter() .filter_map(|v| { - let t = v.to_cow(); + let t = v.to_string(); if t.starts_with("<img") { let mut dimensions = [200u32, 200u32]; diff --git a/crates/smtp/src/scripts/functions/image.rs b/crates/smtp/src/scripts/functions/image.rs index d01924d1..ffba52bd 100644 --- a/crates/smtp/src/scripts/functions/image.rs +++ b/crates/smtp/src/scripts/functions/image.rs @@ -25,18 +25,15 @@ use sieve::{runtime::Variable, Context}; use crate::config::scripts::SieveContext; -pub fn fn_img_metadata<'x>( - ctx: &'x Context<'x, SieveContext>, - v: Vec<Variable<'x>>, -) -> Variable<'x> { +pub fn fn_img_metadata<'x>(ctx: &'x Context<'x, SieveContext>, v: Vec<Variable>) -> Variable { ctx.message() .part(ctx.part()) .map(|p| p.contents()) .and_then(|bytes| { - let arg = v[1].to_cow(); + let arg = v[1].to_string(); match arg.as_ref() { "type" => imagesize::image_type(bytes).ok().map(|t| { - Variable::StringRef(match t { + Variable::from(match t { imagesize::ImageType::Aseprite => "aseprite", imagesize::ImageType::Avif => "avif", imagesize::ImageType::Bmp => "bmp", diff --git a/crates/smtp/src/scripts/functions/misc.rs b/crates/smtp/src/scripts/functions/misc.rs index 5ad80087..ed1ae329 100644 --- a/crates/smtp/src/scripts/functions/misc.rs +++ b/crates/smtp/src/scripts/functions/misc.rs @@ -32,56 +32,48 @@ use crate::config::scripts::SieveContext; use super::ApplyString; -pub fn fn_is_empty<'x>(_: &'x Context<'x, SieveContext>, v: Vec<Variable<'x>>) -> Variable<'x> { +pub fn fn_is_empty<'x>(_: &'x Context<'x, SieveContext>, v: Vec<Variable>) -> Variable { match &v[0] { Variable::String(s) => s.is_empty(), - Variable::StringRef(s) => s.is_empty(), Variable::Integer(_) | Variable::Float(_) => false, Variable::Array(a) => a.is_empty(), - Variable::ArrayRef(a) => a.is_empty(), } .into() } -pub fn fn_is_ip_addr<'x>(_: &'x Context<'x, SieveContext>, v: Vec<Variable<'x>>) -> Variable<'x> { - v[0].to_cow().parse::<std::net::IpAddr>().is_ok().into() +pub fn fn_is_ip_addr<'x>(_: &'x Context<'x, SieveContext>, v: Vec<Variable>) -> Variable { + v[0].to_string().parse::<std::net::IpAddr>().is_ok().into() } -pub fn fn_is_ipv4_addr<'x>(_: &'x Context<'x, SieveContext>, v: Vec<Variable<'x>>) -> Variable<'x> { - v[0].to_cow() +pub fn fn_is_ipv4_addr<'x>(_: &'x Context<'x, SieveContext>, v: Vec<Variable>) -> Variable { + v[0].to_string() .parse::<std::net::IpAddr>() .map_or(false, |ip| matches!(ip, IpAddr::V4(_))) .into() } -pub fn fn_is_ipv6_addr<'x>(_: &'x Context<'x, SieveContext>, v: Vec<Variable<'x>>) -> Variable<'x> { - v[0].to_cow() +pub fn fn_is_ipv6_addr<'x>(_: &'x Context<'x, SieveContext>, v: Vec<Variable>) -> Variable { + v[0].to_string() .parse::<std::net::IpAddr>() .map_or(false, |ip| matches!(ip, IpAddr::V6(_))) .into() } -pub fn fn_ip_reverse_name<'x>( - _: &'x Context<'x, SieveContext>, - v: Vec<Variable<'x>>, -) -> Variable<'x> { - v[0].to_cow() +pub fn fn_ip_reverse_name<'x>(_: &'x Context<'x, SieveContext>, v: Vec<Variable>) -> Variable { + v[0].to_string() .parse::<std::net::IpAddr>() .map(|ip| ip.to_reverse_name()) .unwrap_or_default() .into() } -pub fn fn_detect_file_type<'x>( - ctx: &'x Context<'x, SieveContext>, - v: Vec<Variable<'x>>, -) -> Variable<'x> { +pub fn fn_detect_file_type<'x>(ctx: &'x Context<'x, SieveContext>, v: Vec<Variable>) -> Variable { ctx.message() .part(ctx.part()) .and_then(|p| infer::get(p.contents())) .map(|t| { - Variable::String( - if v[0].to_cow() != "ext" { + Variable::from( + if v[0].to_string() != "ext" { t.mime_type() } else { t.extension() @@ -92,9 +84,9 @@ pub fn fn_detect_file_type<'x>( .unwrap_or_default() } -pub fn fn_hash<'x>(_: &'x Context<'x, SieveContext>, v: Vec<Variable<'x>>) -> Variable<'x> { +pub fn fn_hash<'x>(_: &'x Context<'x, SieveContext>, v: Vec<Variable>) -> Variable { use sha1::Digest; - let hash = v[1].to_cow(); + let hash = v[1].to_string(); v[0].transform(|value| match hash.as_ref() { "md5" => format!("{:x}", md5::compute(value.as_bytes())).into(), @@ -117,13 +109,11 @@ pub fn fn_hash<'x>(_: &'x Context<'x, SieveContext>, v: Vec<Variable<'x>>) -> Va }) } -pub fn fn_is_var_names<'x>( - ctx: &'x Context<'x, SieveContext>, - _: Vec<Variable<'x>>, -) -> Variable<'x> { +pub fn fn_is_var_names<'x>(ctx: &'x Context<'x, SieveContext>, _: Vec<Variable>) -> Variable { Variable::Array( ctx.global_variable_names() .map(|v| Variable::from(v.to_string())) - .collect(), + .collect::<Vec<_>>() + .into(), ) } diff --git a/crates/smtp/src/scripts/functions/mod.rs b/crates/smtp/src/scripts/functions/mod.rs index b6632e98..c09a9239 100644 --- a/crates/smtp/src/scripts/functions/mod.rs +++ b/crates/smtp/src/scripts/functions/mod.rs @@ -112,33 +112,22 @@ pub fn register_functions() -> FunctionMap<SieveContext> { } pub trait ApplyString<'x> { - fn transform(&self, f: impl Fn(&'_ str) -> Variable<'_>) -> Variable<'x>; + fn transform(&self, f: impl Fn(&'_ str) -> Variable) -> Variable; } -impl<'x> ApplyString<'x> for Variable<'x> { - fn transform(&self, f: impl Fn(&'_ str) -> Variable<'_>) -> Variable<'x> { +impl<'x> ApplyString<'x> for Variable { + fn transform(&self, f: impl Fn(&'_ str) -> Variable) -> Variable { match self { - Variable::StringRef(s) => f(s), - Variable::String(s) => f(s).into_owned(), - Variable::ArrayRef(list) => list - .iter() - .map(|v| match v { - Variable::String(s) => f(s), - Variable::StringRef(s) => f(s), - v => f(v.to_cow().as_ref()).into_owned(), - }) - .collect::<Vec<_>>() - .into(), + Variable::String(s) => f(s), Variable::Array(list) => list .iter() .map(|v| match v { - Variable::StringRef(s) => f(s), - Variable::String(s) => f(s).into_owned(), - v => f(v.to_cow().as_ref()).into_owned(), + Variable::String(s) => f(s), + v => f(v.to_string().as_ref()), }) .collect::<Vec<_>>() .into(), - v => f(v.to_cow().as_ref()).into_owned(), + v => f(v.to_string().as_ref()), } } } diff --git a/crates/smtp/src/scripts/functions/text.rs b/crates/smtp/src/scripts/functions/text.rs index 1fc06be3..567a43cf 100644 --- a/crates/smtp/src/scripts/functions/text.rs +++ b/crates/smtp/src/scripts/functions/text.rs @@ -28,38 +28,36 @@ use crate::config::scripts::SieveContext; use super::{html::html_to_tokens, ApplyString}; -pub fn fn_trim<'x>(_: &'x Context<'x, SieveContext>, v: Vec<Variable<'x>>) -> Variable<'x> { - v[0].transform(|s| Variable::StringRef(s.trim())) +pub fn fn_trim<'x>(_: &'x Context<'x, SieveContext>, v: Vec<Variable>) -> Variable { + v[0].transform(|s| Variable::from(s.trim())) } -pub fn fn_trim_end<'x>(_: &'x Context<'x, SieveContext>, v: Vec<Variable<'x>>) -> Variable<'x> { - v[0].transform(|s| Variable::StringRef(s.trim_end())) +pub fn fn_trim_end<'x>(_: &'x Context<'x, SieveContext>, v: Vec<Variable>) -> Variable { + v[0].transform(|s| Variable::from(s.trim_end())) } -pub fn fn_trim_start<'x>(_: &'x Context<'x, SieveContext>, v: Vec<Variable<'x>>) -> Variable<'x> { - v[0].transform(|s| Variable::StringRef(s.trim_start())) +pub fn fn_trim_start<'x>(_: &'x Context<'x, SieveContext>, v: Vec<Variable>) -> Variable { + v[0].transform(|s| Variable::from(s.trim_start())) } -pub fn fn_len<'x>(_: &'x Context<'x, SieveContext>, v: Vec<Variable<'x>>) -> Variable<'x> { +pub fn fn_len<'x>(_: &'x Context<'x, SieveContext>, v: Vec<Variable>) -> Variable { match &v[0] { Variable::String(s) => s.len(), - Variable::StringRef(s) => s.len(), Variable::Array(a) => a.len(), - Variable::ArrayRef(a) => a.len(), v => v.to_string().len(), } .into() } -pub fn fn_to_lowercase<'x>(_: &'x Context<'x, SieveContext>, v: Vec<Variable<'x>>) -> Variable<'x> { - v[0].transform(|s| Variable::String(s.to_lowercase())) +pub fn fn_to_lowercase<'x>(_: &'x Context<'x, SieveContext>, v: Vec<Variable>) -> Variable { + v[0].transform(|s| Variable::from(s.to_lowercase())) } -pub fn fn_to_uppercase<'x>(_: &'x Context<'x, SieveContext>, v: Vec<Variable<'x>>) -> Variable<'x> { - v[0].transform(|s| Variable::String(s.to_uppercase())) +pub fn fn_to_uppercase<'x>(_: &'x Context<'x, SieveContext>, v: Vec<Variable>) -> Variable { + v[0].transform(|s| Variable::from(s.to_uppercase())) } -pub fn fn_is_uppercase<'x>(_: &'x Context<'x, SieveContext>, v: Vec<Variable<'x>>) -> Variable<'x> { +pub fn fn_is_uppercase<'x>(_: &'x Context<'x, SieveContext>, v: Vec<Variable>) -> Variable { v[0].transform(|s| { s.chars() .filter(|c| c.is_alphabetic()) @@ -68,7 +66,7 @@ pub fn fn_is_uppercase<'x>(_: &'x Context<'x, SieveContext>, v: Vec<Variable<'x> }) } -pub fn fn_is_lowercase<'x>(_: &'x Context<'x, SieveContext>, v: Vec<Variable<'x>>) -> Variable<'x> { +pub fn fn_is_lowercase<'x>(_: &'x Context<'x, SieveContext>, v: Vec<Variable>) -> Variable { v[0].transform(|s| { s.chars() .filter(|c| c.is_alphabetic()) @@ -77,38 +75,22 @@ pub fn fn_is_lowercase<'x>(_: &'x Context<'x, SieveContext>, v: Vec<Variable<'x> }) } -pub fn fn_has_digits<'x>(_: &'x Context<'x, SieveContext>, v: Vec<Variable<'x>>) -> Variable<'x> { +pub fn fn_has_digits<'x>(_: &'x Context<'x, SieveContext>, v: Vec<Variable>) -> Variable { v[0].transform(|s| s.chars().any(|c| c.is_ascii_digit()).into()) } -pub fn tokenize_words<'x>(v: &Variable<'x>) -> Variable<'x> { - match v { - Variable::StringRef(s) => s - .split_whitespace() - .filter(|word| word.chars().all(|c| c.is_alphanumeric())) - .map(Variable::from) - .collect::<Vec<_>>(), - Variable::String(s) => s - .split_whitespace() - .filter(|word| word.chars().all(|c| c.is_alphanumeric())) - .map(|word| Variable::from(word.to_string())) - .collect::<Vec<_>>(), - v => v - .to_string() - .split_whitespace() - .filter(|word| word.chars().all(|c| c.is_alphanumeric())) - .map(|word| Variable::from(word.to_string())) - .collect::<Vec<_>>(), - } - .into() +pub fn tokenize_words(v: &Variable) -> Variable { + v.to_string() + .split_whitespace() + .filter(|word| word.chars().all(|c| c.is_alphanumeric())) + .map(|word| Variable::from(word.to_string())) + .collect::<Vec<_>>() + .into() } -pub fn fn_tokenize<'x>( - ctx: &'x Context<'x, SieveContext>, - mut v: Vec<Variable<'x>>, -) -> Variable<'x> { - let (urls, urls_without_scheme, emails) = match v[1].to_cow().as_ref() { - "html" => return html_to_tokens(v[0].to_cow().as_ref()).into(), +pub fn fn_tokenize<'x>(ctx: &'x Context<'x, SieveContext>, mut v: Vec<Variable>) -> Variable { + let (urls, urls_without_scheme, emails) = match v[1].to_string().as_ref() { + "html" => return html_to_tokens(v[0].to_string().as_ref()).into(), "words" => return tokenize_words(&v[0]), "uri" | "url" => (true, true, true), "uri_strict" | "url_strict" => (true, false, false), @@ -117,33 +99,18 @@ pub fn fn_tokenize<'x>( }; match v.remove(0) { - Variable::StringRef(text) => TypesTokenizer::new(text, &ctx.context().psl) - .tokenize_numbers(false) - .tokenize_urls(urls) - .tokenize_urls_without_scheme(urls_without_scheme) - .tokenize_emails(emails) - .filter_map(|t| match t.word { - TokenType::Url(text) if urls => Variable::StringRef(text).into(), - TokenType::UrlNoScheme(text) if urls_without_scheme => { - Variable::String(format!("https://{text}")).into() - } - TokenType::Email(text) if emails => Variable::StringRef(text).into(), - _ => None, - }) - .collect::<Vec<_>>() - .into(), - v @ (Variable::String(_) | Variable::Array(_) | Variable::ArrayRef(_)) => { - TypesTokenizer::new(v.to_cow().as_ref(), &ctx.context().psl) + v @ (Variable::String(_) | Variable::Array(_)) => { + TypesTokenizer::new(v.to_string().as_ref(), &ctx.context().psl) .tokenize_numbers(false) .tokenize_urls(urls) .tokenize_urls_without_scheme(urls_without_scheme) .tokenize_emails(emails) .filter_map(|t| match t.word { - TokenType::Url(text) if urls => Variable::String(text.to_string()).into(), + TokenType::Url(text) if urls => Variable::from(text.to_string()).into(), TokenType::UrlNoScheme(text) if urls_without_scheme => { - Variable::String(format!("https://{text}")).into() + Variable::from(format!("https://{text}")).into() } - TokenType::Email(text) if emails => Variable::String(text.to_string()).into(), + TokenType::Email(text) if emails => Variable::from(text.to_string()).into(), _ => None, }) .collect::<Vec<_>>() @@ -153,8 +120,8 @@ pub fn fn_tokenize<'x>( } } -pub fn fn_count_spaces<'x>(_: &'x Context<'x, SieveContext>, v: Vec<Variable<'x>>) -> Variable<'x> { - v[0].to_cow() +pub fn fn_count_spaces<'x>(_: &'x Context<'x, SieveContext>, v: Vec<Variable>) -> Variable { + v[0].to_string() .as_ref() .chars() .filter(|c| c.is_whitespace()) @@ -162,11 +129,8 @@ pub fn fn_count_spaces<'x>(_: &'x Context<'x, SieveContext>, v: Vec<Variable<'x> .into() } -pub fn fn_count_uppercase<'x>( - _: &'x Context<'x, SieveContext>, - v: Vec<Variable<'x>>, -) -> Variable<'x> { - v[0].to_cow() +pub fn fn_count_uppercase<'x>(_: &'x Context<'x, SieveContext>, v: Vec<Variable>) -> Variable { + v[0].to_string() .as_ref() .chars() .filter(|c| c.is_alphabetic() && c.is_uppercase()) @@ -174,11 +138,8 @@ pub fn fn_count_uppercase<'x>( .into() } -pub fn fn_count_lowercase<'x>( - _: &'x Context<'x, SieveContext>, - v: Vec<Variable<'x>>, -) -> Variable<'x> { - v[0].to_cow() +pub fn fn_count_lowercase<'x>(_: &'x Context<'x, SieveContext>, v: Vec<Variable>) -> Variable { + v[0].to_string() .as_ref() .chars() .filter(|c| c.is_alphabetic() && c.is_lowercase()) @@ -186,46 +147,31 @@ pub fn fn_count_lowercase<'x>( .into() } -pub fn fn_count_chars<'x>(_: &'x Context<'x, SieveContext>, v: Vec<Variable<'x>>) -> Variable<'x> { - v[0].to_cow().as_ref().chars().count().into() +pub fn fn_count_chars<'x>(_: &'x Context<'x, SieveContext>, v: Vec<Variable>) -> Variable { + v[0].to_string().as_ref().chars().count().into() } -pub fn fn_eq_ignore_case<'x>( - _: &'x Context<'x, SieveContext>, - v: Vec<Variable<'x>>, -) -> Variable<'x> { - v[0].to_cow() - .eq_ignore_ascii_case(v[1].to_cow().as_ref()) +pub fn fn_eq_ignore_case<'x>(_: &'x Context<'x, SieveContext>, v: Vec<Variable>) -> Variable { + v[0].to_string() + .eq_ignore_ascii_case(v[1].to_string().as_ref()) .into() } -pub fn fn_contains<'x>(_: &'x Context<'x, SieveContext>, v: Vec<Variable<'x>>) -> Variable<'x> { +pub fn fn_contains<'x>(_: &'x Context<'x, SieveContext>, v: Vec<Variable>) -> Variable { match &v[0] { - Variable::String(s) => s.contains(v[1].to_cow().as_ref()), - Variable::StringRef(s) => s.contains(v[1].to_cow().as_ref()), + Variable::String(s) => s.contains(v[1].to_string().as_ref()), Variable::Array(arr) => arr.contains(&v[1]), - Variable::ArrayRef(arr) => arr.contains(&v[1]), - val => val.to_string().contains(v[1].to_cow().as_ref()), + val => val.to_string().contains(v[1].to_string().as_ref()), } .into() } -pub fn fn_contains_ignore_case<'x>( - _: &'x Context<'x, SieveContext>, - v: Vec<Variable<'x>>, -) -> Variable<'x> { - let needle = v[1].to_cow(); +pub fn fn_contains_ignore_case<'x>(_: &'x Context<'x, SieveContext>, v: Vec<Variable>) -> Variable { + let needle = v[1].to_string(); match &v[0] { Variable::String(s) => s.to_lowercase().contains(&needle.to_lowercase()), - Variable::StringRef(s) => s.to_lowercase().contains(&needle.to_lowercase()), Variable::Array(arr) => arr.iter().any(|v| match v { Variable::String(s) => s.eq_ignore_ascii_case(needle.as_ref()), - Variable::StringRef(s) => s.eq_ignore_ascii_case(needle.as_ref()), - _ => false, - }), - Variable::ArrayRef(arr) => arr.iter().any(|v| match v { - Variable::String(s) => s.eq_ignore_ascii_case(needle.as_ref()), - Variable::StringRef(s) => s.eq_ignore_ascii_case(needle.as_ref()), _ => false, }), val => val.to_string().contains(needle.as_ref()), @@ -233,28 +179,29 @@ pub fn fn_contains_ignore_case<'x>( .into() } -pub fn fn_starts_with<'x>(_: &'x Context<'x, SieveContext>, v: Vec<Variable<'x>>) -> Variable<'x> { - v[0].to_cow().starts_with(v[1].to_cow().as_ref()).into() +pub fn fn_starts_with<'x>(_: &'x Context<'x, SieveContext>, v: Vec<Variable>) -> Variable { + v[0].to_string() + .starts_with(v[1].to_string().as_ref()) + .into() } -pub fn fn_ends_with<'x>(_: &'x Context<'x, SieveContext>, v: Vec<Variable<'x>>) -> Variable<'x> { - v[0].to_cow().ends_with(v[1].to_cow().as_ref()).into() +pub fn fn_ends_with<'x>(_: &'x Context<'x, SieveContext>, v: Vec<Variable>) -> Variable { + v[0].to_string().ends_with(v[1].to_string().as_ref()).into() } -pub fn fn_lines<'x>(_: &'x Context<'x, SieveContext>, mut v: Vec<Variable<'x>>) -> Variable<'x> { +pub fn fn_lines<'x>(_: &'x Context<'x, SieveContext>, mut v: Vec<Variable>) -> Variable { match v.remove(0) { - Variable::StringRef(s) => s.lines().map(Variable::from).collect::<Vec<_>>().into(), Variable::String(s) => s .lines() - .map(|s| Variable::String(s.to_string())) + .map(|s| Variable::from(s.to_string())) .collect::<Vec<_>>() .into(), val => val, } } -pub fn fn_substring<'x>(_: &'x Context<'x, SieveContext>, v: Vec<Variable<'x>>) -> Variable<'x> { - v[0].to_cow() +pub fn fn_substring<'x>(_: &'x Context<'x, SieveContext>, v: Vec<Variable>) -> Variable { + v[0].to_string() .chars() .skip(v[1].to_usize()) .take(v[2].to_usize()) @@ -262,120 +209,60 @@ pub fn fn_substring<'x>(_: &'x Context<'x, SieveContext>, v: Vec<Variable<'x>>) .into() } -pub fn fn_strip_prefix<'x>(_: &'x Context<'x, SieveContext>, v: Vec<Variable<'x>>) -> Variable<'x> { - let prefix = v[1].to_cow(); +pub fn fn_strip_prefix<'x>(_: &'x Context<'x, SieveContext>, v: Vec<Variable>) -> Variable { + let prefix = v[1].to_string(); v[0].transform(|s| { s.strip_prefix(prefix.as_ref()) - .map(Variable::StringRef) + .map(Variable::from) .unwrap_or_default() }) } -pub fn fn_strip_suffix<'x>(_: &'x Context<'x, SieveContext>, v: Vec<Variable<'x>>) -> Variable<'x> { - let suffix = v[1].to_cow(); +pub fn fn_strip_suffix<'x>(_: &'x Context<'x, SieveContext>, v: Vec<Variable>) -> Variable { + let suffix = v[1].to_string(); v[0].transform(|s| { s.strip_suffix(suffix.as_ref()) - .map(Variable::StringRef) + .map(Variable::from) .unwrap_or_default() }) } -pub fn fn_split<'x>(_: &'x Context<'x, SieveContext>, v: Vec<Variable<'x>>) -> Variable<'x> { - match &v[0] { - Variable::StringRef(s) => s - .split(v[1].to_cow().as_ref()) - .map(Variable::from) - .collect::<Vec<_>>() - .into(), - Variable::String(s) => s - .split(v[1].to_cow().as_ref()) - .map(|s| Variable::String(s.to_string())) - .collect::<Vec<_>>() - .into(), - val => val - .to_string() - .split(v[1].to_cow().as_ref()) - .map(|s| Variable::String(s.to_string())) - .collect::<Vec<_>>() - .into(), - } +pub fn fn_split<'x>(_: &'x Context<'x, SieveContext>, v: Vec<Variable>) -> Variable { + v[0].to_string() + .split(v[1].to_string().as_ref()) + .map(|s| Variable::from(s.to_string())) + .collect::<Vec<_>>() + .into() } -pub fn fn_rsplit<'x>(_: &'x Context<'x, SieveContext>, v: Vec<Variable<'x>>) -> Variable<'x> { - match &v[0] { - Variable::StringRef(s) => s - .rsplit(v[1].to_cow().as_ref()) - .map(Variable::from) - .collect::<Vec<_>>() - .into(), - Variable::String(s) => s - .rsplit(v[1].to_cow().as_ref()) - .map(|s| Variable::String(s.to_string())) - .collect::<Vec<_>>() - .into(), - val => val - .to_string() - .rsplit(v[1].to_cow().as_ref()) - .map(|s| Variable::String(s.to_string())) - .collect::<Vec<_>>() - .into(), - } +pub fn fn_rsplit<'x>(_: &'x Context<'x, SieveContext>, v: Vec<Variable>) -> Variable { + v[0].to_string() + .rsplit(v[1].to_string().as_ref()) + .map(|s| Variable::from(s.to_string())) + .collect::<Vec<_>>() + .into() } -pub fn fn_split_once<'x>(_: &'x Context<'x, SieveContext>, v: Vec<Variable<'x>>) -> Variable<'x> { - match &v[0] { - Variable::StringRef(s) => s - .split_once(v[1].to_cow().as_ref()) - .map(|(a, b)| Variable::Array(vec![Variable::StringRef(a), Variable::StringRef(b)])) - .unwrap_or_default(), - Variable::String(s) => s - .split_once(v[1].to_cow().as_ref()) - .map(|(a, b)| { - Variable::Array(vec![ - Variable::String(a.to_string()), - Variable::String(b.to_string()), - ]) - }) - .unwrap_or_default(), - val => val - .to_string() - .split_once(v[1].to_cow().as_ref()) - .map(|(a, b)| { - Variable::Array(vec![ - Variable::String(a.to_string()), - Variable::String(b.to_string()), - ]) - }) - .unwrap_or_default(), - } +pub fn fn_split_once<'x>(_: &'x Context<'x, SieveContext>, v: Vec<Variable>) -> Variable { + v[0].to_string() + .split_once(v[1].to_string().as_ref()) + .map(|(a, b)| { + Variable::Array( + vec![Variable::from(a.to_string()), Variable::from(b.to_string())].into(), + ) + }) + .unwrap_or_default() } -pub fn fn_rsplit_once<'x>(_: &'x Context<'x, SieveContext>, v: Vec<Variable<'x>>) -> Variable<'x> { - match &v[0] { - Variable::StringRef(s) => s - .rsplit_once(v[1].to_cow().as_ref()) - .map(|(a, b)| Variable::Array(vec![Variable::StringRef(a), Variable::StringRef(b)])) - .unwrap_or_default(), - Variable::String(s) => s - .rsplit_once(v[1].to_cow().as_ref()) - .map(|(a, b)| { - Variable::Array(vec![ - Variable::String(a.to_string()), - Variable::String(b.to_string()), - ]) - }) - .unwrap_or_default(), - val => val - .to_string() - .rsplit_once(v[1].to_cow().as_ref()) - .map(|(a, b)| { - Variable::Array(vec![ - Variable::String(a.to_string()), - Variable::String(b.to_string()), - ]) - }) - .unwrap_or_default(), - } +pub fn fn_rsplit_once<'x>(_: &'x Context<'x, SieveContext>, v: Vec<Variable>) -> Variable { + v[0].to_string() + .rsplit_once(v[1].to_string().as_ref()) + .map(|(a, b)| { + Variable::Array( + vec![Variable::from(a.to_string()), Variable::from(b.to_string())].into(), + ) + }) + .unwrap_or_default() } /** @@ -385,12 +272,9 @@ pub fn fn_rsplit_once<'x>(_: &'x Context<'x, SieveContext>, v: Vec<Variable<'x>> * * Copyright (c) 2016 Titus Wormer <tituswormer@gmail.com> */ -pub fn fn_levenshtein_distance<'x>( - _: &'x Context<'x, SieveContext>, - v: Vec<Variable<'x>>, -) -> Variable<'x> { - let a = v[0].to_cow(); - let b = v[1].to_cow(); +pub fn fn_levenshtein_distance<'x>(_: &'x Context<'x, SieveContext>, v: Vec<Variable>) -> Variable { + let a = v[0].to_string(); + let b = v[1].to_string(); let mut result = 0; @@ -449,11 +333,8 @@ pub fn fn_levenshtein_distance<'x>( result.into() } -pub fn fn_detect_language<'x>( - _: &'x Context<'x, SieveContext>, - v: Vec<Variable<'x>>, -) -> Variable<'x> { - whatlang::detect_lang(v[0].to_cow().as_ref()) +pub fn fn_detect_language<'x>(_: &'x Context<'x, SieveContext>, v: Vec<Variable>) -> Variable { + whatlang::detect_lang(v[0].to_string().as_ref()) .map(|l| l.code()) .unwrap_or("unknown") .into() diff --git a/crates/smtp/src/scripts/functions/unicode.rs b/crates/smtp/src/scripts/functions/unicode.rs index 21fbca67..c11721e3 100644 --- a/crates/smtp/src/scripts/functions/unicode.rs +++ b/crates/smtp/src/scripts/functions/unicode.rs @@ -26,37 +26,23 @@ use unicode_security::MixedScript; use crate::config::scripts::SieveContext; -pub fn fn_is_ascii<'x>(_: &'x Context<'x, SieveContext>, v: Vec<Variable<'x>>) -> Variable<'x> { +pub fn fn_is_ascii<'x>(_: &'x Context<'x, SieveContext>, v: Vec<Variable>) -> Variable { match &v[0] { Variable::String(s) => s.chars().all(|c| c.is_ascii()), - Variable::StringRef(s) => s.chars().all(|c| c.is_ascii()), Variable::Integer(_) | Variable::Float(_) => true, Variable::Array(a) => a.iter().all(|v| match v { Variable::String(s) => s.chars().all(|c| c.is_ascii()), - Variable::StringRef(s) => s.chars().all(|c| c.is_ascii()), - _ => true, - }), - Variable::ArrayRef(a) => a.iter().all(|v| match v { - Variable::String(s) => s.chars().all(|c| c.is_ascii()), - Variable::StringRef(s) => s.chars().all(|c| c.is_ascii()), _ => true, }), } .into() } -pub fn fn_has_zwsp<'x>(_: &'x Context<'x, SieveContext>, v: Vec<Variable<'x>>) -> Variable<'x> { +pub fn fn_has_zwsp<'x>(_: &'x Context<'x, SieveContext>, v: Vec<Variable>) -> Variable { match &v[0] { Variable::String(s) => s.chars().any(|c| c.is_zwsp()), - Variable::StringRef(s) => s.chars().any(|c| c.is_zwsp()), Variable::Array(a) => a.iter().any(|v| match v { Variable::String(s) => s.chars().any(|c| c.is_zwsp()), - Variable::StringRef(s) => s.chars().any(|c| c.is_zwsp()), - _ => true, - }), - Variable::ArrayRef(a) => a.iter().any(|v| match v { - Variable::String(s) => s.chars().any(|c| c.is_zwsp()), - Variable::StringRef(s) => s.chars().any(|c| c.is_zwsp()), _ => true, }), Variable::Integer(_) | Variable::Float(_) => false, @@ -64,18 +50,11 @@ pub fn fn_has_zwsp<'x>(_: &'x Context<'x, SieveContext>, v: Vec<Variable<'x>>) - .into() } -pub fn fn_has_obscured<'x>(_: &'x Context<'x, SieveContext>, v: Vec<Variable<'x>>) -> Variable<'x> { +pub fn fn_has_obscured<'x>(_: &'x Context<'x, SieveContext>, v: Vec<Variable>) -> Variable { match &v[0] { Variable::String(s) => s.chars().any(|c| c.is_obscured()), - Variable::StringRef(s) => s.chars().any(|c| c.is_obscured()), Variable::Array(a) => a.iter().any(|v| match v { Variable::String(s) => s.chars().any(|c| c.is_obscured()), - Variable::StringRef(s) => s.chars().any(|c| c.is_obscured()), - _ => true, - }), - Variable::ArrayRef(a) => a.iter().any(|v| match v { - Variable::String(s) => s.chars().any(|c| c.is_obscured()), - Variable::StringRef(s) => s.chars().any(|c| c.is_obscured()), _ => true, }), Variable::Integer(_) | Variable::Float(_) => false, @@ -107,24 +86,18 @@ impl CharUtils for char { } } -pub fn fn_cure_text<'x>(_: &'x Context<'x, SieveContext>, v: Vec<Variable<'x>>) -> Variable<'x> { - decancer::cure(v[0].to_cow().as_ref()).into_str().into() +pub fn fn_cure_text<'x>(_: &'x Context<'x, SieveContext>, v: Vec<Variable>) -> Variable { + decancer::cure(v[0].to_string().as_ref()).into_str().into() } -pub fn fn_unicode_skeleton<'x>( - _: &'x Context<'x, SieveContext>, - v: Vec<Variable<'x>>, -) -> Variable<'x> { - unicode_security::skeleton(v[0].to_cow().as_ref()) +pub fn fn_unicode_skeleton<'x>(_: &'x Context<'x, SieveContext>, v: Vec<Variable>) -> Variable { + unicode_security::skeleton(v[0].to_string().as_ref()) .collect::<String>() .into() } -pub fn fn_is_single_script<'x>( - _: &'x Context<'x, SieveContext>, - v: Vec<Variable<'x>>, -) -> Variable<'x> { - let text = v[0].to_cow(); +pub fn fn_is_single_script<'x>(_: &'x Context<'x, SieveContext>, v: Vec<Variable>) -> Variable { + let text = v[0].to_string(); if !text.is_empty() { text.as_ref().is_single_script() } else { diff --git a/crates/smtp/src/scripts/functions/url.rs b/crates/smtp/src/scripts/functions/url.rs index 9ecbfd5a..b1f9ef3b 100644 --- a/crates/smtp/src/scripts/functions/url.rs +++ b/crates/smtp/src/scripts/functions/url.rs @@ -28,32 +28,30 @@ use crate::config::scripts::SieveContext; use super::ApplyString; -pub fn fn_uri_part<'x>(_: &'x Context<'x, SieveContext>, v: Vec<Variable<'x>>) -> Variable<'x> { - let part = v[1].to_cow(); +pub fn fn_uri_part<'x>(_: &'x Context<'x, SieveContext>, v: Vec<Variable>) -> Variable { + let part = v[1].to_string(); v[0].transform(|uri| { uri.parse::<Uri>() .ok() .and_then(|uri| match part.as_ref() { - "scheme" => uri.scheme_str().map(|s| Variable::String(s.to_string())), - "host" => uri.host().map(|s| Variable::String(s.to_string())), + "scheme" => uri.scheme_str().map(|s| Variable::from(s.to_string())), + "host" => uri.host().map(|s| Variable::from(s.to_string())), "scheme_host" => uri .scheme_str() .and_then(|s| (s, uri.host()?).into()) - .map(|(s, h)| Variable::String(format!("{}://{}", s, h))), - "path" => Variable::String(uri.path().to_string()).into(), + .map(|(s, h)| Variable::from(format!("{}://{}", s, h))), + "path" => Variable::from(uri.path().to_string()).into(), "port" => uri.port_u16().map(|port| Variable::Integer(port as i64)), - "query" => uri.query().map(|s| Variable::String(s.to_string())), - "path_query" => uri - .path_and_query() - .map(|s| Variable::String(s.to_string())), - "authority" => uri.authority().map(|s| Variable::String(s.to_string())), + "query" => uri.query().map(|s| Variable::from(s.to_string())), + "path_query" => uri.path_and_query().map(|s| Variable::from(s.to_string())), + "authority" => uri.authority().map(|s| Variable::from(s.to_string())), _ => None, }) .unwrap_or_default() }) } -pub fn fn_puny_decode<'x>(_: &'x Context<'x, SieveContext>, v: Vec<Variable<'x>>) -> Variable<'x> { +pub fn fn_puny_decode<'x>(_: &'x Context<'x, SieveContext>, v: Vec<Variable>) -> Variable { v[0].transform(|domain| { if domain.contains("xn--") { let mut decoded = String::with_capacity(domain.len()); diff --git a/crates/smtp/src/scripts/mod.rs b/crates/smtp/src/scripts/mod.rs index d3a9dc9c..c53b686d 100644 --- a/crates/smtp/src/scripts/mod.rs +++ b/crates/smtp/src/scripts/mod.rs @@ -47,10 +47,10 @@ pub enum ScriptResult { pub struct ScriptParameters { message: Option<Arc<Vec<u8>>>, - variables: AHashMap<Cow<'static, str>, Variable<'static>>, - envelope: Vec<(Envelope, Variable<'static>)>, + variables: AHashMap<Cow<'static, str>, Variable>, + envelope: Vec<(Envelope, Variable)>, #[cfg(feature = "test_mode")] - expected_variables: Option<AHashMap<String, Variable<'static>>>, + expected_variables: Option<AHashMap<String, Variable>>, } impl ScriptParameters { @@ -74,7 +74,7 @@ impl ScriptParameters { pub fn set_variable( mut self, name: impl Into<Cow<'static, str>>, - value: impl Into<Variable<'static>>, + value: impl Into<Variable>, ) -> Self { self.variables.insert(name.into(), value.into()); self @@ -83,7 +83,7 @@ impl ScriptParameters { #[cfg(feature = "test_mode")] pub fn with_expected_variables( mut self, - expected_variables: AHashMap<String, Variable<'static>>, + expected_variables: AHashMap<String, Variable>, ) -> Self { self.expected_variables = expected_variables.into(); self diff --git a/crates/smtp/src/scripts/plugins/bayes.rs b/crates/smtp/src/scripts/plugins/bayes.rs index e14f6b37..50648c69 100644 --- a/crates/smtp/src/scripts/plugins/bayes.rs +++ b/crates/smtp/src/scripts/plugins/bayes.rs @@ -34,28 +34,42 @@ use crate::config::scripts::SieveContext; use super::PluginContext; pub fn register_train(plugin_id: u32, fnc_map: &mut FunctionMap<SieveContext>) { - fnc_map.set_external_function("bayes_train", plugin_id, 2); + fnc_map.set_external_function("bayes_train", plugin_id, 3); } pub fn register_untrain(plugin_id: u32, fnc_map: &mut FunctionMap<SieveContext>) { - fnc_map.set_external_function("bayes_untrain", plugin_id, 2); + fnc_map.set_external_function("bayes_untrain", plugin_id, 3); } pub fn register_classify(plugin_id: u32, fnc_map: &mut FunctionMap<SieveContext>) { - fnc_map.set_external_function("bayes_classify", plugin_id, 1); + fnc_map.set_external_function("bayes_classify", plugin_id, 2); } -pub fn exec_train(ctx: PluginContext<'_>) -> Variable<'static> { +pub fn exec_train(ctx: PluginContext<'_>) -> Variable { train(ctx, true) } -pub fn exec_untrain(ctx: PluginContext<'_>) -> Variable<'static> { +pub fn exec_untrain(ctx: PluginContext<'_>) -> Variable { train(ctx, false) } -fn train(ctx: PluginContext<'_>, is_train: bool) -> Variable<'static> { - let mut arguments = ctx.arguments.into_iter(); - let text = arguments.next().unwrap().into_string(); +fn train(ctx: PluginContext<'_>, is_train: bool) -> Variable { + let span = ctx.span; + let lookup_id = ctx.arguments[0].to_string(); + let lookup_train = if let Some(lookup_train) = ctx.core.sieve.lookup.get(lookup_id.as_ref()) { + lookup_train + } else { + tracing::warn!( + parent: span, + context = "sieve:bayes_train", + event = "failed", + reason = "Unknown lookup id", + lookup_id = %lookup_id, + ); + return false.into(); + }; + let text = ctx.arguments[1].to_string(); + let is_spam = ctx.arguments[2].to_bool(); if text.is_empty() { return false.into(); } @@ -63,7 +77,6 @@ fn train(ctx: PluginContext<'_>, is_train: bool) -> Variable<'static> { let ctx = ctx.core.sieve.runtime.context(); // Train the model - let is_spam = arguments.next().unwrap().to_bool(); let mut model = BayesModel::default(); model.train( OsbTokenizer::new(BayesTokenizer::new(text.as_ref(), &ctx.psl), 5), @@ -74,7 +87,6 @@ fn train(ctx: PluginContext<'_>, is_train: bool) -> Variable<'static> { } // Update weight and invalidate cache - let upsert = &ctx.lookup_train; for (hash, weights) in model.weights { let (s_weight, h_weight) = if is_train { (weights.spam as i64, weights.ham as i64) @@ -82,7 +94,7 @@ fn train(ctx: PluginContext<'_>, is_train: bool) -> Variable<'static> { (-(weights.spam as i64), -(weights.ham as i64)) }; if handle - .block_on(upsert.lookup(&[ + .block_on(lookup_train.lookup(&[ hash.h1.into(), hash.h2.into(), s_weight.into(), @@ -103,7 +115,7 @@ fn train(ctx: PluginContext<'_>, is_train: bool) -> Variable<'static> { (0i64, train_val) }; if handle - .block_on(upsert.query(&[ + .block_on(lookup_train.query(&[ 0i64.into(), 0i64.into(), spam_count.into(), @@ -118,29 +130,42 @@ fn train(ctx: PluginContext<'_>, is_train: bool) -> Variable<'static> { true.into() } -pub fn exec_classify(ctx: PluginContext<'_>) -> Variable<'static> { - let mut arguments = ctx.arguments.into_iter(); - let text = arguments.next().unwrap().into_string(); +pub fn exec_classify(ctx: PluginContext<'_>) -> Variable { + let span = ctx.span; + let lookup_id = ctx.arguments[0].to_string(); + let lookup_classify = + if let Some(lookup_classify) = ctx.core.sieve.lookup.get(lookup_id.as_ref()) { + lookup_classify + } else { + tracing::warn!( + parent: span, + context = "sieve:bayes_classify", + event = "failed", + reason = "Unknown lookup id", + lookup_id = %lookup_id, + ); + return Variable::default(); + }; + let text = ctx.arguments[1].to_string(); if text.is_empty() { - return 0.into(); + return Variable::default(); } let handle = ctx.handle; let ctx = ctx.core.sieve.runtime.context(); - let get_token = &ctx.lookup_classify; // Obtain training counts let (spam_learns, ham_learns) = if let Some(weights) = ctx.bayes_cache - .get_or_update(TokenHash::default(), handle, get_token) + .get_or_update(TokenHash::default(), handle, lookup_classify) { (weights.spam, weights.ham) } else { - return 0.into(); + return Variable::default(); }; // Make sure we have enough training data if spam_learns < ctx.bayes_classify.min_learns || ham_learns < ctx.bayes_classify.min_learns { - return 0.into(); + return Variable::default(); } // Classify the text @@ -149,7 +174,9 @@ pub fn exec_classify(ctx: PluginContext<'_>) -> Variable<'static> { OsbTokenizer::<_, TokenHash>::new(BayesTokenizer::new(text.as_ref(), &ctx.psl), 5) .filter_map(|t| { OsbToken { - inner: ctx.bayes_cache.get_or_update(t.inner, handle, get_token)?, + inner: ctx + .bayes_cache + .get_or_update(t.inner, handle, lookup_classify)?, idx: t.idx, } .into() @@ -157,8 +184,8 @@ pub fn exec_classify(ctx: PluginContext<'_>) -> Variable<'static> { ham_learns, spam_learns, ) + .map(Variable::from) .unwrap_or_default() - .into() } trait LookupOrInsert { diff --git a/crates/smtp/src/scripts/plugins/dns.rs b/crates/smtp/src/scripts/plugins/dns.rs index d938be6d..a7fc7e30 100644 --- a/crates/smtp/src/scripts/plugins/dns.rs +++ b/crates/smtp/src/scripts/plugins/dns.rs @@ -38,9 +38,9 @@ pub fn register_exists(plugin_id: u32, fnc_map: &mut FunctionMap<SieveContext>) fnc_map.set_external_function("dns_exists", plugin_id, 2); } -pub fn exec(ctx: PluginContext<'_>) -> Variable<'static> { - let entry = ctx.arguments[0].to_cow(); - let record_type = ctx.arguments[1].to_cow(); +pub fn exec(ctx: PluginContext<'_>) -> Variable { + let entry = ctx.arguments[0].to_string(); + let record_type = ctx.arguments[1].to_string(); if record_type.eq_ignore_ascii_case("ip") { match ctx.handle.block_on(ctx.core.resolvers.dns.ip_lookup( @@ -50,7 +50,7 @@ pub fn exec(ctx: PluginContext<'_>) -> Variable<'static> { )) { Ok(result) => result .iter() - .map(|ip| Variable::String(ip.to_string())) + .map(|ip| Variable::from(ip.to_string())) .collect::<Vec<_>>() .into(), Err(err) => err.short_error().into(), @@ -65,7 +65,7 @@ pub fn exec(ctx: PluginContext<'_>) -> Variable<'static> { .flat_map(|mx| { mx.exchanges .iter() - .map(|host| Variable::String(format!("{} {}", mx.preference, host))) + .map(|host| Variable::from(format!("{} {}", mx.preference, host))) }) .collect::<Vec<_>>() .into(), @@ -75,7 +75,7 @@ pub fn exec(ctx: PluginContext<'_>) -> Variable<'static> { #[cfg(feature = "test_mode")] { if entry.contains("origin") { - return Variable::String("23028|US|arin|2002-01-04".to_string()); + return Variable::from("23028|US|arin|2002-01-04".to_string()); } } @@ -83,7 +83,7 @@ pub fn exec(ctx: PluginContext<'_>) -> Variable<'static> { .handle .block_on(ctx.core.resolvers.dns.txt_raw_lookup(entry.as_ref())) { - Ok(result) => Variable::String(String::from_utf8(result).unwrap_or_default()), + Ok(result) => Variable::from(String::from_utf8(result).unwrap_or_default()), Err(err) => err.short_error().into(), } } else if record_type.eq_ignore_ascii_case("ptr") { @@ -91,7 +91,7 @@ pub fn exec(ctx: PluginContext<'_>) -> Variable<'static> { match ctx.handle.block_on(ctx.core.resolvers.dns.ptr_lookup(addr)) { Ok(result) => result .iter() - .map(|host| Variable::String(host.to_string())) + .map(|host| Variable::from(host.to_string())) .collect::<Vec<_>>() .into(), Err(err) => err.short_error().into(), @@ -104,7 +104,7 @@ pub fn exec(ctx: PluginContext<'_>) -> Variable<'static> { { if entry.contains(".168.192.") { let parts = entry.split('.').collect::<Vec<_>>(); - return vec![Variable::String(format!("127.0.{}.{}", parts[1], parts[0]))].into(); + return vec![Variable::from(format!("127.0.{}.{}", parts[1], parts[0]))].into(); } } @@ -114,7 +114,7 @@ pub fn exec(ctx: PluginContext<'_>) -> Variable<'static> { { Ok(result) => result .iter() - .map(|ip| Variable::String(ip.to_string())) + .map(|ip| Variable::from(ip.to_string())) .collect::<Vec<_>>() .into(), Err(err) => err.short_error().into(), @@ -126,7 +126,7 @@ pub fn exec(ctx: PluginContext<'_>) -> Variable<'static> { { Ok(result) => result .iter() - .map(|ip| Variable::String(ip.to_string())) + .map(|ip| Variable::from(ip.to_string())) .collect::<Vec<_>>() .into(), Err(err) => err.short_error().into(), @@ -136,9 +136,9 @@ pub fn exec(ctx: PluginContext<'_>) -> Variable<'static> { } } -pub fn exec_exists(ctx: PluginContext<'_>) -> Variable<'static> { - let entry = ctx.arguments[0].to_cow(); - let record_type = ctx.arguments[1].to_cow(); +pub fn exec_exists(ctx: PluginContext<'_>) -> Variable { + let entry = ctx.arguments[0].to_string(); + let record_type = ctx.arguments[1].to_string(); if record_type.eq_ignore_ascii_case("ip") { match ctx.handle.block_on(ctx.core.resolvers.dns.ip_lookup( diff --git a/crates/smtp/src/scripts/plugins/exec.rs b/crates/smtp/src/scripts/plugins/exec.rs index 8096829f..053e7912 100644 --- a/crates/smtp/src/scripts/plugins/exec.rs +++ b/crates/smtp/src/scripts/plugins/exec.rs @@ -33,13 +33,13 @@ pub fn register(plugin_id: u32, fnc_map: &mut FunctionMap<SieveContext>) { fnc_map.set_external_function("exec", plugin_id, 2); } -pub fn exec(ctx: PluginContext<'_>) -> Variable<'static> { +pub fn exec(ctx: PluginContext<'_>) -> Variable { let span = ctx.span; let mut arguments = ctx.arguments.into_iter(); match Command::new( arguments .next() - .map(|a| a.into_string()) + .map(|a| a.to_string().into_owned()) .unwrap_or_default(), ) .args( diff --git a/crates/smtp/src/scripts/plugins/http.rs b/crates/smtp/src/scripts/plugins/http.rs index 54ecb4b5..77dc4bad 100644 --- a/crates/smtp/src/scripts/plugins/http.rs +++ b/crates/smtp/src/scripts/plugins/http.rs @@ -34,15 +34,15 @@ pub fn register_header(plugin_id: u32, fnc_map: &mut FunctionMap<SieveContext>) fnc_map.set_external_function("http_header", plugin_id, 4); } -pub fn exec_header(ctx: PluginContext<'_>) -> Variable<'static> { - let url = ctx.arguments[0].to_cow(); - let header = ctx.arguments[1].to_cow(); - let agent = ctx.arguments[2].to_cow(); - let timeout = ctx.arguments[3].to_cow().parse::<u64>().unwrap_or(5000); +pub fn exec_header(ctx: PluginContext<'_>) -> Variable { + let url = ctx.arguments[0].to_string(); + let header = ctx.arguments[1].to_string(); + let agent = ctx.arguments[2].to_string(); + let timeout = ctx.arguments[3].to_string().parse::<u64>().unwrap_or(5000); #[cfg(feature = "test_mode")] if url.contains("redirect.") { - return Variable::String(url.split_once("/?").unwrap().1.to_string()); + return Variable::from(url.split_once("/?").unwrap().1.to_string()); } if let Ok(client) = reqwest::Client::builder() @@ -61,7 +61,7 @@ pub fn exec_header(ctx: PluginContext<'_>) -> Variable<'static> { .headers() .get(header.as_ref()) .and_then(|h| h.to_str().ok()) - .map(|h| Variable::String(h.to_string())) + .map(|h| Variable::from(h.to_string())) }) .unwrap_or_default() } else { diff --git a/crates/smtp/src/scripts/plugins/lookup.rs b/crates/smtp/src/scripts/plugins/lookup.rs index 706cbaed..a300ebdd 100644 --- a/crates/smtp/src/scripts/plugins/lookup.rs +++ b/crates/smtp/src/scripts/plugins/lookup.rs @@ -36,39 +36,43 @@ pub fn register_map(plugin_id: u32, fnc_map: &mut FunctionMap<SieveContext>) { fnc_map.set_external_function("lookup_map", plugin_id, 2); } -pub fn exec(ctx: PluginContext<'_>) -> Variable<'static> { - let lookup_id = ctx.arguments[0].to_cow(); - let item = ctx.arguments[1].to_cow(); +pub fn exec(ctx: PluginContext<'_>) -> Variable { + let lookup_id = ctx.arguments[0].to_string(); let span = ctx.span; - - if !lookup_id.is_empty() && !item.is_empty() { - if let Some(lookup) = ctx.core.sieve.lookup.get(lookup_id.as_ref()) { - return ctx - .handle - .block_on(lookup.contains(item.as_ref())) - .unwrap_or(false) - .into(); - } else { - tracing::warn!( - parent: span, - context = "sieve:lookup", - event = "failed", - reason = "Unknown lookup id", - lookup_id = %lookup_id, - ); + if let Some(lookup) = ctx.core.sieve.lookup.get(lookup_id.as_ref()) { + match &ctx.arguments[1] { + Variable::Array(items) => { + for item in items.iter() { + if !item.is_empty() + && ctx.handle.block_on(lookup.contains(item)).unwrap_or(false) + { + return true.into(); + } + } + false + } + v if !v.is_empty() => ctx.handle.block_on(lookup.contains(v)).unwrap_or(false), + _ => false, } + } else { + tracing::warn!( + parent: span, + context = "sieve:lookup", + event = "failed", + reason = "Unknown lookup id", + lookup_id = %lookup_id, + ); + false } - - false.into() + .into() } -pub fn exec_map(ctx: PluginContext<'_>) -> Variable<'static> { - let mut arguments = ctx.arguments.into_iter(); - let lookup_id = arguments.next().unwrap().into_cow(); - let items = match arguments.next().unwrap() { - Variable::Array(l) => l.into_iter().map(DatabaseColumn::from).collect(), - Variable::ArrayRef(l) => l.iter().map(DatabaseColumn::from).collect(), - v => vec![DatabaseColumn::from(v)], +pub fn exec_map(ctx: PluginContext<'_>) -> Variable { + let lookup_id = ctx.arguments[0].to_string(); + let items = match &ctx.arguments[1] { + Variable::Array(l) => l.iter().map(DatabaseColumn::from).collect(), + v if !v.is_empty() => vec![DatabaseColumn::from(v)], + _ => vec![], }; let span = ctx.span; diff --git a/crates/smtp/src/scripts/plugins/mod.rs b/crates/smtp/src/scripts/plugins/mod.rs index 5909654b..d82b9457 100644 --- a/crates/smtp/src/scripts/plugins/mod.rs +++ b/crates/smtp/src/scripts/plugins/mod.rs @@ -35,14 +35,14 @@ use tokio::runtime::Handle; use crate::{config::scripts::SieveContext, core::SMTP}; type RegisterPluginFnc = fn(u32, &mut FunctionMap<SieveContext>) -> (); -type ExecPluginFnc = fn(PluginContext<'_>) -> Variable<'static>; +type ExecPluginFnc = fn(PluginContext<'_>) -> Variable; pub struct PluginContext<'x> { pub span: &'x tracing::Span, pub handle: &'x Handle, pub core: &'x SMTP, pub message: &'x Message<'x>, - pub arguments: Vec<Variable<'static>>, + pub arguments: Vec<Variable>, } const PLUGINS_EXEC: [ExecPluginFnc; 10] = [ @@ -105,6 +105,6 @@ impl SMTP { #[cfg(feature = "test_mode")] pub fn test_print(ctx: PluginContext<'_>) -> Input { - println!("{}", ctx.arguments[0].to_cow()); + println!("{}", ctx.arguments[0].to_string()); Input::True } diff --git a/crates/smtp/src/scripts/plugins/query.rs b/crates/smtp/src/scripts/plugins/query.rs index 60ad0255..31b55db2 100644 --- a/crates/smtp/src/scripts/plugins/query.rs +++ b/crates/smtp/src/scripts/plugins/query.rs @@ -31,27 +31,27 @@ pub fn register(plugin_id: u32, fnc_map: &mut FunctionMap<SieveContext>) { fnc_map.set_external_function("query", plugin_id, 3); } -pub fn exec(ctx: PluginContext<'_>) -> Variable<'static> { +pub fn exec(ctx: PluginContext<'_>) -> Variable { let span = ctx.span; - let mut arguments = ctx.arguments.into_iter(); // Obtain directory name - let directory = arguments.next().unwrap().into_string(); - let directory = if let Some(directory_) = ctx.core.sieve.config.directories.get(&directory) { - directory_ - } else { - tracing::warn!( - parent: span, - context = "sieve:query", - event = "failed", - reason = "Unknown directory", - directory = %directory, - ); - return false.into(); - }; + let directory = ctx.arguments[0].to_string(); + let directory = + if let Some(directory_) = ctx.core.sieve.config.directories.get(directory.as_ref()) { + directory_ + } else { + tracing::warn!( + parent: span, + context = "sieve:query", + event = "failed", + reason = "Unknown directory", + directory = %directory, + ); + return false.into(); + }; // Obtain query string - let query = arguments.next().unwrap().into_string(); + let query = ctx.arguments[1].to_string(); if query.is_empty() { tracing::warn!( parent: span, @@ -63,9 +63,8 @@ pub fn exec(ctx: PluginContext<'_>) -> Variable<'static> { } // Obtain arguments - let arguments = match arguments.next().unwrap() { - Variable::Array(l) => l.into_iter().map(DatabaseColumn::from).collect(), - Variable::ArrayRef(l) => l.iter().map(DatabaseColumn::from).collect(), + let arguments = match &ctx.arguments[2] { + Variable::Array(l) => l.iter().map(DatabaseColumn::from).collect(), v => vec![DatabaseColumn::from(v)], }; @@ -81,7 +80,13 @@ pub fn exec(ctx: PluginContext<'_>) -> Variable<'static> { query_columns.pop().map(Variable::from).unwrap() } 0 => Variable::default(), - _ => Variable::Array(query_columns.into_iter().map(Variable::from).collect()), + _ => Variable::Array( + query_columns + .into_iter() + .map(Variable::from) + .collect::<Vec<_>>() + .into(), + ), } } else { false.into() diff --git a/crates/utils/src/lib.rs b/crates/utils/src/lib.rs index 7e3cf76a..871af2a1 100644 --- a/crates/utils/src/lib.rs +++ b/crates/utils/src/lib.rs @@ -119,6 +119,7 @@ pub fn enable_tracing(config: &Config, message: &str) -> config::Result<Option<W tracing::subscriber::set_global_default( tracing_subscriber::FmtSubscriber::builder() .with_env_filter(env_filter) + .with_ansi(config.property_or_static("global.tracing.ansi", "true")?) .finish(), ) .failed("Failed to set subscriber"); diff --git a/resources/config/sieve/bayes_classify.sieve b/resources/config/sieve/bayes_classify.sieve new file mode 100644 index 00000000..215608a4 --- /dev/null +++ b/resources/config/sieve/bayes_classify.sieve @@ -0,0 +1,10 @@ +if eval "!t.SPAM_TRAP && !t.TRUSTED_REPLY" { + let "bayes_result" "bayes_classify('spamdb/bayes-classify', body_and_subject)"; + if eval "!is_empty(bayes_result)" { + if eval "bayes_result > 0.7" { + let "t.BAYES_SPAM" "1"; + } elsif eval "bayes_result < 0.5" { + let "t.BAYES_HAM" "1"; + } + } +} diff --git a/resources/config/sieve/prelude.sieve b/resources/config/sieve/prelude.sieve index e87295da..8409d4b3 100644 --- a/resources/config/sieve/prelude.sieve +++ b/resources/config/sieve/prelude.sieve @@ -12,7 +12,8 @@ let "urls" "dedup(tokenize(header.subject, 'uri') + body_urls + html_body_urls)" # Obtain thread name and subject let "subject_lc" "to_lowercase(header.subject)"; -let "thread_name" "thread_name(header.subject)"; +let "subject_clean" "thread_name(header.subject)"; +let "body_and_subject" "subject_clean + text_body"; # Obtain all recipients let "recipients" "to_lowercase(header.to:cc:bcc[*].addr[*])"; diff --git a/resources/config/sieve/replies_in.sieve b/resources/config/sieve/replies_in.sieve new file mode 100644 index 00000000..c3471e55 --- /dev/null +++ b/resources/config/sieve/replies_in.sieve @@ -0,0 +1,4 @@ + +if eval "lookup('spamdb/id-lookup', header.In-Reply-To:References)" { + let "t.TRUSTED_REPLY" "1"; +} diff --git a/resources/config/sieve/replies_out.sieve b/resources/config/sieve/replies_out.sieve new file mode 100644 index 00000000..8c015ef0 --- /dev/null +++ b/resources/config/sieve/replies_out.sieve @@ -0,0 +1,11 @@ + +# This script should be used on authenticated SMTP sessions only +let "message_id" "header.Message-ID"; + +if eval "!is_empty(message_id)" { + eval "lookup('spamdb/id-insert', message_id)"; + + if eval "lookup('spam/options', 'AUTOLEARN_REPLIES')" { + eval "bayes_train('spamdb/bayes-train', thread_name(header.subject) + ' ' + body.to_text, false)"; + } +} diff --git a/resources/config/sieve/spamtrap.sieve b/resources/config/sieve/spamtrap.sieve new file mode 100644 index 00000000..a9abbe6e --- /dev/null +++ b/resources/config/sieve/spamtrap.sieve @@ -0,0 +1,6 @@ + +# Check if the message was sent to a spam trap address +if eval "lookup('spam/trap-address', envelope.to)" { + eval "bayes_train('spamdb/bayes-train', body_and_subject, true)"; + let "t.SPAM_TRAP" "1"; +} diff --git a/resources/config/sieve/subject.sieve b/resources/config/sieve/subject.sieve index 7683c234..bb75c199 100644 --- a/resources/config/sieve/subject.sieve +++ b/resources/config/sieve/subject.sieve @@ -2,12 +2,12 @@ let "raw_subject_lc" "to_lowercase(header.subject.raw)"; let "is_ascii_subject" "is_ascii(subject_lc)"; -if eval "len(thread_name) >= 10 && count(tokenize(thread_name, 'words')) > 1 && is_uppercase(thread_name)" { +if eval "len(subject_clean) >= 10 && count(tokenize(subject_clean, 'words')) > 1 && is_uppercase(subject_clean)" { # Subject contains mostly capital letters let "t.SUBJ_ALL_CAPS" "1"; } -if eval "count_chars(thread_name) > 200" { +if eval "count_chars(subject_clean) > 200" { # Subject is very long let "t.LONG_SUBJ" "1"; } diff --git a/resources/config/smtp.toml b/resources/config/smtp.toml index 2295e784..f5b2bb31 100644 --- a/resources/config/smtp.toml +++ b/resources/config/smtp.toml @@ -191,6 +191,7 @@ ip-strategy = "ipv4-then-ipv6" dane = "optional" mta-sts = "optional" starttls = "require" +allow-invalid-certs = false #[queue.outbound.source-ip] #v4 = ["10.0.0.10", "10.0.0.11"] diff --git a/tests/resources/smtp/antispam/bayes_classify.test b/tests/resources/smtp/antispam/bayes_classify.test new file mode 100644 index 00000000..ac9c90ba --- /dev/null +++ b/tests/resources/smtp/antispam/bayes_classify.test @@ -0,0 +1,19 @@ +expect BAYES_SPAM + +Subject: save up to NUMBER on life insurance + +why spend more than you have to life quote savings ensuring your family s financial security is very important life quote savings makes buying life insurance simple and affordable we provide free access to the very best companies and the lowest rates life quote savings is fast easy and saves you money let us help you get started with the best values in the country on new coverage you can save hundreds or even thousands of dollars by requesting a free quote from lifequote savings our service will take you less than NUMBER minutes to complete shop and compare save up to NUMBER on all types of life insurance hyperlink click here for your free quote protecting your family is the best investment you ll ever make if you are in receipt of this email in error and or wish to be removed from our list hyperlink please click here and type remove if you reside in any state which prohibits e mail solicitations for insurance please disregard this email + +<!-- NEXT TEST --> +expect BAYES_HAM + +Subject: can someone explain + +what type of operating system solaris is as ive never seen or used it i dont know wheather to get a server from sun or from dell i would prefer a linux based server and sun seems to be the one for that but im not sure if solaris is a distro of linux or a completely different operating system can someone explain kiall mac innes irish linux users group ilug URL URL for un subscription information list maintainer listmaster URL +<!-- NEXT TEST --> +expect + +Subject: classifier test + +this is a novel text that the bayes classifier has never seen before, it should be classified as ham or non-ham + diff --git a/tests/resources/smtp/antispam/replies_in.test b/tests/resources/smtp/antispam/replies_in.test new file mode 100644 index 00000000..3ba9f2a0 --- /dev/null +++ b/tests/resources/smtp/antispam/replies_in.test @@ -0,0 +1,24 @@ +expect TRUSTED_REPLY + +In-Reply-To: mid1@foobar.org +Subject: test + +test + +<!-- NEXT TEST --> +expect TRUSTED_REPLY + +References: <mid2@foobar.org> +Subject: test + +test + +<!-- NEXT TEST --> +expect + +In-Reply-To: mid1@foobar.net +References: <mid99@foobar.org> +Subject: test + +test + diff --git a/tests/resources/smtp/antispam/replies_out.test b/tests/resources/smtp/antispam/replies_out.test new file mode 100644 index 00000000..31f325ef --- /dev/null +++ b/tests/resources/smtp/antispam/replies_out.test @@ -0,0 +1,78 @@ +expect + +Message-ID: <mid1@foobar.org> +Subject: i have been trying to research via sa mirrors and search engines + +if a canned script exists giving clients access to their user_prefs options via a web based cgi interface numerous isps provide this feature to clients but so far i can find nothing our configuration uses amavis postfix and clamav for virus filtering and procmail with spamassassin for spam filtering i would prefer not to have to write a script myself but will appreciate any suggestions this URL email is sponsored by osdn tired of that same old cell phone get a new here for free URL _______________________________________________ spamassassin talk mailing list spamassassin talk URL URL + +<!-- NEXT TEST --> +expect + +Message-ID: mid2@foobar.org +Subject: hello + +have you seen and discussed this article and his approach thank you URL hell there are no rules here we re trying to accomplish something thomas alva edison this URL email is sponsored by osdn tired of that same old cell phone get a new here for free URL _______________________________________________ spamassassin devel mailing list spamassassin devel URL URL + +<!-- NEXT TEST --> +expect + +Message-ID: <mid3@foobar.org> +Subject: hi all apologies for the possible silly question + +i don t think it is but but is eircom s adsl service nat ed and what implications would that have for voip i know there are difficulties with voip or connecting to clients connected to a nat ed network from the internet wild i e machines with static real ips any help pointers would be helpful cheers rgrds bernard bernard tyers national centre for sensor research p NUMBER NUMBER NUMBER NUMBER e bernard tyers URL w URL l nNUMBER _______________________________________________ iiu mailing list iiu URL URL + +<!-- NEXT TEST --> +expect + +Message-ID: <mid4@foobar.org> +Subject: can someone explain + +what type of operating system solaris is as ive never seen or used it i dont know wheather to get a server from sun or from dell i would prefer a linux based server and sun seems to be the one for that but im not sure if solaris is a distro of linux or a completely different operating system can someone explain kiall mac innes irish linux users group ilug URL URL for un subscription information list maintainer listmaster URL + +<!-- NEXT TEST --> +expect + +Message-ID: <mid5@foobar.org> +Subject: folks my first time posting + +have a bit of unix experience but am new to linux just got a new pc at home dell box with windows xp added a second hard disk for linux partitioned the disk and have installed suse NUMBER NUMBER from cd which went fine except it didn t pick up my monitor i have a dell branded eNUMBERfpp NUMBER lcd flat panel monitor and a nvidia geforceNUMBER tiNUMBER video card both of which are probably too new to feature in suse s default set i downloaded a driver from the nvidia website and installed it using rpm then i ran saxNUMBER as was recommended in some postings i found on the net but it still doesn t feature my video card in the available list what next another problem i have a dell branded keyboard and if i hit caps lock twice the whole machine crashes in linux not windows even the on off switch is inactive leaving me to reach for the power cable instead if anyone can help me in any way with these probs i d be really grateful i ve searched the net but have run out of ideas or should i be going for a different version of linux such as redhat opinions welcome thanks a lot peter irish linux users group ilug URL URL for un subscription information list maintainer listmaster URL + +<!-- NEXT TEST --> +expect + +Message-ID: <mid6@foobar.org> +Subject: has anyone + +seen heard of used some package that would let a random person go to a webpage create a mailing list then administer that list also of course let ppl sign up for the lists and manage their subscriptions similar to the old URL but i d like to have it running on my server not someone elses chris URL + +<!-- NEXT TEST --> +expect + +Message-ID: <mid7@foobar.org> +Subject: hi thank you for the useful replies + +i have found some interesting tutorials in the ibm developer connection URL and URL registration is needed i will post the same message on the web application security list as suggested by someone for now i thing i will use mdNUMBER for password checking i will use the approach described in secure programmin fo linux and unix how to i will separate the authentication module so i can change its implementation at anytime thank you again mario torre please avoid sending me word or powerpoint attachments see URL + +<!-- NEXT TEST --> +expect + +Message-ID: <mid8@foobar.org> +Subject: hehe sorry + +but if you hit caps lock twice the computer crashes theres one ive never heard before have you tryed dell support yet i think dell computers prefer redhat dell provide some computers pre loaded with red hat i dont know for sure tho so get someone elses opnion as well as mine original message from ilug admin URL mailto ilug admin URL on behalf of peter staunton sent NUMBER august NUMBER NUMBER NUMBER to ilug URL subject ilug newbie seeks advice suse NUMBER NUMBER folks my first time posting have a bit of unix experience but am new to linux just got a new pc at home dell box with windows xp added a second hard disk for linux partitioned the disk and have installed suse NUMBER NUMBER from cd which went fine except it didn t pick up my monitor i have a dell branded eNUMBERfpp NUMBER lcd flat panel monitor and a nvidia geforceNUMBER tiNUMBER video card both of which are probably too new to feature in suse s default set i downloaded a driver from the nvidia website and installed it using rpm then i ran saxNUMBER as was recommended in some postings i found on the net but it still doesn t feature my video card in the available list what next another problem i have a dell branded keyboard and if i hit caps lock twice the whole machine crashes in linux not windows even the on off switch is inactive leaving me to reach for the power cable instead if anyone can help me in any way with these probs i d be really grateful i ve searched the net but have run out of ideas or should i be going for a different version of linux such as redhat opinions welcome thanks a lot peter irish linux users group ilug URL URL for un subscription information list maintainer listmaster URL irish linux users group ilug URL URL for un subscription information list maintainer listmaster URL + +<!-- NEXT TEST --> +expect + +Message-ID: <mid9@foobar.org> +Subject: it will function as a router + +if that is what you wish it even looks like the modem s embedded os is some kind of linux being that it has interesting interfaces like ethNUMBER i don t use it as a router though i just have it do the absolute minimum dsl stuff and do all the really fun stuff like pppoe on my linux box also the manual tells you what the default password is don t forget to run pppoe over the alcatel speedtouch NUMBERi as in my case you have to have a bridge configured in the router modem s software this lists your vci values etc also does anyone know if the high end speedtouch with NUMBER ethernet ports can act as a full router or do i still need to run a pppoe stack on the linux box regards vin irish linux users group ilug URL URL for un subscription information list maintainer listmaster URL irish linux users group ilug URL URL for un subscription information list maintainer listmaster URL + +<!-- NEXT TEST --> +expect + +Message-ID: <mid10@foobar.org> +Subject: all is it just me + +or has there been a massive increase in the amount of email being falsely bounced around the place i ve already received email from a number of people i don t know asking why i am sending them email these can be explained by servers from russia and elsewhere coupled with the false emails i received myself it s really starting to annoy me am i the only one seeing an increase in recent weeks martin martin whelan déise design URL tel NUMBER NUMBER our core product déiseditor allows organisations to publish information to their web site in a fast and cost effective manner there is no need for a full time web developer as the site can be easily updated by the organisations own staff instant updates to keep site information fresh sites which are updated regularly bring users back visit URL for a demonstration déiseditor managing your information _______________________________________________ iiu mailing list iiu URL URL ,0 diff --git a/tests/resources/smtp/antispam/spamtrap.test b/tests/resources/smtp/antispam/spamtrap.test new file mode 100644 index 00000000..d607b631 --- /dev/null +++ b/tests/resources/smtp/antispam/spamtrap.test @@ -0,0 +1,98 @@ +envelope_from spammer@domain.com +envelope_to spamtrap@foobar.org +expect SPAM_TRAP + +Subject: save up to NUMBER on life insurance + +why spend more than you have to life quote savings ensuring your family s financial security is very important life quote savings makes buying life insurance simple and affordable we provide free access to the very best companies and the lowest rates life quote savings is fast easy and saves you money let us help you get started with the best values in the country on new coverage you can save hundreds or even thousands of dollars by requesting a free quote from lifequote savings our service will take you less than NUMBER minutes to complete shop and compare save up to NUMBER on all types of life insurance hyperlink click here for your free quote protecting your family is the best investment you ll ever make if you are in receipt of this email in error and or wish to be removed from our list hyperlink please click here and type remove if you reside in any state which prohibits e mail solicitations for insurance please disregard this email + +<!-- NEXT TEST --> +envelope_from spammer@domain.com +envelope_to spamtrap@foobar.org +expect SPAM_TRAP + +Subject: a powerhouse gifting program + +you don t want to miss get in with the founders the major players are on this one for once be where the players are this is your private invitation experts are calling this the fastest way to huge cash flow ever conceived leverage NUMBER NUMBER into NUMBER NUMBER over and over again the question here is you either want to be wealthy or you don t which one are you i am tossing you a financial lifeline and for your sake i hope you grab onto it and hold on tight for the ride of your life testimonials hear what average people are doing their first few days we ve received NUMBER NUMBER in NUMBER day and we are doing that over and over again q s in al i m a single mother in fl and i ve received NUMBER NUMBER in the last NUMBER days d s in fl i was not sure about this when i sent off my NUMBER NUMBER pledge but i got back NUMBER NUMBER the very next day l l in ky i didn t have the money so i found myself a partner to work this with we have received NUMBER NUMBER over the last NUMBER days i think i made the right decision don t you k c in fl i pick up NUMBER NUMBER my first day and i they gave me free leads and all the training you can too j w in ca announcing we will close your sales for you and help you get a fax blast immediately upon your entry you make the money free leads training don t wait call now fax back to NUMBER NUMBER NUMBER NUMBER or call NUMBER NUMBER NUMBER NUMBER name__________________________________phone___________________________________________ fax_____________________________________email____________________________________________ best time to call_________________________time zone________________________________________ this message is sent in compliance of the new e mail bill per section NUMBER paragraph a NUMBER c of s NUMBER further transmissions by the sender of this email may be stopped at no cost to you by sending a reply to this email address with the word remove in the subject line errors omissions and exceptions excluded this is not spam i have compiled this list from our replicate database relative to seattle marketing group the gigt or turbo team for the sole purpose of these communications your continued inclusion is only by your gracious permission if you wish to not receive this mail from me please send an email to tesrewinter URL with remove in the subject and you will be deleted immediately + +<!-- NEXT TEST --> +envelope_from spammer@domain.com +envelope_to spamtrap@foobar.org +expect SPAM_TRAP + +Subject: help wanted + +we are a NUMBER year old fortune NUMBER company that is growing at a tremendous rate we are looking for individuals who want to work from home this is an opportunity to make an excellent income no experience is required we will train you so if you are looking to be employed from home with a career that has vast opportunities then go URL we are looking for energetic and self motivated people if that is you than click on the link and fill out the form and one of our employement specialist will contact you to be removed from our link simple go to URL + +<!-- NEXT TEST --> +envelope_from spammer@domain.com +envelope_to spamtrap@foobar.org +expect SPAM_TRAP + +Subject: tired of the bull out there + +want to stop losing money want a real money maker receive NUMBER NUMBER NUMBER NUMBER today experts are calling this the fastest way to huge cash flow ever conceived a powerhouse gifting program you don t want to miss we work as a team this is your private invitation get in with the founders this is where the big boys play the major players are on this one for once be where the players are this is a system that will drive NUMBER NUMBER s to your doorstep in a short period of time leverage NUMBER NUMBER into NUMBER NUMBER over and over again the question here is you either want to be wealthy or you don t which one are you i am tossing you a financial lifeline and for your sake i hope you grab onto it and hold on tight for the ride of your life testimonials hear what average people are doing their first few days we ve received NUMBER NUMBER in NUMBER day and we are doing that over and over again q s in al i m a single mother in fl and i ve received NUMBER NUMBER in the last NUMBER days d s in fl i was not sure about this when i sent off my NUMBER NUMBER pledge but i got back NUMBER NUMBER the very next day l l in ky i didn t have the money so i found myself a partner to work this with we have received NUMBER NUMBER over the last NUMBER days i think i made the right decision don t you k c in fl i pick up NUMBER NUMBER my first day and i they gave me free leads and all the training you can too j w in ca this will be the most important call you make this year free leads training announcing we will close your sales for you and help you get a fax blast immediately upon your entry you make the money free leads training don t wait call now NUMBER NUMBER NUMBER NUMBER print and fax to NUMBER NUMBER NUMBER NUMBER or send an email requesting more information to successleads URL please include your name and telephone number receive NUMBER NUMBER free leads just for responding a NUMBER NUMBER value name___________________________________ phone___________________________________ fax_____________________________________ email___________________________________ this message is sent in compliance of the new e mail bill per section NUMBER paragraph a NUMBER c of s NUMBER further transmissions by the sender of this email may be stopped at no cost to you by sending a reply to this email address with the word remove in the subject line errors omissions and exceptions excluded this is not spam i have compiled this list from our replicate database relative to seattle marketing group the gigt or turbo team for the sole purpose of these communications your continued inclusion is only by your gracious permission if you wish to not receive this mail from me please send an email to tesrewinter URL with remove in the subject and you will be deleted immediately + +<!-- NEXT TEST --> +envelope_from spammer@domain.com +envelope_to spamtrap@foobar.org +expect SPAM_TRAP + +Subject: cellular phone accessories + +all at below wholesale prices http NUMBER NUMBER NUMBER NUMBER NUMBER sites merchant sales hands free ear buds NUMBER NUMBER phone holsters NUMBER NUMBER booster antennas only NUMBER NUMBER phone cases NUMBER NUMBER car chargers NUMBER NUMBER face plates as low as NUMBER NUMBER lithium ion batteries as low as NUMBER NUMBER http NUMBER NUMBER NUMBER NUMBER NUMBER sites merchant sales click below for accessories on all nokia motorola lg nextel samsung qualcomm ericsson audiovox phones at below wholesale prices http NUMBER NUMBER NUMBER NUMBER NUMBER sites merchant sales if you need assistance please call us NUMBER NUMBER NUMBER to be removed from future mailings please send your remove request to remove me now NUMBER URL thank you and have a super day + +<!-- NEXT TEST --> +envelope_from spammer@domain.com +envelope_to spamtrap@foobar.org +expect SPAM_TRAP + +Subject: conferencing made easy + +only NUMBER cents per minute including long distance no setup fees no contracts or monthly fees call anytime from anywhere to anywhere connects up to NUMBER participants simplicity in set up and administration operator help available NUMBER NUMBER the highest quality service for the lowest rate in the industry fill out the form below to find out how you can lower your phone bill every month required input field name web address company name state business phone home phone email address type of business to be removed from our distribution lists please hyperlink click here + +<!-- NEXT TEST --> +envelope_from spammer@domain.com +envelope_to spamtrap@foobar.org +expect SPAM_TRAP + +Subject: dear friend + +i am mrs sese seko widow of late president mobutu sese seko of zaire now known as democratic republic of congo drc i am moved to write you this letter this was in confidence considering my presentcircumstance and situation i escaped along with my husband and two of our sons george kongolo and basher out of democratic republic of congo drc to abidjan cote d ivoire where my family and i settled while we later moved to settled in morroco where my husband later died of cancer disease however due to this situation we decided to changed most of my husband s billions of dollars deposited in swiss bank and other countries into other forms of money coded for safe purpose because the new head of state of dr mr laurent kabila has made arrangement with the swiss government and other european countries to freeze all my late husband s treasures deposited in some european countries hence my children and i decided laying low in africa to study the situation till when things gets better like now that president kabila is dead and the son taking over joseph kabila one of my late husband s chateaux in southern france was confiscated by the french government and as such i had to change my identity so that my investment will not be traced and confiscated i have deposited the sum eighteen million united state dollars us NUMBER NUMBER NUMBER NUMBER with a security company for safekeeping the funds are security coded to prevent them from knowing the content what i want you to do is to indicate your interest that you will assist us by receiving the money on our behalf acknowledge this message so that i can introduce you to my son kongolo who has the out modalities for the claim of the said funds i want you to assist in investing this money but i will not want my identity revealed i will also want to buy properties and stock in multi national companies and to engage in other safe and non speculative investments may i at this point emphasise the high level of confidentiality which this business demands and hope you will not betray the trust and confidence which i repose in you in conclusion if you want to assist us my son shall put you in the picture of the business tell you where the funds are currently being maintained and also discuss other modalities including remunerationfor your services for this reason kindly furnish us your contact information that is your personal telephone and fax number for confidential URL regards mrs m sese seko + +<!-- NEXT TEST --> +envelope_from spammer@domain.com +envelope_to spamtrap@foobar.org +expect SPAM_TRAP + +Subject: lowest rates available for term life insurance + +take a moment and fill out our online form to see the low rate you qualify for save up to NUMBER from regular rates smokers accepted URL representing quality nationwide carriers act now to easily remove your address from the list go to URL please allow NUMBER NUMBER hours for removal + +<!-- NEXT TEST --> +envelope_from spammer@domain.com +envelope_to spamtrap@foobar.org +expect SPAM_TRAP + +Subject: central bank of nigeria foreign remittance + +dept tinubu square lagos nigeria email smith_j URL NUMBERth of august NUMBER attn president ceo strictly private business proposal i am mr johnson s abu the bills and exchange director at the foreignremittance department of the central bank of nigeria i am writingyou this letter to ask for your support and cooperation to carrying thisbusiness opportunity in my department we discovered abandoned the sumof us NUMBER NUMBER NUMBER NUMBER thirty seven million four hundred thousand unitedstates dollars in an account that belong to one of our foreign customers an american late engr john creek junior an oil merchant with the federal government of nigeria who died along with his entire family of a wifeand two children in kenya airbus aNUMBER NUMBER flight kqNUMBER in novemberNUMBER since we heard of his death we have been expecting his next of kin tocome over and put claims for his money as the heir because we cannotrelease the fund from his account unless someone applies for claims asthe next of kin to the deceased as indicated in our banking guidelines unfortunately neither their family member nor distant relative hasappeared to claim the said fund upon this discovery i and other officialsin my department have agreed to make business with you release the totalamount into your account as the heir of the fund since no one came forit or discovered either maintained account with our bank other wisethe fund will be returned to the bank treasury as unclaimed fund we have agreed that our ratio of sharing will be as stated thus NUMBER for you as foreign partner and NUMBER for us the officials in my department upon the successful completion of this transfer my colleague and i willcome to your country and mind our share it is from our NUMBER we intendto import computer accessories into my country as way of recycling thefund to commence this transaction we require you to immediately indicateyour interest by calling me or sending me a fax immediately on the abovetelefax and enclose your private contact telephone fax full nameand address and your designated banking co ordinates to enable us fileletter of claim to the appropriate department for necessary approvalsbefore the transfer can be made note also this transaction must be kept strictly confidential becauseof its nature nb please remember to give me your phone and fax no mr johnson smith abu irish linux users group ilug URL URL for un subscription information list maintainer listmaster URL + +<!-- NEXT TEST --> +envelope_from spammer@domain.com +envelope_to other@foobar.org +envelope_to spamtrap@foobar.org +expect SPAM_TRAP + +Subject: dear stuart + +are you tired of searching for love in all the wrong places find love now at URL URL browse through thousands of personals in your area join for free URL search e mail chat use URL to meet cool guys and hot girls go NUMBER on NUMBER or use our private chat rooms click on the link to get started URL find love now you have received this email because you have registerd with emailrewardz or subscribed through one of our marketing partners if you have received this message in error or wish to stop receiving these great offers please click the remove link above to unsubscribe from these mailings please click here URL + +<!-- NEXT TEST --> +envelope_from spammer@domain.com +envelope_to other@foobar.org +expect + +Subject: test + +test diff --git a/tests/src/directory/sql.rs b/tests/src/directory/sql.rs index 18c27722..f94a7fb2 100644 --- a/tests/src/directory/sql.rs +++ b/tests/src/directory/sql.rs @@ -270,7 +270,7 @@ pub async fn create_test_user(handle: &dyn Directory, login: &str, secret: &str, handle .query( "INSERT OR IGNORE INTO accounts (name, secret, description, type, active) VALUES (?, ?, ?, 'individual', true)", - &[login, secret, name], + &[login.into(), secret.into(), name.into()], ) .await .unwrap(); @@ -290,7 +290,7 @@ pub async fn create_test_group(handle: &dyn Directory, login: &str, name: &str) handle .query( "INSERT OR IGNORE INTO accounts (name, description, type, active) VALUES (?, ?, 'group', true)", - &[login, name], + &[login.into(), name.into()], ) .await .unwrap(); @@ -305,7 +305,7 @@ pub async fn link_test_address(handle: &dyn Directory, login: &str, address: &st handle .query( "INSERT OR IGNORE INTO emails (name, address, type) VALUES (?, ?, ?)", - &[login, address, typ], + &[login.into(), address.into(), typ.into()], ) .await .unwrap(); @@ -315,7 +315,7 @@ pub async fn set_test_quota(handle: &dyn Directory, login: &str, quota: u32) { handle .query( &format!("UPDATE accounts SET quota = {} where name = ?", quota,), - &[login], + &[login.into()], ) .await .unwrap(); @@ -325,7 +325,7 @@ pub async fn add_to_group(handle: &dyn Directory, login: &str, group: &str) { handle .query( "INSERT INTO group_members (name, member_of) VALUES (?, ?)", - &[login, group], + &[login.into(), group.into()], ) .await .unwrap(); @@ -335,7 +335,7 @@ pub async fn remove_from_group(handle: &dyn Directory, login: &str, group: &str) handle .query( "DELETE FROM group_members WHERE name = ? AND member_of = ?", - &[login, group], + &[login.into(), group.into()], ) .await .unwrap(); @@ -345,7 +345,7 @@ pub async fn remove_test_alias(handle: &dyn Directory, login: &str, alias: &str) handle .query( "DELETE FROM emails WHERE name = ? AND address = ?", - &[login, alias], + &[login.into(), alias.into()], ) .await .unwrap(); diff --git a/tests/src/smtp/inbound/antispam.rs b/tests/src/smtp/inbound/antispam.rs index 9d6ec281..02ff5bb4 100644 --- a/tests/src/smtp/inbound/antispam.rs +++ b/tests/src/smtp/inbound/antispam.rs @@ -27,15 +27,6 @@ use utils::config::Config; use crate::smtp::{TestConfig, TestSMTP}; const CONFIG: &str = r#" -[directory."sql"] -type = "sql" -address = "sqlite://%PATH%/test_antispam.db?mode=rwc" - -[directory."sql".pool] -max-connections = 10 -min-connections = 0 -idle-timeout = "5m" - [sieve] from-name = "Sieve Daemon" from-addr = "sieve@foobar.org" @@ -51,6 +42,22 @@ cpu = 10000 nested-includes = 5 duplicate-expiry = "7d" +[directory."spamdb"] +type = "sql" +address = "sqlite://%PATH%/test_antispam.db?mode=rwc" + +[directory."spamdb".pool] +max-connections = 10 +min-connections = 0 +idle-timeout = "5m" + +[directory."spamdb".lookup] +bayes-train = "INSERT INTO bayes_weights (h1, h2, ws, wh) VALUES (?, ?, ?, ?) ON CONFLICT(h1, h2) DO UPDATE SET ws = ws + excluded.ws, wh = wh + excluded.wh" +bayes-classify = "SELECT ws, wh FROM bayes_weights WHERE h1 = ? AND h2 = ?" +id-insert = "INSERT INTO id_timestamps (id, timestamp) VALUES (?, CURRENT_TIMESTAMP)" +id-lookup = "SELECT 1 FROM id_timestamps WHERE id = ?" +id-cleanup = "DELETE FROM id_timestamps WHERE (strftime('%s', 'now') - strftime('%s', timestamp)) < ?" + [directory."spam"] type = "memory" @@ -100,12 +107,38 @@ type = "glob" comment = '#' values = ["*://phishing-tank.com", "*://phishing-tank.org"] +[directory."spam".lookup."trap-address"] +type = "glob" +comment = '#' +values = ["spamtrap@*"] + +[directory."spam".lookup."options"] +type = "list" +values = ["AUTOLEARN_REPLIES"] + [resolver] public-suffix = "file://%LIST_PATH%/public-suffix.dat" +[bayes] +min-learns = 10 + [sieve.scripts] "#; +const CREATE_TABLES: &[&str; 2] = &[ + "CREATE TABLE IF NOT EXISTS bayes_weights ( +h1 INTEGER NOT NULL, +h2 INTEGER NOT NULL, +ws INTEGER, +wh INTEGER, +PRIMARY KEY (h1, h2) +)", + "CREATE TABLE IF NOT EXISTS id_timestamps ( + id STRING PRIMARY KEY, + timestamp DATETIME NOT NULL +)", +]; + #[tokio::test(flavor = "multi_thread")] async fn antispam() { /*tracing::subscriber::set_global_default( @@ -133,6 +166,10 @@ async fn antispam() { "ip", "helo", "rbl", + "replies_out", + "replies_in", + "spamtrap", + "bayes_classify", ]; let mut core = SMTP::test(); let qr = core.init_test_queue("smtp_antispam_test"); @@ -168,6 +205,12 @@ async fn antispam() { let config = &mut core.session.config; config.rcpt.relay = IfBlock::new(true); + // Create tables + let sdb = ctx.directory.directories.get("spamdb").unwrap(); + for query in CREATE_TABLES { + sdb.query(query, &[]).await.unwrap(); + } + // Add mock DNS entries for (domain, ip) in [ ("bank.com", "127.0.0.1"), @@ -242,7 +285,7 @@ async fn antispam() { while has_more { let mut message = String::new(); let mut in_params = true; - let mut variables: HashMap<String, Variable<'_>> = HashMap::new(); + let mut variables: HashMap<String, Variable> = HashMap::new(); let mut expected_variables = AHashMap::new(); // Build session @@ -288,7 +331,7 @@ async fn antispam() { param.to_string(), value .split_ascii_whitespace() - .map(|s| Variable::String(s.to_string())) + .map(|s| Variable::from(s.to_string())) .collect::<Vec<_>>() .into(), ); @@ -397,40 +440,40 @@ fn html_tokens() { ( "<html>hello<br/>world<br/></html>", vec![ - Variable::String("<html".to_string()), - Variable::String("_hello".to_string()), - Variable::String("<br/".to_string()), - Variable::String("_world".to_string()), - Variable::String("<br/".to_string()), - Variable::String("</html".to_string()), + Variable::from("<html".to_string()), + Variable::from("_hello".to_string()), + Variable::from("<br/".to_string()), + Variable::from("_world".to_string()), + Variable::from("<br/".to_string()), + Variable::from("</html".to_string()), ], ), ( "<html>using <><br/></html>", vec![ - Variable::String("<html".to_string()), - Variable::String("_using <>".to_string()), - Variable::String("<br/".to_string()), - Variable::String("</html".to_string()), + Variable::from("<html".to_string()), + Variable::from("_using <>".to_string()), + Variable::from("<br/".to_string()), + Variable::from("</html".to_string()), ], ), ( "test <not br/>tag<br />", vec![ - Variable::String("_test".to_string()), - Variable::String("<not br/".to_string()), - Variable::String("_ tag".to_string()), - Variable::String("<br /".to_string()), + Variable::from("_test".to_string()), + Variable::from("<not br/".to_string()), + Variable::from("_ tag".to_string()), + Variable::from("<br /".to_string()), ], ), ( "<>< ><tag\n/>>hello world< br \n />", vec![ - Variable::String("<".to_string()), - Variable::String("<".to_string()), - Variable::String("<tag /".to_string()), - Variable::String("_>hello world".to_string()), - Variable::String("<br /".to_string()), + Variable::from("<".to_string()), + Variable::from("<".to_string()), + Variable::from("<tag /".to_string()), + Variable::from("_>hello world".to_string()), + Variable::from("<br /".to_string()), ], ), ( @@ -439,17 +482,17 @@ fn html_tokens() { "<h1><body></h1>" ), vec![ - Variable::String("<head".to_string()), - Variable::String("<title".to_string()), - Variable::String("_ignore head".to_string()), - Variable::String("</title".to_string()), - Variable::String("<not head".to_string()), - Variable::String("_xyz".to_string()), - Variable::String("</not head".to_string()), - Variable::String("</head".to_string()), - Variable::String("<h1".to_string()), - Variable::String("_<body>".to_string()), - Variable::String("</h1".to_string()), + Variable::from("<head".to_string()), + Variable::from("<title".to_string()), + Variable::from("_ignore head".to_string()), + Variable::from("</title".to_string()), + Variable::from("<not head".to_string()), + Variable::from("_xyz".to_string()), + Variable::from("</not head".to_string()), + Variable::from("</head".to_string()), + Variable::from("<h1".to_string()), + Variable::from("_<body>".to_string()), + Variable::from("</h1".to_string()), ], ), ( @@ -458,12 +501,12 @@ fn html_tokens() { "don't hurt me.</p>" ), vec![ - Variable::String("<p".to_string()), - Variable::String("_what is ♥?".to_string()), - Variable::String("</p".to_string()), - Variable::String("<p".to_string()), - Variable::String("_ßĂΒγ don't hurt me.".to_string()), - Variable::String("</p".to_string()), + Variable::from("<p".to_string()), + Variable::from("_what is ♥?".to_string()), + Variable::from("</p".to_string()), + Variable::from("<p".to_string()), + Variable::from("_ßĂΒγ don't hurt me.".to_string()), + Variable::from("</p".to_string()), ], ), ( @@ -473,7 +516,7 @@ fn html_tokens() { "this is <!-- <> < < < < ignore > -> here -->the actual<!--> text" ), vec![ - Variable::String( + Variable::from( concat!( "<!--[if mso]><style type=\"text/css\">body, table, ", "td, a, p, span, ul, li {font-family: Arial, sans-serif!", @@ -481,36 +524,36 @@ fn html_tokens() { ) .to_string(), ), - Variable::String("_this is".to_string()), - Variable::String("<!-- <> < < < < ignore > -> here --".to_string()), - Variable::String("_ the actual".to_string()), - Variable::String("<!--".to_string()), - Variable::String("_ text".to_string()), + Variable::from("_this is".to_string()), + Variable::from("<!-- <> < < < < ignore > -> here --".to_string()), + Variable::from("_ the actual".to_string()), + Variable::from("<!--".to_string()), + Variable::from("_ text".to_string()), ], ), ( " < p > hello < / p > < p > world < / p > !!! < br > ", vec![ - Variable::String("<p ".to_string()), - Variable::String("_hello".to_string()), - Variable::String("</p ".to_string()), - Variable::String("<p ".to_string()), - Variable::String("_ world".to_string()), - Variable::String("</p ".to_string()), - Variable::String("_ !!!".to_string()), - Variable::String("<br ".to_string()), + Variable::from("<p ".to_string()), + Variable::from("_hello".to_string()), + Variable::from("</p ".to_string()), + Variable::from("<p ".to_string()), + Variable::from("_ world".to_string()), + Variable::from("</p ".to_string()), + Variable::from("_ !!!".to_string()), + Variable::from("<br ".to_string()), ], ), ( " <p>please unsubscribe <a href=#>here</a>.</p> ", vec![ - Variable::String("<p".to_string()), - Variable::String("_please unsubscribe".to_string()), - Variable::String("<a href=#".to_string()), - Variable::String("_ here".to_string()), - Variable::String("</a".to_string()), - Variable::String("_.".to_string()), - Variable::String("</p".to_string()), + Variable::from("<p".to_string()), + Variable::from("_please unsubscribe".to_string()), + Variable::from("<a href=#".to_string()), + Variable::from("_ here".to_string()), + Variable::from("</a".to_string()), + Variable::from("_.".to_string()), + Variable::from("</p".to_string()), ], ), ] { @@ -529,11 +572,11 @@ fn html_tokens() { "< anchor href = \"x\">text</a>", ), vec![ - Variable::String("a".to_string()), - Variable::String("b".to_string()), - Variable::String("c".to_string()), - Variable::String("d".to_string()), - Variable::String("e".to_string()), + Variable::from("a".to_string()), + Variable::from("b".to_string()), + Variable::from("c".to_string()), + Variable::from("d".to_string()), + Variable::from("e".to_string()), ], ), ( @@ -547,11 +590,11 @@ fn html_tokens() { "<anchor href=x>text</a>", ), vec![ - Variable::String("a".to_string()), - Variable::String("b".to_string()), - Variable::String("c".to_string()), - Variable::String("d".to_string()), - Variable::String("e".to_string()), + Variable::from("a".to_string()), + Variable::from("b".to_string()), + Variable::from("c".to_string()), + Variable::from("d".to_string()), + Variable::from("e".to_string()), ], ), ( @@ -565,10 +608,10 @@ fn html_tokens() { "<a href=foobar> a href = \"unknown\" </a>", ), vec![ - Variable::String("hello world".to_string()), - Variable::String("test".to_string()), - Variable::String("fudge".to_string()), - Variable::String("foobar".to_string()), + Variable::from("hello world".to_string()), + Variable::from("test".to_string()), + Variable::from("fudge".to_string()), + Variable::from("foobar".to_string()), ], ), ] { diff --git a/tests/src/smtp/mod.rs b/tests/src/smtp/mod.rs index e10704d2..a73d5ddc 100644 --- a/tests/src/smtp/mod.rs +++ b/tests/src/smtp/mod.rs @@ -315,6 +315,7 @@ impl TestConfig for QueueConfig { dane: IfBlock::new(smtp::config::RequireOptional::Optional), mta_sts: IfBlock::new(smtp::config::RequireOptional::Optional), start: IfBlock::new(smtp::config::RequireOptional::Optional), + invalid_certs: IfBlock::new(false), }, dsn: Dsn { name: IfBlock::new("Mail Delivery Subsystem".to_string()), |