From db0efdaf59177e1b74b47c6f5d781db16885aa7d Mon Sep 17 00:00:00 2001 From: mdecimus Date: Thu, 8 Aug 2024 12:50:00 +0200 Subject: v0.9.1 --- CHANGELOG.md | 21 ++++ Cargo.lock | 26 ++--- crates/cli/Cargo.toml | 2 +- crates/common/Cargo.toml | 2 +- crates/common/src/config/telemetry.rs | 179 ++++++++++++++++----------------- crates/directory/Cargo.toml | 2 +- crates/imap/Cargo.toml | 2 +- crates/jmap/Cargo.toml | 2 +- crates/main/Cargo.toml | 2 +- crates/managesieve/Cargo.toml | 2 +- crates/nlp/Cargo.toml | 2 +- crates/pop3/Cargo.toml | 2 +- crates/smtp/Cargo.toml | 2 +- crates/store/Cargo.toml | 2 +- crates/trc/Cargo.toml | 2 +- crates/trc/src/metrics.rs | 181 +++++++++++++++++++++++++++++++++- crates/utils/Cargo.toml | 2 +- 17 files changed, 315 insertions(+), 118 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 16197448..11ee3384 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,27 @@ All notable changes to this project will be documented in this file. This project adheres to [Semantic Versioning](http://semver.org/). +## [0.9.1] - 2024-08-08 + +To upgrade replace the `stalwart-mail` binary and then upgrade to the latest web-admin. + +## Added +- Metrics support (closes #478) + - OpenTelemetry Push Exporter + - Prometheus Pull Exporter (closes #275) +- HTTP endpoint access controls (closes #266 #329 #542) +- Add `options` setting to PostgreSQL driver (closes #662) +- Add `isActive` property to defaults on Sieve/get JMAP method (closes #624) + +### Changed +- Perform must-match-sender checks after sender rewriting (closes #394) +- Only perform email ingest duplicate check on the target mailbox (#632) + +### Fixed +- Properly parse Forwarded and X-Forwarded for headers (fixes #669) +- Resolve DKIM macros when generating DNS records (fixes #666) +- Fixed `is_local_domain` Sieve function (fixes #622) + ## [0.9.0] - 2024-08-01 To upgrade replace the `stalwart-mail` binary and then upgrade to the latest web-admin. This version includes breaking changes to the Webhooks configuration and produces a slightly different log output, read [UPGRADING.md](UPGRADING.md) for details. diff --git a/Cargo.lock b/Cargo.lock index c43b5510..472b7964 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1017,7 +1017,7 @@ dependencies = [ [[package]] name = "common" -version = "0.9.0" +version = "0.9.1" dependencies = [ "ahash 0.8.11", "arc-swap", @@ -1624,7 +1624,7 @@ dependencies = [ [[package]] name = "directory" -version = "0.9.0" +version = "0.9.1" dependencies = [ "ahash 0.8.11", "argon2", @@ -2947,7 +2947,7 @@ checksum = "edcd27d72f2f071c64249075f42e205ff93c9a4c5f6c6da53e79ed9f9832c285" [[package]] name = "imap" -version = "0.9.0" +version = "0.9.1" dependencies = [ "ahash 0.8.11", "common", @@ -3159,7 +3159,7 @@ dependencies = [ [[package]] name = "jmap" -version = "0.9.0" +version = "0.9.1" dependencies = [ "aes", "aes-gcm", @@ -3596,7 +3596,7 @@ dependencies = [ [[package]] name = "mail-server" -version = "0.9.0" +version = "0.9.1" dependencies = [ "common", "directory", @@ -3615,7 +3615,7 @@ dependencies = [ [[package]] name = "managesieve" -version = "0.9.0" +version = "0.9.1" dependencies = [ "ahash 0.8.11", "bincode", @@ -3895,7 +3895,7 @@ dependencies = [ [[package]] name = "nlp" -version = "0.9.0" +version = "0.9.1" dependencies = [ "ahash 0.8.11", "bincode", @@ -4447,7 +4447,7 @@ dependencies = [ [[package]] name = "pop3" -version = "0.9.0" +version = "0.9.1" dependencies = [ "common", "imap", @@ -5989,7 +5989,7 @@ checksum = "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67" [[package]] name = "smtp" -version = "0.9.0" +version = "0.9.1" dependencies = [ "ahash 0.8.11", "bincode", @@ -6106,7 +6106,7 @@ checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" [[package]] name = "stalwart-cli" -version = "0.9.0" +version = "0.9.1" dependencies = [ "clap", "console", @@ -6137,7 +6137,7 @@ checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" [[package]] name = "store" -version = "0.9.0" +version = "0.9.1" dependencies = [ "ahash 0.8.11", "arc-swap", @@ -6762,7 +6762,7 @@ dependencies = [ [[package]] name = "trc" -version = "0.9.0" +version = "0.9.1" dependencies = [ "ahash 0.8.11", "base64 0.22.1", @@ -7005,7 +7005,7 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" [[package]] name = "utils" -version = "0.9.0" +version = "0.9.1" dependencies = [ "ahash 0.8.11", "base64 0.22.1", diff --git a/crates/cli/Cargo.toml b/crates/cli/Cargo.toml index 3644387d..9cc8418b 100644 --- a/crates/cli/Cargo.toml +++ b/crates/cli/Cargo.toml @@ -5,7 +5,7 @@ authors = ["Stalwart Labs Ltd. "] license = "AGPL-3.0-only OR LicenseRef-SEL" repository = "https://github.com/stalwartlabs/cli" homepage = "https://github.com/stalwartlabs/cli" -version = "0.9.0" +version = "0.9.1" edition = "2021" readme = "README.md" resolver = "2" diff --git a/crates/common/Cargo.toml b/crates/common/Cargo.toml index 331c1545..9595a5ff 100644 --- a/crates/common/Cargo.toml +++ b/crates/common/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "common" -version = "0.9.0" +version = "0.9.1" edition = "2021" resolver = "2" diff --git a/crates/common/src/config/telemetry.rs b/crates/common/src/config/telemetry.rs index 5d264f84..59d5e91e 100644 --- a/crates/common/src/config/telemetry.rs +++ b/crates/common/src/config/telemetry.rs @@ -128,11 +128,13 @@ impl Telemetry { // Parse metrics if config - .property_or_default("metrics.prometheus.enable", "true") - .unwrap_or(true) - || config - .property_or_default("metrics.open-telemetry.enable", "false") - .unwrap_or(false) + .property_or_default("metrics.prometheus.enable", "false") + .unwrap_or(false) + || ["http", "grpc"].contains( + &config + .value("metrics.open-telemetry.transport") + .unwrap_or("disabled"), + ) { apply_events( config @@ -141,7 +143,9 @@ impl Telemetry { .map(|(_, e)| e), false, |event_type| { - telemetry.metrics.set(event_type); + if event_type.is_metric() { + telemetry.metrics.set(event_type); + } }, ); } @@ -577,10 +581,21 @@ impl Metrics { }); } - if config - .property_or_default("metrics.open-telemetry.enable", "false") - .unwrap_or(false) + let otel_enabled = match config + .value("metrics.open-telemetry.transport") + .unwrap_or("disable") { + "grpc" => true.into(), + "http" | "https" => false.into(), + "disable" | "disabled" => None, + transport => { + let err = format!("Invalid transport: {transport}"); + config.new_parse_error("metrics.open-telemetry.transport", err); + None + } + }; + + if let Some(is_grpc) = otel_enabled { let timeout = config .property::("metrics.open-telemetry.timeout") .unwrap_or(Duration::from_secs( @@ -597,92 +612,78 @@ impl Metrics { .with_version(env!("CARGO_PKG_VERSION")) .build(); - match config - .value_require("metrics.open-telemetry.transport") - .unwrap_or_default() - { - "grpc" => { - let mut exporter = opentelemetry_otlp::new_exporter() - .tonic() - .with_protocol(opentelemetry_otlp::Protocol::Grpc) - .with_timeout(timeout); - if let Some(endpoint) = config.value("metrics.open-telemetry.endpoint") { - exporter = exporter.with_endpoint(endpoint); - } + if is_grpc { + let mut exporter = opentelemetry_otlp::new_exporter() + .tonic() + .with_protocol(opentelemetry_otlp::Protocol::Grpc) + .with_timeout(timeout); + if let Some(endpoint) = config.value("metrics.open-telemetry.endpoint") { + exporter = exporter.with_endpoint(endpoint); + } - match exporter.build_metrics_exporter( - Box::new(DefaultAggregationSelector::new()), - Box::new(DefaultTemporalitySelector::new()), - ) { - Ok(exporter) => { - metrics.otel = Some(Arc::new(OtelMetrics { - exporter: Box::new(exporter), - interval, - resource, - instrumentation, - })); - } - Err(err) => { - config.new_build_error( - "metrics.open-telemetry", - format!("Failed to build OpenTelemetry metrics exporter: {err}"), - ); - } + match exporter.build_metrics_exporter( + Box::new(DefaultAggregationSelector::new()), + Box::new(DefaultTemporalitySelector::new()), + ) { + Ok(exporter) => { + metrics.otel = Some(Arc::new(OtelMetrics { + exporter: Box::new(exporter), + interval, + resource, + instrumentation, + })); + } + Err(err) => { + config.new_build_error( + "metrics.open-telemetry", + format!("Failed to build OpenTelemetry metrics exporter: {err}"), + ); } } - "http" => { - if let Some(endpoint) = config - .value_require("metrics.open-telemetry.endpoint") - .map(|s| s.to_string()) - { - let mut headers = HashMap::new(); - let mut err = None; - for (_, value) in config.values("metrics.open-telemetry.headers") { - if let Some((key, value)) = value.split_once(':') { - headers.insert(key.trim().to_string(), value.trim().to_string()); - } else { - err = format!("Invalid open-telemetry header {value:?}").into(); - break; - } - } - if let Some(err) = err { - config.new_parse_error("metrics.open-telemetry.headers", err); - } + } else if let Some(endpoint) = config + .value_require("metrics.open-telemetry.endpoint") + .map(|s| s.to_string()) + { + let mut headers = HashMap::new(); + let mut err = None; + for (_, value) in config.values("metrics.open-telemetry.headers") { + if let Some((key, value)) = value.split_once(':') { + headers.insert(key.trim().to_string(), value.trim().to_string()); + } else { + err = format!("Invalid open-telemetry header {value:?}").into(); + break; + } + } + if let Some(err) = err { + config.new_parse_error("metrics.open-telemetry.headers", err); + } - let mut exporter = opentelemetry_otlp::new_exporter() - .http() - .with_endpoint(&endpoint) - .with_timeout(timeout); - if !headers.is_empty() { - exporter = exporter.with_headers(headers); - } + let mut exporter = opentelemetry_otlp::new_exporter() + .http() + .with_endpoint(&endpoint) + .with_timeout(timeout); + if !headers.is_empty() { + exporter = exporter.with_headers(headers); + } - match exporter.build_metrics_exporter( - Box::new(DefaultAggregationSelector::new()), - Box::new(DefaultTemporalitySelector::new()), - ) { - Ok(exporter) => { - metrics.otel = Some(Arc::new(OtelMetrics { - exporter: Box::new(exporter), - interval, - resource, - instrumentation, - })); - } - Err(err) => { - config.new_build_error( - "metrics.open-telemetry", - format!( - "Failed to build OpenTelemetry metrics exporter: {err}" - ), - ); - } - } + match exporter.build_metrics_exporter( + Box::new(DefaultAggregationSelector::new()), + Box::new(DefaultTemporalitySelector::new()), + ) { + Ok(exporter) => { + metrics.otel = Some(Arc::new(OtelMetrics { + exporter: Box::new(exporter), + interval, + resource, + instrumentation, + })); + } + Err(err) => { + config.new_build_error( + "metrics.open-telemetry", + format!("Failed to build OpenTelemetry metrics exporter: {err}"), + ); } - } - transport => { - let err = format!("Invalid transport: {transport}"); - config.new_parse_error("metrics.open-telemetry.transport", err); } } } diff --git a/crates/directory/Cargo.toml b/crates/directory/Cargo.toml index 0cb4ada0..2f6c814d 100644 --- a/crates/directory/Cargo.toml +++ b/crates/directory/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "directory" -version = "0.9.0" +version = "0.9.1" edition = "2021" resolver = "2" diff --git a/crates/imap/Cargo.toml b/crates/imap/Cargo.toml index 61daf0ca..bb2836d7 100644 --- a/crates/imap/Cargo.toml +++ b/crates/imap/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "imap" -version = "0.9.0" +version = "0.9.1" edition = "2021" resolver = "2" diff --git a/crates/jmap/Cargo.toml b/crates/jmap/Cargo.toml index 418a55dd..26c1b256 100644 --- a/crates/jmap/Cargo.toml +++ b/crates/jmap/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "jmap" -version = "0.9.0" +version = "0.9.1" edition = "2021" resolver = "2" diff --git a/crates/main/Cargo.toml b/crates/main/Cargo.toml index 2288c837..d969e050 100644 --- a/crates/main/Cargo.toml +++ b/crates/main/Cargo.toml @@ -7,7 +7,7 @@ homepage = "https://stalw.art" keywords = ["imap", "jmap", "smtp", "email", "mail", "server"] categories = ["email"] license = "AGPL-3.0-only OR LicenseRef-SEL" -version = "0.9.0" +version = "0.9.1" edition = "2021" resolver = "2" diff --git a/crates/managesieve/Cargo.toml b/crates/managesieve/Cargo.toml index 14b6e8b1..c9a259d1 100644 --- a/crates/managesieve/Cargo.toml +++ b/crates/managesieve/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "managesieve" -version = "0.9.0" +version = "0.9.1" edition = "2021" resolver = "2" diff --git a/crates/nlp/Cargo.toml b/crates/nlp/Cargo.toml index 5d640cdf..897f9a1b 100644 --- a/crates/nlp/Cargo.toml +++ b/crates/nlp/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "nlp" -version = "0.9.0" +version = "0.9.1" edition = "2021" resolver = "2" diff --git a/crates/pop3/Cargo.toml b/crates/pop3/Cargo.toml index f09d0748..a99d11b9 100644 --- a/crates/pop3/Cargo.toml +++ b/crates/pop3/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pop3" -version = "0.9.0" +version = "0.9.1" edition = "2021" resolver = "2" diff --git a/crates/smtp/Cargo.toml b/crates/smtp/Cargo.toml index 3dba684a..19559ef8 100644 --- a/crates/smtp/Cargo.toml +++ b/crates/smtp/Cargo.toml @@ -7,7 +7,7 @@ homepage = "https://stalw.art/smtp" keywords = ["smtp", "email", "mail", "server"] categories = ["email"] license = "AGPL-3.0-only OR LicenseRef-SEL" -version = "0.9.0" +version = "0.9.1" edition = "2021" resolver = "2" diff --git a/crates/store/Cargo.toml b/crates/store/Cargo.toml index e41d6e95..46d5e1ea 100644 --- a/crates/store/Cargo.toml +++ b/crates/store/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "store" -version = "0.9.0" +version = "0.9.1" edition = "2021" resolver = "2" diff --git a/crates/trc/Cargo.toml b/crates/trc/Cargo.toml index 4c003c0a..71a29518 100644 --- a/crates/trc/Cargo.toml +++ b/crates/trc/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "trc" -version = "0.9.0" +version = "0.9.1" edition = "2021" resolver = "2" diff --git a/crates/trc/src/metrics.rs b/crates/trc/src/metrics.rs index fd477784..f3ccd28b 100644 --- a/crates/trc/src/metrics.rs +++ b/crates/trc/src/metrics.rs @@ -10,9 +10,7 @@ use crate::{ atomic::{AtomicCounter, AtomicGauge, AtomicHistogram, AtomicU32Array}, collector::{Collector, GlobalInterests, EVENT_TYPES}, subscriber::Interests, - DeliveryEvent, EventType, FtsIndexEvent, HttpEvent, ImapEvent, Key, ManageSieveEvent, - MessageIngestEvent, NetworkEvent, Pop3Event, Protocol, QueueEvent, SmtpEvent, StoreEvent, - Value, TOTAL_EVENT_COUNT, + *, }; pub(crate) static METRIC_INTERESTS: GlobalInterests = GlobalInterests::new(); @@ -469,3 +467,180 @@ impl Protocol { } } } + +impl EventType { + pub fn is_metric(&self) -> bool { + match self { + EventType::Server(ServerEvent::ThreadError) => true, + EventType::Purge( + PurgeEvent::Started + | PurgeEvent::Error + | PurgeEvent::AutoExpunge + | PurgeEvent::TombstoneCleanup, + ) => true, + EventType::Eval( + EvalEvent::Error | EvalEvent::StoreNotFound | EvalEvent::DirectoryNotFound, + ) => true, + EventType::Acme( + AcmeEvent::TlsAlpnError + | AcmeEvent::OrderStart + | AcmeEvent::OrderCompleted + | AcmeEvent::AuthError + | AcmeEvent::AuthCompleted + | AcmeEvent::AuthTooManyAttempts + | AcmeEvent::DnsRecordCreated + | AcmeEvent::DnsRecordCreationFailed + | AcmeEvent::DnsRecordDeletionFailed + | AcmeEvent::DnsRecordPropagationTimeout + | AcmeEvent::ClientMissingSni + | AcmeEvent::TokenNotFound + | AcmeEvent::DnsRecordLookupFailed + | AcmeEvent::OrderInvalid + | AcmeEvent::Error, + ) => true, + EventType::Store( + StoreEvent::AssertValueFailed + | StoreEvent::FoundationdbError + | StoreEvent::MysqlError + | StoreEvent::PostgresqlError + | StoreEvent::RocksdbError + | StoreEvent::SqliteError + | StoreEvent::LdapError + | StoreEvent::ElasticsearchError + | StoreEvent::RedisError + | StoreEvent::S3Error + | StoreEvent::FilesystemError + | StoreEvent::PoolError + | StoreEvent::DataCorruption + | StoreEvent::DecompressError + | StoreEvent::DeserializeError + | StoreEvent::NotFound + | StoreEvent::NotConfigured + | StoreEvent::NotSupported + | StoreEvent::UnexpectedError + | StoreEvent::CryptoError + | StoreEvent::BlobMissingMarker + | StoreEvent::DataWrite + | StoreEvent::DataIterate + | StoreEvent::BlobRead + | StoreEvent::BlobWrite + | StoreEvent::BlobDelete, + ) => true, + EventType::MessageIngest(_) => true, + EventType::Jmap( + JmapEvent::MethodCall + | JmapEvent::WebsocketStart + | JmapEvent::WebsocketError + | JmapEvent::UnsupportedFilter + | JmapEvent::UnsupportedSort + | JmapEvent::Forbidden + | JmapEvent::NotJson + | JmapEvent::NotRequest + | JmapEvent::InvalidArguments + | JmapEvent::RequestTooLarge + | JmapEvent::UnknownMethod, + ) => true, + EventType::Imap(_) => true, + EventType::ManageSieve(_) => true, + EventType::Pop3(_) => true, + EventType::Smtp(_) => true, + EventType::Http( + HttpEvent::Error + | HttpEvent::RequestBody + | HttpEvent::ResponseBody + | HttpEvent::XForwardedMissing, + ) => true, + EventType::Network(_) => true, + EventType::Limit(_) => true, + EventType::Manage(_) => false, + EventType::Auth( + AuthEvent::Success + | AuthEvent::Failed + | AuthEvent::TooManyAttempts + | AuthEvent::Banned + | AuthEvent::Error, + ) => true, + EventType::Config(_) => false, + EventType::Resource( + ResourceEvent::NotFound | ResourceEvent::BadParameters | ResourceEvent::Error, + ) => true, + EventType::Arc(_) => true, + EventType::Dkim(_) => true, + EventType::Dmarc(_) => true, + EventType::Iprev(_) => true, + EventType::Dane(_) => true, + EventType::Spf(_) => true, + EventType::MailAuth(_) => true, + EventType::Tls(_) => true, + EventType::Sieve(_) => true, + EventType::Spam( + SpamEvent::PyzorError + | SpamEvent::ListUpdated + | SpamEvent::Train + | SpamEvent::TrainError + | SpamEvent::Classify + | SpamEvent::ClassifyError + | SpamEvent::NotEnoughTrainingData, + ) => true, + EventType::PushSubscription(_) => true, + EventType::Cluster( + ClusterEvent::PeerOffline + | ClusterEvent::PeerSuspected + | ClusterEvent::PeerSuspectedIsAlive + | ClusterEvent::EmptyPacket + | ClusterEvent::InvalidPacket + | ClusterEvent::DecryptionError + | ClusterEvent::Error, + ) => true, + EventType::Housekeeper(_) => false, + EventType::FtsIndex( + FtsIndexEvent::Index + | FtsIndexEvent::BlobNotFound + | FtsIndexEvent::MetadataNotFound, + ) => true, + EventType::Milter(_) => true, + EventType::MtaHook(_) => true, + EventType::Delivery(_) => true, + EventType::Queue( + QueueEvent::QueueMessage + | QueueEvent::QueueMessageSubmission + | QueueEvent::QueueReport + | QueueEvent::QueueDsn + | QueueEvent::QueueAutogenerated + | QueueEvent::Rescheduled + | QueueEvent::BlobNotFound + | QueueEvent::RateLimitExceeded + | QueueEvent::ConcurrencyLimitExceeded + | QueueEvent::QuotaExceeded, + ) => true, + EventType::TlsRpt(_) => true, + EventType::MtaSts(_) => true, + EventType::IncomingReport(_) => true, + EventType::OutgoingReport( + OutgoingReportEvent::SpfReport + | OutgoingReportEvent::SpfRateLimited + | OutgoingReportEvent::DkimReport + | OutgoingReportEvent::DkimRateLimited + | OutgoingReportEvent::DmarcReport + | OutgoingReportEvent::DmarcRateLimited + | OutgoingReportEvent::DmarcAggregateReport + | OutgoingReportEvent::TlsAggregate + | OutgoingReportEvent::HttpSubmission + | OutgoingReportEvent::UnauthorizedReportingAddress + | OutgoingReportEvent::ReportingAddressValidationError + | OutgoingReportEvent::NotFound + | OutgoingReportEvent::SubmissionError + | OutgoingReportEvent::NoRecipientsFound, + ) => true, + EventType::Telemetry( + TelemetryEvent::LogError + | TelemetryEvent::WebhookError + | TelemetryEvent::OtelExpoterError + | TelemetryEvent::OtelMetricsExporterError + | TelemetryEvent::PrometheusExporterError + | TelemetryEvent::JournalError, + ) => true, + _ => false, + } + } +} diff --git a/crates/utils/Cargo.toml b/crates/utils/Cargo.toml index 3fbf878d..72055013 100644 --- a/crates/utils/Cargo.toml +++ b/crates/utils/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "utils" -version = "0.9.0" +version = "0.9.1" edition = "2021" resolver = "2" -- cgit v1.2.3-70-g09d2