diff --git a/audit/build.gradle b/audit/build.gradle index 43f3bbf626..9d0ce5adf3 100644 --- a/audit/build.gradle +++ b/audit/build.gradle @@ -23,6 +23,7 @@ plugins { id "java-library" id "maven-publish" id "signing" + id "com.github.maiflai.scalatest" version "0.33" } @@ -58,6 +59,7 @@ dependencies { crossBuildV213Implementation group: 'org.scala-lang', name: 'scala-library', version: '2.13.13' crossBuildV3Implementation group: 'org.scala-lang', name: 'scala3-library_3', version: '3.3.3' compileOnly group: 'org.scala-lang', name: 'scala3-library_3', version: '3.3.3' + testImplementation group: 'org.scalatest', name: 'scalatest_3', version: '3.2.16' } test { diff --git a/audit/src/main/scala/tech/beshu/ror/audit/AuditRequestContext.scala b/audit/src/main/scala/tech/beshu/ror/audit/AuditRequestContext.scala index 1b6c8b38c8..3343ba5f8b 100644 --- a/audit/src/main/scala/tech/beshu/ror/audit/AuditRequestContext.scala +++ b/audit/src/main/scala/tech/beshu/ror/audit/AuditRequestContext.scala @@ -21,7 +21,6 @@ import java.time.Instant import org.json.JSONObject trait AuditRequestContext { - def timestamp: Instant def id: String def correlationId: String @@ -45,4 +44,5 @@ trait AuditRequestContext { def attemptedUserName: Option[String] def rawAuthHeader: Option[String] def generalAuditEvents: JSONObject + def auditEnvironmentContext: AuditEnvironmentContext } \ No newline at end of file diff --git a/audit/src/main/scala/tech/beshu/ror/audit/EnvironmentAwareAuditLogSerializer.scala b/audit/src/main/scala/tech/beshu/ror/audit/EnvironmentAwareAuditLogSerializer.scala index c27f9f3671..81e9887249 100644 --- a/audit/src/main/scala/tech/beshu/ror/audit/EnvironmentAwareAuditLogSerializer.scala +++ b/audit/src/main/scala/tech/beshu/ror/audit/EnvironmentAwareAuditLogSerializer.scala @@ -18,6 +18,8 @@ package tech.beshu.ror.audit import org.json.JSONObject +// The `AuditResponseContext` now contains the `AuditEnvironmentContext` and there is no need to use this trait. +// This trait is preserved and supported for compatibility reasons, but we should not include it in our docs and use `AuditLogSerializer` instead trait EnvironmentAwareAuditLogSerializer { def onResponse(responseContext: AuditResponseContext, environmentContext: AuditEnvironmentContext): Option[JSONObject] diff --git a/audit/src/main/scala/tech/beshu/ror/audit/adapters/DeprecatedAuditLogSerializerAdapter.scala b/audit/src/main/scala/tech/beshu/ror/audit/adapters/DeprecatedAuditLogSerializerAdapter.scala index b2802b7ee8..102e44cc33 100644 --- a/audit/src/main/scala/tech/beshu/ror/audit/adapters/DeprecatedAuditLogSerializerAdapter.scala +++ b/audit/src/main/scala/tech/beshu/ror/audit/adapters/DeprecatedAuditLogSerializerAdapter.scala @@ -18,7 +18,7 @@ package tech.beshu.ror.audit.adapters import org.json.JSONObject import tech.beshu.ror.audit.AuditResponseContext.{Allowed, Verbosity} -import tech.beshu.ror.audit.instances.SerializeUser +import tech.beshu.ror.audit.utils.SerializeUser import tech.beshu.ror.audit.{AuditLogSerializer, AuditRequestContext, AuditResponseContext} import tech.beshu.ror.commons.ResponseContext.FinalState import tech.beshu.ror.commons.shims.request.RequestContextShim diff --git a/audit/src/main/scala/tech/beshu/ror/audit/adapters/EnvironmentAwareAuditLogSerializerAdapter.scala b/audit/src/main/scala/tech/beshu/ror/audit/adapters/EnvironmentAwareAuditLogSerializerAdapter.scala index 3038b83c90..7c0a876e1b 100644 --- a/audit/src/main/scala/tech/beshu/ror/audit/adapters/EnvironmentAwareAuditLogSerializerAdapter.scala +++ b/audit/src/main/scala/tech/beshu/ror/audit/adapters/EnvironmentAwareAuditLogSerializerAdapter.scala @@ -19,11 +19,10 @@ package tech.beshu.ror.audit.adapters import org.json.JSONObject import tech.beshu.ror.audit._ -class EnvironmentAwareAuditLogSerializerAdapter(underlying: EnvironmentAwareAuditLogSerializer, - environmentContext: AuditEnvironmentContext) extends AuditLogSerializer { +class EnvironmentAwareAuditLogSerializerAdapter(underlying: EnvironmentAwareAuditLogSerializer) extends AuditLogSerializer { override def onResponse(responseContext: AuditResponseContext): Option[JSONObject] = { - underlying.onResponse(responseContext, environmentContext) + underlying.onResponse(responseContext, responseContext.requestContext.auditEnvironmentContext) } } diff --git a/audit/src/main/scala/tech/beshu/ror/audit/instances/BlockVerbosityAwareAuditLogSerializer.scala b/audit/src/main/scala/tech/beshu/ror/audit/instances/BlockVerbosityAwareAuditLogSerializer.scala new file mode 100644 index 0000000000..cee6e5a649 --- /dev/null +++ b/audit/src/main/scala/tech/beshu/ror/audit/instances/BlockVerbosityAwareAuditLogSerializer.scala @@ -0,0 +1,91 @@ +/* + * This file is part of ReadonlyREST. + * + * ReadonlyREST is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * ReadonlyREST is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with ReadonlyREST. If not, see http://www.gnu.org/licenses/ + */ +package tech.beshu.ror.audit.instances + +import org.json.JSONObject +import tech.beshu.ror.audit.AuditResponseContext.Verbosity +import tech.beshu.ror.audit.utils.AuditSerializationHelper +import tech.beshu.ror.audit.utils.AuditSerializationHelper.AllowedEventMode.Include +import tech.beshu.ror.audit.utils.AuditSerializationHelper.AuditFieldGroup.{CommonFields, EsEnvironmentFields} +import tech.beshu.ror.audit.{AuditLogSerializer, AuditResponseContext} + +/** + * Serializer for audit events that is aware of **rule-defined verbosity**. + * + * - Serializes all non-`Allowed` events. + * - Serializes `Allowed` events only if the corresponding rule + * specifies that they should be logged at `Verbosity.Info`. + * + * Recommended for standard audit logging where request body capture + * is not required, but cluster and node context are desired. + * + * - Fields included: + * - `match` — whether the request matched a rule (boolean) + * - `block` — reason for blocking, if blocked (string) + * - `id` — audit event identifier (string) + * - `final_state` — final processing state (string) + * - `@timestamp` — event timestamp (ISO-8601 string) + * - `correlation_id` — correlation identifier for tracing (string) + * - `processingMillis` — request processing duration in milliseconds (number) + * - `error_type` — type of error, if any (string) + * - `error_message` — error message, if any (string) + * - `content_len` — request body size in bytes (number) + * - `content_len_kb` — request body size in kilobytes (number) + * - `type` — request type (string) + * - `origin` — client (remote) address (string) + * - `destination` — server (local) address (string) + * - `xff` — `X-Forwarded-For` HTTP header value (string) + * - `task_id` — Elasticsearch task ID (number) + * - `req_method` — HTTP request method (string) + * - `headers` — HTTP header names (array of strings) + * - `path` — HTTP request path (string) + * - `user` — authenticated user (string) + * - `impersonated_by` — impersonating user, if applicable (string) + * - `action` — Elasticsearch action name (string) + * - `indices` — indices involved in the request (array of strings) + * - `acl_history` — access control evaluation history (string) + * - `es_node_name` — Elasticsearch node name (string) + * - `es_cluster_name` — Elasticsearch cluster name (string) + */ +class BlockVerbosityAwareAuditLogSerializer extends DefaultAuditLogSerializer + +@deprecated("Use tech.beshu.ror.audit.instances.BlockVerbosityAwareAuditLogSerializer instead", "1.67.0") +class DefaultAuditLogSerializer extends DefaultAuditLogSerializerV2 + +@deprecated("Use tech.beshu.ror.audit.instances.BlockVerbosityAwareAuditLogSerializer instead", "1.67.0") +class DefaultAuditLogSerializerV2 extends AuditLogSerializer { + + override def onResponse(responseContext: AuditResponseContext): Option[JSONObject] = + AuditSerializationHelper.serialize( + responseContext = responseContext, + fieldGroups = Set(CommonFields, EsEnvironmentFields), + allowedEventMode = Include(Set(Verbosity.Info)) + ) + +} + +@deprecated("Use tech.beshu.ror.audit.instances.BlockVerbosityAwareAuditLogSerializer instead", "1.67.0") +class DefaultAuditLogSerializerV1 extends AuditLogSerializer { + + override def onResponse(responseContext: AuditResponseContext): Option[JSONObject] = + AuditSerializationHelper.serialize( + responseContext = responseContext, + fieldGroups = Set(CommonFields), + allowedEventMode = Include(Set(Verbosity.Info)) + ) + +} diff --git a/audit/src/main/scala/tech/beshu/ror/audit/instances/DefaultAuditLogSerializer.scala b/audit/src/main/scala/tech/beshu/ror/audit/instances/DefaultAuditLogSerializer.scala deleted file mode 100644 index 7a1b070f3d..0000000000 --- a/audit/src/main/scala/tech/beshu/ror/audit/instances/DefaultAuditLogSerializer.scala +++ /dev/null @@ -1,21 +0,0 @@ -/* - * This file is part of ReadonlyREST. - * - * ReadonlyREST is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * ReadonlyREST is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with ReadonlyREST. If not, see http://www.gnu.org/licenses/ - */ -package tech.beshu.ror.audit.instances - -import tech.beshu.ror.audit.AuditEnvironmentContext - -class DefaultAuditLogSerializer(environmentContext: AuditEnvironmentContext) extends DefaultAuditLogSerializerV2(environmentContext) diff --git a/audit/src/main/scala/tech/beshu/ror/audit/instances/DefaultAuditLogSerializerV1.scala b/audit/src/main/scala/tech/beshu/ror/audit/instances/DefaultAuditLogSerializerV1.scala deleted file mode 100644 index fabe3440ea..0000000000 --- a/audit/src/main/scala/tech/beshu/ror/audit/instances/DefaultAuditLogSerializerV1.scala +++ /dev/null @@ -1,98 +0,0 @@ -/* - * This file is part of ReadonlyREST. - * - * ReadonlyREST is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * ReadonlyREST is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with ReadonlyREST. If not, see http://www.gnu.org/licenses/ - */ -package tech.beshu.ror.audit.instances - -import org.json.JSONObject -import tech.beshu.ror.audit.AuditResponseContext._ -import tech.beshu.ror.audit.{AuditLogSerializer, AuditRequestContext, AuditResponseContext} - -import java.time.ZoneId -import java.time.format.DateTimeFormatter -import scala.collection.JavaConverters._ -import scala.concurrent.duration.FiniteDuration - -class DefaultAuditLogSerializerV1 extends AuditLogSerializer { - - private val timestampFormatter = DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss'Z'").withZone(ZoneId.of("GMT")) - - override def onResponse(responseContext: AuditResponseContext): Option[JSONObject] = responseContext match { - case Allowed(requestContext, verbosity, reason) => - verbosity match { - case Verbosity.Info => - Some(createEntry(matched = true, "ALLOWED", reason, responseContext.duration, requestContext, None)) - case Verbosity.Error => - None - } - case ForbiddenBy(requestContext, _, reason) => - Some(createEntry(matched = true, "FORBIDDEN", reason, responseContext.duration, requestContext, None)) - case Forbidden(requestContext) => - Some(createEntry(matched = false, "FORBIDDEN", "default", responseContext.duration, requestContext, None)) - case RequestedIndexNotExist(requestContext) => - Some(createEntry(matched = false, "INDEX NOT EXIST", "Requested index doesn't exist", responseContext.duration, requestContext, None)) - case Errored(requestContext, cause) => - Some(createEntry(matched = false, "ERRORED", "error", responseContext.duration, requestContext, Some(cause))) - } - - private def createEntry(matched: Boolean, - finalState: String, - reason: String, - duration: FiniteDuration, - requestContext: AuditRequestContext, - error: Option[Throwable]) = { - new JSONObject() - .put("match", matched) - .put("block", reason) - .put("id", requestContext.id) - .put("final_state", finalState) - .put("@timestamp", timestampFormatter.format(requestContext.timestamp)) - .put("correlation_id", requestContext.correlationId) - .put("processingMillis", duration.toMillis) - .put("error_type", error.map(_.getClass.getSimpleName).orNull) - .put("error_message", error.map(_.getMessage).orNull) - .put("content_len", requestContext.contentLength) - .put("content_len_kb", requestContext.contentLength / 1024) - .put("type", requestContext.`type`) - .put("origin", requestContext.remoteAddress) - .put("destination", requestContext.localAddress) - .put("xff", requestContext.requestHeaders.getValue("X-Forwarded-For").flatMap(_.headOption).orNull) - .put("task_id", requestContext.taskId) - .put("req_method", requestContext.httpMethod) - .put("headers", requestContext.requestHeaders.names.asJava) - .put("path", requestContext.uriPath) - .put("user", SerializeUser.serialize(requestContext).orNull) - .put("impersonated_by", requestContext.impersonatedByUserName.orNull) - .put("action", requestContext.action) - .put("indices", if (requestContext.involvesIndices) requestContext.indices.toList.asJava else List.empty.asJava) - .put("acl_history", requestContext.history) - .mergeWith(requestContext.generalAuditEvents) - } - - private implicit class JsonObjectOps(val mainJson: JSONObject) { - def mergeWith(secondaryJson: JSONObject): JSONObject = { - jsonKeys(secondaryJson).foldLeft(mainJson) { - case (json, name) if !json.has(name) => - json.put(name, secondaryJson.get(name)) - case (json, _) => - json - } - } - - private def jsonKeys(json: JSONObject) = { - Option(JSONObject.getNames(json)).toList.flatten - } - } -} diff --git a/audit/src/main/scala/tech/beshu/ror/audit/instances/FullAuditLogSerializer.scala b/audit/src/main/scala/tech/beshu/ror/audit/instances/FullAuditLogSerializer.scala new file mode 100644 index 0000000000..c315344c5e --- /dev/null +++ b/audit/src/main/scala/tech/beshu/ror/audit/instances/FullAuditLogSerializer.scala @@ -0,0 +1,67 @@ +/* + * This file is part of ReadonlyREST. + * + * ReadonlyREST is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * ReadonlyREST is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with ReadonlyREST. If not, see http://www.gnu.org/licenses/ + */ +package tech.beshu.ror.audit.instances + +import org.json.JSONObject +import tech.beshu.ror.audit.utils.AuditSerializationHelper +import tech.beshu.ror.audit.utils.AuditSerializationHelper.AllowedEventMode.IncludeAll +import tech.beshu.ror.audit.utils.AuditSerializationHelper.AuditFieldGroup._ +import tech.beshu.ror.audit.{AuditLogSerializer, AuditResponseContext} + +/** + * Serializer for **full audit events**. + * - Serializes all events, including every `Allowed` request, + regardless of rule verbosity. + * - Use this serializer, when you need complete coverage of all events. + * - Fields included: + * - `match` — whether the request matched a rule (boolean) + * - `block` — reason for blocking, if blocked (string) + * - `id` — audit event identifier (string) + * - `final_state` — final processing state (string) + * - `@timestamp` — event timestamp (ISO-8601 string) + * - `correlation_id` — correlation identifier for tracing (string) + * - `processingMillis` — request processing duration in milliseconds (number) + * - `error_type` — type of error, if any (string) + * - `error_message` — error message, if any (string) + * - `content_len` — request body size in bytes (number) + * - `content_len_kb` — request body size in kilobytes (number) + * - `type` — request type (string) + * - `origin` — client (remote) address (string) + * - `destination` — server (local) address (string) + * - `xff` — `X-Forwarded-For` HTTP header value (string) + * - `task_id` — Elasticsearch task ID (number) + * - `req_method` — HTTP request method (string) + * - `headers` — HTTP header names (array of strings) + * - `path` — HTTP request path (string) + * - `user` — authenticated user (string) + * - `impersonated_by` — impersonating user, if applicable (string) + * - `action` — Elasticsearch action name (string) + * - `indices` — indices involved in the request (array of strings) + * - `acl_history` — access control evaluation history (string) + * - `es_node_name` — Elasticsearch node name (string) + * - `es_cluster_name` — Elasticsearch cluster name (string) + */ +class FullAuditLogSerializer extends AuditLogSerializer { + + override def onResponse(responseContext: AuditResponseContext): Option[JSONObject] = + AuditSerializationHelper.serialize( + responseContext = responseContext, + fieldGroups = Set(CommonFields, EsEnvironmentFields), + allowedEventMode = IncludeAll + ) + +} diff --git a/audit/src/main/scala/tech/beshu/ror/audit/instances/FullAuditLogWithQuerySerializer.scala b/audit/src/main/scala/tech/beshu/ror/audit/instances/FullAuditLogWithQuerySerializer.scala new file mode 100644 index 0000000000..9b59ee9610 --- /dev/null +++ b/audit/src/main/scala/tech/beshu/ror/audit/instances/FullAuditLogWithQuerySerializer.scala @@ -0,0 +1,68 @@ +/* + * This file is part of ReadonlyREST. + * + * ReadonlyREST is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * ReadonlyREST is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with ReadonlyREST. If not, see http://www.gnu.org/licenses/ + */ +package tech.beshu.ror.audit.instances + +import org.json.JSONObject +import tech.beshu.ror.audit.utils.AuditSerializationHelper +import tech.beshu.ror.audit.utils.AuditSerializationHelper.AllowedEventMode.IncludeAll +import tech.beshu.ror.audit.utils.AuditSerializationHelper.AuditFieldGroup._ +import tech.beshu.ror.audit.{AuditLogSerializer, AuditResponseContext} + +/** + * Serializer for **full audit events including request content**. + * - Serializes all events, including every `Allowed` request, + * regardless of rule verbosity. + * - Use this serializer, when request body capture is required. + * - Fields included: + * - `match` — whether the request matched a rule (boolean) + * - `block` — reason for blocking, if blocked (string) + * - `id` — audit event identifier (string) + * - `final_state` — final processing state (string) + * - `@timestamp` — event timestamp (ISO-8601 string) + * - `correlation_id` — correlation identifier for tracing (string) + * - `processingMillis` — request processing duration in milliseconds (number) + * - `error_type` — type of error, if any (string) + * - `error_message` — error message, if any (string) + * - `content_len` — request body size in bytes (number) + * - `content_len_kb` — request body size in kilobytes (number) + * - `type` — request type (string) + * - `origin` — client (remote) address (string) + * - `destination` — server (local) address (string) + * - `xff` — `X-Forwarded-For` HTTP header value (string) + * - `task_id` — Elasticsearch task ID (number) + * - `req_method` — HTTP request method (string) + * - `headers` — HTTP header names (array of strings) + * - `path` — HTTP request path (string) + * - `user` — authenticated user (string) + * - `impersonated_by` — impersonating user, if applicable (string) + * - `action` — Elasticsearch action name (string) + * - `indices` — indices involved in the request (array of strings) + * - `acl_history` — access control evaluation history (string) + * - `es_node_name` — Elasticsearch node name (string) + * - `es_cluster_name` — Elasticsearch cluster name (string) + * - `content` — full request body (string) + */ +class FullAuditLogWithQuerySerializer extends AuditLogSerializer { + + override def onResponse(responseContext: AuditResponseContext): Option[JSONObject] = + AuditSerializationHelper.serialize( + responseContext = responseContext, + fieldGroups = Set(CommonFields, EsEnvironmentFields, FullRequestContentFields), + allowedEventMode = IncludeAll + ) + +} diff --git a/audit/src/main/scala/tech/beshu/ror/audit/instances/QueryAuditLogSerializer.scala b/audit/src/main/scala/tech/beshu/ror/audit/instances/QueryAuditLogSerializer.scala index 86dc1e4dea..8c23394fa8 100644 --- a/audit/src/main/scala/tech/beshu/ror/audit/instances/QueryAuditLogSerializer.scala +++ b/audit/src/main/scala/tech/beshu/ror/audit/instances/QueryAuditLogSerializer.scala @@ -16,6 +16,140 @@ */ package tech.beshu.ror.audit.instances -import tech.beshu.ror.audit.AuditEnvironmentContext +import org.json.JSONObject +import tech.beshu.ror.audit.AuditResponseContext.Verbosity +import tech.beshu.ror.audit.utils.AuditSerializationHelper +import tech.beshu.ror.audit.utils.AuditSerializationHelper.AllowedEventMode.Include +import tech.beshu.ror.audit.utils.AuditSerializationHelper.AuditFieldGroup.{CommonFields, EsEnvironmentFields, FullRequestContentFields} +import tech.beshu.ror.audit.{AuditLogSerializer, AuditResponseContext} -class QueryAuditLogSerializer(environmentContext: AuditEnvironmentContext) extends QueryAuditLogSerializerV2(environmentContext) +/** + * Public alias for [[QueryAuditLogSerializerV2]]. + * - Captures full request content along with common and ES environment fields. + * - Respects rule-defined verbosity for `Allowed` events: + * only serializes them if the corresponding rule allows logging at `Verbosity.Info`. + * - Prefer this class name in configurations and client code for full-content auditing. + * - Fields included: + * - `match` — whether the request matched a rule (boolean) + * - `block` — reason for blocking, if blocked (string) + * - `id` — audit event identifier (string) + * - `final_state` — final processing state (string) + * - `@timestamp` — event timestamp (ISO-8601 string) + * - `correlation_id` — correlation identifier for tracing (string) + * - `processingMillis` — request processing duration in milliseconds (number) + * - `error_type` — type of error, if any (string) + * - `error_message` — error message, if any (string) + * - `content_len` — request body size in bytes (number) + * - `content_len_kb` — request body size in kilobytes (number) + * - `type` — request type (string) + * - `origin` — client (remote) address (string) + * - `destination` — server (local) address (string) + * - `xff` — `X-Forwarded-For` HTTP header value (string) + * - `task_id` — Elasticsearch task ID (number) + * - `req_method` — HTTP request method (string) + * - `headers` — HTTP header names (array of strings) + * - `path` — HTTP request path (string) + * - `user` — authenticated user (string) + * - `impersonated_by` — impersonating user, if applicable (string) + * - `action` — Elasticsearch action name (string) + * - `indices` — indices involved in the request (array of strings) + * - `acl_history` — access control evaluation history (string) + * - `es_node_name` — Elasticsearch node name (string) + * - `es_cluster_name` — Elasticsearch cluster name (string) + * - `content` — full request body (string) + */ +class QueryAuditLogSerializer extends QueryAuditLogSerializerV2 + +/** + * Serializer for audit events (V2) that is aware of **rule-defined verbosity** + * and includes **full request content**. + * - Serializes all non-Allowed events. + * - Serializes `Allowed` events only if the corresponding rule + * specifies that they should be logged at `Verbosity.Info`. + * - Recommended when capturing the full request content along with + * cluster and node context is needed. + * - Fields included: + * - `match` — whether the request matched a rule (boolean) + * - `block` — reason for blocking, if blocked (string) + * - `id` — audit event identifier (string) + * - `final_state` — final processing state (string) + * - `@timestamp` — event timestamp (ISO-8601 string) + * - `correlation_id` — correlation identifier for tracing (string) + * - `processingMillis` — request processing duration in milliseconds (number) + * - `error_type` — type of error, if any (string) + * - `error_message` — error message, if any (string) + * - `content_len` — request body size in bytes (number) + * - `content_len_kb` — request body size in kilobytes (number) + * - `type` — request type (string) + * - `origin` — client (remote) address (string) + * - `destination` — server (local) address (string) + * - `xff` — `X-Forwarded-For` HTTP header value (string) + * - `task_id` — Elasticsearch task ID (number) + * - `req_method` — HTTP request method (string) + * - `headers` — HTTP header names (array of strings) + * - `path` — HTTP request path (string) + * - `user` — authenticated user (string) + * - `impersonated_by` — impersonating user, if applicable (string) + * - `action` — Elasticsearch action name (string) + * - `indices` — indices involved in the request (array of strings) + * - `acl_history` — access control evaluation history (string) + * - `es_node_name` — Elasticsearch node name (string) + * - `es_cluster_name` — Elasticsearch cluster name (string) + * - `content` — full request body (string) + */ +class QueryAuditLogSerializerV2 extends AuditLogSerializer { + + override def onResponse(responseContext: AuditResponseContext): Option[JSONObject] = + AuditSerializationHelper.serialize( + responseContext = responseContext, + fieldGroups = Set(CommonFields, EsEnvironmentFields, FullRequestContentFields), + allowedEventMode = Include(Set(Verbosity.Info)) + ) + +} + +/** + * Serializer for audit events (V1) that is aware of **rule-defined verbosity** + * and includes **full request content**. + * - Serializes all non-Allowed events. + * - Serializes `Allowed` events only if the corresponding rule + * specifies that they should be logged at `Verbosity.Info`. + * - Recommended when capturing the full request content is important, + * without cluster context. + * - Fields included: + * - `match` — whether the request matched a rule (boolean) + * - `block` — reason for blocking, if blocked (string) + * - `id` — audit event identifier (string) + * - `final_state` — final processing state (string) + * - `@timestamp` — event timestamp (ISO-8601 string) + * - `correlation_id` — correlation identifier for tracing (string) + * - `processingMillis` — request processing duration in milliseconds (number) + * - `error_type` — type of error, if any (string) + * - `error_message` — error message, if any (string) + * - `content_len` — request body size in bytes (number) + * - `content_len_kb` — request body size in kilobytes (number) + * - `type` — request type (string) + * - `origin` — client (remote) address (string) + * - `destination` — server (local) address (string) + * - `xff` — `X-Forwarded-For` HTTP header value (string) + * - `task_id` — Elasticsearch task ID (number) + * - `req_method` — HTTP request method (string) + * - `headers` — HTTP header names (array of strings) + * - `path` — HTTP request path (string) + * - `user` — authenticated user (string) + * - `impersonated_by` — impersonating user, if applicable (string) + * - `action` — Elasticsearch action name (string) + * - `indices` — indices involved in the request (array of strings) + * - `acl_history` — access control evaluation history (string) + * - `content` — full request body (string) + */ +class QueryAuditLogSerializerV1 extends AuditLogSerializer { + + override def onResponse(responseContext: AuditResponseContext): Option[JSONObject] = + AuditSerializationHelper.serialize( + responseContext = responseContext, + fieldGroups = Set(CommonFields, FullRequestContentFields), + allowedEventMode = Include(Set(Verbosity.Info)) + ) + +} diff --git a/audit/src/main/scala/tech/beshu/ror/audit/instances/QueryAuditLogSerializerV1.scala b/audit/src/main/scala/tech/beshu/ror/audit/instances/QueryAuditLogSerializerV1.scala deleted file mode 100644 index 4937c4413e..0000000000 --- a/audit/src/main/scala/tech/beshu/ror/audit/instances/QueryAuditLogSerializerV1.scala +++ /dev/null @@ -1,28 +0,0 @@ -/* - * This file is part of ReadonlyREST. - * - * ReadonlyREST is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * ReadonlyREST is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with ReadonlyREST. If not, see http://www.gnu.org/licenses/ - */ -package tech.beshu.ror.audit.instances - -import org.json.JSONObject -import tech.beshu.ror.audit.AuditResponseContext - -class QueryAuditLogSerializerV1 extends DefaultAuditLogSerializerV1 { - - override def onResponse(responseContext: AuditResponseContext): Option[JSONObject] = { - super.onResponse(responseContext) - .map(_.put("content", responseContext.requestContext.content)) - } -} \ No newline at end of file diff --git a/audit/src/main/scala/tech/beshu/ror/audit/instances/QueryAuditLogSerializerV2.scala b/audit/src/main/scala/tech/beshu/ror/audit/instances/QueryAuditLogSerializerV2.scala deleted file mode 100644 index 28d47a69da..0000000000 --- a/audit/src/main/scala/tech/beshu/ror/audit/instances/QueryAuditLogSerializerV2.scala +++ /dev/null @@ -1,28 +0,0 @@ -/* - * This file is part of ReadonlyREST. - * - * ReadonlyREST is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * ReadonlyREST is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with ReadonlyREST. If not, see http://www.gnu.org/licenses/ - */ -package tech.beshu.ror.audit.instances - -import org.json.JSONObject -import tech.beshu.ror.audit.{AuditEnvironmentContext, AuditResponseContext} - -class QueryAuditLogSerializerV2(environmentContext: AuditEnvironmentContext) extends DefaultAuditLogSerializerV2(environmentContext) { - - override def onResponse(responseContext: AuditResponseContext): Option[JSONObject] = { - super.onResponse(responseContext) - .map(_.put("content", responseContext.requestContext.content)) - } -} \ No newline at end of file diff --git a/audit/src/main/scala/tech/beshu/ror/audit/utils/AuditSerializationHelper.scala b/audit/src/main/scala/tech/beshu/ror/audit/utils/AuditSerializationHelper.scala new file mode 100644 index 0000000000..38d0b323b5 --- /dev/null +++ b/audit/src/main/scala/tech/beshu/ror/audit/utils/AuditSerializationHelper.scala @@ -0,0 +1,276 @@ +/* + * This file is part of ReadonlyREST. + * + * ReadonlyREST is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * ReadonlyREST is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with ReadonlyREST. If not, see http://www.gnu.org/licenses/ + */ +package tech.beshu.ror.audit.utils + +import org.json.JSONObject +import tech.beshu.ror.audit.AuditResponseContext._ +import tech.beshu.ror.audit.{AuditRequestContext, AuditResponseContext} + +import java.time.ZoneId +import java.time.format.DateTimeFormatter +import scala.collection.JavaConverters._ +import scala.concurrent.duration.FiniteDuration + +private[ror] object AuditSerializationHelper { + + private val timestampFormatter = DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss'Z'").withZone(ZoneId.of("GMT")) + + def serialize(responseContext: AuditResponseContext, + fieldGroups: Set[AuditFieldGroup], + allowedEventMode: AllowedEventMode): Option[JSONObject] = { + val fields = fieldGroups.flatMap { + case AuditFieldGroup.CommonFields => commonFields + case AuditFieldGroup.EsEnvironmentFields => esEnvironmentFields + case AuditFieldGroup.FullRequestContentFields => requestContentFields + }.toMap + serialize( + responseContext = responseContext, + fields = fields, + allowedEventMode = allowedEventMode + ) + } + + def serialize(responseContext: AuditResponseContext, + fields: Map[AuditFieldName, AuditFieldValueDescriptor], + allowedEventMode: AllowedEventMode): Option[JSONObject] = { + responseContext match { + case Allowed(requestContext, verbosity, reason) => + allowedEvent( + allowedEventMode, + verbosity, + createEntry(fields, EventData(matched = true, "ALLOWED", reason, responseContext.duration, requestContext, None)) + ) + case ForbiddenBy(requestContext, _, reason) => + Some(createEntry(fields, EventData(matched = true, "FORBIDDEN", reason, responseContext.duration, requestContext, None))) + case Forbidden(requestContext) => + Some(createEntry(fields, EventData(matched = false, "FORBIDDEN", "default", responseContext.duration, requestContext, None))) + case RequestedIndexNotExist(requestContext) => + Some(createEntry(fields, EventData(matched = false, "INDEX NOT EXIST", "Requested index doesn't exist", responseContext.duration, requestContext, None))) + case Errored(requestContext, cause) => + Some(createEntry(fields, EventData(matched = false, "ERRORED", "error", responseContext.duration, requestContext, Some(cause)))) + } + } + + private def allowedEvent(allowedEventMode: AllowedEventMode, verbosity: Verbosity, entry: JSONObject) = { + allowedEventMode match { + case AllowedEventMode.IncludeAll => + Some(entry) + case AllowedEventMode.Include(types) if types.contains(verbosity) => + Some(entry) + case _ => + None + } + } + + private def createEntry(fields: Map[AuditFieldName, AuditFieldValueDescriptor], + eventData: EventData) = { + val resolveAuditFieldValue = resolver(eventData) + val resolvedFields: Map[String, Any] = + Map("@timestamp" -> timestampFormatter.format(eventData.requestContext.timestamp)) ++ + fields.map { case (name, valueDescriptor) => name.value -> resolveAuditFieldValue(valueDescriptor) } + + resolvedFields + .foldLeft(new JSONObject()) { case (soFar, (key, value)) => soFar.put(key, value) } + .mergeWith(eventData.requestContext.generalAuditEvents) + } + + private def resolver(eventData: EventData): AuditFieldValueDescriptor => Any = auditValue => { + val requestContext = eventData.requestContext + auditValue match { + case AuditFieldValueDescriptor.IsMatched => eventData.matched + case AuditFieldValueDescriptor.FinalState => eventData.finalState + case AuditFieldValueDescriptor.Reason => eventData.reason + case AuditFieldValueDescriptor.User => SerializeUser.serialize(requestContext).orNull + case AuditFieldValueDescriptor.ImpersonatedByUser => requestContext.impersonatedByUserName.orNull + case AuditFieldValueDescriptor.Action => requestContext.action + case AuditFieldValueDescriptor.InvolvedIndices => if (requestContext.involvesIndices) requestContext.indices.toList.asJava else List.empty.asJava + case AuditFieldValueDescriptor.AclHistory => requestContext.history + case AuditFieldValueDescriptor.ProcessingDurationMillis => eventData.duration.toMillis + case AuditFieldValueDescriptor.Timestamp => timestampFormatter.format(requestContext.timestamp) + case AuditFieldValueDescriptor.Id => requestContext.id + case AuditFieldValueDescriptor.CorrelationId => requestContext.correlationId + case AuditFieldValueDescriptor.TaskId => requestContext.taskId + case AuditFieldValueDescriptor.ErrorType => eventData.error.map(_.getClass.getSimpleName).orNull + case AuditFieldValueDescriptor.ErrorMessage => eventData.error.map(_.getMessage).orNull + case AuditFieldValueDescriptor.Type => requestContext.`type` + case AuditFieldValueDescriptor.HttpMethod => requestContext.httpMethod + case AuditFieldValueDescriptor.HttpHeaderNames => requestContext.requestHeaders.names.asJava + case AuditFieldValueDescriptor.HttpPath => requestContext.uriPath + case AuditFieldValueDescriptor.XForwardedForHttpHeader => requestContext.requestHeaders.getValue("X-Forwarded-For").flatMap(_.headOption).orNull + case AuditFieldValueDescriptor.RemoteAddress => requestContext.remoteAddress + case AuditFieldValueDescriptor.LocalAddress => requestContext.localAddress + case AuditFieldValueDescriptor.Content => requestContext.content + case AuditFieldValueDescriptor.ContentLengthInBytes => requestContext.contentLength + case AuditFieldValueDescriptor.ContentLengthInKb => requestContext.contentLength / 1024 + case AuditFieldValueDescriptor.EsNodeName => eventData.requestContext.auditEnvironmentContext.esNodeName + case AuditFieldValueDescriptor.EsClusterName => eventData.requestContext.auditEnvironmentContext.esClusterName + case AuditFieldValueDescriptor.StaticText(text) => text + case AuditFieldValueDescriptor.Combined(values) => values.map(resolver(eventData)).mkString + } + } + + private implicit class JsonObjectOps(val mainJson: JSONObject) { + def mergeWith(secondaryJson: JSONObject): JSONObject = { + jsonKeys(secondaryJson).foldLeft(mainJson) { + case (json, name) if !json.has(name) => + json.put(name, secondaryJson.get(name)) + case (json, _) => + json + } + } + + private def jsonKeys(json: JSONObject) = { + Option(JSONObject.getNames(json)).toList.flatten + } + } + + private final case class EventData(matched: Boolean, + finalState: String, + reason: String, + duration: FiniteDuration, + requestContext: AuditRequestContext, + error: Option[Throwable]) + + sealed trait AllowedEventMode + + object AllowedEventMode { + case object IncludeAll extends AllowedEventMode + + final case class Include(types: Set[Verbosity]) extends AllowedEventMode + } + + final case class AuditFieldName(value: String) + + sealed trait AuditFieldValueDescriptor + + object AuditFieldValueDescriptor { + + // Rule + case object IsMatched extends AuditFieldValueDescriptor + + case object FinalState extends AuditFieldValueDescriptor + + case object Reason extends AuditFieldValueDescriptor + + case object User extends AuditFieldValueDescriptor + + case object ImpersonatedByUser extends AuditFieldValueDescriptor + + case object Action extends AuditFieldValueDescriptor + + case object InvolvedIndices extends AuditFieldValueDescriptor + + case object AclHistory extends AuditFieldValueDescriptor + + case object ProcessingDurationMillis extends AuditFieldValueDescriptor + + // Identifiers + case object Timestamp extends AuditFieldValueDescriptor + + case object Id extends AuditFieldValueDescriptor + + case object CorrelationId extends AuditFieldValueDescriptor + + case object TaskId extends AuditFieldValueDescriptor + + // Error details + case object ErrorType extends AuditFieldValueDescriptor + + case object ErrorMessage extends AuditFieldValueDescriptor + + case object Type extends AuditFieldValueDescriptor + + // HTTP protocol values + case object HttpMethod extends AuditFieldValueDescriptor + + case object HttpHeaderNames extends AuditFieldValueDescriptor + + case object HttpPath extends AuditFieldValueDescriptor + + case object XForwardedForHttpHeader extends AuditFieldValueDescriptor + + case object RemoteAddress extends AuditFieldValueDescriptor + + case object LocalAddress extends AuditFieldValueDescriptor + + case object Content extends AuditFieldValueDescriptor + + case object ContentLengthInBytes extends AuditFieldValueDescriptor + + case object ContentLengthInKb extends AuditFieldValueDescriptor + + // ES environment + + case object EsNodeName extends AuditFieldValueDescriptor + + case object EsClusterName extends AuditFieldValueDescriptor + + // Technical + + final case class StaticText(value: String) extends AuditFieldValueDescriptor + + final case class Combined(values: List[AuditFieldValueDescriptor]) extends AuditFieldValueDescriptor + + } + + sealed trait AuditFieldGroup + + object AuditFieldGroup { + case object CommonFields extends AuditFieldGroup + + case object EsEnvironmentFields extends AuditFieldGroup + + case object FullRequestContentFields extends AuditFieldGroup + } + + private val commonFields: Map[AuditFieldName, AuditFieldValueDescriptor] = Map( + AuditFieldName("match") -> AuditFieldValueDescriptor.IsMatched, + AuditFieldName("block") -> AuditFieldValueDescriptor.Reason, + AuditFieldName("id") -> AuditFieldValueDescriptor.Id, + AuditFieldName("final_state") -> AuditFieldValueDescriptor.FinalState, + AuditFieldName("@timestamp") -> AuditFieldValueDescriptor.Timestamp, + AuditFieldName("correlation_id") -> AuditFieldValueDescriptor.CorrelationId, + AuditFieldName("processingMillis") -> AuditFieldValueDescriptor.ProcessingDurationMillis, + AuditFieldName("error_type") -> AuditFieldValueDescriptor.ErrorType, + AuditFieldName("error_message") -> AuditFieldValueDescriptor.ErrorMessage, + AuditFieldName("content_len") -> AuditFieldValueDescriptor.ContentLengthInBytes, + AuditFieldName("content_len_kb") -> AuditFieldValueDescriptor.ContentLengthInKb, + AuditFieldName("type") -> AuditFieldValueDescriptor.Type, + AuditFieldName("origin") -> AuditFieldValueDescriptor.RemoteAddress, + AuditFieldName("destination") -> AuditFieldValueDescriptor.LocalAddress, + AuditFieldName("xff") -> AuditFieldValueDescriptor.XForwardedForHttpHeader, + AuditFieldName("task_id") -> AuditFieldValueDescriptor.TaskId, + AuditFieldName("req_method") -> AuditFieldValueDescriptor.HttpMethod, + AuditFieldName("headers") -> AuditFieldValueDescriptor.HttpHeaderNames, + AuditFieldName("path") -> AuditFieldValueDescriptor.HttpPath, + AuditFieldName("user") -> AuditFieldValueDescriptor.User, + AuditFieldName("impersonated_by") -> AuditFieldValueDescriptor.ImpersonatedByUser, + AuditFieldName("action") -> AuditFieldValueDescriptor.Action, + AuditFieldName("indices") -> AuditFieldValueDescriptor.InvolvedIndices, + AuditFieldName("acl_history") -> AuditFieldValueDescriptor.AclHistory + ) + + private val esEnvironmentFields: Map[AuditFieldName, AuditFieldValueDescriptor] = Map( + AuditFieldName("es_node_name") -> AuditFieldValueDescriptor.EsNodeName, + AuditFieldName("es_cluster_name") -> AuditFieldValueDescriptor.EsClusterName + ) + + private val requestContentFields: Map[AuditFieldName, AuditFieldValueDescriptor] = Map( + AuditFieldName("content") -> AuditFieldValueDescriptor.Content + ) + +} diff --git a/audit/src/main/scala/tech/beshu/ror/audit/instances/SerializeUser.scala b/audit/src/main/scala/tech/beshu/ror/audit/utils/SerializeUser.scala similarity index 93% rename from audit/src/main/scala/tech/beshu/ror/audit/instances/SerializeUser.scala rename to audit/src/main/scala/tech/beshu/ror/audit/utils/SerializeUser.scala index 0e4330df41..a580dc74d8 100644 --- a/audit/src/main/scala/tech/beshu/ror/audit/instances/SerializeUser.scala +++ b/audit/src/main/scala/tech/beshu/ror/audit/utils/SerializeUser.scala @@ -14,11 +14,11 @@ * You should have received a copy of the GNU General Public License * along with ReadonlyREST. If not, see http://www.gnu.org/licenses/ */ -package tech.beshu.ror.audit.instances +package tech.beshu.ror.audit.utils import tech.beshu.ror.audit.AuditRequestContext -object SerializeUser { +private[ror] object SerializeUser { def serialize(requestContext: AuditRequestContext): Option[String] = { requestContext.loggedInUserName.orElse(requestContext.attemptedUserName).orElse(requestContext.rawAuthHeader) diff --git a/audit/src/test/scala/tech/beshu/ror/audit/SerializerTest.scala b/audit/src/test/scala/tech/beshu/ror/audit/SerializerTest.scala new file mode 100644 index 0000000000..ec7e399338 --- /dev/null +++ b/audit/src/test/scala/tech/beshu/ror/audit/SerializerTest.scala @@ -0,0 +1,345 @@ +/* + * This file is part of ReadonlyREST. + * + * ReadonlyREST is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * ReadonlyREST is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with ReadonlyREST. If not, see http://www.gnu.org/licenses/ + */ +package tech.beshu.ror.audit + +import org.json.JSONObject +import org.scalatest.matchers.should.Matchers._ +import org.scalatest.wordspec.AnyWordSpec +import tech.beshu.ror.audit.instances._ + +import java.time.Instant +import scala.annotation.nowarn +import scala.jdk.CollectionConverters._ + +class SerializerTest extends AnyWordSpec { + + "Serializer" when { + "serialized event contains only expected fields" when { + "QueryAuditLogSerializer" in { + testSerializerFieldsWithTypes( + serializer = new QueryAuditLogSerializer, + expectedFieldsWithTypes = Map( + "match" -> "boolean", + "block" -> "string", + "id" -> "string", + "final_state" -> "string", + "@timestamp" -> "string", + "correlation_id" -> "string", + "processingMillis" -> "number", + "content_len" -> "number", + "content_len_kb" -> "number", + "type" -> "string", + "origin" -> "string", + "destination" -> "string", + "task_id" -> "number", + "req_method" -> "string", + "headers" -> "array", + "path" -> "string", + "user" -> "string", + "action" -> "string", + "indices" -> "array", + "acl_history" -> "string", + "es_node_name" -> "string", + "es_cluster_name" -> "string", + "content" -> "string", + ) + ) + } + "QueryAuditLogSerializerV2" in { + testSerializerFieldsWithTypes( + serializer = new QueryAuditLogSerializerV2, + expectedFieldsWithTypes = Map( + "match" -> "boolean", + "block" -> "string", + "id" -> "string", + "final_state" -> "string", + "@timestamp" -> "string", + "correlation_id" -> "string", + "processingMillis" -> "number", + "content_len" -> "number", + "content_len_kb" -> "number", + "type" -> "string", + "origin" -> "string", + "destination" -> "string", + "task_id" -> "number", + "req_method" -> "string", + "headers" -> "array", + "path" -> "string", + "user" -> "string", + "action" -> "string", + "indices" -> "array", + "acl_history" -> "string", + "es_node_name" -> "string", + "es_cluster_name" -> "string", + "content" -> "string", + ) + ) + } + "QueryAuditLogSerializerV1" in { + testSerializerFieldsWithTypes( + serializer = new QueryAuditLogSerializerV1, + expectedFieldsWithTypes = Map( + "match" -> "boolean", + "block" -> "string", + "id" -> "string", + "final_state" -> "string", + "@timestamp" -> "string", + "correlation_id" -> "string", + "processingMillis" -> "number", + "content_len" -> "number", + "content_len_kb" -> "number", + "type" -> "string", + "origin" -> "string", + "destination" -> "string", + "task_id" -> "number", + "req_method" -> "string", + "headers" -> "array", + "path" -> "string", + "user" -> "string", + "action" -> "string", + "indices" -> "array", + "acl_history" -> "string", + "content" -> "string", + ) + ) + } + "FullAuditLogWithQuerySerializer" in { + testSerializerFieldsWithTypes( + serializer = new FullAuditLogWithQuerySerializer, + expectedFieldsWithTypes = Map( + "match" -> "boolean", + "block" -> "string", + "id" -> "string", + "final_state" -> "string", + "@timestamp" -> "string", + "correlation_id" -> "string", + "processingMillis" -> "number", + "content_len" -> "number", + "content_len_kb" -> "number", + "type" -> "string", + "origin" -> "string", + "destination" -> "string", + "task_id" -> "number", + "req_method" -> "string", + "headers" -> "array", + "path" -> "string", + "user" -> "string", + "action" -> "string", + "indices" -> "array", + "acl_history" -> "string", + "es_node_name" -> "string", + "es_cluster_name" -> "string", + "content" -> "string", + ) + ) + } + "FullAuditLogSerializer" in { + testSerializerFieldsWithTypes( + serializer = new FullAuditLogSerializer, + expectedFieldsWithTypes = Map( + "match" -> "boolean", + "block" -> "string", + "id" -> "string", + "final_state" -> "string", + "@timestamp" -> "string", + "correlation_id" -> "string", + "processingMillis" -> "number", + "content_len" -> "number", + "content_len_kb" -> "number", + "type" -> "string", + "origin" -> "string", + "destination" -> "string", + "task_id" -> "number", + "req_method" -> "string", + "headers" -> "array", + "path" -> "string", + "user" -> "string", + "action" -> "string", + "indices" -> "array", + "acl_history" -> "string", + "es_node_name" -> "string", + "es_cluster_name" -> "string", + ) + ) + } + "BlockVerbosityAwareAuditLogSerializer" in { + testSerializerFieldsWithTypes( + serializer = new BlockVerbosityAwareAuditLogSerializer, + expectedFieldsWithTypes = Map( + "match" -> "boolean", + "block" -> "string", + "id" -> "string", + "final_state" -> "string", + "@timestamp" -> "string", + "correlation_id" -> "string", + "processingMillis" -> "number", + "content_len" -> "number", + "content_len_kb" -> "number", + "type" -> "string", + "origin" -> "string", + "destination" -> "string", + "task_id" -> "number", + "req_method" -> "string", + "headers" -> "array", + "path" -> "string", + "user" -> "string", + "action" -> "string", + "indices" -> "array", + "acl_history" -> "string", + "es_node_name" -> "string", + "es_cluster_name" -> "string", + ) + ) + } + "DefaultAuditLogSerializerV2" in { + testSerializerFieldsWithTypes( + serializer = new DefaultAuditLogSerializerV2 @nowarn("cat=deprecation"), + expectedFieldsWithTypes = Map( + "match" -> "boolean", + "block" -> "string", + "id" -> "string", + "final_state" -> "string", + "@timestamp" -> "string", + "correlation_id" -> "string", + "processingMillis" -> "number", + "content_len" -> "number", + "content_len_kb" -> "number", + "type" -> "string", + "origin" -> "string", + "destination" -> "string", + "task_id" -> "number", + "req_method" -> "string", + "headers" -> "array", + "path" -> "string", + "user" -> "string", + "action" -> "string", + "indices" -> "array", + "acl_history" -> "string", + "es_node_name" -> "string", + "es_cluster_name" -> "string", + ) + ) + } + "DefaultAuditLogSerializerV1" in { + testSerializerFieldsWithTypes( + serializer = new DefaultAuditLogSerializerV1 @nowarn("cat=deprecation"), + expectedFieldsWithTypes = Map( + "match" -> "boolean", + "block" -> "string", + "id" -> "string", + "final_state" -> "string", + "@timestamp" -> "string", + "correlation_id" -> "string", + "processingMillis" -> "number", + "content_len" -> "number", + "content_len_kb" -> "number", + "type" -> "string", + "origin" -> "string", + "destination" -> "string", + "task_id" -> "number", + "req_method" -> "string", + "headers" -> "array", + "path" -> "string", + "user" -> "string", + "action" -> "string", + "indices" -> "array", + "acl_history" -> "string", + ), + ) + } + } + } + + private def testSerializerFieldsWithTypes(serializer: AuditLogSerializer, + expectedFieldsWithTypes: Map[String, String]): Unit = { + val serialized = serializer.onResponse(AuditResponseContext.Forbidden(DummyAuditRequestContext)).get + val entryFields = serialized.keySet.asScala.toSet + + if (entryFields != expectedFieldsWithTypes.keySet) { + fail(s"Serialized event does not contains exactly fields that were expected") + } else { + expectedFieldsWithTypes.foreach { case (fieldName, expectedType) => + expectedType match { + case "string" => noException should be thrownBy serialized.getString(fieldName) + case "boolean" => noException should be thrownBy serialized.getBoolean(fieldName) + case "number" => noException should be thrownBy serialized.getDouble(fieldName) + case "array" => + val value = serialized.get(fieldName) + value match { + case _: org.json.JSONArray => succeed + case s: java.util.Collection[_] => noException should be thrownBy new org.json.JSONArray(s) + case other => fail(s"Expected '$fieldName' to be JSONArray, but got ${other.getClass.getName}: $other") + } + case other => fail(s"Unknown expected type: $other") + } + } + } + } + +} + +private object DummyAuditRequestContext extends AuditRequestContext { + override def timestamp: Instant = Instant.now() + + override def id: String = "" + + override def correlationId: String = "" + + override def indices: Set[String] = Set.empty + + override def action: String = "" + + override def headers: Map[String, String] = Map.empty + + override def requestHeaders: Headers = Headers(Map.empty) + + override def uriPath: String = "" + + override def history: String = "" + + override def content: String = "" + + override def contentLength: Integer = 0 + + override def remoteAddress: String = "" + + override def localAddress: String = "" + + override def `type`: String = "" + + override def taskId: Long = 0 + + override def httpMethod: String = "" + + override def loggedInUserName: Option[String] = Some("logged_user") + + override def impersonatedByUserName: Option[String] = None + + override def involvesIndices: Boolean = false + + override def attemptedUserName: Option[String] = None + + override def rawAuthHeader: Option[String] = None + + override def generalAuditEvents: JSONObject = new JSONObject + + override def auditEnvironmentContext: AuditEnvironmentContext = new AuditEnvironmentContext { + override def esNodeName: String = "testEsNode" + + override def esClusterName: String = "testEsCluster" + } +} diff --git a/ci/run-pipeline.sh b/ci/run-pipeline.sh index 1163e7e2b8..d9dad1ffb5 100755 --- a/ci/run-pipeline.sh +++ b/ci/run-pipeline.sh @@ -42,7 +42,7 @@ fi if [[ -z $TRAVIS ]] || [[ $ROR_TASK == "core_tests" ]]; then echo ">>> Running unit tests.." - ./gradlew --no-daemon --stacktrace core:test + ./gradlew --no-daemon --stacktrace core:test audit:test fi run_integration_tests() { diff --git a/core/src/main/scala/tech/beshu/ror/accesscontrol/audit/AuditRequestContextBasedOnAclResult.scala b/core/src/main/scala/tech/beshu/ror/accesscontrol/audit/AuditRequestContextBasedOnAclResult.scala index fe88dd7725..005283c492 100644 --- a/core/src/main/scala/tech/beshu/ror/accesscontrol/audit/AuditRequestContextBasedOnAclResult.scala +++ b/core/src/main/scala/tech/beshu/ror/accesscontrol/audit/AuditRequestContextBasedOnAclResult.scala @@ -26,7 +26,7 @@ import tech.beshu.ror.accesscontrol.domain.LoggedUser.{DirectlyLoggedUser, Imper import tech.beshu.ror.accesscontrol.domain.{Address, Header} import tech.beshu.ror.accesscontrol.request.RequestContext import tech.beshu.ror.accesscontrol.request.RequestContextOps.* -import tech.beshu.ror.audit.{AuditRequestContext, Headers} +import tech.beshu.ror.audit.{AuditEnvironmentContext, AuditRequestContext, Headers} import tech.beshu.ror.implicits.* import java.time.Instant @@ -36,6 +36,7 @@ private[audit] class AuditRequestContextBasedOnAclResult[B <: BlockContext](requ userMetadata: Option[UserMetadata], historyEntries: Vector[History[B]], loggingContext: LoggingContext, + override val auditEnvironmentContext: AuditEnvironmentContext, override val generalAuditEvents: JSONObject, override val involvesIndices: Boolean) extends AuditRequestContext { diff --git a/core/src/main/scala/tech/beshu/ror/accesscontrol/audit/AuditingTool.scala b/core/src/main/scala/tech/beshu/ror/accesscontrol/audit/AuditingTool.scala index 80dcd2ccfa..0ba7e544d1 100644 --- a/core/src/main/scala/tech/beshu/ror/accesscontrol/audit/AuditingTool.scala +++ b/core/src/main/scala/tech/beshu/ror/accesscontrol/audit/AuditingTool.scala @@ -31,7 +31,7 @@ import tech.beshu.ror.accesscontrol.blocks.{Block, BlockContext} import tech.beshu.ror.accesscontrol.domain.{AuditCluster, RorAuditDataStream, RorAuditIndexTemplate, RorAuditLoggerName} import tech.beshu.ror.accesscontrol.logging.ResponseContext import tech.beshu.ror.accesscontrol.request.RequestContext -import tech.beshu.ror.audit.instances.DefaultAuditLogSerializer +import tech.beshu.ror.audit.instances.BlockVerbosityAwareAuditLogSerializer import tech.beshu.ror.audit.{AuditEnvironmentContext, AuditLogSerializer, AuditRequestContext, AuditResponseContext} import tech.beshu.ror.implicits.* @@ -40,8 +40,8 @@ import java.time.Clock final class AuditingTool private(auditSinks: NonEmptyList[BaseAuditSink]) (implicit loggingContext: LoggingContext) { - def audit[B <: BlockContext](response: ResponseContext[B]): Task[Unit] = { - val auditResponseContext = toAuditResponse(response) + def audit[B <: BlockContext](response: ResponseContext[B], auditEnvironmentContext: AuditEnvironmentContext): Task[Unit] = { + val auditResponseContext = toAuditResponse(response, auditEnvironmentContext) auditSinks .parTraverse(_.submit(auditResponseContext)) .map((_: NonEmptyList[Unit]) => ()) @@ -53,12 +53,13 @@ final class AuditingTool private(auditSinks: NonEmptyList[BaseAuditSink]) .map((_: NonEmptyList[Unit]) => ()) } - private def toAuditResponse[B <: BlockContext](responseContext: ResponseContext[B]): AuditResponseContext = { + private def toAuditResponse[B <: BlockContext](responseContext: ResponseContext[B], auditEnvironmentContext: AuditEnvironmentContext): AuditResponseContext = { responseContext match { case allowedBy: ResponseContext.AllowedBy[B] => AuditResponseContext.Allowed( requestContext = toAuditRequestContext( requestContext = allowedBy.requestContext, + auditEnvironmentContext = auditEnvironmentContext, blockContext = Some(allowedBy.blockContext), userMetadata = Some(allowedBy.blockContext.userMetadata), historyEntries = allowedBy.history, @@ -71,6 +72,7 @@ final class AuditingTool private(auditSinks: NonEmptyList[BaseAuditSink]) AuditResponseContext.Allowed( requestContext = toAuditRequestContext( requestContext = allow.requestContext, + auditEnvironmentContext = auditEnvironmentContext, blockContext = None, userMetadata = Some(allow.userMetadata), historyEntries = allow.history, @@ -83,6 +85,7 @@ final class AuditingTool private(auditSinks: NonEmptyList[BaseAuditSink]) AuditResponseContext.ForbiddenBy( requestContext = toAuditRequestContext( requestContext = forbiddenBy.requestContext, + auditEnvironmentContext = auditEnvironmentContext, blockContext = Some(forbiddenBy.blockContext), userMetadata = Some(forbiddenBy.blockContext.userMetadata), historyEntries = forbiddenBy.history), @@ -92,6 +95,7 @@ final class AuditingTool private(auditSinks: NonEmptyList[BaseAuditSink]) case forbidden: ResponseContext.Forbidden[B] => AuditResponseContext.Forbidden(toAuditRequestContext( requestContext = forbidden.requestContext, + auditEnvironmentContext = auditEnvironmentContext, blockContext = None, userMetadata = None, historyEntries = forbidden.history)) @@ -99,6 +103,7 @@ final class AuditingTool private(auditSinks: NonEmptyList[BaseAuditSink]) AuditResponseContext.RequestedIndexNotExist( toAuditRequestContext( requestContext = requestedIndexNotExist.requestContext, + auditEnvironmentContext = auditEnvironmentContext, blockContext = None, userMetadata = None, historyEntries = requestedIndexNotExist.history) @@ -107,6 +112,7 @@ final class AuditingTool private(auditSinks: NonEmptyList[BaseAuditSink]) AuditResponseContext.Errored( requestContext = toAuditRequestContext( requestContext = errored.requestContext, + auditEnvironmentContext = auditEnvironmentContext, blockContext = None, userMetadata = None, historyEntries = Vector.empty), @@ -120,6 +126,7 @@ final class AuditingTool private(auditSinks: NonEmptyList[BaseAuditSink]) } private def toAuditRequestContext[B <: BlockContext](requestContext: RequestContext.Aux[B], + auditEnvironmentContext: AuditEnvironmentContext, blockContext: Option[B], userMetadata: Option[UserMetadata], historyEntries: Vector[History[B]], @@ -129,6 +136,7 @@ final class AuditingTool private(auditSinks: NonEmptyList[BaseAuditSink]) userMetadata, historyEntries, loggingContext, + auditEnvironmentContext, generalAuditEvents, involvesIndices(blockContext) ) @@ -162,8 +170,8 @@ object AuditingTool extends Logging { auditCluster: AuditCluster) extends Config object EsIndexBasedSink { - def default(implicit auditEnvironmentContext: AuditEnvironmentContext): EsIndexBasedSink = EsIndexBasedSink( - logSerializer = new DefaultAuditLogSerializer(auditEnvironmentContext), + val default: EsIndexBasedSink = EsIndexBasedSink( + logSerializer = new BlockVerbosityAwareAuditLogSerializer, rorAuditIndexTemplate = RorAuditIndexTemplate.default, auditCluster = AuditCluster.LocalAuditCluster, ) @@ -174,8 +182,8 @@ object AuditingTool extends Logging { auditCluster: AuditCluster) extends Config object EsDataStreamBasedSink { - def default(implicit auditEnvironmentContext: AuditEnvironmentContext): EsDataStreamBasedSink = EsDataStreamBasedSink( - logSerializer = new DefaultAuditLogSerializer(auditEnvironmentContext), + val default: EsDataStreamBasedSink = EsDataStreamBasedSink( + logSerializer = new BlockVerbosityAwareAuditLogSerializer, rorAuditDataStream = RorAuditDataStream.default, auditCluster = AuditCluster.LocalAuditCluster, ) @@ -185,8 +193,8 @@ object AuditingTool extends Logging { loggerName: RorAuditLoggerName) extends Config object LogBasedSink { - def default(implicit auditEnvironmentContext: AuditEnvironmentContext): LogBasedSink = LogBasedSink( - logSerializer = new DefaultAuditLogSerializer(auditEnvironmentContext), + val default: LogBasedSink = LogBasedSink( + logSerializer = new BlockVerbosityAwareAuditLogSerializer, loggerName = RorAuditLoggerName.default ) } diff --git a/core/src/main/scala/tech/beshu/ror/accesscontrol/audit/configurable/AuditFieldValueDescriptorParser.scala b/core/src/main/scala/tech/beshu/ror/accesscontrol/audit/configurable/AuditFieldValueDescriptorParser.scala new file mode 100644 index 0000000000..e07f1bb7a3 --- /dev/null +++ b/core/src/main/scala/tech/beshu/ror/accesscontrol/audit/configurable/AuditFieldValueDescriptorParser.scala @@ -0,0 +1,91 @@ +/* + * This file is part of ReadonlyREST. + * + * ReadonlyREST is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * ReadonlyREST is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with ReadonlyREST. If not, see http://www.gnu.org/licenses/ + */ +package tech.beshu.ror.accesscontrol.audit.configurable + +import cats.parse.{Parser0, Parser as P} +import cats.syntax.list.* +import tech.beshu.ror.audit.utils.AuditSerializationHelper.AuditFieldValueDescriptor + +object AuditFieldValueDescriptorParser { + + private val lbrace = P.char('{') + private val rbrace = P.char('}') + private val key = P.charsWhile(c => c != '{' && c != '}') + + private val placeholder: P[Either[String, AuditFieldValueDescriptor]] = + (lbrace *> key <* rbrace).map(k => deserializerAuditFieldValueDescriptor(k.trim.toUpperCase).toRight(k)) + + private val text: P[AuditFieldValueDescriptor] = + P.charsWhile(_ != '{').map(AuditFieldValueDescriptor.StaticText.apply) + + private val segment: P[Either[String, AuditFieldValueDescriptor]] = + placeholder.orElse(text.map(Right(_))) + + private val parser: Parser0[List[Either[String, AuditFieldValueDescriptor]]] = + segment.rep0 <* P.end + + def parse(s: String): Either[String, AuditFieldValueDescriptor] = + parser.parseAll(s) match { + case Left(err) => + Left(err.toString) + case Right(segments) => + val (missing, ok) = segments.partitionMap(identity) + missing.toNel match { + case Some(missing) => Left(s"There are invalid placeholder values: ${missing.toList.distinct.mkString(", ")}") + case None => ok match { + case Nil => Right(AuditFieldValueDescriptor.StaticText("")) + case single :: Nil => Right(single) + case many => Right(AuditFieldValueDescriptor.Combined(many)) + } + } + } + + private def deserializerAuditFieldValueDescriptor(str: String): Option[AuditFieldValueDescriptor] = { + str.toUpperCase match { + case "IS_MATCHED" => Some(AuditFieldValueDescriptor.IsMatched) + case "FINAL_STATE" => Some(AuditFieldValueDescriptor.FinalState) + case "REASON" => Some(AuditFieldValueDescriptor.Reason) + case "USER" => Some(AuditFieldValueDescriptor.User) + case "IMPERSONATED_BY_USER" => Some(AuditFieldValueDescriptor.ImpersonatedByUser) + case "ACTION" => Some(AuditFieldValueDescriptor.Action) + case "INVOLVED_INDICES" => Some(AuditFieldValueDescriptor.InvolvedIndices) + case "ACL_HISTORY" => Some(AuditFieldValueDescriptor.AclHistory) + case "PROCESSING_DURATION_MILLIS" => Some(AuditFieldValueDescriptor.ProcessingDurationMillis) + case "TIMESTAMP" => Some(AuditFieldValueDescriptor.Timestamp) + case "ID" => Some(AuditFieldValueDescriptor.Id) + case "CORRELATION_ID" => Some(AuditFieldValueDescriptor.CorrelationId) + case "TASK_ID" => Some(AuditFieldValueDescriptor.TaskId) + case "ERROR_TYPE" => Some(AuditFieldValueDescriptor.ErrorType) + case "ERROR_MESSAGE" => Some(AuditFieldValueDescriptor.ErrorMessage) + case "TYPE" => Some(AuditFieldValueDescriptor.Type) + case "HTTP_METHOD" => Some(AuditFieldValueDescriptor.HttpMethod) + case "HTTP_HEADER_NAMES" => Some(AuditFieldValueDescriptor.HttpHeaderNames) + case "HTTP_PATH" => Some(AuditFieldValueDescriptor.HttpPath) + case "X_FORWARDED_FOR_HTTP_HEADER" => Some(AuditFieldValueDescriptor.XForwardedForHttpHeader) + case "REMOTE_ADDRESS" => Some(AuditFieldValueDescriptor.RemoteAddress) + case "LOCAL_ADDRESS" => Some(AuditFieldValueDescriptor.LocalAddress) + case "CONTENT" => Some(AuditFieldValueDescriptor.Content) + case "CONTENT_LENGTH_IN_BYTES" => Some(AuditFieldValueDescriptor.ContentLengthInBytes) + case "CONTENT_LENGTH_IN_KB" => Some(AuditFieldValueDescriptor.ContentLengthInKb) + case "ES_NODE_NAME" => Some(AuditFieldValueDescriptor.EsNodeName) + case "ES_CLUSTER_NAME" => Some(AuditFieldValueDescriptor.EsClusterName) + case _ => None + } + + } + +} diff --git a/audit/src/main/scala/tech/beshu/ror/audit/instances/DefaultAuditLogSerializerV2.scala b/core/src/main/scala/tech/beshu/ror/accesscontrol/audit/configurable/ConfigurableAuditLogSerializer.scala similarity index 57% rename from audit/src/main/scala/tech/beshu/ror/audit/instances/DefaultAuditLogSerializerV2.scala rename to core/src/main/scala/tech/beshu/ror/accesscontrol/audit/configurable/ConfigurableAuditLogSerializer.scala index 9772b5e6ee..3e2d6a76b8 100644 --- a/audit/src/main/scala/tech/beshu/ror/audit/instances/DefaultAuditLogSerializerV2.scala +++ b/core/src/main/scala/tech/beshu/ror/accesscontrol/audit/configurable/ConfigurableAuditLogSerializer.scala @@ -14,19 +14,17 @@ * You should have received a copy of the GNU General Public License * along with ReadonlyREST. If not, see http://www.gnu.org/licenses/ */ -package tech.beshu.ror.audit.instances +package tech.beshu.ror.accesscontrol.audit.configurable import org.json.JSONObject -import tech.beshu.ror.audit.{AuditEnvironmentContext, AuditResponseContext} +import tech.beshu.ror.audit.utils.AuditSerializationHelper +import tech.beshu.ror.audit.utils.AuditSerializationHelper.{AllowedEventMode, AuditFieldName, AuditFieldValueDescriptor} +import tech.beshu.ror.audit.{AuditLogSerializer, AuditResponseContext} -class DefaultAuditLogSerializerV2(environmentContext: AuditEnvironmentContext) extends DefaultAuditLogSerializerV1 { +class ConfigurableAuditLogSerializer(val allowedEventMode: AllowedEventMode, + val fields: Map[AuditFieldName, AuditFieldValueDescriptor]) extends AuditLogSerializer { + + override def onResponse(responseContext: AuditResponseContext): Option[JSONObject] = + AuditSerializationHelper.serialize(responseContext, fields, allowedEventMode) - override def onResponse(responseContext: AuditResponseContext): Option[JSONObject] = { - lazy val additionalFields = Map( - "es_node_name" -> environmentContext.esNodeName, - "es_cluster_name" -> environmentContext.esClusterName - ) - super.onResponse(responseContext) - .map(additionalFields.foldLeft(_) { case (soFar, (key, value)) => soFar.put(key, value) }) - } } diff --git a/core/src/main/scala/tech/beshu/ror/accesscontrol/factory/RawRorConfigBasedCoreFactory.scala b/core/src/main/scala/tech/beshu/ror/accesscontrol/factory/RawRorConfigBasedCoreFactory.scala index 7bf1a6d24a..98b63fc471 100644 --- a/core/src/main/scala/tech/beshu/ror/accesscontrol/factory/RawRorConfigBasedCoreFactory.scala +++ b/core/src/main/scala/tech/beshu/ror/accesscontrol/factory/RawRorConfigBasedCoreFactory.scala @@ -23,7 +23,7 @@ import monix.eval.Task import org.apache.logging.log4j.scala.Logging import tech.beshu.ror.accesscontrol.* import tech.beshu.ror.accesscontrol.EnabledAccessControlList.AccessControlListStaticContext -import tech.beshu.ror.accesscontrol.audit.{AuditEnvironmentContextBasedOnEsNodeSettings, LoggingContext} +import tech.beshu.ror.accesscontrol.audit.LoggingContext import tech.beshu.ror.accesscontrol.blocks.Block.{RuleDefinition, Verbosity} import tech.beshu.ror.accesscontrol.blocks.ImpersonationWarning.ImpersonationWarningSupport import tech.beshu.ror.accesscontrol.blocks.definitions.UserDef.Mode @@ -50,7 +50,7 @@ import tech.beshu.ror.accesscontrol.utils.CirceOps.* import tech.beshu.ror.accesscontrol.utils.CirceOps.DecoderHelpers.FieldListResult.{FieldListValue, NoField} import tech.beshu.ror.configuration.RorConfig.ImpersonationWarningsReader import tech.beshu.ror.configuration.{EnvironmentConfig, RawRorConfig, RorConfig} -import tech.beshu.ror.es.{EsNodeSettings, EsVersion} +import tech.beshu.ror.es.EsVersion import tech.beshu.ror.implicits.* import tech.beshu.ror.syntax.* import tech.beshu.ror.utils.ScalaOps.* @@ -67,8 +67,7 @@ trait CoreFactory { mocksProvider: MocksProvider): Task[Either[NonEmptyList[CoreCreationError], Core]] } -class RawRorConfigBasedCoreFactory(esVersion: EsVersion, - esNodeSettings: EsNodeSettings) +class RawRorConfigBasedCoreFactory(esVersion: EsVersion) (implicit environmentConfig: EnvironmentConfig) extends CoreFactory with Logging { @@ -318,8 +317,7 @@ class RawRorConfigBasedCoreFactory(esVersion: EsVersion, dynamicVariableTransformationAliases.items.map(_.alias) ) ) - auditEnvironmentContext = new AuditEnvironmentContextBasedOnEsNodeSettings(esNodeSettings) - auditingTools <- AsyncDecoderCreator.from(AuditingSettingsDecoder.instance(esVersion)(auditEnvironmentContext)) + auditingTools <- AsyncDecoderCreator.from(AuditingSettingsDecoder.instance(esVersion)) authProxies <- AsyncDecoderCreator.from(ProxyAuthDefinitionsDecoder.instance) authenticationServices <- AsyncDecoderCreator.from(ExternalAuthenticationServicesDecoder.instance(httpClientFactory)) authorizationServices <- AsyncDecoderCreator.from(ExternalAuthorizationServicesDecoder.instance(httpClientFactory)) diff --git a/core/src/main/scala/tech/beshu/ror/accesscontrol/factory/decoders/AuditingSettingsDecoder.scala b/core/src/main/scala/tech/beshu/ror/accesscontrol/factory/decoders/AuditingSettingsDecoder.scala index 4c06893350..15bd689760 100644 --- a/core/src/main/scala/tech/beshu/ror/accesscontrol/factory/decoders/AuditingSettingsDecoder.scala +++ b/core/src/main/scala/tech/beshu/ror/accesscontrol/factory/decoders/AuditingSettingsDecoder.scala @@ -17,22 +17,26 @@ package tech.beshu.ror.accesscontrol.factory.decoders import cats.data.NonEmptyList -import io.circe.{Decoder, HCursor} +import io.circe.Decoder.* +import io.circe.{Decoder, DecodingFailure, Json, HCursor, KeyDecoder} import io.lemonlabs.uri.Uri import org.apache.logging.log4j.scala.Logging import tech.beshu.ror.accesscontrol.audit.AuditingTool import tech.beshu.ror.accesscontrol.audit.AuditingTool.AuditSettings.AuditSink import tech.beshu.ror.accesscontrol.audit.AuditingTool.AuditSettings.AuditSink.Config import tech.beshu.ror.accesscontrol.audit.AuditingTool.AuditSettings.AuditSink.Config.{EsDataStreamBasedSink, EsIndexBasedSink, LogBasedSink} +import tech.beshu.ror.accesscontrol.audit.configurable.{AuditFieldValueDescriptorParser, ConfigurableAuditLogSerializer} import tech.beshu.ror.accesscontrol.domain.RorAuditIndexTemplate.CreationError import tech.beshu.ror.accesscontrol.domain.{AuditCluster, RorAuditDataStream, RorAuditIndexTemplate, RorAuditLoggerName} import tech.beshu.ror.accesscontrol.factory.RawRorConfigBasedCoreFactory.CoreCreationError.Reason.Message import tech.beshu.ror.accesscontrol.factory.RawRorConfigBasedCoreFactory.CoreCreationError.{AuditingSettingsCreationError, Reason} import tech.beshu.ror.accesscontrol.factory.decoders.common.{lemonLabsUriDecoder, nonEmptyStringDecoder} -import tech.beshu.ror.accesscontrol.utils.CirceOps.DecodingFailureOps +import tech.beshu.ror.accesscontrol.utils.CirceOps.{AclCreationErrorCoders, DecodingFailureOps} import tech.beshu.ror.accesscontrol.utils.SyncDecoderCreator +import tech.beshu.ror.audit.AuditResponseContext.Verbosity +import tech.beshu.ror.audit.utils.AuditSerializationHelper.{AllowedEventMode, AuditFieldName, AuditFieldValueDescriptor} import tech.beshu.ror.audit.adapters.* -import tech.beshu.ror.audit.{AuditEnvironmentContext, AuditLogSerializer} +import tech.beshu.ror.audit.AuditLogSerializer import tech.beshu.ror.es.EsVersion import tech.beshu.ror.implicits.* import tech.beshu.ror.utils.yaml.YamlKeyDecoder @@ -42,42 +46,40 @@ import scala.util.{Failure, Success, Try} object AuditingSettingsDecoder extends Logging { - def instance(esVersion: EsVersion) - (implicit context: AuditEnvironmentContext): Decoder[Option[AuditingTool.AuditSettings]] = { + def instance(esVersion: EsVersion): Decoder[Option[AuditingTool.AuditSettings]] = { for { auditSettings <- auditSettingsDecoder(esVersion) deprecatedAuditSettings <- DeprecatedAuditSettingsDecoder.instance } yield auditSettings.orElse(deprecatedAuditSettings) } - private def auditSettingsDecoder(esVersion: EsVersion) - (using AuditEnvironmentContext): Decoder[Option[AuditingTool.AuditSettings]] = Decoder.instance { c => + private def auditSettingsDecoder(esVersion: EsVersion): Decoder[Option[AuditingTool.AuditSettings]] = Decoder.instance { c => for { isAuditEnabled <- YamlKeyDecoder[Boolean]( segments = NonEmptyList.of("audit", "enabled"), default = false ).apply(c) result <- if (isAuditEnabled) { - decodeAuditSettings(using esVersion, summon[AuditEnvironmentContext])(c).map(Some.apply) + decodeAuditSettings(using esVersion)(c).map(Some.apply) } else { Right(None) } } yield result } - private def decodeAuditSettings(using EsVersion, AuditEnvironmentContext) = { - decodeAuditSettingsWith(using auditSinkConfigSimpleDecoder, summon[AuditEnvironmentContext]) + private def decodeAuditSettings(using EsVersion) = { + decodeAuditSettingsWith(using auditSinkConfigSimpleDecoder) .handleErrorWith { error => if (error.aclCreationError.isDefined) { // the schema was valid, but the config not Decoder.failed(error) } else { - decodeAuditSettingsWith(using auditSinkConfigExtendedDecoder, summon[AuditEnvironmentContext]) + decodeAuditSettingsWith(using auditSinkConfigExtendedDecoder) } } } - private def decodeAuditSettingsWith(using Decoder[AuditSink], AuditEnvironmentContext) = { + private def decodeAuditSettingsWith(using Decoder[AuditSink]) = { SyncDecoderCreator .instance { _.downField("audit").downField("outputs").as[Option[List[AuditSink]]] @@ -96,7 +98,7 @@ object AuditingSettingsDecoder extends Logging { .decoder } - private def auditSinkConfigSimpleDecoder(using EsVersion, AuditEnvironmentContext): Decoder[AuditSink] = { + private def auditSinkConfigSimpleDecoder(using EsVersion): Decoder[AuditSink] = { Decoder[AuditSinkType] .emap[AuditSink.Config] { case AuditSinkType.DataStream => @@ -109,7 +111,7 @@ object AuditingSettingsDecoder extends Logging { .map(AuditSink.Enabled.apply) } - private def auditSinkConfigExtendedDecoder(using EsVersion, AuditEnvironmentContext): Decoder[AuditSink] = { + private def auditSinkConfigExtendedDecoder(using EsVersion): Decoder[AuditSink] = { given Decoder[RorAuditLoggerName] = { SyncDecoderCreator .from(nonEmptyStringDecoder) @@ -120,10 +122,10 @@ object AuditingSettingsDecoder extends Logging { given Decoder[LogBasedSink] = Decoder.instance { c => for { - logSerializerCreator <- c.downField("serializer").as[Option[AuditLogSerializer]] + logSerializer <- c.as[Option[AuditLogSerializer]] loggerName <- c.downField("logger_name").as[Option[RorAuditLoggerName]] } yield LogBasedSink( - logSerializer = logSerializerCreator.getOrElse(LogBasedSink.default.logSerializer), + logSerializer = logSerializer.getOrElse(LogBasedSink.default.logSerializer), loggerName = loggerName.getOrElse(LogBasedSink.default.loggerName) ) } @@ -131,10 +133,10 @@ object AuditingSettingsDecoder extends Logging { given Decoder[EsIndexBasedSink] = Decoder.instance { c => for { auditIndexTemplate <- c.downField("index_template").as[Option[RorAuditIndexTemplate]] - customAuditSerializer <- c.downField("serializer").as[Option[AuditLogSerializer]] + logSerializer <- c.as[Option[AuditLogSerializer]] remoteAuditCluster <- c.downField("cluster").as[Option[AuditCluster.RemoteAuditCluster]] } yield EsIndexBasedSink( - customAuditSerializer.getOrElse(EsIndexBasedSink.default.logSerializer), + logSerializer.getOrElse(EsIndexBasedSink.default.logSerializer), auditIndexTemplate.getOrElse(EsIndexBasedSink.default.rorAuditIndexTemplate), remoteAuditCluster.getOrElse(EsIndexBasedSink.default.auditCluster), ) @@ -143,10 +145,10 @@ object AuditingSettingsDecoder extends Logging { given Decoder[EsDataStreamBasedSink] = Decoder.instance { c => for { rorAuditDataStream <- c.downField("data_stream").as[Option[RorAuditDataStream]] - customAuditSerializer <- c.downField("serializer").as[Option[AuditLogSerializer]] + logSerializer <- c.as[Option[AuditLogSerializer]] remoteAuditCluster <- c.downField("cluster").as[Option[AuditCluster.RemoteAuditCluster]] } yield EsDataStreamBasedSink( - customAuditSerializer.getOrElse(EsDataStreamBasedSink.default.logSerializer), + logSerializer.getOrElse(EsDataStreamBasedSink.default.logSerializer), rorAuditDataStream.getOrElse(EsDataStreamBasedSink.default.rorAuditDataStream), remoteAuditCluster.getOrElse(EsDataStreamBasedSink.default.auditCluster), ) @@ -205,41 +207,157 @@ object AuditingSettingsDecoder extends Logging { } .decoder + given auditLogSerializerDecoder: Decoder[Option[AuditLogSerializer]] = Decoder.instance { c => + for { + serializerTypeStr <- c.as[SerializerType] + result <- serializerTypeStr match { + case SerializerType.SimpleSyntaxStaticSerializer => + c.as[Option[AuditLogSerializer]](simpleSyntaxSerializerDecoder) + case SerializerType.ExtendedSyntaxStaticSerializer => + c.downField("serializer").as[Option[AuditLogSerializer]](extendedSyntaxStaticSerializerDecoder) + case SerializerType.ExtendedSyntaxConfigurableSerializer => + c.downField("serializer").as[Option[AuditLogSerializer]](extendedSyntaxConfigurableSerializerDecoder) + } + } yield result + } + + private def extendedSyntaxConfigurableSerializerDecoder: Decoder[Option[AuditLogSerializer]] = Decoder.instance { c => + for { + allowedEventMode <- c.downField("verbosity_level_serialization_mode").as[AllowedEventMode] + .left.map(withAuditingSettingsCreationErrorMessage(msg => s"Configurable serializer is used, but the 'verbosity_level_serialization_mode' setting is invalid: $msg")) + fields <- c.downField("fields").as[Map[AuditFieldName, AuditFieldValueDescriptor]] + .left.map(withAuditingSettingsCreationErrorMessage(msg => s"Configurable serializer is used, but the 'fields' setting is missing or invalid: $msg")) + serializer = new ConfigurableAuditLogSerializer(allowedEventMode, fields) + } yield Some(serializer) + } + + private def extendedSyntaxStaticSerializerDecoder: Decoder[Option[AuditLogSerializer]] = Decoder.instance { c => + for { + fullClassNameOpt <- c.downField("class_name").as[Option[String]] + serializerOpt <- fullClassNameOpt match { + case Some(fullClassName) => serializerByClassName(fullClassName) + case None => Right(None) + } + } yield serializerOpt + } + + private def simpleSyntaxSerializerDecoder: Decoder[Option[AuditLogSerializer]] = Decoder.instance { c => + for { + fullClassNameOpt <- c.downField("serializer").as[Option[String]] + legacyFullClassNameOpt <- c.downField("audit_serializer").as[Option[String]] + serializerOpt <- fullClassNameOpt.orElse(legacyFullClassNameOpt) match { + case Some(fullClassName) => serializerByClassName(fullClassName) + case None => Right(None) + } + } yield serializerOpt + } + + private def serializerByClassName(className: String): Either[DecodingFailure, Some[AuditLogSerializer]] = { + createSerializerInstanceFromClassName(className).map(Some(_)) + .left.map(error => DecodingFailure(AclCreationErrorCoders.stringify(error), Nil)) + } + + private given serializerTypeDecoder: Decoder[SerializerType] = Decoder.instance { c => + c.downField("serializer").as[Option[Json]].flatMap { + case Some(json) if json.isObject => + json.hcursor.downField("type").as[String].map(_.toLowerCase).flatMap { + case "static" => + Right(SerializerType.ExtendedSyntaxStaticSerializer) + case "configurable" => + Right(SerializerType.ExtendedSyntaxConfigurableSerializer) + case other => + Left(DecodingFailure(AclCreationErrorCoders.stringify( + AuditingSettingsCreationError(Message(s"Invalid serializer type '$other', allowed values [static, configurable]")) + ), Nil)) + } + case Some(_) | None => + Right(SerializerType.SimpleSyntaxStaticSerializer) + } + } + + private sealed trait SerializerType + + private object SerializerType { + case object SimpleSyntaxStaticSerializer extends SerializerType + + case object ExtendedSyntaxStaticSerializer extends SerializerType + + case object ExtendedSyntaxConfigurableSerializer extends SerializerType + } + + private def withAuditingSettingsCreationErrorMessage(message: String => String)(decodingFailure: DecodingFailure) = { + decodingFailure.withMessage(AclCreationErrorCoders.stringify(AuditingSettingsCreationError(Message(message(decodingFailure.message))))) + } + @nowarn("cat=deprecation") - private given customAuditLogSerializer(using AuditEnvironmentContext): Decoder[AuditLogSerializer] = + private def createSerializerInstanceFromClassName(fullClassName: String): Either[AuditingSettingsCreationError, AuditLogSerializer] = { + val clazz = Try(Class.forName(fullClassName)).fold( + { + case _: ClassNotFoundException => throw new IllegalStateException(s"Serializer with class name $fullClassName not found.") + case other => throw other + }, + identity + ) + + def createInstanceOfSimpleSerializer(): Try[Any] = + Try(clazz.getDeclaredConstructor()).map(_.newInstance()) + + val serializer = createInstanceOfSimpleSerializer().getOrElse( + throw new IllegalStateException( + s"Class ${clazz.getName} is required to have either one (AuditEnvironmentContext) parameter constructor or constructor without parameters" + ) + ) + + Try { + serializer match { + case serializer: tech.beshu.ror.audit.AuditLogSerializer => + Some(serializer) + case serializer: tech.beshu.ror.audit.EnvironmentAwareAuditLogSerializer => + Some(new EnvironmentAwareAuditLogSerializerAdapter(serializer)) + case serializer: tech.beshu.ror.requestcontext.AuditLogSerializer[_] => + Some(new DeprecatedAuditLogSerializerAdapter(serializer)) + case _ => None + } + } match { + case Success(Some(customSerializer)) => + logger.info(s"Using custom serializer: ${customSerializer.getClass.getName}") + Right(customSerializer) + case Success(None) => Left(AuditingSettingsCreationError(Message(s"Class ${fullClassName.show} is not a subclass of ${classOf[AuditLogSerializer].getName.show} or ${classOf[tech.beshu.ror.requestcontext.AuditLogSerializer[_]].getName.show}"))) + case Failure(ex) => Left(AuditingSettingsCreationError(Message(s"Cannot create instance of class '${fullClassName.show}', error: ${ex.getMessage.show}"))) + } + } + + given allowedEventModeDecoder: Decoder[AllowedEventMode] = { + SyncDecoderCreator + .from(Decoder[Option[Set[Verbosity]]]) + .map[AllowedEventMode] { + case Some(verbosityLevels) => AllowedEventMode.Include(verbosityLevels) + case None => AllowedEventMode.IncludeAll + } + .decoder + } + + given auditFieldNameDecoder: KeyDecoder[AuditFieldName] = { + KeyDecoder.decodeKeyString.map(AuditFieldName.apply) + } + + given auditFieldValueDecoder: Decoder[AuditFieldValueDescriptor] = { SyncDecoderCreator .from(Decoder.decodeString) - .emapE { fullClassName => - val clazz = Class.forName(fullClassName) - val serializer = - Try(clazz.getConstructor(classOf[AuditEnvironmentContext])) - .map(_.newInstance(summon[AuditEnvironmentContext])) - .orElse(Try(clazz.getDeclaredConstructor()).map(_.newInstance())) - .getOrElse( - throw new IllegalStateException( - s"Class ${Class.forName(fullClassName).getName} is required to have either one (AuditEnvironmentContext) parameter constructor or constructor without parameters" - ) - ) + .emap(AuditFieldValueDescriptorParser.parse) + .decoder + } - Try { - serializer match { - case serializer: tech.beshu.ror.audit.AuditLogSerializer => - Some(serializer) - case serializer: tech.beshu.ror.audit.EnvironmentAwareAuditLogSerializer => - Some(new EnvironmentAwareAuditLogSerializerAdapter(serializer, summon[AuditEnvironmentContext])) - case serializer: tech.beshu.ror.requestcontext.AuditLogSerializer[_] => - Some(new DeprecatedAuditLogSerializerAdapter(serializer)) - case _ => None - } - } match { - case Success(Some(customSerializer)) => - logger.info(s"Using custom serializer: ${customSerializer.getClass.getName}") - Right(customSerializer) - case Success(None) => Left(AuditingSettingsCreationError(Message(s"Class ${fullClassName.show} is not a subclass of ${classOf[AuditLogSerializer].getName.show} or ${classOf[tech.beshu.ror.requestcontext.AuditLogSerializer[_]].getName.show}"))) - case Failure(ex) => Left(AuditingSettingsCreationError(Message(s"Cannot create instance of class '${fullClassName.show}', error: ${ex.getMessage.show}"))) - } + given verbosityDecoder: Decoder[Verbosity] = { + SyncDecoderCreator + .from(Decoder.decodeString) + .emap { + case "ERROR" => Right(Verbosity.Error: Verbosity) + case "INFO" => Right(Verbosity.Info: Verbosity) + case other => Left(s"Unknown verbosity level [$other], allowed values are: [ERROR, INFO]") } .decoder + } private given Decoder[AuditCluster.RemoteAuditCluster] = SyncDecoderCreator @@ -300,17 +418,19 @@ object AuditingSettingsDecoder extends Logging { } private object DeprecatedAuditSettingsDecoder { - def instance(using AuditEnvironmentContext): Decoder[Option[AuditingTool.AuditSettings]] = Decoder.instance { c => + def instance: Decoder[Option[AuditingTool.AuditSettings]] = Decoder.instance { c => whenEnabled(c) { for { auditIndexTemplate <- decodeOptionalSetting[RorAuditIndexTemplate](c)("index_template", fallbackKey = "audit_index_template") - customAuditSerializer <- decodeOptionalSetting[AuditLogSerializer](c)("serializer", fallbackKey = "audit_serializer") + logSerializerOutsideAuditSection <- c.as[Option[AuditLogSerializer]] + logSerializerInAuditSection <- c.downField("audit").success.map(_.as[Option[AuditLogSerializer]]).getOrElse(Right(None)) + logSerializer = logSerializerOutsideAuditSection.orElse(logSerializerInAuditSection) remoteAuditCluster <- decodeOptionalSetting[AuditCluster.RemoteAuditCluster](c)("cluster", fallbackKey = "audit_cluster") } yield AuditingTool.AuditSettings( auditSinks = NonEmptyList.one( AuditSink.Enabled( EsIndexBasedSink( - logSerializer = customAuditSerializer.getOrElse(EsIndexBasedSink.default.logSerializer), + logSerializer = logSerializer.getOrElse(EsIndexBasedSink.default.logSerializer), rorAuditIndexTemplate = auditIndexTemplate.getOrElse(EsIndexBasedSink.default.rorAuditIndexTemplate), auditCluster = remoteAuditCluster.getOrElse(EsIndexBasedSink.default.auditCluster), ) @@ -335,4 +455,5 @@ object AuditingSettingsDecoder extends Logging { } } } + } diff --git a/core/src/main/scala/tech/beshu/ror/accesscontrol/logging/AccessControlListLoggingDecorator.scala b/core/src/main/scala/tech/beshu/ror/accesscontrol/logging/AccessControlListLoggingDecorator.scala index 4d8c8442f4..57d9c89c2f 100644 --- a/core/src/main/scala/tech/beshu/ror/accesscontrol/logging/AccessControlListLoggingDecorator.scala +++ b/core/src/main/scala/tech/beshu/ror/accesscontrol/logging/AccessControlListLoggingDecorator.scala @@ -30,6 +30,7 @@ import tech.beshu.ror.accesscontrol.blocks.{Block, BlockContext, BlockContextUpd import tech.beshu.ror.accesscontrol.domain.Header import tech.beshu.ror.accesscontrol.logging.ResponseContext.* import tech.beshu.ror.accesscontrol.request.RequestContext +import tech.beshu.ror.audit.AuditEnvironmentContext import tech.beshu.ror.constants import tech.beshu.ror.implicits.* import tech.beshu.ror.utils.TaskOps.* @@ -39,6 +40,7 @@ import scala.util.{Failure, Success} class AccessControlListLoggingDecorator(val underlying: AccessControlList, auditingTool: Option[AuditingTool]) (implicit loggingContext: LoggingContext, + auditEnvironmentContext: AuditEnvironmentContext, scheduler: Scheduler) extends AccessControlList with Logging { @@ -105,7 +107,7 @@ class AccessControlListLoggingDecorator(val underlying: AccessControlList, } auditingTool.foreach { _ - .audit(responseContext) + .audit(responseContext, auditEnvironmentContext) .runAsync { case Right(_) => case Left(ex) => diff --git a/core/src/main/scala/tech/beshu/ror/boot/ReadonlyRest.scala b/core/src/main/scala/tech/beshu/ror/boot/ReadonlyRest.scala index db757d790b..4032538787 100644 --- a/core/src/main/scala/tech/beshu/ror/boot/ReadonlyRest.scala +++ b/core/src/main/scala/tech/beshu/ror/boot/ReadonlyRest.scala @@ -21,7 +21,7 @@ import monix.eval.Task import monix.execution.Scheduler import org.apache.logging.log4j.scala.Logging import tech.beshu.ror.accesscontrol.audit.sink.AuditSinkServiceCreator -import tech.beshu.ror.accesscontrol.audit.{AuditingTool, LoggingContext} +import tech.beshu.ror.accesscontrol.audit.{AuditEnvironmentContextBasedOnEsNodeSettings, AuditingTool, LoggingContext} import tech.beshu.ror.accesscontrol.blocks.definitions.ldap.implementations.UnboundidLdapConnectionPoolProvider import tech.beshu.ror.accesscontrol.blocks.mocks.{AuthServicesMocks, MutableMocksProviderWithCachePerRequest} import tech.beshu.ror.accesscontrol.domain.RorConfigurationIndex @@ -31,6 +31,7 @@ import tech.beshu.ror.accesscontrol.factory.RawRorConfigBasedCoreFactory.CoreCre import tech.beshu.ror.accesscontrol.factory.RawRorConfigBasedCoreFactory.CoreCreationError.Reason.Message import tech.beshu.ror.accesscontrol.factory.{AsyncHttpClientsFactory, Core, CoreFactory, RawRorConfigBasedCoreFactory} import tech.beshu.ror.accesscontrol.logging.AccessControlListLoggingDecorator +import tech.beshu.ror.audit.AuditEnvironmentContext import tech.beshu.ror.boot.ReadonlyRest.* import tech.beshu.ror.configuration.* import tech.beshu.ror.configuration.ConfigLoading.{ErrorOr, LoadRorConfig} @@ -233,6 +234,7 @@ class ReadonlyRest(coreFactory: CoreFactory, ldapConnectionPoolProvider: UnboundidLdapConnectionPoolProvider, core: Core): EitherT[Task, NonEmptyList[CoreCreationError], Engine] = { implicit val loggingContext: LoggingContext = LoggingContext(core.accessControl.staticContext.obfuscatedHeaders) + implicit val auditEnvironmentContext: AuditEnvironmentContext = new AuditEnvironmentContextBasedOnEsNodeSettings(esEnv.esNodeSettings) EitherT(createAuditingTool(core)) .map { auditingTool => val decoratedCore = Core( @@ -329,7 +331,7 @@ object ReadonlyRest { env: EsEnv) (implicit scheduler: Scheduler, environmentConfig: EnvironmentConfig): ReadonlyRest = { - val coreFactory: CoreFactory = new RawRorConfigBasedCoreFactory(env.esVersion, env.esNodeSettings) + val coreFactory: CoreFactory = new RawRorConfigBasedCoreFactory(env.esVersion) create(coreFactory, indexContentService, auditSinkServiceCreator, env) } diff --git a/core/src/test/scala/tech/beshu/ror/integration/AuditOutputFormatTests.scala b/core/src/test/scala/tech/beshu/ror/integration/AuditOutputFormatTests.scala index 0e01a0ac98..58fec29d31 100644 --- a/core/src/test/scala/tech/beshu/ror/integration/AuditOutputFormatTests.scala +++ b/core/src/test/scala/tech/beshu/ror/integration/AuditOutputFormatTests.scala @@ -29,7 +29,8 @@ import tech.beshu.ror.accesscontrol.audit.sink.{AuditDataStreamCreator, DataStre import tech.beshu.ror.accesscontrol.audit.{AuditingTool, LoggingContext} import tech.beshu.ror.accesscontrol.domain.* import tech.beshu.ror.accesscontrol.logging.AccessControlListLoggingDecorator -import tech.beshu.ror.audit.instances.DefaultAuditLogSerializer +import tech.beshu.ror.audit.AuditEnvironmentContext +import tech.beshu.ror.audit.instances.BlockVerbosityAwareAuditLogSerializer import tech.beshu.ror.es.{DataStreamBasedAuditSinkService, DataStreamService, IndexBasedAuditSinkService} import tech.beshu.ror.mocks.MockRequestContext import tech.beshu.ror.syntax.* @@ -157,15 +158,16 @@ class AuditOutputFormatTests extends AnyWordSpec with BaseYamlLoadedAccessContro private def auditedAcl(indexBasedAuditSinkService: IndexBasedAuditSinkService, dataStreamBasedAuditSinkService: DataStreamBasedAuditSinkService) = { implicit val loggingContext: LoggingContext = LoggingContext(Set.empty) + implicit val auditEnvironmentContext: AuditEnvironmentContext = testAuditEnvironmentContext val settings = AuditingTool.AuditSettings( NonEmptyList.of( AuditSink.Enabled(Config.EsIndexBasedSink( - new DefaultAuditLogSerializer(testAuditEnvironmentContext), + new BlockVerbosityAwareAuditLogSerializer, RorAuditIndexTemplate.default, AuditCluster.LocalAuditCluster )), AuditSink.Enabled(Config.EsDataStreamBasedSink( - new DefaultAuditLogSerializer(testAuditEnvironmentContext), + new BlockVerbosityAwareAuditLogSerializer, RorAuditDataStream.default, AuditCluster.LocalAuditCluster )) diff --git a/core/src/test/scala/tech/beshu/ror/integration/BaseYamlLoadedAccessControlTest.scala b/core/src/test/scala/tech/beshu/ror/integration/BaseYamlLoadedAccessControlTest.scala index d72436a940..73b770df4a 100644 --- a/core/src/test/scala/tech/beshu/ror/integration/BaseYamlLoadedAccessControlTest.scala +++ b/core/src/test/scala/tech/beshu/ror/integration/BaseYamlLoadedAccessControlTest.scala @@ -27,7 +27,7 @@ import tech.beshu.ror.configuration.{EnvironmentConfig, RawRorConfig} import tech.beshu.ror.mocks.{MockHttpClientsFactory, MockLdapConnectionPoolProvider} import tech.beshu.ror.providers.* import tech.beshu.ror.utils.TestsPropertiesProvider -import tech.beshu.ror.utils.TestsUtils.{BlockContextAssertion, defaultEsVersionForTests, testEsNodeSettings, unsafeNes} +import tech.beshu.ror.utils.TestsUtils.{BlockContextAssertion, defaultEsVersionForTests, unsafeNes} trait BaseYamlLoadedAccessControlTest extends BlockContextAssertion { @@ -41,7 +41,7 @@ trait BaseYamlLoadedAccessControlTest extends BlockContextAssertion { envVarsProvider = envVarsProvider, propertiesProvider = propertiesProvider ) - private val factory = new RawRorConfigBasedCoreFactory(defaultEsVersionForTests, testEsNodeSettings) + private val factory = new RawRorConfigBasedCoreFactory(defaultEsVersionForTests) protected val ldapConnectionPoolProvider: UnboundidLdapConnectionPoolProvider = MockLdapConnectionPoolProvider protected val httpClientsFactory: HttpClientsFactory = MockHttpClientsFactory protected val mockProvider: MocksProvider = NoOpMocksProvider diff --git a/core/src/test/scala/tech/beshu/ror/unit/acl/factory/AuditSettingsTests.scala b/core/src/test/scala/tech/beshu/ror/unit/acl/factory/AuditSettingsTests.scala index 6b50ef3b6f..61f35fd39c 100644 --- a/core/src/test/scala/tech/beshu/ror/unit/acl/factory/AuditSettingsTests.scala +++ b/core/src/test/scala/tech/beshu/ror/unit/acl/factory/AuditSettingsTests.scala @@ -26,6 +26,7 @@ import org.scalatest.matchers.should.Matchers.* import org.scalatest.wordspec.AnyWordSpec import tech.beshu.ror.accesscontrol.audit.AuditingTool.AuditSettings.AuditSink import tech.beshu.ror.accesscontrol.audit.AuditingTool.AuditSettings.AuditSink.Config +import tech.beshu.ror.accesscontrol.audit.configurable.ConfigurableAuditLogSerializer import tech.beshu.ror.accesscontrol.blocks.mocks.NoOpMocksProvider import tech.beshu.ror.accesscontrol.domain.AuditCluster.{LocalAuditCluster, RemoteAuditCluster} import tech.beshu.ror.accesscontrol.domain.{AuditCluster, IndexName, RorAuditLoggerName, RorConfigurationIndex} @@ -33,8 +34,10 @@ import tech.beshu.ror.accesscontrol.factory.RawRorConfigBasedCoreFactory.CoreCre import tech.beshu.ror.accesscontrol.factory.RawRorConfigBasedCoreFactory.CoreCreationError.Reason.Message import tech.beshu.ror.accesscontrol.factory.{Core, RawRorConfigBasedCoreFactory} import tech.beshu.ror.audit.* +import tech.beshu.ror.audit.AuditResponseContext.Verbosity import tech.beshu.ror.audit.adapters.{DeprecatedAuditLogSerializerAdapter, EnvironmentAwareAuditLogSerializerAdapter} -import tech.beshu.ror.audit.instances.{DefaultAuditLogSerializer, QueryAuditLogSerializer} +import tech.beshu.ror.audit.instances.* +import tech.beshu.ror.audit.utils.AuditSerializationHelper.{AllowedEventMode, AuditFieldName, AuditFieldValueDescriptor} import tech.beshu.ror.configuration.{EnvironmentConfig, RawRorConfig, RorConfig} import tech.beshu.ror.es.EsVersion import tech.beshu.ror.mocks.{MockHttpClientsFactory, MockLdapConnectionPoolProvider} @@ -47,7 +50,7 @@ class AuditSettingsTests extends AnyWordSpec with Inside { private def factory(esVersion: EsVersion = defaultEsVersionForTests) = { implicit val environmentConfig: EnvironmentConfig = EnvironmentConfig.default - new RawRorConfigBasedCoreFactory(esVersion, testEsNodeSettings) + new RawRorConfigBasedCoreFactory(esVersion) } private val zonedDateTime = ZonedDateTime.of(2019, 1, 1, 0, 1, 59, 0, ZoneId.of("+1")) @@ -125,7 +128,7 @@ class AuditSettingsTests extends AnyWordSpec with Inside { | """.stripMargin) - assertIndexBasedAuditSinkSettingsPresent[DefaultAuditLogSerializer]( + assertIndexBasedAuditSinkSettingsPresent[BlockVerbosityAwareAuditLogSerializer]( config, expectedIndexName = "readonlyrest_audit-2018-12-31", expectedAuditCluster = LocalAuditCluster @@ -146,7 +149,7 @@ class AuditSettingsTests extends AnyWordSpec with Inside { | """.stripMargin) - assertIndexBasedAuditSinkSettingsPresent[DefaultAuditLogSerializer]( + assertIndexBasedAuditSinkSettingsPresent[BlockVerbosityAwareAuditLogSerializer]( config, expectedIndexName = "readonlyrest_audit-2018-12-31", expectedAuditCluster = LocalAuditCluster @@ -188,7 +191,7 @@ class AuditSettingsTests extends AnyWordSpec with Inside { enabledSink1 shouldBe a[Config.EsIndexBasedSink] val sink1Config = enabledSink1.asInstanceOf[Config.EsIndexBasedSink] sink1Config.rorAuditIndexTemplate.indexName(zonedDateTime.toInstant) should be(indexName("readonlyrest_audit-2018-12-31")) - sink1Config.logSerializer shouldBe a[DefaultAuditLogSerializer] + sink1Config.logSerializer shouldBe a[BlockVerbosityAwareAuditLogSerializer] sink1Config.auditCluster shouldBe AuditCluster.LocalAuditCluster val sink2 = auditingSettings.auditSinks.toList(1) @@ -197,7 +200,7 @@ class AuditSettingsTests extends AnyWordSpec with Inside { enabledSink2 shouldBe a[Config.LogBasedSink] val sink2Config = enabledSink2.asInstanceOf[Config.LogBasedSink] sink2Config.loggerName should be(RorAuditLoggerName("readonlyrest_audit")) - sink2Config.logSerializer shouldBe a[DefaultAuditLogSerializer] + sink2Config.logSerializer shouldBe a[BlockVerbosityAwareAuditLogSerializer] val sink3 = auditingSettings.auditSinks.toList(2) sink3 shouldBe a[AuditSink.Enabled] @@ -205,7 +208,7 @@ class AuditSettingsTests extends AnyWordSpec with Inside { enabledSink3 shouldBe a[Config.EsDataStreamBasedSink] val sink3Config = enabledSink3.asInstanceOf[Config.EsDataStreamBasedSink] sink3Config.rorAuditDataStream.dataStream should be(fullDataStreamName("readonlyrest_audit")) - sink3Config.logSerializer shouldBe a[DefaultAuditLogSerializer] + sink3Config.logSerializer shouldBe a[BlockVerbosityAwareAuditLogSerializer] sink3Config.auditCluster shouldBe AuditCluster.LocalAuditCluster } } @@ -227,7 +230,7 @@ class AuditSettingsTests extends AnyWordSpec with Inside { | """.stripMargin) - assertLogBasedAuditSinkSettingsPresent[DefaultAuditLogSerializer]( + assertLogBasedAuditSinkSettingsPresent[BlockVerbosityAwareAuditLogSerializer]( config, expectedLoggerName = "readonlyrest_audit" ) @@ -251,7 +254,7 @@ class AuditSettingsTests extends AnyWordSpec with Inside { | """.stripMargin) - assertLogBasedAuditSinkSettingsPresent[DefaultAuditLogSerializer]( + assertLogBasedAuditSinkSettingsPresent[BlockVerbosityAwareAuditLogSerializer]( config, expectedLoggerName = "readonlyrest_audit" ) @@ -298,7 +301,7 @@ class AuditSettingsTests extends AnyWordSpec with Inside { | """.stripMargin) - assertLogBasedAuditSinkSettingsPresent[DefaultAuditLogSerializer]( + assertLogBasedAuditSinkSettingsPresent[BlockVerbosityAwareAuditLogSerializer]( config, expectedLoggerName = "custom_logger" ) @@ -373,6 +376,46 @@ class AuditSettingsTests extends AnyWordSpec with Inside { expectedLoggerName = "custom_logger" ) } + "configurable serializer is set" in { + val config = rorConfigFromUnsafe( + """ + |readonlyrest: + | audit: + | enabled: true + | outputs: + | - type: log + | serializer: + | type: configurable + | verbosity_level_serialization_mode: [INFO] + | fields: + | node_name_with_static_suffix: "{ES_NODE_NAME} with suffix" + | another_field: "{ES_CLUSTER_NAME} {HTTP_METHOD}" + | tid: "{TASK_ID}" + | bytes: "{CONTENT_LENGTH_IN_BYTES}" + + | access_control_rules: + | + | - name: test_block + | type: allow + | auth_key: admin:container + | + """.stripMargin) + + assertLogBasedAuditSinkSettingsPresent[ConfigurableAuditLogSerializer]( + config, + expectedLoggerName = "readonlyrest_audit" + ) + + val configuredSerializer = serializer(config).asInstanceOf[ConfigurableAuditLogSerializer] + + configuredSerializer.allowedEventMode shouldBe AllowedEventMode.Include(Set(Verbosity.Info)) + configuredSerializer.fields shouldBe Map( + AuditFieldName("node_name_with_static_suffix") -> AuditFieldValueDescriptor.Combined(List(AuditFieldValueDescriptor.EsNodeName, AuditFieldValueDescriptor.StaticText(" with suffix"))), + AuditFieldName("another_field") -> AuditFieldValueDescriptor.Combined(List(AuditFieldValueDescriptor.EsClusterName, AuditFieldValueDescriptor.StaticText(" "), AuditFieldValueDescriptor.HttpMethod)), + AuditFieldName("tid") -> AuditFieldValueDescriptor.TaskId, + AuditFieldName("bytes") -> AuditFieldValueDescriptor.ContentLengthInBytes, + ) + } } "'index' output type defined" when { "only type is set" in { @@ -392,7 +435,7 @@ class AuditSettingsTests extends AnyWordSpec with Inside { | """.stripMargin) - assertIndexBasedAuditSinkSettingsPresent[DefaultAuditLogSerializer]( + assertIndexBasedAuditSinkSettingsPresent[BlockVerbosityAwareAuditLogSerializer]( config, expectedIndexName = "readonlyrest_audit-2018-12-31", expectedAuditCluster = LocalAuditCluster @@ -417,7 +460,7 @@ class AuditSettingsTests extends AnyWordSpec with Inside { | """.stripMargin) - assertIndexBasedAuditSinkSettingsPresent[DefaultAuditLogSerializer]( + assertIndexBasedAuditSinkSettingsPresent[BlockVerbosityAwareAuditLogSerializer]( config, expectedIndexName = "readonlyrest_audit-2018-12-31", expectedAuditCluster = LocalAuditCluster @@ -465,7 +508,7 @@ class AuditSettingsTests extends AnyWordSpec with Inside { | """.stripMargin) - assertIndexBasedAuditSinkSettingsPresent[DefaultAuditLogSerializer]( + assertIndexBasedAuditSinkSettingsPresent[BlockVerbosityAwareAuditLogSerializer]( config, expectedIndexName = "custom_template_20181231", expectedAuditCluster = LocalAuditCluster @@ -567,7 +610,7 @@ class AuditSettingsTests extends AnyWordSpec with Inside { | """.stripMargin) - assertIndexBasedAuditSinkSettingsPresent[DefaultAuditLogSerializer]( + assertIndexBasedAuditSinkSettingsPresent[BlockVerbosityAwareAuditLogSerializer]( config, expectedIndexName = "readonlyrest_audit-2018-12-31", expectedAuditCluster = RemoteAuditCluster(NonEmptyList.one(Uri.parse("1.1.1.1"))) @@ -618,7 +661,7 @@ class AuditSettingsTests extends AnyWordSpec with Inside { | """.stripMargin) - assertDataStreamAuditSinkSettingsPresent[DefaultAuditLogSerializer]( + assertDataStreamAuditSinkSettingsPresent[BlockVerbosityAwareAuditLogSerializer]( config, expectedDataStreamName = "readonlyrest_audit", expectedAuditCluster = LocalAuditCluster @@ -643,7 +686,7 @@ class AuditSettingsTests extends AnyWordSpec with Inside { | """.stripMargin) - assertDataStreamAuditSinkSettingsPresent[DefaultAuditLogSerializer]( + assertDataStreamAuditSinkSettingsPresent[BlockVerbosityAwareAuditLogSerializer]( config, expectedDataStreamName = "readonlyrest_audit", expectedAuditCluster = LocalAuditCluster @@ -691,7 +734,7 @@ class AuditSettingsTests extends AnyWordSpec with Inside { | """.stripMargin) - assertDataStreamAuditSinkSettingsPresent[DefaultAuditLogSerializer]( + assertDataStreamAuditSinkSettingsPresent[BlockVerbosityAwareAuditLogSerializer]( config, expectedDataStreamName = "custom_audit_data_stream", expectedAuditCluster = LocalAuditCluster @@ -763,7 +806,7 @@ class AuditSettingsTests extends AnyWordSpec with Inside { | """.stripMargin) - assertDataStreamAuditSinkSettingsPresent[DefaultAuditLogSerializer]( + assertDataStreamAuditSinkSettingsPresent[BlockVerbosityAwareAuditLogSerializer]( config, expectedDataStreamName = "readonlyrest_audit", expectedAuditCluster = RemoteAuditCluster(NonEmptyList.one(Uri.parse("1.1.1.1"))) @@ -875,7 +918,7 @@ class AuditSettingsTests extends AnyWordSpec with Inside { enabledSink1 shouldBe a[Config.EsIndexBasedSink] val sink1Config = enabledSink1.asInstanceOf[Config.EsIndexBasedSink] sink1Config.rorAuditIndexTemplate.indexName(zonedDateTime.toInstant) should be(indexName("readonlyrest_audit-2018-12-31")) - sink1Config.logSerializer shouldBe a[DefaultAuditLogSerializer] + sink1Config.logSerializer shouldBe a[BlockVerbosityAwareAuditLogSerializer] sink1Config.auditCluster shouldBe AuditCluster.LocalAuditCluster val sink2 = auditingSettings.auditSinks.toList(1) @@ -892,7 +935,7 @@ class AuditSettingsTests extends AnyWordSpec with Inside { enabledSink3 shouldBe a[Config.EsDataStreamBasedSink] val sink3Config = enabledSink3.asInstanceOf[Config.EsDataStreamBasedSink] sink3Config.rorAuditDataStream.dataStream should be(fullDataStreamName("readonlyrest_audit")) - sink3Config.logSerializer shouldBe a[DefaultAuditLogSerializer] + sink3Config.logSerializer shouldBe a[BlockVerbosityAwareAuditLogSerializer] sink3Config.auditCluster shouldBe AuditCluster.LocalAuditCluster } } @@ -1242,6 +1285,59 @@ class AuditSettingsTests extends AnyWordSpec with Inside { expectedErrorMessage = "The audit 'outputs' array cannot be empty" ) } + "configurable serializer is set with invalid value descriptor" in { + val config = rorConfigFromUnsafe( + """ + |readonlyrest: + | audit: + | enabled: true + | outputs: + | - type: log + | serializer: + | type: configurable + | verbosity_level_serialization_mode: [INFO] + | fields: + | node_name_with_static_suffix: "{ES_NODE_NAME} with suffix" + | another_field: "{ES_CLUSTER_NAME} {HTTP_METHOD2}" + | tid: "{TASK_ID}" + | bytes: "{CONTENT_LENGTH_IN_BYTES}" + | access_control_rules: + | + | - name: test_block + | type: allow + | auth_key: admin:container + | + """.stripMargin) + + assertInvalidSettings( + config, + expectedErrorMessage = "Configurable serializer is used, but the 'fields' setting is missing or invalid: There are invalid placeholder values: HTTP_METHOD2" + ) + } + "configurable serializer is set, but without fields setting" in { + val config = rorConfigFromUnsafe( + """ + |readonlyrest: + | audit: + | enabled: true + | outputs: + | - type: log + | serializer: + | type: configurable + | verbosity_level_serialization_mode: [INFO] + | access_control_rules: + | + | - name: test_block + | type: allow + | auth_key: admin:container + | + """.stripMargin) + + assertInvalidSettings( + config, + expectedErrorMessage = "Configurable serializer is used, but the 'fields' setting is missing or invalid: Missing required field" + ) + } } } "deprecated format is used" should { @@ -1265,7 +1361,7 @@ class AuditSettingsTests extends AnyWordSpec with Inside { | """.stripMargin) - assertIndexBasedAuditSinkSettingsPresent[DefaultAuditLogSerializer]( + assertIndexBasedAuditSinkSettingsPresent[BlockVerbosityAwareAuditLogSerializer]( config, expectedIndexName = "readonlyrest_audit-2018-12-31", expectedAuditCluster = LocalAuditCluster @@ -1325,7 +1421,7 @@ class AuditSettingsTests extends AnyWordSpec with Inside { | """.stripMargin) - assertIndexBasedAuditSinkSettingsPresent[DefaultAuditLogSerializer]( + assertIndexBasedAuditSinkSettingsPresent[BlockVerbosityAwareAuditLogSerializer]( config, expectedIndexName = "readonlyrest_audit-2018-12-31", expectedAuditCluster = LocalAuditCluster @@ -1347,7 +1443,7 @@ class AuditSettingsTests extends AnyWordSpec with Inside { | """.stripMargin) - assertIndexBasedAuditSinkSettingsPresent[DefaultAuditLogSerializer]( + assertIndexBasedAuditSinkSettingsPresent[BlockVerbosityAwareAuditLogSerializer]( config, expectedIndexName = "custom_template_20181231", expectedAuditCluster = LocalAuditCluster @@ -1413,7 +1509,7 @@ class AuditSettingsTests extends AnyWordSpec with Inside { | """.stripMargin) - assertIndexBasedAuditSinkSettingsPresent[DefaultAuditLogSerializer]( + assertIndexBasedAuditSinkSettingsPresent[BlockVerbosityAwareAuditLogSerializer]( config, expectedIndexName = "readonlyrest_audit-2018-12-31", expectedAuditCluster = RemoteAuditCluster(NonEmptyList.one(Uri.parse("1.1.1.1"))) @@ -1459,7 +1555,7 @@ class AuditSettingsTests extends AnyWordSpec with Inside { | """.stripMargin) - assertIndexBasedAuditSinkSettingsPresent[DefaultAuditLogSerializer]( + assertIndexBasedAuditSinkSettingsPresent[BlockVerbosityAwareAuditLogSerializer]( config, expectedIndexName = "readonlyrest_audit-2018-12-31", expectedAuditCluster = LocalAuditCluster @@ -1480,7 +1576,7 @@ class AuditSettingsTests extends AnyWordSpec with Inside { | """.stripMargin) - assertIndexBasedAuditSinkSettingsPresent[DefaultAuditLogSerializer]( + assertIndexBasedAuditSinkSettingsPresent[BlockVerbosityAwareAuditLogSerializer]( config, expectedIndexName = "custom_template_20181231", expectedAuditCluster = LocalAuditCluster @@ -1543,7 +1639,7 @@ class AuditSettingsTests extends AnyWordSpec with Inside { | """.stripMargin) - assertIndexBasedAuditSinkSettingsPresent[DefaultAuditLogSerializer]( + assertIndexBasedAuditSinkSettingsPresent[BlockVerbosityAwareAuditLogSerializer]( config, expectedIndexName = "readonlyrest_audit-2018-12-31", expectedAuditCluster = RemoteAuditCluster(NonEmptyList.one(Uri.parse("user:test@1.1.1.1"))) @@ -1906,7 +2002,7 @@ private object DummyAuditRequestContext extends AuditRequestContext { override def httpMethod: String = "" - override def loggedInUserName: Option[String] = None + override def loggedInUserName: Option[String] = Some("logged_user") override def impersonatedByUserName: Option[String] = None @@ -1917,4 +2013,6 @@ private object DummyAuditRequestContext extends AuditRequestContext { override def rawAuthHeader: Option[String] = None override def generalAuditEvents: JSONObject = new JSONObject + + override def auditEnvironmentContext: AuditEnvironmentContext = testAuditEnvironmentContext } diff --git a/core/src/test/scala/tech/beshu/ror/unit/acl/factory/CoreFactoryTests.scala b/core/src/test/scala/tech/beshu/ror/unit/acl/factory/CoreFactoryTests.scala index 1bfd2c5869..2982cc6393 100644 --- a/core/src/test/scala/tech/beshu/ror/unit/acl/factory/CoreFactoryTests.scala +++ b/core/src/test/scala/tech/beshu/ror/unit/acl/factory/CoreFactoryTests.scala @@ -40,7 +40,7 @@ class CoreFactoryTests extends AnyWordSpec with Inside with MockFactory { private val factory: CoreFactory = { implicit val environmentConfig: EnvironmentConfig = EnvironmentConfig.default - new RawRorConfigBasedCoreFactory(defaultEsVersionForTests, testEsNodeSettings) + new RawRorConfigBasedCoreFactory(defaultEsVersionForTests) } "A RorAclFactory" should { diff --git a/core/src/test/scala/tech/beshu/ror/unit/acl/factory/ImpersonationWarningsTests.scala b/core/src/test/scala/tech/beshu/ror/unit/acl/factory/ImpersonationWarningsTests.scala index bce32288eb..0c5d7bc7b9 100644 --- a/core/src/test/scala/tech/beshu/ror/unit/acl/factory/ImpersonationWarningsTests.scala +++ b/core/src/test/scala/tech/beshu/ror/unit/acl/factory/ImpersonationWarningsTests.scala @@ -387,6 +387,6 @@ class ImpersonationWarningsTests extends AnyWordSpec with Inside { private val factory: CoreFactory = { implicit val environmentConfig: EnvironmentConfig = EnvironmentConfig.default - new RawRorConfigBasedCoreFactory(defaultEsVersionForTests, testEsNodeSettings) + new RawRorConfigBasedCoreFactory(defaultEsVersionForTests) } } diff --git a/core/src/test/scala/tech/beshu/ror/unit/acl/factory/LocalUsersTest.scala b/core/src/test/scala/tech/beshu/ror/unit/acl/factory/LocalUsersTest.scala index c44813f69f..a02b11634b 100644 --- a/core/src/test/scala/tech/beshu/ror/unit/acl/factory/LocalUsersTest.scala +++ b/core/src/test/scala/tech/beshu/ror/unit/acl/factory/LocalUsersTest.scala @@ -319,7 +319,7 @@ class LocalUsersTest extends AnyWordSpec with Inside { private val factory = { implicit val environmentConfig: EnvironmentConfig = EnvironmentConfig.default - new RawRorConfigBasedCoreFactory(defaultEsVersionForTests, testEsNodeSettings) + new RawRorConfigBasedCoreFactory(defaultEsVersionForTests) } } diff --git a/core/src/test/scala/tech/beshu/ror/unit/acl/factory/decoders/rules/BaseRuleSettingsDecoderTest.scala b/core/src/test/scala/tech/beshu/ror/unit/acl/factory/decoders/rules/BaseRuleSettingsDecoderTest.scala index 4751f42ce2..c2332d1965 100644 --- a/core/src/test/scala/tech/beshu/ror/unit/acl/factory/decoders/rules/BaseRuleSettingsDecoderTest.scala +++ b/core/src/test/scala/tech/beshu/ror/unit/acl/factory/decoders/rules/BaseRuleSettingsDecoderTest.scala @@ -50,7 +50,7 @@ abstract class BaseRuleSettingsDecoderTest[T <: Rule : ClassTag] extends AnyWord protected def factory: RawRorConfigBasedCoreFactory = { implicit val environmentConfig: EnvironmentConfig = new EnvironmentConfig(envVarsProvider = envVarsProvider) - new RawRorConfigBasedCoreFactory(defaultEsVersionForTests, testEsNodeSettings) + new RawRorConfigBasedCoreFactory(defaultEsVersionForTests) } def assertDecodingSuccess(yaml: String, diff --git a/core/src/test/scala/tech/beshu/ror/unit/acl/logging/AuditingToolTests.scala b/core/src/test/scala/tech/beshu/ror/unit/acl/logging/AuditingToolTests.scala index 43bd90d959..1d97c650cb 100644 --- a/core/src/test/scala/tech/beshu/ror/unit/acl/logging/AuditingToolTests.scala +++ b/core/src/test/scala/tech/beshu/ror/unit/acl/logging/AuditingToolTests.scala @@ -49,6 +49,7 @@ import tech.beshu.ror.utils.TestsUtils.{fullDataStreamName, fullIndexName, nes, import java.time.* import java.util.UUID +import scala.annotation.nowarn class AuditingToolTests extends AnyWordSpec with MockFactory with BeforeAndAfterAll { @@ -61,8 +62,9 @@ class AuditingToolTests extends AnyWordSpec with MockFactory with BeforeAndAfter "es index sink is used" should { "not submit any audit entry" when { "request was allowed and verbosity level was ERROR" in { + @nowarn("cat=deprecation") val auditingTool = AuditingTool.create( - settings = auditSettings(new DefaultAuditLogSerializer(testAuditEnvironmentContext)), + settings = auditSettings(new DefaultAuditLogSerializer), auditSinkServiceCreator = new DataStreamAndIndexBasedAuditSinkServiceCreator { override def dataStream(cluster: AuditCluster): DataStreamBasedAuditSinkService = mockedDataStreamBasedAuditSinkService @@ -70,7 +72,7 @@ class AuditingToolTests extends AnyWordSpec with MockFactory with BeforeAndAfter override def index(cluster: AuditCluster): IndexBasedAuditSinkService = mock[IndexBasedAuditSinkService] } ).runSyncUnsafe().toOption.flatten.get - auditingTool.audit(createAllowedResponseContext(Policy.Allow, Verbosity.Error)).runSyncUnsafe() + auditingTool.audit(createAllowedResponseContext(Policy.Allow, Verbosity.Error), testAuditEnvironmentContext).runSyncUnsafe() } "custom serializer throws exception" in { val auditingTool = AuditingTool.create( @@ -83,7 +85,7 @@ class AuditingToolTests extends AnyWordSpec with MockFactory with BeforeAndAfter } ).runSyncUnsafe().toOption.flatten.get an[IllegalArgumentException] should be thrownBy { - auditingTool.audit(createAllowedResponseContext(Policy.Allow, Verbosity.Info)).runSyncUnsafe() + auditingTool.audit(createAllowedResponseContext(Policy.Allow, Verbosity.Info), testAuditEnvironmentContext).runSyncUnsafe() } } } @@ -93,25 +95,25 @@ class AuditingToolTests extends AnyWordSpec with MockFactory with BeforeAndAfter (indexAuditSink.submit _).expects(fullIndexName("test_2018-12-31"), "mock-1", *).returning(()) val dataStreamAuditSink = mockedDataStreamBasedAuditSinkService (dataStreamAuditSink.submit _).expects(fullDataStreamName("test_ds"), "mock-1", *).returning(()) - + @nowarn("cat=deprecation") val auditingTool = AuditingTool.create( - settings = auditSettings(new DefaultAuditLogSerializer(testAuditEnvironmentContext)), + settings = auditSettings(new DefaultAuditLogSerializer), auditSinkServiceCreator = new DataStreamAndIndexBasedAuditSinkServiceCreator { override def dataStream(cluster: AuditCluster): DataStreamBasedAuditSinkService = dataStreamAuditSink override def index(cluster: AuditCluster): IndexBasedAuditSinkService = indexAuditSink } ).runSyncUnsafe().toOption.flatten.get - auditingTool.audit(createAllowedResponseContext(Policy.Allow, Verbosity.Info)).runSyncUnsafe() + auditingTool.audit(createAllowedResponseContext(Policy.Allow, Verbosity.Info), testAuditEnvironmentContext).runSyncUnsafe() } "request was matched by forbidden rule" in { val indexAuditSink = mock[IndexBasedAuditSinkService] (indexAuditSink.submit _).expects(fullIndexName("test_2018-12-31"), "mock-1", *).returning(()) val dataStreamAuditSink = mockedDataStreamBasedAuditSinkService (dataStreamAuditSink.submit _).expects(fullDataStreamName("test_ds"), "mock-1", *).returning(()) - + @nowarn("cat=deprecation") val auditingTool = AuditingTool.create( - settings = auditSettings(new DefaultAuditLogSerializer(testAuditEnvironmentContext)), + settings = auditSettings(new DefaultAuditLogSerializer), auditSinkServiceCreator = new DataStreamAndIndexBasedAuditSinkServiceCreator { override def dataStream(cluster: AuditCluster): DataStreamBasedAuditSinkService = dataStreamAuditSink @@ -132,16 +134,16 @@ class AuditingToolTests extends AnyWordSpec with MockFactory with BeforeAndAfter Vector.empty ) - auditingTool.audit(responseContext).runSyncUnsafe() + auditingTool.audit(responseContext, testAuditEnvironmentContext).runSyncUnsafe() } "request was forbidden" in { val indexAuditSink = mock[IndexBasedAuditSinkService] (indexAuditSink.submit _).expects(fullIndexName("test_2018-12-31"), "mock-1", *).returning(()) val dataStreamAuditSink = mockedDataStreamBasedAuditSinkService (dataStreamAuditSink.submit _).expects(fullDataStreamName("test_ds"), "mock-1", *).returning(()) - + @nowarn("cat=deprecation") val auditingTool = AuditingTool.create( - settings = auditSettings(new DefaultAuditLogSerializer(testAuditEnvironmentContext)), + settings = auditSettings(new DefaultAuditLogSerializer), auditSinkServiceCreator = new DataStreamAndIndexBasedAuditSinkServiceCreator { override def dataStream(cluster: AuditCluster): DataStreamBasedAuditSinkService = dataStreamAuditSink @@ -152,16 +154,16 @@ class AuditingToolTests extends AnyWordSpec with MockFactory with BeforeAndAfter val requestContext = MockRequestContext.indices.copy(timestamp = someday.toInstant, id = RequestContext.Id.fromString("mock-1")) val responseContext = Forbidden(requestContext, Vector.empty) - auditingTool.audit(responseContext).runSyncUnsafe() + auditingTool.audit(responseContext, testAuditEnvironmentContext).runSyncUnsafe() } "request was finished with error" in { val indexAuditSink = mock[IndexBasedAuditSinkService] (indexAuditSink.submit _).expects(fullIndexName("test_2018-12-31"), "mock-1", *).returning(()) val dataStreamAuditSink = mockedDataStreamBasedAuditSinkService (dataStreamAuditSink.submit _).expects(fullDataStreamName("test_ds"), "mock-1", *).returning(()) - + @nowarn("cat=deprecation") val auditingTool = AuditingTool.create( - settings = auditSettings(new DefaultAuditLogSerializer(testAuditEnvironmentContext)), + settings = auditSettings(new DefaultAuditLogSerializer), auditSinkServiceCreator = new DataStreamAndIndexBasedAuditSinkServiceCreator { override def dataStream(cluster: AuditCluster): DataStreamBasedAuditSinkService = dataStreamAuditSink @@ -172,17 +174,18 @@ class AuditingToolTests extends AnyWordSpec with MockFactory with BeforeAndAfter val requestContext = MockRequestContext.indices.copy(timestamp = someday.toInstant, id = RequestContext.Id.fromString("mock-1")) val responseContext = Errored(requestContext, new Exception("error")) - auditingTool.audit(responseContext).runSyncUnsafe() + auditingTool.audit(responseContext, testAuditEnvironmentContext).runSyncUnsafe() } } } "log sink is used" should { "saved audit log to file defined in log4j config" in { + @nowarn("cat=deprecation") val auditingTool = AuditingTool.create( settings = AuditSettings( NonEmptyList.of( AuditSink.Enabled(Config.LogBasedSink( - new DefaultAuditLogSerializer(testAuditEnvironmentContext), + new DefaultAuditLogSerializer, RorAuditLoggerName.default )) ) @@ -200,7 +203,7 @@ class AuditingToolTests extends AnyWordSpec with MockFactory with BeforeAndAfter auditLogFile.overwrite("") - auditingTool.audit(responseContext).runSyncUnsafe() + auditingTool.audit(responseContext, testAuditEnvironmentContext).runSyncUnsafe() val logFileContent = auditLogFile.contentAsString logFileContent should include(requestContextId.value) diff --git a/core/src/test/scala/tech/beshu/ror/unit/boot/ReadonlyRestStartingTests.scala b/core/src/test/scala/tech/beshu/ror/unit/boot/ReadonlyRestStartingTests.scala index 2b588070f2..8a84192d6e 100644 --- a/core/src/test/scala/tech/beshu/ror/unit/boot/ReadonlyRestStartingTests.scala +++ b/core/src/test/scala/tech/beshu/ror/unit/boot/ReadonlyRestStartingTests.scala @@ -1315,7 +1315,7 @@ class ReadonlyRestStartingTests val mockedIndexJsonContentManager = mock[IndexJsonContentService] mockIndexJsonContentManagerSourceOfCallTestConfig(mockedIndexJsonContentManager) - val dataStreamSinkConfig1 = AuditSink.Config.EsDataStreamBasedSink.default(testAuditEnvironmentContext) + val dataStreamSinkConfig1 = AuditSink.Config.EsDataStreamBasedSink.default val dataStreamSinkConfig2 = dataStreamSinkConfig1.copy( auditCluster = AuditCluster.RemoteAuditCluster(NonEmptyList.one(Uri.parse("0.0.0.0"))) ) diff --git a/gradle.properties b/gradle.properties index 61393c34ff..152ac548a1 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,5 +1,5 @@ publishedPluginVersion=1.66.1 -pluginVersion=1.67.0-pre2 +pluginVersion=1.67.0-pre3 pluginName=readonlyrest org.gradle.jvmargs=-Xmx6144m diff --git a/integration-tests/src/test/resources/ror_audit/enabled_auditing_tools/readonlyrest.yml b/integration-tests/src/test/resources/ror_audit/enabled_auditing_tools/readonlyrest.yml index e413c7174d..0e8e383748 100644 --- a/integration-tests/src/test/resources/ror_audit/enabled_auditing_tools/readonlyrest.yml +++ b/integration-tests/src/test/resources/ror_audit/enabled_auditing_tools/readonlyrest.yml @@ -5,10 +5,26 @@ readonlyrest: outputs: - type: index index_template: "'audit_index'" - serializer: "tech.beshu.ror.audit.instances.DefaultAuditLogSerializerV1" + serializer: + type: "static" + class_name: "tech.beshu.ror.audit.instances.DefaultAuditLogSerializerV1" + verbosity_level_serialization_mode: [INFO] + fields: + node_name_with_static_suffix: "{ES_NODE_NAME} with suffix" + another_field: "{ES_CLUSTER_NAME} {HTTP_METHOD}" + tid: "{TASK_ID}" + bytes: "{CONTENT_LENGTH_IN_BYTES}" - type: data_stream data_stream: "audit_data_stream" - serializer: "tech.beshu.ror.audit.instances.DefaultAuditLogSerializerV1" + serializer: + type: "static" + class_name: "tech.beshu.ror.audit.instances.DefaultAuditLogSerializerV1" + verbosity_level_serialization_mode: [INFO] + fields: + node_name_with_static_suffix: "{ES_NODE_NAME} with suffix" + another_field: "{ES_CLUSTER_NAME} {HTTP_METHOD}" + tid: "{TASK_ID}" + bytes: "{CONTENT_LENGTH_IN_BYTES}" access_control_rules: diff --git a/integration-tests/src/test/resources/ror_audit/enabled_auditing_tools/readonlyrest_audit_index.yml b/integration-tests/src/test/resources/ror_audit/enabled_auditing_tools/readonlyrest_audit_index.yml index b9dd7d0084..a6b93beda9 100644 --- a/integration-tests/src/test/resources/ror_audit/enabled_auditing_tools/readonlyrest_audit_index.yml +++ b/integration-tests/src/test/resources/ror_audit/enabled_auditing_tools/readonlyrest_audit_index.yml @@ -5,7 +5,15 @@ readonlyrest: outputs: - type: index index_template: "'audit_index'" - serializer: "tech.beshu.ror.audit.instances.DefaultAuditLogSerializerV1" + serializer: + type: "static" + class_name: "tech.beshu.ror.audit.instances.DefaultAuditLogSerializerV1" + verbosity_level_serialization_mode: [INFO] + fields: + node_name_with_static_suffix: "{ES_NODE_NAME} with suffix" + another_field: "{ES_CLUSTER_NAME} {HTTP_METHOD}" + tid: "{TASK_ID}" + bytes: "{CONTENT_LENGTH_IN_BYTES}" access_control_rules: diff --git a/integration-tests/src/test/scala/tech/beshu/ror/integration/suites/audit/LocalClusterAuditingToolsSuite.scala b/integration-tests/src/test/scala/tech/beshu/ror/integration/suites/audit/LocalClusterAuditingToolsSuite.scala index 91f909e07c..185f8f1baa 100644 --- a/integration-tests/src/test/scala/tech/beshu/ror/integration/suites/audit/LocalClusterAuditingToolsSuite.scala +++ b/integration-tests/src/test/scala/tech/beshu/ror/integration/suites/audit/LocalClusterAuditingToolsSuite.scala @@ -73,7 +73,6 @@ class LocalClusterAuditingToolsSuite eventually { val auditEntries = adminAuditManager.getEntries.force().jsons auditEntries.size shouldBe 2 - auditEntries.exists(entry => entry("final_state").str == "ALLOWED" && entry("user").str == "username" && @@ -81,7 +80,6 @@ class LocalClusterAuditingToolsSuite entry.obj.get("es_node_name").isEmpty && entry.obj.get("es_cluster_name").isEmpty ) shouldBe true - auditEntries.exists(entry => entry("final_state").str == "ALLOWED" && entry("user").str == "username" && @@ -91,6 +89,86 @@ class LocalClusterAuditingToolsSuite ) shouldBe true } } + updateRorConfigToUseSerializer("tech.beshu.ror.audit.instances.DefaultAuditLogSerializerV1") + } + "using ReportingAllEventsAuditLogSerializer" in { + val indexManager = new IndexManager(basicAuthClient("username", "dev"), esVersionUsed) + + updateRorConfigToUseSerializer("tech.beshu.ror.audit.instances.FullAuditLogSerializer") + performAndAssertExampleSearchRequest(indexManager) + + forEachAuditManager { adminAuditManager => + eventually { + val auditEntries = adminAuditManager.getEntries.force().jsons + assert(auditEntries.size >= 3) + + auditEntries.exists(entry => + entry("final_state").str == "ALLOWED" && + entry("user").str == "username" && + entry("block").str.contains("name: 'Rule 1'") && + Try(entry("es_node_name")).map(_.str) == Success("ROR_SINGLE_1") && + Try(entry("es_cluster_name")).map(_.str) == Success("ROR_SINGLE") && + entry.obj.get("content").isEmpty + ) shouldBe true + + auditEntries.exists(entry => entry("path").str == "/_readonlyrest/admin/refreshconfig/") shouldBe true + auditEntries.exists(entry => entry("path").str == "/audit_index/_search/") shouldBe true + } + } + updateRorConfigToUseSerializer("tech.beshu.ror.audit.instances.DefaultAuditLogSerializerV1") + // This test uses serializer, that reports all events. We need to wait a moment, to ensure that there will be no more events using that serializer + Thread.sleep(3000) + } + "using ReportingAllEventsWithQueryAuditLogSerializer" in { + val indexManager = new IndexManager(basicAuthClient("username", "dev"), esVersionUsed) + + updateRorConfigToUseSerializer("tech.beshu.ror.audit.instances.FullAuditLogWithQuerySerializer") + performAndAssertExampleSearchRequest(indexManager) + + forEachAuditManager { adminAuditManager => + eventually { + val auditEntries = adminAuditManager.getEntries.force().jsons + assert(auditEntries.size >= 3) + auditEntries.exists(entry => + entry("final_state").str == "ALLOWED" && + entry("user").str == "username" && + entry("block").str.contains("name: 'Rule 1'") && + Try(entry("es_node_name")).map(_.str) == Success("ROR_SINGLE_1") && + Try(entry("es_cluster_name")).map(_.str) == Success("ROR_SINGLE") && + Try(entry("content")).map(_.str) == Success("") + ) shouldBe true + + auditEntries.exists(entry => entry("path").str == "/_readonlyrest/admin/refreshconfig/") shouldBe true + auditEntries.exists(entry => entry("path").str == "/audit_index/_search/") shouldBe true + } + } + updateRorConfigToUseSerializer("tech.beshu.ror.audit.instances.DefaultAuditLogSerializerV1") + // This test uses serializer, that reports all events. We need to wait a moment, to ensure that there will be no more events using that serializer + Thread.sleep(3000) + } + "using ConfigurableQueryAuditLogSerializer" in { + val indexManager = new IndexManager(basicAuthClient("username", "dev"), esVersionUsed) + + updateRorConfig( + originalString = """type: "static"""", + newString = """type: "configurable"""", + ) + performAndAssertExampleSearchRequest(indexManager) + + forEachAuditManager { adminAuditManager => + eventually { + val auditEntries = adminAuditManager.getEntries.force().jsons + auditEntries.size shouldBe 1 + + auditEntries.exists(entry => + entry("node_name_with_static_suffix").str == "ROR_SINGLE_1 with suffix" && + entry("another_field").str == "ROR_SINGLE GET" && + entry("tid").numOpt.isDefined && + entry("bytes").num == 0 + ) shouldBe true + } + } + updateRorConfigToUseSerializer("tech.beshu.ror.audit.instances.DefaultAuditLogSerializerV1") } } } @@ -100,11 +178,15 @@ class LocalClusterAuditingToolsSuite response should have statusCode 200 } - private def updateRorConfigToUseSerializer(serializer: String) = { + private def updateRorConfigToUseSerializer(serializer: String) = updateRorConfig( + originalString = """class_name: "tech.beshu.ror.audit.instances.DefaultAuditLogSerializerV1"""", + newString = s"""class_name: "$serializer"""" + ) + + private def updateRorConfig(originalString: String, newString: String) = { val initialConfig = getResourceContent(rorConfigFileName) - val serializerUsedInOriginalConfigFile = "tech.beshu.ror.audit.instances.DefaultAuditLogSerializerV1" - val firstModifiedConfig = initialConfig.replace(serializerUsedInOriginalConfigFile, serializer) - rorApiManager.updateRorInIndexConfig(firstModifiedConfig).forceOKStatusOrConfigAlreadyLoaded() + val modifiedConfig = initialConfig.replace(originalString, newString) + rorApiManager.updateRorInIndexConfig(modifiedConfig).forceOKStatusOrConfigAlreadyLoaded() rorApiManager.reloadRorConfig().force() } }