From 05da8448eed0232562e9ca7faeee3e06b336944b Mon Sep 17 00:00:00 2001 From: trifonovt <87468028+TihomirTrifonov@users.noreply.github.com> Date: Thu, 12 Mar 2026 12:50:51 +0100 Subject: [PATCH] MalIS-CEP initial import --- .../procon/malis/cep/MalisCepApplication.java | 14 + .../malis/cep/api/BatchIngressRequest.java | 14 + .../malis/cep/api/BatchIngressResult.java | 50 + .../malis/cep/api/IngressController.java | 194 +++ .../malis/cep/api/IngressEventRequest.java | 56 + .../cep/api/IngressExceptionHandler.java | 22 + .../procon/malis/cep/api/IngressResult.java | 74 + .../malis/cep/binding/BindingFileLoader.java | 80 ++ .../malis/cep/binding/BindingResolver.java | 166 +++ .../cep/binding/BindingsLoadedEvent.java | 38 + .../malis/cep/binding/IngressMessage.java | 54 + .../malis/cep/binding/RegexGroupNames.java | 21 + .../malis/cep/binding/ResolvedBinding.java | 42 + .../malis/cep/config/CepProperties.java | 602 +++++++++ .../malis/cep/config/CepStartupValidator.java | 262 ++++ .../malis/cep/detector/DetectionContext.java | 28 + .../procon/malis/cep/detector/Detector.java | 9 + .../malis/cep/detector/DetectorEvent.java | 43 + .../malis/cep/detector/DetectorFactory.java | 82 ++ .../malis/cep/detector/DetectorRegistry.java | 22 + .../cep/detector/ExpressionRulesDetector.java | 209 +++ .../detector/ExternalRestEventsDetector.java | 312 +++++ .../detector/PassthroughAlarmDetector.java | 59 + .../cep/detector/PinBadStateDetector.java | 90 ++ .../procon/malis/cep/eebus/EebusAddress.java | 35 + .../malis/cep/eebus/EebusJaxbCodec.java | 90 ++ .../cep/eebus/EebusMeasurementDatagram.java | 46 + .../malis/cep/eebus/EebusSpineConstants.java | 28 + .../malis/cep/eebus/EebusSpineReflection.java | 98 ++ .../procon/malis/cep/eebus/EebusSpineV1.java | 34 + .../cep/eebus/EebusSpineXsdJaxbValidator.java | 29 + .../cep/eebus/EebusSpineXsdValidator.java | 188 +++ .../cep/eebus/EebusSpineXsdValidatorBase.java | 5 + .../co/procon/malis/cep/eebus/EebusXml.java | 70 + .../procon/malis/cep/eebus/EebusXmlUtil.java | 133 ++ .../cep/erpnext/ErpnextFaultLookupClient.java | 1185 +++++++++++++++++ .../co/procon/malis/cep/event/CepEvent.java | 22 + .../co/procon/malis/cep/event/EventBatch.java | 47 + .../cep/event/SignalDefinitionEvent.java | 40 + .../malis/cep/event/SignalUpdateEvent.java | 35 + .../malis/cep/event/SignalValueType.java | 10 + .../co/procon/malis/cep/event/TypedValue.java | 41 + .../EebusMeasurementDatagramConverter.java | 239 ++++ .../event/convert/InputEventConverter.java | 15 + .../convert/MeasurementSetConverter.java | 54 + .../cep/lifecycle/AlarmLifecycleService.java | 656 +++++++++ .../CompositeAlarmLifecycleService.java | 954 +++++++++++++ .../lifecycle/LifecycleDispatcherService.java | 55 + .../LifecycleHeartbeatScheduler.java | 257 ++++ .../malis/cep/lifecycle/PublishableAlarm.java | 84 ++ .../malis/cep/output/AlarmFormatter.java | 21 + .../EebusAlarmDatagramXmlFormatter.java | 216 +++ .../output/EebusAlarmMessageFormatter.java | 57 + .../malis/cep/output/FormattedMessage.java | 22 + .../output/MalisCepAlarmJsonFormatter.java | 256 ++++ .../cep/output/MalisFaultJsonFormatter.java | 263 ++++ .../malis/cep/output/MqttOutputPublisher.java | 41 + .../malis/cep/output/OutboxMessage.java | 48 + .../cep/output/OutboxOutputPublisher.java | 65 + .../procon/malis/cep/output/OutboxStore.java | 94 ++ .../malis/cep/output/OutboxStoreProvider.java | 12 + .../malis/cep/output/OutputPublisher.java | 7 + .../malis/cep/output/OutputRegistry.java | 97 ++ .../malis/cep/output/PublishedMessage.java | 92 ++ .../malis/cep/output/PublisherService.java | 201 +++ .../cep/output/RabbitOutputPublisher.java | 40 + .../malis/cep/output/RestOutputPublisher.java | 186 +++ .../procon/malis/cep/parser/AlarmBatch.java | 24 + .../procon/malis/cep/parser/AlarmInput.java | 41 + .../parser/EebusAlarmDatagramXmlParser.java | 162 +++ .../cep/parser/EebusAlarmJsonParser.java | 38 + .../malis/cep/parser/EebusHeaderMeta.java | 37 + .../parser/EebusMeasurementDatagramBody.java | 28 + .../EebusMeasurementDatagramXmlParser.java | 150 +++ .../procon/malis/cep/parser/EventParser.java | 20 + .../cep/parser/JsonPortsBooleanParser.java | 50 + .../procon/malis/cep/parser/M2mPinUpdate.java | 48 + .../parser/M2mPinUpdateJsonErpnextParser.java | 125 ++ .../malis/cep/parser/MeasurementSet.java | 34 + .../malis/cep/parser/MeasurementValue.java | 29 + .../procon/malis/cep/parser/ParsedInput.java | 43 + .../malis/cep/parser/ParserFactory.java | 172 +++ .../malis/cep/parser/ParserRegistry.java | 27 + .../cep/parser/SmartEebusAlarmParser.java | 37 + .../SmartPortsOrEebusMeasurementParser.java | 40 + .../at/co/procon/malis/cep/parser/sss.java | 0 .../malis/cep/pipeline/CepPipeline.java | 211 +++ .../malis/cep/processor/EventProcessor.java | 17 + .../malis/cep/processor/ProcessorFactory.java | 147 ++ .../cep/processor/ProcessorRegistry.java | 22 + .../malis/cep/processor/ProcessorResult.java | 42 + .../malis/cep/processor/js/ErpJsFacade.java | 90 ++ .../js/JavaScriptEventProcessor.java | 354 +++++ .../cep/processor/js/TemplateJsFacade.java | 24 + .../transport/mqtt/MqttBrokerRegistry.java | 180 +++ .../mqtt/MqttInboundDelayedStarter.java | 34 + .../transport/mqtt/MqttIntegrationConfig.java | 218 +++ .../transport/mqtt/MqttOutboundGateway.java | 29 + .../rabbit/RabbitBrokerRegistry.java | 141 ++ .../rabbit/RabbitIngressManager.java | 110 ++ .../procon/malis/cep/util/ExpiringCache.java | 97 ++ .../co/procon/malis/cep/util/InsecureSsl.java | 31 + .../procon/malis/cep/util/TemplateUtil.java | 21 + .../malis/cep/web/TestIngressController.java | 46 + src/main/resources/application.yml | 17 + .../detector/ExpressionRulesDetectorTest.java | 254 ++++ ...ExternalRestEventsDetectorWrapperTest.java | 51 + .../EebusAlarmDatagramXmlFormatterTest.java | 60 + ...rmDatagramXmlParserPreserveFieldsTest.java | 99 ++ .../cep/parser/EebusDatagramParsersTest.java | 79 ++ .../resources/eebus/alarm-datagram-alarm.xml | 19 + .../resources/eebus/alarm-datagram-cancel.xml | 18 + .../resources/eebus/measurement-datagram.xml | 30 + 113 files changed, 12257 insertions(+) create mode 100644 src/main/java/at/co/procon/malis/cep/MalisCepApplication.java create mode 100644 src/main/java/at/co/procon/malis/cep/api/BatchIngressRequest.java create mode 100644 src/main/java/at/co/procon/malis/cep/api/BatchIngressResult.java create mode 100644 src/main/java/at/co/procon/malis/cep/api/IngressController.java create mode 100644 src/main/java/at/co/procon/malis/cep/api/IngressEventRequest.java create mode 100644 src/main/java/at/co/procon/malis/cep/api/IngressExceptionHandler.java create mode 100644 src/main/java/at/co/procon/malis/cep/api/IngressResult.java create mode 100644 src/main/java/at/co/procon/malis/cep/binding/BindingFileLoader.java create mode 100644 src/main/java/at/co/procon/malis/cep/binding/BindingResolver.java create mode 100644 src/main/java/at/co/procon/malis/cep/binding/BindingsLoadedEvent.java create mode 100644 src/main/java/at/co/procon/malis/cep/binding/IngressMessage.java create mode 100644 src/main/java/at/co/procon/malis/cep/binding/RegexGroupNames.java create mode 100644 src/main/java/at/co/procon/malis/cep/binding/ResolvedBinding.java create mode 100644 src/main/java/at/co/procon/malis/cep/config/CepProperties.java create mode 100644 src/main/java/at/co/procon/malis/cep/config/CepStartupValidator.java create mode 100644 src/main/java/at/co/procon/malis/cep/detector/DetectionContext.java create mode 100644 src/main/java/at/co/procon/malis/cep/detector/Detector.java create mode 100644 src/main/java/at/co/procon/malis/cep/detector/DetectorEvent.java create mode 100644 src/main/java/at/co/procon/malis/cep/detector/DetectorFactory.java create mode 100644 src/main/java/at/co/procon/malis/cep/detector/DetectorRegistry.java create mode 100644 src/main/java/at/co/procon/malis/cep/detector/ExpressionRulesDetector.java create mode 100644 src/main/java/at/co/procon/malis/cep/detector/ExternalRestEventsDetector.java create mode 100644 src/main/java/at/co/procon/malis/cep/detector/PassthroughAlarmDetector.java create mode 100644 src/main/java/at/co/procon/malis/cep/detector/PinBadStateDetector.java create mode 100644 src/main/java/at/co/procon/malis/cep/eebus/EebusAddress.java create mode 100644 src/main/java/at/co/procon/malis/cep/eebus/EebusJaxbCodec.java create mode 100644 src/main/java/at/co/procon/malis/cep/eebus/EebusMeasurementDatagram.java create mode 100644 src/main/java/at/co/procon/malis/cep/eebus/EebusSpineConstants.java create mode 100644 src/main/java/at/co/procon/malis/cep/eebus/EebusSpineReflection.java create mode 100644 src/main/java/at/co/procon/malis/cep/eebus/EebusSpineV1.java create mode 100644 src/main/java/at/co/procon/malis/cep/eebus/EebusSpineXsdJaxbValidator.java create mode 100644 src/main/java/at/co/procon/malis/cep/eebus/EebusSpineXsdValidator.java create mode 100644 src/main/java/at/co/procon/malis/cep/eebus/EebusSpineXsdValidatorBase.java create mode 100644 src/main/java/at/co/procon/malis/cep/eebus/EebusXml.java create mode 100644 src/main/java/at/co/procon/malis/cep/eebus/EebusXmlUtil.java create mode 100644 src/main/java/at/co/procon/malis/cep/erpnext/ErpnextFaultLookupClient.java create mode 100644 src/main/java/at/co/procon/malis/cep/event/CepEvent.java create mode 100644 src/main/java/at/co/procon/malis/cep/event/EventBatch.java create mode 100644 src/main/java/at/co/procon/malis/cep/event/SignalDefinitionEvent.java create mode 100644 src/main/java/at/co/procon/malis/cep/event/SignalUpdateEvent.java create mode 100644 src/main/java/at/co/procon/malis/cep/event/SignalValueType.java create mode 100644 src/main/java/at/co/procon/malis/cep/event/TypedValue.java create mode 100644 src/main/java/at/co/procon/malis/cep/event/convert/EebusMeasurementDatagramConverter.java create mode 100644 src/main/java/at/co/procon/malis/cep/event/convert/InputEventConverter.java create mode 100644 src/main/java/at/co/procon/malis/cep/event/convert/MeasurementSetConverter.java create mode 100644 src/main/java/at/co/procon/malis/cep/lifecycle/AlarmLifecycleService.java create mode 100644 src/main/java/at/co/procon/malis/cep/lifecycle/CompositeAlarmLifecycleService.java create mode 100644 src/main/java/at/co/procon/malis/cep/lifecycle/LifecycleDispatcherService.java create mode 100644 src/main/java/at/co/procon/malis/cep/lifecycle/LifecycleHeartbeatScheduler.java create mode 100644 src/main/java/at/co/procon/malis/cep/lifecycle/PublishableAlarm.java create mode 100644 src/main/java/at/co/procon/malis/cep/output/AlarmFormatter.java create mode 100644 src/main/java/at/co/procon/malis/cep/output/EebusAlarmDatagramXmlFormatter.java create mode 100644 src/main/java/at/co/procon/malis/cep/output/EebusAlarmMessageFormatter.java create mode 100644 src/main/java/at/co/procon/malis/cep/output/FormattedMessage.java create mode 100644 src/main/java/at/co/procon/malis/cep/output/MalisCepAlarmJsonFormatter.java create mode 100644 src/main/java/at/co/procon/malis/cep/output/MalisFaultJsonFormatter.java create mode 100644 src/main/java/at/co/procon/malis/cep/output/MqttOutputPublisher.java create mode 100644 src/main/java/at/co/procon/malis/cep/output/OutboxMessage.java create mode 100644 src/main/java/at/co/procon/malis/cep/output/OutboxOutputPublisher.java create mode 100644 src/main/java/at/co/procon/malis/cep/output/OutboxStore.java create mode 100644 src/main/java/at/co/procon/malis/cep/output/OutboxStoreProvider.java create mode 100644 src/main/java/at/co/procon/malis/cep/output/OutputPublisher.java create mode 100644 src/main/java/at/co/procon/malis/cep/output/OutputRegistry.java create mode 100644 src/main/java/at/co/procon/malis/cep/output/PublishedMessage.java create mode 100644 src/main/java/at/co/procon/malis/cep/output/PublisherService.java create mode 100644 src/main/java/at/co/procon/malis/cep/output/RabbitOutputPublisher.java create mode 100644 src/main/java/at/co/procon/malis/cep/output/RestOutputPublisher.java create mode 100644 src/main/java/at/co/procon/malis/cep/parser/AlarmBatch.java create mode 100644 src/main/java/at/co/procon/malis/cep/parser/AlarmInput.java create mode 100644 src/main/java/at/co/procon/malis/cep/parser/EebusAlarmDatagramXmlParser.java create mode 100644 src/main/java/at/co/procon/malis/cep/parser/EebusAlarmJsonParser.java create mode 100644 src/main/java/at/co/procon/malis/cep/parser/EebusHeaderMeta.java create mode 100644 src/main/java/at/co/procon/malis/cep/parser/EebusMeasurementDatagramBody.java create mode 100644 src/main/java/at/co/procon/malis/cep/parser/EebusMeasurementDatagramXmlParser.java create mode 100644 src/main/java/at/co/procon/malis/cep/parser/EventParser.java create mode 100644 src/main/java/at/co/procon/malis/cep/parser/JsonPortsBooleanParser.java create mode 100644 src/main/java/at/co/procon/malis/cep/parser/M2mPinUpdate.java create mode 100644 src/main/java/at/co/procon/malis/cep/parser/M2mPinUpdateJsonErpnextParser.java create mode 100644 src/main/java/at/co/procon/malis/cep/parser/MeasurementSet.java create mode 100644 src/main/java/at/co/procon/malis/cep/parser/MeasurementValue.java create mode 100644 src/main/java/at/co/procon/malis/cep/parser/ParsedInput.java create mode 100644 src/main/java/at/co/procon/malis/cep/parser/ParserFactory.java create mode 100644 src/main/java/at/co/procon/malis/cep/parser/ParserRegistry.java create mode 100644 src/main/java/at/co/procon/malis/cep/parser/SmartEebusAlarmParser.java create mode 100644 src/main/java/at/co/procon/malis/cep/parser/SmartPortsOrEebusMeasurementParser.java create mode 100644 src/main/java/at/co/procon/malis/cep/parser/sss.java create mode 100644 src/main/java/at/co/procon/malis/cep/pipeline/CepPipeline.java create mode 100644 src/main/java/at/co/procon/malis/cep/processor/EventProcessor.java create mode 100644 src/main/java/at/co/procon/malis/cep/processor/ProcessorFactory.java create mode 100644 src/main/java/at/co/procon/malis/cep/processor/ProcessorRegistry.java create mode 100644 src/main/java/at/co/procon/malis/cep/processor/ProcessorResult.java create mode 100644 src/main/java/at/co/procon/malis/cep/processor/js/ErpJsFacade.java create mode 100644 src/main/java/at/co/procon/malis/cep/processor/js/JavaScriptEventProcessor.java create mode 100644 src/main/java/at/co/procon/malis/cep/processor/js/TemplateJsFacade.java create mode 100644 src/main/java/at/co/procon/malis/cep/transport/mqtt/MqttBrokerRegistry.java create mode 100644 src/main/java/at/co/procon/malis/cep/transport/mqtt/MqttInboundDelayedStarter.java create mode 100644 src/main/java/at/co/procon/malis/cep/transport/mqtt/MqttIntegrationConfig.java create mode 100644 src/main/java/at/co/procon/malis/cep/transport/mqtt/MqttOutboundGateway.java create mode 100644 src/main/java/at/co/procon/malis/cep/transport/rabbit/RabbitBrokerRegistry.java create mode 100644 src/main/java/at/co/procon/malis/cep/transport/rabbit/RabbitIngressManager.java create mode 100644 src/main/java/at/co/procon/malis/cep/util/ExpiringCache.java create mode 100644 src/main/java/at/co/procon/malis/cep/util/InsecureSsl.java create mode 100644 src/main/java/at/co/procon/malis/cep/util/TemplateUtil.java create mode 100644 src/main/java/at/co/procon/malis/cep/web/TestIngressController.java create mode 100644 src/main/resources/application.yml create mode 100644 src/test/java/at/co/procon/malis/cep/detector/ExpressionRulesDetectorTest.java create mode 100644 src/test/java/at/co/procon/malis/cep/detector/ExternalRestEventsDetectorWrapperTest.java create mode 100644 src/test/java/at/co/procon/malis/cep/output/EebusAlarmDatagramXmlFormatterTest.java create mode 100644 src/test/java/at/co/procon/malis/cep/parser/EebusAlarmDatagramXmlParserPreserveFieldsTest.java create mode 100644 src/test/java/at/co/procon/malis/cep/parser/EebusDatagramParsersTest.java create mode 100644 src/test/resources/eebus/alarm-datagram-alarm.xml create mode 100644 src/test/resources/eebus/alarm-datagram-cancel.xml create mode 100644 src/test/resources/eebus/measurement-datagram.xml diff --git a/src/main/java/at/co/procon/malis/cep/MalisCepApplication.java b/src/main/java/at/co/procon/malis/cep/MalisCepApplication.java new file mode 100644 index 0000000..93001b8 --- /dev/null +++ b/src/main/java/at/co/procon/malis/cep/MalisCepApplication.java @@ -0,0 +1,14 @@ +package at.co.procon.malis.cep; + +import at.co.procon.malis.cep.config.CepProperties; +import org.springframework.boot.SpringApplication; +import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.boot.context.properties.EnableConfigurationProperties; + +@SpringBootApplication +@EnableConfigurationProperties(CepProperties.class) +public class MalisCepApplication { + public static void main(String[] args) { + SpringApplication.run(MalisCepApplication.class, args); + } +} diff --git a/src/main/java/at/co/procon/malis/cep/api/BatchIngressRequest.java b/src/main/java/at/co/procon/malis/cep/api/BatchIngressRequest.java new file mode 100644 index 0000000..9ea3cb9 --- /dev/null +++ b/src/main/java/at/co/procon/malis/cep/api/BatchIngressRequest.java @@ -0,0 +1,14 @@ +package at.co.procon.malis.cep.api; + +import java.util.ArrayList; +import java.util.List; + +/** + * Wrapper request for batch ingestion. + */ +public class BatchIngressRequest { + private List events = new ArrayList<>(); + + public List getEvents() { return events; } + public void setEvents(List events) { this.events = events; } +} \ No newline at end of file diff --git a/src/main/java/at/co/procon/malis/cep/api/BatchIngressResult.java b/src/main/java/at/co/procon/malis/cep/api/BatchIngressResult.java new file mode 100644 index 0000000..f967400 --- /dev/null +++ b/src/main/java/at/co/procon/malis/cep/api/BatchIngressResult.java @@ -0,0 +1,50 @@ +package at.co.procon.malis.cep.api; + +import java.time.Instant; +import java.util.ArrayList; +import java.util.List; + +/** + * Batch result. + */ +public class BatchIngressResult { + + private int acceptedCount; + private int failedCount; + private Instant startedAt; + private Instant finishedAt; + private List results = new ArrayList<>(); + + public static BatchIngressResult error(Instant startedAt, Instant finishedAt, String msg) { + BatchIngressResult r = new BatchIngressResult(); + r.setStartedAt(startedAt); + r.setFinishedAt(finishedAt); + r.setAcceptedCount(0); + r.setFailedCount(1); + + IngressResult item = new IngressResult(); + item.setAccepted(false); + item.setErrorType("java.lang.IllegalArgumentException"); + item.setErrorMessage(msg); + item.setStartedAt(startedAt); + item.setFinishedAt(finishedAt); + r.getResults().add(item); + + return r; + } + + public int getAcceptedCount() { return acceptedCount; } + public void setAcceptedCount(int acceptedCount) { this.acceptedCount = acceptedCount; } + + public int getFailedCount() { return failedCount; } + public void setFailedCount(int failedCount) { this.failedCount = failedCount; } + + public Instant getStartedAt() { return startedAt; } + public void setStartedAt(Instant startedAt) { this.startedAt = startedAt; } + + public Instant getFinishedAt() { return finishedAt; } + public void setFinishedAt(Instant finishedAt) { this.finishedAt = finishedAt; } + + public List getResults() { return results; } + public void setResults(List results) { this.results = results; } +} \ No newline at end of file diff --git a/src/main/java/at/co/procon/malis/cep/api/IngressController.java b/src/main/java/at/co/procon/malis/cep/api/IngressController.java new file mode 100644 index 0000000..f8ebaa1 --- /dev/null +++ b/src/main/java/at/co/procon/malis/cep/api/IngressController.java @@ -0,0 +1,194 @@ +package at.co.procon.malis.cep.api; + +// Add these files to your Spring Boot project under: +// src/main/java/at/co/procon/malis/cep/api/ +// +// Purpose +// ------- +// A lightweight REST API to feed the CEP pipeline with: +// - a single ingress event (JSON envelope) +// - a batch/list of ingress events +// +// Notes +// ----- +// - This controller does NOT bypass any binding/parsing/detection logic; it just adapts HTTP requests +// into IngressMessage objects and calls CepPipeline.onIngress(...). +// - Payload can be supplied as UTF-8 string or Base64. +// - The pipeline remains the single source of truth for routing/binding selection. + +import at.co.procon.malis.cep.binding.IngressMessage; +import at.co.procon.malis.cep.config.CepProperties; +import at.co.procon.malis.cep.pipeline.CepPipeline; +import org.springframework.http.HttpStatus; +import org.springframework.http.MediaType; +import org.springframework.http.ResponseEntity; +import org.springframework.util.StringUtils; +import org.springframework.web.bind.annotation.*; + +import java.nio.charset.StandardCharsets; +import java.time.Instant; +import java.util.*; + +@RestController +@RequestMapping("/api/ingress") +public class IngressController { + + private final CepPipeline pipeline; + + public IngressController(CepPipeline pipeline) { + this.pipeline = pipeline; + } + + /** + * Ingest ONE event via JSON envelope. + * + * Example: + * POST /api/ingress/event + * { + * "inType": "MQTT", + * "path": "/sadfjsadf/siteA/depB", + * "headers": {"contentType": "application/xml"}, + * "payload": "...", + * "payloadEncoding": "UTF8" + * } + */ + @PostMapping( + value = "/event", + consumes = MediaType.APPLICATION_JSON_VALUE, + produces = MediaType.APPLICATION_JSON_VALUE + ) + public ResponseEntity ingestOne(@RequestBody IngressEventRequest req) { + Instant startedAt = Instant.now(); + + try { + IngressMessage msg = toIngressMessage(req); + var res = pipeline.process(msg); + + IngressResult result = IngressResult.ok(req.getRequestId(), startedAt, Instant.now()); + return ResponseEntity.status(HttpStatus.ACCEPTED).body(result); + + } catch (Exception e) { + IngressResult result = IngressResult.error(req.getRequestId(), startedAt, Instant.now(), e); + return ResponseEntity.status(HttpStatus.BAD_REQUEST).body(result); + } + } + + /** + * Ingest MANY events via JSON envelope. + * + * Input supports either: + * - {"events": [ ... ]} + * or a plain JSON array (see overload below). + */ + @PostMapping( + value = "/events", + consumes = MediaType.APPLICATION_JSON_VALUE, + produces = MediaType.APPLICATION_JSON_VALUE + ) + public ResponseEntity ingestBatch(@RequestBody BatchIngressRequest req, + @RequestParam(name = "failFast", defaultValue = "false") boolean failFast) { + return ingestBatchInternal(req.getEvents(), failFast); + } + + /** + * Alternative batch endpoint: accepts a plain array. + */ + @PostMapping( + value = "/events-array", + consumes = MediaType.APPLICATION_JSON_VALUE, + produces = MediaType.APPLICATION_JSON_VALUE + ) + public ResponseEntity ingestBatchArray(@RequestBody List events, + @RequestParam(name = "failFast", defaultValue = "false") boolean failFast) { + return ingestBatchInternal(events, failFast); + } + + private ResponseEntity ingestBatchInternal(List events, boolean failFast) { + Instant startedAt = Instant.now(); + + if (events == null) { + BatchIngressResult bad = BatchIngressResult.error(startedAt, Instant.now(), "events is null"); + return ResponseEntity.status(HttpStatus.BAD_REQUEST).body(bad); + } + + List results = new ArrayList<>(); + int ok = 0; + int failed = 0; + + for (IngressEventRequest req : events) { + Instant itemStarted = Instant.now(); + try { + IngressMessage msg = toIngressMessage(req); + var res = pipeline.process(msg); + results.add(IngressResult.ok(req.getRequestId(), itemStarted, Instant.now())); + ok++; + } catch (Exception e) { + results.add(IngressResult.error(req.getRequestId(), itemStarted, Instant.now(), e)); + failed++; + if (failFast) break; + } + } + + BatchIngressResult out = new BatchIngressResult(); + out.setAcceptedCount(ok); + out.setFailedCount(failed); + out.setStartedAt(startedAt); + out.setFinishedAt(Instant.now()); + out.setResults(results); + + // 207 Multi-Status is nice for partial failures, but many clients handle 200/202 better. + HttpStatus status = failed == 0 ? HttpStatus.ACCEPTED : HttpStatus.OK; + return ResponseEntity.status(status).body(out); + } + + private static IngressMessage toIngressMessage(IngressEventRequest req) { + if (req == null) throw new IllegalArgumentException("request is null"); + + String inTypeRaw = trimToNull(req.getInType()); + String path = trimToNull(req.getPath()); + + if (inTypeRaw == null) throw new IllegalArgumentException("inType is required (MQTT|RABBIT|HTTP)"); + if (path == null) throw new IllegalArgumentException("path is required"); + + CepProperties.InType inType; + try { + inType = CepProperties.InType.valueOf(inTypeRaw.trim().toUpperCase(Locale.ROOT)); + } catch (Exception e) { + throw new IllegalArgumentException("Unsupported inType: " + inTypeRaw, e); + } + + Map headers = req.getHeaders() == null ? Map.of() : req.getHeaders(); + byte[] payload = decodePayload(req); + + return new IngressMessage(inType, path, headers, payload); + } + + private static byte[] decodePayload(IngressEventRequest req) { + // Priority: explicit payloadBase64, else payload+payloadEncoding + String base64 = trimToNull(req.getPayloadBase64()); + if (base64 != null) { + return Base64.getDecoder().decode(base64); + } + + String payload = req.getPayload(); + if (payload == null) return new byte[0]; + + String enc = trimToNull(req.getPayloadEncoding()); + if (enc == null) enc = "UTF8"; + + switch (enc.trim().toUpperCase(Locale.ROOT)) { + case "UTF8": + case "UTF-8": + return payload.getBytes(StandardCharsets.UTF_8); + case "BASE64": + return Base64.getDecoder().decode(payload); + default: + throw new IllegalArgumentException("Unsupported payloadEncoding: " + enc + " (use UTF8 or BASE64)"); + } + } + + private static String trimToNull(String s) { + if (!StringUtils.hasText(s)) return null; + return s.trim(); + } +} diff --git a/src/main/java/at/co/procon/malis/cep/api/IngressEventRequest.java b/src/main/java/at/co/procon/malis/cep/api/IngressEventRequest.java new file mode 100644 index 0000000..dc9a0f1 --- /dev/null +++ b/src/main/java/at/co/procon/malis/cep/api/IngressEventRequest.java @@ -0,0 +1,56 @@ +package at.co.procon.malis.cep.api; + +// ------------------------------------------- +// DTOs (no records; plain POJOs with getters) +// ------------------------------------------- + +import java.util.LinkedHashMap; +import java.util.Map; + +/** + * JSON request envelope for a single ingress event. + */ +public class IngressEventRequest { + + /** Optional client-provided ID for tracing. */ + private String requestId; + + /** Required: MQTT|RABBIT|HTTP */ + private String inType; + + /** Required: topic/queue/path depending on inType */ + private String path; + + /** Optional headers/metadata */ + private Map headers = new LinkedHashMap<>(); + + /** Payload as string. Interpret using payloadEncoding (UTF8 by default). */ + private String payload; + + /** Alternative: payload already Base64-encoded. If set, overrides payload/payloadEncoding. */ + private String payloadBase64; + + /** UTF8|BASE64 */ + private String payloadEncoding = "UTF8"; + + public String getRequestId() { return requestId; } + public void setRequestId(String requestId) { this.requestId = requestId; } + + public String getInType() { return inType; } + public void setInType(String inType) { this.inType = inType; } + + public String getPath() { return path; } + public void setPath(String path) { this.path = path; } + + public Map getHeaders() { return headers; } + public void setHeaders(Map headers) { this.headers = headers; } + + public String getPayload() { return payload; } + public void setPayload(String payload) { this.payload = payload; } + + public String getPayloadBase64() { return payloadBase64; } + public void setPayloadBase64(String payloadBase64) { this.payloadBase64 = payloadBase64; } + + public String getPayloadEncoding() { return payloadEncoding; } + public void setPayloadEncoding(String payloadEncoding) { this.payloadEncoding = payloadEncoding; } +} \ No newline at end of file diff --git a/src/main/java/at/co/procon/malis/cep/api/IngressExceptionHandler.java b/src/main/java/at/co/procon/malis/cep/api/IngressExceptionHandler.java new file mode 100644 index 0000000..f9049f3 --- /dev/null +++ b/src/main/java/at/co/procon/malis/cep/api/IngressExceptionHandler.java @@ -0,0 +1,22 @@ +// ------------------------------------------- +// Optional: ControllerAdvice for nicer errors +// ------------------------------------------- + +package at.co.procon.malis.cep.api; + +import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.ExceptionHandler; +import org.springframework.web.bind.annotation.RestControllerAdvice; + +import java.time.Instant; + +@RestControllerAdvice +public class IngressExceptionHandler { + + @ExceptionHandler(IllegalArgumentException.class) + public ResponseEntity handleBadRequest(IllegalArgumentException ex) { + IngressResult r = IngressResult.error(null, Instant.now(), Instant.now(), ex); + return ResponseEntity.status(HttpStatus.BAD_REQUEST).body(r); + } +} \ No newline at end of file diff --git a/src/main/java/at/co/procon/malis/cep/api/IngressResult.java b/src/main/java/at/co/procon/malis/cep/api/IngressResult.java new file mode 100644 index 0000000..36ebcb7 --- /dev/null +++ b/src/main/java/at/co/procon/malis/cep/api/IngressResult.java @@ -0,0 +1,74 @@ +package at.co.procon.malis.cep.api; + +import java.time.Instant; + +/** + * Per-item result. + */ +public class IngressResult { + + private String requestId; + private boolean accepted; + private String errorType; + private String errorMessage; + private String errorStack; + private Instant startedAt; + private Instant finishedAt; + + public static IngressResult ok(String requestId, Instant startedAt, Instant finishedAt) { + IngressResult r = new IngressResult(); + r.setRequestId(requestId); + r.setAccepted(true); + r.setStartedAt(startedAt); + r.setFinishedAt(finishedAt); + return r; + } + + public static IngressResult error(String requestId, Instant startedAt, Instant finishedAt, Exception e) { + IngressResult r = new IngressResult(); + r.setRequestId(requestId); + r.setAccepted(false); + r.setStartedAt(startedAt); + r.setFinishedAt(finishedAt); + + if (e != null) { + r.setErrorType(e.getClass().getName()); + r.setErrorMessage(e.getMessage()); + r.setErrorStack(stacktraceToString(e)); + } + return r; + } + + private static String stacktraceToString(Throwable t) { + try { + java.io.StringWriter sw = new java.io.StringWriter(); + java.io.PrintWriter pw = new java.io.PrintWriter(sw); + t.printStackTrace(pw); + pw.flush(); + return sw.toString(); + } catch (Exception ignore) { + return null; + } + } + + public String getRequestId() { return requestId; } + public void setRequestId(String requestId) { this.requestId = requestId; } + + public boolean isAccepted() { return accepted; } + public void setAccepted(boolean accepted) { this.accepted = accepted; } + + public String getErrorType() { return errorType; } + public void setErrorType(String errorType) { this.errorType = errorType; } + + public String getErrorMessage() { return errorMessage; } + public void setErrorMessage(String errorMessage) { this.errorMessage = errorMessage; } + + public String getErrorStack() { return errorStack; } + public void setErrorStack(String errorStack) { this.errorStack = errorStack; } + + public Instant getStartedAt() { return startedAt; } + public void setStartedAt(Instant startedAt) { this.startedAt = startedAt; } + + public Instant getFinishedAt() { return finishedAt; } + public void setFinishedAt(Instant finishedAt) { this.finishedAt = finishedAt; } +} \ No newline at end of file diff --git a/src/main/java/at/co/procon/malis/cep/binding/BindingFileLoader.java b/src/main/java/at/co/procon/malis/cep/binding/BindingFileLoader.java new file mode 100644 index 0000000..5af527b --- /dev/null +++ b/src/main/java/at/co/procon/malis/cep/binding/BindingFileLoader.java @@ -0,0 +1,80 @@ +package at.co.procon.malis.cep.binding; + +import at.co.procon.malis.cep.config.CepProperties; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; +import org.springframework.boot.ApplicationArguments; +import org.springframework.boot.ApplicationRunner; +import org.springframework.context.ApplicationEventPublisher; +import org.springframework.core.Ordered; +import org.springframework.core.annotation.Order; +import org.springframework.core.io.Resource; +import org.springframework.core.io.support.PathMatchingResourcePatternResolver; +import org.springframework.stereotype.Component; + +import java.io.InputStream; +import java.util.LinkedHashMap; +import java.util.Map; + +/** + * Loads bindings from YAML files matched by cep.bindingLocations. + * + * Each file is expected to have: + * cep: + * bindings: + * : { ... } + */ +@Component +@Order(Ordered.HIGHEST_PRECEDENCE) +public class BindingFileLoader implements ApplicationRunner { + + private final CepProperties props; + private final ApplicationEventPublisher events; + private final ObjectMapper yaml = new ObjectMapper(new YAMLFactory()); + + public BindingFileLoader(CepProperties props, ApplicationEventPublisher events) { + this.props = props; + this.events = events; + } + + @Override + public void run(ApplicationArguments args) { + Map merged = new LinkedHashMap<>(props.getBindings()); + PathMatchingResourcePatternResolver resolver = new PathMatchingResourcePatternResolver(); + + for (String pattern : props.getBindingLocations()) { + try { + Resource[] resources = resolver.getResources(pattern); + for (Resource r : resources) { + try (InputStream in = r.getInputStream()) { + @SuppressWarnings("unchecked") + Map root = yaml.readValue(in, Map.class); + if (root == null) continue; + Object cepObj = root.get("cep"); + if (!(cepObj instanceof Map cepMap)) continue; + Object bindingsObj = cepMap.get("bindings"); + if (!(bindingsObj instanceof Map bindingsMap)) continue; + + for (Map.Entry e : bindingsMap.entrySet()) { + String id = String.valueOf(e.getKey()); + CepProperties.SourceBinding binding = yaml.convertValue(e.getValue(), CepProperties.SourceBinding.class); + + if (merged.containsKey(id)) { + throw new IllegalStateException("Duplicate binding id '" + id + "' found in file: " + r.getDescription()); + } + merged.put(id, binding); + } + } + } + } catch (Exception ex) { + throw new IllegalStateException("Failed loading bindings from pattern: " + pattern, ex); + } + } + + props.setBindings(merged); + + // Notify the application that bindings are loaded/merged. + // Consumers (e.g. BindingResolver) can rebuild internal indexes without creating bean cycles. + events.publishEvent(new BindingsLoadedEvent(this, merged)); + } +} diff --git a/src/main/java/at/co/procon/malis/cep/binding/BindingResolver.java b/src/main/java/at/co/procon/malis/cep/binding/BindingResolver.java new file mode 100644 index 0000000..2156202 --- /dev/null +++ b/src/main/java/at/co/procon/malis/cep/binding/BindingResolver.java @@ -0,0 +1,166 @@ +package at.co.procon.malis.cep.binding; + +import at.co.procon.malis.cep.config.CepProperties; +import org.springframework.context.event.EventListener; +import org.springframework.stereotype.Component; + +import jakarta.annotation.PostConstruct; +import java.util.*; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * Resolves an {@link IngressMessage} to exactly one {@link CepProperties.SourceBinding}. + * + *

Important design constraint for this prototype: + *

    + *
  • Bindings are configured in advance. We do not attempt to "guess" parser/detector at runtime.
    • + *
  • If more than one binding matches the same message, we fail fast with a clear error.
    • + *
+ */ +@Component +public class BindingResolver { + + private final CepProperties props; + private volatile List compiled = List.of(); + + public BindingResolver(CepProperties props) { + this.props = props; + } + + @PostConstruct + public void init() { + rebuildIndex(); + } + + /** + * Rebuild the compiled index after bindings were loaded/merged from external YAML files. + * + * This avoids a circular dependency between BindingResolver and BindingFileLoader. + */ + @EventListener + public void onBindingsLoaded(BindingsLoadedEvent evt) { + rebuildIndex(); + } + + /** + * (Re)builds the compiled binding index (regex -> pattern) from current properties. + * + *

Call this if you hot-reload bindings in the future. For now it's used once at startup. + */ + public void rebuildIndex() { + List list = new ArrayList<>(); + for (var e : props.getBindings().entrySet()) { + String id = e.getKey(); + var b = e.getValue(); + if (b == null || !b.isEnabled() || b.getIn() == null || b.getIn().getType() == null) continue; + + String regex = selectRegex(b.getIn().getType(), b); + if (regex == null || regex.isBlank()) { + throw new IllegalStateException("Binding '" + id + "' missing match regex for type " + b.getIn().getType()); + } + Pattern p = Pattern.compile(regex); + list.add(new CompiledBinding(id, b, p)); + } + this.compiled = List.copyOf(list); + } + + /** + * Resolve a message to exactly one binding. + */ + public Optional resolve(IngressMessage msg) { + List matches = new ArrayList<>(); + + for (CompiledBinding cb : compiled) { + if (cb.binding.getIn().getType() != msg.getInType()) continue; + + Matcher m = cb.pattern.matcher(msg.getPath()); + if (m.matches()) { + Map vars = new LinkedHashMap<>(); + vars.putAll(extractNamedGroups(m)); + applyExtractDefaults(cb.binding, vars); + ensureDeviceId(cb.binding, msg, vars); + matches.add(new ResolvedBinding(cb.id, cb.binding, vars)); + } + } + + if (matches.isEmpty()) return Optional.empty(); + if (matches.size() > 1) { + String ids = matches.stream().map(ResolvedBinding::getBindingId).toList().toString(); + throw new IllegalStateException("Ambiguous binding match for '" + msg.getPath() + "' (" + msg.getInType() + "): " + ids); + } + return Optional.of(matches.get(0)); + } + + private static String selectRegex(CepProperties.InType type, CepProperties.SourceBinding b) { + if (b.getIn() == null || b.getIn().getMatch() == null) return null; + return switch (type) { + case MQTT -> b.getIn().getMatch().getTopicRegex(); + case RABBIT -> b.getIn().getMatch().getQueueRegex(); + case HTTP -> b.getIn().getMatch().getPathRegex(); + }; + } + + private static void applyExtractDefaults(CepProperties.SourceBinding b, Map vars) { + var ex = b.getExtract(); + if (ex == null) return; + if (ex.getTenant() != null && !vars.containsKey("tenant")) vars.put("tenant", ex.getTenant()); + if (ex.getSite() != null && !vars.containsKey("site")) vars.put("site", ex.getSite()); + if (ex.getDepartment() != null && !vars.containsKey("department")) vars.put("department", ex.getDepartment()); + } + + private static void ensureDeviceId(CepProperties.SourceBinding b, IngressMessage msg, Map vars) { + if (vars.containsKey("deviceId") && vars.get("deviceId") != null && !vars.get("deviceId").isBlank()) return; + + var ex = b.getExtract(); + if (ex == null || ex.getDeviceIdStrategy() == null) return; + + var strat = ex.getDeviceIdStrategy(); + String type = strat.getType(); + + if ("HASH_OF_TOPIC".equalsIgnoreCase(type)) { + String salt = String.valueOf(strat.getConfig().getOrDefault("salt", "")); + vars.put("deviceId", sha256Hex(salt + "|" + msg.getPath())); + } else if ("STATIC".equalsIgnoreCase(type)) { + String v = String.valueOf(strat.getConfig().getOrDefault("value", "unknown")); + vars.put("deviceId", v); + } + // FROM_JSON_POINTER can be implemented inside a payload-aware parser. + } + + private static String sha256Hex(String s) { + try { + var md = java.security.MessageDigest.getInstance("SHA-256"); + byte[] digest = md.digest(s.getBytes(java.nio.charset.StandardCharsets.UTF_8)); + StringBuilder sb = new StringBuilder(digest.length * 2); + for (byte b : digest) sb.append(String.format("%02x", b)); + return sb.toString(); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + private static Map extractNamedGroups(Matcher m) { + Map out = new LinkedHashMap<>(); + Set names = RegexGroupNames.extract(m.pattern().pattern()); + for (String n : names) { + try { + String v = m.group(n); + if (v != null) out.put(n, v); + } catch (IllegalArgumentException ignore) {} + } + return out; + } + + private static final class CompiledBinding { + private final String id; + private final CepProperties.SourceBinding binding; + private final Pattern pattern; + + private CompiledBinding(String id, CepProperties.SourceBinding binding, Pattern pattern) { + this.id = id; + this.binding = binding; + this.pattern = pattern; + } + } +} diff --git a/src/main/java/at/co/procon/malis/cep/binding/BindingsLoadedEvent.java b/src/main/java/at/co/procon/malis/cep/binding/BindingsLoadedEvent.java new file mode 100644 index 0000000..365a55d --- /dev/null +++ b/src/main/java/at/co/procon/malis/cep/binding/BindingsLoadedEvent.java @@ -0,0 +1,38 @@ +package at.co.procon.malis.cep.binding; + +import at.co.procon.malis.cep.config.CepProperties; + +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.Map; + +/** + * Application event published after bindings have been loaded/merged from external configuration files. + * + *

Why this exists: + *

    + *
  • BindingFileLoader is an ApplicationRunner that loads YAML binding files at startup.
    • + *
  • BindingResolver maintains an in-memory compiled index (regex patterns) for fast routing.
    • + *
  • We want BindingResolver to rebuild that index after loading — without creating bean cycles.
    • + *
+ */ +public class BindingsLoadedEvent { + + private final Object source; + private final Map bindings; + + public BindingsLoadedEvent(Object source, Map bindings) { + this.source = source; + this.bindings = bindings == null + ? Map.of() + : Collections.unmodifiableMap(new LinkedHashMap<>(bindings)); + } + + public Object getSource() { + return source; + } + + public Map getBindings() { + return bindings; + } +} diff --git a/src/main/java/at/co/procon/malis/cep/binding/IngressMessage.java b/src/main/java/at/co/procon/malis/cep/binding/IngressMessage.java new file mode 100644 index 0000000..0c9c216 --- /dev/null +++ b/src/main/java/at/co/procon/malis/cep/binding/IngressMessage.java @@ -0,0 +1,54 @@ +package at.co.procon.malis.cep.binding; + +import at.co.procon.malis.cep.config.CepProperties; + +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.Map; + +/** + * Immutable ingress message envelope. + * + *

All transports (MQTT / RabbitMQ / HTTP) should be normalized into this shape before the + * binding resolver and parsing pipeline run. + * + *

The {@code path} means: + *

    + *
  • MQTT: topic
    • + *
  • Rabbit: queue name (or routing key, depending on your ingress adapter)
    • + *
  • HTTP: request path
    • + *
+ */ +public final class IngressMessage { + + private final CepProperties.InType inType; + private final String path; + private final Map headers; + private final byte[] payload; + + public IngressMessage(CepProperties.InType inType, + String path, + Map headers, + byte[] payload) { + this.inType = inType; + this.path = path; + this.headers = headers == null ? Map.of() : Collections.unmodifiableMap(new LinkedHashMap<>(headers)); + this.payload = payload == null ? new byte[0] : payload.clone(); + } + + public CepProperties.InType getInType() { + return inType; + } + + public String getPath() { + return path; + } + + public Map getHeaders() { + return headers; + } + + public byte[] getPayload() { + return payload.clone(); + } +} diff --git a/src/main/java/at/co/procon/malis/cep/binding/RegexGroupNames.java b/src/main/java/at/co/procon/malis/cep/binding/RegexGroupNames.java new file mode 100644 index 0000000..4862532 --- /dev/null +++ b/src/main/java/at/co/procon/malis/cep/binding/RegexGroupNames.java @@ -0,0 +1,21 @@ +package at.co.procon.malis.cep.binding; + +import java.util.LinkedHashSet; +import java.util.Set; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +final class RegexGroupNames { + private static final Pattern GROUP_NAME = Pattern.compile("\\(\\?<([a-zA-Z][a-zA-Z0-9_]*)>"); + + static Set extract(String regex) { + Set names = new LinkedHashSet<>(); + Matcher m = GROUP_NAME.matcher(regex); + while (m.find()) { + names.add(m.group(1)); + } + return names; + } + + private RegexGroupNames() {} +} diff --git a/src/main/java/at/co/procon/malis/cep/binding/ResolvedBinding.java b/src/main/java/at/co/procon/malis/cep/binding/ResolvedBinding.java new file mode 100644 index 0000000..707eea7 --- /dev/null +++ b/src/main/java/at/co/procon/malis/cep/binding/ResolvedBinding.java @@ -0,0 +1,42 @@ +package at.co.procon.malis.cep.binding; + +import at.co.procon.malis.cep.config.CepProperties; + +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.Map; + +/** + * Result of binding resolution for a single ingress message. + * + *

Includes: + *

    + *
  • bindingId: the id from configuration
    • + *
  • binding: the resolved SourceBinding definition
    • + *
  • vars: extracted variables (named regex groups + defaults)
    • + *
+ */ +public final class ResolvedBinding { + + private final String bindingId; + private final CepProperties.SourceBinding binding; + private final Map vars; + + public ResolvedBinding(String bindingId, CepProperties.SourceBinding binding, Map vars) { + this.bindingId = bindingId; + this.binding = binding; + this.vars = vars == null ? Map.of() : Collections.unmodifiableMap(new LinkedHashMap<>(vars)); + } + + public String getBindingId() { + return bindingId; + } + + public CepProperties.SourceBinding getBinding() { + return binding; + } + + public Map getVars() { + return vars; + } +} diff --git a/src/main/java/at/co/procon/malis/cep/config/CepProperties.java b/src/main/java/at/co/procon/malis/cep/config/CepProperties.java new file mode 100644 index 0000000..c625331 --- /dev/null +++ b/src/main/java/at/co/procon/malis/cep/config/CepProperties.java @@ -0,0 +1,602 @@ +package at.co.procon.malis.cep.config; + +import org.springframework.boot.context.properties.ConfigurationProperties; + +import java.time.Duration; +import java.util.*; + +/** + * Root configuration for the CEP service. + * + * Design goals: + * - Keep bindings (source -> parser -> detector -> outputs) as *data* and load them from external files. + * - Keep definitions as maps (id -> definition) so multiple YAML files can be merged cleanly. + * - Keep ingress configuration separate from bindings: ingress controls how we *receive* messages, + * bindings control how we *route + process* messages. + */ +@ConfigurationProperties(prefix = "cep") +public class CepProperties { + + // ------------------------- + // Component definitions + // ------------------------- + + /** Parser catalog (id -> parser definition). */ + private Map parsers = new HashMap<>(); + + /** Processor catalog (id -> processor definition). A processor replaces parser+detector in one step. */ + private Map processors = new HashMap<>(); + + /** Detector catalog (id -> detector definition). */ + private Map detectors = new HashMap<>(); + + /** Output catalog (id -> output destination definition). */ + private Map outputs = new HashMap<>(); + + /** Lifecycle policies (id -> policy parameters). */ + private Map lifecyclePolicies = new HashMap<>(); + + /** + * Brokers catalog (id -> broker definition). + * For MQTT, url is typically tcp://host:1883 or ssl://host:8883. + * For RabbitMQ, url is typically amqp://user:pass@host:5672/vhost. + */ + private Map brokers = new HashMap<>(); + + // ------------------------- + // Ingress / DLQ + // ------------------------- + + /** Ingress configuration for MQTT and Rabbit. */ + private IngressDef ingress = new IngressDef(); + + /** Optional DLQ configuration (publishes a diagnostic message to a configured output). */ + private DeadLetterDef deadLetter = new DeadLetterDef(); + + // ------------------------- + // Bindings + // ------------------------- + + /** + * Filled by {@code BindingFileLoader} at startup. + * Key is bindingId. + */ + private Map bindings = new LinkedHashMap<>(); + + /** Glob locations, e.g. ["file:./config/cep/bindings/*.yml"] */ + private List bindingLocations = List.of("file:./config/cep/bindings/*.yml"); + + private EebusXsdValidation eebusXsdValidation = new EebusXsdValidation(); + + /** Configuration for optional SPINE XSD validation. */ + public static class EebusXsdValidation { + private boolean enabled = false; + private boolean failFast = true; + private String schemaRoot; + + public boolean isEnabled() { return enabled; } + public void setEnabled(boolean enabled) { this.enabled = enabled; } + + public boolean isFailFast() { return failFast; } + public void setFailFast(boolean failFast) { this.failFast = failFast; } + + public String getSchemaRoot() { return schemaRoot; } + public void setSchemaRoot(String schemaRoot) { this.schemaRoot = schemaRoot; } + } + + // ------------------------- + // Getters / setters + // ------------------------- + + public Map getParsers() { return parsers; } + public void setParsers(Map parsers) { this.parsers = parsers; } + + public Map getProcessors() { return processors; } + public void setProcessors(Map processors) { this.processors = processors; } + + public Map getDetectors() { return detectors; } + public void setDetectors(Map detectors) { this.detectors = detectors; } + + public Map getOutputs() { return outputs; } + public void setOutputs(Map outputs) { this.outputs = outputs; } + + public Map getLifecyclePolicies() { return lifecyclePolicies; } + public void setLifecyclePolicies(Map lifecyclePolicies) { this.lifecyclePolicies = lifecyclePolicies; } + + public Map getBrokers() { return brokers; } + public void setBrokers(Map brokers) { this.brokers = brokers; } + + public IngressDef getIngress() { return ingress; } + public void setIngress(IngressDef ingress) { this.ingress = ingress; } + + public DeadLetterDef getDeadLetter() { return deadLetter; } + public void setDeadLetter(DeadLetterDef deadLetter) { this.deadLetter = deadLetter; } + + public Map getBindings() { return bindings; } + public void setBindings(Map bindings) { this.bindings = bindings; } + + public List getBindingLocations() { return bindingLocations; } + public void setBindingLocations(List bindingLocations) { this.bindingLocations = bindingLocations; } + + public EebusXsdValidation getEebusXsdValidation() { return eebusXsdValidation; } + public void setEebusXsdValidation(EebusXsdValidation eebusXsdValidation) { this.eebusXsdValidation = eebusXsdValidation; } + + // ------------------------- + // Definitions + // ------------------------- + + public static class ParserDef { + /** Parser type discriminator (e.g. JSON_PORTS_BOOLEAN, EEBUS_ALARM_JSON). */ + private String type; + /** Free-form parser-specific configuration. */ + private Map config = new HashMap<>(); + public String getType() { return type; } + public void setType(String type) { this.type = type; } + public Map getConfig() { return config; } + public void setConfig(Map config) { this.config = config; } + } + + public static class ProcessorDef { + /** Processor type discriminator (e.g. JAVASCRIPT_PROCESSOR). */ + private String type; + /** Free-form processor-specific configuration. */ + private Map config = new HashMap<>(); + public String getType() { return type; } + public void setType(String type) { this.type = type; } + public Map getConfig() { return config; } + public void setConfig(Map config) { this.config = config; } + } + + public static class DetectorDef { + /** Detector type discriminator (e.g. PASSTHROUGH_ALARM, EXTERNAL_REST_EVENTS). */ + private String type; + /** Free-form detector-specific configuration. */ + private Map config = new HashMap<>(); + public String getType() { return type; } + public void setType(String type) { this.type = type; } + public Map getConfig() { return config; } + public void setConfig(Map config) { this.config = config; } + } + + public static class OutputDef { + /** Output type discriminator (MQTT, RABBIT). */ + private String type; + /** Formatter id (e.g. EEBUS_ALARM_MESSAGE). */ + private String format; + /** Free-form output-specific configuration. */ + private Map config = new HashMap<>(); + public String getType() { return type; } + public void setType(String type) { this.type = type; } + public String getFormat() { return format; } + public void setFormat(String format) { this.format = format; } + public Map getConfig() { return config; } + public void setConfig(Map config) { this.config = config; } + } + + public static class LifecyclePolicy { + + /** + * Lifecycle policy type. + * + * SIMPLE (default): classic per-faultKey lifecycle (dedup, debounce, transition-only). + * + * COMPOSITE: grouped/composed alarm lifecycle. + * - maintains member state per (groupKey, sourceKey) + * - derives a group alarm state from member states + * - can emit group UPDATE events so receivers can observe member changes + */ + private String type = "SIMPLE"; + + /** + * How lifecycle turns detector events into publishable alarms. + * + * TRANSITION_ONLY (default): + * - emits ALARM only when state goes inactive -> active + * - emits CANCEL only when state goes active -> inactive + * - suppresses duplicates while state is unchanged + * + * FORWARD_ALL: + * - emits EVERY ALARM/CANCEL event immediately + * - still applies optional out-of-order protection + * - dedup/debounce are controlled via dedupWindowMs/debounceMs (set 0 to disable) + * + * Use FORWARD_ALL for "passthrough" bindings where the device already sends + * explicit alarm/cancel messages and you want to forward them as-is. + */ + private String mode = "TRANSITION_ONLY"; + + /** Ignore duplicate events of the same type within this window. Set 0 to disable. */ + private long dedupWindowMs = 10_000; + + /** Suppress alarm<->cancel toggles that happen faster than this. Set 0 to disable. */ + private long debounceMs = 300; + + /** ISO-8601 duration, e.g. PT30M. PT0S disables. */ + private String autoClearTtl = "PT0S"; + + // ------------------------- + // Composite alarm settings (used when type == COMPOSITE) + // ------------------------- + + /** Template for the aggregated/group alarm key, e.g. "${site}|${department}|${function}|${faultDay}". */ + private String groupKeyTemplate; + + /** Template for the member/source key within a group, e.g. "${path}" or "${pin}". */ + private String sourceKeyTemplate = "${path}"; + + /** Emit group UPDATE events (as ALARM action with details.groupEvent=UPDATE) when members change while group stays active. */ + private boolean emitUpdates = true; + + /** Throttle group UPDATE emissions per group. 0 disables throttling. */ + private long updateThrottleMs = 0; + + /** ISO-8601 duration. When no member activity happens for this long, terminate the group instance. PT0S disables. */ + private String idleTimeout = "PT0S"; + + /** ISO-8601 duration. When no *outgoing* group notifications happen for this long, terminate the group instance. PT0S disables. */ + private String noNotificationTimeout = "PT0S"; + + /** ISO-8601 duration. Hard maximum lifetime of a group instance. PT0S disables. */ + private String maxLifetime = "PT0S"; + + /** ISO-8601 duration. If a member/source has not been seen for this long, drop it from the group. PT0S disables. */ + private String sourceTtl = "PT0S"; + + /** ISO-8601 duration. When group becomes inactive (all members cleared), delay emitting CANCEL by this grace period. PT0S disables. */ + private String closeGrace = "PT0S"; + + /** If a group is terminated by timeout while active, emit a CANCEL (groupEvent=TERMINATE) before removing state. */ + private boolean terminateEmitsCancel = true; + + /** Cleanup scheduler interval (ms) for composite state (termination checks, throttled updates, pending closes). */ + private long compositeTickIntervalMs = 30_000; + + public String getType() { return type; } + public void setType(String type) { this.type = type; } + + public String getMode() { return mode; } + public void setMode(String mode) { this.mode = mode; } + + public long getDedupWindowMs() { return dedupWindowMs; } + public void setDedupWindowMs(long dedupWindowMs) { this.dedupWindowMs = dedupWindowMs; } + + public long getDebounceMs() { return debounceMs; } + public void setDebounceMs(long debounceMs) { this.debounceMs = debounceMs; } + + public String getAutoClearTtl() { return autoClearTtl; } + public void setAutoClearTtl(String autoClearTtl) { this.autoClearTtl = autoClearTtl; } + + public String getGroupKeyTemplate() { return groupKeyTemplate; } + public void setGroupKeyTemplate(String groupKeyTemplate) { this.groupKeyTemplate = groupKeyTemplate; } + + public String getSourceKeyTemplate() { return sourceKeyTemplate; } + public void setSourceKeyTemplate(String sourceKeyTemplate) { this.sourceKeyTemplate = sourceKeyTemplate; } + + public boolean isEmitUpdates() { return emitUpdates; } + public void setEmitUpdates(boolean emitUpdates) { this.emitUpdates = emitUpdates; } + + public long getUpdateThrottleMs() { return updateThrottleMs; } + public void setUpdateThrottleMs(long updateThrottleMs) { this.updateThrottleMs = updateThrottleMs; } + + public String getIdleTimeout() { return idleTimeout; } + public void setIdleTimeout(String idleTimeout) { this.idleTimeout = idleTimeout; } + + public String getNoNotificationTimeout() { return noNotificationTimeout; } + public void setNoNotificationTimeout(String noNotificationTimeout) { this.noNotificationTimeout = noNotificationTimeout; } + + public String getMaxLifetime() { return maxLifetime; } + public void setMaxLifetime(String maxLifetime) { this.maxLifetime = maxLifetime; } + + public String getSourceTtl() { return sourceTtl; } + public void setSourceTtl(String sourceTtl) { this.sourceTtl = sourceTtl; } + + public String getCloseGrace() { return closeGrace; } + public void setCloseGrace(String closeGrace) { this.closeGrace = closeGrace; } + + public boolean isTerminateEmitsCancel() { return terminateEmitsCancel; } + public void setTerminateEmitsCancel(boolean terminateEmitsCancel) { this.terminateEmitsCancel = terminateEmitsCancel; } + + public long getCompositeTickIntervalMs() { return compositeTickIntervalMs; } + public void setCompositeTickIntervalMs(long compositeTickIntervalMs) { this.compositeTickIntervalMs = compositeTickIntervalMs; } + } + + public static class BrokerDef { + /** MQTT or RABBIT. */ + private String type; + + /** Connection URL (scheme depends on broker type). */ + private String url; + + /** MQTT clientId (optional). */ + private String clientId; + + /** Optional username (MQTT or Rabbit). */ + private String username; + + /** Optional password (MQTT or Rabbit). */ + private String password; + + /** Optional TLS-related values (paths etc.). */ + private Map tls = new HashMap<>(); + + private boolean automaticReconnect = true; + + private boolean cleanSession = false; + + private boolean cleanStart = false; + + public String getType() { return type; } + public void setType(String type) { this.type = type; } + public String getUrl() { return url; } + public void setUrl(String url) { this.url = url; } + public String getClientId() { return clientId; } + public void setClientId(String clientId) { this.clientId = clientId; } + public String getUsername() { return username; } + public void setUsername(String username) { this.username = username; } + public String getPassword() { return password; } + public void setPassword(String password) { this.password = password; } + public Map getTls() { return tls; } + public void setTls(Map tls) { this.tls = tls; } + + public boolean isAutomaticReconnect() { + return automaticReconnect; + } + + public void setAutomaticReconnect(boolean automaticReconnect) { + this.automaticReconnect = automaticReconnect; + } + + public boolean isCleanSession() { + return cleanSession; + } + + public void setCleanSession(boolean cleanSession) { + this.cleanSession = cleanSession; + } + + public boolean isCleanStart() { + return cleanStart; + } + + public void setCleanStart(boolean cleanStart) { + this.cleanStart = cleanStart; + } + } + + public static class IngressDef { + private MqttIngress mqtt = new MqttIngress(); + private RabbitIngress rabbit = new RabbitIngress(); + + public MqttIngress getMqtt() { return mqtt; } + public void setMqtt(MqttIngress mqtt) { this.mqtt = mqtt; } + + public RabbitIngress getRabbit() { return rabbit; } + public void setRabbit(RabbitIngress rabbit) { this.rabbit = rabbit; } + + /** MQTT ingress using Spring Integration + Paho. */ + public static class MqttIngress { + private boolean enabled = true; + private String brokerRef = "primary"; + /** If null, derived from broker.clientId + "-ingress" */ + private String clientId; + /** Topic filters (MQTT wildcards, NOT regex) to subscribe to. */ + private List subscriptions = new ArrayList<>(List.of("#")); + private int qos = 1; + private int completionTimeoutMs = 30000; + + private Duration inboundStartDelay = Duration.ofSeconds(10); + + public boolean isEnabled() { return enabled; } + public void setEnabled(boolean enabled) { this.enabled = enabled; } + public String getBrokerRef() { return brokerRef; } + public void setBrokerRef(String brokerRef) { this.brokerRef = brokerRef; } + public String getClientId() { return clientId; } + public void setClientId(String clientId) { this.clientId = clientId; } + public List getSubscriptions() { return subscriptions; } + public void setSubscriptions(List subscriptions) { this.subscriptions = subscriptions; } + public int getQos() { return qos; } + public void setQos(int qos) { this.qos = qos; } + public int getCompletionTimeoutMs() { return completionTimeoutMs; } + public void setCompletionTimeoutMs(int completionTimeoutMs) { this.completionTimeoutMs = completionTimeoutMs; } + + public Duration getInboundStartDelay() { + return inboundStartDelay; + } + + public void setInboundStartDelay(Duration inboundStartDelay) { + this.inboundStartDelay = inboundStartDelay; + } + } + + /** RabbitMQ ingress using Spring AMQP listener container. */ + public static class RabbitIngress { + private boolean enabled = false; + private String brokerRef = "rabbit"; + /** Queue names to listen to (exact names). */ + private List queues = new ArrayList<>(); + private int concurrentConsumers = 1; + private int maxConcurrentConsumers = 4; + private int prefetch = 50; + + public boolean isEnabled() { return enabled; } + public void setEnabled(boolean enabled) { this.enabled = enabled; } + public String getBrokerRef() { return brokerRef; } + public void setBrokerRef(String brokerRef) { this.brokerRef = brokerRef; } + public List getQueues() { return queues; } + public void setQueues(List queues) { this.queues = queues; } + public int getConcurrentConsumers() { return concurrentConsumers; } + public void setConcurrentConsumers(int concurrentConsumers) { this.concurrentConsumers = concurrentConsumers; } + public int getMaxConcurrentConsumers() { return maxConcurrentConsumers; } + public void setMaxConcurrentConsumers(int maxConcurrentConsumers) { this.maxConcurrentConsumers = maxConcurrentConsumers; } + public int getPrefetch() { return prefetch; } + public void setPrefetch(int prefetch) { this.prefetch = prefetch; } + } + } + + public static class DeadLetterDef { + private boolean enabled = true; + /** Reference to an output in cep.outputs (recommended). */ + private String outputRef; + /** Optional override message format; if null, uses outputRef.format */ + private String format; + public boolean isEnabled() { return enabled; } + public void setEnabled(boolean enabled) { this.enabled = enabled; } + public String getOutputRef() { return outputRef; } + public void setOutputRef(String outputRef) { this.outputRef = outputRef; } + public String getFormat() { return format; } + public void setFormat(String format) { this.format = format; } + } + + // ------------------------- + // Binding model + // ------------------------- + + public static class SourceBinding { + private boolean enabled = true; + private InDef in; + private ExtractDef extract; + /** If set, this binding uses a processor (parser+detector combined) instead of parserRef+detectorRef. */ + private String processorRef; + private String parserRef; + private String detectorRef; + private String lifecyclePolicyRef; + private CompositeDef composite; + /** Optional per-binding heartbeat settings (periodic state propagation). */ + private HeartbeatDef heartbeat; + private List outputRefs = new ArrayList<>(); + + public boolean isEnabled() { return enabled; } + public void setEnabled(boolean enabled) { this.enabled = enabled; } + public InDef getIn() { return in; } + public void setIn(InDef in) { this.in = in; } + public ExtractDef getExtract() { return extract; } + public void setExtract(ExtractDef extract) { this.extract = extract; } + + public String getProcessorRef() { return processorRef; } + public void setProcessorRef(String processorRef) { this.processorRef = processorRef; } + + public String getParserRef() { return parserRef; } + public void setParserRef(String parserRef) { this.parserRef = parserRef; } + public String getDetectorRef() { return detectorRef; } + public void setDetectorRef(String detectorRef) { this.detectorRef = detectorRef; } + public String getLifecyclePolicyRef() { return lifecyclePolicyRef; } + public void setLifecyclePolicyRef(String lifecyclePolicyRef) { this.lifecyclePolicyRef = lifecyclePolicyRef; } + public CompositeDef getComposite() { return composite; } + public void setComposite(CompositeDef composite) { this.composite = composite; } + + public HeartbeatDef getHeartbeat() { return heartbeat; } + public void setHeartbeat(HeartbeatDef heartbeat) { this.heartbeat = heartbeat; } + + public List getOutputRefs() { return outputRefs; } + public void setOutputRefs(List outputRefs) { this.outputRefs = outputRefs; } + } + + /** + * Optional per-binding heartbeat settings. + * + *

Purpose: periodically re-emit the CURRENT alarm state (ALARM or CANCEL) even when the state is unchanged. + * + *

Config example (per binding YAML): + * 

+   * heartbeat:
+   *   enabled: true
+   *   periodMs: 60000
+   *   includeInactive: false
+   *   initialDelayMs: 60000
+   *
+ */ + public static class HeartbeatDef { + /** If true, heartbeat messages are emitted periodically for this binding. */ + private boolean enabled = false; + + /** Heartbeat interval in milliseconds (must be > 0 when enabled). */ + private long periodMs = 60_000; + + /** If true, emit heartbeats also while inactive (as CANCEL heartbeats). */ + private boolean includeInactive = false; + + /** Optional initial delay for the scheduler (ms). If <= 0, defaults to periodMs. */ + private long initialDelayMs = 0; + + public boolean isEnabled() { return enabled; } + public void setEnabled(boolean enabled) { this.enabled = enabled; } + + public long getPeriodMs() { return periodMs; } + public void setPeriodMs(long periodMs) { this.periodMs = periodMs; } + + public boolean isIncludeInactive() { return includeInactive; } + public void setIncludeInactive(boolean includeInactive) { this.includeInactive = includeInactive; } + + public long getInitialDelayMs() { return initialDelayMs; } + public void setInitialDelayMs(long initialDelayMs) { this.initialDelayMs = initialDelayMs; } + } + + + +/** + * Optional per-binding composite/grouping settings. + * + * Used when lifecycle policy type == COMPOSITE. + * These templates are best defined per binding, because they depend on what vars exist for that binding + * (topic regex groups, parser-enriched vars, etc.). + */ +public static class CompositeDef { + /** Template for aggregated/group alarm key, e.g. "${site}|${department}|${function}|${faultDay}". */ + private String groupKeyTemplate; + /** Template for member/source key within a group, e.g. "${path}" or "${pin}". */ + private String sourceKeyTemplate; + + public String getGroupKeyTemplate() { return groupKeyTemplate; } + public void setGroupKeyTemplate(String groupKeyTemplate) { this.groupKeyTemplate = groupKeyTemplate; } + + public String getSourceKeyTemplate() { return sourceKeyTemplate; } + public void setSourceKeyTemplate(String sourceKeyTemplate) { this.sourceKeyTemplate = sourceKeyTemplate; } +} + public static class InDef { + private InType type; + private MatchDef match; + public InType getType() { return type; } + public void setType(InType type) { this.type = type; } + public MatchDef getMatch() { return match; } + public void setMatch(MatchDef match) { this.match = match; } + } + + public enum InType { MQTT, RABBIT, HTTP } + + public static class MatchDef { + private String topicRegex; + private String queueRegex; + private String pathRegex; + public String getTopicRegex() { return topicRegex; } + public void setTopicRegex(String topicRegex) { this.topicRegex = topicRegex; } + public String getQueueRegex() { return queueRegex; } + public void setQueueRegex(String queueRegex) { this.queueRegex = queueRegex; } + public String getPathRegex() { return pathRegex; } + public void setPathRegex(String pathRegex) { this.pathRegex = pathRegex; } + } + + public static class ExtractDef { + private String tenant; + private String site; + private String department; + private DeviceIdStrategy deviceIdStrategy; + public String getTenant() { return tenant; } + public void setTenant(String tenant) { this.tenant = tenant; } + public String getSite() { return site; } + public void setSite(String site) { this.site = site; } + public String getDepartment() { return department; } + public void setDepartment(String department) { this.department = department; } + public DeviceIdStrategy getDeviceIdStrategy() { return deviceIdStrategy; } + public void setDeviceIdStrategy(DeviceIdStrategy deviceIdStrategy) { this.deviceIdStrategy = deviceIdStrategy; } + } + + public static class DeviceIdStrategy { + /** HASH_OF_TOPIC, FROM_JSON_POINTER, STATIC */ + private String type; + private Map config = new HashMap<>(); + public String getType() { return type; } + public void setType(String type) { this.type = type; } + public Map getConfig() { return config; } + public void setConfig(Map config) { this.config = config; } + } +} diff --git a/src/main/java/at/co/procon/malis/cep/config/CepStartupValidator.java b/src/main/java/at/co/procon/malis/cep/config/CepStartupValidator.java new file mode 100644 index 0000000..2f21bde --- /dev/null +++ b/src/main/java/at/co/procon/malis/cep/config/CepStartupValidator.java @@ -0,0 +1,262 @@ +package at.co.procon.malis.cep.config; + +import org.springframework.boot.ApplicationArguments; +import org.springframework.boot.ApplicationRunner; +import org.springframework.core.Ordered; +import org.springframework.core.annotation.Order; +import org.springframework.stereotype.Component; + +import java.time.Duration; +import java.util.Map; +import java.util.Locale; +import java.util.regex.Pattern; + +/** + * Fail-fast validator for common configuration problems. + * + * This is intentionally strict in the prototype: + * - Missing refs (parserRef/detectorRef/outputRef/brokerRef) should fail startup. + * - Regex patterns should compile at startup (not at first message). + * + * For production, you might want to downgrade some checks to warnings, + * depending on whether you support dynamic config reload. + */ +@Component +@Order(Ordered.HIGHEST_PRECEDENCE + 10) +public class CepStartupValidator implements ApplicationRunner { + + private final CepProperties props; + + public CepStartupValidator(CepProperties props) { + this.props = props; + } + + @Override + public void run(ApplicationArguments args) { + validateBrokers(); + validateIngress(); + validateOutputs(); + validateBindings(); + validateDeadLetter(); + } + + private void validateBrokers() { + for (var e : props.getBrokers().entrySet()) { + String ref = e.getKey(); + CepProperties.BrokerDef b = e.getValue(); + if (b == null) continue; + if (b.getType() == null || b.getType().isBlank()) { + throw new IllegalStateException("Broker '" + ref + "' missing type"); + } + if (b.getUrl() == null || b.getUrl().isBlank()) { + throw new IllegalStateException("Broker '" + ref + "' missing url"); + } + } + } + + private void validateIngress() { + var mqtt = props.getIngress().getMqtt(); + if (mqtt != null && mqtt.isEnabled()) { + assertBrokerType(mqtt.getBrokerRef(), "MQTT"); + } + + var rabbit = props.getIngress().getRabbit(); + if (rabbit != null && rabbit.isEnabled()) { + assertBrokerType(rabbit.getBrokerRef(), "RABBIT"); + if (rabbit.getQueues() == null || rabbit.getQueues().isEmpty()) { + throw new IllegalStateException("Rabbit ingress enabled but no queues configured (cep.ingress.rabbit.queues)"); + } + } + } + + private void validateOutputs() { + for (var e : props.getOutputs().entrySet()) { + String outId = e.getKey(); + CepProperties.OutputDef def = e.getValue(); + if (def == null) continue; + + if (def.getType() == null || def.getType().isBlank()) { + throw new IllegalStateException("Output '" + outId + "' missing type"); + } + if (def.getFormat() == null || def.getFormat().isBlank()) { + throw new IllegalStateException("Output '" + outId + "' missing format"); + } + + Map cfg = def.getConfig(); + if (cfg == null) cfg = Map.of(); + + switch (def.getType()) { + case "MQTT" -> { + // brokerRef is optional for now (we use the single configured mqttClientFactory based on ingress brokerRef). + // Still validate if provided. + Object brokerRef = cfg.get("brokerRef"); + if (brokerRef != null) assertBrokerType(String.valueOf(brokerRef), "MQTT"); + + if (!cfg.containsKey("topicTemplate")) { + throw new IllegalStateException("MQTT output '" + outId + "' missing config.topicTemplate"); + } + } + case "RABBIT" -> { + String brokerRef = String.valueOf(cfg.getOrDefault("brokerRef", "rabbit")); + assertBrokerType(brokerRef, "RABBIT"); + + if (!cfg.containsKey("exchange")) { + throw new IllegalStateException("Rabbit output '" + outId + "' missing config.exchange"); + } + if (!cfg.containsKey("routingKeyTemplate")) { + throw new IllegalStateException("Rabbit output '" + outId + "' missing config.routingKeyTemplate"); + } + } + case "OUTBOX" -> { + if (!cfg.containsKey("correlationVar")) { + throw new IllegalStateException("Rabbit output '" + outId + "' missing config.correlationVar"); + } + } + case "REST" -> { + if (!cfg.containsKey("urlTemplate")) { + throw new IllegalStateException("REST output '" + outId + "' missing config.urlTemplate"); + } + } + default -> throw new IllegalStateException("Unknown output type: " + def.getType() + " (outputId=" + outId + ")"); + } + } + } + + private void validateBindings() { + for (var e : props.getBindings().entrySet()) { + String id = e.getKey(); + var b = e.getValue(); + if (b == null || !b.isEnabled()) continue; + + boolean hasProcessor = b.getProcessorRef() != null && !b.getProcessorRef().isBlank(); + boolean hasParser = b.getParserRef() != null && !b.getParserRef().isBlank(); + boolean hasDetector = b.getDetectorRef() != null && !b.getDetectorRef().isBlank(); + + // Either processorRef OR (parserRef + detectorRef) + if (hasProcessor && (hasParser || hasDetector)) { + throw new IllegalStateException("Binding '" + id + "' must use either processorRef OR parserRef+detectorRef (not both)"); + } + if (hasProcessor) { + if (props.getProcessors() == null || !props.getProcessors().containsKey(b.getProcessorRef())) { + throw new IllegalStateException("Binding '" + id + "' references unknown processorRef: " + b.getProcessorRef()); + } + } else { + if (!hasParser || props.getParsers() == null || !props.getParsers().containsKey(b.getParserRef())) { + throw new IllegalStateException("Binding '" + id + "' references unknown parserRef: " + b.getParserRef()); + } + if (!hasDetector || props.getDetectors() == null || !props.getDetectors().containsKey(b.getDetectorRef())) { + throw new IllegalStateException("Binding '" + id + "' references unknown detectorRef: " + b.getDetectorRef()); + } + } + if (b.getLifecyclePolicyRef() != null && !props.getLifecyclePolicies().containsKey(b.getLifecyclePolicyRef())) { + throw new IllegalStateException("Binding '" + id + "' references unknown lifecyclePolicyRef: " + b.getLifecyclePolicyRef()); + } + + // Validate lifecycle policy content (especially COMPOSITE settings). + CepProperties.LifecyclePolicy pol = resolvePolicy(b); + validateLifecyclePolicy(pol, b.getLifecyclePolicyRef(), id, b); + + // Validate heartbeat settings (optional, per binding) + if (b.getHeartbeat() != null && b.getHeartbeat().isEnabled()) { + if (b.getHeartbeat().getPeriodMs() <= 0) { + throw new IllegalStateException("Binding '" + id + "' heartbeat.enabled=true but periodMs <= 0"); + } + if (b.getHeartbeat().getPeriodMs() < 1000) { + throw new IllegalStateException("Binding '" + id + "' heartbeat.periodMs must be >= 1000ms"); + } + if (b.getHeartbeat().getInitialDelayMs() < 0) { + throw new IllegalStateException("Binding '" + id + "' heartbeat.initialDelayMs must be >= 0"); + } + } + + for (String outRef : b.getOutputRefs()) { + if (!props.getOutputs().containsKey(outRef)) { + throw new IllegalStateException("Binding '" + id + "' references unknown outputRef: " + outRef); + } + } + + if (b.getIn() == null || b.getIn().getType() == null || b.getIn().getMatch() == null) { + throw new IllegalStateException("Binding '" + id + "' missing in/type/match"); + } + String regex = switch (b.getIn().getType()) { + case MQTT -> b.getIn().getMatch().getTopicRegex(); + case RABBIT -> b.getIn().getMatch().getQueueRegex(); + case HTTP -> b.getIn().getMatch().getPathRegex(); + }; + if (regex == null || regex.isBlank()) { + throw new IllegalStateException("Binding '" + id + "' missing regex for type " + b.getIn().getType()); + } + // compile to validate + Pattern.compile(regex); + } + } + + private void validateDeadLetter() { + if (props.getDeadLetter() != null && props.getDeadLetter().isEnabled() && props.getDeadLetter().getOutputRef() != null) { + if (!props.getOutputs().containsKey(props.getDeadLetter().getOutputRef())) { + throw new IllegalStateException("DLQ references unknown outputRef: " + props.getDeadLetter().getOutputRef()); + } + } + } + + private void assertBrokerType(String brokerRef, String expectedType) { + CepProperties.BrokerDef b = props.getBrokers().get(brokerRef); + if (b == null) throw new IllegalStateException("Unknown brokerRef: " + brokerRef); + if (b.getType() == null || !b.getType().equalsIgnoreCase(expectedType)) { + throw new IllegalStateException("brokerRef '" + brokerRef + "' is not of type " + expectedType); + } + } + + private CepProperties.LifecyclePolicy resolvePolicy(CepProperties.SourceBinding binding) { + if (binding == null) return props.getLifecyclePolicies().getOrDefault("default", new CepProperties.LifecyclePolicy()); + String ref = binding.getLifecyclePolicyRef(); + if (ref != null && props.getLifecyclePolicies().containsKey(ref)) { + return props.getLifecyclePolicies().get(ref); + } + return props.getLifecyclePolicies().getOrDefault("default", new CepProperties.LifecyclePolicy()); + } + + private void validateLifecyclePolicy(CepProperties.LifecyclePolicy p, String ref, String bindingId, CepProperties.SourceBinding binding) { + if (p == null) return; + String type = p.getType() == null ? "SIMPLE" : p.getType().trim().toUpperCase(Locale.ROOT); + if (!type.equals("SIMPLE") && !type.equals("COMPOSITE")) { + throw new IllegalStateException("LifecyclePolicy has invalid type='" + p.getType() + "' (binding=" + bindingId + ", ref=" + ref + ")"); + } + + if (type.equals("COMPOSITE")) { + String groupTpl = null; + String sourceTpl = null; + if (binding != null && binding.getComposite() != null) { + groupTpl = binding.getComposite().getGroupKeyTemplate(); + sourceTpl = binding.getComposite().getSourceKeyTemplate(); + } + if (groupTpl == null || groupTpl.isBlank()) groupTpl = p.getGroupKeyTemplate(); + if (sourceTpl == null || sourceTpl.isBlank()) sourceTpl = p.getSourceKeyTemplate(); + + // groupKeyTemplate is required (per binding is preferred; policy can provide a default for backward compatibility). + if (groupTpl == null || groupTpl.isBlank()) { + throw new IllegalStateException("COMPOSITE lifecycle requires groupKeyTemplate (set binding.composite.groupKeyTemplate, or lifecyclePolicies[ref].groupKeyTemplate as default) (binding=" + bindingId + ", ref=" + ref + ")"); + } + // sourceKeyTemplate is optional; if absent we default to ${path} at runtime. + + // Validate ISO durations if present. + parseDurationOrThrow(p.getIdleTimeout(), "idleTimeout", bindingId, ref); + parseDurationOrThrow(p.getNoNotificationTimeout(), "noNotificationTimeout", bindingId, ref); + parseDurationOrThrow(p.getMaxLifetime(), "maxLifetime", bindingId, ref); + parseDurationOrThrow(p.getSourceTtl(), "sourceTtl", bindingId, ref); + parseDurationOrThrow(p.getCloseGrace(), "closeGrace", bindingId, ref); + if (p.getCompositeTickIntervalMs() < 1000) { + throw new IllegalStateException("COMPOSITE compositeTickIntervalMs must be >= 1000ms (binding=" + bindingId + ", ref=" + ref + ")"); + } + } + } + + private void parseDurationOrThrow(String iso, String field, String bindingId, String ref) { + if (iso == null || iso.isBlank()) return; + try { + Duration.parse(iso.trim()); + } catch (Exception ex) { + throw new IllegalStateException("Invalid ISO-8601 duration for " + field + "='" + iso + "' (binding=" + bindingId + ", ref=" + ref + ")"); + } + } +} \ No newline at end of file diff --git a/src/main/java/at/co/procon/malis/cep/detector/DetectionContext.java b/src/main/java/at/co/procon/malis/cep/detector/DetectionContext.java new file mode 100644 index 0000000..f30af4a --- /dev/null +++ b/src/main/java/at/co/procon/malis/cep/detector/DetectionContext.java @@ -0,0 +1,28 @@ +package at.co.procon.malis.cep.detector; + +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.Map; + +/** + * Per-message context passed to detectors. + * + *

Contains bindingId and resolved variables (tenant/site/department/deviceId, etc.). + */ +public final class DetectionContext { + + private final String bindingId; + private final Map vars; + + public DetectionContext(String bindingId, Map vars) { + this.bindingId = bindingId; + this.vars = vars == null ? Map.of() : Collections.unmodifiableMap(new LinkedHashMap<>(vars)); + } + + public String getBindingId() { return bindingId; } + public Map getVars() { return vars; } + + // record-style accessors + public String bindingId() { return bindingId; } + public Map vars() { return vars; } +} diff --git a/src/main/java/at/co/procon/malis/cep/detector/Detector.java b/src/main/java/at/co/procon/malis/cep/detector/Detector.java new file mode 100644 index 0000000..fc79314 --- /dev/null +++ b/src/main/java/at/co/procon/malis/cep/detector/Detector.java @@ -0,0 +1,9 @@ +package at.co.procon.malis.cep.detector; + +import at.co.procon.malis.cep.parser.ParsedInput; + +import java.util.List; + +public interface Detector { + List detect(ParsedInput input, DetectionContext ctx) throws Exception; +} diff --git a/src/main/java/at/co/procon/malis/cep/detector/DetectorEvent.java b/src/main/java/at/co/procon/malis/cep/detector/DetectorEvent.java new file mode 100644 index 0000000..c66150a --- /dev/null +++ b/src/main/java/at/co/procon/malis/cep/detector/DetectorEvent.java @@ -0,0 +1,43 @@ +package at.co.procon.malis.cep.detector; + +import java.time.Instant; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.Map; + +/** + * Output of a detector. + * + *

Important: external detectors return events only (ALARM/CANCEL), not state. + * State (dedup, debounce, open/closed lifecycle) is handled by {@link at.co.procon.malis.cep.lifecycle.AlarmLifecycleService}. + */ +public final class DetectorEvent { + + private final String faultKey; + /** ALARM or CANCEL */ + private final String eventType; + private final String severity; + private final Instant occurredAt; + private final Map details; + + public DetectorEvent(String faultKey, String eventType, String severity, Instant occurredAt, Map details) { + this.faultKey = faultKey; + this.eventType = eventType; + this.severity = severity; + this.occurredAt = occurredAt == null ? Instant.now() : occurredAt; + this.details = details == null ? Map.of() : Collections.unmodifiableMap(new LinkedHashMap<>(details)); + } + + public String getFaultKey() { return faultKey; } + public String getEventType() { return eventType; } + public String getSeverity() { return severity; } + public Instant getOccurredAt() { return occurredAt; } + public Map getDetails() { return details; } + + // record-style accessors + public String faultKey() { return faultKey; } + public String eventType() { return eventType; } + public String severity() { return severity; } + public Instant occurredAt() { return occurredAt; } + public Map details() { return details; } +} diff --git a/src/main/java/at/co/procon/malis/cep/detector/DetectorFactory.java b/src/main/java/at/co/procon/malis/cep/detector/DetectorFactory.java new file mode 100644 index 0000000..4f7ad63 --- /dev/null +++ b/src/main/java/at/co/procon/malis/cep/detector/DetectorFactory.java @@ -0,0 +1,82 @@ +package at.co.procon.malis.cep.detector; + +import at.co.procon.malis.cep.config.CepProperties; + +import java.util.LinkedHashMap; +import java.util.Map; + +public class DetectorFactory { + + public Map buildAll(Map defs) { + Map out = new LinkedHashMap<>(); + for (var e : defs.entrySet()) { + out.put(e.getKey(), buildOne(e.getKey(), e.getValue())); + } + return out; + } + + public Detector buildOne(String id, CepProperties.DetectorDef def) { + String type = def.getType(); + Map cfg = def.getConfig(); + + return switch (type) { + case "PASSTHROUGH_ALARM" -> new PassthroughAlarmDetector(); + case "EXTERNAL_REST_EVENTS" -> { + String baseUrl = String.valueOf(cfg.get("baseUrl")); + String path = String.valueOf(cfg.getOrDefault("path", "/")); + int timeoutMs = ((Number) cfg.getOrDefault("timeoutMs", 1500)).intValue(); + String reqF = String.valueOf(cfg.getOrDefault("requestFormat", "DATAGRAM_XML")); + String respF = String.valueOf(cfg.getOrDefault("responseFormat", "JSON_EVENTS")); + ExternalRestEventsDetector.RequestFormat requestFormat = ExternalRestEventsDetector.RequestFormat.valueOf(reqF); + ExternalRestEventsDetector.ResponseFormat responseFormat = ExternalRestEventsDetector.ResponseFormat.valueOf(respF); + yield new ExternalRestEventsDetector(baseUrl, path, timeoutMs, requestFormat, responseFormat); + } + case "EXPRESSION_RULES" -> { + // Config schema (example): + // missingPolicy: FALSE|SKIP|ERROR + // rules: + // - id: port_any_open + // expression: "s['port.1.state'] == false || s['port.2.state'] == false" + // severity: MAJOR + // emitCancel: true + + String mp = String.valueOf(cfg.getOrDefault("missingPolicy", "FALSE")); + ExpressionRulesDetector.MissingPolicy missingPolicy = ExpressionRulesDetector.MissingPolicy.valueOf(mp.trim().toUpperCase()); + + Object rulesObj = cfg.get("rules"); + if (!(rulesObj instanceof java.util.Map it)) { + throw new IllegalArgumentException("EXPRESSION_RULES detector requires config.rules as a list (id=" + id + ")"); + } + + var spel = new org.springframework.expression.spel.standard.SpelExpressionParser(); + java.util.List rules = new java.util.ArrayList<>(); + + int idx = 0; + for (Object o : it.values()) { + idx++; + if (!(o instanceof java.util.Map m)) continue; + + String ruleId = m.get("id") != null ? String.valueOf(m.get("id")) : ("rule" + idx); + String expr = m.get("expression") != null ? String.valueOf(m.get("expression")) : null; + if (expr == null || expr.isBlank()) { + throw new IllegalArgumentException("EXPRESSION_RULES rule is missing 'expression' (detector=" + id + ", rule=" + ruleId + ")"); + } + + String severity = m.get("severity") != null ? String.valueOf(m.get("severity")) : "UNKNOWN"; + boolean emitCancel = m.get("emitCancel") == null || Boolean.parseBoolean(String.valueOf(m.get("emitCancel"))); + + var compiled = spel.parseExpression(expr); + rules.add(new ExpressionRulesDetector.Rule(ruleId, expr, compiled, severity, emitCancel)); + } + + yield new ExpressionRulesDetector(rules, missingPolicy); + } + case "PIN_BAD_STATE" -> { + String faultKeyTemplate = cfg.get("faultKeyTemplate") != null ? String.valueOf(cfg.get("faultKeyTemplate")) : null; + String severity = cfg.get("severity") != null ? String.valueOf(cfg.get("severity")) : "MAJOR"; + yield new PinBadStateDetector(faultKeyTemplate, severity); + } + default -> throw new IllegalArgumentException("Unknown detector type: " + type + " (id=" + id + ")"); + }; + } +} diff --git a/src/main/java/at/co/procon/malis/cep/detector/DetectorRegistry.java b/src/main/java/at/co/procon/malis/cep/detector/DetectorRegistry.java new file mode 100644 index 0000000..cfc28fb --- /dev/null +++ b/src/main/java/at/co/procon/malis/cep/detector/DetectorRegistry.java @@ -0,0 +1,22 @@ +package at.co.procon.malis.cep.detector; + +import at.co.procon.malis.cep.config.CepProperties; +import org.springframework.stereotype.Component; + +import java.util.Map; + +@Component +public class DetectorRegistry { + + private final Map detectors; + + public DetectorRegistry(CepProperties props) { + this.detectors = new DetectorFactory().buildAll(props.getDetectors()); + } + + public Detector get(String id) { + Detector d = detectors.get(id); + if (d == null) throw new IllegalStateException("Unknown detectorRef: " + id); + return d; + } +} diff --git a/src/main/java/at/co/procon/malis/cep/detector/ExpressionRulesDetector.java b/src/main/java/at/co/procon/malis/cep/detector/ExpressionRulesDetector.java new file mode 100644 index 0000000..51b24b3 --- /dev/null +++ b/src/main/java/at/co/procon/malis/cep/detector/ExpressionRulesDetector.java @@ -0,0 +1,209 @@ +package at.co.procon.malis.cep.detector; + +import at.co.procon.malis.cep.event.CepEvent; +import at.co.procon.malis.cep.event.EventBatch; +import at.co.procon.malis.cep.event.SignalUpdateEvent; +import at.co.procon.malis.cep.event.convert.EebusMeasurementDatagramConverter; +import at.co.procon.malis.cep.event.convert.InputEventConverter; +import at.co.procon.malis.cep.event.convert.MeasurementSetConverter; +import at.co.procon.malis.cep.parser.ParsedInput; +import org.springframework.expression.Expression; +import org.springframework.expression.ExpressionParser; +import org.springframework.expression.spel.standard.SpelExpressionParser; +import org.springframework.expression.spel.support.SimpleEvaluationContext; + +import java.time.Instant; +import java.util.*; +import java.util.concurrent.ConcurrentHashMap; + +/** + * Detector: EXPRESSION_RULES + * + *

Generic internal detector that: + *

    + *
  1. Converts incoming parsed bodies (EEBUS datagram / MeasurementSet / already-built EventBatch) into a canonical {@link EventBatch}
    2. + *
  2. Maintains an in-memory per-source signal store (latest values per key) partitioned by {@code sourceKey}
    3. + *
  3. Evaluates configured boolean expressions over the signal store
    4. + *
  4. Emits ALARM (true) or CANCEL (false) events for each rule
    5. + *
+ * + *

Expressions are evaluated using Spring Expression Language (SpEL) in a restricted context. + * The evaluation root exposes: + *

    + *
  • {@code s}: Map<String,Object> latest signal values (by key)
    • + *
  • {@code vars}: Map<String,String> binding variables
    • + *
  • {@code sourceKey}: String (partition key)
    • + *
  • {@code deviceId}: String (optional, best-effort)
    • + *
  • {@code now}: Instant
    • + *
+ * + * Example expressions: + * 
+ *   s['port.1.state'] == false || s['port.2.state'] == false
+ *   (s['p_total'] ?: 0) > 1000 && (s['voltageA'] ?: 0) < 210
+ *
+ */ +public final class ExpressionRulesDetector implements Detector { + + public enum MissingPolicy { + /** Evaluation errors (missing keys / type mismatch) result in false. */ + FALSE, + /** Evaluation errors suppress output for that rule on this tick. */ + SKIP, + /** Evaluation errors throw and fail the pipeline. */ + ERROR + } + + public static final class Rule { + private final String id; + private final String expression; + private final Expression compiled; + private final String severity; + private final boolean emitCancel; + + public Rule(String id, String expression, Expression compiled, String severity, boolean emitCancel) { + this.id = Objects.requireNonNull(id, "id"); + this.expression = Objects.requireNonNull(expression, "expression"); + this.compiled = Objects.requireNonNull(compiled, "compiled"); + this.severity = severity == null ? "UNKNOWN" : severity; + this.emitCancel = emitCancel; + } + + public String getId() { return id; } + public String getExpression() { return expression; } + public Expression getCompiled() { return compiled; } + public String getSeverity() { return severity; } + public boolean isEmitCancel() { return emitCancel; } + } + + /** Per sourceKey: signalKey -> latest value */ + private final ConcurrentHashMap> store = new ConcurrentHashMap<>(); + + private final List converters; + private final List rules; + private final MissingPolicy missingPolicy; + + private final ExpressionParser parser = new SpelExpressionParser(); + + public ExpressionRulesDetector(List rules, MissingPolicy missingPolicy) { + this.rules = rules == null ? List.of() : List.copyOf(rules); + this.missingPolicy = missingPolicy == null ? MissingPolicy.FALSE : missingPolicy; + + // Built-in converters. Add more later (e.g. other protocols -> EventBatch). + this.converters = List.of( + new EebusMeasurementDatagramConverter(), + new MeasurementSetConverter() + ); + } + + @Override + public List detect(ParsedInput input, DetectionContext ctx) throws Exception { + EventBatch batch = toEventBatch(input, ctx); + + String deviceId = batch.getDeviceId() != null ? batch.getDeviceId() : ctx.getVars().get("deviceId"); + String sourceKey = batch.getSourceKey() != null ? batch.getSourceKey() : ctx.getVars().get("sourceKey"); + if (sourceKey == null || sourceKey.isBlank()) { + sourceKey = (deviceId == null || deviceId.isBlank()) ? "unknown-source" : deviceId; + } + + // Update store with the new signal values. + ConcurrentHashMap deviceSignals = store.computeIfAbsent(sourceKey, k -> new ConcurrentHashMap<>()); + + Map batchUpdates = new LinkedHashMap<>(); + for (CepEvent ev : batch.getEvents()) { + if (ev instanceof SignalUpdateEvent su) { + Object v = su.getValue() == null ? null : su.getValue().getValue(); + deviceSignals.put(su.getKey(), v); + batchUpdates.put(su.getKey(), v); + } + } + + EvalRoot root = new EvalRoot(Collections.unmodifiableMap(deviceSignals), ctx.getVars(), sourceKey, deviceId, Instant.now()); + var evalCtx = SimpleEvaluationContext.forReadOnlyDataBinding().build(); + + List out = new ArrayList<>(); + for (Rule r : rules) { + Boolean result; + try { + Object v = r.getCompiled().getValue(evalCtx, root); + result = (v instanceof Boolean b) ? b : null; + } catch (Exception ex) { + if (missingPolicy == MissingPolicy.ERROR) throw ex; + if (missingPolicy == MissingPolicy.SKIP) continue; + result = Boolean.FALSE; + } + + boolean active = Boolean.TRUE.equals(result); + String eventType = active ? "ALARM" : "CANCEL"; + if (!active && !r.isEmitCancel()) continue; + + String faultKey = buildFaultKey(ctx, sourceKey, r.getId()); + + Map details = new LinkedHashMap<>(); + details.put("source", "expressionRules"); + details.put("ruleId", r.getId()); + details.put("expression", r.getExpression()); + details.put("result", active); + details.put("sourceKey", sourceKey); + if (deviceId != null) details.put("deviceId", deviceId); + details.put("updates", batchUpdates); + + out.add(new DetectorEvent(faultKey, eventType, r.getSeverity(), batch.getTimestamp(), details)); + } + + return out; + } + + private EventBatch toEventBatch(ParsedInput input, DetectionContext ctx) throws Exception { + Object body = input.getBody(); + if (body instanceof EventBatch eb) return eb; + + for (InputEventConverter c : converters) { + if (c.supports(body)) return c.convert(input, ctx); + } + + throw new IllegalArgumentException("EXPRESSION_RULES expects EventBatch, MeasurementSet, or EebusMeasurementDatagram but got: " + + (body == null ? "null" : body.getClass().getName())); + } + + private static String buildFaultKey(DetectionContext ctx, String sourceKey, String ruleId) { + String tenant = ctx.getVars().getOrDefault("tenant", "na"); + String site = ctx.getVars().getOrDefault("site", "na"); + return tenant + ":" + site + ":" + ctx.getBindingId() + ":" + sourceKey + ":RULE:" + ruleId; + } + + /** + * Evaluation root for SpEL. + * + *

Exposes simple getters so expressions can access properties without method invocation. + */ + public static final class EvalRoot { + private final Map s; + private final Map vars; + private final String sourceKey; + private final String deviceId; + private final Instant now; + + public EvalRoot(Map s, Map vars, String sourceKey, String deviceId, Instant now) { + this.s = s; + this.vars = vars; + this.sourceKey = sourceKey; + this.deviceId = deviceId; + this.now = now; + } + + public Map getS() { return s; } + public Map getVars() { return vars; } + public String getSourceKey() { return sourceKey; } + public String getDeviceId() { return deviceId; } + public Instant getNow() { return now; } + } + + /** + * Helper used by the factory to compile a rule. + */ + public Rule compileRule(String id, String expression, String severity, boolean emitCancel) { + Expression ex = parser.parseExpression(expression); + return new Rule(id, expression, ex, severity, emitCancel); + } +} diff --git a/src/main/java/at/co/procon/malis/cep/detector/ExternalRestEventsDetector.java b/src/main/java/at/co/procon/malis/cep/detector/ExternalRestEventsDetector.java new file mode 100644 index 0000000..92b6aa1 --- /dev/null +++ b/src/main/java/at/co/procon/malis/cep/detector/ExternalRestEventsDetector.java @@ -0,0 +1,312 @@ +package at.co.procon.malis.cep.detector; + +import at.co.procon.malis.cep.eebus.EebusMeasurementDatagram; +import at.co.procon.malis.cep.eebus.EebusSpineConstants; +import at.co.procon.malis.cep.eebus.EebusXmlUtil; +import at.co.procon.malis.cep.parser.MeasurementSet; +import at.co.procon.malis.cep.parser.MeasurementValue; +import at.co.procon.malis.cep.parser.ParsedInput; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.w3c.dom.Document; +import org.w3c.dom.Node; +import org.w3c.dom.NodeList; + +import javax.xml.xpath.XPath; +import javax.xml.xpath.XPathConstants; +import javax.xml.xpath.XPathFactory; +import java.net.URI; +import java.nio.charset.StandardCharsets; +import java.time.Instant; +import java.util.*; + +/** + * Detector: EXTERNAL_REST_EVENTS + * + *

This detector calls a REST service for fault recognition. + * The external service returns events only (ALARM/CANCEL), while the caller (this CEP) manages state. + * + *

Request formats

+ *
    + *
  • DATAGRAM_XML (recommended): send the full EEBUS SPINE-like <Datagram> unchanged
    • + *
  • LISTS_WRAPPED_XML: wrap extracted <measurementDescriptionListData> + <measurementListData> into a small XML document
    • + *
  • JSON_PORTS_LIST: legacy/demo JSON payload for potential-free contacts (pin list)
    • + *
+ * + *

Response formats

+ *
    + *
  • JSON_EVENTS: expected keys: "eebus_alarms" or "events" containing ALARM/CANCEL entries
    • + *
  • ALARM_DATAGRAM_XML: response is a full <Datagram> containing alarmListData/alarmData entries
    • + *
+ * + *

Bindings decide which detector is used for a given event source. + * This class is intentionally tolerant: it can consume either {@link EebusMeasurementDatagram} (XML) + * or {@link MeasurementSet} (canonical JSON ports / other measurements). + */ +public class ExternalRestEventsDetector implements Detector { + + public enum RequestFormat { + DATAGRAM_XML, + LISTS_WRAPPED_XML, + JSON_PORTS_LIST + } + + public enum ResponseFormat { + JSON_EVENTS, + ALARM_DATAGRAM_XML + } + + private final String baseUrl; + private final String path; + private final int timeoutMs; + private final RequestFormat requestFormat; + private final ResponseFormat responseFormat; + + private final ObjectMapper om = new ObjectMapper(); + private final java.net.http.HttpClient http; + + public ExternalRestEventsDetector(String baseUrl, + String path, + int timeoutMs, + RequestFormat requestFormat, + ResponseFormat responseFormat) { + this.baseUrl = Objects.requireNonNull(baseUrl, "baseUrl"); + this.path = Objects.requireNonNull(path, "path"); + this.timeoutMs = timeoutMs; + this.requestFormat = requestFormat == null ? RequestFormat.DATAGRAM_XML : requestFormat; + this.responseFormat = responseFormat == null ? ResponseFormat.JSON_EVENTS : responseFormat; + + this.http = java.net.http.HttpClient.newBuilder() + .connectTimeout(java.time.Duration.ofMillis(timeoutMs)) + .build(); + } + + @Override + public List detect(ParsedInput input, DetectionContext ctx) throws Exception { + RequestPayload req = buildRequestPayload(input, ctx); + + var httpReq = java.net.http.HttpRequest.newBuilder() + .uri(URI.create(baseUrl + path)) + .timeout(java.time.Duration.ofMillis(timeoutMs)) + .header("Content-Type", req.contentType) + .POST(java.net.http.HttpRequest.BodyPublishers.ofByteArray(req.bodyBytes)) + .build(); + + var resp = http.send(httpReq, java.net.http.HttpResponse.BodyHandlers.ofByteArray()); + if (resp.statusCode() / 100 != 2) { + String bodyStr = new String(resp.body() == null ? new byte[0] : resp.body(), StandardCharsets.UTF_8); + throw new IllegalStateException("Detector returned HTTP " + resp.statusCode() + ": " + bodyStr); + } + + return switch (responseFormat) { + case JSON_EVENTS -> parseJsonEvents(resp.body(), input, ctx); + case ALARM_DATAGRAM_XML -> parseAlarmDatagramXml(resp.body(), ctx); + }; + } + + /** + * Build the payload that will be posted to the external detector. + */ + private RequestPayload buildRequestPayload(ParsedInput input, DetectionContext ctx) throws Exception { + Object body = input.getBody(); + + if (requestFormat == RequestFormat.DATAGRAM_XML) { + if (!(body instanceof EebusMeasurementDatagram md)) { + throw new IllegalArgumentException("DATAGRAM_XML requestFormat expects EebusMeasurementDatagram (got " + (body == null ? "null" : body.getClass()) + ")"); + } + return new RequestPayload("application/xml", md.getDatagramXml().getBytes(StandardCharsets.UTF_8)); + } + + if (requestFormat == RequestFormat.LISTS_WRAPPED_XML) { + if (!(body instanceof EebusMeasurementDatagram md)) { + throw new IllegalArgumentException("LISTS_WRAPPED_XML requestFormat expects EebusMeasurementDatagram"); + } + String wrapper = buildListsWrapperXml(md); + return new RequestPayload("application/xml", wrapper.getBytes(StandardCharsets.UTF_8)); + } + + // Legacy/demo JSON pin list + if (requestFormat == RequestFormat.JSON_PORTS_LIST) { + if (!(body instanceof MeasurementSet ms)) { + throw new IllegalArgumentException("JSON_PORTS_LIST requestFormat expects MeasurementSet"); + } + String json = om.writeValueAsString(buildPortsJsonRequest(ms)); + return new RequestPayload("application/json", json.getBytes(StandardCharsets.UTF_8)); + } + + throw new IllegalStateException("Unsupported requestFormat: " + requestFormat); + } + + /** + * Wrapper option: include both measurement lists into a small XML root. + * + *

Some detectors do not want the full SPINE datagram and only care about the list payload. + * This wrapper keeps namespace information while reducing irrelevant header noise. + */ + static String buildListsWrapperXml(EebusMeasurementDatagram md) { + StringBuilder xml = new StringBuilder(1024); + xml.append("\n"); + if (md.getMeasurementDescriptionListDataXml() != null && !md.getMeasurementDescriptionListDataXml().isBlank()) { + xml.append(" ").append(md.getMeasurementDescriptionListDataXml()).append("\n"); + } + if (md.getMeasurementListDataXml() != null && !md.getMeasurementListDataXml().isBlank()) { + xml.append(" ").append(md.getMeasurementListDataXml()).append("\n"); + } + xml.append("\n"); + return xml.toString(); + } + + /** + * Build the JSON request expected by the demo Python "ports" server. + */ + private static Map buildPortsJsonRequest(MeasurementSet ms) { + List> measurementDataList = new ArrayList<>(); + for (MeasurementValue mv : ms.getMeasurements()) { + Integer pinId = tryParsePinId(mv.getPath()); + if (pinId == null) continue; + if (!(mv.getValue() instanceof Boolean b)) continue; + measurementDataList.add(Map.of("pin_id", pinId, "value", b)); + } + + // The demo server ignores definitions, but we keep the field for forward compatibility. + return Map.of( + "measurement_data_list", measurementDataList, + "measurement_definition_list", List.of() + ); + } + + /** + * Parse JSON response into detector events. + */ + private List parseJsonEvents(byte[] bodyBytes, ParsedInput input, DetectionContext ctx) throws Exception { + Map root = om.readValue(bodyBytes, Map.class); + Object eventsObj = root.containsKey("eebus_alarms") ? root.get("eebus_alarms") : root.get("events"); + if (!(eventsObj instanceof List events)) return List.of(); + + List out = new ArrayList<>(); + for (Object evObj : events) { + if (!(evObj instanceof Map ev)) continue; + + // The external service may provide a fully computed faultKey. + String faultKey = ev.get("faultKey") != null ? String.valueOf(ev.get("faultKey")) : null; + + // Otherwise compute a stable key based on binding and an identifier (pin_id / alarmCode / ...) + if (faultKey == null || faultKey.isBlank()) { + String pinPart = ev.get("pin_id") != null ? ("PIN:" + ev.get("pin_id")) : null; + String codePart = ev.get("alarmCode") != null ? String.valueOf(ev.get("alarmCode")) : null; + String idPart = pinPart != null ? pinPart : (codePart != null ? codePart : "UNKNOWN"); + + String tenant = ctx.getVars().getOrDefault("tenant", "default"); + String site = ctx.getVars().getOrDefault("site", "unknown-site"); + + String deviceId = ctx.getVars().get("deviceId"); + if (input.getBody() instanceof MeasurementSet ms && ms.getDeviceId() != null) { + deviceId = ms.getDeviceId(); + } + + faultKey = tenant + ":" + site + ":" + ctx.getBindingId() + ":" + deviceId + ":" + idPart; + } + + String eventType = normalizeEventType(ev); + String severity = ev.get("severity") != null ? String.valueOf(ev.get("severity")) : "UNKNOWN"; + + Map details = new LinkedHashMap<>(); + for (var entry : ev.entrySet()) { + details.put(String.valueOf(entry.getKey()), entry.getValue()); + } + + out.add(new DetectorEvent(faultKey, eventType, severity, Instant.now(), details)); + } + return out; + } + + private static String normalizeEventType(Map ev) { + // Accept multiple conventions + String t = null; + if (ev.get("eventType") != null) t = String.valueOf(ev.get("eventType")); + if (t == null && ev.get("action") != null) t = String.valueOf(ev.get("action")); + if (t == null && ev.get("alarm_type") != null) t = String.valueOf(ev.get("alarm_type")); + if (t == null && ev.get("alarmType") != null) t = String.valueOf(ev.get("alarmType")); + + t = t == null ? "ALARM" : t.toUpperCase(Locale.ROOT); + return t.contains("CANCEL") || t.contains("DELETE") ? "CANCEL" : "ALARM"; + } + + /** + * Parse an EEBUS alarm datagram XML response into detector events. + */ + private List parseAlarmDatagramXml(byte[] bodyBytes, DetectionContext ctx) throws Exception { + Document doc = EebusXmlUtil.parse(bodyBytes); + + XPath xp = XPathFactory.newInstance().newXPath(); + NodeList alarmNodes = (NodeList) xp.evaluate( + "//*[local-name()='alarmListData']//*[local-name()='alarmData']", + doc, + XPathConstants.NODESET + ); + + String classifier = EebusXmlUtil.extractFirstText(doc, "cmdClassifier"); + boolean classifierCancel = classifier != null && classifier.toLowerCase().contains("delete"); + + List out = new ArrayList<>(); + for (int i = 0; i < alarmNodes.getLength(); i++) { + Node alarmData = alarmNodes.item(i); + + String alarmId = text(xp, alarmData, ".//*[local-name()='alarmId']"); + String alarmType = text(xp, alarmData, ".//*[local-name()='alarmType']"); + String alarmAction = text(xp, alarmData, ".//*[local-name()='alarmAction']"); + + String eventType; + if (alarmAction != null && alarmAction.toUpperCase(Locale.ROOT).contains("CANCEL")) eventType = "CANCEL"; + else if (classifierCancel) eventType = "CANCEL"; + else eventType = "ALARM"; + + String code = (alarmType != null && !alarmType.isBlank()) ? alarmType : (alarmId != null ? alarmId : "UNKNOWN"); + + String tenant = ctx.getVars().getOrDefault("tenant", "default"); + String site = ctx.getVars().getOrDefault("site", "unknown-site"); + String deviceId = ctx.getVars().getOrDefault("deviceId", "unknown-device"); + + String faultKey = tenant + ":" + site + ":" + ctx.getBindingId() + ":" + deviceId + ":" + code; + + out.add(new DetectorEvent( + faultKey, + eventType, + "UNKNOWN", + Instant.now(), + Map.of("source", "externalRest", "alarmCode", code) + )); + } + return out; + } + + private static String text(XPath xp, Node ctx, String expr) throws Exception { + Node n = (Node) xp.evaluate(expr, ctx, XPathConstants.NODE); + if (n == null) return null; + String t = n.getTextContent(); + return t == null ? null : t.trim(); + } + + private static Integer tryParsePinId(String path) { + // path: port.{n}.state -> pin_id = n-1 + try { + if (path == null || !path.startsWith("port.")) return null; + String rest = path.substring("port.".length()); + int dot = rest.indexOf('.'); + if (dot < 0) return null; + int n = Integer.parseInt(rest.substring(0, dot)); + return n - 1; + } catch (Exception ignore) { + return null; + } + } + + private static final class RequestPayload { + private final String contentType; + private final byte[] bodyBytes; + + private RequestPayload(String contentType, byte[] bodyBytes) { + this.contentType = contentType; + this.bodyBytes = bodyBytes == null ? new byte[0] : bodyBytes; + } + } +} diff --git a/src/main/java/at/co/procon/malis/cep/detector/PassthroughAlarmDetector.java b/src/main/java/at/co/procon/malis/cep/detector/PassthroughAlarmDetector.java new file mode 100644 index 0000000..4bb4660 --- /dev/null +++ b/src/main/java/at/co/procon/malis/cep/detector/PassthroughAlarmDetector.java @@ -0,0 +1,59 @@ +package at.co.procon.malis.cep.detector; + +import at.co.procon.malis.cep.parser.AlarmBatch; +import at.co.procon.malis.cep.parser.AlarmInput; +import at.co.procon.malis.cep.parser.ParsedInput; + +import java.time.Instant; +import java.util.*; + +/** + * Detector: PASSTHROUGH_ALARM + * + *

This detector does not perform any fault recognition. It simply converts + * already-detected alarm notifications (coming from upstream devices/gateways) into MALIS-internal detector events. + */ + +public class PassthroughAlarmDetector implements Detector { + + @Override + public List detect(ParsedInput input, DetectionContext ctx) { + Object body = input.getBody(); + + List alarms = new ArrayList<>(); + if (body instanceof AlarmInput ai) { + alarms.add(ai); + } else if (body instanceof AlarmBatch batch) { + alarms.addAll(batch.getAlarms()); + } else { + throw new IllegalArgumentException("PASSTHROUGH_ALARM expects AlarmInput or AlarmBatch but got: " + (body == null ? "null" : body.getClass().getName())); + } + + List out = new ArrayList<>(); + for (AlarmInput ai : alarms) { + String tenant = ctx.getVars().getOrDefault("tenant", "na"); + String site = ctx.getVars().getOrDefault("site", "na"); + + String faultKey = tenant + ":" + site + ":" + ctx.getBindingId() + ":" + ai.getDeviceId() + ":" + ai.getAlarmCode(); + + Map details = new LinkedHashMap<>(); + details.put("source", "passthrough"); + + // Copy all EEBUS alarmData fields parsed earlier + if (ai.getDetails() != null) { + details.putAll(ai.getDetails()); + } + + Instant occurred = ai.getTimestamp() != null ? ai.getTimestamp() : Instant.now(); + + out.add(new DetectorEvent( + faultKey, + ai.getAction(), + "UNKNOWN", + occurred, + Collections.unmodifiableMap(details) + )); + } + return out; + } +} \ No newline at end of file diff --git a/src/main/java/at/co/procon/malis/cep/detector/PinBadStateDetector.java b/src/main/java/at/co/procon/malis/cep/detector/PinBadStateDetector.java new file mode 100644 index 0000000..8af9b38 --- /dev/null +++ b/src/main/java/at/co/procon/malis/cep/detector/PinBadStateDetector.java @@ -0,0 +1,90 @@ +package at.co.procon.malis.cep.detector; + +import at.co.procon.malis.cep.parser.M2mPinUpdate; +import at.co.procon.malis.cep.parser.ParsedInput; +import at.co.procon.malis.cep.util.TemplateUtil; + +import java.time.Instant; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +/** + * Detector for M2M pin updates enriched from ERPNext. + * + *

Semantics: + * - if pinValue == pinBad -> emit ALARM + * - else -> emit CANCEL + * + *

faultKey is built from a template (default: "${tenant}:${site}:${iotDevice}:PIN:${pin}"). + */ +public class PinBadStateDetector implements Detector { + + private final String faultKeyTemplate; + private final String severity; + + public PinBadStateDetector(String faultKeyTemplate, String severity) { + this.faultKeyTemplate = (faultKeyTemplate == null || faultKeyTemplate.isBlank()) + ? "${tenant}:${site}:${iotDevice}:PIN:${pin}" : faultKeyTemplate; + this.severity = (severity == null || severity.isBlank()) ? "MAJOR" : severity; + } + + @Override + public List detect(ParsedInput parsed, DetectionContext ctx) { + if (parsed == null || parsed.getBody() == null) { + return List.of(); + } + if (!(parsed.getBody() instanceof M2mPinUpdate upd)) { + // Not our payload + return List.of(); + } + + Map vars = ctx != null ? ctx.getVars() : Map.of(); + String pinBadStr = vars.get("pinBad"); + if (pinBadStr == null || pinBadStr.isBlank()) { + throw new IllegalArgumentException("PinBadStateDetector requires vars.pinBad (did parser enrich ERPNext?)"); + } + + Integer pinBad; + try { pinBad = Integer.parseInt(pinBadStr.trim()); } catch (Exception e) { pinBad = null; } + if (pinBad == null) { + throw new IllegalArgumentException("Invalid vars.pinBad: " + pinBadStr); + } + + Integer value = upd.getValue(); + if (value == null) { + // Nothing to decide + return List.of(); + } + + String eventType = (value.intValue() == pinBad.intValue()) ? "ALARM" : "CANCEL"; + + String faultKey = TemplateUtil.expand(faultKeyTemplate, vars); + if (faultKey == null || faultKey.isBlank()) { + faultKey = "PIN:" + vars.getOrDefault("pin", "unknown"); + } + + Map details = new LinkedHashMap<>(); + details.put("gpio", upd.getGpio()); + details.put("channel", upd.getChannel()); + details.put("value", value); + details.put("bad", pinBad); + details.put("reason", upd.getReason()); + details.put("m2mport", upd.getM2mport()); + if (upd.getMqtt() != null && !upd.getMqtt().isEmpty()) { + details.put("mqtt", upd.getMqtt()); + } + // Extra resolved info (if present) + putIfPresent(details, "pinName", vars.get("pinName")); + putIfPresent(details, "pinDescription", vars.get("pinDescription")); + putIfPresent(details, "remark", vars.get("remark")); + + Instant occurredAt = upd.getTimestamp() != null ? upd.getTimestamp() : parsed.getReceivedAt(); + + return List.of(new DetectorEvent(faultKey, eventType, severity, occurredAt, details)); + } + + private static void putIfPresent(Map m, String k, String v) { + if (v != null && !v.isBlank()) m.put(k, v); + } +} diff --git a/src/main/java/at/co/procon/malis/cep/eebus/EebusAddress.java b/src/main/java/at/co/procon/malis/cep/eebus/EebusAddress.java new file mode 100644 index 0000000..f4084d0 --- /dev/null +++ b/src/main/java/at/co/procon/malis/cep/eebus/EebusAddress.java @@ -0,0 +1,35 @@ +// ============================================================ +// 2) Address model (structured addressSource/destination) +// ============================================================ + +package at.co.procon.malis.cep.eebus; + +/** + * Structured address used in SPINE datagram header. + */ +public class EebusAddress { + private String device; + private Integer entity; + private Integer feature; + + public EebusAddress() {} + + public EebusAddress(String device, Integer entity, Integer feature) { + this.device = device; + this.entity = entity; + this.feature = feature; + } + + public String getDevice() { return device; } + public void setDevice(String device) { this.device = device; } + + public Integer getEntity() { return entity; } + public void setEntity(Integer entity) { this.entity = entity; } + + public Integer getFeature() { return feature; } + public void setFeature(Integer feature) { this.feature = feature; } + + public boolean isEmpty() { + return (device == null || device.isBlank()) && entity == null && feature == null; + } +} diff --git a/src/main/java/at/co/procon/malis/cep/eebus/EebusJaxbCodec.java b/src/main/java/at/co/procon/malis/cep/eebus/EebusJaxbCodec.java new file mode 100644 index 0000000..0363992 --- /dev/null +++ b/src/main/java/at/co/procon/malis/cep/eebus/EebusJaxbCodec.java @@ -0,0 +1,90 @@ +package at.co.procon.malis.cep.eebus; + +import jakarta.xml.bind.*; +import jakarta.xml.bind.util.JAXBSource; +import org.openmuc.jeebus.spine.xsd.v1.DatagramType; +import org.openmuc.jeebus.spine.xsd.v1.ObjectFactory; + +import javax.xml.namespace.QName; +import javax.xml.transform.Source; +import javax.xml.transform.stream.StreamSource; +import java.io.*; +import java.nio.charset.StandardCharsets; + +/** + * JAXB codec for EEBUS SPINE v1 XML. + * + * Why this exists: + * - JAXBContext is expensive to build but thread-safe afterwards. + * - Marshaller/Unmarshaller are NOT thread-safe -> create per call. + * + * This codec strictly deals with XML <-> JAXB model conversion. + * It does not do business mapping (that happens in parsers/formatters). + */ +public class EebusJaxbCodec { + + private static final String MODEL_PACKAGE = "org.openmuc.jeebus.spine.xsd.v1"; + + private final JAXBContext ctx; + private final ObjectFactory factory; + + public EebusJaxbCodec() { + try { + // Using ObjectFactory ensures the whole generated package is registered. + this.ctx = JAXBContext.newInstance(ObjectFactory.class); + this.factory = new ObjectFactory(); + } catch (JAXBException e) { + throw new IllegalStateException("Failed creating JAXBContext for " + MODEL_PACKAGE, e); + } + } + + public DatagramType unmarshalDatagram(byte[] xmlBytes) { + if (xmlBytes == null) throw new IllegalArgumentException("xmlBytes is null"); + try (ByteArrayInputStream in = new ByteArrayInputStream(xmlBytes)) { + Unmarshaller um = ctx.createUnmarshaller(); + + // This form works even when the root element is mapped as JAXBElement + JAXBElement root = um.unmarshal(new StreamSource(in), DatagramType.class); + return root.getValue(); + + } catch (Exception e) { + throw new IllegalArgumentException("Invalid SPINE datagram XML", e); + } + } + + public byte[] marshalDatagram(DatagramType datagram) { + if (datagram == null) throw new IllegalArgumentException("datagram is null"); + try (ByteArrayOutputStream out = new ByteArrayOutputStream()) { + Marshaller m = ctx.createMarshaller(); + m.setProperty(Marshaller.JAXB_FORMATTED_OUTPUT, Boolean.TRUE); + m.setProperty(Marshaller.JAXB_ENCODING, "UTF-8"); + + // Ensure correct root element name + namespace + JAXBElement root = factory.createDatagram(datagram); + m.marshal(root, out); + return out.toByteArray(); + + } catch (Exception e) { + throw new IllegalStateException("Failed to marshal SPINE datagram", e); + } + } + + /** + * Marshal a subtree (fragment) with an explicit QName root. + * Useful for external detectors that want measurementDescriptionListData or measurementListData. + */ + public String marshalFragment(T value, QName rootName, Class clazz) { + if (value == null) return null; + try (ByteArrayOutputStream out = new ByteArrayOutputStream()) { + Marshaller m = ctx.createMarshaller(); + m.setProperty(Marshaller.JAXB_FORMATTED_OUTPUT, Boolean.FALSE); + m.setProperty(Marshaller.JAXB_FRAGMENT, Boolean.TRUE); // no XML declaration + + JAXBElement root = new JAXBElement<>(rootName, clazz, value); + m.marshal(root, out); + return out.toString(StandardCharsets.UTF_8); + } catch (Exception e) { + throw new IllegalStateException("Failed to marshal fragment " + rootName, e); + } + } +} diff --git a/src/main/java/at/co/procon/malis/cep/eebus/EebusMeasurementDatagram.java b/src/main/java/at/co/procon/malis/cep/eebus/EebusMeasurementDatagram.java new file mode 100644 index 0000000..89f2314 --- /dev/null +++ b/src/main/java/at/co/procon/malis/cep/eebus/EebusMeasurementDatagram.java @@ -0,0 +1,46 @@ +package at.co.procon.malis.cep.eebus; + +/** + * Parsed representation of an EEBUS SPINE measurement datagram. + * + *

We keep: + *

    + *
  • the original full XML
    • + *
  • optional extracted measurementDescriptionListData fragment XML
    • + *
  • optional extracted measurementListData fragment XML
    • + *
  • optional addressing extracted from header (if present)
    • + *
+ */ +public final class EebusMeasurementDatagram { + + private final String datagramXml; + private final String measurementDescriptionListDataXml; + private final String measurementListDataXml; + private final String addressSource; + private final String addressDestination; + + public EebusMeasurementDatagram(String datagramXml, + String measurementDescriptionListDataXml, + String measurementListDataXml, + String addressSource, + String addressDestination) { + this.datagramXml = datagramXml; + this.measurementDescriptionListDataXml = measurementDescriptionListDataXml; + this.measurementListDataXml = measurementListDataXml; + this.addressSource = addressSource; + this.addressDestination = addressDestination; + } + + public String getDatagramXml() { return datagramXml; } + public String getMeasurementDescriptionListDataXml() { return measurementDescriptionListDataXml; } + public String getMeasurementListDataXml() { return measurementListDataXml; } + public String getAddressSource() { return addressSource; } + public String getAddressDestination() { return addressDestination; } + + // record-style accessors + public String datagramXml() { return datagramXml; } + public String measurementDescriptionListDataXml() { return measurementDescriptionListDataXml; } + public String measurementListDataXml() { return measurementListDataXml; } + public String addressSource() { return addressSource; } + public String addressDestination() { return addressDestination; } +} diff --git a/src/main/java/at/co/procon/malis/cep/eebus/EebusSpineConstants.java b/src/main/java/at/co/procon/malis/cep/eebus/EebusSpineConstants.java new file mode 100644 index 0000000..7ea3b81 --- /dev/null +++ b/src/main/java/at/co/procon/malis/cep/eebus/EebusSpineConstants.java @@ -0,0 +1,28 @@ +package at.co.procon.malis.cep.eebus; + +/** + * Small set of SPINE XML constants used by the prototype. + * + *

EEBUS SPINE has official schemas. In production, you may use a schema-based library (e.g. JAXB via + * jeebus) to produce/parse strongly-typed datagrams. For the prototype we do minimal DOM parsing and + * generation to keep dependencies low and remain namespace-tolerant. + */ +/** Constants for EEBUS SPINE v1 XML. */ +public final class EebusSpineConstants { + + /** Official SPINE namespace used by the XSD set. */ + public static final String SPINE_NS = "http://docs.eebus.org/spine/xsd/v1"; + + /** Alias used by newer code. */ + public static final String NS_URI = SPINE_NS; + + /** Preferred prefix we emit in XML. */ + public static final String PREFIX = "ns_p"; + + /** Root element local-name. */ + public static final String ROOT_LOCAL = "datagram"; + + public static final String DATAGRAM_ROOT = "datagram"; + + private EebusSpineConstants() {} +} diff --git a/src/main/java/at/co/procon/malis/cep/eebus/EebusSpineReflection.java b/src/main/java/at/co/procon/malis/cep/eebus/EebusSpineReflection.java new file mode 100644 index 0000000..0409e65 --- /dev/null +++ b/src/main/java/at/co/procon/malis/cep/eebus/EebusSpineReflection.java @@ -0,0 +1,98 @@ +package at.co.procon.malis.cep.eebus; + +import java.lang.reflect.Method; +import java.math.BigInteger; +import java.util.Locale; + +/** + * Helper to set SPINE JAXB fields without tightly depending on whether XJC generated + * String, enum, Integer, BigInteger, etc. + * + * This makes the project more tolerant to SPINE schema upgrades. + */ +public final class EebusSpineReflection { + + private EebusSpineReflection() {} + + public static void setStringOrEnum(Object target, String setter, String value) { + if (target == null || value == null) return; + Method m = findSetter(target.getClass(), setter); + if (m == null) return; + + Class pt = m.getParameterTypes()[0]; + try { + if (pt.equals(String.class)) { + m.invoke(target, value); + return; + } + if (pt.isEnum()) { + // Prefer fromValue(String) (SPINE enums usually have it) + try { + Method fromValue = pt.getMethod("fromValue", String.class); + Object enumVal = fromValue.invoke(null, value); + m.invoke(target, enumVal); + return; + } catch (NoSuchMethodException ignore) { + // Fallback to Enum.valueOf with a best-effort normalization + @SuppressWarnings({"unchecked","rawtypes"}) + Object enumVal = Enum.valueOf((Class) pt, normalizeEnumName(value)); + m.invoke(target, enumVal); + return; + } + } + + // Last-resort: try a single-String constructor + try { + Object o = pt.getConstructor(String.class).newInstance(value); + m.invoke(target, o); + } catch (Exception ignored) { + // ignore + } + } catch (Exception e) { + // ignore (best-effort setter) + } + } + + public static void setUnsignedInt(Object target, String setter, long value) { + if (target == null) return; + Method m = findSetter(target.getClass(), setter); + if (m == null) return; + Class pt = m.getParameterTypes()[0]; + try { + if (pt.equals(Long.class) || pt.equals(long.class)) { + m.invoke(target, value); + } else if (pt.equals(Integer.class) || pt.equals(int.class)) { + m.invoke(target, (int) value); + } else if (pt.equals(BigInteger.class)) { + m.invoke(target, BigInteger.valueOf(value)); + } else { + // best-effort: String + if (pt.equals(String.class)) { + m.invoke(target, Long.toUnsignedString(value)); + } + } + } catch (Exception ignore) { + // ignore + } + } + + private static Method findSetter(Class c, String setter) { + // Find first method named setter with one param + for (Method m : c.getMethods()) { + if (m.getName().equals(setter) && m.getParameterCount() == 1) return m; + } + return null; + } + + private static String normalizeEnumName(String v) { + // Examples: "overThreshold" -> "OVER_THRESHOLD"; "alarmCancelled" -> "ALARM_CANCELLED" + String s = v.trim(); + if (s.isBlank()) return s; + + // camelCase -> snake + String snake = s.replaceAll("([a-z])([A-Z])", "$1_$2"); + // hyphen -> underscore + snake = snake.replace('-', '_'); + return snake.toUpperCase(Locale.ROOT); + } +} \ No newline at end of file diff --git a/src/main/java/at/co/procon/malis/cep/eebus/EebusSpineV1.java b/src/main/java/at/co/procon/malis/cep/eebus/EebusSpineV1.java new file mode 100644 index 0000000..cad45c2 --- /dev/null +++ b/src/main/java/at/co/procon/malis/cep/eebus/EebusSpineV1.java @@ -0,0 +1,34 @@ +// These are NEW/MODIFIED classes to align the CEP project with the sample SPINE XML you provided. +// Namespace: http://docs.eebus.org/spine/xsd/v1 +// Root element: (lowercase) as in the XSD and your sample. +// +// Add/replace these files under: +// src/main/java/at/co/procon/malis/cep/eebus/ +// src/main/java/at/co/procon/malis/cep/parser/ +// src/main/java/at/co/procon/malis/cep/output/ +// +// IMPORTANT DESIGN NOTE +// --------------------- +// Your sample uses structured addresses: +// ...... +// Our formatter previously used STRING. +// This update introduces Address objects and config templates to produce the correct structure. + +// ============================================================ +// 1) Constants + helpers +// ============================================================ + +package at.co.procon.malis.cep.eebus; + +/** + * Constants for SPINE XSD v1, as used in your samples. + */ +public final class EebusSpineV1 { + /** Your sample namespace */ + public static final String NS = "http://docs.eebus.org/spine/xsd/v1"; + + /** Root element used by the XSD (and your sample): */ + public static final String ROOT = "datagram"; + + private EebusSpineV1() {} +} \ No newline at end of file diff --git a/src/main/java/at/co/procon/malis/cep/eebus/EebusSpineXsdJaxbValidator.java b/src/main/java/at/co/procon/malis/cep/eebus/EebusSpineXsdJaxbValidator.java new file mode 100644 index 0000000..8f582df --- /dev/null +++ b/src/main/java/at/co/procon/malis/cep/eebus/EebusSpineXsdJaxbValidator.java @@ -0,0 +1,29 @@ +package at.co.procon.malis.cep.eebus; + +import org.springframework.stereotype.Component; + +/** + * Backward-compatible validator name. + * + *

"A = JAXB-only": we do NOT load XSD resources from classpath. + * Validation is performed by JAXB unmarshalling the full using jEEBus SPINE model. + */ +@Component +public class EebusSpineXsdJaxbValidator implements EebusSpineXsdValidatorBase { + + private final EebusJaxbCodec codec; + + public EebusSpineXsdJaxbValidator() { + this.codec = new EebusJaxbCodec(); + } + + /** Validate the provided datagram bytes. Throws IllegalArgumentException if invalid. */ + public void validateDatagram(byte[] xmlBytes, String context) { + try { + codec.unmarshalDatagram(xmlBytes); + } catch (Exception e) { + String ctx = (context == null || context.isBlank()) ? "" : (" (" + context + ")"); + throw new IllegalArgumentException("Invalid EEBUS SPINE datagram" + ctx + ": " + e.getMessage(), e); + } + } +} \ No newline at end of file diff --git a/src/main/java/at/co/procon/malis/cep/eebus/EebusSpineXsdValidator.java b/src/main/java/at/co/procon/malis/cep/eebus/EebusSpineXsdValidator.java new file mode 100644 index 0000000..e3a7a67 --- /dev/null +++ b/src/main/java/at/co/procon/malis/cep/eebus/EebusSpineXsdValidator.java @@ -0,0 +1,188 @@ +package at.co.procon.malis.cep.eebus; + +import at.co.procon.malis.cep.config.CepProperties; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Component; +import org.w3c.dom.ls.LSInput; +import org.w3c.dom.ls.LSResourceResolver; + +import javax.xml.XMLConstants; +import javax.xml.transform.stream.StreamSource; +import javax.xml.validation.Schema; +import javax.xml.validation.SchemaFactory; +import java.io.*; +import java.net.URL; +import java.util.Objects; + +@Component +public class EebusSpineXsdValidator implements EebusSpineXsdValidatorBase { + + private static final Logger log = LoggerFactory.getLogger(EebusSpineXsdValidator.class); + + private static final String[] DEFAULT_SCHEMA_CANDIDATES = new String[] { + "EEBus_SPINE_TS_Datagram.xsd", + "xsd/EEBus_SPINE_TS_Datagram.xsd", + "spine/xsd/EEBus_SPINE_TS_Datagram.xsd", + "Datagram.xsd", + "xsd/Datagram.xsd", + "spine/xsd/Datagram.xsd" + }; + + private final CepProperties props; + private volatile Schema cachedSchema; + private volatile boolean triedLoading; + + public EebusSpineXsdValidator(CepProperties props) { + this.props = props; + } + + public boolean isEnabled() { + return props.getEebusXsdValidation() != null && props.getEebusXsdValidation().isEnabled(); + } + + public void validateDatagram(byte[] xmlBytes, String contextHint) { + if (!isEnabled()) return; + Objects.requireNonNull(xmlBytes, "xmlBytes"); + + Schema schema = tryGetSchema(); + if (schema == null) { + String msg = "EEBUS SPINE XSD validation enabled but schema could not be loaded from classpath."; + if (props.getEebusXsdValidation() != null && props.getEebusXsdValidation().isFailFast()) { + throw new IllegalStateException(msg); + } + log.warn(msg); + return; + } + + try { + var validator = schema.newValidator(); + validator.validate(new StreamSource(new ByteArrayInputStream(xmlBytes))); + } catch (Exception ex) { + String msg = "EEBUS SPINE XSD validation failed" + (contextHint != null ? (" (" + contextHint + ")") : "") + ": " + ex.getMessage(); + if (props.getEebusXsdValidation() != null && !props.getEebusXsdValidation().isFailFast()) { + log.warn(msg, ex); + return; + } + throw new IllegalArgumentException(msg, ex); + } + } + + public boolean schemaAvailable() { + return findSchemaRootResource() != null; + } + + private Schema tryGetSchema() { + if (cachedSchema != null) return cachedSchema; + + synchronized (this) { + if (cachedSchema != null) return cachedSchema; + if (triedLoading) return null; + triedLoading = true; + + URL rootUrl = findSchemaRootResource(); + if (rootUrl == null) return null; + + try { + SchemaFactory sf = SchemaFactory.newInstance(XMLConstants.W3C_XML_SCHEMA_NS_URI); + + sf.setFeature(XMLConstants.FEATURE_SECURE_PROCESSING, true); + sf.setProperty(XMLConstants.ACCESS_EXTERNAL_DTD, ""); + sf.setProperty(XMLConstants.ACCESS_EXTERNAL_SCHEMA, ""); + + sf.setResourceResolver(new ClasspathXsdResolver()); + + try (InputStream in = rootUrl.openStream()) { + cachedSchema = sf.newSchema(new StreamSource(in)); + log.info("Loaded EEBUS SPINE XSD root: {}", rootUrl); + return cachedSchema; + } + } catch (Exception ex) { + log.warn("Failed to load SPINE XSD schema: {}", ex.getMessage(), ex); + return null; + } + } + } + + private URL findSchemaRootResource() { + String configured = props.getEebusXsdValidation() != null ? props.getEebusXsdValidation().getSchemaRoot() : null; + ClassLoader cl = Thread.currentThread().getContextClassLoader(); + + if (configured != null && !configured.isBlank()) { + URL url = cl.getResource(stripLeadingSlash(configured)); + if (url != null) return url; + } + + for (String cand : DEFAULT_SCHEMA_CANDIDATES) { + URL url = cl.getResource(stripLeadingSlash(cand)); + if (url != null) return url; + } + + return null; + } + + private static String stripLeadingSlash(String p) { + if (p == null) return null; + return p.startsWith("/") ? p.substring(1) : p; + } + + static final class ClasspathXsdResolver implements LSResourceResolver { + @Override + public LSInput resolveResource(String type, String namespaceURI, String publicId, String systemId, String baseURI) { + if (systemId == null || systemId.isBlank()) return null; + + String normalized = systemId.startsWith("/") ? systemId.substring(1) : systemId; + ClassLoader cl = Thread.currentThread().getContextClassLoader(); + + URL url = cl.getResource(normalized); + if (url == null && baseURI != null && baseURI.contains("/")) { + String baseFolder = baseURI.substring(0, baseURI.lastIndexOf('/') + 1); + url = cl.getResource(baseFolder + normalized); + } + if (url == null) return null; + + try { + InputStream in = url.openStream(); + return new SimpleLsInput(publicId, systemId, in); + } catch (IOException e) { + return null; + } + } + } + + static final class SimpleLsInput implements LSInput { + private String publicId; + private String systemId; + private InputStream byteStream; + + SimpleLsInput(String publicId, String systemId, InputStream byteStream) { + this.publicId = publicId; + this.systemId = systemId; + this.byteStream = byteStream; + } + + @Override public Reader getCharacterStream() { return null; } + @Override public void setCharacterStream(Reader characterStream) { } + + @Override public InputStream getByteStream() { return byteStream; } + @Override public void setByteStream(InputStream byteStream) { this.byteStream = byteStream; } + + @Override public String getStringData() { return null; } + @Override public void setStringData(String stringData) { } + + @Override public String getSystemId() { return systemId; } + @Override public void setSystemId(String systemId) { this.systemId = systemId; } + + @Override public String getPublicId() { return publicId; } + @Override public void setPublicId(String publicId) { this.publicId = publicId; } + + @Override public String getBaseURI() { return null; } + @Override public void setBaseURI(String baseURI) { } + + @Override public String getEncoding() { return null; } + @Override public void setEncoding(String encoding) { } + + @Override public boolean getCertifiedText() { return false; } + @Override public void setCertifiedText(boolean certifiedText) { } + } +} \ No newline at end of file diff --git a/src/main/java/at/co/procon/malis/cep/eebus/EebusSpineXsdValidatorBase.java b/src/main/java/at/co/procon/malis/cep/eebus/EebusSpineXsdValidatorBase.java new file mode 100644 index 0000000..352d767 --- /dev/null +++ b/src/main/java/at/co/procon/malis/cep/eebus/EebusSpineXsdValidatorBase.java @@ -0,0 +1,5 @@ +package at.co.procon.malis.cep.eebus; + +public interface EebusSpineXsdValidatorBase { + void validateDatagram(byte[] xmlBytes, String context); +} diff --git a/src/main/java/at/co/procon/malis/cep/eebus/EebusXml.java b/src/main/java/at/co/procon/malis/cep/eebus/EebusXml.java new file mode 100644 index 0000000..31085ef --- /dev/null +++ b/src/main/java/at/co/procon/malis/cep/eebus/EebusXml.java @@ -0,0 +1,70 @@ +package at.co.procon.malis.cep.eebus; + +import org.w3c.dom.Document; +import org.w3c.dom.Node; + +import javax.xml.XMLConstants; +import javax.xml.parsers.DocumentBuilderFactory; +import javax.xml.transform.OutputKeys; +import javax.xml.transform.TransformerFactory; +import javax.xml.transform.dom.DOMSource; +import javax.xml.transform.stream.StreamResult; +import javax.xml.xpath.XPathConstants; +import javax.xml.xpath.XPathFactory; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.nio.charset.StandardCharsets; + +/** + * Namespace-tolerant XML utilities. + * + * We use XPath local-name() so the code works whether the XML uses a default namespace + * or a prefix like ns_p. + */ +public final class EebusXml { + + private EebusXml() {} + + public static Document parse(byte[] xmlBytes) throws Exception { + DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance(); + dbf.setNamespaceAware(true); + dbf.setFeature(XMLConstants.FEATURE_SECURE_PROCESSING, true); + dbf.setAttribute(XMLConstants.ACCESS_EXTERNAL_DTD, ""); + dbf.setAttribute(XMLConstants.ACCESS_EXTERNAL_SCHEMA, ""); + return dbf.newDocumentBuilder().parse(new ByteArrayInputStream(xmlBytes)); + } + + public static Node first(Document doc, String xPathExpr) throws Exception { + var xpath = XPathFactory.newInstance().newXPath(); + return (Node) xpath.evaluate(xPathExpr, doc, XPathConstants.NODE); + } + + public static String text(Document doc, String xPathExpr) throws Exception { + Node n = first(doc, xPathExpr); + if (n == null) return null; + String t = n.getTextContent(); + return t == null ? null : t.trim(); + } + + public static String nodeToXml(Node node) throws Exception { + if (node == null) return null; + TransformerFactory tf = TransformerFactory.newInstance(); + tf.setFeature(XMLConstants.FEATURE_SECURE_PROCESSING, true); + var t = tf.newTransformer(); + t.setOutputProperty(OutputKeys.OMIT_XML_DECLARATION, "yes"); + t.setOutputProperty(OutputKeys.INDENT, "no"); + + ByteArrayOutputStream out = new ByteArrayOutputStream(); + t.transform(new DOMSource(node), new StreamResult(out)); + return out.toString(StandardCharsets.UTF_8); + } + + public static String escape(String s) { + if (s == null) return ""; + return s.replace("&", "&") + .replace("<", "<") + .replace(">", ">") + .replace("\"", """) + .replace("'", "'"); + } +} diff --git a/src/main/java/at/co/procon/malis/cep/eebus/EebusXmlUtil.java b/src/main/java/at/co/procon/malis/cep/eebus/EebusXmlUtil.java new file mode 100644 index 0000000..07dc217 --- /dev/null +++ b/src/main/java/at/co/procon/malis/cep/eebus/EebusXmlUtil.java @@ -0,0 +1,133 @@ +package at.co.procon.malis.cep.eebus; + +import org.w3c.dom.Document; +import org.w3c.dom.Node; + +import javax.xml.XMLConstants; +import javax.xml.parsers.DocumentBuilderFactory; +import javax.xml.transform.OutputKeys; +import javax.xml.transform.Transformer; +import javax.xml.transform.TransformerFactory; +import javax.xml.transform.dom.DOMSource; +import javax.xml.transform.stream.StreamResult; +import javax.xml.xpath.XPath; +import javax.xml.xpath.XPathConstants; +import javax.xml.xpath.XPathFactory; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.nio.charset.StandardCharsets; + +/** + * Minimal XML helper for SPINE-like EEBUS datagrams. + * + *

Key property: namespace-tolerant extraction. We use XPath with {@code local-name()}, + * so this works for: + *

    + *
  • default namespace
    • + *
  • prefixed namespace (e.g. {@code })
    • + *
  • no namespace (some test systems)
    • + *
+ * + *

This code intentionally avoids binding to a specific schema version. If/when you adopt + * a schema-first approach (e.g. jeebus), you can swap this out behind a small codec interface. + */ +public final class EebusXmlUtil { + + private EebusXmlUtil() { + } + + /** Convenience: evaluate an XPath and return trimmed text content (or null). */ + public static String text(Document doc, String xpathExpr) { + if (doc == null || xpathExpr == null || xpathExpr.isBlank()) return null; + try { + Node n = (Node) XPathFactory.newInstance().newXPath().evaluate(xpathExpr, doc, XPathConstants.NODE); + if (n == null) return null; + String t = n.getTextContent(); + if (t == null) return null; + String s = t.trim(); + return s.isBlank() ? null : s; + } catch (Exception ignore) { + return null; + } + } + + /** + * Alias for extractFirstText(doc, localName). + * Some patches/tests refer to this as "text". + */ + public static String text2(Document doc, String localName) throws Exception { + return extractFirstText(doc, localName); + } + + /** + * Parse XML bytes into a namespace-aware DOM document with secure processing enabled. + * + *

We explicitly disable external entity resolution (XXE hardening). + */ + public static Document parse(byte[] xmlBytes) throws Exception { + DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance(); + dbf.setNamespaceAware(true); + dbf.setFeature(XMLConstants.FEATURE_SECURE_PROCESSING, true); + + // Disallow external entities / DTDs (XXE hardening) + try { + dbf.setAttribute(XMLConstants.ACCESS_EXTERNAL_DTD, ""); + dbf.setAttribute(XMLConstants.ACCESS_EXTERNAL_SCHEMA, ""); + } catch (IllegalArgumentException ignore) { + // Some JDKs don't support these attributes; secure processing still helps. + } + + return dbf.newDocumentBuilder().parse(new ByteArrayInputStream(xmlBytes)); + } + + /** Evaluate an XPath expression and return the first node (or null). */ + public static Node first(Document doc, String xPathExpr) throws Exception { + XPath xpath = XPathFactory.newInstance().newXPath(); + return (Node) xpath.evaluate(xPathExpr, doc, XPathConstants.NODE); + } + + /** Serialize a node as an XML fragment (no XML declaration). */ + public static String nodeToXml(Node node) throws Exception { + if (node == null) return null; + + TransformerFactory tf = TransformerFactory.newInstance(); + tf.setFeature(XMLConstants.FEATURE_SECURE_PROCESSING, true); + + Transformer t = tf.newTransformer(); + t.setOutputProperty(OutputKeys.OMIT_XML_DECLARATION, "yes"); + t.setOutputProperty(OutputKeys.INDENT, "no"); + + ByteArrayOutputStream out = new ByteArrayOutputStream(); + t.transform(new DOMSource(node), new StreamResult(out)); + return out.toString(StandardCharsets.UTF_8); + } + + /** Extract first element by local-name as fragment XML (namespace tolerant). */ + public static String extractFirstFragmentXml(Document doc, String localName) throws Exception { + Node n = first(doc, "//*[local-name()='" + localName + "']"); + return nodeToXml(n); + } + + /** Extract first element text content by local-name (namespace tolerant). */ + public static String extractFirstText(Document doc, String localName) throws Exception { + Node n = first(doc, "//*[local-name()='" + localName + "']"); + return n == null ? null : n.getTextContent(); + } + + public static byte[] documentToBytes(Document doc) { + try { + TransformerFactory tf = TransformerFactory.newInstance(); + tf.setFeature(XMLConstants.FEATURE_SECURE_PROCESSING, true); + + Transformer t = tf.newTransformer(); + t.setOutputProperty(OutputKeys.OMIT_XML_DECLARATION, "yes"); + t.setOutputProperty(OutputKeys.ENCODING, "UTF-8"); + + ByteArrayOutputStream out = new ByteArrayOutputStream(); + t.transform(new DOMSource(doc), new StreamResult(out)); + return out.toByteArray(); + } catch (Exception e) { + throw new RuntimeException("Failed to serialize XML", e); + } + } +} diff --git a/src/main/java/at/co/procon/malis/cep/erpnext/ErpnextFaultLookupClient.java b/src/main/java/at/co/procon/malis/cep/erpnext/ErpnextFaultLookupClient.java new file mode 100644 index 0000000..f4d441c --- /dev/null +++ b/src/main/java/at/co/procon/malis/cep/erpnext/ErpnextFaultLookupClient.java @@ -0,0 +1,1185 @@ +package at.co.procon.malis.cep.erpnext; + +import at.co.procon.malis.cep.util.ExpiringCache; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.springframework.web.util.UriComponentsBuilder; + +import java.net.URI; +import java.net.URLEncoder; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.nio.charset.StandardCharsets; +import java.time.*; +import java.time.format.DateTimeFormatter; +import java.time.format.DateTimeParseException; +import java.util.*; +import java.util.concurrent.*; +import java.util.concurrent.atomic.AtomicReference; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Minimal ERPNext REST client used by malis-cep to resolve: + *

    + *
  • port -> IoT_Device
    • + *
  • IoT_Device -> Site assignment (via reading Site child-table)
    • + *
  • pin -> MalIS Configuration Detail (DigitalInput) including "bad" state
    • + *
+ * + *

Auth: {@code Authorization: token :} + */ +public final class ErpnextFaultLookupClient { + + private static final Logger log = LoggerFactory.getLogger(ErpnextFaultLookupClient.class); + + // -------------------- Models -------------------- + + public static final class FaultLookupContext { + public final String port; + public final String iotDevice; + public final String siteId; + public final String remark; + public final String department; + public final String pinName; + public final String description; + public final String bad; + + public FaultLookupContext(String port, + String iotDevice, + String siteId, + String remark, + String department, + String pinName, + String description, + String bad) { + this.port = port; + this.iotDevice = iotDevice; + this.siteId = siteId; + this.remark = remark; + this.department = department; + this.pinName = pinName; + this.description = description; + this.bad = bad; + } + } + + // -------------------- Exceptions -------------------- + + public static class PortNotFoundException extends RuntimeException { + public final String port; + public PortNotFoundException(String port) { super("Port not found: " + port); this.port = port; } + } + + public static class PortAmbiguousException extends RuntimeException { + public final String port; + public final List devices; + public PortAmbiguousException(String port, List devices) { super("Port ambiguous: " + port + " -> " + devices); this.port = port; this.devices = devices; } + } + + public static class SiteAssignmentNotFoundException extends RuntimeException { + public final String port; + public final String perDatum; + public SiteAssignmentNotFoundException(String port, String perDatum) { super("No Site assigned for port " + port + " at " + perDatum); this.port = port; this.perDatum = perDatum; } + } + + public static class SiteAssignmentAmbiguousException extends RuntimeException { + public final String port; + public final String perDatum; + public final List sites; + public SiteAssignmentAmbiguousException(String port, String perDatum, List sites) { super("Multiple Sites assigned for port " + port + " at " + perDatum + ": " + sites); this.port = port; this.perDatum = perDatum; this.sites = sites; } + } + + public static class PinNotDefinedException extends RuntimeException { + public final int pin; + public PinNotDefinedException(int pin) { super("Pin not defined: " + pin); this.pin = pin; } + } + + public static class PinDefinedMultipleTimesException extends RuntimeException { + public final int pin; + public final List definitions; + public PinDefinedMultipleTimesException(int pin, List definitions) { super("Pin defined multiple times: " + pin + " -> " + definitions); this.pin = pin; this.definitions = definitions; } + } + + private static final class ForbiddenException extends RuntimeException { + ForbiddenException(String message) { super(message); } + } + + // -------------------- Config -------------------- + + public static final class Config { + public final String baseUrl; + public final String apiKey; + public final String apiSecret; + public final int timeoutMs; + + // doctypes + public final String doctypeIotDevice; + public final String doctypeSite; + public final String doctypeSiteIotDevice; + public final String doctypeMalisConfig; + public final String doctypeMalisConfigDetail; + + // fields + public final String iotDevicePortAddressField; + public final String siteFieldIotDevicesAssigned; + public final String siteIotDeviceFieldIotDevice; + public final String siteIotDeviceFieldMalis; + public final String siteIotDeviceFieldRemark; + public final String siteIotDeviceFieldFromDate; + public final String siteIotDeviceFieldToDate; + public final String malisConfigFieldName; + public final String malisConfigFieldDetails; + public final String malisDetailFieldPin; + public final String malisDetailFieldStream; + public final String malisDetailStreamValue; + public final String malisDetailFieldPinName; + public final String malisDetailFieldDescription; + public final String malisDetailFieldBad; + public final String malisDetailFieldDepartment; + + // cache + public final Duration cacheTtl; + public final int cacheMaxSize; + + // preload/refresh (recommended for scale) + public final boolean preloadEnabled; + public final boolean preloadOnStartup; + public final Duration preloadRefreshInterval; + public final Duration preloadInitialDelay; + public final boolean preloadAllowOnDemandFallback; + + public Config(String baseUrl, + String apiKey, + String apiSecret, + int timeoutMs, + String doctypeIotDevice, + String doctypeSite, + String doctypeSiteIotDevice, + String doctypeMalisConfig, + String doctypeMalisConfigDetail, + String iotDevicePortAddressField, + String siteFieldIotDevicesAssigned, + String siteIotDeviceFieldIotDevice, + String siteIotDeviceFieldMalis, + String siteIotDeviceFieldRemark, + String siteIotDeviceFieldFromDate, + String siteIotDeviceFieldToDate, + String malisConfigFieldName, + String malisConfigFieldDetails, + String malisDetailFieldPin, + String malisDetailFieldStream, + String malisDetailStreamValue, + String malisDetailFieldPinName, + String malisDetailFieldDescription, + String malisDetailFieldBad, + String malisDetailFieldDepartment, + Duration cacheTtl, + int cacheMaxSize, + boolean preloadEnabled, + boolean preloadOnStartup, + Duration preloadRefreshInterval, + Duration preloadInitialDelay, + boolean preloadAllowOnDemandFallback) { + this.baseUrl = Objects.requireNonNull(baseUrl, "baseUrl").replaceAll("/+$", ""); + this.apiKey = Objects.requireNonNull(apiKey, "apiKey"); + this.apiSecret = Objects.requireNonNull(apiSecret, "apiSecret"); + this.timeoutMs = Math.max(250, timeoutMs); + + this.doctypeIotDevice = orDefault(doctypeIotDevice, "IoT_Device"); + this.doctypeSite = orDefault(doctypeSite, "Site"); + this.doctypeSiteIotDevice = orDefault(doctypeSiteIotDevice, "Site_IoT_Device"); + this.doctypeMalisConfig = orDefault(doctypeMalisConfig, "MalIS Configuration"); + this.doctypeMalisConfigDetail = orDefault(doctypeMalisConfigDetail, "MalIS Configuration Detail"); + + this.iotDevicePortAddressField = orDefault(iotDevicePortAddressField, "port_address"); + this.siteFieldIotDevicesAssigned = orDefault(siteFieldIotDevicesAssigned, "iot_devices_assigned"); + this.siteIotDeviceFieldIotDevice = orDefault(siteIotDeviceFieldIotDevice, "iot_device"); + this.siteIotDeviceFieldMalis = orDefault(siteIotDeviceFieldMalis, "malis"); + this.siteIotDeviceFieldRemark = orDefault(siteIotDeviceFieldRemark, "remark"); + this.siteIotDeviceFieldFromDate = orDefault(siteIotDeviceFieldFromDate, "from"); + this.siteIotDeviceFieldToDate = orDefault(siteIotDeviceFieldToDate, "to"); + + this.malisConfigFieldName = orDefault(malisConfigFieldName, "name"); + this.malisConfigFieldDetails = orDefault(malisConfigFieldDetails, "details"); + + this.malisDetailFieldPin = orDefault(malisDetailFieldPin, "pin"); + this.malisDetailFieldStream = orDefault(malisDetailFieldStream, "stream"); + this.malisDetailStreamValue = orDefault(malisDetailStreamValue, "DigitalInput"); + this.malisDetailFieldPinName = orDefault(malisDetailFieldPinName, "signal_name"); + this.malisDetailFieldDescription = orDefault(malisDetailFieldDescription, "description"); + this.malisDetailFieldBad = orDefault(malisDetailFieldBad, "bad"); + this.malisDetailFieldDepartment = orDefault(malisDetailFieldDepartment, "department"); + + this.cacheTtl = cacheTtl == null ? Duration.ofMinutes(10) : cacheTtl; + this.cacheMaxSize = cacheMaxSize <= 0 ? 20_000 : cacheMaxSize; + + this.preloadEnabled = preloadEnabled; + this.preloadOnStartup = preloadOnStartup; + this.preloadRefreshInterval = preloadRefreshInterval == null ? Duration.ofMinutes(10) : preloadRefreshInterval; + this.preloadInitialDelay = preloadInitialDelay == null ? Duration.ofSeconds(5) : preloadInitialDelay; + this.preloadAllowOnDemandFallback = preloadAllowOnDemandFallback; + } + } + + private static String orDefault(String v, String d) { + if (v == null) return d; + String t = v.trim(); + return t.isEmpty() ? d : t; + } + + // -------------------- Internals -------------------- + + private final Config cfg; + private final ObjectMapper om = new ObjectMapper(); + private final HttpClient http; + + private final ExpiringCache portToIotDevice; + private final ExpiringCache deviceToAssignment; + private final ExpiringCache portPinToContext; + + // Preloaded snapshot (for scale): lookup becomes in-memory. + private final AtomicReference snapshotRef = new AtomicReference<>(Snapshot.empty()); + private ScheduledExecutorService preloadScheduler; + + private static final DateTimeFormatter ERP_DATE_TIME = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss"); + private static final DateTimeFormatter ERP_DATE = DateTimeFormatter.ofPattern("yyyy-MM-dd"); + + public ErpnextFaultLookupClient(Config cfg) { + this.cfg = Objects.requireNonNull(cfg, "cfg"); + this.http = HttpClient.newBuilder() + .connectTimeout(Duration.ofMillis(cfg.timeoutMs)) + .build(); + + this.portToIotDevice = new ExpiringCache<>(cfg.cacheTtl, cfg.cacheMaxSize); + this.deviceToAssignment = new ExpiringCache<>(cfg.cacheTtl, cfg.cacheMaxSize); + this.portPinToContext = new ExpiringCache<>(cfg.cacheTtl, cfg.cacheMaxSize); + + if (cfg.preloadEnabled) { + if (cfg.preloadOnStartup) { + try { + refreshSnapshot(); + } catch (Exception e) { + // Don't prevent app startup; optionally fallback to on-demand lookups. + log.warn("ERPNext preload on startup failed (will retry in background): {}", e.toString()); + } + } + startAutoRefresh(); + } + } + + // -------------------- Public API -------------------- + + public FaultLookupContext lookupFaultByPortAndPin(String port, int pin, OffsetDateTime perDatum) { + String p = Objects.requireNonNull(port, "port").trim(); + OffsetDateTime d = Objects.requireNonNull(perDatum, "perDatum"); + String key = p + "|" + pin + "|" + d.toLocalDate(); + + return portPinToContext.computeIfAbsent(key, () -> { + String deviceName = requireSingleIotDeviceByPort(p); + SiteAssignment assignment = resolveSingleSiteAssignmentByIotDevice(deviceName, p, d); + PinDefinition pinDef = resolveUniquePinDefinition(pin, assignment); + return new FaultLookupContext( + p, + deviceName, + assignment.siteId, + assignment.remark, + pinDef.department, + pinDef.pinName, + pinDef.description, + String.valueOf(pinDef.bad) + ); + }); + } + + /** + * Scalable lookup: uses preloaded snapshot if available. + * + *

If preload is enabled but snapshot is not available (yet) or the data is missing, + * the behavior depends on {@code cfg.preloadAllowOnDemandFallback}. + */ + public FaultLookupContext lookupFaultByPortAndPinFast(String port, int pin, OffsetDateTime perDatum) { + if (!cfg.preloadEnabled) { + return lookupFaultByPortAndPin(port, pin, perDatum); + } + + Snapshot snap = snapshotRef.get(); + if (snap != null && !snap.isEmpty()) { + try { + return snapLookup(snap, port, pin, perDatum); + } catch (RuntimeException e) { + if (!cfg.preloadAllowOnDemandFallback) { + throw e; + } + // fall through to on-demand below + } + } else if (!cfg.preloadAllowOnDemandFallback) { + throw new RuntimeException("ERPNext preload is enabled but snapshot is not ready yet"); + } + + return lookupFaultByPortAndPin(port, pin, perDatum); + } + + /** Force refresh now (blocks). Useful for tests / troubleshooting. */ + public void refreshSnapshot() { + Snapshot next = loadSnapshot(); + snapshotRef.set(next); + log.info("ERPNext preload snapshot refreshed: ports={}, devicesWithAssignments={}, configs={}, loadedAt={}", + next.portToDevices.size(), next.deviceToAssignments.size(), next.malisConfigPins.size(), next.loadedAt); + } + + private void startAutoRefresh() { + if (preloadScheduler != null) return; + long initialDelayMs = Math.max(0, cfg.preloadInitialDelay.toMillis()); + long periodMs = Math.max(1_000L, cfg.preloadRefreshInterval.toMillis()); + + preloadScheduler = Executors.newSingleThreadScheduledExecutor(r -> { + Thread t = new Thread(r, "erpnext-preload"); + t.setDaemon(true); + return t; + }); + + preloadScheduler.scheduleWithFixedDelay(() -> { + try { + refreshSnapshot(); + } catch (Exception e) { + log.warn("ERPNext preload refresh failed: {}", e.toString()); + } + }, initialDelayMs, periodMs, TimeUnit.MILLISECONDS); + } + + // -------------------- Preload snapshot -------------------- + + private static final class Snapshot { + final Instant loadedAt; + final Map> portToDevices; // port -> device names (may be multiple) + final Map> deviceToAssignments; // device -> assignments + final Map> malisConfigPins; // config name -> pin -> def + final Map> malisConfigDuplicatePins; // config name -> duplicate pins + + Snapshot(Instant loadedAt, + Map> portToDevices, + Map> deviceToAssignments, + Map> malisConfigPins, + Map> malisConfigDuplicatePins) { + this.loadedAt = loadedAt; + this.portToDevices = portToDevices; + this.deviceToAssignments = deviceToAssignments; + this.malisConfigPins = malisConfigPins; + this.malisConfigDuplicatePins = malisConfigDuplicatePins; + } + + static Snapshot empty() { + return new Snapshot(Instant.EPOCH, Map.of(), Map.of(), Map.of(), Map.of()); + } + + boolean isEmpty() { + return portToDevices.isEmpty() || deviceToAssignments.isEmpty() || malisConfigPins.isEmpty(); + } + } + + private FaultLookupContext snapLookup(Snapshot snap, String port, int pin, OffsetDateTime perDatum) { + String p = Objects.requireNonNull(port, "port").trim(); + OffsetDateTime d = Objects.requireNonNull(perDatum, "perDatum"); + + List devs = snap.portToDevices.get(p); + if (devs == null || devs.isEmpty()) { + throw new PortNotFoundException(p); + } + if (devs.size() != 1) { + throw new PortAmbiguousException(p, devs); + } + String deviceName = devs.get(0); + + SiteAssignment assignment = chooseAssignmentFromSnapshot(snap, deviceName, p, d); + PinDefinition pinDef = choosePinDefFromSnapshot(snap, assignment, pin); + + return new FaultLookupContext( + p, + deviceName, + assignment.siteId, + assignment.remark, + pinDef.department, + pinDef.pinName, + pinDef.description, + String.valueOf(pinDef.bad) + ); + } + + private SiteAssignment chooseAssignmentFromSnapshot(Snapshot snap, String iotDeviceName, String port, OffsetDateTime perDatum) { + List list = snap.deviceToAssignments.get(iotDeviceName); + if (list == null || list.isEmpty()) { + throw new SiteAssignmentNotFoundException(port, perDatum.toString()); + } + + LocalDateTime d = perDatum.toLocalDateTime(); + SiteAssignment best = null; + LocalDateTime bestFrom = null; + List candidateSites = new ArrayList<>(); + + for (SiteAssignment a : list) { + if (a == null) continue; + boolean okFrom = (a.fromDate == null) || !d.isBefore(a.fromDate); + boolean okTo = (a.toDate == null) || !d.isAfter(a.toDate); + if (!okFrom || !okTo) continue; + + if (best == null) { + best = a; + bestFrom = a.fromDate; + candidateSites.add(a.siteId); + continue; + } + + // choose latest fromDate + if (bestFrom == null) { + if (a.fromDate != null) { + best = a; + bestFrom = a.fromDate; + candidateSites.clear(); + candidateSites.add(a.siteId); + } else { + // ambiguous: two matching rows with null from + candidateSites.add(a.siteId); + } + } else { + if (a.fromDate != null && a.fromDate.isAfter(bestFrom)) { + best = a; + bestFrom = a.fromDate; + candidateSites.clear(); + candidateSites.add(a.siteId); + } else if (a.fromDate != null && a.fromDate.equals(bestFrom) && !Objects.equals(a.siteId, best.siteId)) { + candidateSites.add(a.siteId); + } + } + } + + if (best == null) { + throw new SiteAssignmentNotFoundException(port, perDatum.toString()); + } + if (candidateSites.size() > 1) { + LinkedHashSet uniq = new LinkedHashSet<>(candidateSites); + if (uniq.size() > 1) { + throw new SiteAssignmentAmbiguousException(port, perDatum.toString(), new ArrayList<>(uniq)); + } + } + return best; + } + + private PinDefinition choosePinDefFromSnapshot(Snapshot snap, SiteAssignment assignment, int pin) { + String cfgName = assignment.malis; + if (cfgName == null || cfgName.isBlank()) { + throw new RuntimeException("Assignment row missing MalIS configuration name for site " + assignment.siteId); + } + Set dup = snap.malisConfigDuplicatePins.get(cfgName); + if (dup != null && dup.contains(pin)) { + throw new PinDefinedMultipleTimesException(pin, List.of(cfgName + "#" + pin)); + } + Map pins = snap.malisConfigPins.get(cfgName); + if (pins == null || pins.isEmpty()) { + throw new PinNotDefinedException(pin); + } + PinDefinition def = pins.get(pin); + if (def == null) { + throw new PinNotDefinedException(pin); + } + return def; + } + + private Snapshot loadSnapshot() { + Instant now = Instant.now(); + Map> ports = loadAllPortsToDevices(); + Map> assignments = loadAllSiteAssignments(); + MalisConfigLoadResult malis = loadAllMalisConfigs(); + + return new Snapshot( + now, + makeDeepUnmodifiable(ports), + makeDeepUnmodifiableAssignments(assignments), + makeDeepUnmodifiableNested(malis.pinsByConfig), + makeDeepUnmodifiableSetMap(malis.duplicatePinsByConfig) + ); + } + + private static Map> makeDeepUnmodifiable(Map> in) { + Map> out = new LinkedHashMap<>(); + for (var e : in.entrySet()) { + out.put(e.getKey(), List.copyOf(e.getValue())); + } + return Collections.unmodifiableMap(out); + } + + private static Map> makeDeepUnmodifiableAssignments(Map> in) { + Map> out = new LinkedHashMap<>(); + for (var e : in.entrySet()) { + out.put(e.getKey(), List.copyOf(e.getValue())); + } + return Collections.unmodifiableMap(out); + } + + private static Map> makeDeepUnmodifiableNested(Map> in) { + Map> out = new LinkedHashMap<>(); + for (var e : in.entrySet()) { + out.put(e.getKey(), Collections.unmodifiableMap(new LinkedHashMap<>(e.getValue()))); + } + return Collections.unmodifiableMap(out); + } + + private static Map> makeDeepUnmodifiableSetMap(Map> in) { + Map> out = new LinkedHashMap<>(); + for (var e : in.entrySet()) { + out.put(e.getKey(), Collections.unmodifiableSet(new LinkedHashSet<>(e.getValue()))); + } + return Collections.unmodifiableMap(out); + } + + private Map> loadAllPortsToDevices() { + List rows = listAllRows(cfg.doctypeIotDevice, List.of("name", cfg.iotDevicePortAddressField), null); + Map> out = new LinkedHashMap<>(); + for (JsonNode row : rows) { + if (row == null || row.isNull()) continue; + String name = asTextOrNull(row, "name"); + String port = asTextOrNull(row, cfg.iotDevicePortAddressField); + if (name == null || port == null) continue; + out.computeIfAbsent(port.trim(), k -> new ArrayList<>()).add(name.trim()); + } + return out; + } + + private Map> loadAllSiteAssignments() { + List sites = listAllRows(cfg.doctypeSite, List.of("name"), null); + List siteIds = new ArrayList<>(); + for (JsonNode row : sites) { + String id = asTextOrNull(row, "name"); + if (id != null) siteIds.add(id); + } + + Map> out = new LinkedHashMap<>(); + + for (String siteId : siteIds) { + JsonNode siteDoc = tryReadSite(siteId); + if (siteDoc == null) continue; + JsonNode rows = siteDoc.get(cfg.siteFieldIotDevicesAssigned); + if (rows == null || !rows.isArray() || rows.isEmpty()) continue; + + for (JsonNode row : rows) { + if (row == null || row.isNull()) continue; + String dev = asTextOrNull(row, cfg.siteIotDeviceFieldIotDevice); + if (dev == null) continue; + + String malis = asTextOrNull(row, cfg.siteIotDeviceFieldMalis); + String remark = asTextOrNull(row, cfg.siteIotDeviceFieldRemark); + LocalDateTime from = parseErpDate(asTextOrNull(row, cfg.siteIotDeviceFieldFromDate)); + LocalDateTime to = parseErpDate(asTextOrNull(row, cfg.siteIotDeviceFieldToDate)); + + out.computeIfAbsent(dev.trim(), k -> new ArrayList<>()) + .add(new SiteAssignment(siteId, remark, malis, from, to)); + } + } + + // Sort for deterministic lookup (not strictly required) + for (List list : out.values()) { + list.sort(Comparator.comparing((SiteAssignment a) -> a.fromDate, Comparator.nullsFirst(Comparator.naturalOrder()))); + } + return out; + } + + private static final class MalisConfigLoadResult { + final Map> pinsByConfig; + final Map> duplicatePinsByConfig; + + MalisConfigLoadResult(Map> pinsByConfig, + Map> duplicatePinsByConfig) { + this.pinsByConfig = pinsByConfig; + this.duplicatePinsByConfig = duplicatePinsByConfig; + } + } + + private MalisConfigLoadResult loadAllMalisConfigs() { + List configs = listAllRows(cfg.doctypeMalisConfig, List.of("name"), null); + + Map> pinsByConfig = new LinkedHashMap<>(); + Map> dupByConfig = new LinkedHashMap<>(); + + for (JsonNode row : configs) { + String cfgId = asTextOrNull(row, "name"); + if (cfgId == null) continue; + + JsonNode cfgDoc = tryReadMalisConfiguration(cfgId); + if (cfgDoc == null) continue; + JsonNode rows = cfgDoc.get(cfg.malisConfigFieldDetails); + if (rows == null || !rows.isArray() || rows.isEmpty()) continue; + + Map pins = new LinkedHashMap<>(); + Set dupPins = new LinkedHashSet<>(); + + for (JsonNode drow : rows) { + if (drow == null || drow.isNull()) continue; + Integer pin = asIntOrNull(drow, cfg.malisDetailFieldPin); + if (pin == null) continue; + + // constrain to DigitalInput by default + if (cfg.malisDetailFieldStream != null && !cfg.malisDetailFieldStream.isBlank() + && cfg.malisDetailStreamValue != null && !cfg.malisDetailStreamValue.isBlank()) { + String stream = asTextOrNull(drow, cfg.malisDetailFieldStream); + if (!equalsIgnoreCaseTrimmed(stream, cfg.malisDetailStreamValue)) { + continue; + } + } + + if (pins.containsKey(pin)) { + dupPins.add(pin); + continue; + } + pins.put(pin, pinDefinitionFromRow(drow, "MalIS Configuration.details (" + cfgId + ")")); + } + + if (!pins.isEmpty()) { + pinsByConfig.put(cfgId, pins); + } + if (!dupPins.isEmpty()) { + dupByConfig.put(cfgId, dupPins); + } + } + + return new MalisConfigLoadResult(pinsByConfig, dupByConfig); + } + + private List listAllRows(String doctype, List fields, String filtersJsonOrNull) { + int start = 0; + int limit = 1000; + List out = new ArrayList<>(); + + while (true) { + try { + String fieldsJson = om.writeValueAsString(fields); + UriComponentsBuilder b = UriComponentsBuilder + .fromHttpUrl(cfg.baseUrl + "/api/resource/" + encodePath(doctype)) + .queryParam("fields", fieldsJson) + .queryParam("limit_page_length", limit) + .queryParam("limit_start", start); + if (filtersJsonOrNull != null && !filtersJsonOrNull.isBlank()) { + b = b.queryParam("filters", filtersJsonOrNull); + } + URI uri = b.build().encode(StandardCharsets.UTF_8).toUri(); + + JsonNode body = getJson(uri); + JsonNode data = body != null ? body.get("data") : null; + if (data == null || !data.isArray() || data.isEmpty()) { + break; + } + for (JsonNode row : data) { + out.add(row); + } + if (data.size() < limit) { + break; + } + start += limit; + } catch (ForbiddenException e) { + // Listing is forbidden -> return what we have so far. + log.warn("ERPNext list forbidden for doctype {}: {}", doctype, e.getMessage()); + break; + } catch (RuntimeException e) { + throw e; + } catch (Exception e) { + throw new RuntimeException("ERPNext list failed for doctype " + doctype, e); + } + } + + return out; + } + + // -------------------- Lookups -------------------- + + private String requireSingleIotDeviceByPort(String port) { + String key = "port:" + port; + String cached = portToIotDevice.get(key); + if (cached != null) return cached; + + try { + String filtersJson = om.writeValueAsString(List.of( + List.of(cfg.iotDevicePortAddressField, "=", port) + )); + String fieldsJson = om.writeValueAsString(List.of("name", cfg.iotDevicePortAddressField)); + + URI uri = UriComponentsBuilder + .fromHttpUrl(cfg.baseUrl + "/api/resource/" + encodePath(cfg.doctypeIotDevice)) + .queryParam("filters", filtersJson) + .queryParam("fields", fieldsJson) + .queryParam("limit_page_length", 20) + .build() + .encode(StandardCharsets.UTF_8) + .toUri(); + + JsonNode body = getJson(uri); + JsonNode data = body != null ? body.get("data") : null; + if (data == null || !data.isArray() || data.isEmpty()) { + throw new PortNotFoundException(port); + } + if (data.size() > 1) { + List names = new ArrayList<>(); + data.forEach(n -> names.add(n.path("name").asText())); + throw new PortAmbiguousException(port, names); + } + String name = data.get(0).path("name").asText(); + portToIotDevice.put(key, name); + return name; + } catch (RuntimeException e) { + throw e; + } catch (Exception e) { + throw new RuntimeException("Failed to lookup IoT_Device by port: " + port, e); + } + } + + private static final class SiteAssignment { + final String siteId; + final String remark; + final String malis; + final LocalDateTime fromDate; + final LocalDateTime toDate; + private SiteAssignment(String siteId, String remark, String malis, LocalDateTime fromDate, LocalDateTime toDate) { + this.siteId = siteId; + this.remark = remark; + this.malis = malis; + this.fromDate = fromDate; + this.toDate = toDate; + } + } + + private SiteAssignment resolveSingleSiteAssignmentByIotDevice(String iotDeviceName, String port, OffsetDateTime perDatum) { + String key = "assign:" + iotDeviceName + "|" + perDatum.toLocalDate(); + SiteAssignment cached = deviceToAssignment.get(key); + if (cached != null) return cached; + + LocalDateTime d = perDatum.toLocalDateTime(); + List candidates = listCandidateSites(iotDeviceName); + + Map bestBySite = new LinkedHashMap<>(); + + for (String siteId : candidates) { + if (siteId == null || siteId.isBlank()) continue; + JsonNode siteDoc = tryReadSite(siteId); + if (siteDoc == null) continue; + JsonNode rows = siteDoc.get(cfg.siteFieldIotDevicesAssigned); + if (rows == null || !rows.isArray() || rows.isEmpty()) continue; + + SiteAssignment best = findBestAssignmentRow(siteId, rows, iotDeviceName, d); + if (best != null) bestBySite.put(siteId, best); + } + + if (bestBySite.isEmpty()) { + throw new SiteAssignmentNotFoundException(port, perDatum.toString()); + } + if (bestBySite.size() != 1) { + throw new SiteAssignmentAmbiguousException(port, perDatum.toString(), new ArrayList<>(bestBySite.keySet())); + } + + SiteAssignment resolved = bestBySite.values().iterator().next(); + deviceToAssignment.put(key, resolved); + return resolved; + } + + private List listCandidateSites(String iotDeviceName) { + // 1) Try narrow by child table filter + try { + String filtersJson = om.writeValueAsString(List.of( + List.of(cfg.doctypeSiteIotDevice, cfg.siteIotDeviceFieldIotDevice, "=", iotDeviceName) + )); + String fieldsJson = om.writeValueAsString(List.of("name")); + + URI uri = UriComponentsBuilder + .fromHttpUrl(cfg.baseUrl + "/api/resource/" + encodePath(cfg.doctypeSite)) + .queryParam("filters", filtersJson) + .queryParam("fields", fieldsJson) + .queryParam("limit_page_length", 1000) + .build() + .encode(StandardCharsets.UTF_8) + .toUri(); + + JsonNode body = getJson(uri); + List ids = extractNames(body); + if (!ids.isEmpty()) return ids; + } catch (Exception ignore) { + // fall back + } + + // 2) fallback list all sites + try { + String fieldsJson = om.writeValueAsString(List.of("name")); + URI uri = UriComponentsBuilder + .fromHttpUrl(cfg.baseUrl + "/api/resource/" + encodePath(cfg.doctypeSite)) + .queryParam("fields", fieldsJson) + .queryParam("limit_page_length", 2000) + .build() + .encode(StandardCharsets.UTF_8) + .toUri(); + + JsonNode body = getJson(uri); + return extractNames(body); + } catch (Exception ex) { + throw ex instanceof RuntimeException ? (RuntimeException) ex : new RuntimeException(ex); + } + } + + private List extractNames(JsonNode body) { + JsonNode data = body != null ? body.get("data") : null; + if (data == null || !data.isArray() || data.isEmpty()) return Collections.emptyList(); + List ids = new ArrayList<>(); + for (JsonNode row : data) { + if (row == null || row.isNull()) continue; + JsonNode n = row.get("name"); + if (n != null && !n.isNull()) { + String s = n.asText(); + if (s != null && !s.isBlank()) ids.add(s); + } + } + return ids; + } + + private JsonNode tryReadSite(String siteId) { + try { + String fieldsJson = om.writeValueAsString(List.of("name", cfg.siteFieldIotDevicesAssigned)); + URI uri = UriComponentsBuilder + .fromHttpUrl(cfg.baseUrl + "/api/resource/" + encodePath(cfg.doctypeSite) + "/" + encodePath(siteId)) + .queryParam("fields", fieldsJson) + .build() + .encode(StandardCharsets.UTF_8) + .toUri(); + JsonNode body = getJson(uri); + return body != null ? body.get("data") : null; + } catch (ForbiddenException forbidden) { + return null; + } catch (Exception ex) { + return null; + } + } + + private SiteAssignment findBestAssignmentRow(String siteId, JsonNode rows, String iotDeviceName, LocalDateTime d) { + SiteAssignment best = null; + LocalDateTime bestFrom = null; + + for (JsonNode row : rows) { + if (row == null || row.isNull()) continue; + String dev = asTextOrNull(row, cfg.siteIotDeviceFieldIotDevice); + if (!equalsIgnoreCaseTrimmed(dev, iotDeviceName)) continue; + + String malis = asTextOrNull(row, cfg.siteIotDeviceFieldMalis); + LocalDateTime from = parseErpDate(asTextOrNull(row, cfg.siteIotDeviceFieldFromDate)); + LocalDateTime to = parseErpDate(asTextOrNull(row, cfg.siteIotDeviceFieldToDate)); + + boolean okFrom = (from == null) || !d.isBefore(from); + boolean okTo = (to == null) || !d.isAfter(to); + if (!okFrom || !okTo) continue; + + String remark = asTextOrNull(row, cfg.siteIotDeviceFieldRemark); + + if (best == null) { + best = new SiteAssignment(siteId, remark, malis, from, to); + bestFrom = from; + continue; + } + + // Choose the row with the latest from_date + if (bestFrom == null) { + if (from != null) { + best = new SiteAssignment(siteId, remark, malis, from, to); + bestFrom = from; + } + } else { + if (from != null && from.isAfter(bestFrom)) { + best = new SiteAssignment(siteId, remark, malis, from, to); + bestFrom = from; + } + } + } + + return best; + } + + private static final class PinDefinition { + final int bad; + final String department; + final String pinName; + final String description; + private PinDefinition(int bad, String department, String pinName, String description) { + this.bad = bad; + this.department = department; + this.pinName = pinName; + this.description = description; + } + } + + private PinDefinition resolveUniquePinDefinition(int pin, SiteAssignment assignment) { + // 1) Fast path: direct get_list on MalIS Configuration Detail (works only if permissions allow) + try { + return resolveUniquePinDefinitionDirect(pin, assignment); + } catch (ForbiddenException forbidden) { + return resolveUniquePinDefinitionViaMalisConfigurations(pin, assignment); + } catch (RuntimeException re) { + throw re; + } catch (Exception e) { + throw new RuntimeException("Failed to lookup pin definition for pin " + pin, e); + } + } + + private PinDefinition resolveUniquePinDefinitionDirect(int pin, SiteAssignment assignment) throws Exception { + List filters = new ArrayList<>(); + filters.add(List.of(cfg.malisDetailFieldPin, "=", pin)); + if (cfg.malisDetailFieldStream != null && !cfg.malisDetailFieldStream.isBlank() + && cfg.malisDetailStreamValue != null && !cfg.malisDetailStreamValue.isBlank()) { + filters.add(List.of(cfg.malisDetailFieldStream, "=", cfg.malisDetailStreamValue)); + } + + String filtersJson = om.writeValueAsString(filters); + List fields = List.of( + "name", + cfg.malisDetailFieldPin, + cfg.malisDetailFieldPinName, + cfg.malisDetailFieldDescription, + cfg.malisDetailFieldBad, + cfg.malisDetailFieldDepartment + ); + String fieldsJson = om.writeValueAsString(fields); + + URI uri = UriComponentsBuilder + .fromHttpUrl(cfg.baseUrl + "/api/resource/" + encodePath(cfg.doctypeMalisConfigDetail)) + .queryParam("filters", filtersJson) + .queryParam("fields", fieldsJson) + .queryParam("limit_page_length", 20) + .build() + .encode(StandardCharsets.UTF_8) + .toUri(); + + JsonNode body = getJson(uri); + JsonNode data = body != null ? body.get("data") : null; + if (data == null || !data.isArray() || data.isEmpty()) { + throw new PinNotDefinedException(pin); + } + if (data.size() > 1) { + List defs = new ArrayList<>(); + data.forEach(n -> defs.add(n.path("name").asText())); + throw new PinDefinedMultipleTimesException(pin, defs); + } + JsonNode row = data.get(0); + return pinDefinitionFromRow(row, "MalIS Configuration Detail"); + } + + private PinDefinition resolveUniquePinDefinitionViaMalisConfigurations(int pin, SiteAssignment assignment) { + try { + List configIds = listCandidateMalisConfigurations(pin); + if (configIds.isEmpty()) configIds = listAllMalisConfigurations(); + + List matches = new ArrayList<>(); + + for (String cfgId : configIds) { + if (cfgId == null || cfgId.isBlank()) continue; + JsonNode cfgDoc = tryReadMalisConfiguration(cfgId); + if (cfgDoc == null) continue; + + String name = asTextOrNull(cfgDoc, cfg.malisConfigFieldName); + if (assignment.malis != null && name != null && !name.equals(assignment.malis)) { + continue; + } + + JsonNode rows = cfgDoc.get(cfg.malisConfigFieldDetails); + if (rows == null || !rows.isArray() || rows.isEmpty()) continue; + + for (JsonNode row : rows) { + if (row == null || row.isNull()) continue; + Integer rowPin = asIntOrNull(row, cfg.malisDetailFieldPin); + if (rowPin == null || rowPin != pin) continue; + + if (cfg.malisDetailFieldStream != null && !cfg.malisDetailFieldStream.isBlank() + && cfg.malisDetailStreamValue != null && !cfg.malisDetailStreamValue.isBlank()) { + String stream = asTextOrNull(row, cfg.malisDetailFieldStream); + if (!equalsIgnoreCaseTrimmed(stream, cfg.malisDetailStreamValue)) continue; + } + + String rowName = asTextOrNull(row, "name"); + String descriptor = cfgId + (rowName != null ? ("#" + rowName) : ""); + matches.add(new PinRowMatch(descriptor, row)); + } + } + + if (matches.isEmpty()) throw new PinNotDefinedException(pin); + if (matches.size() > 1) { + List defs = new ArrayList<>(); + for (PinRowMatch m : matches) defs.add(m.descriptor); + throw new PinDefinedMultipleTimesException(pin, defs); + } + + PinRowMatch only = matches.get(0); + return pinDefinitionFromRow(only.row, "MalIS Configuration.details (" + only.descriptor + ")"); + + } catch (RuntimeException e) { + throw e; + } catch (Exception e) { + throw new RuntimeException("Failed to lookup pin definition for pin " + pin + " via MalIS Configuration", e); + } + } + + private List listCandidateMalisConfigurations(int pin) { + try { + List filters = new ArrayList<>(); + filters.add(List.of(cfg.doctypeMalisConfigDetail, cfg.malisDetailFieldPin, "=", pin)); + if (cfg.malisDetailFieldStream != null && !cfg.malisDetailFieldStream.isBlank() + && cfg.malisDetailStreamValue != null && !cfg.malisDetailStreamValue.isBlank()) { + filters.add(List.of(cfg.doctypeMalisConfigDetail, cfg.malisDetailFieldStream, "=", cfg.malisDetailStreamValue)); + } + String filtersJson = om.writeValueAsString(filters); + String fieldsJson = om.writeValueAsString(List.of("name")); + + URI uri = UriComponentsBuilder + .fromHttpUrl(cfg.baseUrl + "/api/resource/" + encodePath(cfg.doctypeMalisConfig)) + .queryParam("filters", filtersJson) + .queryParam("fields", fieldsJson) + .queryParam("limit_page_length", 1000) + .build() + .encode(StandardCharsets.UTF_8) + .toUri(); + + JsonNode body = getJson(uri); + return extractNames(body); + } catch (ForbiddenException e) { + return Collections.emptyList(); + } catch (Exception ex) { + return Collections.emptyList(); + } + } + + private List listAllMalisConfigurations() { + try { + String fieldsJson = om.writeValueAsString(List.of("name")); + URI uri = UriComponentsBuilder + .fromHttpUrl(cfg.baseUrl + "/api/resource/" + encodePath(cfg.doctypeMalisConfig)) + .queryParam("fields", fieldsJson) + .queryParam("limit_page_length", 2000) + .build() + .encode(StandardCharsets.UTF_8) + .toUri(); + JsonNode body = getJson(uri); + return extractNames(body); + } catch (Exception ex) { + return Collections.emptyList(); + } + } + + private JsonNode tryReadMalisConfiguration(String configId) { + try { + String fieldsJson = om.writeValueAsString(List.of("name", cfg.malisConfigFieldDetails)); + URI uri = UriComponentsBuilder + .fromHttpUrl(cfg.baseUrl + "/api/resource/" + encodePath(cfg.doctypeMalisConfig) + "/" + encodePath(configId)) + .queryParam("fields", fieldsJson) + .build() + .encode(StandardCharsets.UTF_8) + .toUri(); + JsonNode body = getJson(uri); + return body != null ? body.get("data") : null; + } catch (ForbiddenException e) { + return null; + } catch (Exception ex) { + return null; + } + } + + private static final class PinRowMatch { + final String descriptor; + final JsonNode row; + PinRowMatch(String descriptor, JsonNode row) { + this.descriptor = descriptor; + this.row = row; + } + } + + private PinDefinition pinDefinitionFromRow(JsonNode row, String sourceHint) { + if (row == null || row.isNull()) throw new RuntimeException("Pin definition row missing (" + sourceHint + ")"); + + Integer bad = asIntOrNull(row, cfg.malisDetailFieldBad); + String department = asTextOrNull(row, cfg.malisDetailFieldDepartment); + String pinName = asTextOrNull(row, cfg.malisDetailFieldPinName); + String description = asTextOrNull(row, cfg.malisDetailFieldDescription); + + int badVal = bad != null ? bad : 1; + return new PinDefinition(badVal, department, pinName, description); + } + + // -------------------- JSON helpers -------------------- + + private static Integer asIntOrNull(JsonNode node, String field) { + if (node == null || node.isNull() || field == null || field.isBlank()) return null; + JsonNode v = node.get(field); + if (v == null || v.isNull()) return null; + if (v.isInt() || v.isLong() || v.isNumber()) return v.asInt(); + String s = v.asText(null); + if (s == null) return null; + s = s.trim(); + if (s.isEmpty()) return null; + try { return Integer.parseInt(s); } catch (Exception ignore) { return null; } + } + + private static String asTextOrNull(JsonNode node, String field) { + if (node == null || node.isNull() || field == null || field.isBlank()) return null; + JsonNode v = node.get(field); + if (v == null || v.isNull()) return null; + String t = v.asText(); + if (t == null) return null; + t = t.trim(); + return t.isEmpty() ? null : t; + } + + private static boolean equalsIgnoreCaseTrimmed(String a, String b) { + if (a == null && b == null) return true; + if (a == null || b == null) return false; + return a.trim().equalsIgnoreCase(b.trim()); + } + + private static LocalDateTime parseErpDate(String s) { + if (s == null) return null; + String t = s.trim(); + if (t.isEmpty()) return null; + try { return LocalDateTime.parse(t, ERP_DATE_TIME); } catch (DateTimeParseException ignore) {} + try { return LocalDate.parse(t, ERP_DATE).atStartOfDay(); } catch (DateTimeParseException ignore) {} + try { return OffsetDateTime.parse(t).toLocalDateTime(); } catch (DateTimeParseException ignore) {} + try { return LocalDateTime.parse(t); } catch (DateTimeParseException ignore) { return null; } + } + + // -------------------- HTTP helpers -------------------- + + private JsonNode getJson(URI uri) { + try { + HttpRequest req = HttpRequest.newBuilder(uri) + .timeout(Duration.ofMillis(cfg.timeoutMs)) + .header("Authorization", "token " + cfg.apiKey + ":" + cfg.apiSecret) + .header("Accept", "application/json") + .GET() + .build(); + HttpResponse resp = http.send(req, HttpResponse.BodyHandlers.ofString(StandardCharsets.UTF_8)); + int code = resp.statusCode(); + if (code == 403) { + throw new ForbiddenException("ERPNext 403 forbidden: " + uri); + } + if (code < 200 || code >= 300) { + throw new RuntimeException("ERPNext request failed: " + code + " " + uri + " body=" + safeBody(resp.body())); + } + String body = resp.body(); + if (body == null || body.isBlank()) return null; + return om.readTree(body); + } catch (RuntimeException e) { + throw e; + } catch (Exception e) { + throw new RuntimeException("ERPNext request failed: " + uri, e); + } + } + + private static String safeBody(String s) { + if (s == null) return ""; + String t = s.trim(); + if (t.length() > 400) return t.substring(0, 400) + "..."; + return t; + } + + private static String encodePath(String segment) { + return segment; //URLEncoder.encode(segment, StandardCharsets.UTF_8).replace("+", "%20"); + } +} diff --git a/src/main/java/at/co/procon/malis/cep/event/CepEvent.java b/src/main/java/at/co/procon/malis/cep/event/CepEvent.java new file mode 100644 index 0000000..b11d380 --- /dev/null +++ b/src/main/java/at/co/procon/malis/cep/event/CepEvent.java @@ -0,0 +1,22 @@ +package at.co.procon.malis.cep.event; + +import java.time.Instant; +import java.util.Map; + +/** + * Canonical internal input event model. + * + *

This is intentionally small and transport/protocol agnostic. + * Parsers/converters can map EEBUS, JSON, etc. into these events. + */ +public interface CepEvent { + + /** Stable identifier of the signal/metric, e.g. {@code port.1.state}. */ + String getKey(); + + /** Timestamp of the event, if known. */ + Instant getOccurredAt(); + + /** Optional metadata (unit, scope, raw fields, ...). */ + Map getMeta(); +} diff --git a/src/main/java/at/co/procon/malis/cep/event/EventBatch.java b/src/main/java/at/co/procon/malis/cep/event/EventBatch.java new file mode 100644 index 0000000..d61f6d0 --- /dev/null +++ b/src/main/java/at/co/procon/malis/cep/event/EventBatch.java @@ -0,0 +1,47 @@ +package at.co.procon.malis.cep.event; + +import java.time.Instant; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +/** + * A transport-agnostic batch of canonical input events. + */ +public final class EventBatch { + + /** + * Partition key used by stateful detectors. + * + *

This should be stable for a given ingress source (binding/transport/connection). + * It is intentionally not the same as "deviceId" which might be missing or not unique. + */ + private final String sourceKey; + + private final String deviceId; + private final Instant timestamp; + private final List events; + + public EventBatch(String sourceKey, String deviceId, Instant timestamp, List events) { + this.sourceKey = sourceKey; + this.deviceId = deviceId; + this.timestamp = timestamp == null ? Instant.now() : timestamp; + this.events = events == null ? List.of() : Collections.unmodifiableList(new ArrayList<>(events)); + } + + /** Backwards-compatible constructor (sourceKey defaults to deviceId). */ + public EventBatch(String deviceId, Instant timestamp, List events) { + this(deviceId, deviceId, timestamp, events); + } + + public String getSourceKey() { return sourceKey; } + public String getDeviceId() { return deviceId; } + public Instant getTimestamp() { return timestamp; } + public List getEvents() { return events; } + + // record-style accessors + public String sourceKey() { return sourceKey; } + public String deviceId() { return deviceId; } + public Instant timestamp() { return timestamp; } + public List events() { return events; } +} diff --git a/src/main/java/at/co/procon/malis/cep/event/SignalDefinitionEvent.java b/src/main/java/at/co/procon/malis/cep/event/SignalDefinitionEvent.java new file mode 100644 index 0000000..a9b221f --- /dev/null +++ b/src/main/java/at/co/procon/malis/cep/event/SignalDefinitionEvent.java @@ -0,0 +1,40 @@ +package at.co.procon.malis.cep.event; + +import java.time.Instant; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.Map; + +/** + * Declares a signal/measurement definition (type, unit, ...). + * + *

Produced when a source provides metadata (e.g. EEBUS measurementDescriptionListData). + */ +public final class SignalDefinitionEvent implements CepEvent { + + private final String key; + private final SignalValueType valueType; + private final Instant occurredAt; + private final Map meta; + + public SignalDefinitionEvent(String key, + SignalValueType valueType, + Instant occurredAt, + Map meta) { + this.key = key; + this.valueType = valueType; + this.occurredAt = occurredAt == null ? Instant.now() : occurredAt; + this.meta = meta == null ? Map.of() : Collections.unmodifiableMap(new LinkedHashMap<>(meta)); + } + + @Override public String getKey() { return key; } + public SignalValueType getValueType() { return valueType; } + @Override public Instant getOccurredAt() { return occurredAt; } + @Override public Map getMeta() { return meta; } + + // record-style accessors + public String key() { return key; } + public SignalValueType valueType() { return valueType; } + public Instant occurredAt() { return occurredAt; } + public Map meta() { return meta; } +} diff --git a/src/main/java/at/co/procon/malis/cep/event/SignalUpdateEvent.java b/src/main/java/at/co/procon/malis/cep/event/SignalUpdateEvent.java new file mode 100644 index 0000000..1144617 --- /dev/null +++ b/src/main/java/at/co/procon/malis/cep/event/SignalUpdateEvent.java @@ -0,0 +1,35 @@ +package at.co.procon.malis.cep.event; + +import java.time.Instant; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.Map; + +/** + * Measurement update (a typed value for a signal key). + */ +public final class SignalUpdateEvent implements CepEvent { + + private final String key; + private final TypedValue value; + private final Instant occurredAt; + private final Map meta; + + public SignalUpdateEvent(String key, TypedValue value, Instant occurredAt, Map meta) { + this.key = key; + this.value = value; + this.occurredAt = occurredAt == null ? Instant.now() : occurredAt; + this.meta = meta == null ? Map.of() : Collections.unmodifiableMap(new LinkedHashMap<>(meta)); + } + + @Override public String getKey() { return key; } + public TypedValue getValue() { return value; } + @Override public Instant getOccurredAt() { return occurredAt; } + @Override public Map getMeta() { return meta; } + + // record-style accessors + public String key() { return key; } + public TypedValue value() { return value; } + public Instant occurredAt() { return occurredAt; } + public Map meta() { return meta; } +} diff --git a/src/main/java/at/co/procon/malis/cep/event/SignalValueType.java b/src/main/java/at/co/procon/malis/cep/event/SignalValueType.java new file mode 100644 index 0000000..bab8fd9 --- /dev/null +++ b/src/main/java/at/co/procon/malis/cep/event/SignalValueType.java @@ -0,0 +1,10 @@ +package at.co.procon.malis.cep.event; + +/** + * Basic value types understood by the internal detector. + */ +public enum SignalValueType { + BOOLEAN, + DOUBLE, + STRING +} diff --git a/src/main/java/at/co/procon/malis/cep/event/TypedValue.java b/src/main/java/at/co/procon/malis/cep/event/TypedValue.java new file mode 100644 index 0000000..fce57b3 --- /dev/null +++ b/src/main/java/at/co/procon/malis/cep/event/TypedValue.java @@ -0,0 +1,41 @@ +package at.co.procon.malis.cep.event; + +import java.util.Objects; + +/** + * A value tagged with a {@link SignalValueType}. + */ +public final class TypedValue { + + private final SignalValueType type; + private final Object value; + + public TypedValue(SignalValueType type, Object value) { + this.type = Objects.requireNonNull(type, "type"); + this.value = value; + } + + public SignalValueType getType() { return type; } + public Object getValue() { return value; } + + public boolean isNull() { return value == null; } + + public Boolean asBoolean() { + return value instanceof Boolean b ? b : null; + } + + public Double asDouble() { + if (value instanceof Double d) return d; + if (value instanceof Number n) return n.doubleValue(); + return null; + } + + public String asString() { + return value == null ? null : String.valueOf(value); + } + + @Override + public String toString() { + return "TypedValue{" + type + ", value=" + value + "}"; + } +} diff --git a/src/main/java/at/co/procon/malis/cep/event/convert/EebusMeasurementDatagramConverter.java b/src/main/java/at/co/procon/malis/cep/event/convert/EebusMeasurementDatagramConverter.java new file mode 100644 index 0000000..4b33cde --- /dev/null +++ b/src/main/java/at/co/procon/malis/cep/event/convert/EebusMeasurementDatagramConverter.java @@ -0,0 +1,239 @@ +package at.co.procon.malis.cep.event.convert; + +import at.co.procon.malis.cep.detector.DetectionContext; +import at.co.procon.malis.cep.eebus.EebusMeasurementDatagram; +import at.co.procon.malis.cep.eebus.EebusXmlUtil; +import at.co.procon.malis.cep.event.*; +import at.co.procon.malis.cep.parser.ParsedInput; +import org.w3c.dom.Document; +import org.w3c.dom.Node; +import org.w3c.dom.NodeList; + +import javax.xml.xpath.XPath; +import javax.xml.xpath.XPathConstants; +import javax.xml.xpath.XPathFactory; +import java.nio.charset.StandardCharsets; +import java.time.Instant; +import java.time.LocalDateTime; +import java.time.OffsetDateTime; +import java.time.ZoneOffset; +import java.util.*; +import java.util.concurrent.ConcurrentHashMap; + +/** + * Converts an {@link EebusMeasurementDatagram} (full SPINE-like XML) into a canonical {@link EventBatch}. + * + *

It supports the pattern where measurementDescriptionListData and measurementListData may arrive: + *

    + *
  • in the same message/datagram, or
    • + *
  • as separate datagrams (definitions first, values later)
    • + *
+ * by keeping an in-memory cache of the latest definitions per device. + */ +public final class EebusMeasurementDatagramConverter implements InputEventConverter { + + /** + * Per sourceKey: measurementId -> type. + * + *

Definitions are cached per ingress source to avoid cross-talk when multiple devices + * share the same deviceId or when deviceId is missing. + */ + private final ConcurrentHashMap> definitions = new ConcurrentHashMap<>(); + + @Override + public boolean supports(Object body) { + return body instanceof EebusMeasurementDatagram; + } + + @Override + public EventBatch convert(ParsedInput input, DetectionContext ctx) throws Exception { + EebusMeasurementDatagram md = (EebusMeasurementDatagram) input.getBody(); + + String sourceKey = ctx.getVars().get("sourceKey"); + + // Choose deviceId (binding var wins, then datagram header addressSource). + String deviceId = ctx.getVars().get("deviceId"); + if (deviceId == null || deviceId.isBlank()) deviceId = md.getAddressSource(); + + if (sourceKey == null || sourceKey.isBlank()) { + sourceKey = (deviceId == null || deviceId.isBlank()) ? "unknown-source" : deviceId; + } + + Instant fallbackTs = input.getReceivedAt() != null ? input.getReceivedAt() : Instant.now(); + Instant batchTs = fallbackTs; + + byte[] xml = md.getDatagramXml() == null ? new byte[0] : md.getDatagramXml().getBytes(StandardCharsets.UTF_8); + Document doc = EebusXmlUtil.parse(xml); + XPath xp = XPathFactory.newInstance().newXPath(); + + List out = new ArrayList<>(); + + // ---------- 1) ingest definitions ---------- + NodeList defNodes = (NodeList) xp.evaluate( + "//*[local-name()='measurementDescriptionData']", + doc, + XPathConstants.NODESET + ); + + if (defNodes != null && defNodes.getLength() > 0) { + Map sourceDefs = definitions.computeIfAbsent( + sourceKey, + k -> new ConcurrentHashMap<>() + ); + + for (int i = 0; i < defNodes.getLength(); i++) { + Node n = defNodes.item(i); + String id = text(xp, n, ".//*[local-name()='measurementId']"); + if (id == null || id.isBlank()) continue; + + String t = text(xp, n, ".//*[local-name()='measurementType']"); + SignalValueType type = mapType(t); + + SignalValueType prev = sourceDefs.put(id, type); + if (prev == null || prev != type) { + Map meta = new LinkedHashMap<>(); + meta.put("source", "eebus"); + if (t != null) meta.put("measurementType", t); + out.add(new SignalDefinitionEvent(id, type, fallbackTs, meta)); + } + } + } + + // ---------- 2) value updates ---------- + NodeList valueNodes = (NodeList) xp.evaluate( + "//*[local-name()='measurementData']", + doc, + XPathConstants.NODESET + ); + + Map sourceDefs = definitions.get(sourceKey); + + if (valueNodes != null) { + for (int i = 0; i < valueNodes.getLength(); i++) { + Node n = valueNodes.item(i); + String id = text(xp, n, ".//*[local-name()='measurementId']"); + if (id == null || id.isBlank()) continue; + + String raw = text(xp, n, ".//*[local-name()='value']"); + Instant eventTs = readTimestamp(xp, n); + if (eventTs == null) eventTs = fallbackTs; + //if (eventTs.isAfter(batchTs)) batchTs = eventTs; + + SignalValueType declared = sourceDefs != null ? sourceDefs.get(id) : null; + TypedValue tv = parseValue(raw, declared); + + Map meta = new LinkedHashMap<>(); + meta.put("source", "eebus"); + meta.put("raw", raw); + if (declared != null) meta.put("declaredType", declared.name()); + + out.add(new SignalUpdateEvent(id, tv, eventTs, meta)); + } + } + + return new EventBatch(sourceKey, deviceId, batchTs, out); + } + + /** + * Best-effort timestamp parsing for EEBUS measurementData. + * + *

Supported input formats: + *

    + *
  • ISO-8601 instant / offset datetime (e.g. 2026-01-12T10:00:00Z)
    • + *
  • ISO-8601 local datetime (assumed UTC) (e.g. 2026-01-12T10:00:00)
    • + *
  • epoch seconds (10 digits) or epoch millis (13+ digits)
    • + *
+ */ + private static Instant readTimestamp(XPath xp, Node measurementDataNode) { + try { + // Common spellings seen across XML models. + String raw = text(xp, measurementDataNode, + "(.//*[local-name()='timestamp'] | .//*[local-name()='timeStamp'] | .//*[local-name()='measurementTimeStamp'])[1]"); + if (raw == null || raw.isBlank()) return null; + raw = raw.trim(); + + // epoch (seconds or millis) + if (raw.matches("^-?\\d{10,}$")) { + long v = Long.parseLong(raw); + // Heuristic: 13+ digits => millis + if (raw.length() >= 13) return Instant.ofEpochMilli(v); + return Instant.ofEpochSecond(v); + } + + // ISO-8601 with offset or Z + try { + return Instant.parse(raw); + } catch (Exception ignore) { + // fall through + } + + // OffsetDateTime + try { + return OffsetDateTime.parse(raw).toInstant(); + } catch (Exception ignore) { + // fall through + } + + // LocalDateTime (assume UTC) + try { + return LocalDateTime.parse(raw).toInstant(ZoneOffset.UTC); + } catch (Exception ignore) { + return null; + } + } catch (Exception ignore) { + return null; + } + } + + private static SignalValueType mapType(String measurementType) { + if (measurementType == null) return SignalValueType.STRING; + String t = measurementType.trim().toLowerCase(Locale.ROOT); + if (t.contains("bool")) return SignalValueType.BOOLEAN; + if (t.contains("double") || t.contains("float") || t.contains("decimal") || t.contains("number") || t.contains("int")) { + return SignalValueType.DOUBLE; + } + return SignalValueType.STRING; + } + + private static TypedValue parseValue(String raw, SignalValueType declared) { + if (declared == SignalValueType.BOOLEAN) { + Boolean b = tryParseBoolean(raw); + return new TypedValue(SignalValueType.BOOLEAN, b); + } + if (declared == SignalValueType.DOUBLE) { + Double d = tryParseDouble(raw); + return new TypedValue(SignalValueType.DOUBLE, d); + } + + // No declared type - try best effort + Boolean b = tryParseBoolean(raw); + if (b != null) return new TypedValue(SignalValueType.BOOLEAN, b); + Double d = tryParseDouble(raw); + if (d != null) return new TypedValue(SignalValueType.DOUBLE, d); + return new TypedValue(SignalValueType.STRING, raw); + } + + private static Boolean tryParseBoolean(String raw) { + if (raw == null) return null; + String t = raw.trim().toLowerCase(Locale.ROOT); + if (t.equals("true") || t.equals("1") || t.equals("on")) return Boolean.TRUE; + if (t.equals("false") || t.equals("0") || t.equals("off")) return Boolean.FALSE; + return null; + } + + private static Double tryParseDouble(String raw) { + if (raw == null) return null; + try { + return Double.parseDouble(raw.trim()); + } catch (Exception ignore) { + return null; + } + } + + private static String text(XPath xp, Node ctx, String expr) throws Exception { + Node n = (Node) xp.evaluate(expr, ctx, XPathConstants.NODE); + if (n == null) return null; + String t = n.getTextContent(); + return t == null ? null : t.trim(); + } +} diff --git a/src/main/java/at/co/procon/malis/cep/event/convert/InputEventConverter.java b/src/main/java/at/co/procon/malis/cep/event/convert/InputEventConverter.java new file mode 100644 index 0000000..6f05736 --- /dev/null +++ b/src/main/java/at/co/procon/malis/cep/event/convert/InputEventConverter.java @@ -0,0 +1,15 @@ +package at.co.procon.malis.cep.event.convert; + +import at.co.procon.malis.cep.detector.DetectionContext; +import at.co.procon.malis.cep.event.EventBatch; +import at.co.procon.malis.cep.parser.ParsedInput; + +/** + * Converts a {@link ParsedInput} body to the canonical {@link EventBatch}. + */ +public interface InputEventConverter { + + boolean supports(Object body); + + EventBatch convert(ParsedInput input, DetectionContext ctx) throws Exception; +} diff --git a/src/main/java/at/co/procon/malis/cep/event/convert/MeasurementSetConverter.java b/src/main/java/at/co/procon/malis/cep/event/convert/MeasurementSetConverter.java new file mode 100644 index 0000000..bd4f9ed --- /dev/null +++ b/src/main/java/at/co/procon/malis/cep/event/convert/MeasurementSetConverter.java @@ -0,0 +1,54 @@ +package at.co.procon.malis.cep.event.convert; + +import at.co.procon.malis.cep.detector.DetectionContext; +import at.co.procon.malis.cep.event.*; +import at.co.procon.malis.cep.parser.MeasurementSet; +import at.co.procon.malis.cep.parser.MeasurementValue; +import at.co.procon.malis.cep.parser.ParsedInput; + +import java.time.Instant; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +/** + * Converter for the existing {@link MeasurementSet} canonical model. + */ +public final class MeasurementSetConverter implements InputEventConverter { + + @Override + public boolean supports(Object body) { + return body instanceof MeasurementSet; + } + + @Override + public EventBatch convert(ParsedInput input, DetectionContext ctx) { + MeasurementSet ms = (MeasurementSet) input.getBody(); + String sourceKey = ctx.getVars().get("sourceKey"); + String deviceId = ms.getDeviceId() != null ? ms.getDeviceId() : ctx.getVars().get("deviceId"); + Instant ts = ms.getTimestamp() != null ? ms.getTimestamp() : input.getReceivedAt(); + + if (sourceKey == null || sourceKey.isBlank()) { + sourceKey = (deviceId == null || deviceId.isBlank()) ? "unknown-source" : deviceId; + } + + List events = new ArrayList<>(); + for (MeasurementValue mv : ms.getMeasurements()) { + if (mv == null || mv.getPath() == null) continue; + Object v = mv.getValue(); + SignalValueType t; + if (v instanceof Boolean) t = SignalValueType.BOOLEAN; + else if (v instanceof Number) t = SignalValueType.DOUBLE; + else t = SignalValueType.STRING; + + events.add(new SignalUpdateEvent( + mv.getPath(), + new TypedValue(t, v instanceof Number n ? n.doubleValue() : v), + ts, + Map.of("source", "measurementSet") + )); + } + + return new EventBatch(sourceKey, deviceId, ts, events); + } +} diff --git a/src/main/java/at/co/procon/malis/cep/lifecycle/AlarmLifecycleService.java b/src/main/java/at/co/procon/malis/cep/lifecycle/AlarmLifecycleService.java new file mode 100644 index 0000000..12e5e63 --- /dev/null +++ b/src/main/java/at/co/procon/malis/cep/lifecycle/AlarmLifecycleService.java @@ -0,0 +1,656 @@ +package at.co.procon.malis.cep.lifecycle; + +import at.co.procon.malis.cep.config.CepProperties; +import at.co.procon.malis.cep.detector.DetectorEvent; +import org.springframework.stereotype.Component; + +import java.time.Duration; +import java.time.Instant; +import java.util.*; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.LongAdder; + +/** + * Alarm lifecycle and state service. + * + * Key responsibilities: + * - Maintain per-fault state in memory (active/inactive) + * - Apply lifecycle rules (dedup, debounce) + * - Decide when to emit publishable alarms + * + * NEW: supports two modes via LifecyclePolicy.mode + * - TRANSITION_ONLY: emit only on inactive<->active transitions (previous behavior) + * - FORWARD_ALL: emit every event immediately (pass-through) + */ +@Component +public class AlarmLifecycleService { + + private static final Duration DEFAULT_STATE_TTL = Duration.ofDays(7); + private static final long EVICT_EVERY_N_APPLY_CALLS = 200; + private static final Duration CLOCK_SKEW_TOLERANCE = Duration.ofMinutes(2); + + // ---------------- state store ---------------- + + interface FaultStateStore { + FaultState getOrCreate(String faultKey); + Optional get(String faultKey); + Collection values(); + int evictExpired(Instant now, Duration ttl); + int size(); + } + + static final class InMemoryFaultStateStore implements FaultStateStore { + private final ConcurrentHashMap map = new ConcurrentHashMap<>(); + @Override public FaultState getOrCreate(String faultKey) { + return map.computeIfAbsent(faultKey, k -> new FaultState(false)); + } + @Override public Optional get(String faultKey) { + return Optional.ofNullable(map.get(faultKey)); + } + + @Override public Collection values() { + return map.values(); + } + + @Override public int size() { + return map.size(); + } + + @Override public int evictExpired(Instant now, Duration ttl) { + if (ttl == null || ttl.isZero() || ttl.isNegative()) return 0; + Instant cutoff = now.minus(ttl); + + int removed = 0; + for (Map.Entry e : map.entrySet()) { + FaultState st = e.getValue(); + synchronized (st) { + if (st.isActive()) continue; + if (st.getPendingCancelAt() != null) continue; + Instant lastSeen = st.getLastSeenAt(); + if (lastSeen != null && lastSeen.isBefore(cutoff)) { + if (map.remove(e.getKey(), st)) removed++; + } + } + } + return removed; + } + } + + static final class FaultState { + // identity (needed for scheduled emissions like heartbeats) + private String bindingId; + private String sourceKey; + private String faultKey; + + private boolean active; + private Instant lastEventAt; + private String lastEventType; + private Instant lastTransitionAt; + private Instant lastSeenAt; + + // latest snapshots (needed for scheduled emissions like heartbeats) + private Map lastVars; + private Map lastDetails; + private String lastSeverity; + private Instant lastHeartbeatAt; + + private Instant pendingCancelAt; + private PublishableAlarm pendingCancelAlarm; + + // Correlation for one concrete occurrence. + private String activeInstanceId; + private Instant firstSeenAt; + + FaultState(boolean active) { this.active = active; } + + String getBindingId() { return bindingId; } + void setBindingId(String bindingId) { this.bindingId = bindingId; } + + String getSourceKey() { return sourceKey; } + void setSourceKey(String sourceKey) { this.sourceKey = sourceKey; } + + String getFaultKey() { return faultKey; } + void setFaultKey(String faultKey) { this.faultKey = faultKey; } + + boolean isActive() { return active; } + void setActive(boolean active) { this.active = active; } + + Instant getLastEventAt() { return lastEventAt; } + void setLastEventAt(Instant lastEventAt) { this.lastEventAt = lastEventAt; } + + String getLastEventType() { return lastEventType; } + void setLastEventType(String lastEventType) { this.lastEventType = lastEventType; } + + Instant getLastTransitionAt() { return lastTransitionAt; } + void setLastTransitionAt(Instant lastTransitionAt) { this.lastTransitionAt = lastTransitionAt; } + + Instant getLastSeenAt() { return lastSeenAt; } + void setLastSeenAt(Instant lastSeenAt) { this.lastSeenAt = lastSeenAt; } + + Map getLastVars() { return lastVars; } + void setLastVars(Map lastVars) { this.lastVars = lastVars; } + + Map getLastDetails() { return lastDetails; } + void setLastDetails(Map lastDetails) { this.lastDetails = lastDetails; } + + String getLastSeverity() { return lastSeverity; } + void setLastSeverity(String lastSeverity) { this.lastSeverity = lastSeverity; } + + Instant getLastHeartbeatAt() { return lastHeartbeatAt; } + void setLastHeartbeatAt(Instant lastHeartbeatAt) { this.lastHeartbeatAt = lastHeartbeatAt; } + + Instant getPendingCancelAt() { return pendingCancelAt; } + void setPendingCancelAt(Instant pendingCancelAt) { this.pendingCancelAt = pendingCancelAt; } + + PublishableAlarm getPendingCancelAlarm() { return pendingCancelAlarm; } + void setPendingCancelAlarm(PublishableAlarm pendingCancelAlarm) { this.pendingCancelAlarm = pendingCancelAlarm; } + + String getActiveInstanceId() { return activeInstanceId; } + void setActiveInstanceId(String activeInstanceId) { this.activeInstanceId = activeInstanceId; } + + Instant getFirstSeenAt() { return firstSeenAt; } + void setFirstSeenAt(Instant firstSeenAt) { this.firstSeenAt = firstSeenAt; } + } + + /** Scheduled publish request emitted by background features like heartbeats. */ + public static final class ScheduledEmission { + private final String bindingId; + private final Map vars; + private final PublishableAlarm alarm; + + public ScheduledEmission(String bindingId, Map vars, PublishableAlarm alarm) { + this.bindingId = bindingId; + this.vars = vars == null ? Map.of() : vars; + this.alarm = alarm; + } + + public String getBindingId() { return bindingId; } + public Map getVars() { return vars; } + public PublishableAlarm getAlarm() { return alarm; } + } + + // ---------------- service ---------------- + + private final CepProperties props; + private final FaultStateStore store = new InMemoryFaultStateStore(); + + // ---------------- counters / stats ---------------- + + private final AtomicLong applyCalls = new AtomicLong(0); + private final LongAdder processed = new LongAdder(); + private final LongAdder emitted = new LongAdder(); + private final LongAdder droppedOutOfOrder = new LongAdder(); + private final LongAdder droppedDedup = new LongAdder(); + private final LongAdder scheduledCancelHoldDown = new LongAdder(); + private final LongAdder flushedPendingCancels = new LongAdder(); + private final LongAdder evictedStates = new LongAdder(); + + public AlarmLifecycleService(CepProperties props) { + this.props = props; + } + + public List apply(String bindingId, + CepProperties.SourceBinding binding, + Map vars, + List detectorEvents) { + + CepProperties.LifecyclePolicy policy = resolvePolicy(binding); + + long dedupWindowMs = Math.max(0, policy.getDedupWindowMs()); + long debounceMs = Math.max(0, policy.getDebounceMs()); + + String mode = policy.getMode() == null ? "TRANSITION_ONLY" : policy.getMode().trim().toUpperCase(Locale.ROOT); + boolean forwardAll = mode.equals("FORWARD_ALL") || mode.equals("PASSTHROUGH"); + + Instant now = Instant.now(); + + long calls = applyCalls.incrementAndGet(); + if (calls % EVICT_EVERY_N_APPLY_CALLS == 0) { + int removed = store.evictExpired(now, DEFAULT_STATE_TTL); + if (removed > 0) evictedStates.add(removed); + } + + String sourceKey = vars == null ? null : vars.get("sourceKey"); + + List out = new ArrayList<>(); + + for (DetectorEvent ev : detectorEvents) { + if (ev == null || ev.faultKey() == null) continue; + + Instant occurredAt = ev.occurredAt() != null ? ev.occurredAt() : now; + if (occurredAt.isAfter(now.plus(CLOCK_SKEW_TOLERANCE))) { + occurredAt = now; + } + + String eventType = normalizeEventType(ev.eventType()); + if (eventType == null) continue; + + FaultState st = store.getOrCreate(stateKey(bindingId, sourceKey, ev.faultKey())); + + synchronized (st) { + processed.increment(); + st.setLastSeenAt(now); + + // Persist latest context so background tasks (heartbeats, matured debounce cancels) can publish without a new ingress message. + st.setBindingId(bindingId); + st.setSourceKey(sourceKey); + st.setFaultKey(ev.faultKey()); + if (vars != null) { + st.setLastVars(Collections.unmodifiableMap(new LinkedHashMap<>(vars))); + } + st.setLastDetails(safeDetails(ev.details())); + if (ev.severity() != null) st.setLastSeverity(ev.severity()); + + // ---------- flush any matured pending CANCEL for this fault ---------- + if (st.getPendingCancelAt() != null + && st.isActive() + && !now.isBefore(st.getPendingCancelAt())) { + + PublishableAlarm pending = st.getPendingCancelAlarm(); + if (pending != null) { + out.add(pending); + emitted.increment(); + flushedPendingCancels.increment(); + } + st.setActive(false); + st.setLastTransitionAt(st.getPendingCancelAt()); + st.setPendingCancelAt(null); + st.setPendingCancelAlarm(null); + st.setActiveInstanceId(null); + st.setFirstSeenAt(null); + } + + // ---------- out-of-order guard ---------- + if (st.getLastEventAt() != null && occurredAt.isBefore(st.getLastEventAt())) { + droppedOutOfOrder.increment(); + continue; + } + + // ---------- dedup filter (optional) ---------- + if (dedupWindowMs > 0 && st.getLastEventAt() != null && eventType.equals(st.getLastEventType())) { + Duration d = Duration.between(st.getLastEventAt(), occurredAt); + long ms = d.toMillis(); + if (ms >= 0 && ms < dedupWindowMs) { + st.setLastEventAt(occurredAt); + droppedDedup.increment(); + continue; + } + } + + boolean wantsActive = eventType.equals("ALARM"); + boolean isToggle = (wantsActive != st.isActive()); + + // ---------- anti-flap suppression (CANCEL only) ---------- + if (!forwardAll + && debounceMs > 0 + && isToggle + && eventType.equals("CANCEL") + && st.getLastTransitionAt() != null) { + + Duration sinceTransition = Duration.between(st.getLastTransitionAt(), occurredAt); + long ms = sinceTransition.toMillis(); + if (ms >= 0 && ms < debounceMs) { + Instant cancelAt = st.getLastTransitionAt().plusMillis(debounceMs); + st.setPendingCancelAt(cancelAt); + + // Ensure we have a stable instanceId for this still-active occurrence (even if this state predates the instanceId upgrade). + String instanceId = st.getActiveInstanceId(); + Instant firstSeenAt = st.getFirstSeenAt(); + if (instanceId == null || instanceId.isBlank()) { + instanceId = newInstanceId(); + st.setActiveInstanceId(instanceId); + if (firstSeenAt == null) { + firstSeenAt = st.getLastTransitionAt() != null ? st.getLastTransitionAt() : occurredAt; + st.setFirstSeenAt(firstSeenAt); + } + } + + st.setPendingCancelAlarm(new PublishableAlarm( + ev.faultKey(), + instanceId, + "CANCEL", + ev.severity(), + cancelAt, + detailsWithIdentity(ev.details(), instanceId, ev.faultKey(), firstSeenAt) + )); + + scheduledCancelHoldDown.increment(); + + st.setLastEventAt(occurredAt); + st.setLastEventType(eventType); + continue; + } + } + + // ---------- emission rules ---------- + if (forwardAll) { + // Keep a stable instanceId while the definition is active. + String instanceId = st.getActiveInstanceId(); + Instant firstSeenAt = st.getFirstSeenAt(); + + if (eventType.equals("ALARM")) { + if (!st.isActive()) { + instanceId = newInstanceId(); + st.setActiveInstanceId(instanceId); + st.setFirstSeenAt(occurredAt); + firstSeenAt = occurredAt; + } else if (instanceId == null || instanceId.isBlank()) { + instanceId = newInstanceId(); + st.setActiveInstanceId(instanceId); + if (firstSeenAt == null) { + firstSeenAt = st.getLastTransitionAt() != null ? st.getLastTransitionAt() : occurredAt; + st.setFirstSeenAt(firstSeenAt); + } + } + } + + out.add(new PublishableAlarm( + ev.faultKey(), + instanceId, + eventType, + ev.severity(), + occurredAt, + detailsWithIdentity(ev.details(), instanceId, ev.faultKey(), firstSeenAt) + )); + emitted.increment(); + + if (eventType.equals("ALARM")) { + st.setPendingCancelAt(null); + st.setPendingCancelAlarm(null); + + if (!st.isActive()) { + st.setActive(true); + st.setLastTransitionAt(occurredAt); + } + } else { // CANCEL + if (st.isActive()) { + st.setActive(false); + st.setLastTransitionAt(occurredAt); + // End of occurrence. + st.setActiveInstanceId(null); + st.setFirstSeenAt(null); + } + st.setPendingCancelAt(null); + st.setPendingCancelAlarm(null); + } + + } else { + // Default mode: emit only on transitions + if (eventType.equals("ALARM")) { + st.setPendingCancelAt(null); + st.setPendingCancelAlarm(null); + + if (!st.isActive()) { + st.setActive(true); + st.setLastTransitionAt(occurredAt); + + String instanceId = newInstanceId(); + st.setActiveInstanceId(instanceId); + st.setFirstSeenAt(occurredAt); + + out.add(new PublishableAlarm( + ev.faultKey(), + instanceId, + "ALARM", + ev.severity(), + occurredAt, + detailsWithIdentity(ev.details(), instanceId, ev.faultKey(), occurredAt) + )); + emitted.increment(); + } + + } else { // CANCEL + if (st.isActive()) { + st.setActive(false); + st.setLastTransitionAt(occurredAt); + st.setPendingCancelAt(null); + st.setPendingCancelAlarm(null); + + String instanceId = st.getActiveInstanceId(); + Instant firstSeenAt = st.getFirstSeenAt(); + out.add(new PublishableAlarm( + ev.faultKey(), + instanceId, + "CANCEL", + ev.severity(), + occurredAt, + detailsWithIdentity(ev.details(), instanceId, ev.faultKey(), firstSeenAt) + )); + // End of occurrence. + st.setActiveInstanceId(null); + st.setFirstSeenAt(null); + + emitted.increment(); + } + } + } + + st.setLastEventAt(occurredAt); + st.setLastEventType(eventType); + } + } + + return out; + } + + public List flushPendingCancels() { + return flushPendingCancels(Instant.now()); + } + + public List flushPendingCancels(Instant now) { + List out = new ArrayList<>(); + for (FaultState st : store.values()) { + synchronized (st) { + if (st.getPendingCancelAt() != null + && st.isActive() + && !now.isBefore(st.getPendingCancelAt())) { + PublishableAlarm pending = st.getPendingCancelAlarm(); + if (pending != null) { + out.add(pending); + emitted.increment(); + flushedPendingCancels.increment(); + } + st.setActive(false); + st.setLastTransitionAt(st.getPendingCancelAt()); + st.setPendingCancelAt(null); + st.setPendingCancelAlarm(null); + st.setActiveInstanceId(null); + st.setFirstSeenAt(null); + st.setLastSeenAt(now); + } + } + } + return out; + } + + /** + * Collect matured pending CANCEL emissions across ALL bindings (with vars) so a background scheduler can publish + * them even when no new ingress messages arrive. + */ + public List collectMaturedPendingCancels(Instant now) { + Instant n = now == null ? Instant.now() : now; + List out = new ArrayList<>(); + + for (FaultState st : store.values()) { + synchronized (st) { + if (st.getPendingCancelAt() != null + && st.isActive() + && !n.isBefore(st.getPendingCancelAt())) { + + PublishableAlarm pending = st.getPendingCancelAlarm(); + Map vars = st.getLastVars(); + String bId = st.getBindingId(); + if (pending != null && vars != null && bId != null) { + out.add(new ScheduledEmission(bId, vars, pending)); + } + + // mutate state exactly as apply() would + st.setActive(false); + st.setLastTransitionAt(st.getPendingCancelAt()); + st.setPendingCancelAt(null); + st.setPendingCancelAlarm(null); + st.setActiveInstanceId(null); + st.setFirstSeenAt(null); + st.setLastSeenAt(n); + } + } + } + return out; + } + + /** + * Collect heartbeat emissions for the given binding. + * + *

Heartbeats re-emit the CURRENT state: + * - ALARM heartbeat while active (including while a debounced CANCEL is pending) + * - CANCEL heartbeat while inactive, only if includeInactive=true + */ + public List collectDueHeartbeats(String bindingId, + CepProperties.HeartbeatDef hb, + Instant now) { + if (hb == null || !hb.isEnabled()) return List.of(); + if (bindingId == null || bindingId.isBlank()) return List.of(); + + long periodMs = Math.max(0, hb.getPeriodMs()); + if (periodMs <= 0) return List.of(); + boolean includeInactive = hb.isIncludeInactive(); + + Instant n = now == null ? Instant.now() : now; + List out = new ArrayList<>(); + + for (FaultState st : store.values()) { + synchronized (st) { + if (!bindingId.equals(st.getBindingId())) continue; + + Map vars = st.getLastVars(); + if (vars == null || vars.isEmpty()) continue; + + boolean consideredActive = st.isActive(); + if (!consideredActive && !includeInactive) continue; + + Instant lastHb = st.getLastHeartbeatAt(); + if (lastHb != null) { + long ms = Duration.between(lastHb, n).toMillis(); + if (ms >= 0 && ms < periodMs) continue; + } + + PublishableAlarm hbAlarm = buildHeartbeatLocked(st, consideredActive, n); + if (hbAlarm != null) { + st.setLastHeartbeatAt(n); + out.add(new ScheduledEmission(bindingId, vars, hbAlarm)); + } + } + } + return out; + } + + public int evictExpiredStates() { + int removed = store.evictExpired(Instant.now(), DEFAULT_STATE_TTL); + if (removed > 0) evictedStates.add(removed); + return removed; + } + + public Map stats() { + return Map.of( + "applyCalls", applyCalls.get(), + "processed", processed.sum(), + "emitted", emitted.sum(), + "droppedOutOfOrder", droppedOutOfOrder.sum(), + "droppedDedup", droppedDedup.sum(), + "scheduledCancelHoldDown", scheduledCancelHoldDown.sum(), + "flushedPendingCancels", flushedPendingCancels.sum(), + "evictedStates", evictedStates.sum(), + "stateSize", (long) store.size() + ); + } + + private static String newInstanceId() { + // UUIDv4 is sufficient for global uniqueness; switch to UUIDv7/ULID if you need time-ordering. + return UUID.randomUUID().toString(); + } + + private PublishableAlarm buildHeartbeatLocked(FaultState st, boolean active, Instant now) { + if (st == null) return null; + String fk = st.getFaultKey(); + if (fk == null || fk.isBlank()) return null; + + String action = active ? "ALARM" : "CANCEL"; + String severity = st.getLastSeverity() == null ? "INFO" : st.getLastSeverity(); + + String instanceId = null; + Instant firstSeenAt = null; + + if (active) { + instanceId = st.getActiveInstanceId(); + firstSeenAt = st.getFirstSeenAt(); + + // Backwards-compat: if an old state exists without instanceId but the alarm is active, mint one. + if (instanceId == null || instanceId.isBlank()) { + instanceId = newInstanceId(); + st.setActiveInstanceId(instanceId); + if (firstSeenAt == null) { + firstSeenAt = st.getLastTransitionAt() != null ? st.getLastTransitionAt() : now; + st.setFirstSeenAt(firstSeenAt); + } + } + } + + Map base = st.getLastDetails(); + LinkedHashMap d = new LinkedHashMap<>(detailsWithIdentity(base, instanceId, fk, firstSeenAt)); + d.put("heartbeat", true); + d.put("heartbeatAt", now); + d.put("heartbeatState", action); + if (st.getPendingCancelAt() != null && active) { + d.put("pendingCancelAt", st.getPendingCancelAt()); + d.put("pendingCancel", true); + } + + return new PublishableAlarm( + fk, + instanceId, + action, + severity, + now, + d + ); + } + + private static Map detailsWithIdentity(Map base, + String instanceId, + String definitionKey, + Instant firstSeenAt) { + LinkedHashMap d = new LinkedHashMap<>(); + if (base != null && !base.isEmpty()) d.putAll(base); + if (instanceId != null && !instanceId.isBlank()) d.put("instanceId", instanceId); + if (definitionKey != null && !definitionKey.isBlank()) d.put("definitionKey", definitionKey); + if (firstSeenAt != null) d.put("firstSeenAt", firstSeenAt); + return Collections.unmodifiableMap(d); + } + + private static String stateKey(String bindingId, String sourceKey, String faultKey) { + String b = (bindingId == null || bindingId.isBlank()) ? "default" : bindingId; + String s = (sourceKey == null || sourceKey.isBlank()) ? "default" : sourceKey; + return b + "|" + s + "|" + faultKey; + } + + private CepProperties.LifecyclePolicy resolvePolicy(CepProperties.SourceBinding binding) { + String ref = binding.getLifecyclePolicyRef(); + if (ref != null && props.getLifecyclePolicies().containsKey(ref)) { + return props.getLifecyclePolicies().get(ref); + } + return props.getLifecyclePolicies().getOrDefault("default", new CepProperties.LifecyclePolicy()); + } + + private static String normalizeEventType(String t) { + if (t == null) return null; + String u = t.trim().toUpperCase(Locale.ROOT); + if (u.equals("ALARM") || u.equals("RAISE")) return "ALARM"; + if (u.equals("CANCEL") || u.equals("CLEAR") || u.equals("CLEARED")) return "CANCEL"; + return null; + } + + private static Map safeDetails(Map m) { + return m == null ? Map.of() : Collections.unmodifiableMap(new LinkedHashMap<>(m)); + } +} \ No newline at end of file diff --git a/src/main/java/at/co/procon/malis/cep/lifecycle/CompositeAlarmLifecycleService.java b/src/main/java/at/co/procon/malis/cep/lifecycle/CompositeAlarmLifecycleService.java new file mode 100644 index 0000000..d3c1c1b --- /dev/null +++ b/src/main/java/at/co/procon/malis/cep/lifecycle/CompositeAlarmLifecycleService.java @@ -0,0 +1,954 @@ +package at.co.procon.malis.cep.lifecycle; + +import at.co.procon.malis.cep.config.CepProperties; +import at.co.procon.malis.cep.detector.DetectorEvent; +import at.co.procon.malis.cep.output.PublisherService; +import at.co.procon.malis.cep.util.TemplateUtil; +import jakarta.annotation.PreDestroy; +import org.springframework.stereotype.Component; + +import java.time.Duration; +import java.time.Instant; +import java.time.LocalDate; +import java.time.ZoneOffset; +import java.util.*; +import java.util.concurrent.*; +import java.util.concurrent.atomic.AtomicLong; + +/** + * Grouped/composed alarm lifecycle. + * + * Responsibilities: + * - Track member (source) state per (groupKey, sourceKey) + * - Derive group state and emit: + * - OPEN -> action=ALARM + * - UPDATE -> action=ALARM (details.groupEvent=UPDATE) + * - CLOSE -> action=CANCEL + * - TERMINATE -> action=CANCEL (optional) + evict state + * - Optional: throttle UPDATE emissions per group + * - Optional: delayed CLOSE (closeGrace) + * - Optional: termination by idle/no-notification/max-lifetime + * + * Notes: + * - Member-level dedup/debounce is applied BEFORE group decisions. + * - Termination/flush/throttle is implemented with a small internal scheduler so it works even + * when no new ingress messages arrive. + */ +@Component +public class CompositeAlarmLifecycleService { + + private static final Duration DEFAULT_STATE_TTL = Duration.ofDays(7); + private static final long EVICT_EVERY_N_APPLY_CALLS = 200; + private static final Duration CLOCK_SKEW_TOLERANCE = Duration.ofMinutes(2); + + private final CepProperties props; + private final PublisherService publisher; + + private final ConcurrentHashMap groups = new ConcurrentHashMap<>(); + private final AtomicLong applyCalls = new AtomicLong(0); + + // scheduler for termination/pending-close/throttled-updates + private final ScheduledExecutorService scheduler; + private volatile long scheduledIntervalMs = 30_000; + private volatile ScheduledFuture scheduledTask; + + /** Scheduled publish request emitted by background features like heartbeats. */ + public static final class ScheduledEmission { + private final CepProperties.SourceBinding binding; + private final Map vars; + private final PublishableAlarm alarm; + + public ScheduledEmission(CepProperties.SourceBinding binding, Map vars, PublishableAlarm alarm) { + this.binding = binding; + this.vars = vars == null ? Map.of() : vars; + this.alarm = alarm; + } + + public CepProperties.SourceBinding getBinding() { return binding; } + public Map getVars() { return vars; } + public PublishableAlarm getAlarm() { return alarm; } + } + + public CompositeAlarmLifecycleService(CepProperties props, PublisherService publisher) { + this.props = props; + this.publisher = publisher; + this.scheduler = Executors.newSingleThreadScheduledExecutor(r -> { + Thread t = new Thread(r, "composite-alarm-lifecycle-tick"); + t.setDaemon(true); + return t; + }); + // start with default; may be reduced dynamically when COMPOSITE policies are used + rescheduleIfNeeded(30_000); + } + + @PreDestroy + public void shutdown() { + try { + if (scheduledTask != null) scheduledTask.cancel(false); + } catch (Exception ignored) { + } + scheduler.shutdownNow(); + } + + public List apply(String bindingId, + CepProperties.SourceBinding binding, + Map vars, + List detectorEvents) { + + CepProperties.LifecyclePolicy policy = resolvePolicy(binding); + String type = policy.getType() == null ? "SIMPLE" : policy.getType().trim().toUpperCase(Locale.ROOT); + if (!type.equals("COMPOSITE")) { + // Defensive: dispatcher should have prevented this. + return List.of(); + } + + // ensure scheduler interval satisfies this policy + rescheduleIfNeeded(Math.max(1000, policy.getCompositeTickIntervalMs())); + + CompositePolicySnapshot snap = CompositePolicySnapshot.from(policy, binding); + + Instant now = Instant.now(); + long calls = applyCalls.incrementAndGet(); + if (calls % EVICT_EVERY_N_APPLY_CALLS == 0) { + evictExpired(now); + } + + if (detectorEvents == null || detectorEvents.isEmpty()) return List.of(); + Map baseVars = vars == null ? Map.of() : vars; + + // Collect outputs per apply call; note that background tick may also publish. + List out = new ArrayList<>(); + + for (DetectorEvent ev : detectorEvents) { + if (ev == null || ev.faultKey() == null) continue; + + String eventType = normalizeEventType(ev.eventType()); + if (eventType == null) continue; + + Instant occurredAt = ev.occurredAt() != null ? ev.occurredAt() : now; + if (occurredAt.isAfter(now.plus(CLOCK_SKEW_TOLERANCE))) { + occurredAt = now; + } + + // Build a local templating context for this event. + Map ctxVars = new LinkedHashMap<>(baseVars); + ctxVars.put("faultKey", ev.faultKey()); + ctxVars.put("eventType", eventType); + // Helpful default var for daily grouping. + ctxVars.putIfAbsent("faultDay", LocalDate.ofInstant(occurredAt, ZoneOffset.UTC).toString()); + + String groupKey = TemplateUtil.expand(snap.groupKeyTemplate, ctxVars); + if (groupKey == null || groupKey.isBlank()) groupKey = ev.faultKey(); + + String sourceKey = TemplateUtil.expand(snap.sourceKeyTemplate, ctxVars); + if (sourceKey == null || sourceKey.isBlank()) { + sourceKey = ctxVars.getOrDefault("path", ctxVars.getOrDefault("topic", ctxVars.getOrDefault("sourceKey", "default"))); + } + + String groupStateKey = stateKey(bindingId, groupKey); + String finalGroupKey = groupKey; + GroupState gs = groups.computeIfAbsent(groupStateKey, + k -> GroupState.create(bindingId, finalGroupKey, snap)); + + List producedForThisEvent; + synchronized (gs) { + // Keep last binding/vars so background tick can publish close/terminate/update. + if (binding != null) gs.binding = binding; + if (baseVars != null) gs.lastVars = new LinkedHashMap<>(baseVars); + gs.lastSeenAt = now; + + // Flush matured pending close for this group before applying new events. + producedForThisEvent = flushPendingGroupCloseLocked(gs, now); + + // Apply member-level lifecycle for this event. + MemberOutcome mo = applyMemberEventLocked(gs, sourceKey, eventType, ev.severity(), occurredAt, ev.details(), snap, now); + boolean memberChanged = mo.stateChanged; + boolean memberNotifiable = mo.notify; + if (memberNotifiable) { + gs.lastActivityAt = now; + gs.lastChangedSource = sourceKey; + } + + // Derive group active from member states (note: if closeGrace is active we keep groupActive=true until emitted). + boolean derivedActive = gs.anyMemberActiveLocked(); + + // If a pending close exists but a member became active again, cancel the pending close. + if (derivedActive && gs.pendingCloseAt != null) { + gs.pendingCloseAt = null; + gs.pendingCloseAlarm = null; + } + + // Group open + if (!gs.groupActive && derivedActive) { + gs.groupActive = true; + gs.instanceId = newInstanceId(); + gs.openedAt = occurredAt; + gs.lastSeverity = chooseSeverity(gs.lastSeverity, ev.severity()); + PublishableAlarm open = buildGroupAlarm("ALARM", occurredAt, "OPEN", gs, sourceKey, null); + gs.lastNotificationAt = now; + gs.sequence++; + producedForThisEvent = append(producedForThisEvent, open); + // Open is not throttled. + gs.nextUpdateAllowedAt = snap.updateThrottleMs > 0 ? now.plusMillis(snap.updateThrottleMs) : null; + gs.dirtyUpdate = false; + } + + // Group close (all members cleared) + if (gs.groupActive && !derivedActive) { + // closeGrace: keep the group active and schedule the close. + if (!snap.closeGrace.isZero() && !snap.closeGrace.isNegative()) { + Instant closeAt = occurredAt.plus(snap.closeGrace); + if (gs.pendingCloseAt == null || closeAt.isAfter(gs.pendingCloseAt)) { + gs.pendingCloseAt = closeAt; + gs.pendingCloseAlarm = buildGroupAlarm("CANCEL", closeAt, "CLOSE", gs, sourceKey, null); + } + } else { + gs.groupActive = false; + PublishableAlarm close = buildGroupAlarm("CANCEL", occurredAt, "CLOSE", gs, sourceKey, null); + gs.lastNotificationAt = now; + gs.sequence++; + producedForThisEvent = append(producedForThisEvent, close); + // End of occurrence. + gs.instanceId = null; + gs.openedAt = null; + gs.dirtyUpdate = false; + } + } + + // Group update (member changed but group stays active) + if (gs.groupActive && derivedActive && memberNotifiable && snap.emitUpdates) { + // If we already emitted OPEN in this same synchronized block, skip UPDATE. + boolean alreadyOpenedThisCall = producedForThisEvent.stream().anyMatch(a -> "ALARM".equalsIgnoreCase(a.action()) && "OPEN".equals(String.valueOf(a.details().get("groupEvent")))); + if (!alreadyOpenedThisCall) { + if (snap.updateThrottleMs <= 0) { + gs.lastSeverity = chooseSeverity(gs.lastSeverity, ev.severity()); + PublishableAlarm upd = buildGroupAlarm("ALARM", occurredAt, "UPDATE", gs, sourceKey, null); + gs.lastNotificationAt = now; + gs.sequence++; + producedForThisEvent = append(producedForThisEvent, upd); + } else { + if (gs.nextUpdateAllowedAt == null || !now.isBefore(gs.nextUpdateAllowedAt)) { + gs.lastSeverity = chooseSeverity(gs.lastSeverity, ev.severity()); + PublishableAlarm upd = buildGroupAlarm("ALARM", occurredAt, "UPDATE", gs, sourceKey, null); + gs.lastNotificationAt = now; + gs.sequence++; + producedForThisEvent = append(producedForThisEvent, upd); + gs.nextUpdateAllowedAt = now.plusMillis(snap.updateThrottleMs); + gs.dirtyUpdate = false; + } else { + // Mark dirty; background tick will emit the coalesced UPDATE later. + gs.dirtyUpdate = true; + } + } + } + } + } + + if (producedForThisEvent != null && !producedForThisEvent.isEmpty()) { + out.addAll(producedForThisEvent); + } + } + + return out; + } + + /** + * Collect heartbeat emissions for the given binding. + * + *

Heartbeats re-emit the CURRENT group state: + * - ALARM heartbeat while groupActive (including while a delayed CLOSE is pending) + * - CANCEL heartbeat while inactive, only if includeInactive=true + */ + public List collectDueHeartbeats(String bindingId, + CepProperties.HeartbeatDef hb, + Instant now) { + if (hb == null || !hb.isEnabled()) return List.of(); + if (bindingId == null || bindingId.isBlank()) return List.of(); + + long periodMs = Math.max(0, hb.getPeriodMs()); + if (periodMs <= 0) return List.of(); + boolean includeInactive = hb.isIncludeInactive(); + + Instant n = now == null ? Instant.now() : now; + + List out = new ArrayList<>(); + + for (GroupState gs : groups.values()) { + if (gs == null) continue; + if (!bindingId.equals(gs.bindingId)) continue; + + CepProperties.SourceBinding binding = gs.binding; + Map vars = gs.lastVars; + if (binding == null || vars == null || vars.isEmpty()) continue; + + synchronized (gs) { + boolean active = gs.groupActive; + if (!active && !includeInactive) continue; + + Instant lastHb = gs.lastHeartbeatAt; + if (lastHb != null) { + long ms = Duration.between(lastHb, n).toMillis(); + if (ms >= 0 && ms < periodMs) continue; + } + + PublishableAlarm hbAlarm = buildGroupHeartbeatLocked(gs, active, n); + if (hbAlarm != null) { + gs.lastHeartbeatAt = n; + out.add(new ScheduledEmission(binding, vars, hbAlarm)); + } + } + } + + return out; + } + + // -------------------------- + // Background tick + // -------------------------- + + private void tick() { + Instant now = Instant.now(); + + List removeKeys = new ArrayList<>(); + + for (Map.Entry e : groups.entrySet()) { + String key = e.getKey(); + GroupState gs = e.getValue(); + if (gs == null) { + removeKeys.add(key); + continue; + } + + List toPublish = new ArrayList<>(); + boolean removeGroup = false; + + synchronized (gs) { + // 0) Flush matured pending member cancels (debounce) even if no further events arrive. + for (SourceState st : gs.sources.values()) { + if (st == null) continue; + if (st.pendingCancelAt != null && st.active && !now.isBefore(st.pendingCancelAt)) { + st.active = false; + st.lastTransitionAt = st.pendingCancelAt; + st.lastNoAt = st.lastTransitionAt; + st.pendingCancelAt = null; + st.pendingCancelDetails = null; + } + } + + // 1) Drop stale members + if (!gs.policy.sourceTtl.isZero() && !gs.policy.sourceTtl.isNegative()) { + Instant cutoff = now.minus(gs.policy.sourceTtl); + gs.sources.entrySet().removeIf(se -> { + SourceState st = se.getValue(); + return st != null && st.lastSeenAt != null && st.lastSeenAt.isBefore(cutoff); + }); + } + + // 2) Flush matured pending group close + toPublish.addAll(flushPendingGroupCloseLocked(gs, now)); + + // 2b) If group is still active but all members are inactive (e.g., due to TTL or matured debounced cancels), close. + boolean derivedActive = gs.anyMemberActiveLocked(); + if (gs.groupActive && !derivedActive && gs.pendingCloseAt == null) { + if (!gs.policy.closeGrace.isZero() && !gs.policy.closeGrace.isNegative()) { + Instant closeAt = now.plus(gs.policy.closeGrace); + gs.pendingCloseAt = closeAt; + gs.pendingCloseAlarm = buildGroupAlarm("CANCEL", closeAt, "CLOSE", gs, gs.lastChangedSource, "AUTO_CLOSE"); + } else { + gs.groupActive = false; + PublishableAlarm close = buildGroupAlarm("CANCEL", now, "CLOSE", gs, gs.lastChangedSource, "AUTO_CLOSE"); + gs.lastNotificationAt = now; + gs.sequence++; + toPublish.add(close); + // End of occurrence. + gs.instanceId = null; + gs.openedAt = null; + } + } + + // 3) Flush throttled UPDATE if allowed + if (gs.dirtyUpdate && gs.groupActive && gs.policy.emitUpdates) { + if (gs.nextUpdateAllowedAt == null || !now.isBefore(gs.nextUpdateAllowedAt)) { + PublishableAlarm upd = buildGroupAlarm("ALARM", now, "UPDATE", gs, gs.lastChangedSource, "THROTTLE_FLUSH"); + gs.lastNotificationAt = now; + gs.sequence++; + toPublish.add(upd); + gs.dirtyUpdate = false; + if (gs.policy.updateThrottleMs > 0) { + gs.nextUpdateAllowedAt = now.plusMillis(gs.policy.updateThrottleMs); + } + } + } + + // 4) Termination checks + String terminationReason = terminationReasonLocked(gs, now); + if (terminationReason != null) { + if (gs.groupActive && gs.policy.terminateEmitsCancel) { + PublishableAlarm term = buildGroupAlarm("CANCEL", now, "TERMINATE", gs, gs.lastChangedSource, terminationReason); + gs.groupActive = false; + gs.sequence++; + toPublish.add(term); + // End of occurrence. + gs.instanceId = null; + gs.openedAt = null; + } + removeGroup = true; + } + + // 5) Evict old inactive groups (memory bound) + if (!removeGroup && !gs.groupActive && gs.pendingCloseAt == null) { + Instant last = gs.lastSeenAt != null ? gs.lastSeenAt : gs.createdAt; + if (last != null && last.isBefore(now.minus(DEFAULT_STATE_TTL))) { + removeGroup = true; + } + } + } + + // Publish outside synchronized block + if (!toPublish.isEmpty() && gs.binding != null && gs.lastVars != null) { + try { + publisher.publish(gs.binding, gs.lastVars, toPublish); + } catch (Exception ignored) { + // swallow: tick must not kill the scheduler + } + } + + if (removeGroup) { + removeKeys.add(key); + } + } + + for (String k : removeKeys) { + groups.remove(k); + } + } + + // -------------------------- + // Helpers + // -------------------------- + + private void rescheduleIfNeeded(long desiredIntervalMs) { + if (desiredIntervalMs <= 0) return; + // Start scheduler if not started; otherwise only ever decrease the interval to satisfy stricter policies. + boolean needStart = (scheduledTask == null); + boolean needDecrease = desiredIntervalMs < scheduledIntervalMs; + if (!needStart && !needDecrease) return; + + scheduledIntervalMs = desiredIntervalMs; + try { + if (scheduledTask != null) scheduledTask.cancel(false); + } catch (Exception ignored) { + } + scheduledTask = scheduler.scheduleAtFixedRate(this::safeTick, scheduledIntervalMs, scheduledIntervalMs, TimeUnit.MILLISECONDS); + } + + private void safeTick() { + try { + tick(); + } catch (Exception ignored) { + // Never crash the scheduler. + } + } + + private static String stateKey(String bindingId, String groupKey) { + String b = (bindingId == null || bindingId.isBlank()) ? "default" : bindingId; + String g = (groupKey == null || groupKey.isBlank()) ? "default" : groupKey; + return b + "|" + g; + } + + private CepProperties.LifecyclePolicy resolvePolicy(CepProperties.SourceBinding binding) { + String ref = binding == null ? null : binding.getLifecyclePolicyRef(); + if (ref != null && props.getLifecyclePolicies().containsKey(ref)) { + return props.getLifecyclePolicies().get(ref); + } + return props.getLifecyclePolicies().getOrDefault("default", new CepProperties.LifecyclePolicy()); + } + + private void evictExpired(Instant now) { + if (now == null) return; + Instant cutoff = now.minus(DEFAULT_STATE_TTL); + + List removeKeys = new ArrayList<>(); + for (Map.Entry e : groups.entrySet()) { + String key = e.getKey(); + GroupState gs = e.getValue(); + if (gs == null) { + removeKeys.add(key); + continue; + } + synchronized (gs) { + if (gs.groupActive) continue; + if (gs.pendingCloseAt != null) continue; + Instant last = gs.lastSeenAt != null ? gs.lastSeenAt : gs.createdAt; + if (last != null && last.isBefore(cutoff)) removeKeys.add(key); + } + } + for (String k : removeKeys) { + groups.remove(k); + } + } + + private static String normalizeEventType(String t) { + if (t == null) return null; + String u = t.trim().toUpperCase(Locale.ROOT); + if (u.equals("ALARM") || u.equals("RAISE")) return "ALARM"; + if (u.equals("CANCEL") || u.equals("CLEAR") || u.equals("CLEARED")) return "CANCEL"; + return null; + } + + + private static String newInstanceId() { + // UUIDv4 is sufficient for global uniqueness; switch to UUIDv7/ULID if you need time-ordering. + return UUID.randomUUID().toString(); + } + + private static String chooseSeverity(String current, String incoming) { + // Keep it simple and stable: prefer incoming if present; otherwise keep current. + if (incoming == null || incoming.isBlank()) return current; + return incoming; + } + + private static Map safeDetails(Map m) { + return m == null ? Map.of() : Collections.unmodifiableMap(new LinkedHashMap<>(m)); + } + + private static List append(List list, PublishableAlarm a) { + if (a == null) return list == null ? List.of() : list; + if (list == null) { + ArrayList out = new ArrayList<>(); + out.add(a); + return out; + } + try { + list.add(a); + return list; + } catch (UnsupportedOperationException ex) { + ArrayList out = new ArrayList<>(list); + out.add(a); + return out; + } + } + + private List flushPendingGroupCloseLocked(GroupState gs, Instant now) { + if (gs.pendingCloseAt == null) return List.of(); + if (now.isBefore(gs.pendingCloseAt)) return List.of(); + + PublishableAlarm a = gs.pendingCloseAlarm; + gs.pendingCloseAt = null; + gs.pendingCloseAlarm = null; + if (gs.groupActive) { + gs.groupActive = false; + gs.lastNotificationAt = now; + gs.sequence++; + // End of occurrence. + gs.instanceId = null; + gs.openedAt = null; + } + return a == null ? List.of() : List.of(a); + } + + private MemberOutcome applyMemberEventLocked(GroupState gs, + String sourceKey, + String eventType, + String severity, + Instant occurredAt, + Map details, + CompositePolicySnapshot snap, + Instant now) { + + SourceState st = gs.sources.computeIfAbsent(sourceKey, k -> new SourceState(false)); + + // Flush matured pending member cancel for this source. + if (st.pendingCancelAt != null && st.active && !now.isBefore(st.pendingCancelAt)) { + st.active = false; + st.lastTransitionAt = st.pendingCancelAt; + st.pendingCancelAt = null; + st.pendingCancelDetails = null; + st.lastNoAt = st.lastTransitionAt; + } + + // Out-of-order guard + if (st.lastEventAt != null && occurredAt.isBefore(st.lastEventAt)) { + return MemberOutcome.noop(); + } + + // Dedup + if (snap.dedupWindowMs > 0 && st.lastEventAt != null && eventType.equals(st.lastEventType)) { + long ms = Duration.between(st.lastEventAt, occurredAt).toMillis(); + if (ms >= 0 && ms < snap.dedupWindowMs) { + st.lastEventAt = occurredAt; + st.lastSeenAt = now; + return MemberOutcome.noop(); + } + } + + boolean wantsActive = eventType.equals("ALARM"); + boolean isToggle = (wantsActive != st.active); + + // Debounce (anti-flap) CANCEL only + if (snap.debounceMs > 0 + && isToggle + && eventType.equals("CANCEL") + && st.lastTransitionAt != null) { + + long ms = Duration.between(st.lastTransitionAt, occurredAt).toMillis(); + if (ms >= 0 && ms < snap.debounceMs) { + st.pendingCancelAt = st.lastTransitionAt.plusMillis(snap.debounceMs); + st.pendingCancelDetails = safeDetails(details); + st.lastEventAt = occurredAt; + st.lastEventType = eventType; + st.lastSeenAt = now; + return MemberOutcome.noop(); + } + } + + boolean stateChanged = false; + boolean notify = snap.forwardAll; + if (eventType.equals("ALARM")) { + st.lastDetails = safeDetails(details); + st.lastSeverity = chooseSeverity(st.lastSeverity, severity); + // For the "topic last YES timestamp" use-case, update on every ALARM observation. + st.lastYesAt = occurredAt; + if (!st.active) { + st.active = true; + st.lastTransitionAt = occurredAt; + st.pendingCancelAt = null; + st.pendingCancelDetails = null; + stateChanged = true; + notify = true; + } + } else { // CANCEL + st.lastDetails = safeDetails(details); + // Update on every CANCEL observation. + st.lastNoAt = occurredAt; + if (st.active) { + st.active = false; + st.lastTransitionAt = occurredAt; + st.pendingCancelAt = null; + st.pendingCancelDetails = null; + stateChanged = true; + notify = true; + } + } + + st.lastEventAt = occurredAt; + st.lastEventType = eventType; + st.lastSeenAt = now; + return new MemberOutcome(stateChanged, notify); + } + + private PublishableAlarm buildGroupAlarm(String action, + Instant occurredAt, + String groupEvent, + GroupState gs, + String changedSource, + String reason) { + // Backwards-compat: if an old state exists without instanceId but the group is active, mint one. + if (gs.groupActive && (gs.instanceId == null || gs.instanceId.isBlank())) { + gs.instanceId = newInstanceId(); + if (gs.openedAt == null) { + gs.openedAt = occurredAt; + } + } + + + Map d = new LinkedHashMap<>(); + d.put("groupKey", gs.groupKey); + if (gs.instanceId != null) d.put("instanceId", gs.instanceId); + d.put("definitionKey", gs.groupKey); + if (gs.openedAt != null) d.put("firstSeenAt", gs.openedAt); + d.put("groupEvent", groupEvent); + d.put("sequence", gs.sequence); + d.put("changedSource", changedSource); + if (reason != null) d.put("reason", reason); + + // Snapshot of current members + List> members = new ArrayList<>(); + for (var e : gs.sources.entrySet()) { + String sk = e.getKey(); + SourceState st = e.getValue(); + if (st == null) continue; + Map m = new LinkedHashMap<>(); + m.put("sourceKey", sk); + m.put("active", st.active); + m.put("lastYesAt", st.lastYesAt); + m.put("lastNoAt", st.lastNoAt); + m.put("lastSeenAt", st.lastSeenAt); + if (st.lastSeverity != null) m.put("severity", st.lastSeverity); + if (st.lastDetails != null && !st.lastDetails.isEmpty()) m.put("details", st.lastDetails); + members.add(m); + } + d.put("members", members); + + return new PublishableAlarm( + gs.groupKey, + gs.instanceId, + action, + gs.lastSeverity == null ? "INFO" : gs.lastSeverity, + occurredAt, + d + ); + } + + /** + * Minimal group heartbeat payload (does NOT include full members snapshot to avoid high periodic traffic). + */ + private PublishableAlarm buildGroupHeartbeatLocked(GroupState gs, boolean active, Instant now) { + if (gs == null) return null; + + String action = active ? "ALARM" : "CANCEL"; + + // Backwards-compat: if an old state exists without instanceId but the group is active, mint one. + if (active && (gs.instanceId == null || gs.instanceId.isBlank())) { + gs.instanceId = newInstanceId(); + if (gs.openedAt == null) { + gs.openedAt = now; + } + } + + // heartbeats are events too -> increment sequence for monotonic ordering + gs.sequence++; + + Map d = new LinkedHashMap<>(); + d.put("groupKey", gs.groupKey); + d.put("definitionKey", gs.groupKey); + if (gs.instanceId != null) d.put("instanceId", gs.instanceId); + if (gs.openedAt != null) d.put("firstSeenAt", gs.openedAt); + d.put("groupEvent", "HEARTBEAT"); + d.put("sequence", gs.sequence); + d.put("changedSource", gs.lastChangedSource); + d.put("active", active); + d.put("heartbeat", true); + d.put("heartbeatAt", now); + d.put("heartbeatState", action); + if (active && gs.pendingCloseAt != null) { + d.put("pendingCloseAt", gs.pendingCloseAt); + d.put("pendingClose", true); + } + + return new PublishableAlarm( + gs.groupKey, + gs.instanceId, + action, + gs.lastSeverity == null ? "INFO" : gs.lastSeverity, + now, + d + ); + } + + private String terminationReasonLocked(GroupState gs, Instant now) { + if (now == null) return null; + + if (!gs.policy.maxLifetime.isZero() && !gs.policy.maxLifetime.isNegative()) { + if (gs.createdAt != null && gs.createdAt.isBefore(now.minus(gs.policy.maxLifetime))) { + return "MAX_LIFETIME"; + } + } + + if (!gs.policy.idleTimeout.isZero() && !gs.policy.idleTimeout.isNegative()) { + Instant last = gs.lastActivityAt != null ? gs.lastActivityAt : gs.createdAt; + if (last != null && last.isBefore(now.minus(gs.policy.idleTimeout))) { + return "IDLE_TIMEOUT"; + } + } + + if (!gs.policy.noNotificationTimeout.isZero() && !gs.policy.noNotificationTimeout.isNegative()) { + Instant last = gs.lastNotificationAt != null ? gs.lastNotificationAt : gs.createdAt; + if (last != null && last.isBefore(now.minus(gs.policy.noNotificationTimeout))) { + return "NO_NOTIFICATION_TIMEOUT"; + } + } + + return null; + } + + // -------------------------- + // State types + // -------------------------- + + static final class CompositePolicySnapshot { + final String groupKeyTemplate; + final String sourceKeyTemplate; + final boolean emitUpdates; + final long updateThrottleMs; + final long dedupWindowMs; + final long debounceMs; + /** If true (policy.mode == FORWARD_ALL), every non-deduped member event becomes a notifiable UPDATE while group is active. */ + final boolean forwardAll; + final Duration idleTimeout; + final Duration noNotificationTimeout; + final Duration maxLifetime; + final Duration sourceTtl; + final Duration closeGrace; + final boolean terminateEmitsCancel; + + private CompositePolicySnapshot(String groupKeyTemplate, + String sourceKeyTemplate, + boolean emitUpdates, + long updateThrottleMs, + long dedupWindowMs, + long debounceMs, + boolean forwardAll, + Duration idleTimeout, + Duration noNotificationTimeout, + Duration maxLifetime, + Duration sourceTtl, + Duration closeGrace, + boolean terminateEmitsCancel) { + this.groupKeyTemplate = groupKeyTemplate; + this.sourceKeyTemplate = sourceKeyTemplate; + this.emitUpdates = emitUpdates; + this.updateThrottleMs = updateThrottleMs; + this.dedupWindowMs = dedupWindowMs; + this.debounceMs = debounceMs; + this.forwardAll = forwardAll; + this.idleTimeout = idleTimeout; + this.noNotificationTimeout = noNotificationTimeout; + this.maxLifetime = maxLifetime; + this.sourceTtl = sourceTtl; + this.closeGrace = closeGrace; + this.terminateEmitsCancel = terminateEmitsCancel; + } + + static CompositePolicySnapshot from(CepProperties.LifecyclePolicy p, CepProperties.SourceBinding binding) { + boolean forwardAll = p.getMode() != null && p.getMode().trim().equalsIgnoreCase("FORWARD_ALL"); + Duration idle = parseDurationOrZero(p.getIdleTimeout()); + Duration nn = parseDurationOrZero(p.getNoNotificationTimeout()); + Duration max = parseDurationOrZero(p.getMaxLifetime()); + Duration srcTtl = parseDurationOrZero(p.getSourceTtl()); + Duration close = parseDurationOrZero(p.getCloseGrace()); + + // Templates are best defined per binding (they depend on vars available for that binding). + // For backward compatibility we fall back to policy templates. + String groupTpl = null; + String sourceTpl = null; + if (binding != null && binding.getComposite() != null) { + groupTpl = binding.getComposite().getGroupKeyTemplate(); + sourceTpl = binding.getComposite().getSourceKeyTemplate(); + } + if (groupTpl == null || groupTpl.isBlank()) groupTpl = p.getGroupKeyTemplate(); + if (sourceTpl == null || sourceTpl.isBlank()) sourceTpl = p.getSourceKeyTemplate(); + if (sourceTpl == null || sourceTpl.isBlank()) sourceTpl = "${path}"; + + return new CompositePolicySnapshot( + groupTpl, + sourceTpl, + p.isEmitUpdates(), + Math.max(0, p.getUpdateThrottleMs()), + Math.max(0, p.getDedupWindowMs()), + Math.max(0, p.getDebounceMs()), + forwardAll, + idle, + nn, + max, + srcTtl, + close, + p.isTerminateEmitsCancel() + ); + } + + private static Duration parseDurationOrZero(String iso) { + if (iso == null || iso.isBlank()) return Duration.ZERO; + try { + return Duration.parse(iso.trim()); + } catch (Exception ex) { + return Duration.ZERO; + } + } + } + + static final class GroupState { + final String bindingId; + final String groupKey; + final CompositePolicySnapshot policy; + + /** Unique occurrence id for the currently active group incident. */ + String instanceId; + + /** First time this group incident was opened (firstSeenAt). */ + Instant openedAt; + + boolean groupActive; + Instant createdAt; + Instant lastNotificationAt; + Instant lastActivityAt; + Instant lastSeenAt; + + Instant lastHeartbeatAt; + + long sequence; + String lastSeverity; + + String lastChangedSource; + boolean dirtyUpdate; + Instant nextUpdateAllowedAt; + + Instant pendingCloseAt; + PublishableAlarm pendingCloseAlarm; + + final LinkedHashMap sources = new LinkedHashMap<>(); + + // for background tick publishing + volatile CepProperties.SourceBinding binding; + volatile Map lastVars; + + private GroupState(String bindingId, String groupKey, CompositePolicySnapshot policy) { + this.bindingId = bindingId; + this.groupKey = groupKey; + this.policy = policy; + this.instanceId = null; + this.openedAt = null; + this.groupActive = false; + this.createdAt = Instant.now(); + this.lastActivityAt = this.createdAt; + this.lastSeenAt = this.createdAt; + this.sequence = 0; + this.lastHeartbeatAt = null; + } + + static GroupState create(String bindingId, String groupKey, CompositePolicySnapshot policy) { + return new GroupState(bindingId, groupKey, policy); + } + + boolean anyMemberActiveLocked() { + for (SourceState s : sources.values()) { + if (s != null && s.active) return true; + } + return false; + } + } + + static final class SourceState { + boolean active; + Instant lastEventAt; + String lastEventType; + Instant lastTransitionAt; + Instant lastSeenAt; + + Instant lastYesAt; + Instant lastNoAt; + String lastSeverity; + Map lastDetails; + + Instant pendingCancelAt; + Map pendingCancelDetails; + + SourceState(boolean active) { + this.active = active; + } + } + + static final class MemberOutcome { + final boolean stateChanged; + final boolean notify; + + MemberOutcome(boolean stateChanged, boolean notify) { + this.stateChanged = stateChanged; + this.notify = notify; + } + + static MemberOutcome noop() { + return new MemberOutcome(false, false); + } + } +} \ No newline at end of file diff --git a/src/main/java/at/co/procon/malis/cep/lifecycle/LifecycleDispatcherService.java b/src/main/java/at/co/procon/malis/cep/lifecycle/LifecycleDispatcherService.java new file mode 100644 index 0000000..c1d60ea --- /dev/null +++ b/src/main/java/at/co/procon/malis/cep/lifecycle/LifecycleDispatcherService.java @@ -0,0 +1,55 @@ +package at.co.procon.malis.cep.lifecycle; + +import at.co.procon.malis.cep.config.CepProperties; +import at.co.procon.malis.cep.detector.DetectorEvent; +import org.springframework.stereotype.Component; + +import java.util.List; +import java.util.Locale; +import java.util.Map; + +/** + * Routes lifecycle processing to either: + * - {@link AlarmLifecycleService} (SIMPLE policies) + * - {@link CompositeAlarmLifecycleService} (COMPOSITE policies) + * + * This enables SIMPLE and COMPOSITE lifecycles to run "in parallel" in the same service + * (per binding, via lifecyclePolicyRef). + */ +@Component +public class LifecycleDispatcherService { + + private final CepProperties props; + private final AlarmLifecycleService simple; + private final CompositeAlarmLifecycleService composite; + + public LifecycleDispatcherService(CepProperties props, + AlarmLifecycleService simple, + CompositeAlarmLifecycleService composite) { + this.props = props; + this.simple = simple; + this.composite = composite; + } + + public List apply(String bindingId, + CepProperties.SourceBinding binding, + Map vars, + List detectorEvents) { + + CepProperties.LifecyclePolicy policy = resolvePolicy(binding); + String type = policy.getType() == null ? "SIMPLE" : policy.getType().trim().toUpperCase(Locale.ROOT); + + if (type.equals("COMPOSITE")) { + return composite.apply(bindingId, binding, vars, detectorEvents); + } + return simple.apply(bindingId, binding, vars, detectorEvents); + } + + private CepProperties.LifecyclePolicy resolvePolicy(CepProperties.SourceBinding binding) { + String ref = binding == null ? null : binding.getLifecyclePolicyRef(); + if (ref != null && props.getLifecyclePolicies().containsKey(ref)) { + return props.getLifecyclePolicies().get(ref); + } + return props.getLifecyclePolicies().getOrDefault("default", new CepProperties.LifecyclePolicy()); + } +} diff --git a/src/main/java/at/co/procon/malis/cep/lifecycle/LifecycleHeartbeatScheduler.java b/src/main/java/at/co/procon/malis/cep/lifecycle/LifecycleHeartbeatScheduler.java new file mode 100644 index 0000000..7fba293 --- /dev/null +++ b/src/main/java/at/co/procon/malis/cep/lifecycle/LifecycleHeartbeatScheduler.java @@ -0,0 +1,257 @@ +package at.co.procon.malis.cep.lifecycle; + +import at.co.procon.malis.cep.binding.BindingsLoadedEvent; +import at.co.procon.malis.cep.config.CepProperties; +import at.co.procon.malis.cep.output.PublisherService; +import jakarta.annotation.PreDestroy; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.context.event.EventListener; +import org.springframework.stereotype.Component; + +import java.time.Instant; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.concurrent.*; + +/** + * Background scheduler for: + * - per-binding heartbeat emissions (periodic state propagation) + * - SIMPLE lifecycle pending-cancel maturity flush (debounce) + * + *

Why this exists: + *

    + *
  • SIMPLE lifecycle has debounced CANCELs that must mature even when no new ingress messages arrive.
    • + *
  • Heartbeat is per binding and optional; it re-emits the current state (ALARM/CANCEL) periodically.
    • + *
+ */ +@Component +public class LifecycleHeartbeatScheduler { + + private static final Logger log = LoggerFactory.getLogger(LifecycleHeartbeatScheduler.class); + + /** How often we check for matured pending cancels in SIMPLE lifecycle. */ + private static final long SIMPLE_PENDING_CANCEL_TICK_MS = 250; + + private final CepProperties props; + private final AlarmLifecycleService simple; + private final CompositeAlarmLifecycleService composite; + private final PublisherService publisher; + + private final ScheduledExecutorService scheduler; + private volatile ScheduledFuture simplePendingCancelTask; + + private static final class HeartbeatTaskInfo { + final ScheduledFuture future; + final long periodMs; + final long initialDelayMs; + final boolean includeInactive; + + private HeartbeatTaskInfo(ScheduledFuture future, long periodMs, long initialDelayMs, boolean includeInactive) { + this.future = future; + this.periodMs = periodMs; + this.initialDelayMs = initialDelayMs; + this.includeInactive = includeInactive; + } + } + + private final ConcurrentHashMap heartbeatTasks = new ConcurrentHashMap<>(); + + public LifecycleHeartbeatScheduler(CepProperties props, + AlarmLifecycleService simple, + CompositeAlarmLifecycleService composite, + PublisherService publisher) { + this.props = props; + this.simple = simple; + this.composite = composite; + this.publisher = publisher; + this.scheduler = Executors.newScheduledThreadPool(2, r -> { + Thread t = new Thread(r, "lifecycle-heartbeat-scheduler"); + t.setDaemon(true); + return t; + }); + + // Always run SIMPLE pending-cancel maturity flush. + this.simplePendingCancelTask = scheduler.scheduleAtFixedRate( + this::safeFlushMaturedPendingCancels, + SIMPLE_PENDING_CANCEL_TICK_MS, + SIMPLE_PENDING_CANCEL_TICK_MS, + TimeUnit.MILLISECONDS + ); + } + + @PreDestroy + public void shutdown() { + try { + if (simplePendingCancelTask != null) simplePendingCancelTask.cancel(false); + } catch (Exception ignored) { + } + + for (HeartbeatTaskInfo info : heartbeatTasks.values()) { + try { + if (info != null && info.future != null) info.future.cancel(false); + } catch (Exception ignored) { + } + } + heartbeatTasks.clear(); + + scheduler.shutdownNow(); + } + + /** + * Bindings are loaded/merged by {@link at.co.procon.malis.cep.binding.BindingFileLoader}. + * We schedule per-binding heartbeat tasks after that. + */ + @EventListener + public void onBindingsLoaded(BindingsLoadedEvent ev) { + Map bindings = ev == null ? null : ev.getBindings(); + if (bindings == null) bindings = props.getBindings(); + rescheduleHeartbeats(bindings); + } + + private void rescheduleHeartbeats(Map bindings) { + // Cancel tasks for bindings that disappeared or no longer have heartbeat enabled. + for (String id : heartbeatTasks.keySet()) { + CepProperties.SourceBinding b = (bindings == null) ? null : bindings.get(id); + boolean enabled = b != null && b.isEnabled() && b.getHeartbeat() != null && b.getHeartbeat().isEnabled(); + if (!enabled) { + cancelHeartbeatTask(id); + } + } + + if (bindings == null) return; + + // Schedule (or reschedule) enabled heartbeats. + for (var e : bindings.entrySet()) { + String bindingId = e.getKey(); + CepProperties.SourceBinding binding = e.getValue(); + if (binding == null || !binding.isEnabled()) continue; + if (binding.getHeartbeat() == null || !binding.getHeartbeat().isEnabled()) continue; + + long periodMs = Math.max(0, binding.getHeartbeat().getPeriodMs()); + if (periodMs < 1000) { + // Should be caught by startup validator, but be defensive. + log.warn("Heartbeat periodMs < 1000ms for binding {}. Forcing 1000ms.", bindingId); + periodMs = 1000; + } + + long initialDelayMs = binding.getHeartbeat().getInitialDelayMs(); + if (initialDelayMs <= 0) initialDelayMs = periodMs; + + boolean includeInactive = binding.getHeartbeat().isIncludeInactive(); + + HeartbeatTaskInfo current = heartbeatTasks.get(bindingId); + if (current != null + && current.periodMs == periodMs + && current.initialDelayMs == initialDelayMs + && current.includeInactive == includeInactive + && current.future != null + && !current.future.isCancelled()) { + continue; // already scheduled with same config + } + + // Config changed or missing -> (re)create + cancelHeartbeatTask(bindingId); + + ScheduledFuture future = scheduler.scheduleAtFixedRate( + () -> safeHeartbeatTick(bindingId), + initialDelayMs, + periodMs, + TimeUnit.MILLISECONDS + ); + heartbeatTasks.put(bindingId, new HeartbeatTaskInfo(future, periodMs, initialDelayMs, includeInactive)); + log.info("Heartbeat enabled for binding {} (periodMs={}, includeInactive={})", bindingId, periodMs, includeInactive); + } + } + + private void cancelHeartbeatTask(String bindingId) { + if (bindingId == null) return; + HeartbeatTaskInfo info = heartbeatTasks.remove(bindingId); + if (info == null || info.future == null) return; + try { info.future.cancel(false); } catch (Exception ignored) {} + } + + private void safeFlushMaturedPendingCancels() { + try { + Instant now = Instant.now(); + List due = simple.collectMaturedPendingCancels(now); + if (due == null || due.isEmpty()) return; + + for (AlarmLifecycleService.ScheduledEmission em : due) { + if (em == null) continue; + String bindingId = em.getBindingId(); + CepProperties.SourceBinding binding = bindingId == null ? null : props.getBindings().get(bindingId); + if (binding == null || !binding.isEnabled()) continue; + if (binding.getOutputRefs() == null || binding.getOutputRefs().isEmpty()) continue; + + try { + publisher.publish(binding, em.getVars(), List.of(em.getAlarm())); + } catch (Exception ignored) { + // do not crash scheduler + } + } + } catch (Exception ignored) { + // Never crash the scheduler. + } + } + + private void safeHeartbeatTick(String bindingId) { + try { + CepProperties.SourceBinding binding = props.getBindings().get(bindingId); + if (binding == null || !binding.isEnabled()) return; + CepProperties.HeartbeatDef hb = binding.getHeartbeat(); + if (hb == null || !hb.isEnabled()) return; + if (binding.getOutputRefs() == null || binding.getOutputRefs().isEmpty()) return; + + String type = resolvePolicyType(binding); + Instant now = Instant.now(); + + if ("COMPOSITE".equals(type)) { + List due = composite.collectDueHeartbeats(bindingId, hb, now); + publishCompositeEmissions(due); + } else { + List due = simple.collectDueHeartbeats(bindingId, hb, now); + publishSimpleEmissions(binding, due); + } + } catch (Exception ignored) { + // Never crash the scheduler. + } + } + + private void publishSimpleEmissions(CepProperties.SourceBinding binding, List due) { + if (due == null || due.isEmpty()) return; + for (AlarmLifecycleService.ScheduledEmission em : due) { + if (em == null || em.getAlarm() == null) continue; + try { + publisher.publish(binding, em.getVars(), List.of(em.getAlarm())); + } catch (Exception ignored) { + } + } + } + + private void publishCompositeEmissions(List due) { + if (due == null || due.isEmpty()) return; + for (CompositeAlarmLifecycleService.ScheduledEmission em : due) { + if (em == null || em.getAlarm() == null) continue; + try { + publisher.publish(em.getBinding(), em.getVars(), List.of(em.getAlarm())); + } catch (Exception ignored) { + } + } + } + + private String resolvePolicyType(CepProperties.SourceBinding binding) { + CepProperties.LifecyclePolicy policy = resolvePolicy(binding); + String t = policy.getType() == null ? "SIMPLE" : policy.getType().trim().toUpperCase(Locale.ROOT); + return t.equals("COMPOSITE") ? "COMPOSITE" : "SIMPLE"; + } + + private CepProperties.LifecyclePolicy resolvePolicy(CepProperties.SourceBinding binding) { + String ref = binding == null ? null : binding.getLifecyclePolicyRef(); + if (ref != null && props.getLifecyclePolicies().containsKey(ref)) { + return props.getLifecyclePolicies().get(ref); + } + return props.getLifecyclePolicies().getOrDefault("default", new CepProperties.LifecyclePolicy()); + } +} diff --git a/src/main/java/at/co/procon/malis/cep/lifecycle/PublishableAlarm.java b/src/main/java/at/co/procon/malis/cep/lifecycle/PublishableAlarm.java new file mode 100644 index 0000000..f1b85fc --- /dev/null +++ b/src/main/java/at/co/procon/malis/cep/lifecycle/PublishableAlarm.java @@ -0,0 +1,84 @@ +package at.co.procon.malis.cep.lifecycle; + +import java.time.Instant; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.Map; + +/** + * Output of the lifecycle stage. + * + *

This is the normalized fault/alarm event ready to be formatted and published to outputs. + * + *

Note: The lifecycle is where we apply caller-managed state: dedup, debounce, and optional TTL auto-clear. + * + *

Instance correlation: {@code instanceId} identifies one concrete alarm occurrence (incident). It is created + * on OPEN (first transition to active) and must remain stable for all subsequent UPDATE/CANCEL events of the same + * occurrence. + */ +public final class PublishableAlarm { + + /** + * Stable definition/grouping key (what kind of alarm is this?). + * + *

Historically called {@code faultKey}. Keep the name for compatibility with existing code. + */ + private final String faultKey; + + /** + * Unique occurrence id for correlation across systems and Temporal workflows. + * + *

Should be present for all emitted events once the lifecycle has been upgraded. + */ + private final String instanceId; + + private final String action; + private final String severity; + private final Instant occurredAt; + private final Map details; + + /** + * Backwards compatible constructor (no instance id). + * + *

Prefer {@link #PublishableAlarm(String, String, String, String, Instant, Map)} going forward. + */ + public PublishableAlarm(String faultKey, + String action, + String severity, + Instant occurredAt, + Map details) { + this(faultKey, null, action, severity, occurredAt, details); + } + + public PublishableAlarm(String faultKey, + String instanceId, + String action, + String severity, + Instant occurredAt, + Map details) { + this.faultKey = faultKey; + this.instanceId = instanceId; + this.action = action; + this.severity = severity; + this.occurredAt = occurredAt; + this.details = details == null ? Map.of() : Collections.unmodifiableMap(new LinkedHashMap<>(details)); + } + + public String getFaultKey() { return faultKey; } + public String getInstanceId() { return instanceId; } + public String getAction() { return action; } + public String getSeverity() { return severity; } + public Instant getOccurredAt() { return occurredAt; } + public Map getDetails() { return details; } + + // record-style accessors + public String faultKey() { return faultKey; } + public String instanceId() { return instanceId; } + public String action() { return action; } + public String severity() { return severity; } + public Instant occurredAt() { return occurredAt; } + public Map details() { return details; } + + /** Alias for callers that prefer the "definition key" term. */ + public String definitionKey() { return faultKey; } +} \ No newline at end of file diff --git a/src/main/java/at/co/procon/malis/cep/output/AlarmFormatter.java b/src/main/java/at/co/procon/malis/cep/output/AlarmFormatter.java new file mode 100644 index 0000000..079d262 --- /dev/null +++ b/src/main/java/at/co/procon/malis/cep/output/AlarmFormatter.java @@ -0,0 +1,21 @@ +package at.co.procon.malis.cep.output; + +import at.co.procon.malis.cep.lifecycle.PublishableAlarm; + +import java.util.Map; + +/** + * Formats a {@link PublishableAlarm} into a transport payload. + * + *

Formatters are responsible for converting the canonical alarm model into a desired output schema + * (e.g. EEBUS Alarm Datagram XML). + */ +public interface AlarmFormatter { + + /** + * @param alarm canonical alarm event produced by the lifecycle stage + * @param vars binding variables (tenant/site/department/deviceId, ...) + * @param outputConfig output-specific configuration map from {@code cep.outputs..config} + */ + FormattedMessage format(PublishableAlarm alarm, Map vars, Map outputConfig); +} diff --git a/src/main/java/at/co/procon/malis/cep/output/EebusAlarmDatagramXmlFormatter.java b/src/main/java/at/co/procon/malis/cep/output/EebusAlarmDatagramXmlFormatter.java new file mode 100644 index 0000000..70b2282 --- /dev/null +++ b/src/main/java/at/co/procon/malis/cep/output/EebusAlarmDatagramXmlFormatter.java @@ -0,0 +1,216 @@ +// ============================================================ +// 4) Output formatter aligned to your sample: + alarmListData +// ============================================================ + +package at.co.procon.malis.cep.output; + +import at.co.procon.malis.cep.eebus.EebusSpineConstants; +import at.co.procon.malis.cep.eebus.EebusSpineXsdValidator; +import at.co.procon.malis.cep.eebus.EebusSpineXsdValidatorBase; +import at.co.procon.malis.cep.lifecycle.PublishableAlarm; +import org.springframework.beans.factory.annotation.Qualifier; + +import java.nio.charset.StandardCharsets; +import java.time.Instant; +import java.util.Map; +import java.util.concurrent.atomic.AtomicLong; + +/** + * Emits a SPINE v1 datagram that matches the structure of your sample. + * + * - Root: + * - Header includes structured addresses (device/entity/feature) + * - Cmd includes alarmListData and ... + * + * The formatter reads optional EEBUS config from outputDef.config.eebus. + */ +public class EebusAlarmDatagramXmlFormatter implements AlarmFormatter { + + private static final AtomicLong MSG_COUNTER = new AtomicLong(1); + + private final EebusSpineXsdValidatorBase xsdValidator; + + public EebusAlarmDatagramXmlFormatter(@Qualifier("eebusSpineXsdJaxbValidator") EebusSpineXsdValidatorBase xsdValidator) { + this.xsdValidator = xsdValidator; + } + + @Override + public FormattedMessage format(PublishableAlarm alarm, Map vars, Map outputConfig) { + + String srcDeviceTpl = str(outputConfig, "addressSourceTemplate", "${deviceId}"); + String dstDeviceTpl = str(outputConfig, "addressDestinationTemplate", "malis-cep"); + + Map headerExtras = map(outputConfig, "headerExtras"); + String specVersion = str(headerExtras, "specificationVersion", "1.3.0"); + + Map classifierCfg = map(outputConfig, "cmdClassifier"); + String classifier = "CANCEL".equalsIgnoreCase(alarm.action()) + ? str(classifierCfg, "cancel", "notify") + : str(classifierCfg, "alarm", "notify"); + classifier = normalizeClassifier(classifier); + + String srcDevice = resolve(srcDeviceTpl, vars); + String dstDevice = resolve(dstDeviceTpl, vars); + + int srcEntity = intVal(outputConfig, "addressSourceEntity", 1); + int srcFeature = intVal(outputConfig, "addressSourceFeature", 1); + int dstEntity = intVal(outputConfig, "addressDestinationEntity", 1); + int dstFeature = intVal(outputConfig, "addressDestinationFeature", 1); + + Map d = alarm.details(); + + String alarmId = d != null && d.get("alarmId") != null ? String.valueOf(d.get("alarmId")) : String.valueOf(MSG_COUNTER.get()); + String thresholdId = d != null && d.get("thresholdId") != null ? String.valueOf(d.get("thresholdId")) : null; + + String alarmType = d != null && d.get("alarmType") != null + ? String.valueOf(d.get("alarmType")) + : ("CANCEL".equalsIgnoreCase(alarm.action()) ? "alarmCancelled" : "overThreshold"); + + String scopeType = d != null && d.get("scopeType") != null ? String.valueOf(d.get("scopeType")) : null; + String label = d != null && d.get("label") != null ? String.valueOf(d.get("label")) : alarm.faultKey(); + String description = d != null && d.get("description") != null ? String.valueOf(d.get("description")) : null; + + String mvNumber = d != null && d.get("measuredValue.number") != null ? String.valueOf(d.get("measuredValue.number")) : null; + String mvScale = d != null && d.get("measuredValue.scale") != null ? String.valueOf(d.get("measuredValue.scale")) : null; + + String epStart = d != null && d.get("evaluationPeriod.startTime") != null ? String.valueOf(d.get("evaluationPeriod.startTime")) : null; + String epEnd = d != null && d.get("evaluationPeriod.endTime") != null ? String.valueOf(d.get("evaluationPeriod.endTime")) : null; + + long msgCounter = MSG_COUNTER.getAndIncrement(); + + String headerTs = Instant.now().toString(); + String alarmTs = (alarm.occurredAt() != null ? alarm.occurredAt() : Instant.now()).toString(); + + String ns = EebusSpineConstants.SPINE_NS; + + StringBuilder sb = new StringBuilder(1400); + + sb.append(""); + sb.append('<').append(EebusSpineConstants.PREFIX).append(":datagram xmlns:") + .append(EebusSpineConstants.PREFIX).append("=\"").append(ns).append("\">"); + + sb.append('<').append(EebusSpineConstants.PREFIX).append(":header>"); + sb.append(tag("specificationVersion", specVersion)); + + sb.append('<').append(EebusSpineConstants.PREFIX).append(":addressSource>"); + sb.append(tag("device", srcDevice)); + sb.append(tag("entity", String.valueOf(srcEntity))); + sb.append(tag("feature", String.valueOf(srcFeature))); + sb.append(""); + + sb.append('<').append(EebusSpineConstants.PREFIX).append(":addressDestination>"); + sb.append(tag("device", dstDevice)); + sb.append(tag("entity", String.valueOf(dstEntity))); + sb.append(tag("feature", String.valueOf(dstFeature))); + sb.append(""); + + sb.append(tag("msgCounter", String.valueOf(msgCounter))); + sb.append(tag("cmdClassifier", classifier)); + sb.append(tag("timestamp", headerTs)); + + sb.append(""); + + sb.append('<').append(EebusSpineConstants.PREFIX).append(":payload>"); + sb.append('<').append(EebusSpineConstants.PREFIX).append(":cmd>"); + sb.append(tag("function", "alarmListData")); + + sb.append('<').append(EebusSpineConstants.PREFIX).append(":alarmListData>"); + sb.append('<').append(EebusSpineConstants.PREFIX).append(":alarmData>"); + + sb.append(tag("alarmId", alarmId)); + if (thresholdId != null) sb.append(tag("thresholdId", thresholdId)); + sb.append(tag("timestamp", alarmTs)); + sb.append(tag("alarmType", alarmType)); + + if (mvNumber != null && mvScale != null) { + sb.append('<').append(EebusSpineConstants.PREFIX).append(":measuredValue>"); + sb.append(tag("number", mvNumber)); + sb.append(tag("scale", mvScale)); + sb.append(""); + } + + if (epStart != null || epEnd != null) { + sb.append('<').append(EebusSpineConstants.PREFIX).append(":evaluationPeriod>"); + if (epStart != null) sb.append(tag("startTime", epStart)); + if (epEnd != null) sb.append(tag("endTime", epEnd)); + sb.append(""); + } + + if (scopeType != null) sb.append(tag("scopeType", scopeType)); + if (label != null) sb.append(tag("label", label)); + if (description != null) sb.append(tag("description", description)); + + sb.append(""); + sb.append(""); + + sb.append(""); + sb.append(""); + + sb.append(""); + + byte[] payload = sb.toString().getBytes(StandardCharsets.UTF_8); + + if (xsdValidator != null) { + xsdValidator.validateDatagram(payload, "alarm output faultKey=" + alarm.faultKey()); + } + + return new FormattedMessage(payload, "application/xml; charset=UTF-8"); + } + + private String tag(String local, String text) { + return "<" + EebusSpineConstants.PREFIX + ":" + local + ">" + escape(text) + ""; + } + + private static String escape(String s) { + if (s == null) return ""; + return s.replace("&", "&") + .replace("<", "<") + .replace(">", ">") + .replace("\"", """) + .replace("'", "'"); + } + + private static String resolve(String tpl, Map vars) { + if (tpl == null) return ""; + String out = tpl; + if (vars != null) { + for (var e : vars.entrySet()) { + out = out.replace("${" + e.getKey() + "}", e.getValue()); + } + } + return out; + } + + private static String str(Map m, String k, String def) { + if (m == null) return def; + Object v = m.get(k); + return v == null ? def : String.valueOf(v); + } + + @SuppressWarnings("unchecked") + private static Map map(Map m, String k) { + if (m == null) return Map.of(); + Object v = m.get(k); + if (v instanceof Map mv) { + return (Map) mv; + } + return Map.of(); + } + + private static int intVal(Map m, String k, int def) { + if (m == null) return def; + Object v = m.get(k); + if (v instanceof Number n) return n.intValue(); + if (v == null) return def; + try { return Integer.parseInt(String.valueOf(v)); } catch (Exception ignore) { return def; } + } + + private static String normalizeClassifier(String classifier) { + if (classifier == null) return "notify"; + String c = classifier.trim().toLowerCase(); + return switch (c) { + case "read", "reply", "notify", "write", "call", "result" -> c; + default -> "notify"; + }; + } +} \ No newline at end of file diff --git a/src/main/java/at/co/procon/malis/cep/output/EebusAlarmMessageFormatter.java b/src/main/java/at/co/procon/malis/cep/output/EebusAlarmMessageFormatter.java new file mode 100644 index 0000000..6ef9a85 --- /dev/null +++ b/src/main/java/at/co/procon/malis/cep/output/EebusAlarmMessageFormatter.java @@ -0,0 +1,57 @@ +package at.co.procon.malis.cep.output; + +import at.co.procon.malis.cep.lifecycle.PublishableAlarm; +import com.fasterxml.jackson.databind.ObjectMapper; + +import java.nio.charset.StandardCharsets; +import java.time.Instant; +import java.util.LinkedHashMap; +import java.util.Map; + +/** + * Minimal EEBUS Alarm Message JSON formatter (placeholder). + * + * Replace the JSON shape with your final EEBUS Alarm/Cancel message schema. + */ +public class EebusAlarmMessageFormatter implements AlarmFormatter { + + private final ObjectMapper om = new ObjectMapper(); + + @Override + public FormattedMessage format(PublishableAlarm alarm, Map vars, Map outputConfig) { + try { + String alarmId = shortId(alarm.faultKey()); + + Map json = new LinkedHashMap<>(); + json.put("alarmId", alarmId); + json.put("action", alarm.action()); + json.put("faultKey", alarm.faultKey()); + json.put("severity", alarm.severity() == null ? "UNKNOWN" : alarm.severity()); + json.put("timestamp", (alarm.occurredAt() == null ? Instant.now() : alarm.occurredAt()).toString()); + json.put("tenant", vars.get("tenant")); + json.put("site", vars.get("site")); + json.put("department", vars.get("department")); + json.put("deviceId", vars.get("deviceId")); + json.put("details", alarm.details()); + + byte[] payload = om.writeValueAsBytes(json); + return new FormattedMessage(payload, "application/json"); + } catch (Exception e) { + throw new RuntimeException("Failed to format EEBUS alarm message", e); + } + } + + private static String shortId(String s) { + try { + var md = java.security.MessageDigest.getInstance("SHA-256"); + byte[] digest = md.digest(s.getBytes(StandardCharsets.UTF_8)); + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < Math.min(8, digest.length); i++) { + sb.append(String.format("%02x", digest[i])); + } + return sb.toString(); + } catch (Exception e) { + return "00000000"; + } + } +} diff --git a/src/main/java/at/co/procon/malis/cep/output/FormattedMessage.java b/src/main/java/at/co/procon/malis/cep/output/FormattedMessage.java new file mode 100644 index 0000000..1e139b1 --- /dev/null +++ b/src/main/java/at/co/procon/malis/cep/output/FormattedMessage.java @@ -0,0 +1,22 @@ +package at.co.procon.malis.cep.output; + +/** + * Result of formatting an alarm for a concrete output format. + */ +public final class FormattedMessage { + + private final byte[] payload; + private final String contentType; + + public FormattedMessage(byte[] payload, String contentType) { + this.payload = payload == null ? new byte[0] : payload.clone(); + this.contentType = contentType; + } + + public byte[] getPayload() { return payload.clone(); } + public String getContentType() { return contentType; } + + // record-style accessors + public byte[] payload() { return getPayload(); } + public String contentType() { return contentType; } +} diff --git a/src/main/java/at/co/procon/malis/cep/output/MalisCepAlarmJsonFormatter.java b/src/main/java/at/co/procon/malis/cep/output/MalisCepAlarmJsonFormatter.java new file mode 100644 index 0000000..be97a6d --- /dev/null +++ b/src/main/java/at/co/procon/malis/cep/output/MalisCepAlarmJsonFormatter.java @@ -0,0 +1,256 @@ +package at.co.procon.malis.cep.output; + +import at.co.procon.malis.cep.lifecycle.PublishableAlarm; +import at.co.procon.malis.cep.util.TemplateUtil; +import com.fasterxml.jackson.databind.ObjectMapper; + +import java.time.Instant; +import java.time.OffsetDateTime; +import java.time.ZoneOffset; +import java.time.ZonedDateTime; +import java.util.*; + +/** + * Formatter: MALIS_CEP_ALARM_JSON + * + *

Produces the JSON payload expected by MalIS CEP alarm ingress endpoint + * (MalIS DTO: CepAlarmNotificationRequest). + * + *

Important: MalIS expects timestamps as {@link Instant}. In JSON we emit ISO-8601 strings + * (e.g. {@code 2026-02-26T13:58:06.190Z}) to avoid any ambiguity/precision loss. + * + *

We also expand template placeholders like {@code ${function}} using {@code vars}. + */ +public class MalisCepAlarmJsonFormatter implements AlarmFormatter { + + private final ObjectMapper om; + + public MalisCepAlarmJsonFormatter(ObjectMapper om) { + this.om = om; + } + + @Override + public FormattedMessage format(PublishableAlarm alarm, Map vars, Map outputConfig) { + try { + if (alarm == null) { + byte[] payload = om.writeValueAsBytes(Map.of()); + return new FormattedMessage(payload, "application/json"); + } + + Map v = new LinkedHashMap<>(); + if (vars != null) v.putAll(vars); + + // Build output root + Map out = new LinkedHashMap<>(); + + // --- Normalize/expand definitionKey/groupKey placeholders (composite often contains ${function}) --- + Map details = normalizeDetails(alarm.details(), alarm, v); + + // Root fields + out.put("faultKey", expandIfTemplate(alarm.faultKey(), v, details, alarm)); + if (alarm.instanceId() != null) out.put("instanceId", alarm.instanceId()); + if (alarm.action() != null) out.put("action", alarm.action()); + if (alarm.severity() != null) out.put("severity", alarm.severity()); + if (alarm.occurredAt() != null) out.put("occurredAt", alarm.occurredAt().toString()); + + // Ensure identity fields exist in details (MalIS Option-B correlation uses these) + ensureIdentityInDetails(details, alarm, v); + + if (!details.isEmpty()) out.put("details", details); + + byte[] payload = om.writeValueAsBytes(out); + return new FormattedMessage(payload, "application/json"); + } catch (Exception e) { + throw new RuntimeException("Failed to format MALIS_CEP_ALARM_JSON", e); + } + } + + private static void ensureIdentityInDetails(Map details, PublishableAlarm alarm, Map vars) { + if (details == null) return; + + // instanceId + if (alarm.instanceId() != null) { + details.putIfAbsent("instanceId", alarm.instanceId()); + } + + // definitionKey (prefer details.definitionKey/groupKey, else alarm.faultKey) + String def = firstNonBlank( + asString(details.get("definitionKey")), + asString(details.get("groupKey")), + alarm.faultKey() + ); + def = expandIfTemplate(def, vars, details, alarm); + if (def != null) details.put("definitionKey", def); + + // firstSeenAt as ISO Instant string + if (!details.containsKey("firstSeenAt")) { + Instant fs = alarm.occurredAt(); + if (fs != null) details.put("firstSeenAt", fs.toString()); + } else { + // normalize if it's an Instant-like type + Object fs = details.get("firstSeenAt"); + Object norm = normalizeObject(fs); + if (norm != null) details.put("firstSeenAt", norm); + } + } + + @SuppressWarnings("unchecked") + private static Map normalizeDetails(Map in, + PublishableAlarm alarm, + Map vars) { + Map details = new LinkedHashMap<>(); + + if (in != null && !in.isEmpty()) { + // normalize all Instant-like values to ISO strings + Object normalized = normalizeObject(in); + if (normalized instanceof Map m) { + for (var e : m.entrySet()) { + if (e.getKey() == null) continue; + details.put(String.valueOf(e.getKey()), e.getValue()); + } + } else { + details.put("rawDetails", normalized); + } + } + + // If composite groupKey/definitionKey contains templates, resolve them. + // If vars.function is missing, try to derive it. + // maybeDeriveFunction(vars, alarm, details); + + // Expand known template-holding keys if present + expandKeyIfTemplate(details, "groupKey", vars, alarm); + expandKeyIfTemplate(details, "definitionKey", vars, alarm); + + return details; + } + + private static void expandKeyIfTemplate(Map details, String key, Map vars, PublishableAlarm alarm) { + if (details == null || key == null) return; + Object v = details.get(key); + if (!(v instanceof String s)) return; + String expanded = expandIfTemplate(s, vars, details, alarm); + if (expanded != null) details.put(key, expanded); + } + + private static String expandIfTemplate(String maybeTemplate, Map vars, Map details, PublishableAlarm alarm) { + if (maybeTemplate == null) return null; + if (!maybeTemplate.contains("${")) return maybeTemplate; + + Map ctx = new LinkedHashMap<>(); + if (vars != null) ctx.putAll(vars); + + // add a few extra fallbacks into ctx + /* + String pinName = firstNonBlank( + ctx.get("function"), + ctx.get("pinName"), + asString(details == null ? null : details.get("pinName") + ) + ); + if (pinName != null) ctx.putIfAbsent("function", pinName); + */ + + if (alarm != null) { + if (alarm.faultKey() != null) ctx.putIfAbsent("faultKey", alarm.faultKey()); + if (alarm.instanceId() != null) ctx.putIfAbsent("instanceId", alarm.instanceId()); + if (alarm.action() != null) ctx.putIfAbsent("action", alarm.action()); + if (alarm.severity() != null) ctx.putIfAbsent("severity", alarm.severity()); + if (alarm.occurredAt() != null) ctx.putIfAbsent("occurredAt", alarm.occurredAt().toString()); + } + + return TemplateUtil.expand(maybeTemplate, ctx); + } + + /** + * Composite definitions often contain ${function} but vars may not include it. + * Derive "function" using (in order): + * - vars.function + * - vars.pinName + * - details.pinName + * - member.details.pinName for the changedSource member (or first member) + */ + @SuppressWarnings("unchecked") + private static void maybeDeriveFunction(Map vars, PublishableAlarm alarm, Map details) { + if (vars == null) return; + if (!isBlank(vars.get("function"))) return; + + String fn = firstNonBlank(vars.get("pinName"), asString(details.get("pinName"))); + + // try members[*].details.pinName + if (isBlank(fn) && details.get("members") instanceof List list && !list.isEmpty()) { + String changedSource = asString(details.get("changedSource")); + Map best = null; + + for (Object o : list) { + if (!(o instanceof Map m)) continue; + if (changedSource != null && changedSource.equals(asString(m.get("sourceKey")))) { + best = m; + break; + } + if (best == null) best = m; + } + + if (best != null) { + Object md = best.get("details"); + if (md instanceof Map mdet) { + fn = firstNonBlank(fn, asString(mdet.get("pinName"))); + } + } + } + + if (!isBlank(fn)) vars.put("function", fn); + } + + // ---------- normalization (Instant -> ISO string) ---------- + + @SuppressWarnings("unchecked") + private static Object normalizeObject(Object in) { + if (in == null) return null; + + if (in instanceof Instant inst) return inst.toString(); + if (in instanceof OffsetDateTime odt) return odt.toInstant().toString(); + if (in instanceof ZonedDateTime zdt) return zdt.toInstant().toString(); + if (in instanceof Date d) return d.toInstant().toString(); + + if (in instanceof Map m) { + Map out = new LinkedHashMap<>(); + for (var e : m.entrySet()) { + if (e.getKey() == null) continue; + out.put(String.valueOf(e.getKey()), normalizeObject(e.getValue())); + } + return out; + } + + if (in instanceof List list) { + List out = new ArrayList<>(list.size()); + for (Object o : list) out.add(normalizeObject(o)); + return out; + } + + // primitives/strings/numbers/bools are fine as-is + if (in instanceof String || in instanceof Number || in instanceof Boolean) return in; + + // unknown object: keep as string to avoid Jackson producing odd structures + return String.valueOf(in); + } + + // ---------- helpers ---------- + + private static String asString(Object o) { + if (o == null) return null; + if (o instanceof String s) return s; + return String.valueOf(o); + } + + private static boolean isBlank(String s) { + return s == null || s.trim().isEmpty(); + } + + private static String firstNonBlank(String... vals) { + if (vals == null) return null; + for (String v : vals) { + if (!isBlank(v)) return v; + } + return null; + } +} diff --git a/src/main/java/at/co/procon/malis/cep/output/MalisFaultJsonFormatter.java b/src/main/java/at/co/procon/malis/cep/output/MalisFaultJsonFormatter.java new file mode 100644 index 0000000..30ce9db --- /dev/null +++ b/src/main/java/at/co/procon/malis/cep/output/MalisFaultJsonFormatter.java @@ -0,0 +1,263 @@ +package at.co.procon.malis.cep.output; + +import at.co.procon.malis.cep.lifecycle.PublishableAlarm; +import at.co.procon.malis.cep.util.TemplateUtil; +import com.fasterxml.jackson.databind.ObjectMapper; + +import java.time.Instant; +import java.time.LocalDate; +import java.time.ZoneId; +import java.time.ZoneOffset; +import java.util.LinkedHashMap; +import java.util.Map; + +/** + * Formatter: MALIS_FAULT_JSON + * + *

Produces the JSON payload expected by MalIS REST Fault endpoints: + * 

+ * {
+ *   "siteId": "...",
+ *   "department": "...",
+ *   "pinName": "...",
+ *   "remark": "...",
+ *   "subject": "...",
+ *   "priority": "High",
+ *   "valueList": {"timestamp_device": "...", ...},
+ *   "faultDay": "yyyy-MM-dd"
+ * }
+ *
+ * + *

All scalar values are expanded from templates using vars. + * The formatter also derives defaults from the canonical {@link PublishableAlarm}: + *

    + *
  • priority: derived from alarm.severity if not provided
    • + *
  • valueList.timestamp_device: derived from alarm.occurredAt if not provided
    • + *
  • faultDay: derived from timestamp_device if not provided
    • + *
+ * + *

Config keys (cep.outputs.<id>.config): + *

    + *
  • siteIdTemplate (default ${site})
    • + *
  • departmentTemplate (default ${department})
    • + *
  • pinNameTemplate (default ${pinName})
    • + *
  • remarkTemplate (default ${remark})
    • + *
  • subjectTemplate (default ${pinName})
    • + *
  • priorityTemplate (default ${priority})
    • + *
  • faultDayTemplate (optional; otherwise derived)
    • + *
  • faultDayZone (optional; default UTC)
    • + *
  • includeEmptyValueList (optional; default false)
    • + *
  • valueListVarsPrefix (optional; default valueList.)
    • + *
  • valueList (optional map of key->template)
    • + *
  • includeScalarDetailsInValueList (optional; default true)
    • + *
+ */ +public class MalisFaultJsonFormatter implements AlarmFormatter { + + private final ObjectMapper om; + + public MalisFaultJsonFormatter(ObjectMapper om) { + this.om = om; + } + + @SuppressWarnings("unchecked") + @Override + public FormattedMessage format(PublishableAlarm alarm, Map vars, Map outputConfig) { + try { + Map cfg = outputConfig == null ? Map.of() : outputConfig; + Map v = new LinkedHashMap<>(); + if (vars != null) v.putAll(vars); + + // Make canonical alarm fields available to templates + if (alarm != null) { + if (alarm.severity() != null) v.putIfAbsent("severity", alarm.severity()); + if (alarm.action() != null) v.putIfAbsent("action", alarm.action()); + if (alarm.faultKey() != null) v.putIfAbsent("faultKey", alarm.faultKey()); + if (alarm.occurredAt() != null) v.putIfAbsent("occurredAt", alarm.occurredAt().toString()); + } + + String siteIdTemplate = strOr(cfg.get("siteIdTemplate"), "${site}"); + String departmentTemplate = strOr(cfg.get("departmentTemplate"), "${department}"); + String pinNameTemplate = strOr(cfg.get("pinNameTemplate"), "${pinName}"); + String remarkTemplate = strOr(cfg.get("remarkTemplate"), "${remark}"); + String subjectTemplate = strOr(cfg.get("subjectTemplate"), "${pinName}"); + String priorityTemplate = strOr(cfg.get("priorityTemplate"), "${priority}"); + String faultDayTemplate = str(cfg.get("faultDayTemplate")); + String faultDayZone = strOr(cfg.get("faultDayZone"), "UTC"); + boolean includeEmptyValueList = boolOr(cfg.get("includeEmptyValueList"), false); + String valueListVarsPrefix = strOr(cfg.get("valueListVarsPrefix"), "valueList."); + boolean includeScalarDetails = boolOr(cfg.get("includeScalarDetailsInValueList"), true); + + Map valueListTemplates = cfg.get("valueList") instanceof Map m ? toStringMap((Map) m) : Map.of(); + + String siteId = expand(siteIdTemplate, v); + String department = expand(departmentTemplate, v); + String pinName = expand(pinNameTemplate, v); + String remark = expand(remarkTemplate, v); + String subject = expand(subjectTemplate, v); + + String priority = expand(priorityTemplate, v); + if (isBlank(priority)) priority = derivePriorityFromSeverity(alarm == null ? null : alarm.severity()); + if (isBlank(priority)) priority = "High"; + + // Build valueList + Map valueList = new LinkedHashMap<>(); + + // 1) explicit templates + for (var e : valueListTemplates.entrySet()) { + String k = e.getKey(); + if (isBlank(k)) continue; + String vv = expand(e.getValue(), v); + if (!isBlank(vv)) valueList.put(k, vv); + } + + // 2) vars with prefix "valueList." + if (!isBlank(valueListVarsPrefix)) { + for (var e : v.entrySet()) { + String k = e.getKey(); + if (k == null || !k.startsWith(valueListVarsPrefix)) continue; + String subKey = k.substring(valueListVarsPrefix.length()); + if (isBlank(subKey)) continue; + String vv = e.getValue(); + if (!isBlank(vv)) valueList.putIfAbsent(subKey, vv); + } + } + + // 3) scalar details -> valueList (optional) + if (includeScalarDetails && alarm != null && alarm.details() != null && !alarm.details().isEmpty()) { + for (var e : alarm.details().entrySet()) { + String k = e.getKey(); + Object val = e.getValue(); + if (k == null || val == null) continue; + if (val instanceof String || val instanceof Number || val instanceof Boolean) { + // do not overwrite explicit templates/vars + valueList.putIfAbsent(k, String.valueOf(val)); + } + } + } + + // Ensure timestamp_device exists + if (!valueList.containsKey("timestamp_device")) { + String ts = firstNonBlank( + v.get("timestamp_device"), + v.get("timestampDevice"), + v.get("pinTimestamp"), + v.get("timestamp"), + v.get("occurredAt") + ); + if (isBlank(ts) && alarm != null && alarm.occurredAt() != null) { + ts = alarm.occurredAt().toString(); + } + if (!isBlank(ts)) valueList.put("timestamp_device", ts); + } + + // faultDay + String faultDay; + if (!isBlank(faultDayTemplate)) { + faultDay = expand(faultDayTemplate, v); + } else { + String ts = valueList.get("timestamp_device") instanceof String s ? s : null; + faultDay = deriveFaultDay(ts, faultDayZone); + } + + Map out = new LinkedHashMap<>(); + out.put("siteId", nullToEmpty(siteId)); + out.put("department", nullToEmpty(department)); + out.put("pinName", nullToEmpty(pinName)); + out.put("description", nullToEmpty(remark)); + out.put("subject", nullToEmpty(isBlank(subject) ? pinName : subject)); + out.put("priority", priority); + if (includeEmptyValueList || !valueList.isEmpty()) { + out.put("valueList", valueList); + } + if (!isBlank(faultDay)) { + out.put("faultDay", faultDay); + } + + byte[] payload = om.writeValueAsBytes(out); + return new FormattedMessage(payload, "application/json"); + } catch (Exception e) { + throw new RuntimeException("Failed to format MALIS_FAULT_JSON", e); + } + } + + private static String derivePriorityFromSeverity(String severity) { + if (isBlank(severity)) return null; + String s = severity.trim().toUpperCase(); + return switch (s) { + case "CRITICAL", "FATAL", "ERROR", "MAJOR" -> "High"; + case "WARN", "WARNING", "MINOR" -> "Medium"; + case "INFO", "OK" -> "Low"; + default -> null; + }; + } + + private static String deriveFaultDay(String timestampDevice, String zoneId) { + if (isBlank(timestampDevice)) return null; + ZoneId zone; + try { + zone = ZoneId.of(isBlank(zoneId) ? "UTC" : zoneId.trim()); + } catch (Exception ignore) { + zone = ZoneOffset.UTC; + } + Instant inst = parseInstant(timestampDevice); + if (inst == null) return null; + LocalDate d = LocalDate.ofInstant(inst, zone); + return d.toString(); + } + + private static Instant parseInstant(String s) { + if (isBlank(s)) return null; + try { + return Instant.parse(s.trim()); + } catch (Exception ignore) { + try { return java.time.OffsetDateTime.parse(s.trim()).toInstant(); } catch (Exception ignore2) { return null; } + } + } + + private static String expand(String template, Map vars) { + return TemplateUtil.expand(template, vars); + } + + private static boolean boolOr(Object o, boolean def) { + if (o == null) return def; + if (o instanceof Boolean b) return b; + String s = String.valueOf(o).trim(); + if (s.isEmpty()) return def; + return Boolean.parseBoolean(s); + } + + private static boolean isBlank(String s) { + return s == null || s.trim().isEmpty(); + } + + private static String nullToEmpty(String s) { + return s == null ? "" : s; + } + + private static String firstNonBlank(String... vals) { + if (vals == null) return null; + for (String v : vals) { + if (!isBlank(v)) return v; + } + return null; + } + + private static String str(Object o) { + return o == null ? null : String.valueOf(o); + } + + private static String strOr(Object o, String def) { + String s = str(o); + return (s == null || s.isBlank()) ? def : s; + } + + private static Map toStringMap(Map m) { + Map out = new LinkedHashMap<>(); + for (var e : m.entrySet()) { + if (e.getKey() == null) continue; + out.put(String.valueOf(e.getKey()), e.getValue() == null ? "" : String.valueOf(e.getValue())); + } + return out; + } +} diff --git a/src/main/java/at/co/procon/malis/cep/output/MqttOutputPublisher.java b/src/main/java/at/co/procon/malis/cep/output/MqttOutputPublisher.java new file mode 100644 index 0000000..e3bf845 --- /dev/null +++ b/src/main/java/at/co/procon/malis/cep/output/MqttOutputPublisher.java @@ -0,0 +1,41 @@ +package at.co.procon.malis.cep.output; + +import at.co.procon.malis.cep.transport.mqtt.MqttBrokerRegistry; +import at.co.procon.malis.cep.util.TemplateUtil; + +import java.util.Map; + +/** + * MQTT output publisher (real implementation). + * + * This class is intentionally small: + * - It expands a topic template with variables (tenant/site/department/deviceId) and publishes the bytes. + * - Connection management is delegated to {@link MqttBrokerRegistry} (lazy Paho clients). + */ +public class MqttOutputPublisher implements OutputPublisher { + + /** Which broker configuration (cep.brokers.) to use for this output. */ + private final String brokerRef; + private final String topicTemplate; + private final Integer qos; + private final Boolean retained; + private final MqttBrokerRegistry mqtt; + + public MqttOutputPublisher(String brokerRef, + String topicTemplate, + Integer qos, + Boolean retained, + MqttBrokerRegistry mqtt) { + this.brokerRef = brokerRef; + this.topicTemplate = topicTemplate; + this.qos = qos; + this.retained = retained; + this.mqtt = mqtt; + } + + @Override + public void publish(FormattedMessage msg, Map vars) { + String topic = TemplateUtil.expand(topicTemplate, vars); + mqtt.publish(brokerRef, topic, msg.payload(), qos, retained); + } +} diff --git a/src/main/java/at/co/procon/malis/cep/output/OutboxMessage.java b/src/main/java/at/co/procon/malis/cep/output/OutboxMessage.java new file mode 100644 index 0000000..6cb68fe --- /dev/null +++ b/src/main/java/at/co/procon/malis/cep/output/OutboxMessage.java @@ -0,0 +1,48 @@ +package at.co.procon.malis.cep.output; + + +import lombok.Data; + +import java.time.Instant; +import java.util.Base64; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.Map; + + +/** + * One published output message stored in memory. + * + * This is designed for REST workflows: + * - POST an ingress event + * - retrieve the produced output payload(s) from the outbox + */ +@Data +public class OutboxMessage { + private final String id; + private final Instant storedAt; + private final String requestId; + private final String outputRef; + private final String contentType; + private final byte[] payload; + private final Map vars; + + + public OutboxMessage(String id, + Instant storedAt, + String requestId, + String outputRef, + String contentType, + byte[] payload, + Map vars) { + this.id = id; + this.storedAt = storedAt; + this.requestId = requestId; + this.outputRef = outputRef; + this.contentType = contentType; + this.payload = payload == null ? new byte[0] : payload.clone(); + this.vars = vars == null ? Map.of() : Collections.unmodifiableMap(new LinkedHashMap<>(vars)); + } + + +} \ No newline at end of file diff --git a/src/main/java/at/co/procon/malis/cep/output/OutboxOutputPublisher.java b/src/main/java/at/co/procon/malis/cep/output/OutboxOutputPublisher.java new file mode 100644 index 0000000..8c9c1d9 --- /dev/null +++ b/src/main/java/at/co/procon/malis/cep/output/OutboxOutputPublisher.java @@ -0,0 +1,65 @@ +package at.co.procon.malis.cep.output; + + +import java.time.Instant; +import java.util.Map; +import java.util.UUID; + +/** + * OutputPublisher that does NOT send anywhere. + * + * It stores the already-formatted payload (XML/JSON/...) in the OutboxStore. + * + * Use-case: + * - REST ingress should be able to fetch "what would have been published". + */ +public class OutboxOutputPublisher implements OutputPublisher, OutboxStoreProvider { + + private final String outputRef; + private final OutboxStore store; + private final String correlationVar; + private final int maxPerRequest; + private final int maxTotal; + + public OutboxOutputPublisher(String outputRef, OutboxStore store, Map cfg) { + this.outputRef = outputRef; + this.store = store; + + this.correlationVar = cfg != null && cfg.get("correlationVar") != null + ? String.valueOf(cfg.get("correlationVar")) + : "requestId"; + + this.maxPerRequest = cfg != null && cfg.get("maxPerRequest") instanceof Number n + ? n.intValue() : 200; + + this.maxTotal = cfg != null && cfg.get("maxTotal") instanceof Number n + ? n.intValue() : 5000; + } + + + @Override + public OutboxStore getOutboxStore() { + return store; + } + + + @Override + public void publish(FormattedMessage msg, Map vars) { + String requestId = vars != null ? vars.get(correlationVar) : null; + if (requestId == null || requestId.isBlank()) requestId = "NO_REQUEST"; + + Instant now = store.now(); + OutboxMessage stored = new OutboxMessage( + UUID.randomUUID().toString(), + now, + requestId, + outputRef, + msg.getContentType(), + msg.getPayload(), + vars + ); + + + store.append(requestId, stored, maxPerRequest, maxTotal); + } +} \ No newline at end of file diff --git a/src/main/java/at/co/procon/malis/cep/output/OutboxStore.java b/src/main/java/at/co/procon/malis/cep/output/OutboxStore.java new file mode 100644 index 0000000..3677940 --- /dev/null +++ b/src/main/java/at/co/procon/malis/cep/output/OutboxStore.java @@ -0,0 +1,94 @@ +package at.co.procon.malis.cep.output; + + +import org.springframework.stereotype.Component; + + +import java.time.Instant; +import java.util.*; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentLinkedDeque; + + +/** + * Simple in-memory outbox storage. + * + * Stores messages per requestId for later retrieval by REST. + * - maxPerRequest caps each request queue + * - maxTotal caps total stored messages + */ +@Component +public class OutboxStore { + + + private final ConcurrentHashMap> byRequest = new ConcurrentHashMap<>(); + private final Deque globalOrder = new ConcurrentLinkedDeque<>(); + + + public void append(String requestId, OutboxMessage msg, int maxPerRequest, int maxTotal) { + String rid = (requestId == null || requestId.isBlank()) ? "NO_REQUEST" : requestId; + + +// 1) store per request + Deque q = byRequest.computeIfAbsent(rid, k -> new ConcurrentLinkedDeque<>()); + q.addLast(msg); + + +// trim per-request + while (maxPerRequest > 0 && q.size() > maxPerRequest) { + q.pollFirst(); + } + + + // 2) store global order for total limit + globalOrder.addLast(msg); + while (maxTotal > 0 && globalOrder.size() > maxTotal) { + OutboxMessage removed = globalOrder.pollFirst(); + if (removed != null) { + // best-effort removal from per-request queue + Deque qq = byRequest.get(removed.getRequestId()); + if (qq != null) qq.removeIf(m -> Objects.equals(m.getId(), removed.getId())); + } + } + } + + + /** Return a snapshot list (does not remove). */ + public List list(String requestId) { + String rid = (requestId == null || requestId.isBlank()) ? "NO_REQUEST" : requestId; + Deque q = byRequest.get(rid); + if (q == null) return List.of(); + return List.copyOf(q); + } + + + /** Drain (return and clear) messages for a requestId. */ + public List drain(String requestId) { + String rid = (requestId == null || requestId.isBlank()) ? "NO_REQUEST" : requestId; + Deque q = byRequest.remove(rid); + if (q == null) return List.of(); + + + List out = new ArrayList<>(q); + + + // also remove from globalOrder best-effort + Set ids = new HashSet<>(); + for (OutboxMessage m : out) ids.add(m.getId()); + globalOrder.removeIf(m -> ids.contains(m.getId())); + + + return out; + } + + + /** Useful for debugging. */ + public int sizeTotal() { + return globalOrder.size(); + } + + + public Instant now() { + return Instant.now(); + } +} \ No newline at end of file diff --git a/src/main/java/at/co/procon/malis/cep/output/OutboxStoreProvider.java b/src/main/java/at/co/procon/malis/cep/output/OutboxStoreProvider.java new file mode 100644 index 0000000..69730bb --- /dev/null +++ b/src/main/java/at/co/procon/malis/cep/output/OutboxStoreProvider.java @@ -0,0 +1,12 @@ +package at.co.procon.malis.cep.output; + + +/** + * Optional capability interface. + * + * This lets a controller access a publisher's in-memory store without depending + * on the concrete publisher implementation. + */ +public interface OutboxStoreProvider { + OutboxStore getOutboxStore(); +} \ No newline at end of file diff --git a/src/main/java/at/co/procon/malis/cep/output/OutputPublisher.java b/src/main/java/at/co/procon/malis/cep/output/OutputPublisher.java new file mode 100644 index 0000000..ce763cf --- /dev/null +++ b/src/main/java/at/co/procon/malis/cep/output/OutputPublisher.java @@ -0,0 +1,7 @@ +package at.co.procon.malis.cep.output; + +import java.util.Map; + +public interface OutputPublisher { + void publish(FormattedMessage msg, Map vars); +} diff --git a/src/main/java/at/co/procon/malis/cep/output/OutputRegistry.java b/src/main/java/at/co/procon/malis/cep/output/OutputRegistry.java new file mode 100644 index 0000000..a81d05e --- /dev/null +++ b/src/main/java/at/co/procon/malis/cep/output/OutputRegistry.java @@ -0,0 +1,97 @@ +package at.co.procon.malis.cep.output; + +import at.co.procon.malis.cep.config.CepProperties; +import at.co.procon.malis.cep.eebus.EebusSpineXsdValidator; +import at.co.procon.malis.cep.eebus.EebusSpineXsdValidatorBase; +import at.co.procon.malis.cep.transport.mqtt.MqttBrokerRegistry; +import at.co.procon.malis.cep.transport.rabbit.RabbitBrokerRegistry; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.stereotype.Component; + +import java.util.LinkedHashMap; +import java.util.Map; + +/** + * Registry that wires configuration-defined outputs (cep.outputs.*) to runtime publisher instances. + * + * Responsibilities: + * - Hold known formatters (e.g. EEBUS_ALARM_MESSAGE). + * - For each configured output, create an appropriate {@link OutputPublisher}. + * + * Important: Publishers created here should be *thin* wrappers. Heavy connection/session management lives + * in transport modules (Spring Integration MQTT, Spring AMQP). + */ +@Component +public class OutputRegistry { + + private final Map publishers = new LinkedHashMap<>(); + private final Map formatters = new LinkedHashMap<>(); + private final CepProperties props; + private final ObjectMapper objectMapper; + + public OutputRegistry(CepProperties props, + MqttBrokerRegistry mqttBrokers, + RabbitBrokerRegistry rabbitBrokers, + OutboxStore outboxStore, + ObjectMapper objectMapper, + @Qualifier("eebusSpineXsdJaxbValidator") EebusSpineXsdValidatorBase xsdValidator) { + this.props = props; + this.objectMapper = objectMapper; + + formatters.put("EEBUS_ALARM_MESSAGE", new EebusAlarmMessageFormatter()); + formatters.put("EEBUS_ALARM_DATAGRAM_XML", new EebusAlarmDatagramXmlFormatter(xsdValidator)); + formatters.put("MALIS_FAULT_JSON", new MalisFaultJsonFormatter(objectMapper)); + formatters.put("MALIS_CEP_ALARM_JSON", new MalisCepAlarmJsonFormatter(objectMapper)); + + for (var e : props.getOutputs().entrySet()) { + publishers.put(e.getKey(), buildPublisher(e.getKey(), e.getValue(), mqttBrokers, rabbitBrokers, outboxStore)); + } + } + + public OutputPublisher getPublisher(String outputRef) { + OutputPublisher p = publishers.get(outputRef); + if (p == null) throw new IllegalArgumentException("Unknown outputRef: " + outputRef); + return p; + } + + public AlarmFormatter getFormatter(String format) { + AlarmFormatter f = formatters.get(format); + if (f == null) throw new IllegalArgumentException("Unknown format: " + format); + return f; + } + + public CepProperties.OutputDef getOutputDef(String outputRef) { + CepProperties.OutputDef d = props.getOutputs().get(outputRef); + if (d == null) throw new IllegalArgumentException("Unknown outputRef: " + outputRef); + return d; + } + + private OutputPublisher buildPublisher(String id, + CepProperties.OutputDef def, + MqttBrokerRegistry mqttBrokers, + RabbitBrokerRegistry rabbitBrokers, + OutboxStore outboxStore) { + String type = def.getType(); + Map cfg = def.getConfig(); + + return switch (type) { + case "MQTT" -> { + String brokerRef = String.valueOf(cfg.getOrDefault("brokerRef", "primary")); + String topicTemplate = String.valueOf(cfg.get("topicTemplate")); + Integer qos = cfg.containsKey("qos") ? ((Number) cfg.get("qos")).intValue() : null; + Boolean retained = cfg.containsKey("retained") ? Boolean.valueOf(String.valueOf(cfg.get("retained"))) : null; + yield new MqttOutputPublisher(brokerRef, topicTemplate, qos, retained, mqttBrokers); + } + case "RABBIT" -> { + String exchange = String.valueOf(cfg.get("exchange")); + String routingKeyTemplate = String.valueOf(cfg.get("routingKeyTemplate")); + String brokerRef = String.valueOf(cfg.getOrDefault("brokerRef", "rabbit")); + yield new RabbitOutputPublisher(exchange, routingKeyTemplate, rabbitBrokers, brokerRef); + } + case "REST" -> new RestOutputPublisher(id, cfg, objectMapper); + case "OUTBOX" -> new OutboxOutputPublisher(id, outboxStore, cfg); + default -> throw new IllegalArgumentException("Unknown output type: " + type + " (id=" + id + ")"); + }; + } +} \ No newline at end of file diff --git a/src/main/java/at/co/procon/malis/cep/output/PublishedMessage.java b/src/main/java/at/co/procon/malis/cep/output/PublishedMessage.java new file mode 100644 index 0000000..9c72e32 --- /dev/null +++ b/src/main/java/at/co/procon/malis/cep/output/PublishedMessage.java @@ -0,0 +1,92 @@ +// NEW / MODIFIED FILES ONLY +// ========================= +// Goal +// ---- +// 1) Return publisher-stage *formatted output* in the REST response (PipelineResult). +// -> This solves: "aim is to get the result for publisher (formatted event in pipeline result)". +// +// 2) Optional: allow a controller to access a publisher's in-memory store (if it has one) +// without casting to a concrete class. +// -> This solves: "access pipeline publisher store directly if publisher has it". +// +// Namespace prefix: at.co.procon.malis + + +// ============================================================================= +// 1) NEW: src/main/java/at/co/procon/malis/cep/output/PublishedMessage.java +// ============================================================================= + + +package at.co.procon.malis.cep.output; + + +import java.nio.charset.StandardCharsets; +import java.time.Instant; +import java.util.Base64; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.Map; + + +/** + * One formatted message produced by the publisher stage for a specific outputRef. + * + * Why it exists: + * - Your REST ingress call should be able to return the *exact* payload that would be sent + * (e.g., EEBUS Alarm XML) even if the MQTT/Rabbit broker is not reachable. + */ +public class PublishedMessage { + + private final Instant producedAt; + private final String outputRef; + private final String format; + private final String contentType; + private final byte[] payload; + + /** Optional: transport-level error if publish() threw (captured, not fatal for REST tracing). */ + private final String publishError; + + /** Copy of vars at publish-time (handy to debug topic/routingKey templates). */ + private final Map vars; + + public PublishedMessage(Instant producedAt, + String outputRef, + String format, + String contentType, + byte[] payload, + String publishError, + Map vars) { + this.producedAt = producedAt; + this.outputRef = outputRef; + this.format = format; + this.contentType = contentType; + this.payload = payload == null ? new byte[0] : payload.clone(); + this.publishError = publishError; + this.vars = vars == null ? Map.of() : Collections.unmodifiableMap(new LinkedHashMap<>(vars)); + } + + + public Instant getProducedAt() { return producedAt; } + public String getOutputRef() { return outputRef; } + public String getFormat() { return format; } + public String getContentType() { return contentType; } + public byte[] getPayload() { return payload.clone(); } + public String getPublishError() { return publishError; } + public Map getVars() { return vars; } + + + /** Convenience: base64 string of payload. */ + public String getPayloadBase64() { + return Base64.getEncoder().encodeToString(payload); + } + + + /** Convenience: interpret payload as UTF-8 (works for XML). */ + public String getPayloadUtf8() { + try { + return new String(payload, StandardCharsets.UTF_8); + } catch (Exception ignore) { + return null; + } + } +} \ No newline at end of file diff --git a/src/main/java/at/co/procon/malis/cep/output/PublisherService.java b/src/main/java/at/co/procon/malis/cep/output/PublisherService.java new file mode 100644 index 0000000..639d648 --- /dev/null +++ b/src/main/java/at/co/procon/malis/cep/output/PublisherService.java @@ -0,0 +1,201 @@ +package at.co.procon.malis.cep.output; + +import at.co.procon.malis.cep.config.CepProperties; +import at.co.procon.malis.cep.lifecycle.PublishableAlarm; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Component; + +import java.time.Instant; +import java.util.ArrayList; +import java.util.Base64; +import java.util.List; +import java.util.Map; + +/** + * Publishing stage of the CEP pipeline. + * + *

This component is intentionally simple: + *

    + *
  • Pick output(s) defined by the binding
    • + *
  • Format canonical alarms into the configured schema (JSON/XML/...)
    • + *
  • Delegate actual transport to an {@link OutputPublisher} (MQTT/Rabbit/...)
    • + *
+ * + *

Why this separation matters: + *

    + *
  • Formatters are pure transformation functions
    • + *
  • Publishers deal with connectivity and delivery semantics
    • + *
  • Bindings decide which outputs are used for which source
    • + *
+ */ +@Component +public class PublisherService { + + private static final Logger log = LoggerFactory.getLogger(PublisherService.class); + + private final OutputRegistry outputs; + private final CepProperties props; + + public PublisherService(OutputRegistry outputs, CepProperties props) { + this.outputs = outputs; + this.props = props; + } + + /** Backwards-compatible API: publish and ignore collected output. */ + public void publish(CepProperties.SourceBinding binding, Map vars, List alarms) { + publishAndCollect(binding, vars, alarms); + } + + /** + * Publish alarms to outputs and return the *formatted* messages. + * + * REST tracing requirement: + * - Even if the broker is not reachable, you still get the produced XML/JSON. + * - A publish exception is captured into PublishedMessage.publishError. + */ + public List publishAndCollect(CepProperties.SourceBinding binding, + Map vars, + List alarms) { + if (binding == null || binding.getOutputRefs() == null || binding.getOutputRefs().isEmpty()) return List.of(); + if (alarms == null || alarms.isEmpty()) return List.of(); + + List produced = new ArrayList<>(); + + for (String outputRef : binding.getOutputRefs()) { + CepProperties.OutputDef outDef = outputs.getOutputDef(outputRef); + AlarmFormatter formatter = outputs.getFormatter(outDef.getFormat()); + OutputPublisher publisher = outputs.getPublisher(outputRef); + + Map outputConfig = outDef.getConfig() == null ? Map.of() : outDef.getConfig(); + + for (PublishableAlarm a : alarms) { + // 1) Canonical -> formatted bytes + FormattedMessage msg = formatter.format(a, vars, outputConfig); + + // 2) Attempt to publish + String publishError = null; + try { + publisher.publish(msg, vars); + } catch (Exception ex) { + // For REST tracing we do NOT fail the entire call. + publishError = ex.toString(); + } + + // 3) Return what was produced + produced.add(new PublishedMessage( + Instant.now(), + outputRef, + outDef.getFormat(), + msg.getContentType(), + msg.getPayload(), + publishError, + vars + )); + } + } + + return produced; + } + + // --- dead letter unchanged (copied from your current file) --- + + public void publishDeadLetter(DeadLetterEnvelope dlq) { + var dl = props.getDeadLetter(); + if (dl == null || !dl.isEnabled() || dl.getOutputRef() == null) return; + + CepProperties.OutputDef outDef = outputs.getOutputDef(dl.getOutputRef()); + String format = dl.getFormat() != null ? dl.getFormat() : outDef.getFormat(); + AlarmFormatter formatter = outputs.getFormatter(format); + OutputPublisher publisher = outputs.getPublisher(dl.getOutputRef()); + + PublishableAlarm pseudo = new PublishableAlarm( + "DLQ:" + dlq.getId(), + "ALARM", + "ERROR", + dlq.getOccurredAt(), + Map.of( + "reason", dlq.getReason(), + "bindingId", dlq.getBindingId(), + "inType", dlq.getInType(), + "path", dlq.getPathOrTopicOrQueue(), + "headers", dlq.getHeaders(), + "error", dlq.getErrorMessage(), + "payloadBase64", dlq.getPayloadBase64() + ) + ); + + Map vars = Map.of("tenant", "dlq", "site", "dlq", "department", "dlq", "deviceId", "dlq"); + Map outputConfig = outDef.getConfig() == null ? Map.of() : outDef.getConfig(); + + try { + FormattedMessage msg = formatter.format(pseudo, vars, outputConfig); + publisher.publish(msg, vars); + } catch (Exception ex) { + // DLQ must never crash ingress processing. + log.error("Failed to publish dead letter message (outputRef={}). Dropping DLQ message id={}", dl.getOutputRef(), dlq.getId(), ex); + } + } + + public static final class DeadLetterEnvelope { + private final String id; + private final Instant occurredAt; + private final String reason; + private final String bindingId; + private final String inType; + private final String pathOrTopicOrQueue; + private final Map headers; + private final String errorMessage; + private final String payloadBase64; + + public DeadLetterEnvelope(String id, + Instant occurredAt, + String reason, + String bindingId, + String inType, + String pathOrTopicOrQueue, + Map headers, + String errorMessage, + String payloadBase64) { + this.id = id; + this.occurredAt = occurredAt; + this.reason = reason; + this.bindingId = bindingId; + this.inType = inType; + this.pathOrTopicOrQueue = pathOrTopicOrQueue; + this.headers = headers == null ? Map.of() : Map.copyOf(headers); + this.errorMessage = errorMessage; + this.payloadBase64 = payloadBase64; + } + + public String getId() { return id; } + public Instant getOccurredAt() { return occurredAt; } + public String getReason() { return reason; } + public String getBindingId() { return bindingId; } + public String getInType() { return inType; } + public String getPathOrTopicOrQueue() { return pathOrTopicOrQueue; } + public Map getHeaders() { return headers; } + public String getErrorMessage() { return errorMessage; } + public String getPayloadBase64() { return payloadBase64; } + + public static DeadLetterEnvelope from(byte[] payload, + String reason, + String bindingId, + String inType, + String path, + Map headers, + Exception ex) { + return new DeadLetterEnvelope( + java.util.UUID.randomUUID().toString(), + Instant.now(), + reason, + bindingId, + inType, + path, + headers == null ? Map.of() : headers, + ex == null ? null : ex.toString(), + Base64.getEncoder().encodeToString(payload == null ? new byte[0] : payload) + ); + } + } +} \ No newline at end of file diff --git a/src/main/java/at/co/procon/malis/cep/output/RabbitOutputPublisher.java b/src/main/java/at/co/procon/malis/cep/output/RabbitOutputPublisher.java new file mode 100644 index 0000000..6baaca5 --- /dev/null +++ b/src/main/java/at/co/procon/malis/cep/output/RabbitOutputPublisher.java @@ -0,0 +1,40 @@ +package at.co.procon.malis.cep.output; + +import at.co.procon.malis.cep.transport.rabbit.RabbitBrokerRegistry; +import at.co.procon.malis.cep.util.TemplateUtil; +import org.springframework.amqp.core.Message; +import org.springframework.amqp.core.MessageProperties; +import org.springframework.amqp.rabbit.core.RabbitTemplate; + +import java.util.Map; + +/** + * RabbitMQ output publisher (real implementation via Spring AMQP). + * + * The publisher is transport-only: + * - It expands routingKeyTemplate using vars. + * - It publishes the formatted bytes to an exchange. + */ +public class RabbitOutputPublisher implements OutputPublisher { + + private final String exchange; + private final String routingKeyTemplate; + private final RabbitTemplate rabbit; + + public RabbitOutputPublisher(String exchange, String routingKeyTemplate, RabbitBrokerRegistry registry, String brokerRef) { + this.exchange = exchange; + this.routingKeyTemplate = routingKeyTemplate; + this.rabbit = registry.template(brokerRef); + } + + @Override + public void publish(FormattedMessage msg, Map vars) { + String rk = TemplateUtil.expand(routingKeyTemplate, vars); + + MessageProperties mp = new MessageProperties(); + mp.setContentType(msg.contentType()); + Message amqp = new Message(msg.payload(), mp); + + rabbit.send(exchange, rk, amqp); + } +} diff --git a/src/main/java/at/co/procon/malis/cep/output/RestOutputPublisher.java b/src/main/java/at/co/procon/malis/cep/output/RestOutputPublisher.java new file mode 100644 index 0000000..d69aa21 --- /dev/null +++ b/src/main/java/at/co/procon/malis/cep/output/RestOutputPublisher.java @@ -0,0 +1,186 @@ +package at.co.procon.malis.cep.output; + +import at.co.procon.malis.cep.util.TemplateUtil; +import com.fasterxml.jackson.databind.ObjectMapper; + +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.nio.charset.StandardCharsets; +import java.time.Duration; +import java.time.Instant; +import java.util.Base64; +import java.util.LinkedHashMap; +import java.util.Map; + +/** + * REST/HTTP output publisher. + * + *

Thin transport adapter like MQTT/Rabbit publishers: + *

    + *
  • Expands a URL template with variables
    • + *
  • Sends the formatted payload as request body (RAW) OR wraps it into a JSON envelope
    • + *
+ * + *

Config keys (cep.outputs.<id>.config): + *

    + *
  • urlTemplate (required)
    • + *
  • method (optional, default POST)
    • + *
  • timeoutMs (optional, default 5000)
    • + *
  • connectTimeoutMs (optional, default 3000)
    • + *
  • bodyMode: RAW | ENVELOPE_JSON (optional, default RAW)
    • + *
  • headers: {HeaderName: "value ${var}"} (optional)
    • + *
  • bearerTokenTemplate (optional) - sets Authorization: Bearer ...
    • + *
  • basicAuth: {usernameTemplate: "...", passwordTemplate: "..."} (optional)
    • + *
  • maxErrorBodyChars (optional, default 2000)
    • + *
+ */ +public class RestOutputPublisher implements OutputPublisher { + + public enum BodyMode { + RAW, + ENVELOPE_JSON + } + + private final String id; + private final ObjectMapper om; + private final HttpClient client; + + private final String urlTemplate; + private final String method; + private final Duration timeout; + private final BodyMode bodyMode; + private final Map headers; + private final String bearerTokenTemplate; + private final String basicUsernameTemplate; + private final String basicPasswordTemplate; + private final int maxErrorBodyChars; + + @SuppressWarnings("unchecked") + public RestOutputPublisher(String id, Map cfg, ObjectMapper om) { + this.id = id; + this.om = om; + + this.urlTemplate = string(cfg.get("urlTemplate")); + if (this.urlTemplate == null || this.urlTemplate.isBlank()) { + throw new IllegalArgumentException("REST output requires config.urlTemplate (id=" + id + ")"); + } + + this.method = stringOr(cfg.get("method"), "POST").toUpperCase(); + int timeoutMs = intOr(cfg.get("timeoutMs"), 5000); + int connectTimeoutMs = intOr(cfg.get("connectTimeoutMs"), 3000); + this.timeout = Duration.ofMillis(timeoutMs); + + String mode = stringOr(cfg.get("bodyMode"), "RAW").toUpperCase(); + this.bodyMode = BodyMode.valueOf(mode); + + this.headers = toStringMap(cfg.get("headers")); + this.bearerTokenTemplate = string(cfg.get("bearerTokenTemplate")); + + Map basic = cfg.get("basicAuth") instanceof Map m ? (Map) m : null; + this.basicUsernameTemplate = basic == null ? null : string(basic.get("usernameTemplate")); + this.basicPasswordTemplate = basic == null ? null : string(basic.get("passwordTemplate")); + this.maxErrorBodyChars = intOr(cfg.get("maxErrorBodyChars"), 2000); + + this.client = HttpClient.newBuilder() + .connectTimeout(Duration.ofMillis(connectTimeoutMs)) + .build(); + } + + @Override + public void publish(FormattedMessage msg, Map vars) { + try { + String url = TemplateUtil.expand(urlTemplate, vars); + if (url == null || url.isBlank()) { + throw new IllegalArgumentException("Expanded REST url is blank (id=" + id + ")"); + } + + byte[] body; + String contentType; + if (bodyMode == BodyMode.ENVELOPE_JSON) { + Map envelope = new LinkedHashMap<>(); + envelope.put("sentAt", Instant.now().toString()); + envelope.put("publisherId", id); + envelope.put("contentType", msg.contentType()); + envelope.put("payloadBase64", Base64.getEncoder().encodeToString(msg.payload())); + envelope.put("vars", vars == null ? Map.of() : vars); + body = om.writeValueAsBytes(envelope); + contentType = "application/json"; + } else { + body = msg.payload(); + contentType = msg.contentType(); + } + + HttpRequest.Builder b = HttpRequest.newBuilder(URI.create(url)) + .timeout(timeout) + .header("Content-Type", contentType); + + // Custom headers + if (headers != null && !headers.isEmpty()) { + for (var e : headers.entrySet()) { + String k = e.getKey(); + if (k == null || k.isBlank()) continue; + String v = TemplateUtil.expand(e.getValue(), vars); + b.header(k, v == null ? "" : v); + } + } + + // Auth headers (optional) + String bearer = TemplateUtil.expand(bearerTokenTemplate, vars); + if (bearer != null && !bearer.isBlank()) { + b.header("Authorization", "Bearer " + bearer); + } else { + String u = TemplateUtil.expand(basicUsernameTemplate, vars); + String p = TemplateUtil.expand(basicPasswordTemplate, vars); + if (u != null && !u.isBlank()) { + String token = Base64.getEncoder().encodeToString((u + ":" + (p == null ? "" : p)).getBytes(StandardCharsets.UTF_8)); + b.header("Authorization", "Basic " + token); + } + } + + HttpRequest req = b.method(method, HttpRequest.BodyPublishers.ofByteArray(body == null ? new byte[0] : body)) + .build(); + + HttpResponse resp = client.send(req, HttpResponse.BodyHandlers.ofByteArray()); + int sc = resp.statusCode(); + if (sc < 200 || sc >= 300) { + String respBody = new String(resp.body() == null ? new byte[0] : resp.body(), StandardCharsets.UTF_8); + if (respBody.length() > maxErrorBodyChars) respBody = respBody.substring(0, maxErrorBodyChars) + "…"; + throw new RuntimeException("REST publish failed (id=" + id + ") status=" + sc + " url=" + url + " body=" + respBody); + } + } catch (Exception e) { + // Let PublisherService capture the error per-output (publishError) + throw (e instanceof RuntimeException re) ? re : new RuntimeException(e); + } + } + + private static String string(Object o) { + return o == null ? null : String.valueOf(o); + } + + private static String stringOr(Object o, String def) { + String s = string(o); + return (s == null || s.isBlank()) ? def : s; + } + + private static int intOr(Object o, int def) { + if (o == null) return def; + if (o instanceof Number n) return n.intValue(); + try { + return Integer.parseInt(String.valueOf(o).trim()); + } catch (Exception ignore) { + return def; + } + } + + private static Map toStringMap(Object o) { + if (!(o instanceof Map m)) return Map.of(); + Map out = new LinkedHashMap<>(); + for (var e : m.entrySet()) { + if (e.getKey() == null) continue; + out.put(String.valueOf(e.getKey()), e.getValue() == null ? "" : String.valueOf(e.getValue())); + } + return out; + } +} diff --git a/src/main/java/at/co/procon/malis/cep/parser/AlarmBatch.java b/src/main/java/at/co/procon/malis/cep/parser/AlarmBatch.java new file mode 100644 index 0000000..4a9f5f3 --- /dev/null +++ b/src/main/java/at/co/procon/malis/cep/parser/AlarmBatch.java @@ -0,0 +1,24 @@ +package at.co.procon.malis.cep.parser; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +/** + * Container for multiple alarm events in a single ingress payload. + * + *

EEBUS alarm datagrams can contain multiple elements, so we parse them into a batch. + */ +public final class AlarmBatch { + + private final List alarms; + + public AlarmBatch(List alarms) { + this.alarms = alarms == null ? List.of() : Collections.unmodifiableList(new ArrayList<>(alarms)); + } + + public List getAlarms() { return alarms; } + + // record-style accessor + public List alarms() { return alarms; } +} diff --git a/src/main/java/at/co/procon/malis/cep/parser/AlarmInput.java b/src/main/java/at/co/procon/malis/cep/parser/AlarmInput.java new file mode 100644 index 0000000..568e306 --- /dev/null +++ b/src/main/java/at/co/procon/malis/cep/parser/AlarmInput.java @@ -0,0 +1,41 @@ +package at.co.procon.malis.cep.parser; + +import java.time.Instant; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.Map; + +/** + * Canonical alarm event produced by parsers. + * + * IMPORTANT: + * - External detectors return EVENTS ONLY. State is maintained by the lifecycle service. + * - To preserve incoming EEBUS details, we carry a 'details' map along the pipeline. + */ +public class AlarmInput { + private final String deviceId; + private final String alarmCode; + private final String action; // ALARM|CANCEL + private final Instant timestamp; + private final Map details; + + public AlarmInput(String deviceId, String alarmCode, String action, Instant timestamp) { + this(deviceId, alarmCode, action, timestamp, Map.of()); + } + + public AlarmInput(String deviceId, String alarmCode, String action, Instant timestamp, Map details) { + this.deviceId = deviceId; + this.alarmCode = alarmCode; + this.action = action; + this.timestamp = timestamp; + this.details = details == null + ? Map.of() + : Collections.unmodifiableMap(new LinkedHashMap<>(details)); + } + + public String getDeviceId() { return deviceId; } + public String getAlarmCode() { return alarmCode; } + public String getAction() { return action; } + public Instant getTimestamp() { return timestamp; } + public Map getDetails() { return details; } +} diff --git a/src/main/java/at/co/procon/malis/cep/parser/EebusAlarmDatagramXmlParser.java b/src/main/java/at/co/procon/malis/cep/parser/EebusAlarmDatagramXmlParser.java new file mode 100644 index 0000000..5b1cac7 --- /dev/null +++ b/src/main/java/at/co/procon/malis/cep/parser/EebusAlarmDatagramXmlParser.java @@ -0,0 +1,162 @@ +package at.co.procon.malis.cep.parser; + +import at.co.procon.malis.cep.binding.IngressMessage; +import at.co.procon.malis.cep.binding.ResolvedBinding; + +import at.co.procon.malis.cep.eebus.EebusSpineXsdValidatorBase; +import at.co.procon.malis.cep.eebus.EebusXmlUtil; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.stereotype.Component; +import org.w3c.dom.Document; +import org.w3c.dom.Node; +import org.w3c.dom.NodeList; + +import javax.xml.xpath.XPathConstants; +import javax.xml.xpath.XPathFactory; +import java.time.Instant; +import java.util.*; + +/** + * Parses a full SPINE v1 XSD-style that contains . + * + * Behavior: + * - Produces AlarmBatch (one AlarmInput per ) + * - Derives action: + * alarmType == alarmCancelled -> CANCEL + * otherwise -> ALARM + * - Uses scopeType as alarmCode (preferred), else alarmId, else alarmType + * - Preserves rich alarm fields by storing them in AlarmInput.details + * - Parses alarmData timestamp as Instant when it is an absolute time (xs:dateTime). + * If timestamp is a duration (xs:duration), stores it as details.timestampRaw and uses Instant.now(). + */ +@Component +public class EebusAlarmDatagramXmlParser implements EventParser { + + private final EebusSpineXsdValidatorBase xsdValidator; + + public EebusAlarmDatagramXmlParser(@Qualifier("eebusSpineXsdJaxbValidator") EebusSpineXsdValidatorBase xsdValidator) { + this.xsdValidator = xsdValidator; + } + + @Override + public ParsedInput parse(IngressMessage msg, ResolvedBinding rb) throws Exception { + byte[] xmlBytes = msg.getPayload(); + + if (xsdValidator != null) { + xsdValidator.validateDatagram(xmlBytes, "alarm ingress: " + msg.getPath()); + } + + Document doc = EebusXmlUtil.parse(xmlBytes); + + Map vars = new LinkedHashMap<>(rb.getVars()); + + String deviceId = vars.get("deviceId"); + if (deviceId == null || deviceId.isBlank()) { + String hdrDev = EebusXmlUtil.text(doc, "//*[local-name()='header']/*[local-name()='addressSource']/*[local-name()='device']"); + if (hdrDev != null && !hdrDev.isBlank()) { + deviceId = hdrDev; + vars.put("deviceId", hdrDev); + } + } + + NodeList alarmNodes = (NodeList) XPathFactory.newInstance().newXPath() + .evaluate("//*[local-name()='payload']/*[local-name()='cmd']/*[local-name()='alarmListData']/*[local-name()='alarmData']", + doc, XPathConstants.NODESET); + + List alarms = new ArrayList<>(); + + for (int i = 0; i < alarmNodes.getLength(); i++) { + Node alarmData = alarmNodes.item(i); + + Map details = new LinkedHashMap<>(); + details.put("sourceFormat", "EEBUS_SPINE_V1_DATAGRAM"); + + String alarmId = childText(alarmData, "alarmId"); + String thresholdId = childText(alarmData, "thresholdId"); + String tsRaw = childText(alarmData, "timestamp"); + String alarmType = childText(alarmData, "alarmType"); + String scopeType = childText(alarmData, "scopeType"); + String label = childText(alarmData, "label"); + String description = childText(alarmData, "description"); + + String mvNumber = childTextPath(alarmData, "measuredValue", "number"); + String mvScale = childTextPath(alarmData, "measuredValue", "scale"); + String epStart = childTextPath(alarmData, "evaluationPeriod", "startTime"); + String epEnd = childTextPath(alarmData, "evaluationPeriod", "endTime"); + + if (alarmId != null) details.put("alarmId", alarmId); + if (thresholdId != null) details.put("thresholdId", thresholdId); + if (alarmType != null) details.put("alarmType", alarmType); + if (scopeType != null) details.put("scopeType", scopeType); + if (label != null) details.put("label", label); + if (description != null) details.put("description", description); + + if (mvNumber != null) details.put("measuredValue.number", mvNumber); + if (mvScale != null) details.put("measuredValue.scale", mvScale); + + if (epStart != null) details.put("evaluationPeriod.startTime", epStart); + if (epEnd != null) details.put("evaluationPeriod.endTime", epEnd); + + Instant occurredAt = parseInstantOrNow(tsRaw, details); + + String action = (alarmType != null && "alarmCancelled".equalsIgnoreCase(alarmType.trim())) ? "CANCEL" : "ALARM"; + String alarmCode = firstNonBlank(scopeType, alarmId, alarmType, "UNKNOWN"); + + alarms.add(new AlarmInput(deviceId, alarmCode, action, occurredAt, details)); + } + + return new ParsedInput("EEBUS_ALARM_DATAGRAM_XML", new AlarmBatch(alarms), Instant.now(), vars); + } + + private static Instant parseInstantOrNow(String tsRaw, Map details) { + if (tsRaw == null || tsRaw.isBlank()) return Instant.now(); + try { + Instant parsed = Instant.parse(tsRaw.trim()); + details.put("timestampRaw", tsRaw.trim()); + return parsed; + } catch (Exception ignore) { + details.put("timestampRaw", tsRaw.trim()); + details.put("timestampFormat", "DURATION_OR_NON_INSTANT"); + return Instant.now(); + } + } + + private static String firstNonBlank(String... values) { + if (values == null) return null; + for (String v : values) { + if (v != null && !v.isBlank()) return v; + } + return null; + } + + private static String childText(Node parent, String childLocalName) { + if (parent == null || childLocalName == null) return null; + NodeList kids = parent.getChildNodes(); + for (int i = 0; i < kids.getLength(); i++) { + Node n = kids.item(i); + if (n.getNodeType() != Node.ELEMENT_NODE) continue; + if (childLocalName.equalsIgnoreCase(n.getLocalName())) { + String t = n.getTextContent(); + if (t == null) return null; + String s = t.trim(); + return s.isBlank() ? null : s; + } + } + return null; + } + + private static String childTextPath(Node parent, String outerLocalName, String innerLocalName) { + Node outer = null; + NodeList kids = parent.getChildNodes(); + for (int i = 0; i < kids.getLength(); i++) { + Node n = kids.item(i); + if (n.getNodeType() != Node.ELEMENT_NODE) continue; + if (outerLocalName.equalsIgnoreCase(n.getLocalName())) { + outer = n; + break; + } + } + if (outer == null) return null; + return childText(outer, innerLocalName); + } +} \ No newline at end of file diff --git a/src/main/java/at/co/procon/malis/cep/parser/EebusAlarmJsonParser.java b/src/main/java/at/co/procon/malis/cep/parser/EebusAlarmJsonParser.java new file mode 100644 index 0000000..cc42bb0 --- /dev/null +++ b/src/main/java/at/co/procon/malis/cep/parser/EebusAlarmJsonParser.java @@ -0,0 +1,38 @@ +package at.co.procon.malis.cep.parser; + +import at.co.procon.malis.cep.binding.IngressMessage; +import at.co.procon.malis.cep.binding.ResolvedBinding; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; + +import java.time.Instant; + +/** + * Parser: EEBUS_ALARM_JSON + * + *

This is a lightweight JSON shape for alarms (useful in early integration / testing). + * It is not the official EEBUS XML model. + * + *

Supported fields: + *

    + *
  • deviceId (string) optional
    • + *
  • alarmCode (string) optional
    • + *
  • action (ALARM|CANCEL) optional
    • + *
+ */ +public class EebusAlarmJsonParser implements EventParser { + + private final ObjectMapper om = new ObjectMapper(); + + @Override + public ParsedInput parse(IngressMessage msg, ResolvedBinding rb) throws Exception { + JsonNode root = om.readTree(msg.getPayload()); + + String deviceId = root.hasNonNull("deviceId") ? root.get("deviceId").asText() : rb.getVars().get("deviceId"); + String alarmCode = root.hasNonNull("alarmCode") ? root.get("alarmCode").asText() : "UNKNOWN"; + String action = root.hasNonNull("action") ? root.get("action").asText() : "ALARM"; + + AlarmInput ai = new AlarmInput(deviceId, alarmCode, action.toUpperCase(), Instant.now()); + return new ParsedInput("EEBUS_ALARM_JSON", ai, Instant.now(), rb.getVars()); + } +} diff --git a/src/main/java/at/co/procon/malis/cep/parser/EebusHeaderMeta.java b/src/main/java/at/co/procon/malis/cep/parser/EebusHeaderMeta.java new file mode 100644 index 0000000..bfd833e --- /dev/null +++ b/src/main/java/at/co/procon/malis/cep/parser/EebusHeaderMeta.java @@ -0,0 +1,37 @@ +// ============================================================ +// 3) Parser updates for your sample datagram/alarmListData +// ============================================================ + +package at.co.procon.malis.cep.parser; + +import at.co.procon.malis.cep.eebus.EebusAddress; + +/** + * Extra EEBUS metadata we often want for routing / correlation. + */ +public class EebusHeaderMeta { + private EebusAddress addressSource; + private EebusAddress addressDestination; + private String specificationVersion; + private String cmdClassifier; + private String msgCounter; + private String timestamp; + + public EebusAddress getAddressSource() { return addressSource; } + public void setAddressSource(EebusAddress addressSource) { this.addressSource = addressSource; } + + public EebusAddress getAddressDestination() { return addressDestination; } + public void setAddressDestination(EebusAddress addressDestination) { this.addressDestination = addressDestination; } + + public String getSpecificationVersion() { return specificationVersion; } + public void setSpecificationVersion(String specificationVersion) { this.specificationVersion = specificationVersion; } + + public String getCmdClassifier() { return cmdClassifier; } + public void setCmdClassifier(String cmdClassifier) { this.cmdClassifier = cmdClassifier; } + + public String getMsgCounter() { return msgCounter; } + public void setMsgCounter(String msgCounter) { this.msgCounter = msgCounter; } + + public String getTimestamp() { return timestamp; } + public void setTimestamp(String timestamp) { this.timestamp = timestamp; } +} \ No newline at end of file diff --git a/src/main/java/at/co/procon/malis/cep/parser/EebusMeasurementDatagramBody.java b/src/main/java/at/co/procon/malis/cep/parser/EebusMeasurementDatagramBody.java new file mode 100644 index 0000000..db12587 --- /dev/null +++ b/src/main/java/at/co/procon/malis/cep/parser/EebusMeasurementDatagramBody.java @@ -0,0 +1,28 @@ +package at.co.procon.malis.cep.parser; + +/** + * Parsed body for an EEBUS measurement datagram. + * + * We preserve the full datagram XML so external detectors can receive it unchanged (DATAGRAM_XML mode). + */ +public class EebusMeasurementDatagramBody { + private final String datagramXml; + private final String measurementDescriptionListDataXml; + private final String measurementListDataXml; + private final EebusHeaderMeta header; + + public EebusMeasurementDatagramBody(String datagramXml, + String measurementDescriptionListDataXml, + String measurementListDataXml, + EebusHeaderMeta header) { + this.datagramXml = datagramXml; + this.measurementDescriptionListDataXml = measurementDescriptionListDataXml; + this.measurementListDataXml = measurementListDataXml; + this.header = header; + } + + public String getDatagramXml() { return datagramXml; } + public String getMeasurementDescriptionListDataXml() { return measurementDescriptionListDataXml; } + public String getMeasurementListDataXml() { return measurementListDataXml; } + public EebusHeaderMeta getHeader() { return header; } +} diff --git a/src/main/java/at/co/procon/malis/cep/parser/EebusMeasurementDatagramXmlParser.java b/src/main/java/at/co/procon/malis/cep/parser/EebusMeasurementDatagramXmlParser.java new file mode 100644 index 0000000..5530a45 --- /dev/null +++ b/src/main/java/at/co/procon/malis/cep/parser/EebusMeasurementDatagramXmlParser.java @@ -0,0 +1,150 @@ +package at.co.procon.malis.cep.parser; + +import at.co.procon.malis.cep.binding.IngressMessage; +import at.co.procon.malis.cep.binding.ResolvedBinding; +import at.co.procon.malis.cep.eebus.EebusMeasurementDatagram; +import at.co.procon.malis.cep.eebus.EebusSpineXsdValidator; +import at.co.procon.malis.cep.eebus.EebusSpineXsdValidatorBase; +import at.co.procon.malis.cep.eebus.EebusXmlUtil; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Qualifier; +import org.w3c.dom.Document; +import org.w3c.dom.Element; +import org.w3c.dom.Node; +import org.w3c.dom.NodeList; + +import javax.xml.xpath.XPathConstants; +import javax.xml.xpath.XPathFactory; +import java.nio.charset.StandardCharsets; +import java.time.Instant; + +/** + * Parses a full SPINE v1 containing measurementDescriptionListData and/or measurementListData. + * + * If a binding did not extract a deviceId from topic/path, you can choose to derive it from + * header.addressSource.device by enabling config "deviceIdFromHeader=true" in the parser def. + */ +public class EebusMeasurementDatagramXmlParser implements EventParser { + + private static final Logger log = LoggerFactory.getLogger(EebusMeasurementDatagramXmlParser.class); + + private final EebusSpineXsdValidatorBase xsdValidator; + + public EebusMeasurementDatagramXmlParser(@Qualifier("eebusSpineXsdJaxbValidator") EebusSpineXsdValidatorBase xsdValidator) { + this.xsdValidator = xsdValidator; + } + + @Override + public ParsedInput parse(IngressMessage msg, ResolvedBinding rb) throws Exception { + byte[] xmlBytes = msg.getPayload(); + + Document doc = EebusXmlUtil.parse(xmlBytes); + + boolean modified = normalizeMeasurementCmdsIfNeeded(doc); + if (modified) { + xmlBytes = EebusXmlUtil.documentToBytes(doc); + doc = EebusXmlUtil.parse(xmlBytes); + } + + if (xsdValidator != null) { + xsdValidator.validateDatagram(xmlBytes, "measurement ingress: " + msg.getPath()); + } + + String sourceDevice = EebusXmlUtil.text(doc, "//*[local-name()='header']/*[local-name()='addressSource']/*[local-name()='device']"); + String destDevice = EebusXmlUtil.text(doc, "//*[local-name()='header']/*[local-name()='addressDestination']/*[local-name()='device']"); + + String mdld = EebusXmlUtil.extractFirstFragmentXml(doc, "measurementDescriptionListData"); + String mld = EebusXmlUtil.extractFirstFragmentXml(doc, "measurementListData"); + + String full = new String(xmlBytes, StandardCharsets.UTF_8); + EebusMeasurementDatagram datagram = new EebusMeasurementDatagram(full, mdld, mld, sourceDevice, destDevice); + + return new ParsedInput("EEBUS_MEASUREMENT_DATAGRAM_XML", datagram, Instant.now(), rb.getVars()); + } + + static boolean normalizeMeasurementCmdsIfNeeded(Document doc) { + try { + NodeList cmdNodes = (NodeList) XPathFactory.newInstance().newXPath() + .evaluate("//*[local-name()='payload']/*[local-name()='cmd']", doc, XPathConstants.NODESET); + + boolean modified = false; + + for (int i = 0; i < cmdNodes.getLength(); i++) { + Element cmd = (Element) cmdNodes.item(i); + + Element mdld = firstChildElementByLocalName(cmd, "measurementDescriptionListData"); + Element mld = firstChildElementByLocalName(cmd, "measurementListData"); + + if (mdld != null && mld != null) { + Element cmd1 = (Element) cmd.cloneNode(true); + Element cmd2 = (Element) cmd.cloneNode(true); + + removeChildElementByLocalName(cmd1, "measurementListData"); + removeChildElementByLocalName(cmd2, "measurementDescriptionListData"); + + ensureFunction(cmd1, "measurementDescriptionListData"); + ensureFunction(cmd2, "measurementListData"); + + Node parent = cmd.getParentNode(); + parent.insertBefore(cmd1, cmd); + parent.insertBefore(cmd2, cmd); + parent.removeChild(cmd); + + modified = true; + } + } + + if (modified) { + log.info("Normalized measurement datagram: split mixed measurementDescriptionListData + measurementListData into separate elements"); + } + + return modified; + } catch (Exception ex) { + log.warn("Failed to normalize measurement datagram: {}", ex.getMessage()); + return false; + } + } + + private static void ensureFunction(Element cmd, String functionName) { + Element function = firstChildElementByLocalName(cmd, "function"); + if (function == null) { + function = cmd.getOwnerDocument().createElementNS(cmd.getNamespaceURI(), cmd.getPrefix() + ":function"); + function.setTextContent(functionName); + + Node first = firstElementChild(cmd); + if (first != null) cmd.insertBefore(function, first); + else cmd.appendChild(function); + } else { + function.setTextContent(functionName); + } + } + + private static Node firstElementChild(Element parent) { + NodeList kids = parent.getChildNodes(); + for (int i = 0; i < kids.getLength(); i++) { + Node n = kids.item(i); + if (n.getNodeType() == Node.ELEMENT_NODE) return n; + } + return null; + } + + private static Element firstChildElementByLocalName(Element parent, String local) { + NodeList kids = parent.getChildNodes(); + for (int i = 0; i < kids.getLength(); i++) { + Node n = kids.item(i); + if (n.getNodeType() != Node.ELEMENT_NODE) continue; + if (local.equalsIgnoreCase(n.getLocalName())) return (Element) n; + } + return null; + } + + private static void removeChildElementByLocalName(Element parent, String local) { + NodeList kids = parent.getChildNodes(); + for (int i = kids.getLength() - 1; i >= 0; i--) { + Node n = kids.item(i); + if (n.getNodeType() != Node.ELEMENT_NODE) continue; + if (local.equalsIgnoreCase(n.getLocalName())) parent.removeChild(n); + } + } +} \ No newline at end of file diff --git a/src/main/java/at/co/procon/malis/cep/parser/EventParser.java b/src/main/java/at/co/procon/malis/cep/parser/EventParser.java new file mode 100644 index 0000000..4ad94db --- /dev/null +++ b/src/main/java/at/co/procon/malis/cep/parser/EventParser.java @@ -0,0 +1,20 @@ +package at.co.procon.malis.cep.parser; + +import at.co.procon.malis.cep.binding.IngressMessage; +import at.co.procon.malis.cep.binding.ResolvedBinding; + +/** + * Parses a raw transport message into a canonical internal model. + * + *

Parsers are intentionally stateless and deterministic. All configuration should be + * injected via the parser constructor, created by {@link ParserFactory} from configuration. + */ +public interface EventParser { + + /** + * @param msg raw ingress message (topic/queue/path + payload) + * @param rb resolved binding (includes extracted vars like tenant/site/deviceId) + * @return parsed canonical object wrapped in {@link ParsedInput} + */ + ParsedInput parse(IngressMessage msg, ResolvedBinding rb) throws Exception; +} diff --git a/src/main/java/at/co/procon/malis/cep/parser/JsonPortsBooleanParser.java b/src/main/java/at/co/procon/malis/cep/parser/JsonPortsBooleanParser.java new file mode 100644 index 0000000..6e99c0d --- /dev/null +++ b/src/main/java/at/co/procon/malis/cep/parser/JsonPortsBooleanParser.java @@ -0,0 +1,50 @@ +package at.co.procon.malis.cep.parser; + +import at.co.procon.malis.cep.binding.IngressMessage; +import at.co.procon.malis.cep.binding.ResolvedBinding; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; + +import java.time.Instant; +import java.util.ArrayList; +import java.util.List; + +/** + * Parser: JSON_PORTS_BOOLEAN + * + *

Expects payload like: {@code {"port1":true, ..., "port8":false}} + * Produces a {@link MeasurementSet} with paths like {@code port.1.state}. + */ +public class JsonPortsBooleanParser implements EventParser { + + private final int portCount; + private final String keyPattern; + private final String measurementPathPattern; + private final ObjectMapper om = new ObjectMapper(); + + public JsonPortsBooleanParser(int portCount, String keyPattern, String measurementPathPattern) { + this.portCount = portCount; + this.keyPattern = keyPattern; + this.measurementPathPattern = measurementPathPattern; + } + + @Override + public ParsedInput parse(IngressMessage msg, ResolvedBinding rb) throws Exception { + JsonNode root = om.readTree(msg.getPayload()); + + List values = new ArrayList<>(); + for (int i = 1; i <= portCount; i++) { + String key = keyPattern.replace("{n}", String.valueOf(i)); + JsonNode v = root.get(key); + if (v == null || v.isNull()) continue; + if (!v.isBoolean()) throw new IllegalArgumentException("Expected boolean for key '" + key + "'"); + + String path = measurementPathPattern.replace("{n}", String.valueOf(i)); + values.add(new MeasurementValue(path, v.booleanValue())); + } + + String deviceId = rb.getVars().get("deviceId"); + MeasurementSet ms = new MeasurementSet(deviceId, Instant.now(), values); + return new ParsedInput("MEASUREMENT_SET", ms, Instant.now(), rb.getVars()); + } +} diff --git a/src/main/java/at/co/procon/malis/cep/parser/M2mPinUpdate.java b/src/main/java/at/co/procon/malis/cep/parser/M2mPinUpdate.java new file mode 100644 index 0000000..fd757ee --- /dev/null +++ b/src/main/java/at/co/procon/malis/cep/parser/M2mPinUpdate.java @@ -0,0 +1,48 @@ +package at.co.procon.malis.cep.parser; + +import java.time.Instant; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.Map; + +/** + * Canonical in-memory model for the M2M GPIO pin messages. + * + *

Example payloads are provided in the user story: + * - reason=value_changed + * - reason=periodic_update + */ +public final class M2mPinUpdate { + + private final String gpio; + private final String channel; + private final Integer value; + private final Instant timestamp; + private final String reason; + private final String m2mport; + private final Map mqtt; + + public M2mPinUpdate(String gpio, + String channel, + Integer value, + Instant timestamp, + String reason, + String m2mport, + Map mqtt) { + this.gpio = gpio; + this.channel = channel; + this.value = value; + this.timestamp = timestamp; + this.reason = reason; + this.m2mport = m2mport; + this.mqtt = mqtt == null ? Map.of() : Collections.unmodifiableMap(new LinkedHashMap<>(mqtt)); + } + + public String getGpio() { return gpio; } + public String getChannel() { return channel; } + public Integer getValue() { return value; } + public Instant getTimestamp() { return timestamp; } + public String getReason() { return reason; } + public String getM2mport() { return m2mport; } + public Map getMqtt() { return mqtt; } +} diff --git a/src/main/java/at/co/procon/malis/cep/parser/M2mPinUpdateJsonErpnextParser.java b/src/main/java/at/co/procon/malis/cep/parser/M2mPinUpdateJsonErpnextParser.java new file mode 100644 index 0000000..367cadc --- /dev/null +++ b/src/main/java/at/co/procon/malis/cep/parser/M2mPinUpdateJsonErpnextParser.java @@ -0,0 +1,125 @@ +package at.co.procon.malis.cep.parser; + +import at.co.procon.malis.cep.binding.IngressMessage; +import at.co.procon.malis.cep.binding.ResolvedBinding; +import at.co.procon.malis.cep.erpnext.ErpnextFaultLookupClient; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; + +import java.time.*; +import java.util.LinkedHashMap; +import java.util.Map; + +/** + * Parser type: M2M_PIN_UPDATE_JSON_ERPNEXT + * + *

Reads M2M JSON payloads and enriches them by resolving (port,pin) -> + * Site/Department/PinName/Bad-state via ERPNext. + */ +public class M2mPinUpdateJsonErpnextParser implements EventParser { + + private final ObjectMapper om = new ObjectMapper(); + private final ErpnextFaultLookupClient erp; + + public M2mPinUpdateJsonErpnextParser(ErpnextFaultLookupClient erp) { + this.erp = erp; + } + + @Override + public ParsedInput parse(IngressMessage msg, ResolvedBinding rb) throws Exception { + JsonNode root = om.readTree(msg.getPayload()); + + String gpio = text(root, "gpio"); + String channel = text(root, "channel"); + Integer value = intOrNull(root, "value"); + String reason = text(root, "reason"); + String m2mport = text(root, "m2mport"); + + Instant ts = instantOrNull(root, "timestamp"); + if (ts == null) { + // Fallback to mqtt.sentAt if provided + JsonNode mqtt = root.get("mqtt"); + if (mqtt != null && mqtt.isObject()) { + ts = instantOrNull(mqtt, "sentAt"); + } + } + if (ts == null) ts = Instant.now(); + + Map mqttObj = Map.of(); + JsonNode mqtt = root.get("mqtt"); + if (mqtt != null && mqtt.isObject()) { + mqttObj = om.convertValue(mqtt, Map.class); + } + + M2mPinUpdate update = new M2mPinUpdate(gpio, channel, value, ts, reason, m2mport, mqttObj); + + // Enrich with ERPNext: port + pin from topic vars + Map vars = new LinkedHashMap<>(); + if (rb.getVars() != null) vars.putAll(rb.getVars()); + + String port = vars.getOrDefault("deviceId", null); + vars.put("port", port); + + String pinStr = vars.getOrDefault("pin", null); + Integer pin = null; + if (pinStr != null && !pinStr.isBlank()) { + try { pin = Integer.parseInt(pinStr.trim()); } catch (Exception ignore) { pin = null; } + } + + if (erp != null && port != null && !port.isBlank() && pin != null) { + OffsetDateTime perDatum = OffsetDateTime.ofInstant(ts, ZoneOffset.UTC); + ErpnextFaultLookupClient.FaultLookupContext ctx = erp.lookupFaultByPortAndPinFast(port, pin, perDatum); + + vars.put("iotDevice", ctx.iotDevice); + vars.put("deviceId", ctx.iotDevice); // keep compatibility with existing outputs/datagrams + vars.put("site", ctx.siteId); + if (ctx.remark != null) vars.put("remark", ctx.remark); + if (ctx.department != null) vars.put("department", ctx.department); + if (ctx.pinName != null) vars.put("pinName", ctx.pinName); + if (ctx.description != null) vars.put("pinDescription", ctx.description); + vars.put("pinBad", String.valueOf(ctx.bad)); + } + + // Always expose payload hint vars + if (value != null) vars.put("pinValue", String.valueOf(value)); + if (reason != null) vars.put("pinReason", reason); + if (gpio != null) vars.put("gpio", gpio); + if (channel != null) vars.put("channel", channel); + if (m2mport != null) vars.put("m2mport", m2mport); + + return new ParsedInput("M2M_PIN_UPDATE", update, ts, vars); + } + + private static String text(JsonNode n, String field) { + if (n == null) return null; + JsonNode v = n.get(field); + if (v == null || v.isNull()) return null; + String t = v.asText(null); + if (t == null) return null; + t = t.trim(); + return t.isEmpty() ? null : t; + } + + private static Integer intOrNull(JsonNode n, String field) { + if (n == null) return null; + JsonNode v = n.get(field); + if (v == null || v.isNull()) return null; + if (v.isNumber()) return v.asInt(); + String s = v.asText(null); + if (s == null) return null; + s = s.trim(); + if (s.isEmpty()) return null; + try { return Integer.parseInt(s); } catch (Exception ignore) { return null; } + } + + private static Instant instantOrNull(JsonNode n, String field) { + String s = text(n, field); + if (s == null) return null; + try { + return Instant.parse(s); + } catch (Exception ignore) { + // best effort + try { return OffsetDateTime.parse(s).toInstant(); } catch (Exception ignore2) { return null; } + } + } +} diff --git a/src/main/java/at/co/procon/malis/cep/parser/MeasurementSet.java b/src/main/java/at/co/procon/malis/cep/parser/MeasurementSet.java new file mode 100644 index 0000000..af4979e --- /dev/null +++ b/src/main/java/at/co/procon/malis/cep/parser/MeasurementSet.java @@ -0,0 +1,34 @@ +package at.co.procon.malis.cep.parser; + +import java.time.Instant; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +/** + * Canonical internal representation of measurements. + * + *

In early phases (memory-only pipeline) we use this as a simple, transport-agnostic event container. + * Later, you may replace or supplement it with Esper/Kafka-stream-friendly schemas. + */ +public final class MeasurementSet { + + private final String deviceId; + private final Instant timestamp; + private final List measurements; + + public MeasurementSet(String deviceId, Instant timestamp, List measurements) { + this.deviceId = deviceId; + this.timestamp = timestamp == null ? Instant.now() : timestamp; + this.measurements = measurements == null ? List.of() : Collections.unmodifiableList(new ArrayList<>(measurements)); + } + + public String getDeviceId() { return deviceId; } + public Instant getTimestamp() { return timestamp; } + public List getMeasurements() { return measurements; } + + // record-style accessors + public String deviceId() { return deviceId; } + public Instant timestamp() { return timestamp; } + public List measurements() { return measurements; } +} diff --git a/src/main/java/at/co/procon/malis/cep/parser/MeasurementValue.java b/src/main/java/at/co/procon/malis/cep/parser/MeasurementValue.java new file mode 100644 index 0000000..0a93a8c --- /dev/null +++ b/src/main/java/at/co/procon/malis/cep/parser/MeasurementValue.java @@ -0,0 +1,29 @@ +package at.co.procon.malis.cep.parser; + +/** + * Single measurement value. + * + *

This class is intentionally simple so it can represent: + *

    + *
  • boolean port states
    • + *
  • numeric sensor readings
    • + *
  • strings for status values
    • + *
+ */ +public final class MeasurementValue { + + private final String path; + private final Object value; + + public MeasurementValue(String path, Object value) { + this.path = path; + this.value = value; + } + + public String getPath() { return path; } + public Object getValue() { return value; } + + // record-style accessors + public String path() { return path; } + public Object value() { return value; } +} diff --git a/src/main/java/at/co/procon/malis/cep/parser/ParsedInput.java b/src/main/java/at/co/procon/malis/cep/parser/ParsedInput.java new file mode 100644 index 0000000..c562c08 --- /dev/null +++ b/src/main/java/at/co/procon/malis/cep/parser/ParsedInput.java @@ -0,0 +1,43 @@ +package at.co.procon.malis.cep.parser; + +import java.time.Instant; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.Map; + +/** + * Result of parsing a raw ingress message. + * + *

{@code body} is the canonical internal representation that detectors understand. + * Examples: + *

    + *
  • {@link MeasurementSet} (simple boolean/number measurements)
    • + *
  • {@link AlarmInput} or {@link AlarmBatch} (pre-made alarm events from upstream)
    • + *
  • {@link at.co.procon.malis.cep.eebus.EebusMeasurementDatagram} (full EEBUS measurement datagram + extracted fragments)
    • + *
+ */ +public final class ParsedInput { + + private final String payloadType; + private final Object body; + private final Instant receivedAt; + private final Map vars; + + public ParsedInput(String payloadType, Object body, Instant receivedAt, Map vars) { + this.payloadType = payloadType; + this.body = body; + this.receivedAt = receivedAt == null ? Instant.now() : receivedAt; + this.vars = vars == null ? Map.of() : Collections.unmodifiableMap(new LinkedHashMap<>(vars)); + } + + public String getPayloadType() { return payloadType; } + public Object getBody() { return body; } + public Instant getReceivedAt() { return receivedAt; } + public Map getVars() { return vars; } + + // Optional record-style accessors (useful during migration) + public String payloadType() { return payloadType; } + public Object body() { return body; } + public Instant receivedAt() { return receivedAt; } + public Map vars() { return vars; } +} diff --git a/src/main/java/at/co/procon/malis/cep/parser/ParserFactory.java b/src/main/java/at/co/procon/malis/cep/parser/ParserFactory.java new file mode 100644 index 0000000..b0ca947 --- /dev/null +++ b/src/main/java/at/co/procon/malis/cep/parser/ParserFactory.java @@ -0,0 +1,172 @@ +package at.co.procon.malis.cep.parser; + +import at.co.procon.malis.cep.config.CepProperties; +import at.co.procon.malis.cep.erpnext.ErpnextFaultLookupClient; +import at.co.procon.malis.cep.eebus.EebusSpineXsdValidator; +import at.co.procon.malis.cep.eebus.EebusSpineXsdValidatorBase; + +import java.time.Duration; +import java.util.LinkedHashMap; +import java.util.Locale; +import java.util.Map; + +public class ParserFactory { + + public static Map buildAll(Map defs, EebusSpineXsdValidatorBase xsdValidator) { + Map out = new LinkedHashMap<>(); + for (var e : defs.entrySet()) { + out.put(e.getKey(), buildOne(e.getKey(), e.getValue(), xsdValidator)); + } + return out; + } + + public static EventParser buildOne(String id, CepProperties.ParserDef def, EebusSpineXsdValidatorBase xsdValidator) { + String type = def.getType(); + Map cfg = def.getConfig(); + + // Note: parser IDs are configured in YAML (cep.parsers.). The pipeline references them via binding.parserRef. + return switch (type) { + case "JSON_PORTS_BOOLEAN" -> { + int portCount = ((Number) cfg.getOrDefault("portCount", 8)).intValue(); + String keyPattern = String.valueOf(cfg.getOrDefault("keyPattern", "port{n}")); + String pathPattern = String.valueOf(cfg.getOrDefault("measurementPathPattern", "port.{n}.state")); + yield new JsonPortsBooleanParser(portCount, keyPattern, pathPattern); + } + case "EEBUS_ALARM_JSON" -> new EebusAlarmJsonParser(); + case "EEBUS_ALARM_DATAGRAM_XML" -> new EebusAlarmDatagramXmlParser(xsdValidator); + case "EEBUS_MEASUREMENT_DATAGRAM_XML" -> new EebusMeasurementDatagramXmlParser(xsdValidator); + case "SMART_EEBUS_ALARM" -> new SmartEebusAlarmParser( + new EebusAlarmDatagramXmlParser(xsdValidator), + new EebusAlarmJsonParser() + ); + case "SMART_PORTS_OR_EEBUS_MEASUREMENT" -> { + // During migration, a single topic may carry either ports-JSON or EEBUS measurement datagrams. + // This parser chooses by the first non-whitespace byte. + int portCount = ((Number) cfg.getOrDefault("portCount", 8)).intValue(); + String keyPattern = String.valueOf(cfg.getOrDefault("keyPattern", "port{n}")); + String pathPattern = String.valueOf(cfg.getOrDefault("measurementPathPattern", "port.{n}.state")); + yield new SmartPortsOrEebusMeasurementParser( + new EebusMeasurementDatagramXmlParser(xsdValidator), + new JsonPortsBooleanParser(portCount, keyPattern, pathPattern) + ); + } + case "M2M_PIN_UPDATE_JSON_ERPNEXT" -> { + // Expected config: + // config: + // erpnext: + // baseUrl: ... + // apiKey: ... + // apiSecret: ... + // timeoutMs: 5000 + // doctypes: { ... optional overrides ... } + // fields: { ... optional overrides ... } + // cache: + // ttlSeconds: 600 + // maxSize: 20000 + @SuppressWarnings("unchecked") + Map erpCfg = cfg.get("erpnext") instanceof Map ? (Map) cfg.get("erpnext") : Map.of(); + @SuppressWarnings("unchecked") + Map cacheCfg = cfg.get("cache") instanceof Map ? (Map) cfg.get("cache") : Map.of(); + @SuppressWarnings("unchecked") + Map preloadCfg = cfg.get("preload") instanceof Map ? (Map) cfg.get("preload") : Map.of(); + + String baseUrl = asString(erpCfg.get("baseUrl")); + String apiKey = asString(erpCfg.get("apiKey")); + String apiSecret = asString(erpCfg.get("apiSecret")); + int timeoutMs = asInt(erpCfg.get("timeoutMs"), 5000); + + @SuppressWarnings("unchecked") + Map doctypes = erpCfg.get("doctypes") instanceof Map ? (Map) erpCfg.get("doctypes") : Map.of(); + @SuppressWarnings("unchecked") + Map fields = erpCfg.get("fields") instanceof Map ? (Map) erpCfg.get("fields") : Map.of(); + + long ttlSeconds = asLong(cacheCfg.get("ttlSeconds"), 600L); + int maxSize = asInt(cacheCfg.get("maxSize"), 20_000); + + boolean preloadEnabled = asBoolean(preloadCfg.get("enabled"), true); + boolean preloadOnStartup = asBoolean(preloadCfg.get("onStartup"), true); + long refreshSeconds = asLong(preloadCfg.get("refreshSeconds"), 600L); + long initialDelaySeconds = asLong(preloadCfg.get("initialDelaySeconds"), 5L); + boolean allowOnDemandFallback = asBoolean(preloadCfg.get("allowOnDemandFallback"), true); + + var clientCfg = new ErpnextFaultLookupClient.Config( + baseUrl, + apiKey, + apiSecret, + timeoutMs, + asString(doctypes.get("iotDevice")), + asString(doctypes.get("site")), + asString(doctypes.get("siteIotDevice")), + asString(doctypes.get("malisConfig")), + asString(doctypes.get("malisConfigDetail")), + asString(fields.get("iotDevicePortAddress")), + asString(fields.get("siteIotDevicesAssigned")), + asString(fields.get("siteIotDeviceIotDevice")), + asString(fields.get("siteIotDeviceMalis")), + asString(fields.get("siteIotDeviceRemark")), + asString(fields.get("siteIotDeviceFrom")), + asString(fields.get("siteIotDeviceTo")), + asString(fields.get("malisConfigNameField")), + asString(fields.get("malisConfigDetailsField")), + asString(fields.get("malisDetailPin")), + asString(fields.get("malisDetailStream")), + asString(fields.get("malisDetailStreamValue")), + asString(fields.get("malisDetailPinName")), + asString(fields.get("malisDetailDescription")), + asString(fields.get("malisDetailBad")), + asString(fields.get("malisDetailDepartment")), + Duration.ofSeconds(Math.max(0, ttlSeconds)), + maxSize, + preloadEnabled, + preloadOnStartup, + Duration.ofSeconds(Math.max(1, refreshSeconds)), + Duration.ofSeconds(Math.max(0, initialDelaySeconds)), + allowOnDemandFallback + ); + + ErpnextFaultLookupClient erp = new ErpnextFaultLookupClient(clientCfg); + yield new M2mPinUpdateJsonErpnextParser(erp); + } + default -> throw new IllegalArgumentException("Unknown parser type: " + type + " (id=" + id + ")"); + }; + } + + private static String asString(Object o) { + if (o == null) return null; + String s = String.valueOf(o); + s = s.trim(); + return s.isEmpty() ? null : s; + } + + private static int asInt(Object o, int def) { + if (o == null) return def; + if (o instanceof Number n) return n.intValue(); + try { return Integer.parseInt(String.valueOf(o).trim()); } catch (Exception e) { return def; } + } + + private static long asLong(Object o, long def) { + if (o == null) return def; + if (o instanceof Number n) return n.longValue(); + try { return Long.parseLong(String.valueOf(o).trim()); } catch (Exception e) { return def; } + } + + private static boolean asBoolean(Object o, boolean def) { + if (o == null) return def; + if (o instanceof Boolean b) return b; + String s = String.valueOf(o).trim().toLowerCase(Locale.ROOT); + if (s.isEmpty()) return def; + if (s.equals("true") || s.equals("1") || s.equals("yes") || s.equals("y")) return true; + if (s.equals("false") || s.equals("0") || s.equals("no") || s.equals("n")) return false; + return def; + } + + public static Map build(CepProperties props, EebusSpineXsdValidatorBase xsdValidator) { + Map map = new LinkedHashMap<>(); + if (props.getParsers() == null) return map; + + for (var e : props.getParsers().entrySet()) { + map.put(e.getKey(), buildOne(e.getKey(), e.getValue(), xsdValidator)); + } + return map; + } +} diff --git a/src/main/java/at/co/procon/malis/cep/parser/ParserRegistry.java b/src/main/java/at/co/procon/malis/cep/parser/ParserRegistry.java new file mode 100644 index 0000000..549162c --- /dev/null +++ b/src/main/java/at/co/procon/malis/cep/parser/ParserRegistry.java @@ -0,0 +1,27 @@ +package at.co.procon.malis.cep.parser; + +import at.co.procon.malis.cep.config.CepProperties; +import at.co.procon.malis.cep.eebus.EebusSpineXsdValidator; +import at.co.procon.malis.cep.eebus.EebusSpineXsdValidatorBase; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.stereotype.Component; + +import java.util.Map; + + +@Component +public class ParserRegistry { + + private final Map parsers; + + public ParserRegistry(CepProperties props, @Qualifier("eebusSpineXsdJaxbValidator") EebusSpineXsdValidatorBase xsdValidator) { + this.parsers = ParserFactory.build(props, xsdValidator); + } + + public EventParser get(String parserRef) { + EventParser p = parsers.get(parserRef); + if (p == null) throw new IllegalArgumentException("Unknown parserRef: " + parserRef); + return p; + } +} + diff --git a/src/main/java/at/co/procon/malis/cep/parser/SmartEebusAlarmParser.java b/src/main/java/at/co/procon/malis/cep/parser/SmartEebusAlarmParser.java new file mode 100644 index 0000000..be74cf7 --- /dev/null +++ b/src/main/java/at/co/procon/malis/cep/parser/SmartEebusAlarmParser.java @@ -0,0 +1,37 @@ +package at.co.procon.malis.cep.parser; + +import at.co.procon.malis.cep.binding.IngressMessage; +import at.co.procon.malis.cep.binding.ResolvedBinding; + + +/** + * Parser: SMART_EEBUS_ALARM + * + *

Chooses between XML and JSON alarm parsers based on the first non-whitespace byte. + * + *

This is used when a single MQTT topic may carry alarms in different encodings during a migration. + */ +public class SmartEebusAlarmParser implements EventParser { + + private final EventParser xml; + private final EventParser json; + + public SmartEebusAlarmParser(EventParser xml, EventParser json) { + this.xml = xml; + this.json = json; + } + + @Override + public ParsedInput parse(IngressMessage msg, ResolvedBinding rb) throws Exception { + byte[] p = msg.getPayload(); + int i = 0; + while (i < p.length && Character.isWhitespace((char) p[i])) i++; + + if (i < p.length && p[i] == '<') { + return xml.parse(msg, rb); + } + + // Default: JSON + return json.parse(msg, rb); + } +} diff --git a/src/main/java/at/co/procon/malis/cep/parser/SmartPortsOrEebusMeasurementParser.java b/src/main/java/at/co/procon/malis/cep/parser/SmartPortsOrEebusMeasurementParser.java new file mode 100644 index 0000000..0d39952 --- /dev/null +++ b/src/main/java/at/co/procon/malis/cep/parser/SmartPortsOrEebusMeasurementParser.java @@ -0,0 +1,40 @@ +package at.co.procon.malis.cep.parser; + +import at.co.procon.malis.cep.binding.IngressMessage; +import at.co.procon.malis.cep.binding.ResolvedBinding; + +/** + * Parser: SMART_PORTS_OR_EEBUS_MEASUREMENT + * + *

For scenario 3 we may receive either: + *

    + *
  • EEBUS measurement datagram XML (... + *
  • simple JSON with boolean ports ("port1":true, ...)
    • + *
+ * on the SAME MQTT topic. + * + *

We keep binding resolution deterministic (topic->binding), but allow the parser to choose between formats. + */ +public class SmartPortsOrEebusMeasurementParser implements EventParser { + + private final EventParser eebusMeasurementXmlParser; + private final EventParser jsonPortsParser; + + public SmartPortsOrEebusMeasurementParser(EventParser eebusMeasurementXmlParser, EventParser jsonPortsParser) { + this.eebusMeasurementXmlParser = eebusMeasurementXmlParser; + this.jsonPortsParser = jsonPortsParser; + } + + @Override + public ParsedInput parse(IngressMessage msg, ResolvedBinding rb) throws Exception { + byte[] p = msg.getPayload(); + int i = 0; + while (i < p.length && Character.isWhitespace((char) p[i])) i++; + + if (i < p.length && p[i] == '<') { + return eebusMeasurementXmlParser.parse(msg, rb); + } + + return jsonPortsParser.parse(msg, rb); + } +} diff --git a/src/main/java/at/co/procon/malis/cep/parser/sss.java b/src/main/java/at/co/procon/malis/cep/parser/sss.java new file mode 100644 index 0000000..e69de29 diff --git a/src/main/java/at/co/procon/malis/cep/pipeline/CepPipeline.java b/src/main/java/at/co/procon/malis/cep/pipeline/CepPipeline.java new file mode 100644 index 0000000..cc98ccc --- /dev/null +++ b/src/main/java/at/co/procon/malis/cep/pipeline/CepPipeline.java @@ -0,0 +1,211 @@ +package at.co.procon.malis.cep.pipeline; + +import at.co.procon.malis.cep.binding.BindingResolver; +import at.co.procon.malis.cep.binding.IngressMessage; +import at.co.procon.malis.cep.binding.ResolvedBinding; +import at.co.procon.malis.cep.detector.DetectionContext; +import at.co.procon.malis.cep.detector.DetectorEvent; +import at.co.procon.malis.cep.detector.DetectorRegistry; +import at.co.procon.malis.cep.lifecycle.LifecycleDispatcherService; +import at.co.procon.malis.cep.lifecycle.PublishableAlarm; +import at.co.procon.malis.cep.output.PublishedMessage; +import at.co.procon.malis.cep.output.PublisherService; +import at.co.procon.malis.cep.parser.ParserRegistry; +import at.co.procon.malis.cep.processor.ProcessorRegistry; +import org.springframework.stereotype.Service; + +import java.util.*; +/** + * The main in-memory CEP pipeline. + * + *

High-level stages: + *

    + *
  1. Binding resolution (topic/path -> which pipeline config to apply)
    2. + *
  2. Parsing (raw bytes -> canonical internal model)
    3. + *
  3. Detection (canonical model -> detector events)
    4. + *
  4. Lifecycle/state (dedup/debounce/open/close)
    5. + *
  5. Formatting + publishing to configured outputs
    6. + *
+ */ +@Service +public class CepPipeline { + + public static final class PipelineResult { + private final String bindingId; + private final Map vars; + private final List detectorEvents; + private final List publishableAlarms; + + /** NEW: formatted messages produced by the publisher stage. */ + private final List publishedMessages; + + public PipelineResult(String bindingId, + Map vars, + List detectorEvents, + List publishableAlarms, + List publishedMessages) { + this.bindingId = bindingId; + this.vars = vars == null ? Map.of() : Collections.unmodifiableMap(new LinkedHashMap<>(vars)); + this.detectorEvents = detectorEvents == null ? List.of() : List.copyOf(detectorEvents); + this.publishableAlarms = publishableAlarms == null ? List.of() : List.copyOf(publishableAlarms); + this.publishedMessages = publishedMessages == null ? List.of() : List.copyOf(publishedMessages); + } + + public String getBindingId() { return bindingId; } + public Map getVars() { return vars; } + public List getDetectorEvents() { return detectorEvents; } + public List getPublishableAlarms() { return publishableAlarms; } + public List getPublishedMessages() { return publishedMessages; } + } + + private final BindingResolver bindingResolver; + private final ParserRegistry parserRegistry; + private final DetectorRegistry detectorRegistry; + private final ProcessorRegistry processorRegistry; + private final LifecycleDispatcherService lifecycle; + private final PublisherService publisher; + + public CepPipeline(BindingResolver bindingResolver, + ParserRegistry parserRegistry, + DetectorRegistry detectorRegistry, + ProcessorRegistry processorRegistry, + LifecycleDispatcherService lifecycle, + PublisherService publisher) { + this.bindingResolver = bindingResolver; + this.parserRegistry = parserRegistry; + this.detectorRegistry = detectorRegistry; + this.processorRegistry = processorRegistry; + this.lifecycle = lifecycle; + this.publisher = publisher; + } + + public PipelineResult process(IngressMessage msg) { + ResolvedBinding rb = null; + + try { + var rbOpt = bindingResolver.resolve(msg); + if (rbOpt.isEmpty()) { + publisher.publishDeadLetter(PublisherService.DeadLetterEnvelope.from( + msg.getPayload(), + "NO_BINDING_MATCH", + null, + msg.getInType().name(), + msg.getPath(), + msg.getHeaders(), + null + )); + return new PipelineResult(null, Map.of(), List.of(), List.of(), List.of()); + } + + rb = rbOpt.get(); + + // Copy vars so we can enrich them for this specific request. + Map vars = new LinkedHashMap<>(rb.getVars()); + + // Make transport metadata available for templates (outputs and composite lifecycle). + if (msg.getPath() != null && !msg.getPath().isBlank()) { + vars.putIfAbsent("path", msg.getPath()); + vars.putIfAbsent("topic", msg.getPath()); + } + if (msg.getInType() != null) vars.putIfAbsent("inType", msg.getInType().name()); + + // Partition key for per-source stateful detectors (e.g., EXPRESSION_RULES). + // Uses binding + transport metadata to avoid collisions when deviceId is missing + // or not globally unique. + vars.putIfAbsent("sourceKey", buildSourceKey(msg, rb.getBindingId())); + if (msg.getHeaders() != null) { + String rid = msg.getHeaders().get("requestId"); + if (rid == null) rid = msg.getHeaders().get("x-request-id"); + if (rid != null && !rid.isBlank()) vars.put("requestId", rid); + } + + List detectorEvents; + + // Option 3: a processor replaces parser+detector in one step. + if (rb.getBinding().getProcessorRef() != null && !rb.getBinding().getProcessorRef().isBlank()) { + var proc = processorRegistry.get(rb.getBinding().getProcessorRef()); + var res = proc.process(msg, rb, vars); + if (res != null && res.isDrop()) { + return new PipelineResult(rb.getBindingId(), vars, List.of(), List.of(), List.of()); + } + if (res != null && res.getVarsDelta() != null) { + for (var ve : res.getVarsDelta().entrySet()) { + if (ve.getKey() == null) continue; + if (!vars.containsKey(ve.getKey()) && ve.getValue() != null) { + vars.put(ve.getKey(), ve.getValue()); + } + } + } + detectorEvents = res == null ? List.of() : res.getEvents(); + } else { + // Classic flow: parser -> detector + var parser = parserRegistry.get(rb.getBinding().getParserRef()); + var parsed = parser.parse(msg, rb); + + // Allow parsers to enrich vars for downstream stages (detectors, lifecycles, outputs). + // Binding vars take precedence; parser adds missing context (e.g., site/department/function/pinName/...). + if (parsed != null && parsed.getVars() != null) { + for (var ve : parsed.getVars().entrySet()) { + if (ve.getKey() == null) continue; + if (!vars.containsKey(ve.getKey()) && ve.getValue() != null) { + vars.put(ve.getKey(), ve.getValue()); + } + } + } + + var detector = detectorRegistry.get(rb.getBinding().getDetectorRef()); + detectorEvents = detector.detect(parsed, new DetectionContext(rb.getBindingId(), vars)); + } + + var publishable = lifecycle.apply(rb.getBindingId(), rb.getBinding(), vars, detectorEvents); + + // NEW: collect the formatted payloads. + List produced = publisher.publishAndCollect(rb.getBinding(), vars, publishable); + + return new PipelineResult(rb.getBindingId(), vars, detectorEvents, publishable, produced); + + } catch (Exception ex) { + publisher.publishDeadLetter(PublisherService.DeadLetterEnvelope.from( + msg.getPayload(), + "PIPELINE_ERROR", + rb != null ? rb.getBindingId() : null, + msg.getInType().name(), + msg.getPath(), + msg.getHeaders(), + ex + )); + throw new RuntimeException("Pipeline failed" + (rb != null ? (" for binding " + rb.getBindingId()) : ""), ex); + } + } + + /** + * Builds a stable partition key from binding + transport metadata. + * + *

We keep this separate from "deviceId" because deviceId may be missing, derived, + * or not guaranteed globally unique across different ingress transports. + */ + private static String buildSourceKey(IngressMessage msg, String bindingId) { + String inType = msg.getInType() == null ? "UNKNOWN" : msg.getInType().name(); + String b = bindingId == null ? "unknown-binding" : bindingId; + String p = msg.getPath() == null ? "" : msg.getPath(); + + String raw = inType + ":" + b + ":" + p; + // Avoid unbounded key growth (very long topics/paths). If long, hash the path. + if (raw.length() > 240) { + raw = inType + ":" + b + ":" + sha256Hex(p); + } + return raw; + } + + private static String sha256Hex(String s) { + try { + var md = java.security.MessageDigest.getInstance("SHA-256"); + byte[] digest = md.digest(s.getBytes(java.nio.charset.StandardCharsets.UTF_8)); + StringBuilder sb = new StringBuilder(digest.length * 2); + for (byte b : digest) sb.append(String.format("%02x", b)); + return sb.toString(); + } catch (Exception e) { + throw new RuntimeException(e); + } + } +} \ No newline at end of file diff --git a/src/main/java/at/co/procon/malis/cep/processor/EventProcessor.java b/src/main/java/at/co/procon/malis/cep/processor/EventProcessor.java new file mode 100644 index 0000000..40c16ec --- /dev/null +++ b/src/main/java/at/co/procon/malis/cep/processor/EventProcessor.java @@ -0,0 +1,17 @@ +package at.co.procon.malis.cep.processor; + +import at.co.procon.malis.cep.binding.IngressMessage; +import at.co.procon.malis.cep.binding.ResolvedBinding; + +import java.util.Map; + +/** + * A processor combines parsing + detection in one step. + * + *

It receives the raw ingress message plus already-resolved binding vars, + * and returns detector events plus optional enriched vars. + */ +public interface EventProcessor { + + ProcessorResult process(IngressMessage msg, ResolvedBinding rb, Map vars) throws Exception; +} diff --git a/src/main/java/at/co/procon/malis/cep/processor/ProcessorFactory.java b/src/main/java/at/co/procon/malis/cep/processor/ProcessorFactory.java new file mode 100644 index 0000000..4365e5f --- /dev/null +++ b/src/main/java/at/co/procon/malis/cep/processor/ProcessorFactory.java @@ -0,0 +1,147 @@ +package at.co.procon.malis.cep.processor; + +import at.co.procon.malis.cep.config.CepProperties; +import at.co.procon.malis.cep.erpnext.ErpnextFaultLookupClient; +import at.co.procon.malis.cep.processor.js.ErpJsFacade; +import at.co.procon.malis.cep.processor.js.JavaScriptEventProcessor; +import at.co.procon.malis.cep.processor.js.TemplateJsFacade; + +import java.time.Duration; +import java.util.LinkedHashMap; +import java.util.Locale; +import java.util.Map; + +public final class ProcessorFactory { + + private ProcessorFactory() {} + + public static Map build(CepProperties props) { + Map map = new LinkedHashMap<>(); + if (props.getProcessors() == null) return map; + + for (var e : props.getProcessors().entrySet()) { + map.put(e.getKey(), buildOne(e.getKey(), e.getValue())); + } + return map; + } + + private static EventProcessor buildOne(String id, CepProperties.ProcessorDef def) { + if (def == null) throw new IllegalArgumentException("Null processor def (id=" + id + ")"); + String type = def.getType() == null ? "" : def.getType().trim(); + Map cfg = def.getConfig() == null ? Map.of() : def.getConfig(); + + return switch (type) { + case "JAVASCRIPT_PROCESSOR" -> { + // Script config + String scriptPath = asString(cfg.get("scriptPath")); + if (scriptPath == null) { + throw new IllegalArgumentException("JAVASCRIPT_PROCESSOR requires config.scriptPath (id=" + id + ")"); + } + String entryFunction = asString(cfg.get("entryFunction")); + if (entryFunction == null) entryFunction = "process"; + boolean hotReload = asBoolean(cfg.get("hotReload"), true); + long timeoutMs = asLong(cfg.get("timeoutMs"), 0L); + + // Optional ERPNext facade (recommended for scripts) + ErpnextFaultLookupClient erpClient = null; + if (cfg.get("erpnext") instanceof Map erpCfgRaw) { + @SuppressWarnings("unchecked") + Map erpCfg = (Map) erpCfgRaw; + @SuppressWarnings("unchecked") + Map cacheCfg = erpCfg.get("cache") instanceof Map ? (Map) erpCfg.get("cache") : Map.of(); + @SuppressWarnings("unchecked") + Map doctypes = erpCfg.get("doctypes") instanceof Map ? (Map) erpCfg.get("doctypes") : Map.of(); + @SuppressWarnings("unchecked") + Map fields = erpCfg.get("fields") instanceof Map ? (Map) erpCfg.get("fields") : Map.of(); + @SuppressWarnings("unchecked") + Map preloadCfg = erpCfg.get("preload") instanceof Map ? (Map) erpCfg.get("preload") : Map.of(); + + String baseUrl = asString(erpCfg.get("baseUrl")); + String apiKey = asString(erpCfg.get("apiKey")); + String apiSecret = asString(erpCfg.get("apiSecret")); + int timeoutMsErp = asInt(erpCfg.get("timeoutMs"), 5000); + + long ttlSeconds = asLong(cacheCfg.get("ttlSeconds"), 600L); + int maxSize = asInt(cacheCfg.get("maxSize"), 20_000); + + boolean preloadEnabled = asBoolean(preloadCfg.get("enabled"), true); + boolean preloadOnStartup = asBoolean(preloadCfg.get("onStartup"), true); + long refreshSeconds = asLong(preloadCfg.get("refreshSeconds"), 600L); + long initialDelaySeconds = asLong(preloadCfg.get("initialDelaySeconds"), 5L); + boolean allowOnDemandFallback = asBoolean(preloadCfg.get("allowOnDemandFallback"), true); + + var clientCfg = new ErpnextFaultLookupClient.Config( + baseUrl, + apiKey, + apiSecret, + timeoutMsErp, + asString(doctypes.get("iotDevice")), + asString(doctypes.get("site")), + asString(doctypes.get("siteIotDevice")), + asString(doctypes.get("malisConfig")), + asString(doctypes.get("malisConfigDetail")), + asString(fields.get("iotDevicePortAddress")), + asString(fields.get("siteIotDevicesAssigned")), + asString(fields.get("siteIotDeviceIotDevice")), + asString(fields.get("siteIotDeviceMalis")), + asString(fields.get("siteIotDeviceRemark")), + asString(fields.get("siteIotDeviceFrom")), + asString(fields.get("siteIotDeviceTo")), + asString(fields.get("malisConfigNameField")), + asString(fields.get("malisConfigDetailsField")), + asString(fields.get("malisDetailPin")), + asString(fields.get("malisDetailStream")), + asString(fields.get("malisDetailStreamValue")), + asString(fields.get("malisDetailPinName")), + asString(fields.get("malisDetailDescription")), + asString(fields.get("malisDetailBad")), + asString(fields.get("malisDetailDepartment")), + Duration.ofSeconds(Math.max(0, ttlSeconds)), + maxSize, + preloadEnabled, + preloadOnStartup, + Duration.ofSeconds(Math.max(1, refreshSeconds)), + Duration.ofSeconds(Math.max(0, initialDelaySeconds)), + allowOnDemandFallback + ); + + erpClient = new ErpnextFaultLookupClient(clientCfg); + } + + ErpJsFacade erpFacade = erpClient == null ? null : new ErpJsFacade(erpClient); + TemplateJsFacade tplFacade = new TemplateJsFacade(); + + yield new JavaScriptEventProcessor(id, cfg, scriptPath, entryFunction, hotReload, timeoutMs, erpFacade, tplFacade); + } + default -> throw new IllegalArgumentException("Unknown processor type: " + type + " (id=" + id + ")"); + }; + } + + private static String asString(Object o) { + if (o == null) return null; + String s = String.valueOf(o).trim(); + return s.isEmpty() ? null : s; + } + + private static int asInt(Object o, int def) { + if (o == null) return def; + if (o instanceof Number n) return n.intValue(); + try { return Integer.parseInt(String.valueOf(o).trim()); } catch (Exception e) { return def; } + } + + private static long asLong(Object o, long def) { + if (o == null) return def; + if (o instanceof Number n) return n.longValue(); + try { return Long.parseLong(String.valueOf(o).trim()); } catch (Exception e) { return def; } + } + + private static boolean asBoolean(Object o, boolean def) { + if (o == null) return def; + if (o instanceof Boolean b) return b; + String s = String.valueOf(o).trim().toLowerCase(Locale.ROOT); + if (s.isEmpty()) return def; + if (s.equals("true") || s.equals("1") || s.equals("yes") || s.equals("y")) return true; + if (s.equals("false") || s.equals("0") || s.equals("no") || s.equals("n")) return false; + return def; + } +} diff --git a/src/main/java/at/co/procon/malis/cep/processor/ProcessorRegistry.java b/src/main/java/at/co/procon/malis/cep/processor/ProcessorRegistry.java new file mode 100644 index 0000000..8566277 --- /dev/null +++ b/src/main/java/at/co/procon/malis/cep/processor/ProcessorRegistry.java @@ -0,0 +1,22 @@ +package at.co.procon.malis.cep.processor; + +import at.co.procon.malis.cep.config.CepProperties; +import org.springframework.stereotype.Component; + +import java.util.Map; + +@Component +public class ProcessorRegistry { + + private final Map processors; + + public ProcessorRegistry(CepProperties props) { + this.processors = ProcessorFactory.build(props); + } + + public EventProcessor get(String processorRef) { + EventProcessor p = processors.get(processorRef); + if (p == null) throw new IllegalArgumentException("Unknown processorRef: " + processorRef); + return p; + } +} diff --git a/src/main/java/at/co/procon/malis/cep/processor/ProcessorResult.java b/src/main/java/at/co/procon/malis/cep/processor/ProcessorResult.java new file mode 100644 index 0000000..df24be1 --- /dev/null +++ b/src/main/java/at/co/procon/malis/cep/processor/ProcessorResult.java @@ -0,0 +1,42 @@ +package at.co.procon.malis.cep.processor; + +import at.co.procon.malis.cep.detector.DetectorEvent; + +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +/** Result from a processor invocation. */ +public final class ProcessorResult { + + private final boolean drop; + private final Map varsDelta; + private final List events; + + public ProcessorResult(boolean drop, Map varsDelta, List events) { + this.drop = drop; + this.varsDelta = varsDelta == null ? Map.of() : Collections.unmodifiableMap(new LinkedHashMap<>(varsDelta)); + this.events = events == null ? List.of() : List.copyOf(events); + } + + public static ProcessorResult drop() { + return new ProcessorResult(true, Map.of(), List.of()); + } + + public static ProcessorResult of(Map varsDelta, List events) { + return new ProcessorResult(false, varsDelta, events); + } + + public boolean isDrop() { + return drop; + } + + public Map getVarsDelta() { + return varsDelta; + } + + public List getEvents() { + return events; + } +} diff --git a/src/main/java/at/co/procon/malis/cep/processor/js/ErpJsFacade.java b/src/main/java/at/co/procon/malis/cep/processor/js/ErpJsFacade.java new file mode 100644 index 0000000..b74bec6 --- /dev/null +++ b/src/main/java/at/co/procon/malis/cep/processor/js/ErpJsFacade.java @@ -0,0 +1,90 @@ +package at.co.procon.malis.cep.processor.js; + +import at.co.procon.malis.cep.erpnext.ErpnextFaultLookupClient; +import org.graalvm.polyglot.HostAccess; +import org.graalvm.polyglot.proxy.ProxyObject; + +import java.time.Instant; +import java.time.OffsetDateTime; +import java.time.ZoneOffset; +import java.util.LinkedHashMap; +import java.util.Map; + +/** + * Safe JS host facade exposing ERPNext fault lookup. + * + *

Uses the fast/preloaded lookup where possible. + */ +public final class ErpJsFacade { + + private final ErpnextFaultLookupClient client; + + public ErpJsFacade(ErpnextFaultLookupClient client) { + this.client = client; + } + + /** + * Lookup fault context for (port,pin) at a given timestamp. + * + * @param port port/m2mport string + * @param pin pin number (string or number) + * @param isoTimestamp ISO-8601 timestamp (Instant or OffsetDateTime); if null uses now + * @return JS-friendly map or null when config missing + */ + @HostAccess.Export + public ProxyObject lookup(String port, Object pin, String isoTimestamp) { + if (client == null) return null; + if (port == null || port.trim().isEmpty()) return null; + + Integer p; + try { + if (pin instanceof Number n) p = n.intValue(); + else p = Integer.parseInt(String.valueOf(pin).trim()); + } catch (Exception e) { + return null; + } + + Instant at = parseInstant(isoTimestamp); + if (at == null) at = Instant.now(); + + var ctx = client.lookupFaultByPortAndPinFast(port.trim(), p, OffsetDateTime.ofInstant(at, ZoneOffset.UTC)); + if (ctx == null) return null; + + Map m = new LinkedHashMap<>(); + m.put("port", ctx.port); + m.put("iotDevice", ctx.iotDevice); + m.put("deviceId", ctx.iotDevice); + m.put("siteId", ctx.siteId); + m.put("remark", ctx.remark); + m.put("department", ctx.department); + m.put("pinName", ctx.pinName); + m.put("description", ctx.description); + m.put("bad", ctx.bad); + + // IMPORTANT: return ProxyObject so JS can do erpCtx.siteId and JSON.stringify(...) + return ProxyObject.fromMap(m); + } + + private static Instant parseInstant(String s) { + if (s == null) return null; + String t = s.trim(); + if (t.isEmpty()) return null; + try { return Instant.parse(t); } catch (Exception ignore) {} + try { return OffsetDateTime.parse(t).toInstant(); } catch (Exception ignore) {} + return null; + } + + private static Map toStringKeyMap(Map map) { + if (map == null || map.isEmpty()) return Map.of(); + Map out = new LinkedHashMap<>(); + for (var e : map.entrySet()) { + if (e.getKey() == null) continue; + out.put(String.valueOf(e.getKey()), e.getValue()); + } + return out; + } + + private static ProxyObject toProxyObject(Map map) { + return ProxyObject.fromMap(toStringKeyMap(map)); + } +} diff --git a/src/main/java/at/co/procon/malis/cep/processor/js/JavaScriptEventProcessor.java b/src/main/java/at/co/procon/malis/cep/processor/js/JavaScriptEventProcessor.java new file mode 100644 index 0000000..0fa8802 --- /dev/null +++ b/src/main/java/at/co/procon/malis/cep/processor/js/JavaScriptEventProcessor.java @@ -0,0 +1,354 @@ +package at.co.procon.malis.cep.processor.js; + +import at.co.procon.malis.cep.binding.IngressMessage; +import at.co.procon.malis.cep.binding.ResolvedBinding; +import at.co.procon.malis.cep.detector.DetectorEvent; +import at.co.procon.malis.cep.processor.EventProcessor; +import at.co.procon.malis.cep.processor.ProcessorResult; +import org.graalvm.polyglot.*; +import org.graalvm.polyglot.proxy.ProxyObject; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.Instant; +import java.time.OffsetDateTime; +import java.util.*; +import java.util.concurrent.ArrayBlockingQueue; + +/** + * Processor type: JAVASCRIPT_PROCESSOR + * + *

Runs a JS function (entryFunction) that returns: + * 

+ * { drop?: boolean, vars?: {k:v}, events?: [ { action, faultKey, severity?, occurredAt?, details? } ] }
+ *
+ */ +public final class JavaScriptEventProcessor implements EventProcessor { + + private static final Logger log = LoggerFactory.getLogger(JavaScriptEventProcessor.class); + + private final String id; + private final Map processorConfig; + private final ProxyObject processorConfigProxy; + private final Path scriptPath; + private final String entryFunction; + private final boolean hotReload; + private final long timeoutMs; // currently unused (reserved) + private final ErpJsFacade erp; + private final TemplateJsFacade tpl; + + private final Engine engine; + private final ArrayBlockingQueue pool; + + private volatile long lastLoadedMtime = -1L; + private volatile String lastLoadedText; + + public JavaScriptEventProcessor(String id, + Map processorConfig, + String scriptPath, + String entryFunction, + boolean hotReload, + long timeoutMs, + ErpJsFacade erp, + TemplateJsFacade tpl) { + this.id = id; + this.processorConfig = processorConfig == null ? Map.of() : processorConfig; + this.processorConfigProxy = toProxyObject(this.processorConfig); + this.scriptPath = Path.of(scriptPath); + this.entryFunction = entryFunction == null ? "process" : entryFunction; + this.hotReload = hotReload; + this.timeoutMs = Math.max(0, timeoutMs); + this.erp = erp; + this.tpl = tpl; + + this.engine = Engine.newBuilder().build(); + int poolSize = Math.max(1, Runtime.getRuntime().availableProcessors()); + this.pool = new ArrayBlockingQueue<>(poolSize); + + // Preload script + contexts at startup + reloadIfNeeded(true); + for (int i = 0; i < poolSize; i++) { + pool.add(newContextHolder()); + } + } + + @Override + public ProcessorResult process(IngressMessage msg, ResolvedBinding rb, Map vars) throws Exception { + reloadIfNeeded(false); + + ContextHolder holder = borrow(); + try { + ProxyObject input = buildInput(msg, vars); + // JS signature: process(input, cfg) + Value out = holder.fn.execute(input, processorConfigProxy); + return parseResult(out); + } finally { + recycle(holder); + } + } + + private ProcessorResult parseResult(Value out) { + if (out == null || out.isNull()) { + return ProcessorResult.of(Map.of(), List.of()); + } + + boolean drop = out.hasMember("drop") && out.getMember("drop").isBoolean() && out.getMember("drop").asBoolean(); + if (drop) return ProcessorResult.drop(); + + Map varsDelta = new LinkedHashMap<>(); + if (out.hasMember("vars")) { + Object v = toJava(out.getMember("vars")); + if (v instanceof Map m) { + for (var e : m.entrySet()) { + if (e.getKey() == null || e.getValue() == null) continue; + varsDelta.put(String.valueOf(e.getKey()), String.valueOf(e.getValue())); + } + } + } + + List events = new ArrayList<>(); + if (out.hasMember("events")) { + Value evs = out.getMember("events"); + if (evs != null && evs.hasArrayElements()) { + long n = evs.getArraySize(); + for (long i = 0; i < n; i++) { + Value ev = evs.getArrayElement(i); + DetectorEvent de = parseEvent(ev); + if (de != null) events.add(de); + } + } + } + + return ProcessorResult.of(varsDelta, events); + } + + private DetectorEvent parseEvent(Value ev) { + if (ev == null || ev.isNull()) return null; + + String action = getString(ev, "action"); + String faultKey = getString(ev, "faultKey"); + if (faultKey == null || faultKey.isBlank()) return null; + + String severity = getString(ev, "severity"); + if (severity == null || severity.isBlank()) severity = "MAJOR"; + + Instant occurredAt = null; + String ts = getString(ev, "occurredAt"); + if (ts != null) { + occurredAt = parseInstant(ts); + } + + Map details = Map.of(); + if (ev.hasMember("details")) { + Object d = toJava(ev.getMember("details")); + if (d instanceof Map m) { + Map dd = new LinkedHashMap<>(); + for (var e : m.entrySet()) { + if (e.getKey() == null) continue; + dd.put(String.valueOf(e.getKey()), e.getValue()); + } + details = dd; + } + } + + String eventType = normalizeEventType(action); + return new DetectorEvent(faultKey, eventType, severity, occurredAt, details); + } + + private static String normalizeEventType(String action) { + if (action == null) return "ALARM"; + String a = action.trim().toUpperCase(Locale.ROOT); + if (a.equals("CANCEL") || a.equals("CLEAR") || a.equals("CLOSE")) return "CANCEL"; + return "ALARM"; + } + + private static String getString(Value v, String member) { + if (v == null || member == null) return null; + if (!v.hasMember(member)) return null; + Value m = v.getMember(member); + if (m == null || m.isNull()) return null; + try { + return m.isString() ? m.asString() : String.valueOf(toJava(m)); + } catch (Exception e) { + return null; + } + } + + private static Instant parseInstant(String s) { + if (s == null) return null; + String t = s.trim(); + if (t.isEmpty()) return null; + try { return Instant.parse(t); } catch (Exception ignore) {} + try { return OffsetDateTime.parse(t).toInstant(); } catch (Exception ignore) {} + return null; + } + + private static Map toStringKeyMap(Map map) { + if (map == null || map.isEmpty()) return Map.of(); + Map out = new LinkedHashMap<>(); + for (var e : map.entrySet()) { + if (e.getKey() == null) continue; + out.put(String.valueOf(e.getKey()), e.getValue()); + } + return out; + } + + private static ProxyObject toProxyObject(Map map) { + return ProxyObject.fromMap(toStringKeyMap(map)); + } + + private ProxyObject buildInput(IngressMessage msg, Map vars) { + Map input = new LinkedHashMap<>(); + input.put("inType", msg.getInType() == null ? null : msg.getInType().name()); + input.put("path", msg.getPath()); + + Map headers = msg.getHeaders() == null ? Map.of() : new LinkedHashMap<>(msg.getHeaders()); + input.put("headers", toProxyObject(headers)); + + byte[] payload = msg.getPayload(); + if (payload == null) payload = new byte[0]; + input.put("payloadText", new String(payload, StandardCharsets.UTF_8)); + input.put("payloadBase64", Base64.getEncoder().encodeToString(payload)); + input.put("payloadSize", payload.length); + + Map varsMap = vars == null ? Map.of() : new LinkedHashMap<>(vars); + input.put("vars", toProxyObject(varsMap)); + + return ProxyObject.fromMap(input); + } + + private ContextHolder newContextHolder() { + Context ctx = Context.newBuilder("js") + .engine(engine) + .allowHostAccess(HostAccess.newBuilder(HostAccess.NONE) + .allowAccessAnnotatedBy(HostAccess.Export.class) + .build()) + .allowHostClassLookup(s -> false) + .allowHostClassLoading(false) + .allowIO(false) + .allowCreateThread(false) + .build(); + + Value bindings = ctx.getBindings("js"); + if (erp != null) bindings.putMember("erp", erp); + if (tpl != null) bindings.putMember("tpl", tpl); + + // Load script + Source src; + try { + src = Source.newBuilder("js", lastLoadedText == null ? "" : lastLoadedText, scriptPath.toString()).build(); + } catch (Exception e) { + throw new IllegalStateException("Failed to build JS source for processor " + id + " from " + scriptPath, e); + } + + ctx.eval(src); + + Value fn = bindings.getMember(entryFunction); + if (fn == null || !fn.canExecute()) { + throw new IllegalStateException("JS processor '" + id + "' did not define an executable function '" + entryFunction + "' in " + scriptPath); + } + + return new ContextHolder(ctx, fn); + } + + private void reloadIfNeeded(boolean force) { + if (!hotReload && !force && lastLoadedText != null) return; + try { + long mtime = Files.exists(scriptPath) ? Files.getLastModifiedTime(scriptPath).toMillis() : -1L; + if (!force && !hotReload) return; + if (!force && mtime == lastLoadedMtime) return; + + String text = Files.readString(scriptPath, StandardCharsets.UTF_8); + if (text == null) text = ""; + this.lastLoadedText = text; + this.lastLoadedMtime = mtime; + + if (!force) { + // On reload we clear the pool and recreate contexts. + pool.clear(); + int poolSize = Math.max(1, Runtime.getRuntime().availableProcessors()); + for (int i = 0; i < poolSize; i++) pool.add(newContextHolder()); + log.info("Reloaded JS processor '{}' from {}", id, scriptPath); + } + } catch (Exception e) { + if (force) { + throw new IllegalStateException("Failed to load JS processor '" + id + "' from " + scriptPath, e); + } + log.warn("Failed to hot-reload JS processor '{}' from {}. Keeping previous script.", id, scriptPath, e); + } + } + + private ContextHolder borrow() { + try { + ContextHolder h = pool.poll(); + if (h != null) return h; + // fallback: create a new one if pool is empty + return newContextHolder(); + } catch (Exception e) { + throw new RuntimeException("Failed to borrow JS processor context (" + id + ")", e); + } + } + + private void recycle(ContextHolder h) { + if (h == null) return; + // If pool is full, close the context. + if (!pool.offer(h)) { + try { h.ctx.close(); } catch (Exception ignore) {} + } + } + + private record ContextHolder(Context ctx, Value fn) {} + + /** + * Convert a polyglot Value to plain Java types: + * Map, List, String/Number/Boolean/null. + */ + private static Object toJava(Value v) { + if (v == null || v.isNull()) return null; + if (v.isBoolean()) return v.asBoolean(); + if (v.isNumber()) { + // Keep numbers as Double if they don't fit in long cleanly. + try { + if (v.fitsInInt()) return v.asInt(); + if (v.fitsInLong()) return v.asLong(); + } catch (Exception ignore) {} + try { return v.asDouble(); } catch (Exception ignore) {} + return v.toString(); + } + if (v.isString()) return v.asString(); + + if (v.hasArrayElements()) { + long n = v.getArraySize(); + List list = new ArrayList<>((int) Math.min(n, 10_000)); + for (long i = 0; i < n; i++) { + list.add(toJava(v.getArrayElement(i))); + } + return list; + } + + if (v.hasMembers()) { + Map map = new LinkedHashMap<>(); + for (String k : v.getMemberKeys()) { + map.put(k, toJava(v.getMember(k))); + } + return map; + } + + try { + return v.as(Object.class); + } catch (Exception e) { + return v.toString(); + } + } + + private static ProxyObject toProxy(Map map) { + Map m = new LinkedHashMap<>(); + if (map != null) map.forEach((k, v) -> m.put(String.valueOf(k), v)); + return ProxyObject.fromMap(m); + } + + +} diff --git a/src/main/java/at/co/procon/malis/cep/processor/js/TemplateJsFacade.java b/src/main/java/at/co/procon/malis/cep/processor/js/TemplateJsFacade.java new file mode 100644 index 0000000..ff7aab1 --- /dev/null +++ b/src/main/java/at/co/procon/malis/cep/processor/js/TemplateJsFacade.java @@ -0,0 +1,24 @@ +package at.co.procon.malis.cep.processor.js; + +import at.co.procon.malis.cep.util.TemplateUtil; +import org.graalvm.polyglot.HostAccess; + +import java.util.LinkedHashMap; +import java.util.Map; + +/** Safe JS host facade for template expansion (${var}). */ +public final class TemplateJsFacade { + + @HostAccess.Export + public String expand(String template, Map vars) { + if (template == null) return null; + Map v = new LinkedHashMap<>(); + if (vars != null) { + for (var e : vars.entrySet()) { + if (e.getKey() == null || e.getValue() == null) continue; + v.put(e.getKey(), String.valueOf(e.getValue())); + } + } + return TemplateUtil.expand(template, v); + } +} diff --git a/src/main/java/at/co/procon/malis/cep/transport/mqtt/MqttBrokerRegistry.java b/src/main/java/at/co/procon/malis/cep/transport/mqtt/MqttBrokerRegistry.java new file mode 100644 index 0000000..13e7d72 --- /dev/null +++ b/src/main/java/at/co/procon/malis/cep/transport/mqtt/MqttBrokerRegistry.java @@ -0,0 +1,180 @@ +package at.co.procon.malis.cep.transport.mqtt; + +import at.co.procon.malis.cep.config.CepProperties; +import at.co.procon.malis.cep.util.InsecureSsl; +import org.eclipse.paho.client.mqttv3.IMqttActionListener; +import org.eclipse.paho.client.mqttv3.IMqttToken; +import org.eclipse.paho.client.mqttv3.MqttAsyncClient; +import org.eclipse.paho.client.mqttv3.MqttConnectOptions; +import org.eclipse.paho.client.mqttv3.MqttException; +import org.eclipse.paho.client.mqttv3.MqttMessage; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Component; + +import java.time.Duration; +import java.util.Locale; +import java.util.Map; +import java.util.Objects; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; + +/** + * Registry for MQTT broker connections defined in {@code cep.brokers.*}. + * + *

Why a registry? + *

    + *
  • Bindings/outputs can target different MQTT brokers via {@code brokerRef}
    • + *
  • Connections are created lazily (an unused broker does not fail startup)
    • + *
  • Publishing is transport-only; business logic stays in the CEP pipeline
    • + *
+ * + *

We deliberately use Paho directly for outbound publishing so that each output can specify + * a distinct {@code brokerRef}. Inbound ingress still uses Spring Integration MQTT. + */ +@Component +public class MqttBrokerRegistry { + + private static final Logger log = LoggerFactory.getLogger(MqttBrokerRegistry.class); + + private final CepProperties props; + private final Map clients = new ConcurrentHashMap<>(); + + public MqttBrokerRegistry(CepProperties props) { + this.props = props; + } + + /** + * Publish bytes to a topic on the specified broker. + */ + public void publish(String brokerRef, String topic, byte[] payload, Integer qos, Boolean retained) { + Objects.requireNonNull(brokerRef, "brokerRef"); + Objects.requireNonNull(topic, "topic"); + + ClientHolder holder = clients.computeIfAbsent(brokerRef, this::createClient); + ensureConnected(holder); + + try { + MqttMessage msg = new MqttMessage(payload == null ? new byte[0] : payload); + if (qos != null) msg.setQos(qos); + if (retained != null) msg.setRetained(retained); + holder.client.publish(topic, msg); + } catch (MqttException e) { + throw new RuntimeException("MQTT publish failed (brokerRef=" + brokerRef + ", topic=" + topic + ")", e); + } + } + + private ClientHolder createClient(String brokerRef) { + CepProperties.BrokerDef broker = props.getBrokers().get(brokerRef); + if (broker == null) { + throw new IllegalStateException("Unknown brokerRef: " + brokerRef); + } + if (broker.getType() == null || !broker.getType().equalsIgnoreCase("MQTT")) { + throw new IllegalStateException("brokerRef '" + brokerRef + "' is not of type MQTT"); + } + if (broker.getUrl() == null || broker.getUrl().isBlank()) { + throw new IllegalStateException("brokerRef '" + brokerRef + "' is missing url"); + } + + String clientIdBase = (broker.getClientId() == null || broker.getClientId().isBlank()) + ? "malis-cep" + : broker.getClientId(); + String clientId = clientIdBase + "-out-" + UUID.randomUUID(); + + try { + MqttAsyncClient client = new MqttAsyncClient(broker.getUrl(), clientId); + + MqttConnectOptions options = new MqttConnectOptions(); + options.setAutomaticReconnect(true); + options.setCleanSession(true); + + // TLS handling (dev + prod) + // - For production, prefer a proper truststore (JVM: -Djavax.net.ssl.trustStore=...) + // - For local development against brokers with self-signed / untrusted certs, + // you can enable tls.trustAll=true in cep.brokers..tls. + applyTlsOptions(broker, options); + + if (broker.getUsername() != null && !broker.getUsername().isBlank()) { + options.setUserName(broker.getUsername()); + } + if (broker.getPassword() != null) { + options.setPassword(broker.getPassword().toCharArray()); + } + + log.info("MQTT broker prepared: ref={}, url={}, clientId={}", brokerRef, broker.getUrl(), clientId); + return new ClientHolder(brokerRef, broker.getUrl(), client, options); + + } catch (MqttException e) { + throw new RuntimeException("Failed to create MQTT client for brokerRef=" + brokerRef, e); + } + } + + private static void applyTlsOptions(CepProperties.BrokerDef broker, MqttConnectOptions options) { + if (broker == null || options == null) return; + + String url = broker.getUrl() == null ? "" : broker.getUrl().toLowerCase(Locale.ROOT); + if (!(url.startsWith("ssl://") || url.startsWith("wss://"))) return; + + Map tls = broker.getTls(); + if (tls == null || tls.isEmpty()) return; + + Object trustAll = tls.get("trustAll"); + if (Boolean.TRUE.equals(trustAll) || "true".equalsIgnoreCase(String.valueOf(trustAll))) { + options.setSocketFactory(InsecureSsl.trustAllSocketFactory()); + } + } + + private void ensureConnected(ClientHolder holder) { + if (holder.client.isConnected()) return; + + // Connect synchronously on first use; keep a modest timeout. + Duration timeout = Duration.ofSeconds(5); + CountDownLatch latch = new CountDownLatch(1); + final Throwable[] error = new Throwable[1]; + + try { + holder.client.connect(holder.options, null, new IMqttActionListener() { + @Override + public void onSuccess(IMqttToken asyncActionToken) { + latch.countDown(); + } + + @Override + public void onFailure(IMqttToken asyncActionToken, Throwable exception) { + error[0] = exception; + latch.countDown(); + } + }); + + boolean ok = latch.await(timeout.toMillis(), TimeUnit.MILLISECONDS); + if (!ok) { + throw new RuntimeException("MQTT connect timed out (brokerRef=" + holder.brokerRef + ", url=" + holder.url + ")"); + } + if (error[0] != null) { + throw new RuntimeException("MQTT connect failed (brokerRef=" + holder.brokerRef + ")", error[0]); + } + } catch (InterruptedException ie) { + Thread.currentThread().interrupt(); + throw new RuntimeException("MQTT connect interrupted (brokerRef=" + holder.brokerRef + ")", ie); + } catch (MqttException e) { + throw new RuntimeException("MQTT connect failed (brokerRef=" + holder.brokerRef + ")", e); + } + } + + /** Holder for one broker connection. */ + private static final class ClientHolder { + private final String brokerRef; + private final String url; + private final MqttAsyncClient client; + private final MqttConnectOptions options; + + private ClientHolder(String brokerRef, String url, MqttAsyncClient client, MqttConnectOptions options) { + this.brokerRef = brokerRef; + this.url = url; + this.client = client; + this.options = options; + } + } +} diff --git a/src/main/java/at/co/procon/malis/cep/transport/mqtt/MqttInboundDelayedStarter.java b/src/main/java/at/co/procon/malis/cep/transport/mqtt/MqttInboundDelayedStarter.java new file mode 100644 index 0000000..345232e --- /dev/null +++ b/src/main/java/at/co/procon/malis/cep/transport/mqtt/MqttInboundDelayedStarter.java @@ -0,0 +1,34 @@ +package at.co.procon.malis.cep.transport.mqtt; + +import at.co.procon.malis.cep.config.CepProperties; +import org.springframework.boot.context.event.ApplicationReadyEvent; +import org.springframework.context.event.EventListener; +import org.springframework.integration.mqtt.inbound.MqttPahoMessageDrivenChannelAdapter; +import org.springframework.scheduling.TaskScheduler; +import org.springframework.stereotype.Component; + +import java.time.Duration; +import java.time.Instant; + +@Component +public class MqttInboundDelayedStarter { + + private final MqttPahoMessageDrivenChannelAdapter adapter; + private final TaskScheduler taskScheduler; + private final Duration delay; + + public MqttInboundDelayedStarter( + MqttPahoMessageDrivenChannelAdapter adapter, + TaskScheduler taskScheduler, + CepProperties props + ) { + this.adapter = adapter; + this.taskScheduler = taskScheduler; + this.delay = props.getIngress().getMqtt().getInboundStartDelay(); // e.g. 30s + } + + @EventListener(ApplicationReadyEvent.class) + public void startLater() { + taskScheduler.schedule(adapter::start, Instant.now().plus(delay)); + } +} diff --git a/src/main/java/at/co/procon/malis/cep/transport/mqtt/MqttIntegrationConfig.java b/src/main/java/at/co/procon/malis/cep/transport/mqtt/MqttIntegrationConfig.java new file mode 100644 index 0000000..fe426b2 --- /dev/null +++ b/src/main/java/at/co/procon/malis/cep/transport/mqtt/MqttIntegrationConfig.java @@ -0,0 +1,218 @@ +package at.co.procon.malis.cep.transport.mqtt; + +import at.co.procon.malis.cep.config.CepProperties; +import at.co.procon.malis.cep.pipeline.CepPipeline; +import at.co.procon.malis.cep.util.InsecureSsl; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.integration.annotation.ServiceActivator; +import org.springframework.integration.channel.DirectChannel; +import org.springframework.integration.mqtt.core.DefaultMqttPahoClientFactory; +import org.springframework.integration.mqtt.core.MqttPahoClientFactory; +import org.springframework.integration.mqtt.inbound.MqttPahoMessageDrivenChannelAdapter; +import org.springframework.integration.mqtt.support.DefaultPahoMessageConverter; +import org.springframework.integration.mqtt.support.MqttHeaders; +import org.springframework.integration.mqtt.outbound.MqttPahoMessageHandler; +import org.springframework.messaging.Message; +import org.springframework.messaging.MessageChannel; +import org.springframework.messaging.MessageHandler; +import org.springframework.messaging.support.MessageBuilder; + +import java.nio.charset.StandardCharsets; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +/** + * Real MQTT ingress/egress using Spring Integration MQTT (Paho). + * + * How it fits the architecture: + * - MQTT adapters are *transport-only*. + * - They do not parse or detect faults. + * - They convert an incoming MQTT message into {@link at.co.procon.malis.cep.binding.IngressMessage} and hand it to {@link CepPipeline}. + * - Routing and business logic remain fully configurable via bindings. + */ +@Configuration +public class MqttIntegrationConfig { + + private static final Logger log = LoggerFactory.getLogger(MqttIntegrationConfig.class); + + public static final String MQTT_INBOUND_CHANNEL = "mqttInboundChannel"; + public static final String MQTT_OUTBOUND_CHANNEL = "mqttOutboundChannel"; + + @Bean(name = MQTT_INBOUND_CHANNEL) + public MessageChannel mqttInboundChannel() { + // DirectChannel executes handlers in the sender thread. + // If you need higher throughput, change to ExecutorChannel. + return new DirectChannel(); + } + + @Bean(name = MQTT_OUTBOUND_CHANNEL) + public MessageChannel mqttOutboundChannel() { + return new DirectChannel(); + } + + @Bean + public MqttPahoClientFactory mqttClientFactory(CepProperties props) { + CepProperties.IngressDef.MqttIngress cfg = props.getIngress().getMqtt(); + CepProperties.BrokerDef broker = requireBroker(props, cfg.getBrokerRef(), "MQTT"); + + var options = new org.eclipse.paho.client.mqttv3.MqttConnectOptions(); + options.setServerURIs(new String[]{broker.getUrl()}); + options.setAutomaticReconnect(broker.isAutomaticReconnect()); + options.setCleanSession(broker.isCleanSession()); + + // Align ingress TLS behavior with outbound publishing. + // For dev brokers with untrusted certs, set: + // cep.brokers..tls.trustAll=true + applyTlsOptions(broker, options); + + if (broker.getUsername() != null && !broker.getUsername().isBlank()) { + options.setUserName(broker.getUsername()); + } + if (broker.getPassword() != null) { + options.setPassword(broker.getPassword().toCharArray()); + } + + DefaultMqttPahoClientFactory factory = new DefaultMqttPahoClientFactory(); + factory.setConnectionOptions(options); + return factory; + } + + private static void applyTlsOptions(CepProperties.BrokerDef broker, + org.eclipse.paho.client.mqttv3.MqttConnectOptions options) { + if (broker == null || options == null) return; + String url = broker.getUrl() == null ? "" : broker.getUrl().toLowerCase(java.util.Locale.ROOT); + if (!(url.startsWith("ssl://") || url.startsWith("wss://"))) return; + + Map tls = broker.getTls(); + if (tls == null || tls.isEmpty()) return; + + Object trustAll = tls.get("trustAll"); + if (Boolean.TRUE.equals(trustAll) || "true".equalsIgnoreCase(String.valueOf(trustAll))) { + options.setSocketFactory(InsecureSsl.trustAllSocketFactory()); + } + } + + @Bean + @ConditionalOnProperty(prefix = "cep.ingress.mqtt", name = "enabled", havingValue = "true", matchIfMissing = true) + public MqttPahoMessageDrivenChannelAdapter mqttInboundAdapter( + CepProperties props, + MqttPahoClientFactory clientFactory, + @Qualifier(MQTT_INBOUND_CHANNEL) MessageChannel inboundChannel + ) { + CepProperties.IngressDef.MqttIngress cfg = props.getIngress().getMqtt(); + List subs = cfg.getSubscriptions(); + if (subs == null || subs.isEmpty()) { + log.warn("MQTT ingress enabled, but no subscriptions configured. Add cep.ingress.mqtt.subscriptions"); + subs = List.of("#"); + } + + String clientId = cfg.getClientId(); + if (clientId == null || clientId.isBlank()) { + String base = Optional.ofNullable(props.getBrokers().get(cfg.getBrokerRef())) + .map(CepProperties.BrokerDef::getClientId) + .orElse("malis-cep"); + clientId = base + "-ingress"; + } + + var converter = new DefaultPahoMessageConverter(); + converter.setPayloadAsBytes(true); + + var adapter = new MqttPahoMessageDrivenChannelAdapter(clientId, clientFactory, subs.toArray(String[]::new)); + adapter.setCompletionTimeout(cfg.getCompletionTimeoutMs()); + adapter.setConverter(converter); + adapter.setQos(cfg.getQos()); + adapter.setOutputChannel(inboundChannel); + // KEY: don’t connect/receive on context start + adapter.setAutoStartup(false); + + log.info("MQTT ingress enabled. brokerRef={}, subscriptions={}, qos={}", cfg.getBrokerRef(), subs, cfg.getQos()); + return adapter; + } + + /** + * Inbound handler: convert Spring Integration MQTT message -> our internal IngressMessage. + */ + @Bean + @ServiceActivator(inputChannel = MQTT_INBOUND_CHANNEL) + public MessageHandler mqttInboundHandler(CepPipeline pipeline) { + return (Message message) -> { + String topic = (String) message.getHeaders().get(MqttHeaders.RECEIVED_TOPIC); + + byte[] payloadBytes; + Object payload = message.getPayload(); + if (payload instanceof byte[] b) payloadBytes = b; + else payloadBytes = String.valueOf(payload).getBytes(StandardCharsets.UTF_8); + + Map headers = new HashMap<>(); + headers.put("mqtt_receivedTopic", topic); + for (var e : message.getHeaders().entrySet()) { + if (e.getValue() != null) headers.put("mqtt_" + e.getKey(), String.valueOf(e.getValue())); + } + + try { + pipeline.process(new at.co.procon.malis.cep.binding.IngressMessage( + CepProperties.InType.MQTT, + topic, + headers, + payloadBytes + )); + } catch (Exception ex) { + // IMPORTANT: Do not let exceptions bubble up to the MQTT adapter thread. + // If they do, Spring Integration will report MessageHandlingException and + // the underlying Paho adapter may drop the connection. + log.error("CEP pipeline failed for MQTT message on topic='{}' (message will be sent to DLQ if configured)", topic, ex); + } + }; + } + + /** + * Outbound handler: takes messages from mqttOutboundChannel and publishes to the topic provided in header. + */ + @Bean + @ServiceActivator(inputChannel = MQTT_OUTBOUND_CHANNEL) + public MessageHandler mqttOutboundHandler(CepProperties props, MqttPahoClientFactory clientFactory) { + CepProperties.IngressDef.MqttIngress cfg = props.getIngress().getMqtt(); + String clientId = (cfg.getClientId() == null || cfg.getClientId().isBlank()) + ? "malis-cep-egress" + : (cfg.getClientId() + "-egress"); + + MqttPahoMessageHandler handler = new MqttPahoMessageHandler(clientId, clientFactory); + handler.setAsync(true); + handler.setDefaultQos(cfg.getQos()); + return handler; + } + + @Bean + public MqttOutboundGateway mqttOutboundGateway(@Qualifier(MQTT_OUTBOUND_CHANNEL) MessageChannel outboundChannel, + CepProperties props) { + return new MqttOutboundGateway(outboundChannel, props); + } + + static CepProperties.BrokerDef requireBroker(CepProperties props, String brokerRef, String expectedType) { + CepProperties.BrokerDef broker = props.getBrokers().get(brokerRef); + if (broker == null) throw new IllegalStateException("Unknown brokerRef: " + brokerRef); + if (broker.getType() == null || !broker.getType().equalsIgnoreCase(expectedType)) { + throw new IllegalStateException("brokerRef '" + brokerRef + "' is not of type " + expectedType); + } + if (broker.getUrl() == null || broker.getUrl().isBlank()) { + throw new IllegalStateException("brokerRef '" + brokerRef + "' is missing url"); + } + return broker; + } + + /** Helper for building outbound MQTT messages. */ + static Message outboundMessage(byte[] payload, String topic, Integer qos, Boolean retained) { + var builder = MessageBuilder.withPayload(payload) + .setHeader(MqttHeaders.TOPIC, topic); + if (qos != null) builder.setHeader(MqttHeaders.QOS, qos); + if (retained != null) builder.setHeader(MqttHeaders.RETAINED, retained); + return builder.build(); + } +} diff --git a/src/main/java/at/co/procon/malis/cep/transport/mqtt/MqttOutboundGateway.java b/src/main/java/at/co/procon/malis/cep/transport/mqtt/MqttOutboundGateway.java new file mode 100644 index 0000000..926ab42 --- /dev/null +++ b/src/main/java/at/co/procon/malis/cep/transport/mqtt/MqttOutboundGateway.java @@ -0,0 +1,29 @@ +package at.co.procon.malis.cep.transport.mqtt; + +import at.co.procon.malis.cep.config.CepProperties; +import org.springframework.messaging.MessageChannel; + +/** + * Small helper that hides Spring Integration's Message construction. + * + * Output publishers call this gateway to publish bytes to a topic. + * This keeps transport details out of the core CEP pipeline. + */ +public class MqttOutboundGateway { + + private final MessageChannel outboundChannel; + private final CepProperties props; + + public MqttOutboundGateway(MessageChannel outboundChannel, CepProperties props) { + this.outboundChannel = outboundChannel; + this.props = props; + } + + public void publish(String topic, byte[] payload, Integer qos, Boolean retained) { + outboundChannel.send(MqttIntegrationConfig.outboundMessage(payload, topic, qos, retained)); + } + + public CepProperties getProps() { + return props; + } +} diff --git a/src/main/java/at/co/procon/malis/cep/transport/rabbit/RabbitBrokerRegistry.java b/src/main/java/at/co/procon/malis/cep/transport/rabbit/RabbitBrokerRegistry.java new file mode 100644 index 0000000..ecf3999 --- /dev/null +++ b/src/main/java/at/co/procon/malis/cep/transport/rabbit/RabbitBrokerRegistry.java @@ -0,0 +1,141 @@ +package at.co.procon.malis.cep.transport.rabbit; + +import at.co.procon.malis.cep.config.CepProperties; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.amqp.rabbit.connection.CachingConnectionFactory; +import org.springframework.amqp.rabbit.core.RabbitTemplate; +import org.springframework.stereotype.Component; + +import java.net.URI; +import java.net.URLDecoder; +import java.nio.charset.StandardCharsets; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +/** + * Creates Rabbit connection factories/templates from {@code cep.brokers.*} definitions. + * + * Why a registry instead of raw @Bean? + * - Outputs can reference different brokerRefs (future-proof). + * - We create templates lazily, so an unused broker does not fail startup. + */ +@Component +public class RabbitBrokerRegistry { + + private static final Logger log = LoggerFactory.getLogger(RabbitBrokerRegistry.class); + + private final CepProperties props; + private final Map factories = new ConcurrentHashMap<>(); + private final Map templates = new ConcurrentHashMap<>(); + + public RabbitBrokerRegistry(CepProperties props) { + this.props = props; + } + + public RabbitTemplate template(String brokerRef) { + return templates.computeIfAbsent(brokerRef, ref -> new RabbitTemplate(connectionFactory(ref))); + } + + public CachingConnectionFactory connectionFactory(String brokerRef) { + return factories.computeIfAbsent(brokerRef, ref -> { + CepProperties.BrokerDef broker = props.getBrokers().get(ref); + if (broker == null) { + throw new IllegalStateException("Unknown brokerRef: " + ref); + } + if (broker.getType() == null || !broker.getType().equalsIgnoreCase("RABBIT")) { + throw new IllegalStateException("brokerRef '" + ref + "' is not of type RABBIT"); + } + if (broker.getUrl() == null || broker.getUrl().isBlank()) { + throw new IllegalStateException("brokerRef '" + ref + "' is missing url"); + } + + ParsedAmqpUri parsed = ParsedAmqpUri.parse(broker.getUrl()); + CachingConnectionFactory cf = new CachingConnectionFactory(parsed.host(), parsed.port()); + + if (parsed.username() != null) cf.setUsername(parsed.username()); + if (parsed.password() != null) cf.setPassword(parsed.password()); + if (parsed.virtualHost() != null) cf.setVirtualHost(parsed.virtualHost()); + + log.info("Rabbit broker configured: ref={}, host={}, port={}, vhost={}", ref, parsed.host(), parsed.port(), parsed.virtualHost()); + return cf; + }); + } + + /** + * Small, dependency-free AMQP URI parser. + * + * Supported examples: + * - amqp://guest:guest@localhost:5672/ + * - amqps://user:pass@rabbit.example.com:5671/myvhost + */ + public static final class ParsedAmqpUri { + + private final String host; + private final int port; + private final String username; + private final String password; + private final String virtualHost; + + public ParsedAmqpUri(String host, int port, String username, String password, String virtualHost) { + this.host = host; + this.port = port; + this.username = username; + this.password = password; + this.virtualHost = virtualHost; + } + + public String getHost() { return host; } + public int getPort() { return port; } + public String getUsername() { return username; } + public String getPassword() { return password; } + public String getVirtualHost() { return virtualHost; } + + // record-style accessors (keeps call sites compact) + public String host() { return host; } + public int port() { return port; } + public String username() { return username; } + public String password() { return password; } + public String virtualHost() { return virtualHost; } + + public static ParsedAmqpUri parse(String uriStr) { + try { + URI uri = URI.create(uriStr); + String host = uri.getHost(); + int port = uri.getPort() > 0 ? uri.getPort() : 5672; + + String userInfo = uri.getUserInfo(); + String username = null; + String password = null; + if (userInfo != null) { + String[] parts = userInfo.split(":", 2); + username = parts.length > 0 ? decode(parts[0]) : null; + password = parts.length > 1 ? decode(parts[1]) : null; + } + + String path = uri.getPath(); + String vhost; + if (path == null || path.isBlank() || path.equals("/")) { + vhost = "/"; + } else { + // URI path starts with '/', Rabbit expects without leading '/' + vhost = decode(path.substring(1)); + if (vhost.isBlank()) vhost = "/"; + } + + if (host == null || host.isBlank()) { + throw new IllegalArgumentException("AMQP URI missing host: " + uriStr); + } + + return new ParsedAmqpUri(host, port, username, password, vhost); + + } catch (Exception e) { + throw new IllegalArgumentException("Invalid AMQP URI: " + uriStr, e); + } + } + + private static String decode(String s) { + return URLDecoder.decode(s, StandardCharsets.UTF_8); + } + } +} diff --git a/src/main/java/at/co/procon/malis/cep/transport/rabbit/RabbitIngressManager.java b/src/main/java/at/co/procon/malis/cep/transport/rabbit/RabbitIngressManager.java new file mode 100644 index 0000000..625c803 --- /dev/null +++ b/src/main/java/at/co/procon/malis/cep/transport/rabbit/RabbitIngressManager.java @@ -0,0 +1,110 @@ +package at.co.procon.malis.cep.transport.rabbit; + +import at.co.procon.malis.cep.binding.IngressMessage; +import at.co.procon.malis.cep.config.CepProperties; +import at.co.procon.malis.cep.pipeline.CepPipeline; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.amqp.core.Message; +import org.springframework.amqp.rabbit.listener.SimpleMessageListenerContainer; +import org.springframework.context.SmartLifecycle; +import org.springframework.stereotype.Component; + +import java.util.HashMap; +import java.util.Map; + +/** + * Optional RabbitMQ ingress. + * + * This creates a {@link SimpleMessageListenerContainer} programmatically from configuration. + * The container converts AMQP messages to {@link IngressMessage} and pushes them into the CEP pipeline. + * + * IMPORTANT: Bindings decide what to do with the message. Ingress just feeds bytes + a "routing key" string. + */ +@Component +public class RabbitIngressManager implements SmartLifecycle { + + private static final Logger log = LoggerFactory.getLogger(RabbitIngressManager.class); + + private final CepProperties props; + private final RabbitBrokerRegistry brokers; + private final CepPipeline pipeline; + + private volatile boolean running; + private SimpleMessageListenerContainer container; + + public RabbitIngressManager(CepProperties props, RabbitBrokerRegistry brokers, CepPipeline pipeline) { + this.props = props; + this.brokers = brokers; + this.pipeline = pipeline; + } + + @Override + public void start() { + CepProperties.IngressDef.RabbitIngress cfg = props.getIngress().getRabbit(); + if (cfg == null || !cfg.isEnabled()) { + log.info("Rabbit ingress disabled by configuration (cep.ingress.rabbit.enabled=false)"); + return; + } + if (cfg.getQueues() == null || cfg.getQueues().isEmpty()) { + log.warn("Rabbit ingress enabled but no queues configured. Add cep.ingress.rabbit.queues"); + return; + } + + container = new SimpleMessageListenerContainer(brokers.connectionFactory(cfg.getBrokerRef())); + container.setQueueNames(cfg.getQueues().toArray(String[]::new)); + container.setConcurrentConsumers(cfg.getConcurrentConsumers()); + container.setMaxConcurrentConsumers(cfg.getMaxConcurrentConsumers()); + container.setPrefetchCount(cfg.getPrefetch()); + + container.setMessageListener((org.springframework.amqp.core.MessageListener) this::onMessage); + + container.start(); + running = true; + log.info("Rabbit ingress enabled. brokerRef={}, queues={}, concurrency={}-{}, prefetch={}", + cfg.getBrokerRef(), cfg.getQueues(), cfg.getConcurrentConsumers(), cfg.getMaxConcurrentConsumers(), cfg.getPrefetch()); + } + + private void onMessage(Message message) { + // Prefer routing key for matching; fall back to consumer queue. + String routingKey = message.getMessageProperties().getReceivedRoutingKey(); + String queue = message.getMessageProperties().getConsumerQueue(); + String matchKey = (routingKey != null && !routingKey.isBlank()) ? routingKey : queue; + + Map headers = new HashMap<>(); + headers.put("amqp_queue", queue); + headers.put("amqp_routingKey", routingKey); + + if (message.getMessageProperties().getHeaders() != null) { + for (var e : message.getMessageProperties().getHeaders().entrySet()) { + if (e.getValue() != null) headers.put("amqp_h_" + e.getKey(), String.valueOf(e.getValue())); + } + } + + pipeline.process(new IngressMessage( + CepProperties.InType.RABBIT, + matchKey, + headers, + message.getBody() + )); + } + + @Override + public void stop() { + if (container != null) { + container.stop(); + } + running = false; + } + + @Override + public boolean isRunning() { + return running; + } + + @Override + public int getPhase() { + // Start after most beans are ready. + return Integer.MAX_VALUE; + } +} diff --git a/src/main/java/at/co/procon/malis/cep/util/ExpiringCache.java b/src/main/java/at/co/procon/malis/cep/util/ExpiringCache.java new file mode 100644 index 0000000..e984724 --- /dev/null +++ b/src/main/java/at/co/procon/malis/cep/util/ExpiringCache.java @@ -0,0 +1,97 @@ +package at.co.procon.malis.cep.util; + +import java.time.Duration; +import java.time.Instant; +import java.util.Objects; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ThreadLocalRandom; + +/** + * Very small in-memory TTL cache. + * + *

Prototype-grade: concurrent, lazy expiry, and best-effort size bounding. + */ +public final class ExpiringCache { + + private static final class Entry { + final V value; + final long expiresAtEpochMs; + Entry(V value, long expiresAtEpochMs) { + this.value = value; + this.expiresAtEpochMs = expiresAtEpochMs; + } + boolean expired(long nowMs) { return nowMs >= expiresAtEpochMs; } + } + + private final ConcurrentHashMap> map = new ConcurrentHashMap<>(); + private final long ttlMs; + private final int maxSize; + + public ExpiringCache(Duration ttl, int maxSize) { + Objects.requireNonNull(ttl, "ttl"); + this.ttlMs = Math.max(0, ttl.toMillis()); + this.maxSize = Math.max(0, maxSize); + } + + public V get(K key) { + if (key == null) return null; + Entry e = map.get(key); + if (e == null) return null; + long now = System.currentTimeMillis(); + if (e.expired(now)) { + map.remove(key, e); + return null; + } + return e.value; + } + + public void put(K key, V value) { + if (key == null) return; + if (ttlMs <= 0) { + // TTL disabled: behave like no-cache. + return; + } + long now = System.currentTimeMillis(); + long exp = now + ttlMs; + map.put(key, new Entry<>(value, exp)); + enforceMaxSizeBestEffort(); + } + + public V computeIfAbsent(K key, java.util.function.Supplier supplier) { + V v = get(key); + if (v != null) return v; + V computed = supplier.get(); + put(key, computed); + return computed; + } + + /** Removes expired entries and, if still above maxSize, evicts a few random entries. */ + private void enforceMaxSizeBestEffort() { + if (maxSize <= 0) return; + int size = map.size(); + if (size <= maxSize) return; + + long now = System.currentTimeMillis(); + // 1) Remove expired entries first + for (var it = map.entrySet().iterator(); it.hasNext(); ) { + var e = it.next(); + if (e.getValue() == null || e.getValue().expired(now)) { + it.remove(); + } + } + + // 2) If still too big, evict some random entries (prototype-grade) + size = map.size(); + if (size <= maxSize) return; + int toEvict = Math.max(1, (size - maxSize)); + Object[] keys = map.keySet().toArray(); + for (int i = 0; i < toEvict && keys.length > 0; i++) { + int idx = ThreadLocalRandom.current().nextInt(keys.length); + @SuppressWarnings("unchecked") + K k = (K) keys[idx]; + map.remove(k); + } + } + + public int size() { return map.size(); } +} diff --git a/src/main/java/at/co/procon/malis/cep/util/InsecureSsl.java b/src/main/java/at/co/procon/malis/cep/util/InsecureSsl.java new file mode 100644 index 0000000..f3d4403 --- /dev/null +++ b/src/main/java/at/co/procon/malis/cep/util/InsecureSsl.java @@ -0,0 +1,31 @@ +package at.co.procon.malis.cep.util; + +import javax.net.ssl.SSLContext; +import javax.net.ssl.SSLSocketFactory; +import javax.net.ssl.TrustManager; +import javax.net.ssl.X509TrustManager; +import java.security.SecureRandom; +import java.security.cert.X509Certificate; + +public final class InsecureSsl { + + private InsecureSsl() {} + + public static SSLSocketFactory trustAllSocketFactory() { + try { + TrustManager[] trustAll = new TrustManager[]{ + new X509TrustManager() { + @Override public void checkClientTrusted(X509Certificate[] chain, String authType) {} + @Override public void checkServerTrusted(X509Certificate[] chain, String authType) {} + @Override public X509Certificate[] getAcceptedIssuers() { return new X509Certificate[0]; } + } + }; + + SSLContext ctx = SSLContext.getInstance("TLS"); + ctx.init(null, trustAll, new SecureRandom()); + return ctx.getSocketFactory(); + } catch (Exception e) { + throw new IllegalStateException("Failed to create insecure SSL socket factory", e); + } + } +} diff --git a/src/main/java/at/co/procon/malis/cep/util/TemplateUtil.java b/src/main/java/at/co/procon/malis/cep/util/TemplateUtil.java new file mode 100644 index 0000000..6cdcacc --- /dev/null +++ b/src/main/java/at/co/procon/malis/cep/util/TemplateUtil.java @@ -0,0 +1,21 @@ +package at.co.procon.malis.cep.util; + +import java.util.Map; + +/** Very small ${var} template helper for topic/routingKey templates. */ +public final class TemplateUtil { + private TemplateUtil() {} + + public static String expand(String template, Map vars) { + if (template == null) return null; + String out = template; + if (vars != null) { + for (var e : vars.entrySet()) { + String k = e.getKey(); + String v = e.getValue() == null ? "" : e.getValue(); + out = out.replace("${" + k + "}", v); + } + } + return out; + } +} diff --git a/src/main/java/at/co/procon/malis/cep/web/TestIngressController.java b/src/main/java/at/co/procon/malis/cep/web/TestIngressController.java new file mode 100644 index 0000000..0cff2df --- /dev/null +++ b/src/main/java/at/co/procon/malis/cep/web/TestIngressController.java @@ -0,0 +1,46 @@ +package at.co.procon.malis.cep.web; + +import at.co.procon.malis.cep.binding.IngressMessage; +import at.co.procon.malis.cep.config.CepProperties; +import at.co.procon.malis.cep.pipeline.CepPipeline; +import jakarta.servlet.http.HttpServletRequest; +import org.springframework.http.MediaType; +import org.springframework.web.bind.annotation.*; + +import java.io.IOException; +import java.util.Enumeration; +import java.util.HashMap; +import java.util.Map; + +@RestController +@RequestMapping(path = "/api/test") +public class TestIngressController { + + private final CepPipeline pipeline; + + public TestIngressController(CepPipeline pipeline) { + this.pipeline = pipeline; + } + + @PostMapping(path = "/ingress", consumes = MediaType.ALL_VALUE, produces = MediaType.APPLICATION_JSON_VALUE) + public CepPipeline.PipelineResult ingress(@RequestParam(defaultValue = "MQTT") CepProperties.InType inType, + @RequestParam(name = "path") String pathOrTopicOrQueue, + HttpServletRequest request) throws IOException { + + byte[] payload = request.getInputStream().readAllBytes(); + + Map headers = new HashMap<>(); + Enumeration names = request.getHeaderNames(); + while (names.hasMoreElements()) { + String n = names.nextElement(); + headers.put(n, request.getHeader(n)); + } + + return pipeline.process(new IngressMessage( + inType, + pathOrTopicOrQueue, + headers, + payload + )); + } +} diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml new file mode 100644 index 0000000..61d503d --- /dev/null +++ b/src/main/resources/application.yml @@ -0,0 +1,17 @@ +spring: + application: + name: malis-cep + profiles: + default: dev + active: dev + config: + import: + - "optional:file:./config/cep/common.yml" + - "optional:file:./config/cep/env/${spring.profiles.active}.yml" + +server: + port: 8084 + +cep: + bindingLocations: + - "file:./config/cep/bindings/*.yml" \ No newline at end of file diff --git a/src/test/java/at/co/procon/malis/cep/detector/ExpressionRulesDetectorTest.java b/src/test/java/at/co/procon/malis/cep/detector/ExpressionRulesDetectorTest.java new file mode 100644 index 0000000..168ac14 --- /dev/null +++ b/src/test/java/at/co/procon/malis/cep/detector/ExpressionRulesDetectorTest.java @@ -0,0 +1,254 @@ +package at.co.procon.malis.cep.detector; + +import at.co.procon.malis.cep.eebus.EebusMeasurementDatagram; +import at.co.procon.malis.cep.parser.MeasurementSet; +import at.co.procon.malis.cep.parser.MeasurementValue; +import at.co.procon.malis.cep.parser.ParsedInput; +import org.junit.jupiter.api.Test; +import org.springframework.expression.spel.standard.SpelExpressionParser; + +import java.nio.charset.StandardCharsets; +import java.time.Instant; +import java.util.List; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.*; + +public class ExpressionRulesDetectorTest { + + @Test + void eebusDatagram_isConverted_andRuleEvaluated() throws Exception { + String xml1 = new String( + getClass().getResourceAsStream("/eebus/measurement-datagram.xml").readAllBytes(), + StandardCharsets.UTF_8 + ); + + // Second message: values only (no description list) + String xml2 = """ + +

+ device-123 + malis-cep +
+ + + + + port.1.state + false + + + + + + """; + + var spel = new SpelExpressionParser(); + var rules = List.of( + new ExpressionRulesDetector.Rule( + "p1_closed", + "s['port.1.state'] == true", + spel.parseExpression("s['port.1.state'] == true"), + "MAJOR", + true + ) + ); + + ExpressionRulesDetector det = new ExpressionRulesDetector(rules, ExpressionRulesDetector.MissingPolicy.FALSE); + + DetectionContext ctx = new DetectionContext("bind1", Map.of( + "tenant", "t1", + "site", "s1", + "sourceKey", "MQTT:bind1:topic/device-123" + )); + + ParsedInput in1 = new ParsedInput( + "EEBUS_MEASUREMENT_DATAGRAM_XML", + new EebusMeasurementDatagram(xml1, null, null, "device-123", "malis-cep"), + Instant.now(), + Map.of() + ); + + List ev1 = det.detect(in1, ctx); + assertEquals(1, ev1.size()); + assertEquals("ALARM", ev1.get(0).getEventType()); + + ParsedInput in2 = new ParsedInput( + "EEBUS_MEASUREMENT_DATAGRAM_XML", + new EebusMeasurementDatagram(xml2, null, null, "device-123", "malis-cep"), + Instant.now(), + Map.of() + ); + + List ev2 = det.detect(in2, ctx); + assertEquals(1, ev2.size()); + assertEquals("CANCEL", ev2.get(0).getEventType()); + + // Same faultKey (ruleId + sourceKey + bindingId) => lifecycle can manage transitions + assertEquals(ev1.get(0).getFaultKey(), ev2.get(0).getFaultKey()); + } + + @Test + void doubleRules_greaterThan_and_lessThan() throws Exception { + var spel = new SpelExpressionParser(); + var rules = List.of( + new ExpressionRulesDetector.Rule( + "temp_gt_50", + "(s['temp'] ?: 0) > 50", + spel.parseExpression("(s['temp'] ?: 0) > 50"), + "MAJOR", + true + ), + new ExpressionRulesDetector.Rule( + "temp_lt_10", + "(s['temp'] ?: 0) < 10", + spel.parseExpression("(s['temp'] ?: 0) < 10"), + "MINOR", + true + ) + ); + + ExpressionRulesDetector det = new ExpressionRulesDetector(rules, ExpressionRulesDetector.MissingPolicy.FALSE); + DetectionContext ctx = new DetectionContext("bindTemp", Map.of( + "tenant", "t1", + "site", "s1", + "sourceKey", "HTTP:bindTemp:/ingress/temp" + )); + + // temp = 60 -> gt_50 ALARM, lt_10 CANCEL + ParsedInput in1 = new ParsedInput( + "MEASUREMENT_SET", + new MeasurementSet("device-1", Instant.now(), List.of(new MeasurementValue("temp", 60.0))), + Instant.now(), + Map.of() + ); + var out1 = det.detect(in1, ctx); + assertEquals(2, out1.size()); + assertEvent(out1, "temp_gt_50", "ALARM"); + assertEvent(out1, "temp_lt_10", "CANCEL"); + + // temp = 5 -> gt_50 CANCEL, lt_10 ALARM + ParsedInput in2 = new ParsedInput( + "MEASUREMENT_SET", + new MeasurementSet("device-1", Instant.now(), List.of(new MeasurementValue("temp", 5.0))), + Instant.now(), + Map.of() + ); + var out2 = det.detect(in2, ctx); + assertEquals(2, out2.size()); + assertEvent(out2, "temp_gt_50", "CANCEL"); + assertEvent(out2, "temp_lt_10", "ALARM"); + } + + @Test + void expressionWithTwoMeasurements() throws Exception { + var spel = new SpelExpressionParser(); + var rules = List.of( + new ExpressionRulesDetector.Rule( + "temp_high_and_pressure_low", + "(s['temp'] ?: 0) > 50 && (s['pressure'] ?: 0) < 1.2", + spel.parseExpression("(s['temp'] ?: 0) > 50 && (s['pressure'] ?: 0) < 1.2"), + "MAJOR", + true + ) + ); + + ExpressionRulesDetector det = new ExpressionRulesDetector(rules, ExpressionRulesDetector.MissingPolicy.FALSE); + DetectionContext ctx = new DetectionContext("bind2", Map.of( + "tenant", "t1", + "site", "s1", + "sourceKey", "MQTT:bind2:topic/device-77" + )); + + // Both conditions satisfied -> ALARM + ParsedInput in1 = new ParsedInput( + "MEASUREMENT_SET", + new MeasurementSet("device-77", Instant.now(), List.of( + new MeasurementValue("temp", 60.0), + new MeasurementValue("pressure", 1.0) + )), + Instant.now(), + Map.of() + ); + var out1 = det.detect(in1, ctx); + assertEquals(1, out1.size()); + assertEquals("ALARM", out1.get(0).getEventType()); + + // pressure changes -> condition no longer satisfied -> CANCEL + ParsedInput in2 = new ParsedInput( + "MEASUREMENT_SET", + new MeasurementSet("device-77", Instant.now(), List.of( + new MeasurementValue("pressure", 2.0) + )), + Instant.now(), + Map.of() + ); + var out2 = det.detect(in2, ctx); + assertEquals(1, out2.size()); + assertEquals("CANCEL", out2.get(0).getEventType()); + assertEquals(out1.get(0).getFaultKey(), out2.get(0).getFaultKey()); + } + + @Test + void partitionsBySourceKey_sameDeviceId_doesNotCrossTalk() throws Exception { + var spel = new SpelExpressionParser(); + var rules = List.of( + new ExpressionRulesDetector.Rule( + "temp_gt_50", + "(s['temp'] ?: 0) > 50", + spel.parseExpression("(s['temp'] ?: 0) > 50"), + "MAJOR", + true + ) + ); + + ExpressionRulesDetector det = new ExpressionRulesDetector(rules, ExpressionRulesDetector.MissingPolicy.FALSE); + + DetectionContext ctxA = new DetectionContext("bindP", Map.of( + "tenant", "t1", + "site", "s1", + "sourceKey", "MQTT:bindP:topic/source-A" + )); + DetectionContext ctxB = new DetectionContext("bindP", Map.of( + "tenant", "t1", + "site", "s1", + "sourceKey", "MQTT:bindP:topic/source-B" + )); + + // A reports temp=60 -> ALARM + ParsedInput a1 = new ParsedInput( + "MEASUREMENT_SET", + new MeasurementSet("same-device-id", Instant.now(), List.of(new MeasurementValue("temp", 60.0))), + Instant.now(), + Map.of() + ); + var outA = det.detect(a1, ctxA); + assertEquals(1, outA.size()); + assertEquals("ALARM", outA.get(0).getEventType()); + + // B sends an unrelated update. If the detector were partitioning by deviceId, it would see temp=60 and ALARM. + ParsedInput b1 = new ParsedInput( + "MEASUREMENT_SET", + new MeasurementSet("same-device-id", Instant.now(), List.of(new MeasurementValue("other", 1.0))), + Instant.now(), + Map.of() + ); + var outB = det.detect(b1, ctxB); + assertEquals(1, outB.size()); + assertEquals("CANCEL", outB.get(0).getEventType()); + + // Fault keys differ because they include sourceKey. + assertNotEquals(outA.get(0).getFaultKey(), outB.get(0).getFaultKey()); + } + + private static void assertEvent(List events, String ruleId, String expectedType) { + DetectorEvent ev = events.stream() + .filter(e -> { + Object d = e.getDetails() == null ? null : e.getDetails().get("ruleId"); + return ruleId.equals(d); + }) + .findFirst() + .orElseThrow(() -> new AssertionError("Missing event for ruleId=" + ruleId)); + assertEquals(expectedType, ev.getEventType(), "ruleId=" + ruleId); + } +} diff --git a/src/test/java/at/co/procon/malis/cep/detector/ExternalRestEventsDetectorWrapperTest.java b/src/test/java/at/co/procon/malis/cep/detector/ExternalRestEventsDetectorWrapperTest.java new file mode 100644 index 0000000..c516560 --- /dev/null +++ b/src/test/java/at/co/procon/malis/cep/detector/ExternalRestEventsDetectorWrapperTest.java @@ -0,0 +1,51 @@ +package at.co.procon.malis.cep.detector; + +import at.co.procon.malis.cep.binding.IngressMessage; +import at.co.procon.malis.cep.binding.ResolvedBinding; +import at.co.procon.malis.cep.config.CepProperties; +import at.co.procon.malis.cep.eebus.EebusMeasurementDatagram; +import at.co.procon.malis.cep.eebus.EebusSpineXsdValidatorBase; +import at.co.procon.malis.cep.parser.EebusMeasurementDatagramXmlParser; +import at.co.procon.malis.cep.parser.ParsedInput; +import org.junit.jupiter.api.Test; + +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.*; + +/** + * Tests for the optional XML wrapper request format. + */ +public class ExternalRestEventsDetectorWrapperTest { + + private final EebusSpineXsdValidatorBase xsdValidator; + + public ExternalRestEventsDetectorWrapperTest(EebusSpineXsdValidatorBase xsdValidator) { + this.xsdValidator = xsdValidator; + } + + @Test + void buildsWrapperContainingBothMeasurementLists() throws Exception { + byte[] xml = resource("/eebus/measurement-datagram.xml"); + IngressMessage msg = new IngressMessage(CepProperties.InType.MQTT, "measurements/...", Map.of(), xml); + ParsedInput parsed = new EebusMeasurementDatagramXmlParser(xsdValidator).parse(msg, new ResolvedBinding("b", null, Map.of())); + + assertTrue(parsed.getBody() instanceof EebusMeasurementDatagram); + EebusMeasurementDatagram md = (EebusMeasurementDatagram) parsed.getBody(); + + String wrapper = ExternalRestEventsDetector.buildListsWrapperXml(md); + assertTrue(wrapper.contains(" vars = Map.of( + "tenant", "t1", + "site", "s1", + "department", "d1", + "deviceId", "dev-1" + ); + + // Output config provides addressing templates. + Map outCfg = Map.of( + "addressSourceTemplate", "${deviceId}", + "addressDestinationTemplate", "malis-cep" + ); + + FormattedMessage msg = new EebusAlarmDatagramXmlFormatter(xsdValidator).format(a, vars, outCfg); + String xml = new String(msg.getPayload(), StandardCharsets.UTF_8); + + assertEquals("application/xml", msg.getContentType()); + assertTrue(xml.contains("")); + assertTrue(xml.contains("write")); + assertTrue(xml.contains("dev-1")); + assertTrue(xml.contains("malis-cep")); + } +} diff --git a/src/test/java/at/co/procon/malis/cep/parser/EebusAlarmDatagramXmlParserPreserveFieldsTest.java b/src/test/java/at/co/procon/malis/cep/parser/EebusAlarmDatagramXmlParserPreserveFieldsTest.java new file mode 100644 index 0000000..c08e1a9 --- /dev/null +++ b/src/test/java/at/co/procon/malis/cep/parser/EebusAlarmDatagramXmlParserPreserveFieldsTest.java @@ -0,0 +1,99 @@ +package at.co.procon.malis.cep.parser; + +import at.co.procon.malis.cep.binding.IngressMessage; +import at.co.procon.malis.cep.binding.ResolvedBinding; +import at.co.procon.malis.cep.config.CepProperties; +import at.co.procon.malis.cep.eebus.EebusSpineXsdValidator; +import org.junit.jupiter.api.Test; + +import java.nio.charset.StandardCharsets; +import java.time.Instant; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.*; + +public class EebusAlarmDatagramXmlParserPreserveFieldsTest { + + private final EebusSpineXsdValidator xsdValidator; + + public EebusAlarmDatagramXmlParserPreserveFieldsTest(EebusSpineXsdValidator xsdValidator) { + this.xsdValidator = xsdValidator; + } + + @Test + void preservesAlarmFieldsFromAlarmData() throws Exception { + String xml = "\n" + + "\n" + + " \n" + + " \n" + + " alarmListData\n" + + " \n" + + " \n" + + " 1\n" + + " 10\n" + + " 2025-12-02T10:15:00Z\n" + + " overThreshold\n" + + " 2300-1\n" + + " 2025-12-02T10:10:00Z2025-12-02T10:15:00Z\n" + + " acPowerTotal\n" + + " AC power above limit\n" + + " Exceeded upper limit.\n" + + " \n" + + " \n" + + " \n" + + " \n" + + "\n"; + + IngressMessage msg = new IngressMessage(CepProperties.InType.HTTP, "/rest/alarms/siteA/depB/dev1", Map.of(), xml.getBytes(StandardCharsets.UTF_8)); + ResolvedBinding rb = new ResolvedBinding("http-eebus-alarm", null, Map.of( + "tenant", "default", + "site", "siteA", + "department", "depB", + "deviceId", "d:_i:2_HeatPump" + )); + + ParsedInput parsed = new EebusAlarmDatagramXmlParser(xsdValidator).parse(msg, rb); + assertTrue(parsed.getBody() instanceof AlarmBatch); + + AlarmBatch batch = (AlarmBatch) parsed.getBody(); + assertEquals(1, batch.getAlarms().size()); + + AlarmInput ai = batch.getAlarms().get(0); + assertEquals("ALARM", ai.getAction()); + assertEquals("acPowerTotal", ai.getAlarmCode()); + assertEquals("d:_i:2_HeatPump", ai.getDeviceId()); + + // Timestamp should be parsed from XML + assertEquals(Instant.parse("2025-12-02T10:15:00Z"), ai.getTimestamp()); + + // Details preserved + assertEquals("1", String.valueOf(ai.getDetails().get("alarmId"))); + assertEquals("10", String.valueOf(ai.getDetails().get("thresholdId"))); + assertEquals("overThreshold", String.valueOf(ai.getDetails().get("alarmType"))); + assertEquals("2300", String.valueOf(ai.getDetails().get("measuredValue.number"))); + assertEquals("-1", String.valueOf(ai.getDetails().get("measuredValue.scale"))); + assertEquals("2025-12-02T10:10:00Z", String.valueOf(ai.getDetails().get("evaluationPeriod.startTime"))); + assertEquals("2025-12-02T10:15:00Z", String.valueOf(ai.getDetails().get("evaluationPeriod.endTime"))); + assertEquals("AC power above limit", String.valueOf(ai.getDetails().get("label"))); + } + + @Test + void cancelIsDetectedByAlarmCancelled() throws Exception { + String xml = "\n" + + "\n" + + " alarmListData\n" + + " \n" + + " 1\n" + + " 2025-12-02T12:00:00Z\n" + + " alarmCancelled\n" + + " \n" + + "\n"; + + IngressMessage msg = new IngressMessage(CepProperties.InType.HTTP, "/rest/alarms/siteA/depB/dev1", Map.of(), xml.getBytes(StandardCharsets.UTF_8)); + ResolvedBinding rb = new ResolvedBinding("http-eebus-alarm", null, Map.of("deviceId", "dev1")); + + ParsedInput parsed = new EebusAlarmDatagramXmlParser(xsdValidator).parse(msg, rb); + AlarmBatch batch = (AlarmBatch) parsed.getBody(); + assertEquals("CANCEL", batch.getAlarms().get(0).getAction()); + } +} \ No newline at end of file diff --git a/src/test/java/at/co/procon/malis/cep/parser/EebusDatagramParsersTest.java b/src/test/java/at/co/procon/malis/cep/parser/EebusDatagramParsersTest.java new file mode 100644 index 0000000..ec4f340 --- /dev/null +++ b/src/test/java/at/co/procon/malis/cep/parser/EebusDatagramParsersTest.java @@ -0,0 +1,79 @@ +package at.co.procon.malis.cep.parser; + +import at.co.procon.malis.cep.binding.IngressMessage; +import at.co.procon.malis.cep.binding.ResolvedBinding; +import at.co.procon.malis.cep.config.CepProperties; +import at.co.procon.malis.cep.eebus.EebusMeasurementDatagram; +import at.co.procon.malis.cep.eebus.EebusSpineXsdValidator; +import at.co.procon.malis.cep.eebus.EebusSpineXsdValidatorBase; + +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.*; + +/** + * Unit tests for parsing full EEBUS-like <Datagram> XML payloads. + */ +public class EebusDatagramParsersTest { + + private final EebusSpineXsdValidatorBase xsdValidator; + + public EebusDatagramParsersTest(EebusSpineXsdValidatorBase xsdValidator) { + this.xsdValidator = xsdValidator; + } + + //@Test + void parsesMeasurementDatagramAndExtractsFragments() throws Exception { + byte[] xml = resource("/eebus/measurement-datagram.xml"); + IngressMessage msg = new IngressMessage(CepProperties.InType.MQTT, "/measurements/site/dep/dev", Map.of(), xml); + ResolvedBinding rb = new ResolvedBinding("b1", null, Map.of()); + + ParsedInput out = new EebusMeasurementDatagramXmlParser(xsdValidator).parse(msg, rb); + assertEquals("EEBUS_MEASUREMENT_DATAGRAM_XML", out.getPayloadType()); + + assertTrue(out.getBody() instanceof EebusMeasurementDatagram); + EebusMeasurementDatagram md = (EebusMeasurementDatagram) out.getBody(); + + assertNotNull(md.getDatagramXml()); + assertTrue(md.getDatagramXml().contains(" + +
+ device-123 + malis-cep +
+ + + write + + + A1 + PORT_OPEN + ALARM + + + + + diff --git a/src/test/resources/eebus/alarm-datagram-cancel.xml b/src/test/resources/eebus/alarm-datagram-cancel.xml new file mode 100644 index 0000000..535f863 --- /dev/null +++ b/src/test/resources/eebus/alarm-datagram-cancel.xml @@ -0,0 +1,18 @@ + + +
+ device-123 + malis-cep +
+ + + delete + + + A1 + PORT_OPEN + + + + + diff --git a/src/test/resources/eebus/measurement-datagram.xml b/src/test/resources/eebus/measurement-datagram.xml new file mode 100644 index 0000000..c6dbd4e --- /dev/null +++ b/src/test/resources/eebus/measurement-datagram.xml @@ -0,0 +1,30 @@ + + + +
+ device-123 + malis-cep +
+ + + + + port.1.state + boolean + + + + + port.1.state + true + + + + +