TriTih
/
eventhub
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Compare commits

base: TriTih:14a6f8d42ea865fdefa7cb1d24cfa9edc38e18f9
Branches Tags
TriTih:main
...
compare: TriTih:e84dfef614b18c60f1540ffc17e83e946ca68a1a
Branches Tags
TriTih:main

5 Commits
14a6f8d42e ... e84dfef614

Author SHA1 Message Date
trifonovt e84dfef614 Fix YellowFox bounded import cursor and ignition handling 2026-05-08 16:50:34 +02:00
trifonovt a0242eedee Adjust import runtime defaults 2026-05-08 13:19:51 +02:00
trifonovt ddc45f3c30 Add full-EPL operating period mode 2026-05-08 13:19:34 +02:00
trifonovt 519711b214 Add YellowFox master-data refresh flow 2026-05-08 13:19:25 +02:00
trifonovt de9c884578 Introduce driver card identity model 2026-05-08 13:19:00 +02:00
50 changed files with 4798 additions and 777 deletions
Show Stats Expand all files Collapse all files

265
docs/db/create_eventhub_schema.sql
View File

@ -1,6 +1,8 @@
create extension if not exists pgcrypto;
create extension if not exists postgis;
create extension if not exists timescaledb;
drop schema if exists eventhub cascade;
create schema if not exists eventhub;
create table if not exists eventhub.event_source (
@ -143,11 +145,56 @@ create table if not exists eventhub.source_master_relation (
constraint chk_source_master_relation_valid_time_order check (valid_from is null or valid_to is null or valid_from <= valid_to)
);
create table if not exists eventhub.vehicle (
create table if not exists eventhub.driver (
id uuid primary key,
first_names text,
last_name text,
birth_date date,
source_updated_at timestamptz,
payload jsonb not null default '{}'::jsonb,
created_at timestamptz not null default now(),
updated_at timestamptz not null default now()
);
create table if not exists eventhub.driver_card (
id uuid primary key,
driver_id uuid references eventhub.driver(id),
nation text not null,
card_number text not null,
source_updated_at timestamptz,
payload jsonb not null default '{}'::jsonb,
created_at timestamptz not null default now(),
updated_at timestamptz not null default now()
);
create table if not exists eventhub.source_driver_identity (
id uuid primary key,
tenant_key text not null,
event_source_id integer not null references eventhub.event_source(id),
source_vehicle_entity_id text,
source_driver_entity_id text not null,
driver_id uuid not null references eventhub.driver(id) on delete cascade,
source_updated_at timestamptz,
payload jsonb not null default '{}'::jsonb,
created_at timestamptz not null default now(),
updated_at timestamptz not null default now(),
constraint ux_source_driver_identity unique (tenant_key, event_source_id, source_driver_entity_id)
);
create table if not exists eventhub.source_driver_card_identity (
id uuid primary key,
tenant_key text not null,
event_source_id integer not null references eventhub.event_source(id),
source_driver_card_entity_id text not null,
driver_card_id uuid not null references eventhub.driver_card(id) on delete cascade,
source_updated_at timestamptz,
payload jsonb not null default '{}'::jsonb,
created_at timestamptz not null default now(),
updated_at timestamptz not null default now(),
constraint ux_source_driver_card_identity unique (tenant_key, event_source_id, source_driver_card_entity_id)
);
create table if not exists eventhub.vehicle (
id uuid primary key,
vin text,
created_at timestamptz not null default now(),
updated_at timestamptz not null default now()
@ -155,9 +202,6 @@ create table if not exists eventhub.vehicle (
create table if not exists eventhub.vehicle_registration (
id uuid primary key,
tenant_key text not null,
event_source_id integer not null references eventhub.event_source(id),
source_registration_entity_id text,
nation text not null,
registration_number text not null,
source_updated_at timestamptz,
@ -166,6 +210,32 @@ create table if not exists eventhub.vehicle_registration (
updated_at timestamptz not null default now()
);
create table if not exists eventhub.source_vehicle_identity (
id uuid primary key,
tenant_key text not null,
event_source_id integer not null references eventhub.event_source(id),
source_vehicle_entity_id text not null,
vehicle_id uuid not null references eventhub.vehicle(id) on delete cascade,
source_updated_at timestamptz,
payload jsonb not null default '{}'::jsonb,
created_at timestamptz not null default now(),
updated_at timestamptz not null default now(),
constraint ux_source_vehicle_identity unique (tenant_key, event_source_id, source_vehicle_entity_id)
);
create table if not exists eventhub.source_vehicle_registration_identity (
id uuid primary key,
tenant_key text not null,
event_source_id integer not null references eventhub.event_source(id),
source_registration_entity_id text not null,
vehicle_registration_id uuid not null references eventhub.vehicle_registration(id) on delete cascade,
source_updated_at timestamptz,
payload jsonb not null default '{}'::jsonb,
created_at timestamptz not null default now(),
updated_at timestamptz not null default now(),
constraint ux_source_vehicle_registration_identity unique (tenant_key, event_source_id, source_registration_entity_id)
);
create table if not exists eventhub.vehicle_registration_assignment (
id uuid primary key,
tenant_key text not null,
@ -186,9 +256,11 @@ create table if not exists eventhub.event (
event_source_id integer not null references eventhub.event_source(id),
data_package_id uuid not null references eventhub.data_package(id),
external_source_event_id text not null,
driver_entity_id uuid references eventhub.source_master_entity(id),
driver_id uuid references eventhub.driver(id),
driver_card_id uuid references eventhub.driver_card(id),
vehicle_id uuid references eventhub.vehicle(id),
vehicle_registration_id uuid references eventhub.vehicle_registration(id),
source_package_id text,
source_package_entity_id uuid references eventhub.source_master_entity(id),
occurred_at timestamptz not null,
received_partner_at timestamptz,
@ -205,26 +277,90 @@ create table if not exists eventhub.event (
created_at timestamptz not null default now(),
constraint pk_event primary key (occurred_at, id),
constraint chk_event_driver_or_vehicle_ref check (
driver_entity_id is not null
driver_id is not null
or driver_card_id is not null
or vehicle_id is not null
or vehicle_registration_id is not null
)
);
create table if not exists eventhub.event_source_record (
source_record_key_hash text primary key,
event_occurred_at timestamptz not null,
event_id uuid not null,
created_at timestamptz not null default now()
);
create table if not exists eventhub.event_detail (
event_occurred_at timestamptz not null,
event_id uuid not null,
detail_type text not null,
attributes jsonb not null default '{}'::jsonb,
created_at timestamptz not null default now(),
constraint pk_event_detail primary key (event_occurred_at, event_id, detail_type),
constraint fk_event_detail_event foreign key (event_occurred_at, event_id)
references eventhub.event(occurred_at, id)
on delete cascade
constraint pk_event_detail primary key (event_occurred_at, event_id, detail_type)
);
create unique index if not exists ux_event_source_record
on eventhub.event(source_record_key_hash);
select create_hypertable(
'eventhub.event',
'occurred_at',
chunk_time_interval => interval '7 days',
if_not_exists => true
);
alter table eventhub.event_source_record
add constraint fk_event_source_record_event foreign key (event_occurred_at, event_id)
references eventhub.event(occurred_at, id)
on delete cascade
deferrable initially deferred;
alter table eventhub.event_detail
add constraint fk_event_detail_event foreign key (event_occurred_at, event_id)
references eventhub.event(occurred_at, id)
on delete cascade;
create index if not exists idx_data_package_source_time
on eventhub.data_package(tenant_key, event_source_id, received_at desc);
create index if not exists idx_data_package_scope
on eventhub.data_package(tenant_key, import_scope_type, root_source_org_entity_id, occurred_from, occurred_to);
create index if not exists idx_data_package_extraction
on eventhub.data_package(tenant_key, event_source_id, import_run_id, event_family, extraction_source_kind, extraction_code, batch_no);
create index if not exists idx_import_run_source_status
on eventhub.import_run(tenant_key, event_source_id, status, started_at desc);
create index if not exists idx_source_master_entity_type_key
on eventhub.source_master_entity(tenant_key, event_source_id, entity_type, source_external_key)
where source_external_key is not null;
create index if not exists idx_source_master_entity_payload_gin
on eventhub.source_master_entity using gin(payload);
create index if not exists idx_source_master_relation_from
on eventhub.source_master_relation(tenant_key, event_source_id, from_entity_type, from_source_entity_id, relation_type);
create index if not exists idx_source_master_relation_to
on eventhub.source_master_relation(tenant_key, event_source_id, to_entity_type, to_source_entity_id, relation_type);
create index if not exists idx_source_master_relation_payload_gin
on eventhub.source_master_relation using gin(payload);
create index if not exists idx_vehicle_vin
on eventhub.vehicle(vin)
where vin is not null;
create index if not exists idx_vehicle_registration_plate
on eventhub.vehicle_registration(nation, registration_number);
create index if not exists idx_vehicle_registration_assignment_registration_time
on eventhub.vehicle_registration_assignment(vehicle_registration_id, valid_from desc, valid_to);
create index if not exists idx_vehicle_registration_assignment_vehicle_time
on eventhub.vehicle_registration_assignment(vehicle_id, valid_from desc, valid_to);
create index if not exists idx_event_source_record_event
on eventhub.event_source_record(event_occurred_at, event_id);
create index if not exists idx_event_signature
on eventhub.event(event_signature_hash)
@ -236,12 +372,49 @@ create index if not exists idx_event_source_time
create index if not exists idx_event_package_time
on eventhub.event(data_package_id, occurred_at desc);
create index if not exists idx_event_source_package_id
on eventhub.event(source_package_id)
where source_package_id is not null;
create index if not exists idx_event_domain_type_time
on eventhub.event(event_domain, event_type, occurred_at desc);
create index if not exists idx_driver_card_key
on eventhub.driver_card(nation, card_number);
create index if not exists idx_driver_card_driver
on eventhub.driver_card(driver_id)
where driver_id is not null;
create unique index if not exists ux_driver_card_key
on eventhub.driver_card(nation, card_number);
create unique index if not exists ux_vehicle_vin
on eventhub.vehicle(vin)
where vin is not null;
create unique index if not exists ux_vehicle_registration_plate
on eventhub.vehicle_registration(nation, registration_number);
create index if not exists idx_source_driver_identity_driver
on eventhub.source_driver_identity(driver_id);
create index if not exists idx_source_driver_card_identity_card
on eventhub.source_driver_card_identity(driver_card_id);
create index if not exists idx_source_vehicle_identity_vehicle
on eventhub.source_vehicle_identity(vehicle_id);
create index if not exists idx_source_vehicle_registration_identity_registration
on eventhub.source_vehicle_registration_identity(vehicle_registration_id);
create index if not exists idx_event_driver_time
on eventhub.event(driver_entity_id, occurred_at desc)
where driver_entity_id is not null;
on eventhub.event(driver_id, occurred_at desc)
where driver_id is not null;
create index if not exists idx_event_driver_card_time
on eventhub.event(driver_card_id, occurred_at desc)
where driver_card_id is not null;
create index if not exists idx_event_vehicle_time
on eventhub.event(vehicle_id, occurred_at desc)
@ -264,54 +437,18 @@ create index if not exists idx_event_detail_type
create index if not exists idx_event_detail_attributes_gin
on eventhub.event_detail using gin(attributes);
create index if not exists idx_source_master_entity_type_key
on eventhub.source_master_entity(tenant_key, event_source_id, entity_type, source_external_key)
where source_external_key is not null;
create index if not exists idx_event_detail_yellowfox_slot
on eventhub.event_detail(detail_type, (attributes ->> 'slot'), event_occurred_at)
where detail_type in ('DRIVER_ACTIVITY', 'DRIVER_CARD');
create index if not exists idx_source_master_entity_payload_gin
on eventhub.source_master_entity using gin(payload);
create index if not exists idx_event_detail_yellowfox_eventtype_state
on eventhub.event_detail(
(attributes ->> 'yellowFoxEventType'),
(attributes ->> 'yellowFoxState'),
event_occurred_at
)
where attributes ? 'yellowFoxEventType';
create index if not exists idx_source_master_relation_from
on eventhub.source_master_relation(tenant_key, event_source_id, from_entity_type, from_source_entity_id, relation_type);
create index if not exists idx_source_master_relation_to
on eventhub.source_master_relation(tenant_key, event_source_id, to_entity_type, to_source_entity_id, relation_type);
create index if not exists idx_source_master_relation_payload_gin
on eventhub.source_master_relation using gin(payload);
create index if not exists idx_vehicle_lookup_ctx
on eventhub.vehicle(tenant_key, event_source_id, updated_at desc);
create index if not exists idx_vehicle_source_entity
on eventhub.vehicle(tenant_key, event_source_id, source_vehicle_entity_id)
where source_vehicle_entity_id is not null;
create index if not exists idx_vehicle_vin
on eventhub.vehicle(tenant_key, event_source_id, vin)
where vin is not null;
create index if not exists idx_vehicle_registration_source_entity
on eventhub.vehicle_registration(tenant_key, event_source_id, source_registration_entity_id)
where source_registration_entity_id is not null;
create index if not exists idx_vehicle_registration_plate
on eventhub.vehicle_registration(tenant_key, event_source_id, nation, registration_number);
create index if not exists idx_vehicle_registration_assignment_registration_time
on eventhub.vehicle_registration_assignment(vehicle_registration_id, valid_from desc, valid_to);
create index if not exists idx_vehicle_registration_assignment_vehicle_time
on eventhub.vehicle_registration_assignment(vehicle_id, valid_from desc, valid_to);
create index if not exists idx_data_package_source_time
on eventhub.data_package(tenant_key, event_source_id, received_at desc);
create index if not exists idx_data_package_scope
on eventhub.data_package(tenant_key, import_scope_type, root_source_org_entity_id, occurred_from, occurred_to);
create index if not exists idx_data_package_extraction
on eventhub.data_package(tenant_key, event_source_id, import_run_id, event_family, extraction_source_kind, extraction_code, batch_no);
create index if not exists idx_import_run_source_status
on eventhub.import_run(tenant_key, event_source_id, status, started_at desc);
create index if not exists idx_event_detail_yellowfox_ignition
on eventhub.event_detail(detail_type, (attributes ->> 'ignitionState'), event_occurred_at)
where attributes ? 'ignitionState';

21
docs/sql/truncate-eventhub-data.sql Normal file
View File

@ -0,0 +1,21 @@
-- Truncate all application data in the eventhub schema.
-- Keeps the schema and Flyway migration history intact.
-- Intended for PostgreSQL / TimescaleDB environments.
DO $$
DECLARE
table_list text;
BEGIN
SELECT string_agg(format('%I.%I', schemaname, tablename), ', ' ORDER BY tablename)
INTO table_list
FROM pg_tables
WHERE schemaname = 'eventhub';
IF table_list IS NULL THEN
RAISE NOTICE 'No tables found in schema eventhub.';
RETURN;
END IF;
EXECUTE 'TRUNCATE TABLE ' || table_list || ' RESTART IDENTITY CASCADE';
END
$$;

217
postman/eventhub-rest-methods.postman_collection.json
View File

@ -12,6 +12,10 @@
{
"key": "planKey",
"value": "kralowetz-tachograph-org-147"
},
{
"key": "yellowFoxPlanKey",
"value": "yellowfox-d8-default"
}
],
"item": [
@ -376,6 +380,219 @@
]
}
}
},
{
"name": "POST /api/eventhub/acquisition/yellowfox/d8/imports/plan",
"request": {
"method": "POST",
"header": [
{
"key": "Content-Type",
"value": "application/json"
}
],
"body": {
"mode": "raw",
"raw": "{\n \"tenantKey\": \"Procon\",\n \"eventSource\": {\n \"providerKey\": \"YELLOWFOX\",\n \"sourceKind\": \"TELEMATICS_PLATFORM\",\n \"sourceKey\": \"YELLOWFOX_D8\",\n \"sourceInstanceKey\": \"logistics-db-prod\",\n \"tenantProviderSettingKey\": \"yellowfox-main\",\n \"externalFleetKey\": null\n },\n \"sourceGroup\": null,\n \"importScope\": {\n \"type\": \"TENANT_ALL\",\n \"rootSourceOrganisation\": null,\n \"includeChildren\": false,\n \"occurredFrom\": null,\n \"occurredTo\": null\n },\n \"eventFamilies\": [\n \"DRIVER_ACTIVITY\",\n \"DRIVER_CARD\"\n ],\n \"mode\": \"INCREMENTAL_UPDATE\",\n \"refreshMasterDataFirst\": false,\n \"acquisitionStrategy\": \"SOURCE_ROW_WATERMARK\"\n}"
},
"url": {
"raw": "{{baseUrl}}/api/eventhub/acquisition/yellowfox/d8/imports/plan",
"host": [
"{{baseUrl}}"
],
"path": [
"api",
"eventhub",
"acquisition",
"yellowfox",
"d8",
"imports",
"plan"
]
}
}
},
{
"name": "POST /api/eventhub/acquisition/yellowfox/d8/imports/start?execute=true",
"request": {
"method": "POST",
"header": [
{
"key": "Content-Type",
"value": "application/json"
}
],
"body": {
"mode": "raw",
"raw": "{\n \"tenantKey\": \"Procon\",\n \"eventSource\": {\n \"providerKey\": \"YELLOWFOX\",\n \"sourceKind\": \"TELEMATICS_PLATFORM\",\n \"sourceKey\": \"YELLOWFOX_D8\",\n \"sourceInstanceKey\": \"logistics-db-prod\",\n \"tenantProviderSettingKey\": \"yellowfox-main\",\n \"externalFleetKey\": null\n },\n \"sourceGroup\": null,\n \"importScope\": {\n \"type\": \"TENANT_ALL\",\n \"rootSourceOrganisation\": null,\n \"includeChildren\": false,\n \"occurredFrom\": null,\n \"occurredTo\": null\n },\n \"eventFamilies\": [\n \"DRIVER_ACTIVITY\",\n \"DRIVER_CARD\"\n ],\n \"mode\": \"INCREMENTAL_UPDATE\",\n \"refreshMasterDataFirst\": false,\n \"acquisitionStrategy\": \"SOURCE_ROW_WATERMARK\"\n}"
},
"url": {
"raw": "{{baseUrl}}/api/eventhub/acquisition/yellowfox/d8/imports/start?execute=true",
"host": [
"{{baseUrl}}"
],
"path": [
"api",
"eventhub",
"acquisition",
"yellowfox",
"d8",
"imports",
"start"
],
"query": [
{
"key": "execute",
"value": "true"
}
]
}
}
},
{
"name": "POST /api/eventhub/acquisition/yellowfox/d8/master-data/refresh",
"request": {
"method": "POST",
"header": [
{
"key": "Content-Type",
"value": "application/json"
}
],
"body": {
"mode": "raw",
"raw": "{\n \"tenantKey\": \"Procon\",\n \"eventSource\": {\n \"providerKey\": \"YELLOWFOX\",\n \"sourceKind\": \"TELEMATICS_PLATFORM\",\n \"sourceKey\": \"YELLOWFOX_D8\",\n \"sourceInstanceKey\": \"logistics-db-prod\",\n \"tenantProviderSettingKey\": \"yellowfox-main\",\n \"externalFleetKey\": null\n },\n \"sourceGroup\": null,\n \"importScope\": {\n \"type\": \"TENANT_ALL\",\n \"rootSourceOrganisation\": null,\n \"includeChildren\": false,\n \"occurredFrom\": null,\n \"occurredTo\": null\n },\n \"eventFamilies\": [\n \"DRIVER_ACTIVITY\",\n \"DRIVER_CARD\"\n ],\n \"mode\": \"INCREMENTAL_UPDATE\",\n \"refreshMasterDataFirst\": true,\n \"acquisitionStrategy\": \"SOURCE_ROW_WATERMARK\"\n}"
},
"url": {
"raw": "{{baseUrl}}/api/eventhub/acquisition/yellowfox/d8/master-data/refresh",
"host": [
"{{baseUrl}}"
],
"path": [
"api",
"eventhub",
"acquisition",
"yellowfox",
"d8",
"master-data",
"refresh"
]
}
}
},
{
"name": "GET /api/eventhub/acquisition/yellowfox/d8/imports/configured-plans",
"request": {
"method": "GET",
"url": {
"raw": "{{baseUrl}}/api/eventhub/acquisition/yellowfox/d8/imports/configured-plans",
"host": [
"{{baseUrl}}"
],
"path": [
"api",
"eventhub",
"acquisition",
"yellowfox",
"d8",
"imports",
"configured-plans"
]
}
}
},
{
"name": "GET /api/eventhub/acquisition/yellowfox/d8/imports/configured-plans/{planKey}",
"request": {
"method": "GET",
"url": {
"raw": "{{baseUrl}}/api/eventhub/acquisition/yellowfox/d8/imports/configured-plans/{{yellowFoxPlanKey}}",
"host": [
"{{baseUrl}}"
],
"path": [
"api",
"eventhub",
"acquisition",
"yellowfox",
"d8",
"imports",
"configured-plans",
"{{yellowFoxPlanKey}}"
]
}
}
},
{
"name": "POST /api/eventhub/acquisition/yellowfox/d8/imports/configured-plans/{planKey}/start",
"request": {
"method": "POST",
"url": {
"raw": "{{baseUrl}}/api/eventhub/acquisition/yellowfox/d8/imports/configured-plans/{{yellowFoxPlanKey}}/start?triggerMode=EXECUTE&mode=INCREMENTAL_UPDATE&strategy=SOURCE_ROW_WATERMARK",
"host": [
"{{baseUrl}}"
],
"path": [
"api",
"eventhub",
"acquisition",
"yellowfox",
"d8",
"imports",
"configured-plans",
"{{yellowFoxPlanKey}}",
"start"
],
"query": [
{
"key": "triggerMode",
"value": "EXECUTE"
},
{
"key": "mode",
"value": "INCREMENTAL_UPDATE"
},
{
"key": "strategy",
"value": "SOURCE_ROW_WATERMARK"
}
]
}
}
},
{
"name": "POST /api/eventhub/acquisition/yellowfox/d8/imports/configured-plans/{planKey}/master-data/refresh",
"request": {
"method": "POST",
"url": {
"raw": "{{baseUrl}}/api/eventhub/acquisition/yellowfox/d8/imports/configured-plans/{{yellowFoxPlanKey}}/master-data/refresh?mode=INCREMENTAL_UPDATE&strategy=SOURCE_ROW_WATERMARK",
"host": [
"{{baseUrl}}"
],
"path": [
"api",
"eventhub",
"acquisition",
"yellowfox",
"d8",
"imports",
"configured-plans",
"{{yellowFoxPlanKey}}",
"master-data",
"refresh"
],
"query": [
{
"key": "mode",
"value": "INCREMENTAL_UPDATE"
},
{
"key": "strategy",
"value": "SOURCE_ROW_WATERMARK"
}
]
}
}
}
]
},

34
src/main/java/at/procon/eventhub/config/EventHubProperties.java
View File

@ -1,6 +1,7 @@
package at.procon.eventhub.config;
import at.procon.eventhub.esperpoc.dto.EsperActivityMergeMode;
import at.procon.eventhub.esperpoc.dto.EsperOperatingPeriodEngineMode;
import at.procon.eventhub.esperpoc.dto.EsperUnknownTreatmentMode;
import at.procon.eventhub.esperpoc.dto.EsperShiftResolutionMode;
import java.time.Duration;
@ -80,6 +81,7 @@ public class EventHubProperties {
private int mergeGapSeconds = 0;
private int gapDetectionToleranceSeconds = 0;
private EsperUnknownTreatmentMode unknownTreatmentMode = EsperUnknownTreatmentMode.AS_BREAK_REST;
private EsperOperatingPeriodEngineMode engineMode = EsperOperatingPeriodEngineMode.STREAM_COLLECTOR;
public int getOperatingSplitIdleHours() {
return operatingSplitIdleHours;
@ -122,6 +124,16 @@ public class EventHubProperties {
? EsperUnknownTreatmentMode.AS_BREAK_REST
: unknownTreatmentMode;
}
public EsperOperatingPeriodEngineMode getEngineMode() {
return engineMode;
}
public void setEngineMode(EsperOperatingPeriodEngineMode engineMode) {
this.engineMode = engineMode == null
? EsperOperatingPeriodEngineMode.STREAM_COLLECTOR
: engineMode;
}
}
public static class Batch {
@ -212,6 +224,9 @@ public class EventHubProperties {
/** How JDBC extraction batches are handed over to the ingest pipeline. */
private JdbcExtractionIngestMode jdbcExtractionIngestMode = JdbcExtractionIngestMode.SYNC_DIRECT;
/** Whether master-data refresh reconciles vehicle registrations and assignments. */
private boolean syncVehicleRegistrationsOnMasterDataUpdate = true;
/** Configured tenant/source import plans. */
private List<ConfiguredImportPlan> importPlans = new ArrayList<>();
@ -268,6 +283,14 @@ public class EventHubProperties {
: jdbcExtractionIngestMode;
}
public boolean isSyncVehicleRegistrationsOnMasterDataUpdate() {
return syncVehicleRegistrationsOnMasterDataUpdate;
}
public void setSyncVehicleRegistrationsOnMasterDataUpdate(boolean syncVehicleRegistrationsOnMasterDataUpdate) {
this.syncVehicleRegistrationsOnMasterDataUpdate = syncVehicleRegistrationsOnMasterDataUpdate;
}
public List<ConfiguredImportPlan> getImportPlans() {
return importPlans;
}
@ -342,6 +365,9 @@ public class EventHubProperties {
/** Emit a first ignition snapshot per vehicle if no previous D8 ignition state exists in the imported window. */
private boolean emitInitialIgnitionSnapshot = false;
/** Whether master-data refresh reconciles vehicle registrations and assignments. */
private boolean syncVehicleRegistrationsOnMasterDataUpdate = true;
/** Configured tenant/source import plans. */
private List<ConfiguredImportPlan> importPlans = new ArrayList<>();
@ -400,6 +426,14 @@ public class EventHubProperties {
this.emitInitialIgnitionSnapshot = emitInitialIgnitionSnapshot;
}
public boolean isSyncVehicleRegistrationsOnMasterDataUpdate() {
return syncVehicleRegistrationsOnMasterDataUpdate;
}
public void setSyncVehicleRegistrationsOnMasterDataUpdate(boolean syncVehicleRegistrationsOnMasterDataUpdate) {
this.syncVehicleRegistrationsOnMasterDataUpdate = syncVehicleRegistrationsOnMasterDataUpdate;
}
public List<ConfiguredImportPlan> getImportPlans() {
return importPlans;
}

7
src/main/java/at/procon/eventhub/esperpoc/api/EsperPocController.java
View File

@ -1,6 +1,7 @@
package at.procon.eventhub.esperpoc.api;
import at.procon.eventhub.esperpoc.dto.EsperActivityMergeMode;
import at.procon.eventhub.esperpoc.dto.EsperOperatingPeriodEngineMode;
import at.procon.eventhub.esperpoc.dto.EsperOperatingPeriodRequest;
import at.procon.eventhub.esperpoc.dto.EsperOperatingPeriodResultDto;
import at.procon.eventhub.esperpoc.dto.EsperPocRequest;
@ -76,7 +77,8 @@ public class EsperPocController {
@RequestParam(required = false) Integer significantDrivingMinutes,
@RequestParam(required = false) Integer mergeGapSeconds,
@RequestParam(required = false) Integer gapDetectionToleranceSeconds,
@RequestParam(required = false) EsperUnknownTreatmentMode unknownTreatmentMode
@RequestParam(required = false) EsperUnknownTreatmentMode unknownTreatmentMode,
@RequestParam(required = false) EsperOperatingPeriodEngineMode engineMode
) {
EsperOperatingPeriodRequest request = new EsperOperatingPeriodRequest(
tenantKey,
@ -88,7 +90,8 @@ public class EsperPocController {
significantDrivingMinutes,
mergeGapSeconds,
gapDetectionToleranceSeconds,
unknownTreatmentMode
unknownTreatmentMode,
engineMode
);
return ResponseEntity.ok(operatingPeriodEvaluationService.evaluate(request));
}

6
src/main/java/at/procon/eventhub/esperpoc/dto/EsperOperatingPeriodEngineMode.java Normal file
View File

@ -0,0 +1,6 @@
package at.procon.eventhub.esperpoc.dto;
public enum EsperOperatingPeriodEngineMode {
STREAM_COLLECTOR,
FULL_EPL
}

3
src/main/java/at/procon/eventhub/esperpoc/dto/EsperOperatingPeriodRequest.java
View File

@ -15,7 +15,8 @@ public record EsperOperatingPeriodRequest(
Integer significantDrivingMinutes,
Integer mergeGapSeconds,
Integer gapDetectionToleranceSeconds,
EsperUnknownTreatmentMode unknownTreatmentMode
EsperUnknownTreatmentMode unknownTreatmentMode,
EsperOperatingPeriodEngineMode engineMode
) {
public EsperOperatingPeriodRequest {
if (occurredFrom != null && occurredTo != null && !occurredFrom.isBefore(occurredTo)) {

1
src/main/java/at/procon/eventhub/esperpoc/dto/EsperOperatingPeriodResultDto.java
View File

@ -27,6 +27,7 @@ public record EsperOperatingPeriodResultDto(
int mergeGapSeconds,
int gapDetectionToleranceSeconds,
EsperUnknownTreatmentMode unknownTreatmentMode,
EsperOperatingPeriodEngineMode engineMode,
List<RawActivityEventDto> rawEvents,
List<ActivityIntervalDto> resolvedKnownIntervals,
List<ActivityIntervalDto> evaluationIntervals,

207
src/main/java/at/procon/eventhub/esperpoc/service/EsperOperatingPeriodEngine.java
View File

@ -1,6 +1,7 @@
package at.procon.eventhub.esperpoc.service;
import at.procon.eventhub.esperpoc.dto.ActivityIntervalDto;
import at.procon.eventhub.esperpoc.dto.EsperOperatingPeriodEngineMode;
import at.procon.eventhub.esperpoc.dto.OperatingPeriodActivityIntervalDto;
import com.espertech.esper.common.client.EPCompiled;
import com.espertech.esper.common.client.EventBean;
@ -12,37 +13,66 @@ import com.espertech.esper.runtime.client.EPDeployException;
import com.espertech.esper.runtime.client.EPDeployment;
import com.espertech.esper.runtime.client.EPRuntime;
import com.espertech.esper.runtime.client.EPRuntimeProvider;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.time.Instant;
import java.time.Duration;
import java.time.OffsetDateTime;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.atomic.AtomicLong;
import org.springframework.core.io.ClassPathResource;
import org.springframework.stereotype.Component;
import org.springframework.util.StreamUtils;
@Component
public class EsperOperatingPeriodEngine {
private static final AtomicLong RUNTIME_COUNTER = new AtomicLong();
// Minimal stream-only mode: Esper preserves event ordering, Java owns the period state machine.
private static final String INPUT_STREAM_EPL = """
@name('operatingPeriodIntervalStream')
select * from OperatingPeriodIntervalInputEvent
""";
// Full-EPL mode: Esper owns the operating-period state machine and emits periodized intervals/closures.
private static final String FULL_EPL_TEMPLATE = loadResource("esper/operating-period-state-machine.epl");
public EsperOperatingPeriodEvaluation evaluate(
List<ActivityIntervalDto> intervals,
Duration operatingSplitIdleThreshold
Duration operatingSplitIdleThreshold,
EsperOperatingPeriodEngineMode mode
) {
List<ActivityIntervalDto> sorted = sortedPositiveIntervals(intervals);
if (sorted.isEmpty()) {
return new EsperOperatingPeriodEvaluation(List.of(), List.of());
}
if (mode == EsperOperatingPeriodEngineMode.FULL_EPL) {
return evaluateFullEpl(sorted, operatingSplitIdleThreshold);
}
return evaluateStreamCollector(sorted, operatingSplitIdleThreshold);
}
public EsperOperatingPeriodEvaluation evaluate(
List<ActivityIntervalDto> intervals,
Duration operatingSplitIdleThreshold
) {
return evaluate(intervals, operatingSplitIdleThreshold, EsperOperatingPeriodEngineMode.STREAM_COLLECTOR);
}
private EsperOperatingPeriodEvaluation evaluateStreamCollector(
List<ActivityIntervalDto> sorted,
Duration operatingSplitIdleThreshold
) {
// In stream-collector mode Esper only serializes the stream; period transitions are evaluated in Java.
PeriodizationCollector collector = new PeriodizationCollector(operatingSplitIdleThreshold);
executeWithRuntime(
configuration -> configuration.getCommon().addEventType("OperatingPeriodIntervalInputEvent", EsperOperatingPeriodIntervalInputEvent.class),
INPUT_STREAM_EPL,
"operatingPeriodIntervalStream",
newData -> collectInputIntervals(newData, collector),
List.of("operatingPeriodIntervalStream"),
(statementName, newData) -> collectInputIntervals(newData, collector),
runtime -> {
for (ActivityIntervalDto interval : sorted) {
runtime.getEventService().sendEventBean(toInputEvent(interval), "OperatingPeriodIntervalInputEvent");
@ -52,11 +82,72 @@ public class EsperOperatingPeriodEngine {
return collector.finish();
}
private EsperOperatingPeriodEvaluation evaluateFullEpl(
List<ActivityIntervalDto> sorted,
Duration operatingSplitIdleThreshold
) {
// The full-EPL script is parameterized per request so the idle-split threshold matches the request/config.
String epl = FULL_EPL_TEMPLATE.replace("${operatingSplitIdleMs}", Long.toString(operatingSplitIdleThreshold.toMillis()));
List<OperatingPeriodActivityIntervalDto> periodizedIntervals = new ArrayList<>();
List<EsperClosedOperatingPeriod> closedPeriods = new ArrayList<>();
executeWithRuntime(
configuration -> {
// Full-EPL mode uses explicit map schemas so EPL can own the whole state machine without
// relying on Java bean-property resolution during compilation.
Map<String, Object> inputDefinition = new LinkedHashMap<>();
inputDefinition.put("driverId", java.util.UUID.class);
inputDefinition.put("vehicleId", java.util.UUID.class);
inputDefinition.put("vehicleRegistrationId", java.util.UUID.class);
inputDefinition.put("activityType", String.class);
inputDefinition.put("cardSlot", String.class);
inputDefinition.put("cardStatus", String.class);
inputDefinition.put("drivingStatus", String.class);
inputDefinition.put("sourceKind", String.class);
inputDefinition.put("startTs", long.class);
inputDefinition.put("endTs", long.class);
inputDefinition.put("durationMs", long.class);
inputDefinition.put("sourceRowId", String.class);
inputDefinition.put("sourceRowIds", java.util.List.class);
inputDefinition.put("clippedToRequestedPeriod", boolean.class);
inputDefinition.put("synthetic", boolean.class);
configuration.getCommon().addEventType("OperatingPeriodInputMap", inputDefinition);
configuration.getCommon().addEventType("OperatingPeriodFlushEvent", Map.of("reason", String.class));
},
epl,
List.of("periodizedActivityIntervals", "operatingPeriodClosed"),
(statementName, newData) -> {
if ("periodizedActivityIntervals".equals(statementName)) {
collectPeriodizedOutputs(newData, periodizedIntervals);
} else if ("operatingPeriodClosed".equals(statementName)) {
collectClosedOutputs(newData, closedPeriods);
}
},
runtime -> {
// Historical evaluation sends the complete interval timeline first and then a single flush event
// so the EPL state machine can emit the final still-open operating period.
for (ActivityIntervalDto interval : sorted) {
runtime.getEventService().sendEventMap(toInputMap(interval), "OperatingPeriodInputMap");
}
runtime.getEventService().sendEventMap(Map.of("reason", "HISTORICAL_EVALUATION"), "OperatingPeriodFlushEvent");
}
);
return new EsperOperatingPeriodEvaluation(
periodizedIntervals.stream()
.sorted(Comparator.comparing(OperatingPeriodActivityIntervalDto::startedAt)
.thenComparing(OperatingPeriodActivityIntervalDto::endedAt)
.thenComparing(OperatingPeriodActivityIntervalDto::activityType, Comparator.nullsLast(String::compareTo)))
.toList(),
closedPeriods.stream()
.sorted(Comparator.comparing(EsperClosedOperatingPeriod::startedAt))
.toList()
);
}
private void executeWithRuntime(
java.util.function.Consumer<Configuration> configurationSetup,
String epl,
String statementName,
java.util.function.Consumer<EventBean[]> listener,
List<String> statementNames,
StatementListener listener,
java.util.function.Consumer<EPRuntime> sender
) {
EPRuntime runtime = null;
@ -69,9 +160,12 @@ public class EsperOperatingPeriodEngine {
CompilerArguments arguments = new CompilerArguments(configuration);
EPCompiled compiled = EPCompilerProvider.getCompiler().compile(epl, arguments);
EPDeployment deployment = runtime.getDeploymentService().deploy(compiled);
runtime.getDeploymentService()
.getStatement(deployment.getDeploymentId(), statementName)
.addListener((newData, oldData, statement, rt) -> listener.accept(newData));
// Multiple statements may emit outputs from a single deployment; we dispatch by statement name.
for (String statementName : statementNames) {
runtime.getDeploymentService()
.getStatement(deployment.getDeploymentId(), statementName)
.addListener((newData, oldData, statement, rt) -> listener.accept(statementName, newData));
}
sender.accept(runtime);
} catch (EPCompileException | EPDeployException e) {
throw new IllegalStateException("Cannot compile/deploy Esper operating-period EPL", e);
@ -87,10 +181,66 @@ public class EsperOperatingPeriodEngine {
return;
}
for (EventBean event : newData) {
// Stream-collector mode receives the ordered interval stream back from Esper and applies the
// deterministic Java state machine to it.
collector.accept((EsperOperatingPeriodIntervalInputEvent) event.getUnderlying());
}
}
private void collectPeriodizedOutputs(EventBean[] newData, List<OperatingPeriodActivityIntervalDto> target) {
if (newData == null) {
return;
}
for (EventBean event : newData) {
target.add(new OperatingPeriodActivityIntervalDto(
(java.util.UUID) event.get("driverId"),
(java.util.UUID) event.get("vehicleId"),
(java.util.UUID) event.get("vehicleRegistrationId"),
(String) event.get("activityType"),
(String) event.get("cardSlot"),
(String) event.get("cardStatus"),
(String) event.get("drivingStatus"),
(String) event.get("sourceKind"),
OffsetDateTime.ofInstant(Instant.ofEpochMilli((Long) event.get("startedAtTs")), java.time.ZoneOffset.UTC),
OffsetDateTime.ofInstant(Instant.ofEpochMilli((Long) event.get("endedAtTs")), java.time.ZoneOffset.UTC),
((Long) event.get("durationMs")) / 1000L,
(String) event.get("sourceRowId"),
castSourceRowIds(event.get("sourceRowIds")),
(Boolean) event.get("clippedToRequestedPeriod"),
"PERIODIZED_ACTIVITY",
(Long) event.get("operatingPeriodNo"),
OffsetDateTime.ofInstant(Instant.ofEpochMilli((Long) event.get("operatingPeriodStartedAtTs")), java.time.ZoneOffset.UTC),
(Boolean) event.get("newOperatingPeriod"),
nullableMillisToSeconds((Long) event.get("gapSincePreviousActivityMs")),
(Boolean) event.get("synthetic")
));
}
}
private void collectClosedOutputs(EventBean[] newData, List<EsperClosedOperatingPeriod> target) {
if (newData == null) {
return;
}
for (EventBean event : newData) {
target.add(new EsperClosedOperatingPeriod(
(Long) event.get("operatingPeriodNo"),
OffsetDateTime.ofInstant(Instant.ofEpochMilli((Long) event.get("operatingPeriodStartedAtTs")), java.time.ZoneOffset.UTC),
OffsetDateTime.ofInstant(Instant.ofEpochMilli((Long) event.get("operatingPeriodEndedAtTs")), java.time.ZoneOffset.UTC),
((Long) event.get("durationMs")) / 1000L,
(String) event.get("closedBy")
));
}
}
@SuppressWarnings("unchecked")
private List<String> castSourceRowIds(Object value) {
return value == null ? List.of() : List.copyOf((List<String>) value);
}
private Long nullableMillisToSeconds(Long value) {
return value == null ? null : value / 1000L;
}
private EsperOperatingPeriodIntervalInputEvent toInputEvent(ActivityIntervalDto interval) {
return new EsperOperatingPeriodIntervalInputEvent(
interval.driverEntityId(),
@ -111,6 +261,26 @@ public class EsperOperatingPeriodEngine {
);
}
private Map<String, Object> toInputMap(ActivityIntervalDto interval) {
Map<String, Object> map = new LinkedHashMap<>();
map.put("driverId", interval.driverEntityId());
map.put("vehicleId", interval.vehicleId());
map.put("vehicleRegistrationId", interval.vehicleRegistrationId());
map.put("activityType", interval.activityType());
map.put("cardSlot", interval.cardSlot());
map.put("cardStatus", interval.cardStatus());
map.put("drivingStatus", interval.drivingStatus());
map.put("sourceKind", interval.sourceKind());
map.put("startTs", interval.startedAt().toInstant().toEpochMilli());
map.put("endTs", interval.endedAt().toInstant().toEpochMilli());
map.put("durationMs", interval.durationSeconds() * 1000L);
map.put("sourceRowId", interval.sourceRowId());
map.put("sourceRowIds", interval.sourceRowIds());
map.put("clippedToRequestedPeriod", interval.clippedToRequestedPeriod());
map.put("synthetic", "UNKNOWN_GAP".equals(interval.level()));
return map;
}
private List<ActivityIntervalDto> sortedPositiveIntervals(List<ActivityIntervalDto> intervals) {
if (intervals == null || intervals.isEmpty()) {
return List.of();
@ -129,6 +299,18 @@ public class EsperOperatingPeriodEngine {
) {
}
private interface StatementListener {
void accept(String statementName, EventBean[] newData);
}
private static String loadResource(String path) {
try {
return StreamUtils.copyToString(new ClassPathResource(path).getInputStream(), StandardCharsets.UTF_8);
} catch (IOException e) {
throw new IllegalStateException("Cannot load Esper resource " + path, e);
}
}
private static final class PeriodizationCollector {
private final Duration operatingSplitIdleThreshold;
private final List<OperatingPeriodActivityIntervalDto> periodizedIntervals = new ArrayList<>();
@ -163,12 +345,16 @@ public class EsperOperatingPeriodEngine {
if ("UNKNOWN".equals(dto.activityType())) {
if (!hasOpenPeriod) {
// Unknown time before the first known activity does not belong to any operating period.
return;
}
if (dto.durationSeconds() >= operatingSplitIdleThreshold.getSeconds()) {
// Long UNKNOWN behaves like a closing gap: close the current period and wait for the next
// known activity to reopen a new period number.
closeCurrent("UNKNOWN_GAP");
return;
}
// Short UNKNOWN stays inside the current period as explicit uncertainty.
periodizedIntervals.add(OperatingPeriodActivityIntervalDto.periodized(
dto,
operatingPeriodNo,
@ -180,6 +366,7 @@ public class EsperOperatingPeriodEngine {
}
if (!hasOpenPeriod) {
// First known activity, or first activity after a long closing gap, opens a new operating period.
operatingPeriodNo = operatingPeriodNo < 1 ? 1 : operatingPeriodNo + 1;
hasOpenPeriod = true;
operatingPeriodStartedAt = dto.startedAt();
@ -196,6 +383,7 @@ public class EsperOperatingPeriodEngine {
long gapSeconds = Math.max(0, Duration.between(lastKnownActivityEndAt, dto.startedAt()).getSeconds());
if (gapSeconds >= operatingSplitIdleThreshold.getSeconds()) {
// Long idle time between known activities closes the current period and starts the next one.
closeCurrent("IDLE_GAP");
operatingPeriodNo++;
hasOpenPeriod = true;
@ -211,6 +399,7 @@ public class EsperOperatingPeriodEngine {
return;
}
// Normal forward continuity inside the same period.
periodizedIntervals.add(OperatingPeriodActivityIntervalDto.periodized(
dto,
operatingPeriodNo,
@ -225,6 +414,7 @@ public class EsperOperatingPeriodEngine {
private EsperOperatingPeriodEvaluation finish() {
if (hasOpenPeriod) {
// Historical evaluation has no future event to close the final period, so emit it explicitly.
closeCurrent("FLUSH");
}
return new EsperOperatingPeriodEvaluation(
@ -244,6 +434,7 @@ public class EsperOperatingPeriodEngine {
hasOpenPeriod = false;
return;
}
// A closed period always ends at the last known non-rest activity end, never at the synthetic UNKNOWN.
closedPeriods.add(new EsperClosedOperatingPeriod(
operatingPeriodNo,
operatingPeriodStartedAt,

21
src/main/java/at/procon/eventhub/esperpoc/service/EsperOperatingPeriodEvaluationService.java
View File

@ -3,6 +3,7 @@ package at.procon.eventhub.esperpoc.service;
import at.procon.eventhub.config.EventHubProperties;
import at.procon.eventhub.esperpoc.dto.ActivityIntervalDto;
import at.procon.eventhub.esperpoc.dto.DrivingInterruptionDto;
import at.procon.eventhub.esperpoc.dto.EsperOperatingPeriodEngineMode;
import at.procon.eventhub.esperpoc.dto.EsperOperatingPeriodRequest;
import at.procon.eventhub.esperpoc.dto.EsperOperatingPeriodResultDto;
import at.procon.eventhub.esperpoc.dto.EsperUnknownTreatmentMode;
@ -68,6 +69,7 @@ public class EsperOperatingPeriodEvaluationService {
Duration mergeGapTolerance = Duration.ofSeconds(resolveMergeGapSeconds(request));
Duration gapDetectionTolerance = Duration.ofSeconds(resolveGapDetectionToleranceSeconds(request));
EsperUnknownTreatmentMode unknownTreatmentMode = resolveUnknownTreatmentMode(request);
EsperOperatingPeriodEngineMode engineMode = resolveEngineMode(request);
long dbStartedNanos = System.nanoTime();
List<RawActivityEventDto> rawEvents = activityRepository.findDriverActivityEvents(
@ -105,7 +107,8 @@ public class EsperOperatingPeriodEvaluationService {
long periodizeStartedNanos = System.nanoTime();
EsperOperatingPeriodEngine.EsperOperatingPeriodEvaluation evaluation = operatingPeriodEngine.evaluate(
evaluationLoadedIntervals,
splitIdleThreshold
splitIdleThreshold,
engineMode
);
long periodizeElapsedMs = elapsedMillis(periodizeStartedNanos);
@ -150,7 +153,7 @@ public class EsperOperatingPeriodEvaluationService {
);
long totalElapsedMs = elapsedMillis(startedNanos);
log.info("Esper operating-period evaluation tenant={} driverId={} requestedFrom={} requestedTo={} loadedFrom={} loadedTo={} unknownMode={} rawEvents={} cardRawEvents={} vuRawEvents={} cardIntervals={} vuIntervals={} resolvedKnownIntervals={} evaluationIntervals={} periodizedIntervals={} mergedIntervals={} nonDrivingIntervals={} operatingPeriods={} timingsMs={{dbRetrieve={}, cardIntervalEsper={}, vuIntervalEsper={}, vuGapFill={}, synthUnknown={}, periodizeEsper={}, merge={}, nonDriving={}, total={}}}",
log.info("Esper operating-period evaluation tenant={} driverId={} requestedFrom={} requestedTo={} loadedFrom={} loadedTo={} unknownMode={} engineMode={} rawEvents={} cardRawEvents={} vuRawEvents={} cardIntervals={} vuIntervals={} resolvedKnownIntervals={} evaluationIntervals={} periodizedIntervals={} mergedIntervals={} nonDrivingIntervals={} operatingPeriods={} timingsMs={{dbRetrieve={}, cardIntervalEsper={}, vuIntervalEsper={}, vuGapFill={}, synthUnknown={}, periodizeEsper={}, merge={}, nonDriving={}, total={}}}",
request.tenantKey(),
request.driverId(),
requestedFrom,
@ -158,6 +161,7 @@ public class EsperOperatingPeriodEvaluationService {
loadedFrom,
loadedTo,
unknownTreatmentMode,
engineMode,
rawEvents.size(),
driverCardRawEvents.size(),
vehicleUnitRawEvents.size(),
@ -202,6 +206,7 @@ public class EsperOperatingPeriodEvaluationService {
resolveMergeGapSeconds(request),
resolveGapDetectionToleranceSeconds(request),
unknownTreatmentMode,
engineMode,
rawEvents,
resolvedKnownLoadedIntervals,
evaluationLoadedIntervals,
@ -210,6 +215,7 @@ public class EsperOperatingPeriodEvaluationService {
nonDrivingIntervals,
operatingPeriods,
notes(
engineMode,
unknownTreatmentMode,
resolveOperatingSplitIdleHours(request),
resolveSignificantDrivingMinutes(request),
@ -708,7 +714,17 @@ public class EsperOperatingPeriodEvaluationService {
: properties.getEsperPoc().getOperatingPeriodEvaluation().getUnknownTreatmentMode();
}
private EsperOperatingPeriodEngineMode resolveEngineMode(EsperOperatingPeriodRequest request) {
if (request.engineMode() != null) {
return request.engineMode();
}
return properties == null
? EsperOperatingPeriodEngineMode.STREAM_COLLECTOR
: properties.getEsperPoc().getOperatingPeriodEvaluation().getEngineMode();
}
private List<String> notes(
EsperOperatingPeriodEngineMode engineMode,
EsperUnknownTreatmentMode unknownTreatmentMode,
int operatingSplitIdleHours,
int significantDrivingMinutes,
@ -719,6 +735,7 @@ public class EsperOperatingPeriodEvaluationService {
"BREAK_REST events are ignored for activity evaluation but still prevent synthetic UNKNOWN intervals from being created over covered rest spans.",
"Synthetic UNKNOWN intervals are created only for uncovered gaps between non-rest activities.",
"UNKNOWN treatment mode is " + unknownTreatmentMode + ".",
"Operating-period engine mode is " + engineMode + ".",
"Operating periods split after " + operatingSplitIdleHours + " hours of no non-rest activity; significant driving closes non-driving intervals from " + significantDrivingMinutes + " minutes onward.",
"Synthetic UNKNOWN gaps are only emitted when uncovered time exceeds " + gapDetectionToleranceSeconds + " seconds."
);

27
src/main/java/at/procon/eventhub/importing/AbstractConfiguredImportPlanService.java
View File

@ -64,10 +64,15 @@ public abstract class AbstractConfiguredImportPlanService<R extends ImportRunReq
AcquisitionStrategy strategy = strategyOverride == null
? (mode == ImportMode.INCREMENTAL_UPDATE ? plan.getScheduledStrategy() : plan.getInitialStrategy())
: strategyOverride;
return buildRequest(plan, mode, strategy, scopedForRequest(plan, applyInitialOccurredWindow));
return buildRequest(plan, mode, strategy, scopedForRequest(plan, mode, strategy, applyInitialOccurredWindow));
}
private ImportScopeDto scopedForRequest(EventHubProperties.ConfiguredImportPlan plan, boolean applyInitialOccurredWindow) {
private ImportScopeDto scopedForRequest(
EventHubProperties.ConfiguredImportPlan plan,
ImportMode mode,
AcquisitionStrategy strategy,
boolean applyInitialOccurredWindow
) {
ImportScopeDto scope = plan.getImportScope();
if (applyInitialOccurredWindow && scope != null
&& (plan.getInitialOccurredFrom() != null || plan.getInitialOccurredTo() != null)) {
@ -79,9 +84,27 @@ public abstract class AbstractConfiguredImportPlanService<R extends ImportRunReq
plan.getInitialOccurredTo() == null ? scope.occurredTo() : plan.getInitialOccurredTo()
);
}
if (mode == ImportMode.INCREMENTAL_UPDATE
&& isWatermarkStrategy(strategy)
&& scope != null
&& scope.occurredFrom() == null
&& plan.getInitialOccurredFrom() != null) {
return new ImportScopeDto(
scope.type(),
scope.rootSourceOrganisation(),
scope.includeChildren(),
plan.getInitialOccurredFrom(),
scope.occurredTo() == null ? plan.getInitialOccurredTo() : scope.occurredTo()
);
}
return scope;
}
private boolean isWatermarkStrategy(AcquisitionStrategy strategy) {
return strategy == AcquisitionStrategy.SOURCE_PACKAGE_WATERMARK
|| strategy == AcquisitionStrategy.SOURCE_ROW_WATERMARK;
}
private D planByKey(String planKey) {
return toDto(rawPlanByKey(planKey));
}

7
src/main/java/at/procon/eventhub/importing/ImportChunkPlanner.java
View File

@ -17,7 +17,7 @@ public class ImportChunkPlanner {
OffsetDateTime to = scope == null ? null : scope.occurredTo();
if (request.mode() == ImportMode.INCREMENTAL_UPDATE
&& request.acquisitionStrategy() == AcquisitionStrategy.SOURCE_PACKAGE_WATERMARK) {
&& isWatermarkStrategy(request.acquisitionStrategy())) {
return List.of(new ImportTimeChunkDto(1, from, to));
}
@ -39,4 +39,9 @@ public class ImportChunkPlanner {
}
return chunks.isEmpty() ? List.of(new ImportTimeChunkDto(1, from, to)) : chunks;
}
private boolean isWatermarkStrategy(AcquisitionStrategy strategy) {
return strategy == AcquisitionStrategy.SOURCE_PACKAGE_WATERMARK
|| strategy == AcquisitionStrategy.SOURCE_ROW_WATERMARK;
}
}

38
src/main/java/at/procon/eventhub/importing/extraction/AbstractJdbcExtractionBatchExecutor.java
View File

@ -88,15 +88,7 @@ public abstract class AbstractJdbcExtractionBatchExecutor<R extends ImportRunReq
packageInfo
);
String scopeHash = request.importScope() == null ? "NO_SCOPE" : request.importScope().stableKey();
ImportCursorStateDto cursor = importCursorRepository.findCursor(
request.tenantKey(),
eventSourceId,
scopeHash,
planItem.eventFamily(),
planItem.sourceKind(),
request.acquisitionStrategy()
);
ImportCursorStateDto cursor = findCursor(eventSourceId, request, planItem);
Map<String, Object> params = parameters(request, chunkScope, cursor);
String sql = loadSql(definition.sqlResource());
@ -290,6 +282,34 @@ public abstract class AbstractJdbcExtractionBatchExecutor<R extends ImportRunReq
return params;
}
protected ImportCursorStateDto findCursor(int eventSourceId, R request, ImportPlanItemDto planItem) {
String scopeHash = request.importScope() == null ? "NO_SCOPE" : request.importScope().stableKey();
ImportCursorStateDto cursor = importCursorRepository.findCursor(
request.tenantKey(),
eventSourceId,
scopeHash,
planItem.eventFamily(),
planItem.sourceKind(),
request.acquisitionStrategy()
);
if (cursor != null || !shouldBootstrapWatermarkCursor(request)) {
return cursor;
}
return importCursorRepository.findLatestCursor(
request.tenantKey(),
eventSourceId,
scopeHash,
planItem.eventFamily(),
planItem.sourceKind()
);
}
private boolean shouldBootstrapWatermarkCursor(R request) {
return request.mode() == at.procon.eventhub.dto.ImportMode.INCREMENTAL_UPDATE
&& (request.acquisitionStrategy() == AcquisitionStrategy.SOURCE_ROW_WATERMARK
|| request.acquisitionStrategy() == AcquisitionStrategy.SOURCE_PACKAGE_WATERMARK);
}
protected OffsetDateTime lastSourcePackageImportedAt(ExtractedEventStats stats, ImportCursorStateDto cursor) {
return stats.lastSourcePackageImportedAt() == null
? cursor == null ? null : cursor.lastSourcePackageImportedAt()

39
src/main/java/at/procon/eventhub/importing/persistence/ImportCursorRepository.java
View File

@ -56,6 +56,45 @@ public class ImportCursorRepository {
);
}
public ImportCursorStateDto findLatestCursor(
String tenantKey,
int eventSourceId,
String scopeHash,
EventFamily eventFamily,
String sourceKind
) {
return jdbcTemplate.query(
"""
select last_source_package_imported_at,
last_source_package_id,
last_source_row_updated_at,
last_occurred_to
from eventhub.import_cursor
where tenant_key = ?
and event_source_id = ?
and scope_hash = ?
and event_family = ?
and source_kind = ?
order by coalesce(last_occurred_to, last_source_row_updated_at, last_source_package_imported_at) desc nulls last,
updated_at desc
limit 1
""",
rs -> rs.next()
? new ImportCursorStateDto(
rs.getObject("last_source_package_imported_at", java.time.OffsetDateTime.class),
rs.getString("last_source_package_id"),
rs.getObject("last_source_row_updated_at", java.time.OffsetDateTime.class),
rs.getObject("last_occurred_to", java.time.OffsetDateTime.class)
)
: null,
tenantKey,
eventSourceId,
scopeHash,
eventFamily.name(),
sourceKind
);
}
public void advanceCursor(
String tenantKey,
int eventSourceId,

877
src/main/java/at/procon/eventhub/persistence/DriverIdentityRepository.java
View File

File diff suppressed because it is too large Load Diff

74
src/main/java/at/procon/eventhub/persistence/EventRepository.java
View File

@ -57,11 +57,12 @@ public class EventRepository {
*/
public int batchInsert(UUID packageId, String tenantKey, int eventSourceId, List<EventHubEventDto> events) {
Map<String, UUID> entityIdCache = new HashMap<>();
Map<String, DriverIdentityRepository.ResolvedDriverReference> driverRefCache = new HashMap<>();
Map<String, List<VehicleRefCacheEntry>> vehicleRefCache = new HashMap<>();
List<ResolvedEventImportRow> rows = new ArrayList<>(events.size());
for (EventHubEventDto event : events) {
ResolvedEntityRefs refs = resolveEntityRefs(tenantKey, eventSourceId, event, entityIdCache, vehicleRefCache);
ResolvedEntityRefs refs = resolveEntityRefs(tenantKey, eventSourceId, event, entityIdCache, driverRefCache, vehicleRefCache);
rows.add(resolveEventImportRow(packageId, eventSourceId, event, refs));
}
@ -93,6 +94,7 @@ public class EventRepository {
eventSourceId,
event.externalSourceEventId(),
refs.driverId(),
refs.driverCardId(),
refs.vehicleId(),
refs.vehicleRegistrationId(),
sourcePackageId,
@ -125,6 +127,7 @@ public class EventRepository {
event_source_id integer not null,
external_source_event_id text not null,
driver_id uuid,
driver_card_id uuid,
vehicle_id uuid,
vehicle_registration_id uuid,
source_package_id text,
@ -152,11 +155,11 @@ public class EventRepository {
"""
insert into eventhub_event_import_stage(
row_no, source_record_key_hash, requested_event_id, data_package_id, event_source_id,
external_source_event_id, driver_id, vehicle_id, vehicle_registration_id,
external_source_event_id, driver_id, driver_card_id, vehicle_id, vehicle_registration_id,
source_package_id, source_package_entity_id, occurred_at, received_partner_at, received_hub_at,
event_domain, event_type, lifecycle, odometer_m, longitude, latitude,
payload, manual_entry, event_signature_hash
) values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?::jsonb, ?, ?)
) values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?::jsonb, ?, ?)
""",
new BatchPreparedStatementSetter() {
@Override
@ -169,22 +172,23 @@ public class EventRepository {
ps.setInt(5, row.eventSourceId());
ps.setString(6, row.externalSourceEventId());
ps.setObject(7, row.driverId());
ps.setObject(8, row.vehicleId());
ps.setObject(9, row.vehicleRegistrationId());
ps.setString(10, row.sourcePackageId());
ps.setObject(11, row.sourcePackageEntityId());
ps.setObject(12, row.occurredAt());
ps.setObject(13, row.receivedPartnerAt());
ps.setObject(14, row.receivedHubAt());
ps.setString(15, row.eventDomain());
ps.setString(16, row.eventType());
ps.setString(17, row.lifecycle());
ps.setObject(18, row.odometerM());
ps.setObject(19, row.longitude());
ps.setObject(20, row.latitude());
ps.setString(21, row.payloadJson());
ps.setBoolean(22, row.manualEntry());
ps.setString(23, row.eventSignatureHash());
ps.setObject(8, row.driverCardId());
ps.setObject(9, row.vehicleId());
ps.setObject(10, row.vehicleRegistrationId());
ps.setString(11, row.sourcePackageId());
ps.setObject(12, row.sourcePackageEntityId());
ps.setObject(13, row.occurredAt());
ps.setObject(14, row.receivedPartnerAt());
ps.setObject(15, row.receivedHubAt());
ps.setString(16, row.eventDomain());
ps.setString(17, row.eventType());
ps.setString(18, row.lifecycle());
ps.setObject(19, row.odometerM());
ps.setObject(20, row.longitude());
ps.setObject(21, row.latitude());
ps.setString(22, row.payloadJson());
ps.setBoolean(23, row.manualEntry());
ps.setString(24, row.eventSignatureHash());
}
@Override
@ -236,7 +240,7 @@ public class EventRepository {
insert into eventhub.event(
id, event_source_id, data_package_id,
external_source_event_id,
driver_id, vehicle_id, vehicle_registration_id,
driver_id, driver_card_id, vehicle_id, vehicle_registration_id,
source_package_id, source_package_entity_id,
occurred_at, received_partner_at, received_hub_at,
event_domain, event_type, lifecycle,
@ -247,7 +251,7 @@ public class EventRepository {
select
source_record.event_id, stage.event_source_id, stage.data_package_id,
stage.external_source_event_id,
stage.driver_id, stage.vehicle_id, stage.vehicle_registration_id,
stage.driver_id, stage.driver_card_id, stage.vehicle_id, stage.vehicle_registration_id,
stage.source_package_id, stage.source_package_entity_id,
source_record.event_occurred_at, stage.received_partner_at, stage.received_hub_at,
stage.event_domain, stage.event_type, stage.lifecycle,
@ -358,33 +362,39 @@ public class EventRepository {
int eventSourceId,
EventHubEventDto event,
Map<String, UUID> entityIdCache,
Map<String, DriverIdentityRepository.ResolvedDriverReference> driverRefCache,
Map<String, List<VehicleRefCacheEntry>> vehicleRefCache
) {
UUID driverId = resolveDriverId(tenantKey, eventSourceId, event, entityIdCache);
DriverIdentityRepository.ResolvedDriverReference driverRef = resolveDriverReference(tenantKey, eventSourceId, event, driverRefCache);
ResolvedVehicleReference vehicleRef = resolveVehicleReference(tenantKey, eventSourceId, event, vehicleRefCache);
UUID sourcePackageEntityId = resolveSourcePackageEntityId(tenantKey, eventSourceId, event, entityIdCache);
return new ResolvedEntityRefs(driverId, vehicleRef.vehicleId(), vehicleRef.vehicleRegistrationId(), sourcePackageEntityId);
return new ResolvedEntityRefs(
driverRef.driverId(),
driverRef.driverCardId(),
vehicleRef.vehicleId(),
vehicleRef.vehicleRegistrationId(),
sourcePackageEntityId
);
}
private UUID resolveDriverId(
private DriverIdentityRepository.ResolvedDriverReference resolveDriverReference(
String tenantKey,
int eventSourceId,
EventHubEventDto event,
Map<String, UUID> entityIdCache
Map<String, DriverIdentityRepository.ResolvedDriverReference> driverRefCache
) {
DriverRefDto driverRef = event.driverRef();
if (driverRef == null || !driverRef.hasAnyReference()) {
return null;
return DriverIdentityRepository.ResolvedDriverReference.empty();
}
String cacheKey = "DRIVER|" + driverRef.stableKey();
UUID cached = entityIdCache.get(cacheKey);
DriverIdentityRepository.ResolvedDriverReference cached = driverRefCache.get(cacheKey);
if (cached != null) {
return cached;
}
UUID resolved = driverIdentityRepository.resolveOrCreateDriverId(tenantKey, eventSourceId, driverRef);
if (resolved != null) {
entityIdCache.put(cacheKey, resolved);
}
DriverIdentityRepository.ResolvedDriverReference resolved =
driverIdentityRepository.resolveOrCreateDriverReference(tenantKey, eventSourceId, driverRef);
driverRefCache.put(cacheKey, resolved);
return resolved;
}
@ -587,6 +597,7 @@ public class EventRepository {
private record ResolvedEntityRefs(
UUID driverId,
UUID driverCardId,
UUID vehicleId,
UUID vehicleRegistrationId,
UUID sourcePackageEntityId
@ -606,6 +617,7 @@ public class EventRepository {
int eventSourceId,
String externalSourceEventId,
UUID driverId,
UUID driverCardId,
UUID vehicleId,
UUID vehicleRegistrationId,
String sourcePackageId,

849
src/main/java/at/procon/eventhub/persistence/VehicleIdentityRepository.java
View File

File diff suppressed because it is too large Load Diff

17
src/main/java/at/procon/eventhub/tachograph/service/TachographMasterDataRefreshService.java
View File

@ -1,5 +1,6 @@
package at.procon.eventhub.tachograph.service;
import at.procon.eventhub.config.EventHubProperties;
import at.procon.eventhub.importing.masterdata.MasterDataRefreshResult;
import at.procon.eventhub.importing.masterdata.SourceMasterEntityUpsert;
import at.procon.eventhub.importing.masterdata.SourceMasterRelationUpsert;
@ -56,6 +57,7 @@ public class TachographMasterDataRefreshService {
private final DriverIdentityRepository driverIdentityRepository;
private final VehicleIdentityRepository vehicleIdentityRepository;
private final ResourceLoader resourceLoader;
private final EventHubProperties properties;
public TachographMasterDataRefreshService(
@Qualifier("tachographNamedParameterJdbcTemplate") ObjectProvider<NamedParameterJdbcTemplate> tachographJdbcTemplateProvider,
@ -63,7 +65,8 @@ public class TachographMasterDataRefreshService {
EventSourceRepository eventSourceRepository,
DriverIdentityRepository driverIdentityRepository,
VehicleIdentityRepository vehicleIdentityRepository,
ResourceLoader resourceLoader
ResourceLoader resourceLoader,
EventHubProperties properties
) {
this.tachographJdbcTemplateProvider = tachographJdbcTemplateProvider;
this.sourceMasterDataRepository = sourceMasterDataRepository;
@ -71,6 +74,7 @@ public class TachographMasterDataRefreshService {
this.driverIdentityRepository = driverIdentityRepository;
this.vehicleIdentityRepository = vehicleIdentityRepository;
this.resourceLoader = resourceLoader;
this.properties = properties;
}
public MasterDataRefreshResult refreshIfRequested(TachographImportRequest request) {
@ -101,14 +105,19 @@ public class TachographMasterDataRefreshService {
}
int relationCount = streamRelations(tachographJdbcTemplate, tenantKey, eventSourceId, RELATIONS_SQL_RESOURCE, loadSql(RELATIONS_SQL_RESOURCE));
boolean syncVehicleRegistrations = properties.getTachograph().isSyncVehicleRegistrationsOnMasterDataUpdate();
log.info("Reconciling tachograph driver identities from source master data tenant={} source={}",
tenantKey, masterDataSource.stableKey());
int reconciledDrivers = driverIdentityRepository.reconcileFromMasterData(tenantKey, eventSourceId);
log.info("Reconciling tachograph vehicle identities from source master data tenant={} source={}",
tenantKey, masterDataSource.stableKey());
int reconciledVehicles = vehicleIdentityRepository.reconcileFromMasterData(tenantKey, eventSourceId);
log.info("Reconciling tachograph vehicle identities from source master data tenant={} source={} syncVehicleRegistrations={}",
tenantKey, masterDataSource.stableKey(), syncVehicleRegistrations);
int reconciledVehicles = vehicleIdentityRepository.reconcileFromMasterData(
tenantKey,
eventSourceId,
syncVehicleRegistrations
);
MasterDataRefreshResult result = new MasterDataRefreshResult(entities, relationCount);
log.info("Refreshed tachograph source master data tenant={} source={} entities={} relations={} reconciledDrivers={} reconciledVehicles={}",

24
src/main/java/at/procon/eventhub/yellowfox/api/YellowFoxD8ImportController.java
View File

@ -3,6 +3,7 @@ package at.procon.eventhub.yellowfox.api;
import at.procon.eventhub.dto.AcquisitionStrategy;
import at.procon.eventhub.dto.ImportMode;
import at.procon.eventhub.dto.SchedulerTriggerMode;
import at.procon.eventhub.importing.masterdata.MasterDataRefreshResult;
import at.procon.eventhub.yellowfox.dto.ConfiguredYellowFoxD8ImportPlanDto;
import at.procon.eventhub.yellowfox.dto.YellowFoxD8ImportRequest;
import at.procon.eventhub.yellowfox.dto.YellowFoxD8ImportRunResultDto;
@ -10,6 +11,7 @@ import at.procon.eventhub.yellowfox.dto.YellowFoxD8ImportTriggerResultDto;
import at.procon.eventhub.yellowfox.service.YellowFoxD8ConfiguredImportPlanService;
import at.procon.eventhub.yellowfox.service.YellowFoxD8ImportExecutionService;
import at.procon.eventhub.yellowfox.service.YellowFoxD8ImportPlanService;
import at.procon.eventhub.yellowfox.service.YellowFoxMasterDataRefreshService;
import jakarta.validation.Valid;
import java.time.OffsetDateTime;
import java.util.List;
@ -31,17 +33,20 @@ public class YellowFoxD8ImportController {
private final YellowFoxD8ImportPlanService importPlanService;
private final YellowFoxD8ConfiguredImportPlanService configuredImportPlanService;
private final YellowFoxD8ImportExecutionService importExecutionService;
private final YellowFoxMasterDataRefreshService masterDataRefreshService;
public YellowFoxD8ImportController(
ProducerTemplate producerTemplate,
YellowFoxD8ImportPlanService importPlanService,
YellowFoxD8ConfiguredImportPlanService configuredImportPlanService,
YellowFoxD8ImportExecutionService importExecutionService
YellowFoxD8ImportExecutionService importExecutionService,
YellowFoxMasterDataRefreshService masterDataRefreshService
) {
this.producerTemplate = producerTemplate;
this.importPlanService = importPlanService;
this.configuredImportPlanService = configuredImportPlanService;
this.importExecutionService = importExecutionService;
this.masterDataRefreshService = masterDataRefreshService;
}
@PostMapping("/imports/plan")
@ -60,6 +65,13 @@ public class YellowFoxD8ImportController {
return ResponseEntity.accepted().body(result);
}
@PostMapping("/master-data/refresh")
public ResponseEntity<MasterDataRefreshResult> refreshYellowFoxMasterData(
@Valid @RequestBody YellowFoxD8ImportRequest request
) {
return ResponseEntity.ok(masterDataRefreshService.refresh(request));
}
@GetMapping("/imports/configured-plans")
public ResponseEntity<List<ConfiguredYellowFoxD8ImportPlanDto>> listConfiguredYellowFoxPlans() {
return ResponseEntity.ok(configuredImportPlanService.listPlans());
@ -90,4 +102,14 @@ public class YellowFoxD8ImportController {
result
));
}
@PostMapping("/imports/configured-plans/{planKey}/master-data/refresh")
public ResponseEntity<MasterDataRefreshResult> refreshConfiguredYellowFoxMasterData(
@PathVariable String planKey,
@RequestParam(required = false) ImportMode mode,
@RequestParam(required = false) AcquisitionStrategy strategy
) {
YellowFoxD8ImportRequest request = configuredImportPlanService.createRequest(planKey, mode, strategy);
return ResponseEntity.ok(masterDataRefreshService.refresh(request));
}
}

1
src/main/java/at/procon/eventhub/yellowfox/dto/YellowFoxD8BookingDto.java
View File

@ -15,6 +15,7 @@ public record YellowFoxD8BookingDto(
String eventId,
String key,
Integer ignition,
Integer previousIgnition,
Integer eventType,
Integer state,
OffsetDateTime occurredAt,

121
src/main/java/at/procon/eventhub/yellowfox/service/JdbcYellowFoxD8BookingExtractionBatchExecutor.java
View File

@ -83,15 +83,15 @@ public class JdbcYellowFoxD8BookingExtractionBatchExecutor implements YellowFoxD
ImportScopeDto chunkScope = chunkScope(request.importScope(), chunk);
ImportCursorStateDto cursor = findCursor(eventSourceId, request, planItem);
Map<String, Object> params = parameters(request, chunkScope, cursor);
QuerySpec query = buildQuerySpec(request, chunkScope, cursor);
Stats stats = new Stats();
YellowFoxD8IgnitionTransitionDetector.Session ignitionSession = ignitionTransitionDetector
.newSession(properties.getYellowFox().isEmitInitialIgnitionSnapshot());
log.info("Reading YellowFox D8 bookings tenant={} importRunId={} packageId={} chunk={} occurredFrom={} occurredTo={} fleetId={} strategy={}",
request.tenantKey(), importRunId, packageId, chunk.sequence(), chunk.occurredFrom(), chunk.occurredTo(), params.get("fleetId"), request.acquisitionStrategy());
request.tenantKey(), importRunId, packageId, chunk.sequence(), chunk.occurredFrom(), chunk.occurredTo(), query.fleetId(), request.acquisitionStrategy());
jdbcTemplate.query(loadSql(), params, rs -> {
jdbcTemplate.query(query.sql(), query.params(), rs -> {
stats.sourceRowsRead++;
YellowFoxD8BookingDto booking = rowMapper.map(
rs,
@ -136,9 +136,33 @@ public class JdbcYellowFoxD8BookingExtractionBatchExecutor implements YellowFoxD
);
}
private ImportCursorStateDto findCursor(int eventSourceId, YellowFoxD8ImportRequest request, ImportPlanItemDto planItem) {
QuerySpec buildQuerySpec(YellowFoxD8ImportRequest request, ImportScopeDto scope, ImportCursorStateDto cursor) {
Map<String, Object> params = new HashMap<>();
StringBuilder filters = new StringBuilder("where 1 = 1");
if (scope != null && scope.occurredFrom() != null) {
params.put("occurredFrom", scope.occurredFrom());
filters.append("\n and b.utc >= :occurredFrom");
}
if (scope != null && scope.occurredTo() != null) {
params.put("occurredTo", scope.occurredTo());
filters.append("\n and b.utc < :occurredTo");
}
Integer fleetId = fleetId(request);
if (fleetId != null) {
params.put("fleetId", fleetId);
filters.append("\n and f.id = :fleetId");
}
appendCursorFilter(filters, params, request, cursor);
return new QuerySpec(applyFilters(loadSqlTemplate(), filters.toString()), params, fleetId);
}
ImportCursorStateDto findCursor(int eventSourceId, YellowFoxD8ImportRequest request, ImportPlanItemDto planItem) {
String scopeHash = request.importScope() == null ? "NO_SCOPE" : request.importScope().stableKey();
return importCursorRepository.findCursor(
ImportCursorStateDto cursor = importCursorRepository.findCursor(
request.tenantKey(),
eventSourceId,
scopeHash,
@ -146,28 +170,16 @@ public class JdbcYellowFoxD8BookingExtractionBatchExecutor implements YellowFoxD
planItem.sourceKind(),
request.acquisitionStrategy()
);
}
private Map<String, Object> parameters(YellowFoxD8ImportRequest request, ImportScopeDto scope, ImportCursorStateDto cursor) {
Map<String, Object> params = new HashMap<>();
params.put("occurredFrom", scope == null ? null : scope.occurredFrom());
params.put("occurredTo", scope == null ? null : scope.occurredTo());
params.put("fleetId", fleetId(request));
if (request.acquisitionStrategy() == AcquisitionStrategy.SOURCE_ROW_WATERMARK && cursor != null) {
OffsetDateTime lastOccurredTo = cursor.lastOccurredTo();
String lastSourceRowId = cursor.lastSourcePackageId();
if (lastOccurredTo != null && properties.getYellowFox().getOccurredAtOverlap() != null
&& !properties.getYellowFox().getOccurredAtOverlap().isZero()) {
lastOccurredTo = lastOccurredTo.minus(properties.getYellowFox().getOccurredAtOverlap());
lastSourceRowId = null;
}
params.put("lastOccurredTo", lastOccurredTo);
params.put("lastSourceRowId", lastSourceRowId);
} else {
params.put("lastOccurredTo", null);
params.put("lastSourceRowId", null);
if (cursor != null || !shouldBootstrapWatermarkCursor(request)) {
return cursor;
}
return params;
return importCursorRepository.findLatestCursor(
request.tenantKey(),
eventSourceId,
scopeHash,
planItem.eventFamily(),
planItem.sourceKind()
);
}
private Integer fleetId(YellowFoxD8ImportRequest request) {
@ -204,7 +216,59 @@ public class JdbcYellowFoxD8BookingExtractionBatchExecutor implements YellowFoxD
stats.acceptEvent(event);
}
private String loadSql() {
private void appendCursorFilter(
StringBuilder filters,
Map<String, Object> params,
YellowFoxD8ImportRequest request,
ImportCursorStateDto cursor
) {
if (request.acquisitionStrategy() != AcquisitionStrategy.SOURCE_ROW_WATERMARK || cursor == null) {
return;
}
OffsetDateTime lastOccurredTo = cursor.lastOccurredTo();
String lastSourceRowId = cursor.lastSourcePackageId();
if (lastOccurredTo == null) {
return;
}
if (properties.getYellowFox().getOccurredAtOverlap() != null
&& !properties.getYellowFox().getOccurredAtOverlap().isZero()) {
lastOccurredTo = lastOccurredTo.minus(properties.getYellowFox().getOccurredAtOverlap());
lastSourceRowId = null;
}
params.put("lastOccurredTo", lastOccurredTo);
if (lastSourceRowId == null || lastSourceRowId.isBlank()) {
filters.append("\n and b.utc >= :lastOccurredTo");
return;
}
params.put("lastSourceRowId", lastSourceRowId);
filters.append("""
and (
b.utc > :lastOccurredTo
or (
b.utc = :lastOccurredTo
and b.eventid > :lastSourceRowId
)
)""");
}
private boolean shouldBootstrapWatermarkCursor(YellowFoxD8ImportRequest request) {
return request.mode() == at.procon.eventhub.dto.ImportMode.INCREMENTAL_UPDATE
&& (request.acquisitionStrategy() == AcquisitionStrategy.SOURCE_ROW_WATERMARK
|| request.acquisitionStrategy() == AcquisitionStrategy.SOURCE_PACKAGE_WATERMARK);
}
private String applyFilters(String sqlTemplate, String filters) {
if (!sqlTemplate.contains("/*__FILTERS__*/")) {
throw new IllegalStateException("YellowFox D8 extraction SQL template is missing filter marker");
}
return sqlTemplate.replace("/*__FILTERS__*/", filters);
}
private String loadSqlTemplate() {
Resource resource = resourceLoader.getResource("classpath:sql/yellowfox/d8-bookings.sql");
try (var in = resource.getInputStream()) {
return StreamUtils.copyToString(in, StandardCharsets.UTF_8);
@ -213,6 +277,9 @@ public class JdbcYellowFoxD8BookingExtractionBatchExecutor implements YellowFoxD
}
}
static record QuerySpec(String sql, Map<String, Object> params, Integer fleetId) {
}
private static class Stats {
private int sourceRowsRead;
private int eventsSent;

65
src/main/java/at/procon/eventhub/yellowfox/service/YellowFoxD8BookingRowMapper.java
View File

@ -28,40 +28,43 @@ public class YellowFoxD8BookingRowMapper {
}
public YellowFoxD8BookingDto map(ResultSet rs, String tenantKey, String sourceInstanceKey, String tenantProviderSettingKey) throws SQLException {
String eventId = rs.getString("eventid");
String eventId = string(rs, "eventid");
OffsetDateTime occurredAt = rs.getObject("utc", OffsetDateTime.class);
Integer vehicleId = getInteger(rs, "vehicle_id");
Integer driverId = getInteger(rs, "driver_id");
String vehicleVrn = rs.getString("vehicle_vrn");
String vehicleVin = rs.getString("vehicle_vin");
String driverCard = rs.getString("driver_card_number");
String vehicleVrn = string(rs, "vehicle_vrn");
String vehicleVin = string(rs, "vehicle_vin");
String driverCard = normalizeBookingDriverCardNumber(string(rs, "driver_card_number"));
Integer fleetId = getInteger(rs, "fleet_id");
String fleetName = rs.getString("fleet_name");
String fleetName = string(rs, "fleet_name");
Integer odometer = getInteger(rs, "odometer");
Map<String, Object> payload = payload(rs.getString("payload"));
Map<String, Object> payload = trimPayloadStrings(payload(rs.getString("payload")));
put(payload, "yellowFoxEventId", eventId);
put(payload, "yellowFoxOdometerRaw", odometer);
put(payload, "vehicleVrn", vehicleVrn);
put(payload, "vehicleVin", vehicleVin);
put(payload, "driverCardNumber", driverCard);
put(payload, "driverFirstName", rs.getString("driver_firstname"));
put(payload, "driverLastName", rs.getString("driver_lastname"));
put(payload, "driverFirstName", string(rs, "driver_firstname"));
put(payload, "driverLastName", string(rs, "driver_lastname"));
put(payload, "fleetId", fleetId);
put(payload, "fleetName", fleetName);
put(payload, "telematicProviderId", getInteger(rs, "telematic_provider_id"));
put(payload, "telematicProviderName", rs.getString("telematic_provider_name"));
put(payload, "telematicProviderName", string(rs, "telematic_provider_name"));
DriverRefDto driverRef = driverId == null && isBlank(driverCard)
? null
: new DriverRefDto(driverId == null ? null : driverId.toString(), new DriverCardRefDto(null, driverCard));
: new DriverRefDto(
driverId == null ? null : driverId.toString(),
new DriverCardRefDto(YellowFoxReferenceSemantics.SYNTHETIC_REFERENCE_NATION, driverCard)
);
VehicleRefDto vehicleRef = vehicleId == null && isBlank(vehicleVin) && isBlank(vehicleVrn)
? null
: new VehicleRefDto(
vehicleId == null ? null : vehicleId.toString(),
vehicleVin,
vehicleId == null ? null : vehicleId.toString(),
new VehicleRegistrationRefDto(null, vehicleVrn)
new VehicleRegistrationRefDto(YellowFoxReferenceSemantics.SYNTHETIC_REFERENCE_NATION, vehicleVrn)
);
return new YellowFoxD8BookingDto(
@ -71,8 +74,9 @@ public class YellowFoxD8BookingRowMapper {
fleetId == null ? null : fleetId.toString(),
fleetName,
eventId,
rs.getString("key"),
string(rs, "key"),
getInteger(rs, "ignition"),
getInteger(rs, "previous_ignition"),
getInteger(rs, "eventtype"),
getInteger(rs, "state"),
occurredAt,
@ -105,6 +109,35 @@ public class YellowFoxD8BookingRowMapper {
}
}
private String string(ResultSet rs, String column) throws SQLException {
String value = rs.getString(column);
return value == null || value.isBlank() ? null : value.trim();
}
@SuppressWarnings("unchecked")
private Map<String, Object> trimPayloadStrings(Map<String, Object> payload) {
payload.replaceAll((key, value) -> trimPayloadValue(value));
return payload;
}
@SuppressWarnings("unchecked")
private Object trimPayloadValue(Object value) {
if (value instanceof String text) {
return text.trim();
}
if (value instanceof Map<?, ?> nestedMap) {
((Map<Object, Object>) nestedMap).replaceAll((key, nestedValue) -> trimPayloadValue(nestedValue));
return nestedMap;
}
if (value instanceof java.util.List<?> list) {
for (int i = 0; i < list.size(); i++) {
((java.util.List<Object>) list).set(i, trimPayloadValue(list.get(i)));
}
return list;
}
return value;
}
private void put(Map<String, Object> target, String key, Object value) {
if (value != null) {
target.put(key, value instanceof BigDecimal bd ? bd.stripTrailingZeros().toPlainString() : value);
@ -114,4 +147,12 @@ public class YellowFoxD8BookingRowMapper {
private boolean isBlank(String value) {
return value == null || value.isBlank();
}
private String normalizeBookingDriverCardNumber(String value) {
if (value == null || value.isBlank()) {
return null;
}
String trimmed = value.trim();
return trimmed.length() <= 14 ? trimmed : trimmed.substring(0, 14);
}
}

4
src/main/java/at/procon/eventhub/yellowfox/service/YellowFoxD8ConfiguredImportPlanService.java
View File

@ -32,7 +32,7 @@ public class YellowFoxD8ConfiguredImportPlanService
scope,
plan.getEventFamilies(),
mode,
false,
plan.isRefreshMasterDataFirst(),
strategy
);
}
@ -53,7 +53,7 @@ public class YellowFoxD8ConfiguredImportPlanService
dto.scheduledMode(),
dto.initialStrategy(),
dto.scheduledStrategy(),
false,
dto.refreshMasterDataFirst(),
dto.initialOccurredFrom(),
dto.initialOccurredTo(),
dto.runInitialOnStartup()

5
src/main/java/at/procon/eventhub/yellowfox/service/YellowFoxD8IgnitionTransitionDetector.java
View File

@ -36,7 +36,10 @@ public class YellowFoxD8IgnitionTransitionDetector {
String vehicleKey = booking.vehicleRef().stableKey();
Integer previous = lastIgnitionByVehicle.put(vehicleKey, booking.ignition());
if (previous == null) {
return emitInitialSnapshot ? mapper.mapIgnitionTransition(booking, null) : null;
previous = booking.previousIgnition();
if (previous == null) {
return emitInitialSnapshot ? mapper.mapIgnitionTransition(booking, null) : null;
}
}
if (!previous.equals(booking.ignition())) {
return mapper.mapIgnitionTransition(booking, previous);

8
src/main/java/at/procon/eventhub/yellowfox/service/YellowFoxD8ImportExecutionService.java
View File

@ -23,6 +23,7 @@ public class YellowFoxD8ImportExecutionService
extends AbstractImportExecutionService<YellowFoxD8ImportRequest, YellowFoxD8ExtractionBatchResultDto> {
private final YellowFoxD8ImportPlanService planService;
private final YellowFoxMasterDataRefreshService masterDataRefreshService;
private final YellowFoxD8ExtractionBatchExecutor extractionBatchExecutor;
public YellowFoxD8ImportExecutionService(
@ -31,10 +32,12 @@ public class YellowFoxD8ImportExecutionService
ImportRunRepository importRunRepository,
DataPackageRepository dataPackageRepository,
ImportCursorRepository importCursorRepository,
YellowFoxMasterDataRefreshService masterDataRefreshService,
YellowFoxD8ExtractionBatchExecutor extractionBatchExecutor
) {
super(eventSourceRepository, importRunRepository, dataPackageRepository, importCursorRepository);
this.planService = planService;
this.masterDataRefreshService = masterDataRefreshService;
this.extractionBatchExecutor = extractionBatchExecutor;
}
@ -64,6 +67,11 @@ public class YellowFoxD8ImportExecutionService
return extractionBatchExecutor.execute(importRunId, packageId, eventSourceId, request, planItem, chunk);
}
@Override
protected void beforeExecute(YellowFoxD8ImportRequest request) {
masterDataRefreshService.refreshIfRequested(request);
}
@Override
protected EventSourceDto eventSourceForItem(EventSourceDto base, ImportPlanItemDto item) {
return new EventSourceDto(

335
src/main/java/at/procon/eventhub/yellowfox/service/YellowFoxMasterDataRefreshService.java Normal file
View File

@ -0,0 +1,335 @@
package at.procon.eventhub.yellowfox.service;
import at.procon.eventhub.config.EventHubProperties;
import at.procon.eventhub.dto.EventSourceDto;
import at.procon.eventhub.importing.masterdata.MasterDataRefreshResult;
import at.procon.eventhub.importing.masterdata.SourceMasterEntityUpsert;
import at.procon.eventhub.importing.masterdata.SourceMasterRelationUpsert;
import at.procon.eventhub.persistence.DriverIdentityRepository;
import at.procon.eventhub.persistence.EventSourceRepository;
import at.procon.eventhub.persistence.SourceMasterDataRepository;
import at.procon.eventhub.persistence.VehicleIdentityRepository;
import at.procon.eventhub.yellowfox.dto.YellowFoxD8ImportRequest;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.sql.ResultSet;
import java.sql.ResultSetMetaData;
import java.sql.SQLException;
import java.sql.Timestamp;
import java.time.LocalDateTime;
import java.time.OffsetDateTime;
import java.time.ZoneOffset;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.ObjectProvider;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.core.io.Resource;
import org.springframework.core.io.ResourceLoader;
import org.springframework.jdbc.core.namedparam.NamedParameterJdbcTemplate;
import org.springframework.stereotype.Service;
import org.springframework.util.StreamUtils;
@Service
public class YellowFoxMasterDataRefreshService {
private static final Logger log = LoggerFactory.getLogger(YellowFoxMasterDataRefreshService.class);
private static final List<String> ENTITY_SQL_RESOURCES = List.of(
"classpath:sql/yellowfox/master-data/fleets.sql",
"classpath:sql/yellowfox/master-data/drivers.sql",
"classpath:sql/yellowfox/master-data/driver-cards.sql",
"classpath:sql/yellowfox/master-data/vehicles.sql",
"classpath:sql/yellowfox/master-data/vehicle-registrations.sql",
"classpath:sql/yellowfox/master-data/telematic-providers.sql"
);
private static final String RELATIONS_SQL_RESOURCE = "classpath:sql/yellowfox/master-data/relations.sql";
private static final String MASTER_DATA_SOURCE_KIND = "MASTER_DATA";
private static final String MASTER_DATA_SOURCE_KEY = "YELLOWFOX_MASTER_DATA";
private static final int MASTER_DATA_UPSERT_BATCH_SIZE = 5000;
private final ObjectProvider<NamedParameterJdbcTemplate> yellowFoxJdbcTemplateProvider;
private final SourceMasterDataRepository sourceMasterDataRepository;
private final EventSourceRepository eventSourceRepository;
private final DriverIdentityRepository driverIdentityRepository;
private final VehicleIdentityRepository vehicleIdentityRepository;
private final ResourceLoader resourceLoader;
private final EventHubProperties properties;
public YellowFoxMasterDataRefreshService(
@Qualifier("yellowFoxNamedParameterJdbcTemplate") ObjectProvider<NamedParameterJdbcTemplate> yellowFoxJdbcTemplateProvider,
SourceMasterDataRepository sourceMasterDataRepository,
EventSourceRepository eventSourceRepository,
DriverIdentityRepository driverIdentityRepository,
VehicleIdentityRepository vehicleIdentityRepository,
ResourceLoader resourceLoader,
EventHubProperties properties
) {
this.yellowFoxJdbcTemplateProvider = yellowFoxJdbcTemplateProvider;
this.sourceMasterDataRepository = sourceMasterDataRepository;
this.eventSourceRepository = eventSourceRepository;
this.driverIdentityRepository = driverIdentityRepository;
this.vehicleIdentityRepository = vehicleIdentityRepository;
this.resourceLoader = resourceLoader;
this.properties = properties;
}
public MasterDataRefreshResult refreshIfRequested(YellowFoxD8ImportRequest request) {
if (!request.refreshMasterDataFirst()) {
return MasterDataRefreshResult.empty();
}
return refresh(request);
}
public MasterDataRefreshResult refresh(YellowFoxD8ImportRequest request) {
NamedParameterJdbcTemplate yellowFoxJdbcTemplate = yellowFoxJdbcTemplateProvider.getIfAvailable();
if (yellowFoxJdbcTemplate == null) {
log.info("Skipping YellowFox master-data refresh for tenant={} because no YellowFox datasource is configured.",
request.tenantKey());
return MasterDataRefreshResult.empty();
}
String tenantKey = request.tenantKey() == null || request.tenantKey().isBlank() ? "default" : request.tenantKey().trim();
EventSourceDto masterDataSource = masterDataSourceFor(request.eventSource());
int eventSourceId = eventSourceRepository.resolveSourceId(tenantKey, masterDataSource);
log.info("Starting YellowFox source master-data refresh tenant={} source={} batchSize={}",
tenantKey, masterDataSource.stableKey(), MASTER_DATA_UPSERT_BATCH_SIZE);
int entities = 0;
for (String sqlResource : ENTITY_SQL_RESOURCES) {
entities += streamEntities(yellowFoxJdbcTemplate, tenantKey, eventSourceId, sqlResource, loadSql(sqlResource));
}
int relationCount = streamRelations(yellowFoxJdbcTemplate, tenantKey, eventSourceId, RELATIONS_SQL_RESOURCE, loadSql(RELATIONS_SQL_RESOURCE));
boolean syncVehicleRegistrations = properties.getYellowFox().isSyncVehicleRegistrationsOnMasterDataUpdate();
log.info("Reconciling YellowFox driver identities from source master data tenant={} source={}",
tenantKey, masterDataSource.stableKey());
int reconciledDrivers = driverIdentityRepository.reconcileFromMasterData(tenantKey, eventSourceId);
log.info("Reconciling YellowFox vehicle identities from source master data tenant={} source={} syncVehicleRegistrations={}",
tenantKey, masterDataSource.stableKey(), syncVehicleRegistrations);
int reconciledVehicles = vehicleIdentityRepository.reconcileFromMasterData(
tenantKey,
eventSourceId,
syncVehicleRegistrations
);
MasterDataRefreshResult result = new MasterDataRefreshResult(entities, relationCount);
log.info("Refreshed YellowFox source master data tenant={} source={} entities={} relations={} reconciledDrivers={} reconciledVehicles={}",
tenantKey, masterDataSource.stableKey(), result.entitiesUpserted(), result.relationsUpserted(), reconciledDrivers, reconciledVehicles);
return result;
}
private int streamEntities(
NamedParameterJdbcTemplate yellowFoxJdbcTemplate,
String tenantKey,
int eventSourceId,
String sqlResource,
String sql
) {
String section = masterDataSection(sqlResource);
List<SourceMasterEntityUpsert> buffer = new ArrayList<>(MASTER_DATA_UPSERT_BATCH_SIZE);
Map<String, Integer> typeCounts = new LinkedHashMap<>();
int[] count = {0};
int[] chunk = {0};
log.info("Reading YellowFox master-data entities tenant={} section={}", tenantKey, section);
yellowFoxJdbcTemplate.query(sql, Map.of(), rs -> {
SourceMasterEntityUpsert entity = entity(rs);
buffer.add(entity);
increment(typeCounts, entity.entityType());
if (buffer.size() >= MASTER_DATA_UPSERT_BATCH_SIZE) {
count[0] += flushEntities(tenantKey, eventSourceId, section, ++chunk[0], buffer, count[0], typeCounts);
buffer.clear();
}
});
if (!buffer.isEmpty()) {
count[0] += flushEntities(tenantKey, eventSourceId, section, ++chunk[0], buffer, count[0], typeCounts);
}
log.info("Finished YellowFox master-data entities tenant={} section={} processed={} byType={}",
tenantKey, section, count[0], typeCounts);
return count[0];
}
private int streamRelations(
NamedParameterJdbcTemplate yellowFoxJdbcTemplate,
String tenantKey,
int eventSourceId,
String sqlResource,
String sql
) {
String section = masterDataSection(sqlResource);
List<SourceMasterRelationUpsert> buffer = new ArrayList<>(MASTER_DATA_UPSERT_BATCH_SIZE);
Map<String, Integer> typeCounts = new LinkedHashMap<>();
int[] count = {0};
int[] chunk = {0};
log.info("Reading YellowFox master-data relations tenant={} section={}", tenantKey, section);
yellowFoxJdbcTemplate.query(sql, Map.of(), rs -> {
SourceMasterRelationUpsert relation = relation(rs);
buffer.add(relation);
increment(typeCounts, relation.relationType());
if (buffer.size() >= MASTER_DATA_UPSERT_BATCH_SIZE) {
count[0] += flushRelations(tenantKey, eventSourceId, section, ++chunk[0], buffer, count[0], typeCounts);
buffer.clear();
}
});
if (!buffer.isEmpty()) {
count[0] += flushRelations(tenantKey, eventSourceId, section, ++chunk[0], buffer, count[0], typeCounts);
}
log.info("Finished YellowFox master-data relations tenant={} section={} processed={} byType={}",
tenantKey, section, count[0], typeCounts);
return count[0];
}
private int flushEntities(
String tenantKey,
int eventSourceId,
String section,
int chunk,
List<SourceMasterEntityUpsert> buffer,
int alreadyProcessed,
Map<String, Integer> typeCounts
) {
int upserted = sourceMasterDataRepository.upsertEntities(tenantKey, eventSourceId, buffer);
log.info("YellowFox master-data entity progress tenant={} section={} chunk={} chunkSize={} processed={} byType={}",
tenantKey, section, chunk, buffer.size(), alreadyProcessed + upserted, typeCounts);
return upserted;
}
private int flushRelations(
String tenantKey,
int eventSourceId,
String section,
int chunk,
List<SourceMasterRelationUpsert> buffer,
int alreadyProcessed,
Map<String, Integer> typeCounts
) {
int upserted = sourceMasterDataRepository.upsertRelations(tenantKey, eventSourceId, buffer);
log.info("YellowFox master-data relation progress tenant={} section={} chunk={} chunkSize={} processed={} byType={}",
tenantKey, section, chunk, buffer.size(), alreadyProcessed + upserted, typeCounts);
return upserted;
}
private void increment(Map<String, Integer> counts, String type) {
String key = type == null || type.isBlank() ? "UNKNOWN" : type;
counts.merge(key, 1, Integer::sum);
}
private String masterDataSection(String sqlResource) {
int slash = sqlResource.lastIndexOf('/');
String fileName = slash < 0 ? sqlResource : sqlResource.substring(slash + 1);
return fileName.endsWith(".sql") ? fileName.substring(0, fileName.length() - 4) : fileName;
}
private EventSourceDto masterDataSourceFor(EventSourceDto source) {
return new EventSourceDto(
source.providerKey(),
MASTER_DATA_SOURCE_KIND,
MASTER_DATA_SOURCE_KEY,
source.sourceInstanceKey(),
source.tenantProviderSettingKey(),
source.externalFleetKey()
);
}
private SourceMasterEntityUpsert entity(ResultSet rs) throws SQLException {
String entityType = string(rs, "entity_type");
String sourceEntityId = string(rs, "source_entity_id");
return new SourceMasterEntityUpsert(
entityType,
sourceEntityId,
string(rs, "source_external_key"),
string(rs, "display_name"),
bool(rs, "active"),
offsetDateTime(rs, "valid_from"),
offsetDateTime(rs, "valid_to"),
offsetDateTime(rs, "source_updated_at"),
payload(rs)
);
}
private SourceMasterRelationUpsert relation(ResultSet rs) throws SQLException {
return new SourceMasterRelationUpsert(
string(rs, "relation_type"),
string(rs, "from_entity_type"),
string(rs, "from_source_entity_id"),
string(rs, "to_entity_type"),
string(rs, "to_source_entity_id"),
offsetDateTime(rs, "valid_from"),
offsetDateTime(rs, "valid_to"),
offsetDateTime(rs, "source_updated_at"),
payload(rs)
);
}
private Map<String, Object> payload(ResultSet rs) throws SQLException {
ResultSetMetaData metaData = rs.getMetaData();
Map<String, Object> payload = new LinkedHashMap<>();
for (int i = 1; i <= metaData.getColumnCount(); i++) {
String name = metaData.getColumnLabel(i);
Object value = rs.getObject(i);
if (value != null) {
payload.put(name, value);
}
}
return payload;
}
private String loadSql(String location) {
Resource resource = resourceLoader.getResource(location);
try (var inputStream = resource.getInputStream()) {
return StreamUtils.copyToString(inputStream, StandardCharsets.UTF_8);
} catch (IOException e) {
throw new IllegalStateException("Cannot load YellowFox master-data SQL resource " + location, e);
}
}
private String string(ResultSet rs, String column) throws SQLException {
String value = rs.getString(column);
return value == null || value.isBlank() ? null : value.trim();
}
private Boolean bool(ResultSet rs, String column) throws SQLException {
Object value = rs.getObject(column);
if (value == null) {
return null;
}
if (value instanceof Boolean bool) {
return bool;
}
if (value instanceof Number number) {
return number.intValue() != 0;
}
return Boolean.parseBoolean(value.toString());
}
private OffsetDateTime offsetDateTime(ResultSet rs, String column) throws SQLException {
Object value = rs.getObject(column);
if (value == null) {
return null;
}
if (value instanceof OffsetDateTime offsetDateTime) {
return offsetDateTime.withOffsetSameInstant(ZoneOffset.UTC);
}
if (value instanceof Timestamp timestamp) {
return timestamp.toLocalDateTime().atOffset(ZoneOffset.UTC);
}
if (value instanceof LocalDateTime localDateTime) {
return localDateTime.atOffset(ZoneOffset.UTC);
}
String text = value.toString();
try {
return OffsetDateTime.parse(text).withOffsetSameInstant(ZoneOffset.UTC);
} catch (RuntimeException ignored) {
return LocalDateTime.parse(text).atOffset(ZoneOffset.UTC);
}
}
}

9
src/main/java/at/procon/eventhub/yellowfox/service/YellowFoxReferenceSemantics.java Normal file
View File

@ -0,0 +1,9 @@
package at.procon.eventhub.yellowfox.service;
final class YellowFoxReferenceSemantics {
static final String SYNTHETIC_REFERENCE_NATION = "YELLOWFOX";
private YellowFoxReferenceSemantics() {
}
}

34
src/main/resources/application.yml
View File

@ -53,6 +53,7 @@ eventhub:
tachograph:
default-chunk-days: 1
occurred-at-overlap: 7d
sync-vehicle-registrations-on-master-data-update: true
# Set TACHOGRAPH_DB_JDBC_URL to enable JdbcTachographExtractionBatchExecutor.
datasource:
@ -63,7 +64,7 @@ eventhub:
# Enables the scheduler that regularly triggers configured tachograph import plans.
# Default is safe: no scheduled import starts unless explicitly enabled.
scheduler-enabled: true
scheduler-enabled: false
scheduler-poll-interval-ms: 3600000
# PLAN_ONLY creates import_run + planned extraction packages.
@ -78,7 +79,7 @@ eventhub:
# Example plan. Keep disabled until the tachograph datasource/extractor is wired.
import-plans:
- plan-key: tachograph-org-14708
enabled: true
enabled: false
cron: "0 15 * * * *" # hourly at minute 15
tenant-key: Procon
event-source:
@ -115,10 +116,10 @@ eventhub:
scheduled-mode: INCREMENTAL_UPDATE
initial-strategy: OCCURRED_AT_WINDOW_WITH_OVERLAP
scheduled-strategy: SOURCE_PACKAGE_WATERMARK
refresh-master-data-first: true
initial-occurred-from: "2026-01-21T00:00:00+01:00"
initial-occurred-to:
run-initial-on-startup: true
refresh-master-data-first: false
initial-occurred-from: "2026-04-01T00:00:00+01:00"
initial-occurred-to: "2026-04-10T00:00:00+01:00"
run-initial-on-startup: false
esper-poc:
activity-merge-mode: JAVA
@ -129,11 +130,13 @@ eventhub:
merge-gap-seconds: 0
gap-detection-tolerance-seconds: 0
unknown-treatment-mode: AS_BREAK_REST
engine-mode: STREAM_COLLECTOR
yellow-fox:
default-chunk-days: 1
occurred-at-overlap: 2h
emit-initial-ignition-snapshot: false
sync-vehicle-registrations-on-master-data-update: false
datasource:
jdbc-url: ${YELLOWFOX_DB_JDBC_URL:}
@ -141,21 +144,24 @@ eventhub:
password: ${YELLOWFOX_DB_PASSWORD:}
driver-class-name: org.postgresql.Driver
scheduler-enabled: false
scheduler-enabled: true
scheduler-poll-interval-ms: 60000
scheduler-trigger-mode: PLAN_ONLY
scheduler-trigger-mode: EXECUTE
import-plans:
- plan-key: yellowfox-d8-default
enabled: false
enabled: true
cron: "0 */5 * * * *"
tenant-key: default
tenant-key: Procon
event-source:
provider-key: YELLOWFOX
source-kind: TELEMATICS_PLATFORM
source-key: YELLOWFOX_D8
source-instance-key: logistics-db-prod
tenant-provider-setting-key: yellowfox-main
source-group:
type: FLEET
source-entity-id: "7"
import-scope:
type: TENANT_ALL
include-children: false
@ -163,8 +169,10 @@ eventhub:
- DRIVER_ACTIVITY
- DRIVER_CARD
initial-mode: INITIAL_BACKFILL
scheduled-mode: INCREMENTAL_UPDATE
scheduled-mode: INITIAL_BACKFILL # INITIAL_BACKFILL, INCREMENTAL_UPDATE
initial-strategy: OCCURRED_AT_WINDOW_WITH_OVERLAP
scheduled-strategy: SOURCE_ROW_WATERMARK
refresh-master-data-first: false
run-initial-on-startup: false
initial-occurred-from: "2026-04-01T00:00:00+01:00"
initial-occurred-to: "2026-04-10T00:00:00+01:00"
refresh-master-data-first: true
run-initial-on-startup: true

152
src/main/resources/db/eventhub_schema_create.sql
View File

@ -145,11 +145,56 @@ create table if not exists eventhub.source_master_relation (
constraint chk_source_master_relation_valid_time_order check (valid_from is null or valid_to is null or valid_from <= valid_to)
);
create table if not exists eventhub.vehicle (
create table if not exists eventhub.driver (
id uuid primary key,
first_names text,
last_name text,
birth_date date,
source_updated_at timestamptz,
payload jsonb not null default '{}'::jsonb,
created_at timestamptz not null default now(),
updated_at timestamptz not null default now()
);
create table if not exists eventhub.driver_card (
id uuid primary key,
driver_id uuid references eventhub.driver(id),
nation text not null,
card_number text not null,
source_updated_at timestamptz,
payload jsonb not null default '{}'::jsonb,
created_at timestamptz not null default now(),
updated_at timestamptz not null default now()
);
create table if not exists eventhub.source_driver_identity (
id uuid primary key,
tenant_key text not null,
event_source_id integer not null references eventhub.event_source(id),
source_vehicle_entity_id text,
source_driver_entity_id text not null,
driver_id uuid not null references eventhub.driver(id) on delete cascade,
source_updated_at timestamptz,
payload jsonb not null default '{}'::jsonb,
created_at timestamptz not null default now(),
updated_at timestamptz not null default now(),
constraint ux_source_driver_identity unique (tenant_key, event_source_id, source_driver_entity_id)
);
create table if not exists eventhub.source_driver_card_identity (
id uuid primary key,
tenant_key text not null,
event_source_id integer not null references eventhub.event_source(id),
source_driver_card_entity_id text not null,
driver_card_id uuid not null references eventhub.driver_card(id) on delete cascade,
source_updated_at timestamptz,
payload jsonb not null default '{}'::jsonb,
created_at timestamptz not null default now(),
updated_at timestamptz not null default now(),
constraint ux_source_driver_card_identity unique (tenant_key, event_source_id, source_driver_card_entity_id)
);
create table if not exists eventhub.vehicle (
id uuid primary key,
vin text,
created_at timestamptz not null default now(),
updated_at timestamptz not null default now()
@ -157,9 +202,6 @@ create table if not exists eventhub.vehicle (
create table if not exists eventhub.vehicle_registration (
id uuid primary key,
tenant_key text not null,
event_source_id integer not null references eventhub.event_source(id),
source_registration_entity_id text,
nation text not null,
registration_number text not null,
source_updated_at timestamptz,
@ -168,6 +210,32 @@ create table if not exists eventhub.vehicle_registration (
updated_at timestamptz not null default now()
);
create table if not exists eventhub.source_vehicle_identity (
id uuid primary key,
tenant_key text not null,
event_source_id integer not null references eventhub.event_source(id),
source_vehicle_entity_id text not null,
vehicle_id uuid not null references eventhub.vehicle(id) on delete cascade,
source_updated_at timestamptz,
payload jsonb not null default '{}'::jsonb,
created_at timestamptz not null default now(),
updated_at timestamptz not null default now(),
constraint ux_source_vehicle_identity unique (tenant_key, event_source_id, source_vehicle_entity_id)
);
create table if not exists eventhub.source_vehicle_registration_identity (
id uuid primary key,
tenant_key text not null,
event_source_id integer not null references eventhub.event_source(id),
source_registration_entity_id text not null,
vehicle_registration_id uuid not null references eventhub.vehicle_registration(id) on delete cascade,
source_updated_at timestamptz,
payload jsonb not null default '{}'::jsonb,
created_at timestamptz not null default now(),
updated_at timestamptz not null default now(),
constraint ux_source_vehicle_registration_identity unique (tenant_key, event_source_id, source_registration_entity_id)
);
create table if not exists eventhub.vehicle_registration_assignment (
id uuid primary key,
tenant_key text not null,
@ -188,7 +256,8 @@ create table if not exists eventhub.event (
event_source_id integer not null references eventhub.event_source(id),
data_package_id uuid not null references eventhub.data_package(id),
external_source_event_id text not null,
driver_entity_id uuid references eventhub.source_master_entity(id),
driver_id uuid references eventhub.driver(id),
driver_card_id uuid references eventhub.driver_card(id),
vehicle_id uuid references eventhub.vehicle(id),
vehicle_registration_id uuid references eventhub.vehicle_registration(id),
source_package_id text,
@ -208,7 +277,8 @@ create table if not exists eventhub.event (
created_at timestamptz not null default now(),
constraint pk_event primary key (occurred_at, id),
constraint chk_event_driver_or_vehicle_ref check (
driver_entity_id is not null
driver_id is not null
or driver_card_id is not null
or vehicle_id is not null
or vehicle_registration_id is not null
)
@ -276,23 +346,12 @@ create index if not exists idx_source_master_relation_to
create index if not exists idx_source_master_relation_payload_gin
on eventhub.source_master_relation using gin(payload);
create index if not exists idx_vehicle_lookup_ctx
on eventhub.vehicle(tenant_key, event_source_id, updated_at desc);
create index if not exists idx_vehicle_source_entity
on eventhub.vehicle(tenant_key, event_source_id, source_vehicle_entity_id)
where source_vehicle_entity_id is not null;
create index if not exists idx_vehicle_vin
on eventhub.vehicle(tenant_key, event_source_id, vin)
on eventhub.vehicle(vin)
where vin is not null;
create index if not exists idx_vehicle_registration_source_entity
on eventhub.vehicle_registration(tenant_key, event_source_id, source_registration_entity_id)
where source_registration_entity_id is not null;
create index if not exists idx_vehicle_registration_plate
on eventhub.vehicle_registration(tenant_key, event_source_id, nation, registration_number);
on eventhub.vehicle_registration(nation, registration_number);
create index if not exists idx_vehicle_registration_assignment_registration_time
on eventhub.vehicle_registration_assignment(vehicle_registration_id, valid_from desc, valid_to);
@ -320,9 +379,42 @@ create index if not exists idx_event_source_package_id
create index if not exists idx_event_domain_type_time
on eventhub.event(event_domain, event_type, occurred_at desc);
create index if not exists idx_driver_card_key
on eventhub.driver_card(nation, card_number);
create index if not exists idx_driver_card_driver
on eventhub.driver_card(driver_id)
where driver_id is not null;
create unique index if not exists ux_driver_card_key
on eventhub.driver_card(nation, card_number);
create unique index if not exists ux_vehicle_vin
on eventhub.vehicle(vin)
where vin is not null;
create unique index if not exists ux_vehicle_registration_plate
on eventhub.vehicle_registration(nation, registration_number);
create index if not exists idx_source_driver_identity_driver
on eventhub.source_driver_identity(driver_id);
create index if not exists idx_source_driver_card_identity_card
on eventhub.source_driver_card_identity(driver_card_id);
create index if not exists idx_source_vehicle_identity_vehicle
on eventhub.source_vehicle_identity(vehicle_id);
create index if not exists idx_source_vehicle_registration_identity_registration
on eventhub.source_vehicle_registration_identity(vehicle_registration_id);
create index if not exists idx_event_driver_time
on eventhub.event(driver_entity_id, occurred_at desc)
where driver_entity_id is not null;
on eventhub.event(driver_id, occurred_at desc)
where driver_id is not null;
create index if not exists idx_event_driver_card_time
on eventhub.event(driver_card_id, occurred_at desc)
where driver_card_id is not null;
create index if not exists idx_event_vehicle_time
on eventhub.event(vehicle_id, occurred_at desc)
@ -344,3 +436,19 @@ create index if not exists idx_event_detail_type
create index if not exists idx_event_detail_attributes_gin
on eventhub.event_detail using gin(attributes);
create index if not exists idx_event_detail_yellowfox_slot
on eventhub.event_detail(detail_type, (attributes ->> 'slot'), event_occurred_at)
where detail_type in ('DRIVER_ACTIVITY', 'DRIVER_CARD');
create index if not exists idx_event_detail_yellowfox_eventtype_state
on eventhub.event_detail(
(attributes ->> 'yellowFoxEventType'),
(attributes ->> 'yellowFoxState'),
event_occurred_at
)
where attributes ? 'yellowFoxEventType';
create index if not exists idx_event_detail_yellowfox_ignition
on eventhub.event_detail(detail_type, (attributes ->> 'ignitionState'), event_occurred_at)
where attributes ? 'ignitionState';

363
src/main/resources/db/migration/V10__introduce_driver_card_model.sql Normal file
View File

@ -0,0 +1,363 @@
create table if not exists eventhub.driver_card (
id uuid primary key,
driver_id uuid references eventhub.driver(id),
nation text not null,
card_number text not null,
source_updated_at timestamptz,
payload jsonb not null default '{}'::jsonb,
created_at timestamptz not null default now(),
updated_at timestamptz not null default now()
);
create table if not exists eventhub.source_driver_identity (
id uuid primary key,
tenant_key text not null,
event_source_id integer not null references eventhub.event_source(id),
source_driver_entity_id text not null,
driver_id uuid not null references eventhub.driver(id) on delete cascade,
source_updated_at timestamptz,
payload jsonb not null default '{}'::jsonb,
created_at timestamptz not null default now(),
updated_at timestamptz not null default now(),
constraint ux_source_driver_identity unique (tenant_key, event_source_id, source_driver_entity_id)
);
create table if not exists eventhub.source_driver_card_identity (
id uuid primary key,
tenant_key text not null,
event_source_id integer not null references eventhub.event_source(id),
source_driver_card_entity_id text not null,
driver_card_id uuid not null references eventhub.driver_card(id) on delete cascade,
source_updated_at timestamptz,
payload jsonb not null default '{}'::jsonb,
created_at timestamptz not null default now(),
updated_at timestamptz not null default now(),
constraint ux_source_driver_card_identity unique (tenant_key, event_source_id, source_driver_card_entity_id)
);
create table if not exists eventhub.source_vehicle_identity (
id uuid primary key,
tenant_key text not null,
event_source_id integer not null references eventhub.event_source(id),
source_vehicle_entity_id text not null,
vehicle_id uuid not null references eventhub.vehicle(id) on delete cascade,
source_updated_at timestamptz,
payload jsonb not null default '{}'::jsonb,
created_at timestamptz not null default now(),
updated_at timestamptz not null default now(),
constraint ux_source_vehicle_identity unique (tenant_key, event_source_id, source_vehicle_entity_id)
);
create table if not exists eventhub.source_vehicle_registration_identity (
id uuid primary key,
tenant_key text not null,
event_source_id integer not null references eventhub.event_source(id),
source_registration_entity_id text not null,
vehicle_registration_id uuid not null references eventhub.vehicle_registration(id) on delete cascade,
source_updated_at timestamptz,
payload jsonb not null default '{}'::jsonb,
created_at timestamptz not null default now(),
updated_at timestamptz not null default now(),
constraint ux_source_vehicle_registration_identity unique (tenant_key, event_source_id, source_registration_entity_id)
);
alter table eventhub.event
add column if not exists driver_card_id uuid references eventhub.driver_card(id);
insert into eventhub.source_driver_identity(
id, tenant_key, event_source_id, source_driver_entity_id,
driver_id, source_updated_at, payload, created_at, updated_at
)
select gen_random_uuid(),
driver.tenant_key,
driver.event_source_id,
driver.source_driver_entity_id,
driver.id,
driver.source_updated_at,
driver.payload,
coalesce(driver.created_at, now()),
coalesce(driver.updated_at, now())
from eventhub.driver driver
where exists (
select 1
from information_schema.columns
where table_schema = 'eventhub'
and table_name = 'driver'
and column_name = 'source_driver_entity_id'
)
and driver.source_driver_entity_id is not null
on conflict (tenant_key, event_source_id, source_driver_entity_id)
do update set
driver_id = excluded.driver_id,
source_updated_at = coalesce(excluded.source_updated_at, eventhub.source_driver_identity.source_updated_at),
payload = eventhub.source_driver_identity.payload || excluded.payload,
updated_at = now();
with legacy_driver_cards as (
select distinct on (driver.card_nation, driver.card_number)
driver.id as driver_id,
driver.card_nation as nation,
driver.card_number,
driver.source_updated_at,
driver.payload,
driver.created_at,
driver.updated_at
from eventhub.driver driver
where exists (
select 1
from information_schema.columns
where table_schema = 'eventhub'
and table_name = 'driver'
and column_name = 'card_nation'
)
and driver.card_nation is not null
and driver.card_number is not null
order by driver.card_nation,
driver.card_number,
case when driver.source_driver_entity_id is null then 1 else 0 end,
driver.updated_at desc,
driver.created_at desc,
driver.id
),
existing_driver_cards as (
select distinct on (card.nation, card.card_number)
card.id,
card.nation,
card.card_number
from eventhub.driver_card card
order by card.nation,
card.card_number,
case when card.driver_id is null then 1 else 0 end,
card.updated_at desc,
card.created_at desc,
card.id
),
resolved_cards as (
select legacy.*,
coalesce(existing.id, gen_random_uuid()) as driver_card_id
from legacy_driver_cards legacy
left join existing_driver_cards existing
on existing.nation = legacy.nation
and existing.card_number = legacy.card_number
),
inserted_cards as (
insert into eventhub.driver_card(
id, driver_id, nation, card_number, source_updated_at, payload, created_at, updated_at
)
select distinct on (resolved.driver_card_id)
resolved.driver_card_id,
resolved.driver_id,
resolved.nation,
resolved.card_number,
resolved.source_updated_at,
jsonb_build_object('migrated_from', 'eventhub.driver') || coalesce(resolved.payload, '{}'::jsonb),
resolved.created_at,
resolved.updated_at
from resolved_cards resolved
where not exists (
select 1
from eventhub.driver_card existing
where existing.id = resolved.driver_card_id
)
returning id
),
updated_cards as (
update eventhub.driver_card card
set driver_id = coalesce(card.driver_id, resolved.driver_id),
source_updated_at = coalesce(resolved.source_updated_at, card.source_updated_at),
payload = card.payload || jsonb_build_object('migrated_from', 'eventhub.driver') || coalesce(resolved.payload, '{}'::jsonb),
updated_at = now()
from resolved_cards resolved
where card.id = resolved.driver_card_id
returning card.id
)
select count(*) from inserted_cards
union all
select count(*) from updated_cards;
with single_card_per_driver as (
select driver_id,
min(id::text)::uuid as driver_card_id
from eventhub.driver_card
where driver_id is not null
group by driver_id
having count(*) = 1
)
update eventhub.event event
set driver_card_id = card.driver_card_id
from single_card_per_driver card
where event.driver_id = card.driver_id
and event.driver_card_id is null;
with ranked_driver_cards as (
select card.id,
card.driver_id,
card.source_updated_at,
first_value(card.id) over (
partition by card.nation, card.card_number
order by case when card.driver_id is null then 1 else 0 end,
card.updated_at desc,
card.created_at desc,
card.id
) as canonical_id,
row_number() over (
partition by card.nation, card.card_number
order by case when card.driver_id is null then 1 else 0 end,
card.updated_at desc,
card.created_at desc,
card.id
) as duplicate_rank
from eventhub.driver_card card
),
duplicate_driver_cards as (
select id, canonical_id, driver_id, source_updated_at
from ranked_driver_cards
where duplicate_rank > 1
),
duplicate_card_rollup as (
select duplicate.canonical_id,
(array_agg(duplicate.driver_id order by case when duplicate.driver_id is null then 1 else 0 end, duplicate.id))[1] as preferred_driver_id,
max(duplicate.source_updated_at) as latest_source_updated_at
from duplicate_driver_cards duplicate
group by duplicate.canonical_id
),
updated_canonical_cards as (
update eventhub.driver_card card
set driver_id = coalesce(card.driver_id, rollup.preferred_driver_id),
source_updated_at = case
when card.source_updated_at is null then rollup.latest_source_updated_at
when rollup.latest_source_updated_at is null then card.source_updated_at
else greatest(card.source_updated_at, rollup.latest_source_updated_at)
end,
updated_at = now()
from duplicate_card_rollup rollup
where card.id = rollup.canonical_id
returning card.id
),
relinked_source_driver_card_identity as (
update eventhub.source_driver_card_identity identity
set driver_card_id = duplicate.canonical_id,
updated_at = now()
from duplicate_driver_cards duplicate
where identity.driver_card_id = duplicate.id
returning identity.id
),
relinked_events as (
update eventhub.event event
set driver_card_id = duplicate.canonical_id
from duplicate_driver_cards duplicate
where event.driver_card_id = duplicate.id
returning event.id
),
deleted_duplicate_driver_cards as (
delete from eventhub.driver_card card
using duplicate_driver_cards duplicate
where card.id = duplicate.id
returning card.id
)
select (select count(*) from updated_canonical_cards) as updated_canonical_cards,
(select count(*) from relinked_source_driver_card_identity) as relinked_source_driver_card_identity,
(select count(*) from relinked_events) as relinked_events,
(select count(*) from deleted_duplicate_driver_cards) as deleted_duplicate_driver_cards;
insert into eventhub.source_vehicle_identity(
id, tenant_key, event_source_id, source_vehicle_entity_id,
vehicle_id, source_updated_at, payload, created_at, updated_at
)
select gen_random_uuid(),
vehicle.tenant_key,
vehicle.event_source_id,
vehicle.source_vehicle_entity_id,
vehicle.id,
null,
'{}'::jsonb,
coalesce(vehicle.created_at, now()),
coalesce(vehicle.updated_at, now())
from eventhub.vehicle vehicle
where exists (
select 1
from information_schema.columns
where table_schema = 'eventhub'
and table_name = 'vehicle'
and column_name = 'source_vehicle_entity_id'
)
and vehicle.source_vehicle_entity_id is not null
on conflict (tenant_key, event_source_id, source_vehicle_entity_id)
do update set
vehicle_id = excluded.vehicle_id,
updated_at = now();
insert into eventhub.source_vehicle_registration_identity(
id, tenant_key, event_source_id, source_registration_entity_id,
vehicle_registration_id, source_updated_at, payload, created_at, updated_at
)
select gen_random_uuid(),
registration.tenant_key,
registration.event_source_id,
registration.source_registration_entity_id,
registration.id,
registration.source_updated_at,
registration.payload,
coalesce(registration.created_at, now()),
coalesce(registration.updated_at, now())
from eventhub.vehicle_registration registration
where exists (
select 1
from information_schema.columns
where table_schema = 'eventhub'
and table_name = 'vehicle_registration'
and column_name = 'source_registration_entity_id'
)
and registration.source_registration_entity_id is not null
on conflict (tenant_key, event_source_id, source_registration_entity_id)
do update set
vehicle_registration_id = excluded.vehicle_registration_id,
source_updated_at = coalesce(excluded.source_updated_at, eventhub.source_vehicle_registration_identity.source_updated_at),
payload = eventhub.source_vehicle_registration_identity.payload || excluded.payload,
updated_at = now();
create index if not exists idx_driver_card_key
on eventhub.driver_card(nation, card_number);
create index if not exists idx_driver_card_driver
on eventhub.driver_card(driver_id)
where driver_id is not null;
create unique index if not exists ux_driver_card_key
on eventhub.driver_card(nation, card_number);
create unique index if not exists ux_vehicle_vin
on eventhub.vehicle(vin)
where vin is not null;
create unique index if not exists ux_vehicle_registration_plate
on eventhub.vehicle_registration(nation, registration_number);
create index if not exists idx_source_driver_identity_driver
on eventhub.source_driver_identity(driver_id);
create index if not exists idx_source_driver_card_identity_card
on eventhub.source_driver_card_identity(driver_card_id);
create index if not exists idx_source_vehicle_identity_vehicle
on eventhub.source_vehicle_identity(vehicle_id);
create index if not exists idx_source_vehicle_registration_identity_registration
on eventhub.source_vehicle_registration_identity(vehicle_registration_id);
create index if not exists idx_event_driver_card_time
on eventhub.event(driver_card_id, occurred_at desc)
where driver_card_id is not null;
alter table eventhub.event
drop constraint if exists chk_event_driver_or_vehicle_ref;
alter table eventhub.event
add constraint chk_event_driver_or_vehicle_ref
check (
driver_id is not null
or driver_card_id is not null
or driver_entity_id is not null
or vehicle_id is not null
or vehicle_registration_id is not null
);

355
src/main/resources/esper/operating-period-state-machine.epl Normal file
View File

@ -0,0 +1,355 @@
create variable long operatingSplitIdleMs = ${operatingSplitIdleMs};
/*
* Full-EPL operating-period state machine.
*
* Input contract:
* - Java sends already-resolved intervals, not raw START/END boundaries.
* - DRIVER_CARD remains authoritative and VU is only used to fill uncovered gaps before events reach this EPL.
* - Synthetic uncovered gaps arrive as activityType = 'UNKNOWN'.
*
* Output contract:
* - PeriodizedActivityInterval: every input interval assigned to an operating period
* - OperatingPeriodClosed: closed operating periods, including the final FLUSH period
*/
create schema KnownOperatingInput(
driverId java.util.UUID,
vehicleId java.util.UUID,
vehicleRegistrationId java.util.UUID,
activityType string,
cardSlot string,
cardStatus string,
drivingStatus string,
sourceKind string,
startTs long,
endTs long,
durationMs long,
sourceRowId string,
sourceRowIds java.util.List,
clippedToRequestedPeriod boolean,
synthetic boolean
);
create schema UnknownOperatingInput(
driverId java.util.UUID,
vehicleId java.util.UUID,
vehicleRegistrationId java.util.UUID,
activityType string,
cardSlot string,
cardStatus string,
drivingStatus string,
sourceKind string,
startTs long,
endTs long,
durationMs long,
sourceRowId string,
sourceRowIds java.util.List,
clippedToRequestedPeriod boolean,
synthetic boolean
);
create schema PeriodizedActivityInterval(
driverId java.util.UUID,
vehicleId java.util.UUID,
vehicleRegistrationId java.util.UUID,
activityType string,
cardSlot string,
cardStatus string,
drivingStatus string,
sourceKind string,
startedAtTs long,
endedAtTs long,
durationMs long,
sourceRowId string,
sourceRowIds java.util.List,
clippedToRequestedPeriod boolean,
operatingPeriodNo long,
operatingPeriodStartedAtTs long,
newOperatingPeriod boolean,
gapSincePreviousActivityMs java.lang.Long,
synthetic boolean
);
create schema OperatingPeriodClosed(
driverId java.util.UUID,
operatingPeriodNo long,
operatingPeriodStartedAtTs long,
operatingPeriodEndedAtTs long,
durationMs long,
closedBy string
);
create window OperatingPeriodState#unique(driverId) as (
driverId java.util.UUID,
hasOpen boolean,
operatingPeriodNo long,
operatingPeriodStartedAtTs long,
lastKnownActivityEndTs long
);
/* Split the timeline into known activities and synthetic UNKNOWN gaps. */
insert into KnownOperatingInput
select * from OperatingPeriodInputMap as i where i.activityType != 'UNKNOWN';
insert into UnknownOperatingInput
select * from OperatingPeriodInputMap as i where i.activityType = 'UNKNOWN';
/* First known activity for a driver opens operating period 1. */
@Priority(200)
on KnownOperatingInput as i
insert into PeriodizedActivityInterval
select
i.driverId as driverId,
i.vehicleId as vehicleId,
i.vehicleRegistrationId as vehicleRegistrationId,
i.activityType as activityType,
i.cardSlot as cardSlot,
i.cardStatus as cardStatus,
i.drivingStatus as drivingStatus,
i.sourceKind as sourceKind,
i.startTs as startedAtTs,
i.endTs as endedAtTs,
i.durationMs as durationMs,
i.sourceRowId as sourceRowId,
i.sourceRowIds as sourceRowIds,
i.clippedToRequestedPeriod as clippedToRequestedPeriod,
1L as operatingPeriodNo,
i.startTs as operatingPeriodStartedAtTs,
true as newOperatingPeriod,
cast(null, java.lang.Long) as gapSincePreviousActivityMs,
i.synthetic as synthetic
where not exists (select * from OperatingPeriodState as s where s.driverId = i.driverId);
@Priority(190)
on KnownOperatingInput as i
insert into OperatingPeriodState
select
i.driverId as driverId,
true as hasOpen,
1L as operatingPeriodNo,
i.startTs as operatingPeriodStartedAtTs,
i.endTs as lastKnownActivityEndTs
where not exists (select * from OperatingPeriodState as s where s.driverId = i.driverId);
/* A long forward gap between known activities closes the current period with reason IDLE_GAP. */
@Priority(180)
on KnownOperatingInput as i
insert into OperatingPeriodClosed
select
s.driverId as driverId,
s.operatingPeriodNo as operatingPeriodNo,
s.operatingPeriodStartedAtTs as operatingPeriodStartedAtTs,
s.lastKnownActivityEndTs as operatingPeriodEndedAtTs,
s.lastKnownActivityEndTs - s.operatingPeriodStartedAtTs as durationMs,
'IDLE_GAP' as closedBy
from OperatingPeriodState as s
where s.driverId = i.driverId
and s.hasOpen = true
and i.startTs - s.lastKnownActivityEndTs >= operatingSplitIdleMs;
/* After a long idle gap, the next known interval is emitted as the first interval of the next period. */
@Priority(170)
on KnownOperatingInput as i
insert into PeriodizedActivityInterval
select
i.driverId as driverId,
i.vehicleId as vehicleId,
i.vehicleRegistrationId as vehicleRegistrationId,
i.activityType as activityType,
i.cardSlot as cardSlot,
i.cardStatus as cardStatus,
i.drivingStatus as drivingStatus,
i.sourceKind as sourceKind,
i.startTs as startedAtTs,
i.endTs as endedAtTs,
i.durationMs as durationMs,
i.sourceRowId as sourceRowId,
i.sourceRowIds as sourceRowIds,
i.clippedToRequestedPeriod as clippedToRequestedPeriod,
s.operatingPeriodNo + 1 as operatingPeriodNo,
i.startTs as operatingPeriodStartedAtTs,
true as newOperatingPeriod,
cast(i.startTs - s.lastKnownActivityEndTs, java.lang.Long) as gapSincePreviousActivityMs,
i.synthetic as synthetic
from OperatingPeriodState as s
where s.driverId = i.driverId
and s.hasOpen = true
and i.startTs - s.lastKnownActivityEndTs >= operatingSplitIdleMs;
/* Update the window state to the newly opened period after an IDLE_GAP close. */
@Priority(160)
on KnownOperatingInput as i
update OperatingPeriodState as s
set
hasOpen = true,
operatingPeriodNo = s.operatingPeriodNo + 1,
operatingPeriodStartedAtTs = i.startTs,
lastKnownActivityEndTs = i.endTs
where s.driverId = i.driverId
and s.hasOpen = true
and i.startTs - s.lastKnownActivityEndTs >= operatingSplitIdleMs;
/* After a long UNKNOWN gap we keep the counter but mark the period closed. The next known activity reopens
* the next period number using the retained state row. */
@Priority(155)
on KnownOperatingInput as i
insert into PeriodizedActivityInterval
select
i.driverId as driverId,
i.vehicleId as vehicleId,
i.vehicleRegistrationId as vehicleRegistrationId,
i.activityType as activityType,
i.cardSlot as cardSlot,
i.cardStatus as cardStatus,
i.drivingStatus as drivingStatus,
i.sourceKind as sourceKind,
i.startTs as startedAtTs,
i.endTs as endedAtTs,
i.durationMs as durationMs,
i.sourceRowId as sourceRowId,
i.sourceRowIds as sourceRowIds,
i.clippedToRequestedPeriod as clippedToRequestedPeriod,
s.operatingPeriodNo + 1 as operatingPeriodNo,
i.startTs as operatingPeriodStartedAtTs,
true as newOperatingPeriod,
cast(null, java.lang.Long) as gapSincePreviousActivityMs,
i.synthetic as synthetic
from OperatingPeriodState as s
where s.driverId = i.driverId
and s.hasOpen = false;
@Priority(145)
on KnownOperatingInput as i
update OperatingPeriodState as s
set
hasOpen = true,
operatingPeriodNo = s.operatingPeriodNo + 1,
operatingPeriodStartedAtTs = i.startTs,
lastKnownActivityEndTs = i.endTs
where s.driverId = i.driverId
and s.hasOpen = false;
/* Normal same-period continuity: the gap is forward, non-negative, and still below the split threshold. */
@Priority(150)
on KnownOperatingInput as i
insert into PeriodizedActivityInterval
select
i.driverId as driverId,
i.vehicleId as vehicleId,
i.vehicleRegistrationId as vehicleRegistrationId,
i.activityType as activityType,
i.cardSlot as cardSlot,
i.cardStatus as cardStatus,
i.drivingStatus as drivingStatus,
i.sourceKind as sourceKind,
i.startTs as startedAtTs,
i.endTs as endedAtTs,
i.durationMs as durationMs,
i.sourceRowId as sourceRowId,
i.sourceRowIds as sourceRowIds,
i.clippedToRequestedPeriod as clippedToRequestedPeriod,
s.operatingPeriodNo as operatingPeriodNo,
s.operatingPeriodStartedAtTs as operatingPeriodStartedAtTs,
false as newOperatingPeriod,
cast(i.startTs - s.lastKnownActivityEndTs, java.lang.Long) as gapSincePreviousActivityMs,
i.synthetic as synthetic
from OperatingPeriodState as s
where s.driverId = i.driverId
and s.hasOpen = true
and i.startTs - s.lastKnownActivityEndTs >= 0
and i.startTs - s.lastKnownActivityEndTs < operatingSplitIdleMs;
@Priority(140)
on KnownOperatingInput as i
update OperatingPeriodState as s
set
lastKnownActivityEndTs = case
when i.endTs > s.lastKnownActivityEndTs then i.endTs
else s.lastKnownActivityEndTs
end
where s.driverId = i.driverId
and s.hasOpen = true
and i.startTs - s.lastKnownActivityEndTs >= 0
and i.startTs - s.lastKnownActivityEndTs < operatingSplitIdleMs;
/* A long UNKNOWN interval closes the current period, but the state row remains so the next known activity
* can reopen with the next period number. */
@Priority(130)
on UnknownOperatingInput as i
insert into OperatingPeriodClosed
select
s.driverId as driverId,
s.operatingPeriodNo as operatingPeriodNo,
s.operatingPeriodStartedAtTs as operatingPeriodStartedAtTs,
s.lastKnownActivityEndTs as operatingPeriodEndedAtTs,
s.lastKnownActivityEndTs - s.operatingPeriodStartedAtTs as durationMs,
'UNKNOWN_GAP' as closedBy
from OperatingPeriodState as s
where s.driverId = i.driverId
and s.hasOpen = true
and i.durationMs >= operatingSplitIdleMs;
@Priority(120)
on UnknownOperatingInput as i
update OperatingPeriodState as s
set hasOpen = false
where s.driverId = i.driverId
and s.hasOpen = true
and i.durationMs >= operatingSplitIdleMs;
/* Short UNKNOWN stays inside the open period as explicit uncertainty, without changing period state. */
@Priority(110)
on UnknownOperatingInput as i
insert into PeriodizedActivityInterval
select
i.driverId as driverId,
i.vehicleId as vehicleId,
i.vehicleRegistrationId as vehicleRegistrationId,
i.activityType as activityType,
i.cardSlot as cardSlot,
i.cardStatus as cardStatus,
i.drivingStatus as drivingStatus,
i.sourceKind as sourceKind,
i.startTs as startedAtTs,
i.endTs as endedAtTs,
i.durationMs as durationMs,
i.sourceRowId as sourceRowId,
i.sourceRowIds as sourceRowIds,
i.clippedToRequestedPeriod as clippedToRequestedPeriod,
s.operatingPeriodNo as operatingPeriodNo,
s.operatingPeriodStartedAtTs as operatingPeriodStartedAtTs,
false as newOperatingPeriod,
cast(i.startTs - s.lastKnownActivityEndTs, java.lang.Long) as gapSincePreviousActivityMs,
i.synthetic as synthetic
from OperatingPeriodState as s
where s.driverId = i.driverId
and s.hasOpen = true
and i.durationMs < operatingSplitIdleMs;
/* Historical evaluation ends with a flush event so the final still-open period is emitted. */
@Priority(100)
on OperatingPeriodFlushEvent as f
insert into OperatingPeriodClosed
select
s.driverId as driverId,
s.operatingPeriodNo as operatingPeriodNo,
s.operatingPeriodStartedAtTs as operatingPeriodStartedAtTs,
s.lastKnownActivityEndTs as operatingPeriodEndedAtTs,
s.lastKnownActivityEndTs - s.operatingPeriodStartedAtTs as durationMs,
'FLUSH' as closedBy
from OperatingPeriodState as s
where s.hasOpen = true;
@Priority(90)
on OperatingPeriodFlushEvent as f
update OperatingPeriodState as s
set hasOpen = false
where s.hasOpen = true;
/* Listener-facing output statements consumed by Java. */
@name('periodizedActivityIntervals')
select * from PeriodizedActivityInterval;
@name('operatingPeriodClosed')
select * from OperatingPeriodClosed;

413
src/main/resources/sql/maintenance/repair-tachograph-driver-entities.sql
View File

@ -1,19 +1,22 @@
/*
* Repairs and normalizes tachograph driver aggregates after introducing eventhub.driver.
* Repairs and normalizes tachograph driver identities after introducing:
* - global eventhub.driver
* - global eventhub.driver_card
* - source_driver_identity
* - source_driver_card_identity
*
* What it does:
* 1. Ensures tachograph DRIVER master-data payload carries last_name while keeping source_master_entity.display_name unchanged.
* 2. Upserts eventhub.driver rows from MASTER_DATA DRIVER entities.
* 3. Projects card nation/number onto eventhub.driver from DRIVER_CARD_DRIVER relations.
* 4. Remaps event.driver_id from provisional card-only drivers to proper source-driver aggregates when possible.
* 5. Deletes now-unreferenced provisional tachograph driver rows with no source_driver_entity_id.
*
* Assumptions:
* - Tachograph master-data source is provider_key=TACHOGRAPH, source_kind=MASTER_DATA, source_key=TACHOGRAPH_MASTER_DATA.
* - eventhub.driver and event.driver_id already exist.
* 1. Ensures tachograph DRIVER master-data payload carries last_name while
* keeping source_master_entity.display_name unchanged.
* 2. Upserts global eventhub.driver rows from tachograph DRIVER entities.
* 3. Upserts global eventhub.driver_card rows from tachograph DRIVER_CARD entities.
* 4. Upserts source identity links for drivers and cards.
* 5. Links cards to drivers using DRIVER_CARD_DRIVER master-data relations.
* 6. Backfills event.driver_card_id where a driver has exactly one card.
* 7. Remaps event.driver_id from provisional rows to the linked canonical driver.
* 8. Deletes now-unreferenced provisional driver rows.
*/
-- 1) Keep display_name, but ensure DRIVER payload has last_name.
with master_sources as (
select es.id, es.tenant_key
from eventhub.event_source es
@ -40,22 +43,16 @@ updated_master_payload as (
select count(*) as updated_master_payload
from updated_master_payload;
-- 2) Upsert driver aggregates from tachograph master data.
with master_sources as (
select es.id,
es.tenant_key,
es.source_instance_key,
coalesce(es.tenant_provider_setting_key, '') as tenant_provider_setting_key
select es.id as master_event_source_id, es.tenant_key
from eventhub.event_source es
where es.provider_key = 'TACHOGRAPH'
and es.source_kind = 'MASTER_DATA'
and es.source_key = 'TACHOGRAPH_MASTER_DATA'
),
master_drivers as (
select ms.id as master_event_source_id,
select ms.master_event_source_id,
ms.tenant_key,
ms.source_instance_key,
ms.tenant_provider_setting_key,
d.source_entity_id as source_driver_entity_id,
coalesce(nullif(trim(d.payload ->> 'first_names'), ''), nullif(trim(d.payload ->> 'firstnames'), '')) as first_names,
coalesce(nullif(trim(d.payload ->> 'last_name'), ''), nullif(trim(d.payload ->> 'surname'), '')) as last_name,
@ -65,174 +62,298 @@ master_drivers as (
from master_sources ms
join eventhub.source_master_entity d
on d.tenant_key = ms.tenant_key
and d.event_source_id = ms.id
and d.event_source_id = ms.master_event_source_id
and d.entity_type = 'DRIVER'
and d.source_entity_id not like 'DRIVER_CARD:%'
),
compatible_targets as (
resolved_drivers as (
select md.*,
es.id as target_event_source_id
coalesce(identity.driver_id, gen_random_uuid()) as driver_id
from master_drivers md
join eventhub.event_source es
on es.tenant_key = md.tenant_key
and es.provider_key = 'TACHOGRAPH'
and es.source_instance_key = md.source_instance_key
and coalesce(es.tenant_provider_setting_key, '') = md.tenant_provider_setting_key
),
updated_drivers as (
update eventhub.driver driver
set first_names = coalesce(ct.first_names, driver.first_names),
last_name = coalesce(ct.last_name, driver.last_name),
birth_date = coalesce(ct.birth_date, driver.birth_date),
source_updated_at = ct.source_updated_at,
payload = driver.payload || ct.payload,
updated_at = now()
from compatible_targets ct
where driver.tenant_key = ct.tenant_key
and driver.event_source_id = ct.target_event_source_id
and driver.source_driver_entity_id = ct.source_driver_entity_id
returning driver.id
left join eventhub.source_driver_identity identity
on identity.tenant_key = md.tenant_key
and identity.event_source_id = md.master_event_source_id
and identity.source_driver_entity_id = md.source_driver_entity_id
),
inserted_drivers as (
insert into eventhub.driver(
id, tenant_key, event_source_id, source_driver_entity_id,
first_names, last_name, birth_date, source_updated_at, payload, updated_at
id, first_names, last_name, birth_date, source_updated_at, payload, updated_at
)
select gen_random_uuid(),
ct.tenant_key,
ct.target_event_source_id,
ct.source_driver_entity_id,
ct.first_names,
ct.last_name,
ct.birth_date,
ct.source_updated_at,
ct.payload,
select distinct on (rd.driver_id)
rd.driver_id,
rd.first_names,
rd.last_name,
rd.birth_date,
rd.source_updated_at,
rd.payload,
now()
from compatible_targets ct
from resolved_drivers rd
where not exists (
select 1
from eventhub.driver existing
where existing.tenant_key = ct.tenant_key
and existing.event_source_id = ct.target_event_source_id
and existing.source_driver_entity_id = ct.source_driver_entity_id
where existing.id = rd.driver_id
)
returning id
),
updated_drivers as (
update eventhub.driver driver
set first_names = coalesce(rd.first_names, driver.first_names),
last_name = coalesce(rd.last_name, driver.last_name),
birth_date = coalesce(rd.birth_date, driver.birth_date),
source_updated_at = coalesce(rd.source_updated_at, driver.source_updated_at),
payload = driver.payload || rd.payload,
updated_at = now()
from resolved_drivers rd
where driver.id = rd.driver_id
returning driver.id
),
upserted_source_driver_identity as (
insert into eventhub.source_driver_identity(
id, tenant_key, event_source_id, source_driver_entity_id,
driver_id, source_updated_at, payload, updated_at
)
select gen_random_uuid(),
rd.tenant_key,
rd.master_event_source_id,
rd.source_driver_entity_id,
rd.driver_id,
rd.source_updated_at,
rd.payload,
now()
from resolved_drivers rd
on conflict (tenant_key, event_source_id, source_driver_entity_id)
do update set
driver_id = excluded.driver_id,
source_updated_at = coalesce(excluded.source_updated_at, eventhub.source_driver_identity.source_updated_at),
payload = eventhub.source_driver_identity.payload || excluded.payload,
updated_at = now()
returning id
)
select (select count(*) from updated_drivers) as updated_drivers,
(select count(*) from inserted_drivers) as inserted_drivers;
select (select count(*) from inserted_drivers) as inserted_drivers,
(select count(*) from updated_drivers) as updated_drivers,
(select count(*) from upserted_source_driver_identity) as upserted_source_driver_identity;
-- 3) Project driver-card identifiers from master-data relations.
with master_sources as (
select es.id,
es.tenant_key,
es.source_instance_key,
coalesce(es.tenant_provider_setting_key, '') as tenant_provider_setting_key
select es.id as master_event_source_id, es.tenant_key
from eventhub.event_source es
where es.provider_key = 'TACHOGRAPH'
and es.source_kind = 'MASTER_DATA'
and es.source_key = 'TACHOGRAPH_MASTER_DATA'
),
card_projection as (
select distinct on (ms.tenant_key, ms.source_instance_key, ms.tenant_provider_setting_key, rel.to_source_entity_id)
master_cards as (
select ms.master_event_source_id,
ms.tenant_key,
ms.source_instance_key,
ms.tenant_provider_setting_key,
rel.to_source_entity_id as source_driver_entity_id,
card.source_entity_id as source_driver_card_entity_id,
nullif(trim(card.payload ->> 'card_nation'), '') as card_nation,
nullif(trim(card.payload ->> 'card_number'), '') as card_number,
rel.source_updated_at
card.source_updated_at,
card.payload
from master_sources ms
join eventhub.source_master_relation rel
on rel.tenant_key = ms.tenant_key
and rel.event_source_id = ms.id
and rel.relation_type = 'DRIVER_CARD_DRIVER'
and rel.from_entity_type = 'DRIVER_CARD'
and rel.to_entity_type = 'DRIVER'
join eventhub.source_master_entity card
on card.tenant_key = ms.tenant_key
and card.event_source_id = ms.id
and card.event_source_id = ms.master_event_source_id
and card.entity_type = 'DRIVER_CARD'
and card.source_entity_id = rel.from_source_entity_id
order by ms.tenant_key,
ms.source_instance_key,
ms.tenant_provider_setting_key,
rel.to_source_entity_id,
rel.valid_to desc nulls last,
rel.valid_from desc nulls last,
rel.updated_at desc
and nullif(trim(card.payload ->> 'card_nation'), '') is not null
and nullif(trim(card.payload ->> 'card_number'), '') is not null
),
updated_driver_cards as (
update eventhub.driver driver
set card_nation = coalesce(driver.card_nation, projection.card_nation),
card_number = coalesce(driver.card_number, projection.card_number),
source_updated_at = coalesce(projection.source_updated_at, driver.source_updated_at),
canonical_driver_cards as (
select distinct on (mc.card_nation, mc.card_number)
mc.card_nation,
mc.card_number,
mc.source_updated_at,
mc.payload
from master_cards mc
order by mc.card_nation,
mc.card_number,
case when mc.source_driver_card_entity_id is null then 1 else 0 end,
mc.source_updated_at desc,
mc.source_driver_card_entity_id
),
existing_driver_cards as (
select distinct on (card.nation, card.card_number)
card.id,
card.nation,
card.card_number
from eventhub.driver_card card
order by card.nation,
card.card_number,
case when card.driver_id is null then 1 else 0 end,
card.updated_at desc,
card.created_at desc,
card.id
),
resolved_cards as (
select canonical.card_nation,
canonical.card_number,
canonical.source_updated_at,
canonical.payload,
coalesce(existing.id, gen_random_uuid()) as driver_card_id
from canonical_driver_cards canonical
left join existing_driver_cards existing
on existing.nation = canonical.card_nation
and existing.card_number = canonical.card_number
),
inserted_cards as (
insert into eventhub.driver_card(
id, driver_id, nation, card_number, source_updated_at, payload, updated_at
)
select distinct on (rc.driver_card_id)
rc.driver_card_id,
null,
rc.card_nation,
rc.card_number,
rc.source_updated_at,
rc.payload,
now()
from resolved_cards rc
where not exists (
select 1
from eventhub.driver_card existing
where existing.id = rc.driver_card_id
)
returning id
),
updated_cards as (
update eventhub.driver_card card
set source_updated_at = coalesce(rc.source_updated_at, card.source_updated_at),
payload = card.payload || rc.payload,
updated_at = now()
from card_projection projection
join eventhub.event_source es
on es.id = driver.event_source_id
where driver.tenant_key = projection.tenant_key
and es.provider_key = 'TACHOGRAPH'
and es.source_instance_key = projection.source_instance_key
and coalesce(es.tenant_provider_setting_key, '') = projection.tenant_provider_setting_key
and driver.source_driver_entity_id = projection.source_driver_entity_id
and (
(driver.card_nation is null and projection.card_nation is not null)
or (driver.card_number is null and projection.card_number is not null)
)
returning driver.id
from resolved_cards rc
where card.id = rc.driver_card_id
returning card.id
),
upserted_source_driver_card_identity as (
insert into eventhub.source_driver_card_identity(
id, tenant_key, event_source_id, source_driver_card_entity_id,
driver_card_id, source_updated_at, payload, updated_at
)
select gen_random_uuid(),
mc.tenant_key,
mc.master_event_source_id,
mc.source_driver_card_entity_id,
coalesce(identity.driver_card_id, existing.id),
mc.source_updated_at,
mc.payload,
now()
from master_cards mc
left join eventhub.source_driver_card_identity identity
on identity.tenant_key = mc.tenant_key
and identity.event_source_id = mc.master_event_source_id
and identity.source_driver_card_entity_id = mc.source_driver_card_entity_id
left join existing_driver_cards existing
on existing.nation = mc.card_nation
and existing.card_number = mc.card_number
where mc.source_driver_card_entity_id is not null
and coalesce(identity.driver_card_id, existing.id) is not null
on conflict (tenant_key, event_source_id, source_driver_card_entity_id)
do update set
driver_card_id = excluded.driver_card_id,
source_updated_at = coalesce(excluded.source_updated_at, eventhub.source_driver_card_identity.source_updated_at),
payload = eventhub.source_driver_card_identity.payload || excluded.payload,
updated_at = now()
returning id
)
select count(*) as updated_driver_cards
from updated_driver_cards;
select (select count(*) from inserted_cards) as inserted_cards,
(select count(*) from updated_cards) as updated_cards,
(select count(*) from upserted_source_driver_card_identity) as upserted_source_driver_card_identity;
-- 4) Remap events from provisional card-only drivers to proper source-driver aggregates.
with provisional_to_real as (
select provisional.id as provisional_driver_id,
real.id as real_driver_id
from eventhub.driver provisional
join eventhub.event_source provisional_source
on provisional_source.id = provisional.event_source_id
and provisional_source.provider_key = 'TACHOGRAPH'
join eventhub.driver real
on real.tenant_key = provisional.tenant_key
and real.source_driver_entity_id is not null
and real.card_nation = provisional.card_nation
and real.card_number = provisional.card_number
join eventhub.event_source real_source
on real_source.id = real.event_source_id
and real_source.provider_key = provisional_source.provider_key
and real_source.tenant_key = provisional_source.tenant_key
and real_source.source_instance_key = provisional_source.source_instance_key
and coalesce(real_source.tenant_provider_setting_key, '') = coalesce(provisional_source.tenant_provider_setting_key, '')
where provisional.source_driver_entity_id is null
and provisional.card_nation is not null
and provisional.card_number is not null
and provisional.id <> real.id
with master_sources as (
select es.id as master_event_source_id, es.tenant_key
from eventhub.event_source es
where es.provider_key = 'TACHOGRAPH'
and es.source_kind = 'MASTER_DATA'
and es.source_key = 'TACHOGRAPH_MASTER_DATA'
),
updated_card_links as (
update eventhub.driver_card card
set driver_id = driver_identity.driver_id,
source_updated_at = coalesce(rel.source_updated_at, card.source_updated_at),
updated_at = now()
from master_sources ms
join eventhub.source_master_relation rel
on rel.tenant_key = ms.tenant_key
and rel.event_source_id = ms.master_event_source_id
and rel.relation_type = 'DRIVER_CARD_DRIVER'
and rel.from_entity_type = 'DRIVER_CARD'
and rel.to_entity_type = 'DRIVER'
join eventhub.source_driver_card_identity card_identity
on card_identity.tenant_key = rel.tenant_key
and card_identity.event_source_id = rel.event_source_id
and card_identity.source_driver_card_entity_id = rel.from_source_entity_id
join eventhub.source_driver_identity driver_identity
on driver_identity.tenant_key = rel.tenant_key
and driver_identity.event_source_id = rel.event_source_id
and driver_identity.source_driver_entity_id = rel.to_source_entity_id
where card.id = card_identity.driver_card_id
and (
card.driver_id is null
or card.driver_id = driver_identity.driver_id
or not exists (
select 1
from eventhub.source_driver_identity existing_identity
where existing_identity.driver_id = card.driver_id
)
)
returning card.id
)
select count(*) as updated_card_links
from updated_card_links;
with single_card_per_driver as (
select driver_id,
min(id::text)::uuid as driver_card_id
from eventhub.driver_card
where driver_id is not null
group by driver_id
having count(*) = 1
),
updated_events as (
update eventhub.event e
set driver_id = map.real_driver_id
from provisional_to_real map
where e.driver_id = map.provisional_driver_id
and e.driver_id <> map.real_driver_id
returning e.id
update eventhub.event event
set driver_card_id = card.driver_card_id
from single_card_per_driver card
where event.driver_id = card.driver_id
and event.driver_card_id is null
returning event.id
)
select count(*) as remapped_events
select count(*) as backfilled_event_driver_cards
from updated_events;
-- 5) Delete now-unreferenced provisional tachograph driver rows.
with remapped_events as (
update eventhub.event event
set driver_id = card.driver_id
from eventhub.driver_card card
where event.driver_card_id = card.id
and card.driver_id is not null
and (
event.driver_id is null
or not exists (
select 1
from eventhub.source_driver_identity source_identity
where source_identity.driver_id = event.driver_id
)
)
and event.driver_id is distinct from card.driver_id
returning event.id
)
select count(*) as remapped_events
from remapped_events;
with deleted_drivers as (
delete from eventhub.driver driver
using eventhub.event_source es
where es.id = driver.event_source_id
and es.provider_key = 'TACHOGRAPH'
and driver.source_driver_entity_id is null
and driver.card_nation is not null
and driver.card_number is not null
where not exists (
select 1
from eventhub.event event
where event.driver_id = driver.id
)
and not exists (
select 1
from eventhub.event e
where e.driver_id = driver.id
)
select 1
from eventhub.driver_card card
where card.driver_id = driver.id
)
and not exists (
select 1
from eventhub.source_driver_identity source_identity
where source_identity.driver_id = driver.id
)
returning driver.id
)
select count(*) as deleted_provisional_drivers

110
src/main/resources/sql/maintenance/restore-driver-card-columns.sql Normal file
View File

@ -0,0 +1,110 @@
/*
* Restores legacy columns on eventhub.driver:
* - card_nation
* - card_number
*
* The script is conservative:
* - it recreates the columns and the old check constraint
* - it backfills values only when a driver has exactly one distinct card
* recoverable from:
* 1. eventhub.driver_card.driver_id
* 2. eventhub.event.driver_id + event.driver_card_id
*
* If a driver is associated with multiple distinct cards, the columns remain null.
*/
alter table eventhub.driver
add column if not exists card_nation text;
alter table eventhub.driver
add column if not exists card_number text;
do $restore$
begin
if not exists (
select 1
from information_schema.table_constraints
where constraint_schema = 'eventhub'
and table_name = 'driver'
and constraint_name = 'chk_driver_card_nation_when_number'
) then
alter table eventhub.driver
add constraint chk_driver_card_nation_when_number
check (card_number is null or card_nation is not null);
end if;
end
$restore$;
with direct_driver_cards as (
select driver.id as driver_id,
card.nation,
card.card_number
from eventhub.driver driver
join eventhub.driver_card card
on card.driver_id = driver.id
),
event_driver_cards as (
select distinct event.driver_id,
card.nation,
card.card_number
from eventhub.event event
join eventhub.driver_card card
on card.id = event.driver_card_id
where event.driver_id is not null
and event.driver_card_id is not null
),
all_driver_cards as (
select driver_id, nation, card_number
from direct_driver_cards
union
select driver_id, nation, card_number
from event_driver_cards
),
single_card_per_driver as (
select driver_id,
max(nation) as card_nation,
max(card_number) as card_number
from all_driver_cards
group by driver_id
having count(*) = 1
),
updated_drivers as (
update eventhub.driver driver
set card_nation = single.card_nation,
card_number = single.card_number,
updated_at = now()
from single_card_per_driver single
where driver.id = single.driver_id
and (
driver.card_nation is distinct from single.card_nation
or driver.card_number is distinct from single.card_number
)
returning driver.id
)
select count(*) as restored_driver_card_columns
from updated_drivers;
with ambiguous_drivers as (
select driver_id,
count(*) as distinct_card_count
from (
select distinct driver_id, nation, card_number
from (
select driver.id as driver_id, card.nation, card.card_number
from eventhub.driver driver
join eventhub.driver_card card
on card.driver_id = driver.id
union all
select distinct event.driver_id, card.nation, card.card_number
from eventhub.event event
join eventhub.driver_card card
on card.id = event.driver_card_id
where event.driver_id is not null
and event.driver_card_id is not null
) cards
) distinct_cards
group by driver_id
having count(*) > 1
)
select count(*) as ambiguous_driver_count
from ambiguous_drivers;

26
src/main/resources/sql/yellowfox/d8-bookings.sql
View File

@ -1,3 +1,12 @@
with bookings as (
select
b.*,
lag(b.ignition) over (
partition by b.vehicle_id
order by b.utc asc, b.eventid asc
) as previous_ignition
from data.d8_booking b
)
select
b.eventid,
b.utc,
@ -5,6 +14,7 @@ select
b.driver_id,
b.key,
b.ignition,
b.previous_ignition,
b.eventtype,
b.state,
b.odometer,
@ -18,7 +28,7 @@ select
d.firstname as driver_firstname,
d.name as driver_lastname,
d.drivers_card as driver_card_number,
left(trim(d.drivers_card), 14) as driver_card_number,
d.fleet_id as driver_fleet_id,
f.id as fleet_id,
@ -26,7 +36,7 @@ select
tp.id as telematic_provider_id,
tp.name as telematic_provider_name
from data.d8_booking b
from bookings b
left join data.vehicle v
on v.id = b.vehicle_id
left join data.driver d
@ -35,15 +45,5 @@ left join data.fleet f
on f.id = coalesce(v.fleet_id, d.fleet_id)
left join data.telematic_provider tp
on tp.id = v.telematic_provider_id
where (:occurredFrom is null or b.utc >= :occurredFrom)
and (:occurredTo is null or b.utc < :occurredTo)
and (:fleetId is null or f.id = :fleetId)
and (
:lastOccurredTo is null
or b.utc > :lastOccurredTo
or (
b.utc = :lastOccurredTo
and (:lastSourceRowId is null or b.eventid > :lastSourceRowId)
)
)
/*__FILTERS__*/
order by b.utc asc, b.eventid asc;

15
src/main/resources/sql/yellowfox/master-data/driver-cards.sql Normal file
View File

@ -0,0 +1,15 @@
select
'DRIVER_CARD' as entity_type,
cast(d.id as varchar(128)) as source_entity_id,
concat('YELLOWFOX:', left(trim(d.drivers_card), 14)) as source_external_key,
left(trim(d.drivers_card), 14) as display_name,
true as active,
null::timestamptz as valid_from,
null::timestamptz as valid_to,
null::timestamptz as source_updated_at,
d.id as driver_id,
left(trim(d.drivers_card), 14) as card_number,
'UNKNOWN' as card_nation,
d.fleet_id as fleet_id
from data.driver d
where nullif(trim(d.drivers_card), '') is not null;

14
src/main/resources/sql/yellowfox/master-data/drivers.sql Normal file
View File

@ -0,0 +1,14 @@
select
'DRIVER' as entity_type,
cast(d.id as varchar(128)) as source_entity_id,
coalesce(nullif(left(trim(d.drivers_card), 14), ''), cast(d.id as varchar(128))) as source_external_key,
nullif(trim(concat_ws(' ', d.firstname, d.name)), '') as display_name,
true as active,
null::timestamptz as valid_from,
null::timestamptz as valid_to,
null::timestamptz as source_updated_at,
d.id as driver_id,
d.firstname as first_names,
d.name as last_name,
d.fleet_id as fleet_id
from data.driver d;

12
src/main/resources/sql/yellowfox/master-data/fleets.sql Normal file
View File

@ -0,0 +1,12 @@
select
'FLEET' as entity_type,
cast(f.id as varchar(128)) as source_entity_id,
cast(f.id as varchar(128)) as source_external_key,
nullif(trim(f.name), '') as display_name,
true as active,
null::timestamptz as valid_from,
null::timestamptz as valid_to,
null::timestamptz as source_updated_at,
f.id as fleet_id,
f.name as fleet_name
from data.fleet f;

80
src/main/resources/sql/yellowfox/master-data/relations.sql Normal file
View File

@ -0,0 +1,80 @@
select
'DRIVER_FLEET' as relation_type,
'DRIVER' as from_entity_type,
cast(d.id as varchar(128)) as from_source_entity_id,
'FLEET' as to_entity_type,
cast(d.fleet_id as varchar(128)) as to_source_entity_id,
null::timestamptz as valid_from,
null::timestamptz as valid_to,
null::timestamptz as source_updated_at,
'data.driver' as source_table,
cast(d.id as varchar(128)) as source_row_id
from data.driver d
where d.fleet_id is not null
union all
select
'DRIVER_CARD_DRIVER' as relation_type,
'DRIVER_CARD' as from_entity_type,
cast(d.id as varchar(128)) as from_source_entity_id,
'DRIVER' as to_entity_type,
cast(d.id as varchar(128)) as to_source_entity_id,
null::timestamptz as valid_from,
null::timestamptz as valid_to,
null::timestamptz as source_updated_at,
'data.driver' as source_table,
cast(d.id as varchar(128)) as source_row_id
from data.driver d
where nullif(trim(d.drivers_card), '') is not null
union all
select
'VEHICLE_FLEET' as relation_type,
'VEHICLE' as from_entity_type,
cast(v.id as varchar(128)) as from_source_entity_id,
'FLEET' as to_entity_type,
cast(v.fleet_id as varchar(128)) as to_source_entity_id,
null::timestamptz as valid_from,
null::timestamptz as valid_to,
null::timestamptz as source_updated_at,
'data.vehicle' as source_table,
cast(v.id as varchar(128)) as source_row_id
from data.vehicle v
where v.fleet_id is not null
and nullif(trim(v.vin), '') is not null
union all
select
'VEHICLE_REGISTRATION_VEHICLE' as relation_type,
'VEHICLE_REGISTRATION' as from_entity_type,
cast(v.id as varchar(128)) as from_source_entity_id,
'VEHICLE' as to_entity_type,
cast(v.id as varchar(128)) as to_source_entity_id,
null::timestamptz as valid_from,
null::timestamptz as valid_to,
null::timestamptz as source_updated_at,
'data.vehicle' as source_table,
cast(v.id as varchar(128)) as source_row_id
from data.vehicle v
where nullif(trim(v.vrn), '') is not null
and nullif(trim(v.vin), '') is not null
union all
select
'VEHICLE_TELEMATIC_PROVIDER' as relation_type,
'VEHICLE' as from_entity_type,
cast(v.id as varchar(128)) as from_source_entity_id,
'TELEMATIC_PROVIDER' as to_entity_type,
cast(v.telematic_provider_id as varchar(128)) as to_source_entity_id,
null::timestamptz as valid_from,
null::timestamptz as valid_to,
null::timestamptz as source_updated_at,
'data.vehicle' as source_table,
cast(v.id as varchar(128)) as source_row_id
from data.vehicle v
where v.telematic_provider_id is not null
and nullif(trim(v.vin), '') is not null;

12
src/main/resources/sql/yellowfox/master-data/telematic-providers.sql Normal file
View File

@ -0,0 +1,12 @@
select
'TELEMATIC_PROVIDER' as entity_type,
cast(tp.id as varchar(128)) as source_entity_id,
cast(tp.id as varchar(128)) as source_external_key,
nullif(trim(tp.name), '') as display_name,
true as active,
null::timestamptz as valid_from,
null::timestamptz as valid_to,
null::timestamptz as source_updated_at,
tp.id as telematic_provider_id,
tp.name as telematic_provider_name
from data.telematic_provider tp;

16
src/main/resources/sql/yellowfox/master-data/vehicle-registrations.sql Normal file
View File

@ -0,0 +1,16 @@
select
'VEHICLE_REGISTRATION' as entity_type,
cast(v.id as varchar(128)) as source_entity_id,
concat('YELLOWFOX:', trim(v.vrn)) as source_external_key,
trim(v.vrn) as display_name,
true as active,
null::timestamptz as valid_from,
null::timestamptz as valid_to,
null::timestamptz as source_updated_at,
v.id as vehicle_id,
trim(v.vin) as vin,
'UNKNOWN' as registration_nation,
trim(v.vrn) as registration_number,
v.fleet_id as fleet_id
from data.vehicle v
where nullif(trim(v.vrn), '') is not null;

17
src/main/resources/sql/yellowfox/master-data/vehicles.sql Normal file
View File

@ -0,0 +1,17 @@
select
'VEHICLE' as entity_type,
cast(v.id as varchar(128)) as source_entity_id,
trim(v.vin) as source_external_key,
coalesce(nullif(trim(v.vrn), ''), trim(v.vin)) as display_name,
true as active,
null::timestamptz as valid_from,
null::timestamptz as valid_to,
null::timestamptz as source_updated_at,
v.id as vehicle_id,
trim(v.vin) as vin,
trim(v.vrn) as registration_number,
'UNKNOWN' as registration_nation,
v.fleet_id as fleet_id,
v.telematic_provider_id as telematic_provider_id
from data.vehicle v
where nullif(trim(v.vin), '') is not null;

1
src/test/java/at/procon/eventhub/YellowFoxD8BookingEventMapperTest.java
View File

@ -83,6 +83,7 @@ class YellowFoxD8BookingEventMapperTest {
"event-1",
"key-1",
ignition,
null,
eventType,
state,
OffsetDateTime.parse("2026-04-29T08:15:00+02:00"),

35
src/test/java/at/procon/eventhub/esperpoc/service/EsperOperatingPeriodEvaluationServiceTest.java
View File

@ -3,6 +3,7 @@ package at.procon.eventhub.esperpoc.service;
import static org.assertj.core.api.Assertions.assertThat;
import at.procon.eventhub.esperpoc.dto.ActivityIntervalDto;
import at.procon.eventhub.esperpoc.dto.EsperOperatingPeriodEngineMode;
import at.procon.eventhub.esperpoc.dto.EsperUnknownTreatmentMode;
import at.procon.eventhub.esperpoc.dto.NonDrivingIntervalDto;
import at.procon.eventhub.esperpoc.dto.OperatingPeriodActivityIntervalDto;
@ -53,7 +54,8 @@ class EsperOperatingPeriodEvaluationServiceTest {
EsperOperatingPeriodEngine.EsperOperatingPeriodEvaluation evaluation = operatingPeriodEngine.evaluate(
evaluationIntervals,
Duration.ofHours(7)
Duration.ofHours(7),
EsperOperatingPeriodEngineMode.STREAM_COLLECTOR
);
assertThat(evaluation.periodizedIntervals()).extracting(OperatingPeriodActivityIntervalDto::activityType)
@ -99,6 +101,37 @@ class EsperOperatingPeriodEvaluationServiceTest {
assertThat(nonDrivingIntervals.get(1).startedAt()).isEqualTo(OffsetDateTime.parse("2026-04-01T09:30:00Z"));
}
@Test
void fullEplModeMatchesStreamCollectorMode() {
UUID driverId = UUID.randomUUID();
List<ActivityIntervalDto> evaluationIntervals = List.of(
activity(driverId, "WORK", "2026-04-01T08:00:00Z", "2026-04-01T09:00:00Z", "w1", "DRIVER_CARD"),
activity(driverId, "AVAILABILITY", "2026-04-01T10:00:00Z", "2026-04-01T11:00:00Z", "a1", "DRIVER_CARD"),
unknown(driverId, "2026-04-01T11:00:00Z", "2026-04-01T11:30:00Z"),
activity(driverId, "WORK", "2026-04-01T11:30:00Z", "2026-04-01T12:00:00Z", "w2", "DRIVER_CARD"),
unknown(driverId, "2026-04-01T12:00:00Z", "2026-04-01T20:00:00Z"),
activity(driverId, "DRIVE", "2026-04-01T20:00:00Z", "2026-04-01T20:30:00Z", "d1", "DRIVER_CARD")
);
EsperOperatingPeriodEngine.EsperOperatingPeriodEvaluation collectorEvaluation = operatingPeriodEngine.evaluate(
evaluationIntervals,
Duration.ofHours(7),
EsperOperatingPeriodEngineMode.STREAM_COLLECTOR
);
EsperOperatingPeriodEngine.EsperOperatingPeriodEvaluation fullEplEvaluation = operatingPeriodEngine.evaluate(
evaluationIntervals,
Duration.ofHours(7),
EsperOperatingPeriodEngineMode.FULL_EPL
);
assertThat(fullEplEvaluation.periodizedIntervals())
.usingRecursiveComparison()
.isEqualTo(collectorEvaluation.periodizedIntervals());
assertThat(fullEplEvaluation.closedPeriods())
.usingRecursiveComparison()
.isEqualTo(collectorEvaluation.closedPeriods());
}
private ActivityIntervalDto activity(
UUID driverId,
String activity,

42
src/test/java/at/procon/eventhub/importing/ImportChunkPlannerTest.java Normal file
View File

@ -0,0 +1,42 @@
package at.procon.eventhub.importing;
import at.procon.eventhub.dto.AcquisitionStrategy;
import at.procon.eventhub.dto.EventFamily;
import at.procon.eventhub.dto.EventSourceDto;
import at.procon.eventhub.dto.ImportMode;
import at.procon.eventhub.dto.ImportScopeDto;
import at.procon.eventhub.dto.SourceGroupRefDto;
import at.procon.eventhub.yellowfox.dto.YellowFoxD8ImportRequest;
import java.time.OffsetDateTime;
import java.util.EnumSet;
import org.junit.jupiter.api.Test;
import static org.assertj.core.api.Assertions.assertThat;
class ImportChunkPlannerTest {
@Test
void keepsScheduledRowWatermarkImportsAsSingleChunkEvenWhenScopeHasWindow() {
ImportChunkPlanner planner = new ImportChunkPlanner();
YellowFoxD8ImportRequest request = new YellowFoxD8ImportRequest(
"tenant-1",
new EventSourceDto("YELLOWFOX", "TELEMATICS_PLATFORM", "YELLOWFOX_D8", "instance-1", "setting-1", null),
(SourceGroupRefDto) null,
ImportScopeDto.tenantAll(
OffsetDateTime.parse("2026-04-01T00:00:00+02:00"),
OffsetDateTime.parse("2026-04-10T00:00:00+02:00")
),
EnumSet.noneOf(EventFamily.class),
ImportMode.INCREMENTAL_UPDATE,
false,
AcquisitionStrategy.SOURCE_ROW_WATERMARK
);
assertThat(planner.chunksFor(request, 1))
.containsExactly(new ImportTimeChunkDto(
1,
OffsetDateTime.parse("2026-04-01T00:00:00+02:00"),
OffsetDateTime.parse("2026-04-10T00:00:00+02:00")
));
}
}

187
src/test/java/at/procon/eventhub/importing/extraction/AbstractJdbcExtractionBatchExecutorCursorTest.java Normal file
View File

@ -0,0 +1,187 @@
package at.procon.eventhub.importing.extraction;
import at.procon.eventhub.config.EventHubProperties;
import at.procon.eventhub.dto.AcquisitionStrategy;
import at.procon.eventhub.dto.EventFamily;
import at.procon.eventhub.dto.EventSourceDto;
import at.procon.eventhub.dto.ImportCursorStateDto;
import at.procon.eventhub.dto.ImportMode;
import at.procon.eventhub.dto.ImportScopeDto;
import at.procon.eventhub.importing.ExtractionBatchResult;
import at.procon.eventhub.importing.ImportPlanItemDto;
import at.procon.eventhub.importing.ImportRunRequest;
import at.procon.eventhub.importing.ImportTimeChunkDto;
import at.procon.eventhub.importing.persistence.ImportCursorRepository;
import java.time.OffsetDateTime;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.UUID;
import org.junit.jupiter.api.Test;
import static org.assertj.core.api.Assertions.assertThat;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
class AbstractJdbcExtractionBatchExecutorCursorTest {
@Test
void bootstrapsWatermarkCursorFromLatestExistingCursorWhenExactCursorIsMissing() {
ImportCursorRepository repository = mock(ImportCursorRepository.class);
TestExecutor executor = new TestExecutor(repository);
TestRequest request = new TestRequest();
ImportPlanItemDto planItem = new ImportPlanItemDto(
EventFamily.DRIVER_ACTIVITY,
"VEHICLE_UNIT",
"VU_ACTIVITY",
List.of("VUActivity"),
"VEHICLE",
"Vehicle activity",
AcquisitionStrategy.SOURCE_ROW_WATERMARK
);
ImportCursorStateDto fallbackCursor = new ImportCursorStateDto(
null,
"9001",
null,
OffsetDateTime.parse("2026-04-09T23:59:59+02:00")
);
when(repository.findCursor(
"tenant-1",
21,
request.importScope().stableKey(),
EventFamily.DRIVER_ACTIVITY,
"VEHICLE_UNIT",
AcquisitionStrategy.SOURCE_ROW_WATERMARK
)).thenReturn(null);
when(repository.findLatestCursor(
"tenant-1",
21,
request.importScope().stableKey(),
EventFamily.DRIVER_ACTIVITY,
"VEHICLE_UNIT"
)).thenReturn(fallbackCursor);
assertThat(executor.resolveCursor(21, request, planItem)).isEqualTo(fallbackCursor);
verify(repository).findLatestCursor(
"tenant-1",
21,
request.importScope().stableKey(),
EventFamily.DRIVER_ACTIVITY,
"VEHICLE_UNIT"
);
}
private static final class TestExecutor extends AbstractJdbcExtractionBatchExecutor<TestRequest, TestResult> {
private TestExecutor(ImportCursorRepository repository) {
super(null, null, null, new EventHubProperties(), null, repository);
}
private ImportCursorStateDto resolveCursor(int eventSourceId, TestRequest request, ImportPlanItemDto planItem) {
return findCursor(eventSourceId, request, planItem);
}
@Override
protected Optional<ExtractionDefinition<TestRequest>> findDefinition(String code) {
return Optional.empty();
}
@Override
protected EventSourceDto eventSourceFor(TestRequest request, ImportPlanItemDto planItem) {
return request.eventSource();
}
@Override
protected TestResult resultFor(UUID packageId, ImportPlanItemDto planItem, ImportTimeChunkDto chunk, ImportCursorStateDto cursor, ExtractedEventStats stats) {
return new TestResult();
}
}
private record TestRequest() implements ImportRunRequest {
@Override
public String tenantKey() {
return "tenant-1";
}
@Override
public EventSourceDto eventSource() {
return new EventSourceDto("TACHOGRAPH", "VEHICLE_UNIT", "TACHOGRAPH_VEHICLE_UNIT", "instance-1", "setting-1", null);
}
@Override
public at.procon.eventhub.dto.SourceGroupRefDto sourceGroup() {
return null;
}
@Override
public ImportScopeDto importScope() {
return ImportScopeDto.tenantAll(
OffsetDateTime.parse("2026-04-01T00:00:00+02:00"),
OffsetDateTime.parse("2026-04-10T00:00:00+02:00")
);
}
@Override
public Set<EventFamily> eventFamilies() {
return Set.of(EventFamily.DRIVER_ACTIVITY);
}
@Override
public ImportMode mode() {
return ImportMode.INCREMENTAL_UPDATE;
}
@Override
public boolean refreshMasterDataFirst() {
return false;
}
@Override
public AcquisitionStrategy acquisitionStrategy() {
return AcquisitionStrategy.SOURCE_ROW_WATERMARK;
}
}
private static final class TestResult implements ExtractionBatchResult {
@Override
public int eventsInserted() {
return 0;
}
@Override
public boolean executed() {
return false;
}
@Override
public OffsetDateTime lastSourcePackageImportedAt() {
return null;
}
@Override
public String lastSourcePackageId() {
return null;
}
@Override
public OffsetDateTime lastSourceRowUpdatedAt() {
return null;
}
@Override
public OffsetDateTime lastOccurredTo() {
return null;
}
@Override
public Map<String, Integer> eventTypeCounts() {
return Map.of();
}
}
}

105
src/test/java/at/procon/eventhub/yellowfox/service/JdbcYellowFoxD8BookingExtractionBatchExecutorCursorTest.java Normal file
View File

@ -0,0 +1,105 @@
package at.procon.eventhub.yellowfox.service;
import at.procon.eventhub.config.EventHubProperties;
import at.procon.eventhub.dto.AcquisitionStrategy;
import at.procon.eventhub.dto.EventFamily;
import at.procon.eventhub.dto.EventSourceDto;
import at.procon.eventhub.dto.ImportCursorStateDto;
import at.procon.eventhub.dto.ImportMode;
import at.procon.eventhub.dto.ImportScopeDto;
import at.procon.eventhub.dto.SourceGroupRefDto;
import at.procon.eventhub.dto.SourceGroupType;
import at.procon.eventhub.importing.ImportPlanItemDto;
import at.procon.eventhub.importing.persistence.ImportCursorRepository;
import at.procon.eventhub.yellowfox.dto.YellowFoxD8ImportRequest;
import java.time.OffsetDateTime;
import java.util.EnumSet;
import java.util.List;
import org.junit.jupiter.api.Test;
import org.springframework.core.io.DefaultResourceLoader;
import static org.assertj.core.api.Assertions.assertThat;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
class JdbcYellowFoxD8BookingExtractionBatchExecutorCursorTest {
@Test
void bootstrapsScheduledWatermarkCursorFromLatestExistingCursorWhenExactCursorIsMissing() {
ImportCursorRepository repository = mock(ImportCursorRepository.class);
JdbcYellowFoxD8BookingExtractionBatchExecutor executor = executor(repository);
YellowFoxD8ImportRequest request = request();
ImportPlanItemDto planItem = new ImportPlanItemDto(
EventFamily.DRIVER_ACTIVITY,
"TELEMATICS_PLATFORM",
"YELLOWFOX_D8_BOOKING",
List.of("data.d8_booking"),
"BOTH",
"YellowFox bookings",
AcquisitionStrategy.SOURCE_ROW_WATERMARK
);
ImportCursorStateDto fallbackCursor = new ImportCursorStateDto(
null,
"4711",
null,
OffsetDateTime.parse("2026-04-09T23:59:59+02:00")
);
when(repository.findCursor(
"tenant-1",
17,
request.importScope().stableKey(),
EventFamily.DRIVER_ACTIVITY,
"TELEMATICS_PLATFORM",
AcquisitionStrategy.SOURCE_ROW_WATERMARK
)).thenReturn(null);
when(repository.findLatestCursor(
"tenant-1",
17,
request.importScope().stableKey(),
EventFamily.DRIVER_ACTIVITY,
"TELEMATICS_PLATFORM"
)).thenReturn(fallbackCursor);
assertThat(executor.findCursor(17, request, planItem)).isEqualTo(fallbackCursor);
verify(repository).findLatestCursor(
"tenant-1",
17,
request.importScope().stableKey(),
EventFamily.DRIVER_ACTIVITY,
"TELEMATICS_PLATFORM"
);
}
private JdbcYellowFoxD8BookingExtractionBatchExecutor executor(ImportCursorRepository repository) {
EventHubProperties properties = new EventHubProperties();
return new JdbcYellowFoxD8BookingExtractionBatchExecutor(
null,
null,
new DefaultResourceLoader(),
repository,
properties,
null,
null,
null
);
}
private YellowFoxD8ImportRequest request() {
return new YellowFoxD8ImportRequest(
"tenant-1",
new EventSourceDto("YELLOWFOX", "TELEMATICS_PLATFORM", "YELLOWFOX_D8", "instance-1", "setting-1", null),
new SourceGroupRefDto(SourceGroupType.FLEET, "7", null, "Fleet 7"),
ImportScopeDto.tenantAll(
OffsetDateTime.parse("2026-04-01T00:00:00+02:00"),
OffsetDateTime.parse("2026-04-10T00:00:00+02:00")
),
EnumSet.of(EventFamily.DRIVER_ACTIVITY),
ImportMode.INCREMENTAL_UPDATE,
false,
AcquisitionStrategy.SOURCE_ROW_WATERMARK
);
}
}

110
src/test/java/at/procon/eventhub/yellowfox/service/JdbcYellowFoxD8BookingExtractionBatchExecutorTest.java Normal file
View File

@ -0,0 +1,110 @@
package at.procon.eventhub.yellowfox.service;
import at.procon.eventhub.config.EventHubProperties;
import at.procon.eventhub.dto.AcquisitionStrategy;
import at.procon.eventhub.dto.EventSourceDto;
import at.procon.eventhub.dto.ImportMode;
import at.procon.eventhub.dto.ImportScopeDto;
import at.procon.eventhub.dto.ImportCursorStateDto;
import at.procon.eventhub.dto.SourceGroupRefDto;
import at.procon.eventhub.dto.SourceGroupType;
import at.procon.eventhub.yellowfox.dto.YellowFoxD8ImportRequest;
import java.time.Duration;
import java.time.OffsetDateTime;
import java.util.EnumSet;
import org.junit.jupiter.api.Test;
import org.springframework.core.io.DefaultResourceLoader;
import static org.assertj.core.api.Assertions.assertThat;
class JdbcYellowFoxD8BookingExtractionBatchExecutorTest {
@Test
void omitsCursorParametersWhenNoCursorExists() {
JdbcYellowFoxD8BookingExtractionBatchExecutor executor = executor(Duration.ofHours(2));
JdbcYellowFoxD8BookingExtractionBatchExecutor.QuerySpec query = executor.buildQuerySpec(
request(AcquisitionStrategy.SOURCE_ROW_WATERMARK),
ImportScopeDto.tenantAll(
OffsetDateTime.parse("2026-04-01T00:00:00+02:00"),
OffsetDateTime.parse("2026-04-02T00:00:00+02:00")
),
null
);
assertThat(query.sql()).contains("b.utc >= :occurredFrom");
assertThat(query.sql()).contains("b.utc < :occurredTo");
assertThat(query.sql()).contains("f.id = :fleetId");
assertThat(query.sql()).doesNotContain(":lastOccurredTo");
assertThat(query.sql()).doesNotContain(" is null");
assertThat(query.params()).containsOnlyKeys("occurredFrom", "occurredTo", "fleetId");
assertThat(query.fleetId()).isEqualTo(7);
}
@Test
void keepsStrictUtcEventIdCursorWhenOverlapIsDisabled() {
JdbcYellowFoxD8BookingExtractionBatchExecutor executor = executor(Duration.ZERO);
OffsetDateTime cursorTime = OffsetDateTime.parse("2026-04-02T09:15:00+02:00");
JdbcYellowFoxD8BookingExtractionBatchExecutor.QuerySpec query = executor.buildQuerySpec(
request(AcquisitionStrategy.SOURCE_ROW_WATERMARK),
ImportScopeDto.tenantAll(null, null),
new ImportCursorStateDto(null, "4711", null, cursorTime)
);
assertThat(query.sql()).contains("b.utc > :lastOccurredTo");
assertThat(query.sql()).contains("b.utc = :lastOccurredTo");
assertThat(query.sql()).contains("b.eventid > :lastSourceRowId");
assertThat(query.sql()).doesNotContain(" is null");
assertThat(query.params()).containsEntry("lastOccurredTo", cursorTime);
assertThat(query.params()).containsEntry("lastSourceRowId", "4711");
}
@Test
void keepsOccurredToCapOnceCursorExistsForWatermarkIncrementalRuns() {
JdbcYellowFoxD8BookingExtractionBatchExecutor executor = executor(Duration.ZERO);
OffsetDateTime cursorTime = OffsetDateTime.parse("2026-04-02T09:15:00+02:00");
JdbcYellowFoxD8BookingExtractionBatchExecutor.QuerySpec query = executor.buildQuerySpec(
request(AcquisitionStrategy.SOURCE_ROW_WATERMARK),
ImportScopeDto.tenantAll(
OffsetDateTime.parse("2026-04-01T00:00:00+02:00"),
OffsetDateTime.parse("2026-04-02T00:00:00+02:00")
),
new ImportCursorStateDto(null, "4711", null, cursorTime)
);
assertThat(query.sql()).contains("b.utc >= :occurredFrom");
assertThat(query.sql()).contains("b.utc < :occurredTo");
assertThat(query.params()).containsEntry("occurredFrom", OffsetDateTime.parse("2026-04-01T00:00:00+02:00"));
assertThat(query.params()).containsEntry("occurredTo", OffsetDateTime.parse("2026-04-02T00:00:00+02:00"));
}
private JdbcYellowFoxD8BookingExtractionBatchExecutor executor(Duration overlap) {
EventHubProperties properties = new EventHubProperties();
properties.getYellowFox().setOccurredAtOverlap(overlap);
return new JdbcYellowFoxD8BookingExtractionBatchExecutor(
null,
null,
new DefaultResourceLoader(),
null,
properties,
null,
null,
null
);
}
private YellowFoxD8ImportRequest request(AcquisitionStrategy strategy) {
return new YellowFoxD8ImportRequest(
"tenant-1",
new EventSourceDto("YELLOWFOX", "TELEMATICS_PLATFORM", "YELLOWFOX_D8", "instance-1", "setting-1", null),
new SourceGroupRefDto(SourceGroupType.FLEET, "7", null, "Fleet 7"),
ImportScopeDto.tenantAll(null, null),
EnumSet.noneOf(at.procon.eventhub.dto.EventFamily.class),
ImportMode.INCREMENTAL_UPDATE,
false,
strategy
);
}
}

90
src/test/java/at/procon/eventhub/yellowfox/service/YellowFoxD8BookingRowMapperTest.java Normal file
View File

@ -0,0 +1,90 @@
package at.procon.eventhub.yellowfox.service;
import com.fasterxml.jackson.databind.ObjectMapper;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.time.OffsetDateTime;
import java.util.List;
import java.util.Map;
import org.junit.jupiter.api.Test;
import static org.assertj.core.api.Assertions.assertThat;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
class YellowFoxD8BookingRowMapperTest {
@Test
void trimsVehicleRegistrationValuesInReferencesAndPayload() throws SQLException {
YellowFoxD8BookingRowMapper mapper = new YellowFoxD8BookingRowMapper(new ObjectMapper());
ResultSet rs = mock(ResultSet.class);
OffsetDateTime occurredAt = OffsetDateTime.parse("2026-05-08T10:15:30+02:00");
when(rs.getString("eventid")).thenReturn(" evt-1 ");
when(rs.getObject("utc", OffsetDateTime.class)).thenReturn(occurredAt);
when(rs.getInt("vehicle_id")).thenReturn(42);
when(rs.getInt("driver_id")).thenReturn(7);
when(rs.getInt("fleet_id")).thenReturn(9);
when(rs.getInt("odometer")).thenReturn(1234);
when(rs.getInt("telematic_provider_id")).thenReturn(3);
when(rs.getInt("ignition")).thenReturn(1);
when(rs.getInt("previous_ignition")).thenReturn(0);
when(rs.getInt("eventtype")).thenReturn(2);
when(rs.getInt("state")).thenReturn(3);
when(rs.wasNull()).thenReturn(false);
when(rs.getString("vehicle_vrn")).thenReturn(" W-12345 ");
when(rs.getString("vehicle_vin")).thenReturn(" vin-123 ");
when(rs.getString("driver_card_number")).thenReturn(" card-9 ");
when(rs.getString("fleet_name")).thenReturn(" Fleet 9 ");
when(rs.getString("driver_firstname")).thenReturn(" Ada ");
when(rs.getString("driver_lastname")).thenReturn(" Lovelace ");
when(rs.getString("telematic_provider_name")).thenReturn(" YellowFox ");
when(rs.getString("key")).thenReturn(" booking-key ");
when(rs.getString("payload")).thenReturn("""
{"vrn":" W-12345 ","nested":{"label":" value "},"list":[" x ",1]}
""");
var booking = mapper.map(rs, "tenant-1", "instance-1", "setting-1");
assertThat(booking.eventId()).isEqualTo("evt-1");
assertThat(booking.key()).isEqualTo("booking-key");
assertThat(booking.previousIgnition()).isEqualTo(0);
assertThat(booking.vehicleRef().vin()).isEqualTo("VIN-123");
assertThat(booking.vehicleRef().vehicleRegistration().number()).isEqualTo("W-12345");
assertThat(booking.payload()).containsEntry("vrn", "W-12345");
assertThat(booking.payload()).containsEntry("vehicleVrn", "W-12345");
assertThat(booking.payload()).containsEntry("driverFirstName", "Ada");
assertThat(booking.payload()).containsEntry("telematicProviderName", "YellowFox");
assertThat(((Map<?, ?>) booking.payload().get("nested")).get("label")).isEqualTo("value");
assertThat((List<Object>) booking.payload().get("list")).containsExactly("x", 1);
}
@Test
void truncatesBookingDriverCardNumberToFirst14Characters() throws SQLException {
YellowFoxD8BookingRowMapper mapper = new YellowFoxD8BookingRowMapper(new ObjectMapper());
ResultSet rs = mock(ResultSet.class);
OffsetDateTime occurredAt = OffsetDateTime.parse("2026-05-08T10:15:30+02:00");
when(rs.getString("eventid")).thenReturn("evt-2");
when(rs.getObject("utc", OffsetDateTime.class)).thenReturn(occurredAt);
when(rs.getInt("vehicle_id")).thenReturn(0);
when(rs.getInt("driver_id")).thenReturn(0);
when(rs.getInt("fleet_id")).thenReturn(0);
when(rs.getInt("odometer")).thenReturn(0);
when(rs.getInt("telematic_provider_id")).thenReturn(0);
when(rs.getInt("ignition")).thenReturn(1);
when(rs.getInt("previous_ignition")).thenReturn(0);
when(rs.getInt("eventtype")).thenReturn(2);
when(rs.getInt("state")).thenReturn(3);
when(rs.wasNull()).thenReturn(true);
when(rs.getString("driver_card_number")).thenReturn(" 12345678901234AB ");
when(rs.getString("payload")).thenReturn("{}");
var booking = mapper.map(rs, "tenant-1", "instance-1", "setting-1");
assertThat(booking.driverRef()).isNotNull();
assertThat(booking.driverRef().driverCard()).isNotNull();
assertThat(booking.driverRef().driverCard().number()).isEqualTo("12345678901234");
assertThat(booking.payload()).containsEntry("driverCardNumber", "12345678901234");
}
}

39
src/test/java/at/procon/eventhub/yellowfox/service/YellowFoxD8ConfiguredImportPlanServiceTest.java Normal file
View File

@ -0,0 +1,39 @@
package at.procon.eventhub.yellowfox.service;
import at.procon.eventhub.config.EventHubProperties;
import at.procon.eventhub.dto.AcquisitionStrategy;
import at.procon.eventhub.dto.EventSourceDto;
import at.procon.eventhub.dto.ImportMode;
import at.procon.eventhub.dto.ImportScopeDto;
import at.procon.eventhub.dto.ImportScopeType;
import at.procon.eventhub.yellowfox.dto.YellowFoxD8ImportRequest;
import java.time.OffsetDateTime;
import org.junit.jupiter.api.Test;
import static org.assertj.core.api.Assertions.assertThat;
class YellowFoxD8ConfiguredImportPlanServiceTest {
@Test
void scheduledWatermarkRequestKeepsInitialOccurredWindowAsBootstrapScope() {
EventHubProperties properties = new EventHubProperties();
EventHubProperties.ConfiguredImportPlan plan = new EventHubProperties.ConfiguredImportPlan();
plan.setPlanKey("yellowfox-d8-default");
plan.setTenantKey("Procon");
plan.setEventSource(new EventSourceDto("YELLOWFOX", "TELEMATICS_PLATFORM", "YELLOWFOX_D8", "logistics-db-prod", "yellowfox-main", null));
plan.setImportScope(new ImportScopeDto(ImportScopeType.TENANT_ALL, null, false, null, null));
plan.setScheduledMode(ImportMode.INCREMENTAL_UPDATE);
plan.setScheduledStrategy(AcquisitionStrategy.SOURCE_ROW_WATERMARK);
plan.setInitialOccurredFrom(OffsetDateTime.parse("2026-04-01T00:00:00+02:00"));
plan.setInitialOccurredTo(OffsetDateTime.parse("2026-04-02T00:00:00+02:00"));
properties.getYellowFox().getImportPlans().add(plan);
YellowFoxD8ConfiguredImportPlanService service = new YellowFoxD8ConfiguredImportPlanService(properties);
YellowFoxD8ImportRequest request = service.createScheduledRequest(plan);
assertThat(request.mode()).isEqualTo(ImportMode.INCREMENTAL_UPDATE);
assertThat(request.acquisitionStrategy()).isEqualTo(AcquisitionStrategy.SOURCE_ROW_WATERMARK);
assertThat(request.importScope().occurredFrom()).isEqualTo(OffsetDateTime.parse("2026-04-01T00:00:00+02:00"));
assertThat(request.importScope().occurredTo()).isEqualTo(OffsetDateTime.parse("2026-04-02T00:00:00+02:00"));
}
}

65
src/test/java/at/procon/eventhub/yellowfox/service/YellowFoxD8IgnitionTransitionDetectorTest.java Normal file
View File

@ -0,0 +1,65 @@
package at.procon.eventhub.yellowfox.service;
import at.procon.eventhub.dto.DriverCardRefDto;
import at.procon.eventhub.dto.DriverRefDto;
import at.procon.eventhub.dto.EventDomain;
import at.procon.eventhub.dto.EventType;
import at.procon.eventhub.dto.VehicleRefDto;
import at.procon.eventhub.dto.VehicleRegistrationRefDto;
import at.procon.eventhub.service.EventDetailsFactory;
import at.procon.eventhub.yellowfox.dto.YellowFoxD8BookingDto;
import com.fasterxml.jackson.databind.ObjectMapper;
import java.time.OffsetDateTime;
import java.util.Map;
import org.junit.jupiter.api.Test;
import static org.assertj.core.api.Assertions.assertThat;
class YellowFoxD8IgnitionTransitionDetectorTest {
@Test
void emitsBoundaryTransitionFromPreviousIgnitionFromSourceRow() {
YellowFoxD8BookingEventMapper mapper = new YellowFoxD8BookingEventMapper(new EventDetailsFactory(new ObjectMapper()));
YellowFoxD8IgnitionTransitionDetector detector = new YellowFoxD8IgnitionTransitionDetector(mapper);
var event = detector.newSession(false).detect(booking(1, 0));
assertThat(event).isNotNull();
assertThat(event.eventDomain()).isEqualTo(EventDomain.IGNITION);
assertThat(event.eventType()).isEqualTo(EventType.IGNITION_ON);
}
@Test
void doesNotEmitWhenCurrentIgnitionMatchesPreviousIgnitionFromSourceRow() {
YellowFoxD8BookingEventMapper mapper = new YellowFoxD8BookingEventMapper(new EventDetailsFactory(new ObjectMapper()));
YellowFoxD8IgnitionTransitionDetector detector = new YellowFoxD8IgnitionTransitionDetector(mapper);
var event = detector.newSession(false).detect(booking(1, 1));
assertThat(event).isNull();
}
private YellowFoxD8BookingDto booking(Integer ignition, Integer previousIgnition) {
return new YellowFoxD8BookingDto(
"tenant-1",
"instance-1",
"setting-1",
"7",
"Fleet 7",
"evt-1",
"key-1",
ignition,
previousIgnition,
2,
3,
OffsetDateTime.parse("2026-05-08T10:15:30+02:00"),
null,
new DriverRefDto("1", new DriverCardRefDto(YellowFoxReferenceSemantics.SYNTHETIC_REFERENCE_NATION, "12345678901234")),
new VehicleRefDto("42", "VIN-42", "42", new VehicleRegistrationRefDto(YellowFoxReferenceSemantics.SYNTHETIC_REFERENCE_NATION, "W-4242")),
123_000L,
null,
null,
Map.of()
);
}
}