TriTih
/
eventhub
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Compare commits

base: TriTih:e84dfef614b18c60f1540ffc17e83e946ca68a1a
Branches Tags
TriTih:main
..
compare: TriTih:14a6f8d42ea865fdefa7cb1d24cfa9edc38e18f9
Branches Tags
TriTih:main

No commits in common. "e84dfef614b18c60f1540ffc17e83e946ca68a1a" and "14a6f8d42ea865fdefa7cb1d24cfa9edc38e18f9" have entirely different histories.
e84dfef614 ... 14a6f8d42e

50 changed files with 773 additions and 4794 deletions
Show Stats Expand all files Collapse all files

265
docs/db/create_eventhub_schema.sql
View File

@ -1,8 +1,6 @@
create extension if not exists pgcrypto; create extension if not exists pgcrypto;
create extension if not exists postgis; create extension if not exists postgis;
create extension if not exists timescaledb;
drop schema if exists eventhub cascade;
create schema if not exists eventhub; create schema if not exists eventhub;
create table if not exists eventhub.event_source ( create table if not exists eventhub.event_source (
@ -145,56 +143,11 @@ create table if not exists eventhub.source_master_relation (
constraint chk_source_master_relation_valid_time_order check (valid_from is null or valid_to is null or valid_from <= valid_to) constraint chk_source_master_relation_valid_time_order check (valid_from is null or valid_to is null or valid_from <= valid_to)
); );
create table if not exists eventhub.driver (
id uuid primary key,
first_names text,
last_name text,
birth_date date,
source_updated_at timestamptz,
payload jsonb not null default '{}'::jsonb,
created_at timestamptz not null default now(),
updated_at timestamptz not null default now()
);
create table if not exists eventhub.driver_card (
id uuid primary key,
driver_id uuid references eventhub.driver(id),
nation text not null,
card_number text not null,
source_updated_at timestamptz,
payload jsonb not null default '{}'::jsonb,
created_at timestamptz not null default now(),
updated_at timestamptz not null default now()
);
create table if not exists eventhub.source_driver_identity (
id uuid primary key,
tenant_key text not null,
event_source_id integer not null references eventhub.event_source(id),
source_driver_entity_id text not null,
driver_id uuid not null references eventhub.driver(id) on delete cascade,
source_updated_at timestamptz,
payload jsonb not null default '{}'::jsonb,
created_at timestamptz not null default now(),
updated_at timestamptz not null default now(),
constraint ux_source_driver_identity unique (tenant_key, event_source_id, source_driver_entity_id)
);
create table if not exists eventhub.source_driver_card_identity (
id uuid primary key,
tenant_key text not null,
event_source_id integer not null references eventhub.event_source(id),
source_driver_card_entity_id text not null,
driver_card_id uuid not null references eventhub.driver_card(id) on delete cascade,
source_updated_at timestamptz,
payload jsonb not null default '{}'::jsonb,
created_at timestamptz not null default now(),
updated_at timestamptz not null default now(),
constraint ux_source_driver_card_identity unique (tenant_key, event_source_id, source_driver_card_entity_id)
);
create table if not exists eventhub.vehicle ( create table if not exists eventhub.vehicle (
id uuid primary key, id uuid primary key,
tenant_key text not null,
event_source_id integer not null references eventhub.event_source(id),
source_vehicle_entity_id text,
vin text, vin text,
created_at timestamptz not null default now(), created_at timestamptz not null default now(),
updated_at timestamptz not null default now() updated_at timestamptz not null default now()
@ -202,6 +155,9 @@ create table if not exists eventhub.vehicle (
create table if not exists eventhub.vehicle_registration ( create table if not exists eventhub.vehicle_registration (
id uuid primary key, id uuid primary key,
tenant_key text not null,
event_source_id integer not null references eventhub.event_source(id),
source_registration_entity_id text,
nation text not null, nation text not null,
registration_number text not null, registration_number text not null,
source_updated_at timestamptz, source_updated_at timestamptz,
@ -210,32 +166,6 @@ create table if not exists eventhub.vehicle_registration (
updated_at timestamptz not null default now() updated_at timestamptz not null default now()
); );
create table if not exists eventhub.source_vehicle_identity (
id uuid primary key,
tenant_key text not null,
event_source_id integer not null references eventhub.event_source(id),
source_vehicle_entity_id text not null,
vehicle_id uuid not null references eventhub.vehicle(id) on delete cascade,
source_updated_at timestamptz,
payload jsonb not null default '{}'::jsonb,
created_at timestamptz not null default now(),
updated_at timestamptz not null default now(),
constraint ux_source_vehicle_identity unique (tenant_key, event_source_id, source_vehicle_entity_id)
);
create table if not exists eventhub.source_vehicle_registration_identity (
id uuid primary key,
tenant_key text not null,
event_source_id integer not null references eventhub.event_source(id),
source_registration_entity_id text not null,
vehicle_registration_id uuid not null references eventhub.vehicle_registration(id) on delete cascade,
source_updated_at timestamptz,
payload jsonb not null default '{}'::jsonb,
created_at timestamptz not null default now(),
updated_at timestamptz not null default now(),
constraint ux_source_vehicle_registration_identity unique (tenant_key, event_source_id, source_registration_entity_id)
);
create table if not exists eventhub.vehicle_registration_assignment ( create table if not exists eventhub.vehicle_registration_assignment (
id uuid primary key, id uuid primary key,
tenant_key text not null, tenant_key text not null,
@ -256,11 +186,9 @@ create table if not exists eventhub.event (
event_source_id integer not null references eventhub.event_source(id), event_source_id integer not null references eventhub.event_source(id),
data_package_id uuid not null references eventhub.data_package(id), data_package_id uuid not null references eventhub.data_package(id),
external_source_event_id text not null, external_source_event_id text not null,
driver_id uuid references eventhub.driver(id), driver_entity_id uuid references eventhub.source_master_entity(id),
driver_card_id uuid references eventhub.driver_card(id),
vehicle_id uuid references eventhub.vehicle(id), vehicle_id uuid references eventhub.vehicle(id),
vehicle_registration_id uuid references eventhub.vehicle_registration(id), vehicle_registration_id uuid references eventhub.vehicle_registration(id),
source_package_id text,
source_package_entity_id uuid references eventhub.source_master_entity(id), source_package_entity_id uuid references eventhub.source_master_entity(id),
occurred_at timestamptz not null, occurred_at timestamptz not null,
received_partner_at timestamptz, received_partner_at timestamptz,
@ -277,90 +205,26 @@ create table if not exists eventhub.event (
created_at timestamptz not null default now(), created_at timestamptz not null default now(),
constraint pk_event primary key (occurred_at, id), constraint pk_event primary key (occurred_at, id),
constraint chk_event_driver_or_vehicle_ref check ( constraint chk_event_driver_or_vehicle_ref check (
driver_id is not null driver_entity_id is not null
or driver_card_id is not null
or vehicle_id is not null or vehicle_id is not null
or vehicle_registration_id is not null or vehicle_registration_id is not null
) )
); );
create table if not exists eventhub.event_source_record (
source_record_key_hash text primary key,
event_occurred_at timestamptz not null,
event_id uuid not null,
created_at timestamptz not null default now()
);
create table if not exists eventhub.event_detail ( create table if not exists eventhub.event_detail (
event_occurred_at timestamptz not null, event_occurred_at timestamptz not null,
event_id uuid not null, event_id uuid not null,
detail_type text not null, detail_type text not null,
attributes jsonb not null default '{}'::jsonb, attributes jsonb not null default '{}'::jsonb,
created_at timestamptz not null default now(), created_at timestamptz not null default now(),
constraint pk_event_detail primary key (event_occurred_at, event_id, detail_type) constraint pk_event_detail primary key (event_occurred_at, event_id, detail_type),
); constraint fk_event_detail_event foreign key (event_occurred_at, event_id)
select create_hypertable(
'eventhub.event',
'occurred_at',
chunk_time_interval => interval '7 days',
if_not_exists => true
);
alter table eventhub.event_source_record
add constraint fk_event_source_record_event foreign key (event_occurred_at, event_id)
references eventhub.event(occurred_at, id) references eventhub.event(occurred_at, id)
on delete cascade on delete cascade
deferrable initially deferred; );
alter table eventhub.event_detail create unique index if not exists ux_event_source_record
add constraint fk_event_detail_event foreign key (event_occurred_at, event_id) on eventhub.event(source_record_key_hash);
references eventhub.event(occurred_at, id)
on delete cascade;
create index if not exists idx_data_package_source_time
on eventhub.data_package(tenant_key, event_source_id, received_at desc);
create index if not exists idx_data_package_scope
on eventhub.data_package(tenant_key, import_scope_type, root_source_org_entity_id, occurred_from, occurred_to);
create index if not exists idx_data_package_extraction
on eventhub.data_package(tenant_key, event_source_id, import_run_id, event_family, extraction_source_kind, extraction_code, batch_no);
create index if not exists idx_import_run_source_status
on eventhub.import_run(tenant_key, event_source_id, status, started_at desc);
create index if not exists idx_source_master_entity_type_key
on eventhub.source_master_entity(tenant_key, event_source_id, entity_type, source_external_key)
where source_external_key is not null;
create index if not exists idx_source_master_entity_payload_gin
on eventhub.source_master_entity using gin(payload);
create index if not exists idx_source_master_relation_from
on eventhub.source_master_relation(tenant_key, event_source_id, from_entity_type, from_source_entity_id, relation_type);
create index if not exists idx_source_master_relation_to
on eventhub.source_master_relation(tenant_key, event_source_id, to_entity_type, to_source_entity_id, relation_type);
create index if not exists idx_source_master_relation_payload_gin
on eventhub.source_master_relation using gin(payload);
create index if not exists idx_vehicle_vin
on eventhub.vehicle(vin)
where vin is not null;
create index if not exists idx_vehicle_registration_plate
on eventhub.vehicle_registration(nation, registration_number);
create index if not exists idx_vehicle_registration_assignment_registration_time
on eventhub.vehicle_registration_assignment(vehicle_registration_id, valid_from desc, valid_to);
create index if not exists idx_vehicle_registration_assignment_vehicle_time
on eventhub.vehicle_registration_assignment(vehicle_id, valid_from desc, valid_to);
create index if not exists idx_event_source_record_event
on eventhub.event_source_record(event_occurred_at, event_id);
create index if not exists idx_event_signature create index if not exists idx_event_signature
on eventhub.event(event_signature_hash) on eventhub.event(event_signature_hash)
@ -372,49 +236,12 @@ create index if not exists idx_event_source_time
create index if not exists idx_event_package_time create index if not exists idx_event_package_time
on eventhub.event(data_package_id, occurred_at desc); on eventhub.event(data_package_id, occurred_at desc);
create index if not exists idx_event_source_package_id
on eventhub.event(source_package_id)
where source_package_id is not null;
create index if not exists idx_event_domain_type_time create index if not exists idx_event_domain_type_time
on eventhub.event(event_domain, event_type, occurred_at desc); on eventhub.event(event_domain, event_type, occurred_at desc);
create index if not exists idx_driver_card_key
on eventhub.driver_card(nation, card_number);
create index if not exists idx_driver_card_driver
on eventhub.driver_card(driver_id)
where driver_id is not null;
create unique index if not exists ux_driver_card_key
on eventhub.driver_card(nation, card_number);
create unique index if not exists ux_vehicle_vin
on eventhub.vehicle(vin)
where vin is not null;
create unique index if not exists ux_vehicle_registration_plate
on eventhub.vehicle_registration(nation, registration_number);
create index if not exists idx_source_driver_identity_driver
on eventhub.source_driver_identity(driver_id);
create index if not exists idx_source_driver_card_identity_card
on eventhub.source_driver_card_identity(driver_card_id);
create index if not exists idx_source_vehicle_identity_vehicle
on eventhub.source_vehicle_identity(vehicle_id);
create index if not exists idx_source_vehicle_registration_identity_registration
on eventhub.source_vehicle_registration_identity(vehicle_registration_id);
create index if not exists idx_event_driver_time create index if not exists idx_event_driver_time
on eventhub.event(driver_id, occurred_at desc) on eventhub.event(driver_entity_id, occurred_at desc)
where driver_id is not null; where driver_entity_id is not null;
create index if not exists idx_event_driver_card_time
on eventhub.event(driver_card_id, occurred_at desc)
where driver_card_id is not null;
create index if not exists idx_event_vehicle_time create index if not exists idx_event_vehicle_time
on eventhub.event(vehicle_id, occurred_at desc) on eventhub.event(vehicle_id, occurred_at desc)
@ -437,18 +264,54 @@ create index if not exists idx_event_detail_type
create index if not exists idx_event_detail_attributes_gin create index if not exists idx_event_detail_attributes_gin
on eventhub.event_detail using gin(attributes); on eventhub.event_detail using gin(attributes);
create index if not exists idx_event_detail_yellowfox_slot create index if not exists idx_source_master_entity_type_key
on eventhub.event_detail(detail_type, (attributes ->> 'slot'), event_occurred_at) on eventhub.source_master_entity(tenant_key, event_source_id, entity_type, source_external_key)
where detail_type in ('DRIVER_ACTIVITY', 'DRIVER_CARD'); where source_external_key is not null;
create index if not exists idx_event_detail_yellowfox_eventtype_state create index if not exists idx_source_master_entity_payload_gin
on eventhub.event_detail( on eventhub.source_master_entity using gin(payload);
(attributes ->> 'yellowFoxEventType'),
(attributes ->> 'yellowFoxState'),
event_occurred_at
)
where attributes ? 'yellowFoxEventType';
create index if not exists idx_event_detail_yellowfox_ignition create index if not exists idx_source_master_relation_from
on eventhub.event_detail(detail_type, (attributes ->> 'ignitionState'), event_occurred_at) on eventhub.source_master_relation(tenant_key, event_source_id, from_entity_type, from_source_entity_id, relation_type);
where attributes ? 'ignitionState';
create index if not exists idx_source_master_relation_to
on eventhub.source_master_relation(tenant_key, event_source_id, to_entity_type, to_source_entity_id, relation_type);
create index if not exists idx_source_master_relation_payload_gin
on eventhub.source_master_relation using gin(payload);
create index if not exists idx_vehicle_lookup_ctx
on eventhub.vehicle(tenant_key, event_source_id, updated_at desc);
create index if not exists idx_vehicle_source_entity
on eventhub.vehicle(tenant_key, event_source_id, source_vehicle_entity_id)
where source_vehicle_entity_id is not null;
create index if not exists idx_vehicle_vin
on eventhub.vehicle(tenant_key, event_source_id, vin)
where vin is not null;
create index if not exists idx_vehicle_registration_source_entity
on eventhub.vehicle_registration(tenant_key, event_source_id, source_registration_entity_id)
where source_registration_entity_id is not null;
create index if not exists idx_vehicle_registration_plate
on eventhub.vehicle_registration(tenant_key, event_source_id, nation, registration_number);
create index if not exists idx_vehicle_registration_assignment_registration_time
on eventhub.vehicle_registration_assignment(vehicle_registration_id, valid_from desc, valid_to);
create index if not exists idx_vehicle_registration_assignment_vehicle_time
on eventhub.vehicle_registration_assignment(vehicle_id, valid_from desc, valid_to);
create index if not exists idx_data_package_source_time
on eventhub.data_package(tenant_key, event_source_id, received_at desc);
create index if not exists idx_data_package_scope
on eventhub.data_package(tenant_key, import_scope_type, root_source_org_entity_id, occurred_from, occurred_to);
create index if not exists idx_data_package_extraction
on eventhub.data_package(tenant_key, event_source_id, import_run_id, event_family, extraction_source_kind, extraction_code, batch_no);
create index if not exists idx_import_run_source_status
on eventhub.import_run(tenant_key, event_source_id, status, started_at desc);

21
docs/sql/truncate-eventhub-data.sql
View File

@ -1,21 +0,0 @@
-- Truncate all application data in the eventhub schema.
-- Keeps the schema and Flyway migration history intact.
-- Intended for PostgreSQL / TimescaleDB environments.
DO $$
DECLARE
table_list text;
BEGIN
SELECT string_agg(format('%I.%I', schemaname, tablename), ', ' ORDER BY tablename)
INTO table_list
FROM pg_tables
WHERE schemaname = 'eventhub';
IF table_list IS NULL THEN
RAISE NOTICE 'No tables found in schema eventhub.';
RETURN;
END IF;
EXECUTE 'TRUNCATE TABLE ' || table_list || ' RESTART IDENTITY CASCADE';
END
$$;

217
postman/eventhub-rest-methods.postman_collection.json
View File

@ -12,10 +12,6 @@
{ {
"key": "planKey", "key": "planKey",
"value": "kralowetz-tachograph-org-147" "value": "kralowetz-tachograph-org-147"
},
{
"key": "yellowFoxPlanKey",
"value": "yellowfox-d8-default"
} }
], ],
"item": [ "item": [
@ -380,219 +376,6 @@
] ]
} }
} }
},
{
"name": "POST /api/eventhub/acquisition/yellowfox/d8/imports/plan",
"request": {
"method": "POST",
"header": [
{
"key": "Content-Type",
"value": "application/json"
}
],
"body": {
"mode": "raw",
"raw": "{\n \"tenantKey\": \"Procon\",\n \"eventSource\": {\n \"providerKey\": \"YELLOWFOX\",\n \"sourceKind\": \"TELEMATICS_PLATFORM\",\n \"sourceKey\": \"YELLOWFOX_D8\",\n \"sourceInstanceKey\": \"logistics-db-prod\",\n \"tenantProviderSettingKey\": \"yellowfox-main\",\n \"externalFleetKey\": null\n },\n \"sourceGroup\": null,\n \"importScope\": {\n \"type\": \"TENANT_ALL\",\n \"rootSourceOrganisation\": null,\n \"includeChildren\": false,\n \"occurredFrom\": null,\n \"occurredTo\": null\n },\n \"eventFamilies\": [\n \"DRIVER_ACTIVITY\",\n \"DRIVER_CARD\"\n ],\n \"mode\": \"INCREMENTAL_UPDATE\",\n \"refreshMasterDataFirst\": false,\n \"acquisitionStrategy\": \"SOURCE_ROW_WATERMARK\"\n}"
},
"url": {
"raw": "{{baseUrl}}/api/eventhub/acquisition/yellowfox/d8/imports/plan",
"host": [
"{{baseUrl}}"
],
"path": [
"api",
"eventhub",
"acquisition",
"yellowfox",
"d8",
"imports",
"plan"
]
}
}
},
{
"name": "POST /api/eventhub/acquisition/yellowfox/d8/imports/start?execute=true",
"request": {
"method": "POST",
"header": [
{
"key": "Content-Type",
"value": "application/json"
}
],
"body": {
"mode": "raw",
"raw": "{\n \"tenantKey\": \"Procon\",\n \"eventSource\": {\n \"providerKey\": \"YELLOWFOX\",\n \"sourceKind\": \"TELEMATICS_PLATFORM\",\n \"sourceKey\": \"YELLOWFOX_D8\",\n \"sourceInstanceKey\": \"logistics-db-prod\",\n \"tenantProviderSettingKey\": \"yellowfox-main\",\n \"externalFleetKey\": null\n },\n \"sourceGroup\": null,\n \"importScope\": {\n \"type\": \"TENANT_ALL\",\n \"rootSourceOrganisation\": null,\n \"includeChildren\": false,\n \"occurredFrom\": null,\n \"occurredTo\": null\n },\n \"eventFamilies\": [\n \"DRIVER_ACTIVITY\",\n \"DRIVER_CARD\"\n ],\n \"mode\": \"INCREMENTAL_UPDATE\",\n \"refreshMasterDataFirst\": false,\n \"acquisitionStrategy\": \"SOURCE_ROW_WATERMARK\"\n}"
},
"url": {
"raw": "{{baseUrl}}/api/eventhub/acquisition/yellowfox/d8/imports/start?execute=true",
"host": [
"{{baseUrl}}"
],
"path": [
"api",
"eventhub",
"acquisition",
"yellowfox",
"d8",
"imports",
"start"
],
"query": [
{
"key": "execute",
"value": "true"
}
]
}
}
},
{
"name": "POST /api/eventhub/acquisition/yellowfox/d8/master-data/refresh",
"request": {
"method": "POST",
"header": [
{
"key": "Content-Type",
"value": "application/json"
}
],
"body": {
"mode": "raw",
"raw": "{\n \"tenantKey\": \"Procon\",\n \"eventSource\": {\n \"providerKey\": \"YELLOWFOX\",\n \"sourceKind\": \"TELEMATICS_PLATFORM\",\n \"sourceKey\": \"YELLOWFOX_D8\",\n \"sourceInstanceKey\": \"logistics-db-prod\",\n \"tenantProviderSettingKey\": \"yellowfox-main\",\n \"externalFleetKey\": null\n },\n \"sourceGroup\": null,\n \"importScope\": {\n \"type\": \"TENANT_ALL\",\n \"rootSourceOrganisation\": null,\n \"includeChildren\": false,\n \"occurredFrom\": null,\n \"occurredTo\": null\n },\n \"eventFamilies\": [\n \"DRIVER_ACTIVITY\",\n \"DRIVER_CARD\"\n ],\n \"mode\": \"INCREMENTAL_UPDATE\",\n \"refreshMasterDataFirst\": true,\n \"acquisitionStrategy\": \"SOURCE_ROW_WATERMARK\"\n}"
},
"url": {
"raw": "{{baseUrl}}/api/eventhub/acquisition/yellowfox/d8/master-data/refresh",
"host": [
"{{baseUrl}}"
],
"path": [
"api",
"eventhub",
"acquisition",
"yellowfox",
"d8",
"master-data",
"refresh"
]
}
}
},
{
"name": "GET /api/eventhub/acquisition/yellowfox/d8/imports/configured-plans",
"request": {
"method": "GET",
"url": {
"raw": "{{baseUrl}}/api/eventhub/acquisition/yellowfox/d8/imports/configured-plans",
"host": [
"{{baseUrl}}"
],
"path": [
"api",
"eventhub",
"acquisition",
"yellowfox",
"d8",
"imports",
"configured-plans"
]
}
}
},
{
"name": "GET /api/eventhub/acquisition/yellowfox/d8/imports/configured-plans/{planKey}",
"request": {
"method": "GET",
"url": {
"raw": "{{baseUrl}}/api/eventhub/acquisition/yellowfox/d8/imports/configured-plans/{{yellowFoxPlanKey}}",
"host": [
"{{baseUrl}}"
],
"path": [
"api",
"eventhub",
"acquisition",
"yellowfox",
"d8",
"imports",
"configured-plans",
"{{yellowFoxPlanKey}}"
]
}
}
},
{
"name": "POST /api/eventhub/acquisition/yellowfox/d8/imports/configured-plans/{planKey}/start",
"request": {
"method": "POST",
"url": {
"raw": "{{baseUrl}}/api/eventhub/acquisition/yellowfox/d8/imports/configured-plans/{{yellowFoxPlanKey}}/start?triggerMode=EXECUTE&mode=INCREMENTAL_UPDATE&strategy=SOURCE_ROW_WATERMARK",
"host": [
"{{baseUrl}}"
],
"path": [
"api",
"eventhub",
"acquisition",
"yellowfox",
"d8",
"imports",
"configured-plans",
"{{yellowFoxPlanKey}}",
"start"
],
"query": [
{
"key": "triggerMode",
"value": "EXECUTE"
},
{
"key": "mode",
"value": "INCREMENTAL_UPDATE"
},
{
"key": "strategy",
"value": "SOURCE_ROW_WATERMARK"
}
]
}
}
},
{
"name": "POST /api/eventhub/acquisition/yellowfox/d8/imports/configured-plans/{planKey}/master-data/refresh",
"request": {
"method": "POST",
"url": {
"raw": "{{baseUrl}}/api/eventhub/acquisition/yellowfox/d8/imports/configured-plans/{{yellowFoxPlanKey}}/master-data/refresh?mode=INCREMENTAL_UPDATE&strategy=SOURCE_ROW_WATERMARK",
"host": [
"{{baseUrl}}"
],
"path": [
"api",
"eventhub",
"acquisition",
"yellowfox",
"d8",
"imports",
"configured-plans",
"{{yellowFoxPlanKey}}",
"master-data",
"refresh"
],
"query": [
{
"key": "mode",
"value": "INCREMENTAL_UPDATE"
},
{
"key": "strategy",
"value": "SOURCE_ROW_WATERMARK"
}
]
}
}
} }
] ]
}, },

34
src/main/java/at/procon/eventhub/config/EventHubProperties.java
View File

@ -1,7 +1,6 @@
package at.procon.eventhub.config; package at.procon.eventhub.config;
import at.procon.eventhub.esperpoc.dto.EsperActivityMergeMode; import at.procon.eventhub.esperpoc.dto.EsperActivityMergeMode;
import at.procon.eventhub.esperpoc.dto.EsperOperatingPeriodEngineMode;
import at.procon.eventhub.esperpoc.dto.EsperUnknownTreatmentMode; import at.procon.eventhub.esperpoc.dto.EsperUnknownTreatmentMode;
import at.procon.eventhub.esperpoc.dto.EsperShiftResolutionMode; import at.procon.eventhub.esperpoc.dto.EsperShiftResolutionMode;
import java.time.Duration; import java.time.Duration;
@ -81,7 +80,6 @@ public class EventHubProperties {
private int mergeGapSeconds = 0; private int mergeGapSeconds = 0;
private int gapDetectionToleranceSeconds = 0; private int gapDetectionToleranceSeconds = 0;
private EsperUnknownTreatmentMode unknownTreatmentMode = EsperUnknownTreatmentMode.AS_BREAK_REST; private EsperUnknownTreatmentMode unknownTreatmentMode = EsperUnknownTreatmentMode.AS_BREAK_REST;
private EsperOperatingPeriodEngineMode engineMode = EsperOperatingPeriodEngineMode.STREAM_COLLECTOR;
public int getOperatingSplitIdleHours() { public int getOperatingSplitIdleHours() {
return operatingSplitIdleHours; return operatingSplitIdleHours;
@ -124,16 +122,6 @@ public class EventHubProperties {
? EsperUnknownTreatmentMode.AS_BREAK_REST ? EsperUnknownTreatmentMode.AS_BREAK_REST
: unknownTreatmentMode; : unknownTreatmentMode;
} }
public EsperOperatingPeriodEngineMode getEngineMode() {
return engineMode;
}
public void setEngineMode(EsperOperatingPeriodEngineMode engineMode) {
this.engineMode = engineMode == null
? EsperOperatingPeriodEngineMode.STREAM_COLLECTOR
: engineMode;
}
} }
public static class Batch { public static class Batch {
@ -224,9 +212,6 @@ public class EventHubProperties {
/** How JDBC extraction batches are handed over to the ingest pipeline. */ /** How JDBC extraction batches are handed over to the ingest pipeline. */
private JdbcExtractionIngestMode jdbcExtractionIngestMode = JdbcExtractionIngestMode.SYNC_DIRECT; private JdbcExtractionIngestMode jdbcExtractionIngestMode = JdbcExtractionIngestMode.SYNC_DIRECT;
/** Whether master-data refresh reconciles vehicle registrations and assignments. */
private boolean syncVehicleRegistrationsOnMasterDataUpdate = true;
/** Configured tenant/source import plans. */ /** Configured tenant/source import plans. */
private List<ConfiguredImportPlan> importPlans = new ArrayList<>(); private List<ConfiguredImportPlan> importPlans = new ArrayList<>();
@ -283,14 +268,6 @@ public class EventHubProperties {
: jdbcExtractionIngestMode; : jdbcExtractionIngestMode;
} }
public boolean isSyncVehicleRegistrationsOnMasterDataUpdate() {
return syncVehicleRegistrationsOnMasterDataUpdate;
}
public void setSyncVehicleRegistrationsOnMasterDataUpdate(boolean syncVehicleRegistrationsOnMasterDataUpdate) {
this.syncVehicleRegistrationsOnMasterDataUpdate = syncVehicleRegistrationsOnMasterDataUpdate;
}
public List<ConfiguredImportPlan> getImportPlans() { public List<ConfiguredImportPlan> getImportPlans() {
return importPlans; return importPlans;
} }
@ -365,9 +342,6 @@ public class EventHubProperties {
/** Emit a first ignition snapshot per vehicle if no previous D8 ignition state exists in the imported window. */ /** Emit a first ignition snapshot per vehicle if no previous D8 ignition state exists in the imported window. */
private boolean emitInitialIgnitionSnapshot = false; private boolean emitInitialIgnitionSnapshot = false;
/** Whether master-data refresh reconciles vehicle registrations and assignments. */
private boolean syncVehicleRegistrationsOnMasterDataUpdate = true;
/** Configured tenant/source import plans. */ /** Configured tenant/source import plans. */
private List<ConfiguredImportPlan> importPlans = new ArrayList<>(); private List<ConfiguredImportPlan> importPlans = new ArrayList<>();
@ -426,14 +400,6 @@ public class EventHubProperties {
this.emitInitialIgnitionSnapshot = emitInitialIgnitionSnapshot; this.emitInitialIgnitionSnapshot = emitInitialIgnitionSnapshot;
} }
public boolean isSyncVehicleRegistrationsOnMasterDataUpdate() {
return syncVehicleRegistrationsOnMasterDataUpdate;
}
public void setSyncVehicleRegistrationsOnMasterDataUpdate(boolean syncVehicleRegistrationsOnMasterDataUpdate) {
this.syncVehicleRegistrationsOnMasterDataUpdate = syncVehicleRegistrationsOnMasterDataUpdate;
}
public List<ConfiguredImportPlan> getImportPlans() { public List<ConfiguredImportPlan> getImportPlans() {
return importPlans; return importPlans;
} }

7
src/main/java/at/procon/eventhub/esperpoc/api/EsperPocController.java
View File

@ -1,7 +1,6 @@
package at.procon.eventhub.esperpoc.api; package at.procon.eventhub.esperpoc.api;
import at.procon.eventhub.esperpoc.dto.EsperActivityMergeMode; import at.procon.eventhub.esperpoc.dto.EsperActivityMergeMode;
import at.procon.eventhub.esperpoc.dto.EsperOperatingPeriodEngineMode;
import at.procon.eventhub.esperpoc.dto.EsperOperatingPeriodRequest; import at.procon.eventhub.esperpoc.dto.EsperOperatingPeriodRequest;
import at.procon.eventhub.esperpoc.dto.EsperOperatingPeriodResultDto; import at.procon.eventhub.esperpoc.dto.EsperOperatingPeriodResultDto;
import at.procon.eventhub.esperpoc.dto.EsperPocRequest; import at.procon.eventhub.esperpoc.dto.EsperPocRequest;
@ -77,8 +76,7 @@ public class EsperPocController {
@RequestParam(required = false) Integer significantDrivingMinutes, @RequestParam(required = false) Integer significantDrivingMinutes,
@RequestParam(required = false) Integer mergeGapSeconds, @RequestParam(required = false) Integer mergeGapSeconds,
@RequestParam(required = false) Integer gapDetectionToleranceSeconds, @RequestParam(required = false) Integer gapDetectionToleranceSeconds,
@RequestParam(required = false) EsperUnknownTreatmentMode unknownTreatmentMode, @RequestParam(required = false) EsperUnknownTreatmentMode unknownTreatmentMode
@RequestParam(required = false) EsperOperatingPeriodEngineMode engineMode
) { ) {
EsperOperatingPeriodRequest request = new EsperOperatingPeriodRequest( EsperOperatingPeriodRequest request = new EsperOperatingPeriodRequest(
tenantKey, tenantKey,
@ -90,8 +88,7 @@ public class EsperPocController {
significantDrivingMinutes, significantDrivingMinutes,
mergeGapSeconds, mergeGapSeconds,
gapDetectionToleranceSeconds, gapDetectionToleranceSeconds,
unknownTreatmentMode, unknownTreatmentMode
engineMode
); );
return ResponseEntity.ok(operatingPeriodEvaluationService.evaluate(request)); return ResponseEntity.ok(operatingPeriodEvaluationService.evaluate(request));
} }

6
src/main/java/at/procon/eventhub/esperpoc/dto/EsperOperatingPeriodEngineMode.java
View File

@ -1,6 +0,0 @@
package at.procon.eventhub.esperpoc.dto;
public enum EsperOperatingPeriodEngineMode {
STREAM_COLLECTOR,
FULL_EPL
}

3
src/main/java/at/procon/eventhub/esperpoc/dto/EsperOperatingPeriodRequest.java
View File

@ -15,8 +15,7 @@ public record EsperOperatingPeriodRequest(
Integer significantDrivingMinutes, Integer significantDrivingMinutes,
Integer mergeGapSeconds, Integer mergeGapSeconds,
Integer gapDetectionToleranceSeconds, Integer gapDetectionToleranceSeconds,
EsperUnknownTreatmentMode unknownTreatmentMode, EsperUnknownTreatmentMode unknownTreatmentMode
EsperOperatingPeriodEngineMode engineMode
) { ) {
public EsperOperatingPeriodRequest { public EsperOperatingPeriodRequest {
if (occurredFrom != null && occurredTo != null && !occurredFrom.isBefore(occurredTo)) { if (occurredFrom != null && occurredTo != null && !occurredFrom.isBefore(occurredTo)) {

1
src/main/java/at/procon/eventhub/esperpoc/dto/EsperOperatingPeriodResultDto.java
View File

@ -27,7 +27,6 @@ public record EsperOperatingPeriodResultDto(
int mergeGapSeconds, int mergeGapSeconds,
int gapDetectionToleranceSeconds, int gapDetectionToleranceSeconds,
EsperUnknownTreatmentMode unknownTreatmentMode, EsperUnknownTreatmentMode unknownTreatmentMode,
EsperOperatingPeriodEngineMode engineMode,
List<RawActivityEventDto> rawEvents, List<RawActivityEventDto> rawEvents,
List<ActivityIntervalDto> resolvedKnownIntervals, List<ActivityIntervalDto> resolvedKnownIntervals,
List<ActivityIntervalDto> evaluationIntervals, List<ActivityIntervalDto> evaluationIntervals,

203
src/main/java/at/procon/eventhub/esperpoc/service/EsperOperatingPeriodEngine.java
View File

@ -1,7 +1,6 @@
package at.procon.eventhub.esperpoc.service; package at.procon.eventhub.esperpoc.service;
import at.procon.eventhub.esperpoc.dto.ActivityIntervalDto; import at.procon.eventhub.esperpoc.dto.ActivityIntervalDto;
import at.procon.eventhub.esperpoc.dto.EsperOperatingPeriodEngineMode;
import at.procon.eventhub.esperpoc.dto.OperatingPeriodActivityIntervalDto; import at.procon.eventhub.esperpoc.dto.OperatingPeriodActivityIntervalDto;
import com.espertech.esper.common.client.EPCompiled; import com.espertech.esper.common.client.EPCompiled;
import com.espertech.esper.common.client.EventBean; import com.espertech.esper.common.client.EventBean;
@ -13,66 +12,37 @@ import com.espertech.esper.runtime.client.EPDeployException;
import com.espertech.esper.runtime.client.EPDeployment; import com.espertech.esper.runtime.client.EPDeployment;
import com.espertech.esper.runtime.client.EPRuntime; import com.espertech.esper.runtime.client.EPRuntime;
import com.espertech.esper.runtime.client.EPRuntimeProvider; import com.espertech.esper.runtime.client.EPRuntimeProvider;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.time.Instant;
import java.time.Duration; import java.time.Duration;
import java.time.OffsetDateTime; import java.time.OffsetDateTime;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Comparator; import java.util.Comparator;
import java.util.LinkedHashMap;
import java.util.List; import java.util.List;
import java.util.Map;
import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicLong;
import org.springframework.core.io.ClassPathResource;
import org.springframework.stereotype.Component; import org.springframework.stereotype.Component;
import org.springframework.util.StreamUtils;
@Component @Component
public class EsperOperatingPeriodEngine { public class EsperOperatingPeriodEngine {
private static final AtomicLong RUNTIME_COUNTER = new AtomicLong(); private static final AtomicLong RUNTIME_COUNTER = new AtomicLong();
// Minimal stream-only mode: Esper preserves event ordering, Java owns the period state machine.
private static final String INPUT_STREAM_EPL = """ private static final String INPUT_STREAM_EPL = """
@name('operatingPeriodIntervalStream') @name('operatingPeriodIntervalStream')
select * from OperatingPeriodIntervalInputEvent select * from OperatingPeriodIntervalInputEvent
"""; """;
// Full-EPL mode: Esper owns the operating-period state machine and emits periodized intervals/closures.
private static final String FULL_EPL_TEMPLATE = loadResource("esper/operating-period-state-machine.epl");
public EsperOperatingPeriodEvaluation evaluate( public EsperOperatingPeriodEvaluation evaluate(
List<ActivityIntervalDto> intervals, List<ActivityIntervalDto> intervals,
Duration operatingSplitIdleThreshold, Duration operatingSplitIdleThreshold
EsperOperatingPeriodEngineMode mode
) { ) {
List<ActivityIntervalDto> sorted = sortedPositiveIntervals(intervals); List<ActivityIntervalDto> sorted = sortedPositiveIntervals(intervals);
if (sorted.isEmpty()) { if (sorted.isEmpty()) {
return new EsperOperatingPeriodEvaluation(List.of(), List.of()); return new EsperOperatingPeriodEvaluation(List.of(), List.of());
} }
if (mode == EsperOperatingPeriodEngineMode.FULL_EPL) {
return evaluateFullEpl(sorted, operatingSplitIdleThreshold);
}
return evaluateStreamCollector(sorted, operatingSplitIdleThreshold);
}
public EsperOperatingPeriodEvaluation evaluate(
List<ActivityIntervalDto> intervals,
Duration operatingSplitIdleThreshold
) {
return evaluate(intervals, operatingSplitIdleThreshold, EsperOperatingPeriodEngineMode.STREAM_COLLECTOR);
}
private EsperOperatingPeriodEvaluation evaluateStreamCollector(
List<ActivityIntervalDto> sorted,
Duration operatingSplitIdleThreshold
) {
// In stream-collector mode Esper only serializes the stream; period transitions are evaluated in Java.
PeriodizationCollector collector = new PeriodizationCollector(operatingSplitIdleThreshold); PeriodizationCollector collector = new PeriodizationCollector(operatingSplitIdleThreshold);
executeWithRuntime( executeWithRuntime(
configuration -> configuration.getCommon().addEventType("OperatingPeriodIntervalInputEvent", EsperOperatingPeriodIntervalInputEvent.class), configuration -> configuration.getCommon().addEventType("OperatingPeriodIntervalInputEvent", EsperOperatingPeriodIntervalInputEvent.class),
INPUT_STREAM_EPL, INPUT_STREAM_EPL,
List.of("operatingPeriodIntervalStream"), "operatingPeriodIntervalStream",
(statementName, newData) -> collectInputIntervals(newData, collector), newData -> collectInputIntervals(newData, collector),
runtime -> { runtime -> {
for (ActivityIntervalDto interval : sorted) { for (ActivityIntervalDto interval : sorted) {
runtime.getEventService().sendEventBean(toInputEvent(interval), "OperatingPeriodIntervalInputEvent"); runtime.getEventService().sendEventBean(toInputEvent(interval), "OperatingPeriodIntervalInputEvent");
@ -82,72 +52,11 @@ public class EsperOperatingPeriodEngine {
return collector.finish(); return collector.finish();
} }
private EsperOperatingPeriodEvaluation evaluateFullEpl(
List<ActivityIntervalDto> sorted,
Duration operatingSplitIdleThreshold
) {
// The full-EPL script is parameterized per request so the idle-split threshold matches the request/config.
String epl = FULL_EPL_TEMPLATE.replace("${operatingSplitIdleMs}", Long.toString(operatingSplitIdleThreshold.toMillis()));
List<OperatingPeriodActivityIntervalDto> periodizedIntervals = new ArrayList<>();
List<EsperClosedOperatingPeriod> closedPeriods = new ArrayList<>();
executeWithRuntime(
configuration -> {
// Full-EPL mode uses explicit map schemas so EPL can own the whole state machine without
// relying on Java bean-property resolution during compilation.
Map<String, Object> inputDefinition = new LinkedHashMap<>();
inputDefinition.put("driverId", java.util.UUID.class);
inputDefinition.put("vehicleId", java.util.UUID.class);
inputDefinition.put("vehicleRegistrationId", java.util.UUID.class);
inputDefinition.put("activityType", String.class);
inputDefinition.put("cardSlot", String.class);
inputDefinition.put("cardStatus", String.class);
inputDefinition.put("drivingStatus", String.class);
inputDefinition.put("sourceKind", String.class);
inputDefinition.put("startTs", long.class);
inputDefinition.put("endTs", long.class);
inputDefinition.put("durationMs", long.class);
inputDefinition.put("sourceRowId", String.class);
inputDefinition.put("sourceRowIds", java.util.List.class);
inputDefinition.put("clippedToRequestedPeriod", boolean.class);
inputDefinition.put("synthetic", boolean.class);
configuration.getCommon().addEventType("OperatingPeriodInputMap", inputDefinition);
configuration.getCommon().addEventType("OperatingPeriodFlushEvent", Map.of("reason", String.class));
},
epl,
List.of("periodizedActivityIntervals", "operatingPeriodClosed"),
(statementName, newData) -> {
if ("periodizedActivityIntervals".equals(statementName)) {
collectPeriodizedOutputs(newData, periodizedIntervals);
} else if ("operatingPeriodClosed".equals(statementName)) {
collectClosedOutputs(newData, closedPeriods);
}
},
runtime -> {
// Historical evaluation sends the complete interval timeline first and then a single flush event
// so the EPL state machine can emit the final still-open operating period.
for (ActivityIntervalDto interval : sorted) {
runtime.getEventService().sendEventMap(toInputMap(interval), "OperatingPeriodInputMap");
}
runtime.getEventService().sendEventMap(Map.of("reason", "HISTORICAL_EVALUATION"), "OperatingPeriodFlushEvent");
}
);
return new EsperOperatingPeriodEvaluation(
periodizedIntervals.stream()
.sorted(Comparator.comparing(OperatingPeriodActivityIntervalDto::startedAt)
.thenComparing(OperatingPeriodActivityIntervalDto::endedAt)
.thenComparing(OperatingPeriodActivityIntervalDto::activityType, Comparator.nullsLast(String::compareTo)))
.toList(),
closedPeriods.stream()
.sorted(Comparator.comparing(EsperClosedOperatingPeriod::startedAt))
.toList()
);
}
private void executeWithRuntime( private void executeWithRuntime(
java.util.function.Consumer<Configuration> configurationSetup, java.util.function.Consumer<Configuration> configurationSetup,
String epl, String epl,
List<String> statementNames, String statementName,
StatementListener listener, java.util.function.Consumer<EventBean[]> listener,
java.util.function.Consumer<EPRuntime> sender java.util.function.Consumer<EPRuntime> sender
) { ) {
EPRuntime runtime = null; EPRuntime runtime = null;
@ -160,12 +69,9 @@ public class EsperOperatingPeriodEngine {
CompilerArguments arguments = new CompilerArguments(configuration); CompilerArguments arguments = new CompilerArguments(configuration);
EPCompiled compiled = EPCompilerProvider.getCompiler().compile(epl, arguments); EPCompiled compiled = EPCompilerProvider.getCompiler().compile(epl, arguments);
EPDeployment deployment = runtime.getDeploymentService().deploy(compiled); EPDeployment deployment = runtime.getDeploymentService().deploy(compiled);
// Multiple statements may emit outputs from a single deployment; we dispatch by statement name.
for (String statementName : statementNames) {
runtime.getDeploymentService() runtime.getDeploymentService()
.getStatement(deployment.getDeploymentId(), statementName) .getStatement(deployment.getDeploymentId(), statementName)
.addListener((newData, oldData, statement, rt) -> listener.accept(statementName, newData)); .addListener((newData, oldData, statement, rt) -> listener.accept(newData));
}
sender.accept(runtime); sender.accept(runtime);
} catch (EPCompileException | EPDeployException e) { } catch (EPCompileException | EPDeployException e) {
throw new IllegalStateException("Cannot compile/deploy Esper operating-period EPL", e); throw new IllegalStateException("Cannot compile/deploy Esper operating-period EPL", e);
@ -181,66 +87,10 @@ public class EsperOperatingPeriodEngine {
return; return;
} }
for (EventBean event : newData) { for (EventBean event : newData) {
// Stream-collector mode receives the ordered interval stream back from Esper and applies the
// deterministic Java state machine to it.
collector.accept((EsperOperatingPeriodIntervalInputEvent) event.getUnderlying()); collector.accept((EsperOperatingPeriodIntervalInputEvent) event.getUnderlying());
} }
} }
private void collectPeriodizedOutputs(EventBean[] newData, List<OperatingPeriodActivityIntervalDto> target) {
if (newData == null) {
return;
}
for (EventBean event : newData) {
target.add(new OperatingPeriodActivityIntervalDto(
(java.util.UUID) event.get("driverId"),
(java.util.UUID) event.get("vehicleId"),
(java.util.UUID) event.get("vehicleRegistrationId"),
(String) event.get("activityType"),
(String) event.get("cardSlot"),
(String) event.get("cardStatus"),
(String) event.get("drivingStatus"),
(String) event.get("sourceKind"),
OffsetDateTime.ofInstant(Instant.ofEpochMilli((Long) event.get("startedAtTs")), java.time.ZoneOffset.UTC),
OffsetDateTime.ofInstant(Instant.ofEpochMilli((Long) event.get("endedAtTs")), java.time.ZoneOffset.UTC),
((Long) event.get("durationMs")) / 1000L,
(String) event.get("sourceRowId"),
castSourceRowIds(event.get("sourceRowIds")),
(Boolean) event.get("clippedToRequestedPeriod"),
"PERIODIZED_ACTIVITY",
(Long) event.get("operatingPeriodNo"),
OffsetDateTime.ofInstant(Instant.ofEpochMilli((Long) event.get("operatingPeriodStartedAtTs")), java.time.ZoneOffset.UTC),
(Boolean) event.get("newOperatingPeriod"),
nullableMillisToSeconds((Long) event.get("gapSincePreviousActivityMs")),
(Boolean) event.get("synthetic")
));
}
}
private void collectClosedOutputs(EventBean[] newData, List<EsperClosedOperatingPeriod> target) {
if (newData == null) {
return;
}
for (EventBean event : newData) {
target.add(new EsperClosedOperatingPeriod(
(Long) event.get("operatingPeriodNo"),
OffsetDateTime.ofInstant(Instant.ofEpochMilli((Long) event.get("operatingPeriodStartedAtTs")), java.time.ZoneOffset.UTC),
OffsetDateTime.ofInstant(Instant.ofEpochMilli((Long) event.get("operatingPeriodEndedAtTs")), java.time.ZoneOffset.UTC),
((Long) event.get("durationMs")) / 1000L,
(String) event.get("closedBy")
));
}
}
@SuppressWarnings("unchecked")
private List<String> castSourceRowIds(Object value) {
return value == null ? List.of() : List.copyOf((List<String>) value);
}
private Long nullableMillisToSeconds(Long value) {
return value == null ? null : value / 1000L;
}
private EsperOperatingPeriodIntervalInputEvent toInputEvent(ActivityIntervalDto interval) { private EsperOperatingPeriodIntervalInputEvent toInputEvent(ActivityIntervalDto interval) {
return new EsperOperatingPeriodIntervalInputEvent( return new EsperOperatingPeriodIntervalInputEvent(
interval.driverEntityId(), interval.driverEntityId(),
@ -261,26 +111,6 @@ public class EsperOperatingPeriodEngine {
); );
} }
private Map<String, Object> toInputMap(ActivityIntervalDto interval) {
Map<String, Object> map = new LinkedHashMap<>();
map.put("driverId", interval.driverEntityId());
map.put("vehicleId", interval.vehicleId());
map.put("vehicleRegistrationId", interval.vehicleRegistrationId());
map.put("activityType", interval.activityType());
map.put("cardSlot", interval.cardSlot());
map.put("cardStatus", interval.cardStatus());
map.put("drivingStatus", interval.drivingStatus());
map.put("sourceKind", interval.sourceKind());
map.put("startTs", interval.startedAt().toInstant().toEpochMilli());
map.put("endTs", interval.endedAt().toInstant().toEpochMilli());
map.put("durationMs", interval.durationSeconds() * 1000L);
map.put("sourceRowId", interval.sourceRowId());
map.put("sourceRowIds", interval.sourceRowIds());
map.put("clippedToRequestedPeriod", interval.clippedToRequestedPeriod());
map.put("synthetic", "UNKNOWN_GAP".equals(interval.level()));
return map;
}
private List<ActivityIntervalDto> sortedPositiveIntervals(List<ActivityIntervalDto> intervals) { private List<ActivityIntervalDto> sortedPositiveIntervals(List<ActivityIntervalDto> intervals) {
if (intervals == null || intervals.isEmpty()) { if (intervals == null || intervals.isEmpty()) {
return List.of(); return List.of();
@ -299,18 +129,6 @@ public class EsperOperatingPeriodEngine {
) { ) {
} }
private interface StatementListener {
void accept(String statementName, EventBean[] newData);
}
private static String loadResource(String path) {
try {
return StreamUtils.copyToString(new ClassPathResource(path).getInputStream(), StandardCharsets.UTF_8);
} catch (IOException e) {
throw new IllegalStateException("Cannot load Esper resource " + path, e);
}
}
private static final class PeriodizationCollector { private static final class PeriodizationCollector {
private final Duration operatingSplitIdleThreshold; private final Duration operatingSplitIdleThreshold;
private final List<OperatingPeriodActivityIntervalDto> periodizedIntervals = new ArrayList<>(); private final List<OperatingPeriodActivityIntervalDto> periodizedIntervals = new ArrayList<>();
@ -345,16 +163,12 @@ public class EsperOperatingPeriodEngine {
if ("UNKNOWN".equals(dto.activityType())) { if ("UNKNOWN".equals(dto.activityType())) {
if (!hasOpenPeriod) { if (!hasOpenPeriod) {
// Unknown time before the first known activity does not belong to any operating period.
return; return;
} }
if (dto.durationSeconds() >= operatingSplitIdleThreshold.getSeconds()) { if (dto.durationSeconds() >= operatingSplitIdleThreshold.getSeconds()) {
// Long UNKNOWN behaves like a closing gap: close the current period and wait for the next
// known activity to reopen a new period number.
closeCurrent("UNKNOWN_GAP"); closeCurrent("UNKNOWN_GAP");
return; return;
} }
// Short UNKNOWN stays inside the current period as explicit uncertainty.
periodizedIntervals.add(OperatingPeriodActivityIntervalDto.periodized( periodizedIntervals.add(OperatingPeriodActivityIntervalDto.periodized(
dto, dto,
operatingPeriodNo, operatingPeriodNo,
@ -366,7 +180,6 @@ public class EsperOperatingPeriodEngine {
} }
if (!hasOpenPeriod) { if (!hasOpenPeriod) {
// First known activity, or first activity after a long closing gap, opens a new operating period.
operatingPeriodNo = operatingPeriodNo < 1 ? 1 : operatingPeriodNo + 1; operatingPeriodNo = operatingPeriodNo < 1 ? 1 : operatingPeriodNo + 1;
hasOpenPeriod = true; hasOpenPeriod = true;
operatingPeriodStartedAt = dto.startedAt(); operatingPeriodStartedAt = dto.startedAt();
@ -383,7 +196,6 @@ public class EsperOperatingPeriodEngine {
long gapSeconds = Math.max(0, Duration.between(lastKnownActivityEndAt, dto.startedAt()).getSeconds()); long gapSeconds = Math.max(0, Duration.between(lastKnownActivityEndAt, dto.startedAt()).getSeconds());
if (gapSeconds >= operatingSplitIdleThreshold.getSeconds()) { if (gapSeconds >= operatingSplitIdleThreshold.getSeconds()) {
// Long idle time between known activities closes the current period and starts the next one.
closeCurrent("IDLE_GAP"); closeCurrent("IDLE_GAP");
operatingPeriodNo++; operatingPeriodNo++;
hasOpenPeriod = true; hasOpenPeriod = true;
@ -399,7 +211,6 @@ public class EsperOperatingPeriodEngine {
return; return;
} }
// Normal forward continuity inside the same period.
periodizedIntervals.add(OperatingPeriodActivityIntervalDto.periodized( periodizedIntervals.add(OperatingPeriodActivityIntervalDto.periodized(
dto, dto,
operatingPeriodNo, operatingPeriodNo,
@ -414,7 +225,6 @@ public class EsperOperatingPeriodEngine {
private EsperOperatingPeriodEvaluation finish() { private EsperOperatingPeriodEvaluation finish() {
if (hasOpenPeriod) { if (hasOpenPeriod) {
// Historical evaluation has no future event to close the final period, so emit it explicitly.
closeCurrent("FLUSH"); closeCurrent("FLUSH");
} }
return new EsperOperatingPeriodEvaluation( return new EsperOperatingPeriodEvaluation(
@ -434,7 +244,6 @@ public class EsperOperatingPeriodEngine {
hasOpenPeriod = false; hasOpenPeriod = false;
return; return;
} }
// A closed period always ends at the last known non-rest activity end, never at the synthetic UNKNOWN.
closedPeriods.add(new EsperClosedOperatingPeriod( closedPeriods.add(new EsperClosedOperatingPeriod(
operatingPeriodNo, operatingPeriodNo,
operatingPeriodStartedAt, operatingPeriodStartedAt,

21
src/main/java/at/procon/eventhub/esperpoc/service/EsperOperatingPeriodEvaluationService.java
View File

@ -3,7 +3,6 @@ package at.procon.eventhub.esperpoc.service;
import at.procon.eventhub.config.EventHubProperties; import at.procon.eventhub.config.EventHubProperties;
import at.procon.eventhub.esperpoc.dto.ActivityIntervalDto; import at.procon.eventhub.esperpoc.dto.ActivityIntervalDto;
import at.procon.eventhub.esperpoc.dto.DrivingInterruptionDto; import at.procon.eventhub.esperpoc.dto.DrivingInterruptionDto;
import at.procon.eventhub.esperpoc.dto.EsperOperatingPeriodEngineMode;
import at.procon.eventhub.esperpoc.dto.EsperOperatingPeriodRequest; import at.procon.eventhub.esperpoc.dto.EsperOperatingPeriodRequest;
import at.procon.eventhub.esperpoc.dto.EsperOperatingPeriodResultDto; import at.procon.eventhub.esperpoc.dto.EsperOperatingPeriodResultDto;
import at.procon.eventhub.esperpoc.dto.EsperUnknownTreatmentMode; import at.procon.eventhub.esperpoc.dto.EsperUnknownTreatmentMode;
@ -69,7 +68,6 @@ public class EsperOperatingPeriodEvaluationService {
Duration mergeGapTolerance = Duration.ofSeconds(resolveMergeGapSeconds(request)); Duration mergeGapTolerance = Duration.ofSeconds(resolveMergeGapSeconds(request));
Duration gapDetectionTolerance = Duration.ofSeconds(resolveGapDetectionToleranceSeconds(request)); Duration gapDetectionTolerance = Duration.ofSeconds(resolveGapDetectionToleranceSeconds(request));
EsperUnknownTreatmentMode unknownTreatmentMode = resolveUnknownTreatmentMode(request); EsperUnknownTreatmentMode unknownTreatmentMode = resolveUnknownTreatmentMode(request);
EsperOperatingPeriodEngineMode engineMode = resolveEngineMode(request);
long dbStartedNanos = System.nanoTime(); long dbStartedNanos = System.nanoTime();
List<RawActivityEventDto> rawEvents = activityRepository.findDriverActivityEvents( List<RawActivityEventDto> rawEvents = activityRepository.findDriverActivityEvents(
@ -107,8 +105,7 @@ public class EsperOperatingPeriodEvaluationService {
long periodizeStartedNanos = System.nanoTime(); long periodizeStartedNanos = System.nanoTime();
EsperOperatingPeriodEngine.EsperOperatingPeriodEvaluation evaluation = operatingPeriodEngine.evaluate( EsperOperatingPeriodEngine.EsperOperatingPeriodEvaluation evaluation = operatingPeriodEngine.evaluate(
evaluationLoadedIntervals, evaluationLoadedIntervals,
splitIdleThreshold, splitIdleThreshold
engineMode
); );
long periodizeElapsedMs = elapsedMillis(periodizeStartedNanos); long periodizeElapsedMs = elapsedMillis(periodizeStartedNanos);
@ -153,7 +150,7 @@ public class EsperOperatingPeriodEvaluationService {
); );
long totalElapsedMs = elapsedMillis(startedNanos); long totalElapsedMs = elapsedMillis(startedNanos);
log.info("Esper operating-period evaluation tenant={} driverId={} requestedFrom={} requestedTo={} loadedFrom={} loadedTo={} unknownMode={} engineMode={} rawEvents={} cardRawEvents={} vuRawEvents={} cardIntervals={} vuIntervals={} resolvedKnownIntervals={} evaluationIntervals={} periodizedIntervals={} mergedIntervals={} nonDrivingIntervals={} operatingPeriods={} timingsMs={{dbRetrieve={}, cardIntervalEsper={}, vuIntervalEsper={}, vuGapFill={}, synthUnknown={}, periodizeEsper={}, merge={}, nonDriving={}, total={}}}", log.info("Esper operating-period evaluation tenant={} driverId={} requestedFrom={} requestedTo={} loadedFrom={} loadedTo={} unknownMode={} rawEvents={} cardRawEvents={} vuRawEvents={} cardIntervals={} vuIntervals={} resolvedKnownIntervals={} evaluationIntervals={} periodizedIntervals={} mergedIntervals={} nonDrivingIntervals={} operatingPeriods={} timingsMs={{dbRetrieve={}, cardIntervalEsper={}, vuIntervalEsper={}, vuGapFill={}, synthUnknown={}, periodizeEsper={}, merge={}, nonDriving={}, total={}}}",
request.tenantKey(), request.tenantKey(),
request.driverId(), request.driverId(),
requestedFrom, requestedFrom,
@ -161,7 +158,6 @@ public class EsperOperatingPeriodEvaluationService {
loadedFrom, loadedFrom,
loadedTo, loadedTo,
unknownTreatmentMode, unknownTreatmentMode,
engineMode,
rawEvents.size(), rawEvents.size(),
driverCardRawEvents.size(), driverCardRawEvents.size(),
vehicleUnitRawEvents.size(), vehicleUnitRawEvents.size(),
@ -206,7 +202,6 @@ public class EsperOperatingPeriodEvaluationService {
resolveMergeGapSeconds(request), resolveMergeGapSeconds(request),
resolveGapDetectionToleranceSeconds(request), resolveGapDetectionToleranceSeconds(request),
unknownTreatmentMode, unknownTreatmentMode,
engineMode,
rawEvents, rawEvents,
resolvedKnownLoadedIntervals, resolvedKnownLoadedIntervals,
evaluationLoadedIntervals, evaluationLoadedIntervals,
@ -215,7 +210,6 @@ public class EsperOperatingPeriodEvaluationService {
nonDrivingIntervals, nonDrivingIntervals,
operatingPeriods, operatingPeriods,
notes( notes(
engineMode,
unknownTreatmentMode, unknownTreatmentMode,
resolveOperatingSplitIdleHours(request), resolveOperatingSplitIdleHours(request),
resolveSignificantDrivingMinutes(request), resolveSignificantDrivingMinutes(request),
@ -714,17 +708,7 @@ public class EsperOperatingPeriodEvaluationService {
: properties.getEsperPoc().getOperatingPeriodEvaluation().getUnknownTreatmentMode(); : properties.getEsperPoc().getOperatingPeriodEvaluation().getUnknownTreatmentMode();
} }
private EsperOperatingPeriodEngineMode resolveEngineMode(EsperOperatingPeriodRequest request) {
if (request.engineMode() != null) {
return request.engineMode();
}
return properties == null
? EsperOperatingPeriodEngineMode.STREAM_COLLECTOR
: properties.getEsperPoc().getOperatingPeriodEvaluation().getEngineMode();
}
private List<String> notes( private List<String> notes(
EsperOperatingPeriodEngineMode engineMode,
EsperUnknownTreatmentMode unknownTreatmentMode, EsperUnknownTreatmentMode unknownTreatmentMode,
int operatingSplitIdleHours, int operatingSplitIdleHours,
int significantDrivingMinutes, int significantDrivingMinutes,
@ -735,7 +719,6 @@ public class EsperOperatingPeriodEvaluationService {
"BREAK_REST events are ignored for activity evaluation but still prevent synthetic UNKNOWN intervals from being created over covered rest spans.", "BREAK_REST events are ignored for activity evaluation but still prevent synthetic UNKNOWN intervals from being created over covered rest spans.",
"Synthetic UNKNOWN intervals are created only for uncovered gaps between non-rest activities.", "Synthetic UNKNOWN intervals are created only for uncovered gaps between non-rest activities.",
"UNKNOWN treatment mode is " + unknownTreatmentMode + ".", "UNKNOWN treatment mode is " + unknownTreatmentMode + ".",
"Operating-period engine mode is " + engineMode + ".",
"Operating periods split after " + operatingSplitIdleHours + " hours of no non-rest activity; significant driving closes non-driving intervals from " + significantDrivingMinutes + " minutes onward.", "Operating periods split after " + operatingSplitIdleHours + " hours of no non-rest activity; significant driving closes non-driving intervals from " + significantDrivingMinutes + " minutes onward.",
"Synthetic UNKNOWN gaps are only emitted when uncovered time exceeds " + gapDetectionToleranceSeconds + " seconds." "Synthetic UNKNOWN gaps are only emitted when uncovered time exceeds " + gapDetectionToleranceSeconds + " seconds."
); );

27
src/main/java/at/procon/eventhub/importing/AbstractConfiguredImportPlanService.java
View File

@ -64,15 +64,10 @@ public abstract class AbstractConfiguredImportPlanService<R extends ImportRunReq
AcquisitionStrategy strategy = strategyOverride == null AcquisitionStrategy strategy = strategyOverride == null
? (mode == ImportMode.INCREMENTAL_UPDATE ? plan.getScheduledStrategy() : plan.getInitialStrategy()) ? (mode == ImportMode.INCREMENTAL_UPDATE ? plan.getScheduledStrategy() : plan.getInitialStrategy())
: strategyOverride; : strategyOverride;
return buildRequest(plan, mode, strategy, scopedForRequest(plan, mode, strategy, applyInitialOccurredWindow)); return buildRequest(plan, mode, strategy, scopedForRequest(plan, applyInitialOccurredWindow));
} }
private ImportScopeDto scopedForRequest( private ImportScopeDto scopedForRequest(EventHubProperties.ConfiguredImportPlan plan, boolean applyInitialOccurredWindow) {
EventHubProperties.ConfiguredImportPlan plan,
ImportMode mode,
AcquisitionStrategy strategy,
boolean applyInitialOccurredWindow
) {
ImportScopeDto scope = plan.getImportScope(); ImportScopeDto scope = plan.getImportScope();
if (applyInitialOccurredWindow && scope != null if (applyInitialOccurredWindow && scope != null
&& (plan.getInitialOccurredFrom() != null || plan.getInitialOccurredTo() != null)) { && (plan.getInitialOccurredFrom() != null || plan.getInitialOccurredTo() != null)) {
@ -84,27 +79,9 @@ public abstract class AbstractConfiguredImportPlanService<R extends ImportRunReq
plan.getInitialOccurredTo() == null ? scope.occurredTo() : plan.getInitialOccurredTo() plan.getInitialOccurredTo() == null ? scope.occurredTo() : plan.getInitialOccurredTo()
); );
} }
if (mode == ImportMode.INCREMENTAL_UPDATE
&& isWatermarkStrategy(strategy)
&& scope != null
&& scope.occurredFrom() == null
&& plan.getInitialOccurredFrom() != null) {
return new ImportScopeDto(
scope.type(),
scope.rootSourceOrganisation(),
scope.includeChildren(),
plan.getInitialOccurredFrom(),
scope.occurredTo() == null ? plan.getInitialOccurredTo() : scope.occurredTo()
);
}
return scope; return scope;
} }
private boolean isWatermarkStrategy(AcquisitionStrategy strategy) {
return strategy == AcquisitionStrategy.SOURCE_PACKAGE_WATERMARK
|| strategy == AcquisitionStrategy.SOURCE_ROW_WATERMARK;
}
private D planByKey(String planKey) { private D planByKey(String planKey) {
return toDto(rawPlanByKey(planKey)); return toDto(rawPlanByKey(planKey));
} }

7
src/main/java/at/procon/eventhub/importing/ImportChunkPlanner.java
View File

@ -17,7 +17,7 @@ public class ImportChunkPlanner {
OffsetDateTime to = scope == null ? null : scope.occurredTo(); OffsetDateTime to = scope == null ? null : scope.occurredTo();
if (request.mode() == ImportMode.INCREMENTAL_UPDATE if (request.mode() == ImportMode.INCREMENTAL_UPDATE
&& isWatermarkStrategy(request.acquisitionStrategy())) { && request.acquisitionStrategy() == AcquisitionStrategy.SOURCE_PACKAGE_WATERMARK) {
return List.of(new ImportTimeChunkDto(1, from, to)); return List.of(new ImportTimeChunkDto(1, from, to));
} }
@ -39,9 +39,4 @@ public class ImportChunkPlanner {
} }
return chunks.isEmpty() ? List.of(new ImportTimeChunkDto(1, from, to)) : chunks; return chunks.isEmpty() ? List.of(new ImportTimeChunkDto(1, from, to)) : chunks;
} }
private boolean isWatermarkStrategy(AcquisitionStrategy strategy) {
return strategy == AcquisitionStrategy.SOURCE_PACKAGE_WATERMARK
|| strategy == AcquisitionStrategy.SOURCE_ROW_WATERMARK;
}
} }

38
src/main/java/at/procon/eventhub/importing/extraction/AbstractJdbcExtractionBatchExecutor.java
View File

@ -88,7 +88,15 @@ public abstract class AbstractJdbcExtractionBatchExecutor<R extends ImportRunReq
packageInfo packageInfo
); );
ImportCursorStateDto cursor = findCursor(eventSourceId, request, planItem); String scopeHash = request.importScope() == null ? "NO_SCOPE" : request.importScope().stableKey();
ImportCursorStateDto cursor = importCursorRepository.findCursor(
request.tenantKey(),
eventSourceId,
scopeHash,
planItem.eventFamily(),
planItem.sourceKind(),
request.acquisitionStrategy()
);
Map<String, Object> params = parameters(request, chunkScope, cursor); Map<String, Object> params = parameters(request, chunkScope, cursor);
String sql = loadSql(definition.sqlResource()); String sql = loadSql(definition.sqlResource());
@ -282,34 +290,6 @@ public abstract class AbstractJdbcExtractionBatchExecutor<R extends ImportRunReq
return params; return params;
} }
protected ImportCursorStateDto findCursor(int eventSourceId, R request, ImportPlanItemDto planItem) {
String scopeHash = request.importScope() == null ? "NO_SCOPE" : request.importScope().stableKey();
ImportCursorStateDto cursor = importCursorRepository.findCursor(
request.tenantKey(),
eventSourceId,
scopeHash,
planItem.eventFamily(),
planItem.sourceKind(),
request.acquisitionStrategy()
);
if (cursor != null || !shouldBootstrapWatermarkCursor(request)) {
return cursor;
}
return importCursorRepository.findLatestCursor(
request.tenantKey(),
eventSourceId,
scopeHash,
planItem.eventFamily(),
planItem.sourceKind()
);
}
private boolean shouldBootstrapWatermarkCursor(R request) {
return request.mode() == at.procon.eventhub.dto.ImportMode.INCREMENTAL_UPDATE
&& (request.acquisitionStrategy() == AcquisitionStrategy.SOURCE_ROW_WATERMARK
|| request.acquisitionStrategy() == AcquisitionStrategy.SOURCE_PACKAGE_WATERMARK);
}
protected OffsetDateTime lastSourcePackageImportedAt(ExtractedEventStats stats, ImportCursorStateDto cursor) { protected OffsetDateTime lastSourcePackageImportedAt(ExtractedEventStats stats, ImportCursorStateDto cursor) {
return stats.lastSourcePackageImportedAt() == null return stats.lastSourcePackageImportedAt() == null
? cursor == null ? null : cursor.lastSourcePackageImportedAt() ? cursor == null ? null : cursor.lastSourcePackageImportedAt()

39
src/main/java/at/procon/eventhub/importing/persistence/ImportCursorRepository.java
View File

@ -56,45 +56,6 @@ public class ImportCursorRepository {
); );
} }
public ImportCursorStateDto findLatestCursor(
String tenantKey,
int eventSourceId,
String scopeHash,
EventFamily eventFamily,
String sourceKind
) {
return jdbcTemplate.query(
"""
select last_source_package_imported_at,
last_source_package_id,
last_source_row_updated_at,
last_occurred_to
from eventhub.import_cursor
where tenant_key = ?
and event_source_id = ?
and scope_hash = ?
and event_family = ?
and source_kind = ?
order by coalesce(last_occurred_to, last_source_row_updated_at, last_source_package_imported_at) desc nulls last,
updated_at desc
limit 1
""",
rs -> rs.next()
? new ImportCursorStateDto(
rs.getObject("last_source_package_imported_at", java.time.OffsetDateTime.class),
rs.getString("last_source_package_id"),
rs.getObject("last_source_row_updated_at", java.time.OffsetDateTime.class),
rs.getObject("last_occurred_to", java.time.OffsetDateTime.class)
)
: null,
tenantKey,
eventSourceId,
scopeHash,
eventFamily.name(),
sourceKind
);
}
public void advanceCursor( public void advanceCursor(
String tenantKey, String tenantKey,
int eventSourceId, int eventSourceId,

821
src/main/java/at/procon/eventhub/persistence/DriverIdentityRepository.java
View File

@ -4,7 +4,6 @@ import at.procon.eventhub.dto.DriverCardRefDto;
import at.procon.eventhub.dto.DriverRefDto; import at.procon.eventhub.dto.DriverRefDto;
import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectMapper;
import java.time.LocalDate;
import java.time.OffsetDateTime; import java.time.OffsetDateTime;
import java.util.LinkedHashMap; import java.util.LinkedHashMap;
import java.util.Map; import java.util.Map;
@ -16,9 +15,6 @@ import org.springframework.transaction.annotation.Transactional;
@Repository @Repository
public class DriverIdentityRepository { public class DriverIdentityRepository {
private static final String YELLOWFOX_SYNTHETIC_REFERENCE_NATION = "YELLOWFOX";
private static final String UNKNOWN_CARD_NATION = "UNKNOWN";
private final JdbcTemplate jdbcTemplate; private final JdbcTemplate jdbcTemplate;
private final ObjectMapper objectMapper; private final ObjectMapper objectMapper;
@ -27,36 +23,34 @@ public class DriverIdentityRepository {
this.objectMapper = objectMapper; this.objectMapper = objectMapper;
} }
public ResolvedDriverReference resolveOrCreateDriverReference( public UUID resolveOrCreateDriverId(
String tenantKey, String tenantKey,
int eventSourceId, int eventSourceId,
DriverRefDto driverRef DriverRefDto driverRef
) { ) {
if (driverRef == null || !driverRef.hasAnyReference()) { if (driverRef == null || !driverRef.hasAnyReference()) {
return ResolvedDriverReference.empty(); return null;
} }
String normalizedTenantKey = normalizeRequired(tenantKey, "tenantKey"); String normalizedTenantKey = normalizeRequired(tenantKey, "tenantKey");
String sourceDriverEntityId = normalizeNullable(driverRef.sourceEntityId()); String sourceDriverEntityId = normalizeNullable(driverRef.sourceEntityId());
DriverCardRefDto driverCard = driverRef.driverCard(); DriverCardRefDto driverCard = driverRef.driverCard();
String cardNation = driverCard == null ? null : normalizeNullable(driverCard.nation()); String cardNation = driverCard == null ? null : normalizeNullable(driverCard.nation());
String cardNumber = driverCard == null ? null : normalizeDriverCardNumber(cardNation, driverCard.number()); String cardNumber = driverCard == null ? null : normalizeNullable(driverCard.number());
UUID driverId = findBySourceDriverEntityId(normalizedTenantKey, eventSourceId, sourceDriverEntityId); UUID driverId = resolveDriverId(normalizedTenantKey, eventSourceId, sourceDriverEntityId, cardNation, cardNumber);
UUID driverCardId = resolveOrCreateDriverCardId(cardNation, cardNumber, driverId); if (driverId == null) {
if (driverId == null && driverCardId != null) {
driverId = findDriverIdByCardId(driverCardId);
}
if (driverId == null && sourceDriverEntityId != null) {
Map<String, Object> payload = new LinkedHashMap<>(); Map<String, Object> payload = new LinkedHashMap<>();
put(payload, "source", "event"); put(payload, "source", "event");
put(payload, "source_driver_entity_id", sourceDriverEntityId); put(payload, "source_driver_entity_id", sourceDriverEntityId);
put(payload, "card_nation", cardNation);
put(payload, "card_number", cardNumber);
driverId = createDriver( driverId = createDriver(
normalizedTenantKey, normalizedTenantKey,
eventSourceId, eventSourceId,
sourceDriverEntityId, sourceDriverEntityId,
null, cardNation,
cardNumber,
null, null,
null, null,
null, null,
@ -64,35 +58,20 @@ public class DriverIdentityRepository {
); );
} }
if (driverId != null && sourceDriverEntityId != null) { touchDriver(driverId, sourceDriverEntityId, cardNation, cardNumber);
upsertSourceDriverIdentity( return driverId;
normalizedTenantKey,
eventSourceId,
sourceDriverEntityId,
driverId,
null,
Map.of("source", "event")
);
}
if (driverCardId != null && driverId != null) {
linkDriverCard(driverCardId, driverId);
}
return new ResolvedDriverReference(driverId, driverCardId);
} }
@Transactional @Transactional
public int reconcileFromMasterData(String tenantKey, int eventSourceId) { public int reconcileFromMasterData(String tenantKey, int eventSourceId) {
String normalizedTenantKey = normalizeRequired(tenantKey, "tenantKey"); String normalizedTenantKey = normalizeRequired(tenantKey, "tenantKey");
int updates = reconcileDriversFromMasterData(normalizedTenantKey, eventSourceId); int updates = reconcileDriversFromMasterData(normalizedTenantKey, eventSourceId);
updates += reconcileDriverCardsFromMasterData(normalizedTenantKey, eventSourceId); updates += projectDriverCardsFromMasterData(normalizedTenantKey, eventSourceId);
updates += projectDriverCardLinksFromMasterData(normalizedTenantKey, eventSourceId);
return updates; return updates;
} }
private int reconcileDriversFromMasterData(String tenantKey, int eventSourceId) { private int reconcileDriversFromMasterData(String tenantKey, int eventSourceId) {
int insertedDrivers; Long count = jdbcTemplate.queryForObject(
if (driverUsesLegacySchema()) {
insertedDrivers = jdbcTemplate.update(
compatibleSourcesCte() + """ compatibleSourcesCte() + """
, master_drivers as ( , master_drivers as (
select distinct on (nullif(trim(source_entity_id), '')) select distinct on (nullif(trim(source_entity_id), ''))
@ -111,540 +90,125 @@ public class DriverIdentityRepository {
and event_source_id in (select id from compatible_sources) and event_source_id in (select id from compatible_sources)
and entity_type = 'DRIVER' and entity_type = 'DRIVER'
and nullif(trim(source_entity_id), '') is not null and nullif(trim(source_entity_id), '') is not null
and source_entity_id not like 'DRIVER_CARD:%'
order by nullif(trim(source_entity_id), ''), updated_at desc order by nullif(trim(source_entity_id), ''), updated_at desc
), ),
resolved_drivers as ( updated_by_source as (
select master.event_source_id,
master.source_driver_entity_id,
master.first_names,
master.last_name,
master.birth_date,
master.source_updated_at,
master.payload,
coalesce(identity.driver_id, gen_random_uuid()) as driver_id
from master_drivers master
left join lateral (
select identity.driver_id
from eventhub.source_driver_identity identity
where identity.tenant_key = ?
and identity.event_source_id in (select id from compatible_sources)
and identity.source_driver_entity_id = master.source_driver_entity_id
order by identity.updated_at desc, identity.id desc
limit 1
) identity on true
)
insert into eventhub.driver(
id, tenant_key, event_source_id, source_driver_entity_id,
first_names, last_name, birth_date, source_updated_at, payload, updated_at
)
select distinct on (resolved.driver_id)
resolved.driver_id,
?,
resolved.event_source_id,
resolved.source_driver_entity_id,
resolved.first_names,
resolved.last_name,
resolved.birth_date,
resolved.source_updated_at,
resolved.payload,
now()
from resolved_drivers resolved
where not exists (
select 1
from eventhub.driver existing
where existing.id = resolved.driver_id
)
""",
eventSourceId,
tenantKey,
tenantKey,
tenantKey
);
} else {
insertedDrivers = jdbcTemplate.update(
compatibleSourcesCte() + """
, master_drivers as (
select distinct on (nullif(trim(source_entity_id), ''))
event_source_id,
nullif(trim(source_entity_id), '') as source_driver_entity_id,
nullif(trim(payload ->> 'first_names'), '') as first_names,
coalesce(
nullif(trim(payload ->> 'last_name'), ''),
nullif(trim(payload ->> 'surname'), '')
) as last_name,
cast(nullif(trim(payload ->> 'birth_date'), '') as date) as birth_date,
source_updated_at,
payload
from eventhub.source_master_entity
where tenant_key = ?
and event_source_id in (select id from compatible_sources)
and entity_type = 'DRIVER'
and nullif(trim(source_entity_id), '') is not null
order by nullif(trim(source_entity_id), ''), updated_at desc
),
resolved_drivers as (
select master.event_source_id,
master.source_driver_entity_id,
master.first_names,
master.last_name,
master.birth_date,
master.source_updated_at,
master.payload,
coalesce(identity.driver_id, gen_random_uuid()) as driver_id
from master_drivers master
left join lateral (
select identity.driver_id
from eventhub.source_driver_identity identity
where identity.tenant_key = ?
and identity.event_source_id in (select id from compatible_sources)
and identity.source_driver_entity_id = master.source_driver_entity_id
order by identity.updated_at desc, identity.id desc
limit 1
) identity on true
)
insert into eventhub.driver(
id, first_names, last_name, birth_date, source_updated_at, payload, updated_at
)
select distinct on (resolved.driver_id)
resolved.driver_id,
resolved.first_names,
resolved.last_name,
resolved.birth_date,
resolved.source_updated_at,
resolved.payload,
now()
from resolved_drivers resolved
where not exists (
select 1
from eventhub.driver existing
where existing.id = resolved.driver_id
)
""",
eventSourceId,
tenantKey,
tenantKey
);
}
int updatedDrivers = jdbcTemplate.update(
compatibleSourcesCte() + """
, master_drivers as (
select distinct on (nullif(trim(source_entity_id), ''))
nullif(trim(source_entity_id), '') as source_driver_entity_id,
nullif(trim(payload ->> 'first_names'), '') as first_names,
coalesce(
nullif(trim(payload ->> 'last_name'), ''),
nullif(trim(payload ->> 'surname'), '')
) as last_name,
cast(nullif(trim(payload ->> 'birth_date'), '') as date) as birth_date,
source_updated_at,
payload
from eventhub.source_master_entity
where tenant_key = ?
and event_source_id in (select id from compatible_sources)
and entity_type = 'DRIVER'
and nullif(trim(source_entity_id), '') is not null
order by nullif(trim(source_entity_id), ''), updated_at desc
)
update eventhub.driver driver update eventhub.driver driver
set first_names = coalesce(master.first_names, driver.first_names), set first_names = coalesce(master.first_names, driver.first_names),
last_name = coalesce(master.last_name, driver.last_name), last_name = coalesce(master.last_name, driver.last_name),
birth_date = coalesce(master.birth_date, driver.birth_date), birth_date = coalesce(master.birth_date, driver.birth_date),
source_updated_at = coalesce(master.source_updated_at, driver.source_updated_at), source_updated_at = master.source_updated_at,
payload = driver.payload || master.payload, payload = driver.payload || master.payload,
updated_at = now() updated_at = now()
from master_drivers master from master_drivers master
join lateral ( where driver.tenant_key = ?
select identity.driver_id and driver.event_source_id in (select id from compatible_sources)
from eventhub.source_driver_identity identity and driver.source_driver_entity_id = master.source_driver_entity_id
where identity.tenant_key = ? returning driver.id
and identity.event_source_id in (select id from compatible_sources)
and identity.source_driver_entity_id = master.source_driver_entity_id
order by identity.updated_at desc, identity.id desc
limit 1
) identity on true
where identity.driver_id = driver.id
""",
eventSourceId,
tenantKey,
tenantKey
);
int linkedDrivers = jdbcTemplate.update(
compatibleSourcesCte() + """
, master_drivers as (
select distinct on (nullif(trim(source_entity_id), ''))
event_source_id,
nullif(trim(source_entity_id), '') as source_driver_entity_id,
source_updated_at,
payload
from eventhub.source_master_entity
where tenant_key = ?
and event_source_id in (select id from compatible_sources)
and entity_type = 'DRIVER'
and nullif(trim(source_entity_id), '') is not null
order by nullif(trim(source_entity_id), ''), updated_at desc
), ),
resolved_driver_ids as ( inserted as (
select master.event_source_id, insert into eventhub.driver(
master.source_driver_entity_id,
master.source_updated_at,
master.payload,
coalesce(identity.driver_id, (
select created.id
from eventhub.driver created
where created.payload = master.payload
order by created.updated_at desc
limit 1
)) as driver_id
from master_drivers master
left join lateral (
select identity.driver_id
from eventhub.source_driver_identity identity
where identity.tenant_key = ?
and identity.event_source_id in (select id from compatible_sources)
and identity.source_driver_entity_id = master.source_driver_entity_id
order by identity.updated_at desc, identity.id desc
limit 1
) identity on true
)
insert into eventhub.source_driver_identity(
id, tenant_key, event_source_id, source_driver_entity_id, id, tenant_key, event_source_id, source_driver_entity_id,
driver_id, source_updated_at, payload, updated_at first_names, last_name, birth_date, source_updated_at, payload, updated_at
) )
select gen_random_uuid(), select gen_random_uuid(),
?, ?,
resolved.event_source_id, master.event_source_id,
resolved.source_driver_entity_id, master.source_driver_entity_id,
resolved.driver_id, master.first_names,
resolved.source_updated_at, master.last_name,
resolved.payload, master.birth_date,
now()
from resolved_driver_ids resolved
where resolved.driver_id is not null
on conflict (tenant_key, event_source_id, source_driver_entity_id)
do update set
driver_id = excluded.driver_id,
source_updated_at = coalesce(excluded.source_updated_at, eventhub.source_driver_identity.source_updated_at),
payload = eventhub.source_driver_identity.payload || excluded.payload,
updated_at = now()
""",
eventSourceId,
tenantKey,
tenantKey,
tenantKey
);
return insertedDrivers + updatedDrivers + linkedDrivers;
}
private int reconcileDriverCardsFromMasterData(String tenantKey, int eventSourceId) {
int insertedCards = jdbcTemplate.update(
compatibleSourcesCte() + """
, master_driver_cards as (
select distinct on (
nullif(trim(source_entity_id), ''),
nullif(trim(payload ->> 'card_nation'), ''),
nullif(trim(payload ->> 'card_number'), '')
)
event_source_id,
nullif(trim(source_entity_id), '') as source_driver_card_entity_id,
nullif(trim(payload ->> 'card_nation'), '') as card_nation,
nullif(trim(payload ->> 'card_number'), '') as card_number,
source_updated_at,
payload
from eventhub.source_master_entity
where tenant_key = ?
and event_source_id in (select id from compatible_sources)
and entity_type = 'DRIVER_CARD'
and nullif(trim(payload ->> 'card_nation'), '') is not null
and nullif(trim(payload ->> 'card_number'), '') is not null
order by nullif(trim(source_entity_id), ''),
nullif(trim(payload ->> 'card_nation'), ''),
nullif(trim(payload ->> 'card_number'), ''),
updated_at desc
),
canonical_driver_cards as (
select distinct on (master.card_nation, master.card_number)
master.card_nation,
master.card_number,
master.source_updated_at, master.source_updated_at,
master.payload master.payload,
from master_driver_cards master
order by master.card_nation,
master.card_number,
case when master.source_driver_card_entity_id is null then 1 else 0 end,
master.source_updated_at desc,
master.source_driver_card_entity_id
),
existing_driver_cards as (
select distinct on (existing.nation, existing.card_number)
existing.id,
existing.nation,
existing.card_number
from eventhub.driver_card existing
order by existing.nation,
existing.card_number,
case when existing.driver_id is null then 1 else 0 end,
existing.updated_at desc,
existing.created_at desc,
existing.id
),
resolved_cards as (
select canonical.card_nation,
canonical.card_number,
canonical.source_updated_at,
canonical.payload,
coalesce(existing.id, gen_random_uuid()) as driver_card_id
from canonical_driver_cards canonical
left join existing_driver_cards existing
on existing.nation = canonical.card_nation
and existing.card_number = canonical.card_number
)
insert into eventhub.driver_card(
id, driver_id, nation, card_number, source_updated_at, payload, updated_at
)
select distinct on (resolved.driver_card_id)
resolved.driver_card_id,
null,
resolved.card_nation,
resolved.card_number,
resolved.source_updated_at,
resolved.payload,
now() now()
from resolved_cards resolved from master_drivers master
where not exists ( where not exists (
select 1 select 1
from eventhub.driver_card existing from eventhub.driver existing
where existing.id = resolved.driver_card_id where existing.tenant_key = ?
and existing.event_source_id in (select id from compatible_sources)
and existing.source_driver_entity_id = master.source_driver_entity_id
) )
returning id
)
select (select count(*) from updated_by_source)
+ (select count(*) from inserted)
""", """,
Long.class,
eventSourceId, eventSourceId,
tenantKey tenantKey,
);
int updatedCards = jdbcTemplate.update(
compatibleSourcesCte() + """
, master_driver_cards as (
select distinct on (
nullif(trim(source_entity_id), ''),
nullif(trim(payload ->> 'card_nation'), ''),
nullif(trim(payload ->> 'card_number'), '')
)
nullif(trim(source_entity_id), '') as source_driver_card_entity_id,
nullif(trim(payload ->> 'card_nation'), '') as card_nation,
nullif(trim(payload ->> 'card_number'), '') as card_number,
source_updated_at,
payload
from eventhub.source_master_entity
where tenant_key = ?
and event_source_id in (select id from compatible_sources)
and entity_type = 'DRIVER_CARD'
and nullif(trim(payload ->> 'card_nation'), '') is not null
and nullif(trim(payload ->> 'card_number'), '') is not null
order by nullif(trim(source_entity_id), ''),
nullif(trim(payload ->> 'card_nation'), ''),
nullif(trim(payload ->> 'card_number'), ''),
updated_at desc
),
canonical_driver_cards as (
select distinct on (master.card_nation, master.card_number)
master.card_nation,
master.card_number,
master.source_updated_at,
master.payload
from master_driver_cards master
order by master.card_nation,
master.card_number,
case when master.source_driver_card_entity_id is null then 1 else 0 end,
master.source_updated_at desc,
master.source_driver_card_entity_id
),
existing_driver_cards as (
select distinct on (existing.nation, existing.card_number)
existing.id,
existing.nation,
existing.card_number
from eventhub.driver_card existing
order by existing.nation,
existing.card_number,
case when existing.driver_id is null then 1 else 0 end,
existing.updated_at desc,
existing.created_at desc,
existing.id
),
resolved_cards as (
select canonical.source_updated_at,
canonical.payload,
existing.id as driver_card_id
from canonical_driver_cards canonical
join existing_driver_cards existing
on existing.nation = canonical.card_nation
and existing.card_number = canonical.card_number
)
update eventhub.driver_card card
set source_updated_at = coalesce(resolved.source_updated_at, card.source_updated_at),
payload = card.payload || resolved.payload,
updated_at = now()
from resolved_cards resolved
where card.id = resolved.driver_card_id
""",
eventSourceId,
tenantKey
);
int linkedCards = jdbcTemplate.update(
compatibleSourcesCte() + """
, master_driver_cards as (
select distinct on (
nullif(trim(source_entity_id), ''),
nullif(trim(payload ->> 'card_nation'), ''),
nullif(trim(payload ->> 'card_number'), '')
)
event_source_id,
nullif(trim(source_entity_id), '') as source_driver_card_entity_id,
nullif(trim(payload ->> 'card_nation'), '') as card_nation,
nullif(trim(payload ->> 'card_number'), '') as card_number,
source_updated_at,
payload
from eventhub.source_master_entity
where tenant_key = ?
and event_source_id in (select id from compatible_sources)
and entity_type = 'DRIVER_CARD'
and nullif(trim(payload ->> 'card_nation'), '') is not null
and nullif(trim(payload ->> 'card_number'), '') is not null
order by nullif(trim(source_entity_id), ''),
nullif(trim(payload ->> 'card_nation'), ''),
nullif(trim(payload ->> 'card_number'), ''),
updated_at desc
),
existing_driver_cards as (
select distinct on (existing.nation, existing.card_number)
existing.id,
existing.nation,
existing.card_number
from eventhub.driver_card existing
order by existing.nation,
existing.card_number,
case when existing.driver_id is null then 1 else 0 end,
existing.updated_at desc,
existing.created_at desc,
existing.id
),
resolved_cards as (
select master.event_source_id,
master.source_driver_card_entity_id,
master.source_updated_at,
master.payload,
coalesce(identity.driver_card_id, existing.id) as driver_card_id
from master_driver_cards master
left join lateral (
select identity.driver_card_id
from eventhub.source_driver_card_identity identity
where identity.tenant_key = ?
and identity.event_source_id in (select id from compatible_sources)
and identity.source_driver_card_entity_id = master.source_driver_card_entity_id
order by identity.updated_at desc, identity.id desc
limit 1
) identity on true
left join existing_driver_cards existing
on existing.nation = master.card_nation
and existing.card_number = master.card_number
)
insert into eventhub.source_driver_card_identity(
id, tenant_key, event_source_id, source_driver_card_entity_id,
driver_card_id, source_updated_at, payload, updated_at
)
select gen_random_uuid(),
?,
resolved.event_source_id,
resolved.source_driver_card_entity_id,
resolved.driver_card_id,
resolved.source_updated_at,
resolved.payload,
now()
from resolved_cards resolved
where resolved.source_driver_card_entity_id is not null
and resolved.driver_card_id is not null
on conflict (tenant_key, event_source_id, source_driver_card_entity_id)
do update set
driver_card_id = excluded.driver_card_id,
source_updated_at = coalesce(excluded.source_updated_at, eventhub.source_driver_card_identity.source_updated_at),
payload = eventhub.source_driver_card_identity.payload || excluded.payload,
updated_at = now()
""",
eventSourceId,
tenantKey, tenantKey,
tenantKey, tenantKey,
tenantKey tenantKey
); );
return count == null ? 0 : Math.toIntExact(count);
return insertedCards + updatedCards + linkedCards;
} }
private int projectDriverCardLinksFromMasterData(String tenantKey, int eventSourceId) { private int projectDriverCardsFromMasterData(String tenantKey, int eventSourceId) {
return jdbcTemplate.update( Long count = jdbcTemplate.queryForObject(
compatibleSourcesCte() + """ compatibleSourcesCte() + """
update eventhub.driver_card card , driver_card_projection as (
set driver_id = driver_identity.driver_id, select distinct on (rel.to_source_entity_id)
source_updated_at = coalesce(relation.source_updated_at, card.source_updated_at), rel.to_source_entity_id as source_driver_entity_id,
nullif(trim(card.payload ->> 'card_nation'), '') as card_nation,
nullif(trim(card.payload ->> 'card_number'), '') as card_number,
rel.source_updated_at
from eventhub.source_master_relation rel
join eventhub.source_master_entity card
on card.tenant_key = rel.tenant_key
and card.event_source_id = rel.event_source_id
and card.entity_type = 'DRIVER_CARD'
and card.source_entity_id = rel.from_source_entity_id
where rel.tenant_key = ?
and rel.event_source_id in (select id from compatible_sources)
and rel.relation_type = 'DRIVER_CARD_DRIVER'
and rel.from_entity_type = 'DRIVER_CARD'
and rel.to_entity_type = 'DRIVER'
order by rel.to_source_entity_id,
rel.valid_to desc nulls last,
rel.valid_from desc nulls last,
rel.updated_at desc
),
updated_by_source as (
update eventhub.driver driver
set card_nation = coalesce(driver.card_nation, projection.card_nation),
card_number = coalesce(driver.card_number, projection.card_number),
source_updated_at = coalesce(projection.source_updated_at, driver.source_updated_at),
updated_at = now() updated_at = now()
from eventhub.source_master_relation relation from driver_card_projection projection
join eventhub.source_driver_card_identity card_identity where driver.tenant_key = ?
on card_identity.tenant_key = relation.tenant_key and driver.event_source_id in (select id from compatible_sources)
and card_identity.event_source_id = relation.event_source_id and driver.source_driver_entity_id = projection.source_driver_entity_id
and card_identity.source_driver_card_entity_id = relation.from_source_entity_id
join eventhub.source_driver_identity driver_identity
on driver_identity.tenant_key = relation.tenant_key
and driver_identity.event_source_id = relation.event_source_id
and driver_identity.source_driver_entity_id = relation.to_source_entity_id
where relation.tenant_key = ?
and relation.event_source_id in (select id from compatible_sources)
and relation.relation_type = 'DRIVER_CARD_DRIVER'
and relation.from_entity_type = 'DRIVER_CARD'
and relation.to_entity_type = 'DRIVER'
and card.id = card_identity.driver_card_id
and ( and (
card.driver_id is null (driver.card_nation is null and projection.card_nation is not null)
or card.driver_id = driver_identity.driver_id or (driver.card_number is null and projection.card_number is not null)
or not exists (
select 1
from eventhub.source_driver_identity existing_identity
where existing_identity.driver_id = card.driver_id
) )
returning driver.id
) )
select count(*)
from updated_by_source
""", """,
Long.class,
eventSourceId, eventSourceId,
tenantKey,
tenantKey tenantKey
); );
return count == null ? 0 : Math.toIntExact(count);
} }
private UUID resolveOrCreateDriverCardId( private UUID resolveDriverId(
String tenantKey,
int eventSourceId,
String sourceDriverEntityId,
String cardNation, String cardNation,
String cardNumber, String cardNumber
UUID preferredDriverId
) { ) {
if (cardNation == null || cardNumber == null) { UUID driverId = findBySourceDriverEntityId(tenantKey, eventSourceId, sourceDriverEntityId);
return null; if (driverId == null) {
driverId = findByCard(tenantKey, eventSourceId, cardNation, cardNumber);
} }
UUID driverCardId = findDriverCardByCard(cardNation, cardNumber); return driverId;
if (driverCardId == null) {
Map<String, Object> payload = new LinkedHashMap<>();
put(payload, "source", "event");
put(payload, "card_nation", cardNation);
put(payload, "card_number", cardNumber);
return createDriverCard(
preferredDriverId,
cardNation,
cardNumber,
null,
payload
);
}
if (preferredDriverId != null) {
linkDriverCard(driverCardId, preferredDriverId);
}
return driverCardId;
} }
private UUID findBySourceDriverEntityId(String tenantKey, int eventSourceId, String sourceDriverEntityId) { private UUID findBySourceDriverEntityId(String tenantKey, int eventSourceId, String sourceDriverEntityId) {
@ -653,50 +217,39 @@ public class DriverIdentityRepository {
} }
return jdbcTemplate.query( return jdbcTemplate.query(
compatibleSourcesCte() + """ compatibleSourcesCte() + """
select identity.driver_id select d.id
from eventhub.source_driver_identity identity from eventhub.driver d
where identity.tenant_key = ? where d.tenant_key = ?
and identity.event_source_id in (select id from compatible_sources) and d.event_source_id in (select id from compatible_sources)
and identity.source_driver_entity_id = ? and d.source_driver_entity_id = ?
order by identity.updated_at desc order by d.updated_at desc
limit 1 limit 1
""", """,
rs -> rs.next() ? (UUID) rs.getObject("driver_id") : null, rs -> rs.next() ? (UUID) rs.getObject("id") : null,
eventSourceId, eventSourceId,
tenantKey, tenantKey,
sourceDriverEntityId sourceDriverEntityId
); );
} }
private UUID findDriverIdByCardId(UUID driverCardId) { private UUID findByCard(String tenantKey, int eventSourceId, String cardNation, String cardNumber) {
if (driverCardId == null) {
return null;
}
return jdbcTemplate.query(
"""
select driver_id
from eventhub.driver_card
where id = ?
""",
rs -> rs.next() ? (UUID) rs.getObject("driver_id") : null,
driverCardId
);
}
private UUID findDriverCardByCard(String cardNation, String cardNumber) {
if (cardNation == null || cardNumber == null) { if (cardNation == null || cardNumber == null) {
return null; return null;
} }
return jdbcTemplate.query( return jdbcTemplate.query(
""" compatibleSourcesCte() + """
select card.id select d.id
from eventhub.driver_card card from eventhub.driver d
where card.nation = ? where d.tenant_key = ?
and card.card_number = ? and d.event_source_id in (select id from compatible_sources)
order by card.updated_at desc and d.card_nation = ?
and d.card_number = ?
order by d.updated_at desc
limit 1 limit 1
""", """,
rs -> rs.next() ? (UUID) rs.getObject("id") : null, rs -> rs.next() ? (UUID) rs.getObject("id") : null,
eventSourceId,
tenantKey,
cardNation, cardNation,
cardNumber cardNumber
); );
@ -706,168 +259,66 @@ public class DriverIdentityRepository {
String tenantKey, String tenantKey,
int eventSourceId, int eventSourceId,
String sourceDriverEntityId, String sourceDriverEntityId,
String cardNation,
String cardNumber,
String firstNames, String firstNames,
String lastName, String lastName,
OffsetDateTime sourceUpdatedAt, OffsetDateTime sourceUpdatedAt,
LocalDate birthDate,
Map<String, Object> payload Map<String, Object> payload
) { ) {
UUID driverId = UUID.randomUUID(); UUID driverId = UUID.randomUUID();
if (driverUsesLegacySchema()) {
jdbcTemplate.update( jdbcTemplate.update(
""" """
insert into eventhub.driver( insert into eventhub.driver(
id, tenant_key, event_source_id, source_driver_entity_id, id, tenant_key, event_source_id, source_driver_entity_id,
first_names, last_name, birth_date, source_updated_at, payload, updated_at card_nation, card_number, first_names, last_name,
) values (?, ?, ?, ?, ?, ?, ?, ?, ?::jsonb, now()) source_updated_at, payload, updated_at
) values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?::jsonb, now())
""", """,
driverId, driverId,
tenantKey, tenantKey,
eventSourceId, eventSourceId,
sourceDriverEntityId, sourceDriverEntityId,
cardNation,
cardNumber,
firstNames, firstNames,
lastName, lastName,
birthDate,
sourceUpdatedAt, sourceUpdatedAt,
toJson(payload) toJson(payload)
); );
} else {
jdbcTemplate.update(
"""
insert into eventhub.driver(
id, first_names, last_name, birth_date, source_updated_at, payload, updated_at
) values (?, ?, ?, ?, ?, ?::jsonb, now())
""",
driverId,
firstNames,
lastName,
birthDate,
sourceUpdatedAt,
toJson(payload)
);
}
return driverId; return driverId;
} }
private boolean driverUsesLegacySchema() { private void touchDriver(
Integer legacyColumns = jdbcTemplate.queryForObject(
"""
select count(*)
from information_schema.columns
where table_schema = 'eventhub'
and table_name = 'driver'
and column_name in ('tenant_key', 'event_source_id', 'source_driver_entity_id')
""",
Integer.class
);
return legacyColumns != null && legacyColumns == 3;
}
private String normalizeDriverCardNumber(String cardNation, String cardNumber) {
String normalized = normalizeNullable(cardNumber);
if (normalized == null) {
return null;
}
if (isSyntheticYellowFoxCardNation(cardNation)) {
return normalized.length() <= 14 ? normalized : normalized.substring(0, 14);
}
return normalized;
}
private boolean isSyntheticYellowFoxCardNation(String cardNation) {
return YELLOWFOX_SYNTHETIC_REFERENCE_NATION.equalsIgnoreCase(cardNation)
|| UNKNOWN_CARD_NATION.equalsIgnoreCase(cardNation);
}
private UUID createDriverCard(
UUID driverId, UUID driverId,
String cardNation,
String cardNumber,
OffsetDateTime sourceUpdatedAt,
Map<String, Object> payload
) {
UUID driverCardId = UUID.randomUUID();
jdbcTemplate.update(
"""
insert into eventhub.driver_card(
id, driver_id, nation, card_number, source_updated_at, payload, updated_at
) values (?, ?, ?, ?, ?, ?::jsonb, now())
""",
driverCardId,
driverId,
cardNation,
cardNumber,
sourceUpdatedAt,
toJson(payload)
);
return driverCardId;
}
private void upsertSourceDriverIdentity(
String tenantKey,
int eventSourceId,
String sourceDriverEntityId, String sourceDriverEntityId,
UUID driverId, String cardNation,
OffsetDateTime sourceUpdatedAt, String cardNumber
Map<String, Object> payload
) { ) {
if (sourceDriverEntityId == null || driverId == null) { if (sourceDriverEntityId == null && cardNation == null && cardNumber == null) {
return; return;
} }
jdbcTemplate.update( jdbcTemplate.update(
""" """
insert into eventhub.source_driver_identity( update eventhub.driver
id, tenant_key, event_source_id, source_driver_entity_id, set source_driver_entity_id = coalesce(source_driver_entity_id, cast(? as text)),
driver_id, source_updated_at, payload, updated_at card_nation = coalesce(card_nation, cast(? as text)),
) values (?, ?, ?, ?, ?, ?, ?::jsonb, now()) card_number = coalesce(card_number, cast(? as text)),
on conflict (tenant_key, event_source_id, source_driver_entity_id)
do update set
driver_id = excluded.driver_id,
source_updated_at = coalesce(excluded.source_updated_at, eventhub.source_driver_identity.source_updated_at),
payload = eventhub.source_driver_identity.payload || excluded.payload,
updated_at = now()
""",
UUID.randomUUID(),
tenantKey,
eventSourceId,
sourceDriverEntityId,
driverId,
sourceUpdatedAt,
toJson(payload)
);
}
private void linkDriverCard(UUID driverCardId, UUID driverId) {
if (driverCardId == null || driverId == null) {
return;
}
jdbcTemplate.update(
"""
update eventhub.driver_card
set driver_id = ?,
updated_at = now() updated_at = now()
where id = ? where id = ?
and ( and (
driver_id is null (source_driver_entity_id is null and cast(? as text) is not null)
or driver_id = ? or (card_nation is null and cast(? as text) is not null)
or ( or (card_number is null and cast(? as text) is not null)
exists (
select 1
from eventhub.source_driver_identity preferred_identity
where preferred_identity.driver_id = ?
)
and not exists (
select 1
from eventhub.source_driver_identity current_identity
where current_identity.driver_id = eventhub.driver_card.driver_id
)
)
) )
""", """,
sourceDriverEntityId,
cardNation,
cardNumber,
driverId, driverId,
driverCardId, sourceDriverEntityId,
driverId, cardNation,
driverId cardNumber
); );
} }
@ -918,10 +369,4 @@ public class DriverIdentityRepository {
String trimmed = value.trim(); String trimmed = value.trim();
return trimmed.isEmpty() ? null : trimmed; return trimmed.isEmpty() ? null : trimmed;
} }
public record ResolvedDriverReference(UUID driverId, UUID driverCardId) {
public static ResolvedDriverReference empty() {
return new ResolvedDriverReference(null, null);
}
}
} }

74
src/main/java/at/procon/eventhub/persistence/EventRepository.java
View File

@ -57,12 +57,11 @@ public class EventRepository {
*/ */
public int batchInsert(UUID packageId, String tenantKey, int eventSourceId, List<EventHubEventDto> events) { public int batchInsert(UUID packageId, String tenantKey, int eventSourceId, List<EventHubEventDto> events) {
Map<String, UUID> entityIdCache = new HashMap<>(); Map<String, UUID> entityIdCache = new HashMap<>();
Map<String, DriverIdentityRepository.ResolvedDriverReference> driverRefCache = new HashMap<>();
Map<String, List<VehicleRefCacheEntry>> vehicleRefCache = new HashMap<>(); Map<String, List<VehicleRefCacheEntry>> vehicleRefCache = new HashMap<>();
List<ResolvedEventImportRow> rows = new ArrayList<>(events.size()); List<ResolvedEventImportRow> rows = new ArrayList<>(events.size());
for (EventHubEventDto event : events) { for (EventHubEventDto event : events) {
ResolvedEntityRefs refs = resolveEntityRefs(tenantKey, eventSourceId, event, entityIdCache, driverRefCache, vehicleRefCache); ResolvedEntityRefs refs = resolveEntityRefs(tenantKey, eventSourceId, event, entityIdCache, vehicleRefCache);
rows.add(resolveEventImportRow(packageId, eventSourceId, event, refs)); rows.add(resolveEventImportRow(packageId, eventSourceId, event, refs));
} }
@ -94,7 +93,6 @@ public class EventRepository {
eventSourceId, eventSourceId,
event.externalSourceEventId(), event.externalSourceEventId(),
refs.driverId(), refs.driverId(),
refs.driverCardId(),
refs.vehicleId(), refs.vehicleId(),
refs.vehicleRegistrationId(), refs.vehicleRegistrationId(),
sourcePackageId, sourcePackageId,
@ -127,7 +125,6 @@ public class EventRepository {
event_source_id integer not null, event_source_id integer not null,
external_source_event_id text not null, external_source_event_id text not null,
driver_id uuid, driver_id uuid,
driver_card_id uuid,
vehicle_id uuid, vehicle_id uuid,
vehicle_registration_id uuid, vehicle_registration_id uuid,
source_package_id text, source_package_id text,
@ -155,11 +152,11 @@ public class EventRepository {
""" """
insert into eventhub_event_import_stage( insert into eventhub_event_import_stage(
row_no, source_record_key_hash, requested_event_id, data_package_id, event_source_id, row_no, source_record_key_hash, requested_event_id, data_package_id, event_source_id,
external_source_event_id, driver_id, driver_card_id, vehicle_id, vehicle_registration_id, external_source_event_id, driver_id, vehicle_id, vehicle_registration_id,
source_package_id, source_package_entity_id, occurred_at, received_partner_at, received_hub_at, source_package_id, source_package_entity_id, occurred_at, received_partner_at, received_hub_at,
event_domain, event_type, lifecycle, odometer_m, longitude, latitude, event_domain, event_type, lifecycle, odometer_m, longitude, latitude,
payload, manual_entry, event_signature_hash payload, manual_entry, event_signature_hash
) values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?::jsonb, ?, ?) ) values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?::jsonb, ?, ?)
""", """,
new BatchPreparedStatementSetter() { new BatchPreparedStatementSetter() {
@Override @Override
@ -172,23 +169,22 @@ public class EventRepository {
ps.setInt(5, row.eventSourceId()); ps.setInt(5, row.eventSourceId());
ps.setString(6, row.externalSourceEventId()); ps.setString(6, row.externalSourceEventId());
ps.setObject(7, row.driverId()); ps.setObject(7, row.driverId());
ps.setObject(8, row.driverCardId()); ps.setObject(8, row.vehicleId());
ps.setObject(9, row.vehicleId()); ps.setObject(9, row.vehicleRegistrationId());
ps.setObject(10, row.vehicleRegistrationId()); ps.setString(10, row.sourcePackageId());
ps.setString(11, row.sourcePackageId()); ps.setObject(11, row.sourcePackageEntityId());
ps.setObject(12, row.sourcePackageEntityId()); ps.setObject(12, row.occurredAt());
ps.setObject(13, row.occurredAt()); ps.setObject(13, row.receivedPartnerAt());
ps.setObject(14, row.receivedPartnerAt()); ps.setObject(14, row.receivedHubAt());
ps.setObject(15, row.receivedHubAt()); ps.setString(15, row.eventDomain());
ps.setString(16, row.eventDomain()); ps.setString(16, row.eventType());
ps.setString(17, row.eventType()); ps.setString(17, row.lifecycle());
ps.setString(18, row.lifecycle()); ps.setObject(18, row.odometerM());
ps.setObject(19, row.odometerM()); ps.setObject(19, row.longitude());
ps.setObject(20, row.longitude()); ps.setObject(20, row.latitude());
ps.setObject(21, row.latitude()); ps.setString(21, row.payloadJson());
ps.setString(22, row.payloadJson()); ps.setBoolean(22, row.manualEntry());
ps.setBoolean(23, row.manualEntry()); ps.setString(23, row.eventSignatureHash());
ps.setString(24, row.eventSignatureHash());
} }
@Override @Override
@ -240,7 +236,7 @@ public class EventRepository {
insert into eventhub.event( insert into eventhub.event(
id, event_source_id, data_package_id, id, event_source_id, data_package_id,
external_source_event_id, external_source_event_id,
driver_id, driver_card_id, vehicle_id, vehicle_registration_id, driver_id, vehicle_id, vehicle_registration_id,
source_package_id, source_package_entity_id, source_package_id, source_package_entity_id,
occurred_at, received_partner_at, received_hub_at, occurred_at, received_partner_at, received_hub_at,
event_domain, event_type, lifecycle, event_domain, event_type, lifecycle,
@ -251,7 +247,7 @@ public class EventRepository {
select select
source_record.event_id, stage.event_source_id, stage.data_package_id, source_record.event_id, stage.event_source_id, stage.data_package_id,
stage.external_source_event_id, stage.external_source_event_id,
stage.driver_id, stage.driver_card_id, stage.vehicle_id, stage.vehicle_registration_id, stage.driver_id, stage.vehicle_id, stage.vehicle_registration_id,
stage.source_package_id, stage.source_package_entity_id, stage.source_package_id, stage.source_package_entity_id,
source_record.event_occurred_at, stage.received_partner_at, stage.received_hub_at, source_record.event_occurred_at, stage.received_partner_at, stage.received_hub_at,
stage.event_domain, stage.event_type, stage.lifecycle, stage.event_domain, stage.event_type, stage.lifecycle,
@ -362,39 +358,33 @@ public class EventRepository {
int eventSourceId, int eventSourceId,
EventHubEventDto event, EventHubEventDto event,
Map<String, UUID> entityIdCache, Map<String, UUID> entityIdCache,
Map<String, DriverIdentityRepository.ResolvedDriverReference> driverRefCache,
Map<String, List<VehicleRefCacheEntry>> vehicleRefCache Map<String, List<VehicleRefCacheEntry>> vehicleRefCache
) { ) {
DriverIdentityRepository.ResolvedDriverReference driverRef = resolveDriverReference(tenantKey, eventSourceId, event, driverRefCache); UUID driverId = resolveDriverId(tenantKey, eventSourceId, event, entityIdCache);
ResolvedVehicleReference vehicleRef = resolveVehicleReference(tenantKey, eventSourceId, event, vehicleRefCache); ResolvedVehicleReference vehicleRef = resolveVehicleReference(tenantKey, eventSourceId, event, vehicleRefCache);
UUID sourcePackageEntityId = resolveSourcePackageEntityId(tenantKey, eventSourceId, event, entityIdCache); UUID sourcePackageEntityId = resolveSourcePackageEntityId(tenantKey, eventSourceId, event, entityIdCache);
return new ResolvedEntityRefs( return new ResolvedEntityRefs(driverId, vehicleRef.vehicleId(), vehicleRef.vehicleRegistrationId(), sourcePackageEntityId);
driverRef.driverId(),
driverRef.driverCardId(),
vehicleRef.vehicleId(),
vehicleRef.vehicleRegistrationId(),
sourcePackageEntityId
);
} }
private DriverIdentityRepository.ResolvedDriverReference resolveDriverReference( private UUID resolveDriverId(
String tenantKey, String tenantKey,
int eventSourceId, int eventSourceId,
EventHubEventDto event, EventHubEventDto event,
Map<String, DriverIdentityRepository.ResolvedDriverReference> driverRefCache Map<String, UUID> entityIdCache
) { ) {
DriverRefDto driverRef = event.driverRef(); DriverRefDto driverRef = event.driverRef();
if (driverRef == null || !driverRef.hasAnyReference()) { if (driverRef == null || !driverRef.hasAnyReference()) {
return DriverIdentityRepository.ResolvedDriverReference.empty(); return null;
} }
String cacheKey = "DRIVER|" + driverRef.stableKey(); String cacheKey = "DRIVER|" + driverRef.stableKey();
DriverIdentityRepository.ResolvedDriverReference cached = driverRefCache.get(cacheKey); UUID cached = entityIdCache.get(cacheKey);
if (cached != null) { if (cached != null) {
return cached; return cached;
} }
DriverIdentityRepository.ResolvedDriverReference resolved = UUID resolved = driverIdentityRepository.resolveOrCreateDriverId(tenantKey, eventSourceId, driverRef);
driverIdentityRepository.resolveOrCreateDriverReference(tenantKey, eventSourceId, driverRef); if (resolved != null) {
driverRefCache.put(cacheKey, resolved); entityIdCache.put(cacheKey, resolved);
}
return resolved; return resolved;
} }
@ -597,7 +587,6 @@ public class EventRepository {
private record ResolvedEntityRefs( private record ResolvedEntityRefs(
UUID driverId, UUID driverId,
UUID driverCardId,
UUID vehicleId, UUID vehicleId,
UUID vehicleRegistrationId, UUID vehicleRegistrationId,
UUID sourcePackageEntityId UUID sourcePackageEntityId
@ -617,7 +606,6 @@ public class EventRepository {
int eventSourceId, int eventSourceId,
String externalSourceEventId, String externalSourceEventId,
UUID driverId, UUID driverId,
UUID driverCardId,
UUID vehicleId, UUID vehicleId,
UUID vehicleRegistrationId, UUID vehicleRegistrationId,
String sourcePackageId, String sourcePackageId,

837
src/main/java/at/procon/eventhub/persistence/VehicleIdentityRepository.java
View File

File diff suppressed because it is too large Load Diff

17
src/main/java/at/procon/eventhub/tachograph/service/TachographMasterDataRefreshService.java
View File

@ -1,6 +1,5 @@
package at.procon.eventhub.tachograph.service; package at.procon.eventhub.tachograph.service;
import at.procon.eventhub.config.EventHubProperties;
import at.procon.eventhub.importing.masterdata.MasterDataRefreshResult; import at.procon.eventhub.importing.masterdata.MasterDataRefreshResult;
import at.procon.eventhub.importing.masterdata.SourceMasterEntityUpsert; import at.procon.eventhub.importing.masterdata.SourceMasterEntityUpsert;
import at.procon.eventhub.importing.masterdata.SourceMasterRelationUpsert; import at.procon.eventhub.importing.masterdata.SourceMasterRelationUpsert;
@ -57,7 +56,6 @@ public class TachographMasterDataRefreshService {
private final DriverIdentityRepository driverIdentityRepository; private final DriverIdentityRepository driverIdentityRepository;
private final VehicleIdentityRepository vehicleIdentityRepository; private final VehicleIdentityRepository vehicleIdentityRepository;
private final ResourceLoader resourceLoader; private final ResourceLoader resourceLoader;
private final EventHubProperties properties;
public TachographMasterDataRefreshService( public TachographMasterDataRefreshService(
@Qualifier("tachographNamedParameterJdbcTemplate") ObjectProvider<NamedParameterJdbcTemplate> tachographJdbcTemplateProvider, @Qualifier("tachographNamedParameterJdbcTemplate") ObjectProvider<NamedParameterJdbcTemplate> tachographJdbcTemplateProvider,
@ -65,8 +63,7 @@ public class TachographMasterDataRefreshService {
EventSourceRepository eventSourceRepository, EventSourceRepository eventSourceRepository,
DriverIdentityRepository driverIdentityRepository, DriverIdentityRepository driverIdentityRepository,
VehicleIdentityRepository vehicleIdentityRepository, VehicleIdentityRepository vehicleIdentityRepository,
ResourceLoader resourceLoader, ResourceLoader resourceLoader
EventHubProperties properties
) { ) {
this.tachographJdbcTemplateProvider = tachographJdbcTemplateProvider; this.tachographJdbcTemplateProvider = tachographJdbcTemplateProvider;
this.sourceMasterDataRepository = sourceMasterDataRepository; this.sourceMasterDataRepository = sourceMasterDataRepository;
@ -74,7 +71,6 @@ public class TachographMasterDataRefreshService {
this.driverIdentityRepository = driverIdentityRepository; this.driverIdentityRepository = driverIdentityRepository;
this.vehicleIdentityRepository = vehicleIdentityRepository; this.vehicleIdentityRepository = vehicleIdentityRepository;
this.resourceLoader = resourceLoader; this.resourceLoader = resourceLoader;
this.properties = properties;
} }
public MasterDataRefreshResult refreshIfRequested(TachographImportRequest request) { public MasterDataRefreshResult refreshIfRequested(TachographImportRequest request) {
@ -105,19 +101,14 @@ public class TachographMasterDataRefreshService {
} }
int relationCount = streamRelations(tachographJdbcTemplate, tenantKey, eventSourceId, RELATIONS_SQL_RESOURCE, loadSql(RELATIONS_SQL_RESOURCE)); int relationCount = streamRelations(tachographJdbcTemplate, tenantKey, eventSourceId, RELATIONS_SQL_RESOURCE, loadSql(RELATIONS_SQL_RESOURCE));
boolean syncVehicleRegistrations = properties.getTachograph().isSyncVehicleRegistrationsOnMasterDataUpdate();
log.info("Reconciling tachograph driver identities from source master data tenant={} source={}", log.info("Reconciling tachograph driver identities from source master data tenant={} source={}",
tenantKey, masterDataSource.stableKey()); tenantKey, masterDataSource.stableKey());
int reconciledDrivers = driverIdentityRepository.reconcileFromMasterData(tenantKey, eventSourceId); int reconciledDrivers = driverIdentityRepository.reconcileFromMasterData(tenantKey, eventSourceId);
log.info("Reconciling tachograph vehicle identities from source master data tenant={} source={} syncVehicleRegistrations={}", log.info("Reconciling tachograph vehicle identities from source master data tenant={} source={}",
tenantKey, masterDataSource.stableKey(), syncVehicleRegistrations); tenantKey, masterDataSource.stableKey());
int reconciledVehicles = vehicleIdentityRepository.reconcileFromMasterData( int reconciledVehicles = vehicleIdentityRepository.reconcileFromMasterData(tenantKey, eventSourceId);
tenantKey,
eventSourceId,
syncVehicleRegistrations
);
MasterDataRefreshResult result = new MasterDataRefreshResult(entities, relationCount); MasterDataRefreshResult result = new MasterDataRefreshResult(entities, relationCount);
log.info("Refreshed tachograph source master data tenant={} source={} entities={} relations={} reconciledDrivers={} reconciledVehicles={}", log.info("Refreshed tachograph source master data tenant={} source={} entities={} relations={} reconciledDrivers={} reconciledVehicles={}",

24
src/main/java/at/procon/eventhub/yellowfox/api/YellowFoxD8ImportController.java
View File

@ -3,7 +3,6 @@ package at.procon.eventhub.yellowfox.api;
import at.procon.eventhub.dto.AcquisitionStrategy; import at.procon.eventhub.dto.AcquisitionStrategy;
import at.procon.eventhub.dto.ImportMode; import at.procon.eventhub.dto.ImportMode;
import at.procon.eventhub.dto.SchedulerTriggerMode; import at.procon.eventhub.dto.SchedulerTriggerMode;
import at.procon.eventhub.importing.masterdata.MasterDataRefreshResult;
import at.procon.eventhub.yellowfox.dto.ConfiguredYellowFoxD8ImportPlanDto; import at.procon.eventhub.yellowfox.dto.ConfiguredYellowFoxD8ImportPlanDto;
import at.procon.eventhub.yellowfox.dto.YellowFoxD8ImportRequest; import at.procon.eventhub.yellowfox.dto.YellowFoxD8ImportRequest;
import at.procon.eventhub.yellowfox.dto.YellowFoxD8ImportRunResultDto; import at.procon.eventhub.yellowfox.dto.YellowFoxD8ImportRunResultDto;
@ -11,7 +10,6 @@ import at.procon.eventhub.yellowfox.dto.YellowFoxD8ImportTriggerResultDto;
import at.procon.eventhub.yellowfox.service.YellowFoxD8ConfiguredImportPlanService; import at.procon.eventhub.yellowfox.service.YellowFoxD8ConfiguredImportPlanService;
import at.procon.eventhub.yellowfox.service.YellowFoxD8ImportExecutionService; import at.procon.eventhub.yellowfox.service.YellowFoxD8ImportExecutionService;
import at.procon.eventhub.yellowfox.service.YellowFoxD8ImportPlanService; import at.procon.eventhub.yellowfox.service.YellowFoxD8ImportPlanService;
import at.procon.eventhub.yellowfox.service.YellowFoxMasterDataRefreshService;
import jakarta.validation.Valid; import jakarta.validation.Valid;
import java.time.OffsetDateTime; import java.time.OffsetDateTime;
import java.util.List; import java.util.List;
@ -33,20 +31,17 @@ public class YellowFoxD8ImportController {
private final YellowFoxD8ImportPlanService importPlanService; private final YellowFoxD8ImportPlanService importPlanService;
private final YellowFoxD8ConfiguredImportPlanService configuredImportPlanService; private final YellowFoxD8ConfiguredImportPlanService configuredImportPlanService;
private final YellowFoxD8ImportExecutionService importExecutionService; private final YellowFoxD8ImportExecutionService importExecutionService;
private final YellowFoxMasterDataRefreshService masterDataRefreshService;
public YellowFoxD8ImportController( public YellowFoxD8ImportController(
ProducerTemplate producerTemplate, ProducerTemplate producerTemplate,
YellowFoxD8ImportPlanService importPlanService, YellowFoxD8ImportPlanService importPlanService,
YellowFoxD8ConfiguredImportPlanService configuredImportPlanService, YellowFoxD8ConfiguredImportPlanService configuredImportPlanService,
YellowFoxD8ImportExecutionService importExecutionService, YellowFoxD8ImportExecutionService importExecutionService
YellowFoxMasterDataRefreshService masterDataRefreshService
) { ) {
this.producerTemplate = producerTemplate; this.producerTemplate = producerTemplate;
this.importPlanService = importPlanService; this.importPlanService = importPlanService;
this.configuredImportPlanService = configuredImportPlanService; this.configuredImportPlanService = configuredImportPlanService;
this.importExecutionService = importExecutionService; this.importExecutionService = importExecutionService;
this.masterDataRefreshService = masterDataRefreshService;
} }
@PostMapping("/imports/plan") @PostMapping("/imports/plan")
@ -65,13 +60,6 @@ public class YellowFoxD8ImportController {
return ResponseEntity.accepted().body(result); return ResponseEntity.accepted().body(result);
} }
@PostMapping("/master-data/refresh")
public ResponseEntity<MasterDataRefreshResult> refreshYellowFoxMasterData(
@Valid @RequestBody YellowFoxD8ImportRequest request
) {
return ResponseEntity.ok(masterDataRefreshService.refresh(request));
}
@GetMapping("/imports/configured-plans") @GetMapping("/imports/configured-plans")
public ResponseEntity<List<ConfiguredYellowFoxD8ImportPlanDto>> listConfiguredYellowFoxPlans() { public ResponseEntity<List<ConfiguredYellowFoxD8ImportPlanDto>> listConfiguredYellowFoxPlans() {
return ResponseEntity.ok(configuredImportPlanService.listPlans()); return ResponseEntity.ok(configuredImportPlanService.listPlans());
@ -102,14 +90,4 @@ public class YellowFoxD8ImportController {
result result
)); ));
} }
@PostMapping("/imports/configured-plans/{planKey}/master-data/refresh")
public ResponseEntity<MasterDataRefreshResult> refreshConfiguredYellowFoxMasterData(
@PathVariable String planKey,
@RequestParam(required = false) ImportMode mode,
@RequestParam(required = false) AcquisitionStrategy strategy
) {
YellowFoxD8ImportRequest request = configuredImportPlanService.createRequest(planKey, mode, strategy);
return ResponseEntity.ok(masterDataRefreshService.refresh(request));
}
} }

1
src/main/java/at/procon/eventhub/yellowfox/dto/YellowFoxD8BookingDto.java
View File

@ -15,7 +15,6 @@ public record YellowFoxD8BookingDto(
String eventId, String eventId,
String key, String key,
Integer ignition, Integer ignition,
Integer previousIgnition,
Integer eventType, Integer eventType,
Integer state, Integer state,
OffsetDateTime occurredAt, OffsetDateTime occurredAt,

121
src/main/java/at/procon/eventhub/yellowfox/service/JdbcYellowFoxD8BookingExtractionBatchExecutor.java
View File

@ -83,15 +83,15 @@ public class JdbcYellowFoxD8BookingExtractionBatchExecutor implements YellowFoxD
ImportScopeDto chunkScope = chunkScope(request.importScope(), chunk); ImportScopeDto chunkScope = chunkScope(request.importScope(), chunk);
ImportCursorStateDto cursor = findCursor(eventSourceId, request, planItem); ImportCursorStateDto cursor = findCursor(eventSourceId, request, planItem);
QuerySpec query = buildQuerySpec(request, chunkScope, cursor); Map<String, Object> params = parameters(request, chunkScope, cursor);
Stats stats = new Stats(); Stats stats = new Stats();
YellowFoxD8IgnitionTransitionDetector.Session ignitionSession = ignitionTransitionDetector YellowFoxD8IgnitionTransitionDetector.Session ignitionSession = ignitionTransitionDetector
.newSession(properties.getYellowFox().isEmitInitialIgnitionSnapshot()); .newSession(properties.getYellowFox().isEmitInitialIgnitionSnapshot());
log.info("Reading YellowFox D8 bookings tenant={} importRunId={} packageId={} chunk={} occurredFrom={} occurredTo={} fleetId={} strategy={}", log.info("Reading YellowFox D8 bookings tenant={} importRunId={} packageId={} chunk={} occurredFrom={} occurredTo={} fleetId={} strategy={}",
request.tenantKey(), importRunId, packageId, chunk.sequence(), chunk.occurredFrom(), chunk.occurredTo(), query.fleetId(), request.acquisitionStrategy()); request.tenantKey(), importRunId, packageId, chunk.sequence(), chunk.occurredFrom(), chunk.occurredTo(), params.get("fleetId"), request.acquisitionStrategy());
jdbcTemplate.query(query.sql(), query.params(), rs -> { jdbcTemplate.query(loadSql(), params, rs -> {
stats.sourceRowsRead++; stats.sourceRowsRead++;
YellowFoxD8BookingDto booking = rowMapper.map( YellowFoxD8BookingDto booking = rowMapper.map(
rs, rs,
@ -136,33 +136,9 @@ public class JdbcYellowFoxD8BookingExtractionBatchExecutor implements YellowFoxD
); );
} }
QuerySpec buildQuerySpec(YellowFoxD8ImportRequest request, ImportScopeDto scope, ImportCursorStateDto cursor) { private ImportCursorStateDto findCursor(int eventSourceId, YellowFoxD8ImportRequest request, ImportPlanItemDto planItem) {
Map<String, Object> params = new HashMap<>();
StringBuilder filters = new StringBuilder("where 1 = 1");
if (scope != null && scope.occurredFrom() != null) {
params.put("occurredFrom", scope.occurredFrom());
filters.append("\n and b.utc >= :occurredFrom");
}
if (scope != null && scope.occurredTo() != null) {
params.put("occurredTo", scope.occurredTo());
filters.append("\n and b.utc < :occurredTo");
}
Integer fleetId = fleetId(request);
if (fleetId != null) {
params.put("fleetId", fleetId);
filters.append("\n and f.id = :fleetId");
}
appendCursorFilter(filters, params, request, cursor);
return new QuerySpec(applyFilters(loadSqlTemplate(), filters.toString()), params, fleetId);
}
ImportCursorStateDto findCursor(int eventSourceId, YellowFoxD8ImportRequest request, ImportPlanItemDto planItem) {
String scopeHash = request.importScope() == null ? "NO_SCOPE" : request.importScope().stableKey(); String scopeHash = request.importScope() == null ? "NO_SCOPE" : request.importScope().stableKey();
ImportCursorStateDto cursor = importCursorRepository.findCursor( return importCursorRepository.findCursor(
request.tenantKey(), request.tenantKey(),
eventSourceId, eventSourceId,
scopeHash, scopeHash,
@ -170,16 +146,28 @@ public class JdbcYellowFoxD8BookingExtractionBatchExecutor implements YellowFoxD
planItem.sourceKind(), planItem.sourceKind(),
request.acquisitionStrategy() request.acquisitionStrategy()
); );
if (cursor != null || !shouldBootstrapWatermarkCursor(request)) {
return cursor;
} }
return importCursorRepository.findLatestCursor(
request.tenantKey(), private Map<String, Object> parameters(YellowFoxD8ImportRequest request, ImportScopeDto scope, ImportCursorStateDto cursor) {
eventSourceId, Map<String, Object> params = new HashMap<>();
scopeHash, params.put("occurredFrom", scope == null ? null : scope.occurredFrom());
planItem.eventFamily(), params.put("occurredTo", scope == null ? null : scope.occurredTo());
planItem.sourceKind() params.put("fleetId", fleetId(request));
); if (request.acquisitionStrategy() == AcquisitionStrategy.SOURCE_ROW_WATERMARK && cursor != null) {
OffsetDateTime lastOccurredTo = cursor.lastOccurredTo();
String lastSourceRowId = cursor.lastSourcePackageId();
if (lastOccurredTo != null && properties.getYellowFox().getOccurredAtOverlap() != null
&& !properties.getYellowFox().getOccurredAtOverlap().isZero()) {
lastOccurredTo = lastOccurredTo.minus(properties.getYellowFox().getOccurredAtOverlap());
lastSourceRowId = null;
}
params.put("lastOccurredTo", lastOccurredTo);
params.put("lastSourceRowId", lastSourceRowId);
} else {
params.put("lastOccurredTo", null);
params.put("lastSourceRowId", null);
}
return params;
} }
private Integer fleetId(YellowFoxD8ImportRequest request) { private Integer fleetId(YellowFoxD8ImportRequest request) {
@ -216,59 +204,7 @@ public class JdbcYellowFoxD8BookingExtractionBatchExecutor implements YellowFoxD
stats.acceptEvent(event); stats.acceptEvent(event);
} }
private void appendCursorFilter( private String loadSql() {
StringBuilder filters,
Map<String, Object> params,
YellowFoxD8ImportRequest request,
ImportCursorStateDto cursor
) {
if (request.acquisitionStrategy() != AcquisitionStrategy.SOURCE_ROW_WATERMARK || cursor == null) {
return;
}
OffsetDateTime lastOccurredTo = cursor.lastOccurredTo();
String lastSourceRowId = cursor.lastSourcePackageId();
if (lastOccurredTo == null) {
return;
}
if (properties.getYellowFox().getOccurredAtOverlap() != null
&& !properties.getYellowFox().getOccurredAtOverlap().isZero()) {
lastOccurredTo = lastOccurredTo.minus(properties.getYellowFox().getOccurredAtOverlap());
lastSourceRowId = null;
}
params.put("lastOccurredTo", lastOccurredTo);
if (lastSourceRowId == null || lastSourceRowId.isBlank()) {
filters.append("\n and b.utc >= :lastOccurredTo");
return;
}
params.put("lastSourceRowId", lastSourceRowId);
filters.append("""
and (
b.utc > :lastOccurredTo
or (
b.utc = :lastOccurredTo
and b.eventid > :lastSourceRowId
)
)""");
}
private boolean shouldBootstrapWatermarkCursor(YellowFoxD8ImportRequest request) {
return request.mode() == at.procon.eventhub.dto.ImportMode.INCREMENTAL_UPDATE
&& (request.acquisitionStrategy() == AcquisitionStrategy.SOURCE_ROW_WATERMARK
|| request.acquisitionStrategy() == AcquisitionStrategy.SOURCE_PACKAGE_WATERMARK);
}
private String applyFilters(String sqlTemplate, String filters) {
if (!sqlTemplate.contains("/*__FILTERS__*/")) {
throw new IllegalStateException("YellowFox D8 extraction SQL template is missing filter marker");
}
return sqlTemplate.replace("/*__FILTERS__*/", filters);
}
private String loadSqlTemplate() {
Resource resource = resourceLoader.getResource("classpath:sql/yellowfox/d8-bookings.sql"); Resource resource = resourceLoader.getResource("classpath:sql/yellowfox/d8-bookings.sql");
try (var in = resource.getInputStream()) { try (var in = resource.getInputStream()) {
return StreamUtils.copyToString(in, StandardCharsets.UTF_8); return StreamUtils.copyToString(in, StandardCharsets.UTF_8);
@ -277,9 +213,6 @@ public class JdbcYellowFoxD8BookingExtractionBatchExecutor implements YellowFoxD
} }
} }
static record QuerySpec(String sql, Map<String, Object> params, Integer fleetId) {
}
private static class Stats { private static class Stats {
private int sourceRowsRead; private int sourceRowsRead;
private int eventsSent; private int eventsSent;

65
src/main/java/at/procon/eventhub/yellowfox/service/YellowFoxD8BookingRowMapper.java
View File

@ -28,43 +28,40 @@ public class YellowFoxD8BookingRowMapper {
} }
public YellowFoxD8BookingDto map(ResultSet rs, String tenantKey, String sourceInstanceKey, String tenantProviderSettingKey) throws SQLException { public YellowFoxD8BookingDto map(ResultSet rs, String tenantKey, String sourceInstanceKey, String tenantProviderSettingKey) throws SQLException {
String eventId = string(rs, "eventid"); String eventId = rs.getString("eventid");
OffsetDateTime occurredAt = rs.getObject("utc", OffsetDateTime.class); OffsetDateTime occurredAt = rs.getObject("utc", OffsetDateTime.class);
Integer vehicleId = getInteger(rs, "vehicle_id"); Integer vehicleId = getInteger(rs, "vehicle_id");
Integer driverId = getInteger(rs, "driver_id"); Integer driverId = getInteger(rs, "driver_id");
String vehicleVrn = string(rs, "vehicle_vrn"); String vehicleVrn = rs.getString("vehicle_vrn");
String vehicleVin = string(rs, "vehicle_vin"); String vehicleVin = rs.getString("vehicle_vin");
String driverCard = normalizeBookingDriverCardNumber(string(rs, "driver_card_number")); String driverCard = rs.getString("driver_card_number");
Integer fleetId = getInteger(rs, "fleet_id"); Integer fleetId = getInteger(rs, "fleet_id");
String fleetName = string(rs, "fleet_name"); String fleetName = rs.getString("fleet_name");
Integer odometer = getInteger(rs, "odometer"); Integer odometer = getInteger(rs, "odometer");
Map<String, Object> payload = trimPayloadStrings(payload(rs.getString("payload"))); Map<String, Object> payload = payload(rs.getString("payload"));
put(payload, "yellowFoxEventId", eventId); put(payload, "yellowFoxEventId", eventId);
put(payload, "yellowFoxOdometerRaw", odometer); put(payload, "yellowFoxOdometerRaw", odometer);
put(payload, "vehicleVrn", vehicleVrn); put(payload, "vehicleVrn", vehicleVrn);
put(payload, "vehicleVin", vehicleVin); put(payload, "vehicleVin", vehicleVin);
put(payload, "driverCardNumber", driverCard); put(payload, "driverCardNumber", driverCard);
put(payload, "driverFirstName", string(rs, "driver_firstname")); put(payload, "driverFirstName", rs.getString("driver_firstname"));
put(payload, "driverLastName", string(rs, "driver_lastname")); put(payload, "driverLastName", rs.getString("driver_lastname"));
put(payload, "fleetId", fleetId); put(payload, "fleetId", fleetId);
put(payload, "fleetName", fleetName); put(payload, "fleetName", fleetName);
put(payload, "telematicProviderId", getInteger(rs, "telematic_provider_id")); put(payload, "telematicProviderId", getInteger(rs, "telematic_provider_id"));
put(payload, "telematicProviderName", string(rs, "telematic_provider_name")); put(payload, "telematicProviderName", rs.getString("telematic_provider_name"));
DriverRefDto driverRef = driverId == null && isBlank(driverCard) DriverRefDto driverRef = driverId == null && isBlank(driverCard)
? null ? null
: new DriverRefDto( : new DriverRefDto(driverId == null ? null : driverId.toString(), new DriverCardRefDto(null, driverCard));
driverId == null ? null : driverId.toString(),
new DriverCardRefDto(YellowFoxReferenceSemantics.SYNTHETIC_REFERENCE_NATION, driverCard)
);
VehicleRefDto vehicleRef = vehicleId == null && isBlank(vehicleVin) && isBlank(vehicleVrn) VehicleRefDto vehicleRef = vehicleId == null && isBlank(vehicleVin) && isBlank(vehicleVrn)
? null ? null
: new VehicleRefDto( : new VehicleRefDto(
vehicleId == null ? null : vehicleId.toString(), vehicleId == null ? null : vehicleId.toString(),
vehicleVin, vehicleVin,
vehicleId == null ? null : vehicleId.toString(), vehicleId == null ? null : vehicleId.toString(),
new VehicleRegistrationRefDto(YellowFoxReferenceSemantics.SYNTHETIC_REFERENCE_NATION, vehicleVrn) new VehicleRegistrationRefDto(null, vehicleVrn)
); );
return new YellowFoxD8BookingDto( return new YellowFoxD8BookingDto(
@ -74,9 +71,8 @@ public class YellowFoxD8BookingRowMapper {
fleetId == null ? null : fleetId.toString(), fleetId == null ? null : fleetId.toString(),
fleetName, fleetName,
eventId, eventId,
string(rs, "key"), rs.getString("key"),
getInteger(rs, "ignition"), getInteger(rs, "ignition"),
getInteger(rs, "previous_ignition"),
getInteger(rs, "eventtype"), getInteger(rs, "eventtype"),
getInteger(rs, "state"), getInteger(rs, "state"),
occurredAt, occurredAt,
@ -109,35 +105,6 @@ public class YellowFoxD8BookingRowMapper {
} }
} }
private String string(ResultSet rs, String column) throws SQLException {
String value = rs.getString(column);
return value == null || value.isBlank() ? null : value.trim();
}
@SuppressWarnings("unchecked")
private Map<String, Object> trimPayloadStrings(Map<String, Object> payload) {
payload.replaceAll((key, value) -> trimPayloadValue(value));
return payload;
}
@SuppressWarnings("unchecked")
private Object trimPayloadValue(Object value) {
if (value instanceof String text) {
return text.trim();
}
if (value instanceof Map<?, ?> nestedMap) {
((Map<Object, Object>) nestedMap).replaceAll((key, nestedValue) -> trimPayloadValue(nestedValue));
return nestedMap;
}
if (value instanceof java.util.List<?> list) {
for (int i = 0; i < list.size(); i++) {
((java.util.List<Object>) list).set(i, trimPayloadValue(list.get(i)));
}
return list;
}
return value;
}
private void put(Map<String, Object> target, String key, Object value) { private void put(Map<String, Object> target, String key, Object value) {
if (value != null) { if (value != null) {
target.put(key, value instanceof BigDecimal bd ? bd.stripTrailingZeros().toPlainString() : value); target.put(key, value instanceof BigDecimal bd ? bd.stripTrailingZeros().toPlainString() : value);
@ -147,12 +114,4 @@ public class YellowFoxD8BookingRowMapper {
private boolean isBlank(String value) { private boolean isBlank(String value) {
return value == null || value.isBlank(); return value == null || value.isBlank();
} }
private String normalizeBookingDriverCardNumber(String value) {
if (value == null || value.isBlank()) {
return null;
}
String trimmed = value.trim();
return trimmed.length() <= 14 ? trimmed : trimmed.substring(0, 14);
}
} }

4
src/main/java/at/procon/eventhub/yellowfox/service/YellowFoxD8ConfiguredImportPlanService.java
View File

@ -32,7 +32,7 @@ public class YellowFoxD8ConfiguredImportPlanService
scope, scope,
plan.getEventFamilies(), plan.getEventFamilies(),
mode, mode,
plan.isRefreshMasterDataFirst(), false,
strategy strategy
); );
} }
@ -53,7 +53,7 @@ public class YellowFoxD8ConfiguredImportPlanService
dto.scheduledMode(), dto.scheduledMode(),
dto.initialStrategy(), dto.initialStrategy(),
dto.scheduledStrategy(), dto.scheduledStrategy(),
dto.refreshMasterDataFirst(), false,
dto.initialOccurredFrom(), dto.initialOccurredFrom(),
dto.initialOccurredTo(), dto.initialOccurredTo(),
dto.runInitialOnStartup() dto.runInitialOnStartup()

3
src/main/java/at/procon/eventhub/yellowfox/service/YellowFoxD8IgnitionTransitionDetector.java
View File

@ -35,12 +35,9 @@ public class YellowFoxD8IgnitionTransitionDetector {
} }
String vehicleKey = booking.vehicleRef().stableKey(); String vehicleKey = booking.vehicleRef().stableKey();
Integer previous = lastIgnitionByVehicle.put(vehicleKey, booking.ignition()); Integer previous = lastIgnitionByVehicle.put(vehicleKey, booking.ignition());
if (previous == null) {
previous = booking.previousIgnition();
if (previous == null) { if (previous == null) {
return emitInitialSnapshot ? mapper.mapIgnitionTransition(booking, null) : null; return emitInitialSnapshot ? mapper.mapIgnitionTransition(booking, null) : null;
} }
}
if (!previous.equals(booking.ignition())) { if (!previous.equals(booking.ignition())) {
return mapper.mapIgnitionTransition(booking, previous); return mapper.mapIgnitionTransition(booking, previous);
} }

8
src/main/java/at/procon/eventhub/yellowfox/service/YellowFoxD8ImportExecutionService.java
View File

@ -23,7 +23,6 @@ public class YellowFoxD8ImportExecutionService
extends AbstractImportExecutionService<YellowFoxD8ImportRequest, YellowFoxD8ExtractionBatchResultDto> { extends AbstractImportExecutionService<YellowFoxD8ImportRequest, YellowFoxD8ExtractionBatchResultDto> {
private final YellowFoxD8ImportPlanService planService; private final YellowFoxD8ImportPlanService planService;
private final YellowFoxMasterDataRefreshService masterDataRefreshService;
private final YellowFoxD8ExtractionBatchExecutor extractionBatchExecutor; private final YellowFoxD8ExtractionBatchExecutor extractionBatchExecutor;
public YellowFoxD8ImportExecutionService( public YellowFoxD8ImportExecutionService(
@ -32,12 +31,10 @@ public class YellowFoxD8ImportExecutionService
ImportRunRepository importRunRepository, ImportRunRepository importRunRepository,
DataPackageRepository dataPackageRepository, DataPackageRepository dataPackageRepository,
ImportCursorRepository importCursorRepository, ImportCursorRepository importCursorRepository,
YellowFoxMasterDataRefreshService masterDataRefreshService,
YellowFoxD8ExtractionBatchExecutor extractionBatchExecutor YellowFoxD8ExtractionBatchExecutor extractionBatchExecutor
) { ) {
super(eventSourceRepository, importRunRepository, dataPackageRepository, importCursorRepository); super(eventSourceRepository, importRunRepository, dataPackageRepository, importCursorRepository);
this.planService = planService; this.planService = planService;
this.masterDataRefreshService = masterDataRefreshService;
this.extractionBatchExecutor = extractionBatchExecutor; this.extractionBatchExecutor = extractionBatchExecutor;
} }
@ -67,11 +64,6 @@ public class YellowFoxD8ImportExecutionService
return extractionBatchExecutor.execute(importRunId, packageId, eventSourceId, request, planItem, chunk); return extractionBatchExecutor.execute(importRunId, packageId, eventSourceId, request, planItem, chunk);
} }
@Override
protected void beforeExecute(YellowFoxD8ImportRequest request) {
masterDataRefreshService.refreshIfRequested(request);
}
@Override @Override
protected EventSourceDto eventSourceForItem(EventSourceDto base, ImportPlanItemDto item) { protected EventSourceDto eventSourceForItem(EventSourceDto base, ImportPlanItemDto item) {
return new EventSourceDto( return new EventSourceDto(

335
src/main/java/at/procon/eventhub/yellowfox/service/YellowFoxMasterDataRefreshService.java
View File

@ -1,335 +0,0 @@
package at.procon.eventhub.yellowfox.service;
import at.procon.eventhub.config.EventHubProperties;
import at.procon.eventhub.dto.EventSourceDto;
import at.procon.eventhub.importing.masterdata.MasterDataRefreshResult;
import at.procon.eventhub.importing.masterdata.SourceMasterEntityUpsert;
import at.procon.eventhub.importing.masterdata.SourceMasterRelationUpsert;
import at.procon.eventhub.persistence.DriverIdentityRepository;
import at.procon.eventhub.persistence.EventSourceRepository;
import at.procon.eventhub.persistence.SourceMasterDataRepository;
import at.procon.eventhub.persistence.VehicleIdentityRepository;
import at.procon.eventhub.yellowfox.dto.YellowFoxD8ImportRequest;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.sql.ResultSet;
import java.sql.ResultSetMetaData;
import java.sql.SQLException;
import java.sql.Timestamp;
import java.time.LocalDateTime;
import java.time.OffsetDateTime;
import java.time.ZoneOffset;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.ObjectProvider;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.core.io.Resource;
import org.springframework.core.io.ResourceLoader;
import org.springframework.jdbc.core.namedparam.NamedParameterJdbcTemplate;
import org.springframework.stereotype.Service;
import org.springframework.util.StreamUtils;
@Service
public class YellowFoxMasterDataRefreshService {
private static final Logger log = LoggerFactory.getLogger(YellowFoxMasterDataRefreshService.class);
private static final List<String> ENTITY_SQL_RESOURCES = List.of(
"classpath:sql/yellowfox/master-data/fleets.sql",
"classpath:sql/yellowfox/master-data/drivers.sql",
"classpath:sql/yellowfox/master-data/driver-cards.sql",
"classpath:sql/yellowfox/master-data/vehicles.sql",
"classpath:sql/yellowfox/master-data/vehicle-registrations.sql",
"classpath:sql/yellowfox/master-data/telematic-providers.sql"
);
private static final String RELATIONS_SQL_RESOURCE = "classpath:sql/yellowfox/master-data/relations.sql";
private static final String MASTER_DATA_SOURCE_KIND = "MASTER_DATA";
private static final String MASTER_DATA_SOURCE_KEY = "YELLOWFOX_MASTER_DATA";
private static final int MASTER_DATA_UPSERT_BATCH_SIZE = 5000;
private final ObjectProvider<NamedParameterJdbcTemplate> yellowFoxJdbcTemplateProvider;
private final SourceMasterDataRepository sourceMasterDataRepository;
private final EventSourceRepository eventSourceRepository;
private final DriverIdentityRepository driverIdentityRepository;
private final VehicleIdentityRepository vehicleIdentityRepository;
private final ResourceLoader resourceLoader;
private final EventHubProperties properties;
public YellowFoxMasterDataRefreshService(
@Qualifier("yellowFoxNamedParameterJdbcTemplate") ObjectProvider<NamedParameterJdbcTemplate> yellowFoxJdbcTemplateProvider,
SourceMasterDataRepository sourceMasterDataRepository,
EventSourceRepository eventSourceRepository,
DriverIdentityRepository driverIdentityRepository,
VehicleIdentityRepository vehicleIdentityRepository,
ResourceLoader resourceLoader,
EventHubProperties properties
) {
this.yellowFoxJdbcTemplateProvider = yellowFoxJdbcTemplateProvider;
this.sourceMasterDataRepository = sourceMasterDataRepository;
this.eventSourceRepository = eventSourceRepository;
this.driverIdentityRepository = driverIdentityRepository;
this.vehicleIdentityRepository = vehicleIdentityRepository;
this.resourceLoader = resourceLoader;
this.properties = properties;
}
public MasterDataRefreshResult refreshIfRequested(YellowFoxD8ImportRequest request) {
if (!request.refreshMasterDataFirst()) {
return MasterDataRefreshResult.empty();
}
return refresh(request);
}
public MasterDataRefreshResult refresh(YellowFoxD8ImportRequest request) {
NamedParameterJdbcTemplate yellowFoxJdbcTemplate = yellowFoxJdbcTemplateProvider.getIfAvailable();
if (yellowFoxJdbcTemplate == null) {
log.info("Skipping YellowFox master-data refresh for tenant={} because no YellowFox datasource is configured.",
request.tenantKey());
return MasterDataRefreshResult.empty();
}
String tenantKey = request.tenantKey() == null || request.tenantKey().isBlank() ? "default" : request.tenantKey().trim();
EventSourceDto masterDataSource = masterDataSourceFor(request.eventSource());
int eventSourceId = eventSourceRepository.resolveSourceId(tenantKey, masterDataSource);
log.info("Starting YellowFox source master-data refresh tenant={} source={} batchSize={}",
tenantKey, masterDataSource.stableKey(), MASTER_DATA_UPSERT_BATCH_SIZE);
int entities = 0;
for (String sqlResource : ENTITY_SQL_RESOURCES) {
entities += streamEntities(yellowFoxJdbcTemplate, tenantKey, eventSourceId, sqlResource, loadSql(sqlResource));
}
int relationCount = streamRelations(yellowFoxJdbcTemplate, tenantKey, eventSourceId, RELATIONS_SQL_RESOURCE, loadSql(RELATIONS_SQL_RESOURCE));
boolean syncVehicleRegistrations = properties.getYellowFox().isSyncVehicleRegistrationsOnMasterDataUpdate();
log.info("Reconciling YellowFox driver identities from source master data tenant={} source={}",
tenantKey, masterDataSource.stableKey());
int reconciledDrivers = driverIdentityRepository.reconcileFromMasterData(tenantKey, eventSourceId);
log.info("Reconciling YellowFox vehicle identities from source master data tenant={} source={} syncVehicleRegistrations={}",
tenantKey, masterDataSource.stableKey(), syncVehicleRegistrations);
int reconciledVehicles = vehicleIdentityRepository.reconcileFromMasterData(
tenantKey,
eventSourceId,
syncVehicleRegistrations
);
MasterDataRefreshResult result = new MasterDataRefreshResult(entities, relationCount);
log.info("Refreshed YellowFox source master data tenant={} source={} entities={} relations={} reconciledDrivers={} reconciledVehicles={}",
tenantKey, masterDataSource.stableKey(), result.entitiesUpserted(), result.relationsUpserted(), reconciledDrivers, reconciledVehicles);
return result;
}
private int streamEntities(
NamedParameterJdbcTemplate yellowFoxJdbcTemplate,
String tenantKey,
int eventSourceId,
String sqlResource,
String sql
) {
String section = masterDataSection(sqlResource);
List<SourceMasterEntityUpsert> buffer = new ArrayList<>(MASTER_DATA_UPSERT_BATCH_SIZE);
Map<String, Integer> typeCounts = new LinkedHashMap<>();
int[] count = {0};
int[] chunk = {0};
log.info("Reading YellowFox master-data entities tenant={} section={}", tenantKey, section);
yellowFoxJdbcTemplate.query(sql, Map.of(), rs -> {
SourceMasterEntityUpsert entity = entity(rs);
buffer.add(entity);
increment(typeCounts, entity.entityType());
if (buffer.size() >= MASTER_DATA_UPSERT_BATCH_SIZE) {
count[0] += flushEntities(tenantKey, eventSourceId, section, ++chunk[0], buffer, count[0], typeCounts);
buffer.clear();
}
});
if (!buffer.isEmpty()) {
count[0] += flushEntities(tenantKey, eventSourceId, section, ++chunk[0], buffer, count[0], typeCounts);
}
log.info("Finished YellowFox master-data entities tenant={} section={} processed={} byType={}",
tenantKey, section, count[0], typeCounts);
return count[0];
}
private int streamRelations(
NamedParameterJdbcTemplate yellowFoxJdbcTemplate,
String tenantKey,
int eventSourceId,
String sqlResource,
String sql
) {
String section = masterDataSection(sqlResource);
List<SourceMasterRelationUpsert> buffer = new ArrayList<>(MASTER_DATA_UPSERT_BATCH_SIZE);
Map<String, Integer> typeCounts = new LinkedHashMap<>();
int[] count = {0};
int[] chunk = {0};
log.info("Reading YellowFox master-data relations tenant={} section={}", tenantKey, section);
yellowFoxJdbcTemplate.query(sql, Map.of(), rs -> {
SourceMasterRelationUpsert relation = relation(rs);
buffer.add(relation);
increment(typeCounts, relation.relationType());
if (buffer.size() >= MASTER_DATA_UPSERT_BATCH_SIZE) {
count[0] += flushRelations(tenantKey, eventSourceId, section, ++chunk[0], buffer, count[0], typeCounts);
buffer.clear();
}
});
if (!buffer.isEmpty()) {
count[0] += flushRelations(tenantKey, eventSourceId, section, ++chunk[0], buffer, count[0], typeCounts);
}
log.info("Finished YellowFox master-data relations tenant={} section={} processed={} byType={}",
tenantKey, section, count[0], typeCounts);
return count[0];
}
private int flushEntities(
String tenantKey,
int eventSourceId,
String section,
int chunk,
List<SourceMasterEntityUpsert> buffer,
int alreadyProcessed,
Map<String, Integer> typeCounts
) {
int upserted = sourceMasterDataRepository.upsertEntities(tenantKey, eventSourceId, buffer);
log.info("YellowFox master-data entity progress tenant={} section={} chunk={} chunkSize={} processed={} byType={}",
tenantKey, section, chunk, buffer.size(), alreadyProcessed + upserted, typeCounts);
return upserted;
}
private int flushRelations(
String tenantKey,
int eventSourceId,
String section,
int chunk,
List<SourceMasterRelationUpsert> buffer,
int alreadyProcessed,
Map<String, Integer> typeCounts
) {
int upserted = sourceMasterDataRepository.upsertRelations(tenantKey, eventSourceId, buffer);
log.info("YellowFox master-data relation progress tenant={} section={} chunk={} chunkSize={} processed={} byType={}",
tenantKey, section, chunk, buffer.size(), alreadyProcessed + upserted, typeCounts);
return upserted;
}
private void increment(Map<String, Integer> counts, String type) {
String key = type == null || type.isBlank() ? "UNKNOWN" : type;
counts.merge(key, 1, Integer::sum);
}
private String masterDataSection(String sqlResource) {
int slash = sqlResource.lastIndexOf('/');
String fileName = slash < 0 ? sqlResource : sqlResource.substring(slash + 1);
return fileName.endsWith(".sql") ? fileName.substring(0, fileName.length() - 4) : fileName;
}
private EventSourceDto masterDataSourceFor(EventSourceDto source) {
return new EventSourceDto(
source.providerKey(),
MASTER_DATA_SOURCE_KIND,
MASTER_DATA_SOURCE_KEY,
source.sourceInstanceKey(),
source.tenantProviderSettingKey(),
source.externalFleetKey()
);
}
private SourceMasterEntityUpsert entity(ResultSet rs) throws SQLException {
String entityType = string(rs, "entity_type");
String sourceEntityId = string(rs, "source_entity_id");
return new SourceMasterEntityUpsert(
entityType,
sourceEntityId,
string(rs, "source_external_key"),
string(rs, "display_name"),
bool(rs, "active"),
offsetDateTime(rs, "valid_from"),
offsetDateTime(rs, "valid_to"),
offsetDateTime(rs, "source_updated_at"),
payload(rs)
);
}
private SourceMasterRelationUpsert relation(ResultSet rs) throws SQLException {
return new SourceMasterRelationUpsert(
string(rs, "relation_type"),
string(rs, "from_entity_type"),
string(rs, "from_source_entity_id"),
string(rs, "to_entity_type"),
string(rs, "to_source_entity_id"),
offsetDateTime(rs, "valid_from"),
offsetDateTime(rs, "valid_to"),
offsetDateTime(rs, "source_updated_at"),
payload(rs)
);
}
private Map<String, Object> payload(ResultSet rs) throws SQLException {
ResultSetMetaData metaData = rs.getMetaData();
Map<String, Object> payload = new LinkedHashMap<>();
for (int i = 1; i <= metaData.getColumnCount(); i++) {
String name = metaData.getColumnLabel(i);
Object value = rs.getObject(i);
if (value != null) {
payload.put(name, value);
}
}
return payload;
}
private String loadSql(String location) {
Resource resource = resourceLoader.getResource(location);
try (var inputStream = resource.getInputStream()) {
return StreamUtils.copyToString(inputStream, StandardCharsets.UTF_8);
} catch (IOException e) {
throw new IllegalStateException("Cannot load YellowFox master-data SQL resource " + location, e);
}
}
private String string(ResultSet rs, String column) throws SQLException {
String value = rs.getString(column);
return value == null || value.isBlank() ? null : value.trim();
}
private Boolean bool(ResultSet rs, String column) throws SQLException {
Object value = rs.getObject(column);
if (value == null) {
return null;
}
if (value instanceof Boolean bool) {
return bool;
}
if (value instanceof Number number) {
return number.intValue() != 0;
}
return Boolean.parseBoolean(value.toString());
}
private OffsetDateTime offsetDateTime(ResultSet rs, String column) throws SQLException {
Object value = rs.getObject(column);
if (value == null) {
return null;
}
if (value instanceof OffsetDateTime offsetDateTime) {
return offsetDateTime.withOffsetSameInstant(ZoneOffset.UTC);
}
if (value instanceof Timestamp timestamp) {
return timestamp.toLocalDateTime().atOffset(ZoneOffset.UTC);
}
if (value instanceof LocalDateTime localDateTime) {
return localDateTime.atOffset(ZoneOffset.UTC);
}
String text = value.toString();
try {
return OffsetDateTime.parse(text).withOffsetSameInstant(ZoneOffset.UTC);
} catch (RuntimeException ignored) {
return LocalDateTime.parse(text).atOffset(ZoneOffset.UTC);
}
}
}

9
src/main/java/at/procon/eventhub/yellowfox/service/YellowFoxReferenceSemantics.java
View File

@ -1,9 +0,0 @@
package at.procon.eventhub.yellowfox.service;
final class YellowFoxReferenceSemantics {
static final String SYNTHETIC_REFERENCE_NATION = "YELLOWFOX";
private YellowFoxReferenceSemantics() {
}
}

34
src/main/resources/application.yml
View File

@ -53,7 +53,6 @@ eventhub:
tachograph: tachograph:
default-chunk-days: 1 default-chunk-days: 1
occurred-at-overlap: 7d occurred-at-overlap: 7d
sync-vehicle-registrations-on-master-data-update: true
# Set TACHOGRAPH_DB_JDBC_URL to enable JdbcTachographExtractionBatchExecutor. # Set TACHOGRAPH_DB_JDBC_URL to enable JdbcTachographExtractionBatchExecutor.
datasource: datasource:
@ -64,7 +63,7 @@ eventhub:
# Enables the scheduler that regularly triggers configured tachograph import plans. # Enables the scheduler that regularly triggers configured tachograph import plans.
# Default is safe: no scheduled import starts unless explicitly enabled. # Default is safe: no scheduled import starts unless explicitly enabled.
scheduler-enabled: false scheduler-enabled: true
scheduler-poll-interval-ms: 3600000 scheduler-poll-interval-ms: 3600000
# PLAN_ONLY creates import_run + planned extraction packages. # PLAN_ONLY creates import_run + planned extraction packages.
@ -79,7 +78,7 @@ eventhub:
# Example plan. Keep disabled until the tachograph datasource/extractor is wired. # Example plan. Keep disabled until the tachograph datasource/extractor is wired.
import-plans: import-plans:
- plan-key: tachograph-org-14708 - plan-key: tachograph-org-14708
enabled: false enabled: true
cron: "0 15 * * * *" # hourly at minute 15 cron: "0 15 * * * *" # hourly at minute 15
tenant-key: Procon tenant-key: Procon
event-source: event-source:
@ -116,10 +115,10 @@ eventhub:
scheduled-mode: INCREMENTAL_UPDATE scheduled-mode: INCREMENTAL_UPDATE
initial-strategy: OCCURRED_AT_WINDOW_WITH_OVERLAP initial-strategy: OCCURRED_AT_WINDOW_WITH_OVERLAP
scheduled-strategy: SOURCE_PACKAGE_WATERMARK scheduled-strategy: SOURCE_PACKAGE_WATERMARK
refresh-master-data-first: false refresh-master-data-first: true
initial-occurred-from: "2026-04-01T00:00:00+01:00" initial-occurred-from: "2026-01-21T00:00:00+01:00"
initial-occurred-to: "2026-04-10T00:00:00+01:00" initial-occurred-to:
run-initial-on-startup: false run-initial-on-startup: true
esper-poc: esper-poc:
activity-merge-mode: JAVA activity-merge-mode: JAVA
@ -130,13 +129,11 @@ eventhub:
merge-gap-seconds: 0 merge-gap-seconds: 0
gap-detection-tolerance-seconds: 0 gap-detection-tolerance-seconds: 0
unknown-treatment-mode: AS_BREAK_REST unknown-treatment-mode: AS_BREAK_REST
engine-mode: STREAM_COLLECTOR
yellow-fox: yellow-fox:
default-chunk-days: 1 default-chunk-days: 1
occurred-at-overlap: 2h occurred-at-overlap: 2h
emit-initial-ignition-snapshot: false emit-initial-ignition-snapshot: false
sync-vehicle-registrations-on-master-data-update: false
datasource: datasource:
jdbc-url: ${YELLOWFOX_DB_JDBC_URL:} jdbc-url: ${YELLOWFOX_DB_JDBC_URL:}
@ -144,24 +141,21 @@ eventhub:
password: ${YELLOWFOX_DB_PASSWORD:} password: ${YELLOWFOX_DB_PASSWORD:}
driver-class-name: org.postgresql.Driver driver-class-name: org.postgresql.Driver
scheduler-enabled: true scheduler-enabled: false
scheduler-poll-interval-ms: 60000 scheduler-poll-interval-ms: 60000
scheduler-trigger-mode: EXECUTE scheduler-trigger-mode: PLAN_ONLY
import-plans: import-plans:
- plan-key: yellowfox-d8-default - plan-key: yellowfox-d8-default
enabled: true enabled: false
cron: "0 */5 * * * *" cron: "0 */5 * * * *"
tenant-key: Procon tenant-key: default
event-source: event-source:
provider-key: YELLOWFOX provider-key: YELLOWFOX
source-kind: TELEMATICS_PLATFORM source-kind: TELEMATICS_PLATFORM
source-key: YELLOWFOX_D8 source-key: YELLOWFOX_D8
source-instance-key: logistics-db-prod source-instance-key: logistics-db-prod
tenant-provider-setting-key: yellowfox-main tenant-provider-setting-key: yellowfox-main
source-group:
type: FLEET
source-entity-id: "7"
import-scope: import-scope:
type: TENANT_ALL type: TENANT_ALL
include-children: false include-children: false
@ -169,10 +163,8 @@ eventhub:
- DRIVER_ACTIVITY - DRIVER_ACTIVITY
- DRIVER_CARD - DRIVER_CARD
initial-mode: INITIAL_BACKFILL initial-mode: INITIAL_BACKFILL
scheduled-mode: INITIAL_BACKFILL # INITIAL_BACKFILL, INCREMENTAL_UPDATE scheduled-mode: INCREMENTAL_UPDATE
initial-strategy: OCCURRED_AT_WINDOW_WITH_OVERLAP initial-strategy: OCCURRED_AT_WINDOW_WITH_OVERLAP
scheduled-strategy: SOURCE_ROW_WATERMARK scheduled-strategy: SOURCE_ROW_WATERMARK
initial-occurred-from: "2026-04-01T00:00:00+01:00" refresh-master-data-first: false
initial-occurred-to: "2026-04-10T00:00:00+01:00" run-initial-on-startup: false
refresh-master-data-first: true
run-initial-on-startup: true

154
src/main/resources/db/eventhub_schema_create.sql
View File

@ -145,56 +145,11 @@ create table if not exists eventhub.source_master_relation (
constraint chk_source_master_relation_valid_time_order check (valid_from is null or valid_to is null or valid_from <= valid_to) constraint chk_source_master_relation_valid_time_order check (valid_from is null or valid_to is null or valid_from <= valid_to)
); );
create table if not exists eventhub.driver (
id uuid primary key,
first_names text,
last_name text,
birth_date date,
source_updated_at timestamptz,
payload jsonb not null default '{}'::jsonb,
created_at timestamptz not null default now(),
updated_at timestamptz not null default now()
);
create table if not exists eventhub.driver_card (
id uuid primary key,
driver_id uuid references eventhub.driver(id),
nation text not null,
card_number text not null,
source_updated_at timestamptz,
payload jsonb not null default '{}'::jsonb,
created_at timestamptz not null default now(),
updated_at timestamptz not null default now()
);
create table if not exists eventhub.source_driver_identity (
id uuid primary key,
tenant_key text not null,
event_source_id integer not null references eventhub.event_source(id),
source_driver_entity_id text not null,
driver_id uuid not null references eventhub.driver(id) on delete cascade,
source_updated_at timestamptz,
payload jsonb not null default '{}'::jsonb,
created_at timestamptz not null default now(),
updated_at timestamptz not null default now(),
constraint ux_source_driver_identity unique (tenant_key, event_source_id, source_driver_entity_id)
);
create table if not exists eventhub.source_driver_card_identity (
id uuid primary key,
tenant_key text not null,
event_source_id integer not null references eventhub.event_source(id),
source_driver_card_entity_id text not null,
driver_card_id uuid not null references eventhub.driver_card(id) on delete cascade,
source_updated_at timestamptz,
payload jsonb not null default '{}'::jsonb,
created_at timestamptz not null default now(),
updated_at timestamptz not null default now(),
constraint ux_source_driver_card_identity unique (tenant_key, event_source_id, source_driver_card_entity_id)
);
create table if not exists eventhub.vehicle ( create table if not exists eventhub.vehicle (
id uuid primary key, id uuid primary key,
tenant_key text not null,
event_source_id integer not null references eventhub.event_source(id),
source_vehicle_entity_id text,
vin text, vin text,
created_at timestamptz not null default now(), created_at timestamptz not null default now(),
updated_at timestamptz not null default now() updated_at timestamptz not null default now()
@ -202,6 +157,9 @@ create table if not exists eventhub.vehicle (
create table if not exists eventhub.vehicle_registration ( create table if not exists eventhub.vehicle_registration (
id uuid primary key, id uuid primary key,
tenant_key text not null,
event_source_id integer not null references eventhub.event_source(id),
source_registration_entity_id text,
nation text not null, nation text not null,
registration_number text not null, registration_number text not null,
source_updated_at timestamptz, source_updated_at timestamptz,
@ -210,32 +168,6 @@ create table if not exists eventhub.vehicle_registration (
updated_at timestamptz not null default now() updated_at timestamptz not null default now()
); );
create table if not exists eventhub.source_vehicle_identity (
id uuid primary key,
tenant_key text not null,
event_source_id integer not null references eventhub.event_source(id),
source_vehicle_entity_id text not null,
vehicle_id uuid not null references eventhub.vehicle(id) on delete cascade,
source_updated_at timestamptz,
payload jsonb not null default '{}'::jsonb,
created_at timestamptz not null default now(),
updated_at timestamptz not null default now(),
constraint ux_source_vehicle_identity unique (tenant_key, event_source_id, source_vehicle_entity_id)
);
create table if not exists eventhub.source_vehicle_registration_identity (
id uuid primary key,
tenant_key text not null,
event_source_id integer not null references eventhub.event_source(id),
source_registration_entity_id text not null,
vehicle_registration_id uuid not null references eventhub.vehicle_registration(id) on delete cascade,
source_updated_at timestamptz,
payload jsonb not null default '{}'::jsonb,
created_at timestamptz not null default now(),
updated_at timestamptz not null default now(),
constraint ux_source_vehicle_registration_identity unique (tenant_key, event_source_id, source_registration_entity_id)
);
create table if not exists eventhub.vehicle_registration_assignment ( create table if not exists eventhub.vehicle_registration_assignment (
id uuid primary key, id uuid primary key,
tenant_key text not null, tenant_key text not null,
@ -256,8 +188,7 @@ create table if not exists eventhub.event (
event_source_id integer not null references eventhub.event_source(id), event_source_id integer not null references eventhub.event_source(id),
data_package_id uuid not null references eventhub.data_package(id), data_package_id uuid not null references eventhub.data_package(id),
external_source_event_id text not null, external_source_event_id text not null,
driver_id uuid references eventhub.driver(id), driver_entity_id uuid references eventhub.source_master_entity(id),
driver_card_id uuid references eventhub.driver_card(id),
vehicle_id uuid references eventhub.vehicle(id), vehicle_id uuid references eventhub.vehicle(id),
vehicle_registration_id uuid references eventhub.vehicle_registration(id), vehicle_registration_id uuid references eventhub.vehicle_registration(id),
source_package_id text, source_package_id text,
@ -277,8 +208,7 @@ create table if not exists eventhub.event (
created_at timestamptz not null default now(), created_at timestamptz not null default now(),
constraint pk_event primary key (occurred_at, id), constraint pk_event primary key (occurred_at, id),
constraint chk_event_driver_or_vehicle_ref check ( constraint chk_event_driver_or_vehicle_ref check (
driver_id is not null driver_entity_id is not null
or driver_card_id is not null
or vehicle_id is not null or vehicle_id is not null
or vehicle_registration_id is not null or vehicle_registration_id is not null
) )
@ -346,12 +276,23 @@ create index if not exists idx_source_master_relation_to
create index if not exists idx_source_master_relation_payload_gin create index if not exists idx_source_master_relation_payload_gin
on eventhub.source_master_relation using gin(payload); on eventhub.source_master_relation using gin(payload);
create index if not exists idx_vehicle_lookup_ctx
on eventhub.vehicle(tenant_key, event_source_id, updated_at desc);
create index if not exists idx_vehicle_source_entity
on eventhub.vehicle(tenant_key, event_source_id, source_vehicle_entity_id)
where source_vehicle_entity_id is not null;
create index if not exists idx_vehicle_vin create index if not exists idx_vehicle_vin
on eventhub.vehicle(vin) on eventhub.vehicle(tenant_key, event_source_id, vin)
where vin is not null; where vin is not null;
create index if not exists idx_vehicle_registration_source_entity
on eventhub.vehicle_registration(tenant_key, event_source_id, source_registration_entity_id)
where source_registration_entity_id is not null;
create index if not exists idx_vehicle_registration_plate create index if not exists idx_vehicle_registration_plate
on eventhub.vehicle_registration(nation, registration_number); on eventhub.vehicle_registration(tenant_key, event_source_id, nation, registration_number);
create index if not exists idx_vehicle_registration_assignment_registration_time create index if not exists idx_vehicle_registration_assignment_registration_time
on eventhub.vehicle_registration_assignment(vehicle_registration_id, valid_from desc, valid_to); on eventhub.vehicle_registration_assignment(vehicle_registration_id, valid_from desc, valid_to);
@ -379,42 +320,9 @@ create index if not exists idx_event_source_package_id
create index if not exists idx_event_domain_type_time create index if not exists idx_event_domain_type_time
on eventhub.event(event_domain, event_type, occurred_at desc); on eventhub.event(event_domain, event_type, occurred_at desc);
create index if not exists idx_driver_card_key
on eventhub.driver_card(nation, card_number);
create index if not exists idx_driver_card_driver
on eventhub.driver_card(driver_id)
where driver_id is not null;
create unique index if not exists ux_driver_card_key
on eventhub.driver_card(nation, card_number);
create unique index if not exists ux_vehicle_vin
on eventhub.vehicle(vin)
where vin is not null;
create unique index if not exists ux_vehicle_registration_plate
on eventhub.vehicle_registration(nation, registration_number);
create index if not exists idx_source_driver_identity_driver
on eventhub.source_driver_identity(driver_id);
create index if not exists idx_source_driver_card_identity_card
on eventhub.source_driver_card_identity(driver_card_id);
create index if not exists idx_source_vehicle_identity_vehicle
on eventhub.source_vehicle_identity(vehicle_id);
create index if not exists idx_source_vehicle_registration_identity_registration
on eventhub.source_vehicle_registration_identity(vehicle_registration_id);
create index if not exists idx_event_driver_time create index if not exists idx_event_driver_time
on eventhub.event(driver_id, occurred_at desc) on eventhub.event(driver_entity_id, occurred_at desc)
where driver_id is not null; where driver_entity_id is not null;
create index if not exists idx_event_driver_card_time
on eventhub.event(driver_card_id, occurred_at desc)
where driver_card_id is not null;
create index if not exists idx_event_vehicle_time create index if not exists idx_event_vehicle_time
on eventhub.event(vehicle_id, occurred_at desc) on eventhub.event(vehicle_id, occurred_at desc)
@ -436,19 +344,3 @@ create index if not exists idx_event_detail_type
create index if not exists idx_event_detail_attributes_gin create index if not exists idx_event_detail_attributes_gin
on eventhub.event_detail using gin(attributes); on eventhub.event_detail using gin(attributes);
create index if not exists idx_event_detail_yellowfox_slot
on eventhub.event_detail(detail_type, (attributes ->> 'slot'), event_occurred_at)
where detail_type in ('DRIVER_ACTIVITY', 'DRIVER_CARD');
create index if not exists idx_event_detail_yellowfox_eventtype_state
on eventhub.event_detail(
(attributes ->> 'yellowFoxEventType'),
(attributes ->> 'yellowFoxState'),
event_occurred_at
)
where attributes ? 'yellowFoxEventType';
create index if not exists idx_event_detail_yellowfox_ignition
on eventhub.event_detail(detail_type, (attributes ->> 'ignitionState'), event_occurred_at)
where attributes ? 'ignitionState';

363
src/main/resources/db/migration/V10__introduce_driver_card_model.sql
View File

@ -1,363 +0,0 @@
create table if not exists eventhub.driver_card (
id uuid primary key,
driver_id uuid references eventhub.driver(id),
nation text not null,
card_number text not null,
source_updated_at timestamptz,
payload jsonb not null default '{}'::jsonb,
created_at timestamptz not null default now(),
updated_at timestamptz not null default now()
);
create table if not exists eventhub.source_driver_identity (
id uuid primary key,
tenant_key text not null,
event_source_id integer not null references eventhub.event_source(id),
source_driver_entity_id text not null,
driver_id uuid not null references eventhub.driver(id) on delete cascade,
source_updated_at timestamptz,
payload jsonb not null default '{}'::jsonb,
created_at timestamptz not null default now(),
updated_at timestamptz not null default now(),
constraint ux_source_driver_identity unique (tenant_key, event_source_id, source_driver_entity_id)
);
create table if not exists eventhub.source_driver_card_identity (
id uuid primary key,
tenant_key text not null,
event_source_id integer not null references eventhub.event_source(id),
source_driver_card_entity_id text not null,
driver_card_id uuid not null references eventhub.driver_card(id) on delete cascade,
source_updated_at timestamptz,
payload jsonb not null default '{}'::jsonb,
created_at timestamptz not null default now(),
updated_at timestamptz not null default now(),
constraint ux_source_driver_card_identity unique (tenant_key, event_source_id, source_driver_card_entity_id)
);
create table if not exists eventhub.source_vehicle_identity (
id uuid primary key,
tenant_key text not null,
event_source_id integer not null references eventhub.event_source(id),
source_vehicle_entity_id text not null,
vehicle_id uuid not null references eventhub.vehicle(id) on delete cascade,
source_updated_at timestamptz,
payload jsonb not null default '{}'::jsonb,
created_at timestamptz not null default now(),
updated_at timestamptz not null default now(),
constraint ux_source_vehicle_identity unique (tenant_key, event_source_id, source_vehicle_entity_id)
);
create table if not exists eventhub.source_vehicle_registration_identity (
id uuid primary key,
tenant_key text not null,
event_source_id integer not null references eventhub.event_source(id),
source_registration_entity_id text not null,
vehicle_registration_id uuid not null references eventhub.vehicle_registration(id) on delete cascade,
source_updated_at timestamptz,
payload jsonb not null default '{}'::jsonb,
created_at timestamptz not null default now(),
updated_at timestamptz not null default now(),
constraint ux_source_vehicle_registration_identity unique (tenant_key, event_source_id, source_registration_entity_id)
);
alter table eventhub.event
add column if not exists driver_card_id uuid references eventhub.driver_card(id);
insert into eventhub.source_driver_identity(
id, tenant_key, event_source_id, source_driver_entity_id,
driver_id, source_updated_at, payload, created_at, updated_at
)
select gen_random_uuid(),
driver.tenant_key,
driver.event_source_id,
driver.source_driver_entity_id,
driver.id,
driver.source_updated_at,
driver.payload,
coalesce(driver.created_at, now()),
coalesce(driver.updated_at, now())
from eventhub.driver driver
where exists (
select 1
from information_schema.columns
where table_schema = 'eventhub'
and table_name = 'driver'
and column_name = 'source_driver_entity_id'
)
and driver.source_driver_entity_id is not null
on conflict (tenant_key, event_source_id, source_driver_entity_id)
do update set
driver_id = excluded.driver_id,
source_updated_at = coalesce(excluded.source_updated_at, eventhub.source_driver_identity.source_updated_at),
payload = eventhub.source_driver_identity.payload || excluded.payload,
updated_at = now();
with legacy_driver_cards as (
select distinct on (driver.card_nation, driver.card_number)
driver.id as driver_id,
driver.card_nation as nation,
driver.card_number,
driver.source_updated_at,
driver.payload,
driver.created_at,
driver.updated_at
from eventhub.driver driver
where exists (
select 1
from information_schema.columns
where table_schema = 'eventhub'
and table_name = 'driver'
and column_name = 'card_nation'
)
and driver.card_nation is not null
and driver.card_number is not null
order by driver.card_nation,
driver.card_number,
case when driver.source_driver_entity_id is null then 1 else 0 end,
driver.updated_at desc,
driver.created_at desc,
driver.id
),
existing_driver_cards as (
select distinct on (card.nation, card.card_number)
card.id,
card.nation,
card.card_number
from eventhub.driver_card card
order by card.nation,
card.card_number,
case when card.driver_id is null then 1 else 0 end,
card.updated_at desc,
card.created_at desc,
card.id
),
resolved_cards as (
select legacy.*,
coalesce(existing.id, gen_random_uuid()) as driver_card_id
from legacy_driver_cards legacy
left join existing_driver_cards existing
on existing.nation = legacy.nation
and existing.card_number = legacy.card_number
),
inserted_cards as (
insert into eventhub.driver_card(
id, driver_id, nation, card_number, source_updated_at, payload, created_at, updated_at
)
select distinct on (resolved.driver_card_id)
resolved.driver_card_id,
resolved.driver_id,
resolved.nation,
resolved.card_number,
resolved.source_updated_at,
jsonb_build_object('migrated_from', 'eventhub.driver') || coalesce(resolved.payload, '{}'::jsonb),
resolved.created_at,
resolved.updated_at
from resolved_cards resolved
where not exists (
select 1
from eventhub.driver_card existing
where existing.id = resolved.driver_card_id
)
returning id
),
updated_cards as (
update eventhub.driver_card card
set driver_id = coalesce(card.driver_id, resolved.driver_id),
source_updated_at = coalesce(resolved.source_updated_at, card.source_updated_at),
payload = card.payload || jsonb_build_object('migrated_from', 'eventhub.driver') || coalesce(resolved.payload, '{}'::jsonb),
updated_at = now()
from resolved_cards resolved
where card.id = resolved.driver_card_id
returning card.id
)
select count(*) from inserted_cards
union all
select count(*) from updated_cards;
with single_card_per_driver as (
select driver_id,
min(id::text)::uuid as driver_card_id
from eventhub.driver_card
where driver_id is not null
group by driver_id
having count(*) = 1
)
update eventhub.event event
set driver_card_id = card.driver_card_id
from single_card_per_driver card
where event.driver_id = card.driver_id
and event.driver_card_id is null;
with ranked_driver_cards as (
select card.id,
card.driver_id,
card.source_updated_at,
first_value(card.id) over (
partition by card.nation, card.card_number
order by case when card.driver_id is null then 1 else 0 end,
card.updated_at desc,
card.created_at desc,
card.id
) as canonical_id,
row_number() over (
partition by card.nation, card.card_number
order by case when card.driver_id is null then 1 else 0 end,
card.updated_at desc,
card.created_at desc,
card.id
) as duplicate_rank
from eventhub.driver_card card
),
duplicate_driver_cards as (
select id, canonical_id, driver_id, source_updated_at
from ranked_driver_cards
where duplicate_rank > 1
),
duplicate_card_rollup as (
select duplicate.canonical_id,
(array_agg(duplicate.driver_id order by case when duplicate.driver_id is null then 1 else 0 end, duplicate.id))[1] as preferred_driver_id,
max(duplicate.source_updated_at) as latest_source_updated_at
from duplicate_driver_cards duplicate
group by duplicate.canonical_id
),
updated_canonical_cards as (
update eventhub.driver_card card
set driver_id = coalesce(card.driver_id, rollup.preferred_driver_id),
source_updated_at = case
when card.source_updated_at is null then rollup.latest_source_updated_at
when rollup.latest_source_updated_at is null then card.source_updated_at
else greatest(card.source_updated_at, rollup.latest_source_updated_at)
end,
updated_at = now()
from duplicate_card_rollup rollup
where card.id = rollup.canonical_id
returning card.id
),
relinked_source_driver_card_identity as (
update eventhub.source_driver_card_identity identity
set driver_card_id = duplicate.canonical_id,
updated_at = now()
from duplicate_driver_cards duplicate
where identity.driver_card_id = duplicate.id
returning identity.id
),
relinked_events as (
update eventhub.event event
set driver_card_id = duplicate.canonical_id
from duplicate_driver_cards duplicate
where event.driver_card_id = duplicate.id
returning event.id
),
deleted_duplicate_driver_cards as (
delete from eventhub.driver_card card
using duplicate_driver_cards duplicate
where card.id = duplicate.id
returning card.id
)
select (select count(*) from updated_canonical_cards) as updated_canonical_cards,
(select count(*) from relinked_source_driver_card_identity) as relinked_source_driver_card_identity,
(select count(*) from relinked_events) as relinked_events,
(select count(*) from deleted_duplicate_driver_cards) as deleted_duplicate_driver_cards;
insert into eventhub.source_vehicle_identity(
id, tenant_key, event_source_id, source_vehicle_entity_id,
vehicle_id, source_updated_at, payload, created_at, updated_at
)
select gen_random_uuid(),
vehicle.tenant_key,
vehicle.event_source_id,
vehicle.source_vehicle_entity_id,
vehicle.id,
null,
'{}'::jsonb,
coalesce(vehicle.created_at, now()),
coalesce(vehicle.updated_at, now())
from eventhub.vehicle vehicle
where exists (
select 1
from information_schema.columns
where table_schema = 'eventhub'
and table_name = 'vehicle'
and column_name = 'source_vehicle_entity_id'
)
and vehicle.source_vehicle_entity_id is not null
on conflict (tenant_key, event_source_id, source_vehicle_entity_id)
do update set
vehicle_id = excluded.vehicle_id,
updated_at = now();
insert into eventhub.source_vehicle_registration_identity(
id, tenant_key, event_source_id, source_registration_entity_id,
vehicle_registration_id, source_updated_at, payload, created_at, updated_at
)
select gen_random_uuid(),
registration.tenant_key,
registration.event_source_id,
registration.source_registration_entity_id,
registration.id,
registration.source_updated_at,
registration.payload,
coalesce(registration.created_at, now()),
coalesce(registration.updated_at, now())
from eventhub.vehicle_registration registration
where exists (
select 1
from information_schema.columns
where table_schema = 'eventhub'
and table_name = 'vehicle_registration'
and column_name = 'source_registration_entity_id'
)
and registration.source_registration_entity_id is not null
on conflict (tenant_key, event_source_id, source_registration_entity_id)
do update set
vehicle_registration_id = excluded.vehicle_registration_id,
source_updated_at = coalesce(excluded.source_updated_at, eventhub.source_vehicle_registration_identity.source_updated_at),
payload = eventhub.source_vehicle_registration_identity.payload || excluded.payload,
updated_at = now();
create index if not exists idx_driver_card_key
on eventhub.driver_card(nation, card_number);
create index if not exists idx_driver_card_driver
on eventhub.driver_card(driver_id)
where driver_id is not null;
create unique index if not exists ux_driver_card_key
on eventhub.driver_card(nation, card_number);
create unique index if not exists ux_vehicle_vin
on eventhub.vehicle(vin)
where vin is not null;
create unique index if not exists ux_vehicle_registration_plate
on eventhub.vehicle_registration(nation, registration_number);
create index if not exists idx_source_driver_identity_driver
on eventhub.source_driver_identity(driver_id);
create index if not exists idx_source_driver_card_identity_card
on eventhub.source_driver_card_identity(driver_card_id);
create index if not exists idx_source_vehicle_identity_vehicle
on eventhub.source_vehicle_identity(vehicle_id);
create index if not exists idx_source_vehicle_registration_identity_registration
on eventhub.source_vehicle_registration_identity(vehicle_registration_id);
create index if not exists idx_event_driver_card_time
on eventhub.event(driver_card_id, occurred_at desc)
where driver_card_id is not null;
alter table eventhub.event
drop constraint if exists chk_event_driver_or_vehicle_ref;
alter table eventhub.event
add constraint chk_event_driver_or_vehicle_ref
check (
driver_id is not null
or driver_card_id is not null
or driver_entity_id is not null
or vehicle_id is not null
or vehicle_registration_id is not null
);

355
src/main/resources/esper/operating-period-state-machine.epl
View File

@ -1,355 +0,0 @@
create variable long operatingSplitIdleMs = ${operatingSplitIdleMs};
/*
* Full-EPL operating-period state machine.
*
* Input contract:
* - Java sends already-resolved intervals, not raw START/END boundaries.
* - DRIVER_CARD remains authoritative and VU is only used to fill uncovered gaps before events reach this EPL.
* - Synthetic uncovered gaps arrive as activityType = 'UNKNOWN'.
*
* Output contract:
* - PeriodizedActivityInterval: every input interval assigned to an operating period
* - OperatingPeriodClosed: closed operating periods, including the final FLUSH period
*/
create schema KnownOperatingInput(
driverId java.util.UUID,
vehicleId java.util.UUID,
vehicleRegistrationId java.util.UUID,
activityType string,
cardSlot string,
cardStatus string,
drivingStatus string,
sourceKind string,
startTs long,
endTs long,
durationMs long,
sourceRowId string,
sourceRowIds java.util.List,
clippedToRequestedPeriod boolean,
synthetic boolean
);
create schema UnknownOperatingInput(
driverId java.util.UUID,
vehicleId java.util.UUID,
vehicleRegistrationId java.util.UUID,
activityType string,
cardSlot string,
cardStatus string,
drivingStatus string,
sourceKind string,
startTs long,
endTs long,
durationMs long,
sourceRowId string,
sourceRowIds java.util.List,
clippedToRequestedPeriod boolean,
synthetic boolean
);
create schema PeriodizedActivityInterval(
driverId java.util.UUID,
vehicleId java.util.UUID,
vehicleRegistrationId java.util.UUID,
activityType string,
cardSlot string,
cardStatus string,
drivingStatus string,
sourceKind string,
startedAtTs long,
endedAtTs long,
durationMs long,
sourceRowId string,
sourceRowIds java.util.List,
clippedToRequestedPeriod boolean,
operatingPeriodNo long,
operatingPeriodStartedAtTs long,
newOperatingPeriod boolean,
gapSincePreviousActivityMs java.lang.Long,
synthetic boolean
);
create schema OperatingPeriodClosed(
driverId java.util.UUID,
operatingPeriodNo long,
operatingPeriodStartedAtTs long,
operatingPeriodEndedAtTs long,
durationMs long,
closedBy string
);
create window OperatingPeriodState#unique(driverId) as (
driverId java.util.UUID,
hasOpen boolean,
operatingPeriodNo long,
operatingPeriodStartedAtTs long,
lastKnownActivityEndTs long
);
/* Split the timeline into known activities and synthetic UNKNOWN gaps. */
insert into KnownOperatingInput
select * from OperatingPeriodInputMap as i where i.activityType != 'UNKNOWN';
insert into UnknownOperatingInput
select * from OperatingPeriodInputMap as i where i.activityType = 'UNKNOWN';
/* First known activity for a driver opens operating period 1. */
@Priority(200)
on KnownOperatingInput as i
insert into PeriodizedActivityInterval
select
i.driverId as driverId,
i.vehicleId as vehicleId,
i.vehicleRegistrationId as vehicleRegistrationId,
i.activityType as activityType,
i.cardSlot as cardSlot,
i.cardStatus as cardStatus,
i.drivingStatus as drivingStatus,
i.sourceKind as sourceKind,
i.startTs as startedAtTs,
i.endTs as endedAtTs,
i.durationMs as durationMs,
i.sourceRowId as sourceRowId,
i.sourceRowIds as sourceRowIds,
i.clippedToRequestedPeriod as clippedToRequestedPeriod,
1L as operatingPeriodNo,
i.startTs as operatingPeriodStartedAtTs,
true as newOperatingPeriod,
cast(null, java.lang.Long) as gapSincePreviousActivityMs,
i.synthetic as synthetic
where not exists (select * from OperatingPeriodState as s where s.driverId = i.driverId);
@Priority(190)
on KnownOperatingInput as i
insert into OperatingPeriodState
select
i.driverId as driverId,
true as hasOpen,
1L as operatingPeriodNo,
i.startTs as operatingPeriodStartedAtTs,
i.endTs as lastKnownActivityEndTs
where not exists (select * from OperatingPeriodState as s where s.driverId = i.driverId);
/* A long forward gap between known activities closes the current period with reason IDLE_GAP. */
@Priority(180)
on KnownOperatingInput as i
insert into OperatingPeriodClosed
select
s.driverId as driverId,
s.operatingPeriodNo as operatingPeriodNo,
s.operatingPeriodStartedAtTs as operatingPeriodStartedAtTs,
s.lastKnownActivityEndTs as operatingPeriodEndedAtTs,
s.lastKnownActivityEndTs - s.operatingPeriodStartedAtTs as durationMs,
'IDLE_GAP' as closedBy
from OperatingPeriodState as s
where s.driverId = i.driverId
and s.hasOpen = true
and i.startTs - s.lastKnownActivityEndTs >= operatingSplitIdleMs;
/* After a long idle gap, the next known interval is emitted as the first interval of the next period. */
@Priority(170)
on KnownOperatingInput as i
insert into PeriodizedActivityInterval
select
i.driverId as driverId,
i.vehicleId as vehicleId,
i.vehicleRegistrationId as vehicleRegistrationId,
i.activityType as activityType,
i.cardSlot as cardSlot,
i.cardStatus as cardStatus,
i.drivingStatus as drivingStatus,
i.sourceKind as sourceKind,
i.startTs as startedAtTs,
i.endTs as endedAtTs,
i.durationMs as durationMs,
i.sourceRowId as sourceRowId,
i.sourceRowIds as sourceRowIds,
i.clippedToRequestedPeriod as clippedToRequestedPeriod,
s.operatingPeriodNo + 1 as operatingPeriodNo,
i.startTs as operatingPeriodStartedAtTs,
true as newOperatingPeriod,
cast(i.startTs - s.lastKnownActivityEndTs, java.lang.Long) as gapSincePreviousActivityMs,
i.synthetic as synthetic
from OperatingPeriodState as s
where s.driverId = i.driverId
and s.hasOpen = true
and i.startTs - s.lastKnownActivityEndTs >= operatingSplitIdleMs;
/* Update the window state to the newly opened period after an IDLE_GAP close. */
@Priority(160)
on KnownOperatingInput as i
update OperatingPeriodState as s
set
hasOpen = true,
operatingPeriodNo = s.operatingPeriodNo + 1,
operatingPeriodStartedAtTs = i.startTs,
lastKnownActivityEndTs = i.endTs
where s.driverId = i.driverId
and s.hasOpen = true
and i.startTs - s.lastKnownActivityEndTs >= operatingSplitIdleMs;
/* After a long UNKNOWN gap we keep the counter but mark the period closed. The next known activity reopens
* the next period number using the retained state row. */
@Priority(155)
on KnownOperatingInput as i
insert into PeriodizedActivityInterval
select
i.driverId as driverId,
i.vehicleId as vehicleId,
i.vehicleRegistrationId as vehicleRegistrationId,
i.activityType as activityType,
i.cardSlot as cardSlot,
i.cardStatus as cardStatus,
i.drivingStatus as drivingStatus,
i.sourceKind as sourceKind,
i.startTs as startedAtTs,
i.endTs as endedAtTs,
i.durationMs as durationMs,
i.sourceRowId as sourceRowId,
i.sourceRowIds as sourceRowIds,
i.clippedToRequestedPeriod as clippedToRequestedPeriod,
s.operatingPeriodNo + 1 as operatingPeriodNo,
i.startTs as operatingPeriodStartedAtTs,
true as newOperatingPeriod,
cast(null, java.lang.Long) as gapSincePreviousActivityMs,
i.synthetic as synthetic
from OperatingPeriodState as s
where s.driverId = i.driverId
and s.hasOpen = false;
@Priority(145)
on KnownOperatingInput as i
update OperatingPeriodState as s
set
hasOpen = true,
operatingPeriodNo = s.operatingPeriodNo + 1,
operatingPeriodStartedAtTs = i.startTs,
lastKnownActivityEndTs = i.endTs
where s.driverId = i.driverId
and s.hasOpen = false;
/* Normal same-period continuity: the gap is forward, non-negative, and still below the split threshold. */
@Priority(150)
on KnownOperatingInput as i
insert into PeriodizedActivityInterval
select
i.driverId as driverId,
i.vehicleId as vehicleId,
i.vehicleRegistrationId as vehicleRegistrationId,
i.activityType as activityType,
i.cardSlot as cardSlot,
i.cardStatus as cardStatus,
i.drivingStatus as drivingStatus,
i.sourceKind as sourceKind,
i.startTs as startedAtTs,
i.endTs as endedAtTs,
i.durationMs as durationMs,
i.sourceRowId as sourceRowId,
i.sourceRowIds as sourceRowIds,
i.clippedToRequestedPeriod as clippedToRequestedPeriod,
s.operatingPeriodNo as operatingPeriodNo,
s.operatingPeriodStartedAtTs as operatingPeriodStartedAtTs,
false as newOperatingPeriod,
cast(i.startTs - s.lastKnownActivityEndTs, java.lang.Long) as gapSincePreviousActivityMs,
i.synthetic as synthetic
from OperatingPeriodState as s
where s.driverId = i.driverId
and s.hasOpen = true
and i.startTs - s.lastKnownActivityEndTs >= 0
and i.startTs - s.lastKnownActivityEndTs < operatingSplitIdleMs;
@Priority(140)
on KnownOperatingInput as i
update OperatingPeriodState as s
set
lastKnownActivityEndTs = case
when i.endTs > s.lastKnownActivityEndTs then i.endTs
else s.lastKnownActivityEndTs
end
where s.driverId = i.driverId
and s.hasOpen = true
and i.startTs - s.lastKnownActivityEndTs >= 0
and i.startTs - s.lastKnownActivityEndTs < operatingSplitIdleMs;
/* A long UNKNOWN interval closes the current period, but the state row remains so the next known activity
* can reopen with the next period number. */
@Priority(130)
on UnknownOperatingInput as i
insert into OperatingPeriodClosed
select
s.driverId as driverId,
s.operatingPeriodNo as operatingPeriodNo,
s.operatingPeriodStartedAtTs as operatingPeriodStartedAtTs,
s.lastKnownActivityEndTs as operatingPeriodEndedAtTs,
s.lastKnownActivityEndTs - s.operatingPeriodStartedAtTs as durationMs,
'UNKNOWN_GAP' as closedBy
from OperatingPeriodState as s
where s.driverId = i.driverId
and s.hasOpen = true
and i.durationMs >= operatingSplitIdleMs;
@Priority(120)
on UnknownOperatingInput as i
update OperatingPeriodState as s
set hasOpen = false
where s.driverId = i.driverId
and s.hasOpen = true
and i.durationMs >= operatingSplitIdleMs;
/* Short UNKNOWN stays inside the open period as explicit uncertainty, without changing period state. */
@Priority(110)
on UnknownOperatingInput as i
insert into PeriodizedActivityInterval
select
i.driverId as driverId,
i.vehicleId as vehicleId,
i.vehicleRegistrationId as vehicleRegistrationId,
i.activityType as activityType,
i.cardSlot as cardSlot,
i.cardStatus as cardStatus,
i.drivingStatus as drivingStatus,
i.sourceKind as sourceKind,
i.startTs as startedAtTs,
i.endTs as endedAtTs,
i.durationMs as durationMs,
i.sourceRowId as sourceRowId,
i.sourceRowIds as sourceRowIds,
i.clippedToRequestedPeriod as clippedToRequestedPeriod,
s.operatingPeriodNo as operatingPeriodNo,
s.operatingPeriodStartedAtTs as operatingPeriodStartedAtTs,
false as newOperatingPeriod,
cast(i.startTs - s.lastKnownActivityEndTs, java.lang.Long) as gapSincePreviousActivityMs,
i.synthetic as synthetic
from OperatingPeriodState as s
where s.driverId = i.driverId
and s.hasOpen = true
and i.durationMs < operatingSplitIdleMs;
/* Historical evaluation ends with a flush event so the final still-open period is emitted. */
@Priority(100)
on OperatingPeriodFlushEvent as f
insert into OperatingPeriodClosed
select
s.driverId as driverId,
s.operatingPeriodNo as operatingPeriodNo,
s.operatingPeriodStartedAtTs as operatingPeriodStartedAtTs,
s.lastKnownActivityEndTs as operatingPeriodEndedAtTs,
s.lastKnownActivityEndTs - s.operatingPeriodStartedAtTs as durationMs,
'FLUSH' as closedBy
from OperatingPeriodState as s
where s.hasOpen = true;
@Priority(90)
on OperatingPeriodFlushEvent as f
update OperatingPeriodState as s
set hasOpen = false
where s.hasOpen = true;
/* Listener-facing output statements consumed by Java. */
@name('periodizedActivityIntervals')
select * from PeriodizedActivityInterval;
@name('operatingPeriodClosed')
select * from OperatingPeriodClosed;

403
src/main/resources/sql/maintenance/repair-tachograph-driver-entities.sql
View File

@ -1,22 +1,19 @@
/* /*
* Repairs and normalizes tachograph driver identities after introducing: * Repairs and normalizes tachograph driver aggregates after introducing eventhub.driver.
* - global eventhub.driver
* - global eventhub.driver_card
* - source_driver_identity
* - source_driver_card_identity
* *
* What it does: * What it does:
* 1. Ensures tachograph DRIVER master-data payload carries last_name while * 1. Ensures tachograph DRIVER master-data payload carries last_name while keeping source_master_entity.display_name unchanged.
* keeping source_master_entity.display_name unchanged. * 2. Upserts eventhub.driver rows from MASTER_DATA DRIVER entities.
* 2. Upserts global eventhub.driver rows from tachograph DRIVER entities. * 3. Projects card nation/number onto eventhub.driver from DRIVER_CARD_DRIVER relations.
* 3. Upserts global eventhub.driver_card rows from tachograph DRIVER_CARD entities. * 4. Remaps event.driver_id from provisional card-only drivers to proper source-driver aggregates when possible.
* 4. Upserts source identity links for drivers and cards. * 5. Deletes now-unreferenced provisional tachograph driver rows with no source_driver_entity_id.
* 5. Links cards to drivers using DRIVER_CARD_DRIVER master-data relations. *
* 6. Backfills event.driver_card_id where a driver has exactly one card. * Assumptions:
* 7. Remaps event.driver_id from provisional rows to the linked canonical driver. * - Tachograph master-data source is provider_key=TACHOGRAPH, source_kind=MASTER_DATA, source_key=TACHOGRAPH_MASTER_DATA.
* 8. Deletes now-unreferenced provisional driver rows. * - eventhub.driver and event.driver_id already exist.
*/ */
-- 1) Keep display_name, but ensure DRIVER payload has last_name.
with master_sources as ( with master_sources as (
select es.id, es.tenant_key select es.id, es.tenant_key
from eventhub.event_source es from eventhub.event_source es
@ -43,16 +40,22 @@ updated_master_payload as (
select count(*) as updated_master_payload select count(*) as updated_master_payload
from updated_master_payload; from updated_master_payload;
-- 2) Upsert driver aggregates from tachograph master data.
with master_sources as ( with master_sources as (
select es.id as master_event_source_id, es.tenant_key select es.id,
es.tenant_key,
es.source_instance_key,
coalesce(es.tenant_provider_setting_key, '') as tenant_provider_setting_key
from eventhub.event_source es from eventhub.event_source es
where es.provider_key = 'TACHOGRAPH' where es.provider_key = 'TACHOGRAPH'
and es.source_kind = 'MASTER_DATA' and es.source_kind = 'MASTER_DATA'
and es.source_key = 'TACHOGRAPH_MASTER_DATA' and es.source_key = 'TACHOGRAPH_MASTER_DATA'
), ),
master_drivers as ( master_drivers as (
select ms.master_event_source_id, select ms.id as master_event_source_id,
ms.tenant_key, ms.tenant_key,
ms.source_instance_key,
ms.tenant_provider_setting_key,
d.source_entity_id as source_driver_entity_id, d.source_entity_id as source_driver_entity_id,
coalesce(nullif(trim(d.payload ->> 'first_names'), ''), nullif(trim(d.payload ->> 'firstnames'), '')) as first_names, coalesce(nullif(trim(d.payload ->> 'first_names'), ''), nullif(trim(d.payload ->> 'firstnames'), '')) as first_names,
coalesce(nullif(trim(d.payload ->> 'last_name'), ''), nullif(trim(d.payload ->> 'surname'), '')) as last_name, coalesce(nullif(trim(d.payload ->> 'last_name'), ''), nullif(trim(d.payload ->> 'surname'), '')) as last_name,
@ -62,297 +65,173 @@ master_drivers as (
from master_sources ms from master_sources ms
join eventhub.source_master_entity d join eventhub.source_master_entity d
on d.tenant_key = ms.tenant_key on d.tenant_key = ms.tenant_key
and d.event_source_id = ms.master_event_source_id and d.event_source_id = ms.id
and d.entity_type = 'DRIVER' and d.entity_type = 'DRIVER'
and d.source_entity_id not like 'DRIVER_CARD:%'
), ),
resolved_drivers as ( compatible_targets as (
select md.*, select md.*,
coalesce(identity.driver_id, gen_random_uuid()) as driver_id es.id as target_event_source_id
from master_drivers md from master_drivers md
left join eventhub.source_driver_identity identity join eventhub.event_source es
on identity.tenant_key = md.tenant_key on es.tenant_key = md.tenant_key
and identity.event_source_id = md.master_event_source_id and es.provider_key = 'TACHOGRAPH'
and identity.source_driver_entity_id = md.source_driver_entity_id and es.source_instance_key = md.source_instance_key
), and coalesce(es.tenant_provider_setting_key, '') = md.tenant_provider_setting_key
inserted_drivers as (
insert into eventhub.driver(
id, first_names, last_name, birth_date, source_updated_at, payload, updated_at
)
select distinct on (rd.driver_id)
rd.driver_id,
rd.first_names,
rd.last_name,
rd.birth_date,
rd.source_updated_at,
rd.payload,
now()
from resolved_drivers rd
where not exists (
select 1
from eventhub.driver existing
where existing.id = rd.driver_id
)
returning id
), ),
updated_drivers as ( updated_drivers as (
update eventhub.driver driver update eventhub.driver driver
set first_names = coalesce(rd.first_names, driver.first_names), set first_names = coalesce(ct.first_names, driver.first_names),
last_name = coalesce(rd.last_name, driver.last_name), last_name = coalesce(ct.last_name, driver.last_name),
birth_date = coalesce(rd.birth_date, driver.birth_date), birth_date = coalesce(ct.birth_date, driver.birth_date),
source_updated_at = coalesce(rd.source_updated_at, driver.source_updated_at), source_updated_at = ct.source_updated_at,
payload = driver.payload || rd.payload, payload = driver.payload || ct.payload,
updated_at = now() updated_at = now()
from resolved_drivers rd from compatible_targets ct
where driver.id = rd.driver_id where driver.tenant_key = ct.tenant_key
and driver.event_source_id = ct.target_event_source_id
and driver.source_driver_entity_id = ct.source_driver_entity_id
returning driver.id returning driver.id
), ),
upserted_source_driver_identity as ( inserted_drivers as (
insert into eventhub.source_driver_identity( insert into eventhub.driver(
id, tenant_key, event_source_id, source_driver_entity_id, id, tenant_key, event_source_id, source_driver_entity_id,
driver_id, source_updated_at, payload, updated_at first_names, last_name, birth_date, source_updated_at, payload, updated_at
) )
select gen_random_uuid(), select gen_random_uuid(),
rd.tenant_key, ct.tenant_key,
rd.master_event_source_id, ct.target_event_source_id,
rd.source_driver_entity_id, ct.source_driver_entity_id,
rd.driver_id, ct.first_names,
rd.source_updated_at, ct.last_name,
rd.payload, ct.birth_date,
ct.source_updated_at,
ct.payload,
now() now()
from resolved_drivers rd from compatible_targets ct
on conflict (tenant_key, event_source_id, source_driver_entity_id)
do update set
driver_id = excluded.driver_id,
source_updated_at = coalesce(excluded.source_updated_at, eventhub.source_driver_identity.source_updated_at),
payload = eventhub.source_driver_identity.payload || excluded.payload,
updated_at = now()
returning id
)
select (select count(*) from inserted_drivers) as inserted_drivers,
(select count(*) from updated_drivers) as updated_drivers,
(select count(*) from upserted_source_driver_identity) as upserted_source_driver_identity;
with master_sources as (
select es.id as master_event_source_id, es.tenant_key
from eventhub.event_source es
where es.provider_key = 'TACHOGRAPH'
and es.source_kind = 'MASTER_DATA'
and es.source_key = 'TACHOGRAPH_MASTER_DATA'
),
master_cards as (
select ms.master_event_source_id,
ms.tenant_key,
card.source_entity_id as source_driver_card_entity_id,
nullif(trim(card.payload ->> 'card_nation'), '') as card_nation,
nullif(trim(card.payload ->> 'card_number'), '') as card_number,
card.source_updated_at,
card.payload
from master_sources ms
join eventhub.source_master_entity card
on card.tenant_key = ms.tenant_key
and card.event_source_id = ms.master_event_source_id
and card.entity_type = 'DRIVER_CARD'
and nullif(trim(card.payload ->> 'card_nation'), '') is not null
and nullif(trim(card.payload ->> 'card_number'), '') is not null
),
canonical_driver_cards as (
select distinct on (mc.card_nation, mc.card_number)
mc.card_nation,
mc.card_number,
mc.source_updated_at,
mc.payload
from master_cards mc
order by mc.card_nation,
mc.card_number,
case when mc.source_driver_card_entity_id is null then 1 else 0 end,
mc.source_updated_at desc,
mc.source_driver_card_entity_id
),
existing_driver_cards as (
select distinct on (card.nation, card.card_number)
card.id,
card.nation,
card.card_number
from eventhub.driver_card card
order by card.nation,
card.card_number,
case when card.driver_id is null then 1 else 0 end,
card.updated_at desc,
card.created_at desc,
card.id
),
resolved_cards as (
select canonical.card_nation,
canonical.card_number,
canonical.source_updated_at,
canonical.payload,
coalesce(existing.id, gen_random_uuid()) as driver_card_id
from canonical_driver_cards canonical
left join existing_driver_cards existing
on existing.nation = canonical.card_nation
and existing.card_number = canonical.card_number
),
inserted_cards as (
insert into eventhub.driver_card(
id, driver_id, nation, card_number, source_updated_at, payload, updated_at
)
select distinct on (rc.driver_card_id)
rc.driver_card_id,
null,
rc.card_nation,
rc.card_number,
rc.source_updated_at,
rc.payload,
now()
from resolved_cards rc
where not exists ( where not exists (
select 1 select 1
from eventhub.driver_card existing from eventhub.driver existing
where existing.id = rc.driver_card_id where existing.tenant_key = ct.tenant_key
and existing.event_source_id = ct.target_event_source_id
and existing.source_driver_entity_id = ct.source_driver_entity_id
) )
returning id returning id
),
updated_cards as (
update eventhub.driver_card card
set source_updated_at = coalesce(rc.source_updated_at, card.source_updated_at),
payload = card.payload || rc.payload,
updated_at = now()
from resolved_cards rc
where card.id = rc.driver_card_id
returning card.id
),
upserted_source_driver_card_identity as (
insert into eventhub.source_driver_card_identity(
id, tenant_key, event_source_id, source_driver_card_entity_id,
driver_card_id, source_updated_at, payload, updated_at
)
select gen_random_uuid(),
mc.tenant_key,
mc.master_event_source_id,
mc.source_driver_card_entity_id,
coalesce(identity.driver_card_id, existing.id),
mc.source_updated_at,
mc.payload,
now()
from master_cards mc
left join eventhub.source_driver_card_identity identity
on identity.tenant_key = mc.tenant_key
and identity.event_source_id = mc.master_event_source_id
and identity.source_driver_card_entity_id = mc.source_driver_card_entity_id
left join existing_driver_cards existing
on existing.nation = mc.card_nation
and existing.card_number = mc.card_number
where mc.source_driver_card_entity_id is not null
and coalesce(identity.driver_card_id, existing.id) is not null
on conflict (tenant_key, event_source_id, source_driver_card_entity_id)
do update set
driver_card_id = excluded.driver_card_id,
source_updated_at = coalesce(excluded.source_updated_at, eventhub.source_driver_card_identity.source_updated_at),
payload = eventhub.source_driver_card_identity.payload || excluded.payload,
updated_at = now()
returning id
) )
select (select count(*) from inserted_cards) as inserted_cards, select (select count(*) from updated_drivers) as updated_drivers,
(select count(*) from updated_cards) as updated_cards, (select count(*) from inserted_drivers) as inserted_drivers;
(select count(*) from upserted_source_driver_card_identity) as upserted_source_driver_card_identity;
-- 3) Project driver-card identifiers from master-data relations.
with master_sources as ( with master_sources as (
select es.id as master_event_source_id, es.tenant_key select es.id,
es.tenant_key,
es.source_instance_key,
coalesce(es.tenant_provider_setting_key, '') as tenant_provider_setting_key
from eventhub.event_source es from eventhub.event_source es
where es.provider_key = 'TACHOGRAPH' where es.provider_key = 'TACHOGRAPH'
and es.source_kind = 'MASTER_DATA' and es.source_kind = 'MASTER_DATA'
and es.source_key = 'TACHOGRAPH_MASTER_DATA' and es.source_key = 'TACHOGRAPH_MASTER_DATA'
), ),
updated_card_links as ( card_projection as (
update eventhub.driver_card card select distinct on (ms.tenant_key, ms.source_instance_key, ms.tenant_provider_setting_key, rel.to_source_entity_id)
set driver_id = driver_identity.driver_id, ms.tenant_key,
source_updated_at = coalesce(rel.source_updated_at, card.source_updated_at), ms.source_instance_key,
updated_at = now() ms.tenant_provider_setting_key,
rel.to_source_entity_id as source_driver_entity_id,
nullif(trim(card.payload ->> 'card_nation'), '') as card_nation,
nullif(trim(card.payload ->> 'card_number'), '') as card_number,
rel.source_updated_at
from master_sources ms from master_sources ms
join eventhub.source_master_relation rel join eventhub.source_master_relation rel
on rel.tenant_key = ms.tenant_key on rel.tenant_key = ms.tenant_key
and rel.event_source_id = ms.master_event_source_id and rel.event_source_id = ms.id
and rel.relation_type = 'DRIVER_CARD_DRIVER' and rel.relation_type = 'DRIVER_CARD_DRIVER'
and rel.from_entity_type = 'DRIVER_CARD' and rel.from_entity_type = 'DRIVER_CARD'
and rel.to_entity_type = 'DRIVER' and rel.to_entity_type = 'DRIVER'
join eventhub.source_driver_card_identity card_identity join eventhub.source_master_entity card
on card_identity.tenant_key = rel.tenant_key on card.tenant_key = ms.tenant_key
and card_identity.event_source_id = rel.event_source_id and card.event_source_id = ms.id
and card_identity.source_driver_card_entity_id = rel.from_source_entity_id and card.entity_type = 'DRIVER_CARD'
join eventhub.source_driver_identity driver_identity and card.source_entity_id = rel.from_source_entity_id
on driver_identity.tenant_key = rel.tenant_key order by ms.tenant_key,
and driver_identity.event_source_id = rel.event_source_id ms.source_instance_key,
and driver_identity.source_driver_entity_id = rel.to_source_entity_id ms.tenant_provider_setting_key,
where card.id = card_identity.driver_card_id rel.to_source_entity_id,
rel.valid_to desc nulls last,
rel.valid_from desc nulls last,
rel.updated_at desc
),
updated_driver_cards as (
update eventhub.driver driver
set card_nation = coalesce(driver.card_nation, projection.card_nation),
card_number = coalesce(driver.card_number, projection.card_number),
source_updated_at = coalesce(projection.source_updated_at, driver.source_updated_at),
updated_at = now()
from card_projection projection
join eventhub.event_source es
on es.id = driver.event_source_id
where driver.tenant_key = projection.tenant_key
and es.provider_key = 'TACHOGRAPH'
and es.source_instance_key = projection.source_instance_key
and coalesce(es.tenant_provider_setting_key, '') = projection.tenant_provider_setting_key
and driver.source_driver_entity_id = projection.source_driver_entity_id
and ( and (
card.driver_id is null (driver.card_nation is null and projection.card_nation is not null)
or card.driver_id = driver_identity.driver_id or (driver.card_number is null and projection.card_number is not null)
or not exists (
select 1
from eventhub.source_driver_identity existing_identity
where existing_identity.driver_id = card.driver_id
) )
) returning driver.id
returning card.id
) )
select count(*) as updated_card_links select count(*) as updated_driver_cards
from updated_card_links; from updated_driver_cards;
with single_card_per_driver as ( -- 4) Remap events from provisional card-only drivers to proper source-driver aggregates.
select driver_id, with provisional_to_real as (
min(id::text)::uuid as driver_card_id select provisional.id as provisional_driver_id,
from eventhub.driver_card real.id as real_driver_id
where driver_id is not null from eventhub.driver provisional
group by driver_id join eventhub.event_source provisional_source
having count(*) = 1 on provisional_source.id = provisional.event_source_id
and provisional_source.provider_key = 'TACHOGRAPH'
join eventhub.driver real
on real.tenant_key = provisional.tenant_key
and real.source_driver_entity_id is not null
and real.card_nation = provisional.card_nation
and real.card_number = provisional.card_number
join eventhub.event_source real_source
on real_source.id = real.event_source_id
and real_source.provider_key = provisional_source.provider_key
and real_source.tenant_key = provisional_source.tenant_key
and real_source.source_instance_key = provisional_source.source_instance_key
and coalesce(real_source.tenant_provider_setting_key, '') = coalesce(provisional_source.tenant_provider_setting_key, '')
where provisional.source_driver_entity_id is null
and provisional.card_nation is not null
and provisional.card_number is not null
and provisional.id <> real.id
), ),
updated_events as ( updated_events as (
update eventhub.event event update eventhub.event e
set driver_card_id = card.driver_card_id set driver_id = map.real_driver_id
from single_card_per_driver card from provisional_to_real map
where event.driver_id = card.driver_id where e.driver_id = map.provisional_driver_id
and event.driver_card_id is null and e.driver_id <> map.real_driver_id
returning event.id returning e.id
)
select count(*) as backfilled_event_driver_cards
from updated_events;
with remapped_events as (
update eventhub.event event
set driver_id = card.driver_id
from eventhub.driver_card card
where event.driver_card_id = card.id
and card.driver_id is not null
and (
event.driver_id is null
or not exists (
select 1
from eventhub.source_driver_identity source_identity
where source_identity.driver_id = event.driver_id
)
)
and event.driver_id is distinct from card.driver_id
returning event.id
) )
select count(*) as remapped_events select count(*) as remapped_events
from remapped_events; from updated_events;
-- 5) Delete now-unreferenced provisional tachograph driver rows.
with deleted_drivers as ( with deleted_drivers as (
delete from eventhub.driver driver delete from eventhub.driver driver
where not exists ( using eventhub.event_source es
select 1 where es.id = driver.event_source_id
from eventhub.event event and es.provider_key = 'TACHOGRAPH'
where event.driver_id = driver.id and driver.source_driver_entity_id is null
) and driver.card_nation is not null
and driver.card_number is not null
and not exists ( and not exists (
select 1 select 1
from eventhub.driver_card card from eventhub.event e
where card.driver_id = driver.id where e.driver_id = driver.id
)
and not exists (
select 1
from eventhub.source_driver_identity source_identity
where source_identity.driver_id = driver.id
) )
returning driver.id returning driver.id
) )

110
src/main/resources/sql/maintenance/restore-driver-card-columns.sql
View File

@ -1,110 +0,0 @@
/*
* Restores legacy columns on eventhub.driver:
* - card_nation
* - card_number
*
* The script is conservative:
* - it recreates the columns and the old check constraint
* - it backfills values only when a driver has exactly one distinct card
* recoverable from:
* 1. eventhub.driver_card.driver_id
* 2. eventhub.event.driver_id + event.driver_card_id
*
* If a driver is associated with multiple distinct cards, the columns remain null.
*/
alter table eventhub.driver
add column if not exists card_nation text;
alter table eventhub.driver
add column if not exists card_number text;
do $restore$
begin
if not exists (
select 1
from information_schema.table_constraints
where constraint_schema = 'eventhub'
and table_name = 'driver'
and constraint_name = 'chk_driver_card_nation_when_number'
) then
alter table eventhub.driver
add constraint chk_driver_card_nation_when_number
check (card_number is null or card_nation is not null);
end if;
end
$restore$;
with direct_driver_cards as (
select driver.id as driver_id,
card.nation,
card.card_number
from eventhub.driver driver
join eventhub.driver_card card
on card.driver_id = driver.id
),
event_driver_cards as (
select distinct event.driver_id,
card.nation,
card.card_number
from eventhub.event event
join eventhub.driver_card card
on card.id = event.driver_card_id
where event.driver_id is not null
and event.driver_card_id is not null
),
all_driver_cards as (
select driver_id, nation, card_number
from direct_driver_cards
union
select driver_id, nation, card_number
from event_driver_cards
),
single_card_per_driver as (
select driver_id,
max(nation) as card_nation,
max(card_number) as card_number
from all_driver_cards
group by driver_id
having count(*) = 1
),
updated_drivers as (
update eventhub.driver driver
set card_nation = single.card_nation,
card_number = single.card_number,
updated_at = now()
from single_card_per_driver single
where driver.id = single.driver_id
and (
driver.card_nation is distinct from single.card_nation
or driver.card_number is distinct from single.card_number
)
returning driver.id
)
select count(*) as restored_driver_card_columns
from updated_drivers;
with ambiguous_drivers as (
select driver_id,
count(*) as distinct_card_count
from (
select distinct driver_id, nation, card_number
from (
select driver.id as driver_id, card.nation, card.card_number
from eventhub.driver driver
join eventhub.driver_card card
on card.driver_id = driver.id
union all
select distinct event.driver_id, card.nation, card.card_number
from eventhub.event event
join eventhub.driver_card card
on card.id = event.driver_card_id
where event.driver_id is not null
and event.driver_card_id is not null
) cards
) distinct_cards
group by driver_id
having count(*) > 1
)
select count(*) as ambiguous_driver_count
from ambiguous_drivers;

26
src/main/resources/sql/yellowfox/d8-bookings.sql
View File

@ -1,12 +1,3 @@
with bookings as (
select
b.*,
lag(b.ignition) over (
partition by b.vehicle_id
order by b.utc asc, b.eventid asc
) as previous_ignition
from data.d8_booking b
)
select select
b.eventid, b.eventid,
b.utc, b.utc,
@ -14,7 +5,6 @@ select
b.driver_id, b.driver_id,
b.key, b.key,
b.ignition, b.ignition,
b.previous_ignition,
b.eventtype, b.eventtype,
b.state, b.state,
b.odometer, b.odometer,
@ -28,7 +18,7 @@ select
d.firstname as driver_firstname, d.firstname as driver_firstname,
d.name as driver_lastname, d.name as driver_lastname,
left(trim(d.drivers_card), 14) as driver_card_number, d.drivers_card as driver_card_number,
d.fleet_id as driver_fleet_id, d.fleet_id as driver_fleet_id,
f.id as fleet_id, f.id as fleet_id,
@ -36,7 +26,7 @@ select
tp.id as telematic_provider_id, tp.id as telematic_provider_id,
tp.name as telematic_provider_name tp.name as telematic_provider_name
from bookings b from data.d8_booking b
left join data.vehicle v left join data.vehicle v
on v.id = b.vehicle_id on v.id = b.vehicle_id
left join data.driver d left join data.driver d
@ -45,5 +35,15 @@ left join data.fleet f
on f.id = coalesce(v.fleet_id, d.fleet_id) on f.id = coalesce(v.fleet_id, d.fleet_id)
left join data.telematic_provider tp left join data.telematic_provider tp
on tp.id = v.telematic_provider_id on tp.id = v.telematic_provider_id
/*__FILTERS__*/ where (:occurredFrom is null or b.utc >= :occurredFrom)
and (:occurredTo is null or b.utc < :occurredTo)
and (:fleetId is null or f.id = :fleetId)
and (
:lastOccurredTo is null
or b.utc > :lastOccurredTo
or (
b.utc = :lastOccurredTo
and (:lastSourceRowId is null or b.eventid > :lastSourceRowId)
)
)
order by b.utc asc, b.eventid asc; order by b.utc asc, b.eventid asc;

15
src/main/resources/sql/yellowfox/master-data/driver-cards.sql
View File

@ -1,15 +0,0 @@
select
'DRIVER_CARD' as entity_type,
cast(d.id as varchar(128)) as source_entity_id,
concat('YELLOWFOX:', left(trim(d.drivers_card), 14)) as source_external_key,
left(trim(d.drivers_card), 14) as display_name,
true as active,
null::timestamptz as valid_from,
null::timestamptz as valid_to,
null::timestamptz as source_updated_at,
d.id as driver_id,
left(trim(d.drivers_card), 14) as card_number,
'UNKNOWN' as card_nation,
d.fleet_id as fleet_id
from data.driver d
where nullif(trim(d.drivers_card), '') is not null;

14
src/main/resources/sql/yellowfox/master-data/drivers.sql
View File

@ -1,14 +0,0 @@
select
'DRIVER' as entity_type,
cast(d.id as varchar(128)) as source_entity_id,
coalesce(nullif(left(trim(d.drivers_card), 14), ''), cast(d.id as varchar(128))) as source_external_key,
nullif(trim(concat_ws(' ', d.firstname, d.name)), '') as display_name,
true as active,
null::timestamptz as valid_from,
null::timestamptz as valid_to,
null::timestamptz as source_updated_at,
d.id as driver_id,
d.firstname as first_names,
d.name as last_name,
d.fleet_id as fleet_id
from data.driver d;

12
src/main/resources/sql/yellowfox/master-data/fleets.sql
View File

@ -1,12 +0,0 @@
select
'FLEET' as entity_type,
cast(f.id as varchar(128)) as source_entity_id,
cast(f.id as varchar(128)) as source_external_key,
nullif(trim(f.name), '') as display_name,
true as active,
null::timestamptz as valid_from,
null::timestamptz as valid_to,
null::timestamptz as source_updated_at,
f.id as fleet_id,
f.name as fleet_name
from data.fleet f;

80
src/main/resources/sql/yellowfox/master-data/relations.sql
View File

@ -1,80 +0,0 @@
select
'DRIVER_FLEET' as relation_type,
'DRIVER' as from_entity_type,
cast(d.id as varchar(128)) as from_source_entity_id,
'FLEET' as to_entity_type,
cast(d.fleet_id as varchar(128)) as to_source_entity_id,
null::timestamptz as valid_from,
null::timestamptz as valid_to,
null::timestamptz as source_updated_at,
'data.driver' as source_table,
cast(d.id as varchar(128)) as source_row_id
from data.driver d
where d.fleet_id is not null
union all
select
'DRIVER_CARD_DRIVER' as relation_type,
'DRIVER_CARD' as from_entity_type,
cast(d.id as varchar(128)) as from_source_entity_id,
'DRIVER' as to_entity_type,
cast(d.id as varchar(128)) as to_source_entity_id,
null::timestamptz as valid_from,
null::timestamptz as valid_to,
null::timestamptz as source_updated_at,
'data.driver' as source_table,
cast(d.id as varchar(128)) as source_row_id
from data.driver d
where nullif(trim(d.drivers_card), '') is not null
union all
select
'VEHICLE_FLEET' as relation_type,
'VEHICLE' as from_entity_type,
cast(v.id as varchar(128)) as from_source_entity_id,
'FLEET' as to_entity_type,
cast(v.fleet_id as varchar(128)) as to_source_entity_id,
null::timestamptz as valid_from,
null::timestamptz as valid_to,
null::timestamptz as source_updated_at,
'data.vehicle' as source_table,
cast(v.id as varchar(128)) as source_row_id
from data.vehicle v
where v.fleet_id is not null
and nullif(trim(v.vin), '') is not null
union all
select
'VEHICLE_REGISTRATION_VEHICLE' as relation_type,
'VEHICLE_REGISTRATION' as from_entity_type,
cast(v.id as varchar(128)) as from_source_entity_id,
'VEHICLE' as to_entity_type,
cast(v.id as varchar(128)) as to_source_entity_id,
null::timestamptz as valid_from,
null::timestamptz as valid_to,
null::timestamptz as source_updated_at,
'data.vehicle' as source_table,
cast(v.id as varchar(128)) as source_row_id
from data.vehicle v
where nullif(trim(v.vrn), '') is not null
and nullif(trim(v.vin), '') is not null
union all
select
'VEHICLE_TELEMATIC_PROVIDER' as relation_type,
'VEHICLE' as from_entity_type,
cast(v.id as varchar(128)) as from_source_entity_id,
'TELEMATIC_PROVIDER' as to_entity_type,
cast(v.telematic_provider_id as varchar(128)) as to_source_entity_id,
null::timestamptz as valid_from,
null::timestamptz as valid_to,
null::timestamptz as source_updated_at,
'data.vehicle' as source_table,
cast(v.id as varchar(128)) as source_row_id
from data.vehicle v
where v.telematic_provider_id is not null
and nullif(trim(v.vin), '') is not null;

12
src/main/resources/sql/yellowfox/master-data/telematic-providers.sql
View File

@ -1,12 +0,0 @@
select
'TELEMATIC_PROVIDER' as entity_type,
cast(tp.id as varchar(128)) as source_entity_id,
cast(tp.id as varchar(128)) as source_external_key,
nullif(trim(tp.name), '') as display_name,
true as active,
null::timestamptz as valid_from,
null::timestamptz as valid_to,
null::timestamptz as source_updated_at,
tp.id as telematic_provider_id,
tp.name as telematic_provider_name
from data.telematic_provider tp;

16
src/main/resources/sql/yellowfox/master-data/vehicle-registrations.sql
View File

@ -1,16 +0,0 @@
select
'VEHICLE_REGISTRATION' as entity_type,
cast(v.id as varchar(128)) as source_entity_id,
concat('YELLOWFOX:', trim(v.vrn)) as source_external_key,
trim(v.vrn) as display_name,
true as active,
null::timestamptz as valid_from,
null::timestamptz as valid_to,
null::timestamptz as source_updated_at,
v.id as vehicle_id,
trim(v.vin) as vin,
'UNKNOWN' as registration_nation,
trim(v.vrn) as registration_number,
v.fleet_id as fleet_id
from data.vehicle v
where nullif(trim(v.vrn), '') is not null;

17
src/main/resources/sql/yellowfox/master-data/vehicles.sql
View File

@ -1,17 +0,0 @@
select
'VEHICLE' as entity_type,
cast(v.id as varchar(128)) as source_entity_id,
trim(v.vin) as source_external_key,
coalesce(nullif(trim(v.vrn), ''), trim(v.vin)) as display_name,
true as active,
null::timestamptz as valid_from,
null::timestamptz as valid_to,
null::timestamptz as source_updated_at,
v.id as vehicle_id,
trim(v.vin) as vin,
trim(v.vrn) as registration_number,
'UNKNOWN' as registration_nation,
v.fleet_id as fleet_id,
v.telematic_provider_id as telematic_provider_id
from data.vehicle v
where nullif(trim(v.vin), '') is not null;

1
src/test/java/at/procon/eventhub/YellowFoxD8BookingEventMapperTest.java
View File

@ -83,7 +83,6 @@ class YellowFoxD8BookingEventMapperTest {
"event-1", "event-1",
"key-1", "key-1",
ignition, ignition,
null,
eventType, eventType,
state, state,
OffsetDateTime.parse("2026-04-29T08:15:00+02:00"), OffsetDateTime.parse("2026-04-29T08:15:00+02:00"),

35
src/test/java/at/procon/eventhub/esperpoc/service/EsperOperatingPeriodEvaluationServiceTest.java
View File

@ -3,7 +3,6 @@ package at.procon.eventhub.esperpoc.service;
import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThat;
import at.procon.eventhub.esperpoc.dto.ActivityIntervalDto; import at.procon.eventhub.esperpoc.dto.ActivityIntervalDto;
import at.procon.eventhub.esperpoc.dto.EsperOperatingPeriodEngineMode;
import at.procon.eventhub.esperpoc.dto.EsperUnknownTreatmentMode; import at.procon.eventhub.esperpoc.dto.EsperUnknownTreatmentMode;
import at.procon.eventhub.esperpoc.dto.NonDrivingIntervalDto; import at.procon.eventhub.esperpoc.dto.NonDrivingIntervalDto;
import at.procon.eventhub.esperpoc.dto.OperatingPeriodActivityIntervalDto; import at.procon.eventhub.esperpoc.dto.OperatingPeriodActivityIntervalDto;
@ -54,8 +53,7 @@ class EsperOperatingPeriodEvaluationServiceTest {
EsperOperatingPeriodEngine.EsperOperatingPeriodEvaluation evaluation = operatingPeriodEngine.evaluate( EsperOperatingPeriodEngine.EsperOperatingPeriodEvaluation evaluation = operatingPeriodEngine.evaluate(
evaluationIntervals, evaluationIntervals,
Duration.ofHours(7), Duration.ofHours(7)
EsperOperatingPeriodEngineMode.STREAM_COLLECTOR
); );
assertThat(evaluation.periodizedIntervals()).extracting(OperatingPeriodActivityIntervalDto::activityType) assertThat(evaluation.periodizedIntervals()).extracting(OperatingPeriodActivityIntervalDto::activityType)
@ -101,37 +99,6 @@ class EsperOperatingPeriodEvaluationServiceTest {
assertThat(nonDrivingIntervals.get(1).startedAt()).isEqualTo(OffsetDateTime.parse("2026-04-01T09:30:00Z")); assertThat(nonDrivingIntervals.get(1).startedAt()).isEqualTo(OffsetDateTime.parse("2026-04-01T09:30:00Z"));
} }
@Test
void fullEplModeMatchesStreamCollectorMode() {
UUID driverId = UUID.randomUUID();
List<ActivityIntervalDto> evaluationIntervals = List.of(
activity(driverId, "WORK", "2026-04-01T08:00:00Z", "2026-04-01T09:00:00Z", "w1", "DRIVER_CARD"),
activity(driverId, "AVAILABILITY", "2026-04-01T10:00:00Z", "2026-04-01T11:00:00Z", "a1", "DRIVER_CARD"),
unknown(driverId, "2026-04-01T11:00:00Z", "2026-04-01T11:30:00Z"),
activity(driverId, "WORK", "2026-04-01T11:30:00Z", "2026-04-01T12:00:00Z", "w2", "DRIVER_CARD"),
unknown(driverId, "2026-04-01T12:00:00Z", "2026-04-01T20:00:00Z"),
activity(driverId, "DRIVE", "2026-04-01T20:00:00Z", "2026-04-01T20:30:00Z", "d1", "DRIVER_CARD")
);
EsperOperatingPeriodEngine.EsperOperatingPeriodEvaluation collectorEvaluation = operatingPeriodEngine.evaluate(
evaluationIntervals,
Duration.ofHours(7),
EsperOperatingPeriodEngineMode.STREAM_COLLECTOR
);
EsperOperatingPeriodEngine.EsperOperatingPeriodEvaluation fullEplEvaluation = operatingPeriodEngine.evaluate(
evaluationIntervals,
Duration.ofHours(7),
EsperOperatingPeriodEngineMode.FULL_EPL
);
assertThat(fullEplEvaluation.periodizedIntervals())
.usingRecursiveComparison()
.isEqualTo(collectorEvaluation.periodizedIntervals());
assertThat(fullEplEvaluation.closedPeriods())
.usingRecursiveComparison()
.isEqualTo(collectorEvaluation.closedPeriods());
}
private ActivityIntervalDto activity( private ActivityIntervalDto activity(
UUID driverId, UUID driverId,
String activity, String activity,

42
src/test/java/at/procon/eventhub/importing/ImportChunkPlannerTest.java
View File

@ -1,42 +0,0 @@
package at.procon.eventhub.importing;
import at.procon.eventhub.dto.AcquisitionStrategy;
import at.procon.eventhub.dto.EventFamily;
import at.procon.eventhub.dto.EventSourceDto;
import at.procon.eventhub.dto.ImportMode;
import at.procon.eventhub.dto.ImportScopeDto;
import at.procon.eventhub.dto.SourceGroupRefDto;
import at.procon.eventhub.yellowfox.dto.YellowFoxD8ImportRequest;
import java.time.OffsetDateTime;
import java.util.EnumSet;
import org.junit.jupiter.api.Test;
import static org.assertj.core.api.Assertions.assertThat;
class ImportChunkPlannerTest {
@Test
void keepsScheduledRowWatermarkImportsAsSingleChunkEvenWhenScopeHasWindow() {
ImportChunkPlanner planner = new ImportChunkPlanner();
YellowFoxD8ImportRequest request = new YellowFoxD8ImportRequest(
"tenant-1",
new EventSourceDto("YELLOWFOX", "TELEMATICS_PLATFORM", "YELLOWFOX_D8", "instance-1", "setting-1", null),
(SourceGroupRefDto) null,
ImportScopeDto.tenantAll(
OffsetDateTime.parse("2026-04-01T00:00:00+02:00"),
OffsetDateTime.parse("2026-04-10T00:00:00+02:00")
),
EnumSet.noneOf(EventFamily.class),
ImportMode.INCREMENTAL_UPDATE,
false,
AcquisitionStrategy.SOURCE_ROW_WATERMARK
);
assertThat(planner.chunksFor(request, 1))
.containsExactly(new ImportTimeChunkDto(
1,
OffsetDateTime.parse("2026-04-01T00:00:00+02:00"),
OffsetDateTime.parse("2026-04-10T00:00:00+02:00")
));
}
}

187
src/test/java/at/procon/eventhub/importing/extraction/AbstractJdbcExtractionBatchExecutorCursorTest.java
View File

@ -1,187 +0,0 @@
package at.procon.eventhub.importing.extraction;
import at.procon.eventhub.config.EventHubProperties;
import at.procon.eventhub.dto.AcquisitionStrategy;
import at.procon.eventhub.dto.EventFamily;
import at.procon.eventhub.dto.EventSourceDto;
import at.procon.eventhub.dto.ImportCursorStateDto;
import at.procon.eventhub.dto.ImportMode;
import at.procon.eventhub.dto.ImportScopeDto;
import at.procon.eventhub.importing.ExtractionBatchResult;
import at.procon.eventhub.importing.ImportPlanItemDto;
import at.procon.eventhub.importing.ImportRunRequest;
import at.procon.eventhub.importing.ImportTimeChunkDto;
import at.procon.eventhub.importing.persistence.ImportCursorRepository;
import java.time.OffsetDateTime;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.UUID;
import org.junit.jupiter.api.Test;
import static org.assertj.core.api.Assertions.assertThat;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
class AbstractJdbcExtractionBatchExecutorCursorTest {
@Test
void bootstrapsWatermarkCursorFromLatestExistingCursorWhenExactCursorIsMissing() {
ImportCursorRepository repository = mock(ImportCursorRepository.class);
TestExecutor executor = new TestExecutor(repository);
TestRequest request = new TestRequest();
ImportPlanItemDto planItem = new ImportPlanItemDto(
EventFamily.DRIVER_ACTIVITY,
"VEHICLE_UNIT",
"VU_ACTIVITY",
List.of("VUActivity"),
"VEHICLE",
"Vehicle activity",
AcquisitionStrategy.SOURCE_ROW_WATERMARK
);
ImportCursorStateDto fallbackCursor = new ImportCursorStateDto(
null,
"9001",
null,
OffsetDateTime.parse("2026-04-09T23:59:59+02:00")
);
when(repository.findCursor(
"tenant-1",
21,
request.importScope().stableKey(),
EventFamily.DRIVER_ACTIVITY,
"VEHICLE_UNIT",
AcquisitionStrategy.SOURCE_ROW_WATERMARK
)).thenReturn(null);
when(repository.findLatestCursor(
"tenant-1",
21,
request.importScope().stableKey(),
EventFamily.DRIVER_ACTIVITY,
"VEHICLE_UNIT"
)).thenReturn(fallbackCursor);
assertThat(executor.resolveCursor(21, request, planItem)).isEqualTo(fallbackCursor);
verify(repository).findLatestCursor(
"tenant-1",
21,
request.importScope().stableKey(),
EventFamily.DRIVER_ACTIVITY,
"VEHICLE_UNIT"
);
}
private static final class TestExecutor extends AbstractJdbcExtractionBatchExecutor<TestRequest, TestResult> {
private TestExecutor(ImportCursorRepository repository) {
super(null, null, null, new EventHubProperties(), null, repository);
}
private ImportCursorStateDto resolveCursor(int eventSourceId, TestRequest request, ImportPlanItemDto planItem) {
return findCursor(eventSourceId, request, planItem);
}
@Override
protected Optional<ExtractionDefinition<TestRequest>> findDefinition(String code) {
return Optional.empty();
}
@Override
protected EventSourceDto eventSourceFor(TestRequest request, ImportPlanItemDto planItem) {
return request.eventSource();
}
@Override
protected TestResult resultFor(UUID packageId, ImportPlanItemDto planItem, ImportTimeChunkDto chunk, ImportCursorStateDto cursor, ExtractedEventStats stats) {
return new TestResult();
}
}
private record TestRequest() implements ImportRunRequest {
@Override
public String tenantKey() {
return "tenant-1";
}
@Override
public EventSourceDto eventSource() {
return new EventSourceDto("TACHOGRAPH", "VEHICLE_UNIT", "TACHOGRAPH_VEHICLE_UNIT", "instance-1", "setting-1", null);
}
@Override
public at.procon.eventhub.dto.SourceGroupRefDto sourceGroup() {
return null;
}
@Override
public ImportScopeDto importScope() {
return ImportScopeDto.tenantAll(
OffsetDateTime.parse("2026-04-01T00:00:00+02:00"),
OffsetDateTime.parse("2026-04-10T00:00:00+02:00")
);
}
@Override
public Set<EventFamily> eventFamilies() {
return Set.of(EventFamily.DRIVER_ACTIVITY);
}
@Override
public ImportMode mode() {
return ImportMode.INCREMENTAL_UPDATE;
}
@Override
public boolean refreshMasterDataFirst() {
return false;
}
@Override
public AcquisitionStrategy acquisitionStrategy() {
return AcquisitionStrategy.SOURCE_ROW_WATERMARK;
}
}
private static final class TestResult implements ExtractionBatchResult {
@Override
public int eventsInserted() {
return 0;
}
@Override
public boolean executed() {
return false;
}
@Override
public OffsetDateTime lastSourcePackageImportedAt() {
return null;
}
@Override
public String lastSourcePackageId() {
return null;
}
@Override
public OffsetDateTime lastSourceRowUpdatedAt() {
return null;
}
@Override
public OffsetDateTime lastOccurredTo() {
return null;
}
@Override
public Map<String, Integer> eventTypeCounts() {
return Map.of();
}
}
}

105
src/test/java/at/procon/eventhub/yellowfox/service/JdbcYellowFoxD8BookingExtractionBatchExecutorCursorTest.java
View File

@ -1,105 +0,0 @@
package at.procon.eventhub.yellowfox.service;
import at.procon.eventhub.config.EventHubProperties;
import at.procon.eventhub.dto.AcquisitionStrategy;
import at.procon.eventhub.dto.EventFamily;
import at.procon.eventhub.dto.EventSourceDto;
import at.procon.eventhub.dto.ImportCursorStateDto;
import at.procon.eventhub.dto.ImportMode;
import at.procon.eventhub.dto.ImportScopeDto;
import at.procon.eventhub.dto.SourceGroupRefDto;
import at.procon.eventhub.dto.SourceGroupType;
import at.procon.eventhub.importing.ImportPlanItemDto;
import at.procon.eventhub.importing.persistence.ImportCursorRepository;
import at.procon.eventhub.yellowfox.dto.YellowFoxD8ImportRequest;
import java.time.OffsetDateTime;
import java.util.EnumSet;
import java.util.List;
import org.junit.jupiter.api.Test;
import org.springframework.core.io.DefaultResourceLoader;
import static org.assertj.core.api.Assertions.assertThat;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
class JdbcYellowFoxD8BookingExtractionBatchExecutorCursorTest {
@Test
void bootstrapsScheduledWatermarkCursorFromLatestExistingCursorWhenExactCursorIsMissing() {
ImportCursorRepository repository = mock(ImportCursorRepository.class);
JdbcYellowFoxD8BookingExtractionBatchExecutor executor = executor(repository);
YellowFoxD8ImportRequest request = request();
ImportPlanItemDto planItem = new ImportPlanItemDto(
EventFamily.DRIVER_ACTIVITY,
"TELEMATICS_PLATFORM",
"YELLOWFOX_D8_BOOKING",
List.of("data.d8_booking"),
"BOTH",
"YellowFox bookings",
AcquisitionStrategy.SOURCE_ROW_WATERMARK
);
ImportCursorStateDto fallbackCursor = new ImportCursorStateDto(
null,
"4711",
null,
OffsetDateTime.parse("2026-04-09T23:59:59+02:00")
);
when(repository.findCursor(
"tenant-1",
17,
request.importScope().stableKey(),
EventFamily.DRIVER_ACTIVITY,
"TELEMATICS_PLATFORM",
AcquisitionStrategy.SOURCE_ROW_WATERMARK
)).thenReturn(null);
when(repository.findLatestCursor(
"tenant-1",
17,
request.importScope().stableKey(),
EventFamily.DRIVER_ACTIVITY,
"TELEMATICS_PLATFORM"
)).thenReturn(fallbackCursor);
assertThat(executor.findCursor(17, request, planItem)).isEqualTo(fallbackCursor);
verify(repository).findLatestCursor(
"tenant-1",
17,
request.importScope().stableKey(),
EventFamily.DRIVER_ACTIVITY,
"TELEMATICS_PLATFORM"
);
}
private JdbcYellowFoxD8BookingExtractionBatchExecutor executor(ImportCursorRepository repository) {
EventHubProperties properties = new EventHubProperties();
return new JdbcYellowFoxD8BookingExtractionBatchExecutor(
null,
null,
new DefaultResourceLoader(),
repository,
properties,
null,
null,
null
);
}
private YellowFoxD8ImportRequest request() {
return new YellowFoxD8ImportRequest(
"tenant-1",
new EventSourceDto("YELLOWFOX", "TELEMATICS_PLATFORM", "YELLOWFOX_D8", "instance-1", "setting-1", null),
new SourceGroupRefDto(SourceGroupType.FLEET, "7", null, "Fleet 7"),
ImportScopeDto.tenantAll(
OffsetDateTime.parse("2026-04-01T00:00:00+02:00"),
OffsetDateTime.parse("2026-04-10T00:00:00+02:00")
),
EnumSet.of(EventFamily.DRIVER_ACTIVITY),
ImportMode.INCREMENTAL_UPDATE,
false,
AcquisitionStrategy.SOURCE_ROW_WATERMARK
);
}
}

110
src/test/java/at/procon/eventhub/yellowfox/service/JdbcYellowFoxD8BookingExtractionBatchExecutorTest.java
View File

@ -1,110 +0,0 @@
package at.procon.eventhub.yellowfox.service;
import at.procon.eventhub.config.EventHubProperties;
import at.procon.eventhub.dto.AcquisitionStrategy;
import at.procon.eventhub.dto.EventSourceDto;
import at.procon.eventhub.dto.ImportMode;
import at.procon.eventhub.dto.ImportScopeDto;
import at.procon.eventhub.dto.ImportCursorStateDto;
import at.procon.eventhub.dto.SourceGroupRefDto;
import at.procon.eventhub.dto.SourceGroupType;
import at.procon.eventhub.yellowfox.dto.YellowFoxD8ImportRequest;
import java.time.Duration;
import java.time.OffsetDateTime;
import java.util.EnumSet;
import org.junit.jupiter.api.Test;
import org.springframework.core.io.DefaultResourceLoader;
import static org.assertj.core.api.Assertions.assertThat;
class JdbcYellowFoxD8BookingExtractionBatchExecutorTest {
@Test
void omitsCursorParametersWhenNoCursorExists() {
JdbcYellowFoxD8BookingExtractionBatchExecutor executor = executor(Duration.ofHours(2));
JdbcYellowFoxD8BookingExtractionBatchExecutor.QuerySpec query = executor.buildQuerySpec(
request(AcquisitionStrategy.SOURCE_ROW_WATERMARK),
ImportScopeDto.tenantAll(
OffsetDateTime.parse("2026-04-01T00:00:00+02:00"),
OffsetDateTime.parse("2026-04-02T00:00:00+02:00")
),
null
);
assertThat(query.sql()).contains("b.utc >= :occurredFrom");
assertThat(query.sql()).contains("b.utc < :occurredTo");
assertThat(query.sql()).contains("f.id = :fleetId");
assertThat(query.sql()).doesNotContain(":lastOccurredTo");
assertThat(query.sql()).doesNotContain(" is null");
assertThat(query.params()).containsOnlyKeys("occurredFrom", "occurredTo", "fleetId");
assertThat(query.fleetId()).isEqualTo(7);
}
@Test
void keepsStrictUtcEventIdCursorWhenOverlapIsDisabled() {
JdbcYellowFoxD8BookingExtractionBatchExecutor executor = executor(Duration.ZERO);
OffsetDateTime cursorTime = OffsetDateTime.parse("2026-04-02T09:15:00+02:00");
JdbcYellowFoxD8BookingExtractionBatchExecutor.QuerySpec query = executor.buildQuerySpec(
request(AcquisitionStrategy.SOURCE_ROW_WATERMARK),
ImportScopeDto.tenantAll(null, null),
new ImportCursorStateDto(null, "4711", null, cursorTime)
);
assertThat(query.sql()).contains("b.utc > :lastOccurredTo");
assertThat(query.sql()).contains("b.utc = :lastOccurredTo");
assertThat(query.sql()).contains("b.eventid > :lastSourceRowId");
assertThat(query.sql()).doesNotContain(" is null");
assertThat(query.params()).containsEntry("lastOccurredTo", cursorTime);
assertThat(query.params()).containsEntry("lastSourceRowId", "4711");
}
@Test
void keepsOccurredToCapOnceCursorExistsForWatermarkIncrementalRuns() {
JdbcYellowFoxD8BookingExtractionBatchExecutor executor = executor(Duration.ZERO);
OffsetDateTime cursorTime = OffsetDateTime.parse("2026-04-02T09:15:00+02:00");
JdbcYellowFoxD8BookingExtractionBatchExecutor.QuerySpec query = executor.buildQuerySpec(
request(AcquisitionStrategy.SOURCE_ROW_WATERMARK),
ImportScopeDto.tenantAll(
OffsetDateTime.parse("2026-04-01T00:00:00+02:00"),
OffsetDateTime.parse("2026-04-02T00:00:00+02:00")
),
new ImportCursorStateDto(null, "4711", null, cursorTime)
);
assertThat(query.sql()).contains("b.utc >= :occurredFrom");
assertThat(query.sql()).contains("b.utc < :occurredTo");
assertThat(query.params()).containsEntry("occurredFrom", OffsetDateTime.parse("2026-04-01T00:00:00+02:00"));
assertThat(query.params()).containsEntry("occurredTo", OffsetDateTime.parse("2026-04-02T00:00:00+02:00"));
}
private JdbcYellowFoxD8BookingExtractionBatchExecutor executor(Duration overlap) {
EventHubProperties properties = new EventHubProperties();
properties.getYellowFox().setOccurredAtOverlap(overlap);
return new JdbcYellowFoxD8BookingExtractionBatchExecutor(
null,
null,
new DefaultResourceLoader(),
null,
properties,
null,
null,
null
);
}
private YellowFoxD8ImportRequest request(AcquisitionStrategy strategy) {
return new YellowFoxD8ImportRequest(
"tenant-1",
new EventSourceDto("YELLOWFOX", "TELEMATICS_PLATFORM", "YELLOWFOX_D8", "instance-1", "setting-1", null),
new SourceGroupRefDto(SourceGroupType.FLEET, "7", null, "Fleet 7"),
ImportScopeDto.tenantAll(null, null),
EnumSet.noneOf(at.procon.eventhub.dto.EventFamily.class),
ImportMode.INCREMENTAL_UPDATE,
false,
strategy
);
}
}

90
src/test/java/at/procon/eventhub/yellowfox/service/YellowFoxD8BookingRowMapperTest.java
View File

@ -1,90 +0,0 @@
package at.procon.eventhub.yellowfox.service;
import com.fasterxml.jackson.databind.ObjectMapper;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.time.OffsetDateTime;
import java.util.List;
import java.util.Map;
import org.junit.jupiter.api.Test;
import static org.assertj.core.api.Assertions.assertThat;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
class YellowFoxD8BookingRowMapperTest {
@Test
void trimsVehicleRegistrationValuesInReferencesAndPayload() throws SQLException {
YellowFoxD8BookingRowMapper mapper = new YellowFoxD8BookingRowMapper(new ObjectMapper());
ResultSet rs = mock(ResultSet.class);
OffsetDateTime occurredAt = OffsetDateTime.parse("2026-05-08T10:15:30+02:00");
when(rs.getString("eventid")).thenReturn(" evt-1 ");
when(rs.getObject("utc", OffsetDateTime.class)).thenReturn(occurredAt);
when(rs.getInt("vehicle_id")).thenReturn(42);
when(rs.getInt("driver_id")).thenReturn(7);
when(rs.getInt("fleet_id")).thenReturn(9);
when(rs.getInt("odometer")).thenReturn(1234);
when(rs.getInt("telematic_provider_id")).thenReturn(3);
when(rs.getInt("ignition")).thenReturn(1);
when(rs.getInt("previous_ignition")).thenReturn(0);
when(rs.getInt("eventtype")).thenReturn(2);
when(rs.getInt("state")).thenReturn(3);
when(rs.wasNull()).thenReturn(false);
when(rs.getString("vehicle_vrn")).thenReturn(" W-12345 ");
when(rs.getString("vehicle_vin")).thenReturn(" vin-123 ");
when(rs.getString("driver_card_number")).thenReturn(" card-9 ");
when(rs.getString("fleet_name")).thenReturn(" Fleet 9 ");
when(rs.getString("driver_firstname")).thenReturn(" Ada ");
when(rs.getString("driver_lastname")).thenReturn(" Lovelace ");
when(rs.getString("telematic_provider_name")).thenReturn(" YellowFox ");
when(rs.getString("key")).thenReturn(" booking-key ");
when(rs.getString("payload")).thenReturn("""
{"vrn":" W-12345 ","nested":{"label":" value "},"list":[" x ",1]}
""");
var booking = mapper.map(rs, "tenant-1", "instance-1", "setting-1");
assertThat(booking.eventId()).isEqualTo("evt-1");
assertThat(booking.key()).isEqualTo("booking-key");
assertThat(booking.previousIgnition()).isEqualTo(0);
assertThat(booking.vehicleRef().vin()).isEqualTo("VIN-123");
assertThat(booking.vehicleRef().vehicleRegistration().number()).isEqualTo("W-12345");
assertThat(booking.payload()).containsEntry("vrn", "W-12345");
assertThat(booking.payload()).containsEntry("vehicleVrn", "W-12345");
assertThat(booking.payload()).containsEntry("driverFirstName", "Ada");
assertThat(booking.payload()).containsEntry("telematicProviderName", "YellowFox");
assertThat(((Map<?, ?>) booking.payload().get("nested")).get("label")).isEqualTo("value");
assertThat((List<Object>) booking.payload().get("list")).containsExactly("x", 1);
}
@Test
void truncatesBookingDriverCardNumberToFirst14Characters() throws SQLException {
YellowFoxD8BookingRowMapper mapper = new YellowFoxD8BookingRowMapper(new ObjectMapper());
ResultSet rs = mock(ResultSet.class);
OffsetDateTime occurredAt = OffsetDateTime.parse("2026-05-08T10:15:30+02:00");
when(rs.getString("eventid")).thenReturn("evt-2");
when(rs.getObject("utc", OffsetDateTime.class)).thenReturn(occurredAt);
when(rs.getInt("vehicle_id")).thenReturn(0);
when(rs.getInt("driver_id")).thenReturn(0);
when(rs.getInt("fleet_id")).thenReturn(0);
when(rs.getInt("odometer")).thenReturn(0);
when(rs.getInt("telematic_provider_id")).thenReturn(0);
when(rs.getInt("ignition")).thenReturn(1);
when(rs.getInt("previous_ignition")).thenReturn(0);
when(rs.getInt("eventtype")).thenReturn(2);
when(rs.getInt("state")).thenReturn(3);
when(rs.wasNull()).thenReturn(true);
when(rs.getString("driver_card_number")).thenReturn(" 12345678901234AB ");
when(rs.getString("payload")).thenReturn("{}");
var booking = mapper.map(rs, "tenant-1", "instance-1", "setting-1");
assertThat(booking.driverRef()).isNotNull();
assertThat(booking.driverRef().driverCard()).isNotNull();
assertThat(booking.driverRef().driverCard().number()).isEqualTo("12345678901234");
assertThat(booking.payload()).containsEntry("driverCardNumber", "12345678901234");
}
}

39
src/test/java/at/procon/eventhub/yellowfox/service/YellowFoxD8ConfiguredImportPlanServiceTest.java
View File

@ -1,39 +0,0 @@
package at.procon.eventhub.yellowfox.service;
import at.procon.eventhub.config.EventHubProperties;
import at.procon.eventhub.dto.AcquisitionStrategy;
import at.procon.eventhub.dto.EventSourceDto;
import at.procon.eventhub.dto.ImportMode;
import at.procon.eventhub.dto.ImportScopeDto;
import at.procon.eventhub.dto.ImportScopeType;
import at.procon.eventhub.yellowfox.dto.YellowFoxD8ImportRequest;
import java.time.OffsetDateTime;
import org.junit.jupiter.api.Test;
import static org.assertj.core.api.Assertions.assertThat;
class YellowFoxD8ConfiguredImportPlanServiceTest {
@Test
void scheduledWatermarkRequestKeepsInitialOccurredWindowAsBootstrapScope() {
EventHubProperties properties = new EventHubProperties();
EventHubProperties.ConfiguredImportPlan plan = new EventHubProperties.ConfiguredImportPlan();
plan.setPlanKey("yellowfox-d8-default");
plan.setTenantKey("Procon");
plan.setEventSource(new EventSourceDto("YELLOWFOX", "TELEMATICS_PLATFORM", "YELLOWFOX_D8", "logistics-db-prod", "yellowfox-main", null));
plan.setImportScope(new ImportScopeDto(ImportScopeType.TENANT_ALL, null, false, null, null));
plan.setScheduledMode(ImportMode.INCREMENTAL_UPDATE);
plan.setScheduledStrategy(AcquisitionStrategy.SOURCE_ROW_WATERMARK);
plan.setInitialOccurredFrom(OffsetDateTime.parse("2026-04-01T00:00:00+02:00"));
plan.setInitialOccurredTo(OffsetDateTime.parse("2026-04-02T00:00:00+02:00"));
properties.getYellowFox().getImportPlans().add(plan);
YellowFoxD8ConfiguredImportPlanService service = new YellowFoxD8ConfiguredImportPlanService(properties);
YellowFoxD8ImportRequest request = service.createScheduledRequest(plan);
assertThat(request.mode()).isEqualTo(ImportMode.INCREMENTAL_UPDATE);
assertThat(request.acquisitionStrategy()).isEqualTo(AcquisitionStrategy.SOURCE_ROW_WATERMARK);
assertThat(request.importScope().occurredFrom()).isEqualTo(OffsetDateTime.parse("2026-04-01T00:00:00+02:00"));
assertThat(request.importScope().occurredTo()).isEqualTo(OffsetDateTime.parse("2026-04-02T00:00:00+02:00"));
}
}

65
src/test/java/at/procon/eventhub/yellowfox/service/YellowFoxD8IgnitionTransitionDetectorTest.java
View File

@ -1,65 +0,0 @@
package at.procon.eventhub.yellowfox.service;
import at.procon.eventhub.dto.DriverCardRefDto;
import at.procon.eventhub.dto.DriverRefDto;
import at.procon.eventhub.dto.EventDomain;
import at.procon.eventhub.dto.EventType;
import at.procon.eventhub.dto.VehicleRefDto;
import at.procon.eventhub.dto.VehicleRegistrationRefDto;
import at.procon.eventhub.service.EventDetailsFactory;
import at.procon.eventhub.yellowfox.dto.YellowFoxD8BookingDto;
import com.fasterxml.jackson.databind.ObjectMapper;
import java.time.OffsetDateTime;
import java.util.Map;
import org.junit.jupiter.api.Test;
import static org.assertj.core.api.Assertions.assertThat;
class YellowFoxD8IgnitionTransitionDetectorTest {
@Test
void emitsBoundaryTransitionFromPreviousIgnitionFromSourceRow() {
YellowFoxD8BookingEventMapper mapper = new YellowFoxD8BookingEventMapper(new EventDetailsFactory(new ObjectMapper()));
YellowFoxD8IgnitionTransitionDetector detector = new YellowFoxD8IgnitionTransitionDetector(mapper);
var event = detector.newSession(false).detect(booking(1, 0));
assertThat(event).isNotNull();
assertThat(event.eventDomain()).isEqualTo(EventDomain.IGNITION);
assertThat(event.eventType()).isEqualTo(EventType.IGNITION_ON);
}
@Test
void doesNotEmitWhenCurrentIgnitionMatchesPreviousIgnitionFromSourceRow() {
YellowFoxD8BookingEventMapper mapper = new YellowFoxD8BookingEventMapper(new EventDetailsFactory(new ObjectMapper()));
YellowFoxD8IgnitionTransitionDetector detector = new YellowFoxD8IgnitionTransitionDetector(mapper);
var event = detector.newSession(false).detect(booking(1, 1));
assertThat(event).isNull();
}
private YellowFoxD8BookingDto booking(Integer ignition, Integer previousIgnition) {
return new YellowFoxD8BookingDto(
"tenant-1",
"instance-1",
"setting-1",
"7",
"Fleet 7",
"evt-1",
"key-1",
ignition,
previousIgnition,
2,
3,
OffsetDateTime.parse("2026-05-08T10:15:30+02:00"),
null,
new DriverRefDto("1", new DriverCardRefDto(YellowFoxReferenceSemantics.SYNTHETIC_REFERENCE_NATION, "12345678901234")),
new VehicleRefDto("42", "VIN-42", "42", new VehicleRegistrationRefDto(YellowFoxReferenceSemantics.SYNTHETIC_REFERENCE_NATION, "W-4242")),
123_000L,
null,
null,
Map.of()
);
}
}