Compare commits

..

11 Commits

94 changed files with 9321 additions and 433 deletions

View File

@ -264,8 +264,8 @@ DRIVER_ACTIVITY / VEHICLE_UNIT -> VUActivity
DRIVER_ACTIVITY / DRIVER_CARD -> CardActivity
DRIVER_CARD / VEHICLE_UNIT -> IWCycle
DRIVER_CARD / DRIVER_CARD -> CardVehiclesUsed
POSITION / VEHICLE_UNIT -> VUPlaces, VULoadUnload, VUGnssAccumulatedDriving, VUBorderCrossing
POSITION / DRIVER_CARD -> CardPlaces, CardLoadUnload, CardGnssAccumulatedDriving, CardBorderCrossing
POSITION / VEHICLE_UNIT -> VUGnssAccumulatedDriving
POSITION / DRIVER_CARD -> CardGnssAccumulatedDriving
BORDER_CROSSING / VEHICLE_UNIT -> VUBorderCrossing
BORDER_CROSSING / DRIVER_CARD -> CardBorderCrossing
LOAD_UNLOAD / VEHICLE_UNIT -> VULoadUnload
@ -624,6 +624,10 @@ This allows a single EventHub import run to process many original tachograph pac
}
```
For tachograph JDBC extraction, `sourcePackageId` is the original `FileLog.ID`. It is stored directly on `eventhub.event.source_package_id` and also resolved to `source_package_entity_id` for joins to source master data.
`eventhub.event` is designed as a TimescaleDB hypertable partitioned by `occurred_at`. Source-record idempotency is therefore enforced through `eventhub.event_source_record(source_record_key_hash)` instead of a unique index on the hypertable, because Timescale unique indexes must include the partitioning column.
### Tachograph import endpoints
`POST /api/eventhub/acquisition/tachograph/imports/plan` returns the calculated event-family extraction plan and time chunks.
@ -814,14 +818,40 @@ The first concrete extractor is `JdbcTachographExtractionBatchExecutor`. It is e
Currently implemented extraction definitions:
```text
CARD_ACTIVITY -> DRIVER_ACTIVITY / DRIVER_CARD / CardActivity
VU_ACTIVITY -> DRIVER_ACTIVITY / VEHICLE_UNIT / VUActivity
CARD_ACTIVITY -> DRIVER_ACTIVITY / DRIVER_CARD / CardActivity
VU_ACTIVITY -> DRIVER_ACTIVITY / VEHICLE_UNIT / VUActivity
CARD_VEHICLES_USED -> DRIVER_CARD / DRIVER_CARD / CardVehiclesUsed
IW_CYCLE -> DRIVER_CARD / VEHICLE_UNIT / IWCycle
CARD_BORDER_CROSSING -> BORDER_CROSSING / DRIVER_CARD / CardBorderCrossing
VU_BORDER_CROSSING -> BORDER_CROSSING / VEHICLE_UNIT / VUBorderCrossing
CARD_LOAD_UNLOAD -> LOAD_UNLOAD / DRIVER_CARD / CardLoadUnload
VU_LOAD_UNLOAD -> LOAD_UNLOAD / VEHICLE_UNIT / VULoadUnload
CARD_SPECIFIC_CONDITION -> SPECIFIC_CONDITION / DRIVER_CARD / CardSpecificCondition
VU_SPECIFIC_CONDITION -> SPECIFIC_CONDITION / VEHICLE_UNIT / VUSpecificCondition
CARD_POSITION -> POSITION / DRIVER_CARD / CardGnssAccumulatedDriving
VU_POSITION -> POSITION / VEHICLE_UNIT / VUGnssAccumulatedDriving
CARD_PLACE -> PLACE / DRIVER_CARD / CardPlaces
VU_PLACE -> PLACE / VEHICLE_UNIT / VUPlaces
SPEEDING_EVENTS -> SPEEDING / VEHICLE_UNIT / SpeedingEvents
```
SQL resources:
```text
src/main/resources/sql/tachograph/card-border-crossing.sql
src/main/resources/sql/tachograph/card-load-unload.sql
src/main/resources/sql/tachograph/card-place.sql
src/main/resources/sql/tachograph/card-position.sql
src/main/resources/sql/tachograph/card-specific-condition.sql
src/main/resources/sql/tachograph/card-vehicles-used.sql
src/main/resources/sql/tachograph/card-activity.sql
src/main/resources/sql/tachograph/iw-cycle.sql
src/main/resources/sql/tachograph/speeding-events.sql
src/main/resources/sql/tachograph/vu-border-crossing.sql
src/main/resources/sql/tachograph/vu-load-unload.sql
src/main/resources/sql/tachograph/vu-place.sql
src/main/resources/sql/tachograph/vu-position.sql
src/main/resources/sql/tachograph/vu-specific-condition.sql
src/main/resources/sql/tachograph/vu-activity.sql
```

View File

@ -0,0 +1,338 @@
-- Async ingest diagnostics without filesystem logs.
--
-- Replace the values in params before running.
-- completion_size must match eventhub.batch.completion-size from application.yml.
-- 1) Import-run overview.
with params as (
select
'667b191a-4ef4-49d4-9837-05a7737b4fbb'::uuid as import_run_id,
'54ba1294-1061-4b3a-84b6-5a604aa651e6'::uuid as extraction_package_id,
5000::integer as completion_size
)
select
p.id,
p.package_type,
p.status,
p.batch_no,
p.event_family,
p.extraction_code,
p.extraction_source_kind,
p.chunk_from,
p.chunk_to,
p.event_count,
p.error_message,
p.received_at,
p.completed_at
from eventhub.data_package p
join params prm on prm.import_run_id = p.import_run_id
order by p.package_type, p.batch_no, p.received_at, p.package_key;
-- 2) Exact async-ingest status for one DB_EXTRACT package.
-- event_count on DB_EXTRACT is populated after executeBatch returns and before
-- async wait starts, so it represents extracted/mapped events while waiting.
with params as (
select
'667b191a-4ef4-49d4-9837-05a7737b4fbb'::uuid as import_run_id,
'54ba1294-1061-4b3a-84b6-5a604aa651e6'::uuid as extraction_package_id,
5000::integer as completion_size
),
extract_package as (
select
p.id,
p.import_run_id,
p.event_source_id,
p.tenant_key,
p.package_key,
p.status,
p.event_count as extracted_event_count,
p.event_family,
p.business_date,
p.extraction_code,
p.extraction_source_kind,
p.source_group_type,
p.source_group_entity_id,
p.source_group_code,
p.import_scope_type,
p.root_source_org_entity_id,
p.root_source_org_code,
p.include_children,
p.chunk_from,
p.chunk_to,
coalesce((p.metadata ->> 'chunkSequence')::integer, 1) as chunk_sequence,
es.provider_key,
es.source_kind,
es.source_key,
es.source_instance_key,
es.provider_key || ':' || p.extraction_source_kind || ':' || p.extraction_code
|| ':RUN-' || p.import_run_id::text
|| ':CHUNK-' || coalesce((p.metadata ->> 'chunkSequence')::integer, 1)::text as async_external_package_id,
p.tenant_key || ':'
|| es.provider_key || ':' || es.source_kind || ':' || es.source_key || ':' || coalesce(es.source_instance_key, 'default') || ':'
|| case
when p.source_group_type is null and p.source_group_entity_id is null and p.source_group_code is null
then 'NO_GROUP'
else coalesce(p.source_group_type, '') || '|' || coalesce(p.source_group_entity_id, '') || '|' || coalesce(p.source_group_code, '')
end || ':'
|| coalesce(p.import_scope_type, 'TENANT_ALL') || ':'
|| case
when p.root_source_org_entity_id is null and p.root_source_org_code is null
then 'ALL'
else 'ORGANISATION|' || coalesce(p.root_source_org_entity_id, '') || '|' || coalesce(p.root_source_org_code, '')
end || ':'
|| case when coalesce(p.include_children, false) then 'WITH_CHILDREN' else 'NO_CHILDREN' end || ':'
|| coalesce(p.metadata ->> 'chunkOccurredFrom', 'BEGIN') || ':'
|| coalesce(p.metadata ->> 'chunkOccurredTo', 'END') || ':'
|| p.event_family || ':'
|| coalesce(p.business_date::text, 'NO_DATE') || ':'
|| es.provider_key || ':' || p.extraction_source_kind || ':' || p.extraction_code
|| ':RUN-' || p.import_run_id::text
|| ':CHUNK-' || coalesce((p.metadata ->> 'chunkSequence')::integer, 1)::text as aggregate_package_key
from eventhub.data_package p
join eventhub.event_source es
on es.id = p.event_source_id
join params prm
on prm.extraction_package_id = p.id
where p.package_type = 'DB_EXTRACT'
),
camel_batches as (
select
c.id,
c.package_key,
c.status,
c.received_at,
c.completed_at,
c.event_count as inserted_count,
coalesce((c.metadata ->> 'eventCount')::integer, 0) as received_count,
c.metadata ->> 'externalPackageId' as external_package_id,
c.metadata ->> 'aggregatePackageKey' as aggregate_package_key,
c.error_message
from eventhub.data_package c
join extract_package e
on c.event_source_id = e.event_source_id
and c.tenant_key = e.tenant_key
where c.package_type = 'CAMEL_BATCH'
and c.package_key like e.aggregate_package_key || ':CAMEL-%'
),
camel_summary as (
select
count(*) as observed_camel_batches,
count(*) filter (where status in ('IMPORTED', 'EMPTY')) as successful_camel_batches,
count(*) filter (where status = 'FAILED') as failed_camel_batches,
count(*) filter (where status = 'IMPORTING') as importing_camel_batches,
coalesce(sum(received_count), 0) as received_events,
coalesce(sum(inserted_count), 0) as inserted_events
from camel_batches
)
select
e.id as extraction_package_id,
e.import_run_id,
e.package_key as extraction_package_key,
e.status as extraction_status,
e.extracted_event_count,
case
when e.extracted_event_count <= 0 then 0
else ((e.extracted_event_count - 1) / prm.completion_size) + 1
end as expected_camel_batches,
s.observed_camel_batches,
s.successful_camel_batches,
s.failed_camel_batches,
s.importing_camel_batches,
s.received_events,
s.inserted_events,
e.async_external_package_id,
e.aggregate_package_key
from extract_package e
cross join camel_summary s
cross join params prm;
-- 3) Child CAMEL_BATCH rows for the selected DB_EXTRACT package.
with params as (
select
'667b191a-4ef4-49d4-9837-05a7737b4fbb'::uuid as import_run_id,
'54ba1294-1061-4b3a-84b6-5a604aa651e6'::uuid as extraction_package_id,
5000::integer as completion_size
),
extract_package as (
select
p.id,
p.event_source_id,
p.tenant_key,
p.tenant_key || ':'
|| es.provider_key || ':' || es.source_kind || ':' || es.source_key || ':' || coalesce(es.source_instance_key, 'default') || ':'
|| case
when p.source_group_type is null and p.source_group_entity_id is null and p.source_group_code is null
then 'NO_GROUP'
else coalesce(p.source_group_type, '') || '|' || coalesce(p.source_group_entity_id, '') || '|' || coalesce(p.source_group_code, '')
end || ':'
|| coalesce(p.import_scope_type, 'TENANT_ALL') || ':'
|| case
when p.root_source_org_entity_id is null and p.root_source_org_code is null
then 'ALL'
else 'ORGANISATION|' || coalesce(p.root_source_org_entity_id, '') || '|' || coalesce(p.root_source_org_code, '')
end || ':'
|| case when coalesce(p.include_children, false) then 'WITH_CHILDREN' else 'NO_CHILDREN' end || ':'
|| coalesce(p.metadata ->> 'chunkOccurredFrom', 'BEGIN') || ':'
|| coalesce(p.metadata ->> 'chunkOccurredTo', 'END') || ':'
|| p.event_family || ':'
|| coalesce(p.business_date::text, 'NO_DATE') || ':'
|| es.provider_key || ':' || p.extraction_source_kind || ':' || p.extraction_code
|| ':RUN-' || p.import_run_id::text
|| ':CHUNK-' || coalesce((p.metadata ->> 'chunkSequence')::integer, 1)::text as aggregate_package_key
from eventhub.data_package p
join eventhub.event_source es
on es.id = p.event_source_id
join params prm
on prm.extraction_package_id = p.id
where p.package_type = 'DB_EXTRACT'
)
select
c.id,
c.package_key,
c.status,
c.received_at,
c.completed_at,
coalesce((c.metadata ->> 'eventCount')::integer, 0) as received_count,
c.event_count as inserted_count,
c.metadata ->> 'externalPackageId' as external_package_id,
c.metadata ->> 'aggregatePackageKey' as aggregate_package_key,
c.error_message
from eventhub.data_package c
join extract_package e
on c.event_source_id = e.event_source_id
and c.tenant_key = e.tenant_key
where c.package_type = 'CAMEL_BATCH'
and c.package_key like e.aggregate_package_key || ':CAMEL-%'
order by c.received_at, c.package_key;
-- 4) Recent CAMEL_BATCH rows for the run, independent of a single DB_EXTRACT package.
with params as (
select
'667b191a-4ef4-49d4-9837-05a7737b4fbb'::uuid as import_run_id,
'54ba1294-1061-4b3a-84b6-5a604aa651e6'::uuid as extraction_package_id,
5000::integer as completion_size
)
select
c.id,
c.event_source_id,
c.package_key,
c.status,
c.received_at,
c.completed_at,
c.event_count as inserted_count,
c.metadata ->> 'eventFamily' as event_family,
c.metadata ->> 'eventSource' as event_source,
c.metadata ->> 'externalPackageId' as external_package_id,
c.metadata ->> 'aggregatePackageKey' as aggregate_package_key
from eventhub.data_package c
join params prm
on c.package_type = 'CAMEL_BATCH'
where c.package_key like '%RUN-' || prm.import_run_id::text || '%'
or coalesce(c.metadata ->> 'externalPackageId', '') like '%RUN-' || prm.import_run_id::text || '%'
order by c.received_at desc, c.package_key desc;
-- 5) Per event type: extracted vs received by CAMEL vs imported into eventhub.event.
with params as (
select
'4d21ef91-c979-451b-9055-d574506843bf'::uuid as import_run_id,
'20679175-b481-4439-9f66-9b7c7ed336fb'::uuid as extraction_package_id,
5000::integer as completion_size
),
extract_package as (
select
p.id,
p.import_run_id,
p.event_source_id,
p.tenant_key,
p.event_family,
p.business_date,
p.extraction_code,
p.extraction_source_kind,
p.source_group_type,
p.source_group_entity_id,
p.source_group_code,
p.import_scope_type,
p.root_source_org_entity_id,
p.root_source_org_code,
p.include_children,
p.metadata,
es.provider_key,
es.source_kind,
es.source_key,
es.source_instance_key,
p.tenant_key || ':'
|| es.provider_key || ':' || es.source_kind || ':' || es.source_key || ':' || coalesce(es.source_instance_key, 'default') || ':'
|| case
when p.source_group_type is null and p.source_group_entity_id is null and p.source_group_code is null
then 'NO_GROUP'
else coalesce(p.source_group_type, '') || '|' || coalesce(p.source_group_entity_id, '') || '|' || coalesce(p.source_group_code, '')
end || ':'
|| coalesce(p.import_scope_type, 'TENANT_ALL') || ':'
|| case
when p.root_source_org_entity_id is null and p.root_source_org_code is null
then 'ALL'
else 'ORGANISATION|' || coalesce(p.root_source_org_entity_id, '') || '|' || coalesce(p.root_source_org_code, '')
end || ':'
|| case when coalesce(p.include_children, false) then 'WITH_CHILDREN' else 'NO_CHILDREN' end || ':'
|| coalesce(p.metadata ->> 'chunkOccurredFrom', 'BEGIN') || ':'
|| coalesce(p.metadata ->> 'chunkOccurredTo', 'END') || ':'
|| p.event_family || ':'
|| coalesce(p.business_date::text, 'NO_DATE') || ':'
|| es.provider_key || ':' || p.extraction_source_kind || ':' || p.extraction_code
|| ':RUN-' || p.import_run_id::text
|| ':CHUNK-' || coalesce((p.metadata ->> 'chunkSequence')::integer, 1)::text as aggregate_package_key
from eventhub.data_package p
join eventhub.event_source es
on es.id = p.event_source_id
join params prm
on prm.extraction_package_id = p.id
where p.package_type = 'DB_EXTRACT'
),
camel_batches as (
select c.*
from eventhub.data_package c
join extract_package e
on c.event_source_id = e.event_source_id
and c.tenant_key = e.tenant_key
where c.package_type = 'CAMEL_BATCH'
and c.package_key like e.aggregate_package_key || ':CAMEL-%'
),
extracted_by_type as (
select
key as event_type_key,
value::integer as extracted_events
from extract_package e,
jsonb_each_text(coalesce(e.metadata -> 'extractedEventTypeCounts', '{}'::jsonb))
),
received_by_type as (
select
kv.key as event_type_key,
sum((kv.value)::integer) as received_events
from camel_batches c,
jsonb_each_text(coalesce(c.metadata -> 'receivedEventTypeCounts', '{}'::jsonb)) kv
group by kv.key
),
imported_by_type as (
select
e.event_domain || '/' || e.event_type as event_type_key,
count(*) as imported_events
from camel_batches c
join eventhub.event e
on e.data_package_id = c.id
group by e.event_domain || '/' || e.event_type
)
select
coalesce(x.event_type_key, r.event_type_key, i.event_type_key) as event_type_key,
coalesce(x.extracted_events, 0) as extracted_events,
coalesce(r.received_events, 0) as received_events,
coalesce(i.imported_events, 0) as imported_events
from extracted_by_type x
full outer join received_by_type r
on r.event_type_key = x.event_type_key
full outer join imported_by_type i
on i.event_type_key = coalesce(x.event_type_key, r.event_type_key)
order by event_type_key;

View File

@ -0,0 +1,317 @@
create extension if not exists pgcrypto;
create extension if not exists postgis;
create schema if not exists eventhub;
create table if not exists eventhub.event_source (
id integer generated always as identity primary key,
tenant_key text not null,
provider_key text not null,
source_kind text not null,
source_key text not null,
source_instance_key text not null default 'default',
tenant_provider_setting_key text,
external_fleet_key text,
created_at timestamptz not null default now(),
constraint ux_event_source unique (tenant_key, provider_key, source_kind, source_key, source_instance_key)
);
create table if not exists eventhub.import_run (
id uuid primary key,
tenant_key text not null,
event_source_id integer not null references eventhub.event_source(id),
mode text not null,
status text not null,
refresh_master_data_first boolean not null default true,
source_group_type text,
source_group_entity_id text,
source_group_code text,
source_group_name text,
import_scope_type text not null,
root_source_org_entity_id text,
root_source_org_code text,
root_source_org_name text,
include_children boolean not null default false,
occurred_from timestamptz,
occurred_to timestamptz,
requested_event_families text[] not null default '{}',
acquisition_strategy text,
metadata jsonb not null default '{}'::jsonb,
planned_package_count integer not null default 0,
started_at timestamptz not null default now(),
finished_at timestamptz,
error_message text,
constraint chk_import_run_occ_time_order check (occurred_from is null or occurred_to is null or occurred_from < occurred_to)
);
create table if not exists eventhub.import_cursor (
id uuid primary key,
tenant_key text not null,
event_source_id integer not null references eventhub.event_source(id),
scope_hash text not null,
event_family text not null,
source_kind text not null,
cursor_type text not null,
last_source_package_imported_at timestamptz,
last_source_package_id text,
last_source_row_updated_at timestamptz,
last_occurred_to timestamptz,
updated_at timestamptz not null default now(),
constraint ux_import_cursor unique (tenant_key, event_source_id, scope_hash, event_family, source_kind, cursor_type)
);
create table if not exists eventhub.data_package (
id uuid primary key,
event_source_id integer not null references eventhub.event_source(id),
import_run_id uuid references eventhub.import_run(id),
tenant_key text not null,
package_key text not null,
package_type text not null,
status text not null,
source_group_type text,
source_group_entity_id text,
source_group_code text,
source_group_name text,
import_scope_type text,
root_source_org_entity_id text,
root_source_org_code text,
root_source_org_name text,
include_children boolean not null default false,
occurred_from timestamptz,
occurred_to timestamptz,
event_family text,
business_date date,
external_package_id text,
extraction_code text,
extraction_source_kind text,
entity_axis text,
batch_no integer,
chunk_from timestamptz,
chunk_to timestamptz,
source_package_kind text,
source_package_id text,
source_package_entity_id text,
source_package_period_from timestamptz,
source_package_period_to timestamptz,
source_package_imported_at timestamptz,
received_at timestamptz not null default now(),
completed_at timestamptz,
event_count integer not null default 0,
metadata jsonb not null default '{}'::jsonb,
error_message text,
constraint ux_data_package_package_key unique (tenant_key, event_source_id, package_key),
constraint chk_data_package_occ_time_order check (occurred_from is null or occurred_to is null or occurred_from < occurred_to),
constraint chk_data_package_chunk_time_order check (chunk_from is null or chunk_to is null or chunk_from < chunk_to)
);
create table if not exists eventhub.source_master_entity (
id uuid primary key,
tenant_key text not null,
event_source_id integer not null references eventhub.event_source(id),
entity_type text not null,
source_entity_id text not null,
source_external_key text,
display_name text,
active boolean,
valid_from timestamptz,
valid_to timestamptz,
source_updated_at timestamptz,
payload jsonb not null default '{}'::jsonb,
created_at timestamptz not null default now(),
updated_at timestamptz not null default now(),
constraint ux_source_master_entity unique (tenant_key, event_source_id, entity_type, source_entity_id),
constraint chk_source_master_entity_valid_time_order check (valid_from is null or valid_to is null or valid_from <= valid_to)
);
create table if not exists eventhub.source_master_relation (
id uuid primary key,
tenant_key text not null,
event_source_id integer not null references eventhub.event_source(id),
relation_key text not null,
relation_type text not null,
from_entity_type text not null,
from_source_entity_id text not null,
to_entity_type text not null,
to_source_entity_id text not null,
valid_from timestamptz,
valid_to timestamptz,
source_updated_at timestamptz,
payload jsonb not null default '{}'::jsonb,
created_at timestamptz not null default now(),
updated_at timestamptz not null default now(),
constraint ux_source_master_relation unique (tenant_key, event_source_id, relation_key),
constraint chk_source_master_relation_valid_time_order check (valid_from is null or valid_to is null or valid_from <= valid_to)
);
create table if not exists eventhub.vehicle (
id uuid primary key,
tenant_key text not null,
event_source_id integer not null references eventhub.event_source(id),
source_vehicle_entity_id text,
vin text,
created_at timestamptz not null default now(),
updated_at timestamptz not null default now()
);
create table if not exists eventhub.vehicle_registration (
id uuid primary key,
tenant_key text not null,
event_source_id integer not null references eventhub.event_source(id),
source_registration_entity_id text,
nation text not null,
registration_number text not null,
source_updated_at timestamptz,
payload jsonb not null default '{}'::jsonb,
created_at timestamptz not null default now(),
updated_at timestamptz not null default now()
);
create table if not exists eventhub.vehicle_registration_assignment (
id uuid primary key,
tenant_key text not null,
event_source_id integer not null references eventhub.event_source(id),
vehicle_registration_id uuid not null references eventhub.vehicle_registration(id) on delete cascade,
vehicle_id uuid not null references eventhub.vehicle(id) on delete cascade,
valid_from timestamptz,
valid_to timestamptz,
source_updated_at timestamptz,
payload jsonb not null default '{}'::jsonb,
created_at timestamptz not null default now(),
updated_at timestamptz not null default now(),
constraint chk_vehicle_registration_assignment_valid_time_order check (valid_from is null or valid_to is null or valid_from <= valid_to)
);
create table if not exists eventhub.event (
id uuid not null,
event_source_id integer not null references eventhub.event_source(id),
data_package_id uuid not null references eventhub.data_package(id),
external_source_event_id text not null,
driver_entity_id uuid references eventhub.source_master_entity(id),
vehicle_id uuid references eventhub.vehicle(id),
vehicle_registration_id uuid references eventhub.vehicle_registration(id),
source_package_entity_id uuid references eventhub.source_master_entity(id),
occurred_at timestamptz not null,
received_partner_at timestamptz,
received_hub_at timestamptz not null default now(),
event_domain text not null,
event_type text not null,
lifecycle text not null,
odometer_m bigint,
position geography(Point, 4326),
payload jsonb not null default '{}'::jsonb,
manual_entry boolean not null default false,
source_record_key_hash text not null,
event_signature_hash text,
created_at timestamptz not null default now(),
constraint pk_event primary key (occurred_at, id),
constraint chk_event_driver_or_vehicle_ref check (
driver_entity_id is not null
or vehicle_id is not null
or vehicle_registration_id is not null
)
);
create table if not exists eventhub.event_detail (
event_occurred_at timestamptz not null,
event_id uuid not null,
detail_type text not null,
attributes jsonb not null default '{}'::jsonb,
created_at timestamptz not null default now(),
constraint pk_event_detail primary key (event_occurred_at, event_id, detail_type),
constraint fk_event_detail_event foreign key (event_occurred_at, event_id)
references eventhub.event(occurred_at, id)
on delete cascade
);
create unique index if not exists ux_event_source_record
on eventhub.event(source_record_key_hash);
create index if not exists idx_event_signature
on eventhub.event(event_signature_hash)
where event_signature_hash is not null;
create index if not exists idx_event_source_time
on eventhub.event(event_source_id, occurred_at desc);
create index if not exists idx_event_package_time
on eventhub.event(data_package_id, occurred_at desc);
create index if not exists idx_event_domain_type_time
on eventhub.event(event_domain, event_type, occurred_at desc);
create index if not exists idx_event_driver_time
on eventhub.event(driver_entity_id, occurred_at desc)
where driver_entity_id is not null;
create index if not exists idx_event_vehicle_time
on eventhub.event(vehicle_id, occurred_at desc)
where vehicle_id is not null;
create index if not exists idx_event_vehicle_registration_time
on eventhub.event(vehicle_registration_id, occurred_at desc)
where vehicle_registration_id is not null;
create index if not exists idx_event_position_gist
on eventhub.event using gist(position)
where position is not null;
create index if not exists idx_event_payload_gin
on eventhub.event using gin(payload);
create index if not exists idx_event_detail_type
on eventhub.event_detail(detail_type);
create index if not exists idx_event_detail_attributes_gin
on eventhub.event_detail using gin(attributes);
create index if not exists idx_source_master_entity_type_key
on eventhub.source_master_entity(tenant_key, event_source_id, entity_type, source_external_key)
where source_external_key is not null;
create index if not exists idx_source_master_entity_payload_gin
on eventhub.source_master_entity using gin(payload);
create index if not exists idx_source_master_relation_from
on eventhub.source_master_relation(tenant_key, event_source_id, from_entity_type, from_source_entity_id, relation_type);
create index if not exists idx_source_master_relation_to
on eventhub.source_master_relation(tenant_key, event_source_id, to_entity_type, to_source_entity_id, relation_type);
create index if not exists idx_source_master_relation_payload_gin
on eventhub.source_master_relation using gin(payload);
create index if not exists idx_vehicle_lookup_ctx
on eventhub.vehicle(tenant_key, event_source_id, updated_at desc);
create index if not exists idx_vehicle_source_entity
on eventhub.vehicle(tenant_key, event_source_id, source_vehicle_entity_id)
where source_vehicle_entity_id is not null;
create index if not exists idx_vehicle_vin
on eventhub.vehicle(tenant_key, event_source_id, vin)
where vin is not null;
create index if not exists idx_vehicle_registration_source_entity
on eventhub.vehicle_registration(tenant_key, event_source_id, source_registration_entity_id)
where source_registration_entity_id is not null;
create index if not exists idx_vehicle_registration_plate
on eventhub.vehicle_registration(tenant_key, event_source_id, nation, registration_number);
create index if not exists idx_vehicle_registration_assignment_registration_time
on eventhub.vehicle_registration_assignment(vehicle_registration_id, valid_from desc, valid_to);
create index if not exists idx_vehicle_registration_assignment_vehicle_time
on eventhub.vehicle_registration_assignment(vehicle_id, valid_from desc, valid_to);
create index if not exists idx_data_package_source_time
on eventhub.data_package(tenant_key, event_source_id, received_at desc);
create index if not exists idx_data_package_scope
on eventhub.data_package(tenant_key, import_scope_type, root_source_org_entity_id, occurred_from, occurred_to);
create index if not exists idx_data_package_extraction
on eventhub.data_package(tenant_key, event_source_id, import_run_id, event_family, extraction_source_kind, extraction_code, batch_no);
create index if not exists idx_import_run_source_status
on eventhub.import_run(tenant_key, event_source_id, status, started_at desc);

View File

@ -0,0 +1,459 @@
# Import Performance and Error Fixes
## Scope
This document summarizes the event-ingest, master-data, schema, locking, retry, and correctness fixes made in the current optimization round for the EventHub import pipeline.
It covers:
- event schema and migration fixes
- event import throughput fixes
- master-data import throughput fixes
- deadlock and transaction-visibility fixes
- connection-reset and retry handling
- async import cursor correctness fixes
- logging and operational visibility improvements
Main committed changes:
- `2e6e1aa` - `Optimize ingestion pipeline and reduce import contention`
- `bd3620b` - `Improve vehicle reference caching during ingest`
Additional related fixes are currently present in the workspace but may not yet be committed.
## 1. Schema and Migration Fixes
### 1.1 `event_detail` / hypertable ordering
Problem:
- executing `eventhub_schema_create.sql` on an empty database failed with `relation "eventhub.event_detail" does not exist`
Fix:
- create `eventhub.event_detail` before `create_hypertable(...)`
- add its foreign key after the hypertable conversion
Files:
- `src/main/resources/db/eventhub_schema_create.sql`
### 1.2 Explicit migrations for event hypertable and source-record support
Problem:
- the runtime schema evolution needed explicit migrations for hypertable conversion and `event_source_record`
Fix:
- add migration for `source_package_id` on `event`
- add migration for `event` hypertable conversion and FK recreation
- add migration to ensure `event_source_record` exists and is backfilled
Files:
- `src/main/resources/db/migration/V9__add_event_source_package_id.sql`
- `src/main/resources/db/migration/V10__make_event_hypertable.sql`
- `src/main/resources/db/migration/V11__ensure_event_source_record.sql`
## 2. Event Import Throughput Fixes
### 2.1 Replace per-event inserts with staged set-based writes
Problem:
- `EventRepository.batchInsert(...)` originally processed events one by one despite the batch API
- this caused one insert/query cycle per event and poor throughput
Fix:
- stage a whole ingest batch into `eventhub_event_import_stage`
- reserve `event_source_record` rows set-wise
- insert `eventhub.event` rows set-wise
- upsert `eventhub.event_detail` rows in batch
Files:
- `src/main/java/at/procon/eventhub/persistence/EventRepository.java`
### 2.2 Fix missing event rows when source records were reserved
Problem:
- after the set-based refactor, some runs created `event_source_record` rows without creating `event` rows
Cause:
- the insert statement reserved source records and then tried to re-read them through the base table in the same data-modifying CTE chain
Fix:
- use the `RETURNING` rows from the source-record reservation CTE directly
- also support already-existing source records that still miss the `event` row
Files:
- `src/main/java/at/procon/eventhub/persistence/EventRepository.java`
### 2.3 Stream extraction instead of materializing full result sets
Problem:
- extraction loaded full SQL chunks into memory before handing them to Camel
Fix:
- stream rows directly from JDBC to `direct:eventhub-normalized-input`
- keep only counters and watermark information
Files:
- `src/main/java/at/procon/eventhub/importing/extraction/AbstractJdbcExtractionBatchExecutor.java`
- `src/main/java/at/procon/eventhub/tachograph/service/JdbcTachographExtractionBatchExecutor.java`
### 2.4 Increase batch size and enable parallel queue draining
Problem:
- the async ingest route drained too slowly with `1000`-event batches and a single consumer
Fix:
- raise Camel completion size from `1000` to `5000`
- enable `4` concurrent SEDA consumers
Files:
- `src/main/java/at/procon/eventhub/config/EventHubProperties.java`
- `src/main/java/at/procon/eventhub/camel/EventHubCommonIngestionRoute.java`
- `src/main/resources/application.yml`
### 2.5 Give each Camel flush its own package key
Problem:
- multiple flushes of the same extraction package reused the same `data_package` identity
- logs were misleading and `event_count` on the package row was overwritten by later flushes
Fix:
- derive a unique `packageKey` per completed Camel batch using the aggregate package key plus the Camel exchange id
- preserve both the aggregate key and the child key in metadata
Files:
- `src/main/java/at/procon/eventhub/camel/EventHubBatchBuildProcessor.java`
### 2.6 Improve batch-local entity and vehicle caching
Problem:
- after the bulk insert refactor, the main remaining hot path was still reference resolution
Fixes:
- cache entity ids in the batch by `entityType + sourceEntityId`
- cache vehicle resolutions inside a batch
- later extend vehicle caching to be range-aware for registration-based assignment lookups:
- direct vehicle identifiers cache without time sensitivity
- registration-based resolutions cache over assignment validity intervals
Files:
- `src/main/java/at/procon/eventhub/persistence/EventRepository.java`
- `src/main/java/at/procon/eventhub/persistence/VehicleIdentityRepository.java`
## 3. Master-Data Import Throughput Fixes
### 3.1 Set-based master entity and relation upserts
Problem:
- source master data was previously written row by row
Fix:
- stage master entities and relations into temporary tables
- run set-based `insert ... select ... on conflict do update`
Files:
- `src/main/java/at/procon/eventhub/persistence/SourceMasterDataRepository.java`
### 3.2 Stream and chunk master-data refresh
Problem:
- the refresh path loaded large source master-data result sets into memory
Fix:
- stream source rows
- flush in chunks of `5000`
Files:
- `src/main/java/at/procon/eventhub/tachograph/service/TachographMasterDataRefreshService.java`
### 3.3 Bulk vehicle reconciliation from master data
Problem:
- reconciling vehicles and registrations from master data was done row by row
Fix:
- replace the loop with set-based SQL for vehicles, registrations, and projected assignments
Files:
- `src/main/java/at/procon/eventhub/persistence/VehicleIdentityRepository.java`
## 4. Deadlock and Contention Fixes
### 4.1 Remove unnecessary hot-row updates on vehicle and registration rows
Problem:
- event import updated `vehicle.updated_at` and `vehicle_registration.updated_at` even when no new information was being added
- this created deadlocks under parallel ingest
Fix:
- only update `vehicle` when missing `source_vehicle_entity_id` or `vin` can actually be filled
- only update `vehicle_registration` when missing source id, nation, or registration number can actually be filled
- stop using event import as a generic "touch row" path
Files:
- `src/main/java/at/procon/eventhub/persistence/VehicleIdentityRepository.java`
### 4.2 Make event-time source master entity resolution "find or create", not "update on conflict"
Problem:
- concurrent event batches could deadlock on `eventhub.source_master_entity` through `INSERT ... ON CONFLICT DO UPDATE`
Fix:
- first `SELECT id`
- if missing, `INSERT ... ON CONFLICT DO NOTHING RETURNING id`
- if another transaction won the race, select again
- do not update existing master entity rows during event ingest
Files:
- `src/main/java/at/procon/eventhub/persistence/SourceMasterDataRepository.java`
### 4.3 Fix race handling when `RETURNING` returns no row
Problem:
- if a concurrent transaction inserted the entity first, the resolver could still fail unexpectedly
Fix:
- allow the `RETURNING` path to yield `null`
- retry with a follow-up `SELECT`
Files:
- `src/main/java/at/procon/eventhub/persistence/SourceMasterDataRepository.java`
## 5. Transaction Visibility and Correctness Fixes
### 5.1 Remove outer transaction around full tachograph execution
Problem:
- master-data refresh logs showed completion, but master-data rows were not visible yet because the outer import method still held the transaction open
Fix:
- remove the outer transaction from `startAndExecuteImport(...)`
- keep chunk-level and package-level transactions independent
Files:
- `src/main/java/at/procon/eventhub/tachograph/service/TachographImportExecutionService.java`
### 5.2 Preserve the original ingest exception if package failure marking also fails
Problem:
- when ingest failed and `markFailed(...)` also failed because of a broken connection, the secondary bookkeeping error hid the real root cause
Fix:
- wrap `dataPackageRepository.markFailed(...)` in its own `try/catch`
- log the bookkeeping failure
- keep the original ingest exception and attach the bookkeeping failure as suppressed
Files:
- `src/main/java/at/procon/eventhub/service/EventHubIngestionService.java`
### 5.3 Do not advance import cursors before async ingest really finishes
Problem:
- extraction previously marked packages imported and advanced `import_cursor` before the async `CAMEL_BATCH` ingest was durably finished
- this could skip source data on the next run if async ingest later failed
Fix:
- add grouped child-batch status lookup on `data_package`
- make extraction package completion wait for all derived `CAMEL_BATCH` rows to reach terminal success
- fail the planned extraction package if child batches fail or time out
- only advance the cursor after the async ingest succeeded
- make "Completed import run" mean durable ingest completion instead of extraction completion
Files:
- `src/main/java/at/procon/eventhub/importing/AbstractImportExecutionService.java`
- `src/main/java/at/procon/eventhub/persistence/DataPackageRepository.java`
- `src/main/java/at/procon/eventhub/tachograph/service/TachographImportExecutionService.java`
## 6. Connection Reset and Retry Hardening
### 6.1 Retry transient DB failures in the Camel ingest route
Problem:
- long-running imports hit transient failures such as deadlocks and connection resets
Fix:
- add Camel redelivery with exponential backoff for:
- `CannotAcquireLockException`
- `PessimisticLockingFailureException`
- `DataAccessResourceFailureException`
- `TransientDataAccessException`
Files:
- `src/main/java/at/procon/eventhub/camel/EventHubCommonIngestionRoute.java`
### 6.2 Tune Hikari for shorter-lived and healthier pooled connections
Problem:
- `SQLSTATE 08006` / `Connection reset` events left broken pool entries behind
Fix:
- configure Hikari with explicit pool sizing and connection lifetime / keepalive settings:
- `maximum-pool-size: 16`
- `minimum-idle: 4`
- `connection-timeout: 30000`
- `validation-timeout: 5000`
- `idle-timeout: 300000`
- `keepalive-time: 120000`
- `max-lifetime: 540000`
Files:
- `src/main/resources/application.yml`
## 7. Observability Improvements
### 7.1 Master-data progress logging
Added logs for:
- refresh start
- per-section progress
- per-chunk counts
- `byType` breakdowns
- section completion
- reconciliation start and result
Files:
- `src/main/java/at/procon/eventhub/tachograph/service/TachographMasterDataRefreshService.java`
### 7.2 Event extraction progress logging
Added logs for:
- extraction start
- progress every `5000` mapped events
- final mapped totals with `byType`
Files:
- `src/main/java/at/procon/eventhub/importing/extraction/AbstractJdbcExtractionBatchExecutor.java`
### 7.3 Event ingest throughput logging
Added logs for:
- `receivedCount`
- `insertedCount`
- `elapsedMs`
- `receivedPerSecond`
- `byType`
Files:
- `src/main/java/at/procon/eventhub/service/EventHubIngestionService.java`
### 7.4 Async-ingest wait progress logging
Added logs for:
- number of expected child batches
- observed child batches
- successful / failed / importing child-batch counts while the import executor waits for durable completion
Files:
- `src/main/java/at/procon/eventhub/importing/AbstractImportExecutionService.java`
## 8. Operational Notes
### Throughput effect seen during the optimization round
Observed progression during the work:
- roughly `30` events/sec before the later cache and blocking fixes
- roughly `300` rows/sec after the main contention and stuck-session cleanup work
This is a major improvement, but large historical backfills are still expensive.
### What remains expensive
The main remaining bottleneck is still reference resolution in the ingest hot path, especially:
- driver entity resolution
- source-package entity resolution
- vehicle / registration lookup and creation
The next major optimization step would be set-based pre-resolution of references per ingest batch instead of resolving them one event at a time.
### Safe rerun behavior
- event ingest remains idempotent through `event_source_record.source_record_key_hash`
- already imported events should generally be kept
- when historical cursor corruption existed, repair should target `import_cursor`, not wholesale deletion of imported events
## 9. Main Files Touched
- `src/main/java/at/procon/eventhub/persistence/EventRepository.java`
- `src/main/java/at/procon/eventhub/persistence/VehicleIdentityRepository.java`
- `src/main/java/at/procon/eventhub/persistence/SourceMasterDataRepository.java`
- `src/main/java/at/procon/eventhub/persistence/DataPackageRepository.java`
- `src/main/java/at/procon/eventhub/service/EventHubIngestionService.java`
- `src/main/java/at/procon/eventhub/camel/EventHubCommonIngestionRoute.java`
- `src/main/java/at/procon/eventhub/camel/EventHubBatchBuildProcessor.java`
- `src/main/java/at/procon/eventhub/importing/extraction/AbstractJdbcExtractionBatchExecutor.java`
- `src/main/java/at/procon/eventhub/importing/AbstractImportExecutionService.java`
- `src/main/java/at/procon/eventhub/tachograph/service/JdbcTachographExtractionBatchExecutor.java`
- `src/main/java/at/procon/eventhub/tachograph/service/TachographMasterDataRefreshService.java`
- `src/main/java/at/procon/eventhub/tachograph/service/TachographImportExecutionService.java`
- `src/main/java/at/procon/eventhub/config/EventHubProperties.java`
- `src/main/resources/application.yml`
- `src/main/resources/db/eventhub_schema_create.sql`
- `src/main/resources/db/migration/V9__add_event_source_package_id.sql`
- `src/main/resources/db/migration/V10__make_event_hypertable.sql`
- `src/main/resources/db/migration/V11__ensure_event_source_record.sql`

View File

@ -3,7 +3,7 @@
create extension if not exists timescaledb;
select create_hypertable(
'eventhub.acquired_event',
'eventhub.event',
'occurred_at',
if_not_exists => true,
migrate_data => true

21
pom.xml
View File

@ -97,6 +97,27 @@
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-enforcer-plugin</artifactId>
<version>3.6.1</version>
<executions>
<execution>
<id>enforce-java</id>
<goals>
<goal>enforce</goal>
</goals>
<configuration>
<rules>
<requireJavaVersion>
<version>[21,)</version>
<message>This project requires Java 21+ to build and run.</message>
</requireJavaVersion>
</rules>
</configuration>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-maven-plugin</artifactId>

View File

@ -8,6 +8,7 @@ import at.procon.eventhub.dto.ImportScopeDto;
import at.procon.eventhub.service.EventHubEventSorter;
import java.time.OffsetDateTime;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import org.apache.camel.Exchange;
@ -29,7 +30,8 @@ public class EventHubBatchBuildProcessor implements Processor {
List<EventHubEventDto> events = exchange.getMessage().getBody(List.class);
List<EventHubEventDto> sortedEvents = sorter.sort(events);
String packageKey = exchange.getMessage().getHeader(EventHubHeaders.PACKAGE_KEY, String.class);
String aggregatePackageKey = exchange.getMessage().getHeader(EventHubHeaders.PACKAGE_KEY, String.class);
String packageKey = aggregatePackageKey + ":CAMEL-" + exchange.getExchangeId();
EventHubPackageRequest packageInfo = exchange.getMessage().getHeader(EventHubHeaders.PACKAGE_INFO, EventHubPackageRequest.class);
if (packageInfo == null && !sortedEvents.isEmpty()) {
packageInfo = sortedEvents.getFirst().packageInfo();
@ -46,7 +48,10 @@ public class EventHubBatchBuildProcessor implements Processor {
Map<String, Object> metadata = new HashMap<>();
metadata.put("camelRouteId", exchange.getFromRouteId());
metadata.put("packageKey", packageKey);
metadata.put("aggregatePackageKey", aggregatePackageKey);
metadata.put("camelExchangeId", exchange.getExchangeId());
metadata.put("eventCount", sortedEvents.size());
metadata.put("receivedEventTypeCounts", eventTypeCounts(sortedEvents));
if (packageInfo != null) {
metadata.put("tenantKey", packageInfo.tenantKey());
metadata.put("eventSource", packageInfo.eventSource().stableKey());
@ -67,4 +72,14 @@ public class EventHubBatchBuildProcessor implements Processor {
metadata
));
}
private Map<String, Integer> eventTypeCounts(List<EventHubEventDto> events) {
Map<String, Integer> counts = new LinkedHashMap<>();
for (EventHubEventDto event : events) {
String domain = event.eventDomain() == null ? "UNKNOWN_DOMAIN" : event.eventDomain().name();
String type = event.eventType() == null ? "UNKNOWN_EVENT" : event.eventType().name();
counts.merge(domain + "/" + type, 1, Integer::sum);
}
return counts;
}
}

View File

@ -2,12 +2,19 @@ package at.procon.eventhub.camel;
import at.procon.eventhub.config.EventHubProperties;
import at.procon.eventhub.service.EventHubIngestionService;
import org.springframework.dao.CannotAcquireLockException;
import org.springframework.dao.DataAccessResourceFailureException;
import org.springframework.dao.PessimisticLockingFailureException;
import org.springframework.dao.TransientDataAccessException;
import org.apache.camel.builder.RouteBuilder;
import org.springframework.stereotype.Component;
import org.springframework.transaction.TransactionSystemException;
@Component
public class EventHubCommonIngestionRoute extends RouteBuilder {
private static final String BATCH_INPUT_QUEUE = "seda:eventhub-batch-input";
private final EventHubProperties properties;
private final EventHubEventValidationProcessor validationProcessor;
private final EventHubPackageKeyProcessor packageKeyProcessor;
@ -33,13 +40,28 @@ public class EventHubCommonIngestionRoute extends RouteBuilder {
@Override
public void configure() {
String batchInputUri = batchInputUri();
onException(
CannotAcquireLockException.class,
PessimisticLockingFailureException.class,
DataAccessResourceFailureException.class,
TransientDataAccessException.class,
TransactionSystemException.class
)
.maximumRedeliveries(5)
.redeliveryDelay(2000)
.backOffMultiplier(2.0)
.useExponentialBackOff()
.retryAttemptedLogLevel(org.apache.camel.LoggingLevel.WARN);
from("direct:eventhub-normalized-input")
.routeId("eventhub-normalized-input-route")
.process(validationProcessor)
.process(packageKeyProcessor)
.to("seda:eventhub-batch-input");
.to(batchInputUri);
from("seda:eventhub-batch-input")
from(batchInputUri)
.routeId("eventhub-batch-and-persist-route")
.aggregate(header(EventHubHeaders.PACKAGE_KEY), aggregationStrategy)
.completionSize(properties.getBatch().getCompletionSize())
@ -48,4 +70,13 @@ public class EventHubCommonIngestionRoute extends RouteBuilder {
.process(batchBuildProcessor)
.bean(ingestionService, "ingest");
}
private String batchInputUri() {
EventHubProperties.Batch batch = properties.getBatch();
return BATCH_INPUT_QUEUE
+ "?size=" + batch.getQueueSize()
+ "&concurrentConsumers=" + batch.getConcurrentConsumers()
+ "&blockWhenFull=" + batch.isBlockWhenFull()
+ "&offerTimeout=" + batch.getQueueOfferTimeout().toMillis();
}
}

View File

@ -32,11 +32,23 @@ public class EventHubProperties {
public static class Batch {
/** Number of events collected before a package is persisted. */
private int completionSize = 1000;
private int completionSize = 5000;
/** Maximum time to wait for more events belonging to the same package key. */
private Duration completionTimeout = Duration.ofSeconds(5);
/** Maximum number of normalized events buffered before producers apply backpressure. */
private int queueSize = 10000;
/** Number of parallel consumers draining the normalized-event queue. */
private int concurrentConsumers = 4;
/** Whether producers should wait for queue capacity instead of failing immediately. */
private boolean blockWhenFull = true;
/** Maximum time a producer waits for queue capacity before failing the import. */
private Duration queueOfferTimeout = Duration.ofMinutes(5);
public int getCompletionSize() {
return completionSize;
}
@ -52,6 +64,40 @@ public class EventHubProperties {
public void setCompletionTimeout(Duration completionTimeout) {
this.completionTimeout = completionTimeout;
}
public int getQueueSize() {
return queueSize;
}
public void setQueueSize(int queueSize) {
this.queueSize = Math.max(1, queueSize);
}
public int getConcurrentConsumers() {
return concurrentConsumers;
}
public void setConcurrentConsumers(int concurrentConsumers) {
this.concurrentConsumers = Math.max(1, concurrentConsumers);
}
public boolean isBlockWhenFull() {
return blockWhenFull;
}
public void setBlockWhenFull(boolean blockWhenFull) {
this.blockWhenFull = blockWhenFull;
}
public Duration getQueueOfferTimeout() {
return queueOfferTimeout;
}
public void setQueueOfferTimeout(Duration queueOfferTimeout) {
if (queueOfferTimeout != null && !queueOfferTimeout.isNegative()) {
this.queueOfferTimeout = queueOfferTimeout;
}
}
}
public static class Tachograph {

View File

@ -0,0 +1,17 @@
package at.procon.eventhub.config;
import com.fasterxml.jackson.databind.ObjectMapper;
import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
@Configuration
public class JacksonConfig {
@Bean
@ConditionalOnMissingBean(ObjectMapper.class)
public ObjectMapper objectMapper() {
return new ObjectMapper().findAndRegisterModules();
}
}

View File

@ -7,8 +7,7 @@ public enum EventDomain {
POSITION,
BORDER_CROSSING,
LOAD_UNLOAD,
OUT_OF_SCOPE,
FERRY_TRAIN,
SPECIFIC_CONDITION,
SPEEDING,
PLACE,
VEHICLE_DATA,

View File

@ -1,6 +1,8 @@
package at.procon.eventhub.dto;
import java.time.OffsetDateTime;
import java.util.Collections;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
@ -15,6 +17,6 @@ public record EventHubEventBatchDto(
) {
public EventHubEventBatchDto {
events = events == null ? List.of() : List.copyOf(events);
metadata = metadata == null ? Map.of() : Map.copyOf(metadata);
metadata = metadata == null ? Map.of() : Collections.unmodifiableMap(new LinkedHashMap<>(metadata));
}
}

View File

@ -20,8 +20,7 @@ public enum EventType {
OUT,
FERRY_TRAIN,
SPEEDING,
START_PLACE,
END_PLACE,
WORKING_DAY_PLACE_RECORDED,
VEHICLE_DATA,
TELEMATICS_DATA,
MANUAL_ENTRY,

View File

@ -3,36 +3,56 @@ package at.procon.eventhub.dto;
import jakarta.validation.Valid;
/**
* Source-side vehicle reference. VIN can be missing for driver-card-only data;
* VRN/registration is nation-scoped and can be resolved to VIN later.
* Source-side vehicle reference. A physical tachograph vehicle is identified by
* VehicleIdentification.ID/VIN. A registration/plate is identified separately by
* Vehicle.ID/VRN because plates can move between vehicles over time.
*
* Organisation assignment is intentionally not stored on the event. Vehicle
* organisation relation belongs to master data and can be resolved by
* sourceEntityId/VIN/VRN + occurredAt when needed.
* Organisation assignment is intentionally not stored on the event. Vehicle and
* registration relations belong to master data and can be resolved historically.
*/
public record VehicleRefDto(
String sourceEntityId,
String sourceVehicleEntityId,
String vin,
@Valid VehicleRegistrationRefDto vehicleRegistration
String sourceRegistrationEntityId,
@Valid at.procon.eventhub.dto.VehicleRegistrationRefDto vehicleRegistration
) {
public VehicleRefDto(
String sourceEntityId,
String vin,
at.procon.eventhub.dto.VehicleRegistrationRefDto vehicleRegistration
) {
this(sourceEntityId, vin, null, vehicleRegistration);
}
public VehicleRefDto {
sourceEntityId = normalizeNullable(sourceEntityId);
sourceVehicleEntityId = normalizeNullable(sourceVehicleEntityId);
vin = normalizeVin(vin);
sourceRegistrationEntityId = normalizeNullable(sourceRegistrationEntityId);
}
public boolean hasAnyReference() {
return (sourceEntityId != null && !sourceEntityId.isBlank())
return (sourceVehicleEntityId != null && !sourceVehicleEntityId.isBlank())
|| (vin != null && !vin.isBlank())
|| (sourceRegistrationEntityId != null && !sourceRegistrationEntityId.isBlank())
|| (vehicleRegistration != null && vehicleRegistration.hasValue());
}
public String stableKey() {
String registrationKey = vehicleRegistration == null ? "" : vehicleRegistration.stableKey();
return (sourceEntityId == null ? "" : sourceEntityId) + "|"
return (sourceVehicleEntityId == null ? "" : sourceVehicleEntityId) + "|"
+ (vin == null ? "" : vin) + "|"
+ (sourceRegistrationEntityId == null ? "" : sourceRegistrationEntityId) + "|"
+ registrationKey;
}
/**
* Backward-compatible accessor for older code paths that treated vehicle and
* registration source IDs as one field.
*/
public String sourceEntityId() {
return sourceVehicleEntityId != null ? sourceVehicleEntityId : sourceRegistrationEntityId;
}
private static String normalizeNullable(String value) {
return value == null || value.isBlank() ? null : value.trim();
}

View File

@ -1,12 +1,16 @@
package at.procon.eventhub.importing;
import at.procon.eventhub.config.EventHubProperties;
import at.procon.eventhub.dto.EventHubPackageRequest;
import at.procon.eventhub.dto.EventSourceDto;
import at.procon.eventhub.dto.ImportRunStatus;
import at.procon.eventhub.importing.persistence.ImportCursorRepository;
import at.procon.eventhub.importing.persistence.ImportRunRepository;
import at.procon.eventhub.persistence.DataPackageRepository.CamelBatchGroupStatus;
import at.procon.eventhub.persistence.DataPackageRepository;
import at.procon.eventhub.persistence.EventSourceRepository;
import java.time.Duration;
import java.time.Instant;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
@ -22,23 +26,32 @@ import org.slf4j.LoggerFactory;
*/
public abstract class AbstractImportExecutionService<R extends ImportRunRequest, B extends ExtractionBatchResult> {
private static final Duration ASYNC_INGEST_AWAIT_TIMEOUT = Duration.ofHours(6);
private static final Duration ASYNC_INGEST_POLL_INTERVAL = Duration.ofSeconds(2);
private static final Duration ASYNC_INGEST_FAILURE_GRACE_PERIOD = Duration.ofSeconds(90);
private static final Duration ASYNC_INGEST_STALL_GRACE_PERIOD = Duration.ofSeconds(90);
private static final Duration ASYNC_INGEST_PROGRESS_LOG_INTERVAL = Duration.ofSeconds(30);
private final Logger log = LoggerFactory.getLogger(getClass());
private final EventSourceRepository eventSourceRepository;
private final ImportRunRepository importRunRepository;
private final DataPackageRepository dataPackageRepository;
private final ImportCursorRepository importCursorRepository;
private final EventHubProperties eventHubProperties;
protected AbstractImportExecutionService(
EventSourceRepository eventSourceRepository,
ImportRunRepository importRunRepository,
DataPackageRepository dataPackageRepository,
ImportCursorRepository importCursorRepository
ImportCursorRepository importCursorRepository,
EventHubProperties eventHubProperties
) {
this.eventSourceRepository = eventSourceRepository;
this.importRunRepository = importRunRepository;
this.dataPackageRepository = dataPackageRepository;
this.importCursorRepository = importCursorRepository;
this.eventHubProperties = eventHubProperties;
}
protected ImportRunResultDto createImportRun(R request, boolean executeImmediately) {
@ -165,26 +178,38 @@ public abstract class AbstractImportExecutionService<R extends ImportRunRequest,
List<B> results = new ArrayList<>();
for (PlannedPackage plannedPackage : plannedPackages) {
dataPackageRepository.markImporting(plannedPackage.packageId());
B result = executeBatch(
importRunId,
plannedPackage.packageId(),
plannedPackage.eventSourceId(),
request,
plannedPackage.planItem(),
plannedPackage.chunk()
);
results.add(result);
dataPackageRepository.markImported(plannedPackage.packageId(), result.eventsInserted());
if (result.executed()) {
importCursorRepository.advanceCursor(
request.tenantKey(),
try {
B result = executeBatch(
importRunId,
plannedPackage.packageId(),
plannedPackage.eventSourceId(),
request.importScope() == null ? "NO_SCOPE" : request.importScope().stableKey(),
plannedPackage.planItem().eventFamily(),
plannedPackage.planItem().sourceKind(),
request.acquisitionStrategy(),
result
request,
plannedPackage.planItem(),
plannedPackage.chunk()
);
dataPackageRepository.updateEventCount(plannedPackage.packageId(), result.eventsInserted());
if (!result.eventTypeCounts().isEmpty()) {
dataPackageRepository.mergeMetadata(plannedPackage.packageId(), Map.of(
"extractedEventTypeCounts", result.eventTypeCounts()
));
}
awaitAsyncIngestCompletion(importRunId, request, plannedPackage, result);
results.add(result);
dataPackageRepository.markImported(plannedPackage.packageId(), result.eventsInserted());
if (result.executed()) {
importCursorRepository.advanceCursor(
request.tenantKey(),
plannedPackage.eventSourceId(),
request.importScope() == null ? "NO_SCOPE" : request.importScope().stableKey(),
plannedPackage.planItem().eventFamily(),
plannedPackage.planItem().sourceKind(),
request.acquisitionStrategy(),
result
);
}
} catch (RuntimeException ex) {
dataPackageRepository.markFailed(plannedPackage.packageId(), ex.getMessage());
throw ex;
}
}
importRunRepository.markCompleted(importRunId);
@ -196,6 +221,107 @@ public abstract class AbstractImportExecutionService<R extends ImportRunRequest,
results.stream().filter(ExtractionBatchResult::executed).count());
}
private void awaitAsyncIngestCompletion(UUID importRunId, R request, PlannedPackage plannedPackage, B result) {
int expectedCamelBatches = expectedCamelBatchCount(result.eventsInserted());
if (expectedCamelBatches <= 0) {
return;
}
EventHubPackageRequest packageInfo = extractionAggregatePackageInfo(
importRunId,
request,
eventSourceForItem(request.eventSource(), plannedPackage.planItem()),
plannedPackage.planItem(),
plannedPackage.chunk()
);
String aggregatePackageKey = aggregatePackageKey(packageInfo);
Instant deadline = Instant.now().plus(ASYNC_INGEST_AWAIT_TIMEOUT);
Instant nextProgressLogAt = Instant.now().plus(ASYNC_INGEST_PROGRESS_LOG_INTERVAL);
Instant failedStateObservedAt = null;
Instant lastStateChangeAt = Instant.now();
CamelBatchGroupStatus previousState = null;
while (Instant.now().isBefore(deadline)) {
CamelBatchGroupStatus state = dataPackageRepository.findCamelBatchGroupStatus(
plannedPackage.eventSourceId(),
request.tenantKey(),
aggregatePackageKey
);
if (state.totalCount() >= expectedCamelBatches && state.successCount() >= expectedCamelBatches) {
return;
}
boolean stateChanged = previousState == null || !previousState.equals(state);
if (stateChanged) {
lastStateChangeAt = Instant.now();
}
if (state.failedCount() > 0 && stateChanged) {
failedStateObservedAt = Instant.now();
} else if (state.failedCount() == 0) {
failedStateObservedAt = null;
}
if (state.failedCount() > 0
&& failedStateObservedAt != null
&& Instant.now().isAfter(failedStateObservedAt.plus(ASYNC_INGEST_FAILURE_GRACE_PERIOD))) {
throw new IllegalStateException(
"Async EventHub ingest failed for importRunId=" + importRunId
+ " aggregatePackageKey=" + aggregatePackageKey
+ " expectedCamelBatches=" + expectedCamelBatches
+ " observedCamelBatches=" + state.totalCount()
+ " failedCamelBatches=" + state.failedCount()
+ " failedMessage=" + state.failedMessage()
);
}
if (state.totalCount() < expectedCamelBatches
&& state.importingCount() == 0
&& state.failedCount() == 0
&& Instant.now().isAfter(lastStateChangeAt.plus(ASYNC_INGEST_STALL_GRACE_PERIOD))) {
throw new IllegalStateException(
"Async EventHub ingest stalled for importRunId=" + importRunId
+ " aggregatePackageKey=" + aggregatePackageKey
+ " expectedCamelBatches=" + expectedCamelBatches
+ " observedCamelBatches=" + state.totalCount()
+ " successfulCamelBatches=" + state.successCount()
+ " importingCamelBatches=" + state.importingCount()
+ " failedCamelBatches=" + state.failedCount()
);
}
if (Instant.now().isAfter(nextProgressLogAt)) {
log.info("Waiting for async EventHub ingest provider={} importRunId={} extractionPackageId={} aggregatePackageKey={} expectedCamelBatches={} observedCamelBatches={} successfulCamelBatches={} failedCamelBatches={} importingCamelBatches={}",
providerPackagePrefix(),
importRunId,
plannedPackage.packageId(),
aggregatePackageKey,
expectedCamelBatches,
state.totalCount(),
state.successCount(),
state.failedCount(),
state.importingCount());
nextProgressLogAt = Instant.now().plus(ASYNC_INGEST_PROGRESS_LOG_INTERVAL);
}
previousState = state;
sleepQuietly(ASYNC_INGEST_POLL_INTERVAL);
}
CamelBatchGroupStatus finalState = dataPackageRepository.findCamelBatchGroupStatus(
plannedPackage.eventSourceId(),
request.tenantKey(),
aggregatePackageKey
);
throw new IllegalStateException(
"Timed out waiting for async EventHub ingest for importRunId=" + importRunId
+ " aggregatePackageKey=" + aggregatePackageKey
+ " expectedCamelBatches=" + expectedCamelBatches
+ " observedCamelBatches=" + finalState.totalCount()
+ " successfulCamelBatches=" + finalState.successCount()
+ " failedCamelBatches=" + finalState.failedCount()
);
}
private EventHubPackageRequest packageRequestFor(
R request,
EventSourceDto itemEventSource,
@ -206,13 +332,72 @@ public abstract class AbstractImportExecutionService<R extends ImportRunRequest,
request.tenantKey(),
itemEventSource,
request.sourceGroup(),
request.importScope(),
chunkScope(request.importScope(), chunk),
item.eventFamily().name(),
null,
chunk.occurredFrom() == null ? null : chunk.occurredFrom().toLocalDate(),
externalPackageId(request, item, chunk)
);
}
private EventHubPackageRequest extractionAggregatePackageInfo(
UUID importRunId,
R request,
EventSourceDto itemEventSource,
ImportPlanItemDto item,
ImportTimeChunkDto chunk
) {
return new EventHubPackageRequest(
request.tenantKey(),
itemEventSource,
request.sourceGroup(),
chunkScope(request.importScope(), chunk),
item.eventFamily().name(),
chunk.occurredFrom() == null ? null : chunk.occurredFrom().toLocalDate(),
providerPackagePrefix() + ":" + item.sourceKind() + ":" + item.extractionCode()
+ ":RUN-" + importRunId + ":CHUNK-" + chunk.sequence()
);
}
private int expectedCamelBatchCount(int extractedEventCount) {
if (extractedEventCount <= 0) {
return 0;
}
int completionSize = Math.max(1, eventHubProperties.getBatch().getCompletionSize());
return ((extractedEventCount - 1) / completionSize) + 1;
}
private String aggregatePackageKey(EventHubPackageRequest packageInfo) {
return packageInfo.tenantKey()
+ ":" + packageInfo.eventSource().stableKey()
+ ":" + (packageInfo.sourceGroup() == null ? "NO_GROUP" : packageInfo.sourceGroup().stableKey())
+ ":" + (packageInfo.importScope() == null ? "NO_SCOPE" : packageInfo.importScope().stableKey())
+ ":" + packageInfo.eventFamily()
+ ":" + (packageInfo.businessDate() == null ? "NO_DATE" : packageInfo.businessDate())
+ ":" + packageInfo.externalPackageId();
}
private at.procon.eventhub.dto.ImportScopeDto chunkScope(at.procon.eventhub.dto.ImportScopeDto scope, ImportTimeChunkDto chunk) {
if (scope == null) {
return at.procon.eventhub.dto.ImportScopeDto.tenantAll(chunk.occurredFrom(), chunk.occurredTo());
}
return new at.procon.eventhub.dto.ImportScopeDto(
scope.type(),
scope.rootSourceOrganisation(),
scope.includeChildren(),
chunk.occurredFrom(),
chunk.occurredTo()
);
}
private void sleepQuietly(Duration duration) {
try {
Thread.sleep(duration.toMillis());
} catch (InterruptedException ex) {
Thread.currentThread().interrupt();
throw new IllegalStateException("Interrupted while waiting for async EventHub ingest", ex);
}
}
private record PlannedPackage(UUID packageId, int eventSourceId, ImportPlanItemDto planItem, ImportTimeChunkDto chunk) {
}
}

View File

@ -1,6 +1,7 @@
package at.procon.eventhub.importing;
import java.time.OffsetDateTime;
import java.util.Map;
public interface ExtractionBatchResult {
@ -15,4 +16,8 @@ public interface ExtractionBatchResult {
OffsetDateTime lastSourceRowUpdatedAt();
OffsetDateTime lastOccurredTo();
default Map<String, Integer> eventTypeCounts() {
return Map.of();
}
}

View File

@ -14,11 +14,13 @@ import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.time.OffsetDateTime;
import java.util.HashMap;
import java.util.List;
import java.util.LinkedHashMap;
import java.util.Map;
import java.util.Optional;
import java.util.UUID;
import org.apache.camel.ProducerTemplate;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.core.io.Resource;
import org.springframework.core.io.ResourceLoader;
import org.springframework.jdbc.core.namedparam.NamedParameterJdbcTemplate;
@ -27,6 +29,9 @@ import org.springframework.util.StreamUtils;
public abstract class AbstractJdbcExtractionBatchExecutor<R extends ImportRunRequest, B extends ExtractionBatchResult>
implements ExtractionBatchExecutor<R, B> {
private static final Logger log = LoggerFactory.getLogger(AbstractJdbcExtractionBatchExecutor.class);
private static final int EVENT_EXTRACTION_PROGRESS_LOG_INTERVAL = 5000;
private final NamedParameterJdbcTemplate jdbcTemplate;
private final ProducerTemplate producerTemplate;
private final ResourceLoader resourceLoader;
@ -81,10 +86,21 @@ public abstract class AbstractJdbcExtractionBatchExecutor<R extends ImportRunReq
Map<String, Object> params = parameters(request, chunkScope, cursor);
String sql = loadSql(definition.sqlResource());
List<EventHubEventDto> events = jdbcTemplate.query(sql, params, (rs, rowNum) -> definition.rowMapper().map(rs, rowNum, context));
events.forEach(event -> producerTemplate.sendBody(normalizedInputUri(), event));
ExtractedEventStats stats = new ExtractedEventStats();
log.info("Reading EventHub events provider={} tenant={} importRunId={} packageId={} extractionCode={} sourceKind={} chunk={} occurredFrom={} occurredTo={}",
providerPackagePrefix(), request.tenantKey(), importRunId, packageId, planItem.extractionCode(), planItem.sourceKind(),
chunk.sequence(), chunk.occurredFrom(), chunk.occurredTo());
jdbcTemplate.query(sql, params, rs -> {
EventHubEventDto event = definition.rowMapper().map(rs, stats.eventsMapped(), context);
producerTemplate.sendBody(normalizedInputUri(), event);
stats.accept(event);
if (stats.eventsMapped() % EVENT_EXTRACTION_PROGRESS_LOG_INTERVAL == 0) {
logEventExtractionProgress(request, importRunId, packageId, planItem, chunk, stats);
}
});
logEventExtractionFinished(request, importRunId, packageId, planItem, chunk, stats);
return resultFor(packageId, planItem, chunk, cursor, events);
return resultFor(packageId, planItem, chunk, cursor, stats);
}
protected abstract Optional<ExtractionDefinition<R>> findDefinition(String code);
@ -96,7 +112,7 @@ public abstract class AbstractJdbcExtractionBatchExecutor<R extends ImportRunReq
ImportPlanItemDto planItem,
ImportTimeChunkDto chunk,
ImportCursorStateDto cursor,
List<EventHubEventDto> events
ExtractedEventStats stats
);
protected String providerPackagePrefix() {
@ -107,38 +123,64 @@ public abstract class AbstractJdbcExtractionBatchExecutor<R extends ImportRunReq
return "direct:eventhub-normalized-input";
}
private void logEventExtractionProgress(
R request,
UUID importRunId,
UUID packageId,
ImportPlanItemDto planItem,
ImportTimeChunkDto chunk,
ExtractedEventStats stats
) {
log.info("EventHub event extraction progress provider={} tenant={} importRunId={} packageId={} extractionCode={} sourceKind={} chunk={} mapped={} byType={}",
providerPackagePrefix(), request.tenantKey(), importRunId, packageId, planItem.extractionCode(), planItem.sourceKind(),
chunk.sequence(), stats.eventsMapped(), stats.eventTypeCounts());
}
private void logEventExtractionFinished(
R request,
UUID importRunId,
UUID packageId,
ImportPlanItemDto planItem,
ImportTimeChunkDto chunk,
ExtractedEventStats stats
) {
log.info("Finished EventHub event extraction provider={} tenant={} importRunId={} packageId={} extractionCode={} sourceKind={} chunk={} mapped={} byType={}",
providerPackagePrefix(), request.tenantKey(), importRunId, packageId, planItem.extractionCode(), planItem.sourceKind(),
chunk.sequence(), stats.eventsMapped(), stats.eventTypeCounts());
}
protected Map<String, Object> parameters(R request, ImportScopeDto scope, ImportCursorStateDto cursor) {
Map<String, Object> params = new HashMap<>();
String organisationId = scope == null || scope.rootSourceOrganisation() == null
? null
: scope.rootSourceOrganisation().sourceEntityId();
params.put("tenantKey", request.tenantKey());
params.put("occurredFrom", scope == null ? null : scope.occurredFrom());
params.put("occurredTo", scope == null ? null : scope.occurredTo());
params.put("rootOrganisationId", scope == null || scope.rootSourceOrganisation() == null ? null : scope.rootSourceOrganisation().sourceEntityId());
params.put("organisationId", organisationId);
params.put("includeChildren", scope != null && scope.includeChildren());
params.put("lastSourcePackageImportedAt", cursor == null ? null : cursor.lastSourcePackageImportedAt());
params.put("lastSourcePackageId", cursor == null ? null : cursor.lastSourcePackageId());
params.put("lastSourcePackageIdNumeric", parseLong(cursor == null ? null : cursor.lastSourcePackageId()));
params.put("lastSourceRowUpdatedAt", cursor == null ? null : cursor.lastSourceRowUpdatedAt());
params.put("lastOccurredTo", cursor == null ? null : cursor.lastOccurredTo());
return params;
}
protected OffsetDateTime lastSourcePackageImportedAt(List<EventHubEventDto> events, ImportCursorStateDto cursor) {
return events.stream()
.map(event -> event.sourcePackageRef() == null ? null : event.sourcePackageRef().importedIntoSourceAt())
.filter(value -> value != null)
.max(OffsetDateTime::compareTo)
.orElse(cursor == null ? null : cursor.lastSourcePackageImportedAt());
protected OffsetDateTime lastSourcePackageImportedAt(ExtractedEventStats stats, ImportCursorStateDto cursor) {
return stats.lastSourcePackageImportedAt() == null
? cursor == null ? null : cursor.lastSourcePackageImportedAt()
: stats.lastSourcePackageImportedAt();
}
protected String lastSourcePackageId(List<EventHubEventDto> events, ImportCursorStateDto cursor) {
return events.stream()
.filter(event -> event.sourcePackageRef() != null && event.sourcePackageRef().importedIntoSourceAt() != null)
.max((left, right) -> left.sourcePackageRef().importedIntoSourceAt().compareTo(right.sourcePackageRef().importedIntoSourceAt()))
.map(event -> event.sourcePackageRef().sourcePackageId())
.orElseGet(() -> events.stream()
.map(event -> event.sourcePackageRef() == null ? null : event.sourcePackageRef().sourcePackageId())
.filter(value -> value != null && !value.isBlank())
.max(this::compareSourcePackageId)
.orElse(cursor == null ? null : cursor.lastSourcePackageId()));
protected String lastSourcePackageId(ExtractedEventStats stats, ImportCursorStateDto cursor) {
if (stats.lastSourcePackageIdByImportedAt() != null) {
return stats.lastSourcePackageIdByImportedAt();
}
if (stats.maxSourcePackageId() != null) {
return stats.maxSourcePackageId();
}
return cursor == null ? null : cursor.lastSourcePackageId();
}
private EventHubPackageRequest packageInfo(
@ -200,4 +242,70 @@ public abstract class AbstractJdbcExtractionBatchExecutor<R extends ImportRunReq
return null;
}
}
private Long parseLong(String value) {
if (value == null || value.isBlank()) {
return null;
}
try {
return Long.parseLong(value.trim());
} catch (NumberFormatException ignored) {
return null;
}
}
protected final class ExtractedEventStats {
private int eventsMapped;
private OffsetDateTime lastSourcePackageImportedAt;
private String lastSourcePackageIdByImportedAt;
private String maxSourcePackageId;
private final Map<String, Integer> eventTypeCounts = new LinkedHashMap<>();
public int eventsMapped() {
return eventsMapped;
}
public OffsetDateTime lastSourcePackageImportedAt() {
return lastSourcePackageImportedAt;
}
public String lastSourcePackageIdByImportedAt() {
return lastSourcePackageIdByImportedAt;
}
public String maxSourcePackageId() {
return maxSourcePackageId;
}
public Map<String, Integer> eventTypeCounts() {
return eventTypeCounts;
}
private void accept(EventHubEventDto event) {
eventsMapped++;
eventTypeCounts.merge(eventTypeKey(event), 1, Integer::sum);
if (event.sourcePackageRef() == null) {
return;
}
OffsetDateTime importedAt = event.sourcePackageRef().importedIntoSourceAt();
String sourcePackageId = event.sourcePackageRef().sourcePackageId();
if (importedAt != null
&& (lastSourcePackageImportedAt == null || importedAt.compareTo(lastSourcePackageImportedAt) > 0)) {
lastSourcePackageImportedAt = importedAt;
lastSourcePackageIdByImportedAt = sourcePackageId;
}
if (sourcePackageId != null
&& !sourcePackageId.isBlank()
&& (maxSourcePackageId == null || compareSourcePackageId(sourcePackageId, maxSourcePackageId) > 0)) {
maxSourcePackageId = sourcePackageId;
}
}
private String eventTypeKey(EventHubEventDto event) {
String domain = event.eventDomain() == null ? "UNKNOWN_DOMAIN" : event.eventDomain().name();
String type = event.eventType() == null ? "UNKNOWN_EVENT" : event.eventType().name();
return domain + "/" + type;
}
}
}

View File

@ -1,10 +1,12 @@
package at.procon.eventhub.importing.extraction;
import at.procon.eventhub.importing.ImportRunRequest;
import java.util.Comparator;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.function.Function;
import java.util.stream.Collectors;
@ -21,6 +23,16 @@ public class ExtractionDefinitionRegistry<R extends ImportRunRequest> {
return Optional.ofNullable(definitionsByCode.get(normalize(code)));
}
public Set<String> supportedCodes() {
return definitionsByCode.keySet();
}
public List<ExtractionDefinition<R>> definitions() {
return definitionsByCode.values().stream()
.sorted(Comparator.comparing(ExtractionDefinition::code))
.toList();
}
private String normalize(String value) {
return value == null ? "" : value.trim().toUpperCase(Locale.ROOT);
}

View File

@ -0,0 +1,10 @@
package at.procon.eventhub.importing.masterdata;
public record MasterDataRefreshResult(
int entitiesUpserted,
int relationsUpserted
) {
public static MasterDataRefreshResult empty() {
return new MasterDataRefreshResult(0, 0);
}
}

View File

@ -0,0 +1,17 @@
package at.procon.eventhub.importing.masterdata;
import java.time.OffsetDateTime;
import java.util.Map;
public record SourceMasterEntityUpsert(
String entityType,
String sourceEntityId,
String sourceExternalKey,
String displayName,
Boolean active,
OffsetDateTime validFrom,
OffsetDateTime validTo,
OffsetDateTime sourceUpdatedAt,
Map<String, Object> payload
) {
}

View File

@ -0,0 +1,17 @@
package at.procon.eventhub.importing.masterdata;
import java.time.OffsetDateTime;
import java.util.Map;
public record SourceMasterRelationUpsert(
String relationType,
String fromEntityType,
String fromSourceEntityId,
String toEntityType,
String toSourceEntityId,
OffsetDateTime validFrom,
OffsetDateTime validTo,
OffsetDateTime sourceUpdatedAt,
Map<String, Object> payload
) {
}

View File

@ -7,12 +7,21 @@ import at.procon.eventhub.dto.ImportScopeDto;
import at.procon.eventhub.dto.SourceGroupRefDto;
import at.procon.eventhub.dto.SourcePackageRefDto;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import java.lang.reflect.Array;
import java.time.OffsetDateTime;
import java.time.temporal.TemporalAccessor;
import java.util.ArrayList;
import java.util.Date;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.UUID;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.stereotype.Repository;
import org.springframework.transaction.annotation.Propagation;
import org.springframework.transaction.annotation.Transactional;
@Repository
public class DataPackageRepository {
@ -25,6 +34,7 @@ public class DataPackageRepository {
this.objectMapper = objectMapper;
}
@Transactional(propagation = Propagation.REQUIRES_NEW)
public UUID createPackage(
int eventSourceId,
String packageKey,
@ -34,24 +44,78 @@ public class DataPackageRepository {
OffsetDateTime occurredTo,
Map<String, Object> metadata
) {
return insertPackage(
eventSourceId,
null,
packageKey,
packageInfo,
packageType,
DataPackageStatus.IMPORTING,
occurredFrom,
occurredTo,
null,
null,
null,
null,
null,
null,
null,
null,
metadata
UUID id = UUID.randomUUID();
SourceGroupRefDto sourceGroup = packageInfo == null ? null : packageInfo.sourceGroup();
ImportScopeDto importScope = packageInfo == null ? null : packageInfo.importScope();
SourceGroupRefDto rootOrg = importScope == null ? null : importScope.rootSourceOrganisation();
return jdbcTemplate.query(
con -> {
var ps = con.prepareStatement("""
insert into eventhub.data_package(
id, event_source_id, import_run_id, tenant_key, package_key, package_type, status,
source_group_type, source_group_entity_id, source_group_code, source_group_name,
import_scope_type, root_source_org_entity_id, root_source_org_code, root_source_org_name,
include_children, occurred_from, occurred_to,
event_family, business_date, external_package_id,
extraction_code, extraction_source_kind, entity_axis, batch_no, chunk_from, chunk_to,
source_package_kind, source_package_id, source_package_entity_id,
source_package_period_from, source_package_period_to, source_package_imported_at,
received_at, event_count, metadata
) values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, now(), ?, ?::jsonb)
on conflict (tenant_key, event_source_id, package_key) do update
set status = excluded.status,
occurred_from = excluded.occurred_from,
occurred_to = excluded.occurred_to,
event_count = excluded.event_count,
metadata = excluded.metadata,
error_message = null,
completed_at = null
returning id
""");
ps.setObject(1, id);
ps.setInt(2, eventSourceId);
ps.setObject(3, null);
ps.setString(4, packageInfo == null ? "default" : packageInfo.tenantKey());
ps.setString(5, packageKey);
ps.setString(6, packageType.name());
ps.setString(7, DataPackageStatus.IMPORTING.name());
ps.setString(8, sourceGroup == null || sourceGroup.type() == null ? null : sourceGroup.type().name());
ps.setString(9, sourceGroup == null ? null : sourceGroup.sourceEntityId());
ps.setString(10, sourceGroup == null ? null : sourceGroup.code());
ps.setString(11, sourceGroup == null ? null : sourceGroup.name());
ps.setString(12, importScope == null || importScope.type() == null ? null : importScope.type().name());
ps.setString(13, rootOrg == null ? null : rootOrg.sourceEntityId());
ps.setString(14, rootOrg == null ? null : rootOrg.code());
ps.setString(15, rootOrg == null ? null : rootOrg.name());
ps.setBoolean(16, importScope != null && importScope.includeChildren());
ps.setObject(17, occurredFrom);
ps.setObject(18, occurredTo);
ps.setString(19, packageInfo == null ? null : packageInfo.eventFamily());
ps.setObject(20, packageInfo == null ? null : packageInfo.businessDate());
ps.setString(21, packageInfo == null ? packageKey : packageInfo.externalPackageId());
ps.setString(22, null);
ps.setString(23, null);
ps.setString(24, null);
ps.setObject(25, null);
ps.setObject(26, null);
ps.setObject(27, null);
ps.setString(28, null);
ps.setString(29, null);
ps.setString(30, null);
ps.setObject(31, null);
ps.setObject(32, null);
ps.setObject(33, null);
ps.setInt(34, 0);
ps.setString(35, toJson(metadata));
return ps;
},
rs -> {
if (!rs.next()) {
throw new IllegalStateException("Could not create or resolve data package " + packageKey);
}
return (UUID) rs.getObject(1);
}
);
}
@ -189,6 +253,31 @@ public class DataPackageRepository {
);
}
public void updateEventCount(UUID packageId, int eventCount) {
jdbcTemplate.update(
"""
update eventhub.data_package
set event_count = ?
where id = ?
""",
eventCount,
packageId
);
}
public void mergeMetadata(UUID packageId, Map<String, Object> metadata) {
jdbcTemplate.update(
"""
update eventhub.data_package
set metadata = coalesce(metadata, '{}'::jsonb) || ?::jsonb
where id = ?
""",
toJson(metadata),
packageId
);
}
@Transactional(propagation = Propagation.REQUIRES_NEW)
public void markImported(UUID packageId, int insertedCount) {
jdbcTemplate.update(
"""
@ -202,6 +291,7 @@ public class DataPackageRepository {
);
}
@Transactional(propagation = Propagation.REQUIRES_NEW)
public void markFailed(UUID packageId, String errorMessage) {
jdbcTemplate.update(
"""
@ -215,11 +305,120 @@ public class DataPackageRepository {
);
}
public CamelBatchGroupStatus findCamelBatchGroupStatus(int eventSourceId, String tenantKey, String aggregatePackageKey) {
return jdbcTemplate.query(
"""
select count(*) as total_count,
count(*) filter (where status in (?, ?)) as success_count,
count(*) filter (where status = ?) as failed_count,
count(*) filter (where status = ?) as importing_count,
coalesce(max(error_message) filter (where status = ?), '') as failed_message
from eventhub.data_package
where event_source_id = ?
and tenant_key = ?
and package_type = ?
and package_key like ?
""",
rs -> {
if (!rs.next()) {
return CamelBatchGroupStatus.empty();
}
return new CamelBatchGroupStatus(
rs.getInt("total_count"),
rs.getInt("success_count"),
rs.getInt("failed_count"),
rs.getInt("importing_count"),
rs.getString("failed_message")
);
},
DataPackageStatus.IMPORTED.name(),
DataPackageStatus.EMPTY.name(),
DataPackageStatus.FAILED.name(),
DataPackageStatus.IMPORTING.name(),
DataPackageStatus.FAILED.name(),
eventSourceId,
tenantKey,
DataPackageType.CAMEL_BATCH.name(),
aggregatePackageKey + ":CAMEL-%"
);
}
private String toJson(Map<String, Object> value) {
try {
return objectMapper.writeValueAsString(value == null ? Map.of() : value);
return objectMapper.writeValueAsString(normalizeMetadataMap(value));
} catch (JsonProcessingException e) {
throw new IllegalArgumentException("Cannot serialize package metadata", e);
}
}
private Map<String, Object> normalizeMetadataMap(Map<String, Object> value) {
if (value == null || value.isEmpty()) {
return Map.of();
}
Map<String, Object> normalized = new LinkedHashMap<>();
for (Map.Entry<String, Object> entry : value.entrySet()) {
normalized.put(entry.getKey(), normalizeMetadataValue(entry.getValue()));
}
return normalized;
}
private Object normalizeMetadataValue(Object value) {
if (value == null
|| value instanceof String
|| value instanceof Number
|| value instanceof Boolean
|| value instanceof JsonNode) {
return value;
}
if (value instanceof CharSequence charSequence) {
return charSequence.toString();
}
if (value instanceof Enum<?> enumValue) {
return enumValue.name();
}
if (value instanceof UUID uuid) {
return uuid.toString();
}
if (value instanceof TemporalAccessor temporalValue) {
return temporalValue.toString();
}
if (value instanceof Date dateValue) {
return dateValue.toInstant().toString();
}
if (value instanceof Map<?, ?> mapValue) {
Map<String, Object> nested = new LinkedHashMap<>();
for (Map.Entry<?, ?> entry : mapValue.entrySet()) {
nested.put(String.valueOf(entry.getKey()), normalizeMetadataValue(entry.getValue()));
}
return nested;
}
if (value instanceof Iterable<?> iterable) {
List<Object> nested = new ArrayList<>();
for (Object item : iterable) {
nested.add(normalizeMetadataValue(item));
}
return nested;
}
if (value.getClass().isArray()) {
int length = Array.getLength(value);
List<Object> nested = new ArrayList<>(length);
for (int i = 0; i < length; i++) {
nested.add(normalizeMetadataValue(Array.get(value, i)));
}
return nested;
}
return value.toString();
}
public record CamelBatchGroupStatus(
int totalCount,
int successCount,
int failedCount,
int importingCount,
String failedMessage
) {
public static CamelBatchGroupStatus empty() {
return new CamelBatchGroupStatus(0, 0, 0, 0, "");
}
}
}

View File

@ -6,14 +6,20 @@ import at.procon.eventhub.dto.EventHubEventDto;
import at.procon.eventhub.dto.SourcePackageRefDto;
import at.procon.eventhub.dto.VehicleRefDto;
import at.procon.eventhub.dto.VehicleRegistrationRefDto;
import at.procon.eventhub.persistence.VehicleIdentityRepository.ResolvedVehicleReference;
import at.procon.eventhub.persistence.VehicleIdentityRepository.ResolvedVehicleReferenceResolution;
import at.procon.eventhub.service.EventAcquisitionRecordKeyService;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import java.sql.PreparedStatement;
import java.sql.Types;
import java.sql.SQLException;
import java.time.OffsetDateTime;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.UUID;
import org.springframework.jdbc.core.BatchPreparedStatementSetter;
import org.springframework.jdbc.core.JdbcTemplate;
@ -25,126 +31,563 @@ public class EventRepository {
private final JdbcTemplate jdbcTemplate;
private final ObjectMapper objectMapper;
private final EventAcquisitionRecordKeyService recordKeyService;
private final SourceMasterDataRepository sourceMasterDataRepository;
private final VehicleIdentityRepository vehicleIdentityRepository;
public EventRepository(JdbcTemplate jdbcTemplate, ObjectMapper objectMapper, EventAcquisitionRecordKeyService recordKeyService) {
public EventRepository(
JdbcTemplate jdbcTemplate,
ObjectMapper objectMapper,
EventAcquisitionRecordKeyService recordKeyService,
SourceMasterDataRepository sourceMasterDataRepository,
VehicleIdentityRepository vehicleIdentityRepository
) {
this.jdbcTemplate = jdbcTemplate;
this.objectMapper = objectMapper;
this.recordKeyService = recordKeyService;
this.sourceMasterDataRepository = sourceMasterDataRepository;
this.vehicleIdentityRepository = vehicleIdentityRepository;
}
/**
* Acquisition-stage persistence. This table stores source records as imported.
* It does not merge or deduplicate equivalent events from different sources;
* later query/read models can combine sources when a preferred source has gaps.
* Organisation assignment is not stored per event; it belongs to master-data
* relations for driver/vehicle and can be resolved by occurredAt later.
* Persists normalized events and resolves master-data references on the fly.
* The source-record hash is unique and provides source-level import idempotency.
*/
public int batchInsert(UUID packageId, int eventSourceId, List<EventHubEventDto> events) {
int[] counts = jdbcTemplate.batchUpdate(
public int batchInsert(UUID packageId, String tenantKey, int eventSourceId, List<EventHubEventDto> events) {
Map<String, UUID> entityIdCache = new HashMap<>();
Map<String, List<VehicleRefCacheEntry>> vehicleRefCache = new HashMap<>();
List<ResolvedEventImportRow> rows = new ArrayList<>(events.size());
for (EventHubEventDto event : events) {
ResolvedEntityRefs refs = resolveEntityRefs(tenantKey, eventSourceId, event, entityIdCache, vehicleRefCache);
rows.add(resolveEventImportRow(packageId, eventSourceId, event, refs));
}
createEventImportStage();
stageEvents(rows);
int insertedCount = insertStagedEvents();
Map<String, InsertedEventRow> eventRowsBySourceRecord = resolveStagedEventRows();
batchUpsertEventDetails(rows, eventRowsBySourceRecord);
return insertedCount;
}
private ResolvedEventImportRow resolveEventImportRow(
UUID packageId,
int eventSourceId,
EventHubEventDto event,
ResolvedEntityRefs refs
) {
UUID requestedEventId = event.eventId() == null ? UUID.randomUUID() : event.eventId();
OffsetDateTime receivedHubAt = event.receivedHubAt() == null ? OffsetDateTime.now() : event.receivedHubAt();
String sourceRecordKeyHash = recordKeyService.buildSourceRecordKeyHash(event, eventSourceId);
Double longitude = event.position() == null ? null : event.position().longitude().doubleValue();
Double latitude = event.position() == null ? null : event.position().latitude().doubleValue();
String sourcePackageId = event.sourcePackageRef() == null ? null : event.sourcePackageRef().sourcePackageId();
return new ResolvedEventImportRow(
sourceRecordKeyHash,
requestedEventId,
packageId,
eventSourceId,
event.externalSourceEventId(),
refs.driverEntityId(),
refs.vehicleId(),
refs.vehicleRegistrationId(),
sourcePackageId,
refs.sourcePackageEntityId(),
event.occurredAt(),
event.receivedPartnerAt(),
receivedHubAt,
event.eventDomain().name(),
event.eventType().name(),
event.lifecycle().name(),
event.odometerM(),
longitude,
latitude,
toJson(event.payload()),
event.manualEntry(),
recordKeyService.buildEventSignatureHash(event),
event.eventDetails() == null ? null : event.eventDetails().type(),
event.eventDetails() == null ? null : toJson(event.eventDetails().attributes())
);
}
private void createEventImportStage() {
jdbcTemplate.execute(
"""
insert into eventhub.acquired_event(
id, event_source_id, data_package_id,
external_source_event_id,
driver_source_entity_id, driver_card_nation, driver_card_number,
vehicle_source_entity_id, vehicle_vin, vehicle_registration_nation, vehicle_registration_number,
source_package_kind, source_package_id, source_package_entity_id,
source_package_period_from, source_package_period_to, source_package_imported_at,
occurred_at, received_partner_at, received_hub_at,
event_domain, event_type, lifecycle,
odometer_m, latitude, longitude,
event_details, payload, manual_entry,
source_record_key_hash, event_signature_hash
) values (
?, ?, ?,
?,
?, ?, ?,
?, ?, ?, ?,
?, ?, ?,
?, ?, ?,
?, ?, ?,
?, ?, ?,
?, ?, ?,
?::jsonb, ?::jsonb, ?,
?, ?
)
on conflict do nothing
create temporary table if not exists eventhub_event_import_stage (
row_no integer not null,
source_record_key_hash text not null,
requested_event_id uuid not null,
data_package_id uuid not null,
event_source_id integer not null,
external_source_event_id text not null,
driver_entity_id uuid,
vehicle_id uuid,
vehicle_registration_id uuid,
source_package_id text,
source_package_entity_id uuid,
occurred_at timestamptz not null,
received_partner_at timestamptz,
received_hub_at timestamptz not null,
event_domain text not null,
event_type text not null,
lifecycle text not null,
odometer_m bigint,
longitude double precision,
latitude double precision,
payload jsonb not null,
manual_entry boolean not null,
event_signature_hash text
) on commit drop
"""
);
jdbcTemplate.execute("truncate table eventhub_event_import_stage");
}
private void stageEvents(List<ResolvedEventImportRow> rows) {
jdbcTemplate.batchUpdate(
"""
insert into eventhub_event_import_stage(
row_no, source_record_key_hash, requested_event_id, data_package_id, event_source_id,
external_source_event_id, driver_entity_id, vehicle_id, vehicle_registration_id,
source_package_id, source_package_entity_id, occurred_at, received_partner_at, received_hub_at,
event_domain, event_type, lifecycle, odometer_m, longitude, latitude,
payload, manual_entry, event_signature_hash
) values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?::jsonb, ?, ?)
""",
new BatchPreparedStatementSetter() {
@Override
public void setValues(PreparedStatement ps, int i) throws java.sql.SQLException {
EventHubEventDto event = events.get(i);
UUID eventId = event.eventId() == null ? UUID.randomUUID() : event.eventId();
OffsetDateTime receivedHubAt = event.receivedHubAt() == null ? OffsetDateTime.now() : event.receivedHubAt();
DriverRefDto driverRef = event.driverRef();
DriverCardRefDto driverCard = driverRef == null ? null : driverRef.driverCard();
VehicleRefDto vehicleRef = event.vehicleRef();
VehicleRegistrationRefDto vehicleRegistration = vehicleRef == null ? null : vehicleRef.vehicleRegistration();
SourcePackageRefDto sourcePackageRef = event.sourcePackageRef();
ps.setObject(1, eventId);
ps.setInt(2, eventSourceId);
ps.setObject(3, packageId);
ps.setString(4, event.externalSourceEventId());
ps.setString(5, driverRef == null ? null : driverRef.sourceEntityId());
ps.setString(6, driverCard == null ? null : driverCard.nation());
ps.setString(7, driverCard == null ? null : driverCard.number());
ps.setString(8, vehicleRef == null ? null : vehicleRef.sourceEntityId());
ps.setString(9, vehicleRef == null ? null : vehicleRef.vin());
ps.setString(10, vehicleRegistration == null ? null : vehicleRegistration.nation());
ps.setString(11, vehicleRegistration == null ? null : vehicleRegistration.number());
ps.setString(12, sourcePackageRef == null ? null : sourcePackageRef.packageKind());
ps.setString(13, sourcePackageRef == null ? null : sourcePackageRef.sourcePackageId());
ps.setString(14, sourcePackageRef == null ? null : sourcePackageRef.sourceEntityId());
ps.setObject(15, sourcePackageRef == null ? null : sourcePackageRef.packagePeriodFrom());
ps.setObject(16, sourcePackageRef == null ? null : sourcePackageRef.packagePeriodTo());
ps.setObject(17, sourcePackageRef == null ? null : sourcePackageRef.importedIntoSourceAt());
ps.setObject(18, event.occurredAt());
ps.setObject(19, event.receivedPartnerAt());
ps.setObject(20, receivedHubAt);
ps.setString(21, event.eventDomain().name());
ps.setString(22, event.eventType().name());
ps.setString(23, event.lifecycle().name());
setNullableLong(ps, 24, event.odometerM());
if (event.position() == null) {
ps.setNull(25, Types.NUMERIC);
ps.setNull(26, Types.NUMERIC);
} else {
ps.setObject(25, event.position().latitude());
ps.setObject(26, event.position().longitude());
}
ps.setString(27, toJson(objectMapper.valueToTree(event.eventDetails())));
ps.setString(28, toJson(event.payload()));
ps.setBoolean(29, event.manualEntry());
ps.setString(30, recordKeyService.buildSourceRecordKeyHash(event, eventSourceId));
ps.setString(31, recordKeyService.buildEventSignatureHash(event));
public void setValues(PreparedStatement ps, int i) throws SQLException {
ResolvedEventImportRow row = rows.get(i);
ps.setInt(1, i);
ps.setString(2, row.sourceRecordKeyHash());
ps.setObject(3, row.requestedEventId());
ps.setObject(4, row.packageId());
ps.setInt(5, row.eventSourceId());
ps.setString(6, row.externalSourceEventId());
ps.setObject(7, row.driverEntityId());
ps.setObject(8, row.vehicleId());
ps.setObject(9, row.vehicleRegistrationId());
ps.setString(10, row.sourcePackageId());
ps.setObject(11, row.sourcePackageEntityId());
ps.setObject(12, row.occurredAt());
ps.setObject(13, row.receivedPartnerAt());
ps.setObject(14, row.receivedHubAt());
ps.setString(15, row.eventDomain());
ps.setString(16, row.eventType());
ps.setString(17, row.lifecycle());
ps.setObject(18, row.odometerM());
ps.setObject(19, row.longitude());
ps.setObject(20, row.latitude());
ps.setString(21, row.payloadJson());
ps.setBoolean(22, row.manualEntry());
ps.setString(23, row.eventSignatureHash());
}
@Override
public int getBatchSize() {
return events.size();
return rows.size();
}
}
);
int inserted = 0;
for (int count : counts) {
if (count > 0 || count == PreparedStatement.SUCCESS_NO_INFO) {
inserted++;
}
}
return inserted;
}
private void setNullableLong(PreparedStatement ps, int index, Long value) throws java.sql.SQLException {
if (value == null) {
ps.setNull(index, Types.BIGINT);
} else {
ps.setLong(index, value);
private int insertStagedEvents() {
Long insertedCount = jdbcTemplate.queryForObject(
"""
with stage_one as (
select distinct on (source_record_key_hash) *
from eventhub_event_import_stage
order by source_record_key_hash, row_no
),
reserved_source_record as (
insert into eventhub.event_source_record(
source_record_key_hash, event_occurred_at, event_id
)
select source_record_key_hash, occurred_at, requested_event_id
from stage_one
on conflict (source_record_key_hash) do nothing
returning source_record_key_hash, event_occurred_at, event_id
),
existing_source_record as (
select source_record.source_record_key_hash,
source_record.event_occurred_at,
source_record.event_id
from stage_one stage
join eventhub.event_source_record source_record
on source_record.source_record_key_hash = stage.source_record_key_hash
where not exists (
select 1
from reserved_source_record reserved
where reserved.source_record_key_hash = stage.source_record_key_hash
)
),
source_record_for_stage as (
select *
from reserved_source_record
union all
select *
from existing_source_record
),
inserted_event as (
insert into eventhub.event(
id, event_source_id, data_package_id,
external_source_event_id,
driver_entity_id, vehicle_id, vehicle_registration_id,
source_package_id, source_package_entity_id,
occurred_at, received_partner_at, received_hub_at,
event_domain, event_type, lifecycle,
odometer_m, position,
payload, manual_entry,
source_record_key_hash, event_signature_hash
)
select
source_record.event_id, stage.event_source_id, stage.data_package_id,
stage.external_source_event_id,
stage.driver_entity_id, stage.vehicle_id, stage.vehicle_registration_id,
stage.source_package_id, stage.source_package_entity_id,
source_record.event_occurred_at, stage.received_partner_at, stage.received_hub_at,
stage.event_domain, stage.event_type, stage.lifecycle,
stage.odometer_m, case
when stage.longitude is null or stage.latitude is null then null
else ST_SetSRID(ST_MakePoint(stage.longitude, stage.latitude), 4326)::geography
end,
stage.payload, stage.manual_entry,
stage.source_record_key_hash, stage.event_signature_hash
from stage_one stage
join source_record_for_stage source_record
on source_record.source_record_key_hash = stage.source_record_key_hash
where not exists (
select 1
from eventhub.event existing_event
where existing_event.occurred_at = source_record.event_occurred_at
and existing_event.id = source_record.event_id
)
returning id
)
select count(*)
from inserted_event
""",
Long.class
);
return insertedCount == null ? 0 : Math.toIntExact(insertedCount);
}
private Map<String, InsertedEventRow> resolveStagedEventRows() {
return jdbcTemplate.query(
"""
select distinct
stage.source_record_key_hash,
source_record.event_id,
source_record.event_occurred_at
from eventhub_event_import_stage stage
join eventhub.event_source_record source_record
on source_record.source_record_key_hash = stage.source_record_key_hash
""",
rs -> {
Map<String, InsertedEventRow> rows = new HashMap<>();
while (rs.next()) {
rows.put(
rs.getString("source_record_key_hash"),
new InsertedEventRow(
(UUID) rs.getObject("event_id"),
rs.getObject("event_occurred_at", OffsetDateTime.class)
)
);
}
return rows;
}
);
}
private void batchUpsertEventDetails(
List<ResolvedEventImportRow> rows,
Map<String, InsertedEventRow> eventRowsBySourceRecord
) {
List<EventDetailImportRow> details = new ArrayList<>();
for (ResolvedEventImportRow row : rows) {
if (row.detailType() == null) {
continue;
}
InsertedEventRow insertedEventRow = eventRowsBySourceRecord.get(row.sourceRecordKeyHash());
if (insertedEventRow == null) {
throw new IllegalStateException("Could not insert or resolve event row for source record hash " + row.sourceRecordKeyHash());
}
details.add(new EventDetailImportRow(
insertedEventRow.eventId(),
insertedEventRow.occurredAt(),
row.detailType(),
row.detailAttributesJson()
));
}
if (details.isEmpty()) {
return;
}
jdbcTemplate.batchUpdate(
"""
insert into eventhub.event_detail(
event_occurred_at, event_id, detail_type, attributes
) values (?, ?, ?, ?::jsonb)
on conflict (event_occurred_at, event_id, detail_type)
do update set
attributes = excluded.attributes
""",
new BatchPreparedStatementSetter() {
@Override
public void setValues(PreparedStatement ps, int i) throws SQLException {
EventDetailImportRow detail = details.get(i);
ps.setObject(1, detail.occurredAt());
ps.setObject(2, detail.eventId());
ps.setString(3, detail.detailType());
ps.setString(4, detail.attributesJson());
}
@Override
public int getBatchSize() {
return details.size();
}
}
);
}
private ResolvedEntityRefs resolveEntityRefs(
String tenantKey,
int eventSourceId,
EventHubEventDto event,
Map<String, UUID> entityIdCache,
Map<String, List<VehicleRefCacheEntry>> vehicleRefCache
) {
UUID driverEntityId = resolveDriverEntityId(tenantKey, eventSourceId, event, entityIdCache);
ResolvedVehicleReference vehicleRef = resolveVehicleReference(tenantKey, eventSourceId, event, vehicleRefCache);
UUID sourcePackageEntityId = resolveSourcePackageEntityId(tenantKey, eventSourceId, event, entityIdCache);
return new ResolvedEntityRefs(driverEntityId, vehicleRef.vehicleId(), vehicleRef.vehicleRegistrationId(), sourcePackageEntityId);
}
private UUID resolveDriverEntityId(
String tenantKey,
int eventSourceId,
EventHubEventDto event,
Map<String, UUID> entityIdCache
) {
DriverRefDto driverRef = event.driverRef();
if (driverRef == null || !driverRef.hasAnyReference()) {
return null;
}
DriverCardRefDto card = driverRef.driverCard();
String cardKey = card == null || !card.hasValue() ? null : card.stableKey();
String sourceEntityId = normalizeNullable(driverRef.sourceEntityId());
if (sourceEntityId == null && cardKey != null) {
sourceEntityId = "DRIVER_CARD:" + cardKey;
}
if (sourceEntityId == null) {
return null;
}
Map<String, Object> payload = new LinkedHashMap<>();
put(payload, "source_entity_id", driverRef.sourceEntityId());
put(payload, "driver_card_nation", card == null ? null : card.nation());
put(payload, "driver_card_number", card == null ? null : card.number());
return resolveEntityId(
tenantKey,
eventSourceId,
"DRIVER",
sourceEntityId,
cardKey,
sourceEntityId,
null,
payload,
entityIdCache
);
}
private ResolvedVehicleReference resolveVehicleReference(
String tenantKey,
int eventSourceId,
EventHubEventDto event,
Map<String, List<VehicleRefCacheEntry>> vehicleRefCache
) {
String cacheKey = vehicleRefCacheKey(event.vehicleRef());
if (cacheKey != null) {
ResolvedVehicleReference cached = findCachedVehicleReference(
vehicleRefCache.get(cacheKey),
event.occurredAt()
);
if (cached != null) {
return cached;
}
}
ResolvedVehicleReferenceResolution resolved = vehicleIdentityRepository.resolveOrCreateVehicleReference(
tenantKey,
eventSourceId,
event.vehicleRef(),
event.occurredAt()
);
if (cacheKey != null) {
vehicleRefCache
.computeIfAbsent(cacheKey, ignored -> new ArrayList<>())
.add(buildVehicleRefCacheEntry(event.vehicleRef(), event.occurredAt(), resolved));
}
return resolved.reference();
}
private UUID resolveSourcePackageEntityId(
String tenantKey,
int eventSourceId,
EventHubEventDto event,
Map<String, UUID> entityIdCache
) {
SourcePackageRefDto sourcePackageRef = event.sourcePackageRef();
if (sourcePackageRef == null || !sourcePackageRef.hasAnyReference()) {
return null;
}
String packageKind = normalizeNullable(sourcePackageRef.packageKind());
String sourcePackageId = normalizeNullable(sourcePackageRef.sourcePackageId());
String sourceEntityId = normalizeNullable(sourcePackageRef.sourceEntityId());
if (sourceEntityId == null && sourcePackageId != null) {
sourceEntityId = "SOURCE_PACKAGE:" + (packageKind == null ? "UNKNOWN" : packageKind) + ":" + sourcePackageId;
}
if (sourceEntityId == null) {
return null;
}
Map<String, Object> payload = new LinkedHashMap<>();
put(payload, "package_kind", sourcePackageRef.packageKind());
put(payload, "source_package_id", sourcePackageRef.sourcePackageId());
put(payload, "source_entity_id", sourcePackageRef.sourceEntityId());
put(payload, "package_period_from", sourcePackageRef.packagePeriodFrom());
put(payload, "package_period_to", sourcePackageRef.packagePeriodTo());
put(payload, "imported_into_source_at", sourcePackageRef.importedIntoSourceAt());
String displayName = packageKind == null ? sourceEntityId : packageKind + ":" + (sourcePackageId == null ? sourceEntityId : sourcePackageId);
return resolveEntityId(
tenantKey,
eventSourceId,
"SOURCE_PACKAGE",
sourceEntityId,
sourcePackageId,
displayName,
null,
payload,
entityIdCache
);
}
private UUID resolveEntityId(
String tenantKey,
int eventSourceId,
String entityType,
String sourceEntityId,
String sourceExternalKey,
String displayName,
Boolean active,
Map<String, Object> payload,
Map<String, UUID> entityIdCache
) {
String normalizedSourceEntityId = normalizeNullable(sourceEntityId);
if (normalizedSourceEntityId == null) {
return null;
}
String cacheKey = entityType + "|" + normalizedSourceEntityId;
UUID cached = entityIdCache.get(cacheKey);
if (cached != null) {
return cached;
}
UUID resolved = sourceMasterDataRepository.resolveOrCreateEntityId(
tenantKey,
eventSourceId,
entityType,
normalizedSourceEntityId,
normalizeNullable(sourceExternalKey),
normalizeNullable(displayName),
active,
payload
);
entityIdCache.put(cacheKey, resolved);
return resolved;
}
private String normalizeNullable(String value) {
if (value == null) {
return null;
}
String trimmed = value.trim();
return trimmed.isEmpty() ? null : trimmed;
}
private void put(Map<String, Object> target, String key, Object value) {
if (value != null) {
target.put(key, value);
}
}
private String vehicleRefCacheKey(VehicleRefDto vehicleRef) {
if (vehicleRef == null || !vehicleRef.hasAnyReference()) {
return null;
}
String sourceVehicleEntityId = normalizeNullable(vehicleRef.sourceVehicleEntityId());
String vin = normalizeNullable(vehicleRef.vin());
String sourceRegistrationEntityId = normalizeNullable(vehicleRef.sourceRegistrationEntityId());
VehicleRegistrationRefDto registration = vehicleRef.vehicleRegistration();
String registrationNation = registration == null ? null : normalizeNullable(registration.nation());
String registrationNumber = registration == null ? null : normalizeNullable(registration.number());
return String.join("|",
sourceVehicleEntityId == null ? "" : sourceVehicleEntityId,
vin == null ? "" : vin,
sourceRegistrationEntityId == null ? "" : sourceRegistrationEntityId,
registrationNation == null ? "" : registrationNation,
registrationNumber == null ? "" : registrationNumber
);
}
private ResolvedVehicleReference findCachedVehicleReference(
List<VehicleRefCacheEntry> cacheEntries,
OffsetDateTime occurredAt
) {
if (cacheEntries == null || cacheEntries.isEmpty()) {
return null;
}
for (VehicleRefCacheEntry cacheEntry : cacheEntries) {
if (cacheEntry.matches(occurredAt)) {
return cacheEntry.reference();
}
}
return null;
}
private VehicleRefCacheEntry buildVehicleRefCacheEntry(
VehicleRefDto vehicleRef,
OffsetDateTime occurredAt,
ResolvedVehicleReferenceResolution resolved
) {
boolean hasDirectVehicleIdentity = hasDirectVehicleIdentity(vehicleRef);
if (hasDirectVehicleIdentity) {
return new VehicleRefCacheEntry(resolved.reference(), null, null, null);
}
if (resolved.resolvedFromAssignment()) {
return new VehicleRefCacheEntry(
resolved.reference(),
resolved.assignmentValidFrom(),
resolved.assignmentValidTo(),
null
);
}
return new VehicleRefCacheEntry(resolved.reference(), null, null, occurredAt);
}
private boolean hasDirectVehicleIdentity(VehicleRefDto vehicleRef) {
if (vehicleRef == null) {
return false;
}
return normalizeNullable(vehicleRef.sourceVehicleEntityId()) != null
|| normalizeNullable(vehicleRef.vin()) != null;
}
private String toJson(JsonNode value) {
@ -154,4 +597,73 @@ public class EventRepository {
throw new IllegalArgumentException("Cannot serialize JSONB value", e);
}
}
private record ResolvedEntityRefs(
UUID driverEntityId,
UUID vehicleId,
UUID vehicleRegistrationId,
UUID sourcePackageEntityId
) {
}
private record InsertedEventRow(
UUID eventId,
OffsetDateTime occurredAt
) {
}
private record ResolvedEventImportRow(
String sourceRecordKeyHash,
UUID requestedEventId,
UUID packageId,
int eventSourceId,
String externalSourceEventId,
UUID driverEntityId,
UUID vehicleId,
UUID vehicleRegistrationId,
String sourcePackageId,
UUID sourcePackageEntityId,
OffsetDateTime occurredAt,
OffsetDateTime receivedPartnerAt,
OffsetDateTime receivedHubAt,
String eventDomain,
String eventType,
String lifecycle,
Long odometerM,
Double longitude,
Double latitude,
String payloadJson,
boolean manualEntry,
String eventSignatureHash,
String detailType,
String detailAttributesJson
) {
}
private record EventDetailImportRow(
UUID eventId,
OffsetDateTime occurredAt,
String detailType,
String attributesJson
) {
}
private record VehicleRefCacheEntry(
ResolvedVehicleReference reference,
OffsetDateTime validFrom,
OffsetDateTime validTo,
OffsetDateTime exactOccurredAt
) {
private boolean matches(OffsetDateTime occurredAt) {
if (exactOccurredAt != null) {
return exactOccurredAt.equals(occurredAt);
}
if (validFrom == null && validTo == null) {
return true;
}
boolean fromMatches = validFrom == null || occurredAt == null || !occurredAt.isBefore(validFrom);
boolean toMatches = validTo == null || occurredAt == null || occurredAt.isBefore(validTo);
return fromMatches && toMatches;
}
}
}

View File

@ -0,0 +1,536 @@
package at.procon.eventhub.persistence;
import at.procon.eventhub.importing.masterdata.SourceMasterEntityUpsert;
import at.procon.eventhub.importing.masterdata.SourceMasterRelationUpsert;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.node.POJONode;
import java.lang.reflect.Array;
import java.sql.PreparedStatement;
import java.sql.SQLException;
import java.sql.Timestamp;
import java.time.OffsetDateTime;
import java.time.temporal.TemporalAccessor;
import java.time.temporal.TemporalAmount;
import java.util.ArrayList;
import java.util.Base64;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.UUID;
import java.util.stream.Collectors;
import org.springframework.jdbc.core.BatchPreparedStatementSetter;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.stereotype.Repository;
import org.springframework.transaction.annotation.Transactional;
@Repository
public class SourceMasterDataRepository {
private final JdbcTemplate jdbcTemplate;
private final ObjectMapper objectMapper;
public SourceMasterDataRepository(JdbcTemplate jdbcTemplate, ObjectMapper objectMapper) {
this.jdbcTemplate = jdbcTemplate;
this.objectMapper = objectMapper;
}
@Transactional
public int upsertEntities(String tenantKey, int eventSourceId, List<SourceMasterEntityUpsert> entities) {
List<SourceMasterEntityStageRow> rows = new ArrayList<>(entities.size());
for (SourceMasterEntityUpsert entity : entities) {
if (entity.sourceEntityId() == null || entity.sourceEntityId().isBlank()) {
continue;
}
rows.add(new SourceMasterEntityStageRow(
UUID.randomUUID(),
entity.entityType(),
entity.sourceEntityId(),
entity.sourceExternalKey(),
entity.displayName(),
entity.active(),
entity.validFrom(),
entity.validTo(),
entity.sourceUpdatedAt(),
toJson(entity.payload())
));
}
if (rows.isEmpty()) {
return 0;
}
createEntityStage();
stageEntities(rows);
upsertStagedEntities(tenantKey, eventSourceId);
return rows.size();
}
@Transactional
public int upsertRelations(String tenantKey, int eventSourceId, List<SourceMasterRelationUpsert> relations) {
List<SourceMasterRelationStageRow> rows = new ArrayList<>(relations.size());
for (SourceMasterRelationUpsert relation : relations) {
if (relation.fromSourceEntityId() == null || relation.toSourceEntityId() == null) {
continue;
}
rows.add(new SourceMasterRelationStageRow(
UUID.randomUUID(),
relationKey(relation),
relation.relationType(),
relation.fromEntityType(),
relation.fromSourceEntityId(),
relation.toEntityType(),
relation.toSourceEntityId(),
relation.validFrom(),
relation.validTo(),
relation.sourceUpdatedAt(),
toJson(relation.payload())
));
}
if (rows.isEmpty()) {
return 0;
}
createRelationStage();
stageRelations(rows);
upsertStagedRelations(tenantKey, eventSourceId);
return rows.size();
}
private void createEntityStage() {
jdbcTemplate.execute(
"""
create temporary table if not exists eventhub_source_master_entity_stage (
row_no integer not null,
id uuid not null,
entity_type text not null,
source_entity_id text not null,
source_external_key text,
display_name text,
active boolean,
valid_from timestamptz,
valid_to timestamptz,
source_updated_at timestamptz,
payload jsonb not null
) on commit drop
"""
);
jdbcTemplate.execute("truncate table eventhub_source_master_entity_stage");
}
private void stageEntities(List<SourceMasterEntityStageRow> rows) {
jdbcTemplate.batchUpdate(
"""
insert into eventhub_source_master_entity_stage(
row_no, id, entity_type, source_entity_id, source_external_key, display_name,
active, valid_from, valid_to, source_updated_at, payload
) values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?::jsonb)
""",
new BatchPreparedStatementSetter() {
@Override
public void setValues(PreparedStatement ps, int i) throws SQLException {
SourceMasterEntityStageRow row = rows.get(i);
ps.setInt(1, i);
ps.setObject(2, row.id());
ps.setString(3, row.entityType());
ps.setString(4, row.sourceEntityId());
ps.setString(5, row.sourceExternalKey());
ps.setString(6, row.displayName());
ps.setObject(7, row.active());
ps.setObject(8, row.validFrom());
ps.setObject(9, row.validTo());
ps.setObject(10, row.sourceUpdatedAt());
ps.setString(11, row.payloadJson());
}
@Override
public int getBatchSize() {
return rows.size();
}
}
);
}
private void upsertStagedEntities(String tenantKey, int eventSourceId) {
jdbcTemplate.update(
"""
insert into eventhub.source_master_entity(
id, tenant_key, event_source_id, entity_type, source_entity_id,
source_external_key, display_name, active, valid_from, valid_to,
source_updated_at, payload, updated_at
)
select
id, ?, ?, entity_type, source_entity_id,
source_external_key, display_name, active, valid_from, valid_to,
source_updated_at, payload, now()
from (
select distinct on (entity_type, source_entity_id) *
from eventhub_source_master_entity_stage
order by entity_type, source_entity_id, row_no desc
) stage
on conflict (tenant_key, event_source_id, entity_type, source_entity_id)
do update set
source_external_key = excluded.source_external_key,
display_name = excluded.display_name,
active = excluded.active,
valid_from = excluded.valid_from,
valid_to = excluded.valid_to,
source_updated_at = excluded.source_updated_at,
payload = excluded.payload,
updated_at = now()
""",
tenantKey,
eventSourceId
);
}
private void createRelationStage() {
jdbcTemplate.execute(
"""
create temporary table if not exists eventhub_source_master_relation_stage (
row_no integer not null,
id uuid not null,
relation_key text not null,
relation_type text not null,
from_entity_type text not null,
from_source_entity_id text not null,
to_entity_type text not null,
to_source_entity_id text not null,
valid_from timestamptz,
valid_to timestamptz,
source_updated_at timestamptz,
payload jsonb not null
) on commit drop
"""
);
jdbcTemplate.execute("truncate table eventhub_source_master_relation_stage");
}
private void stageRelations(List<SourceMasterRelationStageRow> rows) {
jdbcTemplate.batchUpdate(
"""
insert into eventhub_source_master_relation_stage(
row_no, id, relation_key, relation_type, from_entity_type, from_source_entity_id,
to_entity_type, to_source_entity_id, valid_from, valid_to, source_updated_at, payload
) values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?::jsonb)
""",
new BatchPreparedStatementSetter() {
@Override
public void setValues(PreparedStatement ps, int i) throws SQLException {
SourceMasterRelationStageRow row = rows.get(i);
ps.setInt(1, i);
ps.setObject(2, row.id());
ps.setString(3, row.relationKey());
ps.setString(4, row.relationType());
ps.setString(5, row.fromEntityType());
ps.setString(6, row.fromSourceEntityId());
ps.setString(7, row.toEntityType());
ps.setString(8, row.toSourceEntityId());
ps.setObject(9, row.validFrom());
ps.setObject(10, row.validTo());
ps.setObject(11, row.sourceUpdatedAt());
ps.setString(12, row.payloadJson());
}
@Override
public int getBatchSize() {
return rows.size();
}
}
);
}
private void upsertStagedRelations(String tenantKey, int eventSourceId) {
jdbcTemplate.update(
"""
insert into eventhub.source_master_relation(
id, tenant_key, event_source_id, relation_key, relation_type,
from_entity_type, from_source_entity_id, to_entity_type, to_source_entity_id,
valid_from, valid_to, source_updated_at, payload, updated_at
)
select
id, ?, ?, relation_key, relation_type,
from_entity_type, from_source_entity_id, to_entity_type, to_source_entity_id,
valid_from, valid_to, source_updated_at, payload, now()
from (
select distinct on (relation_key) *
from eventhub_source_master_relation_stage
order by relation_key, row_no desc
) stage
on conflict (tenant_key, event_source_id, relation_key)
do update set
relation_type = excluded.relation_type,
from_entity_type = excluded.from_entity_type,
from_source_entity_id = excluded.from_source_entity_id,
to_entity_type = excluded.to_entity_type,
to_source_entity_id = excluded.to_source_entity_id,
valid_from = excluded.valid_from,
valid_to = excluded.valid_to,
source_updated_at = excluded.source_updated_at,
payload = excluded.payload,
updated_at = now()
""",
tenantKey,
eventSourceId
);
}
public UUID resolveOrCreateEntityId(
String tenantKey,
int eventSourceId,
String entityType,
String sourceEntityId,
String sourceExternalKey,
String displayName,
Boolean active,
Map<String, Object> payload
) {
String normalizedTenantKey = normalizeRequired(tenantKey, "tenantKey");
String normalizedEntityType = normalizeRequired(entityType, "entityType").toUpperCase();
String normalizedSourceEntityId = normalizeRequired(sourceEntityId, "sourceEntityId");
String normalizedSourceExternalKey = normalizeNullable(sourceExternalKey);
String normalizedDisplayName = normalizeNullable(displayName);
UUID existing = findEntityId(normalizedTenantKey, eventSourceId, normalizedEntityType, normalizedSourceEntityId);
if (existing != null) {
return existing;
}
UUID created = jdbcTemplate.query(
con -> {
PreparedStatement ps = con.prepareStatement(
"""
insert into eventhub.source_master_entity(
id, tenant_key, event_source_id, entity_type, source_entity_id,
source_external_key, display_name, active, payload, updated_at
) values (?, ?, ?, ?, ?, ?, ?, ?, ?::jsonb, now())
on conflict (tenant_key, event_source_id, entity_type, source_entity_id) do nothing
returning id
"""
);
ps.setObject(1, UUID.randomUUID());
ps.setString(2, normalizedTenantKey);
ps.setInt(3, eventSourceId);
ps.setString(4, normalizedEntityType);
ps.setString(5, normalizedSourceEntityId);
ps.setString(6, normalizedSourceExternalKey);
ps.setString(7, normalizedDisplayName);
ps.setObject(8, active);
ps.setString(9, toJson(payload));
return ps;
},
rs -> {
if (!rs.next()) {
return null;
}
return (UUID) rs.getObject(1);
}
);
if (created != null) {
return created;
}
UUID resolved = findEntityId(normalizedTenantKey, eventSourceId, normalizedEntityType, normalizedSourceEntityId);
if (resolved != null) {
return resolved;
}
throw new IllegalStateException(
"Could not resolve source master entity id for "
+ normalizedTenantKey + ":" + eventSourceId + ":" + normalizedEntityType + ":" + normalizedSourceEntityId
);
}
private UUID findEntityId(String tenantKey, int eventSourceId, String entityType, String sourceEntityId) {
return jdbcTemplate.query(
"""
select id
from eventhub.source_master_entity
where tenant_key = ?
and event_source_id = ?
and entity_type = ?
and source_entity_id = ?
""",
rs -> rs.next() ? (UUID) rs.getObject("id") : null,
tenantKey,
eventSourceId,
entityType,
sourceEntityId
);
}
private String relationKey(SourceMasterRelationUpsert relation) {
String validFrom = relation.validFrom() == null ? "MIN" : relation.validFrom().toString();
String validTo = relation.validTo() == null ? "MAX" : relation.validTo().toString();
return relation.relationType()
+ ":" + relation.fromEntityType()
+ ":" + relation.fromSourceEntityId()
+ ":" + relation.toEntityType()
+ ":" + relation.toSourceEntityId()
+ ":" + validFrom
+ ":" + validTo;
}
private String toJson(Map<String, Object> value) {
Map<String, Object> source = value == null ? Map.of() : value;
try {
return objectMapper.writeValueAsString(normalizeJsonMap(source));
} catch (JsonProcessingException e) {
throw new IllegalArgumentException(
"Cannot serialize source master data payload. payloadTypes=" + describePayloadTypes(source),
e
);
}
}
private Map<String, Object> normalizeJsonMap(Map<?, ?> source) {
Map<String, Object> normalized = new LinkedHashMap<>(source.size());
for (Map.Entry<?, ?> entry : source.entrySet()) {
if (entry.getKey() == null) {
continue;
}
normalized.put(String.valueOf(entry.getKey()), normalizeJsonValue(entry.getValue()));
}
return normalized;
}
private Object normalizeJsonValue(Object value) {
if (value == null
|| value instanceof String
|| value instanceof Number
|| value instanceof Boolean) {
return value;
}
if (value instanceof JsonNode node) {
return normalizeJsonNode(node);
}
if (value instanceof Enum<?> enumValue) {
return enumValue.name();
}
if (value instanceof UUID uuid) {
return uuid.toString();
}
if (value instanceof byte[] bytes) {
return Base64.getEncoder().encodeToString(bytes);
}
if (value instanceof Timestamp timestamp) {
return timestamp.toInstant().toString();
}
if (value instanceof java.util.Date date) {
return date.toInstant().toString();
}
if (value instanceof TemporalAccessor || value instanceof TemporalAmount) {
return value.toString();
}
if (value instanceof Map<?, ?> map) {
return normalizeJsonMap(map);
}
if (value instanceof Iterable<?> iterable) {
List<Object> values = new ArrayList<>();
for (Object entry : iterable) {
values.add(normalizeJsonValue(entry));
}
return values;
}
if (value.getClass().isArray()) {
int length = Array.getLength(value);
List<Object> values = new ArrayList<>(length);
for (int i = 0; i < length; i++) {
values.add(normalizeJsonValue(Array.get(value, i)));
}
return values;
}
return String.valueOf(value);
}
private Object normalizeJsonNode(JsonNode node) {
if (node == null || node.isNull() || node.isMissingNode()) {
return null;
}
if (node instanceof POJONode pojoNode) {
return normalizeJsonValue(pojoNode.getPojo());
}
if (node.isObject()) {
return normalizeJsonMap(objectNodeToMap(node));
}
if (node.isArray()) {
List<Object> values = new ArrayList<>();
for (JsonNode child : node) {
values.add(normalizeJsonNode(child));
}
return values;
}
if (node.isTextual()) {
return node.textValue();
}
if (node.isNumber()) {
return node.numberValue();
}
if (node.isBoolean()) {
return node.booleanValue();
}
if (node.isBinary()) {
return node.asText();
}
return node.asText();
}
private Map<String, Object> objectNodeToMap(JsonNode node) {
Map<String, Object> values = new LinkedHashMap<>();
node.fields().forEachRemaining(entry -> values.put(entry.getKey(), entry.getValue()));
return values;
}
private String describePayloadTypes(Map<String, Object> payload) {
if (payload.isEmpty()) {
return "{}";
}
return payload.entrySet()
.stream()
.map(entry -> entry.getKey() + "=" + (entry.getValue() == null ? "null" : entry.getValue().getClass().getName()))
.collect(Collectors.joining(", ", "{", "}"));
}
private String normalizeRequired(String value, String fieldName) {
String normalized = normalizeNullable(value);
if (normalized == null) {
throw new IllegalArgumentException(fieldName + " must not be blank");
}
return normalized;
}
private String normalizeNullable(String value) {
if (value == null) {
return null;
}
String trimmed = value.trim();
return trimmed.isEmpty() ? null : trimmed;
}
private record SourceMasterEntityStageRow(
UUID id,
String entityType,
String sourceEntityId,
String sourceExternalKey,
String displayName,
Boolean active,
OffsetDateTime validFrom,
OffsetDateTime validTo,
OffsetDateTime sourceUpdatedAt,
String payloadJson
) {
}
private record SourceMasterRelationStageRow(
UUID id,
String relationKey,
String relationType,
String fromEntityType,
String fromSourceEntityId,
String toEntityType,
String toSourceEntityId,
OffsetDateTime validFrom,
OffsetDateTime validTo,
OffsetDateTime sourceUpdatedAt,
String payloadJson
) {
}
}

View File

@ -0,0 +1,710 @@
package at.procon.eventhub.persistence;
import at.procon.eventhub.dto.VehicleRefDto;
import at.procon.eventhub.dto.VehicleRegistrationRefDto;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import java.time.OffsetDateTime;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.UUID;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.stereotype.Repository;
import org.springframework.transaction.annotation.Transactional;
@Repository
public class VehicleIdentityRepository {
private final JdbcTemplate jdbcTemplate;
private final ObjectMapper objectMapper;
public VehicleIdentityRepository(JdbcTemplate jdbcTemplate, ObjectMapper objectMapper) {
this.jdbcTemplate = jdbcTemplate;
this.objectMapper = objectMapper;
}
public ResolvedVehicleReferenceResolution resolveOrCreateVehicleReference(
String tenantKey,
int eventSourceId,
VehicleRefDto vehicleRef,
OffsetDateTime occurredAt
) {
if (vehicleRef == null || !vehicleRef.hasAnyReference()) {
return ResolvedVehicleReferenceResolution.empty();
}
String normalizedTenantKey = normalizeRequired(tenantKey, "tenantKey");
String sourceVehicleEntityId = normalizeNullable(vehicleRef.sourceVehicleEntityId());
String vin = normalizeNullable(vehicleRef.vin());
String sourceRegistrationEntityId = normalizeNullable(vehicleRef.sourceRegistrationEntityId());
VehicleRegistrationRefDto registration = vehicleRef.vehicleRegistration();
String registrationNation = registration == null ? null : normalizeNullable(registration.nation());
String registrationNumber = registration == null ? null : normalizeNullable(registration.number());
UUID registrationId = resolveRegistrationId(
normalizedTenantKey,
eventSourceId,
sourceRegistrationEntityId,
registrationNation,
registrationNumber,
occurredAt
);
if (registrationId == null && (sourceRegistrationEntityId != null || registrationNumber != null)) {
registrationId = createRegistration(
normalizedTenantKey,
eventSourceId,
sourceRegistrationEntityId,
registrationNation,
registrationNumber,
null,
Map.of("source", "event")
);
}
UUID vehicleId = resolveVehicleId(
normalizedTenantKey,
eventSourceId,
sourceVehicleEntityId,
vin
);
AssignedVehicleReference assignedVehicle = null;
if (vehicleId == null && registrationId != null) {
assignedVehicle = resolveAssignedVehicleReference(registrationId, occurredAt);
vehicleId = assignedVehicle == null ? null : assignedVehicle.vehicleId();
}
if (vehicleId == null && (sourceVehicleEntityId != null || vin != null)) {
vehicleId = createVehicle(
normalizedTenantKey,
eventSourceId,
sourceVehicleEntityId,
vin
);
}
if (vehicleId != null) {
touchVehicle(vehicleId, sourceVehicleEntityId, vin);
}
if (registrationId != null) {
touchRegistration(registrationId, sourceRegistrationEntityId, registrationNation, registrationNumber);
}
return new ResolvedVehicleReferenceResolution(
new ResolvedVehicleReference(vehicleId, registrationId),
assignedVehicle != null,
assignedVehicle == null ? null : assignedVehicle.validFrom(),
assignedVehicle == null ? null : assignedVehicle.validTo()
);
}
@Transactional
public int reconcileFromMasterData(String tenantKey, int eventSourceId) {
String normalizedTenantKey = normalizeRequired(tenantKey, "tenantKey");
int updates = reconcileVehiclesFromMasterData(normalizedTenantKey, eventSourceId);
updates += reconcileRegistrationsFromMasterData(normalizedTenantKey, eventSourceId);
updates += projectVehicleRegistrationAssignments(normalizedTenantKey, eventSourceId);
return updates;
}
private int reconcileVehiclesFromMasterData(String tenantKey, int eventSourceId) {
Long count = jdbcTemplate.queryForObject(
compatibleSourcesCte() + """
, master_vehicles as (
select distinct on (
event_source_id,
nullif(trim(source_entity_id), ''),
nullif(trim(source_external_key), '')
)
event_source_id,
nullif(trim(source_entity_id), '') as source_vehicle_entity_id,
nullif(trim(source_external_key), '') as vin
from eventhub.source_master_entity
where tenant_key = ?
and event_source_id in (select id from compatible_sources)
and entity_type = 'VEHICLE'
and (
nullif(trim(source_entity_id), '') is not null
or nullif(trim(source_external_key), '') is not null
)
order by event_source_id,
nullif(trim(source_entity_id), ''),
nullif(trim(source_external_key), ''),
updated_at desc
),
updated_by_source as (
update eventhub.vehicle vehicle
set source_vehicle_entity_id = coalesce(vehicle.source_vehicle_entity_id, master.source_vehicle_entity_id),
vin = coalesce(vehicle.vin, master.vin),
updated_at = now()
from master_vehicles master
where vehicle.tenant_key = ?
and vehicle.event_source_id in (select id from compatible_sources)
and master.source_vehicle_entity_id is not null
and vehicle.source_vehicle_entity_id = master.source_vehicle_entity_id
returning vehicle.id
),
updated_by_vin as (
update eventhub.vehicle vehicle
set source_vehicle_entity_id = coalesce(vehicle.source_vehicle_entity_id, master.source_vehicle_entity_id),
vin = coalesce(vehicle.vin, master.vin),
updated_at = now()
from master_vehicles master
where vehicle.tenant_key = ?
and vehicle.event_source_id in (select id from compatible_sources)
and master.vin is not null
and vehicle.vin = master.vin
returning vehicle.id
),
inserted as (
insert into eventhub.vehicle(id, tenant_key, event_source_id, source_vehicle_entity_id, vin, updated_at)
select gen_random_uuid(), ?, master.event_source_id, master.source_vehicle_entity_id, master.vin, now()
from master_vehicles master
where not exists (
select 1
from eventhub.vehicle existing
where existing.tenant_key = ?
and existing.event_source_id in (select id from compatible_sources)
and (
(master.source_vehicle_entity_id is not null and existing.source_vehicle_entity_id = master.source_vehicle_entity_id)
or (master.vin is not null and existing.vin = master.vin)
)
)
returning id
)
select (select count(*) from updated_by_source)
+ (select count(*) from updated_by_vin)
+ (select count(*) from inserted)
""",
Long.class,
eventSourceId,
tenantKey,
tenantKey,
tenantKey,
tenantKey,
tenantKey
);
return count == null ? 0 : Math.toIntExact(count);
}
private int reconcileRegistrationsFromMasterData(String tenantKey, int eventSourceId) {
Long count = jdbcTemplate.queryForObject(
compatibleSourcesCte() + """
, master_registrations as (
select distinct on (
event_source_id,
nullif(trim(source_entity_id), ''),
coalesce(
nullif(trim(payload ->> 'registration_nation'), ''),
nullif(split_part(source_external_key, ':', 1), '')
),
coalesce(
nullif(trim(payload ->> 'registration_number'), ''),
nullif(substring(source_external_key from position(':' in source_external_key) + 1), '')
)
)
event_source_id,
nullif(trim(source_entity_id), '') as source_registration_entity_id,
coalesce(
nullif(trim(payload ->> 'registration_nation'), ''),
nullif(split_part(source_external_key, ':', 1), '')
) as nation,
coalesce(
nullif(trim(payload ->> 'registration_number'), ''),
nullif(substring(source_external_key from position(':' in source_external_key) + 1), '')
) as registration_number,
source_updated_at
from eventhub.source_master_entity
where tenant_key = ?
and event_source_id in (select id from compatible_sources)
and entity_type = 'VEHICLE_REGISTRATION'
and (
nullif(trim(payload ->> 'registration_nation'), '') is not null
or source_external_key like '%:%'
)
and (
nullif(trim(payload ->> 'registration_number'), '') is not null
or source_external_key like '%:%'
)
order by event_source_id,
nullif(trim(source_entity_id), ''),
coalesce(
nullif(trim(payload ->> 'registration_nation'), ''),
nullif(split_part(source_external_key, ':', 1), '')
),
coalesce(
nullif(trim(payload ->> 'registration_number'), ''),
nullif(substring(source_external_key from position(':' in source_external_key) + 1), '')
),
updated_at desc
),
updated_by_source as (
update eventhub.vehicle_registration registration
set source_registration_entity_id = coalesce(registration.source_registration_entity_id, master.source_registration_entity_id),
nation = coalesce(master.nation, registration.nation),
registration_number = coalesce(master.registration_number, registration.registration_number),
source_updated_at = master.source_updated_at,
updated_at = now()
from master_registrations master
where registration.tenant_key = ?
and registration.event_source_id in (select id from compatible_sources)
and master.source_registration_entity_id is not null
and registration.source_registration_entity_id = master.source_registration_entity_id
returning registration.id
),
updated_by_plate as (
update eventhub.vehicle_registration registration
set source_registration_entity_id = coalesce(registration.source_registration_entity_id, master.source_registration_entity_id),
nation = coalesce(master.nation, registration.nation),
registration_number = coalesce(master.registration_number, registration.registration_number),
source_updated_at = master.source_updated_at,
updated_at = now()
from master_registrations master
where registration.tenant_key = ?
and registration.event_source_id in (select id from compatible_sources)
and registration.nation = master.nation
and registration.registration_number = master.registration_number
returning registration.id
),
inserted as (
insert into eventhub.vehicle_registration(
id, tenant_key, event_source_id, source_registration_entity_id, nation, registration_number,
source_updated_at, payload, updated_at
)
select gen_random_uuid(),
?,
master.event_source_id,
master.source_registration_entity_id,
master.nation,
master.registration_number,
master.source_updated_at,
jsonb_build_object('source', 'master-data'),
now()
from master_registrations master
where master.nation is not null
and master.registration_number is not null
and not exists (
select 1
from eventhub.vehicle_registration existing
where existing.tenant_key = ?
and existing.event_source_id in (select id from compatible_sources)
and (
(master.source_registration_entity_id is not null
and existing.source_registration_entity_id = master.source_registration_entity_id)
or (
existing.nation = master.nation
and existing.registration_number = master.registration_number
)
)
)
returning id
)
select (select count(*) from updated_by_source)
+ (select count(*) from updated_by_plate)
+ (select count(*) from inserted)
""",
Long.class,
eventSourceId,
tenantKey,
tenantKey,
tenantKey,
tenantKey,
tenantKey
);
return count == null ? 0 : Math.toIntExact(count);
}
private UUID resolveVehicleId(
String tenantKey,
int eventSourceId,
String sourceVehicleEntityId,
String vin
) {
UUID vehicleId = findVehicleBySourceVehicleEntityId(tenantKey, eventSourceId, sourceVehicleEntityId);
if (vehicleId == null) {
vehicleId = findVehicleByVin(tenantKey, eventSourceId, vin);
}
return vehicleId;
}
private UUID resolveRegistrationId(
String tenantKey,
int eventSourceId,
String sourceRegistrationEntityId,
String nation,
String registrationNumber,
OffsetDateTime occurredAt
) {
UUID registrationId = findRegistrationBySourceRegistrationEntityId(tenantKey, eventSourceId, sourceRegistrationEntityId, occurredAt);
if (registrationId == null) {
registrationId = findRegistrationByPlate(tenantKey, eventSourceId, nation, registrationNumber, occurredAt);
}
return registrationId;
}
private UUID findVehicleBySourceVehicleEntityId(String tenantKey, int eventSourceId, String sourceVehicleEntityId) {
if (sourceVehicleEntityId == null) {
return null;
}
return jdbcTemplate.query(
compatibleSourcesCte() + """
select v.id
from eventhub.vehicle v
where v.tenant_key = ?
and v.event_source_id in (select id from compatible_sources)
and v.source_vehicle_entity_id = ?
order by v.updated_at desc
limit 1
""",
rs -> rs.next() ? (UUID) rs.getObject("id") : null,
eventSourceId,
tenantKey,
sourceVehicleEntityId
);
}
private UUID findVehicleByVin(String tenantKey, int eventSourceId, String vin) {
if (vin == null) {
return null;
}
return jdbcTemplate.query(
compatibleSourcesCte() + """
select v.id
from eventhub.vehicle v
where v.tenant_key = ?
and v.event_source_id in (select id from compatible_sources)
and v.vin = ?
order by v.updated_at desc
limit 1
""",
rs -> rs.next() ? (UUID) rs.getObject("id") : null,
eventSourceId,
tenantKey,
vin
);
}
private UUID findRegistrationBySourceRegistrationEntityId(
String tenantKey,
int eventSourceId,
String sourceRegistrationEntityId,
OffsetDateTime occurredAt
) {
if (sourceRegistrationEntityId == null) {
return null;
}
return jdbcTemplate.query(
compatibleSourcesCte() + """
select r.id
from eventhub.vehicle_registration r
where r.tenant_key = ?
and r.event_source_id in (select id from compatible_sources)
and r.source_registration_entity_id = ?
order by r.updated_at desc
limit 1
""",
rs -> rs.next() ? (UUID) rs.getObject("id") : null,
eventSourceId,
tenantKey,
sourceRegistrationEntityId
);
}
private UUID findRegistrationByPlate(
String tenantKey,
int eventSourceId,
String nation,
String registrationNumber,
OffsetDateTime occurredAt
) {
if (nation == null || registrationNumber == null) {
return null;
}
return jdbcTemplate.query(
compatibleSourcesCte() + """
select r.id
from eventhub.vehicle_registration r
where r.tenant_key = ?
and r.event_source_id in (select id from compatible_sources)
and r.nation = ?
and r.registration_number = ?
order by r.updated_at desc
limit 1
""",
rs -> rs.next() ? (UUID) rs.getObject("id") : null,
eventSourceId,
tenantKey,
nation,
registrationNumber
);
}
private AssignedVehicleReference resolveAssignedVehicleReference(UUID registrationId, OffsetDateTime occurredAt) {
return jdbcTemplate.query(
"""
select vehicle_id, valid_from, valid_to
from eventhub.vehicle_registration_assignment
where vehicle_registration_id = ?
and (? is null or valid_from is null or valid_from <= ?)
and (? is null or valid_to is null or ? < valid_to)
order by valid_from desc nulls last, updated_at desc
limit 1
""",
rs -> rs.next()
? new AssignedVehicleReference(
(UUID) rs.getObject("vehicle_id"),
rs.getObject("valid_from", OffsetDateTime.class),
rs.getObject("valid_to", OffsetDateTime.class)
)
: null,
registrationId,
occurredAt,
occurredAt,
occurredAt,
occurredAt
);
}
private UUID createVehicle(
String tenantKey,
int eventSourceId,
String sourceVehicleEntityId,
String vin
) {
UUID vehicleId = UUID.randomUUID();
jdbcTemplate.update(
"""
insert into eventhub.vehicle(id, tenant_key, event_source_id, source_vehicle_entity_id, vin, updated_at)
values (?, ?, ?, ?, ?, now())
""",
vehicleId,
tenantKey,
eventSourceId,
sourceVehicleEntityId,
vin
);
return vehicleId;
}
private UUID createRegistration(
String tenantKey,
int eventSourceId,
String sourceRegistrationEntityId,
String nation,
String registrationNumber,
OffsetDateTime sourceUpdatedAt,
Map<String, Object> payload
) {
if (nation == null || registrationNumber == null) {
return null;
}
UUID registrationId = UUID.randomUUID();
jdbcTemplate.update(
"""
insert into eventhub.vehicle_registration(
id, tenant_key, event_source_id, source_registration_entity_id, nation, registration_number,
source_updated_at, payload, updated_at
) values (?, ?, ?, ?, ?, ?, ?, ?::jsonb, now())
""",
registrationId,
tenantKey,
eventSourceId,
sourceRegistrationEntityId,
nation,
registrationNumber,
sourceUpdatedAt,
toJson(payload)
);
return registrationId;
}
private void touchVehicle(UUID vehicleId, String sourceVehicleEntityId, String vin) {
if (sourceVehicleEntityId == null && vin == null) {
return;
}
jdbcTemplate.update(
"""
update eventhub.vehicle
set source_vehicle_entity_id = coalesce(source_vehicle_entity_id, cast(? as text)),
vin = coalesce(vin, cast(? as text)),
updated_at = now()
where id = ?
and (
(source_vehicle_entity_id is null and cast(? as text) is not null)
or (vin is null and cast(? as text) is not null)
)
""",
sourceVehicleEntityId,
vin,
vehicleId,
sourceVehicleEntityId,
vin
);
}
private int projectVehicleRegistrationAssignments(String tenantKey, int eventSourceId) {
return jdbcTemplate.update(
compatibleSourcesCte() + """
insert into eventhub.vehicle_registration_assignment(
id, tenant_key, event_source_id, vehicle_registration_id, vehicle_id,
valid_from, valid_to, source_updated_at, payload, updated_at
)
select gen_random_uuid(),
relation.tenant_key,
relation.event_source_id,
registration.id,
vehicle.id,
relation.valid_from,
relation.valid_to,
relation.source_updated_at,
jsonb_build_object(
'source', 'master-data',
'sourceRelationId', relation.id,
'relationKey', relation.relation_key
),
now()
from eventhub.source_master_relation relation
join eventhub.vehicle_registration registration on registration.tenant_key = relation.tenant_key
and registration.event_source_id = relation.event_source_id
and registration.source_registration_entity_id = relation.from_source_entity_id
join eventhub.vehicle vehicle on vehicle.tenant_key = relation.tenant_key
and vehicle.event_source_id = relation.event_source_id
and vehicle.source_vehicle_entity_id = relation.to_source_entity_id
where relation.tenant_key = ?
and relation.event_source_id in (select id from compatible_sources)
and relation.relation_type = 'VEHICLE_REGISTRATION_VEHICLE'
and relation.from_entity_type = 'VEHICLE_REGISTRATION'
and relation.to_entity_type = 'VEHICLE'
and not exists (
select 1
from eventhub.vehicle_registration_assignment existing
where existing.vehicle_registration_id = registration.id
and existing.vehicle_id = vehicle.id
and existing.valid_from is not distinct from relation.valid_from
and existing.valid_to is not distinct from relation.valid_to
)
""",
eventSourceId,
tenantKey
);
}
private void touchRegistration(UUID registrationId, String sourceRegistrationEntityId, String nation, String registrationNumber) {
if (sourceRegistrationEntityId == null && nation == null && registrationNumber == null) {
return;
}
jdbcTemplate.update(
"""
update eventhub.vehicle_registration
set source_registration_entity_id = coalesce(source_registration_entity_id, cast(? as text)),
nation = coalesce(cast(? as text), nation),
registration_number = coalesce(cast(? as text), registration_number),
updated_at = now()
where id = ?
and (
(source_registration_entity_id is null and cast(? as text) is not null)
or (nation is null and cast(? as text) is not null)
or (registration_number is null and cast(? as text) is not null)
)
""",
sourceRegistrationEntityId,
nation,
registrationNumber,
registrationId,
sourceRegistrationEntityId,
nation,
registrationNumber
);
}
private void updateRegistrationFromMasterData(
UUID registrationId,
String sourceRegistrationEntityId,
String nation,
String registrationNumber,
OffsetDateTime sourceUpdatedAt
) {
jdbcTemplate.update(
"""
update eventhub.vehicle_registration
set source_registration_entity_id = coalesce(source_registration_entity_id, ?),
nation = coalesce(?, nation),
registration_number = coalesce(?, registration_number),
source_updated_at = ?,
updated_at = now()
where id = ?
""",
sourceRegistrationEntityId,
nation,
registrationNumber,
sourceUpdatedAt,
registrationId
);
}
private String compatibleSourcesCte() {
return """
with source_context as (
select tenant_key, provider_key, source_instance_key, coalesce(tenant_provider_setting_key, '') as tenant_provider_setting_key
from eventhub.event_source
where id = ?
),
compatible_sources as (
select es.id
from eventhub.event_source es
join source_context ctx on ctx.tenant_key = es.tenant_key
and ctx.provider_key = es.provider_key
and ctx.source_instance_key = es.source_instance_key
and ctx.tenant_provider_setting_key = coalesce(es.tenant_provider_setting_key, '')
)
""";
}
private String toJson(Map<String, Object> value) {
try {
return objectMapper.writeValueAsString(value == null ? Map.of() : value);
} catch (JsonProcessingException e) {
throw new IllegalArgumentException("Cannot serialize vehicle identity payload", e);
}
}
private String normalizeRequired(String value, String fieldName) {
String normalized = normalizeNullable(value);
if (normalized == null) {
throw new IllegalArgumentException(fieldName + " must not be blank");
}
return normalized;
}
private String normalizeNullable(String value) {
if (value == null) {
return null;
}
String trimmed = value.trim();
return trimmed.isEmpty() ? null : trimmed;
}
public record ResolvedVehicleReference(UUID vehicleId, UUID vehicleRegistrationId) {
public static ResolvedVehicleReference empty() {
return new ResolvedVehicleReference(null, null);
}
}
public record ResolvedVehicleReferenceResolution(
ResolvedVehicleReference reference,
boolean resolvedFromAssignment,
OffsetDateTime assignmentValidFrom,
OffsetDateTime assignmentValidTo
) {
public static ResolvedVehicleReferenceResolution empty() {
return new ResolvedVehicleReferenceResolution(ResolvedVehicleReference.empty(), false, null, null);
}
}
private record AssignedVehicleReference(
UUID vehicleId,
OffsetDateTime validFrom,
OffsetDateTime validTo
) {
}
}

View File

@ -68,13 +68,20 @@ public class EventAcquisitionRecordKeyService {
if (event.vehicleRef() == null) {
return "";
}
if (event.vehicleRef().vehicleRegistration() != null && event.vehicleRef().vehicleRegistration().hasValue()) {
return "VRN:" + event.vehicleRef().vehicleRegistration().stableKey();
}
if (event.vehicleRef().vin() != null && !event.vehicleRef().vin().isBlank()) {
boolean hasVin = event.vehicleRef().vin() != null && !event.vehicleRef().vin().isBlank();
boolean hasRegistration = event.vehicleRef().vehicleRegistration() != null
&& event.vehicleRef().vehicleRegistration().hasValue();
if (hasVin) {
return "VIN:" + event.vehicleRef().vin();
}
return "SOURCE_VEHICLE:" + nullToEmpty(event.vehicleRef().sourceEntityId());
if (hasRegistration) {
return "VRN:" + event.vehicleRef().vehicleRegistration().stableKey();
}
if (event.vehicleRef().sourceVehicleEntityId() != null) {
return "SOURCE_VEHICLE:" + event.vehicleRef().sourceVehicleEntityId();
}
return "SOURCE_REGISTRATION:" + nullToEmpty(event.vehicleRef().sourceRegistrationEntityId());
}
private String normalizeTime(OffsetDateTime value) {

View File

@ -2,6 +2,7 @@ package at.procon.eventhub.service;
import at.procon.eventhub.dto.CardSlot;
import at.procon.eventhub.dto.CardStatus;
import at.procon.eventhub.dto.DriverCardRefDto;
import at.procon.eventhub.dto.DrivingStatus;
import at.procon.eventhub.dto.EventDetailsDto;
import com.fasterxml.jackson.databind.JsonNode;
@ -29,9 +30,17 @@ public class EventDetailsFactory {
}
public EventDetailsDto driverCard(CardSlot cardSlot, CardStatus cardStatus) {
return driverCard(cardSlot, cardStatus, null);
}
public EventDetailsDto driverCard(CardSlot cardSlot, CardStatus cardStatus, DriverCardRefDto driverCard) {
Map<String, Object> attributes = new LinkedHashMap<>();
put(attributes, "cardSlot", cardSlot);
put(attributes, "cardStatus", cardStatus);
if (driverCard != null && driverCard.hasValue()) {
put(attributes, "cardNation", driverCard.nation());
put(attributes, "cardNumber", driverCard.number());
}
return new EventDetailsDto("DRIVER_CARD", objectMapper.valueToTree(attributes));
}
@ -41,6 +50,13 @@ public class EventDetailsFactory {
return new EventDetailsDto("POSITION", objectMapper.valueToTree(attributes));
}
public EventDetailsDto place(String country, String region) {
Map<String, Object> attributes = new LinkedHashMap<>();
put(attributes, "country", country);
put(attributes, "region", region);
return new EventDetailsDto("PLACE", objectMapper.valueToTree(attributes));
}
public EventDetailsDto borderCrossing(String countryFrom, String countryTo) {
Map<String, Object> attributes = new LinkedHashMap<>();
put(attributes, "countryFrom", countryFrom);
@ -54,12 +70,21 @@ public class EventDetailsFactory {
return new EventDetailsDto("LOAD_UNLOAD", objectMapper.valueToTree(attributes));
}
public EventDetailsDto specificCondition() {
return new EventDetailsDto("SPECIFIC_CONDITION", objectMapper.createObjectNode());
}
public EventDetailsDto speeding(BigDecimal speedKmh, BigDecimal permittedSpeedKmh) {
return speeding(null, speedKmh, permittedSpeedKmh);
}
public EventDetailsDto speeding(BigDecimal avgSpeedKmh, BigDecimal maxSpeedKmh, BigDecimal permittedSpeedKmh) {
Map<String, Object> attributes = new LinkedHashMap<>();
put(attributes, "speedKmh", speedKmh);
put(attributes, "avgSpeedKmh", avgSpeedKmh);
put(attributes, "maxSpeedKmh", maxSpeedKmh);
put(attributes, "permittedSpeedKmh", permittedSpeedKmh);
if (speedKmh != null && permittedSpeedKmh != null) {
put(attributes, "overspeedKmh", speedKmh.subtract(permittedSpeedKmh));
if (maxSpeedKmh != null && permittedSpeedKmh != null) {
put(attributes, "overspeedKmh", maxSpeedKmh.subtract(permittedSpeedKmh));
}
return new EventDetailsDto("SPEEDING", objectMapper.valueToTree(attributes));
}

View File

@ -74,9 +74,18 @@ public class EventHubEventValidator {
if (event.eventDomain() == EventDomain.BORDER_CROSSING && !"BORDER_CROSSING".equals(detailType)) {
throw new IllegalArgumentException("BORDER_CROSSING events must use eventDetails.type=BORDER_CROSSING");
}
if (event.eventDomain() == EventDomain.POSITION && !"POSITION".equals(detailType)) {
throw new IllegalArgumentException("POSITION events must use eventDetails.type=POSITION");
}
if (event.eventDomain() == EventDomain.PLACE && !"PLACE".equals(detailType)) {
throw new IllegalArgumentException("PLACE events must use eventDetails.type=PLACE");
}
if (event.eventDomain() == EventDomain.LOAD_UNLOAD && !"LOAD_UNLOAD".equals(detailType)) {
throw new IllegalArgumentException("LOAD_UNLOAD events must use eventDetails.type=LOAD_UNLOAD");
}
if (event.eventDomain() == EventDomain.SPECIFIC_CONDITION && !"SPECIFIC_CONDITION".equals(detailType)) {
throw new IllegalArgumentException("SPECIFIC_CONDITION events must use eventDetails.type=SPECIFIC_CONDITION");
}
if (event.eventDomain() == EventDomain.SPEEDING && !"SPEEDING".equals(detailType)) {
throw new IllegalArgumentException("SPEEDING events must use eventDetails.type=SPEEDING");
}

View File

@ -8,12 +8,15 @@ import at.procon.eventhub.dto.EventSourceDto;
import at.procon.eventhub.persistence.DataPackageRepository;
import at.procon.eventhub.persistence.EventRepository;
import at.procon.eventhub.persistence.EventSourceRepository;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.UUID;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import org.springframework.transaction.PlatformTransactionManager;
import org.springframework.transaction.support.TransactionTemplate;
@Service
public class EventHubIngestionService {
@ -25,26 +28,29 @@ public class EventHubIngestionService {
private final EventRepository eventRepository;
private final EventHubEventValidator validator;
private final EventHubEventSorter sorter;
private final TransactionTemplate transactionTemplate;
public EventHubIngestionService(
EventSourceRepository eventSourceRepository,
DataPackageRepository dataPackageRepository,
EventRepository eventRepository,
EventHubEventValidator validator,
EventHubEventSorter sorter
EventHubEventSorter sorter,
PlatformTransactionManager transactionManager
) {
this.eventSourceRepository = eventSourceRepository;
this.dataPackageRepository = dataPackageRepository;
this.eventRepository = eventRepository;
this.validator = validator;
this.sorter = sorter;
this.transactionTemplate = new TransactionTemplate(transactionManager);
}
@Transactional
public EventHubPackageResult ingest(EventHubEventBatchDto batch) {
if (batch == null || batch.events().isEmpty()) {
return new EventHubPackageResult(null, batch == null ? null : batch.packageKey(), 0, 0);
}
long startedAtNanos = System.nanoTime();
EventHubPackageRequest packageInfo = batch.packageInfo();
if (packageInfo == null) {
@ -55,6 +61,7 @@ public class EventHubIngestionService {
}
EventSourceDto eventSource = packageInfo.eventSource();
String tenantKey = packageInfo.tenantKey();
int eventSourceId = eventSourceRepository.resolveSourceId(packageInfo.tenantKey(), eventSource);
List<EventHubEventDto> sortedEvents = sorter.sort(batch.events());
sortedEvents.forEach(validator::validate);
@ -70,14 +77,37 @@ public class EventHubIngestionService {
);
try {
int insertedCount = eventRepository.batchInsert(packageId, eventSourceId, sortedEvents);
Integer insertedCount = transactionTemplate.execute(status ->
eventRepository.batchInsert(packageId, tenantKey, eventSourceId, sortedEvents)
);
if (insertedCount == null) {
throw new IllegalStateException("EventHub batch insert returned no result for package " + batch.packageKey());
}
dataPackageRepository.markImported(packageId, insertedCount);
log.info("Imported EventHub acquisition package packageId={} packageKey={} source={} receivedCount={} insertedCount={}",
packageId, batch.packageKey(), eventSource.stableKey(), sortedEvents.size(), insertedCount);
long elapsedMs = Math.max(1L, (System.nanoTime() - startedAtNanos) / 1_000_000L);
long receivedPerSecond = Math.max(1L, Math.round(sortedEvents.size() * 1000.0 / elapsedMs));
log.info("Imported EventHub acquisition package packageId={} packageKey={} source={} receivedCount={} insertedCount={} elapsedMs={} receivedPerSecond={} byType={}",
packageId, batch.packageKey(), eventSource.stableKey(), sortedEvents.size(), insertedCount, elapsedMs, receivedPerSecond, eventTypeCounts(sortedEvents));
return new EventHubPackageResult(packageId, batch.packageKey(), sortedEvents.size(), insertedCount);
} catch (RuntimeException ex) {
dataPackageRepository.markFailed(packageId, ex.getMessage());
try {
dataPackageRepository.markFailed(packageId, ex.getMessage());
} catch (RuntimeException markFailedEx) {
ex.addSuppressed(markFailedEx);
log.warn("Failed to mark EventHub acquisition package as failed packageId={} packageKey={}",
packageId, batch.packageKey(), markFailedEx);
}
throw ex;
}
}
private Map<String, Integer> eventTypeCounts(List<EventHubEventDto> events) {
Map<String, Integer> counts = new LinkedHashMap<>();
for (EventHubEventDto event : events) {
String domain = event.eventDomain() == null ? "UNKNOWN_DOMAIN" : event.eventDomain().name();
String type = event.eventType() == null ? "UNKNOWN_EVENT" : event.eventType().name();
counts.merge(domain + "/" + type, 1, Integer::sum);
}
return counts;
}
}

View File

@ -0,0 +1,22 @@
package at.procon.eventhub.tachograph.api;
import at.procon.eventhub.tachograph.service.UnsupportedTachographExtractionException;
import java.time.OffsetDateTime;
import java.util.Map;
import org.springframework.http.HttpStatus;
import org.springframework.http.ResponseEntity;
import org.springframework.web.bind.annotation.ExceptionHandler;
import org.springframework.web.bind.annotation.RestControllerAdvice;
@RestControllerAdvice(basePackageClasses = TachographIngestionController.class)
public class TachographApiExceptionHandler {
@ExceptionHandler(UnsupportedTachographExtractionException.class)
public ResponseEntity<Map<String, Object>> handleUnsupportedExtraction(UnsupportedTachographExtractionException ex) {
return ResponseEntity.status(HttpStatus.BAD_REQUEST).body(Map.of(
"error", "UNSUPPORTED_TACHOGRAPH_EXTRACTION",
"message", ex.getMessage(),
"timestamp", OffsetDateTime.now().toString()
));
}
}

View File

@ -3,6 +3,7 @@ package at.procon.eventhub.tachograph.api;
import at.procon.eventhub.dto.AcquisitionStrategy;
import at.procon.eventhub.dto.ImportMode;
import at.procon.eventhub.dto.SchedulerTriggerMode;
import at.procon.eventhub.importing.masterdata.MasterDataRefreshResult;
import at.procon.eventhub.tachograph.dto.ConfiguredTachographImportPlanDto;
import at.procon.eventhub.tachograph.dto.TachographImportRequest;
import at.procon.eventhub.tachograph.dto.TachographImportRunResultDto;
@ -11,6 +12,7 @@ import at.procon.eventhub.tachograph.dto.source.TachographActivityDto;
import at.procon.eventhub.tachograph.service.TachographConfiguredImportPlanService;
import at.procon.eventhub.tachograph.service.TachographImportExecutionService;
import at.procon.eventhub.tachograph.service.TachographImportPlanService;
import at.procon.eventhub.tachograph.service.TachographMasterDataRefreshService;
import jakarta.validation.Valid;
import java.time.OffsetDateTime;
import java.util.List;
@ -33,17 +35,20 @@ public class TachographIngestionController {
private final TachographImportPlanService tachographImportPlanService;
private final TachographConfiguredImportPlanService configuredImportPlanService;
private final TachographImportExecutionService tachographImportExecutionService;
private final TachographMasterDataRefreshService masterDataRefreshService;
public TachographIngestionController(
ProducerTemplate producerTemplate,
TachographImportPlanService tachographImportPlanService,
TachographConfiguredImportPlanService configuredImportPlanService,
TachographImportExecutionService tachographImportExecutionService
TachographImportExecutionService tachographImportExecutionService,
TachographMasterDataRefreshService masterDataRefreshService
) {
this.producerTemplate = producerTemplate;
this.tachographImportPlanService = tachographImportPlanService;
this.configuredImportPlanService = configuredImportPlanService;
this.tachographImportExecutionService = tachographImportExecutionService;
this.masterDataRefreshService = masterDataRefreshService;
}
@PostMapping("/activities")
@ -68,6 +73,13 @@ public class TachographIngestionController {
return ResponseEntity.accepted().body(result);
}
@PostMapping("/master-data/refresh")
public ResponseEntity<MasterDataRefreshResult> refreshTachographMasterData(
@Valid @RequestBody TachographImportRequest request
) {
return ResponseEntity.ok(masterDataRefreshService.refresh(request));
}
@GetMapping("/imports/configured-plans")
public ResponseEntity<List<ConfiguredTachographImportPlanDto>> listConfiguredTachographPlans() {
return ResponseEntity.ok(configuredImportPlanService.listPlans());
@ -99,6 +111,16 @@ public class TachographIngestionController {
));
}
@PostMapping("/imports/configured-plans/{planKey}/master-data/refresh")
public ResponseEntity<MasterDataRefreshResult> refreshConfiguredTachographMasterData(
@PathVariable String planKey,
@RequestParam(required = false) ImportMode mode,
@RequestParam(required = false) AcquisitionStrategy strategy
) {
TachographImportRequest request = configuredImportPlanService.createRequest(planKey, mode, strategy);
return ResponseEntity.ok(masterDataRefreshService.refresh(request));
}
private ResponseEntity<Map<String, Object>> accepted(int count, String route) {
return ResponseEntity.accepted().body(Map.of(
"accepted", count,

View File

@ -1,7 +1,7 @@
package at.procon.eventhub.tachograph.config;
import at.procon.eventhub.config.EventHubProperties;
import javax.sql.DataSource;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression;
import org.springframework.context.annotation.Bean;
@ -13,15 +13,24 @@ import org.springframework.jdbc.datasource.DriverManagerDataSource;
@ConditionalOnExpression("'${eventhub.tachograph.datasource.jdbc-url:}' != ''")
public class TachographDataSourceConfig {
private static final String SQL_SERVER_DRIVER_CLASS = "com.microsoft.sqlserver.jdbc.SQLServerDriver";
private static final String SQL_SERVER_JDBC_PREFIX = "jdbc:sqlserver://";
@Bean
public DataSource tachographDataSource(EventHubProperties properties) {
EventHubProperties.TachographDataSource config = properties.getTachograph().getDatasource();
@ConfigurationProperties(prefix = "eventhub.tachograph.datasource")
public TachographDataSourceProperties tachographDataSourceProperties() {
return new TachographDataSourceProperties();
}
@Bean(defaultCandidate = false)
public DataSource tachographDataSource(TachographDataSourceProperties config) {
DriverManagerDataSource dataSource = new DriverManagerDataSource();
dataSource.setUrl(config.getJdbcUrl());
dataSource.setUrl(validateJdbcUrl(config));
dataSource.setUsername(config.getUsername());
dataSource.setPassword(config.getPassword());
if (config.getDriverClassName() != null && !config.getDriverClassName().isBlank()) {
dataSource.setDriverClassName(config.getDriverClassName());
String driverClassName = trimToNull(config.getDriverClassName());
if (driverClassName != null) {
dataSource.setDriverClassName(driverClassName);
}
return dataSource;
}
@ -32,4 +41,75 @@ public class TachographDataSourceConfig {
) {
return new NamedParameterJdbcTemplate(tachographDataSource);
}
private String validateJdbcUrl(TachographDataSourceProperties config) {
String jdbcUrl = trimToNull(config.getJdbcUrl());
if (jdbcUrl == null) {
throw new IllegalStateException("eventhub.tachograph.datasource.jdbc-url must not be empty");
}
String driverClassName = trimToNull(config.getDriverClassName());
if (SQL_SERVER_DRIVER_CLASS.equals(driverClassName) && !jdbcUrl.startsWith(SQL_SERVER_JDBC_PREFIX)) {
if (jdbcUrl.startsWith("jdbc:")) {
String suggestedUrl = SQL_SERVER_JDBC_PREFIX + jdbcUrl.substring("jdbc:".length());
throw new IllegalStateException(
"Invalid SQL Server JDBC URL '" + jdbcUrl + "'. Expected prefix '"
+ SQL_SERVER_JDBC_PREFIX + "'. Example: " + suggestedUrl
);
}
throw new IllegalStateException(
"Invalid SQL Server JDBC URL '" + jdbcUrl + "'. Expected prefix '"
+ SQL_SERVER_JDBC_PREFIX + "'"
);
}
return jdbcUrl;
}
private String trimToNull(String value) {
if (value == null) {
return null;
}
String trimmedValue = value.trim();
return trimmedValue.isEmpty() ? null : trimmedValue;
}
public static class TachographDataSourceProperties {
private String jdbcUrl;
private String username;
private String password;
private String driverClassName;
public String getJdbcUrl() {
return jdbcUrl;
}
public void setJdbcUrl(String jdbcUrl) {
this.jdbcUrl = jdbcUrl;
}
public String getUsername() {
return username;
}
public void setUsername(String username) {
this.username = username;
}
public String getPassword() {
return password;
}
public void setPassword(String password) {
this.password = password;
}
public String getDriverClassName() {
return driverClassName;
}
public void setDriverClassName(String driverClassName) {
this.driverClassName = driverClassName;
}
}
}

View File

@ -2,6 +2,7 @@ package at.procon.eventhub.tachograph.dto;
import at.procon.eventhub.importing.ExtractionBatchResult;
import java.time.OffsetDateTime;
import java.util.Map;
import java.util.UUID;
public record TachographExtractionBatchResultDto(
@ -16,6 +17,11 @@ public record TachographExtractionBatchResultDto(
OffsetDateTime lastSourcePackageImportedAt,
String lastSourcePackageId,
OffsetDateTime lastSourceRowUpdatedAt,
OffsetDateTime lastOccurredTo
OffsetDateTime lastOccurredTo,
Map<String, Integer> eventTypeCounts
) implements ExtractionBatchResult {
public TachographExtractionBatchResultDto {
eventTypeCounts = eventTypeCounts == null ? Map.of() : Map.copyOf(eventTypeCounts);
}
}

View File

@ -19,6 +19,7 @@ import at.procon.eventhub.tachograph.dto.TachographImportRequest;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Timestamp;
import java.time.LocalDateTime;
import java.time.OffsetDateTime;
import java.time.ZoneOffset;
import java.util.LinkedHashMap;
@ -40,6 +41,9 @@ abstract class AbstractTachographActivityRowMapper implements ExtractionRowMappe
SourcePackageRefDto sourcePackageRef = sourcePackageRef(rs);
DriverRefDto driverRef = driverRef(rs);
VehicleRefDto vehicleRef = vehicleRef(rs);
CardSlot cardSlot = cardSlot(rs);
CardStatus cardStatus = cardStatus(rs);
DrivingStatus drivingStatus = drivingStatus(rs);
String externalSourceEventId = string(rs, "external_source_event_id");
if (externalSourceEventId == null) {
@ -59,15 +63,21 @@ abstract class AbstractTachographActivityRowMapper implements ExtractionRowMappe
lifecycle(rs),
longValue(rs, "odometer_m"),
null,
detailsFactory.driverActivity(cardSlot(rs), cardStatus(rs), drivingStatus(rs)),
detailsFactory.driverActivity(cardSlot, cardStatus, drivingStatus),
sourcePackageRef != null && sourcePackageRef.hasAnyReference() ? sourcePackageRef : null,
detailsFactory.payloadFromMap(payload(rs, context)),
false,
isManualEntry(cardStatus, drivingStatus),
context.packageInfo()
);
}
protected abstract Map<String, Object> sourceSpecificPayload(ResultSet rs) throws SQLException;
protected Map<String, Object> sourceSpecificPayload(ResultSet rs) throws SQLException {
return Map.of();
}
private boolean isManualEntry(CardStatus cardStatus, DrivingStatus drivingStatus) {
return cardStatus == CardStatus.NOT_INSERTED && drivingStatus == DrivingStatus.KNOWN;
}
private DriverRefDto driverRef(ResultSet rs) throws SQLException {
DriverCardRefDto driverCard = null;
@ -85,7 +95,12 @@ abstract class AbstractTachographActivityRowMapper implements ExtractionRowMappe
if (registrationNumber != null) {
registration = new VehicleRegistrationRefDto(string(rs, "vehicle_registration_nation"), registrationNumber);
}
VehicleRefDto vehicleRef = new VehicleRefDto(string(rs, "vehicle_source_entity_id"), string(rs, "vehicle_vin"), registration);
VehicleRefDto vehicleRef = new VehicleRefDto(
string(rs, "vehicle_source_entity_id"),
string(rs, "vehicle_vin"),
string(rs, "vehicle_registration_source_entity_id"),
registration
);
return vehicleRef.hasAnyReference() ? vehicleRef : null;
}
@ -102,24 +117,7 @@ abstract class AbstractTachographActivityRowMapper implements ExtractionRowMappe
private Map<String, Object> payload(ResultSet rs, ExtractionContext<TachographImportRequest> context) throws SQLException {
Map<String, Object> raw = new LinkedHashMap<>();
raw.put("extractionCode", context.planItem().extractionCode());
raw.put("sourceKind", context.planItem().sourceKind());
raw.put("sourceTables", context.planItem().sourceTables());
put(raw, "sourceRowId", string(rs, "source_row_id"));
put(raw, "activityCode", object(rs, "activity_code"));
put(raw, "activityText", string(rs, "activity_text"));
put(raw, "eventType", string(rs, "event_type"));
put(raw, "lifecycle", string(rs, "lifecycle"));
put(raw, "cardSlot", object(rs, "card_slot"));
put(raw, "cardStatus", object(rs, "card_status"));
put(raw, "drivingStatus", object(rs, "driving_status"));
put(raw, "driverSourceEntityId", string(rs, "driver_source_entity_id"));
put(raw, "driverCardNation", string(rs, "driver_card_nation"));
put(raw, "driverCardNumber", string(rs, "driver_card_number"));
put(raw, "vehicleSourceEntityId", string(rs, "vehicle_source_entity_id"));
put(raw, "vehicleVin", string(rs, "vehicle_vin"));
put(raw, "vehicleRegistrationNation", string(rs, "vehicle_registration_nation"));
put(raw, "vehicleRegistrationNumber", string(rs, "vehicle_registration_number"));
raw.putAll(sourceSpecificPayload(rs));
return Map.of("raw", raw);
}
@ -160,15 +158,20 @@ abstract class AbstractTachographActivityRowMapper implements ExtractionRowMappe
return null;
}
if (value instanceof OffsetDateTime offsetDateTime) {
return offsetDateTime;
return offsetDateTime.withOffsetSameInstant(ZoneOffset.UTC);
}
if (value instanceof Timestamp timestamp) {
return timestamp.toInstant().atOffset(ZoneOffset.UTC);
return timestamp.toLocalDateTime().atOffset(ZoneOffset.UTC);
}
if (value instanceof java.time.LocalDateTime localDateTime) {
return localDateTime.atOffset(ZoneOffset.UTC);
}
return OffsetDateTime.parse(value.toString());
String text = value.toString();
try {
return OffsetDateTime.parse(text).withOffsetSameInstant(ZoneOffset.UTC);
} catch (RuntimeException ignored) {
return LocalDateTime.parse(text).atOffset(ZoneOffset.UTC);
}
}
private Long longValue(ResultSet rs, String column) throws SQLException {

View File

@ -0,0 +1,200 @@
package at.procon.eventhub.tachograph.service;
import at.procon.eventhub.dto.DriverCardRefDto;
import at.procon.eventhub.dto.DriverRefDto;
import at.procon.eventhub.dto.EventDomain;
import at.procon.eventhub.dto.EventHubEventDto;
import at.procon.eventhub.dto.EventLifecycle;
import at.procon.eventhub.dto.EventType;
import at.procon.eventhub.dto.GeoPointDto;
import at.procon.eventhub.dto.SourcePackageRefDto;
import at.procon.eventhub.dto.VehicleRefDto;
import at.procon.eventhub.dto.VehicleRegistrationRefDto;
import at.procon.eventhub.importing.extraction.ExtractionContext;
import at.procon.eventhub.importing.extraction.ExtractionRowMapper;
import at.procon.eventhub.service.EventDetailsFactory;
import at.procon.eventhub.tachograph.dto.TachographImportRequest;
import java.math.BigDecimal;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Timestamp;
import java.time.LocalDateTime;
import java.time.OffsetDateTime;
import java.time.ZoneOffset;
import java.util.LinkedHashMap;
import java.util.Map;
import java.util.UUID;
abstract class AbstractTachographBorderCrossingRowMapper implements ExtractionRowMapper<TachographImportRequest> {
private final EventDetailsFactory detailsFactory;
protected AbstractTachographBorderCrossingRowMapper(EventDetailsFactory detailsFactory) {
this.detailsFactory = detailsFactory;
}
@Override
public EventHubEventDto map(ResultSet rs, int rowNum, ExtractionContext<TachographImportRequest> context) throws SQLException {
OffsetDateTime occurredAt = offsetDateTime(rs, "occurred_at");
SourcePackageRefDto sourcePackageRef = sourcePackageRef(rs);
DriverRefDto driverRef = driverRef(rs);
VehicleRefDto vehicleRef = vehicleRef(rs);
String externalSourceEventId = string(rs, "external_source_event_id");
if (externalSourceEventId == null) {
externalSourceEventId = defaultExternalSourceEventId(context, rowNum, occurredAt, sourcePackageRef, driverRef, vehicleRef);
}
return new EventHubEventDto(
UUID.randomUUID(),
externalSourceEventId,
driverRef,
vehicleRef,
occurredAt,
offsetDateTime(rs, "received_partner_at"),
OffsetDateTime.now(),
EventDomain.BORDER_CROSSING,
EventType.BORDER_OUTBOUND,
EventLifecycle.OUTBOUND,
longValue(rs, "odometer_m"),
position(rs),
detailsFactory.borderCrossing(string(rs, "country_from"), string(rs, "country_to")),
sourcePackageRef != null && sourcePackageRef.hasAnyReference() ? sourcePackageRef : null,
detailsFactory.payloadFromMap(payload(rs)),
false,
context.packageInfo()
);
}
protected Map<String, Object> sourceSpecificPayload(ResultSet rs) throws SQLException {
return Map.of();
}
private DriverRefDto driverRef(ResultSet rs) throws SQLException {
DriverCardRefDto driverCard = null;
String cardNumber = string(rs, "driver_card_number");
if (cardNumber != null) {
driverCard = new DriverCardRefDto(string(rs, "driver_card_nation"), cardNumber);
}
DriverRefDto driverRef = new DriverRefDto(string(rs, "driver_source_entity_id"), driverCard);
return driverRef.hasAnyReference() ? driverRef : null;
}
private VehicleRefDto vehicleRef(ResultSet rs) throws SQLException {
VehicleRegistrationRefDto registration = null;
String registrationNumber = string(rs, "vehicle_registration_number");
if (registrationNumber != null) {
registration = new VehicleRegistrationRefDto(string(rs, "vehicle_registration_nation"), registrationNumber);
}
VehicleRefDto vehicleRef = new VehicleRefDto(
string(rs, "vehicle_source_entity_id"),
string(rs, "vehicle_vin"),
string(rs, "vehicle_registration_source_entity_id"),
registration
);
return vehicleRef.hasAnyReference() ? vehicleRef : null;
}
private SourcePackageRefDto sourcePackageRef(ResultSet rs) throws SQLException {
return new SourcePackageRefDto(
string(rs, "source_package_kind"),
string(rs, "source_package_id"),
string(rs, "source_package_entity_id"),
offsetDateTime(rs, "source_package_period_from"),
offsetDateTime(rs, "source_package_period_to"),
offsetDateTime(rs, "source_package_imported_at")
);
}
private GeoPointDto position(ResultSet rs) throws SQLException {
BigDecimal latitude = decimal(rs, "latitude");
BigDecimal longitude = decimal(rs, "longitude");
return latitude == null || longitude == null ? null : new GeoPointDto(latitude, longitude);
}
private Map<String, Object> payload(ResultSet rs) throws SQLException {
Map<String, Object> raw = new LinkedHashMap<>();
put(raw, "sourceRowId", string(rs, "source_row_id"));
raw.putAll(sourceSpecificPayload(rs));
return Map.of("raw", raw);
}
private String defaultExternalSourceEventId(
ExtractionContext<TachographImportRequest> context,
int rowNum,
OffsetDateTime occurredAt,
SourcePackageRefDto sourcePackageRef,
DriverRefDto driverRef,
VehicleRefDto vehicleRef
) {
String sourcePackageId = sourcePackageRef == null || sourcePackageRef.sourcePackageId() == null
? "NO_SOURCE_PACKAGE"
: sourcePackageRef.sourcePackageId();
String subject = driverRef != null && driverRef.hasAnyReference()
? driverRef.stableKey()
: vehicleRef == null ? "NO_SUBJECT" : vehicleRef.stableKey();
return "TACHOGRAPH:" + context.planItem().extractionCode()
+ ":" + sourcePackageId
+ ":" + occurredAt
+ ":" + subject
+ ":ROW-" + rowNum;
}
protected String string(ResultSet rs, String column) throws SQLException {
String value = rs.getString(column);
return value == null || value.isBlank() ? null : value.trim();
}
protected OffsetDateTime offsetDateTime(ResultSet rs, String column) throws SQLException {
Object value = rs.getObject(column);
if (value == null) {
return null;
}
if (value instanceof OffsetDateTime offsetDateTime) {
return offsetDateTime.withOffsetSameInstant(ZoneOffset.UTC);
}
if (value instanceof Timestamp timestamp) {
return timestamp.toLocalDateTime().atOffset(ZoneOffset.UTC);
}
if (value instanceof LocalDateTime localDateTime) {
return localDateTime.atOffset(ZoneOffset.UTC);
}
String text = value.toString();
try {
return OffsetDateTime.parse(text).withOffsetSameInstant(ZoneOffset.UTC);
} catch (RuntimeException ignored) {
return LocalDateTime.parse(text).atOffset(ZoneOffset.UTC);
}
}
private Long longValue(ResultSet rs, String column) throws SQLException {
Object value = rs.getObject(column);
if (value == null) {
return null;
}
if (value instanceof Number number) {
return number.longValue();
}
return Long.parseLong(value.toString());
}
private BigDecimal decimal(ResultSet rs, String column) throws SQLException {
Object value = rs.getObject(column);
if (value == null) {
return null;
}
if (value instanceof BigDecimal bigDecimal) {
return bigDecimal;
}
if (value instanceof Number number) {
return BigDecimal.valueOf(number.doubleValue());
}
return new BigDecimal(value.toString());
}
protected void put(Map<String, Object> target, String key, Object value) {
if (value != null) {
target.put(key, value);
}
}
}

View File

@ -0,0 +1,246 @@
package at.procon.eventhub.tachograph.service;
import at.procon.eventhub.dto.CardSlot;
import at.procon.eventhub.dto.CardStatus;
import at.procon.eventhub.dto.DriverCardRefDto;
import at.procon.eventhub.dto.DriverRefDto;
import at.procon.eventhub.dto.EventDomain;
import at.procon.eventhub.dto.EventHubEventDto;
import at.procon.eventhub.dto.EventLifecycle;
import at.procon.eventhub.dto.EventType;
import at.procon.eventhub.dto.SourcePackageRefDto;
import at.procon.eventhub.dto.VehicleRefDto;
import at.procon.eventhub.dto.VehicleRegistrationRefDto;
import at.procon.eventhub.importing.extraction.ExtractionContext;
import at.procon.eventhub.importing.extraction.ExtractionRowMapper;
import at.procon.eventhub.service.EventDetailsFactory;
import at.procon.eventhub.tachograph.dto.TachographImportRequest;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Timestamp;
import java.time.LocalDateTime;
import java.time.OffsetDateTime;
import java.time.ZoneOffset;
import java.util.LinkedHashMap;
import java.util.Locale;
import java.util.Map;
import java.util.UUID;
abstract class AbstractTachographCardEventRowMapper implements ExtractionRowMapper<TachographImportRequest> {
private final EventDetailsFactory detailsFactory;
protected AbstractTachographCardEventRowMapper(EventDetailsFactory detailsFactory) {
this.detailsFactory = detailsFactory;
}
@Override
public EventHubEventDto map(ResultSet rs, int rowNum, ExtractionContext<TachographImportRequest> context) throws SQLException {
OffsetDateTime occurredAt = offsetDateTime(rs, "occurred_at");
SourcePackageRefDto sourcePackageRef = sourcePackageRef(rs);
DriverRefDto driverRef = driverRef(rs);
VehicleRefDto vehicleRef = vehicleRef(rs);
DriverCardRefDto driverCard = driverRef == null ? null : driverRef.driverCard();
EventType eventType = eventType(rs);
EventLifecycle lifecycle = lifecycle(rs);
CardStatus cardStatus = cardStatus(lifecycle);
String externalSourceEventId = string(rs, "external_source_event_id");
if (externalSourceEventId == null) {
externalSourceEventId = defaultExternalSourceEventId(context, rowNum, occurredAt, sourcePackageRef, driverRef, vehicleRef, lifecycle);
}
return new EventHubEventDto(
UUID.randomUUID(),
externalSourceEventId,
driverRef,
vehicleRef,
occurredAt,
offsetDateTime(rs, "received_partner_at"),
OffsetDateTime.now(),
EventDomain.DRIVER_CARD,
eventType,
lifecycle,
longValue(rs, "odometer_m"),
null,
detailsFactory.driverCard(cardSlot(rs), cardStatus, driverCard),
sourcePackageRef != null && sourcePackageRef.hasAnyReference() ? sourcePackageRef : null,
detailsFactory.payloadFromMap(payload(rs)),
false,
context.packageInfo()
);
}
protected Map<String, Object> sourceSpecificPayload(ResultSet rs) throws SQLException {
return Map.of();
}
private DriverRefDto driverRef(ResultSet rs) throws SQLException {
DriverCardRefDto driverCard = null;
String cardNumber = string(rs, "driver_card_number");
if (cardNumber != null) {
driverCard = new DriverCardRefDto(string(rs, "driver_card_nation"), cardNumber);
}
DriverRefDto driverRef = new DriverRefDto(string(rs, "driver_source_entity_id"), driverCard);
return driverRef.hasAnyReference() ? driverRef : null;
}
private VehicleRefDto vehicleRef(ResultSet rs) throws SQLException {
VehicleRegistrationRefDto registration = null;
String registrationNumber = string(rs, "vehicle_registration_number");
if (registrationNumber != null) {
registration = new VehicleRegistrationRefDto(string(rs, "vehicle_registration_nation"), registrationNumber);
}
VehicleRefDto vehicleRef = new VehicleRefDto(
string(rs, "vehicle_source_entity_id"),
string(rs, "vehicle_vin"),
string(rs, "vehicle_registration_source_entity_id"),
registration
);
return vehicleRef.hasAnyReference() ? vehicleRef : null;
}
private SourcePackageRefDto sourcePackageRef(ResultSet rs) throws SQLException {
return new SourcePackageRefDto(
string(rs, "source_package_kind"),
string(rs, "source_package_id"),
string(rs, "source_package_entity_id"),
offsetDateTime(rs, "source_package_period_from"),
offsetDateTime(rs, "source_package_period_to"),
offsetDateTime(rs, "source_package_imported_at")
);
}
private Map<String, Object> payload(ResultSet rs) throws SQLException {
Map<String, Object> raw = new LinkedHashMap<>();
put(raw, "sourceRowId", string(rs, "source_row_id"));
raw.putAll(sourceSpecificPayload(rs));
return Map.of("raw", raw);
}
private String defaultExternalSourceEventId(
ExtractionContext<TachographImportRequest> context,
int rowNum,
OffsetDateTime occurredAt,
SourcePackageRefDto sourcePackageRef,
DriverRefDto driverRef,
VehicleRefDto vehicleRef,
EventLifecycle lifecycle
) {
String sourcePackageId = sourcePackageRef == null || sourcePackageRef.sourcePackageId() == null
? "NO_SOURCE_PACKAGE"
: sourcePackageRef.sourcePackageId();
String subject = driverRef != null && driverRef.hasAnyReference()
? driverRef.stableKey()
: vehicleRef == null ? "NO_SUBJECT" : vehicleRef.stableKey();
return "TACHOGRAPH:" + context.planItem().extractionCode()
+ ":" + sourcePackageId
+ ":" + lifecycle
+ ":" + occurredAt
+ ":" + subject
+ ":ROW-" + rowNum;
}
protected String string(ResultSet rs, String column) throws SQLException {
String value = rs.getString(column);
return value == null || value.isBlank() ? null : value.trim();
}
protected Object object(ResultSet rs, String column) throws SQLException {
return rs.getObject(column);
}
protected OffsetDateTime offsetDateTime(ResultSet rs, String column) throws SQLException {
Object value = rs.getObject(column);
if (value == null) {
return null;
}
if (value instanceof OffsetDateTime offsetDateTime) {
return offsetDateTime.withOffsetSameInstant(ZoneOffset.UTC);
}
if (value instanceof Timestamp timestamp) {
return timestamp.toLocalDateTime().atOffset(ZoneOffset.UTC);
}
if (value instanceof LocalDateTime localDateTime) {
return localDateTime.atOffset(ZoneOffset.UTC);
}
String text = value.toString();
try {
return OffsetDateTime.parse(text).withOffsetSameInstant(ZoneOffset.UTC);
} catch (RuntimeException ignored) {
return LocalDateTime.parse(text).atOffset(ZoneOffset.UTC);
}
}
private Long longValue(ResultSet rs, String column) throws SQLException {
Object value = rs.getObject(column);
if (value == null) {
return null;
}
if (value instanceof Number number) {
return number.longValue();
}
return Long.parseLong(value.toString());
}
private CardSlot cardSlot(ResultSet rs) throws SQLException {
Object value = object(rs, "card_slot");
if (value instanceof Number number) {
return switch (number.intValue()) {
case 0 -> CardSlot.DRIVER;
case 1 -> CardSlot.CO_DRIVER;
default -> null;
};
}
String text = value == null ? null : value.toString();
Integer parsed = parseInteger(text);
if (parsed != null) {
return parsed == 0 ? CardSlot.DRIVER : parsed == 1 ? CardSlot.CO_DRIVER : null;
}
return parseEnum(CardSlot.class, text, null);
}
private EventType eventType(ResultSet rs) throws SQLException {
return parseEnum(EventType.class, string(rs, "event_type"), EventType.UNKNOWN_EVENT);
}
private EventLifecycle lifecycle(ResultSet rs) throws SQLException {
return parseEnum(EventLifecycle.class, string(rs, "lifecycle"), EventLifecycle.SNAPSHOT);
}
private CardStatus cardStatus(EventLifecycle lifecycle) {
return lifecycle == EventLifecycle.INSERT ? CardStatus.INSERTED : lifecycle == EventLifecycle.WITHDRAW ? CardStatus.NOT_INSERTED : null;
}
private <T extends Enum<T>> T parseEnum(Class<T> type, String value, T fallback) {
if (value == null) {
return fallback;
}
String normalized = value.trim().toUpperCase(Locale.ROOT).replace('-', '_').replace(' ', '_');
if ("CODRIVER".equals(normalized)) {
normalized = "CO_DRIVER";
}
try {
return Enum.valueOf(type, normalized);
} catch (IllegalArgumentException ignored) {
return fallback;
}
}
private Integer parseInteger(String value) {
if (value == null || value.isBlank()) {
return null;
}
try {
return Integer.parseInt(value.trim());
} catch (NumberFormatException ignored) {
return null;
}
}
protected void put(Map<String, Object> target, String key, Object value) {
if (value != null) {
target.put(key, value);
}
}
}

View File

@ -0,0 +1,220 @@
package at.procon.eventhub.tachograph.service;
import at.procon.eventhub.dto.DriverCardRefDto;
import at.procon.eventhub.dto.DriverRefDto;
import at.procon.eventhub.dto.EventDomain;
import at.procon.eventhub.dto.EventHubEventDto;
import at.procon.eventhub.dto.EventLifecycle;
import at.procon.eventhub.dto.EventType;
import at.procon.eventhub.dto.GeoPointDto;
import at.procon.eventhub.dto.SourcePackageRefDto;
import at.procon.eventhub.dto.VehicleRefDto;
import at.procon.eventhub.dto.VehicleRegistrationRefDto;
import at.procon.eventhub.importing.extraction.ExtractionContext;
import at.procon.eventhub.importing.extraction.ExtractionRowMapper;
import at.procon.eventhub.service.EventDetailsFactory;
import at.procon.eventhub.tachograph.dto.TachographImportRequest;
import java.math.BigDecimal;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Timestamp;
import java.time.LocalDateTime;
import java.time.OffsetDateTime;
import java.time.ZoneOffset;
import java.util.LinkedHashMap;
import java.util.Locale;
import java.util.Map;
import java.util.UUID;
abstract class AbstractTachographLoadUnloadRowMapper implements ExtractionRowMapper<TachographImportRequest> {
private final EventDetailsFactory detailsFactory;
protected AbstractTachographLoadUnloadRowMapper(EventDetailsFactory detailsFactory) {
this.detailsFactory = detailsFactory;
}
@Override
public EventHubEventDto map(ResultSet rs, int rowNum, ExtractionContext<TachographImportRequest> context) throws SQLException {
OffsetDateTime occurredAt = offsetDateTime(rs, "occurred_at");
SourcePackageRefDto sourcePackageRef = sourcePackageRef(rs);
DriverRefDto driverRef = driverRef(rs);
VehicleRefDto vehicleRef = vehicleRef(rs);
EventType eventType = eventType(rs);
String externalSourceEventId = string(rs, "external_source_event_id");
if (externalSourceEventId == null) {
externalSourceEventId = defaultExternalSourceEventId(context, rowNum, occurredAt, sourcePackageRef, driverRef, vehicleRef, eventType);
}
return new EventHubEventDto(
UUID.randomUUID(),
externalSourceEventId,
driverRef,
vehicleRef,
occurredAt,
offsetDateTime(rs, "received_partner_at"),
OffsetDateTime.now(),
EventDomain.LOAD_UNLOAD,
eventType,
EventLifecycle.SNAPSHOT,
longValue(rs, "odometer_m"),
position(rs),
detailsFactory.loadUnload(string(rs, "operation")),
sourcePackageRef != null && sourcePackageRef.hasAnyReference() ? sourcePackageRef : null,
detailsFactory.payloadFromMap(payload(rs)),
false,
context.packageInfo()
);
}
protected Map<String, Object> sourceSpecificPayload(ResultSet rs) throws SQLException {
return Map.of();
}
private DriverRefDto driverRef(ResultSet rs) throws SQLException {
DriverCardRefDto driverCard = null;
String cardNumber = string(rs, "driver_card_number");
if (cardNumber != null) {
driverCard = new DriverCardRefDto(string(rs, "driver_card_nation"), cardNumber);
}
DriverRefDto driverRef = new DriverRefDto(string(rs, "driver_source_entity_id"), driverCard);
return driverRef.hasAnyReference() ? driverRef : null;
}
private VehicleRefDto vehicleRef(ResultSet rs) throws SQLException {
VehicleRegistrationRefDto registration = null;
String registrationNumber = string(rs, "vehicle_registration_number");
if (registrationNumber != null) {
registration = new VehicleRegistrationRefDto(string(rs, "vehicle_registration_nation"), registrationNumber);
}
VehicleRefDto vehicleRef = new VehicleRefDto(
string(rs, "vehicle_source_entity_id"),
string(rs, "vehicle_vin"),
string(rs, "vehicle_registration_source_entity_id"),
registration
);
return vehicleRef.hasAnyReference() ? vehicleRef : null;
}
private SourcePackageRefDto sourcePackageRef(ResultSet rs) throws SQLException {
return new SourcePackageRefDto(
string(rs, "source_package_kind"),
string(rs, "source_package_id"),
string(rs, "source_package_entity_id"),
offsetDateTime(rs, "source_package_period_from"),
offsetDateTime(rs, "source_package_period_to"),
offsetDateTime(rs, "source_package_imported_at")
);
}
private GeoPointDto position(ResultSet rs) throws SQLException {
BigDecimal latitude = decimal(rs, "latitude");
BigDecimal longitude = decimal(rs, "longitude");
return latitude == null || longitude == null ? null : new GeoPointDto(latitude, longitude);
}
private Map<String, Object> payload(ResultSet rs) throws SQLException {
Map<String, Object> raw = new LinkedHashMap<>();
put(raw, "sourceRowId", string(rs, "source_row_id"));
raw.putAll(sourceSpecificPayload(rs));
return Map.of("raw", raw);
}
private String defaultExternalSourceEventId(
ExtractionContext<TachographImportRequest> context,
int rowNum,
OffsetDateTime occurredAt,
SourcePackageRefDto sourcePackageRef,
DriverRefDto driverRef,
VehicleRefDto vehicleRef,
EventType eventType
) {
String sourcePackageId = sourcePackageRef == null || sourcePackageRef.sourcePackageId() == null
? "NO_SOURCE_PACKAGE"
: sourcePackageRef.sourcePackageId();
String subject = driverRef != null && driverRef.hasAnyReference()
? driverRef.stableKey()
: vehicleRef == null ? "NO_SUBJECT" : vehicleRef.stableKey();
return "TACHOGRAPH:" + context.planItem().extractionCode()
+ ":" + sourcePackageId
+ ":" + eventType
+ ":" + occurredAt
+ ":" + subject
+ ":ROW-" + rowNum;
}
protected String string(ResultSet rs, String column) throws SQLException {
String value = rs.getString(column);
return value == null || value.isBlank() ? null : value.trim();
}
protected OffsetDateTime offsetDateTime(ResultSet rs, String column) throws SQLException {
Object value = rs.getObject(column);
if (value == null) {
return null;
}
if (value instanceof OffsetDateTime offsetDateTime) {
return offsetDateTime.withOffsetSameInstant(ZoneOffset.UTC);
}
if (value instanceof Timestamp timestamp) {
return timestamp.toLocalDateTime().atOffset(ZoneOffset.UTC);
}
if (value instanceof LocalDateTime localDateTime) {
return localDateTime.atOffset(ZoneOffset.UTC);
}
String text = value.toString();
try {
return OffsetDateTime.parse(text).withOffsetSameInstant(ZoneOffset.UTC);
} catch (RuntimeException ignored) {
return LocalDateTime.parse(text).atOffset(ZoneOffset.UTC);
}
}
private Long longValue(ResultSet rs, String column) throws SQLException {
Object value = rs.getObject(column);
if (value == null) {
return null;
}
if (value instanceof Number number) {
return number.longValue();
}
return Long.parseLong(value.toString());
}
private BigDecimal decimal(ResultSet rs, String column) throws SQLException {
Object value = rs.getObject(column);
if (value == null) {
return null;
}
if (value instanceof BigDecimal bigDecimal) {
return bigDecimal;
}
if (value instanceof Number number) {
return BigDecimal.valueOf(number.doubleValue());
}
return new BigDecimal(value.toString());
}
private EventType eventType(ResultSet rs) throws SQLException {
return parseEnum(EventType.class, string(rs, "event_type"), EventType.LOAD_UNLOAD);
}
private <T extends Enum<T>> T parseEnum(Class<T> type, String value, T fallback) {
if (value == null) {
return fallback;
}
String normalized = value.trim().toUpperCase(Locale.ROOT).replace('-', '_').replace(' ', '_');
try {
return Enum.valueOf(type, normalized);
} catch (IllegalArgumentException ignored) {
return fallback;
}
}
protected void put(Map<String, Object> target, String key, Object value) {
if (value != null) {
target.put(key, value);
}
}
}

View File

@ -0,0 +1,239 @@
package at.procon.eventhub.tachograph.service;
import at.procon.eventhub.dto.DriverCardRefDto;
import at.procon.eventhub.dto.DriverRefDto;
import at.procon.eventhub.dto.EventDomain;
import at.procon.eventhub.dto.EventHubEventDto;
import at.procon.eventhub.dto.EventLifecycle;
import at.procon.eventhub.dto.EventType;
import at.procon.eventhub.dto.GeoPointDto;
import at.procon.eventhub.dto.SourcePackageRefDto;
import at.procon.eventhub.dto.VehicleRefDto;
import at.procon.eventhub.dto.VehicleRegistrationRefDto;
import at.procon.eventhub.importing.extraction.ExtractionContext;
import at.procon.eventhub.importing.extraction.ExtractionRowMapper;
import at.procon.eventhub.service.EventDetailsFactory;
import at.procon.eventhub.tachograph.dto.TachographImportRequest;
import java.math.BigDecimal;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Timestamp;
import java.time.LocalDateTime;
import java.time.OffsetDateTime;
import java.time.ZoneOffset;
import java.util.LinkedHashMap;
import java.util.Locale;
import java.util.Map;
import java.util.UUID;
abstract class AbstractTachographPlaceRowMapper implements ExtractionRowMapper<TachographImportRequest> {
private final EventDetailsFactory detailsFactory;
protected AbstractTachographPlaceRowMapper(EventDetailsFactory detailsFactory) {
this.detailsFactory = detailsFactory;
}
@Override
public EventHubEventDto map(ResultSet rs, int rowNum, ExtractionContext<TachographImportRequest> context) throws SQLException {
OffsetDateTime occurredAt = offsetDateTime(rs, "occurred_at");
SourcePackageRefDto sourcePackageRef = sourcePackageRef(rs);
DriverRefDto driverRef = driverRef(rs);
VehicleRefDto vehicleRef = vehicleRef(rs);
EventType eventType = eventType(rs);
EventLifecycle lifecycle = lifecycle(rs);
String externalSourceEventId = string(rs, "external_source_event_id");
if (externalSourceEventId == null) {
externalSourceEventId = defaultExternalSourceEventId(context, rowNum, occurredAt, sourcePackageRef, driverRef, vehicleRef, eventType);
}
return new EventHubEventDto(
UUID.randomUUID(),
externalSourceEventId,
driverRef,
vehicleRef,
occurredAt,
offsetDateTime(rs, "received_partner_at"),
OffsetDateTime.now(),
EventDomain.PLACE,
eventType,
lifecycle,
longValue(rs, "odometer_m"),
position(rs),
detailsFactory.place(string(rs, "country"), string(rs, "region")),
sourcePackageRef != null && sourcePackageRef.hasAnyReference() ? sourcePackageRef : null,
detailsFactory.payloadFromMap(payload(rs)),
booleanValue(rs, "manual_entry"),
context.packageInfo()
);
}
protected Map<String, Object> sourceSpecificPayload(ResultSet rs) throws SQLException {
return Map.of();
}
private DriverRefDto driverRef(ResultSet rs) throws SQLException {
DriverCardRefDto driverCard = null;
String cardNumber = string(rs, "driver_card_number");
if (cardNumber != null) {
driverCard = new DriverCardRefDto(string(rs, "driver_card_nation"), cardNumber);
}
DriverRefDto driverRef = new DriverRefDto(string(rs, "driver_source_entity_id"), driverCard);
return driverRef.hasAnyReference() ? driverRef : null;
}
private VehicleRefDto vehicleRef(ResultSet rs) throws SQLException {
VehicleRegistrationRefDto registration = null;
String registrationNumber = string(rs, "vehicle_registration_number");
if (registrationNumber != null) {
registration = new VehicleRegistrationRefDto(string(rs, "vehicle_registration_nation"), registrationNumber);
}
VehicleRefDto vehicleRef = new VehicleRefDto(
string(rs, "vehicle_source_entity_id"),
string(rs, "vehicle_vin"),
string(rs, "vehicle_registration_source_entity_id"),
registration
);
return vehicleRef.hasAnyReference() ? vehicleRef : null;
}
private SourcePackageRefDto sourcePackageRef(ResultSet rs) throws SQLException {
return new SourcePackageRefDto(
string(rs, "source_package_kind"),
string(rs, "source_package_id"),
string(rs, "source_package_entity_id"),
offsetDateTime(rs, "source_package_period_from"),
offsetDateTime(rs, "source_package_period_to"),
offsetDateTime(rs, "source_package_imported_at")
);
}
private GeoPointDto position(ResultSet rs) throws SQLException {
BigDecimal latitude = decimal(rs, "latitude");
BigDecimal longitude = decimal(rs, "longitude");
return latitude == null || longitude == null ? null : new GeoPointDto(latitude, longitude);
}
private Map<String, Object> payload(ResultSet rs) throws SQLException {
Map<String, Object> raw = new LinkedHashMap<>();
put(raw, "sourceRowId", string(rs, "source_row_id"));
raw.putAll(sourceSpecificPayload(rs));
return Map.of("raw", raw);
}
private String defaultExternalSourceEventId(
ExtractionContext<TachographImportRequest> context,
int rowNum,
OffsetDateTime occurredAt,
SourcePackageRefDto sourcePackageRef,
DriverRefDto driverRef,
VehicleRefDto vehicleRef,
EventType eventType
) {
String sourcePackageId = sourcePackageRef == null || sourcePackageRef.sourcePackageId() == null
? "NO_SOURCE_PACKAGE"
: sourcePackageRef.sourcePackageId();
String subject = driverRef != null && driverRef.hasAnyReference()
? driverRef.stableKey()
: vehicleRef == null ? "NO_SUBJECT" : vehicleRef.stableKey();
return "TACHOGRAPH:" + context.planItem().extractionCode()
+ ":" + sourcePackageId
+ ":" + eventType
+ ":" + occurredAt
+ ":" + subject
+ ":ROW-" + rowNum;
}
protected String string(ResultSet rs, String column) throws SQLException {
String value = rs.getString(column);
return value == null || value.isBlank() ? null : value.trim();
}
protected OffsetDateTime offsetDateTime(ResultSet rs, String column) throws SQLException {
Object value = rs.getObject(column);
if (value == null) {
return null;
}
if (value instanceof OffsetDateTime offsetDateTime) {
return offsetDateTime.withOffsetSameInstant(ZoneOffset.UTC);
}
if (value instanceof Timestamp timestamp) {
return timestamp.toLocalDateTime().atOffset(ZoneOffset.UTC);
}
if (value instanceof LocalDateTime localDateTime) {
return localDateTime.atOffset(ZoneOffset.UTC);
}
String text = value.toString();
try {
return OffsetDateTime.parse(text).withOffsetSameInstant(ZoneOffset.UTC);
} catch (RuntimeException ignored) {
return LocalDateTime.parse(text).atOffset(ZoneOffset.UTC);
}
}
private Long longValue(ResultSet rs, String column) throws SQLException {
Object value = rs.getObject(column);
if (value == null) {
return null;
}
if (value instanceof Number number) {
return number.longValue();
}
return Long.parseLong(value.toString());
}
private BigDecimal decimal(ResultSet rs, String column) throws SQLException {
Object value = rs.getObject(column);
if (value == null) {
return null;
}
if (value instanceof BigDecimal bigDecimal) {
return bigDecimal;
}
if (value instanceof Number number) {
return BigDecimal.valueOf(number.doubleValue());
}
return new BigDecimal(value.toString());
}
private Boolean booleanValue(ResultSet rs, String column) throws SQLException {
Object value = rs.getObject(column);
if (value == null) {
return false;
}
if (value instanceof Boolean bool) {
return bool;
}
if (value instanceof Number number) {
return number.intValue() != 0;
}
return Boolean.parseBoolean(value.toString());
}
private EventType eventType(ResultSet rs) throws SQLException {
return parseEnum(EventType.class, string(rs, "event_type"), EventType.POSITION_RECORDED);
}
private EventLifecycle lifecycle(ResultSet rs) throws SQLException {
return parseEnum(EventLifecycle.class, string(rs, "lifecycle"), EventLifecycle.SNAPSHOT);
}
private <T extends Enum<T>> T parseEnum(Class<T> type, String value, T fallback) {
if (value == null) {
return fallback;
}
String normalized = value.trim().toUpperCase(Locale.ROOT).replace('-', '_').replace(' ', '_');
try {
return Enum.valueOf(type, normalized);
} catch (IllegalArgumentException ignored) {
return fallback;
}
}
protected void put(Map<String, Object> target, String key, Object value) {
if (value != null) {
target.put(key, value);
}
}
}

View File

@ -0,0 +1,201 @@
package at.procon.eventhub.tachograph.service;
import at.procon.eventhub.dto.DriverCardRefDto;
import at.procon.eventhub.dto.DriverRefDto;
import at.procon.eventhub.dto.EventDomain;
import at.procon.eventhub.dto.EventHubEventDto;
import at.procon.eventhub.dto.EventLifecycle;
import at.procon.eventhub.dto.EventType;
import at.procon.eventhub.dto.GeoPointDto;
import at.procon.eventhub.dto.SourcePackageRefDto;
import at.procon.eventhub.dto.VehicleRefDto;
import at.procon.eventhub.dto.VehicleRegistrationRefDto;
import at.procon.eventhub.importing.extraction.ExtractionContext;
import at.procon.eventhub.importing.extraction.ExtractionRowMapper;
import at.procon.eventhub.service.EventDetailsFactory;
import at.procon.eventhub.tachograph.dto.TachographImportRequest;
import java.math.BigDecimal;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Timestamp;
import java.time.LocalDateTime;
import java.time.OffsetDateTime;
import java.time.ZoneOffset;
import java.util.LinkedHashMap;
import java.util.Map;
import java.util.UUID;
abstract class AbstractTachographPositionRowMapper implements ExtractionRowMapper<TachographImportRequest> {
private final EventDetailsFactory detailsFactory;
protected AbstractTachographPositionRowMapper(EventDetailsFactory detailsFactory) {
this.detailsFactory = detailsFactory;
}
@Override
public EventHubEventDto map(ResultSet rs, int rowNum, ExtractionContext<TachographImportRequest> context) throws SQLException {
OffsetDateTime occurredAt = offsetDateTime(rs, "occurred_at");
SourcePackageRefDto sourcePackageRef = sourcePackageRef(rs);
DriverRefDto driverRef = driverRef(rs);
VehicleRefDto vehicleRef = vehicleRef(rs);
String externalSourceEventId = string(rs, "external_source_event_id");
if (externalSourceEventId == null) {
externalSourceEventId = defaultExternalSourceEventId(context, rowNum, occurredAt, sourcePackageRef, driverRef, vehicleRef);
}
return new EventHubEventDto(
UUID.randomUUID(),
externalSourceEventId,
driverRef,
vehicleRef,
occurredAt,
offsetDateTime(rs, "received_partner_at"),
OffsetDateTime.now(),
EventDomain.POSITION,
EventType.POSITION_RECORDED,
EventLifecycle.SNAPSHOT,
longValue(rs, "odometer_m"),
position(rs),
detailsFactory.position("GNSS_ACCUMULATED_DRIVING"),
sourcePackageRef != null && sourcePackageRef.hasAnyReference() ? sourcePackageRef : null,
detailsFactory.payloadFromMap(payload(rs)),
false,
context.packageInfo()
);
}
protected Map<String, Object> sourceSpecificPayload(ResultSet rs) throws SQLException {
return Map.of();
}
private DriverRefDto driverRef(ResultSet rs) throws SQLException {
DriverCardRefDto driverCard = null;
String cardNumber = string(rs, "driver_card_number");
if (cardNumber != null) {
driverCard = new DriverCardRefDto(string(rs, "driver_card_nation"), cardNumber);
}
DriverRefDto driverRef = new DriverRefDto(string(rs, "driver_source_entity_id"), driverCard);
return driverRef.hasAnyReference() ? driverRef : null;
}
private VehicleRefDto vehicleRef(ResultSet rs) throws SQLException {
VehicleRegistrationRefDto registration = null;
String registrationNumber = string(rs, "vehicle_registration_number");
if (registrationNumber != null) {
registration = new VehicleRegistrationRefDto(string(rs, "vehicle_registration_nation"), registrationNumber);
}
VehicleRefDto vehicleRef = new VehicleRefDto(
string(rs, "vehicle_source_entity_id"),
string(rs, "vehicle_vin"),
string(rs, "vehicle_registration_source_entity_id"),
registration
);
return vehicleRef.hasAnyReference() ? vehicleRef : null;
}
private SourcePackageRefDto sourcePackageRef(ResultSet rs) throws SQLException {
return new SourcePackageRefDto(
string(rs, "source_package_kind"),
string(rs, "source_package_id"),
string(rs, "source_package_entity_id"),
offsetDateTime(rs, "source_package_period_from"),
offsetDateTime(rs, "source_package_period_to"),
offsetDateTime(rs, "source_package_imported_at")
);
}
private GeoPointDto position(ResultSet rs) throws SQLException {
BigDecimal latitude = decimal(rs, "latitude");
BigDecimal longitude = decimal(rs, "longitude");
return latitude == null || longitude == null ? null : new GeoPointDto(latitude, longitude);
}
private Map<String, Object> payload(ResultSet rs) throws SQLException {
Map<String, Object> raw = new LinkedHashMap<>();
put(raw, "sourceRowId", string(rs, "source_row_id"));
raw.putAll(sourceSpecificPayload(rs));
return Map.of("raw", raw);
}
private String defaultExternalSourceEventId(
ExtractionContext<TachographImportRequest> context,
int rowNum,
OffsetDateTime occurredAt,
SourcePackageRefDto sourcePackageRef,
DriverRefDto driverRef,
VehicleRefDto vehicleRef
) {
String sourcePackageId = sourcePackageRef == null || sourcePackageRef.sourcePackageId() == null
? "NO_SOURCE_PACKAGE"
: sourcePackageRef.sourcePackageId();
String subject = driverRef != null && driverRef.hasAnyReference()
? driverRef.stableKey()
: vehicleRef == null ? "NO_SUBJECT" : vehicleRef.stableKey();
return "TACHOGRAPH:" + context.planItem().extractionCode()
+ ":" + sourcePackageId
+ ":POSITION_RECORDED"
+ ":" + occurredAt
+ ":" + subject
+ ":ROW-" + rowNum;
}
protected String string(ResultSet rs, String column) throws SQLException {
String value = rs.getString(column);
return value == null || value.isBlank() ? null : value.trim();
}
protected OffsetDateTime offsetDateTime(ResultSet rs, String column) throws SQLException {
Object value = rs.getObject(column);
if (value == null) {
return null;
}
if (value instanceof OffsetDateTime offsetDateTime) {
return offsetDateTime.withOffsetSameInstant(ZoneOffset.UTC);
}
if (value instanceof Timestamp timestamp) {
return timestamp.toLocalDateTime().atOffset(ZoneOffset.UTC);
}
if (value instanceof LocalDateTime localDateTime) {
return localDateTime.atOffset(ZoneOffset.UTC);
}
String text = value.toString();
try {
return OffsetDateTime.parse(text).withOffsetSameInstant(ZoneOffset.UTC);
} catch (RuntimeException ignored) {
return LocalDateTime.parse(text).atOffset(ZoneOffset.UTC);
}
}
private Long longValue(ResultSet rs, String column) throws SQLException {
Object value = rs.getObject(column);
if (value == null) {
return null;
}
if (value instanceof Number number) {
return number.longValue();
}
return Long.parseLong(value.toString());
}
private BigDecimal decimal(ResultSet rs, String column) throws SQLException {
Object value = rs.getObject(column);
if (value == null) {
return null;
}
if (value instanceof BigDecimal bigDecimal) {
return bigDecimal;
}
if (value instanceof Number number) {
return BigDecimal.valueOf(number.doubleValue());
}
return new BigDecimal(value.toString());
}
protected void put(Map<String, Object> target, String key, Object value) {
if (value != null) {
target.put(key, value);
}
}
}

View File

@ -0,0 +1,199 @@
package at.procon.eventhub.tachograph.service;
import at.procon.eventhub.dto.DriverCardRefDto;
import at.procon.eventhub.dto.DriverRefDto;
import at.procon.eventhub.dto.EventDomain;
import at.procon.eventhub.dto.EventHubEventDto;
import at.procon.eventhub.dto.EventLifecycle;
import at.procon.eventhub.dto.EventType;
import at.procon.eventhub.dto.SourcePackageRefDto;
import at.procon.eventhub.dto.VehicleRefDto;
import at.procon.eventhub.dto.VehicleRegistrationRefDto;
import at.procon.eventhub.importing.extraction.ExtractionContext;
import at.procon.eventhub.importing.extraction.ExtractionRowMapper;
import at.procon.eventhub.service.EventDetailsFactory;
import at.procon.eventhub.tachograph.dto.TachographImportRequest;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Timestamp;
import java.time.LocalDateTime;
import java.time.OffsetDateTime;
import java.time.ZoneOffset;
import java.util.LinkedHashMap;
import java.util.Locale;
import java.util.Map;
import java.util.UUID;
abstract class AbstractTachographSpecificConditionRowMapper implements ExtractionRowMapper<TachographImportRequest> {
private final EventDetailsFactory detailsFactory;
protected AbstractTachographSpecificConditionRowMapper(EventDetailsFactory detailsFactory) {
this.detailsFactory = detailsFactory;
}
@Override
public EventHubEventDto map(ResultSet rs, int rowNum, ExtractionContext<TachographImportRequest> context) throws SQLException {
OffsetDateTime occurredAt = offsetDateTime(rs, "occurred_at");
SourcePackageRefDto sourcePackageRef = sourcePackageRef(rs);
DriverRefDto driverRef = driverRef(rs);
VehicleRefDto vehicleRef = vehicleRef(rs);
EventDomain eventDomain = eventDomain(rs);
EventType eventType = eventType(rs);
EventLifecycle lifecycle = lifecycle(rs);
String externalSourceEventId = string(rs, "external_source_event_id");
if (externalSourceEventId == null) {
externalSourceEventId = defaultExternalSourceEventId(context, rowNum, occurredAt, sourcePackageRef, driverRef, vehicleRef, eventType, lifecycle);
}
return new EventHubEventDto(
UUID.randomUUID(),
externalSourceEventId,
driverRef,
vehicleRef,
occurredAt,
offsetDateTime(rs, "received_partner_at"),
OffsetDateTime.now(),
eventDomain,
eventType,
lifecycle,
null,
null,
detailsFactory.specificCondition(),
sourcePackageRef != null && sourcePackageRef.hasAnyReference() ? sourcePackageRef : null,
detailsFactory.payloadFromMap(payload(rs)),
false,
context.packageInfo()
);
}
protected Map<String, Object> sourceSpecificPayload(ResultSet rs) throws SQLException {
return Map.of();
}
private DriverRefDto driverRef(ResultSet rs) throws SQLException {
DriverCardRefDto driverCard = null;
String cardNumber = string(rs, "driver_card_number");
if (cardNumber != null) {
driverCard = new DriverCardRefDto(string(rs, "driver_card_nation"), cardNumber);
}
DriverRefDto driverRef = new DriverRefDto(string(rs, "driver_source_entity_id"), driverCard);
return driverRef.hasAnyReference() ? driverRef : null;
}
private VehicleRefDto vehicleRef(ResultSet rs) throws SQLException {
VehicleRegistrationRefDto registration = null;
String registrationNumber = string(rs, "vehicle_registration_number");
if (registrationNumber != null) {
registration = new VehicleRegistrationRefDto(string(rs, "vehicle_registration_nation"), registrationNumber);
}
VehicleRefDto vehicleRef = new VehicleRefDto(
string(rs, "vehicle_source_entity_id"),
string(rs, "vehicle_vin"),
string(rs, "vehicle_registration_source_entity_id"),
registration
);
return vehicleRef.hasAnyReference() ? vehicleRef : null;
}
private SourcePackageRefDto sourcePackageRef(ResultSet rs) throws SQLException {
return new SourcePackageRefDto(
string(rs, "source_package_kind"),
string(rs, "source_package_id"),
string(rs, "source_package_entity_id"),
offsetDateTime(rs, "source_package_period_from"),
offsetDateTime(rs, "source_package_period_to"),
offsetDateTime(rs, "source_package_imported_at")
);
}
private Map<String, Object> payload(ResultSet rs) throws SQLException {
Map<String, Object> raw = new LinkedHashMap<>();
put(raw, "sourceRowId", string(rs, "source_row_id"));
raw.putAll(sourceSpecificPayload(rs));
return Map.of("raw", raw);
}
private String defaultExternalSourceEventId(
ExtractionContext<TachographImportRequest> context,
int rowNum,
OffsetDateTime occurredAt,
SourcePackageRefDto sourcePackageRef,
DriverRefDto driverRef,
VehicleRefDto vehicleRef,
EventType eventType,
EventLifecycle lifecycle
) {
String sourcePackageId = sourcePackageRef == null || sourcePackageRef.sourcePackageId() == null
? "NO_SOURCE_PACKAGE"
: sourcePackageRef.sourcePackageId();
String subject = driverRef != null && driverRef.hasAnyReference()
? driverRef.stableKey()
: vehicleRef == null ? "NO_SUBJECT" : vehicleRef.stableKey();
return "TACHOGRAPH:" + context.planItem().extractionCode()
+ ":" + sourcePackageId
+ ":" + eventType
+ ":" + lifecycle
+ ":" + occurredAt
+ ":" + subject
+ ":ROW-" + rowNum;
}
protected String string(ResultSet rs, String column) throws SQLException {
String value = rs.getString(column);
return value == null || value.isBlank() ? null : value.trim();
}
protected OffsetDateTime offsetDateTime(ResultSet rs, String column) throws SQLException {
Object value = rs.getObject(column);
if (value == null) {
return null;
}
if (value instanceof OffsetDateTime offsetDateTime) {
return offsetDateTime.withOffsetSameInstant(ZoneOffset.UTC);
}
if (value instanceof Timestamp timestamp) {
return timestamp.toLocalDateTime().atOffset(ZoneOffset.UTC);
}
if (value instanceof LocalDateTime localDateTime) {
return localDateTime.atOffset(ZoneOffset.UTC);
}
String text = value.toString();
try {
return OffsetDateTime.parse(text).withOffsetSameInstant(ZoneOffset.UTC);
} catch (RuntimeException ignored) {
return LocalDateTime.parse(text).atOffset(ZoneOffset.UTC);
}
}
private EventDomain eventDomain(ResultSet rs) throws SQLException {
return parseEnum(EventDomain.class, string(rs, "event_domain"), EventDomain.SPECIFIC_CONDITION);
}
private EventType eventType(ResultSet rs) throws SQLException {
return parseEnum(EventType.class, string(rs, "event_type"), EventType.UNKNOWN_EVENT);
}
private EventLifecycle lifecycle(ResultSet rs) throws SQLException {
return parseEnum(EventLifecycle.class, string(rs, "lifecycle"), EventLifecycle.SNAPSHOT);
}
private <T extends Enum<T>> T parseEnum(Class<T> type, String value, T fallback) {
if (value == null) {
return fallback;
}
String normalized = value.trim().toUpperCase(Locale.ROOT).replace('-', '_').replace(' ', '_');
try {
return Enum.valueOf(type, normalized);
} catch (IllegalArgumentException ignored) {
return fallback;
}
}
protected void put(Map<String, Object> target, String key, Object value) {
if (value != null) {
target.put(key, value);
}
}
}

View File

@ -1,10 +1,6 @@
package at.procon.eventhub.tachograph.service;
import at.procon.eventhub.service.EventDetailsFactory;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.LinkedHashMap;
import java.util.Map;
import org.springframework.stereotype.Component;
@Component
@ -13,11 +9,4 @@ public class CardActivityRowMapper extends AbstractTachographActivityRowMapper {
public CardActivityRowMapper(EventDetailsFactory detailsFactory) {
super(detailsFactory);
}
@Override
protected Map<String, Object> sourceSpecificPayload(ResultSet rs) throws SQLException {
Map<String, Object> raw = new LinkedHashMap<>();
put(raw, "cardActivityId", string(rs, "card_activity_id"));
return raw;
}
}

View File

@ -0,0 +1,12 @@
package at.procon.eventhub.tachograph.service;
import at.procon.eventhub.service.EventDetailsFactory;
import org.springframework.stereotype.Component;
@Component
public class CardBorderCrossingRowMapper extends AbstractTachographBorderCrossingRowMapper {
public CardBorderCrossingRowMapper(EventDetailsFactory detailsFactory) {
super(detailsFactory);
}
}

View File

@ -0,0 +1,12 @@
package at.procon.eventhub.tachograph.service;
import at.procon.eventhub.service.EventDetailsFactory;
import org.springframework.stereotype.Component;
@Component
public class CardLoadUnloadRowMapper extends AbstractTachographLoadUnloadRowMapper {
public CardLoadUnloadRowMapper(EventDetailsFactory detailsFactory) {
super(detailsFactory);
}
}

View File

@ -0,0 +1,12 @@
package at.procon.eventhub.tachograph.service;
import at.procon.eventhub.service.EventDetailsFactory;
import org.springframework.stereotype.Component;
@Component
public class CardPlaceRowMapper extends AbstractTachographPlaceRowMapper {
public CardPlaceRowMapper(EventDetailsFactory detailsFactory) {
super(detailsFactory);
}
}

View File

@ -0,0 +1,12 @@
package at.procon.eventhub.tachograph.service;
import at.procon.eventhub.service.EventDetailsFactory;
import org.springframework.stereotype.Component;
@Component
public class CardPositionRowMapper extends AbstractTachographPositionRowMapper {
public CardPositionRowMapper(EventDetailsFactory detailsFactory) {
super(detailsFactory);
}
}

View File

@ -0,0 +1,12 @@
package at.procon.eventhub.tachograph.service;
import at.procon.eventhub.service.EventDetailsFactory;
import org.springframework.stereotype.Component;
@Component
public class CardSpecificConditionRowMapper extends AbstractTachographSpecificConditionRowMapper {
public CardSpecificConditionRowMapper(EventDetailsFactory detailsFactory) {
super(detailsFactory);
}
}

View File

@ -0,0 +1,12 @@
package at.procon.eventhub.tachograph.service;
import at.procon.eventhub.service.EventDetailsFactory;
import org.springframework.stereotype.Component;
@Component
public class CardVehiclesUsedCardEventRowMapper extends AbstractTachographCardEventRowMapper {
public CardVehiclesUsedCardEventRowMapper(EventDetailsFactory detailsFactory) {
super(detailsFactory);
}
}

View File

@ -0,0 +1,12 @@
package at.procon.eventhub.tachograph.service;
import at.procon.eventhub.service.EventDetailsFactory;
import org.springframework.stereotype.Component;
@Component
public class IwCycleCardEventRowMapper extends AbstractTachographCardEventRowMapper {
public IwCycleCardEventRowMapper(EventDetailsFactory detailsFactory) {
super(detailsFactory);
}
}

View File

@ -1,8 +1,8 @@
package at.procon.eventhub.tachograph.service;
import at.procon.eventhub.dto.EventHubEventDto;
import at.procon.eventhub.dto.EventSourceDto;
import at.procon.eventhub.dto.ImportCursorStateDto;
import at.procon.eventhub.dto.ImportScopeDto;
import at.procon.eventhub.importing.ImportPlanItemDto;
import at.procon.eventhub.importing.ImportTimeChunkDto;
import at.procon.eventhub.importing.extraction.AbstractJdbcExtractionBatchExecutor;
@ -10,7 +10,10 @@ import at.procon.eventhub.importing.extraction.ExtractionDefinition;
import at.procon.eventhub.importing.persistence.ImportCursorRepository;
import at.procon.eventhub.tachograph.dto.TachographExtractionBatchResultDto;
import at.procon.eventhub.tachograph.dto.TachographImportRequest;
import java.util.List;
import java.time.LocalDateTime;
import java.time.OffsetDateTime;
import java.time.ZoneOffset;
import java.util.Map;
import java.util.Optional;
import java.util.UUID;
import org.apache.camel.ProducerTemplate;
@ -46,27 +49,43 @@ public class JdbcTachographExtractionBatchExecutor
return definitionRegistry.findByCode(code);
}
@Override
protected Map<String, Object> parameters(
TachographImportRequest request,
ImportScopeDto scope,
ImportCursorStateDto cursor
) {
Map<String, Object> params = super.parameters(request, scope, cursor);
params.put("occurredFrom", utcLocalDateTime(scope == null ? null : scope.occurredFrom()));
params.put("occurredTo", utcLocalDateTime(scope == null ? null : scope.occurredTo()));
params.put("lastSourcePackageImportedAt", utcLocalDateTime(cursor == null ? null : cursor.lastSourcePackageImportedAt()));
params.put("lastSourceRowUpdatedAt", utcLocalDateTime(cursor == null ? null : cursor.lastSourceRowUpdatedAt()));
params.put("lastOccurredTo", utcLocalDateTime(cursor == null ? null : cursor.lastOccurredTo()));
return params;
}
@Override
protected TachographExtractionBatchResultDto resultFor(
UUID packageId,
ImportPlanItemDto planItem,
ImportTimeChunkDto chunk,
ImportCursorStateDto cursor,
List<EventHubEventDto> events
ExtractedEventStats stats
) {
return new TachographExtractionBatchResultDto(
packageId,
planItem.extractionCode(),
planItem.sourceKind(),
events.size(),
events.size(),
events.size(),
stats.eventsMapped(),
stats.eventsMapped(),
stats.eventsMapped(),
0,
true,
lastSourcePackageImportedAt(events, cursor),
lastSourcePackageId(events, cursor),
lastSourcePackageImportedAt(stats, cursor),
lastSourcePackageId(stats, cursor),
null,
chunk.occurredTo()
chunk.occurredTo(),
stats.eventTypeCounts()
);
}
@ -91,4 +110,8 @@ public class JdbcTachographExtractionBatchExecutor
protected String providerPackagePrefix() {
return "TACHOGRAPH";
}
private LocalDateTime utcLocalDateTime(OffsetDateTime value) {
return value == null ? null : value.withOffsetSameInstant(ZoneOffset.UTC).toLocalDateTime();
}
}

View File

@ -42,7 +42,8 @@ public class NoopTachographExtractionBatchExecutor
null,
null,
null,
null
null,
java.util.Map.of()
);
}

View File

@ -0,0 +1,200 @@
package at.procon.eventhub.tachograph.service;
import at.procon.eventhub.dto.DriverCardRefDto;
import at.procon.eventhub.dto.DriverRefDto;
import at.procon.eventhub.dto.EventDomain;
import at.procon.eventhub.dto.EventHubEventDto;
import at.procon.eventhub.dto.EventLifecycle;
import at.procon.eventhub.dto.EventType;
import at.procon.eventhub.dto.SourcePackageRefDto;
import at.procon.eventhub.dto.VehicleRefDto;
import at.procon.eventhub.dto.VehicleRegistrationRefDto;
import at.procon.eventhub.importing.extraction.ExtractionContext;
import at.procon.eventhub.importing.extraction.ExtractionRowMapper;
import at.procon.eventhub.service.EventDetailsFactory;
import at.procon.eventhub.tachograph.dto.TachographImportRequest;
import java.math.BigDecimal;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Timestamp;
import java.time.LocalDateTime;
import java.time.OffsetDateTime;
import java.time.ZoneOffset;
import java.util.LinkedHashMap;
import java.util.Locale;
import java.util.Map;
import java.util.UUID;
import org.springframework.stereotype.Component;
@Component
public class SpeedingEventRowMapper implements ExtractionRowMapper<TachographImportRequest> {
private final EventDetailsFactory detailsFactory;
public SpeedingEventRowMapper(EventDetailsFactory detailsFactory) {
this.detailsFactory = detailsFactory;
}
@Override
public EventHubEventDto map(ResultSet rs, int rowNum, ExtractionContext<TachographImportRequest> context) throws SQLException {
OffsetDateTime occurredAt = offsetDateTime(rs, "occurred_at");
SourcePackageRefDto sourcePackageRef = sourcePackageRef(rs);
DriverRefDto driverRef = driverRef(rs);
VehicleRefDto vehicleRef = vehicleRef(rs);
EventLifecycle lifecycle = lifecycle(rs);
String externalSourceEventId = string(rs, "external_source_event_id");
if (externalSourceEventId == null) {
externalSourceEventId = defaultExternalSourceEventId(context, rowNum, occurredAt, sourcePackageRef, driverRef, vehicleRef, lifecycle);
}
return new EventHubEventDto(
UUID.randomUUID(),
externalSourceEventId,
driverRef,
vehicleRef,
occurredAt,
offsetDateTime(rs, "received_partner_at"),
OffsetDateTime.now(),
EventDomain.SPEEDING,
EventType.SPEEDING,
lifecycle,
null,
null,
detailsFactory.speeding(decimal(rs, "avg_speed_kmh"), decimal(rs, "max_speed_kmh"), null),
sourcePackageRef != null && sourcePackageRef.hasAnyReference() ? sourcePackageRef : null,
detailsFactory.payloadFromMap(payload(rs)),
false,
context.packageInfo()
);
}
private DriverRefDto driverRef(ResultSet rs) throws SQLException {
DriverCardRefDto driverCard = null;
String cardNumber = string(rs, "driver_card_number");
if (cardNumber != null) {
driverCard = new DriverCardRefDto(string(rs, "driver_card_nation"), cardNumber);
}
DriverRefDto driverRef = new DriverRefDto(string(rs, "driver_source_entity_id"), driverCard);
return driverRef.hasAnyReference() ? driverRef : null;
}
private VehicleRefDto vehicleRef(ResultSet rs) throws SQLException {
VehicleRegistrationRefDto registration = null;
String registrationNumber = string(rs, "vehicle_registration_number");
if (registrationNumber != null) {
registration = new VehicleRegistrationRefDto(string(rs, "vehicle_registration_nation"), registrationNumber);
}
VehicleRefDto vehicleRef = new VehicleRefDto(
string(rs, "vehicle_source_entity_id"),
string(rs, "vehicle_vin"),
string(rs, "vehicle_registration_source_entity_id"),
registration
);
return vehicleRef.hasAnyReference() ? vehicleRef : null;
}
private SourcePackageRefDto sourcePackageRef(ResultSet rs) throws SQLException {
return new SourcePackageRefDto(
string(rs, "source_package_kind"),
string(rs, "source_package_id"),
string(rs, "source_package_entity_id"),
offsetDateTime(rs, "source_package_period_from"),
offsetDateTime(rs, "source_package_period_to"),
offsetDateTime(rs, "source_package_imported_at")
);
}
private Map<String, Object> payload(ResultSet rs) throws SQLException {
Map<String, Object> raw = new LinkedHashMap<>();
put(raw, "sourceRowId", string(rs, "source_row_id"));
return Map.of("raw", raw);
}
private String defaultExternalSourceEventId(
ExtractionContext<TachographImportRequest> context,
int rowNum,
OffsetDateTime occurredAt,
SourcePackageRefDto sourcePackageRef,
DriverRefDto driverRef,
VehicleRefDto vehicleRef,
EventLifecycle lifecycle
) {
String sourcePackageId = sourcePackageRef == null || sourcePackageRef.sourcePackageId() == null
? "NO_SOURCE_PACKAGE"
: sourcePackageRef.sourcePackageId();
String subject = driverRef != null && driverRef.hasAnyReference()
? driverRef.stableKey()
: vehicleRef == null ? "NO_SUBJECT" : vehicleRef.stableKey();
return "TACHOGRAPH:" + context.planItem().extractionCode()
+ ":" + sourcePackageId
+ ":SPEEDING"
+ ":" + lifecycle
+ ":" + occurredAt
+ ":" + subject
+ ":ROW-" + rowNum;
}
private String string(ResultSet rs, String column) throws SQLException {
String value = rs.getString(column);
return value == null || value.isBlank() ? null : value.trim();
}
private OffsetDateTime offsetDateTime(ResultSet rs, String column) throws SQLException {
Object value = rs.getObject(column);
if (value == null) {
return null;
}
if (value instanceof OffsetDateTime offsetDateTime) {
return offsetDateTime.withOffsetSameInstant(ZoneOffset.UTC);
}
if (value instanceof Timestamp timestamp) {
return timestamp.toLocalDateTime().atOffset(ZoneOffset.UTC);
}
if (value instanceof LocalDateTime localDateTime) {
return localDateTime.atOffset(ZoneOffset.UTC);
}
String text = value.toString();
try {
return OffsetDateTime.parse(text).withOffsetSameInstant(ZoneOffset.UTC);
} catch (RuntimeException ignored) {
return LocalDateTime.parse(text).atOffset(ZoneOffset.UTC);
}
}
private BigDecimal decimal(ResultSet rs, String column) throws SQLException {
Object value = rs.getObject(column);
if (value == null) {
return null;
}
if (value instanceof BigDecimal bigDecimal) {
return bigDecimal;
}
if (value instanceof Number number) {
return BigDecimal.valueOf(number.doubleValue());
}
return new BigDecimal(value.toString());
}
private EventLifecycle lifecycle(ResultSet rs) throws SQLException {
return parseEnum(EventLifecycle.class, string(rs, "lifecycle"), EventLifecycle.BEGIN);
}
private <T extends Enum<T>> T parseEnum(Class<T> type, String value, T fallback) {
if (value == null) {
return fallback;
}
String normalized = value.trim().toUpperCase(Locale.ROOT).replace('-', '_').replace(' ', '_');
try {
return Enum.valueOf(type, normalized);
} catch (IllegalArgumentException ignored) {
return fallback;
}
}
private void put(Map<String, Object> target, String key, Object value) {
if (value != null) {
target.put(key, value);
}
}
}

View File

@ -12,7 +12,20 @@ public class TachographExtractionDefinitionRegistry extends ExtractionDefinition
public TachographExtractionDefinitionRegistry(
CardActivityRowMapper cardActivityRowMapper,
VuActivityRowMapper vuActivityRowMapper
VuActivityRowMapper vuActivityRowMapper,
IwCycleCardEventRowMapper iwCycleCardEventRowMapper,
CardVehiclesUsedCardEventRowMapper cardVehiclesUsedCardEventRowMapper,
VuBorderCrossingRowMapper vuBorderCrossingRowMapper,
CardBorderCrossingRowMapper cardBorderCrossingRowMapper,
VuLoadUnloadRowMapper vuLoadUnloadRowMapper,
CardLoadUnloadRowMapper cardLoadUnloadRowMapper,
VuSpecificConditionRowMapper vuSpecificConditionRowMapper,
CardSpecificConditionRowMapper cardSpecificConditionRowMapper,
VuPositionRowMapper vuPositionRowMapper,
CardPositionRowMapper cardPositionRowMapper,
VuPlaceRowMapper vuPlaceRowMapper,
CardPlaceRowMapper cardPlaceRowMapper,
SpeedingEventRowMapper speedingEventRowMapper
) {
super(List.of(
new ExtractionDefinition<>(
@ -30,6 +43,110 @@ public class TachographExtractionDefinitionRegistry extends ExtractionDefinition
"VEHICLE",
"classpath:sql/tachograph/vu-activity.sql",
vuActivityRowMapper
),
new ExtractionDefinition<>(
"IW_CYCLE",
EventFamily.DRIVER_CARD,
"VEHICLE_UNIT",
"BOTH",
"classpath:sql/tachograph/iw-cycle.sql",
iwCycleCardEventRowMapper
),
new ExtractionDefinition<>(
"CARD_VEHICLES_USED",
EventFamily.DRIVER_CARD,
"DRIVER_CARD",
"DRIVER",
"classpath:sql/tachograph/card-vehicles-used.sql",
cardVehiclesUsedCardEventRowMapper
),
new ExtractionDefinition<>(
"VU_BORDER_CROSSING",
EventFamily.BORDER_CROSSING,
"VEHICLE_UNIT",
"VEHICLE",
"classpath:sql/tachograph/vu-border-crossing.sql",
vuBorderCrossingRowMapper
),
new ExtractionDefinition<>(
"CARD_BORDER_CROSSING",
EventFamily.BORDER_CROSSING,
"DRIVER_CARD",
"DRIVER",
"classpath:sql/tachograph/card-border-crossing.sql",
cardBorderCrossingRowMapper
),
new ExtractionDefinition<>(
"VU_LOAD_UNLOAD",
EventFamily.LOAD_UNLOAD,
"VEHICLE_UNIT",
"VEHICLE",
"classpath:sql/tachograph/vu-load-unload.sql",
vuLoadUnloadRowMapper
),
new ExtractionDefinition<>(
"CARD_LOAD_UNLOAD",
EventFamily.LOAD_UNLOAD,
"DRIVER_CARD",
"DRIVER",
"classpath:sql/tachograph/card-load-unload.sql",
cardLoadUnloadRowMapper
),
new ExtractionDefinition<>(
"VU_SPECIFIC_CONDITION",
EventFamily.SPECIFIC_CONDITION,
"VEHICLE_UNIT",
"VEHICLE",
"classpath:sql/tachograph/vu-specific-condition.sql",
vuSpecificConditionRowMapper
),
new ExtractionDefinition<>(
"CARD_SPECIFIC_CONDITION",
EventFamily.SPECIFIC_CONDITION,
"DRIVER_CARD",
"DRIVER",
"classpath:sql/tachograph/card-specific-condition.sql",
cardSpecificConditionRowMapper
),
new ExtractionDefinition<>(
"VU_POSITION",
EventFamily.POSITION,
"VEHICLE_UNIT",
"VEHICLE",
"classpath:sql/tachograph/vu-position.sql",
vuPositionRowMapper
),
new ExtractionDefinition<>(
"CARD_POSITION",
EventFamily.POSITION,
"DRIVER_CARD",
"DRIVER",
"classpath:sql/tachograph/card-position.sql",
cardPositionRowMapper
),
new ExtractionDefinition<>(
"VU_PLACE",
EventFamily.PLACE,
"VEHICLE_UNIT",
"VEHICLE",
"classpath:sql/tachograph/vu-place.sql",
vuPlaceRowMapper
),
new ExtractionDefinition<>(
"CARD_PLACE",
EventFamily.PLACE,
"DRIVER_CARD",
"DRIVER",
"classpath:sql/tachograph/card-place.sql",
cardPlaceRowMapper
),
new ExtractionDefinition<>(
"SPEEDING_EVENTS",
EventFamily.SPEEDING,
"VEHICLE_UNIT",
"VEHICLE",
"classpath:sql/tachograph/speeding-events.sql",
speedingEventRowMapper
)
));
}

View File

@ -1,5 +1,6 @@
package at.procon.eventhub.tachograph.service;
import at.procon.eventhub.config.EventHubProperties;
import at.procon.eventhub.dto.EventSourceDto;
import at.procon.eventhub.importing.AbstractImportExecutionService;
import at.procon.eventhub.importing.ImportPlanDto;
@ -32,10 +33,11 @@ public class TachographImportExecutionService
ImportRunRepository importRunRepository,
DataPackageRepository dataPackageRepository,
ImportCursorRepository importCursorRepository,
EventHubProperties eventHubProperties,
TachographMasterDataRefreshService masterDataRefreshService,
TachographExtractionBatchExecutor extractionBatchExecutor
) {
super(eventSourceRepository, importRunRepository, dataPackageRepository, importCursorRepository);
super(eventSourceRepository, importRunRepository, dataPackageRepository, importCursorRepository, eventHubProperties);
this.planService = planService;
this.masterDataRefreshService = masterDataRefreshService;
this.extractionBatchExecutor = extractionBatchExecutor;
@ -46,7 +48,6 @@ public class TachographImportExecutionService
return toTachographResult(createImportRun(request, false));
}
@Transactional
public TachographImportRunResultDto startAndExecuteImport(TachographImportRequest request) {
return toTachographResult(createImportRun(request, true));
}

View File

@ -9,6 +9,8 @@ import at.procon.eventhub.importing.ImportPlanItemDto;
import at.procon.eventhub.tachograph.dto.TachographImportRequest;
import java.util.ArrayList;
import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;
import org.springframework.stereotype.Service;
@Service
@ -16,10 +18,16 @@ public class TachographImportPlanService {
private final EventHubProperties properties;
private final ImportChunkPlanner chunkPlanner;
private final TachographExtractionDefinitionRegistry definitionRegistry;
public TachographImportPlanService(EventHubProperties properties, ImportChunkPlanner chunkPlanner) {
public TachographImportPlanService(
EventHubProperties properties,
ImportChunkPlanner chunkPlanner,
TachographExtractionDefinitionRegistry definitionRegistry
) {
this.properties = properties;
this.chunkPlanner = chunkPlanner;
this.definitionRegistry = definitionRegistry;
}
public ImportPlanDto createPlan(TachographImportRequest request) {
@ -27,6 +35,7 @@ public class TachographImportPlanService {
for (EventFamily family : request.eventFamilies()) {
items.addAll(itemsFor(family, request.acquisitionStrategy()));
}
validateJdbcExtractions(items, request.eventFamilies());
return new ImportPlanDto(
request.tenantKey(),
request.mode(),
@ -51,8 +60,8 @@ public class TachographImportPlanService {
item(family, "DRIVER_CARD", "CARD_VEHICLES_USED", List.of("CardVehiclesUsed"), "DRIVER", "Card insert/withdraw/use events from card vehicle usage", strategy)
);
case POSITION -> List.of(
item(family, "VEHICLE_UNIT", "VU_POSITION", List.of("VUPlaces", "VULoadUnload", "VUGnssAccumulatedDriving", "VUBorderCrossing"), "VEHICLE", "Position points from VU tachograph sources", strategy),
item(family, "DRIVER_CARD", "CARD_POSITION", List.of("CardPlaces", "CardLoadUnload", "CardGnssAccumulatedDriving", "CardBorderCrossing"), "DRIVER", "Position points from driver-card tachograph sources", strategy)
item(family, "VEHICLE_UNIT", "VU_POSITION", List.of("VUGnssAccumulatedDriving"), "VEHICLE", "Periodic GNSS position points from VU", strategy),
item(family, "DRIVER_CARD", "CARD_POSITION", List.of("CardGnssAccumulatedDriving"), "DRIVER", "Periodic GNSS position points from driver card", strategy)
);
case BORDER_CROSSING -> List.of(
item(family, "VEHICLE_UNIT", "VU_BORDER_CROSSING", List.of("VUBorderCrossing"), "VEHICLE", "Border crossing events from VU", strategy),
@ -87,4 +96,47 @@ public class TachographImportPlanService {
) {
return new ImportPlanItemDto(family, sourceKind, extractionCode, sourceTables, entityAxis, description, strategy);
}
private void validateJdbcExtractions(List<ImportPlanItemDto> items, Set<EventFamily> requestedFamilies) {
String jdbcUrl = properties.getTachograph().getDatasource().getJdbcUrl();
if (jdbcUrl == null || jdbcUrl.isBlank()) {
return;
}
List<String> unsupportedCodes = items.stream()
.map(ImportPlanItemDto::extractionCode)
.filter(code -> definitionRegistry.findByCode(code).isEmpty())
.distinct()
.sorted()
.toList();
if (unsupportedCodes.isEmpty()) {
return;
}
List<String> supportedCodes = definitionRegistry.supportedCodes().stream()
.sorted()
.toList();
List<String> supportedFamilies = definitionRegistry.definitions().stream()
.map(definition -> definition.eventFamily().name())
.distinct()
.sorted()
.toList();
String requestedFamilyNames = requestedFamilies.stream()
.map(EventFamily::name)
.sorted()
.collect(Collectors.joining(", "));
throw new UnsupportedTachographExtractionException(
"Tachograph JDBC extraction is enabled, but the plan contains extraction codes without JDBC definitions: "
+ unsupportedCodes
+ ". Supported JDBC extraction codes are: "
+ supportedCodes
+ ". Requested event families: ["
+ requestedFamilyNames
+ "]. "
+ "Use only supported event families "
+ supportedFamilies
+ " for JDBC execution, or add the missing codes to TachographExtractionDefinitionRegistry."
);
}
}

View File

@ -1,27 +1,361 @@
package at.procon.eventhub.tachograph.service;
import at.procon.eventhub.importing.masterdata.MasterDataRefreshResult;
import at.procon.eventhub.importing.masterdata.SourceMasterEntityUpsert;
import at.procon.eventhub.importing.masterdata.SourceMasterRelationUpsert;
import at.procon.eventhub.dto.EventSourceDto;
import at.procon.eventhub.persistence.EventSourceRepository;
import at.procon.eventhub.persistence.SourceMasterDataRepository;
import at.procon.eventhub.persistence.VehicleIdentityRepository;
import at.procon.eventhub.tachograph.dto.TachographImportRequest;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.sql.ResultSet;
import java.sql.ResultSetMetaData;
import java.sql.SQLException;
import java.sql.Timestamp;
import java.time.LocalDateTime;
import java.time.OffsetDateTime;
import java.time.ZoneOffset;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.ObjectProvider;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.core.io.Resource;
import org.springframework.core.io.ResourceLoader;
import org.springframework.jdbc.core.namedparam.NamedParameterJdbcTemplate;
import org.springframework.stereotype.Service;
import org.springframework.util.StreamUtils;
/**
* Hook for refreshing tachograph master data before event extraction.
*
* The generated project does not yet know the concrete tachograph master-data
* schema. Replace/extend this service with SQL readers for organisations,
* vehicles, vehicle registrations, drivers, and driver cards.
*/
@Service
public class TachographMasterDataRefreshService {
private static final Logger log = LoggerFactory.getLogger(TachographMasterDataRefreshService.class);
public void refreshIfRequested(TachographImportRequest request) {
private static final List<String> ENTITY_SQL_RESOURCES = List.of(
"classpath:sql/tachograph/master-data/organisations.sql",
"classpath:sql/tachograph/master-data/drivers.sql",
"classpath:sql/tachograph/master-data/driver-cards.sql",
"classpath:sql/tachograph/master-data/vehicles.sql",
"classpath:sql/tachograph/master-data/vehicle-registrations.sql"
);
private static final String RELATIONS_SQL_RESOURCE = "classpath:sql/tachograph/master-data/relations.sql";
private static final String MASTER_DATA_SOURCE_KIND = "MASTER_DATA";
private static final String MASTER_DATA_SOURCE_KEY = "TACHOGRAPH_MASTER_DATA";
private static final int MASTER_DATA_UPSERT_BATCH_SIZE = 5000;
private final ObjectProvider<NamedParameterJdbcTemplate> tachographJdbcTemplateProvider;
private final SourceMasterDataRepository sourceMasterDataRepository;
private final EventSourceRepository eventSourceRepository;
private final VehicleIdentityRepository vehicleIdentityRepository;
private final ResourceLoader resourceLoader;
public TachographMasterDataRefreshService(
@Qualifier("tachographNamedParameterJdbcTemplate") ObjectProvider<NamedParameterJdbcTemplate> tachographJdbcTemplateProvider,
SourceMasterDataRepository sourceMasterDataRepository,
EventSourceRepository eventSourceRepository,
VehicleIdentityRepository vehicleIdentityRepository,
ResourceLoader resourceLoader
) {
this.tachographJdbcTemplateProvider = tachographJdbcTemplateProvider;
this.sourceMasterDataRepository = sourceMasterDataRepository;
this.eventSourceRepository = eventSourceRepository;
this.vehicleIdentityRepository = vehicleIdentityRepository;
this.resourceLoader = resourceLoader;
}
public MasterDataRefreshResult refreshIfRequested(TachographImportRequest request) {
if (!request.refreshMasterDataFirst()) {
return;
return MasterDataRefreshResult.empty();
}
log.info("Tachograph master-data refresh requested for tenant={} source={}. Concrete SQL refresh is a project-specific extension point.",
request.tenantKey(), request.eventSource().stableKey());
return refresh(request);
}
public MasterDataRefreshResult refresh(TachographImportRequest request) {
NamedParameterJdbcTemplate tachographJdbcTemplate = tachographJdbcTemplateProvider.getIfAvailable();
if (tachographJdbcTemplate == null) {
log.info("Skipping tachograph master-data refresh for tenant={} because no tachograph datasource is configured.",
request.tenantKey());
return MasterDataRefreshResult.empty();
}
String tenantKey = request.tenantKey() == null || request.tenantKey().isBlank() ? "default" : request.tenantKey().trim();
EventSourceDto masterDataSource = masterDataSourceFor(request.eventSource());
int eventSourceId = eventSourceRepository.resolveSourceId(tenantKey, masterDataSource);
log.info("Starting tachograph source master-data refresh tenant={} source={} batchSize={}",
tenantKey, masterDataSource.stableKey(), MASTER_DATA_UPSERT_BATCH_SIZE);
int entities = 0;
for (String sqlResource : ENTITY_SQL_RESOURCES) {
entities += streamEntities(tachographJdbcTemplate, tenantKey, eventSourceId, sqlResource, loadSql(sqlResource));
}
int relationCount = streamRelations(tachographJdbcTemplate, tenantKey, eventSourceId, RELATIONS_SQL_RESOURCE, loadSql(RELATIONS_SQL_RESOURCE));
log.info("Reconciling tachograph vehicle identities from source master data tenant={} source={}",
tenantKey, masterDataSource.stableKey());
int reconciledVehicles = vehicleIdentityRepository.reconcileFromMasterData(tenantKey, eventSourceId);
MasterDataRefreshResult result = new MasterDataRefreshResult(entities, relationCount);
log.info("Refreshed tachograph source master data tenant={} source={} entities={} relations={} reconciledVehicles={}",
tenantKey, masterDataSource.stableKey(), result.entitiesUpserted(), result.relationsUpserted(), reconciledVehicles);
return result;
}
private int streamEntities(
NamedParameterJdbcTemplate tachographJdbcTemplate,
String tenantKey,
int eventSourceId,
String sqlResource,
String sql
) {
String section = masterDataSection(sqlResource);
List<SourceMasterEntityUpsert> buffer = new ArrayList<>(MASTER_DATA_UPSERT_BATCH_SIZE);
Map<String, Integer> typeCounts = new LinkedHashMap<>();
int[] count = {0};
int[] chunk = {0};
log.info("Reading tachograph master-data entities tenant={} section={}", tenantKey, section);
tachographJdbcTemplate.query(sql, Map.of(), rs -> {
SourceMasterEntityUpsert entity = entity(rs);
buffer.add(entity);
increment(typeCounts, entity.entityType());
if (buffer.size() >= MASTER_DATA_UPSERT_BATCH_SIZE) {
count[0] += flushEntities(tenantKey, eventSourceId, section, ++chunk[0], buffer, count[0], typeCounts);
buffer.clear();
}
});
if (!buffer.isEmpty()) {
count[0] += flushEntities(tenantKey, eventSourceId, section, ++chunk[0], buffer, count[0], typeCounts);
}
log.info("Finished tachograph master-data entities tenant={} section={} processed={} byType={}",
tenantKey, section, count[0], typeCounts);
return count[0];
}
private int streamRelations(
NamedParameterJdbcTemplate tachographJdbcTemplate,
String tenantKey,
int eventSourceId,
String sqlResource,
String sql
) {
String section = masterDataSection(sqlResource);
List<SourceMasterRelationUpsert> buffer = new ArrayList<>(MASTER_DATA_UPSERT_BATCH_SIZE);
Map<String, Integer> typeCounts = new LinkedHashMap<>();
int[] count = {0};
int[] chunk = {0};
log.info("Reading tachograph master-data relations tenant={} section={}", tenantKey, section);
tachographJdbcTemplate.query(sql, Map.of(), rs -> {
SourceMasterRelationUpsert relation = relation(rs);
buffer.add(relation);
increment(typeCounts, relation.relationType());
if (buffer.size() >= MASTER_DATA_UPSERT_BATCH_SIZE) {
count[0] += flushRelations(tenantKey, eventSourceId, section, ++chunk[0], buffer, count[0], typeCounts);
buffer.clear();
}
});
if (!buffer.isEmpty()) {
count[0] += flushRelations(tenantKey, eventSourceId, section, ++chunk[0], buffer, count[0], typeCounts);
}
log.info("Finished tachograph master-data relations tenant={} section={} processed={} byType={}",
tenantKey, section, count[0], typeCounts);
return count[0];
}
private int flushEntities(
String tenantKey,
int eventSourceId,
String section,
int chunk,
List<SourceMasterEntityUpsert> buffer,
int alreadyProcessed,
Map<String, Integer> typeCounts
) {
int upserted = sourceMasterDataRepository.upsertEntities(tenantKey, eventSourceId, buffer);
log.info("Tachograph master-data entity progress tenant={} section={} chunk={} chunkSize={} processed={} byType={}",
tenantKey, section, chunk, buffer.size(), alreadyProcessed + upserted, typeCounts);
return upserted;
}
private int flushRelations(
String tenantKey,
int eventSourceId,
String section,
int chunk,
List<SourceMasterRelationUpsert> buffer,
int alreadyProcessed,
Map<String, Integer> typeCounts
) {
int upserted = sourceMasterDataRepository.upsertRelations(tenantKey, eventSourceId, buffer);
log.info("Tachograph master-data relation progress tenant={} section={} chunk={} chunkSize={} processed={} byType={}",
tenantKey, section, chunk, buffer.size(), alreadyProcessed + upserted, typeCounts);
return upserted;
}
private void increment(Map<String, Integer> counts, String type) {
String key = type == null || type.isBlank() ? "UNKNOWN" : type;
counts.merge(key, 1, Integer::sum);
}
private String masterDataSection(String sqlResource) {
int slash = sqlResource.lastIndexOf('/');
String fileName = slash < 0 ? sqlResource : sqlResource.substring(slash + 1);
return fileName.endsWith(".sql") ? fileName.substring(0, fileName.length() - 4) : fileName;
}
private EventSourceDto masterDataSourceFor(EventSourceDto source) {
return new EventSourceDto(
source.providerKey(),
MASTER_DATA_SOURCE_KIND,
MASTER_DATA_SOURCE_KEY,
source.sourceInstanceKey(),
source.tenantProviderSettingKey(),
source.externalFleetKey()
);
}
private SourceMasterEntityUpsert entity(ResultSet rs) throws SQLException {
String entityType = string(rs, "entity_type");
String sourceEntityId = string(rs, "source_entity_id");
OffsetDateTime validFrom = offsetDateTime(rs, "valid_from");
OffsetDateTime validTo = offsetDateTime(rs, "valid_to");
ValidityRange validityRange = normalizeValidityRange(
validFrom,
validTo,
"entity",
entityType + ":" + sourceEntityId
);
return new SourceMasterEntityUpsert(
entityType,
sourceEntityId,
string(rs, "source_external_key"),
string(rs, "display_name"),
bool(rs, "active"),
validityRange.validFrom(),
validityRange.validTo(),
offsetDateTime(rs, "source_updated_at"),
payload(rs)
);
}
private SourceMasterRelationUpsert relation(ResultSet rs) throws SQLException {
String relationType = string(rs, "relation_type");
String fromEntityType = string(rs, "from_entity_type");
String fromSourceEntityId = string(rs, "from_source_entity_id");
String toEntityType = string(rs, "to_entity_type");
String toSourceEntityId = string(rs, "to_source_entity_id");
OffsetDateTime validFrom = offsetDateTime(rs, "valid_from");
OffsetDateTime validTo = offsetDateTime(rs, "valid_to");
ValidityRange validityRange = normalizeValidityRange(
validFrom,
validTo,
"relation",
relationType + ":" + fromEntityType + ":" + fromSourceEntityId + "->" + toEntityType + ":" + toSourceEntityId
);
return new SourceMasterRelationUpsert(
relationType,
fromEntityType,
fromSourceEntityId,
toEntityType,
toSourceEntityId,
validityRange.validFrom(),
validityRange.validTo(),
offsetDateTime(rs, "source_updated_at"),
payload(rs)
);
}
private Map<String, Object> payload(ResultSet rs) throws SQLException {
ResultSetMetaData metaData = rs.getMetaData();
Map<String, Object> payload = new LinkedHashMap<>();
for (int i = 1; i <= metaData.getColumnCount(); i++) {
String name = metaData.getColumnLabel(i);
Object value = rs.getObject(i);
if (value != null) {
payload.put(name, value);
}
}
return payload;
}
private String loadSql(String location) {
Resource resource = resourceLoader.getResource(location);
try (var inputStream = resource.getInputStream()) {
return StreamUtils.copyToString(inputStream, StandardCharsets.UTF_8);
} catch (IOException e) {
throw new IllegalStateException("Cannot load tachograph master-data SQL resource " + location, e);
}
}
private String string(ResultSet rs, String column) throws SQLException {
String value = rs.getString(column);
return value == null || value.isBlank() ? null : value.trim();
}
private Boolean bool(ResultSet rs, String column) throws SQLException {
Object value = rs.getObject(column);
if (value == null) {
return null;
}
if (value instanceof Boolean bool) {
return bool;
}
if (value instanceof Number number) {
return number.intValue() != 0;
}
return Boolean.parseBoolean(value.toString());
}
private OffsetDateTime offsetDateTime(ResultSet rs, String column) throws SQLException {
Object value = rs.getObject(column);
if (value == null) {
return null;
}
if (value instanceof OffsetDateTime offsetDateTime) {
return offsetDateTime.withOffsetSameInstant(ZoneOffset.UTC);
}
if (value instanceof Timestamp timestamp) {
return timestamp.toLocalDateTime().atOffset(ZoneOffset.UTC);
}
if (value instanceof java.time.LocalDateTime localDateTime) {
return localDateTime.atOffset(ZoneOffset.UTC);
}
String text = value.toString();
try {
return OffsetDateTime.parse(text).withOffsetSameInstant(ZoneOffset.UTC);
} catch (RuntimeException ignored) {
return LocalDateTime.parse(text).atOffset(ZoneOffset.UTC);
}
}
private ValidityRange normalizeValidityRange(
OffsetDateTime validFrom,
OffsetDateTime validTo,
String rowKind,
String rowKey
) {
if (validFrom == null || validTo == null || !validFrom.isAfter(validTo)) {
return new ValidityRange(validFrom, validTo);
}
log.warn(
"Ignoring invalid validity end for {} {} because valid_from {} is after valid_to {}. Keeping valid_from and setting valid_to=null.",
rowKind,
rowKey,
validFrom,
validTo
);
return new ValidityRange(validFrom, null);
}
private record ValidityRange(OffsetDateTime validFrom, OffsetDateTime validTo) {
}
}

View File

@ -0,0 +1,8 @@
package at.procon.eventhub.tachograph.service;
public class UnsupportedTachographExtractionException extends IllegalArgumentException {
public UnsupportedTachographExtractionException(String message) {
super(message);
}
}

View File

@ -1,10 +1,6 @@
package at.procon.eventhub.tachograph.service;
import at.procon.eventhub.service.EventDetailsFactory;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.LinkedHashMap;
import java.util.Map;
import org.springframework.stereotype.Component;
@Component
@ -13,11 +9,4 @@ public class VuActivityRowMapper extends AbstractTachographActivityRowMapper {
public VuActivityRowMapper(EventDetailsFactory detailsFactory) {
super(detailsFactory);
}
@Override
protected Map<String, Object> sourceSpecificPayload(ResultSet rs) throws SQLException {
Map<String, Object> raw = new LinkedHashMap<>();
put(raw, "vuActivityId", string(rs, "vu_activity_id"));
return raw;
}
}

View File

@ -0,0 +1,12 @@
package at.procon.eventhub.tachograph.service;
import at.procon.eventhub.service.EventDetailsFactory;
import org.springframework.stereotype.Component;
@Component
public class VuBorderCrossingRowMapper extends AbstractTachographBorderCrossingRowMapper {
public VuBorderCrossingRowMapper(EventDetailsFactory detailsFactory) {
super(detailsFactory);
}
}

View File

@ -0,0 +1,12 @@
package at.procon.eventhub.tachograph.service;
import at.procon.eventhub.service.EventDetailsFactory;
import org.springframework.stereotype.Component;
@Component
public class VuLoadUnloadRowMapper extends AbstractTachographLoadUnloadRowMapper {
public VuLoadUnloadRowMapper(EventDetailsFactory detailsFactory) {
super(detailsFactory);
}
}

View File

@ -0,0 +1,12 @@
package at.procon.eventhub.tachograph.service;
import at.procon.eventhub.service.EventDetailsFactory;
import org.springframework.stereotype.Component;
@Component
public class VuPlaceRowMapper extends AbstractTachographPlaceRowMapper {
public VuPlaceRowMapper(EventDetailsFactory detailsFactory) {
super(detailsFactory);
}
}

View File

@ -0,0 +1,12 @@
package at.procon.eventhub.tachograph.service;
import at.procon.eventhub.service.EventDetailsFactory;
import org.springframework.stereotype.Component;
@Component
public class VuPositionRowMapper extends AbstractTachographPositionRowMapper {
public VuPositionRowMapper(EventDetailsFactory detailsFactory) {
super(detailsFactory);
}
}

View File

@ -0,0 +1,12 @@
package at.procon.eventhub.tachograph.service;
import at.procon.eventhub.service.EventDetailsFactory;
import org.springframework.stereotype.Component;
@Component
public class VuSpecificConditionRowMapper extends AbstractTachographSpecificConditionRowMapper {
public VuSpecificConditionRowMapper(EventDetailsFactory detailsFactory) {
super(detailsFactory);
}
}

View File

@ -3,16 +3,33 @@ spring:
name: eventhub-ingestion-service
datasource:
url: jdbc:postgresql://localhost:5432/eventhub
username: eventhub
password: eventhub
username: postgres
password: P54!pcd#Wi
hikari:
maximum-pool-size: 16
minimum-idle: 4
connection-timeout: 30000
validation-timeout: 5000
idle-timeout: 60000
keepalive-time: 30000
max-lifetime: 90000
flyway:
enabled: true
default-schema: eventhub
schemas: eventhub
create-schemas: true
logging:
file:
name: ${EVENTHUB_LOG_FILE:logs/eventhub-ingestion-service.log}
logback:
rollingpolicy:
max-file-size: ${EVENTHUB_LOG_MAX_FILE_SIZE:50MB}
max-history: ${EVENTHUB_LOG_MAX_HISTORY:14}
total-size-cap: ${EVENTHUB_LOG_TOTAL_SIZE_CAP:1GB}
server:
port: 8080
port: 8085
camel:
springboot:
@ -27,39 +44,43 @@ management:
eventhub:
batch:
completion-size: 1000
completion-size: 5000
completion-timeout: 5s
queue-size: 10000
concurrent-consumers: 4
block-when-full: true
queue-offer-timeout: 5m
tachograph:
default-chunk-days: 1
occurred-at-overlap: 7d
# Configure this block to enable JdbcTachographExtractionBatchExecutor.
# datasource:
# jdbc-url: jdbc:sqlserver://localhost:1433;databaseName=tachograph;encrypt=true;trustServerCertificate=true
# username: tachograph_user
# password: change-me
# driver-class-name: com.microsoft.sqlserver.jdbc.SQLServerDriver
datasource:
jdbc-url: jdbc:sqlserver://db.bytebar.eu:22996;databaseName=ByteBarDriverSettlement;trustServerCertificate=true
username: ReadOnly
password: p2=race!
driver-class-name: com.microsoft.sqlserver.jdbc.SQLServerDriver
# Enables the scheduler that regularly triggers configured tachograph import plans.
scheduler-enabled: false
scheduler-enabled: true
scheduler-poll-interval-ms: 60000
# PLAN_ONLY creates import_run + planned extraction packages.
# EXECUTE also invokes the configured TachographExtractionBatchExecutor.
scheduler-trigger-mode: PLAN_ONLY
scheduler-trigger-mode: EXECUTE
# Example plan. Keep disabled until the tachograph datasource/extractor is wired.
import-plans:
- plan-key: kralowetz-tachograph-org-147
enabled: false
enabled: true
cron: "0 15 * * * *" # hourly at minute 15
tenant-key: kralowetz
event-source:
provider-key: TACHOGRAPH
source-kind: MIXED
source-key: TACHOGRAPH_DB
source-instance-key: tachograph-prod-db
tenant-provider-setting-key: kralowetz-tachograph-prod
source-instance-key: ByteBar-DriverSettlement
tenant-provider-setting-key: ByteBar-DriverSettlement
source-group:
type: ORGANISATION
source-entity-id: "147"
@ -88,7 +109,7 @@ eventhub:
scheduled-mode: INCREMENTAL_UPDATE
initial-strategy: OCCURRED_AT_WINDOW_WITH_OVERLAP
scheduled-strategy: SOURCE_PACKAGE_WATERMARK
refresh-master-data-first: true
initial-occurred-from: "2025-01-01T00:00:00+01:00"
initial-occurred-to: null
run-initial-on-startup: false
refresh-master-data-first: false
initial-occurred-from: "2026-01-21T00:00:00+01:00"
initial-occurred-to: "2026-01-25T00:00:00+01:00"
run-initial-on-startup: true

View File

@ -0,0 +1,346 @@
create extension if not exists pgcrypto;
create extension if not exists postgis;
create extension if not exists timescaledb;
drop schema if exists eventhub cascade;
create schema if not exists eventhub;
create table if not exists eventhub.event_source (
id integer generated always as identity primary key,
tenant_key text not null,
provider_key text not null,
source_kind text not null,
source_key text not null,
source_instance_key text not null default 'default',
tenant_provider_setting_key text,
external_fleet_key text,
created_at timestamptz not null default now(),
constraint ux_event_source unique (tenant_key, provider_key, source_kind, source_key, source_instance_key)
);
create table if not exists eventhub.import_run (
id uuid primary key,
tenant_key text not null,
event_source_id integer not null references eventhub.event_source(id),
mode text not null,
status text not null,
refresh_master_data_first boolean not null default true,
source_group_type text,
source_group_entity_id text,
source_group_code text,
source_group_name text,
import_scope_type text not null,
root_source_org_entity_id text,
root_source_org_code text,
root_source_org_name text,
include_children boolean not null default false,
occurred_from timestamptz,
occurred_to timestamptz,
requested_event_families text[] not null default '{}',
acquisition_strategy text,
metadata jsonb not null default '{}'::jsonb,
planned_package_count integer not null default 0,
started_at timestamptz not null default now(),
finished_at timestamptz,
error_message text,
constraint chk_import_run_occ_time_order check (occurred_from is null or occurred_to is null or occurred_from < occurred_to)
);
create table if not exists eventhub.import_cursor (
id uuid primary key,
tenant_key text not null,
event_source_id integer not null references eventhub.event_source(id),
scope_hash text not null,
event_family text not null,
source_kind text not null,
cursor_type text not null,
last_source_package_imported_at timestamptz,
last_source_package_id text,
last_source_row_updated_at timestamptz,
last_occurred_to timestamptz,
updated_at timestamptz not null default now(),
constraint ux_import_cursor unique (tenant_key, event_source_id, scope_hash, event_family, source_kind, cursor_type)
);
create table if not exists eventhub.data_package (
id uuid primary key,
event_source_id integer not null references eventhub.event_source(id),
import_run_id uuid references eventhub.import_run(id),
tenant_key text not null,
package_key text not null,
package_type text not null,
status text not null,
source_group_type text,
source_group_entity_id text,
source_group_code text,
source_group_name text,
import_scope_type text,
root_source_org_entity_id text,
root_source_org_code text,
root_source_org_name text,
include_children boolean not null default false,
occurred_from timestamptz,
occurred_to timestamptz,
event_family text,
business_date date,
external_package_id text,
extraction_code text,
extraction_source_kind text,
entity_axis text,
batch_no integer,
chunk_from timestamptz,
chunk_to timestamptz,
source_package_kind text,
source_package_id text,
source_package_entity_id text,
source_package_period_from timestamptz,
source_package_period_to timestamptz,
source_package_imported_at timestamptz,
received_at timestamptz not null default now(),
completed_at timestamptz,
event_count integer not null default 0,
metadata jsonb not null default '{}'::jsonb,
error_message text,
constraint ux_data_package_package_key unique (tenant_key, event_source_id, package_key),
constraint chk_data_package_occ_time_order check (occurred_from is null or occurred_to is null or occurred_from < occurred_to),
constraint chk_data_package_chunk_time_order check (chunk_from is null or chunk_to is null or chunk_from < chunk_to)
);
create table if not exists eventhub.source_master_entity (
id uuid primary key,
tenant_key text not null,
event_source_id integer not null references eventhub.event_source(id),
entity_type text not null,
source_entity_id text not null,
source_external_key text,
display_name text,
active boolean,
valid_from timestamptz,
valid_to timestamptz,
source_updated_at timestamptz,
payload jsonb not null default '{}'::jsonb,
created_at timestamptz not null default now(),
updated_at timestamptz not null default now(),
constraint ux_source_master_entity unique (tenant_key, event_source_id, entity_type, source_entity_id),
constraint chk_source_master_entity_valid_time_order check (valid_from is null or valid_to is null or valid_from <= valid_to)
);
create table if not exists eventhub.source_master_relation (
id uuid primary key,
tenant_key text not null,
event_source_id integer not null references eventhub.event_source(id),
relation_key text not null,
relation_type text not null,
from_entity_type text not null,
from_source_entity_id text not null,
to_entity_type text not null,
to_source_entity_id text not null,
valid_from timestamptz,
valid_to timestamptz,
source_updated_at timestamptz,
payload jsonb not null default '{}'::jsonb,
created_at timestamptz not null default now(),
updated_at timestamptz not null default now(),
constraint ux_source_master_relation unique (tenant_key, event_source_id, relation_key),
constraint chk_source_master_relation_valid_time_order check (valid_from is null or valid_to is null or valid_from <= valid_to)
);
create table if not exists eventhub.vehicle (
id uuid primary key,
tenant_key text not null,
event_source_id integer not null references eventhub.event_source(id),
source_vehicle_entity_id text,
vin text,
created_at timestamptz not null default now(),
updated_at timestamptz not null default now()
);
create table if not exists eventhub.vehicle_registration (
id uuid primary key,
tenant_key text not null,
event_source_id integer not null references eventhub.event_source(id),
source_registration_entity_id text,
nation text not null,
registration_number text not null,
source_updated_at timestamptz,
payload jsonb not null default '{}'::jsonb,
created_at timestamptz not null default now(),
updated_at timestamptz not null default now()
);
create table if not exists eventhub.vehicle_registration_assignment (
id uuid primary key,
tenant_key text not null,
event_source_id integer not null references eventhub.event_source(id),
vehicle_registration_id uuid not null references eventhub.vehicle_registration(id) on delete cascade,
vehicle_id uuid not null references eventhub.vehicle(id) on delete cascade,
valid_from timestamptz,
valid_to timestamptz,
source_updated_at timestamptz,
payload jsonb not null default '{}'::jsonb,
created_at timestamptz not null default now(),
updated_at timestamptz not null default now(),
constraint chk_vehicle_registration_assignment_valid_time_order check (valid_from is null or valid_to is null or valid_from <= valid_to)
);
create table if not exists eventhub.event (
id uuid not null,
event_source_id integer not null references eventhub.event_source(id),
data_package_id uuid not null references eventhub.data_package(id),
external_source_event_id text not null,
driver_entity_id uuid references eventhub.source_master_entity(id),
vehicle_id uuid references eventhub.vehicle(id),
vehicle_registration_id uuid references eventhub.vehicle_registration(id),
source_package_id text,
source_package_entity_id uuid references eventhub.source_master_entity(id),
occurred_at timestamptz not null,
received_partner_at timestamptz,
received_hub_at timestamptz not null default now(),
event_domain text not null,
event_type text not null,
lifecycle text not null,
odometer_m bigint,
position geography(Point, 4326),
payload jsonb not null default '{}'::jsonb,
manual_entry boolean not null default false,
source_record_key_hash text not null,
event_signature_hash text,
created_at timestamptz not null default now(),
constraint pk_event primary key (occurred_at, id),
constraint chk_event_driver_or_vehicle_ref check (
driver_entity_id is not null
or vehicle_id is not null
or vehicle_registration_id is not null
)
);
create table if not exists eventhub.event_source_record (
source_record_key_hash text primary key,
event_occurred_at timestamptz not null,
event_id uuid not null,
created_at timestamptz not null default now()
);
create table if not exists eventhub.event_detail (
event_occurred_at timestamptz not null,
event_id uuid not null,
detail_type text not null,
attributes jsonb not null default '{}'::jsonb,
created_at timestamptz not null default now(),
constraint pk_event_detail primary key (event_occurred_at, event_id, detail_type)
);
select create_hypertable(
'eventhub.event',
'occurred_at',
chunk_time_interval => interval '7 days',
if_not_exists => true
);
alter table eventhub.event_source_record
add constraint fk_event_source_record_event foreign key (event_occurred_at, event_id)
references eventhub.event(occurred_at, id)
on delete cascade
deferrable initially deferred;
alter table eventhub.event_detail
add constraint fk_event_detail_event foreign key (event_occurred_at, event_id)
references eventhub.event(occurred_at, id)
on delete cascade;
create index if not exists idx_data_package_source_time
on eventhub.data_package(tenant_key, event_source_id, received_at desc);
create index if not exists idx_data_package_scope
on eventhub.data_package(tenant_key, import_scope_type, root_source_org_entity_id, occurred_from, occurred_to);
create index if not exists idx_data_package_extraction
on eventhub.data_package(tenant_key, event_source_id, import_run_id, event_family, extraction_source_kind, extraction_code, batch_no);
create index if not exists idx_import_run_source_status
on eventhub.import_run(tenant_key, event_source_id, status, started_at desc);
create index if not exists idx_source_master_entity_type_key
on eventhub.source_master_entity(tenant_key, event_source_id, entity_type, source_external_key)
where source_external_key is not null;
create index if not exists idx_source_master_entity_payload_gin
on eventhub.source_master_entity using gin(payload);
create index if not exists idx_source_master_relation_from
on eventhub.source_master_relation(tenant_key, event_source_id, from_entity_type, from_source_entity_id, relation_type);
create index if not exists idx_source_master_relation_to
on eventhub.source_master_relation(tenant_key, event_source_id, to_entity_type, to_source_entity_id, relation_type);
create index if not exists idx_source_master_relation_payload_gin
on eventhub.source_master_relation using gin(payload);
create index if not exists idx_vehicle_lookup_ctx
on eventhub.vehicle(tenant_key, event_source_id, updated_at desc);
create index if not exists idx_vehicle_source_entity
on eventhub.vehicle(tenant_key, event_source_id, source_vehicle_entity_id)
where source_vehicle_entity_id is not null;
create index if not exists idx_vehicle_vin
on eventhub.vehicle(tenant_key, event_source_id, vin)
where vin is not null;
create index if not exists idx_vehicle_registration_source_entity
on eventhub.vehicle_registration(tenant_key, event_source_id, source_registration_entity_id)
where source_registration_entity_id is not null;
create index if not exists idx_vehicle_registration_plate
on eventhub.vehicle_registration(tenant_key, event_source_id, nation, registration_number);
create index if not exists idx_vehicle_registration_assignment_registration_time
on eventhub.vehicle_registration_assignment(vehicle_registration_id, valid_from desc, valid_to);
create index if not exists idx_vehicle_registration_assignment_vehicle_time
on eventhub.vehicle_registration_assignment(vehicle_id, valid_from desc, valid_to);
create index if not exists idx_event_source_record_event
on eventhub.event_source_record(event_occurred_at, event_id);
create index if not exists idx_event_signature
on eventhub.event(event_signature_hash)
where event_signature_hash is not null;
create index if not exists idx_event_source_time
on eventhub.event(event_source_id, occurred_at desc);
create index if not exists idx_event_package_time
on eventhub.event(data_package_id, occurred_at desc);
create index if not exists idx_event_source_package_id
on eventhub.event(source_package_id)
where source_package_id is not null;
create index if not exists idx_event_domain_type_time
on eventhub.event(event_domain, event_type, occurred_at desc);
create index if not exists idx_event_driver_time
on eventhub.event(driver_entity_id, occurred_at desc)
where driver_entity_id is not null;
create index if not exists idx_event_vehicle_time
on eventhub.event(vehicle_id, occurred_at desc)
where vehicle_id is not null;
create index if not exists idx_event_vehicle_registration_time
on eventhub.event(vehicle_registration_id, occurred_at desc)
where vehicle_registration_id is not null;
create index if not exists idx_event_position_gist
on eventhub.event using gist(position)
where position is not null;
create index if not exists idx_event_payload_gin
on eventhub.event using gin(payload);
create index if not exists idx_event_detail_type
on eventhub.event_detail(detail_type);
create index if not exists idx_event_detail_attributes_gin
on eventhub.event_detail using gin(attributes);

View File

@ -0,0 +1,54 @@
create table if not exists eventhub.source_master_entity (
id uuid primary key,
tenant_key text not null,
event_source_id integer not null references eventhub.event_source(id),
entity_type text not null,
source_entity_id text not null,
source_external_key text,
display_name text,
active boolean,
valid_from timestamptz,
valid_to timestamptz,
source_updated_at timestamptz,
payload jsonb not null default '{}'::jsonb,
created_at timestamptz not null default now(),
updated_at timestamptz not null default now(),
constraint ux_source_master_entity unique (tenant_key, event_source_id, entity_type, source_entity_id),
constraint chk_source_master_entity_valid_time_order check (valid_from is null or valid_to is null or valid_from < valid_to)
);
create table if not exists eventhub.source_master_relation (
id uuid primary key,
tenant_key text not null,
event_source_id integer not null references eventhub.event_source(id),
relation_key text not null,
relation_type text not null,
from_entity_type text not null,
from_source_entity_id text not null,
to_entity_type text not null,
to_source_entity_id text not null,
valid_from timestamptz,
valid_to timestamptz,
source_updated_at timestamptz,
payload jsonb not null default '{}'::jsonb,
created_at timestamptz not null default now(),
updated_at timestamptz not null default now(),
constraint ux_source_master_relation unique (tenant_key, event_source_id, relation_key),
constraint chk_source_master_relation_valid_time_order check (valid_from is null or valid_to is null or valid_from < valid_to)
);
create index if not exists idx_source_master_entity_type_key
on eventhub.source_master_entity(tenant_key, event_source_id, entity_type, source_external_key)
where source_external_key is not null;
create index if not exists idx_source_master_entity_payload_gin
on eventhub.source_master_entity using gin(payload);
create index if not exists idx_source_master_relation_from
on eventhub.source_master_relation(tenant_key, event_source_id, from_entity_type, from_source_entity_id, relation_type);
create index if not exists idx_source_master_relation_to
on eventhub.source_master_relation(tenant_key, event_source_id, to_entity_type, to_source_entity_id, relation_type);
create index if not exists idx_source_master_relation_payload_gin
on eventhub.source_master_relation using gin(payload);

View File

@ -0,0 +1,13 @@
alter table eventhub.source_master_entity
drop constraint if exists chk_source_master_entity_valid_time_order;
alter table eventhub.source_master_entity
add constraint chk_source_master_entity_valid_time_order
check (valid_from is null or valid_to is null or valid_from <= valid_to);
alter table eventhub.source_master_relation
drop constraint if exists chk_source_master_relation_valid_time_order;
alter table eventhub.source_master_relation
add constraint chk_source_master_relation_valid_time_order
check (valid_from is null or valid_to is null or valid_from <= valid_to);

View File

@ -0,0 +1,91 @@
create extension if not exists postgis;
drop table if exists eventhub.event_detail;
drop table if exists eventhub.acquired_event;
drop table if exists eventhub.event;
create table eventhub.event (
id uuid not null,
event_source_id integer not null references eventhub.event_source(id),
data_package_id uuid not null references eventhub.data_package(id),
external_source_event_id text not null,
driver_entity_id uuid references eventhub.source_master_entity(id),
vehicle_entity_id uuid references eventhub.source_master_entity(id),
source_package_entity_id uuid references eventhub.source_master_entity(id),
occurred_at timestamptz not null,
received_partner_at timestamptz,
received_hub_at timestamptz not null default now(),
event_domain text not null,
event_type text not null,
lifecycle text not null,
odometer_m bigint,
position geography(Point, 4326),
payload jsonb not null default '{}'::jsonb,
manual_entry boolean not null default false,
source_record_key_hash text not null,
event_signature_hash text,
created_at timestamptz not null default now(),
constraint pk_event primary key (occurred_at, id),
constraint chk_event_driver_or_vehicle_ref check (
driver_entity_id is not null
or vehicle_entity_id is not null
)
);
create unique index ux_event_source_record
on eventhub.event(source_record_key_hash);
create index idx_event_signature
on eventhub.event(event_signature_hash)
where event_signature_hash is not null;
create index idx_event_source_time
on eventhub.event(event_source_id, occurred_at desc);
create index idx_event_package_time
on eventhub.event(data_package_id, occurred_at desc);
create index idx_event_domain_type_time
on eventhub.event(event_domain, event_type, occurred_at desc);
create index idx_event_driver_time
on eventhub.event(driver_entity_id, occurred_at desc)
where driver_entity_id is not null;
create index idx_event_vehicle_time
on eventhub.event(vehicle_entity_id, occurred_at desc)
where vehicle_entity_id is not null;
create index idx_event_position_gist
on eventhub.event using gist(position)
where position is not null;
create index idx_event_payload_gin
on eventhub.event using gin(payload);
create table eventhub.event_detail (
event_occurred_at timestamptz not null,
event_id uuid not null,
detail_type text not null,
attributes jsonb not null default '{}'::jsonb,
created_at timestamptz not null default now(),
constraint pk_event_detail primary key (event_occurred_at, event_id, detail_type),
constraint fk_event_detail_event foreign key (event_occurred_at, event_id)
references eventhub.event(occurred_at, id)
on delete cascade
);
create index idx_event_detail_type
on eventhub.event_detail(detail_type);
create index idx_event_detail_attributes_gin
on eventhub.event_detail using gin(attributes);

View File

@ -0,0 +1,75 @@
create table eventhub.vehicle (
id uuid primary key,
tenant_key text not null,
event_source_id integer not null references eventhub.event_source(id),
created_at timestamptz not null default now(),
updated_at timestamptz not null default now()
);
create table eventhub.vehicle_identifier (
id uuid primary key,
vehicle_id uuid not null references eventhub.vehicle(id) on delete cascade,
tenant_key text not null,
event_source_id integer not null references eventhub.event_source(id),
identifier_type text not null,
nation text,
identifier_value text not null,
valid_from timestamptz,
valid_to timestamptz,
source_updated_at timestamptz,
payload jsonb not null default '{}'::jsonb,
created_at timestamptz not null default now(),
updated_at timestamptz not null default now(),
constraint chk_vehicle_identifier_valid_time_order
check (valid_from is null or valid_to is null or valid_from <= valid_to),
constraint chk_vehicle_identifier_nation_for_vrn
check (
identifier_type <> 'VRN'
or nation is not null
)
);
create index idx_vehicle_lookup_ctx
on eventhub.vehicle(tenant_key, event_source_id, updated_at desc);
create index idx_vehicle_identifier_lookup
on eventhub.vehicle_identifier(
tenant_key, event_source_id, identifier_type, nation, identifier_value, valid_from desc
);
create unique index ux_vehicle_identifier_exact
on eventhub.vehicle_identifier(
vehicle_id,
identifier_type,
coalesce(nation, ''),
identifier_value,
coalesce(valid_from, '-infinity'::timestamptz),
coalesce(valid_to, 'infinity'::timestamptz)
);
alter table eventhub.event
rename column vehicle_entity_id to vehicle_id;
alter table eventhub.event
drop constraint if exists event_vehicle_entity_id_fkey;
alter table eventhub.event
add constraint fk_event_vehicle
foreign key (vehicle_id)
references eventhub.vehicle(id);
drop index if exists eventhub.idx_event_vehicle_time;
create index idx_event_vehicle_time
on eventhub.event(vehicle_id, occurred_at desc)
where vehicle_id is not null;
alter table eventhub.event
drop constraint if exists chk_event_driver_or_vehicle_ref;
alter table eventhub.event
add constraint chk_event_driver_or_vehicle_ref
check (
driver_entity_id is not null
or vehicle_id is not null
);

View File

@ -0,0 +1,241 @@
alter table eventhub.vehicle
add column if not exists source_vehicle_entity_id text,
add column if not exists vin text;
do $migration$
begin
if to_regclass('eventhub.vehicle_identifier') is not null
and exists (
select 1
from information_schema.columns
where table_schema = 'eventhub'
and table_name = 'vehicle_identifier'
and column_name = 'vehicle_id'
)
then
execute $sql$
update eventhub.vehicle v
set source_vehicle_entity_id = src.identifier_value
from eventhub.vehicle_identifier src
where src.vehicle_id = v.id
and src.identifier_type = 'SOURCE_VEHICLE'
and v.source_vehicle_entity_id is null
$sql$;
execute $sql$
update eventhub.vehicle v
set vin = src.identifier_value
from eventhub.vehicle_identifier src
where src.vehicle_id = v.id
and src.identifier_type = 'VIN'
and v.vin is null
$sql$;
end if;
end
$migration$;
create table if not exists eventhub.vehicle_registration (
id uuid primary key,
tenant_key text not null,
event_source_id integer not null references eventhub.event_source(id),
source_registration_entity_id text,
nation text not null,
registration_number text not null,
valid_from timestamptz,
valid_to timestamptz,
source_updated_at timestamptz,
payload jsonb not null default '{}'::jsonb,
created_at timestamptz not null default now(),
updated_at timestamptz not null default now(),
constraint chk_vehicle_registration_valid_time_order
check (valid_from is null or valid_to is null or valid_from <= valid_to)
);
create table if not exists eventhub.vehicle_registration_assignment (
id uuid primary key,
tenant_key text not null,
event_source_id integer not null references eventhub.event_source(id),
vehicle_registration_id uuid not null references eventhub.vehicle_registration(id) on delete cascade,
vehicle_id uuid not null references eventhub.vehicle(id) on delete cascade,
valid_from timestamptz,
valid_to timestamptz,
source_updated_at timestamptz,
payload jsonb not null default '{}'::jsonb,
created_at timestamptz not null default now(),
updated_at timestamptz not null default now(),
constraint chk_vehicle_registration_assignment_valid_time_order
check (valid_from is null or valid_to is null or valid_from <= valid_to)
);
do $migration$
begin
if to_regclass('eventhub.vehicle_identifier') is not null
and exists (
select 1
from information_schema.columns
where table_schema = 'eventhub'
and table_name = 'vehicle_identifier'
and column_name = 'vehicle_id'
)
then
execute $sql$
insert into eventhub.vehicle_registration(
id, tenant_key, event_source_id, source_registration_entity_id, nation, registration_number,
valid_from, valid_to, payload
)
select gen_random_uuid(),
src.tenant_key,
src.event_source_id,
source_reg.identifier_value,
src.nation,
src.identifier_value,
src.valid_from,
src.valid_to,
src.payload
from eventhub.vehicle_identifier src
left join eventhub.vehicle_identifier source_reg on source_reg.vehicle_id = src.vehicle_id
and source_reg.identifier_type = 'SOURCE_REGISTRATION'
where src.identifier_type = 'VRN'
and src.nation is not null
and not exists (
select 1
from eventhub.vehicle_registration existing
where existing.tenant_key = src.tenant_key
and existing.event_source_id = src.event_source_id
and existing.nation = src.nation
and existing.registration_number = src.identifier_value
and existing.valid_from is not distinct from src.valid_from
and existing.valid_to is not distinct from src.valid_to
)
$sql$;
execute $sql$
insert into eventhub.vehicle_registration_assignment(
id, tenant_key, event_source_id, vehicle_registration_id, vehicle_id, valid_from, valid_to, payload
)
select gen_random_uuid(),
reg.tenant_key,
reg.event_source_id,
reg.id,
v.id,
reg.valid_from,
reg.valid_to,
jsonb_build_object('migrated_from', 'vehicle_identifier')
from eventhub.vehicle_registration reg
join eventhub.vehicle_identifier vrn on vrn.tenant_key = reg.tenant_key
and vrn.event_source_id = reg.event_source_id
and vrn.identifier_type = 'VRN'
and vrn.nation = reg.nation
and vrn.identifier_value = reg.registration_number
and vrn.valid_from is not distinct from reg.valid_from
and vrn.valid_to is not distinct from reg.valid_to
join eventhub.vehicle v on v.id = vrn.vehicle_id
where not exists (
select 1
from eventhub.vehicle_registration_assignment existing
where existing.vehicle_registration_id = reg.id
and existing.vehicle_id = v.id
and existing.valid_from is not distinct from reg.valid_from
and existing.valid_to is not distinct from reg.valid_to
)
$sql$;
end if;
end
$migration$;
do $migration$
begin
if exists (
select 1
from information_schema.columns
where table_schema = 'eventhub'
and table_name = 'event'
and column_name = 'vehicle_entity_id'
)
and not exists (
select 1
from information_schema.columns
where table_schema = 'eventhub'
and table_name = 'event'
and column_name = 'vehicle_id'
)
then
alter table eventhub.event rename column vehicle_entity_id to vehicle_id;
end if;
if not exists (
select 1
from information_schema.columns
where table_schema = 'eventhub'
and table_name = 'event'
and column_name = 'vehicle_id'
)
then
alter table eventhub.event add column vehicle_id uuid references eventhub.vehicle(id);
end if;
end
$migration$;
alter table eventhub.event
drop constraint if exists event_vehicle_entity_id_fkey;
alter table eventhub.event
drop constraint if exists fk_event_vehicle;
do $migration$
begin
if exists (
select 1
from information_schema.columns
where table_schema = 'eventhub'
and table_name = 'event'
and column_name = 'vehicle_id'
)
then
alter table eventhub.event
add constraint fk_event_vehicle
foreign key (vehicle_id)
references eventhub.vehicle(id)
not valid;
end if;
end
$migration$;
alter table eventhub.event
add column if not exists vehicle_registration_id uuid references eventhub.vehicle_registration(id);
create index if not exists idx_vehicle_source_entity
on eventhub.vehicle(tenant_key, event_source_id, source_vehicle_entity_id)
where source_vehicle_entity_id is not null;
create index if not exists idx_vehicle_vin
on eventhub.vehicle(tenant_key, event_source_id, vin)
where vin is not null;
create index if not exists idx_vehicle_registration_source_entity
on eventhub.vehicle_registration(tenant_key, event_source_id, source_registration_entity_id)
where source_registration_entity_id is not null;
create index if not exists idx_vehicle_registration_plate
on eventhub.vehicle_registration(tenant_key, event_source_id, nation, registration_number, valid_from desc);
create index if not exists idx_vehicle_registration_assignment_registration_time
on eventhub.vehicle_registration_assignment(vehicle_registration_id, valid_from desc, valid_to);
create index if not exists idx_vehicle_registration_assignment_vehicle_time
on eventhub.vehicle_registration_assignment(vehicle_id, valid_from desc, valid_to);
create index if not exists idx_event_vehicle_registration_time
on eventhub.event(vehicle_registration_id, occurred_at desc)
where vehicle_registration_id is not null;
alter table eventhub.event
drop constraint if exists chk_event_driver_or_vehicle_ref;
alter table eventhub.event
add constraint chk_event_driver_or_vehicle_ref
check (
driver_entity_id is not null
or vehicle_id is not null
or vehicle_registration_id is not null
);

View File

@ -0,0 +1,24 @@
alter table eventhub.event
drop constraint if exists event_vehicle_entity_id_fkey;
alter table eventhub.event
drop constraint if exists fk_event_vehicle;
do $migration$
begin
if exists (
select 1
from information_schema.columns
where table_schema = 'eventhub'
and table_name = 'event'
and column_name = 'vehicle_id'
)
then
alter table eventhub.event
add constraint fk_event_vehicle
foreign key (vehicle_id)
references eventhub.vehicle(id)
not valid;
end if;
end
$migration$;

View File

@ -0,0 +1,193 @@
drop index if exists eventhub.idx_vehicle_registration_plate;
alter table eventhub.vehicle_registration
drop constraint if exists chk_vehicle_registration_valid_time_order;
alter table eventhub.vehicle_registration
drop column if exists valid_from,
drop column if exists valid_to;
create index if not exists idx_vehicle_registration_plate
on eventhub.vehicle_registration(tenant_key, event_source_id, nation, registration_number);
update eventhub.source_master_entity
set valid_from = null,
valid_to = null,
updated_at = now()
where entity_type = 'VEHICLE_REGISTRATION'
and (valid_from is not null or valid_to is not null);
with vehicle_entities as (
select tenant_key,
event_source_id,
nullif(source_entity_id, '') as source_vehicle_entity_id,
nullif(source_external_key, '') as vin
from eventhub.source_master_entity
where entity_type = 'VEHICLE'
and (nullif(source_entity_id, '') is not null or nullif(source_external_key, '') is not null)
)
update eventhub.vehicle vehicle
set source_vehicle_entity_id = coalesce(vehicle.source_vehicle_entity_id, vehicle_entities.source_vehicle_entity_id),
vin = coalesce(vehicle.vin, vehicle_entities.vin),
updated_at = now()
from vehicle_entities
where vehicle.tenant_key = vehicle_entities.tenant_key
and vehicle.event_source_id = vehicle_entities.event_source_id
and (
vehicle.source_vehicle_entity_id = vehicle_entities.source_vehicle_entity_id
or vehicle.vin = vehicle_entities.vin
);
with vehicle_entities as (
select tenant_key,
event_source_id,
nullif(source_entity_id, '') as source_vehicle_entity_id,
nullif(source_external_key, '') as vin
from eventhub.source_master_entity
where entity_type = 'VEHICLE'
and (nullif(source_entity_id, '') is not null or nullif(source_external_key, '') is not null)
)
insert into eventhub.vehicle(id, tenant_key, event_source_id, source_vehicle_entity_id, vin)
select gen_random_uuid(),
vehicle_entities.tenant_key,
vehicle_entities.event_source_id,
vehicle_entities.source_vehicle_entity_id,
vehicle_entities.vin
from vehicle_entities
where not exists (
select 1
from eventhub.vehicle vehicle
where vehicle.tenant_key = vehicle_entities.tenant_key
and vehicle.event_source_id = vehicle_entities.event_source_id
and (
vehicle.source_vehicle_entity_id = vehicle_entities.source_vehicle_entity_id
or vehicle.vin = vehicle_entities.vin
)
);
with registration_entities as (
select tenant_key,
event_source_id,
nullif(source_entity_id, '') as source_registration_entity_id,
coalesce(
nullif(payload ->> 'registration_nation', ''),
nullif(split_part(source_external_key, ':', 1), '')
) as nation,
coalesce(
nullif(payload ->> 'registration_number', ''),
case
when position(':' in source_external_key) > 0
then nullif(substring(source_external_key from position(':' in source_external_key) + 1), '')
else nullif(source_external_key, '')
end
) as registration_number,
source_updated_at
from eventhub.source_master_entity
where entity_type = 'VEHICLE_REGISTRATION'
)
update eventhub.vehicle_registration registration
set source_registration_entity_id = coalesce(
registration.source_registration_entity_id,
registration_entities.source_registration_entity_id
),
nation = coalesce(registration_entities.nation, registration.nation),
registration_number = coalesce(registration_entities.registration_number, registration.registration_number),
source_updated_at = registration_entities.source_updated_at,
updated_at = now()
from registration_entities
where registration_entities.nation is not null
and registration_entities.registration_number is not null
and registration.tenant_key = registration_entities.tenant_key
and registration.event_source_id = registration_entities.event_source_id
and (
registration.source_registration_entity_id = registration_entities.source_registration_entity_id
or (
registration.nation = registration_entities.nation
and registration.registration_number = registration_entities.registration_number
)
);
with registration_entities as (
select tenant_key,
event_source_id,
nullif(source_entity_id, '') as source_registration_entity_id,
coalesce(
nullif(payload ->> 'registration_nation', ''),
nullif(split_part(source_external_key, ':', 1), '')
) as nation,
coalesce(
nullif(payload ->> 'registration_number', ''),
case
when position(':' in source_external_key) > 0
then nullif(substring(source_external_key from position(':' in source_external_key) + 1), '')
else nullif(source_external_key, '')
end
) as registration_number,
source_updated_at
from eventhub.source_master_entity
where entity_type = 'VEHICLE_REGISTRATION'
)
insert into eventhub.vehicle_registration(
id, tenant_key, event_source_id, source_registration_entity_id,
nation, registration_number, source_updated_at, payload
)
select gen_random_uuid(),
registration_entities.tenant_key,
registration_entities.event_source_id,
registration_entities.source_registration_entity_id,
registration_entities.nation,
registration_entities.registration_number,
registration_entities.source_updated_at,
jsonb_build_object('source', 'master-data')
from registration_entities
where registration_entities.nation is not null
and registration_entities.registration_number is not null
and not exists (
select 1
from eventhub.vehicle_registration registration
where registration.tenant_key = registration_entities.tenant_key
and registration.event_source_id = registration_entities.event_source_id
and (
registration.source_registration_entity_id = registration_entities.source_registration_entity_id
or (
registration.nation = registration_entities.nation
and registration.registration_number = registration_entities.registration_number
)
)
);
insert into eventhub.vehicle_registration_assignment(
id, tenant_key, event_source_id, vehicle_registration_id, vehicle_id,
valid_from, valid_to, source_updated_at, payload
)
select gen_random_uuid(),
relation.tenant_key,
relation.event_source_id,
registration.id,
vehicle.id,
relation.valid_from,
relation.valid_to,
relation.source_updated_at,
jsonb_build_object(
'source', 'master-data',
'sourceRelationId', relation.id,
'relationKey', relation.relation_key
)
from eventhub.source_master_relation relation
join eventhub.vehicle_registration registration on registration.tenant_key = relation.tenant_key
and registration.event_source_id = relation.event_source_id
and registration.source_registration_entity_id = relation.from_source_entity_id
join eventhub.vehicle vehicle on vehicle.tenant_key = relation.tenant_key
and vehicle.event_source_id = relation.event_source_id
and vehicle.source_vehicle_entity_id = relation.to_source_entity_id
where relation.relation_type = 'VEHICLE_REGISTRATION_VEHICLE'
and relation.from_entity_type = 'VEHICLE_REGISTRATION'
and relation.to_entity_type = 'VEHICLE'
and not exists (
select 1
from eventhub.vehicle_registration_assignment existing
where existing.vehicle_registration_id = registration.id
and existing.vehicle_id = vehicle.id
and existing.valid_from is not distinct from relation.valid_from
and existing.valid_to is not distinct from relation.valid_to
);

View File

@ -0,0 +1,37 @@
<?xml version="1.0" encoding="UTF-8"?>
<configuration>
<springProperty scope="context" name="logFile" source="logging.file.name" defaultValue="logs/eventhub-ingestion-service.log"/>
<springProperty scope="context" name="maxFileSize" source="logging.logback.rollingpolicy.max-file-size" defaultValue="50MB"/>
<springProperty scope="context" name="maxHistory" source="logging.logback.rollingpolicy.max-history" defaultValue="14"/>
<springProperty scope="context" name="totalSizeCap" source="logging.logback.rollingpolicy.total-size-cap" defaultValue="1GB"/>
<appender name="CONSOLE" class="ch.qos.logback.core.ConsoleAppender">
<encoder>
<pattern>%d{yyyy-MM-dd'T'HH:mm:ss.SSSXXX} %-5level ${PID:-unknown} --- [%thread] %logger{39} : %msg%n%ex</pattern>
<charset>UTF-8</charset>
</encoder>
</appender>
<appender name="FILE" class="ch.qos.logback.core.rolling.RollingFileAppender">
<file>${logFile}</file>
<append>true</append>
<immediateFlush>true</immediateFlush>
<encoder>
<pattern>%d{yyyy-MM-dd'T'HH:mm:ss.SSSXXX} %-5level ${PID:-unknown} --- [%thread] %logger{39} : %msg%n%ex</pattern>
<charset>UTF-8</charset>
</encoder>
<rollingPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy">
<fileNamePattern>${logFile}.%d{yyyy-MM-dd}.%i.gz</fileNamePattern>
<maxFileSize>${maxFileSize}</maxFileSize>
<maxHistory>${maxHistory}</maxHistory>
<totalSizeCap>${totalSizeCap}</totalSizeCap>
</rollingPolicy>
</appender>
<root level="INFO">
<appender-ref ref="CONSOLE"/>
<appender-ref ref="FILE"/>
</root>
</configuration>

View File

@ -8,16 +8,104 @@
* the best matching CardVehiclesUsed row for the activity timestamp when one is
* available.
*/
select
cast(ca.ID as varchar(128)) as source_row_id,
cast(ca.ID as varchar(128)) as card_activity_id,
concat('TACHOGRAPH:CARD_ACTIVITY:', ca.ID) as external_source_event_id,
with OrgTree as (
select org.I_90021_OID
from dbo.GetOrganisationTree(null, :organisationId, 0, null) org
where :organisationId is not null
)
,
CandidateActivity as (
select
ca.ID,
ca.BeginTime,
activity_time.EndTime,
ca.Activity,
ca.Slot,
ca.CardStatus,
ca.DrivingStatus,
ca.ID_FileLog,
cda.RecordDate,
cda.RecordDateTo,
cda.ID_FileLog as cda_filelog_id,
c.ID as card_id,
c.ID_Driver as driver_id,
c.ID_Nation as driver_card_nation_id,
c.CardNumber as driver_card_number,
c.ID_FileLog as card_filelog_id,
coalesce(ca.ID_FileLog, cda.ID_FileLog, c.ID_FileLog) as file_log_id
from dbo.CardActivity ca
join dbo.CardDailyActivity cda on cda.ID = ca.ID_DailyActivity
join dbo.Card c on c.ID = cda.ID_Card
cross apply (values (dateadd(minute, coalesce(ca.Duration, 0), ca.BeginTime))) activity_time(EndTime)
where (:occurredTo is null or ca.BeginTime < :occurredTo)
and (
:occurredFrom is null
or ca.BeginTime >= :occurredFrom
or activity_time.EndTime >= :occurredFrom
)
and (
:organisationId is null
or exists (
select 1
from dbo.Driver_I_90021 rel
join OrgTree on OrgTree.I_90021_OID = rel.ID_I_90021
where rel.ID_Driver = c.ID_Driver
and rel.GILT_BIS is null
)
)
)
,
Base as (
select
ca.ID,
ca.BeginTime,
ca.EndTime,
ca.Activity,
ca.Slot,
ca.CardStatus,
ca.DrivingStatus,
ca.ID_FileLog,
ca.cda_filelog_id,
ca.card_filelog_id,
ca.BeginTime as occurred_at,
coalesce(fl.DownloadDate, fl.OriginalDownloadDate, fl.TStamp, fl.CreationDate) as received_partner_at,
ca.Activity as activity_code,
ca.Activity as activity_text,
case upper(coalesce(ca.Activity, ''))
coalesce(fl.DownloadDate, fl.OriginalDownloadDate, fl.TStamp, fl.CreationDate) as received_partner_at,
coalesce(fl.ID, ca.ID_FileLog, ca.cda_filelog_id, ca.card_filelog_id) as source_package_id_raw,
coalesce(fl.ID_Card, ca.card_id) as source_package_entity_id_raw,
coalesce(fl.DownloadFrom, ca.RecordDate) as source_package_period_from,
coalesce(fl.DownloadTo, ca.RecordDateTo, dateadd(day, 1, ca.RecordDate)) as source_package_period_to,
coalesce(fl.CreationDate, fl.TStamp) as source_package_imported_at,
ca.driver_id,
cn.AlphaCode as driver_card_nation,
ca.driver_card_number,
coalesce(cvu.ID_Vehicle, v.ID) as vehicle_registration_id,
null as vehicle_vin,
vn.AlphaCode as vehicle_registration_nation,
v.VRN as vehicle_registration_number
from CandidateActivity ca
left join dbo.FileLog fl on fl.ID = ca.file_log_id
left join dbo.Nation cn on cn.ID = ca.driver_card_nation_id
outer apply (
select top 1 used.ID_Vehicle
from dbo.CardVehiclesUsed used
where used.ID_Card = ca.card_id
and (used.FirstUse is null or used.FirstUse <= ca.BeginTime)
and (used.LastUse is null or used.LastUse >= ca.BeginTime)
order by
used.FirstUse desc,
used.ID desc
) cvu
left join dbo.Vehicle v on v.ID = cvu.ID_Vehicle
left join dbo.Nation vn on vn.ID = v.ID_Nation
)
select
cast(base.ID as varchar(128)) as source_row_id,
concat('TACHOGRAPH:CARD_ACTIVITY:', base.ID, ':', evt.lifecycle) as external_source_event_id,
evt.occurred_at as occurred_at,
base.received_partner_at,
base.Activity as activity_code,
case upper(coalesce(base.Activity, ''))
when 'DRIVING' then 'DRIVE'
when 'DRIVE' then 'DRIVE'
when 'WORK' then 'WORK'
@ -28,67 +116,35 @@ select
when 'REST' then 'BREAK_REST'
else 'UNKNOWN_ACTIVITY'
end as event_type,
'SNAPSHOT' as lifecycle,
ca.Slot as card_slot,
ca.CardStatus as card_status,
ca.DrivingStatus as driving_status,
evt.lifecycle as lifecycle,
base.Slot as card_slot,
base.CardStatus as card_status,
base.DrivingStatus as driving_status,
cast(null as bigint) as odometer_m,
cast(d.ID as varchar(128)) as driver_source_entity_id,
cn.AlphaCode as driver_card_nation,
c.CardNumber as driver_card_number,
cast(base.driver_id as varchar(128)) as driver_source_entity_id,
base.driver_card_nation,
base.driver_card_number,
cast(coalesce(cvu.ID_Vehicle, v.ID) as varchar(128)) as vehicle_source_entity_id,
coalesce(cvu.VIN, vi.VIN) as vehicle_vin,
vn.AlphaCode as vehicle_registration_nation,
v.VRN as vehicle_registration_number,
cast(null as varchar(128)) as vehicle_source_entity_id,
base.vehicle_vin,
cast(base.vehicle_registration_id as varchar(128)) as vehicle_registration_source_entity_id,
base.vehicle_registration_nation,
base.vehicle_registration_number,
'DRIVER_CARD' as source_package_kind,
cast(coalesce(fl.ID, ca.ID_FileLog, cda.ID_FileLog, c.ID_FileLog, ca.ID) as varchar(128)) as source_package_id,
cast(coalesce(fl.ID_Card, c.ID) as varchar(128)) as source_package_entity_id,
coalesce(fl.DownloadFrom, cda.RecordDate) as source_package_period_from,
coalesce(fl.DownloadTo, cda.RecordDateTo, dateadd(day, 1, cda.RecordDate)) as source_package_period_to,
coalesce(fl.CreationDate, fl.TStamp) as source_package_imported_at
from dbo.CardActivity ca
join dbo.CardDailyActivity cda on cda.ID = ca.ID_DailyActivity
join dbo.Card c on c.ID = cda.ID_Card
left join dbo.FileLog fl on fl.ID = coalesce(ca.ID_FileLog, cda.ID_FileLog, c.ID_FileLog)
left join dbo.Driver d on d.ID = c.ID_Driver
left join dbo.Nation cn on cn.ID = c.ID_Nation
outer apply (
select top 1 used.ID_Vehicle,
used.VIN,
used.OdoBegin
from dbo.CardVehiclesUsed used
where used.ID_Card = c.ID
and (used.FirstUse is null or used.FirstUse <= ca.BeginTime)
and (used.LastUse is null or used.LastUse >= ca.BeginTime)
order by
case when used.FirstUse is null then 1 else 0 end,
used.FirstUse desc,
used.ID desc
) cvu
left join dbo.Vehicle v on v.ID = cvu.ID_Vehicle
left join dbo.VehicleIdentification vi on vi.ID = v.ID_VehicleIdentification
left join dbo.Nation vn on vn.ID = v.ID_Nation
where (:occurredFrom is null or ca.BeginTime >= :occurredFrom)
and (:occurredTo is null or ca.BeginTime < :occurredTo)
and (
:lastSourcePackageImportedAt is null
or coalesce(fl.CreationDate, fl.TStamp) > :lastSourcePackageImportedAt
or (
coalesce(fl.CreationDate, fl.TStamp) = :lastSourcePackageImportedAt
and coalesce(fl.ID, ca.ID_FileLog, cda.ID_FileLog, c.ID_FileLog, ca.ID) > try_convert(int, :lastSourcePackageId)
)
or (
coalesce(fl.CreationDate, fl.TStamp) is null
and (
:lastSourcePackageId is null
or coalesce(fl.ID, ca.ID_FileLog, cda.ID_FileLog, c.ID_FileLog, ca.ID) > try_convert(int, :lastSourcePackageId)
)
)
)
cast(base.source_package_id_raw as varchar(128)) as source_package_id,
cast(base.source_package_entity_id_raw as varchar(128)) as source_package_entity_id,
base.source_package_period_from,
base.source_package_period_to,
base.source_package_imported_at
from Base base
cross apply (values
('START', base.BeginTime),
('END', base.EndTime)
) evt(lifecycle, occurred_at)
where (:occurredFrom is null or evt.occurred_at >= :occurredFrom)
and (:occurredTo is null or evt.occurred_at < :occurredTo)
/*
* Organisation filtering can use FileLog.I_90021_ID / FileLog.OrgID or
* Driver_I_90021 / Vehicle_I_90021 once subtree semantics are confirmed.
* Organisation filter: driver membership in GetOrganisationTree(null, :organisationId, 0, null).
*/

View File

@ -0,0 +1,96 @@
/*
* CardBorderCrossing BORDER_CROSSING extraction for the bytebar tachograph schema.
*/
with OrgTree as (
select org.I_90021_OID
from dbo.GetOrganisationTree(null, :organisationId, 0, null) org
where :organisationId is not null
)
,
Base as (
select
border.ID,
border.Timestamp as occurred_at,
border.Odo,
border.ID_FileLog,
c.ID as card_id,
c.ID_Driver as driver_id,
cn.AlphaCode as driver_card_nation,
c.CardNumber as driver_card_number,
leftNation.AlphaCode as country_from,
enteredNation.AlphaCode as country_to,
gnss.Latitude as latitude,
gnss.Longitude as longitude,
v.ID as vehicle_registration_id,
v.ID_VehicleIdentification as vehicle_identification_id,
vi.VIN as vehicle_vin,
vehicleNation.AlphaCode as vehicle_registration_nation,
v.VRN as vehicle_registration_number,
coalesce(fl.DownloadDate, fl.OriginalDownloadDate, fl.TStamp, fl.CreationDate) as received_partner_at,
coalesce(fl.ID, border.ID_FileLog) as source_package_id_raw,
coalesce(fl.ID_Card, border.ID_Card) as source_package_entity_id_raw,
coalesce(fl.DownloadFrom, border.Timestamp) as source_package_period_from,
coalesce(fl.DownloadTo, border.Timestamp) as source_package_period_to,
coalesce(fl.CreationDate, fl.TStamp) as source_package_imported_at
from dbo.CardBorderCrossing border
join dbo.Card c on c.ID = border.ID_Card
left join dbo.Nation cn on cn.ID = c.ID_Nation
left join dbo.Nation leftNation on leftNation.ID = border.ID_NationLeft
left join dbo.Nation enteredNation on enteredNation.ID = border.ID_NationEntered
left join dbo.GnssPlace gnss on gnss.ID = border.ID_GnssPlace
left join dbo.FileLog fl on fl.ID = border.ID_FileLog
outer apply (
select top 1 used.ID_Vehicle
from dbo.CardVehiclesUsed used
where used.ID_Card = border.ID_Card
and (used.FirstUse is null or used.FirstUse <= border.Timestamp)
and (used.LastUse is null or used.LastUse >= border.Timestamp)
order by
used.FirstUse desc,
used.ID desc
) cvu
left join dbo.Vehicle v on v.ID = cvu.ID_Vehicle
left join dbo.VehicleIdentification vi on vi.ID = v.ID_VehicleIdentification
left join dbo.Nation vehicleNation on vehicleNation.ID = v.ID_Nation
where (:occurredFrom is null or border.Timestamp >= :occurredFrom)
and (:occurredTo is null or border.Timestamp < :occurredTo)
and (
:organisationId is null
or exists (
select 1
from dbo.Driver_I_90021 rel
join OrgTree on OrgTree.I_90021_OID = rel.ID_I_90021
where rel.ID_Driver = c.ID_Driver
and rel.GILT_BIS is null
)
)
)
select
cast(base.ID as varchar(128)) as source_row_id,
concat('TACHOGRAPH:CARD_BORDER_CROSSING:', base.ID) as external_source_event_id,
base.occurred_at,
base.received_partner_at,
cast(base.Odo as bigint) * 1000 as odometer_m,
base.latitude,
base.longitude,
base.country_from,
base.country_to,
cast(base.driver_id as varchar(128)) as driver_source_entity_id,
base.driver_card_nation,
base.driver_card_number,
cast(base.vehicle_identification_id as varchar(128)) as vehicle_source_entity_id,
base.vehicle_vin,
cast(base.vehicle_registration_id as varchar(128)) as vehicle_registration_source_entity_id,
base.vehicle_registration_nation,
base.vehicle_registration_number,
'DRIVER_CARD' as source_package_kind,
cast(base.source_package_id_raw as varchar(128)) as source_package_id,
cast(base.source_package_entity_id_raw as varchar(128)) as source_package_entity_id,
base.source_package_period_from,
base.source_package_period_to,
base.source_package_imported_at
from Base base

View File

@ -0,0 +1,101 @@
/*
* CardLoadUnload LOAD_UNLOAD extraction for the bytebar tachograph schema.
*/
with OrgTree as (
select org.I_90021_OID
from dbo.GetOrganisationTree(null, :organisationId, 0, null) org
where :organisationId is not null
)
,
Base as (
select
lu.ID,
lu.Timestamp as occurred_at,
lu.OperationType,
lu.Odo,
lu.ID_FileLog,
c.ID as card_id,
c.ID_Driver as driver_id,
cn.AlphaCode as driver_card_nation,
c.CardNumber as driver_card_number,
gnss.Latitude as latitude,
gnss.Longitude as longitude,
v.ID as vehicle_registration_id,
v.ID_VehicleIdentification as vehicle_identification_id,
vi.VIN as vehicle_vin,
vehicleNation.AlphaCode as vehicle_registration_nation,
v.VRN as vehicle_registration_number,
coalesce(fl.DownloadDate, fl.OriginalDownloadDate, fl.TStamp, fl.CreationDate) as received_partner_at,
coalesce(fl.ID, lu.ID_FileLog) as source_package_id_raw,
coalesce(fl.ID_Card, lu.ID_Card) as source_package_entity_id_raw,
coalesce(fl.DownloadFrom, lu.Timestamp) as source_package_period_from,
coalesce(fl.DownloadTo, lu.Timestamp) as source_package_period_to,
coalesce(fl.CreationDate, fl.TStamp) as source_package_imported_at
from dbo.CardLoadUnload lu
join dbo.Card c on c.ID = lu.ID_Card
left join dbo.Nation cn on cn.ID = c.ID_Nation
left join dbo.GnssPlace gnss on gnss.ID = lu.ID_GnssPlace
left join dbo.FileLog fl on fl.ID = lu.ID_FileLog
outer apply (
select top 1 used.ID_Vehicle
from dbo.CardVehiclesUsed used
where used.ID_Card = lu.ID_Card
and (used.FirstUse is null or used.FirstUse <= lu.Timestamp)
and (used.LastUse is null or used.LastUse >= lu.Timestamp)
order by
used.FirstUse desc,
used.ID desc
) cvu
left join dbo.Vehicle v on v.ID = cvu.ID_Vehicle
left join dbo.VehicleIdentification vi on vi.ID = v.ID_VehicleIdentification
left join dbo.Nation vehicleNation on vehicleNation.ID = v.ID_Nation
where (:occurredFrom is null or lu.Timestamp >= :occurredFrom)
and (:occurredTo is null or lu.Timestamp < :occurredTo)
and (
:organisationId is null
or exists (
select 1
from dbo.Driver_I_90021 rel
join OrgTree on OrgTree.I_90021_OID = rel.ID_I_90021
where rel.ID_Driver = c.ID_Driver
and rel.GILT_BIS is null
)
)
)
select
cast(base.ID as varchar(128)) as source_row_id,
concat('TACHOGRAPH:CARD_LOAD_UNLOAD:', base.ID) as external_source_event_id,
base.occurred_at,
base.received_partner_at,
cast(base.Odo as bigint) * 1000 as odometer_m,
base.latitude,
base.longitude,
case base.OperationType
when 1 then 'LOAD'
when 2 then 'UNLOAD'
else 'LOAD_UNLOAD'
end as operation,
case base.OperationType
when 1 then 'LOAD'
when 2 then 'UNLOAD'
else 'LOAD_UNLOAD'
end as event_type,
cast(base.driver_id as varchar(128)) as driver_source_entity_id,
base.driver_card_nation,
base.driver_card_number,
cast(base.vehicle_identification_id as varchar(128)) as vehicle_source_entity_id,
base.vehicle_vin,
cast(base.vehicle_registration_id as varchar(128)) as vehicle_registration_source_entity_id,
base.vehicle_registration_nation,
base.vehicle_registration_number,
'DRIVER_CARD' as source_package_kind,
cast(base.source_package_id_raw as varchar(128)) as source_package_id,
cast(base.source_package_entity_id_raw as varchar(128)) as source_package_entity_id,
base.source_package_period_from,
base.source_package_period_to,
base.source_package_imported_at
from Base base

View File

@ -0,0 +1,100 @@
/*
* CardPlaces PLACE extraction for the bytebar tachograph schema.
*/
with OrgTree as (
select org.I_90021_OID
from dbo.GetOrganisationTree(null, :organisationId, 0, null) org
where :organisationId is not null
)
,
Base as (
select
place.ID,
place.EntryTime as occurred_at,
cast(place.EntryType as int) as entry_type,
place.Odo,
place.ID_FileLog,
c.ID as card_id,
c.ID_Driver as driver_id,
cn.AlphaCode as driver_card_nation,
c.CardNumber as driver_card_number,
placeNation.AlphaCode as country,
coalesce(region.AlphaCode, region.Name) as region,
gnss.Latitude as latitude,
gnss.Longitude as longitude,
v.ID as vehicle_registration_id,
v.ID_VehicleIdentification as vehicle_identification_id,
vi.VIN as vehicle_vin,
vehicleNation.AlphaCode as vehicle_registration_nation,
v.VRN as vehicle_registration_number,
coalesce(fl.DownloadDate, fl.OriginalDownloadDate, fl.TStamp, fl.CreationDate) as received_partner_at,
coalesce(fl.ID, place.ID_FileLog) as source_package_id_raw,
coalesce(fl.ID_Card, place.ID_Card) as source_package_entity_id_raw,
coalesce(fl.DownloadFrom, place.EntryTime) as source_package_period_from,
coalesce(fl.DownloadTo, place.EntryTime) as source_package_period_to,
coalesce(fl.CreationDate, fl.TStamp) as source_package_imported_at
from dbo.CardPlaces place
join dbo.Card c on c.ID = place.ID_Card
left join dbo.Nation cn on cn.ID = c.ID_Nation
left join dbo.Nation placeNation on placeNation.ID = place.ID_Nation
left join dbo.Region region on region.ID = place.ID_Region
left join dbo.GnssPlace gnss on gnss.ID = place.ID_GnssPlace
left join dbo.FileLog fl on fl.ID = place.ID_FileLog
outer apply (
select top 1 used.ID_Vehicle
from dbo.CardVehiclesUsed used
where used.ID_Card = place.ID_Card
and (used.FirstUse is null or used.FirstUse <= place.EntryTime)
and (used.LastUse is null or used.LastUse >= place.EntryTime)
order by
used.FirstUse desc,
used.ID desc
) cvu
left join dbo.Vehicle v on v.ID = cvu.ID_Vehicle
left join dbo.VehicleIdentification vi on vi.ID = v.ID_VehicleIdentification
left join dbo.Nation vehicleNation on vehicleNation.ID = v.ID_Nation
where (:occurredFrom is null or place.EntryTime >= :occurredFrom)
and (:occurredTo is null or place.EntryTime < :occurredTo)
and (
:organisationId is null
or exists (
select 1
from dbo.Driver_I_90021 rel
join OrgTree on OrgTree.I_90021_OID = rel.ID_I_90021
where rel.ID_Driver = c.ID_Driver
and rel.GILT_BIS is null
)
)
)
select
cast(base.ID as varchar(128)) as source_row_id,
concat('TACHOGRAPH:CARD_PLACE:', base.ID) as external_source_event_id,
base.occurred_at,
base.received_partner_at,
cast(base.Odo as bigint) * 1000 as odometer_m,
base.latitude,
base.longitude,
base.country,
base.region,
'WORKING_DAY_PLACE_RECORDED' as event_type,
case when base.entry_type in (0, 2, 4) then 'START' else 'END' end as lifecycle,
cast(case when base.entry_type in (2, 3, 4, 5) then 1 else 0 end as bit) as manual_entry,
cast(base.driver_id as varchar(128)) as driver_source_entity_id,
base.driver_card_nation,
base.driver_card_number,
cast(base.vehicle_identification_id as varchar(128)) as vehicle_source_entity_id,
base.vehicle_vin,
cast(base.vehicle_registration_id as varchar(128)) as vehicle_registration_source_entity_id,
base.vehicle_registration_nation,
base.vehicle_registration_number,
'DRIVER_CARD' as source_package_kind,
cast(base.source_package_id_raw as varchar(128)) as source_package_id,
cast(base.source_package_entity_id_raw as varchar(128)) as source_package_entity_id,
base.source_package_period_from,
base.source_package_period_to,
base.source_package_imported_at
from Base base

View File

@ -0,0 +1,90 @@
/*
* CardGnssAccumulatedDriving POSITION extraction for the bytebar tachograph schema.
*/
with OrgTree as (
select org.I_90021_OID
from dbo.GetOrganisationTree(null, :organisationId, 0, null) org
where :organisationId is not null
)
,
Base as (
select
pos.ID,
pos.Timestamp as occurred_at,
pos.Odo,
pos.ID_FileLog,
c.ID as card_id,
c.ID_Driver as driver_id,
cn.AlphaCode as driver_card_nation,
c.CardNumber as driver_card_number,
gnss.Latitude as latitude,
gnss.Longitude as longitude,
v.ID as vehicle_registration_id,
v.ID_VehicleIdentification as vehicle_identification_id,
vi.VIN as vehicle_vin,
vehicleNation.AlphaCode as vehicle_registration_nation,
v.VRN as vehicle_registration_number,
coalesce(fl.DownloadDate, fl.OriginalDownloadDate, fl.TStamp, fl.CreationDate) as received_partner_at,
coalesce(fl.ID, pos.ID_FileLog) as source_package_id_raw,
coalesce(fl.ID_Card, pos.ID_Card) as source_package_entity_id_raw,
coalesce(fl.DownloadFrom, pos.Timestamp) as source_package_period_from,
coalesce(fl.DownloadTo, pos.Timestamp) as source_package_period_to,
coalesce(fl.CreationDate, fl.TStamp) as source_package_imported_at
from dbo.CardGnssAccumulatedDriving pos
join dbo.Card c on c.ID = pos.ID_Card
left join dbo.Nation cn on cn.ID = c.ID_Nation
join dbo.GnssPlace gnss on gnss.ID = pos.ID_GnssPlace
left join dbo.FileLog fl on fl.ID = pos.ID_FileLog
outer apply (
select top 1 used.ID_Vehicle
from dbo.CardVehiclesUsed used
where used.ID_Card = pos.ID_Card
and (used.FirstUse is null or used.FirstUse <= pos.Timestamp)
and (used.LastUse is null or used.LastUse >= pos.Timestamp)
order by
used.FirstUse desc,
used.ID desc
) cvu
left join dbo.Vehicle v on v.ID = cvu.ID_Vehicle
left join dbo.VehicleIdentification vi on vi.ID = v.ID_VehicleIdentification
left join dbo.Nation vehicleNation on vehicleNation.ID = v.ID_Nation
where (:occurredFrom is null or pos.Timestamp >= :occurredFrom)
and (:occurredTo is null or pos.Timestamp < :occurredTo)
and (
:organisationId is null
or exists (
select 1
from dbo.Driver_I_90021 rel
join OrgTree on OrgTree.I_90021_OID = rel.ID_I_90021
where rel.ID_Driver = c.ID_Driver
and rel.GILT_BIS is null
)
)
)
select
cast(base.ID as varchar(128)) as source_row_id,
concat('TACHOGRAPH:CARD_POSITION:', base.ID) as external_source_event_id,
base.occurred_at,
base.received_partner_at,
cast(base.Odo as bigint) * 1000 as odometer_m,
base.latitude,
base.longitude,
cast(base.driver_id as varchar(128)) as driver_source_entity_id,
base.driver_card_nation,
base.driver_card_number,
cast(base.vehicle_identification_id as varchar(128)) as vehicle_source_entity_id,
base.vehicle_vin,
cast(base.vehicle_registration_id as varchar(128)) as vehicle_registration_source_entity_id,
base.vehicle_registration_nation,
base.vehicle_registration_number,
'DRIVER_CARD' as source_package_kind,
cast(base.source_package_id_raw as varchar(128)) as source_package_id,
cast(base.source_package_entity_id_raw as varchar(128)) as source_package_entity_id,
base.source_package_period_from,
base.source_package_period_to,
base.source_package_imported_at
from Base base

View File

@ -0,0 +1,88 @@
/*
* CardSpecificCondition SPECIFIC_CONDITION extraction for the bytebar tachograph schema.
*/
with OrgTree as (
select org.I_90021_OID
from dbo.GetOrganisationTree(null, :organisationId, 0, null) org
where :organisationId is not null
)
,
Base as (
select
cond.ID,
cond.EntryTime as occurred_at,
sc.Condition as condition_code,
cond.ID_FileLog,
c.ID as card_id,
c.ID_Driver as driver_id,
cn.AlphaCode as driver_card_nation,
c.CardNumber as driver_card_number,
v.ID as vehicle_registration_id,
v.ID_VehicleIdentification as vehicle_identification_id,
vi.VIN as vehicle_vin,
vehicleNation.AlphaCode as vehicle_registration_nation,
v.VRN as vehicle_registration_number,
coalesce(fl.DownloadDate, fl.OriginalDownloadDate, fl.TStamp, fl.CreationDate) as received_partner_at,
coalesce(fl.ID, cond.ID_FileLog) as source_package_id_raw,
coalesce(fl.ID_Card, cond.ID_Card) as source_package_entity_id_raw,
coalesce(fl.DownloadFrom, cond.EntryTime) as source_package_period_from,
coalesce(fl.DownloadTo, cond.EntryTime) as source_package_period_to,
coalesce(fl.CreationDate, fl.TStamp) as source_package_imported_at
from dbo.CardSpecificConditions cond
join dbo.SpecificCondition sc on sc.ID = cond.ID_SpecificCondition
join dbo.Card c on c.ID = cond.ID_Card
left join dbo.Nation cn on cn.ID = c.ID_Nation
left join dbo.FileLog fl on fl.ID = cond.ID_FileLog
outer apply (
select top 1 used.ID_Vehicle
from dbo.CardVehiclesUsed used
where used.ID_Card = cond.ID_Card
and (used.FirstUse is null or used.FirstUse <= cond.EntryTime)
and (used.LastUse is null or used.LastUse >= cond.EntryTime)
order by
used.FirstUse desc,
used.ID desc
) cvu
left join dbo.Vehicle v on v.ID = cvu.ID_Vehicle
left join dbo.VehicleIdentification vi on vi.ID = v.ID_VehicleIdentification
left join dbo.Nation vehicleNation on vehicleNation.ID = v.ID_Nation
where (:occurredFrom is null or cond.EntryTime >= :occurredFrom)
and (:occurredTo is null or cond.EntryTime < :occurredTo)
and (
:organisationId is null
or exists (
select 1
from dbo.Driver_I_90021 rel
join OrgTree on OrgTree.I_90021_OID = rel.ID_I_90021
where rel.ID_Driver = c.ID_Driver
and rel.GILT_BIS is null
)
)
)
select
cast(base.ID as varchar(128)) as source_row_id,
concat('TACHOGRAPH:CARD_SPECIFIC_CONDITION:', base.ID) as external_source_event_id,
base.occurred_at,
base.received_partner_at,
'SPECIFIC_CONDITION' as event_domain,
case when base.condition_code in (3, 4) then 'FERRY_TRAIN' else 'OUT' end as event_type,
case when base.condition_code in (1, 3) then 'BEGIN' else 'END' end as lifecycle,
cast(base.driver_id as varchar(128)) as driver_source_entity_id,
base.driver_card_nation,
base.driver_card_number,
cast(base.vehicle_identification_id as varchar(128)) as vehicle_source_entity_id,
base.vehicle_vin,
cast(base.vehicle_registration_id as varchar(128)) as vehicle_registration_source_entity_id,
base.vehicle_registration_nation,
base.vehicle_registration_number,
'DRIVER_CARD' as source_package_kind,
cast(base.source_package_id_raw as varchar(128)) as source_package_id,
cast(base.source_package_entity_id_raw as varchar(128)) as source_package_entity_id,
base.source_package_period_from,
base.source_package_period_to,
base.source_package_imported_at
from Base base

View File

@ -0,0 +1,114 @@
/*
* CardVehiclesUsed DRIVER_CARD extraction for the bytebar tachograph schema.
*
* Each row is emitted as INSERT at FirstUse and WITHDRAW at LastUse when those
* timestamps are available.
*/
with OrgTree as (
select org.I_90021_OID
from dbo.GetOrganisationTree(null, :organisationId, 0, null) org
where :organisationId is not null
)
,
CandidateUse as (
select
used.ID,
used.ID_Card,
used.ID_Vehicle,
used.OdoBegin,
used.OdoEnd,
used.FirstUse,
used.LastUse,
used.ID_FileLog,
used.VIN as used_vin,
c.ID_Driver as driver_id,
cn.AlphaCode as driver_card_nation,
c.CardNumber as driver_card_number,
v.ID_VehicleIdentification,
vi.ID as vehicle_identification_id,
coalesce(used.VIN, vi.VIN) as vehicle_vin,
vn.AlphaCode as vehicle_registration_nation,
v.VRN as vehicle_registration_number
from dbo.CardVehiclesUsed used
join dbo.Card c on c.ID = used.ID_Card
left join dbo.Nation cn on cn.ID = c.ID_Nation
left join dbo.Vehicle v on v.ID = used.ID_Vehicle
left join dbo.VehicleIdentification vi on vi.ID = v.ID_VehicleIdentification
left join dbo.Nation vn on vn.ID = v.ID_Nation
where (:occurredTo is null or used.FirstUse < :occurredTo or used.LastUse < :occurredTo)
and (:occurredFrom is null or used.FirstUse >= :occurredFrom or used.LastUse >= :occurredFrom)
and (
:organisationId is null
or exists (
select 1
from dbo.Driver_I_90021 rel
join OrgTree on OrgTree.I_90021_OID = rel.ID_I_90021
where rel.ID_Driver = c.ID_Driver
and rel.GILT_BIS is null
)
)
)
,
Base as (
select
used.ID,
used.ID_Card,
used.ID_Vehicle,
used.driver_id,
used.driver_card_nation,
used.driver_card_number,
used.vehicle_identification_id,
used.vehicle_vin,
used.vehicle_registration_nation,
used.vehicle_registration_number,
evt.lifecycle,
evt.occurred_at,
evt.odometer_m,
coalesce(fl.DownloadDate, fl.OriginalDownloadDate, fl.TStamp, fl.CreationDate) as received_partner_at,
coalesce(fl.ID, used.ID_FileLog) as source_package_id_raw,
coalesce(fl.ID_Card, used.ID_Card) as source_package_entity_id_raw,
coalesce(fl.DownloadFrom, used.FirstUse) as source_package_period_from,
coalesce(fl.DownloadTo, used.LastUse, used.FirstUse) as source_package_period_to,
coalesce(fl.CreationDate, fl.TStamp) as source_package_imported_at
from CandidateUse used
cross apply (values
('INSERT', used.FirstUse, cast(used.OdoBegin as bigint) * 1000),
('WITHDRAW', used.LastUse, cast(used.OdoEnd as bigint) * 1000)
) evt(lifecycle, occurred_at, odometer_m)
left join dbo.FileLog fl on fl.ID = used.ID_FileLog
where evt.occurred_at is not null
and (:occurredFrom is null or evt.occurred_at >= :occurredFrom)
and (:occurredTo is null or evt.occurred_at < :occurredTo)
)
select
cast(base.ID as varchar(128)) as source_row_id,
concat('TACHOGRAPH:CARD_VEHICLES_USED:', base.ID, ':', base.lifecycle) as external_source_event_id,
base.occurred_at,
base.received_partner_at,
case base.lifecycle
when 'INSERT' then 'CARD_INSERTED'
when 'WITHDRAW' then 'CARD_WITHDRAWN'
else 'UNKNOWN_EVENT'
end as event_type,
base.lifecycle,
cast(null as varchar(32)) as card_slot,
base.odometer_m,
cast(base.driver_id as varchar(128)) as driver_source_entity_id,
base.driver_card_nation,
base.driver_card_number,
cast(base.vehicle_identification_id as varchar(128)) as vehicle_source_entity_id,
base.vehicle_vin,
cast(base.ID_Vehicle as varchar(128)) as vehicle_registration_source_entity_id,
base.vehicle_registration_nation,
base.vehicle_registration_number,
'DRIVER_CARD' as source_package_kind,
cast(base.source_package_id_raw as varchar(128)) as source_package_id,
cast(base.source_package_entity_id_raw as varchar(128)) as source_package_entity_id,
base.source_package_period_from,
base.source_package_period_to,
base.source_package_imported_at
from Base base

View File

@ -0,0 +1,128 @@
/*
* IWCycle DRIVER_CARD extraction for the bytebar tachograph schema.
*
* Each IWCycle row describes a card inserted into a vehicle unit. It is emitted
* as two point events when timestamps are available: INSERT at BeginTime and
* WITHDRAW at EndTime.
*/
with OrgTree as (
select org.I_90021_OID
from dbo.GetOrganisationTree(null, :organisationId, 0, null) org
where :organisationId is not null
)
,
CandidateCycle as (
select
iw.ID,
iw.BeginTime,
iw.EndTime,
iw.OdoBegin,
iw.OdoEnd,
iw.Slot,
iw.ID_FileLog,
iw.ID_Card,
iw.ID_VUInstallation,
vui.ID_VehicleIdentification,
vui.ID_FileLog as vui_filelog_id,
vi.ID as vehicle_identification_id,
vi.VIN as vehicle_vin,
c.ID_Driver as driver_id,
cn.AlphaCode as driver_card_nation,
c.CardNumber as driver_card_number,
coalesce(iw.ID_FileLog, vui.ID_FileLog) as file_log_id
from dbo.IWCycle iw
join dbo.VUInstallation vui on vui.ID = iw.ID_VUInstallation
join dbo.VehicleIdentification vi on vi.ID = vui.ID_VehicleIdentification
left join dbo.Card c on c.ID = iw.ID_Card
left join dbo.Nation cn on cn.ID = c.ID_Nation
where (:occurredTo is null or iw.BeginTime < :occurredTo or iw.EndTime < :occurredTo)
and (:occurredFrom is null or iw.BeginTime >= :occurredFrom or iw.EndTime >= :occurredFrom)
)
,
Base as (
select
cycle.ID,
cycle.Slot,
cycle.ID_FileLog,
cycle.vui_filelog_id,
cycle.ID_Card,
cycle.ID_VUInstallation,
cycle.vehicle_identification_id,
cycle.vehicle_vin,
cycle.driver_id,
cycle.driver_card_nation,
cycle.driver_card_number,
evt.lifecycle,
evt.occurred_at,
evt.odometer_m,
coalesce(fl.DownloadDate, fl.OriginalDownloadDate, fl.TStamp, fl.CreationDate) as received_partner_at,
coalesce(fl.ID, cycle.ID_FileLog, cycle.vui_filelog_id) as source_package_id_raw,
coalesce(fl.ID_VehicleIdentification, cycle.ID_VehicleIdentification) as source_package_entity_id_raw,
coalesce(fl.DownloadFrom, cycle.BeginTime) as source_package_period_from,
coalesce(fl.DownloadTo, cycle.EndTime, cycle.BeginTime) as source_package_period_to,
coalesce(fl.CreationDate, fl.TStamp) as source_package_imported_at
from CandidateCycle cycle
cross apply (values
('INSERT', cycle.BeginTime, cast(cycle.OdoBegin as bigint) * 1000),
('WITHDRAW', cycle.EndTime, cast(cycle.OdoEnd as bigint) * 1000)
) evt(lifecycle, occurred_at, odometer_m)
left join dbo.FileLog fl on fl.ID = cycle.file_log_id
where evt.occurred_at is not null
and (:occurredFrom is null or evt.occurred_at >= :occurredFrom)
and (:occurredTo is null or evt.occurred_at < :occurredTo)
)
select
cast(base.ID as varchar(128)) as source_row_id,
concat('TACHOGRAPH:IW_CYCLE:', base.ID, ':', base.lifecycle) as external_source_event_id,
base.occurred_at,
base.received_partner_at,
case base.lifecycle
when 'INSERT' then 'CARD_INSERTED'
when 'WITHDRAW' then 'CARD_WITHDRAWN'
else 'UNKNOWN_EVENT'
end as event_type,
base.lifecycle,
base.Slot as card_slot,
base.odometer_m,
cast(base.driver_id as varchar(128)) as driver_source_entity_id,
base.driver_card_nation,
base.driver_card_number,
cast(base.vehicle_identification_id as varchar(128)) as vehicle_source_entity_id,
base.vehicle_vin,
cast(v.ID as varchar(128)) as vehicle_registration_source_entity_id,
vn.AlphaCode as vehicle_registration_nation,
v.VRN as vehicle_registration_number,
'VEHICLE_UNIT' as source_package_kind,
cast(base.source_package_id_raw as varchar(128)) as source_package_id,
cast(base.source_package_entity_id_raw as varchar(128)) as source_package_entity_id,
base.source_package_period_from,
base.source_package_period_to,
base.source_package_imported_at
from Base base
outer apply (
select top 1 vehicle.ID,
vehicle.VRN,
vehicle.ID_Nation
from dbo.Vehicle vehicle
where vehicle.ID_VehicleIdentification = base.vehicle_identification_id
and (vehicle.ValidFrom is null or vehicle.ValidFrom <= base.occurred_at)
and (vehicle.ValidTo is null or vehicle.ValidTo > base.occurred_at)
order by
vehicle.ValidFrom desc,
vehicle.ID desc
) v
left join dbo.Nation vn on vn.ID = v.ID_Nation
where (
:organisationId is null
or exists (
select 1
from dbo.Vehicle_I_90021 rel
join OrgTree on OrgTree.I_90021_OID = rel.ID_I_90021
where rel.ID_Vehicle = v.ID
and rel.GILT_BIS is null
)
)

View File

@ -0,0 +1,27 @@
select
'DRIVER_CARD' as entity_type,
cast(c.ID as varchar(128)) as source_entity_id,
concat(coalesce(n.AlphaCode, ''), ':', c.CardNumber) as source_external_key,
concat(coalesce(n.AlphaCode, ''), ':', c.CardNumber) as display_name,
cast(case when c.IsLastCard = 1 and (c.ExpiryDate is null or c.ExpiryDate > getutcdate()) then 1 else 0 end as bit) as active,
c.IssueDate as valid_from,
c.ExpiryDate as valid_to,
cast(null as datetime) as source_updated_at,
c.ID as card_id,
cast(c.ID_Driver as varchar(128)) as driver_id,
n.AlphaCode as card_nation,
c.CardNumber as card_number,
c.Consecutive as consecutive,
c.Replacement as replacement,
c.Renewal as renewal,
c.IssueDate as issue_date,
c.ExpiryDate as expiry_date,
c.ValidityDate as validity_date,
c.Authorityname as authority_name,
c.ID_FileLog as file_log_id,
c.IsAnalog as is_analog,
c.IsLastCard as is_last_card,
c.Generation as generation
from dbo.Card c
left join dbo.Nation n on n.ID = c.ID_Nation
where c.CardNumber is not null

View File

@ -0,0 +1,35 @@
select
'DRIVER' as entity_type,
cast(d.ID as varchar(128)) as source_entity_id,
coalesce(nullif(d.IdentificationNumber, ''), nullif(d.LicenseNumber, ''), cast(d.ID as varchar(128))) as source_external_key,
ltrim(rtrim(concat(d.Surname, ' ', d.Firstnames))) as display_name,
cast(case when d.IsActive = 1 then 1 else 0 end as bit) as active,
d.EmploymentStartDate as valid_from,
d.RetirementDate as valid_to,
d.LastUpdate as source_updated_at,
d.ID as driver_id,
d.Surname as surname,
d.Firstnames as first_names,
d.Birthdate as birth_date,
d.BirthPlace as birth_place,
d.LicenseNumber as license_number,
ln.AlphaCode as license_nation,
d.LicenseAuthority as license_authority,
d.LExpiryDate as license_expiry_date,
d.Phone as phone,
d.Mobile as mobile,
d.Email as email,
d.CostCenter as cost_center,
d.OrganisationUserID as organisation_user_id,
d.IdentificationNumber as identification_number,
d.Region as region,
d.ID_FileLog as file_log_id,
d.LastFileLog_ID as last_file_log_id,
d.FirstDrivingTime as first_driving_time,
d.LastDrivingTime as last_driving_time,
d.LastUsedVehicle_ID as last_used_vehicle_id,
d.LastUsedVehicleTime as last_used_vehicle_time,
d.IsDigital as is_digital,
d.IsAnalog as is_analog
from dbo.Driver d
left join dbo.Nation ln on ln.ID = d.ID_LicenseNation

View File

@ -0,0 +1,22 @@
select
'ORGANISATION' as entity_type,
cast(o.oid as varchar(128)) as source_entity_id,
cast(o.GUID as varchar(64)) as source_external_key,
coalesce(nullif(o.name, ''), nullif(o.kurzbez, ''), cast(o.oid as varchar(128))) as display_name,
cast(case when o.gueltigbis is null or o.gueltigbis > getutcdate() then 1 else 0 end as bit) as active,
o.VertragGiltAb as valid_from,
o.gueltigbis as valid_to,
cast(null as datetime) as source_updated_at,
o.oid as organisation_id,
cast(o.GUID as varchar(64)) as organisation_guid,
o.kurzbez as organisation_code,
o.name as organisation_name,
cast(o.n_rekey01 as varchar(128)) as parent_organisation_id,
o.Unternehmens_ID as company_id,
o.kostenstelle as cost_center,
o.Country as country,
o.PostalCode as postal_code,
o.City as city,
o.strasse as street,
o.Email as email
from dbo.I_90021 o

View File

@ -0,0 +1,75 @@
select
'ORGANISATION_PARENT' as relation_type,
'ORGANISATION' as from_entity_type,
cast(o.oid as varchar(128)) as from_source_entity_id,
'ORGANISATION' as to_entity_type,
cast(o.n_rekey01 as varchar(128)) as to_source_entity_id,
o.VertragGiltAb as valid_from,
o.gueltigbis as valid_to,
cast(null as datetime) as source_updated_at,
'I_90021' as source_table,
cast(o.oid as varchar(128)) as source_row_id
from dbo.I_90021 o
where o.n_rekey01 is not null
union all
select
'DRIVER_ORGANISATION' as relation_type,
'DRIVER' as from_entity_type,
cast(rel.ID_Driver as varchar(128)) as from_source_entity_id,
'ORGANISATION' as to_entity_type,
cast(rel.ID_I_90021 as varchar(128)) as to_source_entity_id,
rel.GILT_AB as valid_from,
rel.GILT_BIS as valid_to,
rel.LastUpdate as source_updated_at,
'Driver_I_90021' as source_table,
cast(rel.ID as varchar(128)) as source_row_id
from dbo.Driver_I_90021 rel
union all
select
'DRIVER_CARD_DRIVER' as relation_type,
'DRIVER_CARD' as from_entity_type,
cast(c.ID as varchar(128)) as from_source_entity_id,
'DRIVER' as to_entity_type,
cast(c.ID_Driver as varchar(128)) as to_source_entity_id,
c.IssueDate as valid_from,
c.ExpiryDate as valid_to,
cast(null as datetime) as source_updated_at,
'Card' as source_table,
cast(c.ID as varchar(128)) as source_row_id
from dbo.Card c
where c.ID_Driver is not null
union all
select
'VEHICLE_ORGANISATION' as relation_type,
'VEHICLE_REGISTRATION' as from_entity_type,
cast(rel.ID_Vehicle as varchar(128)) as from_source_entity_id,
'ORGANISATION' as to_entity_type,
cast(rel.ID_I_90021 as varchar(128)) as to_source_entity_id,
rel.GILT_AB as valid_from,
rel.GILT_BIS as valid_to,
rel.LastUpdate as source_updated_at,
'Vehicle_I_90021' as source_table,
cast(rel.ID as varchar(128)) as source_row_id
from dbo.Vehicle_I_90021 rel
union all
select
'VEHICLE_REGISTRATION_VEHICLE' as relation_type,
'VEHICLE_REGISTRATION' as from_entity_type,
cast(v.ID as varchar(128)) as from_source_entity_id,
'VEHICLE' as to_entity_type,
cast(v.ID_VehicleIdentification as varchar(128)) as to_source_entity_id,
v.ValidFrom as valid_from,
v.ValidTo as valid_to,
cast(null as datetime) as source_updated_at,
'Vehicle' as source_table,
cast(v.ID as varchar(128)) as source_row_id
from dbo.Vehicle v
where v.ID_VehicleIdentification is not null

View File

@ -0,0 +1,20 @@
select
'VEHICLE_REGISTRATION' as entity_type,
cast(v.ID as varchar(128)) as source_entity_id,
concat(coalesce(n.AlphaCode, ''), ':', v.VRN) as source_external_key,
concat(coalesce(n.AlphaCode, ''), ':', v.VRN) as display_name,
cast(case when v.ValidTo is null or v.ValidTo > getutcdate() then 1 else 0 end as bit) as active,
cast(null as datetime) as valid_from,
cast(null as datetime) as valid_to,
cast(null as datetime) as source_updated_at,
v.ID as vehicle_registration_id,
cast(v.ID_VehicleIdentification as varchar(128)) as vehicle_identification_id,
n.AlphaCode as registration_nation,
v.VRN as registration_number,
v.VrnNormalized as registration_number_normalized,
v.ID_FileLog as file_log_id,
v.LastDriver_ID as last_driver_id,
v.LastDriverTime as last_driver_time
from dbo.Vehicle v
left join dbo.Nation n on n.ID = v.ID_Nation
where v.VRN is not null

View File

@ -0,0 +1,28 @@
select
'VEHICLE' as entity_type,
cast(vi.ID as varchar(128)) as source_entity_id,
vi.VIN as source_external_key,
vi.VIN as display_name,
cast(case when vi.IsActive = 1 then 1 else 0 end as bit) as active,
cast(null as datetime) as valid_from,
cast(null as datetime) as valid_to,
cast(null as datetime) as source_updated_at,
vi.ID as vehicle_identification_id,
vi.VIN as vin,
vi.CostCenter as cost_center,
vi.ID_FileLog as file_log_id,
vi.BillingDate as billing_date,
vi.Mobile as mobile,
vi.Email as email,
vi.GrossVehicleMass as gross_vehicle_mass,
vi.VehicleType_ID as vehicle_type_id,
vi.FirstDrivingTime as first_driving_time,
vi.LastDrivingTime as last_driving_time,
vi.LastOdoEnd as last_odo_end,
vi.LastOdoEndTime as last_odo_end_time,
vi.LastDriver_ID as last_driver_id,
vi.LastDriverTime as last_driver_time,
vi.LastFileLog_ID as last_file_log_id,
vi.IsDigital as is_digital,
vi.IsAnalog as is_analog
from dbo.VehicleIdentification vi

View File

@ -0,0 +1,118 @@
/*
* SpeedingEvents SPEEDING extraction for the bytebar tachograph schema.
*
* The source row is an interval. The normalized event contract is point-based,
* so this query emits a BEGIN point at BeginTime and an END point at EndTime.
*/
with OrgTree as (
select org.I_90021_OID
from dbo.GetOrganisationTree(null, :organisationId, 0, null) org
where :organisationId is not null
)
,
Base as (
select
speeding.ID,
speeding.BeginTime,
speeding.EndTime,
speeding.AvgSpeed,
speeding.MaxSpeed,
speeding.ID_FileLog,
speeding.ID_VUInstallation,
c.ID as card_id,
c.ID_Driver as driver_id,
cn.AlphaCode as driver_card_nation,
c.CardNumber as driver_card_number,
vui.ID_FileLog as vui_filelog_id,
vui.ID_VehicleIdentification,
vi.ID as vehicle_identification_id,
vi.VIN as vehicle_vin,
coalesce(fl.DownloadDate, fl.OriginalDownloadDate, fl.TStamp, fl.CreationDate) as received_partner_at,
coalesce(fl.ID, speeding.ID_FileLog, vui.ID_FileLog) as source_package_id_raw,
coalesce(fl.ID_VehicleIdentification, vui.ID_VehicleIdentification) as source_package_entity_id_raw,
coalesce(fl.DownloadFrom, speeding.BeginTime) as source_package_period_from,
coalesce(fl.DownloadTo, speeding.EndTime, speeding.BeginTime) as source_package_period_to,
coalesce(fl.CreationDate, fl.TStamp) as source_package_imported_at
from dbo.SpeedingEvents speeding
join dbo.VUInstallation vui on vui.ID = speeding.ID_VUInstallation
join dbo.VehicleIdentification vi on vi.ID = vui.ID_VehicleIdentification
left join dbo.Card c on c.ID = speeding.ID_Card
left join dbo.Nation cn on cn.ID = c.ID_Nation
left join dbo.FileLog fl on fl.ID = coalesce(speeding.ID_FileLog, vui.ID_FileLog)
where (
:occurredFrom is null
or speeding.BeginTime >= :occurredFrom
or speeding.EndTime >= :occurredFrom
)
and (
:occurredTo is null
or speeding.BeginTime < :occurredTo
or speeding.EndTime < :occurredTo
)
)
,
Events as (
select
base.*,
base.BeginTime as occurred_at,
'BEGIN' as lifecycle
from Base base
union all
select
base.*,
base.EndTime as occurred_at,
'END' as lifecycle
from Base base
)
select
concat(cast(events.ID as varchar(128)), ':', events.lifecycle) as source_row_id,
concat('TACHOGRAPH:SPEEDING_EVENTS:', events.ID, ':', events.lifecycle) as external_source_event_id,
events.occurred_at,
events.received_partner_at,
cast(events.AvgSpeed as decimal(10, 2)) as avg_speed_kmh,
cast(events.MaxSpeed as decimal(10, 2)) as max_speed_kmh,
events.lifecycle,
cast(events.driver_id as varchar(128)) as driver_source_entity_id,
events.driver_card_nation,
events.driver_card_number,
cast(events.vehicle_identification_id as varchar(128)) as vehicle_source_entity_id,
events.vehicle_vin,
cast(v.ID as varchar(128)) as vehicle_registration_source_entity_id,
vn.AlphaCode as vehicle_registration_nation,
v.VRN as vehicle_registration_number,
'VEHICLE_UNIT' as source_package_kind,
cast(events.source_package_id_raw as varchar(128)) as source_package_id,
cast(events.source_package_entity_id_raw as varchar(128)) as source_package_entity_id,
events.source_package_period_from,
events.source_package_period_to,
events.source_package_imported_at
from Events events
outer apply (
select top 1 vehicle.ID,
vehicle.VRN,
vehicle.ID_Nation
from dbo.Vehicle vehicle
where vehicle.ID_VehicleIdentification = events.vehicle_identification_id
and (vehicle.ValidFrom is null or vehicle.ValidFrom <= events.occurred_at)
and (vehicle.ValidTo is null or vehicle.ValidTo > events.occurred_at)
order by
vehicle.ValidFrom desc,
vehicle.ID desc
) v
left join dbo.Nation vn on vn.ID = v.ID_Nation
where (:occurredFrom is null or events.occurred_at >= :occurredFrom)
and (:occurredTo is null or events.occurred_at < :occurredTo)
and (
:organisationId is null
or exists (
select 1
from dbo.Vehicle_I_90021 rel
join OrgTree on OrgTree.I_90021_OID = rel.ID_I_90021
where rel.ID_Vehicle = v.ID
and rel.GILT_BIS is null
)
)

View File

@ -5,16 +5,133 @@
* VUActivity -> VUDailyActivity -> VUInstallation -> VehicleIdentification
* Optional driver/card context comes from VUActivity.ID_IWCycle -> IWCycle -> Card.
*/
select
cast(va.ID as varchar(128)) as source_row_id,
cast(va.ID as varchar(128)) as vu_activity_id,
concat('TACHOGRAPH:VU_ACTIVITY:', va.ID) as external_source_event_id,
with OrgTree as (
select org.I_90021_OID
from dbo.GetOrganisationTree(null, :organisationId, 0, null) org
where :organisationId is not null
)
,
CandidateActivity as (
select
va.ID,
va.BeginTime,
activity_time.EndTime,
va.Activity,
va.Slot,
va.CardStatus,
va.DrivingStatus,
va.ID_FileLog,
va.ID_IWCycle,
vda.RecordDate,
vda.ID_FileLog as vda_filelog_id,
vui.ID_VehicleIdentification,
vui.ID_FileLog as vui_filelog_id,
vi.ID as vehicle_identification_id,
vi.VIN as vehicle_vin,
coalesce(va.ID_FileLog, vda.ID_FileLog, vui.ID_FileLog) as file_log_id
from dbo.VUActivity va
join dbo.VUDailyActivity vda on vda.ID = va.ID_VUDailyActivity
join dbo.VUInstallation vui on vui.ID = vda.ID_VUInstallation
join dbo.VehicleIdentification vi on vi.ID = vui.ID_VehicleIdentification
cross apply (values (dateadd(minute, coalesce(va.Duration, 0), va.BeginTime))) activity_time(EndTime)
where (:occurredTo is null or va.BeginTime < :occurredTo)
and (
:occurredFrom is null
or va.BeginTime >= :occurredFrom
or activity_time.EndTime >= :occurredFrom
)
)
,
CandidateVehicle as (
select
va.ID,
va.BeginTime,
va.EndTime,
va.Activity,
va.Slot,
va.CardStatus,
va.DrivingStatus,
va.ID_FileLog,
va.ID_IWCycle,
va.RecordDate,
va.vda_filelog_id,
va.ID_VehicleIdentification,
va.vui_filelog_id,
va.vehicle_identification_id,
va.vehicle_vin,
va.file_log_id,
v.ID as vehicle_registration_id,
v.ID_Nation as vehicle_registration_nation_id,
v.VRN as vehicle_registration_number
from CandidateActivity va
outer apply (
select top 1 vehicle.ID,
vehicle.VRN,
vehicle.ID_Nation
from dbo.Vehicle vehicle
where vehicle.ID_VehicleIdentification = va.vehicle_identification_id
and (vehicle.ValidFrom is null or vehicle.ValidFrom <= va.BeginTime)
and (vehicle.ValidTo is null or vehicle.ValidTo > va.BeginTime)
order by
vehicle.ValidFrom desc,
vehicle.ID desc
) v
where (
:organisationId is null
or exists (
select 1
from dbo.Vehicle_I_90021 rel
join OrgTree on OrgTree.I_90021_OID = rel.ID_I_90021
where rel.ID_Vehicle = v.ID
and rel.GILT_BIS is null
)
)
)
,
Base as (
select
va.ID,
va.BeginTime,
va.EndTime,
va.Activity,
va.Slot,
va.CardStatus,
va.DrivingStatus,
va.ID_FileLog,
va.vda_filelog_id,
va.vui_filelog_id,
va.BeginTime as occurred_at,
coalesce(fl.DownloadDate, fl.OriginalDownloadDate, fl.TStamp, fl.CreationDate) as received_partner_at,
va.Activity as activity_code,
va.Activity as activity_text,
case upper(coalesce(va.Activity, ''))
coalesce(fl.DownloadDate, fl.OriginalDownloadDate, fl.TStamp, fl.CreationDate) as received_partner_at,
coalesce(fl.ID, va.ID_FileLog, va.vda_filelog_id, va.vui_filelog_id) as source_package_id_raw,
coalesce(fl.ID_VehicleIdentification, va.ID_VehicleIdentification) as source_package_entity_id_raw,
coalesce(fl.DownloadFrom, va.RecordDate) as source_package_period_from,
coalesce(fl.DownloadTo, dateadd(day, 1, va.RecordDate)) as source_package_period_to,
coalesce(fl.CreationDate, fl.TStamp) as source_package_imported_at,
null as odometer_m,
c.ID_Driver as driver_id,
cn.AlphaCode as driver_card_nation,
c.CardNumber as driver_card_number,
va.vehicle_identification_id,
va.vehicle_registration_id,
va.vehicle_vin,
vn.AlphaCode as vehicle_registration_nation,
va.vehicle_registration_number
from CandidateVehicle va
left join dbo.FileLog fl on fl.ID = va.file_log_id
left join dbo.Nation vn on vn.ID = va.vehicle_registration_nation_id
left join dbo.IWCycle iw on iw.ID = va.ID_IWCycle
left join dbo.Card c on c.ID = iw.ID_Card
left join dbo.Nation cn on cn.ID = c.ID_Nation
)
select
cast(base.ID as varchar(128)) as source_row_id,
concat('TACHOGRAPH:VU_ACTIVITY:', base.ID, ':', evt.lifecycle) as external_source_event_id,
evt.occurred_at as occurred_at,
base.received_partner_at,
base.Activity as activity_code,
case upper(coalesce(base.Activity, ''))
when 'DRIVING' then 'DRIVE'
when 'DRIVE' then 'DRIVE'
when 'WORK' then 'WORK'
@ -25,68 +142,35 @@ select
when 'REST' then 'BREAK_REST'
else 'UNKNOWN_ACTIVITY'
end as event_type,
'SNAPSHOT' as lifecycle,
va.Slot as card_slot,
va.CardStatus as card_status,
va.DrivingStatus as driving_status,
cast(iw.OdoBegin as bigint) as odometer_m,
evt.lifecycle as lifecycle,
base.Slot as card_slot,
base.CardStatus as card_status,
base.DrivingStatus as driving_status,
base.odometer_m,
cast(d.ID as varchar(128)) as driver_source_entity_id,
cn.AlphaCode as driver_card_nation,
c.CardNumber as driver_card_number,
cast(base.driver_id as varchar(128)) as driver_source_entity_id,
base.driver_card_nation,
base.driver_card_number,
cast(v.ID as varchar(128)) as vehicle_source_entity_id,
vi.VIN as vehicle_vin,
vn.AlphaCode as vehicle_registration_nation,
v.VRN as vehicle_registration_number,
cast(base.vehicle_identification_id as varchar(128)) as vehicle_source_entity_id,
base.vehicle_vin,
cast(base.vehicle_registration_id as varchar(128)) as vehicle_registration_source_entity_id,
base.vehicle_registration_nation,
base.vehicle_registration_number,
'VEHICLE_UNIT' as source_package_kind,
cast(coalesce(fl.ID, va.ID_FileLog, vda.ID_FileLog, vui.ID_FileLog, va.ID) as varchar(128)) as source_package_id,
cast(coalesce(fl.ID_VehicleIdentification, vui.ID_VehicleIdentification) as varchar(128)) as source_package_entity_id,
coalesce(fl.DownloadFrom, vda.RecordDate) as source_package_period_from,
coalesce(fl.DownloadTo, dateadd(day, 1, vda.RecordDate)) as source_package_period_to,
coalesce(fl.CreationDate, fl.TStamp) as source_package_imported_at
from dbo.VUActivity va
join dbo.VUDailyActivity vda on vda.ID = va.ID_VUDailyActivity
join dbo.VUInstallation vui on vui.ID = vda.ID_VUInstallation
left join dbo.FileLog fl on fl.ID = coalesce(va.ID_FileLog, vda.ID_FileLog, vui.ID_FileLog)
join dbo.VehicleIdentification vi on vi.ID = vui.ID_VehicleIdentification
outer apply (
select top 1 vehicle.ID,
vehicle.VRN,
vehicle.ID_Nation
from dbo.Vehicle vehicle
where vehicle.ID_VehicleIdentification = vi.ID
and (vehicle.ValidFrom is null or vehicle.ValidFrom <= va.BeginTime)
and (vehicle.ValidTo is null or vehicle.ValidTo > va.BeginTime)
order by
case when vehicle.ValidFrom is null then 1 else 0 end,
vehicle.ValidFrom desc,
vehicle.ID desc
) v
left join dbo.Nation vn on vn.ID = v.ID_Nation
left join dbo.IWCycle iw on iw.ID = va.ID_IWCycle
left join dbo.Card c on c.ID = iw.ID_Card
left join dbo.Driver d on d.ID = c.ID_Driver
left join dbo.Nation cn on cn.ID = c.ID_Nation
where (:occurredFrom is null or va.BeginTime >= :occurredFrom)
and (:occurredTo is null or va.BeginTime < :occurredTo)
and (
:lastSourcePackageImportedAt is null
or coalesce(fl.CreationDate, fl.TStamp) > :lastSourcePackageImportedAt
or (
coalesce(fl.CreationDate, fl.TStamp) = :lastSourcePackageImportedAt
and coalesce(fl.ID, va.ID_FileLog, vda.ID_FileLog, vui.ID_FileLog, va.ID) > try_convert(int, :lastSourcePackageId)
)
or (
coalesce(fl.CreationDate, fl.TStamp) is null
and (
:lastSourcePackageId is null
or coalesce(fl.ID, va.ID_FileLog, vda.ID_FileLog, vui.ID_FileLog, va.ID) > try_convert(int, :lastSourcePackageId)
)
)
)
cast(base.source_package_id_raw as varchar(128)) as source_package_id,
cast(base.source_package_entity_id_raw as varchar(128)) as source_package_entity_id,
base.source_package_period_from,
base.source_package_period_to,
base.source_package_imported_at
from Base base
cross apply (values
('START', base.BeginTime),
('END', base.EndTime)
) evt(lifecycle, occurred_at)
where (:occurredFrom is null or evt.occurred_at >= :occurredFrom)
and (:occurredTo is null or evt.occurred_at < :occurredTo)
/*
* Organisation filtering can use FileLog.I_90021_ID / FileLog.OrgID or
* Vehicle_I_90021 / Driver_I_90021 once subtree semantics are confirmed.
* Organisation filter: vehicle membership in GetOrganisationTree(null, :organisationId, 0, null).
*/

View File

@ -0,0 +1,100 @@
/*
* VUBorderCrossing BORDER_CROSSING extraction for the bytebar tachograph schema.
*/
with OrgTree as (
select org.I_90021_OID
from dbo.GetOrganisationTree(null, :organisationId, 0, null) org
where :organisationId is not null
)
,
Base as (
select
border.ID,
border.Timestamp as occurred_at,
border.Odo,
border.ID_FileLog,
border.ID_VUInstallation,
coalesce(driverCard.ID, coDriverCard.ID) as card_id,
coalesce(driverCard.ID_Driver, coDriverCard.ID_Driver) as driver_id,
coalesce(driverNation.AlphaCode, coDriverNation.AlphaCode) as driver_card_nation,
coalesce(driverCard.CardNumber, coDriverCard.CardNumber) as driver_card_number,
leftNation.AlphaCode as country_from,
enteredNation.AlphaCode as country_to,
gnss.Latitude as latitude,
gnss.Longitude as longitude,
vui.ID_FileLog as vui_filelog_id,
vui.ID_VehicleIdentification,
vi.ID as vehicle_identification_id,
vi.VIN as vehicle_vin,
coalesce(fl.DownloadDate, fl.OriginalDownloadDate, fl.TStamp, fl.CreationDate) as received_partner_at,
coalesce(fl.ID, border.ID_FileLog, vui.ID_FileLog) as source_package_id_raw,
coalesce(fl.ID_VehicleIdentification, vui.ID_VehicleIdentification) as source_package_entity_id_raw,
coalesce(fl.DownloadFrom, border.Timestamp) as source_package_period_from,
coalesce(fl.DownloadTo, border.Timestamp) as source_package_period_to,
coalesce(fl.CreationDate, fl.TStamp) as source_package_imported_at
from dbo.VUBorderCrossing border
join dbo.VUInstallation vui on vui.ID = border.ID_VUInstallation
join dbo.VehicleIdentification vi on vi.ID = vui.ID_VehicleIdentification
left join dbo.Card driverCard on driverCard.ID = border.ID_DriverCard
left join dbo.Nation driverNation on driverNation.ID = driverCard.ID_Nation
left join dbo.Card coDriverCard on coDriverCard.ID = border.ID_CoDriverCard
left join dbo.Nation coDriverNation on coDriverNation.ID = coDriverCard.ID_Nation
left join dbo.Nation leftNation on leftNation.ID = border.ID_NationLeft
left join dbo.Nation enteredNation on enteredNation.ID = border.ID_NationEntered
left join dbo.GnssPlace gnss on gnss.ID = border.ID_GnssPlace
left join dbo.FileLog fl on fl.ID = coalesce(border.ID_FileLog, vui.ID_FileLog)
where (:occurredFrom is null or border.Timestamp >= :occurredFrom)
and (:occurredTo is null or border.Timestamp < :occurredTo)
)
select
cast(base.ID as varchar(128)) as source_row_id,
concat('TACHOGRAPH:VU_BORDER_CROSSING:', base.ID) as external_source_event_id,
base.occurred_at,
base.received_partner_at,
cast(base.Odo as bigint) * 1000 as odometer_m,
base.latitude,
base.longitude,
base.country_from,
base.country_to,
cast(base.driver_id as varchar(128)) as driver_source_entity_id,
base.driver_card_nation,
base.driver_card_number,
cast(base.vehicle_identification_id as varchar(128)) as vehicle_source_entity_id,
base.vehicle_vin,
cast(v.ID as varchar(128)) as vehicle_registration_source_entity_id,
vn.AlphaCode as vehicle_registration_nation,
v.VRN as vehicle_registration_number,
'VEHICLE_UNIT' as source_package_kind,
cast(base.source_package_id_raw as varchar(128)) as source_package_id,
cast(base.source_package_entity_id_raw as varchar(128)) as source_package_entity_id,
base.source_package_period_from,
base.source_package_period_to,
base.source_package_imported_at
from Base base
outer apply (
select top 1 vehicle.ID,
vehicle.VRN,
vehicle.ID_Nation
from dbo.Vehicle vehicle
where vehicle.ID_VehicleIdentification = base.vehicle_identification_id
and (vehicle.ValidFrom is null or vehicle.ValidFrom <= base.occurred_at)
and (vehicle.ValidTo is null or vehicle.ValidTo > base.occurred_at)
order by
vehicle.ValidFrom desc,
vehicle.ID desc
) v
left join dbo.Nation vn on vn.ID = v.ID_Nation
where (
:organisationId is null
or exists (
select 1
from dbo.Vehicle_I_90021 rel
join OrgTree on OrgTree.I_90021_OID = rel.ID_I_90021
where rel.ID_Vehicle = v.ID
and rel.GILT_BIS is null
)
)

View File

@ -0,0 +1,105 @@
/*
* VULoadUnload LOAD_UNLOAD extraction for the bytebar tachograph schema.
*/
with OrgTree as (
select org.I_90021_OID
from dbo.GetOrganisationTree(null, :organisationId, 0, null) org
where :organisationId is not null
)
,
Base as (
select
lu.ID,
lu.Timestamp as occurred_at,
lu.OperationType,
lu.Odo,
lu.ID_FileLog,
lu.ID_VUInstallation,
coalesce(driverCard.ID, coDriverCard.ID) as card_id,
coalesce(driverCard.ID_Driver, coDriverCard.ID_Driver) as driver_id,
coalesce(driverNation.AlphaCode, coDriverNation.AlphaCode) as driver_card_nation,
coalesce(driverCard.CardNumber, coDriverCard.CardNumber) as driver_card_number,
gnss.Latitude as latitude,
gnss.Longitude as longitude,
vui.ID_FileLog as vui_filelog_id,
vui.ID_VehicleIdentification,
vi.ID as vehicle_identification_id,
vi.VIN as vehicle_vin,
coalesce(fl.DownloadDate, fl.OriginalDownloadDate, fl.TStamp, fl.CreationDate) as received_partner_at,
coalesce(fl.ID, lu.ID_FileLog, vui.ID_FileLog) as source_package_id_raw,
coalesce(fl.ID_VehicleIdentification, vui.ID_VehicleIdentification) as source_package_entity_id_raw,
coalesce(fl.DownloadFrom, lu.Timestamp) as source_package_period_from,
coalesce(fl.DownloadTo, lu.Timestamp) as source_package_period_to,
coalesce(fl.CreationDate, fl.TStamp) as source_package_imported_at
from dbo.VULoadUnload lu
join dbo.VUInstallation vui on vui.ID = lu.ID_VUInstallation
join dbo.VehicleIdentification vi on vi.ID = vui.ID_VehicleIdentification
left join dbo.Card driverCard on driverCard.ID = lu.ID_DriverCard
left join dbo.Nation driverNation on driverNation.ID = driverCard.ID_Nation
left join dbo.Card coDriverCard on coDriverCard.ID = lu.ID_CoDriverCard
left join dbo.Nation coDriverNation on coDriverNation.ID = coDriverCard.ID_Nation
left join dbo.GnssPlace gnss on gnss.ID = lu.ID_GnssPlace
left join dbo.FileLog fl on fl.ID = coalesce(lu.ID_FileLog, vui.ID_FileLog)
where (:occurredFrom is null or lu.Timestamp >= :occurredFrom)
and (:occurredTo is null or lu.Timestamp < :occurredTo)
)
select
cast(base.ID as varchar(128)) as source_row_id,
concat('TACHOGRAPH:VU_LOAD_UNLOAD:', base.ID) as external_source_event_id,
base.occurred_at,
base.received_partner_at,
cast(base.Odo as bigint) * 1000 as odometer_m,
base.latitude,
base.longitude,
case base.OperationType
when 1 then 'LOAD'
when 2 then 'UNLOAD'
else 'LOAD_UNLOAD'
end as operation,
case base.OperationType
when 1 then 'LOAD'
when 2 then 'UNLOAD'
else 'LOAD_UNLOAD'
end as event_type,
cast(base.driver_id as varchar(128)) as driver_source_entity_id,
base.driver_card_nation,
base.driver_card_number,
cast(base.vehicle_identification_id as varchar(128)) as vehicle_source_entity_id,
base.vehicle_vin,
cast(v.ID as varchar(128)) as vehicle_registration_source_entity_id,
vn.AlphaCode as vehicle_registration_nation,
v.VRN as vehicle_registration_number,
'VEHICLE_UNIT' as source_package_kind,
cast(base.source_package_id_raw as varchar(128)) as source_package_id,
cast(base.source_package_entity_id_raw as varchar(128)) as source_package_entity_id,
base.source_package_period_from,
base.source_package_period_to,
base.source_package_imported_at
from Base base
outer apply (
select top 1 vehicle.ID,
vehicle.VRN,
vehicle.ID_Nation
from dbo.Vehicle vehicle
where vehicle.ID_VehicleIdentification = base.vehicle_identification_id
and (vehicle.ValidFrom is null or vehicle.ValidFrom <= base.occurred_at)
and (vehicle.ValidTo is null or vehicle.ValidTo > base.occurred_at)
order by
vehicle.ValidFrom desc,
vehicle.ID desc
) v
left join dbo.Nation vn on vn.ID = v.ID_Nation
where (
:organisationId is null
or exists (
select 1
from dbo.Vehicle_I_90021 rel
join OrgTree on OrgTree.I_90021_OID = rel.ID_I_90021
where rel.ID_Vehicle = v.ID
and rel.GILT_BIS is null
)
)

View File

@ -0,0 +1,101 @@
/*
* VUPlaces PLACE extraction for the bytebar tachograph schema.
*/
with OrgTree as (
select org.I_90021_OID
from dbo.GetOrganisationTree(null, :organisationId, 0, null) org
where :organisationId is not null
)
,
Base as (
select
place.ID,
place.EntryTime as occurred_at,
place.EntryType as entry_type,
place.Odo,
place.ID_FileLog,
c.ID as card_id,
c.ID_Driver as driver_id,
cn.AlphaCode as driver_card_nation,
c.CardNumber as driver_card_number,
placeNation.AlphaCode as country,
coalesce(region.AlphaCode, region.Name) as region,
gnss.Latitude as latitude,
gnss.Longitude as longitude,
vui.ID_FileLog as vui_filelog_id,
vui.ID_VehicleIdentification,
vi.ID as vehicle_identification_id,
vi.VIN as vehicle_vin,
coalesce(fl.DownloadDate, fl.OriginalDownloadDate, fl.TStamp, fl.CreationDate) as received_partner_at,
coalesce(fl.ID, place.ID_FileLog, vui.ID_FileLog) as source_package_id_raw,
coalesce(fl.ID_VehicleIdentification, vui.ID_VehicleIdentification) as source_package_entity_id_raw,
coalesce(fl.DownloadFrom, place.EntryTime) as source_package_period_from,
coalesce(fl.DownloadTo, place.EntryTime) as source_package_period_to,
coalesce(fl.CreationDate, fl.TStamp) as source_package_imported_at
from dbo.VUPlaces place
join dbo.VUInstallation vui on vui.ID = place.ID_VUInstallation
join dbo.VehicleIdentification vi on vi.ID = vui.ID_VehicleIdentification
left join dbo.Card c on c.ID = place.ID_Card
left join dbo.Nation cn on cn.ID = c.ID_Nation
left join dbo.Nation placeNation on placeNation.ID = place.ID_Nation
left join dbo.Region region on region.ID = place.ID_Region
left join dbo.GnssPlace gnss on gnss.ID = place.ID_GnssPlace
left join dbo.FileLog fl on fl.ID = coalesce(place.ID_FileLog, vui.ID_FileLog)
where (:occurredFrom is null or place.EntryTime >= :occurredFrom)
and (:occurredTo is null or place.EntryTime < :occurredTo)
)
select
cast(base.ID as varchar(128)) as source_row_id,
concat('TACHOGRAPH:VU_PLACE:', base.ID) as external_source_event_id,
base.occurred_at,
base.received_partner_at,
cast(base.Odo as bigint) * 1000 as odometer_m,
base.latitude,
base.longitude,
base.country,
base.region,
'WORKING_DAY_PLACE_RECORDED' as event_type,
case when base.entry_type in (0, 2, 4) then 'START' else 'END' end as lifecycle,
cast(case when base.entry_type in (2, 3, 4, 5) then 1 else 0 end as bit) as manual_entry,
cast(base.driver_id as varchar(128)) as driver_source_entity_id,
base.driver_card_nation,
base.driver_card_number,
cast(base.vehicle_identification_id as varchar(128)) as vehicle_source_entity_id,
base.vehicle_vin,
cast(v.ID as varchar(128)) as vehicle_registration_source_entity_id,
vn.AlphaCode as vehicle_registration_nation,
v.VRN as vehicle_registration_number,
'VEHICLE_UNIT' as source_package_kind,
cast(base.source_package_id_raw as varchar(128)) as source_package_id,
cast(base.source_package_entity_id_raw as varchar(128)) as source_package_entity_id,
base.source_package_period_from,
base.source_package_period_to,
base.source_package_imported_at
from Base base
outer apply (
select top 1 vehicle.ID,
vehicle.VRN,
vehicle.ID_Nation
from dbo.Vehicle vehicle
where vehicle.ID_VehicleIdentification = base.vehicle_identification_id
and (vehicle.ValidFrom is null or vehicle.ValidFrom <= base.occurred_at)
and (vehicle.ValidTo is null or vehicle.ValidTo > base.occurred_at)
order by
vehicle.ValidFrom desc,
vehicle.ID desc
) v
left join dbo.Nation vn on vn.ID = v.ID_Nation
where (
:organisationId is null
or exists (
select 1
from dbo.Vehicle_I_90021 rel
join OrgTree on OrgTree.I_90021_OID = rel.ID_I_90021
where rel.ID_Vehicle = v.ID
and rel.GILT_BIS is null
)
)

View File

@ -0,0 +1,94 @@
/*
* VUGnssAccumulatedDriving POSITION extraction for the bytebar tachograph schema.
*/
with OrgTree as (
select org.I_90021_OID
from dbo.GetOrganisationTree(null, :organisationId, 0, null) org
where :organisationId is not null
)
,
Base as (
select
pos.ID,
pos.Timestamp as occurred_at,
pos.Odo,
pos.ID_FileLog,
pos.ID_VUInstallation,
coalesce(driverCard.ID, coDriverCard.ID) as card_id,
coalesce(driverCard.ID_Driver, coDriverCard.ID_Driver) as driver_id,
coalesce(driverNation.AlphaCode, coDriverNation.AlphaCode) as driver_card_nation,
coalesce(driverCard.CardNumber, coDriverCard.CardNumber) as driver_card_number,
gnss.Latitude as latitude,
gnss.Longitude as longitude,
vui.ID_FileLog as vui_filelog_id,
vui.ID_VehicleIdentification,
vi.ID as vehicle_identification_id,
vi.VIN as vehicle_vin,
coalesce(fl.DownloadDate, fl.OriginalDownloadDate, fl.TStamp, fl.CreationDate) as received_partner_at,
coalesce(fl.ID, pos.ID_FileLog, vui.ID_FileLog) as source_package_id_raw,
coalesce(fl.ID_VehicleIdentification, vui.ID_VehicleIdentification) as source_package_entity_id_raw,
coalesce(fl.DownloadFrom, pos.Timestamp) as source_package_period_from,
coalesce(fl.DownloadTo, pos.Timestamp) as source_package_period_to,
coalesce(fl.CreationDate, fl.TStamp) as source_package_imported_at
from dbo.VUGnssAccumulatedDriving pos
join dbo.VUInstallation vui on vui.ID = pos.ID_VUInstallation
join dbo.VehicleIdentification vi on vi.ID = vui.ID_VehicleIdentification
left join dbo.Card driverCard on driverCard.ID = pos.ID_DriverCard
left join dbo.Nation driverNation on driverNation.ID = driverCard.ID_Nation
left join dbo.Card coDriverCard on coDriverCard.ID = pos.ID_CoDriverCard
left join dbo.Nation coDriverNation on coDriverNation.ID = coDriverCard.ID_Nation
join dbo.GnssPlace gnss on gnss.ID = pos.ID_GnssPlace
left join dbo.FileLog fl on fl.ID = coalesce(pos.ID_FileLog, vui.ID_FileLog)
where (:occurredFrom is null or pos.Timestamp >= :occurredFrom)
and (:occurredTo is null or pos.Timestamp < :occurredTo)
)
select
cast(base.ID as varchar(128)) as source_row_id,
concat('TACHOGRAPH:VU_POSITION:', base.ID) as external_source_event_id,
base.occurred_at,
base.received_partner_at,
cast(base.Odo as bigint) * 1000 as odometer_m,
base.latitude,
base.longitude,
cast(base.driver_id as varchar(128)) as driver_source_entity_id,
base.driver_card_nation,
base.driver_card_number,
cast(base.vehicle_identification_id as varchar(128)) as vehicle_source_entity_id,
base.vehicle_vin,
cast(v.ID as varchar(128)) as vehicle_registration_source_entity_id,
vn.AlphaCode as vehicle_registration_nation,
v.VRN as vehicle_registration_number,
'VEHICLE_UNIT' as source_package_kind,
cast(base.source_package_id_raw as varchar(128)) as source_package_id,
cast(base.source_package_entity_id_raw as varchar(128)) as source_package_entity_id,
base.source_package_period_from,
base.source_package_period_to,
base.source_package_imported_at
from Base base
outer apply (
select top 1 vehicle.ID,
vehicle.VRN,
vehicle.ID_Nation
from dbo.Vehicle vehicle
where vehicle.ID_VehicleIdentification = base.vehicle_identification_id
and (vehicle.ValidFrom is null or vehicle.ValidFrom <= base.occurred_at)
and (vehicle.ValidTo is null or vehicle.ValidTo > base.occurred_at)
order by
vehicle.ValidFrom desc,
vehicle.ID desc
) v
left join dbo.Nation vn on vn.ID = v.ID_Nation
where (
:organisationId is null
or exists (
select 1
from dbo.Vehicle_I_90021 rel
join OrgTree on OrgTree.I_90021_OID = rel.ID_I_90021
where rel.ID_Vehicle = v.ID
and rel.GILT_BIS is null
)
)

View File

@ -0,0 +1,84 @@
/*
* VUSpecificCondition SPECIFIC_CONDITION extraction for the bytebar tachograph schema.
*/
with OrgTree as (
select org.I_90021_OID
from dbo.GetOrganisationTree(null, :organisationId, 0, null) org
where :organisationId is not null
)
,
Base as (
select
cond.ID,
cond.EntryTime as occurred_at,
sc.Condition as condition_code,
cond.ID_FileLog,
cond.ID_VUInstallation,
vui.ID_FileLog as vui_filelog_id,
vui.ID_VehicleIdentification,
vi.ID as vehicle_identification_id,
vi.VIN as vehicle_vin,
coalesce(fl.DownloadDate, fl.OriginalDownloadDate, fl.TStamp, fl.CreationDate) as received_partner_at,
coalesce(fl.ID, cond.ID_FileLog, vui.ID_FileLog) as source_package_id_raw,
coalesce(fl.ID_VehicleIdentification, vui.ID_VehicleIdentification) as source_package_entity_id_raw,
coalesce(fl.DownloadFrom, cond.EntryTime) as source_package_period_from,
coalesce(fl.DownloadTo, cond.EntryTime) as source_package_period_to,
coalesce(fl.CreationDate, fl.TStamp) as source_package_imported_at
from dbo.VUSpecificConditions cond
join dbo.SpecificCondition sc on sc.ID = cond.ID_SpecificCondition
join dbo.VUInstallation vui on vui.ID = cond.ID_VUInstallation
join dbo.VehicleIdentification vi on vi.ID = vui.ID_VehicleIdentification
left join dbo.FileLog fl on fl.ID = coalesce(cond.ID_FileLog, vui.ID_FileLog)
where (:occurredFrom is null or cond.EntryTime >= :occurredFrom)
and (:occurredTo is null or cond.EntryTime < :occurredTo)
)
select
cast(base.ID as varchar(128)) as source_row_id,
concat('TACHOGRAPH:VU_SPECIFIC_CONDITION:', base.ID) as external_source_event_id,
base.occurred_at,
base.received_partner_at,
'SPECIFIC_CONDITION' as event_domain,
case when base.condition_code in (3, 4) then 'FERRY_TRAIN' else 'OUT' end as event_type,
case when base.condition_code in (1, 3) then 'BEGIN' else 'END' end as lifecycle,
cast(null as varchar(128)) as driver_source_entity_id,
cast(null as varchar(16)) as driver_card_nation,
cast(null as varchar(64)) as driver_card_number,
cast(base.vehicle_identification_id as varchar(128)) as vehicle_source_entity_id,
base.vehicle_vin,
cast(v.ID as varchar(128)) as vehicle_registration_source_entity_id,
vn.AlphaCode as vehicle_registration_nation,
v.VRN as vehicle_registration_number,
'VEHICLE_UNIT' as source_package_kind,
cast(base.source_package_id_raw as varchar(128)) as source_package_id,
cast(base.source_package_entity_id_raw as varchar(128)) as source_package_entity_id,
base.source_package_period_from,
base.source_package_period_to,
base.source_package_imported_at
from Base base
outer apply (
select top 1 vehicle.ID,
vehicle.VRN,
vehicle.ID_Nation
from dbo.Vehicle vehicle
where vehicle.ID_VehicleIdentification = base.vehicle_identification_id
and (vehicle.ValidFrom is null or vehicle.ValidFrom <= base.occurred_at)
and (vehicle.ValidTo is null or vehicle.ValidTo > base.occurred_at)
order by
vehicle.ValidFrom desc,
vehicle.ID desc
) v
left join dbo.Nation vn on vn.ID = v.ID_Nation
where (
:organisationId is null
or exists (
select 1
from dbo.Vehicle_I_90021 rel
join OrgTree on OrgTree.I_90021_OID = rel.ID_I_90021
where rel.ID_Vehicle = v.ID
and rel.GILT_BIS is null
)
)

View File

@ -0,0 +1,155 @@
package at.procon.eventhub.tachograph.service;
import at.procon.eventhub.config.EventHubProperties;
import at.procon.eventhub.dto.EventFamily;
import at.procon.eventhub.dto.EventSourceDto;
import at.procon.eventhub.dto.ImportScopeDto;
import at.procon.eventhub.importing.ImportChunkPlanner;
import at.procon.eventhub.service.EventDetailsFactory;
import at.procon.eventhub.tachograph.dto.TachographImportRequest;
import com.fasterxml.jackson.databind.ObjectMapper;
import java.util.EnumSet;
import java.util.List;
import org.junit.jupiter.api.Test;
import static org.assertj.core.api.Assertions.assertThat;
class TachographImportPlanServiceTest {
private final EventDetailsFactory detailsFactory = new EventDetailsFactory(new ObjectMapper());
private final TachographExtractionDefinitionRegistry definitionRegistry = new TachographExtractionDefinitionRegistry(
new CardActivityRowMapper(detailsFactory),
new VuActivityRowMapper(detailsFactory),
new IwCycleCardEventRowMapper(detailsFactory),
new CardVehiclesUsedCardEventRowMapper(detailsFactory),
new VuBorderCrossingRowMapper(detailsFactory),
new CardBorderCrossingRowMapper(detailsFactory),
new VuLoadUnloadRowMapper(detailsFactory),
new CardLoadUnloadRowMapper(detailsFactory),
new VuSpecificConditionRowMapper(detailsFactory),
new CardSpecificConditionRowMapper(detailsFactory),
new VuPositionRowMapper(detailsFactory),
new CardPositionRowMapper(detailsFactory),
new VuPlaceRowMapper(detailsFactory),
new CardPlaceRowMapper(detailsFactory),
new SpeedingEventRowMapper(detailsFactory)
);
@Test
void allowsSupportedDriverActivityFamilyWhenJdbcExtractionIsEnabled() {
TachographImportPlanService service = serviceWithJdbcExtractor();
TachographImportRequest request = requestForFamilies(EventFamily.DRIVER_ACTIVITY);
var plan = service.createPlan(request);
assertThat(plan.items())
.extracting(item -> item.extractionCode())
.containsExactlyInAnyOrder("VU_ACTIVITY", "CARD_ACTIVITY");
}
@Test
void allowsSupportedDriverCardFamilyWhenJdbcExtractionIsEnabled() {
TachographImportPlanService service = serviceWithJdbcExtractor();
TachographImportRequest request = requestForFamilies(EventFamily.DRIVER_CARD);
var plan = service.createPlan(request);
assertThat(plan.items())
.extracting(item -> item.extractionCode())
.containsExactlyInAnyOrder("IW_CYCLE", "CARD_VEHICLES_USED");
}
@Test
void allowsSupportedBorderCrossingFamilyWhenJdbcExtractionIsEnabled() {
TachographImportPlanService service = serviceWithJdbcExtractor();
TachographImportRequest request = requestForFamilies(EventFamily.BORDER_CROSSING);
var plan = service.createPlan(request);
assertThat(plan.items())
.extracting(item -> item.extractionCode())
.containsExactlyInAnyOrder("VU_BORDER_CROSSING", "CARD_BORDER_CROSSING");
}
@Test
void allowsSupportedLoadUnloadFamilyWhenJdbcExtractionIsEnabled() {
TachographImportPlanService service = serviceWithJdbcExtractor();
TachographImportRequest request = requestForFamilies(EventFamily.LOAD_UNLOAD);
var plan = service.createPlan(request);
assertThat(plan.items())
.extracting(item -> item.extractionCode())
.containsExactlyInAnyOrder("VU_LOAD_UNLOAD", "CARD_LOAD_UNLOAD");
}
@Test
void allowsSupportedSpecificConditionFamilyWhenJdbcExtractionIsEnabled() {
TachographImportPlanService service = serviceWithJdbcExtractor();
TachographImportRequest request = requestForFamilies(EventFamily.SPECIFIC_CONDITION);
var plan = service.createPlan(request);
assertThat(plan.items())
.extracting(item -> item.extractionCode())
.containsExactlyInAnyOrder("VU_SPECIFIC_CONDITION", "CARD_SPECIFIC_CONDITION");
}
@Test
void allowsSupportedPositionFamilyWhenJdbcExtractionIsEnabled() {
TachographImportPlanService service = serviceWithJdbcExtractor();
TachographImportRequest request = requestForFamilies(EventFamily.POSITION);
var plan = service.createPlan(request);
assertThat(plan.items())
.extracting(item -> item.extractionCode())
.containsExactlyInAnyOrder("VU_POSITION", "CARD_POSITION");
}
@Test
void allowsSupportedPlaceFamilyWhenJdbcExtractionIsEnabled() {
TachographImportPlanService service = serviceWithJdbcExtractor();
TachographImportRequest request = requestForFamilies(EventFamily.PLACE);
var plan = service.createPlan(request);
assertThat(plan.items())
.extracting(item -> item.extractionCode())
.containsExactlyInAnyOrder("VU_PLACE", "CARD_PLACE");
}
@Test
void allowsSupportedSpeedingFamilyWhenJdbcExtractionIsEnabled() {
TachographImportPlanService service = serviceWithJdbcExtractor();
TachographImportRequest request = requestForFamilies(EventFamily.SPEEDING);
var plan = service.createPlan(request);
assertThat(plan.items())
.extracting(item -> item.extractionCode())
.containsExactly("SPEEDING_EVENTS");
}
private TachographImportPlanService serviceWithJdbcExtractor() {
EventHubProperties properties = new EventHubProperties();
properties.getTachograph().getDatasource().setJdbcUrl("jdbc:sqlserver://tachograph-db");
return new TachographImportPlanService(properties, new ImportChunkPlanner(), definitionRegistry);
}
private TachographImportRequest requestForFamilies(EventFamily... families) {
EnumSet<EventFamily> selectedFamilies = families.length == 0
? EnumSet.noneOf(EventFamily.class)
: EnumSet.copyOf(List.of(families));
return new TachographImportRequest(
"tenant-1",
new EventSourceDto("TACHOGRAPH", "MIXED", "TACHOGRAPH_DB", "tachograph-db", null, null),
null,
ImportScopeDto.tenantAll(null, null),
selectedFamilies,
null,
false,
null
);
}
}