Compare commits
11 Commits
e3ffa56932
...
b8817e18d9
| Author | SHA1 | Date |
|---|---|---|
|
|
b8817e18d9 | |
|
|
d0a8d44082 | |
|
|
bd3620b9af | |
|
|
2e6e1aa5c6 | |
|
|
c21530826c | |
|
|
9d9541bac9 | |
|
|
7ed0c73107 | |
|
|
dfdc5cdeb9 | |
|
|
d27be66d0d | |
|
|
866e275691 | |
|
|
ec533bb24f |
38
README.md
38
README.md
|
|
@ -264,8 +264,8 @@ DRIVER_ACTIVITY / VEHICLE_UNIT -> VUActivity
|
||||||
DRIVER_ACTIVITY / DRIVER_CARD -> CardActivity
|
DRIVER_ACTIVITY / DRIVER_CARD -> CardActivity
|
||||||
DRIVER_CARD / VEHICLE_UNIT -> IWCycle
|
DRIVER_CARD / VEHICLE_UNIT -> IWCycle
|
||||||
DRIVER_CARD / DRIVER_CARD -> CardVehiclesUsed
|
DRIVER_CARD / DRIVER_CARD -> CardVehiclesUsed
|
||||||
POSITION / VEHICLE_UNIT -> VUPlaces, VULoadUnload, VUGnssAccumulatedDriving, VUBorderCrossing
|
POSITION / VEHICLE_UNIT -> VUGnssAccumulatedDriving
|
||||||
POSITION / DRIVER_CARD -> CardPlaces, CardLoadUnload, CardGnssAccumulatedDriving, CardBorderCrossing
|
POSITION / DRIVER_CARD -> CardGnssAccumulatedDriving
|
||||||
BORDER_CROSSING / VEHICLE_UNIT -> VUBorderCrossing
|
BORDER_CROSSING / VEHICLE_UNIT -> VUBorderCrossing
|
||||||
BORDER_CROSSING / DRIVER_CARD -> CardBorderCrossing
|
BORDER_CROSSING / DRIVER_CARD -> CardBorderCrossing
|
||||||
LOAD_UNLOAD / VEHICLE_UNIT -> VULoadUnload
|
LOAD_UNLOAD / VEHICLE_UNIT -> VULoadUnload
|
||||||
|
|
@ -624,6 +624,10 @@ This allows a single EventHub import run to process many original tachograph pac
|
||||||
}
|
}
|
||||||
```
|
```
|
||||||
|
|
||||||
|
For tachograph JDBC extraction, `sourcePackageId` is the original `FileLog.ID`. It is stored directly on `eventhub.event.source_package_id` and also resolved to `source_package_entity_id` for joins to source master data.
|
||||||
|
|
||||||
|
`eventhub.event` is designed as a TimescaleDB hypertable partitioned by `occurred_at`. Source-record idempotency is therefore enforced through `eventhub.event_source_record(source_record_key_hash)` instead of a unique index on the hypertable, because Timescale unique indexes must include the partitioning column.
|
||||||
|
|
||||||
### Tachograph import endpoints
|
### Tachograph import endpoints
|
||||||
|
|
||||||
`POST /api/eventhub/acquisition/tachograph/imports/plan` returns the calculated event-family extraction plan and time chunks.
|
`POST /api/eventhub/acquisition/tachograph/imports/plan` returns the calculated event-family extraction plan and time chunks.
|
||||||
|
|
@ -814,14 +818,40 @@ The first concrete extractor is `JdbcTachographExtractionBatchExecutor`. It is e
|
||||||
Currently implemented extraction definitions:
|
Currently implemented extraction definitions:
|
||||||
|
|
||||||
```text
|
```text
|
||||||
CARD_ACTIVITY -> DRIVER_ACTIVITY / DRIVER_CARD / CardActivity
|
CARD_ACTIVITY -> DRIVER_ACTIVITY / DRIVER_CARD / CardActivity
|
||||||
VU_ACTIVITY -> DRIVER_ACTIVITY / VEHICLE_UNIT / VUActivity
|
VU_ACTIVITY -> DRIVER_ACTIVITY / VEHICLE_UNIT / VUActivity
|
||||||
|
CARD_VEHICLES_USED -> DRIVER_CARD / DRIVER_CARD / CardVehiclesUsed
|
||||||
|
IW_CYCLE -> DRIVER_CARD / VEHICLE_UNIT / IWCycle
|
||||||
|
CARD_BORDER_CROSSING -> BORDER_CROSSING / DRIVER_CARD / CardBorderCrossing
|
||||||
|
VU_BORDER_CROSSING -> BORDER_CROSSING / VEHICLE_UNIT / VUBorderCrossing
|
||||||
|
CARD_LOAD_UNLOAD -> LOAD_UNLOAD / DRIVER_CARD / CardLoadUnload
|
||||||
|
VU_LOAD_UNLOAD -> LOAD_UNLOAD / VEHICLE_UNIT / VULoadUnload
|
||||||
|
CARD_SPECIFIC_CONDITION -> SPECIFIC_CONDITION / DRIVER_CARD / CardSpecificCondition
|
||||||
|
VU_SPECIFIC_CONDITION -> SPECIFIC_CONDITION / VEHICLE_UNIT / VUSpecificCondition
|
||||||
|
CARD_POSITION -> POSITION / DRIVER_CARD / CardGnssAccumulatedDriving
|
||||||
|
VU_POSITION -> POSITION / VEHICLE_UNIT / VUGnssAccumulatedDriving
|
||||||
|
CARD_PLACE -> PLACE / DRIVER_CARD / CardPlaces
|
||||||
|
VU_PLACE -> PLACE / VEHICLE_UNIT / VUPlaces
|
||||||
|
SPEEDING_EVENTS -> SPEEDING / VEHICLE_UNIT / SpeedingEvents
|
||||||
```
|
```
|
||||||
|
|
||||||
SQL resources:
|
SQL resources:
|
||||||
|
|
||||||
```text
|
```text
|
||||||
|
src/main/resources/sql/tachograph/card-border-crossing.sql
|
||||||
|
src/main/resources/sql/tachograph/card-load-unload.sql
|
||||||
|
src/main/resources/sql/tachograph/card-place.sql
|
||||||
|
src/main/resources/sql/tachograph/card-position.sql
|
||||||
|
src/main/resources/sql/tachograph/card-specific-condition.sql
|
||||||
|
src/main/resources/sql/tachograph/card-vehicles-used.sql
|
||||||
src/main/resources/sql/tachograph/card-activity.sql
|
src/main/resources/sql/tachograph/card-activity.sql
|
||||||
|
src/main/resources/sql/tachograph/iw-cycle.sql
|
||||||
|
src/main/resources/sql/tachograph/speeding-events.sql
|
||||||
|
src/main/resources/sql/tachograph/vu-border-crossing.sql
|
||||||
|
src/main/resources/sql/tachograph/vu-load-unload.sql
|
||||||
|
src/main/resources/sql/tachograph/vu-place.sql
|
||||||
|
src/main/resources/sql/tachograph/vu-position.sql
|
||||||
|
src/main/resources/sql/tachograph/vu-specific-condition.sql
|
||||||
src/main/resources/sql/tachograph/vu-activity.sql
|
src/main/resources/sql/tachograph/vu-activity.sql
|
||||||
```
|
```
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,338 @@
|
||||||
|
-- Async ingest diagnostics without filesystem logs.
|
||||||
|
--
|
||||||
|
-- Replace the values in params before running.
|
||||||
|
-- completion_size must match eventhub.batch.completion-size from application.yml.
|
||||||
|
|
||||||
|
-- 1) Import-run overview.
|
||||||
|
with params as (
|
||||||
|
select
|
||||||
|
'667b191a-4ef4-49d4-9837-05a7737b4fbb'::uuid as import_run_id,
|
||||||
|
'54ba1294-1061-4b3a-84b6-5a604aa651e6'::uuid as extraction_package_id,
|
||||||
|
5000::integer as completion_size
|
||||||
|
)
|
||||||
|
select
|
||||||
|
p.id,
|
||||||
|
p.package_type,
|
||||||
|
p.status,
|
||||||
|
p.batch_no,
|
||||||
|
p.event_family,
|
||||||
|
p.extraction_code,
|
||||||
|
p.extraction_source_kind,
|
||||||
|
p.chunk_from,
|
||||||
|
p.chunk_to,
|
||||||
|
p.event_count,
|
||||||
|
p.error_message,
|
||||||
|
p.received_at,
|
||||||
|
p.completed_at
|
||||||
|
from eventhub.data_package p
|
||||||
|
join params prm on prm.import_run_id = p.import_run_id
|
||||||
|
order by p.package_type, p.batch_no, p.received_at, p.package_key;
|
||||||
|
|
||||||
|
|
||||||
|
-- 2) Exact async-ingest status for one DB_EXTRACT package.
|
||||||
|
-- event_count on DB_EXTRACT is populated after executeBatch returns and before
|
||||||
|
-- async wait starts, so it represents extracted/mapped events while waiting.
|
||||||
|
with params as (
|
||||||
|
select
|
||||||
|
'667b191a-4ef4-49d4-9837-05a7737b4fbb'::uuid as import_run_id,
|
||||||
|
'54ba1294-1061-4b3a-84b6-5a604aa651e6'::uuid as extraction_package_id,
|
||||||
|
5000::integer as completion_size
|
||||||
|
),
|
||||||
|
extract_package as (
|
||||||
|
select
|
||||||
|
p.id,
|
||||||
|
p.import_run_id,
|
||||||
|
p.event_source_id,
|
||||||
|
p.tenant_key,
|
||||||
|
p.package_key,
|
||||||
|
p.status,
|
||||||
|
p.event_count as extracted_event_count,
|
||||||
|
p.event_family,
|
||||||
|
p.business_date,
|
||||||
|
p.extraction_code,
|
||||||
|
p.extraction_source_kind,
|
||||||
|
p.source_group_type,
|
||||||
|
p.source_group_entity_id,
|
||||||
|
p.source_group_code,
|
||||||
|
p.import_scope_type,
|
||||||
|
p.root_source_org_entity_id,
|
||||||
|
p.root_source_org_code,
|
||||||
|
p.include_children,
|
||||||
|
p.chunk_from,
|
||||||
|
p.chunk_to,
|
||||||
|
coalesce((p.metadata ->> 'chunkSequence')::integer, 1) as chunk_sequence,
|
||||||
|
es.provider_key,
|
||||||
|
es.source_kind,
|
||||||
|
es.source_key,
|
||||||
|
es.source_instance_key,
|
||||||
|
es.provider_key || ':' || p.extraction_source_kind || ':' || p.extraction_code
|
||||||
|
|| ':RUN-' || p.import_run_id::text
|
||||||
|
|| ':CHUNK-' || coalesce((p.metadata ->> 'chunkSequence')::integer, 1)::text as async_external_package_id,
|
||||||
|
p.tenant_key || ':'
|
||||||
|
|| es.provider_key || ':' || es.source_kind || ':' || es.source_key || ':' || coalesce(es.source_instance_key, 'default') || ':'
|
||||||
|
|| case
|
||||||
|
when p.source_group_type is null and p.source_group_entity_id is null and p.source_group_code is null
|
||||||
|
then 'NO_GROUP'
|
||||||
|
else coalesce(p.source_group_type, '') || '|' || coalesce(p.source_group_entity_id, '') || '|' || coalesce(p.source_group_code, '')
|
||||||
|
end || ':'
|
||||||
|
|| coalesce(p.import_scope_type, 'TENANT_ALL') || ':'
|
||||||
|
|| case
|
||||||
|
when p.root_source_org_entity_id is null and p.root_source_org_code is null
|
||||||
|
then 'ALL'
|
||||||
|
else 'ORGANISATION|' || coalesce(p.root_source_org_entity_id, '') || '|' || coalesce(p.root_source_org_code, '')
|
||||||
|
end || ':'
|
||||||
|
|| case when coalesce(p.include_children, false) then 'WITH_CHILDREN' else 'NO_CHILDREN' end || ':'
|
||||||
|
|| coalesce(p.metadata ->> 'chunkOccurredFrom', 'BEGIN') || ':'
|
||||||
|
|| coalesce(p.metadata ->> 'chunkOccurredTo', 'END') || ':'
|
||||||
|
|| p.event_family || ':'
|
||||||
|
|| coalesce(p.business_date::text, 'NO_DATE') || ':'
|
||||||
|
|| es.provider_key || ':' || p.extraction_source_kind || ':' || p.extraction_code
|
||||||
|
|| ':RUN-' || p.import_run_id::text
|
||||||
|
|| ':CHUNK-' || coalesce((p.metadata ->> 'chunkSequence')::integer, 1)::text as aggregate_package_key
|
||||||
|
from eventhub.data_package p
|
||||||
|
join eventhub.event_source es
|
||||||
|
on es.id = p.event_source_id
|
||||||
|
join params prm
|
||||||
|
on prm.extraction_package_id = p.id
|
||||||
|
where p.package_type = 'DB_EXTRACT'
|
||||||
|
),
|
||||||
|
camel_batches as (
|
||||||
|
select
|
||||||
|
c.id,
|
||||||
|
c.package_key,
|
||||||
|
c.status,
|
||||||
|
c.received_at,
|
||||||
|
c.completed_at,
|
||||||
|
c.event_count as inserted_count,
|
||||||
|
coalesce((c.metadata ->> 'eventCount')::integer, 0) as received_count,
|
||||||
|
c.metadata ->> 'externalPackageId' as external_package_id,
|
||||||
|
c.metadata ->> 'aggregatePackageKey' as aggregate_package_key,
|
||||||
|
c.error_message
|
||||||
|
from eventhub.data_package c
|
||||||
|
join extract_package e
|
||||||
|
on c.event_source_id = e.event_source_id
|
||||||
|
and c.tenant_key = e.tenant_key
|
||||||
|
where c.package_type = 'CAMEL_BATCH'
|
||||||
|
and c.package_key like e.aggregate_package_key || ':CAMEL-%'
|
||||||
|
),
|
||||||
|
camel_summary as (
|
||||||
|
select
|
||||||
|
count(*) as observed_camel_batches,
|
||||||
|
count(*) filter (where status in ('IMPORTED', 'EMPTY')) as successful_camel_batches,
|
||||||
|
count(*) filter (where status = 'FAILED') as failed_camel_batches,
|
||||||
|
count(*) filter (where status = 'IMPORTING') as importing_camel_batches,
|
||||||
|
coalesce(sum(received_count), 0) as received_events,
|
||||||
|
coalesce(sum(inserted_count), 0) as inserted_events
|
||||||
|
from camel_batches
|
||||||
|
)
|
||||||
|
select
|
||||||
|
e.id as extraction_package_id,
|
||||||
|
e.import_run_id,
|
||||||
|
e.package_key as extraction_package_key,
|
||||||
|
e.status as extraction_status,
|
||||||
|
e.extracted_event_count,
|
||||||
|
case
|
||||||
|
when e.extracted_event_count <= 0 then 0
|
||||||
|
else ((e.extracted_event_count - 1) / prm.completion_size) + 1
|
||||||
|
end as expected_camel_batches,
|
||||||
|
s.observed_camel_batches,
|
||||||
|
s.successful_camel_batches,
|
||||||
|
s.failed_camel_batches,
|
||||||
|
s.importing_camel_batches,
|
||||||
|
s.received_events,
|
||||||
|
s.inserted_events,
|
||||||
|
e.async_external_package_id,
|
||||||
|
e.aggregate_package_key
|
||||||
|
from extract_package e
|
||||||
|
cross join camel_summary s
|
||||||
|
cross join params prm;
|
||||||
|
|
||||||
|
|
||||||
|
-- 3) Child CAMEL_BATCH rows for the selected DB_EXTRACT package.
|
||||||
|
with params as (
|
||||||
|
select
|
||||||
|
'667b191a-4ef4-49d4-9837-05a7737b4fbb'::uuid as import_run_id,
|
||||||
|
'54ba1294-1061-4b3a-84b6-5a604aa651e6'::uuid as extraction_package_id,
|
||||||
|
5000::integer as completion_size
|
||||||
|
),
|
||||||
|
extract_package as (
|
||||||
|
select
|
||||||
|
p.id,
|
||||||
|
p.event_source_id,
|
||||||
|
p.tenant_key,
|
||||||
|
p.tenant_key || ':'
|
||||||
|
|| es.provider_key || ':' || es.source_kind || ':' || es.source_key || ':' || coalesce(es.source_instance_key, 'default') || ':'
|
||||||
|
|| case
|
||||||
|
when p.source_group_type is null and p.source_group_entity_id is null and p.source_group_code is null
|
||||||
|
then 'NO_GROUP'
|
||||||
|
else coalesce(p.source_group_type, '') || '|' || coalesce(p.source_group_entity_id, '') || '|' || coalesce(p.source_group_code, '')
|
||||||
|
end || ':'
|
||||||
|
|| coalesce(p.import_scope_type, 'TENANT_ALL') || ':'
|
||||||
|
|| case
|
||||||
|
when p.root_source_org_entity_id is null and p.root_source_org_code is null
|
||||||
|
then 'ALL'
|
||||||
|
else 'ORGANISATION|' || coalesce(p.root_source_org_entity_id, '') || '|' || coalesce(p.root_source_org_code, '')
|
||||||
|
end || ':'
|
||||||
|
|| case when coalesce(p.include_children, false) then 'WITH_CHILDREN' else 'NO_CHILDREN' end || ':'
|
||||||
|
|| coalesce(p.metadata ->> 'chunkOccurredFrom', 'BEGIN') || ':'
|
||||||
|
|| coalesce(p.metadata ->> 'chunkOccurredTo', 'END') || ':'
|
||||||
|
|| p.event_family || ':'
|
||||||
|
|| coalesce(p.business_date::text, 'NO_DATE') || ':'
|
||||||
|
|| es.provider_key || ':' || p.extraction_source_kind || ':' || p.extraction_code
|
||||||
|
|| ':RUN-' || p.import_run_id::text
|
||||||
|
|| ':CHUNK-' || coalesce((p.metadata ->> 'chunkSequence')::integer, 1)::text as aggregate_package_key
|
||||||
|
from eventhub.data_package p
|
||||||
|
join eventhub.event_source es
|
||||||
|
on es.id = p.event_source_id
|
||||||
|
join params prm
|
||||||
|
on prm.extraction_package_id = p.id
|
||||||
|
where p.package_type = 'DB_EXTRACT'
|
||||||
|
)
|
||||||
|
select
|
||||||
|
c.id,
|
||||||
|
c.package_key,
|
||||||
|
c.status,
|
||||||
|
c.received_at,
|
||||||
|
c.completed_at,
|
||||||
|
coalesce((c.metadata ->> 'eventCount')::integer, 0) as received_count,
|
||||||
|
c.event_count as inserted_count,
|
||||||
|
c.metadata ->> 'externalPackageId' as external_package_id,
|
||||||
|
c.metadata ->> 'aggregatePackageKey' as aggregate_package_key,
|
||||||
|
c.error_message
|
||||||
|
from eventhub.data_package c
|
||||||
|
join extract_package e
|
||||||
|
on c.event_source_id = e.event_source_id
|
||||||
|
and c.tenant_key = e.tenant_key
|
||||||
|
where c.package_type = 'CAMEL_BATCH'
|
||||||
|
and c.package_key like e.aggregate_package_key || ':CAMEL-%'
|
||||||
|
order by c.received_at, c.package_key;
|
||||||
|
|
||||||
|
|
||||||
|
-- 4) Recent CAMEL_BATCH rows for the run, independent of a single DB_EXTRACT package.
|
||||||
|
with params as (
|
||||||
|
select
|
||||||
|
'667b191a-4ef4-49d4-9837-05a7737b4fbb'::uuid as import_run_id,
|
||||||
|
'54ba1294-1061-4b3a-84b6-5a604aa651e6'::uuid as extraction_package_id,
|
||||||
|
5000::integer as completion_size
|
||||||
|
)
|
||||||
|
select
|
||||||
|
c.id,
|
||||||
|
c.event_source_id,
|
||||||
|
c.package_key,
|
||||||
|
c.status,
|
||||||
|
c.received_at,
|
||||||
|
c.completed_at,
|
||||||
|
c.event_count as inserted_count,
|
||||||
|
c.metadata ->> 'eventFamily' as event_family,
|
||||||
|
c.metadata ->> 'eventSource' as event_source,
|
||||||
|
c.metadata ->> 'externalPackageId' as external_package_id,
|
||||||
|
c.metadata ->> 'aggregatePackageKey' as aggregate_package_key
|
||||||
|
from eventhub.data_package c
|
||||||
|
join params prm
|
||||||
|
on c.package_type = 'CAMEL_BATCH'
|
||||||
|
where c.package_key like '%RUN-' || prm.import_run_id::text || '%'
|
||||||
|
or coalesce(c.metadata ->> 'externalPackageId', '') like '%RUN-' || prm.import_run_id::text || '%'
|
||||||
|
order by c.received_at desc, c.package_key desc;
|
||||||
|
|
||||||
|
|
||||||
|
-- 5) Per event type: extracted vs received by CAMEL vs imported into eventhub.event.
|
||||||
|
with params as (
|
||||||
|
select
|
||||||
|
'4d21ef91-c979-451b-9055-d574506843bf'::uuid as import_run_id,
|
||||||
|
'20679175-b481-4439-9f66-9b7c7ed336fb'::uuid as extraction_package_id,
|
||||||
|
5000::integer as completion_size
|
||||||
|
),
|
||||||
|
extract_package as (
|
||||||
|
select
|
||||||
|
p.id,
|
||||||
|
p.import_run_id,
|
||||||
|
p.event_source_id,
|
||||||
|
p.tenant_key,
|
||||||
|
p.event_family,
|
||||||
|
p.business_date,
|
||||||
|
p.extraction_code,
|
||||||
|
p.extraction_source_kind,
|
||||||
|
p.source_group_type,
|
||||||
|
p.source_group_entity_id,
|
||||||
|
p.source_group_code,
|
||||||
|
p.import_scope_type,
|
||||||
|
p.root_source_org_entity_id,
|
||||||
|
p.root_source_org_code,
|
||||||
|
p.include_children,
|
||||||
|
p.metadata,
|
||||||
|
es.provider_key,
|
||||||
|
es.source_kind,
|
||||||
|
es.source_key,
|
||||||
|
es.source_instance_key,
|
||||||
|
p.tenant_key || ':'
|
||||||
|
|| es.provider_key || ':' || es.source_kind || ':' || es.source_key || ':' || coalesce(es.source_instance_key, 'default') || ':'
|
||||||
|
|| case
|
||||||
|
when p.source_group_type is null and p.source_group_entity_id is null and p.source_group_code is null
|
||||||
|
then 'NO_GROUP'
|
||||||
|
else coalesce(p.source_group_type, '') || '|' || coalesce(p.source_group_entity_id, '') || '|' || coalesce(p.source_group_code, '')
|
||||||
|
end || ':'
|
||||||
|
|| coalesce(p.import_scope_type, 'TENANT_ALL') || ':'
|
||||||
|
|| case
|
||||||
|
when p.root_source_org_entity_id is null and p.root_source_org_code is null
|
||||||
|
then 'ALL'
|
||||||
|
else 'ORGANISATION|' || coalesce(p.root_source_org_entity_id, '') || '|' || coalesce(p.root_source_org_code, '')
|
||||||
|
end || ':'
|
||||||
|
|| case when coalesce(p.include_children, false) then 'WITH_CHILDREN' else 'NO_CHILDREN' end || ':'
|
||||||
|
|| coalesce(p.metadata ->> 'chunkOccurredFrom', 'BEGIN') || ':'
|
||||||
|
|| coalesce(p.metadata ->> 'chunkOccurredTo', 'END') || ':'
|
||||||
|
|| p.event_family || ':'
|
||||||
|
|| coalesce(p.business_date::text, 'NO_DATE') || ':'
|
||||||
|
|| es.provider_key || ':' || p.extraction_source_kind || ':' || p.extraction_code
|
||||||
|
|| ':RUN-' || p.import_run_id::text
|
||||||
|
|| ':CHUNK-' || coalesce((p.metadata ->> 'chunkSequence')::integer, 1)::text as aggregate_package_key
|
||||||
|
from eventhub.data_package p
|
||||||
|
join eventhub.event_source es
|
||||||
|
on es.id = p.event_source_id
|
||||||
|
join params prm
|
||||||
|
on prm.extraction_package_id = p.id
|
||||||
|
where p.package_type = 'DB_EXTRACT'
|
||||||
|
),
|
||||||
|
camel_batches as (
|
||||||
|
select c.*
|
||||||
|
from eventhub.data_package c
|
||||||
|
join extract_package e
|
||||||
|
on c.event_source_id = e.event_source_id
|
||||||
|
and c.tenant_key = e.tenant_key
|
||||||
|
where c.package_type = 'CAMEL_BATCH'
|
||||||
|
and c.package_key like e.aggregate_package_key || ':CAMEL-%'
|
||||||
|
),
|
||||||
|
extracted_by_type as (
|
||||||
|
select
|
||||||
|
key as event_type_key,
|
||||||
|
value::integer as extracted_events
|
||||||
|
from extract_package e,
|
||||||
|
jsonb_each_text(coalesce(e.metadata -> 'extractedEventTypeCounts', '{}'::jsonb))
|
||||||
|
),
|
||||||
|
received_by_type as (
|
||||||
|
select
|
||||||
|
kv.key as event_type_key,
|
||||||
|
sum((kv.value)::integer) as received_events
|
||||||
|
from camel_batches c,
|
||||||
|
jsonb_each_text(coalesce(c.metadata -> 'receivedEventTypeCounts', '{}'::jsonb)) kv
|
||||||
|
group by kv.key
|
||||||
|
),
|
||||||
|
imported_by_type as (
|
||||||
|
select
|
||||||
|
e.event_domain || '/' || e.event_type as event_type_key,
|
||||||
|
count(*) as imported_events
|
||||||
|
from camel_batches c
|
||||||
|
join eventhub.event e
|
||||||
|
on e.data_package_id = c.id
|
||||||
|
group by e.event_domain || '/' || e.event_type
|
||||||
|
)
|
||||||
|
select
|
||||||
|
coalesce(x.event_type_key, r.event_type_key, i.event_type_key) as event_type_key,
|
||||||
|
coalesce(x.extracted_events, 0) as extracted_events,
|
||||||
|
coalesce(r.received_events, 0) as received_events,
|
||||||
|
coalesce(i.imported_events, 0) as imported_events
|
||||||
|
from extracted_by_type x
|
||||||
|
full outer join received_by_type r
|
||||||
|
on r.event_type_key = x.event_type_key
|
||||||
|
full outer join imported_by_type i
|
||||||
|
on i.event_type_key = coalesce(x.event_type_key, r.event_type_key)
|
||||||
|
order by event_type_key;
|
||||||
|
|
@ -0,0 +1,317 @@
|
||||||
|
create extension if not exists pgcrypto;
|
||||||
|
create extension if not exists postgis;
|
||||||
|
|
||||||
|
create schema if not exists eventhub;
|
||||||
|
|
||||||
|
create table if not exists eventhub.event_source (
|
||||||
|
id integer generated always as identity primary key,
|
||||||
|
tenant_key text not null,
|
||||||
|
provider_key text not null,
|
||||||
|
source_kind text not null,
|
||||||
|
source_key text not null,
|
||||||
|
source_instance_key text not null default 'default',
|
||||||
|
tenant_provider_setting_key text,
|
||||||
|
external_fleet_key text,
|
||||||
|
created_at timestamptz not null default now(),
|
||||||
|
constraint ux_event_source unique (tenant_key, provider_key, source_kind, source_key, source_instance_key)
|
||||||
|
);
|
||||||
|
|
||||||
|
create table if not exists eventhub.import_run (
|
||||||
|
id uuid primary key,
|
||||||
|
tenant_key text not null,
|
||||||
|
event_source_id integer not null references eventhub.event_source(id),
|
||||||
|
mode text not null,
|
||||||
|
status text not null,
|
||||||
|
refresh_master_data_first boolean not null default true,
|
||||||
|
source_group_type text,
|
||||||
|
source_group_entity_id text,
|
||||||
|
source_group_code text,
|
||||||
|
source_group_name text,
|
||||||
|
import_scope_type text not null,
|
||||||
|
root_source_org_entity_id text,
|
||||||
|
root_source_org_code text,
|
||||||
|
root_source_org_name text,
|
||||||
|
include_children boolean not null default false,
|
||||||
|
occurred_from timestamptz,
|
||||||
|
occurred_to timestamptz,
|
||||||
|
requested_event_families text[] not null default '{}',
|
||||||
|
acquisition_strategy text,
|
||||||
|
metadata jsonb not null default '{}'::jsonb,
|
||||||
|
planned_package_count integer not null default 0,
|
||||||
|
started_at timestamptz not null default now(),
|
||||||
|
finished_at timestamptz,
|
||||||
|
error_message text,
|
||||||
|
constraint chk_import_run_occ_time_order check (occurred_from is null or occurred_to is null or occurred_from < occurred_to)
|
||||||
|
);
|
||||||
|
|
||||||
|
create table if not exists eventhub.import_cursor (
|
||||||
|
id uuid primary key,
|
||||||
|
tenant_key text not null,
|
||||||
|
event_source_id integer not null references eventhub.event_source(id),
|
||||||
|
scope_hash text not null,
|
||||||
|
event_family text not null,
|
||||||
|
source_kind text not null,
|
||||||
|
cursor_type text not null,
|
||||||
|
last_source_package_imported_at timestamptz,
|
||||||
|
last_source_package_id text,
|
||||||
|
last_source_row_updated_at timestamptz,
|
||||||
|
last_occurred_to timestamptz,
|
||||||
|
updated_at timestamptz not null default now(),
|
||||||
|
constraint ux_import_cursor unique (tenant_key, event_source_id, scope_hash, event_family, source_kind, cursor_type)
|
||||||
|
);
|
||||||
|
|
||||||
|
create table if not exists eventhub.data_package (
|
||||||
|
id uuid primary key,
|
||||||
|
event_source_id integer not null references eventhub.event_source(id),
|
||||||
|
import_run_id uuid references eventhub.import_run(id),
|
||||||
|
tenant_key text not null,
|
||||||
|
package_key text not null,
|
||||||
|
package_type text not null,
|
||||||
|
status text not null,
|
||||||
|
source_group_type text,
|
||||||
|
source_group_entity_id text,
|
||||||
|
source_group_code text,
|
||||||
|
source_group_name text,
|
||||||
|
import_scope_type text,
|
||||||
|
root_source_org_entity_id text,
|
||||||
|
root_source_org_code text,
|
||||||
|
root_source_org_name text,
|
||||||
|
include_children boolean not null default false,
|
||||||
|
occurred_from timestamptz,
|
||||||
|
occurred_to timestamptz,
|
||||||
|
event_family text,
|
||||||
|
business_date date,
|
||||||
|
external_package_id text,
|
||||||
|
extraction_code text,
|
||||||
|
extraction_source_kind text,
|
||||||
|
entity_axis text,
|
||||||
|
batch_no integer,
|
||||||
|
chunk_from timestamptz,
|
||||||
|
chunk_to timestamptz,
|
||||||
|
source_package_kind text,
|
||||||
|
source_package_id text,
|
||||||
|
source_package_entity_id text,
|
||||||
|
source_package_period_from timestamptz,
|
||||||
|
source_package_period_to timestamptz,
|
||||||
|
source_package_imported_at timestamptz,
|
||||||
|
received_at timestamptz not null default now(),
|
||||||
|
completed_at timestamptz,
|
||||||
|
event_count integer not null default 0,
|
||||||
|
metadata jsonb not null default '{}'::jsonb,
|
||||||
|
error_message text,
|
||||||
|
constraint ux_data_package_package_key unique (tenant_key, event_source_id, package_key),
|
||||||
|
constraint chk_data_package_occ_time_order check (occurred_from is null or occurred_to is null or occurred_from < occurred_to),
|
||||||
|
constraint chk_data_package_chunk_time_order check (chunk_from is null or chunk_to is null or chunk_from < chunk_to)
|
||||||
|
);
|
||||||
|
|
||||||
|
create table if not exists eventhub.source_master_entity (
|
||||||
|
id uuid primary key,
|
||||||
|
tenant_key text not null,
|
||||||
|
event_source_id integer not null references eventhub.event_source(id),
|
||||||
|
entity_type text not null,
|
||||||
|
source_entity_id text not null,
|
||||||
|
source_external_key text,
|
||||||
|
display_name text,
|
||||||
|
active boolean,
|
||||||
|
valid_from timestamptz,
|
||||||
|
valid_to timestamptz,
|
||||||
|
source_updated_at timestamptz,
|
||||||
|
payload jsonb not null default '{}'::jsonb,
|
||||||
|
created_at timestamptz not null default now(),
|
||||||
|
updated_at timestamptz not null default now(),
|
||||||
|
constraint ux_source_master_entity unique (tenant_key, event_source_id, entity_type, source_entity_id),
|
||||||
|
constraint chk_source_master_entity_valid_time_order check (valid_from is null or valid_to is null or valid_from <= valid_to)
|
||||||
|
);
|
||||||
|
|
||||||
|
create table if not exists eventhub.source_master_relation (
|
||||||
|
id uuid primary key,
|
||||||
|
tenant_key text not null,
|
||||||
|
event_source_id integer not null references eventhub.event_source(id),
|
||||||
|
relation_key text not null,
|
||||||
|
relation_type text not null,
|
||||||
|
from_entity_type text not null,
|
||||||
|
from_source_entity_id text not null,
|
||||||
|
to_entity_type text not null,
|
||||||
|
to_source_entity_id text not null,
|
||||||
|
valid_from timestamptz,
|
||||||
|
valid_to timestamptz,
|
||||||
|
source_updated_at timestamptz,
|
||||||
|
payload jsonb not null default '{}'::jsonb,
|
||||||
|
created_at timestamptz not null default now(),
|
||||||
|
updated_at timestamptz not null default now(),
|
||||||
|
constraint ux_source_master_relation unique (tenant_key, event_source_id, relation_key),
|
||||||
|
constraint chk_source_master_relation_valid_time_order check (valid_from is null or valid_to is null or valid_from <= valid_to)
|
||||||
|
);
|
||||||
|
|
||||||
|
create table if not exists eventhub.vehicle (
|
||||||
|
id uuid primary key,
|
||||||
|
tenant_key text not null,
|
||||||
|
event_source_id integer not null references eventhub.event_source(id),
|
||||||
|
source_vehicle_entity_id text,
|
||||||
|
vin text,
|
||||||
|
created_at timestamptz not null default now(),
|
||||||
|
updated_at timestamptz not null default now()
|
||||||
|
);
|
||||||
|
|
||||||
|
create table if not exists eventhub.vehicle_registration (
|
||||||
|
id uuid primary key,
|
||||||
|
tenant_key text not null,
|
||||||
|
event_source_id integer not null references eventhub.event_source(id),
|
||||||
|
source_registration_entity_id text,
|
||||||
|
nation text not null,
|
||||||
|
registration_number text not null,
|
||||||
|
source_updated_at timestamptz,
|
||||||
|
payload jsonb not null default '{}'::jsonb,
|
||||||
|
created_at timestamptz not null default now(),
|
||||||
|
updated_at timestamptz not null default now()
|
||||||
|
);
|
||||||
|
|
||||||
|
create table if not exists eventhub.vehicle_registration_assignment (
|
||||||
|
id uuid primary key,
|
||||||
|
tenant_key text not null,
|
||||||
|
event_source_id integer not null references eventhub.event_source(id),
|
||||||
|
vehicle_registration_id uuid not null references eventhub.vehicle_registration(id) on delete cascade,
|
||||||
|
vehicle_id uuid not null references eventhub.vehicle(id) on delete cascade,
|
||||||
|
valid_from timestamptz,
|
||||||
|
valid_to timestamptz,
|
||||||
|
source_updated_at timestamptz,
|
||||||
|
payload jsonb not null default '{}'::jsonb,
|
||||||
|
created_at timestamptz not null default now(),
|
||||||
|
updated_at timestamptz not null default now(),
|
||||||
|
constraint chk_vehicle_registration_assignment_valid_time_order check (valid_from is null or valid_to is null or valid_from <= valid_to)
|
||||||
|
);
|
||||||
|
|
||||||
|
create table if not exists eventhub.event (
|
||||||
|
id uuid not null,
|
||||||
|
event_source_id integer not null references eventhub.event_source(id),
|
||||||
|
data_package_id uuid not null references eventhub.data_package(id),
|
||||||
|
external_source_event_id text not null,
|
||||||
|
driver_entity_id uuid references eventhub.source_master_entity(id),
|
||||||
|
vehicle_id uuid references eventhub.vehicle(id),
|
||||||
|
vehicle_registration_id uuid references eventhub.vehicle_registration(id),
|
||||||
|
source_package_entity_id uuid references eventhub.source_master_entity(id),
|
||||||
|
occurred_at timestamptz not null,
|
||||||
|
received_partner_at timestamptz,
|
||||||
|
received_hub_at timestamptz not null default now(),
|
||||||
|
event_domain text not null,
|
||||||
|
event_type text not null,
|
||||||
|
lifecycle text not null,
|
||||||
|
odometer_m bigint,
|
||||||
|
position geography(Point, 4326),
|
||||||
|
payload jsonb not null default '{}'::jsonb,
|
||||||
|
manual_entry boolean not null default false,
|
||||||
|
source_record_key_hash text not null,
|
||||||
|
event_signature_hash text,
|
||||||
|
created_at timestamptz not null default now(),
|
||||||
|
constraint pk_event primary key (occurred_at, id),
|
||||||
|
constraint chk_event_driver_or_vehicle_ref check (
|
||||||
|
driver_entity_id is not null
|
||||||
|
or vehicle_id is not null
|
||||||
|
or vehicle_registration_id is not null
|
||||||
|
)
|
||||||
|
);
|
||||||
|
|
||||||
|
create table if not exists eventhub.event_detail (
|
||||||
|
event_occurred_at timestamptz not null,
|
||||||
|
event_id uuid not null,
|
||||||
|
detail_type text not null,
|
||||||
|
attributes jsonb not null default '{}'::jsonb,
|
||||||
|
created_at timestamptz not null default now(),
|
||||||
|
constraint pk_event_detail primary key (event_occurred_at, event_id, detail_type),
|
||||||
|
constraint fk_event_detail_event foreign key (event_occurred_at, event_id)
|
||||||
|
references eventhub.event(occurred_at, id)
|
||||||
|
on delete cascade
|
||||||
|
);
|
||||||
|
|
||||||
|
create unique index if not exists ux_event_source_record
|
||||||
|
on eventhub.event(source_record_key_hash);
|
||||||
|
|
||||||
|
create index if not exists idx_event_signature
|
||||||
|
on eventhub.event(event_signature_hash)
|
||||||
|
where event_signature_hash is not null;
|
||||||
|
|
||||||
|
create index if not exists idx_event_source_time
|
||||||
|
on eventhub.event(event_source_id, occurred_at desc);
|
||||||
|
|
||||||
|
create index if not exists idx_event_package_time
|
||||||
|
on eventhub.event(data_package_id, occurred_at desc);
|
||||||
|
|
||||||
|
create index if not exists idx_event_domain_type_time
|
||||||
|
on eventhub.event(event_domain, event_type, occurred_at desc);
|
||||||
|
|
||||||
|
create index if not exists idx_event_driver_time
|
||||||
|
on eventhub.event(driver_entity_id, occurred_at desc)
|
||||||
|
where driver_entity_id is not null;
|
||||||
|
|
||||||
|
create index if not exists idx_event_vehicle_time
|
||||||
|
on eventhub.event(vehicle_id, occurred_at desc)
|
||||||
|
where vehicle_id is not null;
|
||||||
|
|
||||||
|
create index if not exists idx_event_vehicle_registration_time
|
||||||
|
on eventhub.event(vehicle_registration_id, occurred_at desc)
|
||||||
|
where vehicle_registration_id is not null;
|
||||||
|
|
||||||
|
create index if not exists idx_event_position_gist
|
||||||
|
on eventhub.event using gist(position)
|
||||||
|
where position is not null;
|
||||||
|
|
||||||
|
create index if not exists idx_event_payload_gin
|
||||||
|
on eventhub.event using gin(payload);
|
||||||
|
|
||||||
|
create index if not exists idx_event_detail_type
|
||||||
|
on eventhub.event_detail(detail_type);
|
||||||
|
|
||||||
|
create index if not exists idx_event_detail_attributes_gin
|
||||||
|
on eventhub.event_detail using gin(attributes);
|
||||||
|
|
||||||
|
create index if not exists idx_source_master_entity_type_key
|
||||||
|
on eventhub.source_master_entity(tenant_key, event_source_id, entity_type, source_external_key)
|
||||||
|
where source_external_key is not null;
|
||||||
|
|
||||||
|
create index if not exists idx_source_master_entity_payload_gin
|
||||||
|
on eventhub.source_master_entity using gin(payload);
|
||||||
|
|
||||||
|
create index if not exists idx_source_master_relation_from
|
||||||
|
on eventhub.source_master_relation(tenant_key, event_source_id, from_entity_type, from_source_entity_id, relation_type);
|
||||||
|
|
||||||
|
create index if not exists idx_source_master_relation_to
|
||||||
|
on eventhub.source_master_relation(tenant_key, event_source_id, to_entity_type, to_source_entity_id, relation_type);
|
||||||
|
|
||||||
|
create index if not exists idx_source_master_relation_payload_gin
|
||||||
|
on eventhub.source_master_relation using gin(payload);
|
||||||
|
|
||||||
|
create index if not exists idx_vehicle_lookup_ctx
|
||||||
|
on eventhub.vehicle(tenant_key, event_source_id, updated_at desc);
|
||||||
|
|
||||||
|
create index if not exists idx_vehicle_source_entity
|
||||||
|
on eventhub.vehicle(tenant_key, event_source_id, source_vehicle_entity_id)
|
||||||
|
where source_vehicle_entity_id is not null;
|
||||||
|
|
||||||
|
create index if not exists idx_vehicle_vin
|
||||||
|
on eventhub.vehicle(tenant_key, event_source_id, vin)
|
||||||
|
where vin is not null;
|
||||||
|
|
||||||
|
create index if not exists idx_vehicle_registration_source_entity
|
||||||
|
on eventhub.vehicle_registration(tenant_key, event_source_id, source_registration_entity_id)
|
||||||
|
where source_registration_entity_id is not null;
|
||||||
|
|
||||||
|
create index if not exists idx_vehicle_registration_plate
|
||||||
|
on eventhub.vehicle_registration(tenant_key, event_source_id, nation, registration_number);
|
||||||
|
|
||||||
|
create index if not exists idx_vehicle_registration_assignment_registration_time
|
||||||
|
on eventhub.vehicle_registration_assignment(vehicle_registration_id, valid_from desc, valid_to);
|
||||||
|
|
||||||
|
create index if not exists idx_vehicle_registration_assignment_vehicle_time
|
||||||
|
on eventhub.vehicle_registration_assignment(vehicle_id, valid_from desc, valid_to);
|
||||||
|
|
||||||
|
create index if not exists idx_data_package_source_time
|
||||||
|
on eventhub.data_package(tenant_key, event_source_id, received_at desc);
|
||||||
|
|
||||||
|
create index if not exists idx_data_package_scope
|
||||||
|
on eventhub.data_package(tenant_key, import_scope_type, root_source_org_entity_id, occurred_from, occurred_to);
|
||||||
|
|
||||||
|
create index if not exists idx_data_package_extraction
|
||||||
|
on eventhub.data_package(tenant_key, event_source_id, import_run_id, event_family, extraction_source_kind, extraction_code, batch_no);
|
||||||
|
|
||||||
|
create index if not exists idx_import_run_source_status
|
||||||
|
on eventhub.import_run(tenant_key, event_source_id, status, started_at desc);
|
||||||
|
|
@ -0,0 +1,459 @@
|
||||||
|
# Import Performance and Error Fixes
|
||||||
|
|
||||||
|
## Scope
|
||||||
|
|
||||||
|
This document summarizes the event-ingest, master-data, schema, locking, retry, and correctness fixes made in the current optimization round for the EventHub import pipeline.
|
||||||
|
|
||||||
|
It covers:
|
||||||
|
|
||||||
|
- event schema and migration fixes
|
||||||
|
- event import throughput fixes
|
||||||
|
- master-data import throughput fixes
|
||||||
|
- deadlock and transaction-visibility fixes
|
||||||
|
- connection-reset and retry handling
|
||||||
|
- async import cursor correctness fixes
|
||||||
|
- logging and operational visibility improvements
|
||||||
|
|
||||||
|
Main committed changes:
|
||||||
|
|
||||||
|
- `2e6e1aa` - `Optimize ingestion pipeline and reduce import contention`
|
||||||
|
- `bd3620b` - `Improve vehicle reference caching during ingest`
|
||||||
|
|
||||||
|
Additional related fixes are currently present in the workspace but may not yet be committed.
|
||||||
|
|
||||||
|
## 1. Schema and Migration Fixes
|
||||||
|
|
||||||
|
### 1.1 `event_detail` / hypertable ordering
|
||||||
|
|
||||||
|
Problem:
|
||||||
|
|
||||||
|
- executing `eventhub_schema_create.sql` on an empty database failed with `relation "eventhub.event_detail" does not exist`
|
||||||
|
|
||||||
|
Fix:
|
||||||
|
|
||||||
|
- create `eventhub.event_detail` before `create_hypertable(...)`
|
||||||
|
- add its foreign key after the hypertable conversion
|
||||||
|
|
||||||
|
Files:
|
||||||
|
|
||||||
|
- `src/main/resources/db/eventhub_schema_create.sql`
|
||||||
|
|
||||||
|
### 1.2 Explicit migrations for event hypertable and source-record support
|
||||||
|
|
||||||
|
Problem:
|
||||||
|
|
||||||
|
- the runtime schema evolution needed explicit migrations for hypertable conversion and `event_source_record`
|
||||||
|
|
||||||
|
Fix:
|
||||||
|
|
||||||
|
- add migration for `source_package_id` on `event`
|
||||||
|
- add migration for `event` hypertable conversion and FK recreation
|
||||||
|
- add migration to ensure `event_source_record` exists and is backfilled
|
||||||
|
|
||||||
|
Files:
|
||||||
|
|
||||||
|
- `src/main/resources/db/migration/V9__add_event_source_package_id.sql`
|
||||||
|
- `src/main/resources/db/migration/V10__make_event_hypertable.sql`
|
||||||
|
- `src/main/resources/db/migration/V11__ensure_event_source_record.sql`
|
||||||
|
|
||||||
|
## 2. Event Import Throughput Fixes
|
||||||
|
|
||||||
|
### 2.1 Replace per-event inserts with staged set-based writes
|
||||||
|
|
||||||
|
Problem:
|
||||||
|
|
||||||
|
- `EventRepository.batchInsert(...)` originally processed events one by one despite the batch API
|
||||||
|
- this caused one insert/query cycle per event and poor throughput
|
||||||
|
|
||||||
|
Fix:
|
||||||
|
|
||||||
|
- stage a whole ingest batch into `eventhub_event_import_stage`
|
||||||
|
- reserve `event_source_record` rows set-wise
|
||||||
|
- insert `eventhub.event` rows set-wise
|
||||||
|
- upsert `eventhub.event_detail` rows in batch
|
||||||
|
|
||||||
|
Files:
|
||||||
|
|
||||||
|
- `src/main/java/at/procon/eventhub/persistence/EventRepository.java`
|
||||||
|
|
||||||
|
### 2.2 Fix missing event rows when source records were reserved
|
||||||
|
|
||||||
|
Problem:
|
||||||
|
|
||||||
|
- after the set-based refactor, some runs created `event_source_record` rows without creating `event` rows
|
||||||
|
|
||||||
|
Cause:
|
||||||
|
|
||||||
|
- the insert statement reserved source records and then tried to re-read them through the base table in the same data-modifying CTE chain
|
||||||
|
|
||||||
|
Fix:
|
||||||
|
|
||||||
|
- use the `RETURNING` rows from the source-record reservation CTE directly
|
||||||
|
- also support already-existing source records that still miss the `event` row
|
||||||
|
|
||||||
|
Files:
|
||||||
|
|
||||||
|
- `src/main/java/at/procon/eventhub/persistence/EventRepository.java`
|
||||||
|
|
||||||
|
### 2.3 Stream extraction instead of materializing full result sets
|
||||||
|
|
||||||
|
Problem:
|
||||||
|
|
||||||
|
- extraction loaded full SQL chunks into memory before handing them to Camel
|
||||||
|
|
||||||
|
Fix:
|
||||||
|
|
||||||
|
- stream rows directly from JDBC to `direct:eventhub-normalized-input`
|
||||||
|
- keep only counters and watermark information
|
||||||
|
|
||||||
|
Files:
|
||||||
|
|
||||||
|
- `src/main/java/at/procon/eventhub/importing/extraction/AbstractJdbcExtractionBatchExecutor.java`
|
||||||
|
- `src/main/java/at/procon/eventhub/tachograph/service/JdbcTachographExtractionBatchExecutor.java`
|
||||||
|
|
||||||
|
### 2.4 Increase batch size and enable parallel queue draining
|
||||||
|
|
||||||
|
Problem:
|
||||||
|
|
||||||
|
- the async ingest route drained too slowly with `1000`-event batches and a single consumer
|
||||||
|
|
||||||
|
Fix:
|
||||||
|
|
||||||
|
- raise Camel completion size from `1000` to `5000`
|
||||||
|
- enable `4` concurrent SEDA consumers
|
||||||
|
|
||||||
|
Files:
|
||||||
|
|
||||||
|
- `src/main/java/at/procon/eventhub/config/EventHubProperties.java`
|
||||||
|
- `src/main/java/at/procon/eventhub/camel/EventHubCommonIngestionRoute.java`
|
||||||
|
- `src/main/resources/application.yml`
|
||||||
|
|
||||||
|
### 2.5 Give each Camel flush its own package key
|
||||||
|
|
||||||
|
Problem:
|
||||||
|
|
||||||
|
- multiple flushes of the same extraction package reused the same `data_package` identity
|
||||||
|
- logs were misleading and `event_count` on the package row was overwritten by later flushes
|
||||||
|
|
||||||
|
Fix:
|
||||||
|
|
||||||
|
- derive a unique `packageKey` per completed Camel batch using the aggregate package key plus the Camel exchange id
|
||||||
|
- preserve both the aggregate key and the child key in metadata
|
||||||
|
|
||||||
|
Files:
|
||||||
|
|
||||||
|
- `src/main/java/at/procon/eventhub/camel/EventHubBatchBuildProcessor.java`
|
||||||
|
|
||||||
|
### 2.6 Improve batch-local entity and vehicle caching
|
||||||
|
|
||||||
|
Problem:
|
||||||
|
|
||||||
|
- after the bulk insert refactor, the main remaining hot path was still reference resolution
|
||||||
|
|
||||||
|
Fixes:
|
||||||
|
|
||||||
|
- cache entity ids in the batch by `entityType + sourceEntityId`
|
||||||
|
- cache vehicle resolutions inside a batch
|
||||||
|
- later extend vehicle caching to be range-aware for registration-based assignment lookups:
|
||||||
|
- direct vehicle identifiers cache without time sensitivity
|
||||||
|
- registration-based resolutions cache over assignment validity intervals
|
||||||
|
|
||||||
|
Files:
|
||||||
|
|
||||||
|
- `src/main/java/at/procon/eventhub/persistence/EventRepository.java`
|
||||||
|
- `src/main/java/at/procon/eventhub/persistence/VehicleIdentityRepository.java`
|
||||||
|
|
||||||
|
## 3. Master-Data Import Throughput Fixes
|
||||||
|
|
||||||
|
### 3.1 Set-based master entity and relation upserts
|
||||||
|
|
||||||
|
Problem:
|
||||||
|
|
||||||
|
- source master data was previously written row by row
|
||||||
|
|
||||||
|
Fix:
|
||||||
|
|
||||||
|
- stage master entities and relations into temporary tables
|
||||||
|
- run set-based `insert ... select ... on conflict do update`
|
||||||
|
|
||||||
|
Files:
|
||||||
|
|
||||||
|
- `src/main/java/at/procon/eventhub/persistence/SourceMasterDataRepository.java`
|
||||||
|
|
||||||
|
### 3.2 Stream and chunk master-data refresh
|
||||||
|
|
||||||
|
Problem:
|
||||||
|
|
||||||
|
- the refresh path loaded large source master-data result sets into memory
|
||||||
|
|
||||||
|
Fix:
|
||||||
|
|
||||||
|
- stream source rows
|
||||||
|
- flush in chunks of `5000`
|
||||||
|
|
||||||
|
Files:
|
||||||
|
|
||||||
|
- `src/main/java/at/procon/eventhub/tachograph/service/TachographMasterDataRefreshService.java`
|
||||||
|
|
||||||
|
### 3.3 Bulk vehicle reconciliation from master data
|
||||||
|
|
||||||
|
Problem:
|
||||||
|
|
||||||
|
- reconciling vehicles and registrations from master data was done row by row
|
||||||
|
|
||||||
|
Fix:
|
||||||
|
|
||||||
|
- replace the loop with set-based SQL for vehicles, registrations, and projected assignments
|
||||||
|
|
||||||
|
Files:
|
||||||
|
|
||||||
|
- `src/main/java/at/procon/eventhub/persistence/VehicleIdentityRepository.java`
|
||||||
|
|
||||||
|
## 4. Deadlock and Contention Fixes
|
||||||
|
|
||||||
|
### 4.1 Remove unnecessary hot-row updates on vehicle and registration rows
|
||||||
|
|
||||||
|
Problem:
|
||||||
|
|
||||||
|
- event import updated `vehicle.updated_at` and `vehicle_registration.updated_at` even when no new information was being added
|
||||||
|
- this created deadlocks under parallel ingest
|
||||||
|
|
||||||
|
Fix:
|
||||||
|
|
||||||
|
- only update `vehicle` when missing `source_vehicle_entity_id` or `vin` can actually be filled
|
||||||
|
- only update `vehicle_registration` when missing source id, nation, or registration number can actually be filled
|
||||||
|
- stop using event import as a generic "touch row" path
|
||||||
|
|
||||||
|
Files:
|
||||||
|
|
||||||
|
- `src/main/java/at/procon/eventhub/persistence/VehicleIdentityRepository.java`
|
||||||
|
|
||||||
|
### 4.2 Make event-time source master entity resolution "find or create", not "update on conflict"
|
||||||
|
|
||||||
|
Problem:
|
||||||
|
|
||||||
|
- concurrent event batches could deadlock on `eventhub.source_master_entity` through `INSERT ... ON CONFLICT DO UPDATE`
|
||||||
|
|
||||||
|
Fix:
|
||||||
|
|
||||||
|
- first `SELECT id`
|
||||||
|
- if missing, `INSERT ... ON CONFLICT DO NOTHING RETURNING id`
|
||||||
|
- if another transaction won the race, select again
|
||||||
|
- do not update existing master entity rows during event ingest
|
||||||
|
|
||||||
|
Files:
|
||||||
|
|
||||||
|
- `src/main/java/at/procon/eventhub/persistence/SourceMasterDataRepository.java`
|
||||||
|
|
||||||
|
### 4.3 Fix race handling when `RETURNING` returns no row
|
||||||
|
|
||||||
|
Problem:
|
||||||
|
|
||||||
|
- if a concurrent transaction inserted the entity first, the resolver could still fail unexpectedly
|
||||||
|
|
||||||
|
Fix:
|
||||||
|
|
||||||
|
- allow the `RETURNING` path to yield `null`
|
||||||
|
- retry with a follow-up `SELECT`
|
||||||
|
|
||||||
|
Files:
|
||||||
|
|
||||||
|
- `src/main/java/at/procon/eventhub/persistence/SourceMasterDataRepository.java`
|
||||||
|
|
||||||
|
## 5. Transaction Visibility and Correctness Fixes
|
||||||
|
|
||||||
|
### 5.1 Remove outer transaction around full tachograph execution
|
||||||
|
|
||||||
|
Problem:
|
||||||
|
|
||||||
|
- master-data refresh logs showed completion, but master-data rows were not visible yet because the outer import method still held the transaction open
|
||||||
|
|
||||||
|
Fix:
|
||||||
|
|
||||||
|
- remove the outer transaction from `startAndExecuteImport(...)`
|
||||||
|
- keep chunk-level and package-level transactions independent
|
||||||
|
|
||||||
|
Files:
|
||||||
|
|
||||||
|
- `src/main/java/at/procon/eventhub/tachograph/service/TachographImportExecutionService.java`
|
||||||
|
|
||||||
|
### 5.2 Preserve the original ingest exception if package failure marking also fails
|
||||||
|
|
||||||
|
Problem:
|
||||||
|
|
||||||
|
- when ingest failed and `markFailed(...)` also failed because of a broken connection, the secondary bookkeeping error hid the real root cause
|
||||||
|
|
||||||
|
Fix:
|
||||||
|
|
||||||
|
- wrap `dataPackageRepository.markFailed(...)` in its own `try/catch`
|
||||||
|
- log the bookkeeping failure
|
||||||
|
- keep the original ingest exception and attach the bookkeeping failure as suppressed
|
||||||
|
|
||||||
|
Files:
|
||||||
|
|
||||||
|
- `src/main/java/at/procon/eventhub/service/EventHubIngestionService.java`
|
||||||
|
|
||||||
|
### 5.3 Do not advance import cursors before async ingest really finishes
|
||||||
|
|
||||||
|
Problem:
|
||||||
|
|
||||||
|
- extraction previously marked packages imported and advanced `import_cursor` before the async `CAMEL_BATCH` ingest was durably finished
|
||||||
|
- this could skip source data on the next run if async ingest later failed
|
||||||
|
|
||||||
|
Fix:
|
||||||
|
|
||||||
|
- add grouped child-batch status lookup on `data_package`
|
||||||
|
- make extraction package completion wait for all derived `CAMEL_BATCH` rows to reach terminal success
|
||||||
|
- fail the planned extraction package if child batches fail or time out
|
||||||
|
- only advance the cursor after the async ingest succeeded
|
||||||
|
- make "Completed import run" mean durable ingest completion instead of extraction completion
|
||||||
|
|
||||||
|
Files:
|
||||||
|
|
||||||
|
- `src/main/java/at/procon/eventhub/importing/AbstractImportExecutionService.java`
|
||||||
|
- `src/main/java/at/procon/eventhub/persistence/DataPackageRepository.java`
|
||||||
|
- `src/main/java/at/procon/eventhub/tachograph/service/TachographImportExecutionService.java`
|
||||||
|
|
||||||
|
## 6. Connection Reset and Retry Hardening
|
||||||
|
|
||||||
|
### 6.1 Retry transient DB failures in the Camel ingest route
|
||||||
|
|
||||||
|
Problem:
|
||||||
|
|
||||||
|
- long-running imports hit transient failures such as deadlocks and connection resets
|
||||||
|
|
||||||
|
Fix:
|
||||||
|
|
||||||
|
- add Camel redelivery with exponential backoff for:
|
||||||
|
- `CannotAcquireLockException`
|
||||||
|
- `PessimisticLockingFailureException`
|
||||||
|
- `DataAccessResourceFailureException`
|
||||||
|
- `TransientDataAccessException`
|
||||||
|
|
||||||
|
Files:
|
||||||
|
|
||||||
|
- `src/main/java/at/procon/eventhub/camel/EventHubCommonIngestionRoute.java`
|
||||||
|
|
||||||
|
### 6.2 Tune Hikari for shorter-lived and healthier pooled connections
|
||||||
|
|
||||||
|
Problem:
|
||||||
|
|
||||||
|
- `SQLSTATE 08006` / `Connection reset` events left broken pool entries behind
|
||||||
|
|
||||||
|
Fix:
|
||||||
|
|
||||||
|
- configure Hikari with explicit pool sizing and connection lifetime / keepalive settings:
|
||||||
|
- `maximum-pool-size: 16`
|
||||||
|
- `minimum-idle: 4`
|
||||||
|
- `connection-timeout: 30000`
|
||||||
|
- `validation-timeout: 5000`
|
||||||
|
- `idle-timeout: 300000`
|
||||||
|
- `keepalive-time: 120000`
|
||||||
|
- `max-lifetime: 540000`
|
||||||
|
|
||||||
|
Files:
|
||||||
|
|
||||||
|
- `src/main/resources/application.yml`
|
||||||
|
|
||||||
|
## 7. Observability Improvements
|
||||||
|
|
||||||
|
### 7.1 Master-data progress logging
|
||||||
|
|
||||||
|
Added logs for:
|
||||||
|
|
||||||
|
- refresh start
|
||||||
|
- per-section progress
|
||||||
|
- per-chunk counts
|
||||||
|
- `byType` breakdowns
|
||||||
|
- section completion
|
||||||
|
- reconciliation start and result
|
||||||
|
|
||||||
|
Files:
|
||||||
|
|
||||||
|
- `src/main/java/at/procon/eventhub/tachograph/service/TachographMasterDataRefreshService.java`
|
||||||
|
|
||||||
|
### 7.2 Event extraction progress logging
|
||||||
|
|
||||||
|
Added logs for:
|
||||||
|
|
||||||
|
- extraction start
|
||||||
|
- progress every `5000` mapped events
|
||||||
|
- final mapped totals with `byType`
|
||||||
|
|
||||||
|
Files:
|
||||||
|
|
||||||
|
- `src/main/java/at/procon/eventhub/importing/extraction/AbstractJdbcExtractionBatchExecutor.java`
|
||||||
|
|
||||||
|
### 7.3 Event ingest throughput logging
|
||||||
|
|
||||||
|
Added logs for:
|
||||||
|
|
||||||
|
- `receivedCount`
|
||||||
|
- `insertedCount`
|
||||||
|
- `elapsedMs`
|
||||||
|
- `receivedPerSecond`
|
||||||
|
- `byType`
|
||||||
|
|
||||||
|
Files:
|
||||||
|
|
||||||
|
- `src/main/java/at/procon/eventhub/service/EventHubIngestionService.java`
|
||||||
|
|
||||||
|
### 7.4 Async-ingest wait progress logging
|
||||||
|
|
||||||
|
Added logs for:
|
||||||
|
|
||||||
|
- number of expected child batches
|
||||||
|
- observed child batches
|
||||||
|
- successful / failed / importing child-batch counts while the import executor waits for durable completion
|
||||||
|
|
||||||
|
Files:
|
||||||
|
|
||||||
|
- `src/main/java/at/procon/eventhub/importing/AbstractImportExecutionService.java`
|
||||||
|
|
||||||
|
## 8. Operational Notes
|
||||||
|
|
||||||
|
### Throughput effect seen during the optimization round
|
||||||
|
|
||||||
|
Observed progression during the work:
|
||||||
|
|
||||||
|
- roughly `30` events/sec before the later cache and blocking fixes
|
||||||
|
- roughly `300` rows/sec after the main contention and stuck-session cleanup work
|
||||||
|
|
||||||
|
This is a major improvement, but large historical backfills are still expensive.
|
||||||
|
|
||||||
|
### What remains expensive
|
||||||
|
|
||||||
|
The main remaining bottleneck is still reference resolution in the ingest hot path, especially:
|
||||||
|
|
||||||
|
- driver entity resolution
|
||||||
|
- source-package entity resolution
|
||||||
|
- vehicle / registration lookup and creation
|
||||||
|
|
||||||
|
The next major optimization step would be set-based pre-resolution of references per ingest batch instead of resolving them one event at a time.
|
||||||
|
|
||||||
|
### Safe rerun behavior
|
||||||
|
|
||||||
|
- event ingest remains idempotent through `event_source_record.source_record_key_hash`
|
||||||
|
- already imported events should generally be kept
|
||||||
|
- when historical cursor corruption existed, repair should target `import_cursor`, not wholesale deletion of imported events
|
||||||
|
|
||||||
|
## 9. Main Files Touched
|
||||||
|
|
||||||
|
- `src/main/java/at/procon/eventhub/persistence/EventRepository.java`
|
||||||
|
- `src/main/java/at/procon/eventhub/persistence/VehicleIdentityRepository.java`
|
||||||
|
- `src/main/java/at/procon/eventhub/persistence/SourceMasterDataRepository.java`
|
||||||
|
- `src/main/java/at/procon/eventhub/persistence/DataPackageRepository.java`
|
||||||
|
- `src/main/java/at/procon/eventhub/service/EventHubIngestionService.java`
|
||||||
|
- `src/main/java/at/procon/eventhub/camel/EventHubCommonIngestionRoute.java`
|
||||||
|
- `src/main/java/at/procon/eventhub/camel/EventHubBatchBuildProcessor.java`
|
||||||
|
- `src/main/java/at/procon/eventhub/importing/extraction/AbstractJdbcExtractionBatchExecutor.java`
|
||||||
|
- `src/main/java/at/procon/eventhub/importing/AbstractImportExecutionService.java`
|
||||||
|
- `src/main/java/at/procon/eventhub/tachograph/service/JdbcTachographExtractionBatchExecutor.java`
|
||||||
|
- `src/main/java/at/procon/eventhub/tachograph/service/TachographMasterDataRefreshService.java`
|
||||||
|
- `src/main/java/at/procon/eventhub/tachograph/service/TachographImportExecutionService.java`
|
||||||
|
- `src/main/java/at/procon/eventhub/config/EventHubProperties.java`
|
||||||
|
- `src/main/resources/application.yml`
|
||||||
|
- `src/main/resources/db/eventhub_schema_create.sql`
|
||||||
|
- `src/main/resources/db/migration/V9__add_event_source_package_id.sql`
|
||||||
|
- `src/main/resources/db/migration/V10__make_event_hypertable.sql`
|
||||||
|
- `src/main/resources/db/migration/V11__ensure_event_source_record.sql`
|
||||||
|
|
@ -3,7 +3,7 @@
|
||||||
create extension if not exists timescaledb;
|
create extension if not exists timescaledb;
|
||||||
|
|
||||||
select create_hypertable(
|
select create_hypertable(
|
||||||
'eventhub.acquired_event',
|
'eventhub.event',
|
||||||
'occurred_at',
|
'occurred_at',
|
||||||
if_not_exists => true,
|
if_not_exists => true,
|
||||||
migrate_data => true
|
migrate_data => true
|
||||||
|
|
|
||||||
21
pom.xml
21
pom.xml
|
|
@ -97,6 +97,27 @@
|
||||||
|
|
||||||
<build>
|
<build>
|
||||||
<plugins>
|
<plugins>
|
||||||
|
<plugin>
|
||||||
|
<groupId>org.apache.maven.plugins</groupId>
|
||||||
|
<artifactId>maven-enforcer-plugin</artifactId>
|
||||||
|
<version>3.6.1</version>
|
||||||
|
<executions>
|
||||||
|
<execution>
|
||||||
|
<id>enforce-java</id>
|
||||||
|
<goals>
|
||||||
|
<goal>enforce</goal>
|
||||||
|
</goals>
|
||||||
|
<configuration>
|
||||||
|
<rules>
|
||||||
|
<requireJavaVersion>
|
||||||
|
<version>[21,)</version>
|
||||||
|
<message>This project requires Java 21+ to build and run.</message>
|
||||||
|
</requireJavaVersion>
|
||||||
|
</rules>
|
||||||
|
</configuration>
|
||||||
|
</execution>
|
||||||
|
</executions>
|
||||||
|
</plugin>
|
||||||
<plugin>
|
<plugin>
|
||||||
<groupId>org.springframework.boot</groupId>
|
<groupId>org.springframework.boot</groupId>
|
||||||
<artifactId>spring-boot-maven-plugin</artifactId>
|
<artifactId>spring-boot-maven-plugin</artifactId>
|
||||||
|
|
|
||||||
|
|
@ -8,6 +8,7 @@ import at.procon.eventhub.dto.ImportScopeDto;
|
||||||
import at.procon.eventhub.service.EventHubEventSorter;
|
import at.procon.eventhub.service.EventHubEventSorter;
|
||||||
import java.time.OffsetDateTime;
|
import java.time.OffsetDateTime;
|
||||||
import java.util.HashMap;
|
import java.util.HashMap;
|
||||||
|
import java.util.LinkedHashMap;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import java.util.Map;
|
import java.util.Map;
|
||||||
import org.apache.camel.Exchange;
|
import org.apache.camel.Exchange;
|
||||||
|
|
@ -29,7 +30,8 @@ public class EventHubBatchBuildProcessor implements Processor {
|
||||||
List<EventHubEventDto> events = exchange.getMessage().getBody(List.class);
|
List<EventHubEventDto> events = exchange.getMessage().getBody(List.class);
|
||||||
List<EventHubEventDto> sortedEvents = sorter.sort(events);
|
List<EventHubEventDto> sortedEvents = sorter.sort(events);
|
||||||
|
|
||||||
String packageKey = exchange.getMessage().getHeader(EventHubHeaders.PACKAGE_KEY, String.class);
|
String aggregatePackageKey = exchange.getMessage().getHeader(EventHubHeaders.PACKAGE_KEY, String.class);
|
||||||
|
String packageKey = aggregatePackageKey + ":CAMEL-" + exchange.getExchangeId();
|
||||||
EventHubPackageRequest packageInfo = exchange.getMessage().getHeader(EventHubHeaders.PACKAGE_INFO, EventHubPackageRequest.class);
|
EventHubPackageRequest packageInfo = exchange.getMessage().getHeader(EventHubHeaders.PACKAGE_INFO, EventHubPackageRequest.class);
|
||||||
if (packageInfo == null && !sortedEvents.isEmpty()) {
|
if (packageInfo == null && !sortedEvents.isEmpty()) {
|
||||||
packageInfo = sortedEvents.getFirst().packageInfo();
|
packageInfo = sortedEvents.getFirst().packageInfo();
|
||||||
|
|
@ -46,7 +48,10 @@ public class EventHubBatchBuildProcessor implements Processor {
|
||||||
Map<String, Object> metadata = new HashMap<>();
|
Map<String, Object> metadata = new HashMap<>();
|
||||||
metadata.put("camelRouteId", exchange.getFromRouteId());
|
metadata.put("camelRouteId", exchange.getFromRouteId());
|
||||||
metadata.put("packageKey", packageKey);
|
metadata.put("packageKey", packageKey);
|
||||||
|
metadata.put("aggregatePackageKey", aggregatePackageKey);
|
||||||
|
metadata.put("camelExchangeId", exchange.getExchangeId());
|
||||||
metadata.put("eventCount", sortedEvents.size());
|
metadata.put("eventCount", sortedEvents.size());
|
||||||
|
metadata.put("receivedEventTypeCounts", eventTypeCounts(sortedEvents));
|
||||||
if (packageInfo != null) {
|
if (packageInfo != null) {
|
||||||
metadata.put("tenantKey", packageInfo.tenantKey());
|
metadata.put("tenantKey", packageInfo.tenantKey());
|
||||||
metadata.put("eventSource", packageInfo.eventSource().stableKey());
|
metadata.put("eventSource", packageInfo.eventSource().stableKey());
|
||||||
|
|
@ -67,4 +72,14 @@ public class EventHubBatchBuildProcessor implements Processor {
|
||||||
metadata
|
metadata
|
||||||
));
|
));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private Map<String, Integer> eventTypeCounts(List<EventHubEventDto> events) {
|
||||||
|
Map<String, Integer> counts = new LinkedHashMap<>();
|
||||||
|
for (EventHubEventDto event : events) {
|
||||||
|
String domain = event.eventDomain() == null ? "UNKNOWN_DOMAIN" : event.eventDomain().name();
|
||||||
|
String type = event.eventType() == null ? "UNKNOWN_EVENT" : event.eventType().name();
|
||||||
|
counts.merge(domain + "/" + type, 1, Integer::sum);
|
||||||
|
}
|
||||||
|
return counts;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -2,12 +2,19 @@ package at.procon.eventhub.camel;
|
||||||
|
|
||||||
import at.procon.eventhub.config.EventHubProperties;
|
import at.procon.eventhub.config.EventHubProperties;
|
||||||
import at.procon.eventhub.service.EventHubIngestionService;
|
import at.procon.eventhub.service.EventHubIngestionService;
|
||||||
|
import org.springframework.dao.CannotAcquireLockException;
|
||||||
|
import org.springframework.dao.DataAccessResourceFailureException;
|
||||||
|
import org.springframework.dao.PessimisticLockingFailureException;
|
||||||
|
import org.springframework.dao.TransientDataAccessException;
|
||||||
import org.apache.camel.builder.RouteBuilder;
|
import org.apache.camel.builder.RouteBuilder;
|
||||||
import org.springframework.stereotype.Component;
|
import org.springframework.stereotype.Component;
|
||||||
|
import org.springframework.transaction.TransactionSystemException;
|
||||||
|
|
||||||
@Component
|
@Component
|
||||||
public class EventHubCommonIngestionRoute extends RouteBuilder {
|
public class EventHubCommonIngestionRoute extends RouteBuilder {
|
||||||
|
|
||||||
|
private static final String BATCH_INPUT_QUEUE = "seda:eventhub-batch-input";
|
||||||
|
|
||||||
private final EventHubProperties properties;
|
private final EventHubProperties properties;
|
||||||
private final EventHubEventValidationProcessor validationProcessor;
|
private final EventHubEventValidationProcessor validationProcessor;
|
||||||
private final EventHubPackageKeyProcessor packageKeyProcessor;
|
private final EventHubPackageKeyProcessor packageKeyProcessor;
|
||||||
|
|
@ -33,13 +40,28 @@ public class EventHubCommonIngestionRoute extends RouteBuilder {
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void configure() {
|
public void configure() {
|
||||||
|
String batchInputUri = batchInputUri();
|
||||||
|
|
||||||
|
onException(
|
||||||
|
CannotAcquireLockException.class,
|
||||||
|
PessimisticLockingFailureException.class,
|
||||||
|
DataAccessResourceFailureException.class,
|
||||||
|
TransientDataAccessException.class,
|
||||||
|
TransactionSystemException.class
|
||||||
|
)
|
||||||
|
.maximumRedeliveries(5)
|
||||||
|
.redeliveryDelay(2000)
|
||||||
|
.backOffMultiplier(2.0)
|
||||||
|
.useExponentialBackOff()
|
||||||
|
.retryAttemptedLogLevel(org.apache.camel.LoggingLevel.WARN);
|
||||||
|
|
||||||
from("direct:eventhub-normalized-input")
|
from("direct:eventhub-normalized-input")
|
||||||
.routeId("eventhub-normalized-input-route")
|
.routeId("eventhub-normalized-input-route")
|
||||||
.process(validationProcessor)
|
.process(validationProcessor)
|
||||||
.process(packageKeyProcessor)
|
.process(packageKeyProcessor)
|
||||||
.to("seda:eventhub-batch-input");
|
.to(batchInputUri);
|
||||||
|
|
||||||
from("seda:eventhub-batch-input")
|
from(batchInputUri)
|
||||||
.routeId("eventhub-batch-and-persist-route")
|
.routeId("eventhub-batch-and-persist-route")
|
||||||
.aggregate(header(EventHubHeaders.PACKAGE_KEY), aggregationStrategy)
|
.aggregate(header(EventHubHeaders.PACKAGE_KEY), aggregationStrategy)
|
||||||
.completionSize(properties.getBatch().getCompletionSize())
|
.completionSize(properties.getBatch().getCompletionSize())
|
||||||
|
|
@ -48,4 +70,13 @@ public class EventHubCommonIngestionRoute extends RouteBuilder {
|
||||||
.process(batchBuildProcessor)
|
.process(batchBuildProcessor)
|
||||||
.bean(ingestionService, "ingest");
|
.bean(ingestionService, "ingest");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private String batchInputUri() {
|
||||||
|
EventHubProperties.Batch batch = properties.getBatch();
|
||||||
|
return BATCH_INPUT_QUEUE
|
||||||
|
+ "?size=" + batch.getQueueSize()
|
||||||
|
+ "&concurrentConsumers=" + batch.getConcurrentConsumers()
|
||||||
|
+ "&blockWhenFull=" + batch.isBlockWhenFull()
|
||||||
|
+ "&offerTimeout=" + batch.getQueueOfferTimeout().toMillis();
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -32,11 +32,23 @@ public class EventHubProperties {
|
||||||
|
|
||||||
public static class Batch {
|
public static class Batch {
|
||||||
/** Number of events collected before a package is persisted. */
|
/** Number of events collected before a package is persisted. */
|
||||||
private int completionSize = 1000;
|
private int completionSize = 5000;
|
||||||
|
|
||||||
/** Maximum time to wait for more events belonging to the same package key. */
|
/** Maximum time to wait for more events belonging to the same package key. */
|
||||||
private Duration completionTimeout = Duration.ofSeconds(5);
|
private Duration completionTimeout = Duration.ofSeconds(5);
|
||||||
|
|
||||||
|
/** Maximum number of normalized events buffered before producers apply backpressure. */
|
||||||
|
private int queueSize = 10000;
|
||||||
|
|
||||||
|
/** Number of parallel consumers draining the normalized-event queue. */
|
||||||
|
private int concurrentConsumers = 4;
|
||||||
|
|
||||||
|
/** Whether producers should wait for queue capacity instead of failing immediately. */
|
||||||
|
private boolean blockWhenFull = true;
|
||||||
|
|
||||||
|
/** Maximum time a producer waits for queue capacity before failing the import. */
|
||||||
|
private Duration queueOfferTimeout = Duration.ofMinutes(5);
|
||||||
|
|
||||||
public int getCompletionSize() {
|
public int getCompletionSize() {
|
||||||
return completionSize;
|
return completionSize;
|
||||||
}
|
}
|
||||||
|
|
@ -52,6 +64,40 @@ public class EventHubProperties {
|
||||||
public void setCompletionTimeout(Duration completionTimeout) {
|
public void setCompletionTimeout(Duration completionTimeout) {
|
||||||
this.completionTimeout = completionTimeout;
|
this.completionTimeout = completionTimeout;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public int getQueueSize() {
|
||||||
|
return queueSize;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setQueueSize(int queueSize) {
|
||||||
|
this.queueSize = Math.max(1, queueSize);
|
||||||
|
}
|
||||||
|
|
||||||
|
public int getConcurrentConsumers() {
|
||||||
|
return concurrentConsumers;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setConcurrentConsumers(int concurrentConsumers) {
|
||||||
|
this.concurrentConsumers = Math.max(1, concurrentConsumers);
|
||||||
|
}
|
||||||
|
|
||||||
|
public boolean isBlockWhenFull() {
|
||||||
|
return blockWhenFull;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setBlockWhenFull(boolean blockWhenFull) {
|
||||||
|
this.blockWhenFull = blockWhenFull;
|
||||||
|
}
|
||||||
|
|
||||||
|
public Duration getQueueOfferTimeout() {
|
||||||
|
return queueOfferTimeout;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setQueueOfferTimeout(Duration queueOfferTimeout) {
|
||||||
|
if (queueOfferTimeout != null && !queueOfferTimeout.isNegative()) {
|
||||||
|
this.queueOfferTimeout = queueOfferTimeout;
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
public static class Tachograph {
|
public static class Tachograph {
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,17 @@
|
||||||
|
package at.procon.eventhub.config;
|
||||||
|
|
||||||
|
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||||
|
import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean;
|
||||||
|
import org.springframework.context.annotation.Bean;
|
||||||
|
import org.springframework.context.annotation.Configuration;
|
||||||
|
|
||||||
|
@Configuration
|
||||||
|
public class JacksonConfig {
|
||||||
|
|
||||||
|
@Bean
|
||||||
|
@ConditionalOnMissingBean(ObjectMapper.class)
|
||||||
|
public ObjectMapper objectMapper() {
|
||||||
|
return new ObjectMapper().findAndRegisterModules();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
@ -7,8 +7,7 @@ public enum EventDomain {
|
||||||
POSITION,
|
POSITION,
|
||||||
BORDER_CROSSING,
|
BORDER_CROSSING,
|
||||||
LOAD_UNLOAD,
|
LOAD_UNLOAD,
|
||||||
OUT_OF_SCOPE,
|
SPECIFIC_CONDITION,
|
||||||
FERRY_TRAIN,
|
|
||||||
SPEEDING,
|
SPEEDING,
|
||||||
PLACE,
|
PLACE,
|
||||||
VEHICLE_DATA,
|
VEHICLE_DATA,
|
||||||
|
|
|
||||||
|
|
@ -1,6 +1,8 @@
|
||||||
package at.procon.eventhub.dto;
|
package at.procon.eventhub.dto;
|
||||||
|
|
||||||
import java.time.OffsetDateTime;
|
import java.time.OffsetDateTime;
|
||||||
|
import java.util.Collections;
|
||||||
|
import java.util.LinkedHashMap;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import java.util.Map;
|
import java.util.Map;
|
||||||
|
|
||||||
|
|
@ -15,6 +17,6 @@ public record EventHubEventBatchDto(
|
||||||
) {
|
) {
|
||||||
public EventHubEventBatchDto {
|
public EventHubEventBatchDto {
|
||||||
events = events == null ? List.of() : List.copyOf(events);
|
events = events == null ? List.of() : List.copyOf(events);
|
||||||
metadata = metadata == null ? Map.of() : Map.copyOf(metadata);
|
metadata = metadata == null ? Map.of() : Collections.unmodifiableMap(new LinkedHashMap<>(metadata));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -20,8 +20,7 @@ public enum EventType {
|
||||||
OUT,
|
OUT,
|
||||||
FERRY_TRAIN,
|
FERRY_TRAIN,
|
||||||
SPEEDING,
|
SPEEDING,
|
||||||
START_PLACE,
|
WORKING_DAY_PLACE_RECORDED,
|
||||||
END_PLACE,
|
|
||||||
VEHICLE_DATA,
|
VEHICLE_DATA,
|
||||||
TELEMATICS_DATA,
|
TELEMATICS_DATA,
|
||||||
MANUAL_ENTRY,
|
MANUAL_ENTRY,
|
||||||
|
|
|
||||||
|
|
@ -3,36 +3,56 @@ package at.procon.eventhub.dto;
|
||||||
import jakarta.validation.Valid;
|
import jakarta.validation.Valid;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Source-side vehicle reference. VIN can be missing for driver-card-only data;
|
* Source-side vehicle reference. A physical tachograph vehicle is identified by
|
||||||
* VRN/registration is nation-scoped and can be resolved to VIN later.
|
* VehicleIdentification.ID/VIN. A registration/plate is identified separately by
|
||||||
|
* Vehicle.ID/VRN because plates can move between vehicles over time.
|
||||||
*
|
*
|
||||||
* Organisation assignment is intentionally not stored on the event. Vehicle ↔
|
* Organisation assignment is intentionally not stored on the event. Vehicle and
|
||||||
* organisation relation belongs to master data and can be resolved by
|
* registration relations belong to master data and can be resolved historically.
|
||||||
* sourceEntityId/VIN/VRN + occurredAt when needed.
|
|
||||||
*/
|
*/
|
||||||
public record VehicleRefDto(
|
public record VehicleRefDto(
|
||||||
String sourceEntityId,
|
String sourceVehicleEntityId,
|
||||||
String vin,
|
String vin,
|
||||||
@Valid VehicleRegistrationRefDto vehicleRegistration
|
String sourceRegistrationEntityId,
|
||||||
|
@Valid at.procon.eventhub.dto.VehicleRegistrationRefDto vehicleRegistration
|
||||||
) {
|
) {
|
||||||
|
public VehicleRefDto(
|
||||||
|
String sourceEntityId,
|
||||||
|
String vin,
|
||||||
|
at.procon.eventhub.dto.VehicleRegistrationRefDto vehicleRegistration
|
||||||
|
) {
|
||||||
|
this(sourceEntityId, vin, null, vehicleRegistration);
|
||||||
|
}
|
||||||
|
|
||||||
public VehicleRefDto {
|
public VehicleRefDto {
|
||||||
sourceEntityId = normalizeNullable(sourceEntityId);
|
sourceVehicleEntityId = normalizeNullable(sourceVehicleEntityId);
|
||||||
vin = normalizeVin(vin);
|
vin = normalizeVin(vin);
|
||||||
|
sourceRegistrationEntityId = normalizeNullable(sourceRegistrationEntityId);
|
||||||
}
|
}
|
||||||
|
|
||||||
public boolean hasAnyReference() {
|
public boolean hasAnyReference() {
|
||||||
return (sourceEntityId != null && !sourceEntityId.isBlank())
|
return (sourceVehicleEntityId != null && !sourceVehicleEntityId.isBlank())
|
||||||
|| (vin != null && !vin.isBlank())
|
|| (vin != null && !vin.isBlank())
|
||||||
|
|| (sourceRegistrationEntityId != null && !sourceRegistrationEntityId.isBlank())
|
||||||
|| (vehicleRegistration != null && vehicleRegistration.hasValue());
|
|| (vehicleRegistration != null && vehicleRegistration.hasValue());
|
||||||
}
|
}
|
||||||
|
|
||||||
public String stableKey() {
|
public String stableKey() {
|
||||||
String registrationKey = vehicleRegistration == null ? "" : vehicleRegistration.stableKey();
|
String registrationKey = vehicleRegistration == null ? "" : vehicleRegistration.stableKey();
|
||||||
return (sourceEntityId == null ? "" : sourceEntityId) + "|"
|
return (sourceVehicleEntityId == null ? "" : sourceVehicleEntityId) + "|"
|
||||||
+ (vin == null ? "" : vin) + "|"
|
+ (vin == null ? "" : vin) + "|"
|
||||||
|
+ (sourceRegistrationEntityId == null ? "" : sourceRegistrationEntityId) + "|"
|
||||||
+ registrationKey;
|
+ registrationKey;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Backward-compatible accessor for older code paths that treated vehicle and
|
||||||
|
* registration source IDs as one field.
|
||||||
|
*/
|
||||||
|
public String sourceEntityId() {
|
||||||
|
return sourceVehicleEntityId != null ? sourceVehicleEntityId : sourceRegistrationEntityId;
|
||||||
|
}
|
||||||
|
|
||||||
private static String normalizeNullable(String value) {
|
private static String normalizeNullable(String value) {
|
||||||
return value == null || value.isBlank() ? null : value.trim();
|
return value == null || value.isBlank() ? null : value.trim();
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -1,12 +1,16 @@
|
||||||
package at.procon.eventhub.importing;
|
package at.procon.eventhub.importing;
|
||||||
|
|
||||||
|
import at.procon.eventhub.config.EventHubProperties;
|
||||||
import at.procon.eventhub.dto.EventHubPackageRequest;
|
import at.procon.eventhub.dto.EventHubPackageRequest;
|
||||||
import at.procon.eventhub.dto.EventSourceDto;
|
import at.procon.eventhub.dto.EventSourceDto;
|
||||||
import at.procon.eventhub.dto.ImportRunStatus;
|
import at.procon.eventhub.dto.ImportRunStatus;
|
||||||
import at.procon.eventhub.importing.persistence.ImportCursorRepository;
|
import at.procon.eventhub.importing.persistence.ImportCursorRepository;
|
||||||
import at.procon.eventhub.importing.persistence.ImportRunRepository;
|
import at.procon.eventhub.importing.persistence.ImportRunRepository;
|
||||||
|
import at.procon.eventhub.persistence.DataPackageRepository.CamelBatchGroupStatus;
|
||||||
import at.procon.eventhub.persistence.DataPackageRepository;
|
import at.procon.eventhub.persistence.DataPackageRepository;
|
||||||
import at.procon.eventhub.persistence.EventSourceRepository;
|
import at.procon.eventhub.persistence.EventSourceRepository;
|
||||||
|
import java.time.Duration;
|
||||||
|
import java.time.Instant;
|
||||||
import java.util.ArrayList;
|
import java.util.ArrayList;
|
||||||
import java.util.LinkedHashMap;
|
import java.util.LinkedHashMap;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
|
|
@ -22,23 +26,32 @@ import org.slf4j.LoggerFactory;
|
||||||
*/
|
*/
|
||||||
public abstract class AbstractImportExecutionService<R extends ImportRunRequest, B extends ExtractionBatchResult> {
|
public abstract class AbstractImportExecutionService<R extends ImportRunRequest, B extends ExtractionBatchResult> {
|
||||||
|
|
||||||
|
private static final Duration ASYNC_INGEST_AWAIT_TIMEOUT = Duration.ofHours(6);
|
||||||
|
private static final Duration ASYNC_INGEST_POLL_INTERVAL = Duration.ofSeconds(2);
|
||||||
|
private static final Duration ASYNC_INGEST_FAILURE_GRACE_PERIOD = Duration.ofSeconds(90);
|
||||||
|
private static final Duration ASYNC_INGEST_STALL_GRACE_PERIOD = Duration.ofSeconds(90);
|
||||||
|
private static final Duration ASYNC_INGEST_PROGRESS_LOG_INTERVAL = Duration.ofSeconds(30);
|
||||||
|
|
||||||
private final Logger log = LoggerFactory.getLogger(getClass());
|
private final Logger log = LoggerFactory.getLogger(getClass());
|
||||||
|
|
||||||
private final EventSourceRepository eventSourceRepository;
|
private final EventSourceRepository eventSourceRepository;
|
||||||
private final ImportRunRepository importRunRepository;
|
private final ImportRunRepository importRunRepository;
|
||||||
private final DataPackageRepository dataPackageRepository;
|
private final DataPackageRepository dataPackageRepository;
|
||||||
private final ImportCursorRepository importCursorRepository;
|
private final ImportCursorRepository importCursorRepository;
|
||||||
|
private final EventHubProperties eventHubProperties;
|
||||||
|
|
||||||
protected AbstractImportExecutionService(
|
protected AbstractImportExecutionService(
|
||||||
EventSourceRepository eventSourceRepository,
|
EventSourceRepository eventSourceRepository,
|
||||||
ImportRunRepository importRunRepository,
|
ImportRunRepository importRunRepository,
|
||||||
DataPackageRepository dataPackageRepository,
|
DataPackageRepository dataPackageRepository,
|
||||||
ImportCursorRepository importCursorRepository
|
ImportCursorRepository importCursorRepository,
|
||||||
|
EventHubProperties eventHubProperties
|
||||||
) {
|
) {
|
||||||
this.eventSourceRepository = eventSourceRepository;
|
this.eventSourceRepository = eventSourceRepository;
|
||||||
this.importRunRepository = importRunRepository;
|
this.importRunRepository = importRunRepository;
|
||||||
this.dataPackageRepository = dataPackageRepository;
|
this.dataPackageRepository = dataPackageRepository;
|
||||||
this.importCursorRepository = importCursorRepository;
|
this.importCursorRepository = importCursorRepository;
|
||||||
|
this.eventHubProperties = eventHubProperties;
|
||||||
}
|
}
|
||||||
|
|
||||||
protected ImportRunResultDto createImportRun(R request, boolean executeImmediately) {
|
protected ImportRunResultDto createImportRun(R request, boolean executeImmediately) {
|
||||||
|
|
@ -165,26 +178,38 @@ public abstract class AbstractImportExecutionService<R extends ImportRunRequest,
|
||||||
List<B> results = new ArrayList<>();
|
List<B> results = new ArrayList<>();
|
||||||
for (PlannedPackage plannedPackage : plannedPackages) {
|
for (PlannedPackage plannedPackage : plannedPackages) {
|
||||||
dataPackageRepository.markImporting(plannedPackage.packageId());
|
dataPackageRepository.markImporting(plannedPackage.packageId());
|
||||||
B result = executeBatch(
|
try {
|
||||||
importRunId,
|
B result = executeBatch(
|
||||||
plannedPackage.packageId(),
|
importRunId,
|
||||||
plannedPackage.eventSourceId(),
|
plannedPackage.packageId(),
|
||||||
request,
|
|
||||||
plannedPackage.planItem(),
|
|
||||||
plannedPackage.chunk()
|
|
||||||
);
|
|
||||||
results.add(result);
|
|
||||||
dataPackageRepository.markImported(plannedPackage.packageId(), result.eventsInserted());
|
|
||||||
if (result.executed()) {
|
|
||||||
importCursorRepository.advanceCursor(
|
|
||||||
request.tenantKey(),
|
|
||||||
plannedPackage.eventSourceId(),
|
plannedPackage.eventSourceId(),
|
||||||
request.importScope() == null ? "NO_SCOPE" : request.importScope().stableKey(),
|
request,
|
||||||
plannedPackage.planItem().eventFamily(),
|
plannedPackage.planItem(),
|
||||||
plannedPackage.planItem().sourceKind(),
|
plannedPackage.chunk()
|
||||||
request.acquisitionStrategy(),
|
|
||||||
result
|
|
||||||
);
|
);
|
||||||
|
dataPackageRepository.updateEventCount(plannedPackage.packageId(), result.eventsInserted());
|
||||||
|
if (!result.eventTypeCounts().isEmpty()) {
|
||||||
|
dataPackageRepository.mergeMetadata(plannedPackage.packageId(), Map.of(
|
||||||
|
"extractedEventTypeCounts", result.eventTypeCounts()
|
||||||
|
));
|
||||||
|
}
|
||||||
|
awaitAsyncIngestCompletion(importRunId, request, plannedPackage, result);
|
||||||
|
results.add(result);
|
||||||
|
dataPackageRepository.markImported(plannedPackage.packageId(), result.eventsInserted());
|
||||||
|
if (result.executed()) {
|
||||||
|
importCursorRepository.advanceCursor(
|
||||||
|
request.tenantKey(),
|
||||||
|
plannedPackage.eventSourceId(),
|
||||||
|
request.importScope() == null ? "NO_SCOPE" : request.importScope().stableKey(),
|
||||||
|
plannedPackage.planItem().eventFamily(),
|
||||||
|
plannedPackage.planItem().sourceKind(),
|
||||||
|
request.acquisitionStrategy(),
|
||||||
|
result
|
||||||
|
);
|
||||||
|
}
|
||||||
|
} catch (RuntimeException ex) {
|
||||||
|
dataPackageRepository.markFailed(plannedPackage.packageId(), ex.getMessage());
|
||||||
|
throw ex;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
importRunRepository.markCompleted(importRunId);
|
importRunRepository.markCompleted(importRunId);
|
||||||
|
|
@ -196,6 +221,107 @@ public abstract class AbstractImportExecutionService<R extends ImportRunRequest,
|
||||||
results.stream().filter(ExtractionBatchResult::executed).count());
|
results.stream().filter(ExtractionBatchResult::executed).count());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private void awaitAsyncIngestCompletion(UUID importRunId, R request, PlannedPackage plannedPackage, B result) {
|
||||||
|
int expectedCamelBatches = expectedCamelBatchCount(result.eventsInserted());
|
||||||
|
if (expectedCamelBatches <= 0) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
EventHubPackageRequest packageInfo = extractionAggregatePackageInfo(
|
||||||
|
importRunId,
|
||||||
|
request,
|
||||||
|
eventSourceForItem(request.eventSource(), plannedPackage.planItem()),
|
||||||
|
plannedPackage.planItem(),
|
||||||
|
plannedPackage.chunk()
|
||||||
|
);
|
||||||
|
String aggregatePackageKey = aggregatePackageKey(packageInfo);
|
||||||
|
Instant deadline = Instant.now().plus(ASYNC_INGEST_AWAIT_TIMEOUT);
|
||||||
|
Instant nextProgressLogAt = Instant.now().plus(ASYNC_INGEST_PROGRESS_LOG_INTERVAL);
|
||||||
|
Instant failedStateObservedAt = null;
|
||||||
|
Instant lastStateChangeAt = Instant.now();
|
||||||
|
CamelBatchGroupStatus previousState = null;
|
||||||
|
|
||||||
|
while (Instant.now().isBefore(deadline)) {
|
||||||
|
CamelBatchGroupStatus state = dataPackageRepository.findCamelBatchGroupStatus(
|
||||||
|
plannedPackage.eventSourceId(),
|
||||||
|
request.tenantKey(),
|
||||||
|
aggregatePackageKey
|
||||||
|
);
|
||||||
|
if (state.totalCount() >= expectedCamelBatches && state.successCount() >= expectedCamelBatches) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
boolean stateChanged = previousState == null || !previousState.equals(state);
|
||||||
|
if (stateChanged) {
|
||||||
|
lastStateChangeAt = Instant.now();
|
||||||
|
}
|
||||||
|
if (state.failedCount() > 0 && stateChanged) {
|
||||||
|
failedStateObservedAt = Instant.now();
|
||||||
|
} else if (state.failedCount() == 0) {
|
||||||
|
failedStateObservedAt = null;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (state.failedCount() > 0
|
||||||
|
&& failedStateObservedAt != null
|
||||||
|
&& Instant.now().isAfter(failedStateObservedAt.plus(ASYNC_INGEST_FAILURE_GRACE_PERIOD))) {
|
||||||
|
throw new IllegalStateException(
|
||||||
|
"Async EventHub ingest failed for importRunId=" + importRunId
|
||||||
|
+ " aggregatePackageKey=" + aggregatePackageKey
|
||||||
|
+ " expectedCamelBatches=" + expectedCamelBatches
|
||||||
|
+ " observedCamelBatches=" + state.totalCount()
|
||||||
|
+ " failedCamelBatches=" + state.failedCount()
|
||||||
|
+ " failedMessage=" + state.failedMessage()
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (state.totalCount() < expectedCamelBatches
|
||||||
|
&& state.importingCount() == 0
|
||||||
|
&& state.failedCount() == 0
|
||||||
|
&& Instant.now().isAfter(lastStateChangeAt.plus(ASYNC_INGEST_STALL_GRACE_PERIOD))) {
|
||||||
|
throw new IllegalStateException(
|
||||||
|
"Async EventHub ingest stalled for importRunId=" + importRunId
|
||||||
|
+ " aggregatePackageKey=" + aggregatePackageKey
|
||||||
|
+ " expectedCamelBatches=" + expectedCamelBatches
|
||||||
|
+ " observedCamelBatches=" + state.totalCount()
|
||||||
|
+ " successfulCamelBatches=" + state.successCount()
|
||||||
|
+ " importingCamelBatches=" + state.importingCount()
|
||||||
|
+ " failedCamelBatches=" + state.failedCount()
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (Instant.now().isAfter(nextProgressLogAt)) {
|
||||||
|
log.info("Waiting for async EventHub ingest provider={} importRunId={} extractionPackageId={} aggregatePackageKey={} expectedCamelBatches={} observedCamelBatches={} successfulCamelBatches={} failedCamelBatches={} importingCamelBatches={}",
|
||||||
|
providerPackagePrefix(),
|
||||||
|
importRunId,
|
||||||
|
plannedPackage.packageId(),
|
||||||
|
aggregatePackageKey,
|
||||||
|
expectedCamelBatches,
|
||||||
|
state.totalCount(),
|
||||||
|
state.successCount(),
|
||||||
|
state.failedCount(),
|
||||||
|
state.importingCount());
|
||||||
|
nextProgressLogAt = Instant.now().plus(ASYNC_INGEST_PROGRESS_LOG_INTERVAL);
|
||||||
|
}
|
||||||
|
|
||||||
|
previousState = state;
|
||||||
|
sleepQuietly(ASYNC_INGEST_POLL_INTERVAL);
|
||||||
|
}
|
||||||
|
|
||||||
|
CamelBatchGroupStatus finalState = dataPackageRepository.findCamelBatchGroupStatus(
|
||||||
|
plannedPackage.eventSourceId(),
|
||||||
|
request.tenantKey(),
|
||||||
|
aggregatePackageKey
|
||||||
|
);
|
||||||
|
throw new IllegalStateException(
|
||||||
|
"Timed out waiting for async EventHub ingest for importRunId=" + importRunId
|
||||||
|
+ " aggregatePackageKey=" + aggregatePackageKey
|
||||||
|
+ " expectedCamelBatches=" + expectedCamelBatches
|
||||||
|
+ " observedCamelBatches=" + finalState.totalCount()
|
||||||
|
+ " successfulCamelBatches=" + finalState.successCount()
|
||||||
|
+ " failedCamelBatches=" + finalState.failedCount()
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
private EventHubPackageRequest packageRequestFor(
|
private EventHubPackageRequest packageRequestFor(
|
||||||
R request,
|
R request,
|
||||||
EventSourceDto itemEventSource,
|
EventSourceDto itemEventSource,
|
||||||
|
|
@ -206,13 +332,72 @@ public abstract class AbstractImportExecutionService<R extends ImportRunRequest,
|
||||||
request.tenantKey(),
|
request.tenantKey(),
|
||||||
itemEventSource,
|
itemEventSource,
|
||||||
request.sourceGroup(),
|
request.sourceGroup(),
|
||||||
request.importScope(),
|
chunkScope(request.importScope(), chunk),
|
||||||
item.eventFamily().name(),
|
item.eventFamily().name(),
|
||||||
null,
|
chunk.occurredFrom() == null ? null : chunk.occurredFrom().toLocalDate(),
|
||||||
externalPackageId(request, item, chunk)
|
externalPackageId(request, item, chunk)
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private EventHubPackageRequest extractionAggregatePackageInfo(
|
||||||
|
UUID importRunId,
|
||||||
|
R request,
|
||||||
|
EventSourceDto itemEventSource,
|
||||||
|
ImportPlanItemDto item,
|
||||||
|
ImportTimeChunkDto chunk
|
||||||
|
) {
|
||||||
|
return new EventHubPackageRequest(
|
||||||
|
request.tenantKey(),
|
||||||
|
itemEventSource,
|
||||||
|
request.sourceGroup(),
|
||||||
|
chunkScope(request.importScope(), chunk),
|
||||||
|
item.eventFamily().name(),
|
||||||
|
chunk.occurredFrom() == null ? null : chunk.occurredFrom().toLocalDate(),
|
||||||
|
providerPackagePrefix() + ":" + item.sourceKind() + ":" + item.extractionCode()
|
||||||
|
+ ":RUN-" + importRunId + ":CHUNK-" + chunk.sequence()
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
private int expectedCamelBatchCount(int extractedEventCount) {
|
||||||
|
if (extractedEventCount <= 0) {
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
int completionSize = Math.max(1, eventHubProperties.getBatch().getCompletionSize());
|
||||||
|
return ((extractedEventCount - 1) / completionSize) + 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
private String aggregatePackageKey(EventHubPackageRequest packageInfo) {
|
||||||
|
return packageInfo.tenantKey()
|
||||||
|
+ ":" + packageInfo.eventSource().stableKey()
|
||||||
|
+ ":" + (packageInfo.sourceGroup() == null ? "NO_GROUP" : packageInfo.sourceGroup().stableKey())
|
||||||
|
+ ":" + (packageInfo.importScope() == null ? "NO_SCOPE" : packageInfo.importScope().stableKey())
|
||||||
|
+ ":" + packageInfo.eventFamily()
|
||||||
|
+ ":" + (packageInfo.businessDate() == null ? "NO_DATE" : packageInfo.businessDate())
|
||||||
|
+ ":" + packageInfo.externalPackageId();
|
||||||
|
}
|
||||||
|
|
||||||
|
private at.procon.eventhub.dto.ImportScopeDto chunkScope(at.procon.eventhub.dto.ImportScopeDto scope, ImportTimeChunkDto chunk) {
|
||||||
|
if (scope == null) {
|
||||||
|
return at.procon.eventhub.dto.ImportScopeDto.tenantAll(chunk.occurredFrom(), chunk.occurredTo());
|
||||||
|
}
|
||||||
|
return new at.procon.eventhub.dto.ImportScopeDto(
|
||||||
|
scope.type(),
|
||||||
|
scope.rootSourceOrganisation(),
|
||||||
|
scope.includeChildren(),
|
||||||
|
chunk.occurredFrom(),
|
||||||
|
chunk.occurredTo()
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
private void sleepQuietly(Duration duration) {
|
||||||
|
try {
|
||||||
|
Thread.sleep(duration.toMillis());
|
||||||
|
} catch (InterruptedException ex) {
|
||||||
|
Thread.currentThread().interrupt();
|
||||||
|
throw new IllegalStateException("Interrupted while waiting for async EventHub ingest", ex);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
private record PlannedPackage(UUID packageId, int eventSourceId, ImportPlanItemDto planItem, ImportTimeChunkDto chunk) {
|
private record PlannedPackage(UUID packageId, int eventSourceId, ImportPlanItemDto planItem, ImportTimeChunkDto chunk) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -1,6 +1,7 @@
|
||||||
package at.procon.eventhub.importing;
|
package at.procon.eventhub.importing;
|
||||||
|
|
||||||
import java.time.OffsetDateTime;
|
import java.time.OffsetDateTime;
|
||||||
|
import java.util.Map;
|
||||||
|
|
||||||
public interface ExtractionBatchResult {
|
public interface ExtractionBatchResult {
|
||||||
|
|
||||||
|
|
@ -15,4 +16,8 @@ public interface ExtractionBatchResult {
|
||||||
OffsetDateTime lastSourceRowUpdatedAt();
|
OffsetDateTime lastSourceRowUpdatedAt();
|
||||||
|
|
||||||
OffsetDateTime lastOccurredTo();
|
OffsetDateTime lastOccurredTo();
|
||||||
|
|
||||||
|
default Map<String, Integer> eventTypeCounts() {
|
||||||
|
return Map.of();
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -14,11 +14,13 @@ import java.io.IOException;
|
||||||
import java.nio.charset.StandardCharsets;
|
import java.nio.charset.StandardCharsets;
|
||||||
import java.time.OffsetDateTime;
|
import java.time.OffsetDateTime;
|
||||||
import java.util.HashMap;
|
import java.util.HashMap;
|
||||||
import java.util.List;
|
import java.util.LinkedHashMap;
|
||||||
import java.util.Map;
|
import java.util.Map;
|
||||||
import java.util.Optional;
|
import java.util.Optional;
|
||||||
import java.util.UUID;
|
import java.util.UUID;
|
||||||
import org.apache.camel.ProducerTemplate;
|
import org.apache.camel.ProducerTemplate;
|
||||||
|
import org.slf4j.Logger;
|
||||||
|
import org.slf4j.LoggerFactory;
|
||||||
import org.springframework.core.io.Resource;
|
import org.springframework.core.io.Resource;
|
||||||
import org.springframework.core.io.ResourceLoader;
|
import org.springframework.core.io.ResourceLoader;
|
||||||
import org.springframework.jdbc.core.namedparam.NamedParameterJdbcTemplate;
|
import org.springframework.jdbc.core.namedparam.NamedParameterJdbcTemplate;
|
||||||
|
|
@ -27,6 +29,9 @@ import org.springframework.util.StreamUtils;
|
||||||
public abstract class AbstractJdbcExtractionBatchExecutor<R extends ImportRunRequest, B extends ExtractionBatchResult>
|
public abstract class AbstractJdbcExtractionBatchExecutor<R extends ImportRunRequest, B extends ExtractionBatchResult>
|
||||||
implements ExtractionBatchExecutor<R, B> {
|
implements ExtractionBatchExecutor<R, B> {
|
||||||
|
|
||||||
|
private static final Logger log = LoggerFactory.getLogger(AbstractJdbcExtractionBatchExecutor.class);
|
||||||
|
private static final int EVENT_EXTRACTION_PROGRESS_LOG_INTERVAL = 5000;
|
||||||
|
|
||||||
private final NamedParameterJdbcTemplate jdbcTemplate;
|
private final NamedParameterJdbcTemplate jdbcTemplate;
|
||||||
private final ProducerTemplate producerTemplate;
|
private final ProducerTemplate producerTemplate;
|
||||||
private final ResourceLoader resourceLoader;
|
private final ResourceLoader resourceLoader;
|
||||||
|
|
@ -81,10 +86,21 @@ public abstract class AbstractJdbcExtractionBatchExecutor<R extends ImportRunReq
|
||||||
|
|
||||||
Map<String, Object> params = parameters(request, chunkScope, cursor);
|
Map<String, Object> params = parameters(request, chunkScope, cursor);
|
||||||
String sql = loadSql(definition.sqlResource());
|
String sql = loadSql(definition.sqlResource());
|
||||||
List<EventHubEventDto> events = jdbcTemplate.query(sql, params, (rs, rowNum) -> definition.rowMapper().map(rs, rowNum, context));
|
ExtractedEventStats stats = new ExtractedEventStats();
|
||||||
events.forEach(event -> producerTemplate.sendBody(normalizedInputUri(), event));
|
log.info("Reading EventHub events provider={} tenant={} importRunId={} packageId={} extractionCode={} sourceKind={} chunk={} occurredFrom={} occurredTo={}",
|
||||||
|
providerPackagePrefix(), request.tenantKey(), importRunId, packageId, planItem.extractionCode(), planItem.sourceKind(),
|
||||||
|
chunk.sequence(), chunk.occurredFrom(), chunk.occurredTo());
|
||||||
|
jdbcTemplate.query(sql, params, rs -> {
|
||||||
|
EventHubEventDto event = definition.rowMapper().map(rs, stats.eventsMapped(), context);
|
||||||
|
producerTemplate.sendBody(normalizedInputUri(), event);
|
||||||
|
stats.accept(event);
|
||||||
|
if (stats.eventsMapped() % EVENT_EXTRACTION_PROGRESS_LOG_INTERVAL == 0) {
|
||||||
|
logEventExtractionProgress(request, importRunId, packageId, planItem, chunk, stats);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
logEventExtractionFinished(request, importRunId, packageId, planItem, chunk, stats);
|
||||||
|
|
||||||
return resultFor(packageId, planItem, chunk, cursor, events);
|
return resultFor(packageId, planItem, chunk, cursor, stats);
|
||||||
}
|
}
|
||||||
|
|
||||||
protected abstract Optional<ExtractionDefinition<R>> findDefinition(String code);
|
protected abstract Optional<ExtractionDefinition<R>> findDefinition(String code);
|
||||||
|
|
@ -96,7 +112,7 @@ public abstract class AbstractJdbcExtractionBatchExecutor<R extends ImportRunReq
|
||||||
ImportPlanItemDto planItem,
|
ImportPlanItemDto planItem,
|
||||||
ImportTimeChunkDto chunk,
|
ImportTimeChunkDto chunk,
|
||||||
ImportCursorStateDto cursor,
|
ImportCursorStateDto cursor,
|
||||||
List<EventHubEventDto> events
|
ExtractedEventStats stats
|
||||||
);
|
);
|
||||||
|
|
||||||
protected String providerPackagePrefix() {
|
protected String providerPackagePrefix() {
|
||||||
|
|
@ -107,38 +123,64 @@ public abstract class AbstractJdbcExtractionBatchExecutor<R extends ImportRunReq
|
||||||
return "direct:eventhub-normalized-input";
|
return "direct:eventhub-normalized-input";
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private void logEventExtractionProgress(
|
||||||
|
R request,
|
||||||
|
UUID importRunId,
|
||||||
|
UUID packageId,
|
||||||
|
ImportPlanItemDto planItem,
|
||||||
|
ImportTimeChunkDto chunk,
|
||||||
|
ExtractedEventStats stats
|
||||||
|
) {
|
||||||
|
log.info("EventHub event extraction progress provider={} tenant={} importRunId={} packageId={} extractionCode={} sourceKind={} chunk={} mapped={} byType={}",
|
||||||
|
providerPackagePrefix(), request.tenantKey(), importRunId, packageId, planItem.extractionCode(), planItem.sourceKind(),
|
||||||
|
chunk.sequence(), stats.eventsMapped(), stats.eventTypeCounts());
|
||||||
|
}
|
||||||
|
|
||||||
|
private void logEventExtractionFinished(
|
||||||
|
R request,
|
||||||
|
UUID importRunId,
|
||||||
|
UUID packageId,
|
||||||
|
ImportPlanItemDto planItem,
|
||||||
|
ImportTimeChunkDto chunk,
|
||||||
|
ExtractedEventStats stats
|
||||||
|
) {
|
||||||
|
log.info("Finished EventHub event extraction provider={} tenant={} importRunId={} packageId={} extractionCode={} sourceKind={} chunk={} mapped={} byType={}",
|
||||||
|
providerPackagePrefix(), request.tenantKey(), importRunId, packageId, planItem.extractionCode(), planItem.sourceKind(),
|
||||||
|
chunk.sequence(), stats.eventsMapped(), stats.eventTypeCounts());
|
||||||
|
}
|
||||||
|
|
||||||
protected Map<String, Object> parameters(R request, ImportScopeDto scope, ImportCursorStateDto cursor) {
|
protected Map<String, Object> parameters(R request, ImportScopeDto scope, ImportCursorStateDto cursor) {
|
||||||
Map<String, Object> params = new HashMap<>();
|
Map<String, Object> params = new HashMap<>();
|
||||||
|
String organisationId = scope == null || scope.rootSourceOrganisation() == null
|
||||||
|
? null
|
||||||
|
: scope.rootSourceOrganisation().sourceEntityId();
|
||||||
params.put("tenantKey", request.tenantKey());
|
params.put("tenantKey", request.tenantKey());
|
||||||
params.put("occurredFrom", scope == null ? null : scope.occurredFrom());
|
params.put("occurredFrom", scope == null ? null : scope.occurredFrom());
|
||||||
params.put("occurredTo", scope == null ? null : scope.occurredTo());
|
params.put("occurredTo", scope == null ? null : scope.occurredTo());
|
||||||
params.put("rootOrganisationId", scope == null || scope.rootSourceOrganisation() == null ? null : scope.rootSourceOrganisation().sourceEntityId());
|
params.put("organisationId", organisationId);
|
||||||
params.put("includeChildren", scope != null && scope.includeChildren());
|
params.put("includeChildren", scope != null && scope.includeChildren());
|
||||||
params.put("lastSourcePackageImportedAt", cursor == null ? null : cursor.lastSourcePackageImportedAt());
|
params.put("lastSourcePackageImportedAt", cursor == null ? null : cursor.lastSourcePackageImportedAt());
|
||||||
params.put("lastSourcePackageId", cursor == null ? null : cursor.lastSourcePackageId());
|
params.put("lastSourcePackageId", cursor == null ? null : cursor.lastSourcePackageId());
|
||||||
|
params.put("lastSourcePackageIdNumeric", parseLong(cursor == null ? null : cursor.lastSourcePackageId()));
|
||||||
params.put("lastSourceRowUpdatedAt", cursor == null ? null : cursor.lastSourceRowUpdatedAt());
|
params.put("lastSourceRowUpdatedAt", cursor == null ? null : cursor.lastSourceRowUpdatedAt());
|
||||||
params.put("lastOccurredTo", cursor == null ? null : cursor.lastOccurredTo());
|
params.put("lastOccurredTo", cursor == null ? null : cursor.lastOccurredTo());
|
||||||
return params;
|
return params;
|
||||||
}
|
}
|
||||||
|
|
||||||
protected OffsetDateTime lastSourcePackageImportedAt(List<EventHubEventDto> events, ImportCursorStateDto cursor) {
|
protected OffsetDateTime lastSourcePackageImportedAt(ExtractedEventStats stats, ImportCursorStateDto cursor) {
|
||||||
return events.stream()
|
return stats.lastSourcePackageImportedAt() == null
|
||||||
.map(event -> event.sourcePackageRef() == null ? null : event.sourcePackageRef().importedIntoSourceAt())
|
? cursor == null ? null : cursor.lastSourcePackageImportedAt()
|
||||||
.filter(value -> value != null)
|
: stats.lastSourcePackageImportedAt();
|
||||||
.max(OffsetDateTime::compareTo)
|
|
||||||
.orElse(cursor == null ? null : cursor.lastSourcePackageImportedAt());
|
|
||||||
}
|
}
|
||||||
|
|
||||||
protected String lastSourcePackageId(List<EventHubEventDto> events, ImportCursorStateDto cursor) {
|
protected String lastSourcePackageId(ExtractedEventStats stats, ImportCursorStateDto cursor) {
|
||||||
return events.stream()
|
if (stats.lastSourcePackageIdByImportedAt() != null) {
|
||||||
.filter(event -> event.sourcePackageRef() != null && event.sourcePackageRef().importedIntoSourceAt() != null)
|
return stats.lastSourcePackageIdByImportedAt();
|
||||||
.max((left, right) -> left.sourcePackageRef().importedIntoSourceAt().compareTo(right.sourcePackageRef().importedIntoSourceAt()))
|
}
|
||||||
.map(event -> event.sourcePackageRef().sourcePackageId())
|
if (stats.maxSourcePackageId() != null) {
|
||||||
.orElseGet(() -> events.stream()
|
return stats.maxSourcePackageId();
|
||||||
.map(event -> event.sourcePackageRef() == null ? null : event.sourcePackageRef().sourcePackageId())
|
}
|
||||||
.filter(value -> value != null && !value.isBlank())
|
return cursor == null ? null : cursor.lastSourcePackageId();
|
||||||
.max(this::compareSourcePackageId)
|
|
||||||
.orElse(cursor == null ? null : cursor.lastSourcePackageId()));
|
|
||||||
}
|
}
|
||||||
|
|
||||||
private EventHubPackageRequest packageInfo(
|
private EventHubPackageRequest packageInfo(
|
||||||
|
|
@ -200,4 +242,70 @@ public abstract class AbstractJdbcExtractionBatchExecutor<R extends ImportRunReq
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private Long parseLong(String value) {
|
||||||
|
if (value == null || value.isBlank()) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
return Long.parseLong(value.trim());
|
||||||
|
} catch (NumberFormatException ignored) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
protected final class ExtractedEventStats {
|
||||||
|
private int eventsMapped;
|
||||||
|
private OffsetDateTime lastSourcePackageImportedAt;
|
||||||
|
private String lastSourcePackageIdByImportedAt;
|
||||||
|
private String maxSourcePackageId;
|
||||||
|
private final Map<String, Integer> eventTypeCounts = new LinkedHashMap<>();
|
||||||
|
|
||||||
|
public int eventsMapped() {
|
||||||
|
return eventsMapped;
|
||||||
|
}
|
||||||
|
|
||||||
|
public OffsetDateTime lastSourcePackageImportedAt() {
|
||||||
|
return lastSourcePackageImportedAt;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String lastSourcePackageIdByImportedAt() {
|
||||||
|
return lastSourcePackageIdByImportedAt;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String maxSourcePackageId() {
|
||||||
|
return maxSourcePackageId;
|
||||||
|
}
|
||||||
|
|
||||||
|
public Map<String, Integer> eventTypeCounts() {
|
||||||
|
return eventTypeCounts;
|
||||||
|
}
|
||||||
|
|
||||||
|
private void accept(EventHubEventDto event) {
|
||||||
|
eventsMapped++;
|
||||||
|
eventTypeCounts.merge(eventTypeKey(event), 1, Integer::sum);
|
||||||
|
if (event.sourcePackageRef() == null) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
OffsetDateTime importedAt = event.sourcePackageRef().importedIntoSourceAt();
|
||||||
|
String sourcePackageId = event.sourcePackageRef().sourcePackageId();
|
||||||
|
if (importedAt != null
|
||||||
|
&& (lastSourcePackageImportedAt == null || importedAt.compareTo(lastSourcePackageImportedAt) > 0)) {
|
||||||
|
lastSourcePackageImportedAt = importedAt;
|
||||||
|
lastSourcePackageIdByImportedAt = sourcePackageId;
|
||||||
|
}
|
||||||
|
if (sourcePackageId != null
|
||||||
|
&& !sourcePackageId.isBlank()
|
||||||
|
&& (maxSourcePackageId == null || compareSourcePackageId(sourcePackageId, maxSourcePackageId) > 0)) {
|
||||||
|
maxSourcePackageId = sourcePackageId;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private String eventTypeKey(EventHubEventDto event) {
|
||||||
|
String domain = event.eventDomain() == null ? "UNKNOWN_DOMAIN" : event.eventDomain().name();
|
||||||
|
String type = event.eventType() == null ? "UNKNOWN_EVENT" : event.eventType().name();
|
||||||
|
return domain + "/" + type;
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -1,10 +1,12 @@
|
||||||
package at.procon.eventhub.importing.extraction;
|
package at.procon.eventhub.importing.extraction;
|
||||||
|
|
||||||
import at.procon.eventhub.importing.ImportRunRequest;
|
import at.procon.eventhub.importing.ImportRunRequest;
|
||||||
|
import java.util.Comparator;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import java.util.Locale;
|
import java.util.Locale;
|
||||||
import java.util.Map;
|
import java.util.Map;
|
||||||
import java.util.Optional;
|
import java.util.Optional;
|
||||||
|
import java.util.Set;
|
||||||
import java.util.function.Function;
|
import java.util.function.Function;
|
||||||
import java.util.stream.Collectors;
|
import java.util.stream.Collectors;
|
||||||
|
|
||||||
|
|
@ -21,6 +23,16 @@ public class ExtractionDefinitionRegistry<R extends ImportRunRequest> {
|
||||||
return Optional.ofNullable(definitionsByCode.get(normalize(code)));
|
return Optional.ofNullable(definitionsByCode.get(normalize(code)));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public Set<String> supportedCodes() {
|
||||||
|
return definitionsByCode.keySet();
|
||||||
|
}
|
||||||
|
|
||||||
|
public List<ExtractionDefinition<R>> definitions() {
|
||||||
|
return definitionsByCode.values().stream()
|
||||||
|
.sorted(Comparator.comparing(ExtractionDefinition::code))
|
||||||
|
.toList();
|
||||||
|
}
|
||||||
|
|
||||||
private String normalize(String value) {
|
private String normalize(String value) {
|
||||||
return value == null ? "" : value.trim().toUpperCase(Locale.ROOT);
|
return value == null ? "" : value.trim().toUpperCase(Locale.ROOT);
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,10 @@
|
||||||
|
package at.procon.eventhub.importing.masterdata;
|
||||||
|
|
||||||
|
public record MasterDataRefreshResult(
|
||||||
|
int entitiesUpserted,
|
||||||
|
int relationsUpserted
|
||||||
|
) {
|
||||||
|
public static MasterDataRefreshResult empty() {
|
||||||
|
return new MasterDataRefreshResult(0, 0);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,17 @@
|
||||||
|
package at.procon.eventhub.importing.masterdata;
|
||||||
|
|
||||||
|
import java.time.OffsetDateTime;
|
||||||
|
import java.util.Map;
|
||||||
|
|
||||||
|
public record SourceMasterEntityUpsert(
|
||||||
|
String entityType,
|
||||||
|
String sourceEntityId,
|
||||||
|
String sourceExternalKey,
|
||||||
|
String displayName,
|
||||||
|
Boolean active,
|
||||||
|
OffsetDateTime validFrom,
|
||||||
|
OffsetDateTime validTo,
|
||||||
|
OffsetDateTime sourceUpdatedAt,
|
||||||
|
Map<String, Object> payload
|
||||||
|
) {
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,17 @@
|
||||||
|
package at.procon.eventhub.importing.masterdata;
|
||||||
|
|
||||||
|
import java.time.OffsetDateTime;
|
||||||
|
import java.util.Map;
|
||||||
|
|
||||||
|
public record SourceMasterRelationUpsert(
|
||||||
|
String relationType,
|
||||||
|
String fromEntityType,
|
||||||
|
String fromSourceEntityId,
|
||||||
|
String toEntityType,
|
||||||
|
String toSourceEntityId,
|
||||||
|
OffsetDateTime validFrom,
|
||||||
|
OffsetDateTime validTo,
|
||||||
|
OffsetDateTime sourceUpdatedAt,
|
||||||
|
Map<String, Object> payload
|
||||||
|
) {
|
||||||
|
}
|
||||||
|
|
@ -7,12 +7,21 @@ import at.procon.eventhub.dto.ImportScopeDto;
|
||||||
import at.procon.eventhub.dto.SourceGroupRefDto;
|
import at.procon.eventhub.dto.SourceGroupRefDto;
|
||||||
import at.procon.eventhub.dto.SourcePackageRefDto;
|
import at.procon.eventhub.dto.SourcePackageRefDto;
|
||||||
import com.fasterxml.jackson.core.JsonProcessingException;
|
import com.fasterxml.jackson.core.JsonProcessingException;
|
||||||
|
import com.fasterxml.jackson.databind.JsonNode;
|
||||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||||
|
import java.lang.reflect.Array;
|
||||||
import java.time.OffsetDateTime;
|
import java.time.OffsetDateTime;
|
||||||
|
import java.time.temporal.TemporalAccessor;
|
||||||
|
import java.util.ArrayList;
|
||||||
|
import java.util.Date;
|
||||||
|
import java.util.LinkedHashMap;
|
||||||
|
import java.util.List;
|
||||||
import java.util.Map;
|
import java.util.Map;
|
||||||
import java.util.UUID;
|
import java.util.UUID;
|
||||||
import org.springframework.jdbc.core.JdbcTemplate;
|
import org.springframework.jdbc.core.JdbcTemplate;
|
||||||
import org.springframework.stereotype.Repository;
|
import org.springframework.stereotype.Repository;
|
||||||
|
import org.springframework.transaction.annotation.Propagation;
|
||||||
|
import org.springframework.transaction.annotation.Transactional;
|
||||||
|
|
||||||
@Repository
|
@Repository
|
||||||
public class DataPackageRepository {
|
public class DataPackageRepository {
|
||||||
|
|
@ -25,6 +34,7 @@ public class DataPackageRepository {
|
||||||
this.objectMapper = objectMapper;
|
this.objectMapper = objectMapper;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Transactional(propagation = Propagation.REQUIRES_NEW)
|
||||||
public UUID createPackage(
|
public UUID createPackage(
|
||||||
int eventSourceId,
|
int eventSourceId,
|
||||||
String packageKey,
|
String packageKey,
|
||||||
|
|
@ -34,24 +44,78 @@ public class DataPackageRepository {
|
||||||
OffsetDateTime occurredTo,
|
OffsetDateTime occurredTo,
|
||||||
Map<String, Object> metadata
|
Map<String, Object> metadata
|
||||||
) {
|
) {
|
||||||
return insertPackage(
|
UUID id = UUID.randomUUID();
|
||||||
eventSourceId,
|
SourceGroupRefDto sourceGroup = packageInfo == null ? null : packageInfo.sourceGroup();
|
||||||
null,
|
ImportScopeDto importScope = packageInfo == null ? null : packageInfo.importScope();
|
||||||
packageKey,
|
SourceGroupRefDto rootOrg = importScope == null ? null : importScope.rootSourceOrganisation();
|
||||||
packageInfo,
|
|
||||||
packageType,
|
return jdbcTemplate.query(
|
||||||
DataPackageStatus.IMPORTING,
|
con -> {
|
||||||
occurredFrom,
|
var ps = con.prepareStatement("""
|
||||||
occurredTo,
|
insert into eventhub.data_package(
|
||||||
null,
|
id, event_source_id, import_run_id, tenant_key, package_key, package_type, status,
|
||||||
null,
|
source_group_type, source_group_entity_id, source_group_code, source_group_name,
|
||||||
null,
|
import_scope_type, root_source_org_entity_id, root_source_org_code, root_source_org_name,
|
||||||
null,
|
include_children, occurred_from, occurred_to,
|
||||||
null,
|
event_family, business_date, external_package_id,
|
||||||
null,
|
extraction_code, extraction_source_kind, entity_axis, batch_no, chunk_from, chunk_to,
|
||||||
null,
|
source_package_kind, source_package_id, source_package_entity_id,
|
||||||
null,
|
source_package_period_from, source_package_period_to, source_package_imported_at,
|
||||||
metadata
|
received_at, event_count, metadata
|
||||||
|
) values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, now(), ?, ?::jsonb)
|
||||||
|
on conflict (tenant_key, event_source_id, package_key) do update
|
||||||
|
set status = excluded.status,
|
||||||
|
occurred_from = excluded.occurred_from,
|
||||||
|
occurred_to = excluded.occurred_to,
|
||||||
|
event_count = excluded.event_count,
|
||||||
|
metadata = excluded.metadata,
|
||||||
|
error_message = null,
|
||||||
|
completed_at = null
|
||||||
|
returning id
|
||||||
|
""");
|
||||||
|
ps.setObject(1, id);
|
||||||
|
ps.setInt(2, eventSourceId);
|
||||||
|
ps.setObject(3, null);
|
||||||
|
ps.setString(4, packageInfo == null ? "default" : packageInfo.tenantKey());
|
||||||
|
ps.setString(5, packageKey);
|
||||||
|
ps.setString(6, packageType.name());
|
||||||
|
ps.setString(7, DataPackageStatus.IMPORTING.name());
|
||||||
|
ps.setString(8, sourceGroup == null || sourceGroup.type() == null ? null : sourceGroup.type().name());
|
||||||
|
ps.setString(9, sourceGroup == null ? null : sourceGroup.sourceEntityId());
|
||||||
|
ps.setString(10, sourceGroup == null ? null : sourceGroup.code());
|
||||||
|
ps.setString(11, sourceGroup == null ? null : sourceGroup.name());
|
||||||
|
ps.setString(12, importScope == null || importScope.type() == null ? null : importScope.type().name());
|
||||||
|
ps.setString(13, rootOrg == null ? null : rootOrg.sourceEntityId());
|
||||||
|
ps.setString(14, rootOrg == null ? null : rootOrg.code());
|
||||||
|
ps.setString(15, rootOrg == null ? null : rootOrg.name());
|
||||||
|
ps.setBoolean(16, importScope != null && importScope.includeChildren());
|
||||||
|
ps.setObject(17, occurredFrom);
|
||||||
|
ps.setObject(18, occurredTo);
|
||||||
|
ps.setString(19, packageInfo == null ? null : packageInfo.eventFamily());
|
||||||
|
ps.setObject(20, packageInfo == null ? null : packageInfo.businessDate());
|
||||||
|
ps.setString(21, packageInfo == null ? packageKey : packageInfo.externalPackageId());
|
||||||
|
ps.setString(22, null);
|
||||||
|
ps.setString(23, null);
|
||||||
|
ps.setString(24, null);
|
||||||
|
ps.setObject(25, null);
|
||||||
|
ps.setObject(26, null);
|
||||||
|
ps.setObject(27, null);
|
||||||
|
ps.setString(28, null);
|
||||||
|
ps.setString(29, null);
|
||||||
|
ps.setString(30, null);
|
||||||
|
ps.setObject(31, null);
|
||||||
|
ps.setObject(32, null);
|
||||||
|
ps.setObject(33, null);
|
||||||
|
ps.setInt(34, 0);
|
||||||
|
ps.setString(35, toJson(metadata));
|
||||||
|
return ps;
|
||||||
|
},
|
||||||
|
rs -> {
|
||||||
|
if (!rs.next()) {
|
||||||
|
throw new IllegalStateException("Could not create or resolve data package " + packageKey);
|
||||||
|
}
|
||||||
|
return (UUID) rs.getObject(1);
|
||||||
|
}
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -189,6 +253,31 @@ public class DataPackageRepository {
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public void updateEventCount(UUID packageId, int eventCount) {
|
||||||
|
jdbcTemplate.update(
|
||||||
|
"""
|
||||||
|
update eventhub.data_package
|
||||||
|
set event_count = ?
|
||||||
|
where id = ?
|
||||||
|
""",
|
||||||
|
eventCount,
|
||||||
|
packageId
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
public void mergeMetadata(UUID packageId, Map<String, Object> metadata) {
|
||||||
|
jdbcTemplate.update(
|
||||||
|
"""
|
||||||
|
update eventhub.data_package
|
||||||
|
set metadata = coalesce(metadata, '{}'::jsonb) || ?::jsonb
|
||||||
|
where id = ?
|
||||||
|
""",
|
||||||
|
toJson(metadata),
|
||||||
|
packageId
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Transactional(propagation = Propagation.REQUIRES_NEW)
|
||||||
public void markImported(UUID packageId, int insertedCount) {
|
public void markImported(UUID packageId, int insertedCount) {
|
||||||
jdbcTemplate.update(
|
jdbcTemplate.update(
|
||||||
"""
|
"""
|
||||||
|
|
@ -202,6 +291,7 @@ public class DataPackageRepository {
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Transactional(propagation = Propagation.REQUIRES_NEW)
|
||||||
public void markFailed(UUID packageId, String errorMessage) {
|
public void markFailed(UUID packageId, String errorMessage) {
|
||||||
jdbcTemplate.update(
|
jdbcTemplate.update(
|
||||||
"""
|
"""
|
||||||
|
|
@ -215,11 +305,120 @@ public class DataPackageRepository {
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public CamelBatchGroupStatus findCamelBatchGroupStatus(int eventSourceId, String tenantKey, String aggregatePackageKey) {
|
||||||
|
return jdbcTemplate.query(
|
||||||
|
"""
|
||||||
|
select count(*) as total_count,
|
||||||
|
count(*) filter (where status in (?, ?)) as success_count,
|
||||||
|
count(*) filter (where status = ?) as failed_count,
|
||||||
|
count(*) filter (where status = ?) as importing_count,
|
||||||
|
coalesce(max(error_message) filter (where status = ?), '') as failed_message
|
||||||
|
from eventhub.data_package
|
||||||
|
where event_source_id = ?
|
||||||
|
and tenant_key = ?
|
||||||
|
and package_type = ?
|
||||||
|
and package_key like ?
|
||||||
|
""",
|
||||||
|
rs -> {
|
||||||
|
if (!rs.next()) {
|
||||||
|
return CamelBatchGroupStatus.empty();
|
||||||
|
}
|
||||||
|
return new CamelBatchGroupStatus(
|
||||||
|
rs.getInt("total_count"),
|
||||||
|
rs.getInt("success_count"),
|
||||||
|
rs.getInt("failed_count"),
|
||||||
|
rs.getInt("importing_count"),
|
||||||
|
rs.getString("failed_message")
|
||||||
|
);
|
||||||
|
},
|
||||||
|
DataPackageStatus.IMPORTED.name(),
|
||||||
|
DataPackageStatus.EMPTY.name(),
|
||||||
|
DataPackageStatus.FAILED.name(),
|
||||||
|
DataPackageStatus.IMPORTING.name(),
|
||||||
|
DataPackageStatus.FAILED.name(),
|
||||||
|
eventSourceId,
|
||||||
|
tenantKey,
|
||||||
|
DataPackageType.CAMEL_BATCH.name(),
|
||||||
|
aggregatePackageKey + ":CAMEL-%"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
private String toJson(Map<String, Object> value) {
|
private String toJson(Map<String, Object> value) {
|
||||||
try {
|
try {
|
||||||
return objectMapper.writeValueAsString(value == null ? Map.of() : value);
|
return objectMapper.writeValueAsString(normalizeMetadataMap(value));
|
||||||
} catch (JsonProcessingException e) {
|
} catch (JsonProcessingException e) {
|
||||||
throw new IllegalArgumentException("Cannot serialize package metadata", e);
|
throw new IllegalArgumentException("Cannot serialize package metadata", e);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private Map<String, Object> normalizeMetadataMap(Map<String, Object> value) {
|
||||||
|
if (value == null || value.isEmpty()) {
|
||||||
|
return Map.of();
|
||||||
|
}
|
||||||
|
Map<String, Object> normalized = new LinkedHashMap<>();
|
||||||
|
for (Map.Entry<String, Object> entry : value.entrySet()) {
|
||||||
|
normalized.put(entry.getKey(), normalizeMetadataValue(entry.getValue()));
|
||||||
|
}
|
||||||
|
return normalized;
|
||||||
|
}
|
||||||
|
|
||||||
|
private Object normalizeMetadataValue(Object value) {
|
||||||
|
if (value == null
|
||||||
|
|| value instanceof String
|
||||||
|
|| value instanceof Number
|
||||||
|
|| value instanceof Boolean
|
||||||
|
|| value instanceof JsonNode) {
|
||||||
|
return value;
|
||||||
|
}
|
||||||
|
if (value instanceof CharSequence charSequence) {
|
||||||
|
return charSequence.toString();
|
||||||
|
}
|
||||||
|
if (value instanceof Enum<?> enumValue) {
|
||||||
|
return enumValue.name();
|
||||||
|
}
|
||||||
|
if (value instanceof UUID uuid) {
|
||||||
|
return uuid.toString();
|
||||||
|
}
|
||||||
|
if (value instanceof TemporalAccessor temporalValue) {
|
||||||
|
return temporalValue.toString();
|
||||||
|
}
|
||||||
|
if (value instanceof Date dateValue) {
|
||||||
|
return dateValue.toInstant().toString();
|
||||||
|
}
|
||||||
|
if (value instanceof Map<?, ?> mapValue) {
|
||||||
|
Map<String, Object> nested = new LinkedHashMap<>();
|
||||||
|
for (Map.Entry<?, ?> entry : mapValue.entrySet()) {
|
||||||
|
nested.put(String.valueOf(entry.getKey()), normalizeMetadataValue(entry.getValue()));
|
||||||
|
}
|
||||||
|
return nested;
|
||||||
|
}
|
||||||
|
if (value instanceof Iterable<?> iterable) {
|
||||||
|
List<Object> nested = new ArrayList<>();
|
||||||
|
for (Object item : iterable) {
|
||||||
|
nested.add(normalizeMetadataValue(item));
|
||||||
|
}
|
||||||
|
return nested;
|
||||||
|
}
|
||||||
|
if (value.getClass().isArray()) {
|
||||||
|
int length = Array.getLength(value);
|
||||||
|
List<Object> nested = new ArrayList<>(length);
|
||||||
|
for (int i = 0; i < length; i++) {
|
||||||
|
nested.add(normalizeMetadataValue(Array.get(value, i)));
|
||||||
|
}
|
||||||
|
return nested;
|
||||||
|
}
|
||||||
|
return value.toString();
|
||||||
|
}
|
||||||
|
|
||||||
|
public record CamelBatchGroupStatus(
|
||||||
|
int totalCount,
|
||||||
|
int successCount,
|
||||||
|
int failedCount,
|
||||||
|
int importingCount,
|
||||||
|
String failedMessage
|
||||||
|
) {
|
||||||
|
public static CamelBatchGroupStatus empty() {
|
||||||
|
return new CamelBatchGroupStatus(0, 0, 0, 0, "");
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -6,14 +6,20 @@ import at.procon.eventhub.dto.EventHubEventDto;
|
||||||
import at.procon.eventhub.dto.SourcePackageRefDto;
|
import at.procon.eventhub.dto.SourcePackageRefDto;
|
||||||
import at.procon.eventhub.dto.VehicleRefDto;
|
import at.procon.eventhub.dto.VehicleRefDto;
|
||||||
import at.procon.eventhub.dto.VehicleRegistrationRefDto;
|
import at.procon.eventhub.dto.VehicleRegistrationRefDto;
|
||||||
|
import at.procon.eventhub.persistence.VehicleIdentityRepository.ResolvedVehicleReference;
|
||||||
|
import at.procon.eventhub.persistence.VehicleIdentityRepository.ResolvedVehicleReferenceResolution;
|
||||||
import at.procon.eventhub.service.EventAcquisitionRecordKeyService;
|
import at.procon.eventhub.service.EventAcquisitionRecordKeyService;
|
||||||
import com.fasterxml.jackson.core.JsonProcessingException;
|
import com.fasterxml.jackson.core.JsonProcessingException;
|
||||||
import com.fasterxml.jackson.databind.JsonNode;
|
import com.fasterxml.jackson.databind.JsonNode;
|
||||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||||
import java.sql.PreparedStatement;
|
import java.sql.PreparedStatement;
|
||||||
import java.sql.Types;
|
import java.sql.SQLException;
|
||||||
import java.time.OffsetDateTime;
|
import java.time.OffsetDateTime;
|
||||||
|
import java.util.ArrayList;
|
||||||
|
import java.util.HashMap;
|
||||||
|
import java.util.LinkedHashMap;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
|
import java.util.Map;
|
||||||
import java.util.UUID;
|
import java.util.UUID;
|
||||||
import org.springframework.jdbc.core.BatchPreparedStatementSetter;
|
import org.springframework.jdbc.core.BatchPreparedStatementSetter;
|
||||||
import org.springframework.jdbc.core.JdbcTemplate;
|
import org.springframework.jdbc.core.JdbcTemplate;
|
||||||
|
|
@ -25,126 +31,563 @@ public class EventRepository {
|
||||||
private final JdbcTemplate jdbcTemplate;
|
private final JdbcTemplate jdbcTemplate;
|
||||||
private final ObjectMapper objectMapper;
|
private final ObjectMapper objectMapper;
|
||||||
private final EventAcquisitionRecordKeyService recordKeyService;
|
private final EventAcquisitionRecordKeyService recordKeyService;
|
||||||
|
private final SourceMasterDataRepository sourceMasterDataRepository;
|
||||||
|
private final VehicleIdentityRepository vehicleIdentityRepository;
|
||||||
|
|
||||||
public EventRepository(JdbcTemplate jdbcTemplate, ObjectMapper objectMapper, EventAcquisitionRecordKeyService recordKeyService) {
|
public EventRepository(
|
||||||
|
JdbcTemplate jdbcTemplate,
|
||||||
|
ObjectMapper objectMapper,
|
||||||
|
EventAcquisitionRecordKeyService recordKeyService,
|
||||||
|
SourceMasterDataRepository sourceMasterDataRepository,
|
||||||
|
VehicleIdentityRepository vehicleIdentityRepository
|
||||||
|
) {
|
||||||
this.jdbcTemplate = jdbcTemplate;
|
this.jdbcTemplate = jdbcTemplate;
|
||||||
this.objectMapper = objectMapper;
|
this.objectMapper = objectMapper;
|
||||||
this.recordKeyService = recordKeyService;
|
this.recordKeyService = recordKeyService;
|
||||||
|
this.sourceMasterDataRepository = sourceMasterDataRepository;
|
||||||
|
this.vehicleIdentityRepository = vehicleIdentityRepository;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Acquisition-stage persistence. This table stores source records as imported.
|
* Persists normalized events and resolves master-data references on the fly.
|
||||||
* It does not merge or deduplicate equivalent events from different sources;
|
* The source-record hash is unique and provides source-level import idempotency.
|
||||||
* later query/read models can combine sources when a preferred source has gaps.
|
|
||||||
* Organisation assignment is not stored per event; it belongs to master-data
|
|
||||||
* relations for driver/vehicle and can be resolved by occurredAt later.
|
|
||||||
*/
|
*/
|
||||||
public int batchInsert(UUID packageId, int eventSourceId, List<EventHubEventDto> events) {
|
public int batchInsert(UUID packageId, String tenantKey, int eventSourceId, List<EventHubEventDto> events) {
|
||||||
int[] counts = jdbcTemplate.batchUpdate(
|
Map<String, UUID> entityIdCache = new HashMap<>();
|
||||||
|
Map<String, List<VehicleRefCacheEntry>> vehicleRefCache = new HashMap<>();
|
||||||
|
List<ResolvedEventImportRow> rows = new ArrayList<>(events.size());
|
||||||
|
|
||||||
|
for (EventHubEventDto event : events) {
|
||||||
|
ResolvedEntityRefs refs = resolveEntityRefs(tenantKey, eventSourceId, event, entityIdCache, vehicleRefCache);
|
||||||
|
rows.add(resolveEventImportRow(packageId, eventSourceId, event, refs));
|
||||||
|
}
|
||||||
|
|
||||||
|
createEventImportStage();
|
||||||
|
stageEvents(rows);
|
||||||
|
int insertedCount = insertStagedEvents();
|
||||||
|
Map<String, InsertedEventRow> eventRowsBySourceRecord = resolveStagedEventRows();
|
||||||
|
batchUpsertEventDetails(rows, eventRowsBySourceRecord);
|
||||||
|
return insertedCount;
|
||||||
|
}
|
||||||
|
|
||||||
|
private ResolvedEventImportRow resolveEventImportRow(
|
||||||
|
UUID packageId,
|
||||||
|
int eventSourceId,
|
||||||
|
EventHubEventDto event,
|
||||||
|
ResolvedEntityRefs refs
|
||||||
|
) {
|
||||||
|
UUID requestedEventId = event.eventId() == null ? UUID.randomUUID() : event.eventId();
|
||||||
|
OffsetDateTime receivedHubAt = event.receivedHubAt() == null ? OffsetDateTime.now() : event.receivedHubAt();
|
||||||
|
String sourceRecordKeyHash = recordKeyService.buildSourceRecordKeyHash(event, eventSourceId);
|
||||||
|
Double longitude = event.position() == null ? null : event.position().longitude().doubleValue();
|
||||||
|
Double latitude = event.position() == null ? null : event.position().latitude().doubleValue();
|
||||||
|
String sourcePackageId = event.sourcePackageRef() == null ? null : event.sourcePackageRef().sourcePackageId();
|
||||||
|
|
||||||
|
return new ResolvedEventImportRow(
|
||||||
|
sourceRecordKeyHash,
|
||||||
|
requestedEventId,
|
||||||
|
packageId,
|
||||||
|
eventSourceId,
|
||||||
|
event.externalSourceEventId(),
|
||||||
|
refs.driverEntityId(),
|
||||||
|
refs.vehicleId(),
|
||||||
|
refs.vehicleRegistrationId(),
|
||||||
|
sourcePackageId,
|
||||||
|
refs.sourcePackageEntityId(),
|
||||||
|
event.occurredAt(),
|
||||||
|
event.receivedPartnerAt(),
|
||||||
|
receivedHubAt,
|
||||||
|
event.eventDomain().name(),
|
||||||
|
event.eventType().name(),
|
||||||
|
event.lifecycle().name(),
|
||||||
|
event.odometerM(),
|
||||||
|
longitude,
|
||||||
|
latitude,
|
||||||
|
toJson(event.payload()),
|
||||||
|
event.manualEntry(),
|
||||||
|
recordKeyService.buildEventSignatureHash(event),
|
||||||
|
event.eventDetails() == null ? null : event.eventDetails().type(),
|
||||||
|
event.eventDetails() == null ? null : toJson(event.eventDetails().attributes())
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
private void createEventImportStage() {
|
||||||
|
jdbcTemplate.execute(
|
||||||
"""
|
"""
|
||||||
insert into eventhub.acquired_event(
|
create temporary table if not exists eventhub_event_import_stage (
|
||||||
id, event_source_id, data_package_id,
|
row_no integer not null,
|
||||||
external_source_event_id,
|
source_record_key_hash text not null,
|
||||||
driver_source_entity_id, driver_card_nation, driver_card_number,
|
requested_event_id uuid not null,
|
||||||
vehicle_source_entity_id, vehicle_vin, vehicle_registration_nation, vehicle_registration_number,
|
data_package_id uuid not null,
|
||||||
source_package_kind, source_package_id, source_package_entity_id,
|
event_source_id integer not null,
|
||||||
source_package_period_from, source_package_period_to, source_package_imported_at,
|
external_source_event_id text not null,
|
||||||
occurred_at, received_partner_at, received_hub_at,
|
driver_entity_id uuid,
|
||||||
event_domain, event_type, lifecycle,
|
vehicle_id uuid,
|
||||||
odometer_m, latitude, longitude,
|
vehicle_registration_id uuid,
|
||||||
event_details, payload, manual_entry,
|
source_package_id text,
|
||||||
source_record_key_hash, event_signature_hash
|
source_package_entity_id uuid,
|
||||||
) values (
|
occurred_at timestamptz not null,
|
||||||
?, ?, ?,
|
received_partner_at timestamptz,
|
||||||
?,
|
received_hub_at timestamptz not null,
|
||||||
?, ?, ?,
|
event_domain text not null,
|
||||||
?, ?, ?, ?,
|
event_type text not null,
|
||||||
?, ?, ?,
|
lifecycle text not null,
|
||||||
?, ?, ?,
|
odometer_m bigint,
|
||||||
?, ?, ?,
|
longitude double precision,
|
||||||
?, ?, ?,
|
latitude double precision,
|
||||||
?, ?, ?,
|
payload jsonb not null,
|
||||||
?::jsonb, ?::jsonb, ?,
|
manual_entry boolean not null,
|
||||||
?, ?
|
event_signature_hash text
|
||||||
)
|
) on commit drop
|
||||||
on conflict do nothing
|
"""
|
||||||
|
);
|
||||||
|
jdbcTemplate.execute("truncate table eventhub_event_import_stage");
|
||||||
|
}
|
||||||
|
|
||||||
|
private void stageEvents(List<ResolvedEventImportRow> rows) {
|
||||||
|
jdbcTemplate.batchUpdate(
|
||||||
|
"""
|
||||||
|
insert into eventhub_event_import_stage(
|
||||||
|
row_no, source_record_key_hash, requested_event_id, data_package_id, event_source_id,
|
||||||
|
external_source_event_id, driver_entity_id, vehicle_id, vehicle_registration_id,
|
||||||
|
source_package_id, source_package_entity_id, occurred_at, received_partner_at, received_hub_at,
|
||||||
|
event_domain, event_type, lifecycle, odometer_m, longitude, latitude,
|
||||||
|
payload, manual_entry, event_signature_hash
|
||||||
|
) values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?::jsonb, ?, ?)
|
||||||
""",
|
""",
|
||||||
new BatchPreparedStatementSetter() {
|
new BatchPreparedStatementSetter() {
|
||||||
@Override
|
@Override
|
||||||
public void setValues(PreparedStatement ps, int i) throws java.sql.SQLException {
|
public void setValues(PreparedStatement ps, int i) throws SQLException {
|
||||||
EventHubEventDto event = events.get(i);
|
ResolvedEventImportRow row = rows.get(i);
|
||||||
UUID eventId = event.eventId() == null ? UUID.randomUUID() : event.eventId();
|
ps.setInt(1, i);
|
||||||
OffsetDateTime receivedHubAt = event.receivedHubAt() == null ? OffsetDateTime.now() : event.receivedHubAt();
|
ps.setString(2, row.sourceRecordKeyHash());
|
||||||
DriverRefDto driverRef = event.driverRef();
|
ps.setObject(3, row.requestedEventId());
|
||||||
DriverCardRefDto driverCard = driverRef == null ? null : driverRef.driverCard();
|
ps.setObject(4, row.packageId());
|
||||||
VehicleRefDto vehicleRef = event.vehicleRef();
|
ps.setInt(5, row.eventSourceId());
|
||||||
VehicleRegistrationRefDto vehicleRegistration = vehicleRef == null ? null : vehicleRef.vehicleRegistration();
|
ps.setString(6, row.externalSourceEventId());
|
||||||
SourcePackageRefDto sourcePackageRef = event.sourcePackageRef();
|
ps.setObject(7, row.driverEntityId());
|
||||||
|
ps.setObject(8, row.vehicleId());
|
||||||
ps.setObject(1, eventId);
|
ps.setObject(9, row.vehicleRegistrationId());
|
||||||
ps.setInt(2, eventSourceId);
|
ps.setString(10, row.sourcePackageId());
|
||||||
ps.setObject(3, packageId);
|
ps.setObject(11, row.sourcePackageEntityId());
|
||||||
ps.setString(4, event.externalSourceEventId());
|
ps.setObject(12, row.occurredAt());
|
||||||
|
ps.setObject(13, row.receivedPartnerAt());
|
||||||
ps.setString(5, driverRef == null ? null : driverRef.sourceEntityId());
|
ps.setObject(14, row.receivedHubAt());
|
||||||
ps.setString(6, driverCard == null ? null : driverCard.nation());
|
ps.setString(15, row.eventDomain());
|
||||||
ps.setString(7, driverCard == null ? null : driverCard.number());
|
ps.setString(16, row.eventType());
|
||||||
|
ps.setString(17, row.lifecycle());
|
||||||
ps.setString(8, vehicleRef == null ? null : vehicleRef.sourceEntityId());
|
ps.setObject(18, row.odometerM());
|
||||||
ps.setString(9, vehicleRef == null ? null : vehicleRef.vin());
|
ps.setObject(19, row.longitude());
|
||||||
ps.setString(10, vehicleRegistration == null ? null : vehicleRegistration.nation());
|
ps.setObject(20, row.latitude());
|
||||||
ps.setString(11, vehicleRegistration == null ? null : vehicleRegistration.number());
|
ps.setString(21, row.payloadJson());
|
||||||
|
ps.setBoolean(22, row.manualEntry());
|
||||||
ps.setString(12, sourcePackageRef == null ? null : sourcePackageRef.packageKind());
|
ps.setString(23, row.eventSignatureHash());
|
||||||
ps.setString(13, sourcePackageRef == null ? null : sourcePackageRef.sourcePackageId());
|
|
||||||
ps.setString(14, sourcePackageRef == null ? null : sourcePackageRef.sourceEntityId());
|
|
||||||
ps.setObject(15, sourcePackageRef == null ? null : sourcePackageRef.packagePeriodFrom());
|
|
||||||
ps.setObject(16, sourcePackageRef == null ? null : sourcePackageRef.packagePeriodTo());
|
|
||||||
ps.setObject(17, sourcePackageRef == null ? null : sourcePackageRef.importedIntoSourceAt());
|
|
||||||
|
|
||||||
ps.setObject(18, event.occurredAt());
|
|
||||||
ps.setObject(19, event.receivedPartnerAt());
|
|
||||||
ps.setObject(20, receivedHubAt);
|
|
||||||
ps.setString(21, event.eventDomain().name());
|
|
||||||
ps.setString(22, event.eventType().name());
|
|
||||||
ps.setString(23, event.lifecycle().name());
|
|
||||||
setNullableLong(ps, 24, event.odometerM());
|
|
||||||
if (event.position() == null) {
|
|
||||||
ps.setNull(25, Types.NUMERIC);
|
|
||||||
ps.setNull(26, Types.NUMERIC);
|
|
||||||
} else {
|
|
||||||
ps.setObject(25, event.position().latitude());
|
|
||||||
ps.setObject(26, event.position().longitude());
|
|
||||||
}
|
|
||||||
ps.setString(27, toJson(objectMapper.valueToTree(event.eventDetails())));
|
|
||||||
ps.setString(28, toJson(event.payload()));
|
|
||||||
ps.setBoolean(29, event.manualEntry());
|
|
||||||
ps.setString(30, recordKeyService.buildSourceRecordKeyHash(event, eventSourceId));
|
|
||||||
ps.setString(31, recordKeyService.buildEventSignatureHash(event));
|
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public int getBatchSize() {
|
public int getBatchSize() {
|
||||||
return events.size();
|
return rows.size();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
);
|
);
|
||||||
|
|
||||||
int inserted = 0;
|
|
||||||
for (int count : counts) {
|
|
||||||
if (count > 0 || count == PreparedStatement.SUCCESS_NO_INFO) {
|
|
||||||
inserted++;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return inserted;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
private void setNullableLong(PreparedStatement ps, int index, Long value) throws java.sql.SQLException {
|
private int insertStagedEvents() {
|
||||||
if (value == null) {
|
Long insertedCount = jdbcTemplate.queryForObject(
|
||||||
ps.setNull(index, Types.BIGINT);
|
"""
|
||||||
} else {
|
with stage_one as (
|
||||||
ps.setLong(index, value);
|
select distinct on (source_record_key_hash) *
|
||||||
|
from eventhub_event_import_stage
|
||||||
|
order by source_record_key_hash, row_no
|
||||||
|
),
|
||||||
|
reserved_source_record as (
|
||||||
|
insert into eventhub.event_source_record(
|
||||||
|
source_record_key_hash, event_occurred_at, event_id
|
||||||
|
)
|
||||||
|
select source_record_key_hash, occurred_at, requested_event_id
|
||||||
|
from stage_one
|
||||||
|
on conflict (source_record_key_hash) do nothing
|
||||||
|
returning source_record_key_hash, event_occurred_at, event_id
|
||||||
|
),
|
||||||
|
existing_source_record as (
|
||||||
|
select source_record.source_record_key_hash,
|
||||||
|
source_record.event_occurred_at,
|
||||||
|
source_record.event_id
|
||||||
|
from stage_one stage
|
||||||
|
join eventhub.event_source_record source_record
|
||||||
|
on source_record.source_record_key_hash = stage.source_record_key_hash
|
||||||
|
where not exists (
|
||||||
|
select 1
|
||||||
|
from reserved_source_record reserved
|
||||||
|
where reserved.source_record_key_hash = stage.source_record_key_hash
|
||||||
|
)
|
||||||
|
),
|
||||||
|
source_record_for_stage as (
|
||||||
|
select *
|
||||||
|
from reserved_source_record
|
||||||
|
union all
|
||||||
|
select *
|
||||||
|
from existing_source_record
|
||||||
|
),
|
||||||
|
inserted_event as (
|
||||||
|
insert into eventhub.event(
|
||||||
|
id, event_source_id, data_package_id,
|
||||||
|
external_source_event_id,
|
||||||
|
driver_entity_id, vehicle_id, vehicle_registration_id,
|
||||||
|
source_package_id, source_package_entity_id,
|
||||||
|
occurred_at, received_partner_at, received_hub_at,
|
||||||
|
event_domain, event_type, lifecycle,
|
||||||
|
odometer_m, position,
|
||||||
|
payload, manual_entry,
|
||||||
|
source_record_key_hash, event_signature_hash
|
||||||
|
)
|
||||||
|
select
|
||||||
|
source_record.event_id, stage.event_source_id, stage.data_package_id,
|
||||||
|
stage.external_source_event_id,
|
||||||
|
stage.driver_entity_id, stage.vehicle_id, stage.vehicle_registration_id,
|
||||||
|
stage.source_package_id, stage.source_package_entity_id,
|
||||||
|
source_record.event_occurred_at, stage.received_partner_at, stage.received_hub_at,
|
||||||
|
stage.event_domain, stage.event_type, stage.lifecycle,
|
||||||
|
stage.odometer_m, case
|
||||||
|
when stage.longitude is null or stage.latitude is null then null
|
||||||
|
else ST_SetSRID(ST_MakePoint(stage.longitude, stage.latitude), 4326)::geography
|
||||||
|
end,
|
||||||
|
stage.payload, stage.manual_entry,
|
||||||
|
stage.source_record_key_hash, stage.event_signature_hash
|
||||||
|
from stage_one stage
|
||||||
|
join source_record_for_stage source_record
|
||||||
|
on source_record.source_record_key_hash = stage.source_record_key_hash
|
||||||
|
where not exists (
|
||||||
|
select 1
|
||||||
|
from eventhub.event existing_event
|
||||||
|
where existing_event.occurred_at = source_record.event_occurred_at
|
||||||
|
and existing_event.id = source_record.event_id
|
||||||
|
)
|
||||||
|
returning id
|
||||||
|
)
|
||||||
|
select count(*)
|
||||||
|
from inserted_event
|
||||||
|
""",
|
||||||
|
Long.class
|
||||||
|
);
|
||||||
|
return insertedCount == null ? 0 : Math.toIntExact(insertedCount);
|
||||||
|
}
|
||||||
|
|
||||||
|
private Map<String, InsertedEventRow> resolveStagedEventRows() {
|
||||||
|
return jdbcTemplate.query(
|
||||||
|
"""
|
||||||
|
select distinct
|
||||||
|
stage.source_record_key_hash,
|
||||||
|
source_record.event_id,
|
||||||
|
source_record.event_occurred_at
|
||||||
|
from eventhub_event_import_stage stage
|
||||||
|
join eventhub.event_source_record source_record
|
||||||
|
on source_record.source_record_key_hash = stage.source_record_key_hash
|
||||||
|
""",
|
||||||
|
rs -> {
|
||||||
|
Map<String, InsertedEventRow> rows = new HashMap<>();
|
||||||
|
while (rs.next()) {
|
||||||
|
rows.put(
|
||||||
|
rs.getString("source_record_key_hash"),
|
||||||
|
new InsertedEventRow(
|
||||||
|
(UUID) rs.getObject("event_id"),
|
||||||
|
rs.getObject("event_occurred_at", OffsetDateTime.class)
|
||||||
|
)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
return rows;
|
||||||
|
}
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
private void batchUpsertEventDetails(
|
||||||
|
List<ResolvedEventImportRow> rows,
|
||||||
|
Map<String, InsertedEventRow> eventRowsBySourceRecord
|
||||||
|
) {
|
||||||
|
List<EventDetailImportRow> details = new ArrayList<>();
|
||||||
|
for (ResolvedEventImportRow row : rows) {
|
||||||
|
if (row.detailType() == null) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
InsertedEventRow insertedEventRow = eventRowsBySourceRecord.get(row.sourceRecordKeyHash());
|
||||||
|
if (insertedEventRow == null) {
|
||||||
|
throw new IllegalStateException("Could not insert or resolve event row for source record hash " + row.sourceRecordKeyHash());
|
||||||
|
}
|
||||||
|
details.add(new EventDetailImportRow(
|
||||||
|
insertedEventRow.eventId(),
|
||||||
|
insertedEventRow.occurredAt(),
|
||||||
|
row.detailType(),
|
||||||
|
row.detailAttributesJson()
|
||||||
|
));
|
||||||
}
|
}
|
||||||
|
if (details.isEmpty()) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
jdbcTemplate.batchUpdate(
|
||||||
|
"""
|
||||||
|
insert into eventhub.event_detail(
|
||||||
|
event_occurred_at, event_id, detail_type, attributes
|
||||||
|
) values (?, ?, ?, ?::jsonb)
|
||||||
|
on conflict (event_occurred_at, event_id, detail_type)
|
||||||
|
do update set
|
||||||
|
attributes = excluded.attributes
|
||||||
|
""",
|
||||||
|
new BatchPreparedStatementSetter() {
|
||||||
|
@Override
|
||||||
|
public void setValues(PreparedStatement ps, int i) throws SQLException {
|
||||||
|
EventDetailImportRow detail = details.get(i);
|
||||||
|
ps.setObject(1, detail.occurredAt());
|
||||||
|
ps.setObject(2, detail.eventId());
|
||||||
|
ps.setString(3, detail.detailType());
|
||||||
|
ps.setString(4, detail.attributesJson());
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public int getBatchSize() {
|
||||||
|
return details.size();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
private ResolvedEntityRefs resolveEntityRefs(
|
||||||
|
String tenantKey,
|
||||||
|
int eventSourceId,
|
||||||
|
EventHubEventDto event,
|
||||||
|
Map<String, UUID> entityIdCache,
|
||||||
|
Map<String, List<VehicleRefCacheEntry>> vehicleRefCache
|
||||||
|
) {
|
||||||
|
UUID driverEntityId = resolveDriverEntityId(tenantKey, eventSourceId, event, entityIdCache);
|
||||||
|
ResolvedVehicleReference vehicleRef = resolveVehicleReference(tenantKey, eventSourceId, event, vehicleRefCache);
|
||||||
|
UUID sourcePackageEntityId = resolveSourcePackageEntityId(tenantKey, eventSourceId, event, entityIdCache);
|
||||||
|
return new ResolvedEntityRefs(driverEntityId, vehicleRef.vehicleId(), vehicleRef.vehicleRegistrationId(), sourcePackageEntityId);
|
||||||
|
}
|
||||||
|
|
||||||
|
private UUID resolveDriverEntityId(
|
||||||
|
String tenantKey,
|
||||||
|
int eventSourceId,
|
||||||
|
EventHubEventDto event,
|
||||||
|
Map<String, UUID> entityIdCache
|
||||||
|
) {
|
||||||
|
DriverRefDto driverRef = event.driverRef();
|
||||||
|
if (driverRef == null || !driverRef.hasAnyReference()) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
DriverCardRefDto card = driverRef.driverCard();
|
||||||
|
String cardKey = card == null || !card.hasValue() ? null : card.stableKey();
|
||||||
|
String sourceEntityId = normalizeNullable(driverRef.sourceEntityId());
|
||||||
|
if (sourceEntityId == null && cardKey != null) {
|
||||||
|
sourceEntityId = "DRIVER_CARD:" + cardKey;
|
||||||
|
}
|
||||||
|
if (sourceEntityId == null) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
Map<String, Object> payload = new LinkedHashMap<>();
|
||||||
|
put(payload, "source_entity_id", driverRef.sourceEntityId());
|
||||||
|
put(payload, "driver_card_nation", card == null ? null : card.nation());
|
||||||
|
put(payload, "driver_card_number", card == null ? null : card.number());
|
||||||
|
return resolveEntityId(
|
||||||
|
tenantKey,
|
||||||
|
eventSourceId,
|
||||||
|
"DRIVER",
|
||||||
|
sourceEntityId,
|
||||||
|
cardKey,
|
||||||
|
sourceEntityId,
|
||||||
|
null,
|
||||||
|
payload,
|
||||||
|
entityIdCache
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
private ResolvedVehicleReference resolveVehicleReference(
|
||||||
|
String tenantKey,
|
||||||
|
int eventSourceId,
|
||||||
|
EventHubEventDto event,
|
||||||
|
Map<String, List<VehicleRefCacheEntry>> vehicleRefCache
|
||||||
|
) {
|
||||||
|
String cacheKey = vehicleRefCacheKey(event.vehicleRef());
|
||||||
|
if (cacheKey != null) {
|
||||||
|
ResolvedVehicleReference cached = findCachedVehicleReference(
|
||||||
|
vehicleRefCache.get(cacheKey),
|
||||||
|
event.occurredAt()
|
||||||
|
);
|
||||||
|
if (cached != null) {
|
||||||
|
return cached;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
ResolvedVehicleReferenceResolution resolved = vehicleIdentityRepository.resolveOrCreateVehicleReference(
|
||||||
|
tenantKey,
|
||||||
|
eventSourceId,
|
||||||
|
event.vehicleRef(),
|
||||||
|
event.occurredAt()
|
||||||
|
);
|
||||||
|
if (cacheKey != null) {
|
||||||
|
vehicleRefCache
|
||||||
|
.computeIfAbsent(cacheKey, ignored -> new ArrayList<>())
|
||||||
|
.add(buildVehicleRefCacheEntry(event.vehicleRef(), event.occurredAt(), resolved));
|
||||||
|
}
|
||||||
|
return resolved.reference();
|
||||||
|
}
|
||||||
|
|
||||||
|
private UUID resolveSourcePackageEntityId(
|
||||||
|
String tenantKey,
|
||||||
|
int eventSourceId,
|
||||||
|
EventHubEventDto event,
|
||||||
|
Map<String, UUID> entityIdCache
|
||||||
|
) {
|
||||||
|
SourcePackageRefDto sourcePackageRef = event.sourcePackageRef();
|
||||||
|
if (sourcePackageRef == null || !sourcePackageRef.hasAnyReference()) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
String packageKind = normalizeNullable(sourcePackageRef.packageKind());
|
||||||
|
String sourcePackageId = normalizeNullable(sourcePackageRef.sourcePackageId());
|
||||||
|
|
||||||
|
String sourceEntityId = normalizeNullable(sourcePackageRef.sourceEntityId());
|
||||||
|
if (sourceEntityId == null && sourcePackageId != null) {
|
||||||
|
sourceEntityId = "SOURCE_PACKAGE:" + (packageKind == null ? "UNKNOWN" : packageKind) + ":" + sourcePackageId;
|
||||||
|
}
|
||||||
|
if (sourceEntityId == null) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
Map<String, Object> payload = new LinkedHashMap<>();
|
||||||
|
put(payload, "package_kind", sourcePackageRef.packageKind());
|
||||||
|
put(payload, "source_package_id", sourcePackageRef.sourcePackageId());
|
||||||
|
put(payload, "source_entity_id", sourcePackageRef.sourceEntityId());
|
||||||
|
put(payload, "package_period_from", sourcePackageRef.packagePeriodFrom());
|
||||||
|
put(payload, "package_period_to", sourcePackageRef.packagePeriodTo());
|
||||||
|
put(payload, "imported_into_source_at", sourcePackageRef.importedIntoSourceAt());
|
||||||
|
|
||||||
|
String displayName = packageKind == null ? sourceEntityId : packageKind + ":" + (sourcePackageId == null ? sourceEntityId : sourcePackageId);
|
||||||
|
return resolveEntityId(
|
||||||
|
tenantKey,
|
||||||
|
eventSourceId,
|
||||||
|
"SOURCE_PACKAGE",
|
||||||
|
sourceEntityId,
|
||||||
|
sourcePackageId,
|
||||||
|
displayName,
|
||||||
|
null,
|
||||||
|
payload,
|
||||||
|
entityIdCache
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
private UUID resolveEntityId(
|
||||||
|
String tenantKey,
|
||||||
|
int eventSourceId,
|
||||||
|
String entityType,
|
||||||
|
String sourceEntityId,
|
||||||
|
String sourceExternalKey,
|
||||||
|
String displayName,
|
||||||
|
Boolean active,
|
||||||
|
Map<String, Object> payload,
|
||||||
|
Map<String, UUID> entityIdCache
|
||||||
|
) {
|
||||||
|
String normalizedSourceEntityId = normalizeNullable(sourceEntityId);
|
||||||
|
if (normalizedSourceEntityId == null) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
String cacheKey = entityType + "|" + normalizedSourceEntityId;
|
||||||
|
UUID cached = entityIdCache.get(cacheKey);
|
||||||
|
if (cached != null) {
|
||||||
|
return cached;
|
||||||
|
}
|
||||||
|
|
||||||
|
UUID resolved = sourceMasterDataRepository.resolveOrCreateEntityId(
|
||||||
|
tenantKey,
|
||||||
|
eventSourceId,
|
||||||
|
entityType,
|
||||||
|
normalizedSourceEntityId,
|
||||||
|
normalizeNullable(sourceExternalKey),
|
||||||
|
normalizeNullable(displayName),
|
||||||
|
active,
|
||||||
|
payload
|
||||||
|
);
|
||||||
|
entityIdCache.put(cacheKey, resolved);
|
||||||
|
return resolved;
|
||||||
|
}
|
||||||
|
|
||||||
|
private String normalizeNullable(String value) {
|
||||||
|
if (value == null) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
String trimmed = value.trim();
|
||||||
|
return trimmed.isEmpty() ? null : trimmed;
|
||||||
|
}
|
||||||
|
|
||||||
|
private void put(Map<String, Object> target, String key, Object value) {
|
||||||
|
if (value != null) {
|
||||||
|
target.put(key, value);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private String vehicleRefCacheKey(VehicleRefDto vehicleRef) {
|
||||||
|
if (vehicleRef == null || !vehicleRef.hasAnyReference()) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
String sourceVehicleEntityId = normalizeNullable(vehicleRef.sourceVehicleEntityId());
|
||||||
|
String vin = normalizeNullable(vehicleRef.vin());
|
||||||
|
String sourceRegistrationEntityId = normalizeNullable(vehicleRef.sourceRegistrationEntityId());
|
||||||
|
VehicleRegistrationRefDto registration = vehicleRef.vehicleRegistration();
|
||||||
|
String registrationNation = registration == null ? null : normalizeNullable(registration.nation());
|
||||||
|
String registrationNumber = registration == null ? null : normalizeNullable(registration.number());
|
||||||
|
|
||||||
|
return String.join("|",
|
||||||
|
sourceVehicleEntityId == null ? "" : sourceVehicleEntityId,
|
||||||
|
vin == null ? "" : vin,
|
||||||
|
sourceRegistrationEntityId == null ? "" : sourceRegistrationEntityId,
|
||||||
|
registrationNation == null ? "" : registrationNation,
|
||||||
|
registrationNumber == null ? "" : registrationNumber
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
private ResolvedVehicleReference findCachedVehicleReference(
|
||||||
|
List<VehicleRefCacheEntry> cacheEntries,
|
||||||
|
OffsetDateTime occurredAt
|
||||||
|
) {
|
||||||
|
if (cacheEntries == null || cacheEntries.isEmpty()) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
for (VehicleRefCacheEntry cacheEntry : cacheEntries) {
|
||||||
|
if (cacheEntry.matches(occurredAt)) {
|
||||||
|
return cacheEntry.reference();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
private VehicleRefCacheEntry buildVehicleRefCacheEntry(
|
||||||
|
VehicleRefDto vehicleRef,
|
||||||
|
OffsetDateTime occurredAt,
|
||||||
|
ResolvedVehicleReferenceResolution resolved
|
||||||
|
) {
|
||||||
|
boolean hasDirectVehicleIdentity = hasDirectVehicleIdentity(vehicleRef);
|
||||||
|
if (hasDirectVehicleIdentity) {
|
||||||
|
return new VehicleRefCacheEntry(resolved.reference(), null, null, null);
|
||||||
|
}
|
||||||
|
if (resolved.resolvedFromAssignment()) {
|
||||||
|
return new VehicleRefCacheEntry(
|
||||||
|
resolved.reference(),
|
||||||
|
resolved.assignmentValidFrom(),
|
||||||
|
resolved.assignmentValidTo(),
|
||||||
|
null
|
||||||
|
);
|
||||||
|
}
|
||||||
|
return new VehicleRefCacheEntry(resolved.reference(), null, null, occurredAt);
|
||||||
|
}
|
||||||
|
|
||||||
|
private boolean hasDirectVehicleIdentity(VehicleRefDto vehicleRef) {
|
||||||
|
if (vehicleRef == null) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
return normalizeNullable(vehicleRef.sourceVehicleEntityId()) != null
|
||||||
|
|| normalizeNullable(vehicleRef.vin()) != null;
|
||||||
}
|
}
|
||||||
|
|
||||||
private String toJson(JsonNode value) {
|
private String toJson(JsonNode value) {
|
||||||
|
|
@ -154,4 +597,73 @@ public class EventRepository {
|
||||||
throw new IllegalArgumentException("Cannot serialize JSONB value", e);
|
throw new IllegalArgumentException("Cannot serialize JSONB value", e);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private record ResolvedEntityRefs(
|
||||||
|
UUID driverEntityId,
|
||||||
|
UUID vehicleId,
|
||||||
|
UUID vehicleRegistrationId,
|
||||||
|
UUID sourcePackageEntityId
|
||||||
|
) {
|
||||||
|
}
|
||||||
|
|
||||||
|
private record InsertedEventRow(
|
||||||
|
UUID eventId,
|
||||||
|
OffsetDateTime occurredAt
|
||||||
|
) {
|
||||||
|
}
|
||||||
|
|
||||||
|
private record ResolvedEventImportRow(
|
||||||
|
String sourceRecordKeyHash,
|
||||||
|
UUID requestedEventId,
|
||||||
|
UUID packageId,
|
||||||
|
int eventSourceId,
|
||||||
|
String externalSourceEventId,
|
||||||
|
UUID driverEntityId,
|
||||||
|
UUID vehicleId,
|
||||||
|
UUID vehicleRegistrationId,
|
||||||
|
String sourcePackageId,
|
||||||
|
UUID sourcePackageEntityId,
|
||||||
|
OffsetDateTime occurredAt,
|
||||||
|
OffsetDateTime receivedPartnerAt,
|
||||||
|
OffsetDateTime receivedHubAt,
|
||||||
|
String eventDomain,
|
||||||
|
String eventType,
|
||||||
|
String lifecycle,
|
||||||
|
Long odometerM,
|
||||||
|
Double longitude,
|
||||||
|
Double latitude,
|
||||||
|
String payloadJson,
|
||||||
|
boolean manualEntry,
|
||||||
|
String eventSignatureHash,
|
||||||
|
String detailType,
|
||||||
|
String detailAttributesJson
|
||||||
|
) {
|
||||||
|
}
|
||||||
|
|
||||||
|
private record EventDetailImportRow(
|
||||||
|
UUID eventId,
|
||||||
|
OffsetDateTime occurredAt,
|
||||||
|
String detailType,
|
||||||
|
String attributesJson
|
||||||
|
) {
|
||||||
|
}
|
||||||
|
|
||||||
|
private record VehicleRefCacheEntry(
|
||||||
|
ResolvedVehicleReference reference,
|
||||||
|
OffsetDateTime validFrom,
|
||||||
|
OffsetDateTime validTo,
|
||||||
|
OffsetDateTime exactOccurredAt
|
||||||
|
) {
|
||||||
|
private boolean matches(OffsetDateTime occurredAt) {
|
||||||
|
if (exactOccurredAt != null) {
|
||||||
|
return exactOccurredAt.equals(occurredAt);
|
||||||
|
}
|
||||||
|
if (validFrom == null && validTo == null) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
boolean fromMatches = validFrom == null || occurredAt == null || !occurredAt.isBefore(validFrom);
|
||||||
|
boolean toMatches = validTo == null || occurredAt == null || occurredAt.isBefore(validTo);
|
||||||
|
return fromMatches && toMatches;
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,536 @@
|
||||||
|
package at.procon.eventhub.persistence;
|
||||||
|
|
||||||
|
import at.procon.eventhub.importing.masterdata.SourceMasterEntityUpsert;
|
||||||
|
import at.procon.eventhub.importing.masterdata.SourceMasterRelationUpsert;
|
||||||
|
import com.fasterxml.jackson.core.JsonProcessingException;
|
||||||
|
import com.fasterxml.jackson.databind.JsonNode;
|
||||||
|
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||||
|
import com.fasterxml.jackson.databind.node.POJONode;
|
||||||
|
import java.lang.reflect.Array;
|
||||||
|
import java.sql.PreparedStatement;
|
||||||
|
import java.sql.SQLException;
|
||||||
|
import java.sql.Timestamp;
|
||||||
|
import java.time.OffsetDateTime;
|
||||||
|
import java.time.temporal.TemporalAccessor;
|
||||||
|
import java.time.temporal.TemporalAmount;
|
||||||
|
import java.util.ArrayList;
|
||||||
|
import java.util.Base64;
|
||||||
|
import java.util.LinkedHashMap;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.Map;
|
||||||
|
import java.util.UUID;
|
||||||
|
import java.util.stream.Collectors;
|
||||||
|
import org.springframework.jdbc.core.BatchPreparedStatementSetter;
|
||||||
|
import org.springframework.jdbc.core.JdbcTemplate;
|
||||||
|
import org.springframework.stereotype.Repository;
|
||||||
|
import org.springframework.transaction.annotation.Transactional;
|
||||||
|
|
||||||
|
@Repository
|
||||||
|
public class SourceMasterDataRepository {
|
||||||
|
|
||||||
|
private final JdbcTemplate jdbcTemplate;
|
||||||
|
private final ObjectMapper objectMapper;
|
||||||
|
|
||||||
|
public SourceMasterDataRepository(JdbcTemplate jdbcTemplate, ObjectMapper objectMapper) {
|
||||||
|
this.jdbcTemplate = jdbcTemplate;
|
||||||
|
this.objectMapper = objectMapper;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Transactional
|
||||||
|
public int upsertEntities(String tenantKey, int eventSourceId, List<SourceMasterEntityUpsert> entities) {
|
||||||
|
List<SourceMasterEntityStageRow> rows = new ArrayList<>(entities.size());
|
||||||
|
for (SourceMasterEntityUpsert entity : entities) {
|
||||||
|
if (entity.sourceEntityId() == null || entity.sourceEntityId().isBlank()) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
rows.add(new SourceMasterEntityStageRow(
|
||||||
|
UUID.randomUUID(),
|
||||||
|
entity.entityType(),
|
||||||
|
entity.sourceEntityId(),
|
||||||
|
entity.sourceExternalKey(),
|
||||||
|
entity.displayName(),
|
||||||
|
entity.active(),
|
||||||
|
entity.validFrom(),
|
||||||
|
entity.validTo(),
|
||||||
|
entity.sourceUpdatedAt(),
|
||||||
|
toJson(entity.payload())
|
||||||
|
));
|
||||||
|
}
|
||||||
|
if (rows.isEmpty()) {
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
createEntityStage();
|
||||||
|
stageEntities(rows);
|
||||||
|
upsertStagedEntities(tenantKey, eventSourceId);
|
||||||
|
return rows.size();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Transactional
|
||||||
|
public int upsertRelations(String tenantKey, int eventSourceId, List<SourceMasterRelationUpsert> relations) {
|
||||||
|
List<SourceMasterRelationStageRow> rows = new ArrayList<>(relations.size());
|
||||||
|
for (SourceMasterRelationUpsert relation : relations) {
|
||||||
|
if (relation.fromSourceEntityId() == null || relation.toSourceEntityId() == null) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
rows.add(new SourceMasterRelationStageRow(
|
||||||
|
UUID.randomUUID(),
|
||||||
|
relationKey(relation),
|
||||||
|
relation.relationType(),
|
||||||
|
relation.fromEntityType(),
|
||||||
|
relation.fromSourceEntityId(),
|
||||||
|
relation.toEntityType(),
|
||||||
|
relation.toSourceEntityId(),
|
||||||
|
relation.validFrom(),
|
||||||
|
relation.validTo(),
|
||||||
|
relation.sourceUpdatedAt(),
|
||||||
|
toJson(relation.payload())
|
||||||
|
));
|
||||||
|
}
|
||||||
|
if (rows.isEmpty()) {
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
createRelationStage();
|
||||||
|
stageRelations(rows);
|
||||||
|
upsertStagedRelations(tenantKey, eventSourceId);
|
||||||
|
return rows.size();
|
||||||
|
}
|
||||||
|
|
||||||
|
private void createEntityStage() {
|
||||||
|
jdbcTemplate.execute(
|
||||||
|
"""
|
||||||
|
create temporary table if not exists eventhub_source_master_entity_stage (
|
||||||
|
row_no integer not null,
|
||||||
|
id uuid not null,
|
||||||
|
entity_type text not null,
|
||||||
|
source_entity_id text not null,
|
||||||
|
source_external_key text,
|
||||||
|
display_name text,
|
||||||
|
active boolean,
|
||||||
|
valid_from timestamptz,
|
||||||
|
valid_to timestamptz,
|
||||||
|
source_updated_at timestamptz,
|
||||||
|
payload jsonb not null
|
||||||
|
) on commit drop
|
||||||
|
"""
|
||||||
|
);
|
||||||
|
jdbcTemplate.execute("truncate table eventhub_source_master_entity_stage");
|
||||||
|
}
|
||||||
|
|
||||||
|
private void stageEntities(List<SourceMasterEntityStageRow> rows) {
|
||||||
|
jdbcTemplate.batchUpdate(
|
||||||
|
"""
|
||||||
|
insert into eventhub_source_master_entity_stage(
|
||||||
|
row_no, id, entity_type, source_entity_id, source_external_key, display_name,
|
||||||
|
active, valid_from, valid_to, source_updated_at, payload
|
||||||
|
) values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?::jsonb)
|
||||||
|
""",
|
||||||
|
new BatchPreparedStatementSetter() {
|
||||||
|
@Override
|
||||||
|
public void setValues(PreparedStatement ps, int i) throws SQLException {
|
||||||
|
SourceMasterEntityStageRow row = rows.get(i);
|
||||||
|
ps.setInt(1, i);
|
||||||
|
ps.setObject(2, row.id());
|
||||||
|
ps.setString(3, row.entityType());
|
||||||
|
ps.setString(4, row.sourceEntityId());
|
||||||
|
ps.setString(5, row.sourceExternalKey());
|
||||||
|
ps.setString(6, row.displayName());
|
||||||
|
ps.setObject(7, row.active());
|
||||||
|
ps.setObject(8, row.validFrom());
|
||||||
|
ps.setObject(9, row.validTo());
|
||||||
|
ps.setObject(10, row.sourceUpdatedAt());
|
||||||
|
ps.setString(11, row.payloadJson());
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public int getBatchSize() {
|
||||||
|
return rows.size();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
private void upsertStagedEntities(String tenantKey, int eventSourceId) {
|
||||||
|
jdbcTemplate.update(
|
||||||
|
"""
|
||||||
|
insert into eventhub.source_master_entity(
|
||||||
|
id, tenant_key, event_source_id, entity_type, source_entity_id,
|
||||||
|
source_external_key, display_name, active, valid_from, valid_to,
|
||||||
|
source_updated_at, payload, updated_at
|
||||||
|
)
|
||||||
|
select
|
||||||
|
id, ?, ?, entity_type, source_entity_id,
|
||||||
|
source_external_key, display_name, active, valid_from, valid_to,
|
||||||
|
source_updated_at, payload, now()
|
||||||
|
from (
|
||||||
|
select distinct on (entity_type, source_entity_id) *
|
||||||
|
from eventhub_source_master_entity_stage
|
||||||
|
order by entity_type, source_entity_id, row_no desc
|
||||||
|
) stage
|
||||||
|
on conflict (tenant_key, event_source_id, entity_type, source_entity_id)
|
||||||
|
do update set
|
||||||
|
source_external_key = excluded.source_external_key,
|
||||||
|
display_name = excluded.display_name,
|
||||||
|
active = excluded.active,
|
||||||
|
valid_from = excluded.valid_from,
|
||||||
|
valid_to = excluded.valid_to,
|
||||||
|
source_updated_at = excluded.source_updated_at,
|
||||||
|
payload = excluded.payload,
|
||||||
|
updated_at = now()
|
||||||
|
""",
|
||||||
|
tenantKey,
|
||||||
|
eventSourceId
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
private void createRelationStage() {
|
||||||
|
jdbcTemplate.execute(
|
||||||
|
"""
|
||||||
|
create temporary table if not exists eventhub_source_master_relation_stage (
|
||||||
|
row_no integer not null,
|
||||||
|
id uuid not null,
|
||||||
|
relation_key text not null,
|
||||||
|
relation_type text not null,
|
||||||
|
from_entity_type text not null,
|
||||||
|
from_source_entity_id text not null,
|
||||||
|
to_entity_type text not null,
|
||||||
|
to_source_entity_id text not null,
|
||||||
|
valid_from timestamptz,
|
||||||
|
valid_to timestamptz,
|
||||||
|
source_updated_at timestamptz,
|
||||||
|
payload jsonb not null
|
||||||
|
) on commit drop
|
||||||
|
"""
|
||||||
|
);
|
||||||
|
jdbcTemplate.execute("truncate table eventhub_source_master_relation_stage");
|
||||||
|
}
|
||||||
|
|
||||||
|
private void stageRelations(List<SourceMasterRelationStageRow> rows) {
|
||||||
|
jdbcTemplate.batchUpdate(
|
||||||
|
"""
|
||||||
|
insert into eventhub_source_master_relation_stage(
|
||||||
|
row_no, id, relation_key, relation_type, from_entity_type, from_source_entity_id,
|
||||||
|
to_entity_type, to_source_entity_id, valid_from, valid_to, source_updated_at, payload
|
||||||
|
) values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?::jsonb)
|
||||||
|
""",
|
||||||
|
new BatchPreparedStatementSetter() {
|
||||||
|
@Override
|
||||||
|
public void setValues(PreparedStatement ps, int i) throws SQLException {
|
||||||
|
SourceMasterRelationStageRow row = rows.get(i);
|
||||||
|
ps.setInt(1, i);
|
||||||
|
ps.setObject(2, row.id());
|
||||||
|
ps.setString(3, row.relationKey());
|
||||||
|
ps.setString(4, row.relationType());
|
||||||
|
ps.setString(5, row.fromEntityType());
|
||||||
|
ps.setString(6, row.fromSourceEntityId());
|
||||||
|
ps.setString(7, row.toEntityType());
|
||||||
|
ps.setString(8, row.toSourceEntityId());
|
||||||
|
ps.setObject(9, row.validFrom());
|
||||||
|
ps.setObject(10, row.validTo());
|
||||||
|
ps.setObject(11, row.sourceUpdatedAt());
|
||||||
|
ps.setString(12, row.payloadJson());
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public int getBatchSize() {
|
||||||
|
return rows.size();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
private void upsertStagedRelations(String tenantKey, int eventSourceId) {
|
||||||
|
jdbcTemplate.update(
|
||||||
|
"""
|
||||||
|
insert into eventhub.source_master_relation(
|
||||||
|
id, tenant_key, event_source_id, relation_key, relation_type,
|
||||||
|
from_entity_type, from_source_entity_id, to_entity_type, to_source_entity_id,
|
||||||
|
valid_from, valid_to, source_updated_at, payload, updated_at
|
||||||
|
)
|
||||||
|
select
|
||||||
|
id, ?, ?, relation_key, relation_type,
|
||||||
|
from_entity_type, from_source_entity_id, to_entity_type, to_source_entity_id,
|
||||||
|
valid_from, valid_to, source_updated_at, payload, now()
|
||||||
|
from (
|
||||||
|
select distinct on (relation_key) *
|
||||||
|
from eventhub_source_master_relation_stage
|
||||||
|
order by relation_key, row_no desc
|
||||||
|
) stage
|
||||||
|
on conflict (tenant_key, event_source_id, relation_key)
|
||||||
|
do update set
|
||||||
|
relation_type = excluded.relation_type,
|
||||||
|
from_entity_type = excluded.from_entity_type,
|
||||||
|
from_source_entity_id = excluded.from_source_entity_id,
|
||||||
|
to_entity_type = excluded.to_entity_type,
|
||||||
|
to_source_entity_id = excluded.to_source_entity_id,
|
||||||
|
valid_from = excluded.valid_from,
|
||||||
|
valid_to = excluded.valid_to,
|
||||||
|
source_updated_at = excluded.source_updated_at,
|
||||||
|
payload = excluded.payload,
|
||||||
|
updated_at = now()
|
||||||
|
""",
|
||||||
|
tenantKey,
|
||||||
|
eventSourceId
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
public UUID resolveOrCreateEntityId(
|
||||||
|
String tenantKey,
|
||||||
|
int eventSourceId,
|
||||||
|
String entityType,
|
||||||
|
String sourceEntityId,
|
||||||
|
String sourceExternalKey,
|
||||||
|
String displayName,
|
||||||
|
Boolean active,
|
||||||
|
Map<String, Object> payload
|
||||||
|
) {
|
||||||
|
String normalizedTenantKey = normalizeRequired(tenantKey, "tenantKey");
|
||||||
|
String normalizedEntityType = normalizeRequired(entityType, "entityType").toUpperCase();
|
||||||
|
String normalizedSourceEntityId = normalizeRequired(sourceEntityId, "sourceEntityId");
|
||||||
|
String normalizedSourceExternalKey = normalizeNullable(sourceExternalKey);
|
||||||
|
String normalizedDisplayName = normalizeNullable(displayName);
|
||||||
|
UUID existing = findEntityId(normalizedTenantKey, eventSourceId, normalizedEntityType, normalizedSourceEntityId);
|
||||||
|
if (existing != null) {
|
||||||
|
return existing;
|
||||||
|
}
|
||||||
|
|
||||||
|
UUID created = jdbcTemplate.query(
|
||||||
|
con -> {
|
||||||
|
PreparedStatement ps = con.prepareStatement(
|
||||||
|
"""
|
||||||
|
insert into eventhub.source_master_entity(
|
||||||
|
id, tenant_key, event_source_id, entity_type, source_entity_id,
|
||||||
|
source_external_key, display_name, active, payload, updated_at
|
||||||
|
) values (?, ?, ?, ?, ?, ?, ?, ?, ?::jsonb, now())
|
||||||
|
on conflict (tenant_key, event_source_id, entity_type, source_entity_id) do nothing
|
||||||
|
returning id
|
||||||
|
"""
|
||||||
|
);
|
||||||
|
ps.setObject(1, UUID.randomUUID());
|
||||||
|
ps.setString(2, normalizedTenantKey);
|
||||||
|
ps.setInt(3, eventSourceId);
|
||||||
|
ps.setString(4, normalizedEntityType);
|
||||||
|
ps.setString(5, normalizedSourceEntityId);
|
||||||
|
ps.setString(6, normalizedSourceExternalKey);
|
||||||
|
ps.setString(7, normalizedDisplayName);
|
||||||
|
ps.setObject(8, active);
|
||||||
|
ps.setString(9, toJson(payload));
|
||||||
|
return ps;
|
||||||
|
},
|
||||||
|
rs -> {
|
||||||
|
if (!rs.next()) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
return (UUID) rs.getObject(1);
|
||||||
|
}
|
||||||
|
);
|
||||||
|
if (created != null) {
|
||||||
|
return created;
|
||||||
|
}
|
||||||
|
|
||||||
|
UUID resolved = findEntityId(normalizedTenantKey, eventSourceId, normalizedEntityType, normalizedSourceEntityId);
|
||||||
|
if (resolved != null) {
|
||||||
|
return resolved;
|
||||||
|
}
|
||||||
|
throw new IllegalStateException(
|
||||||
|
"Could not resolve source master entity id for "
|
||||||
|
+ normalizedTenantKey + ":" + eventSourceId + ":" + normalizedEntityType + ":" + normalizedSourceEntityId
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
private UUID findEntityId(String tenantKey, int eventSourceId, String entityType, String sourceEntityId) {
|
||||||
|
return jdbcTemplate.query(
|
||||||
|
"""
|
||||||
|
select id
|
||||||
|
from eventhub.source_master_entity
|
||||||
|
where tenant_key = ?
|
||||||
|
and event_source_id = ?
|
||||||
|
and entity_type = ?
|
||||||
|
and source_entity_id = ?
|
||||||
|
""",
|
||||||
|
rs -> rs.next() ? (UUID) rs.getObject("id") : null,
|
||||||
|
tenantKey,
|
||||||
|
eventSourceId,
|
||||||
|
entityType,
|
||||||
|
sourceEntityId
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
private String relationKey(SourceMasterRelationUpsert relation) {
|
||||||
|
String validFrom = relation.validFrom() == null ? "MIN" : relation.validFrom().toString();
|
||||||
|
String validTo = relation.validTo() == null ? "MAX" : relation.validTo().toString();
|
||||||
|
return relation.relationType()
|
||||||
|
+ ":" + relation.fromEntityType()
|
||||||
|
+ ":" + relation.fromSourceEntityId()
|
||||||
|
+ ":" + relation.toEntityType()
|
||||||
|
+ ":" + relation.toSourceEntityId()
|
||||||
|
+ ":" + validFrom
|
||||||
|
+ ":" + validTo;
|
||||||
|
}
|
||||||
|
|
||||||
|
private String toJson(Map<String, Object> value) {
|
||||||
|
Map<String, Object> source = value == null ? Map.of() : value;
|
||||||
|
try {
|
||||||
|
return objectMapper.writeValueAsString(normalizeJsonMap(source));
|
||||||
|
} catch (JsonProcessingException e) {
|
||||||
|
throw new IllegalArgumentException(
|
||||||
|
"Cannot serialize source master data payload. payloadTypes=" + describePayloadTypes(source),
|
||||||
|
e
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private Map<String, Object> normalizeJsonMap(Map<?, ?> source) {
|
||||||
|
Map<String, Object> normalized = new LinkedHashMap<>(source.size());
|
||||||
|
for (Map.Entry<?, ?> entry : source.entrySet()) {
|
||||||
|
if (entry.getKey() == null) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
normalized.put(String.valueOf(entry.getKey()), normalizeJsonValue(entry.getValue()));
|
||||||
|
}
|
||||||
|
return normalized;
|
||||||
|
}
|
||||||
|
|
||||||
|
private Object normalizeJsonValue(Object value) {
|
||||||
|
if (value == null
|
||||||
|
|| value instanceof String
|
||||||
|
|| value instanceof Number
|
||||||
|
|| value instanceof Boolean) {
|
||||||
|
return value;
|
||||||
|
}
|
||||||
|
if (value instanceof JsonNode node) {
|
||||||
|
return normalizeJsonNode(node);
|
||||||
|
}
|
||||||
|
if (value instanceof Enum<?> enumValue) {
|
||||||
|
return enumValue.name();
|
||||||
|
}
|
||||||
|
if (value instanceof UUID uuid) {
|
||||||
|
return uuid.toString();
|
||||||
|
}
|
||||||
|
if (value instanceof byte[] bytes) {
|
||||||
|
return Base64.getEncoder().encodeToString(bytes);
|
||||||
|
}
|
||||||
|
if (value instanceof Timestamp timestamp) {
|
||||||
|
return timestamp.toInstant().toString();
|
||||||
|
}
|
||||||
|
if (value instanceof java.util.Date date) {
|
||||||
|
return date.toInstant().toString();
|
||||||
|
}
|
||||||
|
if (value instanceof TemporalAccessor || value instanceof TemporalAmount) {
|
||||||
|
return value.toString();
|
||||||
|
}
|
||||||
|
if (value instanceof Map<?, ?> map) {
|
||||||
|
return normalizeJsonMap(map);
|
||||||
|
}
|
||||||
|
if (value instanceof Iterable<?> iterable) {
|
||||||
|
List<Object> values = new ArrayList<>();
|
||||||
|
for (Object entry : iterable) {
|
||||||
|
values.add(normalizeJsonValue(entry));
|
||||||
|
}
|
||||||
|
return values;
|
||||||
|
}
|
||||||
|
if (value.getClass().isArray()) {
|
||||||
|
int length = Array.getLength(value);
|
||||||
|
List<Object> values = new ArrayList<>(length);
|
||||||
|
for (int i = 0; i < length; i++) {
|
||||||
|
values.add(normalizeJsonValue(Array.get(value, i)));
|
||||||
|
}
|
||||||
|
return values;
|
||||||
|
}
|
||||||
|
return String.valueOf(value);
|
||||||
|
}
|
||||||
|
|
||||||
|
private Object normalizeJsonNode(JsonNode node) {
|
||||||
|
if (node == null || node.isNull() || node.isMissingNode()) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
if (node instanceof POJONode pojoNode) {
|
||||||
|
return normalizeJsonValue(pojoNode.getPojo());
|
||||||
|
}
|
||||||
|
if (node.isObject()) {
|
||||||
|
return normalizeJsonMap(objectNodeToMap(node));
|
||||||
|
}
|
||||||
|
if (node.isArray()) {
|
||||||
|
List<Object> values = new ArrayList<>();
|
||||||
|
for (JsonNode child : node) {
|
||||||
|
values.add(normalizeJsonNode(child));
|
||||||
|
}
|
||||||
|
return values;
|
||||||
|
}
|
||||||
|
if (node.isTextual()) {
|
||||||
|
return node.textValue();
|
||||||
|
}
|
||||||
|
if (node.isNumber()) {
|
||||||
|
return node.numberValue();
|
||||||
|
}
|
||||||
|
if (node.isBoolean()) {
|
||||||
|
return node.booleanValue();
|
||||||
|
}
|
||||||
|
if (node.isBinary()) {
|
||||||
|
return node.asText();
|
||||||
|
}
|
||||||
|
return node.asText();
|
||||||
|
}
|
||||||
|
|
||||||
|
private Map<String, Object> objectNodeToMap(JsonNode node) {
|
||||||
|
Map<String, Object> values = new LinkedHashMap<>();
|
||||||
|
node.fields().forEachRemaining(entry -> values.put(entry.getKey(), entry.getValue()));
|
||||||
|
return values;
|
||||||
|
}
|
||||||
|
|
||||||
|
private String describePayloadTypes(Map<String, Object> payload) {
|
||||||
|
if (payload.isEmpty()) {
|
||||||
|
return "{}";
|
||||||
|
}
|
||||||
|
return payload.entrySet()
|
||||||
|
.stream()
|
||||||
|
.map(entry -> entry.getKey() + "=" + (entry.getValue() == null ? "null" : entry.getValue().getClass().getName()))
|
||||||
|
.collect(Collectors.joining(", ", "{", "}"));
|
||||||
|
}
|
||||||
|
|
||||||
|
private String normalizeRequired(String value, String fieldName) {
|
||||||
|
String normalized = normalizeNullable(value);
|
||||||
|
if (normalized == null) {
|
||||||
|
throw new IllegalArgumentException(fieldName + " must not be blank");
|
||||||
|
}
|
||||||
|
return normalized;
|
||||||
|
}
|
||||||
|
|
||||||
|
private String normalizeNullable(String value) {
|
||||||
|
if (value == null) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
String trimmed = value.trim();
|
||||||
|
return trimmed.isEmpty() ? null : trimmed;
|
||||||
|
}
|
||||||
|
|
||||||
|
private record SourceMasterEntityStageRow(
|
||||||
|
UUID id,
|
||||||
|
String entityType,
|
||||||
|
String sourceEntityId,
|
||||||
|
String sourceExternalKey,
|
||||||
|
String displayName,
|
||||||
|
Boolean active,
|
||||||
|
OffsetDateTime validFrom,
|
||||||
|
OffsetDateTime validTo,
|
||||||
|
OffsetDateTime sourceUpdatedAt,
|
||||||
|
String payloadJson
|
||||||
|
) {
|
||||||
|
}
|
||||||
|
|
||||||
|
private record SourceMasterRelationStageRow(
|
||||||
|
UUID id,
|
||||||
|
String relationKey,
|
||||||
|
String relationType,
|
||||||
|
String fromEntityType,
|
||||||
|
String fromSourceEntityId,
|
||||||
|
String toEntityType,
|
||||||
|
String toSourceEntityId,
|
||||||
|
OffsetDateTime validFrom,
|
||||||
|
OffsetDateTime validTo,
|
||||||
|
OffsetDateTime sourceUpdatedAt,
|
||||||
|
String payloadJson
|
||||||
|
) {
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,710 @@
|
||||||
|
package at.procon.eventhub.persistence;
|
||||||
|
|
||||||
|
import at.procon.eventhub.dto.VehicleRefDto;
|
||||||
|
import at.procon.eventhub.dto.VehicleRegistrationRefDto;
|
||||||
|
import com.fasterxml.jackson.core.JsonProcessingException;
|
||||||
|
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||||
|
import java.time.OffsetDateTime;
|
||||||
|
import java.util.LinkedHashMap;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.Map;
|
||||||
|
import java.util.UUID;
|
||||||
|
import org.springframework.jdbc.core.JdbcTemplate;
|
||||||
|
import org.springframework.stereotype.Repository;
|
||||||
|
import org.springframework.transaction.annotation.Transactional;
|
||||||
|
|
||||||
|
@Repository
|
||||||
|
public class VehicleIdentityRepository {
|
||||||
|
|
||||||
|
private final JdbcTemplate jdbcTemplate;
|
||||||
|
private final ObjectMapper objectMapper;
|
||||||
|
|
||||||
|
public VehicleIdentityRepository(JdbcTemplate jdbcTemplate, ObjectMapper objectMapper) {
|
||||||
|
this.jdbcTemplate = jdbcTemplate;
|
||||||
|
this.objectMapper = objectMapper;
|
||||||
|
}
|
||||||
|
|
||||||
|
public ResolvedVehicleReferenceResolution resolveOrCreateVehicleReference(
|
||||||
|
String tenantKey,
|
||||||
|
int eventSourceId,
|
||||||
|
VehicleRefDto vehicleRef,
|
||||||
|
OffsetDateTime occurredAt
|
||||||
|
) {
|
||||||
|
if (vehicleRef == null || !vehicleRef.hasAnyReference()) {
|
||||||
|
return ResolvedVehicleReferenceResolution.empty();
|
||||||
|
}
|
||||||
|
|
||||||
|
String normalizedTenantKey = normalizeRequired(tenantKey, "tenantKey");
|
||||||
|
String sourceVehicleEntityId = normalizeNullable(vehicleRef.sourceVehicleEntityId());
|
||||||
|
String vin = normalizeNullable(vehicleRef.vin());
|
||||||
|
String sourceRegistrationEntityId = normalizeNullable(vehicleRef.sourceRegistrationEntityId());
|
||||||
|
VehicleRegistrationRefDto registration = vehicleRef.vehicleRegistration();
|
||||||
|
String registrationNation = registration == null ? null : normalizeNullable(registration.nation());
|
||||||
|
String registrationNumber = registration == null ? null : normalizeNullable(registration.number());
|
||||||
|
|
||||||
|
UUID registrationId = resolveRegistrationId(
|
||||||
|
normalizedTenantKey,
|
||||||
|
eventSourceId,
|
||||||
|
sourceRegistrationEntityId,
|
||||||
|
registrationNation,
|
||||||
|
registrationNumber,
|
||||||
|
occurredAt
|
||||||
|
);
|
||||||
|
if (registrationId == null && (sourceRegistrationEntityId != null || registrationNumber != null)) {
|
||||||
|
registrationId = createRegistration(
|
||||||
|
normalizedTenantKey,
|
||||||
|
eventSourceId,
|
||||||
|
sourceRegistrationEntityId,
|
||||||
|
registrationNation,
|
||||||
|
registrationNumber,
|
||||||
|
null,
|
||||||
|
Map.of("source", "event")
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
UUID vehicleId = resolveVehicleId(
|
||||||
|
normalizedTenantKey,
|
||||||
|
eventSourceId,
|
||||||
|
sourceVehicleEntityId,
|
||||||
|
vin
|
||||||
|
);
|
||||||
|
AssignedVehicleReference assignedVehicle = null;
|
||||||
|
if (vehicleId == null && registrationId != null) {
|
||||||
|
assignedVehicle = resolveAssignedVehicleReference(registrationId, occurredAt);
|
||||||
|
vehicleId = assignedVehicle == null ? null : assignedVehicle.vehicleId();
|
||||||
|
}
|
||||||
|
if (vehicleId == null && (sourceVehicleEntityId != null || vin != null)) {
|
||||||
|
vehicleId = createVehicle(
|
||||||
|
normalizedTenantKey,
|
||||||
|
eventSourceId,
|
||||||
|
sourceVehicleEntityId,
|
||||||
|
vin
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (vehicleId != null) {
|
||||||
|
touchVehicle(vehicleId, sourceVehicleEntityId, vin);
|
||||||
|
}
|
||||||
|
if (registrationId != null) {
|
||||||
|
touchRegistration(registrationId, sourceRegistrationEntityId, registrationNation, registrationNumber);
|
||||||
|
}
|
||||||
|
return new ResolvedVehicleReferenceResolution(
|
||||||
|
new ResolvedVehicleReference(vehicleId, registrationId),
|
||||||
|
assignedVehicle != null,
|
||||||
|
assignedVehicle == null ? null : assignedVehicle.validFrom(),
|
||||||
|
assignedVehicle == null ? null : assignedVehicle.validTo()
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Transactional
|
||||||
|
public int reconcileFromMasterData(String tenantKey, int eventSourceId) {
|
||||||
|
String normalizedTenantKey = normalizeRequired(tenantKey, "tenantKey");
|
||||||
|
int updates = reconcileVehiclesFromMasterData(normalizedTenantKey, eventSourceId);
|
||||||
|
updates += reconcileRegistrationsFromMasterData(normalizedTenantKey, eventSourceId);
|
||||||
|
updates += projectVehicleRegistrationAssignments(normalizedTenantKey, eventSourceId);
|
||||||
|
|
||||||
|
return updates;
|
||||||
|
}
|
||||||
|
|
||||||
|
private int reconcileVehiclesFromMasterData(String tenantKey, int eventSourceId) {
|
||||||
|
Long count = jdbcTemplate.queryForObject(
|
||||||
|
compatibleSourcesCte() + """
|
||||||
|
, master_vehicles as (
|
||||||
|
select distinct on (
|
||||||
|
event_source_id,
|
||||||
|
nullif(trim(source_entity_id), ''),
|
||||||
|
nullif(trim(source_external_key), '')
|
||||||
|
)
|
||||||
|
event_source_id,
|
||||||
|
nullif(trim(source_entity_id), '') as source_vehicle_entity_id,
|
||||||
|
nullif(trim(source_external_key), '') as vin
|
||||||
|
from eventhub.source_master_entity
|
||||||
|
where tenant_key = ?
|
||||||
|
and event_source_id in (select id from compatible_sources)
|
||||||
|
and entity_type = 'VEHICLE'
|
||||||
|
and (
|
||||||
|
nullif(trim(source_entity_id), '') is not null
|
||||||
|
or nullif(trim(source_external_key), '') is not null
|
||||||
|
)
|
||||||
|
order by event_source_id,
|
||||||
|
nullif(trim(source_entity_id), ''),
|
||||||
|
nullif(trim(source_external_key), ''),
|
||||||
|
updated_at desc
|
||||||
|
),
|
||||||
|
updated_by_source as (
|
||||||
|
update eventhub.vehicle vehicle
|
||||||
|
set source_vehicle_entity_id = coalesce(vehicle.source_vehicle_entity_id, master.source_vehicle_entity_id),
|
||||||
|
vin = coalesce(vehicle.vin, master.vin),
|
||||||
|
updated_at = now()
|
||||||
|
from master_vehicles master
|
||||||
|
where vehicle.tenant_key = ?
|
||||||
|
and vehicle.event_source_id in (select id from compatible_sources)
|
||||||
|
and master.source_vehicle_entity_id is not null
|
||||||
|
and vehicle.source_vehicle_entity_id = master.source_vehicle_entity_id
|
||||||
|
returning vehicle.id
|
||||||
|
),
|
||||||
|
updated_by_vin as (
|
||||||
|
update eventhub.vehicle vehicle
|
||||||
|
set source_vehicle_entity_id = coalesce(vehicle.source_vehicle_entity_id, master.source_vehicle_entity_id),
|
||||||
|
vin = coalesce(vehicle.vin, master.vin),
|
||||||
|
updated_at = now()
|
||||||
|
from master_vehicles master
|
||||||
|
where vehicle.tenant_key = ?
|
||||||
|
and vehicle.event_source_id in (select id from compatible_sources)
|
||||||
|
and master.vin is not null
|
||||||
|
and vehicle.vin = master.vin
|
||||||
|
returning vehicle.id
|
||||||
|
),
|
||||||
|
inserted as (
|
||||||
|
insert into eventhub.vehicle(id, tenant_key, event_source_id, source_vehicle_entity_id, vin, updated_at)
|
||||||
|
select gen_random_uuid(), ?, master.event_source_id, master.source_vehicle_entity_id, master.vin, now()
|
||||||
|
from master_vehicles master
|
||||||
|
where not exists (
|
||||||
|
select 1
|
||||||
|
from eventhub.vehicle existing
|
||||||
|
where existing.tenant_key = ?
|
||||||
|
and existing.event_source_id in (select id from compatible_sources)
|
||||||
|
and (
|
||||||
|
(master.source_vehicle_entity_id is not null and existing.source_vehicle_entity_id = master.source_vehicle_entity_id)
|
||||||
|
or (master.vin is not null and existing.vin = master.vin)
|
||||||
|
)
|
||||||
|
)
|
||||||
|
returning id
|
||||||
|
)
|
||||||
|
select (select count(*) from updated_by_source)
|
||||||
|
+ (select count(*) from updated_by_vin)
|
||||||
|
+ (select count(*) from inserted)
|
||||||
|
""",
|
||||||
|
Long.class,
|
||||||
|
eventSourceId,
|
||||||
|
tenantKey,
|
||||||
|
tenantKey,
|
||||||
|
tenantKey,
|
||||||
|
tenantKey,
|
||||||
|
tenantKey
|
||||||
|
);
|
||||||
|
return count == null ? 0 : Math.toIntExact(count);
|
||||||
|
}
|
||||||
|
|
||||||
|
private int reconcileRegistrationsFromMasterData(String tenantKey, int eventSourceId) {
|
||||||
|
Long count = jdbcTemplate.queryForObject(
|
||||||
|
compatibleSourcesCte() + """
|
||||||
|
, master_registrations as (
|
||||||
|
select distinct on (
|
||||||
|
event_source_id,
|
||||||
|
nullif(trim(source_entity_id), ''),
|
||||||
|
coalesce(
|
||||||
|
nullif(trim(payload ->> 'registration_nation'), ''),
|
||||||
|
nullif(split_part(source_external_key, ':', 1), '')
|
||||||
|
),
|
||||||
|
coalesce(
|
||||||
|
nullif(trim(payload ->> 'registration_number'), ''),
|
||||||
|
nullif(substring(source_external_key from position(':' in source_external_key) + 1), '')
|
||||||
|
)
|
||||||
|
)
|
||||||
|
event_source_id,
|
||||||
|
nullif(trim(source_entity_id), '') as source_registration_entity_id,
|
||||||
|
coalesce(
|
||||||
|
nullif(trim(payload ->> 'registration_nation'), ''),
|
||||||
|
nullif(split_part(source_external_key, ':', 1), '')
|
||||||
|
) as nation,
|
||||||
|
coalesce(
|
||||||
|
nullif(trim(payload ->> 'registration_number'), ''),
|
||||||
|
nullif(substring(source_external_key from position(':' in source_external_key) + 1), '')
|
||||||
|
) as registration_number,
|
||||||
|
source_updated_at
|
||||||
|
from eventhub.source_master_entity
|
||||||
|
where tenant_key = ?
|
||||||
|
and event_source_id in (select id from compatible_sources)
|
||||||
|
and entity_type = 'VEHICLE_REGISTRATION'
|
||||||
|
and (
|
||||||
|
nullif(trim(payload ->> 'registration_nation'), '') is not null
|
||||||
|
or source_external_key like '%:%'
|
||||||
|
)
|
||||||
|
and (
|
||||||
|
nullif(trim(payload ->> 'registration_number'), '') is not null
|
||||||
|
or source_external_key like '%:%'
|
||||||
|
)
|
||||||
|
order by event_source_id,
|
||||||
|
nullif(trim(source_entity_id), ''),
|
||||||
|
coalesce(
|
||||||
|
nullif(trim(payload ->> 'registration_nation'), ''),
|
||||||
|
nullif(split_part(source_external_key, ':', 1), '')
|
||||||
|
),
|
||||||
|
coalesce(
|
||||||
|
nullif(trim(payload ->> 'registration_number'), ''),
|
||||||
|
nullif(substring(source_external_key from position(':' in source_external_key) + 1), '')
|
||||||
|
),
|
||||||
|
updated_at desc
|
||||||
|
),
|
||||||
|
updated_by_source as (
|
||||||
|
update eventhub.vehicle_registration registration
|
||||||
|
set source_registration_entity_id = coalesce(registration.source_registration_entity_id, master.source_registration_entity_id),
|
||||||
|
nation = coalesce(master.nation, registration.nation),
|
||||||
|
registration_number = coalesce(master.registration_number, registration.registration_number),
|
||||||
|
source_updated_at = master.source_updated_at,
|
||||||
|
updated_at = now()
|
||||||
|
from master_registrations master
|
||||||
|
where registration.tenant_key = ?
|
||||||
|
and registration.event_source_id in (select id from compatible_sources)
|
||||||
|
and master.source_registration_entity_id is not null
|
||||||
|
and registration.source_registration_entity_id = master.source_registration_entity_id
|
||||||
|
returning registration.id
|
||||||
|
),
|
||||||
|
updated_by_plate as (
|
||||||
|
update eventhub.vehicle_registration registration
|
||||||
|
set source_registration_entity_id = coalesce(registration.source_registration_entity_id, master.source_registration_entity_id),
|
||||||
|
nation = coalesce(master.nation, registration.nation),
|
||||||
|
registration_number = coalesce(master.registration_number, registration.registration_number),
|
||||||
|
source_updated_at = master.source_updated_at,
|
||||||
|
updated_at = now()
|
||||||
|
from master_registrations master
|
||||||
|
where registration.tenant_key = ?
|
||||||
|
and registration.event_source_id in (select id from compatible_sources)
|
||||||
|
and registration.nation = master.nation
|
||||||
|
and registration.registration_number = master.registration_number
|
||||||
|
returning registration.id
|
||||||
|
),
|
||||||
|
inserted as (
|
||||||
|
insert into eventhub.vehicle_registration(
|
||||||
|
id, tenant_key, event_source_id, source_registration_entity_id, nation, registration_number,
|
||||||
|
source_updated_at, payload, updated_at
|
||||||
|
)
|
||||||
|
select gen_random_uuid(),
|
||||||
|
?,
|
||||||
|
master.event_source_id,
|
||||||
|
master.source_registration_entity_id,
|
||||||
|
master.nation,
|
||||||
|
master.registration_number,
|
||||||
|
master.source_updated_at,
|
||||||
|
jsonb_build_object('source', 'master-data'),
|
||||||
|
now()
|
||||||
|
from master_registrations master
|
||||||
|
where master.nation is not null
|
||||||
|
and master.registration_number is not null
|
||||||
|
and not exists (
|
||||||
|
select 1
|
||||||
|
from eventhub.vehicle_registration existing
|
||||||
|
where existing.tenant_key = ?
|
||||||
|
and existing.event_source_id in (select id from compatible_sources)
|
||||||
|
and (
|
||||||
|
(master.source_registration_entity_id is not null
|
||||||
|
and existing.source_registration_entity_id = master.source_registration_entity_id)
|
||||||
|
or (
|
||||||
|
existing.nation = master.nation
|
||||||
|
and existing.registration_number = master.registration_number
|
||||||
|
)
|
||||||
|
)
|
||||||
|
)
|
||||||
|
returning id
|
||||||
|
)
|
||||||
|
select (select count(*) from updated_by_source)
|
||||||
|
+ (select count(*) from updated_by_plate)
|
||||||
|
+ (select count(*) from inserted)
|
||||||
|
""",
|
||||||
|
Long.class,
|
||||||
|
eventSourceId,
|
||||||
|
tenantKey,
|
||||||
|
tenantKey,
|
||||||
|
tenantKey,
|
||||||
|
tenantKey,
|
||||||
|
tenantKey
|
||||||
|
);
|
||||||
|
return count == null ? 0 : Math.toIntExact(count);
|
||||||
|
}
|
||||||
|
|
||||||
|
private UUID resolveVehicleId(
|
||||||
|
String tenantKey,
|
||||||
|
int eventSourceId,
|
||||||
|
String sourceVehicleEntityId,
|
||||||
|
String vin
|
||||||
|
) {
|
||||||
|
UUID vehicleId = findVehicleBySourceVehicleEntityId(tenantKey, eventSourceId, sourceVehicleEntityId);
|
||||||
|
if (vehicleId == null) {
|
||||||
|
vehicleId = findVehicleByVin(tenantKey, eventSourceId, vin);
|
||||||
|
}
|
||||||
|
return vehicleId;
|
||||||
|
}
|
||||||
|
|
||||||
|
private UUID resolveRegistrationId(
|
||||||
|
String tenantKey,
|
||||||
|
int eventSourceId,
|
||||||
|
String sourceRegistrationEntityId,
|
||||||
|
String nation,
|
||||||
|
String registrationNumber,
|
||||||
|
OffsetDateTime occurredAt
|
||||||
|
) {
|
||||||
|
UUID registrationId = findRegistrationBySourceRegistrationEntityId(tenantKey, eventSourceId, sourceRegistrationEntityId, occurredAt);
|
||||||
|
if (registrationId == null) {
|
||||||
|
registrationId = findRegistrationByPlate(tenantKey, eventSourceId, nation, registrationNumber, occurredAt);
|
||||||
|
}
|
||||||
|
return registrationId;
|
||||||
|
}
|
||||||
|
|
||||||
|
private UUID findVehicleBySourceVehicleEntityId(String tenantKey, int eventSourceId, String sourceVehicleEntityId) {
|
||||||
|
if (sourceVehicleEntityId == null) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
return jdbcTemplate.query(
|
||||||
|
compatibleSourcesCte() + """
|
||||||
|
select v.id
|
||||||
|
from eventhub.vehicle v
|
||||||
|
where v.tenant_key = ?
|
||||||
|
and v.event_source_id in (select id from compatible_sources)
|
||||||
|
and v.source_vehicle_entity_id = ?
|
||||||
|
order by v.updated_at desc
|
||||||
|
limit 1
|
||||||
|
""",
|
||||||
|
rs -> rs.next() ? (UUID) rs.getObject("id") : null,
|
||||||
|
eventSourceId,
|
||||||
|
tenantKey,
|
||||||
|
sourceVehicleEntityId
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
private UUID findVehicleByVin(String tenantKey, int eventSourceId, String vin) {
|
||||||
|
if (vin == null) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
return jdbcTemplate.query(
|
||||||
|
compatibleSourcesCte() + """
|
||||||
|
select v.id
|
||||||
|
from eventhub.vehicle v
|
||||||
|
where v.tenant_key = ?
|
||||||
|
and v.event_source_id in (select id from compatible_sources)
|
||||||
|
and v.vin = ?
|
||||||
|
order by v.updated_at desc
|
||||||
|
limit 1
|
||||||
|
""",
|
||||||
|
rs -> rs.next() ? (UUID) rs.getObject("id") : null,
|
||||||
|
eventSourceId,
|
||||||
|
tenantKey,
|
||||||
|
vin
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
private UUID findRegistrationBySourceRegistrationEntityId(
|
||||||
|
String tenantKey,
|
||||||
|
int eventSourceId,
|
||||||
|
String sourceRegistrationEntityId,
|
||||||
|
OffsetDateTime occurredAt
|
||||||
|
) {
|
||||||
|
if (sourceRegistrationEntityId == null) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
return jdbcTemplate.query(
|
||||||
|
compatibleSourcesCte() + """
|
||||||
|
select r.id
|
||||||
|
from eventhub.vehicle_registration r
|
||||||
|
where r.tenant_key = ?
|
||||||
|
and r.event_source_id in (select id from compatible_sources)
|
||||||
|
and r.source_registration_entity_id = ?
|
||||||
|
order by r.updated_at desc
|
||||||
|
limit 1
|
||||||
|
""",
|
||||||
|
rs -> rs.next() ? (UUID) rs.getObject("id") : null,
|
||||||
|
eventSourceId,
|
||||||
|
tenantKey,
|
||||||
|
sourceRegistrationEntityId
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
private UUID findRegistrationByPlate(
|
||||||
|
String tenantKey,
|
||||||
|
int eventSourceId,
|
||||||
|
String nation,
|
||||||
|
String registrationNumber,
|
||||||
|
OffsetDateTime occurredAt
|
||||||
|
) {
|
||||||
|
if (nation == null || registrationNumber == null) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
return jdbcTemplate.query(
|
||||||
|
compatibleSourcesCte() + """
|
||||||
|
select r.id
|
||||||
|
from eventhub.vehicle_registration r
|
||||||
|
where r.tenant_key = ?
|
||||||
|
and r.event_source_id in (select id from compatible_sources)
|
||||||
|
and r.nation = ?
|
||||||
|
and r.registration_number = ?
|
||||||
|
order by r.updated_at desc
|
||||||
|
limit 1
|
||||||
|
""",
|
||||||
|
rs -> rs.next() ? (UUID) rs.getObject("id") : null,
|
||||||
|
eventSourceId,
|
||||||
|
tenantKey,
|
||||||
|
nation,
|
||||||
|
registrationNumber
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
private AssignedVehicleReference resolveAssignedVehicleReference(UUID registrationId, OffsetDateTime occurredAt) {
|
||||||
|
return jdbcTemplate.query(
|
||||||
|
"""
|
||||||
|
select vehicle_id, valid_from, valid_to
|
||||||
|
from eventhub.vehicle_registration_assignment
|
||||||
|
where vehicle_registration_id = ?
|
||||||
|
and (? is null or valid_from is null or valid_from <= ?)
|
||||||
|
and (? is null or valid_to is null or ? < valid_to)
|
||||||
|
order by valid_from desc nulls last, updated_at desc
|
||||||
|
limit 1
|
||||||
|
""",
|
||||||
|
rs -> rs.next()
|
||||||
|
? new AssignedVehicleReference(
|
||||||
|
(UUID) rs.getObject("vehicle_id"),
|
||||||
|
rs.getObject("valid_from", OffsetDateTime.class),
|
||||||
|
rs.getObject("valid_to", OffsetDateTime.class)
|
||||||
|
)
|
||||||
|
: null,
|
||||||
|
registrationId,
|
||||||
|
occurredAt,
|
||||||
|
occurredAt,
|
||||||
|
occurredAt,
|
||||||
|
occurredAt
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
private UUID createVehicle(
|
||||||
|
String tenantKey,
|
||||||
|
int eventSourceId,
|
||||||
|
String sourceVehicleEntityId,
|
||||||
|
String vin
|
||||||
|
) {
|
||||||
|
UUID vehicleId = UUID.randomUUID();
|
||||||
|
jdbcTemplate.update(
|
||||||
|
"""
|
||||||
|
insert into eventhub.vehicle(id, tenant_key, event_source_id, source_vehicle_entity_id, vin, updated_at)
|
||||||
|
values (?, ?, ?, ?, ?, now())
|
||||||
|
""",
|
||||||
|
vehicleId,
|
||||||
|
tenantKey,
|
||||||
|
eventSourceId,
|
||||||
|
sourceVehicleEntityId,
|
||||||
|
vin
|
||||||
|
);
|
||||||
|
return vehicleId;
|
||||||
|
}
|
||||||
|
|
||||||
|
private UUID createRegistration(
|
||||||
|
String tenantKey,
|
||||||
|
int eventSourceId,
|
||||||
|
String sourceRegistrationEntityId,
|
||||||
|
String nation,
|
||||||
|
String registrationNumber,
|
||||||
|
OffsetDateTime sourceUpdatedAt,
|
||||||
|
Map<String, Object> payload
|
||||||
|
) {
|
||||||
|
if (nation == null || registrationNumber == null) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
UUID registrationId = UUID.randomUUID();
|
||||||
|
jdbcTemplate.update(
|
||||||
|
"""
|
||||||
|
insert into eventhub.vehicle_registration(
|
||||||
|
id, tenant_key, event_source_id, source_registration_entity_id, nation, registration_number,
|
||||||
|
source_updated_at, payload, updated_at
|
||||||
|
) values (?, ?, ?, ?, ?, ?, ?, ?::jsonb, now())
|
||||||
|
""",
|
||||||
|
registrationId,
|
||||||
|
tenantKey,
|
||||||
|
eventSourceId,
|
||||||
|
sourceRegistrationEntityId,
|
||||||
|
nation,
|
||||||
|
registrationNumber,
|
||||||
|
sourceUpdatedAt,
|
||||||
|
toJson(payload)
|
||||||
|
);
|
||||||
|
return registrationId;
|
||||||
|
}
|
||||||
|
|
||||||
|
private void touchVehicle(UUID vehicleId, String sourceVehicleEntityId, String vin) {
|
||||||
|
if (sourceVehicleEntityId == null && vin == null) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
jdbcTemplate.update(
|
||||||
|
"""
|
||||||
|
update eventhub.vehicle
|
||||||
|
set source_vehicle_entity_id = coalesce(source_vehicle_entity_id, cast(? as text)),
|
||||||
|
vin = coalesce(vin, cast(? as text)),
|
||||||
|
updated_at = now()
|
||||||
|
where id = ?
|
||||||
|
and (
|
||||||
|
(source_vehicle_entity_id is null and cast(? as text) is not null)
|
||||||
|
or (vin is null and cast(? as text) is not null)
|
||||||
|
)
|
||||||
|
""",
|
||||||
|
sourceVehicleEntityId,
|
||||||
|
vin,
|
||||||
|
vehicleId,
|
||||||
|
sourceVehicleEntityId,
|
||||||
|
vin
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
private int projectVehicleRegistrationAssignments(String tenantKey, int eventSourceId) {
|
||||||
|
return jdbcTemplate.update(
|
||||||
|
compatibleSourcesCte() + """
|
||||||
|
insert into eventhub.vehicle_registration_assignment(
|
||||||
|
id, tenant_key, event_source_id, vehicle_registration_id, vehicle_id,
|
||||||
|
valid_from, valid_to, source_updated_at, payload, updated_at
|
||||||
|
)
|
||||||
|
select gen_random_uuid(),
|
||||||
|
relation.tenant_key,
|
||||||
|
relation.event_source_id,
|
||||||
|
registration.id,
|
||||||
|
vehicle.id,
|
||||||
|
relation.valid_from,
|
||||||
|
relation.valid_to,
|
||||||
|
relation.source_updated_at,
|
||||||
|
jsonb_build_object(
|
||||||
|
'source', 'master-data',
|
||||||
|
'sourceRelationId', relation.id,
|
||||||
|
'relationKey', relation.relation_key
|
||||||
|
),
|
||||||
|
now()
|
||||||
|
from eventhub.source_master_relation relation
|
||||||
|
join eventhub.vehicle_registration registration on registration.tenant_key = relation.tenant_key
|
||||||
|
and registration.event_source_id = relation.event_source_id
|
||||||
|
and registration.source_registration_entity_id = relation.from_source_entity_id
|
||||||
|
join eventhub.vehicle vehicle on vehicle.tenant_key = relation.tenant_key
|
||||||
|
and vehicle.event_source_id = relation.event_source_id
|
||||||
|
and vehicle.source_vehicle_entity_id = relation.to_source_entity_id
|
||||||
|
where relation.tenant_key = ?
|
||||||
|
and relation.event_source_id in (select id from compatible_sources)
|
||||||
|
and relation.relation_type = 'VEHICLE_REGISTRATION_VEHICLE'
|
||||||
|
and relation.from_entity_type = 'VEHICLE_REGISTRATION'
|
||||||
|
and relation.to_entity_type = 'VEHICLE'
|
||||||
|
and not exists (
|
||||||
|
select 1
|
||||||
|
from eventhub.vehicle_registration_assignment existing
|
||||||
|
where existing.vehicle_registration_id = registration.id
|
||||||
|
and existing.vehicle_id = vehicle.id
|
||||||
|
and existing.valid_from is not distinct from relation.valid_from
|
||||||
|
and existing.valid_to is not distinct from relation.valid_to
|
||||||
|
)
|
||||||
|
""",
|
||||||
|
eventSourceId,
|
||||||
|
tenantKey
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
private void touchRegistration(UUID registrationId, String sourceRegistrationEntityId, String nation, String registrationNumber) {
|
||||||
|
if (sourceRegistrationEntityId == null && nation == null && registrationNumber == null) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
jdbcTemplate.update(
|
||||||
|
"""
|
||||||
|
update eventhub.vehicle_registration
|
||||||
|
set source_registration_entity_id = coalesce(source_registration_entity_id, cast(? as text)),
|
||||||
|
nation = coalesce(cast(? as text), nation),
|
||||||
|
registration_number = coalesce(cast(? as text), registration_number),
|
||||||
|
updated_at = now()
|
||||||
|
where id = ?
|
||||||
|
and (
|
||||||
|
(source_registration_entity_id is null and cast(? as text) is not null)
|
||||||
|
or (nation is null and cast(? as text) is not null)
|
||||||
|
or (registration_number is null and cast(? as text) is not null)
|
||||||
|
)
|
||||||
|
""",
|
||||||
|
sourceRegistrationEntityId,
|
||||||
|
nation,
|
||||||
|
registrationNumber,
|
||||||
|
registrationId,
|
||||||
|
sourceRegistrationEntityId,
|
||||||
|
nation,
|
||||||
|
registrationNumber
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
private void updateRegistrationFromMasterData(
|
||||||
|
UUID registrationId,
|
||||||
|
String sourceRegistrationEntityId,
|
||||||
|
String nation,
|
||||||
|
String registrationNumber,
|
||||||
|
OffsetDateTime sourceUpdatedAt
|
||||||
|
) {
|
||||||
|
jdbcTemplate.update(
|
||||||
|
"""
|
||||||
|
update eventhub.vehicle_registration
|
||||||
|
set source_registration_entity_id = coalesce(source_registration_entity_id, ?),
|
||||||
|
nation = coalesce(?, nation),
|
||||||
|
registration_number = coalesce(?, registration_number),
|
||||||
|
source_updated_at = ?,
|
||||||
|
updated_at = now()
|
||||||
|
where id = ?
|
||||||
|
""",
|
||||||
|
sourceRegistrationEntityId,
|
||||||
|
nation,
|
||||||
|
registrationNumber,
|
||||||
|
sourceUpdatedAt,
|
||||||
|
registrationId
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
private String compatibleSourcesCte() {
|
||||||
|
return """
|
||||||
|
with source_context as (
|
||||||
|
select tenant_key, provider_key, source_instance_key, coalesce(tenant_provider_setting_key, '') as tenant_provider_setting_key
|
||||||
|
from eventhub.event_source
|
||||||
|
where id = ?
|
||||||
|
),
|
||||||
|
compatible_sources as (
|
||||||
|
select es.id
|
||||||
|
from eventhub.event_source es
|
||||||
|
join source_context ctx on ctx.tenant_key = es.tenant_key
|
||||||
|
and ctx.provider_key = es.provider_key
|
||||||
|
and ctx.source_instance_key = es.source_instance_key
|
||||||
|
and ctx.tenant_provider_setting_key = coalesce(es.tenant_provider_setting_key, '')
|
||||||
|
)
|
||||||
|
""";
|
||||||
|
}
|
||||||
|
|
||||||
|
private String toJson(Map<String, Object> value) {
|
||||||
|
try {
|
||||||
|
return objectMapper.writeValueAsString(value == null ? Map.of() : value);
|
||||||
|
} catch (JsonProcessingException e) {
|
||||||
|
throw new IllegalArgumentException("Cannot serialize vehicle identity payload", e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private String normalizeRequired(String value, String fieldName) {
|
||||||
|
String normalized = normalizeNullable(value);
|
||||||
|
if (normalized == null) {
|
||||||
|
throw new IllegalArgumentException(fieldName + " must not be blank");
|
||||||
|
}
|
||||||
|
return normalized;
|
||||||
|
}
|
||||||
|
|
||||||
|
private String normalizeNullable(String value) {
|
||||||
|
if (value == null) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
String trimmed = value.trim();
|
||||||
|
return trimmed.isEmpty() ? null : trimmed;
|
||||||
|
}
|
||||||
|
|
||||||
|
public record ResolvedVehicleReference(UUID vehicleId, UUID vehicleRegistrationId) {
|
||||||
|
public static ResolvedVehicleReference empty() {
|
||||||
|
return new ResolvedVehicleReference(null, null);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public record ResolvedVehicleReferenceResolution(
|
||||||
|
ResolvedVehicleReference reference,
|
||||||
|
boolean resolvedFromAssignment,
|
||||||
|
OffsetDateTime assignmentValidFrom,
|
||||||
|
OffsetDateTime assignmentValidTo
|
||||||
|
) {
|
||||||
|
public static ResolvedVehicleReferenceResolution empty() {
|
||||||
|
return new ResolvedVehicleReferenceResolution(ResolvedVehicleReference.empty(), false, null, null);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private record AssignedVehicleReference(
|
||||||
|
UUID vehicleId,
|
||||||
|
OffsetDateTime validFrom,
|
||||||
|
OffsetDateTime validTo
|
||||||
|
) {
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
|
@ -68,13 +68,20 @@ public class EventAcquisitionRecordKeyService {
|
||||||
if (event.vehicleRef() == null) {
|
if (event.vehicleRef() == null) {
|
||||||
return "";
|
return "";
|
||||||
}
|
}
|
||||||
if (event.vehicleRef().vehicleRegistration() != null && event.vehicleRef().vehicleRegistration().hasValue()) {
|
boolean hasVin = event.vehicleRef().vin() != null && !event.vehicleRef().vin().isBlank();
|
||||||
return "VRN:" + event.vehicleRef().vehicleRegistration().stableKey();
|
boolean hasRegistration = event.vehicleRef().vehicleRegistration() != null
|
||||||
}
|
&& event.vehicleRef().vehicleRegistration().hasValue();
|
||||||
if (event.vehicleRef().vin() != null && !event.vehicleRef().vin().isBlank()) {
|
|
||||||
|
if (hasVin) {
|
||||||
return "VIN:" + event.vehicleRef().vin();
|
return "VIN:" + event.vehicleRef().vin();
|
||||||
}
|
}
|
||||||
return "SOURCE_VEHICLE:" + nullToEmpty(event.vehicleRef().sourceEntityId());
|
if (hasRegistration) {
|
||||||
|
return "VRN:" + event.vehicleRef().vehicleRegistration().stableKey();
|
||||||
|
}
|
||||||
|
if (event.vehicleRef().sourceVehicleEntityId() != null) {
|
||||||
|
return "SOURCE_VEHICLE:" + event.vehicleRef().sourceVehicleEntityId();
|
||||||
|
}
|
||||||
|
return "SOURCE_REGISTRATION:" + nullToEmpty(event.vehicleRef().sourceRegistrationEntityId());
|
||||||
}
|
}
|
||||||
|
|
||||||
private String normalizeTime(OffsetDateTime value) {
|
private String normalizeTime(OffsetDateTime value) {
|
||||||
|
|
|
||||||
|
|
@ -2,6 +2,7 @@ package at.procon.eventhub.service;
|
||||||
|
|
||||||
import at.procon.eventhub.dto.CardSlot;
|
import at.procon.eventhub.dto.CardSlot;
|
||||||
import at.procon.eventhub.dto.CardStatus;
|
import at.procon.eventhub.dto.CardStatus;
|
||||||
|
import at.procon.eventhub.dto.DriverCardRefDto;
|
||||||
import at.procon.eventhub.dto.DrivingStatus;
|
import at.procon.eventhub.dto.DrivingStatus;
|
||||||
import at.procon.eventhub.dto.EventDetailsDto;
|
import at.procon.eventhub.dto.EventDetailsDto;
|
||||||
import com.fasterxml.jackson.databind.JsonNode;
|
import com.fasterxml.jackson.databind.JsonNode;
|
||||||
|
|
@ -29,9 +30,17 @@ public class EventDetailsFactory {
|
||||||
}
|
}
|
||||||
|
|
||||||
public EventDetailsDto driverCard(CardSlot cardSlot, CardStatus cardStatus) {
|
public EventDetailsDto driverCard(CardSlot cardSlot, CardStatus cardStatus) {
|
||||||
|
return driverCard(cardSlot, cardStatus, null);
|
||||||
|
}
|
||||||
|
|
||||||
|
public EventDetailsDto driverCard(CardSlot cardSlot, CardStatus cardStatus, DriverCardRefDto driverCard) {
|
||||||
Map<String, Object> attributes = new LinkedHashMap<>();
|
Map<String, Object> attributes = new LinkedHashMap<>();
|
||||||
put(attributes, "cardSlot", cardSlot);
|
put(attributes, "cardSlot", cardSlot);
|
||||||
put(attributes, "cardStatus", cardStatus);
|
put(attributes, "cardStatus", cardStatus);
|
||||||
|
if (driverCard != null && driverCard.hasValue()) {
|
||||||
|
put(attributes, "cardNation", driverCard.nation());
|
||||||
|
put(attributes, "cardNumber", driverCard.number());
|
||||||
|
}
|
||||||
return new EventDetailsDto("DRIVER_CARD", objectMapper.valueToTree(attributes));
|
return new EventDetailsDto("DRIVER_CARD", objectMapper.valueToTree(attributes));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -41,6 +50,13 @@ public class EventDetailsFactory {
|
||||||
return new EventDetailsDto("POSITION", objectMapper.valueToTree(attributes));
|
return new EventDetailsDto("POSITION", objectMapper.valueToTree(attributes));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public EventDetailsDto place(String country, String region) {
|
||||||
|
Map<String, Object> attributes = new LinkedHashMap<>();
|
||||||
|
put(attributes, "country", country);
|
||||||
|
put(attributes, "region", region);
|
||||||
|
return new EventDetailsDto("PLACE", objectMapper.valueToTree(attributes));
|
||||||
|
}
|
||||||
|
|
||||||
public EventDetailsDto borderCrossing(String countryFrom, String countryTo) {
|
public EventDetailsDto borderCrossing(String countryFrom, String countryTo) {
|
||||||
Map<String, Object> attributes = new LinkedHashMap<>();
|
Map<String, Object> attributes = new LinkedHashMap<>();
|
||||||
put(attributes, "countryFrom", countryFrom);
|
put(attributes, "countryFrom", countryFrom);
|
||||||
|
|
@ -54,12 +70,21 @@ public class EventDetailsFactory {
|
||||||
return new EventDetailsDto("LOAD_UNLOAD", objectMapper.valueToTree(attributes));
|
return new EventDetailsDto("LOAD_UNLOAD", objectMapper.valueToTree(attributes));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public EventDetailsDto specificCondition() {
|
||||||
|
return new EventDetailsDto("SPECIFIC_CONDITION", objectMapper.createObjectNode());
|
||||||
|
}
|
||||||
|
|
||||||
public EventDetailsDto speeding(BigDecimal speedKmh, BigDecimal permittedSpeedKmh) {
|
public EventDetailsDto speeding(BigDecimal speedKmh, BigDecimal permittedSpeedKmh) {
|
||||||
|
return speeding(null, speedKmh, permittedSpeedKmh);
|
||||||
|
}
|
||||||
|
|
||||||
|
public EventDetailsDto speeding(BigDecimal avgSpeedKmh, BigDecimal maxSpeedKmh, BigDecimal permittedSpeedKmh) {
|
||||||
Map<String, Object> attributes = new LinkedHashMap<>();
|
Map<String, Object> attributes = new LinkedHashMap<>();
|
||||||
put(attributes, "speedKmh", speedKmh);
|
put(attributes, "avgSpeedKmh", avgSpeedKmh);
|
||||||
|
put(attributes, "maxSpeedKmh", maxSpeedKmh);
|
||||||
put(attributes, "permittedSpeedKmh", permittedSpeedKmh);
|
put(attributes, "permittedSpeedKmh", permittedSpeedKmh);
|
||||||
if (speedKmh != null && permittedSpeedKmh != null) {
|
if (maxSpeedKmh != null && permittedSpeedKmh != null) {
|
||||||
put(attributes, "overspeedKmh", speedKmh.subtract(permittedSpeedKmh));
|
put(attributes, "overspeedKmh", maxSpeedKmh.subtract(permittedSpeedKmh));
|
||||||
}
|
}
|
||||||
return new EventDetailsDto("SPEEDING", objectMapper.valueToTree(attributes));
|
return new EventDetailsDto("SPEEDING", objectMapper.valueToTree(attributes));
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -74,9 +74,18 @@ public class EventHubEventValidator {
|
||||||
if (event.eventDomain() == EventDomain.BORDER_CROSSING && !"BORDER_CROSSING".equals(detailType)) {
|
if (event.eventDomain() == EventDomain.BORDER_CROSSING && !"BORDER_CROSSING".equals(detailType)) {
|
||||||
throw new IllegalArgumentException("BORDER_CROSSING events must use eventDetails.type=BORDER_CROSSING");
|
throw new IllegalArgumentException("BORDER_CROSSING events must use eventDetails.type=BORDER_CROSSING");
|
||||||
}
|
}
|
||||||
|
if (event.eventDomain() == EventDomain.POSITION && !"POSITION".equals(detailType)) {
|
||||||
|
throw new IllegalArgumentException("POSITION events must use eventDetails.type=POSITION");
|
||||||
|
}
|
||||||
|
if (event.eventDomain() == EventDomain.PLACE && !"PLACE".equals(detailType)) {
|
||||||
|
throw new IllegalArgumentException("PLACE events must use eventDetails.type=PLACE");
|
||||||
|
}
|
||||||
if (event.eventDomain() == EventDomain.LOAD_UNLOAD && !"LOAD_UNLOAD".equals(detailType)) {
|
if (event.eventDomain() == EventDomain.LOAD_UNLOAD && !"LOAD_UNLOAD".equals(detailType)) {
|
||||||
throw new IllegalArgumentException("LOAD_UNLOAD events must use eventDetails.type=LOAD_UNLOAD");
|
throw new IllegalArgumentException("LOAD_UNLOAD events must use eventDetails.type=LOAD_UNLOAD");
|
||||||
}
|
}
|
||||||
|
if (event.eventDomain() == EventDomain.SPECIFIC_CONDITION && !"SPECIFIC_CONDITION".equals(detailType)) {
|
||||||
|
throw new IllegalArgumentException("SPECIFIC_CONDITION events must use eventDetails.type=SPECIFIC_CONDITION");
|
||||||
|
}
|
||||||
if (event.eventDomain() == EventDomain.SPEEDING && !"SPEEDING".equals(detailType)) {
|
if (event.eventDomain() == EventDomain.SPEEDING && !"SPEEDING".equals(detailType)) {
|
||||||
throw new IllegalArgumentException("SPEEDING events must use eventDetails.type=SPEEDING");
|
throw new IllegalArgumentException("SPEEDING events must use eventDetails.type=SPEEDING");
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -8,12 +8,15 @@ import at.procon.eventhub.dto.EventSourceDto;
|
||||||
import at.procon.eventhub.persistence.DataPackageRepository;
|
import at.procon.eventhub.persistence.DataPackageRepository;
|
||||||
import at.procon.eventhub.persistence.EventRepository;
|
import at.procon.eventhub.persistence.EventRepository;
|
||||||
import at.procon.eventhub.persistence.EventSourceRepository;
|
import at.procon.eventhub.persistence.EventSourceRepository;
|
||||||
|
import java.util.LinkedHashMap;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
|
import java.util.Map;
|
||||||
import java.util.UUID;
|
import java.util.UUID;
|
||||||
import org.slf4j.Logger;
|
import org.slf4j.Logger;
|
||||||
import org.slf4j.LoggerFactory;
|
import org.slf4j.LoggerFactory;
|
||||||
import org.springframework.stereotype.Service;
|
import org.springframework.stereotype.Service;
|
||||||
import org.springframework.transaction.annotation.Transactional;
|
import org.springframework.transaction.PlatformTransactionManager;
|
||||||
|
import org.springframework.transaction.support.TransactionTemplate;
|
||||||
|
|
||||||
@Service
|
@Service
|
||||||
public class EventHubIngestionService {
|
public class EventHubIngestionService {
|
||||||
|
|
@ -25,26 +28,29 @@ public class EventHubIngestionService {
|
||||||
private final EventRepository eventRepository;
|
private final EventRepository eventRepository;
|
||||||
private final EventHubEventValidator validator;
|
private final EventHubEventValidator validator;
|
||||||
private final EventHubEventSorter sorter;
|
private final EventHubEventSorter sorter;
|
||||||
|
private final TransactionTemplate transactionTemplate;
|
||||||
|
|
||||||
public EventHubIngestionService(
|
public EventHubIngestionService(
|
||||||
EventSourceRepository eventSourceRepository,
|
EventSourceRepository eventSourceRepository,
|
||||||
DataPackageRepository dataPackageRepository,
|
DataPackageRepository dataPackageRepository,
|
||||||
EventRepository eventRepository,
|
EventRepository eventRepository,
|
||||||
EventHubEventValidator validator,
|
EventHubEventValidator validator,
|
||||||
EventHubEventSorter sorter
|
EventHubEventSorter sorter,
|
||||||
|
PlatformTransactionManager transactionManager
|
||||||
) {
|
) {
|
||||||
this.eventSourceRepository = eventSourceRepository;
|
this.eventSourceRepository = eventSourceRepository;
|
||||||
this.dataPackageRepository = dataPackageRepository;
|
this.dataPackageRepository = dataPackageRepository;
|
||||||
this.eventRepository = eventRepository;
|
this.eventRepository = eventRepository;
|
||||||
this.validator = validator;
|
this.validator = validator;
|
||||||
this.sorter = sorter;
|
this.sorter = sorter;
|
||||||
|
this.transactionTemplate = new TransactionTemplate(transactionManager);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Transactional
|
|
||||||
public EventHubPackageResult ingest(EventHubEventBatchDto batch) {
|
public EventHubPackageResult ingest(EventHubEventBatchDto batch) {
|
||||||
if (batch == null || batch.events().isEmpty()) {
|
if (batch == null || batch.events().isEmpty()) {
|
||||||
return new EventHubPackageResult(null, batch == null ? null : batch.packageKey(), 0, 0);
|
return new EventHubPackageResult(null, batch == null ? null : batch.packageKey(), 0, 0);
|
||||||
}
|
}
|
||||||
|
long startedAtNanos = System.nanoTime();
|
||||||
|
|
||||||
EventHubPackageRequest packageInfo = batch.packageInfo();
|
EventHubPackageRequest packageInfo = batch.packageInfo();
|
||||||
if (packageInfo == null) {
|
if (packageInfo == null) {
|
||||||
|
|
@ -55,6 +61,7 @@ public class EventHubIngestionService {
|
||||||
}
|
}
|
||||||
|
|
||||||
EventSourceDto eventSource = packageInfo.eventSource();
|
EventSourceDto eventSource = packageInfo.eventSource();
|
||||||
|
String tenantKey = packageInfo.tenantKey();
|
||||||
int eventSourceId = eventSourceRepository.resolveSourceId(packageInfo.tenantKey(), eventSource);
|
int eventSourceId = eventSourceRepository.resolveSourceId(packageInfo.tenantKey(), eventSource);
|
||||||
List<EventHubEventDto> sortedEvents = sorter.sort(batch.events());
|
List<EventHubEventDto> sortedEvents = sorter.sort(batch.events());
|
||||||
sortedEvents.forEach(validator::validate);
|
sortedEvents.forEach(validator::validate);
|
||||||
|
|
@ -70,14 +77,37 @@ public class EventHubIngestionService {
|
||||||
);
|
);
|
||||||
|
|
||||||
try {
|
try {
|
||||||
int insertedCount = eventRepository.batchInsert(packageId, eventSourceId, sortedEvents);
|
Integer insertedCount = transactionTemplate.execute(status ->
|
||||||
|
eventRepository.batchInsert(packageId, tenantKey, eventSourceId, sortedEvents)
|
||||||
|
);
|
||||||
|
if (insertedCount == null) {
|
||||||
|
throw new IllegalStateException("EventHub batch insert returned no result for package " + batch.packageKey());
|
||||||
|
}
|
||||||
dataPackageRepository.markImported(packageId, insertedCount);
|
dataPackageRepository.markImported(packageId, insertedCount);
|
||||||
log.info("Imported EventHub acquisition package packageId={} packageKey={} source={} receivedCount={} insertedCount={}",
|
long elapsedMs = Math.max(1L, (System.nanoTime() - startedAtNanos) / 1_000_000L);
|
||||||
packageId, batch.packageKey(), eventSource.stableKey(), sortedEvents.size(), insertedCount);
|
long receivedPerSecond = Math.max(1L, Math.round(sortedEvents.size() * 1000.0 / elapsedMs));
|
||||||
|
log.info("Imported EventHub acquisition package packageId={} packageKey={} source={} receivedCount={} insertedCount={} elapsedMs={} receivedPerSecond={} byType={}",
|
||||||
|
packageId, batch.packageKey(), eventSource.stableKey(), sortedEvents.size(), insertedCount, elapsedMs, receivedPerSecond, eventTypeCounts(sortedEvents));
|
||||||
return new EventHubPackageResult(packageId, batch.packageKey(), sortedEvents.size(), insertedCount);
|
return new EventHubPackageResult(packageId, batch.packageKey(), sortedEvents.size(), insertedCount);
|
||||||
} catch (RuntimeException ex) {
|
} catch (RuntimeException ex) {
|
||||||
dataPackageRepository.markFailed(packageId, ex.getMessage());
|
try {
|
||||||
|
dataPackageRepository.markFailed(packageId, ex.getMessage());
|
||||||
|
} catch (RuntimeException markFailedEx) {
|
||||||
|
ex.addSuppressed(markFailedEx);
|
||||||
|
log.warn("Failed to mark EventHub acquisition package as failed packageId={} packageKey={}",
|
||||||
|
packageId, batch.packageKey(), markFailedEx);
|
||||||
|
}
|
||||||
throw ex;
|
throw ex;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private Map<String, Integer> eventTypeCounts(List<EventHubEventDto> events) {
|
||||||
|
Map<String, Integer> counts = new LinkedHashMap<>();
|
||||||
|
for (EventHubEventDto event : events) {
|
||||||
|
String domain = event.eventDomain() == null ? "UNKNOWN_DOMAIN" : event.eventDomain().name();
|
||||||
|
String type = event.eventType() == null ? "UNKNOWN_EVENT" : event.eventType().name();
|
||||||
|
counts.merge(domain + "/" + type, 1, Integer::sum);
|
||||||
|
}
|
||||||
|
return counts;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,22 @@
|
||||||
|
package at.procon.eventhub.tachograph.api;
|
||||||
|
|
||||||
|
import at.procon.eventhub.tachograph.service.UnsupportedTachographExtractionException;
|
||||||
|
import java.time.OffsetDateTime;
|
||||||
|
import java.util.Map;
|
||||||
|
import org.springframework.http.HttpStatus;
|
||||||
|
import org.springframework.http.ResponseEntity;
|
||||||
|
import org.springframework.web.bind.annotation.ExceptionHandler;
|
||||||
|
import org.springframework.web.bind.annotation.RestControllerAdvice;
|
||||||
|
|
||||||
|
@RestControllerAdvice(basePackageClasses = TachographIngestionController.class)
|
||||||
|
public class TachographApiExceptionHandler {
|
||||||
|
|
||||||
|
@ExceptionHandler(UnsupportedTachographExtractionException.class)
|
||||||
|
public ResponseEntity<Map<String, Object>> handleUnsupportedExtraction(UnsupportedTachographExtractionException ex) {
|
||||||
|
return ResponseEntity.status(HttpStatus.BAD_REQUEST).body(Map.of(
|
||||||
|
"error", "UNSUPPORTED_TACHOGRAPH_EXTRACTION",
|
||||||
|
"message", ex.getMessage(),
|
||||||
|
"timestamp", OffsetDateTime.now().toString()
|
||||||
|
));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -3,6 +3,7 @@ package at.procon.eventhub.tachograph.api;
|
||||||
import at.procon.eventhub.dto.AcquisitionStrategy;
|
import at.procon.eventhub.dto.AcquisitionStrategy;
|
||||||
import at.procon.eventhub.dto.ImportMode;
|
import at.procon.eventhub.dto.ImportMode;
|
||||||
import at.procon.eventhub.dto.SchedulerTriggerMode;
|
import at.procon.eventhub.dto.SchedulerTriggerMode;
|
||||||
|
import at.procon.eventhub.importing.masterdata.MasterDataRefreshResult;
|
||||||
import at.procon.eventhub.tachograph.dto.ConfiguredTachographImportPlanDto;
|
import at.procon.eventhub.tachograph.dto.ConfiguredTachographImportPlanDto;
|
||||||
import at.procon.eventhub.tachograph.dto.TachographImportRequest;
|
import at.procon.eventhub.tachograph.dto.TachographImportRequest;
|
||||||
import at.procon.eventhub.tachograph.dto.TachographImportRunResultDto;
|
import at.procon.eventhub.tachograph.dto.TachographImportRunResultDto;
|
||||||
|
|
@ -11,6 +12,7 @@ import at.procon.eventhub.tachograph.dto.source.TachographActivityDto;
|
||||||
import at.procon.eventhub.tachograph.service.TachographConfiguredImportPlanService;
|
import at.procon.eventhub.tachograph.service.TachographConfiguredImportPlanService;
|
||||||
import at.procon.eventhub.tachograph.service.TachographImportExecutionService;
|
import at.procon.eventhub.tachograph.service.TachographImportExecutionService;
|
||||||
import at.procon.eventhub.tachograph.service.TachographImportPlanService;
|
import at.procon.eventhub.tachograph.service.TachographImportPlanService;
|
||||||
|
import at.procon.eventhub.tachograph.service.TachographMasterDataRefreshService;
|
||||||
import jakarta.validation.Valid;
|
import jakarta.validation.Valid;
|
||||||
import java.time.OffsetDateTime;
|
import java.time.OffsetDateTime;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
|
|
@ -33,17 +35,20 @@ public class TachographIngestionController {
|
||||||
private final TachographImportPlanService tachographImportPlanService;
|
private final TachographImportPlanService tachographImportPlanService;
|
||||||
private final TachographConfiguredImportPlanService configuredImportPlanService;
|
private final TachographConfiguredImportPlanService configuredImportPlanService;
|
||||||
private final TachographImportExecutionService tachographImportExecutionService;
|
private final TachographImportExecutionService tachographImportExecutionService;
|
||||||
|
private final TachographMasterDataRefreshService masterDataRefreshService;
|
||||||
|
|
||||||
public TachographIngestionController(
|
public TachographIngestionController(
|
||||||
ProducerTemplate producerTemplate,
|
ProducerTemplate producerTemplate,
|
||||||
TachographImportPlanService tachographImportPlanService,
|
TachographImportPlanService tachographImportPlanService,
|
||||||
TachographConfiguredImportPlanService configuredImportPlanService,
|
TachographConfiguredImportPlanService configuredImportPlanService,
|
||||||
TachographImportExecutionService tachographImportExecutionService
|
TachographImportExecutionService tachographImportExecutionService,
|
||||||
|
TachographMasterDataRefreshService masterDataRefreshService
|
||||||
) {
|
) {
|
||||||
this.producerTemplate = producerTemplate;
|
this.producerTemplate = producerTemplate;
|
||||||
this.tachographImportPlanService = tachographImportPlanService;
|
this.tachographImportPlanService = tachographImportPlanService;
|
||||||
this.configuredImportPlanService = configuredImportPlanService;
|
this.configuredImportPlanService = configuredImportPlanService;
|
||||||
this.tachographImportExecutionService = tachographImportExecutionService;
|
this.tachographImportExecutionService = tachographImportExecutionService;
|
||||||
|
this.masterDataRefreshService = masterDataRefreshService;
|
||||||
}
|
}
|
||||||
|
|
||||||
@PostMapping("/activities")
|
@PostMapping("/activities")
|
||||||
|
|
@ -68,6 +73,13 @@ public class TachographIngestionController {
|
||||||
return ResponseEntity.accepted().body(result);
|
return ResponseEntity.accepted().body(result);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@PostMapping("/master-data/refresh")
|
||||||
|
public ResponseEntity<MasterDataRefreshResult> refreshTachographMasterData(
|
||||||
|
@Valid @RequestBody TachographImportRequest request
|
||||||
|
) {
|
||||||
|
return ResponseEntity.ok(masterDataRefreshService.refresh(request));
|
||||||
|
}
|
||||||
|
|
||||||
@GetMapping("/imports/configured-plans")
|
@GetMapping("/imports/configured-plans")
|
||||||
public ResponseEntity<List<ConfiguredTachographImportPlanDto>> listConfiguredTachographPlans() {
|
public ResponseEntity<List<ConfiguredTachographImportPlanDto>> listConfiguredTachographPlans() {
|
||||||
return ResponseEntity.ok(configuredImportPlanService.listPlans());
|
return ResponseEntity.ok(configuredImportPlanService.listPlans());
|
||||||
|
|
@ -99,6 +111,16 @@ public class TachographIngestionController {
|
||||||
));
|
));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@PostMapping("/imports/configured-plans/{planKey}/master-data/refresh")
|
||||||
|
public ResponseEntity<MasterDataRefreshResult> refreshConfiguredTachographMasterData(
|
||||||
|
@PathVariable String planKey,
|
||||||
|
@RequestParam(required = false) ImportMode mode,
|
||||||
|
@RequestParam(required = false) AcquisitionStrategy strategy
|
||||||
|
) {
|
||||||
|
TachographImportRequest request = configuredImportPlanService.createRequest(planKey, mode, strategy);
|
||||||
|
return ResponseEntity.ok(masterDataRefreshService.refresh(request));
|
||||||
|
}
|
||||||
|
|
||||||
private ResponseEntity<Map<String, Object>> accepted(int count, String route) {
|
private ResponseEntity<Map<String, Object>> accepted(int count, String route) {
|
||||||
return ResponseEntity.accepted().body(Map.of(
|
return ResponseEntity.accepted().body(Map.of(
|
||||||
"accepted", count,
|
"accepted", count,
|
||||||
|
|
|
||||||
|
|
@ -1,7 +1,7 @@
|
||||||
package at.procon.eventhub.tachograph.config;
|
package at.procon.eventhub.tachograph.config;
|
||||||
|
|
||||||
import at.procon.eventhub.config.EventHubProperties;
|
|
||||||
import javax.sql.DataSource;
|
import javax.sql.DataSource;
|
||||||
|
import org.springframework.boot.context.properties.ConfigurationProperties;
|
||||||
import org.springframework.beans.factory.annotation.Qualifier;
|
import org.springframework.beans.factory.annotation.Qualifier;
|
||||||
import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression;
|
import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression;
|
||||||
import org.springframework.context.annotation.Bean;
|
import org.springframework.context.annotation.Bean;
|
||||||
|
|
@ -13,15 +13,24 @@ import org.springframework.jdbc.datasource.DriverManagerDataSource;
|
||||||
@ConditionalOnExpression("'${eventhub.tachograph.datasource.jdbc-url:}' != ''")
|
@ConditionalOnExpression("'${eventhub.tachograph.datasource.jdbc-url:}' != ''")
|
||||||
public class TachographDataSourceConfig {
|
public class TachographDataSourceConfig {
|
||||||
|
|
||||||
|
private static final String SQL_SERVER_DRIVER_CLASS = "com.microsoft.sqlserver.jdbc.SQLServerDriver";
|
||||||
|
private static final String SQL_SERVER_JDBC_PREFIX = "jdbc:sqlserver://";
|
||||||
|
|
||||||
@Bean
|
@Bean
|
||||||
public DataSource tachographDataSource(EventHubProperties properties) {
|
@ConfigurationProperties(prefix = "eventhub.tachograph.datasource")
|
||||||
EventHubProperties.TachographDataSource config = properties.getTachograph().getDatasource();
|
public TachographDataSourceProperties tachographDataSourceProperties() {
|
||||||
|
return new TachographDataSourceProperties();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Bean(defaultCandidate = false)
|
||||||
|
public DataSource tachographDataSource(TachographDataSourceProperties config) {
|
||||||
DriverManagerDataSource dataSource = new DriverManagerDataSource();
|
DriverManagerDataSource dataSource = new DriverManagerDataSource();
|
||||||
dataSource.setUrl(config.getJdbcUrl());
|
dataSource.setUrl(validateJdbcUrl(config));
|
||||||
dataSource.setUsername(config.getUsername());
|
dataSource.setUsername(config.getUsername());
|
||||||
dataSource.setPassword(config.getPassword());
|
dataSource.setPassword(config.getPassword());
|
||||||
if (config.getDriverClassName() != null && !config.getDriverClassName().isBlank()) {
|
String driverClassName = trimToNull(config.getDriverClassName());
|
||||||
dataSource.setDriverClassName(config.getDriverClassName());
|
if (driverClassName != null) {
|
||||||
|
dataSource.setDriverClassName(driverClassName);
|
||||||
}
|
}
|
||||||
return dataSource;
|
return dataSource;
|
||||||
}
|
}
|
||||||
|
|
@ -32,4 +41,75 @@ public class TachographDataSourceConfig {
|
||||||
) {
|
) {
|
||||||
return new NamedParameterJdbcTemplate(tachographDataSource);
|
return new NamedParameterJdbcTemplate(tachographDataSource);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private String validateJdbcUrl(TachographDataSourceProperties config) {
|
||||||
|
String jdbcUrl = trimToNull(config.getJdbcUrl());
|
||||||
|
if (jdbcUrl == null) {
|
||||||
|
throw new IllegalStateException("eventhub.tachograph.datasource.jdbc-url must not be empty");
|
||||||
|
}
|
||||||
|
|
||||||
|
String driverClassName = trimToNull(config.getDriverClassName());
|
||||||
|
if (SQL_SERVER_DRIVER_CLASS.equals(driverClassName) && !jdbcUrl.startsWith(SQL_SERVER_JDBC_PREFIX)) {
|
||||||
|
if (jdbcUrl.startsWith("jdbc:")) {
|
||||||
|
String suggestedUrl = SQL_SERVER_JDBC_PREFIX + jdbcUrl.substring("jdbc:".length());
|
||||||
|
throw new IllegalStateException(
|
||||||
|
"Invalid SQL Server JDBC URL '" + jdbcUrl + "'. Expected prefix '"
|
||||||
|
+ SQL_SERVER_JDBC_PREFIX + "'. Example: " + suggestedUrl
|
||||||
|
);
|
||||||
|
}
|
||||||
|
throw new IllegalStateException(
|
||||||
|
"Invalid SQL Server JDBC URL '" + jdbcUrl + "'. Expected prefix '"
|
||||||
|
+ SQL_SERVER_JDBC_PREFIX + "'"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
return jdbcUrl;
|
||||||
|
}
|
||||||
|
|
||||||
|
private String trimToNull(String value) {
|
||||||
|
if (value == null) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
String trimmedValue = value.trim();
|
||||||
|
return trimmedValue.isEmpty() ? null : trimmedValue;
|
||||||
|
}
|
||||||
|
|
||||||
|
public static class TachographDataSourceProperties {
|
||||||
|
private String jdbcUrl;
|
||||||
|
private String username;
|
||||||
|
private String password;
|
||||||
|
private String driverClassName;
|
||||||
|
|
||||||
|
public String getJdbcUrl() {
|
||||||
|
return jdbcUrl;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setJdbcUrl(String jdbcUrl) {
|
||||||
|
this.jdbcUrl = jdbcUrl;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getUsername() {
|
||||||
|
return username;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setUsername(String username) {
|
||||||
|
this.username = username;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getPassword() {
|
||||||
|
return password;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setPassword(String password) {
|
||||||
|
this.password = password;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getDriverClassName() {
|
||||||
|
return driverClassName;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setDriverClassName(String driverClassName) {
|
||||||
|
this.driverClassName = driverClassName;
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -2,6 +2,7 @@ package at.procon.eventhub.tachograph.dto;
|
||||||
|
|
||||||
import at.procon.eventhub.importing.ExtractionBatchResult;
|
import at.procon.eventhub.importing.ExtractionBatchResult;
|
||||||
import java.time.OffsetDateTime;
|
import java.time.OffsetDateTime;
|
||||||
|
import java.util.Map;
|
||||||
import java.util.UUID;
|
import java.util.UUID;
|
||||||
|
|
||||||
public record TachographExtractionBatchResultDto(
|
public record TachographExtractionBatchResultDto(
|
||||||
|
|
@ -16,6 +17,11 @@ public record TachographExtractionBatchResultDto(
|
||||||
OffsetDateTime lastSourcePackageImportedAt,
|
OffsetDateTime lastSourcePackageImportedAt,
|
||||||
String lastSourcePackageId,
|
String lastSourcePackageId,
|
||||||
OffsetDateTime lastSourceRowUpdatedAt,
|
OffsetDateTime lastSourceRowUpdatedAt,
|
||||||
OffsetDateTime lastOccurredTo
|
OffsetDateTime lastOccurredTo,
|
||||||
|
Map<String, Integer> eventTypeCounts
|
||||||
) implements ExtractionBatchResult {
|
) implements ExtractionBatchResult {
|
||||||
|
|
||||||
|
public TachographExtractionBatchResultDto {
|
||||||
|
eventTypeCounts = eventTypeCounts == null ? Map.of() : Map.copyOf(eventTypeCounts);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -19,6 +19,7 @@ import at.procon.eventhub.tachograph.dto.TachographImportRequest;
|
||||||
import java.sql.ResultSet;
|
import java.sql.ResultSet;
|
||||||
import java.sql.SQLException;
|
import java.sql.SQLException;
|
||||||
import java.sql.Timestamp;
|
import java.sql.Timestamp;
|
||||||
|
import java.time.LocalDateTime;
|
||||||
import java.time.OffsetDateTime;
|
import java.time.OffsetDateTime;
|
||||||
import java.time.ZoneOffset;
|
import java.time.ZoneOffset;
|
||||||
import java.util.LinkedHashMap;
|
import java.util.LinkedHashMap;
|
||||||
|
|
@ -40,6 +41,9 @@ abstract class AbstractTachographActivityRowMapper implements ExtractionRowMappe
|
||||||
SourcePackageRefDto sourcePackageRef = sourcePackageRef(rs);
|
SourcePackageRefDto sourcePackageRef = sourcePackageRef(rs);
|
||||||
DriverRefDto driverRef = driverRef(rs);
|
DriverRefDto driverRef = driverRef(rs);
|
||||||
VehicleRefDto vehicleRef = vehicleRef(rs);
|
VehicleRefDto vehicleRef = vehicleRef(rs);
|
||||||
|
CardSlot cardSlot = cardSlot(rs);
|
||||||
|
CardStatus cardStatus = cardStatus(rs);
|
||||||
|
DrivingStatus drivingStatus = drivingStatus(rs);
|
||||||
|
|
||||||
String externalSourceEventId = string(rs, "external_source_event_id");
|
String externalSourceEventId = string(rs, "external_source_event_id");
|
||||||
if (externalSourceEventId == null) {
|
if (externalSourceEventId == null) {
|
||||||
|
|
@ -59,15 +63,21 @@ abstract class AbstractTachographActivityRowMapper implements ExtractionRowMappe
|
||||||
lifecycle(rs),
|
lifecycle(rs),
|
||||||
longValue(rs, "odometer_m"),
|
longValue(rs, "odometer_m"),
|
||||||
null,
|
null,
|
||||||
detailsFactory.driverActivity(cardSlot(rs), cardStatus(rs), drivingStatus(rs)),
|
detailsFactory.driverActivity(cardSlot, cardStatus, drivingStatus),
|
||||||
sourcePackageRef != null && sourcePackageRef.hasAnyReference() ? sourcePackageRef : null,
|
sourcePackageRef != null && sourcePackageRef.hasAnyReference() ? sourcePackageRef : null,
|
||||||
detailsFactory.payloadFromMap(payload(rs, context)),
|
detailsFactory.payloadFromMap(payload(rs, context)),
|
||||||
false,
|
isManualEntry(cardStatus, drivingStatus),
|
||||||
context.packageInfo()
|
context.packageInfo()
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
protected abstract Map<String, Object> sourceSpecificPayload(ResultSet rs) throws SQLException;
|
protected Map<String, Object> sourceSpecificPayload(ResultSet rs) throws SQLException {
|
||||||
|
return Map.of();
|
||||||
|
}
|
||||||
|
|
||||||
|
private boolean isManualEntry(CardStatus cardStatus, DrivingStatus drivingStatus) {
|
||||||
|
return cardStatus == CardStatus.NOT_INSERTED && drivingStatus == DrivingStatus.KNOWN;
|
||||||
|
}
|
||||||
|
|
||||||
private DriverRefDto driverRef(ResultSet rs) throws SQLException {
|
private DriverRefDto driverRef(ResultSet rs) throws SQLException {
|
||||||
DriverCardRefDto driverCard = null;
|
DriverCardRefDto driverCard = null;
|
||||||
|
|
@ -85,7 +95,12 @@ abstract class AbstractTachographActivityRowMapper implements ExtractionRowMappe
|
||||||
if (registrationNumber != null) {
|
if (registrationNumber != null) {
|
||||||
registration = new VehicleRegistrationRefDto(string(rs, "vehicle_registration_nation"), registrationNumber);
|
registration = new VehicleRegistrationRefDto(string(rs, "vehicle_registration_nation"), registrationNumber);
|
||||||
}
|
}
|
||||||
VehicleRefDto vehicleRef = new VehicleRefDto(string(rs, "vehicle_source_entity_id"), string(rs, "vehicle_vin"), registration);
|
VehicleRefDto vehicleRef = new VehicleRefDto(
|
||||||
|
string(rs, "vehicle_source_entity_id"),
|
||||||
|
string(rs, "vehicle_vin"),
|
||||||
|
string(rs, "vehicle_registration_source_entity_id"),
|
||||||
|
registration
|
||||||
|
);
|
||||||
return vehicleRef.hasAnyReference() ? vehicleRef : null;
|
return vehicleRef.hasAnyReference() ? vehicleRef : null;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -102,24 +117,7 @@ abstract class AbstractTachographActivityRowMapper implements ExtractionRowMappe
|
||||||
|
|
||||||
private Map<String, Object> payload(ResultSet rs, ExtractionContext<TachographImportRequest> context) throws SQLException {
|
private Map<String, Object> payload(ResultSet rs, ExtractionContext<TachographImportRequest> context) throws SQLException {
|
||||||
Map<String, Object> raw = new LinkedHashMap<>();
|
Map<String, Object> raw = new LinkedHashMap<>();
|
||||||
raw.put("extractionCode", context.planItem().extractionCode());
|
|
||||||
raw.put("sourceKind", context.planItem().sourceKind());
|
|
||||||
raw.put("sourceTables", context.planItem().sourceTables());
|
|
||||||
put(raw, "sourceRowId", string(rs, "source_row_id"));
|
put(raw, "sourceRowId", string(rs, "source_row_id"));
|
||||||
put(raw, "activityCode", object(rs, "activity_code"));
|
|
||||||
put(raw, "activityText", string(rs, "activity_text"));
|
|
||||||
put(raw, "eventType", string(rs, "event_type"));
|
|
||||||
put(raw, "lifecycle", string(rs, "lifecycle"));
|
|
||||||
put(raw, "cardSlot", object(rs, "card_slot"));
|
|
||||||
put(raw, "cardStatus", object(rs, "card_status"));
|
|
||||||
put(raw, "drivingStatus", object(rs, "driving_status"));
|
|
||||||
put(raw, "driverSourceEntityId", string(rs, "driver_source_entity_id"));
|
|
||||||
put(raw, "driverCardNation", string(rs, "driver_card_nation"));
|
|
||||||
put(raw, "driverCardNumber", string(rs, "driver_card_number"));
|
|
||||||
put(raw, "vehicleSourceEntityId", string(rs, "vehicle_source_entity_id"));
|
|
||||||
put(raw, "vehicleVin", string(rs, "vehicle_vin"));
|
|
||||||
put(raw, "vehicleRegistrationNation", string(rs, "vehicle_registration_nation"));
|
|
||||||
put(raw, "vehicleRegistrationNumber", string(rs, "vehicle_registration_number"));
|
|
||||||
raw.putAll(sourceSpecificPayload(rs));
|
raw.putAll(sourceSpecificPayload(rs));
|
||||||
return Map.of("raw", raw);
|
return Map.of("raw", raw);
|
||||||
}
|
}
|
||||||
|
|
@ -160,15 +158,20 @@ abstract class AbstractTachographActivityRowMapper implements ExtractionRowMappe
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
if (value instanceof OffsetDateTime offsetDateTime) {
|
if (value instanceof OffsetDateTime offsetDateTime) {
|
||||||
return offsetDateTime;
|
return offsetDateTime.withOffsetSameInstant(ZoneOffset.UTC);
|
||||||
}
|
}
|
||||||
if (value instanceof Timestamp timestamp) {
|
if (value instanceof Timestamp timestamp) {
|
||||||
return timestamp.toInstant().atOffset(ZoneOffset.UTC);
|
return timestamp.toLocalDateTime().atOffset(ZoneOffset.UTC);
|
||||||
}
|
}
|
||||||
if (value instanceof java.time.LocalDateTime localDateTime) {
|
if (value instanceof java.time.LocalDateTime localDateTime) {
|
||||||
return localDateTime.atOffset(ZoneOffset.UTC);
|
return localDateTime.atOffset(ZoneOffset.UTC);
|
||||||
}
|
}
|
||||||
return OffsetDateTime.parse(value.toString());
|
String text = value.toString();
|
||||||
|
try {
|
||||||
|
return OffsetDateTime.parse(text).withOffsetSameInstant(ZoneOffset.UTC);
|
||||||
|
} catch (RuntimeException ignored) {
|
||||||
|
return LocalDateTime.parse(text).atOffset(ZoneOffset.UTC);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private Long longValue(ResultSet rs, String column) throws SQLException {
|
private Long longValue(ResultSet rs, String column) throws SQLException {
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,200 @@
|
||||||
|
package at.procon.eventhub.tachograph.service;
|
||||||
|
|
||||||
|
import at.procon.eventhub.dto.DriverCardRefDto;
|
||||||
|
import at.procon.eventhub.dto.DriverRefDto;
|
||||||
|
import at.procon.eventhub.dto.EventDomain;
|
||||||
|
import at.procon.eventhub.dto.EventHubEventDto;
|
||||||
|
import at.procon.eventhub.dto.EventLifecycle;
|
||||||
|
import at.procon.eventhub.dto.EventType;
|
||||||
|
import at.procon.eventhub.dto.GeoPointDto;
|
||||||
|
import at.procon.eventhub.dto.SourcePackageRefDto;
|
||||||
|
import at.procon.eventhub.dto.VehicleRefDto;
|
||||||
|
import at.procon.eventhub.dto.VehicleRegistrationRefDto;
|
||||||
|
import at.procon.eventhub.importing.extraction.ExtractionContext;
|
||||||
|
import at.procon.eventhub.importing.extraction.ExtractionRowMapper;
|
||||||
|
import at.procon.eventhub.service.EventDetailsFactory;
|
||||||
|
import at.procon.eventhub.tachograph.dto.TachographImportRequest;
|
||||||
|
import java.math.BigDecimal;
|
||||||
|
import java.sql.ResultSet;
|
||||||
|
import java.sql.SQLException;
|
||||||
|
import java.sql.Timestamp;
|
||||||
|
import java.time.LocalDateTime;
|
||||||
|
import java.time.OffsetDateTime;
|
||||||
|
import java.time.ZoneOffset;
|
||||||
|
import java.util.LinkedHashMap;
|
||||||
|
import java.util.Map;
|
||||||
|
import java.util.UUID;
|
||||||
|
|
||||||
|
abstract class AbstractTachographBorderCrossingRowMapper implements ExtractionRowMapper<TachographImportRequest> {
|
||||||
|
|
||||||
|
private final EventDetailsFactory detailsFactory;
|
||||||
|
|
||||||
|
protected AbstractTachographBorderCrossingRowMapper(EventDetailsFactory detailsFactory) {
|
||||||
|
this.detailsFactory = detailsFactory;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public EventHubEventDto map(ResultSet rs, int rowNum, ExtractionContext<TachographImportRequest> context) throws SQLException {
|
||||||
|
OffsetDateTime occurredAt = offsetDateTime(rs, "occurred_at");
|
||||||
|
SourcePackageRefDto sourcePackageRef = sourcePackageRef(rs);
|
||||||
|
DriverRefDto driverRef = driverRef(rs);
|
||||||
|
VehicleRefDto vehicleRef = vehicleRef(rs);
|
||||||
|
|
||||||
|
String externalSourceEventId = string(rs, "external_source_event_id");
|
||||||
|
if (externalSourceEventId == null) {
|
||||||
|
externalSourceEventId = defaultExternalSourceEventId(context, rowNum, occurredAt, sourcePackageRef, driverRef, vehicleRef);
|
||||||
|
}
|
||||||
|
|
||||||
|
return new EventHubEventDto(
|
||||||
|
UUID.randomUUID(),
|
||||||
|
externalSourceEventId,
|
||||||
|
driverRef,
|
||||||
|
vehicleRef,
|
||||||
|
occurredAt,
|
||||||
|
offsetDateTime(rs, "received_partner_at"),
|
||||||
|
OffsetDateTime.now(),
|
||||||
|
EventDomain.BORDER_CROSSING,
|
||||||
|
EventType.BORDER_OUTBOUND,
|
||||||
|
EventLifecycle.OUTBOUND,
|
||||||
|
longValue(rs, "odometer_m"),
|
||||||
|
position(rs),
|
||||||
|
detailsFactory.borderCrossing(string(rs, "country_from"), string(rs, "country_to")),
|
||||||
|
sourcePackageRef != null && sourcePackageRef.hasAnyReference() ? sourcePackageRef : null,
|
||||||
|
detailsFactory.payloadFromMap(payload(rs)),
|
||||||
|
false,
|
||||||
|
context.packageInfo()
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
protected Map<String, Object> sourceSpecificPayload(ResultSet rs) throws SQLException {
|
||||||
|
return Map.of();
|
||||||
|
}
|
||||||
|
|
||||||
|
private DriverRefDto driverRef(ResultSet rs) throws SQLException {
|
||||||
|
DriverCardRefDto driverCard = null;
|
||||||
|
String cardNumber = string(rs, "driver_card_number");
|
||||||
|
if (cardNumber != null) {
|
||||||
|
driverCard = new DriverCardRefDto(string(rs, "driver_card_nation"), cardNumber);
|
||||||
|
}
|
||||||
|
DriverRefDto driverRef = new DriverRefDto(string(rs, "driver_source_entity_id"), driverCard);
|
||||||
|
return driverRef.hasAnyReference() ? driverRef : null;
|
||||||
|
}
|
||||||
|
|
||||||
|
private VehicleRefDto vehicleRef(ResultSet rs) throws SQLException {
|
||||||
|
VehicleRegistrationRefDto registration = null;
|
||||||
|
String registrationNumber = string(rs, "vehicle_registration_number");
|
||||||
|
if (registrationNumber != null) {
|
||||||
|
registration = new VehicleRegistrationRefDto(string(rs, "vehicle_registration_nation"), registrationNumber);
|
||||||
|
}
|
||||||
|
VehicleRefDto vehicleRef = new VehicleRefDto(
|
||||||
|
string(rs, "vehicle_source_entity_id"),
|
||||||
|
string(rs, "vehicle_vin"),
|
||||||
|
string(rs, "vehicle_registration_source_entity_id"),
|
||||||
|
registration
|
||||||
|
);
|
||||||
|
return vehicleRef.hasAnyReference() ? vehicleRef : null;
|
||||||
|
}
|
||||||
|
|
||||||
|
private SourcePackageRefDto sourcePackageRef(ResultSet rs) throws SQLException {
|
||||||
|
return new SourcePackageRefDto(
|
||||||
|
string(rs, "source_package_kind"),
|
||||||
|
string(rs, "source_package_id"),
|
||||||
|
string(rs, "source_package_entity_id"),
|
||||||
|
offsetDateTime(rs, "source_package_period_from"),
|
||||||
|
offsetDateTime(rs, "source_package_period_to"),
|
||||||
|
offsetDateTime(rs, "source_package_imported_at")
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
private GeoPointDto position(ResultSet rs) throws SQLException {
|
||||||
|
BigDecimal latitude = decimal(rs, "latitude");
|
||||||
|
BigDecimal longitude = decimal(rs, "longitude");
|
||||||
|
return latitude == null || longitude == null ? null : new GeoPointDto(latitude, longitude);
|
||||||
|
}
|
||||||
|
|
||||||
|
private Map<String, Object> payload(ResultSet rs) throws SQLException {
|
||||||
|
Map<String, Object> raw = new LinkedHashMap<>();
|
||||||
|
put(raw, "sourceRowId", string(rs, "source_row_id"));
|
||||||
|
raw.putAll(sourceSpecificPayload(rs));
|
||||||
|
return Map.of("raw", raw);
|
||||||
|
}
|
||||||
|
|
||||||
|
private String defaultExternalSourceEventId(
|
||||||
|
ExtractionContext<TachographImportRequest> context,
|
||||||
|
int rowNum,
|
||||||
|
OffsetDateTime occurredAt,
|
||||||
|
SourcePackageRefDto sourcePackageRef,
|
||||||
|
DriverRefDto driverRef,
|
||||||
|
VehicleRefDto vehicleRef
|
||||||
|
) {
|
||||||
|
String sourcePackageId = sourcePackageRef == null || sourcePackageRef.sourcePackageId() == null
|
||||||
|
? "NO_SOURCE_PACKAGE"
|
||||||
|
: sourcePackageRef.sourcePackageId();
|
||||||
|
String subject = driverRef != null && driverRef.hasAnyReference()
|
||||||
|
? driverRef.stableKey()
|
||||||
|
: vehicleRef == null ? "NO_SUBJECT" : vehicleRef.stableKey();
|
||||||
|
return "TACHOGRAPH:" + context.planItem().extractionCode()
|
||||||
|
+ ":" + sourcePackageId
|
||||||
|
+ ":" + occurredAt
|
||||||
|
+ ":" + subject
|
||||||
|
+ ":ROW-" + rowNum;
|
||||||
|
}
|
||||||
|
|
||||||
|
protected String string(ResultSet rs, String column) throws SQLException {
|
||||||
|
String value = rs.getString(column);
|
||||||
|
return value == null || value.isBlank() ? null : value.trim();
|
||||||
|
}
|
||||||
|
|
||||||
|
protected OffsetDateTime offsetDateTime(ResultSet rs, String column) throws SQLException {
|
||||||
|
Object value = rs.getObject(column);
|
||||||
|
if (value == null) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
if (value instanceof OffsetDateTime offsetDateTime) {
|
||||||
|
return offsetDateTime.withOffsetSameInstant(ZoneOffset.UTC);
|
||||||
|
}
|
||||||
|
if (value instanceof Timestamp timestamp) {
|
||||||
|
return timestamp.toLocalDateTime().atOffset(ZoneOffset.UTC);
|
||||||
|
}
|
||||||
|
if (value instanceof LocalDateTime localDateTime) {
|
||||||
|
return localDateTime.atOffset(ZoneOffset.UTC);
|
||||||
|
}
|
||||||
|
String text = value.toString();
|
||||||
|
try {
|
||||||
|
return OffsetDateTime.parse(text).withOffsetSameInstant(ZoneOffset.UTC);
|
||||||
|
} catch (RuntimeException ignored) {
|
||||||
|
return LocalDateTime.parse(text).atOffset(ZoneOffset.UTC);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private Long longValue(ResultSet rs, String column) throws SQLException {
|
||||||
|
Object value = rs.getObject(column);
|
||||||
|
if (value == null) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
if (value instanceof Number number) {
|
||||||
|
return number.longValue();
|
||||||
|
}
|
||||||
|
return Long.parseLong(value.toString());
|
||||||
|
}
|
||||||
|
|
||||||
|
private BigDecimal decimal(ResultSet rs, String column) throws SQLException {
|
||||||
|
Object value = rs.getObject(column);
|
||||||
|
if (value == null) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
if (value instanceof BigDecimal bigDecimal) {
|
||||||
|
return bigDecimal;
|
||||||
|
}
|
||||||
|
if (value instanceof Number number) {
|
||||||
|
return BigDecimal.valueOf(number.doubleValue());
|
||||||
|
}
|
||||||
|
return new BigDecimal(value.toString());
|
||||||
|
}
|
||||||
|
|
||||||
|
protected void put(Map<String, Object> target, String key, Object value) {
|
||||||
|
if (value != null) {
|
||||||
|
target.put(key, value);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,246 @@
|
||||||
|
package at.procon.eventhub.tachograph.service;
|
||||||
|
|
||||||
|
import at.procon.eventhub.dto.CardSlot;
|
||||||
|
import at.procon.eventhub.dto.CardStatus;
|
||||||
|
import at.procon.eventhub.dto.DriverCardRefDto;
|
||||||
|
import at.procon.eventhub.dto.DriverRefDto;
|
||||||
|
import at.procon.eventhub.dto.EventDomain;
|
||||||
|
import at.procon.eventhub.dto.EventHubEventDto;
|
||||||
|
import at.procon.eventhub.dto.EventLifecycle;
|
||||||
|
import at.procon.eventhub.dto.EventType;
|
||||||
|
import at.procon.eventhub.dto.SourcePackageRefDto;
|
||||||
|
import at.procon.eventhub.dto.VehicleRefDto;
|
||||||
|
import at.procon.eventhub.dto.VehicleRegistrationRefDto;
|
||||||
|
import at.procon.eventhub.importing.extraction.ExtractionContext;
|
||||||
|
import at.procon.eventhub.importing.extraction.ExtractionRowMapper;
|
||||||
|
import at.procon.eventhub.service.EventDetailsFactory;
|
||||||
|
import at.procon.eventhub.tachograph.dto.TachographImportRequest;
|
||||||
|
import java.sql.ResultSet;
|
||||||
|
import java.sql.SQLException;
|
||||||
|
import java.sql.Timestamp;
|
||||||
|
import java.time.LocalDateTime;
|
||||||
|
import java.time.OffsetDateTime;
|
||||||
|
import java.time.ZoneOffset;
|
||||||
|
import java.util.LinkedHashMap;
|
||||||
|
import java.util.Locale;
|
||||||
|
import java.util.Map;
|
||||||
|
import java.util.UUID;
|
||||||
|
|
||||||
|
abstract class AbstractTachographCardEventRowMapper implements ExtractionRowMapper<TachographImportRequest> {
|
||||||
|
|
||||||
|
private final EventDetailsFactory detailsFactory;
|
||||||
|
|
||||||
|
protected AbstractTachographCardEventRowMapper(EventDetailsFactory detailsFactory) {
|
||||||
|
this.detailsFactory = detailsFactory;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public EventHubEventDto map(ResultSet rs, int rowNum, ExtractionContext<TachographImportRequest> context) throws SQLException {
|
||||||
|
OffsetDateTime occurredAt = offsetDateTime(rs, "occurred_at");
|
||||||
|
SourcePackageRefDto sourcePackageRef = sourcePackageRef(rs);
|
||||||
|
DriverRefDto driverRef = driverRef(rs);
|
||||||
|
VehicleRefDto vehicleRef = vehicleRef(rs);
|
||||||
|
DriverCardRefDto driverCard = driverRef == null ? null : driverRef.driverCard();
|
||||||
|
EventType eventType = eventType(rs);
|
||||||
|
EventLifecycle lifecycle = lifecycle(rs);
|
||||||
|
CardStatus cardStatus = cardStatus(lifecycle);
|
||||||
|
|
||||||
|
String externalSourceEventId = string(rs, "external_source_event_id");
|
||||||
|
if (externalSourceEventId == null) {
|
||||||
|
externalSourceEventId = defaultExternalSourceEventId(context, rowNum, occurredAt, sourcePackageRef, driverRef, vehicleRef, lifecycle);
|
||||||
|
}
|
||||||
|
|
||||||
|
return new EventHubEventDto(
|
||||||
|
UUID.randomUUID(),
|
||||||
|
externalSourceEventId,
|
||||||
|
driverRef,
|
||||||
|
vehicleRef,
|
||||||
|
occurredAt,
|
||||||
|
offsetDateTime(rs, "received_partner_at"),
|
||||||
|
OffsetDateTime.now(),
|
||||||
|
EventDomain.DRIVER_CARD,
|
||||||
|
eventType,
|
||||||
|
lifecycle,
|
||||||
|
longValue(rs, "odometer_m"),
|
||||||
|
null,
|
||||||
|
detailsFactory.driverCard(cardSlot(rs), cardStatus, driverCard),
|
||||||
|
sourcePackageRef != null && sourcePackageRef.hasAnyReference() ? sourcePackageRef : null,
|
||||||
|
detailsFactory.payloadFromMap(payload(rs)),
|
||||||
|
false,
|
||||||
|
context.packageInfo()
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
protected Map<String, Object> sourceSpecificPayload(ResultSet rs) throws SQLException {
|
||||||
|
return Map.of();
|
||||||
|
}
|
||||||
|
|
||||||
|
private DriverRefDto driverRef(ResultSet rs) throws SQLException {
|
||||||
|
DriverCardRefDto driverCard = null;
|
||||||
|
String cardNumber = string(rs, "driver_card_number");
|
||||||
|
if (cardNumber != null) {
|
||||||
|
driverCard = new DriverCardRefDto(string(rs, "driver_card_nation"), cardNumber);
|
||||||
|
}
|
||||||
|
DriverRefDto driverRef = new DriverRefDto(string(rs, "driver_source_entity_id"), driverCard);
|
||||||
|
return driverRef.hasAnyReference() ? driverRef : null;
|
||||||
|
}
|
||||||
|
|
||||||
|
private VehicleRefDto vehicleRef(ResultSet rs) throws SQLException {
|
||||||
|
VehicleRegistrationRefDto registration = null;
|
||||||
|
String registrationNumber = string(rs, "vehicle_registration_number");
|
||||||
|
if (registrationNumber != null) {
|
||||||
|
registration = new VehicleRegistrationRefDto(string(rs, "vehicle_registration_nation"), registrationNumber);
|
||||||
|
}
|
||||||
|
VehicleRefDto vehicleRef = new VehicleRefDto(
|
||||||
|
string(rs, "vehicle_source_entity_id"),
|
||||||
|
string(rs, "vehicle_vin"),
|
||||||
|
string(rs, "vehicle_registration_source_entity_id"),
|
||||||
|
registration
|
||||||
|
);
|
||||||
|
return vehicleRef.hasAnyReference() ? vehicleRef : null;
|
||||||
|
}
|
||||||
|
|
||||||
|
private SourcePackageRefDto sourcePackageRef(ResultSet rs) throws SQLException {
|
||||||
|
return new SourcePackageRefDto(
|
||||||
|
string(rs, "source_package_kind"),
|
||||||
|
string(rs, "source_package_id"),
|
||||||
|
string(rs, "source_package_entity_id"),
|
||||||
|
offsetDateTime(rs, "source_package_period_from"),
|
||||||
|
offsetDateTime(rs, "source_package_period_to"),
|
||||||
|
offsetDateTime(rs, "source_package_imported_at")
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
private Map<String, Object> payload(ResultSet rs) throws SQLException {
|
||||||
|
Map<String, Object> raw = new LinkedHashMap<>();
|
||||||
|
put(raw, "sourceRowId", string(rs, "source_row_id"));
|
||||||
|
raw.putAll(sourceSpecificPayload(rs));
|
||||||
|
return Map.of("raw", raw);
|
||||||
|
}
|
||||||
|
|
||||||
|
private String defaultExternalSourceEventId(
|
||||||
|
ExtractionContext<TachographImportRequest> context,
|
||||||
|
int rowNum,
|
||||||
|
OffsetDateTime occurredAt,
|
||||||
|
SourcePackageRefDto sourcePackageRef,
|
||||||
|
DriverRefDto driverRef,
|
||||||
|
VehicleRefDto vehicleRef,
|
||||||
|
EventLifecycle lifecycle
|
||||||
|
) {
|
||||||
|
String sourcePackageId = sourcePackageRef == null || sourcePackageRef.sourcePackageId() == null
|
||||||
|
? "NO_SOURCE_PACKAGE"
|
||||||
|
: sourcePackageRef.sourcePackageId();
|
||||||
|
String subject = driverRef != null && driverRef.hasAnyReference()
|
||||||
|
? driverRef.stableKey()
|
||||||
|
: vehicleRef == null ? "NO_SUBJECT" : vehicleRef.stableKey();
|
||||||
|
return "TACHOGRAPH:" + context.planItem().extractionCode()
|
||||||
|
+ ":" + sourcePackageId
|
||||||
|
+ ":" + lifecycle
|
||||||
|
+ ":" + occurredAt
|
||||||
|
+ ":" + subject
|
||||||
|
+ ":ROW-" + rowNum;
|
||||||
|
}
|
||||||
|
|
||||||
|
protected String string(ResultSet rs, String column) throws SQLException {
|
||||||
|
String value = rs.getString(column);
|
||||||
|
return value == null || value.isBlank() ? null : value.trim();
|
||||||
|
}
|
||||||
|
|
||||||
|
protected Object object(ResultSet rs, String column) throws SQLException {
|
||||||
|
return rs.getObject(column);
|
||||||
|
}
|
||||||
|
|
||||||
|
protected OffsetDateTime offsetDateTime(ResultSet rs, String column) throws SQLException {
|
||||||
|
Object value = rs.getObject(column);
|
||||||
|
if (value == null) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
if (value instanceof OffsetDateTime offsetDateTime) {
|
||||||
|
return offsetDateTime.withOffsetSameInstant(ZoneOffset.UTC);
|
||||||
|
}
|
||||||
|
if (value instanceof Timestamp timestamp) {
|
||||||
|
return timestamp.toLocalDateTime().atOffset(ZoneOffset.UTC);
|
||||||
|
}
|
||||||
|
if (value instanceof LocalDateTime localDateTime) {
|
||||||
|
return localDateTime.atOffset(ZoneOffset.UTC);
|
||||||
|
}
|
||||||
|
String text = value.toString();
|
||||||
|
try {
|
||||||
|
return OffsetDateTime.parse(text).withOffsetSameInstant(ZoneOffset.UTC);
|
||||||
|
} catch (RuntimeException ignored) {
|
||||||
|
return LocalDateTime.parse(text).atOffset(ZoneOffset.UTC);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private Long longValue(ResultSet rs, String column) throws SQLException {
|
||||||
|
Object value = rs.getObject(column);
|
||||||
|
if (value == null) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
if (value instanceof Number number) {
|
||||||
|
return number.longValue();
|
||||||
|
}
|
||||||
|
return Long.parseLong(value.toString());
|
||||||
|
}
|
||||||
|
|
||||||
|
private CardSlot cardSlot(ResultSet rs) throws SQLException {
|
||||||
|
Object value = object(rs, "card_slot");
|
||||||
|
if (value instanceof Number number) {
|
||||||
|
return switch (number.intValue()) {
|
||||||
|
case 0 -> CardSlot.DRIVER;
|
||||||
|
case 1 -> CardSlot.CO_DRIVER;
|
||||||
|
default -> null;
|
||||||
|
};
|
||||||
|
}
|
||||||
|
String text = value == null ? null : value.toString();
|
||||||
|
Integer parsed = parseInteger(text);
|
||||||
|
if (parsed != null) {
|
||||||
|
return parsed == 0 ? CardSlot.DRIVER : parsed == 1 ? CardSlot.CO_DRIVER : null;
|
||||||
|
}
|
||||||
|
return parseEnum(CardSlot.class, text, null);
|
||||||
|
}
|
||||||
|
|
||||||
|
private EventType eventType(ResultSet rs) throws SQLException {
|
||||||
|
return parseEnum(EventType.class, string(rs, "event_type"), EventType.UNKNOWN_EVENT);
|
||||||
|
}
|
||||||
|
|
||||||
|
private EventLifecycle lifecycle(ResultSet rs) throws SQLException {
|
||||||
|
return parseEnum(EventLifecycle.class, string(rs, "lifecycle"), EventLifecycle.SNAPSHOT);
|
||||||
|
}
|
||||||
|
|
||||||
|
private CardStatus cardStatus(EventLifecycle lifecycle) {
|
||||||
|
return lifecycle == EventLifecycle.INSERT ? CardStatus.INSERTED : lifecycle == EventLifecycle.WITHDRAW ? CardStatus.NOT_INSERTED : null;
|
||||||
|
}
|
||||||
|
|
||||||
|
private <T extends Enum<T>> T parseEnum(Class<T> type, String value, T fallback) {
|
||||||
|
if (value == null) {
|
||||||
|
return fallback;
|
||||||
|
}
|
||||||
|
String normalized = value.trim().toUpperCase(Locale.ROOT).replace('-', '_').replace(' ', '_');
|
||||||
|
if ("CODRIVER".equals(normalized)) {
|
||||||
|
normalized = "CO_DRIVER";
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
return Enum.valueOf(type, normalized);
|
||||||
|
} catch (IllegalArgumentException ignored) {
|
||||||
|
return fallback;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private Integer parseInteger(String value) {
|
||||||
|
if (value == null || value.isBlank()) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
return Integer.parseInt(value.trim());
|
||||||
|
} catch (NumberFormatException ignored) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
protected void put(Map<String, Object> target, String key, Object value) {
|
||||||
|
if (value != null) {
|
||||||
|
target.put(key, value);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,220 @@
|
||||||
|
package at.procon.eventhub.tachograph.service;
|
||||||
|
|
||||||
|
import at.procon.eventhub.dto.DriverCardRefDto;
|
||||||
|
import at.procon.eventhub.dto.DriverRefDto;
|
||||||
|
import at.procon.eventhub.dto.EventDomain;
|
||||||
|
import at.procon.eventhub.dto.EventHubEventDto;
|
||||||
|
import at.procon.eventhub.dto.EventLifecycle;
|
||||||
|
import at.procon.eventhub.dto.EventType;
|
||||||
|
import at.procon.eventhub.dto.GeoPointDto;
|
||||||
|
import at.procon.eventhub.dto.SourcePackageRefDto;
|
||||||
|
import at.procon.eventhub.dto.VehicleRefDto;
|
||||||
|
import at.procon.eventhub.dto.VehicleRegistrationRefDto;
|
||||||
|
import at.procon.eventhub.importing.extraction.ExtractionContext;
|
||||||
|
import at.procon.eventhub.importing.extraction.ExtractionRowMapper;
|
||||||
|
import at.procon.eventhub.service.EventDetailsFactory;
|
||||||
|
import at.procon.eventhub.tachograph.dto.TachographImportRequest;
|
||||||
|
import java.math.BigDecimal;
|
||||||
|
import java.sql.ResultSet;
|
||||||
|
import java.sql.SQLException;
|
||||||
|
import java.sql.Timestamp;
|
||||||
|
import java.time.LocalDateTime;
|
||||||
|
import java.time.OffsetDateTime;
|
||||||
|
import java.time.ZoneOffset;
|
||||||
|
import java.util.LinkedHashMap;
|
||||||
|
import java.util.Locale;
|
||||||
|
import java.util.Map;
|
||||||
|
import java.util.UUID;
|
||||||
|
|
||||||
|
abstract class AbstractTachographLoadUnloadRowMapper implements ExtractionRowMapper<TachographImportRequest> {
|
||||||
|
|
||||||
|
private final EventDetailsFactory detailsFactory;
|
||||||
|
|
||||||
|
protected AbstractTachographLoadUnloadRowMapper(EventDetailsFactory detailsFactory) {
|
||||||
|
this.detailsFactory = detailsFactory;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public EventHubEventDto map(ResultSet rs, int rowNum, ExtractionContext<TachographImportRequest> context) throws SQLException {
|
||||||
|
OffsetDateTime occurredAt = offsetDateTime(rs, "occurred_at");
|
||||||
|
SourcePackageRefDto sourcePackageRef = sourcePackageRef(rs);
|
||||||
|
DriverRefDto driverRef = driverRef(rs);
|
||||||
|
VehicleRefDto vehicleRef = vehicleRef(rs);
|
||||||
|
EventType eventType = eventType(rs);
|
||||||
|
|
||||||
|
String externalSourceEventId = string(rs, "external_source_event_id");
|
||||||
|
if (externalSourceEventId == null) {
|
||||||
|
externalSourceEventId = defaultExternalSourceEventId(context, rowNum, occurredAt, sourcePackageRef, driverRef, vehicleRef, eventType);
|
||||||
|
}
|
||||||
|
|
||||||
|
return new EventHubEventDto(
|
||||||
|
UUID.randomUUID(),
|
||||||
|
externalSourceEventId,
|
||||||
|
driverRef,
|
||||||
|
vehicleRef,
|
||||||
|
occurredAt,
|
||||||
|
offsetDateTime(rs, "received_partner_at"),
|
||||||
|
OffsetDateTime.now(),
|
||||||
|
EventDomain.LOAD_UNLOAD,
|
||||||
|
eventType,
|
||||||
|
EventLifecycle.SNAPSHOT,
|
||||||
|
longValue(rs, "odometer_m"),
|
||||||
|
position(rs),
|
||||||
|
detailsFactory.loadUnload(string(rs, "operation")),
|
||||||
|
sourcePackageRef != null && sourcePackageRef.hasAnyReference() ? sourcePackageRef : null,
|
||||||
|
detailsFactory.payloadFromMap(payload(rs)),
|
||||||
|
false,
|
||||||
|
context.packageInfo()
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
protected Map<String, Object> sourceSpecificPayload(ResultSet rs) throws SQLException {
|
||||||
|
return Map.of();
|
||||||
|
}
|
||||||
|
|
||||||
|
private DriverRefDto driverRef(ResultSet rs) throws SQLException {
|
||||||
|
DriverCardRefDto driverCard = null;
|
||||||
|
String cardNumber = string(rs, "driver_card_number");
|
||||||
|
if (cardNumber != null) {
|
||||||
|
driverCard = new DriverCardRefDto(string(rs, "driver_card_nation"), cardNumber);
|
||||||
|
}
|
||||||
|
DriverRefDto driverRef = new DriverRefDto(string(rs, "driver_source_entity_id"), driverCard);
|
||||||
|
return driverRef.hasAnyReference() ? driverRef : null;
|
||||||
|
}
|
||||||
|
|
||||||
|
private VehicleRefDto vehicleRef(ResultSet rs) throws SQLException {
|
||||||
|
VehicleRegistrationRefDto registration = null;
|
||||||
|
String registrationNumber = string(rs, "vehicle_registration_number");
|
||||||
|
if (registrationNumber != null) {
|
||||||
|
registration = new VehicleRegistrationRefDto(string(rs, "vehicle_registration_nation"), registrationNumber);
|
||||||
|
}
|
||||||
|
VehicleRefDto vehicleRef = new VehicleRefDto(
|
||||||
|
string(rs, "vehicle_source_entity_id"),
|
||||||
|
string(rs, "vehicle_vin"),
|
||||||
|
string(rs, "vehicle_registration_source_entity_id"),
|
||||||
|
registration
|
||||||
|
);
|
||||||
|
return vehicleRef.hasAnyReference() ? vehicleRef : null;
|
||||||
|
}
|
||||||
|
|
||||||
|
private SourcePackageRefDto sourcePackageRef(ResultSet rs) throws SQLException {
|
||||||
|
return new SourcePackageRefDto(
|
||||||
|
string(rs, "source_package_kind"),
|
||||||
|
string(rs, "source_package_id"),
|
||||||
|
string(rs, "source_package_entity_id"),
|
||||||
|
offsetDateTime(rs, "source_package_period_from"),
|
||||||
|
offsetDateTime(rs, "source_package_period_to"),
|
||||||
|
offsetDateTime(rs, "source_package_imported_at")
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
private GeoPointDto position(ResultSet rs) throws SQLException {
|
||||||
|
BigDecimal latitude = decimal(rs, "latitude");
|
||||||
|
BigDecimal longitude = decimal(rs, "longitude");
|
||||||
|
return latitude == null || longitude == null ? null : new GeoPointDto(latitude, longitude);
|
||||||
|
}
|
||||||
|
|
||||||
|
private Map<String, Object> payload(ResultSet rs) throws SQLException {
|
||||||
|
Map<String, Object> raw = new LinkedHashMap<>();
|
||||||
|
put(raw, "sourceRowId", string(rs, "source_row_id"));
|
||||||
|
raw.putAll(sourceSpecificPayload(rs));
|
||||||
|
return Map.of("raw", raw);
|
||||||
|
}
|
||||||
|
|
||||||
|
private String defaultExternalSourceEventId(
|
||||||
|
ExtractionContext<TachographImportRequest> context,
|
||||||
|
int rowNum,
|
||||||
|
OffsetDateTime occurredAt,
|
||||||
|
SourcePackageRefDto sourcePackageRef,
|
||||||
|
DriverRefDto driverRef,
|
||||||
|
VehicleRefDto vehicleRef,
|
||||||
|
EventType eventType
|
||||||
|
) {
|
||||||
|
String sourcePackageId = sourcePackageRef == null || sourcePackageRef.sourcePackageId() == null
|
||||||
|
? "NO_SOURCE_PACKAGE"
|
||||||
|
: sourcePackageRef.sourcePackageId();
|
||||||
|
String subject = driverRef != null && driverRef.hasAnyReference()
|
||||||
|
? driverRef.stableKey()
|
||||||
|
: vehicleRef == null ? "NO_SUBJECT" : vehicleRef.stableKey();
|
||||||
|
return "TACHOGRAPH:" + context.planItem().extractionCode()
|
||||||
|
+ ":" + sourcePackageId
|
||||||
|
+ ":" + eventType
|
||||||
|
+ ":" + occurredAt
|
||||||
|
+ ":" + subject
|
||||||
|
+ ":ROW-" + rowNum;
|
||||||
|
}
|
||||||
|
|
||||||
|
protected String string(ResultSet rs, String column) throws SQLException {
|
||||||
|
String value = rs.getString(column);
|
||||||
|
return value == null || value.isBlank() ? null : value.trim();
|
||||||
|
}
|
||||||
|
|
||||||
|
protected OffsetDateTime offsetDateTime(ResultSet rs, String column) throws SQLException {
|
||||||
|
Object value = rs.getObject(column);
|
||||||
|
if (value == null) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
if (value instanceof OffsetDateTime offsetDateTime) {
|
||||||
|
return offsetDateTime.withOffsetSameInstant(ZoneOffset.UTC);
|
||||||
|
}
|
||||||
|
if (value instanceof Timestamp timestamp) {
|
||||||
|
return timestamp.toLocalDateTime().atOffset(ZoneOffset.UTC);
|
||||||
|
}
|
||||||
|
if (value instanceof LocalDateTime localDateTime) {
|
||||||
|
return localDateTime.atOffset(ZoneOffset.UTC);
|
||||||
|
}
|
||||||
|
String text = value.toString();
|
||||||
|
try {
|
||||||
|
return OffsetDateTime.parse(text).withOffsetSameInstant(ZoneOffset.UTC);
|
||||||
|
} catch (RuntimeException ignored) {
|
||||||
|
return LocalDateTime.parse(text).atOffset(ZoneOffset.UTC);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private Long longValue(ResultSet rs, String column) throws SQLException {
|
||||||
|
Object value = rs.getObject(column);
|
||||||
|
if (value == null) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
if (value instanceof Number number) {
|
||||||
|
return number.longValue();
|
||||||
|
}
|
||||||
|
return Long.parseLong(value.toString());
|
||||||
|
}
|
||||||
|
|
||||||
|
private BigDecimal decimal(ResultSet rs, String column) throws SQLException {
|
||||||
|
Object value = rs.getObject(column);
|
||||||
|
if (value == null) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
if (value instanceof BigDecimal bigDecimal) {
|
||||||
|
return bigDecimal;
|
||||||
|
}
|
||||||
|
if (value instanceof Number number) {
|
||||||
|
return BigDecimal.valueOf(number.doubleValue());
|
||||||
|
}
|
||||||
|
return new BigDecimal(value.toString());
|
||||||
|
}
|
||||||
|
|
||||||
|
private EventType eventType(ResultSet rs) throws SQLException {
|
||||||
|
return parseEnum(EventType.class, string(rs, "event_type"), EventType.LOAD_UNLOAD);
|
||||||
|
}
|
||||||
|
|
||||||
|
private <T extends Enum<T>> T parseEnum(Class<T> type, String value, T fallback) {
|
||||||
|
if (value == null) {
|
||||||
|
return fallback;
|
||||||
|
}
|
||||||
|
String normalized = value.trim().toUpperCase(Locale.ROOT).replace('-', '_').replace(' ', '_');
|
||||||
|
try {
|
||||||
|
return Enum.valueOf(type, normalized);
|
||||||
|
} catch (IllegalArgumentException ignored) {
|
||||||
|
return fallback;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
protected void put(Map<String, Object> target, String key, Object value) {
|
||||||
|
if (value != null) {
|
||||||
|
target.put(key, value);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,239 @@
|
||||||
|
package at.procon.eventhub.tachograph.service;
|
||||||
|
|
||||||
|
import at.procon.eventhub.dto.DriverCardRefDto;
|
||||||
|
import at.procon.eventhub.dto.DriverRefDto;
|
||||||
|
import at.procon.eventhub.dto.EventDomain;
|
||||||
|
import at.procon.eventhub.dto.EventHubEventDto;
|
||||||
|
import at.procon.eventhub.dto.EventLifecycle;
|
||||||
|
import at.procon.eventhub.dto.EventType;
|
||||||
|
import at.procon.eventhub.dto.GeoPointDto;
|
||||||
|
import at.procon.eventhub.dto.SourcePackageRefDto;
|
||||||
|
import at.procon.eventhub.dto.VehicleRefDto;
|
||||||
|
import at.procon.eventhub.dto.VehicleRegistrationRefDto;
|
||||||
|
import at.procon.eventhub.importing.extraction.ExtractionContext;
|
||||||
|
import at.procon.eventhub.importing.extraction.ExtractionRowMapper;
|
||||||
|
import at.procon.eventhub.service.EventDetailsFactory;
|
||||||
|
import at.procon.eventhub.tachograph.dto.TachographImportRequest;
|
||||||
|
import java.math.BigDecimal;
|
||||||
|
import java.sql.ResultSet;
|
||||||
|
import java.sql.SQLException;
|
||||||
|
import java.sql.Timestamp;
|
||||||
|
import java.time.LocalDateTime;
|
||||||
|
import java.time.OffsetDateTime;
|
||||||
|
import java.time.ZoneOffset;
|
||||||
|
import java.util.LinkedHashMap;
|
||||||
|
import java.util.Locale;
|
||||||
|
import java.util.Map;
|
||||||
|
import java.util.UUID;
|
||||||
|
|
||||||
|
abstract class AbstractTachographPlaceRowMapper implements ExtractionRowMapper<TachographImportRequest> {
|
||||||
|
|
||||||
|
private final EventDetailsFactory detailsFactory;
|
||||||
|
|
||||||
|
protected AbstractTachographPlaceRowMapper(EventDetailsFactory detailsFactory) {
|
||||||
|
this.detailsFactory = detailsFactory;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public EventHubEventDto map(ResultSet rs, int rowNum, ExtractionContext<TachographImportRequest> context) throws SQLException {
|
||||||
|
OffsetDateTime occurredAt = offsetDateTime(rs, "occurred_at");
|
||||||
|
SourcePackageRefDto sourcePackageRef = sourcePackageRef(rs);
|
||||||
|
DriverRefDto driverRef = driverRef(rs);
|
||||||
|
VehicleRefDto vehicleRef = vehicleRef(rs);
|
||||||
|
EventType eventType = eventType(rs);
|
||||||
|
EventLifecycle lifecycle = lifecycle(rs);
|
||||||
|
|
||||||
|
String externalSourceEventId = string(rs, "external_source_event_id");
|
||||||
|
if (externalSourceEventId == null) {
|
||||||
|
externalSourceEventId = defaultExternalSourceEventId(context, rowNum, occurredAt, sourcePackageRef, driverRef, vehicleRef, eventType);
|
||||||
|
}
|
||||||
|
|
||||||
|
return new EventHubEventDto(
|
||||||
|
UUID.randomUUID(),
|
||||||
|
externalSourceEventId,
|
||||||
|
driverRef,
|
||||||
|
vehicleRef,
|
||||||
|
occurredAt,
|
||||||
|
offsetDateTime(rs, "received_partner_at"),
|
||||||
|
OffsetDateTime.now(),
|
||||||
|
EventDomain.PLACE,
|
||||||
|
eventType,
|
||||||
|
lifecycle,
|
||||||
|
longValue(rs, "odometer_m"),
|
||||||
|
position(rs),
|
||||||
|
detailsFactory.place(string(rs, "country"), string(rs, "region")),
|
||||||
|
sourcePackageRef != null && sourcePackageRef.hasAnyReference() ? sourcePackageRef : null,
|
||||||
|
detailsFactory.payloadFromMap(payload(rs)),
|
||||||
|
booleanValue(rs, "manual_entry"),
|
||||||
|
context.packageInfo()
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
protected Map<String, Object> sourceSpecificPayload(ResultSet rs) throws SQLException {
|
||||||
|
return Map.of();
|
||||||
|
}
|
||||||
|
|
||||||
|
private DriverRefDto driverRef(ResultSet rs) throws SQLException {
|
||||||
|
DriverCardRefDto driverCard = null;
|
||||||
|
String cardNumber = string(rs, "driver_card_number");
|
||||||
|
if (cardNumber != null) {
|
||||||
|
driverCard = new DriverCardRefDto(string(rs, "driver_card_nation"), cardNumber);
|
||||||
|
}
|
||||||
|
DriverRefDto driverRef = new DriverRefDto(string(rs, "driver_source_entity_id"), driverCard);
|
||||||
|
return driverRef.hasAnyReference() ? driverRef : null;
|
||||||
|
}
|
||||||
|
|
||||||
|
private VehicleRefDto vehicleRef(ResultSet rs) throws SQLException {
|
||||||
|
VehicleRegistrationRefDto registration = null;
|
||||||
|
String registrationNumber = string(rs, "vehicle_registration_number");
|
||||||
|
if (registrationNumber != null) {
|
||||||
|
registration = new VehicleRegistrationRefDto(string(rs, "vehicle_registration_nation"), registrationNumber);
|
||||||
|
}
|
||||||
|
VehicleRefDto vehicleRef = new VehicleRefDto(
|
||||||
|
string(rs, "vehicle_source_entity_id"),
|
||||||
|
string(rs, "vehicle_vin"),
|
||||||
|
string(rs, "vehicle_registration_source_entity_id"),
|
||||||
|
registration
|
||||||
|
);
|
||||||
|
return vehicleRef.hasAnyReference() ? vehicleRef : null;
|
||||||
|
}
|
||||||
|
|
||||||
|
private SourcePackageRefDto sourcePackageRef(ResultSet rs) throws SQLException {
|
||||||
|
return new SourcePackageRefDto(
|
||||||
|
string(rs, "source_package_kind"),
|
||||||
|
string(rs, "source_package_id"),
|
||||||
|
string(rs, "source_package_entity_id"),
|
||||||
|
offsetDateTime(rs, "source_package_period_from"),
|
||||||
|
offsetDateTime(rs, "source_package_period_to"),
|
||||||
|
offsetDateTime(rs, "source_package_imported_at")
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
private GeoPointDto position(ResultSet rs) throws SQLException {
|
||||||
|
BigDecimal latitude = decimal(rs, "latitude");
|
||||||
|
BigDecimal longitude = decimal(rs, "longitude");
|
||||||
|
return latitude == null || longitude == null ? null : new GeoPointDto(latitude, longitude);
|
||||||
|
}
|
||||||
|
|
||||||
|
private Map<String, Object> payload(ResultSet rs) throws SQLException {
|
||||||
|
Map<String, Object> raw = new LinkedHashMap<>();
|
||||||
|
put(raw, "sourceRowId", string(rs, "source_row_id"));
|
||||||
|
raw.putAll(sourceSpecificPayload(rs));
|
||||||
|
return Map.of("raw", raw);
|
||||||
|
}
|
||||||
|
|
||||||
|
private String defaultExternalSourceEventId(
|
||||||
|
ExtractionContext<TachographImportRequest> context,
|
||||||
|
int rowNum,
|
||||||
|
OffsetDateTime occurredAt,
|
||||||
|
SourcePackageRefDto sourcePackageRef,
|
||||||
|
DriverRefDto driverRef,
|
||||||
|
VehicleRefDto vehicleRef,
|
||||||
|
EventType eventType
|
||||||
|
) {
|
||||||
|
String sourcePackageId = sourcePackageRef == null || sourcePackageRef.sourcePackageId() == null
|
||||||
|
? "NO_SOURCE_PACKAGE"
|
||||||
|
: sourcePackageRef.sourcePackageId();
|
||||||
|
String subject = driverRef != null && driverRef.hasAnyReference()
|
||||||
|
? driverRef.stableKey()
|
||||||
|
: vehicleRef == null ? "NO_SUBJECT" : vehicleRef.stableKey();
|
||||||
|
return "TACHOGRAPH:" + context.planItem().extractionCode()
|
||||||
|
+ ":" + sourcePackageId
|
||||||
|
+ ":" + eventType
|
||||||
|
+ ":" + occurredAt
|
||||||
|
+ ":" + subject
|
||||||
|
+ ":ROW-" + rowNum;
|
||||||
|
}
|
||||||
|
|
||||||
|
protected String string(ResultSet rs, String column) throws SQLException {
|
||||||
|
String value = rs.getString(column);
|
||||||
|
return value == null || value.isBlank() ? null : value.trim();
|
||||||
|
}
|
||||||
|
|
||||||
|
protected OffsetDateTime offsetDateTime(ResultSet rs, String column) throws SQLException {
|
||||||
|
Object value = rs.getObject(column);
|
||||||
|
if (value == null) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
if (value instanceof OffsetDateTime offsetDateTime) {
|
||||||
|
return offsetDateTime.withOffsetSameInstant(ZoneOffset.UTC);
|
||||||
|
}
|
||||||
|
if (value instanceof Timestamp timestamp) {
|
||||||
|
return timestamp.toLocalDateTime().atOffset(ZoneOffset.UTC);
|
||||||
|
}
|
||||||
|
if (value instanceof LocalDateTime localDateTime) {
|
||||||
|
return localDateTime.atOffset(ZoneOffset.UTC);
|
||||||
|
}
|
||||||
|
String text = value.toString();
|
||||||
|
try {
|
||||||
|
return OffsetDateTime.parse(text).withOffsetSameInstant(ZoneOffset.UTC);
|
||||||
|
} catch (RuntimeException ignored) {
|
||||||
|
return LocalDateTime.parse(text).atOffset(ZoneOffset.UTC);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private Long longValue(ResultSet rs, String column) throws SQLException {
|
||||||
|
Object value = rs.getObject(column);
|
||||||
|
if (value == null) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
if (value instanceof Number number) {
|
||||||
|
return number.longValue();
|
||||||
|
}
|
||||||
|
return Long.parseLong(value.toString());
|
||||||
|
}
|
||||||
|
|
||||||
|
private BigDecimal decimal(ResultSet rs, String column) throws SQLException {
|
||||||
|
Object value = rs.getObject(column);
|
||||||
|
if (value == null) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
if (value instanceof BigDecimal bigDecimal) {
|
||||||
|
return bigDecimal;
|
||||||
|
}
|
||||||
|
if (value instanceof Number number) {
|
||||||
|
return BigDecimal.valueOf(number.doubleValue());
|
||||||
|
}
|
||||||
|
return new BigDecimal(value.toString());
|
||||||
|
}
|
||||||
|
|
||||||
|
private Boolean booleanValue(ResultSet rs, String column) throws SQLException {
|
||||||
|
Object value = rs.getObject(column);
|
||||||
|
if (value == null) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
if (value instanceof Boolean bool) {
|
||||||
|
return bool;
|
||||||
|
}
|
||||||
|
if (value instanceof Number number) {
|
||||||
|
return number.intValue() != 0;
|
||||||
|
}
|
||||||
|
return Boolean.parseBoolean(value.toString());
|
||||||
|
}
|
||||||
|
|
||||||
|
private EventType eventType(ResultSet rs) throws SQLException {
|
||||||
|
return parseEnum(EventType.class, string(rs, "event_type"), EventType.POSITION_RECORDED);
|
||||||
|
}
|
||||||
|
|
||||||
|
private EventLifecycle lifecycle(ResultSet rs) throws SQLException {
|
||||||
|
return parseEnum(EventLifecycle.class, string(rs, "lifecycle"), EventLifecycle.SNAPSHOT);
|
||||||
|
}
|
||||||
|
|
||||||
|
private <T extends Enum<T>> T parseEnum(Class<T> type, String value, T fallback) {
|
||||||
|
if (value == null) {
|
||||||
|
return fallback;
|
||||||
|
}
|
||||||
|
String normalized = value.trim().toUpperCase(Locale.ROOT).replace('-', '_').replace(' ', '_');
|
||||||
|
try {
|
||||||
|
return Enum.valueOf(type, normalized);
|
||||||
|
} catch (IllegalArgumentException ignored) {
|
||||||
|
return fallback;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
protected void put(Map<String, Object> target, String key, Object value) {
|
||||||
|
if (value != null) {
|
||||||
|
target.put(key, value);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,201 @@
|
||||||
|
package at.procon.eventhub.tachograph.service;
|
||||||
|
|
||||||
|
import at.procon.eventhub.dto.DriverCardRefDto;
|
||||||
|
import at.procon.eventhub.dto.DriverRefDto;
|
||||||
|
import at.procon.eventhub.dto.EventDomain;
|
||||||
|
import at.procon.eventhub.dto.EventHubEventDto;
|
||||||
|
import at.procon.eventhub.dto.EventLifecycle;
|
||||||
|
import at.procon.eventhub.dto.EventType;
|
||||||
|
import at.procon.eventhub.dto.GeoPointDto;
|
||||||
|
import at.procon.eventhub.dto.SourcePackageRefDto;
|
||||||
|
import at.procon.eventhub.dto.VehicleRefDto;
|
||||||
|
import at.procon.eventhub.dto.VehicleRegistrationRefDto;
|
||||||
|
import at.procon.eventhub.importing.extraction.ExtractionContext;
|
||||||
|
import at.procon.eventhub.importing.extraction.ExtractionRowMapper;
|
||||||
|
import at.procon.eventhub.service.EventDetailsFactory;
|
||||||
|
import at.procon.eventhub.tachograph.dto.TachographImportRequest;
|
||||||
|
import java.math.BigDecimal;
|
||||||
|
import java.sql.ResultSet;
|
||||||
|
import java.sql.SQLException;
|
||||||
|
import java.sql.Timestamp;
|
||||||
|
import java.time.LocalDateTime;
|
||||||
|
import java.time.OffsetDateTime;
|
||||||
|
import java.time.ZoneOffset;
|
||||||
|
import java.util.LinkedHashMap;
|
||||||
|
import java.util.Map;
|
||||||
|
import java.util.UUID;
|
||||||
|
|
||||||
|
abstract class AbstractTachographPositionRowMapper implements ExtractionRowMapper<TachographImportRequest> {
|
||||||
|
|
||||||
|
private final EventDetailsFactory detailsFactory;
|
||||||
|
|
||||||
|
protected AbstractTachographPositionRowMapper(EventDetailsFactory detailsFactory) {
|
||||||
|
this.detailsFactory = detailsFactory;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public EventHubEventDto map(ResultSet rs, int rowNum, ExtractionContext<TachographImportRequest> context) throws SQLException {
|
||||||
|
OffsetDateTime occurredAt = offsetDateTime(rs, "occurred_at");
|
||||||
|
SourcePackageRefDto sourcePackageRef = sourcePackageRef(rs);
|
||||||
|
DriverRefDto driverRef = driverRef(rs);
|
||||||
|
VehicleRefDto vehicleRef = vehicleRef(rs);
|
||||||
|
|
||||||
|
String externalSourceEventId = string(rs, "external_source_event_id");
|
||||||
|
if (externalSourceEventId == null) {
|
||||||
|
externalSourceEventId = defaultExternalSourceEventId(context, rowNum, occurredAt, sourcePackageRef, driverRef, vehicleRef);
|
||||||
|
}
|
||||||
|
|
||||||
|
return new EventHubEventDto(
|
||||||
|
UUID.randomUUID(),
|
||||||
|
externalSourceEventId,
|
||||||
|
driverRef,
|
||||||
|
vehicleRef,
|
||||||
|
occurredAt,
|
||||||
|
offsetDateTime(rs, "received_partner_at"),
|
||||||
|
OffsetDateTime.now(),
|
||||||
|
EventDomain.POSITION,
|
||||||
|
EventType.POSITION_RECORDED,
|
||||||
|
EventLifecycle.SNAPSHOT,
|
||||||
|
longValue(rs, "odometer_m"),
|
||||||
|
position(rs),
|
||||||
|
detailsFactory.position("GNSS_ACCUMULATED_DRIVING"),
|
||||||
|
sourcePackageRef != null && sourcePackageRef.hasAnyReference() ? sourcePackageRef : null,
|
||||||
|
detailsFactory.payloadFromMap(payload(rs)),
|
||||||
|
false,
|
||||||
|
context.packageInfo()
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
protected Map<String, Object> sourceSpecificPayload(ResultSet rs) throws SQLException {
|
||||||
|
return Map.of();
|
||||||
|
}
|
||||||
|
|
||||||
|
private DriverRefDto driverRef(ResultSet rs) throws SQLException {
|
||||||
|
DriverCardRefDto driverCard = null;
|
||||||
|
String cardNumber = string(rs, "driver_card_number");
|
||||||
|
if (cardNumber != null) {
|
||||||
|
driverCard = new DriverCardRefDto(string(rs, "driver_card_nation"), cardNumber);
|
||||||
|
}
|
||||||
|
DriverRefDto driverRef = new DriverRefDto(string(rs, "driver_source_entity_id"), driverCard);
|
||||||
|
return driverRef.hasAnyReference() ? driverRef : null;
|
||||||
|
}
|
||||||
|
|
||||||
|
private VehicleRefDto vehicleRef(ResultSet rs) throws SQLException {
|
||||||
|
VehicleRegistrationRefDto registration = null;
|
||||||
|
String registrationNumber = string(rs, "vehicle_registration_number");
|
||||||
|
if (registrationNumber != null) {
|
||||||
|
registration = new VehicleRegistrationRefDto(string(rs, "vehicle_registration_nation"), registrationNumber);
|
||||||
|
}
|
||||||
|
VehicleRefDto vehicleRef = new VehicleRefDto(
|
||||||
|
string(rs, "vehicle_source_entity_id"),
|
||||||
|
string(rs, "vehicle_vin"),
|
||||||
|
string(rs, "vehicle_registration_source_entity_id"),
|
||||||
|
registration
|
||||||
|
);
|
||||||
|
return vehicleRef.hasAnyReference() ? vehicleRef : null;
|
||||||
|
}
|
||||||
|
|
||||||
|
private SourcePackageRefDto sourcePackageRef(ResultSet rs) throws SQLException {
|
||||||
|
return new SourcePackageRefDto(
|
||||||
|
string(rs, "source_package_kind"),
|
||||||
|
string(rs, "source_package_id"),
|
||||||
|
string(rs, "source_package_entity_id"),
|
||||||
|
offsetDateTime(rs, "source_package_period_from"),
|
||||||
|
offsetDateTime(rs, "source_package_period_to"),
|
||||||
|
offsetDateTime(rs, "source_package_imported_at")
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
private GeoPointDto position(ResultSet rs) throws SQLException {
|
||||||
|
BigDecimal latitude = decimal(rs, "latitude");
|
||||||
|
BigDecimal longitude = decimal(rs, "longitude");
|
||||||
|
return latitude == null || longitude == null ? null : new GeoPointDto(latitude, longitude);
|
||||||
|
}
|
||||||
|
|
||||||
|
private Map<String, Object> payload(ResultSet rs) throws SQLException {
|
||||||
|
Map<String, Object> raw = new LinkedHashMap<>();
|
||||||
|
put(raw, "sourceRowId", string(rs, "source_row_id"));
|
||||||
|
raw.putAll(sourceSpecificPayload(rs));
|
||||||
|
return Map.of("raw", raw);
|
||||||
|
}
|
||||||
|
|
||||||
|
private String defaultExternalSourceEventId(
|
||||||
|
ExtractionContext<TachographImportRequest> context,
|
||||||
|
int rowNum,
|
||||||
|
OffsetDateTime occurredAt,
|
||||||
|
SourcePackageRefDto sourcePackageRef,
|
||||||
|
DriverRefDto driverRef,
|
||||||
|
VehicleRefDto vehicleRef
|
||||||
|
) {
|
||||||
|
String sourcePackageId = sourcePackageRef == null || sourcePackageRef.sourcePackageId() == null
|
||||||
|
? "NO_SOURCE_PACKAGE"
|
||||||
|
: sourcePackageRef.sourcePackageId();
|
||||||
|
String subject = driverRef != null && driverRef.hasAnyReference()
|
||||||
|
? driverRef.stableKey()
|
||||||
|
: vehicleRef == null ? "NO_SUBJECT" : vehicleRef.stableKey();
|
||||||
|
return "TACHOGRAPH:" + context.planItem().extractionCode()
|
||||||
|
+ ":" + sourcePackageId
|
||||||
|
+ ":POSITION_RECORDED"
|
||||||
|
+ ":" + occurredAt
|
||||||
|
+ ":" + subject
|
||||||
|
+ ":ROW-" + rowNum;
|
||||||
|
}
|
||||||
|
|
||||||
|
protected String string(ResultSet rs, String column) throws SQLException {
|
||||||
|
String value = rs.getString(column);
|
||||||
|
return value == null || value.isBlank() ? null : value.trim();
|
||||||
|
}
|
||||||
|
|
||||||
|
protected OffsetDateTime offsetDateTime(ResultSet rs, String column) throws SQLException {
|
||||||
|
Object value = rs.getObject(column);
|
||||||
|
if (value == null) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
if (value instanceof OffsetDateTime offsetDateTime) {
|
||||||
|
return offsetDateTime.withOffsetSameInstant(ZoneOffset.UTC);
|
||||||
|
}
|
||||||
|
if (value instanceof Timestamp timestamp) {
|
||||||
|
return timestamp.toLocalDateTime().atOffset(ZoneOffset.UTC);
|
||||||
|
}
|
||||||
|
if (value instanceof LocalDateTime localDateTime) {
|
||||||
|
return localDateTime.atOffset(ZoneOffset.UTC);
|
||||||
|
}
|
||||||
|
String text = value.toString();
|
||||||
|
try {
|
||||||
|
return OffsetDateTime.parse(text).withOffsetSameInstant(ZoneOffset.UTC);
|
||||||
|
} catch (RuntimeException ignored) {
|
||||||
|
return LocalDateTime.parse(text).atOffset(ZoneOffset.UTC);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private Long longValue(ResultSet rs, String column) throws SQLException {
|
||||||
|
Object value = rs.getObject(column);
|
||||||
|
if (value == null) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
if (value instanceof Number number) {
|
||||||
|
return number.longValue();
|
||||||
|
}
|
||||||
|
return Long.parseLong(value.toString());
|
||||||
|
}
|
||||||
|
|
||||||
|
private BigDecimal decimal(ResultSet rs, String column) throws SQLException {
|
||||||
|
Object value = rs.getObject(column);
|
||||||
|
if (value == null) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
if (value instanceof BigDecimal bigDecimal) {
|
||||||
|
return bigDecimal;
|
||||||
|
}
|
||||||
|
if (value instanceof Number number) {
|
||||||
|
return BigDecimal.valueOf(number.doubleValue());
|
||||||
|
}
|
||||||
|
return new BigDecimal(value.toString());
|
||||||
|
}
|
||||||
|
|
||||||
|
protected void put(Map<String, Object> target, String key, Object value) {
|
||||||
|
if (value != null) {
|
||||||
|
target.put(key, value);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,199 @@
|
||||||
|
package at.procon.eventhub.tachograph.service;
|
||||||
|
|
||||||
|
import at.procon.eventhub.dto.DriverCardRefDto;
|
||||||
|
import at.procon.eventhub.dto.DriverRefDto;
|
||||||
|
import at.procon.eventhub.dto.EventDomain;
|
||||||
|
import at.procon.eventhub.dto.EventHubEventDto;
|
||||||
|
import at.procon.eventhub.dto.EventLifecycle;
|
||||||
|
import at.procon.eventhub.dto.EventType;
|
||||||
|
import at.procon.eventhub.dto.SourcePackageRefDto;
|
||||||
|
import at.procon.eventhub.dto.VehicleRefDto;
|
||||||
|
import at.procon.eventhub.dto.VehicleRegistrationRefDto;
|
||||||
|
import at.procon.eventhub.importing.extraction.ExtractionContext;
|
||||||
|
import at.procon.eventhub.importing.extraction.ExtractionRowMapper;
|
||||||
|
import at.procon.eventhub.service.EventDetailsFactory;
|
||||||
|
import at.procon.eventhub.tachograph.dto.TachographImportRequest;
|
||||||
|
import java.sql.ResultSet;
|
||||||
|
import java.sql.SQLException;
|
||||||
|
import java.sql.Timestamp;
|
||||||
|
import java.time.LocalDateTime;
|
||||||
|
import java.time.OffsetDateTime;
|
||||||
|
import java.time.ZoneOffset;
|
||||||
|
import java.util.LinkedHashMap;
|
||||||
|
import java.util.Locale;
|
||||||
|
import java.util.Map;
|
||||||
|
import java.util.UUID;
|
||||||
|
|
||||||
|
abstract class AbstractTachographSpecificConditionRowMapper implements ExtractionRowMapper<TachographImportRequest> {
|
||||||
|
|
||||||
|
private final EventDetailsFactory detailsFactory;
|
||||||
|
|
||||||
|
protected AbstractTachographSpecificConditionRowMapper(EventDetailsFactory detailsFactory) {
|
||||||
|
this.detailsFactory = detailsFactory;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public EventHubEventDto map(ResultSet rs, int rowNum, ExtractionContext<TachographImportRequest> context) throws SQLException {
|
||||||
|
OffsetDateTime occurredAt = offsetDateTime(rs, "occurred_at");
|
||||||
|
SourcePackageRefDto sourcePackageRef = sourcePackageRef(rs);
|
||||||
|
DriverRefDto driverRef = driverRef(rs);
|
||||||
|
VehicleRefDto vehicleRef = vehicleRef(rs);
|
||||||
|
EventDomain eventDomain = eventDomain(rs);
|
||||||
|
EventType eventType = eventType(rs);
|
||||||
|
EventLifecycle lifecycle = lifecycle(rs);
|
||||||
|
|
||||||
|
String externalSourceEventId = string(rs, "external_source_event_id");
|
||||||
|
if (externalSourceEventId == null) {
|
||||||
|
externalSourceEventId = defaultExternalSourceEventId(context, rowNum, occurredAt, sourcePackageRef, driverRef, vehicleRef, eventType, lifecycle);
|
||||||
|
}
|
||||||
|
|
||||||
|
return new EventHubEventDto(
|
||||||
|
UUID.randomUUID(),
|
||||||
|
externalSourceEventId,
|
||||||
|
driverRef,
|
||||||
|
vehicleRef,
|
||||||
|
occurredAt,
|
||||||
|
offsetDateTime(rs, "received_partner_at"),
|
||||||
|
OffsetDateTime.now(),
|
||||||
|
eventDomain,
|
||||||
|
eventType,
|
||||||
|
lifecycle,
|
||||||
|
null,
|
||||||
|
null,
|
||||||
|
detailsFactory.specificCondition(),
|
||||||
|
sourcePackageRef != null && sourcePackageRef.hasAnyReference() ? sourcePackageRef : null,
|
||||||
|
detailsFactory.payloadFromMap(payload(rs)),
|
||||||
|
false,
|
||||||
|
context.packageInfo()
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
protected Map<String, Object> sourceSpecificPayload(ResultSet rs) throws SQLException {
|
||||||
|
return Map.of();
|
||||||
|
}
|
||||||
|
|
||||||
|
private DriverRefDto driverRef(ResultSet rs) throws SQLException {
|
||||||
|
DriverCardRefDto driverCard = null;
|
||||||
|
String cardNumber = string(rs, "driver_card_number");
|
||||||
|
if (cardNumber != null) {
|
||||||
|
driverCard = new DriverCardRefDto(string(rs, "driver_card_nation"), cardNumber);
|
||||||
|
}
|
||||||
|
DriverRefDto driverRef = new DriverRefDto(string(rs, "driver_source_entity_id"), driverCard);
|
||||||
|
return driverRef.hasAnyReference() ? driverRef : null;
|
||||||
|
}
|
||||||
|
|
||||||
|
private VehicleRefDto vehicleRef(ResultSet rs) throws SQLException {
|
||||||
|
VehicleRegistrationRefDto registration = null;
|
||||||
|
String registrationNumber = string(rs, "vehicle_registration_number");
|
||||||
|
if (registrationNumber != null) {
|
||||||
|
registration = new VehicleRegistrationRefDto(string(rs, "vehicle_registration_nation"), registrationNumber);
|
||||||
|
}
|
||||||
|
VehicleRefDto vehicleRef = new VehicleRefDto(
|
||||||
|
string(rs, "vehicle_source_entity_id"),
|
||||||
|
string(rs, "vehicle_vin"),
|
||||||
|
string(rs, "vehicle_registration_source_entity_id"),
|
||||||
|
registration
|
||||||
|
);
|
||||||
|
return vehicleRef.hasAnyReference() ? vehicleRef : null;
|
||||||
|
}
|
||||||
|
|
||||||
|
private SourcePackageRefDto sourcePackageRef(ResultSet rs) throws SQLException {
|
||||||
|
return new SourcePackageRefDto(
|
||||||
|
string(rs, "source_package_kind"),
|
||||||
|
string(rs, "source_package_id"),
|
||||||
|
string(rs, "source_package_entity_id"),
|
||||||
|
offsetDateTime(rs, "source_package_period_from"),
|
||||||
|
offsetDateTime(rs, "source_package_period_to"),
|
||||||
|
offsetDateTime(rs, "source_package_imported_at")
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
private Map<String, Object> payload(ResultSet rs) throws SQLException {
|
||||||
|
Map<String, Object> raw = new LinkedHashMap<>();
|
||||||
|
put(raw, "sourceRowId", string(rs, "source_row_id"));
|
||||||
|
raw.putAll(sourceSpecificPayload(rs));
|
||||||
|
return Map.of("raw", raw);
|
||||||
|
}
|
||||||
|
|
||||||
|
private String defaultExternalSourceEventId(
|
||||||
|
ExtractionContext<TachographImportRequest> context,
|
||||||
|
int rowNum,
|
||||||
|
OffsetDateTime occurredAt,
|
||||||
|
SourcePackageRefDto sourcePackageRef,
|
||||||
|
DriverRefDto driverRef,
|
||||||
|
VehicleRefDto vehicleRef,
|
||||||
|
EventType eventType,
|
||||||
|
EventLifecycle lifecycle
|
||||||
|
) {
|
||||||
|
String sourcePackageId = sourcePackageRef == null || sourcePackageRef.sourcePackageId() == null
|
||||||
|
? "NO_SOURCE_PACKAGE"
|
||||||
|
: sourcePackageRef.sourcePackageId();
|
||||||
|
String subject = driverRef != null && driverRef.hasAnyReference()
|
||||||
|
? driverRef.stableKey()
|
||||||
|
: vehicleRef == null ? "NO_SUBJECT" : vehicleRef.stableKey();
|
||||||
|
return "TACHOGRAPH:" + context.planItem().extractionCode()
|
||||||
|
+ ":" + sourcePackageId
|
||||||
|
+ ":" + eventType
|
||||||
|
+ ":" + lifecycle
|
||||||
|
+ ":" + occurredAt
|
||||||
|
+ ":" + subject
|
||||||
|
+ ":ROW-" + rowNum;
|
||||||
|
}
|
||||||
|
|
||||||
|
protected String string(ResultSet rs, String column) throws SQLException {
|
||||||
|
String value = rs.getString(column);
|
||||||
|
return value == null || value.isBlank() ? null : value.trim();
|
||||||
|
}
|
||||||
|
|
||||||
|
protected OffsetDateTime offsetDateTime(ResultSet rs, String column) throws SQLException {
|
||||||
|
Object value = rs.getObject(column);
|
||||||
|
if (value == null) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
if (value instanceof OffsetDateTime offsetDateTime) {
|
||||||
|
return offsetDateTime.withOffsetSameInstant(ZoneOffset.UTC);
|
||||||
|
}
|
||||||
|
if (value instanceof Timestamp timestamp) {
|
||||||
|
return timestamp.toLocalDateTime().atOffset(ZoneOffset.UTC);
|
||||||
|
}
|
||||||
|
if (value instanceof LocalDateTime localDateTime) {
|
||||||
|
return localDateTime.atOffset(ZoneOffset.UTC);
|
||||||
|
}
|
||||||
|
String text = value.toString();
|
||||||
|
try {
|
||||||
|
return OffsetDateTime.parse(text).withOffsetSameInstant(ZoneOffset.UTC);
|
||||||
|
} catch (RuntimeException ignored) {
|
||||||
|
return LocalDateTime.parse(text).atOffset(ZoneOffset.UTC);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private EventDomain eventDomain(ResultSet rs) throws SQLException {
|
||||||
|
return parseEnum(EventDomain.class, string(rs, "event_domain"), EventDomain.SPECIFIC_CONDITION);
|
||||||
|
}
|
||||||
|
|
||||||
|
private EventType eventType(ResultSet rs) throws SQLException {
|
||||||
|
return parseEnum(EventType.class, string(rs, "event_type"), EventType.UNKNOWN_EVENT);
|
||||||
|
}
|
||||||
|
|
||||||
|
private EventLifecycle lifecycle(ResultSet rs) throws SQLException {
|
||||||
|
return parseEnum(EventLifecycle.class, string(rs, "lifecycle"), EventLifecycle.SNAPSHOT);
|
||||||
|
}
|
||||||
|
|
||||||
|
private <T extends Enum<T>> T parseEnum(Class<T> type, String value, T fallback) {
|
||||||
|
if (value == null) {
|
||||||
|
return fallback;
|
||||||
|
}
|
||||||
|
String normalized = value.trim().toUpperCase(Locale.ROOT).replace('-', '_').replace(' ', '_');
|
||||||
|
try {
|
||||||
|
return Enum.valueOf(type, normalized);
|
||||||
|
} catch (IllegalArgumentException ignored) {
|
||||||
|
return fallback;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
protected void put(Map<String, Object> target, String key, Object value) {
|
||||||
|
if (value != null) {
|
||||||
|
target.put(key, value);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -1,10 +1,6 @@
|
||||||
package at.procon.eventhub.tachograph.service;
|
package at.procon.eventhub.tachograph.service;
|
||||||
|
|
||||||
import at.procon.eventhub.service.EventDetailsFactory;
|
import at.procon.eventhub.service.EventDetailsFactory;
|
||||||
import java.sql.ResultSet;
|
|
||||||
import java.sql.SQLException;
|
|
||||||
import java.util.LinkedHashMap;
|
|
||||||
import java.util.Map;
|
|
||||||
import org.springframework.stereotype.Component;
|
import org.springframework.stereotype.Component;
|
||||||
|
|
||||||
@Component
|
@Component
|
||||||
|
|
@ -13,11 +9,4 @@ public class CardActivityRowMapper extends AbstractTachographActivityRowMapper {
|
||||||
public CardActivityRowMapper(EventDetailsFactory detailsFactory) {
|
public CardActivityRowMapper(EventDetailsFactory detailsFactory) {
|
||||||
super(detailsFactory);
|
super(detailsFactory);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
|
||||||
protected Map<String, Object> sourceSpecificPayload(ResultSet rs) throws SQLException {
|
|
||||||
Map<String, Object> raw = new LinkedHashMap<>();
|
|
||||||
put(raw, "cardActivityId", string(rs, "card_activity_id"));
|
|
||||||
return raw;
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,12 @@
|
||||||
|
package at.procon.eventhub.tachograph.service;
|
||||||
|
|
||||||
|
import at.procon.eventhub.service.EventDetailsFactory;
|
||||||
|
import org.springframework.stereotype.Component;
|
||||||
|
|
||||||
|
@Component
|
||||||
|
public class CardBorderCrossingRowMapper extends AbstractTachographBorderCrossingRowMapper {
|
||||||
|
|
||||||
|
public CardBorderCrossingRowMapper(EventDetailsFactory detailsFactory) {
|
||||||
|
super(detailsFactory);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,12 @@
|
||||||
|
package at.procon.eventhub.tachograph.service;
|
||||||
|
|
||||||
|
import at.procon.eventhub.service.EventDetailsFactory;
|
||||||
|
import org.springframework.stereotype.Component;
|
||||||
|
|
||||||
|
@Component
|
||||||
|
public class CardLoadUnloadRowMapper extends AbstractTachographLoadUnloadRowMapper {
|
||||||
|
|
||||||
|
public CardLoadUnloadRowMapper(EventDetailsFactory detailsFactory) {
|
||||||
|
super(detailsFactory);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,12 @@
|
||||||
|
package at.procon.eventhub.tachograph.service;
|
||||||
|
|
||||||
|
import at.procon.eventhub.service.EventDetailsFactory;
|
||||||
|
import org.springframework.stereotype.Component;
|
||||||
|
|
||||||
|
@Component
|
||||||
|
public class CardPlaceRowMapper extends AbstractTachographPlaceRowMapper {
|
||||||
|
|
||||||
|
public CardPlaceRowMapper(EventDetailsFactory detailsFactory) {
|
||||||
|
super(detailsFactory);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,12 @@
|
||||||
|
package at.procon.eventhub.tachograph.service;
|
||||||
|
|
||||||
|
import at.procon.eventhub.service.EventDetailsFactory;
|
||||||
|
import org.springframework.stereotype.Component;
|
||||||
|
|
||||||
|
@Component
|
||||||
|
public class CardPositionRowMapper extends AbstractTachographPositionRowMapper {
|
||||||
|
|
||||||
|
public CardPositionRowMapper(EventDetailsFactory detailsFactory) {
|
||||||
|
super(detailsFactory);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,12 @@
|
||||||
|
package at.procon.eventhub.tachograph.service;
|
||||||
|
|
||||||
|
import at.procon.eventhub.service.EventDetailsFactory;
|
||||||
|
import org.springframework.stereotype.Component;
|
||||||
|
|
||||||
|
@Component
|
||||||
|
public class CardSpecificConditionRowMapper extends AbstractTachographSpecificConditionRowMapper {
|
||||||
|
|
||||||
|
public CardSpecificConditionRowMapper(EventDetailsFactory detailsFactory) {
|
||||||
|
super(detailsFactory);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,12 @@
|
||||||
|
package at.procon.eventhub.tachograph.service;
|
||||||
|
|
||||||
|
import at.procon.eventhub.service.EventDetailsFactory;
|
||||||
|
import org.springframework.stereotype.Component;
|
||||||
|
|
||||||
|
@Component
|
||||||
|
public class CardVehiclesUsedCardEventRowMapper extends AbstractTachographCardEventRowMapper {
|
||||||
|
|
||||||
|
public CardVehiclesUsedCardEventRowMapper(EventDetailsFactory detailsFactory) {
|
||||||
|
super(detailsFactory);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,12 @@
|
||||||
|
package at.procon.eventhub.tachograph.service;
|
||||||
|
|
||||||
|
import at.procon.eventhub.service.EventDetailsFactory;
|
||||||
|
import org.springframework.stereotype.Component;
|
||||||
|
|
||||||
|
@Component
|
||||||
|
public class IwCycleCardEventRowMapper extends AbstractTachographCardEventRowMapper {
|
||||||
|
|
||||||
|
public IwCycleCardEventRowMapper(EventDetailsFactory detailsFactory) {
|
||||||
|
super(detailsFactory);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -1,8 +1,8 @@
|
||||||
package at.procon.eventhub.tachograph.service;
|
package at.procon.eventhub.tachograph.service;
|
||||||
|
|
||||||
import at.procon.eventhub.dto.EventHubEventDto;
|
|
||||||
import at.procon.eventhub.dto.EventSourceDto;
|
import at.procon.eventhub.dto.EventSourceDto;
|
||||||
import at.procon.eventhub.dto.ImportCursorStateDto;
|
import at.procon.eventhub.dto.ImportCursorStateDto;
|
||||||
|
import at.procon.eventhub.dto.ImportScopeDto;
|
||||||
import at.procon.eventhub.importing.ImportPlanItemDto;
|
import at.procon.eventhub.importing.ImportPlanItemDto;
|
||||||
import at.procon.eventhub.importing.ImportTimeChunkDto;
|
import at.procon.eventhub.importing.ImportTimeChunkDto;
|
||||||
import at.procon.eventhub.importing.extraction.AbstractJdbcExtractionBatchExecutor;
|
import at.procon.eventhub.importing.extraction.AbstractJdbcExtractionBatchExecutor;
|
||||||
|
|
@ -10,7 +10,10 @@ import at.procon.eventhub.importing.extraction.ExtractionDefinition;
|
||||||
import at.procon.eventhub.importing.persistence.ImportCursorRepository;
|
import at.procon.eventhub.importing.persistence.ImportCursorRepository;
|
||||||
import at.procon.eventhub.tachograph.dto.TachographExtractionBatchResultDto;
|
import at.procon.eventhub.tachograph.dto.TachographExtractionBatchResultDto;
|
||||||
import at.procon.eventhub.tachograph.dto.TachographImportRequest;
|
import at.procon.eventhub.tachograph.dto.TachographImportRequest;
|
||||||
import java.util.List;
|
import java.time.LocalDateTime;
|
||||||
|
import java.time.OffsetDateTime;
|
||||||
|
import java.time.ZoneOffset;
|
||||||
|
import java.util.Map;
|
||||||
import java.util.Optional;
|
import java.util.Optional;
|
||||||
import java.util.UUID;
|
import java.util.UUID;
|
||||||
import org.apache.camel.ProducerTemplate;
|
import org.apache.camel.ProducerTemplate;
|
||||||
|
|
@ -46,27 +49,43 @@ public class JdbcTachographExtractionBatchExecutor
|
||||||
return definitionRegistry.findByCode(code);
|
return definitionRegistry.findByCode(code);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
protected Map<String, Object> parameters(
|
||||||
|
TachographImportRequest request,
|
||||||
|
ImportScopeDto scope,
|
||||||
|
ImportCursorStateDto cursor
|
||||||
|
) {
|
||||||
|
Map<String, Object> params = super.parameters(request, scope, cursor);
|
||||||
|
params.put("occurredFrom", utcLocalDateTime(scope == null ? null : scope.occurredFrom()));
|
||||||
|
params.put("occurredTo", utcLocalDateTime(scope == null ? null : scope.occurredTo()));
|
||||||
|
params.put("lastSourcePackageImportedAt", utcLocalDateTime(cursor == null ? null : cursor.lastSourcePackageImportedAt()));
|
||||||
|
params.put("lastSourceRowUpdatedAt", utcLocalDateTime(cursor == null ? null : cursor.lastSourceRowUpdatedAt()));
|
||||||
|
params.put("lastOccurredTo", utcLocalDateTime(cursor == null ? null : cursor.lastOccurredTo()));
|
||||||
|
return params;
|
||||||
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
protected TachographExtractionBatchResultDto resultFor(
|
protected TachographExtractionBatchResultDto resultFor(
|
||||||
UUID packageId,
|
UUID packageId,
|
||||||
ImportPlanItemDto planItem,
|
ImportPlanItemDto planItem,
|
||||||
ImportTimeChunkDto chunk,
|
ImportTimeChunkDto chunk,
|
||||||
ImportCursorStateDto cursor,
|
ImportCursorStateDto cursor,
|
||||||
List<EventHubEventDto> events
|
ExtractedEventStats stats
|
||||||
) {
|
) {
|
||||||
return new TachographExtractionBatchResultDto(
|
return new TachographExtractionBatchResultDto(
|
||||||
packageId,
|
packageId,
|
||||||
planItem.extractionCode(),
|
planItem.extractionCode(),
|
||||||
planItem.sourceKind(),
|
planItem.sourceKind(),
|
||||||
events.size(),
|
stats.eventsMapped(),
|
||||||
events.size(),
|
stats.eventsMapped(),
|
||||||
events.size(),
|
stats.eventsMapped(),
|
||||||
0,
|
0,
|
||||||
true,
|
true,
|
||||||
lastSourcePackageImportedAt(events, cursor),
|
lastSourcePackageImportedAt(stats, cursor),
|
||||||
lastSourcePackageId(events, cursor),
|
lastSourcePackageId(stats, cursor),
|
||||||
null,
|
null,
|
||||||
chunk.occurredTo()
|
chunk.occurredTo(),
|
||||||
|
stats.eventTypeCounts()
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -91,4 +110,8 @@ public class JdbcTachographExtractionBatchExecutor
|
||||||
protected String providerPackagePrefix() {
|
protected String providerPackagePrefix() {
|
||||||
return "TACHOGRAPH";
|
return "TACHOGRAPH";
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private LocalDateTime utcLocalDateTime(OffsetDateTime value) {
|
||||||
|
return value == null ? null : value.withOffsetSameInstant(ZoneOffset.UTC).toLocalDateTime();
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -42,7 +42,8 @@ public class NoopTachographExtractionBatchExecutor
|
||||||
null,
|
null,
|
||||||
null,
|
null,
|
||||||
null,
|
null,
|
||||||
null
|
null,
|
||||||
|
java.util.Map.of()
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,200 @@
|
||||||
|
package at.procon.eventhub.tachograph.service;
|
||||||
|
|
||||||
|
import at.procon.eventhub.dto.DriverCardRefDto;
|
||||||
|
import at.procon.eventhub.dto.DriverRefDto;
|
||||||
|
import at.procon.eventhub.dto.EventDomain;
|
||||||
|
import at.procon.eventhub.dto.EventHubEventDto;
|
||||||
|
import at.procon.eventhub.dto.EventLifecycle;
|
||||||
|
import at.procon.eventhub.dto.EventType;
|
||||||
|
import at.procon.eventhub.dto.SourcePackageRefDto;
|
||||||
|
import at.procon.eventhub.dto.VehicleRefDto;
|
||||||
|
import at.procon.eventhub.dto.VehicleRegistrationRefDto;
|
||||||
|
import at.procon.eventhub.importing.extraction.ExtractionContext;
|
||||||
|
import at.procon.eventhub.importing.extraction.ExtractionRowMapper;
|
||||||
|
import at.procon.eventhub.service.EventDetailsFactory;
|
||||||
|
import at.procon.eventhub.tachograph.dto.TachographImportRequest;
|
||||||
|
import java.math.BigDecimal;
|
||||||
|
import java.sql.ResultSet;
|
||||||
|
import java.sql.SQLException;
|
||||||
|
import java.sql.Timestamp;
|
||||||
|
import java.time.LocalDateTime;
|
||||||
|
import java.time.OffsetDateTime;
|
||||||
|
import java.time.ZoneOffset;
|
||||||
|
import java.util.LinkedHashMap;
|
||||||
|
import java.util.Locale;
|
||||||
|
import java.util.Map;
|
||||||
|
import java.util.UUID;
|
||||||
|
import org.springframework.stereotype.Component;
|
||||||
|
|
||||||
|
@Component
|
||||||
|
public class SpeedingEventRowMapper implements ExtractionRowMapper<TachographImportRequest> {
|
||||||
|
|
||||||
|
private final EventDetailsFactory detailsFactory;
|
||||||
|
|
||||||
|
public SpeedingEventRowMapper(EventDetailsFactory detailsFactory) {
|
||||||
|
this.detailsFactory = detailsFactory;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public EventHubEventDto map(ResultSet rs, int rowNum, ExtractionContext<TachographImportRequest> context) throws SQLException {
|
||||||
|
OffsetDateTime occurredAt = offsetDateTime(rs, "occurred_at");
|
||||||
|
SourcePackageRefDto sourcePackageRef = sourcePackageRef(rs);
|
||||||
|
DriverRefDto driverRef = driverRef(rs);
|
||||||
|
VehicleRefDto vehicleRef = vehicleRef(rs);
|
||||||
|
EventLifecycle lifecycle = lifecycle(rs);
|
||||||
|
|
||||||
|
String externalSourceEventId = string(rs, "external_source_event_id");
|
||||||
|
if (externalSourceEventId == null) {
|
||||||
|
externalSourceEventId = defaultExternalSourceEventId(context, rowNum, occurredAt, sourcePackageRef, driverRef, vehicleRef, lifecycle);
|
||||||
|
}
|
||||||
|
|
||||||
|
return new EventHubEventDto(
|
||||||
|
UUID.randomUUID(),
|
||||||
|
externalSourceEventId,
|
||||||
|
driverRef,
|
||||||
|
vehicleRef,
|
||||||
|
occurredAt,
|
||||||
|
offsetDateTime(rs, "received_partner_at"),
|
||||||
|
OffsetDateTime.now(),
|
||||||
|
EventDomain.SPEEDING,
|
||||||
|
EventType.SPEEDING,
|
||||||
|
lifecycle,
|
||||||
|
null,
|
||||||
|
null,
|
||||||
|
detailsFactory.speeding(decimal(rs, "avg_speed_kmh"), decimal(rs, "max_speed_kmh"), null),
|
||||||
|
sourcePackageRef != null && sourcePackageRef.hasAnyReference() ? sourcePackageRef : null,
|
||||||
|
detailsFactory.payloadFromMap(payload(rs)),
|
||||||
|
false,
|
||||||
|
context.packageInfo()
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
private DriverRefDto driverRef(ResultSet rs) throws SQLException {
|
||||||
|
DriverCardRefDto driverCard = null;
|
||||||
|
String cardNumber = string(rs, "driver_card_number");
|
||||||
|
if (cardNumber != null) {
|
||||||
|
driverCard = new DriverCardRefDto(string(rs, "driver_card_nation"), cardNumber);
|
||||||
|
}
|
||||||
|
DriverRefDto driverRef = new DriverRefDto(string(rs, "driver_source_entity_id"), driverCard);
|
||||||
|
return driverRef.hasAnyReference() ? driverRef : null;
|
||||||
|
}
|
||||||
|
|
||||||
|
private VehicleRefDto vehicleRef(ResultSet rs) throws SQLException {
|
||||||
|
VehicleRegistrationRefDto registration = null;
|
||||||
|
String registrationNumber = string(rs, "vehicle_registration_number");
|
||||||
|
if (registrationNumber != null) {
|
||||||
|
registration = new VehicleRegistrationRefDto(string(rs, "vehicle_registration_nation"), registrationNumber);
|
||||||
|
}
|
||||||
|
VehicleRefDto vehicleRef = new VehicleRefDto(
|
||||||
|
string(rs, "vehicle_source_entity_id"),
|
||||||
|
string(rs, "vehicle_vin"),
|
||||||
|
string(rs, "vehicle_registration_source_entity_id"),
|
||||||
|
registration
|
||||||
|
);
|
||||||
|
return vehicleRef.hasAnyReference() ? vehicleRef : null;
|
||||||
|
}
|
||||||
|
|
||||||
|
private SourcePackageRefDto sourcePackageRef(ResultSet rs) throws SQLException {
|
||||||
|
return new SourcePackageRefDto(
|
||||||
|
string(rs, "source_package_kind"),
|
||||||
|
string(rs, "source_package_id"),
|
||||||
|
string(rs, "source_package_entity_id"),
|
||||||
|
offsetDateTime(rs, "source_package_period_from"),
|
||||||
|
offsetDateTime(rs, "source_package_period_to"),
|
||||||
|
offsetDateTime(rs, "source_package_imported_at")
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
private Map<String, Object> payload(ResultSet rs) throws SQLException {
|
||||||
|
Map<String, Object> raw = new LinkedHashMap<>();
|
||||||
|
put(raw, "sourceRowId", string(rs, "source_row_id"));
|
||||||
|
return Map.of("raw", raw);
|
||||||
|
}
|
||||||
|
|
||||||
|
private String defaultExternalSourceEventId(
|
||||||
|
ExtractionContext<TachographImportRequest> context,
|
||||||
|
int rowNum,
|
||||||
|
OffsetDateTime occurredAt,
|
||||||
|
SourcePackageRefDto sourcePackageRef,
|
||||||
|
DriverRefDto driverRef,
|
||||||
|
VehicleRefDto vehicleRef,
|
||||||
|
EventLifecycle lifecycle
|
||||||
|
) {
|
||||||
|
String sourcePackageId = sourcePackageRef == null || sourcePackageRef.sourcePackageId() == null
|
||||||
|
? "NO_SOURCE_PACKAGE"
|
||||||
|
: sourcePackageRef.sourcePackageId();
|
||||||
|
String subject = driverRef != null && driverRef.hasAnyReference()
|
||||||
|
? driverRef.stableKey()
|
||||||
|
: vehicleRef == null ? "NO_SUBJECT" : vehicleRef.stableKey();
|
||||||
|
return "TACHOGRAPH:" + context.planItem().extractionCode()
|
||||||
|
+ ":" + sourcePackageId
|
||||||
|
+ ":SPEEDING"
|
||||||
|
+ ":" + lifecycle
|
||||||
|
+ ":" + occurredAt
|
||||||
|
+ ":" + subject
|
||||||
|
+ ":ROW-" + rowNum;
|
||||||
|
}
|
||||||
|
|
||||||
|
private String string(ResultSet rs, String column) throws SQLException {
|
||||||
|
String value = rs.getString(column);
|
||||||
|
return value == null || value.isBlank() ? null : value.trim();
|
||||||
|
}
|
||||||
|
|
||||||
|
private OffsetDateTime offsetDateTime(ResultSet rs, String column) throws SQLException {
|
||||||
|
Object value = rs.getObject(column);
|
||||||
|
if (value == null) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
if (value instanceof OffsetDateTime offsetDateTime) {
|
||||||
|
return offsetDateTime.withOffsetSameInstant(ZoneOffset.UTC);
|
||||||
|
}
|
||||||
|
if (value instanceof Timestamp timestamp) {
|
||||||
|
return timestamp.toLocalDateTime().atOffset(ZoneOffset.UTC);
|
||||||
|
}
|
||||||
|
if (value instanceof LocalDateTime localDateTime) {
|
||||||
|
return localDateTime.atOffset(ZoneOffset.UTC);
|
||||||
|
}
|
||||||
|
String text = value.toString();
|
||||||
|
try {
|
||||||
|
return OffsetDateTime.parse(text).withOffsetSameInstant(ZoneOffset.UTC);
|
||||||
|
} catch (RuntimeException ignored) {
|
||||||
|
return LocalDateTime.parse(text).atOffset(ZoneOffset.UTC);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private BigDecimal decimal(ResultSet rs, String column) throws SQLException {
|
||||||
|
Object value = rs.getObject(column);
|
||||||
|
if (value == null) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
if (value instanceof BigDecimal bigDecimal) {
|
||||||
|
return bigDecimal;
|
||||||
|
}
|
||||||
|
if (value instanceof Number number) {
|
||||||
|
return BigDecimal.valueOf(number.doubleValue());
|
||||||
|
}
|
||||||
|
return new BigDecimal(value.toString());
|
||||||
|
}
|
||||||
|
|
||||||
|
private EventLifecycle lifecycle(ResultSet rs) throws SQLException {
|
||||||
|
return parseEnum(EventLifecycle.class, string(rs, "lifecycle"), EventLifecycle.BEGIN);
|
||||||
|
}
|
||||||
|
|
||||||
|
private <T extends Enum<T>> T parseEnum(Class<T> type, String value, T fallback) {
|
||||||
|
if (value == null) {
|
||||||
|
return fallback;
|
||||||
|
}
|
||||||
|
String normalized = value.trim().toUpperCase(Locale.ROOT).replace('-', '_').replace(' ', '_');
|
||||||
|
try {
|
||||||
|
return Enum.valueOf(type, normalized);
|
||||||
|
} catch (IllegalArgumentException ignored) {
|
||||||
|
return fallback;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private void put(Map<String, Object> target, String key, Object value) {
|
||||||
|
if (value != null) {
|
||||||
|
target.put(key, value);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -12,7 +12,20 @@ public class TachographExtractionDefinitionRegistry extends ExtractionDefinition
|
||||||
|
|
||||||
public TachographExtractionDefinitionRegistry(
|
public TachographExtractionDefinitionRegistry(
|
||||||
CardActivityRowMapper cardActivityRowMapper,
|
CardActivityRowMapper cardActivityRowMapper,
|
||||||
VuActivityRowMapper vuActivityRowMapper
|
VuActivityRowMapper vuActivityRowMapper,
|
||||||
|
IwCycleCardEventRowMapper iwCycleCardEventRowMapper,
|
||||||
|
CardVehiclesUsedCardEventRowMapper cardVehiclesUsedCardEventRowMapper,
|
||||||
|
VuBorderCrossingRowMapper vuBorderCrossingRowMapper,
|
||||||
|
CardBorderCrossingRowMapper cardBorderCrossingRowMapper,
|
||||||
|
VuLoadUnloadRowMapper vuLoadUnloadRowMapper,
|
||||||
|
CardLoadUnloadRowMapper cardLoadUnloadRowMapper,
|
||||||
|
VuSpecificConditionRowMapper vuSpecificConditionRowMapper,
|
||||||
|
CardSpecificConditionRowMapper cardSpecificConditionRowMapper,
|
||||||
|
VuPositionRowMapper vuPositionRowMapper,
|
||||||
|
CardPositionRowMapper cardPositionRowMapper,
|
||||||
|
VuPlaceRowMapper vuPlaceRowMapper,
|
||||||
|
CardPlaceRowMapper cardPlaceRowMapper,
|
||||||
|
SpeedingEventRowMapper speedingEventRowMapper
|
||||||
) {
|
) {
|
||||||
super(List.of(
|
super(List.of(
|
||||||
new ExtractionDefinition<>(
|
new ExtractionDefinition<>(
|
||||||
|
|
@ -30,6 +43,110 @@ public class TachographExtractionDefinitionRegistry extends ExtractionDefinition
|
||||||
"VEHICLE",
|
"VEHICLE",
|
||||||
"classpath:sql/tachograph/vu-activity.sql",
|
"classpath:sql/tachograph/vu-activity.sql",
|
||||||
vuActivityRowMapper
|
vuActivityRowMapper
|
||||||
|
),
|
||||||
|
new ExtractionDefinition<>(
|
||||||
|
"IW_CYCLE",
|
||||||
|
EventFamily.DRIVER_CARD,
|
||||||
|
"VEHICLE_UNIT",
|
||||||
|
"BOTH",
|
||||||
|
"classpath:sql/tachograph/iw-cycle.sql",
|
||||||
|
iwCycleCardEventRowMapper
|
||||||
|
),
|
||||||
|
new ExtractionDefinition<>(
|
||||||
|
"CARD_VEHICLES_USED",
|
||||||
|
EventFamily.DRIVER_CARD,
|
||||||
|
"DRIVER_CARD",
|
||||||
|
"DRIVER",
|
||||||
|
"classpath:sql/tachograph/card-vehicles-used.sql",
|
||||||
|
cardVehiclesUsedCardEventRowMapper
|
||||||
|
),
|
||||||
|
new ExtractionDefinition<>(
|
||||||
|
"VU_BORDER_CROSSING",
|
||||||
|
EventFamily.BORDER_CROSSING,
|
||||||
|
"VEHICLE_UNIT",
|
||||||
|
"VEHICLE",
|
||||||
|
"classpath:sql/tachograph/vu-border-crossing.sql",
|
||||||
|
vuBorderCrossingRowMapper
|
||||||
|
),
|
||||||
|
new ExtractionDefinition<>(
|
||||||
|
"CARD_BORDER_CROSSING",
|
||||||
|
EventFamily.BORDER_CROSSING,
|
||||||
|
"DRIVER_CARD",
|
||||||
|
"DRIVER",
|
||||||
|
"classpath:sql/tachograph/card-border-crossing.sql",
|
||||||
|
cardBorderCrossingRowMapper
|
||||||
|
),
|
||||||
|
new ExtractionDefinition<>(
|
||||||
|
"VU_LOAD_UNLOAD",
|
||||||
|
EventFamily.LOAD_UNLOAD,
|
||||||
|
"VEHICLE_UNIT",
|
||||||
|
"VEHICLE",
|
||||||
|
"classpath:sql/tachograph/vu-load-unload.sql",
|
||||||
|
vuLoadUnloadRowMapper
|
||||||
|
),
|
||||||
|
new ExtractionDefinition<>(
|
||||||
|
"CARD_LOAD_UNLOAD",
|
||||||
|
EventFamily.LOAD_UNLOAD,
|
||||||
|
"DRIVER_CARD",
|
||||||
|
"DRIVER",
|
||||||
|
"classpath:sql/tachograph/card-load-unload.sql",
|
||||||
|
cardLoadUnloadRowMapper
|
||||||
|
),
|
||||||
|
new ExtractionDefinition<>(
|
||||||
|
"VU_SPECIFIC_CONDITION",
|
||||||
|
EventFamily.SPECIFIC_CONDITION,
|
||||||
|
"VEHICLE_UNIT",
|
||||||
|
"VEHICLE",
|
||||||
|
"classpath:sql/tachograph/vu-specific-condition.sql",
|
||||||
|
vuSpecificConditionRowMapper
|
||||||
|
),
|
||||||
|
new ExtractionDefinition<>(
|
||||||
|
"CARD_SPECIFIC_CONDITION",
|
||||||
|
EventFamily.SPECIFIC_CONDITION,
|
||||||
|
"DRIVER_CARD",
|
||||||
|
"DRIVER",
|
||||||
|
"classpath:sql/tachograph/card-specific-condition.sql",
|
||||||
|
cardSpecificConditionRowMapper
|
||||||
|
),
|
||||||
|
new ExtractionDefinition<>(
|
||||||
|
"VU_POSITION",
|
||||||
|
EventFamily.POSITION,
|
||||||
|
"VEHICLE_UNIT",
|
||||||
|
"VEHICLE",
|
||||||
|
"classpath:sql/tachograph/vu-position.sql",
|
||||||
|
vuPositionRowMapper
|
||||||
|
),
|
||||||
|
new ExtractionDefinition<>(
|
||||||
|
"CARD_POSITION",
|
||||||
|
EventFamily.POSITION,
|
||||||
|
"DRIVER_CARD",
|
||||||
|
"DRIVER",
|
||||||
|
"classpath:sql/tachograph/card-position.sql",
|
||||||
|
cardPositionRowMapper
|
||||||
|
),
|
||||||
|
new ExtractionDefinition<>(
|
||||||
|
"VU_PLACE",
|
||||||
|
EventFamily.PLACE,
|
||||||
|
"VEHICLE_UNIT",
|
||||||
|
"VEHICLE",
|
||||||
|
"classpath:sql/tachograph/vu-place.sql",
|
||||||
|
vuPlaceRowMapper
|
||||||
|
),
|
||||||
|
new ExtractionDefinition<>(
|
||||||
|
"CARD_PLACE",
|
||||||
|
EventFamily.PLACE,
|
||||||
|
"DRIVER_CARD",
|
||||||
|
"DRIVER",
|
||||||
|
"classpath:sql/tachograph/card-place.sql",
|
||||||
|
cardPlaceRowMapper
|
||||||
|
),
|
||||||
|
new ExtractionDefinition<>(
|
||||||
|
"SPEEDING_EVENTS",
|
||||||
|
EventFamily.SPEEDING,
|
||||||
|
"VEHICLE_UNIT",
|
||||||
|
"VEHICLE",
|
||||||
|
"classpath:sql/tachograph/speeding-events.sql",
|
||||||
|
speedingEventRowMapper
|
||||||
)
|
)
|
||||||
));
|
));
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -1,5 +1,6 @@
|
||||||
package at.procon.eventhub.tachograph.service;
|
package at.procon.eventhub.tachograph.service;
|
||||||
|
|
||||||
|
import at.procon.eventhub.config.EventHubProperties;
|
||||||
import at.procon.eventhub.dto.EventSourceDto;
|
import at.procon.eventhub.dto.EventSourceDto;
|
||||||
import at.procon.eventhub.importing.AbstractImportExecutionService;
|
import at.procon.eventhub.importing.AbstractImportExecutionService;
|
||||||
import at.procon.eventhub.importing.ImportPlanDto;
|
import at.procon.eventhub.importing.ImportPlanDto;
|
||||||
|
|
@ -32,10 +33,11 @@ public class TachographImportExecutionService
|
||||||
ImportRunRepository importRunRepository,
|
ImportRunRepository importRunRepository,
|
||||||
DataPackageRepository dataPackageRepository,
|
DataPackageRepository dataPackageRepository,
|
||||||
ImportCursorRepository importCursorRepository,
|
ImportCursorRepository importCursorRepository,
|
||||||
|
EventHubProperties eventHubProperties,
|
||||||
TachographMasterDataRefreshService masterDataRefreshService,
|
TachographMasterDataRefreshService masterDataRefreshService,
|
||||||
TachographExtractionBatchExecutor extractionBatchExecutor
|
TachographExtractionBatchExecutor extractionBatchExecutor
|
||||||
) {
|
) {
|
||||||
super(eventSourceRepository, importRunRepository, dataPackageRepository, importCursorRepository);
|
super(eventSourceRepository, importRunRepository, dataPackageRepository, importCursorRepository, eventHubProperties);
|
||||||
this.planService = planService;
|
this.planService = planService;
|
||||||
this.masterDataRefreshService = masterDataRefreshService;
|
this.masterDataRefreshService = masterDataRefreshService;
|
||||||
this.extractionBatchExecutor = extractionBatchExecutor;
|
this.extractionBatchExecutor = extractionBatchExecutor;
|
||||||
|
|
@ -46,7 +48,6 @@ public class TachographImportExecutionService
|
||||||
return toTachographResult(createImportRun(request, false));
|
return toTachographResult(createImportRun(request, false));
|
||||||
}
|
}
|
||||||
|
|
||||||
@Transactional
|
|
||||||
public TachographImportRunResultDto startAndExecuteImport(TachographImportRequest request) {
|
public TachographImportRunResultDto startAndExecuteImport(TachographImportRequest request) {
|
||||||
return toTachographResult(createImportRun(request, true));
|
return toTachographResult(createImportRun(request, true));
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -9,6 +9,8 @@ import at.procon.eventhub.importing.ImportPlanItemDto;
|
||||||
import at.procon.eventhub.tachograph.dto.TachographImportRequest;
|
import at.procon.eventhub.tachograph.dto.TachographImportRequest;
|
||||||
import java.util.ArrayList;
|
import java.util.ArrayList;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
|
import java.util.Set;
|
||||||
|
import java.util.stream.Collectors;
|
||||||
import org.springframework.stereotype.Service;
|
import org.springframework.stereotype.Service;
|
||||||
|
|
||||||
@Service
|
@Service
|
||||||
|
|
@ -16,10 +18,16 @@ public class TachographImportPlanService {
|
||||||
|
|
||||||
private final EventHubProperties properties;
|
private final EventHubProperties properties;
|
||||||
private final ImportChunkPlanner chunkPlanner;
|
private final ImportChunkPlanner chunkPlanner;
|
||||||
|
private final TachographExtractionDefinitionRegistry definitionRegistry;
|
||||||
|
|
||||||
public TachographImportPlanService(EventHubProperties properties, ImportChunkPlanner chunkPlanner) {
|
public TachographImportPlanService(
|
||||||
|
EventHubProperties properties,
|
||||||
|
ImportChunkPlanner chunkPlanner,
|
||||||
|
TachographExtractionDefinitionRegistry definitionRegistry
|
||||||
|
) {
|
||||||
this.properties = properties;
|
this.properties = properties;
|
||||||
this.chunkPlanner = chunkPlanner;
|
this.chunkPlanner = chunkPlanner;
|
||||||
|
this.definitionRegistry = definitionRegistry;
|
||||||
}
|
}
|
||||||
|
|
||||||
public ImportPlanDto createPlan(TachographImportRequest request) {
|
public ImportPlanDto createPlan(TachographImportRequest request) {
|
||||||
|
|
@ -27,6 +35,7 @@ public class TachographImportPlanService {
|
||||||
for (EventFamily family : request.eventFamilies()) {
|
for (EventFamily family : request.eventFamilies()) {
|
||||||
items.addAll(itemsFor(family, request.acquisitionStrategy()));
|
items.addAll(itemsFor(family, request.acquisitionStrategy()));
|
||||||
}
|
}
|
||||||
|
validateJdbcExtractions(items, request.eventFamilies());
|
||||||
return new ImportPlanDto(
|
return new ImportPlanDto(
|
||||||
request.tenantKey(),
|
request.tenantKey(),
|
||||||
request.mode(),
|
request.mode(),
|
||||||
|
|
@ -51,8 +60,8 @@ public class TachographImportPlanService {
|
||||||
item(family, "DRIVER_CARD", "CARD_VEHICLES_USED", List.of("CardVehiclesUsed"), "DRIVER", "Card insert/withdraw/use events from card vehicle usage", strategy)
|
item(family, "DRIVER_CARD", "CARD_VEHICLES_USED", List.of("CardVehiclesUsed"), "DRIVER", "Card insert/withdraw/use events from card vehicle usage", strategy)
|
||||||
);
|
);
|
||||||
case POSITION -> List.of(
|
case POSITION -> List.of(
|
||||||
item(family, "VEHICLE_UNIT", "VU_POSITION", List.of("VUPlaces", "VULoadUnload", "VUGnssAccumulatedDriving", "VUBorderCrossing"), "VEHICLE", "Position points from VU tachograph sources", strategy),
|
item(family, "VEHICLE_UNIT", "VU_POSITION", List.of("VUGnssAccumulatedDriving"), "VEHICLE", "Periodic GNSS position points from VU", strategy),
|
||||||
item(family, "DRIVER_CARD", "CARD_POSITION", List.of("CardPlaces", "CardLoadUnload", "CardGnssAccumulatedDriving", "CardBorderCrossing"), "DRIVER", "Position points from driver-card tachograph sources", strategy)
|
item(family, "DRIVER_CARD", "CARD_POSITION", List.of("CardGnssAccumulatedDriving"), "DRIVER", "Periodic GNSS position points from driver card", strategy)
|
||||||
);
|
);
|
||||||
case BORDER_CROSSING -> List.of(
|
case BORDER_CROSSING -> List.of(
|
||||||
item(family, "VEHICLE_UNIT", "VU_BORDER_CROSSING", List.of("VUBorderCrossing"), "VEHICLE", "Border crossing events from VU", strategy),
|
item(family, "VEHICLE_UNIT", "VU_BORDER_CROSSING", List.of("VUBorderCrossing"), "VEHICLE", "Border crossing events from VU", strategy),
|
||||||
|
|
@ -87,4 +96,47 @@ public class TachographImportPlanService {
|
||||||
) {
|
) {
|
||||||
return new ImportPlanItemDto(family, sourceKind, extractionCode, sourceTables, entityAxis, description, strategy);
|
return new ImportPlanItemDto(family, sourceKind, extractionCode, sourceTables, entityAxis, description, strategy);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private void validateJdbcExtractions(List<ImportPlanItemDto> items, Set<EventFamily> requestedFamilies) {
|
||||||
|
String jdbcUrl = properties.getTachograph().getDatasource().getJdbcUrl();
|
||||||
|
if (jdbcUrl == null || jdbcUrl.isBlank()) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
List<String> unsupportedCodes = items.stream()
|
||||||
|
.map(ImportPlanItemDto::extractionCode)
|
||||||
|
.filter(code -> definitionRegistry.findByCode(code).isEmpty())
|
||||||
|
.distinct()
|
||||||
|
.sorted()
|
||||||
|
.toList();
|
||||||
|
if (unsupportedCodes.isEmpty()) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
List<String> supportedCodes = definitionRegistry.supportedCodes().stream()
|
||||||
|
.sorted()
|
||||||
|
.toList();
|
||||||
|
List<String> supportedFamilies = definitionRegistry.definitions().stream()
|
||||||
|
.map(definition -> definition.eventFamily().name())
|
||||||
|
.distinct()
|
||||||
|
.sorted()
|
||||||
|
.toList();
|
||||||
|
String requestedFamilyNames = requestedFamilies.stream()
|
||||||
|
.map(EventFamily::name)
|
||||||
|
.sorted()
|
||||||
|
.collect(Collectors.joining(", "));
|
||||||
|
|
||||||
|
throw new UnsupportedTachographExtractionException(
|
||||||
|
"Tachograph JDBC extraction is enabled, but the plan contains extraction codes without JDBC definitions: "
|
||||||
|
+ unsupportedCodes
|
||||||
|
+ ". Supported JDBC extraction codes are: "
|
||||||
|
+ supportedCodes
|
||||||
|
+ ". Requested event families: ["
|
||||||
|
+ requestedFamilyNames
|
||||||
|
+ "]. "
|
||||||
|
+ "Use only supported event families "
|
||||||
|
+ supportedFamilies
|
||||||
|
+ " for JDBC execution, or add the missing codes to TachographExtractionDefinitionRegistry."
|
||||||
|
);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -1,27 +1,361 @@
|
||||||
package at.procon.eventhub.tachograph.service;
|
package at.procon.eventhub.tachograph.service;
|
||||||
|
|
||||||
|
import at.procon.eventhub.importing.masterdata.MasterDataRefreshResult;
|
||||||
|
import at.procon.eventhub.importing.masterdata.SourceMasterEntityUpsert;
|
||||||
|
import at.procon.eventhub.importing.masterdata.SourceMasterRelationUpsert;
|
||||||
|
import at.procon.eventhub.dto.EventSourceDto;
|
||||||
|
import at.procon.eventhub.persistence.EventSourceRepository;
|
||||||
|
import at.procon.eventhub.persistence.SourceMasterDataRepository;
|
||||||
|
import at.procon.eventhub.persistence.VehicleIdentityRepository;
|
||||||
import at.procon.eventhub.tachograph.dto.TachographImportRequest;
|
import at.procon.eventhub.tachograph.dto.TachographImportRequest;
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.nio.charset.StandardCharsets;
|
||||||
|
import java.sql.ResultSet;
|
||||||
|
import java.sql.ResultSetMetaData;
|
||||||
|
import java.sql.SQLException;
|
||||||
|
import java.sql.Timestamp;
|
||||||
|
import java.time.LocalDateTime;
|
||||||
|
import java.time.OffsetDateTime;
|
||||||
|
import java.time.ZoneOffset;
|
||||||
|
import java.util.ArrayList;
|
||||||
|
import java.util.LinkedHashMap;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.Map;
|
||||||
import org.slf4j.Logger;
|
import org.slf4j.Logger;
|
||||||
import org.slf4j.LoggerFactory;
|
import org.slf4j.LoggerFactory;
|
||||||
|
import org.springframework.beans.factory.ObjectProvider;
|
||||||
|
import org.springframework.beans.factory.annotation.Qualifier;
|
||||||
|
import org.springframework.core.io.Resource;
|
||||||
|
import org.springframework.core.io.ResourceLoader;
|
||||||
|
import org.springframework.jdbc.core.namedparam.NamedParameterJdbcTemplate;
|
||||||
import org.springframework.stereotype.Service;
|
import org.springframework.stereotype.Service;
|
||||||
|
import org.springframework.util.StreamUtils;
|
||||||
|
|
||||||
/**
|
|
||||||
* Hook for refreshing tachograph master data before event extraction.
|
|
||||||
*
|
|
||||||
* The generated project does not yet know the concrete tachograph master-data
|
|
||||||
* schema. Replace/extend this service with SQL readers for organisations,
|
|
||||||
* vehicles, vehicle registrations, drivers, and driver cards.
|
|
||||||
*/
|
|
||||||
@Service
|
@Service
|
||||||
public class TachographMasterDataRefreshService {
|
public class TachographMasterDataRefreshService {
|
||||||
|
|
||||||
private static final Logger log = LoggerFactory.getLogger(TachographMasterDataRefreshService.class);
|
private static final Logger log = LoggerFactory.getLogger(TachographMasterDataRefreshService.class);
|
||||||
|
|
||||||
public void refreshIfRequested(TachographImportRequest request) {
|
private static final List<String> ENTITY_SQL_RESOURCES = List.of(
|
||||||
|
"classpath:sql/tachograph/master-data/organisations.sql",
|
||||||
|
"classpath:sql/tachograph/master-data/drivers.sql",
|
||||||
|
"classpath:sql/tachograph/master-data/driver-cards.sql",
|
||||||
|
"classpath:sql/tachograph/master-data/vehicles.sql",
|
||||||
|
"classpath:sql/tachograph/master-data/vehicle-registrations.sql"
|
||||||
|
);
|
||||||
|
|
||||||
|
private static final String RELATIONS_SQL_RESOURCE = "classpath:sql/tachograph/master-data/relations.sql";
|
||||||
|
private static final String MASTER_DATA_SOURCE_KIND = "MASTER_DATA";
|
||||||
|
private static final String MASTER_DATA_SOURCE_KEY = "TACHOGRAPH_MASTER_DATA";
|
||||||
|
private static final int MASTER_DATA_UPSERT_BATCH_SIZE = 5000;
|
||||||
|
|
||||||
|
private final ObjectProvider<NamedParameterJdbcTemplate> tachographJdbcTemplateProvider;
|
||||||
|
private final SourceMasterDataRepository sourceMasterDataRepository;
|
||||||
|
private final EventSourceRepository eventSourceRepository;
|
||||||
|
private final VehicleIdentityRepository vehicleIdentityRepository;
|
||||||
|
private final ResourceLoader resourceLoader;
|
||||||
|
|
||||||
|
public TachographMasterDataRefreshService(
|
||||||
|
@Qualifier("tachographNamedParameterJdbcTemplate") ObjectProvider<NamedParameterJdbcTemplate> tachographJdbcTemplateProvider,
|
||||||
|
SourceMasterDataRepository sourceMasterDataRepository,
|
||||||
|
EventSourceRepository eventSourceRepository,
|
||||||
|
VehicleIdentityRepository vehicleIdentityRepository,
|
||||||
|
ResourceLoader resourceLoader
|
||||||
|
) {
|
||||||
|
this.tachographJdbcTemplateProvider = tachographJdbcTemplateProvider;
|
||||||
|
this.sourceMasterDataRepository = sourceMasterDataRepository;
|
||||||
|
this.eventSourceRepository = eventSourceRepository;
|
||||||
|
this.vehicleIdentityRepository = vehicleIdentityRepository;
|
||||||
|
this.resourceLoader = resourceLoader;
|
||||||
|
}
|
||||||
|
|
||||||
|
public MasterDataRefreshResult refreshIfRequested(TachographImportRequest request) {
|
||||||
if (!request.refreshMasterDataFirst()) {
|
if (!request.refreshMasterDataFirst()) {
|
||||||
return;
|
return MasterDataRefreshResult.empty();
|
||||||
}
|
}
|
||||||
log.info("Tachograph master-data refresh requested for tenant={} source={}. Concrete SQL refresh is a project-specific extension point.",
|
return refresh(request);
|
||||||
request.tenantKey(), request.eventSource().stableKey());
|
}
|
||||||
|
|
||||||
|
public MasterDataRefreshResult refresh(TachographImportRequest request) {
|
||||||
|
NamedParameterJdbcTemplate tachographJdbcTemplate = tachographJdbcTemplateProvider.getIfAvailable();
|
||||||
|
if (tachographJdbcTemplate == null) {
|
||||||
|
log.info("Skipping tachograph master-data refresh for tenant={} because no tachograph datasource is configured.",
|
||||||
|
request.tenantKey());
|
||||||
|
return MasterDataRefreshResult.empty();
|
||||||
|
}
|
||||||
|
|
||||||
|
String tenantKey = request.tenantKey() == null || request.tenantKey().isBlank() ? "default" : request.tenantKey().trim();
|
||||||
|
EventSourceDto masterDataSource = masterDataSourceFor(request.eventSource());
|
||||||
|
int eventSourceId = eventSourceRepository.resolveSourceId(tenantKey, masterDataSource);
|
||||||
|
|
||||||
|
log.info("Starting tachograph source master-data refresh tenant={} source={} batchSize={}",
|
||||||
|
tenantKey, masterDataSource.stableKey(), MASTER_DATA_UPSERT_BATCH_SIZE);
|
||||||
|
|
||||||
|
int entities = 0;
|
||||||
|
for (String sqlResource : ENTITY_SQL_RESOURCES) {
|
||||||
|
entities += streamEntities(tachographJdbcTemplate, tenantKey, eventSourceId, sqlResource, loadSql(sqlResource));
|
||||||
|
}
|
||||||
|
|
||||||
|
int relationCount = streamRelations(tachographJdbcTemplate, tenantKey, eventSourceId, RELATIONS_SQL_RESOURCE, loadSql(RELATIONS_SQL_RESOURCE));
|
||||||
|
|
||||||
|
log.info("Reconciling tachograph vehicle identities from source master data tenant={} source={}",
|
||||||
|
tenantKey, masterDataSource.stableKey());
|
||||||
|
int reconciledVehicles = vehicleIdentityRepository.reconcileFromMasterData(tenantKey, eventSourceId);
|
||||||
|
|
||||||
|
MasterDataRefreshResult result = new MasterDataRefreshResult(entities, relationCount);
|
||||||
|
log.info("Refreshed tachograph source master data tenant={} source={} entities={} relations={} reconciledVehicles={}",
|
||||||
|
tenantKey, masterDataSource.stableKey(), result.entitiesUpserted(), result.relationsUpserted(), reconciledVehicles);
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
|
private int streamEntities(
|
||||||
|
NamedParameterJdbcTemplate tachographJdbcTemplate,
|
||||||
|
String tenantKey,
|
||||||
|
int eventSourceId,
|
||||||
|
String sqlResource,
|
||||||
|
String sql
|
||||||
|
) {
|
||||||
|
String section = masterDataSection(sqlResource);
|
||||||
|
List<SourceMasterEntityUpsert> buffer = new ArrayList<>(MASTER_DATA_UPSERT_BATCH_SIZE);
|
||||||
|
Map<String, Integer> typeCounts = new LinkedHashMap<>();
|
||||||
|
int[] count = {0};
|
||||||
|
int[] chunk = {0};
|
||||||
|
|
||||||
|
log.info("Reading tachograph master-data entities tenant={} section={}", tenantKey, section);
|
||||||
|
tachographJdbcTemplate.query(sql, Map.of(), rs -> {
|
||||||
|
SourceMasterEntityUpsert entity = entity(rs);
|
||||||
|
buffer.add(entity);
|
||||||
|
increment(typeCounts, entity.entityType());
|
||||||
|
if (buffer.size() >= MASTER_DATA_UPSERT_BATCH_SIZE) {
|
||||||
|
count[0] += flushEntities(tenantKey, eventSourceId, section, ++chunk[0], buffer, count[0], typeCounts);
|
||||||
|
buffer.clear();
|
||||||
|
}
|
||||||
|
});
|
||||||
|
if (!buffer.isEmpty()) {
|
||||||
|
count[0] += flushEntities(tenantKey, eventSourceId, section, ++chunk[0], buffer, count[0], typeCounts);
|
||||||
|
}
|
||||||
|
log.info("Finished tachograph master-data entities tenant={} section={} processed={} byType={}",
|
||||||
|
tenantKey, section, count[0], typeCounts);
|
||||||
|
return count[0];
|
||||||
|
}
|
||||||
|
|
||||||
|
private int streamRelations(
|
||||||
|
NamedParameterJdbcTemplate tachographJdbcTemplate,
|
||||||
|
String tenantKey,
|
||||||
|
int eventSourceId,
|
||||||
|
String sqlResource,
|
||||||
|
String sql
|
||||||
|
) {
|
||||||
|
String section = masterDataSection(sqlResource);
|
||||||
|
List<SourceMasterRelationUpsert> buffer = new ArrayList<>(MASTER_DATA_UPSERT_BATCH_SIZE);
|
||||||
|
Map<String, Integer> typeCounts = new LinkedHashMap<>();
|
||||||
|
int[] count = {0};
|
||||||
|
int[] chunk = {0};
|
||||||
|
|
||||||
|
log.info("Reading tachograph master-data relations tenant={} section={}", tenantKey, section);
|
||||||
|
tachographJdbcTemplate.query(sql, Map.of(), rs -> {
|
||||||
|
SourceMasterRelationUpsert relation = relation(rs);
|
||||||
|
buffer.add(relation);
|
||||||
|
increment(typeCounts, relation.relationType());
|
||||||
|
if (buffer.size() >= MASTER_DATA_UPSERT_BATCH_SIZE) {
|
||||||
|
count[0] += flushRelations(tenantKey, eventSourceId, section, ++chunk[0], buffer, count[0], typeCounts);
|
||||||
|
buffer.clear();
|
||||||
|
}
|
||||||
|
});
|
||||||
|
if (!buffer.isEmpty()) {
|
||||||
|
count[0] += flushRelations(tenantKey, eventSourceId, section, ++chunk[0], buffer, count[0], typeCounts);
|
||||||
|
}
|
||||||
|
log.info("Finished tachograph master-data relations tenant={} section={} processed={} byType={}",
|
||||||
|
tenantKey, section, count[0], typeCounts);
|
||||||
|
return count[0];
|
||||||
|
}
|
||||||
|
|
||||||
|
private int flushEntities(
|
||||||
|
String tenantKey,
|
||||||
|
int eventSourceId,
|
||||||
|
String section,
|
||||||
|
int chunk,
|
||||||
|
List<SourceMasterEntityUpsert> buffer,
|
||||||
|
int alreadyProcessed,
|
||||||
|
Map<String, Integer> typeCounts
|
||||||
|
) {
|
||||||
|
int upserted = sourceMasterDataRepository.upsertEntities(tenantKey, eventSourceId, buffer);
|
||||||
|
log.info("Tachograph master-data entity progress tenant={} section={} chunk={} chunkSize={} processed={} byType={}",
|
||||||
|
tenantKey, section, chunk, buffer.size(), alreadyProcessed + upserted, typeCounts);
|
||||||
|
return upserted;
|
||||||
|
}
|
||||||
|
|
||||||
|
private int flushRelations(
|
||||||
|
String tenantKey,
|
||||||
|
int eventSourceId,
|
||||||
|
String section,
|
||||||
|
int chunk,
|
||||||
|
List<SourceMasterRelationUpsert> buffer,
|
||||||
|
int alreadyProcessed,
|
||||||
|
Map<String, Integer> typeCounts
|
||||||
|
) {
|
||||||
|
int upserted = sourceMasterDataRepository.upsertRelations(tenantKey, eventSourceId, buffer);
|
||||||
|
log.info("Tachograph master-data relation progress tenant={} section={} chunk={} chunkSize={} processed={} byType={}",
|
||||||
|
tenantKey, section, chunk, buffer.size(), alreadyProcessed + upserted, typeCounts);
|
||||||
|
return upserted;
|
||||||
|
}
|
||||||
|
|
||||||
|
private void increment(Map<String, Integer> counts, String type) {
|
||||||
|
String key = type == null || type.isBlank() ? "UNKNOWN" : type;
|
||||||
|
counts.merge(key, 1, Integer::sum);
|
||||||
|
}
|
||||||
|
|
||||||
|
private String masterDataSection(String sqlResource) {
|
||||||
|
int slash = sqlResource.lastIndexOf('/');
|
||||||
|
String fileName = slash < 0 ? sqlResource : sqlResource.substring(slash + 1);
|
||||||
|
return fileName.endsWith(".sql") ? fileName.substring(0, fileName.length() - 4) : fileName;
|
||||||
|
}
|
||||||
|
|
||||||
|
private EventSourceDto masterDataSourceFor(EventSourceDto source) {
|
||||||
|
return new EventSourceDto(
|
||||||
|
source.providerKey(),
|
||||||
|
MASTER_DATA_SOURCE_KIND,
|
||||||
|
MASTER_DATA_SOURCE_KEY,
|
||||||
|
source.sourceInstanceKey(),
|
||||||
|
source.tenantProviderSettingKey(),
|
||||||
|
source.externalFleetKey()
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
private SourceMasterEntityUpsert entity(ResultSet rs) throws SQLException {
|
||||||
|
String entityType = string(rs, "entity_type");
|
||||||
|
String sourceEntityId = string(rs, "source_entity_id");
|
||||||
|
OffsetDateTime validFrom = offsetDateTime(rs, "valid_from");
|
||||||
|
OffsetDateTime validTo = offsetDateTime(rs, "valid_to");
|
||||||
|
ValidityRange validityRange = normalizeValidityRange(
|
||||||
|
validFrom,
|
||||||
|
validTo,
|
||||||
|
"entity",
|
||||||
|
entityType + ":" + sourceEntityId
|
||||||
|
);
|
||||||
|
return new SourceMasterEntityUpsert(
|
||||||
|
entityType,
|
||||||
|
sourceEntityId,
|
||||||
|
string(rs, "source_external_key"),
|
||||||
|
string(rs, "display_name"),
|
||||||
|
bool(rs, "active"),
|
||||||
|
validityRange.validFrom(),
|
||||||
|
validityRange.validTo(),
|
||||||
|
offsetDateTime(rs, "source_updated_at"),
|
||||||
|
payload(rs)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
private SourceMasterRelationUpsert relation(ResultSet rs) throws SQLException {
|
||||||
|
String relationType = string(rs, "relation_type");
|
||||||
|
String fromEntityType = string(rs, "from_entity_type");
|
||||||
|
String fromSourceEntityId = string(rs, "from_source_entity_id");
|
||||||
|
String toEntityType = string(rs, "to_entity_type");
|
||||||
|
String toSourceEntityId = string(rs, "to_source_entity_id");
|
||||||
|
OffsetDateTime validFrom = offsetDateTime(rs, "valid_from");
|
||||||
|
OffsetDateTime validTo = offsetDateTime(rs, "valid_to");
|
||||||
|
ValidityRange validityRange = normalizeValidityRange(
|
||||||
|
validFrom,
|
||||||
|
validTo,
|
||||||
|
"relation",
|
||||||
|
relationType + ":" + fromEntityType + ":" + fromSourceEntityId + "->" + toEntityType + ":" + toSourceEntityId
|
||||||
|
);
|
||||||
|
return new SourceMasterRelationUpsert(
|
||||||
|
relationType,
|
||||||
|
fromEntityType,
|
||||||
|
fromSourceEntityId,
|
||||||
|
toEntityType,
|
||||||
|
toSourceEntityId,
|
||||||
|
validityRange.validFrom(),
|
||||||
|
validityRange.validTo(),
|
||||||
|
offsetDateTime(rs, "source_updated_at"),
|
||||||
|
payload(rs)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
private Map<String, Object> payload(ResultSet rs) throws SQLException {
|
||||||
|
ResultSetMetaData metaData = rs.getMetaData();
|
||||||
|
Map<String, Object> payload = new LinkedHashMap<>();
|
||||||
|
for (int i = 1; i <= metaData.getColumnCount(); i++) {
|
||||||
|
String name = metaData.getColumnLabel(i);
|
||||||
|
Object value = rs.getObject(i);
|
||||||
|
if (value != null) {
|
||||||
|
payload.put(name, value);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return payload;
|
||||||
|
}
|
||||||
|
|
||||||
|
private String loadSql(String location) {
|
||||||
|
Resource resource = resourceLoader.getResource(location);
|
||||||
|
try (var inputStream = resource.getInputStream()) {
|
||||||
|
return StreamUtils.copyToString(inputStream, StandardCharsets.UTF_8);
|
||||||
|
} catch (IOException e) {
|
||||||
|
throw new IllegalStateException("Cannot load tachograph master-data SQL resource " + location, e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private String string(ResultSet rs, String column) throws SQLException {
|
||||||
|
String value = rs.getString(column);
|
||||||
|
return value == null || value.isBlank() ? null : value.trim();
|
||||||
|
}
|
||||||
|
|
||||||
|
private Boolean bool(ResultSet rs, String column) throws SQLException {
|
||||||
|
Object value = rs.getObject(column);
|
||||||
|
if (value == null) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
if (value instanceof Boolean bool) {
|
||||||
|
return bool;
|
||||||
|
}
|
||||||
|
if (value instanceof Number number) {
|
||||||
|
return number.intValue() != 0;
|
||||||
|
}
|
||||||
|
return Boolean.parseBoolean(value.toString());
|
||||||
|
}
|
||||||
|
|
||||||
|
private OffsetDateTime offsetDateTime(ResultSet rs, String column) throws SQLException {
|
||||||
|
Object value = rs.getObject(column);
|
||||||
|
if (value == null) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
if (value instanceof OffsetDateTime offsetDateTime) {
|
||||||
|
return offsetDateTime.withOffsetSameInstant(ZoneOffset.UTC);
|
||||||
|
}
|
||||||
|
if (value instanceof Timestamp timestamp) {
|
||||||
|
return timestamp.toLocalDateTime().atOffset(ZoneOffset.UTC);
|
||||||
|
}
|
||||||
|
if (value instanceof java.time.LocalDateTime localDateTime) {
|
||||||
|
return localDateTime.atOffset(ZoneOffset.UTC);
|
||||||
|
}
|
||||||
|
String text = value.toString();
|
||||||
|
try {
|
||||||
|
return OffsetDateTime.parse(text).withOffsetSameInstant(ZoneOffset.UTC);
|
||||||
|
} catch (RuntimeException ignored) {
|
||||||
|
return LocalDateTime.parse(text).atOffset(ZoneOffset.UTC);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private ValidityRange normalizeValidityRange(
|
||||||
|
OffsetDateTime validFrom,
|
||||||
|
OffsetDateTime validTo,
|
||||||
|
String rowKind,
|
||||||
|
String rowKey
|
||||||
|
) {
|
||||||
|
if (validFrom == null || validTo == null || !validFrom.isAfter(validTo)) {
|
||||||
|
return new ValidityRange(validFrom, validTo);
|
||||||
|
}
|
||||||
|
|
||||||
|
log.warn(
|
||||||
|
"Ignoring invalid validity end for {} {} because valid_from {} is after valid_to {}. Keeping valid_from and setting valid_to=null.",
|
||||||
|
rowKind,
|
||||||
|
rowKey,
|
||||||
|
validFrom,
|
||||||
|
validTo
|
||||||
|
);
|
||||||
|
return new ValidityRange(validFrom, null);
|
||||||
|
}
|
||||||
|
|
||||||
|
private record ValidityRange(OffsetDateTime validFrom, OffsetDateTime validTo) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,8 @@
|
||||||
|
package at.procon.eventhub.tachograph.service;
|
||||||
|
|
||||||
|
public class UnsupportedTachographExtractionException extends IllegalArgumentException {
|
||||||
|
|
||||||
|
public UnsupportedTachographExtractionException(String message) {
|
||||||
|
super(message);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -1,10 +1,6 @@
|
||||||
package at.procon.eventhub.tachograph.service;
|
package at.procon.eventhub.tachograph.service;
|
||||||
|
|
||||||
import at.procon.eventhub.service.EventDetailsFactory;
|
import at.procon.eventhub.service.EventDetailsFactory;
|
||||||
import java.sql.ResultSet;
|
|
||||||
import java.sql.SQLException;
|
|
||||||
import java.util.LinkedHashMap;
|
|
||||||
import java.util.Map;
|
|
||||||
import org.springframework.stereotype.Component;
|
import org.springframework.stereotype.Component;
|
||||||
|
|
||||||
@Component
|
@Component
|
||||||
|
|
@ -13,11 +9,4 @@ public class VuActivityRowMapper extends AbstractTachographActivityRowMapper {
|
||||||
public VuActivityRowMapper(EventDetailsFactory detailsFactory) {
|
public VuActivityRowMapper(EventDetailsFactory detailsFactory) {
|
||||||
super(detailsFactory);
|
super(detailsFactory);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
|
||||||
protected Map<String, Object> sourceSpecificPayload(ResultSet rs) throws SQLException {
|
|
||||||
Map<String, Object> raw = new LinkedHashMap<>();
|
|
||||||
put(raw, "vuActivityId", string(rs, "vu_activity_id"));
|
|
||||||
return raw;
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,12 @@
|
||||||
|
package at.procon.eventhub.tachograph.service;
|
||||||
|
|
||||||
|
import at.procon.eventhub.service.EventDetailsFactory;
|
||||||
|
import org.springframework.stereotype.Component;
|
||||||
|
|
||||||
|
@Component
|
||||||
|
public class VuBorderCrossingRowMapper extends AbstractTachographBorderCrossingRowMapper {
|
||||||
|
|
||||||
|
public VuBorderCrossingRowMapper(EventDetailsFactory detailsFactory) {
|
||||||
|
super(detailsFactory);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,12 @@
|
||||||
|
package at.procon.eventhub.tachograph.service;
|
||||||
|
|
||||||
|
import at.procon.eventhub.service.EventDetailsFactory;
|
||||||
|
import org.springframework.stereotype.Component;
|
||||||
|
|
||||||
|
@Component
|
||||||
|
public class VuLoadUnloadRowMapper extends AbstractTachographLoadUnloadRowMapper {
|
||||||
|
|
||||||
|
public VuLoadUnloadRowMapper(EventDetailsFactory detailsFactory) {
|
||||||
|
super(detailsFactory);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,12 @@
|
||||||
|
package at.procon.eventhub.tachograph.service;
|
||||||
|
|
||||||
|
import at.procon.eventhub.service.EventDetailsFactory;
|
||||||
|
import org.springframework.stereotype.Component;
|
||||||
|
|
||||||
|
@Component
|
||||||
|
public class VuPlaceRowMapper extends AbstractTachographPlaceRowMapper {
|
||||||
|
|
||||||
|
public VuPlaceRowMapper(EventDetailsFactory detailsFactory) {
|
||||||
|
super(detailsFactory);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,12 @@
|
||||||
|
package at.procon.eventhub.tachograph.service;
|
||||||
|
|
||||||
|
import at.procon.eventhub.service.EventDetailsFactory;
|
||||||
|
import org.springframework.stereotype.Component;
|
||||||
|
|
||||||
|
@Component
|
||||||
|
public class VuPositionRowMapper extends AbstractTachographPositionRowMapper {
|
||||||
|
|
||||||
|
public VuPositionRowMapper(EventDetailsFactory detailsFactory) {
|
||||||
|
super(detailsFactory);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,12 @@
|
||||||
|
package at.procon.eventhub.tachograph.service;
|
||||||
|
|
||||||
|
import at.procon.eventhub.service.EventDetailsFactory;
|
||||||
|
import org.springframework.stereotype.Component;
|
||||||
|
|
||||||
|
@Component
|
||||||
|
public class VuSpecificConditionRowMapper extends AbstractTachographSpecificConditionRowMapper {
|
||||||
|
|
||||||
|
public VuSpecificConditionRowMapper(EventDetailsFactory detailsFactory) {
|
||||||
|
super(detailsFactory);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -3,16 +3,33 @@ spring:
|
||||||
name: eventhub-ingestion-service
|
name: eventhub-ingestion-service
|
||||||
datasource:
|
datasource:
|
||||||
url: jdbc:postgresql://localhost:5432/eventhub
|
url: jdbc:postgresql://localhost:5432/eventhub
|
||||||
username: eventhub
|
username: postgres
|
||||||
password: eventhub
|
password: P54!pcd#Wi
|
||||||
|
hikari:
|
||||||
|
maximum-pool-size: 16
|
||||||
|
minimum-idle: 4
|
||||||
|
connection-timeout: 30000
|
||||||
|
validation-timeout: 5000
|
||||||
|
idle-timeout: 60000
|
||||||
|
keepalive-time: 30000
|
||||||
|
max-lifetime: 90000
|
||||||
flyway:
|
flyway:
|
||||||
enabled: true
|
enabled: true
|
||||||
default-schema: eventhub
|
default-schema: eventhub
|
||||||
schemas: eventhub
|
schemas: eventhub
|
||||||
create-schemas: true
|
create-schemas: true
|
||||||
|
|
||||||
|
logging:
|
||||||
|
file:
|
||||||
|
name: ${EVENTHUB_LOG_FILE:logs/eventhub-ingestion-service.log}
|
||||||
|
logback:
|
||||||
|
rollingpolicy:
|
||||||
|
max-file-size: ${EVENTHUB_LOG_MAX_FILE_SIZE:50MB}
|
||||||
|
max-history: ${EVENTHUB_LOG_MAX_HISTORY:14}
|
||||||
|
total-size-cap: ${EVENTHUB_LOG_TOTAL_SIZE_CAP:1GB}
|
||||||
|
|
||||||
server:
|
server:
|
||||||
port: 8080
|
port: 8085
|
||||||
|
|
||||||
camel:
|
camel:
|
||||||
springboot:
|
springboot:
|
||||||
|
|
@ -27,39 +44,43 @@ management:
|
||||||
|
|
||||||
eventhub:
|
eventhub:
|
||||||
batch:
|
batch:
|
||||||
completion-size: 1000
|
completion-size: 5000
|
||||||
completion-timeout: 5s
|
completion-timeout: 5s
|
||||||
|
queue-size: 10000
|
||||||
|
concurrent-consumers: 4
|
||||||
|
block-when-full: true
|
||||||
|
queue-offer-timeout: 5m
|
||||||
tachograph:
|
tachograph:
|
||||||
default-chunk-days: 1
|
default-chunk-days: 1
|
||||||
occurred-at-overlap: 7d
|
occurred-at-overlap: 7d
|
||||||
|
|
||||||
# Configure this block to enable JdbcTachographExtractionBatchExecutor.
|
# Configure this block to enable JdbcTachographExtractionBatchExecutor.
|
||||||
# datasource:
|
datasource:
|
||||||
# jdbc-url: jdbc:sqlserver://localhost:1433;databaseName=tachograph;encrypt=true;trustServerCertificate=true
|
jdbc-url: jdbc:sqlserver://db.bytebar.eu:22996;databaseName=ByteBarDriverSettlement;trustServerCertificate=true
|
||||||
# username: tachograph_user
|
username: ReadOnly
|
||||||
# password: change-me
|
password: p2=race!
|
||||||
# driver-class-name: com.microsoft.sqlserver.jdbc.SQLServerDriver
|
driver-class-name: com.microsoft.sqlserver.jdbc.SQLServerDriver
|
||||||
|
|
||||||
# Enables the scheduler that regularly triggers configured tachograph import plans.
|
# Enables the scheduler that regularly triggers configured tachograph import plans.
|
||||||
scheduler-enabled: false
|
scheduler-enabled: true
|
||||||
scheduler-poll-interval-ms: 60000
|
scheduler-poll-interval-ms: 60000
|
||||||
|
|
||||||
# PLAN_ONLY creates import_run + planned extraction packages.
|
# PLAN_ONLY creates import_run + planned extraction packages.
|
||||||
# EXECUTE also invokes the configured TachographExtractionBatchExecutor.
|
# EXECUTE also invokes the configured TachographExtractionBatchExecutor.
|
||||||
scheduler-trigger-mode: PLAN_ONLY
|
scheduler-trigger-mode: EXECUTE
|
||||||
|
|
||||||
# Example plan. Keep disabled until the tachograph datasource/extractor is wired.
|
# Example plan. Keep disabled until the tachograph datasource/extractor is wired.
|
||||||
import-plans:
|
import-plans:
|
||||||
- plan-key: kralowetz-tachograph-org-147
|
- plan-key: kralowetz-tachograph-org-147
|
||||||
enabled: false
|
enabled: true
|
||||||
cron: "0 15 * * * *" # hourly at minute 15
|
cron: "0 15 * * * *" # hourly at minute 15
|
||||||
tenant-key: kralowetz
|
tenant-key: kralowetz
|
||||||
event-source:
|
event-source:
|
||||||
provider-key: TACHOGRAPH
|
provider-key: TACHOGRAPH
|
||||||
source-kind: MIXED
|
source-kind: MIXED
|
||||||
source-key: TACHOGRAPH_DB
|
source-key: TACHOGRAPH_DB
|
||||||
source-instance-key: tachograph-prod-db
|
source-instance-key: ByteBar-DriverSettlement
|
||||||
tenant-provider-setting-key: kralowetz-tachograph-prod
|
tenant-provider-setting-key: ByteBar-DriverSettlement
|
||||||
source-group:
|
source-group:
|
||||||
type: ORGANISATION
|
type: ORGANISATION
|
||||||
source-entity-id: "147"
|
source-entity-id: "147"
|
||||||
|
|
@ -88,7 +109,7 @@ eventhub:
|
||||||
scheduled-mode: INCREMENTAL_UPDATE
|
scheduled-mode: INCREMENTAL_UPDATE
|
||||||
initial-strategy: OCCURRED_AT_WINDOW_WITH_OVERLAP
|
initial-strategy: OCCURRED_AT_WINDOW_WITH_OVERLAP
|
||||||
scheduled-strategy: SOURCE_PACKAGE_WATERMARK
|
scheduled-strategy: SOURCE_PACKAGE_WATERMARK
|
||||||
refresh-master-data-first: true
|
refresh-master-data-first: false
|
||||||
initial-occurred-from: "2025-01-01T00:00:00+01:00"
|
initial-occurred-from: "2026-01-21T00:00:00+01:00"
|
||||||
initial-occurred-to: null
|
initial-occurred-to: "2026-01-25T00:00:00+01:00"
|
||||||
run-initial-on-startup: false
|
run-initial-on-startup: true
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,346 @@
|
||||||
|
create extension if not exists pgcrypto;
|
||||||
|
create extension if not exists postgis;
|
||||||
|
create extension if not exists timescaledb;
|
||||||
|
|
||||||
|
drop schema if exists eventhub cascade;
|
||||||
|
create schema if not exists eventhub;
|
||||||
|
|
||||||
|
create table if not exists eventhub.event_source (
|
||||||
|
id integer generated always as identity primary key,
|
||||||
|
tenant_key text not null,
|
||||||
|
provider_key text not null,
|
||||||
|
source_kind text not null,
|
||||||
|
source_key text not null,
|
||||||
|
source_instance_key text not null default 'default',
|
||||||
|
tenant_provider_setting_key text,
|
||||||
|
external_fleet_key text,
|
||||||
|
created_at timestamptz not null default now(),
|
||||||
|
constraint ux_event_source unique (tenant_key, provider_key, source_kind, source_key, source_instance_key)
|
||||||
|
);
|
||||||
|
|
||||||
|
create table if not exists eventhub.import_run (
|
||||||
|
id uuid primary key,
|
||||||
|
tenant_key text not null,
|
||||||
|
event_source_id integer not null references eventhub.event_source(id),
|
||||||
|
mode text not null,
|
||||||
|
status text not null,
|
||||||
|
refresh_master_data_first boolean not null default true,
|
||||||
|
source_group_type text,
|
||||||
|
source_group_entity_id text,
|
||||||
|
source_group_code text,
|
||||||
|
source_group_name text,
|
||||||
|
import_scope_type text not null,
|
||||||
|
root_source_org_entity_id text,
|
||||||
|
root_source_org_code text,
|
||||||
|
root_source_org_name text,
|
||||||
|
include_children boolean not null default false,
|
||||||
|
occurred_from timestamptz,
|
||||||
|
occurred_to timestamptz,
|
||||||
|
requested_event_families text[] not null default '{}',
|
||||||
|
acquisition_strategy text,
|
||||||
|
metadata jsonb not null default '{}'::jsonb,
|
||||||
|
planned_package_count integer not null default 0,
|
||||||
|
started_at timestamptz not null default now(),
|
||||||
|
finished_at timestamptz,
|
||||||
|
error_message text,
|
||||||
|
constraint chk_import_run_occ_time_order check (occurred_from is null or occurred_to is null or occurred_from < occurred_to)
|
||||||
|
);
|
||||||
|
|
||||||
|
create table if not exists eventhub.import_cursor (
|
||||||
|
id uuid primary key,
|
||||||
|
tenant_key text not null,
|
||||||
|
event_source_id integer not null references eventhub.event_source(id),
|
||||||
|
scope_hash text not null,
|
||||||
|
event_family text not null,
|
||||||
|
source_kind text not null,
|
||||||
|
cursor_type text not null,
|
||||||
|
last_source_package_imported_at timestamptz,
|
||||||
|
last_source_package_id text,
|
||||||
|
last_source_row_updated_at timestamptz,
|
||||||
|
last_occurred_to timestamptz,
|
||||||
|
updated_at timestamptz not null default now(),
|
||||||
|
constraint ux_import_cursor unique (tenant_key, event_source_id, scope_hash, event_family, source_kind, cursor_type)
|
||||||
|
);
|
||||||
|
|
||||||
|
create table if not exists eventhub.data_package (
|
||||||
|
id uuid primary key,
|
||||||
|
event_source_id integer not null references eventhub.event_source(id),
|
||||||
|
import_run_id uuid references eventhub.import_run(id),
|
||||||
|
tenant_key text not null,
|
||||||
|
package_key text not null,
|
||||||
|
package_type text not null,
|
||||||
|
status text not null,
|
||||||
|
source_group_type text,
|
||||||
|
source_group_entity_id text,
|
||||||
|
source_group_code text,
|
||||||
|
source_group_name text,
|
||||||
|
import_scope_type text,
|
||||||
|
root_source_org_entity_id text,
|
||||||
|
root_source_org_code text,
|
||||||
|
root_source_org_name text,
|
||||||
|
include_children boolean not null default false,
|
||||||
|
occurred_from timestamptz,
|
||||||
|
occurred_to timestamptz,
|
||||||
|
event_family text,
|
||||||
|
business_date date,
|
||||||
|
external_package_id text,
|
||||||
|
extraction_code text,
|
||||||
|
extraction_source_kind text,
|
||||||
|
entity_axis text,
|
||||||
|
batch_no integer,
|
||||||
|
chunk_from timestamptz,
|
||||||
|
chunk_to timestamptz,
|
||||||
|
source_package_kind text,
|
||||||
|
source_package_id text,
|
||||||
|
source_package_entity_id text,
|
||||||
|
source_package_period_from timestamptz,
|
||||||
|
source_package_period_to timestamptz,
|
||||||
|
source_package_imported_at timestamptz,
|
||||||
|
received_at timestamptz not null default now(),
|
||||||
|
completed_at timestamptz,
|
||||||
|
event_count integer not null default 0,
|
||||||
|
metadata jsonb not null default '{}'::jsonb,
|
||||||
|
error_message text,
|
||||||
|
constraint ux_data_package_package_key unique (tenant_key, event_source_id, package_key),
|
||||||
|
constraint chk_data_package_occ_time_order check (occurred_from is null or occurred_to is null or occurred_from < occurred_to),
|
||||||
|
constraint chk_data_package_chunk_time_order check (chunk_from is null or chunk_to is null or chunk_from < chunk_to)
|
||||||
|
);
|
||||||
|
|
||||||
|
create table if not exists eventhub.source_master_entity (
|
||||||
|
id uuid primary key,
|
||||||
|
tenant_key text not null,
|
||||||
|
event_source_id integer not null references eventhub.event_source(id),
|
||||||
|
entity_type text not null,
|
||||||
|
source_entity_id text not null,
|
||||||
|
source_external_key text,
|
||||||
|
display_name text,
|
||||||
|
active boolean,
|
||||||
|
valid_from timestamptz,
|
||||||
|
valid_to timestamptz,
|
||||||
|
source_updated_at timestamptz,
|
||||||
|
payload jsonb not null default '{}'::jsonb,
|
||||||
|
created_at timestamptz not null default now(),
|
||||||
|
updated_at timestamptz not null default now(),
|
||||||
|
constraint ux_source_master_entity unique (tenant_key, event_source_id, entity_type, source_entity_id),
|
||||||
|
constraint chk_source_master_entity_valid_time_order check (valid_from is null or valid_to is null or valid_from <= valid_to)
|
||||||
|
);
|
||||||
|
|
||||||
|
create table if not exists eventhub.source_master_relation (
|
||||||
|
id uuid primary key,
|
||||||
|
tenant_key text not null,
|
||||||
|
event_source_id integer not null references eventhub.event_source(id),
|
||||||
|
relation_key text not null,
|
||||||
|
relation_type text not null,
|
||||||
|
from_entity_type text not null,
|
||||||
|
from_source_entity_id text not null,
|
||||||
|
to_entity_type text not null,
|
||||||
|
to_source_entity_id text not null,
|
||||||
|
valid_from timestamptz,
|
||||||
|
valid_to timestamptz,
|
||||||
|
source_updated_at timestamptz,
|
||||||
|
payload jsonb not null default '{}'::jsonb,
|
||||||
|
created_at timestamptz not null default now(),
|
||||||
|
updated_at timestamptz not null default now(),
|
||||||
|
constraint ux_source_master_relation unique (tenant_key, event_source_id, relation_key),
|
||||||
|
constraint chk_source_master_relation_valid_time_order check (valid_from is null or valid_to is null or valid_from <= valid_to)
|
||||||
|
);
|
||||||
|
|
||||||
|
create table if not exists eventhub.vehicle (
|
||||||
|
id uuid primary key,
|
||||||
|
tenant_key text not null,
|
||||||
|
event_source_id integer not null references eventhub.event_source(id),
|
||||||
|
source_vehicle_entity_id text,
|
||||||
|
vin text,
|
||||||
|
created_at timestamptz not null default now(),
|
||||||
|
updated_at timestamptz not null default now()
|
||||||
|
);
|
||||||
|
|
||||||
|
create table if not exists eventhub.vehicle_registration (
|
||||||
|
id uuid primary key,
|
||||||
|
tenant_key text not null,
|
||||||
|
event_source_id integer not null references eventhub.event_source(id),
|
||||||
|
source_registration_entity_id text,
|
||||||
|
nation text not null,
|
||||||
|
registration_number text not null,
|
||||||
|
source_updated_at timestamptz,
|
||||||
|
payload jsonb not null default '{}'::jsonb,
|
||||||
|
created_at timestamptz not null default now(),
|
||||||
|
updated_at timestamptz not null default now()
|
||||||
|
);
|
||||||
|
|
||||||
|
create table if not exists eventhub.vehicle_registration_assignment (
|
||||||
|
id uuid primary key,
|
||||||
|
tenant_key text not null,
|
||||||
|
event_source_id integer not null references eventhub.event_source(id),
|
||||||
|
vehicle_registration_id uuid not null references eventhub.vehicle_registration(id) on delete cascade,
|
||||||
|
vehicle_id uuid not null references eventhub.vehicle(id) on delete cascade,
|
||||||
|
valid_from timestamptz,
|
||||||
|
valid_to timestamptz,
|
||||||
|
source_updated_at timestamptz,
|
||||||
|
payload jsonb not null default '{}'::jsonb,
|
||||||
|
created_at timestamptz not null default now(),
|
||||||
|
updated_at timestamptz not null default now(),
|
||||||
|
constraint chk_vehicle_registration_assignment_valid_time_order check (valid_from is null or valid_to is null or valid_from <= valid_to)
|
||||||
|
);
|
||||||
|
|
||||||
|
create table if not exists eventhub.event (
|
||||||
|
id uuid not null,
|
||||||
|
event_source_id integer not null references eventhub.event_source(id),
|
||||||
|
data_package_id uuid not null references eventhub.data_package(id),
|
||||||
|
external_source_event_id text not null,
|
||||||
|
driver_entity_id uuid references eventhub.source_master_entity(id),
|
||||||
|
vehicle_id uuid references eventhub.vehicle(id),
|
||||||
|
vehicle_registration_id uuid references eventhub.vehicle_registration(id),
|
||||||
|
source_package_id text,
|
||||||
|
source_package_entity_id uuid references eventhub.source_master_entity(id),
|
||||||
|
occurred_at timestamptz not null,
|
||||||
|
received_partner_at timestamptz,
|
||||||
|
received_hub_at timestamptz not null default now(),
|
||||||
|
event_domain text not null,
|
||||||
|
event_type text not null,
|
||||||
|
lifecycle text not null,
|
||||||
|
odometer_m bigint,
|
||||||
|
position geography(Point, 4326),
|
||||||
|
payload jsonb not null default '{}'::jsonb,
|
||||||
|
manual_entry boolean not null default false,
|
||||||
|
source_record_key_hash text not null,
|
||||||
|
event_signature_hash text,
|
||||||
|
created_at timestamptz not null default now(),
|
||||||
|
constraint pk_event primary key (occurred_at, id),
|
||||||
|
constraint chk_event_driver_or_vehicle_ref check (
|
||||||
|
driver_entity_id is not null
|
||||||
|
or vehicle_id is not null
|
||||||
|
or vehicle_registration_id is not null
|
||||||
|
)
|
||||||
|
);
|
||||||
|
|
||||||
|
create table if not exists eventhub.event_source_record (
|
||||||
|
source_record_key_hash text primary key,
|
||||||
|
event_occurred_at timestamptz not null,
|
||||||
|
event_id uuid not null,
|
||||||
|
created_at timestamptz not null default now()
|
||||||
|
);
|
||||||
|
|
||||||
|
create table if not exists eventhub.event_detail (
|
||||||
|
event_occurred_at timestamptz not null,
|
||||||
|
event_id uuid not null,
|
||||||
|
detail_type text not null,
|
||||||
|
attributes jsonb not null default '{}'::jsonb,
|
||||||
|
created_at timestamptz not null default now(),
|
||||||
|
constraint pk_event_detail primary key (event_occurred_at, event_id, detail_type)
|
||||||
|
);
|
||||||
|
|
||||||
|
select create_hypertable(
|
||||||
|
'eventhub.event',
|
||||||
|
'occurred_at',
|
||||||
|
chunk_time_interval => interval '7 days',
|
||||||
|
if_not_exists => true
|
||||||
|
);
|
||||||
|
|
||||||
|
alter table eventhub.event_source_record
|
||||||
|
add constraint fk_event_source_record_event foreign key (event_occurred_at, event_id)
|
||||||
|
references eventhub.event(occurred_at, id)
|
||||||
|
on delete cascade
|
||||||
|
deferrable initially deferred;
|
||||||
|
|
||||||
|
alter table eventhub.event_detail
|
||||||
|
add constraint fk_event_detail_event foreign key (event_occurred_at, event_id)
|
||||||
|
references eventhub.event(occurred_at, id)
|
||||||
|
on delete cascade;
|
||||||
|
|
||||||
|
create index if not exists idx_data_package_source_time
|
||||||
|
on eventhub.data_package(tenant_key, event_source_id, received_at desc);
|
||||||
|
|
||||||
|
create index if not exists idx_data_package_scope
|
||||||
|
on eventhub.data_package(tenant_key, import_scope_type, root_source_org_entity_id, occurred_from, occurred_to);
|
||||||
|
|
||||||
|
create index if not exists idx_data_package_extraction
|
||||||
|
on eventhub.data_package(tenant_key, event_source_id, import_run_id, event_family, extraction_source_kind, extraction_code, batch_no);
|
||||||
|
|
||||||
|
create index if not exists idx_import_run_source_status
|
||||||
|
on eventhub.import_run(tenant_key, event_source_id, status, started_at desc);
|
||||||
|
|
||||||
|
create index if not exists idx_source_master_entity_type_key
|
||||||
|
on eventhub.source_master_entity(tenant_key, event_source_id, entity_type, source_external_key)
|
||||||
|
where source_external_key is not null;
|
||||||
|
|
||||||
|
create index if not exists idx_source_master_entity_payload_gin
|
||||||
|
on eventhub.source_master_entity using gin(payload);
|
||||||
|
|
||||||
|
create index if not exists idx_source_master_relation_from
|
||||||
|
on eventhub.source_master_relation(tenant_key, event_source_id, from_entity_type, from_source_entity_id, relation_type);
|
||||||
|
|
||||||
|
create index if not exists idx_source_master_relation_to
|
||||||
|
on eventhub.source_master_relation(tenant_key, event_source_id, to_entity_type, to_source_entity_id, relation_type);
|
||||||
|
|
||||||
|
create index if not exists idx_source_master_relation_payload_gin
|
||||||
|
on eventhub.source_master_relation using gin(payload);
|
||||||
|
|
||||||
|
create index if not exists idx_vehicle_lookup_ctx
|
||||||
|
on eventhub.vehicle(tenant_key, event_source_id, updated_at desc);
|
||||||
|
|
||||||
|
create index if not exists idx_vehicle_source_entity
|
||||||
|
on eventhub.vehicle(tenant_key, event_source_id, source_vehicle_entity_id)
|
||||||
|
where source_vehicle_entity_id is not null;
|
||||||
|
|
||||||
|
create index if not exists idx_vehicle_vin
|
||||||
|
on eventhub.vehicle(tenant_key, event_source_id, vin)
|
||||||
|
where vin is not null;
|
||||||
|
|
||||||
|
create index if not exists idx_vehicle_registration_source_entity
|
||||||
|
on eventhub.vehicle_registration(tenant_key, event_source_id, source_registration_entity_id)
|
||||||
|
where source_registration_entity_id is not null;
|
||||||
|
|
||||||
|
create index if not exists idx_vehicle_registration_plate
|
||||||
|
on eventhub.vehicle_registration(tenant_key, event_source_id, nation, registration_number);
|
||||||
|
|
||||||
|
create index if not exists idx_vehicle_registration_assignment_registration_time
|
||||||
|
on eventhub.vehicle_registration_assignment(vehicle_registration_id, valid_from desc, valid_to);
|
||||||
|
|
||||||
|
create index if not exists idx_vehicle_registration_assignment_vehicle_time
|
||||||
|
on eventhub.vehicle_registration_assignment(vehicle_id, valid_from desc, valid_to);
|
||||||
|
|
||||||
|
create index if not exists idx_event_source_record_event
|
||||||
|
on eventhub.event_source_record(event_occurred_at, event_id);
|
||||||
|
|
||||||
|
create index if not exists idx_event_signature
|
||||||
|
on eventhub.event(event_signature_hash)
|
||||||
|
where event_signature_hash is not null;
|
||||||
|
|
||||||
|
create index if not exists idx_event_source_time
|
||||||
|
on eventhub.event(event_source_id, occurred_at desc);
|
||||||
|
|
||||||
|
create index if not exists idx_event_package_time
|
||||||
|
on eventhub.event(data_package_id, occurred_at desc);
|
||||||
|
|
||||||
|
create index if not exists idx_event_source_package_id
|
||||||
|
on eventhub.event(source_package_id)
|
||||||
|
where source_package_id is not null;
|
||||||
|
|
||||||
|
create index if not exists idx_event_domain_type_time
|
||||||
|
on eventhub.event(event_domain, event_type, occurred_at desc);
|
||||||
|
|
||||||
|
create index if not exists idx_event_driver_time
|
||||||
|
on eventhub.event(driver_entity_id, occurred_at desc)
|
||||||
|
where driver_entity_id is not null;
|
||||||
|
|
||||||
|
create index if not exists idx_event_vehicle_time
|
||||||
|
on eventhub.event(vehicle_id, occurred_at desc)
|
||||||
|
where vehicle_id is not null;
|
||||||
|
|
||||||
|
create index if not exists idx_event_vehicle_registration_time
|
||||||
|
on eventhub.event(vehicle_registration_id, occurred_at desc)
|
||||||
|
where vehicle_registration_id is not null;
|
||||||
|
|
||||||
|
create index if not exists idx_event_position_gist
|
||||||
|
on eventhub.event using gist(position)
|
||||||
|
where position is not null;
|
||||||
|
|
||||||
|
create index if not exists idx_event_payload_gin
|
||||||
|
on eventhub.event using gin(payload);
|
||||||
|
|
||||||
|
create index if not exists idx_event_detail_type
|
||||||
|
on eventhub.event_detail(detail_type);
|
||||||
|
|
||||||
|
create index if not exists idx_event_detail_attributes_gin
|
||||||
|
on eventhub.event_detail using gin(attributes);
|
||||||
|
|
@ -0,0 +1,54 @@
|
||||||
|
create table if not exists eventhub.source_master_entity (
|
||||||
|
id uuid primary key,
|
||||||
|
tenant_key text not null,
|
||||||
|
event_source_id integer not null references eventhub.event_source(id),
|
||||||
|
entity_type text not null,
|
||||||
|
source_entity_id text not null,
|
||||||
|
source_external_key text,
|
||||||
|
display_name text,
|
||||||
|
active boolean,
|
||||||
|
valid_from timestamptz,
|
||||||
|
valid_to timestamptz,
|
||||||
|
source_updated_at timestamptz,
|
||||||
|
payload jsonb not null default '{}'::jsonb,
|
||||||
|
created_at timestamptz not null default now(),
|
||||||
|
updated_at timestamptz not null default now(),
|
||||||
|
constraint ux_source_master_entity unique (tenant_key, event_source_id, entity_type, source_entity_id),
|
||||||
|
constraint chk_source_master_entity_valid_time_order check (valid_from is null or valid_to is null or valid_from < valid_to)
|
||||||
|
);
|
||||||
|
|
||||||
|
create table if not exists eventhub.source_master_relation (
|
||||||
|
id uuid primary key,
|
||||||
|
tenant_key text not null,
|
||||||
|
event_source_id integer not null references eventhub.event_source(id),
|
||||||
|
relation_key text not null,
|
||||||
|
relation_type text not null,
|
||||||
|
from_entity_type text not null,
|
||||||
|
from_source_entity_id text not null,
|
||||||
|
to_entity_type text not null,
|
||||||
|
to_source_entity_id text not null,
|
||||||
|
valid_from timestamptz,
|
||||||
|
valid_to timestamptz,
|
||||||
|
source_updated_at timestamptz,
|
||||||
|
payload jsonb not null default '{}'::jsonb,
|
||||||
|
created_at timestamptz not null default now(),
|
||||||
|
updated_at timestamptz not null default now(),
|
||||||
|
constraint ux_source_master_relation unique (tenant_key, event_source_id, relation_key),
|
||||||
|
constraint chk_source_master_relation_valid_time_order check (valid_from is null or valid_to is null or valid_from < valid_to)
|
||||||
|
);
|
||||||
|
|
||||||
|
create index if not exists idx_source_master_entity_type_key
|
||||||
|
on eventhub.source_master_entity(tenant_key, event_source_id, entity_type, source_external_key)
|
||||||
|
where source_external_key is not null;
|
||||||
|
|
||||||
|
create index if not exists idx_source_master_entity_payload_gin
|
||||||
|
on eventhub.source_master_entity using gin(payload);
|
||||||
|
|
||||||
|
create index if not exists idx_source_master_relation_from
|
||||||
|
on eventhub.source_master_relation(tenant_key, event_source_id, from_entity_type, from_source_entity_id, relation_type);
|
||||||
|
|
||||||
|
create index if not exists idx_source_master_relation_to
|
||||||
|
on eventhub.source_master_relation(tenant_key, event_source_id, to_entity_type, to_source_entity_id, relation_type);
|
||||||
|
|
||||||
|
create index if not exists idx_source_master_relation_payload_gin
|
||||||
|
on eventhub.source_master_relation using gin(payload);
|
||||||
|
|
@ -0,0 +1,13 @@
|
||||||
|
alter table eventhub.source_master_entity
|
||||||
|
drop constraint if exists chk_source_master_entity_valid_time_order;
|
||||||
|
|
||||||
|
alter table eventhub.source_master_entity
|
||||||
|
add constraint chk_source_master_entity_valid_time_order
|
||||||
|
check (valid_from is null or valid_to is null or valid_from <= valid_to);
|
||||||
|
|
||||||
|
alter table eventhub.source_master_relation
|
||||||
|
drop constraint if exists chk_source_master_relation_valid_time_order;
|
||||||
|
|
||||||
|
alter table eventhub.source_master_relation
|
||||||
|
add constraint chk_source_master_relation_valid_time_order
|
||||||
|
check (valid_from is null or valid_to is null or valid_from <= valid_to);
|
||||||
|
|
@ -0,0 +1,91 @@
|
||||||
|
create extension if not exists postgis;
|
||||||
|
|
||||||
|
drop table if exists eventhub.event_detail;
|
||||||
|
drop table if exists eventhub.acquired_event;
|
||||||
|
drop table if exists eventhub.event;
|
||||||
|
|
||||||
|
create table eventhub.event (
|
||||||
|
id uuid not null,
|
||||||
|
event_source_id integer not null references eventhub.event_source(id),
|
||||||
|
data_package_id uuid not null references eventhub.data_package(id),
|
||||||
|
|
||||||
|
external_source_event_id text not null,
|
||||||
|
|
||||||
|
driver_entity_id uuid references eventhub.source_master_entity(id),
|
||||||
|
vehicle_entity_id uuid references eventhub.source_master_entity(id),
|
||||||
|
source_package_entity_id uuid references eventhub.source_master_entity(id),
|
||||||
|
|
||||||
|
occurred_at timestamptz not null,
|
||||||
|
received_partner_at timestamptz,
|
||||||
|
received_hub_at timestamptz not null default now(),
|
||||||
|
|
||||||
|
event_domain text not null,
|
||||||
|
event_type text not null,
|
||||||
|
lifecycle text not null,
|
||||||
|
|
||||||
|
odometer_m bigint,
|
||||||
|
position geography(Point, 4326),
|
||||||
|
|
||||||
|
payload jsonb not null default '{}'::jsonb,
|
||||||
|
manual_entry boolean not null default false,
|
||||||
|
|
||||||
|
source_record_key_hash text not null,
|
||||||
|
event_signature_hash text,
|
||||||
|
|
||||||
|
created_at timestamptz not null default now(),
|
||||||
|
|
||||||
|
constraint pk_event primary key (occurred_at, id),
|
||||||
|
constraint chk_event_driver_or_vehicle_ref check (
|
||||||
|
driver_entity_id is not null
|
||||||
|
or vehicle_entity_id is not null
|
||||||
|
)
|
||||||
|
);
|
||||||
|
|
||||||
|
create unique index ux_event_source_record
|
||||||
|
on eventhub.event(source_record_key_hash);
|
||||||
|
|
||||||
|
create index idx_event_signature
|
||||||
|
on eventhub.event(event_signature_hash)
|
||||||
|
where event_signature_hash is not null;
|
||||||
|
|
||||||
|
create index idx_event_source_time
|
||||||
|
on eventhub.event(event_source_id, occurred_at desc);
|
||||||
|
|
||||||
|
create index idx_event_package_time
|
||||||
|
on eventhub.event(data_package_id, occurred_at desc);
|
||||||
|
|
||||||
|
create index idx_event_domain_type_time
|
||||||
|
on eventhub.event(event_domain, event_type, occurred_at desc);
|
||||||
|
|
||||||
|
create index idx_event_driver_time
|
||||||
|
on eventhub.event(driver_entity_id, occurred_at desc)
|
||||||
|
where driver_entity_id is not null;
|
||||||
|
|
||||||
|
create index idx_event_vehicle_time
|
||||||
|
on eventhub.event(vehicle_entity_id, occurred_at desc)
|
||||||
|
where vehicle_entity_id is not null;
|
||||||
|
|
||||||
|
create index idx_event_position_gist
|
||||||
|
on eventhub.event using gist(position)
|
||||||
|
where position is not null;
|
||||||
|
|
||||||
|
create index idx_event_payload_gin
|
||||||
|
on eventhub.event using gin(payload);
|
||||||
|
|
||||||
|
create table eventhub.event_detail (
|
||||||
|
event_occurred_at timestamptz not null,
|
||||||
|
event_id uuid not null,
|
||||||
|
detail_type text not null,
|
||||||
|
attributes jsonb not null default '{}'::jsonb,
|
||||||
|
created_at timestamptz not null default now(),
|
||||||
|
constraint pk_event_detail primary key (event_occurred_at, event_id, detail_type),
|
||||||
|
constraint fk_event_detail_event foreign key (event_occurred_at, event_id)
|
||||||
|
references eventhub.event(occurred_at, id)
|
||||||
|
on delete cascade
|
||||||
|
);
|
||||||
|
|
||||||
|
create index idx_event_detail_type
|
||||||
|
on eventhub.event_detail(detail_type);
|
||||||
|
|
||||||
|
create index idx_event_detail_attributes_gin
|
||||||
|
on eventhub.event_detail using gin(attributes);
|
||||||
|
|
@ -0,0 +1,75 @@
|
||||||
|
create table eventhub.vehicle (
|
||||||
|
id uuid primary key,
|
||||||
|
tenant_key text not null,
|
||||||
|
event_source_id integer not null references eventhub.event_source(id),
|
||||||
|
created_at timestamptz not null default now(),
|
||||||
|
updated_at timestamptz not null default now()
|
||||||
|
);
|
||||||
|
|
||||||
|
create table eventhub.vehicle_identifier (
|
||||||
|
id uuid primary key,
|
||||||
|
vehicle_id uuid not null references eventhub.vehicle(id) on delete cascade,
|
||||||
|
tenant_key text not null,
|
||||||
|
event_source_id integer not null references eventhub.event_source(id),
|
||||||
|
identifier_type text not null,
|
||||||
|
nation text,
|
||||||
|
identifier_value text not null,
|
||||||
|
valid_from timestamptz,
|
||||||
|
valid_to timestamptz,
|
||||||
|
source_updated_at timestamptz,
|
||||||
|
payload jsonb not null default '{}'::jsonb,
|
||||||
|
created_at timestamptz not null default now(),
|
||||||
|
updated_at timestamptz not null default now(),
|
||||||
|
constraint chk_vehicle_identifier_valid_time_order
|
||||||
|
check (valid_from is null or valid_to is null or valid_from <= valid_to),
|
||||||
|
constraint chk_vehicle_identifier_nation_for_vrn
|
||||||
|
check (
|
||||||
|
identifier_type <> 'VRN'
|
||||||
|
or nation is not null
|
||||||
|
)
|
||||||
|
);
|
||||||
|
|
||||||
|
create index idx_vehicle_lookup_ctx
|
||||||
|
on eventhub.vehicle(tenant_key, event_source_id, updated_at desc);
|
||||||
|
|
||||||
|
create index idx_vehicle_identifier_lookup
|
||||||
|
on eventhub.vehicle_identifier(
|
||||||
|
tenant_key, event_source_id, identifier_type, nation, identifier_value, valid_from desc
|
||||||
|
);
|
||||||
|
|
||||||
|
create unique index ux_vehicle_identifier_exact
|
||||||
|
on eventhub.vehicle_identifier(
|
||||||
|
vehicle_id,
|
||||||
|
identifier_type,
|
||||||
|
coalesce(nation, ''),
|
||||||
|
identifier_value,
|
||||||
|
coalesce(valid_from, '-infinity'::timestamptz),
|
||||||
|
coalesce(valid_to, 'infinity'::timestamptz)
|
||||||
|
);
|
||||||
|
|
||||||
|
alter table eventhub.event
|
||||||
|
rename column vehicle_entity_id to vehicle_id;
|
||||||
|
|
||||||
|
alter table eventhub.event
|
||||||
|
drop constraint if exists event_vehicle_entity_id_fkey;
|
||||||
|
|
||||||
|
alter table eventhub.event
|
||||||
|
add constraint fk_event_vehicle
|
||||||
|
foreign key (vehicle_id)
|
||||||
|
references eventhub.vehicle(id);
|
||||||
|
|
||||||
|
drop index if exists eventhub.idx_event_vehicle_time;
|
||||||
|
|
||||||
|
create index idx_event_vehicle_time
|
||||||
|
on eventhub.event(vehicle_id, occurred_at desc)
|
||||||
|
where vehicle_id is not null;
|
||||||
|
|
||||||
|
alter table eventhub.event
|
||||||
|
drop constraint if exists chk_event_driver_or_vehicle_ref;
|
||||||
|
|
||||||
|
alter table eventhub.event
|
||||||
|
add constraint chk_event_driver_or_vehicle_ref
|
||||||
|
check (
|
||||||
|
driver_entity_id is not null
|
||||||
|
or vehicle_id is not null
|
||||||
|
);
|
||||||
|
|
@ -0,0 +1,241 @@
|
||||||
|
alter table eventhub.vehicle
|
||||||
|
add column if not exists source_vehicle_entity_id text,
|
||||||
|
add column if not exists vin text;
|
||||||
|
|
||||||
|
do $migration$
|
||||||
|
begin
|
||||||
|
if to_regclass('eventhub.vehicle_identifier') is not null
|
||||||
|
and exists (
|
||||||
|
select 1
|
||||||
|
from information_schema.columns
|
||||||
|
where table_schema = 'eventhub'
|
||||||
|
and table_name = 'vehicle_identifier'
|
||||||
|
and column_name = 'vehicle_id'
|
||||||
|
)
|
||||||
|
then
|
||||||
|
execute $sql$
|
||||||
|
update eventhub.vehicle v
|
||||||
|
set source_vehicle_entity_id = src.identifier_value
|
||||||
|
from eventhub.vehicle_identifier src
|
||||||
|
where src.vehicle_id = v.id
|
||||||
|
and src.identifier_type = 'SOURCE_VEHICLE'
|
||||||
|
and v.source_vehicle_entity_id is null
|
||||||
|
$sql$;
|
||||||
|
|
||||||
|
execute $sql$
|
||||||
|
update eventhub.vehicle v
|
||||||
|
set vin = src.identifier_value
|
||||||
|
from eventhub.vehicle_identifier src
|
||||||
|
where src.vehicle_id = v.id
|
||||||
|
and src.identifier_type = 'VIN'
|
||||||
|
and v.vin is null
|
||||||
|
$sql$;
|
||||||
|
end if;
|
||||||
|
end
|
||||||
|
$migration$;
|
||||||
|
|
||||||
|
create table if not exists eventhub.vehicle_registration (
|
||||||
|
id uuid primary key,
|
||||||
|
tenant_key text not null,
|
||||||
|
event_source_id integer not null references eventhub.event_source(id),
|
||||||
|
source_registration_entity_id text,
|
||||||
|
nation text not null,
|
||||||
|
registration_number text not null,
|
||||||
|
valid_from timestamptz,
|
||||||
|
valid_to timestamptz,
|
||||||
|
source_updated_at timestamptz,
|
||||||
|
payload jsonb not null default '{}'::jsonb,
|
||||||
|
created_at timestamptz not null default now(),
|
||||||
|
updated_at timestamptz not null default now(),
|
||||||
|
constraint chk_vehicle_registration_valid_time_order
|
||||||
|
check (valid_from is null or valid_to is null or valid_from <= valid_to)
|
||||||
|
);
|
||||||
|
|
||||||
|
create table if not exists eventhub.vehicle_registration_assignment (
|
||||||
|
id uuid primary key,
|
||||||
|
tenant_key text not null,
|
||||||
|
event_source_id integer not null references eventhub.event_source(id),
|
||||||
|
vehicle_registration_id uuid not null references eventhub.vehicle_registration(id) on delete cascade,
|
||||||
|
vehicle_id uuid not null references eventhub.vehicle(id) on delete cascade,
|
||||||
|
valid_from timestamptz,
|
||||||
|
valid_to timestamptz,
|
||||||
|
source_updated_at timestamptz,
|
||||||
|
payload jsonb not null default '{}'::jsonb,
|
||||||
|
created_at timestamptz not null default now(),
|
||||||
|
updated_at timestamptz not null default now(),
|
||||||
|
constraint chk_vehicle_registration_assignment_valid_time_order
|
||||||
|
check (valid_from is null or valid_to is null or valid_from <= valid_to)
|
||||||
|
);
|
||||||
|
|
||||||
|
do $migration$
|
||||||
|
begin
|
||||||
|
if to_regclass('eventhub.vehicle_identifier') is not null
|
||||||
|
and exists (
|
||||||
|
select 1
|
||||||
|
from information_schema.columns
|
||||||
|
where table_schema = 'eventhub'
|
||||||
|
and table_name = 'vehicle_identifier'
|
||||||
|
and column_name = 'vehicle_id'
|
||||||
|
)
|
||||||
|
then
|
||||||
|
execute $sql$
|
||||||
|
insert into eventhub.vehicle_registration(
|
||||||
|
id, tenant_key, event_source_id, source_registration_entity_id, nation, registration_number,
|
||||||
|
valid_from, valid_to, payload
|
||||||
|
)
|
||||||
|
select gen_random_uuid(),
|
||||||
|
src.tenant_key,
|
||||||
|
src.event_source_id,
|
||||||
|
source_reg.identifier_value,
|
||||||
|
src.nation,
|
||||||
|
src.identifier_value,
|
||||||
|
src.valid_from,
|
||||||
|
src.valid_to,
|
||||||
|
src.payload
|
||||||
|
from eventhub.vehicle_identifier src
|
||||||
|
left join eventhub.vehicle_identifier source_reg on source_reg.vehicle_id = src.vehicle_id
|
||||||
|
and source_reg.identifier_type = 'SOURCE_REGISTRATION'
|
||||||
|
where src.identifier_type = 'VRN'
|
||||||
|
and src.nation is not null
|
||||||
|
and not exists (
|
||||||
|
select 1
|
||||||
|
from eventhub.vehicle_registration existing
|
||||||
|
where existing.tenant_key = src.tenant_key
|
||||||
|
and existing.event_source_id = src.event_source_id
|
||||||
|
and existing.nation = src.nation
|
||||||
|
and existing.registration_number = src.identifier_value
|
||||||
|
and existing.valid_from is not distinct from src.valid_from
|
||||||
|
and existing.valid_to is not distinct from src.valid_to
|
||||||
|
)
|
||||||
|
$sql$;
|
||||||
|
|
||||||
|
execute $sql$
|
||||||
|
insert into eventhub.vehicle_registration_assignment(
|
||||||
|
id, tenant_key, event_source_id, vehicle_registration_id, vehicle_id, valid_from, valid_to, payload
|
||||||
|
)
|
||||||
|
select gen_random_uuid(),
|
||||||
|
reg.tenant_key,
|
||||||
|
reg.event_source_id,
|
||||||
|
reg.id,
|
||||||
|
v.id,
|
||||||
|
reg.valid_from,
|
||||||
|
reg.valid_to,
|
||||||
|
jsonb_build_object('migrated_from', 'vehicle_identifier')
|
||||||
|
from eventhub.vehicle_registration reg
|
||||||
|
join eventhub.vehicle_identifier vrn on vrn.tenant_key = reg.tenant_key
|
||||||
|
and vrn.event_source_id = reg.event_source_id
|
||||||
|
and vrn.identifier_type = 'VRN'
|
||||||
|
and vrn.nation = reg.nation
|
||||||
|
and vrn.identifier_value = reg.registration_number
|
||||||
|
and vrn.valid_from is not distinct from reg.valid_from
|
||||||
|
and vrn.valid_to is not distinct from reg.valid_to
|
||||||
|
join eventhub.vehicle v on v.id = vrn.vehicle_id
|
||||||
|
where not exists (
|
||||||
|
select 1
|
||||||
|
from eventhub.vehicle_registration_assignment existing
|
||||||
|
where existing.vehicle_registration_id = reg.id
|
||||||
|
and existing.vehicle_id = v.id
|
||||||
|
and existing.valid_from is not distinct from reg.valid_from
|
||||||
|
and existing.valid_to is not distinct from reg.valid_to
|
||||||
|
)
|
||||||
|
$sql$;
|
||||||
|
end if;
|
||||||
|
end
|
||||||
|
$migration$;
|
||||||
|
|
||||||
|
do $migration$
|
||||||
|
begin
|
||||||
|
if exists (
|
||||||
|
select 1
|
||||||
|
from information_schema.columns
|
||||||
|
where table_schema = 'eventhub'
|
||||||
|
and table_name = 'event'
|
||||||
|
and column_name = 'vehicle_entity_id'
|
||||||
|
)
|
||||||
|
and not exists (
|
||||||
|
select 1
|
||||||
|
from information_schema.columns
|
||||||
|
where table_schema = 'eventhub'
|
||||||
|
and table_name = 'event'
|
||||||
|
and column_name = 'vehicle_id'
|
||||||
|
)
|
||||||
|
then
|
||||||
|
alter table eventhub.event rename column vehicle_entity_id to vehicle_id;
|
||||||
|
end if;
|
||||||
|
|
||||||
|
if not exists (
|
||||||
|
select 1
|
||||||
|
from information_schema.columns
|
||||||
|
where table_schema = 'eventhub'
|
||||||
|
and table_name = 'event'
|
||||||
|
and column_name = 'vehicle_id'
|
||||||
|
)
|
||||||
|
then
|
||||||
|
alter table eventhub.event add column vehicle_id uuid references eventhub.vehicle(id);
|
||||||
|
end if;
|
||||||
|
end
|
||||||
|
$migration$;
|
||||||
|
|
||||||
|
alter table eventhub.event
|
||||||
|
drop constraint if exists event_vehicle_entity_id_fkey;
|
||||||
|
|
||||||
|
alter table eventhub.event
|
||||||
|
drop constraint if exists fk_event_vehicle;
|
||||||
|
|
||||||
|
do $migration$
|
||||||
|
begin
|
||||||
|
if exists (
|
||||||
|
select 1
|
||||||
|
from information_schema.columns
|
||||||
|
where table_schema = 'eventhub'
|
||||||
|
and table_name = 'event'
|
||||||
|
and column_name = 'vehicle_id'
|
||||||
|
)
|
||||||
|
then
|
||||||
|
alter table eventhub.event
|
||||||
|
add constraint fk_event_vehicle
|
||||||
|
foreign key (vehicle_id)
|
||||||
|
references eventhub.vehicle(id)
|
||||||
|
not valid;
|
||||||
|
end if;
|
||||||
|
end
|
||||||
|
$migration$;
|
||||||
|
|
||||||
|
alter table eventhub.event
|
||||||
|
add column if not exists vehicle_registration_id uuid references eventhub.vehicle_registration(id);
|
||||||
|
|
||||||
|
create index if not exists idx_vehicle_source_entity
|
||||||
|
on eventhub.vehicle(tenant_key, event_source_id, source_vehicle_entity_id)
|
||||||
|
where source_vehicle_entity_id is not null;
|
||||||
|
|
||||||
|
create index if not exists idx_vehicle_vin
|
||||||
|
on eventhub.vehicle(tenant_key, event_source_id, vin)
|
||||||
|
where vin is not null;
|
||||||
|
|
||||||
|
create index if not exists idx_vehicle_registration_source_entity
|
||||||
|
on eventhub.vehicle_registration(tenant_key, event_source_id, source_registration_entity_id)
|
||||||
|
where source_registration_entity_id is not null;
|
||||||
|
|
||||||
|
create index if not exists idx_vehicle_registration_plate
|
||||||
|
on eventhub.vehicle_registration(tenant_key, event_source_id, nation, registration_number, valid_from desc);
|
||||||
|
|
||||||
|
create index if not exists idx_vehicle_registration_assignment_registration_time
|
||||||
|
on eventhub.vehicle_registration_assignment(vehicle_registration_id, valid_from desc, valid_to);
|
||||||
|
|
||||||
|
create index if not exists idx_vehicle_registration_assignment_vehicle_time
|
||||||
|
on eventhub.vehicle_registration_assignment(vehicle_id, valid_from desc, valid_to);
|
||||||
|
|
||||||
|
create index if not exists idx_event_vehicle_registration_time
|
||||||
|
on eventhub.event(vehicle_registration_id, occurred_at desc)
|
||||||
|
where vehicle_registration_id is not null;
|
||||||
|
|
||||||
|
alter table eventhub.event
|
||||||
|
drop constraint if exists chk_event_driver_or_vehicle_ref;
|
||||||
|
|
||||||
|
alter table eventhub.event
|
||||||
|
add constraint chk_event_driver_or_vehicle_ref
|
||||||
|
check (
|
||||||
|
driver_entity_id is not null
|
||||||
|
or vehicle_id is not null
|
||||||
|
or vehicle_registration_id is not null
|
||||||
|
);
|
||||||
|
|
@ -0,0 +1,24 @@
|
||||||
|
alter table eventhub.event
|
||||||
|
drop constraint if exists event_vehicle_entity_id_fkey;
|
||||||
|
|
||||||
|
alter table eventhub.event
|
||||||
|
drop constraint if exists fk_event_vehicle;
|
||||||
|
|
||||||
|
do $migration$
|
||||||
|
begin
|
||||||
|
if exists (
|
||||||
|
select 1
|
||||||
|
from information_schema.columns
|
||||||
|
where table_schema = 'eventhub'
|
||||||
|
and table_name = 'event'
|
||||||
|
and column_name = 'vehicle_id'
|
||||||
|
)
|
||||||
|
then
|
||||||
|
alter table eventhub.event
|
||||||
|
add constraint fk_event_vehicle
|
||||||
|
foreign key (vehicle_id)
|
||||||
|
references eventhub.vehicle(id)
|
||||||
|
not valid;
|
||||||
|
end if;
|
||||||
|
end
|
||||||
|
$migration$;
|
||||||
|
|
@ -0,0 +1,193 @@
|
||||||
|
drop index if exists eventhub.idx_vehicle_registration_plate;
|
||||||
|
|
||||||
|
alter table eventhub.vehicle_registration
|
||||||
|
drop constraint if exists chk_vehicle_registration_valid_time_order;
|
||||||
|
|
||||||
|
alter table eventhub.vehicle_registration
|
||||||
|
drop column if exists valid_from,
|
||||||
|
drop column if exists valid_to;
|
||||||
|
|
||||||
|
create index if not exists idx_vehicle_registration_plate
|
||||||
|
on eventhub.vehicle_registration(tenant_key, event_source_id, nation, registration_number);
|
||||||
|
|
||||||
|
update eventhub.source_master_entity
|
||||||
|
set valid_from = null,
|
||||||
|
valid_to = null,
|
||||||
|
updated_at = now()
|
||||||
|
where entity_type = 'VEHICLE_REGISTRATION'
|
||||||
|
and (valid_from is not null or valid_to is not null);
|
||||||
|
|
||||||
|
with vehicle_entities as (
|
||||||
|
select tenant_key,
|
||||||
|
event_source_id,
|
||||||
|
nullif(source_entity_id, '') as source_vehicle_entity_id,
|
||||||
|
nullif(source_external_key, '') as vin
|
||||||
|
from eventhub.source_master_entity
|
||||||
|
where entity_type = 'VEHICLE'
|
||||||
|
and (nullif(source_entity_id, '') is not null or nullif(source_external_key, '') is not null)
|
||||||
|
)
|
||||||
|
update eventhub.vehicle vehicle
|
||||||
|
set source_vehicle_entity_id = coalesce(vehicle.source_vehicle_entity_id, vehicle_entities.source_vehicle_entity_id),
|
||||||
|
vin = coalesce(vehicle.vin, vehicle_entities.vin),
|
||||||
|
updated_at = now()
|
||||||
|
from vehicle_entities
|
||||||
|
where vehicle.tenant_key = vehicle_entities.tenant_key
|
||||||
|
and vehicle.event_source_id = vehicle_entities.event_source_id
|
||||||
|
and (
|
||||||
|
vehicle.source_vehicle_entity_id = vehicle_entities.source_vehicle_entity_id
|
||||||
|
or vehicle.vin = vehicle_entities.vin
|
||||||
|
);
|
||||||
|
|
||||||
|
with vehicle_entities as (
|
||||||
|
select tenant_key,
|
||||||
|
event_source_id,
|
||||||
|
nullif(source_entity_id, '') as source_vehicle_entity_id,
|
||||||
|
nullif(source_external_key, '') as vin
|
||||||
|
from eventhub.source_master_entity
|
||||||
|
where entity_type = 'VEHICLE'
|
||||||
|
and (nullif(source_entity_id, '') is not null or nullif(source_external_key, '') is not null)
|
||||||
|
)
|
||||||
|
insert into eventhub.vehicle(id, tenant_key, event_source_id, source_vehicle_entity_id, vin)
|
||||||
|
select gen_random_uuid(),
|
||||||
|
vehicle_entities.tenant_key,
|
||||||
|
vehicle_entities.event_source_id,
|
||||||
|
vehicle_entities.source_vehicle_entity_id,
|
||||||
|
vehicle_entities.vin
|
||||||
|
from vehicle_entities
|
||||||
|
where not exists (
|
||||||
|
select 1
|
||||||
|
from eventhub.vehicle vehicle
|
||||||
|
where vehicle.tenant_key = vehicle_entities.tenant_key
|
||||||
|
and vehicle.event_source_id = vehicle_entities.event_source_id
|
||||||
|
and (
|
||||||
|
vehicle.source_vehicle_entity_id = vehicle_entities.source_vehicle_entity_id
|
||||||
|
or vehicle.vin = vehicle_entities.vin
|
||||||
|
)
|
||||||
|
);
|
||||||
|
|
||||||
|
with registration_entities as (
|
||||||
|
select tenant_key,
|
||||||
|
event_source_id,
|
||||||
|
nullif(source_entity_id, '') as source_registration_entity_id,
|
||||||
|
coalesce(
|
||||||
|
nullif(payload ->> 'registration_nation', ''),
|
||||||
|
nullif(split_part(source_external_key, ':', 1), '')
|
||||||
|
) as nation,
|
||||||
|
coalesce(
|
||||||
|
nullif(payload ->> 'registration_number', ''),
|
||||||
|
case
|
||||||
|
when position(':' in source_external_key) > 0
|
||||||
|
then nullif(substring(source_external_key from position(':' in source_external_key) + 1), '')
|
||||||
|
else nullif(source_external_key, '')
|
||||||
|
end
|
||||||
|
) as registration_number,
|
||||||
|
source_updated_at
|
||||||
|
from eventhub.source_master_entity
|
||||||
|
where entity_type = 'VEHICLE_REGISTRATION'
|
||||||
|
)
|
||||||
|
update eventhub.vehicle_registration registration
|
||||||
|
set source_registration_entity_id = coalesce(
|
||||||
|
registration.source_registration_entity_id,
|
||||||
|
registration_entities.source_registration_entity_id
|
||||||
|
),
|
||||||
|
nation = coalesce(registration_entities.nation, registration.nation),
|
||||||
|
registration_number = coalesce(registration_entities.registration_number, registration.registration_number),
|
||||||
|
source_updated_at = registration_entities.source_updated_at,
|
||||||
|
updated_at = now()
|
||||||
|
from registration_entities
|
||||||
|
where registration_entities.nation is not null
|
||||||
|
and registration_entities.registration_number is not null
|
||||||
|
and registration.tenant_key = registration_entities.tenant_key
|
||||||
|
and registration.event_source_id = registration_entities.event_source_id
|
||||||
|
and (
|
||||||
|
registration.source_registration_entity_id = registration_entities.source_registration_entity_id
|
||||||
|
or (
|
||||||
|
registration.nation = registration_entities.nation
|
||||||
|
and registration.registration_number = registration_entities.registration_number
|
||||||
|
)
|
||||||
|
);
|
||||||
|
|
||||||
|
with registration_entities as (
|
||||||
|
select tenant_key,
|
||||||
|
event_source_id,
|
||||||
|
nullif(source_entity_id, '') as source_registration_entity_id,
|
||||||
|
coalesce(
|
||||||
|
nullif(payload ->> 'registration_nation', ''),
|
||||||
|
nullif(split_part(source_external_key, ':', 1), '')
|
||||||
|
) as nation,
|
||||||
|
coalesce(
|
||||||
|
nullif(payload ->> 'registration_number', ''),
|
||||||
|
case
|
||||||
|
when position(':' in source_external_key) > 0
|
||||||
|
then nullif(substring(source_external_key from position(':' in source_external_key) + 1), '')
|
||||||
|
else nullif(source_external_key, '')
|
||||||
|
end
|
||||||
|
) as registration_number,
|
||||||
|
source_updated_at
|
||||||
|
from eventhub.source_master_entity
|
||||||
|
where entity_type = 'VEHICLE_REGISTRATION'
|
||||||
|
)
|
||||||
|
insert into eventhub.vehicle_registration(
|
||||||
|
id, tenant_key, event_source_id, source_registration_entity_id,
|
||||||
|
nation, registration_number, source_updated_at, payload
|
||||||
|
)
|
||||||
|
select gen_random_uuid(),
|
||||||
|
registration_entities.tenant_key,
|
||||||
|
registration_entities.event_source_id,
|
||||||
|
registration_entities.source_registration_entity_id,
|
||||||
|
registration_entities.nation,
|
||||||
|
registration_entities.registration_number,
|
||||||
|
registration_entities.source_updated_at,
|
||||||
|
jsonb_build_object('source', 'master-data')
|
||||||
|
from registration_entities
|
||||||
|
where registration_entities.nation is not null
|
||||||
|
and registration_entities.registration_number is not null
|
||||||
|
and not exists (
|
||||||
|
select 1
|
||||||
|
from eventhub.vehicle_registration registration
|
||||||
|
where registration.tenant_key = registration_entities.tenant_key
|
||||||
|
and registration.event_source_id = registration_entities.event_source_id
|
||||||
|
and (
|
||||||
|
registration.source_registration_entity_id = registration_entities.source_registration_entity_id
|
||||||
|
or (
|
||||||
|
registration.nation = registration_entities.nation
|
||||||
|
and registration.registration_number = registration_entities.registration_number
|
||||||
|
)
|
||||||
|
)
|
||||||
|
);
|
||||||
|
|
||||||
|
insert into eventhub.vehicle_registration_assignment(
|
||||||
|
id, tenant_key, event_source_id, vehicle_registration_id, vehicle_id,
|
||||||
|
valid_from, valid_to, source_updated_at, payload
|
||||||
|
)
|
||||||
|
select gen_random_uuid(),
|
||||||
|
relation.tenant_key,
|
||||||
|
relation.event_source_id,
|
||||||
|
registration.id,
|
||||||
|
vehicle.id,
|
||||||
|
relation.valid_from,
|
||||||
|
relation.valid_to,
|
||||||
|
relation.source_updated_at,
|
||||||
|
jsonb_build_object(
|
||||||
|
'source', 'master-data',
|
||||||
|
'sourceRelationId', relation.id,
|
||||||
|
'relationKey', relation.relation_key
|
||||||
|
)
|
||||||
|
from eventhub.source_master_relation relation
|
||||||
|
join eventhub.vehicle_registration registration on registration.tenant_key = relation.tenant_key
|
||||||
|
and registration.event_source_id = relation.event_source_id
|
||||||
|
and registration.source_registration_entity_id = relation.from_source_entity_id
|
||||||
|
join eventhub.vehicle vehicle on vehicle.tenant_key = relation.tenant_key
|
||||||
|
and vehicle.event_source_id = relation.event_source_id
|
||||||
|
and vehicle.source_vehicle_entity_id = relation.to_source_entity_id
|
||||||
|
where relation.relation_type = 'VEHICLE_REGISTRATION_VEHICLE'
|
||||||
|
and relation.from_entity_type = 'VEHICLE_REGISTRATION'
|
||||||
|
and relation.to_entity_type = 'VEHICLE'
|
||||||
|
and not exists (
|
||||||
|
select 1
|
||||||
|
from eventhub.vehicle_registration_assignment existing
|
||||||
|
where existing.vehicle_registration_id = registration.id
|
||||||
|
and existing.vehicle_id = vehicle.id
|
||||||
|
and existing.valid_from is not distinct from relation.valid_from
|
||||||
|
and existing.valid_to is not distinct from relation.valid_to
|
||||||
|
);
|
||||||
|
|
@ -0,0 +1,37 @@
|
||||||
|
<?xml version="1.0" encoding="UTF-8"?>
|
||||||
|
<configuration>
|
||||||
|
|
||||||
|
<springProperty scope="context" name="logFile" source="logging.file.name" defaultValue="logs/eventhub-ingestion-service.log"/>
|
||||||
|
<springProperty scope="context" name="maxFileSize" source="logging.logback.rollingpolicy.max-file-size" defaultValue="50MB"/>
|
||||||
|
<springProperty scope="context" name="maxHistory" source="logging.logback.rollingpolicy.max-history" defaultValue="14"/>
|
||||||
|
<springProperty scope="context" name="totalSizeCap" source="logging.logback.rollingpolicy.total-size-cap" defaultValue="1GB"/>
|
||||||
|
|
||||||
|
<appender name="CONSOLE" class="ch.qos.logback.core.ConsoleAppender">
|
||||||
|
<encoder>
|
||||||
|
<pattern>%d{yyyy-MM-dd'T'HH:mm:ss.SSSXXX} %-5level ${PID:-unknown} --- [%thread] %logger{39} : %msg%n%ex</pattern>
|
||||||
|
<charset>UTF-8</charset>
|
||||||
|
</encoder>
|
||||||
|
</appender>
|
||||||
|
|
||||||
|
<appender name="FILE" class="ch.qos.logback.core.rolling.RollingFileAppender">
|
||||||
|
<file>${logFile}</file>
|
||||||
|
<append>true</append>
|
||||||
|
<immediateFlush>true</immediateFlush>
|
||||||
|
<encoder>
|
||||||
|
<pattern>%d{yyyy-MM-dd'T'HH:mm:ss.SSSXXX} %-5level ${PID:-unknown} --- [%thread] %logger{39} : %msg%n%ex</pattern>
|
||||||
|
<charset>UTF-8</charset>
|
||||||
|
</encoder>
|
||||||
|
<rollingPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy">
|
||||||
|
<fileNamePattern>${logFile}.%d{yyyy-MM-dd}.%i.gz</fileNamePattern>
|
||||||
|
<maxFileSize>${maxFileSize}</maxFileSize>
|
||||||
|
<maxHistory>${maxHistory}</maxHistory>
|
||||||
|
<totalSizeCap>${totalSizeCap}</totalSizeCap>
|
||||||
|
</rollingPolicy>
|
||||||
|
</appender>
|
||||||
|
|
||||||
|
<root level="INFO">
|
||||||
|
<appender-ref ref="CONSOLE"/>
|
||||||
|
<appender-ref ref="FILE"/>
|
||||||
|
</root>
|
||||||
|
|
||||||
|
</configuration>
|
||||||
|
|
@ -8,16 +8,104 @@
|
||||||
* the best matching CardVehiclesUsed row for the activity timestamp when one is
|
* the best matching CardVehiclesUsed row for the activity timestamp when one is
|
||||||
* available.
|
* available.
|
||||||
*/
|
*/
|
||||||
select
|
with OrgTree as (
|
||||||
cast(ca.ID as varchar(128)) as source_row_id,
|
select org.I_90021_OID
|
||||||
cast(ca.ID as varchar(128)) as card_activity_id,
|
from dbo.GetOrganisationTree(null, :organisationId, 0, null) org
|
||||||
concat('TACHOGRAPH:CARD_ACTIVITY:', ca.ID) as external_source_event_id,
|
where :organisationId is not null
|
||||||
|
)
|
||||||
|
,
|
||||||
|
CandidateActivity as (
|
||||||
|
select
|
||||||
|
ca.ID,
|
||||||
|
ca.BeginTime,
|
||||||
|
activity_time.EndTime,
|
||||||
|
ca.Activity,
|
||||||
|
ca.Slot,
|
||||||
|
ca.CardStatus,
|
||||||
|
ca.DrivingStatus,
|
||||||
|
ca.ID_FileLog,
|
||||||
|
cda.RecordDate,
|
||||||
|
cda.RecordDateTo,
|
||||||
|
cda.ID_FileLog as cda_filelog_id,
|
||||||
|
c.ID as card_id,
|
||||||
|
c.ID_Driver as driver_id,
|
||||||
|
c.ID_Nation as driver_card_nation_id,
|
||||||
|
c.CardNumber as driver_card_number,
|
||||||
|
c.ID_FileLog as card_filelog_id,
|
||||||
|
coalesce(ca.ID_FileLog, cda.ID_FileLog, c.ID_FileLog) as file_log_id
|
||||||
|
from dbo.CardActivity ca
|
||||||
|
join dbo.CardDailyActivity cda on cda.ID = ca.ID_DailyActivity
|
||||||
|
join dbo.Card c on c.ID = cda.ID_Card
|
||||||
|
cross apply (values (dateadd(minute, coalesce(ca.Duration, 0), ca.BeginTime))) activity_time(EndTime)
|
||||||
|
where (:occurredTo is null or ca.BeginTime < :occurredTo)
|
||||||
|
and (
|
||||||
|
:occurredFrom is null
|
||||||
|
or ca.BeginTime >= :occurredFrom
|
||||||
|
or activity_time.EndTime >= :occurredFrom
|
||||||
|
)
|
||||||
|
and (
|
||||||
|
:organisationId is null
|
||||||
|
or exists (
|
||||||
|
select 1
|
||||||
|
from dbo.Driver_I_90021 rel
|
||||||
|
join OrgTree on OrgTree.I_90021_OID = rel.ID_I_90021
|
||||||
|
where rel.ID_Driver = c.ID_Driver
|
||||||
|
and rel.GILT_BIS is null
|
||||||
|
)
|
||||||
|
)
|
||||||
|
)
|
||||||
|
,
|
||||||
|
Base as (
|
||||||
|
select
|
||||||
|
ca.ID,
|
||||||
|
ca.BeginTime,
|
||||||
|
ca.EndTime,
|
||||||
|
ca.Activity,
|
||||||
|
ca.Slot,
|
||||||
|
ca.CardStatus,
|
||||||
|
ca.DrivingStatus,
|
||||||
|
ca.ID_FileLog,
|
||||||
|
ca.cda_filelog_id,
|
||||||
|
ca.card_filelog_id,
|
||||||
|
|
||||||
ca.BeginTime as occurred_at,
|
coalesce(fl.DownloadDate, fl.OriginalDownloadDate, fl.TStamp, fl.CreationDate) as received_partner_at,
|
||||||
coalesce(fl.DownloadDate, fl.OriginalDownloadDate, fl.TStamp, fl.CreationDate) as received_partner_at,
|
coalesce(fl.ID, ca.ID_FileLog, ca.cda_filelog_id, ca.card_filelog_id) as source_package_id_raw,
|
||||||
ca.Activity as activity_code,
|
coalesce(fl.ID_Card, ca.card_id) as source_package_entity_id_raw,
|
||||||
ca.Activity as activity_text,
|
coalesce(fl.DownloadFrom, ca.RecordDate) as source_package_period_from,
|
||||||
case upper(coalesce(ca.Activity, ''))
|
coalesce(fl.DownloadTo, ca.RecordDateTo, dateadd(day, 1, ca.RecordDate)) as source_package_period_to,
|
||||||
|
coalesce(fl.CreationDate, fl.TStamp) as source_package_imported_at,
|
||||||
|
|
||||||
|
ca.driver_id,
|
||||||
|
cn.AlphaCode as driver_card_nation,
|
||||||
|
ca.driver_card_number,
|
||||||
|
coalesce(cvu.ID_Vehicle, v.ID) as vehicle_registration_id,
|
||||||
|
null as vehicle_vin,
|
||||||
|
vn.AlphaCode as vehicle_registration_nation,
|
||||||
|
v.VRN as vehicle_registration_number
|
||||||
|
from CandidateActivity ca
|
||||||
|
left join dbo.FileLog fl on fl.ID = ca.file_log_id
|
||||||
|
left join dbo.Nation cn on cn.ID = ca.driver_card_nation_id
|
||||||
|
outer apply (
|
||||||
|
select top 1 used.ID_Vehicle
|
||||||
|
from dbo.CardVehiclesUsed used
|
||||||
|
where used.ID_Card = ca.card_id
|
||||||
|
and (used.FirstUse is null or used.FirstUse <= ca.BeginTime)
|
||||||
|
and (used.LastUse is null or used.LastUse >= ca.BeginTime)
|
||||||
|
order by
|
||||||
|
used.FirstUse desc,
|
||||||
|
used.ID desc
|
||||||
|
) cvu
|
||||||
|
left join dbo.Vehicle v on v.ID = cvu.ID_Vehicle
|
||||||
|
left join dbo.Nation vn on vn.ID = v.ID_Nation
|
||||||
|
)
|
||||||
|
select
|
||||||
|
cast(base.ID as varchar(128)) as source_row_id,
|
||||||
|
concat('TACHOGRAPH:CARD_ACTIVITY:', base.ID, ':', evt.lifecycle) as external_source_event_id,
|
||||||
|
|
||||||
|
evt.occurred_at as occurred_at,
|
||||||
|
base.received_partner_at,
|
||||||
|
base.Activity as activity_code,
|
||||||
|
case upper(coalesce(base.Activity, ''))
|
||||||
when 'DRIVING' then 'DRIVE'
|
when 'DRIVING' then 'DRIVE'
|
||||||
when 'DRIVE' then 'DRIVE'
|
when 'DRIVE' then 'DRIVE'
|
||||||
when 'WORK' then 'WORK'
|
when 'WORK' then 'WORK'
|
||||||
|
|
@ -28,67 +116,35 @@ select
|
||||||
when 'REST' then 'BREAK_REST'
|
when 'REST' then 'BREAK_REST'
|
||||||
else 'UNKNOWN_ACTIVITY'
|
else 'UNKNOWN_ACTIVITY'
|
||||||
end as event_type,
|
end as event_type,
|
||||||
'SNAPSHOT' as lifecycle,
|
evt.lifecycle as lifecycle,
|
||||||
ca.Slot as card_slot,
|
base.Slot as card_slot,
|
||||||
ca.CardStatus as card_status,
|
base.CardStatus as card_status,
|
||||||
ca.DrivingStatus as driving_status,
|
base.DrivingStatus as driving_status,
|
||||||
cast(null as bigint) as odometer_m,
|
cast(null as bigint) as odometer_m,
|
||||||
|
|
||||||
cast(d.ID as varchar(128)) as driver_source_entity_id,
|
cast(base.driver_id as varchar(128)) as driver_source_entity_id,
|
||||||
cn.AlphaCode as driver_card_nation,
|
base.driver_card_nation,
|
||||||
c.CardNumber as driver_card_number,
|
base.driver_card_number,
|
||||||
|
|
||||||
cast(coalesce(cvu.ID_Vehicle, v.ID) as varchar(128)) as vehicle_source_entity_id,
|
cast(null as varchar(128)) as vehicle_source_entity_id,
|
||||||
coalesce(cvu.VIN, vi.VIN) as vehicle_vin,
|
base.vehicle_vin,
|
||||||
vn.AlphaCode as vehicle_registration_nation,
|
cast(base.vehicle_registration_id as varchar(128)) as vehicle_registration_source_entity_id,
|
||||||
v.VRN as vehicle_registration_number,
|
base.vehicle_registration_nation,
|
||||||
|
base.vehicle_registration_number,
|
||||||
|
|
||||||
'DRIVER_CARD' as source_package_kind,
|
'DRIVER_CARD' as source_package_kind,
|
||||||
cast(coalesce(fl.ID, ca.ID_FileLog, cda.ID_FileLog, c.ID_FileLog, ca.ID) as varchar(128)) as source_package_id,
|
cast(base.source_package_id_raw as varchar(128)) as source_package_id,
|
||||||
cast(coalesce(fl.ID_Card, c.ID) as varchar(128)) as source_package_entity_id,
|
cast(base.source_package_entity_id_raw as varchar(128)) as source_package_entity_id,
|
||||||
coalesce(fl.DownloadFrom, cda.RecordDate) as source_package_period_from,
|
base.source_package_period_from,
|
||||||
coalesce(fl.DownloadTo, cda.RecordDateTo, dateadd(day, 1, cda.RecordDate)) as source_package_period_to,
|
base.source_package_period_to,
|
||||||
coalesce(fl.CreationDate, fl.TStamp) as source_package_imported_at
|
base.source_package_imported_at
|
||||||
from dbo.CardActivity ca
|
from Base base
|
||||||
join dbo.CardDailyActivity cda on cda.ID = ca.ID_DailyActivity
|
cross apply (values
|
||||||
join dbo.Card c on c.ID = cda.ID_Card
|
('START', base.BeginTime),
|
||||||
left join dbo.FileLog fl on fl.ID = coalesce(ca.ID_FileLog, cda.ID_FileLog, c.ID_FileLog)
|
('END', base.EndTime)
|
||||||
left join dbo.Driver d on d.ID = c.ID_Driver
|
) evt(lifecycle, occurred_at)
|
||||||
left join dbo.Nation cn on cn.ID = c.ID_Nation
|
where (:occurredFrom is null or evt.occurred_at >= :occurredFrom)
|
||||||
outer apply (
|
and (:occurredTo is null or evt.occurred_at < :occurredTo)
|
||||||
select top 1 used.ID_Vehicle,
|
|
||||||
used.VIN,
|
|
||||||
used.OdoBegin
|
|
||||||
from dbo.CardVehiclesUsed used
|
|
||||||
where used.ID_Card = c.ID
|
|
||||||
and (used.FirstUse is null or used.FirstUse <= ca.BeginTime)
|
|
||||||
and (used.LastUse is null or used.LastUse >= ca.BeginTime)
|
|
||||||
order by
|
|
||||||
case when used.FirstUse is null then 1 else 0 end,
|
|
||||||
used.FirstUse desc,
|
|
||||||
used.ID desc
|
|
||||||
) cvu
|
|
||||||
left join dbo.Vehicle v on v.ID = cvu.ID_Vehicle
|
|
||||||
left join dbo.VehicleIdentification vi on vi.ID = v.ID_VehicleIdentification
|
|
||||||
left join dbo.Nation vn on vn.ID = v.ID_Nation
|
|
||||||
where (:occurredFrom is null or ca.BeginTime >= :occurredFrom)
|
|
||||||
and (:occurredTo is null or ca.BeginTime < :occurredTo)
|
|
||||||
and (
|
|
||||||
:lastSourcePackageImportedAt is null
|
|
||||||
or coalesce(fl.CreationDate, fl.TStamp) > :lastSourcePackageImportedAt
|
|
||||||
or (
|
|
||||||
coalesce(fl.CreationDate, fl.TStamp) = :lastSourcePackageImportedAt
|
|
||||||
and coalesce(fl.ID, ca.ID_FileLog, cda.ID_FileLog, c.ID_FileLog, ca.ID) > try_convert(int, :lastSourcePackageId)
|
|
||||||
)
|
|
||||||
or (
|
|
||||||
coalesce(fl.CreationDate, fl.TStamp) is null
|
|
||||||
and (
|
|
||||||
:lastSourcePackageId is null
|
|
||||||
or coalesce(fl.ID, ca.ID_FileLog, cda.ID_FileLog, c.ID_FileLog, ca.ID) > try_convert(int, :lastSourcePackageId)
|
|
||||||
)
|
|
||||||
)
|
|
||||||
)
|
|
||||||
/*
|
/*
|
||||||
* Organisation filtering can use FileLog.I_90021_ID / FileLog.OrgID or
|
* Organisation filter: driver membership in GetOrganisationTree(null, :organisationId, 0, null).
|
||||||
* Driver_I_90021 / Vehicle_I_90021 once subtree semantics are confirmed.
|
|
||||||
*/
|
*/
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,96 @@
|
||||||
|
/*
|
||||||
|
* CardBorderCrossing BORDER_CROSSING extraction for the bytebar tachograph schema.
|
||||||
|
*/
|
||||||
|
with OrgTree as (
|
||||||
|
select org.I_90021_OID
|
||||||
|
from dbo.GetOrganisationTree(null, :organisationId, 0, null) org
|
||||||
|
where :organisationId is not null
|
||||||
|
)
|
||||||
|
,
|
||||||
|
Base as (
|
||||||
|
select
|
||||||
|
border.ID,
|
||||||
|
border.Timestamp as occurred_at,
|
||||||
|
border.Odo,
|
||||||
|
border.ID_FileLog,
|
||||||
|
c.ID as card_id,
|
||||||
|
c.ID_Driver as driver_id,
|
||||||
|
cn.AlphaCode as driver_card_nation,
|
||||||
|
c.CardNumber as driver_card_number,
|
||||||
|
leftNation.AlphaCode as country_from,
|
||||||
|
enteredNation.AlphaCode as country_to,
|
||||||
|
gnss.Latitude as latitude,
|
||||||
|
gnss.Longitude as longitude,
|
||||||
|
v.ID as vehicle_registration_id,
|
||||||
|
v.ID_VehicleIdentification as vehicle_identification_id,
|
||||||
|
vi.VIN as vehicle_vin,
|
||||||
|
vehicleNation.AlphaCode as vehicle_registration_nation,
|
||||||
|
v.VRN as vehicle_registration_number,
|
||||||
|
coalesce(fl.DownloadDate, fl.OriginalDownloadDate, fl.TStamp, fl.CreationDate) as received_partner_at,
|
||||||
|
coalesce(fl.ID, border.ID_FileLog) as source_package_id_raw,
|
||||||
|
coalesce(fl.ID_Card, border.ID_Card) as source_package_entity_id_raw,
|
||||||
|
coalesce(fl.DownloadFrom, border.Timestamp) as source_package_period_from,
|
||||||
|
coalesce(fl.DownloadTo, border.Timestamp) as source_package_period_to,
|
||||||
|
coalesce(fl.CreationDate, fl.TStamp) as source_package_imported_at
|
||||||
|
from dbo.CardBorderCrossing border
|
||||||
|
join dbo.Card c on c.ID = border.ID_Card
|
||||||
|
left join dbo.Nation cn on cn.ID = c.ID_Nation
|
||||||
|
left join dbo.Nation leftNation on leftNation.ID = border.ID_NationLeft
|
||||||
|
left join dbo.Nation enteredNation on enteredNation.ID = border.ID_NationEntered
|
||||||
|
left join dbo.GnssPlace gnss on gnss.ID = border.ID_GnssPlace
|
||||||
|
left join dbo.FileLog fl on fl.ID = border.ID_FileLog
|
||||||
|
outer apply (
|
||||||
|
select top 1 used.ID_Vehicle
|
||||||
|
from dbo.CardVehiclesUsed used
|
||||||
|
where used.ID_Card = border.ID_Card
|
||||||
|
and (used.FirstUse is null or used.FirstUse <= border.Timestamp)
|
||||||
|
and (used.LastUse is null or used.LastUse >= border.Timestamp)
|
||||||
|
order by
|
||||||
|
used.FirstUse desc,
|
||||||
|
used.ID desc
|
||||||
|
) cvu
|
||||||
|
left join dbo.Vehicle v on v.ID = cvu.ID_Vehicle
|
||||||
|
left join dbo.VehicleIdentification vi on vi.ID = v.ID_VehicleIdentification
|
||||||
|
left join dbo.Nation vehicleNation on vehicleNation.ID = v.ID_Nation
|
||||||
|
where (:occurredFrom is null or border.Timestamp >= :occurredFrom)
|
||||||
|
and (:occurredTo is null or border.Timestamp < :occurredTo)
|
||||||
|
and (
|
||||||
|
:organisationId is null
|
||||||
|
or exists (
|
||||||
|
select 1
|
||||||
|
from dbo.Driver_I_90021 rel
|
||||||
|
join OrgTree on OrgTree.I_90021_OID = rel.ID_I_90021
|
||||||
|
where rel.ID_Driver = c.ID_Driver
|
||||||
|
and rel.GILT_BIS is null
|
||||||
|
)
|
||||||
|
)
|
||||||
|
)
|
||||||
|
select
|
||||||
|
cast(base.ID as varchar(128)) as source_row_id,
|
||||||
|
concat('TACHOGRAPH:CARD_BORDER_CROSSING:', base.ID) as external_source_event_id,
|
||||||
|
|
||||||
|
base.occurred_at,
|
||||||
|
base.received_partner_at,
|
||||||
|
cast(base.Odo as bigint) * 1000 as odometer_m,
|
||||||
|
base.latitude,
|
||||||
|
base.longitude,
|
||||||
|
base.country_from,
|
||||||
|
base.country_to,
|
||||||
|
|
||||||
|
cast(base.driver_id as varchar(128)) as driver_source_entity_id,
|
||||||
|
base.driver_card_nation,
|
||||||
|
base.driver_card_number,
|
||||||
|
|
||||||
|
cast(base.vehicle_identification_id as varchar(128)) as vehicle_source_entity_id,
|
||||||
|
base.vehicle_vin,
|
||||||
|
cast(base.vehicle_registration_id as varchar(128)) as vehicle_registration_source_entity_id,
|
||||||
|
base.vehicle_registration_nation,
|
||||||
|
base.vehicle_registration_number,
|
||||||
|
|
||||||
|
'DRIVER_CARD' as source_package_kind,
|
||||||
|
cast(base.source_package_id_raw as varchar(128)) as source_package_id,
|
||||||
|
cast(base.source_package_entity_id_raw as varchar(128)) as source_package_entity_id,
|
||||||
|
base.source_package_period_from,
|
||||||
|
base.source_package_period_to,
|
||||||
|
base.source_package_imported_at
|
||||||
|
from Base base
|
||||||
|
|
@ -0,0 +1,101 @@
|
||||||
|
/*
|
||||||
|
* CardLoadUnload LOAD_UNLOAD extraction for the bytebar tachograph schema.
|
||||||
|
*/
|
||||||
|
with OrgTree as (
|
||||||
|
select org.I_90021_OID
|
||||||
|
from dbo.GetOrganisationTree(null, :organisationId, 0, null) org
|
||||||
|
where :organisationId is not null
|
||||||
|
)
|
||||||
|
,
|
||||||
|
Base as (
|
||||||
|
select
|
||||||
|
lu.ID,
|
||||||
|
lu.Timestamp as occurred_at,
|
||||||
|
lu.OperationType,
|
||||||
|
lu.Odo,
|
||||||
|
lu.ID_FileLog,
|
||||||
|
c.ID as card_id,
|
||||||
|
c.ID_Driver as driver_id,
|
||||||
|
cn.AlphaCode as driver_card_nation,
|
||||||
|
c.CardNumber as driver_card_number,
|
||||||
|
gnss.Latitude as latitude,
|
||||||
|
gnss.Longitude as longitude,
|
||||||
|
v.ID as vehicle_registration_id,
|
||||||
|
v.ID_VehicleIdentification as vehicle_identification_id,
|
||||||
|
vi.VIN as vehicle_vin,
|
||||||
|
vehicleNation.AlphaCode as vehicle_registration_nation,
|
||||||
|
v.VRN as vehicle_registration_number,
|
||||||
|
coalesce(fl.DownloadDate, fl.OriginalDownloadDate, fl.TStamp, fl.CreationDate) as received_partner_at,
|
||||||
|
coalesce(fl.ID, lu.ID_FileLog) as source_package_id_raw,
|
||||||
|
coalesce(fl.ID_Card, lu.ID_Card) as source_package_entity_id_raw,
|
||||||
|
coalesce(fl.DownloadFrom, lu.Timestamp) as source_package_period_from,
|
||||||
|
coalesce(fl.DownloadTo, lu.Timestamp) as source_package_period_to,
|
||||||
|
coalesce(fl.CreationDate, fl.TStamp) as source_package_imported_at
|
||||||
|
from dbo.CardLoadUnload lu
|
||||||
|
join dbo.Card c on c.ID = lu.ID_Card
|
||||||
|
left join dbo.Nation cn on cn.ID = c.ID_Nation
|
||||||
|
left join dbo.GnssPlace gnss on gnss.ID = lu.ID_GnssPlace
|
||||||
|
left join dbo.FileLog fl on fl.ID = lu.ID_FileLog
|
||||||
|
outer apply (
|
||||||
|
select top 1 used.ID_Vehicle
|
||||||
|
from dbo.CardVehiclesUsed used
|
||||||
|
where used.ID_Card = lu.ID_Card
|
||||||
|
and (used.FirstUse is null or used.FirstUse <= lu.Timestamp)
|
||||||
|
and (used.LastUse is null or used.LastUse >= lu.Timestamp)
|
||||||
|
order by
|
||||||
|
used.FirstUse desc,
|
||||||
|
used.ID desc
|
||||||
|
) cvu
|
||||||
|
left join dbo.Vehicle v on v.ID = cvu.ID_Vehicle
|
||||||
|
left join dbo.VehicleIdentification vi on vi.ID = v.ID_VehicleIdentification
|
||||||
|
left join dbo.Nation vehicleNation on vehicleNation.ID = v.ID_Nation
|
||||||
|
where (:occurredFrom is null or lu.Timestamp >= :occurredFrom)
|
||||||
|
and (:occurredTo is null or lu.Timestamp < :occurredTo)
|
||||||
|
and (
|
||||||
|
:organisationId is null
|
||||||
|
or exists (
|
||||||
|
select 1
|
||||||
|
from dbo.Driver_I_90021 rel
|
||||||
|
join OrgTree on OrgTree.I_90021_OID = rel.ID_I_90021
|
||||||
|
where rel.ID_Driver = c.ID_Driver
|
||||||
|
and rel.GILT_BIS is null
|
||||||
|
)
|
||||||
|
)
|
||||||
|
)
|
||||||
|
select
|
||||||
|
cast(base.ID as varchar(128)) as source_row_id,
|
||||||
|
concat('TACHOGRAPH:CARD_LOAD_UNLOAD:', base.ID) as external_source_event_id,
|
||||||
|
|
||||||
|
base.occurred_at,
|
||||||
|
base.received_partner_at,
|
||||||
|
cast(base.Odo as bigint) * 1000 as odometer_m,
|
||||||
|
base.latitude,
|
||||||
|
base.longitude,
|
||||||
|
case base.OperationType
|
||||||
|
when 1 then 'LOAD'
|
||||||
|
when 2 then 'UNLOAD'
|
||||||
|
else 'LOAD_UNLOAD'
|
||||||
|
end as operation,
|
||||||
|
case base.OperationType
|
||||||
|
when 1 then 'LOAD'
|
||||||
|
when 2 then 'UNLOAD'
|
||||||
|
else 'LOAD_UNLOAD'
|
||||||
|
end as event_type,
|
||||||
|
|
||||||
|
cast(base.driver_id as varchar(128)) as driver_source_entity_id,
|
||||||
|
base.driver_card_nation,
|
||||||
|
base.driver_card_number,
|
||||||
|
|
||||||
|
cast(base.vehicle_identification_id as varchar(128)) as vehicle_source_entity_id,
|
||||||
|
base.vehicle_vin,
|
||||||
|
cast(base.vehicle_registration_id as varchar(128)) as vehicle_registration_source_entity_id,
|
||||||
|
base.vehicle_registration_nation,
|
||||||
|
base.vehicle_registration_number,
|
||||||
|
|
||||||
|
'DRIVER_CARD' as source_package_kind,
|
||||||
|
cast(base.source_package_id_raw as varchar(128)) as source_package_id,
|
||||||
|
cast(base.source_package_entity_id_raw as varchar(128)) as source_package_entity_id,
|
||||||
|
base.source_package_period_from,
|
||||||
|
base.source_package_period_to,
|
||||||
|
base.source_package_imported_at
|
||||||
|
from Base base
|
||||||
|
|
@ -0,0 +1,100 @@
|
||||||
|
/*
|
||||||
|
* CardPlaces PLACE extraction for the bytebar tachograph schema.
|
||||||
|
*/
|
||||||
|
with OrgTree as (
|
||||||
|
select org.I_90021_OID
|
||||||
|
from dbo.GetOrganisationTree(null, :organisationId, 0, null) org
|
||||||
|
where :organisationId is not null
|
||||||
|
)
|
||||||
|
,
|
||||||
|
Base as (
|
||||||
|
select
|
||||||
|
place.ID,
|
||||||
|
place.EntryTime as occurred_at,
|
||||||
|
cast(place.EntryType as int) as entry_type,
|
||||||
|
place.Odo,
|
||||||
|
place.ID_FileLog,
|
||||||
|
c.ID as card_id,
|
||||||
|
c.ID_Driver as driver_id,
|
||||||
|
cn.AlphaCode as driver_card_nation,
|
||||||
|
c.CardNumber as driver_card_number,
|
||||||
|
placeNation.AlphaCode as country,
|
||||||
|
coalesce(region.AlphaCode, region.Name) as region,
|
||||||
|
gnss.Latitude as latitude,
|
||||||
|
gnss.Longitude as longitude,
|
||||||
|
v.ID as vehicle_registration_id,
|
||||||
|
v.ID_VehicleIdentification as vehicle_identification_id,
|
||||||
|
vi.VIN as vehicle_vin,
|
||||||
|
vehicleNation.AlphaCode as vehicle_registration_nation,
|
||||||
|
v.VRN as vehicle_registration_number,
|
||||||
|
coalesce(fl.DownloadDate, fl.OriginalDownloadDate, fl.TStamp, fl.CreationDate) as received_partner_at,
|
||||||
|
coalesce(fl.ID, place.ID_FileLog) as source_package_id_raw,
|
||||||
|
coalesce(fl.ID_Card, place.ID_Card) as source_package_entity_id_raw,
|
||||||
|
coalesce(fl.DownloadFrom, place.EntryTime) as source_package_period_from,
|
||||||
|
coalesce(fl.DownloadTo, place.EntryTime) as source_package_period_to,
|
||||||
|
coalesce(fl.CreationDate, fl.TStamp) as source_package_imported_at
|
||||||
|
from dbo.CardPlaces place
|
||||||
|
join dbo.Card c on c.ID = place.ID_Card
|
||||||
|
left join dbo.Nation cn on cn.ID = c.ID_Nation
|
||||||
|
left join dbo.Nation placeNation on placeNation.ID = place.ID_Nation
|
||||||
|
left join dbo.Region region on region.ID = place.ID_Region
|
||||||
|
left join dbo.GnssPlace gnss on gnss.ID = place.ID_GnssPlace
|
||||||
|
left join dbo.FileLog fl on fl.ID = place.ID_FileLog
|
||||||
|
outer apply (
|
||||||
|
select top 1 used.ID_Vehicle
|
||||||
|
from dbo.CardVehiclesUsed used
|
||||||
|
where used.ID_Card = place.ID_Card
|
||||||
|
and (used.FirstUse is null or used.FirstUse <= place.EntryTime)
|
||||||
|
and (used.LastUse is null or used.LastUse >= place.EntryTime)
|
||||||
|
order by
|
||||||
|
used.FirstUse desc,
|
||||||
|
used.ID desc
|
||||||
|
) cvu
|
||||||
|
left join dbo.Vehicle v on v.ID = cvu.ID_Vehicle
|
||||||
|
left join dbo.VehicleIdentification vi on vi.ID = v.ID_VehicleIdentification
|
||||||
|
left join dbo.Nation vehicleNation on vehicleNation.ID = v.ID_Nation
|
||||||
|
where (:occurredFrom is null or place.EntryTime >= :occurredFrom)
|
||||||
|
and (:occurredTo is null or place.EntryTime < :occurredTo)
|
||||||
|
and (
|
||||||
|
:organisationId is null
|
||||||
|
or exists (
|
||||||
|
select 1
|
||||||
|
from dbo.Driver_I_90021 rel
|
||||||
|
join OrgTree on OrgTree.I_90021_OID = rel.ID_I_90021
|
||||||
|
where rel.ID_Driver = c.ID_Driver
|
||||||
|
and rel.GILT_BIS is null
|
||||||
|
)
|
||||||
|
)
|
||||||
|
)
|
||||||
|
select
|
||||||
|
cast(base.ID as varchar(128)) as source_row_id,
|
||||||
|
concat('TACHOGRAPH:CARD_PLACE:', base.ID) as external_source_event_id,
|
||||||
|
|
||||||
|
base.occurred_at,
|
||||||
|
base.received_partner_at,
|
||||||
|
cast(base.Odo as bigint) * 1000 as odometer_m,
|
||||||
|
base.latitude,
|
||||||
|
base.longitude,
|
||||||
|
base.country,
|
||||||
|
base.region,
|
||||||
|
'WORKING_DAY_PLACE_RECORDED' as event_type,
|
||||||
|
case when base.entry_type in (0, 2, 4) then 'START' else 'END' end as lifecycle,
|
||||||
|
cast(case when base.entry_type in (2, 3, 4, 5) then 1 else 0 end as bit) as manual_entry,
|
||||||
|
|
||||||
|
cast(base.driver_id as varchar(128)) as driver_source_entity_id,
|
||||||
|
base.driver_card_nation,
|
||||||
|
base.driver_card_number,
|
||||||
|
|
||||||
|
cast(base.vehicle_identification_id as varchar(128)) as vehicle_source_entity_id,
|
||||||
|
base.vehicle_vin,
|
||||||
|
cast(base.vehicle_registration_id as varchar(128)) as vehicle_registration_source_entity_id,
|
||||||
|
base.vehicle_registration_nation,
|
||||||
|
base.vehicle_registration_number,
|
||||||
|
|
||||||
|
'DRIVER_CARD' as source_package_kind,
|
||||||
|
cast(base.source_package_id_raw as varchar(128)) as source_package_id,
|
||||||
|
cast(base.source_package_entity_id_raw as varchar(128)) as source_package_entity_id,
|
||||||
|
base.source_package_period_from,
|
||||||
|
base.source_package_period_to,
|
||||||
|
base.source_package_imported_at
|
||||||
|
from Base base
|
||||||
|
|
@ -0,0 +1,90 @@
|
||||||
|
/*
|
||||||
|
* CardGnssAccumulatedDriving POSITION extraction for the bytebar tachograph schema.
|
||||||
|
*/
|
||||||
|
with OrgTree as (
|
||||||
|
select org.I_90021_OID
|
||||||
|
from dbo.GetOrganisationTree(null, :organisationId, 0, null) org
|
||||||
|
where :organisationId is not null
|
||||||
|
)
|
||||||
|
,
|
||||||
|
Base as (
|
||||||
|
select
|
||||||
|
pos.ID,
|
||||||
|
pos.Timestamp as occurred_at,
|
||||||
|
pos.Odo,
|
||||||
|
pos.ID_FileLog,
|
||||||
|
c.ID as card_id,
|
||||||
|
c.ID_Driver as driver_id,
|
||||||
|
cn.AlphaCode as driver_card_nation,
|
||||||
|
c.CardNumber as driver_card_number,
|
||||||
|
gnss.Latitude as latitude,
|
||||||
|
gnss.Longitude as longitude,
|
||||||
|
v.ID as vehicle_registration_id,
|
||||||
|
v.ID_VehicleIdentification as vehicle_identification_id,
|
||||||
|
vi.VIN as vehicle_vin,
|
||||||
|
vehicleNation.AlphaCode as vehicle_registration_nation,
|
||||||
|
v.VRN as vehicle_registration_number,
|
||||||
|
coalesce(fl.DownloadDate, fl.OriginalDownloadDate, fl.TStamp, fl.CreationDate) as received_partner_at,
|
||||||
|
coalesce(fl.ID, pos.ID_FileLog) as source_package_id_raw,
|
||||||
|
coalesce(fl.ID_Card, pos.ID_Card) as source_package_entity_id_raw,
|
||||||
|
coalesce(fl.DownloadFrom, pos.Timestamp) as source_package_period_from,
|
||||||
|
coalesce(fl.DownloadTo, pos.Timestamp) as source_package_period_to,
|
||||||
|
coalesce(fl.CreationDate, fl.TStamp) as source_package_imported_at
|
||||||
|
from dbo.CardGnssAccumulatedDriving pos
|
||||||
|
join dbo.Card c on c.ID = pos.ID_Card
|
||||||
|
left join dbo.Nation cn on cn.ID = c.ID_Nation
|
||||||
|
join dbo.GnssPlace gnss on gnss.ID = pos.ID_GnssPlace
|
||||||
|
left join dbo.FileLog fl on fl.ID = pos.ID_FileLog
|
||||||
|
outer apply (
|
||||||
|
select top 1 used.ID_Vehicle
|
||||||
|
from dbo.CardVehiclesUsed used
|
||||||
|
where used.ID_Card = pos.ID_Card
|
||||||
|
and (used.FirstUse is null or used.FirstUse <= pos.Timestamp)
|
||||||
|
and (used.LastUse is null or used.LastUse >= pos.Timestamp)
|
||||||
|
order by
|
||||||
|
used.FirstUse desc,
|
||||||
|
used.ID desc
|
||||||
|
) cvu
|
||||||
|
left join dbo.Vehicle v on v.ID = cvu.ID_Vehicle
|
||||||
|
left join dbo.VehicleIdentification vi on vi.ID = v.ID_VehicleIdentification
|
||||||
|
left join dbo.Nation vehicleNation on vehicleNation.ID = v.ID_Nation
|
||||||
|
where (:occurredFrom is null or pos.Timestamp >= :occurredFrom)
|
||||||
|
and (:occurredTo is null or pos.Timestamp < :occurredTo)
|
||||||
|
and (
|
||||||
|
:organisationId is null
|
||||||
|
or exists (
|
||||||
|
select 1
|
||||||
|
from dbo.Driver_I_90021 rel
|
||||||
|
join OrgTree on OrgTree.I_90021_OID = rel.ID_I_90021
|
||||||
|
where rel.ID_Driver = c.ID_Driver
|
||||||
|
and rel.GILT_BIS is null
|
||||||
|
)
|
||||||
|
)
|
||||||
|
)
|
||||||
|
select
|
||||||
|
cast(base.ID as varchar(128)) as source_row_id,
|
||||||
|
concat('TACHOGRAPH:CARD_POSITION:', base.ID) as external_source_event_id,
|
||||||
|
|
||||||
|
base.occurred_at,
|
||||||
|
base.received_partner_at,
|
||||||
|
cast(base.Odo as bigint) * 1000 as odometer_m,
|
||||||
|
base.latitude,
|
||||||
|
base.longitude,
|
||||||
|
|
||||||
|
cast(base.driver_id as varchar(128)) as driver_source_entity_id,
|
||||||
|
base.driver_card_nation,
|
||||||
|
base.driver_card_number,
|
||||||
|
|
||||||
|
cast(base.vehicle_identification_id as varchar(128)) as vehicle_source_entity_id,
|
||||||
|
base.vehicle_vin,
|
||||||
|
cast(base.vehicle_registration_id as varchar(128)) as vehicle_registration_source_entity_id,
|
||||||
|
base.vehicle_registration_nation,
|
||||||
|
base.vehicle_registration_number,
|
||||||
|
|
||||||
|
'DRIVER_CARD' as source_package_kind,
|
||||||
|
cast(base.source_package_id_raw as varchar(128)) as source_package_id,
|
||||||
|
cast(base.source_package_entity_id_raw as varchar(128)) as source_package_entity_id,
|
||||||
|
base.source_package_period_from,
|
||||||
|
base.source_package_period_to,
|
||||||
|
base.source_package_imported_at
|
||||||
|
from Base base
|
||||||
|
|
@ -0,0 +1,88 @@
|
||||||
|
/*
|
||||||
|
* CardSpecificCondition SPECIFIC_CONDITION extraction for the bytebar tachograph schema.
|
||||||
|
*/
|
||||||
|
with OrgTree as (
|
||||||
|
select org.I_90021_OID
|
||||||
|
from dbo.GetOrganisationTree(null, :organisationId, 0, null) org
|
||||||
|
where :organisationId is not null
|
||||||
|
)
|
||||||
|
,
|
||||||
|
Base as (
|
||||||
|
select
|
||||||
|
cond.ID,
|
||||||
|
cond.EntryTime as occurred_at,
|
||||||
|
sc.Condition as condition_code,
|
||||||
|
cond.ID_FileLog,
|
||||||
|
c.ID as card_id,
|
||||||
|
c.ID_Driver as driver_id,
|
||||||
|
cn.AlphaCode as driver_card_nation,
|
||||||
|
c.CardNumber as driver_card_number,
|
||||||
|
v.ID as vehicle_registration_id,
|
||||||
|
v.ID_VehicleIdentification as vehicle_identification_id,
|
||||||
|
vi.VIN as vehicle_vin,
|
||||||
|
vehicleNation.AlphaCode as vehicle_registration_nation,
|
||||||
|
v.VRN as vehicle_registration_number,
|
||||||
|
coalesce(fl.DownloadDate, fl.OriginalDownloadDate, fl.TStamp, fl.CreationDate) as received_partner_at,
|
||||||
|
coalesce(fl.ID, cond.ID_FileLog) as source_package_id_raw,
|
||||||
|
coalesce(fl.ID_Card, cond.ID_Card) as source_package_entity_id_raw,
|
||||||
|
coalesce(fl.DownloadFrom, cond.EntryTime) as source_package_period_from,
|
||||||
|
coalesce(fl.DownloadTo, cond.EntryTime) as source_package_period_to,
|
||||||
|
coalesce(fl.CreationDate, fl.TStamp) as source_package_imported_at
|
||||||
|
from dbo.CardSpecificConditions cond
|
||||||
|
join dbo.SpecificCondition sc on sc.ID = cond.ID_SpecificCondition
|
||||||
|
join dbo.Card c on c.ID = cond.ID_Card
|
||||||
|
left join dbo.Nation cn on cn.ID = c.ID_Nation
|
||||||
|
left join dbo.FileLog fl on fl.ID = cond.ID_FileLog
|
||||||
|
outer apply (
|
||||||
|
select top 1 used.ID_Vehicle
|
||||||
|
from dbo.CardVehiclesUsed used
|
||||||
|
where used.ID_Card = cond.ID_Card
|
||||||
|
and (used.FirstUse is null or used.FirstUse <= cond.EntryTime)
|
||||||
|
and (used.LastUse is null or used.LastUse >= cond.EntryTime)
|
||||||
|
order by
|
||||||
|
used.FirstUse desc,
|
||||||
|
used.ID desc
|
||||||
|
) cvu
|
||||||
|
left join dbo.Vehicle v on v.ID = cvu.ID_Vehicle
|
||||||
|
left join dbo.VehicleIdentification vi on vi.ID = v.ID_VehicleIdentification
|
||||||
|
left join dbo.Nation vehicleNation on vehicleNation.ID = v.ID_Nation
|
||||||
|
where (:occurredFrom is null or cond.EntryTime >= :occurredFrom)
|
||||||
|
and (:occurredTo is null or cond.EntryTime < :occurredTo)
|
||||||
|
and (
|
||||||
|
:organisationId is null
|
||||||
|
or exists (
|
||||||
|
select 1
|
||||||
|
from dbo.Driver_I_90021 rel
|
||||||
|
join OrgTree on OrgTree.I_90021_OID = rel.ID_I_90021
|
||||||
|
where rel.ID_Driver = c.ID_Driver
|
||||||
|
and rel.GILT_BIS is null
|
||||||
|
)
|
||||||
|
)
|
||||||
|
)
|
||||||
|
select
|
||||||
|
cast(base.ID as varchar(128)) as source_row_id,
|
||||||
|
concat('TACHOGRAPH:CARD_SPECIFIC_CONDITION:', base.ID) as external_source_event_id,
|
||||||
|
|
||||||
|
base.occurred_at,
|
||||||
|
base.received_partner_at,
|
||||||
|
'SPECIFIC_CONDITION' as event_domain,
|
||||||
|
case when base.condition_code in (3, 4) then 'FERRY_TRAIN' else 'OUT' end as event_type,
|
||||||
|
case when base.condition_code in (1, 3) then 'BEGIN' else 'END' end as lifecycle,
|
||||||
|
|
||||||
|
cast(base.driver_id as varchar(128)) as driver_source_entity_id,
|
||||||
|
base.driver_card_nation,
|
||||||
|
base.driver_card_number,
|
||||||
|
|
||||||
|
cast(base.vehicle_identification_id as varchar(128)) as vehicle_source_entity_id,
|
||||||
|
base.vehicle_vin,
|
||||||
|
cast(base.vehicle_registration_id as varchar(128)) as vehicle_registration_source_entity_id,
|
||||||
|
base.vehicle_registration_nation,
|
||||||
|
base.vehicle_registration_number,
|
||||||
|
|
||||||
|
'DRIVER_CARD' as source_package_kind,
|
||||||
|
cast(base.source_package_id_raw as varchar(128)) as source_package_id,
|
||||||
|
cast(base.source_package_entity_id_raw as varchar(128)) as source_package_entity_id,
|
||||||
|
base.source_package_period_from,
|
||||||
|
base.source_package_period_to,
|
||||||
|
base.source_package_imported_at
|
||||||
|
from Base base
|
||||||
|
|
@ -0,0 +1,114 @@
|
||||||
|
/*
|
||||||
|
* CardVehiclesUsed DRIVER_CARD extraction for the bytebar tachograph schema.
|
||||||
|
*
|
||||||
|
* Each row is emitted as INSERT at FirstUse and WITHDRAW at LastUse when those
|
||||||
|
* timestamps are available.
|
||||||
|
*/
|
||||||
|
with OrgTree as (
|
||||||
|
select org.I_90021_OID
|
||||||
|
from dbo.GetOrganisationTree(null, :organisationId, 0, null) org
|
||||||
|
where :organisationId is not null
|
||||||
|
)
|
||||||
|
,
|
||||||
|
CandidateUse as (
|
||||||
|
select
|
||||||
|
used.ID,
|
||||||
|
used.ID_Card,
|
||||||
|
used.ID_Vehicle,
|
||||||
|
used.OdoBegin,
|
||||||
|
used.OdoEnd,
|
||||||
|
used.FirstUse,
|
||||||
|
used.LastUse,
|
||||||
|
used.ID_FileLog,
|
||||||
|
used.VIN as used_vin,
|
||||||
|
c.ID_Driver as driver_id,
|
||||||
|
cn.AlphaCode as driver_card_nation,
|
||||||
|
c.CardNumber as driver_card_number,
|
||||||
|
v.ID_VehicleIdentification,
|
||||||
|
vi.ID as vehicle_identification_id,
|
||||||
|
coalesce(used.VIN, vi.VIN) as vehicle_vin,
|
||||||
|
vn.AlphaCode as vehicle_registration_nation,
|
||||||
|
v.VRN as vehicle_registration_number
|
||||||
|
from dbo.CardVehiclesUsed used
|
||||||
|
join dbo.Card c on c.ID = used.ID_Card
|
||||||
|
left join dbo.Nation cn on cn.ID = c.ID_Nation
|
||||||
|
left join dbo.Vehicle v on v.ID = used.ID_Vehicle
|
||||||
|
left join dbo.VehicleIdentification vi on vi.ID = v.ID_VehicleIdentification
|
||||||
|
left join dbo.Nation vn on vn.ID = v.ID_Nation
|
||||||
|
where (:occurredTo is null or used.FirstUse < :occurredTo or used.LastUse < :occurredTo)
|
||||||
|
and (:occurredFrom is null or used.FirstUse >= :occurredFrom or used.LastUse >= :occurredFrom)
|
||||||
|
and (
|
||||||
|
:organisationId is null
|
||||||
|
or exists (
|
||||||
|
select 1
|
||||||
|
from dbo.Driver_I_90021 rel
|
||||||
|
join OrgTree on OrgTree.I_90021_OID = rel.ID_I_90021
|
||||||
|
where rel.ID_Driver = c.ID_Driver
|
||||||
|
and rel.GILT_BIS is null
|
||||||
|
)
|
||||||
|
)
|
||||||
|
)
|
||||||
|
,
|
||||||
|
Base as (
|
||||||
|
select
|
||||||
|
used.ID,
|
||||||
|
used.ID_Card,
|
||||||
|
used.ID_Vehicle,
|
||||||
|
used.driver_id,
|
||||||
|
used.driver_card_nation,
|
||||||
|
used.driver_card_number,
|
||||||
|
used.vehicle_identification_id,
|
||||||
|
used.vehicle_vin,
|
||||||
|
used.vehicle_registration_nation,
|
||||||
|
used.vehicle_registration_number,
|
||||||
|
evt.lifecycle,
|
||||||
|
evt.occurred_at,
|
||||||
|
evt.odometer_m,
|
||||||
|
coalesce(fl.DownloadDate, fl.OriginalDownloadDate, fl.TStamp, fl.CreationDate) as received_partner_at,
|
||||||
|
coalesce(fl.ID, used.ID_FileLog) as source_package_id_raw,
|
||||||
|
coalesce(fl.ID_Card, used.ID_Card) as source_package_entity_id_raw,
|
||||||
|
coalesce(fl.DownloadFrom, used.FirstUse) as source_package_period_from,
|
||||||
|
coalesce(fl.DownloadTo, used.LastUse, used.FirstUse) as source_package_period_to,
|
||||||
|
coalesce(fl.CreationDate, fl.TStamp) as source_package_imported_at
|
||||||
|
from CandidateUse used
|
||||||
|
cross apply (values
|
||||||
|
('INSERT', used.FirstUse, cast(used.OdoBegin as bigint) * 1000),
|
||||||
|
('WITHDRAW', used.LastUse, cast(used.OdoEnd as bigint) * 1000)
|
||||||
|
) evt(lifecycle, occurred_at, odometer_m)
|
||||||
|
left join dbo.FileLog fl on fl.ID = used.ID_FileLog
|
||||||
|
where evt.occurred_at is not null
|
||||||
|
and (:occurredFrom is null or evt.occurred_at >= :occurredFrom)
|
||||||
|
and (:occurredTo is null or evt.occurred_at < :occurredTo)
|
||||||
|
)
|
||||||
|
select
|
||||||
|
cast(base.ID as varchar(128)) as source_row_id,
|
||||||
|
concat('TACHOGRAPH:CARD_VEHICLES_USED:', base.ID, ':', base.lifecycle) as external_source_event_id,
|
||||||
|
|
||||||
|
base.occurred_at,
|
||||||
|
base.received_partner_at,
|
||||||
|
case base.lifecycle
|
||||||
|
when 'INSERT' then 'CARD_INSERTED'
|
||||||
|
when 'WITHDRAW' then 'CARD_WITHDRAWN'
|
||||||
|
else 'UNKNOWN_EVENT'
|
||||||
|
end as event_type,
|
||||||
|
base.lifecycle,
|
||||||
|
cast(null as varchar(32)) as card_slot,
|
||||||
|
base.odometer_m,
|
||||||
|
|
||||||
|
cast(base.driver_id as varchar(128)) as driver_source_entity_id,
|
||||||
|
base.driver_card_nation,
|
||||||
|
base.driver_card_number,
|
||||||
|
|
||||||
|
cast(base.vehicle_identification_id as varchar(128)) as vehicle_source_entity_id,
|
||||||
|
base.vehicle_vin,
|
||||||
|
cast(base.ID_Vehicle as varchar(128)) as vehicle_registration_source_entity_id,
|
||||||
|
base.vehicle_registration_nation,
|
||||||
|
base.vehicle_registration_number,
|
||||||
|
|
||||||
|
'DRIVER_CARD' as source_package_kind,
|
||||||
|
cast(base.source_package_id_raw as varchar(128)) as source_package_id,
|
||||||
|
cast(base.source_package_entity_id_raw as varchar(128)) as source_package_entity_id,
|
||||||
|
base.source_package_period_from,
|
||||||
|
base.source_package_period_to,
|
||||||
|
base.source_package_imported_at
|
||||||
|
from Base base
|
||||||
|
|
@ -0,0 +1,128 @@
|
||||||
|
/*
|
||||||
|
* IWCycle DRIVER_CARD extraction for the bytebar tachograph schema.
|
||||||
|
*
|
||||||
|
* Each IWCycle row describes a card inserted into a vehicle unit. It is emitted
|
||||||
|
* as two point events when timestamps are available: INSERT at BeginTime and
|
||||||
|
* WITHDRAW at EndTime.
|
||||||
|
*/
|
||||||
|
with OrgTree as (
|
||||||
|
select org.I_90021_OID
|
||||||
|
from dbo.GetOrganisationTree(null, :organisationId, 0, null) org
|
||||||
|
where :organisationId is not null
|
||||||
|
)
|
||||||
|
,
|
||||||
|
CandidateCycle as (
|
||||||
|
select
|
||||||
|
iw.ID,
|
||||||
|
iw.BeginTime,
|
||||||
|
iw.EndTime,
|
||||||
|
iw.OdoBegin,
|
||||||
|
iw.OdoEnd,
|
||||||
|
iw.Slot,
|
||||||
|
iw.ID_FileLog,
|
||||||
|
iw.ID_Card,
|
||||||
|
iw.ID_VUInstallation,
|
||||||
|
vui.ID_VehicleIdentification,
|
||||||
|
vui.ID_FileLog as vui_filelog_id,
|
||||||
|
vi.ID as vehicle_identification_id,
|
||||||
|
vi.VIN as vehicle_vin,
|
||||||
|
c.ID_Driver as driver_id,
|
||||||
|
cn.AlphaCode as driver_card_nation,
|
||||||
|
c.CardNumber as driver_card_number,
|
||||||
|
coalesce(iw.ID_FileLog, vui.ID_FileLog) as file_log_id
|
||||||
|
from dbo.IWCycle iw
|
||||||
|
join dbo.VUInstallation vui on vui.ID = iw.ID_VUInstallation
|
||||||
|
join dbo.VehicleIdentification vi on vi.ID = vui.ID_VehicleIdentification
|
||||||
|
left join dbo.Card c on c.ID = iw.ID_Card
|
||||||
|
left join dbo.Nation cn on cn.ID = c.ID_Nation
|
||||||
|
where (:occurredTo is null or iw.BeginTime < :occurredTo or iw.EndTime < :occurredTo)
|
||||||
|
and (:occurredFrom is null or iw.BeginTime >= :occurredFrom or iw.EndTime >= :occurredFrom)
|
||||||
|
)
|
||||||
|
,
|
||||||
|
Base as (
|
||||||
|
select
|
||||||
|
cycle.ID,
|
||||||
|
cycle.Slot,
|
||||||
|
cycle.ID_FileLog,
|
||||||
|
cycle.vui_filelog_id,
|
||||||
|
cycle.ID_Card,
|
||||||
|
cycle.ID_VUInstallation,
|
||||||
|
cycle.vehicle_identification_id,
|
||||||
|
cycle.vehicle_vin,
|
||||||
|
cycle.driver_id,
|
||||||
|
cycle.driver_card_nation,
|
||||||
|
cycle.driver_card_number,
|
||||||
|
evt.lifecycle,
|
||||||
|
evt.occurred_at,
|
||||||
|
evt.odometer_m,
|
||||||
|
coalesce(fl.DownloadDate, fl.OriginalDownloadDate, fl.TStamp, fl.CreationDate) as received_partner_at,
|
||||||
|
coalesce(fl.ID, cycle.ID_FileLog, cycle.vui_filelog_id) as source_package_id_raw,
|
||||||
|
coalesce(fl.ID_VehicleIdentification, cycle.ID_VehicleIdentification) as source_package_entity_id_raw,
|
||||||
|
coalesce(fl.DownloadFrom, cycle.BeginTime) as source_package_period_from,
|
||||||
|
coalesce(fl.DownloadTo, cycle.EndTime, cycle.BeginTime) as source_package_period_to,
|
||||||
|
coalesce(fl.CreationDate, fl.TStamp) as source_package_imported_at
|
||||||
|
from CandidateCycle cycle
|
||||||
|
cross apply (values
|
||||||
|
('INSERT', cycle.BeginTime, cast(cycle.OdoBegin as bigint) * 1000),
|
||||||
|
('WITHDRAW', cycle.EndTime, cast(cycle.OdoEnd as bigint) * 1000)
|
||||||
|
) evt(lifecycle, occurred_at, odometer_m)
|
||||||
|
left join dbo.FileLog fl on fl.ID = cycle.file_log_id
|
||||||
|
where evt.occurred_at is not null
|
||||||
|
and (:occurredFrom is null or evt.occurred_at >= :occurredFrom)
|
||||||
|
and (:occurredTo is null or evt.occurred_at < :occurredTo)
|
||||||
|
)
|
||||||
|
select
|
||||||
|
cast(base.ID as varchar(128)) as source_row_id,
|
||||||
|
concat('TACHOGRAPH:IW_CYCLE:', base.ID, ':', base.lifecycle) as external_source_event_id,
|
||||||
|
|
||||||
|
base.occurred_at,
|
||||||
|
base.received_partner_at,
|
||||||
|
case base.lifecycle
|
||||||
|
when 'INSERT' then 'CARD_INSERTED'
|
||||||
|
when 'WITHDRAW' then 'CARD_WITHDRAWN'
|
||||||
|
else 'UNKNOWN_EVENT'
|
||||||
|
end as event_type,
|
||||||
|
base.lifecycle,
|
||||||
|
base.Slot as card_slot,
|
||||||
|
base.odometer_m,
|
||||||
|
|
||||||
|
cast(base.driver_id as varchar(128)) as driver_source_entity_id,
|
||||||
|
base.driver_card_nation,
|
||||||
|
base.driver_card_number,
|
||||||
|
|
||||||
|
cast(base.vehicle_identification_id as varchar(128)) as vehicle_source_entity_id,
|
||||||
|
base.vehicle_vin,
|
||||||
|
cast(v.ID as varchar(128)) as vehicle_registration_source_entity_id,
|
||||||
|
vn.AlphaCode as vehicle_registration_nation,
|
||||||
|
v.VRN as vehicle_registration_number,
|
||||||
|
|
||||||
|
'VEHICLE_UNIT' as source_package_kind,
|
||||||
|
cast(base.source_package_id_raw as varchar(128)) as source_package_id,
|
||||||
|
cast(base.source_package_entity_id_raw as varchar(128)) as source_package_entity_id,
|
||||||
|
base.source_package_period_from,
|
||||||
|
base.source_package_period_to,
|
||||||
|
base.source_package_imported_at
|
||||||
|
from Base base
|
||||||
|
outer apply (
|
||||||
|
select top 1 vehicle.ID,
|
||||||
|
vehicle.VRN,
|
||||||
|
vehicle.ID_Nation
|
||||||
|
from dbo.Vehicle vehicle
|
||||||
|
where vehicle.ID_VehicleIdentification = base.vehicle_identification_id
|
||||||
|
and (vehicle.ValidFrom is null or vehicle.ValidFrom <= base.occurred_at)
|
||||||
|
and (vehicle.ValidTo is null or vehicle.ValidTo > base.occurred_at)
|
||||||
|
order by
|
||||||
|
vehicle.ValidFrom desc,
|
||||||
|
vehicle.ID desc
|
||||||
|
) v
|
||||||
|
left join dbo.Nation vn on vn.ID = v.ID_Nation
|
||||||
|
where (
|
||||||
|
:organisationId is null
|
||||||
|
or exists (
|
||||||
|
select 1
|
||||||
|
from dbo.Vehicle_I_90021 rel
|
||||||
|
join OrgTree on OrgTree.I_90021_OID = rel.ID_I_90021
|
||||||
|
where rel.ID_Vehicle = v.ID
|
||||||
|
and rel.GILT_BIS is null
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
@ -0,0 +1,27 @@
|
||||||
|
select
|
||||||
|
'DRIVER_CARD' as entity_type,
|
||||||
|
cast(c.ID as varchar(128)) as source_entity_id,
|
||||||
|
concat(coalesce(n.AlphaCode, ''), ':', c.CardNumber) as source_external_key,
|
||||||
|
concat(coalesce(n.AlphaCode, ''), ':', c.CardNumber) as display_name,
|
||||||
|
cast(case when c.IsLastCard = 1 and (c.ExpiryDate is null or c.ExpiryDate > getutcdate()) then 1 else 0 end as bit) as active,
|
||||||
|
c.IssueDate as valid_from,
|
||||||
|
c.ExpiryDate as valid_to,
|
||||||
|
cast(null as datetime) as source_updated_at,
|
||||||
|
c.ID as card_id,
|
||||||
|
cast(c.ID_Driver as varchar(128)) as driver_id,
|
||||||
|
n.AlphaCode as card_nation,
|
||||||
|
c.CardNumber as card_number,
|
||||||
|
c.Consecutive as consecutive,
|
||||||
|
c.Replacement as replacement,
|
||||||
|
c.Renewal as renewal,
|
||||||
|
c.IssueDate as issue_date,
|
||||||
|
c.ExpiryDate as expiry_date,
|
||||||
|
c.ValidityDate as validity_date,
|
||||||
|
c.Authorityname as authority_name,
|
||||||
|
c.ID_FileLog as file_log_id,
|
||||||
|
c.IsAnalog as is_analog,
|
||||||
|
c.IsLastCard as is_last_card,
|
||||||
|
c.Generation as generation
|
||||||
|
from dbo.Card c
|
||||||
|
left join dbo.Nation n on n.ID = c.ID_Nation
|
||||||
|
where c.CardNumber is not null
|
||||||
|
|
@ -0,0 +1,35 @@
|
||||||
|
select
|
||||||
|
'DRIVER' as entity_type,
|
||||||
|
cast(d.ID as varchar(128)) as source_entity_id,
|
||||||
|
coalesce(nullif(d.IdentificationNumber, ''), nullif(d.LicenseNumber, ''), cast(d.ID as varchar(128))) as source_external_key,
|
||||||
|
ltrim(rtrim(concat(d.Surname, ' ', d.Firstnames))) as display_name,
|
||||||
|
cast(case when d.IsActive = 1 then 1 else 0 end as bit) as active,
|
||||||
|
d.EmploymentStartDate as valid_from,
|
||||||
|
d.RetirementDate as valid_to,
|
||||||
|
d.LastUpdate as source_updated_at,
|
||||||
|
d.ID as driver_id,
|
||||||
|
d.Surname as surname,
|
||||||
|
d.Firstnames as first_names,
|
||||||
|
d.Birthdate as birth_date,
|
||||||
|
d.BirthPlace as birth_place,
|
||||||
|
d.LicenseNumber as license_number,
|
||||||
|
ln.AlphaCode as license_nation,
|
||||||
|
d.LicenseAuthority as license_authority,
|
||||||
|
d.LExpiryDate as license_expiry_date,
|
||||||
|
d.Phone as phone,
|
||||||
|
d.Mobile as mobile,
|
||||||
|
d.Email as email,
|
||||||
|
d.CostCenter as cost_center,
|
||||||
|
d.OrganisationUserID as organisation_user_id,
|
||||||
|
d.IdentificationNumber as identification_number,
|
||||||
|
d.Region as region,
|
||||||
|
d.ID_FileLog as file_log_id,
|
||||||
|
d.LastFileLog_ID as last_file_log_id,
|
||||||
|
d.FirstDrivingTime as first_driving_time,
|
||||||
|
d.LastDrivingTime as last_driving_time,
|
||||||
|
d.LastUsedVehicle_ID as last_used_vehicle_id,
|
||||||
|
d.LastUsedVehicleTime as last_used_vehicle_time,
|
||||||
|
d.IsDigital as is_digital,
|
||||||
|
d.IsAnalog as is_analog
|
||||||
|
from dbo.Driver d
|
||||||
|
left join dbo.Nation ln on ln.ID = d.ID_LicenseNation
|
||||||
|
|
@ -0,0 +1,22 @@
|
||||||
|
select
|
||||||
|
'ORGANISATION' as entity_type,
|
||||||
|
cast(o.oid as varchar(128)) as source_entity_id,
|
||||||
|
cast(o.GUID as varchar(64)) as source_external_key,
|
||||||
|
coalesce(nullif(o.name, ''), nullif(o.kurzbez, ''), cast(o.oid as varchar(128))) as display_name,
|
||||||
|
cast(case when o.gueltigbis is null or o.gueltigbis > getutcdate() then 1 else 0 end as bit) as active,
|
||||||
|
o.VertragGiltAb as valid_from,
|
||||||
|
o.gueltigbis as valid_to,
|
||||||
|
cast(null as datetime) as source_updated_at,
|
||||||
|
o.oid as organisation_id,
|
||||||
|
cast(o.GUID as varchar(64)) as organisation_guid,
|
||||||
|
o.kurzbez as organisation_code,
|
||||||
|
o.name as organisation_name,
|
||||||
|
cast(o.n_rekey01 as varchar(128)) as parent_organisation_id,
|
||||||
|
o.Unternehmens_ID as company_id,
|
||||||
|
o.kostenstelle as cost_center,
|
||||||
|
o.Country as country,
|
||||||
|
o.PostalCode as postal_code,
|
||||||
|
o.City as city,
|
||||||
|
o.strasse as street,
|
||||||
|
o.Email as email
|
||||||
|
from dbo.I_90021 o
|
||||||
|
|
@ -0,0 +1,75 @@
|
||||||
|
select
|
||||||
|
'ORGANISATION_PARENT' as relation_type,
|
||||||
|
'ORGANISATION' as from_entity_type,
|
||||||
|
cast(o.oid as varchar(128)) as from_source_entity_id,
|
||||||
|
'ORGANISATION' as to_entity_type,
|
||||||
|
cast(o.n_rekey01 as varchar(128)) as to_source_entity_id,
|
||||||
|
o.VertragGiltAb as valid_from,
|
||||||
|
o.gueltigbis as valid_to,
|
||||||
|
cast(null as datetime) as source_updated_at,
|
||||||
|
'I_90021' as source_table,
|
||||||
|
cast(o.oid as varchar(128)) as source_row_id
|
||||||
|
from dbo.I_90021 o
|
||||||
|
where o.n_rekey01 is not null
|
||||||
|
|
||||||
|
union all
|
||||||
|
|
||||||
|
select
|
||||||
|
'DRIVER_ORGANISATION' as relation_type,
|
||||||
|
'DRIVER' as from_entity_type,
|
||||||
|
cast(rel.ID_Driver as varchar(128)) as from_source_entity_id,
|
||||||
|
'ORGANISATION' as to_entity_type,
|
||||||
|
cast(rel.ID_I_90021 as varchar(128)) as to_source_entity_id,
|
||||||
|
rel.GILT_AB as valid_from,
|
||||||
|
rel.GILT_BIS as valid_to,
|
||||||
|
rel.LastUpdate as source_updated_at,
|
||||||
|
'Driver_I_90021' as source_table,
|
||||||
|
cast(rel.ID as varchar(128)) as source_row_id
|
||||||
|
from dbo.Driver_I_90021 rel
|
||||||
|
|
||||||
|
union all
|
||||||
|
|
||||||
|
select
|
||||||
|
'DRIVER_CARD_DRIVER' as relation_type,
|
||||||
|
'DRIVER_CARD' as from_entity_type,
|
||||||
|
cast(c.ID as varchar(128)) as from_source_entity_id,
|
||||||
|
'DRIVER' as to_entity_type,
|
||||||
|
cast(c.ID_Driver as varchar(128)) as to_source_entity_id,
|
||||||
|
c.IssueDate as valid_from,
|
||||||
|
c.ExpiryDate as valid_to,
|
||||||
|
cast(null as datetime) as source_updated_at,
|
||||||
|
'Card' as source_table,
|
||||||
|
cast(c.ID as varchar(128)) as source_row_id
|
||||||
|
from dbo.Card c
|
||||||
|
where c.ID_Driver is not null
|
||||||
|
|
||||||
|
union all
|
||||||
|
|
||||||
|
select
|
||||||
|
'VEHICLE_ORGANISATION' as relation_type,
|
||||||
|
'VEHICLE_REGISTRATION' as from_entity_type,
|
||||||
|
cast(rel.ID_Vehicle as varchar(128)) as from_source_entity_id,
|
||||||
|
'ORGANISATION' as to_entity_type,
|
||||||
|
cast(rel.ID_I_90021 as varchar(128)) as to_source_entity_id,
|
||||||
|
rel.GILT_AB as valid_from,
|
||||||
|
rel.GILT_BIS as valid_to,
|
||||||
|
rel.LastUpdate as source_updated_at,
|
||||||
|
'Vehicle_I_90021' as source_table,
|
||||||
|
cast(rel.ID as varchar(128)) as source_row_id
|
||||||
|
from dbo.Vehicle_I_90021 rel
|
||||||
|
|
||||||
|
union all
|
||||||
|
|
||||||
|
select
|
||||||
|
'VEHICLE_REGISTRATION_VEHICLE' as relation_type,
|
||||||
|
'VEHICLE_REGISTRATION' as from_entity_type,
|
||||||
|
cast(v.ID as varchar(128)) as from_source_entity_id,
|
||||||
|
'VEHICLE' as to_entity_type,
|
||||||
|
cast(v.ID_VehicleIdentification as varchar(128)) as to_source_entity_id,
|
||||||
|
v.ValidFrom as valid_from,
|
||||||
|
v.ValidTo as valid_to,
|
||||||
|
cast(null as datetime) as source_updated_at,
|
||||||
|
'Vehicle' as source_table,
|
||||||
|
cast(v.ID as varchar(128)) as source_row_id
|
||||||
|
from dbo.Vehicle v
|
||||||
|
where v.ID_VehicleIdentification is not null
|
||||||
|
|
@ -0,0 +1,20 @@
|
||||||
|
select
|
||||||
|
'VEHICLE_REGISTRATION' as entity_type,
|
||||||
|
cast(v.ID as varchar(128)) as source_entity_id,
|
||||||
|
concat(coalesce(n.AlphaCode, ''), ':', v.VRN) as source_external_key,
|
||||||
|
concat(coalesce(n.AlphaCode, ''), ':', v.VRN) as display_name,
|
||||||
|
cast(case when v.ValidTo is null or v.ValidTo > getutcdate() then 1 else 0 end as bit) as active,
|
||||||
|
cast(null as datetime) as valid_from,
|
||||||
|
cast(null as datetime) as valid_to,
|
||||||
|
cast(null as datetime) as source_updated_at,
|
||||||
|
v.ID as vehicle_registration_id,
|
||||||
|
cast(v.ID_VehicleIdentification as varchar(128)) as vehicle_identification_id,
|
||||||
|
n.AlphaCode as registration_nation,
|
||||||
|
v.VRN as registration_number,
|
||||||
|
v.VrnNormalized as registration_number_normalized,
|
||||||
|
v.ID_FileLog as file_log_id,
|
||||||
|
v.LastDriver_ID as last_driver_id,
|
||||||
|
v.LastDriverTime as last_driver_time
|
||||||
|
from dbo.Vehicle v
|
||||||
|
left join dbo.Nation n on n.ID = v.ID_Nation
|
||||||
|
where v.VRN is not null
|
||||||
|
|
@ -0,0 +1,28 @@
|
||||||
|
select
|
||||||
|
'VEHICLE' as entity_type,
|
||||||
|
cast(vi.ID as varchar(128)) as source_entity_id,
|
||||||
|
vi.VIN as source_external_key,
|
||||||
|
vi.VIN as display_name,
|
||||||
|
cast(case when vi.IsActive = 1 then 1 else 0 end as bit) as active,
|
||||||
|
cast(null as datetime) as valid_from,
|
||||||
|
cast(null as datetime) as valid_to,
|
||||||
|
cast(null as datetime) as source_updated_at,
|
||||||
|
vi.ID as vehicle_identification_id,
|
||||||
|
vi.VIN as vin,
|
||||||
|
vi.CostCenter as cost_center,
|
||||||
|
vi.ID_FileLog as file_log_id,
|
||||||
|
vi.BillingDate as billing_date,
|
||||||
|
vi.Mobile as mobile,
|
||||||
|
vi.Email as email,
|
||||||
|
vi.GrossVehicleMass as gross_vehicle_mass,
|
||||||
|
vi.VehicleType_ID as vehicle_type_id,
|
||||||
|
vi.FirstDrivingTime as first_driving_time,
|
||||||
|
vi.LastDrivingTime as last_driving_time,
|
||||||
|
vi.LastOdoEnd as last_odo_end,
|
||||||
|
vi.LastOdoEndTime as last_odo_end_time,
|
||||||
|
vi.LastDriver_ID as last_driver_id,
|
||||||
|
vi.LastDriverTime as last_driver_time,
|
||||||
|
vi.LastFileLog_ID as last_file_log_id,
|
||||||
|
vi.IsDigital as is_digital,
|
||||||
|
vi.IsAnalog as is_analog
|
||||||
|
from dbo.VehicleIdentification vi
|
||||||
|
|
@ -0,0 +1,118 @@
|
||||||
|
/*
|
||||||
|
* SpeedingEvents SPEEDING extraction for the bytebar tachograph schema.
|
||||||
|
*
|
||||||
|
* The source row is an interval. The normalized event contract is point-based,
|
||||||
|
* so this query emits a BEGIN point at BeginTime and an END point at EndTime.
|
||||||
|
*/
|
||||||
|
with OrgTree as (
|
||||||
|
select org.I_90021_OID
|
||||||
|
from dbo.GetOrganisationTree(null, :organisationId, 0, null) org
|
||||||
|
where :organisationId is not null
|
||||||
|
)
|
||||||
|
,
|
||||||
|
Base as (
|
||||||
|
select
|
||||||
|
speeding.ID,
|
||||||
|
speeding.BeginTime,
|
||||||
|
speeding.EndTime,
|
||||||
|
speeding.AvgSpeed,
|
||||||
|
speeding.MaxSpeed,
|
||||||
|
speeding.ID_FileLog,
|
||||||
|
speeding.ID_VUInstallation,
|
||||||
|
c.ID as card_id,
|
||||||
|
c.ID_Driver as driver_id,
|
||||||
|
cn.AlphaCode as driver_card_nation,
|
||||||
|
c.CardNumber as driver_card_number,
|
||||||
|
vui.ID_FileLog as vui_filelog_id,
|
||||||
|
vui.ID_VehicleIdentification,
|
||||||
|
vi.ID as vehicle_identification_id,
|
||||||
|
vi.VIN as vehicle_vin,
|
||||||
|
coalesce(fl.DownloadDate, fl.OriginalDownloadDate, fl.TStamp, fl.CreationDate) as received_partner_at,
|
||||||
|
coalesce(fl.ID, speeding.ID_FileLog, vui.ID_FileLog) as source_package_id_raw,
|
||||||
|
coalesce(fl.ID_VehicleIdentification, vui.ID_VehicleIdentification) as source_package_entity_id_raw,
|
||||||
|
coalesce(fl.DownloadFrom, speeding.BeginTime) as source_package_period_from,
|
||||||
|
coalesce(fl.DownloadTo, speeding.EndTime, speeding.BeginTime) as source_package_period_to,
|
||||||
|
coalesce(fl.CreationDate, fl.TStamp) as source_package_imported_at
|
||||||
|
from dbo.SpeedingEvents speeding
|
||||||
|
join dbo.VUInstallation vui on vui.ID = speeding.ID_VUInstallation
|
||||||
|
join dbo.VehicleIdentification vi on vi.ID = vui.ID_VehicleIdentification
|
||||||
|
left join dbo.Card c on c.ID = speeding.ID_Card
|
||||||
|
left join dbo.Nation cn on cn.ID = c.ID_Nation
|
||||||
|
left join dbo.FileLog fl on fl.ID = coalesce(speeding.ID_FileLog, vui.ID_FileLog)
|
||||||
|
where (
|
||||||
|
:occurredFrom is null
|
||||||
|
or speeding.BeginTime >= :occurredFrom
|
||||||
|
or speeding.EndTime >= :occurredFrom
|
||||||
|
)
|
||||||
|
and (
|
||||||
|
:occurredTo is null
|
||||||
|
or speeding.BeginTime < :occurredTo
|
||||||
|
or speeding.EndTime < :occurredTo
|
||||||
|
)
|
||||||
|
)
|
||||||
|
,
|
||||||
|
Events as (
|
||||||
|
select
|
||||||
|
base.*,
|
||||||
|
base.BeginTime as occurred_at,
|
||||||
|
'BEGIN' as lifecycle
|
||||||
|
from Base base
|
||||||
|
union all
|
||||||
|
select
|
||||||
|
base.*,
|
||||||
|
base.EndTime as occurred_at,
|
||||||
|
'END' as lifecycle
|
||||||
|
from Base base
|
||||||
|
)
|
||||||
|
select
|
||||||
|
concat(cast(events.ID as varchar(128)), ':', events.lifecycle) as source_row_id,
|
||||||
|
concat('TACHOGRAPH:SPEEDING_EVENTS:', events.ID, ':', events.lifecycle) as external_source_event_id,
|
||||||
|
|
||||||
|
events.occurred_at,
|
||||||
|
events.received_partner_at,
|
||||||
|
cast(events.AvgSpeed as decimal(10, 2)) as avg_speed_kmh,
|
||||||
|
cast(events.MaxSpeed as decimal(10, 2)) as max_speed_kmh,
|
||||||
|
events.lifecycle,
|
||||||
|
|
||||||
|
cast(events.driver_id as varchar(128)) as driver_source_entity_id,
|
||||||
|
events.driver_card_nation,
|
||||||
|
events.driver_card_number,
|
||||||
|
|
||||||
|
cast(events.vehicle_identification_id as varchar(128)) as vehicle_source_entity_id,
|
||||||
|
events.vehicle_vin,
|
||||||
|
cast(v.ID as varchar(128)) as vehicle_registration_source_entity_id,
|
||||||
|
vn.AlphaCode as vehicle_registration_nation,
|
||||||
|
v.VRN as vehicle_registration_number,
|
||||||
|
|
||||||
|
'VEHICLE_UNIT' as source_package_kind,
|
||||||
|
cast(events.source_package_id_raw as varchar(128)) as source_package_id,
|
||||||
|
cast(events.source_package_entity_id_raw as varchar(128)) as source_package_entity_id,
|
||||||
|
events.source_package_period_from,
|
||||||
|
events.source_package_period_to,
|
||||||
|
events.source_package_imported_at
|
||||||
|
from Events events
|
||||||
|
outer apply (
|
||||||
|
select top 1 vehicle.ID,
|
||||||
|
vehicle.VRN,
|
||||||
|
vehicle.ID_Nation
|
||||||
|
from dbo.Vehicle vehicle
|
||||||
|
where vehicle.ID_VehicleIdentification = events.vehicle_identification_id
|
||||||
|
and (vehicle.ValidFrom is null or vehicle.ValidFrom <= events.occurred_at)
|
||||||
|
and (vehicle.ValidTo is null or vehicle.ValidTo > events.occurred_at)
|
||||||
|
order by
|
||||||
|
vehicle.ValidFrom desc,
|
||||||
|
vehicle.ID desc
|
||||||
|
) v
|
||||||
|
left join dbo.Nation vn on vn.ID = v.ID_Nation
|
||||||
|
where (:occurredFrom is null or events.occurred_at >= :occurredFrom)
|
||||||
|
and (:occurredTo is null or events.occurred_at < :occurredTo)
|
||||||
|
and (
|
||||||
|
:organisationId is null
|
||||||
|
or exists (
|
||||||
|
select 1
|
||||||
|
from dbo.Vehicle_I_90021 rel
|
||||||
|
join OrgTree on OrgTree.I_90021_OID = rel.ID_I_90021
|
||||||
|
where rel.ID_Vehicle = v.ID
|
||||||
|
and rel.GILT_BIS is null
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
@ -5,16 +5,133 @@
|
||||||
* VUActivity -> VUDailyActivity -> VUInstallation -> VehicleIdentification
|
* VUActivity -> VUDailyActivity -> VUInstallation -> VehicleIdentification
|
||||||
* Optional driver/card context comes from VUActivity.ID_IWCycle -> IWCycle -> Card.
|
* Optional driver/card context comes from VUActivity.ID_IWCycle -> IWCycle -> Card.
|
||||||
*/
|
*/
|
||||||
select
|
with OrgTree as (
|
||||||
cast(va.ID as varchar(128)) as source_row_id,
|
select org.I_90021_OID
|
||||||
cast(va.ID as varchar(128)) as vu_activity_id,
|
from dbo.GetOrganisationTree(null, :organisationId, 0, null) org
|
||||||
concat('TACHOGRAPH:VU_ACTIVITY:', va.ID) as external_source_event_id,
|
where :organisationId is not null
|
||||||
|
)
|
||||||
|
,
|
||||||
|
CandidateActivity as (
|
||||||
|
select
|
||||||
|
va.ID,
|
||||||
|
va.BeginTime,
|
||||||
|
activity_time.EndTime,
|
||||||
|
va.Activity,
|
||||||
|
va.Slot,
|
||||||
|
va.CardStatus,
|
||||||
|
va.DrivingStatus,
|
||||||
|
va.ID_FileLog,
|
||||||
|
va.ID_IWCycle,
|
||||||
|
vda.RecordDate,
|
||||||
|
vda.ID_FileLog as vda_filelog_id,
|
||||||
|
vui.ID_VehicleIdentification,
|
||||||
|
vui.ID_FileLog as vui_filelog_id,
|
||||||
|
vi.ID as vehicle_identification_id,
|
||||||
|
vi.VIN as vehicle_vin,
|
||||||
|
coalesce(va.ID_FileLog, vda.ID_FileLog, vui.ID_FileLog) as file_log_id
|
||||||
|
from dbo.VUActivity va
|
||||||
|
join dbo.VUDailyActivity vda on vda.ID = va.ID_VUDailyActivity
|
||||||
|
join dbo.VUInstallation vui on vui.ID = vda.ID_VUInstallation
|
||||||
|
join dbo.VehicleIdentification vi on vi.ID = vui.ID_VehicleIdentification
|
||||||
|
cross apply (values (dateadd(minute, coalesce(va.Duration, 0), va.BeginTime))) activity_time(EndTime)
|
||||||
|
where (:occurredTo is null or va.BeginTime < :occurredTo)
|
||||||
|
and (
|
||||||
|
:occurredFrom is null
|
||||||
|
or va.BeginTime >= :occurredFrom
|
||||||
|
or activity_time.EndTime >= :occurredFrom
|
||||||
|
)
|
||||||
|
)
|
||||||
|
,
|
||||||
|
CandidateVehicle as (
|
||||||
|
select
|
||||||
|
va.ID,
|
||||||
|
va.BeginTime,
|
||||||
|
va.EndTime,
|
||||||
|
va.Activity,
|
||||||
|
va.Slot,
|
||||||
|
va.CardStatus,
|
||||||
|
va.DrivingStatus,
|
||||||
|
va.ID_FileLog,
|
||||||
|
va.ID_IWCycle,
|
||||||
|
va.RecordDate,
|
||||||
|
va.vda_filelog_id,
|
||||||
|
va.ID_VehicleIdentification,
|
||||||
|
va.vui_filelog_id,
|
||||||
|
va.vehicle_identification_id,
|
||||||
|
va.vehicle_vin,
|
||||||
|
va.file_log_id,
|
||||||
|
v.ID as vehicle_registration_id,
|
||||||
|
v.ID_Nation as vehicle_registration_nation_id,
|
||||||
|
v.VRN as vehicle_registration_number
|
||||||
|
from CandidateActivity va
|
||||||
|
outer apply (
|
||||||
|
select top 1 vehicle.ID,
|
||||||
|
vehicle.VRN,
|
||||||
|
vehicle.ID_Nation
|
||||||
|
from dbo.Vehicle vehicle
|
||||||
|
where vehicle.ID_VehicleIdentification = va.vehicle_identification_id
|
||||||
|
and (vehicle.ValidFrom is null or vehicle.ValidFrom <= va.BeginTime)
|
||||||
|
and (vehicle.ValidTo is null or vehicle.ValidTo > va.BeginTime)
|
||||||
|
order by
|
||||||
|
vehicle.ValidFrom desc,
|
||||||
|
vehicle.ID desc
|
||||||
|
) v
|
||||||
|
where (
|
||||||
|
:organisationId is null
|
||||||
|
or exists (
|
||||||
|
select 1
|
||||||
|
from dbo.Vehicle_I_90021 rel
|
||||||
|
join OrgTree on OrgTree.I_90021_OID = rel.ID_I_90021
|
||||||
|
where rel.ID_Vehicle = v.ID
|
||||||
|
and rel.GILT_BIS is null
|
||||||
|
)
|
||||||
|
)
|
||||||
|
)
|
||||||
|
,
|
||||||
|
Base as (
|
||||||
|
select
|
||||||
|
va.ID,
|
||||||
|
va.BeginTime,
|
||||||
|
va.EndTime,
|
||||||
|
va.Activity,
|
||||||
|
va.Slot,
|
||||||
|
va.CardStatus,
|
||||||
|
va.DrivingStatus,
|
||||||
|
va.ID_FileLog,
|
||||||
|
va.vda_filelog_id,
|
||||||
|
va.vui_filelog_id,
|
||||||
|
|
||||||
va.BeginTime as occurred_at,
|
coalesce(fl.DownloadDate, fl.OriginalDownloadDate, fl.TStamp, fl.CreationDate) as received_partner_at,
|
||||||
coalesce(fl.DownloadDate, fl.OriginalDownloadDate, fl.TStamp, fl.CreationDate) as received_partner_at,
|
coalesce(fl.ID, va.ID_FileLog, va.vda_filelog_id, va.vui_filelog_id) as source_package_id_raw,
|
||||||
va.Activity as activity_code,
|
coalesce(fl.ID_VehicleIdentification, va.ID_VehicleIdentification) as source_package_entity_id_raw,
|
||||||
va.Activity as activity_text,
|
coalesce(fl.DownloadFrom, va.RecordDate) as source_package_period_from,
|
||||||
case upper(coalesce(va.Activity, ''))
|
coalesce(fl.DownloadTo, dateadd(day, 1, va.RecordDate)) as source_package_period_to,
|
||||||
|
coalesce(fl.CreationDate, fl.TStamp) as source_package_imported_at,
|
||||||
|
|
||||||
|
null as odometer_m,
|
||||||
|
c.ID_Driver as driver_id,
|
||||||
|
cn.AlphaCode as driver_card_nation,
|
||||||
|
c.CardNumber as driver_card_number,
|
||||||
|
va.vehicle_identification_id,
|
||||||
|
va.vehicle_registration_id,
|
||||||
|
va.vehicle_vin,
|
||||||
|
vn.AlphaCode as vehicle_registration_nation,
|
||||||
|
va.vehicle_registration_number
|
||||||
|
from CandidateVehicle va
|
||||||
|
left join dbo.FileLog fl on fl.ID = va.file_log_id
|
||||||
|
left join dbo.Nation vn on vn.ID = va.vehicle_registration_nation_id
|
||||||
|
left join dbo.IWCycle iw on iw.ID = va.ID_IWCycle
|
||||||
|
left join dbo.Card c on c.ID = iw.ID_Card
|
||||||
|
left join dbo.Nation cn on cn.ID = c.ID_Nation
|
||||||
|
)
|
||||||
|
select
|
||||||
|
cast(base.ID as varchar(128)) as source_row_id,
|
||||||
|
concat('TACHOGRAPH:VU_ACTIVITY:', base.ID, ':', evt.lifecycle) as external_source_event_id,
|
||||||
|
|
||||||
|
evt.occurred_at as occurred_at,
|
||||||
|
base.received_partner_at,
|
||||||
|
base.Activity as activity_code,
|
||||||
|
case upper(coalesce(base.Activity, ''))
|
||||||
when 'DRIVING' then 'DRIVE'
|
when 'DRIVING' then 'DRIVE'
|
||||||
when 'DRIVE' then 'DRIVE'
|
when 'DRIVE' then 'DRIVE'
|
||||||
when 'WORK' then 'WORK'
|
when 'WORK' then 'WORK'
|
||||||
|
|
@ -25,68 +142,35 @@ select
|
||||||
when 'REST' then 'BREAK_REST'
|
when 'REST' then 'BREAK_REST'
|
||||||
else 'UNKNOWN_ACTIVITY'
|
else 'UNKNOWN_ACTIVITY'
|
||||||
end as event_type,
|
end as event_type,
|
||||||
'SNAPSHOT' as lifecycle,
|
evt.lifecycle as lifecycle,
|
||||||
va.Slot as card_slot,
|
base.Slot as card_slot,
|
||||||
va.CardStatus as card_status,
|
base.CardStatus as card_status,
|
||||||
va.DrivingStatus as driving_status,
|
base.DrivingStatus as driving_status,
|
||||||
cast(iw.OdoBegin as bigint) as odometer_m,
|
base.odometer_m,
|
||||||
|
|
||||||
cast(d.ID as varchar(128)) as driver_source_entity_id,
|
cast(base.driver_id as varchar(128)) as driver_source_entity_id,
|
||||||
cn.AlphaCode as driver_card_nation,
|
base.driver_card_nation,
|
||||||
c.CardNumber as driver_card_number,
|
base.driver_card_number,
|
||||||
|
|
||||||
cast(v.ID as varchar(128)) as vehicle_source_entity_id,
|
cast(base.vehicle_identification_id as varchar(128)) as vehicle_source_entity_id,
|
||||||
vi.VIN as vehicle_vin,
|
base.vehicle_vin,
|
||||||
vn.AlphaCode as vehicle_registration_nation,
|
cast(base.vehicle_registration_id as varchar(128)) as vehicle_registration_source_entity_id,
|
||||||
v.VRN as vehicle_registration_number,
|
base.vehicle_registration_nation,
|
||||||
|
base.vehicle_registration_number,
|
||||||
|
|
||||||
'VEHICLE_UNIT' as source_package_kind,
|
'VEHICLE_UNIT' as source_package_kind,
|
||||||
cast(coalesce(fl.ID, va.ID_FileLog, vda.ID_FileLog, vui.ID_FileLog, va.ID) as varchar(128)) as source_package_id,
|
cast(base.source_package_id_raw as varchar(128)) as source_package_id,
|
||||||
cast(coalesce(fl.ID_VehicleIdentification, vui.ID_VehicleIdentification) as varchar(128)) as source_package_entity_id,
|
cast(base.source_package_entity_id_raw as varchar(128)) as source_package_entity_id,
|
||||||
coalesce(fl.DownloadFrom, vda.RecordDate) as source_package_period_from,
|
base.source_package_period_from,
|
||||||
coalesce(fl.DownloadTo, dateadd(day, 1, vda.RecordDate)) as source_package_period_to,
|
base.source_package_period_to,
|
||||||
coalesce(fl.CreationDate, fl.TStamp) as source_package_imported_at
|
base.source_package_imported_at
|
||||||
from dbo.VUActivity va
|
from Base base
|
||||||
join dbo.VUDailyActivity vda on vda.ID = va.ID_VUDailyActivity
|
cross apply (values
|
||||||
join dbo.VUInstallation vui on vui.ID = vda.ID_VUInstallation
|
('START', base.BeginTime),
|
||||||
left join dbo.FileLog fl on fl.ID = coalesce(va.ID_FileLog, vda.ID_FileLog, vui.ID_FileLog)
|
('END', base.EndTime)
|
||||||
join dbo.VehicleIdentification vi on vi.ID = vui.ID_VehicleIdentification
|
) evt(lifecycle, occurred_at)
|
||||||
outer apply (
|
where (:occurredFrom is null or evt.occurred_at >= :occurredFrom)
|
||||||
select top 1 vehicle.ID,
|
and (:occurredTo is null or evt.occurred_at < :occurredTo)
|
||||||
vehicle.VRN,
|
|
||||||
vehicle.ID_Nation
|
|
||||||
from dbo.Vehicle vehicle
|
|
||||||
where vehicle.ID_VehicleIdentification = vi.ID
|
|
||||||
and (vehicle.ValidFrom is null or vehicle.ValidFrom <= va.BeginTime)
|
|
||||||
and (vehicle.ValidTo is null or vehicle.ValidTo > va.BeginTime)
|
|
||||||
order by
|
|
||||||
case when vehicle.ValidFrom is null then 1 else 0 end,
|
|
||||||
vehicle.ValidFrom desc,
|
|
||||||
vehicle.ID desc
|
|
||||||
) v
|
|
||||||
left join dbo.Nation vn on vn.ID = v.ID_Nation
|
|
||||||
left join dbo.IWCycle iw on iw.ID = va.ID_IWCycle
|
|
||||||
left join dbo.Card c on c.ID = iw.ID_Card
|
|
||||||
left join dbo.Driver d on d.ID = c.ID_Driver
|
|
||||||
left join dbo.Nation cn on cn.ID = c.ID_Nation
|
|
||||||
where (:occurredFrom is null or va.BeginTime >= :occurredFrom)
|
|
||||||
and (:occurredTo is null or va.BeginTime < :occurredTo)
|
|
||||||
and (
|
|
||||||
:lastSourcePackageImportedAt is null
|
|
||||||
or coalesce(fl.CreationDate, fl.TStamp) > :lastSourcePackageImportedAt
|
|
||||||
or (
|
|
||||||
coalesce(fl.CreationDate, fl.TStamp) = :lastSourcePackageImportedAt
|
|
||||||
and coalesce(fl.ID, va.ID_FileLog, vda.ID_FileLog, vui.ID_FileLog, va.ID) > try_convert(int, :lastSourcePackageId)
|
|
||||||
)
|
|
||||||
or (
|
|
||||||
coalesce(fl.CreationDate, fl.TStamp) is null
|
|
||||||
and (
|
|
||||||
:lastSourcePackageId is null
|
|
||||||
or coalesce(fl.ID, va.ID_FileLog, vda.ID_FileLog, vui.ID_FileLog, va.ID) > try_convert(int, :lastSourcePackageId)
|
|
||||||
)
|
|
||||||
)
|
|
||||||
)
|
|
||||||
/*
|
/*
|
||||||
* Organisation filtering can use FileLog.I_90021_ID / FileLog.OrgID or
|
* Organisation filter: vehicle membership in GetOrganisationTree(null, :organisationId, 0, null).
|
||||||
* Vehicle_I_90021 / Driver_I_90021 once subtree semantics are confirmed.
|
|
||||||
*/
|
*/
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,100 @@
|
||||||
|
/*
|
||||||
|
* VUBorderCrossing BORDER_CROSSING extraction for the bytebar tachograph schema.
|
||||||
|
*/
|
||||||
|
with OrgTree as (
|
||||||
|
select org.I_90021_OID
|
||||||
|
from dbo.GetOrganisationTree(null, :organisationId, 0, null) org
|
||||||
|
where :organisationId is not null
|
||||||
|
)
|
||||||
|
,
|
||||||
|
Base as (
|
||||||
|
select
|
||||||
|
border.ID,
|
||||||
|
border.Timestamp as occurred_at,
|
||||||
|
border.Odo,
|
||||||
|
border.ID_FileLog,
|
||||||
|
border.ID_VUInstallation,
|
||||||
|
coalesce(driverCard.ID, coDriverCard.ID) as card_id,
|
||||||
|
coalesce(driverCard.ID_Driver, coDriverCard.ID_Driver) as driver_id,
|
||||||
|
coalesce(driverNation.AlphaCode, coDriverNation.AlphaCode) as driver_card_nation,
|
||||||
|
coalesce(driverCard.CardNumber, coDriverCard.CardNumber) as driver_card_number,
|
||||||
|
leftNation.AlphaCode as country_from,
|
||||||
|
enteredNation.AlphaCode as country_to,
|
||||||
|
gnss.Latitude as latitude,
|
||||||
|
gnss.Longitude as longitude,
|
||||||
|
vui.ID_FileLog as vui_filelog_id,
|
||||||
|
vui.ID_VehicleIdentification,
|
||||||
|
vi.ID as vehicle_identification_id,
|
||||||
|
vi.VIN as vehicle_vin,
|
||||||
|
coalesce(fl.DownloadDate, fl.OriginalDownloadDate, fl.TStamp, fl.CreationDate) as received_partner_at,
|
||||||
|
coalesce(fl.ID, border.ID_FileLog, vui.ID_FileLog) as source_package_id_raw,
|
||||||
|
coalesce(fl.ID_VehicleIdentification, vui.ID_VehicleIdentification) as source_package_entity_id_raw,
|
||||||
|
coalesce(fl.DownloadFrom, border.Timestamp) as source_package_period_from,
|
||||||
|
coalesce(fl.DownloadTo, border.Timestamp) as source_package_period_to,
|
||||||
|
coalesce(fl.CreationDate, fl.TStamp) as source_package_imported_at
|
||||||
|
from dbo.VUBorderCrossing border
|
||||||
|
join dbo.VUInstallation vui on vui.ID = border.ID_VUInstallation
|
||||||
|
join dbo.VehicleIdentification vi on vi.ID = vui.ID_VehicleIdentification
|
||||||
|
left join dbo.Card driverCard on driverCard.ID = border.ID_DriverCard
|
||||||
|
left join dbo.Nation driverNation on driverNation.ID = driverCard.ID_Nation
|
||||||
|
left join dbo.Card coDriverCard on coDriverCard.ID = border.ID_CoDriverCard
|
||||||
|
left join dbo.Nation coDriverNation on coDriverNation.ID = coDriverCard.ID_Nation
|
||||||
|
left join dbo.Nation leftNation on leftNation.ID = border.ID_NationLeft
|
||||||
|
left join dbo.Nation enteredNation on enteredNation.ID = border.ID_NationEntered
|
||||||
|
left join dbo.GnssPlace gnss on gnss.ID = border.ID_GnssPlace
|
||||||
|
left join dbo.FileLog fl on fl.ID = coalesce(border.ID_FileLog, vui.ID_FileLog)
|
||||||
|
where (:occurredFrom is null or border.Timestamp >= :occurredFrom)
|
||||||
|
and (:occurredTo is null or border.Timestamp < :occurredTo)
|
||||||
|
)
|
||||||
|
select
|
||||||
|
cast(base.ID as varchar(128)) as source_row_id,
|
||||||
|
concat('TACHOGRAPH:VU_BORDER_CROSSING:', base.ID) as external_source_event_id,
|
||||||
|
|
||||||
|
base.occurred_at,
|
||||||
|
base.received_partner_at,
|
||||||
|
cast(base.Odo as bigint) * 1000 as odometer_m,
|
||||||
|
base.latitude,
|
||||||
|
base.longitude,
|
||||||
|
base.country_from,
|
||||||
|
base.country_to,
|
||||||
|
|
||||||
|
cast(base.driver_id as varchar(128)) as driver_source_entity_id,
|
||||||
|
base.driver_card_nation,
|
||||||
|
base.driver_card_number,
|
||||||
|
|
||||||
|
cast(base.vehicle_identification_id as varchar(128)) as vehicle_source_entity_id,
|
||||||
|
base.vehicle_vin,
|
||||||
|
cast(v.ID as varchar(128)) as vehicle_registration_source_entity_id,
|
||||||
|
vn.AlphaCode as vehicle_registration_nation,
|
||||||
|
v.VRN as vehicle_registration_number,
|
||||||
|
|
||||||
|
'VEHICLE_UNIT' as source_package_kind,
|
||||||
|
cast(base.source_package_id_raw as varchar(128)) as source_package_id,
|
||||||
|
cast(base.source_package_entity_id_raw as varchar(128)) as source_package_entity_id,
|
||||||
|
base.source_package_period_from,
|
||||||
|
base.source_package_period_to,
|
||||||
|
base.source_package_imported_at
|
||||||
|
from Base base
|
||||||
|
outer apply (
|
||||||
|
select top 1 vehicle.ID,
|
||||||
|
vehicle.VRN,
|
||||||
|
vehicle.ID_Nation
|
||||||
|
from dbo.Vehicle vehicle
|
||||||
|
where vehicle.ID_VehicleIdentification = base.vehicle_identification_id
|
||||||
|
and (vehicle.ValidFrom is null or vehicle.ValidFrom <= base.occurred_at)
|
||||||
|
and (vehicle.ValidTo is null or vehicle.ValidTo > base.occurred_at)
|
||||||
|
order by
|
||||||
|
vehicle.ValidFrom desc,
|
||||||
|
vehicle.ID desc
|
||||||
|
) v
|
||||||
|
left join dbo.Nation vn on vn.ID = v.ID_Nation
|
||||||
|
where (
|
||||||
|
:organisationId is null
|
||||||
|
or exists (
|
||||||
|
select 1
|
||||||
|
from dbo.Vehicle_I_90021 rel
|
||||||
|
join OrgTree on OrgTree.I_90021_OID = rel.ID_I_90021
|
||||||
|
where rel.ID_Vehicle = v.ID
|
||||||
|
and rel.GILT_BIS is null
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
@ -0,0 +1,105 @@
|
||||||
|
/*
|
||||||
|
* VULoadUnload LOAD_UNLOAD extraction for the bytebar tachograph schema.
|
||||||
|
*/
|
||||||
|
with OrgTree as (
|
||||||
|
select org.I_90021_OID
|
||||||
|
from dbo.GetOrganisationTree(null, :organisationId, 0, null) org
|
||||||
|
where :organisationId is not null
|
||||||
|
)
|
||||||
|
,
|
||||||
|
Base as (
|
||||||
|
select
|
||||||
|
lu.ID,
|
||||||
|
lu.Timestamp as occurred_at,
|
||||||
|
lu.OperationType,
|
||||||
|
lu.Odo,
|
||||||
|
lu.ID_FileLog,
|
||||||
|
lu.ID_VUInstallation,
|
||||||
|
coalesce(driverCard.ID, coDriverCard.ID) as card_id,
|
||||||
|
coalesce(driverCard.ID_Driver, coDriverCard.ID_Driver) as driver_id,
|
||||||
|
coalesce(driverNation.AlphaCode, coDriverNation.AlphaCode) as driver_card_nation,
|
||||||
|
coalesce(driverCard.CardNumber, coDriverCard.CardNumber) as driver_card_number,
|
||||||
|
gnss.Latitude as latitude,
|
||||||
|
gnss.Longitude as longitude,
|
||||||
|
vui.ID_FileLog as vui_filelog_id,
|
||||||
|
vui.ID_VehicleIdentification,
|
||||||
|
vi.ID as vehicle_identification_id,
|
||||||
|
vi.VIN as vehicle_vin,
|
||||||
|
coalesce(fl.DownloadDate, fl.OriginalDownloadDate, fl.TStamp, fl.CreationDate) as received_partner_at,
|
||||||
|
coalesce(fl.ID, lu.ID_FileLog, vui.ID_FileLog) as source_package_id_raw,
|
||||||
|
coalesce(fl.ID_VehicleIdentification, vui.ID_VehicleIdentification) as source_package_entity_id_raw,
|
||||||
|
coalesce(fl.DownloadFrom, lu.Timestamp) as source_package_period_from,
|
||||||
|
coalesce(fl.DownloadTo, lu.Timestamp) as source_package_period_to,
|
||||||
|
coalesce(fl.CreationDate, fl.TStamp) as source_package_imported_at
|
||||||
|
from dbo.VULoadUnload lu
|
||||||
|
join dbo.VUInstallation vui on vui.ID = lu.ID_VUInstallation
|
||||||
|
join dbo.VehicleIdentification vi on vi.ID = vui.ID_VehicleIdentification
|
||||||
|
left join dbo.Card driverCard on driverCard.ID = lu.ID_DriverCard
|
||||||
|
left join dbo.Nation driverNation on driverNation.ID = driverCard.ID_Nation
|
||||||
|
left join dbo.Card coDriverCard on coDriverCard.ID = lu.ID_CoDriverCard
|
||||||
|
left join dbo.Nation coDriverNation on coDriverNation.ID = coDriverCard.ID_Nation
|
||||||
|
left join dbo.GnssPlace gnss on gnss.ID = lu.ID_GnssPlace
|
||||||
|
left join dbo.FileLog fl on fl.ID = coalesce(lu.ID_FileLog, vui.ID_FileLog)
|
||||||
|
where (:occurredFrom is null or lu.Timestamp >= :occurredFrom)
|
||||||
|
and (:occurredTo is null or lu.Timestamp < :occurredTo)
|
||||||
|
)
|
||||||
|
select
|
||||||
|
cast(base.ID as varchar(128)) as source_row_id,
|
||||||
|
concat('TACHOGRAPH:VU_LOAD_UNLOAD:', base.ID) as external_source_event_id,
|
||||||
|
|
||||||
|
base.occurred_at,
|
||||||
|
base.received_partner_at,
|
||||||
|
cast(base.Odo as bigint) * 1000 as odometer_m,
|
||||||
|
base.latitude,
|
||||||
|
base.longitude,
|
||||||
|
case base.OperationType
|
||||||
|
when 1 then 'LOAD'
|
||||||
|
when 2 then 'UNLOAD'
|
||||||
|
else 'LOAD_UNLOAD'
|
||||||
|
end as operation,
|
||||||
|
case base.OperationType
|
||||||
|
when 1 then 'LOAD'
|
||||||
|
when 2 then 'UNLOAD'
|
||||||
|
else 'LOAD_UNLOAD'
|
||||||
|
end as event_type,
|
||||||
|
|
||||||
|
cast(base.driver_id as varchar(128)) as driver_source_entity_id,
|
||||||
|
base.driver_card_nation,
|
||||||
|
base.driver_card_number,
|
||||||
|
|
||||||
|
cast(base.vehicle_identification_id as varchar(128)) as vehicle_source_entity_id,
|
||||||
|
base.vehicle_vin,
|
||||||
|
cast(v.ID as varchar(128)) as vehicle_registration_source_entity_id,
|
||||||
|
vn.AlphaCode as vehicle_registration_nation,
|
||||||
|
v.VRN as vehicle_registration_number,
|
||||||
|
|
||||||
|
'VEHICLE_UNIT' as source_package_kind,
|
||||||
|
cast(base.source_package_id_raw as varchar(128)) as source_package_id,
|
||||||
|
cast(base.source_package_entity_id_raw as varchar(128)) as source_package_entity_id,
|
||||||
|
base.source_package_period_from,
|
||||||
|
base.source_package_period_to,
|
||||||
|
base.source_package_imported_at
|
||||||
|
from Base base
|
||||||
|
outer apply (
|
||||||
|
select top 1 vehicle.ID,
|
||||||
|
vehicle.VRN,
|
||||||
|
vehicle.ID_Nation
|
||||||
|
from dbo.Vehicle vehicle
|
||||||
|
where vehicle.ID_VehicleIdentification = base.vehicle_identification_id
|
||||||
|
and (vehicle.ValidFrom is null or vehicle.ValidFrom <= base.occurred_at)
|
||||||
|
and (vehicle.ValidTo is null or vehicle.ValidTo > base.occurred_at)
|
||||||
|
order by
|
||||||
|
vehicle.ValidFrom desc,
|
||||||
|
vehicle.ID desc
|
||||||
|
) v
|
||||||
|
left join dbo.Nation vn on vn.ID = v.ID_Nation
|
||||||
|
where (
|
||||||
|
:organisationId is null
|
||||||
|
or exists (
|
||||||
|
select 1
|
||||||
|
from dbo.Vehicle_I_90021 rel
|
||||||
|
join OrgTree on OrgTree.I_90021_OID = rel.ID_I_90021
|
||||||
|
where rel.ID_Vehicle = v.ID
|
||||||
|
and rel.GILT_BIS is null
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
@ -0,0 +1,101 @@
|
||||||
|
/*
|
||||||
|
* VUPlaces PLACE extraction for the bytebar tachograph schema.
|
||||||
|
*/
|
||||||
|
with OrgTree as (
|
||||||
|
select org.I_90021_OID
|
||||||
|
from dbo.GetOrganisationTree(null, :organisationId, 0, null) org
|
||||||
|
where :organisationId is not null
|
||||||
|
)
|
||||||
|
,
|
||||||
|
Base as (
|
||||||
|
select
|
||||||
|
place.ID,
|
||||||
|
place.EntryTime as occurred_at,
|
||||||
|
place.EntryType as entry_type,
|
||||||
|
place.Odo,
|
||||||
|
place.ID_FileLog,
|
||||||
|
c.ID as card_id,
|
||||||
|
c.ID_Driver as driver_id,
|
||||||
|
cn.AlphaCode as driver_card_nation,
|
||||||
|
c.CardNumber as driver_card_number,
|
||||||
|
placeNation.AlphaCode as country,
|
||||||
|
coalesce(region.AlphaCode, region.Name) as region,
|
||||||
|
gnss.Latitude as latitude,
|
||||||
|
gnss.Longitude as longitude,
|
||||||
|
vui.ID_FileLog as vui_filelog_id,
|
||||||
|
vui.ID_VehicleIdentification,
|
||||||
|
vi.ID as vehicle_identification_id,
|
||||||
|
vi.VIN as vehicle_vin,
|
||||||
|
coalesce(fl.DownloadDate, fl.OriginalDownloadDate, fl.TStamp, fl.CreationDate) as received_partner_at,
|
||||||
|
coalesce(fl.ID, place.ID_FileLog, vui.ID_FileLog) as source_package_id_raw,
|
||||||
|
coalesce(fl.ID_VehicleIdentification, vui.ID_VehicleIdentification) as source_package_entity_id_raw,
|
||||||
|
coalesce(fl.DownloadFrom, place.EntryTime) as source_package_period_from,
|
||||||
|
coalesce(fl.DownloadTo, place.EntryTime) as source_package_period_to,
|
||||||
|
coalesce(fl.CreationDate, fl.TStamp) as source_package_imported_at
|
||||||
|
from dbo.VUPlaces place
|
||||||
|
join dbo.VUInstallation vui on vui.ID = place.ID_VUInstallation
|
||||||
|
join dbo.VehicleIdentification vi on vi.ID = vui.ID_VehicleIdentification
|
||||||
|
left join dbo.Card c on c.ID = place.ID_Card
|
||||||
|
left join dbo.Nation cn on cn.ID = c.ID_Nation
|
||||||
|
left join dbo.Nation placeNation on placeNation.ID = place.ID_Nation
|
||||||
|
left join dbo.Region region on region.ID = place.ID_Region
|
||||||
|
left join dbo.GnssPlace gnss on gnss.ID = place.ID_GnssPlace
|
||||||
|
left join dbo.FileLog fl on fl.ID = coalesce(place.ID_FileLog, vui.ID_FileLog)
|
||||||
|
where (:occurredFrom is null or place.EntryTime >= :occurredFrom)
|
||||||
|
and (:occurredTo is null or place.EntryTime < :occurredTo)
|
||||||
|
)
|
||||||
|
select
|
||||||
|
cast(base.ID as varchar(128)) as source_row_id,
|
||||||
|
concat('TACHOGRAPH:VU_PLACE:', base.ID) as external_source_event_id,
|
||||||
|
|
||||||
|
base.occurred_at,
|
||||||
|
base.received_partner_at,
|
||||||
|
cast(base.Odo as bigint) * 1000 as odometer_m,
|
||||||
|
base.latitude,
|
||||||
|
base.longitude,
|
||||||
|
base.country,
|
||||||
|
base.region,
|
||||||
|
'WORKING_DAY_PLACE_RECORDED' as event_type,
|
||||||
|
case when base.entry_type in (0, 2, 4) then 'START' else 'END' end as lifecycle,
|
||||||
|
cast(case when base.entry_type in (2, 3, 4, 5) then 1 else 0 end as bit) as manual_entry,
|
||||||
|
|
||||||
|
cast(base.driver_id as varchar(128)) as driver_source_entity_id,
|
||||||
|
base.driver_card_nation,
|
||||||
|
base.driver_card_number,
|
||||||
|
|
||||||
|
cast(base.vehicle_identification_id as varchar(128)) as vehicle_source_entity_id,
|
||||||
|
base.vehicle_vin,
|
||||||
|
cast(v.ID as varchar(128)) as vehicle_registration_source_entity_id,
|
||||||
|
vn.AlphaCode as vehicle_registration_nation,
|
||||||
|
v.VRN as vehicle_registration_number,
|
||||||
|
|
||||||
|
'VEHICLE_UNIT' as source_package_kind,
|
||||||
|
cast(base.source_package_id_raw as varchar(128)) as source_package_id,
|
||||||
|
cast(base.source_package_entity_id_raw as varchar(128)) as source_package_entity_id,
|
||||||
|
base.source_package_period_from,
|
||||||
|
base.source_package_period_to,
|
||||||
|
base.source_package_imported_at
|
||||||
|
from Base base
|
||||||
|
outer apply (
|
||||||
|
select top 1 vehicle.ID,
|
||||||
|
vehicle.VRN,
|
||||||
|
vehicle.ID_Nation
|
||||||
|
from dbo.Vehicle vehicle
|
||||||
|
where vehicle.ID_VehicleIdentification = base.vehicle_identification_id
|
||||||
|
and (vehicle.ValidFrom is null or vehicle.ValidFrom <= base.occurred_at)
|
||||||
|
and (vehicle.ValidTo is null or vehicle.ValidTo > base.occurred_at)
|
||||||
|
order by
|
||||||
|
vehicle.ValidFrom desc,
|
||||||
|
vehicle.ID desc
|
||||||
|
) v
|
||||||
|
left join dbo.Nation vn on vn.ID = v.ID_Nation
|
||||||
|
where (
|
||||||
|
:organisationId is null
|
||||||
|
or exists (
|
||||||
|
select 1
|
||||||
|
from dbo.Vehicle_I_90021 rel
|
||||||
|
join OrgTree on OrgTree.I_90021_OID = rel.ID_I_90021
|
||||||
|
where rel.ID_Vehicle = v.ID
|
||||||
|
and rel.GILT_BIS is null
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
@ -0,0 +1,94 @@
|
||||||
|
/*
|
||||||
|
* VUGnssAccumulatedDriving POSITION extraction for the bytebar tachograph schema.
|
||||||
|
*/
|
||||||
|
with OrgTree as (
|
||||||
|
select org.I_90021_OID
|
||||||
|
from dbo.GetOrganisationTree(null, :organisationId, 0, null) org
|
||||||
|
where :organisationId is not null
|
||||||
|
)
|
||||||
|
,
|
||||||
|
Base as (
|
||||||
|
select
|
||||||
|
pos.ID,
|
||||||
|
pos.Timestamp as occurred_at,
|
||||||
|
pos.Odo,
|
||||||
|
pos.ID_FileLog,
|
||||||
|
pos.ID_VUInstallation,
|
||||||
|
coalesce(driverCard.ID, coDriverCard.ID) as card_id,
|
||||||
|
coalesce(driverCard.ID_Driver, coDriverCard.ID_Driver) as driver_id,
|
||||||
|
coalesce(driverNation.AlphaCode, coDriverNation.AlphaCode) as driver_card_nation,
|
||||||
|
coalesce(driverCard.CardNumber, coDriverCard.CardNumber) as driver_card_number,
|
||||||
|
gnss.Latitude as latitude,
|
||||||
|
gnss.Longitude as longitude,
|
||||||
|
vui.ID_FileLog as vui_filelog_id,
|
||||||
|
vui.ID_VehicleIdentification,
|
||||||
|
vi.ID as vehicle_identification_id,
|
||||||
|
vi.VIN as vehicle_vin,
|
||||||
|
coalesce(fl.DownloadDate, fl.OriginalDownloadDate, fl.TStamp, fl.CreationDate) as received_partner_at,
|
||||||
|
coalesce(fl.ID, pos.ID_FileLog, vui.ID_FileLog) as source_package_id_raw,
|
||||||
|
coalesce(fl.ID_VehicleIdentification, vui.ID_VehicleIdentification) as source_package_entity_id_raw,
|
||||||
|
coalesce(fl.DownloadFrom, pos.Timestamp) as source_package_period_from,
|
||||||
|
coalesce(fl.DownloadTo, pos.Timestamp) as source_package_period_to,
|
||||||
|
coalesce(fl.CreationDate, fl.TStamp) as source_package_imported_at
|
||||||
|
from dbo.VUGnssAccumulatedDriving pos
|
||||||
|
join dbo.VUInstallation vui on vui.ID = pos.ID_VUInstallation
|
||||||
|
join dbo.VehicleIdentification vi on vi.ID = vui.ID_VehicleIdentification
|
||||||
|
left join dbo.Card driverCard on driverCard.ID = pos.ID_DriverCard
|
||||||
|
left join dbo.Nation driverNation on driverNation.ID = driverCard.ID_Nation
|
||||||
|
left join dbo.Card coDriverCard on coDriverCard.ID = pos.ID_CoDriverCard
|
||||||
|
left join dbo.Nation coDriverNation on coDriverNation.ID = coDriverCard.ID_Nation
|
||||||
|
join dbo.GnssPlace gnss on gnss.ID = pos.ID_GnssPlace
|
||||||
|
left join dbo.FileLog fl on fl.ID = coalesce(pos.ID_FileLog, vui.ID_FileLog)
|
||||||
|
where (:occurredFrom is null or pos.Timestamp >= :occurredFrom)
|
||||||
|
and (:occurredTo is null or pos.Timestamp < :occurredTo)
|
||||||
|
)
|
||||||
|
select
|
||||||
|
cast(base.ID as varchar(128)) as source_row_id,
|
||||||
|
concat('TACHOGRAPH:VU_POSITION:', base.ID) as external_source_event_id,
|
||||||
|
|
||||||
|
base.occurred_at,
|
||||||
|
base.received_partner_at,
|
||||||
|
cast(base.Odo as bigint) * 1000 as odometer_m,
|
||||||
|
base.latitude,
|
||||||
|
base.longitude,
|
||||||
|
|
||||||
|
cast(base.driver_id as varchar(128)) as driver_source_entity_id,
|
||||||
|
base.driver_card_nation,
|
||||||
|
base.driver_card_number,
|
||||||
|
|
||||||
|
cast(base.vehicle_identification_id as varchar(128)) as vehicle_source_entity_id,
|
||||||
|
base.vehicle_vin,
|
||||||
|
cast(v.ID as varchar(128)) as vehicle_registration_source_entity_id,
|
||||||
|
vn.AlphaCode as vehicle_registration_nation,
|
||||||
|
v.VRN as vehicle_registration_number,
|
||||||
|
|
||||||
|
'VEHICLE_UNIT' as source_package_kind,
|
||||||
|
cast(base.source_package_id_raw as varchar(128)) as source_package_id,
|
||||||
|
cast(base.source_package_entity_id_raw as varchar(128)) as source_package_entity_id,
|
||||||
|
base.source_package_period_from,
|
||||||
|
base.source_package_period_to,
|
||||||
|
base.source_package_imported_at
|
||||||
|
from Base base
|
||||||
|
outer apply (
|
||||||
|
select top 1 vehicle.ID,
|
||||||
|
vehicle.VRN,
|
||||||
|
vehicle.ID_Nation
|
||||||
|
from dbo.Vehicle vehicle
|
||||||
|
where vehicle.ID_VehicleIdentification = base.vehicle_identification_id
|
||||||
|
and (vehicle.ValidFrom is null or vehicle.ValidFrom <= base.occurred_at)
|
||||||
|
and (vehicle.ValidTo is null or vehicle.ValidTo > base.occurred_at)
|
||||||
|
order by
|
||||||
|
vehicle.ValidFrom desc,
|
||||||
|
vehicle.ID desc
|
||||||
|
) v
|
||||||
|
left join dbo.Nation vn on vn.ID = v.ID_Nation
|
||||||
|
where (
|
||||||
|
:organisationId is null
|
||||||
|
or exists (
|
||||||
|
select 1
|
||||||
|
from dbo.Vehicle_I_90021 rel
|
||||||
|
join OrgTree on OrgTree.I_90021_OID = rel.ID_I_90021
|
||||||
|
where rel.ID_Vehicle = v.ID
|
||||||
|
and rel.GILT_BIS is null
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
@ -0,0 +1,84 @@
|
||||||
|
/*
|
||||||
|
* VUSpecificCondition SPECIFIC_CONDITION extraction for the bytebar tachograph schema.
|
||||||
|
*/
|
||||||
|
with OrgTree as (
|
||||||
|
select org.I_90021_OID
|
||||||
|
from dbo.GetOrganisationTree(null, :organisationId, 0, null) org
|
||||||
|
where :organisationId is not null
|
||||||
|
)
|
||||||
|
,
|
||||||
|
Base as (
|
||||||
|
select
|
||||||
|
cond.ID,
|
||||||
|
cond.EntryTime as occurred_at,
|
||||||
|
sc.Condition as condition_code,
|
||||||
|
cond.ID_FileLog,
|
||||||
|
cond.ID_VUInstallation,
|
||||||
|
vui.ID_FileLog as vui_filelog_id,
|
||||||
|
vui.ID_VehicleIdentification,
|
||||||
|
vi.ID as vehicle_identification_id,
|
||||||
|
vi.VIN as vehicle_vin,
|
||||||
|
coalesce(fl.DownloadDate, fl.OriginalDownloadDate, fl.TStamp, fl.CreationDate) as received_partner_at,
|
||||||
|
coalesce(fl.ID, cond.ID_FileLog, vui.ID_FileLog) as source_package_id_raw,
|
||||||
|
coalesce(fl.ID_VehicleIdentification, vui.ID_VehicleIdentification) as source_package_entity_id_raw,
|
||||||
|
coalesce(fl.DownloadFrom, cond.EntryTime) as source_package_period_from,
|
||||||
|
coalesce(fl.DownloadTo, cond.EntryTime) as source_package_period_to,
|
||||||
|
coalesce(fl.CreationDate, fl.TStamp) as source_package_imported_at
|
||||||
|
from dbo.VUSpecificConditions cond
|
||||||
|
join dbo.SpecificCondition sc on sc.ID = cond.ID_SpecificCondition
|
||||||
|
join dbo.VUInstallation vui on vui.ID = cond.ID_VUInstallation
|
||||||
|
join dbo.VehicleIdentification vi on vi.ID = vui.ID_VehicleIdentification
|
||||||
|
left join dbo.FileLog fl on fl.ID = coalesce(cond.ID_FileLog, vui.ID_FileLog)
|
||||||
|
where (:occurredFrom is null or cond.EntryTime >= :occurredFrom)
|
||||||
|
and (:occurredTo is null or cond.EntryTime < :occurredTo)
|
||||||
|
)
|
||||||
|
select
|
||||||
|
cast(base.ID as varchar(128)) as source_row_id,
|
||||||
|
concat('TACHOGRAPH:VU_SPECIFIC_CONDITION:', base.ID) as external_source_event_id,
|
||||||
|
|
||||||
|
base.occurred_at,
|
||||||
|
base.received_partner_at,
|
||||||
|
'SPECIFIC_CONDITION' as event_domain,
|
||||||
|
case when base.condition_code in (3, 4) then 'FERRY_TRAIN' else 'OUT' end as event_type,
|
||||||
|
case when base.condition_code in (1, 3) then 'BEGIN' else 'END' end as lifecycle,
|
||||||
|
|
||||||
|
cast(null as varchar(128)) as driver_source_entity_id,
|
||||||
|
cast(null as varchar(16)) as driver_card_nation,
|
||||||
|
cast(null as varchar(64)) as driver_card_number,
|
||||||
|
|
||||||
|
cast(base.vehicle_identification_id as varchar(128)) as vehicle_source_entity_id,
|
||||||
|
base.vehicle_vin,
|
||||||
|
cast(v.ID as varchar(128)) as vehicle_registration_source_entity_id,
|
||||||
|
vn.AlphaCode as vehicle_registration_nation,
|
||||||
|
v.VRN as vehicle_registration_number,
|
||||||
|
|
||||||
|
'VEHICLE_UNIT' as source_package_kind,
|
||||||
|
cast(base.source_package_id_raw as varchar(128)) as source_package_id,
|
||||||
|
cast(base.source_package_entity_id_raw as varchar(128)) as source_package_entity_id,
|
||||||
|
base.source_package_period_from,
|
||||||
|
base.source_package_period_to,
|
||||||
|
base.source_package_imported_at
|
||||||
|
from Base base
|
||||||
|
outer apply (
|
||||||
|
select top 1 vehicle.ID,
|
||||||
|
vehicle.VRN,
|
||||||
|
vehicle.ID_Nation
|
||||||
|
from dbo.Vehicle vehicle
|
||||||
|
where vehicle.ID_VehicleIdentification = base.vehicle_identification_id
|
||||||
|
and (vehicle.ValidFrom is null or vehicle.ValidFrom <= base.occurred_at)
|
||||||
|
and (vehicle.ValidTo is null or vehicle.ValidTo > base.occurred_at)
|
||||||
|
order by
|
||||||
|
vehicle.ValidFrom desc,
|
||||||
|
vehicle.ID desc
|
||||||
|
) v
|
||||||
|
left join dbo.Nation vn on vn.ID = v.ID_Nation
|
||||||
|
where (
|
||||||
|
:organisationId is null
|
||||||
|
or exists (
|
||||||
|
select 1
|
||||||
|
from dbo.Vehicle_I_90021 rel
|
||||||
|
join OrgTree on OrgTree.I_90021_OID = rel.ID_I_90021
|
||||||
|
where rel.ID_Vehicle = v.ID
|
||||||
|
and rel.GILT_BIS is null
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
@ -0,0 +1,155 @@
|
||||||
|
package at.procon.eventhub.tachograph.service;
|
||||||
|
|
||||||
|
import at.procon.eventhub.config.EventHubProperties;
|
||||||
|
import at.procon.eventhub.dto.EventFamily;
|
||||||
|
import at.procon.eventhub.dto.EventSourceDto;
|
||||||
|
import at.procon.eventhub.dto.ImportScopeDto;
|
||||||
|
import at.procon.eventhub.importing.ImportChunkPlanner;
|
||||||
|
import at.procon.eventhub.service.EventDetailsFactory;
|
||||||
|
import at.procon.eventhub.tachograph.dto.TachographImportRequest;
|
||||||
|
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||||
|
import java.util.EnumSet;
|
||||||
|
import java.util.List;
|
||||||
|
import org.junit.jupiter.api.Test;
|
||||||
|
|
||||||
|
import static org.assertj.core.api.Assertions.assertThat;
|
||||||
|
|
||||||
|
class TachographImportPlanServiceTest {
|
||||||
|
|
||||||
|
private final EventDetailsFactory detailsFactory = new EventDetailsFactory(new ObjectMapper());
|
||||||
|
private final TachographExtractionDefinitionRegistry definitionRegistry = new TachographExtractionDefinitionRegistry(
|
||||||
|
new CardActivityRowMapper(detailsFactory),
|
||||||
|
new VuActivityRowMapper(detailsFactory),
|
||||||
|
new IwCycleCardEventRowMapper(detailsFactory),
|
||||||
|
new CardVehiclesUsedCardEventRowMapper(detailsFactory),
|
||||||
|
new VuBorderCrossingRowMapper(detailsFactory),
|
||||||
|
new CardBorderCrossingRowMapper(detailsFactory),
|
||||||
|
new VuLoadUnloadRowMapper(detailsFactory),
|
||||||
|
new CardLoadUnloadRowMapper(detailsFactory),
|
||||||
|
new VuSpecificConditionRowMapper(detailsFactory),
|
||||||
|
new CardSpecificConditionRowMapper(detailsFactory),
|
||||||
|
new VuPositionRowMapper(detailsFactory),
|
||||||
|
new CardPositionRowMapper(detailsFactory),
|
||||||
|
new VuPlaceRowMapper(detailsFactory),
|
||||||
|
new CardPlaceRowMapper(detailsFactory),
|
||||||
|
new SpeedingEventRowMapper(detailsFactory)
|
||||||
|
);
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void allowsSupportedDriverActivityFamilyWhenJdbcExtractionIsEnabled() {
|
||||||
|
TachographImportPlanService service = serviceWithJdbcExtractor();
|
||||||
|
TachographImportRequest request = requestForFamilies(EventFamily.DRIVER_ACTIVITY);
|
||||||
|
|
||||||
|
var plan = service.createPlan(request);
|
||||||
|
|
||||||
|
assertThat(plan.items())
|
||||||
|
.extracting(item -> item.extractionCode())
|
||||||
|
.containsExactlyInAnyOrder("VU_ACTIVITY", "CARD_ACTIVITY");
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void allowsSupportedDriverCardFamilyWhenJdbcExtractionIsEnabled() {
|
||||||
|
TachographImportPlanService service = serviceWithJdbcExtractor();
|
||||||
|
TachographImportRequest request = requestForFamilies(EventFamily.DRIVER_CARD);
|
||||||
|
|
||||||
|
var plan = service.createPlan(request);
|
||||||
|
|
||||||
|
assertThat(plan.items())
|
||||||
|
.extracting(item -> item.extractionCode())
|
||||||
|
.containsExactlyInAnyOrder("IW_CYCLE", "CARD_VEHICLES_USED");
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void allowsSupportedBorderCrossingFamilyWhenJdbcExtractionIsEnabled() {
|
||||||
|
TachographImportPlanService service = serviceWithJdbcExtractor();
|
||||||
|
TachographImportRequest request = requestForFamilies(EventFamily.BORDER_CROSSING);
|
||||||
|
|
||||||
|
var plan = service.createPlan(request);
|
||||||
|
|
||||||
|
assertThat(plan.items())
|
||||||
|
.extracting(item -> item.extractionCode())
|
||||||
|
.containsExactlyInAnyOrder("VU_BORDER_CROSSING", "CARD_BORDER_CROSSING");
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void allowsSupportedLoadUnloadFamilyWhenJdbcExtractionIsEnabled() {
|
||||||
|
TachographImportPlanService service = serviceWithJdbcExtractor();
|
||||||
|
TachographImportRequest request = requestForFamilies(EventFamily.LOAD_UNLOAD);
|
||||||
|
|
||||||
|
var plan = service.createPlan(request);
|
||||||
|
|
||||||
|
assertThat(plan.items())
|
||||||
|
.extracting(item -> item.extractionCode())
|
||||||
|
.containsExactlyInAnyOrder("VU_LOAD_UNLOAD", "CARD_LOAD_UNLOAD");
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void allowsSupportedSpecificConditionFamilyWhenJdbcExtractionIsEnabled() {
|
||||||
|
TachographImportPlanService service = serviceWithJdbcExtractor();
|
||||||
|
TachographImportRequest request = requestForFamilies(EventFamily.SPECIFIC_CONDITION);
|
||||||
|
|
||||||
|
var plan = service.createPlan(request);
|
||||||
|
|
||||||
|
assertThat(plan.items())
|
||||||
|
.extracting(item -> item.extractionCode())
|
||||||
|
.containsExactlyInAnyOrder("VU_SPECIFIC_CONDITION", "CARD_SPECIFIC_CONDITION");
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void allowsSupportedPositionFamilyWhenJdbcExtractionIsEnabled() {
|
||||||
|
TachographImportPlanService service = serviceWithJdbcExtractor();
|
||||||
|
TachographImportRequest request = requestForFamilies(EventFamily.POSITION);
|
||||||
|
|
||||||
|
var plan = service.createPlan(request);
|
||||||
|
|
||||||
|
assertThat(plan.items())
|
||||||
|
.extracting(item -> item.extractionCode())
|
||||||
|
.containsExactlyInAnyOrder("VU_POSITION", "CARD_POSITION");
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void allowsSupportedPlaceFamilyWhenJdbcExtractionIsEnabled() {
|
||||||
|
TachographImportPlanService service = serviceWithJdbcExtractor();
|
||||||
|
TachographImportRequest request = requestForFamilies(EventFamily.PLACE);
|
||||||
|
|
||||||
|
var plan = service.createPlan(request);
|
||||||
|
|
||||||
|
assertThat(plan.items())
|
||||||
|
.extracting(item -> item.extractionCode())
|
||||||
|
.containsExactlyInAnyOrder("VU_PLACE", "CARD_PLACE");
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void allowsSupportedSpeedingFamilyWhenJdbcExtractionIsEnabled() {
|
||||||
|
TachographImportPlanService service = serviceWithJdbcExtractor();
|
||||||
|
TachographImportRequest request = requestForFamilies(EventFamily.SPEEDING);
|
||||||
|
|
||||||
|
var plan = service.createPlan(request);
|
||||||
|
|
||||||
|
assertThat(plan.items())
|
||||||
|
.extracting(item -> item.extractionCode())
|
||||||
|
.containsExactly("SPEEDING_EVENTS");
|
||||||
|
}
|
||||||
|
|
||||||
|
private TachographImportPlanService serviceWithJdbcExtractor() {
|
||||||
|
EventHubProperties properties = new EventHubProperties();
|
||||||
|
properties.getTachograph().getDatasource().setJdbcUrl("jdbc:sqlserver://tachograph-db");
|
||||||
|
return new TachographImportPlanService(properties, new ImportChunkPlanner(), definitionRegistry);
|
||||||
|
}
|
||||||
|
|
||||||
|
private TachographImportRequest requestForFamilies(EventFamily... families) {
|
||||||
|
EnumSet<EventFamily> selectedFamilies = families.length == 0
|
||||||
|
? EnumSet.noneOf(EventFamily.class)
|
||||||
|
: EnumSet.copyOf(List.of(families));
|
||||||
|
return new TachographImportRequest(
|
||||||
|
"tenant-1",
|
||||||
|
new EventSourceDto("TACHOGRAPH", "MIXED", "TACHOGRAPH_DB", "tachograph-db", null, null),
|
||||||
|
null,
|
||||||
|
ImportScopeDto.tenantAll(null, null),
|
||||||
|
selectedFamilies,
|
||||||
|
null,
|
||||||
|
false,
|
||||||
|
null
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
Loading…
Reference in New Issue