diff --git a/pom.xml b/pom.xml index bb81542..632827b 100644 --- a/pom.xml +++ b/pom.xml @@ -115,7 +115,7 @@ org.postgresql postgresql - runtime + compile com.pgvector @@ -192,6 +192,11 @@ commons-compress 1.27.1 + + org.apache.commons + commons-math3 + 3.6.1 + diff --git a/src/main/java/at/procon/dip/DocumentIntelligencePlatformApplication.java b/src/main/java/at/procon/dip/DocumentIntelligencePlatformApplication.java index f45bbce..1cc2bc8 100644 --- a/src/main/java/at/procon/dip/DocumentIntelligencePlatformApplication.java +++ b/src/main/java/at/procon/dip/DocumentIntelligencePlatformApplication.java @@ -16,8 +16,8 @@ import org.springframework.scheduling.annotation.EnableAsync; */ @SpringBootApplication(scanBasePackages = {"at.procon.dip", "at.procon.ted"}) @EnableAsync -@EntityScan(basePackages = {"at.procon.ted.model.entity", "at.procon.dip.domain.document.entity", "at.procon.dip.domain.tenant.entity", "at.procon.dip.domain.ted.entity", "at.procon.dip.embedding.job.entity", "at.procon.dip.migration.audit.entity", "at.procon.dip.migration.entity"}) -@EnableJpaRepositories(basePackages = {"at.procon.ted.repository", "at.procon.dip.domain.document.repository", "at.procon.dip.domain.tenant.repository", "at.procon.dip.domain.ted.repository", "at.procon.dip.embedding.job.repository", "at.procon.dip.migration.audit.repository", "at.procon.dip.migration.repository"}) +@EntityScan(basePackages = {"at.procon.ted.model.entity", "at.procon.dip.domain.document.entity", "at.procon.dip.domain.tenant.entity", "at.procon.dip.domain.ted.entity", "at.procon.dip.embedding.job.entity", "at.procon.dip.migration.audit.entity", "at.procon.dip.migration.entity", /*"at.procon.dip.domain.time.entity",*/ "at.procon.dip.clustering.entity"}) +@EnableJpaRepositories(basePackages = {"at.procon.ted.repository", "at.procon.dip.domain.document.repository", "at.procon.dip.domain.tenant.repository", "at.procon.dip.domain.ted.repository", "at.procon.dip.embedding.job.repository", "at.procon.dip.migration.audit.repository", "at.procon.dip.migration.repository", /*"at.procon.dip.domain.time.repository",*/ "at.procon.dip.clustering.repository"}) public class DocumentIntelligencePlatformApplication { public static void main(String[] args) { diff --git a/src/main/java/at/procon/dip/clustering/repository/DocumentEmbeddingClusterSelectionRepositoryImpl.java b/src/main/java/at/procon/dip/clustering/repository/DocumentEmbeddingClusterSelectionRepositoryImpl.java index 2ffe8a6..1119d53 100644 --- a/src/main/java/at/procon/dip/clustering/repository/DocumentEmbeddingClusterSelectionRepositoryImpl.java +++ b/src/main/java/at/procon/dip/clustering/repository/DocumentEmbeddingClusterSelectionRepositoryImpl.java @@ -46,6 +46,7 @@ public class DocumentEmbeddingClusterSelectionRepositoryImpl implements Document where e.embedding_status = 'COMPLETED' and e.embedding_vector is not null and e.prefix_profile_id is not null + and d. """); MapSqlParameterSource params = new MapSqlParameterSource(); applyFilters(spec, sql, params); diff --git a/src/main/java/at/procon/dip/domain/time/config/TimeDomainProperties.java b/src/main/java/at/procon/dip/domain/time/config/TimeDomainProperties.java index 37c746f..27cf14c 100644 --- a/src/main/java/at/procon/dip/domain/time/config/TimeDomainProperties.java +++ b/src/main/java/at/procon/dip/domain/time/config/TimeDomainProperties.java @@ -29,6 +29,10 @@ public class TimeDomainProperties { private boolean buildRepresentations = true; private boolean queueEmbeddings = true; private boolean startupProjectionRebuildEnabled = false; + private boolean startupSelectiveMaterializationEnabled = false; + private String selectiveMaterializationPersonDbk; + private Integer selectiveMaterializationPersonNumber; + private boolean selectiveMaterializationBuildProjection = true; private String representationLanguageCode = "de"; private String scopeKey = "leitstand-default"; private JdbcProperties jdbc = new JdbcProperties(); diff --git a/src/main/java/at/procon/dip/domain/time/entity/TimeEntry.java b/src/main/java/at/procon/dip/domain/time/entity/TimeEntry.java index ba08789..9ca1532 100644 --- a/src/main/java/at/procon/dip/domain/time/entity/TimeEntry.java +++ b/src/main/java/at/procon/dip/domain/time/entity/TimeEntry.java @@ -75,7 +75,7 @@ public class TimeEntry { @Column(name = "duration_seconds") private Long durationSeconds; - @Column(name = "description_short", length = 1000) + @Column(name = "description_short", columnDefinition = "TEXT") private String descriptionShort; @Column(name = "description_long", columnDefinition = "TEXT") @@ -96,7 +96,7 @@ public class TimeEntry { @Column(name = "raw_status", length = 120) private String rawStatus; - @Column(name = "search_anchor_label", length = 500) + @Column(name = "search_anchor_label", columnDefinition = "TEXT") private String searchAnchorLabel; @Builder.Default diff --git a/src/main/java/at/procon/dip/domain/time/leitstand/config/LeitstandTimeImportConfiguration.java b/src/main/java/at/procon/dip/domain/time/leitstand/config/LeitstandTimeImportConfiguration.java index 4159143..2ac852a 100644 --- a/src/main/java/at/procon/dip/domain/time/leitstand/config/LeitstandTimeImportConfiguration.java +++ b/src/main/java/at/procon/dip/domain/time/leitstand/config/LeitstandTimeImportConfiguration.java @@ -8,6 +8,7 @@ import lombok.RequiredArgsConstructor; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Primary; import org.springframework.jdbc.core.JdbcTemplate; import org.springframework.jdbc.core.namedparam.NamedParameterJdbcTemplate; import org.springframework.jdbc.datasource.DriverManagerDataSource; @@ -31,11 +32,18 @@ public class LeitstandTimeImportConfiguration { return ds; } + @Primary @Bean(name = "applicationJdbcTemplate") public JdbcTemplate applicationJdbcTemplate(DataSource dataSource) { return new JdbcTemplate(dataSource); } + @Primary + @Bean(name = "applicationNamedParameterJdbcTemplate") + public NamedParameterJdbcTemplate applicationNamedParameterJdbcTemplate(DataSource dataSource) { + return new NamedParameterJdbcTemplate(dataSource); + } + @Bean(name = "leitstandTimeNamedParameterJdbcTemplate") public NamedParameterJdbcTemplate leitstandNamedParameterJdbcTemplate() { return new NamedParameterJdbcTemplate(createLeitstandDataSource()); diff --git a/src/main/java/at/procon/dip/domain/time/repository/leitstand/LeitstandPersonRepository.java b/src/main/java/at/procon/dip/domain/time/repository/leitstand/LeitstandPersonRepository.java index 239de46..62bc38e 100644 --- a/src/main/java/at/procon/dip/domain/time/repository/leitstand/LeitstandPersonRepository.java +++ b/src/main/java/at/procon/dip/domain/time/repository/leitstand/LeitstandPersonRepository.java @@ -1,6 +1,10 @@ package at.procon.dip.domain.time.repository.leitstand; import at.procon.dip.domain.time.entity.leitstand.LeitstandPerson; +import java.util.Optional; import org.springframework.data.jpa.repository.JpaRepository; -public interface LeitstandPersonRepository extends JpaRepository {} +public interface LeitstandPersonRepository extends JpaRepository { + + Optional findByPersonNumber(Integer personNumber); +} diff --git a/src/main/java/at/procon/dip/domain/time/repository/leitstand/LeitstandTimeRecordingRepository.java b/src/main/java/at/procon/dip/domain/time/repository/leitstand/LeitstandTimeRecordingRepository.java index 2ec0463..46db2a1 100644 --- a/src/main/java/at/procon/dip/domain/time/repository/leitstand/LeitstandTimeRecordingRepository.java +++ b/src/main/java/at/procon/dip/domain/time/repository/leitstand/LeitstandTimeRecordingRepository.java @@ -11,4 +11,8 @@ public interface LeitstandTimeRecordingRepository extends JpaRepository findByTimeEntry_Id(UUID timeEntryId); List findByTimeEntryIsNotNull(); + + List findByPersonDbkOrderByRecordedFromAscDbkAsc(String personDbk); + + List findByPersonDbkAndTimeEntryIsNotNullOrderByRecordedFromAscDbkAsc(String personDbk); } diff --git a/src/main/java/at/procon/dip/domain/time/service/LeitstandTimeImportService.java b/src/main/java/at/procon/dip/domain/time/service/LeitstandTimeImportService.java index 27d45c3..4a8fde4 100644 --- a/src/main/java/at/procon/dip/domain/time/service/LeitstandTimeImportService.java +++ b/src/main/java/at/procon/dip/domain/time/service/LeitstandTimeImportService.java @@ -8,12 +8,16 @@ import at.procon.dip.domain.document.entity.Document; import at.procon.dip.domain.document.repository.DocumentRepository; import at.procon.dip.domain.time.config.TimeDomainProperties; import at.procon.dip.domain.time.entity.*; +import at.procon.dip.domain.time.entity.leitstand.LeitstandPerson; +import at.procon.dip.domain.time.entity.leitstand.LeitstandTimeRecording; import at.procon.dip.domain.time.leitstand.source.LeitstandSourceRows; import at.procon.dip.domain.time.leitstand.source.LeitstandTimeSourceClient; import at.procon.dip.domain.time.repository.TimeEntryRepository; import at.procon.dip.domain.time.repository.TimeEntrySourceLinkRepository; import at.procon.dip.domain.time.repository.TimeSyncRunRepository; import at.procon.dip.domain.time.repository.TimeSyncStateRepository; +import at.procon.dip.domain.time.repository.leitstand.LeitstandPersonRepository; +import at.procon.dip.domain.time.repository.leitstand.LeitstandTimeRecordingRepository; import at.procon.dip.runtime.condition.ConditionalOnRuntimeMode; import at.procon.dip.runtime.config.RuntimeMode; import java.math.BigDecimal; @@ -26,16 +30,17 @@ import java.util.List; import java.util.Map; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.jdbc.core.BatchPreparedStatementSetter; import org.springframework.jdbc.core.JdbcTemplate; import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Propagation; import org.springframework.transaction.annotation.Transactional; @Service -@ConditionalOnRuntimeMode(RuntimeMode.NEW) +//@ConditionalOnRuntimeMode(RuntimeMode.NEW) @ConditionalOnProperty(prefix = "dip.time.leitstand", name = "enabled", havingValue = "true") +@RequiredArgsConstructor @Slf4j public class LeitstandTimeImportService { @@ -49,30 +54,10 @@ public class LeitstandTimeImportService { private final DocumentRepository documentRepository; private final TimeEntryRepository timeEntryRepository; private final TimeEntrySourceLinkRepository sourceLinkRepository; + private final LeitstandTimeRecordingRepository timeRecordingRepository; + private final LeitstandPersonRepository personRepository; private final LeitstandTimeProjectionService projectionService; - public LeitstandTimeImportService( - @Qualifier("applicationJdbcTemplate") JdbcTemplate targetJdbcTemplate, - TimeDomainProperties properties, - LeitstandTimeSourceClient sourceClient, - TimeSyncRunRepository syncRunRepository, - TimeSyncStateRepository syncStateRepository, - DocumentRepository documentRepository, - TimeEntryRepository timeEntryRepository, - TimeEntrySourceLinkRepository sourceLinkRepository, - LeitstandTimeProjectionService projectionService - ) { - this. properties = properties; - this.jdbcTemplate = targetJdbcTemplate; - this.sourceClient = sourceClient; - this.syncRunRepository = syncRunRepository; - this.syncStateRepository = syncStateRepository; - this.documentRepository = documentRepository; - this.timeEntryRepository = timeEntryRepository; - this.sourceLinkRepository = sourceLinkRepository; - this.projectionService = projectionService; - } - public void runSync() { TimeSyncRun run = syncRunRepository.save(TimeSyncRun.builder() .sourceSystem(TimeSourceSystem.LEITSTAND) @@ -116,7 +101,7 @@ public class LeitstandTimeImportService { List rows = sourceClient.fetchActivityTypes(); run.setRowsRead(run.getRowsRead() + rows.size()); jdbcTemplate.batchUpdate(""" - INSERT INTO "time".ls_activity_type(id, l_code, bez) + INSERT INTO TIME.ls_activity_type(id, l_code, bez) VALUES (?, ?, ?) ON CONFLICT (id) DO UPDATE SET l_code = EXCLUDED.l_code, bez = EXCLUDED.bez """, new BatchPreparedStatementSetter() { @@ -134,39 +119,90 @@ public class LeitstandTimeImportService { List rows = sourceClient.fetchPersons(watermark); run.setRowsRead(run.getRowsRead() + rows.size()); batchUpsert(""" - INSERT INTO "time".ls_person(dbk, dbk_uid, upd, upd_anz, upd_uid, status, pers_nr, last_name, first_name, category_dbk, salutation, title, street, postal_code, city, country_code, phone, fax, email, cost_per_hour, organization_dbk) + INSERT INTO TIME.ls_person(dbk, dbk_uid, upd, upd_anz, upd_uid, status, pers_nr, last_name, first_name, category_dbk, salutation, title, street, postal_code, city, country_code, phone, fax, email, cost_per_hour, organization_dbk) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT (dbk) DO UPDATE SET dbk_uid=EXCLUDED.dbk_uid, upd=EXCLUDED.upd, upd_anz=EXCLUDED.upd_anz, upd_uid=EXCLUDED.upd_uid, status=EXCLUDED.status, pers_nr=EXCLUDED.pers_nr, last_name=EXCLUDED.last_name, first_name=EXCLUDED.first_name, category_dbk=EXCLUDED.category_dbk, salutation=EXCLUDED.salutation, title=EXCLUDED.title, street=EXCLUDED.street, postal_code=EXCLUDED.postal_code, city=EXCLUDED.city, country_code=EXCLUDED.country_code, phone=EXCLUDED.phone, fax=EXCLUDED.fax, email=EXCLUDED.email, cost_per_hour=EXCLUDED.cost_per_hour, organization_dbk=EXCLUDED.organization_dbk, updated_at=CURRENT_TIMESTAMP """, rows, (ps, r) -> { set(ps,1,r.dbk()); set(ps,2,r.dbkUid()); set(ps,3,r.upd()); set(ps,4,r.updAnz()); set(ps,5,r.updUid()); set(ps,6,r.status()); set(ps,7,r.personNumber()); set(ps,8,r.lastName()); set(ps,9,r.firstName()); set(ps,10,r.categoryDbk()); set(ps,11,r.salutation()); set(ps,12,r.title()); set(ps,13,r.street()); set(ps,14,r.postalCode()); set(ps,15,r.city()); set(ps,16,r.countryCode()); set(ps,17,r.phone()); set(ps,18,r.fax()); set(ps,19,r.email()); set(ps,20,r.costPerHour()); set(ps,21,r.organizationDbk());}); updateTableState("person", maxUpd(rows, LeitstandSourceRows.PersonRow::upd), lastId(rows, LeitstandSourceRows.PersonRow::dbk)); } - private void syncOrganizations(TimeSyncRun run) { var rows = sourceClient.fetchOrganizations(watermark("organization")); run.setRowsRead(run.getRowsRead()+rows.size()); batchUpsert(""" - INSERT INTO "time".ls_organization(dbk, dbk_uid, upd, upd_anz, upd_uid, status, betreu, name, street, postal_code, city, phone, fax, email, org_nr, short_name) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT (dbk) DO UPDATE SET dbk_uid=EXCLUDED.dbk_uid, upd=EXCLUDED.upd, upd_anz=EXCLUDED.upd_anz, upd_uid=EXCLUDED.upd_uid, status=EXCLUDED.status, betreu=EXCLUDED.betreu, name=EXCLUDED.name, street=EXCLUDED.street, postal_code=EXCLUDED.postal_code, city=EXCLUDED.city, phone=EXCLUDED.phone, fax=EXCLUDED.fax, email=EXCLUDED.email, org_nr=EXCLUDED.org_nr, short_name=EXCLUDED.short_name, updated_at=CURRENT_TIMESTAMP - """, rows, (ps,r)->{set(ps,1,r.dbk());set(ps,2,r.dbkUid());set(ps,3,r.upd());set(ps,4,r.updAnz());set(ps,5,r.updUid());set(ps,6,r.status());set(ps,7,r.betreu());set(ps,8,r.name());set(ps,9,r.street());set(ps,10,r.postalCode());set(ps,11,r.city());set(ps,12,r.phone());set(ps,13,r.fax());set(ps,14,r.email());set(ps,15,r.orgNumber());set(ps,16,r.shortName());}); updateTableState("organization", maxUpd(rows, LeitstandSourceRows.OrganizationRow::upd), lastId(rows, LeitstandSourceRows.OrganizationRow::dbk)); } - private void syncContracts(TimeSyncRun run) { var rows = sourceClient.fetchContracts(watermark("contract")); run.setRowsRead(run.getRowsRead()+rows.size()); batchUpsert(""" - INSERT INTO "time".ls_contract(dbk, dbk_uid, upd, upd_anz, upd_uid, status, organization_dbk, lfdnr, name, iref, eref, description, valid_from, valid_to) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT (dbk) DO UPDATE SET dbk_uid=EXCLUDED.dbk_uid, upd=EXCLUDED.upd, upd_anz=EXCLUDED.upd_anz, upd_uid=EXCLUDED.upd_uid, status=EXCLUDED.status, organization_dbk=EXCLUDED.organization_dbk, lfdnr=EXCLUDED.lfdnr, name=EXCLUDED.name, iref=EXCLUDED.iref, eref=EXCLUDED.eref, description=EXCLUDED.description, valid_from=EXCLUDED.valid_from, valid_to=EXCLUDED.valid_to, updated_at=CURRENT_TIMESTAMP - """, rows, (ps,r)->{set(ps,1,r.dbk());set(ps,2,r.dbkUid());set(ps,3,r.upd());set(ps,4,r.updAnz());set(ps,5,r.updUid());set(ps,6,r.status());set(ps,7,r.organizationDbk());set(ps,8,r.lfdnr());set(ps,9,r.name());set(ps,10,r.iref());set(ps,11,r.eref());set(ps,12,r.description());set(ps,13,ts(r.validFrom()));set(ps,14,ts(r.validTo()));}); updateTableState("contract", maxUpd(rows, LeitstandSourceRows.ContractRow::upd), lastId(rows, LeitstandSourceRows.ContractRow::dbk)); } - private void syncContractPositions(TimeSyncRun run) { var rows = sourceClient.fetchContractPositions(watermark("contract-position")); run.setRowsRead(run.getRowsRead()+rows.size()); batchUpsert(""" - INSERT INTO "time".ls_contract_position(dbk, dbk_uid, upd, upd_anz, upd_uid, status, contract_dbk, lfdnr, project_name, sales_price, purchase_price, description, valid_from, valid_to, name, iref, eref) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT (dbk) DO UPDATE SET dbk_uid=EXCLUDED.dbk_uid, upd=EXCLUDED.upd, upd_anz=EXCLUDED.upd_anz, upd_uid=EXCLUDED.upd_uid, status=EXCLUDED.status, contract_dbk=EXCLUDED.contract_dbk, lfdnr=EXCLUDED.lfdnr, project_name=EXCLUDED.project_name, sales_price=EXCLUDED.sales_price, purchase_price=EXCLUDED.purchase_price, description=EXCLUDED.description, valid_from=EXCLUDED.valid_from, valid_to=EXCLUDED.valid_to, name=EXCLUDED.name, iref=EXCLUDED.iref, eref=EXCLUDED.eref, updated_at=CURRENT_TIMESTAMP - """, rows, (ps,r)->{set(ps,1,r.dbk());set(ps,2,r.dbkUid());set(ps,3,r.upd());set(ps,4,r.updAnz());set(ps,5,r.updUid());set(ps,6,r.status());set(ps,7,r.contractDbk());set(ps,8,r.lfdnr());set(ps,9,r.projectName());set(ps,10,r.salesPrice());set(ps,11,r.purchasePrice());set(ps,12,r.description());set(ps,13,ts(r.validFrom()));set(ps,14,ts(r.validTo()));set(ps,15,r.name());set(ps,16,r.iref());set(ps,17,r.eref());}); updateTableState("contract-position", maxUpd(rows, LeitstandSourceRows.ContractPositionRow::upd), lastId(rows, LeitstandSourceRows.ContractPositionRow::dbk)); } - private void syncCostUnits(TimeSyncRun run) { var rows = sourceClient.fetchCostUnits(watermark("cost-unit")); run.setRowsRead(run.getRowsRead()+rows.size()); batchUpsert(""" - INSERT INTO "time".ls_cost_unit(dbk, dbk_uid, upd, upd_anz, upd_uid, status, person_dbk, organization_dbk, contract_dbk, legacy_relation_4_dbk, legacy_relation_5_dbk, contract_position_dbk, project_name, project_id, project_task, mcl_id, mcl_name, mcl_desc, valid_from, valid_to, effort_plan) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT (dbk) DO UPDATE SET dbk_uid=EXCLUDED.dbk_uid, upd=EXCLUDED.upd, upd_anz=EXCLUDED.upd_anz, upd_uid=EXCLUDED.upd_uid, status=EXCLUDED.status, person_dbk=EXCLUDED.person_dbk, organization_dbk=EXCLUDED.organization_dbk, contract_dbk=EXCLUDED.contract_dbk, legacy_relation_4_dbk=EXCLUDED.legacy_relation_4_dbk, legacy_relation_5_dbk=EXCLUDED.legacy_relation_5_dbk, contract_position_dbk=EXCLUDED.contract_position_dbk, project_name=EXCLUDED.project_name, project_id=EXCLUDED.project_id, project_task=EXCLUDED.project_task, mcl_id=EXCLUDED.mcl_id, mcl_name=EXCLUDED.mcl_name, mcl_desc=EXCLUDED.mcl_desc, valid_from=EXCLUDED.valid_from, valid_to=EXCLUDED.valid_to, effort_plan=EXCLUDED.effort_plan, updated_at=CURRENT_TIMESTAMP - """, rows, (ps,r)->{set(ps,1,r.dbk());set(ps,2,r.dbkUid());set(ps,3,r.upd());set(ps,4,r.updAnz());set(ps,5,r.updUid());set(ps,6,r.status());set(ps,7,r.personDbk());set(ps,8,r.organizationDbk());set(ps,9,r.contractDbk());set(ps,10,r.legacyRelation4Dbk());set(ps,11,r.legacyRelation5Dbk());set(ps,12,r.contractPositionDbk());set(ps,13,r.projectName());set(ps,14,r.projectId());set(ps,15,r.projectTask());set(ps,16,r.mclId());set(ps,17,r.mclName());set(ps,18,r.mclDesc());set(ps,19,ts(r.validFrom()));set(ps,20,ts(r.validTo()));set(ps,21,r.effortPlan());}); updateTableState("cost-unit", maxUpd(rows, LeitstandSourceRows.CostUnitRow::upd), lastId(rows, LeitstandSourceRows.CostUnitRow::dbk)); } - private void syncTasks(TimeSyncRun run) { var rows = sourceClient.fetchTasks(watermark("task")); run.setRowsRead(run.getRowsRead()+rows.size()); batchUpsert(""" - INSERT INTO "time".ls_task(dbk, dbk_uid, upd, upd_anz, upd_uid, status, primary_cost_unit_dbk, secondary_cost_unit_dbk, tertiary_cost_unit_dbk, parent_task_dbk, lfdnr, task_type, mcl_id, mcl_name, mcl_desc, valid_from, valid_to, effort_plan, last_work, completion_percent, done_date, remark, n_rid, legacy_90702_rid, amount, created_source_at, legacy_90700_rid, legacy_90699_rid, quantity) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT (dbk) DO UPDATE SET dbk_uid=EXCLUDED.dbk_uid, upd=EXCLUDED.upd, upd_anz=EXCLUDED.upd_anz, upd_uid=EXCLUDED.upd_uid, status=EXCLUDED.status, primary_cost_unit_dbk=EXCLUDED.primary_cost_unit_dbk, secondary_cost_unit_dbk=EXCLUDED.secondary_cost_unit_dbk, tertiary_cost_unit_dbk=EXCLUDED.tertiary_cost_unit_dbk, parent_task_dbk=EXCLUDED.parent_task_dbk, lfdnr=EXCLUDED.lfdnr, task_type=EXCLUDED.task_type, mcl_id=EXCLUDED.mcl_id, mcl_name=EXCLUDED.mcl_name, mcl_desc=EXCLUDED.mcl_desc, valid_from=EXCLUDED.valid_from, valid_to=EXCLUDED.valid_to, effort_plan=EXCLUDED.effort_plan, last_work=EXCLUDED.last_work, completion_percent=EXCLUDED.completion_percent, done_date=EXCLUDED.done_date, remark=EXCLUDED.remark, n_rid=EXCLUDED.n_rid, legacy_90702_rid=EXCLUDED.legacy_90702_rid, amount=EXCLUDED.amount, created_source_at=EXCLUDED.created_source_at, legacy_90700_rid=EXCLUDED.legacy_90700_rid, legacy_90699_rid=EXCLUDED.legacy_90699_rid, quantity=EXCLUDED.quantity, updated_at=CURRENT_TIMESTAMP - """, rows, (ps,r)->{set(ps,1,r.dbk());set(ps,2,r.dbkUid());set(ps,3,r.upd());set(ps,4,r.updAnz());set(ps,5,r.updUid());set(ps,6,r.status());set(ps,7,r.primaryCostUnitDbk());set(ps,8,r.secondaryCostUnitDbk());set(ps,9,r.tertiaryCostUnitDbk());set(ps,10,r.parentTaskDbk());set(ps,11,r.lfdnr());set(ps,12,r.taskType());set(ps,13,r.mclId());set(ps,14,r.mclName());set(ps,15,r.mclDesc());set(ps,16,ts(r.validFrom()));set(ps,17,ts(r.validTo()));set(ps,18,r.effortPlan());set(ps,19,ts(r.lastWork()));set(ps,20,r.completionPercent());set(ps,21,ts(r.doneDate()));set(ps,22,r.remark());set(ps,23,r.nRid());set(ps,24,r.legacy90702Rid());set(ps,25,r.amount());set(ps,26,ts(r.createdSourceAt()));set(ps,27,r.legacy90700Rid());set(ps,28,r.legacy90699Rid());set(ps,29,r.quantity());}); updateTableState("task", maxUpd(rows, LeitstandSourceRows.TaskRow::upd), lastId(rows, LeitstandSourceRows.TaskRow::dbk)); } - private void syncPersonTaskAssignments(TimeSyncRun run) { var rows = sourceClient.fetchPersonTaskAssignments(watermark("person-task-assignment")); run.setRowsRead(run.getRowsRead()+rows.size()); batchUpsert(""" - INSERT INTO "time".ls_person_task_assignment(dbk, dbk_uid, upd, upd_anz, upd_uid, status, task_dbk, person_dbk, cost_unit_dbk, rtype, last_work, effort_plan) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT (dbk) DO UPDATE SET dbk_uid=EXCLUDED.dbk_uid, upd=EXCLUDED.upd, upd_anz=EXCLUDED.upd_anz, upd_uid=EXCLUDED.upd_uid, status=EXCLUDED.status, task_dbk=EXCLUDED.task_dbk, person_dbk=EXCLUDED.person_dbk, cost_unit_dbk=EXCLUDED.cost_unit_dbk, rtype=EXCLUDED.rtype, last_work=EXCLUDED.last_work, effort_plan=EXCLUDED.effort_plan, updated_at=CURRENT_TIMESTAMP - """, rows, (ps,r)->{set(ps,1,r.dbk());set(ps,2,r.dbkUid());set(ps,3,r.upd());set(ps,4,r.updAnz());set(ps,5,r.updUid());set(ps,6,r.status());set(ps,7,r.taskDbk());set(ps,8,r.personDbk());set(ps,9,r.costUnitDbk());set(ps,10,r.rtype());set(ps,11,ts(r.lastWork()));set(ps,12,r.effortPlan());}); updateTableState("person-task-assignment", maxUpd(rows, LeitstandSourceRows.PersonTaskAssignmentRow::upd), lastId(rows, LeitstandSourceRows.PersonTaskAssignmentRow::dbk)); } - private List syncTimeRecordings(TimeSyncRun run) { var rows = sourceClient.fetchTimeRecordings(watermark("time-recording")); run.setRowsRead(run.getRowsRead()+rows.size()); batchUpsert(""" - INSERT INTO "time".ls_time_recording(dbk, dbk_uid, upd, upd_anz, upd_uid, status, person_dbk, lfdnr, record_type, mcl_id, mcl_desc, recorded_from, recorded_to, effort, remark, url, activity_type_id) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT (dbk) DO UPDATE SET dbk_uid=EXCLUDED.dbk_uid, upd=EXCLUDED.upd, upd_anz=EXCLUDED.upd_anz, upd_uid=EXCLUDED.upd_uid, status=EXCLUDED.status, person_dbk=EXCLUDED.person_dbk, lfdnr=EXCLUDED.lfdnr, record_type=EXCLUDED.record_type, mcl_id=EXCLUDED.mcl_id, mcl_desc=EXCLUDED.mcl_desc, recorded_from=EXCLUDED.recorded_from, recorded_to=EXCLUDED.recorded_to, effort=EXCLUDED.effort, remark=EXCLUDED.remark, url=EXCLUDED.url, activity_type_id=EXCLUDED.activity_type_id, updated_at=CURRENT_TIMESTAMP - """, rows, (ps,r)->{set(ps,1,r.dbk());set(ps,2,r.dbkUid());set(ps,3,r.upd());set(ps,4,r.updAnz());set(ps,5,r.updUid());set(ps,6,r.status());set(ps,7,r.personDbk());set(ps,8,r.lfdnr());set(ps,9,r.recordType());set(ps,10,r.mclId());set(ps,11,r.mclDesc());set(ps,12,ts(r.recordedFrom()));set(ps,13,ts(r.recordedTo()));set(ps,14,r.effort());set(ps,15,r.remark());set(ps,16,r.url());set(ps,17,r.activityTypeId());}); updateTableState("time-recording", maxUpd(rows, LeitstandSourceRows.TimeRecordingRow::upd), lastId(rows, LeitstandSourceRows.TimeRecordingRow::dbk)); return rows; } - private void syncTimeRecordingAssignments(TimeSyncRun run) { var rows = sourceClient.fetchTimeRecordingAssignments(watermark("time-recording-assignment")); run.setRowsRead(run.getRowsRead()+rows.size()); batchUpsert(""" - INSERT INTO "time".ls_time_recording_assignment(dbk, dbk_uid, upd, upd_anz, upd_uid, status, person_task_assignment_dbk, time_recording_dbk, mcl_desc, effort, remark) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT (dbk) DO UPDATE SET dbk_uid=EXCLUDED.dbk_uid, upd=EXCLUDED.upd, upd_anz=EXCLUDED.upd_anz, upd_uid=EXCLUDED.upd_uid, status=EXCLUDED.status, person_task_assignment_dbk=EXCLUDED.person_task_assignment_dbk, time_recording_dbk=EXCLUDED.time_recording_dbk, mcl_desc=EXCLUDED.mcl_desc, effort=EXCLUDED.effort, remark=EXCLUDED.remark, updated_at=CURRENT_TIMESTAMP - """, rows, (ps,r)->{set(ps,1,r.dbk());set(ps,2,r.dbkUid());set(ps,3,r.upd());set(ps,4,r.updAnz());set(ps,5,r.updUid());set(ps,6,r.status());set(ps,7,r.personTaskAssignmentDbk());set(ps,8,r.timeRecordingDbk());set(ps,9,r.mclDesc());set(ps,10,r.effort());set(ps,11,r.remark());}); updateTableState("time-recording-assignment", maxUpd(rows, LeitstandSourceRows.TimeRecordingAssignmentRow::upd), lastId(rows, LeitstandSourceRows.TimeRecordingAssignmentRow::dbk)); } + private void syncOrganizations(TimeSyncRun run) { var rows = sourceClient.fetchOrganizations(watermark("organization")); run.setRowsRead(run.getRowsRead()+rows.size()); batchUpsert("INSERT INTO TIME.ls_organization(dbk, dbk_uid, upd, upd_anz, upd_uid, status, betreu, name, street, postal_code, city, phone, fax, email, org_nr, short_name) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT (dbk) DO UPDATE SET dbk_uid=EXCLUDED.dbk_uid, upd=EXCLUDED.upd, upd_anz=EXCLUDED.upd_anz, upd_uid=EXCLUDED.upd_uid, status=EXCLUDED.status, betreu=EXCLUDED.betreu, name=EXCLUDED.name, street=EXCLUDED.street, postal_code=EXCLUDED.postal_code, city=EXCLUDED.city, phone=EXCLUDED.phone, fax=EXCLUDED.fax, email=EXCLUDED.email, org_nr=EXCLUDED.org_nr, short_name=EXCLUDED.short_name, updated_at=CURRENT_TIMESTAMP", rows, (ps,r)->{set(ps,1,r.dbk());set(ps,2,r.dbkUid());set(ps,3,r.upd());set(ps,4,r.updAnz());set(ps,5,r.updUid());set(ps,6,r.status());set(ps,7,r.betreu());set(ps,8,r.name());set(ps,9,r.street());set(ps,10,r.postalCode());set(ps,11,r.city());set(ps,12,r.phone());set(ps,13,r.fax());set(ps,14,r.email());set(ps,15,r.orgNumber());set(ps,16,r.shortName());}); updateTableState("organization", maxUpd(rows, LeitstandSourceRows.OrganizationRow::upd), lastId(rows, LeitstandSourceRows.OrganizationRow::dbk)); } + private void syncContracts(TimeSyncRun run) { var rows = sourceClient.fetchContracts(watermark("contract")); run.setRowsRead(run.getRowsRead()+rows.size()); batchUpsert("INSERT INTO TIME.ls_contract(dbk, dbk_uid, upd, upd_anz, upd_uid, status, organization_dbk, lfdnr, name, iref, eref, description, valid_from, valid_to) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT (dbk) DO UPDATE SET dbk_uid=EXCLUDED.dbk_uid, upd=EXCLUDED.upd, upd_anz=EXCLUDED.upd_anz, upd_uid=EXCLUDED.upd_uid, status=EXCLUDED.status, organization_dbk=EXCLUDED.organization_dbk, lfdnr=EXCLUDED.lfdnr, name=EXCLUDED.name, iref=EXCLUDED.iref, eref=EXCLUDED.eref, description=EXCLUDED.description, valid_from=EXCLUDED.valid_from, valid_to=EXCLUDED.valid_to, updated_at=CURRENT_TIMESTAMP", rows, (ps,r)->{set(ps,1,r.dbk());set(ps,2,r.dbkUid());set(ps,3,r.upd());set(ps,4,r.updAnz());set(ps,5,r.updUid());set(ps,6,r.status());set(ps,7,r.organizationDbk());set(ps,8,r.lfdnr());set(ps,9,r.name());set(ps,10,r.iref());set(ps,11,r.eref());set(ps,12,r.description());set(ps,13,ts(r.validFrom()));set(ps,14,ts(r.validTo()));}); updateTableState("contract", maxUpd(rows, LeitstandSourceRows.ContractRow::upd), lastId(rows, LeitstandSourceRows.ContractRow::dbk)); } + private void syncContractPositions(TimeSyncRun run) { var rows = sourceClient.fetchContractPositions(watermark("contract-position")); run.setRowsRead(run.getRowsRead()+rows.size()); batchUpsert("INSERT INTO TIME.ls_contract_position(dbk, dbk_uid, upd, upd_anz, upd_uid, status, contract_dbk, lfdnr, project_name, sales_price, purchase_price, description, valid_from, valid_to, name, iref, eref) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT (dbk) DO UPDATE SET dbk_uid=EXCLUDED.dbk_uid, upd=EXCLUDED.upd, upd_anz=EXCLUDED.upd_anz, upd_uid=EXCLUDED.upd_uid, status=EXCLUDED.status, contract_dbk=EXCLUDED.contract_dbk, lfdnr=EXCLUDED.lfdnr, project_name=EXCLUDED.project_name, sales_price=EXCLUDED.sales_price, purchase_price=EXCLUDED.purchase_price, description=EXCLUDED.description, valid_from=EXCLUDED.valid_from, valid_to=EXCLUDED.valid_to, name=EXCLUDED.name, iref=EXCLUDED.iref, eref=EXCLUDED.eref, updated_at=CURRENT_TIMESTAMP", rows, (ps,r)->{set(ps,1,r.dbk());set(ps,2,r.dbkUid());set(ps,3,r.upd());set(ps,4,r.updAnz());set(ps,5,r.updUid());set(ps,6,r.status());set(ps,7,r.contractDbk());set(ps,8,r.lfdnr());set(ps,9,r.projectName());set(ps,10,r.salesPrice());set(ps,11,r.purchasePrice());set(ps,12,r.description());set(ps,13,ts(r.validFrom()));set(ps,14,ts(r.validTo()));set(ps,15,r.name());set(ps,16,r.iref());set(ps,17,r.eref());}); updateTableState("contract-position", maxUpd(rows, LeitstandSourceRows.ContractPositionRow::upd), lastId(rows, LeitstandSourceRows.ContractPositionRow::dbk)); } + private void syncCostUnits(TimeSyncRun run) { var rows = sourceClient.fetchCostUnits(watermark("cost-unit")); run.setRowsRead(run.getRowsRead()+rows.size()); batchUpsert("INSERT INTO TIME.ls_cost_unit(dbk, dbk_uid, upd, upd_anz, upd_uid, status, person_dbk, organization_dbk, contract_dbk, legacy_relation_4_dbk, legacy_relation_5_dbk, contract_position_dbk, project_name, project_id, project_task, mcl_id, mcl_name, mcl_desc, valid_from, valid_to, effort_plan) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT (dbk) DO UPDATE SET dbk_uid=EXCLUDED.dbk_uid, upd=EXCLUDED.upd, upd_anz=EXCLUDED.upd_anz, upd_uid=EXCLUDED.upd_uid, status=EXCLUDED.status, person_dbk=EXCLUDED.person_dbk, organization_dbk=EXCLUDED.organization_dbk, contract_dbk=EXCLUDED.contract_dbk, legacy_relation_4_dbk=EXCLUDED.legacy_relation_4_dbk, legacy_relation_5_dbk=EXCLUDED.legacy_relation_5_dbk, contract_position_dbk=EXCLUDED.contract_position_dbk, project_name=EXCLUDED.project_name, project_id=EXCLUDED.project_id, project_task=EXCLUDED.project_task, mcl_id=EXCLUDED.mcl_id, mcl_name=EXCLUDED.mcl_name, mcl_desc=EXCLUDED.mcl_desc, valid_from=EXCLUDED.valid_from, valid_to=EXCLUDED.valid_to, effort_plan=EXCLUDED.effort_plan, updated_at=CURRENT_TIMESTAMP", rows, (ps,r)->{set(ps,1,r.dbk());set(ps,2,r.dbkUid());set(ps,3,r.upd());set(ps,4,r.updAnz());set(ps,5,r.updUid());set(ps,6,r.status());set(ps,7,r.personDbk());set(ps,8,r.organizationDbk());set(ps,9,r.contractDbk());set(ps,10,r.legacyRelation4Dbk());set(ps,11,r.legacyRelation5Dbk());set(ps,12,r.contractPositionDbk());set(ps,13,r.projectName());set(ps,14,r.projectId());set(ps,15,r.projectTask());set(ps,16,r.mclId());set(ps,17,r.mclName());set(ps,18,r.mclDesc());set(ps,19,ts(r.validFrom()));set(ps,20,ts(r.validTo()));set(ps,21,r.effortPlan());}); updateTableState("cost-unit", maxUpd(rows, LeitstandSourceRows.CostUnitRow::upd), lastId(rows, LeitstandSourceRows.CostUnitRow::dbk)); } + private void syncTasks(TimeSyncRun run) { var rows = sourceClient.fetchTasks(watermark("task")); run.setRowsRead(run.getRowsRead()+rows.size()); batchUpsert("INSERT INTO TIME.ls_task(dbk, dbk_uid, upd, upd_anz, upd_uid, status, primary_cost_unit_dbk, secondary_cost_unit_dbk, tertiary_cost_unit_dbk, parent_task_dbk, lfdnr, task_type, mcl_id, mcl_name, mcl_desc, valid_from, valid_to, effort_plan, last_work, completion_percent, done_date, remark, n_rid, legacy_90702_rid, amount, created_source_at, legacy_90700_rid, legacy_90699_rid, quantity) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT (dbk) DO UPDATE SET dbk_uid=EXCLUDED.dbk_uid, upd=EXCLUDED.upd, upd_anz=EXCLUDED.upd_anz, upd_uid=EXCLUDED.upd_uid, status=EXCLUDED.status, primary_cost_unit_dbk=EXCLUDED.primary_cost_unit_dbk, secondary_cost_unit_dbk=EXCLUDED.secondary_cost_unit_dbk, tertiary_cost_unit_dbk=EXCLUDED.tertiary_cost_unit_dbk, parent_task_dbk=EXCLUDED.parent_task_dbk, lfdnr=EXCLUDED.lfdnr, task_type=EXCLUDED.task_type, mcl_id=EXCLUDED.mcl_id, mcl_name=EXCLUDED.mcl_name, mcl_desc=EXCLUDED.mcl_desc, valid_from=EXCLUDED.valid_from, valid_to=EXCLUDED.valid_to, effort_plan=EXCLUDED.effort_plan, last_work=EXCLUDED.last_work, completion_percent=EXCLUDED.completion_percent, done_date=EXCLUDED.done_date, remark=EXCLUDED.remark, n_rid=EXCLUDED.n_rid, legacy_90702_rid=EXCLUDED.legacy_90702_rid, amount=EXCLUDED.amount, created_source_at=EXCLUDED.created_source_at, legacy_90700_rid=EXCLUDED.legacy_90700_rid, legacy_90699_rid=EXCLUDED.legacy_90699_rid, quantity=EXCLUDED.quantity, updated_at=CURRENT_TIMESTAMP", rows, (ps,r)->{set(ps,1,r.dbk());set(ps,2,r.dbkUid());set(ps,3,r.upd());set(ps,4,r.updAnz());set(ps,5,r.updUid());set(ps,6,r.status());set(ps,7,r.primaryCostUnitDbk());set(ps,8,r.secondaryCostUnitDbk());set(ps,9,r.tertiaryCostUnitDbk());set(ps,10,r.parentTaskDbk());set(ps,11,r.lfdnr());set(ps,12,r.taskType());set(ps,13,r.mclId());set(ps,14,r.mclName());set(ps,15,r.mclDesc());set(ps,16,ts(r.validFrom()));set(ps,17,ts(r.validTo()));set(ps,18,r.effortPlan());set(ps,19,ts(r.lastWork()));set(ps,20,r.completionPercent());set(ps,21,ts(r.doneDate()));set(ps,22,r.remark());set(ps,23,r.nRid());set(ps,24,r.legacy90702Rid());set(ps,25,r.amount());set(ps,26,ts(r.createdSourceAt()));set(ps,27,r.legacy90700Rid());set(ps,28,r.legacy90699Rid());set(ps,29,r.quantity());}); updateTableState("task", maxUpd(rows, LeitstandSourceRows.TaskRow::upd), lastId(rows, LeitstandSourceRows.TaskRow::dbk)); } + private void syncPersonTaskAssignments(TimeSyncRun run) { var rows = sourceClient.fetchPersonTaskAssignments(watermark("person-task-assignment")); run.setRowsRead(run.getRowsRead()+rows.size()); batchUpsert("INSERT INTO TIME.ls_person_task_assignment(dbk, dbk_uid, upd, upd_anz, upd_uid, status, task_dbk, person_dbk, cost_unit_dbk, rtype, last_work, effort_plan) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT (dbk) DO UPDATE SET dbk_uid=EXCLUDED.dbk_uid, upd=EXCLUDED.upd, upd_anz=EXCLUDED.upd_anz, upd_uid=EXCLUDED.upd_uid, status=EXCLUDED.status, task_dbk=EXCLUDED.task_dbk, person_dbk=EXCLUDED.person_dbk, cost_unit_dbk=EXCLUDED.cost_unit_dbk, rtype=EXCLUDED.rtype, last_work=EXCLUDED.last_work, effort_plan=EXCLUDED.effort_plan, updated_at=CURRENT_TIMESTAMP", rows, (ps,r)->{set(ps,1,r.dbk());set(ps,2,r.dbkUid());set(ps,3,r.upd());set(ps,4,r.updAnz());set(ps,5,r.updUid());set(ps,6,r.status());set(ps,7,r.taskDbk());set(ps,8,r.personDbk());set(ps,9,r.costUnitDbk());set(ps,10,r.rtype());set(ps,11,ts(r.lastWork()));set(ps,12,r.effortPlan());}); updateTableState("person-task-assignment", maxUpd(rows, LeitstandSourceRows.PersonTaskAssignmentRow::upd), lastId(rows, LeitstandSourceRows.PersonTaskAssignmentRow::dbk)); } + private List syncTimeRecordings(TimeSyncRun run) { var rows = sourceClient.fetchTimeRecordings(watermark("time-recording")); run.setRowsRead(run.getRowsRead()+rows.size()); batchUpsert("INSERT INTO TIME.ls_time_recording(dbk, dbk_uid, upd, upd_anz, upd_uid, status, person_dbk, lfdnr, record_type, mcl_id, mcl_desc, recorded_from, recorded_to, effort, remark, url, activity_type_id) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT (dbk) DO UPDATE SET dbk_uid=EXCLUDED.dbk_uid, upd=EXCLUDED.upd, upd_anz=EXCLUDED.upd_anz, upd_uid=EXCLUDED.upd_uid, status=EXCLUDED.status, person_dbk=EXCLUDED.person_dbk, lfdnr=EXCLUDED.lfdnr, record_type=EXCLUDED.record_type, mcl_id=EXCLUDED.mcl_id, mcl_desc=EXCLUDED.mcl_desc, recorded_from=EXCLUDED.recorded_from, recorded_to=EXCLUDED.recorded_to, effort=EXCLUDED.effort, remark=EXCLUDED.remark, url=EXCLUDED.url, activity_type_id=EXCLUDED.activity_type_id, updated_at=CURRENT_TIMESTAMP", rows, (ps,r)->{set(ps,1,r.dbk());set(ps,2,r.dbkUid());set(ps,3,r.upd());set(ps,4,r.updAnz());set(ps,5,r.updUid());set(ps,6,r.status());set(ps,7,r.personDbk());set(ps,8,r.lfdnr());set(ps,9,r.recordType());set(ps,10,r.mclId());set(ps,11,r.mclDesc());set(ps,12,ts(r.recordedFrom()));set(ps,13,ts(r.recordedTo()));set(ps,14,r.effort());set(ps,15,r.remark());set(ps,16,r.url());set(ps,17,r.activityTypeId());}); updateTableState("time-recording", maxUpd(rows, LeitstandSourceRows.TimeRecordingRow::upd), lastId(rows, LeitstandSourceRows.TimeRecordingRow::dbk)); return rows; } + private void syncTimeRecordingAssignments(TimeSyncRun run) { var rows = sourceClient.fetchTimeRecordingAssignments(watermark("time-recording-assignment")); run.setRowsRead(run.getRowsRead()+rows.size()); batchUpsert("INSERT INTO TIME.ls_time_recording_assignment(dbk, dbk_uid, upd, upd_anz, upd_uid, status, person_task_assignment_dbk, time_recording_dbk, mcl_desc, effort, remark) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT (dbk) DO UPDATE SET dbk_uid=EXCLUDED.dbk_uid, upd=EXCLUDED.upd, upd_anz=EXCLUDED.upd_anz, upd_uid=EXCLUDED.upd_uid, status=EXCLUDED.status, person_task_assignment_dbk=EXCLUDED.person_task_assignment_dbk, time_recording_dbk=EXCLUDED.time_recording_dbk, mcl_desc=EXCLUDED.mcl_desc, effort=EXCLUDED.effort, remark=EXCLUDED.remark, updated_at=CURRENT_TIMESTAMP", rows, (ps,r)->{set(ps,1,r.dbk());set(ps,2,r.dbkUid());set(ps,3,r.upd());set(ps,4,r.updAnz());set(ps,5,r.updUid());set(ps,6,r.status());set(ps,7,r.personTaskAssignmentDbk());set(ps,8,r.timeRecordingDbk());set(ps,9,r.mclDesc());set(ps,10,r.effort());set(ps,11,r.remark());}); updateTableState("time-recording-assignment", maxUpd(rows, LeitstandSourceRows.TimeRecordingAssignmentRow::upd), lastId(rows, LeitstandSourceRows.TimeRecordingAssignmentRow::dbk)); } + + public int materializeCanonicalTimeEntriesForPersonDbk(String personDbk, boolean rebuildProjection) { + if (personDbk == null || personDbk.isBlank()) { + throw new IllegalArgumentException("personDbk must not be blank"); + } + List recordings = timeRecordingRepository.findByPersonDbkOrderByRecordedFromAscDbkAsc(personDbk); + if (recordings.isEmpty()) { + log.info("No Leitstand time recordings found for personDbk={}", personDbk); + return 0; + } + //upsertCanonicalTimeEntriesForImportedRecordings(recordings); + if (rebuildProjection && properties.getLeitstand().isBuildSearchProjection()) { + projectionService.refreshForPersonDbk(personDbk); + } + return recordings.size(); + } @Transactional + public int materializeCanonicalTimeEntriesForPersonNumber(Integer personNumber, boolean rebuildProjection) { + if (personNumber == null) { + throw new IllegalArgumentException("personNumber must not be null"); + } + LeitstandPerson person = personRepository.findByPersonNumber(personNumber) + .orElseThrow(() -> new IllegalArgumentException("Unknown Leitstand person number: " + personNumber)); + return materializeCanonicalTimeEntriesForPersonDbk(person.getDbk(), rebuildProjection); + } + + @Transactional + protected void upsertCanonicalTimeEntriesForImportedRecordings(List recordings) { + Map personNames = loadPersonNames(); + int count = 0; + for (LeitstandTimeRecording recording : recordings) { + String businessKey = "TIME:LEITSTAND:" + recording.getDbk(); + Document document = documentRepository.findByBusinessKey(businessKey).orElseGet(() -> Document.builder().businessKey(businessKey).documentType(DocumentType.TIME_ENTRY).documentFamily(DocumentFamily.TIME).visibility(DocumentVisibility.PUBLIC).status(DocumentStatus.RECEIVED).build()); + document.setTitle(buildTitle(new LeitstandSourceRows.TimeRecordingRow( + recording.getDbk(), recording.getDbkUid(), recording.getUpd(), recording.getUpdAnz(), recording.getUpdUid(), recording.getStatus(), recording.getPersonDbk(), recording.getLfdnr(), recording.getRecordType(), recording.getMclId(), recording.getMclDesc(), recording.getRecordedFrom(), recording.getRecordedTo(), recording.getEffort(), recording.getRemark(), recording.getUrl(), recording.getActivityTypeId() + ), personNames.get(recording.getPersonDbk()))); + document.setSummary(buildSummary(new LeitstandSourceRows.TimeRecordingRow( + recording.getDbk(), recording.getDbkUid(), recording.getUpd(), recording.getUpdAnz(), recording.getUpdUid(), recording.getStatus(), recording.getPersonDbk(), recording.getLfdnr(), recording.getRecordType(), recording.getMclId(), recording.getMclDesc(), recording.getRecordedFrom(), recording.getRecordedTo(), recording.getEffort(), recording.getRemark(), recording.getUrl(), recording.getActivityTypeId() + ))); + document = documentRepository.save(document); + + Document finalDocument = document; + TimeEntry entry = timeEntryRepository.findBySourceSystemAndExternalId(TimeSourceSystem.LEITSTAND, recording.getDbk()).orElseGet(() -> TimeEntry.builder().sourceSystem(TimeSourceSystem.LEITSTAND).externalId(recording.getDbk()).document(finalDocument).build()); + entry.setDocument(document); + entry.setPersonExternalId(recording.getPersonDbk()); + entry.setPersonDisplayName(personNames.get(recording.getPersonDbk())); + entry.setEntryStart(recording.getRecordedFrom()); + entry.setEntryEnd(recording.getRecordedTo()); + entry.setDurationSeconds(toSeconds(recording.getEffort())); + entry.setDescriptionShort(recording.getMclDesc()); + entry.setDescriptionLong(recording.getRemark()); + entry.setRawStatus(recording.getStatus() == null ? null : String.valueOf(recording.getStatus())); + entry.setSearchAnchorLabel(document.getTitle()); + entry = timeEntryRepository.save(entry); + timeEntryRepository.flush(); + + log.info("Leitstand time recordings businessKey={}, count={}", businessKey, ++count); + + jdbcTemplate.update(""" + update TIME.ls_time_recording set time_entry_id = ? where dbk = ? + """, entry.getId(), recording.getDbk()); + upsertSourceLink(entry, "TIME_RECORDING", recording.getDbk(), null); + if (recording.getPersonDbk() != null) upsertSourceLink(entry, "PERSON", recording.getPersonDbk(), recording.getDbk()); + if (recording.getActivityTypeId() != null) upsertSourceLink(entry, "ACTIVITY_TYPE", String.valueOf(recording.getActivityTypeId()), recording.getDbk()); + } + } + + //@Transactional protected void upsertCanonicalTimeEntries(List rows) { Map personNames = loadPersonNames(); for (var row : rows) { @@ -189,10 +225,11 @@ public class LeitstandTimeImportService { entry.setRawStatus(row.status() == null ? null : String.valueOf(row.status())); entry.setSearchAnchorLabel(document.getTitle()); entry = timeEntryRepository.save(entry); + timeEntryRepository.flush(); jdbcTemplate.update(""" - update "time".ls_time_recording set time_entry_id = ? where dbk = ? - """, entry.getId(), row.dbk()); + update TIME.ls_time_recording set time_entry_id = ? where dbk = ? + """, entry.getId(), row.dbk()); upsertSourceLink(entry, "TIME_RECORDING", row.dbk(), null); if (row.personDbk() != null) upsertSourceLink(entry, "PERSON", row.personDbk(), row.dbk()); if (row.activityTypeId() != null) upsertSourceLink(entry, "ACTIVITY_TYPE", String.valueOf(row.activityTypeId()), row.dbk()); @@ -211,7 +248,7 @@ public class LeitstandTimeImportService { private Map loadPersonNames() { Map result = new HashMap<>(); jdbcTemplate.query(""" - select dbk, first_name, last_name from "time".ls_person + select dbk, first_name, last_name from TIME.ls_person """, rs -> { result.put(rs.getString("dbk"), firstNonBlank(joinName(rs.getString("first_name"), rs.getString("last_name")), rs.getString("last_name"), rs.getString("dbk"))); }); diff --git a/src/main/java/at/procon/dip/domain/time/service/LeitstandTimeProjectionService.java b/src/main/java/at/procon/dip/domain/time/service/LeitstandTimeProjectionService.java index b7e97f3..63e5a0d 100644 --- a/src/main/java/at/procon/dip/domain/time/service/LeitstandTimeProjectionService.java +++ b/src/main/java/at/procon/dip/domain/time/service/LeitstandTimeProjectionService.java @@ -58,6 +58,18 @@ public class LeitstandTimeProjectionService { upsertProjections(recordings); } + + @Transactional + public int refreshForPersonDbk(String personDbk) { + if (personDbk == null || personDbk.isBlank()) { + return 0; + } + List recordings = timeRecordingRepository + .findByPersonDbkAndTimeEntryIsNotNullOrderByRecordedFromAscDbkAsc(personDbk); + upsertProjections(recordings); + return recordings.size(); + } + @Transactional public int refreshAll() { List recordings = timeRecordingRepository.findByTimeEntryIsNotNull(); diff --git a/src/main/java/at/procon/dip/domain/time/service/TimeEntryRepresentationMaterializationService.java b/src/main/java/at/procon/dip/domain/time/service/TimeEntryRepresentationMaterializationService.java index 9ab5019..47c8015 100644 --- a/src/main/java/at/procon/dip/domain/time/service/TimeEntryRepresentationMaterializationService.java +++ b/src/main/java/at/procon/dip/domain/time/service/TimeEntryRepresentationMaterializationService.java @@ -35,7 +35,7 @@ public class TimeEntryRepresentationMaterializationService { private final EmbeddingProperties embeddingProperties; private final EmbeddingModelRegistry modelRegistry; - @Transactional + //@Transactional public void upsertRepresentations(TimeEntrySearchProjection projection) { if (projection.getSemanticText() == null || projection.getSemanticText().isBlank()) { log.debug("Skipping TIME representation for document {} because semantic text is blank", projection.getDocument().getId()); diff --git a/src/main/java/at/procon/dip/domain/time/startup/LeitstandTimeSelectiveMaterializationStartupRunner.java b/src/main/java/at/procon/dip/domain/time/startup/LeitstandTimeSelectiveMaterializationStartupRunner.java new file mode 100644 index 0000000..64c4e1a --- /dev/null +++ b/src/main/java/at/procon/dip/domain/time/startup/LeitstandTimeSelectiveMaterializationStartupRunner.java @@ -0,0 +1,42 @@ +package at.procon.dip.domain.time.startup; + +import at.procon.dip.domain.time.config.TimeDomainProperties; +import at.procon.dip.domain.time.service.LeitstandTimeImportService; +import at.procon.dip.runtime.condition.ConditionalOnRuntimeMode; +import at.procon.dip.runtime.config.RuntimeMode; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.boot.ApplicationArguments; +import org.springframework.boot.ApplicationRunner; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.stereotype.Component; + +@Component +@ConditionalOnRuntimeMode(RuntimeMode.NEW) +@ConditionalOnProperty(prefix = "dip.time.leitstand", name = {"enabled", "startup-selective-materialization-enabled"}, havingValue = "true") +@RequiredArgsConstructor +@Slf4j +public class LeitstandTimeSelectiveMaterializationStartupRunner implements ApplicationRunner { + + private final TimeDomainProperties properties; + private final LeitstandTimeImportService importService; + + @Override + public void run(ApplicationArguments args) { + var cfg = properties.getLeitstand(); + boolean rebuildProjection = cfg.isSelectiveMaterializationBuildProjection(); + if (cfg.getSelectiveMaterializationPersonDbk() != null && !cfg.getSelectiveMaterializationPersonDbk().isBlank()) { + log.info("Starting selective Leitstand TIME materialization for personDbk={} (rebuildProjection={})", cfg.getSelectiveMaterializationPersonDbk(), rebuildProjection); + int count = importService.materializeCanonicalTimeEntriesForPersonDbk(cfg.getSelectiveMaterializationPersonDbk(), rebuildProjection); + log.info("Completed selective Leitstand TIME materialization for personDbk={}. Processed {} recordings", cfg.getSelectiveMaterializationPersonDbk(), count); + return; + } + if (cfg.getSelectiveMaterializationPersonNumber() != null) { + log.info("Starting selective Leitstand TIME materialization for personNumber={} (rebuildProjection={})", cfg.getSelectiveMaterializationPersonNumber(), rebuildProjection); + int count = importService.materializeCanonicalTimeEntriesForPersonNumber(cfg.getSelectiveMaterializationPersonNumber(), rebuildProjection); + log.info("Completed selective Leitstand TIME materialization for personNumber={}. Processed {} recordings", cfg.getSelectiveMaterializationPersonNumber(), count); + return; + } + throw new IllegalStateException("dip.time.leitstand.startup-selective-materialization-enabled=true requires either selective-materialization-person-dbk or selective-materialization-person-number"); + } +} diff --git a/src/main/java/at/procon/dip/search/service/DocumentLexicalIndexService.java b/src/main/java/at/procon/dip/search/service/DocumentLexicalIndexService.java index 0d37c12..7508841 100644 --- a/src/main/java/at/procon/dip/search/service/DocumentLexicalIndexService.java +++ b/src/main/java/at/procon/dip/search/service/DocumentLexicalIndexService.java @@ -1,5 +1,6 @@ package at.procon.dip.search.service; +import java.sql.Types; import java.util.List; import java.util.UUID; @@ -7,6 +8,7 @@ import jakarta.persistence.EntityManager; import jakarta.persistence.PersistenceContext; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.jdbc.core.JdbcTemplate; import org.springframework.jdbc.core.namedparam.MapSqlParameterSource; import org.springframework.jdbc.core.namedparam.NamedParameterJdbcTemplate; @@ -18,7 +20,9 @@ import org.springframework.transaction.annotation.Transactional; @Slf4j public class DocumentLexicalIndexService { + @Qualifier("applicationNamedParameterJdbcTemplate") private final NamedParameterJdbcTemplate namedParameterJdbcTemplate; + @Qualifier("applicationJdbcTemplate") private final JdbcTemplate jdbcTemplate; @PersistenceContext @@ -42,7 +46,7 @@ public class DocumentLexicalIndexService { entityManager.flush(); MapSqlParameterSource params = new MapSqlParameterSource(); - params.addValue("representationId", representationId); + params.addValue("representationId", representationId, Types.OTHER); int updated = namedParameterJdbcTemplate.update(""" UPDATE doc.doc_text_representation diff --git a/src/main/java/at/procon/ted/config/OpenApiConfig.java b/src/main/java/at/procon/ted/config/OpenApiConfig.java index 9a8c211..f92d43a 100644 --- a/src/main/java/at/procon/ted/config/OpenApiConfig.java +++ b/src/main/java/at/procon/ted/config/OpenApiConfig.java @@ -48,7 +48,7 @@ public class OpenApiConfig { .url("https://www.procon.co.at"))) .servers(List.of( new Server() - .url("http://localhost:8080/api") + .url("/api") .description("Local Development Server"), new Server() .url("https://ted-api.procon.co.at/api")