Refactor phases 3
trifonovt
1 month ago
parent
71fb43a5ea
commit
adc4f2da43
@ -0,0 +1,46 @@
|
|||||||
|
# Phase 3 - TED projection model
|
||||||
|
|
||||||
|
## Goal
|
||||||
|
|
||||||
|
Move TED from being the implicit root data model to being a typed projection on top of the generic
|
||||||
|
canonical document model.
|
||||||
|
|
||||||
|
## New persistence model
|
||||||
|
|
||||||
|
### Generic root
|
||||||
|
- `DOC.doc_document`
|
||||||
|
- `DOC.doc_content`
|
||||||
|
- `DOC.doc_text_representation`
|
||||||
|
- `DOC.doc_embedding`
|
||||||
|
|
||||||
|
### TED-specific projection
|
||||||
|
- `TED.ted_notice_projection`
|
||||||
|
- `TED.ted_notice_lot`
|
||||||
|
- `TED.ted_notice_organization`
|
||||||
|
|
||||||
|
## Relationship model
|
||||||
|
|
||||||
|
- one generic `DOC.doc_document`
|
||||||
|
- zero or one `TED.ted_notice_projection`
|
||||||
|
- zero to many `TED.ted_notice_lot`
|
||||||
|
- zero to many `TED.ted_notice_organization`
|
||||||
|
|
||||||
|
The projection also keeps an optional back-reference to the legacy `TED.procurement_document` row to
|
||||||
|
support incremental migration and validation.
|
||||||
|
|
||||||
|
## Runtime behavior
|
||||||
|
|
||||||
|
When a new TED XML document is imported:
|
||||||
|
1. it is parsed into the existing legacy `ProcurementDocument`
|
||||||
|
2. the generic DOC root is ensured/refreshed
|
||||||
|
3. the primary text representation is ensured
|
||||||
|
4. if the generic vectorization pipeline is enabled, a pending embedding is ensured
|
||||||
|
5. the TED structured projection tables are refreshed from the parsed legacy document
|
||||||
|
|
||||||
|
## Why this phase matters
|
||||||
|
|
||||||
|
This is the first phase where TED is explicitly modeled as a document type projection instead of the
|
||||||
|
platform's canonical root entity. That makes the next steps possible:
|
||||||
|
- generic semantic search across multiple document types
|
||||||
|
- future non-TED projections
|
||||||
|
- migration of TED structured search to the new projection tables
|
||||||
@ -0,0 +1,30 @@
|
|||||||
|
# Phase 3 - TED as a structured projection on the generic document core
|
||||||
|
|
||||||
|
Phase 3 makes TED a proper type-specific projection layered on top of the generic `DOC.doc_document`
|
||||||
|
root introduced in Phase 1 and the generic vectorization model introduced in Phase 2.
|
||||||
|
|
||||||
|
## What is implemented
|
||||||
|
- `TED.ted_notice_projection`
|
||||||
|
- `TED.ted_notice_lot`
|
||||||
|
- `TED.ted_notice_organization`
|
||||||
|
- `TedNoticeProjectionService`
|
||||||
|
- optional startup backfill of missing TED projections
|
||||||
|
- processing flow updated so freshly imported TED notices dual-write to:
|
||||||
|
- `DOC` generic document/content/representation model
|
||||||
|
- `TED` structured projection tables
|
||||||
|
|
||||||
|
## Core intent
|
||||||
|
TED is no longer the root model of the platform. Instead:
|
||||||
|
- `DOC.doc_document` is the canonical document root
|
||||||
|
- `TED.ted_notice_projection` holds TED-specific structured metadata
|
||||||
|
- `TED.ted_notice_lot` and `TED.ted_notice_organization` hold normalized child structures
|
||||||
|
|
||||||
|
## Compatibility
|
||||||
|
This phase is additive:
|
||||||
|
- legacy `TED.procurement_document` remains in place
|
||||||
|
- existing search and API behavior continue to work
|
||||||
|
- new imports are now representable in both the legacy and new projection model
|
||||||
|
|
||||||
|
## Important limitation
|
||||||
|
Structured search endpoints still read from the legacy TED model. Moving TED structured reads to the
|
||||||
|
new projection tables is the next migration step.
|
||||||
@ -0,0 +1,95 @@
|
|||||||
|
package at.procon.dip.domain.ted.entity;
|
||||||
|
|
||||||
|
import at.procon.dip.architecture.SchemaNames;
|
||||||
|
import jakarta.persistence.Column;
|
||||||
|
import jakarta.persistence.Entity;
|
||||||
|
import jakarta.persistence.FetchType;
|
||||||
|
import jakarta.persistence.GeneratedValue;
|
||||||
|
import jakarta.persistence.GenerationType;
|
||||||
|
import jakarta.persistence.Id;
|
||||||
|
import jakarta.persistence.Index;
|
||||||
|
import jakarta.persistence.JoinColumn;
|
||||||
|
import jakarta.persistence.ManyToOne;
|
||||||
|
import jakarta.persistence.PrePersist;
|
||||||
|
import jakarta.persistence.Table;
|
||||||
|
import jakarta.persistence.UniqueConstraint;
|
||||||
|
import java.math.BigDecimal;
|
||||||
|
import java.time.OffsetDateTime;
|
||||||
|
import java.util.UUID;
|
||||||
|
import lombok.AllArgsConstructor;
|
||||||
|
import lombok.Builder;
|
||||||
|
import lombok.Getter;
|
||||||
|
import lombok.NoArgsConstructor;
|
||||||
|
import lombok.Setter;
|
||||||
|
import org.hibernate.annotations.JdbcTypeCode;
|
||||||
|
import org.hibernate.type.SqlTypes;
|
||||||
|
|
||||||
|
@Entity
|
||||||
|
@Table(schema = SchemaNames.TED, name = "ted_notice_lot", indexes = {
|
||||||
|
@Index(name = "idx_ted_notice_lot_projection", columnList = "notice_projection_id")
|
||||||
|
}, uniqueConstraints = {
|
||||||
|
@UniqueConstraint(name = "uq_ted_notice_lot_projection_lot", columnNames = {"notice_projection_id", "lot_id"})
|
||||||
|
})
|
||||||
|
@Getter
|
||||||
|
@Setter
|
||||||
|
@NoArgsConstructor
|
||||||
|
@AllArgsConstructor
|
||||||
|
@Builder
|
||||||
|
public class TedNoticeLot {
|
||||||
|
|
||||||
|
@Id
|
||||||
|
@GeneratedValue(strategy = GenerationType.UUID)
|
||||||
|
private UUID id;
|
||||||
|
|
||||||
|
@ManyToOne(fetch = FetchType.LAZY)
|
||||||
|
@JoinColumn(name = "notice_projection_id", nullable = false)
|
||||||
|
private TedNoticeProjection noticeProjection;
|
||||||
|
|
||||||
|
@Column(name = "lot_id", nullable = false, length = 50)
|
||||||
|
private String lotId;
|
||||||
|
|
||||||
|
@Column(name = "internal_id", columnDefinition = "TEXT")
|
||||||
|
private String internalId;
|
||||||
|
|
||||||
|
@Column(name = "title", columnDefinition = "TEXT")
|
||||||
|
private String title;
|
||||||
|
|
||||||
|
@Column(name = "description", columnDefinition = "TEXT")
|
||||||
|
private String description;
|
||||||
|
|
||||||
|
@Column(name = "cpv_codes", columnDefinition = "VARCHAR(100)[]")
|
||||||
|
@JdbcTypeCode(SqlTypes.ARRAY)
|
||||||
|
private String[] cpvCodes;
|
||||||
|
|
||||||
|
@Column(name = "nuts_codes", columnDefinition = "VARCHAR(20)[]")
|
||||||
|
@JdbcTypeCode(SqlTypes.ARRAY)
|
||||||
|
private String[] nutsCodes;
|
||||||
|
|
||||||
|
@Column(name = "estimated_value", precision = 20, scale = 2)
|
||||||
|
private BigDecimal estimatedValue;
|
||||||
|
|
||||||
|
@Column(name = "estimated_value_currency", length = 3)
|
||||||
|
private String estimatedValueCurrency;
|
||||||
|
|
||||||
|
@Column(name = "duration_value")
|
||||||
|
private Double durationValue;
|
||||||
|
|
||||||
|
@Column(name = "duration_unit", length = 20)
|
||||||
|
private String durationUnit;
|
||||||
|
|
||||||
|
@Column(name = "submission_deadline")
|
||||||
|
private OffsetDateTime submissionDeadline;
|
||||||
|
|
||||||
|
@Column(name = "eu_funded")
|
||||||
|
@Builder.Default
|
||||||
|
private Boolean euFunded = false;
|
||||||
|
|
||||||
|
@Builder.Default
|
||||||
|
@Column(name = "created_at", nullable = false, updatable = false)
|
||||||
|
private OffsetDateTime createdAt = OffsetDateTime.now();
|
||||||
|
|
||||||
|
@PrePersist
|
||||||
|
protected void onCreate() {
|
||||||
|
createdAt = OffsetDateTime.now();
|
||||||
|
}
|
||||||
|
}
|
||||||
@ -0,0 +1,90 @@
|
|||||||
|
package at.procon.dip.domain.ted.entity;
|
||||||
|
|
||||||
|
import at.procon.dip.architecture.SchemaNames;
|
||||||
|
import jakarta.persistence.Column;
|
||||||
|
import jakarta.persistence.Entity;
|
||||||
|
import jakarta.persistence.FetchType;
|
||||||
|
import jakarta.persistence.GeneratedValue;
|
||||||
|
import jakarta.persistence.GenerationType;
|
||||||
|
import jakarta.persistence.Id;
|
||||||
|
import jakarta.persistence.Index;
|
||||||
|
import jakarta.persistence.JoinColumn;
|
||||||
|
import jakarta.persistence.ManyToOne;
|
||||||
|
import jakarta.persistence.PrePersist;
|
||||||
|
import jakarta.persistence.Table;
|
||||||
|
import jakarta.persistence.UniqueConstraint;
|
||||||
|
import java.time.OffsetDateTime;
|
||||||
|
import java.util.UUID;
|
||||||
|
import lombok.AllArgsConstructor;
|
||||||
|
import lombok.Builder;
|
||||||
|
import lombok.Getter;
|
||||||
|
import lombok.NoArgsConstructor;
|
||||||
|
import lombok.Setter;
|
||||||
|
|
||||||
|
@Entity
|
||||||
|
@Table(schema = SchemaNames.TED, name = "ted_notice_organization", indexes = {
|
||||||
|
@Index(name = "idx_ted_notice_org_projection", columnList = "notice_projection_id"),
|
||||||
|
@Index(name = "idx_ted_notice_org_country", columnList = "country_code")
|
||||||
|
}, uniqueConstraints = {
|
||||||
|
@UniqueConstraint(name = "uq_ted_notice_org_projection_ref", columnNames = {"notice_projection_id", "org_reference"})
|
||||||
|
})
|
||||||
|
@Getter
|
||||||
|
@Setter
|
||||||
|
@NoArgsConstructor
|
||||||
|
@AllArgsConstructor
|
||||||
|
@Builder
|
||||||
|
public class TedNoticeOrganization {
|
||||||
|
|
||||||
|
@Id
|
||||||
|
@GeneratedValue(strategy = GenerationType.UUID)
|
||||||
|
private UUID id;
|
||||||
|
|
||||||
|
@ManyToOne(fetch = FetchType.LAZY)
|
||||||
|
@JoinColumn(name = "notice_projection_id", nullable = false)
|
||||||
|
private TedNoticeProjection noticeProjection;
|
||||||
|
|
||||||
|
@Column(name = "org_reference", length = 50)
|
||||||
|
private String orgReference;
|
||||||
|
|
||||||
|
@Column(name = "role", length = 50)
|
||||||
|
private String role;
|
||||||
|
|
||||||
|
@Column(name = "name", columnDefinition = "TEXT")
|
||||||
|
private String name;
|
||||||
|
|
||||||
|
@Column(name = "company_id", length = 1000)
|
||||||
|
private String companyId;
|
||||||
|
|
||||||
|
@Column(name = "country_code", length = 10)
|
||||||
|
private String countryCode;
|
||||||
|
|
||||||
|
@Column(name = "city", length = 255)
|
||||||
|
private String city;
|
||||||
|
|
||||||
|
@Column(name = "postal_code", length = 255)
|
||||||
|
private String postalCode;
|
||||||
|
|
||||||
|
@Column(name = "street_name", columnDefinition = "TEXT")
|
||||||
|
private String streetName;
|
||||||
|
|
||||||
|
@Column(name = "nuts_code", length = 10)
|
||||||
|
private String nutsCode;
|
||||||
|
|
||||||
|
@Column(name = "website_uri", columnDefinition = "TEXT")
|
||||||
|
private String websiteUri;
|
||||||
|
|
||||||
|
@Column(name = "email", length = 255)
|
||||||
|
private String email;
|
||||||
|
|
||||||
|
@Column(name = "phone", length = 50)
|
||||||
|
private String phone;
|
||||||
|
|
||||||
|
@Builder.Default
|
||||||
|
@Column(name = "created_at", nullable = false, updatable = false)
|
||||||
|
private OffsetDateTime createdAt = OffsetDateTime.now();
|
||||||
|
|
||||||
|
@PrePersist
|
||||||
|
protected void onCreate() {
|
||||||
|
createdAt = OffsetDateTime.now();
|
||||||
|
}
|
||||||
|
}
|
||||||
@ -0,0 +1,194 @@
|
|||||||
|
package at.procon.dip.domain.ted.entity;
|
||||||
|
|
||||||
|
import at.procon.dip.architecture.SchemaNames;
|
||||||
|
import at.procon.dip.domain.document.entity.Document;
|
||||||
|
import at.procon.ted.model.entity.ContractNature;
|
||||||
|
import at.procon.ted.model.entity.NoticeType;
|
||||||
|
import at.procon.ted.model.entity.ProcedureType;
|
||||||
|
import jakarta.persistence.Column;
|
||||||
|
import jakarta.persistence.Entity;
|
||||||
|
import jakarta.persistence.EnumType;
|
||||||
|
import jakarta.persistence.Enumerated;
|
||||||
|
import jakarta.persistence.FetchType;
|
||||||
|
import jakarta.persistence.GeneratedValue;
|
||||||
|
import jakarta.persistence.GenerationType;
|
||||||
|
import jakarta.persistence.Id;
|
||||||
|
import jakarta.persistence.Index;
|
||||||
|
import jakarta.persistence.JoinColumn;
|
||||||
|
import jakarta.persistence.OneToOne;
|
||||||
|
import jakarta.persistence.PrePersist;
|
||||||
|
import jakarta.persistence.PreUpdate;
|
||||||
|
import jakarta.persistence.Table;
|
||||||
|
import java.math.BigDecimal;
|
||||||
|
import java.time.LocalDate;
|
||||||
|
import java.time.OffsetDateTime;
|
||||||
|
import java.util.UUID;
|
||||||
|
import lombok.AllArgsConstructor;
|
||||||
|
import lombok.Builder;
|
||||||
|
import lombok.Getter;
|
||||||
|
import lombok.NoArgsConstructor;
|
||||||
|
import lombok.Setter;
|
||||||
|
import org.hibernate.annotations.JdbcTypeCode;
|
||||||
|
import org.hibernate.type.SqlTypes;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Phase 3 TED-specific projection that sits on top of the generic DOC document root.
|
||||||
|
*/
|
||||||
|
@Entity
|
||||||
|
@Table(schema = SchemaNames.TED, name = "ted_notice_projection", indexes = {
|
||||||
|
@Index(name = "idx_ted_proj_document", columnList = "document_id"),
|
||||||
|
@Index(name = "idx_ted_proj_legacy_doc", columnList = "legacy_procurement_document_id"),
|
||||||
|
@Index(name = "idx_ted_proj_publication_id", columnList = "publication_id"),
|
||||||
|
@Index(name = "idx_ted_proj_notice_type", columnList = "notice_type"),
|
||||||
|
@Index(name = "idx_ted_proj_buyer_country", columnList = "buyer_country_code"),
|
||||||
|
@Index(name = "idx_ted_proj_publication_date", columnList = "publication_date")
|
||||||
|
})
|
||||||
|
@Getter
|
||||||
|
@Setter
|
||||||
|
@NoArgsConstructor
|
||||||
|
@AllArgsConstructor
|
||||||
|
@Builder
|
||||||
|
public class TedNoticeProjection {
|
||||||
|
|
||||||
|
@Id
|
||||||
|
@GeneratedValue(strategy = GenerationType.UUID)
|
||||||
|
private UUID id;
|
||||||
|
|
||||||
|
@OneToOne(fetch = FetchType.LAZY)
|
||||||
|
@JoinColumn(name = "document_id", nullable = false, unique = true)
|
||||||
|
private Document document;
|
||||||
|
|
||||||
|
@Column(name = "legacy_procurement_document_id", unique = true)
|
||||||
|
private UUID legacyProcurementDocumentId;
|
||||||
|
|
||||||
|
@Column(name = "notice_id", length = 100)
|
||||||
|
private String noticeId;
|
||||||
|
|
||||||
|
@Column(name = "publication_id", length = 50)
|
||||||
|
private String publicationId;
|
||||||
|
|
||||||
|
@Column(name = "notice_url", length = 255)
|
||||||
|
private String noticeUrl;
|
||||||
|
|
||||||
|
@Column(name = "ojs_id", length = 20)
|
||||||
|
private String ojsId;
|
||||||
|
|
||||||
|
@Column(name = "contract_folder_id", length = 100)
|
||||||
|
private String contractFolderId;
|
||||||
|
|
||||||
|
@Enumerated(EnumType.STRING)
|
||||||
|
@Column(name = "notice_type", nullable = false, length = 50)
|
||||||
|
@Builder.Default
|
||||||
|
private NoticeType noticeType = NoticeType.OTHER;
|
||||||
|
|
||||||
|
@Column(name = "notice_subtype_code", length = 10)
|
||||||
|
private String noticeSubtypeCode;
|
||||||
|
|
||||||
|
@Column(name = "sdk_version", length = 20)
|
||||||
|
private String sdkVersion;
|
||||||
|
|
||||||
|
@Column(name = "ubl_version", length = 10)
|
||||||
|
private String ublVersion;
|
||||||
|
|
||||||
|
@Column(name = "language_code", length = 10)
|
||||||
|
private String languageCode;
|
||||||
|
|
||||||
|
@Column(name = "issue_datetime")
|
||||||
|
private OffsetDateTime issueDateTime;
|
||||||
|
|
||||||
|
@Column(name = "publication_date")
|
||||||
|
private LocalDate publicationDate;
|
||||||
|
|
||||||
|
@Column(name = "submission_deadline")
|
||||||
|
private OffsetDateTime submissionDeadline;
|
||||||
|
|
||||||
|
@Column(name = "buyer_name", columnDefinition = "TEXT")
|
||||||
|
private String buyerName;
|
||||||
|
|
||||||
|
@Column(name = "buyer_country_code", length = 10)
|
||||||
|
private String buyerCountryCode;
|
||||||
|
|
||||||
|
@Column(name = "buyer_city", length = 255)
|
||||||
|
private String buyerCity;
|
||||||
|
|
||||||
|
@Column(name = "buyer_postal_code", length = 100)
|
||||||
|
private String buyerPostalCode;
|
||||||
|
|
||||||
|
@Column(name = "buyer_nuts_code", length = 10)
|
||||||
|
private String buyerNutsCode;
|
||||||
|
|
||||||
|
@Column(name = "buyer_activity_type", length = 50)
|
||||||
|
private String buyerActivityType;
|
||||||
|
|
||||||
|
@Column(name = "buyer_legal_type", length = 50)
|
||||||
|
private String buyerLegalType;
|
||||||
|
|
||||||
|
@Column(name = "project_title", columnDefinition = "TEXT")
|
||||||
|
private String projectTitle;
|
||||||
|
|
||||||
|
@Column(name = "project_description", columnDefinition = "TEXT")
|
||||||
|
private String projectDescription;
|
||||||
|
|
||||||
|
@Column(name = "internal_reference", length = 500)
|
||||||
|
private String internalReference;
|
||||||
|
|
||||||
|
@Enumerated(EnumType.STRING)
|
||||||
|
@Column(name = "contract_nature", nullable = false, length = 50)
|
||||||
|
@Builder.Default
|
||||||
|
private ContractNature contractNature = ContractNature.UNKNOWN;
|
||||||
|
|
||||||
|
@Enumerated(EnumType.STRING)
|
||||||
|
@Column(name = "procedure_type", length = 50)
|
||||||
|
@Builder.Default
|
||||||
|
private ProcedureType procedureType = ProcedureType.OTHER;
|
||||||
|
|
||||||
|
@Column(name = "cpv_codes", columnDefinition = "VARCHAR(100)[]")
|
||||||
|
@JdbcTypeCode(SqlTypes.ARRAY)
|
||||||
|
private String[] cpvCodes;
|
||||||
|
|
||||||
|
@Column(name = "nuts_codes", columnDefinition = "VARCHAR(20)[]")
|
||||||
|
@JdbcTypeCode(SqlTypes.ARRAY)
|
||||||
|
private String[] nutsCodes;
|
||||||
|
|
||||||
|
@Column(name = "estimated_value", precision = 20, scale = 2)
|
||||||
|
private BigDecimal estimatedValue;
|
||||||
|
|
||||||
|
@Column(name = "estimated_value_currency", length = 3)
|
||||||
|
private String estimatedValueCurrency;
|
||||||
|
|
||||||
|
@Column(name = "total_lots")
|
||||||
|
@Builder.Default
|
||||||
|
private Integer totalLots = 0;
|
||||||
|
|
||||||
|
@Column(name = "max_lots_awarded")
|
||||||
|
private Integer maxLotsAwarded;
|
||||||
|
|
||||||
|
@Column(name = "max_lots_submitted")
|
||||||
|
private Integer maxLotsSubmitted;
|
||||||
|
|
||||||
|
@Column(name = "regulatory_domain", length = 50)
|
||||||
|
private String regulatoryDomain;
|
||||||
|
|
||||||
|
@Column(name = "eu_funded")
|
||||||
|
@Builder.Default
|
||||||
|
private Boolean euFunded = false;
|
||||||
|
|
||||||
|
@Builder.Default
|
||||||
|
@Column(name = "created_at", nullable = false, updatable = false)
|
||||||
|
private OffsetDateTime createdAt = OffsetDateTime.now();
|
||||||
|
|
||||||
|
@Builder.Default
|
||||||
|
@Column(name = "updated_at", nullable = false)
|
||||||
|
private OffsetDateTime updatedAt = OffsetDateTime.now();
|
||||||
|
|
||||||
|
@PrePersist
|
||||||
|
protected void onCreate() {
|
||||||
|
createdAt = OffsetDateTime.now();
|
||||||
|
updatedAt = OffsetDateTime.now();
|
||||||
|
}
|
||||||
|
|
||||||
|
@PreUpdate
|
||||||
|
protected void onUpdate() {
|
||||||
|
updatedAt = OffsetDateTime.now();
|
||||||
|
}
|
||||||
|
}
|
||||||
@ -0,0 +1,13 @@
|
|||||||
|
package at.procon.dip.domain.ted.repository;
|
||||||
|
|
||||||
|
import at.procon.dip.domain.ted.entity.TedNoticeLot;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.UUID;
|
||||||
|
import org.springframework.data.jpa.repository.JpaRepository;
|
||||||
|
|
||||||
|
public interface TedNoticeLotRepository extends JpaRepository<TedNoticeLot, UUID> {
|
||||||
|
|
||||||
|
List<TedNoticeLot> findByNoticeProjection_Id(UUID noticeProjectionId);
|
||||||
|
|
||||||
|
void deleteByNoticeProjection_Id(UUID noticeProjectionId);
|
||||||
|
}
|
||||||
@ -0,0 +1,13 @@
|
|||||||
|
package at.procon.dip.domain.ted.repository;
|
||||||
|
|
||||||
|
import at.procon.dip.domain.ted.entity.TedNoticeOrganization;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.UUID;
|
||||||
|
import org.springframework.data.jpa.repository.JpaRepository;
|
||||||
|
|
||||||
|
public interface TedNoticeOrganizationRepository extends JpaRepository<TedNoticeOrganization, UUID> {
|
||||||
|
|
||||||
|
List<TedNoticeOrganization> findByNoticeProjection_Id(UUID noticeProjectionId);
|
||||||
|
|
||||||
|
void deleteByNoticeProjection_Id(UUID noticeProjectionId);
|
||||||
|
}
|
||||||
@ -0,0 +1,15 @@
|
|||||||
|
package at.procon.dip.domain.ted.repository;
|
||||||
|
|
||||||
|
import at.procon.dip.domain.ted.entity.TedNoticeProjection;
|
||||||
|
import java.util.Optional;
|
||||||
|
import java.util.UUID;
|
||||||
|
import org.springframework.data.jpa.repository.JpaRepository;
|
||||||
|
|
||||||
|
public interface TedNoticeProjectionRepository extends JpaRepository<TedNoticeProjection, UUID> {
|
||||||
|
|
||||||
|
Optional<TedNoticeProjection> findByDocument_Id(UUID documentId);
|
||||||
|
|
||||||
|
Optional<TedNoticeProjection> findByLegacyProcurementDocumentId(UUID legacyProcurementDocumentId);
|
||||||
|
|
||||||
|
boolean existsByLegacyProcurementDocumentId(UUID legacyProcurementDocumentId);
|
||||||
|
}
|
||||||
@ -0,0 +1,181 @@
|
|||||||
|
package at.procon.dip.domain.ted.service;
|
||||||
|
|
||||||
|
import at.procon.dip.domain.document.entity.Document;
|
||||||
|
import at.procon.dip.domain.document.repository.DocumentRepository;
|
||||||
|
import at.procon.dip.domain.ted.entity.TedNoticeLot;
|
||||||
|
import at.procon.dip.domain.ted.entity.TedNoticeOrganization;
|
||||||
|
import at.procon.dip.domain.ted.entity.TedNoticeProjection;
|
||||||
|
import at.procon.dip.domain.ted.repository.TedNoticeLotRepository;
|
||||||
|
import at.procon.dip.domain.ted.repository.TedNoticeOrganizationRepository;
|
||||||
|
import at.procon.dip.domain.ted.repository.TedNoticeProjectionRepository;
|
||||||
|
import at.procon.ted.config.TedProcessorProperties;
|
||||||
|
import at.procon.ted.model.entity.Organization;
|
||||||
|
import at.procon.ted.model.entity.ProcurementDocument;
|
||||||
|
import at.procon.ted.model.entity.ProcurementLot;
|
||||||
|
import at.procon.ted.service.TedPhase2GenericDocumentService;
|
||||||
|
import java.util.ArrayList;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.Optional;
|
||||||
|
import java.util.UUID;
|
||||||
|
import lombok.RequiredArgsConstructor;
|
||||||
|
import lombok.extern.slf4j.Slf4j;
|
||||||
|
import org.springframework.stereotype.Service;
|
||||||
|
import org.springframework.transaction.annotation.Transactional;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Phase 3 service that materializes TED-specific structured projections on top of the generic DOC document root.
|
||||||
|
*/
|
||||||
|
@Service
|
||||||
|
@RequiredArgsConstructor
|
||||||
|
@Slf4j
|
||||||
|
public class TedNoticeProjectionService {
|
||||||
|
|
||||||
|
private final TedProcessorProperties properties;
|
||||||
|
private final TedPhase2GenericDocumentService tedPhase2GenericDocumentService;
|
||||||
|
private final DocumentRepository documentRepository;
|
||||||
|
private final TedNoticeProjectionRepository projectionRepository;
|
||||||
|
private final TedNoticeLotRepository lotRepository;
|
||||||
|
private final TedNoticeOrganizationRepository organizationRepository;
|
||||||
|
|
||||||
|
@Transactional
|
||||||
|
public UUID registerOrRefreshProjection(ProcurementDocument legacyDocument) {
|
||||||
|
if (!properties.getProjection().isEnabled()) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
TedPhase2GenericDocumentService.TedGenericDocumentSyncResult syncResult =
|
||||||
|
tedPhase2GenericDocumentService.syncTedDocument(legacyDocument);
|
||||||
|
return registerOrRefreshProjection(legacyDocument, syncResult.documentId());
|
||||||
|
}
|
||||||
|
|
||||||
|
@Transactional
|
||||||
|
public UUID registerOrRefreshProjection(ProcurementDocument legacyDocument, UUID genericDocumentId) {
|
||||||
|
if (!properties.getProjection().isEnabled()) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
UUID resolvedDocumentId = genericDocumentId;
|
||||||
|
if (resolvedDocumentId == null) {
|
||||||
|
resolvedDocumentId = tedPhase2GenericDocumentService.ensureGenericTedDocument(legacyDocument);
|
||||||
|
}
|
||||||
|
|
||||||
|
UUID finalResolvedDocumentId = resolvedDocumentId;
|
||||||
|
Document genericDocument = documentRepository.findById(resolvedDocumentId)
|
||||||
|
.orElseThrow(() -> new IllegalArgumentException("Unknown DOC document id: " + finalResolvedDocumentId));
|
||||||
|
|
||||||
|
TedNoticeProjection projection = projectionRepository.findByLegacyProcurementDocumentId(legacyDocument.getId())
|
||||||
|
.or(() -> projectionRepository.findByDocument_Id(genericDocument.getId()))
|
||||||
|
.orElseGet(TedNoticeProjection::new);
|
||||||
|
|
||||||
|
mapProjection(projection, genericDocument, legacyDocument);
|
||||||
|
projection = projectionRepository.save(projection);
|
||||||
|
replaceLots(projection, legacyDocument.getLots());
|
||||||
|
replaceOrganizations(projection, legacyDocument.getOrganizations());
|
||||||
|
|
||||||
|
log.debug("Phase 3 TED projection ensured for legacy {} -> projection {} / doc {}",
|
||||||
|
legacyDocument.getId(), projection.getId(), genericDocument.getId());
|
||||||
|
return projection.getId();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Transactional(readOnly = true)
|
||||||
|
public Optional<TedNoticeProjection> findByLegacyProcurementDocumentId(UUID legacyDocumentId) {
|
||||||
|
return projectionRepository.findByLegacyProcurementDocumentId(legacyDocumentId);
|
||||||
|
}
|
||||||
|
|
||||||
|
private void mapProjection(TedNoticeProjection projection, Document genericDocument, ProcurementDocument legacyDocument) {
|
||||||
|
projection.setDocument(genericDocument);
|
||||||
|
projection.setLegacyProcurementDocumentId(legacyDocument.getId());
|
||||||
|
projection.setNoticeId(legacyDocument.getNoticeId());
|
||||||
|
projection.setPublicationId(legacyDocument.getPublicationId());
|
||||||
|
projection.setNoticeUrl(legacyDocument.getNoticeUrl());
|
||||||
|
projection.setOjsId(legacyDocument.getOjsId());
|
||||||
|
projection.setContractFolderId(legacyDocument.getContractFolderId());
|
||||||
|
projection.setNoticeType(legacyDocument.getNoticeType());
|
||||||
|
projection.setNoticeSubtypeCode(legacyDocument.getNoticeSubtypeCode());
|
||||||
|
projection.setSdkVersion(legacyDocument.getSdkVersion());
|
||||||
|
projection.setUblVersion(legacyDocument.getUblVersion());
|
||||||
|
projection.setLanguageCode(legacyDocument.getLanguageCode());
|
||||||
|
projection.setIssueDateTime(legacyDocument.getIssueDateTime());
|
||||||
|
projection.setPublicationDate(legacyDocument.getPublicationDate());
|
||||||
|
projection.setSubmissionDeadline(legacyDocument.getSubmissionDeadline());
|
||||||
|
projection.setBuyerName(legacyDocument.getBuyerName());
|
||||||
|
projection.setBuyerCountryCode(legacyDocument.getBuyerCountryCode());
|
||||||
|
projection.setBuyerCity(legacyDocument.getBuyerCity());
|
||||||
|
projection.setBuyerPostalCode(legacyDocument.getBuyerPostalCode());
|
||||||
|
projection.setBuyerNutsCode(legacyDocument.getBuyerNutsCode());
|
||||||
|
projection.setBuyerActivityType(legacyDocument.getBuyerActivityType());
|
||||||
|
projection.setBuyerLegalType(legacyDocument.getBuyerLegalType());
|
||||||
|
projection.setProjectTitle(legacyDocument.getProjectTitle());
|
||||||
|
projection.setProjectDescription(legacyDocument.getProjectDescription());
|
||||||
|
projection.setInternalReference(legacyDocument.getInternalReference());
|
||||||
|
projection.setContractNature(legacyDocument.getContractNature());
|
||||||
|
projection.setProcedureType(legacyDocument.getProcedureType());
|
||||||
|
projection.setCpvCodes(copyArray(legacyDocument.getCpvCodes()));
|
||||||
|
projection.setNutsCodes(copyArray(legacyDocument.getNutsCodes()));
|
||||||
|
projection.setEstimatedValue(legacyDocument.getEstimatedValue());
|
||||||
|
projection.setEstimatedValueCurrency(legacyDocument.getEstimatedValueCurrency());
|
||||||
|
projection.setTotalLots(legacyDocument.getTotalLots());
|
||||||
|
projection.setMaxLotsAwarded(legacyDocument.getMaxLotsAwarded());
|
||||||
|
projection.setMaxLotsSubmitted(legacyDocument.getMaxLotsSubmitted());
|
||||||
|
projection.setRegulatoryDomain(legacyDocument.getRegulatoryDomain());
|
||||||
|
projection.setEuFunded(legacyDocument.getEuFunded());
|
||||||
|
}
|
||||||
|
|
||||||
|
private void replaceLots(TedNoticeProjection projection, List<ProcurementLot> legacyLots) {
|
||||||
|
lotRepository.deleteByNoticeProjection_Id(projection.getId());
|
||||||
|
if (legacyLots == null || legacyLots.isEmpty()) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
List<TedNoticeLot> projectedLots = new ArrayList<>();
|
||||||
|
for (ProcurementLot lot : legacyLots) {
|
||||||
|
projectedLots.add(TedNoticeLot.builder()
|
||||||
|
.noticeProjection(projection)
|
||||||
|
.lotId(lot.getLotId())
|
||||||
|
.internalId(lot.getInternalId())
|
||||||
|
.title(lot.getTitle())
|
||||||
|
.description(lot.getDescription())
|
||||||
|
.cpvCodes(copyArray(lot.getCpvCodes()))
|
||||||
|
.nutsCodes(copyArray(lot.getNutsCodes()))
|
||||||
|
.estimatedValue(lot.getEstimatedValue())
|
||||||
|
.estimatedValueCurrency(lot.getEstimatedValueCurrency())
|
||||||
|
.durationValue(lot.getDurationValue())
|
||||||
|
.durationUnit(lot.getDurationUnit())
|
||||||
|
.submissionDeadline(lot.getSubmissionDeadline())
|
||||||
|
.euFunded(lot.getEuFunded())
|
||||||
|
.build());
|
||||||
|
}
|
||||||
|
lotRepository.saveAll(projectedLots);
|
||||||
|
}
|
||||||
|
|
||||||
|
private void replaceOrganizations(TedNoticeProjection projection, List<Organization> legacyOrganizations) {
|
||||||
|
organizationRepository.deleteByNoticeProjection_Id(projection.getId());
|
||||||
|
if (legacyOrganizations == null || legacyOrganizations.isEmpty()) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
List<TedNoticeOrganization> projectedOrganizations = new ArrayList<>();
|
||||||
|
for (Organization organization : legacyOrganizations) {
|
||||||
|
projectedOrganizations.add(TedNoticeOrganization.builder()
|
||||||
|
.noticeProjection(projection)
|
||||||
|
.orgReference(organization.getOrgReference())
|
||||||
|
.role(organization.getRole())
|
||||||
|
.name(organization.getName())
|
||||||
|
.companyId(organization.getCompanyId())
|
||||||
|
.countryCode(organization.getCountryCode())
|
||||||
|
.city(organization.getCity())
|
||||||
|
.postalCode(organization.getPostalCode())
|
||||||
|
.streetName(organization.getStreetName())
|
||||||
|
.nutsCode(organization.getNutsCode())
|
||||||
|
.websiteUri(organization.getWebsiteUri())
|
||||||
|
.email(organization.getEmail())
|
||||||
|
.phone(organization.getPhone())
|
||||||
|
.build());
|
||||||
|
}
|
||||||
|
organizationRepository.saveAll(projectedOrganizations);
|
||||||
|
}
|
||||||
|
|
||||||
|
private String[] copyArray(String[] source) {
|
||||||
|
return source == null ? null : source.clone();
|
||||||
|
}
|
||||||
|
}
|
||||||
@ -0,0 +1,51 @@
|
|||||||
|
package at.procon.dip.domain.ted.startup;
|
||||||
|
|
||||||
|
import at.procon.dip.domain.ted.repository.TedNoticeProjectionRepository;
|
||||||
|
import at.procon.dip.domain.ted.service.TedNoticeProjectionService;
|
||||||
|
import at.procon.ted.config.TedProcessorProperties;
|
||||||
|
import at.procon.ted.repository.ProcurementDocumentRepository;
|
||||||
|
import lombok.RequiredArgsConstructor;
|
||||||
|
import lombok.extern.slf4j.Slf4j;
|
||||||
|
import org.springframework.boot.ApplicationArguments;
|
||||||
|
import org.springframework.boot.ApplicationRunner;
|
||||||
|
import org.springframework.data.domain.PageRequest;
|
||||||
|
import org.springframework.data.domain.Sort;
|
||||||
|
import org.springframework.stereotype.Component;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Optional startup backfill for Phase 3 TED projections.
|
||||||
|
*/
|
||||||
|
@Component
|
||||||
|
@RequiredArgsConstructor
|
||||||
|
@Slf4j
|
||||||
|
public class TedProjectionStartupRunner implements ApplicationRunner {
|
||||||
|
|
||||||
|
private final TedProcessorProperties properties;
|
||||||
|
private final ProcurementDocumentRepository procurementDocumentRepository;
|
||||||
|
private final TedNoticeProjectionRepository projectionRepository;
|
||||||
|
private final TedNoticeProjectionService projectionService;
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void run(ApplicationArguments args) {
|
||||||
|
if (!properties.getProjection().isEnabled() || !properties.getProjection().isStartupBackfillEnabled()) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
int limit = properties.getProjection().getStartupBackfillLimit();
|
||||||
|
log.info("Phase 3 startup backfill enabled - ensuring TED projections for up to {} documents", limit);
|
||||||
|
|
||||||
|
var page = procurementDocumentRepository.findAll(
|
||||||
|
PageRequest.of(0, limit, Sort.by(Sort.Direction.ASC, "createdAt")));
|
||||||
|
|
||||||
|
int synced = 0;
|
||||||
|
for (var legacyDocument : page.getContent()) {
|
||||||
|
if (projectionRepository.existsByLegacyProcurementDocumentId(legacyDocument.getId())) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
projectionService.registerOrRefreshProjection(legacyDocument);
|
||||||
|
synced++;
|
||||||
|
}
|
||||||
|
|
||||||
|
log.info("Phase 3 startup backfill completed - synced {} TED projections", synced);
|
||||||
|
}
|
||||||
|
}
|
||||||
@ -0,0 +1,105 @@
|
|||||||
|
-- Phase 3: TED becomes a structured projection on top of the generic DOC document root.
|
||||||
|
-- Additive migration; legacy TED tables remain in place for compatibility.
|
||||||
|
|
||||||
|
SET search_path TO TED, DOC, public;
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS TED.ted_notice_projection (
|
||||||
|
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||||
|
document_id UUID NOT NULL UNIQUE REFERENCES DOC.doc_document(id) ON DELETE CASCADE,
|
||||||
|
legacy_procurement_document_id UUID UNIQUE REFERENCES TED.procurement_document(id) ON DELETE SET NULL,
|
||||||
|
notice_id VARCHAR(100),
|
||||||
|
publication_id VARCHAR(50),
|
||||||
|
notice_url VARCHAR(255),
|
||||||
|
ojs_id VARCHAR(20),
|
||||||
|
contract_folder_id VARCHAR(100),
|
||||||
|
notice_type VARCHAR(50) NOT NULL DEFAULT 'OTHER',
|
||||||
|
notice_subtype_code VARCHAR(10),
|
||||||
|
sdk_version VARCHAR(20),
|
||||||
|
ubl_version VARCHAR(10),
|
||||||
|
language_code VARCHAR(10),
|
||||||
|
issue_datetime TIMESTAMP WITH TIME ZONE,
|
||||||
|
publication_date DATE,
|
||||||
|
submission_deadline TIMESTAMP WITH TIME ZONE,
|
||||||
|
buyer_name TEXT,
|
||||||
|
buyer_country_code VARCHAR(10),
|
||||||
|
buyer_city VARCHAR(255),
|
||||||
|
buyer_postal_code VARCHAR(100),
|
||||||
|
buyer_nuts_code VARCHAR(10),
|
||||||
|
buyer_activity_type VARCHAR(50),
|
||||||
|
buyer_legal_type VARCHAR(50),
|
||||||
|
project_title TEXT,
|
||||||
|
project_description TEXT,
|
||||||
|
internal_reference VARCHAR(500),
|
||||||
|
contract_nature VARCHAR(50) NOT NULL DEFAULT 'UNKNOWN',
|
||||||
|
procedure_type VARCHAR(50) DEFAULT 'OTHER',
|
||||||
|
cpv_codes VARCHAR(100)[],
|
||||||
|
nuts_codes VARCHAR(20)[],
|
||||||
|
estimated_value NUMERIC(20,2),
|
||||||
|
estimated_value_currency VARCHAR(3),
|
||||||
|
total_lots INTEGER NOT NULL DEFAULT 0,
|
||||||
|
max_lots_awarded INTEGER,
|
||||||
|
max_lots_submitted INTEGER,
|
||||||
|
regulatory_domain VARCHAR(50),
|
||||||
|
eu_funded BOOLEAN NOT NULL DEFAULT FALSE,
|
||||||
|
created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||||
|
updated_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_ted_notice_projection_publication_id
|
||||||
|
ON TED.ted_notice_projection(publication_id);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_ted_notice_projection_notice_type
|
||||||
|
ON TED.ted_notice_projection(notice_type);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_ted_notice_projection_buyer_country
|
||||||
|
ON TED.ted_notice_projection(buyer_country_code);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_ted_notice_projection_publication_date
|
||||||
|
ON TED.ted_notice_projection(publication_date DESC);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_ted_notice_projection_document
|
||||||
|
ON TED.ted_notice_projection(document_id);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_ted_notice_projection_legacy_doc
|
||||||
|
ON TED.ted_notice_projection(legacy_procurement_document_id);
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS TED.ted_notice_lot (
|
||||||
|
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||||
|
notice_projection_id UUID NOT NULL REFERENCES TED.ted_notice_projection(id) ON DELETE CASCADE,
|
||||||
|
lot_id VARCHAR(50) NOT NULL,
|
||||||
|
internal_id TEXT,
|
||||||
|
title TEXT,
|
||||||
|
description TEXT,
|
||||||
|
cpv_codes VARCHAR(100)[],
|
||||||
|
nuts_codes VARCHAR(20)[],
|
||||||
|
estimated_value NUMERIC(20,2),
|
||||||
|
estimated_value_currency VARCHAR(3),
|
||||||
|
duration_value DOUBLE PRECISION,
|
||||||
|
duration_unit VARCHAR(20),
|
||||||
|
submission_deadline TIMESTAMP WITH TIME ZONE,
|
||||||
|
eu_funded BOOLEAN NOT NULL DEFAULT FALSE,
|
||||||
|
created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||||
|
CONSTRAINT uq_ted_notice_lot_projection_lot UNIQUE (notice_projection_id, lot_id)
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_ted_notice_lot_projection
|
||||||
|
ON TED.ted_notice_lot(notice_projection_id);
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS TED.ted_notice_organization (
|
||||||
|
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||||
|
notice_projection_id UUID NOT NULL REFERENCES TED.ted_notice_projection(id) ON DELETE CASCADE,
|
||||||
|
org_reference VARCHAR(50),
|
||||||
|
role VARCHAR(50),
|
||||||
|
name TEXT,
|
||||||
|
company_id VARCHAR(1000),
|
||||||
|
country_code VARCHAR(10),
|
||||||
|
city VARCHAR(255),
|
||||||
|
postal_code VARCHAR(255),
|
||||||
|
street_name TEXT,
|
||||||
|
nuts_code VARCHAR(10),
|
||||||
|
website_uri TEXT,
|
||||||
|
email VARCHAR(255),
|
||||||
|
phone VARCHAR(50),
|
||||||
|
created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||||
|
CONSTRAINT uq_ted_notice_org_projection_ref UNIQUE (notice_projection_id, org_reference)
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_ted_notice_org_projection
|
||||||
|
ON TED.ted_notice_organization(notice_projection_id);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_ted_notice_org_country
|
||||||
|
ON TED.ted_notice_organization(country_code);
|
||||||
Loading…
Reference in New Issue