TED Notice Lot documents embedding

PATCH_NOTES.md

@@ -7,10 +7,10 @@ Included changes:
 - Score normalization and result fusion
 - Generic /search endpoint
 - Lexical index maintenance service and startup backfill runner
-- DOC lexical search migration (V9)
+- DOC lexical search migration (V14)
 - Modified DOC representation write path to refresh search vectors
 
 Important note:
-- Full-text search requires V9__doc_search_slice1_support.sql to be applied.
+- Full-text search requires V14__doc_search_slice1_support.sql to be applied.
 - The lexical index service is guarded and will no-op if the search columns are not yet present.
 - Because Flyway is currently disabled in application.yml, apply the migration manually or enable Flyway before using the new search endpoint.

src/main/java/at/procon/dip/clustering/repository/DocumentEmbeddingClusterSelectionRepositoryImpl.java

@@ -46,7 +46,6 @@ public class DocumentEmbeddingClusterSelectionRepositoryImpl implements Document
                 where e.embedding_status = 'COMPLETED'
                   and e.embedding_vector is not null
                   and e.prefix_profile_id is not null
-                  and d.
                 """);
         MapSqlParameterSource params = new MapSqlParameterSource();
         applyFilters(spec, sql, params);

src/main/java/at/procon/dip/domain/document/DocumentType.java

@@ -6,6 +6,7 @@ package at.procon.dip.domain.document;
 public enum DocumentType {
     TED_PACKAGE,
     TED_NOTICE,
+    TED_NOTICE_LOT,
     TIME_ENTRY,
     EMAIL,
     MIME_MESSAGE,

src/main/java/at/procon/dip/domain/document/repository/DocumentTextRepresentationRepository.java

@@ -1,11 +1,16 @@
 package at.procon.dip.domain.document.repository;
 
 import at.procon.dip.domain.document.RepresentationType;
+import at.procon.dip.domain.document.DocumentType;
+import at.procon.dip.domain.document.EmbeddingStatus;
 import at.procon.dip.domain.document.entity.DocumentTextRepresentation;
 import java.util.List;
 import java.util.Optional;
 import java.util.UUID;
+import org.springframework.data.domain.Pageable;
 import org.springframework.data.jpa.repository.JpaRepository;
+import org.springframework.data.jpa.repository.Query;
+import org.springframework.data.repository.query.Param;
 
 public interface DocumentTextRepresentationRepository extends JpaRepository<DocumentTextRepresentation, UUID> {
 
@@ -20,4 +25,33 @@ public interface DocumentTextRepresentationRepository extends JpaRepository<Docu
     long countByRepresentationType(RepresentationType representationType);
 
     Optional<DocumentTextRepresentation> findFirstByDocument_IdAndPrimaryRepresentationTrue(UUID documentId);
+
+    @Query("""
+            SELECT r
+            FROM DocumentTextRepresentation r
+            JOIN r.document d
+            WHERE (:documentType IS NULL OR d.documentType = :documentType)
+              AND (:representationType IS NULL OR r.representationType = :representationType)
+              AND (:builderKey IS NULL OR r.builderKey = :builderKey)
+              AND (:primaryOnly = false OR r.primaryRepresentation = true)
+              AND r.textBody IS NOT NULL
+              AND r.textBody <> ''
+              AND (:includeCompleted = true OR NOT EXISTS (
+                    SELECT e.id
+                    FROM DocumentEmbedding e
+                    WHERE e.representation.id = r.id
+                      AND e.model.modelKey = :modelKey
+                      AND e.embeddingStatus = :completedStatus
+              ))
+            ORDER BY r.createdAt ASC, r.id ASC
+            """)
+    List<DocumentTextRepresentation> findEmbeddingCandidatesByDocumentType(
+            @Param("documentType") DocumentType documentType,
+            @Param("representationType") RepresentationType representationType,
+            @Param("builderKey") String builderKey,
+            @Param("primaryOnly") boolean primaryOnly,
+            @Param("modelKey") String modelKey,
+            @Param("completedStatus") EmbeddingStatus completedStatus,
+            @Param("includeCompleted") boolean includeCompleted,
+            Pageable pageable);
 }

src/main/java/at/procon/dip/domain/ted/config/TedProjectionProperties.java

@@ -1,20 +1,52 @@
 package at.procon.dip.domain.ted.config;
 
+import jakarta.validation.constraints.Min;
 import jakarta.validation.constraints.Positive;
 import lombok.Data;
 import org.springframework.boot.context.properties.ConfigurationProperties;
 import org.springframework.context.annotation.Configuration;
+import org.springframework.validation.annotation.Validated;
 
 @Configuration
 @ConfigurationProperties(prefix = "dip.ted.projection")
+@Validated
 @Data
 public class TedProjectionProperties {
     private boolean enabled = true;
     private boolean startupBackfillEnabled = false;
-    @Positive
+    /**
+     * Maximum number of legacy TED documents to backfill on startup. 0 means no limit.
+     */
+    @Min(0)
     private int startupBackfillLimit = 250;
     @Positive
     private int structuredSearchHybridCandidateLimit = 5000;
     @Positive
     private int structuredSearchFacetBucketLimit = 12;
+    private LotDocuments lotDocuments = new LotDocuments();
+
+    @Data
+    public static class LotDocuments {
+        /**
+         * Materialize one canonical DOC document of type TED_NOTICE_LOT per TED lot.
+         */
+        private boolean enabled = false;
+        /**
+         * Optional startup/backfill path for notices that were imported before lot documents existed.
+         */
+        private boolean startupBackfillEnabled = false;
+        /**
+         * Maximum number of projections to backfill on startup. 0 means no limit.
+         */
+        @Min(0)
+        private int startupBackfillLimit = 1000;
+        /**
+         * Queue embeddings whenever the lot semantic text representation is created or changed.
+         */
+        private boolean queueEmbeddings = false;
+        /**
+         * Include parent notice project description even when the lot already has its own description.
+         */
+        private boolean includeProjectDescription = false;
+    }
 }

src/main/java/at/procon/dip/domain/ted/service/TedLotDocumentMaterializationService.java

@@ -0,0 +1,277 @@
+package at.procon.dip.domain.ted.service;
+
+import at.procon.dip.domain.document.DocumentFamily;
+import at.procon.dip.domain.document.DocumentStatus;
+import at.procon.dip.domain.document.DocumentType;
+import at.procon.dip.domain.document.RelationType;
+import at.procon.dip.domain.document.RepresentationType;
+import at.procon.dip.domain.document.entity.Document;
+import at.procon.dip.domain.document.entity.DocumentRelation;
+import at.procon.dip.domain.document.entity.DocumentTextRepresentation;
+import at.procon.dip.domain.document.repository.DocumentRelationRepository;
+import at.procon.dip.domain.document.repository.DocumentRepository;
+import at.procon.dip.domain.document.repository.DocumentTextRepresentationRepository;
+import at.procon.dip.domain.document.service.DocumentRepresentationService;
+import at.procon.dip.domain.document.service.command.AddDocumentTextRepresentationCommand;
+import at.procon.dip.domain.ted.config.TedProjectionProperties;
+import at.procon.dip.domain.ted.entity.TedNoticeLot;
+import at.procon.dip.domain.ted.entity.TedNoticeProjection;
+import at.procon.dip.domain.ted.repository.TedNoticeLotRepository;
+import at.procon.dip.embedding.config.EmbeddingProperties;
+import at.procon.dip.embedding.registry.EmbeddingModelRegistry;
+import at.procon.dip.embedding.service.RepresentationEmbeddingOrchestrator;
+import at.procon.dip.search.service.DocumentLexicalIndexService;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Optional;
+import java.util.UUID;
+import java.util.stream.Collectors;
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.stereotype.Service;
+import org.springframework.transaction.annotation.Transactional;
+import org.springframework.util.StringUtils;
+
+@Service
+@RequiredArgsConstructor
+@Slf4j
+public class TedLotDocumentMaterializationService {
+
+    public static final String BUILDER_KEY = "ted-lot-clustering-text-v1";
+
+    private final TedProjectionProperties properties;
+    private final TedNoticeLotRepository lotRepository;
+    private final DocumentRepository documentRepository;
+    private final DocumentRelationRepository relationRepository;
+    private final DocumentTextRepresentationRepository representationRepository;
+    private final DocumentRepresentationService documentRepresentationService;
+    private final DocumentLexicalIndexService lexicalIndexService;
+    private final RepresentationEmbeddingOrchestrator embeddingOrchestrator;
+    private final EmbeddingProperties embeddingProperties;
+    private final EmbeddingModelRegistry modelRegistry;
+
+    @Transactional
+    public int materializeProjectionLots(UUID projectionId) {
+        if (!properties.getLotDocuments().isEnabled()) {
+            return 0;
+        }
+        List<TedNoticeLot> lots = lotRepository.findByNoticeProjection_Id(projectionId);
+        int count = 0;
+        for (TedNoticeLot lot : lots) {
+            materializeLot(lot);
+            count++;
+        }
+        return count;
+    }
+
+    @Transactional
+    public void materializeLots(TedNoticeProjection projection, List<TedNoticeLot> lots) {
+        if (!properties.getLotDocuments().isEnabled() || lots == null || lots.isEmpty()) {
+            return;
+        }
+        for (TedNoticeLot lot : lots) {
+            materializeLot(lot);
+        }
+    }
+
+    private DocumentTextRepresentation materializeLot(TedNoticeLot lot) {
+        TedNoticeProjection projection = lot.getNoticeProjection();
+        Document parent = projection.getDocument();
+        String semanticText = buildSemanticText(projection, lot);
+        if (!StringUtils.hasText(semanticText)) {
+            log.debug("Skipping TED lot document for lot {} / projection {} because semantic text is blank", lot.getLotId(), projection.getId());
+            return null;
+        }
+
+        String businessKey = buildLotBusinessKey(projection, lot);
+        Document lotDocument = documentRepository.findByBusinessKey(businessKey)
+                .orElseGet(() -> newLotDocument(parent, businessKey));
+        lotDocument.setDocumentType(DocumentType.TED_NOTICE_LOT);
+        lotDocument.setDocumentFamily(DocumentFamily.PROCUREMENT);
+        lotDocument.setStatus(DocumentStatus.REPRESENTED);
+        lotDocument.setTitle(firstNonBlank(lot.getTitle(), projection.getProjectTitle(), businessKey));
+        lotDocument.setSummary(firstNonBlank(lot.getDescription(), projection.getProjectDescription()));
+        lotDocument.setLanguageCode(firstNonBlank(projection.getLanguageCode(), parent.getLanguageCode()));
+        lotDocument.setMimeType("application/x-ted-notice-lot");
+        lotDocument = documentRepository.save(lotDocument);
+
+        ensureRelation(parent, lotDocument, lot);
+        return upsertSemanticRepresentation(lotDocument, semanticText, lotDocument.getLanguageCode());
+    }
+
+    private Document newLotDocument(Document parent, String businessKey) {
+        Document document = new Document();
+        document.setOwnerTenant(parent.getOwnerTenant());
+        document.setVisibility(parent.getVisibility());
+        document.setBusinessKey(businessKey);
+        return document;
+    }
+
+    private void ensureRelation(Document parent, Document child, TedNoticeLot lot) {
+        if (!relationRepository.existsByParentDocument_IdAndChildDocument_IdAndRelationType(parent.getId(), child.getId(), RelationType.CONTAINS)) {
+            relationRepository.save(DocumentRelation.builder()
+                    .parentDocument(parent)
+                    .childDocument(child)
+                    .relationType(RelationType.CONTAINS)
+                    .sortOrder(resolveSortOrder(lot))
+                    .relationMetadata("{\"source\":\"ted-lot-materialization\",\"lotId\":\"" + escapeJson(firstNonBlank(lot.getLotId(), lot.getInternalId(), lot.getId().toString())) + "\"}")
+                    .build());
+        }
+    }
+
+    private DocumentTextRepresentation upsertSemanticRepresentation(Document document, String semanticText, String languageCode) {
+        Optional<DocumentTextRepresentation> existing = representationRepository
+                .findByDocument_IdAndRepresentationType(document.getId(), RepresentationType.SEMANTIC_TEXT)
+                .stream()
+                .filter(r -> BUILDER_KEY.equals(r.getBuilderKey()) || r.isPrimaryRepresentation())
+                .findFirst();
+
+        boolean changed = existing.isEmpty()
+                || !semanticText.equals(existing.get().getTextBody())
+                || !equalsNullable(languageCode, existing.get().getLanguageCode())
+                || !BUILDER_KEY.equals(existing.get().getBuilderKey());
+
+        DocumentTextRepresentation semantic = existing
+                .map(found -> changed ? updateRepresentation(found, semanticText, languageCode) : found)
+                .orElseGet(() -> documentRepresentationService.addRepresentation(new AddDocumentTextRepresentationCommand(
+                        document.getId(),
+                        null,
+                        RepresentationType.SEMANTIC_TEXT,
+                        BUILDER_KEY,
+                        languageCode,
+                        null,
+                        null,
+                        null,
+                        null,
+                        true,
+                        semanticText,
+                        false
+                )));
+
+        if (changed && shouldQueueEmbeddings()) {
+            String modelKey = modelRegistry.getRequiredDefaultDocumentModelKey();
+            embeddingOrchestrator.enqueueRepresentation(document.getId(), semantic.getId(), modelKey);
+        }
+        return semantic;
+    }
+
+    private DocumentTextRepresentation updateRepresentation(DocumentTextRepresentation existing, String semanticText, String languageCode) {
+        existing.setBuilderKey(BUILDER_KEY);
+        existing.setLanguageCode(languageCode);
+        existing.setPrimaryRepresentation(true);
+        existing.setTextBody(semanticText);
+        existing.setCharCount(semanticText.length());
+        DocumentTextRepresentation saved = representationRepository.saveAndFlush(existing);
+        lexicalIndexService.indexRepresentation(saved.getId());
+        return saved;
+    }
+
+    private boolean shouldQueueEmbeddings() {
+        return properties.getLotDocuments().isQueueEmbeddings()
+                && embeddingProperties.isEnabled()
+                && StringUtils.hasText(embeddingProperties.getDefaultDocumentModel());
+    }
+
+    private String buildSemanticText(TedNoticeProjection projection, TedNoticeLot lot) {
+        StringBuilder sb = new StringBuilder(1024);
+        append(sb, "Entity type", "Procurement lot");
+        append(sb, "Procurement scope", joinedLines(lot.getTitle(), lot.getDescription()));
+        append(sb, "Procurement category", joined(firstNonEmpty(lot.getCpvCodes(), projection.getCpvCodes())));
+        append(sb, "Relevant domain", projection.getBuyerActivityType());
+        append(sb, "Contract type", projection.getContractNature() == null ? null : projection.getContractNature().name());
+        append(sb, "Geographic context", joined(firstNonEmpty(lot.getNutsCodes(), projection.getNutsCodes()), projection.getBuyerCountryCode()));
+
+        if (!StringUtils.hasText(lot.getDescription())) {
+            append(sb, "Parent notice context", joinedLines(projection.getProjectTitle(), projection.getProjectDescription()));
+        } else if (properties.getLotDocuments().isIncludeProjectDescription()) {
+            append(sb, "Parent notice context", joinedLines(projection.getProjectTitle(), projection.getProjectDescription()));
+        } else {
+            append(sb, "Parent notice context", projection.getProjectTitle());
+        }
+        return sb.toString().trim();
+    }
+
+    private void append(StringBuilder sb, String label, String value) {
+        if (!StringUtils.hasText(value)) {
+            return;
+        }
+        if (sb.length() > 0) {
+            sb.append('\n').append('\n');
+        }
+        sb.append(label).append(':').append('\n').append(value.trim());
+    }
+
+    private String buildLotBusinessKey(TedNoticeProjection projection, TedNoticeLot lot) {
+        String noticeKey = firstNonBlank(projection.getPublicationId(), projection.getNoticeId(), projection.getDocument().getId().toString());
+        String lotKey = firstNonBlank(lot.getLotId(), lot.getInternalId(), lot.getId().toString());
+        return "TED_NOTICE_LOT:" + sanitizeKey(noticeKey) + ":" + sanitizeKey(lotKey);
+    }
+
+    private int resolveSortOrder(TedNoticeLot lot) {
+        String value = firstNonBlank(lot.getLotId(), lot.getInternalId());
+        if (value != null) {
+            String digits = value.replaceAll("\\D+", "");
+            if (StringUtils.hasText(digits)) {
+                try {
+                    return Integer.parseInt(digits);
+                } catch (NumberFormatException ignored) {
+                    // use default below
+                }
+            }
+        }
+        return 0;
+    }
+
+    private String[] firstNonEmpty(String[] first, String[] fallback) {
+        return first != null && first.length > 0 ? first : fallback;
+    }
+
+    private String joined(String[] values) {
+        if (values == null || values.length == 0) {
+            return null;
+        }
+        return Arrays.stream(values)
+                .filter(StringUtils::hasText)
+                .map(String::trim)
+                .distinct()
+                .collect(Collectors.joining(", "));
+    }
+
+    private String joined(String[] values, String fallback) {
+        String joined = joined(values);
+        if (StringUtils.hasText(joined)) {
+            return joined;
+        }
+        return fallback;
+    }
+
+    private String joinedLines(String... values) {
+        String joined = Arrays.stream(values)
+                .filter(StringUtils::hasText)
+                .map(String::trim)
+                .distinct()
+                .collect(Collectors.joining("\n"));
+        return StringUtils.hasText(joined) ? joined : null;
+    }
+
+    private String sanitizeKey(String value) {
+        return value == null ? "unknown" : value.trim().replaceAll("\\s+", "_");
+    }
+
+    private String escapeJson(String value) {
+        return value == null ? "" : value.replace("\\", "\\\\").replace("\"", "\\\"");
+    }
+
+    private boolean equalsNullable(String left, String right) {
+        return left == null ? right == null : left.equals(right);
+    }
+
+    private String firstNonBlank(String... values) {
+        for (String value : values) {
+            if (StringUtils.hasText(value)) {
+                return value.trim();
+            }
+        }
+        return null;
+    }
+}

src/main/java/at/procon/dip/domain/ted/service/TedNoticeProjectionService.java

@@ -39,6 +39,7 @@ public class TedNoticeProjectionService {
     private final TedNoticeProjectionRepository projectionRepository;
     private final TedNoticeLotRepository lotRepository;
     private final TedNoticeOrganizationRepository organizationRepository;
+    private final TedLotDocumentMaterializationService lotDocumentMaterializationService;
 
     @Transactional
     public UUID registerOrRefreshProjection(ProcurementDocument legacyDocument) {
@@ -166,7 +167,8 @@ public class TedNoticeProjectionService {
                     .euFunded(lot.getEuFunded())
                     .build());
         }
-        lotRepository.saveAll(projectedLots);
+        List<TedNoticeLot> savedLots = lotRepository.saveAll(projectedLots);
+        lotDocumentMaterializationService.materializeLots(projection, savedLots);
     }
 
     private void replaceOrganizations(TedNoticeProjection projection, List<Organization> legacyOrganizations) {

src/main/java/at/procon/dip/domain/ted/startup/TedProjectionStartupRunner.java

@@ -1,8 +1,9 @@
 package at.procon.dip.domain.ted.startup;
 
-import at.procon.dip.domain.ted.repository.TedNoticeProjectionRepository;
-import at.procon.dip.domain.ted.service.TedNoticeProjectionService;
 import at.procon.dip.domain.ted.config.TedProjectionProperties;
+import at.procon.dip.domain.ted.repository.TedNoticeProjectionRepository;
+import at.procon.dip.domain.ted.service.TedLotDocumentMaterializationService;
+import at.procon.dip.domain.ted.service.TedNoticeProjectionService;
 import at.procon.dip.runtime.condition.ConditionalOnRuntimeMode;
 import at.procon.dip.runtime.config.RuntimeMode;
 import at.procon.ted.repository.ProcurementDocumentRepository;
@@ -15,7 +16,7 @@ import org.springframework.data.domain.Sort;
 import org.springframework.stereotype.Component;
 
 /**
- * Optional startup backfill for Phase 3 TED projections.
+ * Optional startup backfill for Phase 3 TED projections and optional TED lot documents.
  */
 @Component
 @ConditionalOnRuntimeMode(RuntimeMode.NEW)
@@ -23,32 +24,99 @@ import org.springframework.stereotype.Component;
 @Slf4j
 public class TedProjectionStartupRunner implements ApplicationRunner {
 
+    private static final int STARTUP_BACKFILL_BATCH_SIZE = 1000;
+
     private final TedProjectionProperties properties;
     private final ProcurementDocumentRepository procurementDocumentRepository;
     private final TedNoticeProjectionRepository projectionRepository;
     private final TedNoticeProjectionService projectionService;
+    private final TedLotDocumentMaterializationService lotDocumentMaterializationService;
 
     @Override
     public void run(ApplicationArguments args) {
-        if (!properties.isEnabled() || !properties.isStartupBackfillEnabled()) {
+        if (!properties.isEnabled()) {
             return;
         }
 
-        int limit = properties.getStartupBackfillLimit();
-        log.info("Phase 3 startup backfill enabled - ensuring TED projections for up to {} documents", limit);
+        if (properties.isStartupBackfillEnabled()) {
+            backfillNoticeProjections();
+        }
 
-        var page = procurementDocumentRepository.findAll(
-                PageRequest.of(0, limit, Sort.by(Sort.Direction.ASC, "createdAt")));
+        if (properties.getLotDocuments().isEnabled() && properties.getLotDocuments().isStartupBackfillEnabled()) {
+            backfillLotDocuments();
+        }
+    }
+
+    private void backfillNoticeProjections() {
+        int limit = properties.getStartupBackfillLimit();
+        log.info("Phase 3 startup backfill enabled - ensuring TED projections for {} documents", describeLimit(limit));
 
         int synced = 0;
-        for (var legacyDocument : page.getContent()) {
-            if (projectionRepository.existsByLegacyProcurementDocumentId(legacyDocument.getId())) {
-                continue;
+        int processed = 0;
+        int pageNumber = 0;
+        Sort sort = Sort.by(Sort.Direction.ASC, "createdAt");
+
+        while (limit <= 0 || processed < limit) {
+            int pageSize = pageSizeFor(limit, processed);
+            var page = procurementDocumentRepository.findAll(PageRequest.of(pageNumber++, pageSize, sort));
+            if (page.isEmpty()) {
+                break;
+            }
+
+            for (var legacyDocument : page.getContent()) {
+                processed++;
+                if (projectionRepository.existsByLegacyProcurementDocumentId(legacyDocument.getId())) {
+                    continue;
+                }
+                projectionService.registerOrRefreshProjection(legacyDocument);
+                synced++;
+            }
+
+            if (!page.hasNext()) {
+                break;
             }
-            projectionService.registerOrRefreshProjection(legacyDocument);
-            synced++;
         }
 
         log.info("Phase 3 startup backfill completed - synced {} TED projections", synced);
     }
+
+    private void backfillLotDocuments() {
+        int limit = properties.getLotDocuments().getStartupBackfillLimit();
+        log.info("TED lot document startup backfill enabled - materializing lots for {} projections", describeLimit(limit));
+
+        int lotDocuments = 0;
+        int processed = 0;
+        int pageNumber = 0;
+        Sort sort = Sort.by(Sort.Direction.ASC, "createdAt");
+
+        while (limit <= 0 || processed < limit) {
+            int pageSize = pageSizeFor(limit, processed);
+            var page = projectionRepository.findAll(PageRequest.of(pageNumber++, pageSize, sort));
+            if (page.isEmpty()) {
+                break;
+            }
+
+            for (var projection : page.getContent()) {
+                processed++;
+                lotDocuments += lotDocumentMaterializationService.materializeProjectionLots(projection.getId());
+            }
+
+            if (!page.hasNext()) {
+                break;
+            }
+        }
+
+        log.info("TED lot document startup backfill completed - materialized/updated {} lot documents", lotDocuments);
+    }
+
+    private int pageSizeFor(int limit, int processed) {
+        if (limit <= 0) {
+            return STARTUP_BACKFILL_BATCH_SIZE;
+        }
+        return Math.min(STARTUP_BACKFILL_BATCH_SIZE, limit - processed);
+    }
+
+    private String describeLimit(int limit) {
+        return limit > 0 ? "up to " + limit : "all available";
+    }
 }

src/main/java/at/procon/dip/embedding/config/EmbeddingProperties.java

@@ -1,6 +1,8 @@
 package at.procon.dip.embedding.config;
 
 import at.procon.dip.domain.document.DistanceMetric;
+import at.procon.dip.domain.document.DocumentType;
+import at.procon.dip.domain.document.RepresentationType;
 import at.procon.dip.embedding.model.EmbeddingPrefixMode;
 import java.time.Duration;
 import java.util.LinkedHashMap;
@@ -21,6 +23,7 @@ public class EmbeddingProperties {
     private Map<String, ModelProperties> models = new LinkedHashMap<>();
     private IndexingProperties indexing = new IndexingProperties();
     private JobsProperties jobs = new JobsProperties();
+    private StartupProperties startup = new StartupProperties();
 
     @Data
     public static class ProviderProperties {
@@ -79,4 +82,18 @@ public class EmbeddingProperties {
         private Duration maxRetryDelay = Duration.ofHours(6);
         private long schedulerDelayMs = 5000;
     }
+
+    @Data
+    public static class StartupProperties {
+        private boolean enqueueMissingEnabled = false;
+        private boolean processReadyEnabled = false;
+        private DocumentType documentType;
+        private RepresentationType representationType;
+        private String builderKey;
+        private boolean primaryOnly = false;
+        private String modelKey;
+        private boolean force = false;
+        private int batchSize = 1000;
+        private long maxRepresentationsPerRun = 0;
+    }
 }

src/main/java/at/procon/dip/embedding/service/ScopedEmbeddingEnqueueResult.java

@@ -0,0 +1,17 @@
+package at.procon.dip.embedding.service;
+
+import at.procon.dip.domain.document.DocumentType;
+import at.procon.dip.domain.document.RepresentationType;
+
+public record ScopedEmbeddingEnqueueResult(
+        DocumentType documentType,
+        RepresentationType representationType,
+        String builderKey,
+        boolean primaryOnly,
+        String modelKey,
+        boolean force,
+        int requestedLimit,
+        int matchedRepresentations,
+        int jobsQueuedOrAlreadyActive
+) {
+}

src/main/java/at/procon/dip/embedding/service/ScopedEmbeddingEnqueueService.java

@@ -0,0 +1,91 @@
+package at.procon.dip.embedding.service;
+
+import at.procon.dip.domain.document.DocumentType;
+import at.procon.dip.domain.document.EmbeddingStatus;
+import at.procon.dip.domain.document.RepresentationType;
+import at.procon.dip.domain.document.entity.DocumentTextRepresentation;
+import at.procon.dip.domain.document.repository.DocumentTextRepresentationRepository;
+import at.procon.dip.embedding.job.service.EmbeddingJobService;
+import at.procon.dip.embedding.model.EmbeddingJobType;
+import at.procon.dip.embedding.registry.EmbeddingModelRegistry;
+import java.util.List;
+import lombok.RequiredArgsConstructor;
+import org.springframework.data.domain.PageRequest;
+import org.springframework.stereotype.Service;
+import org.springframework.transaction.annotation.Transactional;
+import org.springframework.util.StringUtils;
+
+@Service
+@RequiredArgsConstructor
+public class ScopedEmbeddingEnqueueService {
+
+    private static final int MAX_LIMIT = 10_000;
+
+    private final DocumentTextRepresentationRepository representationRepository;
+    private final EmbeddingJobService jobService;
+    private final EmbeddingModelRegistry modelRegistry;
+
+    @Transactional
+    public ScopedEmbeddingEnqueueResult enqueueByDocumentType(DocumentType documentType,
+                                                             RepresentationType representationType,
+                                                             String builderKey,
+                                                             boolean primaryOnly,
+                                                             String modelKey,
+                                                             boolean force,
+                                                             int limit) {
+        return enqueueByDocumentType(documentType, representationType, builderKey, primaryOnly, modelKey, force, 0, limit);
+    }
+
+    @Transactional
+    public ScopedEmbeddingEnqueueResult enqueueByDocumentType(DocumentType documentType,
+                                                             RepresentationType representationType,
+                                                             String builderKey,
+                                                             boolean primaryOnly,
+                                                             String modelKey,
+                                                             boolean force,
+                                                             int pageNumber,
+                                                             int limit) {
+        String effectiveModelKey = StringUtils.hasText(modelKey)
+                ? modelKey
+                : modelRegistry.getRequiredDefaultDocumentModelKey();
+        modelRegistry.getRequired(effectiveModelKey);
+
+        int effectiveLimit = Math.max(1, Math.min(limit, MAX_LIMIT));
+        int effectivePageNumber = Math.max(0, pageNumber);
+        String normalizedBuilderKey = StringUtils.hasText(builderKey) ? builderKey.trim() : null;
+
+        List<DocumentTextRepresentation> representations = representationRepository.findEmbeddingCandidatesByDocumentType(
+                documentType,
+                representationType,
+                normalizedBuilderKey,
+                primaryOnly,
+                effectiveModelKey,
+                EmbeddingStatus.COMPLETED,
+                force,
+                PageRequest.of(effectivePageNumber, effectiveLimit)
+        );
+
+        int jobs = 0;
+        for (DocumentTextRepresentation representation : representations) {
+            jobService.enqueueForRepresentation(
+                    representation.getDocument().getId(),
+                    representation.getId(),
+                    effectiveModelKey,
+                    EmbeddingJobType.DOCUMENT_EMBED
+            );
+            jobs++;
+        }
+
+        return new ScopedEmbeddingEnqueueResult(
+                documentType,
+                representationType,
+                normalizedBuilderKey,
+                primaryOnly,
+                effectiveModelKey,
+                force,
+                effectiveLimit,
+                representations.size(),
+                jobs
+        );
+    }
+}

src/main/java/at/procon/dip/embedding/startup/EmbeddingStartupRunner.java

@@ -0,0 +1,92 @@
+package at.procon.dip.embedding.startup;
+
+import at.procon.dip.embedding.config.EmbeddingProperties;
+import at.procon.dip.embedding.service.RepresentationEmbeddingOrchestrator;
+import at.procon.dip.embedding.service.ScopedEmbeddingEnqueueResult;
+import at.procon.dip.embedding.service.ScopedEmbeddingEnqueueService;
+import at.procon.dip.runtime.condition.ConditionalOnRuntimeMode;
+import at.procon.dip.runtime.config.RuntimeMode;
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.boot.ApplicationArguments;
+import org.springframework.boot.ApplicationRunner;
+import org.springframework.core.Ordered;
+import org.springframework.core.annotation.Order;
+import org.springframework.stereotype.Component;
+
+@Component
+@ConditionalOnRuntimeMode(RuntimeMode.NEW)
+@RequiredArgsConstructor
+@Order(Ordered.LOWEST_PRECEDENCE)
+@Slf4j
+public class EmbeddingStartupRunner implements ApplicationRunner {
+
+    private final EmbeddingProperties properties;
+    private final ScopedEmbeddingEnqueueService enqueueService;
+    private final RepresentationEmbeddingOrchestrator orchestrator;
+
+    @Override
+    public void run(ApplicationArguments args) {
+        EmbeddingProperties.StartupProperties startup = properties.getStartup();
+        if (!properties.isEnabled() || startup == null) {
+            return;
+        }
+
+        if (startup.isEnqueueMissingEnabled()) {
+            enqueueMissing(startup);
+        }
+
+        if (startup.isProcessReadyEnabled()) {
+            processReadyJobs();
+        }
+    }
+
+    private void enqueueMissing(EmbeddingProperties.StartupProperties startup) {
+        int batchSize = Math.max(1, startup.getBatchSize());
+        long max = startup.getMaxRepresentationsPerRun();
+        long remaining = max > 0 ? max : Long.MAX_VALUE;
+        int pageNumber = 0;
+        long matched = 0;
+        long queued = 0;
+
+        log.info("Startup embedding enqueue enabled (documentType={}, representationType={}, builderKey={}, primaryOnly={}, force={}, modelKey={}, maxRepresentationsPerRun={})",
+                startup.getDocumentType(), startup.getRepresentationType(), startup.getBuilderKey(), startup.isPrimaryOnly(),
+                startup.isForce(), startup.getModelKey(), max > 0 ? max : "unbounded");
+
+        while (remaining > 0) {
+            int requested = (int) Math.min(batchSize, remaining);
+            ScopedEmbeddingEnqueueResult result = enqueueService.enqueueByDocumentType(
+                    startup.getDocumentType(),
+                    startup.getRepresentationType(),
+                    startup.getBuilderKey(),
+                    startup.isPrimaryOnly(),
+                    startup.getModelKey(),
+                    startup.isForce(),
+                    pageNumber,
+                    requested
+            );
+
+            matched += result.matchedRepresentations();
+            queued += result.jobsQueuedOrAlreadyActive();
+            remaining -= result.matchedRepresentations();
+
+            if (result.matchedRepresentations() < requested) {
+                break;
+            }
+            pageNumber++;
+        }
+
+        log.info("Startup embedding enqueue completed - matched {} representation(s), queued/already-active {} job(s)",
+                matched, queued);
+    }
+
+    private void processReadyJobs() {
+        if (!properties.getJobs().isEnabled()) {
+            log.warn("Startup embedding processing was requested but dip.embedding.jobs.enabled=false; queued jobs will not be processed by the NEW embedding job subsystem");
+            return;
+        }
+
+        int processed = orchestrator.processNextReadyBatch();
+        log.info("Startup embedding processing completed - processed {} ready job(s)", processed);
+    }
+}

src/main/java/at/procon/dip/embedding/web/EmbeddingAdminController.java

@@ -0,0 +1,66 @@
+package at.procon.dip.embedding.web;
+
+import at.procon.dip.domain.document.DocumentType;
+import at.procon.dip.domain.document.RepresentationType;
+import at.procon.dip.embedding.service.ScopedEmbeddingEnqueueResult;
+import at.procon.dip.embedding.service.ScopedEmbeddingEnqueueService;
+import at.procon.dip.runtime.condition.ConditionalOnRuntimeMode;
+import at.procon.dip.runtime.config.RuntimeMode;
+import io.swagger.v3.oas.annotations.Operation;
+import io.swagger.v3.oas.annotations.tags.Tag;
+import lombok.RequiredArgsConstructor;
+import org.springframework.http.HttpStatus;
+import org.springframework.web.bind.annotation.PostMapping;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RequestParam;
+import org.springframework.web.bind.annotation.RestController;
+import org.springframework.web.server.ResponseStatusException;
+
+@RestController
+@ConditionalOnRuntimeMode(RuntimeMode.NEW)
+@RequestMapping("/v1/dip/admin/embeddings")
+@RequiredArgsConstructor
+@Tag(name = "Embedding Admin", description = "Administrative operations for NEW-runtime representation embeddings")
+public class EmbeddingAdminController {
+
+    private final ScopedEmbeddingEnqueueService enqueueService;
+
+    @PostMapping("/enqueue-by-document-type")
+    @Operation(
+            summary = "Queue embeddings by document type",
+            description = "Queues embedding jobs only for DOC text representations belonging to the requested document type."
+    )
+    public ScopedEmbeddingEnqueueResult enqueueByDocumentType(
+            @RequestParam String documentType,
+            @RequestParam(required = false) String representationType,
+            @RequestParam(required = false) String builderKey,
+            @RequestParam(required = false, defaultValue = "false") boolean primaryOnly,
+            @RequestParam(required = false) String modelKey,
+            @RequestParam(required = false, defaultValue = "false") boolean force,
+            @RequestParam(required = false, defaultValue = "1000") int limit) {
+        return enqueueService.enqueueByDocumentType(
+                parseEnum(DocumentType.class, documentType, "documentType"),
+                parseEnum(RepresentationType.class, representationType, "representationType"),
+                builderKey,
+                primaryOnly,
+                modelKey,
+                force,
+                limit
+        );
+    }
+
+    private <E extends Enum<E>> E parseEnum(Class<E> enumType, String value, String parameterName) {
+        if (value == null || value.isBlank()) {
+            return null;
+        }
+        String normalized = value.trim().replace('-', '_').toUpperCase(java.util.Locale.ROOT);
+        try {
+            return Enum.valueOf(enumType, normalized);
+        } catch (IllegalArgumentException ex) {
+            throw new ResponseStatusException(
+                    HttpStatus.BAD_REQUEST,
+                    "Invalid " + parameterName + ": " + value
+            );
+        }
+    }
+}

src/main/java/at/procon/dip/ingestion/util/DocumentImportSupport.java

@@ -174,7 +174,7 @@ public final class DocumentImportSupport {
 
     public static DocumentFamily familyFor(DocumentType documentType) {
         return switch (documentType) {
-            case TED_PACKAGE, TED_NOTICE -> DocumentFamily.PROCUREMENT;
+            case TED_PACKAGE, TED_NOTICE, TED_NOTICE_LOT -> DocumentFamily.PROCUREMENT;
             case TIME_ENTRY -> DocumentFamily.TIME;
             case EMAIL, MIME_MESSAGE -> DocumentFamily.MAIL;
             case PDF, DOCX, HTML, XML_GENERIC, TEXT, MARKDOWN, ZIP_ARCHIVE, GENERIC_BINARY, UNKNOWN ->

src/main/resources/application-legacy.yml

@@ -119,7 +119,7 @@ ted:
     # Mail account username (email address)
     username: archiv@procon.co.at
     # Mail account password
-    password: ${MAIL_PASSWORD:worasigg}
+    password: ${MAIL_PASSWORD:}
     # Use SSL/TLS connection
     ssl: true
     # Mail folder to read from

src/main/resources/application-new.yml

@@ -43,10 +43,29 @@ dip:
       enabled: true
       jobs:
         enabled: false
-        parallel-batch-count: 1
+        parallel-batch-count: 2
         process-in-batches: true
-        batch-size: 16
-        execution-batch-size: 16
+        batch-size: 48
+        execution-batch-size: 48
+      startup:
+        # Enqueue missing DOC representation embeddings on NEW-runtime startup.
+        enqueue-missing-enabled: true
+        # Also process ready embedding jobs during startup. Requires dip.embedding.jobs.enabled=true.
+        process-ready-enabled: true
+        # Leave empty to enqueue missing embeddings for all document types, or set e.g. TED_NOTICE_LOT.
+        document-type: TED_NOTICE_LOT
+        # Optional representation filter, e.g. SEMANTIC_TEXT.
+        representation-type:
+        # Optional builder filter, e.g. ted-lot-clustering-text-v1.
+        builder-key:
+        primary-only: false
+        # Leave empty to use dip.embedding.default-document-model.
+        model-key:
+        # False skips representations that already have a COMPLETED embedding for the model.
+        force: false
+        batch-size: 1000
+        # 0 means enqueue all matching not-vectorized representations.
+        max-representations-per-run: 0
 
       default-document-model: e5-default
       default-query-model: e5-default
@@ -258,7 +277,7 @@ dip:
       # Mailbox username
       username: archiv@procon.co.at
       # Mailbox password
-      password: ${MAIL_PASSWORD:worasigg}
+      password: ${MAIL_PASSWORD}
       # Folder/mailbox name
       folder-name: INBOX
       # Optional stable provider account key; falls back to username
@@ -350,6 +369,17 @@ dip:
       startup-backfill-limit: 250
       structured-search-hybrid-candidate-limit: 5000
       structured-search-facet-bucket-limit: 12
+      lot-documents:
+        # Materialize one canonical DOC document of type TED_NOTICE_LOT per TED lot.
+        enabled: true
+        # Optional startup/backfill path for notices that were imported before lot documents existed.
+        startup-backfill-enabled: true
+        # Maximum number of legacy TED lot documents to backfill during startup (0 = all)
+        startup-backfill-limit: 0
+        # Queue embeddings whenever the lot semantic text representation is created or changed.
+        queue-embeddings-on-change: true
+        # Include parent notice project description even when the lot already has its own description.
+        include-parent-description: false
 
   migration:
     legacy-audit:
@@ -390,4 +420,4 @@ dip:
       batch-size: 500
       max-documents-per-run: 0
       skip-when-primary-representation-missing: true
-      queue-missing-embeddings: true
+      queue-missing-embeddings: true

src/main/resources/application.yml

@@ -14,12 +14,11 @@ spring:
     name: document-intelligence-platform
 
   datasource:
-    #url: jdbc:postgresql://localhost:5432/RELM
-    #username: ${DB_USERNAME:postgres}
-    #password: ${DB_PASSWORD:P54!pcd#Wi}
-    url: jdbc:postgresql://94.130.218.54:32333/RELM
-    username: ${DB_USERNAME:postgres}
-    password: ${DB_PASSWORD:PDmXRx0Rbk9OFOn9qO5Gm/mPCfqW8zwbZ+/YIU1lySc=}
+    url: jdbc:postgresql://localhost:5432/RELM
+    #url: jdbc:postgresql://94.130.218.54:32333/RELM
+    username: ${DB_USERNAME}
+    password: ${DB_PASSWORD}
+
     driver-class-name: org.postgresql.Driver
     hikari:
       maximum-pool-size: 5
@@ -27,7 +26,7 @@ spring:
       connection-timeout: 30000
       idle-timeout: 300000
       max-lifetime: 900000
-      leak-detection-threshold: 120000  # 2 minutes - increased to avoid false positives with batch processing
+      leak-detection-threshold: 300000  # 5 minutes - increased to avoid false positives with batch processing
 
   jpa:
     hibernate:

src/main/resources/db/migration/V32__ted_lot_documents.sql

@@ -0,0 +1,48 @@
+-- Adds the canonical TED_NOTICE_LOT document type used for per-lot semantic representations.
+-- The lot document stores derived semantic text in DOC.doc_text_representation; no DOC.doc_content row is required.
+
+DO $$
+BEGIN
+    IF EXISTS (
+        SELECT 1
+        FROM pg_type t
+        JOIN pg_namespace n ON n.oid = t.typnamespace
+        WHERE n.nspname = 'doc'
+          AND t.typname = 'doc_document_type'
+    ) THEN
+        ALTER TYPE DOC.doc_document_type ADD VALUE IF NOT EXISTS 'TED_NOTICE_LOT';
+    END IF;
+END
+$$;
+
+DO $$
+BEGIN
+    IF EXISTS (
+        SELECT 1
+        FROM pg_constraint c
+        JOIN pg_class r ON r.oid = c.conrelid
+        JOIN pg_namespace n ON n.oid = r.relnamespace
+        WHERE n.nspname = 'doc'
+          AND r.relname = 'doc_document'
+          AND c.conname = 'doc_document_document_type_check'
+    ) THEN
+        ALTER TABLE DOC.doc_document DROP CONSTRAINT doc_document_document_type_check;
+        ALTER TABLE DOC.doc_document
+            ADD CONSTRAINT doc_document_document_type_check
+            CHECK (
+                document_type IN (
+                    'TED_PACKAGE', 'TED_NOTICE', 'TED_NOTICE_LOT', 'EMAIL', 'MIME_MESSAGE', 'PDF', 'DOCX', 'HTML',
+                    'XML_GENERIC', 'TEXT', 'MARKDOWN', 'ZIP_ARCHIVE', 'GENERIC_BINARY', 'TIME_ENTRY', 'UNKNOWN'
+                )
+            );
+    END IF;
+END
+$$;
+
+CREATE INDEX IF NOT EXISTS idx_doc_document_ted_lot_business_key
+    ON DOC.doc_document(business_key)
+    WHERE document_type = 'TED_NOTICE_LOT';
+
+CREATE INDEX IF NOT EXISTS idx_doc_text_repr_ted_lot_builder
+    ON DOC.doc_text_representation(builder_key)
+    WHERE builder_key = 'ted-lot-structured-text';

src/main/resources/db/migration/V36__ted_lot_clustering_text_builder_index.sql

@@ -0,0 +1,7 @@
+-- Adds an index for the TED lot clustering-oriented semantic representation builder.
+-- The original ted-lot-structured-text index is left in place for databases that
+-- already materialized or queried the earlier representation shape.
+
+CREATE INDEX IF NOT EXISTS idx_doc_text_repr_ted_lot_clustering_builder
+    ON DOC.doc_text_representation(builder_key)
+    WHERE builder_key = 'ted-lot-clustering-text-v1';