ted legacy embeddings migration
trifonovt
4 days ago
parent
28c7854ead
commit
0ce5f51382
@ -1,28 +0,0 @@
|
|||||||
package at.procon.dip;
|
|
||||||
|
|
||||||
import at.procon.ted.config.TedProcessorProperties;
|
|
||||||
import org.springframework.boot.SpringApplication;
|
|
||||||
import org.springframework.boot.autoconfigure.SpringBootApplication;
|
|
||||||
import org.springframework.boot.context.properties.EnableConfigurationProperties;
|
|
||||||
import org.springframework.boot.autoconfigure.domain.EntityScan;
|
|
||||||
import org.springframework.data.jpa.repository.config.EnableJpaRepositories;
|
|
||||||
import org.springframework.scheduling.annotation.EnableAsync;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Procon Document Intelligence Platform (DIP).
|
|
||||||
*
|
|
||||||
* <p>Phase 0 introduces a generic platform root namespace and architecture contracts
|
|
||||||
* while keeping the existing TED-specific runtime intact. Subsequent phases can move
|
|
||||||
* modules incrementally from {@code at.procon.ted} into the broader document platform.</p>
|
|
||||||
*/
|
|
||||||
@SpringBootApplication(scanBasePackages = {"at.procon.dip", "at.procon.ted"})
|
|
||||||
@EnableAsync
|
|
||||||
//@EnableConfigurationProperties(TedProcessorProperties.class)
|
|
||||||
@EntityScan(basePackages = {"at.procon.ted.model.entity"})
|
|
||||||
@EnableJpaRepositories(basePackages = {"at.procon.ted.repository"})
|
|
||||||
public class DocumentIntelligencePlatformApplication {
|
|
||||||
|
|
||||||
public static void main(String[] args) {
|
|
||||||
SpringApplication.run(DocumentIntelligencePlatformApplication.class, args);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@ -0,0 +1,29 @@
|
|||||||
|
package at.procon.dip.migration.config;
|
||||||
|
|
||||||
|
import lombok.Data;
|
||||||
|
import org.springframework.boot.context.properties.ConfigurationProperties;
|
||||||
|
import org.springframework.context.annotation.Configuration;
|
||||||
|
|
||||||
|
@Configuration
|
||||||
|
@ConfigurationProperties(prefix = "dip.migration.legacy-ted-embeddings")
|
||||||
|
@Data
|
||||||
|
public class LegacyTedEmbeddingBackfillProperties {
|
||||||
|
|
||||||
|
/** Enable the legacy TED embedding-only backfill subsystem. */
|
||||||
|
private boolean enabled = false;
|
||||||
|
|
||||||
|
/** Run the embedding-only backfill automatically on application startup in NEW runtime. */
|
||||||
|
private boolean startupEnabled = false;
|
||||||
|
|
||||||
|
/** Number of legacy TED documents to inspect per cursor batch. */
|
||||||
|
private int batchSize = 500;
|
||||||
|
|
||||||
|
/** Optional cap for a single invocation. 0 or negative means unlimited. */
|
||||||
|
private long maxDocumentsPerRun = 0;
|
||||||
|
|
||||||
|
/** Skip legacy TED rows that do not yet have a migrated primary SEMANTIC_TEXT representation. */
|
||||||
|
private boolean skipWhenPrimaryRepresentationMissing = true;
|
||||||
|
|
||||||
|
/** Queue a fresh embedding job when no legacy vector exists for a migrated document. */
|
||||||
|
private boolean queueMissingEmbeddings = false;
|
||||||
|
}
|
||||||
@ -0,0 +1,49 @@
|
|||||||
|
package at.procon.dip.migration.repository;
|
||||||
|
|
||||||
|
import at.procon.dip.migration.service.LegacyTedEmbeddingTarget;
|
||||||
|
import jakarta.persistence.EntityManager;
|
||||||
|
import jakarta.persistence.Query;
|
||||||
|
import java.util.ArrayList;
|
||||||
|
import java.util.Collection;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.UUID;
|
||||||
|
import lombok.RequiredArgsConstructor;
|
||||||
|
import org.springframework.stereotype.Repository;
|
||||||
|
|
||||||
|
@Repository
|
||||||
|
@RequiredArgsConstructor
|
||||||
|
public class LegacyTedEmbeddingTargetQueryRepository {
|
||||||
|
|
||||||
|
private final EntityManager entityManager;
|
||||||
|
|
||||||
|
@SuppressWarnings("unchecked")
|
||||||
|
public List<LegacyTedEmbeddingTarget> findPrimarySemanticTargetsByLegacyIds(Collection<UUID> legacyIds) {
|
||||||
|
if (legacyIds == null || legacyIds.isEmpty()) {
|
||||||
|
return List.of();
|
||||||
|
}
|
||||||
|
|
||||||
|
Query query = entityManager.createNativeQuery("""
|
||||||
|
SELECT p.legacy_procurement_document_id AS legacy_procurement_document_id,
|
||||||
|
p.document_id AS document_id,
|
||||||
|
r.id AS representation_id
|
||||||
|
FROM ted.ted_notice_projection p
|
||||||
|
JOIN doc.doc_text_representation r
|
||||||
|
ON r.document_id = p.document_id
|
||||||
|
WHERE p.legacy_procurement_document_id IN (:legacyIds)
|
||||||
|
AND r.representation_type = 'SEMANTIC_TEXT'
|
||||||
|
AND COALESCE(r.is_primary, FALSE) = TRUE
|
||||||
|
""");
|
||||||
|
query.setParameter("legacyIds", legacyIds);
|
||||||
|
|
||||||
|
List<Object[]> rows = query.getResultList();
|
||||||
|
List<LegacyTedEmbeddingTarget> results = new ArrayList<>(rows.size());
|
||||||
|
for (Object[] row : rows) {
|
||||||
|
results.add(new LegacyTedEmbeddingTarget(
|
||||||
|
(UUID) row[0],
|
||||||
|
(UUID) row[1],
|
||||||
|
(UUID) row[2]
|
||||||
|
));
|
||||||
|
}
|
||||||
|
return results;
|
||||||
|
}
|
||||||
|
}
|
||||||
@ -0,0 +1,188 @@
|
|||||||
|
package at.procon.dip.migration.service;
|
||||||
|
|
||||||
|
import at.procon.dip.embedding.config.EmbeddingProperties;
|
||||||
|
import at.procon.dip.embedding.service.EmbeddingPersistenceService;
|
||||||
|
import at.procon.dip.embedding.service.RepresentationEmbeddingOrchestrator;
|
||||||
|
import at.procon.dip.migration.config.LegacyTedEmbeddingBackfillProperties;
|
||||||
|
import at.procon.dip.migration.repository.LegacyTedEmbeddingTargetQueryRepository;
|
||||||
|
import at.procon.dip.runtime.condition.ConditionalOnRuntimeMode;
|
||||||
|
import at.procon.dip.runtime.config.RuntimeMode;
|
||||||
|
import at.procon.ted.repository.LegacyTedEmbeddingSnapshot;
|
||||||
|
import at.procon.ted.repository.LegacyTedMigrationCursor;
|
||||||
|
import at.procon.ted.repository.ProcurementDocumentRepository;
|
||||||
|
import java.time.Instant;
|
||||||
|
import java.time.OffsetDateTime;
|
||||||
|
import java.time.ZoneId;
|
||||||
|
import java.util.ArrayList;
|
||||||
|
import java.util.Collection;
|
||||||
|
import java.util.LinkedHashMap;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.Map;
|
||||||
|
import java.util.UUID;
|
||||||
|
import lombok.RequiredArgsConstructor;
|
||||||
|
import lombok.extern.slf4j.Slf4j;
|
||||||
|
import org.springframework.stereotype.Service;
|
||||||
|
import org.springframework.util.StringUtils;
|
||||||
|
|
||||||
|
@Service
|
||||||
|
@ConditionalOnRuntimeMode(RuntimeMode.NEW)
|
||||||
|
@RequiredArgsConstructor
|
||||||
|
@Slf4j
|
||||||
|
public class LegacyTedEmbeddingBackfillService {
|
||||||
|
|
||||||
|
private final LegacyTedEmbeddingBackfillProperties properties;
|
||||||
|
private final ProcurementDocumentRepository procurementDocumentRepository;
|
||||||
|
private final LegacyTedEmbeddingTargetQueryRepository targetQueryRepository;
|
||||||
|
private final EmbeddingPersistenceService embeddingPersistenceService;
|
||||||
|
private final RepresentationEmbeddingOrchestrator embeddingOrchestrator;
|
||||||
|
private final EmbeddingProperties embeddingProperties;
|
||||||
|
|
||||||
|
public void runBackfill() {
|
||||||
|
if (!properties.isEnabled()) {
|
||||||
|
log.info("Legacy TED embedding-only backfill is disabled");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (!embeddingProperties.isEnabled() || !StringUtils.hasText(embeddingProperties.getDefaultDocumentModel())) {
|
||||||
|
log.warn("Skipping legacy TED embedding-only backfill because no default document model is configured");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
log.info("Starting legacy TED embedding-only backfill (batchSize={}, maxDocumentsPerRun={}, queueMissingEmbeddings={})",
|
||||||
|
properties.getBatchSize(), properties.getMaxDocumentsPerRun(), properties.isQueueMissingEmbeddings());
|
||||||
|
|
||||||
|
Instant lastCreatedAt = null;
|
||||||
|
UUID lastId = null;
|
||||||
|
long inspected = 0;
|
||||||
|
long migrated = 0;
|
||||||
|
long queued = 0;
|
||||||
|
long skippedMissingTarget = 0;
|
||||||
|
long skippedMissingVector = 0;
|
||||||
|
|
||||||
|
while (true) {
|
||||||
|
int limit = effectiveBatchLimit(inspected);
|
||||||
|
if (limit <= 0) {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
|
List<LegacyTedMigrationCursor> cursors = loadNextBatch(
|
||||||
|
lastCreatedAt != null ?
|
||||||
|
lastCreatedAt.atZone(ZoneId.systemDefault()).toOffsetDateTime() : null, lastId, limit);
|
||||||
|
if (cursors.isEmpty()) {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
|
List<UUID> legacyIds = new ArrayList<>(cursors.size());
|
||||||
|
for (LegacyTedMigrationCursor cursor : cursors) {
|
||||||
|
legacyIds.add(cursor.getId());
|
||||||
|
}
|
||||||
|
|
||||||
|
Map<UUID, LegacyTedEmbeddingTarget> targetsByLegacyId = indexTargets(
|
||||||
|
targetQueryRepository.findPrimarySemanticTargetsByLegacyIds(legacyIds)
|
||||||
|
);
|
||||||
|
Map<UUID, LegacyTedEmbeddingSnapshot> snapshotsByLegacyId = indexSnapshots(
|
||||||
|
procurementDocumentRepository.findEmbeddingSnapshotsByIdsForMigration(legacyIds)
|
||||||
|
);
|
||||||
|
|
||||||
|
for (LegacyTedMigrationCursor cursor : cursors) {
|
||||||
|
inspected++;
|
||||||
|
LegacyTedEmbeddingTarget target = targetsByLegacyId.get(cursor.getId());
|
||||||
|
LegacyTedEmbeddingSnapshot snapshot = snapshotsByLegacyId.get(cursor.getId());
|
||||||
|
|
||||||
|
if (target == null) {
|
||||||
|
if (properties.isSkipWhenPrimaryRepresentationMissing()) {
|
||||||
|
skippedMissingTarget++;
|
||||||
|
}
|
||||||
|
lastCreatedAt = cursor.getCreatedAt();
|
||||||
|
lastId = cursor.getId();
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (snapshot != null && StringUtils.hasText(snapshot.getVectorText())) {
|
||||||
|
float[] vector = parseVectorText(snapshot.getVectorText());
|
||||||
|
if (vector.length > 0) {
|
||||||
|
var embedding = embeddingPersistenceService.ensurePending(target.representationId(), embeddingProperties.getDefaultDocumentModel());
|
||||||
|
embeddingPersistenceService.saveCompleted(embedding.getId(), vector, snapshot.getTokenCount());
|
||||||
|
migrated++;
|
||||||
|
} else {
|
||||||
|
skippedMissingVector++;
|
||||||
|
}
|
||||||
|
} else if (properties.isQueueMissingEmbeddings()) {
|
||||||
|
embeddingOrchestrator.enqueueRepresentation(
|
||||||
|
target.documentId(),
|
||||||
|
target.representationId(),
|
||||||
|
embeddingProperties.getDefaultDocumentModel()
|
||||||
|
);
|
||||||
|
queued++;
|
||||||
|
} else {
|
||||||
|
skippedMissingVector++;
|
||||||
|
}
|
||||||
|
|
||||||
|
lastCreatedAt = cursor.getCreatedAt();
|
||||||
|
lastId = cursor.getId();
|
||||||
|
}
|
||||||
|
|
||||||
|
log.info("Legacy TED embedding-only backfill progress: inspected={}, migrated={}, queued={}, skippedMissingTarget={}, skippedMissingVector={}",
|
||||||
|
inspected, migrated, queued, skippedMissingTarget, skippedMissingVector);
|
||||||
|
}
|
||||||
|
|
||||||
|
log.info("Legacy TED embedding-only backfill finished: inspected={}, migrated={}, queued={}, skippedMissingTarget={}, skippedMissingVector={}",
|
||||||
|
inspected, migrated, queued, skippedMissingTarget, skippedMissingVector);
|
||||||
|
}
|
||||||
|
|
||||||
|
protected List<LegacyTedMigrationCursor> loadNextBatch(OffsetDateTime lastCreatedAt, UUID lastId, int limit) {
|
||||||
|
if (lastCreatedAt == null || lastId == null) {
|
||||||
|
return procurementDocumentRepository.findFirstMigrationBatch(limit);
|
||||||
|
}
|
||||||
|
return procurementDocumentRepository.findNextMigrationBatch(lastCreatedAt, lastId, limit);
|
||||||
|
}
|
||||||
|
|
||||||
|
private int effectiveBatchLimit(long processedInThisInvocation) {
|
||||||
|
long maxPerRun = properties.getMaxDocumentsPerRun();
|
||||||
|
if (maxPerRun <= 0) {
|
||||||
|
return Math.max(1, properties.getBatchSize());
|
||||||
|
}
|
||||||
|
long remaining = maxPerRun - processedInThisInvocation;
|
||||||
|
if (remaining <= 0) {
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
return (int) Math.max(1L, Math.min(properties.getBatchSize(), remaining));
|
||||||
|
}
|
||||||
|
|
||||||
|
private Map<UUID, LegacyTedEmbeddingTarget> indexTargets(Collection<LegacyTedEmbeddingTarget> targets) {
|
||||||
|
Map<UUID, LegacyTedEmbeddingTarget> indexed = new LinkedHashMap<>();
|
||||||
|
for (LegacyTedEmbeddingTarget target : targets) {
|
||||||
|
indexed.putIfAbsent(target.legacyProcurementDocumentId(), target);
|
||||||
|
}
|
||||||
|
return indexed;
|
||||||
|
}
|
||||||
|
|
||||||
|
private Map<UUID, LegacyTedEmbeddingSnapshot> indexSnapshots(Collection<LegacyTedEmbeddingSnapshot> snapshots) {
|
||||||
|
Map<UUID, LegacyTedEmbeddingSnapshot> indexed = new LinkedHashMap<>();
|
||||||
|
for (LegacyTedEmbeddingSnapshot snapshot : snapshots) {
|
||||||
|
indexed.put(snapshot.getId(), snapshot);
|
||||||
|
}
|
||||||
|
return indexed;
|
||||||
|
}
|
||||||
|
|
||||||
|
private float[] parseVectorText(String vectorText) {
|
||||||
|
if (!StringUtils.hasText(vectorText)) {
|
||||||
|
return new float[0];
|
||||||
|
}
|
||||||
|
String trimmed = vectorText.trim();
|
||||||
|
if (trimmed.startsWith("[")) {
|
||||||
|
trimmed = trimmed.substring(1);
|
||||||
|
}
|
||||||
|
if (trimmed.endsWith("]")) {
|
||||||
|
trimmed = trimmed.substring(0, trimmed.length() - 1);
|
||||||
|
}
|
||||||
|
if (!StringUtils.hasText(trimmed)) {
|
||||||
|
return new float[0];
|
||||||
|
}
|
||||||
|
String[] parts = trimmed.split(",");
|
||||||
|
float[] result = new float[parts.length];
|
||||||
|
for (int i = 0; i < parts.length; i++) {
|
||||||
|
result[i] = Float.parseFloat(parts[i].trim());
|
||||||
|
}
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
}
|
||||||
@ -0,0 +1,8 @@
|
|||||||
|
package at.procon.dip.migration.service;
|
||||||
|
|
||||||
|
import java.util.UUID;
|
||||||
|
|
||||||
|
public record LegacyTedEmbeddingTarget(UUID legacyProcurementDocumentId,
|
||||||
|
UUID documentId,
|
||||||
|
UUID representationId) {
|
||||||
|
}
|
||||||
@ -0,0 +1,33 @@
|
|||||||
|
package at.procon.dip.migration.startup;
|
||||||
|
|
||||||
|
import at.procon.dip.migration.config.LegacyTedEmbeddingBackfillProperties;
|
||||||
|
import at.procon.dip.migration.service.LegacyTedEmbeddingBackfillService;
|
||||||
|
import at.procon.dip.runtime.condition.ConditionalOnRuntimeMode;
|
||||||
|
import at.procon.dip.runtime.config.RuntimeMode;
|
||||||
|
import lombok.RequiredArgsConstructor;
|
||||||
|
import lombok.extern.slf4j.Slf4j;
|
||||||
|
import org.springframework.boot.ApplicationArguments;
|
||||||
|
import org.springframework.boot.ApplicationRunner;
|
||||||
|
import org.springframework.core.Ordered;
|
||||||
|
import org.springframework.core.annotation.Order;
|
||||||
|
import org.springframework.stereotype.Component;
|
||||||
|
|
||||||
|
@Component
|
||||||
|
@ConditionalOnRuntimeMode(RuntimeMode.NEW)
|
||||||
|
@RequiredArgsConstructor
|
||||||
|
@Order(Ordered.LOWEST_PRECEDENCE)
|
||||||
|
@Slf4j
|
||||||
|
public class LegacyTedEmbeddingBackfillStartupRunner implements ApplicationRunner {
|
||||||
|
|
||||||
|
private final LegacyTedEmbeddingBackfillProperties properties;
|
||||||
|
private final LegacyTedEmbeddingBackfillService backfillService;
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void run(ApplicationArguments args) {
|
||||||
|
if (!properties.isEnabled() || !properties.isStartupEnabled()) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
log.info("Startup-triggered legacy TED embedding-only backfill is enabled");
|
||||||
|
backfillService.runBackfill();
|
||||||
|
}
|
||||||
|
}
|
||||||
@ -0,0 +1,15 @@
|
|||||||
|
package at.procon.ted.repository;
|
||||||
|
|
||||||
|
import java.util.UUID;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Lightweight projection for migrating legacy TED notice embeddings into the new DOC embedding model.
|
||||||
|
*/
|
||||||
|
public interface LegacyTedEmbeddingSnapshot {
|
||||||
|
|
||||||
|
UUID getId();
|
||||||
|
|
||||||
|
String getVectorText();
|
||||||
|
|
||||||
|
Integer getTokenCount();
|
||||||
|
}
|
||||||
@ -0,0 +1,7 @@
|
|||||||
|
SET search_path TO TED, DOC, public;
|
||||||
|
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_ted_procurement_document_created_at_id
|
||||||
|
ON TED.procurement_document (created_at ASC, id ASC);
|
||||||
|
|
||||||
|
ALTER TABLE DOC.doc_legacy_ted_migration_run
|
||||||
|
ADD COLUMN IF NOT EXISTS build_chunk_representations boolean NOT NULL DEFAULT true;
|
||||||
Loading…
Reference in New Issue