clustering
This commit is contained in:
trifonovt
parent
c6efbf40f6
commit
979f1ba18e
|
|
@ -0,0 +1,40 @@
|
|||
# TIME selective materialization by person
|
||||
|
||||
This NEW-only patch adds the ability to materialize canonical `TIME.time_entry` rows and refresh projection/representations only for Leitstand time recordings that belong to one selected person after the raw `TIME.ls_*` import is already present.
|
||||
|
||||
## Service methods
|
||||
|
||||
- `LeitstandTimeImportService.materializeCanonicalTimeEntriesForPersonDbk(String personDbk, boolean rebuildProjection)`
|
||||
- `LeitstandTimeImportService.materializeCanonicalTimeEntriesForPersonNumber(Integer personNumber, boolean rebuildProjection)`
|
||||
- `LeitstandTimeProjectionService.refreshForPersonDbk(String personDbk)`
|
||||
|
||||
## Optional startup runner
|
||||
|
||||
Enable with:
|
||||
|
||||
```yaml
|
||||
dip:
|
||||
time:
|
||||
leitstand:
|
||||
startup-selective-materialization-enabled: true
|
||||
selective-materialization-person-dbk: 100919970619190804070001
|
||||
selective-materialization-build-projection: true
|
||||
```
|
||||
|
||||
or:
|
||||
|
||||
```yaml
|
||||
dip:
|
||||
time:
|
||||
leitstand:
|
||||
startup-selective-materialization-enabled: true
|
||||
selective-materialization-person-number: 12345
|
||||
selective-materialization-build-projection: true
|
||||
```
|
||||
|
||||
## Notes
|
||||
|
||||
- intended for already imported `TIME.ls_*` rows
|
||||
- no legacy code changes
|
||||
- no raw source sync is triggered by this runner
|
||||
- if projection rebuild is enabled, representations/embedding enqueueing continue to use the existing T3 behavior
|
||||
|
|
@ -0,0 +1,57 @@
|
|||
# Python clustering backend for DBSCAN and advanced algorithms
|
||||
|
||||
This patch adds a dedicated Python service for clustering algorithms that are better supported in the Python scientific stack than in Java.
|
||||
|
||||
## Why Python for this step
|
||||
|
||||
The Spring module remains the orchestrator for:
|
||||
- embedding selection
|
||||
- run metadata
|
||||
- result persistence
|
||||
- cluster browsing APIs
|
||||
|
||||
The Python backend executes the actual clustering for algorithms such as:
|
||||
- `DBSCAN`
|
||||
- `HDBSCAN`
|
||||
- `MINI_BATCH_KMEANS`
|
||||
- `AGGLOMERATIVE`
|
||||
- `KMEANS` with optional reduction
|
||||
|
||||
## Spring-side contract changes in this patch
|
||||
|
||||
The Spring request model now supports generic algorithm parameters through `parameters` instead of only `k`.
|
||||
|
||||
Examples:
|
||||
- KMeans: `{ "k": 25 }`
|
||||
- DBSCAN: `{ "eps": 0.25, "minSamples": 5 }`
|
||||
- HDBSCAN: `{ "minClusterSize": 15, "minSamples": 5 }`
|
||||
- Agglomerative: `{ "k": 20, "linkage": "average", "metric": "euclidean" }`
|
||||
|
||||
The Python response is now mapped with:
|
||||
- `noise`
|
||||
- `membershipScore`
|
||||
- `distanceToCentroid`
|
||||
- noise cluster rows
|
||||
- `noiseCount`
|
||||
|
||||
Those values are persisted back into:
|
||||
- `doc.doc_embedding_cluster`
|
||||
- `doc.doc_embedding_cluster_assignment`
|
||||
- `doc.doc_embedding_cluster_run`
|
||||
|
||||
## Recommended defaults for embeddings
|
||||
|
||||
For high-dimensional text embeddings, use:
|
||||
- `normalizeVectors=true`
|
||||
- `reductionMethod=PCA`
|
||||
- `reductionDimensions=50..150`
|
||||
|
||||
Typical starting points:
|
||||
- DBSCAN: `eps=0.20..0.35`, `minSamples=5`
|
||||
- HDBSCAN: `minClusterSize=10..30`, `minSamples=3..10`
|
||||
|
||||
The right values still depend on:
|
||||
- embedding model
|
||||
- whether vectors are normalized
|
||||
- whether full documents or chunks are clustered
|
||||
- the semantic density of the selected dataset
|
||||
|
|
@ -0,0 +1,100 @@
|
|||
{
|
||||
"info": {
|
||||
"name": "DIP Clustering Dual Python Modes",
|
||||
"_postman_id": "4c2f4f68-c9c3-4977-9627-b4f7422dd001",
|
||||
"schema": "https://schema.getpostman.com/json/collection/v2.1.0/collection.json",
|
||||
"description": "Direct vector-upload and compact runId Python endpoints."
|
||||
},
|
||||
"variable": [
|
||||
{
|
||||
"key": "pythonBaseUrl",
|
||||
"value": "http://localhost:8001"
|
||||
},
|
||||
{
|
||||
"key": "runId",
|
||||
"value": ""
|
||||
}
|
||||
],
|
||||
"item": [
|
||||
{
|
||||
"name": "Health",
|
||||
"item": [
|
||||
{
|
||||
"name": "GET /health",
|
||||
"request": {
|
||||
"method": "GET",
|
||||
"url": {
|
||||
"raw": "{{pythonBaseUrl}}/health",
|
||||
"host": [
|
||||
"{{pythonBaseUrl}}"
|
||||
],
|
||||
"path": [
|
||||
"health"
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "Direct vector upload",
|
||||
"item": [
|
||||
{
|
||||
"name": "POST /cluster",
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"header": [
|
||||
{
|
||||
"key": "Content-Type",
|
||||
"value": "application/json"
|
||||
}
|
||||
],
|
||||
"url": {
|
||||
"raw": "{{pythonBaseUrl}}/cluster",
|
||||
"host": [
|
||||
"{{pythonBaseUrl}}"
|
||||
],
|
||||
"path": [
|
||||
"cluster"
|
||||
]
|
||||
},
|
||||
"body": {
|
||||
"mode": "raw",
|
||||
"raw": "{\n \"algorithm\": \"KMEANS\",\n \"parameters\": {\n \"k\": 2,\n \"normalizeVectors\": true\n },\n \"reductionMethod\": \"NONE\",\n \"reductionDimensions\": null,\n \"items\": [\n {\n \"embeddingId\": \"11111111-1111-1111-1111-111111111111\",\n \"documentId\": \"22222222-2222-2222-2222-222222222222\",\n \"representationId\": \"33333333-3333-3333-3333-333333333333\",\n \"vector\": [\n 0.1,\n 0.2,\n 0.3\n ]\n },\n {\n \"embeddingId\": \"11111111-1111-1111-1111-111111111112\",\n \"documentId\": \"22222222-2222-2222-2222-222222222223\",\n \"representationId\": \"33333333-3333-3333-3333-333333333334\",\n \"vector\": [\n 0.11,\n 0.19,\n 0.31\n ]\n }\n ]\n}"
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "Compact runId mode",
|
||||
"item": [
|
||||
{
|
||||
"name": "POST /cluster-run",
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"header": [
|
||||
{
|
||||
"key": "Content-Type",
|
||||
"value": "application/json"
|
||||
}
|
||||
],
|
||||
"url": {
|
||||
"raw": "{{pythonBaseUrl}}/cluster-run",
|
||||
"host": [
|
||||
"{{pythonBaseUrl}}"
|
||||
],
|
||||
"path": [
|
||||
"cluster-run"
|
||||
]
|
||||
},
|
||||
"body": {
|
||||
"mode": "raw",
|
||||
"raw": "{\n \"runId\": \"{{runId}}\"\n}"
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
|
|
@ -0,0 +1,131 @@
|
|||
{
|
||||
"info": {
|
||||
"name": "DIP Clustering Phase E Compact Run",
|
||||
"_postman_id": "57e745df-cb97-4a13-8c74-9e5c689ef0ac",
|
||||
"schema": "https://schema.getpostman.com/json/collection/v2.1.0/collection.json",
|
||||
"description": "Phase E compact run execution: Spring keeps metadata, Python receives only runId."
|
||||
},
|
||||
"variable": [
|
||||
{
|
||||
"key": "springBaseUrl",
|
||||
"value": "http://localhost:8889/api"
|
||||
},
|
||||
{
|
||||
"key": "pythonBaseUrl",
|
||||
"value": "http://localhost:8001"
|
||||
},
|
||||
{
|
||||
"key": "runId",
|
||||
"value": ""
|
||||
}
|
||||
],
|
||||
"item": [
|
||||
{
|
||||
"name": "Python Health",
|
||||
"request": {
|
||||
"method": "GET",
|
||||
"url": {
|
||||
"raw": "{{pythonBaseUrl}}/health",
|
||||
"host": [
|
||||
"{{pythonBaseUrl}}"
|
||||
],
|
||||
"path": [
|
||||
"health"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Python Cluster Run",
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"header": [
|
||||
{
|
||||
"key": "Content-Type",
|
||||
"value": "application/json"
|
||||
}
|
||||
],
|
||||
"url": {
|
||||
"raw": "{{pythonBaseUrl}}/cluster-run",
|
||||
"host": [
|
||||
"{{pythonBaseUrl}}"
|
||||
],
|
||||
"path": [
|
||||
"cluster-run"
|
||||
]
|
||||
},
|
||||
"body": {
|
||||
"mode": "raw",
|
||||
"raw": "{\n \"runId\": \"{{runId}}\"\n}"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Create TED DBSCAN run",
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"header": [
|
||||
{
|
||||
"key": "Content-Type",
|
||||
"value": "application/json"
|
||||
}
|
||||
],
|
||||
"url": {
|
||||
"raw": "{{springBaseUrl}}/v1/dip/clustering/runs",
|
||||
"host": [
|
||||
"{{springBaseUrl}}"
|
||||
],
|
||||
"path": [
|
||||
"v1",
|
||||
"dip",
|
||||
"clustering",
|
||||
"runs"
|
||||
]
|
||||
},
|
||||
"body": {
|
||||
"mode": "raw",
|
||||
"raw": "{\n \"name\": \"TED notices DBSCAN PCA200\",\n \"algorithm\": \"DBSCAN\",\n \"executionBackend\": \"PYTHON_REMOTE\",\n \"reduction\": {\n \"method\": \"PCA\",\n \"targetDimensions\": 200\n },\n \"selection\": {\n \"documentTypes\": [\n \"TED_NOTICE\"\n ],\n \"representationTypes\": [\n \"SEMANTIC_TEXT\"\n ],\n \"embeddingStatuses\": [\n \"COMPLETED\"\n ],\n \"primaryRepresentationOnly\": true\n },\n \"parameters\": {\n \"eps\": 0.25,\n \"minSamples\": 5,\n \"metric\": \"euclidean\",\n \"normalizeVectors\": true\n }\n}"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Queue Run",
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"url": {
|
||||
"raw": "{{springBaseUrl}}/v1/dip/clustering/runs/{{runId}}/start",
|
||||
"host": [
|
||||
"{{springBaseUrl}}"
|
||||
],
|
||||
"path": [
|
||||
"v1",
|
||||
"dip",
|
||||
"clustering",
|
||||
"runs",
|
||||
"{{runId}}",
|
||||
"start"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Get Run",
|
||||
"request": {
|
||||
"method": "GET",
|
||||
"url": {
|
||||
"raw": "{{springBaseUrl}}/v1/dip/clustering/runs/{{runId}}",
|
||||
"host": [
|
||||
"{{springBaseUrl}}"
|
||||
],
|
||||
"path": [
|
||||
"v1",
|
||||
"dip",
|
||||
"clustering",
|
||||
"runs",
|
||||
"{{runId}}"
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
|
|
@ -0,0 +1,157 @@
|
|||
{
|
||||
"info": {
|
||||
"name": "DIP Clustering Phase C",
|
||||
"_postman_id": "fa4b1e24-7d8d-4b1a-bd67-0f3f1b601111",
|
||||
"description": "Operational Postman collection for clustering sets, async runs, cancellation, and text-aware result inspection.",
|
||||
"schema": "https://schema.getpostman.com/json/collection/v2.1.0/collection.json"
|
||||
},
|
||||
"variable": [
|
||||
{
|
||||
"key": "baseUrl",
|
||||
"value": "http://localhost:8080/api"
|
||||
},
|
||||
{
|
||||
"key": "runId",
|
||||
"value": ""
|
||||
},
|
||||
{
|
||||
"key": "clusterSetId",
|
||||
"value": ""
|
||||
},
|
||||
{
|
||||
"key": "clusterId",
|
||||
"value": ""
|
||||
}
|
||||
],
|
||||
"item": [
|
||||
{
|
||||
"name": "Cluster Sets",
|
||||
"item": [
|
||||
{
|
||||
"name": "Create TED cluster set",
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"header": [
|
||||
{
|
||||
"key": "Content-Type",
|
||||
"value": "application/json"
|
||||
}
|
||||
],
|
||||
"url": "{{baseUrl}}/v1/dip/clustering/sets",
|
||||
"body": {
|
||||
"mode": "raw",
|
||||
"raw": "{\n \"code\": \"TED_NOTICE_E5_PRIMARY\",\n \"name\": \"TED notices primary semantic text\",\n \"description\": \"Saved TED notice clustering selection\",\n \"selection\": {\n \"documentTypes\": [\n \"TED_NOTICE\"\n ],\n \"representationTypes\": [\n \"SEMANTIC_TEXT\"\n ],\n \"embeddingStatuses\": [\n \"COMPLETED\"\n ],\n \"primaryRepresentationOnly\": true\n },\n \"active\": true\n}"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "List cluster sets",
|
||||
"request": {
|
||||
"method": "GET",
|
||||
"url": "{{baseUrl}}/v1/dip/clustering/sets"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Get cluster set",
|
||||
"request": {
|
||||
"method": "GET",
|
||||
"url": "{{baseUrl}}/v1/dip/clustering/sets/{{clusterSetId}}"
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "Runs",
|
||||
"item": [
|
||||
{
|
||||
"name": "Create TED KMeans run",
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"header": [
|
||||
{
|
||||
"key": "Content-Type",
|
||||
"value": "application/json"
|
||||
}
|
||||
],
|
||||
"url": "{{baseUrl}}/v1/dip/clustering/runs",
|
||||
"body": {
|
||||
"mode": "raw",
|
||||
"raw": "{\n \"name\": \"TED notices KMeans async run 1\",\n \"algorithm\": \"KMEANS\",\n \"executionBackend\": \"JAVA_LOCAL\",\n \"reduction\": {\n \"method\": \"NONE\"\n },\n \"selection\": {\n \"documentTypes\": [\n \"TED_NOTICE\"\n ],\n \"representationTypes\": [\n \"SEMANTIC_TEXT\"\n ],\n \"embeddingStatuses\": [\n \"COMPLETED\"\n ],\n \"primaryRepresentationOnly\": true\n },\n \"k\": 25\n}"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Create HDBSCAN Python run",
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"header": [
|
||||
{
|
||||
"key": "Content-Type",
|
||||
"value": "application/json"
|
||||
}
|
||||
],
|
||||
"url": "{{baseUrl}}/v1/dip/clustering/runs",
|
||||
"body": {
|
||||
"mode": "raw",
|
||||
"raw": "{\n \"name\": \"TED notices HDBSCAN PCA100\",\n \"algorithm\": \"HDBSCAN\",\n \"executionBackend\": \"PYTHON_REMOTE\",\n \"reduction\": {\n \"method\": \"PCA\",\n \"targetDimensions\": 100\n },\n \"selection\": {\n \"documentTypes\": [\n \"TED_NOTICE\"\n ],\n \"representationTypes\": [\n \"SEMANTIC_TEXT\"\n ],\n \"embeddingStatuses\": [\n \"COMPLETED\"\n ],\n \"primaryRepresentationOnly\": true\n },\n \"k\": 25\n}"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Start run async",
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"url": "{{baseUrl}}/v1/dip/clustering/runs/{{runId}}/start"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Cancel run",
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"url": "{{baseUrl}}/v1/dip/clustering/runs/{{runId}}/cancel"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Get run",
|
||||
"request": {
|
||||
"method": "GET",
|
||||
"url": "{{baseUrl}}/v1/dip/clustering/runs/{{runId}}"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "List runs",
|
||||
"request": {
|
||||
"method": "GET",
|
||||
"url": "{{baseUrl}}/v1/dip/clustering/runs?status=COMPLETED"
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "Results",
|
||||
"item": [
|
||||
{
|
||||
"name": "List clusters",
|
||||
"request": {
|
||||
"method": "GET",
|
||||
"url": "{{baseUrl}}/v1/dip/clustering/runs/{{runId}}/clusters"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Assignments with text",
|
||||
"request": {
|
||||
"method": "GET",
|
||||
"url": "{{baseUrl}}/v1/dip/clustering/runs/{{runId}}/assignments?includeText=true"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Cluster members with text",
|
||||
"request": {
|
||||
"method": "GET",
|
||||
"url": "{{baseUrl}}/v1/dip/clustering/runs/{{runId}}/clusters/{{clusterId}}/members?includeText=true"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
|
|
@ -0,0 +1,150 @@
|
|||
{
|
||||
"info": {
|
||||
"name": "DIP Clustering Phase D",
|
||||
"_postman_id": "0c39a7cf-8fde-43f9-8fdb-5d8890ad7676",
|
||||
"description": "Spring clustering API examples with generic algorithm parameters and remote Python backend.",
|
||||
"schema": "https://schema.getpostman.com/json/collection/v2.1.0/collection.json"
|
||||
},
|
||||
"variable": [
|
||||
{
|
||||
"key": "baseUrl",
|
||||
"value": "http://localhost:8080"
|
||||
},
|
||||
{
|
||||
"key": "runId",
|
||||
"value": ""
|
||||
}
|
||||
],
|
||||
"item": [
|
||||
{
|
||||
"name": "Create DBSCAN run for TED notices",
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"header": [
|
||||
{
|
||||
"key": "Content-Type",
|
||||
"value": "application/json"
|
||||
}
|
||||
],
|
||||
"url": {
|
||||
"raw": "{{baseUrl}}/v1/dip/clustering/runs",
|
||||
"host": [
|
||||
"{{baseUrl}}"
|
||||
],
|
||||
"path": [
|
||||
"v1",
|
||||
"dip",
|
||||
"clustering",
|
||||
"runs"
|
||||
]
|
||||
},
|
||||
"body": {
|
||||
"mode": "raw",
|
||||
"raw": "{\n \"name\": \"TED notices DBSCAN PCA100\",\n \"algorithm\": \"DBSCAN\",\n \"executionBackend\": \"PYTHON_REMOTE\",\n \"reduction\": {\n \"method\": \"PCA\",\n \"targetDimensions\": 100\n },\n \"selection\": {\n \"documentTypes\": [\n \"TED_NOTICE\"\n ],\n \"representationTypes\": [\n \"SEMANTIC_TEXT\"\n ],\n \"embeddingStatuses\": [\n \"COMPLETED\"\n ],\n \"primaryRepresentationOnly\": true\n },\n \"parameters\": {\n \"eps\": 0.25,\n \"minSamples\": 5,\n \"metric\": \"euclidean\",\n \"normalizeVectors\": true\n }\n}"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Create HDBSCAN run for Leitstand TIME",
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"header": [
|
||||
{
|
||||
"key": "Content-Type",
|
||||
"value": "application/json"
|
||||
}
|
||||
],
|
||||
"url": {
|
||||
"raw": "{{baseUrl}}/v1/dip/clustering/runs",
|
||||
"host": [
|
||||
"{{baseUrl}}"
|
||||
],
|
||||
"path": [
|
||||
"v1",
|
||||
"dip",
|
||||
"clustering",
|
||||
"runs"
|
||||
]
|
||||
},
|
||||
"body": {
|
||||
"mode": "raw",
|
||||
"raw": "{\n \"name\": \"Leitstand TIME HDBSCAN PCA50\",\n \"algorithm\": \"HDBSCAN\",\n \"executionBackend\": \"PYTHON_REMOTE\",\n \"reduction\": {\n \"method\": \"PCA\",\n \"targetDimensions\": 50\n },\n \"selection\": {\n \"documentTypes\": [\n \"TIME_ENTRY\"\n ],\n \"builderKeys\": [\n \"time-entry-structured-text\"\n ],\n \"embeddingStatuses\": [\n \"COMPLETED\"\n ],\n \"primaryRepresentationOnly\": true\n },\n \"parameters\": {\n \"minClusterSize\": 15,\n \"minSamples\": 5,\n \"metric\": \"euclidean\",\n \"clusterSelectionMethod\": \"eom\",\n \"normalizeVectors\": true\n }\n}"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Create Agglomerative run",
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"header": [
|
||||
{
|
||||
"key": "Content-Type",
|
||||
"value": "application/json"
|
||||
}
|
||||
],
|
||||
"url": {
|
||||
"raw": "{{baseUrl}}/v1/dip/clustering/runs",
|
||||
"host": [
|
||||
"{{baseUrl}}"
|
||||
],
|
||||
"path": [
|
||||
"v1",
|
||||
"dip",
|
||||
"clustering",
|
||||
"runs"
|
||||
]
|
||||
},
|
||||
"body": {
|
||||
"mode": "raw",
|
||||
"raw": "{\n \"name\": \"TED notices Agglomerative\",\n \"algorithm\": \"AGGLOMERATIVE\",\n \"executionBackend\": \"PYTHON_REMOTE\",\n \"reduction\": {\n \"method\": \"PCA\",\n \"targetDimensions\": 100\n },\n \"selection\": {\n \"documentTypes\": [\n \"TED_NOTICE\"\n ],\n \"representationTypes\": [\n \"SEMANTIC_TEXT\"\n ],\n \"embeddingStatuses\": [\n \"COMPLETED\"\n ],\n \"primaryRepresentationOnly\": true\n },\n \"parameters\": {\n \"k\": 25,\n \"linkage\": \"average\",\n \"metric\": \"euclidean\",\n \"normalizeVectors\": true\n }\n}"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Start run",
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"url": {
|
||||
"raw": "{{baseUrl}}/v1/dip/clustering/runs/{{runId}}/start",
|
||||
"host": [
|
||||
"{{baseUrl}}"
|
||||
],
|
||||
"path": [
|
||||
"v1",
|
||||
"dip",
|
||||
"clustering",
|
||||
"runs",
|
||||
"{{runId}}",
|
||||
"start"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Assignments with text",
|
||||
"request": {
|
||||
"method": "GET",
|
||||
"url": {
|
||||
"raw": "{{baseUrl}}/v1/dip/clustering/runs/{{runId}}/assignments?includeText=true",
|
||||
"host": [
|
||||
"{{baseUrl}}"
|
||||
],
|
||||
"path": [
|
||||
"v1",
|
||||
"dip",
|
||||
"clustering",
|
||||
"runs",
|
||||
"{{runId}}",
|
||||
"assignments"
|
||||
],
|
||||
"query": [
|
||||
{
|
||||
"key": "includeText",
|
||||
"value": "true"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
|
|
@ -0,0 +1,81 @@
|
|||
{
|
||||
"info": {
|
||||
"name": "DIP Python Clustering Service",
|
||||
"_postman_id": "59df0e88-01d6-42a4-9071-195b43f96787",
|
||||
"description": "Direct calls to the remote Python clustering service.",
|
||||
"schema": "https://schema.getpostman.com/json/collection/v2.1.0/collection.json"
|
||||
},
|
||||
"variable": [
|
||||
{
|
||||
"key": "baseUrl",
|
||||
"value": "http://localhost:8001"
|
||||
}
|
||||
],
|
||||
"item": [
|
||||
{
|
||||
"name": "Health",
|
||||
"request": {
|
||||
"method": "GET",
|
||||
"url": {
|
||||
"raw": "{{baseUrl}}/health",
|
||||
"host": [
|
||||
"{{baseUrl}}"
|
||||
],
|
||||
"path": [
|
||||
"health"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "DBSCAN PCA request",
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"header": [
|
||||
{
|
||||
"key": "Content-Type",
|
||||
"value": "application/json"
|
||||
}
|
||||
],
|
||||
"url": {
|
||||
"raw": "{{baseUrl}}/cluster",
|
||||
"host": [
|
||||
"{{baseUrl}}"
|
||||
],
|
||||
"path": [
|
||||
"cluster"
|
||||
]
|
||||
},
|
||||
"body": {
|
||||
"mode": "raw",
|
||||
"raw": "{\n \"algorithm\": \"DBSCAN\",\n \"parameters\": {\n \"eps\": 0.25,\n \"minSamples\": 2,\n \"normalizeVectors\": false\n },\n \"reductionMethod\": \"NONE\",\n \"items\": [\n {\n \"embeddingId\": \"11111111-1111-1111-1111-111111111111\",\n \"documentId\": \"22222222-2222-2222-2222-222222222221\",\n \"representationId\": \"33333333-3333-3333-3333-333333333331\",\n \"vector\": [\n 0.0,\n 0.0\n ]\n },\n {\n \"embeddingId\": \"11111111-1111-1111-1111-111111111112\",\n \"documentId\": \"22222222-2222-2222-2222-222222222222\",\n \"representationId\": \"33333333-3333-3333-3333-333333333332\",\n \"vector\": [\n 0.05,\n 0.0\n ]\n },\n {\n \"embeddingId\": \"11111111-1111-1111-1111-111111111113\",\n \"documentId\": \"22222222-2222-2222-2222-222222222223\",\n \"representationId\": \"33333333-3333-3333-3333-333333333333\",\n \"vector\": [\n 10.0,\n 10.0\n ]\n }\n ]\n}"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "HDBSCAN PCA request",
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"header": [
|
||||
{
|
||||
"key": "Content-Type",
|
||||
"value": "application/json"
|
||||
}
|
||||
],
|
||||
"url": {
|
||||
"raw": "{{baseUrl}}/cluster",
|
||||
"host": [
|
||||
"{{baseUrl}}"
|
||||
],
|
||||
"path": [
|
||||
"cluster"
|
||||
]
|
||||
},
|
||||
"body": {
|
||||
"mode": "raw",
|
||||
"raw": "{\n \"algorithm\": \"HDBSCAN\",\n \"parameters\": {\n \"minClusterSize\": 2,\n \"minSamples\": 1,\n \"normalizeVectors\": false\n },\n \"reductionMethod\": \"NONE\",\n \"items\": [\n {\n \"embeddingId\": \"11111111-1111-1111-1111-111111111121\",\n \"documentId\": \"22222222-2222-2222-2222-222222222231\",\n \"representationId\": \"33333333-3333-3333-3333-333333333341\",\n \"vector\": [\n 0.0,\n 0.0\n ]\n },\n {\n \"embeddingId\": \"11111111-1111-1111-1111-111111111122\",\n \"documentId\": \"22222222-2222-2222-2222-222222222232\",\n \"representationId\": \"33333333-3333-3333-3333-333333333342\",\n \"vector\": [\n 0.03,\n 0.01\n ]\n },\n {\n \"embeddingId\": \"11111111-1111-1111-1111-111111111123\",\n \"documentId\": \"22222222-2222-2222-2222-222222222233\",\n \"representationId\": \"33333333-3333-3333-3333-333333333343\",\n \"vector\": [\n 5.0,\n 5.0\n ]\n },\n {\n \"embeddingId\": \"11111111-1111-1111-1111-111111111124\",\n \"documentId\": \"22222222-2222-2222-2222-222222222234\",\n \"representationId\": \"33333333-3333-3333-3333-333333333344\",\n \"vector\": [\n 5.05,\n 4.98\n ]\n }\n ]\n}"
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
|
|
@ -0,0 +1,248 @@
|
|||
{
|
||||
"info": {
|
||||
"name": "DIP Clustering Phase A",
|
||||
"_postman_id": "5d5f3a8f-1c0c-4f7a-9e14-4d6f2c6d8f3a",
|
||||
"description": "Sample Postman collection for DIP clustering Phase A endpoints.\n\nVariables:\n- baseUrl: Spring Boot base URL\n- runId: cluster run id returned by create run\n\nThis collection contains example requests for TED notice embeddings and Leitstand TIME entry embeddings.",
|
||||
"schema": "https://schema.getpostman.com/json/collection/v2.1.0/collection.json"
|
||||
},
|
||||
"variable": [
|
||||
{
|
||||
"key": "baseUrl",
|
||||
"value": "http://localhost:8080"
|
||||
},
|
||||
{
|
||||
"key": "runId",
|
||||
"value": ""
|
||||
}
|
||||
],
|
||||
"item": [
|
||||
{
|
||||
"name": "Selection",
|
||||
"item": [
|
||||
{
|
||||
"name": "Count TED selection",
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"header": [
|
||||
{
|
||||
"key": "Content-Type",
|
||||
"value": "application/json"
|
||||
}
|
||||
],
|
||||
"url": {
|
||||
"raw": "{{baseUrl}}/v1/dip/clustering/selection/count",
|
||||
"host": [
|
||||
"{{baseUrl}}"
|
||||
],
|
||||
"path": [
|
||||
"v1",
|
||||
"dip",
|
||||
"clustering",
|
||||
"selection",
|
||||
"count"
|
||||
]
|
||||
},
|
||||
"body": {
|
||||
"mode": "raw",
|
||||
"raw": "{\n \"documentTypes\": [\n \"TED_NOTICE\"\n ],\n \"representationTypes\": [\n \"SEMANTIC_TEXT\"\n ],\n \"embeddingStatuses\": [\n \"COMPLETED\"\n ],\n \"primaryRepresentationOnly\": true\n}"
|
||||
},
|
||||
"description": "Counts the number of completed TED_NOTICE embeddings eligible for clustering."
|
||||
},
|
||||
"response": []
|
||||
},
|
||||
{
|
||||
"name": "Count Leitstand TIME selection",
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"header": [
|
||||
{
|
||||
"key": "Content-Type",
|
||||
"value": "application/json"
|
||||
}
|
||||
],
|
||||
"url": {
|
||||
"raw": "{{baseUrl}}/v1/dip/clustering/selection/count",
|
||||
"host": [
|
||||
"{{baseUrl}}"
|
||||
],
|
||||
"path": [
|
||||
"v1",
|
||||
"dip",
|
||||
"clustering",
|
||||
"selection",
|
||||
"count"
|
||||
]
|
||||
},
|
||||
"body": {
|
||||
"mode": "raw",
|
||||
"raw": "{\n \"documentTypes\": [\n \"TIME_ENTRY\"\n ],\n \"builderKeys\": [\n \"time-entry-structured-text\"\n ],\n \"embeddingStatuses\": [\n \"COMPLETED\"\n ],\n \"primaryRepresentationOnly\": true\n}"
|
||||
},
|
||||
"description": "Counts the number of completed Leitstand TIME_ENTRY embeddings eligible for clustering."
|
||||
},
|
||||
"response": []
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "Runs",
|
||||
"item": [
|
||||
{
|
||||
"name": "Create TED KMeans run",
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"header": [
|
||||
{
|
||||
"key": "Content-Type",
|
||||
"value": "application/json"
|
||||
}
|
||||
],
|
||||
"url": {
|
||||
"raw": "{{baseUrl}}/v1/dip/clustering/runs",
|
||||
"host": [
|
||||
"{{baseUrl}}"
|
||||
],
|
||||
"path": [
|
||||
"v1",
|
||||
"dip",
|
||||
"clustering",
|
||||
"runs"
|
||||
]
|
||||
},
|
||||
"body": {
|
||||
"mode": "raw",
|
||||
"raw": "{\n \"name\": \"TED notices KMeans run 1\",\n \"algorithm\": \"KMEANS\",\n \"selection\": {\n \"documentTypes\": [\n \"TED_NOTICE\"\n ],\n \"representationTypes\": [\n \"SEMANTIC_TEXT\"\n ],\n \"embeddingStatuses\": [\n \"COMPLETED\"\n ],\n \"primaryRepresentationOnly\": true\n },\n \"k\": 25\n}"
|
||||
},
|
||||
"description": "Creates a cluster run for TED notice embeddings. Copy the returned id into the Postman variable runId."
|
||||
},
|
||||
"response": []
|
||||
},
|
||||
{
|
||||
"name": "Create Leitstand TIME KMeans run",
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"header": [
|
||||
{
|
||||
"key": "Content-Type",
|
||||
"value": "application/json"
|
||||
}
|
||||
],
|
||||
"url": {
|
||||
"raw": "{{baseUrl}}/v1/dip/clustering/runs",
|
||||
"host": [
|
||||
"{{baseUrl}}"
|
||||
],
|
||||
"path": [
|
||||
"v1",
|
||||
"dip",
|
||||
"clustering",
|
||||
"runs"
|
||||
]
|
||||
},
|
||||
"body": {
|
||||
"mode": "raw",
|
||||
"raw": "{\n \"name\": \"Leitstand TIME KMeans run 1\",\n \"algorithm\": \"KMEANS\",\n \"selection\": {\n \"documentTypes\": [\n \"TIME_ENTRY\"\n ],\n \"builderKeys\": [\n \"time-entry-structured-text\"\n ],\n \"embeddingStatuses\": [\n \"COMPLETED\"\n ],\n \"primaryRepresentationOnly\": true\n },\n \"k\": 15\n}"
|
||||
},
|
||||
"description": "Creates a cluster run for Leitstand TIME_ENTRY embeddings. Copy the returned id into the Postman variable runId."
|
||||
},
|
||||
"response": []
|
||||
},
|
||||
{
|
||||
"name": "Start run",
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"header": [],
|
||||
"url": {
|
||||
"raw": "{{baseUrl}}/v1/dip/clustering/runs/{{runId}}/start",
|
||||
"host": [
|
||||
"{{baseUrl}}"
|
||||
],
|
||||
"path": [
|
||||
"v1",
|
||||
"dip",
|
||||
"clustering",
|
||||
"runs",
|
||||
"{{runId}}",
|
||||
"start"
|
||||
]
|
||||
},
|
||||
"description": "Starts clustering for the run stored in the runId variable."
|
||||
},
|
||||
"response": []
|
||||
},
|
||||
{
|
||||
"name": "Get run",
|
||||
"request": {
|
||||
"method": "GET",
|
||||
"header": [],
|
||||
"url": {
|
||||
"raw": "{{baseUrl}}/v1/dip/clustering/runs/{{runId}}",
|
||||
"host": [
|
||||
"{{baseUrl}}"
|
||||
],
|
||||
"path": [
|
||||
"v1",
|
||||
"dip",
|
||||
"clustering",
|
||||
"runs",
|
||||
"{{runId}}"
|
||||
]
|
||||
},
|
||||
"description": "Returns run metadata and status."
|
||||
},
|
||||
"response": []
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "Results",
|
||||
"item": [
|
||||
{
|
||||
"name": "Get clusters for run",
|
||||
"request": {
|
||||
"method": "GET",
|
||||
"header": [],
|
||||
"url": {
|
||||
"raw": "{{baseUrl}}/v1/dip/clustering/runs/{{runId}}/clusters",
|
||||
"host": [
|
||||
"{{baseUrl}}"
|
||||
],
|
||||
"path": [
|
||||
"v1",
|
||||
"dip",
|
||||
"clustering",
|
||||
"runs",
|
||||
"{{runId}}",
|
||||
"clusters"
|
||||
]
|
||||
},
|
||||
"description": "Lists clusters discovered in the run."
|
||||
},
|
||||
"response": []
|
||||
},
|
||||
{
|
||||
"name": "Get assignments for run",
|
||||
"request": {
|
||||
"method": "GET",
|
||||
"header": [],
|
||||
"url": {
|
||||
"raw": "{{baseUrl}}/v1/dip/clustering/runs/{{runId}}/assignments",
|
||||
"host": [
|
||||
"{{baseUrl}}"
|
||||
],
|
||||
"path": [
|
||||
"v1",
|
||||
"dip",
|
||||
"clustering",
|
||||
"runs",
|
||||
"{{runId}}",
|
||||
"assignments"
|
||||
]
|
||||
},
|
||||
"description": "Lists embedding-to-cluster assignments for the run."
|
||||
},
|
||||
"response": []
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
|
|
@ -0,0 +1,19 @@
|
|||
FROM python:3.11-slim
|
||||
|
||||
ENV PYTHONDONTWRITEBYTECODE=1 \
|
||||
PYTHONUNBUFFERED=1
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
RUN apt-get update \
|
||||
&& apt-get install -y --no-install-recommends build-essential \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
COPY requirements.txt .
|
||||
RUN pip install --no-cache-dir -r requirements.txt
|
||||
|
||||
COPY app ./app
|
||||
|
||||
EXPOSE 8001
|
||||
|
||||
CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "8001"]
|
||||
|
|
@ -0,0 +1,204 @@
|
|||
# DIP Clustering Service
|
||||
|
||||
Remote Python clustering backend for the DIP Spring clustering module.
|
||||
|
||||
## Main execution mode
|
||||
|
||||
The preferred execution mode is now:
|
||||
|
||||
- Spring keeps run metadata, selection snapshot, and lifecycle.
|
||||
- Spring sends only a compact request containing `runId`.
|
||||
- Python loads the run metadata and selected embeddings directly from Postgres.
|
||||
- Python returns compact assignments keyed by `embeddingId`.
|
||||
|
||||
This avoids sending the full embedding matrix through HTTP.
|
||||
|
||||
## Implemented algorithms
|
||||
|
||||
- `KMEANS`
|
||||
- `MINI_BATCH_KMEANS`
|
||||
- `DBSCAN`
|
||||
- `HDBSCAN`
|
||||
- `AGGLOMERATIVE`
|
||||
|
||||
## Implemented reductions
|
||||
|
||||
- `NONE`
|
||||
- `PCA`
|
||||
- `UMAP`
|
||||
|
||||
## API
|
||||
|
||||
### `GET /health`
|
||||
|
||||
Returns service status and supported algorithms/reduction methods.
|
||||
|
||||
### `POST /cluster-run`
|
||||
|
||||
Preferred endpoint. Accepts only the cluster run id.
|
||||
|
||||
Example request body:
|
||||
|
||||
```json
|
||||
{
|
||||
"runId": "6c3bc3a3-24b0-47a5-9e35-92dd4b7275f8"
|
||||
}
|
||||
```
|
||||
|
||||
|
||||
This service supports two remote execution modes at the same time:
|
||||
|
||||
- `POST /cluster`
|
||||
- Spring uploads embeddings in the request body.
|
||||
- This keeps the original implementation intact.
|
||||
- `POST /cluster-run`
|
||||
- Spring sends only `runId`.
|
||||
- Python loads run metadata and embeddings directly from Postgres.
|
||||
|
||||
## Start
|
||||
|
||||
```powershell
|
||||
py -3.11 -m venv .venv
|
||||
.\.venv\Scripts\python.exe -m pip install --upgrade pip
|
||||
.\.venv\Scripts\python.exe -m pip install -r requirements.txt
|
||||
```
|
||||
|
||||
Configure DB access for `/cluster-run` with either:
|
||||
|
||||
|
||||
### `POST /cluster`
|
||||
|
||||
Accepts the Spring `PythonClusteringRequest` payload and returns `PythonClusteringResponse`.
|
||||
|
||||
Example request body:
|
||||
|
||||
```json
|
||||
{
|
||||
"algorithm": "DBSCAN",
|
||||
"parameters": {
|
||||
"eps": 0.25,
|
||||
"minSamples": 5,
|
||||
"metric": "euclidean",
|
||||
"normalizeVectors": true
|
||||
},
|
||||
"reductionMethod": "PCA",
|
||||
"reductionDimensions": 100,
|
||||
"items": [
|
||||
{
|
||||
"embeddingId": "11111111-1111-1111-1111-111111111111",
|
||||
"documentId": "22222222-2222-2222-2222-222222222222",
|
||||
"representationId": "33333333-3333-3333-3333-333333333333",
|
||||
"vector": [0.1, 0.2, 0.3]
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
## Parameters by algorithm
|
||||
|
||||
### KMEANS
|
||||
- `k` required
|
||||
- `randomState` optional, default `42`
|
||||
- `nInit` optional, default `10`
|
||||
- `maxIter` optional, default `300`
|
||||
|
||||
### MINI_BATCH_KMEANS
|
||||
- `k` required
|
||||
- `batchSize` optional
|
||||
- `randomState` optional, default `42`
|
||||
- `nInit` optional, default `10`
|
||||
- `maxIter` optional, default `300`
|
||||
|
||||
### DBSCAN
|
||||
- `eps` required
|
||||
- `minSamples` optional, default `5`
|
||||
- `metric` optional, default `euclidean`
|
||||
- `algorithm` optional, default `auto`
|
||||
- `nJobs` optional, default `-1`
|
||||
|
||||
### HDBSCAN
|
||||
- `minClusterSize` optional, default `10`
|
||||
- `minSamples` optional
|
||||
- `metric` optional, default `euclidean`
|
||||
- `clusterSelectionMethod` optional, default `eom`
|
||||
|
||||
### AGGLOMERATIVE
|
||||
- `k` required
|
||||
- `linkage` optional, default `average`
|
||||
- `metric` optional, default `euclidean`
|
||||
- `computeDistances` optional, default `false`
|
||||
|
||||
## Shared parameters
|
||||
|
||||
- `normalizeVectors` optional, default `true`
|
||||
- `randomState` optional, used by `KMEANS`, `MINI_BATCH_KMEANS`, `PCA`, `UMAP`
|
||||
|
||||
## UMAP reduction parameters
|
||||
|
||||
- `reductionMetric` optional, default `cosine`
|
||||
- `umapNeighbors` optional, default `15`
|
||||
- `umapMinDist` optional, default `0.0`
|
||||
|
||||
## Local run
|
||||
## Required database configuration
|
||||
|
||||
Set either:
|
||||
|
||||
- `CLUSTERING_DB_DSN`
|
||||
- or `DATABASE_URL`
|
||||
- or `CLUSTERING_DB_HOST`, `CLUSTERING_DB_PORT`, `CLUSTERING_DB_NAME`, `CLUSTERING_DB_USER`, `CLUSTERING_DB_PASSWORD`
|
||||
|
||||
Example:
|
||||
|
||||
```bash
|
||||
export CLUSTERING_DB_DSN=postgresql://postgres:postgres@localhost:5432/dip
|
||||
```
|
||||
|
||||
## Local run on Windows
|
||||
|
||||
```powershell
|
||||
$env:CLUSTERING_DB_DSN="postgresql://postgres:postgres@localhost:5432/dip"
|
||||
.\.venv\Scripts\python.exe -m uvicorn app.main:app --host 0.0.0.0 --port 8001 --reload
|
||||
```
|
||||
|
||||
|
||||
## Docker run
|
||||
|
||||
```bash
|
||||
docker build -t dip-clustering-service .
|
||||
docker run --rm -p 8001:8001 dip-clustering-service
|
||||
```
|
||||
|
||||
## Spring configuration
|
||||
|
||||
Use the original request-upload mode:
|
||||
|
||||
```yaml
|
||||
dip:
|
||||
clustering:
|
||||
python:
|
||||
enabled: true
|
||||
base-url: http://localhost:8001
|
||||
cluster-path: /cluster
|
||||
cluster-run-path: /cluster-run
|
||||
request-mode: INLINE_VECTORS
|
||||
connect-timeout: 30s
|
||||
read-timeout: 30m
|
||||
```
|
||||
|
||||
Use compact `runId` mode:
|
||||
|
||||
```yaml
|
||||
dip:
|
||||
clustering:
|
||||
python:
|
||||
enabled: true
|
||||
base-url: http://localhost:8001
|
||||
cluster-path: /cluster
|
||||
cluster-run-path: /cluster-run
|
||||
request-mode: RUN_ID
|
||||
connect-timeout: 30s
|
||||
read-timeout: 30m
|
||||
```
|
||||
|
||||
`INLINE_VECTORS` is the default if `request-mode` is omitted.
|
||||
|
|
@ -0,0 +1,311 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from typing import Any
|
||||
|
||||
import numpy as np
|
||||
from sklearn.cluster import AgglomerativeClustering, DBSCAN, KMeans, MiniBatchKMeans
|
||||
from sklearn.decomposition import PCA
|
||||
from sklearn.preprocessing import normalize
|
||||
|
||||
try:
|
||||
import hdbscan
|
||||
except Exception: # pragma: no cover - runtime dependency guard
|
||||
hdbscan = None
|
||||
|
||||
try:
|
||||
import umap
|
||||
except Exception: # pragma: no cover - runtime dependency guard
|
||||
umap = None
|
||||
|
||||
from .models import (
|
||||
ClusteringAlgorithm,
|
||||
PythonAssignment,
|
||||
PythonCluster,
|
||||
PythonClusteringItem,
|
||||
PythonClusteringRequest,
|
||||
PythonClusteringResponse,
|
||||
ReductionMethod,
|
||||
RunMetadata,
|
||||
)
|
||||
|
||||
|
||||
class ClusteringServiceError(ValueError):
|
||||
pass
|
||||
|
||||
|
||||
@dataclass
|
||||
class PreparedData:
|
||||
original: np.ndarray
|
||||
transformed: np.ndarray
|
||||
items: list[PythonClusteringItem]
|
||||
|
||||
|
||||
def cluster_embeddings(request: PythonClusteringRequest) -> PythonClusteringResponse:
|
||||
return cluster_items(
|
||||
algorithm=request.algorithm,
|
||||
parameters=request.parameters or {},
|
||||
reduction_method=request.reductionMethod,
|
||||
reduction_dimensions=request.reductionDimensions,
|
||||
items=request.items,
|
||||
)
|
||||
|
||||
|
||||
def cluster_run(metadata: RunMetadata, items: list[PythonClusteringItem]) -> PythonClusteringResponse:
|
||||
return cluster_items(
|
||||
algorithm=metadata.algorithm,
|
||||
parameters=metadata.parameters or {},
|
||||
reduction_method=metadata.reductionMethod,
|
||||
reduction_dimensions=metadata.reductionDimensions,
|
||||
items=items,
|
||||
)
|
||||
|
||||
|
||||
def cluster_items(
|
||||
algorithm: ClusteringAlgorithm,
|
||||
parameters: dict[str, Any],
|
||||
reduction_method: ReductionMethod,
|
||||
reduction_dimensions: int | None,
|
||||
items: list[PythonClusteringItem],
|
||||
) -> PythonClusteringResponse:
|
||||
if not items:
|
||||
raise ClusteringServiceError("Request contains no items")
|
||||
|
||||
prepared = _prepare_data(
|
||||
items=items,
|
||||
parameters=parameters,
|
||||
reduction_method=reduction_method,
|
||||
reduction_dimensions=reduction_dimensions,
|
||||
)
|
||||
labels, membership_scores = _run_algorithm(
|
||||
algorithm=algorithm,
|
||||
vectors=prepared.transformed,
|
||||
parameters=parameters,
|
||||
)
|
||||
return _build_response(prepared, labels, membership_scores)
|
||||
|
||||
|
||||
def _prepare_data(
|
||||
items: list[PythonClusteringItem],
|
||||
parameters: dict[str, Any],
|
||||
reduction_method: ReductionMethod,
|
||||
reduction_dimensions: int | None,
|
||||
) -> PreparedData:
|
||||
vectors = np.asarray([item.vector for item in items], dtype=np.float32)
|
||||
if vectors.ndim != 2 or vectors.shape[0] == 0:
|
||||
raise ClusteringServiceError("Vectors must form a non-empty 2D array")
|
||||
|
||||
if _bool_param(parameters, "normalizeVectors", True):
|
||||
vectors = normalize(vectors, norm="l2")
|
||||
|
||||
transformed = vectors
|
||||
if reduction_method == ReductionMethod.PCA:
|
||||
target_dims = reduction_dimensions
|
||||
if target_dims is None:
|
||||
raise ClusteringServiceError("PCA reduction requires reductionDimensions")
|
||||
max_components = min(transformed.shape[0], transformed.shape[1])
|
||||
if target_dims <= 0 or target_dims > max_components:
|
||||
raise ClusteringServiceError(
|
||||
f"PCA reductionDimensions must be between 1 and {max_components}"
|
||||
)
|
||||
pca = PCA(
|
||||
n_components=target_dims,
|
||||
random_state=_int_param(parameters, "randomState", 42),
|
||||
)
|
||||
transformed = pca.fit_transform(transformed)
|
||||
elif reduction_method == ReductionMethod.UMAP:
|
||||
target_dims = reduction_dimensions
|
||||
if target_dims is None:
|
||||
raise ClusteringServiceError("UMAP reduction requires reductionDimensions")
|
||||
if umap is None:
|
||||
raise ClusteringServiceError("UMAP reduction requested but umap-learn is not installed")
|
||||
reducer = umap.UMAP(
|
||||
n_components=target_dims,
|
||||
metric=_str_param(parameters, "reductionMetric", "cosine"),
|
||||
n_neighbors=_int_param(parameters, "umapNeighbors", 15),
|
||||
min_dist=_float_param(parameters, "umapMinDist", 0.0),
|
||||
random_state=_int_param(parameters, "randomState", 42),
|
||||
)
|
||||
transformed = reducer.fit_transform(transformed)
|
||||
|
||||
return PreparedData(original=vectors, transformed=np.asarray(transformed, dtype=np.float32), items=items)
|
||||
|
||||
|
||||
def _run_algorithm(
|
||||
algorithm: ClusteringAlgorithm,
|
||||
vectors: np.ndarray,
|
||||
parameters: dict[str, Any],
|
||||
) -> tuple[np.ndarray, np.ndarray | None]:
|
||||
if algorithm == ClusteringAlgorithm.KMEANS:
|
||||
k = _required_int_param(parameters, "k")
|
||||
model = KMeans(
|
||||
n_clusters=k,
|
||||
random_state=_int_param(parameters, "randomState", 42),
|
||||
n_init=_int_param(parameters, "nInit", 10),
|
||||
max_iter=_int_param(parameters, "maxIter", 300),
|
||||
)
|
||||
labels = model.fit_predict(vectors)
|
||||
return np.asarray(labels, dtype=np.int32), None
|
||||
|
||||
if algorithm == ClusteringAlgorithm.MINI_BATCH_KMEANS:
|
||||
k = _required_int_param(parameters, "k")
|
||||
batch_size = _int_param(parameters, "batchSize", min(max(k * 16, 256), 4096))
|
||||
model = MiniBatchKMeans(
|
||||
n_clusters=k,
|
||||
random_state=_int_param(parameters, "randomState", 42),
|
||||
n_init=_int_param(parameters, "nInit", 10),
|
||||
max_iter=_int_param(parameters, "maxIter", 300),
|
||||
batch_size=batch_size,
|
||||
)
|
||||
labels = model.fit_predict(vectors)
|
||||
return np.asarray(labels, dtype=np.int32), None
|
||||
|
||||
if algorithm == ClusteringAlgorithm.DBSCAN:
|
||||
eps = _required_float_param(parameters, "eps")
|
||||
model = DBSCAN(
|
||||
eps=eps,
|
||||
min_samples=_int_param(parameters, "minSamples", 5),
|
||||
metric=_str_param(parameters, "metric", "euclidean"),
|
||||
algorithm=_str_param(parameters, "algorithm", "auto"),
|
||||
n_jobs=_int_param(parameters, "nJobs", -1),
|
||||
)
|
||||
labels = model.fit_predict(vectors)
|
||||
return np.asarray(labels, dtype=np.int32), None
|
||||
|
||||
if algorithm == ClusteringAlgorithm.HDBSCAN:
|
||||
if hdbscan is None:
|
||||
raise ClusteringServiceError("HDBSCAN requested but hdbscan is not installed")
|
||||
model = hdbscan.HDBSCAN(
|
||||
min_cluster_size=_int_param(parameters, "minClusterSize", 10),
|
||||
min_samples=_nullable_int_param(parameters, "minSamples"),
|
||||
metric=_str_param(parameters, "metric", "euclidean"),
|
||||
cluster_selection_method=_str_param(parameters, "clusterSelectionMethod", "eom"),
|
||||
)
|
||||
labels = model.fit_predict(vectors)
|
||||
probabilities = getattr(model, "probabilities_", None)
|
||||
return np.asarray(labels, dtype=np.int32), None if probabilities is None else np.asarray(probabilities, dtype=np.float32)
|
||||
|
||||
if algorithm == ClusteringAlgorithm.AGGLOMERATIVE:
|
||||
k = _required_int_param(parameters, "k")
|
||||
linkage = _str_param(parameters, "linkage", "average")
|
||||
metric = _str_param(parameters, "metric", "euclidean")
|
||||
if linkage == "ward":
|
||||
metric = "euclidean"
|
||||
model = AgglomerativeClustering(
|
||||
n_clusters=k,
|
||||
linkage=linkage,
|
||||
metric=metric,
|
||||
compute_distances=_bool_param(parameters, "computeDistances", False),
|
||||
)
|
||||
labels = model.fit_predict(vectors)
|
||||
return np.asarray(labels, dtype=np.int32), None
|
||||
|
||||
raise ClusteringServiceError(f"Unsupported algorithm: {algorithm}")
|
||||
|
||||
|
||||
def _build_response(
|
||||
prepared: PreparedData,
|
||||
labels: np.ndarray,
|
||||
membership_scores: np.ndarray | None,
|
||||
) -> PythonClusteringResponse:
|
||||
unique_labels = sorted(int(label) for label in np.unique(labels))
|
||||
clusters: list[PythonCluster] = []
|
||||
assignments: list[PythonAssignment] = []
|
||||
|
||||
centroids: dict[int, np.ndarray] = {}
|
||||
for label in unique_labels:
|
||||
mask = labels == label
|
||||
item_count = int(mask.sum())
|
||||
noise_cluster = label == -1
|
||||
clusters.append(PythonCluster(clusterLabel=label, itemCount=item_count, noiseCluster=noise_cluster))
|
||||
if not noise_cluster:
|
||||
centroids[label] = prepared.transformed[mask].mean(axis=0)
|
||||
|
||||
for index, item in enumerate(prepared.items):
|
||||
label = int(labels[index])
|
||||
noise = label == -1
|
||||
distance = None if noise else float(np.linalg.norm(prepared.transformed[index] - centroids[label]))
|
||||
membership = None
|
||||
if membership_scores is not None:
|
||||
membership = float(membership_scores[index])
|
||||
assignments.append(
|
||||
PythonAssignment(
|
||||
embeddingId=item.embeddingId,
|
||||
clusterLabel=label,
|
||||
distanceToCentroid=distance,
|
||||
membershipScore=membership,
|
||||
noise=noise,
|
||||
)
|
||||
)
|
||||
|
||||
noise_count = int((labels == -1).sum())
|
||||
return PythonClusteringResponse(clusters=clusters, assignments=assignments, noiseCount=noise_count)
|
||||
|
||||
|
||||
def _required_int_param(parameters: dict[str, Any], key: str) -> int:
|
||||
if key not in parameters or parameters[key] is None:
|
||||
raise ClusteringServiceError(f"Missing required parameter: {key}")
|
||||
return _coerce_int(parameters[key], key)
|
||||
|
||||
|
||||
def _required_float_param(parameters: dict[str, Any], key: str) -> float:
|
||||
if key not in parameters or parameters[key] is None:
|
||||
raise ClusteringServiceError(f"Missing required parameter: {key}")
|
||||
return _coerce_float(parameters[key], key)
|
||||
|
||||
|
||||
def _nullable_int_param(parameters: dict[str, Any], key: str) -> int | None:
|
||||
if key not in parameters or parameters[key] is None:
|
||||
return None
|
||||
return _coerce_int(parameters[key], key)
|
||||
|
||||
|
||||
def _int_param(parameters: dict[str, Any], key: str, default: int) -> int:
|
||||
if key not in parameters or parameters[key] is None:
|
||||
return default
|
||||
return _coerce_int(parameters[key], key)
|
||||
|
||||
|
||||
def _float_param(parameters: dict[str, Any], key: str, default: float) -> float:
|
||||
if key not in parameters or parameters[key] is None:
|
||||
return default
|
||||
return _coerce_float(parameters[key], key)
|
||||
|
||||
|
||||
def _bool_param(parameters: dict[str, Any], key: str, default: bool) -> bool:
|
||||
if key not in parameters or parameters[key] is None:
|
||||
return default
|
||||
value = parameters[key]
|
||||
if isinstance(value, bool):
|
||||
return value
|
||||
if isinstance(value, str):
|
||||
normalized = value.strip().lower()
|
||||
if normalized in {"true", "1", "yes", "y"}:
|
||||
return True
|
||||
if normalized in {"false", "0", "no", "n"}:
|
||||
return False
|
||||
raise ClusteringServiceError(f"Parameter {key} must be boolean-compatible")
|
||||
|
||||
|
||||
def _str_param(parameters: dict[str, Any], key: str, default: str) -> str:
|
||||
if key not in parameters or parameters[key] is None:
|
||||
return default
|
||||
return str(parameters[key])
|
||||
|
||||
|
||||
def _coerce_int(value: Any, key: str) -> int:
|
||||
if isinstance(value, bool):
|
||||
raise ClusteringServiceError(f"Parameter {key} must be integer-compatible")
|
||||
try:
|
||||
return int(value)
|
||||
except (TypeError, ValueError) as exc:
|
||||
raise ClusteringServiceError(f"Parameter {key} must be integer-compatible") from exc
|
||||
|
||||
|
||||
def _coerce_float(value: Any, key: str) -> float:
|
||||
if isinstance(value, bool):
|
||||
raise ClusteringServiceError(f"Parameter {key} must be float-compatible")
|
||||
try:
|
||||
return float(value)
|
||||
except (TypeError, ValueError) as exc:
|
||||
raise ClusteringServiceError(f"Parameter {key} must be float-compatible") from exc
|
||||
|
|
@ -0,0 +1,63 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from fastapi import FastAPI, HTTPException
|
||||
from fastapi.middleware.gzip import GZipMiddleware
|
||||
from pydantic import BaseModel
|
||||
|
||||
from .cluster_service import ClusteringServiceError, cluster_embeddings, cluster_run
|
||||
from .models import (
|
||||
ClusteringAlgorithm,
|
||||
PythonClusteringRequest,
|
||||
PythonClusteringResponse,
|
||||
PythonRunExecutionRequest,
|
||||
ReductionMethod,
|
||||
)
|
||||
from .run_db_loader import load_run_and_embeddings
|
||||
|
||||
|
||||
app = FastAPI(
|
||||
title="DIP Clustering Service",
|
||||
version="2.0.0",
|
||||
description="Remote clustering backend for DIP embedding clustering runs.",
|
||||
)
|
||||
app.add_middleware(GZipMiddleware, minimum_size=1024)
|
||||
|
||||
|
||||
class HealthResponse(BaseModel):
|
||||
status: str
|
||||
algorithms: list[str]
|
||||
reductionMethods: list[str]
|
||||
endpoints: list[str]
|
||||
|
||||
|
||||
@app.get("/health", response_model=HealthResponse)
|
||||
def health() -> HealthResponse:
|
||||
return HealthResponse(
|
||||
status="UP",
|
||||
algorithms=[algorithm.value for algorithm in ClusteringAlgorithm],
|
||||
reductionMethods=[method.value for method in ReductionMethod],
|
||||
endpoints=["/cluster", "/cluster-run"],
|
||||
)
|
||||
|
||||
|
||||
@app.post("/cluster", response_model=PythonClusteringResponse)
|
||||
def cluster_direct(request: PythonClusteringRequest) -> PythonClusteringResponse:
|
||||
try:
|
||||
return cluster_embeddings(request)
|
||||
except ClusteringServiceError as exc:
|
||||
raise HTTPException(status_code=400, detail=str(exc)) from exc
|
||||
except Exception as exc: # pragma: no cover - last-resort guard
|
||||
raise HTTPException(status_code=500, detail=f"Unexpected clustering failure: {exc}") from exc
|
||||
|
||||
|
||||
@app.post("/cluster-run", response_model=PythonClusteringResponse)
|
||||
def cluster_by_run(request: PythonRunExecutionRequest) -> PythonClusteringResponse:
|
||||
try:
|
||||
metadata, items = load_run_and_embeddings(request.runId)
|
||||
return cluster_run(metadata, items)
|
||||
except ClusteringServiceError as exc:
|
||||
raise HTTPException(status_code=400, detail=str(exc)) from exc
|
||||
except ValueError as exc:
|
||||
raise HTTPException(status_code=404, detail=str(exc)) from exc
|
||||
except Exception as exc: # pragma: no cover - last-resort guard
|
||||
raise HTTPException(status_code=500, detail=f"Unexpected clustering failure: {exc}") from exc
|
||||
|
|
@ -0,0 +1,75 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from enum import Enum
|
||||
from typing import Any
|
||||
from uuid import UUID
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field
|
||||
|
||||
|
||||
class ClusteringAlgorithm(str, Enum):
|
||||
KMEANS = "KMEANS"
|
||||
MINI_BATCH_KMEANS = "MINI_BATCH_KMEANS"
|
||||
DBSCAN = "DBSCAN"
|
||||
HDBSCAN = "HDBSCAN"
|
||||
AGGLOMERATIVE = "AGGLOMERATIVE"
|
||||
|
||||
|
||||
class ReductionMethod(str, Enum):
|
||||
NONE = "NONE"
|
||||
PCA = "PCA"
|
||||
UMAP = "UMAP"
|
||||
|
||||
|
||||
class PythonClusteringItem(BaseModel):
|
||||
embeddingId: UUID
|
||||
documentId: UUID | None = None
|
||||
representationId: UUID | None = None
|
||||
vector: list[float]
|
||||
|
||||
|
||||
class PythonClusteringRequest(BaseModel):
|
||||
algorithm: ClusteringAlgorithm
|
||||
parameters: dict[str, Any] = Field(default_factory=dict)
|
||||
reductionMethod: ReductionMethod = ReductionMethod.NONE
|
||||
reductionDimensions: int | None = None
|
||||
items: list[PythonClusteringItem]
|
||||
|
||||
model_config = ConfigDict(use_enum_values=True)
|
||||
|
||||
|
||||
class PythonRunExecutionRequest(BaseModel):
|
||||
runId: UUID
|
||||
|
||||
|
||||
class PythonCluster(BaseModel):
|
||||
clusterLabel: int
|
||||
itemCount: int
|
||||
noiseCluster: bool = False
|
||||
|
||||
|
||||
class PythonAssignment(BaseModel):
|
||||
embeddingId: UUID
|
||||
documentId: UUID | None = None
|
||||
representationId: UUID | None = None
|
||||
clusterLabel: int
|
||||
distanceToCentroid: float | None = None
|
||||
membershipScore: float | None = None
|
||||
noise: bool = False
|
||||
|
||||
|
||||
class PythonClusteringResponse(BaseModel):
|
||||
clusters: list[PythonCluster]
|
||||
assignments: list[PythonAssignment]
|
||||
noiseCount: int
|
||||
|
||||
|
||||
class RunMetadata(BaseModel):
|
||||
runId: UUID
|
||||
algorithm: ClusteringAlgorithm
|
||||
parameters: dict[str, Any] = Field(default_factory=dict)
|
||||
reductionMethod: ReductionMethod = ReductionMethod.NONE
|
||||
reductionDimensions: int | None = None
|
||||
selection: dict[str, Any] = Field(default_factory=dict)
|
||||
|
||||
model_config = ConfigDict(use_enum_values=True)
|
||||
|
|
@ -0,0 +1,163 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from typing import Any
|
||||
from uuid import UUID
|
||||
|
||||
import numpy as np
|
||||
import psycopg2
|
||||
|
||||
from .models import ClusteringAlgorithm, PythonClusteringItem, ReductionMethod, RunMetadata
|
||||
from .settings import ServiceSettings
|
||||
|
||||
|
||||
def load_run_and_embeddings(run_id: UUID) -> tuple[RunMetadata, list[PythonClusteringItem]]:
|
||||
settings = ServiceSettings.from_env()
|
||||
with psycopg2.connect(settings.db_dsn) as connection:
|
||||
run = _load_run_metadata(connection, run_id)
|
||||
items = _load_embeddings(connection, run.selection)
|
||||
return run, items
|
||||
|
||||
|
||||
def _load_run_metadata(connection, run_id: UUID) -> RunMetadata:
|
||||
with connection.cursor() as cursor:
|
||||
cursor.execute(
|
||||
"""
|
||||
select
|
||||
id,
|
||||
algorithm,
|
||||
coalesce(parameters_json::text, '{}'),
|
||||
reduction_method,
|
||||
reduction_dimensions,
|
||||
coalesce(selection_json::text, '{}')
|
||||
from doc.doc_embedding_cluster_run
|
||||
where id = %s
|
||||
""",
|
||||
(str(run_id),),
|
||||
)
|
||||
row = cursor.fetchone()
|
||||
|
||||
if row is None:
|
||||
raise ValueError(f"Cluster run not found: {run_id}")
|
||||
|
||||
parameters = _json_to_dict(row[2])
|
||||
selection = _json_to_dict(row[5])
|
||||
|
||||
return RunMetadata(
|
||||
runId=row[0],
|
||||
algorithm=ClusteringAlgorithm(row[1]),
|
||||
parameters=parameters,
|
||||
reductionMethod=ReductionMethod(row[3]) if row[3] else ReductionMethod.NONE,
|
||||
reductionDimensions=row[4],
|
||||
selection=selection,
|
||||
)
|
||||
|
||||
|
||||
def _load_embeddings(connection, selection: dict[str, Any]) -> list[PythonClusteringItem]:
|
||||
sql_parts = [
|
||||
"""
|
||||
select
|
||||
e.id as embedding_id,
|
||||
e.document_id,
|
||||
e.representation_id,
|
||||
e.embedding_vector::text as embedding_vector_text
|
||||
from doc.doc_embedding e
|
||||
join doc.doc_document d on d.id = e.document_id
|
||||
join doc.doc_text_representation r on r.id = e.representation_id
|
||||
where e.embedding_status = 'COMPLETED'
|
||||
and e.embedding_vector is not null
|
||||
"""
|
||||
]
|
||||
params: list[Any] = []
|
||||
|
||||
_apply_selection_filters(selection, sql_parts, params)
|
||||
sql_parts.append(" order by e.created_at asc")
|
||||
sql = "".join(sql_parts)
|
||||
|
||||
items: list[PythonClusteringItem] = []
|
||||
with connection.cursor(name="cluster_embedding_selection") as cursor:
|
||||
cursor.itersize = 2000
|
||||
cursor.execute(sql, params)
|
||||
for embedding_id, document_id, representation_id, vector_text in cursor:
|
||||
items.append(
|
||||
PythonClusteringItem(
|
||||
embeddingId=embedding_id,
|
||||
documentId=document_id,
|
||||
representationId=representation_id,
|
||||
vector=_parse_vector_text(vector_text),
|
||||
)
|
||||
)
|
||||
return items
|
||||
|
||||
|
||||
def _apply_selection_filters(selection: dict[str, Any], sql_parts: list[str], params: list[Any]) -> None:
|
||||
if not selection:
|
||||
return
|
||||
|
||||
_append_in_filter(sql_parts, params, "documentTypes", "d.document_type", selection.get("documentTypes"))
|
||||
_append_in_filter(sql_parts, params, "documentFamilies", "d.document_family", selection.get("documentFamilies"))
|
||||
_append_in_filter(sql_parts, params, "representationTypes", "r.representation_type", selection.get("representationTypes"))
|
||||
_append_in_filter(sql_parts, params, "embeddingStatuses", "e.embedding_status", selection.get("embeddingStatuses"))
|
||||
_append_in_filter(sql_parts, params, "modelIds", "e.model_id", selection.get("modelIds"))
|
||||
_append_in_filter(sql_parts, params, "prefixProfileIds", "e.prefix_profile_id", selection.get("prefixProfileIds"))
|
||||
_append_in_filter(sql_parts, params, "builderKeys", "r.builder_key", selection.get("builderKeys"))
|
||||
_append_in_filter(sql_parts, params, "languageCodes", "r.language_code", selection.get("languageCodes"))
|
||||
_append_in_filter(sql_parts, params, "ownerTenantIds", "d.owner_tenant_id", selection.get("ownerTenantIds"))
|
||||
|
||||
business_key_like = selection.get("businessKeyLike")
|
||||
if business_key_like:
|
||||
sql_parts.append(" and d.business_key like %s")
|
||||
params.append(business_key_like)
|
||||
|
||||
created_from = selection.get("createdFrom")
|
||||
if created_from:
|
||||
sql_parts.append(" and d.created_at >= %s")
|
||||
params.append(created_from)
|
||||
|
||||
created_to = selection.get("createdTo")
|
||||
if created_to:
|
||||
sql_parts.append(" and d.created_at < %s")
|
||||
params.append(created_to)
|
||||
|
||||
if selection.get("primaryRepresentationOnly") is True:
|
||||
sql_parts.append(" and r.is_primary = true")
|
||||
|
||||
|
||||
def _append_in_filter(
|
||||
sql_parts: list[str],
|
||||
params: list[Any],
|
||||
_key: str,
|
||||
column_name: str,
|
||||
values: list[Any] | None,
|
||||
) -> None:
|
||||
if not values:
|
||||
return
|
||||
placeholders = ", ".join(["%s"] * len(values))
|
||||
sql_parts.append(f" and {column_name} in ({placeholders})")
|
||||
params.extend(values)
|
||||
|
||||
|
||||
def _parse_vector_text(raw_value: str) -> list[float]:
|
||||
if raw_value is None:
|
||||
return []
|
||||
|
||||
value = raw_value.strip()
|
||||
if value.startswith("[") and value.endswith("]"):
|
||||
value = value[1:-1]
|
||||
|
||||
if not value:
|
||||
return []
|
||||
|
||||
vector = np.fromstring(value, sep=",", dtype=np.float32)
|
||||
return vector.astype(float).tolist()
|
||||
|
||||
|
||||
def _json_to_dict(raw_json: str | dict[str, Any] | None) -> dict[str, Any]:
|
||||
if raw_json is None:
|
||||
return {}
|
||||
if isinstance(raw_json, dict):
|
||||
return raw_json
|
||||
if not raw_json.strip():
|
||||
return {}
|
||||
loaded = json.loads(raw_json)
|
||||
return loaded if isinstance(loaded, dict) else {}
|
||||
|
|
@ -0,0 +1,39 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from dataclasses import dataclass
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ServiceSettings:
|
||||
db_dsn: str
|
||||
|
||||
@staticmethod
|
||||
def from_env() -> "ServiceSettings":
|
||||
dsn = (
|
||||
os.getenv("CLUSTERING_DB_DSN")
|
||||
or os.getenv("DATABASE_URL")
|
||||
or _build_dsn_from_parts()
|
||||
)
|
||||
if not dsn:
|
||||
raise RuntimeError(
|
||||
"No database connection configured. Set CLUSTERING_DB_DSN or DATABASE_URL, "
|
||||
"or provide CLUSTERING_DB_HOST / CLUSTERING_DB_PORT / CLUSTERING_DB_NAME / "
|
||||
"CLUSTERING_DB_USER / CLUSTERING_DB_PASSWORD."
|
||||
)
|
||||
return ServiceSettings(db_dsn=dsn)
|
||||
|
||||
|
||||
def _build_dsn_from_parts() -> str | None:
|
||||
host = os.getenv("CLUSTERING_DB_HOST")
|
||||
database = os.getenv("CLUSTERING_DB_NAME")
|
||||
user = os.getenv("CLUSTERING_DB_USER")
|
||||
password = os.getenv("CLUSTERING_DB_PASSWORD")
|
||||
port = os.getenv("CLUSTERING_DB_PORT", "5432")
|
||||
|
||||
if not host or not database or not user:
|
||||
return None
|
||||
|
||||
if password:
|
||||
return f"postgresql://{user}:{password}@{host}:{port}/{database}"
|
||||
return f"postgresql://{user}@{host}:{port}/{database}"
|
||||
|
|
@ -0,0 +1,7 @@
|
|||
fastapi==0.115.12
|
||||
uvicorn[standard]==0.34.2
|
||||
numpy==2.2.5
|
||||
scikit-learn==1.7.0
|
||||
hdbscan==0.8.40
|
||||
umap-learn==0.5.7
|
||||
psycopg2-binary==2.9.10
|
||||
|
|
@ -0,0 +1,85 @@
|
|||
from fastapi.testclient import TestClient
|
||||
|
||||
from app.main import app
|
||||
|
||||
|
||||
client = TestClient(app)
|
||||
|
||||
|
||||
def test_health():
|
||||
response = client.get("/health")
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
assert data["status"] == "UP"
|
||||
assert "DBSCAN" in data["algorithms"]
|
||||
|
||||
|
||||
def test_kmeans_cluster():
|
||||
body = {
|
||||
"algorithm": "KMEANS",
|
||||
"parameters": {"k": 2},
|
||||
"reductionMethod": "NONE",
|
||||
"items": [
|
||||
{
|
||||
"embeddingId": "11111111-1111-1111-1111-111111111111",
|
||||
"documentId": "22222222-2222-2222-2222-222222222221",
|
||||
"representationId": "33333333-3333-3333-3333-333333333331",
|
||||
"vector": [1.0, 1.0]
|
||||
},
|
||||
{
|
||||
"embeddingId": "11111111-1111-1111-1111-111111111112",
|
||||
"documentId": "22222222-2222-2222-2222-222222222222",
|
||||
"representationId": "33333333-3333-3333-3333-333333333332",
|
||||
"vector": [1.1, 1.0]
|
||||
},
|
||||
{
|
||||
"embeddingId": "11111111-1111-1111-1111-111111111113",
|
||||
"documentId": "22222222-2222-2222-2222-222222222223",
|
||||
"representationId": "33333333-3333-3333-3333-333333333333",
|
||||
"vector": [-1.0, -1.0]
|
||||
},
|
||||
{
|
||||
"embeddingId": "11111111-1111-1111-1111-111111111114",
|
||||
"documentId": "22222222-2222-2222-2222-222222222224",
|
||||
"representationId": "33333333-3333-3333-3333-333333333334",
|
||||
"vector": [-1.1, -1.0]
|
||||
}
|
||||
]
|
||||
}
|
||||
response = client.post("/cluster", json=body)
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
assert len(data["clusters"]) == 2
|
||||
assert data["noiseCount"] == 0
|
||||
|
||||
|
||||
def test_dbscan_cluster_with_noise():
|
||||
body = {
|
||||
"algorithm": "DBSCAN",
|
||||
"parameters": {"eps": 0.25, "minSamples": 2, "normalizeVectors": False},
|
||||
"reductionMethod": "NONE",
|
||||
"items": [
|
||||
{
|
||||
"embeddingId": "11111111-1111-1111-1111-111111111211",
|
||||
"documentId": "22222222-2222-2222-2222-222222222211",
|
||||
"representationId": "33333333-3333-3333-3333-333333333211",
|
||||
"vector": [0.0, 0.0]
|
||||
},
|
||||
{
|
||||
"embeddingId": "11111111-1111-1111-1111-111111111212",
|
||||
"documentId": "22222222-2222-2222-2222-222222222212",
|
||||
"representationId": "33333333-3333-3333-3333-333333333212",
|
||||
"vector": [0.05, 0.0]
|
||||
},
|
||||
{
|
||||
"embeddingId": "11111111-1111-1111-1111-111111111213",
|
||||
"documentId": "22222222-2222-2222-2222-222222222213",
|
||||
"representationId": "33333333-3333-3333-3333-333333333213",
|
||||
"vector": [10.0, 10.0]
|
||||
}
|
||||
]
|
||||
}
|
||||
response = client.post("/cluster", json=body)
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
assert data["noiseCount"] == 1
|
||||
|
|
@ -0,0 +1,11 @@
|
|||
package at.procon.dip.clustering;
|
||||
|
||||
public enum ClusterRunStatus {
|
||||
CREATED,
|
||||
QUEUED,
|
||||
RUNNING,
|
||||
CANCEL_REQUESTED,
|
||||
COMPLETED,
|
||||
FAILED,
|
||||
CANCELLED
|
||||
}
|
||||
|
|
@ -0,0 +1,9 @@
|
|||
package at.procon.dip.clustering;
|
||||
|
||||
public enum ClusteringAlgorithm {
|
||||
KMEANS,
|
||||
MINI_BATCH_KMEANS,
|
||||
DBSCAN,
|
||||
HDBSCAN,
|
||||
AGGLOMERATIVE
|
||||
}
|
||||
|
|
@ -0,0 +1,6 @@
|
|||
package at.procon.dip.clustering;
|
||||
|
||||
public enum ClusteringExecutionBackend {
|
||||
JAVA_LOCAL,
|
||||
PYTHON_REMOTE
|
||||
}
|
||||
|
|
@ -0,0 +1,6 @@
|
|||
package at.procon.dip.clustering;
|
||||
|
||||
public enum PythonRequestMode {
|
||||
INLINE_VECTORS,
|
||||
RUN_ID
|
||||
}
|
||||
|
|
@ -0,0 +1,7 @@
|
|||
package at.procon.dip.clustering;
|
||||
|
||||
public enum ReductionMethod {
|
||||
NONE,
|
||||
PCA,
|
||||
UMAP
|
||||
}
|
||||
|
|
@ -0,0 +1,12 @@
|
|||
package at.procon.dip.clustering.client;
|
||||
|
||||
import at.procon.dip.clustering.dto.PythonClusteringRequest;
|
||||
import at.procon.dip.clustering.dto.PythonClusteringResponse;
|
||||
import at.procon.dip.clustering.dto.PythonRunExecutionRequest;
|
||||
|
||||
public interface PythonClusteringClient {
|
||||
|
||||
PythonClusteringResponse cluster(PythonClusteringRequest request);
|
||||
|
||||
PythonClusteringResponse clusterRun(PythonRunExecutionRequest request);
|
||||
}
|
||||
|
|
@ -0,0 +1,96 @@
|
|||
package at.procon.dip.clustering.client;
|
||||
|
||||
import at.procon.dip.clustering.config.ClusteringPhaseBProperties;
|
||||
import at.procon.dip.clustering.dto.PythonClusteringRequest;
|
||||
import at.procon.dip.clustering.dto.PythonClusteringResponse;
|
||||
import at.procon.dip.clustering.dto.PythonRunExecutionRequest;
|
||||
import java.net.http.HttpClient;
|
||||
import java.time.Duration;
|
||||
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
|
||||
import org.springframework.http.MediaType;
|
||||
import org.springframework.http.client.JdkClientHttpRequestFactory;
|
||||
import org.springframework.stereotype.Component;
|
||||
import org.springframework.web.client.RestClient;
|
||||
import org.springframework.web.server.ResponseStatusException;
|
||||
|
||||
@Component
|
||||
@ConditionalOnProperty(prefix = "dip.clustering.python", name = "enabled", havingValue = "true")
|
||||
public class RestPythonClusteringClient implements PythonClusteringClient {
|
||||
|
||||
private static final Duration DEFAULT_CONNECT_TIMEOUT = Duration.ofSeconds(30);
|
||||
private static final Duration DEFAULT_READ_TIMEOUT = Duration.ofMinutes(30);
|
||||
|
||||
private final ClusteringPhaseBProperties properties;
|
||||
private final RestClient restClient;
|
||||
|
||||
public RestPythonClusteringClient(ClusteringPhaseBProperties properties) {
|
||||
this.properties = properties;
|
||||
|
||||
Duration connectTimeout = properties.connectTimeout() != null
|
||||
? properties.connectTimeout()
|
||||
: DEFAULT_CONNECT_TIMEOUT;
|
||||
Duration readTimeout = properties.readTimeout() != null
|
||||
? properties.readTimeout()
|
||||
: DEFAULT_READ_TIMEOUT;
|
||||
|
||||
HttpClient httpClient = HttpClient.newBuilder()
|
||||
.connectTimeout(connectTimeout)
|
||||
.version(HttpClient.Version.HTTP_1_1)
|
||||
.build();
|
||||
|
||||
JdkClientHttpRequestFactory requestFactory = new JdkClientHttpRequestFactory(httpClient);
|
||||
requestFactory.setReadTimeout(readTimeout);
|
||||
|
||||
this.restClient = RestClient.builder()
|
||||
.requestFactory(requestFactory)
|
||||
.build();
|
||||
}
|
||||
|
||||
@Override
|
||||
public PythonClusteringResponse cluster(PythonClusteringRequest request) {
|
||||
String url = properties.resolvedClusterUrl();
|
||||
if (url == null || url.isBlank()) {
|
||||
throw new ResponseStatusException(org.springframework.http.HttpStatus.BAD_REQUEST,
|
||||
"Python clustering is enabled but no baseUrl/clusterPath is configured");
|
||||
}
|
||||
|
||||
try {
|
||||
return restClient.post()
|
||||
.uri(url)
|
||||
.contentType(MediaType.APPLICATION_JSON)
|
||||
.accept(MediaType.APPLICATION_JSON)
|
||||
.body(request)
|
||||
.retrieve()
|
||||
.body(PythonClusteringResponse.class);
|
||||
} catch (Exception ex) {
|
||||
throw new ResponseStatusException(
|
||||
org.springframework.http.HttpStatus.BAD_GATEWAY,
|
||||
"Python cluster request failed: " + ex.getMessage(),
|
||||
ex);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public PythonClusteringResponse clusterRun(PythonRunExecutionRequest request) {
|
||||
String url = properties.resolvedClusterRunUrl();
|
||||
if (url == null || url.isBlank()) {
|
||||
throw new ResponseStatusException(org.springframework.http.HttpStatus.BAD_REQUEST,
|
||||
"Python clustering is enabled but no baseUrl/clusterRunPath is configured");
|
||||
}
|
||||
|
||||
try {
|
||||
return restClient.post()
|
||||
.uri(url)
|
||||
.contentType(MediaType.APPLICATION_JSON)
|
||||
.accept(MediaType.APPLICATION_JSON)
|
||||
.body(request)
|
||||
.retrieve()
|
||||
.body(PythonClusteringResponse.class);
|
||||
} catch (Exception ex) {
|
||||
throw new ResponseStatusException(
|
||||
org.springframework.http.HttpStatus.BAD_GATEWAY,
|
||||
"Python cluster-run request failed: " + ex.getMessage(),
|
||||
ex);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,25 @@
|
|||
package at.procon.dip.clustering.config;
|
||||
|
||||
import java.util.concurrent.Executor;
|
||||
import org.springframework.boot.context.properties.EnableConfigurationProperties;
|
||||
import org.springframework.context.annotation.Bean;
|
||||
import org.springframework.context.annotation.Configuration;
|
||||
import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor;
|
||||
|
||||
@Configuration
|
||||
@EnableConfigurationProperties(ClusteringExecutionProperties.class)
|
||||
public class ClusteringExecutionConfiguration {
|
||||
|
||||
@Bean(name = "clusteringRunExecutor")
|
||||
public Executor clusteringRunExecutor(ClusteringExecutionProperties properties) {
|
||||
ThreadPoolTaskExecutor executor = new ThreadPoolTaskExecutor();
|
||||
executor.setThreadNamePrefix("clustering-run-");
|
||||
executor.setCorePoolSize(properties.resolvedCorePoolSize());
|
||||
executor.setMaxPoolSize(Math.max(properties.resolvedCorePoolSize(), properties.resolvedMaxPoolSize()));
|
||||
executor.setQueueCapacity(properties.resolvedQueueCapacity());
|
||||
executor.setWaitForTasksToCompleteOnShutdown(true);
|
||||
executor.setAwaitTerminationSeconds(30);
|
||||
executor.initialize();
|
||||
return executor;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,22 @@
|
|||
package at.procon.dip.clustering.config;
|
||||
|
||||
import org.springframework.boot.context.properties.ConfigurationProperties;
|
||||
|
||||
@ConfigurationProperties(prefix = "dip.clustering.execution")
|
||||
public record ClusteringExecutionProperties(
|
||||
int corePoolSize,
|
||||
int maxPoolSize,
|
||||
int queueCapacity
|
||||
) {
|
||||
public int resolvedCorePoolSize() {
|
||||
return corePoolSize > 0 ? corePoolSize : 1;
|
||||
}
|
||||
|
||||
public int resolvedMaxPoolSize() {
|
||||
return maxPoolSize > 0 ? maxPoolSize : Math.max(1, resolvedCorePoolSize());
|
||||
}
|
||||
|
||||
public int resolvedQueueCapacity() {
|
||||
return queueCapacity >= 0 ? queueCapacity : 50;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,9 @@
|
|||
package at.procon.dip.clustering.config;
|
||||
|
||||
import org.springframework.boot.context.properties.EnableConfigurationProperties;
|
||||
import org.springframework.context.annotation.Configuration;
|
||||
|
||||
@Configuration
|
||||
@EnableConfigurationProperties(ClusteringPhaseBProperties.class)
|
||||
public class ClusteringPhaseBConfig {
|
||||
}
|
||||
|
|
@ -0,0 +1,37 @@
|
|||
package at.procon.dip.clustering.config;
|
||||
|
||||
import at.procon.dip.clustering.PythonRequestMode;
|
||||
import java.time.Duration;
|
||||
import org.springframework.boot.context.properties.ConfigurationProperties;
|
||||
|
||||
@ConfigurationProperties(prefix = "dip.clustering.python")
|
||||
public record ClusteringPhaseBProperties(
|
||||
boolean enabled,
|
||||
String baseUrl,
|
||||
String clusterPath,
|
||||
String clusterRunPath,
|
||||
Duration connectTimeout,
|
||||
Duration readTimeout,
|
||||
PythonRequestMode requestMode
|
||||
) {
|
||||
public String resolvedClusterUrl() {
|
||||
return resolveUrl(clusterPath == null || clusterPath.isBlank() ? "/cluster" : clusterPath);
|
||||
}
|
||||
|
||||
public String resolvedClusterRunUrl() {
|
||||
return resolveUrl(clusterRunPath == null || clusterRunPath.isBlank() ? "/cluster-run" : clusterRunPath);
|
||||
}
|
||||
|
||||
public PythonRequestMode effectiveRequestMode() {
|
||||
return requestMode == null ? PythonRequestMode.INLINE_VECTORS : requestMode;
|
||||
}
|
||||
|
||||
private String resolveUrl(String path) {
|
||||
if (baseUrl == null || baseUrl.isBlank()) {
|
||||
return null;
|
||||
}
|
||||
return baseUrl.endsWith("/")
|
||||
? baseUrl.substring(0, baseUrl.length() - 1) + path
|
||||
: baseUrl + path;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,16 @@
|
|||
package at.procon.dip.clustering.dto;
|
||||
|
||||
import java.util.UUID;
|
||||
|
||||
public record ClusterAssignmentResponse(
|
||||
UUID id,
|
||||
UUID clusterId,
|
||||
UUID embeddingId,
|
||||
UUID documentId,
|
||||
UUID representationId,
|
||||
Integer clusterLabelRaw,
|
||||
Double membershipScore,
|
||||
Double distanceToCentroid,
|
||||
boolean noise
|
||||
) {
|
||||
}
|
||||
|
|
@ -0,0 +1,26 @@
|
|||
package at.procon.dip.clustering.dto;
|
||||
|
||||
import at.procon.dip.domain.document.DocumentType;
|
||||
import at.procon.dip.domain.document.RepresentationType;
|
||||
import java.util.UUID;
|
||||
|
||||
public record ClusterAssignmentViewResponse(
|
||||
UUID id,
|
||||
UUID clusterId,
|
||||
UUID embeddingId,
|
||||
UUID documentId,
|
||||
UUID representationId,
|
||||
Integer clusterLabelRaw,
|
||||
Double membershipScore,
|
||||
Double distanceToCentroid,
|
||||
boolean noise,
|
||||
String businessKey,
|
||||
DocumentType documentType,
|
||||
RepresentationType representationType,
|
||||
String builderKey,
|
||||
String languageCode,
|
||||
Integer textLength,
|
||||
String textPreview,
|
||||
String textBody
|
||||
) {
|
||||
}
|
||||
|
|
@ -0,0 +1,10 @@
|
|||
package at.procon.dip.clustering.dto;
|
||||
|
||||
import java.util.UUID;
|
||||
|
||||
public record ClusterMembersResponse(
|
||||
UUID clusterId,
|
||||
Integer clusterLabel,
|
||||
ClusterAssignmentViewResponse member
|
||||
) {
|
||||
}
|
||||
|
|
@ -0,0 +1,13 @@
|
|||
package at.procon.dip.clustering.dto;
|
||||
|
||||
import java.util.UUID;
|
||||
|
||||
public record ClusterResponse(
|
||||
UUID id,
|
||||
Integer clusterLabel,
|
||||
String displayName,
|
||||
Long itemCount,
|
||||
boolean noiseCluster,
|
||||
String summaryText
|
||||
) {
|
||||
}
|
||||
|
|
@ -0,0 +1,25 @@
|
|||
package at.procon.dip.clustering.dto;
|
||||
|
||||
import at.procon.dip.clustering.ClusterRunStatus;
|
||||
import at.procon.dip.clustering.ClusteringAlgorithm;
|
||||
import at.procon.dip.clustering.ClusteringExecutionBackend;
|
||||
import at.procon.dip.clustering.ReductionMethod;
|
||||
import java.time.OffsetDateTime;
|
||||
import java.util.UUID;
|
||||
|
||||
public record ClusterRunResponse(
|
||||
UUID id,
|
||||
String name,
|
||||
ClusterRunStatus status,
|
||||
ClusteringAlgorithm algorithm,
|
||||
ClusteringExecutionBackend executionBackend,
|
||||
ReductionMethod reductionMethod,
|
||||
Integer reductionDimensions,
|
||||
Long itemCount,
|
||||
Long clusterCount,
|
||||
Long noiseCount,
|
||||
OffsetDateTime startedAt,
|
||||
OffsetDateTime finishedAt,
|
||||
String errorMessage
|
||||
) {
|
||||
}
|
||||
|
|
@ -0,0 +1,16 @@
|
|||
package at.procon.dip.clustering.dto;
|
||||
|
||||
import java.time.OffsetDateTime;
|
||||
import java.util.UUID;
|
||||
|
||||
public record ClusterSetResponse(
|
||||
UUID id,
|
||||
String code,
|
||||
String name,
|
||||
String description,
|
||||
boolean active,
|
||||
EmbeddingSelectionSpec selection,
|
||||
OffsetDateTime createdAt,
|
||||
OffsetDateTime updatedAt
|
||||
) {
|
||||
}
|
||||
|
|
@ -0,0 +1,33 @@
|
|||
package at.procon.dip.clustering.dto;
|
||||
|
||||
import java.util.UUID;
|
||||
|
||||
public record ClusteringEngineAssignment(
|
||||
UUID embeddingId,
|
||||
UUID documentId,
|
||||
UUID representationId,
|
||||
int clusterLabel,
|
||||
Double distanceToCentroid,
|
||||
Double membershipScore,
|
||||
boolean noise
|
||||
) {
|
||||
public ClusteringEngineAssignment(
|
||||
UUID embeddingId,
|
||||
int clusterLabel,
|
||||
Double distanceToCentroid,
|
||||
Double membershipScore,
|
||||
boolean noise
|
||||
) {
|
||||
this(embeddingId, null, null, clusterLabel, distanceToCentroid, membershipScore, noise);
|
||||
}
|
||||
|
||||
public ClusteringEngineAssignment(
|
||||
UUID embeddingId,
|
||||
UUID documentId,
|
||||
UUID representationId,
|
||||
int clusterLabel,
|
||||
Double distanceToCentroid
|
||||
) {
|
||||
this(embeddingId, documentId, representationId, clusterLabel, distanceToCentroid, null, false);
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,11 @@
|
|||
package at.procon.dip.clustering.dto;
|
||||
|
||||
public record ClusteringEngineCluster(
|
||||
int clusterLabel,
|
||||
long itemCount,
|
||||
boolean noiseCluster
|
||||
) {
|
||||
public ClusteringEngineCluster(int clusterLabel, long itemCount) {
|
||||
this(clusterLabel, itemCount, false);
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,36 @@
|
|||
package at.procon.dip.clustering.dto;
|
||||
|
||||
import java.util.Collections;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.Map;
|
||||
|
||||
public record ClusteringEngineRequest(Map<String, Object> parameters) {
|
||||
|
||||
public ClusteringEngineRequest {
|
||||
parameters = parameters == null
|
||||
? Map.of()
|
||||
: Collections.unmodifiableMap(new LinkedHashMap<>(parameters));
|
||||
}
|
||||
|
||||
public int requiredInt(String key) {
|
||||
Object value = parameters.get(key);
|
||||
if (value == null) {
|
||||
throw new IllegalArgumentException("Missing required clustering parameter: " + key);
|
||||
}
|
||||
if (value instanceof Number number) {
|
||||
return number.intValue();
|
||||
}
|
||||
return Integer.parseInt(String.valueOf(value));
|
||||
}
|
||||
|
||||
public int intValue(String key, int defaultValue) {
|
||||
Object value = parameters.get(key);
|
||||
if (value == null) {
|
||||
return defaultValue;
|
||||
}
|
||||
if (value instanceof Number number) {
|
||||
return number.intValue();
|
||||
}
|
||||
return Integer.parseInt(String.valueOf(value));
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,16 @@
|
|||
package at.procon.dip.clustering.dto;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
public record ClusteringEngineResult(
|
||||
List<ClusteringEngineCluster> clusters,
|
||||
List<ClusteringEngineAssignment> assignments,
|
||||
long noiseCount
|
||||
) {
|
||||
public ClusteringEngineResult(
|
||||
List<ClusteringEngineCluster> clusters,
|
||||
List<ClusteringEngineAssignment> assignments
|
||||
) {
|
||||
this(clusters, assignments, 0L);
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,65 @@
|
|||
package at.procon.dip.clustering.dto;
|
||||
|
||||
import at.procon.dip.clustering.ClusteringAlgorithm;
|
||||
import at.procon.dip.clustering.ClusteringExecutionBackend;
|
||||
import com.fasterxml.jackson.annotation.JsonIgnore;
|
||||
import jakarta.validation.Valid;
|
||||
import jakarta.validation.constraints.AssertTrue;
|
||||
import jakarta.validation.constraints.NotBlank;
|
||||
import jakarta.validation.constraints.NotNull;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.Map;
|
||||
|
||||
public record CreateClusterRunRequest(
|
||||
String clusterSetCode,
|
||||
@NotBlank String name,
|
||||
@NotNull ClusteringAlgorithm algorithm,
|
||||
ClusteringExecutionBackend executionBackend,
|
||||
@Valid ReductionConfig reduction,
|
||||
@Valid @NotNull EmbeddingSelectionSpec selection,
|
||||
Integer k,
|
||||
Map<String, Object> parameters
|
||||
) {
|
||||
|
||||
@JsonIgnore
|
||||
public Map<String, Object> resolvedParameters() {
|
||||
Map<String, Object> merged = new LinkedHashMap<>();
|
||||
if (parameters != null) {
|
||||
merged.putAll(parameters);
|
||||
}
|
||||
if (k != null && !merged.containsKey("k")) {
|
||||
merged.put("k", k);
|
||||
}
|
||||
return merged;
|
||||
}
|
||||
|
||||
@AssertTrue(message = "k must be > 0 for KMEANS and MINI_BATCH_KMEANS; for other algorithms it must be omitted or > 0")
|
||||
@JsonIgnore
|
||||
public boolean isValidKConfiguration() {
|
||||
Integer effectiveK = extractPositiveInteger(resolvedParameters().get("k"));
|
||||
|
||||
if (algorithm == ClusteringAlgorithm.KMEANS
|
||||
|| algorithm == ClusteringAlgorithm.MINI_BATCH_KMEANS) {
|
||||
return effectiveK != null;
|
||||
}
|
||||
|
||||
Object rawK = resolvedParameters().get("k");
|
||||
return rawK == null || effectiveK != null;
|
||||
}
|
||||
|
||||
private Integer extractPositiveInteger(Object value) {
|
||||
if (value == null) {
|
||||
return null;
|
||||
}
|
||||
if (value instanceof Number number) {
|
||||
int intValue = number.intValue();
|
||||
return intValue > 0 ? intValue : null;
|
||||
}
|
||||
try {
|
||||
int intValue = Integer.parseInt(String.valueOf(value));
|
||||
return intValue > 0 ? intValue : null;
|
||||
} catch (NumberFormatException ex) {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,14 @@
|
|||
package at.procon.dip.clustering.dto;
|
||||
|
||||
import jakarta.validation.Valid;
|
||||
import jakarta.validation.constraints.NotBlank;
|
||||
import jakarta.validation.constraints.NotNull;
|
||||
|
||||
public record CreateClusterSetRequest(
|
||||
@NotBlank String code,
|
||||
@NotBlank String name,
|
||||
String description,
|
||||
@Valid @NotNull EmbeddingSelectionSpec selection,
|
||||
Boolean active
|
||||
) {
|
||||
}
|
||||
|
|
@ -0,0 +1,26 @@
|
|||
package at.procon.dip.clustering.dto;
|
||||
|
||||
import at.procon.dip.domain.document.DocumentFamily;
|
||||
import at.procon.dip.domain.document.DocumentType;
|
||||
import at.procon.dip.domain.document.EmbeddingStatus;
|
||||
import at.procon.dip.domain.document.RepresentationType;
|
||||
import java.time.OffsetDateTime;
|
||||
import java.util.Set;
|
||||
import java.util.UUID;
|
||||
|
||||
public record EmbeddingSelectionSpec(
|
||||
Set<DocumentType> documentTypes,
|
||||
Set<DocumentFamily> documentFamilies,
|
||||
Set<RepresentationType> representationTypes,
|
||||
Set<EmbeddingStatus> embeddingStatuses,
|
||||
Set<UUID> modelIds,
|
||||
Set<UUID> prefixProfileIds,
|
||||
Set<String> builderKeys,
|
||||
Set<String> languageCodes,
|
||||
Set<UUID> ownerTenantIds,
|
||||
String businessKeyLike,
|
||||
OffsetDateTime createdFrom,
|
||||
OffsetDateTime createdTo,
|
||||
Boolean primaryRepresentationOnly
|
||||
) {
|
||||
}
|
||||
|
|
@ -0,0 +1,23 @@
|
|||
package at.procon.dip.clustering.dto;
|
||||
|
||||
import at.procon.dip.clustering.ClusteringAlgorithm;
|
||||
import at.procon.dip.clustering.ReductionMethod;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.UUID;
|
||||
|
||||
public record PythonClusteringRequest(
|
||||
ClusteringAlgorithm algorithm,
|
||||
Map<String, Object> parameters,
|
||||
ReductionMethod reductionMethod,
|
||||
Integer reductionDimensions,
|
||||
List<PythonClusteringItem> items
|
||||
) {
|
||||
public record PythonClusteringItem(
|
||||
UUID embeddingId,
|
||||
UUID documentId,
|
||||
UUID representationId,
|
||||
float[] vector
|
||||
) {
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,28 @@
|
|||
package at.procon.dip.clustering.dto;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.UUID;
|
||||
|
||||
public record PythonClusteringResponse(
|
||||
List<PythonCluster> clusters,
|
||||
List<PythonAssignment> assignments,
|
||||
Long noiseCount
|
||||
) {
|
||||
public record PythonCluster(
|
||||
Integer clusterLabel,
|
||||
Long itemCount,
|
||||
Boolean noiseCluster
|
||||
) {
|
||||
}
|
||||
|
||||
public record PythonAssignment(
|
||||
UUID embeddingId,
|
||||
UUID documentId,
|
||||
UUID representationId,
|
||||
Integer clusterLabel,
|
||||
Double distanceToCentroid,
|
||||
Double membershipScore,
|
||||
Boolean noise
|
||||
) {
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,8 @@
|
|||
package at.procon.dip.clustering.dto;
|
||||
|
||||
import java.util.UUID;
|
||||
|
||||
public record PythonRunExecutionRequest(
|
||||
UUID runId
|
||||
) {
|
||||
}
|
||||
|
|
@ -0,0 +1,9 @@
|
|||
package at.procon.dip.clustering.dto;
|
||||
|
||||
import at.procon.dip.clustering.ReductionMethod;
|
||||
|
||||
public record ReductionConfig(
|
||||
ReductionMethod method,
|
||||
Integer targetDimensions
|
||||
) {
|
||||
}
|
||||
|
|
@ -0,0 +1,22 @@
|
|||
package at.procon.dip.clustering.dto;
|
||||
|
||||
import at.procon.dip.domain.document.DocumentFamily;
|
||||
import at.procon.dip.domain.document.DocumentType;
|
||||
import at.procon.dip.domain.document.RepresentationType;
|
||||
import java.util.UUID;
|
||||
|
||||
public record SelectedEmbeddingRow(
|
||||
UUID embeddingId,
|
||||
UUID documentId,
|
||||
UUID representationId,
|
||||
UUID modelId,
|
||||
UUID prefixProfileId,
|
||||
DocumentType documentType,
|
||||
DocumentFamily documentFamily,
|
||||
RepresentationType representationType,
|
||||
String builderKey,
|
||||
String languageCode,
|
||||
String businessKey,
|
||||
float[] embeddingVector
|
||||
) {
|
||||
}
|
||||
|
|
@ -0,0 +1,4 @@
|
|||
package at.procon.dip.clustering.dto;
|
||||
|
||||
public record SelectionCountResponse(long count) {
|
||||
}
|
||||
|
|
@ -0,0 +1,13 @@
|
|||
package at.procon.dip.clustering.dto;
|
||||
|
||||
import jakarta.validation.Valid;
|
||||
import jakarta.validation.constraints.NotBlank;
|
||||
import jakarta.validation.constraints.NotNull;
|
||||
|
||||
public record UpdateClusterSetRequest(
|
||||
@NotBlank String name,
|
||||
String description,
|
||||
@Valid @NotNull EmbeddingSelectionSpec selection,
|
||||
Boolean active
|
||||
) {
|
||||
}
|
||||
|
|
@ -0,0 +1,72 @@
|
|||
package at.procon.dip.clustering.entity;
|
||||
|
||||
import at.procon.dip.architecture.SchemaNames;
|
||||
import jakarta.persistence.Column;
|
||||
import jakarta.persistence.Entity;
|
||||
import jakarta.persistence.FetchType;
|
||||
import jakarta.persistence.GeneratedValue;
|
||||
import jakarta.persistence.GenerationType;
|
||||
import jakarta.persistence.Id;
|
||||
import jakarta.persistence.Index;
|
||||
import jakarta.persistence.JoinColumn;
|
||||
import jakarta.persistence.ManyToOne;
|
||||
import jakarta.persistence.PrePersist;
|
||||
import jakarta.persistence.Table;
|
||||
import java.time.OffsetDateTime;
|
||||
import java.util.UUID;
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Builder;
|
||||
import lombok.Getter;
|
||||
import lombok.NoArgsConstructor;
|
||||
import lombok.Setter;
|
||||
import org.hibernate.annotations.JdbcTypeCode;
|
||||
import org.hibernate.type.SqlTypes;
|
||||
|
||||
@Entity
|
||||
@Table(schema = SchemaNames.DOC, name = "doc_embedding_cluster", indexes = {
|
||||
@Index(name = "idx_doc_cluster_cluster_run_jpa", columnList = "cluster_run_id")
|
||||
})
|
||||
@Getter
|
||||
@Setter
|
||||
@NoArgsConstructor
|
||||
@AllArgsConstructor
|
||||
@Builder
|
||||
public class EmbeddingCluster {
|
||||
|
||||
@Id
|
||||
@GeneratedValue(strategy = GenerationType.UUID)
|
||||
private UUID id;
|
||||
|
||||
@ManyToOne(fetch = FetchType.LAZY, optional = false)
|
||||
@JoinColumn(name = "cluster_run_id", nullable = false)
|
||||
private EmbeddingClusterRun clusterRun;
|
||||
|
||||
@Column(name = "cluster_label", nullable = false)
|
||||
private Integer clusterLabel;
|
||||
|
||||
@Column(name = "display_name", length = 255)
|
||||
private String displayName;
|
||||
|
||||
@Column(name = "item_count", nullable = false)
|
||||
private Long itemCount;
|
||||
|
||||
@Builder.Default
|
||||
@Column(name = "is_noise_cluster", nullable = false)
|
||||
private boolean noiseCluster = false;
|
||||
|
||||
@Column(name = "summary_text", columnDefinition = "TEXT")
|
||||
private String summaryText;
|
||||
|
||||
@JdbcTypeCode(SqlTypes.JSON)
|
||||
@Column(name = "top_terms_json", columnDefinition = "jsonb")
|
||||
private String topTermsJson;
|
||||
|
||||
@Builder.Default
|
||||
@Column(name = "created_at", nullable = false, updatable = false)
|
||||
private OffsetDateTime createdAt = OffsetDateTime.now();
|
||||
|
||||
@PrePersist
|
||||
protected void onCreate() {
|
||||
createdAt = OffsetDateTime.now();
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,84 @@
|
|||
package at.procon.dip.clustering.entity;
|
||||
|
||||
import at.procon.dip.architecture.SchemaNames;
|
||||
import at.procon.dip.domain.document.entity.Document;
|
||||
import at.procon.dip.domain.document.entity.DocumentEmbedding;
|
||||
import at.procon.dip.domain.document.entity.DocumentTextRepresentation;
|
||||
import jakarta.persistence.Column;
|
||||
import jakarta.persistence.Entity;
|
||||
import jakarta.persistence.FetchType;
|
||||
import jakarta.persistence.GeneratedValue;
|
||||
import jakarta.persistence.GenerationType;
|
||||
import jakarta.persistence.Id;
|
||||
import jakarta.persistence.Index;
|
||||
import jakarta.persistence.JoinColumn;
|
||||
import jakarta.persistence.ManyToOne;
|
||||
import jakarta.persistence.PrePersist;
|
||||
import jakarta.persistence.Table;
|
||||
import java.time.OffsetDateTime;
|
||||
import java.util.UUID;
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Builder;
|
||||
import lombok.Getter;
|
||||
import lombok.NoArgsConstructor;
|
||||
import lombok.Setter;
|
||||
|
||||
@Entity
|
||||
@Table(schema = SchemaNames.DOC, name = "doc_embedding_cluster_assignment", indexes = {
|
||||
@Index(name = "idx_doc_cluster_assignment_run_jpa", columnList = "cluster_run_id"),
|
||||
@Index(name = "idx_doc_cluster_assignment_cluster_jpa", columnList = "cluster_id"),
|
||||
@Index(name = "idx_doc_cluster_assignment_document_jpa", columnList = "cluster_run_id, document_id")
|
||||
})
|
||||
@Getter
|
||||
@Setter
|
||||
@NoArgsConstructor
|
||||
@AllArgsConstructor
|
||||
@Builder
|
||||
public class EmbeddingClusterAssignment {
|
||||
|
||||
@Id
|
||||
@GeneratedValue(strategy = GenerationType.UUID)
|
||||
private UUID id;
|
||||
|
||||
@ManyToOne(fetch = FetchType.LAZY, optional = false)
|
||||
@JoinColumn(name = "cluster_run_id", nullable = false)
|
||||
private EmbeddingClusterRun clusterRun;
|
||||
|
||||
@ManyToOne(fetch = FetchType.LAZY)
|
||||
@JoinColumn(name = "cluster_id")
|
||||
private EmbeddingCluster cluster;
|
||||
|
||||
@ManyToOne(fetch = FetchType.LAZY, optional = false)
|
||||
@JoinColumn(name = "embedding_id", nullable = false)
|
||||
private DocumentEmbedding embedding;
|
||||
|
||||
@ManyToOne(fetch = FetchType.LAZY, optional = false)
|
||||
@JoinColumn(name = "document_id", nullable = false)
|
||||
private Document document;
|
||||
|
||||
@ManyToOne(fetch = FetchType.LAZY, optional = false)
|
||||
@JoinColumn(name = "representation_id", nullable = false)
|
||||
private DocumentTextRepresentation representation;
|
||||
|
||||
@Column(name = "cluster_label_raw", nullable = false)
|
||||
private Integer clusterLabelRaw;
|
||||
|
||||
@Column(name = "membership_score")
|
||||
private Double membershipScore;
|
||||
|
||||
@Column(name = "distance_to_centroid")
|
||||
private Double distanceToCentroid;
|
||||
|
||||
@Builder.Default
|
||||
@Column(name = "is_noise", nullable = false)
|
||||
private boolean noise = false;
|
||||
|
||||
@Builder.Default
|
||||
@Column(name = "created_at", nullable = false, updatable = false)
|
||||
private OffsetDateTime createdAt = OffsetDateTime.now();
|
||||
|
||||
@PrePersist
|
||||
protected void onCreate() {
|
||||
createdAt = OffsetDateTime.now();
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,147 @@
|
|||
package at.procon.dip.clustering.entity;
|
||||
|
||||
import at.procon.dip.architecture.SchemaNames;
|
||||
import at.procon.dip.clustering.ClusterRunStatus;
|
||||
import at.procon.dip.clustering.ClusteringAlgorithm;
|
||||
import at.procon.dip.clustering.ClusteringExecutionBackend;
|
||||
import at.procon.dip.clustering.ReductionMethod;
|
||||
import at.procon.dip.domain.document.DocumentFamily;
|
||||
import at.procon.dip.domain.document.DocumentType;
|
||||
import at.procon.dip.domain.document.RepresentationType;
|
||||
import at.procon.dip.domain.document.entity.DocumentEmbeddingModel;
|
||||
import at.procon.dip.domain.document.entity.DocumentEmbeddingPrefixProfile;
|
||||
import jakarta.persistence.Column;
|
||||
import jakarta.persistence.Entity;
|
||||
import jakarta.persistence.EnumType;
|
||||
import jakarta.persistence.Enumerated;
|
||||
import jakarta.persistence.FetchType;
|
||||
import jakarta.persistence.GeneratedValue;
|
||||
import jakarta.persistence.GenerationType;
|
||||
import jakarta.persistence.Id;
|
||||
import jakarta.persistence.Index;
|
||||
import jakarta.persistence.JoinColumn;
|
||||
import jakarta.persistence.ManyToOne;
|
||||
import jakarta.persistence.PrePersist;
|
||||
import jakarta.persistence.PreUpdate;
|
||||
import jakarta.persistence.Table;
|
||||
import java.time.OffsetDateTime;
|
||||
import java.util.UUID;
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Builder;
|
||||
import lombok.Getter;
|
||||
import lombok.NoArgsConstructor;
|
||||
import lombok.Setter;
|
||||
import org.hibernate.annotations.JdbcTypeCode;
|
||||
import org.hibernate.type.SqlTypes;
|
||||
|
||||
@Entity
|
||||
@Table(schema = SchemaNames.DOC, name = "doc_embedding_cluster_run", indexes = {
|
||||
@Index(name = "idx_doc_cluster_run_status_jpa", columnList = "status"),
|
||||
@Index(name = "idx_doc_cluster_run_algorithm_jpa", columnList = "algorithm"),
|
||||
@Index(name = "idx_doc_cluster_run_created_at_jpa", columnList = "created_at")
|
||||
})
|
||||
@Getter
|
||||
@Setter
|
||||
@NoArgsConstructor
|
||||
@AllArgsConstructor
|
||||
@Builder
|
||||
public class EmbeddingClusterRun {
|
||||
|
||||
@Id
|
||||
@GeneratedValue(strategy = GenerationType.UUID)
|
||||
private UUID id;
|
||||
|
||||
@ManyToOne(fetch = FetchType.LAZY)
|
||||
@JoinColumn(name = "cluster_set_id")
|
||||
private EmbeddingClusterSet clusterSet;
|
||||
|
||||
@Column(name = "name", nullable = false, length = 255)
|
||||
private String name;
|
||||
|
||||
@Enumerated(EnumType.STRING)
|
||||
@Column(name = "status", nullable = false, length = 32)
|
||||
private ClusterRunStatus status;
|
||||
|
||||
@Enumerated(EnumType.STRING)
|
||||
@Column(name = "algorithm", nullable = false, length = 64)
|
||||
private ClusteringAlgorithm algorithm;
|
||||
|
||||
@Column(name = "algorithm_version", length = 64)
|
||||
private String algorithmVersion;
|
||||
|
||||
@Enumerated(EnumType.STRING)
|
||||
@Column(name = "execution_backend", length = 64)
|
||||
private ClusteringExecutionBackend executionBackend;
|
||||
|
||||
@Enumerated(EnumType.STRING)
|
||||
@Column(name = "reduction_method", length = 32)
|
||||
private ReductionMethod reductionMethod;
|
||||
|
||||
@Column(name = "reduction_dimensions")
|
||||
private Integer reductionDimensions;
|
||||
|
||||
@JdbcTypeCode(SqlTypes.JSON)
|
||||
@Column(name = "selection_json", nullable = false, columnDefinition = "jsonb")
|
||||
private String selectionJson;
|
||||
|
||||
@JdbcTypeCode(SqlTypes.JSON)
|
||||
@Column(name = "parameters_json", nullable = false, columnDefinition = "jsonb")
|
||||
private String parametersJson;
|
||||
|
||||
@ManyToOne(fetch = FetchType.LAZY)
|
||||
@JoinColumn(name = "embedding_model_id")
|
||||
private DocumentEmbeddingModel embeddingModel;
|
||||
|
||||
@ManyToOne(fetch = FetchType.LAZY)
|
||||
@JoinColumn(name = "prefix_profile_id")
|
||||
private DocumentEmbeddingPrefixProfile prefixProfile;
|
||||
|
||||
@Enumerated(EnumType.STRING)
|
||||
@Column(name = "document_type", length = 64)
|
||||
private DocumentType documentType;
|
||||
|
||||
@Enumerated(EnumType.STRING)
|
||||
@Column(name = "document_family", length = 64)
|
||||
private DocumentFamily documentFamily;
|
||||
|
||||
@Enumerated(EnumType.STRING)
|
||||
@Column(name = "representation_type", length = 64)
|
||||
private RepresentationType representationType;
|
||||
|
||||
@Column(name = "builder_key", length = 255)
|
||||
private String builderKey;
|
||||
|
||||
@Column(name = "item_count")
|
||||
private Long itemCount;
|
||||
|
||||
@Column(name = "cluster_count")
|
||||
private Long clusterCount;
|
||||
|
||||
@Column(name = "noise_count")
|
||||
private Long noiseCount;
|
||||
|
||||
@Column(name = "started_at")
|
||||
private OffsetDateTime startedAt;
|
||||
|
||||
@Column(name = "finished_at")
|
||||
private OffsetDateTime finishedAt;
|
||||
|
||||
@Column(name = "error_message", columnDefinition = "TEXT")
|
||||
private String errorMessage;
|
||||
|
||||
@Builder.Default
|
||||
@Column(name = "created_at", nullable = false, updatable = false)
|
||||
private OffsetDateTime createdAt = OffsetDateTime.now();
|
||||
|
||||
@PrePersist
|
||||
protected void onCreate() {
|
||||
createdAt = OffsetDateTime.now();
|
||||
}
|
||||
|
||||
@PreUpdate
|
||||
protected void onUpdate() {
|
||||
if (status == null) {
|
||||
status = ClusterRunStatus.CREATED;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,74 @@
|
|||
package at.procon.dip.clustering.entity;
|
||||
|
||||
import at.procon.dip.architecture.SchemaNames;
|
||||
import jakarta.persistence.Column;
|
||||
import jakarta.persistence.Entity;
|
||||
import jakarta.persistence.GeneratedValue;
|
||||
import jakarta.persistence.GenerationType;
|
||||
import jakarta.persistence.Id;
|
||||
import jakarta.persistence.Index;
|
||||
import jakarta.persistence.PrePersist;
|
||||
import jakarta.persistence.PreUpdate;
|
||||
import jakarta.persistence.Table;
|
||||
import java.time.OffsetDateTime;
|
||||
import java.util.UUID;
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Builder;
|
||||
import lombok.Getter;
|
||||
import lombok.NoArgsConstructor;
|
||||
import lombok.Setter;
|
||||
import org.hibernate.annotations.JdbcTypeCode;
|
||||
import org.hibernate.type.SqlTypes;
|
||||
|
||||
@Entity
|
||||
@Table(schema = SchemaNames.DOC, name = "doc_embedding_cluster_set", indexes = {
|
||||
@Index(name = "idx_doc_embedding_cluster_set_code", columnList = "code", unique = true),
|
||||
@Index(name = "idx_doc_embedding_cluster_set_active", columnList = "active")
|
||||
})
|
||||
@Getter
|
||||
@Setter
|
||||
@NoArgsConstructor
|
||||
@AllArgsConstructor
|
||||
@Builder
|
||||
public class EmbeddingClusterSet {
|
||||
|
||||
@Id
|
||||
@GeneratedValue(strategy = GenerationType.UUID)
|
||||
private UUID id;
|
||||
|
||||
@Column(name = "code", nullable = false, length = 128, unique = true)
|
||||
private String code;
|
||||
|
||||
@Column(name = "name", nullable = false, length = 255)
|
||||
private String name;
|
||||
|
||||
@Column(name = "description", columnDefinition = "TEXT")
|
||||
private String description;
|
||||
|
||||
@JdbcTypeCode(SqlTypes.JSON)
|
||||
@Column(name = "selection_json", nullable = false, columnDefinition = "jsonb")
|
||||
private String selectionJson;
|
||||
|
||||
@Builder.Default
|
||||
@Column(name = "active", nullable = false)
|
||||
private boolean active = true;
|
||||
|
||||
@Builder.Default
|
||||
@Column(name = "created_at", nullable = false, updatable = false)
|
||||
private OffsetDateTime createdAt = OffsetDateTime.now();
|
||||
|
||||
@Builder.Default
|
||||
@Column(name = "updated_at", nullable = false)
|
||||
private OffsetDateTime updatedAt = OffsetDateTime.now();
|
||||
|
||||
@PrePersist
|
||||
protected void onCreate() {
|
||||
createdAt = OffsetDateTime.now();
|
||||
updatedAt = OffsetDateTime.now();
|
||||
}
|
||||
|
||||
@PreUpdate
|
||||
protected void onUpdate() {
|
||||
updatedAt = OffsetDateTime.now();
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,10 @@
|
|||
package at.procon.dip.clustering.repository;
|
||||
|
||||
import at.procon.dip.clustering.dto.EmbeddingSelectionSpec;
|
||||
import at.procon.dip.clustering.dto.SelectedEmbeddingRow;
|
||||
import java.util.List;
|
||||
|
||||
public interface DocumentEmbeddingClusterSelectionRepository {
|
||||
List<SelectedEmbeddingRow> findSelection(EmbeddingSelectionSpec spec);
|
||||
long countSelection(EmbeddingSelectionSpec spec);
|
||||
}
|
||||
|
|
@ -0,0 +1,219 @@
|
|||
package at.procon.dip.clustering.repository;
|
||||
|
||||
import at.procon.dip.clustering.dto.EmbeddingSelectionSpec;
|
||||
import at.procon.dip.clustering.dto.SelectedEmbeddingRow;
|
||||
import at.procon.dip.domain.document.DocumentFamily;
|
||||
import at.procon.dip.domain.document.DocumentType;
|
||||
import at.procon.dip.domain.document.RepresentationType;
|
||||
import java.sql.ResultSet;
|
||||
import java.sql.SQLException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.UUID;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import org.springframework.jdbc.core.RowMapper;
|
||||
import org.springframework.jdbc.core.namedparam.MapSqlParameterSource;
|
||||
import org.springframework.jdbc.core.namedparam.NamedParameterJdbcTemplate;
|
||||
import org.springframework.stereotype.Repository;
|
||||
import org.springframework.util.CollectionUtils;
|
||||
import org.springframework.util.StringUtils;
|
||||
|
||||
@Repository
|
||||
@RequiredArgsConstructor
|
||||
public class DocumentEmbeddingClusterSelectionRepositoryImpl implements DocumentEmbeddingClusterSelectionRepository {
|
||||
|
||||
private final NamedParameterJdbcTemplate jdbcTemplate;
|
||||
|
||||
@Override
|
||||
public List<SelectedEmbeddingRow> findSelection(EmbeddingSelectionSpec spec) {
|
||||
StringBuilder sql = new StringBuilder("""
|
||||
select
|
||||
e.id as embedding_id,
|
||||
d.id as document_id,
|
||||
r.id as representation_id,
|
||||
e.model_id,
|
||||
e.prefix_profile_id,
|
||||
d.document_type,
|
||||
d.document_family,
|
||||
r.representation_type,
|
||||
r.builder_key,
|
||||
r.language_code,
|
||||
d.business_key,
|
||||
e.embedding_vector::text as embedding_vector_text
|
||||
from doc.doc_embedding e
|
||||
join doc.doc_document d on d.id = e.document_id
|
||||
join doc.doc_text_representation r on r.id = e.representation_id
|
||||
where e.embedding_status = 'COMPLETED'
|
||||
and e.embedding_vector is not null
|
||||
and e.prefix_profile_id is not null
|
||||
""");
|
||||
MapSqlParameterSource params = new MapSqlParameterSource();
|
||||
applyFilters(spec, sql, params);
|
||||
sql.append(" order by e.created_at asc");
|
||||
|
||||
List<RawSelectedEmbeddingRow> rawRows = jdbcTemplate.query(
|
||||
sql.toString(),
|
||||
params,
|
||||
new RawSelectedEmbeddingRowMapper());
|
||||
|
||||
return rawRows.stream()
|
||||
.map(this::toSelectedEmbeddingRow)
|
||||
.toList();
|
||||
}
|
||||
|
||||
@Override
|
||||
public long countSelection(EmbeddingSelectionSpec spec) {
|
||||
StringBuilder sql = new StringBuilder("""
|
||||
select count(*)
|
||||
from doc.doc_embedding e
|
||||
join doc.doc_document d on d.id = e.document_id
|
||||
join doc.doc_text_representation r on r.id = e.representation_id
|
||||
where e.embedding_status = 'COMPLETED'
|
||||
and e.embedding_vector is not null
|
||||
and e.prefix_profile_id is not null
|
||||
""");
|
||||
MapSqlParameterSource params = new MapSqlParameterSource();
|
||||
applyFilters(spec, sql, params);
|
||||
Long result = jdbcTemplate.queryForObject(sql.toString(), params, Long.class);
|
||||
return result == null ? 0L : result;
|
||||
}
|
||||
|
||||
private void applyFilters(EmbeddingSelectionSpec spec, StringBuilder sql, MapSqlParameterSource params) {
|
||||
if (spec == null) {
|
||||
return;
|
||||
}
|
||||
if (!CollectionUtils.isEmpty(spec.documentTypes())) {
|
||||
sql.append(" and d.document_type in (:documentTypes)");
|
||||
params.addValue("documentTypes", enumNames(spec.documentTypes()));
|
||||
}
|
||||
if (!CollectionUtils.isEmpty(spec.documentFamilies())) {
|
||||
sql.append(" and d.document_family in (:documentFamilies)");
|
||||
params.addValue("documentFamilies", enumNames(spec.documentFamilies()));
|
||||
}
|
||||
if (!CollectionUtils.isEmpty(spec.representationTypes())) {
|
||||
sql.append(" and r.representation_type in (:representationTypes)");
|
||||
params.addValue("representationTypes", enumNames(spec.representationTypes()));
|
||||
}
|
||||
if (!CollectionUtils.isEmpty(spec.embeddingStatuses())) {
|
||||
sql.append(" and e.embedding_status in (:embeddingStatuses)");
|
||||
params.addValue("embeddingStatuses", enumNames(spec.embeddingStatuses()));
|
||||
}
|
||||
if (!CollectionUtils.isEmpty(spec.modelIds())) {
|
||||
sql.append(" and e.model_id in (:modelIds)");
|
||||
params.addValue("modelIds", spec.modelIds());
|
||||
}
|
||||
if (!CollectionUtils.isEmpty(spec.prefixProfileIds())) {
|
||||
sql.append(" and e.prefix_profile_id in (:prefixProfileIds)");
|
||||
params.addValue("prefixProfileIds", spec.prefixProfileIds());
|
||||
}
|
||||
if (!CollectionUtils.isEmpty(spec.builderKeys())) {
|
||||
sql.append(" and r.builder_key in (:builderKeys)");
|
||||
params.addValue("builderKeys", spec.builderKeys());
|
||||
}
|
||||
if (!CollectionUtils.isEmpty(spec.languageCodes())) {
|
||||
sql.append(" and r.language_code in (:languageCodes)");
|
||||
params.addValue("languageCodes", spec.languageCodes());
|
||||
}
|
||||
if (!CollectionUtils.isEmpty(spec.ownerTenantIds())) {
|
||||
sql.append(" and d.owner_tenant_id in (:ownerTenantIds)");
|
||||
params.addValue("ownerTenantIds", spec.ownerTenantIds());
|
||||
}
|
||||
if (StringUtils.hasText(spec.businessKeyLike())) {
|
||||
sql.append(" and d.business_key like :businessKeyLike");
|
||||
params.addValue("businessKeyLike", spec.businessKeyLike());
|
||||
}
|
||||
if (spec.createdFrom() != null) {
|
||||
sql.append(" and d.created_at >= :createdFrom");
|
||||
params.addValue("createdFrom", spec.createdFrom());
|
||||
}
|
||||
if (spec.createdTo() != null) {
|
||||
sql.append(" and d.created_at < :createdTo");
|
||||
params.addValue("createdTo", spec.createdTo());
|
||||
}
|
||||
if (Boolean.TRUE.equals(spec.primaryRepresentationOnly())) {
|
||||
sql.append(" and r.is_primary = true");
|
||||
}
|
||||
}
|
||||
|
||||
private List<String> enumNames(Iterable<?> values) {
|
||||
List<String> result = new ArrayList<>();
|
||||
for (Object value : values) {
|
||||
result.add(String.valueOf(value));
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
private SelectedEmbeddingRow toSelectedEmbeddingRow(RawSelectedEmbeddingRow row) {
|
||||
return new SelectedEmbeddingRow(
|
||||
row.embeddingId(),
|
||||
row.documentId(),
|
||||
row.representationId(),
|
||||
row.modelId(),
|
||||
row.prefixProfileId(),
|
||||
row.documentType(),
|
||||
row.documentFamily(),
|
||||
row.representationType(),
|
||||
row.builderKey(),
|
||||
row.languageCode(),
|
||||
row.businessKey(),
|
||||
parseVector(row.embeddingVectorText()));
|
||||
}
|
||||
|
||||
private float[] parseVector(String raw) {
|
||||
if (raw == null) {
|
||||
return null;
|
||||
}
|
||||
String value = raw.trim();
|
||||
if (value.length() < 2) {
|
||||
return new float[0];
|
||||
}
|
||||
if (value.charAt(0) == '[' && value.charAt(value.length() - 1) == ']') {
|
||||
value = value.substring(1, value.length() - 1);
|
||||
}
|
||||
if (value.isBlank()) {
|
||||
return new float[0];
|
||||
}
|
||||
String[] parts = value.split(",");
|
||||
float[] result = new float[parts.length];
|
||||
for (int i = 0; i < parts.length; i++) {
|
||||
result[i] = Float.parseFloat(parts[i].trim());
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
private record RawSelectedEmbeddingRow(
|
||||
UUID embeddingId,
|
||||
UUID documentId,
|
||||
UUID representationId,
|
||||
UUID modelId,
|
||||
UUID prefixProfileId,
|
||||
DocumentType documentType,
|
||||
DocumentFamily documentFamily,
|
||||
RepresentationType representationType,
|
||||
String builderKey,
|
||||
String languageCode,
|
||||
String businessKey,
|
||||
String embeddingVectorText
|
||||
) {
|
||||
}
|
||||
|
||||
private static class RawSelectedEmbeddingRowMapper implements RowMapper<RawSelectedEmbeddingRow> {
|
||||
@Override
|
||||
public RawSelectedEmbeddingRow mapRow(ResultSet rs, int rowNum) throws SQLException {
|
||||
return new RawSelectedEmbeddingRow(
|
||||
rs.getObject("embedding_id", UUID.class),
|
||||
rs.getObject("document_id", UUID.class),
|
||||
rs.getObject("representation_id", UUID.class),
|
||||
rs.getObject("model_id", UUID.class),
|
||||
rs.getObject("prefix_profile_id", UUID.class),
|
||||
DocumentType.valueOf(rs.getString("document_type")),
|
||||
DocumentFamily.valueOf(rs.getString("document_family")),
|
||||
RepresentationType.valueOf(rs.getString("representation_type")),
|
||||
rs.getString("builder_key"),
|
||||
rs.getString("language_code"),
|
||||
rs.getString("business_key"),
|
||||
rs.getString("embedding_vector_text")
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,10 @@
|
|||
package at.procon.dip.clustering.repository;
|
||||
|
||||
import at.procon.dip.clustering.entity.EmbeddingClusterAssignment;
|
||||
import java.util.List;
|
||||
import java.util.UUID;
|
||||
import org.springframework.data.jpa.repository.JpaRepository;
|
||||
|
||||
public interface EmbeddingClusterAssignmentRepository extends JpaRepository<EmbeddingClusterAssignment, UUID> {
|
||||
List<EmbeddingClusterAssignment> findByClusterRun_IdOrderByClusterLabelRawAscDocument_IdAsc(UUID clusterRunId);
|
||||
}
|
||||
|
|
@ -0,0 +1,10 @@
|
|||
package at.procon.dip.clustering.repository;
|
||||
|
||||
import at.procon.dip.clustering.entity.EmbeddingCluster;
|
||||
import java.util.List;
|
||||
import java.util.UUID;
|
||||
import org.springframework.data.jpa.repository.JpaRepository;
|
||||
|
||||
public interface EmbeddingClusterRepository extends JpaRepository<EmbeddingCluster, UUID> {
|
||||
List<EmbeddingCluster> findByClusterRun_IdOrderByClusterLabelAsc(UUID clusterRunId);
|
||||
}
|
||||
|
|
@ -0,0 +1,11 @@
|
|||
package at.procon.dip.clustering.repository;
|
||||
|
||||
import at.procon.dip.clustering.dto.ClusterAssignmentViewResponse;
|
||||
import at.procon.dip.clustering.dto.ClusterMembersResponse;
|
||||
import java.util.List;
|
||||
import java.util.UUID;
|
||||
|
||||
public interface EmbeddingClusterResultQueryRepository {
|
||||
List<ClusterAssignmentViewResponse> findAssignments(UUID runId, boolean includeText, int previewLength);
|
||||
List<ClusterMembersResponse> findClusterMembers(UUID runId, UUID clusterId, boolean includeText, int previewLength);
|
||||
}
|
||||
|
|
@ -0,0 +1,87 @@
|
|||
package at.procon.dip.clustering.repository;
|
||||
|
||||
import at.procon.dip.clustering.dto.ClusterAssignmentViewResponse;
|
||||
import at.procon.dip.clustering.dto.ClusterMembersResponse;
|
||||
import at.procon.dip.domain.document.DocumentType;
|
||||
import at.procon.dip.domain.document.RepresentationType;
|
||||
import java.sql.ResultSet;
|
||||
import java.sql.SQLException;
|
||||
import java.util.List;
|
||||
import java.util.UUID;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import org.springframework.jdbc.core.RowMapper;
|
||||
import org.springframework.jdbc.core.namedparam.MapSqlParameterSource;
|
||||
import org.springframework.jdbc.core.namedparam.NamedParameterJdbcTemplate;
|
||||
import org.springframework.stereotype.Repository;
|
||||
|
||||
@Repository
|
||||
@RequiredArgsConstructor
|
||||
public class EmbeddingClusterResultQueryRepositoryImpl implements EmbeddingClusterResultQueryRepository {
|
||||
|
||||
private final NamedParameterJdbcTemplate jdbcTemplate;
|
||||
|
||||
@Override
|
||||
public List<ClusterAssignmentViewResponse> findAssignments(UUID runId, boolean includeText, int previewLength) {
|
||||
String sql = baseSql(includeText) + " where a.cluster_run_id = :runId order by a.cluster_label_raw asc, d.id asc";
|
||||
MapSqlParameterSource params = new MapSqlParameterSource()
|
||||
.addValue("runId", runId)
|
||||
.addValue("previewLength", previewLength);
|
||||
return jdbcTemplate.query(sql, params, assignmentRowMapper(includeText));
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<ClusterMembersResponse> findClusterMembers(UUID runId, UUID clusterId, boolean includeText, int previewLength) {
|
||||
String sql = "select c.id as result_cluster_id, c.cluster_label as result_cluster_label, x.* from doc.doc_embedding_cluster c "
|
||||
+ "join (" + baseSql(includeText) + ") x on x.cluster_id = c.id "
|
||||
+ "where c.cluster_run_id = :runId and c.id = :clusterId order by x.business_key asc, x.document_id asc";
|
||||
MapSqlParameterSource params = new MapSqlParameterSource()
|
||||
.addValue("runId", runId)
|
||||
.addValue("clusterId", clusterId)
|
||||
.addValue("previewLength", previewLength);
|
||||
return jdbcTemplate.query(sql, params, (rs, rowNum) -> new ClusterMembersResponse(
|
||||
rs.getObject("result_cluster_id", UUID.class),
|
||||
rs.getInt("result_cluster_label"),
|
||||
assignmentRowMapper(includeText).mapRow(rs, rowNum)
|
||||
));
|
||||
}
|
||||
|
||||
private String baseSql(boolean includeText) {
|
||||
String textBodyExpression = includeText ? "r.text_body" : "null::text";
|
||||
return "select a.id, a.cluster_id, a.embedding_id, a.document_id, a.representation_id, a.cluster_label_raw, "
|
||||
+ "a.membership_score, a.distance_to_centroid, a.is_noise, d.business_key, d.document_type, "
|
||||
+ "r.representation_type, r.builder_key, r.language_code, r.char_count as text_length, "
|
||||
+ "case when r.text_body is null then null "
|
||||
+ "when char_length(r.text_body) <= :previewLength then r.text_body "
|
||||
+ "else substring(r.text_body from 1 for :previewLength) end as text_preview, "
|
||||
+ textBodyExpression + " as text_body "
|
||||
+ "from doc.doc_embedding_cluster_assignment a "
|
||||
+ "join doc.doc_document d on d.id = a.document_id "
|
||||
+ "join doc.doc_text_representation r on r.id = a.representation_id";
|
||||
}
|
||||
|
||||
private RowMapper<ClusterAssignmentViewResponse> assignmentRowMapper(boolean includeText) {
|
||||
return (rs, rowNum) -> new ClusterAssignmentViewResponse(
|
||||
rs.getObject("id", UUID.class),
|
||||
rs.getObject("cluster_id", UUID.class),
|
||||
rs.getObject("embedding_id", UUID.class),
|
||||
rs.getObject("document_id", UUID.class),
|
||||
rs.getObject("representation_id", UUID.class),
|
||||
rs.getInt("cluster_label_raw"),
|
||||
rs.getObject("membership_score", Double.class),
|
||||
rs.getObject("distance_to_centroid", Double.class),
|
||||
rs.getBoolean("is_noise"),
|
||||
rs.getString("business_key"),
|
||||
enumValue(DocumentType.class, rs.getString("document_type")),
|
||||
enumValue(RepresentationType.class, rs.getString("representation_type")),
|
||||
rs.getString("builder_key"),
|
||||
rs.getString("language_code"),
|
||||
rs.getObject("text_length", Integer.class),
|
||||
rs.getString("text_preview"),
|
||||
includeText ? rs.getString("text_body") : null
|
||||
);
|
||||
}
|
||||
|
||||
private <T extends Enum<T>> T enumValue(Class<T> type, String value) {
|
||||
return value == null ? null : Enum.valueOf(type, value);
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,10 @@
|
|||
package at.procon.dip.clustering.repository;
|
||||
|
||||
import at.procon.dip.clustering.entity.EmbeddingClusterRun;
|
||||
import java.util.UUID;
|
||||
import org.springframework.data.jpa.repository.JpaRepository;
|
||||
import org.springframework.data.jpa.repository.JpaSpecificationExecutor;
|
||||
|
||||
public interface EmbeddingClusterRunRepository
|
||||
extends JpaRepository<EmbeddingClusterRun, UUID>, JpaSpecificationExecutor<EmbeddingClusterRun> {
|
||||
}
|
||||
|
|
@ -0,0 +1,45 @@
|
|||
package at.procon.dip.clustering.repository;
|
||||
|
||||
import at.procon.dip.clustering.ClusterRunStatus;
|
||||
import at.procon.dip.clustering.ClusteringAlgorithm;
|
||||
import at.procon.dip.clustering.ClusteringExecutionBackend;
|
||||
import at.procon.dip.clustering.entity.EmbeddingClusterRun;
|
||||
import at.procon.dip.domain.document.DocumentType;
|
||||
import java.time.OffsetDateTime;
|
||||
import org.springframework.data.jpa.domain.Specification;
|
||||
|
||||
public final class EmbeddingClusterRunSpecifications {
|
||||
|
||||
private EmbeddingClusterRunSpecifications() {
|
||||
}
|
||||
|
||||
public static Specification<EmbeddingClusterRun> hasStatus(ClusterRunStatus status) {
|
||||
return (root, query, cb) -> status == null ? null : cb.equal(root.get("status"), status);
|
||||
}
|
||||
|
||||
public static Specification<EmbeddingClusterRun> hasAlgorithm(ClusteringAlgorithm algorithm) {
|
||||
return (root, query, cb) -> algorithm == null ? null : cb.equal(root.get("algorithm"), algorithm);
|
||||
}
|
||||
|
||||
public static Specification<EmbeddingClusterRun> hasExecutionBackend(ClusteringExecutionBackend executionBackend) {
|
||||
return (root, query, cb) -> executionBackend == null ? null : cb.equal(root.get("executionBackend"), executionBackend);
|
||||
}
|
||||
|
||||
public static Specification<EmbeddingClusterRun> hasDocumentType(DocumentType documentType) {
|
||||
return (root, query, cb) -> documentType == null ? null : cb.equal(root.get("documentType"), documentType);
|
||||
}
|
||||
|
||||
public static Specification<EmbeddingClusterRun> nameContains(String value) {
|
||||
return (root, query, cb) -> value == null || value.isBlank()
|
||||
? null
|
||||
: cb.like(cb.lower(root.get("name")), "%" + value.toLowerCase() + "%");
|
||||
}
|
||||
|
||||
public static Specification<EmbeddingClusterRun> createdAtFrom(OffsetDateTime value) {
|
||||
return (root, query, cb) -> value == null ? null : cb.greaterThanOrEqualTo(root.get("createdAt"), value);
|
||||
}
|
||||
|
||||
public static Specification<EmbeddingClusterRun> createdAtTo(OffsetDateTime value) {
|
||||
return (root, query, cb) -> value == null ? null : cb.lessThan(root.get("createdAt"), value);
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,19 @@
|
|||
package at.procon.dip.clustering.repository;
|
||||
|
||||
import at.procon.dip.clustering.entity.EmbeddingClusterSet;
|
||||
import java.util.List;
|
||||
import java.util.Optional;
|
||||
import java.util.UUID;
|
||||
import org.springframework.data.jpa.repository.JpaRepository;
|
||||
|
||||
public interface EmbeddingClusterSetRepository extends JpaRepository<EmbeddingClusterSet, UUID> {
|
||||
Optional<EmbeddingClusterSet> findByCode(String code);
|
||||
|
||||
boolean existsByCodeIgnoreCaseAndIdNot(String code, UUID id);
|
||||
|
||||
boolean existsByCodeIgnoreCase(String code);
|
||||
|
||||
List<EmbeddingClusterSet> findAllByActiveOrderByCodeAsc(boolean active);
|
||||
|
||||
List<EmbeddingClusterSet> findAllByOrderByCodeAsc();
|
||||
}
|
||||
|
|
@ -0,0 +1,24 @@
|
|||
package at.procon.dip.clustering.service;
|
||||
|
||||
import java.util.UUID;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.scheduling.annotation.Async;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
@Slf4j
|
||||
public class EmbeddingClusterAsyncExecutionService {
|
||||
|
||||
private final EmbeddingClusterRunService runService;
|
||||
|
||||
@Async("clusteringRunExecutor")
|
||||
public void executeRunAsync(UUID runId) {
|
||||
try {
|
||||
runService.executeQueuedRun(runId);
|
||||
} catch (Exception ex) {
|
||||
log.error("Cluster run {} failed during async execution: {}", runId, ex.getMessage(), ex);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,103 @@
|
|||
package at.procon.dip.clustering.service;
|
||||
|
||||
import at.procon.dip.clustering.dto.ClusteringEngineAssignment;
|
||||
import at.procon.dip.clustering.dto.ClusteringEngineCluster;
|
||||
import at.procon.dip.clustering.dto.ClusteringEngineResult;
|
||||
import at.procon.dip.clustering.entity.EmbeddingCluster;
|
||||
import at.procon.dip.clustering.entity.EmbeddingClusterRun;
|
||||
import at.procon.dip.clustering.repository.EmbeddingClusterRepository;
|
||||
import java.time.OffsetDateTime;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.UUID;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import org.springframework.jdbc.core.namedparam.MapSqlParameterSource;
|
||||
import org.springframework.jdbc.core.namedparam.NamedParameterJdbcTemplate;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.transaction.annotation.Transactional;
|
||||
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
public class EmbeddingClusterPersistenceService {
|
||||
|
||||
private final EmbeddingClusterRepository clusterRepository;
|
||||
private final NamedParameterJdbcTemplate jdbcTemplate;
|
||||
|
||||
@Transactional
|
||||
public void persist(EmbeddingClusterRun run,
|
||||
List<ClusteringEngineCluster> clusters,
|
||||
List<ClusteringEngineAssignment> assignments) {
|
||||
persist(run, new ClusteringEngineResult(clusters, assignments));
|
||||
}
|
||||
|
||||
@Transactional
|
||||
public void persist(EmbeddingClusterRun run, ClusteringEngineResult result) {
|
||||
Map<Integer, EmbeddingCluster> clusterByLabel = new HashMap<>();
|
||||
for (ClusteringEngineCluster cluster : result.clusters()) {
|
||||
EmbeddingCluster saved = clusterRepository.save(EmbeddingCluster.builder()
|
||||
.clusterRun(run)
|
||||
.clusterLabel(cluster.clusterLabel())
|
||||
.itemCount(cluster.itemCount())
|
||||
.noiseCluster(cluster.noiseCluster())
|
||||
.build());
|
||||
clusterRepository.flush();
|
||||
clusterByLabel.put(cluster.clusterLabel(), saved);
|
||||
}
|
||||
|
||||
if (result.assignments().isEmpty()) {
|
||||
return;
|
||||
}
|
||||
|
||||
MapSqlParameterSource[] batch = result.assignments().stream()
|
||||
.map(assignment -> toInsertParams(run, clusterByLabel.get(assignment.clusterLabel()), assignment))
|
||||
.toArray(MapSqlParameterSource[]::new);
|
||||
|
||||
jdbcTemplate.batchUpdate("""
|
||||
insert into doc.doc_embedding_cluster_assignment (
|
||||
id,
|
||||
cluster_run_id,
|
||||
cluster_id,
|
||||
embedding_id,
|
||||
document_id,
|
||||
representation_id,
|
||||
cluster_label_raw,
|
||||
membership_score,
|
||||
distance_to_centroid,
|
||||
is_noise,
|
||||
created_at
|
||||
)
|
||||
select
|
||||
:id,
|
||||
:clusterRunId,
|
||||
:clusterId,
|
||||
e.id,
|
||||
e.document_id,
|
||||
e.representation_id,
|
||||
:clusterLabelRaw,
|
||||
:membershipScore,
|
||||
:distanceToCentroid,
|
||||
:isNoise,
|
||||
:createdAt
|
||||
from doc.doc_embedding e
|
||||
where e.id = :embeddingId
|
||||
""", batch);
|
||||
}
|
||||
|
||||
private MapSqlParameterSource toInsertParams(
|
||||
EmbeddingClusterRun run,
|
||||
EmbeddingCluster cluster,
|
||||
ClusteringEngineAssignment assignment
|
||||
) {
|
||||
return new MapSqlParameterSource()
|
||||
.addValue("id", UUID.randomUUID())
|
||||
.addValue("clusterRunId", run.getId())
|
||||
.addValue("clusterId", cluster == null ? null : cluster.getId())
|
||||
.addValue("embeddingId", assignment.embeddingId())
|
||||
.addValue("clusterLabelRaw", assignment.clusterLabel())
|
||||
.addValue("membershipScore", assignment.membershipScore())
|
||||
.addValue("distanceToCentroid", assignment.distanceToCentroid())
|
||||
.addValue("isNoise", assignment.noise())
|
||||
.addValue("createdAt", OffsetDateTime.now());
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,580 @@
|
|||
package at.procon.dip.clustering.service;
|
||||
|
||||
import static at.procon.dip.clustering.repository.EmbeddingClusterRunSpecifications.createdAtFrom;
|
||||
import static at.procon.dip.clustering.repository.EmbeddingClusterRunSpecifications.createdAtTo;
|
||||
import static at.procon.dip.clustering.repository.EmbeddingClusterRunSpecifications.hasAlgorithm;
|
||||
import static at.procon.dip.clustering.repository.EmbeddingClusterRunSpecifications.hasDocumentType;
|
||||
import static at.procon.dip.clustering.repository.EmbeddingClusterRunSpecifications.hasExecutionBackend;
|
||||
import static at.procon.dip.clustering.repository.EmbeddingClusterRunSpecifications.hasStatus;
|
||||
import static at.procon.dip.clustering.repository.EmbeddingClusterRunSpecifications.nameContains;
|
||||
|
||||
import at.procon.dip.clustering.ClusterRunStatus;
|
||||
import at.procon.dip.clustering.ClusteringAlgorithm;
|
||||
import at.procon.dip.clustering.ClusteringExecutionBackend;
|
||||
import at.procon.dip.clustering.ReductionMethod;
|
||||
import at.procon.dip.clustering.PythonRequestMode;
|
||||
import at.procon.dip.clustering.client.PythonClusteringClient;
|
||||
import at.procon.dip.clustering.config.ClusteringPhaseBProperties;
|
||||
import at.procon.dip.clustering.dto.ClusterAssignmentViewResponse;
|
||||
import at.procon.dip.clustering.dto.ClusterMembersResponse;
|
||||
import at.procon.dip.clustering.dto.ClusterResponse;
|
||||
import at.procon.dip.clustering.dto.ClusterRunResponse;
|
||||
import at.procon.dip.clustering.dto.ClusteringEngineAssignment;
|
||||
import at.procon.dip.clustering.dto.ClusteringEngineCluster;
|
||||
import at.procon.dip.clustering.dto.ClusteringEngineRequest;
|
||||
import at.procon.dip.clustering.dto.ClusteringEngineResult;
|
||||
import at.procon.dip.clustering.dto.CreateClusterRunRequest;
|
||||
import at.procon.dip.clustering.dto.EmbeddingSelectionSpec;
|
||||
import at.procon.dip.clustering.dto.PythonClusteringRequest;
|
||||
import at.procon.dip.clustering.dto.PythonClusteringResponse;
|
||||
import at.procon.dip.clustering.dto.PythonRunExecutionRequest;
|
||||
import at.procon.dip.clustering.dto.ReductionConfig;
|
||||
import at.procon.dip.clustering.dto.SelectedEmbeddingRow;
|
||||
import at.procon.dip.clustering.entity.EmbeddingClusterRun;
|
||||
import at.procon.dip.clustering.entity.EmbeddingClusterSet;
|
||||
import at.procon.dip.clustering.repository.EmbeddingClusterRepository;
|
||||
import at.procon.dip.clustering.repository.EmbeddingClusterResultQueryRepository;
|
||||
import at.procon.dip.clustering.repository.EmbeddingClusterRunRepository;
|
||||
import at.procon.dip.clustering.repository.EmbeddingClusterSetRepository;
|
||||
import at.procon.dip.clustering.spi.EmbeddingClusteringEngine;
|
||||
import at.procon.dip.domain.document.DocumentType;
|
||||
import com.fasterxml.jackson.core.JsonProcessingException;
|
||||
import com.fasterxml.jackson.core.type.TypeReference;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import java.time.OffsetDateTime;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Optional;
|
||||
import java.util.Set;
|
||||
import java.util.UUID;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import org.springframework.data.domain.Sort;
|
||||
import org.springframework.data.jpa.domain.Specification;
|
||||
import org.springframework.http.HttpStatus;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.transaction.annotation.Transactional;
|
||||
import org.springframework.web.server.ResponseStatusException;
|
||||
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
public class EmbeddingClusterRunService {
|
||||
|
||||
private static final int DEFAULT_PREVIEW_LENGTH = 1000;
|
||||
private static final TypeReference<Map<String, Object>> PARAMETERS_TYPE = new TypeReference<>() {
|
||||
};
|
||||
private static final Set<ClusterRunStatus> STARTABLE_STATUSES = Set.of(ClusterRunStatus.CREATED);
|
||||
private static final Set<ClusterRunStatus> FINAL_STATUSES = Set.of(
|
||||
ClusterRunStatus.COMPLETED, ClusterRunStatus.FAILED, ClusterRunStatus.CANCELLED);
|
||||
|
||||
private final EmbeddingClusterRunRepository runRepository;
|
||||
private final EmbeddingClusterSetRepository clusterSetRepository;
|
||||
private final EmbeddingClusterRepository clusterRepository;
|
||||
private final EmbeddingClusterResultQueryRepository resultQueryRepository;
|
||||
private final EmbeddingSelectionService selectionService;
|
||||
private final EmbeddingClusterPersistenceService persistenceService;
|
||||
private final List<EmbeddingClusteringEngine> clusteringEngines;
|
||||
private final ClusteringPhaseBProperties pythonProperties;
|
||||
private final Optional<PythonClusteringClient> pythonClusteringClient;
|
||||
private final ObjectMapper objectMapper;
|
||||
|
||||
@Transactional
|
||||
public ClusterRunResponse createRun(CreateClusterRunRequest request) {
|
||||
EmbeddingClusterRun run = EmbeddingClusterRun.builder()
|
||||
.clusterSet(resolveClusterSet(request.clusterSetCode()))
|
||||
.name(request.name())
|
||||
.status(ClusterRunStatus.CREATED)
|
||||
.algorithm(request.algorithm())
|
||||
.algorithmVersion("phase-e-dual-python")
|
||||
.executionBackend(resolveExecutionBackend(request))
|
||||
.reductionMethod(resolveReductionMethod(request.reduction()))
|
||||
.reductionDimensions(resolveReductionDimensions(request.reduction()))
|
||||
.selectionJson(writeJson(request.selection()))
|
||||
.parametersJson(writeJson(request.resolvedParameters()))
|
||||
.documentType(firstOrNull(request.selection() == null ? null : request.selection().documentTypes()))
|
||||
.documentFamily(firstOrNull(request.selection() == null ? null : request.selection().documentFamilies()))
|
||||
.representationType(firstOrNull(request.selection() == null ? null : request.selection().representationTypes()))
|
||||
.builderKey(firstOrNullString(request.selection() == null ? null : request.selection().builderKeys()))
|
||||
.build();
|
||||
return toResponse(runRepository.save(run));
|
||||
}
|
||||
|
||||
public ClusterRunResponse executeRun(UUID runId) {
|
||||
EmbeddingClusterRun run = loadRun(runId);
|
||||
return run.getExecutionBackend() == ClusteringExecutionBackend.PYTHON_REMOTE
|
||||
? executeRunRemote(run)
|
||||
: executeRunLocal(run);
|
||||
}
|
||||
|
||||
@Transactional
|
||||
public ClusterRunResponse queueRun(UUID runId) {
|
||||
EmbeddingClusterRun run = loadRun(runId);
|
||||
if (!STARTABLE_STATUSES.contains(run.getStatus())) {
|
||||
throw new ResponseStatusException(HttpStatus.CONFLICT,
|
||||
"Cluster run cannot be started from status " + run.getStatus());
|
||||
}
|
||||
run.setStatus(ClusterRunStatus.QUEUED);
|
||||
run.setStartedAt(null);
|
||||
run.setFinishedAt(null);
|
||||
run.setErrorMessage(null);
|
||||
run.setItemCount(null);
|
||||
run.setClusterCount(null);
|
||||
run.setNoiseCount(null);
|
||||
return toResponse(runRepository.save(run));
|
||||
}
|
||||
|
||||
public ClusterRunResponse executeQueuedRun(UUID runId) {
|
||||
EmbeddingClusterRun run = loadRun(runId);
|
||||
if (run.getStatus() == ClusterRunStatus.CANCELLED) {
|
||||
return toResponse(run);
|
||||
}
|
||||
if (run.getStatus() != ClusterRunStatus.QUEUED && run.getStatus() != ClusterRunStatus.CANCEL_REQUESTED) {
|
||||
throw new ResponseStatusException(HttpStatus.CONFLICT,
|
||||
"Cluster run is not queued for execution: " + runId);
|
||||
}
|
||||
|
||||
transitionToRunning(runId);
|
||||
|
||||
if (loadRun(runId).getExecutionBackend() == ClusteringExecutionBackend.PYTHON_REMOTE) {
|
||||
return executeQueuedRemoteRun(runId);
|
||||
}
|
||||
return executeQueuedLocalRun(runId);
|
||||
}
|
||||
|
||||
public ClusterRunResponse getRun(UUID runId) {
|
||||
return toResponse(loadRun(runId));
|
||||
}
|
||||
|
||||
public List<ClusterRunResponse> listRuns(
|
||||
ClusterRunStatus status,
|
||||
ClusteringAlgorithm algorithm,
|
||||
ClusteringExecutionBackend executionBackend,
|
||||
DocumentType documentType,
|
||||
String nameLike,
|
||||
OffsetDateTime createdFrom,
|
||||
OffsetDateTime createdTo) {
|
||||
|
||||
Specification<EmbeddingClusterRun> spec = Specification
|
||||
.where(hasStatus(status))
|
||||
.and(hasAlgorithm(algorithm))
|
||||
.and(hasExecutionBackend(executionBackend))
|
||||
.and(hasDocumentType(documentType))
|
||||
.and(nameContains(nameLike))
|
||||
.and(createdAtFrom(createdFrom))
|
||||
.and(createdAtTo(createdTo));
|
||||
|
||||
return runRepository.findAll(spec, Sort.by(Sort.Direction.DESC, "createdAt")).stream()
|
||||
.map(this::toResponse)
|
||||
.toList();
|
||||
}
|
||||
|
||||
@Transactional
|
||||
public ClusterRunResponse requestCancellation(UUID runId) {
|
||||
EmbeddingClusterRun run = loadRun(runId);
|
||||
if (FINAL_STATUSES.contains(run.getStatus())) {
|
||||
return toResponse(run);
|
||||
}
|
||||
if (run.getStatus() == ClusterRunStatus.CREATED || run.getStatus() == ClusterRunStatus.QUEUED) {
|
||||
run.setStatus(ClusterRunStatus.CANCELLED);
|
||||
run.setFinishedAt(OffsetDateTime.now());
|
||||
run.setErrorMessage("Cluster run was cancelled");
|
||||
return toResponse(runRepository.save(run));
|
||||
}
|
||||
if (run.getStatus() == ClusterRunStatus.RUNNING) {
|
||||
run.setStatus(ClusterRunStatus.CANCEL_REQUESTED);
|
||||
run.setErrorMessage("Cluster run cancellation requested");
|
||||
return toResponse(runRepository.save(run));
|
||||
}
|
||||
return toResponse(run);
|
||||
}
|
||||
|
||||
public List<ClusterResponse> listClusters(UUID runId) {
|
||||
ensureRunExists(runId);
|
||||
return clusterRepository.findByClusterRun_IdOrderByClusterLabelAsc(runId).stream()
|
||||
.map(cluster -> new ClusterResponse(
|
||||
cluster.getId(),
|
||||
cluster.getClusterLabel(),
|
||||
cluster.getDisplayName(),
|
||||
cluster.getItemCount(),
|
||||
cluster.isNoiseCluster(),
|
||||
cluster.getSummaryText()))
|
||||
.toList();
|
||||
}
|
||||
|
||||
public List<ClusterAssignmentViewResponse> listAssignments(UUID runId, boolean includeText) {
|
||||
ensureRunExists(runId);
|
||||
return resultQueryRepository.findAssignments(runId, includeText, DEFAULT_PREVIEW_LENGTH);
|
||||
}
|
||||
|
||||
public List<ClusterMembersResponse> listClusterMembers(UUID runId, UUID clusterId, boolean includeText) {
|
||||
ensureRunExists(runId);
|
||||
return resultQueryRepository.findClusterMembers(runId, clusterId, includeText, DEFAULT_PREVIEW_LENGTH);
|
||||
}
|
||||
|
||||
private ClusterRunResponse executeRunRemote(EmbeddingClusterRun run) {
|
||||
EmbeddingSelectionSpec selection = readSelection(run.getSelectionJson());
|
||||
List<SelectedEmbeddingRow> selected = null;
|
||||
long itemCount;
|
||||
|
||||
if (useCompactPythonRunMode()) {
|
||||
itemCount = selectionService.count(selection);
|
||||
} else {
|
||||
selected = selectionService.load(selection);
|
||||
itemCount = selected.size();
|
||||
}
|
||||
|
||||
if (itemCount == 0L) {
|
||||
failRun(run.getId(), "Selection contains no completed embeddings");
|
||||
return getRun(run.getId());
|
||||
}
|
||||
|
||||
run.setStatus(ClusterRunStatus.RUNNING);
|
||||
run.setStartedAt(OffsetDateTime.now());
|
||||
run.setItemCount(itemCount);
|
||||
run.setErrorMessage(null);
|
||||
run.setFinishedAt(null);
|
||||
run.setClusterCount(null);
|
||||
run.setNoiseCount(null);
|
||||
runRepository.save(run);
|
||||
|
||||
try {
|
||||
ClusteringEngineResult result = executePythonClustering(run, selected);
|
||||
persistenceService.persist(run, result);
|
||||
run.setStatus(ClusterRunStatus.COMPLETED);
|
||||
run.setFinishedAt(OffsetDateTime.now());
|
||||
run.setClusterCount(result.clusters().stream().filter(cluster -> !cluster.noiseCluster()).count());
|
||||
run.setNoiseCount(result.noiseCount());
|
||||
runRepository.save(run);
|
||||
return toResponse(run);
|
||||
} catch (Exception ex) {
|
||||
failRun(run.getId(), ex.getMessage());
|
||||
return getRun(run.getId());
|
||||
}
|
||||
}
|
||||
|
||||
private ClusterRunResponse executeRunLocal(EmbeddingClusterRun run) {
|
||||
EmbeddingSelectionSpec selection = readSelection(run.getSelectionJson());
|
||||
List<SelectedEmbeddingRow> selected = selectionService.load(selection);
|
||||
if (selected.isEmpty()) {
|
||||
failRun(run.getId(), "Selection contains no completed embeddings");
|
||||
return getRun(run.getId());
|
||||
}
|
||||
|
||||
run.setStatus(ClusterRunStatus.RUNNING);
|
||||
run.setStartedAt(OffsetDateTime.now());
|
||||
run.setItemCount((long) selected.size());
|
||||
run.setErrorMessage(null);
|
||||
run.setFinishedAt(null);
|
||||
run.setClusterCount(null);
|
||||
run.setNoiseCount(null);
|
||||
runRepository.save(run);
|
||||
|
||||
try {
|
||||
ClusteringEngineResult result = executeLocalClustering(run, selected);
|
||||
persistenceService.persist(run, result);
|
||||
run.setStatus(ClusterRunStatus.COMPLETED);
|
||||
run.setFinishedAt(OffsetDateTime.now());
|
||||
run.setClusterCount(result.clusters().stream().filter(cluster -> !cluster.noiseCluster()).count());
|
||||
run.setNoiseCount(result.noiseCount());
|
||||
runRepository.save(run);
|
||||
return toResponse(run);
|
||||
} catch (Exception ex) {
|
||||
failRun(run.getId(), ex.getMessage());
|
||||
return getRun(run.getId());
|
||||
}
|
||||
}
|
||||
|
||||
private ClusterRunResponse executeQueuedRemoteRun(UUID runId) {
|
||||
EmbeddingClusterRun run = loadRun(runId);
|
||||
EmbeddingSelectionSpec selection = readSelection(run.getSelectionJson());
|
||||
List<SelectedEmbeddingRow> selected = null;
|
||||
long itemCount;
|
||||
|
||||
if (useCompactPythonRunMode()) {
|
||||
itemCount = selectionService.count(selection);
|
||||
} else {
|
||||
selected = selectionService.load(selection);
|
||||
itemCount = selected.size();
|
||||
}
|
||||
|
||||
if (itemCount == 0L) {
|
||||
failRun(runId, "Selection contains no completed embeddings");
|
||||
return getRun(runId);
|
||||
}
|
||||
updateItemCount(runId, itemCount);
|
||||
|
||||
if (isCancellationRequested(runId)) {
|
||||
cancelRunNow(runId, "Cluster run was cancelled before clustering started");
|
||||
return getRun(runId);
|
||||
}
|
||||
|
||||
try {
|
||||
ClusteringEngineResult result = executePythonClustering(loadRun(runId), selected);
|
||||
if (isCancellationRequested(runId)) {
|
||||
cancelRunNow(runId, "Cluster run was cancelled before results were persisted");
|
||||
return getRun(runId);
|
||||
}
|
||||
persistenceService.persist(loadRun(runId), result);
|
||||
completeRun(runId, result);
|
||||
return getRun(runId);
|
||||
} catch (Exception ex) {
|
||||
failRun(runId, ex.getMessage());
|
||||
return getRun(runId);
|
||||
}
|
||||
}
|
||||
|
||||
private ClusterRunResponse executeQueuedLocalRun(UUID runId) {
|
||||
EmbeddingSelectionSpec selection = readSelection(loadRun(runId).getSelectionJson());
|
||||
List<SelectedEmbeddingRow> selected = selectionService.load(selection);
|
||||
if (selected.isEmpty()) {
|
||||
failRun(runId, "Selection contains no completed embeddings");
|
||||
return getRun(runId);
|
||||
}
|
||||
updateItemCount(runId, (long) selected.size());
|
||||
|
||||
if (isCancellationRequested(runId)) {
|
||||
cancelRunNow(runId, "Cluster run was cancelled before clustering started");
|
||||
return getRun(runId);
|
||||
}
|
||||
|
||||
try {
|
||||
ClusteringEngineResult result = executeLocalClustering(loadRun(runId), selected);
|
||||
if (isCancellationRequested(runId)) {
|
||||
cancelRunNow(runId, "Cluster run was cancelled before results were persisted");
|
||||
return getRun(runId);
|
||||
}
|
||||
persistenceService.persist(loadRun(runId), result);
|
||||
completeRun(runId, result);
|
||||
return getRun(runId);
|
||||
} catch (Exception ex) {
|
||||
failRun(runId, ex.getMessage());
|
||||
return getRun(runId);
|
||||
}
|
||||
}
|
||||
|
||||
private void transitionToRunning(UUID runId) {
|
||||
EmbeddingClusterRun run = loadRun(runId);
|
||||
if (run.getStatus() == ClusterRunStatus.CANCELLED) {
|
||||
return;
|
||||
}
|
||||
run.setStatus(ClusterRunStatus.RUNNING);
|
||||
run.setStartedAt(OffsetDateTime.now());
|
||||
run.setFinishedAt(null);
|
||||
run.setErrorMessage(null);
|
||||
runRepository.save(run);
|
||||
}
|
||||
|
||||
private void updateItemCount(UUID runId, Long itemCount) {
|
||||
EmbeddingClusterRun run = loadRun(runId);
|
||||
run.setItemCount(itemCount);
|
||||
runRepository.save(run);
|
||||
}
|
||||
|
||||
private void completeRun(UUID runId, ClusteringEngineResult result) {
|
||||
EmbeddingClusterRun run = loadRun(runId);
|
||||
run.setStatus(ClusterRunStatus.COMPLETED);
|
||||
run.setFinishedAt(OffsetDateTime.now());
|
||||
run.setClusterCount(result.clusters().stream().filter(cluster -> !cluster.noiseCluster()).count());
|
||||
run.setNoiseCount(result.noiseCount());
|
||||
run.setErrorMessage(null);
|
||||
runRepository.save(run);
|
||||
}
|
||||
|
||||
private void cancelRunNow(UUID runId, String message) {
|
||||
EmbeddingClusterRun run = loadRun(runId);
|
||||
run.setStatus(ClusterRunStatus.CANCELLED);
|
||||
run.setFinishedAt(OffsetDateTime.now());
|
||||
run.setErrorMessage(message);
|
||||
runRepository.save(run);
|
||||
}
|
||||
|
||||
private boolean isCancellationRequested(UUID runId) {
|
||||
ClusterRunStatus status = loadRun(runId).getStatus();
|
||||
return status == ClusterRunStatus.CANCEL_REQUESTED || status == ClusterRunStatus.CANCELLED;
|
||||
}
|
||||
|
||||
private ClusteringEngineResult executePythonClustering(EmbeddingClusterRun run, List<SelectedEmbeddingRow> selected) {
|
||||
PythonClusteringClient client = pythonClusteringClient.orElseThrow(() -> new ResponseStatusException(
|
||||
HttpStatus.BAD_REQUEST,
|
||||
"Cluster run requires PYTHON_REMOTE backend but no Python client is configured"));
|
||||
|
||||
PythonClusteringResponse response = useCompactPythonRunMode()
|
||||
? client.clusterRun(new PythonRunExecutionRequest(run.getId()))
|
||||
: client.cluster(new PythonClusteringRequest(
|
||||
run.getAlgorithm(),
|
||||
readParameters(run.getParametersJson()),
|
||||
run.getReductionMethod(),
|
||||
run.getReductionDimensions(),
|
||||
selected == null ? List.of() : selected.stream()
|
||||
.map(item -> new PythonClusteringRequest.PythonClusteringItem(
|
||||
item.embeddingId(),
|
||||
item.documentId(),
|
||||
item.representationId(),
|
||||
item.embeddingVector()))
|
||||
.toList()));
|
||||
|
||||
return mapPythonResponse(response);
|
||||
}
|
||||
|
||||
private boolean useCompactPythonRunMode() {
|
||||
return pythonProperties.effectiveRequestMode() == PythonRequestMode.RUN_ID;
|
||||
}
|
||||
|
||||
private ClusteringEngineResult executeLocalClustering(EmbeddingClusterRun run, List<SelectedEmbeddingRow> selected) {
|
||||
EmbeddingClusteringEngine engine = resolveEngine(run.getAlgorithm());
|
||||
return engine.cluster(selected, new ClusteringEngineRequest(readParameters(run.getParametersJson())));
|
||||
}
|
||||
|
||||
private ClusteringEngineResult mapPythonResponse(PythonClusteringResponse response) {
|
||||
long noiseCount = response.noiseCount() == null
|
||||
? response.assignments().stream().filter(assignment -> Boolean.TRUE.equals(assignment.noise())).count()
|
||||
: response.noiseCount();
|
||||
|
||||
return new ClusteringEngineResult(
|
||||
response.clusters().stream()
|
||||
.map(cluster -> new ClusteringEngineCluster(
|
||||
cluster.clusterLabel(),
|
||||
cluster.itemCount(),
|
||||
Boolean.TRUE.equals(cluster.noiseCluster())))
|
||||
.toList(),
|
||||
response.assignments().stream()
|
||||
.map(assignment -> new ClusteringEngineAssignment(
|
||||
assignment.embeddingId(),
|
||||
assignment.documentId(),
|
||||
assignment.representationId(),
|
||||
assignment.clusterLabel(),
|
||||
assignment.distanceToCentroid(),
|
||||
assignment.membershipScore(),
|
||||
Boolean.TRUE.equals(assignment.noise())))
|
||||
.toList(),
|
||||
noiseCount);
|
||||
}
|
||||
|
||||
private EmbeddingClusterSet resolveClusterSet(String clusterSetCode) {
|
||||
if (clusterSetCode == null || clusterSetCode.isBlank()) {
|
||||
return null;
|
||||
}
|
||||
return clusterSetRepository.findByCode(clusterSetCode)
|
||||
.orElseThrow(() -> new ResponseStatusException(HttpStatus.BAD_REQUEST,
|
||||
"Cluster set not found: " + clusterSetCode));
|
||||
}
|
||||
|
||||
private void ensureRunExists(UUID runId) {
|
||||
if (!runRepository.existsById(runId)) {
|
||||
throw new ResponseStatusException(HttpStatus.NOT_FOUND, "Cluster run not found: " + runId);
|
||||
}
|
||||
}
|
||||
|
||||
private void failRun(UUID runId, String message) {
|
||||
EmbeddingClusterRun run = loadRun(runId);
|
||||
run.setStatus(ClusterRunStatus.FAILED);
|
||||
run.setFinishedAt(OffsetDateTime.now());
|
||||
run.setErrorMessage(message);
|
||||
runRepository.save(run);
|
||||
}
|
||||
|
||||
private EmbeddingClusteringEngine resolveEngine(ClusteringAlgorithm algorithm) {
|
||||
return clusteringEngines.stream()
|
||||
.filter(engine -> engine.algorithm() == algorithm)
|
||||
.findFirst()
|
||||
.orElseThrow(() -> new ResponseStatusException(HttpStatus.BAD_REQUEST,
|
||||
"No clustering engine registered for algorithm " + algorithm));
|
||||
}
|
||||
|
||||
private EmbeddingSelectionSpec readSelection(String json) {
|
||||
try {
|
||||
return objectMapper.readValue(json, EmbeddingSelectionSpec.class);
|
||||
} catch (JsonProcessingException e) {
|
||||
throw new ResponseStatusException(HttpStatus.INTERNAL_SERVER_ERROR,
|
||||
"Cannot parse stored clustering selection", e);
|
||||
}
|
||||
}
|
||||
|
||||
private Map<String, Object> readParameters(String json) {
|
||||
if (json == null || json.isBlank()) {
|
||||
return Map.of();
|
||||
}
|
||||
try {
|
||||
Map<String, Object> values = objectMapper.readValue(json, PARAMETERS_TYPE);
|
||||
return values == null ? Map.of() : values;
|
||||
} catch (JsonProcessingException e) {
|
||||
throw new ResponseStatusException(HttpStatus.INTERNAL_SERVER_ERROR,
|
||||
"Cannot parse clustering parameters", e);
|
||||
}
|
||||
}
|
||||
|
||||
private String writeJson(Object value) {
|
||||
try {
|
||||
return objectMapper.writeValueAsString(value);
|
||||
} catch (JsonProcessingException e) {
|
||||
throw new ResponseStatusException(HttpStatus.INTERNAL_SERVER_ERROR,
|
||||
"Cannot serialize clustering payload", e);
|
||||
}
|
||||
}
|
||||
|
||||
private ClusteringExecutionBackend resolveExecutionBackend(CreateClusterRunRequest request) {
|
||||
if (request.executionBackend() != null) {
|
||||
if (request.executionBackend() == ClusteringExecutionBackend.JAVA_LOCAL
|
||||
&& request.reduction() != null
|
||||
&& request.reduction().method() != null
|
||||
&& request.reduction().method() != ReductionMethod.NONE) {
|
||||
throw new ResponseStatusException(HttpStatus.BAD_REQUEST,
|
||||
"JAVA_LOCAL backend does not support PCA/UMAP reduction; use PYTHON_REMOTE");
|
||||
}
|
||||
return request.executionBackend();
|
||||
}
|
||||
if (request.reduction() != null && request.reduction().method() != null && request.reduction().method() != ReductionMethod.NONE) {
|
||||
return ClusteringExecutionBackend.PYTHON_REMOTE;
|
||||
}
|
||||
return switch (request.algorithm()) {
|
||||
case KMEANS -> ClusteringExecutionBackend.JAVA_LOCAL;
|
||||
default -> ClusteringExecutionBackend.PYTHON_REMOTE;
|
||||
};
|
||||
}
|
||||
|
||||
private ReductionMethod resolveReductionMethod(ReductionConfig reduction) {
|
||||
if (reduction == null || reduction.method() == null) {
|
||||
return ReductionMethod.NONE;
|
||||
}
|
||||
return reduction.method();
|
||||
}
|
||||
|
||||
private Integer resolveReductionDimensions(ReductionConfig reduction) {
|
||||
return reduction == null ? null : reduction.targetDimensions();
|
||||
}
|
||||
|
||||
private <T> T firstOrNull(Iterable<T> values) {
|
||||
if (values == null) {
|
||||
return null;
|
||||
}
|
||||
for (T value : values) {
|
||||
return value;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
private String firstOrNullString(Iterable<String> values) {
|
||||
return firstOrNull(values);
|
||||
}
|
||||
|
||||
private ClusterRunResponse toResponse(EmbeddingClusterRun run) {
|
||||
return new ClusterRunResponse(
|
||||
run.getId(),
|
||||
run.getName(),
|
||||
run.getStatus(),
|
||||
run.getAlgorithm(),
|
||||
run.getExecutionBackend(),
|
||||
run.getReductionMethod(),
|
||||
run.getReductionDimensions(),
|
||||
run.getItemCount(),
|
||||
run.getClusterCount(),
|
||||
run.getNoiseCount(),
|
||||
run.getStartedAt(),
|
||||
run.getFinishedAt(),
|
||||
run.getErrorMessage());
|
||||
}
|
||||
|
||||
private EmbeddingClusterRun loadRun(UUID runId) {
|
||||
return runRepository.findById(runId)
|
||||
.orElseThrow(() -> new ResponseStatusException(HttpStatus.NOT_FOUND, "Cluster run not found: " + runId));
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,95 @@
|
|||
package at.procon.dip.clustering.service;
|
||||
|
||||
import at.procon.dip.clustering.dto.ClusterSetResponse;
|
||||
import at.procon.dip.clustering.dto.CreateClusterSetRequest;
|
||||
import at.procon.dip.clustering.dto.EmbeddingSelectionSpec;
|
||||
import at.procon.dip.clustering.dto.UpdateClusterSetRequest;
|
||||
import at.procon.dip.clustering.entity.EmbeddingClusterSet;
|
||||
import at.procon.dip.clustering.repository.EmbeddingClusterSetRepository;
|
||||
import com.fasterxml.jackson.core.JsonProcessingException;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import org.springframework.transaction.annotation.Transactional;
|
||||
import java.util.List;
|
||||
import java.util.UUID;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import org.springframework.http.HttpStatus;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.web.server.ResponseStatusException;
|
||||
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
public class EmbeddingClusterSetService {
|
||||
|
||||
private final EmbeddingClusterSetRepository clusterSetRepository;
|
||||
private final ObjectMapper objectMapper;
|
||||
|
||||
@Transactional
|
||||
public ClusterSetResponse create(CreateClusterSetRequest request) {
|
||||
if (clusterSetRepository.existsByCodeIgnoreCase(request.code())) {
|
||||
throw new ResponseStatusException(HttpStatus.CONFLICT,
|
||||
"Cluster set code already exists: " + request.code());
|
||||
}
|
||||
EmbeddingClusterSet saved = clusterSetRepository.save(EmbeddingClusterSet.builder()
|
||||
.code(request.code().trim())
|
||||
.name(request.name().trim())
|
||||
.description(request.description())
|
||||
.selectionJson(writeJson(request.selection()))
|
||||
.active(request.active() == null || request.active())
|
||||
.build());
|
||||
return toResponse(saved);
|
||||
}
|
||||
|
||||
@Transactional
|
||||
public ClusterSetResponse update(UUID id, UpdateClusterSetRequest request) {
|
||||
EmbeddingClusterSet existing = clusterSetRepository.findById(id)
|
||||
.orElseThrow(() -> new ResponseStatusException(HttpStatus.NOT_FOUND, "Cluster set not found: " + id));
|
||||
existing.setName(request.name().trim());
|
||||
existing.setDescription(request.description());
|
||||
existing.setSelectionJson(writeJson(request.selection()));
|
||||
existing.setActive(request.active() == null || request.active());
|
||||
return toResponse(clusterSetRepository.save(existing));
|
||||
}
|
||||
|
||||
public ClusterSetResponse get(UUID id) {
|
||||
return toResponse(clusterSetRepository.findById(id)
|
||||
.orElseThrow(() -> new ResponseStatusException(HttpStatus.NOT_FOUND, "Cluster set not found: " + id)));
|
||||
}
|
||||
|
||||
public List<ClusterSetResponse> list(Boolean activeOnly) {
|
||||
List<EmbeddingClusterSet> sets = activeOnly == null
|
||||
? clusterSetRepository.findAllByOrderByCodeAsc()
|
||||
: clusterSetRepository.findAllByActiveOrderByCodeAsc(activeOnly);
|
||||
return sets.stream().map(this::toResponse).toList();
|
||||
}
|
||||
|
||||
private ClusterSetResponse toResponse(EmbeddingClusterSet entity) {
|
||||
return new ClusterSetResponse(
|
||||
entity.getId(),
|
||||
entity.getCode(),
|
||||
entity.getName(),
|
||||
entity.getDescription(),
|
||||
entity.isActive(),
|
||||
readSelection(entity.getSelectionJson()),
|
||||
entity.getCreatedAt(),
|
||||
entity.getUpdatedAt()
|
||||
);
|
||||
}
|
||||
|
||||
private EmbeddingSelectionSpec readSelection(String json) {
|
||||
try {
|
||||
return objectMapper.readValue(json, EmbeddingSelectionSpec.class);
|
||||
} catch (JsonProcessingException e) {
|
||||
throw new ResponseStatusException(HttpStatus.INTERNAL_SERVER_ERROR,
|
||||
"Cannot parse stored cluster set selection", e);
|
||||
}
|
||||
}
|
||||
|
||||
private String writeJson(Object value) {
|
||||
try {
|
||||
return objectMapper.writeValueAsString(value);
|
||||
} catch (JsonProcessingException e) {
|
||||
throw new ResponseStatusException(HttpStatus.INTERNAL_SERVER_ERROR,
|
||||
"Cannot serialize cluster set selection", e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,23 @@
|
|||
package at.procon.dip.clustering.service;
|
||||
|
||||
import at.procon.dip.clustering.dto.EmbeddingSelectionSpec;
|
||||
import at.procon.dip.clustering.dto.SelectedEmbeddingRow;
|
||||
import at.procon.dip.clustering.repository.DocumentEmbeddingClusterSelectionRepository;
|
||||
import java.util.List;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
public class EmbeddingSelectionService {
|
||||
|
||||
private final DocumentEmbeddingClusterSelectionRepository selectionRepository;
|
||||
|
||||
public long count(EmbeddingSelectionSpec spec) {
|
||||
return selectionRepository.countSelection(spec);
|
||||
}
|
||||
|
||||
public List<SelectedEmbeddingRow> load(EmbeddingSelectionSpec spec) {
|
||||
return selectionRepository.findSelection(spec);
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,14 @@
|
|||
package at.procon.dip.clustering.spi;
|
||||
|
||||
import at.procon.dip.clustering.ClusteringAlgorithm;
|
||||
import at.procon.dip.clustering.dto.ClusteringEngineRequest;
|
||||
import at.procon.dip.clustering.dto.ClusteringEngineResult;
|
||||
import at.procon.dip.clustering.dto.SelectedEmbeddingRow;
|
||||
import java.util.List;
|
||||
|
||||
public interface EmbeddingClusteringEngine {
|
||||
|
||||
ClusteringAlgorithm algorithm();
|
||||
|
||||
ClusteringEngineResult cluster(List<SelectedEmbeddingRow> items, ClusteringEngineRequest request);
|
||||
}
|
||||
|
|
@ -0,0 +1,104 @@
|
|||
package at.procon.dip.clustering.spi;
|
||||
|
||||
import at.procon.dip.clustering.ClusteringAlgorithm;
|
||||
import at.procon.dip.clustering.dto.ClusteringEngineAssignment;
|
||||
import at.procon.dip.clustering.dto.ClusteringEngineCluster;
|
||||
import at.procon.dip.clustering.dto.ClusteringEngineRequest;
|
||||
import at.procon.dip.clustering.dto.ClusteringEngineResult;
|
||||
import at.procon.dip.clustering.dto.SelectedEmbeddingRow;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import org.apache.commons.math3.ml.clustering.CentroidCluster;
|
||||
import org.apache.commons.math3.ml.clustering.DoublePoint;
|
||||
import org.apache.commons.math3.ml.clustering.KMeansPlusPlusClusterer;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
@Component
|
||||
public class KMeansClusteringEngine implements EmbeddingClusteringEngine {
|
||||
|
||||
@Override
|
||||
public ClusteringAlgorithm algorithm() {
|
||||
return ClusteringAlgorithm.KMEANS;
|
||||
}
|
||||
|
||||
@Override
|
||||
public ClusteringEngineResult cluster(List<SelectedEmbeddingRow> items, ClusteringEngineRequest request) {
|
||||
if (items == null || items.isEmpty()) {
|
||||
throw new IllegalArgumentException("Selection contains no embeddings to cluster");
|
||||
}
|
||||
if (request == null) {
|
||||
throw new IllegalArgumentException("Missing clustering request");
|
||||
}
|
||||
|
||||
int k = request.requiredInt("k");
|
||||
if (k <= 0) {
|
||||
throw new IllegalArgumentException("KMeans requires k > 0");
|
||||
}
|
||||
if (k > items.size()) {
|
||||
throw new IllegalArgumentException("KMeans k must be <= selected item count");
|
||||
}
|
||||
|
||||
int maxIterations = request.intValue("maxIterations", 100);
|
||||
|
||||
List<IndexedPoint> points = new ArrayList<>(items.size());
|
||||
for (int i = 0; i < items.size(); i++) {
|
||||
points.add(new IndexedPoint(i, toDouble(items.get(i).embeddingVector())));
|
||||
}
|
||||
|
||||
KMeansPlusPlusClusterer<IndexedPoint> clusterer = new KMeansPlusPlusClusterer<>(k, maxIterations);
|
||||
List<CentroidCluster<IndexedPoint>> clusters = clusterer.cluster(points);
|
||||
|
||||
List<ClusteringEngineCluster> resultClusters = new ArrayList<>();
|
||||
List<ClusteringEngineAssignment> assignments = new ArrayList<>();
|
||||
|
||||
for (int label = 0; label < clusters.size(); label++) {
|
||||
CentroidCluster<IndexedPoint> cluster = clusters.get(label);
|
||||
resultClusters.add(new ClusteringEngineCluster(label, cluster.getPoints().size(), false));
|
||||
double[] centroid = cluster.getCenter().getPoint();
|
||||
for (IndexedPoint point : cluster.getPoints()) {
|
||||
SelectedEmbeddingRow item = items.get(point.index());
|
||||
assignments.add(new ClusteringEngineAssignment(
|
||||
item.embeddingId(),
|
||||
item.documentId(),
|
||||
item.representationId(),
|
||||
label,
|
||||
euclidean(point.getPoint(), centroid),
|
||||
null,
|
||||
false
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
return new ClusteringEngineResult(resultClusters, assignments, 0L);
|
||||
}
|
||||
|
||||
private double[] toDouble(float[] values) {
|
||||
double[] result = new double[values.length];
|
||||
for (int i = 0; i < values.length; i++) {
|
||||
result[i] = values[i];
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
private double euclidean(double[] a, double[] b) {
|
||||
double sum = 0.0d;
|
||||
for (int i = 0; i < a.length; i++) {
|
||||
double d = a[i] - b[i];
|
||||
sum += d * d;
|
||||
}
|
||||
return Math.sqrt(sum);
|
||||
}
|
||||
|
||||
private static class IndexedPoint extends DoublePoint {
|
||||
private final int index;
|
||||
|
||||
IndexedPoint(int index, double[] point) {
|
||||
super(point);
|
||||
this.index = index;
|
||||
}
|
||||
|
||||
int index() {
|
||||
return index;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,130 @@
|
|||
package at.procon.dip.clustering.web;
|
||||
|
||||
import at.procon.dip.clustering.ClusterRunStatus;
|
||||
import at.procon.dip.clustering.ClusteringAlgorithm;
|
||||
import at.procon.dip.clustering.ClusteringExecutionBackend;
|
||||
import at.procon.dip.clustering.dto.ClusterAssignmentViewResponse;
|
||||
import at.procon.dip.clustering.dto.ClusterMembersResponse;
|
||||
import at.procon.dip.clustering.dto.ClusterResponse;
|
||||
import at.procon.dip.clustering.dto.ClusterRunResponse;
|
||||
import at.procon.dip.clustering.dto.ClusterSetResponse;
|
||||
import at.procon.dip.clustering.dto.CreateClusterRunRequest;
|
||||
import at.procon.dip.clustering.dto.CreateClusterSetRequest;
|
||||
import at.procon.dip.clustering.dto.EmbeddingSelectionSpec;
|
||||
import at.procon.dip.clustering.dto.SelectionCountResponse;
|
||||
import at.procon.dip.clustering.dto.UpdateClusterSetRequest;
|
||||
import at.procon.dip.clustering.service.EmbeddingClusterAsyncExecutionService;
|
||||
import at.procon.dip.clustering.service.EmbeddingClusterRunService;
|
||||
import at.procon.dip.clustering.service.EmbeddingClusterSetService;
|
||||
import at.procon.dip.clustering.service.EmbeddingSelectionService;
|
||||
import at.procon.dip.domain.document.DocumentType;
|
||||
import jakarta.validation.Valid;
|
||||
import java.time.OffsetDateTime;
|
||||
import java.util.List;
|
||||
import java.util.UUID;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import org.springframework.format.annotation.DateTimeFormat;
|
||||
import org.springframework.http.ResponseEntity;
|
||||
import org.springframework.web.bind.annotation.GetMapping;
|
||||
import org.springframework.web.bind.annotation.PathVariable;
|
||||
import org.springframework.web.bind.annotation.PostMapping;
|
||||
import org.springframework.web.bind.annotation.PutMapping;
|
||||
import org.springframework.web.bind.annotation.RequestBody;
|
||||
import org.springframework.web.bind.annotation.RequestMapping;
|
||||
import org.springframework.web.bind.annotation.RequestParam;
|
||||
import org.springframework.web.bind.annotation.RestController;
|
||||
|
||||
@RestController
|
||||
@RequestMapping("/v1/dip/clustering")
|
||||
@RequiredArgsConstructor
|
||||
public class EmbeddingClusterController {
|
||||
|
||||
private final EmbeddingSelectionService selectionService;
|
||||
private final EmbeddingClusterSetService clusterSetService;
|
||||
private final EmbeddingClusterRunService runService;
|
||||
private final EmbeddingClusterAsyncExecutionService asyncExecutionService;
|
||||
|
||||
@PostMapping("/selection/count")
|
||||
public ResponseEntity<SelectionCountResponse> countSelection(@RequestBody EmbeddingSelectionSpec selection) {
|
||||
return ResponseEntity.ok(new SelectionCountResponse(selectionService.count(selection)));
|
||||
}
|
||||
|
||||
@PostMapping("/sets")
|
||||
public ResponseEntity<ClusterSetResponse> createSet(@Valid @RequestBody CreateClusterSetRequest request) {
|
||||
return ResponseEntity.ok(clusterSetService.create(request));
|
||||
}
|
||||
|
||||
@PutMapping("/sets/{id}")
|
||||
public ResponseEntity<ClusterSetResponse> updateSet(@PathVariable UUID id,
|
||||
@Valid @RequestBody UpdateClusterSetRequest request) {
|
||||
return ResponseEntity.ok(clusterSetService.update(id, request));
|
||||
}
|
||||
|
||||
@GetMapping("/sets")
|
||||
public ResponseEntity<List<ClusterSetResponse>> listSets(
|
||||
@RequestParam(name = "active", required = false) Boolean active) {
|
||||
return ResponseEntity.ok(clusterSetService.list(active));
|
||||
}
|
||||
|
||||
@GetMapping("/sets/{id}")
|
||||
public ResponseEntity<ClusterSetResponse> getSet(@PathVariable UUID id) {
|
||||
return ResponseEntity.ok(clusterSetService.get(id));
|
||||
}
|
||||
|
||||
@PostMapping("/runs")
|
||||
public ResponseEntity<ClusterRunResponse> createRun(@Valid @RequestBody CreateClusterRunRequest request) {
|
||||
return ResponseEntity.ok(runService.createRun(request));
|
||||
}
|
||||
|
||||
@GetMapping("/runs")
|
||||
public ResponseEntity<List<ClusterRunResponse>> listRuns(
|
||||
@RequestParam(name = "status", required = false) ClusterRunStatus status,
|
||||
@RequestParam(name = "algorithm", required = false) ClusteringAlgorithm algorithm,
|
||||
@RequestParam(name = "executionBackend", required = false) ClusteringExecutionBackend executionBackend,
|
||||
@RequestParam(name = "documentType", required = false) DocumentType documentType,
|
||||
@RequestParam(name = "nameLike", required = false) String nameLike,
|
||||
@RequestParam(name = "createdFrom", required = false)
|
||||
@DateTimeFormat(iso = DateTimeFormat.ISO.DATE_TIME) OffsetDateTime createdFrom,
|
||||
@RequestParam(name = "createdTo", required = false)
|
||||
@DateTimeFormat(iso = DateTimeFormat.ISO.DATE_TIME) OffsetDateTime createdTo) {
|
||||
return ResponseEntity.ok(runService.listRuns(
|
||||
status, algorithm, executionBackend, documentType, nameLike, createdFrom, createdTo));
|
||||
}
|
||||
|
||||
@PostMapping("/runs/{id}/start")
|
||||
public ResponseEntity<ClusterRunResponse> startRun(@PathVariable UUID id) {
|
||||
ClusterRunResponse queued = runService.queueRun(id);
|
||||
asyncExecutionService.executeRunAsync(id);
|
||||
return ResponseEntity.accepted().body(queued);
|
||||
}
|
||||
|
||||
@PostMapping("/runs/{id}/cancel")
|
||||
public ResponseEntity<ClusterRunResponse> cancelRun(@PathVariable UUID id) {
|
||||
return ResponseEntity.ok(runService.requestCancellation(id));
|
||||
}
|
||||
|
||||
@GetMapping("/runs/{id}")
|
||||
public ResponseEntity<ClusterRunResponse> getRun(@PathVariable UUID id) {
|
||||
return ResponseEntity.ok(runService.getRun(id));
|
||||
}
|
||||
|
||||
@GetMapping("/runs/{id}/clusters")
|
||||
public ResponseEntity<List<ClusterResponse>> listClusters(@PathVariable UUID id) {
|
||||
return ResponseEntity.ok(runService.listClusters(id));
|
||||
}
|
||||
|
||||
@GetMapping("/runs/{id}/assignments")
|
||||
public ResponseEntity<List<ClusterAssignmentViewResponse>> listAssignments(
|
||||
@PathVariable UUID id,
|
||||
@RequestParam(name = "includeText", defaultValue = "false") boolean includeText) {
|
||||
return ResponseEntity.ok(runService.listAssignments(id, includeText));
|
||||
}
|
||||
|
||||
@GetMapping("/runs/{runId}/clusters/{clusterId}/members")
|
||||
public ResponseEntity<List<ClusterMembersResponse>> listClusterMembers(
|
||||
@PathVariable UUID runId,
|
||||
@PathVariable UUID clusterId,
|
||||
@RequestParam(name = "includeText", defaultValue = "false") boolean includeText) {
|
||||
return ResponseEntity.ok(runService.listClusterMembers(runId, clusterId, includeText));
|
||||
}
|
||||
}
|
||||
|
|
@ -77,7 +77,7 @@ public class Document {
|
|||
@Builder.Default
|
||||
private DocumentStatus status = DocumentStatus.RECEIVED;
|
||||
|
||||
@Column(name = "title", length = 1000)
|
||||
@Column(name = "title", columnDefinition = "TEXT")
|
||||
private String title;
|
||||
|
||||
@Column(name = "summary", columnDefinition = "TEXT")
|
||||
|
|
|
|||
|
|
@ -176,6 +176,16 @@ dip:
|
|||
profile-key: disabled
|
||||
enabled: false
|
||||
|
||||
clustering:
|
||||
python:
|
||||
enabled: true
|
||||
base-url: http://localhost:8001
|
||||
cluster-path: /cluster
|
||||
cluster-run-path: /cluster-run
|
||||
request-mode: INLINE_VECTORS
|
||||
connect-timeout: 30s
|
||||
read-timeout: 30m
|
||||
|
||||
# Phase 4 generic ingestion configuration
|
||||
ingestion:
|
||||
# Master switch for arbitrary document ingestion into the DOC model
|
||||
|
|
@ -275,7 +285,7 @@ dip:
|
|||
# ted packages download configuration
|
||||
ted-download:
|
||||
# Enable/disable automatic package download
|
||||
enabled: true
|
||||
enabled: false
|
||||
# Base URL for TED Daily Packages
|
||||
base-url: https://ted.europa.eu/packages/daily/
|
||||
# Download directory for tar.gz files
|
||||
|
|
@ -304,6 +314,10 @@ dip:
|
|||
leitstand:
|
||||
enabled: false
|
||||
startup-sync-enabled: false
|
||||
startup-selective-materialization-enabled: true
|
||||
selective-materialization-person-dbk: 100920031023144811001000
|
||||
selective-materialization-person-number:
|
||||
selective-materialization-build-projection: true
|
||||
create-canonical-time-entries: true
|
||||
build-search-projection: true
|
||||
build-representations: true
|
||||
|
|
|
|||
|
|
@ -0,0 +1,107 @@
|
|||
CREATE TABLE IF NOT EXISTS doc.doc_embedding_cluster_set (
|
||||
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
code VARCHAR(128) NOT NULL UNIQUE,
|
||||
name VARCHAR(255) NOT NULL,
|
||||
description TEXT,
|
||||
selection_json JSONB NOT NULL,
|
||||
active BOOLEAN NOT NULL DEFAULT TRUE,
|
||||
created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS doc.doc_embedding_cluster_run (
|
||||
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
cluster_set_id UUID,
|
||||
name VARCHAR(255) NOT NULL,
|
||||
status VARCHAR(32) NOT NULL,
|
||||
algorithm VARCHAR(64) NOT NULL,
|
||||
algorithm_version VARCHAR(64),
|
||||
selection_json JSONB NOT NULL,
|
||||
parameters_json JSONB NOT NULL,
|
||||
embedding_model_id UUID,
|
||||
prefix_profile_id UUID,
|
||||
document_type VARCHAR(64),
|
||||
document_family VARCHAR(64),
|
||||
representation_type VARCHAR(64),
|
||||
builder_key VARCHAR(255),
|
||||
item_count BIGINT,
|
||||
cluster_count BIGINT,
|
||||
noise_count BIGINT,
|
||||
started_at TIMESTAMP WITH TIME ZONE,
|
||||
finished_at TIMESTAMP WITH TIME ZONE,
|
||||
error_message TEXT,
|
||||
created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
CONSTRAINT fk_doc_cluster_run_set
|
||||
FOREIGN KEY (cluster_set_id)
|
||||
REFERENCES doc.doc_embedding_cluster_set(id),
|
||||
CONSTRAINT fk_doc_cluster_run_model
|
||||
FOREIGN KEY (embedding_model_id)
|
||||
REFERENCES doc.doc_embedding_model(id),
|
||||
CONSTRAINT fk_doc_cluster_run_prefix_profile
|
||||
FOREIGN KEY (prefix_profile_id)
|
||||
REFERENCES doc.doc_embedding_prefix_profile(id)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_doc_cluster_run_status
|
||||
ON doc.doc_embedding_cluster_run(status);
|
||||
CREATE INDEX IF NOT EXISTS idx_doc_cluster_run_algorithm
|
||||
ON doc.doc_embedding_cluster_run(algorithm);
|
||||
CREATE INDEX IF NOT EXISTS idx_doc_cluster_run_created_at
|
||||
ON doc.doc_embedding_cluster_run(created_at);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS doc.doc_embedding_cluster (
|
||||
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
cluster_run_id UUID NOT NULL,
|
||||
cluster_label INTEGER NOT NULL,
|
||||
display_name VARCHAR(255),
|
||||
item_count BIGINT NOT NULL,
|
||||
is_noise_cluster BOOLEAN NOT NULL DEFAULT FALSE,
|
||||
summary_text TEXT,
|
||||
top_terms_json JSONB,
|
||||
created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
CONSTRAINT fk_doc_cluster_cluster_run
|
||||
FOREIGN KEY (cluster_run_id)
|
||||
REFERENCES doc.doc_embedding_cluster_run(id)
|
||||
ON DELETE CASCADE,
|
||||
CONSTRAINT uq_doc_cluster_run_label
|
||||
UNIQUE (cluster_run_id, cluster_label)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_doc_cluster_cluster_run
|
||||
ON doc.doc_embedding_cluster(cluster_run_id);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS doc.doc_embedding_cluster_assignment (
|
||||
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
cluster_run_id UUID NOT NULL,
|
||||
cluster_id UUID,
|
||||
embedding_id UUID NOT NULL,
|
||||
document_id UUID NOT NULL,
|
||||
representation_id UUID NOT NULL,
|
||||
cluster_label_raw INTEGER NOT NULL,
|
||||
membership_score DOUBLE PRECISION,
|
||||
distance_to_centroid DOUBLE PRECISION,
|
||||
is_noise BOOLEAN NOT NULL DEFAULT FALSE,
|
||||
created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
CONSTRAINT fk_doc_cluster_assignment_run
|
||||
FOREIGN KEY (cluster_run_id)
|
||||
REFERENCES doc.doc_embedding_cluster_run(id)
|
||||
ON DELETE CASCADE,
|
||||
CONSTRAINT fk_doc_cluster_assignment_cluster
|
||||
FOREIGN KEY (cluster_id)
|
||||
REFERENCES doc.doc_embedding_cluster(id)
|
||||
ON DELETE CASCADE,
|
||||
CONSTRAINT fk_doc_cluster_assignment_embedding
|
||||
FOREIGN KEY (embedding_id)
|
||||
REFERENCES doc.doc_embedding(id),
|
||||
CONSTRAINT uq_doc_cluster_run_embedding
|
||||
UNIQUE (cluster_run_id, embedding_id)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_doc_cluster_assignment_run
|
||||
ON doc.doc_embedding_cluster_assignment(cluster_run_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_doc_cluster_assignment_cluster
|
||||
ON doc.doc_embedding_cluster_assignment(cluster_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_doc_cluster_assignment_document
|
||||
ON doc.doc_embedding_cluster_assignment(cluster_run_id, document_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_doc_cluster_assignment_noise
|
||||
ON doc.doc_embedding_cluster_assignment(cluster_run_id, is_noise);
|
||||
|
|
@ -0,0 +1,7 @@
|
|||
ALTER TABLE doc.doc_embedding_cluster_run
|
||||
ADD COLUMN IF NOT EXISTS execution_backend VARCHAR(64),
|
||||
ADD COLUMN IF NOT EXISTS reduction_method VARCHAR(32),
|
||||
ADD COLUMN IF NOT EXISTS reduction_dimensions INTEGER;
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_doc_cluster_run_backend
|
||||
ON doc.doc_embedding_cluster_run(execution_backend);
|
||||
|
|
@ -0,0 +1,52 @@
|
|||
-- V31__doc_embedding_clustering_enum_constraints.sql
|
||||
-- Updates check constraints for clustering run enums after adding new algorithms and statuses.
|
||||
|
||||
ALTER TABLE doc.doc_embedding_cluster_run
|
||||
DROP CONSTRAINT IF EXISTS doc_embedding_cluster_run_algorithm_check;
|
||||
|
||||
ALTER TABLE doc.doc_embedding_cluster_run
|
||||
ADD CONSTRAINT doc_embedding_cluster_run_algorithm_check
|
||||
CHECK (algorithm IN (
|
||||
'KMEANS',
|
||||
'MINI_BATCH_KMEANS',
|
||||
'DBSCAN',
|
||||
'HDBSCAN',
|
||||
'AGGLOMERATIVE'
|
||||
));
|
||||
|
||||
ALTER TABLE doc.doc_embedding_cluster_run
|
||||
DROP CONSTRAINT IF EXISTS doc_embedding_cluster_run_status_check;
|
||||
|
||||
ALTER TABLE doc.doc_embedding_cluster_run
|
||||
ADD CONSTRAINT doc_embedding_cluster_run_status_check
|
||||
CHECK (status IN (
|
||||
'CREATED',
|
||||
'QUEUED',
|
||||
'RUNNING',
|
||||
'CANCEL_REQUESTED',
|
||||
'COMPLETED',
|
||||
'FAILED',
|
||||
'CANCELLED'
|
||||
));
|
||||
|
||||
-- Optional hardening in case these columns were also created with check constraints.
|
||||
ALTER TABLE doc.doc_embedding_cluster_run
|
||||
DROP CONSTRAINT IF EXISTS doc_embedding_cluster_run_execution_backend_check;
|
||||
|
||||
ALTER TABLE doc.doc_embedding_cluster_run
|
||||
ADD CONSTRAINT doc_embedding_cluster_run_execution_backend_check
|
||||
CHECK (execution_backend IN (
|
||||
'JAVA_LOCAL',
|
||||
'PYTHON_REMOTE'
|
||||
));
|
||||
|
||||
ALTER TABLE doc.doc_embedding_cluster_run
|
||||
DROP CONSTRAINT IF EXISTS doc_embedding_cluster_run_reduction_method_check;
|
||||
|
||||
ALTER TABLE doc.doc_embedding_cluster_run
|
||||
ADD CONSTRAINT doc_embedding_cluster_run_reduction_method_check
|
||||
CHECK (reduction_method IN (
|
||||
'NONE',
|
||||
'PCA',
|
||||
'UMAP'
|
||||
));
|
||||
Loading…
Reference in New Issue