From 21edbc35a267f1f3a808de60b8359899c14ea18e Mon Sep 17 00:00:00 2001 From: trifonovt <87468028+TihomirTrifonov@users.noreply.github.com> Date: Tue, 17 Mar 2026 11:31:06 +0100 Subject: [PATCH] Initial import --- DAILY_PACKAGE_DOWNLOAD.md | 109 +++ Dockerfile.embedding | 39 ++ EXECUTE_ENUM_FIX.md | 67 ++ LICENSE | 152 +++++ MEMORY-OPTIMIZATION.md | 139 ++++ README.md | 402 +++++++++++ Search-TED.ps1 | 164 +++++ TED_AUTOMATED_PIPELINE.md | 467 +++++++++++++ TED_NOTICE_URL.md | 149 ++++ TED_PACKAGE_DOWNLOAD_CAMEL_ROUTE.md | 374 +++++++++++ VECTORIZATION.md | 385 +++++++++++ XPATH_EXAMPLES.md | 241 +++++++ docker-compose.yml | 102 +++ docs/architecture/architecture.archimate | 35 + docs/architecture/architecture.archimate.bak | 14 + embedding_service.py | 340 ++++++++++ execute-enum-fix.bat | 30 + fix-organization-schema.bat | 21 + fix-organization-schema.sql | 20 + pom.xml | 265 ++++++++ requirements-embedding.txt | 16 + reset-stuck-packages.sql | 15 + solution-brief-processed.dat | 0 .../TedProcurementProcessorApplication.java | 26 + .../java/at/procon/ted/camel/MailRoute.java | 485 +++++++++++++ .../procon/ted/camel/SolutionBriefRoute.java | 180 +++++ .../at/procon/ted/camel/TedDocumentRoute.java | 164 +++++ .../camel/TedPackageDownloadCamelRoute.java | 635 ++++++++++++++++++ .../ted/camel/TedPackageDownloadRoute.java | 158 +++++ .../procon/ted/camel/VectorizationRoute.java | 360 ++++++++++ .../at/procon/ted/config/AsyncConfig.java | 78 +++ .../at/procon/ted/config/CamelConfig.java | 34 + .../at/procon/ted/config/OpenApiConfig.java | 58 ++ .../ted/config/TedProcessorProperties.java | 431 ++++++++++++ .../ted/controller/AdminController.java | 264 ++++++++ .../ted/controller/DocumentController.java | 314 +++++++++ .../SimilaritySearchController.java | 175 +++++ .../procon/ted/event/DocumentSavedEvent.java | 12 + .../ted/event/VectorizationEventListener.java | 46 ++ .../at/procon/ted/model/dto/DocumentDtos.java | 264 ++++++++ .../ted/model/entity/ContractNature.java | 15 + .../procon/ted/model/entity/NoticeType.java | 15 + .../procon/ted/model/entity/Organization.java | 90 +++ .../ted/model/entity/ProcedureType.java | 17 + .../ted/model/entity/ProcessedAttachment.java | 146 ++++ .../ted/model/entity/ProcessingLog.java | 91 +++ .../ted/model/entity/ProcurementDocument.java | 281 ++++++++ .../ted/model/entity/ProcurementLot.java | 92 +++ .../ted/model/entity/TedDailyPackage.java | 163 +++++ .../ted/model/entity/VectorizationStatus.java | 16 + .../ProcessedAttachmentRepository.java | 58 ++ .../repository/ProcessingLogRepository.java | 31 + .../ProcurementDocumentRepository.java | 232 +++++++ .../repository/TedDailyPackageRepository.java | 90 +++ .../BatchDocumentProcessingService.java | 183 +++++ .../ted/service/DataCleanupService.java | 95 +++ .../service/DocumentProcessingService.java | 195 ++++++ .../ted/service/ExcelExportService.java | 241 +++++++ .../ted/service/ProcessingLogService.java | 40 ++ .../at/procon/ted/service/SearchService.java | 456 +++++++++++++ .../ted/service/SimilaritySearchService.java | 253 +++++++ .../service/TedPackageDownloadService.java | 558 +++++++++++++++ .../VectorizationProcessorService.java | 123 ++++ .../ted/service/VectorizationService.java | 164 +++++ .../procon/ted/service/XmlParserService.java | 597 ++++++++++++++++ .../attachment/AttachmentExtractor.java | 87 +++ .../AttachmentProcessingService.java | 237 +++++++ .../attachment/PdfExtractionService.java | 115 ++++ .../attachment/ZipExtractionService.java | 234 +++++++ .../startup/OrganizationSchemaFixRunner.java | 104 +++ .../startup/VectorizationStartupRunner.java | 112 +++ .../java/at/procon/ted/util/HashUtils.java | 82 +++ .../at/procon/ted/util/InspectDatabase.java | 86 +++ src/main/resources/application - Kopie.yml | 234 +++++++ src/main/resources/application.yml | 240 +++++++ src/main/resources/banner.txt | 12 + .../db/migration/V1__initial_schema.sql | 428 ++++++++++++ ...V2__extend_organization_varchar_fields.sql | 28 + .../V3__add_processed_attachment_table.sql | 72 ++ .../ted/service/XmlParserServiceTest.java | 199 ++++++ start.bat | 9 + start.sh | 7 + ted-procurement-processor.zip | Bin 0 -> 160870 bytes 83 files changed, 13758 insertions(+) create mode 100644 DAILY_PACKAGE_DOWNLOAD.md create mode 100644 Dockerfile.embedding create mode 100644 EXECUTE_ENUM_FIX.md create mode 100644 LICENSE create mode 100644 MEMORY-OPTIMIZATION.md create mode 100644 README.md create mode 100644 Search-TED.ps1 create mode 100644 TED_AUTOMATED_PIPELINE.md create mode 100644 TED_NOTICE_URL.md create mode 100644 TED_PACKAGE_DOWNLOAD_CAMEL_ROUTE.md create mode 100644 VECTORIZATION.md create mode 100644 XPATH_EXAMPLES.md create mode 100644 docker-compose.yml create mode 100644 docs/architecture/architecture.archimate create mode 100644 docs/architecture/architecture.archimate.bak create mode 100644 embedding_service.py create mode 100644 execute-enum-fix.bat create mode 100644 fix-organization-schema.bat create mode 100644 fix-organization-schema.sql create mode 100644 pom.xml create mode 100644 requirements-embedding.txt create mode 100644 reset-stuck-packages.sql create mode 100644 solution-brief-processed.dat create mode 100644 src/main/java/at/procon/ted/TedProcurementProcessorApplication.java create mode 100644 src/main/java/at/procon/ted/camel/MailRoute.java create mode 100644 src/main/java/at/procon/ted/camel/SolutionBriefRoute.java create mode 100644 src/main/java/at/procon/ted/camel/TedDocumentRoute.java create mode 100644 src/main/java/at/procon/ted/camel/TedPackageDownloadCamelRoute.java create mode 100644 src/main/java/at/procon/ted/camel/TedPackageDownloadRoute.java create mode 100644 src/main/java/at/procon/ted/camel/VectorizationRoute.java create mode 100644 src/main/java/at/procon/ted/config/AsyncConfig.java create mode 100644 src/main/java/at/procon/ted/config/CamelConfig.java create mode 100644 src/main/java/at/procon/ted/config/OpenApiConfig.java create mode 100644 src/main/java/at/procon/ted/config/TedProcessorProperties.java create mode 100644 src/main/java/at/procon/ted/controller/AdminController.java create mode 100644 src/main/java/at/procon/ted/controller/DocumentController.java create mode 100644 src/main/java/at/procon/ted/controller/SimilaritySearchController.java create mode 100644 src/main/java/at/procon/ted/event/DocumentSavedEvent.java create mode 100644 src/main/java/at/procon/ted/event/VectorizationEventListener.java create mode 100644 src/main/java/at/procon/ted/model/dto/DocumentDtos.java create mode 100644 src/main/java/at/procon/ted/model/entity/ContractNature.java create mode 100644 src/main/java/at/procon/ted/model/entity/NoticeType.java create mode 100644 src/main/java/at/procon/ted/model/entity/Organization.java create mode 100644 src/main/java/at/procon/ted/model/entity/ProcedureType.java create mode 100644 src/main/java/at/procon/ted/model/entity/ProcessedAttachment.java create mode 100644 src/main/java/at/procon/ted/model/entity/ProcessingLog.java create mode 100644 src/main/java/at/procon/ted/model/entity/ProcurementDocument.java create mode 100644 src/main/java/at/procon/ted/model/entity/ProcurementLot.java create mode 100644 src/main/java/at/procon/ted/model/entity/TedDailyPackage.java create mode 100644 src/main/java/at/procon/ted/model/entity/VectorizationStatus.java create mode 100644 src/main/java/at/procon/ted/repository/ProcessedAttachmentRepository.java create mode 100644 src/main/java/at/procon/ted/repository/ProcessingLogRepository.java create mode 100644 src/main/java/at/procon/ted/repository/ProcurementDocumentRepository.java create mode 100644 src/main/java/at/procon/ted/repository/TedDailyPackageRepository.java create mode 100644 src/main/java/at/procon/ted/service/BatchDocumentProcessingService.java create mode 100644 src/main/java/at/procon/ted/service/DataCleanupService.java create mode 100644 src/main/java/at/procon/ted/service/DocumentProcessingService.java create mode 100644 src/main/java/at/procon/ted/service/ExcelExportService.java create mode 100644 src/main/java/at/procon/ted/service/ProcessingLogService.java create mode 100644 src/main/java/at/procon/ted/service/SearchService.java create mode 100644 src/main/java/at/procon/ted/service/SimilaritySearchService.java create mode 100644 src/main/java/at/procon/ted/service/TedPackageDownloadService.java create mode 100644 src/main/java/at/procon/ted/service/VectorizationProcessorService.java create mode 100644 src/main/java/at/procon/ted/service/VectorizationService.java create mode 100644 src/main/java/at/procon/ted/service/XmlParserService.java create mode 100644 src/main/java/at/procon/ted/service/attachment/AttachmentExtractor.java create mode 100644 src/main/java/at/procon/ted/service/attachment/AttachmentProcessingService.java create mode 100644 src/main/java/at/procon/ted/service/attachment/PdfExtractionService.java create mode 100644 src/main/java/at/procon/ted/service/attachment/ZipExtractionService.java create mode 100644 src/main/java/at/procon/ted/startup/OrganizationSchemaFixRunner.java create mode 100644 src/main/java/at/procon/ted/startup/VectorizationStartupRunner.java create mode 100644 src/main/java/at/procon/ted/util/HashUtils.java create mode 100644 src/main/java/at/procon/ted/util/InspectDatabase.java create mode 100644 src/main/resources/application - Kopie.yml create mode 100644 src/main/resources/application.yml create mode 100644 src/main/resources/banner.txt create mode 100644 src/main/resources/db/migration/V1__initial_schema.sql create mode 100644 src/main/resources/db/migration/V2__extend_organization_varchar_fields.sql create mode 100644 src/main/resources/db/migration/V3__add_processed_attachment_table.sql create mode 100644 src/test/java/at/procon/ted/service/XmlParserServiceTest.java create mode 100644 start.bat create mode 100644 start.sh create mode 100644 ted-procurement-processor.zip diff --git a/DAILY_PACKAGE_DOWNLOAD.md b/DAILY_PACKAGE_DOWNLOAD.md new file mode 100644 index 0000000..fdcd246 --- /dev/null +++ b/DAILY_PACKAGE_DOWNLOAD.md @@ -0,0 +1,109 @@ +# TED Daily Package Download - Implementierung + +## Übersicht + +Das System lädt automatisch TED Daily Packages herunter und verarbeitet sie. + +## Komponenten + +### 1. Entity: TedDailyPackage ✅ +- Tracking von Downloads +- Status-Management +- Idempotenz durch Hash + +### 2. Repository: TedDailyPackageRepository ✅ +- Package-Verwaltung +- Status-Queries +- Latest-Package-Ermittlung + +### 3. Configuration: DownloadProperties ✅ +- Download-Einstellungen +- URL-Konfiguration +- Rate Limiting + +### 4. Service: TedPackageDownloadService (in Arbeit) +- Package-Download +- tar.gz Extraktion +- Fortschritts-Tracking + +### 5. Camel Route: TedPackageDownloadRoute (ausstehend) +- Scheduled Downloads +- Error Handling +- Integration mit bestehender XML-Verarbeitung + +## Workflow + +1. **Initialization** + - Letztes Package aus DB ermitteln + - Start-Punkt berechnen (aktuelles Jahr oder letztes Package +1) + +2. **Download-Loop** + - Current Year: Start bei letztem +1, bis 404 (max 4x) + - Previous Years: Rückwärts downloaden, langsam + +3. **Package Processing** + - Download tar.gz + - Hash berechnen (SHA-256) + - Prüfung gegen DB (Idempotenz) + - Extraktion der XML-Dateien + - Weiterleitung an XML-Verarbeitungsroute + +4. **Status Tracking** + - PENDING → DOWNLOADING → DOWNLOADED → PROCESSING → COMPLETED + - Fehlerbehandlung: FAILED, NOT_FOUND + +## Konfiguration (application.yml) + +```yaml +ted: + download: + enabled: true + base-url: https://ted.europa.eu/packages/daily/ + download-directory: D:/ted.europe/downloads + extract-directory: D:/ted.europe/extracted + start-year: 2024 + max-consecutive-404: 4 + poll-interval: 3600000 # 1 Stunde + download-timeout: 300000 # 5 Minuten + max-concurrent-downloads: 2 + delay-between-downloads: 5000 # 5 Sekunden + delete-after-extraction: true + prioritize-current-year: true +``` + +## Database Migration + +```sql +CREATE TABLE TED.ted_daily_package ( + id UUID PRIMARY KEY, + package_identifier VARCHAR(20) NOT NULL UNIQUE, + year INTEGER NOT NULL, + serial_number INTEGER NOT NULL, + download_url VARCHAR(500) NOT NULL, + file_hash VARCHAR(64), + xml_file_count INTEGER, + processed_count INTEGER DEFAULT 0, + failed_count INTEGER DEFAULT 0, + download_status VARCHAR(30) NOT NULL DEFAULT 'PENDING', + error_message TEXT, + downloaded_at TIMESTAMP WITH TIME ZONE, + processed_at TIMESTAMP WITH TIME ZONE, + download_duration_ms BIGINT, + processing_duration_ms BIGINT, + created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP, + UNIQUE(year, serial_number) +); + +CREATE INDEX idx_package_identifier ON TED.ted_daily_package(package_identifier); +CREATE INDEX idx_package_year_serial ON TED.ted_daily_package(year, serial_number); +CREATE INDEX idx_package_status ON TED.ted_daily_package(download_status); +CREATE INDEX idx_package_downloaded_at ON TED.ted_daily_package(downloaded_at); +``` + +## Nächste Schritte + +1. Package Download Service fertigstellen +2. Camel Route erstellen +3. Database Migration ausführen +4. Testing & Integration diff --git a/Dockerfile.embedding b/Dockerfile.embedding new file mode 100644 index 0000000..63d9364 --- /dev/null +++ b/Dockerfile.embedding @@ -0,0 +1,39 @@ +# Python Embedding Service Dockerfile +# Author: Martin.Schweitzer@procon.co.at and claude.ai +# +# Provides HTTP API for generating text embeddings using sentence-transformers +# Model: intfloat/multilingual-e5-large (1024 dimensions) + +FROM python:3.11-slim + +WORKDIR /app + +# Install system dependencies +RUN apt-get update && apt-get install -y --no-install-recommends \ + curl \ + && rm -rf /var/lib/apt/lists/* + +# Install Python dependencies +COPY requirements-embedding.txt . +RUN pip install --no-cache-dir -r requirements-embedding.txt + +# Copy application code +COPY embedding_service.py . + +# Pre-download model (optional - reduces startup time) +# RUN python -c "from sentence_transformers import SentenceTransformer; SentenceTransformer('intfloat/multilingual-e5-large')" + +# Environment variables +ENV MODEL_NAME=intfloat/multilingual-e5-large +ENV MAX_LENGTH=512 +ENV HOST=0.0.0.0 +ENV PORT=8001 + +EXPOSE 8001 + +# Health check +HEALTHCHECK --interval=30s --timeout=10s --start-period=60s --retries=3 \ + CMD curl -f http://localhost:8001/health || exit 1 + +# Run the service +CMD ["python", "embedding_service.py"] diff --git a/EXECUTE_ENUM_FIX.md b/EXECUTE_ENUM_FIX.md new file mode 100644 index 0000000..205f957 --- /dev/null +++ b/EXECUTE_ENUM_FIX.md @@ -0,0 +1,67 @@ +# ENUM Type Fix - Execution Instructions + +## Problem +The database has PostgreSQL ENUM types, but Hibernate is configured to use VARCHAR with CHECK constraints. This causes the error: +``` +ERROR: column "contract_nature" is of type ted.contract_nature but expression is of type character varying +``` + +## Solution +Execute the `fix-enum-types-comprehensive.sql` script on the remote database. + +## Execution Methods + +### Option 1: Using psql (Recommended) + +```bash +psql -h 94.130.218.54 -p 5432 -U postgres -d Sales -f fix-enum-types-comprehensive.sql +``` + +When prompted, enter password: `PDmXRx0Rbk9OFOn9qO5Gm/mPCfqW8zwbZ+/YIU1lySc=` + +### Option 2: Using psql with inline password (Windows PowerShell) + +```powershell +$env:PGPASSWORD="PDmXRx0Rbk9OFOn9qO5Gm/mPCfqW8zwbZ+/YIU1lySc=" +psql -h 94.130.218.54 -p 5432 -U postgres -d Sales -f fix-enum-types-comprehensive.sql +``` + +### Option 3: Copy-paste into database client + +If you're using DBeaver, pgAdmin, or another GUI tool: + +1. Connect to: + - Host: `94.130.218.54` + - Port: `5432` + - Database: `Sales` + - Username: `postgres` + - Password: `PDmXRx0Rbk9OFOn9qO5Gm/mPCfqW8zwbZ+/YIU1lySc=` + +2. Open `fix-enum-types-comprehensive.sql` in the query editor + +3. Execute the entire script + +## What the script does + +1. ✅ Converts ENUM columns to VARCHAR(50) while preserving existing data +2. ✅ Drops the old ENUM types +3. ✅ Adds CHECK constraints for data validation +4. ✅ Updates the search function to use VARCHAR parameters +5. ✅ Shows verification query at the end + +## Verification + +After execution, you should see: + +``` +status | column_name | data_type | character_maximum_length +---------------------------------------------|-----------------------|-------------------|-------------------------- +ENUM types successfully converted to VARCHAR!| contract_nature | character varying | 50 +ENUM types successfully converted to VARCHAR!| notice_type | character varying | 50 +ENUM types successfully converted to VARCHAR!| procedure_type | character varying | 50 +ENUM types successfully converted to VARCHAR!| vectorization_status | character varying | 50 +``` + +## After execution + +Restart your Spring Boot application. The error should be resolved and the application will be able to insert data into the database. diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..1489db2 --- /dev/null +++ b/LICENSE @@ -0,0 +1,152 @@ +European Union Public Licence +V. 1.2 + +EUPL © the European Union 2007, 2016 + +This European Union Public Licence (the 'EUPL') applies to the Work (as defined below) which is provided under the terms of this Licence. Any use of the Work, other than as authorised under this Licence is prohibited (to the extent such use is covered by a right of the copyright holder of the Work). + +The Work is provided under the terms of this Licence when the Licensor (as defined below) has placed the following notice immediately following the copyright notice for the Work: + +Licensed under the EUPL + +or has expressed by any other means his willingness to license under the EUPL. + +1. Definitions + +In this Licence, the following terms have the following meaning: + +— 'The Licence': this Licence. +— 'The Original Work': the work or software distributed or communicated by the Licensor under this Licence, available as Source Code and also as Executable Code as the case may be. +— 'Derivative Works': the works or software that could be created by the Licensee, based upon the Original Work or modifications thereof. This Licence does not define the extent of modification or dependence on the Original Work required in order to classify a work as a Derivative Work; this extent is determined by copyright law applicable in the country mentioned in Article 15. +— 'The Work': the Original Work or its Derivative Works. +— 'The Source Code': the human-readable form of the Work which is the most convenient for people to study and modify. +— 'The Executable Code': any code which has generally been compiled and which is meant to be interpreted by a computer as a program. +— 'The Licensor': the natural or legal person that distributes or communicates the Work under the Licence. +— 'Contributor(s)': any natural or legal person who modifies the Work under the Licence, or otherwise contributes to the creation of a Derivative Work. +— 'The Licensee' or 'You': any natural or legal person who makes any usage of the Work under the terms of the Licence. +— 'Distribution' or 'Communication': any act of selling, giving, lending, renting, distributing, communicating, transmitting, or otherwise making available, online or offline, copies of the Work or providing access to its essential functionalities at the disposal of any other natural or legal person. + +2. Scope of the rights granted by the Licence + +The Licensor hereby grants You a worldwide, royalty-free, non-exclusive, sublicensable licence to do the following, for the duration of copyright vested in the Original Work: + +— use the Work in any circumstance and for all usage, +— reproduce the Work, +— modify the Work, and make Derivative Works based upon the Work, +— communicate to the public, including the right to make available or display the Work or copies thereof to the public and perform publicly, as the case may be, the Work, +— distribute the Work or copies thereof, +— lend and rent the Work or copies thereof, +— sublicense rights in the Work or copies thereof. + +Those rights can be exercised on any media, supports and formats, whether now known or later invented, as far as the applicable law permits so. + +In the countries where moral rights apply, the Licensor waives his right to exercise his moral right to the extent allowed by law in order to make effective the licence of the economic rights here above listed. + +The Licensor grants to the Licensee royalty-free, non-exclusive usage rights to any patents held by the Licensor, to the extent necessary to make use of the rights granted on the Work under this Licence. + +3. Communication of the Source Code + +The Licensor may provide the Work either in its Source Code form, or as Executable Code. If the Work is provided as Executable Code, the Licensor provides in addition a machine-readable copy of the Source Code of the Work along with each copy of the Work that the Licensor distributes or indicates, in a notice following the copyright notice attached to the Work, a repository where the Source Code is easily and freely accessible for as long as the Licensor continues to distribute or communicate the Work. + +4. Limitations on copyright + +Nothing in this Licence is intended to deprive the Licensee of the benefits from any exception or limitation to the exclusive rights of the rights owners in the Work, of the exhaustion of those rights or of other applicable limitations thereto. + +5. Obligations of the Licensee + +The grant of the rights mentioned above is subject to some restrictions and obligations imposed on the Licensee. Those obligations are the following: + +Attribution right: The Licensee shall keep intact all copyright, patent or trademarks notices and all notices that refer to the Licence and to the disclaimer of warranties. The Licensee must include a copy of such notices and a copy of the Licence with every copy of the Work he/she distributes or communicates. The Licensee must cause any Derivative Work to carry prominent notices stating that the Work has been modified and the date of modification. + +Copyleft clause: If the Licensee distributes or communicates copies of the Original Works or Derivative Works, this Distribution or Communication will be done under the terms of this Licence or of a later version of this Licence unless the Original Work is expressly distributed only under this version of the Licence — for example by communicating 'EUPL v. 1.2 only'. The Licensee (becoming Licensor) cannot offer or impose any additional terms or conditions on the Work or Derivative Work that alter or restrict the terms of the Licence. + +Compatibility clause: If the Licensee Distributes or Communicates Derivative Works or copies thereof based upon both the Work and another work licensed under a Compatible Licence, this Distribution or Communication can be done under the terms of this Compatible Licence. For the sake of this clause, 'Compatible Licence' refers to the licences listed in the appendix attached to this Licence. Should the Licensee's obligations under the Compatible Licence conflict with his/her obligations under this Licence, the obligations of the Compatible Licence shall prevail. + +Provision of Source Code: When distributing or communicating copies of the Work, the Licensee will provide a machine-readable copy of the Source Code or indicate a repository where this Source will be easily and freely available for as long as the Licensee continues to distribute or communicate the Work. + +Legal Protection: This Licence does not grant permission to use the trade names, trademarks, service marks, or names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the copyright notice. + +6. Chain of Authorship + +The original Licensor warrants that the copyright in the Original Work granted hereunder is owned by him/her or licensed to him/her and that he/she has the power and authority to grant the Licence. + +Each Contributor warrants that the copyright in the modifications he/she brings to the Work are owned by him/her or licensed to him/her and that he/she has the power and authority to grant the Licence. + +Each time You accept the Licence, the original Licensor and subsequent Contributors grant You a licence to their contributions to the Work, under the terms of this Licence. + +7. Disclaimer of Warranty + +The Work is a work in progress, which is continuously improved by numerous Contributors. It is not a finished work and may therefore contain defects or 'bugs' inherent to this type of development. + +For the above reason, the Work is provided under the Licence on an 'as is' basis and without warranties of any kind concerning the Work, including without limitation merchantability, fitness for a particular purpose, absence of defects or errors, accuracy, non-infringement of intellectual property rights other than copyright as stated in Article 6 of this Licence. + +This disclaimer of warranty is an essential part of the Licence and a condition for the grant of any rights to the Work. + +8. Disclaimer of Liability + +Except in the cases of wilful misconduct or damages directly caused to natural persons, the Licensor will in no event be liable for any direct or indirect, material or moral, damages of any kind, arising out of the Licence or of the use of the Work, including without limitation, damages for loss of goodwill, work stoppage, computer failure or malfunction, loss of data or any commercial damage, even if the Licensor has been advised of the possibility of such damage. However, the Licensor will be liable under statutory product liability laws as far such laws apply to the Work. + +9. Additional agreements + +While distributing the Work, You may choose to conclude an additional agreement, defining obligations or services consistent with this Licence. However, if accepting obligations, You may act only on your own behalf and on your sole responsibility, not on behalf of the original Licensor or any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against such Contributor by the fact You have accepted any warranty or additional liability. + +10. Acceptance of the Licence + +The provisions of this Licence can be accepted by clicking on an icon 'I agree' placed under the bottom of a window displaying the text of this Licence or by affirming consent in any other similar way, in accordance with the rules of applicable law. Clicking on that icon indicates your clear and irrevocable acceptance of this Licence and all of its terms and conditions. + +Similarly, you irrevocably accept this Licence and all of its terms and conditions by exercising any rights granted to You by Article 2 of this Licence, such as the use of the Work, the creation by You of a Derivative Work or the Distribution or Communication by You of the Work or copies thereof. + +11. Information to the public + +In case of any Distribution or Communication of the Work by means of electronic communication by You (for example, by offering to download the Work from a remote location) the distribution channel or media (for example, a website) must at least provide to the public the information requested by the applicable law regarding the Licensor, the Licence and the way it may be accessible, concluded, stored and reproduced by the Licensee. + +12. Termination of the Licence + +The Licence and the rights granted hereunder will terminate automatically upon any breach by the Licensee of the terms of the Licence. + +Such a termination will not terminate the licences of any person who has received the Work from the Licensee under the Licence, provided such persons remain in full compliance with the Licence. + +13. Miscellaneous + +Without prejudice of Article 9 above, the Licence represents the complete agreement between the Parties as to the Work. + +If any provision of the Licence is invalid or unenforceable under applicable law, this will not affect the validity or enforceability of the Licence as a whole. Such provision will be construed or reformed so as necessary to make it valid and enforceable. + +The European Commission may publish other linguistic versions or new versions of this Licence or updated versions of the Appendix, so far this is required and reasonable, without reducing the scope of the rights granted by the Licence. New versions of the Licence will be published with a unique version number. + +All linguistic versions of this Licence, approved by the European Commission, have identical value. Parties can take advantage of the linguistic version of their choice. + +14. Jurisdiction + +Without prejudice to specific agreement between parties, + +— any litigation resulting from the interpretation of this License, arising between the European Union institutions, bodies, offices or agencies, as a Licensor, and any Licensee, will be subject to the jurisdiction of the Court of Justice of the European Union, as laid down in article 272 of the Treaty on the Functioning of the European Union, + +— any litigation arising between other parties and resulting from the interpretation of this License, will be subject to the exclusive jurisdiction of the competent court where the Licensor resides or conducts its primary business. + +15. Applicable Law + +Without prejudice to specific agreement between parties, + +— this Licence shall be governed by the law of the European Union Member State where the Licensor has his seat, resides or has his registered office, + +— this licence shall be governed by Belgian law if the Licensor has no seat, residence or registered office inside a European Union Member State. + +Appendix + +'Compatible Licences' according to Article 5 EUPL are: + +— GNU General Public License (GPL) v. 2, v. 3 +— GNU Affero General Public License (AGPL) v. 3 +— Open Software License (OSL) v. 2.1, v. 3.0 +— Eclipse Public License (EPL) v. 1.0 +— CeCILL v. 2.0, v. 2.1 +— Mozilla Public Licence (MPL) v. 2 +— GNU Lesser General Public Licence (LGPL) v. 2.1, v. 3 +— Creative Commons Attribution-ShareAlike v. 3.0 Unported (CC BY-SA 3.0) for works other than software +— European Union Public Licence (EUPL) v. 1.1, v. 1.2 +— Québec Free and Open-Source Licence — Reciprocity (LiLiQ-R) or Strong Reciprocity (LiLiQ-R+). + +The European Commission may update this Appendix to later versions of the above licences without producing a new version of the EUPL, as long as they provide the rights granted in Article 2 of this Licence and protect the covered Source Code from exclusive appropriation. + +All other changes or additions to this Appendix require the production of a new EUPL version. diff --git a/MEMORY-OPTIMIZATION.md b/MEMORY-OPTIMIZATION.md new file mode 100644 index 0000000..ff08394 --- /dev/null +++ b/MEMORY-OPTIMIZATION.md @@ -0,0 +1,139 @@ +# Memory Optimization Changes + +## Problem +Persistent OutOfMemoryError crashes after ~30 minutes of operation. + +## Root Causes Identified +1. **Parallel Processing** - Too many concurrent threads processing XML files +2. **Vectorization** - Heavy memory consumption from embedding service calls +3. **Connection Leaks** - HikariCP pool too large (20 connections) +4. **Duplicate File Processing** - File Consumer route was disabled but still causing issues + +## Changes Made (2026-01-07) + +### 1. Vectorization DISABLED +**File**: `application.yml` +```yaml +vectorization: + enabled: false # Was: true +``` + +**Reason**: Vectorization can be re-enabled later after stability is proven + +### 2. Reduced Database Connection Pool +**File**: `application.yml` +```yaml +hikari: + maximum-pool-size: 5 # Was: 20 + minimum-idle: 2 # Was: 5 + idle-timeout: 300000 # Was: 600000 + max-lifetime: 900000 # Was: 1800000 + leak-detection-threshold: 60000 # NEW +``` + +### 3. Sequential Processing (No Parallelism) +**File**: `TedPackageDownloadCamelRoute.java` +- **Parallel Processing DISABLED** in XML file splitter +- Thread pool reduced to 1 thread (was: 3) +- Only 1 package processed at a time (was: 3) + +```java +.split(header("xmlFiles")) + // .parallelProcessing() // DISABLED + .stopOnException(false) +``` + +### 4. File Consumer Already Disabled +**File**: `TedDocumentRoute.java` +- File consumer route commented out to prevent duplicate processing +- Only Package Download Route processes files + +### 5. Start Script with 8GB Heap +**File**: `start.bat` +```batch +java -Xms4g -Xmx8g -XX:+UseG1GC -XX:MaxGCPauseMillis=200 -jar target\ted-procurement-processor-1.0.0-SNAPSHOT.jar +``` + +## Performance Impact + +### Before +- 3 packages in parallel +- 3 XML files in parallel per package +- Vectorization running +- ~150 concurrent operations +- **Crashes after 30 minutes** + +### After +- 1 package at a time +- Sequential XML file processing +- No vectorization +- ~10-20 concurrent operations +- **Should run stable indefinitely** + +## How to Start + +1. **Reset stuck packages** (if any): + ```bash + psql -h 94.130.218.54 -p 32333 -U postgres -d RELM -f reset-stuck-packages.sql + ``` + +2. **Start application**: + ```bash + start.bat + ``` + +3. **Monitor memory**: + - Check logs for OutOfMemoryError + - Monitor with: `jconsole` or `jvisualvm` + +## Re-enabling Features Later + +### Step 1: Test with current settings +Run for 24-48 hours to confirm stability + +### Step 2: Gradually increase parallelism +```java +// In TedPackageDownloadCamelRoute.java +.split(header("xmlFiles")) + .parallelProcessing() + .executorService(executorService()) // Set to 2-3 threads +``` + +### Step 3: Re-enable vectorization +```yaml +# In application.yml +vectorization: + enabled: true +``` + +### Step 4: Increase connection pool (if needed) +```yaml +hikari: + maximum-pool-size: 10 # Increase gradually +``` + +## Monitoring Commands + +### Check running packages +```sql +SELECT package_identifier, download_status, updated_at +FROM ted.ted_daily_package +WHERE download_status IN ('DOWNLOADING', 'PROCESSING') +ORDER BY updated_at DESC; +``` + +### Check memory usage +```bash +jcmd GC.heap_info +``` + +### Check thread count +```bash +jcmd Thread.print | grep "ted-" | wc -l +``` + +## Notes +- **Processing is slower** but stable +- Approx. 50-100 documents/minute (sequential) +- Can process ~100,000 documents/day +- Vectorization can be run as separate batch job later diff --git a/README.md b/README.md new file mode 100644 index 0000000..3c2dc1f --- /dev/null +++ b/README.md @@ -0,0 +1,402 @@ +# TED Procurement Document Processor + +**AI-Powered Semantic Search Demonstrator for EU Public Procurement** + +A production-ready Spring Boot application showcasing advanced AI semantic search capabilities for processing and searching EU eForms public procurement notices from TED (Tenders Electronic Daily). + +**Author:** Martin.Schweitzer@procon.co.at and claude.ai + +--- + +## 🎯 Demonstrator Highlights + +This application demonstrates the integration of cutting-edge technologies for intelligent document processing: + +### 🧠 **AI Semantic Search** +- **Natural Language Queries**: Search 100,000+ procurement documents using plain language + - Example: *"medical equipment for hospitals in Germany"* + - Example: *"IT infrastructure projects in Austria"* +- **Multilingual Support**: 100+ languages supported via `intfloat/multilingual-e5-large` model +- **1024-Dimensional Embeddings**: High-precision vector representations for accurate similarity matching +- **Hybrid Search**: Combine semantic search with traditional filters (country, CPV codes, dates) + +### 🗄️ **PostgreSQL Native XML** +- **Native XML Data Type**: Store complete eForms XML documents without serialization overhead +- **XPath Queries**: Direct XML querying within PostgreSQL for complex data extraction +- **Dual Storage Strategy**: + - Original XML preserved for audit trail and reprocessing + - Extracted metadata in structured columns for fast filtering + - Best of both worlds: flexibility + performance + +### 🚀 **Production-Grade Features** +- **Fully Automated Pipeline**: Downloads and processes 30,000+ documents daily from ted.europa.eu +- **Apache Camel Integration**: Enterprise Integration Patterns (Timer, Splitter, SEDA, Dead Letter Channel) +- **Idempotent Processing**: SHA-256 hashing prevents duplicate imports +- **Async Vectorization**: Non-blocking background processing with 4 concurrent workers +- **pgvector Extension**: IVFFlat indexing for fast cosine similarity search at scale +- **eForms SDK 1.13**: Full schema validation for EU standard compliance + +--- + +## Key Technologies + +| Technology | Purpose | Benefit | +|------------|---------|---------| +| **PostgreSQL 16+** | Database with native XML | Query XML with XPath while maintaining structure | +| **pgvector** | Vector similarity search | Million-scale semantic search with cosine similarity | +| **Apache Camel** | Integration framework | Enterprise patterns for robust data pipelines | +| **Spring Boot 3.x** | Application framework | Modern Java with dependency injection | +| **intfloat/e5-large** | Embedding model | State-of-the-art multilingual semantic understanding | +| **eForms SDK** | EU standard | Compliance with official procurement schemas | + +## Architecture + +``` +┌─────────────────────────────────────────────────────────────────────┐ +│ TED Procurement Processor │ +├─────────────────────────────────────────────────────────────────────┤ +│ │ +│ ┌──────────────┐ ┌─────────────────┐ ┌───────────────────┐ │ +│ │ File System │───▶│ Apache Camel │───▶│ Document │ │ +│ │ (*.xml) │ │ Route │ │ Processing │ │ +│ └──────────────┘ └─────────────────┘ │ Service │ │ +│ └─────────┬─────────┘ │ +│ │ │ +│ ▼ │ +│ ┌──────────────┐ ┌─────────────────┐ ┌───────────────────┐ │ +│ │ REST API │◀───│ Search │◀───│ PostgreSQL │ │ +│ │ Controller │ │ Service │ │ + pgvector │ │ +│ └──────────────┘ └─────────────────┘ └───────────────────┘ │ +│ ▲ │ +│ │ │ +│ ┌──────────────────────────────────────────────────────┐ │ +│ │ Vectorization Service (Async) │ │ +│ │ intfloat/multilingual-e5-large (1024d) │ │ +│ └──────────────────────────────────────────────────────┘ │ +│ │ +└─────────────────────────────────────────────────────────────────────┘ +``` + +## Prerequisites + +- Java 21+ +- Maven 3.9+ +- PostgreSQL 16+ with pgvector extension +- Python 3.11+ (for embedding service) +- Docker & Docker Compose (optional, for easy setup) + +## 🚀 Automated Pipeline + +**See [TED_AUTOMATED_PIPELINE.md](TED_AUTOMATED_PIPELINE.md) for complete documentation on the automated download, processing, and vectorization pipeline.** + +The application automatically: +1. Downloads TED Daily Packages every hour from ted.europa.eu +2. Extracts and processes XML files +3. Stores in PostgreSQL with native XML support +4. Generates 1024-dimensional embeddings for semantic search +5. Enables REST API queries with natural language + +## Quick Start + +### 1. Start PostgreSQL with pgvector + +Using Docker: +```bash +docker-compose up -d postgres +``` + +Or manually install PostgreSQL with pgvector extension. + +### 2. Configure Application + +Edit `src/main/resources/application.yml`: + +```yaml +ted: + input: + directory: D:/ted.europe/2025-11.tar/2025-11/11 # Your TED XML directory + pattern: "**/*.xml" +``` + +### 3. Build and Run + +```bash +# Build +mvn clean package -DskipTests + +# Run +java -jar target/ted-procurement-processor-1.0.0-SNAPSHOT.jar +``` + +### 4. Start Embedding Service (Optional) + +For semantic search capabilities: + +```bash +# Using Docker +docker-compose --profile with-embedding up -d embedding-service + +# Or manually +pip install -r requirements-embedding.txt +python embedding_service.py +``` + +## Database Schema + +### Main Tables + +| Table | Description | +|-------|-------------| +| `procurement_document` | Main table with extracted metadata and original XML | +| `procurement_lot` | Individual lots within procurement notices | +| `organization` | Organizations mentioned in notices (buyers, review bodies) | +| `processing_log` | Audit trail for document processing events | + +### Key Columns in `procurement_document` + +| Column | Type | Description | +|--------|------|-------------| +| `id` | UUID | Primary key | +| `document_hash` | VARCHAR(64) | SHA-256 hash for idempotency | +| `publication_id` | VARCHAR(50) | TED publication ID (e.g., "00786665-2025") | +| `notice_url` | VARCHAR(255) | TED website URL (e.g., "https://ted.europa.eu/en/notice/-/detail/786665-2025") | +| `xml_document` | XML | Original document | +| `text_content` | TEXT | Extracted text for vectorization | +| `content_vector` | vector(1024) | Embedding for semantic search | +| `buyer_country_code` | VARCHAR(10) | ISO 3166-1 alpha-3 country code | +| `cpv_codes` | VARCHAR(100)[] | CPV classification codes | +| `nuts_codes` | VARCHAR(20)[] | NUTS region codes | + +## REST API + +### Search Endpoints + +#### GET /api/v1/documents/search + +Search with structured filters: + +```bash +# Search by country +curl "http://localhost:8080/api/v1/documents/search?countryCode=POL" + +# Search by CPV code prefix (medical supplies) +curl "http://localhost:8080/api/v1/documents/search?cpvPrefix=33" + +# Search by date range +curl "http://localhost:8080/api/v1/documents/search?publicationDateFrom=2025-01-01&publicationDateTo=2025-12-31" + +# Combined filters +curl "http://localhost:8080/api/v1/documents/search?countryCode=DEU&contractNature=SERVICES¬iceType=CONTRACT_NOTICE" +``` + +#### GET /api/v1/documents/semantic-search + +Natural language semantic search: + +```bash +# Search for medical equipment tenders +curl "http://localhost:8080/api/v1/documents/semantic-search?query=medical+equipment+hospital+supplies" + +# Search with similarity threshold +curl "http://localhost:8080/api/v1/documents/semantic-search?query=construction+works+road+infrastructure&threshold=0.75" +``` + +#### POST /api/v1/documents/search + +Complex search with JSON body: + +```bash +curl -X POST "http://localhost:8080/api/v1/documents/search" \ + -H "Content-Type: application/json" \ + -d '{ + "countryCodes": ["DEU", "AUT", "CHE"], + "contractNature": "SERVICES", + "cpvPrefix": "72", + "semanticQuery": "software development IT services", + "similarityThreshold": 0.7, + "page": 0, + "size": 20 + }' +``` + +### Document Retrieval + +```bash +# Get by UUID +curl "http://localhost:8080/api/v1/documents/{uuid}" + +# Get by publication ID +curl "http://localhost:8080/api/v1/documents/publication/00786665-2025" +``` + +### Metadata Endpoints + +```bash +# List countries +curl "http://localhost:8080/api/v1/documents/metadata/countries" + +# Get statistics +curl "http://localhost:8080/api/v1/documents/statistics" + +# Upcoming deadlines +curl "http://localhost:8080/api/v1/documents/upcoming-deadlines?limit=50" +``` + +### Admin Endpoints + +```bash +# Health check +curl "http://localhost:8080/api/v1/admin/health" + +# Vectorization status +curl "http://localhost:8080/api/v1/admin/vectorization/status" + +# Trigger vectorization for pending documents +curl -X POST "http://localhost:8080/api/v1/admin/vectorization/process-pending?batchSize=100" +``` + +## Configuration + +### Application Properties + +| Property | Default | Description | +|----------|---------|-------------| +| `ted.input.directory` | - | Input directory for XML files | +| `ted.input.pattern` | `**/*.xml` | File pattern (Ant-style) | +| `ted.input.poll-interval` | 5000 | Polling interval in ms | +| `ted.schema.enabled` | true | Enable XSD validation | +| `ted.vectorization.enabled` | true | Enable async vectorization | +| `ted.vectorization.model-name` | `intfloat/multilingual-e5-large` | Embedding model | +| `ted.vectorization.dimensions` | 1024 | Vector dimensions | +| `ted.search.default-page-size` | 20 | Default results per page | +| `ted.search.similarity-threshold` | 0.7 | Default similarity threshold | + +### Environment Variables + +| Variable | Description | +|----------|-------------| +| `DB_USERNAME` | PostgreSQL username | +| `DB_PASSWORD` | PostgreSQL password | +| `TED_INPUT_DIR` | Override input directory | + +## Data Model + +### Notice Types + +- `CONTRACT_NOTICE` - Standard contract notices +- `PRIOR_INFORMATION_NOTICE` - Prior information notices +- `CONTRACT_AWARD_NOTICE` - Contract award notices +- `MODIFICATION_NOTICE` - Contract modifications +- `OTHER` - Other notice types + +### Contract Nature + +- `SUPPLIES` - Goods procurement +- `SERVICES` - Service procurement +- `WORKS` - Construction works +- `MIXED` - Mixed contracts +- `UNKNOWN` - Not specified + +### Procedure Types + +- `OPEN` - Open procedure +- `RESTRICTED` - Restricted procedure +- `COMPETITIVE_DIALOGUE` - Competitive dialogue +- `INNOVATION_PARTNERSHIP` - Innovation partnership +- `NEGOTIATED_WITHOUT_PUBLICATION` - Negotiated without prior publication +- `NEGOTIATED_WITH_PUBLICATION` - Negotiated with prior publication +- `OTHER` - Other procedures + +## Semantic Search + +**See [VECTORIZATION.md](VECTORIZATION.md) for detailed documentation on the vectorization pipeline.** + +The application uses the `intfloat/multilingual-e5-large` model for generating document embeddings: + +- **Dimensions**: 1024 +- **Languages**: Supports 100+ languages +- **Normalization**: Embeddings are L2 normalized for cosine similarity + +### Query Prefixes + +For optimal results with e5 models: +- Documents use `passage: ` prefix +- Queries use `query: ` prefix + +This is handled automatically by the vectorization service. + +## Development + +### Running Tests + +```bash +mvn test +``` + +### Building Docker Image + +```bash +docker build -t ted-procurement-processor . +``` + +### OpenAPI Documentation + +Access Swagger UI at: `http://localhost:8080/api/swagger-ui.html` + +## Performance Considerations + +### Indexes + +The schema includes optimized indexes for: +- Hash lookup (idempotent processing) +- Publication/notice ID lookups +- Date range queries +- Geographic searches (country, NUTS codes) +- CPV code classification +- Vector similarity search (IVFFlat) +- Full-text trigram search + +### Batch Processing + +- Configure `ted.input.max-messages-per-poll` for batch sizes +- Vectorization processes documents in batches of 16 by default +- Use the admin API to trigger bulk vectorization + +## Troubleshooting + +### Common Issues + +**Files not being processed:** +- Check directory path in configuration +- Verify file permissions +- Check Camel route status in logs + +**Duplicate detection not working:** +- Ensure `document_hash` column has unique constraint +- Check if XML content is exactly the same + +**Vectorization failing:** +- Verify embedding service is running +- Check Python dependencies +- Ensure sufficient memory for model + +**Slow searches:** +- Ensure pgvector IVFFlat index is created +- Check if `content_vector` column is populated +- Consider adjusting `lists` parameter in index + +## License + +Licensed under the European Union Public Licence (EUPL) v1.2 + +Copyright (c) 2025 PROCON DATA Gesellschaft m.b.H. + +You may use, copy, modify and distribute this work under the terms of the EUPL. +See the [LICENSE](LICENSE) file for details or visit: https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + +## Acknowledgments + +- [eForms SDK](https://github.com/OP-TED/eForms-SDK) - EU Publications Office +- [pgvector](https://github.com/pgvector/pgvector) - Vector similarity search for PostgreSQL +- [sentence-transformers](https://www.sbert.net/) - Text embeddings +- [Apache Camel](https://camel.apache.org/) - Integration framework diff --git a/Search-TED.ps1 b/Search-TED.ps1 new file mode 100644 index 0000000..2eac9c0 --- /dev/null +++ b/Search-TED.ps1 @@ -0,0 +1,164 @@ +<# +.SYNOPSIS + Semantische Suche in TED Ausschreibungen +.DESCRIPTION + Fuehrt eine semantische Aehnlichkeitssuche gegen die TED Procurement Datenbank durch. +.PARAMETER Query + Der Suchtext fuer die semantische Suche +.PARAMETER TopK + Anzahl der zurueckgegebenen Ergebnisse (Standard: 10, Max: 100) +.PARAMETER Threshold + Minimaler Aehnlichkeitsschwellwert 0.0-1.0 (Standard: 0.5) +.PARAMETER ApiUrl + URL des API-Servers (Standard: http://localhost:8888/api) +.EXAMPLE + .\Search-TED.ps1 -Query "Software Entwicklung Cloud Services" +.EXAMPLE + .\Search-TED.ps1 -Query "IT Beratung Digitalisierung" -TopK 20 -Threshold 0.6 +#> + +param( + [Parameter(Mandatory=$true, Position=0, HelpMessage="Suchtext fuer semantische Suche")] + [string]$Query, + + [Parameter(Mandatory=$false)] + [ValidateRange(1, 100)] + [int]$TopK = 100, + + [Parameter(Mandatory=$false)] + [ValidateRange(0.0, 1.0)] + [double]$Threshold = 0.5, + + [Parameter(Mandatory=$false)] + [string]$ApiUrl = "http://localhost:8888/api" +) + +# Farben fuer Ausgabe +$colors = @{ + Header = "Cyan" + Success = "Green" + Warning = "Yellow" + Error = "Red" + Info = "White" + Highlight = "Magenta" +} + +function Write-Header { + param([string]$Text) + Write-Host "" + Write-Host ("=" * 80) -ForegroundColor $colors.Header + Write-Host $Text -ForegroundColor $colors.Header + Write-Host ("=" * 80) -ForegroundColor $colors.Header +} + +function Write-Result { + param( + [int]$Rank, + [PSObject]$Doc + ) + + $similarity = if ($Doc.similarityPercent) { "$($Doc.similarityPercent)%" } else { "N/A" } + $deadline = "-" + if ($Doc.submissionDeadline) { + try { + # Handle ISO 8601 format (e.g., "2025-04-28T06:00:59Z") + $deadline = ([DateTime]::Parse($Doc.submissionDeadline, [System.Globalization.CultureInfo]::InvariantCulture)).ToString("dd.MM.yyyy") + } catch { + $deadline = $Doc.submissionDeadline.Substring(0, 10) # Fallback: first 10 chars (YYYY-MM-DD) + } + } + + Write-Host "" + Write-Host "[$Rank] " -NoNewline -ForegroundColor $colors.Highlight + Write-Host "$similarity Aehnlichkeit" -ForegroundColor $colors.Success + Write-Host " Titel: " -NoNewline -ForegroundColor $colors.Info + Write-Host $Doc.projectTitle -ForegroundColor $colors.Warning + Write-Host " Auftraggeber: " -NoNewline -ForegroundColor $colors.Info + Write-Host "$($Doc.buyerName) ($($Doc.buyerCountryCode))" -ForegroundColor White + Write-Host " Vertragsart: " -NoNewline -ForegroundColor $colors.Info + Write-Host $Doc.contractNature -ForegroundColor White + Write-Host " Einreichfrist:" -NoNewline -ForegroundColor $colors.Info + Write-Host " $deadline" -ForegroundColor White + Write-Host " TED Link: " -NoNewline -ForegroundColor $colors.Info + Write-Host $Doc.noticeUrl -ForegroundColor Cyan + + if ($Doc.projectDescription) { + $desc = $Doc.projectDescription + if ($desc.Length -gt 200) { + $desc = $desc.Substring(0, 197) + "..." + } + Write-Host " Beschreibung: " -NoNewline -ForegroundColor $colors.Info + Write-Host $desc -ForegroundColor DarkGray + } +} + +# Hauptprogramm +try { + Write-Header "TED Semantische Suche" + Write-Host "Suchtext: " -NoNewline -ForegroundColor $colors.Info + Write-Host $Query -ForegroundColor $colors.Warning + Write-Host "Parameter: TopK=$TopK, Threshold=$Threshold" -ForegroundColor DarkGray + Write-Host "" + + # Request Body erstellen + $body = @{ + text = $Query + topK = $TopK + threshold = $Threshold + } | ConvertTo-Json + + # API aufrufen + Write-Host "Suche laeuft..." -ForegroundColor $colors.Info + $stopwatch = [System.Diagnostics.Stopwatch]::StartNew() + + $response = Invoke-RestMethod -Uri "$ApiUrl/similarity/text" ` + -Method Post ` + -ContentType "application/json; charset=utf-8" ` + -Body $body ` + -ErrorAction Stop + + $stopwatch.Stop() + + # Ergebnisse anzeigen + $resultCount = $response.resultCount + $embeddingTime = $response.embeddingTimeMs + $searchTime = $response.searchTimeMs + $totalTime = $stopwatch.ElapsedMilliseconds + + Write-Header "Suchergebnisse: $resultCount Treffer" + Write-Host "Zeiten: Embedding=${embeddingTime}ms, Suche=${searchTime}ms, Gesamt=${totalTime}ms" -ForegroundColor DarkGray + + if ($resultCount -eq 0) { + Write-Host "" + Write-Host "Keine passenden Ausschreibungen gefunden." -ForegroundColor $colors.Warning + Write-Host "Versuchen Sie:" -ForegroundColor $colors.Info + Write-Host " - Andere Suchbegriffe" -ForegroundColor DarkGray + Write-Host " - Niedrigeren Schwellwert (-Threshold 0.3)" -ForegroundColor DarkGray + } + else { + $rank = 1 + foreach ($doc in $response.results) { + Write-Result -Rank $rank -Doc $doc + $rank++ + } + } + + Write-Host "" + Write-Host ("-" * 80) -ForegroundColor DarkGray + Write-Host "Suche abgeschlossen." -ForegroundColor $colors.Success + +} catch { + Write-Host "" + Write-Host "FEHLER: $_" -ForegroundColor $colors.Error + + if ($_.Exception.Response.StatusCode -eq 503) { + Write-Host "Der Vektorisierungsdienst ist nicht verfuegbar." -ForegroundColor $colors.Warning + Write-Host "Stellen Sie sicher, dass der Embedding-Service auf Port 8001 laeuft." -ForegroundColor $colors.Info + } + elseif ($_.Exception.Message -like "*Unable to connect*") { + Write-Host "Konnte keine Verbindung zum Server herstellen." -ForegroundColor $colors.Warning + Write-Host "Stellen Sie sicher, dass die Anwendung auf $ApiUrl laeuft." -ForegroundColor $colors.Info + } + + exit 1 +} diff --git a/TED_AUTOMATED_PIPELINE.md b/TED_AUTOMATED_PIPELINE.md new file mode 100644 index 0000000..884b9f7 --- /dev/null +++ b/TED_AUTOMATED_PIPELINE.md @@ -0,0 +1,467 @@ +# TED Automatisierte Download & Verarbeitungs-Pipeline + +## Übersicht + +Die komplette automatisierte Pipeline für TED (Tenders Electronic Daily) Ausschreibungen: + +``` +┌────────────────────────────────────────────────────────────────────────┐ +│ TED Automatisierte Pipeline │ +├────────────────────────────────────────────────────────────────────────┤ +│ │ +│ ┌─────────────────┐ │ +│ │ Timer (1h) │ Alle 1 Stunde neue Packages prüfen │ +│ └────────┬────────┘ │ +│ │ │ +│ ▼ │ +│ ┌─────────────────┐ │ +│ │ HTTP Download │ https://ted.europa.eu/packages/daily/ │ +│ │ Package │ Format: YYYY-MM-DD_XXXX.tar.gz │ +│ └────────┬────────┘ │ +│ │ │ +│ ▼ │ +│ ┌─────────────────┐ │ +│ │ Extract │ tar.gz → Tausende von XML Files │ +│ │ tar.gz │ Extract to: D:/ted.europe/extracted │ +│ └────────┬────────┘ │ +│ │ │ +│ ▼ │ +│ ┌─────────────────┐ │ +│ │ XML Splitter │ Parallel Processing (Streaming) │ +│ │ (Parallel) │ Each XML → direct:process-document │ +│ └────────┬────────┘ │ +│ │ │ +│ ▼ │ +│ ┌─────────────────┐ │ +│ │ XML Parser │ XPath Parsing + Metadata Extraction │ +│ │ & Validator │ Schema Validation (eForms SDK 1.13) │ +│ └────────┬────────┘ │ +│ │ │ +│ ▼ │ +│ ┌─────────────────┐ │ +│ │ SHA-256 Hash │ Idempotent Processing │ +│ │ Check │ Skip if already imported │ +│ └────────┬────────┘ │ +│ │ │ +│ ▼ │ +│ ┌─────────────────┐ │ +│ │ Save to DB │ PostgreSQL (ted.procurement_document) │ +│ │ (PostgreSQL) │ + Native XML + Metadata │ +│ └────────┬────────┘ │ +│ │ │ +│ ▼ │ +│ ┌─────────────────┐ │ +│ │ wireTap │ Non-blocking Trigger │ +│ │ Vectorization │ direct:vectorize (async) │ +│ └────────┬────────┘ │ +│ │ │ +│ ▼ │ +│ ┌─────────────────┐ │ +│ │ SEDA Queue │ 4 Concurrent Workers │ +│ │ (Async) │ vectorize-async queue │ +│ └────────┬────────┘ │ +│ │ │ +│ ▼ │ +│ ┌─────────────────┐ │ +│ │ Extract Text │ Title + Description + Lots │ +│ │ Content │ Buyer Info + CPV Codes │ +│ └────────┬────────┘ │ +│ │ │ +│ ▼ │ +│ ┌─────────────────┐ │ +│ │ POST to │ http://localhost:8001/embed │ +│ │ Embedding API │ {"text": "...", "is_query": false} │ +│ └────────┬────────┘ │ +│ │ │ +│ ▼ │ +│ ┌─────────────────┐ │ +│ │ Python Service │ intfloat/multilingual-e5-large │ +│ │ (FastAPI) │ Returns: 1024-dimensional vector │ +│ └────────┬────────┘ │ +│ │ │ +│ ▼ │ +│ ┌─────────────────┐ │ +│ │ Save Vector │ content_vector column (pgvector) │ +│ │ to Database │ Status: COMPLETED │ +│ └─────────────────┘ │ +│ │ +└────────────────────────────────────────────────────────────────────────┘ +``` + +## Konfiguration + +**application.yml:** + +```yaml +ted: + # Input directory (points to extract directory) + input: + directory: D:/ted.europe/extracted + pattern: "**/*.xml" # Recursive scanning + poll-interval: 5000 # Check every 5 seconds + max-messages-per-poll: 100 # Process up to 100 XMLs per poll + + # Automatic download from ted.europa.eu + download: + enabled: true # ✅ ENABLED + base-url: https://ted.europa.eu/packages/daily/ + download-directory: D:/ted.europe/downloads + extract-directory: D:/ted.europe/extracted + start-year: 2024 # Start downloading from 2024 + poll-interval: 3600000 # Check every 1 hour + max-consecutive-404: 4 # Stop after 4 consecutive 404s + delete-after-extraction: true # Clean up tar.gz files + + # Vectorization (automatic after save) + vectorization: + enabled: true # ✅ ENABLED + api-url: http://localhost:8001 + model-name: intfloat/multilingual-e5-large + dimensions: 1024 + batch-size: 16 + max-text-length: 8192 +``` + +## Camel Routes + +### 1. **TedPackageDownloadCamelRoute** (Download & Extract) + +**Route ID:** `ted-package-scheduler` + +**Trigger:** Timer alle 1 Stunde + +**Ablauf:** +1. Bestimmt nächstes Package (Jahr + Serial Number) +2. Prüft ob bereits vorhanden (Idempotent Consumer) +3. HTTP GET von `https://ted.europa.eu/packages/daily/YYYY-MM-DD_XXXX.tar.gz` +4. Speichert in `download-directory` +5. Extrahiert nach `extract-directory` +6. Löscht tar.gz (optional) +7. Splittiert XML Files → `direct:process-document` + +**Enterprise Integration Patterns:** +- ✅ Timer Pattern +- ✅ Idempotent Consumer +- ✅ Content-Based Router +- ✅ Splitter Pattern (Parallel + Streaming) +- ✅ Dead Letter Channel + +### 2. **TedDocumentRoute** (XML Processing) + +**Route ID:** `ted-document-processor` + +**Trigger:** +- File Watcher auf `D:/ted.europe/extracted` +- Direct Call von Download Route + +**Ablauf:** +1. Liest XML File +2. Parst mit XPath (eForms UBL Schema) +3. Extrahiert Metadata +4. Berechnet SHA-256 Hash +5. Prüft Duplikat in DB +6. Speichert in `ted.procurement_document` +7. **wireTap** → `direct:vectorize` (non-blocking!) + +### 3. **VectorizationRoute** (Async Embedding) + +**Route ID:** `vectorization-processor` + +**Trigger:** +- wireTap von TedDocumentRoute +- Timer Scheduler (alle 60s für PENDING) + +**Ablauf:** +1. Load document from DB +2. Extract text_content (Document + Lots) +3. POST to Python Embedding Service +4. Parse 1024-dimensional vector +5. Save to `content_vector` column +6. Update status → `COMPLETED` + +**Queue:** SEDA with 4 concurrent workers + +## Verzeichnisstruktur + +``` +D:/ted.europe/ +├── downloads/ # Temporäre tar.gz Downloads +│ └── 2025-11-30_0001.tar.gz +│ └── 2025-11-30_0002.tar.gz +│ +├── extracted/ # Extrahierte XML Files +│ ├── 2025-11-30/ +│ │ ├── 001/ +│ │ │ ├── 00123456_2025.xml +│ │ │ └── 00123457_2025.xml +│ │ └── 002/ +│ │ └── ... +│ └── .processed/ # Erfolgreich verarbeitete XMLs +│ └── .error/ # Fehlgeschlagene XMLs +``` + +## Datenbank-Tracking + +### ted_daily_package (Download-Tracking) + +| Spalte | Typ | Beschreibung | +|--------|-----|--------------| +| `id` | UUID | Primary Key | +| `year` | INT | Package Jahr (2024, 2025) | +| `serial_number` | INT | Package Nummer (1, 2, 3...) | +| `package_id` | VARCHAR | Format: `2025-11-30_0001` | +| `download_url` | VARCHAR | Full URL | +| `download_status` | VARCHAR | PENDING, DOWNLOADING, COMPLETED, NOT_FOUND, FAILED | +| `downloaded_at` | TIMESTAMP | Download-Zeitpunkt | +| `file_size_bytes` | BIGINT | Größe der tar.gz | +| `xml_file_count` | INT | Anzahl extrahierter XMLs | +| `processed_count` | INT | Anzahl verarbeiteter XMLs | + +### procurement_document (XML-Daten) + +| Spalte | Typ | Beschreibung | +|--------|-----|--------------| +| `id` | UUID | Primary Key | +| `document_hash` | VARCHAR(64) | SHA-256 für Idempotenz | +| `publication_id` | VARCHAR(50) | TED ID (00123456-2025) | +| `notice_url` | VARCHAR(255) | Auto-generated TED URL | +| `xml_document` | XML | Native PostgreSQL XML | +| `text_content` | TEXT | Für Vektorisierung | +| `content_vector` | vector(1024) | pgvector Embedding | +| `vectorization_status` | VARCHAR | PENDING, PROCESSING, COMPLETED, FAILED | + +## Monitoring + +### Camel Routes Status + +```bash +curl http://localhost:8888/api/actuator/camel/routes +``` + +**Wichtige Routes:** +- `ted-package-scheduler` - Download Timer +- `ted-document-processor` - XML Processing +- `vectorization-processor` - Embedding Generation +- `vectorization-scheduler` - PENDING Documents + +### Download Status + +```sql +SELECT + year, + COUNT(*) FILTER (WHERE download_status = 'COMPLETED') as completed, + COUNT(*) FILTER (WHERE download_status = 'NOT_FOUND') as not_found, + COUNT(*) FILTER (WHERE download_status = 'FAILED') as failed, + SUM(xml_file_count) as total_xmls, + SUM(processed_count) as processed_xmls +FROM ted.ted_daily_package +GROUP BY year +ORDER BY year DESC; +``` + +### Vectorization Status + +```sql +SELECT + COUNT(*) FILTER (WHERE vectorization_status = 'COMPLETED') as completed, + COUNT(*) FILTER (WHERE vectorization_status = 'PENDING') as pending, + COUNT(*) FILTER (WHERE vectorization_status = 'FAILED') as failed, + COUNT(*) FILTER (WHERE content_vector IS NOT NULL) as has_vector +FROM ted.procurement_document; +``` + +### Heute verarbeitete Dokumente + +```sql +SELECT + COUNT(*) as today_count, + MIN(created_at) as first, + MAX(created_at) as last +FROM ted.procurement_document +WHERE created_at::date = CURRENT_DATE; +``` + +## Python Embedding Service + +**Start:** +```bash +python embedding_service.py +``` + +**Health Check:** +```bash +curl http://localhost:8001/health +``` + +**Expected Response:** +```json +{ + "status": "healthy", + "model_name": "intfloat/multilingual-e5-large", + "dimensions": 1024, + "max_length": 512 +} +``` + +## Start der Pipeline + +1. **Python Embedding Service starten:** + ```bash + python embedding_service.py + ``` + +2. **Spring Boot Anwendung starten:** + ```bash + mvn spring-boot:run + ``` + +3. **Logs beobachten:** + ``` + INFO: Checking for new TED packages... + INFO: Next package to download: 2025-11-30_0001 + INFO: Downloading from https://ted.europa.eu/packages/daily/... + INFO: Extracting package 2025-11-30_0001... + INFO: Processing 1247 XML files from package 2025-11-30_0001 + INFO: Document processed successfully: 00123456_2025.xml + DEBUG: Queueing document for vectorization: xxx + INFO: Successfully vectorized document: xxx + ``` + +## Durchsatz + +**Geschätzte Performance:** + +| Phase | Geschwindigkeit | Bemerkung | +|-------|----------------|-----------| +| **Download** | 1 Package/Stunde | Timer-basiert | +| **Extract** | ~10 Sekunden | tar.gz → XMLs | +| **XML Processing** | ~100-200 XMLs/min | Abhängig von CPU | +| **Vectorization** | ~60-90 Docs/min | 4 Workers, Python Service | + +**Täglich:** +- ~24 Packages heruntergeladen +- ~30.000-50.000 Dokumente verarbeitet (je nach Package-Größe) +- ~30.000-50.000 Vektoren generiert + +## Fehlerbehandlung + +### Download Fehler + +**404 Not Found:** Package existiert (noch) nicht +- Max 4 consecutive 404s → Switch zu Vorjahr +- Automatische Wiederholung nach 1 Stunde + +**Network Error:** Temporäre Verbindungsprobleme +- 3 Retries mit 10s Delay +- Dead Letter Channel + +### Processing Fehler + +**Duplikate:** SHA-256 Hash bereits vorhanden +- Wird übersprungen (Idempotent Processing) +- Log: "Duplicate document skipped" + +**XML Parsing Error:** Ungültiges XML +- 3 Retries +- Move to `.error` directory +- Status: FAILED in DB + +### Vectorization Fehler + +**Embedding Service nicht erreichbar:** +- 2 Retries mit 2s Delay +- Status: FAILED +- Scheduler versucht erneut nach 60s + +**Invalid Embedding Dimension:** +- Status: FAILED mit Error-Message +- Manuelles Eingreifen erforderlich + +## Troubleshooting + +### Pipeline läuft nicht + +```bash +# Prüfe Camel Routes +curl http://localhost:8888/api/actuator/camel/routes | jq '.routes[] | {id: .id, status: .status}' + +# Prüfe Download Route +tail -f logs/ted-procurement-processor.log | grep "ted-package" + +# Prüfe Vectorization Route +tail -f logs/ted-procurement-processor.log | grep "vectoriz" +``` + +### Keine Downloads + +1. Prüfe `ted.download.enabled = true` +2. Prüfe Internet-Verbindung +3. Prüfe ted.europa.eu erreichbar +4. Prüfe Logs für 404/403 Errors + +### Keine Vektorisierung + +1. Prüfe Embedding Service: `curl http://localhost:8001/health` +2. Prüfe `ted.vectorization.enabled = true` +3. Prüfe PENDING Dokumente in DB +4. Prüfe Logs für HTTP 400/500 Errors + +## Semantic Search + +Nach erfolgreicher Vektorisierung sind Dokumente durchsuchbar: + +```bash +# Semantic Search +curl "http://localhost:8888/api/v1/documents/semantic-search?query=medical+equipment" + +# Combined Search (Semantic + Filters) +curl -X POST "http://localhost:8888/api/v1/documents/search" \ + -H "Content-Type: application/json" \ + -d '{ + "countryCodes": ["DEU", "AUT"], + "semanticQuery": "software development", + "similarityThreshold": 0.7 + }' +``` + +## Performance-Optimierung + +### Vectorization beschleunigen + +```yaml +ted: + vectorization: + thread-pool-size: 8 # Mehr Workers (Standard: 4) +``` + +**Achtung:** Mehr Workers = mehr Last auf Python Service! + +### XML Processing beschleunigen + +```yaml +ted: + input: + max-messages-per-poll: 200 # Mehr Files pro Poll (Standard: 100) +``` + +### Download parallelisieren + +```yaml +ted: + download: + max-concurrent-downloads: 4 # Mehr parallele Downloads (Standard: 2) +``` + +**Achtung:** ted.europa.eu Rate Limiting beachten! + +## Zusammenfassung + +✅ **Komplett automatisierte Pipeline** von Download bis Semantic Search +✅ **Idempotent Processing** - Keine Duplikate +✅ **Asynchrone Vektorisierung** - Non-blocking +✅ **Enterprise Integration Patterns** - Production-ready +✅ **Fehlerbehandlung** - Retries & Dead Letter Channel +✅ **Monitoring** - Actuator + SQL Queries +✅ **Skalierbar** - Concurrent Workers & Parallel Processing + +Die Pipeline läuft vollautomatisch 24/7 und verarbeitet alle neuen TED-Ausschreibungen! 🚀 diff --git a/TED_NOTICE_URL.md b/TED_NOTICE_URL.md new file mode 100644 index 0000000..8b5b576 --- /dev/null +++ b/TED_NOTICE_URL.md @@ -0,0 +1,149 @@ +# TED Notice URL Feature + +## Übersicht + +Jedes Dokument in der Datenbank hat jetzt eine automatisch generierte `notice_url` Spalte, die direkt auf die TED-Webseite verlinkt. + +## Format + +``` +https://ted.europa.eu/en/notice/-/detail/{publication_id ohne führende Nullen} +``` + +**Beispiel:** +- `publication_id`: `00786665-2025` +- `notice_url`: `https://ted.europa.eu/en/notice/-/detail/786665-2025` + +## Automatische Generierung + +Die URL wird automatisch beim Speichern eines Dokuments generiert: + +```java +@PrePersist +@PreUpdate +private void generateNoticeUrl() { + if (publicationId != null && !publicationId.isEmpty()) { + String cleanId = publicationId.replaceFirst("^0+", ""); + this.noticeUrl = "https://ted.europa.eu/en/notice/-/detail/" + cleanId; + } +} +``` + +## Datenbankstruktur + +### Spalte + +```sql +ALTER TABLE ted.procurement_document + ADD COLUMN notice_url VARCHAR(255); + +CREATE INDEX idx_doc_notice_url ON ted.procurement_document(notice_url); +``` + +### Existierende Datensätze + +URLs für existierende Datensätze werden automatisch generiert: + +```sql +UPDATE ted.procurement_document +SET notice_url = 'https://ted.europa.eu/en/notice/-/detail/' || + REGEXP_REPLACE(publication_id, '^0+', '') +WHERE publication_id IS NOT NULL + AND notice_url IS NULL; +``` + +## Verwendung + +### Repository-Abfrage + +```java +// Nach URL suchen +Optional doc = repository.findByNoticeUrl( + "https://ted.europa.eu/en/notice/-/detail/786665-2025" +); +``` + +### Entity-Zugriff + +```java +ProcurementDocument doc = new ProcurementDocument(); +doc.setPublicationId("00786665-2025"); +// notice_url wird automatisch beim Speichern generiert +repository.save(doc); + +// URL abrufen +String url = doc.getNoticeUrl(); +// "https://ted.europa.eu/en/notice/-/detail/786665-2025" +``` + +### REST API Beispiel + +```json +{ + "id": "20fde305-844b-46b7-bb72-93e86381978d", + "publicationId": "00786665-2025", + "noticeUrl": "https://ted.europa.eu/en/notice/-/detail/786665-2025", + "buyerName": "Example Organization", + "projectTitle": "Construction Services" +} +``` + +## Vorteile + +✅ **Direkte Verlinkung** zur offiziellen TED-Webseite +✅ **Automatische Generierung** - keine manuelle Pflege nötig +✅ **Indiziert** für schnelle Suche +✅ **Konsistentes Format** - einheitliche URLs +✅ **Integration** in REST API und Such-Ergebnisse + +## SQL-Abfragen + +### URL für Publication ID generieren + +```sql +SELECT + publication_id, + 'https://ted.europa.eu/en/notice/-/detail/' || + REGEXP_REPLACE(publication_id, '^0+', '') AS notice_url +FROM ted.procurement_document +WHERE publication_id = '00786665-2025'; +``` + +### Alle URLs anzeigen + +```sql +SELECT + publication_id, + notice_url, + buyer_name, + project_title +FROM ted.procurement_document +WHERE notice_url IS NOT NULL +ORDER BY created_at DESC +LIMIT 10; +``` + +### Nach URL-Pattern suchen + +```sql +SELECT * +FROM ted.procurement_document +WHERE notice_url LIKE '%/786665-2025'; +``` + +## Migration + +Beim Hinzufügen des Features wurden: + +1. ✅ Spalte `notice_url` zur Tabelle hinzugefügt +2. ✅ Index `idx_doc_notice_url` erstellt +3. ✅ URLs für alle existierenden Datensätze generiert +4. ✅ Entity-Klasse mit automatischer Generierung erweitert +5. ✅ Repository-Methode `findByNoticeUrl()` hinzugefügt + +## Hinweise + +- Die URL wird **nur** generiert wenn `publication_id` vorhanden ist +- Führende Nullen werden automatisch entfernt (`00786665` → `786665`) +- Die URL wird bei jedem Update aktualisiert (falls sich `publication_id` ändert) +- Die Spalte ist **nicht** `NOT NULL` da alte Datensätze möglicherweise keine `publication_id` haben diff --git a/TED_PACKAGE_DOWNLOAD_CAMEL_ROUTE.md b/TED_PACKAGE_DOWNLOAD_CAMEL_ROUTE.md new file mode 100644 index 0000000..5266062 --- /dev/null +++ b/TED_PACKAGE_DOWNLOAD_CAMEL_ROUTE.md @@ -0,0 +1,374 @@ +# TED Package Download - Camel-Native Implementation + +## Übersicht + +Vollständig Camel-basierte Implementierung des automatischen TED Daily Package Downloads unter Verwendung von Apache Camel Enterprise Integration Patterns (EIP). + +## Architektur + +### Verwendete Enterprise Integration Patterns + +1. **Timer Pattern** - Periodischer Trigger für Downloads +2. **Content-Based Router** - Verzweigung basierend auf HTTP-Status +3. **Splitter Pattern** - Parallele Verarbeitung von XML-Dateien +4. **Dead Letter Channel** - Fehlerbehandlung mit Retry-Logik +5. **Message Filter** - Filterung basierend auf Package-Status +6. **Pipes and Filters** - Sequenzielle Verarbeitung + +### Route-Komponenten + +#### 1. **Timer-Scheduler** (`ted-package-scheduler`) +``` +timer → determineNextPackage → choice → download-package +``` +- Läuft alle X Millisekunden (konfigurierbar, Default: 1 Stunde) +- Ermittelt nächstes Package (aktuelles Jahr priorisiert) +- Stoppt automatisch nach 4 aufeinanderfolgenden 404-Fehlern + +#### 2. **HTTP-Downloader** (`ted-package-http-downloader`) +``` +direct:download-package → createPackageRecord → delay → HTTP GET → choice + ├─ 200 OK → process-downloaded-package + ├─ 404 → markPackageNotFound + └─ other → markPackageFailed +``` +- Native HTTP-Component für Downloads +- Rate Limiting via delay() +- Content-Based Routing nach HTTP-Status + +#### 3. **Package-Processor** (`ted-package-processor`) +``` +process-downloaded-package → calculateHash → checkDuplicate → choice + ├─ duplicate → markPackageDuplicate + └─ new → saveDownloadedPackage → extract-tar-gz +``` +- SHA-256 Hash-Berechnung +- Duplikaterkennung via Hash +- Speicherung auf Filesystem + +#### 4. **TAR.GZ-Extractor** (`ted-package-extractor`) +``` +extract-tar-gz → extractTarGz → deleteTarGz (optional) → split-xml-files +``` +- Apache Commons Compress für TAR.GZ +- Extraktion aller XML-Dateien +- Optionales Cleanup + +#### 5. **XML-Splitter** (`ted-package-xml-splitter`) +``` +split-xml-files → split(xmlFiles) → prepareXmlForProcessing → direct:process-document +``` +- Parallele Verarbeitung (.parallelProcessing()) +- Streaming (.streaming()) +- Integration mit bestehender XML-Route + +## Camel-Komponenten + +### Verwendete Camel-Komponenten + +- **timer** - Periodischer Trigger +- **http** - HTTP GET Requests +- **direct** - Synchrone Route-Verbindungen +- **bean** - Processor-Aufrufe +- **file** - Filesystem-Operationen (indirekt via Processor) + +### Dependencies (pom.xml) + +```xml + + org.apache.camel + camel-http + ${camel.version} + + + org.apache.camel + camel-bean + ${camel.version} + + + org.apache.camel + camel-jackson + ${camel.version} + + + org.apache.commons + commons-compress + 1.27.1 + +``` + +## Workflow-Diagramm + +``` +┌─────────────────────┐ +│ Timer (1h) │ +│ Scheduler │ +└──────┬──────────────┘ + │ + ▼ +┌─────────────────────┐ +│ Determine Next │ +│ Package (Bean) │ +└──────┬──────────────┘ + │ + ▼ +┌─────────────────────┐ +│ HTTP GET │ +│ https://ted... │ +└──────┬──────────────┘ + │ + ▼ + ┌──┴───┐ + │Choice│ + └──┬───┘ + │ + ┌──┴─────┬─────────┬─────────┐ + │ │ │ │ + 200 404 Other Error + │ │ │ │ + ▼ ▼ ▼ ▼ + Process NotFound Failed Dead Letter + │ + ▼ +┌─────────────────────┐ +│ Calculate Hash │ +│ (SHA-256) │ +└──────┬──────────────┘ + │ + ▼ +┌─────────────────────┐ +│ Check Duplicate │ +│ (DB Query) │ +└──────┬──────────────┘ + │ + ┌──┴───┐ + │Choice│ + └──┬───┘ + │ + ┌──┴─────┬─────────┐ + │ │ │ + New Duplicate │ + │ │ │ + ▼ ▼ │ + Extract Complete │ + │ │ + ▼ │ +┌─────────────────────┤ +│ Extract TAR.GZ │ +│ (Apache Commons) │ +└──────┬──────────────┘ + │ + ▼ +┌─────────────────────┐ +│ Split XML Files │ +│ (Parallel) │ +└──────┬──────────────┘ + │ + ▼ +┌─────────────────────┐ +│ Process Document │ +│ (existing route) │ +└─────────────────────┘ +``` + +## Message Headers + +### Download Route Headers + +| Header | Type | Beschreibung | +|--------|------|--------------| +| `packageId` | String | YYYYSSSSS Format | +| `year` | Integer | Jahr des Packages | +| `serialNumber` | Integer | Seriennummer | +| `downloadUrl` | String | Vollständige Download-URL | +| `downloadStartTime` | Long | Start-Timestamp | +| `CamelHttpResponseCode` | Integer | HTTP Status | +| `fileHash` | String | SHA-256 Hash | +| `isDuplicate` | Boolean | Duplikat-Flag | +| `duplicateOf` | String | Original Package-ID | + +### Extraction Headers + +| Header | Type | Beschreibung | +|--------|------|--------------| +| `downloadPath` | String | Pfad zur tar.gz Datei | +| `xmlFiles` | List | Liste der XML-Dateien | +| `xmlFileCount` | Integer | Anzahl XML-Dateien | +| `deleteAfterExtraction` | Boolean | Cleanup-Flag | + +## Konfiguration + +### application.yml + +```yaml +ted: + download: + enabled: true # Aktiviert die Camel-native Route + base-url: https://ted.europa.eu/packages/daily/ + download-directory: D:/ted.europe/downloads + extract-directory: D:/ted.europe/extracted + start-year: 2024 + max-consecutive-404: 4 + poll-interval: 3600000 # 1 Stunde + download-timeout: 300000 # 5 Minuten + delay-between-downloads: 5000 # 5 Sekunden + delete-after-extraction: true + prioritize-current-year: true + + # Optional: Service-basierte Route (alte Implementierung) + use-service-based: false # Deaktiviert +``` + +## Error Handling + +### Dead Letter Channel + +```java +errorHandler(deadLetterChannel("direct:package-download-error") + .maximumRedeliveries(3) + .redeliveryDelay(10000) + .retryAttemptedLogLevel(LoggingLevel.WARN)) +``` + +**Retry-Strategie:** +- Maximale Wiederholungen: 3 +- Verzögerung: 10 Sekunden +- Bei Fehler: Dead Letter Channel + +### Fehlerbehandlung + +1. **HTTP-Fehler**: + - 404 → Status: NOT_FOUND (kein Retry) + - 5xx → Retry 3x + - Andere → Status: FAILED + +2. **Verarbeitungsfehler**: + - Hash-Berechnung fehlgeschlagen → Retry + - Extraktion fehlgeschlagen → Retry + - XML-Verarbeitung fehlgeschlagen → Package-Status bleibt PROCESSING + +## Monitoring & Logging + +### Log-Levels + +```yaml +logging: + level: + at.procon.ted.camel: DEBUG + org.apache.camel: INFO +``` + +### Log-Meldungen + +- `INFO`: Package-Start, Completion, Status-Änderungen +- `DEBUG`: HTTP-Responses, Hash-Berechnungen, Extraktionen +- `WARN`: Duplikate, HTTP-Fehler, Retries +- `ERROR`: Dead Letter Channel, kritische Fehler + +## Performance-Optimierung + +### Parallele Verarbeitung + +```java +.split(header("xmlFiles")) + .parallelProcessing() // Parallele Verarbeitung + .streaming() // Streaming für große Listen +``` + +### Rate Limiting + +```java +.delay(simple("{{ted.download.delay-between-downloads:5000}}")) +``` + +Verhindert Server-Überlastung durch konfigurierbare Verzögerung. + +## Database-Integration + +### Package-Tracking + +Alle Statusänderungen werden in `TED.ted_daily_package` gespeichert: + +```sql +SELECT + package_identifier, + download_status, + xml_file_count, + processed_count, + downloaded_at +FROM TED.ted_daily_package +ORDER BY year DESC, serial_number DESC; +``` + +### Status-Workflow + +``` +PENDING → DOWNLOADING → DOWNLOADED → PROCESSING → COMPLETED + ↓ ↓ + NOT_FOUND FAILED +``` + +## Testing + +### Manueller Test + +```bash +# 1. Verzeichnisse erstellen +mkdir -p D:/ted.europe/downloads +mkdir -p D:/ted.europe/extracted + +# 2. Database Migration +psql -h 94.130.218.54 -p 5432 -U postgres -d Sales \ + -f src/main/resources/db/migration/V2__add_ted_daily_package_table.sql + +# 3. Anwendung starten +mvn spring-boot:run + +# 4. Logs überwachen +tail -f logs/spring.log | grep "ted-package" +``` + +### Erfolgreicher Download (Logs) + +``` +INFO - Checking for new TED packages... +INFO - Next package to download: 202400001 +INFO - Downloaded package 202400001 +INFO - Extracting package 202400001... +INFO - Extracted 1234 XML files from package 202400001 +INFO - Processing 1234 XML files from package 202400001 +INFO - Completed processing package 202400001 +``` + +## Vorteile der Camel-Native Implementierung + +1. ✅ **Enterprise Integration Patterns** - Bewährte Muster +2. ✅ **Declarative Configuration** - Route-Definition in Java +3. ✅ **Native HTTP Component** - Optimiert und getestet +4. ✅ **Monitoring** - Camel JMX-Management +5. ✅ **Error Handling** - Dead Letter Channel, Retry +6. ✅ **Parallel Processing** - Split/Aggregate Pattern +7. ✅ **Message Transformation** - Header/Body-Manipulation +8. ✅ **Content-Based Routing** - Dynamische Verzweigungen + +## Unterschied zur Service-basierten Route + +| Feature | Camel-Native | Service-basiert | +|---------|-------------|-----------------| +| HTTP Download | Camel HTTP Component | Java HttpURLConnection | +| Retry | Camel Error Handler | Manuell | +| Routing | Content-Based Router | if/else | +| Parallelisierung | Camel Splitter | Java Executor | +| Monitoring | Camel JMX | Custom | +| Konfiguration | `ted.download.enabled` | `ted.download.use-service-based` | + +## Nächste Schritte + +1. ✅ Database Migration ausführen +2. ✅ Verzeichnisse erstellen +3. ✅ `ted.download.enabled=true` setzen +4. ✅ Anwendung starten +5. ⏳ Logs überwachen +6. ⏳ DB-Status prüfen + +Das System ist produktionsbereit! 🚀 diff --git a/VECTORIZATION.md b/VECTORIZATION.md new file mode 100644 index 0000000..a6dc33f --- /dev/null +++ b/VECTORIZATION.md @@ -0,0 +1,385 @@ +# Vektorisierung mit Apache Camel + +## Übersicht + +Die Vektorisierung erfolgt vollständig asynchron über **Apache Camel Routes** und nutzt einen externen **Python Embedding Service** über REST. + +## Architektur + +``` +┌─────────────────────────────────────────────────────────────────┐ +│ Vektorisierungs-Pipeline │ +├─────────────────────────────────────────────────────────────────┤ +│ │ +│ ┌──────────────┐ ┌─────────────────┐ ┌───────────────┐ │ +│ │ XML File │───▶│ TedDocumentRoute│───▶│ Document │ │ +│ │ Processing │ │ │ │ Saved to DB │ │ +│ └──────────────┘ └────────┬────────┘ └───────┬───────┘ │ +│ │ │ │ +│ │ wireTap │ │ +│ ▼ │ │ +│ ┌──────────────────────────────────────────────────┼──────┐ │ +│ │ direct:vectorize (Trigger) │ │ │ +│ └──────────────────────────┬───────────────────────┘ │ │ +│ │ │ │ +│ ▼ │ │ +│ ┌──────────────────────────────────────────────────┐ │ │ +│ │ seda:vectorize-async (4 concurrent workers) │ │ │ +│ │ │ │ │ +│ │ 1. Load document from DB │ │ │ +│ │ 2. Extract text_content (includes Lots!) │ │ │ +│ │ 3. Set status = PROCESSING │ │ │ +│ │ 4. Add "passage: " prefix │ │ │ +│ │ 5. Call REST API │ │ │ +│ │ 6. Update content_vector │ │ │ +│ └──────────────┬───────────────────────────────────┘ │ │ +│ │ │ │ +│ ▼ │ │ +│ ┌──────────────────────────────────────────────────┐ │ │ +│ │ Python Embedding Service (Port 8001) │ │ │ +│ │ POST /embed │ │ │ +│ │ Model: intfloat/multilingual-e5-large │ │ │ +│ │ Returns: [1024 floats] │ │ │ +│ └──────────────────────────────────────────────────┘ │ │ +│ │ │ +│ ┌──────────────────────────────────────────────────┐ │ │ +│ │ Timer Route (every 60s) │◀─────┘ │ +│ │ │ │ +│ │ SELECT * FROM procurement_document │ │ +│ │ WHERE vectorization_status = 'PENDING' │ │ +│ │ LIMIT 16 │ │ +│ │ │ │ +│ │ → Trigger vectorization for each │ │ +│ └──────────────────────────────────────────────────┘ │ +│ │ +└─────────────────────────────────────────────────────────────────┘ +``` + +## Apache Camel Routes + +### 1. Trigger Route (`direct:vectorize`) + +**Route-ID:** `vectorization-trigger` + +**Funktion:** Empfängt `documentId` und leitet an async Queue weiter + +**Integration:** Wird von `TedDocumentRoute` per `wireTap` aufgerufen (non-blocking) + +```java +from("direct:vectorize") + .to("seda:vectorize-async?concurrentConsumers=4&waitForTaskToComplete=Never"); +``` + +### 2. Async Processor Route (`seda:vectorize-async`) + +**Route-ID:** `vectorization-processor` + +**Concurrent Workers:** 4 (konfigurierbar) + +**Ablauf:** +1. ✅ Load document from DB via `documentId` +2. ✅ Update status → `PROCESSING` +3. ✅ Extract `text_content` (enthält Dokument + Lots!) +4. ✅ Truncate wenn > `max-text-length` (8192 chars) +5. ✅ Add prefix: `"passage: " + text` +6. ✅ POST → `http://localhost:8001/embed` mit JSON body +7. ✅ Parse JSON response → `float[1024]` +8. ✅ Update `content_vector` in DB +9. ✅ Update status → `COMPLETED` + +**Error Handling:** +- Max 2 Retries mit 2s Delay +- Bei Fehler: Status → `FAILED` mit Error-Message + +### 3. Scheduler Route (`timer:vectorization-scheduler`) + +**Route-ID:** `vectorization-scheduler` + +**Interval:** 60 Sekunden (nach 5s Delay beim Start) + +**Funktion:** Verarbeitet noch nicht vektorisierte Dokumente aus der DB + +**Ablauf:** +```java +from("timer:vectorization-scheduler?period=60000&delay=5000") + .process(exchange -> { + // Load PENDING documents from DB + List pending = + documentRepository.findByVectorizationStatus(PENDING, PageRequest.of(0, 16)); + }) + .split(body()) + .to("direct:vectorize") // Trigger für jedes Dokument + .end(); +``` + +## Text-Inhalt für Vektorisierung + +Der `text_content` wird in `XmlParserService.generateTextContent()` erstellt und enthält: + +``` +Title: Mission de maitrise d'œuvre pour la création... + +Description: Désignation d'une équipe de maîtrise d'œuvre... + +Contracting Authority: Société Publique Locale, Bannalec (FRA) + +Contract Type: SERVICES +Procedure: OTHER +CPV Codes: 71200000 + +Lots (1): +- LOT-0001: Mission de maîtrise d'œuvre... - Désignation d'une équipe... +``` + +**Alle Lot-Titel und Beschreibungen werden einbezogen!** + +## REST API: Python Embedding Service + +### Endpoint + +**POST** `http://localhost:8001/embed` + +### Request + +```json +{ + "text": "passage: Title: Mission de maitrise d'œuvre..." +} +``` + +### Response + +```json +[0.123, -0.456, 0.789, ..., 0.321] +``` + +**Format:** JSON Array mit 1024 Floats + +### Model + +- **Name:** `intfloat/multilingual-e5-large` +- **Dimensions:** 1024 +- **Languages:** 100+ (Mehrsprachig) +- **Normalization:** L2-normalized für Cosine Similarity + +### E5 Model Prefixes + +| Typ | Prefix | Verwendung | +|-----|--------|------------| +| **Dokumente** | `passage: ` | Beim Speichern in DB | +| **Queries** | `query: ` | Bei Suchanfragen | + +## Konfiguration + +**application.yml:** + +```yaml +ted: + vectorization: + enabled: true # Aktivierung + use-http-api: true # REST statt Subprocess + api-url: http://localhost:8001 # Embedding Service URL + model-name: intfloat/multilingual-e5-large + dimensions: 1024 + batch-size: 16 # Scheduler batch size + max-text-length: 8192 # Max chars für Vektorisierung +``` + +## Python Embedding Service Starten + +### Option 1: Docker Compose + +```bash +docker-compose up -d embedding-service +``` + +### Option 2: Standalone Python + +**Datei:** `embedding_service.py` + +```python +from flask import Flask, request, jsonify +from sentence_transformers import SentenceTransformer + +app = Flask(__name__) +model = SentenceTransformer('intfloat/multilingual-e5-large') + +@app.route('/embed', methods=['POST']) +def embed(): + data = request.json + text = data['text'] + + # Generate embedding + embedding = model.encode(text, normalize_embeddings=True) + + return jsonify(embedding.tolist()) + +@app.route('/health', methods=['GET']) +def health(): + return jsonify({"status": "ok"}) + +if __name__ == '__main__': + app.run(host='0.0.0.0', port=8001) +``` + +**Start:** +```bash +pip install flask sentence-transformers +python embedding_service.py +``` + +## Monitoring + +### Vektorisierungs-Status prüfen + +```sql +SELECT + vectorization_status, + COUNT(*) as count +FROM ted.procurement_document +GROUP BY vectorization_status; +``` + +**Mögliche Status:** +- `PENDING` - Wartet auf Vektorisierung +- `PROCESSING` - Wird gerade vektorisiert +- `COMPLETED` - Erfolgreich vektorisiert +- `FAILED` - Fehler bei Vektorisierung +- `SKIPPED` - Kein Text-Inhalt vorhanden + +### Admin REST API + +**GET** `/api/v1/admin/vectorization/status` +```json +{ + "enabled": true, + "pending": 42, + "completed": 1523, + "failed": 3 +} +``` + +**POST** `/api/v1/admin/vectorization/process-pending?batchSize=100` + +Trigger manuelle Verarbeitung von PENDING Dokumenten + +### Camel Routes Status + +**Actuator Endpoint:** `http://localhost:8888/api/actuator/camel` + +Zeigt Status aller Camel Routes: +- `vectorization-trigger` +- `vectorization-processor` +- `vectorization-scheduler` + +## Error Handling + +### Retry-Strategie + +```java +onException(Exception.class) + .maximumRedeliveries(2) + .redeliveryDelay(2000) + .handled(true) + .process(exchange -> { + // Update status to FAILED in database + }); +``` + +**Retries:** 2x mit 2 Sekunden Pause + +**Bei endgültigem Fehler:** +- Status → `FAILED` +- Error-Message in `vectorization_error` Spalte gespeichert +- Dokument erscheint nicht mehr im Scheduler (nur PENDING) + +### Häufige Fehler + +| Fehler | Ursache | Lösung | +|--------|---------|--------| +| Connection refused | Embedding Service läuft nicht | Service starten | +| Invalid dimension | Falsches Model | Konfiguration prüfen | +| Timeout | Service überlastet | `concurrentConsumers` reduzieren | +| No text content | Dokument leer | Wird automatisch als SKIPPED markiert | + +## Performance + +### Durchsatz + +**Concurrent Workers:** 4 +- **Pro Worker:** ~2-3 Sekunden pro Dokument +- **Gesamt:** ~60-90 Dokumente/Minute + +**Optimierung:** +```yaml +vectorization: + thread-pool-size: 8 # Mehr concurrent workers +``` + +### Memory + +**E5-Large Model:** +- ~2 GB RAM +- Läuft auf CPU oder GPU +- Einmalig beim Service-Start geladen + +### Netzwerk + +**Request Size:** ~8 KB (8192 chars max) +**Response Size:** ~4 KB (1024 floats) + +## Best Practices + +✅ **DO:** +- Embedding Service separat laufen lassen +- Service-Health über `/health` endpoint prüfen +- Batch-Size an Server-Kapazität anpassen +- Failed Dokumente regelmäßig prüfen und retry + +❌ **DON'T:** +- Nicht mehr als 8 concurrent workers (überlastet Service) +- Nicht zu große `max-text-length` (>10000 chars) +- Service nicht ohne Health-Check deployen + +## Semantic Search + +Nach erfolgreicher Vektorisierung sind Dokumente über Semantic Search auffindbar: + +```bash +curl "http://localhost:8888/api/v1/documents/semantic-search?query=medical+equipment" +``` + +**Technologie:** +- PostgreSQL pgvector Extension +- Cosine Similarity (`1 - (vec1 <=> vec2)`) +- IVFFlat Index für schnelle Suche + +## Troubleshooting + +### Dokumente werden nicht vektorisiert + +1. ✅ Check Embedding Service: `curl http://localhost:8001/health` +2. ✅ Check Logs: `vectorization-processor` Route +3. ✅ Check DB: `SELECT * FROM procurement_document WHERE vectorization_status = 'FAILED'` +4. ✅ Check Config: `vectorization.enabled = true` + +### Embedding Service antwortet nicht + +```bash +# Service Status +curl http://localhost:8001/health + +# Test embedding +curl -X POST http://localhost:8001/embed \ + -H "Content-Type: application/json" \ + -d '{"text": "passage: test"}' +``` + +### Camel Route läuft nicht + +```bash +# Actuator Camel Routes +curl http://localhost:8888/api/actuator/camel/routes +``` + +Prüfen ob Route `vectorization-processor` Status `Started` hat. diff --git a/XPATH_EXAMPLES.md b/XPATH_EXAMPLES.md new file mode 100644 index 0000000..36adc83 --- /dev/null +++ b/XPATH_EXAMPLES.md @@ -0,0 +1,241 @@ +# Native XML-Spalte mit XPath-Abfragen + +## ✅ Implementiert + +Die `xml_document`-Spalte ist jetzt ein nativer PostgreSQL XML-Typ mit voller XPath-Unterstützung. + +## Hibernate-Konfiguration + +```java +@Column(name = "xml_document", nullable = false) +@JdbcTypeCode(SqlTypes.SQLXML) +private String xmlDocument; +``` + +## XPath-Abfrage Beispiele + +### 1. **Einfache XPath-Abfrage (ohne Namespaces)** + +```sql +-- Alle Dokument-IDs extrahieren +SELECT + id, + xpath('//ID/text()', xml_document) as document_ids +FROM ted.procurement_document; +``` + +### 2. **XPath mit Namespaces (eForms UBL)** + +eForms verwendet XML-Namespaces. Sie müssen diese bei XPath-Abfragen angeben: + +```sql +-- Titel extrahieren (mit Namespace) +SELECT + id, + xpath( + '//cbc:Title/text()', + xml_document, + ARRAY[ + ARRAY['cbc', 'urn:oasis:names:specification:ubl:schema:xsd:CommonBasicComponents-2'], + ARRAY['cac', 'urn:oasis:names:specification:ubl:schema:xsd:CommonAggregateComponents-2'] + ] + ) as titles +FROM ted.procurement_document; +``` + +### 3. **Buyer Name extrahieren** + +```sql +SELECT + id, + publication_id, + xpath( + '//cac:ContractingParty/cac:Party/cac:PartyName/cbc:Name/text()', + xml_document, + ARRAY[ + ARRAY['cbc', 'urn:oasis:names:specification:ubl:schema:xsd:CommonBasicComponents-2'], + ARRAY['cac', 'urn:oasis:names:specification:ubl:schema:xsd:CommonAggregateComponents-2'] + ] + ) as buyer_names +FROM ted.procurement_document; +``` + +### 4. **CPV-Codes extrahieren** + +```sql +SELECT + id, + xpath( + '//cac:ProcurementProject/cac:MainCommodityClassification/cbc:ItemClassificationCode/text()', + xml_document, + ARRAY[ + ARRAY['cbc', 'urn:oasis:names:specification:ubl:schema:xsd:CommonBasicComponents-2'], + ARRAY['cac', 'urn:oasis:names:specification:ubl:schema:xsd:CommonAggregateComponents-2'] + ] + ) as cpv_codes +FROM ted.procurement_document; +``` + +### 5. **Filtern nach XML-Inhalt** + +```sql +-- Alle Dokumente finden, die einen bestimmten CPV-Code enthalten +SELECT + id, + publication_id, + buyer_name +FROM ted.procurement_document +WHERE xpath_exists( + '//cac:ProcurementProject/cac:MainCommodityClassification/cbc:ItemClassificationCode[text()="45000000"]', + xml_document, + ARRAY[ + ARRAY['cbc', 'urn:oasis:names:specification:ubl:schema:xsd:CommonBasicComponents-2'], + ARRAY['cac', 'urn:oasis:names:specification:ubl:schema:xsd:CommonAggregateComponents-2'] + ] +); +``` + +### 6. **Estimated Value extrahieren** + +```sql +SELECT + id, + xpath( + '//cac:ProcurementProject/cac:RequestedTenderTotal/cbc:EstimatedOverallContractAmount/text()', + xml_document, + ARRAY[ + ARRAY['cbc', 'urn:oasis:names:specification:ubl:schema:xsd:CommonBasicComponents-2'], + ARRAY['cac', 'urn:oasis:names:specification:ubl:schema:xsd:CommonAggregateComponents-2'] + ] + ) as estimated_values +FROM ted.procurement_document; +``` + +## JPA/Hibernate Native Queries + +Sie können XPath auch in Spring Data JPA Repositories verwenden: + +### Repository-Beispiel + +```java +@Repository +public interface ProcurementDocumentRepository extends JpaRepository { + + /** + * Findet alle Dokumente, die einen bestimmten Text im Titel enthalten (via XPath) + */ + @Query(value = """ + SELECT * FROM ted.procurement_document + WHERE xpath_exists( + '//cbc:Title[contains(text(), :searchText)]', + xml_document, + ARRAY[ + ARRAY['cbc', 'urn:oasis:names:specification:ubl:schema:xsd:CommonBasicComponents-2'] + ] + ) + """, nativeQuery = true) + List findByTitleContaining(@Param("searchText") String searchText); + + /** + * Extrahiert CPV-Codes via XPath + */ + @Query(value = """ + SELECT + unnest(xpath( + '//cac:ProcurementProject/cac:MainCommodityClassification/cbc:ItemClassificationCode/text()', + xml_document, + ARRAY[ + ARRAY['cbc', 'urn:oasis:names:specification:ubl:schema:xsd:CommonBasicComponents-2'], + ARRAY['cac', 'urn:oasis:names:specification:ubl:schema:xsd:CommonAggregateComponents-2'] + ] + ))::text as cpv_code + FROM ted.procurement_document + WHERE id = :documentId + """, nativeQuery = true) + List extractCpvCodes(@Param("documentId") UUID documentId); + + /** + * Findet Dokumente nach CPV-Code + */ + @Query(value = """ + SELECT * FROM ted.procurement_document + WHERE xpath_exists( + '//cbc:ItemClassificationCode[text()=:cpvCode]', + xml_document, + ARRAY[ + ARRAY['cbc', 'urn:oasis:names:specification:ubl:schema:xsd:CommonBasicComponents-2'] + ] + ) + """, nativeQuery = true) + List findByCpvCode(@Param("cpvCode") String cpvCode); +} +``` + +## PostgreSQL XML-Funktionen + +Weitere nützliche XML-Funktionen: + +### `xml_is_well_formed()` +```sql +SELECT xml_is_well_formed(xml_document) FROM ted.procurement_document; +``` + +### `xpath_exists()` - Prüft ob Pfad existiert +```sql +SELECT xpath_exists('//cbc:Title', xml_document, ...) FROM ted.procurement_document; +``` + +### `unnest()` - Array zu Zeilen +```sql +SELECT + id, + unnest(xpath('//cbc:Title/text()', xml_document, ...))::text as title +FROM ted.procurement_document; +``` + +## Häufige eForms Namespaces + +```sql +ARRAY[ + ARRAY['cbc', 'urn:oasis:names:specification:ubl:schema:xsd:CommonBasicComponents-2'], + ARRAY['cac', 'urn:oasis:names:specification:ubl:schema:xsd:CommonAggregateComponents-2'], + ARRAY['ext', 'urn:oasis:names:specification:ubl:schema:xsd:CommonExtensionComponents-2'], + ARRAY['efac', 'http://data.europa.eu/p27/eforms-ubl-extension-aggregate-components/1'], + ARRAY['efbc', 'http://data.europa.eu/p27/eforms-ubl-extension-basic-components/1'] +] +``` + +## Performance-Tipps + +1. **Indizierung**: Für häufige XPath-Abfragen können Sie funktionale Indexe erstellen: +```sql +CREATE INDEX idx_doc_title ON ted.procurement_document + USING GIN ((xpath('//cbc:Title/text()', xml_document, ...))); +``` + +2. **Materialized Views**: Für komplexe XPath-Abfragen: +```sql +CREATE MATERIALIZED VIEW ted.document_titles AS +SELECT + id, + unnest(xpath('//cbc:Title/text()', xml_document, ...))::text as title +FROM ted.procurement_document; +``` + +## Vorteile der nativen XML-Spalte + +✅ Native XPath-Abfragen +✅ XML-Validierung möglich +✅ Effiziente Speicherung +✅ PostgreSQL XML-Funktionen verfügbar +✅ Strukturierte Abfragen auf XML-Elementen +✅ Funktionale Indexe möglich + +## Hibernate funktioniert jetzt korrekt + +Mit `@JdbcTypeCode(SqlTypes.SQLXML)` weiß Hibernate, dass es `SQLXML` verwenden muss für INSERT/UPDATE. + +Das verhindert den Fehler: +``` +ERROR: column "xml_document" is of type xml but expression is of type character varying +``` diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..73f2323 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,102 @@ +# TED Procurement Document Processor - Docker Compose +# Author: Martin.Schweitzer@procon.co.at and claude.ai +# +# Services: +# - PostgreSQL 16 with pgvector extension (nur für lokale Entwicklung) +# - Python embedding service für Vektorisierung +# +# Standard-Verwendung (nur Embedding-Service): +# docker-compose --profile with-embedding up -d +# +# Lokale Datenbank verwenden: +# docker-compose --profile local-db up -d +# +# Remote-Datenbank (94.130.218.54): +# Konfiguration in application.yml (Standard-Einstellung) + +version: '3.8' + +services: + # PostgreSQL database with pgvector extension (NUR FÜR LOKALE ENTWICKLUNG) + # Standardmäßig deaktiviert - verwende Remote-Server 94.130.218.54 + postgres: + image: pgvector/pgvector:pg16 + container_name: ted-postgres + environment: + POSTGRES_DB: sales + POSTGRES_USER: postgresql + POSTGRES_PASSWORD: "PDmXRx0Rbk9OFOn9qO5Gm/mPCfqW8zwbZ+/YIU1lySc=" + ports: + - "5432:5432" + volumes: + - postgres_data:/var/lib/postgresql/data + - ./init-db.sql:/docker-entrypoint-initdb.d/init-db.sql:ro + healthcheck: + test: ["CMD-SHELL", "pg_isready -U postgresql -d sales"] + interval: 10s + timeout: 5s + retries: 5 + restart: unless-stopped + profiles: + - local-db + + # Python embedding service (optional - for production vectorization) + embedding-service: + build: + context: . + dockerfile: Dockerfile.embedding + container_name: ted-embedding-service + ports: + - "8001:8001" + environment: + MODEL_NAME: intfloat/multilingual-e5-large + MAX_LENGTH: 512 + BATCH_SIZE: 16 + volumes: + - model_cache:/root/.cache/huggingface + deploy: + resources: + reservations: + devices: + - driver: nvidia + count: 1 + capabilities: [gpu] + # Remove GPU section if not using NVIDIA GPU + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:8001/health"] + interval: 30s + timeout: 10s + retries: 3 + restart: unless-stopped + profiles: + - with-embedding + + # PgAdmin for database management (nur für lokale Datenbank) + pgadmin: + image: dpage/pgadmin4:latest + container_name: ted-pgadmin + environment: + PGADMIN_DEFAULT_EMAIL: admin@procon.co.at + PGADMIN_DEFAULT_PASSWORD: admin + ports: + - "5050:80" + volumes: + - pgadmin_data:/var/lib/pgadmin + depends_on: + - postgres + restart: unless-stopped + profiles: + - local-db + - tools + +volumes: + postgres_data: + driver: local + model_cache: + driver: local + pgadmin_data: + driver: local + +networks: + default: + name: ted-network diff --git a/docs/architecture/architecture.archimate b/docs/architecture/architecture.archimate new file mode 100644 index 0000000..19e50d2 --- /dev/null +++ b/docs/architecture/architecture.archimate @@ -0,0 +1,35 @@ + + + + + + + + + https://ted.europa.eu/packages/daily/ + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/docs/architecture/architecture.archimate.bak b/docs/architecture/architecture.archimate.bak new file mode 100644 index 0000000..3286841 --- /dev/null +++ b/docs/architecture/architecture.archimate.bak @@ -0,0 +1,14 @@ + + + + + + + + + + + + + + diff --git a/embedding_service.py b/embedding_service.py new file mode 100644 index 0000000..9c2dd3a --- /dev/null +++ b/embedding_service.py @@ -0,0 +1,340 @@ +#!/usr/bin/env python3 +""" +TED Procurement Document Embedding Service + +HTTP API for generating text embeddings using sentence-transformers. +Model: intfloat/multilingual-e5-large (1024 dimensions) + +Author: Martin.Schweitzer@procon.co.at and claude.ai + +Usage: + python embedding_service.py + +Environment Variables: + MODEL_NAME: Model to use (default: intfloat/multilingual-e5-large) + MAX_LENGTH: Maximum token length (default: 512) + HOST: Server host (default: 0.0.0.0) + PORT: Server port (default: 8001) + +API Endpoints: + POST /embed - Generate embedding for single text + POST /embed/batch - Generate embeddings for multiple texts + GET /health - Health check +""" + +import os +import logging +import threading +import time +from typing import List +from contextlib import asynccontextmanager + +import numpy as np +from fastapi import FastAPI, HTTPException +from pydantic import BaseModel, Field +import uvicorn + +# Configure logging +logging.basicConfig( + level=logging.INFO, + format='%(asctime)s - %(name)s - %(levelname)s - %(message)s' +) +logger = logging.getLogger(__name__) + +# Suppress noisy HTTP warnings from uvicorn and asyncio +logging.getLogger("uvicorn.error").setLevel(logging.CRITICAL) +logging.getLogger("uvicorn.access").setLevel(logging.WARNING) +logging.getLogger("asyncio").setLevel(logging.CRITICAL) + +# Configuration from environment +MODEL_NAME = os.getenv("MODEL_NAME", "intfloat/multilingual-e5-large") +#MODEL_NAME = os.getenv("MODEL_NAME", "BAAI/bge-m3") +MAX_LENGTH = int(os.getenv("MAX_LENGTH", "512")) +HOST = os.getenv("HOST", "0.0.0.0") +PORT = int(os.getenv("PORT", "8001")) + +# Global model instance (single model) with thread-safe access +model = None +model_lock = threading.Lock() +model_dimensions = None + +# Statistics +embedding_count = 0 +total_embedding_time = 0.0 +stats_lock = threading.Lock() + + +class EmbedRequest(BaseModel): + """Request model for single text embedding.""" + text: str = Field(..., description="Text to embed") + is_query: bool = Field(False, description="If True, use 'query:' prefix for e5 models") + + +class EmbedBatchRequest(BaseModel): + """Request model for batch text embedding.""" + texts: List[str] = Field(..., description="List of texts to embed") + is_query: bool = Field(False, description="If True, use 'query:' prefix for e5 models") + + +class EmbedResponse(BaseModel): + """Response model for embedding result.""" + embedding: List[float] = Field(..., description="Vector embedding") + dimensions: int = Field(..., description="Number of dimensions") + token_count: int = Field(..., description="Number of input tokens") + + +class EmbedBatchResponse(BaseModel): + """Response model for batch embedding result.""" + embeddings: List[List[float]] = Field(..., description="List of vector embeddings") + dimensions: int = Field(..., description="Number of dimensions") + count: int = Field(..., description="Number of embeddings generated") + token_counts: List[int] = Field(..., description="Number of input tokens for each text") + + +class HealthResponse(BaseModel): + """Health check response.""" + status: str + model_name: str + dimensions: int + max_length: int + embeddings_processed: int + avg_time_ms: float + + +@asynccontextmanager +async def lifespan(app: FastAPI): + """Initialize single model on startup.""" + global model, model_dimensions + from sentence_transformers import SentenceTransformer + + logger.info(f"Loading single model: {MODEL_NAME}") + + try: + model = SentenceTransformer(MODEL_NAME) + model_dimensions = model.get_sentence_embedding_dimension() + logger.info(f"Model loaded successfully. Embedding dimension: {model_dimensions}") + logger.info("Ready to process embeddings - statistics will be logged every 100 embeddings") + except Exception as e: + logger.error(f"Failed to load model: {e}") + raise + + yield + + # Cleanup + with stats_lock: + avg_time_ms = (total_embedding_time / embedding_count * 1000) if embedding_count > 0 else 0.0 + logger.info(f"Shutting down embedding service - Final statistics: {embedding_count} embeddings processed, average time: {avg_time_ms:.2f}ms per embedding") + + +# Create FastAPI app +app = FastAPI( + title="TED Embedding Service", + description="Generate text embeddings using sentence-transformers for semantic search", + version="1.0.0", + lifespan=lifespan +) + + +def add_prefix(text: str, is_query: bool) -> str: + """Add appropriate prefix for e5 models.""" + if "e5" in MODEL_NAME.lower(): + prefix = "query: " if is_query else "passage: " + return prefix + text + return text + + +def check_token_length(text: str, model) -> tuple[int, bool]: + """ + Check if text exceeds MAX_LENGTH tokens and return token count. + + Returns: + tuple: (token_count, is_truncated) + """ + try: + # Get tokenizer from model + tokenizer = model.tokenizer + tokens = tokenizer.encode(text, add_special_tokens=True) + token_count = len(tokens) + byte_count = len(text.encode('utf-8')) + + if token_count > MAX_LENGTH: + logger.warning( + f"Text exceeds MAX_LENGTH ({MAX_LENGTH} tokens). " + f"Actual: {token_count} tokens, {byte_count} bytes ({len(text)} chars). " + f"Text will be truncated by the model. " + f"Preview: {text[:100]}..." + ) + return token_count, True + + return token_count, False + + except Exception as e: + logger.debug(f"Could not check token length: {e}") + return 0, False + + +@app.post("/embed", response_model=EmbedResponse) +async def embed_text(request: EmbedRequest) -> EmbedResponse: + """Generate embedding for a single text using thread-safe single model.""" + global embedding_count, total_embedding_time + + if model is None: + raise HTTPException(status_code=503, detail="Model not initialized") + + try: + start_time = time.time() + + # Thread-safe access to single model + with model_lock: + # Add prefix for e5 models + text = add_prefix(request.text, request.is_query) + + # Check token length and warn if exceeding MAX_LENGTH + token_count, is_truncated = check_token_length(text, model) + byte_count = len(text.encode('utf-8')) + if is_truncated: + logger.info(f"Processing text: {token_count} tokens, {byte_count} bytes ({len(text)} chars) - exceeds {MAX_LENGTH}, will be truncated") + + # Generate embedding + embedding = model.encode( + text, + normalize_embeddings=True, + convert_to_numpy=True + ) + + # Update statistics + elapsed_time = time.time() - start_time + with stats_lock: + embedding_count += 1 + total_embedding_time += elapsed_time + + # Log statistics every 100 embeddings + if embedding_count % 100 == 0: + avg_time = total_embedding_time / embedding_count + logger.info(f"Statistics: {embedding_count} embeddings processed, average time: {avg_time*1000:.2f}ms per embedding") + + return EmbedResponse( + embedding=embedding.tolist(), + dimensions=len(embedding), + token_count=token_count + ) + + except Exception as e: + logger.error(f"Embedding failed: {e}") + raise HTTPException(status_code=500, detail=str(e)) + + +@app.post("/embed/batch", response_model=EmbedBatchResponse) +async def embed_batch(request: EmbedBatchRequest) -> EmbedBatchResponse: + """Generate embeddings for multiple texts using thread-safe single model.""" + global embedding_count, total_embedding_time + + if model is None: + raise HTTPException(status_code=503, detail="Model not initialized") + + if not request.texts: + raise HTTPException(status_code=400, detail="Empty text list") + + try: + start_time = time.time() + batch_count = len(request.texts) + + # Thread-safe access to single model + with model_lock: + # Add prefixes + texts = [add_prefix(text, request.is_query) for text in request.texts] + + # Check token length for each text and warn if exceeding MAX_LENGTH + truncated_count = 0 + token_counts = [] + for i, text in enumerate(texts): + token_count, is_truncated = check_token_length(text, model) + token_counts.append(token_count) + if is_truncated: + truncated_count += 1 + byte_count = len(text.encode('utf-8')) + logger.info( + f"Batch item {i + 1}/{len(texts)}: {token_count} tokens, " + f"{byte_count} bytes ({len(text)} chars) - exceeds {MAX_LENGTH}, will be truncated" + ) + + if truncated_count > 0: + logger.warning( + f"Batch processing: {truncated_count}/{len(texts)} texts exceed " + f"MAX_LENGTH ({MAX_LENGTH} tokens) and will be truncated" + ) + + # Generate embeddings + embeddings = model.encode( + texts, + normalize_embeddings=True, + convert_to_numpy=True, + batch_size=16, + show_progress_bar=False + ) + + # Update statistics + elapsed_time = time.time() - start_time + with stats_lock: + embedding_count += batch_count + total_embedding_time += elapsed_time + + # Log statistics every 100 embeddings + if embedding_count % 100 == 0 or (embedding_count // 100) != ((embedding_count - batch_count) // 100): + avg_time = total_embedding_time / embedding_count + logger.info(f"Statistics: {embedding_count} embeddings processed, average time: {avg_time*1000:.2f}ms per embedding") + + return EmbedBatchResponse( + embeddings=[emb.tolist() for emb in embeddings], + dimensions=embeddings.shape[1] if len(embeddings.shape) > 1 else len(embeddings), + count=len(embeddings), + token_counts=token_counts + ) + + except Exception as e: + logger.error(f"Batch embedding failed: {e}") + raise HTTPException(status_code=500, detail=str(e)) + + +@app.get("/health", response_model=HealthResponse) +async def health_check() -> HealthResponse: + """Health check endpoint.""" + if model is None: + raise HTTPException(status_code=503, detail="Model not initialized") + + with stats_lock: + avg_time_ms = (total_embedding_time / embedding_count * 1000) if embedding_count > 0 else 0.0 + + return HealthResponse( + status="healthy", + model_name=MODEL_NAME, + dimensions=model_dimensions, + max_length=MAX_LENGTH, + embeddings_processed=embedding_count, + avg_time_ms=round(avg_time_ms, 2) + ) + + +@app.get("/") +async def root(): + """Root endpoint with API info.""" + return { + "service": "TED Embedding Service", + "model": MODEL_NAME, + "endpoints": { + "embed": "POST /embed - Generate single embedding", + "embed_batch": "POST /embed/batch - Generate batch embeddings", + "health": "GET /health - Health check" + } + } + + +if __name__ == "__main__": + logger.info(f"Starting embedding service on {HOST}:{PORT}") + uvicorn.run( + "embedding_service:app", + host=HOST, + port=PORT, + log_level="info", + reload=False + ) diff --git a/execute-enum-fix.bat b/execute-enum-fix.bat new file mode 100644 index 0000000..155cb4e --- /dev/null +++ b/execute-enum-fix.bat @@ -0,0 +1,30 @@ +@echo off +REM Batch script to execute ENUM fix using psql +REM If psql is not in PATH, update the PSQL_PATH variable below + +SET PSQL_PATH=psql +SET PGPASSWORD=PDmXRx0Rbk9OFOn9qO5Gm/mPCfqW8zwbZ+/YIU1lySc= +SET PGHOST=94.130.218.54 +SET PGPORT=5432 +SET PGUSER=postgres +SET PGDATABASE=Sales + +echo Executing ENUM fix on remote database... +echo Host: %PGHOST%:%PGPORT% +echo Database: %PGDATABASE% +echo. + +%PSQL_PATH% -h %PGHOST% -p %PGPORT% -U %PGUSER% -d %PGDATABASE% -f "fix-enum-types-comprehensive.sql" + +if %ERRORLEVEL% EQU 0 ( + echo. + echo SUCCESS: ENUM fix executed successfully! + echo Please restart your Spring Boot application. +) else ( + echo. + echo ERROR: Failed to execute ENUM fix. Error code: %ERRORLEVEL% + echo. + echo If psql is not found, please install PostgreSQL client tools or use a GUI tool like DBeaver. +) + +pause diff --git a/fix-organization-schema.bat b/fix-organization-schema.bat new file mode 100644 index 0000000..edf7da1 --- /dev/null +++ b/fix-organization-schema.bat @@ -0,0 +1,21 @@ +@echo off +REM Fix organization table schema - extend VARCHAR fields +REM This applies the V2 migration manually if Flyway hasn't picked it up + +echo Applying schema fix to ted.organization table... + +set PGPASSWORD=PDmXRx0Rbk9OFOn9qO5Gm/mPCfqW8zwbZ+/YIU1lySc= + +psql -h 94.130.218.54 -p 32333 -U postgres -d RELM -c "ALTER TABLE ted.organization ALTER COLUMN postal_code TYPE VARCHAR(255); ALTER TABLE ted.organization ALTER COLUMN street_name TYPE TEXT; ALTER TABLE ted.organization ALTER COLUMN city TYPE VARCHAR(255); ALTER TABLE ted.organization ALTER COLUMN phone TYPE VARCHAR(100); ALTER TABLE ted.organization ALTER COLUMN org_reference TYPE VARCHAR(100); ALTER TABLE ted.organization ALTER COLUMN role TYPE VARCHAR(100); SELECT 'Schema updated successfully!' AS result;" + +if %ERRORLEVEL% EQU 0 ( + echo. + echo Schema fix applied successfully! + echo You can now restart the application. +) else ( + echo. + echo ERROR: Failed to apply schema fix. + echo Please check the error messages above. +) + +pause diff --git a/fix-organization-schema.sql b/fix-organization-schema.sql new file mode 100644 index 0000000..9c4729d --- /dev/null +++ b/fix-organization-schema.sql @@ -0,0 +1,20 @@ +-- Fix organization table schema - extend VARCHAR fields to handle long TED data +-- Run this manually if Flyway migration V2 hasn't been applied yet +-- Usage: psql -h 94.130.218.54 -p 32333 -U postgres -d RELM -f fix-organization-schema.sql + +-- Check current schema +\d ted.organization + +-- Extend VARCHAR fields in organization table +ALTER TABLE ted.organization ALTER COLUMN postal_code TYPE VARCHAR(255); +ALTER TABLE ted.organization ALTER COLUMN street_name TYPE TEXT; +ALTER TABLE ted.organization ALTER COLUMN city TYPE VARCHAR(255); +ALTER TABLE ted.organization ALTER COLUMN phone TYPE VARCHAR(100); +ALTER TABLE ted.organization ALTER COLUMN org_reference TYPE VARCHAR(100); +ALTER TABLE ted.organization ALTER COLUMN role TYPE VARCHAR(100); + +-- Verify changes +\d ted.organization + +-- Show what was changed +SELECT 'Schema updated successfully' AS result; diff --git a/pom.xml b/pom.xml new file mode 100644 index 0000000..0c30c06 --- /dev/null +++ b/pom.xml @@ -0,0 +1,265 @@ + + + + 4.0.0 + + + org.springframework.boot + spring-boot-starter-parent + 3.3.5 + + + + at.procon.ted + ted-procurement-processor + 1.0.0-SNAPSHOT + TED Procurement Processor + EU eForms TED document processor with vector search capabilities + + + 21 + 4.8.1 + 1.13.1 + 0.1.6 + 0.30.0 + + + + + + org.springframework.boot + spring-boot-starter-web + + + org.springframework.boot + spring-boot-starter-data-jpa + + + org.springframework.boot + spring-boot-starter-validation + + + org.springframework.boot + spring-boot-starter-actuator + + + + + org.apache.camel.springboot + camel-spring-boot-starter + ${camel.version} + + + org.apache.camel + camel-file + ${camel.version} + + + org.apache.camel + camel-jaxb + ${camel.version} + + + org.apache.camel + camel-validator + ${camel.version} + + + org.apache.camel + camel-http + ${camel.version} + + + org.apache.camel + camel-bean + ${camel.version} + + + org.apache.camel + camel-jackson + ${camel.version} + + + org.apache.camel + camel-mail + ${camel.version} + + + + + org.jsoup + jsoup + 1.18.1 + + + + + org.apache.pdfbox + pdfbox + 3.0.3 + + + + + org.postgresql + postgresql + runtime + + + com.pgvector + pgvector + ${pgvector.version} + + + + + eu.europa.ted.eforms + eforms-sdk + ${eforms-sdk.version} + + + + + jakarta.xml.bind + jakarta.xml.bind-api + 4.0.2 + + + org.glassfish.jaxb + jaxb-runtime + 4.0.5 + + + com.sun.xml.bind + jaxb-impl + 4.0.5 + runtime + + + + + ai.djl + api + ${djl.version} + + + ai.djl.huggingface + tokenizers + ${djl.version} + + + ai.djl.pytorch + pytorch-engine + ${djl.version} + runtime + + + ai.djl.pytorch + pytorch-model-zoo + ${djl.version} + + + + + org.projectlombok + lombok + true + + + org.mapstruct + mapstruct + 1.6.2 + + + com.google.guava + guava + 33.3.1-jre + + + org.apache.commons + commons-compress + 1.27.1 + + + + + org.apache.poi + poi-ooxml + 5.3.0 + + + + + org.springdoc + springdoc-openapi-starter-webmvc-ui + 2.6.0 + + + + + org.springframework.boot + spring-boot-starter-test + test + + + org.apache.camel + camel-test-spring-junit5 + ${camel.version} + test + + + org.testcontainers + postgresql + test + + + + + + + org.springframework.boot + spring-boot-maven-plugin + + + + org.projectlombok + lombok + + + + + + org.apache.maven.plugins + maven-compiler-plugin + + ${java.version} + ${java.version} + + + org.projectlombok + lombok + ${lombok.version} + + + org.mapstruct + mapstruct-processor + 1.6.2 + + + + + + + diff --git a/requirements-embedding.txt b/requirements-embedding.txt new file mode 100644 index 0000000..ee889e9 --- /dev/null +++ b/requirements-embedding.txt @@ -0,0 +1,16 @@ +# Python dependencies for embedding service +# Author: Martin.Schweitzer@procon.co.at and claude.ai + +# Sentence Transformers for embedding generation +sentence-transformers>=2.7.0 + +# FastAPI for HTTP API +fastapi>=0.109.0 +uvicorn[standard]>=0.27.0 + +# PyTorch (CPU version - use torch for GPU) +torch>=2.0.0 + +# Utilities +numpy>=1.24.0 +pydantic>=2.0.0 diff --git a/reset-stuck-packages.sql b/reset-stuck-packages.sql new file mode 100644 index 0000000..f51a9d9 --- /dev/null +++ b/reset-stuck-packages.sql @@ -0,0 +1,15 @@ +-- Reset stuck packages to PENDING +-- Run this if packages are stuck in DOWNLOADING or PROCESSING status +-- Usage: psql -h 94.130.218.54 -p 32333 -U postgres -d RELM -f reset-stuck-packages.sql + +UPDATE ted.ted_daily_package +SET download_status = 'PENDING', + error_message = 'Reset from stuck state - manual' +WHERE download_status IN ('DOWNLOADING', 'PROCESSING'); + +-- Show what was reset +SELECT package_identifier, year, serial_number, download_status, error_message, updated_at +FROM ted.ted_daily_package +WHERE error_message LIKE '%Reset from stuck state%' +ORDER BY year DESC, serial_number DESC +LIMIT 10; diff --git a/solution-brief-processed.dat b/solution-brief-processed.dat new file mode 100644 index 0000000..e69de29 diff --git a/src/main/java/at/procon/ted/TedProcurementProcessorApplication.java b/src/main/java/at/procon/ted/TedProcurementProcessorApplication.java new file mode 100644 index 0000000..c86a461 --- /dev/null +++ b/src/main/java/at/procon/ted/TedProcurementProcessorApplication.java @@ -0,0 +1,26 @@ +package at.procon.ted; + +import org.springframework.boot.SpringApplication; +import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.scheduling.annotation.EnableAsync; + +/** + * TED Procurement Document Processor Application. + * + * Processes EU eForms public procurement notices from TED (Tenders Electronic Daily). + * Features: + * - Directory watching with Apache Camel for automated XML processing + * - PostgreSQL storage with native XML support and pgvector for semantic search + * - Asynchronous document vectorization using multilingual-e5-large model + * - REST API for structured and semantic search + * + * @author Martin.Schweitzer@procon.co.at and claude.ai + */ +@SpringBootApplication +@EnableAsync +public class TedProcurementProcessorApplication { + + public static void main(String[] args) { + SpringApplication.run(TedProcurementProcessorApplication.class, args); + } +} diff --git a/src/main/java/at/procon/ted/camel/MailRoute.java b/src/main/java/at/procon/ted/camel/MailRoute.java new file mode 100644 index 0000000..2a56137 --- /dev/null +++ b/src/main/java/at/procon/ted/camel/MailRoute.java @@ -0,0 +1,485 @@ +package at.procon.ted.camel; + +import at.procon.ted.config.TedProcessorProperties; +import at.procon.ted.service.attachment.AttachmentExtractor; +import at.procon.ted.service.attachment.AttachmentProcessingService; +import jakarta.mail.BodyPart; +import jakarta.mail.Message; +import jakarta.mail.Multipart; +import jakarta.mail.Part; +import jakarta.mail.Session; +import jakarta.mail.internet.MimeMessage; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.apache.camel.Exchange; +import org.apache.camel.LoggingLevel; +import org.apache.camel.builder.RouteBuilder; +import org.jsoup.Jsoup; +import org.springframework.stereotype.Component; + +import java.io.ByteArrayInputStream; +import java.io.File; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; +import java.util.Properties; + +/** + * Apache Camel route for IMAP mail processing. + * + * Features: + * - IMAP connection with SSL/TLS to mail server + * - MIME message decoding + * - Asynchronous attachment processing with idempotency + * - PDF text extraction + * - ZIP file extraction with recursive processing + * - HTML to plain text conversion + * + * @author Martin.Schweitzer@procon.co.at and claude.ai + */ +@Component +@RequiredArgsConstructor +@Slf4j +public class MailRoute extends RouteBuilder { + + private static final String ROUTE_ID_IMAP = "mail-imap-consumer"; + private static final String ROUTE_ID_MIME_FILE = "mail-mime-file-consumer"; + private static final String ROUTE_ID_MIME = "mail-mime-decoder"; + private static final String ROUTE_ID_ATTACHMENT = "mail-attachment-processor"; + private static final String ROUTE_ID_ATTACHMENT_ASYNC = "mail-attachment-async"; + + private final TedProcessorProperties properties; + private final AttachmentProcessingService attachmentProcessingService; + + @Override + public void configure() throws Exception { + TedProcessorProperties.MailProperties mail = properties.getMail(); + + if (!mail.isEnabled()) { + log.info("Mail processing is disabled, skipping route configuration"); + return; + } + + log.info("Configuring mail routes (host={}, port={}, ssl={}, user={})", + mail.getHost(), mail.getPort(), mail.isSsl(), mail.getUsername()); + + // Ensure attachment output directory exists + File attachmentDir = new File(mail.getAttachmentOutputDirectory()); + if (!attachmentDir.exists()) { + attachmentDir.mkdirs(); + log.info("Created attachment output directory: {}", attachmentDir.getAbsolutePath()); + } + + // Error handler for mail processing + errorHandler(deadLetterChannel("direct:mail-error-handler") + .maximumRedeliveries(3) + .redeliveryDelay(5000) + .retryAttemptedLogLevel(LoggingLevel.WARN) + .logStackTrace(true)); + + // Mail error handler route + from("direct:mail-error-handler") + .routeId("mail-error-handler") + .process(exchange -> { + Exception exception = exchange.getProperty(Exchange.EXCEPTION_CAUGHT, Exception.class); + String subject = exchange.getIn().getHeader("mailSubject", String.class); + if (exception != null) { + log.error("Mail processing error for subject '{}': {}", subject, exception.getMessage(), exception); + } + }) + .log(LoggingLevel.ERROR, "Mail processing failed: ${exception.message}"); + + // IMAP consumer route + from(buildImapUri()) + .routeId(ROUTE_ID_IMAP) + .log(LoggingLevel.INFO, "Received email: ${header.subject} from ${header.from}") + .to("direct:mime"); + + // MIME file consumer route - reads .eml files from directory + if (mail.isMimeInputEnabled()) { + configureMimeFileConsumer(mail); + } + + // MIME decoder route - decodes the email and extracts content/attachments + from("direct:mime") + .routeId(ROUTE_ID_MIME) + .process(exchange -> { + Message mailMessage = exchange.getIn().getBody(Message.class); + + if (mailMessage == null) { + log.warn("Received null mail message, skipping"); + return; + } + + String subject = mailMessage.getSubject(); + String from = mailMessage.getFrom() != null && mailMessage.getFrom().length > 0 + ? mailMessage.getFrom()[0].toString() : "unknown"; + + log.info("Processing MIME message: subject='{}', from='{}'", subject, from); + + // Store mail metadata in headers + exchange.getIn().setHeader("mailSubject", subject); + exchange.getIn().setHeader("mailFrom", from); + exchange.getIn().setHeader("mailReceivedDate", mailMessage.getReceivedDate()); + + // Process the content + List attachments = new ArrayList<>(); + StringBuilder textContent = new StringBuilder(); + StringBuilder htmlContent = new StringBuilder(); + + processMessageContent(mailMessage, textContent, htmlContent, attachments); + + // Convert HTML to plain text if we have HTML but no plain text + String finalTextContent; + if (textContent.length() == 0 && htmlContent.length() > 0) { + finalTextContent = convertHtmlToText(htmlContent.toString()); + log.debug("Converted HTML mail to plain text ({} chars)", finalTextContent.length()); + } else { + finalTextContent = textContent.toString(); + } + + // Store results + exchange.getIn().setHeader("mailTextContent", finalTextContent); + exchange.getIn().setHeader("mailHtmlContent", htmlContent.toString()); + exchange.getIn().setHeader("mailAttachments", attachments); + exchange.getIn().setHeader("mailAttachmentCount", attachments.size()); + + log.info("MIME decoded: subject='{}', textLength={}, htmlLength={}, attachments={}", + subject, finalTextContent.length(), htmlContent.length(), attachments.size()); + }) + // Queue attachments for async processing + .choice() + .when(simple("${header.mailAttachmentCount} > 0")) + .log(LoggingLevel.INFO, "Queueing ${header.mailAttachmentCount} attachments for async processing") + .otherwise() + .log(LoggingLevel.DEBUG, "No attachments in email: ${header.mailSubject}") + .end() + // Process attachments asynchronously via SEDA + .filter(simple("${header.mailAttachmentCount} > 0")) + .split(header("mailAttachments")) + .to("seda:attachment-async?waitForTaskToComplete=Never&size=500") + .end() + .end() + .log(LoggingLevel.INFO, "Mail processing completed: ${header.mailSubject}"); + + // Async attachment processor route via SEDA + from("seda:attachment-async?concurrentConsumers=2&size=500") + .routeId(ROUTE_ID_ATTACHMENT_ASYNC) + .to("direct:attachment"); + + // Attachment processor route - handles individual attachments with idempotency + from("direct:attachment") + .routeId(ROUTE_ID_ATTACHMENT) + .process(exchange -> { + AttachmentInfo attachment = exchange.getIn().getBody(AttachmentInfo.class); + + if (attachment == null) { + log.warn("Received null attachment info, skipping"); + return; + } + + String mailSubject = exchange.getIn().getHeader("mailSubject", String.class); + String mailFrom = exchange.getIn().getHeader("mailFrom", String.class); + String parentHash = exchange.getIn().getHeader("parentHash", String.class); + + log.info("Processing attachment: '{}' ({} bytes, type={}) from email '{}'", + attachment.getFilename(), attachment.getSize(), + attachment.getContentType(), mailSubject); + + // Process attachment with idempotency check + AttachmentProcessingService.ProcessingResult result = attachmentProcessingService.processAttachment( + attachment.getData(), + attachment.getFilename(), + attachment.getContentType(), + mailSubject, + mailFrom, + parentHash + ); + + if (result.isDuplicate()) { + log.info("Attachment is duplicate, skipping: '{}'", attachment.getFilename()); + exchange.setProperty("isDuplicate", true); + return; + } + + if (!result.isSuccess()) { + log.warn("Attachment processing failed: '{}' - {}", + attachment.getFilename(), result.errorMessage()); + return; + } + + // Store result in exchange + exchange.getIn().setHeader("attachmentId", result.attachment().getId()); + exchange.getIn().setHeader("attachmentHash", result.attachment().getContentHash()); + exchange.getIn().setHeader("extractedText", + result.attachment().getExtractedText() != null + ? result.attachment().getExtractedText().length() + " chars" + : "none"); + + // Queue child attachments (from ZIP) for recursive processing + if (result.hasChildren()) { + log.info("Queueing {} child attachments from ZIP '{}'", + result.childAttachments().size(), attachment.getFilename()); + + for (AttachmentExtractor.ChildAttachment child : result.childAttachments()) { + // Create AttachmentInfo for child and send to SEDA queue + AttachmentInfo childInfo = new AttachmentInfo( + child.filename(), + child.contentType(), + child.data(), + child.data().length + ); + + // Send to SEDA for async processing with parent hash + getContext().createProducerTemplate().sendBodyAndHeaders( + "seda:attachment-async?waitForTaskToComplete=Never", + childInfo, + java.util.Map.of( + "mailSubject", mailSubject != null ? mailSubject : "", + "mailFrom", mailFrom != null ? mailFrom : "", + "parentHash", result.attachment().getContentHash(), + "pathInArchive", child.pathInArchive() + ) + ); + } + } + }) + .choice() + .when(exchangeProperty("isDuplicate").isEqualTo(true)) + .log(LoggingLevel.DEBUG, "Skipped duplicate attachment") + .otherwise() + .log(LoggingLevel.INFO, "Attachment processed: ${header.attachmentId}, extracted=${header.extractedText}") + .end(); + } + + /** + * Configure the MIME file consumer route. + */ + private void configureMimeFileConsumer(TedProcessorProperties.MailProperties mail) throws Exception { + // Ensure MIME input directory exists + File mimeInputDir = new File(mail.getMimeInputDirectory()); + if (!mimeInputDir.exists()) { + mimeInputDir.mkdirs(); + log.info("Created MIME input directory: {}", mimeInputDir.getAbsolutePath()); + } + + String mimeFileUri = buildMimeFileUri(mail); + log.info("Configuring MIME file consumer: {}", mimeFileUri); + + // MIME file consumer - reads .eml files and sends to direct:mime + from(mimeFileUri) + .routeId(ROUTE_ID_MIME_FILE) + .log(LoggingLevel.INFO, "Reading MIME file: ${header.CamelFileName}") + .process(exchange -> { + // Read file content as bytes + byte[] fileContent = exchange.getIn().getBody(byte[].class); + String filename = exchange.getIn().getHeader(Exchange.FILE_NAME, String.class); + + if (fileContent == null || fileContent.length == 0) { + log.warn("Empty MIME file: {}", filename); + throw new RuntimeException("Empty MIME file: " + filename); + } + + log.debug("Parsing MIME file: {} ({} bytes)", filename, fileContent.length); + + // Parse the file as a MimeMessage + Session session = Session.getDefaultInstance(new Properties()); + try (ByteArrayInputStream bais = new ByteArrayInputStream(fileContent)) { + MimeMessage mimeMessage = new MimeMessage(session, bais); + + // Set the parsed message as body for direct:mime + exchange.getIn().setBody(mimeMessage); + + log.info("Parsed MIME file: {} -> subject='{}'", + filename, mimeMessage.getSubject()); + } + }) + .to("direct:mime") + .log(LoggingLevel.INFO, "MIME file processed successfully: ${header.CamelFileName}"); + } + + /** + * Build the file URI for MIME file consumer. + */ + private String buildMimeFileUri(TedProcessorProperties.MailProperties mail) { + String directory = mail.getMimeInputDirectory().replace("\\", "/"); + + StringBuilder uri = new StringBuilder("file:"); + uri.append(directory); + uri.append("?"); + // File pattern + uri.append("include=").append(mail.getMimeInputPattern()); + // Polling interval + uri.append("&delay=").append(mail.getMimeInputPollInterval()); + // Move to .processed after successful processing + uri.append("&move=.processed"); + // Move to .error on failure + uri.append("&moveFailed=.error"); + // Read lock to prevent processing incomplete files + uri.append("&readLock=changed"); + uri.append("&readLockCheckInterval=1000"); + uri.append("&readLockTimeout=30000"); + // Sort by name for consistent ordering + uri.append("&sortBy=file:name"); + // Don't process hidden files + uri.append("&exclude=^\\..*"); + // Recursive scanning disabled by default + uri.append("&recursive=false"); + + return uri.toString(); + } + + /** + * Build the IMAP URI for the mail consumer. + */ + private String buildImapUri() { + TedProcessorProperties.MailProperties mail = properties.getMail(); + + StringBuilder uri = new StringBuilder(); + uri.append(mail.isSsl() ? "imaps://" : "imap://"); + uri.append(mail.getHost()); + uri.append(":").append(mail.getPort()); + uri.append("?username=").append(encodeUriComponent(mail.getUsername())); + uri.append("&password=").append(encodeUriComponent(mail.getPassword())); + uri.append("&folderName=").append(mail.getFolderName()); + uri.append("&delete=").append(mail.isDelete()); + // peek=false means messages will be marked as SEEN after fetch + // peek=true means messages will NOT be marked as SEEN (peek only) + uri.append("&peek=").append(!mail.isSeen()); + uri.append("&unseen=").append(mail.isUnseen()); + uri.append("&delay=").append(mail.getDelay()); + uri.append("&maxMessagesPerPoll=").append(mail.getMaxMessagesPerPoll()); + // Connection settings + uri.append("&connectionTimeout=30000"); + uri.append("&fetchSize=-1"); // Fetch entire message + uri.append("&debugMode=false"); + + log.info("IMAP URI configured (password hidden): {}://{}:{}?username={}&folderName={}", + mail.isSsl() ? "imaps" : "imap", mail.getHost(), mail.getPort(), + mail.getUsername(), mail.getFolderName()); + + return uri.toString(); + } + + /** + * URL-encode a URI component. + */ + private String encodeUriComponent(String value) { + if (value == null) return ""; + try { + return java.net.URLEncoder.encode(value, StandardCharsets.UTF_8); + } catch (Exception e) { + return value; + } + } + + /** + * Recursively process message content to extract text, HTML, and attachments. + */ + private void processMessageContent(Part part, StringBuilder textContent, + StringBuilder htmlContent, List attachments) throws Exception { + + String contentType = part.getContentType().toLowerCase(); + String disposition = part.getDisposition(); + + // Check if this is an attachment + if (disposition != null && (disposition.equalsIgnoreCase(Part.ATTACHMENT) + || disposition.equalsIgnoreCase(Part.INLINE))) { + extractAttachment(part, attachments); + return; + } + + Object content = part.getContent(); + + if (content instanceof Multipart multipart) { + // Process each part of the multipart message + for (int i = 0; i < multipart.getCount(); i++) { + BodyPart bodyPart = multipart.getBodyPart(i); + processMessageContent(bodyPart, textContent, htmlContent, attachments); + } + } else if (contentType.contains("text/plain")) { + // Plain text content + String text = content.toString(); + textContent.append(text); + } else if (contentType.contains("text/html")) { + // HTML content + String html = content.toString(); + htmlContent.append(html); + } else if (part.getFileName() != null) { + // Has filename - treat as attachment + extractAttachment(part, attachments); + } + } + + /** + * Extract attachment data from a message part. + */ + private void extractAttachment(Part part, List attachments) throws Exception { + String filename = part.getFileName(); + if (filename == null) { + filename = "unnamed_attachment"; + } + + // Decode filename if necessary (might be MIME-encoded) + try { + filename = jakarta.mail.internet.MimeUtility.decodeText(filename); + } catch (Exception e) { + log.debug("Could not decode filename: {}", filename); + } + + String contentType = part.getContentType(); + + // Read attachment data + byte[] data; + try (InputStream is = part.getInputStream()) { + data = is.readAllBytes(); + } + + AttachmentInfo info = new AttachmentInfo(filename, contentType, data, data.length); + attachments.add(info); + + log.debug("Extracted attachment: '{}' ({} bytes, type={})", filename, data.length, contentType); + } + + /** + * Convert HTML content to plain text using JSoup. + */ + private String convertHtmlToText(String html) { + if (html == null || html.isBlank()) { + return ""; + } + + try { + // Parse HTML and extract text + org.jsoup.nodes.Document doc = Jsoup.parse(html); + + // Remove script and style elements + doc.select("script, style").remove(); + + // Get text with whitespace preservation + String text = doc.text(); + + // Clean up excessive whitespace + text = text.replaceAll("\\s+", " ").trim(); + + return text; + } catch (Exception e) { + log.warn("Failed to convert HTML to text: {}", e.getMessage()); + // Fallback: strip HTML tags with regex + return html.replaceAll("<[^>]+>", " ").replaceAll("\\s+", " ").trim(); + } + } + + /** + * DTO for attachment information. + */ + @lombok.Data + @lombok.AllArgsConstructor + public static class AttachmentInfo { + private String filename; + private String contentType; + private byte[] data; + private int size; + } +} diff --git a/src/main/java/at/procon/ted/camel/SolutionBriefRoute.java b/src/main/java/at/procon/ted/camel/SolutionBriefRoute.java new file mode 100644 index 0000000..e1019c1 --- /dev/null +++ b/src/main/java/at/procon/ted/camel/SolutionBriefRoute.java @@ -0,0 +1,180 @@ +package at.procon.ted.camel; + +import at.procon.ted.config.TedProcessorProperties; +import at.procon.ted.service.ExcelExportService; +import at.procon.ted.service.SimilaritySearchService; +import at.procon.ted.service.SimilaritySearchService.SimilaritySearchResponse; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.apache.camel.Exchange; +import org.apache.camel.LoggingLevel; +import org.apache.camel.builder.RouteBuilder; +import org.springframework.stereotype.Component; + +import java.io.File; +import java.nio.file.Paths; + +/** + * Apache Camel route for processing Solution Brief PDF files. + * + * Features: + * - Scans input directory for PDF files + * - Performs semantic similarity search against TED documents + * - Generates Excel (XLSX) reports with hyperlinks to matching tenders + * - Files are NOT moved (noop mode) + * - Idempotent processing to avoid reprocessing same files + * + * @author Martin.Schweitzer@procon.co.at and claude.ai + */ +@Component +@RequiredArgsConstructor +@Slf4j +public class SolutionBriefRoute extends RouteBuilder { + + private static final String ROUTE_ID = "solution-brief-processor"; + + private final TedProcessorProperties properties; + private final SimilaritySearchService similaritySearchService; + private final ExcelExportService excelExportService; + + @Override + public void configure() throws Exception { + TedProcessorProperties.SolutionBriefProperties config = properties.getSolutionBrief(); + + if (!config.isEnabled()) { + log.info("Solution Brief processing is disabled, skipping route configuration"); + return; + } + + // Determine result directory (absolute or relative to input) + String resultDir = config.getResultDirectory(); + if (!Paths.get(resultDir).isAbsolute()) { + resultDir = Paths.get(config.getInputDirectory(), resultDir).toString(); + } + + // Ensure directories exist + File inputDir = new File(config.getInputDirectory()); + File outputDir = new File(resultDir); + if (!inputDir.exists()) { + inputDir.mkdirs(); + log.info("Created Solution Brief input directory: {}", inputDir.getAbsolutePath()); + } + if (!outputDir.exists()) { + outputDir.mkdirs(); + log.info("Created Solution Brief result directory: {}", outputDir.getAbsolutePath()); + } + + final String finalResultDir = resultDir; + + String fileUri = buildFileUri(config); + log.info("=== Solution Brief Route Configuration ==="); + log.info("Input Directory: {}", config.getInputDirectory()); + log.info("Input Directory exists: {}", inputDir.exists()); + log.info("Input Directory is directory: {}", inputDir.isDirectory()); + log.info("Result Directory: {}", resultDir); + log.info("File Pattern: {}", config.getFilePattern()); + log.info("TopK: {}, Threshold: {}", config.getTopK(), config.getSimilarityThreshold()); + log.info("Poll Interval: {}ms", config.getPollInterval()); + log.info("Idempotent: {}", config.isIdempotent()); + log.info("File URI: {}", fileUri); + log.info("==========================================="); + + // List existing PDFs in directory + File[] pdfFiles = inputDir.listFiles((dir, name) -> name.toLowerCase().endsWith(".pdf")); + if (pdfFiles != null && pdfFiles.length > 0) { + log.info("Found {} PDF files in input directory:", pdfFiles.length); + for (File pdf : pdfFiles) { + log.info(" - {} ({} bytes)", pdf.getName(), pdf.length()); + } + } else { + log.warn("No PDF files found in input directory: {}", inputDir.getAbsolutePath()); + } + + // Error handler - handled(false) ensures file goes to .error directory + onException(Exception.class) + .routeId("solution-brief-error-handler") + .handled(false) + .log(LoggingLevel.ERROR, "Solution Brief processing failed for ${header.CamelFileName}: ${exception.message}") + .process(exchange -> { + Exception exception = exchange.getProperty(Exchange.EXCEPTION_CAUGHT, Exception.class); + String filename = exchange.getIn().getHeader(Exchange.FILE_NAME, String.class); + log.error("Error processing Solution Brief '{}': {}", filename, + exception != null ? exception.getMessage() : "Unknown error", exception); + }); + + // Main processing route + from(fileUri) + .routeId(ROUTE_ID) + .log(LoggingLevel.INFO, "Processing Solution Brief PDF: ${header.CamelFileName}") + .process(exchange -> { + String filename = exchange.getIn().getHeader(Exchange.FILE_NAME, String.class); + byte[] pdfData = exchange.getIn().getBody(byte[].class); + + if (pdfData == null || pdfData.length == 0) { + log.warn("Empty PDF file: {}", filename); + exchange.setProperty("skipProcessing", true); + return; + } + + log.info("Searching similar documents for: {} ({} bytes)", filename, pdfData.length); + + try { + // Perform similarity search + SimilaritySearchResponse response = similaritySearchService.searchByPdf( + pdfData, + filename, + config.getTopK(), + config.getSimilarityThreshold() + ); + + if (response.getResults().isEmpty()) { + log.info("No similar documents found for: {}", filename); + exchange.setProperty("noResults", true); + return; + } + + log.info("Found {} similar documents for: {}", response.getResultCount(), filename); + + // Export to Excel + String excelPath = excelExportService.exportToExcel( + response, + filename, + finalResultDir + ); + + exchange.getIn().setHeader("excelOutputPath", excelPath); + exchange.getIn().setHeader("resultCount", response.getResultCount()); + + log.info("Excel report generated: {} ({} results)", excelPath, response.getResultCount()); + + } catch (Exception e) { + log.error("Failed to process Solution Brief '{}': {}", filename, e.getMessage(), e); + throw e; + } + }) + .choice() + .when(exchangeProperty("skipProcessing").isEqualTo(true)) + .log(LoggingLevel.WARN, "Skipped empty PDF: ${header.CamelFileName}") + .when(exchangeProperty("noResults").isEqualTo(true)) + .log(LoggingLevel.INFO, "No similar documents found for: ${header.CamelFileName}") + .otherwise() + .log(LoggingLevel.INFO, "Solution Brief completed: ${header.CamelFileName} -> ${header.excelOutputPath} (${header.resultCount} results)") + .end(); + } + + /** + * Build the file URI for the Solution Brief consumer. + */ + private String buildFileUri(TedProcessorProperties.SolutionBriefProperties config) { + String directory = config.getInputDirectory().replace("\\", "/"); + + StringBuilder uri = new StringBuilder("file:"); + uri.append(directory); + uri.append("?includeExt=pdf"); + uri.append("&delay=").append(config.getPollInterval()); + uri.append("&move=.done"); + uri.append("&moveFailed=.error"); + + return uri.toString(); + } +} diff --git a/src/main/java/at/procon/ted/camel/TedDocumentRoute.java b/src/main/java/at/procon/ted/camel/TedDocumentRoute.java new file mode 100644 index 0000000..7920b3c --- /dev/null +++ b/src/main/java/at/procon/ted/camel/TedDocumentRoute.java @@ -0,0 +1,164 @@ +package at.procon.ted.camel; + +import at.procon.ted.config.TedProcessorProperties; +import at.procon.ted.service.DocumentProcessingService; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.apache.camel.Exchange; +import org.apache.camel.LoggingLevel; +import org.apache.camel.builder.RouteBuilder; +import org.springframework.stereotype.Component; + +import java.nio.charset.StandardCharsets; +import java.nio.file.Path; + +/** + * Apache Camel route for watching and processing TED XML documents. + * + * Features: + * - Recursive directory scanning for *.xml files + * - File locking to prevent concurrent processing + * - Move to .processed or .error directories after processing + * - Error handling with retry and dead letter channel + * + * @author Martin.Schweitzer@procon.co.at and claude.ai + */ +@Component +@RequiredArgsConstructor +@Slf4j +public class TedDocumentRoute extends RouteBuilder { + + private static final String ROUTE_ID = "ted-document-processor"; + private static final String ROUTE_ID_PROCESSOR = "ted-document-processor-handler"; + + private final TedProcessorProperties properties; + private final DocumentProcessingService documentProcessingService; + + @Override + public void configure() throws Exception { + // Error handler configuration + errorHandler(deadLetterChannel("direct:error-handler") + .maximumRedeliveries(3) + .redeliveryDelay(1000) + .retryAttemptedLogLevel(LoggingLevel.WARN) + .logStackTrace(true) + .logRetryAttempted(true) + .logExhausted(true)); + + // Error handler route + from("direct:error-handler") + .routeId("ted-error-handler") + .process(exchange -> { + Exception exception = exchange.getProperty(Exchange.EXCEPTION_CAUGHT, Exception.class); + if (exception != null) { + log.debug("Document processing error", exception); + } + }) + .to(buildErrorUri()); + + // Main file processing route - DISABLED + // File consumer disabled to prevent memory leak with package download route + // The package download route processes XML files directly after extraction + // from(buildFileUri()) + // from("file://{{ted.input.directory}}") + // .routeId(ROUTE_ID) + // .to("direct:process-document"); + + // Document processing sub-route + from("direct:process-document") + .routeId(ROUTE_ID_PROCESSOR) + .process(exchange -> { + // Extract file information + String filename = exchange.getIn().getHeader(Exchange.FILE_NAME, String.class); + String absolutePath = exchange.getIn().getHeader(Exchange.FILE_PATH, String.class); + Long fileSize = exchange.getIn().getHeader(Exchange.FILE_LENGTH, Long.class); + + // Read XML content + byte[] body = exchange.getIn().getBody(byte[].class); + String xmlContent = new String(body, StandardCharsets.UTF_8); + + log.debug("Processing file: {} ({} bytes)", filename, fileSize); + + // Process the document + DocumentProcessingService.ProcessingResult result = + documentProcessingService.processDocument(xmlContent, filename, absolutePath, fileSize); + + // Set result in exchange for logging + exchange.setProperty("processingResult", result); + + if (result.isError()) { + throw new RuntimeException("Document processing failed: " + result.errorMessage()); + } + }) + .choice() + .when(exchange -> { + DocumentProcessingService.ProcessingResult result = + exchange.getProperty("processingResult", DocumentProcessingService.ProcessingResult.class); + return result != null && result.isDuplicate(); + }) + // Move duplicate to processed directory (it's not an error) + .to(buildProcessedUri()) + .otherwise() + // Vectorization is already triggered in DocumentProcessingService after DB save + .to(buildProcessedUri()) + .end(); + } + + /** + * Build the file component URI for watching the input directory. + */ + private String buildFileUri() { + TedProcessorProperties.InputProperties input = properties.getInput(); + + // Normalize path for Camel (convert backslashes to forward slashes) + String directory = input.getDirectory().replace("\\", "/"); + + StringBuilder uri = new StringBuilder("file:"); + uri.append(directory); + + uri.append("?"); + // Recursive scanning only if pattern contains ** + boolean recursive = input.getPattern().contains("**"); + uri.append("recursive=").append(recursive); + // File pattern (always use antInclude for Ant-style patterns like *.xml) + uri.append("&antInclude=").append(input.getPattern()); + // Polling configuration + uri.append("&delay=").append(input.getPollInterval()); + uri.append("&maxMessagesPerPoll=").append(input.getMaxMessagesPerPoll()); + // Read lock strategy to prevent processing incomplete files + uri.append("&readLock=changed"); + uri.append("&readLockCheckInterval=1000"); + uri.append("&readLockTimeout=30000"); + // Move files after processing + uri.append("&move=").append(input.getProcessedDirectory()); + uri.append("&moveFailed=").append(input.getErrorDirectory()); + // Sort by name for consistent ordering + uri.append("&sortBy=file:name"); + // Don't process hidden files + uri.append("&exclude=.*"); + // Max depth only for recursive scanning + if (recursive) { + uri.append("&maxDepth=10"); + } + + log.info("File consumer URI: {}", uri); + return uri.toString(); + } + + /** + * Build URI for successfully processed files. + */ + private String buildProcessedUri() { + // Files are automatically moved by the file component's 'move' option + // This is a no-op endpoint used for explicit routing + return "log:ted-processed?level=DEBUG"; + } + + /** + * Build URI for failed files. + */ + private String buildErrorUri() { + // Files are automatically moved by the file component's 'moveFailed' option + return "log:ted-error?level=ERROR"; + } +} diff --git a/src/main/java/at/procon/ted/camel/TedPackageDownloadCamelRoute.java b/src/main/java/at/procon/ted/camel/TedPackageDownloadCamelRoute.java new file mode 100644 index 0000000..4eabe9e --- /dev/null +++ b/src/main/java/at/procon/ted/camel/TedPackageDownloadCamelRoute.java @@ -0,0 +1,635 @@ +package at.procon.ted.camel; + +import at.procon.ted.config.TedProcessorProperties; +import at.procon.ted.model.entity.TedDailyPackage; +import at.procon.ted.repository.TedDailyPackageRepository; +import at.procon.ted.service.BatchDocumentProcessingService; +import at.procon.ted.service.TedPackageDownloadService; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.apache.camel.Exchange; +import org.apache.camel.LoggingLevel; +import org.apache.camel.builder.RouteBuilder; +import org.apache.commons.compress.archivers.tar.TarArchiveEntry; +import org.apache.commons.compress.archivers.tar.TarArchiveInputStream; +import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.stereotype.Component; + +import java.io.*; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.security.MessageDigest; +import java.time.OffsetDateTime; +import java.time.Year; +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; + +/** + * Camel-Native Route for automatic download of TED Daily Packages. + * + * Uses Camel HTTP Component for downloads and Enterprise Integration Patterns: + * - Timer-based triggers + * - Idempotent Consumer Pattern + * - Content-Based Router + * - Splitter Pattern + * - Dead Letter Channel + * + * @author Martin.Schweitzer@procon.co.at and claude.ai + */ +@Component +@ConditionalOnProperty(name = "ted.download.enabled", havingValue = "true") +@RequiredArgsConstructor +@Slf4j +public class TedPackageDownloadCamelRoute extends RouteBuilder { + + private static final String ROUTE_ID_SCHEDULER = "ted-package-scheduler"; + private static final String ROUTE_ID_DOWNLOADER = "ted-package-http-downloader"; + private static final String ROUTE_ID_EXTRACTOR = "ted-package-extractor"; + private static final String ROUTE_ID_XML_SPLITTER = "ted-package-xml-splitter"; + + private final TedProcessorProperties properties; + private final TedDailyPackageRepository packageRepository; + private final TedPackageDownloadService downloadService; + private final BatchDocumentProcessingService batchProcessingService; + + /** + * Creates thread pool for parallel XML processing. + * Maximum 1 parallel process to prevent OutOfMemory. + */ + private java.util.concurrent.ExecutorService executorService() { + return java.util.concurrent.Executors.newFixedThreadPool( + 1, + r -> { + Thread thread = new Thread(r); + thread.setName("ted-xml-processor-" + thread.getId()); + thread.setDaemon(true); + thread.setPriority(Thread.NORM_PRIORITY - 1); + return thread; + } + ); + } + + @Override + public void configure() throws Exception { + + // Error Handler with Dead Letter Channel + errorHandler(deadLetterChannel("direct:package-download-error") + .maximumRedeliveries(3) + .redeliveryDelay(10000) + .retryAttemptedLogLevel(LoggingLevel.WARN) + .logStackTrace(true)); + + // Error Handler Route + from("direct:package-download-error") + .routeId("ted-package-error-handler") + .log(LoggingLevel.ERROR, "Failed to process package: ${exception.message}") + .process(this::handleDownloadError); + + // Timer-based Scheduler (starts immediately with delay=0) + from("timer:ted-package-scheduler?period={{ted.download.poll-interval:120000}}&delay=0") + .routeId(ROUTE_ID_SCHEDULER) + .autoStartup(true) + .log(LoggingLevel.INFO, "TED Package Scheduler: Checking for new packages...") + .process(this::checkRunningPackages) + .choice() + .when(header("tooManyRunning").isEqualTo(true)) + .log(LoggingLevel.INFO, "Skipping download - already ${header.runningCount} packages in progress (max 2)") + .otherwise() + .process(this::determineNextPackage) + .choice() + .when(header("packageId").isNotNull()) + .to("direct:download-package") + .otherwise() + .log(LoggingLevel.INFO, "No more packages to download - all complete") + .end() + .end(); + + // Package Download Route (HTTP) + from("direct:download-package") + .routeId(ROUTE_ID_DOWNLOADER) + .log(LoggingLevel.INFO, "Starting download of package ${header.packageId}") + .setHeader("downloadStartTime", constant(System.currentTimeMillis())) + // Check if already downloaded + .process(this::createPackageRecord) + // Rate Limiting + .delay(simple("{{ted.download.delay-between-downloads:5000}}")) + // HTTP Download + .setHeader(Exchange.HTTP_METHOD, constant("GET")) + .setHeader("CamelHttpConnectionClose", constant(true)) + .toD("${header.downloadUrl}?bridgeEndpoint=true&throwExceptionOnFailure=false&socketTimeout={{ted.download.download-timeout:300000}}") + .choice() + // HTTP 200: Success + .when(header(Exchange.HTTP_RESPONSE_CODE).isEqualTo(200)) + .log(LoggingLevel.INFO, "Processing package ${header.packageId}") + .to("direct:process-downloaded-package") + // HTTP 404: Not Found + .when(header(Exchange.HTTP_RESPONSE_CODE).isEqualTo(404)) + .log(LoggingLevel.DEBUG, "Package not found (404): ${header.packageId}") + .process(this::markPackageNotFound) + // Other HTTP errors + .otherwise() + .log(LoggingLevel.WARN, "HTTP ${header.CamelHttpResponseCode} for package ${header.packageId}") + .process(this::markPackageFailed) + .end(); + + // Downloaded Package Processing + from("direct:process-downloaded-package") + .routeId("ted-package-processor") + .process(this::calculateHash) + .process(this::checkDuplicateByHash) + .choice() + .when(header("isDuplicate").isEqualTo(true)) + .log(LoggingLevel.WARN, "Duplicate package detected via hash: ${header.packageId}") + .process(this::markPackageDuplicate) + .otherwise() + .process(this::saveDownloadedPackage) + .to("direct:extract-tar-gz") + .end(); + + // tar.gz Extraction Route + from("direct:extract-tar-gz") + .routeId(ROUTE_ID_EXTRACTOR) + .log(LoggingLevel.DEBUG, "Extracting package ${header.packageId}...") + .process(this::extractTarGz) + .choice() + .when(header("deleteAfterExtraction").isEqualTo(true)) + .log(LoggingLevel.DEBUG, "Deleting tar.gz: ${header.downloadPath}") + .process(this::deleteTarGz) + .end() + .to("direct:split-xml-files"); + + // XML Files Batch Processor + from("direct:split-xml-files") + .routeId(ROUTE_ID_XML_SPLITTER) + .process(this::updatePackageProcessing) + .setHeader("processingStartTime", constant(System.currentTimeMillis())) + .process(this::processBatchDocuments) + .process(this::markPackageCompleted) + .process(this::logPackageStatistics); + } + + /** + * Checks how many packages are currently being processed. + * Resets stuck packages (older than 30 minutes) to PENDING. + * Sets header "tooManyRunning" to true if 3 or more packages are in progress. + */ + private void checkRunningPackages(Exchange exchange) { + OffsetDateTime thirtyMinutesAgo = OffsetDateTime.now().minusMinutes(30); + + // Find stuck DOWNLOADING packages + List stuckDownloading = packageRepository.findByDownloadStatus( + TedDailyPackage.DownloadStatus.DOWNLOADING).stream() + .filter(pkg -> pkg.getUpdatedAt() != null && pkg.getUpdatedAt().isBefore(thirtyMinutesAgo)) + .toList(); + + // Find stuck PROCESSING packages + List stuckProcessing = packageRepository.findByDownloadStatus( + TedDailyPackage.DownloadStatus.PROCESSING).stream() + .filter(pkg -> pkg.getUpdatedAt() != null && pkg.getUpdatedAt().isBefore(thirtyMinutesAgo)) + .toList(); + + List stuckPackages = new ArrayList<>(); + stuckPackages.addAll(stuckDownloading); + stuckPackages.addAll(stuckProcessing); + + if (!stuckPackages.isEmpty()) { + log.warn("Found {} stuck packages (older than 30 minutes), resetting to PENDING", stuckPackages.size()); + stuckPackages.forEach(pkg -> { + log.warn("Resetting stuck package: {} (status: {}, last update: {})", + pkg.getPackageIdentifier(), pkg.getDownloadStatus(), pkg.getUpdatedAt()); + pkg.setDownloadStatus(TedDailyPackage.DownloadStatus.PENDING); + pkg.setErrorMessage("Reset from stuck state"); + packageRepository.save(pkg); + }); + } + + // Count currently running packages (after reset) + long downloadingCount = packageRepository.findByDownloadStatus( + TedDailyPackage.DownloadStatus.DOWNLOADING).size(); + long processingCount = packageRepository.findByDownloadStatus( + TedDailyPackage.DownloadStatus.PROCESSING).size(); + long runningCount = downloadingCount + processingCount; + + exchange.getIn().setHeader("runningCount", runningCount); + exchange.getIn().setHeader("tooManyRunning", runningCount >= 2); + + if (runningCount > 0) { + log.info("Currently {} packages in progress ({} downloading, {} processing)", + runningCount, downloadingCount, processingCount); + } + } + + /** + * Determines the next package to download. + * Strategy: + * 1. First check for PENDING packages (previously failed/stuck) + * 2. Then use download service strategy: + * - Current year: Forward from max(nr) until 404 + * - All years: Fill gaps (if min(nr) > 1, then backward to 1) + * - If current year complete (min=1 and 404 after max) -> previous year + * - Repeat until startYear + */ + private void determineNextPackage(Exchange exchange) { + // First check for PENDING packages + List pendingPackages = packageRepository.findByDownloadStatus( + TedDailyPackage.DownloadStatus.PENDING); + + if (!pendingPackages.isEmpty()) { + TedDailyPackage pkg = pendingPackages.get(0); + log.info("Retrying PENDING package: {}", pkg.getPackageIdentifier()); + setPackageHeaders(exchange, pkg.getYear(), pkg.getSerialNumber()); + return; + } + + // Use download service to find next package + TedPackageDownloadService.PackageInfo packageInfo = downloadService.getNextPackageToDownload(); + + if (packageInfo == null) { + // No more packages + exchange.getIn().setHeader("packageId", null); + return; + } + + setPackageHeaders(exchange, packageInfo.year(), packageInfo.serialNumber()); + } + + /** + * Sets package headers for download. + */ + private void setPackageHeaders(Exchange exchange, int year, int serialNumber) { + String packageId = String.format("%04d%05d", year, serialNumber); + String downloadUrl = properties.getDownload().getBaseUrl() + packageId; + + exchange.getIn().setHeader("packageId", packageId); + exchange.getIn().setHeader("year", year); + exchange.getIn().setHeader("serialNumber", serialNumber); + exchange.getIn().setHeader("downloadUrl", downloadUrl); + } + + /** + * Creates package record in DB. + */ + private void createPackageRecord(Exchange exchange) { + String packageId = exchange.getIn().getHeader("packageId", String.class); + Integer year = exchange.getIn().getHeader("year", Integer.class); + Integer serialNumber = exchange.getIn().getHeader("serialNumber", Integer.class); + String downloadUrl = exchange.getIn().getHeader("downloadUrl", String.class); + + Optional existing = packageRepository.findByPackageIdentifier(packageId); + if (existing.isPresent()) { + TedDailyPackage pkg = existing.get(); + if (pkg.getDownloadStatus() == TedDailyPackage.DownloadStatus.NOT_FOUND) { + log.info("Retrying existing NOT_FOUND package in Camel route: {}", packageId); + pkg.setDownloadUrl(downloadUrl); + pkg.setErrorMessage(null); + pkg.setDownloadStatus(TedDailyPackage.DownloadStatus.DOWNLOADING); + pkg = packageRepository.save(pkg); + exchange.getIn().setHeader("packageDbId", pkg.getId()); + return; + } + + log.debug("Package {} already exists in DB with status {}", packageId, pkg.getDownloadStatus()); + exchange.getIn().setHeader("packageDbId", pkg.getId()); + return; + } + + TedDailyPackage pkg = TedDailyPackage.builder() + .packageIdentifier(packageId) + .year(year) + .serialNumber(serialNumber) + .downloadUrl(downloadUrl) + .downloadStatus(TedDailyPackage.DownloadStatus.DOWNLOADING) + .build(); + + pkg = packageRepository.save(pkg); + exchange.getIn().setHeader("packageDbId", pkg.getId()); + } + + /** + * Calculates SHA-256 hash. + */ + private void calculateHash(Exchange exchange) throws Exception { + byte[] body = exchange.getIn().getBody(byte[].class); + + MessageDigest digest = MessageDigest.getInstance("SHA-256"); + byte[] hashBytes = digest.digest(body); + + StringBuilder sb = new StringBuilder(); + for (byte b : hashBytes) { + sb.append(String.format("%02x", b)); + } + + String hash = sb.toString(); + exchange.getIn().setHeader("fileHash", hash); + log.debug("Calculated hash: {}", hash); + } + + /** + * Checks for duplicate via hash. + */ + private void checkDuplicateByHash(Exchange exchange) { + String hash = exchange.getIn().getHeader("fileHash", String.class); + + Optional duplicate = packageRepository.findAll().stream() + .filter(p -> hash.equals(p.getFileHash())) + .findFirst(); + + exchange.getIn().setHeader("isDuplicate", duplicate.isPresent()); + if (duplicate.isPresent()) { + exchange.getIn().setHeader("duplicateOf", duplicate.get().getPackageIdentifier()); + } + } + + /** + * Saves downloaded package. + */ + private void saveDownloadedPackage(Exchange exchange) throws IOException { + String packageId = exchange.getIn().getHeader("packageId", String.class); + String hash = exchange.getIn().getHeader("fileHash", String.class); + byte[] body = exchange.getIn().getBody(byte[].class); + + // Save tar.gz + Path downloadDir = Paths.get(properties.getDownload().getDownloadDirectory()); + Files.createDirectories(downloadDir); + Path downloadPath = downloadDir.resolve(packageId + ".tar.gz"); + Files.write(downloadPath, body); + + long downloadDuration = System.currentTimeMillis() - + exchange.getIn().getHeader("downloadStartTime", Long.class); + + // Update DB + packageRepository.findByPackageIdentifier(packageId).ifPresent(pkg -> { + pkg.setFileHash(hash); + pkg.setDownloadStatus(TedDailyPackage.DownloadStatus.DOWNLOADED); + pkg.setDownloadedAt(OffsetDateTime.now()); + pkg.setDownloadDurationMs(downloadDuration); + packageRepository.save(pkg); + }); + + exchange.getIn().setHeader("downloadPath", downloadPath.toString()); + exchange.getIn().setHeader("deleteAfterExtraction", + properties.getDownload().isDeleteAfterExtraction()); + } + + /** + * Extracts tar.gz. + */ + private void extractTarGz(Exchange exchange) throws IOException { + String packageId = exchange.getIn().getHeader("packageId", String.class); + String downloadPath = exchange.getIn().getHeader("downloadPath", String.class); + + Path tarGzFile = Paths.get(downloadPath); + Path extractDir = Paths.get(properties.getDownload().getExtractDirectory()) + .resolve(packageId); + Files.createDirectories(extractDir); + + List xmlFiles = new ArrayList<>(); + + try (FileInputStream fis = new FileInputStream(tarGzFile.toFile()); + GzipCompressorInputStream gzis = new GzipCompressorInputStream(fis); + TarArchiveInputStream tais = new TarArchiveInputStream(gzis)) { + + TarArchiveEntry entry; + while ((entry = tais.getNextTarEntry()) != null) { + if (entry.isDirectory()) { + continue; + } + + String name = entry.getName(); + if (!name.toLowerCase().endsWith(".xml")) { + continue; + } + + Path outputPath = extractDir.resolve(new File(name).getName()); + + try (OutputStream os = Files.newOutputStream(outputPath)) { + byte[] buffer = new byte[8192]; + int read; + while ((read = tais.read(buffer)) > 0) { + os.write(buffer, 0, read); + } + } + + xmlFiles.add(outputPath); + } + } + + exchange.getIn().setHeader("xmlFiles", xmlFiles); + exchange.getIn().setHeader("xmlFileCount", xmlFiles.size()); + + // Update DB + packageRepository.findByPackageIdentifier(packageId).ifPresent(pkg -> { + pkg.setXmlFileCount(xmlFiles.size()); + packageRepository.save(pkg); + }); + + log.debug("Extracted {} XML files from package {}", xmlFiles.size(), packageId); + } + + /** + * Deletes tar.gz. + */ + private void deleteTarGz(Exchange exchange) throws IOException { + String downloadPath = exchange.getIn().getHeader("downloadPath", String.class); + Files.deleteIfExists(Paths.get(downloadPath)); + } + + /** + * Process XML files in chunks to avoid connection leaks. + * Splits large packages into batches of 100 files. + */ + @SuppressWarnings("unchecked") + private void processBatchDocuments(Exchange exchange) { + String packageId = exchange.getIn().getHeader("packageId", String.class); + List xmlFiles = (List) exchange.getIn().getHeader("xmlFiles"); + + log.info("Package {}: Processing {} XML files in batches of 25", packageId, xmlFiles.size()); + + int totalInserted = 0; + int totalDuplicates = 0; + int totalErrors = 0; + long totalDuration = 0; + + // Process in chunks of 25 to avoid connection leaks (must complete in <60s) + int chunkSize = 25; + for (int i = 0; i < xmlFiles.size(); i += chunkSize) { + int end = Math.min(i + chunkSize, xmlFiles.size()); + List chunk = xmlFiles.subList(i, end); + + log.debug("Package {}: Processing chunk {}-{} of {}", + packageId, i + 1, end, xmlFiles.size()); + + // Process chunk in one transaction + BatchDocumentProcessingService.BatchProcessingResult result = + batchProcessingService.processBatch(chunk); + + totalInserted += result.insertedCount(); + totalDuplicates += result.duplicateCount(); + totalErrors += result.errorCount(); + totalDuration += result.durationMs(); + + // Update package statistics after each chunk + updatePackageStatistics(packageId, totalInserted + totalDuplicates, totalErrors); + } + + // Store final result in exchange for logging + exchange.setProperty("processedCount", totalInserted + totalDuplicates); + exchange.setProperty("failedCount", totalErrors); + + log.info("Package {}: Batch processing completed - {} inserted, {} duplicates, {} errors in {}ms", + packageId, totalInserted, totalDuplicates, totalErrors, totalDuration); + } + + /** + * Update package processing statistics. + */ + private void updatePackageStatistics(String packageId, int processedCount, int failedCount) { + packageRepository.findByPackageIdentifier(packageId).ifPresent(pkg -> { + pkg.setProcessedCount(processedCount); + pkg.setFailedCount(failedCount); + packageRepository.save(pkg); + }); + } + + /** + * Updates status to PROCESSING. + */ + private void updatePackageProcessing(Exchange exchange) { + String packageId = exchange.getIn().getHeader("packageId", String.class); + packageRepository.findByPackageIdentifier(packageId).ifPresent(pkg -> { + pkg.setDownloadStatus(TedDailyPackage.DownloadStatus.PROCESSING); + packageRepository.save(pkg); + }); + } + + /** + * Marks package as COMPLETED and cleans up extracted XML files. + */ + private void markPackageCompleted(Exchange exchange) { + String packageId = exchange.getIn().getHeader("packageId", String.class); + Long processingStartTime = exchange.getIn().getHeader("processingStartTime", Long.class); + + packageRepository.findByPackageIdentifier(packageId).ifPresent(pkg -> { + pkg.setDownloadStatus(TedDailyPackage.DownloadStatus.COMPLETED); + pkg.setProcessedAt(OffsetDateTime.now()); + + if (processingStartTime != null) { + long processingDuration = System.currentTimeMillis() - processingStartTime; + pkg.setProcessingDurationMs(processingDuration); + } + + packageRepository.save(pkg); + }); + + // Clean up extracted XML files and package directory to free memory + List xmlFiles = exchange.getIn().getHeader("xmlFiles", List.class); + if (xmlFiles != null) { + int deletedCount = 0; + for (Path xmlFile : xmlFiles) { + try { + if (Files.deleteIfExists(xmlFile)) { + deletedCount++; + } + } catch (IOException e) { + log.warn("Failed to delete XML file {}: {}", xmlFile, e.getMessage()); + } + } + + // Delete package directory if empty + if (!xmlFiles.isEmpty()) { + try { + Path packageDir = xmlFiles.get(0).getParent(); + if (packageDir != null && Files.isDirectory(packageDir)) { + try (var stream = Files.list(packageDir)) { + if (stream.findAny().isEmpty()) { + Files.deleteIfExists(packageDir); + log.debug("Deleted empty package directory: {}", packageDir); + } + } + } + } catch (IOException e) { + log.debug("Could not delete package directory: {}", e.getMessage()); + } + } + + if (deletedCount > 0) { + log.debug("Cleaned up {} XML files for package {}", deletedCount, packageId); + } + } + } + + /** + * Marks package as NOT_FOUND. + */ + private void markPackageNotFound(Exchange exchange) { + String packageId = exchange.getIn().getHeader("packageId", String.class); + packageRepository.findByPackageIdentifier(packageId).ifPresent(pkg -> { + pkg.setDownloadStatus(TedDailyPackage.DownloadStatus.NOT_FOUND); + pkg.setErrorMessage("Package not found (404)"); + packageRepository.save(pkg); + }); + } + + /** + * Marks package as FAILED. + */ + private void markPackageFailed(Exchange exchange) { + String packageId = exchange.getIn().getHeader("packageId", String.class); + Integer httpCode = exchange.getIn().getHeader(Exchange.HTTP_RESPONSE_CODE, Integer.class); + packageRepository.findByPackageIdentifier(packageId).ifPresent(pkg -> { + pkg.setDownloadStatus(TedDailyPackage.DownloadStatus.FAILED); + pkg.setErrorMessage("HTTP " + httpCode); + packageRepository.save(pkg); + }); + } + + /** + * Marks package as DUPLICATE. + */ + private void markPackageDuplicate(Exchange exchange) { + String packageId = exchange.getIn().getHeader("packageId", String.class); + String duplicateOf = exchange.getIn().getHeader("duplicateOf", String.class); + packageRepository.findByPackageIdentifier(packageId).ifPresent(pkg -> { + pkg.setDownloadStatus(TedDailyPackage.DownloadStatus.COMPLETED); + pkg.setErrorMessage("Duplicate of " + duplicateOf); + packageRepository.save(pkg); + }); + } + + /** + * Logs package processing statistics. + */ + private void logPackageStatistics(Exchange exchange) { + String packageId = exchange.getIn().getHeader("packageId", String.class); + packageRepository.findByPackageIdentifier(packageId).ifPresent(pkg -> { + long totalDuration = (pkg.getDownloadDurationMs() != null ? pkg.getDownloadDurationMs() : 0) + + (pkg.getProcessingDurationMs() != null ? pkg.getProcessingDurationMs() : 0); + + log.info("Package {} completed: {} XML files, {} processed, {} failed, total duration: {}ms", + packageId, + pkg.getXmlFileCount(), + pkg.getProcessedCount(), + pkg.getFailedCount(), + totalDuration); + }); + } + + /** + * Error Handler. + */ + private void handleDownloadError(Exchange exchange) { + Exception exception = exchange.getProperty(Exchange.EXCEPTION_CAUGHT, Exception.class); + String packageId = exchange.getIn().getHeader("packageId", String.class); + + if (packageId != null) { + packageRepository.findByPackageIdentifier(packageId).ifPresent(pkg -> { + pkg.setDownloadStatus(TedDailyPackage.DownloadStatus.FAILED); + pkg.setErrorMessage(exception != null ? exception.getMessage() : "Unknown error"); + packageRepository.save(pkg); + }); + } + } +} diff --git a/src/main/java/at/procon/ted/camel/TedPackageDownloadRoute.java b/src/main/java/at/procon/ted/camel/TedPackageDownloadRoute.java new file mode 100644 index 0000000..fda70ad --- /dev/null +++ b/src/main/java/at/procon/ted/camel/TedPackageDownloadRoute.java @@ -0,0 +1,158 @@ +package at.procon.ted.camel; + +import at.procon.ted.config.TedProcessorProperties; +import at.procon.ted.service.TedPackageDownloadService; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.apache.camel.Exchange; +import org.apache.camel.LoggingLevel; +import org.apache.camel.builder.RouteBuilder; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.stereotype.Component; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; + +/** + * Apache Camel Route for automatic download of TED Daily Packages. + * + * Features: + * - Scheduled Download (hourly configurable) + * - Idempotent Downloads via Hash + * - Rate Limiting + * - Integration with XML processing + * - Error Handling + * + * @author Martin.Schweitzer@procon.co.at and claude.ai + */ +@Component +@ConditionalOnProperty(name = "ted.download.use-service-based", havingValue = "true") +@RequiredArgsConstructor +@Slf4j +public class TedPackageDownloadRoute extends RouteBuilder { + + private static final String ROUTE_ID_SCHEDULER = "ted-package-download-scheduler"; + private static final String ROUTE_ID_DOWNLOADER = "ted-package-downloader"; + private static final String ROUTE_ID_XML_PROCESSOR = "ted-package-xml-processor"; + + private final TedProcessorProperties properties; + private final TedPackageDownloadService downloadService; + + /** + * Creates thread pool for parallel XML processing. + * Maximum 3 parallel processes for DB loading (lower priority due to vectorization). + */ + private java.util.concurrent.ExecutorService executorService() { + return java.util.concurrent.Executors.newFixedThreadPool( + 3, + r -> { + Thread thread = new Thread(r); + thread.setName("ted-xml-processor-" + thread.getId()); + thread.setDaemon(true); + thread.setPriority(Thread.NORM_PRIORITY - 1); // Lower priority than vectorization + return thread; + } + ); + } + + @Override + public void configure() throws Exception { + + // Error Handler + errorHandler(defaultErrorHandler() + .maximumRedeliveries(3) + .redeliveryDelay(5000) + .retryAttemptedLogLevel(LoggingLevel.WARN)); + + // Scheduler: Periodic check for new packages + from("timer:package-download-scheduler?period={{ted.download.poll-interval:3600000}}") + .routeId(ROUTE_ID_SCHEDULER) + .autoStartup("{{ted.download.enabled:false}}") + .log(LoggingLevel.DEBUG, "Checking for new TED packages to download...") + .bean(downloadService, "getNextPackageToDownload") + .choice() + .when(body().isNull()) + .log(LoggingLevel.DEBUG, "No more packages to download") + .otherwise() + .to("direct:download-package") + .end(); + + // Download Route + from("direct:download-package") + .routeId(ROUTE_ID_DOWNLOADER) + .log(LoggingLevel.INFO, "Processing package: ${body.identifier}") + .process(exchange -> { + TedPackageDownloadService.PackageInfo packageInfo = + exchange.getIn().getBody(TedPackageDownloadService.PackageInfo.class); + + // Rate Limiting + long delay = properties.getDownload().getDelayBetweenDownloads(); + if (delay > 0) { + Thread.sleep(delay); + } + + // Download Package + TedPackageDownloadService.DownloadResult result = + downloadService.downloadPackage(packageInfo.year(), packageInfo.serialNumber()); + + exchange.setProperty("downloadResult", result); + exchange.getIn().setBody(result); + }) + .choice() + .when(simple("${exchangeProperty.downloadResult.success} == true")) + .to("direct:process-package-xml-files") + .when(simple("${exchangeProperty.downloadResult.status.name} == 'NOT_FOUND'")) + .log(LoggingLevel.DEBUG, "Package not found (404): ${body.packageEntity.packageIdentifier}") + .when(simple("${exchangeProperty.downloadResult.status.name} == 'ALREADY_EXISTS'")) + .log(LoggingLevel.DEBUG, "Package already exists: ${body.packageEntity.packageIdentifier}") + .when(simple("${exchangeProperty.downloadResult.status.name} == 'DUPLICATE'")) + .log(LoggingLevel.WARN, "Duplicate package detected: ${body.packageEntity.packageIdentifier}") + .otherwise() + .log(LoggingLevel.ERROR, "Failed to download package: ${exchangeProperty.downloadResult.error.message}") + .end(); + + // XML Files Processing Route + from("direct:process-package-xml-files") + .routeId(ROUTE_ID_XML_PROCESSOR) + .setProperty("processedCount", constant(0)) + .setProperty("failedCount", constant(0)) + .setProperty("packageIdentifier", simple("${body.packageEntity.packageIdentifier}")) + .setProperty("xmlFileCount", simple("${body.xmlFiles.size}")) + .split(simple("${body.xmlFiles}")) + .parallelProcessing() + .executorService(executorService()) + .stopOnException(false) // Continue even if individual documents fail + .shareUnitOfWork() + .doTry() + .process(exchange -> { + Path xmlFile = exchange.getIn().getBody(Path.class); + + // Set headers for existing XML processing route + exchange.getIn().setHeader(Exchange.FILE_NAME, xmlFile.getFileName().toString()); + exchange.getIn().setHeader(Exchange.FILE_PATH, xmlFile.toString()); + exchange.getIn().setHeader(Exchange.FILE_LENGTH, Files.size(xmlFile)); + + // Read XML content + byte[] content = Files.readAllBytes(xmlFile); + exchange.getIn().setBody(content); + }) + // Forward to existing processing route + .to("direct:process-document") + .process(exchange -> { + // Increment success counter + Integer count = exchange.getProperty("processedCount", Integer.class); + exchange.setProperty("processedCount", count + 1); + }) + .doCatch(Exception.class) + .log(LoggingLevel.WARN, "Failed to process ${header.CamelFileName}: ${exception.message}") + .process(exchange -> { + // Increment error counter + Integer count = exchange.getProperty("failedCount", Integer.class); + exchange.setProperty("failedCount", count + 1); + }) + .end() + .end() + .log(LoggingLevel.INFO, "Package ${exchangeProperty.packageIdentifier} completed: ${exchangeProperty.xmlFileCount} XML files, ${exchangeProperty.processedCount} processed, ${exchangeProperty.failedCount} failed"); + } +} diff --git a/src/main/java/at/procon/ted/camel/VectorizationRoute.java b/src/main/java/at/procon/ted/camel/VectorizationRoute.java new file mode 100644 index 0000000..40203af --- /dev/null +++ b/src/main/java/at/procon/ted/camel/VectorizationRoute.java @@ -0,0 +1,360 @@ +package at.procon.ted.camel; + +import at.procon.ted.config.TedProcessorProperties; +import at.procon.ted.model.entity.ProcurementDocument; +import at.procon.ted.model.entity.VectorizationStatus; +import at.procon.ted.repository.ProcurementDocumentRepository; +import at.procon.ted.service.VectorizationProcessorService; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.ObjectMapper; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.apache.camel.Exchange; +import org.apache.camel.LoggingLevel; +import org.apache.camel.builder.RouteBuilder; +import org.apache.camel.model.dataformat.JsonLibrary; +import org.springframework.data.domain.PageRequest; +import org.springframework.stereotype.Component; + +import java.util.List; +import java.util.UUID; + +/** + * Apache Camel route for asynchronous document vectorization. + * + * Features: + * - Async vectorization triggered after document processing + * - Scheduled processing of pending vectorizations from database + * - Direct REST calls to Python embedding service + * - Error handling with retry mechanism + * + * @author Martin.Schweitzer@procon.co.at and claude.ai + */ +@Component +@RequiredArgsConstructor +@Slf4j +public class VectorizationRoute extends RouteBuilder { + + private static final String ROUTE_ID_TRIGGER = "vectorization-trigger"; + private static final String ROUTE_ID_PROCESSOR = "vectorization-processor"; + private static final String ROUTE_ID_SCHEDULER = "vectorization-scheduler"; + + private final TedProcessorProperties properties; + private final ProcurementDocumentRepository documentRepository; + private final VectorizationProcessorService vectorizationProcessorService; + private final ObjectMapper objectMapper; + + /** + * Creates thread pool for vectorization with highest priority. + * Only 1 thread since only one embedding service is available. + */ + private java.util.concurrent.ExecutorService executorService() { + return java.util.concurrent.Executors.newFixedThreadPool( + 1, + r -> { + Thread thread = new Thread(r); + thread.setName("ted-vectorization-" + thread.getId()); + thread.setDaemon(true); + thread.setPriority(Thread.MAX_PRIORITY); // Highest priority + return thread; + } + ); + } + + @Override + public void configure() throws Exception { + + if (!properties.getVectorization().isEnabled()) { + log.info("Vectorization is disabled, skipping route configuration"); + return; + } + + log.info("Configuring vectorization routes (enabled=true, apiUrl={}, connectTimeout={}ms, socketTimeout={}ms, maxRetries={}, scheduler every 6s)", + properties.getVectorization().getApiUrl(), + properties.getVectorization().getConnectTimeout(), + properties.getVectorization().getSocketTimeout(), + properties.getVectorization().getMaxRetries()); + + // Global error handler for unexpected exceptions (like NullPointer, Connection pool shutdown, etc.) + // Only catches severe exceptions that are not handled by route-specific doCatch + onException(NullPointerException.class, IllegalStateException.class) + .routeId("vectorization-error-handler") + .handled(true) + .process(exchange -> { + UUID documentId = exchange.getIn().getHeader("documentId", UUID.class); + Exception exception = exchange.getProperty(Exchange.EXCEPTION_CAUGHT, Exception.class); + String errorMsg = exception != null ? exception.getClass().getSimpleName() + ": " + exception.getMessage() : "Unknown error"; + + // If connection pool is shut down, it's likely during application shutdown - just log warning + if (errorMsg.contains("Connection pool shut down")) { + log.warn("Vectorization aborted for document {} - connection pool shut down (application shutting down?)", documentId); + return; + } + + log.error("Unexpected error in vectorization for document {}: {}", documentId, errorMsg, exception); + + // Update document status to FAILED via service (transactional) + if (documentId != null) { + try { + vectorizationProcessorService.markAsFailed(documentId, errorMsg); + } catch (Exception e) { + log.warn("Failed to mark document {} as failed: {}", documentId, e.getMessage()); + } + } + }) + .to("log:vectorization-error?level=WARN"); + + // Trigger route: Receives document ID and queues for async processing + // Queue size limited to 1000 to prevent memory issues + from("direct:vectorize") + .routeId(ROUTE_ID_TRIGGER) + .doTry() + .to("seda:vectorize-async?waitForTaskToComplete=Never&size=1000&blockWhenFull=true&timeout=5000") + .doCatch(Exception.class) + .log(LoggingLevel.WARN, "Failed to queue document ${header.documentId} for vectorization (queue may be full or shutting down): ${exception.message}") + .end(); + + // Async processor route: Performs actual vectorization with highest priority + // Uses dedicated single-thread pool with MAX_PRIORITY (1 thread for 1 embedding service) + from("seda:vectorize-async?size=1000") + .routeId(ROUTE_ID_PROCESSOR) + .threads().executorService(executorService()) + .process(exchange -> { + UUID documentId = exchange.getIn().getHeader("documentId", UUID.class); + + log.debug("Starting vectorization for document: {}", documentId); + + // Prepare document for vectorization (transactional) + VectorizationProcessorService.DocumentContent docContent = + vectorizationProcessorService.prepareDocumentForVectorization(documentId); + + if (docContent == null) { + // Document was skipped (no content) + log.debug("Document {} has no content, skipping vectorization", documentId); + exchange.setProperty("skipVectorization", true); + return; + } + + // Prepare request object + EmbedRequest embedRequest = new EmbedRequest(); + embedRequest.text = docContent.textContent(); + embedRequest.isQuery = false; + + // Set headers and body for REST call + exchange.getIn().setHeader("documentId", documentId); + exchange.getIn().setHeader(Exchange.HTTP_METHOD, "POST"); + exchange.getIn().setHeader(Exchange.CONTENT_TYPE, "application/json"); + exchange.getIn().setBody(embedRequest); + }) + .choice() + .when(exchangeProperty("skipVectorization").isEqualTo(true)) + .log(LoggingLevel.DEBUG, "Skipping vectorization (no content): ${header.documentId}") + .otherwise() + // Marshal request to JSON + .marshal().json(JsonLibrary.Jackson) + // Initialize retry counter + .setProperty("retryCount", constant(0)) + .setProperty("maxRetries", constant(properties.getVectorization().getMaxRetries())) + .setProperty("vectorizationSuccess", constant(false)) + // Retry loop with exponential backoff + .loopDoWhile(simple("${exchangeProperty.vectorizationSuccess} == false && ${exchangeProperty.retryCount} < ${exchangeProperty.maxRetries}")) + .process(exchange -> { + Integer retryCount = exchange.getProperty("retryCount", Integer.class); + exchange.setProperty("retryCount", retryCount + 1); + + // Exponential backoff: 2s, 4s, 8s, 16s, 32s + if (retryCount > 0) { + long backoffMs = (long) Math.pow(2, retryCount) * 1000; + UUID documentId = exchange.getIn().getHeader("documentId", UUID.class); + log.warn("Retry #{} for document {} after {}ms backoff", retryCount, documentId, backoffMs); + Thread.sleep(backoffMs); + } + }) + .doTry() + // HTTP call with configurable timeouts + .toD(properties.getVectorization().getApiUrl() + "/embed?bridgeEndpoint=true&throwExceptionOnFailure=false&connectTimeout=" + + properties.getVectorization().getConnectTimeout() + "&socketTimeout=" + + properties.getVectorization().getSocketTimeout()) + .process(exchange -> { + UUID documentId = exchange.getIn().getHeader("documentId", UUID.class); + Integer statusCode = exchange.getIn().getHeader(Exchange.HTTP_RESPONSE_CODE, Integer.class); + + if (statusCode == null) { + log.error("No response from embedding service for document {} (service may be down!)", documentId); + throw new RuntimeException("Embedding service not reachable (no HTTP response)"); + } + + if (statusCode != 200) { + String responseBody = exchange.getIn().getBody(String.class); + String errorMsg = "HTTP " + statusCode + " from embedding service: " + responseBody; + log.error("Embedding service error for document {}: {}", documentId, errorMsg); + throw new RuntimeException(errorMsg); + } + }) + .unmarshal().json(JsonLibrary.Jackson, EmbedResponse.class) + .process(exchange -> { + UUID documentId = exchange.getIn().getHeader("documentId", UUID.class); + EmbedResponse response = exchange.getIn().getBody(EmbedResponse.class); + + if (response == null || response.embedding == null) { + throw new RuntimeException("Embedding service returned null response"); + } + + log.debug("Successfully vectorized document {}: {} dimensions, {} tokens", + documentId, response.dimensions, response.tokenCount); + + // Save embedding with token count via service (transactional) + vectorizationProcessorService.saveEmbedding(documentId, response.embedding, response.tokenCount); + + // Mark as successful to stop retry loop + exchange.setProperty("vectorizationSuccess", true); + }) + .doCatch(Exception.class) + .process(exchange -> { + UUID documentId = exchange.getIn().getHeader("documentId", UUID.class); + Integer retryCount = exchange.getProperty("retryCount", Integer.class); + Integer maxRetries = exchange.getProperty("maxRetries", Integer.class); + Exception exception = exchange.getProperty(Exchange.EXCEPTION_CAUGHT, Exception.class); + String errorMsg = exception != null ? exception.getMessage() : "Unknown error"; + + // Check if error is due to shutdown + if (errorMsg != null && errorMsg.contains("Connection pool shut down")) { + log.warn("Vectorization aborted for document {} - connection pool shut down (application shutting down)", documentId); + // Don't mark as failed - it will be retried on next startup + exchange.setProperty("vectorizationSuccess", true); // Stop retry loop + return; + } + + if (retryCount >= maxRetries) { + log.error("Vectorization failed for document {} after {} retries: {}", documentId, maxRetries, errorMsg, exception); + try { + vectorizationProcessorService.markAsFailed(documentId, errorMsg); + } catch (Exception e) { + log.warn("Failed to mark document {} as failed (may be shutting down): {}", documentId, e.getMessage()); + } + } else { + log.warn("Vectorization attempt #{} failed for document {}: {}", retryCount, documentId, errorMsg); + } + }) + .end() + .end() + .end(); + + // Scheduled route: Process pending and failed vectorizations from database + // Runs every 6 seconds to catch documents that need (re-)vectorization + from("timer:vectorization-scheduler?period=6000&delay=500") + .routeId(ROUTE_ID_SCHEDULER) + .log(LoggingLevel.DEBUG, "Vectorization scheduler: Checking for pending/failed documents...") + .process(exchange -> { + int batchSize = properties.getVectorization().getBatchSize(); + + // First get PENDING documents (highest priority) + List pending = documentRepository.findByVectorizationStatus( + VectorizationStatus.PENDING, + PageRequest.of(0, batchSize) + ); + + // If no PENDING, get FAILED documents for retry + List failed = List.of(); + if (pending.isEmpty()) { + failed = documentRepository.findByVectorizationStatus( + VectorizationStatus.FAILED, + PageRequest.of(0, batchSize) + ); + } + + List toProcess = !pending.isEmpty() ? pending : failed; + + if (!toProcess.isEmpty()) { + String status = !pending.isEmpty() ? "PENDING" : "FAILED"; + log.debug("Processing {} {} vectorizations from database", toProcess.size(), status); + exchange.getIn().setBody(toProcess); + } else { + exchange.setProperty("noPendingDocs", true); + } + }) + .choice() + .when(exchangeProperty("noPendingDocs").isEqualTo(true)) + .log(LoggingLevel.DEBUG, "Vectorization scheduler: No pending or failed vectorizations found") + .otherwise() + .split(body()) + .process(exchange -> { + ProcurementDocument doc = exchange.getIn().getBody(ProcurementDocument.class); + exchange.getIn().setHeader("documentId", doc.getId()); + }) + .to("direct:vectorize") + .end() + .end(); + } + + /** + * Request model for embedding service. + * Matches Python FastAPI EmbedRequest model with snake_case field names. + */ + public static class EmbedRequest { + @JsonProperty("text") + public String text; + + @JsonProperty("is_query") + public boolean isQuery; + + public EmbedRequest() {} + + public String getText() { + return text; + } + + public void setText(String text) { + this.text = text; + } + + @JsonProperty("is_query") + public boolean isIsQuery() { + return isQuery; + } + + @JsonProperty("is_query") + public void setIsQuery(boolean isQuery) { + this.isQuery = isQuery; + } + } + + /** + * Response model for embedding service. + */ + public static class EmbedResponse { + public float[] embedding; + public int dimensions; + @JsonProperty("token_count") + public int tokenCount; + + public EmbedResponse() {} + + public float[] getEmbedding() { + return embedding; + } + + public void setEmbedding(float[] embedding) { + this.embedding = embedding; + } + + public int getDimensions() { + return dimensions; + } + + public void setDimensions(int dimensions) { + this.dimensions = dimensions; + } + + @JsonProperty("token_count") + public int getTokenCount() { + return tokenCount; + } + + @JsonProperty("token_count") + public void setTokenCount(int tokenCount) { + this.tokenCount = tokenCount; + } + } + +} diff --git a/src/main/java/at/procon/ted/config/AsyncConfig.java b/src/main/java/at/procon/ted/config/AsyncConfig.java new file mode 100644 index 0000000..33ad8ec --- /dev/null +++ b/src/main/java/at/procon/ted/config/AsyncConfig.java @@ -0,0 +1,78 @@ +package at.procon.ted.config; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.aop.interceptor.AsyncUncaughtExceptionHandler; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.scheduling.annotation.AsyncConfigurer; +import org.springframework.scheduling.annotation.EnableAsync; +import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor; + +import java.lang.reflect.Method; +import java.util.concurrent.Executor; + +/** + * Async configuration for document vectorization processing. + * Provides thread pool executor optimized for ML inference workloads. + * + * @author Martin.Schweitzer@procon.co.at and claude.ai + */ +@Configuration +@EnableAsync +@RequiredArgsConstructor +@Slf4j +public class AsyncConfig implements AsyncConfigurer { + + private final TedProcessorProperties properties; + + /** + * Thread pool executor for async vectorization tasks. + */ + @Bean(name = "vectorizationExecutor") + public Executor vectorizationExecutor() { + ThreadPoolTaskExecutor executor = new ThreadPoolTaskExecutor(); + executor.setCorePoolSize(properties.getVectorization().getThreadPoolSize()); + executor.setMaxPoolSize(properties.getVectorization().getThreadPoolSize() * 2); + executor.setQueueCapacity(500); + executor.setThreadNamePrefix("vectorization-"); + executor.setRejectedExecutionHandler((r, e) -> + log.warn("Vectorization task rejected, queue full")); + executor.initialize(); + return executor; + } + + /** + * Default async executor for general async tasks. + */ + @Override + @Bean(name = "taskExecutor") + public Executor getAsyncExecutor() { + ThreadPoolTaskExecutor executor = new ThreadPoolTaskExecutor(); + executor.setCorePoolSize(4); + executor.setMaxPoolSize(8); + executor.setQueueCapacity(200); + executor.setThreadNamePrefix("async-"); + executor.initialize(); + return executor; + } + + /** + * Exception handler for async tasks. + */ + @Override + public AsyncUncaughtExceptionHandler getAsyncUncaughtExceptionHandler() { + return new AsyncExceptionHandler(); + } + + /** + * Handles uncaught exceptions in async methods. + */ + @Slf4j + private static class AsyncExceptionHandler implements AsyncUncaughtExceptionHandler { + @Override + public void handleUncaughtException(Throwable ex, Method method, Object... params) { + log.error("Async exception in method {}: {}", method.getName(), ex.getMessage(), ex); + } + } +} diff --git a/src/main/java/at/procon/ted/config/CamelConfig.java b/src/main/java/at/procon/ted/config/CamelConfig.java new file mode 100644 index 0000000..d0211c5 --- /dev/null +++ b/src/main/java/at/procon/ted/config/CamelConfig.java @@ -0,0 +1,34 @@ +package at.procon.ted.config; + +import org.apache.camel.spi.IdempotentRepository; +import org.apache.camel.support.processor.idempotent.FileIdempotentRepository; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +import java.io.File; + +/** + * Camel configuration beans. + * + * @author Martin.Schweitzer@procon.co.at and claude.ai + */ +@Configuration +public class CamelConfig { + + @Value("${ted.solution-brief.idempotent-repository:./solution-brief-processed.dat}") + private String idempotentRepositoryPath; + + /** + * File-based idempotent repository for tracking processed files. + */ + @Bean + public IdempotentRepository fileIdempotentRepository() { + File repoFile = new File(idempotentRepositoryPath); + // Ensure parent directory exists + if (repoFile.getParentFile() != null && !repoFile.getParentFile().exists()) { + repoFile.getParentFile().mkdirs(); + } + return FileIdempotentRepository.fileIdempotentRepository(repoFile, 10000); + } +} diff --git a/src/main/java/at/procon/ted/config/OpenApiConfig.java b/src/main/java/at/procon/ted/config/OpenApiConfig.java new file mode 100644 index 0000000..9a8c211 --- /dev/null +++ b/src/main/java/at/procon/ted/config/OpenApiConfig.java @@ -0,0 +1,58 @@ +package at.procon.ted.config; + +import io.swagger.v3.oas.models.OpenAPI; +import io.swagger.v3.oas.models.info.Contact; +import io.swagger.v3.oas.models.info.Info; +import io.swagger.v3.oas.models.info.License; +import io.swagger.v3.oas.models.servers.Server; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +import java.util.List; + +/** + * OpenAPI/Swagger documentation configuration. + * + * @author Martin.Schweitzer@procon.co.at and claude.ai + */ +@Configuration +public class OpenApiConfig { + + @Bean + public OpenAPI customOpenAPI() { + return new OpenAPI() + .info(new Info() + .title("TED Procurement Document API") + .version("1.0.0") + .description(""" + REST API for searching and retrieving EU eForms public procurement documents + from TED (Tenders Electronic Daily). + + ## Features + - **Structured Search**: Filter by country, CPV codes, dates, procedure types + - **Semantic Search**: Natural language queries using vector similarity + - **Document Retrieval**: Full document details with lots and organizations + + ## Authentication + Currently no authentication required (development mode). + + ## Rate Limits + No rate limits currently enforced. + """) + .contact(new Contact() + .name("PROCON DATA") + .email("Martin.Schweitzer@procon.co.at") + .url("https://www.procon.co.at")) + .license(new License() + .name("Proprietary") + .url("https://www.procon.co.at"))) + .servers(List.of( + new Server() + .url("http://localhost:8080/api") + .description("Local Development Server"), + new Server() + .url("https://ted-api.procon.co.at/api") + .description("Production Server") + )); + } +} diff --git a/src/main/java/at/procon/ted/config/TedProcessorProperties.java b/src/main/java/at/procon/ted/config/TedProcessorProperties.java new file mode 100644 index 0000000..0e307cd --- /dev/null +++ b/src/main/java/at/procon/ted/config/TedProcessorProperties.java @@ -0,0 +1,431 @@ +package at.procon.ted.config; + +import lombok.Data; +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.context.annotation.Configuration; +import org.springframework.validation.annotation.Validated; + +import jakarta.validation.constraints.Min; +import jakarta.validation.constraints.NotBlank; +import jakarta.validation.constraints.Positive; + +/** + * Configuration properties for TED Procurement Processor. + * + * @author Martin.Schweitzer@procon.co.at and claude.ai + */ +@Configuration +@ConfigurationProperties(prefix = "ted") +@Data +@Validated +public class TedProcessorProperties { + + private InputProperties input = new InputProperties(); + private SchemaProperties schema = new SchemaProperties(); + private VectorizationProperties vectorization = new VectorizationProperties(); + private SearchProperties search = new SearchProperties(); + private DownloadProperties download = new DownloadProperties(); + private MailProperties mail = new MailProperties(); + private SolutionBriefProperties solutionBrief = new SolutionBriefProperties(); + + /** + * Input directory configuration for Apache Camel file consumer. + */ + @Data + public static class InputProperties { + + /** + * Base directory for watching incoming TED XML files. + */ + @NotBlank + private String directory = "D:/ted.europe/2025-11.tar/2025-11/11"; + + /** + * File pattern to match (supports Ant-style patterns). + */ + private String pattern = "**/*.xml"; + + /** + * Directory to move successfully processed files. + */ + private String processedDirectory = ".processed"; + + /** + * Directory to move failed files. + */ + private String errorDirectory = ".error"; + + /** + * Polling interval in milliseconds. + */ + @Positive + private long pollInterval = 5000; + + /** + * Maximum number of messages per poll. + */ + @Positive + private int maxMessagesPerPoll = 100; + } + + /** + * XML Schema validation configuration. + */ + @Data + public static class SchemaProperties { + + /** + * Enable/disable XSD validation. + */ + private boolean enabled = true; + + /** + * Path to the eForms XSD schema file. + */ + private String path = "classpath:schemas/maindoc/UBL-ContractNotice-2.3.xsd"; + } + + /** + * Document vectorization configuration. + */ + @Data + public static class VectorizationProperties { + + /** + * Enable/disable async vectorization. + */ + private boolean enabled = true; + + /** + * Use external HTTP API instead of Python subprocess. + */ + private boolean useHttpApi = false; + + /** + * Embedding service HTTP API URL. + */ + private String apiUrl = "http://localhost:8001"; + + /** + * Sentence transformer model name. + */ + private String modelName = "intfloat/multilingual-e5-large"; + + /** + * Vector dimensions (must match model output). + */ + @Positive + private int dimensions = 1024; + + /** + * Batch size for vectorization processing. + */ + @Min(1) + private int batchSize = 16; + + /** + * Thread pool size for async vectorization. + */ + @Min(1) + private int threadPoolSize = 4; + + /** + * Maximum text length for vectorization (characters). + */ + @Positive + private int maxTextLength = 8192; + + /** + * HTTP connection timeout in milliseconds. + */ + @Positive + private int connectTimeout = 10000; + + /** + * HTTP socket/read timeout in milliseconds. + */ + @Positive + private int socketTimeout = 60000; + + /** + * Maximum retries on connection failure. + */ + @Min(0) + private int maxRetries = 5; + } + + /** + * Search configuration. + */ + @Data + public static class SearchProperties { + + /** + * Default page size for search results. + */ + @Positive + private int defaultPageSize = 20; + + /** + * Maximum allowed page size. + */ + @Positive + private int maxPageSize = 100; + + /** + * Similarity threshold for vector search (0.0 - 1.0). + */ + private double similarityThreshold = 0.7; + } + + /** + * TED Daily Package Download configuration. + */ + @Data + public static class DownloadProperties { + + /** + * Enable/disable automatic package download. + */ + private boolean enabled = false; + + /** + * Base URL für TED Daily Packages. + */ + private String baseUrl = "https://ted.europa.eu/packages/daily/"; + + /** + * Download-Verzeichnis für tar.gz Files. + */ + private String downloadDirectory = "D:/ted.europe/downloads"; + + /** + * Extrahierungs-Verzeichnis für XML-Dateien. + */ + private String extractDirectory = "D:/ted.europe/extracted"; + + /** + * Start-Jahr für den Download. + */ + @Positive + private int startYear = 2015; + + /** + * Anzahl aufeinanderfolgender 404-Fehler bevor Download stoppt. + * HINWEIS: Wird nicht mehr verwendet. System stoppt jetzt sofort bei erstem 404. + * @deprecated Nicht mehr verwendet seit Update auf sofortige 404-Behandlung + */ + @Positive + @Deprecated + private int maxConsecutive404 = 1; + + /** + * Polling-Interval für neue Packages (Millisekunden). + */ + @Positive + private long pollInterval = 3600000; // 1 Stunde + + /** + * Retry-Intervall für tail-NOT_FOUND Packages. + * Current year packages remain retryable indefinitely. + */ + @Positive + private long notFoundRetryInterval = 21600000; // 6 Stunden + + /** + * Grace period for previous years after year end before a tail-NOT_FOUND is treated as final. + */ + @Min(0) + private int previousYearGracePeriodDays = 30; + + /** + * Keep retrying current-year tail NOT_FOUND packages indefinitely. + */ + private boolean retryCurrentYearNotFoundIndefinitely = true; + + /** + * Download-Timeout (Millisekunden). + */ + @Positive + private long downloadTimeout = 300000; // 5 Minuten + + /** + * Maximale gleichzeitige Downloads. + */ + @Positive + private int maxConcurrentDownloads = 2; + + /** + * Verzögerung zwischen Downloads (Millisekunden) für Rate Limiting. + */ + @Positive + private long delayBetweenDownloads = 5000; // 5 Sekunden + + /** + * Automatisches Löschen von tar.gz nach Extraktion. + */ + private boolean deleteAfterExtraction = true; + + /** + * Priorisierung: Aktuelles Jahr zuerst, dann rückwärts. + * HINWEIS: Wird nicht mehr verwendet. System priorisiert immer das aktuelle Jahr. + * @deprecated Nicht mehr verwendet - immer aktiv + */ + @Deprecated + private boolean prioritizeCurrentYear = true; + } + + /** + * IMAP Mail configuration for email processing. + */ + @Data + public static class MailProperties { + + /** + * Enable/disable mail processing. + */ + private boolean enabled = false; + + /** + * IMAP server hostname. + */ + @NotBlank + private String host = "mail.mymagenta.business"; + + /** + * IMAP server port. + */ + @Positive + private int port = 993; + + /** + * Mail account username (email address). + */ + @NotBlank + private String username = "archiv@procon.co.at"; + + /** + * Mail account password. + */ + @NotBlank + private String password = ""; + + /** + * Use SSL/TLS connection. + */ + private boolean ssl = true; + + /** + * Mail folder to read from. + */ + private String folderName = "INBOX"; + + /** + * Delete messages after processing. + */ + private boolean delete = false; + + /** + * Mark messages as seen after processing. + */ + private boolean seen = true; + + /** + * Only process unseen messages. + */ + private boolean unseen = true; + + /** + * Polling delay in milliseconds. + */ + @Positive + private long delay = 60000; + + /** + * Max messages per poll. + */ + @Positive + private int maxMessagesPerPoll = 10; + + /** + * Output directory for processed attachments. + */ + private String attachmentOutputDirectory = "D:/ted.europe/mail-attachments"; + + /** + * Enable/disable MIME file input processing. + */ + private boolean mimeInputEnabled = false; + + /** + * Input directory for MIME files (.eml, .msg). + */ + private String mimeInputDirectory = "D:/ted.europe/mime-input"; + + /** + * File pattern for MIME files. + */ + private String mimeInputPattern = "*.eml"; + + /** + * Polling interval for MIME input directory (milliseconds). + */ + @Positive + private long mimeInputPollInterval = 10000; + } + + /** + * Solution Brief processing configuration. + * Scans PDF files and generates Excel reports with similar TED documents. + */ + @Data + public static class SolutionBriefProperties { + + /** + * Enable/disable Solution Brief processing. + */ + private boolean enabled = false; + + /** + * Input directory for Solution Brief PDF files. + */ + private String inputDirectory = "C:/work/SolutionBrief"; + + /** + * Output directory for Excel result files (relative to input or absolute). + */ + private String resultDirectory = "./result"; + + /** + * Number of top similar documents to include in results. + */ + @Positive + private int topK = 20; + + /** + * Minimum similarity threshold (0.0-1.0). + */ + private double similarityThreshold = 0.5; + + /** + * Polling interval in milliseconds. + */ + @Positive + private long pollInterval = 30000; + + /** + * File pattern for PDF files. + */ + private String filePattern = ".*\\.pdf"; + + /** + * Process files only once (idempotent based on filename+size+date). + */ + private boolean idempotent = true; + + /** + * Idempotent repository file path. + */ + private String idempotentRepository = "./solution-brief-processed.dat"; + } +} diff --git a/src/main/java/at/procon/ted/controller/AdminController.java b/src/main/java/at/procon/ted/controller/AdminController.java new file mode 100644 index 0000000..acf5c3f --- /dev/null +++ b/src/main/java/at/procon/ted/controller/AdminController.java @@ -0,0 +1,264 @@ +package at.procon.ted.controller; + +import at.procon.ted.model.entity.ProcessingLog; +import at.procon.ted.model.entity.VectorizationStatus; +import at.procon.ted.repository.ProcurementDocumentRepository; +import at.procon.ted.service.DocumentProcessingService; +import at.procon.ted.service.VectorizationService; +import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.tags.Tag; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.apache.camel.ProducerTemplate; +import org.springframework.data.domain.PageRequest; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.*; + +import java.time.OffsetDateTime; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.UUID; + +/** + * REST API controller for administrative operations. + * + * Provides endpoints for: + * - System health checks + * - Manual vectorization triggers + * - Processing log access + * - Document reprocessing + * + * @author Martin.Schweitzer@procon.co.at and claude.ai + */ +@RestController +@RequestMapping("/v1/admin") +@RequiredArgsConstructor +@Slf4j +@Tag(name = "Admin", description = "Administrative Operations API") +public class AdminController { + + private final VectorizationService vectorizationService; + private final DocumentProcessingService documentProcessingService; + private final ProcurementDocumentRepository documentRepository; + private final at.procon.ted.repository.ProcessingLogRepository logRepository; + private final ProducerTemplate producerTemplate; + private final at.procon.ted.service.DataCleanupService dataCleanupService; + + /** + * Health check endpoint. + */ + @GetMapping("/health") + @Operation(summary = "Health check", description = "Check system health and service availability") + public ResponseEntity> healthCheck() { + Map health = new HashMap<>(); + health.put("status", "UP"); + health.put("timestamp", OffsetDateTime.now()); + health.put("vectorizationAvailable", vectorizationService.isAvailable()); + health.put("documentCount", documentRepository.count()); + + return ResponseEntity.ok(health); + } + + /** + * Get vectorization status overview. + */ + @GetMapping("/vectorization/status") + @Operation(summary = "Vectorization status", description = "Get overview of document vectorization status") + public ResponseEntity> getVectorizationStatus() { + Map status = new HashMap<>(); + + List counts = documentRepository.countByVectorizationStatus(); + Map statusCounts = new HashMap<>(); + for (Object[] row : counts) { + statusCounts.put(((VectorizationStatus) row[0]).name(), (Long) row[1]); + } + + status.put("counts", statusCounts); + status.put("serviceAvailable", vectorizationService.isAvailable()); + status.put("timestamp", OffsetDateTime.now()); + + return ResponseEntity.ok(status); + } + + /** + * Manually trigger vectorization for a specific document. + */ + @PostMapping("/vectorization/trigger/{documentId}") + @Operation(summary = "Trigger vectorization", description = "Manually trigger vectorization for a specific document") + public ResponseEntity> triggerVectorization(@PathVariable UUID documentId) { + Map result = new HashMap<>(); + + if (!vectorizationService.isAvailable()) { + result.put("success", false); + result.put("message", "Vectorization service is not available"); + return ResponseEntity.badRequest().body(result); + } + + // Verify document exists + if (!documentRepository.existsById(documentId)) { + result.put("success", false); + result.put("message", "Document not found: " + documentId); + return ResponseEntity.badRequest().body(result); + } + + // Trigger vectorization via Camel route + producerTemplate.sendBodyAndHeader("direct:vectorize", null, "documentId", documentId); + + result.put("success", true); + result.put("message", "Vectorization triggered for document " + documentId); + result.put("documentId", documentId); + + return ResponseEntity.ok(result); + } + + /** + * Trigger vectorization for all pending documents. + */ + @PostMapping("/vectorization/process-pending") + @Operation(summary = "Process pending vectorizations", description = "Trigger vectorization for all pending documents") + public ResponseEntity> processPendingVectorizations( + @RequestParam(required = false, defaultValue = "100") Integer batchSize) { + Map result = new HashMap<>(); + + if (!vectorizationService.isAvailable()) { + result.put("success", false); + result.put("message", "Vectorization service is not available"); + return ResponseEntity.badRequest().body(result); + } + + var pending = documentRepository.findByVectorizationStatus( + VectorizationStatus.PENDING, + PageRequest.of(0, Math.min(batchSize, 500))); + + int count = 0; + for (var doc : pending) { + // Trigger vectorization via Camel route + producerTemplate.sendBodyAndHeader("direct:vectorize", null, "documentId", doc.getId()); + count++; + } + + result.put("success", true); + result.put("message", "Triggered vectorization for " + count + " documents"); + result.put("documentsQueued", count); + + return ResponseEntity.ok(result); + } + + /** + * Reprocess a document by publication ID. + */ + @PostMapping("/reprocess/{publicationId}") + @Operation(summary = "Reprocess document", description = "Reparse and revectorize a document by publication ID") + public ResponseEntity> reprocessDocument(@PathVariable String publicationId) { + Map result = new HashMap<>(); + + var updated = documentProcessingService.reprocessDocument(publicationId); + + if (updated.isPresent()) { + result.put("success", true); + result.put("message", "Document reprocessed successfully"); + result.put("documentId", updated.get().getId()); + result.put("publicationId", publicationId); + return ResponseEntity.ok(result); + } else { + result.put("success", false); + result.put("message", "Document not found or reprocessing failed"); + result.put("publicationId", publicationId); + return ResponseEntity.notFound().build(); + } + } + + /** + * Get recent processing logs. + */ + @GetMapping("/logs/recent") + @Operation(summary = "Recent processing logs", description = "Get recent document processing log entries") + public ResponseEntity> getRecentLogs( + @RequestParam(required = false, defaultValue = "24") Integer hoursBack, + @RequestParam(required = false, defaultValue = "100") Integer limit) { + + OffsetDateTime since = OffsetDateTime.now().minusHours(hoursBack); + List logs = logRepository.findRecentLogs(since); + + if (logs.size() > limit) { + logs = logs.subList(0, limit); + } + + return ResponseEntity.ok(logs); + } + + /** + * Get processing logs for a specific document. + */ + @GetMapping("/logs/document/{documentId}") + @Operation(summary = "Document processing logs", description = "Get processing log entries for a specific document") + public ResponseEntity> getDocumentLogs(@PathVariable UUID documentId) { + List logs = logRepository.findByDocumentIdOrderByCreatedAtDesc(documentId); + return ResponseEntity.ok(logs); + } + + /** + * Get system information. + */ + @GetMapping("/info") + @Operation(summary = "System information", description = "Get system configuration and runtime information") + public ResponseEntity> getSystemInfo() { + Map info = new HashMap<>(); + + info.put("javaVersion", System.getProperty("java.version")); + info.put("osName", System.getProperty("os.name")); + info.put("availableProcessors", Runtime.getRuntime().availableProcessors()); + info.put("maxMemory", Runtime.getRuntime().maxMemory()); + info.put("freeMemory", Runtime.getRuntime().freeMemory()); + info.put("totalMemory", Runtime.getRuntime().totalMemory()); + info.put("vectorizationEnabled", vectorizationService.isAvailable()); + + return ResponseEntity.ok(info); + } + + /** + * Count documents older than specified retention period. + */ + @GetMapping("/cleanup/count") + @Operation(summary = "Count old documents", description = "Count documents older than specified years") + public ResponseEntity> countOldDocuments( + @RequestParam(required = false, defaultValue = "7") Integer years) { + Map result = new HashMap<>(); + + long count = dataCleanupService.countDocumentsOlderThan(years); + + result.put("years", years); + result.put("count", count); + result.put("message", String.format("Found %d documents older than %d years", count, years)); + + return ResponseEntity.ok(result); + } + + /** + * Delete documents older than specified retention period. + */ + @DeleteMapping("/cleanup/delete") + @Operation(summary = "Delete old documents", description = "Delete documents older than specified years (default: 7)") + public ResponseEntity> deleteOldDocuments( + @RequestParam(required = false, defaultValue = "7") Integer years) { + Map result = new HashMap<>(); + + try { + int deletedCount = dataCleanupService.deleteDocumentsOlderThan(years); + + result.put("success", true); + result.put("years", years); + result.put("deletedCount", deletedCount); + result.put("message", String.format("Deleted %d documents older than %d years", deletedCount, years)); + + return ResponseEntity.ok(result); + + } catch (Exception e) { + log.error("Error deleting old documents", e); + result.put("success", false); + result.put("error", e.getMessage()); + return ResponseEntity.status(500).body(result); + } + } +} diff --git a/src/main/java/at/procon/ted/controller/DocumentController.java b/src/main/java/at/procon/ted/controller/DocumentController.java new file mode 100644 index 0000000..a7f19c2 --- /dev/null +++ b/src/main/java/at/procon/ted/controller/DocumentController.java @@ -0,0 +1,314 @@ +package at.procon.ted.controller; + +import at.procon.ted.model.dto.DocumentDtos.*; +import at.procon.ted.model.entity.ContractNature; +import at.procon.ted.model.entity.NoticeType; +import at.procon.ted.model.entity.ProcedureType; +import at.procon.ted.service.SearchService; +import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.Parameter; +import io.swagger.v3.oas.annotations.media.Content; +import io.swagger.v3.oas.annotations.media.Schema; +import io.swagger.v3.oas.annotations.responses.ApiResponse; +import io.swagger.v3.oas.annotations.responses.ApiResponses; +import io.swagger.v3.oas.annotations.tags.Tag; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.format.annotation.DateTimeFormat; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.*; + +import java.time.LocalDate; +import java.time.OffsetDateTime; +import java.util.List; +import java.util.UUID; + +/** + * REST API controller for searching and retrieving TED procurement documents. + * + * Provides endpoints for: + * - Structured search with filters (country, type, dates, etc.) + * - Semantic search using natural language queries + * - Document retrieval by ID or publication ID + * - Statistics and metadata + * + * @author Martin.Schweitzer@procon.co.at and claude.ai + */ +@RestController +@RequestMapping("/v1/documents") +@RequiredArgsConstructor +@Slf4j +@Tag(name = "Documents", description = "TED Procurement Document Search API") +public class DocumentController { + + private final SearchService searchService; + + /** + * Search documents with structured and/or semantic filters. + */ + @GetMapping("/search") + @Operation( + summary = "Search procurement documents", + description = "Search documents using structured filters (country, type, dates) and/or semantic search with natural language queries" + ) + @ApiResponses(value = { + @ApiResponse(responseCode = "200", description = "Search results returned successfully", + content = @Content(schema = @Schema(implementation = SearchResponse.class))), + @ApiResponse(responseCode = "400", description = "Invalid search parameters") + }) + public ResponseEntity searchDocuments( + @Parameter(description = "Country code (ISO 3166-1 alpha-3, e.g., POL, DEU, FRA)") + @RequestParam(required = false) String countryCode, + + @Parameter(description = "Multiple country codes") + @RequestParam(required = false) List countryCodes, + + @Parameter(description = "Notice type filter") + @RequestParam(required = false) NoticeType noticeType, + + @Parameter(description = "Contract nature filter") + @RequestParam(required = false) ContractNature contractNature, + + @Parameter(description = "Procedure type filter") + @RequestParam(required = false) ProcedureType procedureType, + + @Parameter(description = "CPV code prefix (e.g., '33' for medical supplies)") + @RequestParam(required = false) String cpvPrefix, + + @Parameter(description = "NUTS region code") + @RequestParam(required = false) String nutsCode, + + @Parameter(description = "Publication date from (inclusive)") + @RequestParam(required = false) @DateTimeFormat(iso = DateTimeFormat.ISO.DATE) LocalDate publicationDateFrom, + + @Parameter(description = "Publication date to (inclusive)") + @RequestParam(required = false) @DateTimeFormat(iso = DateTimeFormat.ISO.DATE) LocalDate publicationDateTo, + + @Parameter(description = "Only documents with submission deadline after this date") + @RequestParam(required = false) @DateTimeFormat(iso = DateTimeFormat.ISO.DATE_TIME) OffsetDateTime submissionDeadlineAfter, + + @Parameter(description = "Filter by EU funding status") + @RequestParam(required = false) Boolean euFunded, + + @Parameter(description = "Search in buyer name (case-insensitive)") + @RequestParam(required = false) String buyerNameContains, + + @Parameter(description = "Search in project title (case-insensitive)") + @RequestParam(required = false) String projectTitleContains, + + @Parameter(description = "Natural language semantic search query") + @RequestParam(required = false) String q, + + @Parameter(description = "Similarity threshold for semantic search (0.0-1.0)") + @RequestParam(required = false, defaultValue = "0.7") Double similarityThreshold, + + @Parameter(description = "Page number (0-based)") + @RequestParam(required = false, defaultValue = "0") Integer page, + + @Parameter(description = "Page size (max 100)") + @RequestParam(required = false, defaultValue = "20") Integer size, + + @Parameter(description = "Sort field (publicationDate, submissionDeadline, buyerName, projectTitle)") + @RequestParam(required = false, defaultValue = "publicationDate") String sortBy, + + @Parameter(description = "Sort direction (asc, desc)") + @RequestParam(required = false, defaultValue = "desc") String sortDirection + ) { + SearchRequest request = SearchRequest.builder() + .countryCode(countryCode) + .countryCodes(countryCodes) + .noticeType(noticeType) + .contractNature(contractNature) + .procedureType(procedureType) + .cpvPrefix(cpvPrefix) + .nutsCode(nutsCode) + .publicationDateFrom(publicationDateFrom) + .publicationDateTo(publicationDateTo) + .submissionDeadlineAfter(submissionDeadlineAfter) + .euFunded(euFunded) + .buyerNameContains(buyerNameContains) + .projectTitleContains(projectTitleContains) + .semanticQuery(q) + .similarityThreshold(similarityThreshold) + .page(page) + .size(size) + .sortBy(sortBy) + .sortDirection(sortDirection) + .build(); + + log.debug("Search request: {}", request); + SearchResponse response = searchService.search(request); + return ResponseEntity.ok(response); + } + + /** + * Search documents using POST with request body. + * Useful for complex queries with many parameters. + */ + @PostMapping("/search") + @Operation( + summary = "Search procurement documents (POST)", + description = "Search documents using a JSON request body for complex queries" + ) + public ResponseEntity searchDocumentsPost(@RequestBody SearchRequest request) { + log.debug("Search request (POST): {}", request); + SearchResponse response = searchService.search(request); + return ResponseEntity.ok(response); + } + + /** + * Get document by internal UUID. + */ + @GetMapping("/{id}") + @Operation( + summary = "Get document by ID", + description = "Retrieve full document details by internal UUID" + ) + @ApiResponses(value = { + @ApiResponse(responseCode = "200", description = "Document found", + content = @Content(schema = @Schema(implementation = DocumentDetail.class))), + @ApiResponse(responseCode = "404", description = "Document not found") + }) + public ResponseEntity getDocument( + @Parameter(description = "Document UUID") @PathVariable UUID id) { + return searchService.getDocumentDetail(id) + .map(ResponseEntity::ok) + .orElse(ResponseEntity.notFound().build()); + } + + /** + * Get document by TED publication ID. + */ + @GetMapping("/publication/{publicationId}") + @Operation( + summary = "Get document by publication ID", + description = "Retrieve document by TED publication ID (e.g., '00786665-2025')" + ) + @ApiResponses(value = { + @ApiResponse(responseCode = "200", description = "Document found"), + @ApiResponse(responseCode = "404", description = "Document not found") + }) + public ResponseEntity getDocumentByPublicationId( + @Parameter(description = "TED Publication ID") @PathVariable String publicationId) { + return searchService.getDocumentByPublicationId(publicationId) + .map(ResponseEntity::ok) + .orElse(ResponseEntity.notFound().build()); + } + + /** + * Get documents with upcoming submission deadlines. + */ + @GetMapping("/upcoming-deadlines") + @Operation( + summary = "Get documents with upcoming deadlines", + description = "List documents with submission deadlines in the future, sorted by deadline" + ) + public ResponseEntity> getUpcomingDeadlines( + @Parameter(description = "Maximum number of results") + @RequestParam(required = false, defaultValue = "20") Integer limit) { + List documents = searchService.getUpcomingDeadlines(Math.min(limit, 100)); + return ResponseEntity.ok(documents); + } + + /** + * Get collection statistics. + */ + @GetMapping("/statistics") + @Operation( + summary = "Get collection statistics", + description = "Retrieve statistics about the document collection including counts by country, type, and vectorization status" + ) + public ResponseEntity getStatistics() { + StatisticsResponse stats = searchService.getStatistics(); + return ResponseEntity.ok(stats); + } + + /** + * Get list of all countries in the collection. + */ + @GetMapping("/metadata/countries") + @Operation( + summary = "Get available countries", + description = "List all distinct country codes present in the collection" + ) + public ResponseEntity> getCountries() { + List countries = searchService.getDistinctCountries(); + return ResponseEntity.ok(countries); + } + + /** + * Get available notice types. + */ + @GetMapping("/metadata/notice-types") + @Operation( + summary = "Get available notice types", + description = "List all notice type enum values" + ) + public ResponseEntity getNoticeTypes() { + return ResponseEntity.ok(NoticeType.values()); + } + + /** + * Get available contract natures. + */ + @GetMapping("/metadata/contract-natures") + @Operation( + summary = "Get available contract natures", + description = "List all contract nature enum values" + ) + public ResponseEntity getContractNatures() { + return ResponseEntity.ok(ContractNature.values()); + } + + /** + * Get available procedure types. + */ + @GetMapping("/metadata/procedure-types") + @Operation( + summary = "Get available procedure types", + description = "List all procedure type enum values" + ) + public ResponseEntity getProcedureTypes() { + return ResponseEntity.ok(ProcedureType.values()); + } + + /** + * Semantic search endpoint - convenience method for natural language queries. + */ + @GetMapping("/semantic-search") + @Operation( + summary = "Semantic search", + description = "Search documents using natural language query with vector similarity" + ) + public ResponseEntity semanticSearch( + @Parameter(description = "Natural language search query", required = true) + @RequestParam String query, + + @Parameter(description = "Minimum similarity score (0.0-1.0)") + @RequestParam(required = false, defaultValue = "0.7") Double threshold, + + @Parameter(description = "Country code filter") + @RequestParam(required = false) String countryCode, + + @Parameter(description = "Notice type filter") + @RequestParam(required = false) NoticeType noticeType, + + @Parameter(description = "Page number") + @RequestParam(required = false, defaultValue = "0") Integer page, + + @Parameter(description = "Page size") + @RequestParam(required = false, defaultValue = "20") Integer size + ) { + SearchRequest request = SearchRequest.builder() + .semanticQuery(query) + .similarityThreshold(threshold) + .countryCode(countryCode) + .noticeType(noticeType) + .page(page) + .size(size) + .build(); + + SearchResponse response = searchService.search(request); + return ResponseEntity.ok(response); + } +} diff --git a/src/main/java/at/procon/ted/controller/SimilaritySearchController.java b/src/main/java/at/procon/ted/controller/SimilaritySearchController.java new file mode 100644 index 0000000..5ce2e16 --- /dev/null +++ b/src/main/java/at/procon/ted/controller/SimilaritySearchController.java @@ -0,0 +1,175 @@ +package at.procon.ted.controller; + +import at.procon.ted.service.SimilaritySearchService; +import at.procon.ted.service.SimilaritySearchService.SimilaritySearchResponse; +import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.Parameter; +import io.swagger.v3.oas.annotations.media.Content; +import io.swagger.v3.oas.annotations.media.Schema; +import io.swagger.v3.oas.annotations.responses.ApiResponse; +import io.swagger.v3.oas.annotations.responses.ApiResponses; +import io.swagger.v3.oas.annotations.tags.Tag; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.http.MediaType; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.*; +import org.springframework.web.multipart.MultipartFile; + +import java.io.IOException; + +/** + * REST Controller for similarity search on TED procurement documents. + * Provides endpoints for searching similar documents using text or PDF input. + * + * @author Martin.Schweitzer@procon.co.at and claude.ai + */ +@RestController +@RequestMapping("/similarity") +@RequiredArgsConstructor +@Slf4j +@Tag(name = "Similarity Search", description = "Vector-based semantic similarity search on TED procurement documents") +public class SimilaritySearchController { + + private final SimilaritySearchService similaritySearchService; + + /** + * Search for similar documents using text query. + */ + @PostMapping("/text") + @Operation( + summary = "Search by text", + description = "Find similar TED procurement documents based on text content using vector similarity (cosine distance)" + ) + @ApiResponses({ + @ApiResponse(responseCode = "200", description = "Search completed successfully", + content = @Content(schema = @Schema(implementation = SimilaritySearchResponse.class))), + @ApiResponse(responseCode = "400", description = "Invalid request (empty text)"), + @ApiResponse(responseCode = "503", description = "Vectorization service unavailable") + }) + public ResponseEntity searchByText( + @Parameter(description = "Text content to search for similar documents", required = true) + @RequestBody TextSearchRequest request + ) { + log.info("Text similarity search request: {} chars, topK={}, threshold={}", + request.getText() != null ? request.getText().length() : 0, + request.getTopK(), + request.getThreshold()); + + if (request.getText() == null || request.getText().isBlank()) { + return ResponseEntity.badRequest().build(); + } + + try { + SimilaritySearchResponse response = similaritySearchService.searchByText( + request.getText(), + request.getTopK(), + request.getThreshold() + ); + return ResponseEntity.ok(response); + + } catch (IllegalStateException e) { + log.error("Vectorization service unavailable: {}", e.getMessage()); + return ResponseEntity.status(503).build(); + + } catch (Exception e) { + log.error("Text similarity search failed: {}", e.getMessage(), e); + return ResponseEntity.internalServerError().build(); + } + } + + /** + * Search for similar documents using PDF file. + */ + @PostMapping(value = "/pdf", consumes = MediaType.MULTIPART_FORM_DATA_VALUE) + @Operation( + summary = "Search by PDF", + description = "Upload a PDF document to find similar TED procurement documents. " + + "Text is extracted from the PDF and used for vector similarity search." + ) + @ApiResponses({ + @ApiResponse(responseCode = "200", description = "Search completed successfully", + content = @Content(schema = @Schema(implementation = SimilaritySearchResponse.class))), + @ApiResponse(responseCode = "400", description = "Invalid request (no file or not a PDF)"), + @ApiResponse(responseCode = "422", description = "Could not extract text from PDF"), + @ApiResponse(responseCode = "503", description = "Vectorization service unavailable") + }) + public ResponseEntity searchByPdf( + @Parameter(description = "PDF file to search for similar documents", required = true) + @RequestPart("file") MultipartFile file, + + @Parameter(description = "Number of top results to return (default: 20, max: 100)") + @RequestParam(required = false, defaultValue = "20") Integer topK, + + @Parameter(description = "Minimum similarity threshold (0.0-1.0, default: 0.5)") + @RequestParam(required = false, defaultValue = "0.5") Double threshold + ) { + if (file == null || file.isEmpty()) { + log.warn("PDF search request with empty file"); + return ResponseEntity.badRequest().build(); + } + + String filename = file.getOriginalFilename(); + String contentType = file.getContentType(); + + log.info("PDF similarity search request: filename='{}', size={} bytes, topK={}, threshold={}", + filename, file.getSize(), topK, threshold); + + // Validate file type + if (contentType != null && !contentType.toLowerCase().contains("pdf")) { + if (filename == null || !filename.toLowerCase().endsWith(".pdf")) { + log.warn("Invalid file type: {} ({})", filename, contentType); + return ResponseEntity.badRequest().build(); + } + } + + try { + byte[] pdfData = file.getBytes(); + + SimilaritySearchResponse response = similaritySearchService.searchByPdf( + pdfData, + filename, + topK, + threshold + ); + return ResponseEntity.ok(response); + + } catch (IOException e) { + log.error("Failed to read PDF file: {}", e.getMessage()); + return ResponseEntity.badRequest().build(); + + } catch (IllegalStateException e) { + log.error("Vectorization service unavailable: {}", e.getMessage()); + return ResponseEntity.status(503).build(); + + } catch (RuntimeException e) { + if (e.getMessage() != null && e.getMessage().contains("extraction failed")) { + log.error("PDF extraction failed: {}", e.getMessage()); + return ResponseEntity.unprocessableEntity().build(); + } + log.error("PDF similarity search failed: {}", e.getMessage(), e); + return ResponseEntity.internalServerError().build(); + + } catch (Exception e) { + log.error("PDF similarity search failed: {}", e.getMessage(), e); + return ResponseEntity.internalServerError().build(); + } + } + + /** + * Request DTO for text-based similarity search. + */ + @lombok.Data + @lombok.NoArgsConstructor + @lombok.AllArgsConstructor + public static class TextSearchRequest { + @Schema(description = "Text content to search for similar documents", required = true) + private String text; + + @Schema(description = "Number of top results to return (default: 20, max: 100)") + private Integer topK; + + @Schema(description = "Minimum similarity threshold (0.0-1.0, default: 0.5)") + private Double threshold; + } +} diff --git a/src/main/java/at/procon/ted/event/DocumentSavedEvent.java b/src/main/java/at/procon/ted/event/DocumentSavedEvent.java new file mode 100644 index 0000000..52b6576 --- /dev/null +++ b/src/main/java/at/procon/ted/event/DocumentSavedEvent.java @@ -0,0 +1,12 @@ +package at.procon.ted.event; + +import java.util.UUID; + +/** + * Event published after a document has been successfully saved to the database. + * Triggers async vectorization after transaction commit. + * + * @author Martin.Schweitzer@procon.co.at and claude.ai + */ +public record DocumentSavedEvent(UUID documentId, String publicationId) { +} diff --git a/src/main/java/at/procon/ted/event/VectorizationEventListener.java b/src/main/java/at/procon/ted/event/VectorizationEventListener.java new file mode 100644 index 0000000..6823dac --- /dev/null +++ b/src/main/java/at/procon/ted/event/VectorizationEventListener.java @@ -0,0 +1,46 @@ +package at.procon.ted.event; + +import at.procon.ted.config.TedProcessorProperties; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.apache.camel.ProducerTemplate; +import org.springframework.stereotype.Component; +import org.springframework.transaction.event.TransactionPhase; +import org.springframework.transaction.event.TransactionalEventListener; + +/** + * Event listener that triggers vectorization after document save transaction commits. + * This ensures the document is visible in the database before vectorization starts. + * + * @author Martin.Schweitzer@procon.co.at and claude.ai + */ +@Component +@RequiredArgsConstructor +@Slf4j +public class VectorizationEventListener { + + private final ProducerTemplate producerTemplate; + private final TedProcessorProperties properties; + + /** + * Triggered AFTER the transaction commits (document is now visible in DB). + * Queues the document for async vectorization. + */ + @TransactionalEventListener(phase = TransactionPhase.AFTER_COMMIT) + public void onDocumentSaved(DocumentSavedEvent event) { + if (!properties.getVectorization().isEnabled()) { + return; + } + + try { + log.debug("Document saved event received, triggering vectorization for: {}", event.documentId()); + producerTemplate.sendBodyAndHeader("direct:vectorize", null, "documentId", event.documentId()); + log.debug("Vectorization queued for document: {} (publication: {})", + event.documentId(), event.publicationId()); + } catch (Exception e) { + log.warn("Failed to queue document {} for vectorization: {}", + event.documentId(), e.getMessage()); + // Non-critical: scheduler will pick it up later + } + } +} diff --git a/src/main/java/at/procon/ted/model/dto/DocumentDtos.java b/src/main/java/at/procon/ted/model/dto/DocumentDtos.java new file mode 100644 index 0000000..78f9c03 --- /dev/null +++ b/src/main/java/at/procon/ted/model/dto/DocumentDtos.java @@ -0,0 +1,264 @@ +package at.procon.ted.model.dto; + +import at.procon.ted.model.entity.ContractNature; +import at.procon.ted.model.entity.NoticeType; +import at.procon.ted.model.entity.ProcedureType; +import at.procon.ted.model.entity.VectorizationStatus; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +import java.math.BigDecimal; +import java.time.LocalDate; +import java.time.OffsetDateTime; +import java.util.List; +import java.util.UUID; + +/** + * DTOs for procurement document API responses. + * + * @author Martin.Schweitzer@procon.co.at and claude.ai + */ +public class DocumentDtos { + + /** + * Summary DTO for list views and search results. + */ + @Data + @Builder + @NoArgsConstructor + @AllArgsConstructor + public static class DocumentSummary { + private UUID id; + private String publicationId; + private String noticeId; + private NoticeType noticeType; + private String projectTitle; + private String buyerName; + private String buyerCountryCode; + private String buyerCity; + private ContractNature contractNature; + private ProcedureType procedureType; + private LocalDate publicationDate; + private OffsetDateTime submissionDeadline; + private List cpvCodes; + private Integer totalLots; + private BigDecimal estimatedValue; + private String estimatedValueCurrency; + private Double similarity; // For semantic search results + } + + /** + * Detailed DTO for single document view. + */ + @Data + @Builder + @NoArgsConstructor + @AllArgsConstructor + public static class DocumentDetail { + private UUID id; + private String documentHash; + + // Identifiers + private String publicationId; + private String noticeId; + private String ojsId; + private String contractFolderId; + + // Classification + private NoticeType noticeType; + private String noticeSubtypeCode; + private String sdkVersion; + private String ublVersion; + private String languageCode; + + // Dates + private OffsetDateTime issueDateTime; + private LocalDate publicationDate; + private OffsetDateTime submissionDeadline; + + // Buyer information + private String buyerName; + private String buyerCountryCode; + private String buyerCity; + private String buyerPostalCode; + private String buyerNutsCode; + private String buyerActivityType; + private String buyerLegalType; + + // Project information + private String projectTitle; + private String projectDescription; + private String internalReference; + private ContractNature contractNature; + private ProcedureType procedureType; + + // Classification codes + private List cpvCodes; + private List nutsCodes; + + // Financial + private BigDecimal estimatedValue; + private String estimatedValueCurrency; + + // Lots + private Integer totalLots; + private Integer maxLotsAwarded; + private Integer maxLotsSubmitted; + private List lots; + + // Organizations + private List organizations; + + // Legal + private String regulatoryDomain; + private Boolean euFunded; + + // Vectorization + private VectorizationStatus vectorizationStatus; + private OffsetDateTime vectorizedAt; + + // Metadata + private String sourceFilename; + private Long fileSizeBytes; + private OffsetDateTime createdAt; + private OffsetDateTime updatedAt; + } + + /** + * Lot summary for document detail view. + */ + @Data + @Builder + @NoArgsConstructor + @AllArgsConstructor + public static class LotSummary { + private UUID id; + private String lotId; + private String internalId; + private String title; + private String description; + private List cpvCodes; + private List nutsCodes; + private BigDecimal estimatedValue; + private String estimatedValueCurrency; + private Double durationValue; + private String durationUnit; + private OffsetDateTime submissionDeadline; + private Boolean euFunded; + } + + /** + * Organization summary for document detail view. + */ + @Data + @Builder + @NoArgsConstructor + @AllArgsConstructor + public static class OrganizationSummary { + private UUID id; + private String orgReference; + private String role; + private String name; + private String companyId; + private String countryCode; + private String city; + private String postalCode; + private String nutsCode; + private String websiteUri; + private String email; + private String phone; + } + + /** + * Search request for structured + semantic search. + */ + @Data + @Builder + @NoArgsConstructor + @AllArgsConstructor + public static class SearchRequest { + // Structured filters + private String countryCode; + private List countryCodes; + private NoticeType noticeType; + private ContractNature contractNature; + private ProcedureType procedureType; + private String cpvPrefix; + private List cpvCodes; + private String nutsCode; + private List nutsCodes; + private LocalDate publicationDateFrom; + private LocalDate publicationDateTo; + private OffsetDateTime submissionDeadlineAfter; + private Boolean euFunded; + private String buyerNameContains; + private String projectTitleContains; + + // Semantic search + private String semanticQuery; + private Double similarityThreshold; + + // Pagination + private Integer page; + private Integer size; + private String sortBy; + private String sortDirection; + } + + /** + * Search response with pagination. + */ + @Data + @Builder + @NoArgsConstructor + @AllArgsConstructor + public static class SearchResponse { + private List documents; + private int page; + private int size; + private long totalElements; + private int totalPages; + private boolean hasNext; + private boolean hasPrevious; + } + + /** + * Statistics response. + */ + @Data + @Builder + @NoArgsConstructor + @AllArgsConstructor + public static class StatisticsResponse { + private long totalDocuments; + private long vectorizedDocuments; + private long pendingVectorization; + private long failedVectorization; + private int uniqueCountries; + private LocalDate earliestPublication; + private LocalDate latestPublication; + private long totalLots; + private List countryStatistics; + private List noticeTypeStatistics; + } + + @Data + @Builder + @NoArgsConstructor + @AllArgsConstructor + public static class CountryStats { + private String countryCode; + private long documentCount; + } + + @Data + @Builder + @NoArgsConstructor + @AllArgsConstructor + public static class NoticeTypeStats { + private NoticeType noticeType; + private long documentCount; + } +} diff --git a/src/main/java/at/procon/ted/model/entity/ContractNature.java b/src/main/java/at/procon/ted/model/entity/ContractNature.java new file mode 100644 index 0000000..9a47d07 --- /dev/null +++ b/src/main/java/at/procon/ted/model/entity/ContractNature.java @@ -0,0 +1,15 @@ +package at.procon.ted.model.entity; + +/** + * Enum representing the nature of the procurement contract. + * Maps to eForms contract-nature codelist. + * + * @author Martin.Schweitzer@procon.co.at and claude.ai + */ +public enum ContractNature { + SUPPLIES, // Procurement of goods/supplies + SERVICES, // Procurement of services + WORKS, // Procurement of construction works + MIXED, // Mixed contracts + UNKNOWN // Unknown or not specified +} diff --git a/src/main/java/at/procon/ted/model/entity/NoticeType.java b/src/main/java/at/procon/ted/model/entity/NoticeType.java new file mode 100644 index 0000000..8597ada --- /dev/null +++ b/src/main/java/at/procon/ted/model/entity/NoticeType.java @@ -0,0 +1,15 @@ +package at.procon.ted.model.entity; + +/** + * Enum representing the type of TED procurement notice. + * Based on eForms notice categories. + * + * @author Martin.Schweitzer@procon.co.at and claude.ai + */ +public enum NoticeType { + CONTRACT_NOTICE, // Contract notices (cn-*) + PRIOR_INFORMATION_NOTICE, // Prior information notices (pin-*) + CONTRACT_AWARD_NOTICE, // Contract award notices (can-*) + MODIFICATION_NOTICE, // Contract modification notices (mod-*) + OTHER // Other or unrecognized notice types +} diff --git a/src/main/java/at/procon/ted/model/entity/Organization.java b/src/main/java/at/procon/ted/model/entity/Organization.java new file mode 100644 index 0000000..d7f2652 --- /dev/null +++ b/src/main/java/at/procon/ted/model/entity/Organization.java @@ -0,0 +1,90 @@ +package at.procon.ted.model.entity; + +import jakarta.persistence.*; +import lombok.*; + +import java.time.OffsetDateTime; +import java.util.UUID; + +/** + * JPA Entity representing an organization mentioned in a procurement notice. + * Can be buyers, review bodies, service providers, etc. + * + * @author Martin.Schweitzer@procon.co.at and claude.ai + */ +@Entity +@Table(name = "organization", indexes = { + @Index(name = "idx_org_document", columnList = "document_id"), + @Index(name = "idx_org_country", columnList = "country_code") +}, uniqueConstraints = { + @UniqueConstraint(columnNames = {"document_id", "org_reference"}) +}) +@Getter +@Setter +@NoArgsConstructor +@AllArgsConstructor +@Builder +public class Organization { + + @Id + @GeneratedValue(strategy = GenerationType.UUID) + private UUID id; + + @ManyToOne(fetch = FetchType.LAZY) + @JoinColumn(name = "document_id", nullable = false) + private ProcurementDocument document; + + /** + * Internal organization reference from XML (e.g., "ORG-0001"). + */ + @Column(name = "org_reference", length = 50) + private String orgReference; + + /** + * Role of the organization (e.g., "buyer", "review-body", "ted-esen"). + */ + @Column(name = "role", length = 50) + private String role; + + @Column(name = "name", columnDefinition = "TEXT") + private String name; + + /** + * Company/tax registration ID. + */ + @Column(name = "company_id", length = 1000) + private String companyId; + + @Column(name = "country_code", length = 10) + private String countryCode; + + @Column(name = "city", length = 255) + private String city; + + @Column(name = "postal_code", length = 255) + private String postalCode; + + @Column(name = "street_name", columnDefinition = "TEXT") + private String streetName; + + @Column(name = "nuts_code", length = 10) + private String nutsCode; + + @Column(name = "website_uri", columnDefinition = "TEXT") + private String websiteUri; + + @Column(name = "email", length = 255) + private String email; + + @Column(name = "phone", length = 50) + private String phone; + + @Column(name = "created_at", nullable = false, updatable = false) + @Builder.Default + private OffsetDateTime createdAt = OffsetDateTime.now(); + + @PrePersist + protected void onCreate() { + createdAt = OffsetDateTime.now(); + } +} diff --git a/src/main/java/at/procon/ted/model/entity/ProcedureType.java b/src/main/java/at/procon/ted/model/entity/ProcedureType.java new file mode 100644 index 0000000..59d8b53 --- /dev/null +++ b/src/main/java/at/procon/ted/model/entity/ProcedureType.java @@ -0,0 +1,17 @@ +package at.procon.ted.model.entity; + +/** + * Enum representing the procurement procedure type. + * Maps to eForms procurement-procedure-type codelist. + * + * @author Martin.Schweitzer@procon.co.at and claude.ai + */ +public enum ProcedureType { + OPEN, // Open procedure + RESTRICTED, // Restricted procedure + COMPETITIVE_DIALOGUE, // Competitive dialogue + INNOVATION_PARTNERSHIP, // Innovation partnership + NEGOTIATED_WITHOUT_PUBLICATION, // Negotiated without prior publication + NEGOTIATED_WITH_PUBLICATION, // Negotiated with prior publication + OTHER // Other or not specified +} diff --git a/src/main/java/at/procon/ted/model/entity/ProcessedAttachment.java b/src/main/java/at/procon/ted/model/entity/ProcessedAttachment.java new file mode 100644 index 0000000..98af97e --- /dev/null +++ b/src/main/java/at/procon/ted/model/entity/ProcessedAttachment.java @@ -0,0 +1,146 @@ +package at.procon.ted.model.entity; + +import jakarta.persistence.*; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +import java.time.LocalDateTime; +import java.util.UUID; + +/** + * Entity for tracking processed mail attachments. + * Uses content hash for idempotent processing to avoid duplicate handling. + * + * @author Martin.Schweitzer@procon.co.at and claude.ai + */ +@Entity +@Table(name = "processed_attachment", schema = "ted", + indexes = { + @Index(name = "idx_processed_attachment_hash", columnList = "content_hash", unique = true), + @Index(name = "idx_processed_attachment_status", columnList = "processing_status"), + @Index(name = "idx_processed_attachment_type", columnList = "file_type") + }) +@Data +@Builder +@NoArgsConstructor +@AllArgsConstructor +public class ProcessedAttachment { + + @Id + @GeneratedValue(strategy = GenerationType.UUID) + private UUID id; + + /** + * SHA-256 hash of the attachment content for idempotent processing. + */ + @Column(name = "content_hash", nullable = false, unique = true, length = 64) + private String contentHash; + + /** + * Original filename of the attachment. + */ + @Column(name = "original_filename", nullable = false, length = 500) + private String originalFilename; + + /** + * Detected or declared file type (e.g., PDF, ZIP, XML). + */ + @Column(name = "file_type", length = 50) + private String fileType; + + /** + * MIME content type. + */ + @Column(name = "content_type", length = 255) + private String contentType; + + /** + * File size in bytes. + */ + @Column(name = "file_size") + private Long fileSize; + + /** + * Processing status of the attachment. + */ + @Enumerated(EnumType.STRING) + @Column(name = "processing_status", nullable = false, length = 20) + private ProcessingStatus processingStatus; + + /** + * Extracted text content (for PDF, etc.). + */ + @Column(name = "extracted_text", columnDefinition = "TEXT") + private String extractedText; + + /** + * Path where the attachment was saved. + */ + @Column(name = "saved_path", length = 1000) + private String savedPath; + + /** + * Email subject from which the attachment was extracted. + */ + @Column(name = "mail_subject", length = 500) + private String mailSubject; + + /** + * Email sender. + */ + @Column(name = "mail_from", length = 500) + private String mailFrom; + + /** + * Parent attachment hash (for files extracted from ZIP). + */ + @Column(name = "parent_hash", length = 64) + private String parentHash; + + /** + * Error message if processing failed. + */ + @Column(name = "error_message", columnDefinition = "TEXT") + private String errorMessage; + + /** + * Number of child attachments (for ZIP files). + */ + @Column(name = "child_count") + private Integer childCount; + + /** + * When the attachment was first received. + */ + @Column(name = "received_at", nullable = false) + private LocalDateTime receivedAt; + + /** + * When processing was completed. + */ + @Column(name = "processed_at") + private LocalDateTime processedAt; + + /** + * Processing status enum. + */ + public enum ProcessingStatus { + PENDING, + PROCESSING, + COMPLETED, + FAILED, + DUPLICATE + } + + @PrePersist + protected void onCreate() { + if (receivedAt == null) { + receivedAt = LocalDateTime.now(); + } + if (processingStatus == null) { + processingStatus = ProcessingStatus.PENDING; + } + } +} diff --git a/src/main/java/at/procon/ted/model/entity/ProcessingLog.java b/src/main/java/at/procon/ted/model/entity/ProcessingLog.java new file mode 100644 index 0000000..88e6ec0 --- /dev/null +++ b/src/main/java/at/procon/ted/model/entity/ProcessingLog.java @@ -0,0 +1,91 @@ +package at.procon.ted.model.entity; + +import jakarta.persistence.*; +import lombok.*; + +import java.time.OffsetDateTime; +import java.util.UUID; + +/** + * JPA Entity for logging document processing events. + * Provides audit trail and debugging information. + * + * @author Martin.Schweitzer@procon.co.at and claude.ai + */ +@Entity +@Table(name = "processing_log", indexes = { + @Index(name = "idx_log_document", columnList = "document_id"), + @Index(name = "idx_log_created", columnList = "created_at"), + @Index(name = "idx_log_event_type", columnList = "event_type") +}) +@Getter +@Setter +@NoArgsConstructor +@AllArgsConstructor +@Builder +public class ProcessingLog { + + @Id + @GeneratedValue(strategy = GenerationType.UUID) + private UUID id; + + @ManyToOne(fetch = FetchType.LAZY) + @JoinColumn(name = "document_id") + private ProcurementDocument document; + + @Column(name = "document_hash", length = 64) + private String documentHash; + + @Column(name = "event_type", nullable = false, length = 50) + private String eventType; + + @Column(name = "event_status", nullable = false, length = 20) + private String eventStatus; + + @Column(name = "message", columnDefinition = "TEXT") + private String message; + + @Column(name = "error_details", columnDefinition = "TEXT") + private String errorDetails; + + @Column(name = "source_filename", length = 500) + private String sourceFilename; + + @Column(name = "duration_ms") + private Integer durationMs; + + @Column(name = "created_at", nullable = false, updatable = false) + @Builder.Default + private OffsetDateTime createdAt = OffsetDateTime.now(); + + @PrePersist + protected void onCreate() { + createdAt = OffsetDateTime.now(); + } + + /** + * Event types for processing log entries. + */ + public static final class EventType { + public static final String RECEIVED = "RECEIVED"; + public static final String VALIDATED = "VALIDATED"; + public static final String PARSED = "PARSED"; + public static final String STORED = "STORED"; + public static final String VECTORIZED = "VECTORIZED"; + public static final String DUPLICATE = "DUPLICATE"; + public static final String ERROR = "ERROR"; + + private EventType() {} + } + + /** + * Status values for processing log entries. + */ + public static final class EventStatus { + public static final String SUCCESS = "SUCCESS"; + public static final String FAILURE = "FAILURE"; + public static final String SKIPPED = "SKIPPED"; + + private EventStatus() {} + } +} diff --git a/src/main/java/at/procon/ted/model/entity/ProcurementDocument.java b/src/main/java/at/procon/ted/model/entity/ProcurementDocument.java new file mode 100644 index 0000000..b260df9 --- /dev/null +++ b/src/main/java/at/procon/ted/model/entity/ProcurementDocument.java @@ -0,0 +1,281 @@ +package at.procon.ted.model.entity; + +import jakarta.persistence.*; +import lombok.*; +import org.hibernate.annotations.JdbcTypeCode; +import org.hibernate.type.SqlTypes; + +import java.math.BigDecimal; +import java.time.LocalDate; +import java.time.LocalTime; +import java.time.OffsetDateTime; +import java.util.ArrayList; +import java.util.List; +import java.util.UUID; + +/** + * JPA Entity representing an EU eForms procurement document from TED. + * + * Stores the complete XML document along with extracted metadata for efficient querying. + * Supports semantic search via pgvector embeddings. + * + * @author Martin.Schweitzer@procon.co.at and claude.ai + */ +@Entity +@Table(name = "procurement_document", indexes = { + @Index(name = "idx_doc_hash", columnList = "documentHash"), + @Index(name = "idx_doc_publication_id", columnList = "publicationId"), + @Index(name = "idx_doc_buyer_country", columnList = "buyerCountryCode"), + @Index(name = "idx_doc_publication_date", columnList = "publicationDate") +}) +@Getter +@Setter +@NoArgsConstructor +@AllArgsConstructor +@Builder +public class ProcurementDocument { + + @Id + @GeneratedValue(strategy = GenerationType.UUID) + private UUID id; + + /** + * SHA-256 hash of the XML content for idempotent processing. + * Used as the unique key to prevent duplicate document imports. + */ + @Column(name = "document_hash", nullable = false, unique = true, length = 64) + private String documentHash; + + // TED/eForms identifiers + @Column(name = "notice_id", length = 100) + private String noticeId; + + @Column(name = "publication_id", length = 50) + private String publicationId; + + /** + * TED notice URL generated from publication_id. + * Format: https://ted.europa.eu/en/notice/-/detail/{publication_id without leading zeros} + * Example: https://ted.europa.eu/en/notice/-/detail/786665-2025 + */ + @Column(name = "notice_url", length = 255) + private String noticeUrl; + + @Column(name = "ojs_id", length = 20) + private String ojsId; + + @Column(name = "contract_folder_id", length = 100) + private String contractFolderId; + + // Document classification + @Enumerated(EnumType.STRING) + @Column(name = "notice_type", nullable = false, length = 50) + @Builder.Default + private NoticeType noticeType = NoticeType.OTHER; + + @Column(name = "notice_subtype_code", length = 10) + private String noticeSubtypeCode; + + @Column(name = "sdk_version", length = 20) + private String sdkVersion; + + @Column(name = "ubl_version", length = 10) + private String ublVersion; + + @Column(name = "language_code", length = 10) + private String languageCode; + + // Timestamps + @Column(name = "issue_datetime") + private OffsetDateTime issueDateTime; + + @Column(name = "publication_date") + private LocalDate publicationDate; + + @Column(name = "submission_deadline") + private OffsetDateTime submissionDeadline; + + // Contracting authority (buyer) information + @Column(name = "buyer_name", columnDefinition = "TEXT") + private String buyerName; + + @Column(name = "buyer_country_code", length = 10) + private String buyerCountryCode; + + @Column(name = "buyer_city", length = 255) + private String buyerCity; + + @Column(name = "buyer_postal_code", length = 100) + private String buyerPostalCode; + + @Column(name = "buyer_nuts_code", length = 10) + private String buyerNutsCode; + + @Column(name = "buyer_activity_type", length = 50) + private String buyerActivityType; + + @Column(name = "buyer_legal_type", length = 50) + private String buyerLegalType; + + // Procurement project details + @Column(name = "project_title", columnDefinition = "TEXT") + private String projectTitle; + + @Column(name = "project_description", columnDefinition = "TEXT") + private String projectDescription; + + @Column(name = "internal_reference", length = 500) + private String internalReference; + + @Enumerated(EnumType.STRING) + @Column(name = "contract_nature", nullable = false, length = 50) + @Builder.Default + private ContractNature contractNature = ContractNature.UNKNOWN; + + @Enumerated(EnumType.STRING) + @Column(name = "procedure_type", length = 50) + @Builder.Default + private ProcedureType procedureType = ProcedureType.OTHER; + + // Classification codes (stored as PostgreSQL arrays) + @Column(name = "cpv_codes", columnDefinition = "VARCHAR(100)[]") + @JdbcTypeCode(SqlTypes.ARRAY) + private String[] cpvCodes; + + @Column(name = "nuts_codes", columnDefinition = "VARCHAR(20)[]") + @JdbcTypeCode(SqlTypes.ARRAY) + private String[] nutsCodes; + + // Financial information + @Column(name = "estimated_value", precision = 20, scale = 2) + private BigDecimal estimatedValue; + + @Column(name = "estimated_value_currency", length = 3) + private String estimatedValueCurrency; + + // Lot information + @Column(name = "total_lots") + @Builder.Default + private Integer totalLots = 0; + + @Column(name = "max_lots_awarded") + private Integer maxLotsAwarded; + + @Column(name = "max_lots_submitted") + private Integer maxLotsSubmitted; + + // Legal basis + @Column(name = "regulatory_domain", length = 50) + private String regulatoryDomain; + + @Column(name = "eu_funded") + @Builder.Default + private Boolean euFunded = false; + + /** + * Normalized text content extracted from the XML for vectorization. + * Contains title, description, buyer info, and other searchable text. + */ + @Column(name = "text_content", columnDefinition = "TEXT") + private String textContent; + + /** + * Original XML document stored in PostgreSQL native XML type. + * Enables XPath queries like: xpath('/ContractNotice/cbc:ID/text()', xml_document) + */ + @Column(name = "xml_document", nullable = false) + @JdbcTypeCode(SqlTypes.SQLXML) + private String xmlDocument; + + /** + * 1024-dimensional vector embedding for semantic search. + * Generated using intfloat/multilingual-e5-large model. + * + * Note: This field is @Transient because the pgvector type is not natively + * supported by Hibernate/JDBC. Vectors are written via native SQL queries only. + */ + @Transient + private float[] contentVector; + + // Vectorization tracking + @Enumerated(EnumType.STRING) + @Column(name = "vectorization_status", length = 50) + @Builder.Default + private VectorizationStatus vectorizationStatus = VectorizationStatus.PENDING; + + @Column(name = "vectorization_error", columnDefinition = "TEXT") + private String vectorizationError; + + @Column(name = "vectorized_at") + private OffsetDateTime vectorizedAt; + + @Column(name = "embedding_token_count") + private Integer embeddingTokenCount; + + // Processing metadata + @Column(name = "source_filename", length = 500) + private String sourceFilename; + + @Column(name = "source_path", columnDefinition = "TEXT") + private String sourcePath; + + @Column(name = "file_size_bytes") + private Long fileSizeBytes; + + // Audit fields + @Column(name = "created_at", nullable = false, updatable = false) + @Builder.Default + private OffsetDateTime createdAt = OffsetDateTime.now(); + + @Column(name = "updated_at") + @Builder.Default + private OffsetDateTime updatedAt = OffsetDateTime.now(); + + @Column(name = "processing_duration_ms") + private Integer processingDurationMs; + + // Relationships + @OneToMany(mappedBy = "document", cascade = CascadeType.ALL, orphanRemoval = true) + @Builder.Default + private List lots = new ArrayList<>(); + + @OneToMany(mappedBy = "document", cascade = CascadeType.ALL, orphanRemoval = true) + @Builder.Default + private List organizations = new ArrayList<>(); + + // Helper methods + public void addLot(ProcurementLot lot) { + lots.add(lot); + lot.setDocument(this); + } + + public void addOrganization(Organization organization) { + organizations.add(organization); + organization.setDocument(this); + } + + @PrePersist + protected void onCreate() { + createdAt = OffsetDateTime.now(); + updatedAt = OffsetDateTime.now(); + generateNoticeUrl(); + } + + @PreUpdate + protected void onUpdate() { + updatedAt = OffsetDateTime.now(); + generateNoticeUrl(); + } + + /** + * Generates TED notice URL from publication_id. + * Format: https://ted.europa.eu/en/notice/-/detail/{publication_id without leading zeros} + */ + private void generateNoticeUrl() { + if (publicationId != null && !publicationId.isEmpty()) { + // Remove leading zeros from publication_id + String cleanId = publicationId.replaceFirst("^0+", ""); + this.noticeUrl = "https://ted.europa.eu/en/notice/-/detail/" + cleanId; + } + } +} diff --git a/src/main/java/at/procon/ted/model/entity/ProcurementLot.java b/src/main/java/at/procon/ted/model/entity/ProcurementLot.java new file mode 100644 index 0000000..b6ff6ae --- /dev/null +++ b/src/main/java/at/procon/ted/model/entity/ProcurementLot.java @@ -0,0 +1,92 @@ +package at.procon.ted.model.entity; + +import jakarta.persistence.*; +import lombok.*; +import org.hibernate.annotations.JdbcTypeCode; +import org.hibernate.type.SqlTypes; + +import java.math.BigDecimal; +import java.time.OffsetDateTime; +import java.util.UUID; + +/** + * JPA Entity representing a lot within a procurement notice. + * A procurement document can have multiple lots. + * + * @author Martin.Schweitzer@procon.co.at and claude.ai + */ +@Entity +@Table(name = "procurement_lot", indexes = { + @Index(name = "idx_lot_document", columnList = "document_id") +}, uniqueConstraints = { + @UniqueConstraint(columnNames = {"document_id", "lot_id"}) +}) +@Getter +@Setter +@NoArgsConstructor +@AllArgsConstructor +@Builder +public class ProcurementLot { + + @Id + @GeneratedValue(strategy = GenerationType.UUID) + private UUID id; + + @ManyToOne(fetch = FetchType.LAZY) + @JoinColumn(name = "document_id", nullable = false) + private ProcurementDocument document; + + /** + * Lot identifier from the XML (e.g., "LOT-0001"). + */ + @Column(name = "lot_id", nullable = false, length = 50) + private String lotId; + + /** + * Buyer's internal reference for this lot. + */ + @Column(name = "internal_id", columnDefinition = "TEXT") + private String internalId; + + @Column(name = "title", columnDefinition = "TEXT") + private String title; + + @Column(name = "description", columnDefinition = "TEXT") + private String description; + + @Column(name = "cpv_codes", columnDefinition = "VARCHAR(100)[]") + @JdbcTypeCode(SqlTypes.ARRAY) + private String[] cpvCodes; + + @Column(name = "nuts_codes", columnDefinition = "VARCHAR(20)[]") + @JdbcTypeCode(SqlTypes.ARRAY) + private String[] nutsCodes; + + @Column(name = "estimated_value", precision = 20, scale = 2) + private BigDecimal estimatedValue; + + @Column(name = "estimated_value_currency", length = 3) + private String estimatedValueCurrency; + + @Column(name = "duration_value") + private Double durationValue; + + @Column(name = "duration_unit", length = 20) + private String durationUnit; + + @Column(name = "submission_deadline") + private OffsetDateTime submissionDeadline; + + @Column(name = "eu_funded") + @Builder.Default + private Boolean euFunded = false; + + @Column(name = "created_at", nullable = false, updatable = false) + @Builder.Default + private OffsetDateTime createdAt = OffsetDateTime.now(); + + @PrePersist + protected void onCreate() { + createdAt = OffsetDateTime.now(); + } +} diff --git a/src/main/java/at/procon/ted/model/entity/TedDailyPackage.java b/src/main/java/at/procon/ted/model/entity/TedDailyPackage.java new file mode 100644 index 0000000..935883e --- /dev/null +++ b/src/main/java/at/procon/ted/model/entity/TedDailyPackage.java @@ -0,0 +1,163 @@ +package at.procon.ted.model.entity; + +import jakarta.persistence.*; +import lombok.*; +import java.time.OffsetDateTime; +import java.util.UUID; + +/** + * JPA Entity for tracking downloaded TED Daily Packages. + * + * Stores information about each downloaded package to ensure idempotency + * and track download progress. + * + * @author Martin.Schweitzer@procon.co.at and claude.ai + */ +@Entity +@Table(name = "ted_daily_package", indexes = { + @Index(name = "idx_package_identifier", columnList = "packageIdentifier", unique = true), + @Index(name = "idx_package_year_serial", columnList = "year,serialNumber", unique = true), + @Index(name = "idx_package_status", columnList = "downloadStatus"), + @Index(name = "idx_package_downloaded_at", columnList = "downloadedAt") +}) +@Getter +@Setter +@NoArgsConstructor +@AllArgsConstructor +@Builder +public class TedDailyPackage { + + @Id + @GeneratedValue(strategy = GenerationType.UUID) + private UUID id; + + /** + * Package identifier in format YYYYSSSSS (e.g. 202400001) + */ + @Column(name = "package_identifier", nullable = false, unique = true, length = 20) + private String packageIdentifier; + + /** + * Year of the package (e.g. 2024) + */ + @Column(name = "year", nullable = false) + private Integer year; + + /** + * Serial number within the year (e.g. 1, 2, 3...) + */ + @Column(name = "serial_number", nullable = false) + private Integer serialNumber; + + /** + * Download URL + */ + @Column(name = "download_url", nullable = false, length = 500) + private String downloadUrl; + + /** + * SHA-256 hash of the downloaded tar.gz file + */ + @Column(name = "file_hash", length = 64) + private String fileHash; + + /** + * Number of extracted XML files + */ + @Column(name = "xml_file_count") + private Integer xmlFileCount; + + /** + * Number of successfully processed documents + */ + @Column(name = "processed_count") + @Builder.Default + private Integer processedCount = 0; + + /** + * Number of failed documents + */ + @Column(name = "failed_count") + @Builder.Default + private Integer failedCount = 0; + + /** + * Download status + */ + @Enumerated(EnumType.STRING) + @Column(name = "download_status", nullable = false, length = 30) + @Builder.Default + private DownloadStatus downloadStatus = DownloadStatus.PENDING; + + /** + * Error message for failed download + */ + @Column(name = "error_message", columnDefinition = "TEXT") + private String errorMessage; + + /** + * Timestamp of successful download + */ + @Column(name = "downloaded_at") + private OffsetDateTime downloadedAt; + + /** + * Timestamp of complete processing + */ + @Column(name = "processed_at") + private OffsetDateTime processedAt; + + /** + * Download duration in milliseconds + */ + @Column(name = "download_duration_ms") + private Long downloadDurationMs; + + /** + * Processing duration in milliseconds + */ + @Column(name = "processing_duration_ms") + private Long processingDurationMs; + + /** + * Timestamp of creation + */ + @Column(name = "created_at", nullable = false) + @Builder.Default + private OffsetDateTime createdAt = OffsetDateTime.now(); + + /** + * Timestamp of last update + */ + @Column(name = "updated_at", nullable = false) + @Builder.Default + private OffsetDateTime updatedAt = OffsetDateTime.now(); + + /** + * Download status enum + */ + public enum DownloadStatus { + PENDING, // Not yet downloaded + DOWNLOADING, // Download in progress + DOWNLOADED, // Downloaded, not processed + PROCESSING, // Being processed + COMPLETED, // Fully processed + FAILED, // Download or processing failed + NOT_FOUND // Package does not exist (404) + } + + @PrePersist + protected void onCreate() { + if (createdAt == null) { + createdAt = OffsetDateTime.now(); + } + if (updatedAt == null) { + updatedAt = OffsetDateTime.now(); + } + } + + @PreUpdate + protected void onUpdate() { + updatedAt = OffsetDateTime.now(); + } +} diff --git a/src/main/java/at/procon/ted/model/entity/VectorizationStatus.java b/src/main/java/at/procon/ted/model/entity/VectorizationStatus.java new file mode 100644 index 0000000..fe706c4 --- /dev/null +++ b/src/main/java/at/procon/ted/model/entity/VectorizationStatus.java @@ -0,0 +1,16 @@ +package at.procon.ted.model.entity; + +/** + * Enum representing the status of document vectorization. + * Used for tracking asynchronous vectorization processing. + * + * @author Martin.Schweitzer@procon.co.at and claude.ai + */ +public enum VectorizationStatus { + PENDING, // Awaiting vectorization + PROCESSING, // Currently being vectorized + COMPLETED, // Successfully vectorized + COMPLETED_Temporal, // Legacy: Successfully vectorized (temporal) + FAILED, // Vectorization failed + SKIPPED // Skipped (e.g., no text content) +} diff --git a/src/main/java/at/procon/ted/repository/ProcessedAttachmentRepository.java b/src/main/java/at/procon/ted/repository/ProcessedAttachmentRepository.java new file mode 100644 index 0000000..bc3d592 --- /dev/null +++ b/src/main/java/at/procon/ted/repository/ProcessedAttachmentRepository.java @@ -0,0 +1,58 @@ +package at.procon.ted.repository; + +import at.procon.ted.model.entity.ProcessedAttachment; +import at.procon.ted.model.entity.ProcessedAttachment.ProcessingStatus; +import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.data.jpa.repository.Query; +import org.springframework.data.repository.query.Param; +import org.springframework.stereotype.Repository; + +import java.util.List; +import java.util.Optional; +import java.util.UUID; + +/** + * Repository for ProcessedAttachment entity. + * Provides idempotent attachment tracking via content hash. + * + * @author Martin.Schweitzer@procon.co.at and claude.ai + */ +@Repository +public interface ProcessedAttachmentRepository extends JpaRepository { + + /** + * Find attachment by content hash for idempotency check. + */ + Optional findByContentHash(String contentHash); + + /** + * Check if an attachment with given hash already exists. + */ + boolean existsByContentHash(String contentHash); + + /** + * Find all attachments with given processing status. + */ + List findByProcessingStatus(ProcessingStatus status); + + /** + * Find all child attachments (extracted from a ZIP). + */ + List findByParentHash(String parentHash); + + /** + * Find attachments by file type. + */ + List findByFileType(String fileType); + + /** + * Count attachments by status. + */ + long countByProcessingStatus(ProcessingStatus status); + + /** + * Find pending attachments for retry processing. + */ + @Query("SELECT a FROM ProcessedAttachment a WHERE a.processingStatus = 'PENDING' OR a.processingStatus = 'FAILED' ORDER BY a.receivedAt ASC") + List findPendingOrFailed(); +} diff --git a/src/main/java/at/procon/ted/repository/ProcessingLogRepository.java b/src/main/java/at/procon/ted/repository/ProcessingLogRepository.java new file mode 100644 index 0000000..be282d4 --- /dev/null +++ b/src/main/java/at/procon/ted/repository/ProcessingLogRepository.java @@ -0,0 +1,31 @@ +package at.procon.ted.repository; + +import at.procon.ted.model.entity.ProcessingLog; +import org.springframework.data.domain.Page; +import org.springframework.data.domain.Pageable; +import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.data.jpa.repository.Query; +import org.springframework.data.repository.query.Param; +import org.springframework.stereotype.Repository; + +import java.time.OffsetDateTime; +import java.util.List; +import java.util.UUID; + +/** + * Repository for ProcessingLog entities. + * + * @author Martin.Schweitzer@procon.co.at and claude.ai + */ +@Repository +public interface ProcessingLogRepository extends JpaRepository { + + List findByDocumentIdOrderByCreatedAtDesc(UUID documentId); + + List findByDocumentHashOrderByCreatedAtDesc(String documentHash); + + Page findByEventTypeOrderByCreatedAtDesc(String eventType, Pageable pageable); + + @Query("SELECT l FROM ProcessingLog l WHERE l.createdAt >= :since ORDER BY l.createdAt DESC") + List findRecentLogs(@Param("since") OffsetDateTime since); +} diff --git a/src/main/java/at/procon/ted/repository/ProcurementDocumentRepository.java b/src/main/java/at/procon/ted/repository/ProcurementDocumentRepository.java new file mode 100644 index 0000000..e35f16f --- /dev/null +++ b/src/main/java/at/procon/ted/repository/ProcurementDocumentRepository.java @@ -0,0 +1,232 @@ +package at.procon.ted.repository; + +import at.procon.ted.model.entity.*; +import org.springframework.data.domain.Page; +import org.springframework.data.domain.Pageable; +import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.data.jpa.repository.JpaSpecificationExecutor; +import org.springframework.data.jpa.repository.Modifying; +import org.springframework.data.jpa.repository.Query; +import org.springframework.data.repository.query.Param; +import org.springframework.stereotype.Repository; + +import java.time.LocalDate; +import java.time.OffsetDateTime; +import java.util.List; +import java.util.Optional; +import java.util.UUID; + +/** + * Repository for ProcurementDocument entities. + * Provides standard CRUD operations plus custom queries for search and statistics. + * + * @author Martin.Schweitzer@procon.co.at and claude.ai + */ +@Repository +public interface ProcurementDocumentRepository extends + JpaRepository, + JpaSpecificationExecutor { + + /** + * Find document by its SHA-256 hash (for idempotent processing). + */ + Optional findByDocumentHash(String documentHash); + + /** + * Check if a document with the given hash already exists. + */ + boolean existsByDocumentHash(String documentHash); + + /** + * Find document by TED publication ID. + */ + Optional findByPublicationId(String publicationId); + + /** + * Find document by TED notice URL. + * Example: https://ted.europa.eu/en/notice/-/detail/786665-2025 + */ + Optional findByNoticeUrl(String noticeUrl); + + /** + * Find documents by buyer country code. + */ + Page findByBuyerCountryCode(String countryCode, Pageable pageable); + + /** + * Find documents by notice type. + */ + Page findByNoticeType(NoticeType noticeType, Pageable pageable); + + /** + * Find documents pending vectorization (ordered by creation date). + */ + @Query("SELECT d FROM ProcurementDocument d WHERE d.vectorizationStatus = :status ORDER BY d.createdAt ASC") + List findByVectorizationStatus( + @Param("status") VectorizationStatus status, + Pageable pageable); + + /** + * Get only text content for vectorization (memory efficient - does not load XML). + */ + @Query("SELECT p.textContent FROM ProcurementDocument p WHERE p.id = :id") + String findTextContentById(@Param("id") UUID id); + + /** + * Find document IDs by vectorization status (memory efficient - does not load full entities). + */ + @Query("SELECT d.id FROM ProcurementDocument d WHERE d.vectorizationStatus = :status ORDER BY d.createdAt ASC") + List findIdsByVectorizationStatus(@Param("status") VectorizationStatus status, Pageable pageable); + + /** + * Update vectorization status for a document. + */ + @Modifying + @Query("UPDATE ProcurementDocument d SET d.vectorizationStatus = :status, " + + "d.vectorizationError = :error, d.vectorizedAt = :vectorizedAt " + + "WHERE d.id = :id") + int updateVectorizationStatus( + @Param("id") UUID id, + @Param("status") VectorizationStatus status, + @Param("error") String error, + @Param("vectorizedAt") OffsetDateTime vectorizedAt); + + /** + * Update document vector after successful vectorization. + */ + @Modifying + @Query(value = "UPDATE ted.procurement_document SET content_vector = CAST(:vectorData AS vector), " + + "vectorization_status = 'COMPLETED', vectorized_at = CURRENT_TIMESTAMP, " + + "vectorization_error = NULL, embedding_token_count = :tokenCount WHERE id = :id", + nativeQuery = true) + int updateContentVector(@Param("id") UUID id, @Param("vectorData") String vectorData, @Param("tokenCount") Integer tokenCount); + + /** + * Simple semantic search using cosine similarity without filters. + * Returns document IDs and similarity scores sorted by similarity. + * Note: We only select id and similarity to avoid XML column deserialization issues. + */ + @Query(value = """ + SELECT d.id, 1 - (d.content_vector <=> CAST(:queryVector AS vector)) AS similarity + FROM ted.procurement_document d + WHERE d.content_vector IS NOT NULL + AND (1 - (d.content_vector <=> CAST(:queryVector AS vector))) >= :threshold + ORDER BY similarity DESC + LIMIT :limit + """, nativeQuery = true) + List findBySimilarity( + @Param("queryVector") String queryVector, + @Param("threshold") double threshold, + @Param("limit") int limit); + + /** + * Semantic search using cosine similarity with filters. + * Returns documents sorted by similarity score. + */ + @Query(value = """ + SELECT d.*, 1 - (d.content_vector <=> CAST(:queryVector AS vector)) AS similarity + FROM ted.procurement_document d + WHERE d.content_vector IS NOT NULL + AND (1 - (d.content_vector <=> CAST(:queryVector AS vector))) >= :threshold + AND (:countryCode IS NULL OR d.buyer_country_code = :countryCode) + AND (:noticeType IS NULL OR d.notice_type = :noticeType) + AND (:contractNature IS NULL OR d.contract_nature = :contractNature) + AND (:cpvPrefix IS NULL OR EXISTS ( + SELECT 1 FROM unnest(d.cpv_codes) code WHERE code LIKE :cpvPrefix || '%' + )) + AND (CAST(:dateFrom AS DATE) IS NULL OR d.publication_date >= CAST(:dateFrom AS DATE)) + AND (CAST(:dateTo AS DATE) IS NULL OR d.publication_date <= CAST(:dateTo AS DATE)) + ORDER BY similarity DESC + LIMIT :limit OFFSET :offset + """, nativeQuery = true) + List findBySemanticSearch( + @Param("queryVector") String queryVector, + @Param("threshold") double threshold, + @Param("countryCode") String countryCode, + @Param("noticeType") String noticeType, + @Param("contractNature") String contractNature, + @Param("cpvPrefix") String cpvPrefix, + @Param("dateFrom") LocalDate dateFrom, + @Param("dateTo") LocalDate dateTo, + @Param("limit") int limit, + @Param("offset") int offset); + + /** + * Count total documents for semantic search (for pagination). + */ + @Query(value = """ + SELECT COUNT(*) + FROM ted.procurement_document d + WHERE d.content_vector IS NOT NULL + AND (1 - (d.content_vector <=> CAST(:queryVector AS vector))) >= :threshold + AND (:countryCode IS NULL OR d.buyer_country_code = :countryCode) + AND (:noticeType IS NULL OR d.notice_type = :noticeType) + AND (:contractNature IS NULL OR d.contract_nature = :contractNature) + """, nativeQuery = true) + long countBySemanticSearch( + @Param("queryVector") String queryVector, + @Param("threshold") double threshold, + @Param("countryCode") String countryCode, + @Param("noticeType") String noticeType, + @Param("contractNature") String contractNature); + + /** + * Get document count by country. + */ + @Query("SELECT d.buyerCountryCode, COUNT(d) FROM ProcurementDocument d " + + "WHERE d.buyerCountryCode IS NOT NULL " + + "GROUP BY d.buyerCountryCode ORDER BY COUNT(d) DESC") + List countByCountry(); + + /** + * Get document count by notice type. + */ + @Query("SELECT d.noticeType, COUNT(d) FROM ProcurementDocument d " + + "GROUP BY d.noticeType ORDER BY COUNT(d) DESC") + List countByNoticeType(); + + /** + * Get vectorization statistics. + */ + @Query("SELECT d.vectorizationStatus, COUNT(d) FROM ProcurementDocument d " + + "GROUP BY d.vectorizationStatus") + List countByVectorizationStatus(); + + /** + * Find documents with submission deadline in the future. + */ + @Query("SELECT d FROM ProcurementDocument d " + + "WHERE d.submissionDeadline > CURRENT_TIMESTAMP " + + "ORDER BY d.submissionDeadline ASC") + Page findUpcomingDeadlines(Pageable pageable); + + /** + * Full-text search on text content using PostgreSQL trigram matching. + */ + @Query(value = "SELECT * FROM ted.procurement_document d " + + "WHERE d.text_content ILIKE '%' || :query || '%' " + + "ORDER BY similarity(d.text_content, :query) DESC", + nativeQuery = true) + List findByTextContentContaining(@Param("query") String query, Pageable pageable); + + /** + * Delete all documents created before the specified date. + * Cascading deletes will automatically remove related lots, organizations, and logs. + * + * @param cutoffDate Documents created before this date will be deleted + * @return Number of deleted documents + */ + @Modifying + @Query(value = "DELETE FROM ted.procurement_document WHERE created_at < :cutoffDate", nativeQuery = true) + int deleteByCreatedAtBefore(@Param("cutoffDate") OffsetDateTime cutoffDate); + + /** + * Count documents created before the specified date. + * + * @param cutoffDate Documents created before this date will be counted + * @return Number of documents + */ + long countByCreatedAtBefore(OffsetDateTime cutoffDate); +} + + diff --git a/src/main/java/at/procon/ted/repository/TedDailyPackageRepository.java b/src/main/java/at/procon/ted/repository/TedDailyPackageRepository.java new file mode 100644 index 0000000..970a70f --- /dev/null +++ b/src/main/java/at/procon/ted/repository/TedDailyPackageRepository.java @@ -0,0 +1,90 @@ +package at.procon.ted.repository; + +import at.procon.ted.model.entity.TedDailyPackage; +import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.data.jpa.repository.Query; +import org.springframework.data.repository.query.Param; +import org.springframework.stereotype.Repository; + +import java.util.List; +import java.util.Optional; +import java.util.UUID; + +/** + * Repository für TED Daily Package Entities. + * + * @author Martin.Schweitzer@procon.co.at and claude.ai + */ +@Repository +public interface TedDailyPackageRepository extends JpaRepository { + + /** + * Findet ein Package anhand des Identifiers (YYYYSSSSS). + */ + Optional findByPackageIdentifier(String packageIdentifier); + + /** + * Findet ein Package anhand von Jahr und Seriennummer. + */ + Optional findByYearAndSerialNumber(Integer year, Integer serialNumber); + + /** + * Prüft ob ein Package bereits existiert. + */ + boolean existsByPackageIdentifier(String packageIdentifier); + + /** + * Findet das zuletzt erfolgreich heruntergeladene Package. + */ + @Query("SELECT p FROM TedDailyPackage p " + + "WHERE p.downloadStatus IN ('DOWNLOADED', 'PROCESSING', 'COMPLETED') " + + "ORDER BY p.year DESC, p.serialNumber DESC " + + "LIMIT 1") + Optional findLatestDownloaded(); + + /** + * Findet das Package mit der höchsten Seriennummer für ein bestimmtes Jahr. + */ + @Query("SELECT p FROM TedDailyPackage p " + + "WHERE p.year = :year " + + "ORDER BY p.serialNumber DESC " + + "LIMIT 1") + Optional findLatestByYear(@Param("year") Integer year); + + /** + * Findet das Package mit der niedrigsten Seriennummer für ein bestimmtes Jahr. + */ + @Query("SELECT p FROM TedDailyPackage p " + + "WHERE p.year = :year " + + "ORDER BY p.serialNumber ASC " + + "LIMIT 1") + Optional findFirstByYear(@Param("year") Integer year); + + /** + * Findet alle Packages mit einem bestimmten Status. + */ + List findByDownloadStatus(TedDailyPackage.DownloadStatus status); + + /** + * Findet alle Packages die noch verarbeitet werden müssen. + */ + @Query("SELECT p FROM TedDailyPackage p " + + "WHERE p.downloadStatus IN ('DOWNLOADED', 'PROCESSING') " + + "ORDER BY p.year ASC, p.serialNumber ASC") + List findPendingProcessing(); + + /** + * Checks if there is a NOT_FOUND package directly after the given serial number. + * Returns 1 if the next package (fromSerial+1) is NOT_FOUND, 0 otherwise. + */ + @Query("SELECT COUNT(p) FROM TedDailyPackage p " + + "WHERE p.year = :year " + + "AND p.serialNumber = :fromSerial + 1 " + + "AND p.downloadStatus = 'NOT_FOUND'") + long countConsecutiveNotFound(@Param("year") Integer year, @Param("fromSerial") Integer fromSerial); + + /** + * Findet alle Packages eines Jahres. + */ + List findByYearOrderBySerialNumberAsc(Integer year); +} diff --git a/src/main/java/at/procon/ted/service/BatchDocumentProcessingService.java b/src/main/java/at/procon/ted/service/BatchDocumentProcessingService.java new file mode 100644 index 0000000..0412b15 --- /dev/null +++ b/src/main/java/at/procon/ted/service/BatchDocumentProcessingService.java @@ -0,0 +1,183 @@ +package at.procon.ted.service; + +import at.procon.ted.model.entity.ProcurementDocument; +import at.procon.ted.model.entity.ProcessingLog; +import at.procon.ted.repository.ProcurementDocumentRepository; +import at.procon.ted.util.HashUtils; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; +import java.util.UUID; + +/** + * Service for batch processing of TED procurement documents. + * + * Processes all XML files from a Daily Package in a single transaction: + * 1. Parse all XML files + * 2. Check for duplicates + * 3. Batch insert all new documents (saveAll) + * 4. Trigger vectorization for all inserted documents + * + * This is much more efficient than processing documents one by one. + * + * @author Martin.Schweitzer@procon.co.at and claude.ai + */ +@Service +@RequiredArgsConstructor +@Slf4j +public class BatchDocumentProcessingService { + + private final XmlParserService xmlParserService; + private final ProcurementDocumentRepository documentRepository; + private final ProcessingLogService processingLogService; + + /** + * Process a batch of XML files from a Daily Package. + * + * @param xmlFiles List of XML file paths to process + * @return Batch processing result with statistics + */ + @Transactional + public BatchProcessingResult processBatch(List xmlFiles) { + long startTime = System.currentTimeMillis(); + + List documentsToInsert = new ArrayList<>(); + List duplicateHashes = new ArrayList<>(); + List errors = new ArrayList<>(); + + log.debug("Processing batch of {} XML files", xmlFiles.size()); + + // Step 1: Parse all XML files and check for duplicates + for (Path xmlFile : xmlFiles) { + try { + // Read XML content + String xmlContent = Files.readString(xmlFile, StandardCharsets.UTF_8); + String filename = xmlFile.getFileName().toString(); + String filePath = xmlFile.toString(); + long fileSize = Files.size(xmlFile); + + // Compute document hash + String documentHash = HashUtils.computeSha256(xmlContent); + + // Check for duplicate + if (documentRepository.existsByDocumentHash(documentHash)) { + log.debug("Duplicate document detected, skipping: {} (hash: {})", filename, documentHash); + duplicateHashes.add(documentHash); + + processingLogService.logEvent( + null, documentHash, ProcessingLog.EventType.DUPLICATE, + ProcessingLog.EventStatus.SKIPPED, + "Document already exists in database", null, filename, 0); + continue; + } + + // Parse XML document + ProcurementDocument document = xmlParserService.parseDocument(xmlContent); + document.setDocumentHash(documentHash); + document.setSourceFilename(filename); + document.setSourcePath(filePath); + document.setFileSizeBytes(fileSize); + + documentsToInsert.add(document); + + } catch (IOException e) { + log.warn("Failed to read XML file {}: {}", xmlFile, e.getMessage()); + errors.add(new ProcessingError(xmlFile.toString(), "File read error: " + e.getMessage())); + + } catch (XmlParserService.XmlParsingException e) { + log.warn("Failed to parse XML file {}: {}", xmlFile, e.getMessage()); + errors.add(new ProcessingError(xmlFile.toString(), "XML parsing error: " + e.getMessage())); + + String hash = "unknown"; + try { + String xmlContent = Files.readString(xmlFile, StandardCharsets.UTF_8); + hash = HashUtils.computeSha256(xmlContent); + } catch (IOException ignored) {} + + processingLogService.logEvent( + null, hash, ProcessingLog.EventType.ERROR, + ProcessingLog.EventStatus.FAILURE, + "XML parsing failed", e.getMessage(), xmlFile.getFileName().toString(), 0); + + } catch (Exception e) { + log.error("Unexpected error processing file {}: {}", xmlFile, e.getMessage(), e); + errors.add(new ProcessingError(xmlFile.toString(), "Unexpected error: " + e.getMessage())); + } + } + + // Step 2: Batch insert all new documents + List insertedDocumentIds = new ArrayList<>(); + if (!documentsToInsert.isEmpty()) { + log.info("Batch inserting {} new documents into database", documentsToInsert.size()); + + List savedDocuments = documentRepository.saveAll(documentsToInsert); + + // Log success for each document + for (ProcurementDocument doc : savedDocuments) { + insertedDocumentIds.add(doc.getId()); + + processingLogService.logEvent( + doc, doc.getDocumentHash(), ProcessingLog.EventType.STORED, + ProcessingLog.EventStatus.SUCCESS, + "Document parsed and stored successfully (batch)", null, + doc.getSourceFilename(), 0); + } + + log.info("Successfully inserted {} documents in batch", savedDocuments.size()); + } + + // Step 3: Vectorization will be picked up by VectorizationRoute scheduler + // No need to publish individual events - the scheduler checks for PENDING documents + // This avoids creating 149k+ inflight exchanges in the queue + if (!insertedDocumentIds.isEmpty()) { + log.debug("Inserted {} documents with vectorization_status=PENDING, " + + "will be picked up by vectorization scheduler", insertedDocumentIds.size()); + } + + long duration = System.currentTimeMillis() - startTime; + log.info("Batch processing completed: {} inserted, {} duplicates, {} errors in {}ms", + insertedDocumentIds.size(), duplicateHashes.size(), errors.size(), duration); + + return new BatchProcessingResult( + insertedDocumentIds.size(), + duplicateHashes.size(), + errors.size(), + duration, + insertedDocumentIds, + errors + ); + } + + /** + * Result of batch processing operation. + */ + public record BatchProcessingResult( + int insertedCount, + int duplicateCount, + int errorCount, + long durationMs, + List insertedDocumentIds, + List errors + ) { + public int getTotalProcessed() { + return insertedCount + duplicateCount + errorCount; + } + + public boolean hasErrors() { + return errorCount > 0; + } + } + + /** + * Processing error details. + */ + public record ProcessingError(String filename, String errorMessage) {} +} diff --git a/src/main/java/at/procon/ted/service/DataCleanupService.java b/src/main/java/at/procon/ted/service/DataCleanupService.java new file mode 100644 index 0000000..6a36c70 --- /dev/null +++ b/src/main/java/at/procon/ted/service/DataCleanupService.java @@ -0,0 +1,95 @@ +package at.procon.ted.service; + +import at.procon.ted.repository.ProcurementDocumentRepository; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; + +import java.time.OffsetDateTime; + +/** + * Service for cleaning up old procurement documents. + * + * Automatically deletes documents older than a configurable retention period. + * Default: 7 years + * + * @author Martin.Schweitzer@procon.co.at and claude.ai + */ +@Service +@RequiredArgsConstructor +@Slf4j +public class DataCleanupService { + + private final ProcurementDocumentRepository documentRepository; + + @Value("${ted.cleanup.retention-years:7}") + private int retentionYears; + + @Value("${ted.cleanup.enabled:false}") + private boolean cleanupEnabled; + + /** + * Delete procurement documents older than retention period. + * Runs daily at 2 AM. + */ + @Scheduled(cron = "${ted.cleanup.cron:0 0 2 * * *}") + @Transactional + public void deleteOldDocuments() { + if (!cleanupEnabled) { + log.debug("Data cleanup is disabled"); + return; + } + + OffsetDateTime cutoffDate = OffsetDateTime.now().minusYears(retentionYears); + + log.info("Starting cleanup of documents older than {} (retention: {} years)", + cutoffDate, retentionYears); + + try { + int deletedCount = documentRepository.deleteByCreatedAtBefore(cutoffDate); + + if (deletedCount > 0) { + log.info("✅ Deleted {} documents older than {} years", deletedCount, retentionYears); + } else { + log.debug("No documents to delete"); + } + + } catch (Exception e) { + log.error("❌ Error during cleanup: {}", e.getMessage(), e); + } + } + + /** + * Manually trigger cleanup of old documents. + * + * @param years Number of years to retain (documents older than this will be deleted) + * @return Number of deleted documents + */ + @Transactional + public int deleteDocumentsOlderThan(int years) { + OffsetDateTime cutoffDate = OffsetDateTime.now().minusYears(years); + + log.info("Manual cleanup: Deleting documents older than {} (retention: {} years)", + cutoffDate, years); + + int deletedCount = documentRepository.deleteByCreatedAtBefore(cutoffDate); + + log.info("✅ Manually deleted {} documents", deletedCount); + + return deletedCount; + } + + /** + * Get count of documents that would be deleted. + * + * @param years Number of years to check + * @return Number of documents older than specified years + */ + public long countDocumentsOlderThan(int years) { + OffsetDateTime cutoffDate = OffsetDateTime.now().minusYears(years); + return documentRepository.countByCreatedAtBefore(cutoffDate); + } +} diff --git a/src/main/java/at/procon/ted/service/DocumentProcessingService.java b/src/main/java/at/procon/ted/service/DocumentProcessingService.java new file mode 100644 index 0000000..0d480fc --- /dev/null +++ b/src/main/java/at/procon/ted/service/DocumentProcessingService.java @@ -0,0 +1,195 @@ +package at.procon.ted.service; + +import at.procon.ted.config.TedProcessorProperties; +import at.procon.ted.event.DocumentSavedEvent; +import at.procon.ted.model.entity.*; +import at.procon.ted.repository.ProcurementDocumentRepository; +import at.procon.ted.util.HashUtils; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.context.ApplicationEventPublisher; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; + +import java.time.OffsetDateTime; +import java.util.Optional; + +/** + * Service for processing TED procurement documents. + * + * Handles the complete processing pipeline: + * 1. Hash computation for idempotency check + * 2. XML parsing and data extraction + * 3. Database storage + * + * Note: Vectorization is triggered separately by VectorizationRoute via Camel wireTap. + * + * @author Martin.Schweitzer@procon.co.at and claude.ai + */ +@Service +@RequiredArgsConstructor +@Slf4j +public class DocumentProcessingService { + + private final XmlParserService xmlParserService; + private final ProcurementDocumentRepository documentRepository; + private final ProcessingLogService processingLogService; + private final TedProcessorProperties properties; + private final ApplicationEventPublisher eventPublisher; + + /** + * Process an XML document from the file system. + * + * @param xmlContent The XML content + * @param filename Source filename + * @param filePath Source file path + * @param fileSize File size in bytes + * @return Processing result with document ID if successful + */ + @Transactional + public ProcessingResult processDocument(String xmlContent, String filename, String filePath, Long fileSize) { + long startTime = System.currentTimeMillis(); + + // Step 1: Compute document hash + String documentHash = HashUtils.computeSha256(xmlContent); + log.debug("Processing document {} with hash {}", filename, documentHash); + + // Step 2: Check for duplicate (idempotent processing) + if (documentRepository.existsByDocumentHash(documentHash)) { + log.debug("Duplicate document detected, skipping: {} (hash: {})", filename, documentHash); + processingLogService.logEvent( + null, documentHash, ProcessingLog.EventType.DUPLICATE, + ProcessingLog.EventStatus.SKIPPED, + "Document already exists in database", null, filename, + (int) (System.currentTimeMillis() - startTime)); + return ProcessingResult.duplicate(documentHash); + } + + try { + // Step 3: Parse XML document + ProcurementDocument document = xmlParserService.parseDocument(xmlContent); + document.setDocumentHash(documentHash); + document.setSourceFilename(filename); + document.setSourcePath(filePath); + document.setFileSizeBytes(fileSize); + document.setProcessingDurationMs((int) (System.currentTimeMillis() - startTime)); + + // Step 4: Save to database + document = documentRepository.save(document); + + log.debug("Successfully processed document: {} -> {} (publication: {})", + filename, document.getId(), document.getPublicationId()); + + // Log success + processingLogService.logEvent( + document, documentHash, ProcessingLog.EventType.STORED, + ProcessingLog.EventStatus.SUCCESS, + "Document parsed and stored successfully", null, filename, + (int) (System.currentTimeMillis() - startTime)); + + // Publish event to trigger vectorization AFTER transaction commit + // This ensures document is visible in DB and avoids transaction isolation issues + eventPublisher.publishEvent(new DocumentSavedEvent(document.getId(), document.getPublicationId())); + log.debug("Document saved successfully, vectorization event published: {}", document.getId()); + + return ProcessingResult.success(document.getId(), documentHash, document.getPublicationId()); + + } catch (XmlParserService.XmlParsingException e) { + log.error("Failed to parse XML document {}: {}", filename, e.getMessage()); + processingLogService.logEvent( + null, documentHash, ProcessingLog.EventType.ERROR, + ProcessingLog.EventStatus.FAILURE, + "XML parsing failed", e.getMessage(), filename, + (int) (System.currentTimeMillis() - startTime)); + return ProcessingResult.error(documentHash, "XML parsing failed: " + e.getMessage()); + + } catch (Exception e) { + log.error("Unexpected error processing document {}: {}", filename, e.getMessage(), e); + processingLogService.logEvent( + null, documentHash, ProcessingLog.EventType.ERROR, + ProcessingLog.EventStatus.FAILURE, + "Processing failed", e.getMessage(), filename, + (int) (System.currentTimeMillis() - startTime)); + return ProcessingResult.error(documentHash, "Processing failed: " + e.getMessage()); + } + } + + /** + * Reprocess a document (e.g., after schema update). + */ + @Transactional + public Optional reprocessDocument(String publicationId) { + return documentRepository.findByPublicationId(publicationId) + .map(existing -> { + try { + // Re-parse the stored XML + ProcurementDocument updated = xmlParserService.parseDocument(existing.getXmlDocument()); + + // Preserve identity and tracking fields + updated.setId(existing.getId()); + updated.setDocumentHash(existing.getDocumentHash()); + updated.setSourceFilename(existing.getSourceFilename()); + updated.setSourcePath(existing.getSourcePath()); + updated.setFileSizeBytes(existing.getFileSizeBytes()); + updated.setCreatedAt(existing.getCreatedAt()); + + // Reset vectorization + updated.setVectorizationStatus(VectorizationStatus.PENDING); + updated.setContentVector(null); + updated.setVectorizedAt(null); + updated.setVectorizationError(null); + + documentRepository.save(updated); + + // Note: Re-vectorization will be triggered automatically by + // VectorizationRoute scheduler (checks for PENDING documents every 60s) + + return updated; + } catch (Exception e) { + log.error("Failed to reprocess document {}: {}", publicationId, e.getMessage()); + return null; + } + }); + } + + /** + * Result of document processing operation. + */ + public record ProcessingResult( + Status status, + java.util.UUID documentId, + String documentHash, + String publicationId, + String errorMessage + ) { + public enum Status { + SUCCESS, + DUPLICATE, + ERROR + } + + public static ProcessingResult success(java.util.UUID id, String hash, String pubId) { + return new ProcessingResult(Status.SUCCESS, id, hash, pubId, null); + } + + public static ProcessingResult duplicate(String hash) { + return new ProcessingResult(Status.DUPLICATE, null, hash, null, null); + } + + public static ProcessingResult error(String hash, String message) { + return new ProcessingResult(Status.ERROR, null, hash, null, message); + } + + public boolean isSuccess() { + return status == Status.SUCCESS; + } + + public boolean isDuplicate() { + return status == Status.DUPLICATE; + } + + public boolean isError() { + return status == Status.ERROR; + } + } +} diff --git a/src/main/java/at/procon/ted/service/ExcelExportService.java b/src/main/java/at/procon/ted/service/ExcelExportService.java new file mode 100644 index 0000000..9e7af94 --- /dev/null +++ b/src/main/java/at/procon/ted/service/ExcelExportService.java @@ -0,0 +1,241 @@ +package at.procon.ted.service; + +import at.procon.ted.service.SimilaritySearchService.SimilarDocument; +import at.procon.ted.service.SimilaritySearchService.SimilaritySearchResponse; +import lombok.extern.slf4j.Slf4j; +import org.apache.poi.common.usermodel.HyperlinkType; +import org.apache.poi.ss.usermodel.*; +import org.apache.poi.xssf.usermodel.XSSFWorkbook; +import org.springframework.stereotype.Service; + +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.time.LocalDateTime; +import java.time.format.DateTimeFormatter; +import java.util.List; + +/** + * Service for exporting similarity search results to Excel (XLSX) format. + * + * @author Martin.Schweitzer@procon.co.at and claude.ai + */ +@Service +@Slf4j +public class ExcelExportService { + + private static final String[] HEADERS = { + "Rang", + "Ähnlichkeit %", + "Publication ID", + "Projekt Titel", + "Auftraggeber", + "Land", + "Stadt", + "Vertragsart", + "Verfahrensart", + "Publikationsdatum", + "Einreichfrist", + "Geschätzter Wert", + "Währung", + "CPV Codes", + "TED Link" + }; + + /** + * Export similarity search results to an Excel file. + * + * @param response the similarity search response + * @param sourceFilename the name of the source PDF file + * @param outputDir the output directory for the Excel file + * @return the path to the generated Excel file + */ + public String exportToExcel(SimilaritySearchResponse response, String sourceFilename, String outputDir) throws IOException { + // Ensure output directory exists + File dir = new File(outputDir); + if (!dir.exists()) { + dir.mkdirs(); + } + + // Generate output filename + String baseName = sourceFilename != null + ? sourceFilename.replaceAll("\\.[^.]+$", "") // Remove extension + : "search_results"; + String timestamp = LocalDateTime.now().format(DateTimeFormatter.ofPattern("yyyyMMdd_HHmmss")); + String outputFilename = baseName + "_results_" + timestamp + ".xlsx"; + File outputFile = new File(dir, outputFilename); + + log.info("Exporting {} results to Excel: {}", response.getResultCount(), outputFile.getAbsolutePath()); + + try (Workbook workbook = new XSSFWorkbook()) { + Sheet sheet = workbook.createSheet("Ähnliche Ausschreibungen"); + + // Create styles + CellStyle headerStyle = createHeaderStyle(workbook); + CellStyle linkStyle = createLinkStyle(workbook); + CellStyle percentStyle = createPercentStyle(workbook); + CellStyle dateStyle = createDateStyle(workbook); + CellStyle currencyStyle = createCurrencyStyle(workbook); + + // Create header row + Row headerRow = sheet.createRow(0); + for (int i = 0; i < HEADERS.length; i++) { + Cell cell = headerRow.createCell(i); + cell.setCellValue(HEADERS[i]); + cell.setCellStyle(headerStyle); + } + + // Add data rows (source info is in filename, no need to add it here) + List results = response.getResults(); + int rowNum = 1; // Start directly after header + int rank = 1; + + for (SimilarDocument doc : results) { + Row row = sheet.createRow(rowNum++); + int colNum = 0; + + // Rang + row.createCell(colNum++).setCellValue(rank++); + + // Ähnlichkeit % + Cell simCell = row.createCell(colNum++); + if (doc.getSimilarityPercent() != null) { + simCell.setCellValue(doc.getSimilarityPercent()); + simCell.setCellStyle(percentStyle); + } + + // Publication ID + row.createCell(colNum++).setCellValue(doc.getPublicationId() != null ? doc.getPublicationId() : ""); + + // Projekt Titel + row.createCell(colNum++).setCellValue(doc.getProjectTitle() != null ? doc.getProjectTitle() : ""); + + // Auftraggeber + row.createCell(colNum++).setCellValue(doc.getBuyerName() != null ? doc.getBuyerName() : ""); + + // Land + row.createCell(colNum++).setCellValue(doc.getBuyerCountryCode() != null ? doc.getBuyerCountryCode() : ""); + + // Stadt + row.createCell(colNum++).setCellValue(doc.getBuyerCity() != null ? doc.getBuyerCity() : ""); + + // Vertragsart + row.createCell(colNum++).setCellValue(doc.getContractNature() != null ? doc.getContractNature() : ""); + + // Verfahrensart + row.createCell(colNum++).setCellValue(doc.getProcedureType() != null ? doc.getProcedureType() : ""); + + // Publikationsdatum + Cell pubDateCell = row.createCell(colNum++); + if (doc.getPublicationDate() != null) { + pubDateCell.setCellValue(doc.getPublicationDate().toString()); + pubDateCell.setCellStyle(dateStyle); + } + + // Einreichfrist + Cell deadlineCell = row.createCell(colNum++); + if (doc.getSubmissionDeadline() != null) { + deadlineCell.setCellValue(doc.getSubmissionDeadline().toLocalDate().toString()); + deadlineCell.setCellStyle(dateStyle); + } + + // Geschätzter Wert + Cell valueCell = row.createCell(colNum++); + if (doc.getEstimatedValue() != null) { + valueCell.setCellValue(doc.getEstimatedValue().doubleValue()); + valueCell.setCellStyle(currencyStyle); + } + + // Währung + row.createCell(colNum++).setCellValue( + doc.getEstimatedValueCurrency() != null ? doc.getEstimatedValueCurrency() : ""); + + // CPV Codes + String cpvCodes = doc.getCpvCodes() != null && !doc.getCpvCodes().isEmpty() + ? String.join(", ", doc.getCpvCodes()) + : ""; + row.createCell(colNum++).setCellValue(cpvCodes); + + // TED Link (Hyperlink) + Cell linkCell = row.createCell(colNum); + if (doc.getNoticeUrl() != null && !doc.getNoticeUrl().isEmpty()) { + linkCell.setCellValue("Zur Ausschreibung"); + linkCell.setCellStyle(linkStyle); + + CreationHelper createHelper = workbook.getCreationHelper(); + Hyperlink hyperlink = createHelper.createHyperlink(HyperlinkType.URL); + hyperlink.setAddress(doc.getNoticeUrl()); + linkCell.setHyperlink(hyperlink); + } + } + + // Auto-size columns + for (int i = 0; i < HEADERS.length; i++) { + sheet.autoSizeColumn(i); + // Set minimum width for some columns + if (i == 3) { // Projekt Titel + sheet.setColumnWidth(i, Math.max(sheet.getColumnWidth(i), 50 * 256)); + } + if (i == 4) { // Auftraggeber + sheet.setColumnWidth(i, Math.max(sheet.getColumnWidth(i), 30 * 256)); + } + } + + // Add filter + sheet.setAutoFilter(new org.apache.poi.ss.util.CellRangeAddress(0, 0, 0, HEADERS.length - 1)); + + // Freeze header row + sheet.createFreezePane(0, 1); + + // Write to file + try (FileOutputStream fos = new FileOutputStream(outputFile)) { + workbook.write(fos); + } + + log.info("Excel export completed: {} ({} results)", outputFile.getAbsolutePath(), results.size()); + return outputFile.getAbsolutePath(); + } + } + + private CellStyle createHeaderStyle(Workbook workbook) { + CellStyle style = workbook.createCellStyle(); + Font font = workbook.createFont(); + font.setBold(true); + font.setColor(IndexedColors.WHITE.getIndex()); + style.setFont(font); + style.setFillForegroundColor(IndexedColors.DARK_BLUE.getIndex()); + style.setFillPattern(FillPatternType.SOLID_FOREGROUND); + style.setBorderBottom(BorderStyle.THIN); + style.setAlignment(HorizontalAlignment.CENTER); + return style; + } + + private CellStyle createLinkStyle(Workbook workbook) { + CellStyle style = workbook.createCellStyle(); + Font font = workbook.createFont(); + font.setUnderline(Font.U_SINGLE); + font.setColor(IndexedColors.BLUE.getIndex()); + style.setFont(font); + return style; + } + + private CellStyle createPercentStyle(Workbook workbook) { + CellStyle style = workbook.createCellStyle(); + style.setAlignment(HorizontalAlignment.CENTER); + return style; + } + + private CellStyle createDateStyle(Workbook workbook) { + CellStyle style = workbook.createCellStyle(); + style.setAlignment(HorizontalAlignment.CENTER); + return style; + } + + private CellStyle createCurrencyStyle(Workbook workbook) { + CellStyle style = workbook.createCellStyle(); + DataFormat format = workbook.createDataFormat(); + style.setDataFormat(format.getFormat("#,##0.00")); + style.setAlignment(HorizontalAlignment.RIGHT); + return style; + } +} diff --git a/src/main/java/at/procon/ted/service/ProcessingLogService.java b/src/main/java/at/procon/ted/service/ProcessingLogService.java new file mode 100644 index 0000000..b9e604c --- /dev/null +++ b/src/main/java/at/procon/ted/service/ProcessingLogService.java @@ -0,0 +1,40 @@ +package at.procon.ted.service; + +import at.procon.ted.model.entity.ProcurementDocument; +import at.procon.ted.model.entity.ProcessingLog; +import at.procon.ted.repository.ProcessingLogRepository; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; + +/** + * Service for logging processing events. + * + * @author Martin.Schweitzer@procon.co.at and claude.ai + */ +@Service +@RequiredArgsConstructor +@Slf4j +public class ProcessingLogService { + + private final ProcessingLogRepository logRepository; + + @Transactional + public void logEvent(ProcurementDocument document, String documentHash, String eventType, + String eventStatus, String message, String errorDetails, + String sourceFilename, Integer durationMs) { + ProcessingLog logEntry = ProcessingLog.builder() + .document(document) + .documentHash(documentHash) + .eventType(eventType) + .eventStatus(eventStatus) + .message(message) + .errorDetails(errorDetails) + .sourceFilename(sourceFilename) + .durationMs(durationMs) + .build(); + + logRepository.save(logEntry); + } +} diff --git a/src/main/java/at/procon/ted/service/SearchService.java b/src/main/java/at/procon/ted/service/SearchService.java new file mode 100644 index 0000000..da6cbe3 --- /dev/null +++ b/src/main/java/at/procon/ted/service/SearchService.java @@ -0,0 +1,456 @@ +package at.procon.ted.service; + +import at.procon.ted.config.TedProcessorProperties; +import at.procon.ted.model.dto.DocumentDtos.*; +import at.procon.ted.model.entity.*; +import at.procon.ted.repository.ProcurementDocumentRepository; +import jakarta.persistence.EntityManager; +import jakarta.persistence.PersistenceContext; +import jakarta.persistence.criteria.*; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.data.domain.Page; +import org.springframework.data.domain.PageRequest; +import org.springframework.data.domain.Pageable; +import org.springframework.data.domain.Sort; +import org.springframework.data.jpa.domain.Specification; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; + +import java.time.LocalDate; +import java.time.OffsetDateTime; +import java.util.*; +import java.util.stream.Collectors; + +/** + * Service for searching procurement documents. + * + * Supports: + * - Structured search with filters (country, type, CPV codes, dates, etc.) + * - Semantic search using vector similarity + * - Combined structured + semantic search + * + * @author Martin.Schweitzer@procon.co.at and claude.ai + */ +@Service +@RequiredArgsConstructor +@Slf4j +@Transactional(readOnly = true) +public class SearchService { + + private final ProcurementDocumentRepository documentRepository; + private final VectorizationService vectorizationService; + private final TedProcessorProperties properties; + + @PersistenceContext + private EntityManager entityManager; + + /** + * Search documents with combined structured and semantic filters. + */ + public SearchResponse search(SearchRequest request) { + // Normalize pagination parameters + int page = request.getPage() != null ? request.getPage() : 0; + int size = Math.min( + request.getSize() != null ? request.getSize() : properties.getSearch().getDefaultPageSize(), + properties.getSearch().getMaxPageSize() + ); + + // Check if semantic search is requested + boolean hasSemanticQuery = request.getSemanticQuery() != null && !request.getSemanticQuery().isBlank(); + + if (hasSemanticQuery && vectorizationService.isAvailable()) { + return semanticSearch(request, page, size); + } else { + return structuredSearch(request, page, size); + } + } + + /** + * Perform structured search using JPA Specifications. + */ + private SearchResponse structuredSearch(SearchRequest request, int page, int size) { + Specification spec = buildSpecification(request); + + Sort sort = buildSort(request.getSortBy(), request.getSortDirection()); + Pageable pageable = PageRequest.of(page, size, sort); + + Page result = documentRepository.findAll(spec, pageable); + + List summaries = result.getContent().stream() + .map(this::toSummary) + .collect(Collectors.toList()); + + return SearchResponse.builder() + .documents(summaries) + .page(page) + .size(size) + .totalElements(result.getTotalElements()) + .totalPages(result.getTotalPages()) + .hasNext(result.hasNext()) + .hasPrevious(result.hasPrevious()) + .build(); + } + + /** + * Perform semantic search with vector similarity. + */ + private SearchResponse semanticSearch(SearchRequest request, int page, int size) { + try { + // Generate query embedding + float[] queryEmbedding = vectorizationService.generateQueryEmbedding(request.getSemanticQuery()); + String vectorStr = vectorizationService.floatArrayToVectorString(queryEmbedding); + + double threshold = request.getSimilarityThreshold() != null + ? request.getSimilarityThreshold() + : properties.getSearch().getSimilarityThreshold(); + + // Execute native query for vector search + List results = documentRepository.findBySemanticSearch( + vectorStr, + threshold, + request.getCountryCode(), + request.getNoticeType() != null ? request.getNoticeType().name() : null, + request.getContractNature() != null ? request.getContractNature().name() : null, + request.getCpvPrefix(), + request.getPublicationDateFrom(), + request.getPublicationDateTo(), + size, + page * size + ); + + // Count total for pagination + long totalElements = documentRepository.countBySemanticSearch( + vectorStr, + threshold, + request.getCountryCode(), + request.getNoticeType() != null ? request.getNoticeType().name() : null, + request.getContractNature() != null ? request.getContractNature().name() : null + ); + + // Map results to summaries + List summaries = results.stream() + .map(this::mapSemanticResult) + .collect(Collectors.toList()); + + int totalPages = (int) Math.ceil((double) totalElements / size); + + return SearchResponse.builder() + .documents(summaries) + .page(page) + .size(size) + .totalElements(totalElements) + .totalPages(totalPages) + .hasNext(page < totalPages - 1) + .hasPrevious(page > 0) + .build(); + + } catch (Exception e) { + log.error("Semantic search failed, falling back to structured search: {}", e.getMessage()); + // Fallback to structured search without semantic component + return structuredSearch(request, page, size); + } + } + + /** + * Build JPA Specification from search request. + */ + private Specification buildSpecification(SearchRequest request) { + return (root, query, cb) -> { + List predicates = new ArrayList<>(); + + // Country filter + if (request.getCountryCode() != null && !request.getCountryCode().isBlank()) { + predicates.add(cb.equal(root.get("buyerCountryCode"), request.getCountryCode())); + } + + // Multiple countries filter + if (request.getCountryCodes() != null && !request.getCountryCodes().isEmpty()) { + predicates.add(root.get("buyerCountryCode").in(request.getCountryCodes())); + } + + // Notice type filter + if (request.getNoticeType() != null) { + predicates.add(cb.equal(root.get("noticeType"), request.getNoticeType())); + } + + // Contract nature filter + if (request.getContractNature() != null) { + predicates.add(cb.equal(root.get("contractNature"), request.getContractNature())); + } + + // Procedure type filter + if (request.getProcedureType() != null) { + predicates.add(cb.equal(root.get("procedureType"), request.getProcedureType())); + } + + // Publication date range + if (request.getPublicationDateFrom() != null) { + predicates.add(cb.greaterThanOrEqualTo(root.get("publicationDate"), request.getPublicationDateFrom())); + } + if (request.getPublicationDateTo() != null) { + predicates.add(cb.lessThanOrEqualTo(root.get("publicationDate"), request.getPublicationDateTo())); + } + + // Submission deadline filter + if (request.getSubmissionDeadlineAfter() != null) { + predicates.add(cb.greaterThan(root.get("submissionDeadline"), request.getSubmissionDeadlineAfter())); + } + + // EU funded filter + if (request.getEuFunded() != null) { + predicates.add(cb.equal(root.get("euFunded"), request.getEuFunded())); + } + + // Text search on buyer name + if (request.getBuyerNameContains() != null && !request.getBuyerNameContains().isBlank()) { + predicates.add(cb.like(cb.lower(root.get("buyerName")), + "%" + request.getBuyerNameContains().toLowerCase() + "%")); + } + + // Text search on project title + if (request.getProjectTitleContains() != null && !request.getProjectTitleContains().isBlank()) { + predicates.add(cb.like(cb.lower(root.get("projectTitle")), + "%" + request.getProjectTitleContains().toLowerCase() + "%")); + } + + return cb.and(predicates.toArray(new Predicate[0])); + }; + } + + /** + * Build Sort from request parameters. + */ + private Sort buildSort(String sortBy, String sortDirection) { + String field = sortBy != null ? sortBy : "publicationDate"; + Sort.Direction direction = "asc".equalsIgnoreCase(sortDirection) + ? Sort.Direction.ASC + : Sort.Direction.DESC; + return Sort.by(direction, field); + } + + /** + * Convert entity to summary DTO. + */ + private DocumentSummary toSummary(ProcurementDocument doc) { + return DocumentSummary.builder() + .id(doc.getId()) + .publicationId(doc.getPublicationId()) + .noticeId(doc.getNoticeId()) + .noticeType(doc.getNoticeType()) + .projectTitle(doc.getProjectTitle()) + .buyerName(doc.getBuyerName()) + .buyerCountryCode(doc.getBuyerCountryCode()) + .buyerCity(doc.getBuyerCity()) + .contractNature(doc.getContractNature()) + .procedureType(doc.getProcedureType()) + .publicationDate(doc.getPublicationDate()) + .submissionDeadline(doc.getSubmissionDeadline()) + .cpvCodes(doc.getCpvCodes() != null ? Arrays.asList(doc.getCpvCodes()) : List.of()) + .totalLots(doc.getTotalLots()) + .estimatedValue(doc.getEstimatedValue()) + .estimatedValueCurrency(doc.getEstimatedValueCurrency()) + .build(); + } + + /** + * Map semantic search result array to summary DTO. + */ + private DocumentSummary mapSemanticResult(Object[] row) { + // Results from native query: id, publication_id, project_title, buyer_name, buyer_country_code, publication_date, similarity + UUID id = (UUID) row[0]; + + return documentRepository.findById(id) + .map(doc -> { + DocumentSummary summary = toSummary(doc); + // Set similarity score from query result + if (row.length > 6 && row[6] != null) { + summary.setSimilarity(((Number) row[6]).doubleValue()); + } + return summary; + }) + .orElse(DocumentSummary.builder() + .id(id) + .publicationId(row[1] != null ? row[1].toString() : null) + .projectTitle(row[2] != null ? row[2].toString() : null) + .buyerName(row[3] != null ? row[3].toString() : null) + .buyerCountryCode(row[4] != null ? row[4].toString() : null) + .publicationDate(row[5] != null ? (LocalDate) row[5] : null) + .similarity(row.length > 6 && row[6] != null ? ((Number) row[6]).doubleValue() : null) + .build()); + } + + /** + * Get document by ID with full details. + */ + public Optional getDocumentDetail(UUID id) { + return documentRepository.findById(id).map(this::toDetail); + } + + /** + * Get document by publication ID. + */ + public Optional getDocumentByPublicationId(String publicationId) { + return documentRepository.findByPublicationId(publicationId).map(this::toDetail); + } + + /** + * Convert entity to detail DTO. + */ + private DocumentDetail toDetail(ProcurementDocument doc) { + return DocumentDetail.builder() + .id(doc.getId()) + .documentHash(doc.getDocumentHash()) + .publicationId(doc.getPublicationId()) + .noticeId(doc.getNoticeId()) + .ojsId(doc.getOjsId()) + .contractFolderId(doc.getContractFolderId()) + .noticeType(doc.getNoticeType()) + .noticeSubtypeCode(doc.getNoticeSubtypeCode()) + .sdkVersion(doc.getSdkVersion()) + .ublVersion(doc.getUblVersion()) + .languageCode(doc.getLanguageCode()) + .issueDateTime(doc.getIssueDateTime()) + .publicationDate(doc.getPublicationDate()) + .submissionDeadline(doc.getSubmissionDeadline()) + .buyerName(doc.getBuyerName()) + .buyerCountryCode(doc.getBuyerCountryCode()) + .buyerCity(doc.getBuyerCity()) + .buyerPostalCode(doc.getBuyerPostalCode()) + .buyerNutsCode(doc.getBuyerNutsCode()) + .buyerActivityType(doc.getBuyerActivityType()) + .buyerLegalType(doc.getBuyerLegalType()) + .projectTitle(doc.getProjectTitle()) + .projectDescription(doc.getProjectDescription()) + .internalReference(doc.getInternalReference()) + .contractNature(doc.getContractNature()) + .procedureType(doc.getProcedureType()) + .cpvCodes(doc.getCpvCodes() != null ? Arrays.asList(doc.getCpvCodes()) : List.of()) + .nutsCodes(doc.getNutsCodes() != null ? Arrays.asList(doc.getNutsCodes()) : List.of()) + .estimatedValue(doc.getEstimatedValue()) + .estimatedValueCurrency(doc.getEstimatedValueCurrency()) + .totalLots(doc.getTotalLots()) + .maxLotsAwarded(doc.getMaxLotsAwarded()) + .maxLotsSubmitted(doc.getMaxLotsSubmitted()) + .lots(doc.getLots().stream().map(this::toLotSummary).collect(Collectors.toList())) + .organizations(doc.getOrganizations().stream().map(this::toOrgSummary).collect(Collectors.toList())) + .regulatoryDomain(doc.getRegulatoryDomain()) + .euFunded(doc.getEuFunded()) + .vectorizationStatus(doc.getVectorizationStatus()) + .vectorizedAt(doc.getVectorizedAt()) + .sourceFilename(doc.getSourceFilename()) + .fileSizeBytes(doc.getFileSizeBytes()) + .createdAt(doc.getCreatedAt()) + .updatedAt(doc.getUpdatedAt()) + .build(); + } + + private LotSummary toLotSummary(ProcurementLot lot) { + return LotSummary.builder() + .id(lot.getId()) + .lotId(lot.getLotId()) + .internalId(lot.getInternalId()) + .title(lot.getTitle()) + .description(lot.getDescription()) + .cpvCodes(lot.getCpvCodes() != null ? Arrays.asList(lot.getCpvCodes()) : List.of()) + .nutsCodes(lot.getNutsCodes() != null ? Arrays.asList(lot.getNutsCodes()) : List.of()) + .estimatedValue(lot.getEstimatedValue()) + .estimatedValueCurrency(lot.getEstimatedValueCurrency()) + .durationValue(lot.getDurationValue()) + .durationUnit(lot.getDurationUnit()) + .submissionDeadline(lot.getSubmissionDeadline()) + .euFunded(lot.getEuFunded()) + .build(); + } + + private OrganizationSummary toOrgSummary(Organization org) { + return OrganizationSummary.builder() + .id(org.getId()) + .orgReference(org.getOrgReference()) + .role(org.getRole()) + .name(org.getName()) + .companyId(org.getCompanyId()) + .countryCode(org.getCountryCode()) + .city(org.getCity()) + .postalCode(org.getPostalCode()) + .nutsCode(org.getNutsCode()) + .websiteUri(org.getWebsiteUri()) + .email(org.getEmail()) + .phone(org.getPhone()) + .build(); + } + + /** + * Get statistics about the document collection. + */ + public StatisticsResponse getStatistics() { + // Get vectorization stats + List vectorStats = documentRepository.countByVectorizationStatus(); + Map vectorCounts = vectorStats.stream() + .collect(Collectors.toMap( + row -> (VectorizationStatus) row[0], + row -> (Long) row[1] + )); + + // Get country stats + List countryStats = documentRepository.countByCountry(); + List countries = countryStats.stream() + .map(row -> CountryStats.builder() + .countryCode((String) row[0]) + .documentCount((Long) row[1]) + .build()) + .collect(Collectors.toList()); + + // Get notice type stats + List noticeTypeStats = documentRepository.countByNoticeType(); + List noticeTypes = noticeTypeStats.stream() + .map(row -> NoticeTypeStats.builder() + .noticeType((NoticeType) row[0]) + .documentCount((Long) row[1]) + .build()) + .collect(Collectors.toList()); + + long total = documentRepository.count(); + + return StatisticsResponse.builder() + .totalDocuments(total) + .vectorizedDocuments(vectorCounts.getOrDefault(VectorizationStatus.COMPLETED, 0L)) + .pendingVectorization(vectorCounts.getOrDefault(VectorizationStatus.PENDING, 0L)) + .failedVectorization(vectorCounts.getOrDefault(VectorizationStatus.FAILED, 0L)) + .uniqueCountries(countries.size()) + .countryStatistics(countries) + .noticeTypeStatistics(noticeTypes) + .build(); + } + + /** + * Get documents with upcoming deadlines. + */ + public List getUpcomingDeadlines(int limit) { + Page page = documentRepository.findUpcomingDeadlines( + PageRequest.of(0, limit)); + return page.getContent().stream() + .map(this::toSummary) + .collect(Collectors.toList()); + } + + /** + * Get distinct countries in the database. + */ + public List getDistinctCountries() { + return documentRepository.countByCountry().stream() + .map(row -> (String) row[0]) + .filter(Objects::nonNull) + .sorted() + .collect(Collectors.toList()); + } + + /** + * Get distinct CPV codes (main classification). + */ + public List getDistinctCpvCodes() { + // This would require a native query to unnest the array + // For now, return empty list - could be implemented with @Query + return List.of(); + } +} diff --git a/src/main/java/at/procon/ted/service/SimilaritySearchService.java b/src/main/java/at/procon/ted/service/SimilaritySearchService.java new file mode 100644 index 0000000..407f61b --- /dev/null +++ b/src/main/java/at/procon/ted/service/SimilaritySearchService.java @@ -0,0 +1,253 @@ +package at.procon.ted.service; + +import at.procon.ted.config.TedProcessorProperties; +import at.procon.ted.model.entity.ProcurementDocument; +import at.procon.ted.repository.ProcurementDocumentRepository; +import at.procon.ted.service.attachment.PdfExtractionService; +import at.procon.ted.service.attachment.AttachmentExtractor.ExtractionResult; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; + +import java.math.BigDecimal; +import java.time.LocalDate; +import java.util.ArrayList; +import java.util.List; +import java.util.UUID; + +/** + * Service for similarity search on TED procurement documents. + * Uses vector embeddings and cosine similarity for semantic matching. + * + * @author Martin.Schweitzer@procon.co.at and claude.ai + */ +@Service +@RequiredArgsConstructor +@Slf4j +@Transactional(readOnly = true) +public class SimilaritySearchService { + + private final VectorizationService vectorizationService; + private final ProcurementDocumentRepository documentRepository; + private final PdfExtractionService pdfExtractionService; + private final TedProcessorProperties properties; + + private static final int DEFAULT_TOP_K = 20; + private static final double DEFAULT_THRESHOLD = 0.5; + + /** + * Search for similar documents using text query. + * + * @param queryText the text to search for similar documents + * @param topK number of top results to return (default 20) + * @param threshold minimum similarity threshold (default 0.5) + * @return list of similar documents with similarity scores + */ + public SimilaritySearchResponse searchByText(String queryText, Integer topK, Double threshold) { + if (queryText == null || queryText.isBlank()) { + throw new IllegalArgumentException("Query text cannot be empty"); + } + + if (!vectorizationService.isAvailable()) { + throw new IllegalStateException("Vectorization service is not available"); + } + + int limit = topK != null && topK > 0 ? Math.min(topK, 100) : DEFAULT_TOP_K; + double similarityThreshold = threshold != null ? threshold : DEFAULT_THRESHOLD; + + log.info("Similarity search: query='{}...', topK={}, threshold={}", + queryText.substring(0, Math.min(50, queryText.length())), limit, similarityThreshold); + + try { + // Generate query embedding + long startTime = System.currentTimeMillis(); + float[] queryEmbedding = vectorizationService.generateQueryEmbedding(queryText); + String vectorStr = vectorizationService.floatArrayToVectorString(queryEmbedding); + long embeddingTime = System.currentTimeMillis() - startTime; + + log.debug("Query embedding generated in {}ms ({} dimensions)", embeddingTime, queryEmbedding.length); + + // Execute similarity search (simple query without filters) + startTime = System.currentTimeMillis(); + List results = documentRepository.findBySimilarity( + vectorStr, + similarityThreshold, + limit + ); + long searchTime = System.currentTimeMillis() - startTime; + + log.info("Similarity search completed: {} results in {}ms", results.size(), searchTime); + + // Map results to response + List documents = new ArrayList<>(); + for (Object[] row : results) { + SimilarDocument doc = mapToSimilarDocument(row); + if (doc != null) { + documents.add(doc); + } + } + + return SimilaritySearchResponse.builder() + .query(truncateText(queryText, 200)) + .results(documents) + .resultCount(documents.size()) + .threshold(similarityThreshold) + .embeddingTimeMs(embeddingTime) + .searchTimeMs(searchTime) + .build(); + + } catch (Exception e) { + log.error("Similarity search failed: {}", e.getMessage(), e); + throw new RuntimeException("Similarity search failed: " + e.getMessage(), e); + } + } + + /** + * Search for similar documents using PDF content. + * + * @param pdfData the PDF file content + * @param filename original filename + * @param topK number of top results to return (default 20) + * @param threshold minimum similarity threshold (default 0.5) + * @return list of similar documents with similarity scores + */ + public SimilaritySearchResponse searchByPdf(byte[] pdfData, String filename, Integer topK, Double threshold) { + if (pdfData == null || pdfData.length == 0) { + throw new IllegalArgumentException("PDF data cannot be empty"); + } + + log.info("Extracting text from PDF: {} ({} bytes)", filename, pdfData.length); + + // Extract text from PDF + long startTime = System.currentTimeMillis(); + ExtractionResult extractionResult = pdfExtractionService.extract(pdfData, filename, "application/pdf"); + long extractionTime = System.currentTimeMillis() - startTime; + + if (!extractionResult.success()) { + throw new RuntimeException("PDF text extraction failed: " + extractionResult.errorMessage()); + } + + String extractedText = extractionResult.extractedText(); + if (extractedText == null || extractedText.isBlank()) { + throw new RuntimeException("No text content could be extracted from PDF"); + } + + log.info("PDF text extracted in {}ms: {} characters", extractionTime, extractedText.length()); + + // Search using extracted text + SimilaritySearchResponse response = searchByText(extractedText, topK, threshold); + + // Add PDF extraction info to response + return SimilaritySearchResponse.builder() + .query("PDF: " + filename + " (" + extractedText.length() + " chars extracted)") + .results(response.getResults()) + .resultCount(response.getResultCount()) + .threshold(response.getThreshold()) + .embeddingTimeMs(response.getEmbeddingTimeMs()) + .searchTimeMs(response.getSearchTimeMs()) + .pdfExtractionTimeMs(extractionTime) + .extractedTextLength(extractedText.length()) + .build(); + } + + /** + * Map database result row to SimilarDocument DTO. + * Result format: [id (UUID), similarity (Double)] + */ + private SimilarDocument mapToSimilarDocument(Object[] row) { + if (row == null || row.length < 2) { + return null; + } + + try { + UUID id = (UUID) row[0]; + Double similarity = row[1] != null ? ((Number) row[1]).doubleValue() : null; + + // Fetch full document for detailed mapping + return documentRepository.findById(id) + .map(doc -> SimilarDocument.builder() + .id(doc.getId()) + .publicationId(doc.getPublicationId()) + .noticeId(doc.getNoticeId()) + .noticeUrl(doc.getNoticeUrl()) + .noticeType(doc.getNoticeType() != null ? doc.getNoticeType().name() : null) + .projectTitle(doc.getProjectTitle()) + .projectDescription(truncateText(doc.getProjectDescription(), 500)) + .buyerName(doc.getBuyerName()) + .buyerCountryCode(doc.getBuyerCountryCode()) + .buyerCity(doc.getBuyerCity()) + .contractNature(doc.getContractNature() != null ? doc.getContractNature().name() : null) + .procedureType(doc.getProcedureType() != null ? doc.getProcedureType().name() : null) + .publicationDate(doc.getPublicationDate()) + .submissionDeadline(doc.getSubmissionDeadline()) + .cpvCodes(doc.getCpvCodes() != null ? List.of(doc.getCpvCodes()) : List.of()) + .estimatedValue(doc.getEstimatedValue()) + .estimatedValueCurrency(doc.getEstimatedValueCurrency()) + .similarity(similarity) + .similarityPercent(similarity != null ? Math.round(similarity * 100) : null) + .build()) + .orElse(null); + + } catch (Exception e) { + log.warn("Failed to map search result: {}", e.getMessage()); + return null; + } + } + + /** + * Truncate text to specified length. + */ + private String truncateText(String text, int maxLength) { + if (text == null) return null; + if (text.length() <= maxLength) return text; + return text.substring(0, maxLength - 3) + "..."; + } + + /** + * Response DTO for similarity search. + */ + @lombok.Data + @lombok.Builder + @lombok.NoArgsConstructor + @lombok.AllArgsConstructor + public static class SimilaritySearchResponse { + private String query; + private List results; + private int resultCount; + private double threshold; + private long embeddingTimeMs; + private long searchTimeMs; + private Long pdfExtractionTimeMs; + private Integer extractedTextLength; + } + + /** + * DTO for a similar document result. + */ + @lombok.Data + @lombok.Builder + @lombok.NoArgsConstructor + @lombok.AllArgsConstructor + public static class SimilarDocument { + private UUID id; + private String publicationId; + private String noticeId; + private String noticeUrl; + private String noticeType; + private String projectTitle; + private String projectDescription; + private String buyerName; + private String buyerCountryCode; + private String buyerCity; + private String contractNature; + private String procedureType; + private LocalDate publicationDate; + private java.time.OffsetDateTime submissionDeadline; + private List cpvCodes; + private BigDecimal estimatedValue; + private String estimatedValueCurrency; + private Double similarity; + private Long similarityPercent; + } +} diff --git a/src/main/java/at/procon/ted/service/TedPackageDownloadService.java b/src/main/java/at/procon/ted/service/TedPackageDownloadService.java new file mode 100644 index 0000000..50fbf9b --- /dev/null +++ b/src/main/java/at/procon/ted/service/TedPackageDownloadService.java @@ -0,0 +1,558 @@ +package at.procon.ted.service; + +import at.procon.ted.config.TedProcessorProperties; +import at.procon.ted.model.entity.TedDailyPackage; +import at.procon.ted.repository.TedDailyPackageRepository; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; + +import java.io.*; +import java.net.HttpURLConnection; +import java.net.URI; +import java.net.URL; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.StandardCopyOption; +import java.security.MessageDigest; +import java.time.Duration; +import java.time.LocalDate; +import java.time.OffsetDateTime; +import java.time.Year; +import java.time.ZoneOffset; +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; +import java.util.stream.Stream; + +import org.apache.commons.compress.archivers.tar.TarArchiveEntry; +import org.apache.commons.compress.archivers.tar.TarArchiveInputStream; +import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream; + +/** + * Service for downloading and processing TED Daily Packages. + * + * Features: + * - Automatic download from https://ted.europa.eu/packages/daily/ + * - Hash-based idempotency check + * - tar.gz extraction + * - Integration with XML processing + * + * @author Martin.Schweitzer@procon.co.at and claude.ai + */ +@Service +@RequiredArgsConstructor +@Slf4j +public class TedPackageDownloadService { + + private final TedProcessorProperties properties; + private final TedDailyPackageRepository packageRepository; + + /** + * Generates package identifier in YYYYSSSSS format. + */ + public String generatePackageIdentifier(int year, int serialNumber) { + return String.format("%04d%05d", year, serialNumber); + } + + /** + * Determines the next package to download. + * + * Strategy (starts with current year FIRST, then goes backward): + * 1. CURRENT YEAR (2026): Forward from max(nr) until 404 (get today's packages first!) + * 2. All years (2026 -> 2025 -> 2024...): Fill gaps (if min(nr) > 1, then backward to 1) + * 3. If current year complete (min=1 and 404 after max) -> previous year + * 4. Repeat backward until startYear + * + * This ensures we always get the newest data first! + */ + public PackageInfo getNextPackageToDownload() { + int currentYear = Year.now().getValue(); + + log.debug("Determining next package to download (current year: {})", currentYear); + + // 1. PRIORITY: Current year forward crawling (max+1) - GET TODAY'S PACKAGES FIRST! + PackageInfo nextInCurrentYear = getNextForwardPackage(currentYear); + if (nextInCurrentYear != null) { + log.info("Next package: {} (CURRENT YEAR {} forward - newest data first!)", + nextInCurrentYear.getIdentifier(), currentYear); + return nextInCurrentYear; + } + + log.debug("Current year {} complete or has 404, checking older years backward", currentYear); + + // 2. Go through all years BACKWARD (current year -> startYear) for gaps and completion + for (int year = currentYear; year >= properties.getDownload().getStartYear(); year--) { + // 2a. Check if there are gaps (min > 1) + PackageInfo gapFiller = getGapFillerPackage(year); + if (gapFiller != null) { + log.info("Next package: {} (filling gap in year {})", gapFiller.getIdentifier(), year); + return gapFiller; + } + + // 2b. If no gap filler, check if year is complete + if (!isYearComplete(year)) { + // Year not complete, try forward crawling + PackageInfo forwardPackage = getNextForwardPackage(year); + if (forwardPackage != null) { + log.info("Next package: {} (forward in year {})", forwardPackage.getIdentifier(), year); + return forwardPackage; + } + } else { + log.debug("Year {} is complete", year); + } + } + + // 3. Check if we can open a new previous year + int oldestYear = getOldestYearWithData(); + if (oldestYear > properties.getDownload().getStartYear()) { + int previousYear = oldestYear - 1; + if (previousYear >= properties.getDownload().getStartYear()) { + // Open new year, start with 1 + log.info("Next package: {} (opening new year {})", String.format("%04d%05d", previousYear, 1), previousYear); + return new PackageInfo(previousYear, 1); + } + } + + log.info("All years from {} to {} are complete - no more packages", + properties.getDownload().getStartYear(), currentYear); + return null; // No more packages + } + + /** + * Finds the next package for forward crawling (max+1). + * + * Stronger NOT_FOUND handling: + * - Current year: a tail 404 is treated as "not available yet" and retried indefinitely + * - Older years: a tail 404 remains retryable until the configured grace period after year end expires + * - Final year completion is only assumed after that grace period + */ + private PackageInfo getNextForwardPackage(int year) { + Optional latest = packageRepository.findLatestByYear(year); + + if (latest.isEmpty()) { + // No package for this year -> start with 1 + log.debug("Year {} has no packages, starting with 1", year); + return new PackageInfo(year, 1); + } + + TedDailyPackage latestPackage = latest.get(); + + if (latestPackage.getDownloadStatus() == TedDailyPackage.DownloadStatus.NOT_FOUND) { + if (shouldRetryNotFoundPackage(latestPackage)) { + log.info("Retrying tail NOT_FOUND package {} for year {}", latestPackage.getPackageIdentifier(), year); + return new PackageInfo(year, latestPackage.getSerialNumber()); + } + + if (isNotFoundRetryableForYear(latestPackage)) { + log.debug("Year {} waiting until {} before retrying tail package {}", + year, calculateNextRetryAt(latestPackage), latestPackage.getPackageIdentifier()); + return null; + } + + log.debug("Year {} finalized after grace period at tail package {}", + year, latestPackage.getPackageIdentifier()); + return null; + } + + // Next package (max+1) + log.debug("Year {} continues from package {} to {}", year, latestPackage.getSerialNumber(), latestPackage.getSerialNumber() + 1); + return new PackageInfo(year, latestPackage.getSerialNumber() + 1); + } + + /** + * Finds package for gap filling (min-1). + * Returns null if no gap exists (min = 1). + */ + private PackageInfo getGapFillerPackage(int year) { + Optional first = packageRepository.findFirstByYear(year); + + if (first.isEmpty()) { + // No package for this year + return null; + } + + int minSerial = first.get().getSerialNumber(); + if (minSerial <= 1) { + // No gap, already starts at 1 + return null; + } + + // Gap found: Get (min-1) + return new PackageInfo(year, minSerial - 1); + } + + /** + * Checks if a year is complete. + * A year is complete only when: + * - package numbering starts at 1, and + * - the current tail package is NOT_FOUND, and + * - that NOT_FOUND is no longer retryable (grace period expired) + */ + private boolean isYearComplete(int year) { + Optional first = packageRepository.findFirstByYear(year); + Optional latest = packageRepository.findLatestByYear(year); + + if (first.isEmpty() || latest.isEmpty()) { + return false; + } + + if (first.get().getSerialNumber() != 1) { + return false; + } + + TedDailyPackage latestPackage = latest.get(); + return latestPackage.getDownloadStatus() == TedDailyPackage.DownloadStatus.NOT_FOUND + && !isNotFoundRetryableForYear(latestPackage); + } + + + private boolean shouldRetryNotFoundPackage(TedDailyPackage pkg) { + if (!isNotFoundRetryableForYear(pkg)) { + return false; + } + + OffsetDateTime nextRetryAt = calculateNextRetryAt(pkg); + return !nextRetryAt.isAfter(OffsetDateTime.now()); + } + + private boolean isNotFoundRetryableForYear(TedDailyPackage pkg) { + int currentYear = Year.now().getValue(); + int packageYear = pkg.getYear() != null ? pkg.getYear() : currentYear; + + if (packageYear >= currentYear) { + return properties.getDownload().isRetryCurrentYearNotFoundIndefinitely(); + } + + return OffsetDateTime.now().isBefore(getYearRetryGraceDeadline(packageYear)); + } + + private OffsetDateTime calculateNextRetryAt(TedDailyPackage pkg) { + OffsetDateTime lastAttemptAt = pkg.getUpdatedAt() != null + ? pkg.getUpdatedAt() + : (pkg.getCreatedAt() != null ? pkg.getCreatedAt() : OffsetDateTime.now()); + + return lastAttemptAt.plus(Duration.ofMillis(properties.getDownload().getNotFoundRetryInterval())); + } + + private OffsetDateTime getYearRetryGraceDeadline(int year) { + return LocalDate.of(year + 1, 1, 1) + .atStartOfDay() + .atOffset(ZoneOffset.UTC) + .plusDays(properties.getDownload().getPreviousYearGracePeriodDays()); + } + + /** + * Finds the oldest year for which we have data. + */ + private int getOldestYearWithData() { + // Start from startYear and go forward to find the first year with data + int currentYear = Year.now().getValue(); + for (int year = properties.getDownload().getStartYear(); year <= currentYear; year++) { + if (packageRepository.findLatestByYear(year).isPresent()) { + // Found the oldest year with data + return year; + } + } + return currentYear; + } + + /** + * Lädt ein Package herunter und verarbeitet es. + */ + @Transactional + public DownloadResult downloadPackage(int year, int serialNumber) { + String packageId = generatePackageIdentifier(year, serialNumber); + + log.debug("Starting download of package: {}", packageId); + + // Prüfe ob Package bereits existiert + Optional existing = packageRepository.findByPackageIdentifier(packageId); + + String downloadUrl = buildDownloadUrl(packageId); + TedDailyPackage packageEntity; + + if (existing.isPresent()) { + TedDailyPackage existingPackage = existing.get(); + + if (existingPackage.getDownloadStatus() == TedDailyPackage.DownloadStatus.NOT_FOUND + && isNotFoundRetryableForYear(existingPackage)) { + log.info("Retrying previously NOT_FOUND package: {}", packageId); + existingPackage.setDownloadUrl(downloadUrl); + existingPackage.setErrorMessage(null); + existingPackage.setDownloadStatus(TedDailyPackage.DownloadStatus.PENDING); + packageEntity = packageRepository.save(existingPackage); + } else { + log.debug("Package {} already exists with status: {}", packageId, existingPackage.getDownloadStatus()); + return DownloadResult.alreadyExists(existingPackage); + } + } else { + // Erstelle Package-Eintrag + packageEntity = TedDailyPackage.builder() + .packageIdentifier(packageId) + .year(year) + .serialNumber(serialNumber) + .downloadUrl(downloadUrl) + .downloadStatus(TedDailyPackage.DownloadStatus.PENDING) + .build(); + + packageEntity = packageRepository.save(packageEntity); + } + + long startTime = System.currentTimeMillis(); + + try { + // Update Status: DOWNLOADING + updatePackageStatus(packageEntity.getId(), TedDailyPackage.DownloadStatus.DOWNLOADING, null); + + // Download tar.gz file + Path downloadPath = downloadFile(downloadUrl, packageId); + + if (downloadPath == null) { + // 404 - Package existiert nicht + updatePackageStatus(packageEntity.getId(), TedDailyPackage.DownloadStatus.NOT_FOUND, + "Package not found (404)"); + return DownloadResult.notFound(packageEntity); + } + + // Berechne Hash + String fileHash = calculateSHA256(downloadPath); + + // Prüfe auf Duplikat via Hash + Optional duplicateByHash = packageRepository.findAll().stream() + .filter(p -> fileHash.equals(p.getFileHash())) + .findFirst(); + + if (duplicateByHash.isPresent()) { + log.debug("Duplicate package detected via hash: {} = {}", packageId, duplicateByHash.get().getPackageIdentifier()); + cleanupDownload(downloadPath); + updatePackageStatus(packageEntity.getId(), TedDailyPackage.DownloadStatus.COMPLETED, + "Duplicate of " + duplicateByHash.get().getPackageIdentifier()); + return DownloadResult.duplicate(packageEntity); + } + + long downloadDuration = System.currentTimeMillis() - startTime; + + // Update: DOWNLOADED + packageEntity = packageRepository.findById(packageEntity.getId()).orElseThrow(); + packageEntity.setFileHash(fileHash); + packageEntity.setDownloadStatus(TedDailyPackage.DownloadStatus.DOWNLOADED); + packageEntity.setDownloadedAt(OffsetDateTime.now()); + packageEntity.setDownloadDurationMs(downloadDuration); + packageEntity = packageRepository.save(packageEntity); + + // Extrahiere XML-Dateien + List xmlFiles = extractTarGz(downloadPath, packageId); + + packageEntity.setXmlFileCount(xmlFiles.size()); + packageEntity = packageRepository.save(packageEntity); + + // Cleanup tar.gz if configured + if (properties.getDownload().isDeleteAfterExtraction()) { + cleanupDownload(downloadPath); + } + + log.debug("Successfully downloaded package {}: {} XML files", packageId, xmlFiles.size()); + + return DownloadResult.success(packageEntity, xmlFiles); + + } catch (Exception e) { + log.error("Failed to download package {}: {}", packageId, e.getMessage(), e); + updatePackageStatus(packageEntity.getId(), TedDailyPackage.DownloadStatus.FAILED, + e.getMessage()); + return DownloadResult.failed(packageEntity, e); + } + } + + /** + * Baut die Download-URL. + */ + private String buildDownloadUrl(String packageId) { + return properties.getDownload().getBaseUrl() + packageId; + } + + /** + * Lädt eine Datei herunter. + * Gibt null zurück bei 404. + */ + private Path downloadFile(String urlString, String packageId) throws IOException { + URL url = URI.create(urlString).toURL(); + HttpURLConnection connection = (HttpURLConnection) url.openConnection(); + connection.setRequestMethod("GET"); + connection.setConnectTimeout((int) properties.getDownload().getDownloadTimeout()); + connection.setReadTimeout((int) properties.getDownload().getDownloadTimeout()); + connection.setInstanceFollowRedirects(true); + + int responseCode = connection.getResponseCode(); + + if (responseCode == 404) { + log.info("Package not found (404): {}", urlString); + return null; + } + + if (responseCode != 200) { + throw new IOException("HTTP " + responseCode + " for URL: " + urlString); + } + + // Erstelle Download-Verzeichnis + Path downloadDir = Paths.get(properties.getDownload().getDownloadDirectory()); + Files.createDirectories(downloadDir); + + // Download file + Path targetPath = downloadDir.resolve(packageId + ".tar.gz"); + + try (InputStream in = connection.getInputStream()) { + Files.copy(in, targetPath, StandardCopyOption.REPLACE_EXISTING); + } + + log.debug("Downloaded {} bytes to {}", Files.size(targetPath), targetPath); + + return targetPath; + } + + /** + * Berechnet SHA-256 Hash einer Datei. + */ + private String calculateSHA256(Path file) throws Exception { + MessageDigest digest = MessageDigest.getInstance("SHA-256"); + + try (InputStream is = Files.newInputStream(file)) { + byte[] buffer = new byte[8192]; + int read; + while ((read = is.read(buffer)) > 0) { + digest.update(buffer, 0, read); + } + } + + byte[] hashBytes = digest.digest(); + StringBuilder sb = new StringBuilder(); + for (byte b : hashBytes) { + sb.append(String.format("%02x", b)); + } + + return sb.toString(); + } + + /** + * Extrahiert tar.gz und gibt Liste der XML-Dateien zurück. + */ + private List extractTarGz(Path tarGzFile, String packageId) throws IOException { + List xmlFiles = new ArrayList<>(); + + Path extractDir = Paths.get(properties.getDownload().getExtractDirectory()) + .resolve(packageId); + Files.createDirectories(extractDir); + + try (FileInputStream fis = new FileInputStream(tarGzFile.toFile()); + GzipCompressorInputStream gzis = new GzipCompressorInputStream(fis); + TarArchiveInputStream tais = new TarArchiveInputStream(gzis)) { + + TarArchiveEntry entry; + while ((entry = tais.getNextTarEntry()) != null) { + if (entry.isDirectory()) { + continue; + } + + String name = entry.getName(); + if (!name.toLowerCase().endsWith(".xml")) { + continue; + } + + // Extrahiere XML-Datei + Path outputPath = extractDir.resolve(new File(name).getName()); + + try (OutputStream os = Files.newOutputStream(outputPath)) { + byte[] buffer = new byte[8192]; + int read; + while ((read = tais.read(buffer)) > 0) { + os.write(buffer, 0, read); + } + } + + xmlFiles.add(outputPath); + } + } + + log.debug("Extracted {} XML files from {}", xmlFiles.size(), tarGzFile.getFileName()); + + return xmlFiles; + } + + /** + * Aktualisiert den Package-Status. + */ + private void updatePackageStatus(java.util.UUID packageId, TedDailyPackage.DownloadStatus status, String errorMessage) { + packageRepository.findById(packageId).ifPresent(pkg -> { + pkg.setDownloadStatus(status); + if (errorMessage != null) { + pkg.setErrorMessage(errorMessage); + } + packageRepository.save(pkg); + }); + } + + /** + * Löscht heruntergeladene Datei. + */ + private void cleanupDownload(Path file) { + try { + Files.deleteIfExists(file); + log.debug("Cleaned up download: {}", file); + } catch (IOException e) { + log.warn("Failed to delete file {}: {}", file, e.getMessage()); + } + } + + /** + * Package-Info Record. + */ + public record PackageInfo(int year, int serialNumber) { + public String getIdentifier() { + return String.format("%04d%05d", year, serialNumber); + } + } + + /** + * Download-Ergebnis. + */ + public record DownloadResult( + TedDailyPackage packageEntity, + Status status, + List xmlFiles, + Exception error + ) { + public enum Status { + SUCCESS, + ALREADY_EXISTS, + NOT_FOUND, + DUPLICATE, + FAILED + } + + public static DownloadResult success(TedDailyPackage pkg, List files) { + return new DownloadResult(pkg, Status.SUCCESS, files, null); + } + + public static DownloadResult alreadyExists(TedDailyPackage pkg) { + return new DownloadResult(pkg, Status.ALREADY_EXISTS, List.of(), null); + } + + public static DownloadResult notFound(TedDailyPackage pkg) { + return new DownloadResult(pkg, Status.NOT_FOUND, List.of(), null); + } + + public static DownloadResult duplicate(TedDailyPackage pkg) { + return new DownloadResult(pkg, Status.DUPLICATE, List.of(), null); + } + + public static DownloadResult failed(TedDailyPackage pkg, Exception e) { + return new DownloadResult(pkg, Status.FAILED, List.of(), e); + } + + public boolean isSuccess() { + return status == Status.SUCCESS; + } + } +} diff --git a/src/main/java/at/procon/ted/service/VectorizationProcessorService.java b/src/main/java/at/procon/ted/service/VectorizationProcessorService.java new file mode 100644 index 0000000..82b61f3 --- /dev/null +++ b/src/main/java/at/procon/ted/service/VectorizationProcessorService.java @@ -0,0 +1,123 @@ +package at.procon.ted.service; + +import at.procon.ted.config.TedProcessorProperties; +import at.procon.ted.model.entity.ProcurementDocument; +import at.procon.ted.model.entity.VectorizationStatus; +import at.procon.ted.repository.ProcurementDocumentRepository; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Propagation; +import org.springframework.transaction.annotation.Transactional; + +import java.time.OffsetDateTime; +import java.util.UUID; + +/** + * Service for vectorization processing with transactional support. + * Called by VectorizationRoute to ensure proper transaction management. + * + * @author Martin.Schweitzer@procon.co.at and claude.ai + */ +@Service +@RequiredArgsConstructor +@Slf4j +public class VectorizationProcessorService { + + private final ProcurementDocumentRepository documentRepository; + private final TedProcessorProperties properties; + + /** + * Load document text content for vectorization (memory efficient - does NOT load XML). + * Updates status to PROCESSING. + * + * @return DocumentContent with text and documentId, or null if should skip + */ + @Transactional(propagation = Propagation.REQUIRES_NEW) + public DocumentContent prepareDocumentForVectorization(UUID documentId) { + // Update status to PROCESSING first + documentRepository.updateVectorizationStatus( + documentId, + VectorizationStatus.PROCESSING, + null, + null + ); + + // Load ONLY text content (not the whole document with XML) - memory efficient + String textContent = documentRepository.findTextContentById(documentId); + + if (textContent == null || textContent.isBlank()) { + documentRepository.updateVectorizationStatus( + documentId, + VectorizationStatus.SKIPPED, + "No text content available", + OffsetDateTime.now() + ); + return null; // Skip vectorization + } + + // Truncate if necessary + int maxLength = properties.getVectorization().getMaxTextLength(); + if (textContent.length() > maxLength) { + textContent = textContent.substring(0, maxLength); + log.debug("Truncated text content for document {} from {} to {} chars", + documentId, textContent.length(), maxLength); + } + + return new DocumentContent(documentId, textContent); + } + + /** + * Save embedding vector to database. + * Updates status to COMPLETED. + */ + @Transactional(propagation = Propagation.REQUIRES_NEW) + public void saveEmbedding(UUID documentId, float[] embedding, Integer tokenCount) { + if (embedding == null || embedding.length != properties.getVectorization().getDimensions()) { + throw new RuntimeException("Invalid embedding dimension: expected " + + properties.getVectorization().getDimensions() + + ", got " + (embedding != null ? embedding.length : 0)); + } + + // Convert to PostgreSQL vector format + String vectorStr = floatArrayToVectorString(embedding); + + // Update document with vector and token count + documentRepository.updateContentVector(documentId, vectorStr, tokenCount); + + log.debug("Successfully vectorized document: {} ({} tokens)", documentId, tokenCount); + } + + /** + * Mark document as failed with error message. + */ + @Transactional(propagation = Propagation.REQUIRES_NEW) + public void markAsFailed(UUID documentId, String errorMessage) { + log.debug("Vectorization failed for document {}: {}", documentId, errorMessage); + + documentRepository.updateVectorizationStatus( + documentId, + VectorizationStatus.FAILED, + errorMessage, + OffsetDateTime.now() + ); + } + + /** + * Convert float array to PostgreSQL vector format: [0.1,0.2,0.3] + */ + private String floatArrayToVectorString(float[] embedding) { + StringBuilder sb = new StringBuilder("["); + for (int i = 0; i < embedding.length; i++) { + if (i > 0) sb.append(","); + sb.append(embedding[i]); + } + sb.append("]"); + return sb.toString(); + } + + /** + * Document content holder for vectorization. + */ + public record DocumentContent(UUID documentId, String textContent) {} +} diff --git a/src/main/java/at/procon/ted/service/VectorizationService.java b/src/main/java/at/procon/ted/service/VectorizationService.java new file mode 100644 index 0000000..e5285b3 --- /dev/null +++ b/src/main/java/at/procon/ted/service/VectorizationService.java @@ -0,0 +1,164 @@ +package at.procon.ted.service; + +import at.procon.ted.config.TedProcessorProperties; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Service; + +import java.io.BufferedReader; +import java.io.InputStreamReader; +import java.io.OutputStream; +import java.net.HttpURLConnection; +import java.net.URI; +import java.net.URL; +import java.nio.charset.StandardCharsets; +import java.util.stream.Collectors; + +/** + * Service for vectorization-related utilities. + * + * This service provides helper methods for: + * - Generating query embeddings for semantic search + * - Converting float arrays to PostgreSQL vector format + * - Checking embedding service availability + * + * Note: Document vectorization is now handled by VectorizationRoute (Camel-based) + * with proper transaction management and concurrent processing. + * + * @author Martin.Schweitzer@procon.co.at and claude.ai + */ +@Service +@RequiredArgsConstructor +@Slf4j +public class VectorizationService { + + private final TedProcessorProperties properties; + + /** + * Generate query embedding with appropriate prefix for e5 model. + * Used for semantic search queries. + */ + public float[] generateQueryEmbedding(String query) { + if (!properties.getVectorization().isEnabled()) { + throw new IllegalStateException("Vectorization is disabled"); + } + + try { + String embeddingApiUrl = properties.getVectorization().getApiUrl() + "/embed"; + URL url = URI.create(embeddingApiUrl).toURL(); + HttpURLConnection conn = (HttpURLConnection) url.openConnection(); + conn.setRequestMethod("POST"); + conn.setRequestProperty("Content-Type", "application/json"); + conn.setDoOutput(true); + conn.setConnectTimeout(30000); + conn.setReadTimeout(60000); + + // Add query prefix for e5 model + String prefixedQuery = "query: " + query; + + // Send request + String requestBody = "{\"text\": " + escapeJson(prefixedQuery) + ", \"is_query\": true}"; + try (OutputStream os = conn.getOutputStream()) { + os.write(requestBody.getBytes(StandardCharsets.UTF_8)); + } + + // Read response + if (conn.getResponseCode() == 200) { + try (BufferedReader reader = new BufferedReader( + new InputStreamReader(conn.getInputStream(), StandardCharsets.UTF_8))) { + String response = reader.lines().collect(Collectors.joining()); + return parseEmbeddingJson(response); + } + } else { + throw new RuntimeException("Embedding API returned status: " + conn.getResponseCode()); + } + + } catch (Exception e) { + throw new RuntimeException("Failed to generate query embedding", e); + } + } + + /** + * Parse JSON response from embedding service. + * Expected format: {"embedding": [0.1, 0.2, ...], "dimensions": 1024} + */ + private float[] parseEmbeddingJson(String json) { + json = json.trim(); + + // Check for error response + if (json.startsWith("{\"error\"")) { + throw new RuntimeException("Embedding error: " + json); + } + + // Extract embedding array from response + // Format: {"embedding": [...], "dimensions": 1024} + int embeddingStart = json.indexOf("\"embedding\":"); + if (embeddingStart == -1) { + throw new RuntimeException("Invalid embedding response: " + json); + } + + int arrayStart = json.indexOf("[", embeddingStart); + int arrayEnd = json.indexOf("]", arrayStart); + if (arrayStart == -1 || arrayEnd == -1) { + throw new RuntimeException("Invalid embedding array in response"); + } + + String arrayContent = json.substring(arrayStart + 1, arrayEnd); + String[] parts = arrayContent.split(","); + float[] result = new float[parts.length]; + for (int i = 0; i < parts.length; i++) { + result[i] = Float.parseFloat(parts[i].trim()); + } + return result; + } + + /** + * Escape string for JSON. + */ + private String escapeJson(String text) { + return "\"" + text + .replace("\\", "\\\\") + .replace("\"", "\\\"") + .replace("\n", "\\n") + .replace("\r", "\\r") + .replace("\t", "\\t") + + "\""; + } + + /** + * Convert float array to PostgreSQL vector format: [0.1,0.2,0.3] + * Used by VectorizationProcessorService. + */ + public String floatArrayToVectorString(float[] embedding) { + StringBuilder sb = new StringBuilder("["); + for (int i = 0; i < embedding.length; i++) { + if (i > 0) sb.append(","); + sb.append(embedding[i]); + } + sb.append("]"); + return sb.toString(); + } + + /** + * Check if vectorization service is available. + */ + public boolean isAvailable() { + return properties.getVectorization().isEnabled() && isHttpApiAvailable(); + } + + /** + * Check if HTTP embedding API is reachable. + */ + private boolean isHttpApiAvailable() { + try { + String healthUrl = properties.getVectorization().getApiUrl() + "/health"; + URL url = URI.create(healthUrl).toURL(); + HttpURLConnection conn = (HttpURLConnection) url.openConnection(); + conn.setRequestMethod("GET"); + conn.setConnectTimeout(5000); + return conn.getResponseCode() == 200; + } catch (Exception e) { + return false; + } + } +} diff --git a/src/main/java/at/procon/ted/service/XmlParserService.java b/src/main/java/at/procon/ted/service/XmlParserService.java new file mode 100644 index 0000000..34bc28b --- /dev/null +++ b/src/main/java/at/procon/ted/service/XmlParserService.java @@ -0,0 +1,597 @@ +package at.procon.ted.service; + +import at.procon.ted.model.entity.*; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Service; +import org.w3c.dom.*; +import org.xml.sax.InputSource; + +import javax.xml.namespace.NamespaceContext; +import javax.xml.parsers.DocumentBuilder; +import javax.xml.parsers.DocumentBuilderFactory; +import javax.xml.xpath.*; +import java.io.StringReader; +import java.time.LocalDate; +import java.time.LocalTime; +import java.time.OffsetDateTime; +import java.time.ZoneOffset; +import java.time.format.DateTimeFormatter; +import java.time.format.DateTimeParseException; +import java.util.*; + +/** + * Service for parsing EU eForms XML documents. + * Extracts structured data from TED procurement notices. + * + * Uses XPath for navigation through the UBL 2.3 document structure with eForms extensions. + * + * @author Martin.Schweitzer@procon.co.at and claude.ai + */ +@Service +@RequiredArgsConstructor +@Slf4j +public class XmlParserService { + + // Namespace URIs for eForms/UBL documents + private static final String NS_CN = "urn:oasis:names:specification:ubl:schema:xsd:ContractNotice-2"; + private static final String NS_CAC = "urn:oasis:names:specification:ubl:schema:xsd:CommonAggregateComponents-2"; + private static final String NS_CBC = "urn:oasis:names:specification:ubl:schema:xsd:CommonBasicComponents-2"; + private static final String NS_EFAC = "http://data.europa.eu/p27/eforms-ubl-extension-aggregate-components/1"; + private static final String NS_EFBC = "http://data.europa.eu/p27/eforms-ubl-extension-basic-components/1"; + private static final String NS_EFEXT = "http://data.europa.eu/p27/eforms-ubl-extensions/1"; + private static final String NS_EXT = "urn:oasis:names:specification:ubl:schema:xsd:CommonExtensionComponents-2"; + + private final DocumentBuilderFactory documentBuilderFactory; + private final XPathFactory xPathFactory; + + public XmlParserService() { + this.documentBuilderFactory = DocumentBuilderFactory.newInstance(); + this.documentBuilderFactory.setNamespaceAware(true); + this.xPathFactory = XPathFactory.newInstance(); + } + + /** + * Parse an eForms XML document and extract structured data. + * + * @param xmlContent The XML content as string + * @return Populated ProcurementDocument entity (without ID or hash) + */ + public ProcurementDocument parseDocument(String xmlContent) { + try { + DocumentBuilder builder = documentBuilderFactory.newDocumentBuilder(); + Document doc = builder.parse(new InputSource(new StringReader(xmlContent))); + + XPath xpath = xPathFactory.newXPath(); + xpath.setNamespaceContext(createNamespaceContext()); + + ProcurementDocument document = ProcurementDocument.builder() + .xmlDocument(xmlContent) + .build(); + + // Parse basic notice information + parseNoticeMetadata(doc, xpath, document); + + // Parse contracting party (buyer) information + parseContractingParty(doc, xpath, document); + + // Parse procurement project information + parseProcurementProject(doc, xpath, document); + + // Parse tendering process + parseTenderingProcess(doc, xpath, document); + + // Parse organizations from extensions + parseOrganizations(doc, xpath, document); + + // Parse lots + parseLots(doc, xpath, document); + + // Parse publication information + parsePublication(doc, xpath, document); + + // Generate text content for vectorization + document.setTextContent(generateTextContent(document)); + + return document; + + } catch (Exception e) { + log.error("Error parsing XML document: {}", e.getMessage(), e); + throw new XmlParsingException("Failed to parse XML document", e); + } + } + + private void parseNoticeMetadata(Document doc, XPath xpath, ProcurementDocument document) throws XPathExpressionException { + // UBL Version + document.setUblVersion(getTextContent(xpath, doc, "/*[local-name()='ContractNotice']/cbc:UBLVersionID")); + + // SDK Version (customization ID) + document.setSdkVersion(getTextContent(xpath, doc, "/*[local-name()='ContractNotice']/cbc:CustomizationID")); + + // Notice ID + document.setNoticeId(getTextContent(xpath, doc, "/*[local-name()='ContractNotice']/cbc:ID")); + + // Contract Folder ID + document.setContractFolderId(getTextContent(xpath, doc, "/*[local-name()='ContractNotice']/cbc:ContractFolderID")); + + // Issue Date and Time - combined into single OffsetDateTime + String issueDateStr = getTextContent(xpath, doc, "/*[local-name()='ContractNotice']/cbc:IssueDate"); + String issueTimeStr = getTextContent(xpath, doc, "/*[local-name()='ContractNotice']/cbc:IssueTime"); + if (issueDateStr != null) { + document.setIssueDateTime(parseDateTime(issueDateStr, issueTimeStr)); + } + + // Notice Language + document.setLanguageCode(getTextContent(xpath, doc, "/*[local-name()='ContractNotice']/cbc:NoticeLanguageCode")); + + // Notice Type Code + String noticeTypeCode = getTextContent(xpath, doc, "/*[local-name()='ContractNotice']/cbc:NoticeTypeCode"); + document.setNoticeType(mapNoticeType(noticeTypeCode)); + + // Regulatory Domain + document.setRegulatoryDomain(getTextContent(xpath, doc, "/*[local-name()='ContractNotice']/cbc:RegulatoryDomain")); + + // Notice Subtype from extensions + String subtypeCode = getTextContent(xpath, doc, + "//efext:EformsExtension/efac:NoticeSubType/cbc:SubTypeCode"); + document.setNoticeSubtypeCode(subtypeCode); + } + + private void parseContractingParty(Document doc, XPath xpath, ProcurementDocument document) throws XPathExpressionException { + // Activity type + document.setBuyerActivityType(getTextContent(xpath, doc, + "//cac:ContractingParty/cac:ContractingActivity/cbc:ActivityTypeCode")); + + // Legal type + document.setBuyerLegalType(getTextContent(xpath, doc, + "//cac:ContractingParty/cac:ContractingPartyType/cbc:PartyTypeCode")); + + // Organization reference to link with organizations + String orgRef = getTextContent(xpath, doc, + "//cac:ContractingParty/cac:Party/cac:PartyIdentification/cbc:ID"); + + // Buyer details will be populated from organizations + } + + private void parseProcurementProject(Document doc, XPath xpath, ProcurementDocument document) throws XPathExpressionException { + // Project title + document.setProjectTitle(getTextContent(xpath, doc, + "/*[local-name()='ContractNotice']/cac:ProcurementProject/cbc:Name")); + + // Project description + document.setProjectDescription(getTextContent(xpath, doc, + "/*[local-name()='ContractNotice']/cac:ProcurementProject/cbc:Description")); + + // Internal reference + document.setInternalReference(getTextContent(xpath, doc, + "/*[local-name()='ContractNotice']/cac:ProcurementProject/cbc:ID")); + + // Contract nature + String contractNature = getTextContent(xpath, doc, + "/*[local-name()='ContractNotice']/cac:ProcurementProject/cbc:ProcurementTypeCode"); + document.setContractNature(mapContractNature(contractNature)); + + // CPV codes + List cpvCodes = getTextContents(xpath, doc, + "/*[local-name()='ContractNotice']/cac:ProcurementProject/cac:MainCommodityClassification/cbc:ItemClassificationCode"); + cpvCodes.addAll(getTextContents(xpath, doc, + "/*[local-name()='ContractNotice']/cac:ProcurementProject/cac:AdditionalCommodityClassification/cbc:ItemClassificationCode")); + document.setCpvCodes(cpvCodes.toArray(new String[0])); + + // Location - country and NUTS codes + document.setBuyerCountryCode(getTextContent(xpath, doc, + "/*[local-name()='ContractNotice']/cac:ProcurementProject/cac:RealizedLocation/cac:Address/cac:Country/cbc:IdentificationCode")); + document.setBuyerNutsCode(getTextContent(xpath, doc, + "/*[local-name()='ContractNotice']/cac:ProcurementProject/cac:RealizedLocation/cac:Address/cbc:CountrySubentityCode")); + document.setBuyerCity(getTextContent(xpath, doc, + "/*[local-name()='ContractNotice']/cac:ProcurementProject/cac:RealizedLocation/cac:Address/cbc:CityName")); + + // All NUTS codes from project and lots + List nutsCodes = getTextContents(xpath, doc, + "//cac:RealizedLocation/cac:Address/cbc:CountrySubentityCode"); + document.setNutsCodes(nutsCodes.stream().distinct().toArray(String[]::new)); + } + + private void parseTenderingProcess(Document doc, XPath xpath, ProcurementDocument document) throws XPathExpressionException { + // Procedure type + String procedureCode = getTextContent(xpath, doc, + "/*[local-name()='ContractNotice']/cac:TenderingProcess/cbc:ProcedureCode"); + document.setProcedureType(mapProcedureType(procedureCode)); + + // Lot distribution + String maxLotsAwarded = getTextContent(xpath, doc, + "/*[local-name()='ContractNotice']/cac:TenderingTerms/cac:LotDistribution/cbc:MaximumLotsAwardedNumeric"); + if (maxLotsAwarded != null) { + document.setMaxLotsAwarded(Integer.parseInt(maxLotsAwarded)); + } + + String maxLotsSubmitted = getTextContent(xpath, doc, + "/*[local-name()='ContractNotice']/cac:TenderingTerms/cac:LotDistribution/cbc:MaximumLotsSubmittedNumeric"); + if (maxLotsSubmitted != null) { + document.setMaxLotsSubmitted(Integer.parseInt(maxLotsSubmitted)); + } + } + + private void parseOrganizations(Document doc, XPath xpath, ProcurementDocument document) throws XPathExpressionException { + NodeList orgNodes = (NodeList) xpath.evaluate( + "//efac:Organizations/efac:Organization", doc, XPathConstants.NODESET); + + boolean buyerInfoSet = false; + + for (int i = 0; i < orgNodes.getLength(); i++) { + Node orgNode = orgNodes.item(i); + + Organization org = Organization.builder().build(); + + // Organization reference + org.setOrgReference(getTextContent(xpath, orgNode, ".//cac:PartyIdentification/cbc:ID")); + + // Name + org.setName(getTextContent(xpath, orgNode, ".//cac:PartyName/cbc:Name")); + + // Company ID + org.setCompanyId(getTextContent(xpath, orgNode, ".//cac:PartyLegalEntity/cbc:CompanyID")); + + // Address + org.setStreetName(getTextContent(xpath, orgNode, ".//cac:PostalAddress/cbc:StreetName")); + org.setCity(getTextContent(xpath, orgNode, ".//cac:PostalAddress/cbc:CityName")); + org.setPostalCode(getTextContent(xpath, orgNode, ".//cac:PostalAddress/cbc:PostalZone")); + org.setNutsCode(getTextContent(xpath, orgNode, ".//cac:PostalAddress/cbc:CountrySubentityCode")); + org.setCountryCode(getTextContent(xpath, orgNode, ".//cac:PostalAddress/cac:Country/cbc:IdentificationCode")); + + // Contact + org.setWebsiteUri(getTextContent(xpath, orgNode, ".//cbc:WebsiteURI")); + org.setEmail(getTextContent(xpath, orgNode, ".//cac:Contact/cbc:ElectronicMail")); + org.setPhone(getTextContent(xpath, orgNode, ".//cac:Contact/cbc:Telephone")); + + document.addOrganization(org); + + // Set buyer info from first organization (typically ORG-0001) + if (!buyerInfoSet && "ORG-0001".equals(org.getOrgReference())) { + document.setBuyerName(org.getName()); + if (document.getBuyerCountryCode() == null) { + document.setBuyerCountryCode(org.getCountryCode()); + } + if (document.getBuyerCity() == null) { + document.setBuyerCity(org.getCity()); + } + document.setBuyerPostalCode(org.getPostalCode()); + if (document.getBuyerNutsCode() == null) { + document.setBuyerNutsCode(org.getNutsCode()); + } + buyerInfoSet = true; + } + } + } + + private void parseLots(Document doc, XPath xpath, ProcurementDocument document) throws XPathExpressionException { + NodeList lotNodes = (NodeList) xpath.evaluate( + "//cac:ProcurementProjectLot", doc, XPathConstants.NODESET); + + document.setTotalLots(lotNodes.getLength()); + + for (int i = 0; i < lotNodes.getLength(); i++) { + Node lotNode = lotNodes.item(i); + + ProcurementLot lot = ProcurementLot.builder().build(); + + // Lot ID + lot.setLotId(getTextContent(xpath, lotNode, "cbc:ID")); + + // Internal ID + lot.setInternalId(getTextContent(xpath, lotNode, "cac:ProcurementProject/cbc:ID")); + + // Title and description + lot.setTitle(getTextContent(xpath, lotNode, "cac:ProcurementProject/cbc:Name")); + lot.setDescription(getTextContent(xpath, lotNode, "cac:ProcurementProject/cbc:Description")); + + // CPV codes for this lot + List lotCpvCodes = new ArrayList<>(); + NodeList cpvNodes = (NodeList) xpath.evaluate( + ".//cac:MainCommodityClassification/cbc:ItemClassificationCode", + lotNode, XPathConstants.NODESET); + for (int j = 0; j < cpvNodes.getLength(); j++) { + lotCpvCodes.add(cpvNodes.item(j).getTextContent()); + } + lot.setCpvCodes(lotCpvCodes.toArray(new String[0])); + + // NUTS codes for this lot + List lotNutsCodes = new ArrayList<>(); + NodeList nutsNodes = (NodeList) xpath.evaluate( + ".//cac:RealizedLocation/cac:Address/cbc:CountrySubentityCode", + lotNode, XPathConstants.NODESET); + for (int j = 0; j < nutsNodes.getLength(); j++) { + lotNutsCodes.add(nutsNodes.item(j).getTextContent()); + } + lot.setNutsCodes(lotNutsCodes.toArray(new String[0])); + + // Duration + String durationValue = getTextContent(xpath, lotNode, + "cac:ProcurementProject/cac:PlannedPeriod/cbc:DurationMeasure"); + if (durationValue != null) { + try { + lot.setDurationValue(Double.parseDouble(durationValue)); + } catch (NumberFormatException e) { + log.warn("Invalid duration value '{}' in lot {}, skipping", durationValue, lot.getLotId()); + } + } + lot.setDurationUnit(getAttributeValue(xpath, lotNode, + "cac:ProcurementProject/cac:PlannedPeriod/cbc:DurationMeasure", "unitCode")); + + // Submission deadline + String endDate = getTextContent(xpath, lotNode, + "cac:TenderingProcess/cac:TenderSubmissionDeadlinePeriod/cbc:EndDate"); + String endTime = getTextContent(xpath, lotNode, + "cac:TenderingProcess/cac:TenderSubmissionDeadlinePeriod/cbc:EndTime"); + if (endDate != null) { + lot.setSubmissionDeadline(parseDateTime(endDate, endTime)); + // Set document-level deadline from first lot if not set + if (document.getSubmissionDeadline() == null) { + document.setSubmissionDeadline(lot.getSubmissionDeadline()); + } + } + + // EU funded + String euFunded = getTextContent(xpath, lotNode, + "cac:TenderingTerms/cbc:FundingProgramCode"); + lot.setEuFunded(euFunded != null && !euFunded.contains("no-eu-funds")); + + document.addLot(lot); + } + + // Check if any lot is EU funded + document.setEuFunded(document.getLots().stream().anyMatch(l -> Boolean.TRUE.equals(l.getEuFunded()))); + } + + private void parsePublication(Document doc, XPath xpath, ProcurementDocument document) throws XPathExpressionException { + // Publication ID (OJS notice ID) + document.setPublicationId(getTextContent(xpath, doc, + "//efac:Publication/efbc:NoticePublicationID")); + + // OJS ID (gazette ID) + document.setOjsId(getTextContent(xpath, doc, + "//efac:Publication/efbc:GazetteID")); + + // Publication date + String pubDate = getTextContent(xpath, doc, + "//efac:Publication/efbc:PublicationDate"); + if (pubDate != null) { + document.setPublicationDate(parseDate(pubDate)); + } + + // Fallback to requested publication date + if (document.getPublicationDate() == null) { + String requestedPubDate = getTextContent(xpath, doc, + "/*[local-name()='ContractNotice']/cbc:RequestedPublicationDate"); + if (requestedPubDate != null) { + document.setPublicationDate(parseDate(requestedPubDate)); + } + } + } + + /** + * Generate a textual representation for vectorization. + */ + private String generateTextContent(ProcurementDocument document) { + StringBuilder sb = new StringBuilder(); + + // Title (most important) + if (document.getProjectTitle() != null) { + sb.append("Title: ").append(document.getProjectTitle()).append("\n\n"); + } + + // Description + if (document.getProjectDescription() != null) { + sb.append("Description: ").append(document.getProjectDescription()).append("\n\n"); + } + + // Buyer information + if (document.getBuyerName() != null) { + sb.append("Contracting Authority: ").append(document.getBuyerName()); + if (document.getBuyerCity() != null) { + sb.append(", ").append(document.getBuyerCity()); + } + if (document.getBuyerCountryCode() != null) { + sb.append(" (").append(document.getBuyerCountryCode()).append(")"); + } + sb.append("\n\n"); + } + + // Contract type and procedure + if (document.getContractNature() != null) { + sb.append("Contract Type: ").append(document.getContractNature()).append("\n"); + } + if (document.getProcedureType() != null) { + sb.append("Procedure: ").append(document.getProcedureType()).append("\n"); + } + + // CPV classification + if (document.getCpvCodes() != null && document.getCpvCodes().length > 0) { + sb.append("CPV Codes: ").append(String.join(", ", document.getCpvCodes())).append("\n"); + } + + // Lot information + if (document.getLots() != null && !document.getLots().isEmpty()) { + sb.append("\nLots (").append(document.getLots().size()).append("):\n"); + for (ProcurementLot lot : document.getLots()) { + if (lot.getTitle() != null) { + sb.append("- ").append(lot.getLotId()).append(": ").append(lot.getTitle()); + if (lot.getDescription() != null && !lot.getDescription().equals(lot.getTitle())) { + sb.append(" - ").append(lot.getDescription()); + } + sb.append("\n"); + } + } + } + + return sb.toString().trim(); + } + + // Helper methods + + private String getTextContent(XPath xpath, Object item, String expression) throws XPathExpressionException { + Node node = (Node) xpath.evaluate(expression, item, XPathConstants.NODE); + return node != null ? node.getTextContent().trim() : null; + } + + private List getTextContents(XPath xpath, Object item, String expression) throws XPathExpressionException { + List results = new ArrayList<>(); + NodeList nodes = (NodeList) xpath.evaluate(expression, item, XPathConstants.NODESET); + for (int i = 0; i < nodes.getLength(); i++) { + String text = nodes.item(i).getTextContent().trim(); + if (!text.isEmpty()) { + results.add(text); + } + } + return results; + } + + private String getAttributeValue(XPath xpath, Object item, String expression, String attrName) throws XPathExpressionException { + Node node = (Node) xpath.evaluate(expression, item, XPathConstants.NODE); + if (node instanceof Element) { + return ((Element) node).getAttribute(attrName); + } + return null; + } + + private LocalDate parseDate(String dateStr) { + if (dateStr == null || dateStr.isEmpty()) return null; + try { + // Handle various date formats + dateStr = dateStr.trim(); + + // Handle datetime with dash separator (e.g. "2025-04-23-03:00") + // Extract only the date part (first 10 characters: YYYY-MM-DD) + if (dateStr.matches("\\d{4}-\\d{2}-\\d{2}-\\d{2}:\\d{2}.*")) { + dateStr = dateStr.substring(0, 10); + } + + if (dateStr.contains("+")) { + dateStr = dateStr.substring(0, dateStr.indexOf("+")); + } + if (dateStr.endsWith("Z")) { + dateStr = dateStr.substring(0, dateStr.length() - 1); + } + return LocalDate.parse(dateStr); + } catch (DateTimeParseException e) { + log.warn("Failed to parse date: {} . Error: {}", dateStr, e.getMessage()); + return null; + } + } + + private LocalTime parseTime(String timeStr) { + if (timeStr == null || timeStr.isEmpty()) return null; + try { + timeStr = timeStr.trim(); + + // Handle time with offset (e.g. "12:00:00-03:00") + // Extract only the time part (first 8 characters: HH:mm:ss) + if (timeStr.matches("\\d{2}:\\d{2}:\\d{2}[+-]\\d{2}:\\d{2}")) { + timeStr = timeStr.substring(0, 8); + } + + if (timeStr.contains("+")) { + timeStr = timeStr.substring(0, timeStr.indexOf("+")); + } + if (timeStr.endsWith("Z")) { + timeStr = timeStr.substring(0, timeStr.length() - 1); + } + return LocalTime.parse(timeStr); + } catch (DateTimeParseException e) { + log.warn("Failed to parse time: {} . Error: {}", timeStr, e.getMessage()); + return null; + } + } + + private OffsetDateTime parseDateTime(String dateStr, String timeStr) { + LocalDate date = parseDate(dateStr); + if (date == null) return null; + + LocalTime time = timeStr != null ? parseTime(timeStr) : LocalTime.MIDNIGHT; + if (time == null) time = LocalTime.MIDNIGHT; + + // Parse timezone offset if present in date string + ZoneOffset offset = ZoneOffset.UTC; + if (dateStr != null && dateStr.contains("+")) { + try { + String offsetStr = dateStr.substring(dateStr.indexOf("+")); + offset = ZoneOffset.of(offsetStr); + } catch (Exception e) { + // Default to UTC + } + } + + return OffsetDateTime.of(date, time, offset); + } + + private NoticeType mapNoticeType(String code) { + if (code == null) return NoticeType.OTHER; + return switch (code.toLowerCase()) { + case "cn-standard", "cn-social", "cn-defence" -> NoticeType.CONTRACT_NOTICE; + case "pin-only", "pin-rtl", "pin-cfc-standard" -> NoticeType.PRIOR_INFORMATION_NOTICE; + case "can-standard", "can-social", "can-modif" -> NoticeType.CONTRACT_AWARD_NOTICE; + default -> NoticeType.OTHER; + }; + } + + private ContractNature mapContractNature(String code) { + if (code == null) return ContractNature.UNKNOWN; + return switch (code.toLowerCase()) { + case "supplies" -> ContractNature.SUPPLIES; + case "services" -> ContractNature.SERVICES; + case "works" -> ContractNature.WORKS; + case "mixed" -> ContractNature.MIXED; + default -> ContractNature.UNKNOWN; + }; + } + + private ProcedureType mapProcedureType(String code) { + if (code == null) return ProcedureType.OTHER; + return switch (code.toLowerCase()) { + case "open" -> ProcedureType.OPEN; + case "restricted" -> ProcedureType.RESTRICTED; + case "comp-dial" -> ProcedureType.COMPETITIVE_DIALOGUE; + case "innovation" -> ProcedureType.INNOVATION_PARTNERSHIP; + case "neg-wo-pub" -> ProcedureType.NEGOTIATED_WITHOUT_PUBLICATION; + case "neg-w-pub" -> ProcedureType.NEGOTIATED_WITH_PUBLICATION; + default -> ProcedureType.OTHER; + }; + } + + private NamespaceContext createNamespaceContext() { + return new NamespaceContext() { + @Override + public String getNamespaceURI(String prefix) { + return switch (prefix) { + case "cn" -> NS_CN; + case "cac" -> NS_CAC; + case "cbc" -> NS_CBC; + case "efac" -> NS_EFAC; + case "efbc" -> NS_EFBC; + case "efext" -> NS_EFEXT; + case "ext" -> NS_EXT; + default -> null; + }; + } + + @Override + public String getPrefix(String namespaceURI) { + return null; + } + + @Override + public Iterator getPrefixes(String namespaceURI) { + return null; + } + }; + } + + /** + * Exception thrown when XML parsing fails. + */ + public static class XmlParsingException extends RuntimeException { + public XmlParsingException(String message, Throwable cause) { + super(message, cause); + } + } +} diff --git a/src/main/java/at/procon/ted/service/attachment/AttachmentExtractor.java b/src/main/java/at/procon/ted/service/attachment/AttachmentExtractor.java new file mode 100644 index 0000000..d477fba --- /dev/null +++ b/src/main/java/at/procon/ted/service/attachment/AttachmentExtractor.java @@ -0,0 +1,87 @@ +package at.procon.ted.service.attachment; + +import java.util.List; +import java.util.Set; + +/** + * Interface for attachment content extractors. + * Each implementation handles a specific file format (PDF, ZIP, etc.). + * + * @author Martin.Schweitzer@procon.co.at and claude.ai + */ +public interface AttachmentExtractor { + + /** + * Returns the set of file extensions this extractor can handle. + * Extensions should be lowercase without the dot (e.g., "pdf", "zip"). + */ + Set getSupportedExtensions(); + + /** + * Returns the set of MIME types this extractor can handle. + */ + Set getSupportedMimeTypes(); + + /** + * Check if this extractor can handle the given file. + * + * @param filename the filename + * @param contentType the MIME content type + * @return true if this extractor can process the file + */ + boolean canHandle(String filename, String contentType); + + /** + * Extract content from the attachment. + * + * @param data the raw file data + * @param filename the original filename + * @param contentType the MIME content type + * @return extraction result containing text and/or child attachments + */ + ExtractionResult extract(byte[] data, String filename, String contentType); + + /** + * Result of content extraction. + */ + record ExtractionResult( + /** + * Extracted text content (for PDF, etc.). + */ + String extractedText, + /** + * Child attachments (for ZIP files). + */ + List childAttachments, + /** + * Whether extraction was successful. + */ + boolean success, + /** + * Error message if extraction failed. + */ + String errorMessage + ) { + public static ExtractionResult success(String text) { + return new ExtractionResult(text, List.of(), true, null); + } + + public static ExtractionResult successWithChildren(List children) { + return new ExtractionResult(null, children, true, null); + } + + public static ExtractionResult failure(String errorMessage) { + return new ExtractionResult(null, List.of(), false, errorMessage); + } + } + + /** + * Represents a child attachment extracted from a container (e.g., ZIP). + */ + record ChildAttachment( + String filename, + String contentType, + byte[] data, + String pathInArchive + ) {} +} diff --git a/src/main/java/at/procon/ted/service/attachment/AttachmentProcessingService.java b/src/main/java/at/procon/ted/service/attachment/AttachmentProcessingService.java new file mode 100644 index 0000000..0504227 --- /dev/null +++ b/src/main/java/at/procon/ted/service/attachment/AttachmentProcessingService.java @@ -0,0 +1,237 @@ +package at.procon.ted.service.attachment; + +import at.procon.ted.config.TedProcessorProperties; +import at.procon.ted.model.entity.ProcessedAttachment; +import at.procon.ted.model.entity.ProcessedAttachment.ProcessingStatus; +import at.procon.ted.repository.ProcessedAttachmentRepository; +import at.procon.ted.util.HashUtils; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; + +import java.io.File; +import java.io.FileOutputStream; +import java.time.LocalDateTime; +import java.time.format.DateTimeFormatter; +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; + +/** + * Main service for processing mail attachments. + * Handles idempotency via content hash and delegates to format-specific extractors. + * + * @author Martin.Schweitzer@procon.co.at and claude.ai + */ +@Service +@RequiredArgsConstructor +@Slf4j +public class AttachmentProcessingService { + + private final ProcessedAttachmentRepository attachmentRepository; + private final List extractors; + private final TedProcessorProperties properties; + + /** + * Process an attachment with idempotency check. + * Returns the processed attachment entity or null if it's a duplicate. + * + * @param data raw attachment data + * @param filename original filename + * @param contentType MIME content type + * @param mailSubject subject of the source email + * @param mailFrom sender of the source email + * @param parentHash hash of parent attachment (for ZIP-extracted files) + * @return processing result with entity and child attachments + */ + @Transactional + public ProcessingResult processAttachment(byte[] data, String filename, String contentType, + String mailSubject, String mailFrom, String parentHash) { + + // Calculate content hash for idempotency + String contentHash = HashUtils.computeSha256(data); + + log.debug("Processing attachment: filename='{}', contentType='{}', hash={}", + filename, contentType, contentHash); + + // Check if already processed + Optional existing = attachmentRepository.findByContentHash(contentHash); + if (existing.isPresent()) { + ProcessedAttachment existingAttachment = existing.get(); + log.info("Attachment already processed (duplicate): filename='{}', hash={}, status={}", + filename, contentHash, existingAttachment.getProcessingStatus()); + + return ProcessingResult.duplicate(existingAttachment); + } + + // Determine file type from extension + String fileType = extractFileType(filename); + + // Create entity + ProcessedAttachment attachment = ProcessedAttachment.builder() + .contentHash(contentHash) + .originalFilename(filename) + .fileType(fileType) + .contentType(contentType) + .fileSize((long) data.length) + .processingStatus(ProcessingStatus.PROCESSING) + .mailSubject(mailSubject) + .mailFrom(mailFrom) + .parentHash(parentHash) + .receivedAt(LocalDateTime.now()) + .build(); + + // Save immediately to mark as being processed + attachment = attachmentRepository.save(attachment); + + try { + // Save attachment to disk + String savedPath = saveAttachmentToDisk(data, filename, contentHash); + attachment.setSavedPath(savedPath); + + // Find appropriate extractor + AttachmentExtractor extractor = findExtractor(filename, contentType); + List childAttachments = new ArrayList<>(); + + if (extractor != null) { + log.debug("Using extractor {} for file '{}'", + extractor.getClass().getSimpleName(), filename); + + AttachmentExtractor.ExtractionResult result = extractor.extract(data, filename, contentType); + + if (result.success()) { + if (result.extractedText() != null && !result.extractedText().isBlank()) { + attachment.setExtractedText(result.extractedText()); + log.debug("Extracted {} characters of text from '{}'", + result.extractedText().length(), filename); + } + + if (result.childAttachments() != null && !result.childAttachments().isEmpty()) { + childAttachments.addAll(result.childAttachments()); + attachment.setChildCount(childAttachments.size()); + log.debug("Extracted {} child attachments from '{}'", + childAttachments.size(), filename); + } + } else { + attachment.setErrorMessage(result.errorMessage()); + log.warn("Extraction failed for '{}': {}", filename, result.errorMessage()); + } + } else { + log.debug("No extractor available for file type '{}' ({})", fileType, contentType); + } + + // Mark as completed + attachment.setProcessingStatus(ProcessingStatus.COMPLETED); + attachment.setProcessedAt(LocalDateTime.now()); + attachment = attachmentRepository.save(attachment); + + log.info("Successfully processed attachment: filename='{}', hash={}, extractedText={}, children={}", + filename, contentHash, + attachment.getExtractedText() != null ? attachment.getExtractedText().length() + " chars" : "none", + childAttachments.size()); + + return ProcessingResult.success(attachment, childAttachments); + + } catch (Exception e) { + log.error("Failed to process attachment '{}': {}", filename, e.getMessage(), e); + + attachment.setProcessingStatus(ProcessingStatus.FAILED); + attachment.setErrorMessage(e.getMessage()); + attachment.setProcessedAt(LocalDateTime.now()); + attachmentRepository.save(attachment); + + return ProcessingResult.failure(attachment, e.getMessage()); + } + } + + /** + * Find an extractor that can handle the given file. + */ + private AttachmentExtractor findExtractor(String filename, String contentType) { + for (AttachmentExtractor extractor : extractors) { + if (extractor.canHandle(filename, contentType)) { + return extractor; + } + } + return null; + } + + /** + * Extract file type/extension from filename. + */ + private String extractFileType(String filename) { + if (filename == null) { + return "unknown"; + } + int lastDot = filename.lastIndexOf('.'); + if (lastDot > 0 && lastDot < filename.length() - 1) { + return filename.substring(lastDot + 1).toLowerCase(); + } + return "unknown"; + } + + /** + * Save attachment to disk with hash-based naming. + */ + private String saveAttachmentToDisk(byte[] data, String filename, String contentHash) throws Exception { + String outputDir = properties.getMail().getAttachmentOutputDirectory(); + File dir = new File(outputDir); + if (!dir.exists()) { + dir.mkdirs(); + } + + // Create filename with timestamp and hash for uniqueness + String timestamp = LocalDateTime.now().format(DateTimeFormatter.ofPattern("yyyyMMdd_HHmmss")); + String safeFilename = sanitizeFilename(filename); + String shortHash = contentHash.substring(0, 8); + String outputFilename = timestamp + "_" + shortHash + "_" + safeFilename; + + File outputFile = new File(dir, outputFilename); + + try (FileOutputStream fos = new FileOutputStream(outputFile)) { + fos.write(data); + } + + log.debug("Saved attachment to: {}", outputFile.getAbsolutePath()); + return outputFile.getAbsolutePath(); + } + + /** + * Sanitize filename for filesystem safety. + */ + private String sanitizeFilename(String filename) { + if (filename == null) { + return "unnamed"; + } + return filename.replaceAll("[\\\\/:*?\"<>|]", "_"); + } + + /** + * Result of attachment processing. + */ + public record ProcessingResult( + ProcessedAttachment attachment, + List childAttachments, + boolean isDuplicate, + boolean isSuccess, + String errorMessage + ) { + public static ProcessingResult success(ProcessedAttachment attachment, + List children) { + return new ProcessingResult(attachment, children, false, true, null); + } + + public static ProcessingResult duplicate(ProcessedAttachment attachment) { + return new ProcessingResult(attachment, List.of(), true, true, null); + } + + public static ProcessingResult failure(ProcessedAttachment attachment, String error) { + return new ProcessingResult(attachment, List.of(), false, false, error); + } + + public boolean hasChildren() { + return childAttachments != null && !childAttachments.isEmpty(); + } + } +} diff --git a/src/main/java/at/procon/ted/service/attachment/PdfExtractionService.java b/src/main/java/at/procon/ted/service/attachment/PdfExtractionService.java new file mode 100644 index 0000000..e1e87f6 --- /dev/null +++ b/src/main/java/at/procon/ted/service/attachment/PdfExtractionService.java @@ -0,0 +1,115 @@ +package at.procon.ted.service.attachment; + +import lombok.extern.slf4j.Slf4j; +import org.apache.pdfbox.Loader; +import org.apache.pdfbox.pdmodel.PDDocument; +import org.apache.pdfbox.text.PDFTextStripper; +import org.springframework.stereotype.Service; + +import java.io.ByteArrayInputStream; +import java.util.Set; + +/** + * Service for extracting text content from PDF files. + * Uses Apache PDFBox for PDF parsing and text extraction. + * + * @author Martin.Schweitzer@procon.co.at and claude.ai + */ +@Service +@Slf4j +public class PdfExtractionService implements AttachmentExtractor { + + private static final Set SUPPORTED_EXTENSIONS = Set.of("pdf"); + private static final Set SUPPORTED_MIME_TYPES = Set.of( + "application/pdf", + "application/x-pdf" + ); + + @Override + public Set getSupportedExtensions() { + return SUPPORTED_EXTENSIONS; + } + + @Override + public Set getSupportedMimeTypes() { + return SUPPORTED_MIME_TYPES; + } + + @Override + public boolean canHandle(String filename, String contentType) { + if (filename != null) { + String lowerFilename = filename.toLowerCase(); + if (SUPPORTED_EXTENSIONS.stream().anyMatch(ext -> lowerFilename.endsWith("." + ext))) { + return true; + } + } + if (contentType != null) { + String lowerContentType = contentType.toLowerCase().split(";")[0].trim(); + return SUPPORTED_MIME_TYPES.contains(lowerContentType); + } + return false; + } + + @Override + public ExtractionResult extract(byte[] data, String filename, String contentType) { + if (data == null || data.length == 0) { + return ExtractionResult.failure("Empty PDF data"); + } + + log.debug("Extracting text from PDF: {} ({} bytes)", filename, data.length); + + try (PDDocument document = Loader.loadPDF(data)) { + // Check if document is encrypted + if (document.isEncrypted()) { + log.warn("PDF is encrypted, attempting to decrypt with empty password: {}", filename); + try { + document.setAllSecurityToBeRemoved(true); + } catch (Exception e) { + return ExtractionResult.failure("PDF is encrypted and cannot be decrypted: " + e.getMessage()); + } + } + + PDFTextStripper stripper = new PDFTextStripper(); + stripper.setSortByPosition(true); + + String text = stripper.getText(document); + + // Clean up extracted text + text = cleanExtractedText(text); + + int pageCount = document.getNumberOfPages(); + log.info("Successfully extracted {} characters from {} pages of PDF: {}", + text.length(), pageCount, filename); + + return ExtractionResult.success(text); + + } catch (Exception e) { + log.error("Failed to extract text from PDF '{}': {}", filename, e.getMessage(), e); + return ExtractionResult.failure("PDF extraction failed: " + e.getMessage()); + } + } + + /** + * Clean up extracted text by removing excessive whitespace and normalizing line breaks. + */ + private String cleanExtractedText(String text) { + if (text == null) { + return ""; + } + + // Normalize line breaks + text = text.replaceAll("\r\n", "\n"); + text = text.replaceAll("\r", "\n"); + + // Remove excessive blank lines (more than 2 consecutive) + text = text.replaceAll("\n{3,}", "\n\n"); + + // Remove trailing whitespace from each line + text = text.replaceAll("[ \t]+\n", "\n"); + + // Trim leading/trailing whitespace + text = text.trim(); + + return text; + } +} diff --git a/src/main/java/at/procon/ted/service/attachment/ZipExtractionService.java b/src/main/java/at/procon/ted/service/attachment/ZipExtractionService.java new file mode 100644 index 0000000..92157ec --- /dev/null +++ b/src/main/java/at/procon/ted/service/attachment/ZipExtractionService.java @@ -0,0 +1,234 @@ +package at.procon.ted.service.attachment; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Service; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.net.URLConnection; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; +import java.util.Set; +import java.util.zip.ZipEntry; +import java.util.zip.ZipInputStream; + +/** + * Service for extracting files from ZIP archives. + * Extracts all contained files as child attachments for recursive processing. + * + * @author Martin.Schweitzer@procon.co.at and claude.ai + */ +@Service +@Slf4j +public class ZipExtractionService implements AttachmentExtractor { + + private static final Set SUPPORTED_EXTENSIONS = Set.of("zip"); + private static final Set SUPPORTED_MIME_TYPES = Set.of( + "application/zip", + "application/x-zip", + "application/x-zip-compressed", + "application/octet-stream" // Often used for ZIP files + ); + + // Security limits + private static final long MAX_TOTAL_SIZE = 500 * 1024 * 1024; // 500 MB total extracted size + private static final long MAX_SINGLE_FILE_SIZE = 100 * 1024 * 1024; // 100 MB per file + private static final int MAX_FILES = 1000; // Maximum number of files in archive + private static final int MAX_PATH_LENGTH = 500; // Maximum path length + + @Override + public Set getSupportedExtensions() { + return SUPPORTED_EXTENSIONS; + } + + @Override + public Set getSupportedMimeTypes() { + return SUPPORTED_MIME_TYPES; + } + + @Override + public boolean canHandle(String filename, String contentType) { + if (filename != null) { + String lowerFilename = filename.toLowerCase(); + if (SUPPORTED_EXTENSIONS.stream().anyMatch(ext -> lowerFilename.endsWith("." + ext))) { + return true; + } + } + // Only use MIME type if it's explicitly zip, not application/octet-stream + if (contentType != null) { + String lowerContentType = contentType.toLowerCase().split(";")[0].trim(); + if (lowerContentType.contains("zip")) { + return true; + } + } + return false; + } + + @Override + public ExtractionResult extract(byte[] data, String filename, String contentType) { + if (data == null || data.length == 0) { + return ExtractionResult.failure("Empty ZIP data"); + } + + log.debug("Extracting files from ZIP: {} ({} bytes)", filename, data.length); + + List children = new ArrayList<>(); + long totalExtractedSize = 0; + int fileCount = 0; + + try (ZipInputStream zis = new ZipInputStream(new ByteArrayInputStream(data))) { + ZipEntry entry; + + while ((entry = zis.getNextEntry()) != null) { + // Security check: skip directories + if (entry.isDirectory()) { + zis.closeEntry(); + continue; + } + + String entryName = entry.getName(); + + // Security check: path traversal protection + if (entryName.contains("..") || entryName.startsWith("/") || entryName.startsWith("\\")) { + log.warn("Skipping potentially malicious ZIP entry: {}", entryName); + zis.closeEntry(); + continue; + } + + // Security check: path length + if (entryName.length() > MAX_PATH_LENGTH) { + log.warn("Skipping ZIP entry with too long path: {}", entryName.substring(0, 100) + "..."); + zis.closeEntry(); + continue; + } + + // Security check: maximum files + if (fileCount >= MAX_FILES) { + log.warn("ZIP file contains too many files, stopping at {} files", MAX_FILES); + break; + } + + // Read entry content + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + byte[] buffer = new byte[8192]; + int len; + long entrySize = 0; + + while ((len = zis.read(buffer)) > 0) { + entrySize += len; + + // Security check: single file size + if (entrySize > MAX_SINGLE_FILE_SIZE) { + log.warn("Skipping ZIP entry exceeding max file size: {} (> {} MB)", + entryName, MAX_SINGLE_FILE_SIZE / 1024 / 1024); + break; + } + + // Security check: total extracted size (zip bomb protection) + if (totalExtractedSize + entrySize > MAX_TOTAL_SIZE) { + log.warn("ZIP extraction stopped: total extracted size exceeds limit ({} MB)", + MAX_TOTAL_SIZE / 1024 / 1024); + return ExtractionResult.successWithChildren(children); + } + + baos.write(buffer, 0, len); + } + + if (entrySize > MAX_SINGLE_FILE_SIZE) { + zis.closeEntry(); + continue; + } + + byte[] entryData = baos.toByteArray(); + totalExtractedSize += entryData.length; + fileCount++; + + // Determine content type from filename + String childContentType = guessContentType(entryName); + + // Extract just the filename from the path + String childFilename = extractFilename(entryName); + + ChildAttachment child = new ChildAttachment( + childFilename, + childContentType, + entryData, + entryName + ); + children.add(child); + + log.debug("Extracted from ZIP: {} ({} bytes, type={})", + entryName, entryData.length, childContentType); + + zis.closeEntry(); + } + + log.info("Successfully extracted {} files ({} bytes total) from ZIP: {}", + children.size(), totalExtractedSize, filename); + + return ExtractionResult.successWithChildren(children); + + } catch (Exception e) { + log.error("Failed to extract ZIP '{}': {}", filename, e.getMessage(), e); + return ExtractionResult.failure("ZIP extraction failed: " + e.getMessage()); + } + } + + /** + * Guess the MIME content type from a filename. + */ + private String guessContentType(String filename) { + if (filename == null) { + return "application/octet-stream"; + } + + String lowerFilename = filename.toLowerCase(); + + // Common types + if (lowerFilename.endsWith(".pdf")) { + return "application/pdf"; + } else if (lowerFilename.endsWith(".xml")) { + return "application/xml"; + } else if (lowerFilename.endsWith(".zip")) { + return "application/zip"; + } else if (lowerFilename.endsWith(".txt")) { + return "text/plain"; + } else if (lowerFilename.endsWith(".html") || lowerFilename.endsWith(".htm")) { + return "text/html"; + } else if (lowerFilename.endsWith(".json")) { + return "application/json"; + } else if (lowerFilename.endsWith(".doc")) { + return "application/msword"; + } else if (lowerFilename.endsWith(".docx")) { + return "application/vnd.openxmlformats-officedocument.wordprocessingml.document"; + } else if (lowerFilename.endsWith(".xls")) { + return "application/vnd.ms-excel"; + } else if (lowerFilename.endsWith(".xlsx")) { + return "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"; + } else if (lowerFilename.endsWith(".png")) { + return "image/png"; + } else if (lowerFilename.endsWith(".jpg") || lowerFilename.endsWith(".jpeg")) { + return "image/jpeg"; + } + + // Try to guess from URLConnection + String guessed = URLConnection.guessContentTypeFromName(filename); + return guessed != null ? guessed : "application/octet-stream"; + } + + /** + * Extract just the filename from a path (handles both / and \ separators). + */ + private String extractFilename(String path) { + if (path == null) { + return "unnamed"; + } + int lastSlash = Math.max(path.lastIndexOf('/'), path.lastIndexOf('\\')); + if (lastSlash >= 0 && lastSlash < path.length() - 1) { + return path.substring(lastSlash + 1); + } + return path; + } +} diff --git a/src/main/java/at/procon/ted/startup/OrganizationSchemaFixRunner.java b/src/main/java/at/procon/ted/startup/OrganizationSchemaFixRunner.java new file mode 100644 index 0000000..242428a --- /dev/null +++ b/src/main/java/at/procon/ted/startup/OrganizationSchemaFixRunner.java @@ -0,0 +1,104 @@ +package at.procon.ted.startup; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.boot.ApplicationArguments; +import org.springframework.boot.ApplicationRunner; +import org.springframework.core.annotation.Order; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.stereotype.Component; + +/** + * Startup runner that fixes the organization table schema if needed. + * This is a workaround for Flyway V2 migration not being applied automatically. + * + * Extends VARCHAR fields to handle long TED data. + * + * @author Martin.Schweitzer@procon.co.at and claude.ai + */ +@Component +@Order(1) // Run before other startup runners +@RequiredArgsConstructor +@Slf4j +public class OrganizationSchemaFixRunner implements ApplicationRunner { + + private final JdbcTemplate jdbcTemplate; + + @Override + public void run(ApplicationArguments args) throws Exception { + log.info("Checking organization table schema..."); + + try { + // Check if schema fix is needed by trying to query column types + String checkSql = """ + SELECT column_name, character_maximum_length, data_type + FROM information_schema.columns + WHERE table_schema = 'ted' + AND table_name = 'organization' + AND column_name IN ('postal_code', 'company_id', 'name') + ORDER BY column_name + """; + + var columnInfo = jdbcTemplate.queryForList(checkSql); + boolean needsFix = false; + + for (var row : columnInfo) { + String columnName = (String) row.get("column_name"); + Integer maxLength = (Integer) row.get("character_maximum_length"); + String dataType = (String) row.get("data_type"); + + log.debug("Column {}: type={}, max_length={}", columnName, dataType, maxLength); + + // Check if any field is still too small + if ("postal_code".equals(columnName) && maxLength != null && maxLength < 255) { + needsFix = true; + log.warn("Column postal_code is too small: {} chars, needs 255", maxLength); + } + if ("company_id".equals(columnName) && maxLength != null && maxLength < 255) { + needsFix = true; + log.warn("Column company_id is too small: {} chars, needs 255", maxLength); + } + } + + if (needsFix) { + log.warn("Organization table schema needs fixing - applying migration..."); + applySchemaFix(); + log.info("Organization table schema fixed successfully!"); + } else { + log.info("Organization table schema is up to date"); + } + + } catch (Exception e) { + log.error("Failed to check/fix organization table schema: {}", e.getMessage(), e); + throw e; + } + } + + private void applySchemaFix() { + log.info("Applying schema fix to ted.organization table..."); + + // Apply all column type changes + // Use TEXT for fields that can be extremely long + String[] alterStatements = { + "ALTER TABLE ted.organization ALTER COLUMN postal_code TYPE TEXT", + "ALTER TABLE ted.organization ALTER COLUMN street_name TYPE TEXT", + "ALTER TABLE ted.organization ALTER COLUMN city TYPE TEXT", // Some cities have very long names + "ALTER TABLE ted.organization ALTER COLUMN phone TYPE VARCHAR(100)", + "ALTER TABLE ted.organization ALTER COLUMN org_reference TYPE VARCHAR(100)", + "ALTER TABLE ted.organization ALTER COLUMN role TYPE VARCHAR(100)", + "ALTER TABLE ted.organization ALTER COLUMN company_id TYPE TEXT", // Can be very long + "ALTER TABLE ted.organization ALTER COLUMN name TYPE TEXT" + }; + + for (String sql : alterStatements) { + try { + jdbcTemplate.execute(sql); + log.debug("Executed: {}", sql); + } catch (Exception e) { + log.warn("Failed to execute {}: {} (may already be applied)", sql, e.getMessage()); + } + } + + log.info("Schema fix applied successfully"); + } +} diff --git a/src/main/java/at/procon/ted/startup/VectorizationStartupRunner.java b/src/main/java/at/procon/ted/startup/VectorizationStartupRunner.java new file mode 100644 index 0000000..2048f3d --- /dev/null +++ b/src/main/java/at/procon/ted/startup/VectorizationStartupRunner.java @@ -0,0 +1,112 @@ +package at.procon.ted.startup; + +import at.procon.ted.config.TedProcessorProperties; +import at.procon.ted.model.entity.VectorizationStatus; +import at.procon.ted.repository.ProcurementDocumentRepository; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.apache.camel.ProducerTemplate; +import org.springframework.boot.ApplicationArguments; +import org.springframework.boot.ApplicationRunner; +import org.springframework.data.domain.PageRequest; +import org.springframework.data.domain.Pageable; +import org.springframework.stereotype.Component; + +import java.util.List; +import java.util.UUID; + +/** + * Startup runner that processes all pending and failed vectorizations on application start. + * + * This ensures that any documents that were saved but not yet vectorized + * or failed during vectorization (e.g., due to service restart or embedding service issues) + * are immediately queued for (re-)vectorization. + * + * Memory efficient: Only loads document IDs, not full entities. + * Processes in batches to avoid holding database connections for too long. + * + * @author Martin.Schweitzer@procon.co.at and claude.ai + */ +@Component +@RequiredArgsConstructor +@Slf4j +public class VectorizationStartupRunner implements ApplicationRunner { + + private final ProcurementDocumentRepository documentRepository; + private final ProducerTemplate producerTemplate; + private final TedProcessorProperties properties; + + private static final int BATCH_SIZE = 1000; + + @Override + public void run(ApplicationArguments args) throws Exception { + if (!properties.getVectorization().isEnabled()) { + log.info("Vectorization is disabled, skipping startup processing"); + return; + } + + log.info("Checking for pending and failed vectorizations on startup..."); + + try { + int successCount = 0; + int queueFailedCount = 0; + + // Process PENDING documents first (higher priority) + successCount += processDocumentsByStatus(VectorizationStatus.PENDING, "PENDING"); + + // Then process FAILED documents (retry) + successCount += processDocumentsByStatus(VectorizationStatus.FAILED, "FAILED"); + + if (successCount == 0 && queueFailedCount == 0) { + log.info("No pending or failed vectorizations found"); + return; + } + + log.info("Startup vectorization processing completed: {} queued successfully, {} failed to queue", + successCount, queueFailedCount); + + } catch (Exception e) { + log.error("Error during startup vectorization processing: {}", e.getMessage(), e); + } + } + + /** + * Process documents by status in batches to avoid connection leaks. + */ + private int processDocumentsByStatus(VectorizationStatus status, String statusName) { + int successCount = 0; + int page = 0; + List documentIds; + + do { + // Load batch of document IDs (memory efficient - only IDs, not full entities) + Pageable pageable = PageRequest.of(page, BATCH_SIZE); + documentIds = documentRepository.findIdsByVectorizationStatus(status, pageable); + + if (documentIds.isEmpty()) { + break; + } + + if (page == 0) { + log.info("Found {} documents with status {}, processing in batches of {}...", + documentIds.size(), statusName, BATCH_SIZE); + } + + // Queue each document for vectorization + for (UUID documentId : documentIds) { + try { + producerTemplate.sendBodyAndHeader("direct:vectorize", null, "documentId", documentId); + successCount++; + } catch (Exception e) { + log.warn("Failed to queue {} document {} for vectorization: {}", + statusName, documentId, e.getMessage()); + } + } + + page++; + + } while (documentIds.size() == BATCH_SIZE); + + return successCount; + } +} diff --git a/src/main/java/at/procon/ted/util/HashUtils.java b/src/main/java/at/procon/ted/util/HashUtils.java new file mode 100644 index 0000000..04b0241 --- /dev/null +++ b/src/main/java/at/procon/ted/util/HashUtils.java @@ -0,0 +1,82 @@ +package at.procon.ted.util; + +import lombok.extern.slf4j.Slf4j; + +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; + +/** + * Utility class for computing document hashes. + * Uses SHA-256 for generating unique document identifiers. + * + * @author Martin.Schweitzer@procon.co.at and claude.ai + */ +@Slf4j +public final class HashUtils { + + private static final char[] HEX_ARRAY = "0123456789abcdef".toCharArray(); + + private HashUtils() { + // Utility class, no instantiation + } + + /** + * Compute SHA-256 hash of the given content. + * + * @param content The content to hash + * @return Lowercase hex-encoded hash string (64 characters) + */ + public static String computeSha256(String content) { + try { + MessageDigest digest = MessageDigest.getInstance("SHA-256"); + byte[] hashBytes = digest.digest(content.getBytes(StandardCharsets.UTF_8)); + return bytesToHex(hashBytes); + } catch (NoSuchAlgorithmException e) { + // SHA-256 is always available in Java + throw new RuntimeException("SHA-256 algorithm not available", e); + } + } + + /** + * Compute SHA-256 hash of the given byte array. + * + * @param content The content to hash + * @return Lowercase hex-encoded hash string (64 characters) + */ + public static String computeSha256(byte[] content) { + try { + MessageDigest digest = MessageDigest.getInstance("SHA-256"); + byte[] hashBytes = digest.digest(content); + return bytesToHex(hashBytes); + } catch (NoSuchAlgorithmException e) { + throw new RuntimeException("SHA-256 algorithm not available", e); + } + } + + /** + * Convert byte array to lowercase hexadecimal string. + */ + private static String bytesToHex(byte[] bytes) { + char[] hexChars = new char[bytes.length * 2]; + for (int i = 0; i < bytes.length; i++) { + int v = bytes[i] & 0xFF; + hexChars[i * 2] = HEX_ARRAY[v >>> 4]; + hexChars[i * 2 + 1] = HEX_ARRAY[v & 0x0F]; + } + return new String(hexChars); + } + + /** + * Validate a hash string format. + * + * @param hash The hash to validate + * @return true if valid SHA-256 hex string + */ + public static boolean isValidSha256(String hash) { + if (hash == null || hash.length() != 64) { + return false; + } + return hash.matches("^[a-f0-9]{64}$"); + } +} diff --git a/src/main/java/at/procon/ted/util/InspectDatabase.java b/src/main/java/at/procon/ted/util/InspectDatabase.java new file mode 100644 index 0000000..12a91ba --- /dev/null +++ b/src/main/java/at/procon/ted/util/InspectDatabase.java @@ -0,0 +1,86 @@ +package at.procon.ted.util; + +import lombok.extern.slf4j.Slf4j; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.ResultSet; +import java.sql.Statement; + +/** + * Utility to inspect database objects that depend on ENUM types. + */ +@Slf4j +public class InspectDatabase { + + private static final String DB_URL = "jdbc:postgresql://94.130.218.54:5432/Sales"; + private static final String DB_USER = "postgres"; + private static final String DB_PASSWORD = "PDmXRx0Rbk9OFOn9qO5Gm/mPCfqW8zwbZ+/YIU1lySc="; + + public static void main(String[] args) { + try (Connection conn = DriverManager.getConnection(DB_URL, DB_USER, DB_PASSWORD)) { + log.info("Connected to database"); + + // Check for views + log.info("\n=== VIEWS ==="); + try (Statement stmt = conn.createStatement(); + ResultSet rs = stmt.executeQuery( + "SELECT schemaname, viewname FROM pg_views WHERE schemaname ILIKE 'ted'")) { + while (rs.next()) { + log.info("View: {}.{}", rs.getString(1), rs.getString(2)); + } + } + + // Check for functions + log.info("\n=== FUNCTIONS ==="); + try (Statement stmt = conn.createStatement(); + ResultSet rs = stmt.executeQuery( + "SELECT n.nspname, p.proname FROM pg_proc p " + + "JOIN pg_namespace n ON p.pronamespace = n.oid " + + "WHERE n.nspname ILIKE 'ted'")) { + while (rs.next()) { + log.info("Function: {}.{}", rs.getString(1), rs.getString(2)); + } + } + + // Check for indexes + log.info("\n=== INDEXES ==="); + try (Statement stmt = conn.createStatement(); + ResultSet rs = stmt.executeQuery( + "SELECT schemaname, tablename, indexname FROM pg_indexes WHERE schemaname ILIKE 'ted'")) { + while (rs.next()) { + log.info("Index: {}.{} on table {}", rs.getString(1), rs.getString(3), rs.getString(2)); + } + } + + // Check column types + log.info("\n=== ENUM COLUMNS ==="); + try (Statement stmt = conn.createStatement(); + ResultSet rs = stmt.executeQuery( + "SELECT table_schema, table_name, column_name, udt_name " + + "FROM information_schema.columns " + + "WHERE table_schema ILIKE 'ted' AND table_name = 'procurement_document' " + + "AND column_name IN ('notice_type', 'contract_nature', 'procedure_type', 'vectorization_status')")) { + while (rs.next()) { + log.info("Column: {}.{}.{} -> Type: {}", + rs.getString(1), rs.getString(2), rs.getString(3), rs.getString(4)); + } + } + + // Check for ENUM types + log.info("\n=== ENUM TYPES ==="); + try (Statement stmt = conn.createStatement(); + ResultSet rs = stmt.executeQuery( + "SELECT n.nspname, t.typname FROM pg_type t " + + "JOIN pg_namespace n ON t.typnamespace = n.oid " + + "WHERE n.nspname ILIKE 'ted' AND t.typtype = 'e'")) { + while (rs.next()) { + log.info("ENUM Type: {}.{}", rs.getString(1), rs.getString(2)); + } + } + + } catch (Exception e) { + log.error("Error: {}", e.getMessage(), e); + } + } +} diff --git a/src/main/resources/application - Kopie.yml b/src/main/resources/application - Kopie.yml new file mode 100644 index 0000000..5526b1d --- /dev/null +++ b/src/main/resources/application - Kopie.yml @@ -0,0 +1,234 @@ +# TED Procurement Document Processor Configuration +# Author: Martin.Schweitzer@procon.co.at and claude.ai + +server: + port: 8888 + servlet: + context-path: /api + +spring: + application: + name: ted-procurement-processor + + datasource: + url: jdbc:postgresql://localhost:32333/RELM + username: ${DB_USERNAME:postgres} + password: ${DB_PASSWORD:pwd} + driver-class-name: org.postgresql.Driver + hikari: + maximum-pool-size: 5 + minimum-idle: 2 + connection-timeout: 30000 + idle-timeout: 300000 + max-lifetime: 900000 + leak-detection-threshold: 120000 # 2 minutes - increased to avoid false positives with batch processing + + jpa: + hibernate: + ddl-auto: none + show-sql: false + open-in-view: false + properties: + hibernate: + format_sql: true + default_schema: TED + jdbc: + batch_size: 25 # Match chunk size for optimal batch processing + order_inserts: true + order_updates: true + + flyway: + enabled: true + locations: classpath:db/migration + baseline-on-migrate: true + create-schemas: true + schemas: TED + default-schema: TED + +# Apache Camel Configuration +camel: + springboot: + main-run-controller: true + health: + enabled: true + # Weniger strenge Health-Checks für File-Consumer + consumers-enabled: false + +# Custom Application Properties +ted: + # Directory configuration for file processing + input: + # Base directory for watching incoming TED XML files + directory: ${TED_INPUT_DIR:D:/ted.europe/extracted} + # File pattern to match (recursive scanning) + pattern: "**/*.xml" + # Move processed files to this directory + processed-directory: ${TED_PROCESSED_DIR:.processed} + # Move failed files to this directory + error-directory: ${TED_ERROR_DIR:.error} + # Polling interval in milliseconds + poll-interval: 5000 + # Maximum messages per poll (reduced to prevent memory issues) + max-messages-per-poll: 10 + + # Schema validation configuration + schema: + # Enable/disable XSD validation + enabled: true + # Path to eForms SDK schemas (from Maven dependency or custom location) + path: classpath:schemas/maindoc/UBL-ContractNotice-2.3.xsd + + # Vectorization configuration + vectorization: + # Enable/disable async vectorization + enabled: true + # Use external HTTP API instead of subprocess + use-http-api: true + # Embedding service URL + api-url: http://localhost:8001 + # Model name for sentence-transformers + model-name: intfloat/multilingual-e5-large + # Vector dimensions (must match model output) + dimensions: 1024 + # Batch size for vectorization + batch-size: 16 + # Thread pool size for async processing + thread-pool-size: 4 + # Maximum text length for vectorization (characters) + max-text-length: 8192 + # HTTP connection timeout (milliseconds) + connect-timeout: 10000 + # HTTP socket/read timeout (milliseconds) + socket-timeout: 60000 + # Maximum retries on connection failure + max-retries: 5 + + # Search configuration + search: + # Default page size for search results + default-page-size: 20 + # Maximum page size + max-page-size: 100 + # Similarity threshold for vector search (0.0 - 1.0) + similarity-threshold: 0.7 + + # TED Daily Package Download configuration + download: + # Enable/disable automatic package download + enabled: true + # Base URL for TED Daily Packages + base-url: https://ted.europa.eu/packages/daily/ + # Download directory for tar.gz files + download-directory: D:/ted.europe/downloads + # Extract directory for XML files + extract-directory: D:/ted.europe/extracted + # Start year for downloads + start-year: 2015 + # Max consecutive 404 errors before stopping + max-consecutive-404: 4 + # Polling interval (milliseconds) - 2 minutes + poll-interval: 120000 + # Download timeout (milliseconds) - 5 minutes + download-timeout: 300000 + # Max concurrent downloads + max-concurrent-downloads: 2 + # Delay between downloads (milliseconds) for rate limiting - 5 seconds + delay-between-downloads: 3000 + # Delete tar.gz after extraction + delete-after-extraction: true + # Prioritize current year first + prioritize-current-year: false + + # IMAP Mail configuration + mail: + # Enable/disable mail processing + enabled: true + # IMAP server hostname + host: host + # IMAP server port (993 for IMAPS) + port: 993 + # Mail account username (email address) + username: ${MAIL_USERNAME:} + # Mail account password + password: ${MAIL_PASSWORD:} + # Use SSL/TLS connection + ssl: true + # Mail folder to read from + folder-name: INBOX + # Delete messages after processing + delete: false + # Mark messages as seen after processing (false = peek mode, don't mark as read) + seen: false + # Only process unseen messages + unseen: true + # Polling delay in milliseconds (1 minute) + delay: 60000 + # Max messages per poll + max-messages-per-poll: 10 + # Output directory for processed attachments + attachment-output-directory: D:/ted.europe/mail-attachments + # Enable/disable MIME file input processing + mime-input-enabled: true + # Input directory for MIME files (.eml) + mime-input-directory: D:/ted.europe/mime-input + # File pattern for MIME files (regex) + mime-input-pattern: .*\\.eml + # Polling interval for MIME input directory (milliseconds) + mime-input-poll-interval: 10000 + + # Solution Brief processing configuration + solution-brief: + # Enable/disable Solution Brief processing + enabled: true + # Input directory for Solution Brief PDF files + input-directory: C:/work/SolutionBrief + # Output directory for Excel result files (relative to input or absolute) + result-directory: ./result + # Number of top similar documents to include + top-k: 20 + # Minimum similarity threshold (0.0-1.0) + similarity-threshold: 0.5 + # Polling interval in milliseconds (30 seconds) + poll-interval: 30000 + # File pattern for PDF files (regex) + file-pattern: .*\\.pdf + # Process files only once (idempotent) + idempotent: true + # Idempotent repository file path + idempotent-repository: ./solution-brief-processed.dat + + # Data cleanup configuration + cleanup: + # Enable automatic cleanup of old documents + enabled: false + # Retention period in years (default: 10) + retention-years: 10 + # Cron expression for cleanup schedule (default: daily at 2 AM) + cron: "0 0 2 * * *" + +# Actuator endpoints +management: + endpoints: + web: + exposure: + include: health,info,metrics,camel + endpoint: + health: + show-details: when-authorized + +# OpenAPI documentation +springdoc: + api-docs: + path: /v3/api-docs + swagger-ui: + path: /swagger-ui.html + operations-sorter: method + +# Logging configuration +logging: + level: + at.procon.ted: INFO + at.procon.ted.camel.SolutionBriefRoute: INFO + org.apache.camel: INFO + org.hibernate.SQL: WARN + org.hibernate.type.descriptor.sql: WARN diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml new file mode 100644 index 0000000..8edd412 --- /dev/null +++ b/src/main/resources/application.yml @@ -0,0 +1,240 @@ +# TED Procurement Document Processor Configuration +# Author: Martin.Schweitzer@procon.co.at and claude.ai + +server: + port: 8888 + servlet: + context-path: /api + +spring: + application: + name: ted-procurement-processor + + datasource: + url: jdbc:postgresql://94.130.218.54:32333/RELM + username: ${DB_USERNAME:postgres} + password: ${DB_PASSWORD:PDmXRx0Rbk9OFOn9qO5Gm/mPCfqW8zwbZ+/YIU1lySc=} + driver-class-name: org.postgresql.Driver + hikari: + maximum-pool-size: 5 + minimum-idle: 2 + connection-timeout: 30000 + idle-timeout: 300000 + max-lifetime: 900000 + leak-detection-threshold: 120000 # 2 minutes - increased to avoid false positives with batch processing + + jpa: + hibernate: + ddl-auto: none + show-sql: false + open-in-view: false + properties: + hibernate: + format_sql: true + default_schema: TED + jdbc: + batch_size: 25 # Match chunk size for optimal batch processing + order_inserts: true + order_updates: true + + flyway: + enabled: true + locations: classpath:db/migration + baseline-on-migrate: true + create-schemas: true + schemas: TED + default-schema: TED + +# Apache Camel Configuration +camel: + springboot: + main-run-controller: true + health: + enabled: true + # Weniger strenge Health-Checks für File-Consumer + consumers-enabled: false + +# Custom Application Properties +ted: + # Directory configuration for file processing + input: + # Base directory for watching incoming TED XML files + directory: ${TED_INPUT_DIR:/ted.europe/extracted} + # File pattern to match (recursive scanning) + pattern: "**/*.xml" + # Move processed files to this directory + processed-directory: ${TED_PROCESSED_DIR:.processed} + # Move failed files to this directory + error-directory: ${TED_ERROR_DIR:.error} + # Polling interval in milliseconds + poll-interval: 5000 + # Maximum messages per poll (reduced to prevent memory issues) + max-messages-per-poll: 10 + + # Schema validation configuration + schema: + # Enable/disable XSD validation + enabled: true + # Path to eForms SDK schemas (from Maven dependency or custom location) + path: classpath:schemas/maindoc/UBL-ContractNotice-2.3.xsd + + # Vectorization configuration + vectorization: + # Enable/disable async vectorization + enabled: false + # Use external HTTP API instead of subprocess + use-http-api: true + # Embedding service URL + api-url: http://172.20.240.18:8001 + # Model name for sentence-transformers + model-name: intfloat/multilingual-e5-large + # Vector dimensions (must match model output) + dimensions: 1024 + # Batch size for vectorization + batch-size: 16 + # Thread pool size for async processing + thread-pool-size: 4 + # Maximum text length for vectorization (characters) + max-text-length: 8192 + # HTTP connection timeout (milliseconds) + connect-timeout: 10000 + # HTTP socket/read timeout (milliseconds) + socket-timeout: 60000 + # Maximum retries on connection failure + max-retries: 5 + + # Search configuration + search: + # Default page size for search results + default-page-size: 20 + # Maximum page size + max-page-size: 100 + # Similarity threshold for vector search (0.0 - 1.0) + similarity-threshold: 0.7 + + # TED Daily Package Download configuration + download: + # Enable/disable automatic package download + enabled: true + # Base URL for TED Daily Packages + base-url: https://ted.europa.eu/packages/daily/ + # Download directory for tar.gz files + download-directory: /ted.europe/downloads + # Extract directory for XML files + extract-directory: /ted.europe/extracted + # Start year for downloads + start-year: 2023 + # Max consecutive 404 errors before stopping + max-consecutive-404: 4 + # Polling interval (milliseconds) - 2 minutes + poll-interval: 120000 + # Retry interval for tail NOT_FOUND packages - 6 hours + not-found-retry-interval: 21600000 + # Grace period after year end before a previous-year tail 404 is treated as final + previous-year-grace-period-days: 30 + # Keep retrying current-year tail 404 packages indefinitely + retry-current-year-not-found-indefinitely: true + # Download timeout (milliseconds) - 5 minutes + download-timeout: 300000 + # Max concurrent downloads + max-concurrent-downloads: 2 + # Delay between downloads (milliseconds) for rate limiting - 5 seconds + delay-between-downloads: 3000 + # Delete tar.gz after extraction + delete-after-extraction: true + # Prioritize current year first + prioritize-current-year: false + + # IMAP Mail configuration + mail: + # Enable/disable mail processing + enabled: true + # IMAP server hostname + host: mail.mymagenta.business + # IMAP server port (993 for IMAPS) + port: 993 + # Mail account username (email address) + username: archiv@procon.co.at + # Mail account password + password: ${MAIL_PASSWORD:worasigg} + # Use SSL/TLS connection + ssl: true + # Mail folder to read from + folder-name: INBOX + # Delete messages after processing + delete: false + # Mark messages as seen after processing (false = peek mode, don't mark as read) + seen: false + # Only process unseen messages + unseen: true + # Polling delay in milliseconds (1 minute) + delay: 60000 + # Max messages per poll + max-messages-per-poll: 10 + # Output directory for processed attachments + attachment-output-directory: D:/ted.europe/mail-attachments + # Enable/disable MIME file input processing + mime-input-enabled: true + # Input directory for MIME files (.eml) + mime-input-directory: D:/ted.europe/mime-input + # File pattern for MIME files (regex) + mime-input-pattern: .*\\.eml + # Polling interval for MIME input directory (milliseconds) + mime-input-poll-interval: 10000 + + # Solution Brief processing configuration + solution-brief: + # Enable/disable Solution Brief processing + enabled: true + # Input directory for Solution Brief PDF files + input-directory: C:/work/SolutionBrief + # Output directory for Excel result files (relative to input or absolute) + result-directory: ./result + # Number of top similar documents to include + top-k: 20 + # Minimum similarity threshold (0.0-1.0) + similarity-threshold: 0.5 + # Polling interval in milliseconds (30 seconds) + poll-interval: 30000 + # File pattern for PDF files (regex) + file-pattern: .*\\.pdf + # Process files only once (idempotent) + idempotent: true + # Idempotent repository file path + idempotent-repository: ./solution-brief-processed.dat + + # Data cleanup configuration + cleanup: + # Enable automatic cleanup of old documents + enabled: false + # Retention period in years (default: 10) + retention-years: 10 + # Cron expression for cleanup schedule (default: daily at 2 AM) + cron: "0 0 2 * * *" + +# Actuator endpoints +management: + endpoints: + web: + exposure: + include: health,info,metrics,camel + endpoint: + health: + show-details: when-authorized + +# OpenAPI documentation +springdoc: + api-docs: + path: /v3/api-docs + swagger-ui: + path: /swagger-ui.html + operations-sorter: method + +# Logging configuration +logging: + level: + at.procon.ted: INFO + at.procon.ted.camel.SolutionBriefRoute: INFO + org.apache.camel: INFO + org.hibernate.SQL: WARN + org.hibernate.type.descriptor.sql: WARN diff --git a/src/main/resources/banner.txt b/src/main/resources/banner.txt new file mode 100644 index 0000000..6cf2378 --- /dev/null +++ b/src/main/resources/banner.txt @@ -0,0 +1,12 @@ + + ______ _ _ _ __ _ _ + | ____| | | | |/ / | | | | + | |__ _ _ | | | ' / _ __ _____ _| | ___ __| | __ _ ___ + | __| | | || | | < | '_ \ / _ \ \ /\ / / |/ _ \/ _` |/ _` |/ _ \ + | | | |_| || | | . \| | | | (_) \ V V /| | __/ (_| | (_| | __/ + |_| \__,_||_|_|_|\_\_| |_|\___/ \_/\_/ |_|\___|\__,_|\__, |\___| + __/ | + |___/ + TED Procurement Processor :: ${spring-boot.version} + (c) procon.co.at + diff --git a/src/main/resources/db/migration/V1__initial_schema.sql b/src/main/resources/db/migration/V1__initial_schema.sql new file mode 100644 index 0000000..04cca68 --- /dev/null +++ b/src/main/resources/db/migration/V1__initial_schema.sql @@ -0,0 +1,428 @@ +-- TED Procurement Document Database Schema +-- Author: Martin.Schweitzer@procon.co.at and claude.ai +-- Description: PostgreSQL schema for storing EU eForms procurement notices with vector search support + +-- Create TED schema if it doesn't exist +CREATE SCHEMA IF NOT EXISTS TED; + +-- Set search path to use TED schema +SET search_path TO TED; + +-- Enable required PostgreSQL extensions (wenn Berechtigung vorhanden) +-- Falls Extensions nicht erstellt werden können, müssen diese vom DBA manuell erstellt werden +DO $$ +BEGIN + CREATE EXTENSION IF NOT EXISTS pgcrypto SCHEMA public; +EXCEPTION + WHEN insufficient_privilege THEN + RAISE NOTICE 'Skipping pgcrypto extension creation (insufficient privileges)'; + WHEN duplicate_object THEN + RAISE NOTICE 'Extension pgcrypto already exists'; +END +$$; + +DO $$ +BEGIN + CREATE EXTENSION IF NOT EXISTS vector SCHEMA public; +EXCEPTION + WHEN insufficient_privilege THEN + RAISE NOTICE 'Skipping vector extension creation (insufficient privileges)'; + WHEN duplicate_object THEN + RAISE NOTICE 'Extension vector already exists'; + WHEN undefined_file THEN + RAISE WARNING 'Extension vector not available - install pgvector on the database server'; +END +$$; + +DO $$ +BEGIN + CREATE EXTENSION IF NOT EXISTS pg_trgm SCHEMA public; +EXCEPTION + WHEN insufficient_privilege THEN + RAISE NOTICE 'Skipping pg_trgm extension creation (insufficient privileges)'; + WHEN duplicate_object THEN + RAISE NOTICE 'Extension pg_trgm already exists'; +END +$$; + +-- Enum types for notice classifications +CREATE TYPE notice_type AS ENUM ( + 'CONTRACT_NOTICE', -- cn-standard, cn-social, etc. + 'PRIOR_INFORMATION_NOTICE', -- pin-* + 'CONTRACT_AWARD_NOTICE', -- can-* + 'MODIFICATION_NOTICE', -- mod-* + 'OTHER' +); + +CREATE TYPE contract_nature AS ENUM ( + 'SUPPLIES', + 'SERVICES', + 'WORKS', + 'MIXED', + 'UNKNOWN' +); + +CREATE TYPE procedure_type AS ENUM ( + 'OPEN', + 'RESTRICTED', + 'COMPETITIVE_DIALOGUE', + 'INNOVATION_PARTNERSHIP', + 'NEGOTIATED_WITHOUT_PUBLICATION', + 'NEGOTIATED_WITH_PUBLICATION', + 'OTHER' +); + +CREATE TYPE vectorization_status AS ENUM ( + 'PENDING', + 'PROCESSING', + 'COMPLETED', + 'FAILED', + 'SKIPPED' +); + +-- Main procurement document table +CREATE TABLE procurement_document ( + -- Primary key using UUID + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + + -- Document hash for idempotent processing (SHA-256 of XML content) + document_hash VARCHAR(64) NOT NULL UNIQUE, + + -- TED/eForms identifiers + notice_id VARCHAR(100), -- e.g., "9f87fd31-2c94-45cb-92ca-a3f876252149" + publication_id VARCHAR(50), -- e.g., "00786665-2025" + ojs_id VARCHAR(20), -- e.g., "229/2025" (Official Journal Supplement) + contract_folder_id VARCHAR(100), -- Contract folder grouping + + -- Document metadata + notice_type notice_type NOT NULL DEFAULT 'OTHER', + notice_subtype_code VARCHAR(10), -- e.g., "16" for Contract Notice Standard + sdk_version VARCHAR(20), -- e.g., "eforms-sdk-1.13" + ubl_version VARCHAR(10), -- e.g., "2.3" + language_code VARCHAR(10), -- Primary language (e.g., "POL") + + -- Timestamps from document + issue_date DATE, + issue_time TIME, + publication_date DATE, + submission_deadline TIMESTAMP WITH TIME ZONE, + + -- Contracting authority information + buyer_name TEXT, + buyer_country_code VARCHAR(10), -- ISO 3166-1 alpha-3 (e.g., "POL") + buyer_city VARCHAR(255), + buyer_postal_code VARCHAR(20), + buyer_nuts_code VARCHAR(10), -- NUTS region code (e.g., "PL415") + buyer_activity_type VARCHAR(50), -- e.g., "health", "defence" + buyer_legal_type VARCHAR(50), -- e.g., "body-pl" + + -- Procurement project information + project_title TEXT, + project_description TEXT, + internal_reference VARCHAR(100), -- Buyer's internal reference + contract_nature contract_nature NOT NULL DEFAULT 'UNKNOWN', + procedure_type procedure_type DEFAULT 'OTHER', + + -- Classification + cpv_codes VARCHAR(100)[], -- Common Procurement Vocabulary codes + nuts_codes VARCHAR(20)[], -- All NUTS codes for delivery locations + + -- Financial information (if available) + estimated_value DECIMAL(20, 2), + estimated_value_currency VARCHAR(3), + + -- Lot information + total_lots INTEGER DEFAULT 0, + max_lots_awarded INTEGER, + max_lots_submitted INTEGER, + + -- Legal basis + regulatory_domain VARCHAR(50), -- e.g., "32014L0024" (EU directive reference) + eu_funded BOOLEAN DEFAULT FALSE, + + -- Textual representation for vectorization + -- Contains extracted and normalized text content for semantic search + text_content TEXT, + + -- Original XML document stored in native PostgreSQL XML type + xml_document XML NOT NULL, + + -- Vector embedding for semantic search (1024 dimensions for multilingual-e5-large) + content_vector vector(1024), + + -- Vectorization tracking + vectorization_status vectorization_status DEFAULT 'PENDING', + vectorization_error TEXT, + vectorized_at TIMESTAMP WITH TIME ZONE, + + -- Processing metadata + source_filename VARCHAR(500), + source_path TEXT, + file_size_bytes BIGINT, + + -- Audit fields + created_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP, + processing_duration_ms INTEGER +); + +-- Table for storing lot details (denormalized for search performance) +CREATE TABLE procurement_lot ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + document_id UUID NOT NULL REFERENCES procurement_document(id) ON DELETE CASCADE, + + lot_id VARCHAR(50) NOT NULL, -- e.g., "LOT-0001" + internal_id VARCHAR(100), -- Buyer's internal lot reference + + title TEXT, + description TEXT, + + cpv_codes VARCHAR(100)[], + nuts_codes VARCHAR(20)[], + + estimated_value DECIMAL(20, 2), + estimated_value_currency VARCHAR(3), + + duration_value INTEGER, + duration_unit VARCHAR(20), -- e.g., "MONTH", "DAY" + + submission_deadline TIMESTAMP WITH TIME ZONE, + + eu_funded BOOLEAN DEFAULT FALSE, + + created_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP, + + UNIQUE(document_id, lot_id) +); + +-- Table for organizations mentioned in notices +CREATE TABLE organization ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + document_id UUID NOT NULL REFERENCES procurement_document(id) ON DELETE CASCADE, + + org_reference VARCHAR(100), -- Internal reference (e.g., "ORG-0001") + role VARCHAR(100), -- e.g., "buyer", "review-body", "ted-esen" + + name TEXT, -- Full organization name + company_id TEXT, -- Tax/registration ID (can be very long) + + country_code VARCHAR(10), + city TEXT, -- City name (can be extremely long) + postal_code TEXT, -- Address/postal code (can contain full addresses) + street_name TEXT, -- Street address + nuts_code VARCHAR(10), + + website_uri TEXT, + email VARCHAR(255), + phone VARCHAR(100), -- International phone numbers + + created_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP, + + UNIQUE(document_id, org_reference) +); + +-- Processing log for tracking and debugging +CREATE TABLE processing_log ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + document_id UUID REFERENCES procurement_document(id) ON DELETE SET NULL, + document_hash VARCHAR(64), + + event_type VARCHAR(50) NOT NULL, -- e.g., "RECEIVED", "VALIDATED", "PARSED", "STORED", "VECTORIZED", "ERROR" + event_status VARCHAR(20) NOT NULL, -- e.g., "SUCCESS", "FAILURE", "SKIPPED" + + message TEXT, + error_details TEXT, + + source_filename VARCHAR(500), + + duration_ms INTEGER, + + created_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP +); + +-- TED Daily Package Tracking Table +CREATE TABLE ted_daily_package ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + + -- Package identifier (YYYYSSSSS format, e.g., 202400001) + package_identifier VARCHAR(20) NOT NULL UNIQUE, + + -- Year and serial number + year INTEGER NOT NULL, + serial_number INTEGER NOT NULL, + + -- Download information + download_url VARCHAR(500) NOT NULL, + file_hash VARCHAR(64), -- SHA-256 hash for idempotency + + -- Processing statistics + xml_file_count INTEGER, + processed_count INTEGER DEFAULT 0, + failed_count INTEGER DEFAULT 0, + + -- Status + download_status VARCHAR(30) NOT NULL DEFAULT 'PENDING', -- PENDING, DOWNLOADING, DOWNLOADED, PROCESSING, COMPLETED, FAILED, NOT_FOUND + error_message TEXT, + + -- Timestamps + downloaded_at TIMESTAMP WITH TIME ZONE, + processed_at TIMESTAMP WITH TIME ZONE, + + -- Performance metrics + download_duration_ms BIGINT, + processing_duration_ms BIGINT, + + -- Audit fields + created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP, + + -- Unique constraint on year + serial + UNIQUE(year, serial_number) +); + +-- Indexes for efficient querying + +-- Hash lookup for idempotent processing (most critical) +CREATE INDEX idx_doc_hash ON procurement_document(document_hash); + +-- Publication/notice ID lookups +CREATE INDEX idx_doc_publication_id ON procurement_document(publication_id); +CREATE INDEX idx_doc_notice_id ON procurement_document(notice_id); + +-- Date range queries +CREATE INDEX idx_doc_publication_date ON procurement_document(publication_date); +CREATE INDEX idx_doc_issue_date ON procurement_document(issue_date); +CREATE INDEX idx_doc_submission_deadline ON procurement_document(submission_deadline); + +-- Geographic searches +CREATE INDEX idx_doc_buyer_country ON procurement_document(buyer_country_code); +CREATE INDEX idx_doc_buyer_nuts ON procurement_document(buyer_nuts_code); +CREATE INDEX idx_doc_nuts_codes ON procurement_document USING GIN(nuts_codes); + +-- Classification searches +CREATE INDEX idx_doc_notice_type ON procurement_document(notice_type); +CREATE INDEX idx_doc_contract_nature ON procurement_document(contract_nature); +CREATE INDEX idx_doc_procedure_type ON procurement_document(procedure_type); +CREATE INDEX idx_doc_cpv_codes ON procurement_document USING GIN(cpv_codes); + +-- Full-text search on textual content +CREATE INDEX idx_doc_text_content_trgm ON procurement_document USING GIN(text_content gin_trgm_ops); + +-- Vector similarity search using IVFFlat index (efficient for approximate nearest neighbor) +-- Lists parameter: sqrt(number_of_vectors) for optimal performance +CREATE INDEX idx_doc_vector ON procurement_document USING ivfflat (content_vector vector_cosine_ops) WITH (lists = 100); + +-- Vectorization status for async processing +CREATE INDEX idx_doc_vectorization_status ON procurement_document(vectorization_status) WHERE vectorization_status IN ('PENDING', 'PROCESSING'); + +-- Lot indexes +CREATE INDEX idx_lot_document ON procurement_lot(document_id); +CREATE INDEX idx_lot_cpv_codes ON procurement_lot USING GIN(cpv_codes); + +-- Organization indexes +CREATE INDEX idx_org_document ON organization(document_id); +CREATE INDEX idx_org_country ON organization(country_code); + +-- Processing log indexes +CREATE INDEX idx_log_document ON processing_log(document_id); +CREATE INDEX idx_log_created ON processing_log(created_at); +CREATE INDEX idx_log_event_type ON processing_log(event_type); + +-- TED daily package indexes +CREATE INDEX idx_package_identifier ON ted_daily_package(package_identifier); +CREATE INDEX idx_package_year_serial ON ted_daily_package(year, serial_number); +CREATE INDEX idx_package_status ON ted_daily_package(download_status); +CREATE INDEX idx_package_downloaded_at ON ted_daily_package(downloaded_at); + +-- Trigger to update updated_at timestamp +CREATE OR REPLACE FUNCTION update_updated_at_column() +RETURNS TRIGGER AS $$ +BEGIN + NEW.updated_at = CURRENT_TIMESTAMP; + RETURN NEW; +END; +$$ language 'plpgsql'; + +CREATE TRIGGER update_procurement_document_updated_at + BEFORE UPDATE ON procurement_document + FOR EACH ROW + EXECUTE FUNCTION update_updated_at_column(); + +CREATE TRIGGER update_ted_daily_package_updated_at + BEFORE UPDATE ON ted_daily_package + FOR EACH ROW + EXECUTE FUNCTION update_updated_at_column(); + +-- Helper function for semantic search with filtering +CREATE OR REPLACE FUNCTION search_documents_semantic( + query_vector vector(1024), + similarity_threshold FLOAT DEFAULT 0.7, + country_filter VARCHAR(10) DEFAULT NULL, + notice_type_filter notice_type DEFAULT NULL, + contract_nature_filter contract_nature DEFAULT NULL, + cpv_prefix VARCHAR(20) DEFAULT NULL, + date_from DATE DEFAULT NULL, + date_to DATE DEFAULT NULL, + result_limit INTEGER DEFAULT 20 +) +RETURNS TABLE ( + id UUID, + publication_id VARCHAR(50), + project_title TEXT, + buyer_name TEXT, + buyer_country_code VARCHAR(10), + publication_date DATE, + similarity FLOAT +) AS $$ +BEGIN + RETURN QUERY + SELECT + pd.id, + pd.publication_id, + pd.project_title, + pd.buyer_name, + pd.buyer_country_code, + pd.publication_date, + 1 - (pd.content_vector <=> query_vector) AS similarity + FROM procurement_document pd + WHERE + pd.content_vector IS NOT NULL + AND (1 - (pd.content_vector <=> query_vector)) >= similarity_threshold + AND (country_filter IS NULL OR pd.buyer_country_code = country_filter) + AND (notice_type_filter IS NULL OR pd.notice_type = notice_type_filter) + AND (contract_nature_filter IS NULL OR pd.contract_nature = contract_nature_filter) + AND (cpv_prefix IS NULL OR EXISTS ( + SELECT 1 FROM unnest(pd.cpv_codes) code WHERE code LIKE cpv_prefix || '%' + )) + AND (date_from IS NULL OR pd.publication_date >= date_from) + AND (date_to IS NULL OR pd.publication_date <= date_to) + ORDER BY similarity DESC + LIMIT result_limit; +END; +$$ LANGUAGE plpgsql; + +-- Statistics view for monitoring +CREATE VIEW document_statistics AS +SELECT + COUNT(*) AS total_documents, + COUNT(*) FILTER (WHERE vectorization_status = 'COMPLETED') AS vectorized_documents, + COUNT(*) FILTER (WHERE vectorization_status = 'PENDING') AS pending_vectorization, + COUNT(*) FILTER (WHERE vectorization_status = 'FAILED') AS failed_vectorization, + COUNT(DISTINCT buyer_country_code) AS unique_countries, + COUNT(DISTINCT notice_type) AS notice_types, + MIN(publication_date) AS earliest_publication, + MAX(publication_date) AS latest_publication, + AVG(total_lots) AS avg_lots_per_notice, + SUM(total_lots) AS total_lots +FROM procurement_document; + +COMMENT ON TABLE procurement_document IS 'Main table storing EU eForms procurement notices from TED'; +COMMENT ON COLUMN procurement_document.document_hash IS 'SHA-256 hash of XML content for idempotent processing'; +COMMENT ON COLUMN procurement_document.content_vector IS '1024-dimensional vector from multilingual-e5-large model'; +COMMENT ON COLUMN procurement_document.text_content IS 'Normalized text for vectorization and full-text search'; + +COMMENT ON TABLE ted_daily_package IS 'Tracking table for TED daily package downloads'; +COMMENT ON COLUMN ted_daily_package.package_identifier IS 'Unique package identifier in YYYYSSSSS format'; +COMMENT ON COLUMN ted_daily_package.file_hash IS 'SHA-256 hash for idempotency checking'; +COMMENT ON COLUMN ted_daily_package.download_status IS 'Current status: PENDING, DOWNLOADING, DOWNLOADED, PROCESSING, COMPLETED, FAILED, NOT_FOUND'; +COMMENT ON COLUMN organization.postal_code IS 'Postal code or ZIP code. Extended to 255 chars to handle multi-line addresses from TED data.'; diff --git a/src/main/resources/db/migration/V2__extend_organization_varchar_fields.sql b/src/main/resources/db/migration/V2__extend_organization_varchar_fields.sql new file mode 100644 index 0000000..eb49fbb --- /dev/null +++ b/src/main/resources/db/migration/V2__extend_organization_varchar_fields.sql @@ -0,0 +1,28 @@ +-- Extend VARCHAR fields in organization table to handle longer values from TED data +-- Author: Martin.Schweitzer@procon.co.at and claude.ai + +-- Extend postal_code (was VARCHAR(50), sometimes contains full addresses) +ALTER TABLE ted.organization ALTER COLUMN postal_code TYPE TEXT; + +-- Extend street_name (was VARCHAR(50), sometimes very long) +ALTER TABLE ted.organization ALTER COLUMN street_name TYPE TEXT; + +-- Extend city (was VARCHAR(100), can be extremely long - some have >255 chars) +ALTER TABLE ted.organization ALTER COLUMN city TYPE TEXT; + +-- Extend phone (was VARCHAR(50), international numbers can be longer) +ALTER TABLE ted.organization ALTER COLUMN phone TYPE VARCHAR(100); + +-- Extend org_reference (was VARCHAR(50), sometimes longer internal references) +ALTER TABLE ted.organization ALTER COLUMN org_reference TYPE VARCHAR(100); + +-- Extend role (was VARCHAR(50), enum-like field but allow more space) +ALTER TABLE ted.organization ALTER COLUMN role TYPE VARCHAR(100); + +-- Extend company_id (was VARCHAR(100), can contain very long registration info) +-- Example: 'KRS pod numerem: 0000070678, REGON: 016134981, NIP: 9521822413' (67 chars) +ALTER TABLE ted.organization ALTER COLUMN company_id TYPE TEXT; + +-- Extend name (was TEXT already in V1, but ensure it's TEXT) +-- Contains full organization names which can be very long +ALTER TABLE ted.organization ALTER COLUMN name TYPE TEXT; diff --git a/src/main/resources/db/migration/V3__add_processed_attachment_table.sql b/src/main/resources/db/migration/V3__add_processed_attachment_table.sql new file mode 100644 index 0000000..93de7c4 --- /dev/null +++ b/src/main/resources/db/migration/V3__add_processed_attachment_table.sql @@ -0,0 +1,72 @@ +-- Migration: V3__add_processed_attachment_table.sql +-- Author: Martin.Schweitzer@procon.co.at and claude.ai +-- Description: Add table for tracking processed mail attachments with idempotency support + +-- Create processed_attachment table for tracking mail attachments +CREATE TABLE IF NOT EXISTS ted.processed_attachment ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + + -- Content hash for idempotent processing (SHA-256, 64 hex chars) + content_hash VARCHAR(64) NOT NULL UNIQUE, + + -- File metadata + original_filename VARCHAR(500) NOT NULL, + file_type VARCHAR(50), + content_type VARCHAR(255), + file_size BIGINT, + + -- Processing status + processing_status VARCHAR(20) NOT NULL DEFAULT 'PENDING', + + -- Extracted content + extracted_text TEXT, + + -- Storage path + saved_path VARCHAR(1000), + + -- Source email metadata + mail_subject VARCHAR(500), + mail_from VARCHAR(500), + + -- Parent reference (for files extracted from ZIP) + parent_hash VARCHAR(64), + + -- Error handling + error_message TEXT, + + -- Child count (for ZIP files) + child_count INTEGER, + + -- Timestamps + received_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + processed_at TIMESTAMP, + + -- Constraints + CONSTRAINT chk_processing_status CHECK (processing_status IN ('PENDING', 'PROCESSING', 'COMPLETED', 'FAILED', 'DUPLICATE')) +); + +-- Index on content_hash for fast idempotency lookups +CREATE UNIQUE INDEX IF NOT EXISTS idx_processed_attachment_hash + ON ted.processed_attachment(content_hash); + +-- Index on processing_status for finding pending/failed items +CREATE INDEX IF NOT EXISTS idx_processed_attachment_status + ON ted.processed_attachment(processing_status); + +-- Index on file_type for filtering by type +CREATE INDEX IF NOT EXISTS idx_processed_attachment_type + ON ted.processed_attachment(file_type); + +-- Index on parent_hash for finding children of ZIP files +CREATE INDEX IF NOT EXISTS idx_processed_attachment_parent + ON ted.processed_attachment(parent_hash); + +-- Index on received_at for chronological queries +CREATE INDEX IF NOT EXISTS idx_processed_attachment_received + ON ted.processed_attachment(received_at DESC); + +-- Comment on table +COMMENT ON TABLE ted.processed_attachment IS 'Tracks processed mail attachments with idempotency via content hash'; +COMMENT ON COLUMN ted.processed_attachment.content_hash IS 'SHA-256 hash of file content for duplicate detection'; +COMMENT ON COLUMN ted.processed_attachment.parent_hash IS 'Reference to parent attachment (for files extracted from ZIP)'; +COMMENT ON COLUMN ted.processed_attachment.extracted_text IS 'Text content extracted from PDF and other document types'; diff --git a/src/test/java/at/procon/ted/service/XmlParserServiceTest.java b/src/test/java/at/procon/ted/service/XmlParserServiceTest.java new file mode 100644 index 0000000..b088141 --- /dev/null +++ b/src/test/java/at/procon/ted/service/XmlParserServiceTest.java @@ -0,0 +1,199 @@ +package at.procon.ted.service; + +import at.procon.ted.model.entity.*; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.DisplayName; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; + +import static org.junit.jupiter.api.Assertions.*; + +/** + * Unit tests for XmlParserService. + * + * @author Martin.Schweitzer@procon.co.at and claude.ai + */ +class XmlParserServiceTest { + + private XmlParserService parserService; + + @BeforeEach + void setUp() { + parserService = new XmlParserService(); + } + + @Test + @DisplayName("Should parse contract notice XML and extract basic metadata") + void testParseContractNotice() { + // Sample minimal eForms XML + String xml = """ + + + 2.3 + eforms-sdk-1.13 + test-notice-123 + 2025-01-15 + cn-standard + ENG + + Test Procurement Project + This is a test description + supplies + + + open + + + """; + + ProcurementDocument document = parserService.parseDocument(xml); + + assertNotNull(document); + assertEquals("2.3", document.getUblVersion()); + assertEquals("eforms-sdk-1.13", document.getSdkVersion()); + assertEquals("test-notice-123", document.getNoticeId()); + assertEquals(NoticeType.CONTRACT_NOTICE, document.getNoticeType()); + assertEquals("ENG", document.getLanguageCode()); + assertEquals("Test Procurement Project", document.getProjectTitle()); + assertEquals("This is a test description", document.getProjectDescription()); + assertEquals(ContractNature.SUPPLIES, document.getContractNature()); + assertEquals(ProcedureType.OPEN, document.getProcedureType()); + } + + @Test + @DisplayName("Should extract CPV codes from procurement project") + void testParseCpvCodes() { + String xml = """ + + + 2.3 + + Medical Supplies + supplies + + 33140000 + + + 33141000 + + + + """; + + ProcurementDocument document = parserService.parseDocument(xml); + + assertNotNull(document.getCpvCodes()); + assertEquals(2, document.getCpvCodes().length); + assertEquals("33140000", document.getCpvCodes()[0]); + assertEquals("33141000", document.getCpvCodes()[1]); + } + + @Test + @DisplayName("Should generate text content for vectorization") + void testTextContentGeneration() { + String xml = """ + + + 2.3 + + Hospital Equipment Procurement + Procurement of medical imaging equipment + supplies + + + """; + + ProcurementDocument document = parserService.parseDocument(xml); + + assertNotNull(document.getTextContent()); + assertTrue(document.getTextContent().contains("Hospital Equipment Procurement")); + assertTrue(document.getTextContent().contains("medical imaging equipment")); + } + + @Test + @DisplayName("Should handle missing optional fields gracefully") + void testMissingOptionalFields() { + String xml = """ + + + 2.3 + + """; + + ProcurementDocument document = parserService.parseDocument(xml); + + assertNotNull(document); + assertEquals("2.3", document.getUblVersion()); + assertNull(document.getProjectTitle()); + assertNull(document.getPublicationId()); + assertEquals(NoticeType.OTHER, document.getNoticeType()); + } + + @Test + @DisplayName("Should throw exception for invalid XML") + void testInvalidXml() { + String invalidXml = "This is not XML"; + + assertThrows(XmlParserService.XmlParsingException.class, () -> { + parserService.parseDocument(invalidXml); + }); + } + + @Test + @DisplayName("Should store original XML document") + void testXmlDocumentStorage() { + String xml = """ + + + 2.3 + + """; + + ProcurementDocument document = parserService.parseDocument(xml); + + assertNotNull(document.getXmlDocument()); + assertEquals(xml, document.getXmlDocument()); + } + + @Test + @DisplayName("Should map contract nature correctly") + void testContractNatureMapping() { + String[] natures = {"supplies", "services", "works", "mixed", "unknown"}; + ContractNature[] expected = { + ContractNature.SUPPLIES, + ContractNature.SERVICES, + ContractNature.WORKS, + ContractNature.MIXED, + ContractNature.UNKNOWN + }; + + for (int i = 0; i < natures.length; i++) { + String xml = String.format(""" + + + 2.3 + + %s + + + """, natures[i]); + + ProcurementDocument document = parserService.parseDocument(xml); + assertEquals(expected[i], document.getContractNature(), + "Failed for nature: " + natures[i]); + } + } +} diff --git a/start.bat b/start.bat new file mode 100644 index 0000000..03fb1c7 --- /dev/null +++ b/start.bat @@ -0,0 +1,9 @@ +@echo off +REM TED Procurement Processor Startup Script +REM Increases Java heap size to 8GB + +echo Starting TED Procurement Processor with 8GB heap... + +java -Xms4g -Xmx8g -XX:+UseG1GC -XX:MaxGCPauseMillis=200 -jar target\ted-procurement-processor-1.0.0-SNAPSHOT.jar + +pause diff --git a/start.sh b/start.sh new file mode 100644 index 0000000..d4e7c6d --- /dev/null +++ b/start.sh @@ -0,0 +1,7 @@ +#!/bin/bash +# TED Procurement Processor Startup Script +# Increases Java heap size to 4GB + +echo "Starting TED Procurement Processor with 4GB heap..." + +java -Xms2g -Xmx4g -jar target/ted-procurement-processor-1.0.0-SNAPSHOT.jar diff --git a/ted-procurement-processor.zip b/ted-procurement-processor.zip new file mode 100644 index 0000000000000000000000000000000000000000..ae3c9d43d52c333efac06a17d30e020e7154739e GIT binary patch literal 160870 zcmb5VQ;=p+w=7z=ZFSkUZQHi(E?ZyOR+rUf+qP}nb$TEDJ8s;F8+W~}6*=F=$T2f> zEJYbmFf^XMY{ft%V_Va3CNdI3OVGe_us~q-3=W6oo`&ge1faL=`mTWfg=( z>FrFb)hFB+7?HL*ZWDTgs+5&7*N7j$SI#m=OXDKrR=k=_dqop4_Q`^ z6kbup?|(sC1%j_NJ;z+<%yr>hAb@p45**yYywmE|VF#t@M{$}Vx6VqZGLciVN@PT4 z9Bj6MUO1p8M}@mwaT2I7>Qco*4_@2=AX3j*jWR18yW^`2?G0I_M@7LfDKoN(R?3B( zs&pPTtQus&l@s$Vyzj~p`0p;%KtWi zZ8P&ds-d9sdTWWtNzvbe{&>l|>V^Z;s|>kOzl^aL1T}l!bPIWn8OwDv7Wn2A99mji zvcyVvm^Cn<*0{-zah*uvPb)JWHOkJHmJ}+n&8D59l#nN9*TS^&V>a#80}BZS&gEZq zuY)k%K#0QH$(wc-b5{=_9f2dxJ*%s#7Ge-{?t)~hC$Fo=n_+a#%Pw22>>5~&VSsOV zQL{nQ|6x1Q=AvF6StOES3=_&c!_ANj@YE_TujGmvMM50l64#2Ei_7m${zEz>iiR4f zlw`|1kp*#%POfQ~FtF{mew{UMRN}Ey>dX|oATvU&v(>YM(j-KCyn?a+?FR`Y$+Eao ztnEfn&U%{*tbN6Z;kBIL9uzV)*ncYxcXFtGZc@O45}w*)>H__TitHQywy!Aj051!K zT4}?oo%U^pnQE;1mLS7clVG41Ha#R&AOu>&(c2XU61pR3p;EiMf#EKwETch24HV*n zaBtuFE9WqIEr$DWFoX2&CX6sN=v_>-Ooy4(tTtAQ|{&UE~z5q zDW1xXV>02=>iNphu#>#p+gk?BaBt~z z_s)~{*2(R|gvPD6L&ufA%gx%$rOEGEo`>)OX7A;T#|?hDTD()d6DE|0&DXe5T*qzO zzkWBwu0DOQogC-h%Q_ntvsgxLFB1aIPN2uy3~C3r2^h;vMSZ=eGkXnK72yDEmGCC>;4|^;$1%u^l~vlo0cMe- zD)C6go!nfCS=g+asuPXnv=NDsvS%``S$LHK#!1#5!8F^o0Ny6qgViJRT?5ARgTpbv z8}O_ug@J{uyhr6SuaI&obLHU3>|W3y#)htuzwl-SHZ_Gre|q&M^tp^^J-)wgAH0)M zgjhyAjvjjn%2ubwuhmo-nK2b9m3^G8Y_=PL8OB8ItFi9MQ?Vh8htctB{-No$UIz$% zuqg|>9bc4-rVJCBwN^zW1X|R1Lso0A!(sI@E>FbT&lg0yd`aAgQQwHN?6}FS+-EIrIsh}!xJ0X*x7T4Z9isCghN zxaFIztCu%v!uq>>=`wG|I18j>U;=;WSBFULONlMrK8mmlHT1^QlHRAqpZVNZSZOZX>3EB(T~bw~G=q8#7l z3xBQ@S)a?Q{o-a%cP`c{-Jo?=*X+6W`pD1T`Z|Hd$DkPC(dmNw>nANR=^g&pq^@u2 z1)&R$_puD_?9sSx>*+>CA+l0ISK|r+dNAM4;yBoB|L9sL zH>kfFFYV5`N0&jZ@_T%w;c^-4i+3s?A8mHTMm(yA!a8Q-&wG%dS^EmL)*p}z; z>*)ZP?R5L;_@P#o>RE7^@P6EWuHp9djq5X9)U^B^dBkEf${CXP_$Xbg@l`Q9z=x@_ z6LWaMs<~Mo{%uv?-%eOlf04;9H}1$oH}JG6gaqN1EF|y4H88yXj=reg=>Ogh!J_1&un-J%e2T9xOX&(wkUd=HqdX^!Z^gR*>(!s>o5%9h% zqp_@d8Db5}P0ysQPQ)2Ok5!;J45Dk~f-A1d0fCSY7QGSZC%nJI*~!ddG3mVXQasM& zX138?rIEAaJJ9eGdDYeBC?`i!Yb$_mntZU_*iS_jfa}1=ctU#Y)kNLmU7UjNskiI- zyJaJF`?>fg1P3WrP2TZ6AbuMLXwR%BWCC{NQ*vRtJ~}=3&|CXuRs4b?@3QL0c4gJ9 zFLk3WKFG_)D*-tQC}NulKtqNvS?vPAVCBxG|_2v+2ux@T~M#D3c zxHUcI_7HcJ#(ThXrV5M>3OpL;y8dgyCCQND1{-tNc{Wjd(#>cJirUJy zi)8u;!N_f%K~o57FbI%i3)RP~EL*znwevip+9NZQP$AF_vs5cc9i6s%%H=>M&=*q^gJ; zm0tm)FC~%dJ(N7PPU1;9Y_L5~kl*IXdmHSxXyxUHyc7VTaYItlqU#*aN1VzU7N$Q! zmz0RBPaL)h=<{)l^6e0M9)!RlTJ?K#2bVJI5}7bja1jN#O2bZaF&VpB4(30N85f+| zwWdF(UzL1_o<2;^XR>HdW&n@7{&W=@sq7kBS*T97L+^Juwy!GeT0vmMn8Q zuFC+;D#tH0Dh4H;6^P{n8>1;(CzyKVhwfhb2T4K|DLiuLT$r)=?SALwe0rEc;xy`R z30M2O^!Bqk@7|X|K1T?Q4!@ZK(@)s~?-s%R#_|3Bs!jS*f8NM5Ykz_LKfLw7cnHS& zF~kA{2x$8s7t#Lz;30ZrBOAA5ndl8hw9uE(Z^#2Nk4UnM9C>b8Y?#on)afQyjpEA6 z!Ci^EoUy|l4X;(JsA!Jk!iOB6$5xM*TD6Q$!wqz~F4?o}V(8`B*Ft@U1;Uv;bMCI6 z+vlkqJ0lg<<`J8SP# zJNR^c)n6ClJnA2%R422lYl@Nm>*`K!@&3(rxT}9PjkfZ>|0fT@^vJ>rOyIvv<|FjW z5Cmu*g^&_eJ%=z}%-g6lQ83v}rjmLTQ1LZmrj>Qa=3WdyrA)(0P#zMXa|qI*W<$Xa zrxOK4u%01qP+me9u?u~*UPN)T^C(wKNPt7gpgE5OKZHr42|1P7+J6bUVeXMW8FkRX z*7?b@2Jscv&oki&AOO=IWCxCtxwJWhLDY3wQ>@KEWwldfJrR3ZR9<}?ZXC~w32|{F zn~-`&$}*0?Lqnt&u}ju~Y763JzXMpUnylaVkJy7F>2MXQ_Ym5k3^^erXq~7z5vbt~ zt&`;w1vED9UDf1e=$X`q+PmBigi+63=~Fm1VXS*8jRhGGG@&@JHJyS#^u%U=x_X{* zKQD}7kR77Qv#n>SB=HrcnW?Fjy@i2`nX|iDQO*5o#apARTcLY7Zrw>*wA%5ov1#Lh)gv8vu9cUR3gCe_szzQRO*1Lr+y0WV`ngi z`PSqZ&3bS7oL76hQQK=0o4^v|Pesf;bz#sf^Dir*jTkZ3mresty@oc4U5P(-6q$}C znyL=GH+7xUSry3@Khjy*`sgt<-5Rxd_yiObkhKU#j@f+3=%-LgldD1fj{?PV<%4*i zxp;)Rovogc+?^9ES2$o@W@J>FZD=$Sn+IPG=ra>5Czvj6Kvl$*QLy2b42eHzduJCc zGMXB(GOq{Q+x)ADnAdfa=P=|OR_-wSV>PT$Ys3tTQOq{PZL_WE;7QTErt4jWBzkfU z&SNxFN*6#w$82brfe_V-pU8_!MpfiKSdL`dCu4YLv;b#J+P57QRD3rw_f6E8Zz%mUl&rTOEVR3*dgxKs)Xc}N z;q9#z1G!gn#sO^wI&6Gl-`z~Z`k(rF32+Gz;^AHC)2&Uv^B6W{fO#55^zJqKjY38a zB8ng`NE+}q6A7h>3R0RU)N@*?iOt2B#+1p?FGyS(5LK3=wbV9vPorX$uT74rsiB9xanW(&))CT;iY*A! zV3bNkT_6WK>K)!3!mXLG>0Ki}eqy`41}CRz{m|>Z>6-6wb#ZEUwEHiGl3iNm&bz(E*#v0gc>fgPXUCs4Tuh6j4uF+i*2w>7~ zk#SYBLXAvegY*j%C;8P1CSr`eZafI>4TM~FB4GJR7nGrLT`40v8{;i6rgbS?mF#>e zqM6;BKBtiF*T8%bX~STaoXozxHftu&4L5^r9)%WqK-9gn5;*KnYob^QM;Vuw1I;@f zZN4+2r0Q+HfBQTqo3x>qh=@Q}4_d3Ge>Z;Y~W=eKI zBQC44V96eu{g>6;H}>Khy!q=i(`Nb_@n*71FT9vg4H`z6RGrP}k?C&-2fQ;`5(i@- zP7-%^LbzHa^77~GWAIVb6xEq3Xcvpk*=&}-jJPrKPf(I|hE29MzACN5K~OxoZVtK; zhA2UoiNxA(IBpxTQr%c?e8l;zi`*e}I?ETnOEAPM#qCr%x*JzEeGM}Yi5PKyLPvd^ zE!WEFbvc3jRe_et9W=YnMI7($P8|@&%e4k5tQ>F+d2y;BMi+g*Q7%#i+x+NZARqD=YWJ8I+DT7+$F7fUAiUMMs;Q9x*ve#X;-;O~rPU;>!3UJtlg$8Hbl|ng`ojk~ST9pjqpuNH6!4DF%8|R z4fg~AT2Ym;hV+5SRT5r{QA5-5@%=Tvlsjk8e*5WKe*`4)xBT%zoNgO$8JerqZ@0TQ z6Ycwv?Ts7y4jQ0`wBD(ILwRgtN` zy~~3105}Mp7GV$mLALKnASJfPQeK_3IxfpWB@)(chC7B7@b`2%+ERSmam_rDt31PeC7Oep<3U+{(P zfdh!tBP6u>IW(EGge_nc&y&DDQTG@El;z{vuyQ;CAoya*>W7gl#`=)eQIg}w6893R zU`OQr=54V4*cR`&hB_Xv17`tR#*$;GK;WvB19dfVfB2wvSePPP)KK(9i?bn*dF5)B zdpPp{?yxFztM!T-sQc;$k?@eqv}ZW6%PV}39v+lCLEBc+3$Mfc`YL9Us3r>_JR4-U zRE5S#>My;>ERfNN>P_`CY^%kT#>`{{F`@2S?wryjfx zF?wxR)Ye)n-nxHFjMROF8Y99}X*eSfDhs_9Ih#a!}drEd?Rs#50U3PNQUV1^gSVka8 zK=pvHswY!#H7a!5yPS7@&B>?p`K8Jble`WEEzFMXxzGoK^GAD$4%(jq znwAw&SV4DTlH?ePu+8K*E}RIfJzH#8;o&VCb%5XDqY80Z!zOE)hFAOz=cszQgbjCY z)ylZpQ07>YH(64lMrBBMEs|fe^f*Q5{O3EM63U=yfKSb`qvt{DBJ#@{4 z%}m@}&FIYR-R$VhtvvrpR#!U}J%@EhG{2YH)=D(11!-;V1Q_MwyFCyTz4`R24SjOd zY~!WDVR_2Sm7B!ddbz{1WclM-!%c0qS1I)+BK*U4pbU}4tk~KDR@k_esFHvPK84>`9d&!^_iZSQn9@C{ z{im0{IG0}M%arF#&nUwKL^w6wBO4o@-^-DUhoA}|j7_Qp>58ul@HvF$w%K+9G3!x6T-yWU{Sh1i*dUw{LJda^rTD3bStcJQq zNvdnJ)Syj%@lWt?~@dRJfV9+ zCSHfV>wJ&3*}xS5;|w^g2_<{1`Wd>3GwcU&5y^tT0{|GC7#HU>1JF7s0yzFj$vP z_&ULts?)frR-%bMvF~aGs7e8DegjwI?Ixob;faqGTL&gOfQ0$=ar74=3d2c&z5O@+ znC9uP!>S-jJ^8OfXrO_BQ2*&iF-^^LfHk(%F&1{@|d zl3hMsNa;=@Pa=Su00D@b}n4*0(lz8 z{(L6Tw&^roVKoG8iCA^Yb^VZWpM1X~+OuT`r3>`RSZt>;Pnl`*Ui(CuvH^^yY>>=F zH|Hh>PHX{0yvK;vA$wt%Td{|3jV?E0Z-_G&rzSYcdm;wa4!csS*FwuyjGDX>OLAB( z6XS_7F!^$4ouE~A@UZS0Xiy-cU%(gHx-1!j6LUO_gUyqnbvX(pCBY^FAn}kARrZ=u z1Ak~N-N45D_Ec#0d=qiKI$Y=lm=*Sv|Fo%qE3}ZC9CQkd9N@@6^;z>?@rJFHKied! z$qrN!Tow_tMw#T~n8) z^m?}+zPvtCt>)5OvWvpdKX8Se7P;QawUM7|N-4VJzSkZ5Va}mc<8xqf@NLSB8#*wu zQmZ`o^b@tE)R;p6r&@2LY!in(UjYg*?t@kzo>pG{bB_^gdFF)8-)9-y4m?zmcc{xQ zXcbZ1fQ{eRc!sSc6-i)H9bhaISx+Vt4~&$d@W+}vhp=5+RmP-bEAgVX8SX-Q^qi+^ zO6345o9BTb#4nCgXEX z1``{8y0OB5Kn}8q!=U~*01L#fR=lO-<))YX03@)Y;Cba5&v7EQ1oO9Xm0l>0wVLX9 zv8Xetf0au0OVZ$Hl}!Tm)j&c&lstpH3V|xYLQuEOUly!0&funG{ZNKzb7U)SOc^x8 zt#PkJu1rN2-lKX%9Z-!Tp~SsCFQ2C1djIp)7$0!!);G|kKgy)4AkXP}CMHsx@SkfB zZDDTL)t^a(jb3a5 zJ#qd{6Z$w3!@{Hnc3Ya(>1gX-yh`K3s%V9|#_wvYKLnQ&9%nV$MT42=QKtJ)j+a6n zu}JnRq5^C+j`Z+sdytDVjUJqv^E<%*84=OkRJyC)Z>1OiM#TF+0>J&({Qsr`orAN5 zk-e3-k*k%1J)Mh*rJ0@4e@%*#Bm;$i=H}4O1J!kcqL7Nnk76VOF&zz6Rx-U-B1U`K zxSKsrWn9ki^!n}DNog?gF0g1WiQdi**9!j&&Ae=otO|`Q1*17n!ci2Iy8>$x2&A0>bQ7=#ju;3F1)iUWuq5`qC|i!xFj_+4E*$zPoY8V{cJ(+YIYl8!_DZiZ)`Pc< z^=JeN9XxoEjckBbIn0FhBxjg1OTBl3Tr3rxkcn~6qxnX&8Py`vCYU_h*8AR*nq$sR zXoq~OF5*``Lm2LF$E3WYKFY04{Y{6o9)Y~6-?AN@ai=L}=+*-TIeoXA*UvoewLV6O z(*=5hJu5da1$?2x)Lqn5x>sPrD_MeB{eXiZd#L({DHj869yEz=j+{^1b!Y>{45Gn@ zUcc7vq-{e-Z)gWdDm{|(P@*>h#mnuX8(_uKi+C+cW~yAdxhvQ0P{tIoNUBuIQ%I$f z!&(%Q6&DGnrb3Z`3KC)nu!=&o6lt2IK?OQT62^FKCqFM!E%Njcd#v|@lWE->tW5)z z)80t%LAZ((&~5(0rO2D(0}gN86rb%AUBk7 z{rR?=C25*{%AY@T1!g>lWA^0n4fQ5s-v@#lAOZy}gD~;0II&KVMkQ_Ut>3t6XDR3B ze`&gIj<%LpTPb1Y5qa%OfvJm_x{jNSO!fXHD8Sh~bDEL3RK1XIu{}oKog&>091VEs zQYU3n_B5^Hb2tO_{?Q3`^ZcDO-C)I?*Cm6=VaV6um1<^a^SB(o41i23kGdzQibO%Y z{PWBGK2cor9?HO-)L#g08USrkG^P+K$n58R-KwkGR67~UA#>kAPrdjy(%=a(VAcAm zU31feXXes{u8wj}xD)mJljC7%|LO{zh!9U{6{njg_gGv0GkvN9!Rkdd<}v zokYGl4ks!wLZdKyDGh3bmg7*BMR@SI`OqN7cWI6986R=g<@a{#SOsOYS?t*PBU2dv zGatq>qHw~?6jC(5P`5DcLH>ZZ2=ZS`%Swre$*YKU z>1ZV$h$4O6sGQEY62T30`?5_DOs&)QS!hwWz?!BgO0geDi#S5p0CVh-c1~7_ky&NZ z1w}++>(q=_J9}l6czImfm8t3YF5zro@} zQC$d? zL4=!L?!>ve-6}|5wA+S1ISr?I8iFac`u!4`~vTW+HoO3~!%fXmB%B{H|(m*6~7R1=Hi z>LMj@%=}UE+7>U|6@Lq(TtsFq>jyA76PK&XL5!?a3%>tzZ%z)kG*LY5KR}^eblJ&)oyV5Cl6SDZ9fle-ef- z3~ahOTNNuq@YY#=esSZ+UX{1r;xK&ITC5|YxBl9JldW6{#4;Zb$(%iT4>#39xn9v2 z=FQdWz-mkL@Jchi>H#g^?-vQZSWxxQo$Tzyps~1f<;$Seo73&=MMHUKWqXppSy6J^ z0)ngcfyqRR7bU$o^H*qu|H%HiO)O%2ZrL2s?#gPu zvd73P{~$3L`FUuGk8kZWUmci^*Ob$Y&)zR-k97gtma_Vp_)7i*)5&E{N@Cy^Q)2!QOgpxS2M$Ie4MY7f`PFT$cy?7XhV5oaXr zV0#B9RQVN}+=UKCNkUA7*ZC%y6XNet+JWlv8g@IqD{ z61rfuxM}cYr1)+Tn!FDu@RuOmf4gmQ<-K=1Rf3E&&v$6sl0@c}wL%tX*)8!}@>d{? z)D0gt}E?CaV4qPm}pocj8YNVJj6W)Sz0?>gp1ay zyvTw|3g*<7`0|8PrWmibJ~iVvE`?|%z;i`{KW!~v$P{~rq>Qgq91_Bag73$TNNWT! zmg1P`yBOOp`z78+pz?d;|nwCkbx z4UmIlwA0oFe!4CPy685ZwK;OchE9xj%ektb&bv{9uOV%sA~u>CdU>rp%2!Mthp&M5 z;@dE-eK*Wh6>7_OSG5*e)`EK6Ybs8xvOp@k4|RNSzWnsK&PZz(Oa01_k5h1trGgTE zNVK#Gjz1(x4lB!AYvjCR=(}ouO~|10DLh)=6zG?ZgHl^rWZ4BZtwf;2MHX`+(XB|v zlv~m?O=4lCWPSeNvjQ$ki19;^lDyKsl^nZ&l|Qe=p<{zHl)%r*F_)39TE+%SjTW5{ z_UvyS&k!dfXv5-{CT4}hA6hTQ_?{vSM^pVP5lAf+i(MCW-z%cOZOlQk{_FP!xm-;R zzYuU}kfP7FEDO1909evxS(H?#B-n9bNnC#! z+5zRr?{L?_@-2dT#b7Y%z4tt6BA_u^VgY+dQIGfz%rnmjZGh zL3O`prM+n27f{5_S(8j&i6Kpp+h2ZaG^d*>TH0#F9WLwq>0Ff?H1av=p9o$g{2Z?F zDzGnVEqFKMEE)H~KX2plSH!|z)GSAN{+ag=1+fKFX4o5$U!4OdWzSd_g{qFf<`0-h zST_rWVdVjfPR$}{v+yyILyS^DN39e0vEi;-#<-4oFH?@)I_0ycd?A*SQ5?~TEH6H) zX<%h(jtLkHMA}qFe0JD%v<%q1d&I>`1Al*I1|#VNvPo))}$@$_6Ps8 z9inDnASy?~CZBWz>59D}i!K7%kG9BeG)1k$VKB8-Zq!`!I@d(atYaCgjRAiavUUhTNf88ANJM-KD zLJ?^J$7roL2`m(Ulke!tlrm;m-(D5bg}TXMzFK_YFur9lbrQERZr-j(fhj^mjnQwn zXgzF6&6K>d_eH`Hy8$riY45v)Wn#YU3Ecv-lXNT}!)9FEXZ6X!zNU!B!KMs~{<*g8 zo88%fbulaLdBiaSfG8j8=P`@0D!*~5<-ycGE?j;ixxfVVr{P*jTE>1h35ex~1Sef! zrJJcK$U2;sC3LK^POGT6)Zp*Hm}^gUpaJ+y*ypT^1;s#+c}y+}L2}nET}i8NH`3em z3hH&1tvo2QaSchK=)zbPm)$v(#Yj*Ub=1-!sNg1Z6;(gVxsIGcZ)Gyz!#Pf_+%+QK_0~T4NKD5e8sG ztb3^e4@L>-B!05ozOa#EWEL2;pIyKFMvFH#U1a!09q3vuZ@)C|5Kh%k9~{%7hhCqR zfzmZGETiThy%gY88FLl%HYNUJJoV|ANK{+5TlUueMXc@{G;d-zZ*=&yuJk>Ysfn`E zN75^sR)X=~eGcPS8TV?88gSVO5sQ9kGd!(uq?TWSkS@i(q{H?SNJ9s**iB`e!U#qQ zC0SHh?dI?PrEY+r2*;#RMLZ2w{oTU0wOB-ZP%TR(o%SQKaeD~BL!^gQb6C_1rA#|6y&xUm> zZa`#YKt&;X5bf(pi%|%H*HXjc1!9CgAVOql&i;eUY;o5$-1m`6P8;!Zfgr+gBT-p(+W1UTBxK&2O zBv!)Y>rM$*_h-;)(#CVz*3m>)aRKt>38bxMR8JcGX+r>_jyS~>g^Ef^wsMAbg*4ubHK_y z4N++BKxpuJ{Nj3;piCJo_h`tyG4uhSJd*2n9qBzbR$(`%Hzrvb_VJL2qPPu))Ec z0GLK%?73L@llnC`h*{hDU}Q|2MDgx@XG&)21VA5hZq=p6T&_#oKoi`Il`(yJIAQ4; z5W&UK!|SGMar3bE8jXz}r-QzGr_$cX)ay_S$=V)kUlK ztsbz!2N_I}D_*GTygzVeXEG?Wu)>Y7*{gxm$K(RgO+MP7tY_FQhf9dxjCx1@T#M%& z7T8xcna=ui3-%H~2&xNWTdBSQ2nfnk9h)E(xcEe+u)mK)0sKvYqAK59ja^CY7T z1b-cLT>f?&o>66RFrVDJ3*Tl>3Pk(L-l5C6HI~r3&2rE5ZJUP{nZnaNjy{t_GXP?d zDm5;qhhRqTKSG$UCbQ-yg3a+jVd&+g=Mxba-w519C%DtnztGuU%M5Ij!6HR3W|6wt zR5rgi`4vSSY2ospP7Fdn`fPxkSSPd%j(oX z+Fv@Rj9tL(FC#+o(z28O@8U5jCMN(f0Fm>zsgV^nG4No&x2ZEsW?F_53=GiXg*XinWqf7U{!ilYVx zY=?VtgEE>-{OUnPilF3AjQH@xV92Qu|BgZ(&H*wF|mz*PF^H(R;j-OTSINGx(0vwqPd)(JI>- zD@Mi7`~oe#QHBx+8~qAnbuiq^~QI|}@)M#_*HTVHNc%hd9=efBpfg#AB27=B~w z8HR(qRNQzMDtxx<_~rxwOEIwd9hfhwRT-e4U){!%C-5@F z#&yK^M2lG2BjEGJxqYiBv`_X**=E?@t$@znO?A@6UKs2eUR=6LdP%Z0vDT&QJ7SAK zVxF{xxx#I;&f^(_`Q0)7s10azmaYbYOhDpdgN;itIh>9@$+EH+c4TXj2#^?*cZHYI zel!EuvLgSX_>ZxpCPbkd4r>g;0|5kN0}lj*@z2T{2r_;)|x-}GTCr0D0t7v<&Ee`A-2hE2t936=v|!M zx1E}Gg`@#II;D~>X}7wSXrA6vNg;69HEATrNG7c0r-1svapoq8{<3U8w(Z>4@g^); zA`R)FAxg3Z4m_MK_RLx|S(a4_)B%z-67$pn9XMwSi66UNC~AeR+IK&>(coE z`UDmKPK6qEZBS)pCAy4R-KOdlBoMC)@srVX@g9)REg*wmQ_oQuX7)UpMMUk!foFJ? zrS&-9OB1Irk=T;gl~KT*uDFf{sisjT_YJJBKqVKAX&ez7hRL;KE)jb9r$$^E5w_Rb| zigGBsh%wXWyTkOP4kaa`fpKry0t;nxN_eceNgU^-C_Xv87OBv6xCfn8(bgp(->D`a zht_ER*gS{cliBD_!+jw9oIWXv3^di5i|kRX#dfSF=yt}XO&lpm!}MYfG|A+nE~VUK zk9=_^j6TZ(-F18}bQs@k)28s0Gl`~i)Q@&awurZ}H=sgIMr9mL8;Q#x7JPlb{ce{( zB{|J&))qo_jzWQGe4_WF6<4lZOX7y~mx&txwD=2?hwH3ko&ECPt$6q02T5fJ8m!cT zHkE~fC?(oP9k)mF{VC`&6X~FIDuR0MbZ!HbE;5^ja_AcB#bR2OB^Dw9x1|^gA;Ndy z{(fdVBMcjWPAjTAaQKKr1nk399lVs-%4EAy-;O7;;vtg%zcnQIuT8@FnX>PqmoP$) zILK?q;p=!v)cpq*Vr0YGcBtvwRI9qd&}Ue3Ol8Z^<78{ius^P=k?Tr4**S!A<~PHb z$mgnLkelo-C4dJ&sDkof=+jx}-fM;lu{XLVTwm=ui=L|HW;gJra0yJsO+q4l!{o|~ z#Yuiwz1jJZ*(MQOGlpK*i8dYK_S~jWPna{j;ivYD1^R)!hm&$k>oWKvLdS1dt7gSe zSxWGcTuWtB=)7MJTJ#$HjGkrJNV){jfjfUy>^bR@NxrqgWn`*}j{~xr*UH9-hz$E5 z9qhZMpfGQJ1VV!WDC#5TJ@xVK`{AxW5lvm`y_J2~z zRiymEr4OvAWcv=8pB%GOFQOuD=q#G^c@?aj-nwUKX9^a=VeQ9}E89cgbQYQWnnraQ zx4s;Ikm{_~6=h~GbWfYMeQ_Z={5Y3s`#29_I&f^{Z5V3XeC0w%Eh(@+lcs)zGJYRU z+=uNvwDvkreJ+&O``CNy=QxcRZ`En=%~B4>W_y3RF}<;`uJCxrJFl(p{`Lu8tc_bA z^Cy%`KC0L;vPb0#u*jz1wff$v%>QS3HG8PI72j(CN7gYZQaP*dx2+7+k)%B(00?4x zZC$x>bXD^h#GWvgO*?4>rrC7eC?3@Gcs z5^k$`X=~VYF@RU`X2AJ#-D>}LJdL>zv?R{Cc=ZDcVciIlLq)m9{9#aqgLN`9<&2Ek zP-|#xuAS(*dstpDv+Sl6poek+Wa;q4y<5d_(m4=snz9V+of5jUI{6Qw>X`#$nqP<93)IafeR8q8?7aX)+>TauV_9p_cbFx{2ml7T1ZP6 zjP$tFt6YNxW&F^GnM`?_Kyf@JqQHzIOXV$&f>V?LjYCOKKO0nT2xGwBdL9T<0-Q#T z;`+J=Cxt|TECyRzdOESeQpCt%AOsx$K$>zSc~1f#MgKfCKML;+O=(p6rwmN$2+UJa z$=c7CAFqHSSR<1Oj{IJ50P%@2KP>^RwGR}z@3Hs6&R(PPrb# zZgSzm7%j0bG4##|QE=23F-qAXi6fANZd@!BuvfXr-vJTSWD36%R&%=ZSt*-7c4&$+Oq~QEST11RTtO_U`<9|{Xd)YXv6_Pf)m)tXjQ;C}YUIFCi}{1k4}q#s}Q318A0+VJS?yU5Tu~@ipC< zNXvN=tKui<(oNO1{N3uuv^`1y0b4&#-d;|$+9{Av5O5{Eh@Iu4zrQl&yoaXC?MvDHuJwU~Z$joj!? zlu_0(#ZF`{b=&}2-QOiFsmWHiuwrq#OjUyI4bKMYIy@W{5xucKMqhhD3=s>kJa7%! zb38_Y*owJ6esOFbi0h=0=bb8o%0^&Rgq)1ZTS)c;h5G@1yaXw-z>@qun`;(~D*nN2zrkQT z^e@*sVB6<3prhiPx{i)YCn6XZ6jDB=YGiMs?t#r_dW`qhIj!p!YiA?lDnk0Nk8<0I zaU}>+58s7lL6;p#wG23pn3hS-5tWJuKE`l?<1IzxIU)oeZAM^xyJ#Z5e14oEE?sO^ zp40Sxo!150NkRZxnxIW}vlHQR>lDe=|6uH!qHJr{WXrZ~+qP}nwzZ49Y}>Z0cGp$c6>F#^S=+Q50u7@w@%$4h5M#dKrsauU(|JU~v-9U_-X54mG`?9unCq*zT z^r(56EnYU9x)|bZh0RoB?w;Bw)qMqq{e9WYqIqH3F|(NNV|lw7o$TPxqS%N{oI@+Y zQ|q*XE;1k}Kf8NNbLPgAK84^-WZAk(!)1>Gk>&VhiprMmx%IOxT8iMepOv1q;(Uc& z$e8U@`PyZyPjXQUf!^uzw|4#zP1@pf#H4q3NBYG{iD@LwffwRH{?@we_wIk${J_Nc zY9m(lNtOP{)4VtV0DgpO07{~QBC?`C`<*TA4ZC|0gzg#Tst6F<^6a7;=d#}fECFEa zBqWsP4=eTT;eGYd@U{bjLSfP%Is^g`kaTf>1cmm29Qzh-fNmgsm^+@hB%x6dN-Mtc zM>?EW)7REGZLSdxgc_J3Bg>wWJ3n50C8risvUwjA%zFX1(mm@L-9i zJM3c!z(q&6;OYL6lQNERzxquF(b`T#Yf&1}ay~dc2>&seO1Th|ZV8I+<%QLasC-vX zp;<(v);W~S8j9$E6hfj^C(ts&^k9+@AxE%96D|bMgv+mSwMrAslPNydcU8mTm@o-{ z8PDAl=lhf)gGbn@XweaWX)+mv&Cy0~J5swuGf}a0^+)*Q7pk3v>(Ku+ROIC7XVdDao!2~Y9_2#K3JJe05n8*l&{=w%Ee zk1P%Z6dBmt;K+mmeo(Vh=W{k~b4DW1r~#V+(4d$nj<}>4wPoOELYqpF6-bNw756?& zLKawj;u1QlgYOCc~GR@wZ;(r7M(P(i<}*BfR`nL<+y z9%kUOZ1h;$TXF?#qE1vhb%h9`z}wku5@1*_rS}lE?~s@b;0G->W(gW4$awc@*|dRp zvDNu+&G|%zL^RmP+8&bhoJQjZPr*CKkS*Pkm}Er37tooU(^xpDZ5M#m1-oapa{4Wr z`Ji6JN%aiv<%wwCiKKw&c@dOp2F(3hQC7{E0B-`bjPuLncZg^)$=^91XormF2{X!< z7mYm;XzB?YEcOtBfcaAZWkQOvbbm8A(jTdwvKo?a7Phv`rHfxFc*+peo92kr2f>rg zMc|_Qp3Q2G3b;hK5+ULG$nHKVYvLl#(r*hBY#3i;n!t+#FQ?xsmu{^i+olC_q&!IB zOlLDB^}!*Z_wx@c4x6i$gRpfKrZoGV^D(+Bty%3I0Ud`c|iAEb#{B2JxqO!gHyoVAdQiTW?M>`g{f%rh!sB>xZtlv@m%R>t->-nD-mvTNejk^QDLuImj>Z_kItloMCii>ZY|CEG3vaY$-CwRLUj(Z6 zjX-b5^}iklzMcGFxC`qoVB5VE`F(B77ChoB=Vx4%WjdWczc`}em_!8$kbNDSyQ<-~ zXd^I=b{hDNZcG$imPSscctFr-^lV+tnYTy}FB;FGEVU=z2r=(c=2t-K8#{JG zsoY=6G0-3{o}`;FR;h@18!^)jqG}4U-c5Gd#N8gs-Sb@7l zC~ojkUFKgz8>CM*&Q2&Qognv4KI_FxRhq8!Zk_zHp^A)!43>O^a(Rj1ni8zGRIgy# zl|wBU0!W6SQyO=cN>k*QuBjNl*bt?(sxA_yWLwJ0Tl2P|3)tI|y0ziuq#w_3_O`v{ zhybX-n<|!@N#cqY<2==sueWaPRXC9mSjk$dt!T(0E2H{p$ET>it6X)|DkMPN>6}}w zpKW0E3}0C_D<^cbmj$fv^N9EfbJbax#`Ig2*1p_gEKQ(DV4RuWVJ%O{ z5ixP;G8wH`6n%RvL)I)cVY>}rJ!%EbDf-y}4?8&5F8`s_OCzqU{RH{|h(PRFC*vxs zDX=0sPccN)ZxYnlQ?p!NHyOW9anotqj5u1X8<9S*N9xN_q2*v5y|`ei92%!ip|`h= za_gi?v8OU4u1Thd&r+Z_jhY$+<$e@dVoh%kpCKw&IgOB4&fu=3*mK)koA&p=4b0K& zh{GPTVGA$Q!ru|V5Q(acRGMqMRz!ugSJEx)srW2W5E+|*o1 z*husVuUN!0@wu9SUYz8m@Dkr+j={j0f-s&~yc!eMT|zT{?Bgt;K0at2vlO?jH!M%IoV{)FFj-}%)_b4RPK^+yW0 z^`4|`f4#@GQoxOn9ECS*Vb3w#GRM@>OQE(FN5vI$8MUQd_1OgMXt9<60$jCO#niJz zQ5mLu4D;ftB~m7uj9GXl=GIV)nI!{Q8UO)8J1}GejyB0+Z zxi#*QW;77a@T_`yITtfDf_~O(>JGAhUPA%>m7$r zcw86H!vo8EFK)H`!x-QwJyr}u+x1Wrr`5s!=N;AyZb>~GoqYbZ7oP#ewKw++&#CzH zqt|rtJGz_Fe>&>&guaI_WZ@|@iFY%_1D@4y_V0cnEFE@bwhNtZLEZ&*sM8BzV z8SeO1R)h)qJy>f$D>P54uN%onz43BsrHZ{-Q7$(|U4cmx6>0B_TDjndR|_9m(lX@) zS&^yuaYhUA&LF#YO(UMC)Js5o2z1tgHg5LU2R=me!NqcE(_faCR}M#vX$N=Nonjtc z75BYAN3H}fwqZ&NTe3{iWr08v5Cst^4KLxMOIm9IUJExFXiGO1Zd!2G ziN_oc?ou$$yX7!mE}G|ntd>Z@BT~1Hy40yB)Z-R{DP*Za_OJx#iO;jcJ;2(VEaxO^ zr)obxPA^YGs2wX6l9gwgbF)WG5wZfm7u}Y+FT7=_!qPE}D76MP#{AvgPBU1r1t~=Z zwS^eB*dNwJYsJKNUvgFBKPdgOJcFt#t`c^ZgYfbvR|^mwfAh@-B=?UN3j@`a43W7e2OKc6Et&l6A^gG$vrCPFrYNCC)o;HuQNb-~%5-sPsV#(E+ zixS1qS8SAmE1m==$~2l1xU=ofMbsJa-H5>fZ};2c`oP@eJSf5_J~fmns7JeNI|+u| z17_hO;a7+d=ioL|vrEEkMkLgLyT6FZGQO%i-tdA@yP%;EeLtyoL{r18?9oyy<9sSz z*jRsj7lRv_!b&5-CVs$oT8=VX<}1MDjFmoGt|u*8qDfo5~5g| zlk+?~J}1EkmNJ$~JWSB{%-Bg=+wyz8qktRKy3PX z6ORe}*(>){p}KqmD?Qk_yoz&nO(Jr{MqH^_j{1&n2nAB ziDsdyNkB38jDKAC=&Ren4-6i86GpM?_{;$;I!veUxkKediv?u)tu@mWatXC)C^*yx zFh?~i*|YVl%DZgT1}F|S(RUIVCbp>Oxk1}5U> zf#PBDk`Sb<0w9b$#kl^ta$YOB=SLk9>0LBSbKF&9HXCg2OX~b+@`#+b{L|aa1B28! zq9u(kjg;49-Pw(Io{FNT-|XJ$FAobubYzR)OrwU+4pm{Wq+313RU4B^8621{!&{Ee zr*KteuA6=zDif}V6O+g5jmO*+TZcS7KRtt7H7SgykOMt)4z4pehyv`8xi-4mLYc5P z7Q4?OkcD#|xL=N@-|1H`W$;0`S$tkqN#$tN<_hr1HxnIYjBE0&eCaH z0LgB>g0i}8v|Tmrj$MBAJ#=&_3mf}VZtZS_+PW+yu(&F87H%>F0Ync6zYxzT&l;}& z2J^9*|5{K8{_dShHnw5_Y`VE>|L~NqX5`)7PHLNl+lvziu?F9lSwUqG-BMac<}hR) zc=O`-J$A5BShVU0HM)}(MSV0l5|srL+G!h}RwpVyvD+7~nKJiK1(9qwvxnL)=m0nd z>s7WV*STwDam}_Qia%id!biluHI}tUL%@? zm2e(N&&G^FL*2K8Ul(@Wv5y0Z0u1o=#O66&`JzbJ8Jn8Btd8t z6wyPK2xI5^JR4S7XC>aNMrRRN-0<+hsswg;zdYfIl83QDA{4oqaCP;_0SVsQQt6l( z$};U2O73!?pl+qqSx}B_DMU0JnMsk3KT2|V7)dU@eIs;fN>wiXDEm8Q2OSmG{`7zJ z&j&JoS6t7!9w&72A3X@B7LNstO58+@*yP6z!i7I zD|;>0_SfyoD)o8D0_DTm!$gVJ1vFJaDU+yI#5JBWgt9yZm)Es2V{|LS6}+5WK)74r z6A)C$tas+(*hV~-_0E+txn;`lS?vegK~#A{2r|SED7I)Fru7I!@pA>uJ(M9N^ESfo z^Dab;{Mt=`3*p@Z>dS$`PTiEd_koHhGttMKhG5Hvvey_5bVKtal(SJ;UCQy=!yq)Y zU-Y!p=D(m|W&;ESh_79cjR97C(AN{|BaF2@RcmKJaLTJ{r8As+$h~7i<7a(dg0+24 zW@zu-V?D^0pFRS0hW;qN4D5#~HRr2W1S@|AM=L04&oDJyj;Wi}hdR5`%Y~6fYUx+0 z8B~Qyf$w##UN9j{LkWpn*wKg=qP3}KC`JaWFUWUsbTlkQR)M%cn6m2UO-Wr62ysxk zUc*vI?OunAT{|&BdRX!!Q+NmcPbtGdp^x^}pTAkRKP;ZskF5yn=jvqY=xXU?YHMoe z;!OMR03$jV4;S4ydFep`n4#-?N+9pK0k|zugkUuEK7R20x=Tr(DCV`xUN&W>Q>o8~ z5>-JSvg@PggsQxpB54pn@FM?1VlxgBu!|kW-xA`ZqsJ$6RR-)8srGwA^&$=4JHHH) z~eHY&ASOssE~6a9m5N6VsZwnQyT_D7aMgd*b+o3|vQ4rkEAad~%@I z2Gks!sR*6*icA~7m|{o9y9OY5SzL#0F?AACIk0l*N@X3B_+Yj%vl_qr-@}N2M zFty7)@thpT@8j$?aqs18mGC9XP_)gAN&^6MbrqGkBYb-HgUbVNwUxfP9J z1R7X&<=s(5{e5h6DZuhr)+54X!^|vO)-iuyM*8YuG_nd|S`iejUBCkyQR2a^1dKJ# zKngCPMcD~^PxDVTNE}J@R)!AiCmAuco3#_fhH?paj~(ow4;ZCnb-E_K)|squ5rM*B zh4x>dxy&w82({l0+!%ak+8E!L7!+so!n*I)t|=aB@q@UL>3ju961nHZXf(tuG;O%y z93*DkX#P&Ikgk40b^%4}1P`lOmRO8K0{gAfJcZs)HNo`h{R$V-)8W`LjU@>+XJ{W& zvzvW4LpCKM1=ckS#c++n9q zI0U3I_H`lSl=KzG;(>*K1?SMmAUWwv8Lq?<$WT1L%3$zKj*>CSiYTs}iqjV}>BG|?yG^7)q z9M}-ENsziG-?*r0;>Kw|@xQFtj}aJ(-$q*LMvDoXZn8mZ+Ity$AO_)>MGhckM*^>) zY%xLilK&kLmx{Egb1HY2T~ zcY!Fco~Er>p~nTyKBl?sWZV;R4AK@#hX>fk?<-WJW0g(HOsI~c3i-8ZkMyi_zA0Mk z+Q;o~I^Vu%*SwBi+wF$UIZP+~$X?PevkJA{yn!JO^>n*LVn2*sY!NXFEE4X77E{<;l`_Lep~MD{kYd*N5kGroWYeJEp~6z zp>D_WKCXH<$;Y;mto^-jK1wNoX$ScuYQ3Ce#AG_n3Y=!{RtpLlKMeEJ%rj%$cTA2w z$5QgR#69lP&wxC#!Lk$ZHLZq$>dbWB_V*8>rxkjr(=VGgo6&cA!718PP%Pj0uBsDo z0D*omw(|KzV0(*_7Fd`e8Cbu6w)rDkgi{2zV9aOav-XAWI}kiP3;}KJT3sF~mvc9+ zw+xIb(Za@Alys+nQ)DivpKWz)>fl!M4e_EwAI842D&Ndhh1+@fEjRwht4&Z2dFA<+ zJ2~lxlxb_;$s_#JrETx;%(Di(d$Rd~u& znl$#X@PB(i5*{K$k3&19tv7WM&r(!G=lBVKVn&D zT*6NKhy?3MLouAlRhe%m)AATs404!O*hX|nR+<+%xuF<{PKj_U^5rRgp|0{Uw>QBS z(4rRoc(3W_62t2quMLh^W zstEI)s0g#UBXXSrMLv=qJ6D@~E`?n?AK%XLFX)UeFc z9#g)u_i30s(f;Lm5xEo53187y!e6IOU=@}0gz$Mb1GN`7tH-?AIv~x_^`Uux$^hk! z5PatVRHp^R+fK|gO*rHlT7r?i!M?l*_M9Y0Iw_s6Q*MVUx7P*Bvj5{(h47My#~A@c zRWHvva-GE_41+}@(}e=4r>iO?c!@G~37le%wS%&JP8b+KDK9@Pm-`jj(thQ23V7om zm1`o;m7N)g19`yo#kg)x&TsjVPX8?Tyeg950u(`_{N~))C@+<;Xd&2?8_;COu&E6*!q>gk?ne2xRX{?N%$UzX1 zjVg0>B9{EWnB$SFDu(eP@R-F+_ zNj5i$9x|eQaqIi9{2ZvCcRBz-h?RrRzrOt651gN?v%QV$zd{j=oGeYvXdRsFjZK}M zO-<-b3|;;M-vR)r6!`qNeC(ebSpa7z5oV3->watPs83S__}1R`lv+%aj$Wy=$WO`@bxmfkNXN z8GQp8W}YHz@m=tRYr-fBGc(To8Z{OP-Xli`hqXNSK|E7>^P2kzr0&dfRUmhve%I!0 z*VP{sPu+3@Q>dNm)-!-W3i-tiVfPkW>#ewFFt<|3-lj>1EYcQN+2Yh%`22F}t*2e? zvfnfXQ6VF1xY(tT6OBTNlI(?kgcPwuk@8hY!}bN|vAC(vi*2xL^8|=I1bZ^i2MDX=cxOz?+sC z_REXdrL#2oka5b17k$qKi|<~E3kNj!>}usK*<7;QGoDf zZMnaF(PDGFBf5f~w>OBd0>=wrLA39}#V9(v29VQ(Txc@G=auN|4npvQ3 z0N@*iiZ&DrA~K!>X7!Jx+nJu3LAZ6j5QhwnSO9)d^dy1pzKisMbm4jq$O&hMPKjk1 z|1McmpO*unOalkfArZVlC&Y+^c*DK;r3$sQtxkK^^o+Wa-sS3uEe`l(4g(fplw_L) zG0AN*nB;QT)GEFU-te(2jGgk%>7pwVj*pDMMzZ;MH05(==5-gsY$JkWGU?lNxAG1M zqgQ3|7w0iFP{+rai#IbVPtAbiDSnwgL>+WU)R>)MlfiD z`$%Ih-7s4(Uj!`xYrd4jlPV;ZrK*kFnTBlGjvhJU%NddhD=_Mwy;20;JOznSG02uH zI0-nbw+YAvy{>b$N8sO&mB7YOqB8wpz1XO94|kRkwkrU{_#O+v&DKGn0kUvE?1foe zm=}XL^iub_FB+wlzORxN@u=U32U{V(x5Up`lc2a&r-#KKa4{A^(iQ6z!|A-2DNg5x zk$`#+7xw%Ot*)3%Ur$awC3*uRz=w^#S^=cN;DOC4c2Ds`{aIenxH^IiUwuIt#-GEc zG;Ef)0eT1Z0i|UcaB<=wPZZ4VG69_G+yDfbBHWCCO8{Vask!^Sj9d7_nRexc@I8Yo z!4~^n$M&)*9Kr;X?NK#yFsjG8tSXmycfqSk%E+b+kATT5to1?k|i!=d8a%=)7TQDJsJSsX)I))8y1@tR)FGm;OV_y;UJxbYCx8bzP=v3B=8z&=fX`~=VuXO0^V+; zaZL&EO9rwi&mSX68ot;vwgge&N?t&j6u%jDS+ ze&&7#L&|^?LJ6;J9_5Eibe@m-u9$)#*@J6x;*>)f4$N~v%U)IgAeHXh4MM#~GG_ez z3_9@ZX2H%Mc6ZZ6rCe>)cA(l^DgHOj{S>3&4u_`28i@`r*VdBL{iaiU{(ChddJ2N- z^bJc~PT-(cnuPaJ#(pL6 zk;z4i%FYPCPbAZx1wbb1r?(A2{v#V0nz`?JA457VDncoj?_Gd2FtiA>}Nw3|f75L$%wJ1)>2-8Z8E%w=2MldrHuk)_c zMkt)5Ux20-i9zMB10vS7Z(5KWMMoxwQe0}{t=(;-FePZ+!7Y&Z1&s5|i>T)yTvnxc zo5(x~%wA~)GD|g%GW?rOF>qY9cH_3u+-ry}#e@aX55=Qht`?a06ggLKDSa{;ZSD&o zqA|B@139dmxwti7=LrPk;Eaao8~MT1rqX*BF$gzUX0th3VIS(%kV*@fYD3uNB1vld zA2c4=wK}S;WA?a%n}b7~-=d%qKu|zf$5+NJZesRAo^pXU3G;<3f!4{FMxMbj-@Efb zSpjdhAYy}A**D(61+Q2Ff0g^MmHpD0A2dbi2#E_;ZeK-<;VFD{fcfre^_xKyUs#6p zJ)iu^a4f3pH;qLsyrTP)Ez{lo3KrKPK@BV}X^d{<0lusYE)r^Z_=NfgXosEX>NbQ& zgfCrIE+$jK5F02ze#IFKPG=vYszuk;x+`OYQcU=}4$tfJZZ7t2fhF_(!Z#;IS%k5) z5SqZ|2~O$}B9JCQg?DTy(tVYA*}XGBgs6$Rru!U~96yCPR>iJOYFed5(e}Y#6-kbr zi9$h}hwZJbNTl{tjrVG+t7{2^g>0Fx_!p*+5G8zJCOD$4Y*ro8LbRK0uFYGq5J zD*Tc_hqbciWz#SF^89WyYfFJX)zj~7T_?tGKOhSykmlM`K)lRL5bw*vb)WObBeaut zL)b63T7hd3*lnCU06cdNdv*sXMiP&o-4(3P<`7fwhK(Z_i-ZXJ0nfgVp+;r;?U66K z#{91|`{mU+6PMUuxnN7P);D#Q4)(yat|)&`6RfXJ5@bFwU2zjAQCIx8a48&sxZk%d z$WFhhq}*Lu+UmLpD}>#tX_uGHbeE;#@P*>ZL91>7;UOrzm#PsmFh2(6T~|`h6rg;m z9XDvx)r`!tP&mL+0MOtizqE}<;b)~HCL;iX?YMb=e=a{*h|LveuaaxKgKiBA?2^Tx zk0%Cvr+_U*Q`GR0GP`Xjo`%Z^mk#QsIvv2c(yt3hF(}WgHX*bzj zHRm%h3Z8hJs|oh*r1A!w0ZhFQBJRPLt^gp!$j*!4yEI*50g??)P<7`-M&gv5!#FgW z%Fe8rd8F`R6YDVioW>B1jn{?h$GqcjEEg~pzT*?#G@8>z*N_;3^WB}P5S~|J6n+TD&T(MOx5|Zn` zN9#pby`(4jUEI9Rcv1hv?wz!*^0Z;9$x2%;=xWUq*rDq5NtOq_MJ#1a_Q1QV6FS65 z1C%jEd7-<9OBJ{D$C*=3$Z$Y|AMCPQ{yJapW?_DzwoT5L=Xpi1{Uo&OZHbLySle^* zu~;+WsrHy>ab~lqOdPh8#tYIU?)Mg2Q}*8Rgr!p!pfp}@XabWyr7L)sfLm+AxlIpC z@YfhLZ-1N>Wvh2Iu#Dw<<2ssw}^f;v{v{=_Nx$qR{`!PWSoS$_L^;!nln zZOMxQ6Yu#gslR$nN1NIAZ&xPVpoPpUa;)o=OF#5ny0Loy2I-I-s3FNKAvAc&WTMlw zYrG8QT7t`(Nl(AD@Olo*f=nvq7D@=a4lftxD2fBf#fsC z7bp);51k|&MUb%M_~?R2Zn#21HF)jo5&JfvdH`*_^jy0#}-OL^5BvZphzlptYc02A>xHmGAHWR;$JDBugcudJIn zAYn3IjcTQcrxvPD{cZm?XQjKD88`v*(e~2VtXJDs2K6g@HX&{5$a5ZKoX0GQ$BR|f z<6RwQ@hq)qUH*!aNR0|lC&4I9uXajj8l-Der1j48ji^U17x|p_;d_os2gI&oI6VLXszaNCJ-PeO` z6!Qz+1%x@7;7mBH6dnZ^LLNG`$G=2taFSV{8;IJUaULKm1yO{{J>!Oe{w3$KSI!f~ zgT%|iVI4GVVK>K|(e$Cu;U2Vlv90MQ@uDz~-4#qgY!k*Ys{=dc_C*ZQ@qtjsVs+gV z?g9MU>fsjL)Y?|;c76XcByUV`>i`ST+{xhaMKi*I2I5;S)jVuG|?6biXD z@B8ctPppimn$v13XLdRq?31?mbbhJg>FK@49ua^fKsm@1}MJo|0=zW;h%^e${ zs>;@rg~d8_?B~q$#$Se@Ve}^jG=im^tr&2cz?IH|!2v5BmNbj(>L?*_J;c?m#_h2= zAz?KK?8E>Ikoc_kgC{^|af|#WyIYg;J3eN^$r8JTC~d>EJpsR1;O0*(OEdIR52Cfs ziXF~+C2}f8$!7X-?;frRMWU&W{VoMZQ+#|s9>6pQ{05#nzz4eYTA@R@QVi=k)bb#DJa73tw35g@&hXl3&r|D(M(vt;w#R-H5$KwvZS|W9S%ic?O zbF0Z%Fy%Qa>orH2^pfbt-C5I{6z51~WlE}{#F2EmM!@8?D_iDai%%fJL5%)#0P*8P z=Vd7XEr4Xb?y-+7Et^@5(|%;A08)U}i8^@dKrIw_>_J+9;XK(u53JGJ<@I?A=&cFb zg@4h7H#aCO?*=7$*H%G4f9*=TqQ?HKve*zN8dQioCrY(icZ602Eq}2JEN`PWfE;<@ zK}A+s79l5IPk6$Ni@i;bQG0-BH8rkIVYFdHkVratD9n?Do{sC=Mn`bsX1&uZs{NL_ ze)^Q=8C25jOtiHqXQ>;jPW%HOjd3Vjbnkfg+*QW6IZ$3gCGet#Y6?0?PyTQqCh|#g zoH!t`pyBGLw+A@CQtWu_`@+ccDt)>=AW*5dL#-l`Q!=zO701MV-AkuT6?S>;7Sow{ zFD0Vsskv6@FgM{c8dDS&I6_d6c!1#YM3vx-+^%3eyH_$>jdh1>A={4=)N;NP;>&!{ z&%i%0pz4po3!?`(_p_`inWa6Bd$7HzV0Hv*XYS~3%#`b6|AQKRe9R3kyj{~I06G62xrciB4YrK?Z97W z0%1oXrfm|;Pe+UZ&^k=lcoT^qeVG4zp?m*M@pcZ(8%G)6P1E`-8#*Kt&={bx9%T5t zY?qS)Wq4-o9Yc_u0UX0^NZhg^)_rqJEu2{#aZGd!ro!EwQ^m6Og;ScDUtN6llcPI0 zwhISnEZ_m@x8KO3`SjI^KHFgYZ8E0c*Tl8{$I}9~{?BQG|0^a>`TsJnLjMr2|4Hd}VOrPDGdytj6wrx< z_UZD!NCwqXtA~4$`p=0(pKie^Tw9oP3#fc`lwP(ppy!d%%}uP|`#wI<I!|-Ef`LSj5ald`eJkw%uAoVo$2ghcxFS5(Iek_J+X)1ggJJK{#|ZA3_h<&3v+iTkV6^s9_g zVfe@k-U$pfzUv2tfA#nyIN>R(i)G z@-Wkvnx%@DFeO9uB0hA@QxHQqWeeEVA;197;ac%6w#H&W)Tvty`-{d|Gwr~NG3Szx zaBwl54c8`|gJn6|mJ%XC3z`loiG&S3njs_7DV4r^m~O&ATzHHwf~*8sU=4*Xu#gP$ zDA2P>ni%IKiXpE?89sZqKCdmiV5rrLD50ZERbU|LIWANH<+q>VMv?D4KC5rSoifLAs#Pb!>{K5&~!(;kxR+=*Hn#QX0lRCr#jzq_Q$C7)!sr z-+0*2_(1ukCRxpde|mIAE_62tv84o}P$@!BV2(3-j3ikr{dU8Bp)YmMd7eCHd~~~O z-sPI?Cm94U<-`Oe0~!jbZCy#+YT5;*+OsQMq?#=MuDMudpWJ?hfHbnV)0B+`uUQpw zVEsD5g>Yy%MAHaqk32B5RCFxw5TG?jQB<1@&JE5Ia`W11uOW0Q%#XNoRtA=%D}wN@ zLzg(v+bCeehs>uotLnXP9Hpd?3B*f8w~?ezHS;#OMWxVdE0lB}iJ_}adxP_KdV18k zH?YYB2SKiS|1<0PG#L`JS16=|QAr<|ESeT>D@zK^3iHdEOp+g2V#CZv-MPRbyTMV0 zbHU#?U`JG$cS>W$M%Cyb4tIbMs&%GT{0XA8A1X6av310sIx3?dYW@5syPPGI@o`v# zATeNIxC$4ckI8xUq!FuKxyLF$OWoj5!8H;EV8R*&8xXGrWTg#cva!sGNX7aS_|EzG z5voEfe-nFso7C0KR8g^>xPfS}2hw^{B~fjGS90^iyj*HMwH}%Vorx|L*9PXAM3}(& zNq==^@_n?jMzUw8*2*yQ?bUZ33FIE@HguSOh8imu#jaW;%vE7&#qCi9Nv0VbR(S-3 z>2FQ>5zS6mQFV2r9=jJS_sNt^ZSb)QNiB<3tPJ|71>t^~nCQUE)+efa8L9^DgE&#l zSX?29HJ0_bv}!^1^Zg|nQjIdD^l;LKW0~tgCpJ8-5nkC8q32r=qb5~cEu)alg8|z~ zjsklm9fDNy)q#0}2=hk%qprhXm<`%5{9CWz#v@Cl?)d5W8io3iu%pfub#LJ9=bkeR zv0VU?o?B9`G^0DcO6@}DJMF4MjNA0F> z#`@RR9CvyC)0phcP(OgB83hCagvhg}{XTXwF_+Emg0kN9``TI!y zfYZnhR%x)3PfRi2=2q6lYcnEG*f3Mf^LGm#zi%)MnBjxg13b>zyeSId2r?wQFCBoT z6(MKZg@C-RHgA7yg_9-`p3d{@jKPV~w>Vv8M$Rbr^VoAM=gHhiId751o2Dlp!s6@JWzr3fo?hqB2@Y=i6Td~wgkN;;HzKMAZsWiq zr>rG(q&d<08(IT_TK-?Sm_QXY$R6LL#YU${?nmdswr;o-$#z;P8&VSI=(AX5p-a>9 z(rdTrSP4jR(9p6!@7dl|!R?88uZYij+lsvlka5@( zy7S4k_S6X>l2gf5g=#^ys)njT(kP39oz>HZ^$`56d)aVWIE40YK{RkuQp!$Lki zCHI8q>`S#)5!c#M0d46U+>qD`+PPR|hJYCCl?^R$W3OdCko%t=0O<9Lzp1#an{`n~ zLd}zWt?Ay^Vf1Fw50iv81QfY?aPKUh_b3&OVREPH#&|Lv#6?k?J5}8EFM4*{i1YD+ zk^O34C+;65I%NIGuke=HvxR-PZH_+ zZ!&cL`VOW^wt%_?mVikEs#Y@4KW(8}iI)Hh!`BoNHijC=XTJjX^eXtDi-$BZ}pQg403dwD)41HD=e z!H;9(7?D()#Q;macTVs1NzgxIu}z4ez53QNjoRXH=|5Z+{q3Hud(8fy0H?Z!RHsQU z|F`lFz&ZKRHnRV(0FH{OiHNpud>|AaTy8n!=CP6%Jy`i6Ia^Yx|#Fa83cV%i2u z;HuhsBm$$dn`|`EL6=P^gMP=;Zu?D=!ye&H&C%C6+_~3Fn6UJx35XSeGBExrRg`>r) zBp*oqnO9Q=L;ZV=x)T{1)7(;}a2^#MoP>J&h;Wv;vHh=9gNScwD7qBD;LGZ?H93Y4 zv>JrhZk<3W;ub?m?j@es#ISIKA5UgZI==TJ6SDu$5iwmy`%H0p#0OTX^?5S^ zG@!s72fq2(SVfHR0^Ee@P$dy1h^UG?A1Sd`Ns%0eeQ6b_d0BY4uA_*Y3s?z!RuUjq zVrV*Xw5oehNrV8B4f7`^Y&dbx=)_2C_i=!MyF1Qes&r{SVf=EVzk}Q2ZbRZBVt``} z#7!g?P!yVe7w&Fw5thZX;LPk%v zr6JrXj)EQ&xd4__)x)!HLA9jiP1;tei7=45d95H8B1)zS$+`F^kq+tQ&g)eBKL!R~ zm-Mmd*lJdFGv6+@+l>NOut=rN@)!X|2V6JZH}Q;2FQCa^@+@7gj@@}8g^tpb2Bc9E z85GPwe6mNAl)oXp&YdE7^hUCwSQHgNA)q-*Av380J^vhsV#fC2!$-ioFX)qva_o_S zN_fiK&Ng~{d+6%Nk=>osG>~m^&-A-`U#s`84uA+9?P9u?Ghq0-TGdTyE&DFE4?e@;Qr}Iw`(re}WA6uC(xeA%1id_G2{q zYmVbosKj5G#pa7E$mIv^&gm6ow0aOBNZqxb6#;Unz%74Lm*br-=VLkG^ruVaS(EHO z!{|Sf8osRFpx4u6wOAU+-C0x;ugrqNe}gI;%nHVG!l)rachp!?NoAiEOP5B;Xp<+P z`&L%i^Z)Y3`dsGp6t~!&GiTn7jT!%XdmQlur|x-_=tHMTZ}{rG4;`A=d;fCa;r9DH zbN;@VicD=Z4n;cQxaPA;LZuJ8F4{baXOT~^E7$l0dB{#b32HD?A{6PR`d$0IY15SsT=N#>;)F`cc`tZM^k!dDi8GL|rC@a-F*alxz&a z{i`JJA1%V#h*~e%m(rVl90M6U9&)*SbBzreZ9f~+`6kLHCh&ZZoHyMMpD~I>N%Xjx z8S#Oc+G)Cw5wtvrfl8CvwnIo8toJhhcxJxLuFjM!#8&R~;hk>_^1{7JbOK-iU*i0! zLu7#=jNV;khuvtG`)nh8zIe`9H`;ODAkv*U-I2Um@TX{=Ug@>v?q|h*M$QYXAoj-N zT>Ou*VX8HU3m<)P(%0L>mz&V>@fvp>u31M<0Kfmk**gV?7IoXYNmgvzwr$(CZQHhO zJ1e$r+qSKhF=DZgZCWfYoi#sn-X*SjQJRK>KETzo z=2wV}DmRESq_|4n>m_o{yq)JdD3M#~K0SfCtu2j)OpU@rjN5M0%=9=4QY}<--}$4g zJ%|#9^fWDL$&rHt|v4K_W5XFvj$y4AZnj5Cet3O0g8dVGLUzL-JUw z{8*D=(|egHZiw!H)K6Y)JeD&_5)=UdRJVQCQiH)wlxVxwTZP-lthx4SCgyhr1?n9e zjw{bVl4*#GDn#$W#If6HJICB1;@AR9Jz?R3Ol|pV{XPEIYtjHcgRuG4(%_jAfs~`6 z^bQ8IL}ks7NIi~WEUvt{lldl3!`3H#GN&tfw~GQm6^pDzu07O48KZi3(^X;i#6KT& zK$5K3-vB&Hb9*QTPI%j7eU2^8@Nr7$I=tB3I$d^32uQK7NjdmPiBe9@Ht!(^-ht$LPVi7}IOT3vXLg1CS$M~eMaM-_FuTmZr$8b=&n)cok5=eWI>eF1Ku5U(hwv)eMHNRIERf5( zqZ+gk*f6~;L(BfclyHZ&DiqQ(_?%~{x3K+~8q$jzbr*HZ=HO<`Zo^c*Cio#N&aM?yu z%B0AO?c?4rV6FfSaT4*c!z&xs$L@9mKnkkmb%y~+Q+1W!oFX#I_&sYtMU^5W%)AW5 z8JlYr1>NhbNAfyt?n`R;bOZy0xJ26_+es)=aL$5|{oV6==;aWVc9&4wt?bhcr+$eC z(3zZ!n@E2$dUTJveI@PC2b7h)et``?|DeEwoBmR$i;=4Jow)zAe@FlAY8vX+^JQS= z?c-&h;Lgv+=%A)f{u0;a$MlsgTUu9c;_K_`-~R}R3)G;{o-=TTH2jsZznKC62>oyS z6#4(M9SGaG*;?Bf7z_O$sn&n@EX&wdN{6g>T)l$RO*my~Z*4uLFeNncb4H-xNx{6O zi|q{ciEZqz7hufY`@L>zc3Le)H~Dei{=&L7s-G*84G!sQ`^M%+_78HA2(ZC zi+B2Vbn12o#8f=NKT-PUGv$6DG0ZZDz&+OHi6G28*W^M29POjlzNJ81vzhH^#3Ns#V zPp96;B2t=g4!Cq_^eIlBVuPOLS=``0nQckzvu6xH3J^H8tNa|_yTpCOt5PRbd$FVy zSsoEwkBLM(^pOq5pStKXg)+G^jlrIms_sW===3x{kgPXwcc;QED+N+)5Orn=09ZGN zM=RYChf~iF)F&HRWbDdEU>X31d-qbEUNYSml-diKQ9{q);nMICQ8K)1kVe zW_OK1duSIV!zbKOg1^M)u4@kf_G>ei=|dPZlDorWvb?JVxiJVzqo0LI{_o&(LX9B^ z0f*`W2U%|e17h9yb#w9bW+VgO3Mh8?X%|VQ*r=}D>A&Fz+=fc*iU%UR7$W!t@S2k2 z7~VRhxQHO+OicsM{%;?^9tlr&UCAHBQh~>aPbqvHj}-W`@EjY?u=Wj1fBICtH~2(j{XK}&lFGO4P5dut% zGYc^hCEY(;Kb=^={mQyLnY(!LWQK@82(_P;?ofPj7mA(ca5wG$EgrwWnVT1BnXH%6 zVLY)H7v-5H6(y~&4L~Hxaqn#w)qQz8e4Bc@__AiL3xBu6ep0N`3|SDOtW^pz0-T>& zyH&#cHCD2%xX{HSjnMn2v)sRY%@S3?vZ4z?7l(8P zLV&H!L@zTFg6hYmLk~>I3o-pbY&nwDfzZtd^IiNMJ^cVldJX{k<*`ilPg?v(izW2? zZYNPC47e_mmFyfhxgR7Os{Ro1YC|BzII=ClbLBQ34bf7fk!kmwwGhLzZ+uKZZxP%r z1(-pU)odh-b~Gu9O)aVM{>b_T>TOZ2WG6EYNaFrJ=mX@-{mbd5k>#BAf8FHfaRD0h-&Mf5~B19d|!rUfj?aRIr1enGS!d%mi zf7h&0t|_PK%=x!7b9?m}=E$T4Xz*;`s{-xujQ~%hySaQEnnagx*gIJ`F!QW)8g@tT z!J%`d1dXf(b4D~7&jJUCTR4P3byq+i2R>d;i6Yn*i(ETyrF~SaYzyQZGPqpWXhfxi6r{$^{4wmwpr z0bif?`3w|TEw4z3yZf8^p4~w+PLYj%s=b^8?0e|BAQnZoPHwCKcsgigUm*u0fL7t7 zmcPi)jQ6T2X~f@7EqJFZp7A1Z`mw8q!bNYCqNo$O>atfVOb)8Z2%tLdSh_BADtp%s z+xXb;v2J_RM+htI8Z+K*7*;IvB@Kr>cY=;4nln7SHJLYmG6!4}xJU@%aCx&d2+Ep2 zwgKK7*GtkwGpfw(aEea3Tx+LF^2sjn^HSorz(3BXqxJQfIfjI0yoX07**Yf%XW+D~ zh2w_D?8$@hYtXP=Ok}Q-*Nmj`1oAYI)U=TF#BIqi7AwMWQ16Uo1Gcms&?eXmHQ^yC zK~hD9M()r?O+K3%k3`jJ7(C3Vs;-90HA9jcD!j-^{;PsC#HgXDJ|<;h_ZVA7$sCI| zkO>pV5J+C02*C#Nfg9%4nGRba@%jbPk|f{XvOoQZGAR_ZheNg4hJhCQ9(AzsV62LH zBD>7|F7})vX;XMFcIK{f7R7;joNs~cC$i7k@@JArnd%f&QnWAbP7ImdxNtDKuDI^wU@wHP7sJX!omJ`D7yql6?vSJf@)9;)Rcs-o~)YcFnq<%<3{G7tj zc#U0BCOU`UL`=rUVNi6^=c5+fy(Q9n`Hjb8rOrHuvtW7Gw@$FUEBjI}g7(<)7QA9U zYd9E@(Yev3YKN2OXVc^E?2JLc_&#aq1!y91O_h{>6wcao{>?$=Cu66dd2rzD7LJex zV}6Xe02^ZW#_XIu-+&o}>0wh2Gn2Oz2qBl35S~zs>F)LRVA0(@;3)*|K(Ns&n zKNmKMke-D+as#t{ZAhtg7BCn?Kf+MM7EIowa`CLhAS31AtDNo7Ss-f`)KSzMHL%X| z1GoiIteCFux`W_@mKYXLzDA8`{e`n6$hW`%zf_Y4-sANuiK?OC}rK_Q`DUX3xa1f>8 zN_zt_APuy})Uo%-t~qj42fQzO-@WMKP+9@t#JfqiK4t;`9;jk?99}Jtq?q64@g`$& zyRWsQFanr7QqT>xPpcn(SQ9Mxeri#VtUGXtYM>KT_cyCy`Y_R`4`vm*+Qg}Ih>`Q( z;K2(^Bij#oP~sh!Jm;)|!q5Hai;{Efs54-o^MrJYXmJi4<3NSk!LqS8W2P+Lq|iQ0 zZcIpE^$#gn)*GQaxhwC|%M05sr>9T>_=RVWAFxV!gKbOTsv|wPu$CJyx$}TM04TBZX@BObS46F7fu2hV$TdB#V8Dj5S0 zmXqc4PQgoH#PgZDgC6z>{Fc51pc&Xd6NujJZB~V;@PtE>J+Q-9p?-@x%{qh`;fS?4 zB)BB@zbM;0G+af}WKt8ehs@HwR8Kd3`9fg3rzApee>Ex5PnjWDoph7Wmr84ZXf~l^ zvj1jyX1MpmPUme0siH@Y3qgRrzr=fMx-y9XH0T}c2ATtM+(j&8x4Lja-u3B=!U3|7 zRZ6d6cbhRZsGSs5Y_ha($zKW7UHg_l_x-8}tWvjJZa}0^*?$^w#zNI-gI9^C-G8;P zBmaBJLmwCqGGceKs^Mqu& zE>P+-sK&PP`K(0Yf6QU^_U{R?GwAUGkX`1NrFiRjLA>?j<)P4s8+|0X1yu8LRj+Ya z%F5{UZjhH^!+Qfz@)BV9lGyf{HTHF&d${`2?u>{5C8f6KG1%j+R@ZKT-L3$&oy14C zWoua@Hyrc;9paKW6&VpBBG;pPl-AT<1}43Ii00P87%zZUnr*nJLKTQX22*n`;IP^q zE&E{3N_z29;|K_*Lm_7jBgQGD>ik1bDeZ&2jDj40V<3=e|6NKi2Db6>7zAEEh z_+I%oet^=)=1~-Ik{K|#Wc{V^qJ;o0MEdmH|03n<=R63;?J~|}6{_rNiMnTpq)SiY zo5+k?QP_ZIsuWw$L-Cf0S!OQVL4!c;2M`&m+nH3F?6X{a*lQbKZ-R=)Si z{aFlVXameb4Y^VX8#jKVjRq|bX;;v2+hf7M&bVc{^7*40o^8>8yUvjK2gmOEtpHuO z-TL`lrpI}yXKOQTp-x**n#kf7n5xLMT}9WQe3g{-8HqyB5iim&@>+z?@*nviMxsOX ztx35UjbMz4G6GzdEkV3POOuvP%bockmZ|2PT<&xAE4*V64awrF*q2=ITcV913wzi= zYW6j1rK|zvhFuHB*E5?V(&r|o5=W>QELbB=k-b;o+rM3d`V0aikq3_ItKsY;Mcu_d zQx9l8r70eFx$jwl3xMB2!GpH9=b)A1u&I&F!~tCvP&>gN9@+5+00iyQ{-^nR!{ai) zz$e>|WfEZ6>AOUsp~T^?af=FK`Fo>=AO?7dmxi2x1*km0d*Bow8`|BdC!rhegTE)T zA(d;*HYm36b|skWPB!+f54h3IwDXPSS_yR@`+o;0t7qB4$ZrNqWsogu5*iT8? zck;PYuohr+&#kNfEi|6c$rTjJ9kJ;37)PUYwpL=iTBFp40UilgN6;B$Dc%&^2%VZk zmRi4$R82C81NOsqU=cZ(XMw*Q&J~Y?97hX*2WT?ViH{yLovRn^49Wmsq@Q^TyFXrV zUeKSS&FC+Ez^E*+T^FhZn%S=BjB=|XLVUjY>&?@%E~`1l6ru%$*HK|#^-)4LonA8-I!p)D0#4|DGAW3 zz1WGLgveJ`_Y7KPj?4MymR&M_l}bec+)a;DWNQ|ZM&pTG`azH<@!kaMN#c|aZ-FmlM}rT6#6kxk>9 zWParj2fZ_f(ebnse9n+6%-)_owG)Ltl}<1j@((IlmtKJ&JFFM5(4Kh13HY`K-&6!| z9aW?#>2POJg=M>w{3lab)>z%K6blZ@xX(D@(6B!5#BUHIRfDWr0|I*M`k_}_@yUJ# zFIw&MH!xd5b}Urof(PbCdPH9Ud3<1(3_53M{ z`RASU;Cf3`1^q;`B?1Jwj<9N@@M;mnVM!Dl*SnBSu-N9`(A&3v_B51=f(xz=Tdje&A6A$^lOg>V+p5uaRc{;GXy5_f z4*27yF5)%~MX$XG!5-a&cus=i=Rh9n@MnJLkE()V>cORLCA`|A+-A^?P<;Wcu2wil zq;BTqySjspNT)K+C#sk#dA9tKBcoR&QRs?>gIJYthK336(-g!?Zcmn=I9e@B0wWCJ zF=z%s=s(!tlm~r60=@O3nNuhX7uUp9p0$8@9Q}p0CvDeGy;6e_0-EEHJ4vRsY~?cC z?>iKUE3zcM501BbSt{s8S)guFz^BLyCA7uKi^5ZVb6KCiN>!}C6YNDj*cugz}5ZwF=S&25{firDh3a3M20sMoJ%`#v1S@q<>{eIcCUfhmb{?7IIX% zP;Guia*pnIk|wxXkQfzkeg$_6Mc@u#JuspTN#zT=5id%eYl5HvOghYl0fZ;m&ob{Uhj1 zff$thoHMW12E)-LgT^W=)vD4mM+4e97c5Fm5JJ5!bDa@Fr_?HT({}-nw2e>74;?u}Z3^YoA7n{N zo&jT`nPB?gRJF^3?14zhs2Ykf`pB>#8mNE%aJIO3zS-pRxsLoj*L@5VNW+FoJO#td z-O^zr`nsEj?vNsuFlwcXv{O6%fZVGLW&a zhV-iy+r52grL?95^E8e8*1Wco%IY8)km`Eo%;*;Osw*<4J~x0T4WRCC*O8Pe%KSShXL*lj3`#{P6-4i?W#>a0!L@4*yD(#%$q z4O+Rg4UW%SFUhJX!yM+Vpa#sd%eMY01AOPRr!cHplod;c&ljF@9!|eptc2=1BKA3r zR2s&j`M!_IIfb`BE)7@e=&Vt58i@kzA(D~*BaNbaB|f{v@43Imog7L9N4gp$gQBE^ zhlAXJ^jJz=o7(V@U+IFV@qt@scs`>O{ZaK$LUV45RQ~rj#C@c06(gQ^#0N(gPtr5gLoo3SG^Cx&h}8(Zh+&U0l^y|{!s=vrlm<7sm8?X+pm z&2^Alw$&RWFg1vkbJ?_-FumF3Z_y?==wT3&!3 z`#1mqy#GH(*#AAf9>%bVTcCgY6%-PX!t$J8=<%A%itHr=2LSX8r%SEl=4-C3yW(Sh zcgnTe(yzrYGTjWuxXylhc@aGKEMUM_Zp%Z;0sjTveLe$vTa*l;5hQrt5%ZQ?pTY^^ z>ywd}6#s`1xCFhlX!t8A9On*CK#ZlShiuFLCVx-dcc@Pfuf2iq5s?nujb~a$4p?A( zXo~;TztahV)*dc{>un=?WDA0KJW*7(gN@*{{p@5b>xZ5o=rCk8>aT{9+B=p{3U|OO z=0Z*gO&cc!s}YP)E=4R*6ZCZ<6Esf~)g(@hB1XshkY;L_{H-Y!W z0U{()2ME~txPa18WYZtWVT8RQ`BspN>V+xKJ>?V0EI>$SJi6I#S0Zo|CtAs7jyWM- zNtX^G7R>@#o+9dbCPGkRv~3lSV(cMZ8J`fr44op8kMXP`GgTlXDn1v(_Fj;j&uI@Q zHY{Z=M1pMw?l}AdS)niut(`zpbi|_a2y85_P)DmpA5+?(9uS&QJkn%5{FLqOzDQz# zSdSsEB}tGVyZ2~3Yu(F#ebV+7DoK1=SVyJ*0XBxlTFA4nC<7nio-NATmQ1kIJrOm9 z_trR8E9>t{j8<&vNZ6c>MNz&9Tp?Eu)axmF;v50gUBoT9X9B#ec`PEKsjNFxuLn^C zVV0X9N#U4o74aiI18m;E{YJt^g%SuCy0MV1c)r%I6nc=9zzQfH`qg^)qEhC^83HISc|D(9(PZU(wWuz}MSCeaOr-i?fiNPCUjxdmknzNZ_7rMUay0wC4qQRqAec z=GGF9dL#gvAxOX=l>eNFJ~F}%i$)bMYD$oyi-b2Q1=RqIByuH|P1BtG(4FnO2^m1K zC!%PWZX^RjC%B|6F;<&{35r@D2TTpv@4z_$qwg}_6T}x=L(fmdE^hG2%|1y(Qo%Rr zZk-3r#c~WHQ0Y3G&|LPp2a$cQ*bxSCkuE+iCf+T1Q!39ihqGIOL$X*{2OnUx@It=T z$2ovOrq%s@2LozT#&;FXXXw+R=OuZwO|;#$%KTE_HOq+VVWcJN0l~TsDI&#sup0G^ zSZvHYM5QH}8u+duBkG44X;=czsss*m+U7_MC)uGTFJOw-ceNoi8b%S*W_x?H42x`! zCZP`34>lvuD9wH<0GKTexI(k%e9g_t$43Xe=(s(HsRiNLN|i`^d)4Co>SMGf)O&)A zSyFNZfsqeCAXc=O`np`Kimg^~{mX$>jszKj*TPryu<*tmIimIbo?;1-x7Rfb7y&N) zHy$*4da`a^DMFb66kFh~o(>mq_-<5OUnEkXw0#H>aNhErL~#rm>EXaIxmmgF`~I?w zQNbERW6#DCh5^Kwl)jPZ;IK_NfRrE5sD_nr1+9dDl~Pk;l%Tpuh`6JTpBPgRR9nRX zJR|&ORd&L-9L4TtsWrEVat+Y{j|k~fm3oKj3F|nlOsP!}k1D%thPe1Hfl87pkZf`<%}vHM)RH=fA&QCX zi%jQO&!mgap*Rwu>=57u;Ryn7oJ0>QM4-Y-A}jcW!LfTiINXgmD-U20s37RjSbJyy zpgE|$XtwSR35oS5;r=pw^dFCRr|dY_eyrCDe|>lM^++H`4C92?yrH3H-2HZa8@&%F z)?Ssb?8-$(3Ti|)vev|iD#gmlEM)e86ZYu%@ycwn31vf8+L}s&S)#QNEv>j?HG2)~ z`q7lkoRU*l^-jDtW?c9L3t6#x%VziY+eO1q%W8+tS5cr%Q2?-@ky9iPw-GP5pZa8E zf=Y1=iiBC!OpD+bRX66gLFfF3`||f3MvRzW1@ORNT)auAsBxCf@or{S<;p9{msq{WFP{#k92IexCw=G}eS`9!~9 z_dQEtIilyIwKp6j=FEBz>Ky`~PnGt_FMmW!^u7&>{z-Y*xrL<9m1UD<-=E;_UfRR_ zCh$)IS8JyOjb2nSOkLuA>p(U@^oM{9EOcERnhZRJ*!>$3DCT^^5eJguQ{{n8|MTxB z9$ooViJPc;?OX4EQTKE{Qz)eMD6@i*AH)>RvcXt{dlNld`Qo%T`?$c^V>y=8v(pgqCZ8Nl`RRf{U};?)?-NmUf1c zgeGXSV80904jGO>eJnW#WH^`RIx*}<4wa9Wt_Px`ucLkF_fTd~$!pKw$ICs#>$udd z4<1lwP0d8=H1xI3taZR`s#60)(}cN5cXkH>Ho4aH=ejFEBM6JBs2xe0tF1^sDJf1sb> z3BIMs;vl+s-X}!~47)Ru7z|f%P~Dr7l!hP)9 zk!U2y;7GKJ(wacWv0vYw;;+C%lVL~xK26pf^+51yfoJAa&RqzTi-_>xXUMHLfCr@i zAt+$nk?^Tl+X|4TUB+>H0+(sPsdg+Mz=M+;L}MZcuU_Zg_<@ zAWo>kGadjq?0vc3?>*@IA7fxRvskFiJf#wgGB3W2&i-iCIQLmICe*2&MXd|9ILzAg z+xa<+I~C#G@~DY<$WWfA`>#8bUuuwc6KVSabB&Bgs3nDCH zVN-}Ye(ic#(#Lo(Z<6q`>6oq3>9K%w@crlxgtZd(ktiX@NTx<23FL-#=ZvQ_g}x9?dgRNhKpbp=z!FprT7USc4)?%E_RC|DH7d% zRgCDEzbRX`o|>Sv*R&9FI_7_m8-{0{7GNo+{`|=sgi~k8{%mt?WKZFkUAKm{1###{ z3g>5tp6t_0N@D<5x8-LV_0q2%eilr}-xEu}{Z)DlvQO9>{DZCrgSQ0-0xpv#s~Qsz zF|EA@MW=x`ng#_A5x;h?QoHH}-Lbz0Ut zuyCmZ>shpmNp^G=-Ze?V4w#Bk6`C>poURRsa1r{?)5k+XGDA#xrwLNP z2|QsJ0>~kaRsnPY?Hn!(@e`>LSb;R-*pLmIv;$BRDUz~`Aa52MRl42AgtP#`Q+iB7 zhR}A+5rVYRP)#rc+X9eJ1f$$5bC&RLOQ<|J8)OhZs|s+dyp7BO?YhcYD=uzUIJZ9r zQA<8OvMmiT{1_PzGilG54$2_i8;B)7G>^}#6K`PU<4JM@?jl@1dVF<)(Mq= zWS%%u$h#W5B)yg}ZWlyGuqa|toIr`!;XpKr!(1z0(j3#Bc@!S*6`w}63E<&gGh?&> zz;X>4HF9(loDgs-$T(t+Vj?jsHu_Cw;n9cF$)_69{KkNYNJ)$k7%vGbSy4PpQ(Vx}vEe&NtUrRNWskkBUMpPJI_N(>6l&R`Xy3+$`5 zySmQ%>6_YXTfe2wt~T2|-EJQ?OH6VV_k(C5$aC!}fUy2o2w{LZXb+Q{@!^2{rUW6@ zYo+rqti6jGBOqKpt`BVEvza>%?k-4MAbQ4-8Oe3$i7ihh#&HZXqy^pxMtM{pT5wI~ z^pY}cBxB&c=#m~JsZi;cK}r32V2)}mPVJ31 zRf-6RS~e+F1W1thD12GX%4Y^(>|}>CCg&`Y9drv4#g6$Sax_;Ne6pq`_67U#Z=zw^ ziyf}3)I|bqh{RM9fzmGTRA>v!dqzR zi+P1YHOGb<>)|Q}>6|dL0}v*wFEol5OYHVMay4<(uPgV=MFTIcoLff)Y*-=~7)pSO z(r0l`80|Sm8gQi~DYay@+N>BuJ79!BqsQ6*jxYc)IT#c`KQhchDL=T*c*-u8_90PS zEW&F^1^TP0xBKQ0(!o4S<^w9ibyL^HvYspsDj%2g7ehfAj=B^e_fHyftc|T{guoDc z?pV4=FcGlHBLpC@>#?C;jQuc|P{6R_;?$b9utXbAnzs-P{4zg9h|h)O)?i|mQDK~| z{=In=&BkdF*o?vLK_LwDX?cYvloV!(+g9!xc8lgUX;NftL~K<$5J{|S-3J0F*wP#0WQTfr$Ye2J0r z${(Fu3bBkd&fOFlU{umS+{Zm7!Blcvy1j19c(Vu*v4=A3rZa!XL|+IeSEDf|$8oZY zx$JxLd2Ecd(oC*7&&G5MNK7WV6)pOH{#Cl&H2mo=_2a6}-nBt(`Xb|@UQ{cQW&5~AUb^Q+JXE~@`QSNIAJCl8# zUe?-kQSX>wjCyNh*3}|0el(XI+6dSiucHLit_1Xq`%!;Hank{PAoacfPL z#^J`Wfl)&O|AP_jD8ois!W2(<&!Jc89lTSeg9Mt15oG-CtSJjDdw!>P62G=ChdiRR+ulpbB_veLF!XM@6Pl_eJc(nI)J=~EX zjp!$B|1RsUMSm3B=^T>Y#$IWrCHz?v8kYu?n^8%man+AQMfI9cIXv9Q9PsX#$myf> z#pqPV5(mO4&M`61+5R$Ti`ATTnnE_=RLx`KnPmFRiEvsT!G*n1RPK2b!6Oq!jOFskaV+D6ugNHb*AAccbJXs0)vL2M z=#w`6io-5Po(U+}DottvPelzQG^@KntGeZ&HXedbguEroq7%M_6_XD>b&$5DDbUd6 z>G=Xzp*}b1lQu?^+|WYlF}vSbNZuQ2O>-aL_d->5`strw(y1t^X~D87OLv zVs!nGQfq`X`w-saSi#T%&v2w7ip3c3Iw7u3(nc(3V=!?_&npChyTj_h>v4Cb)mqJR z$$}rN*VKKo`c#>g?2P&>GN!%f@3mb&GV+`uf467HP7k!@=lopZ1pOazUXtavc^1GI z=T(Vw2y>@PP+0uFl)kS$QeDx1ptLn7s>&Zpx-O!rpyn5)z%-2=oF4-_;a*vs9YqbK z5W*>$TsF+tE5}Iw911$7t>xyla{G7KY@v0HY2;Aoz^{-hM(N1$T7bh+C8~f`n2w*G zl^c1xue+@%wLhm1n60#0{11L}B8);tLfTA4sesLFZDlCbP=EEBmEnhWc~ z69<8{0*d7jhl~w>jt;YF-|S?uu2M)QuS549#ckEHXe{vqME1M|D2xlo+9}&S0l^23&irhL*f_jPJ+6W@X%S8U?EZ2altY{ptnk9c^3vK$VN`n`FJ|c$SG_0vU$_i3#%j zl-2knF^5+b4>33|XxNWKQGnr!Gg|+`8PwOeF+$!!a0A`k!gjec6m%VF1Y|}XY(tGM z@v$8xdwXCbN#XikEujr4uv>_to!cSNbAu%P0O;E7!XU^+LbEV zO>}%$@l|M1ta?CxhB)bTFAKk8(`k=OZFea@MyWsp!B!1v_}{_S)%pz$kAm{DR*6;P4tMl?ulh zDI?RDNYOm8q%$*t2zfEQ76H}D%hE74OEMccFHE2kUHrFQE#R2!Sy z4>eQvrMuZRV7pj2wkxZNY&Rjc|!B&2NhF`MPco*HHXN0y{4b06~ z5~2?)kUa_}L?EvJ&r8`_ZG0XPr#ml>Q1-|Y;lYgGUQB>x_vJIzRPr*#4N zq)^~Hg|Wb(IDH~0>O}f%(8MEM&ak%(SYFnbFt5xY$|Kt0I9P9@k^@4GxL{!55Y!I` zY1c9tb4e&Stzvfj?3EAtC}|5kV>;9HMIxom`0&+}@G~%m5xta2yKaU$VS<|qT;x-F zV*y2zb!w?z`A-dg6r)Tjo5TeH&`H9DEa4z!Lrn7JTuxw`iUP&*2}8UM?FDskO{Pd% z0(om7N71*YRJVU4c>wf6bwG+_A2%pOdQ_szoBJKy|6NM}0KEQ&Xa8;@{@;h__wD}+ zuKFL?shzEr1cZQ~?wBP>Sgte&(b)iA_cFOIkZ$X$!y_hi*korMa+t|- zZ|XaQumF0_t1F??aHjRA`9LiwzHhDEl+4d;%DWqtu??7C zk6Z35Ki^foA^+GCMc0H_RWG@iI0#S~)dFXViceF3utbyQcHVZpRYO5PZ}sYqs#rW> zEtL>@K~kh%dpHGXV#P5?b>T1hWj-?2!?Et<$hr<^PvpDX#P1@m`Nw}j1+QY#p^#*6 znli-^s!MG|MYWSq&&DXAo&D9VtpKl(!YZz96E28PqE?B7r294g%W!N%fa_KgTmYQz zt>n@GTVQ=(!1Mf=?qK(qM} z_{nF$&+Y^+p*Y}x2pIo`PNnT1nuQL^8>Mg3wY;xLuND8mGd8O!%LwI%yTT_&4ow7> zoas!PcGb+82JJ9wB(j|~9beEuqD`SD%y*(U=co&hB>b^$cLVx2#kd#M*(NL}FA)Pt zTh47)!jS66JVdOXYWh3)F(p}kHyl6%BbE|0*t!e7$(7j?PV=PmnLV&duxdfz4w;vI zE1fQKh`ds_Ro&OO<~22)(+S&drAcC2YU)5({v*T2z6rcP*2|n~ctz$ayaPpK`JfjH za{Qa#YiM^kD#kx+|Mi-_6$~b3&j$?og!pcp^6m9~_Wc#)fHmf}73|~-cg(qKGqgRQ z&=vQTa#(IHav3+m-HMdYu;nvCH@v4d6uuSOE<2`O9`va7Wy+%zt#1G`Hk1e z+ZFPTyE`Ec_uVHhG1*zY@#@ulAv>Pe5Pox8|}Mb$NyMe`_85KecWM8HX|bo6%wPOM3oamg;{@GykJhQ~}|dPYaZ z+KViutUwX3x~j58_A#H=)1M2;r~W$W_Pw(trrM%YZ8$^~Fuiq~PD5(FkLN&%X8$yz zi#_F#I-E_@+(AsHnLHk%qUa@I|1FD9hw;KUXiGc8PKSc(CR(R%h&GN2mBo$8 ztQ9|>;irEJ6SYC8RbzR`^%N><^A}5HghP8x--%__?ofqrr(npal!zy)kR)7~G^^{9 zGfS%PFO5QKq28~8={7qX8#S;zY?thQp9h90lU(dOYKQo%aii$-Pmw`wquEZYtv#Ts zo^UES$zp|fh+Mx_g#{A*2saXtxVx49Md+a1X?A;Oa1;(xGB*NlnD4(&$MbL0jUT4f zJLd024*YwOv;6NaayfewTLF8E|8YKQRQqj_7~s2a)uK};NrB43p)BcWB&Y~dS}Y;m z4s(b>YrEGDgZi(wp#GNEeS7% zae^D_jSC$F3avwMX1@N-JW;lua~YvbhdSL4dMd6-uN;PC`QrE62}!nq zkifTrnHWJue zrXZn%N^J|F>!*&uP>Ok+HT-3A&3PzziwBH}iUAQ$}C7e`g!D9(_|N{5J%=Jf<+vOEfx z8;&OXX-%cxTQ*qlz$|F6=Hr;uTck2kzLExR=Ny$)PsbPxyOSW@tbvKps_E^o>$ewB zQZzWUV}z`_eeLpz!Q(e{q4~xP5Ctq*v7%*5k{1|k`&DNchR#ll*q^<>ZdbxxXYoS$3pTSuwemunYx1P@oPp#7F;SQ3SAyCdAFGtto;w>NOdM}mVE{uRl+t+LI z6kv3DS@wysKknUznO+ZHXm+uBI4YTZh`yEv^z4(pUzENmz5gSdr|P%uk|w`)CiyKi zSc?CJ8sxv)u3z$k{C|P`?Hqq!>`feyd7Z()YzN zAZd)n+fCgStVs~+N#cmvY%soIdIQLI-i)p!dlvLZVdFDok;FE>nqL1V&gn~>M#~?P zNP;T5hlb$(%e#Lc?B>BGQ^Y_Vcc)C-3CH}P_aj~2T8ua_ zLM(+g+wd|yFNwsY9d2{q5w5})3W*=F)h1hPdrubSZknw8+}$j*?|U%TJ4>V#qQZ)= zXN`0;XmXnx9!!G+cN6Qn$2yqxeJpye84rvgBXptw`}wbX)GE3$ zE^Fh0By#!f(g14PM*ydh2uNZeySLFdZ#UVzz1?m*%Av763zK zpa@F5&tACk?#eChcuF`i4Atl0RYH+`b$OYCw1@IK-k@0Ps_&w|wkSpJb;W_TDi8m2 zdr?z+3$cZTrqRf%N*O<*qsg(GY+$8@N|8ABX=X6o5TBr;S!u&O42B* z*N=Y^6pBFz6-NQKOrCp2&L$Ls&vwwZTQ_iFxdoJdw{T&faIY8qd}8m!OX#J*42eVG z=}*p?`d#x!5%fMbGm|186Qmb1(eIYR?$4Ee%Un@KpvXo4gi!;zO@zw;JKm{d3Z0;} z-U32ot*-%&yhK$jJcbWt%}w>sbK00niZxHI@mTC}CdnO3fdYfxVC}@G_A^ZNdku2x zj|lw!NDwPP@;}iNfs@<9fd+X3k3L0@T1X3H%4HBbiWrD}LcnsIK^D+@p zGDC0c)6vb&jvi0CCqS=fjSfA7KXf7w0~Ri`Ko#G(mt*lVkbWTE2QUyN+VVv+3IvlkyW(RFc7WHxM#8Y4~dlqLj3eW~aEf<*2d^%9Z zO3P;yXJm;luDwD`mh50f-)>WhJE}M?e+L=N_8rGyWb$`f&iA+MW$StngH|Hu2J8Ue zm*w$K3K4vi>Pv@2zxFAB2&hjUBGM9U zXEIM2MPY(GtU^Y?Sl0x_>;W~x(5_IL))GUYipVTZXeRC?gJ|tlaYe7k2q^~|x0`_+ z9m6DM?+U$jMRYqmiw&?oL1ohv&k{}0uCEhrgtCfv2)VA4m_&j#6+vM;U5R-X6gAk7 zFTwWwW5@qtd&Khsk!6Y&RK=hLS}jON>7sX|4{ah(UrE#x^QgUw(WFm+v#Y=wylnBb=0wK z+qUhbV{}g5wa&ZO+UFbNTYJwj=Fhu+)IIL1tFEW&s@3XTOA$DF+z@J{(gVwn9>#TU zn9p~zf}=3Wjq2&NkeS!(^fm)NO{gk-juVK^7jTcQHlI;)jZ0T|DpzJSYXAB} zR&F?KTX>J4SrjZ*Ix~CXY}PKWqm_O6XBIt)J1^juGJ2eUmn_PDh616$v^Wl8P3P7Fh> zL{1ySc)(wk7E|{?T)bTh?o70nLyA=N`iy-1`N(Ya&QiBP8bRYI>_JkH@suxTgG2A> zsM|kva-;&TXnQ$b`&jXnQH4%}(Pa#FhLn&Vl+@xF-9 zs|%=upv-K-&Y{L`F1y|6^7eFoem%{Ml_Rf?jyX)rFv_~PU}PAwb;E1{(^Gff)x$&O zy$h~f?|;Ts()W#1-60tF^Ky%?4UntET5Y3N2=?QxrBbmhR~q!|G%=5>%*NMmtEOel z*m;GSW0&th+(mp4s+W;lRZsBp_Vj1$HaiTs<)=LAdNZ5;}(?<_K2J z>(O2_c5u#iugz(0$8d(qI1I9See}1vrgu`sXE=c&+8+}m&nX^<*XvFFj-pb%C$M>( z7q1Jd`G~VRsCBuhRzzHqlvf8^k#^qRclAt(C7TOU^a9Ihg`7a(bacdg6& zd(z!C3+XeEj+z99GVqOWC;|ejD|yFIagP0di>47sfWFd2VtD9UqR^QNbAb{HiWqGY3-V4* z2+)`e1B1OFS}$-cROBsKUjF4zWtY2(BOaKg~;GXIAZsPGK4GGXrc-LgP*ii+X0v+I51rP zW04$i#*03U_wEw|T_}P_BtT{%$ZXidNx=68q$Dtg57bSM)Dr+b@)WxNa9MjhVsC5Q7fFj|0G}%~|Z8-P#zOm)A5$TwEV;17&%z^Pq7)8`jYGVN(o73N zr(t)&LCV;u$oN&5LdiQqF52?f?J~a<32@GwD`On225z3Mcnf%m$(Yw$u;Q&2;o;}V z7KDdKiz{lG;(wPAPo|^%8PMshF2Za;oL-Mqma4K(q)}-d+gm)WQbfB3%#-N{%*XF@ z28M&b1+^2t@k>5yHc*baQ?x3O-}fw<0H7NKLRoqjI>*OoD9|69Xb2n4kRpglE~s9a z&UKW!RT5H}06Ts8M+Al?BLR#xJ+VQ00?21zUNaz%r>C|$l?LPvR!EIzkKWJNcUKEN z=FseG@-?Y)&mueYS$kccvE5(I5Km`FJ}sNmysRJI`QQxsyGf2YhqGS?eW~S1nXB-0 z*?FM8?@z+KSo-Z>(+cs>xb{t<_k+c=c@oi&9lN(I^+N_l;}pMd9#~Z3c2ZA%)(>Aa zKdX}AhUdfFTXfH#w;TFLjxEybx;Hbo;RKtHSozHsw2vDo$jtj4%o_X)ADEpF;rfz9MX z7g%pBL$7lqihqki7s!H$0-KT=0&63sc7`yRe zfBrJSpIOxMC6hgYv)gG3 zhJU(Gh}Nh&8})PP)DU`e8CLN(A3|OLOo{(5E>xm(Lkh)f2Kj!ZN6icHD>V*|lSfiK zs0Q38;ZzwqoD<#2Mw^!gEpq`89hm&&d2J-K%R@3aC6KFNlPQ-mewv@WU7L<9Jdt_l zA5K2#Qt%%mIqKKnA+m2#X-_+hVl;YvT$8ep(xBY?@bPC+Gp*O|zypA?5}DW`ht|Ww z`jL5}(HC*kP!Fq%{l`Al?qME`0xsuNH-(Ti9g! zym#>l1SfVDfQH=}V^4tF{M@;4)JV{1*UthP!*uY8FT}2a_$OSG22aNZZ>KROI|}V> zcCNPs6_t7Isz!~p)FqND8`m0?nky?XDL zMK?a*x76Do)mPodqAXWUF>7R%S7Jy@hy)jUasc5@h-_G#{aIvb7C6Ug;t?NLRR7Iu zV?3lr4Qc@rgrn!SPO05uYr2>(>|9jLGpi^@X@FMqQ2EpUE{D9~+e2_Amk{j8pj_yA z_NI{OMM7Y@R+lSNDxbl#Ot)8)`+NRRaZ;GrSbh1meIb==DMO;~*l0m)7{Z>Pi||2@ zsoJf+!bNn!*Kp*)I!##<@CGktq8x2KQwY8$Ok!GR|`RYi@KvYD{G4Aa^sbGo54 zK-bEQYYFc8C6j(G+p7q{Vo-(ndrxLp&~}2BO}j7#vzq+gg^?9dfPvq)8_p^6`~7dp z)r~&-wg6~Ycv}_+V27K=#iAr{Gv_9BQc!GYa=c<=1(`RhZ|Mj!=~Io^Ge5D`q+_m_ z)n@B>AX3;*9`FhmdUKE;b92-3XK+6z&AaEG3+Z8_gcwpjC@H-H4hN68FdZrKet6Y# zqo^)_9G_uvTqaM4`w_KW@qb{#P?}=hVP>?22&X+8*8>fJP4F{cb)0IV>D~)sPo_1(y`=*` z+z*S*n`8|&X~l-cZcVbWTEu3O3^b1XF5ui z)0&I(bmBYtu9t)f5SPXWkopuZ6v6j1gTe^2c0_>A~PDVnnUk0`~z5#8TvuKy1no7C3%l5|D$ z-L9^A42;ya+d$@oKM@p(LSMTLL~=xJW-oLI&lVPXcZaGEDM7+LSllUtvCRoy*=_M=PLio$0D_2A_(&LARq6YWk10ho&v(ui z`JK@_xd{|z_Gh7Kg0KS^;@AifOOT^Ql30=*o90pz_lf6_K>`Z)u|qg&$Y z5z|HK5Kwz}Q*v%|y4VGcjTsCOE>TWkHDKBHk<(;I@WSEjE8Zl9R3DX~_C~)M(GcujoI8-9N*XiM2k10{b~u1A#m#EKUX471T+3gZ$8- z*wFPSt9OQ44kZrcny{h@9}Q;n`VcI03L$H#;3v=vtWlE-&$`j#BfDOJXrUzqAyN z%5>%EIc9aXX?N}1_VrcTEWzIAf}Y?>X!Y2mjqLYjasPM)v`Pi(f%wu0?~^PeB0jx! zBf*;2OG-UM5Y@&^(?O*r)zd{Ri8E*B5^eh!q>#2F8(MH+@+a7_VxcISEXZC4zIY;# zGG8s7KfL}i+OS65CxZyNA*m#@I$=hFP>V(~~_f>^}Z+jYc=oi`SzIl8nE+{5BSZH&8$Wa|%z&xFBo25_byqv*KRy z!uQpxS1rY3;u7koeolU#=PBYK`p;A)@ya-{ADw3<39~wvuo>Bt&l7R?eDN9cCoFBV@+;HWJSzQ*ru&xg_zvOxf(Ac^)Z18bexTdhbJZtnyG0Mqoq=N07qNf}!d0`Z z0AdGHpGTbpxo)@`-^Bu6h{0m8=*B15>m;C?(Y8!J;tKRMWLZGm@^vbB$#F>mPMg}@ zaq->h$TznYa7D{*!lmI+wtXci6fWJvFjY%E-Iva6<-Id8SR1XLy*o2lz0SUY#&M2U zZQ}6F#cESV8dlwR%b$)eXs!a<=GrtE8!WEO$D%vY@fcO#FFKzw_vu+%I#JpQ>Dh33 zv9_=*b}iTS>;d+;)#F>$4+Mb%%}UnA`Wq}ueWj?ROP|_iA45#zUZ@gzQa@u4na!yeSpMyqdzz?wE^ zPau(QOU-#S*9b8=>+f7=POSNui!9@fib-@)hg}%3omD6P;6ZKTyqxjybE9JUTxMWq zW>?3@`IS6`)Ie;lmx7AVM&7QLXLQH`kE<_{uQ6$RUNnVM)<}EFhZXwON@#g}N(_5W zNw=Svfe-{yp6Hj`ZrWLe_#W>f`q`N?II(QlzH@r~&LZAKrsE2LVbLVx^#AiRRP-D6 zhFx;KyIxEyOC4v`)kg&-O4*>!3I$ctjVaRY zOo-fR2fRizY^o-c-MVfHTdxc9)h24LHk6nroS0sa`m)G$gQfkn4boNs$S_{x@2tyN z<)3`d`mJ28vPf!2w}>T|ENGK0{r$XdcH3P(R`wq^ua&f&-2?`oxt{`Bv1P8m;UO#c zEuE}cDIy*P0k87FbE4|mC!f$bI!-3g7YjtTI!-p{tC_}lXbbIP@}Lr}&yPTJl*j}h zLZ#~xmOnWe=Ce4bf6xV+(@HWh$lYs)(8f416JVS1HJ5A}hdx=%Qh$u4uFkm#4ou|q zx}3l0&A5f{1mvkBH^L}eIu!I+B;TMJh4dhC%s<7E7)i>jLcKU0HdzirTdeT-!l~Ue>P>S564Gwy zU57WMwr;1K8Nv~*pW8QF$21MD9&NyJdB7G?@!kP9sr-HY;$mxntL9zr_ZK&aV071> z-9y`N-b4HjBvSS|TKbJ4(cnssnxRSe0c&yPqBKO6Q!BYbaMQBJ5TQ10c-q!>?6Qd# z9W&Led2U>lyB`S4wOPHjq`8-d?uG=#L`x6tg)b<@Z2Wo)w;c_%6~LfwF)DbUpE0S@ zDs*YGrqUZ@VUXsk;ppRFC$_xGL(}dWafPOjd#~mhU0T1V=hYBt%$pBU>6 z$JgO)nCI-sximP9AN#AEb8CZEA5j$aQiK-62}5Kc&=>PdP5a|4yX3OSUk@S z4IM3hKy59GYm`X}VPu4eM_2aC|KWsK?_Vn7M3wV=QG+C+r)m*Gs4=gNvZzyMzyML|w!7U?YFI|j=9IZkP(py_h9BM}|hNsfHdR|M0uFP1W zsG3vjzlSWNDN8p08=0IXE&A?u5?88mNTEa;TqBu>qK1cZo;vyY0vSWpu&;GSlvOh{ zR+;9~c=%yhG$GiMhPO(=d<1F>{8+K7nw%BJl#Li)tgv2tH#oaFWN{tx_~$Fr!G+(p!@cza~wc{3av| z8IHl*rQwr4&mza7GrHZ)>x6-4l`wkNR*Q=Y1*McFH@*%!8cbsUgWtrGB#|wEPnei5 z12QB!!=`RVtm zYleT;WXKkWNy>2Vt~u<3M6%hP@--roCcdQUre#XUFY(tu!BjYbAJ73Ukju{0J<|13 zaE=BU%I3Ab3wtiJ=fux~`dP?wN&{R!W+TK86&u#|tNKlYUp8EZ(vkG>x$G;lj42xp zyPIRhHkVDcwx`X;#E*Ng&kE|uh#hUSjvuop;*)U~mg#w3foO^Zo7a-55W&JVdxW;Y zv*FD+)E{=-1N^Zggp1jBfgj#50$KC6vN|GBHukbK8tV0s*BU&y@HkM5>d=wMEo2*| zQs|&cl$9`sos2!-RC*w9ZI-9`0bUK*qwS5uQQwSt9e68wa%$ulwx1w3={-6o5;WeU z2Ixdt7}*YH$re4JKK{5g2w;EHR#Uw2vW;;p&2l)gG*}`lX6R)2&8WdO+4N?ejF{8z z=*>+Cs$C7;$YxBvNGrT7 zD{>Hxl>mA=(Bdgo_=dWzMYK0ck{zh<^t>`s#yu_I*o*{WSA%yRMv1Rz&2vzqxd|z( zL6mXLiL2PV%&g{w8b`nKO9}kFJQb~nZ<|wZ)JHS8gQJ8V@@zPkyd=#N8`T%aGi$zV z)0yH$*mnmS+9?)yRcZ(>a8miIE-?SFqtqVY(JOytXr;D4czK|M8`--C*g0Ytr_ta{ zdCkP%AeSFss~m=Ai8deMzedlJQ$dMk%aS5L6>NwQOCC^_ChmMtvqizS{DE^3HbW8c z$IF+p21e0jEa*r%*p>RomPM6% zW8nG4;rDhn3^LJyiV6C!iJLR$KgJ6ZPbo{n1k%Mw-cl%Os9Xso0Y{RP`YBqbJ@`>F z_$ua*1fR!{oI8-ZD|~DngbX|$*44vk1OnbGTV@(24(4na#EobJ;)@sdmGFNk(76!` zs$>hPPPYez4M;B^gh9s^v4l9z zf&Mo{CLnqx#M{L0;IK46mpLOuc4d<=`O7=sh8|qnjD11-S2@|%f2U@vAB#Fpn+${C ziyq5PD^5+@B=O?QWv|XkWq9 z^V_XlDgcux{8FQCbY3c6oJ*%!`~zesa5fZ(NVhrp#tn^M`;H#dMfUnRF}Oz-bSQ;t z+isruwwx(sQ@je~m}0Ej_9mg$MVn;^vY2qT=f5aF(z3fLZv8ca`T)KW`k-;8%Gn9_ zhrZ+d%70L%pz}8VXF$rIBfI={drXFat6bt@9d?MVh>)kf=+$g4TRDw^El}pe`i`CAi->s z|JI&Q@*grS!1Ze>&%b@4!f6+>5miP&Ez|M7i} zKbG0Hp9xOPEj!Q{7i|o<%|sFsF?$2gzZgrWk{$=`kc8UvNxLdO`Tcso8~YU23xZ9c zJdVr`kB>3yVzHHLG9MzhC65t9!mpL>_`&yUnj3GYa>#&T6YF;m@Qr1KKT`0R>UU7n zlQh%=;Q{XN<6K)r`*opmJrJS(q4=hcjW;63+x4bB-ZEgYi9d#TwUZi}GmXY1*F#J) z^4044DCs!D`*_O+4|H(4xYx2D8ZzVE*?Q3Y#pwe0hmv{Q_k6i!XR1*H-I#~-R zXMioh@gFYCR+DpDV?yeD)3D}FheJ=z>2nVCj<-~NyKB9#SajTScTNv=8E$T(mB+F|Ls1Ke_&fA)CGHA)G0>x zhTKwo(u8YLw#Uvp1p9TR)(nbI( zZoNZh%whT%Qfb^V3@M!sfukAmJV6HAqV{MH7gb5AR66$p6fE4B2Gd8D;(S#5X~0kR+-E{=1F9DmnZv)Xw|_e6laa?+~W3_`PG?dyb!9_<*R87 zy+-Xwn}zDvaX*=2#r3sYTbb@)9i>>@3LU}xx~>C)tzas>W=aD-i#3eRVkNwD^Lb4y z!hJjr5G~l7Sg0Y|vi0FLt5LI`tZ{(og>~|3(nNSXG15}mYP^iltgd9&T6d*U`weKT zATva1=DwZy<-Ib>7Gw@N^PD<8h}ZiPc4_@q;zBnke8uDQVKn)9ifB~X5UTqFyaY%L z!NI#~xAx2}^Bv;vmjeXU_%#yr^;rJv_n(56jhzX=`oDq}5Kw5G?|(Z_@nz}%Zzk!V z=S`gL{>ui`KWuQfT+oDo0|7;Ul_>vdv)R9F{2ly6o$dbFSv&c)+3f4h{+X)2IhK?5 z6Z~b-00&=szLNK5n*h4O<9dXtt^Lq_os||G$7LuBC|}@kAXqTfnrxG zHlY(UA^M@54it+*$_NRXVyc|34g}Rd!O@n|M}?EAV_6Y6PHMU#-EWj;S6?urf)_SW zRzUe4ydpW6Q);L@0T)p;ZlPNY`+@||O<0yJKHI3Y)QgC$k#Y-$(@$!7(u+m#t9A*& zL4lJ^x>QR-!njRU zw1d6Pc3=+)@!xhCbVVfc7A50oE@CL&6NI7j&`No%d>|(yyPMaPmXkQYyo<(|kSF&Z zNSS`aoh2@N(~(A`AVL4&ZiE(1+U2nGbT9!|XN4XgaPwGqFbeNdC_L)FXvO?nu%AOcOz5{)Dw9H|pj5%FW6Lr@ca7 zqLcARR8=JQ&8`|S2Ps;`j{IOQxVL77s!z%pqi#pPlR)uy=5j4(p3N@0E;jZhA(IDf zmJf*7pAFmdCzQ&v$%1f3np8Ly*D#HHPi4s~cCVDw&i7^m@Q6iZ%Bv?1Cf(y6QW2N zXG8gbS+2a+zguau_4;vR|WP$-M zpGT`QF)X))XoCex2>(Hd3!PR*!?*myn49D5TQP|Y{d04Dh`$eYwXQX(ED`OT_k$m< z!V|K}kB;3B+D+|(H>LiS8u+UpTk}C30qBRknkvTgByV^H^#v%&kkNv}5rE?HRV)Z& za;i^Y8GB;Xsl>QGlfegOtE>ziOQzk*lJzHiRw1zoDglw#)$Qe?4wZyGgoHUtNUU$l z$%1o{ncQnQX!LpX1J5*+`(u51un4@YN zkQ(^CNzH%cnzf8CE(vxk)`ED8{_u?Fkm0NAJaed$aW>S}xK~}2PtYJRqxZBu`0k8( za*|lcN#!aiYX&4=Hsa-&zJW9LN31exmiJijIG_AXzQ$LAw>XpQz@>-MD*m+FVan&H z{F5?Vj+#D#xa8_?I77Je!kmTpU{m?pxBUBv_Ohyq?j01L#wAiZHdgyznkxk=Z1$Sr zjc7=YxvN#{Z-98r)_t_4Mws^Yr!bq{R=g6V^RU!2fI!HE^sO#Qk7SVo%XfY_HA$a4 zy)hlMh%m}A4i#{jM`sU9&C7>2<9*wjcpTdFKfvA$3Q(#2`eA7>`gTunYYc@4yp6}PPbsC=cd%4;s!3sA#&;QoB zHfsOhJJ-JY&i)74;S0llaZdl1AQHZkR1POiHR=~Cp7{Fn{cp_tgS9d=c9u7Ec5wv! z1KGL6P0N3Qy|E(exb89bSTqsRrhTPJK<9|m-G1DDXO{DENL&H5o*VRYBkhy~fT z#ZQmiKX7v_o$}=U1m!VeA%|Ys5 zuOZPHU`IZE z_}~wf?ov^S27n#&fh4i0gd&Vemj_?6Q7bHa#>$ETp)-e$VRh9@eVXFKhU036-4E)f z;(NFjH{tJts%a*G8Xec*lq!jFEQh-hYqI3RB=XFolD&oH` zquiZTSA03^SnF%hkLjy(ljnbP848YOhPMA#1n!>|jr#Uq=*V~ayWuTZc$>9yZkZI* z3PrJNr8bX^mk`Cbo^oTesq_^2k8L+s;q{vW<4D02pKYH%SKh~Dn5UI#pr|8GW-3-( zfd}cydE8j4Ej)54$%jkCZ`8ISBE3%bAv8#`FdaT^O_rH*mo}R2LyX`O$-v!dlQ2C$ z1+nJu8z62<*t%P>;YV*M)HS8ww0~08wWzp@5^vUuYSoluDmJ7RY ztq&)ov~q5KQBl{Dj@5PNT*6Iju4QtlH70nwd%Qe~7P;V1{FhDz*y3-EpZsh;A}D^N zLr15au<0Knl~<$tIQ~ijGcpXJ_Rwa|fvv|`hXtL)MjsBu`WDK}ll$Q3ZI@g#t)4D4 zF~akK3(c%=MH6O_4y`BXGe3U(imz^KefgdG2Rup+2QKzl4szQTio&WXs~PL#<$Dx+ zz%)1h(BR^)0^8=d;@Lg)VF&7V3XlC6$p#44VWi(u%~el0dMDC--z#_5S*qzmtjypr z6;)ozXdeMoDRs=TCCnOz(#Ni7JfC)s*^6N%WSXV6nZ!{FKFq=CEHqSb#kXO0`$O_w zI}P}kJ*M1?W6Tj=JQ1xh?3k_M{mTWCGQ*k_)3qI^u5J=4y%LTRl9_sXq#JS?y^z6M z5|VDKGWpGvl)UL6wuoZ3uxLEeJi(>G-6TI8Vf_<7c{=A(uPQs)cWb%Wynx>#rg+mZ zU<~)NU(K?UucKAGMu<_R2@|HEHr!ZXtq@Ls`mgH{A@r&7BDd`46r@TsfZKleIoHWn z5e{C~TyOld*i(l8dcU^D2!^KYF!$zRdaWTAD<{409aeMQgz)Mow9TITPJ_t(6j$yb*8uK*g7ux;B%gf?{j zhIYK>`~d7)pi)exoe^<0$K+XTf&o>8(|oZB>Eh>{xhj>>aQNQ!`!)dmPRkl8c)Egh z?qtF^d9k=pFLaI8uR|`uw>ni2G2GV5-W;F9LCUVj4w+<$r=q|53cvgY{Eskk`3H)#;@?!jXu0<#VZqzH5}gq}=P5=`jXJP^k3{Ae?h+<5aw?G;p;s(hY zu@IZ`E}}%Z0UM)VP4%hjZ`b&|(9mSMYj-s=7wN2KrZJQhKmQ=MfeSZgO76m5Swd}S zf+4l`u+PWx4;wm=6jE%TBGRzCpvFZ|!*4)7m1$gvljuhhUJ*q=IF^S7($cGZhxIVO zWdJAvH69`_-rHY_FVsl0xGgNwLAcR)Q&b7p7ZEj9Pe&z3Oa~wgxwN^GE zC7VS_H1n2`J<3A^BP*us(>6Z~V=k`N-FGY(M5c=#LPt?X*S5m|5@)dr=?<6+oZ;;S zK`E3P@HPn{;0w2(Ho$KACg&0k#7al8f7OS09dP=c=*uaksP`88WlOY-CxocGf^vTB>RNP;(c%ZyF}IhBp_B51F$4Mk3Bo=(U(A z0#&x0ke)e8cx#S;#mcALUrNC|>^itO)GHcb-^6>Gc?O0#tDjyN8wL!(vk{>*6b=ha zbcQTMWmDmTJiq-~jleTd16rWg;MVVUpOvih$3CWVDJGrSF(#Z%0`mund)@>i7uP9- zzgoIXcro7~-=_G@@WPYnDzCyAqD8(kF};G5zTh!Q!D_ec804zuhPdC=*7a?%^Jv95 zvCF6%=ohkofTWv9k!j}y51JiFu1~b;Z3Q4}4q19Te%<4yf`jlJ>&TC$F zqiB_rh6sIpwo>Kn(>R=jt~yXq)iN1NJGBy>4b-nc-v%{!R*kwI8ISba-H!M z+bk|`3^ix9$0f=TIt;7{Q{mJ?BSvSrglFbH&c$BJB+*z8cD9AqDxDfOOhrzW!28`} zAD{Y*LtZJ|%dMM*w>X9Aguda7f6nk{)u+*D%6HNowVsk0m8%flWpBvrm_r+}aDsTE z8HsNoZjDTSq+&pStEnHRt*^t(3$?g!CR_BqJ!(3HTi364mVk5zt=?DhhTH+ zSK_Y0>1y(?_fXDMrfFZH!_tWNKm$KcsBcOB(9OK$#7-D%fZNga<|dnTv6oWq3yrD6 zy{^sE%`xX{|aiQUs*{71m%zA7mQ^7>N)-2XC)^m z3tKZ;JF|a;vr4t^vGYuDe%HUk9~5zC24g6`ur_7<$K1HtU4LuC{z*<5S+L1 zDJ`BLt0dw}RLN^U-d1a~O%#4tH-HJtwP>)UlCUWTC)4?d5y7!?CK4Ge>eGmH%~Y#m z<6H?@6vKz(_1zXOI%?^SSF79C3CzsY!1N)mS33YC(#2W`y|wR@KZ5e+B7iiJ z-6}->k?mBM43j>d@FM|@rJ5ElVntHOiZ%!o#@Sq{GyziGpQ3>wjaXc12pS!ndZ+CX zPL}Aq7<^)%(1k!z z{+3u04Z&J(KPI+2cy>Qx`_+xKO#b&?hhiM?<^qFPm?&U$*?LoNMpCu2bhNb+9!?Z7 zt@^X}?u?r|IGMv7BrGgvL7j0`MT&U{MXG25MpJDkvwkjOiT0!JK;DRhXV zHuD1}uR#QJ(Td<2qdBIof<04m0Vpin-UD6}cMAzfebHQj=#Ev~(SeD}Ex^eWiEfL1 z?J`V-;}#Y^Z{x$dSW|&#So#r>v~hvQ<3R#1>siO`_W}>dL$O2AN$$r&HEZf+`=i~{ z0$zTu7P!zq6ba)0wA2$ZR13=KpS^OI3WAR+~U<%ZQuVP`N`!f34 z>B_cNo#%H&Cx$!QHB=m<1oO$skn8ctEH(m?4q{)G5Mf{3n6(~d@KP@}Z4L_}2_kMu zbtG!j7&ERq&caVmMV)e;>z+>BN8H3`1H?Rjs5O3qU*{`bq+rPr0SLIi!)&duNY=Nw z46GElbY&g97&1n9D%Qss_O-Hf2c0rMT)(B=3+anRnh;d-HE8`cV#D%VkoP=tOr0== z0AS{$q;U&ZnK(_bTJ@TLeC`}SVQ+@vt$W`;G=3OXG}tpT@jTg@KXXva)NMib?Qj>6M7Jc9DkN~3-hE+{P~2rbd0)+azs^0qXLAGl zuP2OM?5o4Awzxm}RRubHvGhX!n@aF6A&M_p`4=Afn==2$Lmt#V{Y%N~&-O2-4`6ya zYR+)*20(k3On^;maNb{C<6I6kUtQxJysWNM9Q-LFs=}EAthvKZYsN-re#I){CAUW8 zKG+_i;QCac6nq%iwis{G5QxU5a4(_)Du5D9)}mGpy1R5YUjbZn7QFH4B1C1mPHaL} zbftbi9I-jYen4R-W$y|hgOMxL{MQyJrVIEkD6)9~^3eipX}ZVLOT(}?fpYD4Y&JfV zt=jw26X2<~T4KJ%DE?eoHrA0A={PKgjw)qm2e`;;Y@OdPi{-=onu5pgFA&@cIUOen zq~#jlFyHIT|78F}O=gTh#(z(>*oi9<%E}0vnhU~{qn1Kt%C=0$Kg#qifPmf7Iil>7u+7pH z66GQys=lzY948ITbN_~C2{Xn3>nU)GM=&=sWx9`|(M;yNG5e0qxt-1^yf{F!sjJC*oo+Y~Bb&n!+WFYVs zIh&0-!&ZD#CNm6P>g|O)LPltM;(PnW5q5R1nX8(My5?-ik6trD0ol`n}6b zg{co7S!@bX%~_?ED!>HeB)qD|s^-iD?yQ^BlPtU<%XP4?{euL$cD&*mBT=u*xzv?I z;(dkM2i=Mr3#RA%!TsNPdkM&@2^A`6-#k!A@B-PxG~AyP+(AB1nD@+P+l>~7kG*6W zjViOQ+EJ7rD1b@5R$reWYJKY3A&k33{kq;{Ph6l~?I7Wil?r-1Bxm!-zQWL`QmQZY@qfGE%G7CPyc@9Bwd%d6c|hdMPretjI`=f zZ(2hF9dz6@^mR$D>N&*F+POu}=4+=qS^|cF&%xY?>EVYlo0o@ca%ni+J!Gi}b{=p* z2Y$yN>g^jCG_XWb_ghi3dFEW_O*eBSpp& zB0b@@%d!Q&2P>&llWCl*w}APj-JwVFrpn9V22!gGI0EyzQZZC& zX-IZ!Z6JS0tM%jsD8ho9KrOh4uGd@)^{3gG2SrsVhSEcr$g)>Y-$%%y3K8dH%mhMF z$GVoVfGbSbRn*(FWRM|O*kYn@r=m>RA{&l^jwcokErOa7!qYyxuj);F<29rbkEhHRcu_>XV+aD^yh(*=P3| zJ#P2DH=g1pRy@S|XUsW2%~z`jCVPI4j?Pdyi1slD6O~{SlyL(j%OJR~F!xxO@~Tb< z@#`A*g9gfSS~p`)1SSF(Z~8^9oh`PzcBWn~`xP`2!bD(2qlTO%Ed%M{jRBBd6Y}RJ z+aTH<2y}K2zefOqt=D&ZVg-^+2)NY7P0%k zxgJw#yb!KFci8=U!f>Su^CZ7&h~Jk@wL^_%%x2Eb7PTVFnKPUFWscRb`m_m+P8#eWMg`r>hh){s)rDx3}DF= zB{+|RRFb=oC#<@u^#g1;7O^AF-dxcatrGe}<^tN7y(v@_M}_(h#U)RL03WrpHSH8Z z(hp2E`#AB>hK4H0^Lf}fBt{Y3z0>1O>D3H!ne+q?Erij)jz){@Ec0#&iAFKWrF86f>UG?5rI;pQTBW%J2fo@x9I}APu-ilLH2wC&{a?3%*V0x zBNC(-X6OTfu5gGBuosNDOeG<`hkoJ7iq-V#`^R_)5+H@*>dC)S;=e}e21i>0_xXYh z2@A8S8EL!=Ox;=q3Q(&-Y-19s3hpPpUh4;to&nrsyGQNHBr zo=XB6s2$#6Jj;vKd#zh17$FM>#jv1J8O2*q6tR*E55gvR6%T$5DZA3@QJkIWGqx0R z!`Ys5OZItZTP^Ea7LYbzIG?;Z5^yKKBMKxkofnueXUrt8)*x1)4;RMDCJaCz9FXg3 z0}ym5fZAW?>;9=IDPCP#@%;KOCn!0k%% z2$SBu-?GO2q)GQe)%OmeO;fk#K&KxShs6olF$uQ7yHFneXTy0&1|L--r~nG%!4&tn zE3ngIBVyCq&WZ~18yd5U#xA9tXwlTQ=;p$3|Gu|tjO&HDUP1~d+4!_m4NF@2!y|D z&{iEA4ovuz8r9)Jb7;mykWkbGpgpT7l+~Qv$c9bpYSDdMvXi~24S#3{|n+MWE zl${=vid5tws^%ZNjmdtf)UPLV#o6-OUj}&D7>?j4>9p*QQ&pUFt56O1ar-dsr1=uM z&*02;iohjzrb^Mp*hXezEE;zCOGNvR;ClgE+^&AG8meQ;_n7}RG7fbRX;I<(jO_ia zvTfi20J#61k@3gAP)Oh0%3V(1&{E&@A8>A!>W0IAtg%lhnYj2vIGDORoj8Dp3X25& z$Y&=KMC(>^RhmnG95%8(Jyq|q6$KfeBn>Vuuh+H;l2+w^*!BQ<i)NW zn(!3GKk9XAl*=HcL(VV$qWoo8qd@}DJgDWN46CHbU6s#XIMB}+jR;V^DQ{jNV?|I( zuPPveq1GlHS*s{Vcl$X4)WZV*-0#Ov3Fw$#5no@8a^=NAY#o-sDKSc2A zW@XVS5@KNxP;*SV5#e#Onh2CYi(ARsMroA%cu@wBLGFHoPCA}f;UPMRoGk=u`7jumue{A@!a=4!uw%<1h% zeTGj#8lv(xIFG#HlsFK?MBJykbS{W38`xUCBR<6Bkwuwj)UQ&;?#+3HBeBaZt|ON6 zEL=5Xx0ZLg$w2Fw)rx&h4aivM#DWkIU6Xoc1*a`{wuhoT!*+&RGTYjFC`^DV7h;(zu`9yjw+GUp2`ac)1Q#3!rYR_=B^uf z9l_fyqUhGVD_=_zKY>+;*Je+$A|WF1$ZL-75+D1LW<}hhO;IF?hk=eDn%i-I&WoXG zo53V_7B^EZYC&@m-*<{udmIKEt4}$_U+tplo;*<4=5z3JW(Zx#E`;m`PiLv!dt5ry z=sd-;Qa;ose~M-CK<8+8c}O=itruTW%%}(#HN6RY4d?wS0%Kbj?GQSBnkevMdIlD_ zFp4)Y$GFV}Zx#U4wJjbdu!#DGw;Qpu6UHikzKg*-#h@pIG5>`#R2^v=7+bV`ks#26 z_gl8r-Y>w`B>sR>sUF{OZFeEHqTtg(aR5H+DMRg@EqmIes82S}=<*j23m$I0x%1i}`uFba?Rho4I&roC*Y2tUDdk<%2x3F@5u zPJAX55(|qm`mA$*0=Z^B%(WugvkVmRpm<8!3Htyr(b44Q=@xcSo*J0FHP;jA@ieiMO?o}-}NvMd&^YyGdJm{UINz&e31%eIf6abK8}df zAcI*^mDhvW*Ry@oq)!A5|KvhpP)+)tE#4Yp?veAnL>Lcg8-$IQg(U%%4LHPLftlU5tM|J5uUSvI`*7vjZ#Fqb^0i&c1ka9p_`9QWT^xBV}~ zaz&>f;?Y0Ysm_07ZTsnUvG@W6!;2Wc? zG-uHZHP*+ScN2t~pcsUJ^0CCQD}0We8V<*hn2P()VN`?vu*5iB%WoejP)3y%8t zB$;H?i0awm_6gJOhME?E&}(J*lg$(nOR-SFqPLPHfZQ)OG?NakJ~#nX?HkjlRA~uY z%jkl}0F!3cUR+7Gmlw#yr)QH=H4HaF$6;NZpcFjvi!Y2j!&rX8qZh<0AN?6MIc{<8 z>+;Iyz$X5Pq9hhv{$$zT_NJWmnNK_zb-yZh{j&9RosmmIT{?8e8JWBM_NrHvK4?i8 z9fGickp!nZ-W&bBfGzVilTSxPsNW_0-m0|uMy#awoLPkG7 zFaD2cK>Yvk_xP`e{KMGyUvXZAe}2V35MQM}hjqUHFkWYB3o_!!fXkhV^D{ps@)uyp zdM2?#w>9Y8*kTUfIO|b`0*N|cz<&Z9j?aF4BvLng3}Xax62L?*(_)JV#;>Y$8c)2! zi*u}sst}%3mP*Ur6jzNv*cj7@pv@d0%W7!*;#rN9K5hsXR9%(EF*fP97>F6%&CE`8 zN22FIg2SNn8bXaenA+$&tpV)yE15J5W1uoXBtv&aacbrQ)|O&K z_dk3j=?hC2Q(ezDEk@8>MSmIrn*_g|&m6eH1Mo=(x*KTxV2G{omYK-9?sU);CU7Z6 z;*MZ`w|e_#h%4?tiL>*csP7neA0pTMZQ2<(x9l|UGgNVL;U}e5OVLGorjR-YdPM1w zir&9U`ID>V>Mor$9=1oh!^u|JI9C4)P}4YDmp2uOS}21uO(FSV01CT3#7s5o@vuO9xglG3hwO=O0{^wwYQ^k*-DQJBah!Rf@L;8o*UbsuR>(S%!$%k4X{_k%s@1?z^3 z4P`c1z~JiNA0Q(Ix6Q1rUtZ7ycf17esDT%?C%_(iXf!>6-$J59sNKJ`q}oMEZBqx# zmX2m0|2)NNRu1Rv{_}Z&0I2!#lVMHeFzNfLR^C6|p5VWImH&b6{kKb;q$q8>&i5Y* zm?$>D2$q+5f?qC4N2pEy&#tSW}r) zw`=X9dBoy**7(xoHtaX*dhR*Cez$UCGGF_i#0xD8YnQmpK0!}Yz4R9wgU!l1WV&C1aSiD4q{LRnk(%@6aUqdVcOPbIqhzoH~nY)Z5EkFO*B%>pC zO&1UTD$EMFh0`zNepGpC3+?usoksdA+^hNlVg16dI4V_kB5kFtZ6~|pam3W_-e5O+ zu%JsWQWL@t`G{%4#4h?E@xCR$#2oxpe+{^z4EvY6I+KguYk-H1IMQ<#^zs{|R7bbn z+@wlBG|;(T3I%g?PqV%K3@$s6wVc0F)R^XZ{pXO^85F>8L!3KbPGj^!j}WVprwShNiUf_QNOJzG zB0`k^k_=6V)0rdcG{zHoT5u#UOf5@A4p;eG?E=q$#yK_YRW1*O0F^NtObIK2HU|L4-Nu(?455 z@{YIxtiQ-7mcpfigEe!8SdooAWQBphveg!*Rf&0E0RBt-E4^&2a0i%~<76fW|0G;K zdQw+Bw5G z4L_DW(`lZ=0gU=w4{efe0M$8)DNAXJbgAC*RJTNxIe(L8y*fhLq-Q-{H$sem2VJz@ z#M_!O6+Cl)`ftIps~s9wXIDX}#HW=3mZP#j6{xv_K2MEl#FLXL<6{<1^_9|62eQsAF7 zx2Ai_@VF*jFGrEc%84CyOcf&}wpTI`Ak)9twtAPEzj>q8V}G^YRPh5UwL3iNDP~)$ zCydTaLe%?flZZ9yqtvJgX(519X!O;LjBgJ@*IB=TkM2mXcJyYM@1^IljLkzsE`)X| zNoqs=dF0DNOo{TZF~{N8dUwkdzz>3Ex=d`dn|xaV@sdVxhri}tT!~g=IOSi};VgJ@ z>NaP=^3-!386<=g9WZs)|5dER-7Ly-S2-^M8zd7}<2ovx#swqF?J0!Q2`JD-_i#<= zuye+u;lV!%x8e0zPtge(b$pXO?KCA)uT>yy!6wGAM#FG@v>^)ZV7glq7N7>L21(bi z->l|(6Ho0tm@SxjB<6_F93;LewGVavXJrO#LmOM8RfJXDpkZTR@QQDm`;p;AL6pR8 zSqBr7A5I0Hl>rsMN(MgYCI<5hC!oX1^n8SSGHawehx8a5t>G0n06jf>xBOM@-A_yUH)K{sEV2Nh}dwVHy>2JDKs-RdCRf#aQ%LRzcw7t64mQ^r|qw(ST(pb8mz=Ci%IqRE zFCKL0^&o1~$$>-`<$Xe^;?z23_@@e)GxMlXe%19!fS_!KEA~_s`N?JawBd1(pVR?> z<1jiOMgRuGkZCe-C-1m#92Rce%D~8hI*dTe(RhJPr-lzPsssj5AH8_mHd(?}^9w}P z!C(DpaBdP8>fZVa_DG`&c(OFiA*$^J{gjAWo$>L#lSdA@1pf24$O~_9iFbWsw}a_q zbm6m#ea->E)L(As@U1?5O%d?is zqFL9vfKoMIt(d4IR?bWw>h2;M$(t!AUa;@I*p|_#|Lpi~6HXB_+x+PWy8yMC7I-CQ zz6Vit@|kgIC(h^pR`->%dFRWI_V+ld2~pz|%^e zd)y2dOKKpM{U*m9+6-EH*Sha`HOIa?dC5h$0=8;*>*Oqx{js+q`VaE+IrxQmVz&l1m5*zqeF#}2E?Qd2lr)M=$ zt8#&Ay<_Stx#gO(r%4pl&sXg=6`#g^rXuaMz)+}+B z)-+=(3Fxue?6tydDMFHh1zKlw(lYKOxy6}=(C(!wWP9;gB-?PV*;-rd1kv_^u?PDH zuM1z4*O!Lg<=g$p%f`R46q@lVW}#zhw3cl3@m zB&WRUHw^Ydo>yrzGzc9d?VqwnmZzt?ySql+nu9(XZFUAusBmrBxMp|SkLk114?f;9 za&3|?jVZUQSX?PC*NRsp#55W_y@=C-s?`LnM$wENbE+={&3JVOeLRx=gyk&uN>~Sf z<1c-LvD1Qq0L^-aNgIhLb@;Vap>sH%yj8umoG+By%9oiJE8$LfP#xA~%PLrEtl1_i zDk8B?C~TA!9!_4ZU~D|e4}0m&^4{_?^%_nobZbl~eK1dW&Kd^*#k>!BP*|TNpl1xg zXc20Y=r-!ZTXIv@jFdW+mGmrG@VgoDAw};g@~WK&Y$5|j4w47r>PkKZ$6J4XBQS~0 z5O&T3kZzyk5ozw~grOjQ`tjskXf(8_OVc#DXn>!ulcsgiOk-C+--sC7E)^Rec&}R)r~AO62S#g*5JPDUsB6GH6yu*Qh@u zZXCE&QQFl?h3V}as#;QpLkyYXPeakE`qw9!)>(VqQ)_nIoSGln{x5Uu^RE`nn}CG= z2_EaY0_Ml;?E{2SUM4G()X=h9I4j5ZcI=}tDFH>I&D%XU9!6(l1)|7qAea~MJ6jZJ zA2^9o{d#A(I}B6u9Yi+F>6LWhN_Inud;QqKW6*7F{7YIKGm~ z=OQ4~b*BW2v(Z*~Ls&^*Cxp>|u$^7Yz6w99M>lyNi3g27mn_=rj-sNRNe@3TpC%Y@ zwIu;<)$?<6#~_eJ@gt}NopIV@N0lUnmH{L`gjw*twD-LuejPWi_$^7*2*(g$mv(HV z_B+=DCmW4@rEd!T5mROAI3-ai9vw8$zH zbDOsVrICHJgdv9s9oWo^;rvl^Rf(+yIya>D2D#K(&TqWQ;tQ@4wp3IR9e4IOKvQv#p>T1Q-s?|C+()#?;%BKh z3pbp@(f85aVgWdq*FQV^J7`F0s)|bLjTxbp4>UihDYwq6KUDw{232wZ?{ut|<#HD5;52G*)qF%D%c>S94UUwYq=v?k{wGl6{N+_OU}B_Ff(7L6u`9Gfj`(c(gK zDY;sLn0ZM~LC{JvkDdxzel?|7P;#zJ$*R}F5!hJLsKIR~G1p6bnURZ7+6-m!nHdGV}8?dU%rr+A(E?%sn~_vg3LpuCV4WbS$olI}1S7M0$pNGQLQxNNoPib{oHh zzJ}a*LsfomERL0*;}f&mbPze2z3HHPgS6C3oif zO>q@aYI)j=Nt`s3K=NGdzOgB9Ry?2|YI~ekozmQ>mYsyTU6U)I_u%kv`iQg^#4Kdw zJ=~^ncIxy~%Rb`u%=4KWv)?!&&MUKH_WIk#tY@1cZdZyxDl2Y$*&QloQnIXK!wO}H ztC={jmaQ-jro+;8T#dNuSb7FXzk*4U$B!&HvHwMCdVJRExLh&&}Clj15r zZOyG1P9`)9$|i>4f6p8D&X-qAFs7^uB(BA6 zxudoadf3g40jhFfhKHNip#!o<*(ImCk8n$0Y7jFw&CaaPIfl%s_radW{FRJB(Cv{a zKXKw-Bpt)rbBg{>7=g&T1A4XV$5OV}#oI#e@Tfa>gowU%a)(;xF6d+qzHB}G%&U5dZ@|TTEV?wLo`-<%?5ETt55~KZ_fJP zxNYkWM|sj%;>pfl$w|oJvSR@R`L`0csV8c4Ad(c_^{EE=NZ~2yQ%hTYe#VqEN`Z)f zPptfRW)DHY$`ZADry_n0m3D4SCENuSqXjWE^&w+Q$d)56N+qj*wkO{10C@{cZakOlmv4^xWzJuLbe88>65Wa6X83a}ms(HjJ7QHhd3BPz9&_%$o zN;2}BoiN^toB$I4GoleT1oI8Xp`%z4mVy#ozm-+q)UL?IAj|W%l?tEl=jZFJFkvoV z;HuD5W2$#?oq(7MtCvkOl{-EjKxy%r9w#CfV?q2eB(kSnA8lzKAiRjP1%NCw8V25c z+ROFZ*b&7qGlOIbQJGA-7{~@iIrSfc0A)?$Se=piDZP4ltHEE~WxXkAW%Ip--6jO9 zlKVc4DVB)tZ?jHj5kEV=3oZk20o^G6?vC=8yf^CDhAPZk6U;R$ra5m=X)z2`@*s6= zyIvoCCCwN)@{3U5!)L~eD#f=chA?gA;2n=0Pe~U-OOT=ptD5oAGqR$_dtW->YUV+Jw{w(i zr$Lm^VhLQ^Qk_8Z(&(I#D){yFbP1&JVH_MHyXET?oQZZEAcEjF@U&!Mb-&$lYrUom z@=VP&f6R7#M5ZZWaHIY(1;@CK$fTSG$8tI?N3dL!m?^za=o*hVBSLX17c?L0J()7! z@aD;Z7*UmoNJd-QmF$V3inf8YXr9wC!VuuBb{&;sgkK6|AdrxmDf^Gy*-P zt4_(84TJHxe_1%NW%2mFH3o9?4U)Cq`RD4rqIn3~2pzLD{Om>y0bodJC?^=tgiWn`&JG`J7u#pNys#znma z{R9U1MGhSk*MzcZ&E|jvmqh%YO!|WqcWY}wmqeem(=Wb)IXsB_0Qk~V zsmw@fVq|r&y4k%L+B-C5bpOcT{Y_}k^-)PpCgTh>+ID)Te}{u)9-JdIb`m6h>HrVb zrtkDayXJod*F58b&X+VZi&KG}!)BUvm*F4U5u5X#p>o-{w}{px5LEXg6B?34!uE+g z8#h98wmaKWK2~JnqG*t{zo^vv+IssjOVA?dXPoTjL~IdyoGUn?m14UIXLAPqD~WN5 z`{r%4P^uCZ*I?eZ?!WNKf*Gznl@3l;lsp?At#4+Y-d7CS>^C4}gB9dGGWJ5>AyF-Y z-XSVV4uKCGH}@>?7Sof`yRO?{peD7pdQDzT?YeR56F~y>(ZLZae%HZ0T%Ej^_Z`8x zo1*DZq}!52nZOM9DZP+fapz(;wG6D=?po=jc31*m_eK|nP7BU}EMfl?1nlKW{WR-n zmh9un980qc70nWF+!|Ybo^x^bP^J}S|wLCeXhWH-uqhJkm|JZTycd=~r zUrT>~fZmY%%|m2&meIA8fMWo&=q69F**0L;3R`RW3r)=9hu_`>l7lc>BO~JX zXKM{q2@Yrg#V<#hIBn1YBbnr@S{vFGETmqy<>~Wzx`BM%93m}}-OY+$nLb7Bc&|)B zx}@tdi;K|BaNI}`&|6SE7_7$@%?A0$q_TG5S7=PS!a7QfWK5_!P{pO}8m#;I+Kd?{ zlHAA@zxrnM2|P*Mf5G@Hq|{YEkoP3Dqk>9&x1T;1wQu&8oJ^u=ST`5>GK&1g=d&y? zgqgK&T-1G5L{#)9!|whDSV`hU#q~nl?)ZKzdnGB=bbob+gb)6a zN#qG}_Mm|jP4HU$0Hu>`vgC;jQap9uZ#)6#N^@`Olw4@(FU!K)V8J6uS%WImUbKRI2(*JgK zD`R&}2sY`xg12KDZuBUevpiXso_Eqt!sY>bw)-5#3LUuEP^0r zRbhsJ$2Thd!}~`RZEmQJ`s+F^^Uxk+Eu`_V|KMmlv*nHsH~og_kC{x4KAMOQohIUp z4`j57hEs_sHC4LuTbv}xUyTIHefoAjS0XK@;G}1D@^6d;WLa{QX(NI^iE ze!IG#O*k~Y$pWjbQJiM*H#|#W2GP#+84sg`kvh1hsYKy~*d54Gs_aJNv;55L9^Pd+c4=KTnw-tXO#|M+flFg0?3vC``DGy~aWQQ5x? zG(agy1f6b2s5MEc=cgT|6@iB&NRBs=5M90OmCqN<9#U*w`0m|2^EzdPft&~(`Bs!4 zID#ufksBV8gK&9^c81TiTKn?j%XG16bSzI(S=WQF+P7mmt`g01fkqno;*;sriNaS^F=W^@|+A5Z7zQQ zDt6Dp!$>l;=0oT_FFSI?j^S+c-h{_xHCVIHHa94~>-w|u{d@>c=vNpuiakF?MT06d zR6Pwg+$@&ie*sk+b8&HwuH>>svcw6J(Dd@cJ+UNWb3AMEUV1 zPQJtPk`3$9MWm+86>5b27#M!k?@hqtkW;F_|cg}>Osg@_n|GYUP zqW{36p=pPrELFj_g8f8C()(Ig4Y-Y%19~M+Hl`BYgM^w z2X_&r8O^)IqyXxI;^an9`GW9Cwan{=x^7-XGa09u6P~Mco#%3KGj~AT<0kgn$gu+C ze1UlNrsdvy0}K8d-l`@rqg9amcVd-AJ`werS~yhDlY|^)(F1LJy=0Mu8< zD9n@Nl*{hG@tEDnjW@IvT@R<$Ws5`WCY=~p4L9C-JKI)TsPuuyRRkGZ5nbdMxrJr- zi?+8cxh~Gvv#4uawUK6fWb!J+7QyjZ8>3KQj##js3esDNOQXekQAp5xONC@jHcWHy zIA)T96)3p|%e=B2;u$WWX@8YRcc1IS0fPDJ#b)L-GoL%O_}FcGbJy*XeyO96edN)d z@y5r$sA@%iiZZYNB|g&6yzcg+s{Q{^lr_xl{`W}Rg|wm?qOD!wFS-zrLb%3dr$94+=9?5~)L--b@Ak z6c{vB0%AfvXV`Z#h&w^+sdW_L=Fich_{$cvcBGF+Y>{s;9nd9`TOh>D7pyhMTBC~- zMt9hJFVtEOqoy@;@`&y=8q#H0e0)?T!N+ZE<>n=fCwKXH5& zPrwm)vBG)CA^s$)h48ypV3KdxPDCp)q_qxWG<3kQ-@x-a%TfGpbktA0-z&%O5AuV+ zb)je~5x_YoQA_jUsG$vM9S9MKcAzAr5j?hm*B~M@cq)CQKS$Q;1Jrd{4VO=1`yc9F zPY5V%gAXeE-V`o2)>AHtwNTtsrYy3w_v~ms1w?rDhVrW9G05?|w!QPa_yHtnw%6q| zc(>l2>9lhiliHz$w#zivu=pltUg6ZsH_hGeq<=8qXIsGQaSE>;La^lE{e7ka@}t?Pf* zUE7W5@Ij#_llpNX(pNwGnrqxQH{D_n~gv?+#o`NJU9wb+ct) zd87CsU0q!JxNWYt+p~PG02R78d;}8Z?C6{|^n^Tg-JyUI22^KsV`Or$`$Z*=O{mi{ zLZk)_6Rlw^XqCd$3o_B1Ixq%OsXOr?#Zo~OdkM4(h}hypl%NVnVr6wQyRo`FeC~me z>jT&Wezv1H(~Fy@pL28Uv4%7BsnT4Do7TfiMB#LTcBYL0loE8cjUI;^^Plvp;#kY? zt#@a*J$#mzpr0TN&mIcy(2aP^$o(f&L@ZFcd^Wdj#t@d&p>kC3gH)$fp!PA3k&E0Z zYUosL3?Q6u-PR z^$ohk0`8}yE!pXs+UvxuvkRVG1fG-F&46^xdbREy{O z^WTsI_+8GD=E)V3BP1@STDB&U!2Q}C2yj<<6eo^TQE{!=_-Oq|0}9BM8LJ>O*GMWH z8|HAF>Xla1T{)^#|t?6}l#&ZS1>%>Pr^yV<2yf7~Vu~YRF$~ zH)UJnn!iA((m@skmR7O+2nH(Ud|li+)k-2-kEQ2v+sJ#uRp zj}c&d$g4Va*>mQG=}S1*1Hrx<{Rr5wq~xtc5Sqhcf-DTy{M>0p?D6Px@F6*`r?cF; zWLGU3XGOMRwAa)iWft$0Z6emUD1o)m(h}??3_WRDY;++5w|mtjdYb`9ZsW$l=-5l{ z=`e+KL!fg;4v+G0ae5&dH7t!SNHszMeZpNBr9m&TBEW${3dHa{gIU8~rw{hB?d={0 zkpvy-Ae-4jzyQSVp7l(ty!0Z((cn-pWC8F-6T$^an*bwf|YRA`E(vezY>jAI=(VR0!J0a)J?6A zyrpk*h2Sfgm}Pld$x>8u%BZJ4)}vv?$Nq!fR>w8d}b@0u&{@Jt#3#SuP^5f zqq3a9R_5ItN2T_6#pWE4a(4n(``(XUuIvd}Qt5|L!$EoS&;mPRN zDv`l=i3G1WA+U<|{dy~Mg8H&-qV1shUAI|w!rBzYq-tc))I1npcZIaGh%`PJQWr(z z0&f1>#CS{1mQqk3HWyWoG+RVDhwu6sP`cvrfm8n$wca;9i%C@08Q2Cyi9ph>$?5#l zBGpiw()?+&{zrT+zk|emME3=)cbIH!;vKk<^;&;sbKcv?J>goT4kSWLbu`qSM=ohh zpo<7uc)~?!q1KD6J-SjKr(4*dC#K30r`tw@W>joYd95xDD0iGoKdIup1f7Q2w6m7V zObO3iogj>D9AhX@TN-(6iSCMVZRmYPFks8;v4ukQOsTV8Lk5h#PV=1nC&}?9xY|#{ z_&z3suOzc%*w{2dQeU84HJ(1HL_RQ8uS%zf1S5r2DY0m&%3UyEvSxa4I1RQ=ByN>Z z8{n2i7XcRxoS?aV2JLnaAT;f^-Awo6`2gdWTD!PoCtb8ZILm?p;6)Xi?9E3~d(eei zvR->*KnD{fS*mOgbMFaQHotEZ;=>p_ZMPvXGDNCgq_LBmcc2^-PR;wqTVu~;ptiuT$7T{> zj353E`d+ZYG`lJulaDQ>J<@b~+8xm7EHJj}CU`GepYW^&)Dg|Hh~>SJoHT;0SbwY;`?=mA^`w(N*tu76`JHCR^&|uC(6Yd=_ z8r-`hGMnB%@l;EWX-_^*KL3PJApp|y$h4*nq@T8anD>)d|M!@sfWDKV*?*dt{O=Kx zF7&k@SCiW=UH$h&0Xo)wlv?WAeI-qJ+tG~Hsx>Cq<_0<#Wqb5XT3O*?JGjwpDd90~JGip^F_fsZF{8o?W+9b@1f-KfnDNTne1{QK zriyB1&@%h|`{_VT1Z0_^mi_TTUS3}j06n)&3J#KJX$*fZM?3nPgnvnfaqJZq-*|?d zVgwuJW|QD-w#kgmg@~3UkFm`lBG4Z++sNG`wSi5-lMVM5e^!_#fTU;Lk%}|)9k7zC zQ`G4=3Qx1Or@q=$MWrG%^9>lr=E6I+A*A>ASi=rBE77$#@Hzo(kIr{mg&f~3q;tDd$OKF6a3iDd*!0}%E$(Wi5>Q1gR>;FFG`LN5nE zW^I!gTX4B0t9}@atLe823o!zM5LOD%uIq`#92=rxzhJl0!F7zR7_?ypKD-4i4i-gT z`AtVgo-I^tSy(ZtV8!k(hQcWpruj=nfvYL9MmQbE}TjaWuJ4tDy;9zXC1N4LDHP0rJ-tIRs&P5qR z{AfS>aIJ}n&89Nl<4N@1g@++oak{&I-Rf9#;(cvp`dtVZRf%o?Jm5lN5|TwXHKO0w zs(Q2JleKP#Vm{r`!Ta$!X^bZYlBBR+@?Ku!8WEZWB*XYgAiNHHXutP&Dv{_>>+`9u(uzXrx{5tn1=2ue94TeP#jYXabe3c#*Yv}egW+Fqa zx>fYZKh_csRBuiO$a~B` zlAZ?aHzxWWGcd$<0|6cQknAqf4_?T>ll7%(fa;3p2KoWJzZHbsRGz_w8VHnax2|8` zyBukDrm}Is;bO}%UDPFG4*EX)Z1ZTG9-3wwtW8|}uM*eCQO_j64mcc3Z*X}tF8h-I zJpJ0vn{UU+^pybyZ^vzHSczyhYXKK4yA=T zqFvoGPTU%&CN1r55wE*HcdTWc-DRBUv|~81dK;Q5@YRRq2Cdt5yF6T<5#!ZM40;&m zlE&)6wj7tDNFGa0J8nx6dh={c6Kz#2+vmJrnU%1Meil~sa8m_A(htOVd{xP=NNedx z>uu8<1hPD{p0TN-ENt(=wOy77;Uu{ZOlv`XLEo3z~eW~TKP zC9Fu(w$6Npv7r{>h`&G-?!?Mp;XwtEtK4|v-7Wf`n^)AoyAz7rR-6x3f-zt%DK^&x zLn5B_&_8v6lj+6wBwr=v6#ga=!j1<@n_5!%Y272HK(LEb$^_S%;r|cD-YLkofXmiQ+qP}nwr$(C zZQHhS($-1)q;1=oH>;y7>W}D%zV*7E_gZ_-F~RKs&+LjtNgG;}u zr#0yRJJj&MBUB=WE{4K3riON|4*#j4_oz<(P7NqqC)9MP(I|m}5RjF`3gMW;Rsf6; zk)_x>CR2YLZ;hn{b(;>5cTZlC>K>dH(1n0>C}Xca?m17mP-p8r1N(8SV`f|?aG=cQ zH^*V*lSJsKZ8uli2BfslS~QziTh3wW8eL8v(Y|*cyU2697A8Y`&FpWB)4@h6DK9hW zYMJ?nyKI?159d;*sy^qz-7xcdLfJr@Qxi=Sv3rM8*;Z`=yIW{ZG<6Z^rLX=ZtDJM! z@!+|(uAxHJTiBkuuhkTBTuX}DV7IsUzONe}*xWr;qC$ zb%?sp;P#eMYHs#9lLvCo9f#;(kD~|*to5rGqy|tW0Tr2!Fx^ANm2aPslFt57vPSrg zD#w};0$@3)cZ0wUd(c7K{{rkWnrMVe6$%rz8W65=oUc=}2aGj{+1sX>PPTOwEzvhQ zG=)2b&iINm+qwR-9F$ZmDLidLo9HC%e3@+H(PZK)Kx=|XlBDWWWP;jT2j1?P1A^+i z_Wzc-6$r9yI*4;f0%64cMIEjS$|iWwk~%wlPAB-OksS8oxn3sE=iQbmE6z!}y@}lh z&n~`efMJN%oJk$I(txfF#%vj##ol1ktA)f0jL({*g}ij?k~Tx%jir}_(@ElpCP<3`tf2kgm+{WxSzCmzY4m90V;-NlU*Z++^ks@b^vTCXNQ07LWsMp8vuZ zyWyzk@Xf&@*(=qj8&CHcL&?B;FP)fc9S z6(+tx4i54TgljTj;4%POfK$_=ZbVSXzf9p9*>n-yv>_i?%aGh5G9MBt71N~LNt2cJ zQ-0ajxMkW&cSbRf3z>cRA0G?ETnZhY#2_+|-}^T^0ssJ^|E2r(|G#N0V@~|WxF7TC z=N=CQ6xQHPF$*B)3@1q_hg4hjkpkICI1w&>kG?gX za6_1SRD$f}CrC#*21p2j^AhDWT8Y%sYrq=9Vdn zK19~-KZ${c$cu{Qku|CmIVNr-!|+ihFrpfI+KP9n{7FNI{HKr!g=fDKBzjAVnBQ`S zQZ!d?iRF|}nKwKN^KeOY=mfn_&?cgnP~i*ZVyWwbpac2)Uu8{}GU8a2;i=nUipGWS zZ^A8x8XwTBT~As;soXF)3$6-a-QsQ;>Em;?9-%|r!C{gMk+fbQpg3;DHZr@wN;2+C zOi5(folXNMlptao_$rwaR!|^Ca6}ox6NU`y6OD952b`g43L}gMAXP!|{4%@~^dkwp zQ+7C%y-H#_h#;x5(`?_d58NRIVr@&xDmR;Yoic>5o>fMdo&~q zl2Spo#=3m#(3(?XT+`AZ6WSz*U`=KS#-itpMnu|#sNWIDfyh?fC3nGsdU(#o=R+5q zrNsz_q)CQpPxx&Mkj>mm8v~l&J1c2tIBA4#ch143OLdZtx((4hS%BPTS zx!Yz1yO;sld-5f0CBwEeFEPBt(kE!K8i94MJ6QPnCtIBY>ld%9$C0+N(xlnvMJV-pGNH48Hmk*; zu0@*u&KqOpJcPH%a>^)&qSmLDvUgh_fmN#P885-a_6d7@nuey{UOEFzEGT_xw8Dl#R2zJ2Cy-?85NRsZ5RCBeOR><{sM}@e<7;NI#tuo z93n{9U@eT0)m{BLL@&&=DVvR1ut~oaP_rb`K|)LJnJPHG8HJEg{xIvo9YSB=oAgqL z?zs!jdvxF%^09igRAN`L8)HFdo}+Y9G0zTvD-gLIh2&w+ssd%)-|Gt(h4E8U-? z1_AZ^d2uwn`;KpDP)HXK&8D2*S((Qki`~Dy>b@ex#9g96Y#qY)b zasp(R0bF)R8Nmp6TaR8ajNYspI0E)0tbb-e`*hm!?D)b#@{5zj!!%TeznpG3yLjii zN$!CRU`X%UHKDt926{Xw8hwd(0z(_{% zMQ0a>8LP}t@V%OM^>TfMP7K=BYXH6NDhnMLi=p&OJwcZAlRuYT}8O&^`S4!R%Y38k4sa zXMq4fc#gK`_ZF?%5?Ax#{3J$Ek-f}azWBx>z%&&f6TT}On+A)~P%E+n!e|W%5M^>< zsStilwGHo0c_OUp87-~*XTVfuJn5`zXit~%@(es`DkGYq`6qafiW^RFbYq!H}K;W>KK+^OB(k^tyb1f^o&$&?8U?JZdxx`6o_+ zBTF)8NhoB7ND+-Bj$#ts9Btp4Vrvc+NLhD@Be4R?#z(FMhkqYNFg-;^d9x+?#6gOv z;SlhXNDBOxq`O-le10R0gt zhr(qL9pbw4V^Ru=8GB5K;#2+cFp3Fj%-Ggfpx||flTpU{Rqp#A3r$jbC|yM31yyoBFta+MBS z1)*H84OT$>d4Pz{9&wWaM~))2#`Jni2jwGZkOHG9z5Z8*kS8Hg8}AC2gkkPV4W=-1 zgC?0CbVE|oB($4nz&ST#DI_Tv@H%s9P#-_r*@QI z7E4_PAy7nBAM7S-$dA}?`bMq_s@>LFWs(=nj87zCg&GFP-$e4YBrCoGN8aL*Ufu9p zud7jm0DL@&``VFf%g-S4tRY))H5f``bIvQ&+v^IQypmGpNz<&V6NcY*D8kF*wroJYVEBx8zu+k7!9C4@Nt|mRFp5kNR2G*bW6VB52U!L!UNDjV z6*5Ny$G0PA+MpDC;{|YV!ILEbTA`H6&=C_&;1Nzfc5*47&sppw7i4n^KugupS(8cg zepm-ozq&c4!-O8spiO~x0ia|5^Qijw2f9B3waJ;BMy)`JfI4~x@^>=W$`r67lE9Ihh; z#^f)HGW4hp-EQmJUHF`lygGfGPPOBxY(H1O`76!7<5yB@0wVe!tU0?p_^cKSE}A@; z4;{hWw@e7@-@cSk?h~059Ld(=TJ^{ZH6Q{swIq4+P`ha|k>m`%y{yMRVd2{pUi^*p zBC$em<)g>r{&CzmvNz@Sq7^Pq^G7Ouo7?6Az2HALw;UDBn5tuD{Zk3-rX*4u?kXA8Z*Qk#CVMYveAY^#keZQ z06-OVUEqUBcxX{G^+F$a;jo)kkJ1v2TsTj6xn=`Cp2(U0tpbAR^Qmsa zu2$L#@0NP5RZ8X(ocT!^R@mxTET}0(L zmdSV&S~}WPRNZAs*8JK6F5=AzgI=D}tKL0D#~XIEbjNmPF&eYFvFR^_ zSzuG#s!ixkJm+;itUvts&ri>I9JPN9`7T+KUvZ1S%>G|gnpirDL}gem8C&6t$K$ib z^t+?U+UHqs`-TsP6NAWyUYpPVINt;RRf*gQfy0h|Ia>U`Iqm-&G4;PExc~8?e=BnS zbB2pe*02sY1MB*@ zl}3AWC*e_JVv&HpQdqt{y(0jg-#t(UZsN##qsLS1C~#p}3UitfUk-&|Y-uWa_UbY7 znf{3KIZ13&S5=8(H*Uw#LP8@+#N4hxc>&p`j2WE}y1^e9?hDA)%YIf$6jOJyRizT7 z682oRS)?CgLq|bO-2=ni{57{w1Sdc58_qf26qe}aF=hN%q{wOswn zQtQJA5;b!okpuso-{fNEkytH}$?t&HCQ-h;138wd|Lgf?T?Rb*>qKY9;OeGggT@iG zm(MY}I@od$MqGGm)+B@yU^dI}O}g`!3dC84-Wisao~kzVAWsTO1Tp-(X7B)&pH zNopKvL#^IIggeSz@TOVL!B`q>lS~Nmxik}BD zbVhx*)-9jj+|trf;RvZumefd~B|7}w@Cao_5T1wjG~JfBP)mGKrT0)FL8U~papf<) z;Ov2|pRjjTN_ef%NE4{6iQ)?KWBWx%_0JA6>H0j;bsPG#g3zWJW}-{B>7aPXURye1 zk795kEV9Zq-9lxoL1wr{rYy^Gc;vy^ABLd;SM*K73dt3jB#QizvRH}M49bA0W#fdj z57~PTQU{%p!eMM8oMF;3{N`dF;qdJ>XYmdnqS8#h%@O0s-TkY;?wTGgDoLfr87z-p zP?yBwvrh@!gWL@#Z;^#m_JgG5`+hiq=M_YPZy^Y+f}|rtZ2)*QI<81l+YN)sGgj%m zBH-E2$0y9bSp~q8hBD`?R69o`DX2pVMWkh}+K5R^_J@(H!z-8w!{rDgknA6PWsw7k zWg=^sArhsO>qMtBc#qka^onE@G=|8pw^+kIq0w9ho)1(eEadu5T33yV;0ZbuPRXma zm0a?ki$Z_I5pNM;xM|K9rKB@PG7P!PyX8$O+Vaw5%mN64&E!pwI|4><|H>rIeu@A{ zfO&QF#3M^2)I^;@ja>J7MRWBR*_oyN2p=h3biVU79sWUkHrpPrEsT?yP;(y8uW(v znU4@!r@WgS$ZON42K)`x&mMtw`}8f3wl^6y(oV6-B2F?TrK+A}1;SZMIf%Jsu+#~d zf4St}A0gK&nrr?y5Z@(OG_oVdsOF9SbHWRaIUAT$k1=orj?N)Ts6q$~H8Z0WAhJwL zw$cI<_t8z3QJq%e;w$)w64E)w#L^X1cLE(=oQI~XNSSm%#W9Tbm<_lt^y+h7n&ma=w&-|42eI)BECxC8 zu9IwA&JHa*R7UJiUa`@z5>b5&H$D`n&c_+dU=;==%mBPf)5vv>cq-K;JRB7jBqxDU zBIgJhC`2_B7f|SHdDgwK-jE$^5_{IVU2g);yMabdyMxb5jA!7tW9sUHe^cd@iL@A; zGC7Zm>14`;z-3Y29s5&t1WuD7&YPhhs|eg7Ygv7j4@>DnAg-+^71;AuN{p^PE&nm3 zm*6pv6W%E6lhOFdGbuHI`5)k=*PM{V4d8P+M5pYZr?&%{RSC=$?r_tSPUPd97H|_V z2xf&JHIU7R#Milb)W<~u*_%Da0-4cvY@rwr@-WtB0#~XPT%yVzh)FDZWl{oAHvkqs zKWw@A;Om+5&54RJ8?~_e$`@g!j5QTA#Jif!mwZu^uQ2002DMObea(pNY|&$M+Se>o3=VD3JS#0cMz%JIAeDnmfW zz^PmmAYTH;nz-p-rb&7M{?h|Alq#4m%Owxpxs7Wr#_jN@?{+1^hbgrc{B(ZUi9#=rRl3gKWO+_~pkn&? zg?gkmX8(QSA8qZb{K8pKIejA1TfG8vpcd}HCNqQGA_8%!vK!*0hmIaOy_iLfuo4Oj z#7p+oGBgHUk~<8)#S~I_XN|ouy>@eL@Y8tskA|s{lj(*mpf8iKB^;D;fpXa)xkC%x zvzi#%$NtCAPLrh7ye(V(tVA6!(9(V+mFA-n3qGOz z9tMTcNS#0&4BLC4qx;nO<(N(4yeY>D^j;frF0+na66@)E!Ek7Mdg+$HOHs(7Y)=_^ zVVp5R7WhCIIn3D{l?MKL_L2sQ>x)X;uG+F5UyBcu+i*!ZO{#eRsp!j57tV-_q-I{- zlVvl64c<+l!kTJ&VGK0JqF;;U_8A1Sw;2nwxMitFdXHPc-mAZ+}vY$JuO?`;mnv!Yhp# zb&f8Y^5}NVIt&*r16sS;8?Bi8u7Q~Ql|NMA9sSv74BY&@EU!Mf>4Yi>HY-AZ_`jTi zEQyk0(1Odq)uq(YlTDI2%v>c;SC?}7#iF?*BzXOYEqq4>T~5n5$|4JUIxQ3MXU{PA zf~n>KWgZX$Hf8gxKgM9LfC5vH17P;d3{QdAcqxmAX}9`b@=)A5lnWokyZT60{2%x? zLwTnD>HiLc75N89on7*oN^(uX@#1Rx2>$sJzl;dL`C9tk@Az3o^a(V1!AV6llkS^_`K^rrn&JD5L8t|zb3udfF!DP+SeQ=ws zkYU6-L8Bnzx%n3Ix>+pm&k{)-09aFHvqm>70j1TN>+;GWTF2p|vdlngF?KUB5~do! zpAdr>>6Hj&DpjL)W=j2xQ>IO(twF8h_5`~7Os~hb)aiEoRrQ9J_M@PJ4vIsgMvO@* zVucz?YHao#XL+?9M(VJwjQk!Va?zGhias90Oo@}K`)U0ne?C5Ij|@csJ)jcXPPagX z8~M)}KIyYS`u@chL`uNCO_H_J;|olKkO@9klg-z=5ub}e2|ie7n-##P#3fX0$}PnN z)S|)CK4zUF;FUOEJ=X0YJ0qY@v!B`!!Dlu-8w0KRp7VdWXGR}ncoAC;2I>re*5tE zz#18-w}9;MYo=7*98=Xzg)5@E6z7g2zzySDTk+YfI2Gh4m(Jn()A?AR3m{zrNx7#4 z|MG#=s(R4A5~F)GaK@^Mz?RO4wo&scKjrijnQ2B=GsH4DWokeT<6^-4^9|!VM)`!* zXYMNg@+Tq^4z@mLfluT*rm8B^>hCY)`y4-*Ej)Q^80dV-@o@WrV;gc3$^3q|%e2=!0B_wwFwHee6N+_j^{-0T>a-Jn(>m3>RQisd!Ham1a;Bh%75cj`@CIqKw?`4w=b8Xs*KWP)!0r)F zYB1J@@F2p*WS0Vzc`#475`bqvN@i(Zz;?2i&{bv_r#XJlx84Aj_)cJdx;Kf9JVBIf zPGY~daQyudj9&pg_;~oclK}4dB;ccCFAfCHOgeBNt}myEEMG(}oXq8*UdyVc5+;o@ z8_N#J67Ie1Q7^{2+RHNgtqr-YSi#-jK-m84&9q#zMdD#0e~yVjjEx_1lfIgaSePkw_OkH$(N94fugJ{5S@Y zR+~yKgR#-{zt}1rtJ5Av3yM=MC==)i8%O@zM-f}{n0BIOy!gX+N9c@NCbm-AaM}7C z)Hyf7NksBcYl&P95E4$u-+GRoqo%yW`7%}$+HE%&Cr^cmW^n1dacyXv4mgczTnl&} ziI0|~OysPuEJkV7jZ|Z_My>Y_hlzhKn6eM8!Fe##1656E+d#qJJIiSu8jHlQ1-|l2 z_tC)DToGqiBKmH-`903qz}6phhJePwmk=n0@eW=u0bF!QjhIEl`4o=`(IBXV;2Ai)Lu5Y5>wqdGL)tHO%42`deBEf#Np0l1!rz;kF7LjBeF9CD_|o z_d<>gYIqF*^QJ6aw{}`#mps^!;N{$15h_2&~(0BN@2yL^`x3-;Jz}`oibsW7$$Q-4h!-F4#L&w2ocWg-M)yYlH-eKRD&QM6gp9S$!L62I} z9Xfsp1@m680d2c+RB>)T}_vf^(Qo?{|nH?2q=yqUlSD z@h5*27?V;GilCV0tW$=_fH|~-fqO&)1Lq|Kz!{tk2A+jQB<1i`J+x*K>z4=%uHu_+ z7O^mgk-cgvUqBg^;|Bu9Q7Lp^09bQBvmNq8Be(_+p`YSNag!j#Ep!o1 zCA8BJ_%jHQOwE>uf~=Tpu?Hh$#q zeKG=a>bleAO@)!W>xfEu%f34kLRf|&f^d|udMqVV=uJsM<*Q&gEW^r%q#I5bxqCor zgjV^$3h45RTPm+0oS~#T35uwo5rgR+QrBAq5~a*EE(pX7v!#_b&`nxmp_Qth!S z3CC`w3K7kLtK|=GL2Guey<gv3L zU1$zU4lPv5DP}>Wd$z(Gaw+hhJgs;*#fKNXc8~M*dDb$5>78+~@63|obWMzsrmra2 zTV(DjhsF^eQm4u;AwgQix$a53L?KVsv{0lBgj%tG@yj(`cH_H6bD>O0ZipLVhB*%C zv1v@B3{SA58J_eVcDXi%iDWe_=IEx#GgN}zIFBb)>ACw$MY}%%1kv%k;#i`ql z<8=V$B?Kij5G~bwp`A2))us{gSM&`giy_0V6O5rDs!&Z?Nn+nIO_)h+q4k=1R(fj3 zU*T_jjGY1kOmC>(q;>p8sho6&lw>Uw-uu#DQ3(eI>K#n8;(yb|tzEp_&NiFuo!2lX zw%10aq~%XS=2eSPQ*p`>evbz}BMOfPIHd)}1`WV$o0xCb7_BmuLjWt=ZkO}yNlQ5( z0Wu&ZRI7ODp-nZME+sg(xY%q@pNX@eOPw$0kI0h!4==XL7~t#fFt!Z7f7~chIAyO7 zczB>-!w31sD;9>aAzyX0Ef30o;dY8R0FY4Hz~gTM`(hnqUfjjPkeV;KRdyAPSpo6! zjr-2dXO>;V^}+{KUa(`GXrMbsW?rt_KPd?XO+F(y@fqnNUazZ^8M%PA8gD7f<7=4ZuY6+O27ytd5Tr_5PkgBAxRiS zB4X|CqP3Lb3EFmz+wBVViks>>3|Su(i4U`7P$VGpGU<78nkBw$5*G(4L05$P*_n=4 zVD_Fs*pN{rj=)85@OP0ArKGW>h&l%#QH2tiiX7~Y0j%8#mBE-u!P|HOw}rT_Ytk$A z1lK)Txiz|elVg0D++&;z35$cORY|SH!TP4|w85zG8yTs|g8=R&*T|*k_Vk?esd;VlE zKav88y<|lEL?%Np7_{wdHOtJcYqSXMZ5z-Z7Iw6g-Y$6CfVttSzb$;}x=ugMX|dBKvd=Y7$bx%|t)Fewb0Y|jeNsX`O+$$i zpfc@3TyB_~_sy|Bx}9oKPH4vR-@7#&tN!Tsav*y%7aZG6P%SD>StWxgv)J%9ugA*^ zgjgZEr2?jRU8qj*7z73HGt$Ar%oHPx`z45ON-;Vu_ZhY*^6e2(%fTdvm%_~Yd~#EP z__$37kL(1oKMh5FH#UOWbDzKhwL{h{{8C*)ekXkju~(7j9CYuXg=h|e$t33pLi=BE zr2U5`;6w6e^+rTxJwyWVy1JLS4;ZhWKhpSdS-#v#wJxs%t#nr#?L>?z`Zp2_p z&1wV1mi;etv_qFCQ6_dMnbCCO!+r=hHZ|>JWFCDV17m4c;nTA5q&w5_vC^d!UX+b1 z3h=PTuJliEC#-=v>&Y=&2V2I2ph+Q&&~YSnGLY%E!WlI|3a~aUKMh|ru}!_r0H?{~ z^%pN2+mNeCbzw4iBO-MDJVUTk0Q)j8er88N=j)LD$! zT&*>ggOC*ocR?}cd`JorLMxrVtKPSrNE0k}6qYZhI8&UFAbI?p=$TQM*)zG-U}e;c z;wvL;9wiva&fWWF!{v=}HtW3Xc*#Zm95u)r_=I8}CX6wXQ`ehkVSzkmhSce$R9OL| z=yuG)a$C9h#}nXde~F?+IA9b?)A&_24moJ}+%|lE3uaXg|B>k&B;Yo%!_yMuRN&7> zB)5mRS2zfUcV&9^bKj58JM}QvP^EH@>v?%+Di!{i&PmkIPFSUN0m` zEd#gX>>IyqLsS|3AWl-jz4|7XEHXbqVgB+pW0q2KZ|`}hR5Q^d#_rjo4*tSDSeOMK zdFI!KB2C2k&l10BCRoeM#gd39*1Ah@#qQp{E_RxUQ>y7=qUjt^;wKT>*u^8?_ucqY zdefBohV1DFy{tn~H912)SFEqr0?qxCyUT<8FUD_m<^`MUlWH2N<7OI-g>n|}Tjvbf zMWddg?~G=%gGVChB%Ta3L+J;lW)M?@0CgwP(B)?U31jbj%rGm0mB z0c8P?FAKSc!8~OV>2v*AGfvufK|%7{R*@POlDJ@0als{3ce>vE6{rDnF=Q0o6d;5N;I$01&`%}Y9_(KBmuGB?FnF# zRW%({+=7VtO8_jtxwPKUZ|lp!XdS%6Z~FSgx5=bg))Nbx@FbT7)bdDgq&00u*BFjI*i!a(Ox4?Sr8 zbFa-UGXrq$#CBD8d@*I+jB}7*lJXilOd`#Xt$lny^yZkcLI4Pnf~3-onqks&!*!7u zxTIR$q{&6*1)=*b)&aCDh{#2BX!xb>9F$e46%%>` z2$3-8cyv-w%|Or#q+bvMGU%OZM|Zgy>E=fZ<3`5=e`vqDF}nFDd6f?6VlqvGx8DsD;D~D`bv_7%1eS4+oa~ zGoqEyF=c-rc!T^d$i^7b-$Z$@_Tu?nE1_$!EyO!L-`DfSXGTtW<_roOBh1fpQUQNR znW%{|d_S3PPFjEt293V*;l$vC=5CL2TEo73N%lBQDo$`2I625}f1+*vXJ7o6D1#Lq zje!yiu>fS7W@V>gMfKmCt~iv(cps*mwZ|g;$iVDjV1is?7T7Rb24Fa1RNGCA?kVGx zG)`^eLf=c|IbH`uD|SIzbixsG2z(NlD1nAj zIqcGR(q;V5p9olg?q`*l8nSGEZ0^T1zRs3X^lu$TT! zPnrsg!kQQ@ao)td@%@H#SF639&Q5 z0LY;c17Y%o-^+CmW5?NZ@3kJxsE+1NVuS1ogGspLY>9T0^l4?rm1YE$MuBP`^g%A1 z zc2eL7Yw5t(CJ3jNBwxHg3QCqZ3o+BnbjHbO1*Q5W1uvRfGw37=V*X%*Uv=E)8(N@Z z71LF*GA1kW$5aRhg-Lz?oR88Z$3zZ7;&a_%{sdnxwFU2tI?>lpqvPOM6k1~eIfRY_ zDxkCM!PsDZBQPJ!k5@YJ)kW>IxEQNc9EW~Vp*q)^T3VIr99taJVL^7#VxeEhGa|Gj zvQ-Ooy_*soP^Ifx%zA7)bGXgwI3zV~ncKw95fQ{4ARS4q;_UB4I;BX(^P}@5!Bkv^ zA{&+q2?qa5sKq5OPVZz&N*+nL5Y|ytWo#o1iI6ytEDA{Y5GvR-e&7h{9(1!FCJIjH4i--)>%?isk`2AS8ZI zC^R0YEEjp<`{XhqMQ$>KEMAfrsGnvR8;eDg>RX7Tu?v5|}P>TnxrAKS?933$4|G4NG8jn-7ljIMSu zqjJi+CdY=-Q-Q8gMgmeCGUB!qJeUitlY0BKiJ3`?TV@jI--y2%bWp9tB{|D!*-#QB zrdbBoUS;DV9mo_J91W@jG#d3hD{)g?X;E1RBV#w^2sDVJ7l_IoV0MmHUrflKthTFv z`J%o6=|Rqs)Epo{L|+tDwLzcCA0}M6b1ieNwV%9NG4EcZ(d|9Y?S~=GZg(d5%LemTsAYaj=)SFhfba zwyeigMM1;Rk?d$sqfTAjMx3>sodBO=aBtepC1m|$x5zrtqQ-GS!D->LLIqvcCq%7| z0)zX{hP=MEDwO?UO~KyIVeRDBC3!4t+5WUazeHC8i6L|nI3d(F`%p#p;<1T7pwtzD zY5PWa50dx5VHy~)d88P9528Q-Wpg~Pa7RO;@}|riYc|-7^&pPXJa3*d(T7#Cj0<-e z9(Y0WrD|4M`Pk}m8%@S)uI+;*meOQPt>bAW)rNcAKigV`CH?c8do9QBl!`K-$=vx6 z2~20pA=us%ysYod51>!7>E{H162is!aAE}{GgY-yQz8#--X#!qpU9)6CRwe1rl4K+ zh~RD7gl6(y(`4a_Y{ew2S`T29a)~tzAwqq3h)E)BaFV&0$HqS*LW_ zk=?xPtBcvneXVbgap|2`^4HfY%&=>{2IvNWwNqXM0M-ax9L5O|P;A|+%~evlLY0U$ zWR2R6&c40MOvNC(P{N5OXSGL_&Odbu_GLxbHG5H5pT?trtJ5_BZ0iBQKaf~sRI%I` z0W&58n`+3kGaE3-c^^zsL%n!arUs%cN&A3Z2|}BL34o|Gnth+D%`H^Ss|)-pqq40} zG4#uxg@Orp9bA)X(`on5W)92vdbpZ*SNBrH!MY~^0Y-iETH3K zaGyPq--pj;CrR(IfVP7hk9U1BOG_`M#eGMpIm$$=eG}Teu%kB6S1lShj|JA_o|Q=g zZ)ba!Hw-hayq5AOCFMiPVKjgupnA8Ejytw8NrMWQfO-_3J5mODbKoV75X_?(a%O~xTh*bF)S(fJnJr!7WfjoTbrxTXuwQVO)N=YC3hDIq0_^gtuwY?P04~g!{yn{KXr- zI<&u$-2cVn1#7yuQP@7_8xe7H3i+0Mf>ig($=B5Q9dpk&5~hIu0SoM^n%FVpf6$h4 z*9j`mIT9VrJdU_RqH=Eb7$3g}Z#pg{YR9dY-pWn#&bhN-RvBgrzBKL z>de?u7f`5~W~O(+0{~Fy7+xh69?Q$UjP4#YpbdPQ+@3;|=N_KUs<%C8Fves#YSD7n zpy+Q`aWor*6m=Cr6#?#W(bZN@+=SaMSB189%{vv_lgj+30NSA8t=`bamRDi0JN%rd zV&+oRMgO+!=K`sT`gBu}+Gz33rm?}vN>VO!AZ^ zn_>*cC9NX!0fB!hIe-Q{J<3n##UW!vsiAmbrvDUHma&51X{3yCcftwpCr%W*Rj*h* zMrR2HCU9Dr@GcT`3&H&!fXEu{4O-+k@rPfF2c}Ki*uc-eIXYUxDS+s47#N z_jP}Gts+#&wy|AZYZAYW0Ee5rzkxpO`xNlI;}Ts}*s?3{5|$_W?qs;|z1tO7H%fcg z!*fxW%VQ=#l}!IT4aFd=Xwg+KbqA0y=fB8q+<`dA#XF~`aRnlJ;YJZ{nsc%5>wZID zeB3&FR!2uo!=yGiWyilfdn-zy=6*ZkHifIc>e~`FSEpbJEnm^pt6Y7Aex`pxRy!U( z6_Eoc)2#L@cp9!i&vR@L-6gM8UqmS_YZl#m>&NG-9~oq(*EP;up-cLymg^Y>AX4>l z!NrX2l_b4iL9x9jLu{!PuKsAp8&eXl%G=p>pF-7~qv_ouMf+M==K+A}?8x`Dkw8uU z;mPmxj%-)vxI5G&wWj9S@JMY9w06MP2rpiy06y<()WdSW*|cf%{XmcvkJKwwU^^Gm zMzn^2$k~7WwQ?tvrSk=E4_^buKFb0(Mj)j9qQr?ZMa1|xD{`@(DX!v6&)x$OAT6j& zE1Dk9*Ih66m?a)_G*(X>gw|inEZP-7^hF)r5{68kqjylk10sZlWBZ4k1$#WkQG<;Q zfOtmWJZZ&uVM9sxzM|Z)bW@PZAr^Nfq69HlZ(0xfR&i#TxN`RhyvH0nu;v%|nQ?P@ zS@(-{uq8E+%TDe4NS{kI$z;qmT%4-SSuk^);_sOeg56h?a-X3bX0nOmy$m)g=BP!x zA>@dwO-&Wr+tO+{pl%|?-Q{^_7H_6bb-_^HWGvQ{cj=$Q>6zbE;9FnT z3tMQ%%n#S2=4~t`ug{O|-Ey|w-X9A1+@BsV)G2=%Z}#P$v_siqZ>E_J)r||JIQPmK z;iDHaAv~zEp&#gIt*^a%j=TTW5a(wGr7rmnxktb^o4(<@q9p!r9)zl!{lkLF%G=2G zTk8CO)9W3a|Fxpjtf7ia-O{f3a zN!eXeBBmk13tBSX?!V1V4eg>q2QJ@ zB}uFf*;x!-I(Tj!?1b*u1>LH4AZ1f~g*47#TM;6IH{mx0LH9tCE%R zg|f+hdML4{X+1f-Wf{rRk0v4{+7JYI9OtFWgot4}rhN0Tm>Sa5Gx8DVk%Tbjoc2k0 z#}D)arG8iq#W21|{=G1|#{iZ%Zx3$?F^cXY{*$7Rz_zsF=!8yjoeJ|lHQb+KugmAi zT}T=1gQ`CEEMMfPkt-Ay;jve_BE;){8R`&8?e*_ zU)3^ltXM~4`eC95Ye|d}cTxlO>#TpN|vc1MIh#xx9GevSnjQ0;NTE{pD?g^&P*jOZI>vl6q z3sJu2^M`ax>!a;umrR&IqsD7OK}2F#m#Bw;m3Qknepv!_{4oS=bRDUM8*2|zf%pTF z5*7;PWs83S)IM0z(ISmWv!CO_)bz&74lVYBU$Z!HfwA#H+hwugV;yl%205b><7i!v zrK023DyFlx$-S_lWE&igqceHxk4b0UeUY2u)G7{?DC`YiwPaw&I577thHO9M$mi|w zpTkp60fB-qPrhH^qxJELv6Foh(6-6O@k^R3=0i}eCU*AvwUn4%28@( zL#^w90yf$pG!(cX3U8E}Eh6-#>&NFZjNz!bdYp;JUEh;Er-i3x#P?EC2~gUk=TK2< z6ztuYk7ozgOo{}_0_R4=y1xYzz4@%t_KCT%;KTvqKzBB=$Yy1^iaZHS05|Qqu_nnO z_cGu5NW=psiQ9^ww!D~LJnadwaQv+^YdI#Rb(Z?%&KHrLK|rq&kxsbzm(3aYdK+hG<^ zYDhXfS8uII$GYII7d#g|{|nh=Bic`=;Te~S;9JV`2gbkOHTB;x{$HVeSW_$ho9Xfo zi!-ku)O1(QF!gkM+6+$iuzY2 zEnov|^u>~*^6bT`fqX~M7qQ|#dQcU-lb!}QcCr1Ryd6C|Ie4$6^&_+8{a(`JPH=nZ@1d!KLv{4ElruVp-0T zY?xko2amh^1!)Srj7q8v#>%^Us*QYREZxU`RUBp&Ug9ZB;ajL$5F^QwteM%ooFvB) z9^hI~QgN3gUn4Qrm}I7KA`);QoJNI`4w~k)wGnqdSo-YhGU}vBlnLo8}S}hyM66cdVZ$~&Ye+kbXf>x?1 zF2+6UY221#oux)T^gJK26DsC9H8%r3t0f+Gjv0CO?Qg-+KIK*mBj_4@jW-N)Acid& zKvlP6U?<U#R3qjG8v(A=V}N z=N>Fd9yG-~`AKT{9`oTDk%h}tl;;K4V2K7wGDSG!0kisgl9@_uAz}}OqUOgF*|9oj&)3r|e9!rSWS~HwK$Kj1w3>1j zDUYP^h&x7LI++Gb2`>|%8sYB;-c0V4B7IQje0SeGW7I%#ExV3bdmsXTrg_}1m1D}# zcG%?Big9Hm@LHX*IF-Xe6kd-LHA}_n$I(5*qZrvs*z zj_^PE9{dm@uMVJu5m`#IOSTH{if?-}1jmKgZ(HDL6($*-hEIYDb&g4swdc_A(Q0NS zD>EP)4Y8%N=DN?Y zsbW`MJ{Qe>o5Duh^eK~@yHrmqq$h!TARcLL7$SXbcOGiqeOw%_tTl9;w*H#Tb7#*_ zovnZ#wiz;oQpVhHP5NTZ=g}3h+-UoRYo!G5;I8!&Oph^!?j{kc5{XrG zl|_CHyn3hCy{i)1G9y-a)F0hhPy0HT#ib=2U8erYzJMah`nW7#2NtKsOdD6_Jr6S@ z8_JdqESZWWoop=Qg;cuNcPgAODOu~39JVwph;Hn#R6j_A=Al#imo>Qp1|Xq7eK&@y z`?w!?aMQTTr6a2o4txr`=h0$#-H7R{hMP0!U5jR0l$Z>kkuC(sqjdXLiiIr>MR`GI zGxUSjz0vMGso?2-fP^ys8PWr|K77EYa9zsZ7bVgU*mHV}GmXvy-t15T?c{f>A{NmU z7c-5liCjud4{I@g(>|xA5+@B?;yopdSvset1;s)di6X-ctJc<+G>l;L1jfg+@}s&? z4BRxA;;qfWb-CYU!<@&n9very+qa|#UWRiOHU95dI`+EvOPkJ00F4_i0PfmGU=Dr6 zeVWo@3m?y7cvn^CeuTeCSc>)s5jHI(>uT*&3p!1~m?ns({x$RF4es^6ooOOvbW8GK z9No4&IeyB9dyN0W+^k-`@dY%m(3&%T&SHOAln#!VcMGL0p53?2j3+*{so(#b6I@^PcUxjF9J2A`8KFXB5W6`UOedZj;LnB$7y=)Z>Ji+@jr&)@nUqR$ z-7k@L`ByX`phXKHAnt#2bERo#t7zoxV)ozmQ`7i1F8gdZ-*zQqkdFCIlfaV#LoNUP-b4(xw-`a}4a`eU0><>JOI$nyZ$iIc!FK_vxselmYk> z_GoL=j~5{$D?KvtPfvfBc%l|2_SFvN`$~<@$~>M%B|xYm)bCGX`AhsQ{A9cLJu~16 znWXloE3klcgDk6aQP@RY~#iiEE>C+jmKAE2(sUqqw1F_*}v#O1uvTIY$^qGoeZ z*8n(zX$Kp}jT7h4c$b0xVubz##U@}V^aLLicJ&{(@0ZLHnpI*P)JdyV5rmn%(K>@H z59|((7u+HxOx{=qQ$$*%JPCwjgeFZ(Fqn~#OtayvNA~l~8p+=dxiL!X=OW8Z<}pQ< zpk=|$MZz1Mi_w$BDhbGb9`a{Ig^`@?hlw#f(S4VJ%pf~e!!w(teJMG(8>aR$_g7a; zZ5oS3)N?s<354~64YezbPK=dpfa;8DIhqs_58(`9FO$vrR006L01cV5z6QXI?pZQJP>ySh_ zi`1Zy3SAHjf$8!vZJ2&1U?K5REIltaOcb+Hl+lBJ2V{P~Ew&aFmKPr}dnOlGAZy8HKV)h>%sbGL?&(p73`1dM6SdLY(qdJ@Fa%b6$9dXYUqjvj!Z z!8F~eDmJ}mLufcNF?7t8v*sfbXOF-MBE{0!b=q($5#4+w#|Xsga41?4Ee*1b^`tiJ4B@56+G zEvwXBSC|-YF=r8yDke!o5l5SCI&f5glh6LpVo;q(RZa#HxEF&V=sbXd687Qcnb{~v z_<=RVg8+>>{%UE*9u{oxnKzv#8$FO4ofQ5lS5GNyX8IaGop?2Xx?>0pDeQnx+afR- z+?O^vw5WUVfSoBi3FlA`TJxl8hkTSHut4FGgcF3e-p;|ks?o3D+pB7o>nB7zlLcf z@Nh0!D&uejjjouT6qE)c`R<+uqxDHZ%z+@iSa26G%;z6VydlJ*6Tz*eAR3CZmjK z2$6ZOeoi&W+W-rpH7`G7`u)du6#{MdXr4JVWeG2~mYe`LNAWkCN*%5h+W_ zr_@LIgFQkS^lpL57f7C90?Pu|mK@ll;E5mJDrSN7ts!E8j=8%Oi|TJH&L#-EcqF## zMm@(-j5La$c(~{Te3-FYZ)sUM(;T*L|XIf*;lu#|{ zHY)@syW&-q5ulej+nsyx^~}n%Gfs$Yr^ygol{|JjRZ*~iEge^t^XQOoj8JE?!HSzP zyK|<0v=_3i@6ssD!Y$8VRbSKR;W&`jSAX4LYtt9a(pP_!T zZFUx~-EJ>F3P|vAa{({Po$)agf1W#nzyb&YFg`ErT~n>m7@&Q{_yJXHGtlTFvT>^Z zC{P)^`imdlNNVaXf?&P3A~$%h6#+@k*rG(p=g@@6yCQAj*}J>}nyJ5*QFZBH^Rt};erOfd>Mju}E=Y;lCQjzEVa z5?H){#l<2*zerO-JYdb&L^&rPF*b0gdRIlNEFnM#G_wTp)CO9VGXfdA<1rn2>J}lj5xRF$24rDjeN!_=tV0tL9-MLtU zS#t+^Asr&I0<9WD*&&4Hd+1pcd=)5MB254bJxAic%NA0!D3Gl#F#STRo*fGa#DdBD z4IV=RRcD3pD0=3Qd>Mw2idVID3PW>o%Ax;;WzJ6fOAOC0P&`xG~mG35-4+)DArI7fJhMweC46Grzx#*codZclWI459~a!` zjnfm)KPzFMZHRewZkru#jLuRr?}W$eAwB%`JSb>WwivNsC{qk)+HO)+dV6>gwBF@c zitp%2^oEuTGp*=}@V9jll@GSWc+6sjTS7M0*3YI)m|~hsCiyFx>RuRajP`Y8&>R>7 z=+WPZK+`IFsqgpzX5H%>HC7y%+Pwhz;^ChN_Cb)#7BDg!Hy86mf zfQev8aj6cM$yJ2^)B8^9=8;>Veym@|#aIae(IB2q*46Us!gO@4-iWr8sZnB30fh=w zzOj?8JypFPA%u{Xz!mHa>!OU>Q;JJ;v+D-%WBK!ENS=~rY6At4M5X+-%LV=fp)z#a(uaw`O`Y{jfgKhALBPCGW6W?C-ujb-&C{#>Le8-{oA;Eh%Sr z0*@2PNSjaW-fqyZWJZQ?`a#H;X)gPGbJ~UVSq{^`Z79bAl&Xx`^(Clpt2y1C(4YPY zQ9#Z*#!^xaM(<7?ZFpcMt0=4MRz&|g`Dl!9*^A#`4I*n#P1xR3*&46 zxJ!0XICFN#Df3k6qZL7LIxs(cBq!;5tte$ukl%$Inx*WS?rA4OY)o!{gbI^z0RcM? zW=cf>tMR@Yn!p=^nlLru{t0-x95V?w2JcFwBbag;;B7g~M?F>$Dy-o$94;izy1*vo zPQ{QOyN5ufMfx~NU5n_01Yt){et@2FsXSbr{BaP>9A!5-1VNISLWY65zN5F3Q+FN1Y=4*bQ0;Wu^6 zj;%p2>Duon zpy>SwM~GnBOaMDn%`Ie_l{BA<0WlBTI1_hQ~zqdJ!hd-3tx-W zF9(^inldn~=HArdKZFHWC`iE3@E)KI#NOmD1 z)S4lie8a8I8W*ZT>9^qcIBgI-_LA|`8DD?o$X5VmzV!oXE!k%9!)}36dG;1s`BP}Z zK+Kx5b}fuIaCfPYk(ilYtZHIEl3&2fh*;{#W-2Yci2s$!TQUdiulG{mm}`7Smby#} zJd!^$PZtFyA$G=0_oM;>mZ>)^I`z*bS)S?1EntYonuaNRp!1fUn%=iYU-_uw{F#r6 zsl~ph;6Z3-E%vh#=0zHdzQJK(C^Uo!Qfe+JE|Fiq#|&*+YB%J-Z7r~BYXTm6h#6n> z1NgJ_5yT$J1ME>WT05Qm)x3-jsh3CdaYIRZcr4}o+Qw#4R(eeHU9r6`J)T~}qJLxK zM5&B)c`al?^x>H}pLUg4Y0vH+xZOcz`_5Ld-4wa@w+cMwoZ2D;+E(#fkusJ*3~6o> zOhJFk{#P3&6=)Eh+CP+~DW+u)YVh3jS;C3rP_&V@*U}8)8L=Zg+-NTsk~#I%@T}Ya z*>iFfDpp@m#u`u1x?Hc)H%8;IRMSdkq?zkU=JW@`bPMOAmECM>{SQT;KcVnosS`AP+qJgtST!?M(c#a1yo7C1QK^X zCGT2K|5Z$rcMk3Q9Xe&z@zrv{?@JM*AH!08E9R6LS0&+st1EL z?1L`cPWFgr z#V*n#519=k--Wn@0&B*Y&1ce{_Hj>GsCYJC)@E55yw`i@)%Ubm6m_S8XtU0Uh!d8> z9<524vg(-gYBH>b!>k>NPDKKE5&jR4$w|c=q_9ZFS5iwR~@0pFr{Tmy{^wwqRkYm0gr7#1<*lXazyluQ6no#3zg~ zkB;sns>--7lbMC-FAynU6zwXBmTF?JtX}znJVKP?l#kcvN$mROeyefTJ3Q|Z5s_D~ z<|4!2l@=zL7mEXr{@o9$-gEwWL1a%oqfGRQBqck7SV^?|6M7&(`L5q&%b)A8Tk z?)Sev;exX?g1(<7{u|UAHCp&LGj0PGIfK52()YdE_NG=3$vKwjN8!KsFHHk2wdw5c zhQp#B}Jbq<&PaxU8-VYQ1r-7XV4csay1 zMt!ov>$*bg(hv++G2gzBer}$L$#!mVRO4dh3L4KSXg`~pYbs*J9J{HE4VS)-S=EpX z(eQhN(?$NYx=aozzeqMn(Y;%$`m|nBNWx2G=yjDm1>7A!D|q{-;vD>_K{p^QH?VV; zJXx6`&XO|Pe?oxHS>ra9+2iFVgVWkg2sr-WCL>wFN68{wB9 z`X=SHi;@kda!A1D)jAnCeII~R$y7}Y=|xh2yZ8?4TNEYVp1f-`lF~Ox61?c?QTnzP z)&17tBF+hi6rG%$avX>K8n&Rj7kr&IpFeK@ZEi3FAyH~7uGi3#iEehYpmLe*yqUdZ z+!7*4`aVBE<9Mvs^7=zZiC3R1!J&vXK!AhdG)1+6r8JO|0{J=TBirtNtLU#GBDH@{ z_<1b;L8ZLnTxq=NpQYR>efTnoHI%<2kY+bqby$M2_W8e0$y5#6+vZ>Zb4G|NPKf#! z=NI?uV~;sRy|*VI3XL9y*J>sF=`j`y`~4TcZcRCG;NsyTc6VeDEvSDC)$X$)rCkls zN2oLBDM%!r8Wi15jz3&@a%9HfO18;|dPz7kUfrzj7N)-L=f#&%ivJ}bq?)T2+J6>T zNwcn=+9*lAl+(YmH;D7cQ}81YT)mNM(T5a}Q{+{{!Av_oE-Ts%l{)?FkF2=h!1ZCp z#`HWG|BB1|!I$xAc~O7oP+@WA#PMe4sccaf`^J(br&<6@i4NcST5t69VWl9J8a8?)8}z3r+2%s#gF(QdCi@kor7pm^yj2(*MPEUe_{MN zP5k&_kg3DG*%~xy-jnY&vh(@%)|G=mo>4z`M$S^W<6={kfAxtY9~-e3@|Yij`t!|; z4Zz->{o#f{=!*xBb*AllS!Y%L&`-f^i%e9R$4KF3z)o>ZLmIZUW4&{*?U8Krf>H6y zr%T>1U7MN)n(qZfOQ4H+So5WoW+0Qe3KRzdnU(L05FC%apW7bTlg~0FTV@P1Ho>(& z@o0N#8;DoKkaHJ2melc<`(4};x=QHGq#~&VBe=XHV0C!)cuId@s=(OWXfRgmvt?-4 zP_T+KQn|r2m?FMU(n;-E?v(`h0DU%LLw6oYFipoN+aoZKwGlqgVfO0_#K&a*0bBpm zOqh{)fo|yFmo|S&xomgn18vXP8YQdgTwvoQlYN7zjJ{E6STY_6PxySA!873NpPqz3 zKr5Qmy5A+#e_mwYcP`E*jCMv=_Kem>?naD8u8fY(4kixvjIL&;j4rN5&aQ5bjQ`d5 z|Bncr#~U-Q-*lG|Bp@Kse=~xDvxSlU|7N?Wm{^+G8HrnYD!bX+n>jQ5-Gyj zkbIvt;Joq?U>JxzfoaG?R0Cjyq}rehEKM9xg1c?pSL{G1dR^QvhvYZVlpcYrDxEX< zSzV4UrpQco5nFbKWD}^drIE1+z}J|}?k+Nphpi}53Pej;UrDiU^Sr7(7gb<4(}ZzT z!o@2s0Oo&xJCcLsOcDAcjy!!W3s;Kev!X{2f7_24NIsILOIZc!r;OE#bBHY_M@1%)A|s**PD5iJ~Tz5gkGlfc3)B}C!W{4Q*;N?4(9!B=Kp7=3i zhWBUZqY6zPPO3=>K+eb}jdVBVhxn{25UC4RpDxq8WN>!_IpL{EE`-#y#*uo*Y;BfU zIf`;&m|u1mqefV)S{ZFl3xLtL-St1hgDQ}{LyOC|>;~oFn7|3caQw?!fNI^WjT=2- z?Z|PWApNtVkNtE9N`-DnETi;-MgHD3)3KiE+xKdDvBP>NJ1)#&oTWfH?X_vD9cUT; zUEvo*k&uvrRTNPC+-4k3X}1R>tO#Zzakihxf@gC|P+qL{ znInn57SnNS@cFwZhlZkzJ&W^Cv2+!+rAsa|s$P5SFZs#%^bobyWy(3jYcE-QO?j6@ z`Cb&+&>C*Bw7@akGHly!2ij2%-+~SgO-GG-XO-09zfbgpiVb{LTY&n5aC_Az(X{OX zx-h~R9l!FUSt+4mmHjkDxXJS2O<5Vrs>f+Mc`9oz+^-Zq5X5~_86BacCvN~Bi+T~s z+vjny&93pSHS)26Gs`w*6PEkCbKN1hc!g^`#!{Gk2SE??hgO1Uugso!2Pv6}Lge@)^?`FJnrv zc4iD-yRi!#i5K$xIuqb2`}i$;na?+PhPlB}&CyA*3)A>#^=sQ8XztT)E?l-j#-0nr zS%EI!)&&O&gDpf|-I^J>T{e*8k(mmeKv7V!l1Cad1AK{n0yr`Bu0a^tDTCKUe=LWQ3K{UDoiG7S_ruT$!sO4s6Le z|M%w~fdZ~*bwiJP*hamVO$VP?0^RP8!pPXx?y7)IT#ObAbc>ZO!v8{brEDtQJ%;uG z<8M?)_(pY+e-qXJJHh2Ye-{4&>NOhwDd_U9LC4L608caJSlHLp1kN*9G&c|kTcmj| zy4Vs!Mu}DWm6Kx7VA48=?PF;x_5S&!!+t!EZb0lf>0CpS!Wz6~QDGW>K?;(OP6-s!gDctHA^&O5B9`U;h(L5#;k$FBAm{$%S z!l&r_KXcg|BFfUa-U}lX4U97>H}-al$D60r>r0b$N`9>>Q%}a-@g*vulf5c=*O9pJ<9FdOG?XI+iJZ^LX10I~;_7s1-cxEJwgjux7BnQR9^$3R0pV{-mnAPNRW?P5&V8XHG`2S| zaRa)}96^wpp`wFn$Pe`#PI;^aau`ghR_fS-v#A{C+EZ3JCCLG0WssB_A{plKP#nS; z$dY#Ui<)kjf*NJ(*MOzdP(<%U+y?{P_VzwyTyYZ3%oNedM)7=_aNlkYDoWuVKDI z5xwitN^P*A?U`rnJUTS*)F*BhTNt*9>0DJ)2i}N^**03@{go#A!z~^G{zYzi)~lLX zwTHE%t$(kbIG3QWzw^^)^D$Lr69>jXvu=$NFIGZpOZGE5T=yzq>*%6b?XIDk(uRxK zPRqm@xbbK@{PLqy*JQaeR71kTK|#S=AWRkJL(sR9K0)SD=2(d^VsKGKefRUjujNA= zcF2mUM+2YXXAdw2^C=4br~Od==x$RdO_xuV~ zXdk%fsvOjJyOo{9{4UQ0n0gTw11}wO);9gmyE-`6ZYTMB{psOMo9)PbB882`UAqWO z7t2pqkX|2qK9`%s8BrpwjVG~LMw|~OM1O9FOxYU1?0V=Z8}K-^uBH^&eP)AjS*`3R z&BreIWaJ*VPsi(dEY1i|c2jJ8SKJB0b*xt4kt|K-uMH7pD-rE}qn*CF{Cyy0xl6Nw zN32^<^5rKviLdPkKF#|nUxn*6IC25Iv?3GUn~QtXbiF{u><1T@H*Xh?ZM>8w{Zk^t zGgz0%DLf(buh-gUjRlx22sv^$r;N>~iQkQe^ZNR|#g(wthq6QaO!=aS@g-)?*63;V z_%toPV6gRDxjVFz0CGS5V{M12gTs&Ab@o?;T`VQATjIE&-3y$9yZqg~^PPWD=GH9q z(HW|>j#>CtcF*}X4E>)V{2yut--*7PvzeWlz3V>-=t7Q;wpJ$pK}!FRygXK=-x`-0 zsry=EYM!~-`K+X8wFaMxY-gDPO&ZbU(o;i}HZsw4|L0$TgmxJgjwHsAb&1#YbXEp@ zIbl6%Uoi;aGDmuGCTj$)-vwkgCn_-Z93ciYhflDw`&r8Fe3u2xhK*u>^RFD#kt!HM z>7*=X(5&;Veo{HWx-o?{bmmnR+4|Da!3uj;US_Rz!3tDmwML^3nU{nuL8dOr?$4q~ z)I6(hMI}h$qq7Y1vS7d!A0o2wmJD79Eyp~Eb`ZivI1T%sM4L!(b1BLvc_3Q0Cm)rH zYp7(y{jxje;1r|CJD)IAip^+3u8c()aVmt}=$B^41T>j`h@XCav*7?J6K52G-?l!o z_zO?#{Gvw56me|weE1c@#t6-sC)!Y(^_A*JlAt`y#JHxL@gdD&-KN(rAx|a7q!WBg z(n1VxOr}|~R-)p(Me(fBbZ5a1<8i9nE9>}loCXs0Lt1GdP2`CyO_SOY>-vR+3CT{@ zRqv=7529UdQzcI@60~c^_EH7N8+MJJZb_NLONMMFsIW}pt4aWN+%Jkf7&!k-h=caN z!kEmixUD9pe5YhPny6mC;FmZ~6!^5`-d|llEa8kOukY#h_-zTr`0ostG;*<2`;O!CAJc0|O=lK` z8OirpBW5UlSVmA)6yA79XR#cyP+VjVZZojv%D(>6S*iEys;P4l1X|$5&=vKh1#tA+ z#e9u?;#V8d56V<=nH*S&da7(`$ePrN)HN6SmtTXM0FUNtp(w3Ak;*`%f%ZWEQmA+B^W0G*f-SZ%+RW~9?RZH zmNxX!T9sr|YH?O653DmY4ub>n>aZ_bIL*QmhWH)$%Zw{5S4c&3UlGob0Lzrf7^Z<_JRB_uCEgL%&O^y*}e zYd-Cw*oKrgc;l!=H^z@hCLOhgE_OMy(>HH>Zpi|_?m6}=HD{>Aja$@=r(X+5bhL(i ztK0YORsML)L2PX*t7~c267^E}f_PS_Fd163EEBH$sGuSxXKezc)Mn$ZPB>+cj=UDf zv%e7(H4UpfEPMH;bSG%S^4L!TmbX&a<+Odo;GJg9ug70EJQwQ6HtLrMb*kEJeCpnI zWmB&b$7@u~IxWI4&yHUVBV=k}R^-)|KW*igNU3otXz(Xfx-Zor+n)(Lu)_SZ4z2s5 ze%^)3a1G-wY;p{!p;P-)*9&>{4f3ys-%q5_j?#Jmjk*QqQc`IH+nin~gTz^mym>}S z+U?{uZwPvSz7}zK#bN&UkvCQc2pa zSSKg)6%5B@ysW%)-k6&~^KqS?yndwKd)6F&mHV1$l81H?fD?&=^pJ|^q>MmB4%H2- z_Dn3jxwq(31-UEnF@pNgw{sp^!Bdebd!^$y>zRJZwDiy1Y4&39jej17ISVsHc?AohP}6-(K>IDR)U{ljV6*vQ4~KTA03b}rv*`O~l0 zIGhnLs-I?t3Ly?gLkw26R?{Gy`wB!Ckd`f#KaB9TDJRU0RI*ed&x)U!Hr>cLdv(-6 z#0Niu1|r7NFESHRh@4T$<#op8l<8DJw0oM6hvHpZZYWAobOwF<=)mBNG6h1?OJ3ZM zT~rXo4d`%NM!SuPa_>_tSQWTZ;+FfVu=JGp3q(w1;%2O3qkX>*|D-6gJ~Ma^i8Z4@ zJ^}Azv3-0_YA0Mq-t1fV)GeI##1q09nrg-o@DKeFV+KtMI$1D; zw~_4$5RNGQtfkU>a}^5G(00QFiK`ldWDe+#U{L_;fg6ea^s^)GnYPh zgC?%qM^|cb$LVOG*{vV`Onmp$))zN`7^4$>X&VN&^5kiENQ}SNGVp=&(v5{Z;1D9d)$WGK2Pw`U){N_6i_w6 z%Q}crSoy~NVYg*Y9FeFnwFx0c+?_fs>&)KqnbwZ=M3~XS*!=UNYqE~d(t>t;R39V= zuofoG!IkqrGcH!;R{VMwxUAiHTN7Bs>j0V~ESvJ2R125rEL44)+W}ptEj-$a+excw z)J!Mtbjm#%67-r$ypFQ!nmV6q$`_0oZwisfjZj*vIT$ta3-3Fd1I3!ZOl&ow*;URL z#Dg>$zZBG3n`+4cFKpU_J2AGCCL@z-AHYoH4~UPl9sbG-U)vT(h5dg69&tSyv5bu`TXX|l=x z{CmnO$@%bbic=TxVj&%{rODYmM0C;Xg@}m+L6aSDo)gZ-nz#C(u|1{=#W08SF-yUx zSkoj|GUVXc8e+4UWmDG+dkp#ilmvi)RKN4dZzTBVMe}|4|5qvoeAkT4tt{y598Asr zD@pi&$Km`xiu0YCeoMvvk3r4;?}M7UI?y?rIXbvleYazJ(S37<-K|W_=uC|4%xvkt zzsc3v!PeHyneHFCkMBQ|A0Qxkfa8D5olyVv6`ajn9Ne5u%v}CC0O~&m=n$wANx=XC zMdAVhk^FxT_!idvX9DhJXB(@vX@9_m)^n|*86fH*#uR zh}9%>sNbFEve1|H%9FXrgx8713KJqSXYCj-&);9*p1YGfqmj0zK|ESnHFf$nrKe5I zDCYNtrkYH(MWAdZoNxwP4#cFmMthNL8uQAcmlPO4C!{Wt_dym8wyP7 zG+tiu6UYJ!-k2rnb?kQp!oNb3p>>+0kW=b>YsE+I{3fXSs0KA=&4s@DE1^@_7`a`0 zCF#txcg1lSI{-x=NKx+AjdF2|fkw8D^m@U&PFrhCH zcgn_Q)T*EsgtE8L@1ot!tg#m4p}H^^!aG@?>clU;NwD9-z9N^+G=_Iu0*TB|-0PIF zIJ9sw>D4UsZOGh>QMZuab)|z)WH=5PTEZVqiSE-^ZPUwiLvVqm`5#L6S~o-Z?b}hy z*k1Z%UG>%@G-}YTQ1l$oV)NtAHkBA^Bvl7OQF0FWqlS=Dv~bWH0>nP4(rFu8Jc;U@ zL)ZwIGFxb?2c zDAp+2=%I6)<2L=Uoqpp7vwwaSCeapT)N8BK_WQX$jNfXTK8MkNz3Mc2pr`?>$*P30 zObrlHoQjUf1|g5e)>oQJ)YH~iG4#6Aop_vFv@e!pm*hzrvMq-CM~Z|irK!bM!ddr< zp=oa2@y5ZE!$?;2s75l@27bBw9hqVq$KWl-)%gOvogP1r-2J@yvt}t2>CoY+d4#H# z_yz|?9d~%k5L+?Wh1_&Hnl?h)#BmqtK&Gl-%6dq`Qk_Qx03Y6WJic_%vqAYTk8{`* z&cAS>n{2EK{Y3F9k`lKxR?-{H^L|&pLn2wT#ZAwNVvvmy&^Z~nZ zh3Gz^4|nomzKC6Ym`xnj*nvbxBeV#G#U(#f#8GV@@948q=2Cy-4RpqoGwoXB(qZ*5sJwmoMX}3igb7>z56f_9oHxI8K{f?a6m12PZ znDo<`0(3EA%%FFGp3-a;P~)_ue=RO^t?OB<4XCaLut*F}i>2^0A6Op26rHo_)mFdL zQ$}PcY8}&Lb@>Ct506pqgtamvO$GTS3FQdvK8M8={5-d|!auKI&0X$Q&F~M)?EK^R zP~(5@NCpr(*w?PkFHy7fYxcX8rcSYW5R>baCsfzep!TH(<|p6DA8_A|2jXjvY#HRW z^D9yyBqD7Ch52yiT)AoGMhDf!b+QnFQfu_{JF1!03DZwlz$1IR3T1TCp|vKXTxuw@ zaCS)!87Vt3cc@F8M>^jyWdeXY{*4t6kJsC$HUP=cXA} z-Q*T$;JJCtcs!U5k3M1EC18$cLICK^yJ`-KtpTKFvv!oGZqs&JFK?my&Y5eW5h(hq z`_D6pZ$4>=F)&sArv3+jnO<#38KC0x<{UF|6`GUqmWPBkR+#Njy02dtx`a{#Mk98q zdk=bbSq-BKhU}Y$q-$~RWaBt#7d!Pq9*d$E$$%CJag1}3(QhXoLX7gRI+ z$kBp@<^(!iqg)mmXK=lJ*tFsSPv4L!>sP_oAqYN+f`k*ZChPm zoipG2oOxc%%#VFt`^O!bJ2H1>#Jyszhue8R1>+-+yqwc z`59Bej6`nF5C$)t9iX{d5a#Y|?@KzHfh&+g2>SANQK^%=a z{fJJlX&7z12{-EVTDl4**LOb_W@LNGf+(?NXVIal!u7y{c1fBSWE?{_LFG74DT-qk z|BFigHucv2veTA;bYgNy|2+xab$5(xPn+ahk`H9ZJgbW~B8d%Mq~~A+L8&1BqUJps zDZCa7M4!w;iPrQ=X*%e7P@Nrp2KRx z=P!mLY@^yuHaob{ZCf(|ner?<{Q;(*O+S;%it9J|{Vv1js*c-&J!(YfgYOIG8`$k$ zULq@PFX&SR8ao}JZtK9!(IOhgrdIRaLSD+5H;veSGzdGPrtjo*^g>+Ex~jwT??3Ea z9@+&&p6c6;reXN%{(jjR-hc3lnd%+N$Es`4;S+fI;jX#C_3I8bJogPL6cniU7{#U9 z87VkKTXjf@DvAXa(b)`1LOsGev}$m9;)%-l@+xS}2Q|Xd*)v8D@C$fPNUL9GJ&L2d z^yaP~IsHgS_xZ(b4`H2bjipGZeu%F+>wf+o*$wiA?9>%Ss&g&>v{?Vi9oM@xjBxy zC$xF<1&1Td1dxfNSmHEW6OF@HMORcn9nGQTpK7=Ln0^QDjNGOTIZfJ;hcaQFpOeCf zwMLOreVd!MDmOHMf4DF{KXQnGnmy@F0*o<@#d|^N4w#0FGJo5eP{o~d59VbpgR=&i zhJ2Z{niVZ1J?A?B# zcUqoZ4%#gn4Z}0IcgQ)2VHkG4Q}9_oEdy!BJD;^H9zw(f;M$wJ56MRbS6W#hi_Kc~ zk6-}}8p*IZr>-`&w{nPY@lTP@__;P3?o1Ti--rf+OpX*=hYu}3Y@68vq^}0gbAcNr zXi6{^9iUzTF+$sho^O$1JBaCV%q z+lZYouZO(VdN+U(A6&8h;IGeI=o)kfABg#VOKZEEf>|jLjcu}HHN*Dddv8x~v0T<= z^I{p4Q5^@I-C z2T|48fm2iRTjDggLx;+_XmrQBde_$avaF@KeH(FZXxfbF>3sHah{^M*49((u%$Z5l zNy$qT!OS1%|H$ki02YkxwL>v%=GMLwP;npt0NnpR5j8aUo1Ee7?wlo82iMOJv3X5} zA-y2%+2OFT*~hNqzs5Y=m))j{KJD`iScSC-h4RB5zLdxiRnb5t{z2~rnsa=3GTu~4 z?oY@Z8{bTr(Wd`&sL{Q#roQu5Ya zwt_sreAW#Lkt@jk>e5e?szr2qAs;^aX?EZUFgS9`UgQVv%8A9P$plm!SkyG$%`oF9p0A9YCG-(i3 zv!35gW43w!(H@MJGJyx&H`FCyqHTVia+ti(X1ni?tRxvvEE8*oa5vU9u~oUMbx@fw z6GFQP9yzp*8@3k+NRr+p`8X&Z&q|QjscN-5c>q)xpRz{s5ipU-cJ07`=_=8ErVZ#X6Cu3e8-bMb{zk|#OpWuwFbe=&Wfnib)XB1y(Z~=;) zPYc-vyRRddA1^PE>OqwryR6$zBIle$2KywyRbBw8cjwpUDSNgxmvZ-viGS~*XDD%)F={&bA* zasM$=aVUlrk*LR9xn6@VwW{&6z@9}9FSB~nNucE9bjZe;Hv7XQcv!dtQp*bjz&}qz zV*4W#35##`#jaARv2NbI;oS{glKwSDP163d{*+tUge#s`jJc1Unvf>=n)sHVlGHeU zWOAsYjgsGbkRV({*^7h7i*)jIWd^;NQFlcI@Lv^LZ4eDErkCy;9THYBjP zf=O22)-u?_>aEJ)H;S^4Ko}x`N%U4(onmh~DX4niI0e4huyDuXu!|$d0j;x>IPijH z_7hmn6D4wR7EM%^iY2&g`cRaF*2L?t^0K8=eVPg1m>O<;lAj zII<96g-3NQPJ3o|L|(zh0G}Rh3#8TyPOEM}Ph3iqY6n)2sA;7wRzo4 z4U1ZPj9U2;x)P=Og(kaX%Op1QTb^uNKqY?gm=J0}F($6r!_e0ow-Qt*04#ACbr1Z(yaYWk_+c6vZx3+3F} zOpESogtm)rd4u%9D03l0NHf?c?ZM6UZgFLBH^%I9aa1R^BS0rFDW0nhFsa6VAg<^J z(I(wF+@|06wDJlFCCr_1pJ@@YGmVG67r<&!T@i;YP*KdYV9i6=i=msLU*EI%7um|U zvkRZzN$bDM){bS$JZ~I`y_x%oy!mf!q-MjE4_{xB59aECaOvS(LE-JeWf5aG=}?oA zBH#rVrBLj|1+fOCut-%843u90ghchiU3dbDHKL7M%hJ2oi``Y_so^%LV>Is4^LZ7NthR$%};$q!5Hei z`kBn>-Ol4@7n?VyFCr(|h=1<*K`SVu8I$+U9sZGUSC1Oh2(g)3zKBla18FGfc(>HfEIQrH9ephnYnRibhcztIm^G#=|BQEv=kN-*i#{KqcvPla=neGjt_S@*FV4aCG($c^t{=$S!W$3wWlK}<|N9W8_ z4n;_Uynbg~*tV+|(*76$I@9dlN~6bKDhOl0k33OGE;=XAV>p&bS|FXE4F_gj-#Qhm zI@!79*-B!7*&0g~DwE-s!$z4oFrST5);>UBj}5 zZ~qp?u(Ha7p<>JULisV1yuQigMnNj6KDIPhYv$rc1CFV{4u~Yt5kN0!h8p16;*tdR@Y9Gm)_r3rZs*ZSC9ZV(-w^fY=Nst)@_&=h~S z@6c`UIg6#t5gzvln-)!bYz;O~_t22Z|A$~X$u2KNc zBl92*DXtJK0}#Eg;lZdteF*n}yurrTl<|0jUa6nu$tf9vJfbI{y)SCU#c0K0YhOLp z0s4$8J1UOFa%xw82I6=NM%%&?ufUtT%V>rJ7V|uv(!i+Z)M!t@i6^2<5uk``WkVBc z%N!d_Y)#Fe*?Ja7Z%1(A6Y<^pOBmx6ia_O>k}z!Ym;R@Ev^udB^`pzFQS)lAl1)dl z22k#2fZxIN5WE!X`}J0_S$E|FB$$SbhdpwNL$C4(rSosHlXv)Wa9+X{2@h?j-{6?@jh&78x5 zX7QoHccOR=u(|kK*e}2el;L672{Qa8rnG=ChB_xLE#{2t(cdEGTzfM|u&EkyxH%LA4}M>j^AH=mwU=&tM{W{}#I^+aOx-tmGPD) z5FmyqTy(Jq_b;DR-r!-bvmx3?mUFShGk+wG?C$QMRmTR6wrm?cu9583SZ} zOgzX*p>Q$=AEa$zsXv!pi9`w6K^&^d8RP7n?PH8>Z()`N2+ zBcSS;Vq!LtwKOzKeTGsePqv%OdRlVE>cyWK#7Qbx`_dm6*|?(87^@kwow%8iI|{`J zlG+6`KWy#1AntyMs?5;=QAT9Qf=q7Lf+0f!K(oreT-!9q7);tq$lj_VA6PRGx5-<1 z3C8E_HBJLJ<5r7Yb7J27kqGRf?Ab`ORtf0kq<6cv=o0OFukv!7fsl@2U)$W-yPD!{3 zb^S*m%Emg*w+Vk@p%Pk6b&}@9u~vDj&Yy-ZwY_AqveZ|gVeg!6*H?{Zt+bX*RAly(_g?guml<>{ zTsv1CJb}D`uP*K>b9PGiwN_b4qSxM~^<}=m0(&&(y^znXH{IU5>r`_mviqg7u00^M zOw>V(_<4wJ0B%=@Dm9JX$yO=~kBORV$*Na}p3E7vL~gK+`STgZWA1Q-abQi}#4#z( z&|`5q3Q{%X+$kXd{RTOJmw#cdWt#UZe-GZeUlDerOU7skE6tY(D73L$fAJt+L*zc+ zSJ9b$nM1W4@OAHAb)+)BqLrxLgqlFKu>#J}2h~WPM9pc1R07hsQBv;mp*(tBpaCk- zxx%j9Tq-5y;ZD!{0eU5VNcSda${(OvOB*Cj1Lq-+U6*%LbJDdR(wDRh4-C1Mgaz|mlVO@RyWD(XTGKpPN2nPqX>!lSd~}#iJR1+ zEQz5RI$t!EW*5k`Bl9%cx9i-5TP|zI7%Wvl z+on^XquFhxx?j3~`~Zk3uE9TL+VkP|l!>r*NGdEpA5%u}t{4g^Ikz^2$@E*XE{DCo;xot({e_XxY4HE*p zWV2rMMeEyYQ^Hx%n7OFGuAD~>>R0MRGmRP&NUh&P!#@-&ePls=NsPwzh?rzYsyEM9Lc%Q^WQsM?k5JX-SQAKh8%#;Y-EWBk{Vgp} zCQH0K-H9vqw4)tX+^$4(amR~2hz~Vq$4_EbFDsQB(|KRPW4bg0%UMN#ABZ4!JKoQs zBBY+StYv!f?f3i)KEsGGX#ELz@4ELrxpH}y_b{2Wf9t1%g0KhiyUPit`2D}?#4ek=KX6u|eUEf7oc2IXE#gRObOuPu>U?aB47`y4#8S%FP!yj(gK192e zbP6=GmiZjEhWLfOXPuMxm!=ncD5ER_N2#FOb$Ttv_l;D_yOF))gEag1dfSH`gaqosyd>aPqcx;o7f&9&gv?1WV-LS`iQ`mje&Qz*72Ii(3l!akN74YmE0hVw zP8>iKB}tvvz_#@9?Tr2qrB3|Z( zU?rrFY^)4AHqj~8@0^*tMhD85kUIc^J8=HF3XwA|C|}xSivr3ZOB}VcH}bIs4?7*b z+JP98o4^ww>qBqd&;8-Sb)Vz!ZU57S=X=8CyEmKvxh@-qYlm9jA?U(;z!$`!6p>l} zD`VDc{PQBhotJA256IMprQ$an1?{HrkvKvV)AQz_G zUW7o!=hM1e2bFG@KgqU4dFv=&|MRnf_&z&zVW_%BJ$s?#EcuvvwV;Td5-W{Q39xfL zWfH_IVzM%2wpqP6@$bs1<)tv=#u`=M4QK+KkkhNsnQ$`(Qd}rx^h64;3YO|n! zJaha0D;(S`K>>GAvLK{So22guHC3SD&j$b#6%@g8zu{(*g77}013XA4Al_OX!&i`B z=M($+Xwn_i>wss5GT3P52CbJo z1s~;|*`mmFXuZxmA|f0{8V@i$fi>&FqVoF7*>Wm#lMrtjASxuKHpTzx%X)i6_;ZP!ShP<_iOHbf zp8eK;mQ-75>r*HG$eh)9RE46ky)-UZR1GrL41-5M2w^ois@^Q&3;1$sJ=|ozcHD@= znJylN4~b?RKEC9CQbaFQ#c)r=a}ijg*-fcN+8J$3esM0hKCl*TCRrW2&iRfbfI0=` zl$$C(*s7Y4fA%}xy}M~6^G6|WE3L-suE9j17M$2smno$Ja&`r0dYA6cZYu%`;-Bc* zstH~)7d7;p4EjIlEh`|QCv?^a@6*{lyEdppYgKAeb5*M^&3KVcW4%+)BeRY;uS+a61{o?McX z9-T|7u|cNuTf=|eGhUuSm|^ZUlfkvE2c{D_CJ>^Rt=xtd-*gDV>W$jO{7TkI>5p`G z6`Dm$V3Qg0@Y!}s z)Zy34u^Wy)Q$|J%Umj9$SVZX@c`<_&%*Vgq7Hh5p|MBhP;;XZ#--U;n&R{xbSk zMS}W&RUw@IcV^UoKF&8Y{O?kVvx$@Qzv>_V^*4S~Pybo=w-)(7-gp0X?RUxKKdt@N zE&tEUG5)*d|G&)nA1koJe`mbm{2#3T5=Z~fmj5BE{@dgKPFDTDNM`>!0tCQc=Wc%5 zhb{pF0JtLhr^EFB?Fj$5(WUbDHU687d#Pz-_uXOo&z7Enh~SyN%rYR7Y>VI1HV2o)JhRMP@@+RnGjX@)Z?`8NGcKn(hiEepgS?JxBX()@ z!3B<6v<3Vq#jv&N{jes}b%wAaWYW^sxocKJf{~2V94}GO59c!c>}Z{GJ|d+NA|2~V zXb`f%2%tE)!|I?Kuod0Yb+gW1PM3$RT2BiEHTe)0BpVp4k_EyyijN&k8zSl>22yl& zd7=VzIgUZG^xe5v#mB2nr1ps4{4kI=enj>dT@6);PyF^cgNiAPcVEaC5gZT?4Epq5 z){}G)sCr?~#F4eTCm4jkrH$2B*utVIPE@3B27f}SiNbnOB_U{6i=nH>Cc-$XJEOm+E=9lqe8v_|@wVq=`c3>%v(D9s_uQnSm zCRf3F^Fa0(!a&tI{iunqKFGD%!Kw989OmSBVBV6b6xoi%=f!FTo4}xF)>57KS7G}f zR|}GN5qFQ-&!PrA}Y-ZeNT!^aMjE|xl*j23tsNHj{nXX;)clfG4%;p? zlI*Dxl;|)DE-5qR4yg#HNMm-25%%3dc!MAhEbtpErbAK2lzZc#W;iLO?n7%qv?$MK zh3SReiGjvvq86{NB;>fQN=`$Ow&`PcMHMCDl`w;@wf=!78Y!ya-6lyyFtOrL&OI+H zdnT}@5?tEpb7t}!d>RB|N$`w|@sW*i46M@;1o0aUV2Ht0W~RV{g}E&u0b}PSmt!M+ z_H$Nl3F39ytjXggg=2Nb3zFIbu{u7W{HA<0>@8Mms8+}JK0$SO}~`wg=vDm7fP zK%_zSefc<$KzukZ;1ZdOuD+`l5~wukYXxkzwLr(S3r^OJ@tyWHA6uQVj_5XJBq{PO z6$L)dE{#Hf6s1#MhW<)uSRq5!=jok!TbEjhDZGyiCX~sf)}rf<5sffEE7Q8jkq2cd ztJLh>@|v!;Yk1iGsZNYEOL8+7yWY#Ve_5tm{tT(gdVIcVcna{oT3x?XWA|AD5^!`A z&RxCm`4!4ab%JKV#4t^5J-abq53q&jHL6m5NV&{=p=@p%h$#1i0fL;v5*|MGtkVcE z;Lv@y*s^O}n!g^DhdfhpvUEJdAkv?)EcG1P3EX33~`E&~jzY|+Q1KetF6UqaiT#=*9-0&vjwfox71 z_4uTv6g+h%Zx>Q#+}yD#HeP!2Jwdmdd%f$Wu?5dMqH zkw7lrlf&cmV&GE4sAB}6!*^&#KBjSi)7m)67!$Pear0rSaFN%BZ9iV=*fB%Qrhz3T z7l)emn|NJyQZ2BLF5ck@P(o^d&F8D2-4vqznu5Ec!@1VpTzQC3y&E<=o@~=!bGdMz zBgIdUO69yc)Dk2EqrMNMS1W&ilc~`mesq0#Gw~Ded9Gd`L+8Nm#DUp{Gc*h{3nZLl z4EibDS5pA?!5Lc#ZHt|I>CSqvHef#9Q-Vj>-pOzW>o8}Hg$RW)^_6ZP%IfxkGJsc=$`~~8 zXE`qTisVNiqMX*yz}YKm0;-=L zLFD;0d{s&i6`c@*w+w305LLSs(#4v{2AQ!8t!Jwk>PmZ)i~c#2qPPvrtD|m!(tqC) zvTo7BrfzWD1vjiQXi(zZ5?I1A$zk)pSdvCk(7)=J_^1LF{-dBhmgsIKL(Cr-m84Gt zzNCrId?5$|j1EtHz7*!kU(7^!#}3!R7*7KtZo>7D=yjElL@FrZqyhK<-YICBM!{Em zH{(-sD^E4C&m#Ch*%zh`^;>u%9R`!`NS?s|_*hy-qHX88P>}QO;plG6Nx(Pzv*FcS-5s*AQ};%8VdMWW>F2uN*Zk715~t>&9Y(P(0_ zOysU!=fcTG0fIL&yG71|A`k;d{Ro5_R!YyVgd!&rWuFKEiJB8)wgYjL$8CP;6D91E%|A6NTqwnh@K)t6TKY# zJs}+6A19>3ytVx72`6Zg005G|$7f@HJE1HhtS6wNEGHxI*Ee|yc@b#|SrJ+r<4bJ` zJ1UXCwv)`FKwQSwW}d!?g0dI<0#_*0;_gvrNJHPFdr&Mrd>bxbCO%Uy@TR37xz}EOc z4*m*aeFozvGfjNi&xtfgDU^NE)j(z7w^2UcCdLT4!4 zTGUQypjiqx%tSuA#mH}YPk9OrvBuK=Vm#QAjMuBty^yO2={E8Cm`zAOrVBTP?6dPW z@Y!tx-t)@a_h5_8w@{}^A>t*!Wxpw5{X-#_6gUCQ7<8zD(nMP5?tKDI{~bLz zf1JOplrMAD>{5uh%42oO=|8{K4?BTB6y4H8U3e$%#m4W|#%;^*_l)(qI|>fWI7s;glKf>zyT!{NLiFh{vrnvhY=)erb3hGXxDjeE%#UcnPD zKxY{)SBspAIw^cKiBajY_b9wLxNMpzyyIfZR`ZO(V~jhDK!_Gg zD3hufa~@vwVZ&klv16$!vT)1X@D7MhjJn6~iWDh&#_nWS{GuRD-B}tpV||=!=)Uig zfS&0r#!;CZq(k)aF?TpH zU1fdM_)|5DJpw~Wt!8rt_hOfq#(5j?o8W64lmar`W0BaYxW--uwwydeTJ%B0UKEN* zeb7dUS;)r0d>rnOd`iIleuF9S=HTP<%VSMT!*0%8O3xEQ7hlJpufZImau}HG7 z+tHSQs4P2Fn$^mb72V6t2W!}qyG8_~iI^Zj&j!8CW#s+ypVC+k{X`l^PRh`pI7;x&x8H=49Zw>@gGYMpq?+Xp|9%6M+?C%uk?DpB3WcD z-={=%@Mj2`_Ij(Tm%%}z)8;fOfI$f`^F=JGDC!Y#^GDMixJv77+*5*h8X@pCK#57& ziS$#(Y>x$kvQg{T4UPEVkop87V@;%saI9MDwM;t7!agY)LhvJ(onm2P9V>vNwa7M* z=+WfD7ZraPTBfZ3Ag)>07dlIY3^N-C0o9`NO=k$qDrT~0fqx<^vX#pcK8f%vG!BmG zQqShuUDWCa2dpLnz(pcI41Y7C5?%^Vfk*nGRA|w*fi1F6>1la>dN~-5qtBU=_3j`n z{S(EmS<>35U<$l^y3%tQNWRy`y4JsqsIZ}eFb$vKcl4P`uXCPGvZ5)KFq1w)2lok5 zYWXpMq>fMRF&#I8aMkhJZLBm#a>Mk%Xqq(Pk%|TdoJ1l{w&o{; zV=%#nhgi*$JQL0+gggs5$`j5IZg6_4-$FG6 z)gu9tL^i}}EV9RfgyV<^B*I+_T5&P8?U3BbEXdKG0w7n_nfdZN#4$-V1Z7im)(R(K z2w7KsH4>*-y*bg-VyA#=NjUL%bfr?jJ#xe|DL|6l?wyp{wf{9&BSH8`Ek1|^m^hGaivA%+6dvo)ZKb zD4!HTpR8c-IsO{|by>Ly#Q}1HQ-yn3T--jGfe`(Zjsjnhsp^MEUD`Ax+PNKp9q>aK zI{4QWT!$AT^yry{QVC&&mb^-y^^V3hKblB?W=I4{vDV4xyIj(6aeN01zQCXLI+e#l zOW88F&%A7G{;MHv@HCIV%>S^$4{I!AI{UdfI4F*9+q&stA zlH5)*f8%#Spfp7=YE6l_HwBZ;{y-^Zwa%#Yemj%6ZzSy@b~Q-I%53HeDM2TDaaZRO z3`{45;#ftDv&O?^)4W`}#pET+_W(X)jOv;YCsbhG%yVy;&k)hBH2ymIIaYKWaGq1+ zvjZ^;YywIQ*VsvP^X7da5?sM@?d5sFXe(=puPkLqvepSSMc&Ytklr%c5vJdV)c$5i zo|WCq6zr6ZthU=n%PiVqyPGO){@A_dzA^vPOEU+`6IHNy4$hRue86bxwB}gMo(639yioNRS$sY2vRbV{ej=d5)VLzfFw<5Q}V?^xM>umQkiB-FVz`VEa_1J&EmXcE4JZs z;PmX5{i1QWKUmhIoK)Q(gNAlDNq2S^K{YFUM zho470%MXRw?R}8cL>~!iq(H`x!|OBB$7j|bcExNx$nu@S&CdA|p*ZiKGs)xQ?cp%| zq=&vSnU%eaef)-9!KMj5ZN8mRW=@?lA=4pu+N*~;MrkOSL0m@hEROz#k|)x)Axx@e z$Y&WxWK4!WRh$bjQ5hUbo%}PJ!?vf_A%0##d9w#a{sXLaYT$Ph*xn( z{6(*gf(+u({IcnyG~&nXl+@H%6n3Yj$^rCreL%$h{jfxbQcV-~Sg>j_m2tN)B!Vba zU62ywB#S*+VsFu@bVesSd*+!s+Ly0K24>NW2emx2-|zqhv7iaa581M}lG0A(ep4@i z;e#Spd2|)S-**`gnhSHB%2EY+85I^t1Bvr@_1UDSEAU+Pw!tl)TI+2wNT!g%p)125 zw6Y7oRUH(PHQGXWV#QIJPWE9d3k?_w$UPd10TDNzKOqdJJ=wW#N-9oetGUsjP%6L^ zDLd^1<%~~cd{FXvkDld!N-E^~M9ad-RVcdW??o`}-#!bc=341PxWgVm5(4l5J+ivo zSqGcRpQVN0d63+`YPcxMqM)d|w*S0&oha8@?~sc>Pee6f_@YFD^xgxCdNnoJD3KEq zK4~~Jd=qPZ9R4$OukzC@tau4LF*Tint|0^-Fa1>0sc|ZbSSee)H{pT`PM)x-;Q920 zTTbEPF=qDtVO`({48`ls+urxw2GYJgg+Z#^EhqgDv6Nav!Z(pohqS=%)l|1z(B51b z&2((++`6KK!umJN^$^DRy4=kCP!fm47m)E0E!4~;DgURN#g7XS;d>|o+;j8J@TRJC z&ciUF(WW1)S5cX9rEs{JI^++D0f{T{luPrdNR`8zi|C`Q4OO4;BCuy~BXJS>vZD5Q zcGu~6$59l$QS6c7Rz}Fook|ocwVgb-F|P0yHGNRm_6i@1fz(MHsOS)-@>m~;4qOaK zhAEgF;2f0p9SGW-{7M6j?hi66kd-3@B7!S^*+?2G$l*wgs1I*1Ecs%$lxX(WeA;uK zh=vwhe^hMI({L}mnKog*1h)7S2fSeYDHtcfl1hS9RdV2A38s4GpoIjyIkSUM=j#{J) zZ&X<1la1SpaU%p4&(+eR${%RK0-bQwZ)6p>UfYz^{I2IB+iSGtC~xIt{XvMahXzPzH2(o>~*hHCK$eb6b`%Gb-$4 znYpc6z)W3&i&wqu;pTYsLb|I+lscIwp{`6qt!)yM|XjHSZ2Doz< zDQKKsKz64COm>I!j2mL}Wm)SKpdd`fgfLtw&C`Sj30>drO<&j27X*Wrb$=RT%4Bm5 zXlVP7RT0eFU3HBmp$B?@(N=UhdGZ|~?!*~=sGy0#ACYPPYNgRA_~;PSs=I*{KVV08 z3i!tqC_%(h6Ts?Va86`2a&M5_aZafmjHm}F3UaW*H65j^H1FjZek^Rs8nNvNIg+CQUkR|2;)#uH4aV1lei8W)q+IO4R_=MEnv|CnQ6qfcenK&xiSR30S3S{@%RdhT&$diKm@! zmWhQm7RO&R!Mcxj3CCQ;udO5r{Dj=n2k7~Wo38m4#P*o=g*NsW6%Xeok|K(vg@>E`INFmI5w zGqW;QJnf1s^wdYoITRVU3Rq{pXbW8|XCidLRl62jI%aC@8m(uJWlY!$qi44-eW$XPu3WL}?|jJ~uZ{vdS~}aCMF})AbULXqErhPr_8p8x+f0cYr2#u#< zXb@f_xm~g)szw*rC&cFO)18F^<=%xkFG}_5jrIac!VjHZcJPMix6GG5`yhKwUrh_K zLTYk5`J?S_as}uvciZuu!?e4WsBAY*43Tz%L@1+2JT$Z#^LXjI*|5*UqFK4>VPGx9duW#=_VHcn#v+q1!lvFlVKWZXvt}C zDg9VS-MTq8V_vAg0qP%wAM2DbK}6dbHBedS-07n*z%fUz5i|)sv}o`sav{ZwW%46u zPvtDJ7nuFfBCRsM*D!EVa2toK(tnN<@`tq*a@PZ|vlcEaWu6VB+$IF0}+ClhW$ zeWhaYaB$4Z^0fxa{sBwHtCtw7jkTrAY8V3M?DiTk>VRneVc7=7Fj4YJ=93>Gs15B+~rXrU`L*NFV=VY4PSXbG;U#_6+)x1A0xlWy# zBUgjE{+}${0Mo}|Ck#nrV?%x|%YiE!&8j2U2KW@Udi$e7-AnqS&`OxLFBD|uKTA2P z_i?Zh$5tiz5bN9&BAjMIPKUs>(T3&Jc#pxjMepo7ID;OH^Dy0=Fv=sQR2+2*8Oaw+ z%g&?f*mnvUic6OHLZ5+ly<)h1JD!w_CkkZ(bs9#K~FGiuJfEgot6Me*Uh!!M(vS_Y*899nt&j zl{O4e>F?#sVK${r`h@={CU81?Yw6TYO)GxKgCt4-fNz?_-!Va6KuAhJOhivuPEA%? zPC!^sNI*tJT2E0Z9?Y%HU_p=f;XkKA?LJt|-E_j)ZBKd=`Vzi)T^B5_E zi*ZR-KtAgY@ea<;G7PR6PZm#n5Y0v|+vxr3{jtJ!fUleaQX zQ~I9#lkA~r=O2Yi)&K#ihg@^x}jhjdrHT+4Ho&kD!IytwK0cjWj8k+)gYz zi*V)cLuv#D34@>MP0({#)JDax+I89d8B5CCuPv5)5gd$EeARiR28viB? znr5_c?%Cn3WJ)O?*CGndjB4Cb;T}~L7fFbEhqZ2rLYZq=iih;XJ}Rtb@;0{}71CGM zBD>;ADI>wXE?^$k65~|JCBb=?ga0sqPp}HyuV2(DDAvWKFsqW=pN3o;3RCcrvgd## z7S0}OfJHaB>(=}JhuAora#Q;8bMPv=JQmtL}>_$<^l#WFfH`|EA1=5s#?CcHw{Wki9rd7N`ulMNVgKw9nu}r5(-jENlSy2 zNFxXmO1IJ}5|T=bpx-_R5DrKEasMCp@os$Xedc}VU2E2?nLT@!Mb&apH>R02n*!Ab z{K~hT0XC_^i0j=5Luce(iN#S36e0MFeHuGwt%35ip2!*Z(XDrDIU{PTL1q`nl!;Yd z#kr5xmfnt}uHtrYf8(C0$Y@Gn7Ht@-)XD1NS2FkRwdMB<&64iPyzf?XQ?4+$2<~i8 zl~%iduTm~|l)X?XB+p?r?43Y3R1tK_ii@Z&IE}B^WvOAn0TY$A^BFRM5mS>&zWEl( z$I{TE_Mq}tQ*37Moe+B8H`QAJ(J1i+oiP2W3>ghGT-*IDsZn86uHDShG z*XY973v2(X1UdYwfPi-f;8ex(mc| z4XPnrWoL}@Bx($Kucj=EOgpN-l(xPvW$FqIXMH90iS4849e$6ga6*fkoxFMs6o5pj zZJ=p&n|QL543~G>?>_nChDi>IFWg;E!f#XDqD+ixlvfRkf0*FV62+H){t6xLRTAkl zE^-^?$fhGdhbf{HK1jW`ZO2u`)v7tW+I<1thvrUpAM;t$>;O59gu;iKvl^Cy2yuPY ztfLAh{_(U**$P1&Ubbh^FvC^~V`H$3FWATsAoF@H7pUtl%Puh0xN~8*ihLa+cV&9w z_A@>yMxb#7PiP`#5M6?tddj#C8$X93WmHVh?fkR+UGca67$ukb@zI}9*sgA{(;#7u zwf0cqComKZ@{5H}0Pk?<9*)Sc7WnA~WpSFxQ!n(oARw!Wf3n76d3WKn7uvG>Seq8# z42J6xjy;dK!*<6zXXUV*>|#>uFYY7l${a5WwE_9JK!+_*rsY`2IZWbp1Kn-ul_N=x zp+~i2^^#%bkM%28NomCE2Y#Iux2(hL+0sgF-iqc=Y!Tezo%jFgSu{1+99BG4fle8( zTj4_Ao8isGw;B7uCoQ;vRCafJ>M}Z($MQ03&dLK^nv6WUkgQraCX4Bn4>GN-Zuf`0 z_;6Wr9@YZ77oh)%m)O_G>PiVNO#qGGC&B3D<1%4ww92VS|(YrHST2>vcI~esOQ;f1hlD^}ve5oH+Hta36 z?A(`EH z>%6`^Sh2A<^Li~JMAtX(P2$p8#oLBm9JKnn^ILYam#k)lzDqDIq=lZ-o@NmH5mS4& zn)Y!TD$~1N>4!hUzWK)p%xXWpilL_;irV{8LbFT9-{*^j{cWQSE#u|LZ(Jm2jGCn| zT-XiONmq@T+CFnIqr1+#B3zMGOT);KF@NW#nA}b!nU1S(HA^IBZ5C5-r@Y=1#dxV7 z-G(*z1&`%7ee6w$a9_Y)mBvA@gMUHRsjT8hA~t8%krD^o1NCki(@ zfjlY^&-!ugr3ky*Cf2?IegRM4pCg$^rWW={t`oVEqJ)3(@)@o5`g5XhWZ#5wO%PC< z$`|^r8{A?$kCL3l?e{^>u*{0R4B;){oneNcpN~bAdd-xFBa+xXgsqxA34cpW?tHBj zb^Zd{2$Z{Rh1ldqHzw)NQkvrGG9?5Kow{(ky)iyLLXkUTWZWQZezb+^Ji91wGS&jS zOd{1R;bIo6*<-Y_i>oOcgCZ`~s@Kuf)!8)Cu!Rszu@v;I@giRF4r(T+ZmD{ZuBczfgJxl(KY0rtKTK~~0Puc-2O!X*l0_Cz@q7Jj?l zDU*xdcQ<11EhfIvb=Flxs&Ka!$TyVA(TwWCx~iGgO*wR)yCOH;Xxp<#xW`Uz(5*y) z;hwV?Hb?ll$gAiF40FvMn2qjCV;dXxM&}$pQTeoV)a(4v&i?piVS(TBj6cVzf~Ys& zCu^IdRHXP_BOO$?YCD0>0l^rSb~(fASLy1k1$Cs7u#!#9t5$eqiKiIYOAD*rZkiIE zUwqLSsh)(O5wm+XA*+i<7ro9&MS1^ z)1G$2Bqw}bB$RU*iLz0t!mV>@G%Hr}lLA^gn#J{S88O8<1OrjL9hC3!udHcYx4BK0 zeHk;O@BH%fb~30D{^*;<+@~hFCir3Ak*-y&=eo-TpN|5;hIAd0VpHsLxPSj`hF4!U zRQwfsgd~wdMq#PY^OSt_n9yhO=MyezL%v_OyYw+1;tt zUgni8pR%@|g5jvc4{h0DiRX@;7A1qat5^aPIH&^ys%K~>`GRSuZs>JqT*;hC zFg;s9lA(q1nQ`Z-lXGrKrSr>@f*l2Ow;&}|Ej10ipVt}_y3I1Je|-CqT(?_OYGva= zerF^*!~pjrGs&%YSI~G~gp4$=-@O|gVo>eDbNTaR;F+#-R}vZ)XM-B6?&{s)=K8AQ z<4#fT^ouIE5cNZ>cAv@T`|{SSr`aYhERiIVMAFy@N`1hR!Rio87O&8V9Z^3FXa-Rz z5r$=Zn&69YsH~= zI7olHBZZ4{R^Q`0rLec+BmQ-nhmB)R+UmrZtOD0(DWW{%SuNXih?+1S43*Yt4!evz zn(*cxuF-USE3Z-_Ty|Y(h;9z!(R6Jpe&W09@3Obw<2v&SmD#qgB-r`oA$xP2Nh27u zP3S-G2;MEKb>@Wt3r3K)Q!_a+_5#F1QS%Z#7+>v^bk;Cg z{Th(v8HlyB)FjNz=o7iwnH^;p23P8;+(ZLol48VK%RTz zm&$L_X}DLQ{~9195AtnmI8~A^>k)^8DI4`|I=w=gYJ2(_g0X>odUD-MUUYr`&vo1Q z?b-6$mu6grb~Z-XmtCrl9O$F=c30D?w21tG0{|{8005TG0hEMqDahUtR~1kYmz4q6 zpChW`mOq3Ds(;913Q&r_=($lu_W)7j4y_%AdA~sMdD%1a4L2sOjr)C`jMpO|AYzj< zs6W9Kl^Le2LVxZv7E3ZTFqm$JxD&GN_}+4~l6^ID%%%O!0*X7urHRXY#K!%4NLmHC z{fldyL)K_K`8h7j1}T>9%x zQmuVMP4%2ni%hb@ioUUbT2e!;LKQv9<}Fedck1%DSTscs3qroP)o^a|ke!aVvHs%7 zVQ8Fixvj@RTD**!%S}r#M?pDqlB3sB%XMXlm~3a0|G8io&On^$9PXKlsRo|6MD5h5 z_)Oc=r9;F!n;unn)mAbXnYsd(?p=M(;xNPM;jogS_o@5&Gi;px1hTJEPrcS!o`1O? z>r!$1^2M)mS4CrQh4OZ=>zVjR`f>4yk2_3Tf77y-=Hs$wEN{@ScPO37-}zKJwBE|E z%LPL=22-8bH)*267d}(4$;hdB%{+%&@Y~sj2`6y>W=Jn9t*22u9u@&x)xUQ zJuVKE)srFDmvUwo`Hsz7-;N)7^l3We{0!xDGLMRQ!>1j6B@ps2Airq9&!G-;$vVyKl8DMgqe7_> zO=A{TEEDV)lacX5RCX7ksn^u3WmWl#LsVq{Tzo^#_+SnJ~67p8j7UBob zU6VSu@Ui&gHkbT`u)bfO`i?6R9xH`UUJg%fd-OXFky#~Ob5@#orj9w_oh`nOn&$Dt zq<#9QZ&P^N3YR%4o8zFF|6=?FGRkTpu|?OL6ud2Y0jtj6zD$dU8Ti#ixDzByy~rOn zAzh@zizJ@3&-W-BGc?61KzZraH&&rl?Y2Y0%H(`UzE}9?C`*Bd7ig46Xm{zzZPRd z@?OX>xH*36&v~8PK3)_Yd9{&4cyU}Ahc>HH&ief|!bgf{Kljo!fG@QzU_L4%^~#yhX&^*?;WG~ssNkzLAk=r9VTi~Z_Y{k#@#F> zn;bO4ctkDL*{X1BJp+{K3DjNMr&xm4=OQ_?{C~E7`G&_GQkk^=Bsu`4oRo;iUk6#e z@Xk$fo%Fto^ArB&HZ(2RXku8gro1+bnoAeRO0Z)Vr}Mb_(s7x>Ge$ZuNr@U67T(%? z{1GF2LJptWm{+HQ$Lf(&S({r)!w&>vEC;Ul4wk0B03O0}T@x~!V$o(U#p`}s<7qX@ z7oPZ>Iu~a6Okbh&+&zKl?}cS{NR+Ntgj?ScZ{Pe*JC%f$0&vg#^!!Xzz;#hdbG(*t zhRe(ckH&Y6%Ugp;+OCN;yEguG8!v5P1+UJkyui2!;q#mN-!56M)+nlP-lrO$Mztsu z(^1c_eW@Klp`c5KDf(<#hjYBg&0eFfEsh37149J{YD0bN9wo=GG&!Y88bRdz5)`M3xelsXp zd`Mx@**eHQkVOPuM|#$_)hwYhKcv(2Me3(CM7;)cZ8g)3b;Kb7b3fgsq~h(X8j->e zA{BeHu#g{nPnHDL6uVx_i-;ZQGnMQxR*MKhLBL>GED0I8>rx#=KYx1Z`Q*Kc1xss$ z=NWxIGdqjump19>pDKRNO?Yn~QGy}g)){c=!|S38!YgTMv6E#9205BYo>ZskZMx44 zD8FFtEmVEWl4#!K{8DW0G7T0%j+RDl?ng3Z!wnO2x5*lV5>=Pk3DHOrn(kes_2Rz5 zMZ|k&%g#`U8->$T3@4rH$l4f{FD_LT?%N`2H#7c%z9s!`F!r>&DBkk6)va#dx1PG$ z4{_9SZ|SkVcIMB%uu>(adsb;|S#e#EN<&)^1CR8>VrQVWY{imHc;`^FyvMknVpIWV zDi%Gn=?hn>o2wcHcQ^7#YOuV%tNCHBEq&LR*NA1)g0G)Bi90z82GjhV*g|QBX}C*9`k!;lR`?;F zI+Of7_CpGdUU2*=wg=PBUNJmFWzIi8-;^fp6d#~yohzXUrQAFfb_X>LwL@dz%Mb=; z3Uf7PRVhFm6NUFGh8tz+Twwl4y?QP8xrH{#v(b#2=Q-G7#*<3gCY62ji&5Y3zVYPn z>Ay7jcE+N+l;^sI;~Iygp5diO`s#~9w$v*a!S*Pnf*n}NPp;_{VJuQ|e{a^aB{g~| zN>9pVGNLShO}P_GPK=2xAmwG5s8(cj=Qq>n+dkhaZb?T!;rF@d{?G@PhW5(~@+QNC zm`97Bq77=|2GR{xIk{6<`MI-In-Q7#j3m-r3&q2qK21RGiy=R&d}TgUP16twJE&5B z^&^Y$mz8~kX~sWIm+>%HaRbuX6EYJRB}dBZ0|s)RAvo0MRT zN&1Ss_qnZV!NO-Mq{Zzvwfcov#MvfQg4cNq-{gK4y2<MvS-K$hwERtp!Mgs;*7p<#gBOow0L|{y#-*&9M}S&v-{-;vE>8EVcO_a0 zEh}AS+}NW0n~2RA>6qQcVS3~FNOG%`?@+|CyuU>0&QP0+NT3&?O|EVU)gUw{YhS8O z-=>I2)Mpb`4%aTZ(wW&GUAXe4@JDWBx(kt*F(R{;NOA~ym~bJzCE$h0^R!p?Iw6g! zfQ)jvx|+4Wwt?<*r8PdzFzv?AIiWIqEo}NGIGDM8A8(4XvlDzA6xl_%-f!>5yNnPN zfXPy!cq;6zSz$B&Rrh7dp;|=Q04>QobvItfJmGAr9A=7T>H5TC^s>`f zG@Svtky&4lA`vjf(y#G@^Ng|}2$?<3cFobDP7(YQ56LY}M-ZLCkRuoam# zlL~5h=6bf#NA)iTC{^RPVQt@j>HySJ5su)clwIMo&A2X&#TBz1VPQ^?>lJxAN=N~D zO#8;0F2+W2UOuLusXVTlJxD3|^f-!=GN?E*9T(q!YC{`VV6#MEjh)oI z4)7t+ha1)6s?JrGx}3Q}XXiDyLEmr@F>M`NvMM9s zEYr3kFa=%w#^gq{j$C9$Gh6Z9D!JxKZNrZV>XqcX+FYHBqN3^jrXhQG zIWXBSu>7a^Wix4;bN16~k{25n(kw6alhBAhmDw7QB^jl6;2tj$E|f1l{~-GvvutDgzo#X2)H z1}z4d2TzH;MMGqlWfN7e3i`C`70LRMEk!$bV_aurVZ%u_lF5zGN5xuQF7I0NE5{Z^ zd$XbXytH$dE@lyRk)*{eD`&Hq4q<>rZK=4}vf5<3u)GG#5oNET8@lT(oUMGA_XJ}n znozz}L6L!qY*Z-oyXVi2*ROt*glGx$%=>fbRb`yfu>O#*Z`P%~wmZvR{p^*3G>$+Q ziK^=l|L#wj!@Z9$ObOf?_*!Qh{7mt^D{W6kOotn{0WV%q)#Pf-r;wMqpIFSl>9R|q z`)$)ouD$Q)b{j3f>2V1=If+jPx6U+wx6FEiCDLBVTtzdHgiphX`NPl2p(Nzz)3-!K zbHyv2?-5WFg<~rlTgn*)BX=JD+?$=JCRHA@d)`N-S7LYNfgj z&cfGxvL$2M^`;yR)q>7QJ{!FjhS-yMU2>$5%Mjfmu0QiNZJ-98NvLM2fv%EW-hB;A z8j?&K1(U>0J-55PIb%m|V%czZS5wE5Exd^a0KQ;V7sx>+WjO%_F)d+b0cklYVR>*1 zU{PhLyBoYzWm$e<-1n{3*99rwK=e!}qOUyIJ`Sh5ZsbZoPM%D&oCsv+^S(R!orcdp zHoPSK*XDDwhB{vx1Zr_d_6n^kvm2u0H{@OYzs~lyP1EXD38I&Ii8@BBvT?IA8rl*> z+J8oO{qdC)Rp7Ipr=WFUguyyT?VA0#>~N>J-6~sHO`6wC<|yq=hMQ+EG7n>=#WZld zcI=rw|B^DrjAWZ{J)LX$wC^hgt?Ep6|ESWyIaAk%%osJ(L2^#63H4M=RE6n1Uu;YC zd($jtFF(jUMMSRd*_vQ}YQ~MM1ZL4nu$tMy$*L2lRcpMUI(?m>*j zcW6dHboTt1rAR99)%}%UV`p(&7+J%kbSZgUs-D|L(@O=) zxl)t8VZG*F;yAzbqTku1PqI)naU-vVW8d3c@=l+=d4J1kiL9c+m&MsNIGCla^enOA z4Tihfc^#>(m(7zoMR)Z&l-np%88t)h>&ia_a0x`AuRllWrSpuArICExozw*6jV%_f4s*7w zEmNPh4b}M)xbUv8Ly&Qutw9f?3uS6Pr7`slZ+cWD!mU8TCPlGj!Yc{+Siz6m66!3et208s%pX7>$7=DVkT7TA4O4D4LoWF8giJ`p4{e>w-E!5vi*T{OC9ZzQK38Dm33KWWjsbmy7og!sMB z_$*C$ISRW>{AWMoEODE={e6SVZs~Zzs9eots;4L)B_<*y=(*pUJIxJ~=B%+!%gcQh zy1(Ff+ReDxAJ?Z~j!1_zn8U-u@!IP21n>&wF+W0JttWqT_CUt)o$P@Uhato62&{hN zi?+|mb&TSX9MPgOnBGKL)CSLqNP5>^=<)58XtN?Epf9%N*RJ;3lDZM)?ZPhf%Y=oS zgT64{ZtfTP^IY$Q2G>P@ulimrf@Q}6zG{uT6`_8_&JwS!HgP%4O9@;w0OQX|LxjHOKvb#xu|H_+}t0LkfXyTi9CW$449V!;@ zBoEb6VPD$q36_a7Pswl_roxhLMZTe8=+x6>P#UuQl=lb80PErseR*%8uD@Q+m~L&& zbd)IGyWtN}Xth(8_u}PpD>X|en)?`;>3VMtWaUjaRmtqUxsJdj5hRXUCDLi&s5&5o zK@#`6&fRGEQO%;}FFc)}#i&#D+C)pkd`^sI{O5+NPTfT-O?o}Q(un;ceakhV@}lkZ zD*JX|^S6lBjPH~mO7UK{^bDKs(r=%`=3+;a-R5;-n&oGEXvEd?MQk?k=k~Tc?t2M* z>`uJKbeUPWjqh7)!0wviJ@d8*-mPY=s~TSj#Hd~k#CHWS z1%E@)U>Zpl{Ixb-5p$(d@aMW7O=sM4)CVKOsPtRu@17ER`Mkb&hxvi?2eROy&E*Yr zGWypoy3_Y65;c_1C!=ZApo~Zk(A?ae_}boy+Az#&uEEr0PV7TU%OY9#Ktl8J#!Fg} z7k4)dUlwX}6{c)4s8l8|4mbY9mXkz4W{(d zJ`BpIvd4huFv6I{J!l`(`1e#kiUNQoQ~O*#zFh(;!B>(_-gaw48A%|y=at11$ zsYpO=Rweg1JM_ahT?du~Un-l9`vyiCRnW@kV47}I1OOWYYkT9pqVu-&zXu}&SY`+Y zYt`K_UV9jUdWwPw;!RfP2jTS51?_M%xgu5r#L)$#%~=?%4J8Pct$`iAt)0D|DZQ1B zo~e$J0rWOi7^EWdCWs|i9#7#ykos=HBSqU;sYe#iY*&GcJWH@L5H<{wzcK*G8$eT3y@k` z(CgY58yM36zQdD2|5#j^e#yfR(W_+-<=}Z3z2Gm?L;j5hnbvY4Xf}GVGY%e7TP$b| zip)i*6|&Waj#&e~)kCL(wA!)Xs=1D_#UGo#l`{`*vaaiYH+?8Sr0I>LO+zo+gOPuq z`jGH2w0&W+z30CK!GjTmQfE&PDD{KO{Qd;7X|~t&0o1=B4t*8Uh1-9E*uS6fPe@Pf z_mB<{k0PiUwHOn-RMXBV5qjGtB|&Q8YqJSkI+|k*?-{ zph1i5!eZ*6DG7o8TNgD7HP=SmadV*q9bo-30xCpVS1^3F-vI!o-$;1tOT z!=T0fBn;&DN3wKvurEe7@*b7yd3Y+hdDQAfGD$x2AeGs30Kg7ICCB|FR1^&K zg)H^#Ar}GvnTV&{2Ss1^VT5y3K~PN=^hS=2e{{f0<%)Wzj>9QkFg|V>0>CX8A_|XA zNJMTgObb~$T9{et=-+~rVg55Ipa9uO4$J{w;NON$p?ojlNnOdNR38u#nD z@xhQHPWZo(`ZFKTB8OROaJV$AKkjlYmI-~*sn_zYU4wv6QmwrnFi4%Dx|APv&*f-43 z^>h&LhGZTgoMU^De18ZZf}#W6V3obW>Qo(7F!{3w=RVfopyVLv^2tC2Z0}p>-GUN; zAH5y9DskXUC7~J#_0+a%cq2f1u6rtkqzwT8^uQGgEQy%cC+a!WG5GyM>63Czu*I|?>t2_}2St|nz0W0Ib&;ayGpF?se6|6lPxFmaqIckt> zPzx#AMLTMc0``^T1BCkRiD-cFLvHUg3OgiBP(El%+I&Ht0nez|s7&m{R-zy$(pEQ~`v6H6Ulh|q9%HJqeGJ~nWn zRSOEi{TmqKne!(H7SK00wg4@%PXtZ{Pxf#LcD}$jE7GaeUl>9I z-c5p%tRPfIS45x-(LYgvbpMHb`P(6I@pB3^jR1^U8fQ+@eVEwXI(JmHA|K$8`0(7Xa8fHt@mhb-t|)-OjA;yuwp9|T7wj0T=2 zH95i{Bq!o$=x8{w!BGuDL3<~#Uww%Wj)Ye556^FCNZ8EyFVN6fprdCeqXXITg!4Yn zTbIv3JfUB-2P#BC?{QB+2SPwLhCO-S>>!h?;Gp1uVMDO`AB2#xv@_N-P`GdP2PTza z*bpXR-@Bnq8h7EDWIJ6_7wLsL7#CT?f$Lj@l>!R7L+o`QSZJl>$%|*s1N#)_>pvo#ypH^ZZ$u=GTM}VQMV)OmyoT zJQ)J8a>-b{;;;jlc$tD9a>K?f{O$ir22#jG;CCGpIDI_rah=&gA0}OtfaC%qkpyVU)j8?pi-x!aBr9Z9moO<@Vqc1YZUiCuz{?5 z!F_=H`)f{kafBUTHo1YzUdBCpy&8lU2NHXo+tZc!!NDa54lY=KK1cK)=)g?H@^)Yt zgr5!$VzM&lM`-Lhh|P!N9o&mNHXXp`&}GE`retGaWoc^+t}tOI10b8@Dr7R)@SB1T zpdH%1N#PI}9XJn;Z7=UBA$?E>3NRlLhcV6W$$_C^26klM`ArMnOri+hkTJfR5DLvp ze#*jAdkJRL>``2XHK3W?K+l2QooGCFVomm`?Tzq%k!r%Y4N0r(uk4W$aD*oXF_hsd zD|H9BJ7EHD@Wbjgci}&gIyTb(MQz`dg0ID)D-gFucxpXWRO;p5G?k0NF)s|>$%=N|2~R(5AW*@|RRjjA~LI5400SQ#$ zM}MORtw9COP@=yHkb|Te{yTxg8B^HF&IU}nEdP-14_5?1hxVO28 zo{X8Gj-B4UgS_Je#>7@^==K~&KsQZ=1|`2I@RF&5do;$%X0;!{VK4x`;D=3Q-^iYf zn2?U0&Mh+o9SeJ_zu6CXFOUq`qd9of`MMOI3|>5~I?q!t8e}kdTY$q*5XQ6S}#t5O@AWp(OtYDZ7d zaSA|Quse>r5hvpWlTq(`lBLaGdBrtdM*zKOr?m^JRm^92Uc&+IY6Y9;nMlFdcoTfn z1?$uzvHu5Nf2EVyT}T8vlkd+ks8hc^y_caJ4b>4~(VSnm%4NQTXX8E?9p8+sPCgl( z<3;4)W?DeJxy}=O|M$Q))D*G_b&xC_QUQQIu>S4u3<>-NHZ{X3`yW&wRdDRBU^64y ziA-7W1>aSnIp`>|@`D!uf?6>%+-?W@$$Kyd6oMH|w@-+Abc;n^@1B9Vj)<|-9eayE z?SgvlvkDN|+XdyyglBa-j#m8>L%F9e$O;=|1-sg~Ty-)nV7wPx4?+a`8!5pKi#E`8 z5p(;Xu6rQap@Gl=WpJCLKF=hu2aLz@;0y{&YOe0zr1szO?5{F@k0k*sD>Gv~=&MLL zukn1>&;>a)a*wks1(Kg>F6`OykQC@81ynFzL-E^wkc6J}{l7_pM`B>Nivh}KF4f=~ z;^4?aA@1JMg+oYm00~?Qo)wIKd>_?ZH1br5_u(bkT14B|7T^(EaElOae z*cJ(l9X}uJ(KyiKPzU$}J6B<(=W72q4HrG=3s|~8Y}xOhxVh}R*Y%TNA8wsP% zc$5>O;4S_{zCq{TM!(LzvBF^ zvkQwTi$TZ%_Dn!+?>OE8@*yHRumw(x(dfBdE@1LB#&f3JxS!^mIiKmQB+;N0lHuwl=O+U;XQ>mUFBHVu1TH1=rIe?2d%+p#Ct zNvOk~7j1xRAFNINr!E{o96bM}4(xf+jXyzHfvK8@IxF(-}kZsfM~7U18f)+OWw?m&r67(%c#dAGLI@?gJQjnVMxEt>2Zg^a?kK1hH6q;I zaWDuRT;5n