David Blanc Brioir
b70b796ef8
Implements comprehensive batch upload system with real-time progress tracking: Backend Infrastructure: - Add batch_jobs global dict for batch orchestration - Add BatchFileInfo and BatchJob TypedDicts to utils/types.py - Create run_batch_sequential() worker function with thread.join() synchronization - Modify /upload POST route to detect single vs multi-file uploads - Add 3 batch API routes: /upload/batch/progress, /status, /result - Add timestamp_to_date Jinja2 template filter Frontend: - Update upload.html with 'multiple' attribute and file counter - Create upload_batch_progress.html: Real-time dashboard with SSE per file - Create upload_batch_result.html: Final summary with statistics Architecture: - Backward compatible: single-file upload unchanged - Sequential processing: one file after another (respects API limits) - N parallel SSE connections: one per file for real-time progress - Polling mechanism to discover job IDs as files start processing - 1-hour timeout per file with error handling and continuation Co-Authored-By: Claude Sonnet 4.5 <noreply@anthropic.com>
271 lines
13 KiB
HTML
271 lines
13 KiB
HTML
{% extends "base.html" %}
|
|
|
|
{% block title %}Upload Document{% endblock %}
|
|
|
|
{% block content %}
|
|
<section class="section">
|
|
<h1>📄 Parser PDF/Word/Markdown</h1>
|
|
<p class="lead">Uploadez un document pour l'analyser et l'indexer dans Weaviate</p>
|
|
|
|
{% if error %}
|
|
<div class="alert alert-warning">
|
|
<strong>Erreur :</strong> {{ error }}
|
|
</div>
|
|
{% endif %}
|
|
|
|
<div class="search-box">
|
|
<form method="post" enctype="multipart/form-data">
|
|
<!-- Sélection du fichier -->
|
|
<div class="form-group">
|
|
<label class="form-label" for="file">📎 Sélectionnez votre fichier (ou plusieurs)</label>
|
|
<input
|
|
type="file"
|
|
name="file"
|
|
id="file"
|
|
class="form-control"
|
|
accept=".pdf,.docx,.md"
|
|
required
|
|
multiple
|
|
onchange="updateOptionsForFileType()"
|
|
>
|
|
<div class="caption mt-1">
|
|
Formats acceptés : PDF (.pdf), Word (.docx) ou Markdown (.md) • Max 50 MB par fichier<br>
|
|
<span id="file-count-info" style="font-weight: 600; color: #2196F3;"></span>
|
|
</div>
|
|
</div>
|
|
|
|
<!-- Configuration recommandée (par défaut) -->
|
|
<div class="card mt-4" style="border-left: 3px solid #2196F3;">
|
|
<h4 style="color: #2196F3;">⚙️ Configuration (Recommandée)</h4>
|
|
<p style="font-size: 0.9rem; color: #666; margin-bottom: 1rem;">
|
|
Les options ci-dessous sont pré-configurées pour un traitement optimal.
|
|
<strong>Vous pouvez simplement cliquer sur "Analyser" !</strong>
|
|
</p>
|
|
|
|
<!-- Options communes -->
|
|
<div style="display: flex; flex-direction: column; gap: 0.75rem;">
|
|
<div style="display: flex; align-items: center; gap: 0.5rem;">
|
|
<input
|
|
type="checkbox"
|
|
name="use_llm"
|
|
id="use_llm"
|
|
checked
|
|
style="width: auto;"
|
|
>
|
|
<label for="use_llm" style="margin: 0; font-weight: 600;">
|
|
✅ Structuration intelligente avec LLM
|
|
</label>
|
|
</div>
|
|
<div style="margin-left: 1.5rem; color: #666; font-size: 0.85rem;">
|
|
Extraction automatique des métadonnées, chapitres, et découpage sémantique
|
|
</div>
|
|
|
|
<div style="display: flex; align-items: center; gap: 0.5rem;">
|
|
<input
|
|
type="checkbox"
|
|
name="ingest_weaviate"
|
|
id="ingest_weaviate"
|
|
checked
|
|
style="width: auto;"
|
|
>
|
|
<label for="ingest_weaviate" style="margin: 0; font-weight: 600;">
|
|
✅ Indexer dans Weaviate (recherche sémantique)
|
|
</label>
|
|
</div>
|
|
<div style="margin-left: 1.5rem; color: #666; font-size: 0.85rem;">
|
|
Permet de rechercher le contenu du document via l'interface de recherche
|
|
</div>
|
|
</div>
|
|
|
|
<!-- Options PDF uniquement -->
|
|
<div id="pdf-only-options" style="display: none; margin-top: 1rem; padding-top: 1rem; border-top: 1px solid #eee;">
|
|
<div style="display: flex; align-items: center; gap: 0.5rem;">
|
|
<input
|
|
type="checkbox"
|
|
name="skip_ocr"
|
|
id="skip_ocr"
|
|
style="width: auto;"
|
|
>
|
|
<label for="skip_ocr" style="margin: 0; font-weight: 600;">
|
|
⚡ Skip OCR (réutiliser markdown existant)
|
|
</label>
|
|
</div>
|
|
<div style="margin-left: 1.5rem; color: #666; font-size: 0.85rem;">
|
|
Utile pour retester un PDF déjà traité (évite les frais d'OCR ~0.003€/page)
|
|
</div>
|
|
|
|
<div style="display: flex; align-items: center; gap: 0.5rem; margin-top: 0.75rem;">
|
|
<input
|
|
type="checkbox"
|
|
name="use_ocr_annotations"
|
|
id="use_ocr_annotations"
|
|
checked
|
|
style="width: auto;"
|
|
>
|
|
<label for="use_ocr_annotations" style="margin: 0; font-weight: 600;">
|
|
📑 Extraction TOC améliorée
|
|
</label>
|
|
</div>
|
|
<div style="margin-left: 1.5rem; color: #666; font-size: 0.85rem;">
|
|
Analyse l'indentation pour mieux détecter la table des matières
|
|
</div>
|
|
</div>
|
|
|
|
<!-- Word/Markdown info -->
|
|
<div id="word-info" style="display: none; margin-top: 1rem; padding: 0.75rem; background: #e8f5e9; border-radius: 4px;">
|
|
<strong style="color: #2e7d32;">✨ Fichier Word détecté</strong>
|
|
<p style="margin: 0.5rem 0 0 0; font-size: 0.85rem; color: #555;">
|
|
Extraction directe du contenu • Pas d'OCR nécessaire • TOC depuis les styles Heading
|
|
</p>
|
|
</div>
|
|
<div id="markdown-info" style="display: none; margin-top: 1rem; padding: 0.75rem; background: #fff3e0; border-radius: 4px;">
|
|
<strong style="color: #e65100;">✨ Fichier Markdown détecté</strong>
|
|
<p style="margin: 0.5rem 0 0 0; font-size: 0.85rem; color: #555;">
|
|
Fichier déjà au format Markdown • Pas d'OCR nécessaire • Traitement direct
|
|
</p>
|
|
</div>
|
|
</div>
|
|
|
|
<!-- Options avancées (repliables) -->
|
|
<details class="mt-3" style="cursor: pointer;">
|
|
<summary style="font-weight: 600; color: #666; padding: 0.5rem; border: 1px solid #ddd; border-radius: 4px; background: #f9f9f9;">
|
|
⚙️ Options avancées (cliquer pour afficher)
|
|
</summary>
|
|
<div style="margin-top: 1rem; padding: 1rem; border: 1px solid #ddd; border-radius: 4px;">
|
|
<div class="form-group">
|
|
<label class="form-label" for="llm_provider">Provider LLM</label>
|
|
<select name="llm_provider" id="llm_provider" class="form-control" onchange="updateModelOptions()">
|
|
<option value="mistral" selected>⚡ Mistral API (rapide, recommandé)</option>
|
|
<option value="ollama">🖥️ Ollama (local, gratuit, lent)</option>
|
|
</select>
|
|
</div>
|
|
<div class="form-group">
|
|
<label class="form-label" for="llm_model">Modèle LLM</label>
|
|
<select name="llm_model" id="llm_model" class="form-control">
|
|
<option value="mistral-small-latest" selected>mistral-small (rapide, économique)</option>
|
|
<option value="mistral-medium-latest">mistral-medium (équilibré)</option>
|
|
<option value="mistral-large-latest">mistral-large (puissant)</option>
|
|
</select>
|
|
</div>
|
|
</div>
|
|
</details>
|
|
|
|
<div class="mt-3">
|
|
<button type="submit" class="btn btn-primary" style="font-size: 1.1rem; padding: 0.75rem 2rem;">
|
|
🚀 Analyser le document
|
|
</button>
|
|
</div>
|
|
</form>
|
|
</div>
|
|
|
|
<hr class="divider">
|
|
|
|
<!-- Informations sur le pipeline -->
|
|
<div class="card">
|
|
<h3>📋 Pipeline de traitement</h3>
|
|
<div class="mt-2">
|
|
<div id="pdf-pipeline-info">
|
|
<p><strong>PDF:</strong></p>
|
|
<p style="margin-left: 1rem;">1. OCR Mistral — Extraction du texte et des images</p>
|
|
<p style="margin-left: 1rem;">2. Markdown — Construction du document structuré</p>
|
|
<p style="margin-left: 1rem;">3. LLM — Extraction métadonnées, TOC, classification</p>
|
|
<p style="margin-left: 1rem;">4. Chunking — Découpage sémantique intelligent</p>
|
|
<p style="margin-left: 1rem;">5. Weaviate — Vectorisation et indexation</p>
|
|
</div>
|
|
|
|
<div id="word-pipeline-info" style="display: none;">
|
|
<p><strong>Word (.docx):</strong></p>
|
|
<p style="margin-left: 1rem;">1. Extraction — Lecture directe du contenu Word</p>
|
|
<p style="margin-left: 1rem;">2. Markdown — Conversion avec styles préservés</p>
|
|
<p style="margin-left: 1rem;">3. TOC — Extraction depuis Heading 1-9</p>
|
|
<p style="margin-left: 1rem;">4. LLM — Métadonnées et structuration</p>
|
|
<p style="margin-left: 1rem;">5. Weaviate — Vectorisation et indexation</p>
|
|
</div>
|
|
|
|
<div id="markdown-pipeline-info" style="display: none;">
|
|
<p><strong>Markdown (.md):</strong></p>
|
|
<p style="margin-left: 1rem;">1. Lecture — Fichier déjà au format Markdown</p>
|
|
<p style="margin-left: 1rem;">2. TOC — Analyse des titres # ##</p>
|
|
<p style="margin-left: 1rem;">3. LLM — Métadonnées et structuration</p>
|
|
<p style="margin-left: 1rem;">4. Chunking — Découpage sémantique</p>
|
|
<p style="margin-left: 1rem;">5. Weaviate — Vectorisation et indexation</p>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
|
|
<div class="text-center mt-4">
|
|
<a href="/documents" class="btn">📚 Voir les documents traités</a>
|
|
</div>
|
|
</section>
|
|
|
|
<script>
|
|
function updateModelOptions() {
|
|
const provider = document.getElementById('llm_provider').value;
|
|
const modelSelect = document.getElementById('llm_model');
|
|
|
|
if (provider === 'mistral') {
|
|
modelSelect.innerHTML = `
|
|
<option value="mistral-small-latest" selected>mistral-small (rapide, économique)</option>
|
|
<option value="mistral-medium-latest">mistral-medium (équilibré)</option>
|
|
<option value="mistral-large-latest">mistral-large (puissant)</option>
|
|
`;
|
|
} else {
|
|
modelSelect.innerHTML = `
|
|
<option value="qwen2.5:7b" selected>qwen2.5:7b (recommandé)</option>
|
|
<option value="qwen2.5:14b">qwen2.5:14b</option>
|
|
<option value="llama3.2:3b">llama3.2:3b (rapide)</option>
|
|
<option value="mistral:7b">mistral:7b</option>
|
|
`;
|
|
}
|
|
}
|
|
|
|
function updateOptionsForFileType() {
|
|
const fileInput = document.getElementById('file');
|
|
const fileCountInfo = document.getElementById('file-count-info');
|
|
const fileCount = fileInput.files.length;
|
|
|
|
// Update file count display
|
|
if (fileCount === 0) {
|
|
fileCountInfo.textContent = '';
|
|
} else if (fileCount === 1) {
|
|
fileCountInfo.textContent = '';
|
|
} else {
|
|
fileCountInfo.textContent = `📦 ${fileCount} fichiers sélectionnés (traitement séquentiel : un fichier après l'autre)`;
|
|
}
|
|
|
|
const fileName = fileInput.files[0]?.name || '';
|
|
const isWord = fileName.toLowerCase().endsWith('.docx');
|
|
const isPDF = fileName.toLowerCase().endsWith('.pdf');
|
|
const isMarkdown = fileName.toLowerCase().endsWith('.md');
|
|
|
|
// Récupérer tous les éléments
|
|
const pdfOptions = document.getElementById('pdf-only-options');
|
|
const wordInfo = document.getElementById('word-info');
|
|
const markdownInfo = document.getElementById('markdown-info');
|
|
const pdfPipelineInfo = document.getElementById('pdf-pipeline-info');
|
|
const wordPipelineInfo = document.getElementById('word-pipeline-info');
|
|
const markdownPipelineInfo = document.getElementById('markdown-pipeline-info');
|
|
|
|
// Masquer tout par défaut
|
|
pdfOptions.style.display = 'none';
|
|
wordInfo.style.display = 'none';
|
|
markdownInfo.style.display = 'none';
|
|
pdfPipelineInfo.style.display = 'none';
|
|
wordPipelineInfo.style.display = 'none';
|
|
markdownPipelineInfo.style.display = 'none';
|
|
|
|
// Afficher selon le type (basé sur le premier fichier)
|
|
if (isWord) {
|
|
wordInfo.style.display = 'block';
|
|
wordPipelineInfo.style.display = 'block';
|
|
} else if (isPDF) {
|
|
pdfOptions.style.display = 'block';
|
|
pdfPipelineInfo.style.display = 'block';
|
|
} else if (isMarkdown) {
|
|
markdownInfo.style.display = 'block';
|
|
markdownPipelineInfo.style.display = 'block';
|
|
}
|
|
}
|
|
</script>
|
|
{% endblock %}
|