Data Intelligence Pipeline

from latence import Latence

client = Latence(api_key="lat_xxx")

# Submit files -- smart defaults handle the rest
job = client.pipeline.run(files=["contract.pdf"])

# Wait for the composed Data Package
pkg = job.wait_for_completion()

# Structured, summarized results
print(pkg.document.markdown)                       # Clean extracted text
print(pkg.entities.summary)                        # {"total": 142, "by_type": {"PERSON": 23, ...}}
print(pkg.knowledge_graph.summary.total_relations)  # 156
print(pkg.quality.confidence.entity_avg_confidence)  # 0.87

# Download as organized ZIP archive
pkg.download_archive("./output/contract_results.zip")

# Or save automatically when waiting:
pkg = job.wait_for_completion(save_to_disk="./output/contract_results.zip")

# Download after completion
pkg.download_archive("./results.zip")

# Or save automatically during wait
pkg = job.wait_for_completion(save_to_disk="./results.zip")

# Merge all outputs into a single document-centric JSON
merged = pkg.merge(save_to="./output/consolidated.json")

# Access consolidated data
for doc in merged["documents"]:
    print(doc["filename"])
    print(doc["markdown"][:200])           # Extracted text (once, no redundancy)
    print(doc["entities"])                  # Entities for this document
    print(doc["knowledge_graph"])           # Graph for this document

# Global summary
print(merged["summary"]["entities"]["total"])
print(merged["summary"]["relations"]["total"])
print(merged["summary"]["cost_usd"])

                    ┌─── extraction ──── ontology
                    │
document_intelligence ─┼─── redaction
                    │
                    ├─── compression
                    │
                    ├─── embedding
                    │
                    ├─── colbert
                    │
                    └─── colpali

steps:
  document_intelligence:
    mode: performance
  extraction:
    label_mode: generated
    user_labels: [person, organization]
  ontology:
    resolve_entities: true

# Submit and return immediately
job = client.pipeline.run(files=["big_archive.pdf"])
print(f"Submitted: {job.id}")  # pipe_abc123

# ... do other work ...

# Check progress
status = job.status()
print(f"Stage: {status.current_service} ({status.stages_completed}/{status.total_stages})")

# When ready, collect results
pkg = job.wait_for_completion()

# Or cancel if no longer needed
job.cancel()

from latence import JobError

try:
    pkg = job.wait_for_completion()
except JobError as e:
    if e.is_resumable:
        # Restart from the last completed checkpoint
        pkg = job.resume().wait_for_completion()
    else:
        raise

pkg = job.wait_for_completion(
    poll_interval=5.0,
    timeout=1800.0,
    on_progress=lambda status, elapsed: print(f"  {status} ({elapsed:.0f}s)"),
)

from latence import (
    AuthenticationError,
    InsufficientCreditsError,
    RateLimitError,
    JobError,
    JobTimeoutError,
    TransportError,
    PipelineValidationError,
)

try:
    job = client.pipeline.run(files=["doc.pdf"])
    pkg = job.wait_for_completion(timeout=600)
except AuthenticationError:
    print("Invalid API key")
except InsufficientCreditsError:
    print("No credits remaining")
except RateLimitError as e:
    print(f"Rate limited -- retry after {e.retry_after}s")
except JobTimeoutError as e:
    print(f"Pipeline {e.job_id} did not finish in time")
except JobError as e:
    if e.is_resumable:
        pkg = job.resume().wait_for_completion()
    else:
        print(f"Pipeline failed: {e.message}")
except TransportError:
    print("Network / connection error")
except PipelineValidationError as e:
    print(f"Invalid pipeline config: {e.errors}")