EAP – Connector API Details

"""
Connectors Endpoint — API Service

Thin router layer: handles CRUD mechanics (DB, Pub/Sub, HTTP responses) and
delegates ALL connector-specific logic to the appropriate orchestrator.

Routes:
  GET    /v1/{tenant_id}/connectors                   — list connectors
  GET    /v1/{tenant_id}/connectors/{connector_id}    — get connector detail
  POST   /v1/{tenant_id}/connectors                   — create connector + publish worker event
  PATCH  /v1/{tenant_id}/connectors/{connector_id}    — update connector fields
  DELETE /v1/{tenant_id}/connectors/{connector_id}    — hard-delete connector + all data

To add a new connector type, create app/connectors/<source>.py implementing
ConnectorOrchestrator and register it in app/connectors/registry.py.
This file never needs to change.
"""

Get Connector Details:

# – -------------------------------------------------------------------------
# GET /v1/{tenant_id}/connectors/{connector_id}
# – -------------------------------------------------------------------------


@router.get(
    "/{tenant_id}/connectors/{connector_id}",
    response_model=ConnectorDetailResponse,
    tags=["Connectors"],
    summary="Get full connector config by ID",
)
async def get_connector(
    tenant_id: str,
    connector_id: str,
    db: Annotated[DatabaseAdapter, Depends(get_database)],
    current_user: Annotated[dict, Depends(verify_token)],
):
    instance = f"/v1/{tenant_id}/connectors/{connector_id}"
    try:
        integration = await db.get_integration(connector_id)
        if not integration:
            raise HTTPException(
                status_code=status.HTTP_404_NOT_FOUND,
                detail=_problem(
                    ErrorType.RESOURCE_NOT_FOUND,
                    "Connector Not Found",
                    404,
                    f"No connector with ID '{connector_id}' found for tenant '{tenant_id}'",
                    instance,
                ),
            )
        if integration.platform_tenant_id != tenant_id:
            raise HTTPException(
                status_code=status.HTTP_403_FORBIDDEN,
                detail=_problem(
                    ErrorType.VALIDATION_ERROR,
                    "Tenant Mismatch",
                    403,
                    f"Connector '{connector_id}' does not belong to tenant '{tenant_id}'",
                    instance,
                ),
            )

        jobs = await db.list_jobs(integration_id=connector_id, limit=1)
        latest_job = jobs[0] if jobs else None

        orchestrator = get_orchestrator(integration.source)
        config_out = orchestrator.get_config_response(integration)

        return ConnectorDetailResponse(
            connector_id=integration.integration_id,
            tenant_id=integration.platform_tenant_id,
            source=integration.source,
            name=integration.connector_name,
            update_frequency=integration.update_frequency,
            storage_config={
                "provider": integration.storage_provider,
                "bucket_name": integration.storage_bucket,
                "container_name": integration.storage_container,
                "base_path": integration.storage_base_path,
            },
            config=config_out,
            status=latest_job.status if latest_job else "no_jobs",
            job_id=latest_job.job_id if latest_job else None,
            created_at=integration.created_at,
            updated_at=integration.updated_at,
            webhook_path=f"/v1/{tenant_id}/connectors/{connector_id}/sync",
        )
    except HTTPException:
        raise
    except Exception as exc:
        logger.error(f"get_connector {connector_id}: {exc}", exc_info=True)
        raise HTTPException(
            status_code=500,
            detail=_problem(
                ErrorType.INTERNAL_ERROR,
                "Failed to get connector",
                500,
                str(exc) if settings.DEBUG else "An internal error occurred",
                instance,
            ),
        ) from exc
# – -------------------------------------------------------------------------
# GET /v1/{tenant_id}/connectors
# – -------------------------------------------------------------------------


@router.get(
    "/{tenant_id}/connectors",
    response_model=List[ConnectorResponse],
    tags=["Connectors"],
    summary="List all connectors for a tenant",
)
async def list_connectors(
    tenant_id: str,
    db: Annotated[DatabaseAdapter, Depends(get_database)],
    current_user: Annotated[dict, Depends(verify_token)],
    source: Annotated[
        Optional[str],
        Query(description="Filter by source type: sharepoint | servicenow | confluence"),
    ] = None,
    active_only: Annotated[bool, Query(description="Return only active connectors")] = True,
):
    try:
        integrations = await db.list_integrations_by_tenant(
            tenant_id=tenant_id,
            source=source,
            is_active=active_only,
        )
        results: List[ConnectorResponse] = []
        for intg in integrations:
            jobs = await db.list_jobs(integration_id=intg.integration_id, limit=1)
            latest_job = jobs[0] if jobs else None
            results.append(_integration_to_response(intg, latest_job))
        return results
    except Exception as exc:
        logger.error(f"list_connectors tenant={tenant_id}: {exc}", exc_info=True)
        raise HTTPException(
            status_code=500,
            detail=_problem(
                ErrorType.INTERNAL_ERROR,
                "Failed to list connectors",
                500,
                str(exc) if settings.DEBUG else "An internal error occurred",
                f"/v1/{tenant_id}/connectors",
            ),
        ) from exc
# – -------------------------------------------------------------------------
# POST /v1/{tenant_id}/connectors
# – -------------------------------------------------------------------------


@router.post(
    "/{tenant_id}/connectors",
    response_model=ConnectorResponse,
    status_code=status.HTTP_201_CREATED,
    tags=["Connectors"],
    summary="Add a connector for a tenant",
)
async def create_connector(
    tenant_id: str,
    request: ConnectorCreate,
    db: Annotated[DatabaseAdapter, Depends(get_database)],
    current_user: Annotated[dict, Depends(verify_token)],
):
    """
    Create a new connector for the tenant and immediately dispatch an
    ingestion job to the worker-service via GCP Pub/Sub.

    When PUBSUB_ENABLED=False the job is saved to the database with status
    QUEUED but no message is published — the worker-service must pick it up
    via its database-polling fallback.
    """
    try:
        connector_id = generate_uuid()
        source = request.config.source
        now = _now_iso()

        # ── Resolve tenant storage config ────────────────────────────────────
        # The tenant record in Firestore holds the GCS bucket name.
        # We derive a unique per-connector subfolder: {tenant_id}/{connector_id}/
        tenant = await db.get_tenant(tenant_id)
        if tenant is None:
            raise HTTPException(
                status_code=404,
                detail=_problem(
                    ErrorType.RESOURCE_NOT_FOUND,
                    "Tenant Not Found",
                    404,
                    f"No tenant registered with ID '{tenant_id}'. "
                    "Register the tenant before creating connectors.",
                    f"/v1/{tenant_id}/connectors",
                ),
            )
        storage_path = f"{tenant_id}/{connector_id}/"

        # ── Build Integration record ─────────────────────────────────────────
        # Build the generic base with connector-agnostic fields.
        # The orchestrator's build_integration() returns the connector-specific
        # subclass (e.g. SharePointIntegration) with all extra fields populated.
        orchestrator = get_orchestrator(source)
        base_integration = BaseIntegration(
            integration_id=connector_id,
            platform_tenant_id=tenant_id,
            source=source,
            connector_name=request.connector_name,
            update_frequency=request.update_frequency,
            storage_provider=tenant.storage_provider,
            storage_bucket=tenant.storage_bucket,
            storage_base_path=storage_path,
            created_at=now,
            updated_at=now,
            is_active=True,
        )
        integration = orchestrator.build_integration(request, base_integration)

        await db.create_integration(integration)
        logger.info(f"Created connector {connector_id} (source={source}, tenant={tenant_id})")

        # ── Create initial Job record ────────────────────────────────────────
        # Same pattern: generic BaseJob first, orchestrator builds the typed subclass.
        job_id = generate_uuid()
        base_job = BaseJob(
            job_id=job_id,
            integration_id=connector_id,
            status=JobStatus.QUEUED,
            platform_tenant_id=tenant_id,
            source=source,
            connector_name=request.connector_name,
        )
        job = orchestrator.build_job(request, base_job, integration)
        await db.create_job(job)
        logger.info(f"Created job {job_id} for connector {connector_id}")

        # ── Dispatch to worker-service via Pub/Sub ───────────────────────────
        try:
            if settings.PUBSUB_ENABLED:
                from app.core.pubsub.publisher import PubSubPublisher  # noqa: PLC0415

                payload = orchestrator.build_pubsub_payload(request, job, integration)
                if payload is not None:
                    publisher = PubSubPublisher(project_id=settings.gcp_project_id)
                    await publisher.publish(
                        topic_id=settings.PUBSUB_WORKER_TOPIC_ID,
                        data=payload,
                    )
                    logger.info(f"Published worker event for job {job_id} to worker-service")
            else:
                # PUBSUB_ENABLED=False — job is persisted with QUEUED status.
                # The worker-service must poll the database for QUEUED jobs.
                logger.warning(
                    f"PUBSUB_ENABLED=False: job {job_id} is QUEUED in DB but no Pub/Sub "
                    f"message was published. Start the worker-service to process this job."
                )
        except Exception as dispatch_exc:
            logger.error(f"Failed to dispatch job {job_id}: {dispatch_exc}", exc_info=True)
            # Job record is persisted — can be retried via PATCH or retry endpoint.

        return _integration_to_response(
            integration,
            job,
            message=f"""Connector created and ingestion job dispatched
            to worker-service (source={source})""",
        )

    except HTTPException:
        raise
    except Exception as exc:
        logger.error(f"create_connector tenant={tenant_id}: {exc}", exc_info=True)
        raise HTTPException(
            status_code=500,
            detail=_problem(
                ErrorType.INTERNAL_ERROR,
                "Failed to create connector",
                500,
                str(exc) if settings.DEBUG else "An internal error occurred",
                f"/v1/{tenant_id}/connectors",
            ),
        ) from exc

Leave a Reply

Your email address will not be published. Required fields are marked *