EAP – Connector API Details
"""
Connectors Endpoint — API Service
Thin router layer: handles CRUD mechanics (DB, Pub/Sub, HTTP responses) and
delegates ALL connector-specific logic to the appropriate orchestrator.
Routes:
GET /v1/{tenant_id}/connectors — list connectors
GET /v1/{tenant_id}/connectors/{connector_id} — get connector detail
POST /v1/{tenant_id}/connectors — create connector + publish worker event
PATCH /v1/{tenant_id}/connectors/{connector_id} — update connector fields
DELETE /v1/{tenant_id}/connectors/{connector_id} — hard-delete connector + all data
To add a new connector type, create app/connectors/<source>.py implementing
ConnectorOrchestrator and register it in app/connectors/registry.py.
This file never needs to change.
"""
Get Connector Details:
# – -------------------------------------------------------------------------
# GET /v1/{tenant_id}/connectors/{connector_id}
# – -------------------------------------------------------------------------
@router.get(
"/{tenant_id}/connectors/{connector_id}",
response_model=ConnectorDetailResponse,
tags=["Connectors"],
summary="Get full connector config by ID",
)
async def get_connector(
tenant_id: str,
connector_id: str,
db: Annotated[DatabaseAdapter, Depends(get_database)],
current_user: Annotated[dict, Depends(verify_token)],
):
instance = f"/v1/{tenant_id}/connectors/{connector_id}"
try:
integration = await db.get_integration(connector_id)
if not integration:
raise HTTPException(
status_code=status.HTTP_404_NOT_FOUND,
detail=_problem(
ErrorType.RESOURCE_NOT_FOUND,
"Connector Not Found",
404,
f"No connector with ID '{connector_id}' found for tenant '{tenant_id}'",
instance,
),
)
if integration.platform_tenant_id != tenant_id:
raise HTTPException(
status_code=status.HTTP_403_FORBIDDEN,
detail=_problem(
ErrorType.VALIDATION_ERROR,
"Tenant Mismatch",
403,
f"Connector '{connector_id}' does not belong to tenant '{tenant_id}'",
instance,
),
)
jobs = await db.list_jobs(integration_id=connector_id, limit=1)
latest_job = jobs[0] if jobs else None
orchestrator = get_orchestrator(integration.source)
config_out = orchestrator.get_config_response(integration)
return ConnectorDetailResponse(
connector_id=integration.integration_id,
tenant_id=integration.platform_tenant_id,
source=integration.source,
name=integration.connector_name,
update_frequency=integration.update_frequency,
storage_config={
"provider": integration.storage_provider,
"bucket_name": integration.storage_bucket,
"container_name": integration.storage_container,
"base_path": integration.storage_base_path,
},
config=config_out,
status=latest_job.status if latest_job else "no_jobs",
job_id=latest_job.job_id if latest_job else None,
created_at=integration.created_at,
updated_at=integration.updated_at,
webhook_path=f"/v1/{tenant_id}/connectors/{connector_id}/sync",
)
except HTTPException:
raise
except Exception as exc:
logger.error(f"get_connector {connector_id}: {exc}", exc_info=True)
raise HTTPException(
status_code=500,
detail=_problem(
ErrorType.INTERNAL_ERROR,
"Failed to get connector",
500,
str(exc) if settings.DEBUG else "An internal error occurred",
instance,
),
) from exc
# – -------------------------------------------------------------------------
# GET /v1/{tenant_id}/connectors
# – -------------------------------------------------------------------------
@router.get(
"/{tenant_id}/connectors",
response_model=List[ConnectorResponse],
tags=["Connectors"],
summary="List all connectors for a tenant",
)
async def list_connectors(
tenant_id: str,
db: Annotated[DatabaseAdapter, Depends(get_database)],
current_user: Annotated[dict, Depends(verify_token)],
source: Annotated[
Optional[str],
Query(description="Filter by source type: sharepoint | servicenow | confluence"),
] = None,
active_only: Annotated[bool, Query(description="Return only active connectors")] = True,
):
try:
integrations = await db.list_integrations_by_tenant(
tenant_id=tenant_id,
source=source,
is_active=active_only,
)
results: List[ConnectorResponse] = []
for intg in integrations:
jobs = await db.list_jobs(integration_id=intg.integration_id, limit=1)
latest_job = jobs[0] if jobs else None
results.append(_integration_to_response(intg, latest_job))
return results
except Exception as exc:
logger.error(f"list_connectors tenant={tenant_id}: {exc}", exc_info=True)
raise HTTPException(
status_code=500,
detail=_problem(
ErrorType.INTERNAL_ERROR,
"Failed to list connectors",
500,
str(exc) if settings.DEBUG else "An internal error occurred",
f"/v1/{tenant_id}/connectors",
),
) from exc
# – -------------------------------------------------------------------------
# POST /v1/{tenant_id}/connectors
# – -------------------------------------------------------------------------
@router.post(
"/{tenant_id}/connectors",
response_model=ConnectorResponse,
status_code=status.HTTP_201_CREATED,
tags=["Connectors"],
summary="Add a connector for a tenant",
)
async def create_connector(
tenant_id: str,
request: ConnectorCreate,
db: Annotated[DatabaseAdapter, Depends(get_database)],
current_user: Annotated[dict, Depends(verify_token)],
):
"""
Create a new connector for the tenant and immediately dispatch an
ingestion job to the worker-service via GCP Pub/Sub.
When PUBSUB_ENABLED=False the job is saved to the database with status
QUEUED but no message is published — the worker-service must pick it up
via its database-polling fallback.
"""
try:
connector_id = generate_uuid()
source = request.config.source
now = _now_iso()
# ── Resolve tenant storage config ────────────────────────────────────
# The tenant record in Firestore holds the GCS bucket name.
# We derive a unique per-connector subfolder: {tenant_id}/{connector_id}/
tenant = await db.get_tenant(tenant_id)
if tenant is None:
raise HTTPException(
status_code=404,
detail=_problem(
ErrorType.RESOURCE_NOT_FOUND,
"Tenant Not Found",
404,
f"No tenant registered with ID '{tenant_id}'. "
"Register the tenant before creating connectors.",
f"/v1/{tenant_id}/connectors",
),
)
storage_path = f"{tenant_id}/{connector_id}/"
# ── Build Integration record ─────────────────────────────────────────
# Build the generic base with connector-agnostic fields.
# The orchestrator's build_integration() returns the connector-specific
# subclass (e.g. SharePointIntegration) with all extra fields populated.
orchestrator = get_orchestrator(source)
base_integration = BaseIntegration(
integration_id=connector_id,
platform_tenant_id=tenant_id,
source=source,
connector_name=request.connector_name,
update_frequency=request.update_frequency,
storage_provider=tenant.storage_provider,
storage_bucket=tenant.storage_bucket,
storage_base_path=storage_path,
created_at=now,
updated_at=now,
is_active=True,
)
integration = orchestrator.build_integration(request, base_integration)
await db.create_integration(integration)
logger.info(f"Created connector {connector_id} (source={source}, tenant={tenant_id})")
# ── Create initial Job record ────────────────────────────────────────
# Same pattern: generic BaseJob first, orchestrator builds the typed subclass.
job_id = generate_uuid()
base_job = BaseJob(
job_id=job_id,
integration_id=connector_id,
status=JobStatus.QUEUED,
platform_tenant_id=tenant_id,
source=source,
connector_name=request.connector_name,
)
job = orchestrator.build_job(request, base_job, integration)
await db.create_job(job)
logger.info(f"Created job {job_id} for connector {connector_id}")
# ── Dispatch to worker-service via Pub/Sub ───────────────────────────
try:
if settings.PUBSUB_ENABLED:
from app.core.pubsub.publisher import PubSubPublisher # noqa: PLC0415
payload = orchestrator.build_pubsub_payload(request, job, integration)
if payload is not None:
publisher = PubSubPublisher(project_id=settings.gcp_project_id)
await publisher.publish(
topic_id=settings.PUBSUB_WORKER_TOPIC_ID,
data=payload,
)
logger.info(f"Published worker event for job {job_id} to worker-service")
else:
# PUBSUB_ENABLED=False — job is persisted with QUEUED status.
# The worker-service must poll the database for QUEUED jobs.
logger.warning(
f"PUBSUB_ENABLED=False: job {job_id} is QUEUED in DB but no Pub/Sub "
f"message was published. Start the worker-service to process this job."
)
except Exception as dispatch_exc:
logger.error(f"Failed to dispatch job {job_id}: {dispatch_exc}", exc_info=True)
# Job record is persisted — can be retried via PATCH or retry endpoint.
return _integration_to_response(
integration,
job,
message=f"""Connector created and ingestion job dispatched
to worker-service (source={source})""",
)
except HTTPException:
raise
except Exception as exc:
logger.error(f"create_connector tenant={tenant_id}: {exc}", exc_info=True)
raise HTTPException(
status_code=500,
detail=_problem(
ErrorType.INTERNAL_ERROR,
"Failed to create connector",
500,
str(exc) if settings.DEBUG else "An internal error occurred",
f"/v1/{tenant_id}/connectors",
),
) from exc
