openapi: 3.0.3
info:
  title: dexter-sentinel API
  description: |
    Config-driven HTTP scrape API. Scraper configs resolve from Athena `scraper_overrides` (when enabled) then YAML.
    Several admin routes return **503** when Athena is not configured (`ATHENA_URL`, `ATHENA_API_KEY`).
    Screenshot upload requires `S3_SCREENSHOT_BUCKET` and AWS credentials; otherwise `POST /api/screenshot` returns **503**.
    PwC Albania publication sync requires `S3_PUBLICATIONS_BUCKET` when `dry_run` is false; otherwise `POST /api/publications/pwc-al/sync` returns **503**.
  version: 0.5.1
  license:
    name: MIT
servers:
- url: http://localhost:8080
  description: Default local server (override `HOST` / `PORT`)
tags:
- name: Health
- name: Metrics
- name: Scrape
- name: Cache
- name: HTTP profiles
- name: Provider requests
- name: Scraper overrides
- name: Screenshots
- name: Publications
paths:
  /:
    get:
      tags:
      - Health
      summary: API root index
      operationId: getRoot
      responses:
        '200':
          description: Service metadata and quick endpoint index
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/RootResponse'
  /health:
    get:
      tags:
      - Health
      summary: Liveness and uptime
      operationId: getHealth
      responses:
        '200':
          description: OK
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/HealthResponse'
  /metrics:
    get:
      tags:
      - Metrics
      summary: Prometheus text exposition (OpenMetrics-style 0.0.4)
      description: |
        Full scrape + HTTP + cache + audit + optional Linux `process_*` metrics.
        Includes `http_requests_total` / `http_request_duration_seconds` (paths normalized with `{id}`),
        Moka cache entry count, Athena/Redis/screenshot/publications/audit feature gauges, bloom filter
        dimensions, `dexter_pipeline_*` totals when wired from the server, scrape HTML/JSON body size
        histograms, Redis URL registry latency + outcome counters, and existing scrape/registry counters.
      operationId: getMetrics
      responses:
        '200':
          description: Prometheus text exposition format (`text/plain; version=0.0.4`)
          content:
            text/plain:
              schema:
                type: string
  /scrape:
    post:
      tags:
      - Scrape
      summary: Run scrape with JSON body
      operationId: postScrape
      requestBody:
        required: true
        content:
          application/json:
            schema:
              $ref: '#/components/schemas/ScrapeRequest'
      responses:
        '200':
          description: Selector results as flat string map (plus `url`, `config_name`)
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/ScrapeResult'
        '400':
          description: Bad request (invalid URL, template, etc.)
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/Error'
        '404':
          description: Unknown config name
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/Error'
        '502':
          description: Upstream HTTP failure
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/Error'
        '500':
          description: Internal error
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/Error'
  /scrape/{config_name}:
    get:
      tags:
      - Scrape
      summary: Run scrape with named config (optional URL query)
      operationId: getScrape
      parameters:
      - name: config_name
        in: path
        required: true
        schema:
          type: string
      - name: url
        in: query
        required: false
        description: |
          Optional request URL after template expansion. If omitted, `base_url` from the resolved config is used.
        schema:
          type: string
      responses:
        '200':
          description: Selector results
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/ScrapeResult'
        '400':
          description: Bad request
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/Error'
        '404':
          description: Unknown config
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/Error'
        '502':
          description: Upstream HTTP failure
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/Error'
        '500':
          description: Internal error
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/Error'
  /api/cache/registry/add:
    post:
      tags:
      - Cache
      summary: Register URL (bloom, optional Redis + Athena)
      operationId: postRegistryAdd
      requestBody:
        required: true
        content:
          application/json:
            schema:
              $ref: '#/components/schemas/AddRegistryRequest'
      responses:
        '200':
          description: Added
          content:
            application/json:
              schema:
                type: object
                required:
                - status
                - url
                properties:
                  status:
                    type: string
                    example: added
                  url:
                    type: string
  /api/cache/registry/check:
    get:
      tags:
      - Cache
      summary: Check if URL is in registry
      operationId: getRegistryCheck
      parameters:
      - name: url
        in: query
        required: true
        schema:
          type: string
      responses:
        '200':
          description: Lookup result
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/RegistryCheckResponse'
  /api/http-profiles:
    get:
      tags:
      - HTTP profiles
      summary: List HTTP client profiles (masked proxy credentials)
      operationId: listHttpProfiles
      responses:
        '200':
          description: OK
          content:
            application/json:
              schema:
                type: array
                items:
                  $ref: '#/components/schemas/HttpClientProfile'
        '503':
          description: Athena not configured
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/Error'
        '500':
          description: Server error
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/Error'
    post:
      tags:
      - HTTP profiles
      summary: Create HTTP client profile
      operationId: postHttpProfile
      requestBody:
        required: true
        content:
          application/json:
            schema:
              $ref: '#/components/schemas/HttpClientProfile'
      responses:
        '201':
          description: Created
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/HttpClientProfile'
        '400':
          description: Validation error
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/Error'
        '503':
          description: Athena not configured
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/Error'
  /api/http-profiles/{name}:
    parameters:
    - name: name
      in: path
      required: true
      schema:
        type: string
    get:
      tags:
      - HTTP profiles
      summary: Get profile by name
      operationId: getHttpProfile
      responses:
        '200':
          description: OK
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/HttpClientProfile'
        '404':
          description: Not found
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/Error'
        '503':
          description: Athena not configured
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/Error'
        '500':
          description: Server error
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/Error'
    put:
      tags:
      - HTTP profiles
      summary: Replace profile by name
      operationId: putHttpProfile
      requestBody:
        required: true
        content:
          application/json:
            schema:
              $ref: '#/components/schemas/HttpClientProfile'
      responses:
        '200':
          description: OK
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/HttpClientProfile'
        '400':
          description: Validation error
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/Error'
        '503':
          description: Athena not configured
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/Error'
    delete:
      tags:
      - HTTP profiles
      summary: Soft-disable profile (`enabled` = false)
      operationId: deleteHttpProfile
      responses:
        '204':
          description: No content
        '400':
          description: Error
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/Error'
        '503':
          description: Athena not configured
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/Error'
  /api/provider-requests:
    get:
      tags:
      - Provider requests
      summary: List data-source provider backlog rows
      operationId: listProviderRequests
      responses:
        '200':
          description: OK
          content:
            application/json:
              schema:
                type: array
                items:
                  $ref: '#/components/schemas/DataSourceProviderRequest'
        '503':
          description: Athena not configured
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/Error'
        '500':
          description: Server error
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/Error'
    post:
      tags:
      - Provider requests
      summary: Create backlog row
      operationId: postProviderRequest
      requestBody:
        required: true
        content:
          application/json:
            schema:
              $ref: '#/components/schemas/DataSourceProviderRequestNew'
      responses:
        '201':
          description: Created
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/OkStub'
        '400':
          description: Validation error
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/Error'
        '503':
          description: Athena not configured
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/Error'
  /api/provider-requests/{id}:
    parameters:
    - name: id
      in: path
      required: true
      schema:
        type: integer
        format: int64
    get:
      tags:
      - Provider requests
      summary: Get backlog row by id
      operationId: getProviderRequest
      responses:
        '200':
          description: OK
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/DataSourceProviderRequest'
        '404':
          description: Not found
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/Error'
        '503':
          description: Athena not configured
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/Error'
        '500':
          description: Server error
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/Error'
    put:
      tags:
      - Provider requests
      summary: Update backlog row
      operationId: putProviderRequest
      requestBody:
        required: true
        content:
          application/json:
            schema:
              $ref: '#/components/schemas/DataSourceProviderRequestNew'
      responses:
        '200':
          description: OK
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/OkStub'
        '400':
          description: Validation error
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/Error'
        '503':
          description: Athena not configured
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/Error'
    delete:
      tags:
      - Provider requests
      summary: Delete backlog row
      operationId: deleteProviderRequest
      responses:
        '204':
          description: No content
        '400':
          description: Error
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/Error'
        '503':
          description: Athena not configured
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/Error'
  /api/scraper-overrides/reload:
    post:
      tags:
      - Scraper overrides
      summary: Reload scraper overrides from Athena into process cache
      operationId: postScraperOverridesReload
      responses:
        '200':
          description: Reloaded
          content:
            application/json:
              schema:
                type: object
                required:
                - count
                properties:
                  count:
                    type: integer
                    description: Number of enabled overrides loaded
                    minimum: 0
        '503':
          description: Athena not configured
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/Error'
        '500':
          description: Server error
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/Error'
  /api/scraper-overrides:
    get:
      tags:
      - Scraper overrides
      summary: List all scraper override rows from Athena
      operationId: listScraperOverrides
      responses:
        '200':
          description: OK
          content:
            application/json:
              schema:
                type: array
                items:
                  $ref: '#/components/schemas/ScraperOverrideListItem'
        '503':
          description: Athena not configured
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/Error'
        '500':
          description: Server error
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/Error'
    post:
      tags:
      - Scraper overrides
      summary: Insert scraper override (body is full ScraperConfig; `name` required)
      operationId: postScraperOverride
      requestBody:
        required: true
        content:
          application/json:
            schema:
              $ref: '#/components/schemas/ScraperConfig'
      responses:
        '201':
          description: Created
          content:
            application/json:
              schema:
                type: object
                required:
                - name
                properties:
                  name:
                    type: string
        '400':
          description: Validation error
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/Error'
        '503':
          description: Athena not configured
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/Error'
  /api/scraper-overrides/{name}:
    parameters:
    - name: name
      in: path
      required: true
      schema:
        type: string
    put:
      tags:
      - Scraper overrides
      summary: Update scraper override (full ScraperConfig; path name overrides body `name`)
      operationId: putScraperOverride
      requestBody:
        required: true
        content:
          application/json:
            schema:
              $ref: '#/components/schemas/ScraperConfig'
      responses:
        '200':
          description: OK
          content:
            application/json:
              schema:
                type: object
                required:
                - name
                properties:
                  name:
                    type: string
        '400':
          description: Validation error
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/Error'
        '503':
          description: Athena not configured
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/Error'
    delete:
      tags:
      - Scraper overrides
      summary: Delete scraper override row
      operationId: deleteScraperOverride
      responses:
        '204':
          description: No content
        '400':
          description: Error
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/Error'
        '503':
          description: Athena not configured
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/Error'
  /api/screenshot:
    post:
      tags:
      - Screenshots
      summary: Capture URL to PNG, upload to S3, optionally attach key to provider request
      operationId: postScreenshot
      requestBody:
        required: true
        content:
          application/json:
            schema:
              $ref: '#/components/schemas/ScreenshotRequestBody'
      responses:
        '200':
          description: Uploaded object key
          content:
            application/json:
              schema:
                type: object
                required:
                - s3_key
                properties:
                  s3_key:
                    type: string
        '400':
          description: URL validation or capture/upload failure (message in `error`)
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/Error'
        '503':
          description: S3 bucket not configured
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/Error'
  /api/publications/pwc-al/sync:
    post:
      tags:
      - Publications
      summary: Sync PwC Albania publications (PDFs; optional HTML) to S3
      operationId: postPwcAlPublicationsSync
      requestBody:
        required: false
        content:
          application/json:
            schema:
              $ref: '#/components/schemas/PwcAlPublicationsSyncBody'
      responses:
        '200':
          description: Sync result summary and per-item status
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/PwcAlPublicationsSyncResult'
        '400':
          description: Fetch or validation error
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/Error'
        '503':
          description: S3 publications bucket not configured (when not dry_run)
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/Error'
components:
  schemas:
    Error:
      type: object
      required:
      - error
      properties:
        error:
          type: string
    PwcAlPublicationsSyncBody:
      type: object
      properties:
        dry_run:
          type: boolean
          default: false
        include_html:
          type: boolean
          default: false
    PwcAlPublicationsSyncItem:
      type: object
      required:
      - title
      - source_url
      - content_type
      - kind
      - skipped_existing
      properties:
        title:
          type: string
        source_url:
          type: string
        content_type:
          type: string
        kind:
          type: string
          description: pdf, html, other (dry_run), or error
        s3_key:
          type: string
          nullable: true
        error:
          type: string
          nullable: true
        skipped_existing:
          type: boolean
    PwcAlPublicationsSyncResult:
      type: object
      required:
      - links_total
      - pdfs_eligible
      - html_eligible
      - uploaded
      - skipped_existing
      - errors
      - items
      properties:
        links_total:
          type: integer
          format: int64
        pdfs_eligible:
          type: integer
          format: int64
        html_eligible:
          type: integer
          format: int64
        uploaded:
          type: integer
          format: int64
        skipped_existing:
          type: integer
          format: int64
        errors:
          type: integer
          format: int64
        items:
          type: array
          items:
            $ref: '#/components/schemas/PwcAlPublicationsSyncItem'
    OkStub:
      type: object
      required:
      - ok
      properties:
        ok:
          type: boolean
          example: true
    HealthResponse:
      type: object
      required:
      - status
      - uptime_seconds
      - service
      properties:
        status:
          type: string
          example: ok
        uptime_seconds:
          type: integer
          format: int64
          minimum: 0
        service:
          type: string
          example: dexter-sentinel
    RootResponse:
      type: object
      required:
      - service
      - status
      - uptime_seconds
      - endpoints
      properties:
        service:
          type: string
          example: dexter-sentinel
        status:
          type: string
          example: ok
        uptime_seconds:
          type: integer
          format: int64
          minimum: 0
        endpoints:
          type: object
          additionalProperties:
            type: string
    ScrapeRequest:
      type: object
      required:
      - config_name
      properties:
        config_name:
          type: string
          description: Resolved against DB overrides first, then YAML scrapers
        url:
          type: string
          description: Optional; if omitted, `base_url` from config is used (after template expansion).
        context:
          type: object
          description: Template placeholder values (scalars only). Keys match `{{key}}` / `{{$TABLE.COL$}}` after normalization.
          additionalProperties: true
    ScrapeResult:
      type: object
      description: Flat string map of selector names plus `url` and `config_name`.
      additionalProperties:
        type: string
    AddRegistryRequest:
      type: object
      required:
      - url
      properties:
        url:
          type: string
    RegistryCheckResponse:
      type: object
      required:
      - url
      - found
      - source
      properties:
        url:
          type: string
        found:
          type: boolean
        source:
          type: string
          description: e.g. `bloom_filter`, `redis`, `no_backend`
    HttpClientProfile:
      type: object
      required:
      - name
      - timeout_secs
      - follow_redirects
      - cookie_store
      - danger_accept_invalid_certs
      - enabled
      properties:
        name:
          type: string
        proxy_url:
          type: string
          nullable: true
        timeout_secs:
          type: integer
          format: int64
        connect_timeout_secs:
          type: integer
          format: int64
          nullable: true
        follow_redirects:
          type: boolean
        cookie_store:
          type: boolean
        default_headers:
          type: object
          nullable: true
          additionalProperties: true
        user_agent:
          type: string
          nullable: true
        danger_accept_invalid_certs:
          type: boolean
        enabled:
          type: boolean
    DataSourceProviderRequest:
      type: object
      required:
      - id
      - example_url
      - status
      properties:
        id:
          type: integer
          format: int64
        example_url:
          type: string
        structure_notes:
          type: string
          nullable: true
        harvestable_fields:
          nullable: true
        reliability:
          type: string
          nullable: true
        status:
          type: string
        screenshot_s3_key:
          type: string
          nullable: true
        implementation_notes:
          type: string
          nullable: true
        created_at:
          type: string
          nullable: true
        updated_at:
          type: string
          nullable: true
    DataSourceProviderRequestNew:
      type: object
      required:
      - example_url
      properties:
        example_url:
          type: string
        structure_notes:
          type: string
          nullable: true
        harvestable_fields:
          nullable: true
        reliability:
          type: string
          nullable: true
        status:
          type: string
          nullable: true
          description: Defaults to `pending` when omitted on insert
        screenshot_s3_key:
          type: string
          nullable: true
        implementation_notes:
          type: string
          nullable: true
    ScraperOverrideListItem:
      type: object
      required:
      - name
      - enabled
      - config
      properties:
        name:
          type: string
        enabled:
          type: boolean
        config:
          $ref: '#/components/schemas/ScraperConfig'
    ScraperConfig:
      type: object
      required:
      - name
      - base_url
      - request
      - selectors
      properties:
        name:
          type: string
        version:
          type: string
          nullable: true
        base_url:
          type: string
          description: May include `{{placeholders}}` when used with `context` on scrape
        description:
          type: string
          nullable: true
        request_profile:
          type: string
          nullable: true
        request:
          $ref: '#/components/schemas/RequestConfig'
        proxy:
          nullable: true
          oneOf:
          - $ref: '#/components/schemas/ProxyConfigNone'
          - $ref: '#/components/schemas/ProxyConfigHttp'
          - $ref: '#/components/schemas/ProxyConfigSocks5'
        selectors:
          type: array
          items:
            $ref: '#/components/schemas/SelectorConfig'
        pagination:
          nullable: true
          allOf:
          - $ref: '#/components/schemas/PaginationConfig'
        rate_limit:
          nullable: true
          allOf:
          - $ref: '#/components/schemas/RateLimitConfig'
        output:
          nullable: true
          allOf:
          - $ref: '#/components/schemas/OutputConfig'
    ProxyConfigNone:
      type: object
      required:
      - type
      properties:
        type:
          type: string
          enum:
          - none
    ProxyConfigHttp:
      type: object
      required:
      - type
      - url
      properties:
        type:
          type: string
          enum:
          - http
        url:
          type: string
    ProxyConfigSocks5:
      type: object
      required:
      - type
      - url
      properties:
        type:
          type: string
          enum:
          - socks5
        url:
          type: string
    RequestConfig:
      type: object
      required:
      - method
      properties:
        method:
          type: string
        headers:
          type: object
          nullable: true
          additionalProperties:
            type: string
        timeout_secs:
          type: integer
          format: int64
          nullable: true
        connect_timeout_secs:
          type: integer
          format: int64
          nullable: true
        follow_redirects:
          type: boolean
          nullable: true
    SelectorConfig:
      type: object
      required:
      - name
      - css
      - attribute
      properties:
        name:
          type: string
        css:
          type: string
        attribute:
          type: string
        required:
          type: boolean
          nullable: true
        transform:
          type: string
          nullable: true
    PaginationConfig:
      type: object
      required:
      - enabled
      properties:
        enabled:
          type: boolean
        next_selector:
          type: string
          nullable: true
        max_pages:
          type: integer
          format: int32
          nullable: true
    RateLimitConfig:
      type: object
      required:
      - requests_per_second
      - burst_size
      properties:
        requests_per_second:
          type: integer
          format: int32
        burst_size:
          type: integer
          format: int32
    OutputConfig:
      type: object
      required:
      - table
      properties:
        table:
          type: string
        deduplicate_by:
          type: string
          nullable: true
    ScreenshotRequestBody:
      type: object
      required:
      - url
      properties:
        url:
          type: string
          format: uri
        provider_request_id:
          type: integer
          format: int64
          description: When set and Athena is configured, updates `screenshot_s3_key` on the row
          nullable: true
