REST API Access

#!/usr/bin/env bash
# Usage:
#     export SOTERI_API_KEY="soteri_..."
#     ./export-space-findings.sh --space-id 123456 --output findings.csv

set -o errexit
set -o nounset
set -o pipefail

# The endpoint that starts a findings export for a single space.
readonly SPACE_FINDINGS_EXPORT_PATH="/rest/space/findings"

# The shared endpoints for polling an export job and downloading the file it produced.
readonly EXPORT_JOBS_PATH="/rest/export/jobs"

# How long to wait between polls, and how long to keep polling before giving up. An export of a large
# space takes minutes rather than seconds.
readonly POLL_INTERVAL_SECONDS=5
readonly POLL_TIMEOUT_SECONDS=$((30 * 60))

space_id=""
output_path="findings.csv"
rule_names=""

# The status code and body of the most recent response. They are globals rather than the output of
# ${request}, because a command substitution would run that function in a subshell and lose the body.
response_status=""
response_body=""

# Print a usage message and exit.
usage() {
    cat <<'USAGE'
Usage: export-space-findings.sh --space-id ID [--output PATH] [--rule-names NAMES]

  --space-id ID       The numeric ID of the space to export.
  --output PATH       Path the exported CSV is written to. Defaults to findings.csv.
  --rule-names NAMES  Comma-separated rule names to restrict the export to, for example
                      'AWS Access Key ID,Slack Token'. Defaults to every rule.

Set SOTERI_API_KEY before running this script.
USAGE
}

# Report a fatal error and stop.
#
# Arguments:
#   $1 The message to report.
fail() {
    echo "$1" >&2
    exit 1
}

# Describe a failed response using the app's error body when it sent one.
#
# Errors are returned as a JSON body with a title, an optional description, and any field-level
# violations. A response that carries no such body (a proxy error page, for example) falls back to its
# status code and raw text.
#
# Arguments:
#   $1 The status code of the failed response.
#   $2 The body of the failed response.
describe_error() {
    local status_code="$1"
    local body="$2"

    local description
    description="$(jq -r '
        if type == "object" and has("restErrorTitle") then
            [.restErrorTitle, .restErrorDescription // empty]
                + [(.violations // [])[] | "\(.fieldName): \(.message)"]
            | join(" - ")
        else
            empty
        end' <<<"$body" 2>/dev/null || true)"

    if [[ -z "$description" ]]; then
        description="$(head -c 200 <<<"$body" | tr -d '\n')"
    fi

    echo "HTTP ${status_code}: ${description}"
}

# Make an authenticated request, storing its outcome in ${response_status} and ${response_body}.
#
# The app's administration screen copies keys without the "Bearer " scheme attached. The value is accepted
# with or without it.
#
# A response is only read into ${response_body} when the request failed or when no file was given for it,
# so that a successful download is streamed straight to disk rather than held in memory.
#
# Arguments:
#   $1 The HTTP method.
#   $2 The URL to request.
#   $3 The file the response body is written to, or empty for a temporary one.
request() {
    local method="$1"
    local url="$2"
    local body_file="${3:-}"

    local token="${SOTERI_API_KEY# }"
    if [[ "$token" != [Bb][Ee][Aa][Rr][Ee][Rr]\ * ]]; then
        token="Bearer ${token}"
    fi

    local body_is_temporary=false
    if [[ -z "$body_file" ]]; then
        body_file="$(mktemp)"
        body_is_temporary=true
    fi

    # A 4xx or 5xx response is handled by the caller, so curl is not asked to fail on one.
    response_status="$(curl --silent --show-error --location \
        --request "$method" \
        --header "Authorization: ${token}" \
        --header "Accept: application/json,text/csv" \
        --write-out '%{http_code}' \
        --output "$body_file" \
        "$url")"

    response_body=""
    if [[ "$body_is_temporary" == true || "$response_status" != 2* ]]; then
        response_body="$(cat "$body_file")"
    fi
    if [[ "$body_is_temporary" == true ]]; then
        rm -f "$body_file"
    fi
}

# Start the export of a space's findings, and report the ID of the job it created.
#
# Outputs:
#   The ID of the created export job, on standard output.
start_space_findings_export() {
    local url="${SOTERI_BASE_URL}${SPACE_FINDINGS_EXPORT_PATH}?spaceId=${space_id}"
    if [[ -n "$rule_names" ]]; then
        url+="&ruleNames=$(jq -rn --arg names "$rule_names" '$names | @uri')"
    fi

    request POST "$url"
    if [[ "$response_status" != 2* ]]; then
        fail "Could not start the export. $(describe_error "$response_status" "$response_body")"
    fi

    jq -r '.jobId' <<<"$response_body"
}

# Poll an export job until its file can be downloaded.
#
# Arguments:
#   $1 The ID of the export job.
wait_until_ready() {
    local job_id="$1"
    local deadline=$((SECONDS + POLL_TIMEOUT_SECONDS))
    local last_progress=""

    while true; do
        request GET "${SOTERI_BASE_URL}${EXPORT_JOBS_PATH}/${job_id}"
        if [[ "$response_status" != 2* ]]; then
            fail "Could not read the status of export ${job_id}. $(describe_error "$response_status" "$response_body")"
        fi

        local status
        status="$(jq -r '.status' <<<"$response_body")"

        case "$status" in
            READY)
                return 0
                ;;
            FAILED)
                fail "Export ${job_id} failed: $(jq -r '.errorMessage // "no reason given"' <<<"$response_body")"
                ;;
            DOWNLOADED)
                fail "Export ${job_id} has already been downloaded, and cannot be downloaded again."
                ;;
            # An export that was running on a node that shut down is rescheduled automatically, so
            # INTERRUPTED is not a terminal state.
            PENDING | IN_PROGRESS | INTERRUPTED) ;;
            *)
                fail "Export ${job_id} reported an unexpected status: ${status}"
                ;;
        esac

        # Only report progress when it changes, so a long export does not produce a line per poll.
        local progress
        progress="$(jq -r '
            if .totalTargetsCount then
                "  \(.status): \(.completedTargetsCount // 0) of \(.totalTargetsCount) items exported"
            else
                empty
            end' <<<"$response_body")"
        if [[ -n "$progress" && "$progress" != "$last_progress" ]]; then
            echo "$progress"
            last_progress="$progress"
        fi

        if ((SECONDS >= deadline)); then
            fail "Export ${job_id} was still ${status} after ${POLL_TIMEOUT_SECONDS} seconds."
        fi
        sleep "$POLL_INTERVAL_SECONDS"
    done
}

# Stream a finished export's CSV file to disk.
#
# The file is streamed to a temporary path first, so that a refused download leaves no partial CSV behind
# and its JSON error body can be read instead.
#
# Arguments:
#   $1 The ID of the export job.
download_export() {
    local job_id="$1"

    local download_file
    download_file="$(mktemp)"

    request GET "${SOTERI_BASE_URL}${EXPORT_JOBS_PATH}/${job_id}/download" "$download_file"
    if [[ "$response_status" != 2* ]]; then
        rm -f "$download_file"
        fail "Could not download export ${job_id}. $(describe_error "$response_status" "$response_body")"
    fi

    mv "$download_file" "$output_path"
}

# Parse the command line arguments into the variables declared above.
parse_args() {
    while (($# > 0)); do
        case "$1" in
            --space-id)
                space_id="${2:-}"
                shift 2
                ;;
            --output)
                output_path="${2:-}"
                shift 2
                ;;
            --rule-names)
                rule_names="${2:-}"
                shift 2
                ;;
            --help | -h)
                usage
                exit 0
                ;;
            *)
                usage >&2
                fail "Unrecognized argument: $1"
                ;;
        esac
    done

    if [[ ! "$space_id" =~ ^[0-9]+$ ]]; then
        usage >&2
        fail "--space-id is required, and must be the numeric ID of a space."
    fi
}

main() {
    parse_args "$@"

    command -v curl >/dev/null || fail "This script needs curl."
    command -v jq >/dev/null || fail "This script needs jq."

    if [[ -z "${SOTERI_API_KEY:-}" ]]; then
        fail "Set SOTERI_API_KEY before running this script."
    fi
    SOTERI_BASE_URL="https://security-for-confluence.soteri.io"

    local job_id
    job_id="$(start_space_findings_export)"
    echo "Started export ${job_id} for space ${space_id}."

    wait_until_ready "$job_id"
    echo "Export ${job_id} is ready. Downloading..."

    download_export "$job_id"
    echo "Wrote ${output_path}."
}

main "$@"

#!/usr/bin/env python3
"""
Usage:
    export SOTERI_API_KEY="soteri_..."
    ./export-space-findings.py --space-id 123456 --output findings.csv
"""
import argparse
import os
import sys
import time

import requests

# The endpoint that starts a findings export for a single space.
SPACE_FINDINGS_EXPORT_PATH = "/rest/space/findings"

# The shared endpoints for polling an export job and downloading the file it produced.
EXPORT_JOBS_PATH = "/rest/export/jobs"

# Statuses that mean the export is still being produced. INTERRUPTED is included because an export that
# was running on a node that shut down is rescheduled automatically, so it is not a terminal state.
IN_PROGRESS_STATUSES = frozenset({"PENDING", "IN_PROGRESS", "INTERRUPTED"})

# How long to wait between polls, and how long to keep polling before giving up. A site-wide export of a
# large space takes minutes rather than seconds.
POLL_INTERVAL_SECONDS = 5
POLL_TIMEOUT_SECONDS = 30 * 60

# Size of the chunks the CSV is streamed to disk in, so that a large export is never held in memory.
DOWNLOAD_CHUNK_BYTES = 64 * 1024


def describe_error(response: requests.Response) -> str:
    """Describe a failed response using the app's error body when it sent one.

    Errors are returned as a JSON body with a title, an optional description, and any field-level
    violations. A response that carries no such body (a proxy error page, for example) falls back to its
    status line and raw text.

    :param response: The failed response.
    :return: A single-line description suitable for an error message.
    """
    try:
        body = response.json()
    except ValueError:
        return f"HTTP {response.status_code}: {response.text.strip()[:200]}"

    if not isinstance(body, dict) or "restErrorTitle" not in body:
        return f"HTTP {response.status_code}: {body}"

    parts = [body["restErrorTitle"]]
    if body.get("restErrorDescription"):
        parts.append(body["restErrorDescription"])
    for violation in body.get("violations", []):
        parts.append(f"{violation.get('fieldName')}: {violation.get('message')}")
    return f"HTTP {response.status_code}: " + " - ".join(parts)


def start_space_findings_export(session: requests.Session, base_url: str, space_id: int,
                                rule_names: str | None) -> dict:
    """Start the export of a space's findings.

    :param session: Session carrying the API key.
    :param base_url: Base URL of the app backend.
    :param space_id: The ID of the space to export.
    :param rule_names: Comma-separated rule names to restrict the findings to, or None for every rule.
    :return: The created export job.
    """
    params = {"spaceId": space_id}
    if rule_names:
        params["ruleNames"] = rule_names

    response = session.post(base_url + SPACE_FINDINGS_EXPORT_PATH, params=params)
    if not response.ok:
        raise RuntimeError(f"Could not start the export. {describe_error(response)}")
    return response.json()


def wait_until_ready(session: requests.Session, base_url: str, job_id: str) -> dict:
    """Poll an export job until its file can be downloaded.
j
    :param session: Session carrying the API key.
    :param base_url: Base URL of the app backend.
    :param job_id: The ID of the export job.
    :return: The job once it reports READY.
    """
    deadline = time.monotonic() + POLL_TIMEOUT_SECONDS
    last_progress = None

    while True:
        response = session.get(f"{base_url}{EXPORT_JOBS_PATH}/{job_id}")
        if not response.ok:
            raise RuntimeError(f"Could not read the status of export {job_id}. {describe_error(response)}")
        job = response.json()
        status = job["status"]

        if status == "READY":
            return job
        if status == "FAILED":
            raise RuntimeError(f"Export {job_id} failed: {job.get('errorMessage') or 'no reason given'}")
        if status == "DOWNLOADED":
            raise RuntimeError(f"Export {job_id} has already been downloaded, and cannot be downloaded again.")
        if status not in IN_PROGRESS_STATUSES:
            raise RuntimeError(f"Export {job_id} reported an unexpected status: {status}")

        # Only report progress when it changes, so a long export does not produce a line per poll.
        progress = (job.get("completedTargetsCount"), job.get("totalTargetsCount"))
        if progress != last_progress and progress[1]:
            print(f"  {status}: {progress[0] or 0} of {progress[1]} items exported")
            last_progress = progress

        if time.monotonic() >= deadline:
            raise RuntimeError(f"Export {job_id} was still {status} after {POLL_TIMEOUT_SECONDS} seconds.")
        time.sleep(POLL_INTERVAL_SECONDS)


def download_export(session: requests.Session, base_url: str, job_id: str, output_path: str) -> None:
    """Stream a finished export's CSV file to disk.

    :param session: Session carrying the API key.
    :param base_url: Base URL of the app backend.
    :param job_id: The ID of the export job.
    :param output_path: Path the CSV is written to.
    """
    with session.get(f"{base_url}{EXPORT_JOBS_PATH}/{job_id}/download", stream=True) as response:
        if not response.ok:
            raise RuntimeError(f"Could not download export {job_id}. {describe_error(response)}")
        with open(output_path, "wb") as output_file:
            for chunk in response.iter_content(chunk_size=DOWNLOAD_CHUNK_BYTES):
                output_file.write(chunk)


def build_session(api_key: str) -> requests.Session:
    """Build a session that authenticates every request with the given API key.

    The app's administration screen copies keys without the "Bearer " scheme attached. The value is accepted
    with or without it.

    :param api_key: The raw API key, optionally prefixed with "Bearer ".
    :return: A session carrying the Authorization header.
    """
    token = api_key.strip()
    if not token.lower().startswith("bearer "):
        token = f"Bearer {token}"

    session = requests.Session()
    session.headers.update({"Authorization": token, "Accept": "application/json,text/csv"})
    return session


def parse_args() -> argparse.Namespace:
    """Parse command line arguments.

    :return: The parsed arguments.
    """
    parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
    parser.add_argument("--space-id", type=int, required=True, help="The numeric ID of the space to export.")
    parser.add_argument("--output", default="findings.csv", help="Path the exported CSV is written to.")
    parser.add_argument("--rule-names",
                        help="Comma-separated rule names to restrict the export to, for example"
                             " 'AWS Access Key ID,Slack Token'. Defaults to every rule.")
    return parser.parse_args()


def main() -> None:
    args = parse_args()

    base_url = "https://security-for-confluence.soteri.io"
    api_key = os.environ.get("SOTERI_API_KEY")
    if not api_key:
        sys.exit("Set SOTERI_API_KEY before running this script.")

    session = build_session(api_key)
    base_url = base_url.rstrip("/")

    try:
        job = start_space_findings_export(session, base_url, args.space_id, args.rule_names)
        job_id = job["jobId"]
        print(f"Started export {job_id} for space {args.space_id}.")

        wait_until_ready(session, base_url, job_id)
        print(f"Export {job_id} is ready. Downloading...")

        download_export(session, base_url, job_id, args.output)
        print(f"Wrote {args.output}.")
    except RuntimeError as error:
        sys.exit(str(error))

if __name__ == "__main__":
    main()