Module boosted.api.api_client

Expand source code
# Copyright (C) 2020 Gradient Boosted Investments, Inc. - All Rights Reserved

import base64
import csv
import datetime
import io
import json
import logging
import math
import mimetypes
import os
import sys
import time
from datetime import timedelta
from typing import Any, Dict, List, Literal, Optional, Union
from urllib import parse

import numpy as np
import pandas as pd
import requests
from boosted.api.api_type import (
    BoostedDate,
    ChunkStatus,
    ColumnSubRole,
    DataAddType,
    DataSetConfig,
    DataSetType,
    DateIdentCountryCurrency,
    GbiIdSecurity,
    HedgeExperiment,
    HedgeExperimentDetails,
    HedgeExperimentScenario,
    PortfolioSettings,
    Status,
    hedge_experiment_type,
)
from boosted.api.api_util import (
    infer_dataset_schema,
    protoCubeJsonDataToDataFrame,
    validate_start_and_end_dates,
)
from dateutil import parser

logger = logging.getLogger("boosted.api.client")
g_boosted_api_url = "https://insights.boosted.ai"
g_boosted_api_url_dev = "https://insights-dev.boosted.ai"
WATCHLIST_ROUTE_PREFIX = "/api/dal/watchlist"
ROUTE_PREFIX = WATCHLIST_ROUTE_PREFIX
DAL_WATCHLIST_ROUTE = "/api/v0/watchlist"
DAL_PA_ROUTE = "/api/v0/portfolio-analysis"
PORTFOLIO_GROUP_ROUTE = "/api/v0/portfolio-group"

RISK_FACTOR = "risk-factor"
RISK_FACTOR_V2 = "risk-factor-v2"
RISK_FACTOR_COLUMNS = [
    "depth",
    "identifier",
    "stock_count",
    "volatility",
    "exposure",
    "rating",
    "rating_delta",
]


GET_SEC_INFO_QRY = """
query GetSecurities($ids: [Int!]) {
  securities(ids: $ids) {
    gbiId
    symbol
    name
    isin
    currency
    country
    sector {
      name
      topParentName
    }
  }
}
"""

# watch for changes here:
# https://github.com/GBI-Core/boosted-insights-web-server/blob/master/src/main/resources/graphql/watchlist.graphqls#L1 # noqa
WATCHLIST_ANALYSIS_QRY = """
query WatchlistAnalysis(
  $modelIds: [ID!]
  $portfolioIds: [ID!]
  $gbiIds: [Int]
  $date: String!
) {
  watchlistAnalysis(
    modelIds: $modelIds
    portfolioIds: $portfolioIds
    gbiIds: $gbiIds
    minDate: $date
    maxDate: $date
  ) {
    ...AnalysisFragment
    ...ExplainWeightsFragment
  }
}

fragment AnalysisFragment on WatchlistAnalysis {
  gbiId
  analysisDates {
    date
    aggregateSignal {
      rating
      ratingDelta
    }
    portfoliosSignals {
      portfolioId
      rank
      signalDelta
      rating
      ratingDelta
    }
  }
}


fragment ExplainWeightsFragment on WatchlistAnalysis {
  gbiId
  analysisDates {
    date
    portfoliosSignals {
      explainWeightNeg
      explainWeightPos
    }
  }
}
"""

GET_MODELS_FOR_PORTFOLIOS_QRY = """
query GetPortfoliosAndModels($ids: [ID!]) {
  portfolios(ids: $ids) {
    id
    name
    modelId
    modelName
  }
}
"""

GET_EXCESS_RETURN_QRY = """
query GetExcessReturn($modelIds: [ID!], $gbiIds: [Int], $date: String!) {
  models(ids: $modelIds) {
    id
    equityExplorerData(date: $date, gbiIds: $gbiIds) {
      hasEquityExplorerData
      equityExplorerSummaryStatistics {
        gbiId
        ER {
          SP {
            oneMonth
          }
        }
      }
    }
  }
}
"""


class BoostedAPIException(Exception):
    def __init__(self, value, data=None):
        self.value = value
        self.data = data

    def __str__(self):
        return repr(self.value)


def convert_date(date: Optional[BoostedDate]) -> Optional[datetime.date]:
    if isinstance(date, str):
        try:
            return parser.parse(date)
        except Exception as e:
            raise BoostedAPIException(f"Unable to parse date: {str(e)}")
    return date


class BoostedClient:
    def __init__(
        self, api_key, override_uri=None, debug=False, proxy=None, disable_verify_ssl=False
    ):
        """
        Parameters
        ----------
        api_key: str
            Your API key provided by the Boosted application.  See your profile
            to generate a new key.
        proxy: str
            Your organization may require the use of a proxy for access.
            The address of a HTTPS proxy in the format of <address>:<port>.
            Examples are "123.456.789:123" or "my.proxy.com:123".
            Do not prepend with "https://".
        disable_verify_ssl: bool
            Your networking setup may be behind a firewall which performs SSL
            inspection. Either set the REQUESTS_CA_BUNDLE environment variable
            to point to the location of a custom certificate bundle, or set this
            parameter to True to disable SSL verification as a workaround.
        """
        if override_uri is None:
            self.base_uri = g_boosted_api_url
        else:
            self.base_uri = override_uri
        self.api_key = api_key
        self.debug = debug
        self._request_params = {}
        if debug:
            logger.setLevel(logging.DEBUG)
        else:
            logger.setLevel(logging.INFO)
        if proxy is not None:
            self._request_params["proxies"] = {"https": proxy}
        if disable_verify_ssl:
            self._request_params["verify"] = False

    def __print_json_info(self, json_data, isInference=False):
        if "warnings" in json_data.keys():
            for warning in json_data["warnings"]:
                logger.warning("  {0}".format(warning))
        if "errors" in json_data.keys():
            for error in json_data["errors"]:
                logger.error("  {0}".format(error))
                return Status.FAIL

        if "result" in json_data.keys():
            results_data = json_data["result"]
            if isInference:
                if "inferenceResultsUrl" in results_data.keys():
                    res_url = parse.urlparse(results_data["inferenceResultsUrl"])
                    logger.debug(res_url)
                    logger.info("Inference started.")
            if "updateCount" in results_data.keys():
                logger.info("Updated {0} rows.".format(results_data["updateCount"]))
            if "createCount" in results_data.keys():
                logger.info("Created {0} rows.".format(results_data["createCount"]))
            return Status.SUCCESS

    def __to_date_obj(self, dt):
        if isinstance(dt, datetime.datetime):
            dt = dt.date()
        elif isinstance(dt, datetime.date):
            return dt
        elif isinstance(dt, str):
            try:
                dt = parser.parse(dt).date()
            except ValueError:
                raise ValueError('dt: "' + dt + '" is not a valid date.')
        return dt

    def __iso_format(self, dt):
        date = self.__to_date_obj(dt)
        if date is not None:
            date = date.isoformat()
        return date

    def _check_status_code(self, response, isInference=False):
        has_json = False
        try:
            logger.debug(response.headers)
            if "Content-Type" in response.headers:
                if response.headers["Content-Type"].startswith("application/json"):
                    json_data = response.json()
                    has_json = True
            else:
                has_json = False
        except json.JSONDecodeError:
            logger.error("ERROR: response has no JSON payload.")
        if response.status_code == 200 or response.status_code == 202:
            if has_json:
                self.__print_json_info(json_data, isInference)
            else:
                pass
            return Status.SUCCESS
        if response.status_code == 404:
            if has_json:
                self.__print_json_info(json_data, isInference)
            raise BoostedAPIException(
                'Server "{0}" not reachable.  Code {1}.'.format(
                    self.base_uri, response.status_code
                ),
                data=response,
            )
        if response.status_code == 400:
            if has_json:
                self.__print_json_info(json_data, isInference)
            if isInference:
                return Status.FAIL
            else:
                raise BoostedAPIException("Error, bad request.  Check the dataset ID.", response)
        if response.status_code == 401:
            if has_json:
                self.__print_json_info(json_data, isInference)
            raise BoostedAPIException("Authorization error.", response)
        else:
            if has_json:
                self.__print_json_info(json_data, isInference)
            raise BoostedAPIException(
                "Error in API response.  Status code={0} {1}\n{2}".format(
                    response.status_code, response.reason, response.headers
                ),
                response,
            )

    def _try_extract_error_code(self, result):
        logger.info(result.headers)
        if "Content-Type" in result.headers:
            if result.headers["Content-Type"].startswith("application/json"):
                if "errors" in result.json():
                    return result.json()["errors"]
            if result.headers["Content-Type"].startswith("text/plain"):
                return result.text
        return str(result.reason)

    def _check_ok_or_err_with_msg(self, res, potential_error_msg: str):
        if not res.ok:
            error = self._try_extract_error_code(res)
            logger.error(error)
            raise BoostedAPIException(f"{potential_error_msg}: {error}")

    def query_dataset(self, dataset_id):
        url = self.base_uri + "/api/datasets/{0}".format(dataset_id)
        headers = {"Authorization": "ApiKey " + self.api_key}
        res = requests.get(url, headers=headers, **self._request_params)
        if res.ok:
            return res.json()
        else:
            error_msg = self._try_extract_error_code(res)
            logger.error(error_msg)
            raise BoostedAPIException("Failed to query dataset: {0}.".format(error_msg))

    def export_global_data(
        self,
        dataset_id,
        start=(datetime.date.today() - timedelta(days=365 * 25)),
        end=datetime.date.today(),
        timeout=600,
    ):
        query_info = self.query_dataset(dataset_id)
        if DataSetType[query_info["type"]] != DataSetType.GLOBAL:
            raise BoostedAPIException(
                f"Incorrect dataset type: {query_info['type']}" f" - Expected {DataSetType.GLOBAL}"
            )
        return self.export_data(dataset_id, start, end, timeout)

    def export_independent_data(
        self,
        dataset_id,
        start=(datetime.date.today() - timedelta(days=365 * 25)),
        end=datetime.date.today(),
        timeout=600,
    ):
        query_info = self.query_dataset(dataset_id)
        if DataSetType[query_info["type"]] != DataSetType.STRATEGY:
            raise BoostedAPIException(
                f"Incorrect dataset type: {query_info['type']}"
                f" - Expected {DataSetType.STRATEGY}"
            )
        return self.export_data(dataset_id, start, end, timeout)

    def export_dependent_data(
        self,
        dataset_id,
        start=(datetime.date.today() - timedelta(days=365 * 25)),
        end=datetime.date.today(),
        timeout=600,
    ):
        query_info = self.query_dataset(dataset_id)
        if DataSetType[query_info["type"]] != DataSetType.STOCK:
            raise BoostedAPIException(
                f"Incorrect dataset type: {query_info['type']}" f" - Expected {DataSetType.STOCK}"
            )
        return self.export_data(dataset_id, start, end, timeout)

    def export_data(
        self,
        dataset_id,
        start=(datetime.date.today() - timedelta(days=365 * 25)),
        end=datetime.date.today(),
        timeout=600,
    ):
        logger.info("Requesting start={0} end={1}.".format(start, end))
        request_url = "/api/datasets/" + dataset_id + "/export-data"
        headers = {"Authorization": "ApiKey " + self.api_key}
        start = self.__iso_format(start)
        end = self.__iso_format(end)
        params = {"start": start, "end": end}
        logger.debug("URL={0}, headers={1}, params={2}".format(request_url, headers, params))
        res = requests.get(
            self.base_uri + request_url,
            headers=headers,
            params=params,
            timeout=timeout,
            **self._request_params,
        )
        if res.ok or self._check_status_code(res):
            buf = io.StringIO(res.text)
            df = pd.read_csv(buf, index_col=0, parse_dates=True)
            if "price" in df.columns:
                df = df.drop("price", axis=1)
            return df
        else:
            error_msg = self._try_extract_error_code(res)
            logger.error(error_msg)
            raise BoostedAPIException("Failed to query dataset: {0}.".format(error_msg))

    def _get_inference(self, model_id, inference_date=datetime.date.today()):
        request_url = "/api/models/" + model_id + "/inference-results"
        headers = {"Authorization": "ApiKey " + self.api_key}
        params = {}
        params["date"] = self.__iso_format(inference_date)
        logger.debug(request_url + ", " + str(headers) + ", " + str(params))
        res = requests.get(
            self.base_uri + request_url, headers=headers, params=params, **self._request_params
        )
        status = self._check_status_code(res, isInference=True)
        if status == Status.SUCCESS:
            return res, status
        else:
            return None, status

    def get_inference(
        self, model_id, inference_date=datetime.date.today(), block=False, timeout_minutes=30
    ):
        start_time = datetime.datetime.now()
        while True:
            for numRetries in range(3):
                res, status = self._get_inference(model_id, inference_date)
                if res is not None:
                    continue
                else:
                    if status == Status.FAIL:
                        return Status.FAIL
                    logger.info("Retrying...")
            if res is None:
                logger.error("Max retries reached.  Request failed.")
                return None

            json_data = res.json()
            if "result" in json_data.keys():
                if json_data["result"]["status"] == "RUNNING":
                    still_running = True
                    if not block:
                        logger.warn("Inference job is still running.")
                        return None
                    else:
                        logger.info(
                            "Inference job is still running.  Time elapsed={0}.".format(
                                datetime.datetime.now() - start_time
                            )
                        )
                        time.sleep(10)
                else:
                    still_running = False

                if not still_running and json_data["result"]["status"] == "COMPLETE":
                    csv = json_data["result"]["signals"]
                    logger.info(json_data["result"])
                    if self._check_status_code(res, isInference=True):
                        logger.info(
                            "Total run time = {0}.".format(datetime.datetime.now() - start_time)
                        )
                        return csv
            else:
                if "errors" in json_data.keys():
                    logger.error(json_data["errors"])
                else:
                    logger.error("Error getting inference for date {0}.".format(inference_date))
                return None
            if (datetime.datetime.now() - start_time).total_seconds() / 60.0 > timeout_minutes:
                logger.error("Timeout waiting for job completion.")
                return None

    def createDataset(self, schema):
        request_url = "/api/datasets"
        headers = {"Authorization": "ApiKey " + self.api_key, "Content-Type": "application/json"}
        s = json.dumps(schema)
        logger.info("Creating dataset with schema " + s)
        res = requests.post(
            self.base_uri + request_url, data=s, headers=headers, **self._request_params
        )
        if res.ok:
            return res.json()["result"]
        else:
            raise BoostedAPIException("Dataset creation failed.")

    def getUniverse(self, modelId, date=None):
        if date is not None:
            url = "/api/models/{0}/universe/{1}".format(modelId, self.__iso_format(date))
            logger.info("Getting universe for date: {0}.".format(date))
        else:
            url = "/api/models/{0}/universe/".format(modelId)
        headers = {"Authorization": "ApiKey " + self.api_key}
        res = requests.get(self.base_uri + url, headers=headers, **self._request_params)
        if res.ok:
            buf = io.StringIO(res.text)
            df = pd.read_csv(buf, index_col=0, parse_dates=True)
            return df
        else:
            error = self._try_extract_error_code(res)
            logger.error(
                "There was a problem getting this universe or model ID: {0}.".format(error)
            )
            raise BoostedAPIException("Failed to get universe: {0}".format(error))

    def updateUniverse(self, modelId, universe_df, date=datetime.date.today() + timedelta(1)):
        date = self.__iso_format(date)
        url = self.base_uri + "/api/models/{0}/universe/{1}".format(modelId, date)
        headers = {"Authorization": "ApiKey " + self.api_key}
        logger.info("Updating universe for date {0}.".format(date))
        if isinstance(universe_df, pd.core.frame.DataFrame):
            buf = io.StringIO()
            universe_df.to_csv(buf)
            target = ("uploaded_universe.csv", buf.getvalue(), "text/csv")
            files_req = {}
            files_req["universe"] = target
            res = requests.post(url, files=files_req, headers=headers, **self._request_params)
        elif isinstance(universe_df, str):
            target = ("uploaded_universe.csv", universe_df, "text/csv")
            files_req = {}
            files_req["universe"] = target
            res = requests.post(url, files=files_req, headers=headers, **self._request_params)
        else:
            raise BoostedAPIException("Expected CSV as str or Pandas DataFrame.")
        if res.ok:
            logger.info("Universe update successful.")
            if "warnings" in res.json():
                logger.info("Warnings: {0}.".format(res.json()["warnings"]))
                return res.json()["warnings"]
            else:
                return "No warnings."
        else:
            error_msg = self._try_extract_error_code(res)
            raise BoostedAPIException("Failed to get universe: {0}.".format(error_msg))

    def create_universe(
        self, universe: Union[pd.DataFrame, str], name: str, description: str
    ) -> List[str]:
        PRESENT = "PRESENT"
        ANY = "ANY"
        EARLIST_DATE = "1900-01-01"
        LATEST_DATE = "4000-01-01"

        if isinstance(universe, (str, bytes, os.PathLike)):
            universe = pd.read_csv(universe)

        universe.columns = universe.columns.str.lower()

        # Clients are free to leave out data. Fill in some defaults here.
        if "from" not in universe.columns:
            universe["from"] = EARLIST_DATE
        if "to" not in universe.columns:
            universe["to"] = LATEST_DATE
        if "currency" not in universe.columns:
            universe["currency"] = ANY
        if "country" not in universe.columns:
            universe["country"] = ANY
        if "isin" not in universe.columns:
            universe["isin"] = None
        if "symbol" not in universe.columns:
            universe["symbol"] = None

        # to prevent conflicts with python keywords
        universe.rename(columns={"from": "from_date", "to": "to_date"}, inplace=True)

        universe = universe.replace({np.nan: None})
        security_country_currency_date_list = []
        for i, r in enumerate(universe.itertuples()):
            id_type = ColumnSubRole.ISIN
            identifier = r.isin

            if identifier is None:
                id_type = ColumnSubRole.SYMBOL
                identifier = str(r.symbol)

            # if identifier is still None, it means that there is no ISIN or
            # SYMBOL for this row, in which case we throw an error
            if identifier is None:
                raise BoostedAPIException(
                    (
                        f"Missing identifier column in universe row {i + 1}"
                        " should contain ISIN or Symbol"
                    )
                )

            security_country_currency_date_list.append(
                DateIdentCountryCurrency(
                    date=r.from_date or EARLIST_DATE,
                    identifier=identifier,
                    country=r.country or ANY,
                    currency=r.currency or ANY,
                    id_type=id_type,
                )
            )

        gbi_id_objs = self.getGbiIdFromIdentCountryCurrencyDate(security_country_currency_date_list)

        security_list = []
        for i, r in enumerate(universe.itertuples()):
            # if we have a None here, we failed to map to a gbi id
            if gbi_id_objs[i] is None:
                raise BoostedAPIException(f"Unable to map row: {tuple(r)}")

            security_list.append(
                {
                    "stockId": gbi_id_objs[i].gbi_id,
                    "fromZ": r.from_date or EARLIST_DATE,
                    "toZ": LATEST_DATE if r.to_date in (PRESENT, None) else r.to_date,
                    "removal": False,
                    "source": "UPLOAD",
                }
            )

        url = self.base_uri + "/api/template-universe/save"
        headers = {"Authorization": "ApiKey " + self.api_key, "Content-Type": "application/json"}
        req = {"name": name, "description": description, "modificationDaos": security_list}

        res = requests.post(url, json=req, headers=headers, **self._request_params)
        self._check_ok_or_err_with_msg(res, "Failed to create universe")

        if "warnings" in res.json():
            logger.info("Warnings: {0}.".format(res.json()["warnings"]))
            return res.json()["warnings"].splitlines()
        else:
            return []

    def validate_dataframe(self, df):
        if not isinstance(df, pd.core.frame.DataFrame):
            logger.error("Dataset must be of type Dataframe.")
            return False
        if type(df.index) != pd.core.indexes.datetimes.DatetimeIndex:
            logger.error("Index must be DatetimeIndex.")
            return False
        if len(df.columns) == 0:
            logger.error("No feature columns exist.")
            return False
        if len(df) == 0:
            logger.error("No rows exist.")
        return True

    def get_dataset_schema(self, dataset_id):
        url = self.base_uri + "/api/datasets/{0}/schema".format(dataset_id)
        headers = {"Authorization": "ApiKey " + self.api_key}
        res = requests.get(url, headers=headers, **self._request_params)
        if res.ok:
            json_schema = res.json()
        else:
            error_msg = self._try_extract_error_code(res)
            logger.error(error_msg)
            raise BoostedAPIException("Failed to query dataset: {0}.".format(error_msg))
        return DataSetConfig.fromDict(json_schema["result"])

    def add_dependent_dataset(
        self, dataset, datasetName="DependentDataset", schema=None, timeout=600, block=True
    ):
        result = self.add_dependent_dataset_with_warnings(
            dataset, datasetName, schema, timeout, block
        )
        return result["dataset_id"]

    def add_dependent_dataset_with_warnings(
        self,
        dataset,
        datasetName="DependentDataset",
        schema=None,
        timeout=600,
        block=True,
        no_exception_on_chunk_error=False,
    ):
        if not self.validate_dataframe(dataset):
            logger.error("dataset failed validation.")
            return None
        if schema is None:
            schema = infer_dataset_schema(datasetName, dataset, DataSetType.STOCK)
        dsid = self.createDataset(schema.toDict())
        logger.info("Creating dataset with ID = {0}.".format(dsid))
        result = self.add_dependent_data(
            dsid,
            dataset,
            timeout,
            block,
            data_type=DataAddType.CREATION,
            no_exception_on_chunk_error=no_exception_on_chunk_error,
        )
        return {"dataset_id": dsid, "warnings": result["warnings"], "errors": result["errors"]}

    def add_independent_dataset(
        self, dataset, datasetName="IndependentDataset", schema=None, timeout=600, block=True
    ):
        result = self.add_independent_dataset_with_warnings(
            dataset, datasetName, schema, timeout, block
        )
        return result["dataset_id"]

    def add_independent_dataset_with_warnings(
        self,
        dataset,
        datasetName="IndependentDataset",
        schema=None,
        timeout=600,
        block=True,
        no_exception_on_chunk_error=False,
    ):
        if not self.validate_dataframe(dataset):
            logger.error("dataset failed validation.")
            return None
        if schema is None:
            schema = infer_dataset_schema(datasetName, dataset, DataSetType.STRATEGY)
        schemaDict = schema.toDict()
        if "configurationDataJson" not in schemaDict:
            schemaDict["configurationDataJson"] = "{}"
        dsid = self.createDataset(schemaDict)
        logger.info("Creating dataset with ID = {0}.".format(dsid))
        result = self.add_independent_data(
            dsid,
            dataset,
            timeout,
            block,
            data_type=DataAddType.CREATION,
            no_exception_on_chunk_error=no_exception_on_chunk_error,
        )
        return {"dataset_id": dsid, "warnings": result["warnings"], "errors": result["errors"]}

    def add_global_dataset(
        self, dataset, datasetName="GlobalDataset", schema=None, timeout=600, block=True
    ):
        result = self.add_global_dataset_with_warnings(dataset, datasetName, schema, timeout, block)
        return result["dataset_id"]

    def add_global_dataset_with_warnings(
        self,
        dataset,
        datasetName="GlobalDataset",
        schema=None,
        timeout=600,
        block=True,
        no_exception_on_chunk_error=False,
    ):
        if not self.validate_dataframe(dataset):
            logger.error("dataset failed validation.")
            return None
        if schema is None:
            schema = infer_dataset_schema(datasetName, dataset, DataSetType.GLOBAL)
        dsid = self.createDataset(schema.toDict())
        logger.info("Creating dataset with ID = {0}.".format(dsid))
        result = self.add_global_data(
            dsid,
            dataset,
            timeout,
            block,
            data_type=DataAddType.CREATION,
            no_exception_on_chunk_error=no_exception_on_chunk_error,
        )
        return {"dataset_id": dsid, "warnings": result["warnings"], "errors": result["errors"]}

    def add_independent_data(
        self,
        dataset_id,
        csv_data,
        timeout=600,
        block=True,
        data_type=DataAddType.HISTORICAL,
        no_exception_on_chunk_error=False,
    ):
        query_info = self.query_dataset(dataset_id)
        if DataSetType[query_info["type"]] != DataSetType.STRATEGY:
            raise BoostedAPIException(
                f"Incorrect dataset type: {query_info['type']}"
                f" - Expected {DataSetType.STRATEGY}"
            )
        warnings, errors = self.setup_chunk_and_upload_data(
            dataset_id, csv_data, data_type, timeout, block, no_exception_on_chunk_error
        )
        if len(warnings) > 0:
            logger.warning(
                "Encountered {0} total warnings while uploading dataset.".format(len(warnings))
            )
        if len(errors) > 0:
            raise BoostedAPIException(
                "Encountered {0} total ERRORS while uploading dataset".format(len(errors))
                + "\n".join(errors)
            )
        return {"warnings": warnings, "errors": errors}

    def add_dependent_data(
        self,
        dataset_id,
        csv_data,
        timeout=600,
        block=True,
        data_type=DataAddType.HISTORICAL,
        no_exception_on_chunk_error=False,
    ):
        warnings = []
        query_info = self.query_dataset(dataset_id)
        if DataSetType[query_info["type"]] != DataSetType.STOCK:
            raise BoostedAPIException(
                f"Incorrect dataset type: {query_info['type']}" f" - Expected {DataSetType.STOCK}"
            )
        warnings, errors = self.setup_chunk_and_upload_data(
            dataset_id, csv_data, data_type, timeout, block, no_exception_on_chunk_error
        )
        if len(warnings) > 0:
            logger.warning(
                "Encountered {0} total warnings while uploading dataset.".format(len(warnings))
            )
        if len(errors) > 0:
            raise BoostedAPIException(
                "Encountered {0} total ERRORS while uploading dataset".format(len(errors))
                + "\n".join(errors)
            )
        return {"warnings": warnings, "errors": errors}

    def add_global_data(
        self,
        dataset_id,
        csv_data,
        timeout=600,
        block=True,
        data_type=DataAddType.HISTORICAL,
        no_exception_on_chunk_error=False,
    ):
        query_info = self.query_dataset(dataset_id)
        if DataSetType[query_info["type"]] != DataSetType.GLOBAL:
            raise BoostedAPIException(
                f"Incorrect dataset type: {query_info['type']}" f" - Expected {DataSetType.GLOBAL}"
            )
        warnings, errors = self.setup_chunk_and_upload_data(
            dataset_id, csv_data, data_type, timeout, block, no_exception_on_chunk_error
        )
        if len(warnings) > 0:
            logger.warning(
                "Encountered {0} total warnings while uploading dataset.".format(len(warnings))
            )
        if len(errors) > 0:
            raise BoostedAPIException(
                "Encountered {0} total ERRORS while uploading dataset".format(len(errors))
                + "\n".join(errors)
            )
        return {"warnings": warnings, "errors": errors}

    def get_csv_buffer(self):
        return io.StringIO()

    def start_chunked_upload(self, dataset_id):
        url = self.base_uri + "/api/datasets/{0}/start-chunked-upload".format(dataset_id)
        headers = {"Authorization": "ApiKey " + self.api_key}
        res = requests.post(url, headers=headers, **self._request_params)
        if res.ok:
            return res.json()["result"]
        else:
            error_msg = self._try_extract_error_code(res)
            logger.error(error_msg)
            raise BoostedAPIException(
                "Failed to obtain dataset lock for upload: {0}.".format(error_msg)
            )

    def abort_chunked_upload(self, dataset_id, chunk_id):
        url = self.base_uri + "/api/datasets/{0}/abort-chunked-upload".format(dataset_id)
        headers = {"Authorization": "ApiKey " + self.api_key}
        params = {"uploadGroupId": chunk_id}
        res = requests.post(url, headers=headers, **self._request_params, params=params)
        if not res.ok:
            error_msg = self._try_extract_error_code(res)
            logger.error(error_msg)
            raise BoostedAPIException(
                "Failed to abort dataset lock during error: {0}.".format(error_msg)
            )

    def check_dataset_ingestion_completion(self, dataset_id, chunk_id, start_time):
        url = self.base_uri + "/api/datasets/{0}/upload-chunk-status".format(dataset_id)
        headers = {"Authorization": "ApiKey " + self.api_key}
        params = {"uploadGroupId": chunk_id}
        res = requests.get(url, headers=headers, **self._request_params, params=params)
        res = res.json()

        finished = False
        warnings = []
        errors = []

        if type(res) == dict:
            dataset_status = res["datasetStatus"]
            chunk_status = res["chunkStatus"]
            if chunk_status != ChunkStatus.PROCESSING.value:
                finished = True
                errors = res["errors"]
                warnings = res["warnings"]
                successful_rows = res["successfulRows"]
                total_rows = res["totalRows"]
                logger.info(
                    f"Successfully ingested {successful_rows} out of {total_rows} uploaded rows."
                )
                if chunk_status in [
                    ChunkStatus.SUCCESS.value,
                    ChunkStatus.WARNING.value,
                    ChunkStatus.ERROR.value,
                ]:
                    if dataset_status != "AVAILABLE":
                        raise BoostedAPIException(
                            "Dataset was unexpectedly unavailable after chunk upload finished."
                        )
                    else:
                        logger.info("Ingestion complete.  Uploaded data is ready for use.")
                elif chunk_status == ChunkStatus.ABORTED.value:
                    errors.append(
                        "Dataset chunk upload was aborted by server! Upload did not succeed."
                    )
                else:
                    errors.append("Unexpected data ingestion status: {0}.".format(chunk_status))
            logger.info(
                "Data ingestion still running.  Time elapsed={0}.".format(
                    datetime.datetime.now() - start_time
                )
            )
        else:
            raise BoostedAPIException("Unable to get status of dataset ingestion.")
        return {"finished": finished, "warnings": warnings, "errors": errors}

    def _commit_chunked_upload(self, dataset_id, chunk_id, data_type, block=True, timeout=600):
        url = self.base_uri + "/api/datasets/{0}/commit-chunked-upload".format(dataset_id)
        headers = {"Authorization": "ApiKey " + self.api_key}
        params = {
            "uploadGroupId": chunk_id,
            "dataAddType": data_type,
            "sendCompletionEmail": not block,
        }
        res = requests.post(url, headers=headers, **self._request_params, params=params)
        if not res.ok:
            error_msg = self._try_extract_error_code(res)
            logger.error(error_msg)
            raise BoostedAPIException("Failed to commit dataset files: {0}.".format(error_msg))

        if block:
            start_time = datetime.datetime.now()
            # Keep waiting until upload is no longer in UPDATING state...
            while True:
                result = self.check_dataset_ingestion_completion(dataset_id, chunk_id, start_time)
                if result["finished"]:
                    break

                if (datetime.datetime.now() - start_time).total_seconds() > timeout:
                    err_str = (
                        f"Timeout waiting for commit of dataset: {dataset_id} | chunk: {chunk_id}"
                    )
                    logger.error(err_str)
                    return [], [err_str]

                time.sleep(10)
            return result["warnings"], result["errors"]
        else:
            return [], []

    def setup_chunk_and_upload_data(
        self,
        dataset_id,
        csv_data,
        data_type,
        timeout=600,
        block=True,
        no_exception_on_chunk_error=False,
    ):
        chunk_id = self.start_chunked_upload(dataset_id)
        logger.info("Obtained lock on dataset for upload: " + chunk_id)
        try:
            warnings, errors = self.chunk_and_upload_data(
                dataset_id, chunk_id, csv_data, timeout, no_exception_on_chunk_error
            )
            commit_warnings, commit_errors = self._commit_chunked_upload(
                dataset_id, chunk_id, data_type, block, timeout
            )
            return warnings + commit_warnings, errors + commit_errors
        except Exception:
            self.abort_chunked_upload(dataset_id, chunk_id)
            raise

    def chunk_and_upload_data(
        self, dataset_id, chunk_id, csv_data, timeout=600, no_exception_on_chunk_error=False
    ):
        if isinstance(csv_data, pd.core.frame.DataFrame):
            if not isinstance(csv_data.index, pd.core.indexes.datetimes.DatetimeIndex):
                raise BoostedAPIException("DataFrame must have DatetimeIndex as index type.")

            warnings = []
            errors = []
            logger.info("Uploading yearly.")
            for t in csv_data.index.to_period("Y").unique():
                if t is pd.NaT:
                    continue

                # serialize bit to string
                buf = self.get_csv_buffer()
                yearly_csv = csv_data.loc[str(t)]
                yearly_csv.to_csv(buf, header=True)
                raw_csv = buf.getvalue()

                # we are already chunking yearly... but if the csv still exceeds a healthy
                # limit of 50mb the final line of defence is to ignore date boundaries and
                # just chunk the rows. This is mostly for the cloudflare upload limit.
                size_lim = 50 * 1000 * 1000
                est_csv_size = sys.getsizeof(raw_csv)
                if est_csv_size > size_lim:
                    del raw_csv, buf
                    logger.info("Yearly data too large for single upload, chunking further...")
                    chunks = []
                    nchunks = math.ceil(est_csv_size / size_lim)
                    rows_per_chunk = math.ceil(len(yearly_csv) / nchunks)
                    for i in range(0, len(yearly_csv), rows_per_chunk):
                        buf = self.get_csv_buffer()
                        split_csv = yearly_csv.iloc[i : i + rows_per_chunk]
                        split_csv.to_csv(buf, header=True)
                        split_csv = buf.getvalue()
                        chunks.append(
                            (
                                "{0}-{1}".format(i + 1, min(len(yearly_csv), i + rows_per_chunk)),
                                split_csv,
                            )
                        )
                else:
                    chunks = [("all", raw_csv)]

                for i, (rows_descriptor, chunk_csv) in enumerate(chunks):
                    chunk_descriptor = "{0} in yearly chunk {1}".format(rows_descriptor, t)
                    logger.info(
                        "Uploading rows:"
                        + chunk_descriptor
                        + " (chunk {0} of {1}):".format(i + 1, len(chunks))
                    )
                    _, new_warnings, new_errors = self.upload_dataset_chunk(
                        chunk_descriptor,
                        dataset_id,
                        chunk_id,
                        chunk_csv,
                        timeout,
                        no_exception_on_chunk_error,
                    )
                    warnings.extend(new_warnings)
                    errors.extend(new_errors)
            return warnings, errors

        elif isinstance(csv_data, str):
            _, warnings, errors = self.upload_dataset_chunk(
                "all data", dataset_id, chunk_id, csv_data, timeout, no_exception_on_chunk_error
            )
            return warnings, errors
        else:
            raise BoostedAPIException("Expected CSV as str or Pandas DataFrame.")

    def upload_dataset_chunk(
        self,
        chunk_descriptor,
        dataset_id,
        chunk_id,
        csv_data,
        timeout=600,
        no_exception_on_chunk_error=False,
    ):
        logger.info("Starting upload: " + chunk_descriptor)
        url = self.base_uri + "/api/datasets/{0}/upload-dataset-chunk".format(dataset_id)
        headers = {"Authorization": "ApiKey " + self.api_key}
        files_req = {}
        warnings = []
        errors = []

        # make the network request
        target = ("uploaded_data.csv", csv_data, "text/csv")
        files_req["dataFile"] = target
        params = {"uploadGroupId": chunk_id}
        res = requests.post(
            url,
            params=params,
            files=files_req,
            headers=headers,
            timeout=timeout,
            **self._request_params,
        )

        if res.ok:
            logger.info(
                (
                    "Chunk upload completed.  "
                    "Ingestion started.  "
                    "Please wait until the data is in AVAILABLE state."
                )
            )
            if "warnings" in res.json():
                warnings = res.json()["warnings"]
                if len(warnings) > 0:
                    logger.warning("Uploaded chunk encountered data warnings: ")
                for w in warnings:
                    logger.warning(w)
        else:
            reason = "Upload failed: {0}, {1}".format(res.text, res.reason)
            logger.error(reason)
            if no_exception_on_chunk_error:
                errors.append(
                    "Chunk {0} failed: {1}. ".format(chunk_descriptor, reason)
                    + "Your data was only PARTIALLY uploaded. "
                    + "Please reattempt the upload of this chunk."
                )
            else:
                raise BoostedAPIException("Upload failed.")

        return res, warnings, errors

    def getAllocationsForDate(self, portfolio_id, date, rollback_to_last_available_date):
        date = self.__iso_format(date)
        endpoint = "latest-allocations" if rollback_to_last_available_date else "allocations"
        url = self.base_uri + "/api/portfolios/{0}/{1}".format(portfolio_id, endpoint)
        headers = {"Authorization": "ApiKey " + self.api_key}
        params = {"date": date}
        logger.info("Retrieving allocations information for date {0}.".format(date))
        res = requests.get(url, params=params, headers=headers, **self._request_params)
        if res.ok:
            logger.info("Allocations retrieval successful.")
            return res.json()
        else:
            error_msg = self._try_extract_error_code(res)
            raise BoostedAPIException("Failed to retrieve allocations: {0}.".format(error_msg))

    # New API method for fetching data from portfolio_holdings.pb2 file.
    def getAllocationsForDateV2(self, portfolio_id, date, rollback_to_last_available_date):
        date = self.__iso_format(date)
        endpoint = "latest-allocations-v2" if rollback_to_last_available_date else "allocations-v2"
        url = self.base_uri + "/api/portfolios/{0}/{1}".format(portfolio_id, endpoint)
        headers = {"Authorization": "ApiKey " + self.api_key}
        params = {"date": date}
        logger.info("Retrieving allocations information for date {0}.".format(date))
        res = requests.get(url, params=params, headers=headers, **self._request_params)
        if res.ok:
            logger.info("Allocations retrieval successful.")
            return res.json()
        else:
            error_msg = self._try_extract_error_code(res)
            raise BoostedAPIException("Failed to retrieve allocations: {0}.".format(error_msg))

    def getAllocationsByDates(self, portfolio_id, dates=None):
        url = self.base_uri + "/api/portfolios/{0}/allocationsByDate".format(portfolio_id)
        headers = {"Authorization": "ApiKey " + self.api_key}
        if dates is not None:
            fmt_dates = []
            for d in dates:
                fmt_dates.append(self.__iso_format(d))
            fmt_dates = ",".join(fmt_dates)
            params = {"dates": fmt_dates}
            logger.info("Retrieving allocations information for dates {0}.".format(fmt_dates))
        else:
            params = {"dates": None}
            logger.info("Retrieving allocations information for all dates")
        res = requests.get(url, params=params, headers=headers, **self._request_params)
        if res.ok:
            logger.info("Allocations retrieval successful.")
            return res.json()
        else:
            error_msg = self._try_extract_error_code(res)
            raise BoostedAPIException("Failed to retrieve allocations: {0}.".format(error_msg))

    def getSignalsForDate(self, portfolio_id, date, rollback_to_last_available_date):
        date = self.__iso_format(date)
        endpoint = "latest-signals" if rollback_to_last_available_date else "signals"
        url = self.base_uri + "/api/portfolios/{0}/{1}".format(portfolio_id, endpoint)
        headers = {"Authorization": "ApiKey " + self.api_key}
        params = {"date": date}
        logger.info("Retrieving signals information for date {0}.".format(date))
        res = requests.get(url, params=params, headers=headers, **self._request_params)
        if res.ok:
            logger.info("Signals retrieval successful.")
            return res.json()
        else:
            error_msg = self._try_extract_error_code(res)
            raise BoostedAPIException("Failed to retrieve signals: {0}.".format(error_msg))

    def getSignalsForAllDates(self, portfolio_id, dates=None):
        url = self.base_uri + "/api/portfolios/{0}/signalsByDate".format(portfolio_id)
        headers = {"Authorization": "ApiKey " + self.api_key}
        params = {}
        if dates is not None:
            fmt_dates = []
            for d in dates:
                fmt_dates.append(self.__iso_format(d))
            fmt_dates = ",".join(fmt_dates)
            params = {"dates": fmt_dates}
            logger.info("Retrieving signals information for dates {0}.".format(fmt_dates))
        else:
            params = {"dates": None}
            logger.info("Retrieving signals information for all dates")
        res = requests.get(url, params=params, headers=headers, **self._request_params)
        if res.ok:
            logger.info("Signals retrieval successful.")
            return res.json()
        else:
            error_msg = self._try_extract_error_code(res)
            raise BoostedAPIException("Failed to retrieve signals: {0}.".format(error_msg))

    def getEquityAccuracy(
        self,
        model_id: str,
        portfolio_id: str,
        tickers: List[str],
        start_date: Optional[datetime.datetime] = None,
        end_date: Optional[datetime.datetime] = None,
    ) -> Dict[str, Dict[str, Any]]:
        validate_start_and_end_dates(start_date, end_date)
        data = {}
        if start_date is not None:
            data["startDate"] = start_date
        if end_date is not None:
            data["endDate"] = end_date

        tickers_stream = ",".join(tickers)
        data["tickers"] = tickers_stream
        data["timestamp"] = time.strftime("%H:%M:%S")
        data["shouldRecalc"] = True
        url = self.base_uri + f"/api/analysis/equity-accuracy/{model_id}/{portfolio_id}"
        headers = {"Authorization": "ApiKey " + self.api_key, "Content-Type": "application/json"}

        logger.info(
            f"Retrieving equity accuracy data for date range {start_date} to {end_date} "
            f"for tickers: {tickers}."
        )

        # Now create dataframes from the JSON output.
        metrics = [
            "hit_rate_mean",
            "hit_rate_median",
            "excess_return_mean",
            "excess_return_median",
            "return",
            "excess_return",
        ]

        # send the request, retry if failed
        MAX_RETRIES = 10  # max of number of retries until timeout
        SLEEP_TIME = 3  # waiting time between requests

        num_retries = 0
        success = False
        while not success and num_retries < MAX_RETRIES:
            res = requests.post(url, data=json.dumps(data), headers=headers, **self._request_params)
            if res.ok:
                logger.info("Equity Accuracy Data retrieval successful.")
                info = res.json()
                success = True
            else:
                data["shouldRecalc"] = False
                num_retries += 1
                time.sleep(SLEEP_TIME)

        if not success:
            raise BoostedAPIException("Failed to retrieve equity accuracy: Request timeout.")

        for ticker, accuracy_data in info.items():
            for metric in metrics:
                metric_matrix = accuracy_data[metric]
                if not isinstance(metric_matrix, str):
                    # Set the index to the quintile label, and remove it from the data
                    index = []
                    for row in metric_matrix[1:]:
                        index.append(row.pop(0))

                    # columns are "1D", "5D", etc.
                    df = pd.DataFrame(metric_matrix[1:], columns=metric_matrix[0][1:], index=index)
                    accuracy_data[metric] = df
        return info

    def getHistoricalTradeDates(self, portfolio_id, start_date=None, end_date=None):
        end_date = self.__to_date_obj(end_date or datetime.date.today())
        start_date = self.__iso_format(start_date or (end_date - timedelta(days=365)))
        end_date = self.__iso_format(end_date)

        url = self.base_uri + "/api/portfolios/{0}/tradingDates".format(portfolio_id)
        headers = {"Authorization": "ApiKey " + self.api_key}
        params = {"startDate": start_date, "endDate": end_date}

        logger.info(
            "Retrieving historical trade dates data for date range {0} to {1}.".format(
                start_date, end_date
            )
        )
        res = requests.get(url, params=params, headers=headers, **self._request_params)
        if res.ok:
            logger.info("Trading dates retrieval successful.")
            return res.json()["dates"]
        else:
            error_msg = self._try_extract_error_code(res)
            raise BoostedAPIException("Failed to retrieve trading dates: {0}.".format(error_msg))

    def getRankingsForAllDates(self, portfolio_id, dates=None):
        url = self.base_uri + "/api/portfolios/{0}/rankingsByDate".format(portfolio_id)
        headers = {"Authorization": "ApiKey " + self.api_key}
        params = {}
        if dates is not None:
            fmt_dates = []
            for d in dates:
                fmt_dates.append(self.__iso_format(d))
            fmt_dates = ",".join(fmt_dates)
            params = {"dates": fmt_dates}
            logger.info("Retrieving rankings information for date {0}.".format(fmt_dates))
        else:
            params = {"dates": None}
            logger.info("Retrieving rankings information for all dates")
        res = requests.get(url, params=params, headers=headers, **self._request_params)
        if res.ok:
            logger.info("Rankings retrieval successful.")
            return res.json()
        else:
            error_msg = self._try_extract_error_code(res)
            raise BoostedAPIException("Failed to retrieve rankings: {0}.".format(error_msg))

    def getRankingsForDate(self, portfolio_id, date, rollback_to_last_available_date):
        date = self.__iso_format(date)
        endpoint = "latest-rankings" if rollback_to_last_available_date else "rankings"
        url = self.base_uri + "/api/{0}/{1}/{2}".format(endpoint, portfolio_id, date)
        headers = {"Authorization": "ApiKey " + self.api_key}
        logger.info("Retrieving rankings information for date {0}.".format(date))
        res = requests.get(url, headers=headers, **self._request_params)
        if res.ok:
            logger.info("Rankings retrieval successful.")
            return res.json()
        else:
            error_msg = self._try_extract_error_code(res)
            raise BoostedAPIException("Failed to retrieve rankings: {0}.".format(error_msg))

    def sendModelRecalc(self, model_id):
        url = self.base_uri + "/api/models/{0}/recalc".format(model_id)
        logger.info("Sending model recalc request for model {0}".format(model_id))
        headers = {"Authorization": "ApiKey " + self.api_key}
        res = requests.put(url, headers=headers, **self._request_params)
        if not res.ok:
            error_msg = self._try_extract_error_code(res)
            logger.error(error_msg)
            raise BoostedAPIException(
                "Failed to send model recalc request - "
                + "the model in UI may be out of date: {0}.".format(error_msg)
            )

    def add_uploaded_model_data(self, url, csv_data, request_data, timeout=600):
        logger.info("Starting upload.")
        headers = {"Authorization": "ApiKey " + self.api_key}
        files_req = {}
        target = ("data.csv", None, "text/csv")
        warnings = []
        if isinstance(csv_data, pd.core.frame.DataFrame):
            buf = io.StringIO()
            csv_data.to_csv(buf, header=False)
            if not isinstance(csv_data.index, pd.core.indexes.datetimes.DatetimeIndex):
                raise BoostedAPIException("DataFrame must have DatetimeIndex as index type.")
            target = ("uploaded_data.csv", buf.getvalue(), "text/csv")
            files_req["dataFile"] = target
            res = requests.post(
                url,
                files=files_req,
                data=request_data,
                headers=headers,
                timeout=timeout,
                **self._request_params,
            )
        elif isinstance(csv_data, str):
            target = ("uploaded_data.csv", csv_data, "text/csv")
            files_req["dataFile"] = target
            res = requests.post(
                url,
                files=files_req,
                data=request_data,
                headers=headers,
                timeout=timeout,
                **self._request_params,
            )
        else:
            raise BoostedAPIException("Expected CSV as str or Pandas DataFrame.")
        if res.ok:
            logger.info("Signals upload completed.")
            result = res.json()["result"]
            if "warningMessages" in result:
                warnings = result["warningMessages"]
        else:
            logger.error("Signals upload failed: {0}, {1}".format(res.text, res.reason))
            raise BoostedAPIException("Upload failed.")

        return res, warnings

    def createSignalsModel(self, csv_data, model_name, timeout=600):
        warnings = []
        url = self.base_uri + "/api/models/upload/signals/create"
        request_data = {"modelName": model_name, "uploadName": model_name}
        res, warnings = self.add_uploaded_model_data(url, csv_data, request_data, timeout)
        result = res.json()["result"]
        model_id = result["modelId"]
        self.sendModelRecalc(model_id)
        return model_id, warnings

    def addToUploadedModel(self, model_id, csv_data, timeout=600):
        warnings = []
        url = self.base_uri + "/api/models/{0}/upload/add-data".format(model_id)
        request_data = {}
        _, warnings = self.add_uploaded_model_data(url, csv_data, request_data, timeout)
        self.sendModelRecalc(model_id)
        return warnings

    def addSignalsToUploadedModel(self, model_id, csv_data, timeout=600):
        warnings = self.addToUploadedModel(model_id, csv_data)
        return warnings

    def getSignalsFromUploadedModel(self, model_id, date=None):
        date = self.__iso_format(date)
        url = self.base_uri + "/api/models/{0}/upload/signals".format(model_id)
        headers = {"Authorization": "ApiKey " + self.api_key}
        params = {"date": date}
        logger.info("Retrieving uploaded signals information")
        res = requests.get(url, params=params, headers=headers, **self._request_params)
        if res.ok:
            result = pd.DataFrame.from_dict(res.json()["result"])
            # ensure column order
            result = result[["date", "isin", "country", "currency", "weight"]]
            result["date"] = pd.to_datetime(result["date"], format="%Y-%m-%d")
            result = result.set_index("date")
            logger.info("Signals retrieval successful.")
            return result
        else:
            error_msg = self._try_extract_error_code(res)
            raise BoostedAPIException("Failed to retrieve signals: {0}.".format(error_msg))

    def getPortfolioSettings(self, portfolio_id, timeout=600):
        url = self.base_uri + "/api/portfolio-settings/{0}".format(portfolio_id)
        headers = {"Authorization": "ApiKey " + self.api_key}
        res = requests.get(url, headers=headers, **self._request_params)
        if res.ok:
            return PortfolioSettings(res.json())
        else:
            error_msg = self._try_extract_error_code(res)
            logger.error(error_msg)
            raise BoostedAPIException(
                "Failed to retrieve portfolio settings: {0}.".format(error_msg)
            )

    def createPortfolioWithPortfolioSettings(
        self, model_id, portfolio_name, portfolio_description, portfolio_settings, timeout=600
    ):
        url = self.base_uri + "/api/models/{0}/constraints/add".format(model_id)
        headers = {"Authorization": "ApiKey " + self.api_key, "Content-Type": "application/json"}
        setting_string = json.dumps(portfolio_settings.settings)
        logger.info("Creating new portfolio with specified setting: {}".format(setting_string))
        params = {
            "name": portfolio_name,
            "description": portfolio_description,
            "constraints": setting_string,
            "validate": "true",
        }
        res = requests.put(url, json=params, headers=headers, **self._request_params)
        response = res.json()
        if res.ok:
            return response
        else:
            error_msg = self._try_extract_error_code(res)
            logger.error(error_msg)
            raise BoostedAPIException(
                "Failed to create portfolio with the specified settings: {0}.".format(error_msg)
            )

    def getGbiIdFromIdentCountryCurrencyDate(
        self, ident_country_currency_dates: List[DateIdentCountryCurrency], timeout: int = 600
    ) -> List[GbiIdSecurity]:
        url = self.base_uri + "/api/custom-stock-data/map-identifiers-simple"
        headers = {"Authorization": "ApiKey " + self.api_key, "Content-Type": "application/json"}
        identifiers = [
            {
                "row": idx,
                "date": identifier.date,
                "isin": identifier.identifier if identifier.id_type == ColumnSubRole.ISIN else None,
                "symbol": identifier.identifier
                if identifier.id_type == ColumnSubRole.SYMBOL
                else None,
                "countryPreference": identifier.country,
                "currencyPreference": identifier.currency,
            }
            for idx, identifier in enumerate(ident_country_currency_dates)
        ]
        params = json.dumps({"identifiers": identifiers})
        logger.info(
            "Retrieving GBI-ID mapping for {} identifier tuples...".format(
                len(ident_country_currency_dates)
            )
        )
        res = requests.post(url, data=params, headers=headers, **self._request_params)

        if res.ok:
            result = res.json()
            warnings = result["warnings"]
            if warnings:
                for warning in warnings:
                    logger.warn(f"Mapping warning: {warning}")
            gbiSecurities = []
            for idx, ident in enumerate(result["mappedIdentifiers"]):
                if ident is None:
                    security = None
                else:
                    security = GbiIdSecurity(
                        ident["gbiId"],
                        ident_country_currency_dates[idx],
                        ident["symbol"],
                        ident["companyName"],
                    )
                gbiSecurities.append(security)

            return gbiSecurities
        else:
            error_msg = self._try_extract_error_code(res)
            raise BoostedAPIException(
                "Failed to retrieve identifier mappings: {0}.".format(error_msg)
            )

    # exists for backwards compatibility purposes.
    def getGbiIdFromIsinCountryCurrencyDate(self, isin_country_currency_dates, timeout=600):
        return self.getGbiIdFromIdentCountryCurrencyDate(
            ident_country_currency_dates=isin_country_currency_dates, timeout=timeout
        )

    def getTradeExplain(self, portfolio_id, date=None):
        url = self.base_uri + f"/api/explain/{portfolio_id}"
        explain_date = self.__iso_format(date)
        if explain_date:
            url = self.base_uri + f"/api/explain/{portfolio_id}/{explain_date}"
        headers = {"Authorization": "ApiKey " + self.api_key}
        res = requests.get(url, headers=headers, **self._request_params)
        if res.ok:
            buf = io.StringIO(res.text)
            df = pd.read_csv(buf, index_col=0, parse_dates=True)
            return df
        else:
            error_msg = self._try_extract_error_code(res)
            logger.error(error_msg)
            raise BoostedAPIException("Failed to get trade explain: {0}.".format(error_msg))

    # model_id: str
    # returns: Dict[str, str] representing the translation from the rankings ID (feature refs)
    # to human readable names
    def __get_rankings_ref_translation(self, model_id: str) -> Dict[str, str]:
        headers = {"Authorization": "ApiKey " + self.api_key, "Content-Type": "application/json"}
        feature_name_url = f"/api/models/{model_id}/advanced-explain/translate-feature-ref/"
        feature_name_res = requests.post(
            self.base_uri + feature_name_url,
            data=json.dumps({}),
            headers=headers,
            **self._request_params,
        )

        if feature_name_res.ok:
            feature_name_dict = feature_name_res.json()
            return {
                id: "-".join(
                    [names["variable_name"], names["transform_name"], names["normalization_name"]]
                )
                for id, names in feature_name_dict.items()
            }
        else:
            raise Exception(
                """Failed to get feature names for model,
                    this model doesn't fully support rankings 2.0"""
            )

    def getDatasetDates(self, dataset_id):
        url = self.base_uri + f"/api/datasets/{dataset_id}"
        headers = {"Authorization": "ApiKey " + self.api_key}
        res = requests.get(url, headers=headers, **self._request_params)
        if res.ok:
            dataset = res.json()
            valid_to_array = dataset.get("validTo")
            valid_to_date = None
            valid_from_array = dataset.get("validFrom")
            valid_from_date = None
            if valid_to_array:
                valid_to_date = datetime.date(
                    valid_to_array[0], valid_to_array[1], valid_to_array[2]
                )
            if valid_from_array:
                valid_from_date = datetime.date(
                    valid_from_array[0], valid_from_array[1], valid_from_array[2]
                )
            return {"validTo": valid_to_date, "validFrom": valid_from_date}
        else:
            error_msg = self._try_extract_error_code(res)
            logger.error(error_msg)
            raise BoostedAPIException("Failed to query dataset: {0}.".format(error_msg))

    def getRankingAnalysis(self, model_id, date):
        url = (
            self.base_uri
            + f"/api/explain-trades/analysis/{model_id}/{self.__iso_format(date)}/json"
        )
        headers = {"Authorization": "ApiKey " + self.api_key}
        analysis_res = requests.get(url, headers=headers, **self._request_params)
        if analysis_res.ok:
            ranking_dict = analysis_res.json()
            feature_name_dict = self.__get_rankings_ref_translation(model_id)
            columns = [feature_name_dict[col] for col in ranking_dict["columns"]]

            df = protoCubeJsonDataToDataFrame(
                ranking_dict["data"],
                "Data Buckets",
                ranking_dict["rows"],
                "Feature Names",
                columns,
                ranking_dict["fields"],
            )
            return df
        else:
            error_msg = self._try_extract_error_code(analysis_res)
            logger.error(error_msg)
            raise BoostedAPIException("Failed to get ranking analysis: {0}.".format(error_msg))

    def getRankingExplain(self, model_id, date):
        url = (
            self.base_uri + f"/api/explain-trades/explain/{model_id}/{self.__iso_format(date)}/json"
        )
        headers = {"Authorization": "ApiKey " + self.api_key, "Content-Type": "application/json"}
        explain_res = requests.get(url, headers=headers, **self._request_params)
        if explain_res.ok:
            ranking_dict = explain_res.json()
            rows = ranking_dict["rows"]
            stock_summary_url = f"/api/stock-summaries/{model_id}"
            stock_summary_body = {"gbiIds": ranking_dict["rows"]}
            summary_res = requests.post(
                self.base_uri + stock_summary_url,
                data=json.dumps(stock_summary_body),
                headers=headers,
                **self._request_params,
            )
            if summary_res.ok:
                stock_summary = summary_res.json()
                rows = [stock_summary[row]["isin"] for row in ranking_dict["rows"]]
            else:
                error_msg = self._try_extract_error_code(summary_res)
                logger.error(error_msg)
                raise BoostedAPIException(
                    "Failed to get isin information ranking explain: {0}.".format(error_msg)
                )

            feature_name_dict = self.__get_rankings_ref_translation(model_id)
            columns = [feature_name_dict[col] for col in ranking_dict["columns"]]

            df = protoCubeJsonDataToDataFrame(
                ranking_dict["data"],
                "ISINs",
                rows,
                "Feature Names",
                columns,
                ranking_dict["fields"],
            )
            return df
        else:
            error_msg = self._try_extract_error_code(explain_res)
            logger.error(error_msg)
            raise BoostedAPIException("Failed to get ranking explain: {0}.".format(error_msg))

    def getDenseSignals(self, model_id, portfolio_id, file_name=None, location="./"):
        url = self.base_uri + f"/api/models/{model_id}/{portfolio_id}/dense-signals"
        headers = {"Authorization": "ApiKey " + self.api_key}
        res = requests.get(url, headers=headers, **self._request_params)
        if file_name is None:
            file_name = f"{model_id}-{portfolio_id}_dense_signals.csv"
        download_location = os.path.join(location, file_name)
        if res.ok:
            with open(download_location, "wb") as file:
                file.write(res.content)
            print("Download Complete")
        elif res.status_code == 404:
            raise BoostedAPIException(
                f"""Dense Singals file does not exist for model:
                 {model_id} - portfolio: {portfolio_id}"""
            )
        else:
            error_msg = self._try_extract_error_code(res)
            logger.error(error_msg)
            raise BoostedAPIException(
                f"""Failed to download dense singals file for model:
                 {model_id} - portfolio: {portfolio_id}"""
            )

    def getRanking2DateAnalysisFile(
        self, model_id, portfolio_id, date, file_name=None, location="./"
    ):
        formatted_date = self.__iso_format(date)
        s3_file_name = f"{formatted_date}_analysis.xlsx"
        download_url = (
            self.base_uri + f"/api/models/{model_id}/{portfolio_id}/ranking-file/{s3_file_name}"
        )
        generate_url = (
            self.base_uri
            + f"/api/explain-trades/{model_id}/{portfolio_id}/generate/date-data/{formatted_date}"
        )
        headers = {"Authorization": "ApiKey " + self.api_key}
        if file_name is None:
            file_name = f"{model_id}-{portfolio_id}_statistical_analysis_{formatted_date}.xlsx"
        download_location = os.path.join(location, file_name)

        res = requests.get(download_url, headers=headers, **self._request_params)
        if res.ok:
            with open(download_location, "wb") as file:
                file.write(res.content)
            print("Download Complete")
        elif res.status_code == 404:
            generate_res = requests.get(generate_url, headers=headers, **self._request_params)
            if generate_res.ok:
                download_res = requests.get(download_url, headers=headers, **self._request_params)
                while download_res.status_code == 404 or (
                    download_res.ok and len(download_res.content) == 0
                ):
                    print("waiting for file to be generated")
                    time.sleep(5)
                    download_res = requests.get(
                        download_url, headers=headers, **self._request_params
                    )
                if download_res.ok:
                    with open(download_location, "wb") as file:
                        file.write(download_res.content)
                    print("Download Complete")
            else:
                error_msg = self._try_extract_error_code(res)
                logger.error(error_msg)
                raise BoostedAPIException(
                    f"""Failed to generate ranking analysis file for model:
                    {model_id} - portfolio: {portfolio_id} on date: {formatted_date}"""
                )
        else:
            error_msg = self._try_extract_error_code(res)
            logger.error(error_msg)
            raise BoostedAPIException(
                f"""Failed to download ranking analysis file for model:
                 {model_id} - portfolio: {portfolio_id} on date: {formatted_date}"""
            )

    def getRanking2DateExplainFile(
        self, model_id, portfolio_id, date, file_name=None, location="./", overwrite: bool = False
    ):
        formatted_date = self.__iso_format(date)
        s3_file_name = f"{formatted_date}_explaindata.xlsx"
        download_url = (
            self.base_uri + f"/api/models/{model_id}/{portfolio_id}/ranking-file/{s3_file_name}"
        )
        generate_url = (
            self.base_uri
            + f"/api/explain-trades/{model_id}/{portfolio_id}/generate/date-data/{formatted_date}"
        )
        headers = {"Authorization": "ApiKey " + self.api_key}
        if file_name is None:
            file_name = f"{model_id}-{portfolio_id}_explain_data_{formatted_date}.xlsx"
        download_location = os.path.join(location, file_name)

        if not overwrite:
            res = requests.get(download_url, headers=headers, **self._request_params)
        if not overwrite and res.ok:
            with open(download_location, "wb") as file:
                file.write(res.content)
            print("Download Complete")
        elif overwrite or res.status_code == 404:
            generate_res = requests.get(generate_url, headers=headers, **self._request_params)
            if generate_res.ok:
                download_res = requests.get(download_url, headers=headers, **self._request_params)
                while download_res.status_code == 404 or (
                    download_res.ok and len(download_res.content) == 0
                ):
                    print("waiting for file to be generated")
                    time.sleep(5)
                    download_res = requests.get(
                        download_url, headers=headers, **self._request_params
                    )
                if download_res.ok:
                    with open(download_location, "wb") as file:
                        file.write(download_res.content)
                    print("Download Complete")
            else:
                error_msg = self._try_extract_error_code(res)
                logger.error(error_msg)
                raise BoostedAPIException(
                    f"""Failed to generate ranking explain file for model:
                    {model_id} - portfolio: {portfolio_id} on date: {formatted_date}"""
                )
        else:
            error_msg = self._try_extract_error_code(res)
            logger.error(error_msg)
            raise BoostedAPIException(
                f"""Failed to download ranking explain file for model:
                 {model_id} - portfolio: {portfolio_id} on date: {formatted_date}"""
            )

    def getTearSheet(self, model_id, portfolio_id, start_date=None, end_date=None, block=False):
        if start_date is None or end_date is None:
            if start_date is not None or end_date is not None:
                raise ValueError("start_date and end_date must both be None or both be defined")
            return self._getCurrentTearSheet(model_id, portfolio_id)

        start_date_obj = self.__to_date_obj(start_date)
        end_date_obj = self.__to_date_obj(end_date)
        if start_date_obj >= end_date_obj:
            raise ValueError("end_date must be later than the start_date")

        # get for the given date
        url = self.base_uri + f"/api/analysis/keyfacts/{model_id}/{portfolio_id}"
        data = {
            "startDate": self.__iso_format(start_date),
            "endDate": self.__iso_format(end_date),
            "shouldRecalc": True,
        }
        headers = {"Authorization": "ApiKey " + self.api_key, "Content-Type": "application/json"}
        res = requests.post(url, data=json.dumps(data), headers=headers, **self._request_params)
        if res.status_code == 404 and block:
            retries = 0
            data["shouldRecalc"] = False
            while retries < 10:
                time.sleep(10)
                retries += 1
                res = requests.post(
                    url, data=json.dumps(data), headers=headers, **self._request_params
                )
                if res.status_code != 404:
                    break
        if res.ok:
            return res.json()
        else:
            error_msg = self._try_extract_error_code(res)
            logger.error(error_msg)
            raise BoostedAPIException(
                "Failed to get tear sheet data: {0} {1}.".format(error_msg, str(res.status_code))
            )

    def _getCurrentTearSheet(self, model_id, portfolio_id):
        url = self.base_uri + f"/api/model-summaries/{model_id}/{portfolio_id}"
        headers = {"Authorization": "ApiKey " + self.api_key, "Content-Type": "application/json"}
        res = requests.get(url, headers=headers, **self._request_params)
        if res.ok:
            json = res.json()
            return json.get("tearSheet", {})
        else:
            error_msg = self._try_extract_error_code(res)
            logger.error(error_msg)
            raise BoostedAPIException("Failed to get tear sheet data: {0}.".format(error_msg))

    def getPortfolioStatus(self, model_id, portfolio_id, job_date):
        url = (
            self.base_uri
            + f"/api/analysis/portfolioStatus/{model_id}/{portfolio_id}?jobDate={job_date}"
        )
        headers = {"Authorization": "ApiKey " + self.api_key, "Content-Type": "application/json"}
        res = requests.get(url, headers=headers, **self._request_params)
        if res.ok:
            result = res.json()
            return {
                "is_complete": result["status"],
                "last_update": None if result["lastUpdate"] is None else result["lastUpdate"][:10],
                "next_update": None if result["nextUpdate"] is None else result["nextUpdate"][:10],
            }
        else:
            error_msg = self._try_extract_error_code(res)
            logger.error(error_msg)
            raise BoostedAPIException("Failed to get portfolio status: {0}".format(error_msg))

    def getBlacklist(self, blacklist_id):
        url = self.base_uri + f"/api/blacklist/{blacklist_id}"
        headers = {"Authorization": "ApiKey " + self.api_key}
        res = requests.get(url, headers=headers, **self._request_params)
        if res.ok:
            result = res.json()
            return result
        error_msg = self._try_extract_error_code(res)
        logger.error(error_msg)
        raise BoostedAPIException(f"Failed to get blacklist with id {blacklist_id}: {error_msg}")

    def getBlacklists(self, model_id=None, company_id=None, last_N=None):
        params = {}
        if last_N:
            params["lastN"] = last_N
        if model_id:
            params["modelId"] = model_id
        if company_id:
            params["companyId"] = company_id
        url = self.base_uri + f"/api/blacklist"
        headers = {"Authorization": "ApiKey " + self.api_key}
        res = requests.get(url, headers=headers, params=params, **self._request_params)
        if res.ok:
            result = res.json()
            return result
        error_msg = self._try_extract_error_code(res)
        logger.error(error_msg)
        raise BoostedAPIException(
            f"""Failed to get blacklists with \
            model_id {model_id} company_id {company_id} last_N {last_N}: {error_msg}"""
        )

    def createBlacklist(
        self,
        isin,
        long_short=2,
        start_date=datetime.date.today(),
        end_date="4000-01-01",
        model_id=None,
    ):
        url = self.base_uri + f"/api/blacklist"
        data = {
            "modelId": model_id,
            "isin": isin,
            "longShort": long_short,
            "startDate": self.__iso_format(start_date),
            "endDate": self.__iso_format(end_date),
        }
        headers = {"Authorization": "ApiKey " + self.api_key, "Content-Type": "application/json"}
        res = requests.post(url, data=json.dumps(data), headers=headers, **self._request_params)
        if res.ok:
            return res.json()
        else:
            error_msg = self._try_extract_error_code(res)
            logger.error(error_msg)
            raise BoostedAPIException(
                f"""Failed to create the blacklist with \
                  isin {isin} long_short {long_short} start_date {start_date} end_date {end_date} \
                  model_id {model_id}: {error_msg}."""
            )

    def createBlacklistsFromCSV(self, csv_name):
        url = self.base_uri + f"/api/blacklists"
        data = []
        with open(csv_name, mode="r") as f:
            csv_reader = csv.DictReader(f)
            for row in csv_reader:
                blacklist = {"modelId": row["ModelID"], "isin": row["ISIN"]}
                if row["LongShort"] == "":
                    blacklist["longShort"] = 2
                else:
                    blacklist["longShort"] = row["LongShort"]

                if row["StartDate"] == "":
                    blacklist["startDate"] = self.__iso_format(datetime.date.today())
                else:
                    blacklist["startDate"] = self.__iso_format(row["StartDate"])

                if row["EndDate"] == "":
                    blacklist["endDate"] = self.__iso_format("4000-01-01")
                else:
                    blacklist["endDate"] = self.__iso_format(row["EndDate"])
                data.append(blacklist)
        print(f"Processed {len(data)} blacklists.")
        headers = {"Authorization": "ApiKey " + self.api_key, "Content-Type": "application/json"}
        res = requests.post(url, data=json.dumps(data), headers=headers, **self._request_params)
        if res.ok:
            return res.json()
        else:
            error_msg = self._try_extract_error_code(res)
            logger.error(error_msg)
            raise BoostedAPIException("failed to create blacklists")

    def updateBlacklist(self, blacklist_id, long_short=None, start_date=None, end_date=None):
        params = {}
        if long_short:
            params["longShort"] = long_short
        if start_date:
            params["startDate"] = start_date
        if end_date:
            params["endDate"] = end_date
        url = self.base_uri + f"/api/blacklist/{blacklist_id}"
        headers = {"Authorization": "ApiKey " + self.api_key, "Content-Type": "application/json"}
        res = requests.patch(url, json=params, headers=headers, **self._request_params)
        if res.ok:
            return res.json()
        else:
            error_msg = self._try_extract_error_code(res)
            logger.error(error_msg)
            raise BoostedAPIException(
                f"Failed to update blacklist with id {blacklist_id}: {error_msg}"
            )

    def deleteBlacklist(self, blacklist_id):
        url = self.base_uri + f"/api/blacklist/{blacklist_id}"
        headers = {"Authorization": "ApiKey " + self.api_key}
        res = requests.delete(url, headers=headers, **self._request_params)
        if res.ok:
            result = res.json()
            return result
        else:
            error_msg = self._try_extract_error_code(res)
            logger.error(error_msg)
            raise BoostedAPIException(
                f"Failed to delete blacklist with id {blacklist_id}: {error_msg}"
            )

    def getFeatureImportance(self, model_id, date, N=None):
        url = self.base_uri + f"/api/analysis/explainability/{model_id}"
        headers = {"Authorization": "ApiKey " + self.api_key}
        logger.info("Retrieving rankings information for date {0}.".format(date))
        res = requests.get(url, headers=headers, **self._request_params)
        if not res.ok:
            error_msg = self._try_extract_error_code(res)
            logger.error(error_msg)
            raise BoostedAPIException(
                f"Failed to fetch feature importance for model/portfolio {model_id}: {error_msg}"
            )

        json_data = res.json()
        if "all" not in json_data.keys() or not json_data["all"]:
            raise BoostedAPIException(f"Unexpected formatting of feature importance response")

        feature_data = json_data["all"]
        # find the right period (assuming returned json has dates in descending order)
        date_obj = self.__to_date_obj(date)
        start_date_for_return_data = self.__to_date_obj(feature_data[0]["date"])
        features_for_requested_period = None

        if date_obj > start_date_for_return_data:
            features_for_requested_period = feature_data[0]["variable"]
        else:
            i = 0
            while i < len(feature_data) - 1:
                current_date = self.__to_date_obj(feature_data[i]["date"])
                next_date = self.__to_date_obj(feature_data[i + 1]["date"])
                if next_date <= date_obj <= current_date:
                    features_for_requested_period = feature_data[i + 1]["variable"]
                    start_date_for_return_data = next_date
                    break
                i += 1

        if features_for_requested_period is None:
            raise BoostedAPIException(f"No feature data was found for requested date: {date_obj}")

        features_for_requested_period.sort(key=lambda x: x["value"], reverse=True)

        if type(N) is int and N > 0:
            df = pd.DataFrame.from_dict(features_for_requested_period[0:N])
        else:
            df = pd.DataFrame.from_dict(features_for_requested_period)
        result = df[["feature", "value"]]

        return result.rename(columns={"feature": f"feature ({start_date_for_return_data})"})

    def getAllModelNames(self) -> Dict[str, str]:
        url = f"{self.base_uri}/api/graphql"
        headers = {"Authorization": "ApiKey " + self.api_key, "Content-Type": "application/json"}
        req_json = {"query": "query listOfModels {\n models { id name }}", "variables": {}}
        res = requests.post(url, json=req_json, headers=headers, **self._request_params)
        if not res.ok:
            error_msg = self._try_extract_error_code(res)
            logger.error(error_msg)
            raise BoostedAPIException(f"Failed to get user models: {error_msg}")
        data = res.json()
        if data["data"]["models"] is None:
            return {}
        return {rec["id"]: rec["name"] for rec in data["data"]["models"]}

    def getAllModelDetails(self) -> Dict[str, Dict[str, Any]]:
        url = f"{self.base_uri}/api/graphql"
        headers = {"Authorization": "ApiKey " + self.api_key, "Content-Type": "application/json"}
        req_json = {
            "query": "query listOfModels {\n models { id name lastUpdated portfolios { id name }}}",
            "variables": {},
        }
        res = requests.post(url, json=req_json, headers=headers, **self._request_params)
        if not res.ok:
            error_msg = self._try_extract_error_code(res)
            logger.error(error_msg)
            raise BoostedAPIException(f"Failed to get user models: {error_msg}")
        data = res.json()
        if data["data"]["models"] is None:
            return {}

        output_data = {}
        for rec in data["data"]["models"]:
            model_id = rec["id"]
            output_data[model_id] = {
                "name": rec["name"],
                "last_updated": parser.parse(rec["lastUpdated"]),
                "portfolios": rec["portfolios"],
            }

        return output_data

    def get_hedge_experiments(self):
        url = self.base_uri + "/api/graphql"
        qry = """
            query getHedgeExperiments {
                hedgeExperiments {
                    hedgeExperimentId
                    experimentName
                    userId
                    config
                    description
                    experimentType
                    lastCalculated
                    lastModified
                    status
                    portfolioCalcStatus
                    targetSecurities {
                        gbiId
                        security {
                            gbiId
                            symbol
                            name
                        }
                        weight
                    }
                    targetPortfolios {
                        portfolioId
                    }
                    baselineModel {
                        id
                        name

                    }
                    baselineScenario {
                        hedgeExperimentScenarioId
                        scenarioName
                        description
                        portfolioSettingsJson
                        hedgeExperimentPortfolios {
                            portfolio {
                                id
                                name
                                modelId
                                performanceGridHeader
                                performanceGrid
                                status
                                tearSheet {
                                    groupName
                                    members {
                                        name
                                        value
                                    }
                                }
                            }
                        }
                        status
                    }
                    baselineStockUniverseId
                }
            }
        """

        headers = {"Authorization": "ApiKey " + self.api_key}
        resp = requests.post(url, json={"query": qry}, headers=headers, params=self._request_params)

        json_resp = resp.json()
        # graphql endpoints typically return 200 or 400 status codes, so we must
        # check if we have any errors, even with a 200
        if (resp.ok and "errors" in json_resp) or not resp.ok:
            error_msg = self._try_extract_error_code(resp)
            logger.error(error_msg)
            raise BoostedAPIException(
                (f"Failed to get hedge experiments: {resp.status_code=}; {error_msg=}")
            )

        json_experiments = resp.json()["data"]["hedgeExperiments"]
        experiments = [HedgeExperiment.from_json_dict(exp_json) for exp_json in json_experiments]
        return experiments

    def get_hedge_experiment_details(self, experiment_id: str):
        url = self.base_uri + "/api/graphql"
        qry = """
            query getHedgeExperimentDetails($hedgeExperimentId: ID!) {
                hedgeExperiment(hedgeExperimentId: $hedgeExperimentId) {
                ...HedgeExperimentDetailsSummaryListFragment
                }
            }

            fragment HedgeExperimentDetailsSummaryListFragment on HedgeExperiment {
                hedgeExperimentId
                experimentName
                userId
                config
                description
                experimentType
                lastCalculated
                lastModified
                status
                portfolioCalcStatus
                targetSecurities {
                    gbiId
                    security {
                        gbiId
                        symbol
                        name
                    }
                    weight
                }
                selectedModels {
                    id
                    name
                    stockUniverse {
                        name
                    }
                }
                hedgeExperimentScenarios {
                    ...experimentScenarioFragment
                }
                selectedDummyHedgeExperimentModels {
                    id
                    name
                    stockUniverse {
                        name
                    }
                }
                targetPortfolios {
                    portfolioId
                }
                baselineModel {
                    id
                    name

                }
                baselineScenario {
                    hedgeExperimentScenarioId
                    scenarioName
                    description
                    portfolioSettingsJson
                    hedgeExperimentPortfolios {
                        portfolio {
                            id
                            name
                            modelId
                            performanceGridHeader
                            performanceGrid
                            status
                            tearSheet {
                                groupName
                                members {
                                    name
                                    value
                                }
                            }
                        }
                    }
                    status
                }
                baselineStockUniverseId
            }

            fragment experimentScenarioFragment on HedgeExperimentScenario {
                hedgeExperimentScenarioId
                scenarioName
                status
                description
                portfolioSettingsJson
                hedgeExperimentPortfolios {
                    portfolio {
                        id
                        name
                        modelId
                        performanceGridHeader
                        performanceGrid
                        status
                        tearSheet {
                            groupName
                            members {
                                name
                                value
                            }
                        }
                    }
                }
            }
        """
        headers = {"Authorization": "ApiKey " + self.api_key}
        resp = requests.post(
            url,
            json={"query": qry, "variables": {"hedgeExperimentId": experiment_id}},
            headers=headers,
            params=self._request_params,
        )

        json_resp = resp.json()
        # graphql endpoints typically return 200 or 400 status codes, so we must
        # check if we have any errors, even with a 200
        if (resp.ok and "errors" in json_resp) or not resp.ok:
            error_msg = self._try_extract_error_code(resp)
            logger.error(error_msg)
            raise BoostedAPIException(
                (
                    f"Failed to get hedge experiment results for {experiment_id=}: "
                    f"{resp.status_code=}; {error_msg=}"
                )
            )

        json_exp_results = json_resp["data"]["hedgeExperiment"]
        if json_exp_results is None:
            return None  # issued a request with a non-existent experiment_id
        exp_results = HedgeExperimentDetails.from_json_dict(json_exp_results)
        return exp_results

    def get_portfolio_performance(self, portfolio_id: str) -> pd.DataFrame:
        url = f"{self.base_uri}/api/graphql"
        qry = """
            query getPortfolioPerformance($portfolioId: ID!) {
                portfolio(id: $portfolioId) {
                    id
                    modelId
                    name
                    status
                    performance {
                        benchmark
                        date
                        turnover
                        value
                    }
                }
            }
        """

        headers = {"Authorization": "ApiKey " + self.api_key}
        resp = requests.post(
            url,
            json={"query": qry, "variables": {"portfolioId": portfolio_id}},
            headers=headers,
            params=self._request_params,
        )

        json_resp = resp.json()
        # the webserver returns an error for non-ready portfolios, so we have to check
        # for this prior to the error check below
        pf = json_resp["data"].get("portfolio")
        if pf is not None and pf["status"] != "READY":
            return pd.DataFrame()

        # graphql endpoints typically return 200 or 400 status codes, so we must
        # check if we have any errors, even with a 200
        if (resp.ok and "errors" in json_resp) or not resp.ok:
            error_msg = self._try_extract_error_code(resp)
            logger.error(error_msg)
            raise BoostedAPIException(
                (
                    f"Failed to get portfolio performance for {portfolio_id=}: "
                    f"{resp.status_code=}; {error_msg=}"
                )
            )

        perf = json_resp["data"]["portfolio"]["performance"]
        df = pd.DataFrame(perf).set_index("date").rename(columns={"value": "portfolio"})
        df.index = pd.to_datetime(df.index)
        return df.astype(float)

    def _is_portfolio_still_running(self, error_msg: str) -> bool:
        # this is jank af. a proper fix of this is either at the webserver
        # returning a better response for a portfolio in draft HT2-226, OR
        # a bigger refactor of the API that moves to more OOP, which would allow us
        # to have this data all in one place
        return "Could not find a model with this ID" in error_msg

    def get_portfolio_factors(self, model_id: str, portfolio_id: str) -> pd.DataFrame:
        url = f"{self.base_uri}/api/analysis/factors/{model_id}/{portfolio_id}"
        headers = {"Authorization": "ApiKey " + self.api_key}
        resp = requests.get(url, headers=headers, params=self._request_params)

        json_resp = resp.json()
        if (resp.ok and "errors" in json_resp) or not resp.ok:
            error_msg = json_resp["errors"][0]
            if self._is_portfolio_still_running(error_msg):
                return pd.DataFrame()
            logger.error(error_msg)
            raise BoostedAPIException(
                (
                    f"Failed to get portfolio factors for {portfolio_id=}: "
                    f"{resp.status_code=}; {error_msg=}"
                )
            )

        df = pd.DataFrame(json_resp["data"], columns=json_resp["header_row"])

        def to_lower_snake_case(s):  # why are we linting lambdas? :(
            return "_".join(w.lower() for w in s.split(" "))

        df = df.rename(columns={old: to_lower_snake_case(old) for old in df.columns}).set_index(
            "date"
        )
        df.index = pd.to_datetime(df.index)
        return df

    def get_portfolio_volatility(self, model_id: str, portfolio_id: str) -> pd.DataFrame:
        url = f"{self.base_uri}/api/analysis/volatility_rolling/{model_id}/{portfolio_id}"
        headers = {"Authorization": "ApiKey " + self.api_key}
        resp = requests.get(url, headers=headers, params=self._request_params)

        json_resp = resp.json()
        if (resp.ok and "errors" in json_resp) or not resp.ok:
            error_msg = json_resp["errors"][0]
            if self._is_portfolio_still_running(error_msg):
                return pd.DataFrame()
            logger.error(error_msg)
            raise BoostedAPIException(
                (
                    f"Failed to get portfolio volatility for {portfolio_id=}: "
                    f"{resp.status_code=}; {error_msg=}"
                )
            )

        df = pd.DataFrame(json_resp["data"], columns=json_resp["headerRow"])
        df = df.rename(
            columns={old: old.lower().replace("avg", "avg_") for old in df.columns}
        ).set_index("date")
        df.index = pd.to_datetime(df.index)
        return df

    def get_portfolio_holdings(self, model_id: str, portfolio_id: str) -> pd.DataFrame:
        url = f"{self.base_uri}/api/models/{model_id}/{portfolio_id}/basket-data"
        headers = {"Authorization": "ApiKey " + self.api_key}
        resp = requests.get(url, headers=headers, params=self._request_params)

        # this is a classic abuse of try/except as control flow: we try to get json body
        # from the response so that we can error-check. if this fails, we assume we have
        # a legit text response (corresponding to the csv data we care about)
        try:
            json_resp = resp.json()
        except json.decoder.JSONDecodeError:
            df = pd.read_csv(io.StringIO(resp.text), header=[0])
        else:
            error_msg = json_resp["errors"][0]
            if self._is_portfolio_still_running(error_msg):
                return pd.DataFrame()
            else:
                logger.error(error_msg)
                raise BoostedAPIException(
                    (
                        f"Failed to get portfolio holdings for {portfolio_id=}: "
                        f"{resp.status_code=}; {error_msg=}"
                    )
                )

        df = df.rename(columns={old: old.lower() for old in df.columns}).set_index("date")
        df.index = pd.to_datetime(df.index)
        return df

    def getStockDataTableForDate(
        self, model_id: str, portfolio_id: str, date: datetime.date
    ) -> pd.DataFrame:
        headers = {"Authorization": "ApiKey " + self.api_key, "Content-Type": "application/json"}

        url_base = f"{self.base_uri}/api/analysis"
        url_params = f"{model_id}/{portfolio_id}"
        formatted_date = date.strftime("%Y-%m-%d")

        stock_prices_url = f"{url_base}/stock-prices/{url_params}/{formatted_date}"
        stock_factors_url = f"{url_base}/stock-factors/{url_params}/date/{formatted_date}"

        prices_params = {"useTicker": "true"}

        prices_resp = requests.get(
            stock_prices_url, headers=headers, params=prices_params, **self._request_params
        )
        factors_resp = requests.get(stock_factors_url, headers=headers, **self._request_params)

        frames = []
        for res in (prices_resp, factors_resp):
            if not res.ok:
                error_msg = self._try_extract_error_code(res)
                logger.error(error_msg)
                raise BoostedAPIException(
                    f"Failed to fetch stock data table for model {model_id}: {error_msg=}"
                )
            result = res.json()
            frames.append(pd.DataFrame(result))

        output_df = pd.concat(frames)
        return output_df

    def add_hedge_experiment_scenario(
        self,
        experiment_id: str,
        scenario_name: str,
        scenario_settings: PortfolioSettings,
        run_scenario_immediately: bool,
    ) -> HedgeExperimentScenario:

        add_scenario_input = {
            "hedgeExperimentId": experiment_id,
            "scenarioName": scenario_name,
            "portfolioSettingsJson": str(scenario_settings),
            "runExperimentOnScenario": run_scenario_immediately,
            "createDefaultPortfolio": "false",
        }
        qry = """
            mutation addHedgeExperimentScenario(
                $input: AddHedgeExperimentScenarioInput!
            ) {
                addHedgeExperimentScenario(input: $input) {
                    hedgeExperimentScenario {
                        hedgeExperimentScenarioId
                        scenarioName
                        description
                        portfolioSettingsJson
                    }
                }
            }

        """

        url = f"{self.base_uri}/api/graphql"

        resp = requests.post(
            url,
            headers={"Authorization": "ApiKey " + self.api_key},
            json={"query": qry, "variables": {"input": add_scenario_input}},
        )

        json_resp = resp.json()
        if (resp.ok and "errors" in json_resp) or not resp.ok:
            error_msg = self._try_extract_error_code(resp)
            logger.error(error_msg)
            raise BoostedAPIException(
                (f"Failed to add scenario: {resp.status_code=}; {error_msg=}")
            )

        scenario_dict = json_resp["data"]["addHedgeExperimentScenario"]["hedgeExperimentScenario"]
        if scenario_dict is None:
            raise BoostedAPIException(
                "Failed to add scenario, likely due to bad experiment id or api key"
            )
        s = HedgeExperimentScenario.from_json_dict(scenario_dict)
        return s

    # experiment life cycle has 4 steps:
    # 1. creation - essentially a very simple registration of a new instance, returning an id
    # 2. modify - populate with settings
    # 3. start - run the experiment
    # 4. delete - drop the experiment
    # while i would prefer to just have 2 funcs for (1,2,3) and (4) for a simpler api,
    # we need to expose finer-grained control becuase of how scenarios work.
    def create_hedge_experiment(
        self,
        name: str,
        description: str,
        experiment_type: hedge_experiment_type,
        target_securities: Union[Dict[GbiIdSecurity, float], str],
    ) -> HedgeExperiment:
        # we don't pass target_securities here (as much as id like to) because the
        # graphql input doesn't support it at this point

        # note that this query returns a lot of null fields at this point, but
        # they are necessary for building a HE.
        create_qry = """
            mutation createDraftMutation($input: CreateHedgeExperimentDraftInput!) {
                createHedgeExperimentDraft(input: $input) {
                    hedgeExperiment {
                        hedgeExperimentId
                        experimentName
                        userId
                        config
                        description
                        experimentType
                        lastCalculated
                        lastModified
                        status
                        portfolioCalcStatus
                        targetSecurities {
                            gbiId
                            security {
                                gbiId
                                name
                                symbol
                            }
                            weight
                        }
                        baselineModel {
                            id
                            name
                        }
                        baselineScenario {
                            hedgeExperimentScenarioId
                            scenarioName
                            description
                            portfolioSettingsJson
                            hedgeExperimentPortfolios {
                                portfolio {
                                    id
                                    name
                                    modelId
                                    performanceGridHeader
                                    performanceGrid
                                    status
                                    tearSheet {
                                        groupName
                                        members {
                                            name
                                            value
                                        }
                                    }
                                }
                            }
                            status
                        }
                        baselineStockUniverseId
                    }
                }
            }
        """

        create_input = {"name": name, "experimentType": experiment_type, "description": description}
        if isinstance(target_securities, dict):
            create_input["setTargetSecurities"] = [
                {"gbiId": sec.gbi_id, "weight": weight}
                for (sec, weight) in target_securities.items()
            ]
        elif isinstance(target_securities, str):
            create_input["setTargetPortfolios"] = [{"portfolioId": target_securities}]
        elif target_securities is None:
            pass
        else:
            raise TypeError(
                "Expected value of type Union[Dict[GbiIdSecurity, str], str] for "
                f"argument 'target_securities'; got {type(target_securities)}"
            )
        resp = requests.post(
            f"{self.base_uri}/api/graphql",
            json={"query": create_qry, "variables": {"input": create_input}},
            headers={"Authorization": "ApiKey " + self.api_key},
            params=self._request_params,
        )

        json_resp = resp.json()
        if (resp.ok and "errors" in json_resp) or not resp.ok:
            error_msg = self._try_extract_error_code(resp)
            logger.error(error_msg)
            raise BoostedAPIException(
                (f"Failed to create hedge experiment: {resp.status_code=}; {error_msg=}")
            )

        exp_dict = json_resp["data"]["createHedgeExperimentDraft"]["hedgeExperiment"]
        experiment = HedgeExperiment.from_json_dict(exp_dict)
        return experiment

    def modify_hedge_experiment(
        self,
        experiment_id: str,
        name: Optional[str] = None,
        description: Optional[str] = None,
        experiment_type: Optional[hedge_experiment_type] = None,
        target_securities: Union[Dict[GbiIdSecurity, float], str, None] = None,
        model_ids: Optional[List[str]] = None,
        stock_universe_ids: Optional[List[str]] = None,
        create_default_scenario: bool = True,
        baseline_model_id: Optional[str] = None,
        baseline_stock_universe_id: Optional[str] = None,
        baseline_portfolio_settings: Optional[str] = None,
    ) -> HedgeExperiment:

        mod_qry = """
            mutation modifyHedgeExperimentDraft(
                $input: ModifyHedgeExperimentDraftInput!
            ) {
                modifyHedgeExperimentDraft(input: $input) {
                    hedgeExperiment {
                    ...HedgeExperimentSelectedSecuritiesPageFragment
                    }
                }
            }

            fragment HedgeExperimentSelectedSecuritiesPageFragment on HedgeExperiment {
                hedgeExperimentId
                experimentName
                userId
                config
                description
                experimentType
                lastCalculated
                lastModified
                status
                portfolioCalcStatus
                targetSecurities {
                    gbiId
                    security {
                        gbiId
                        name
                        symbol
                    }
                    weight
                }
                targetPortfolios {
                    portfolioId
                }
                baselineModel {
                    id
                    name
                }
                baselineScenario {
                    hedgeExperimentScenarioId
                    scenarioName
                    description
                    portfolioSettingsJson
                    hedgeExperimentPortfolios {
                        portfolio {
                            id
                            name
                            modelId
                            performanceGridHeader
                            performanceGrid
                            status
                            tearSheet {
                                groupName
                                members {
                                    name
                                    value
                                }
                            }
                        }
                    }
                    status
                }
                baselineStockUniverseId
            }
        """
        mod_input = {
            "hedgeExperimentId": experiment_id,
            "createDefaultScenario": create_default_scenario,
        }
        if name is not None:
            mod_input["newExperimentName"] = name
        if description is not None:
            mod_input["newExperimentDescription"] = description
        if experiment_type is not None:
            mod_input["newExperimentType"] = experiment_type
        if model_ids is not None:
            mod_input["setSelectdModels"] = model_ids
        if stock_universe_ids is not None:
            mod_input["selectedStockUniverseIds"] = stock_universe_ids
        if baseline_model_id is not None:
            mod_input["setBaselineModel"] = baseline_model_id
        if baseline_stock_universe_id is not None:
            mod_input["setBaselineStockUniverse"] = baseline_stock_universe_id
        if baseline_portfolio_settings is not None:
            mod_input["setBaselinePortfolioSettings"] = baseline_portfolio_settings
        # note that the behaviors bound to these data are mutually exclusive,
        # and its possible the opposite was set earlier in the DRAFT phase
        # of experiment creation, so when setting one, we must unset the other
        if isinstance(target_securities, dict):
            mod_input["setTargetSecurities"] = [
                {"gbiId": sec.gbi_id, "weight": weight}
                for (sec, weight) in target_securities.items()
            ]
            mod_input["setTargetPortfolios"] = None
        elif isinstance(target_securities, str):
            mod_input["setTargetPortfolios"] = [{"portfolioId": target_securities}]
            mod_input["setTargetSecurities"] = None
        elif target_securities is None:
            pass
        else:
            raise TypeError(
                "Expected value of type Union[Dict[GbiIdSecurity, str], str] "
                f"for argument 'target_securities'; got {type(target_securities)}"
            )

        resp = requests.post(
            f"{self.base_uri}/api/graphql",
            json={"query": mod_qry, "variables": {"input": mod_input}},
            headers={"Authorization": "ApiKey " + self.api_key},
            params=self._request_params,
        )

        json_resp = resp.json()
        if (resp.ok and "errors" in json_resp) or not resp.ok:
            error_msg = self._try_extract_error_code(resp)
            logger.error(error_msg)
            raise BoostedAPIException(
                (
                    f"Failed to modify hedge experiment in preparation for start {experiment_id=}: "
                    f"{resp.status_code=}; {error_msg=}"
                )
            )

        exp_dict = json_resp["data"]["modifyHedgeExperimentDraft"]["hedgeExperiment"]
        experiment = HedgeExperiment.from_json_dict(exp_dict)
        return experiment

    def start_hedge_experiment(self, experiment_id: str, *scenario_ids: str) -> HedgeExperiment:
        start_qry = """
            mutation startHedgeExperiment($input: StartHedgeExperimentInput!) {
                startHedgeExperiment(input: $input) {
                    hedgeExperiment {
                        hedgeExperimentId
                        experimentName
                        userId
                        config
                        description
                        experimentType
                        lastCalculated
                        lastModified
                        status
                        portfolioCalcStatus
                        targetSecurities {
                            gbiId
                            security {
                                gbiId
                                name
                                symbol
                            }
                            weight
                        }
                        targetPortfolios {
                            portfolioId
                        }
                        baselineModel {
                            id
                            name
                        }
                        baselineScenario {
                            hedgeExperimentScenarioId
                            scenarioName
                            description
                            portfolioSettingsJson
                            hedgeExperimentPortfolios {
                                portfolio {
                                    id
                                    name
                                    modelId
                                    performanceGridHeader
                                    performanceGrid
                                    status
                                    tearSheet {
                                        groupName
                                        members {
                                            name
                                            value
                                        }
                                    }
                                }
                            }
                            status
                        }
                        baselineStockUniverseId
                    }
                }
            }
        """
        start_input = {"hedgeExperimentId": experiment_id}
        if len(scenario_ids) > 0:
            start_input["hedgeExperimentScenarioIds"] = list(scenario_ids)

        resp = requests.post(
            f"{self.base_uri}/api/graphql",
            json={"query": start_qry, "variables": {"input": start_input}},
            headers={"Authorization": "ApiKey " + self.api_key},
            params=self._request_params,
        )

        json_resp = resp.json()
        if (resp.ok and "errors" in json_resp) or not resp.ok:
            error_msg = self._try_extract_error_code(resp)
            logger.error(error_msg)
            raise BoostedAPIException(
                (
                    f"Failed to start hedge experiment {experiment_id=}: "
                    f"{resp.status_code=}; {error_msg=}"
                )
            )

        exp_dict = json_resp["data"]["startHedgeExperiment"]["hedgeExperiment"]
        experiment = HedgeExperiment.from_json_dict(exp_dict)
        return experiment

    def delete_hedge_experiment(self, experiment_id: str) -> bool:
        delete_qry = """
            mutation($input: DeleteHedgeExperimentsInput!) {
                deleteHedgeExperiments(input: $input) {
                    success
                }
            }
        """
        delete_input = {"hedgeExperimentIds": [experiment_id]}
        resp = requests.post(
            f"{self.base_uri}/api/graphql",
            json={"query": delete_qry, "variables": {"input": delete_input}},
            headers={"Authorization": "ApiKey " + self.api_key},
            params=self._request_params,
        )

        json_resp = resp.json()
        if (resp.ok and "errors" in json_resp) or not resp.ok:
            error_msg = self._try_extract_error_code(resp)
            logger.error(error_msg)
            raise BoostedAPIException(
                (
                    f"Failed to delete hedge experiment {experiment_id=}: "
                    + f"status_code={resp.status_code}; error_msg={error_msg}"
                )
            )

        return json_resp["data"]["deleteHedgeExperiments"]["success"]

    def get_portfolio_accuracy(self, model_id: str, portfolio_id: str) -> dict:
        # TODO: Later change this URI to not use the watchlist prefix. It is misnamed.
        url = f"{self.base_uri}{WATCHLIST_ROUTE_PREFIX}{DAL_PA_ROUTE}/get-hit-rate/"
        headers = {"Authorization": "ApiKey " + self.api_key, "Content-Type": "application/json"}
        req_json = {"model_id": model_id, "portfolio_id": portfolio_id}
        res = requests.post(url, json=req_json, headers=headers, **self._request_params)

        if not res.ok:
            error_msg = self._try_extract_error_code(res)
            logger.error(error_msg)
            raise BoostedAPIException(f"Failed to get Hit Rate: {error_msg}")

        data = res.json()
        return data

    def create_watchlist(self, name: str) -> str:
        url = f"{self.base_uri}{WATCHLIST_ROUTE_PREFIX}{DAL_WATCHLIST_ROUTE}/create/"
        headers = {"Authorization": "ApiKey " + self.api_key, "Content-Type": "application/json"}
        req_json = {"name": name}
        res = requests.post(url, json=req_json, headers=headers, **self._request_params)

        if not res.ok:
            error_msg = self._try_extract_error_code(res)
            logger.error(error_msg)
            raise BoostedAPIException(f"Failed to get user models: {error_msg}")

        data = res.json()
        return data["watchlist_id"]

    def _get_graphql(
        self, query: str, variables: Dict, error_msg_prefix: str = "Failed to get graphql result: "
    ) -> Dict:
        headers = {"Authorization": "ApiKey " + self.api_key}
        json_req = {"query": query, "variables": variables}

        url = self.base_uri + "/api/graphql"
        resp = requests.post(
            url,
            json=json_req,
            headers=headers,
            params=self._request_params,
        )

        # graphql endpoints typically return 200 or 400 status codes, so we must
        # check if we have any errors, even with a 200
        if not resp.ok or (resp.ok and "errors" in resp.json()):
            error_msg = self._try_extract_error_code(resp)
            error_str = str(error_msg_prefix) + f" {resp.status_code=}; {error_msg=}"
            logger.error(error_str)
            raise BoostedAPIException(error_str)

        json_resp = resp.json()
        return json_resp

    def _get_security_info(self, gbi_ids: List[int]) -> Dict:
        query = GET_SEC_INFO_QRY
        variables = {
            "ids": [] if not gbi_ids else gbi_ids,
        }

        error_msg_prefix = "Failed to get Security Details:"
        return self._get_graphql(
            query=query, variables=variables, error_msg_prefix=error_msg_prefix
        )

    def _get_sector_info(self) -> Dict:
        """
        Returns a list of sector objects, e.g.
        {
            "id": 1010,
            "parentId": 10,
            "name": "Energy",
            "topParentName": null,
            "spiqSectorId": -1,
            "legacy": false
        }
        """
        url = f"{self.base_uri}/api/sectors"
        headers = {"Authorization": "ApiKey " + self.api_key}
        res = requests.get(url, headers=headers, **self._request_params)
        self._check_ok_or_err_with_msg(res, "Failed to get sectors data")
        return res.json()["sectors"]

    def _get_watchlist_analysis(
        self,
        gbi_ids: List[int],
        model_ids: List[str],
        portfolio_ids: List[str],
        asof_date=datetime.date.today(),
    ) -> Dict:
        query = WATCHLIST_ANALYSIS_QRY
        variables = {
            "gbiIds": gbi_ids,
            "modelIds": model_ids,
            "portfolioIds": portfolio_ids,
            "date": self.__iso_format(asof_date),
        }
        error_msg_prefix = "Failed to get Coverage Analysis:"
        return self._get_graphql(
            query=query, variables=variables, error_msg_prefix=error_msg_prefix
        )

    def _get_models_for_portfolio(self, portfolio_ids: List[str]) -> Dict:
        query = GET_MODELS_FOR_PORTFOLIOS_QRY
        variables = {"ids": portfolio_ids}
        error_msg_prefix = "Failed to get Models for Portfolios: "
        return self._get_graphql(
            query=query, variables=variables, error_msg_prefix=error_msg_prefix
        )

    def _get_excess_return(
        self, model_ids: List[str], gbi_ids: List[int], asof_date=datetime.date.today()
    ) -> Dict:
        query = GET_EXCESS_RETURN_QRY

        variables = {
            "modelIds": model_ids,
            "gbiIds": gbi_ids,
            "date": self.__iso_format(asof_date),
        }
        error_msg_prefix = "Failed to get Excess Return Slugging Pct: "
        return self._get_graphql(
            query=query, variables=variables, error_msg_prefix=error_msg_prefix
        )

    def _coverage_column_name_format(self, in_str) -> str:
        if in_str.upper() == "ISIN":
            return "ISIN"

        return in_str.title()

    def get_coverage_info(self, watchlist_id: str, portfolio_group_id: str) -> pd.DataFrame:

        # get securities list in watchlist
        watchlist_details = self.get_watchlist_details(watchlist_id)
        security_list = watchlist_details["targets"]

        gbi_ids = [x["gbi_id"] for x in security_list]

        gbi_data = {x: {} for x in gbi_ids}

        # get security info ticker, name, industry etc
        sec_info = self._get_security_info(gbi_ids)

        for sec in sec_info["data"]["securities"]:
            gbi_id = sec["gbiId"]
            for k in ["symbol", "name", "isin", "country", "currency"]:
                gbi_data[gbi_id][self._coverage_column_name_format(k)] = sec[k]

            gbi_data[gbi_id][self._coverage_column_name_format("Sector")] = sec["sector"][
                "topParentName"
            ]

        # get portfolios list in portfolio_Group
        portfolio_group = self.get_portfolio_group(portfolio_group_id)
        portfolio_ids = [x["portfolio_id"] for x in portfolio_group["portfolios"]]
        portfolio_info = {x["portfolio_id"]: x for x in portfolio_group["portfolios"]}

        model_resp = self._get_models_for_portfolio(portfolio_ids=portfolio_ids)
        for portfolio in model_resp["data"]["portfolios"]:
            portfolio_info[portfolio["id"]].update(portfolio)

        model_info = {
            x["modelId"]: portfolio_info[x["id"]] for x in model_resp["data"]["portfolios"]
        }

        # model_ids and portfolio_ids are parallel arrays
        model_ids = [portfolio_info[x]["modelId"] for x in portfolio_ids]

        # graphql: get watchlist analysis
        wl_analysis = self._get_watchlist_analysis(
            gbi_ids=gbi_ids,
            model_ids=model_ids,
            portfolio_ids=portfolio_ids,
            asof_date=datetime.date.today(),
        )

        portfolio_gbi_data = {k: {} for k in portfolio_ids}
        for pi, v in portfolio_gbi_data.items():
            v.update({k: {} for k in gbi_data.keys()})

        equity_explorer_date = wl_analysis["data"]["watchlistAnalysis"][0]["analysisDates"][0][
            "date"
        ]
        for wla in wl_analysis["data"]["watchlistAnalysis"]:
            gbi_id = wla["gbiId"]
            gbi_data[gbi_id]["Composite Rating"] = wla["analysisDates"][0]["aggregateSignal"][
                "rating"
            ]
            gbi_data[gbi_id]["Composite Rating Delta"] = wla["analysisDates"][0]["aggregateSignal"][
                "ratingDelta"
            ]

            for p in wla["analysisDates"][0]["portfoliosSignals"]:
                model_name = portfolio_info[p["portfolioId"]]["modelName"]

                portfolio_gbi_data[p["portfolioId"]][gbi_id][
                    model_name + self._coverage_column_name_format(": rank")
                ] = (p["rank"] + 1)
                portfolio_gbi_data[p["portfolioId"]][gbi_id][
                    model_name + self._coverage_column_name_format(": rank delta")
                ] = (-1 * p["signalDelta"])
                portfolio_gbi_data[p["portfolioId"]][gbi_id][
                    model_name + self._coverage_column_name_format(": rating")
                ] = p["rating"]
                portfolio_gbi_data[p["portfolioId"]][gbi_id][
                    model_name + self._coverage_column_name_format(": rating delta")
                ] = p["ratingDelta"]

        neg_rec = {k: {} for k in gbi_data.keys()}
        pos_rec = {k: {} for k in gbi_data.keys()}
        for wla in wl_analysis["data"]["watchlistAnalysis"]:
            gbi_id = wla["gbiId"]

            for pid, signals in zip(portfolio_ids, wla["analysisDates"][0]["portfoliosSignals"]):
                model_name = portfolio_info[pid]["modelName"]
                neg_rec[gbi_id][
                    model_name + self._coverage_column_name_format(": negative recommendation")
                ] = signals["explainWeightNeg"]
                pos_rec[gbi_id][
                    model_name + self._coverage_column_name_format(": positive recommendation")
                ] = signals["explainWeightPos"]

        # graphql: GetExcessReturn - slugging pct
        er_sp = self._get_excess_return(
            model_ids=model_ids, gbi_ids=gbi_ids, asof_date=equity_explorer_date
        )

        for model in er_sp["data"]["models"]:
            model_name = model_info[model["id"]]["modelName"]
            for stat in model["equityExplorerData"]["equityExplorerSummaryStatistics"]:
                portfolioId = model_info[model["id"]]["id"]
                portfolio_gbi_data[portfolioId][int(stat["gbiId"])][
                    model_name + self._coverage_column_name_format(": slugging %")
                ] = (stat["ER"]["SP"]["oneMonth"] * 100)

        # add rank, rating, slugging
        for pid, v in portfolio_gbi_data.items():
            for gbi_id, vv in v.items():
                gbi_data[gbi_id].update(vv)

        # add neg/pos rec scores
        for rec in [neg_rec, pos_rec]:
            for k, v in rec.items():
                gbi_data[k].update(v)

        df = pd.DataFrame.from_records([v for _, v in gbi_data.items()])

        return df

    def get_coverage_csv(
        self, watchlist_id: str, portfolio_group_id: str, filepath: Optional[str] = None
    ) -> Optional[str]:
        """
        Converts the coverage contents to CSV format
        Parameters
        ----------
        watchlist_id: str
            UUID str identifying the coverage watchlist
        portfolio_group_id: str
            UUID str identifying the group of portfolio to use for analysis
        filepath: Optional[str]
            UUID str identifying the group of portfolio to use for analysis

        Returns:
        ----------
        None if filepath is provided, else a string with a csv's contents is returned
        """

        df = self.get_coverage_info(watchlist_id, portfolio_group_id)

        return df.to_csv(filepath, index=False, float_format="%.4f")

    def get_watchlist_details(self, watchlist_id: str) -> Dict:

        url = f"{self.base_uri}{ROUTE_PREFIX}{DAL_WATCHLIST_ROUTE}/details/"
        headers = {"Authorization": "ApiKey " + self.api_key, "Content-Type": "application/json"}
        req_json = {"watchlist_id": watchlist_id}
        res = requests.post(url, json=req_json, headers=headers, **self._request_params)

        if not res.ok:
            error_msg = self._try_extract_error_code(res)
            logger.error(error_msg)
            raise BoostedAPIException(f"Failed to get user models: {error_msg}")

        data = res.json()
        return data

    def create_watchlist_from_file(self, name: str, filepath: str) -> str:

        url = f"{self.base_uri}{ROUTE_PREFIX}{DAL_WATCHLIST_ROUTE}/create_watchlist_from_file/"
        headers = {"Authorization": "ApiKey " + self.api_key}

        with open(filepath, "rb") as fp:
            file_bytes = fp.read()

        file_bytes_base64 = base64.b64encode(file_bytes).decode("ascii")
        json_req = {
            "content_type": mimetypes.guess_type(filepath)[0],
            "file_bytes_base64": file_bytes_base64,
            "name": name,
        }

        res = requests.post(url, json=json_req, headers=headers)

        if not res.ok:
            error_msg = self._try_extract_error_code(res)
            logger.error(error_msg)
            raise BoostedAPIException(f"Failed to create watchlist from file: {error_msg}")

        data = res.json()
        return data["watchlist_id"]

    def get_watchlists(self) -> List[Dict]:

        url = f"{self.base_uri}{ROUTE_PREFIX}{DAL_WATCHLIST_ROUTE}/get_user_watchlists/"
        headers = {"Authorization": "ApiKey " + self.api_key, "Content-Type": "application/json"}
        req_json = {}
        res = requests.post(url, json=req_json, headers=headers, **self._request_params)

        if not res.ok:
            error_msg = self._try_extract_error_code(res)
            logger.error(error_msg)
            raise BoostedAPIException(f"Failed to get user watchlists: {error_msg}")

        data = res.json()
        return data["watchlists"]

    def get_watchlist_contents(self, watchlist_id) -> Dict:

        url = f"{self.base_uri}{ROUTE_PREFIX}{DAL_WATCHLIST_ROUTE}/contents/"
        headers = {"Authorization": "ApiKey " + self.api_key, "Content-Type": "application/json"}
        req_json = {"watchlist_id": watchlist_id}
        res = requests.post(url, json=req_json, headers=headers, **self._request_params)

        if not res.ok:
            error_msg = self._try_extract_error_code(res)
            logger.error(error_msg)
            raise BoostedAPIException(f"Failed to get watchlist contents: {error_msg}")

        data = res.json()
        return data

    def get_watchlist_contents_as_csv(self, watchlist_id, filepath) -> None:
        data = self.get_watchlist_contents(watchlist_id)
        df = pd.DataFrame(data["contents"])
        df.to_csv(filepath, index=False)

    # TODO this will need to be enhanced to accept country/currency overrides
    def add_securities_to_watchlist(
        self, watchlist_id: str, identifiers: List[str], identifier_type: Literal["TICKER", "ISIN"]
    ) -> Dict:

        # should we just make the arg lower? all caps has a flag-like feel to it
        id_type = identifier_type.lower()
        url = f"{self.base_uri}{WATCHLIST_ROUTE_PREFIX}{DAL_WATCHLIST_ROUTE}/add_{id_type}s/"
        headers = {"Authorization": "ApiKey " + self.api_key, "Content-Type": "application/json"}
        req_json = {"watchlist_id": watchlist_id, id_type: identifiers}
        res = requests.post(url, json=req_json, headers=headers, **self._request_params)

        if not res.ok:
            error_msg = self._try_extract_error_code(res)
            logger.error(error_msg)
            raise BoostedAPIException(f"Failed to get user models: {error_msg}")

        data = res.json()
        return data

    def remove_securities_from_watchlist(
        self, watchlist_id: str, identifiers: List[str], identifier_type: Literal["TICKER", "ISIN"]
    ) -> Dict:
        # should we just make the arg lower? all caps has a flag-like feel to it
        id_type = identifier_type.lower()
        url = f"{self.base_uri}{WATCHLIST_ROUTE_PREFIX}{DAL_WATCHLIST_ROUTE}/remove_{id_type}s/"
        headers = {"Authorization": "ApiKey " + self.api_key, "Content-Type": "application/json"}
        req_json = {"watchlist_id": watchlist_id, id_type: identifiers}
        res = requests.post(url, json=req_json, headers=headers, **self._request_params)

        if not res.ok:
            error_msg = self._try_extract_error_code(res)
            logger.error(error_msg)
            raise BoostedAPIException(f"Failed to get user models: {error_msg}")

        data = res.json()
        return data

    def get_portfolio_groups(
        self,
    ) -> Dict:
        """
        Parameters: None


        Returns:
        ----------

        Dict:  {
        user_id: str
        portfolio_groups: List[PortfolioGroup]
        }
        where PortfolioGroup is defined as = Dict {
        group_id: str
        group_name: str
        portfolios: List[PortfolioInGroup]
        }
        where PortfolioInGroup is defined as = Dict {
        portfolio_id: str
        rank_in_group: Optional[int]
        }
        """
        url = f"{self.base_uri}{WATCHLIST_ROUTE_PREFIX}{PORTFOLIO_GROUP_ROUTE}/get"
        headers = {"Authorization": "ApiKey " + self.api_key, "Content-Type": "application/json"}
        req_json = {}
        res = requests.post(url, json=req_json, headers=headers, **self._request_params)

        if not res.ok:
            error_msg = self._try_extract_error_code(res)
            logger.error(error_msg)
            raise BoostedAPIException(f"Failed to get user portfolio groups: {error_msg}")

        data = res.json()
        return data

    def get_portfolio_group(self, portfolio_group_id: str) -> Dict:
        """
        Parameters:
        portfolio_group_id: str
           UUID identifier for the portfolio group


        Returns:
        ----------

        PortfolioGroup: Dict:  {
        group_id: str
        group_name: str
        portfolios: List[PortfolioInGroup]
        }
        where PortfolioInGroup is defined as = Dict {
        portfolio_id: str
        portfolio_name: str
        rank_in_group: Optional[int]
        }
        """
        url = f"{self.base_uri}{WATCHLIST_ROUTE_PREFIX}{PORTFOLIO_GROUP_ROUTE}/get-one"
        headers = {"Authorization": "ApiKey " + self.api_key, "Content-Type": "application/json"}
        req_json = {"portfolio_group_id": portfolio_group_id}
        res = requests.post(url, json=req_json, headers=headers, **self._request_params)

        if not res.ok:
            error_msg = self._try_extract_error_code(res)
            logger.error(error_msg)
            raise BoostedAPIException(f"Failed to get user portfolio groups: {error_msg}")

        data = res.json()
        return data

    def set_sticky_portfolio_group(
        self,
        portfolio_group_id: str,
    ) -> Dict:
        """
        Set sticky portfolio group

        Parameters
        ----------

        group_id: str,
           UUID str identifying a portfolio group

        Returns:
        -------
        Dict {
            changed: int - 1 == success
        }
        """
        url = f"{self.base_uri}{WATCHLIST_ROUTE_PREFIX}{PORTFOLIO_GROUP_ROUTE}/set-sticky"
        headers = {"Authorization": "ApiKey " + self.api_key, "Content-Type": "application/json"}
        req_json = {"portfolio_group_id": portfolio_group_id}
        res = requests.post(url, json=req_json, headers=headers, **self._request_params)

        if not res.ok:
            error_msg = self._try_extract_error_code(res)
            logger.error(error_msg)
            raise BoostedAPIException(f"Failed to set sticky portfolio group: {error_msg}")

        data = res.json()
        return data

    def get_sticky_portfolio_group(
        self,
    ) -> Dict:
        """
        Get sticky portfolio group for the user

        Parameters
        ----------

        Returns:
        -------
        Dict {
            group_id: str
            group_name: str
            portfolios: List[PortfolioInGroup(Dict)]
                  PortfolioInGroup(Dict):
                           portfolio_id: str
                           rank_in_group: Optional[int] = None
                           portfolio_name: Optional[str] = None
        }
        """
        url = f"{self.base_uri}{WATCHLIST_ROUTE_PREFIX}{PORTFOLIO_GROUP_ROUTE}/get-sticky"
        headers = {"Authorization": "ApiKey " + self.api_key, "Content-Type": "application/json"}
        req_json = {}
        res = requests.post(url, json=req_json, headers=headers, **self._request_params)

        if not res.ok:
            error_msg = self._try_extract_error_code(res)
            logger.error(error_msg)
            raise BoostedAPIException(f"Failed to get sticky portfolio group: {error_msg}")

        data = res.json()
        return data

    def create_portfolio_group(
        self,
        group_name: str,
        portfolios: Optional[List[Dict]] = None,
    ) -> Dict:
        """
        Create a new portfolio group

        Parameters
        ----------

        group_name: str
           name of the new group

        portfolios: List of Dict [:

        portfolio_id: str
        rank_in_group: Optional[int] = None
        ]

        Returns:
        ----------

        Dict: {
        group_id: str
           UUID identifier for the portfolio group

        created: int
           num groups created, 1 == success

        added: int
           num portfolios added to the group, should match the length of 'portfolios' argument
        }
        """
        url = f"{self.base_uri}{WATCHLIST_ROUTE_PREFIX}{PORTFOLIO_GROUP_ROUTE}/create"
        headers = {"Authorization": "ApiKey " + self.api_key, "Content-Type": "application/json"}
        req_json = {"group_name": group_name, "portfolios": portfolios}

        res = requests.post(url, json=req_json, headers=headers, **self._request_params)

        if not res.ok:
            error_msg = self._try_extract_error_code(res)
            logger.error(error_msg)
            raise BoostedAPIException(f"Failed to create portfolio group: {error_msg}")

        data = res.json()
        return data

    def rename_portfolio_group(
        self,
        group_id: str,
        group_name: str,
    ) -> Dict:
        """
        Rename a portfolio group

        Parameters
        ----------

        group_id: str,
           UUID str identifying a portfolio group

        group_name: str,
           The new name for the porfolio

        Returns:
        -------
        Dict {
            changed: int - 1 == success
        }
        """
        url = f"{self.base_uri}{WATCHLIST_ROUTE_PREFIX}{PORTFOLIO_GROUP_ROUTE}/rename"
        headers = {"Authorization": "ApiKey " + self.api_key, "Content-Type": "application/json"}
        req_json = {"group_id": group_id, "group_name": group_name}
        res = requests.post(url, json=req_json, headers=headers, **self._request_params)

        if not res.ok:
            error_msg = self._try_extract_error_code(res)
            logger.error(error_msg)
            raise BoostedAPIException(f"Failed to rename portfolio group: {error_msg}")

        data = res.json()
        return data

    def add_to_portfolio_group(
        self,
        group_id: str,
        portfolios: List[Dict],
    ) -> Dict:
        """
        Add portfolios to a group

        Parameters
        ----------

        group_id: str,
           UUID str identifying a portfolio group

        portfolios: List of Dict [:
            portfolio_id: str
            rank_in_group: Optional[int] = None
        ]


        Returns:
        -------
        Dict {
            added: int
               number of successful changes
        }
        """
        url = f"{self.base_uri}{WATCHLIST_ROUTE_PREFIX}{PORTFOLIO_GROUP_ROUTE}/add-to-group"
        headers = {"Authorization": "ApiKey " + self.api_key, "Content-Type": "application/json"}
        req_json = {"group_id": group_id, "portfolios": portfolios}

        res = requests.post(url, json=req_json, headers=headers, **self._request_params)

        if not res.ok:
            error_msg = self._try_extract_error_code(res)
            logger.error(error_msg)
            raise BoostedAPIException(f"Failed to add portfolios to portfolio group: {error_msg}")

        data = res.json()
        return data

    def remove_from_portfolio_group(
        self,
        group_id: str,
        portfolios: List[str],
    ) -> Dict:
        """
        Remove portfolios from a group

        Parameters
        ----------

        group_id: str,
           UUID str identifying a portfolio group

        portfolios: List of str


        Returns:
        -------
        Dict {
            removed: int
               number of successful changes
        }
        """
        url = f"{self.base_uri}{WATCHLIST_ROUTE_PREFIX}{PORTFOLIO_GROUP_ROUTE}/remove-from-group"
        headers = {"Authorization": "ApiKey " + self.api_key, "Content-Type": "application/json"}
        req_json = {"group_id": group_id, "portfolios": portfolios}
        res = requests.post(url, json=req_json, headers=headers, **self._request_params)

        if not res.ok:
            error_msg = self._try_extract_error_code(res)
            logger.error(error_msg)
            raise BoostedAPIException(
                f"Failed to remove portfolios from portfolio group: {error_msg}"
            )

        data = res.json()
        return data

    def delete_portfolio_group(
        self,
        group_id: str,
    ) -> Dict:
        """
        Delete a portfolio group

        Parameters
        ----------

        group_id: str,
           UUID str identifying a portfolio group


        Returns:
        -------
        Dict {
            removed_groups: int
               number of successful changes

            removed_portfolios: int
               number of successful changes
        }
        """
        url = f"{self.base_uri}{WATCHLIST_ROUTE_PREFIX}{PORTFOLIO_GROUP_ROUTE}/remove"
        headers = {"Authorization": "ApiKey " + self.api_key, "Content-Type": "application/json"}
        req_json = {"group_id": group_id}
        res = requests.post(url, json=req_json, headers=headers, **self._request_params)

        if not res.ok:
            error_msg = self._try_extract_error_code(res)
            logger.error(error_msg)
            raise BoostedAPIException(f"Failed to delete portfolio group: {error_msg}")

        data = res.json()
        return data

    def set_portfolio_group_for_watchlist(
        self,
        portfolio_group_id: str,
        watchlist_id: str,
    ) -> Dict:
        """
        Set portfolio group for watchlist.

        Parameters
        ----------

        portfolio_group_id: str,
           UUID str identifying a portfolio group

        watchlist_id: str,
           UUID str identifying a watchlist


        Returns:
        -------
        Dict {
            success: bool
            errors:
            data: Dict
                changed: int
        }
        """
        url = f"{self.base_uri}{WATCHLIST_ROUTE_PREFIX}{DAL_WATCHLIST_ROUTE}/set-portfolio-groups/"
        headers = {"Authorization": "ApiKey " + self.api_key, "Content-Type": "application/json"}
        req_json = {"portfolio_group_id": portfolio_group_id, "watchlist_id": watchlist_id}
        res = requests.post(url, json=req_json, headers=headers, **self._request_params)

        if not res.ok:
            error_msg = self._try_extract_error_code(res)
            logger.error(error_msg)
            raise BoostedAPIException(f"Failed to set portfolio group for watchlist: {error_msg}")

        return res.json()

    def get_ranking_dates(self, model_id: str, portfolio_id: str) -> List[datetime.date]:
        headers = {"Authorization": "ApiKey " + self.api_key, "Content-Type": "application/json"}
        url = self.base_uri + f"/api/analysis/ranking-dates/{model_id}/{portfolio_id}"
        res = requests.get(url, headers=headers, **self._request_params)
        self._check_ok_or_err_with_msg(res, "Failed to get ranking dates")
        data = res.json().get("ranking_dates", [])

        return [parser.parse(d).date() for d in data]

    def get_prior_ranking_date(
        self, ranking_dates: List[datetime.date], starting_date: datetime.date
    ) -> datetime.date:
        """
        Given a starting date and a list of ranking dates, return the most
        recent previous ranking date.
        """
        # order from most recent to least
        ranking_dates.sort(reverse=True)

        for d in ranking_dates:
            if d <= starting_date:
                return d

        # if we get here, the starting date is before the earliest ranking date
        raise BoostedAPIException(f"No rankins exist on or before {starting_date}")

    def _get_risk_factors_descriptors(
        self, model_id: str, portfolio_id: str, use_v2: bool = False
    ) -> Dict[int, str]:
        """Returns a map from descriptor id to descriptor name."""
        headers = {"Authorization": "ApiKey " + self.api_key, "Content-Type": "application/json"}

        risk_factor = RISK_FACTOR_V2 if use_v2 else RISK_FACTOR
        url = self.base_uri + f"/api/{risk_factor}/{model_id}/{portfolio_id}/descriptors"
        res = requests.get(url, headers=headers, **self._request_params)

        self._check_ok_or_err_with_msg(res, "Failed to get risk factor descriptors")

        descriptors = {int(i): name for i, name in res.json().items() if i.isnumeric()}
        return descriptors

    def get_risk_groups(
        self, model_id: str, portfolio_id: str, date: datetime.date, use_v2: bool = False
    ) -> List[Dict[str, Any]]:
        # first get the group descriptors
        descriptors = self._get_risk_factors_descriptors(model_id, portfolio_id, use_v2)

        # calculate the most recent prior rankings date. This is the date
        # we need to use to query for risk group data.
        ranking_dates = self.get_ranking_dates(model_id, portfolio_id)
        ranking_date = self.get_prior_ranking_date(ranking_dates, date)
        date_str = ranking_date.strftime("%Y-%m-%d")

        risk_factor = RISK_FACTOR_V2 if use_v2 else RISK_FACTOR

        headers = {"Authorization": "ApiKey " + self.api_key, "Content-Type": "application/json"}
        url = self.base_uri + f"/api/{risk_factor}/{model_id}/{portfolio_id}/risk-groups/{date_str}"
        res = requests.get(url, headers=headers, **self._request_params)

        self._check_ok_or_err_with_msg(
            res, f"Failed to get risk factors for {model_id=}, {portfolio_id=}, {date=}"
        )

        # Response is a list of objects like:
        # [
        #   [
        #     0,
        #     14,
        #     1
        #   ],
        #   [
        #     25,
        #     12,
        #     13
        #   ],
        # 0.67013
        # ],
        #
        # Where each integer in the lists is a descriptor id.

        groups = []
        for row in res.json():
            row_map = {}
            # map descriptor id to name
            row_map["risk_group_a"] = [descriptors[i] for i in row[0]]
            row_map["risk_group_b"] = [descriptors[i] for i in row[1]]
            row_map["volatility_explained"] = row[2]
            groups.append(row_map)

        return groups

    def get_risk_factors_discovered_descriptors(
        self, model_id: str, portfolio_id: str, date: datetime.date, use_v2: bool = False
    ) -> pd.DataFrame:
        # first get the group descriptors
        descriptors = self._get_risk_factors_descriptors(model_id, portfolio_id)

        # calculate the most recent prior rankings date. This is the date
        # we need to use to query for risk group data.
        ranking_dates = self.get_ranking_dates(model_id, portfolio_id)
        ranking_date = self.get_prior_ranking_date(ranking_dates, date)
        date_str = ranking_date.strftime("%Y-%m-%d")

        risk_factor = RISK_FACTOR_V2 if use_v2 else RISK_FACTOR

        headers = {"Authorization": "ApiKey " + self.api_key, "Content-Type": "application/json"}
        url = (
            self.base_uri
            + f"/api/{risk_factor}/{model_id}/{portfolio_id}/risk-descriptors/json/{date_str}"
        )
        res = requests.get(url, headers=headers, **self._request_params)

        self._check_ok_or_err_with_msg(
            res, f"Failed to get risk factors for {model_id=}, {portfolio_id=}, {date=}"
        )

        # Endpoint returns a nested list of floats
        df = pd.DataFrame(res.json(), columns=RISK_FACTOR_COLUMNS)

        # This flat dataframe represents a potentially doubly nested structure
        # of Sector -> (high/low volatility) -> security. We don't care about
        # the high/low volatility rows, (which will have negative identifiers)
        # so we can filter these out.
        df = df[df["identifier"] >= 0]

        # now, any values that had a depth of 2 should be set to a depth of 1,
        # since we removed the double nesting.
        df.replace(to_replace=2, value=1, inplace=True)

        # This dataframe represents data that is nested on the UI, so the
        # "depth" field indicates which level of nesting each row is at. At this
        # point, a depth of 0 indicates a sector, and following depth 1 rows are
        # securities within the sector.

        # Identifiers in rows with depth 1 will be gbi ids, need to convert to
        # symbols.
        gbi_ids = df[df["depth"] == 1]["identifier"].tolist()
        sec_info = self._get_security_info(gbi_ids)["data"]["securities"]
        sec_map = {s["gbiId"]: s["symbol"] for s in sec_info}

        def convert_ids(row: pd.Series) -> pd.Series:
            # convert each row's "identifier" to the appropriate id type. If the
            # depth is 0, the identifier should be a sector, otherwise it should
            # be a ticker.
            ident = int(row["identifier"])
            row["identifier"] = (
                descriptors.get(ident).title() if row["depth"] == 0 else sec_map.get(ident)
            )
            return row

        df["depth"] = df["depth"].astype(int)
        df["stock_count"] = df["stock_count"].astype(int)
        df = df.apply(convert_ids, axis=1)
        df = df.reset_index(drop=True)
        return df

    def get_risk_factors_sectors(
        self, model_id: str, portfolio_id: str, date: datetime.date, use_v2: bool = False
    ) -> pd.DataFrame:
        # first get the group descriptors
        sectors = {s["id"]: s["name"] for s in self._get_sector_info()}

        # calculate the most recent prior rankings date. This is the date
        # we need to use to query for risk group data.
        ranking_dates = self.get_ranking_dates(model_id, portfolio_id)
        ranking_date = self.get_prior_ranking_date(ranking_dates, date)
        date_str = ranking_date.strftime("%Y-%m-%d")

        risk_factor = RISK_FACTOR_V2 if use_v2 else RISK_FACTOR

        headers = {"Authorization": "ApiKey " + self.api_key, "Content-Type": "application/json"}
        url = (
            self.base_uri
            + f"/api/{risk_factor}/{model_id}/{portfolio_id}/risk-sectors/json/{date_str}"
        )
        res = requests.get(url, headers=headers, **self._request_params)

        self._check_ok_or_err_with_msg(
            res, f"Failed to get risk factors for {model_id=}, {portfolio_id=}, {date=}"
        )

        # Endpoint returns a nested list of floats
        df = pd.DataFrame(res.json(), columns=RISK_FACTOR_COLUMNS)

        # identifier is a gics sector identifier
        df["identifier"] = df["identifier"].apply(lambda i: sectors.get(int(i)))

        # This dataframe represents data that is nested on the UI, so the
        # "depth" field indicates which level of nesting each row is at. For
        # risk factors sectors, each "depth" represents a level of specificity
        # for the sector. E.g. Energy -> Energy Equipment -> Oil & Gas Equipment
        df["depth"] = df["depth"].astype(int)
        df["stock_count"] = df["stock_count"].astype(int)
        df = df.reset_index(drop=True)
        return df

    def download_complete_portfolio_data(
        self, model_id: str, portfolio_id: str, download_filepath: str
    ):
        headers = {"Authorization": "ApiKey " + self.api_key, "Content-Type": "application/json"}
        url = self.base_uri + f"/api/models/{model_id}/{portfolio_id}/excel"

        res = requests.get(url, headers=headers, **self._request_params)
        self._check_ok_or_err_with_msg(
            res, f"Failed to get full data for {model_id=}, {portfolio_id=}"
        )

        with open(download_filepath, "wb") as f:
            f.write(res.content)

    def diff_hedge_experiment_portfolio_data(
        self,
        hedge_experiment_id: str,
        comparison_portfolios: List[str],
        categories: List[str],
    ) -> Dict:
        qry = """
        query diffHedgeExperimentPortfolios(
            $input: DiffHedgeExperimentPortfoliosInput!
        ) {
            diffHedgeExperimentPortfolios(input: $input) {
            data {
                diffs {
                    volatility {
                        date
                        vol5D
                        vol10D
                        vol21D
                        vol21D
                        vol63D
                        vol126D
                        vol189D
                        vol252D
                        vol315D
                        vol378D
                        vol441D
                        vol504D
                    }
                    performance {
                        date
                        value
                    }
                    performanceGrid {
                        headerRow
                        values
                    }
                    factors {
                        date
                        momentum
                        growth
                        size
                        value
                        dividendYield
                        volatility
                    }
                }
            }
            errors
            }
        }
        """
        headers = {"Authorization": "ApiKey " + self.api_key}
        params = {
            "hedgeExperimentId": hedge_experiment_id,
            "portfolioIds": comparison_portfolios,
            "categories": categories,
        }
        resp = requests.post(
            f"{self.base_uri}/api/graphql",
            json={"query": qry, "variables": params},
            headers=headers,
            params=self._request_params,
        )

        json_resp = resp.json()

        # graphql endpoints typically return 200 or 400 status codes, so we must
        # check if we have any errors, even with a 200
        if (resp.ok and "errors" in json_resp) or not resp.ok:
            error_msg = self._try_extract_error_code(resp)
            logger.error(error_msg)
            raise BoostedAPIException(
                (
                    f"Failed to get portfolio diffs for {hedge_experiment_id=}: "
                    f"{resp.status_code=}; {error_msg=}"
                )
            )

        diffs = json_resp["data"]["diffHedgeExperimentPortfolios"]["data"]["diffs"]
        comparisons = {}
        for pf, cmp in zip(comparison_portfolios, diffs):
            res = {
                "performance": None,
                "performanceGrid": None,
                "factors": None,
                "volatility": None,
            }
            if "performanceGrid" in cmp:
                grid = cmp["performanceGrid"]
                grid_df = pd.DataFrame(grid["values"], columns=grid["headerRow"])
                res["performanceGrid"] = grid_df
            if "performance" in cmp:
                perf_df = pd.DataFrame(cmp["performance"]).set_index("date")
                perf_df.index = pd.to_datetime(perf_df.index)
                res["performance"] = perf_df
            if "volatility" in cmp:
                vol_df = pd.DataFrame(cmp["volatility"]).set_index("date")
                vol_df.index = pd.to_datetime(vol_df.index)
                res["volatility"] = vol_df
            if "factors" in cmp:
                factors_df = pd.DataFrame(cmp["factors"]).set_index("date")
                factors_df.index = pd.to_datetime(factors_df.index)
                res["factors"] = factors_df
            comparisons[pf] = res
        return comparisons

    def get_signal_strength(self, model_id: str, portfolio_id: str) -> pd.DataFrame:
        url = self.base_uri + f"/api/analysis/signal_strength/{model_id}/{portfolio_id}"
        headers = {"Authorization": "ApiKey " + self.api_key}

        logger.info(f"Retrieving portfolio signals for {model_id=}, {portfolio_id=}")

        # Response format is a json object with a "header_row" key for column
        # names, and then a nested list of data.
        resp = requests.get(url, headers=headers, **self._request_params)
        self._check_ok_or_err_with_msg(
            resp, f"Failed to get portfolio signals for {model_id=}, {portfolio_id=}"
        )

        data = resp.json()

        df = pd.DataFrame(data=data["data"], columns=data["header_row"])
        df["Date"] = pd.to_datetime(df["Date"])
        df = df.set_index("Date")
        return df.astype(float)

    def get_rolling_signal_strength(self, model_id: str, portfolio_id: str) -> pd.DataFrame:
        url = self.base_uri + f"/api/analysis/signal_strength_rolling/{model_id}/{portfolio_id}"
        headers = {"Authorization": "ApiKey " + self.api_key}

        logger.info(f"Retrieving rolling portfolio signals for {model_id=}, {portfolio_id=}")

        # Response format is a json object with a "header_row" key for column
        # names, and then a nested list of data.
        resp = requests.get(url, headers=headers, **self._request_params)
        self._check_ok_or_err_with_msg(
            resp, f"Failed to get rolling portfolio signals for {model_id=}, {portfolio_id=}"
        )

        data = resp.json()

        df = pd.DataFrame(data=data["data"], columns=data["header_row"])
        df["Date"] = pd.to_datetime(df["Date"])
        df = df.set_index("Date")
        return df.astype(float)

    def get_portfolio_quantiles(
        self,
        model_id: str,
        portfolio_id: str,
        id_type: Literal["TICKER", "ISIN"] = "TICKER",
    ):
        headers = {"Authorization": "ApiKey " + self.api_key, "Content-Type": "application/json"}
        date = datetime.date.today().strftime("%Y-%m-%d")

        payload = {
            "model_id": model_id,
            "portfolio_id": portfolio_id,
            "fields": ["quantile"],
            "min_date": date,
            "max_date": date,
            "return_format": "json",
        }
        # TODO: Later change this URI to not use the watchlist prefix. It is misnamed.
        url = f"{self.base_uri}{WATCHLIST_ROUTE_PREFIX}{DAL_PA_ROUTE}/get-data/"

        resp = requests.post(url, json=payload, headers=headers, **self._request_params)
        self._check_ok_or_err_with_msg(resp, "Unable to get quantile data")

        resp = resp.json()
        quantile_index = resp["field_map"]["Quantile"]
        quantile_data = [[c[quantile_index] for c in r] for r in resp["data"]]
        date_cols = pd.to_datetime(resp["columns"])

        # Need to map gbi id's to isins or tickers
        gbi_ids = [int(i) for i in resp["rows"]]
        security_info = self._get_security_info(gbi_ids)

        # We now have security data, go through and create a map from internal
        # gbi id to client facing identifier
        id_key = "isin" if id_type == "ISIN" else "symbol"
        gbi_identifier_map = {
            sec["gbiId"]: sec[id_key] for sec in security_info["data"]["securities"]
        }

        df = pd.DataFrame(quantile_data, index=gbi_ids, columns=date_cols).transpose()
        df = df.rename(columns=gbi_identifier_map)
        return df

Functions

def convert_date(date: Union[datetime.date, str, None]) ‑> Optional[datetime.date]
Expand source code
def convert_date(date: Optional[BoostedDate]) -> Optional[datetime.date]:
    if isinstance(date, str):
        try:
            return parser.parse(date)
        except Exception as e:
            raise BoostedAPIException(f"Unable to parse date: {str(e)}")
    return date

Classes

class BoostedAPIException (value, data=None)

Common base class for all non-exit exceptions.

Expand source code
class BoostedAPIException(Exception):
    def __init__(self, value, data=None):
        self.value = value
        self.data = data

    def __str__(self):
        return repr(self.value)

Ancestors

  • builtins.Exception
  • builtins.BaseException
class BoostedClient (api_key, override_uri=None, debug=False, proxy=None, disable_verify_ssl=False)

Parameters

api_key : str
Your API key provided by the Boosted application. See your profile to generate a new key.
proxy : str
Your organization may require the use of a proxy for access. The address of a HTTPS proxy in the format of
:. Examples are "123.456.789:123" or "my.proxy.com:123". Do not prepend with "https://".
disable_verify_ssl : bool
Your networking setup may be behind a firewall which performs SSL inspection. Either set the REQUESTS_CA_BUNDLE environment variable to point to the location of a custom certificate bundle, or set this parameter to True to disable SSL verification as a workaround.
Expand source code
class BoostedClient:
    def __init__(
        self, api_key, override_uri=None, debug=False, proxy=None, disable_verify_ssl=False
    ):
        """
        Parameters
        ----------
        api_key: str
            Your API key provided by the Boosted application.  See your profile
            to generate a new key.
        proxy: str
            Your organization may require the use of a proxy for access.
            The address of a HTTPS proxy in the format of <address>:<port>.
            Examples are "123.456.789:123" or "my.proxy.com:123".
            Do not prepend with "https://".
        disable_verify_ssl: bool
            Your networking setup may be behind a firewall which performs SSL
            inspection. Either set the REQUESTS_CA_BUNDLE environment variable
            to point to the location of a custom certificate bundle, or set this
            parameter to True to disable SSL verification as a workaround.
        """
        if override_uri is None:
            self.base_uri = g_boosted_api_url
        else:
            self.base_uri = override_uri
        self.api_key = api_key
        self.debug = debug
        self._request_params = {}
        if debug:
            logger.setLevel(logging.DEBUG)
        else:
            logger.setLevel(logging.INFO)
        if proxy is not None:
            self._request_params["proxies"] = {"https": proxy}
        if disable_verify_ssl:
            self._request_params["verify"] = False

    def __print_json_info(self, json_data, isInference=False):
        if "warnings" in json_data.keys():
            for warning in json_data["warnings"]:
                logger.warning("  {0}".format(warning))
        if "errors" in json_data.keys():
            for error in json_data["errors"]:
                logger.error("  {0}".format(error))
                return Status.FAIL

        if "result" in json_data.keys():
            results_data = json_data["result"]
            if isInference:
                if "inferenceResultsUrl" in results_data.keys():
                    res_url = parse.urlparse(results_data["inferenceResultsUrl"])
                    logger.debug(res_url)
                    logger.info("Inference started.")
            if "updateCount" in results_data.keys():
                logger.info("Updated {0} rows.".format(results_data["updateCount"]))
            if "createCount" in results_data.keys():
                logger.info("Created {0} rows.".format(results_data["createCount"]))
            return Status.SUCCESS

    def __to_date_obj(self, dt):
        if isinstance(dt, datetime.datetime):
            dt = dt.date()
        elif isinstance(dt, datetime.date):
            return dt
        elif isinstance(dt, str):
            try:
                dt = parser.parse(dt).date()
            except ValueError:
                raise ValueError('dt: "' + dt + '" is not a valid date.')
        return dt

    def __iso_format(self, dt):
        date = self.__to_date_obj(dt)
        if date is not None:
            date = date.isoformat()
        return date

    def _check_status_code(self, response, isInference=False):
        has_json = False
        try:
            logger.debug(response.headers)
            if "Content-Type" in response.headers:
                if response.headers["Content-Type"].startswith("application/json"):
                    json_data = response.json()
                    has_json = True
            else:
                has_json = False
        except json.JSONDecodeError:
            logger.error("ERROR: response has no JSON payload.")
        if response.status_code == 200 or response.status_code == 202:
            if has_json:
                self.__print_json_info(json_data, isInference)
            else:
                pass
            return Status.SUCCESS
        if response.status_code == 404:
            if has_json:
                self.__print_json_info(json_data, isInference)
            raise BoostedAPIException(
                'Server "{0}" not reachable.  Code {1}.'.format(
                    self.base_uri, response.status_code
                ),
                data=response,
            )
        if response.status_code == 400:
            if has_json:
                self.__print_json_info(json_data, isInference)
            if isInference:
                return Status.FAIL
            else:
                raise BoostedAPIException("Error, bad request.  Check the dataset ID.", response)
        if response.status_code == 401:
            if has_json:
                self.__print_json_info(json_data, isInference)
            raise BoostedAPIException("Authorization error.", response)
        else:
            if has_json:
                self.__print_json_info(json_data, isInference)
            raise BoostedAPIException(
                "Error in API response.  Status code={0} {1}\n{2}".format(
                    response.status_code, response.reason, response.headers
                ),
                response,
            )

    def _try_extract_error_code(self, result):
        logger.info(result.headers)
        if "Content-Type" in result.headers:
            if result.headers["Content-Type"].startswith("application/json"):
                if "errors" in result.json():
                    return result.json()["errors"]
            if result.headers["Content-Type"].startswith("text/plain"):
                return result.text
        return str(result.reason)

    def _check_ok_or_err_with_msg(self, res, potential_error_msg: str):
        if not res.ok:
            error = self._try_extract_error_code(res)
            logger.error(error)
            raise BoostedAPIException(f"{potential_error_msg}: {error}")

    def query_dataset(self, dataset_id):
        url = self.base_uri + "/api/datasets/{0}".format(dataset_id)
        headers = {"Authorization": "ApiKey " + self.api_key}
        res = requests.get(url, headers=headers, **self._request_params)
        if res.ok:
            return res.json()
        else:
            error_msg = self._try_extract_error_code(res)
            logger.error(error_msg)
            raise BoostedAPIException("Failed to query dataset: {0}.".format(error_msg))

    def export_global_data(
        self,
        dataset_id,
        start=(datetime.date.today() - timedelta(days=365 * 25)),
        end=datetime.date.today(),
        timeout=600,
    ):
        query_info = self.query_dataset(dataset_id)
        if DataSetType[query_info["type"]] != DataSetType.GLOBAL:
            raise BoostedAPIException(
                f"Incorrect dataset type: {query_info['type']}" f" - Expected {DataSetType.GLOBAL}"
            )
        return self.export_data(dataset_id, start, end, timeout)

    def export_independent_data(
        self,
        dataset_id,
        start=(datetime.date.today() - timedelta(days=365 * 25)),
        end=datetime.date.today(),
        timeout=600,
    ):
        query_info = self.query_dataset(dataset_id)
        if DataSetType[query_info["type"]] != DataSetType.STRATEGY:
            raise BoostedAPIException(
                f"Incorrect dataset type: {query_info['type']}"
                f" - Expected {DataSetType.STRATEGY}"
            )
        return self.export_data(dataset_id, start, end, timeout)

    def export_dependent_data(
        self,
        dataset_id,
        start=(datetime.date.today() - timedelta(days=365 * 25)),
        end=datetime.date.today(),
        timeout=600,
    ):
        query_info = self.query_dataset(dataset_id)
        if DataSetType[query_info["type"]] != DataSetType.STOCK:
            raise BoostedAPIException(
                f"Incorrect dataset type: {query_info['type']}" f" - Expected {DataSetType.STOCK}"
            )
        return self.export_data(dataset_id, start, end, timeout)

    def export_data(
        self,
        dataset_id,
        start=(datetime.date.today() - timedelta(days=365 * 25)),
        end=datetime.date.today(),
        timeout=600,
    ):
        logger.info("Requesting start={0} end={1}.".format(start, end))
        request_url = "/api/datasets/" + dataset_id + "/export-data"
        headers = {"Authorization": "ApiKey " + self.api_key}
        start = self.__iso_format(start)
        end = self.__iso_format(end)
        params = {"start": start, "end": end}
        logger.debug("URL={0}, headers={1}, params={2}".format(request_url, headers, params))
        res = requests.get(
            self.base_uri + request_url,
            headers=headers,
            params=params,
            timeout=timeout,
            **self._request_params,
        )
        if res.ok or self._check_status_code(res):
            buf = io.StringIO(res.text)
            df = pd.read_csv(buf, index_col=0, parse_dates=True)
            if "price" in df.columns:
                df = df.drop("price", axis=1)
            return df
        else:
            error_msg = self._try_extract_error_code(res)
            logger.error(error_msg)
            raise BoostedAPIException("Failed to query dataset: {0}.".format(error_msg))

    def _get_inference(self, model_id, inference_date=datetime.date.today()):
        request_url = "/api/models/" + model_id + "/inference-results"
        headers = {"Authorization": "ApiKey " + self.api_key}
        params = {}
        params["date"] = self.__iso_format(inference_date)
        logger.debug(request_url + ", " + str(headers) + ", " + str(params))
        res = requests.get(
            self.base_uri + request_url, headers=headers, params=params, **self._request_params
        )
        status = self._check_status_code(res, isInference=True)
        if status == Status.SUCCESS:
            return res, status
        else:
            return None, status

    def get_inference(
        self, model_id, inference_date=datetime.date.today(), block=False, timeout_minutes=30
    ):
        start_time = datetime.datetime.now()
        while True:
            for numRetries in range(3):
                res, status = self._get_inference(model_id, inference_date)
                if res is not None:
                    continue
                else:
                    if status == Status.FAIL:
                        return Status.FAIL
                    logger.info("Retrying...")
            if res is None:
                logger.error("Max retries reached.  Request failed.")
                return None

            json_data = res.json()
            if "result" in json_data.keys():
                if json_data["result"]["status"] == "RUNNING":
                    still_running = True
                    if not block:
                        logger.warn("Inference job is still running.")
                        return None
                    else:
                        logger.info(
                            "Inference job is still running.  Time elapsed={0}.".format(
                                datetime.datetime.now() - start_time
                            )
                        )
                        time.sleep(10)
                else:
                    still_running = False

                if not still_running and json_data["result"]["status"] == "COMPLETE":
                    csv = json_data["result"]["signals"]
                    logger.info(json_data["result"])
                    if self._check_status_code(res, isInference=True):
                        logger.info(
                            "Total run time = {0}.".format(datetime.datetime.now() - start_time)
                        )
                        return csv
            else:
                if "errors" in json_data.keys():
                    logger.error(json_data["errors"])
                else:
                    logger.error("Error getting inference for date {0}.".format(inference_date))
                return None
            if (datetime.datetime.now() - start_time).total_seconds() / 60.0 > timeout_minutes:
                logger.error("Timeout waiting for job completion.")
                return None

    def createDataset(self, schema):
        request_url = "/api/datasets"
        headers = {"Authorization": "ApiKey " + self.api_key, "Content-Type": "application/json"}
        s = json.dumps(schema)
        logger.info("Creating dataset with schema " + s)
        res = requests.post(
            self.base_uri + request_url, data=s, headers=headers, **self._request_params
        )
        if res.ok:
            return res.json()["result"]
        else:
            raise BoostedAPIException("Dataset creation failed.")

    def getUniverse(self, modelId, date=None):
        if date is not None:
            url = "/api/models/{0}/universe/{1}".format(modelId, self.__iso_format(date))
            logger.info("Getting universe for date: {0}.".format(date))
        else:
            url = "/api/models/{0}/universe/".format(modelId)
        headers = {"Authorization": "ApiKey " + self.api_key}
        res = requests.get(self.base_uri + url, headers=headers, **self._request_params)
        if res.ok:
            buf = io.StringIO(res.text)
            df = pd.read_csv(buf, index_col=0, parse_dates=True)
            return df
        else:
            error = self._try_extract_error_code(res)
            logger.error(
                "There was a problem getting this universe or model ID: {0}.".format(error)
            )
            raise BoostedAPIException("Failed to get universe: {0}".format(error))

    def updateUniverse(self, modelId, universe_df, date=datetime.date.today() + timedelta(1)):
        date = self.__iso_format(date)
        url = self.base_uri + "/api/models/{0}/universe/{1}".format(modelId, date)
        headers = {"Authorization": "ApiKey " + self.api_key}
        logger.info("Updating universe for date {0}.".format(date))
        if isinstance(universe_df, pd.core.frame.DataFrame):
            buf = io.StringIO()
            universe_df.to_csv(buf)
            target = ("uploaded_universe.csv", buf.getvalue(), "text/csv")
            files_req = {}
            files_req["universe"] = target
            res = requests.post(url, files=files_req, headers=headers, **self._request_params)
        elif isinstance(universe_df, str):
            target = ("uploaded_universe.csv", universe_df, "text/csv")
            files_req = {}
            files_req["universe"] = target
            res = requests.post(url, files=files_req, headers=headers, **self._request_params)
        else:
            raise BoostedAPIException("Expected CSV as str or Pandas DataFrame.")
        if res.ok:
            logger.info("Universe update successful.")
            if "warnings" in res.json():
                logger.info("Warnings: {0}.".format(res.json()["warnings"]))
                return res.json()["warnings"]
            else:
                return "No warnings."
        else:
            error_msg = self._try_extract_error_code(res)
            raise BoostedAPIException("Failed to get universe: {0}.".format(error_msg))

    def create_universe(
        self, universe: Union[pd.DataFrame, str], name: str, description: str
    ) -> List[str]:
        PRESENT = "PRESENT"
        ANY = "ANY"
        EARLIST_DATE = "1900-01-01"
        LATEST_DATE = "4000-01-01"

        if isinstance(universe, (str, bytes, os.PathLike)):
            universe = pd.read_csv(universe)

        universe.columns = universe.columns.str.lower()

        # Clients are free to leave out data. Fill in some defaults here.
        if "from" not in universe.columns:
            universe["from"] = EARLIST_DATE
        if "to" not in universe.columns:
            universe["to"] = LATEST_DATE
        if "currency" not in universe.columns:
            universe["currency"] = ANY
        if "country" not in universe.columns:
            universe["country"] = ANY
        if "isin" not in universe.columns:
            universe["isin"] = None
        if "symbol" not in universe.columns:
            universe["symbol"] = None

        # to prevent conflicts with python keywords
        universe.rename(columns={"from": "from_date", "to": "to_date"}, inplace=True)

        universe = universe.replace({np.nan: None})
        security_country_currency_date_list = []
        for i, r in enumerate(universe.itertuples()):
            id_type = ColumnSubRole.ISIN
            identifier = r.isin

            if identifier is None:
                id_type = ColumnSubRole.SYMBOL
                identifier = str(r.symbol)

            # if identifier is still None, it means that there is no ISIN or
            # SYMBOL for this row, in which case we throw an error
            if identifier is None:
                raise BoostedAPIException(
                    (
                        f"Missing identifier column in universe row {i + 1}"
                        " should contain ISIN or Symbol"
                    )
                )

            security_country_currency_date_list.append(
                DateIdentCountryCurrency(
                    date=r.from_date or EARLIST_DATE,
                    identifier=identifier,
                    country=r.country or ANY,
                    currency=r.currency or ANY,
                    id_type=id_type,
                )
            )

        gbi_id_objs = self.getGbiIdFromIdentCountryCurrencyDate(security_country_currency_date_list)

        security_list = []
        for i, r in enumerate(universe.itertuples()):
            # if we have a None here, we failed to map to a gbi id
            if gbi_id_objs[i] is None:
                raise BoostedAPIException(f"Unable to map row: {tuple(r)}")

            security_list.append(
                {
                    "stockId": gbi_id_objs[i].gbi_id,
                    "fromZ": r.from_date or EARLIST_DATE,
                    "toZ": LATEST_DATE if r.to_date in (PRESENT, None) else r.to_date,
                    "removal": False,
                    "source": "UPLOAD",
                }
            )

        url = self.base_uri + "/api/template-universe/save"
        headers = {"Authorization": "ApiKey " + self.api_key, "Content-Type": "application/json"}
        req = {"name": name, "description": description, "modificationDaos": security_list}

        res = requests.post(url, json=req, headers=headers, **self._request_params)
        self._check_ok_or_err_with_msg(res, "Failed to create universe")

        if "warnings" in res.json():
            logger.info("Warnings: {0}.".format(res.json()["warnings"]))
            return res.json()["warnings"].splitlines()
        else:
            return []

    def validate_dataframe(self, df):
        if not isinstance(df, pd.core.frame.DataFrame):
            logger.error("Dataset must be of type Dataframe.")
            return False
        if type(df.index) != pd.core.indexes.datetimes.DatetimeIndex:
            logger.error("Index must be DatetimeIndex.")
            return False
        if len(df.columns) == 0:
            logger.error("No feature columns exist.")
            return False
        if len(df) == 0:
            logger.error("No rows exist.")
        return True

    def get_dataset_schema(self, dataset_id):
        url = self.base_uri + "/api/datasets/{0}/schema".format(dataset_id)
        headers = {"Authorization": "ApiKey " + self.api_key}
        res = requests.get(url, headers=headers, **self._request_params)
        if res.ok:
            json_schema = res.json()
        else:
            error_msg = self._try_extract_error_code(res)
            logger.error(error_msg)
            raise BoostedAPIException("Failed to query dataset: {0}.".format(error_msg))
        return DataSetConfig.fromDict(json_schema["result"])

    def add_dependent_dataset(
        self, dataset, datasetName="DependentDataset", schema=None, timeout=600, block=True
    ):
        result = self.add_dependent_dataset_with_warnings(
            dataset, datasetName, schema, timeout, block
        )
        return result["dataset_id"]

    def add_dependent_dataset_with_warnings(
        self,
        dataset,
        datasetName="DependentDataset",
        schema=None,
        timeout=600,
        block=True,
        no_exception_on_chunk_error=False,
    ):
        if not self.validate_dataframe(dataset):
            logger.error("dataset failed validation.")
            return None
        if schema is None:
            schema = infer_dataset_schema(datasetName, dataset, DataSetType.STOCK)
        dsid = self.createDataset(schema.toDict())
        logger.info("Creating dataset with ID = {0}.".format(dsid))
        result = self.add_dependent_data(
            dsid,
            dataset,
            timeout,
            block,
            data_type=DataAddType.CREATION,
            no_exception_on_chunk_error=no_exception_on_chunk_error,
        )
        return {"dataset_id": dsid, "warnings": result["warnings"], "errors": result["errors"]}

    def add_independent_dataset(
        self, dataset, datasetName="IndependentDataset", schema=None, timeout=600, block=True
    ):
        result = self.add_independent_dataset_with_warnings(
            dataset, datasetName, schema, timeout, block
        )
        return result["dataset_id"]

    def add_independent_dataset_with_warnings(
        self,
        dataset,
        datasetName="IndependentDataset",
        schema=None,
        timeout=600,
        block=True,
        no_exception_on_chunk_error=False,
    ):
        if not self.validate_dataframe(dataset):
            logger.error("dataset failed validation.")
            return None
        if schema is None:
            schema = infer_dataset_schema(datasetName, dataset, DataSetType.STRATEGY)
        schemaDict = schema.toDict()
        if "configurationDataJson" not in schemaDict:
            schemaDict["configurationDataJson"] = "{}"
        dsid = self.createDataset(schemaDict)
        logger.info("Creating dataset with ID = {0}.".format(dsid))
        result = self.add_independent_data(
            dsid,
            dataset,
            timeout,
            block,
            data_type=DataAddType.CREATION,
            no_exception_on_chunk_error=no_exception_on_chunk_error,
        )
        return {"dataset_id": dsid, "warnings": result["warnings"], "errors": result["errors"]}

    def add_global_dataset(
        self, dataset, datasetName="GlobalDataset", schema=None, timeout=600, block=True
    ):
        result = self.add_global_dataset_with_warnings(dataset, datasetName, schema, timeout, block)
        return result["dataset_id"]

    def add_global_dataset_with_warnings(
        self,
        dataset,
        datasetName="GlobalDataset",
        schema=None,
        timeout=600,
        block=True,
        no_exception_on_chunk_error=False,
    ):
        if not self.validate_dataframe(dataset):
            logger.error("dataset failed validation.")
            return None
        if schema is None:
            schema = infer_dataset_schema(datasetName, dataset, DataSetType.GLOBAL)
        dsid = self.createDataset(schema.toDict())
        logger.info("Creating dataset with ID = {0}.".format(dsid))
        result = self.add_global_data(
            dsid,
            dataset,
            timeout,
            block,
            data_type=DataAddType.CREATION,
            no_exception_on_chunk_error=no_exception_on_chunk_error,
        )
        return {"dataset_id": dsid, "warnings": result["warnings"], "errors": result["errors"]}

    def add_independent_data(
        self,
        dataset_id,
        csv_data,
        timeout=600,
        block=True,
        data_type=DataAddType.HISTORICAL,
        no_exception_on_chunk_error=False,
    ):
        query_info = self.query_dataset(dataset_id)
        if DataSetType[query_info["type"]] != DataSetType.STRATEGY:
            raise BoostedAPIException(
                f"Incorrect dataset type: {query_info['type']}"
                f" - Expected {DataSetType.STRATEGY}"
            )
        warnings, errors = self.setup_chunk_and_upload_data(
            dataset_id, csv_data, data_type, timeout, block, no_exception_on_chunk_error
        )
        if len(warnings) > 0:
            logger.warning(
                "Encountered {0} total warnings while uploading dataset.".format(len(warnings))
            )
        if len(errors) > 0:
            raise BoostedAPIException(
                "Encountered {0} total ERRORS while uploading dataset".format(len(errors))
                + "\n".join(errors)
            )
        return {"warnings": warnings, "errors": errors}

    def add_dependent_data(
        self,
        dataset_id,
        csv_data,
        timeout=600,
        block=True,
        data_type=DataAddType.HISTORICAL,
        no_exception_on_chunk_error=False,
    ):
        warnings = []
        query_info = self.query_dataset(dataset_id)
        if DataSetType[query_info["type"]] != DataSetType.STOCK:
            raise BoostedAPIException(
                f"Incorrect dataset type: {query_info['type']}" f" - Expected {DataSetType.STOCK}"
            )
        warnings, errors = self.setup_chunk_and_upload_data(
            dataset_id, csv_data, data_type, timeout, block, no_exception_on_chunk_error
        )
        if len(warnings) > 0:
            logger.warning(
                "Encountered {0} total warnings while uploading dataset.".format(len(warnings))
            )
        if len(errors) > 0:
            raise BoostedAPIException(
                "Encountered {0} total ERRORS while uploading dataset".format(len(errors))
                + "\n".join(errors)
            )
        return {"warnings": warnings, "errors": errors}

    def add_global_data(
        self,
        dataset_id,
        csv_data,
        timeout=600,
        block=True,
        data_type=DataAddType.HISTORICAL,
        no_exception_on_chunk_error=False,
    ):
        query_info = self.query_dataset(dataset_id)
        if DataSetType[query_info["type"]] != DataSetType.GLOBAL:
            raise BoostedAPIException(
                f"Incorrect dataset type: {query_info['type']}" f" - Expected {DataSetType.GLOBAL}"
            )
        warnings, errors = self.setup_chunk_and_upload_data(
            dataset_id, csv_data, data_type, timeout, block, no_exception_on_chunk_error
        )
        if len(warnings) > 0:
            logger.warning(
                "Encountered {0} total warnings while uploading dataset.".format(len(warnings))
            )
        if len(errors) > 0:
            raise BoostedAPIException(
                "Encountered {0} total ERRORS while uploading dataset".format(len(errors))
                + "\n".join(errors)
            )
        return {"warnings": warnings, "errors": errors}

    def get_csv_buffer(self):
        return io.StringIO()

    def start_chunked_upload(self, dataset_id):
        url = self.base_uri + "/api/datasets/{0}/start-chunked-upload".format(dataset_id)
        headers = {"Authorization": "ApiKey " + self.api_key}
        res = requests.post(url, headers=headers, **self._request_params)
        if res.ok:
            return res.json()["result"]
        else:
            error_msg = self._try_extract_error_code(res)
            logger.error(error_msg)
            raise BoostedAPIException(
                "Failed to obtain dataset lock for upload: {0}.".format(error_msg)
            )

    def abort_chunked_upload(self, dataset_id, chunk_id):
        url = self.base_uri + "/api/datasets/{0}/abort-chunked-upload".format(dataset_id)
        headers = {"Authorization": "ApiKey " + self.api_key}
        params = {"uploadGroupId": chunk_id}
        res = requests.post(url, headers=headers, **self._request_params, params=params)
        if not res.ok:
            error_msg = self._try_extract_error_code(res)
            logger.error(error_msg)
            raise BoostedAPIException(
                "Failed to abort dataset lock during error: {0}.".format(error_msg)
            )

    def check_dataset_ingestion_completion(self, dataset_id, chunk_id, start_time):
        url = self.base_uri + "/api/datasets/{0}/upload-chunk-status".format(dataset_id)
        headers = {"Authorization": "ApiKey " + self.api_key}
        params = {"uploadGroupId": chunk_id}
        res = requests.get(url, headers=headers, **self._request_params, params=params)
        res = res.json()

        finished = False
        warnings = []
        errors = []

        if type(res) == dict:
            dataset_status = res["datasetStatus"]
            chunk_status = res["chunkStatus"]
            if chunk_status != ChunkStatus.PROCESSING.value:
                finished = True
                errors = res["errors"]
                warnings = res["warnings"]
                successful_rows = res["successfulRows"]
                total_rows = res["totalRows"]
                logger.info(
                    f"Successfully ingested {successful_rows} out of {total_rows} uploaded rows."
                )
                if chunk_status in [
                    ChunkStatus.SUCCESS.value,
                    ChunkStatus.WARNING.value,
                    ChunkStatus.ERROR.value,
                ]:
                    if dataset_status != "AVAILABLE":
                        raise BoostedAPIException(
                            "Dataset was unexpectedly unavailable after chunk upload finished."
                        )
                    else:
                        logger.info("Ingestion complete.  Uploaded data is ready for use.")
                elif chunk_status == ChunkStatus.ABORTED.value:
                    errors.append(
                        "Dataset chunk upload was aborted by server! Upload did not succeed."
                    )
                else:
                    errors.append("Unexpected data ingestion status: {0}.".format(chunk_status))
            logger.info(
                "Data ingestion still running.  Time elapsed={0}.".format(
                    datetime.datetime.now() - start_time
                )
            )
        else:
            raise BoostedAPIException("Unable to get status of dataset ingestion.")
        return {"finished": finished, "warnings": warnings, "errors": errors}

    def _commit_chunked_upload(self, dataset_id, chunk_id, data_type, block=True, timeout=600):
        url = self.base_uri + "/api/datasets/{0}/commit-chunked-upload".format(dataset_id)
        headers = {"Authorization": "ApiKey " + self.api_key}
        params = {
            "uploadGroupId": chunk_id,
            "dataAddType": data_type,
            "sendCompletionEmail": not block,
        }
        res = requests.post(url, headers=headers, **self._request_params, params=params)
        if not res.ok:
            error_msg = self._try_extract_error_code(res)
            logger.error(error_msg)
            raise BoostedAPIException("Failed to commit dataset files: {0}.".format(error_msg))

        if block:
            start_time = datetime.datetime.now()
            # Keep waiting until upload is no longer in UPDATING state...
            while True:
                result = self.check_dataset_ingestion_completion(dataset_id, chunk_id, start_time)
                if result["finished"]:
                    break

                if (datetime.datetime.now() - start_time).total_seconds() > timeout:
                    err_str = (
                        f"Timeout waiting for commit of dataset: {dataset_id} | chunk: {chunk_id}"
                    )
                    logger.error(err_str)
                    return [], [err_str]

                time.sleep(10)
            return result["warnings"], result["errors"]
        else:
            return [], []

    def setup_chunk_and_upload_data(
        self,
        dataset_id,
        csv_data,
        data_type,
        timeout=600,
        block=True,
        no_exception_on_chunk_error=False,
    ):
        chunk_id = self.start_chunked_upload(dataset_id)
        logger.info("Obtained lock on dataset for upload: " + chunk_id)
        try:
            warnings, errors = self.chunk_and_upload_data(
                dataset_id, chunk_id, csv_data, timeout, no_exception_on_chunk_error
            )
            commit_warnings, commit_errors = self._commit_chunked_upload(
                dataset_id, chunk_id, data_type, block, timeout
            )
            return warnings + commit_warnings, errors + commit_errors
        except Exception:
            self.abort_chunked_upload(dataset_id, chunk_id)
            raise

    def chunk_and_upload_data(
        self, dataset_id, chunk_id, csv_data, timeout=600, no_exception_on_chunk_error=False
    ):
        if isinstance(csv_data, pd.core.frame.DataFrame):
            if not isinstance(csv_data.index, pd.core.indexes.datetimes.DatetimeIndex):
                raise BoostedAPIException("DataFrame must have DatetimeIndex as index type.")

            warnings = []
            errors = []
            logger.info("Uploading yearly.")
            for t in csv_data.index.to_period("Y").unique():
                if t is pd.NaT:
                    continue

                # serialize bit to string
                buf = self.get_csv_buffer()
                yearly_csv = csv_data.loc[str(t)]
                yearly_csv.to_csv(buf, header=True)
                raw_csv = buf.getvalue()

                # we are already chunking yearly... but if the csv still exceeds a healthy
                # limit of 50mb the final line of defence is to ignore date boundaries and
                # just chunk the rows. This is mostly for the cloudflare upload limit.
                size_lim = 50 * 1000 * 1000
                est_csv_size = sys.getsizeof(raw_csv)
                if est_csv_size > size_lim:
                    del raw_csv, buf
                    logger.info("Yearly data too large for single upload, chunking further...")
                    chunks = []
                    nchunks = math.ceil(est_csv_size / size_lim)
                    rows_per_chunk = math.ceil(len(yearly_csv) / nchunks)
                    for i in range(0, len(yearly_csv), rows_per_chunk):
                        buf = self.get_csv_buffer()
                        split_csv = yearly_csv.iloc[i : i + rows_per_chunk]
                        split_csv.to_csv(buf, header=True)
                        split_csv = buf.getvalue()
                        chunks.append(
                            (
                                "{0}-{1}".format(i + 1, min(len(yearly_csv), i + rows_per_chunk)),
                                split_csv,
                            )
                        )
                else:
                    chunks = [("all", raw_csv)]

                for i, (rows_descriptor, chunk_csv) in enumerate(chunks):
                    chunk_descriptor = "{0} in yearly chunk {1}".format(rows_descriptor, t)
                    logger.info(
                        "Uploading rows:"
                        + chunk_descriptor
                        + " (chunk {0} of {1}):".format(i + 1, len(chunks))
                    )
                    _, new_warnings, new_errors = self.upload_dataset_chunk(
                        chunk_descriptor,
                        dataset_id,
                        chunk_id,
                        chunk_csv,
                        timeout,
                        no_exception_on_chunk_error,
                    )
                    warnings.extend(new_warnings)
                    errors.extend(new_errors)
            return warnings, errors

        elif isinstance(csv_data, str):
            _, warnings, errors = self.upload_dataset_chunk(
                "all data", dataset_id, chunk_id, csv_data, timeout, no_exception_on_chunk_error
            )
            return warnings, errors
        else:
            raise BoostedAPIException("Expected CSV as str or Pandas DataFrame.")

    def upload_dataset_chunk(
        self,
        chunk_descriptor,
        dataset_id,
        chunk_id,
        csv_data,
        timeout=600,
        no_exception_on_chunk_error=False,
    ):
        logger.info("Starting upload: " + chunk_descriptor)
        url = self.base_uri + "/api/datasets/{0}/upload-dataset-chunk".format(dataset_id)
        headers = {"Authorization": "ApiKey " + self.api_key}
        files_req = {}
        warnings = []
        errors = []

        # make the network request
        target = ("uploaded_data.csv", csv_data, "text/csv")
        files_req["dataFile"] = target
        params = {"uploadGroupId": chunk_id}
        res = requests.post(
            url,
            params=params,
            files=files_req,
            headers=headers,
            timeout=timeout,
            **self._request_params,
        )

        if res.ok:
            logger.info(
                (
                    "Chunk upload completed.  "
                    "Ingestion started.  "
                    "Please wait until the data is in AVAILABLE state."
                )
            )
            if "warnings" in res.json():
                warnings = res.json()["warnings"]
                if len(warnings) > 0:
                    logger.warning("Uploaded chunk encountered data warnings: ")
                for w in warnings:
                    logger.warning(w)
        else:
            reason = "Upload failed: {0}, {1}".format(res.text, res.reason)
            logger.error(reason)
            if no_exception_on_chunk_error:
                errors.append(
                    "Chunk {0} failed: {1}. ".format(chunk_descriptor, reason)
                    + "Your data was only PARTIALLY uploaded. "
                    + "Please reattempt the upload of this chunk."
                )
            else:
                raise BoostedAPIException("Upload failed.")

        return res, warnings, errors

    def getAllocationsForDate(self, portfolio_id, date, rollback_to_last_available_date):
        date = self.__iso_format(date)
        endpoint = "latest-allocations" if rollback_to_last_available_date else "allocations"
        url = self.base_uri + "/api/portfolios/{0}/{1}".format(portfolio_id, endpoint)
        headers = {"Authorization": "ApiKey " + self.api_key}
        params = {"date": date}
        logger.info("Retrieving allocations information for date {0}.".format(date))
        res = requests.get(url, params=params, headers=headers, **self._request_params)
        if res.ok:
            logger.info("Allocations retrieval successful.")
            return res.json()
        else:
            error_msg = self._try_extract_error_code(res)
            raise BoostedAPIException("Failed to retrieve allocations: {0}.".format(error_msg))

    # New API method for fetching data from portfolio_holdings.pb2 file.
    def getAllocationsForDateV2(self, portfolio_id, date, rollback_to_last_available_date):
        date = self.__iso_format(date)
        endpoint = "latest-allocations-v2" if rollback_to_last_available_date else "allocations-v2"
        url = self.base_uri + "/api/portfolios/{0}/{1}".format(portfolio_id, endpoint)
        headers = {"Authorization": "ApiKey " + self.api_key}
        params = {"date": date}
        logger.info("Retrieving allocations information for date {0}.".format(date))
        res = requests.get(url, params=params, headers=headers, **self._request_params)
        if res.ok:
            logger.info("Allocations retrieval successful.")
            return res.json()
        else:
            error_msg = self._try_extract_error_code(res)
            raise BoostedAPIException("Failed to retrieve allocations: {0}.".format(error_msg))

    def getAllocationsByDates(self, portfolio_id, dates=None):
        url = self.base_uri + "/api/portfolios/{0}/allocationsByDate".format(portfolio_id)
        headers = {"Authorization": "ApiKey " + self.api_key}
        if dates is not None:
            fmt_dates = []
            for d in dates:
                fmt_dates.append(self.__iso_format(d))
            fmt_dates = ",".join(fmt_dates)
            params = {"dates": fmt_dates}
            logger.info("Retrieving allocations information for dates {0}.".format(fmt_dates))
        else:
            params = {"dates": None}
            logger.info("Retrieving allocations information for all dates")
        res = requests.get(url, params=params, headers=headers, **self._request_params)
        if res.ok:
            logger.info("Allocations retrieval successful.")
            return res.json()
        else:
            error_msg = self._try_extract_error_code(res)
            raise BoostedAPIException("Failed to retrieve allocations: {0}.".format(error_msg))

    def getSignalsForDate(self, portfolio_id, date, rollback_to_last_available_date):
        date = self.__iso_format(date)
        endpoint = "latest-signals" if rollback_to_last_available_date else "signals"
        url = self.base_uri + "/api/portfolios/{0}/{1}".format(portfolio_id, endpoint)
        headers = {"Authorization": "ApiKey " + self.api_key}
        params = {"date": date}
        logger.info("Retrieving signals information for date {0}.".format(date))
        res = requests.get(url, params=params, headers=headers, **self._request_params)
        if res.ok:
            logger.info("Signals retrieval successful.")
            return res.json()
        else:
            error_msg = self._try_extract_error_code(res)
            raise BoostedAPIException("Failed to retrieve signals: {0}.".format(error_msg))

    def getSignalsForAllDates(self, portfolio_id, dates=None):
        url = self.base_uri + "/api/portfolios/{0}/signalsByDate".format(portfolio_id)
        headers = {"Authorization": "ApiKey " + self.api_key}
        params = {}
        if dates is not None:
            fmt_dates = []
            for d in dates:
                fmt_dates.append(self.__iso_format(d))
            fmt_dates = ",".join(fmt_dates)
            params = {"dates": fmt_dates}
            logger.info("Retrieving signals information for dates {0}.".format(fmt_dates))
        else:
            params = {"dates": None}
            logger.info("Retrieving signals information for all dates")
        res = requests.get(url, params=params, headers=headers, **self._request_params)
        if res.ok:
            logger.info("Signals retrieval successful.")
            return res.json()
        else:
            error_msg = self._try_extract_error_code(res)
            raise BoostedAPIException("Failed to retrieve signals: {0}.".format(error_msg))

    def getEquityAccuracy(
        self,
        model_id: str,
        portfolio_id: str,
        tickers: List[str],
        start_date: Optional[datetime.datetime] = None,
        end_date: Optional[datetime.datetime] = None,
    ) -> Dict[str, Dict[str, Any]]:
        validate_start_and_end_dates(start_date, end_date)
        data = {}
        if start_date is not None:
            data["startDate"] = start_date
        if end_date is not None:
            data["endDate"] = end_date

        tickers_stream = ",".join(tickers)
        data["tickers"] = tickers_stream
        data["timestamp"] = time.strftime("%H:%M:%S")
        data["shouldRecalc"] = True
        url = self.base_uri + f"/api/analysis/equity-accuracy/{model_id}/{portfolio_id}"
        headers = {"Authorization": "ApiKey " + self.api_key, "Content-Type": "application/json"}

        logger.info(
            f"Retrieving equity accuracy data for date range {start_date} to {end_date} "
            f"for tickers: {tickers}."
        )

        # Now create dataframes from the JSON output.
        metrics = [
            "hit_rate_mean",
            "hit_rate_median",
            "excess_return_mean",
            "excess_return_median",
            "return",
            "excess_return",
        ]

        # send the request, retry if failed
        MAX_RETRIES = 10  # max of number of retries until timeout
        SLEEP_TIME = 3  # waiting time between requests

        num_retries = 0
        success = False
        while not success and num_retries < MAX_RETRIES:
            res = requests.post(url, data=json.dumps(data), headers=headers, **self._request_params)
            if res.ok:
                logger.info("Equity Accuracy Data retrieval successful.")
                info = res.json()
                success = True
            else:
                data["shouldRecalc"] = False
                num_retries += 1
                time.sleep(SLEEP_TIME)

        if not success:
            raise BoostedAPIException("Failed to retrieve equity accuracy: Request timeout.")

        for ticker, accuracy_data in info.items():
            for metric in metrics:
                metric_matrix = accuracy_data[metric]
                if not isinstance(metric_matrix, str):
                    # Set the index to the quintile label, and remove it from the data
                    index = []
                    for row in metric_matrix[1:]:
                        index.append(row.pop(0))

                    # columns are "1D", "5D", etc.
                    df = pd.DataFrame(metric_matrix[1:], columns=metric_matrix[0][1:], index=index)
                    accuracy_data[metric] = df
        return info

    def getHistoricalTradeDates(self, portfolio_id, start_date=None, end_date=None):
        end_date = self.__to_date_obj(end_date or datetime.date.today())
        start_date = self.__iso_format(start_date or (end_date - timedelta(days=365)))
        end_date = self.__iso_format(end_date)

        url = self.base_uri + "/api/portfolios/{0}/tradingDates".format(portfolio_id)
        headers = {"Authorization": "ApiKey " + self.api_key}
        params = {"startDate": start_date, "endDate": end_date}

        logger.info(
            "Retrieving historical trade dates data for date range {0} to {1}.".format(
                start_date, end_date
            )
        )
        res = requests.get(url, params=params, headers=headers, **self._request_params)
        if res.ok:
            logger.info("Trading dates retrieval successful.")
            return res.json()["dates"]
        else:
            error_msg = self._try_extract_error_code(res)
            raise BoostedAPIException("Failed to retrieve trading dates: {0}.".format(error_msg))

    def getRankingsForAllDates(self, portfolio_id, dates=None):
        url = self.base_uri + "/api/portfolios/{0}/rankingsByDate".format(portfolio_id)
        headers = {"Authorization": "ApiKey " + self.api_key}
        params = {}
        if dates is not None:
            fmt_dates = []
            for d in dates:
                fmt_dates.append(self.__iso_format(d))
            fmt_dates = ",".join(fmt_dates)
            params = {"dates": fmt_dates}
            logger.info("Retrieving rankings information for date {0}.".format(fmt_dates))
        else:
            params = {"dates": None}
            logger.info("Retrieving rankings information for all dates")
        res = requests.get(url, params=params, headers=headers, **self._request_params)
        if res.ok:
            logger.info("Rankings retrieval successful.")
            return res.json()
        else:
            error_msg = self._try_extract_error_code(res)
            raise BoostedAPIException("Failed to retrieve rankings: {0}.".format(error_msg))

    def getRankingsForDate(self, portfolio_id, date, rollback_to_last_available_date):
        date = self.__iso_format(date)
        endpoint = "latest-rankings" if rollback_to_last_available_date else "rankings"
        url = self.base_uri + "/api/{0}/{1}/{2}".format(endpoint, portfolio_id, date)
        headers = {"Authorization": "ApiKey " + self.api_key}
        logger.info("Retrieving rankings information for date {0}.".format(date))
        res = requests.get(url, headers=headers, **self._request_params)
        if res.ok:
            logger.info("Rankings retrieval successful.")
            return res.json()
        else:
            error_msg = self._try_extract_error_code(res)
            raise BoostedAPIException("Failed to retrieve rankings: {0}.".format(error_msg))

    def sendModelRecalc(self, model_id):
        url = self.base_uri + "/api/models/{0}/recalc".format(model_id)
        logger.info("Sending model recalc request for model {0}".format(model_id))
        headers = {"Authorization": "ApiKey " + self.api_key}
        res = requests.put(url, headers=headers, **self._request_params)
        if not res.ok:
            error_msg = self._try_extract_error_code(res)
            logger.error(error_msg)
            raise BoostedAPIException(
                "Failed to send model recalc request - "
                + "the model in UI may be out of date: {0}.".format(error_msg)
            )

    def add_uploaded_model_data(self, url, csv_data, request_data, timeout=600):
        logger.info("Starting upload.")
        headers = {"Authorization": "ApiKey " + self.api_key}
        files_req = {}
        target = ("data.csv", None, "text/csv")
        warnings = []
        if isinstance(csv_data, pd.core.frame.DataFrame):
            buf = io.StringIO()
            csv_data.to_csv(buf, header=False)
            if not isinstance(csv_data.index, pd.core.indexes.datetimes.DatetimeIndex):
                raise BoostedAPIException("DataFrame must have DatetimeIndex as index type.")
            target = ("uploaded_data.csv", buf.getvalue(), "text/csv")
            files_req["dataFile"] = target
            res = requests.post(
                url,
                files=files_req,
                data=request_data,
                headers=headers,
                timeout=timeout,
                **self._request_params,
            )
        elif isinstance(csv_data, str):
            target = ("uploaded_data.csv", csv_data, "text/csv")
            files_req["dataFile"] = target
            res = requests.post(
                url,
                files=files_req,
                data=request_data,
                headers=headers,
                timeout=timeout,
                **self._request_params,
            )
        else:
            raise BoostedAPIException("Expected CSV as str or Pandas DataFrame.")
        if res.ok:
            logger.info("Signals upload completed.")
            result = res.json()["result"]
            if "warningMessages" in result:
                warnings = result["warningMessages"]
        else:
            logger.error("Signals upload failed: {0}, {1}".format(res.text, res.reason))
            raise BoostedAPIException("Upload failed.")

        return res, warnings

    def createSignalsModel(self, csv_data, model_name, timeout=600):
        warnings = []
        url = self.base_uri + "/api/models/upload/signals/create"
        request_data = {"modelName": model_name, "uploadName": model_name}
        res, warnings = self.add_uploaded_model_data(url, csv_data, request_data, timeout)
        result = res.json()["result"]
        model_id = result["modelId"]
        self.sendModelRecalc(model_id)
        return model_id, warnings

    def addToUploadedModel(self, model_id, csv_data, timeout=600):
        warnings = []
        url = self.base_uri + "/api/models/{0}/upload/add-data".format(model_id)
        request_data = {}
        _, warnings = self.add_uploaded_model_data(url, csv_data, request_data, timeout)
        self.sendModelRecalc(model_id)
        return warnings

    def addSignalsToUploadedModel(self, model_id, csv_data, timeout=600):
        warnings = self.addToUploadedModel(model_id, csv_data)
        return warnings

    def getSignalsFromUploadedModel(self, model_id, date=None):
        date = self.__iso_format(date)
        url = self.base_uri + "/api/models/{0}/upload/signals".format(model_id)
        headers = {"Authorization": "ApiKey " + self.api_key}
        params = {"date": date}
        logger.info("Retrieving uploaded signals information")
        res = requests.get(url, params=params, headers=headers, **self._request_params)
        if res.ok:
            result = pd.DataFrame.from_dict(res.json()["result"])
            # ensure column order
            result = result[["date", "isin", "country", "currency", "weight"]]
            result["date"] = pd.to_datetime(result["date"], format="%Y-%m-%d")
            result = result.set_index("date")
            logger.info("Signals retrieval successful.")
            return result
        else:
            error_msg = self._try_extract_error_code(res)
            raise BoostedAPIException("Failed to retrieve signals: {0}.".format(error_msg))

    def getPortfolioSettings(self, portfolio_id, timeout=600):
        url = self.base_uri + "/api/portfolio-settings/{0}".format(portfolio_id)
        headers = {"Authorization": "ApiKey " + self.api_key}
        res = requests.get(url, headers=headers, **self._request_params)
        if res.ok:
            return PortfolioSettings(res.json())
        else:
            error_msg = self._try_extract_error_code(res)
            logger.error(error_msg)
            raise BoostedAPIException(
                "Failed to retrieve portfolio settings: {0}.".format(error_msg)
            )

    def createPortfolioWithPortfolioSettings(
        self, model_id, portfolio_name, portfolio_description, portfolio_settings, timeout=600
    ):
        url = self.base_uri + "/api/models/{0}/constraints/add".format(model_id)
        headers = {"Authorization": "ApiKey " + self.api_key, "Content-Type": "application/json"}
        setting_string = json.dumps(portfolio_settings.settings)
        logger.info("Creating new portfolio with specified setting: {}".format(setting_string))
        params = {
            "name": portfolio_name,
            "description": portfolio_description,
            "constraints": setting_string,
            "validate": "true",
        }
        res = requests.put(url, json=params, headers=headers, **self._request_params)
        response = res.json()
        if res.ok:
            return response
        else:
            error_msg = self._try_extract_error_code(res)
            logger.error(error_msg)
            raise BoostedAPIException(
                "Failed to create portfolio with the specified settings: {0}.".format(error_msg)
            )

    def getGbiIdFromIdentCountryCurrencyDate(
        self, ident_country_currency_dates: List[DateIdentCountryCurrency], timeout: int = 600
    ) -> List[GbiIdSecurity]:
        url = self.base_uri + "/api/custom-stock-data/map-identifiers-simple"
        headers = {"Authorization": "ApiKey " + self.api_key, "Content-Type": "application/json"}
        identifiers = [
            {
                "row": idx,
                "date": identifier.date,
                "isin": identifier.identifier if identifier.id_type == ColumnSubRole.ISIN else None,
                "symbol": identifier.identifier
                if identifier.id_type == ColumnSubRole.SYMBOL
                else None,
                "countryPreference": identifier.country,
                "currencyPreference": identifier.currency,
            }
            for idx, identifier in enumerate(ident_country_currency_dates)
        ]
        params = json.dumps({"identifiers": identifiers})
        logger.info(
            "Retrieving GBI-ID mapping for {} identifier tuples...".format(
                len(ident_country_currency_dates)
            )
        )
        res = requests.post(url, data=params, headers=headers, **self._request_params)

        if res.ok:
            result = res.json()
            warnings = result["warnings"]
            if warnings:
                for warning in warnings:
                    logger.warn(f"Mapping warning: {warning}")
            gbiSecurities = []
            for idx, ident in enumerate(result["mappedIdentifiers"]):
                if ident is None:
                    security = None
                else:
                    security = GbiIdSecurity(
                        ident["gbiId"],
                        ident_country_currency_dates[idx],
                        ident["symbol"],
                        ident["companyName"],
                    )
                gbiSecurities.append(security)

            return gbiSecurities
        else:
            error_msg = self._try_extract_error_code(res)
            raise BoostedAPIException(
                "Failed to retrieve identifier mappings: {0}.".format(error_msg)
            )

    # exists for backwards compatibility purposes.
    def getGbiIdFromIsinCountryCurrencyDate(self, isin_country_currency_dates, timeout=600):
        return self.getGbiIdFromIdentCountryCurrencyDate(
            ident_country_currency_dates=isin_country_currency_dates, timeout=timeout
        )

    def getTradeExplain(self, portfolio_id, date=None):
        url = self.base_uri + f"/api/explain/{portfolio_id}"
        explain_date = self.__iso_format(date)
        if explain_date:
            url = self.base_uri + f"/api/explain/{portfolio_id}/{explain_date}"
        headers = {"Authorization": "ApiKey " + self.api_key}
        res = requests.get(url, headers=headers, **self._request_params)
        if res.ok:
            buf = io.StringIO(res.text)
            df = pd.read_csv(buf, index_col=0, parse_dates=True)
            return df
        else:
            error_msg = self._try_extract_error_code(res)
            logger.error(error_msg)
            raise BoostedAPIException("Failed to get trade explain: {0}.".format(error_msg))

    # model_id: str
    # returns: Dict[str, str] representing the translation from the rankings ID (feature refs)
    # to human readable names
    def __get_rankings_ref_translation(self, model_id: str) -> Dict[str, str]:
        headers = {"Authorization": "ApiKey " + self.api_key, "Content-Type": "application/json"}
        feature_name_url = f"/api/models/{model_id}/advanced-explain/translate-feature-ref/"
        feature_name_res = requests.post(
            self.base_uri + feature_name_url,
            data=json.dumps({}),
            headers=headers,
            **self._request_params,
        )

        if feature_name_res.ok:
            feature_name_dict = feature_name_res.json()
            return {
                id: "-".join(
                    [names["variable_name"], names["transform_name"], names["normalization_name"]]
                )
                for id, names in feature_name_dict.items()
            }
        else:
            raise Exception(
                """Failed to get feature names for model,
                    this model doesn't fully support rankings 2.0"""
            )

    def getDatasetDates(self, dataset_id):
        url = self.base_uri + f"/api/datasets/{dataset_id}"
        headers = {"Authorization": "ApiKey " + self.api_key}
        res = requests.get(url, headers=headers, **self._request_params)
        if res.ok:
            dataset = res.json()
            valid_to_array = dataset.get("validTo")
            valid_to_date = None
            valid_from_array = dataset.get("validFrom")
            valid_from_date = None
            if valid_to_array:
                valid_to_date = datetime.date(
                    valid_to_array[0], valid_to_array[1], valid_to_array[2]
                )
            if valid_from_array:
                valid_from_date = datetime.date(
                    valid_from_array[0], valid_from_array[1], valid_from_array[2]
                )
            return {"validTo": valid_to_date, "validFrom": valid_from_date}
        else:
            error_msg = self._try_extract_error_code(res)
            logger.error(error_msg)
            raise BoostedAPIException("Failed to query dataset: {0}.".format(error_msg))

    def getRankingAnalysis(self, model_id, date):
        url = (
            self.base_uri
            + f"/api/explain-trades/analysis/{model_id}/{self.__iso_format(date)}/json"
        )
        headers = {"Authorization": "ApiKey " + self.api_key}
        analysis_res = requests.get(url, headers=headers, **self._request_params)
        if analysis_res.ok:
            ranking_dict = analysis_res.json()
            feature_name_dict = self.__get_rankings_ref_translation(model_id)
            columns = [feature_name_dict[col] for col in ranking_dict["columns"]]

            df = protoCubeJsonDataToDataFrame(
                ranking_dict["data"],
                "Data Buckets",
                ranking_dict["rows"],
                "Feature Names",
                columns,
                ranking_dict["fields"],
            )
            return df
        else:
            error_msg = self._try_extract_error_code(analysis_res)
            logger.error(error_msg)
            raise BoostedAPIException("Failed to get ranking analysis: {0}.".format(error_msg))

    def getRankingExplain(self, model_id, date):
        url = (
            self.base_uri + f"/api/explain-trades/explain/{model_id}/{self.__iso_format(date)}/json"
        )
        headers = {"Authorization": "ApiKey " + self.api_key, "Content-Type": "application/json"}
        explain_res = requests.get(url, headers=headers, **self._request_params)
        if explain_res.ok:
            ranking_dict = explain_res.json()
            rows = ranking_dict["rows"]
            stock_summary_url = f"/api/stock-summaries/{model_id}"
            stock_summary_body = {"gbiIds": ranking_dict["rows"]}
            summary_res = requests.post(
                self.base_uri + stock_summary_url,
                data=json.dumps(stock_summary_body),
                headers=headers,
                **self._request_params,
            )
            if summary_res.ok:
                stock_summary = summary_res.json()
                rows = [stock_summary[row]["isin"] for row in ranking_dict["rows"]]
            else:
                error_msg = self._try_extract_error_code(summary_res)
                logger.error(error_msg)
                raise BoostedAPIException(
                    "Failed to get isin information ranking explain: {0}.".format(error_msg)
                )

            feature_name_dict = self.__get_rankings_ref_translation(model_id)
            columns = [feature_name_dict[col] for col in ranking_dict["columns"]]

            df = protoCubeJsonDataToDataFrame(
                ranking_dict["data"],
                "ISINs",
                rows,
                "Feature Names",
                columns,
                ranking_dict["fields"],
            )
            return df
        else:
            error_msg = self._try_extract_error_code(explain_res)
            logger.error(error_msg)
            raise BoostedAPIException("Failed to get ranking explain: {0}.".format(error_msg))

    def getDenseSignals(self, model_id, portfolio_id, file_name=None, location="./"):
        url = self.base_uri + f"/api/models/{model_id}/{portfolio_id}/dense-signals"
        headers = {"Authorization": "ApiKey " + self.api_key}
        res = requests.get(url, headers=headers, **self._request_params)
        if file_name is None:
            file_name = f"{model_id}-{portfolio_id}_dense_signals.csv"
        download_location = os.path.join(location, file_name)
        if res.ok:
            with open(download_location, "wb") as file:
                file.write(res.content)
            print("Download Complete")
        elif res.status_code == 404:
            raise BoostedAPIException(
                f"""Dense Singals file does not exist for model:
                 {model_id} - portfolio: {portfolio_id}"""
            )
        else:
            error_msg = self._try_extract_error_code(res)
            logger.error(error_msg)
            raise BoostedAPIException(
                f"""Failed to download dense singals file for model:
                 {model_id} - portfolio: {portfolio_id}"""
            )

    def getRanking2DateAnalysisFile(
        self, model_id, portfolio_id, date, file_name=None, location="./"
    ):
        formatted_date = self.__iso_format(date)
        s3_file_name = f"{formatted_date}_analysis.xlsx"
        download_url = (
            self.base_uri + f"/api/models/{model_id}/{portfolio_id}/ranking-file/{s3_file_name}"
        )
        generate_url = (
            self.base_uri
            + f"/api/explain-trades/{model_id}/{portfolio_id}/generate/date-data/{formatted_date}"
        )
        headers = {"Authorization": "ApiKey " + self.api_key}
        if file_name is None:
            file_name = f"{model_id}-{portfolio_id}_statistical_analysis_{formatted_date}.xlsx"
        download_location = os.path.join(location, file_name)

        res = requests.get(download_url, headers=headers, **self._request_params)
        if res.ok:
            with open(download_location, "wb") as file:
                file.write(res.content)
            print("Download Complete")
        elif res.status_code == 404:
            generate_res = requests.get(generate_url, headers=headers, **self._request_params)
            if generate_res.ok:
                download_res = requests.get(download_url, headers=headers, **self._request_params)
                while download_res.status_code == 404 or (
                    download_res.ok and len(download_res.content) == 0
                ):
                    print("waiting for file to be generated")
                    time.sleep(5)
                    download_res = requests.get(
                        download_url, headers=headers, **self._request_params
                    )
                if download_res.ok:
                    with open(download_location, "wb") as file:
                        file.write(download_res.content)
                    print("Download Complete")
            else:
                error_msg = self._try_extract_error_code(res)
                logger.error(error_msg)
                raise BoostedAPIException(
                    f"""Failed to generate ranking analysis file for model:
                    {model_id} - portfolio: {portfolio_id} on date: {formatted_date}"""
                )
        else:
            error_msg = self._try_extract_error_code(res)
            logger.error(error_msg)
            raise BoostedAPIException(
                f"""Failed to download ranking analysis file for model:
                 {model_id} - portfolio: {portfolio_id} on date: {formatted_date}"""
            )

    def getRanking2DateExplainFile(
        self, model_id, portfolio_id, date, file_name=None, location="./", overwrite: bool = False
    ):
        formatted_date = self.__iso_format(date)
        s3_file_name = f"{formatted_date}_explaindata.xlsx"
        download_url = (
            self.base_uri + f"/api/models/{model_id}/{portfolio_id}/ranking-file/{s3_file_name}"
        )
        generate_url = (
            self.base_uri
            + f"/api/explain-trades/{model_id}/{portfolio_id}/generate/date-data/{formatted_date}"
        )
        headers = {"Authorization": "ApiKey " + self.api_key}
        if file_name is None:
            file_name = f"{model_id}-{portfolio_id}_explain_data_{formatted_date}.xlsx"
        download_location = os.path.join(location, file_name)

        if not overwrite:
            res = requests.get(download_url, headers=headers, **self._request_params)
        if not overwrite and res.ok:
            with open(download_location, "wb") as file:
                file.write(res.content)
            print("Download Complete")
        elif overwrite or res.status_code == 404:
            generate_res = requests.get(generate_url, headers=headers, **self._request_params)
            if generate_res.ok:
                download_res = requests.get(download_url, headers=headers, **self._request_params)
                while download_res.status_code == 404 or (
                    download_res.ok and len(download_res.content) == 0
                ):
                    print("waiting for file to be generated")
                    time.sleep(5)
                    download_res = requests.get(
                        download_url, headers=headers, **self._request_params
                    )
                if download_res.ok:
                    with open(download_location, "wb") as file:
                        file.write(download_res.content)
                    print("Download Complete")
            else:
                error_msg = self._try_extract_error_code(res)
                logger.error(error_msg)
                raise BoostedAPIException(
                    f"""Failed to generate ranking explain file for model:
                    {model_id} - portfolio: {portfolio_id} on date: {formatted_date}"""
                )
        else:
            error_msg = self._try_extract_error_code(res)
            logger.error(error_msg)
            raise BoostedAPIException(
                f"""Failed to download ranking explain file for model:
                 {model_id} - portfolio: {portfolio_id} on date: {formatted_date}"""
            )

    def getTearSheet(self, model_id, portfolio_id, start_date=None, end_date=None, block=False):
        if start_date is None or end_date is None:
            if start_date is not None or end_date is not None:
                raise ValueError("start_date and end_date must both be None or both be defined")
            return self._getCurrentTearSheet(model_id, portfolio_id)

        start_date_obj = self.__to_date_obj(start_date)
        end_date_obj = self.__to_date_obj(end_date)
        if start_date_obj >= end_date_obj:
            raise ValueError("end_date must be later than the start_date")

        # get for the given date
        url = self.base_uri + f"/api/analysis/keyfacts/{model_id}/{portfolio_id}"
        data = {
            "startDate": self.__iso_format(start_date),
            "endDate": self.__iso_format(end_date),
            "shouldRecalc": True,
        }
        headers = {"Authorization": "ApiKey " + self.api_key, "Content-Type": "application/json"}
        res = requests.post(url, data=json.dumps(data), headers=headers, **self._request_params)
        if res.status_code == 404 and block:
            retries = 0
            data["shouldRecalc"] = False
            while retries < 10:
                time.sleep(10)
                retries += 1
                res = requests.post(
                    url, data=json.dumps(data), headers=headers, **self._request_params
                )
                if res.status_code != 404:
                    break
        if res.ok:
            return res.json()
        else:
            error_msg = self._try_extract_error_code(res)
            logger.error(error_msg)
            raise BoostedAPIException(
                "Failed to get tear sheet data: {0} {1}.".format(error_msg, str(res.status_code))
            )

    def _getCurrentTearSheet(self, model_id, portfolio_id):
        url = self.base_uri + f"/api/model-summaries/{model_id}/{portfolio_id}"
        headers = {"Authorization": "ApiKey " + self.api_key, "Content-Type": "application/json"}
        res = requests.get(url, headers=headers, **self._request_params)
        if res.ok:
            json = res.json()
            return json.get("tearSheet", {})
        else:
            error_msg = self._try_extract_error_code(res)
            logger.error(error_msg)
            raise BoostedAPIException("Failed to get tear sheet data: {0}.".format(error_msg))

    def getPortfolioStatus(self, model_id, portfolio_id, job_date):
        url = (
            self.base_uri
            + f"/api/analysis/portfolioStatus/{model_id}/{portfolio_id}?jobDate={job_date}"
        )
        headers = {"Authorization": "ApiKey " + self.api_key, "Content-Type": "application/json"}
        res = requests.get(url, headers=headers, **self._request_params)
        if res.ok:
            result = res.json()
            return {
                "is_complete": result["status"],
                "last_update": None if result["lastUpdate"] is None else result["lastUpdate"][:10],
                "next_update": None if result["nextUpdate"] is None else result["nextUpdate"][:10],
            }
        else:
            error_msg = self._try_extract_error_code(res)
            logger.error(error_msg)
            raise BoostedAPIException("Failed to get portfolio status: {0}".format(error_msg))

    def getBlacklist(self, blacklist_id):
        url = self.base_uri + f"/api/blacklist/{blacklist_id}"
        headers = {"Authorization": "ApiKey " + self.api_key}
        res = requests.get(url, headers=headers, **self._request_params)
        if res.ok:
            result = res.json()
            return result
        error_msg = self._try_extract_error_code(res)
        logger.error(error_msg)
        raise BoostedAPIException(f"Failed to get blacklist with id {blacklist_id}: {error_msg}")

    def getBlacklists(self, model_id=None, company_id=None, last_N=None):
        params = {}
        if last_N:
            params["lastN"] = last_N
        if model_id:
            params["modelId"] = model_id
        if company_id:
            params["companyId"] = company_id
        url = self.base_uri + f"/api/blacklist"
        headers = {"Authorization": "ApiKey " + self.api_key}
        res = requests.get(url, headers=headers, params=params, **self._request_params)
        if res.ok:
            result = res.json()
            return result
        error_msg = self._try_extract_error_code(res)
        logger.error(error_msg)
        raise BoostedAPIException(
            f"""Failed to get blacklists with \
            model_id {model_id} company_id {company_id} last_N {last_N}: {error_msg}"""
        )

    def createBlacklist(
        self,
        isin,
        long_short=2,
        start_date=datetime.date.today(),
        end_date="4000-01-01",
        model_id=None,
    ):
        url = self.base_uri + f"/api/blacklist"
        data = {
            "modelId": model_id,
            "isin": isin,
            "longShort": long_short,
            "startDate": self.__iso_format(start_date),
            "endDate": self.__iso_format(end_date),
        }
        headers = {"Authorization": "ApiKey " + self.api_key, "Content-Type": "application/json"}
        res = requests.post(url, data=json.dumps(data), headers=headers, **self._request_params)
        if res.ok:
            return res.json()
        else:
            error_msg = self._try_extract_error_code(res)
            logger.error(error_msg)
            raise BoostedAPIException(
                f"""Failed to create the blacklist with \
                  isin {isin} long_short {long_short} start_date {start_date} end_date {end_date} \
                  model_id {model_id}: {error_msg}."""
            )

    def createBlacklistsFromCSV(self, csv_name):
        url = self.base_uri + f"/api/blacklists"
        data = []
        with open(csv_name, mode="r") as f:
            csv_reader = csv.DictReader(f)
            for row in csv_reader:
                blacklist = {"modelId": row["ModelID"], "isin": row["ISIN"]}
                if row["LongShort"] == "":
                    blacklist["longShort"] = 2
                else:
                    blacklist["longShort"] = row["LongShort"]

                if row["StartDate"] == "":
                    blacklist["startDate"] = self.__iso_format(datetime.date.today())
                else:
                    blacklist["startDate"] = self.__iso_format(row["StartDate"])

                if row["EndDate"] == "":
                    blacklist["endDate"] = self.__iso_format("4000-01-01")
                else:
                    blacklist["endDate"] = self.__iso_format(row["EndDate"])
                data.append(blacklist)
        print(f"Processed {len(data)} blacklists.")
        headers = {"Authorization": "ApiKey " + self.api_key, "Content-Type": "application/json"}
        res = requests.post(url, data=json.dumps(data), headers=headers, **self._request_params)
        if res.ok:
            return res.json()
        else:
            error_msg = self._try_extract_error_code(res)
            logger.error(error_msg)
            raise BoostedAPIException("failed to create blacklists")

    def updateBlacklist(self, blacklist_id, long_short=None, start_date=None, end_date=None):
        params = {}
        if long_short:
            params["longShort"] = long_short
        if start_date:
            params["startDate"] = start_date
        if end_date:
            params["endDate"] = end_date
        url = self.base_uri + f"/api/blacklist/{blacklist_id}"
        headers = {"Authorization": "ApiKey " + self.api_key, "Content-Type": "application/json"}
        res = requests.patch(url, json=params, headers=headers, **self._request_params)
        if res.ok:
            return res.json()
        else:
            error_msg = self._try_extract_error_code(res)
            logger.error(error_msg)
            raise BoostedAPIException(
                f"Failed to update blacklist with id {blacklist_id}: {error_msg}"
            )

    def deleteBlacklist(self, blacklist_id):
        url = self.base_uri + f"/api/blacklist/{blacklist_id}"
        headers = {"Authorization": "ApiKey " + self.api_key}
        res = requests.delete(url, headers=headers, **self._request_params)
        if res.ok:
            result = res.json()
            return result
        else:
            error_msg = self._try_extract_error_code(res)
            logger.error(error_msg)
            raise BoostedAPIException(
                f"Failed to delete blacklist with id {blacklist_id}: {error_msg}"
            )

    def getFeatureImportance(self, model_id, date, N=None):
        url = self.base_uri + f"/api/analysis/explainability/{model_id}"
        headers = {"Authorization": "ApiKey " + self.api_key}
        logger.info("Retrieving rankings information for date {0}.".format(date))
        res = requests.get(url, headers=headers, **self._request_params)
        if not res.ok:
            error_msg = self._try_extract_error_code(res)
            logger.error(error_msg)
            raise BoostedAPIException(
                f"Failed to fetch feature importance for model/portfolio {model_id}: {error_msg}"
            )

        json_data = res.json()
        if "all" not in json_data.keys() or not json_data["all"]:
            raise BoostedAPIException(f"Unexpected formatting of feature importance response")

        feature_data = json_data["all"]
        # find the right period (assuming returned json has dates in descending order)
        date_obj = self.__to_date_obj(date)
        start_date_for_return_data = self.__to_date_obj(feature_data[0]["date"])
        features_for_requested_period = None

        if date_obj > start_date_for_return_data:
            features_for_requested_period = feature_data[0]["variable"]
        else:
            i = 0
            while i < len(feature_data) - 1:
                current_date = self.__to_date_obj(feature_data[i]["date"])
                next_date = self.__to_date_obj(feature_data[i + 1]["date"])
                if next_date <= date_obj <= current_date:
                    features_for_requested_period = feature_data[i + 1]["variable"]
                    start_date_for_return_data = next_date
                    break
                i += 1

        if features_for_requested_period is None:
            raise BoostedAPIException(f"No feature data was found for requested date: {date_obj}")

        features_for_requested_period.sort(key=lambda x: x["value"], reverse=True)

        if type(N) is int and N > 0:
            df = pd.DataFrame.from_dict(features_for_requested_period[0:N])
        else:
            df = pd.DataFrame.from_dict(features_for_requested_period)
        result = df[["feature", "value"]]

        return result.rename(columns={"feature": f"feature ({start_date_for_return_data})"})

    def getAllModelNames(self) -> Dict[str, str]:
        url = f"{self.base_uri}/api/graphql"
        headers = {"Authorization": "ApiKey " + self.api_key, "Content-Type": "application/json"}
        req_json = {"query": "query listOfModels {\n models { id name }}", "variables": {}}
        res = requests.post(url, json=req_json, headers=headers, **self._request_params)
        if not res.ok:
            error_msg = self._try_extract_error_code(res)
            logger.error(error_msg)
            raise BoostedAPIException(f"Failed to get user models: {error_msg}")
        data = res.json()
        if data["data"]["models"] is None:
            return {}
        return {rec["id"]: rec["name"] for rec in data["data"]["models"]}

    def getAllModelDetails(self) -> Dict[str, Dict[str, Any]]:
        url = f"{self.base_uri}/api/graphql"
        headers = {"Authorization": "ApiKey " + self.api_key, "Content-Type": "application/json"}
        req_json = {
            "query": "query listOfModels {\n models { id name lastUpdated portfolios { id name }}}",
            "variables": {},
        }
        res = requests.post(url, json=req_json, headers=headers, **self._request_params)
        if not res.ok:
            error_msg = self._try_extract_error_code(res)
            logger.error(error_msg)
            raise BoostedAPIException(f"Failed to get user models: {error_msg}")
        data = res.json()
        if data["data"]["models"] is None:
            return {}

        output_data = {}
        for rec in data["data"]["models"]:
            model_id = rec["id"]
            output_data[model_id] = {
                "name": rec["name"],
                "last_updated": parser.parse(rec["lastUpdated"]),
                "portfolios": rec["portfolios"],
            }

        return output_data

    def get_hedge_experiments(self):
        url = self.base_uri + "/api/graphql"
        qry = """
            query getHedgeExperiments {
                hedgeExperiments {
                    hedgeExperimentId
                    experimentName
                    userId
                    config
                    description
                    experimentType
                    lastCalculated
                    lastModified
                    status
                    portfolioCalcStatus
                    targetSecurities {
                        gbiId
                        security {
                            gbiId
                            symbol
                            name
                        }
                        weight
                    }
                    targetPortfolios {
                        portfolioId
                    }
                    baselineModel {
                        id
                        name

                    }
                    baselineScenario {
                        hedgeExperimentScenarioId
                        scenarioName
                        description
                        portfolioSettingsJson
                        hedgeExperimentPortfolios {
                            portfolio {
                                id
                                name
                                modelId
                                performanceGridHeader
                                performanceGrid
                                status
                                tearSheet {
                                    groupName
                                    members {
                                        name
                                        value
                                    }
                                }
                            }
                        }
                        status
                    }
                    baselineStockUniverseId
                }
            }
        """

        headers = {"Authorization": "ApiKey " + self.api_key}
        resp = requests.post(url, json={"query": qry}, headers=headers, params=self._request_params)

        json_resp = resp.json()
        # graphql endpoints typically return 200 or 400 status codes, so we must
        # check if we have any errors, even with a 200
        if (resp.ok and "errors" in json_resp) or not resp.ok:
            error_msg = self._try_extract_error_code(resp)
            logger.error(error_msg)
            raise BoostedAPIException(
                (f"Failed to get hedge experiments: {resp.status_code=}; {error_msg=}")
            )

        json_experiments = resp.json()["data"]["hedgeExperiments"]
        experiments = [HedgeExperiment.from_json_dict(exp_json) for exp_json in json_experiments]
        return experiments

    def get_hedge_experiment_details(self, experiment_id: str):
        url = self.base_uri + "/api/graphql"
        qry = """
            query getHedgeExperimentDetails($hedgeExperimentId: ID!) {
                hedgeExperiment(hedgeExperimentId: $hedgeExperimentId) {
                ...HedgeExperimentDetailsSummaryListFragment
                }
            }

            fragment HedgeExperimentDetailsSummaryListFragment on HedgeExperiment {
                hedgeExperimentId
                experimentName
                userId
                config
                description
                experimentType
                lastCalculated
                lastModified
                status
                portfolioCalcStatus
                targetSecurities {
                    gbiId
                    security {
                        gbiId
                        symbol
                        name
                    }
                    weight
                }
                selectedModels {
                    id
                    name
                    stockUniverse {
                        name
                    }
                }
                hedgeExperimentScenarios {
                    ...experimentScenarioFragment
                }
                selectedDummyHedgeExperimentModels {
                    id
                    name
                    stockUniverse {
                        name
                    }
                }
                targetPortfolios {
                    portfolioId
                }
                baselineModel {
                    id
                    name

                }
                baselineScenario {
                    hedgeExperimentScenarioId
                    scenarioName
                    description
                    portfolioSettingsJson
                    hedgeExperimentPortfolios {
                        portfolio {
                            id
                            name
                            modelId
                            performanceGridHeader
                            performanceGrid
                            status
                            tearSheet {
                                groupName
                                members {
                                    name
                                    value
                                }
                            }
                        }
                    }
                    status
                }
                baselineStockUniverseId
            }

            fragment experimentScenarioFragment on HedgeExperimentScenario {
                hedgeExperimentScenarioId
                scenarioName
                status
                description
                portfolioSettingsJson
                hedgeExperimentPortfolios {
                    portfolio {
                        id
                        name
                        modelId
                        performanceGridHeader
                        performanceGrid
                        status
                        tearSheet {
                            groupName
                            members {
                                name
                                value
                            }
                        }
                    }
                }
            }
        """
        headers = {"Authorization": "ApiKey " + self.api_key}
        resp = requests.post(
            url,
            json={"query": qry, "variables": {"hedgeExperimentId": experiment_id}},
            headers=headers,
            params=self._request_params,
        )

        json_resp = resp.json()
        # graphql endpoints typically return 200 or 400 status codes, so we must
        # check if we have any errors, even with a 200
        if (resp.ok and "errors" in json_resp) or not resp.ok:
            error_msg = self._try_extract_error_code(resp)
            logger.error(error_msg)
            raise BoostedAPIException(
                (
                    f"Failed to get hedge experiment results for {experiment_id=}: "
                    f"{resp.status_code=}; {error_msg=}"
                )
            )

        json_exp_results = json_resp["data"]["hedgeExperiment"]
        if json_exp_results is None:
            return None  # issued a request with a non-existent experiment_id
        exp_results = HedgeExperimentDetails.from_json_dict(json_exp_results)
        return exp_results

    def get_portfolio_performance(self, portfolio_id: str) -> pd.DataFrame:
        url = f"{self.base_uri}/api/graphql"
        qry = """
            query getPortfolioPerformance($portfolioId: ID!) {
                portfolio(id: $portfolioId) {
                    id
                    modelId
                    name
                    status
                    performance {
                        benchmark
                        date
                        turnover
                        value
                    }
                }
            }
        """

        headers = {"Authorization": "ApiKey " + self.api_key}
        resp = requests.post(
            url,
            json={"query": qry, "variables": {"portfolioId": portfolio_id}},
            headers=headers,
            params=self._request_params,
        )

        json_resp = resp.json()
        # the webserver returns an error for non-ready portfolios, so we have to check
        # for this prior to the error check below
        pf = json_resp["data"].get("portfolio")
        if pf is not None and pf["status"] != "READY":
            return pd.DataFrame()

        # graphql endpoints typically return 200 or 400 status codes, so we must
        # check if we have any errors, even with a 200
        if (resp.ok and "errors" in json_resp) or not resp.ok:
            error_msg = self._try_extract_error_code(resp)
            logger.error(error_msg)
            raise BoostedAPIException(
                (
                    f"Failed to get portfolio performance for {portfolio_id=}: "
                    f"{resp.status_code=}; {error_msg=}"
                )
            )

        perf = json_resp["data"]["portfolio"]["performance"]
        df = pd.DataFrame(perf).set_index("date").rename(columns={"value": "portfolio"})
        df.index = pd.to_datetime(df.index)
        return df.astype(float)

    def _is_portfolio_still_running(self, error_msg: str) -> bool:
        # this is jank af. a proper fix of this is either at the webserver
        # returning a better response for a portfolio in draft HT2-226, OR
        # a bigger refactor of the API that moves to more OOP, which would allow us
        # to have this data all in one place
        return "Could not find a model with this ID" in error_msg

    def get_portfolio_factors(self, model_id: str, portfolio_id: str) -> pd.DataFrame:
        url = f"{self.base_uri}/api/analysis/factors/{model_id}/{portfolio_id}"
        headers = {"Authorization": "ApiKey " + self.api_key}
        resp = requests.get(url, headers=headers, params=self._request_params)

        json_resp = resp.json()
        if (resp.ok and "errors" in json_resp) or not resp.ok:
            error_msg = json_resp["errors"][0]
            if self._is_portfolio_still_running(error_msg):
                return pd.DataFrame()
            logger.error(error_msg)
            raise BoostedAPIException(
                (
                    f"Failed to get portfolio factors for {portfolio_id=}: "
                    f"{resp.status_code=}; {error_msg=}"
                )
            )

        df = pd.DataFrame(json_resp["data"], columns=json_resp["header_row"])

        def to_lower_snake_case(s):  # why are we linting lambdas? :(
            return "_".join(w.lower() for w in s.split(" "))

        df = df.rename(columns={old: to_lower_snake_case(old) for old in df.columns}).set_index(
            "date"
        )
        df.index = pd.to_datetime(df.index)
        return df

    def get_portfolio_volatility(self, model_id: str, portfolio_id: str) -> pd.DataFrame:
        url = f"{self.base_uri}/api/analysis/volatility_rolling/{model_id}/{portfolio_id}"
        headers = {"Authorization": "ApiKey " + self.api_key}
        resp = requests.get(url, headers=headers, params=self._request_params)

        json_resp = resp.json()
        if (resp.ok and "errors" in json_resp) or not resp.ok:
            error_msg = json_resp["errors"][0]
            if self._is_portfolio_still_running(error_msg):
                return pd.DataFrame()
            logger.error(error_msg)
            raise BoostedAPIException(
                (
                    f"Failed to get portfolio volatility for {portfolio_id=}: "
                    f"{resp.status_code=}; {error_msg=}"
                )
            )

        df = pd.DataFrame(json_resp["data"], columns=json_resp["headerRow"])
        df = df.rename(
            columns={old: old.lower().replace("avg", "avg_") for old in df.columns}
        ).set_index("date")
        df.index = pd.to_datetime(df.index)
        return df

    def get_portfolio_holdings(self, model_id: str, portfolio_id: str) -> pd.DataFrame:
        url = f"{self.base_uri}/api/models/{model_id}/{portfolio_id}/basket-data"
        headers = {"Authorization": "ApiKey " + self.api_key}
        resp = requests.get(url, headers=headers, params=self._request_params)

        # this is a classic abuse of try/except as control flow: we try to get json body
        # from the response so that we can error-check. if this fails, we assume we have
        # a legit text response (corresponding to the csv data we care about)
        try:
            json_resp = resp.json()
        except json.decoder.JSONDecodeError:
            df = pd.read_csv(io.StringIO(resp.text), header=[0])
        else:
            error_msg = json_resp["errors"][0]
            if self._is_portfolio_still_running(error_msg):
                return pd.DataFrame()
            else:
                logger.error(error_msg)
                raise BoostedAPIException(
                    (
                        f"Failed to get portfolio holdings for {portfolio_id=}: "
                        f"{resp.status_code=}; {error_msg=}"
                    )
                )

        df = df.rename(columns={old: old.lower() for old in df.columns}).set_index("date")
        df.index = pd.to_datetime(df.index)
        return df

    def getStockDataTableForDate(
        self, model_id: str, portfolio_id: str, date: datetime.date
    ) -> pd.DataFrame:
        headers = {"Authorization": "ApiKey " + self.api_key, "Content-Type": "application/json"}

        url_base = f"{self.base_uri}/api/analysis"
        url_params = f"{model_id}/{portfolio_id}"
        formatted_date = date.strftime("%Y-%m-%d")

        stock_prices_url = f"{url_base}/stock-prices/{url_params}/{formatted_date}"
        stock_factors_url = f"{url_base}/stock-factors/{url_params}/date/{formatted_date}"

        prices_params = {"useTicker": "true"}

        prices_resp = requests.get(
            stock_prices_url, headers=headers, params=prices_params, **self._request_params
        )
        factors_resp = requests.get(stock_factors_url, headers=headers, **self._request_params)

        frames = []
        for res in (prices_resp, factors_resp):
            if not res.ok:
                error_msg = self._try_extract_error_code(res)
                logger.error(error_msg)
                raise BoostedAPIException(
                    f"Failed to fetch stock data table for model {model_id}: {error_msg=}"
                )
            result = res.json()
            frames.append(pd.DataFrame(result))

        output_df = pd.concat(frames)
        return output_df

    def add_hedge_experiment_scenario(
        self,
        experiment_id: str,
        scenario_name: str,
        scenario_settings: PortfolioSettings,
        run_scenario_immediately: bool,
    ) -> HedgeExperimentScenario:

        add_scenario_input = {
            "hedgeExperimentId": experiment_id,
            "scenarioName": scenario_name,
            "portfolioSettingsJson": str(scenario_settings),
            "runExperimentOnScenario": run_scenario_immediately,
            "createDefaultPortfolio": "false",
        }
        qry = """
            mutation addHedgeExperimentScenario(
                $input: AddHedgeExperimentScenarioInput!
            ) {
                addHedgeExperimentScenario(input: $input) {
                    hedgeExperimentScenario {
                        hedgeExperimentScenarioId
                        scenarioName
                        description
                        portfolioSettingsJson
                    }
                }
            }

        """

        url = f"{self.base_uri}/api/graphql"

        resp = requests.post(
            url,
            headers={"Authorization": "ApiKey " + self.api_key},
            json={"query": qry, "variables": {"input": add_scenario_input}},
        )

        json_resp = resp.json()
        if (resp.ok and "errors" in json_resp) or not resp.ok:
            error_msg = self._try_extract_error_code(resp)
            logger.error(error_msg)
            raise BoostedAPIException(
                (f"Failed to add scenario: {resp.status_code=}; {error_msg=}")
            )

        scenario_dict = json_resp["data"]["addHedgeExperimentScenario"]["hedgeExperimentScenario"]
        if scenario_dict is None:
            raise BoostedAPIException(
                "Failed to add scenario, likely due to bad experiment id or api key"
            )
        s = HedgeExperimentScenario.from_json_dict(scenario_dict)
        return s

    # experiment life cycle has 4 steps:
    # 1. creation - essentially a very simple registration of a new instance, returning an id
    # 2. modify - populate with settings
    # 3. start - run the experiment
    # 4. delete - drop the experiment
    # while i would prefer to just have 2 funcs for (1,2,3) and (4) for a simpler api,
    # we need to expose finer-grained control becuase of how scenarios work.
    def create_hedge_experiment(
        self,
        name: str,
        description: str,
        experiment_type: hedge_experiment_type,
        target_securities: Union[Dict[GbiIdSecurity, float], str],
    ) -> HedgeExperiment:
        # we don't pass target_securities here (as much as id like to) because the
        # graphql input doesn't support it at this point

        # note that this query returns a lot of null fields at this point, but
        # they are necessary for building a HE.
        create_qry = """
            mutation createDraftMutation($input: CreateHedgeExperimentDraftInput!) {
                createHedgeExperimentDraft(input: $input) {
                    hedgeExperiment {
                        hedgeExperimentId
                        experimentName
                        userId
                        config
                        description
                        experimentType
                        lastCalculated
                        lastModified
                        status
                        portfolioCalcStatus
                        targetSecurities {
                            gbiId
                            security {
                                gbiId
                                name
                                symbol
                            }
                            weight
                        }
                        baselineModel {
                            id
                            name
                        }
                        baselineScenario {
                            hedgeExperimentScenarioId
                            scenarioName
                            description
                            portfolioSettingsJson
                            hedgeExperimentPortfolios {
                                portfolio {
                                    id
                                    name
                                    modelId
                                    performanceGridHeader
                                    performanceGrid
                                    status
                                    tearSheet {
                                        groupName
                                        members {
                                            name
                                            value
                                        }
                                    }
                                }
                            }
                            status
                        }
                        baselineStockUniverseId
                    }
                }
            }
        """

        create_input = {"name": name, "experimentType": experiment_type, "description": description}
        if isinstance(target_securities, dict):
            create_input["setTargetSecurities"] = [
                {"gbiId": sec.gbi_id, "weight": weight}
                for (sec, weight) in target_securities.items()
            ]
        elif isinstance(target_securities, str):
            create_input["setTargetPortfolios"] = [{"portfolioId": target_securities}]
        elif target_securities is None:
            pass
        else:
            raise TypeError(
                "Expected value of type Union[Dict[GbiIdSecurity, str], str] for "
                f"argument 'target_securities'; got {type(target_securities)}"
            )
        resp = requests.post(
            f"{self.base_uri}/api/graphql",
            json={"query": create_qry, "variables": {"input": create_input}},
            headers={"Authorization": "ApiKey " + self.api_key},
            params=self._request_params,
        )

        json_resp = resp.json()
        if (resp.ok and "errors" in json_resp) or not resp.ok:
            error_msg = self._try_extract_error_code(resp)
            logger.error(error_msg)
            raise BoostedAPIException(
                (f"Failed to create hedge experiment: {resp.status_code=}; {error_msg=}")
            )

        exp_dict = json_resp["data"]["createHedgeExperimentDraft"]["hedgeExperiment"]
        experiment = HedgeExperiment.from_json_dict(exp_dict)
        return experiment

    def modify_hedge_experiment(
        self,
        experiment_id: str,
        name: Optional[str] = None,
        description: Optional[str] = None,
        experiment_type: Optional[hedge_experiment_type] = None,
        target_securities: Union[Dict[GbiIdSecurity, float], str, None] = None,
        model_ids: Optional[List[str]] = None,
        stock_universe_ids: Optional[List[str]] = None,
        create_default_scenario: bool = True,
        baseline_model_id: Optional[str] = None,
        baseline_stock_universe_id: Optional[str] = None,
        baseline_portfolio_settings: Optional[str] = None,
    ) -> HedgeExperiment:

        mod_qry = """
            mutation modifyHedgeExperimentDraft(
                $input: ModifyHedgeExperimentDraftInput!
            ) {
                modifyHedgeExperimentDraft(input: $input) {
                    hedgeExperiment {
                    ...HedgeExperimentSelectedSecuritiesPageFragment
                    }
                }
            }

            fragment HedgeExperimentSelectedSecuritiesPageFragment on HedgeExperiment {
                hedgeExperimentId
                experimentName
                userId
                config
                description
                experimentType
                lastCalculated
                lastModified
                status
                portfolioCalcStatus
                targetSecurities {
                    gbiId
                    security {
                        gbiId
                        name
                        symbol
                    }
                    weight
                }
                targetPortfolios {
                    portfolioId
                }
                baselineModel {
                    id
                    name
                }
                baselineScenario {
                    hedgeExperimentScenarioId
                    scenarioName
                    description
                    portfolioSettingsJson
                    hedgeExperimentPortfolios {
                        portfolio {
                            id
                            name
                            modelId
                            performanceGridHeader
                            performanceGrid
                            status
                            tearSheet {
                                groupName
                                members {
                                    name
                                    value
                                }
                            }
                        }
                    }
                    status
                }
                baselineStockUniverseId
            }
        """
        mod_input = {
            "hedgeExperimentId": experiment_id,
            "createDefaultScenario": create_default_scenario,
        }
        if name is not None:
            mod_input["newExperimentName"] = name
        if description is not None:
            mod_input["newExperimentDescription"] = description
        if experiment_type is not None:
            mod_input["newExperimentType"] = experiment_type
        if model_ids is not None:
            mod_input["setSelectdModels"] = model_ids
        if stock_universe_ids is not None:
            mod_input["selectedStockUniverseIds"] = stock_universe_ids
        if baseline_model_id is not None:
            mod_input["setBaselineModel"] = baseline_model_id
        if baseline_stock_universe_id is not None:
            mod_input["setBaselineStockUniverse"] = baseline_stock_universe_id
        if baseline_portfolio_settings is not None:
            mod_input["setBaselinePortfolioSettings"] = baseline_portfolio_settings
        # note that the behaviors bound to these data are mutually exclusive,
        # and its possible the opposite was set earlier in the DRAFT phase
        # of experiment creation, so when setting one, we must unset the other
        if isinstance(target_securities, dict):
            mod_input["setTargetSecurities"] = [
                {"gbiId": sec.gbi_id, "weight": weight}
                for (sec, weight) in target_securities.items()
            ]
            mod_input["setTargetPortfolios"] = None
        elif isinstance(target_securities, str):
            mod_input["setTargetPortfolios"] = [{"portfolioId": target_securities}]
            mod_input["setTargetSecurities"] = None
        elif target_securities is None:
            pass
        else:
            raise TypeError(
                "Expected value of type Union[Dict[GbiIdSecurity, str], str] "
                f"for argument 'target_securities'; got {type(target_securities)}"
            )

        resp = requests.post(
            f"{self.base_uri}/api/graphql",
            json={"query": mod_qry, "variables": {"input": mod_input}},
            headers={"Authorization": "ApiKey " + self.api_key},
            params=self._request_params,
        )

        json_resp = resp.json()
        if (resp.ok and "errors" in json_resp) or not resp.ok:
            error_msg = self._try_extract_error_code(resp)
            logger.error(error_msg)
            raise BoostedAPIException(
                (
                    f"Failed to modify hedge experiment in preparation for start {experiment_id=}: "
                    f"{resp.status_code=}; {error_msg=}"
                )
            )

        exp_dict = json_resp["data"]["modifyHedgeExperimentDraft"]["hedgeExperiment"]
        experiment = HedgeExperiment.from_json_dict(exp_dict)
        return experiment

    def start_hedge_experiment(self, experiment_id: str, *scenario_ids: str) -> HedgeExperiment:
        start_qry = """
            mutation startHedgeExperiment($input: StartHedgeExperimentInput!) {
                startHedgeExperiment(input: $input) {
                    hedgeExperiment {
                        hedgeExperimentId
                        experimentName
                        userId
                        config
                        description
                        experimentType
                        lastCalculated
                        lastModified
                        status
                        portfolioCalcStatus
                        targetSecurities {
                            gbiId
                            security {
                                gbiId
                                name
                                symbol
                            }
                            weight
                        }
                        targetPortfolios {
                            portfolioId
                        }
                        baselineModel {
                            id
                            name
                        }
                        baselineScenario {
                            hedgeExperimentScenarioId
                            scenarioName
                            description
                            portfolioSettingsJson
                            hedgeExperimentPortfolios {
                                portfolio {
                                    id
                                    name
                                    modelId
                                    performanceGridHeader
                                    performanceGrid
                                    status
                                    tearSheet {
                                        groupName
                                        members {
                                            name
                                            value
                                        }
                                    }
                                }
                            }
                            status
                        }
                        baselineStockUniverseId
                    }
                }
            }
        """
        start_input = {"hedgeExperimentId": experiment_id}
        if len(scenario_ids) > 0:
            start_input["hedgeExperimentScenarioIds"] = list(scenario_ids)

        resp = requests.post(
            f"{self.base_uri}/api/graphql",
            json={"query": start_qry, "variables": {"input": start_input}},
            headers={"Authorization": "ApiKey " + self.api_key},
            params=self._request_params,
        )

        json_resp = resp.json()
        if (resp.ok and "errors" in json_resp) or not resp.ok:
            error_msg = self._try_extract_error_code(resp)
            logger.error(error_msg)
            raise BoostedAPIException(
                (
                    f"Failed to start hedge experiment {experiment_id=}: "
                    f"{resp.status_code=}; {error_msg=}"
                )
            )

        exp_dict = json_resp["data"]["startHedgeExperiment"]["hedgeExperiment"]
        experiment = HedgeExperiment.from_json_dict(exp_dict)
        return experiment

    def delete_hedge_experiment(self, experiment_id: str) -> bool:
        delete_qry = """
            mutation($input: DeleteHedgeExperimentsInput!) {
                deleteHedgeExperiments(input: $input) {
                    success
                }
            }
        """
        delete_input = {"hedgeExperimentIds": [experiment_id]}
        resp = requests.post(
            f"{self.base_uri}/api/graphql",
            json={"query": delete_qry, "variables": {"input": delete_input}},
            headers={"Authorization": "ApiKey " + self.api_key},
            params=self._request_params,
        )

        json_resp = resp.json()
        if (resp.ok and "errors" in json_resp) or not resp.ok:
            error_msg = self._try_extract_error_code(resp)
            logger.error(error_msg)
            raise BoostedAPIException(
                (
                    f"Failed to delete hedge experiment {experiment_id=}: "
                    + f"status_code={resp.status_code}; error_msg={error_msg}"
                )
            )

        return json_resp["data"]["deleteHedgeExperiments"]["success"]

    def get_portfolio_accuracy(self, model_id: str, portfolio_id: str) -> dict:
        # TODO: Later change this URI to not use the watchlist prefix. It is misnamed.
        url = f"{self.base_uri}{WATCHLIST_ROUTE_PREFIX}{DAL_PA_ROUTE}/get-hit-rate/"
        headers = {"Authorization": "ApiKey " + self.api_key, "Content-Type": "application/json"}
        req_json = {"model_id": model_id, "portfolio_id": portfolio_id}
        res = requests.post(url, json=req_json, headers=headers, **self._request_params)

        if not res.ok:
            error_msg = self._try_extract_error_code(res)
            logger.error(error_msg)
            raise BoostedAPIException(f"Failed to get Hit Rate: {error_msg}")

        data = res.json()
        return data

    def create_watchlist(self, name: str) -> str:
        url = f"{self.base_uri}{WATCHLIST_ROUTE_PREFIX}{DAL_WATCHLIST_ROUTE}/create/"
        headers = {"Authorization": "ApiKey " + self.api_key, "Content-Type": "application/json"}
        req_json = {"name": name}
        res = requests.post(url, json=req_json, headers=headers, **self._request_params)

        if not res.ok:
            error_msg = self._try_extract_error_code(res)
            logger.error(error_msg)
            raise BoostedAPIException(f"Failed to get user models: {error_msg}")

        data = res.json()
        return data["watchlist_id"]

    def _get_graphql(
        self, query: str, variables: Dict, error_msg_prefix: str = "Failed to get graphql result: "
    ) -> Dict:
        headers = {"Authorization": "ApiKey " + self.api_key}
        json_req = {"query": query, "variables": variables}

        url = self.base_uri + "/api/graphql"
        resp = requests.post(
            url,
            json=json_req,
            headers=headers,
            params=self._request_params,
        )

        # graphql endpoints typically return 200 or 400 status codes, so we must
        # check if we have any errors, even with a 200
        if not resp.ok or (resp.ok and "errors" in resp.json()):
            error_msg = self._try_extract_error_code(resp)
            error_str = str(error_msg_prefix) + f" {resp.status_code=}; {error_msg=}"
            logger.error(error_str)
            raise BoostedAPIException(error_str)

        json_resp = resp.json()
        return json_resp

    def _get_security_info(self, gbi_ids: List[int]) -> Dict:
        query = GET_SEC_INFO_QRY
        variables = {
            "ids": [] if not gbi_ids else gbi_ids,
        }

        error_msg_prefix = "Failed to get Security Details:"
        return self._get_graphql(
            query=query, variables=variables, error_msg_prefix=error_msg_prefix
        )

    def _get_sector_info(self) -> Dict:
        """
        Returns a list of sector objects, e.g.
        {
            "id": 1010,
            "parentId": 10,
            "name": "Energy",
            "topParentName": null,
            "spiqSectorId": -1,
            "legacy": false
        }
        """
        url = f"{self.base_uri}/api/sectors"
        headers = {"Authorization": "ApiKey " + self.api_key}
        res = requests.get(url, headers=headers, **self._request_params)
        self._check_ok_or_err_with_msg(res, "Failed to get sectors data")
        return res.json()["sectors"]

    def _get_watchlist_analysis(
        self,
        gbi_ids: List[int],
        model_ids: List[str],
        portfolio_ids: List[str],
        asof_date=datetime.date.today(),
    ) -> Dict:
        query = WATCHLIST_ANALYSIS_QRY
        variables = {
            "gbiIds": gbi_ids,
            "modelIds": model_ids,
            "portfolioIds": portfolio_ids,
            "date": self.__iso_format(asof_date),
        }
        error_msg_prefix = "Failed to get Coverage Analysis:"
        return self._get_graphql(
            query=query, variables=variables, error_msg_prefix=error_msg_prefix
        )

    def _get_models_for_portfolio(self, portfolio_ids: List[str]) -> Dict:
        query = GET_MODELS_FOR_PORTFOLIOS_QRY
        variables = {"ids": portfolio_ids}
        error_msg_prefix = "Failed to get Models for Portfolios: "
        return self._get_graphql(
            query=query, variables=variables, error_msg_prefix=error_msg_prefix
        )

    def _get_excess_return(
        self, model_ids: List[str], gbi_ids: List[int], asof_date=datetime.date.today()
    ) -> Dict:
        query = GET_EXCESS_RETURN_QRY

        variables = {
            "modelIds": model_ids,
            "gbiIds": gbi_ids,
            "date": self.__iso_format(asof_date),
        }
        error_msg_prefix = "Failed to get Excess Return Slugging Pct: "
        return self._get_graphql(
            query=query, variables=variables, error_msg_prefix=error_msg_prefix
        )

    def _coverage_column_name_format(self, in_str) -> str:
        if in_str.upper() == "ISIN":
            return "ISIN"

        return in_str.title()

    def get_coverage_info(self, watchlist_id: str, portfolio_group_id: str) -> pd.DataFrame:

        # get securities list in watchlist
        watchlist_details = self.get_watchlist_details(watchlist_id)
        security_list = watchlist_details["targets"]

        gbi_ids = [x["gbi_id"] for x in security_list]

        gbi_data = {x: {} for x in gbi_ids}

        # get security info ticker, name, industry etc
        sec_info = self._get_security_info(gbi_ids)

        for sec in sec_info["data"]["securities"]:
            gbi_id = sec["gbiId"]
            for k in ["symbol", "name", "isin", "country", "currency"]:
                gbi_data[gbi_id][self._coverage_column_name_format(k)] = sec[k]

            gbi_data[gbi_id][self._coverage_column_name_format("Sector")] = sec["sector"][
                "topParentName"
            ]

        # get portfolios list in portfolio_Group
        portfolio_group = self.get_portfolio_group(portfolio_group_id)
        portfolio_ids = [x["portfolio_id"] for x in portfolio_group["portfolios"]]
        portfolio_info = {x["portfolio_id"]: x for x in portfolio_group["portfolios"]}

        model_resp = self._get_models_for_portfolio(portfolio_ids=portfolio_ids)
        for portfolio in model_resp["data"]["portfolios"]:
            portfolio_info[portfolio["id"]].update(portfolio)

        model_info = {
            x["modelId"]: portfolio_info[x["id"]] for x in model_resp["data"]["portfolios"]
        }

        # model_ids and portfolio_ids are parallel arrays
        model_ids = [portfolio_info[x]["modelId"] for x in portfolio_ids]

        # graphql: get watchlist analysis
        wl_analysis = self._get_watchlist_analysis(
            gbi_ids=gbi_ids,
            model_ids=model_ids,
            portfolio_ids=portfolio_ids,
            asof_date=datetime.date.today(),
        )

        portfolio_gbi_data = {k: {} for k in portfolio_ids}
        for pi, v in portfolio_gbi_data.items():
            v.update({k: {} for k in gbi_data.keys()})

        equity_explorer_date = wl_analysis["data"]["watchlistAnalysis"][0]["analysisDates"][0][
            "date"
        ]
        for wla in wl_analysis["data"]["watchlistAnalysis"]:
            gbi_id = wla["gbiId"]
            gbi_data[gbi_id]["Composite Rating"] = wla["analysisDates"][0]["aggregateSignal"][
                "rating"
            ]
            gbi_data[gbi_id]["Composite Rating Delta"] = wla["analysisDates"][0]["aggregateSignal"][
                "ratingDelta"
            ]

            for p in wla["analysisDates"][0]["portfoliosSignals"]:
                model_name = portfolio_info[p["portfolioId"]]["modelName"]

                portfolio_gbi_data[p["portfolioId"]][gbi_id][
                    model_name + self._coverage_column_name_format(": rank")
                ] = (p["rank"] + 1)
                portfolio_gbi_data[p["portfolioId"]][gbi_id][
                    model_name + self._coverage_column_name_format(": rank delta")
                ] = (-1 * p["signalDelta"])
                portfolio_gbi_data[p["portfolioId"]][gbi_id][
                    model_name + self._coverage_column_name_format(": rating")
                ] = p["rating"]
                portfolio_gbi_data[p["portfolioId"]][gbi_id][
                    model_name + self._coverage_column_name_format(": rating delta")
                ] = p["ratingDelta"]

        neg_rec = {k: {} for k in gbi_data.keys()}
        pos_rec = {k: {} for k in gbi_data.keys()}
        for wla in wl_analysis["data"]["watchlistAnalysis"]:
            gbi_id = wla["gbiId"]

            for pid, signals in zip(portfolio_ids, wla["analysisDates"][0]["portfoliosSignals"]):
                model_name = portfolio_info[pid]["modelName"]
                neg_rec[gbi_id][
                    model_name + self._coverage_column_name_format(": negative recommendation")
                ] = signals["explainWeightNeg"]
                pos_rec[gbi_id][
                    model_name + self._coverage_column_name_format(": positive recommendation")
                ] = signals["explainWeightPos"]

        # graphql: GetExcessReturn - slugging pct
        er_sp = self._get_excess_return(
            model_ids=model_ids, gbi_ids=gbi_ids, asof_date=equity_explorer_date
        )

        for model in er_sp["data"]["models"]:
            model_name = model_info[model["id"]]["modelName"]
            for stat in model["equityExplorerData"]["equityExplorerSummaryStatistics"]:
                portfolioId = model_info[model["id"]]["id"]
                portfolio_gbi_data[portfolioId][int(stat["gbiId"])][
                    model_name + self._coverage_column_name_format(": slugging %")
                ] = (stat["ER"]["SP"]["oneMonth"] * 100)

        # add rank, rating, slugging
        for pid, v in portfolio_gbi_data.items():
            for gbi_id, vv in v.items():
                gbi_data[gbi_id].update(vv)

        # add neg/pos rec scores
        for rec in [neg_rec, pos_rec]:
            for k, v in rec.items():
                gbi_data[k].update(v)

        df = pd.DataFrame.from_records([v for _, v in gbi_data.items()])

        return df

    def get_coverage_csv(
        self, watchlist_id: str, portfolio_group_id: str, filepath: Optional[str] = None
    ) -> Optional[str]:
        """
        Converts the coverage contents to CSV format
        Parameters
        ----------
        watchlist_id: str
            UUID str identifying the coverage watchlist
        portfolio_group_id: str
            UUID str identifying the group of portfolio to use for analysis
        filepath: Optional[str]
            UUID str identifying the group of portfolio to use for analysis

        Returns:
        ----------
        None if filepath is provided, else a string with a csv's contents is returned
        """

        df = self.get_coverage_info(watchlist_id, portfolio_group_id)

        return df.to_csv(filepath, index=False, float_format="%.4f")

    def get_watchlist_details(self, watchlist_id: str) -> Dict:

        url = f"{self.base_uri}{ROUTE_PREFIX}{DAL_WATCHLIST_ROUTE}/details/"
        headers = {"Authorization": "ApiKey " + self.api_key, "Content-Type": "application/json"}
        req_json = {"watchlist_id": watchlist_id}
        res = requests.post(url, json=req_json, headers=headers, **self._request_params)

        if not res.ok:
            error_msg = self._try_extract_error_code(res)
            logger.error(error_msg)
            raise BoostedAPIException(f"Failed to get user models: {error_msg}")

        data = res.json()
        return data

    def create_watchlist_from_file(self, name: str, filepath: str) -> str:

        url = f"{self.base_uri}{ROUTE_PREFIX}{DAL_WATCHLIST_ROUTE}/create_watchlist_from_file/"
        headers = {"Authorization": "ApiKey " + self.api_key}

        with open(filepath, "rb") as fp:
            file_bytes = fp.read()

        file_bytes_base64 = base64.b64encode(file_bytes).decode("ascii")
        json_req = {
            "content_type": mimetypes.guess_type(filepath)[0],
            "file_bytes_base64": file_bytes_base64,
            "name": name,
        }

        res = requests.post(url, json=json_req, headers=headers)

        if not res.ok:
            error_msg = self._try_extract_error_code(res)
            logger.error(error_msg)
            raise BoostedAPIException(f"Failed to create watchlist from file: {error_msg}")

        data = res.json()
        return data["watchlist_id"]

    def get_watchlists(self) -> List[Dict]:

        url = f"{self.base_uri}{ROUTE_PREFIX}{DAL_WATCHLIST_ROUTE}/get_user_watchlists/"
        headers = {"Authorization": "ApiKey " + self.api_key, "Content-Type": "application/json"}
        req_json = {}
        res = requests.post(url, json=req_json, headers=headers, **self._request_params)

        if not res.ok:
            error_msg = self._try_extract_error_code(res)
            logger.error(error_msg)
            raise BoostedAPIException(f"Failed to get user watchlists: {error_msg}")

        data = res.json()
        return data["watchlists"]

    def get_watchlist_contents(self, watchlist_id) -> Dict:

        url = f"{self.base_uri}{ROUTE_PREFIX}{DAL_WATCHLIST_ROUTE}/contents/"
        headers = {"Authorization": "ApiKey " + self.api_key, "Content-Type": "application/json"}
        req_json = {"watchlist_id": watchlist_id}
        res = requests.post(url, json=req_json, headers=headers, **self._request_params)

        if not res.ok:
            error_msg = self._try_extract_error_code(res)
            logger.error(error_msg)
            raise BoostedAPIException(f"Failed to get watchlist contents: {error_msg}")

        data = res.json()
        return data

    def get_watchlist_contents_as_csv(self, watchlist_id, filepath) -> None:
        data = self.get_watchlist_contents(watchlist_id)
        df = pd.DataFrame(data["contents"])
        df.to_csv(filepath, index=False)

    # TODO this will need to be enhanced to accept country/currency overrides
    def add_securities_to_watchlist(
        self, watchlist_id: str, identifiers: List[str], identifier_type: Literal["TICKER", "ISIN"]
    ) -> Dict:

        # should we just make the arg lower? all caps has a flag-like feel to it
        id_type = identifier_type.lower()
        url = f"{self.base_uri}{WATCHLIST_ROUTE_PREFIX}{DAL_WATCHLIST_ROUTE}/add_{id_type}s/"
        headers = {"Authorization": "ApiKey " + self.api_key, "Content-Type": "application/json"}
        req_json = {"watchlist_id": watchlist_id, id_type: identifiers}
        res = requests.post(url, json=req_json, headers=headers, **self._request_params)

        if not res.ok:
            error_msg = self._try_extract_error_code(res)
            logger.error(error_msg)
            raise BoostedAPIException(f"Failed to get user models: {error_msg}")

        data = res.json()
        return data

    def remove_securities_from_watchlist(
        self, watchlist_id: str, identifiers: List[str], identifier_type: Literal["TICKER", "ISIN"]
    ) -> Dict:
        # should we just make the arg lower? all caps has a flag-like feel to it
        id_type = identifier_type.lower()
        url = f"{self.base_uri}{WATCHLIST_ROUTE_PREFIX}{DAL_WATCHLIST_ROUTE}/remove_{id_type}s/"
        headers = {"Authorization": "ApiKey " + self.api_key, "Content-Type": "application/json"}
        req_json = {"watchlist_id": watchlist_id, id_type: identifiers}
        res = requests.post(url, json=req_json, headers=headers, **self._request_params)

        if not res.ok:
            error_msg = self._try_extract_error_code(res)
            logger.error(error_msg)
            raise BoostedAPIException(f"Failed to get user models: {error_msg}")

        data = res.json()
        return data

    def get_portfolio_groups(
        self,
    ) -> Dict:
        """
        Parameters: None


        Returns:
        ----------

        Dict:  {
        user_id: str
        portfolio_groups: List[PortfolioGroup]
        }
        where PortfolioGroup is defined as = Dict {
        group_id: str
        group_name: str
        portfolios: List[PortfolioInGroup]
        }
        where PortfolioInGroup is defined as = Dict {
        portfolio_id: str
        rank_in_group: Optional[int]
        }
        """
        url = f"{self.base_uri}{WATCHLIST_ROUTE_PREFIX}{PORTFOLIO_GROUP_ROUTE}/get"
        headers = {"Authorization": "ApiKey " + self.api_key, "Content-Type": "application/json"}
        req_json = {}
        res = requests.post(url, json=req_json, headers=headers, **self._request_params)

        if not res.ok:
            error_msg = self._try_extract_error_code(res)
            logger.error(error_msg)
            raise BoostedAPIException(f"Failed to get user portfolio groups: {error_msg}")

        data = res.json()
        return data

    def get_portfolio_group(self, portfolio_group_id: str) -> Dict:
        """
        Parameters:
        portfolio_group_id: str
           UUID identifier for the portfolio group


        Returns:
        ----------

        PortfolioGroup: Dict:  {
        group_id: str
        group_name: str
        portfolios: List[PortfolioInGroup]
        }
        where PortfolioInGroup is defined as = Dict {
        portfolio_id: str
        portfolio_name: str
        rank_in_group: Optional[int]
        }
        """
        url = f"{self.base_uri}{WATCHLIST_ROUTE_PREFIX}{PORTFOLIO_GROUP_ROUTE}/get-one"
        headers = {"Authorization": "ApiKey " + self.api_key, "Content-Type": "application/json"}
        req_json = {"portfolio_group_id": portfolio_group_id}
        res = requests.post(url, json=req_json, headers=headers, **self._request_params)

        if not res.ok:
            error_msg = self._try_extract_error_code(res)
            logger.error(error_msg)
            raise BoostedAPIException(f"Failed to get user portfolio groups: {error_msg}")

        data = res.json()
        return data

    def set_sticky_portfolio_group(
        self,
        portfolio_group_id: str,
    ) -> Dict:
        """
        Set sticky portfolio group

        Parameters
        ----------

        group_id: str,
           UUID str identifying a portfolio group

        Returns:
        -------
        Dict {
            changed: int - 1 == success
        }
        """
        url = f"{self.base_uri}{WATCHLIST_ROUTE_PREFIX}{PORTFOLIO_GROUP_ROUTE}/set-sticky"
        headers = {"Authorization": "ApiKey " + self.api_key, "Content-Type": "application/json"}
        req_json = {"portfolio_group_id": portfolio_group_id}
        res = requests.post(url, json=req_json, headers=headers, **self._request_params)

        if not res.ok:
            error_msg = self._try_extract_error_code(res)
            logger.error(error_msg)
            raise BoostedAPIException(f"Failed to set sticky portfolio group: {error_msg}")

        data = res.json()
        return data

    def get_sticky_portfolio_group(
        self,
    ) -> Dict:
        """
        Get sticky portfolio group for the user

        Parameters
        ----------

        Returns:
        -------
        Dict {
            group_id: str
            group_name: str
            portfolios: List[PortfolioInGroup(Dict)]
                  PortfolioInGroup(Dict):
                           portfolio_id: str
                           rank_in_group: Optional[int] = None
                           portfolio_name: Optional[str] = None
        }
        """
        url = f"{self.base_uri}{WATCHLIST_ROUTE_PREFIX}{PORTFOLIO_GROUP_ROUTE}/get-sticky"
        headers = {"Authorization": "ApiKey " + self.api_key, "Content-Type": "application/json"}
        req_json = {}
        res = requests.post(url, json=req_json, headers=headers, **self._request_params)

        if not res.ok:
            error_msg = self._try_extract_error_code(res)
            logger.error(error_msg)
            raise BoostedAPIException(f"Failed to get sticky portfolio group: {error_msg}")

        data = res.json()
        return data

    def create_portfolio_group(
        self,
        group_name: str,
        portfolios: Optional[List[Dict]] = None,
    ) -> Dict:
        """
        Create a new portfolio group

        Parameters
        ----------

        group_name: str
           name of the new group

        portfolios: List of Dict [:

        portfolio_id: str
        rank_in_group: Optional[int] = None
        ]

        Returns:
        ----------

        Dict: {
        group_id: str
           UUID identifier for the portfolio group

        created: int
           num groups created, 1 == success

        added: int
           num portfolios added to the group, should match the length of 'portfolios' argument
        }
        """
        url = f"{self.base_uri}{WATCHLIST_ROUTE_PREFIX}{PORTFOLIO_GROUP_ROUTE}/create"
        headers = {"Authorization": "ApiKey " + self.api_key, "Content-Type": "application/json"}
        req_json = {"group_name": group_name, "portfolios": portfolios}

        res = requests.post(url, json=req_json, headers=headers, **self._request_params)

        if not res.ok:
            error_msg = self._try_extract_error_code(res)
            logger.error(error_msg)
            raise BoostedAPIException(f"Failed to create portfolio group: {error_msg}")

        data = res.json()
        return data

    def rename_portfolio_group(
        self,
        group_id: str,
        group_name: str,
    ) -> Dict:
        """
        Rename a portfolio group

        Parameters
        ----------

        group_id: str,
           UUID str identifying a portfolio group

        group_name: str,
           The new name for the porfolio

        Returns:
        -------
        Dict {
            changed: int - 1 == success
        }
        """
        url = f"{self.base_uri}{WATCHLIST_ROUTE_PREFIX}{PORTFOLIO_GROUP_ROUTE}/rename"
        headers = {"Authorization": "ApiKey " + self.api_key, "Content-Type": "application/json"}
        req_json = {"group_id": group_id, "group_name": group_name}
        res = requests.post(url, json=req_json, headers=headers, **self._request_params)

        if not res.ok:
            error_msg = self._try_extract_error_code(res)
            logger.error(error_msg)
            raise BoostedAPIException(f"Failed to rename portfolio group: {error_msg}")

        data = res.json()
        return data

    def add_to_portfolio_group(
        self,
        group_id: str,
        portfolios: List[Dict],
    ) -> Dict:
        """
        Add portfolios to a group

        Parameters
        ----------

        group_id: str,
           UUID str identifying a portfolio group

        portfolios: List of Dict [:
            portfolio_id: str
            rank_in_group: Optional[int] = None
        ]


        Returns:
        -------
        Dict {
            added: int
               number of successful changes
        }
        """
        url = f"{self.base_uri}{WATCHLIST_ROUTE_PREFIX}{PORTFOLIO_GROUP_ROUTE}/add-to-group"
        headers = {"Authorization": "ApiKey " + self.api_key, "Content-Type": "application/json"}
        req_json = {"group_id": group_id, "portfolios": portfolios}

        res = requests.post(url, json=req_json, headers=headers, **self._request_params)

        if not res.ok:
            error_msg = self._try_extract_error_code(res)
            logger.error(error_msg)
            raise BoostedAPIException(f"Failed to add portfolios to portfolio group: {error_msg}")

        data = res.json()
        return data

    def remove_from_portfolio_group(
        self,
        group_id: str,
        portfolios: List[str],
    ) -> Dict:
        """
        Remove portfolios from a group

        Parameters
        ----------

        group_id: str,
           UUID str identifying a portfolio group

        portfolios: List of str


        Returns:
        -------
        Dict {
            removed: int
               number of successful changes
        }
        """
        url = f"{self.base_uri}{WATCHLIST_ROUTE_PREFIX}{PORTFOLIO_GROUP_ROUTE}/remove-from-group"
        headers = {"Authorization": "ApiKey " + self.api_key, "Content-Type": "application/json"}
        req_json = {"group_id": group_id, "portfolios": portfolios}
        res = requests.post(url, json=req_json, headers=headers, **self._request_params)

        if not res.ok:
            error_msg = self._try_extract_error_code(res)
            logger.error(error_msg)
            raise BoostedAPIException(
                f"Failed to remove portfolios from portfolio group: {error_msg}"
            )

        data = res.json()
        return data

    def delete_portfolio_group(
        self,
        group_id: str,
    ) -> Dict:
        """
        Delete a portfolio group

        Parameters
        ----------

        group_id: str,
           UUID str identifying a portfolio group


        Returns:
        -------
        Dict {
            removed_groups: int
               number of successful changes

            removed_portfolios: int
               number of successful changes
        }
        """
        url = f"{self.base_uri}{WATCHLIST_ROUTE_PREFIX}{PORTFOLIO_GROUP_ROUTE}/remove"
        headers = {"Authorization": "ApiKey " + self.api_key, "Content-Type": "application/json"}
        req_json = {"group_id": group_id}
        res = requests.post(url, json=req_json, headers=headers, **self._request_params)

        if not res.ok:
            error_msg = self._try_extract_error_code(res)
            logger.error(error_msg)
            raise BoostedAPIException(f"Failed to delete portfolio group: {error_msg}")

        data = res.json()
        return data

    def set_portfolio_group_for_watchlist(
        self,
        portfolio_group_id: str,
        watchlist_id: str,
    ) -> Dict:
        """
        Set portfolio group for watchlist.

        Parameters
        ----------

        portfolio_group_id: str,
           UUID str identifying a portfolio group

        watchlist_id: str,
           UUID str identifying a watchlist


        Returns:
        -------
        Dict {
            success: bool
            errors:
            data: Dict
                changed: int
        }
        """
        url = f"{self.base_uri}{WATCHLIST_ROUTE_PREFIX}{DAL_WATCHLIST_ROUTE}/set-portfolio-groups/"
        headers = {"Authorization": "ApiKey " + self.api_key, "Content-Type": "application/json"}
        req_json = {"portfolio_group_id": portfolio_group_id, "watchlist_id": watchlist_id}
        res = requests.post(url, json=req_json, headers=headers, **self._request_params)

        if not res.ok:
            error_msg = self._try_extract_error_code(res)
            logger.error(error_msg)
            raise BoostedAPIException(f"Failed to set portfolio group for watchlist: {error_msg}")

        return res.json()

    def get_ranking_dates(self, model_id: str, portfolio_id: str) -> List[datetime.date]:
        headers = {"Authorization": "ApiKey " + self.api_key, "Content-Type": "application/json"}
        url = self.base_uri + f"/api/analysis/ranking-dates/{model_id}/{portfolio_id}"
        res = requests.get(url, headers=headers, **self._request_params)
        self._check_ok_or_err_with_msg(res, "Failed to get ranking dates")
        data = res.json().get("ranking_dates", [])

        return [parser.parse(d).date() for d in data]

    def get_prior_ranking_date(
        self, ranking_dates: List[datetime.date], starting_date: datetime.date
    ) -> datetime.date:
        """
        Given a starting date and a list of ranking dates, return the most
        recent previous ranking date.
        """
        # order from most recent to least
        ranking_dates.sort(reverse=True)

        for d in ranking_dates:
            if d <= starting_date:
                return d

        # if we get here, the starting date is before the earliest ranking date
        raise BoostedAPIException(f"No rankins exist on or before {starting_date}")

    def _get_risk_factors_descriptors(
        self, model_id: str, portfolio_id: str, use_v2: bool = False
    ) -> Dict[int, str]:
        """Returns a map from descriptor id to descriptor name."""
        headers = {"Authorization": "ApiKey " + self.api_key, "Content-Type": "application/json"}

        risk_factor = RISK_FACTOR_V2 if use_v2 else RISK_FACTOR
        url = self.base_uri + f"/api/{risk_factor}/{model_id}/{portfolio_id}/descriptors"
        res = requests.get(url, headers=headers, **self._request_params)

        self._check_ok_or_err_with_msg(res, "Failed to get risk factor descriptors")

        descriptors = {int(i): name for i, name in res.json().items() if i.isnumeric()}
        return descriptors

    def get_risk_groups(
        self, model_id: str, portfolio_id: str, date: datetime.date, use_v2: bool = False
    ) -> List[Dict[str, Any]]:
        # first get the group descriptors
        descriptors = self._get_risk_factors_descriptors(model_id, portfolio_id, use_v2)

        # calculate the most recent prior rankings date. This is the date
        # we need to use to query for risk group data.
        ranking_dates = self.get_ranking_dates(model_id, portfolio_id)
        ranking_date = self.get_prior_ranking_date(ranking_dates, date)
        date_str = ranking_date.strftime("%Y-%m-%d")

        risk_factor = RISK_FACTOR_V2 if use_v2 else RISK_FACTOR

        headers = {"Authorization": "ApiKey " + self.api_key, "Content-Type": "application/json"}
        url = self.base_uri + f"/api/{risk_factor}/{model_id}/{portfolio_id}/risk-groups/{date_str}"
        res = requests.get(url, headers=headers, **self._request_params)

        self._check_ok_or_err_with_msg(
            res, f"Failed to get risk factors for {model_id=}, {portfolio_id=}, {date=}"
        )

        # Response is a list of objects like:
        # [
        #   [
        #     0,
        #     14,
        #     1
        #   ],
        #   [
        #     25,
        #     12,
        #     13
        #   ],
        # 0.67013
        # ],
        #
        # Where each integer in the lists is a descriptor id.

        groups = []
        for row in res.json():
            row_map = {}
            # map descriptor id to name
            row_map["risk_group_a"] = [descriptors[i] for i in row[0]]
            row_map["risk_group_b"] = [descriptors[i] for i in row[1]]
            row_map["volatility_explained"] = row[2]
            groups.append(row_map)

        return groups

    def get_risk_factors_discovered_descriptors(
        self, model_id: str, portfolio_id: str, date: datetime.date, use_v2: bool = False
    ) -> pd.DataFrame:
        # first get the group descriptors
        descriptors = self._get_risk_factors_descriptors(model_id, portfolio_id)

        # calculate the most recent prior rankings date. This is the date
        # we need to use to query for risk group data.
        ranking_dates = self.get_ranking_dates(model_id, portfolio_id)
        ranking_date = self.get_prior_ranking_date(ranking_dates, date)
        date_str = ranking_date.strftime("%Y-%m-%d")

        risk_factor = RISK_FACTOR_V2 if use_v2 else RISK_FACTOR

        headers = {"Authorization": "ApiKey " + self.api_key, "Content-Type": "application/json"}
        url = (
            self.base_uri
            + f"/api/{risk_factor}/{model_id}/{portfolio_id}/risk-descriptors/json/{date_str}"
        )
        res = requests.get(url, headers=headers, **self._request_params)

        self._check_ok_or_err_with_msg(
            res, f"Failed to get risk factors for {model_id=}, {portfolio_id=}, {date=}"
        )

        # Endpoint returns a nested list of floats
        df = pd.DataFrame(res.json(), columns=RISK_FACTOR_COLUMNS)

        # This flat dataframe represents a potentially doubly nested structure
        # of Sector -> (high/low volatility) -> security. We don't care about
        # the high/low volatility rows, (which will have negative identifiers)
        # so we can filter these out.
        df = df[df["identifier"] >= 0]

        # now, any values that had a depth of 2 should be set to a depth of 1,
        # since we removed the double nesting.
        df.replace(to_replace=2, value=1, inplace=True)

        # This dataframe represents data that is nested on the UI, so the
        # "depth" field indicates which level of nesting each row is at. At this
        # point, a depth of 0 indicates a sector, and following depth 1 rows are
        # securities within the sector.

        # Identifiers in rows with depth 1 will be gbi ids, need to convert to
        # symbols.
        gbi_ids = df[df["depth"] == 1]["identifier"].tolist()
        sec_info = self._get_security_info(gbi_ids)["data"]["securities"]
        sec_map = {s["gbiId"]: s["symbol"] for s in sec_info}

        def convert_ids(row: pd.Series) -> pd.Series:
            # convert each row's "identifier" to the appropriate id type. If the
            # depth is 0, the identifier should be a sector, otherwise it should
            # be a ticker.
            ident = int(row["identifier"])
            row["identifier"] = (
                descriptors.get(ident).title() if row["depth"] == 0 else sec_map.get(ident)
            )
            return row

        df["depth"] = df["depth"].astype(int)
        df["stock_count"] = df["stock_count"].astype(int)
        df = df.apply(convert_ids, axis=1)
        df = df.reset_index(drop=True)
        return df

    def get_risk_factors_sectors(
        self, model_id: str, portfolio_id: str, date: datetime.date, use_v2: bool = False
    ) -> pd.DataFrame:
        # first get the group descriptors
        sectors = {s["id"]: s["name"] for s in self._get_sector_info()}

        # calculate the most recent prior rankings date. This is the date
        # we need to use to query for risk group data.
        ranking_dates = self.get_ranking_dates(model_id, portfolio_id)
        ranking_date = self.get_prior_ranking_date(ranking_dates, date)
        date_str = ranking_date.strftime("%Y-%m-%d")

        risk_factor = RISK_FACTOR_V2 if use_v2 else RISK_FACTOR

        headers = {"Authorization": "ApiKey " + self.api_key, "Content-Type": "application/json"}
        url = (
            self.base_uri
            + f"/api/{risk_factor}/{model_id}/{portfolio_id}/risk-sectors/json/{date_str}"
        )
        res = requests.get(url, headers=headers, **self._request_params)

        self._check_ok_or_err_with_msg(
            res, f"Failed to get risk factors for {model_id=}, {portfolio_id=}, {date=}"
        )

        # Endpoint returns a nested list of floats
        df = pd.DataFrame(res.json(), columns=RISK_FACTOR_COLUMNS)

        # identifier is a gics sector identifier
        df["identifier"] = df["identifier"].apply(lambda i: sectors.get(int(i)))

        # This dataframe represents data that is nested on the UI, so the
        # "depth" field indicates which level of nesting each row is at. For
        # risk factors sectors, each "depth" represents a level of specificity
        # for the sector. E.g. Energy -> Energy Equipment -> Oil & Gas Equipment
        df["depth"] = df["depth"].astype(int)
        df["stock_count"] = df["stock_count"].astype(int)
        df = df.reset_index(drop=True)
        return df

    def download_complete_portfolio_data(
        self, model_id: str, portfolio_id: str, download_filepath: str
    ):
        headers = {"Authorization": "ApiKey " + self.api_key, "Content-Type": "application/json"}
        url = self.base_uri + f"/api/models/{model_id}/{portfolio_id}/excel"

        res = requests.get(url, headers=headers, **self._request_params)
        self._check_ok_or_err_with_msg(
            res, f"Failed to get full data for {model_id=}, {portfolio_id=}"
        )

        with open(download_filepath, "wb") as f:
            f.write(res.content)

    def diff_hedge_experiment_portfolio_data(
        self,
        hedge_experiment_id: str,
        comparison_portfolios: List[str],
        categories: List[str],
    ) -> Dict:
        qry = """
        query diffHedgeExperimentPortfolios(
            $input: DiffHedgeExperimentPortfoliosInput!
        ) {
            diffHedgeExperimentPortfolios(input: $input) {
            data {
                diffs {
                    volatility {
                        date
                        vol5D
                        vol10D
                        vol21D
                        vol21D
                        vol63D
                        vol126D
                        vol189D
                        vol252D
                        vol315D
                        vol378D
                        vol441D
                        vol504D
                    }
                    performance {
                        date
                        value
                    }
                    performanceGrid {
                        headerRow
                        values
                    }
                    factors {
                        date
                        momentum
                        growth
                        size
                        value
                        dividendYield
                        volatility
                    }
                }
            }
            errors
            }
        }
        """
        headers = {"Authorization": "ApiKey " + self.api_key}
        params = {
            "hedgeExperimentId": hedge_experiment_id,
            "portfolioIds": comparison_portfolios,
            "categories": categories,
        }
        resp = requests.post(
            f"{self.base_uri}/api/graphql",
            json={"query": qry, "variables": params},
            headers=headers,
            params=self._request_params,
        )

        json_resp = resp.json()

        # graphql endpoints typically return 200 or 400 status codes, so we must
        # check if we have any errors, even with a 200
        if (resp.ok and "errors" in json_resp) or not resp.ok:
            error_msg = self._try_extract_error_code(resp)
            logger.error(error_msg)
            raise BoostedAPIException(
                (
                    f"Failed to get portfolio diffs for {hedge_experiment_id=}: "
                    f"{resp.status_code=}; {error_msg=}"
                )
            )

        diffs = json_resp["data"]["diffHedgeExperimentPortfolios"]["data"]["diffs"]
        comparisons = {}
        for pf, cmp in zip(comparison_portfolios, diffs):
            res = {
                "performance": None,
                "performanceGrid": None,
                "factors": None,
                "volatility": None,
            }
            if "performanceGrid" in cmp:
                grid = cmp["performanceGrid"]
                grid_df = pd.DataFrame(grid["values"], columns=grid["headerRow"])
                res["performanceGrid"] = grid_df
            if "performance" in cmp:
                perf_df = pd.DataFrame(cmp["performance"]).set_index("date")
                perf_df.index = pd.to_datetime(perf_df.index)
                res["performance"] = perf_df
            if "volatility" in cmp:
                vol_df = pd.DataFrame(cmp["volatility"]).set_index("date")
                vol_df.index = pd.to_datetime(vol_df.index)
                res["volatility"] = vol_df
            if "factors" in cmp:
                factors_df = pd.DataFrame(cmp["factors"]).set_index("date")
                factors_df.index = pd.to_datetime(factors_df.index)
                res["factors"] = factors_df
            comparisons[pf] = res
        return comparisons

    def get_signal_strength(self, model_id: str, portfolio_id: str) -> pd.DataFrame:
        url = self.base_uri + f"/api/analysis/signal_strength/{model_id}/{portfolio_id}"
        headers = {"Authorization": "ApiKey " + self.api_key}

        logger.info(f"Retrieving portfolio signals for {model_id=}, {portfolio_id=}")

        # Response format is a json object with a "header_row" key for column
        # names, and then a nested list of data.
        resp = requests.get(url, headers=headers, **self._request_params)
        self._check_ok_or_err_with_msg(
            resp, f"Failed to get portfolio signals for {model_id=}, {portfolio_id=}"
        )

        data = resp.json()

        df = pd.DataFrame(data=data["data"], columns=data["header_row"])
        df["Date"] = pd.to_datetime(df["Date"])
        df = df.set_index("Date")
        return df.astype(float)

    def get_rolling_signal_strength(self, model_id: str, portfolio_id: str) -> pd.DataFrame:
        url = self.base_uri + f"/api/analysis/signal_strength_rolling/{model_id}/{portfolio_id}"
        headers = {"Authorization": "ApiKey " + self.api_key}

        logger.info(f"Retrieving rolling portfolio signals for {model_id=}, {portfolio_id=}")

        # Response format is a json object with a "header_row" key for column
        # names, and then a nested list of data.
        resp = requests.get(url, headers=headers, **self._request_params)
        self._check_ok_or_err_with_msg(
            resp, f"Failed to get rolling portfolio signals for {model_id=}, {portfolio_id=}"
        )

        data = resp.json()

        df = pd.DataFrame(data=data["data"], columns=data["header_row"])
        df["Date"] = pd.to_datetime(df["Date"])
        df = df.set_index("Date")
        return df.astype(float)

    def get_portfolio_quantiles(
        self,
        model_id: str,
        portfolio_id: str,
        id_type: Literal["TICKER", "ISIN"] = "TICKER",
    ):
        headers = {"Authorization": "ApiKey " + self.api_key, "Content-Type": "application/json"}
        date = datetime.date.today().strftime("%Y-%m-%d")

        payload = {
            "model_id": model_id,
            "portfolio_id": portfolio_id,
            "fields": ["quantile"],
            "min_date": date,
            "max_date": date,
            "return_format": "json",
        }
        # TODO: Later change this URI to not use the watchlist prefix. It is misnamed.
        url = f"{self.base_uri}{WATCHLIST_ROUTE_PREFIX}{DAL_PA_ROUTE}/get-data/"

        resp = requests.post(url, json=payload, headers=headers, **self._request_params)
        self._check_ok_or_err_with_msg(resp, "Unable to get quantile data")

        resp = resp.json()
        quantile_index = resp["field_map"]["Quantile"]
        quantile_data = [[c[quantile_index] for c in r] for r in resp["data"]]
        date_cols = pd.to_datetime(resp["columns"])

        # Need to map gbi id's to isins or tickers
        gbi_ids = [int(i) for i in resp["rows"]]
        security_info = self._get_security_info(gbi_ids)

        # We now have security data, go through and create a map from internal
        # gbi id to client facing identifier
        id_key = "isin" if id_type == "ISIN" else "symbol"
        gbi_identifier_map = {
            sec["gbiId"]: sec[id_key] for sec in security_info["data"]["securities"]
        }

        df = pd.DataFrame(quantile_data, index=gbi_ids, columns=date_cols).transpose()
        df = df.rename(columns=gbi_identifier_map)
        return df

Methods

def abort_chunked_upload(self, dataset_id, chunk_id)
Expand source code
def abort_chunked_upload(self, dataset_id, chunk_id):
    url = self.base_uri + "/api/datasets/{0}/abort-chunked-upload".format(dataset_id)
    headers = {"Authorization": "ApiKey " + self.api_key}
    params = {"uploadGroupId": chunk_id}
    res = requests.post(url, headers=headers, **self._request_params, params=params)
    if not res.ok:
        error_msg = self._try_extract_error_code(res)
        logger.error(error_msg)
        raise BoostedAPIException(
            "Failed to abort dataset lock during error: {0}.".format(error_msg)
        )
def addSignalsToUploadedModel(self, model_id, csv_data, timeout=600)
Expand source code
def addSignalsToUploadedModel(self, model_id, csv_data, timeout=600):
    warnings = self.addToUploadedModel(model_id, csv_data)
    return warnings
def addToUploadedModel(self, model_id, csv_data, timeout=600)
Expand source code
def addToUploadedModel(self, model_id, csv_data, timeout=600):
    warnings = []
    url = self.base_uri + "/api/models/{0}/upload/add-data".format(model_id)
    request_data = {}
    _, warnings = self.add_uploaded_model_data(url, csv_data, request_data, timeout)
    self.sendModelRecalc(model_id)
    return warnings
def add_dependent_data(self, dataset_id, csv_data, timeout=600, block=True, data_type=HISTORICAL, no_exception_on_chunk_error=False)
Expand source code
def add_dependent_data(
    self,
    dataset_id,
    csv_data,
    timeout=600,
    block=True,
    data_type=DataAddType.HISTORICAL,
    no_exception_on_chunk_error=False,
):
    warnings = []
    query_info = self.query_dataset(dataset_id)
    if DataSetType[query_info["type"]] != DataSetType.STOCK:
        raise BoostedAPIException(
            f"Incorrect dataset type: {query_info['type']}" f" - Expected {DataSetType.STOCK}"
        )
    warnings, errors = self.setup_chunk_and_upload_data(
        dataset_id, csv_data, data_type, timeout, block, no_exception_on_chunk_error
    )
    if len(warnings) > 0:
        logger.warning(
            "Encountered {0} total warnings while uploading dataset.".format(len(warnings))
        )
    if len(errors) > 0:
        raise BoostedAPIException(
            "Encountered {0} total ERRORS while uploading dataset".format(len(errors))
            + "\n".join(errors)
        )
    return {"warnings": warnings, "errors": errors}
def add_dependent_dataset(self, dataset, datasetName='DependentDataset', schema=None, timeout=600, block=True)
Expand source code
def add_dependent_dataset(
    self, dataset, datasetName="DependentDataset", schema=None, timeout=600, block=True
):
    result = self.add_dependent_dataset_with_warnings(
        dataset, datasetName, schema, timeout, block
    )
    return result["dataset_id"]
def add_dependent_dataset_with_warnings(self, dataset, datasetName='DependentDataset', schema=None, timeout=600, block=True, no_exception_on_chunk_error=False)
Expand source code
def add_dependent_dataset_with_warnings(
    self,
    dataset,
    datasetName="DependentDataset",
    schema=None,
    timeout=600,
    block=True,
    no_exception_on_chunk_error=False,
):
    if not self.validate_dataframe(dataset):
        logger.error("dataset failed validation.")
        return None
    if schema is None:
        schema = infer_dataset_schema(datasetName, dataset, DataSetType.STOCK)
    dsid = self.createDataset(schema.toDict())
    logger.info("Creating dataset with ID = {0}.".format(dsid))
    result = self.add_dependent_data(
        dsid,
        dataset,
        timeout,
        block,
        data_type=DataAddType.CREATION,
        no_exception_on_chunk_error=no_exception_on_chunk_error,
    )
    return {"dataset_id": dsid, "warnings": result["warnings"], "errors": result["errors"]}
def add_global_data(self, dataset_id, csv_data, timeout=600, block=True, data_type=HISTORICAL, no_exception_on_chunk_error=False)
Expand source code
def add_global_data(
    self,
    dataset_id,
    csv_data,
    timeout=600,
    block=True,
    data_type=DataAddType.HISTORICAL,
    no_exception_on_chunk_error=False,
):
    query_info = self.query_dataset(dataset_id)
    if DataSetType[query_info["type"]] != DataSetType.GLOBAL:
        raise BoostedAPIException(
            f"Incorrect dataset type: {query_info['type']}" f" - Expected {DataSetType.GLOBAL}"
        )
    warnings, errors = self.setup_chunk_and_upload_data(
        dataset_id, csv_data, data_type, timeout, block, no_exception_on_chunk_error
    )
    if len(warnings) > 0:
        logger.warning(
            "Encountered {0} total warnings while uploading dataset.".format(len(warnings))
        )
    if len(errors) > 0:
        raise BoostedAPIException(
            "Encountered {0} total ERRORS while uploading dataset".format(len(errors))
            + "\n".join(errors)
        )
    return {"warnings": warnings, "errors": errors}
def add_global_dataset(self, dataset, datasetName='GlobalDataset', schema=None, timeout=600, block=True)
Expand source code
def add_global_dataset(
    self, dataset, datasetName="GlobalDataset", schema=None, timeout=600, block=True
):
    result = self.add_global_dataset_with_warnings(dataset, datasetName, schema, timeout, block)
    return result["dataset_id"]
def add_global_dataset_with_warnings(self, dataset, datasetName='GlobalDataset', schema=None, timeout=600, block=True, no_exception_on_chunk_error=False)
Expand source code
def add_global_dataset_with_warnings(
    self,
    dataset,
    datasetName="GlobalDataset",
    schema=None,
    timeout=600,
    block=True,
    no_exception_on_chunk_error=False,
):
    if not self.validate_dataframe(dataset):
        logger.error("dataset failed validation.")
        return None
    if schema is None:
        schema = infer_dataset_schema(datasetName, dataset, DataSetType.GLOBAL)
    dsid = self.createDataset(schema.toDict())
    logger.info("Creating dataset with ID = {0}.".format(dsid))
    result = self.add_global_data(
        dsid,
        dataset,
        timeout,
        block,
        data_type=DataAddType.CREATION,
        no_exception_on_chunk_error=no_exception_on_chunk_error,
    )
    return {"dataset_id": dsid, "warnings": result["warnings"], "errors": result["errors"]}
def add_hedge_experiment_scenario(self, experiment_id: str, scenario_name: str, scenario_settings: PortfolioSettings, run_scenario_immediately: bool) ‑> HedgeExperimentScenario
Expand source code
def add_hedge_experiment_scenario(
    self,
    experiment_id: str,
    scenario_name: str,
    scenario_settings: PortfolioSettings,
    run_scenario_immediately: bool,
) -> HedgeExperimentScenario:

    add_scenario_input = {
        "hedgeExperimentId": experiment_id,
        "scenarioName": scenario_name,
        "portfolioSettingsJson": str(scenario_settings),
        "runExperimentOnScenario": run_scenario_immediately,
        "createDefaultPortfolio": "false",
    }
    qry = """
        mutation addHedgeExperimentScenario(
            $input: AddHedgeExperimentScenarioInput!
        ) {
            addHedgeExperimentScenario(input: $input) {
                hedgeExperimentScenario {
                    hedgeExperimentScenarioId
                    scenarioName
                    description
                    portfolioSettingsJson
                }
            }
        }

    """

    url = f"{self.base_uri}/api/graphql"

    resp = requests.post(
        url,
        headers={"Authorization": "ApiKey " + self.api_key},
        json={"query": qry, "variables": {"input": add_scenario_input}},
    )

    json_resp = resp.json()
    if (resp.ok and "errors" in json_resp) or not resp.ok:
        error_msg = self._try_extract_error_code(resp)
        logger.error(error_msg)
        raise BoostedAPIException(
            (f"Failed to add scenario: {resp.status_code=}; {error_msg=}")
        )

    scenario_dict = json_resp["data"]["addHedgeExperimentScenario"]["hedgeExperimentScenario"]
    if scenario_dict is None:
        raise BoostedAPIException(
            "Failed to add scenario, likely due to bad experiment id or api key"
        )
    s = HedgeExperimentScenario.from_json_dict(scenario_dict)
    return s
def add_independent_data(self, dataset_id, csv_data, timeout=600, block=True, data_type=HISTORICAL, no_exception_on_chunk_error=False)
Expand source code
def add_independent_data(
    self,
    dataset_id,
    csv_data,
    timeout=600,
    block=True,
    data_type=DataAddType.HISTORICAL,
    no_exception_on_chunk_error=False,
):
    query_info = self.query_dataset(dataset_id)
    if DataSetType[query_info["type"]] != DataSetType.STRATEGY:
        raise BoostedAPIException(
            f"Incorrect dataset type: {query_info['type']}"
            f" - Expected {DataSetType.STRATEGY}"
        )
    warnings, errors = self.setup_chunk_and_upload_data(
        dataset_id, csv_data, data_type, timeout, block, no_exception_on_chunk_error
    )
    if len(warnings) > 0:
        logger.warning(
            "Encountered {0} total warnings while uploading dataset.".format(len(warnings))
        )
    if len(errors) > 0:
        raise BoostedAPIException(
            "Encountered {0} total ERRORS while uploading dataset".format(len(errors))
            + "\n".join(errors)
        )
    return {"warnings": warnings, "errors": errors}
def add_independent_dataset(self, dataset, datasetName='IndependentDataset', schema=None, timeout=600, block=True)
Expand source code
def add_independent_dataset(
    self, dataset, datasetName="IndependentDataset", schema=None, timeout=600, block=True
):
    result = self.add_independent_dataset_with_warnings(
        dataset, datasetName, schema, timeout, block
    )
    return result["dataset_id"]
def add_independent_dataset_with_warnings(self, dataset, datasetName='IndependentDataset', schema=None, timeout=600, block=True, no_exception_on_chunk_error=False)
Expand source code
def add_independent_dataset_with_warnings(
    self,
    dataset,
    datasetName="IndependentDataset",
    schema=None,
    timeout=600,
    block=True,
    no_exception_on_chunk_error=False,
):
    if not self.validate_dataframe(dataset):
        logger.error("dataset failed validation.")
        return None
    if schema is None:
        schema = infer_dataset_schema(datasetName, dataset, DataSetType.STRATEGY)
    schemaDict = schema.toDict()
    if "configurationDataJson" not in schemaDict:
        schemaDict["configurationDataJson"] = "{}"
    dsid = self.createDataset(schemaDict)
    logger.info("Creating dataset with ID = {0}.".format(dsid))
    result = self.add_independent_data(
        dsid,
        dataset,
        timeout,
        block,
        data_type=DataAddType.CREATION,
        no_exception_on_chunk_error=no_exception_on_chunk_error,
    )
    return {"dataset_id": dsid, "warnings": result["warnings"], "errors": result["errors"]}
def add_securities_to_watchlist(self, watchlist_id: str, identifiers: List[str], identifier_type: Literal['TICKER', 'ISIN']) ‑> Dict[~KT, ~VT]
Expand source code
def add_securities_to_watchlist(
    self, watchlist_id: str, identifiers: List[str], identifier_type: Literal["TICKER", "ISIN"]
) -> Dict:

    # should we just make the arg lower? all caps has a flag-like feel to it
    id_type = identifier_type.lower()
    url = f"{self.base_uri}{WATCHLIST_ROUTE_PREFIX}{DAL_WATCHLIST_ROUTE}/add_{id_type}s/"
    headers = {"Authorization": "ApiKey " + self.api_key, "Content-Type": "application/json"}
    req_json = {"watchlist_id": watchlist_id, id_type: identifiers}
    res = requests.post(url, json=req_json, headers=headers, **self._request_params)

    if not res.ok:
        error_msg = self._try_extract_error_code(res)
        logger.error(error_msg)
        raise BoostedAPIException(f"Failed to get user models: {error_msg}")

    data = res.json()
    return data
def add_to_portfolio_group(self, group_id: str, portfolios: List[Dict[~KT, ~VT]]) ‑> Dict[~KT, ~VT]

Add portfolios to a group

Parameters

group_id : str,
 

UUID str identifying a portfolio group

portfolios : List of Dict [:
portfolio_id: str rank_in_group: Optional[int] = None

]

Returns:

Dict { added: int number of successful changes }

Expand source code
def add_to_portfolio_group(
    self,
    group_id: str,
    portfolios: List[Dict],
) -> Dict:
    """
    Add portfolios to a group

    Parameters
    ----------

    group_id: str,
       UUID str identifying a portfolio group

    portfolios: List of Dict [:
        portfolio_id: str
        rank_in_group: Optional[int] = None
    ]


    Returns:
    -------
    Dict {
        added: int
           number of successful changes
    }
    """
    url = f"{self.base_uri}{WATCHLIST_ROUTE_PREFIX}{PORTFOLIO_GROUP_ROUTE}/add-to-group"
    headers = {"Authorization": "ApiKey " + self.api_key, "Content-Type": "application/json"}
    req_json = {"group_id": group_id, "portfolios": portfolios}

    res = requests.post(url, json=req_json, headers=headers, **self._request_params)

    if not res.ok:
        error_msg = self._try_extract_error_code(res)
        logger.error(error_msg)
        raise BoostedAPIException(f"Failed to add portfolios to portfolio group: {error_msg}")

    data = res.json()
    return data
def add_uploaded_model_data(self, url, csv_data, request_data, timeout=600)
Expand source code
def add_uploaded_model_data(self, url, csv_data, request_data, timeout=600):
    logger.info("Starting upload.")
    headers = {"Authorization": "ApiKey " + self.api_key}
    files_req = {}
    target = ("data.csv", None, "text/csv")
    warnings = []
    if isinstance(csv_data, pd.core.frame.DataFrame):
        buf = io.StringIO()
        csv_data.to_csv(buf, header=False)
        if not isinstance(csv_data.index, pd.core.indexes.datetimes.DatetimeIndex):
            raise BoostedAPIException("DataFrame must have DatetimeIndex as index type.")
        target = ("uploaded_data.csv", buf.getvalue(), "text/csv")
        files_req["dataFile"] = target
        res = requests.post(
            url,
            files=files_req,
            data=request_data,
            headers=headers,
            timeout=timeout,
            **self._request_params,
        )
    elif isinstance(csv_data, str):
        target = ("uploaded_data.csv", csv_data, "text/csv")
        files_req["dataFile"] = target
        res = requests.post(
            url,
            files=files_req,
            data=request_data,
            headers=headers,
            timeout=timeout,
            **self._request_params,
        )
    else:
        raise BoostedAPIException("Expected CSV as str or Pandas DataFrame.")
    if res.ok:
        logger.info("Signals upload completed.")
        result = res.json()["result"]
        if "warningMessages" in result:
            warnings = result["warningMessages"]
    else:
        logger.error("Signals upload failed: {0}, {1}".format(res.text, res.reason))
        raise BoostedAPIException("Upload failed.")

    return res, warnings
def check_dataset_ingestion_completion(self, dataset_id, chunk_id, start_time)
Expand source code
def check_dataset_ingestion_completion(self, dataset_id, chunk_id, start_time):
    url = self.base_uri + "/api/datasets/{0}/upload-chunk-status".format(dataset_id)
    headers = {"Authorization": "ApiKey " + self.api_key}
    params = {"uploadGroupId": chunk_id}
    res = requests.get(url, headers=headers, **self._request_params, params=params)
    res = res.json()

    finished = False
    warnings = []
    errors = []

    if type(res) == dict:
        dataset_status = res["datasetStatus"]
        chunk_status = res["chunkStatus"]
        if chunk_status != ChunkStatus.PROCESSING.value:
            finished = True
            errors = res["errors"]
            warnings = res["warnings"]
            successful_rows = res["successfulRows"]
            total_rows = res["totalRows"]
            logger.info(
                f"Successfully ingested {successful_rows} out of {total_rows} uploaded rows."
            )
            if chunk_status in [
                ChunkStatus.SUCCESS.value,
                ChunkStatus.WARNING.value,
                ChunkStatus.ERROR.value,
            ]:
                if dataset_status != "AVAILABLE":
                    raise BoostedAPIException(
                        "Dataset was unexpectedly unavailable after chunk upload finished."
                    )
                else:
                    logger.info("Ingestion complete.  Uploaded data is ready for use.")
            elif chunk_status == ChunkStatus.ABORTED.value:
                errors.append(
                    "Dataset chunk upload was aborted by server! Upload did not succeed."
                )
            else:
                errors.append("Unexpected data ingestion status: {0}.".format(chunk_status))
        logger.info(
            "Data ingestion still running.  Time elapsed={0}.".format(
                datetime.datetime.now() - start_time
            )
        )
    else:
        raise BoostedAPIException("Unable to get status of dataset ingestion.")
    return {"finished": finished, "warnings": warnings, "errors": errors}
def chunk_and_upload_data(self, dataset_id, chunk_id, csv_data, timeout=600, no_exception_on_chunk_error=False)
Expand source code
def chunk_and_upload_data(
    self, dataset_id, chunk_id, csv_data, timeout=600, no_exception_on_chunk_error=False
):
    if isinstance(csv_data, pd.core.frame.DataFrame):
        if not isinstance(csv_data.index, pd.core.indexes.datetimes.DatetimeIndex):
            raise BoostedAPIException("DataFrame must have DatetimeIndex as index type.")

        warnings = []
        errors = []
        logger.info("Uploading yearly.")
        for t in csv_data.index.to_period("Y").unique():
            if t is pd.NaT:
                continue

            # serialize bit to string
            buf = self.get_csv_buffer()
            yearly_csv = csv_data.loc[str(t)]
            yearly_csv.to_csv(buf, header=True)
            raw_csv = buf.getvalue()

            # we are already chunking yearly... but if the csv still exceeds a healthy
            # limit of 50mb the final line of defence is to ignore date boundaries and
            # just chunk the rows. This is mostly for the cloudflare upload limit.
            size_lim = 50 * 1000 * 1000
            est_csv_size = sys.getsizeof(raw_csv)
            if est_csv_size > size_lim:
                del raw_csv, buf
                logger.info("Yearly data too large for single upload, chunking further...")
                chunks = []
                nchunks = math.ceil(est_csv_size / size_lim)
                rows_per_chunk = math.ceil(len(yearly_csv) / nchunks)
                for i in range(0, len(yearly_csv), rows_per_chunk):
                    buf = self.get_csv_buffer()
                    split_csv = yearly_csv.iloc[i : i + rows_per_chunk]
                    split_csv.to_csv(buf, header=True)
                    split_csv = buf.getvalue()
                    chunks.append(
                        (
                            "{0}-{1}".format(i + 1, min(len(yearly_csv), i + rows_per_chunk)),
                            split_csv,
                        )
                    )
            else:
                chunks = [("all", raw_csv)]

            for i, (rows_descriptor, chunk_csv) in enumerate(chunks):
                chunk_descriptor = "{0} in yearly chunk {1}".format(rows_descriptor, t)
                logger.info(
                    "Uploading rows:"
                    + chunk_descriptor
                    + " (chunk {0} of {1}):".format(i + 1, len(chunks))
                )
                _, new_warnings, new_errors = self.upload_dataset_chunk(
                    chunk_descriptor,
                    dataset_id,
                    chunk_id,
                    chunk_csv,
                    timeout,
                    no_exception_on_chunk_error,
                )
                warnings.extend(new_warnings)
                errors.extend(new_errors)
        return warnings, errors

    elif isinstance(csv_data, str):
        _, warnings, errors = self.upload_dataset_chunk(
            "all data", dataset_id, chunk_id, csv_data, timeout, no_exception_on_chunk_error
        )
        return warnings, errors
    else:
        raise BoostedAPIException("Expected CSV as str or Pandas DataFrame.")
def createBlacklist(self, isin, long_short=2, start_date=datetime.date(2022, 9, 15), end_date='4000-01-01', model_id=None)
Expand source code
def createBlacklist(
    self,
    isin,
    long_short=2,
    start_date=datetime.date.today(),
    end_date="4000-01-01",
    model_id=None,
):
    url = self.base_uri + f"/api/blacklist"
    data = {
        "modelId": model_id,
        "isin": isin,
        "longShort": long_short,
        "startDate": self.__iso_format(start_date),
        "endDate": self.__iso_format(end_date),
    }
    headers = {"Authorization": "ApiKey " + self.api_key, "Content-Type": "application/json"}
    res = requests.post(url, data=json.dumps(data), headers=headers, **self._request_params)
    if res.ok:
        return res.json()
    else:
        error_msg = self._try_extract_error_code(res)
        logger.error(error_msg)
        raise BoostedAPIException(
            f"""Failed to create the blacklist with \
              isin {isin} long_short {long_short} start_date {start_date} end_date {end_date} \
              model_id {model_id}: {error_msg}."""
        )
def createBlacklistsFromCSV(self, csv_name)
Expand source code
def createBlacklistsFromCSV(self, csv_name):
    url = self.base_uri + f"/api/blacklists"
    data = []
    with open(csv_name, mode="r") as f:
        csv_reader = csv.DictReader(f)
        for row in csv_reader:
            blacklist = {"modelId": row["ModelID"], "isin": row["ISIN"]}
            if row["LongShort"] == "":
                blacklist["longShort"] = 2
            else:
                blacklist["longShort"] = row["LongShort"]

            if row["StartDate"] == "":
                blacklist["startDate"] = self.__iso_format(datetime.date.today())
            else:
                blacklist["startDate"] = self.__iso_format(row["StartDate"])

            if row["EndDate"] == "":
                blacklist["endDate"] = self.__iso_format("4000-01-01")
            else:
                blacklist["endDate"] = self.__iso_format(row["EndDate"])
            data.append(blacklist)
    print(f"Processed {len(data)} blacklists.")
    headers = {"Authorization": "ApiKey " + self.api_key, "Content-Type": "application/json"}
    res = requests.post(url, data=json.dumps(data), headers=headers, **self._request_params)
    if res.ok:
        return res.json()
    else:
        error_msg = self._try_extract_error_code(res)
        logger.error(error_msg)
        raise BoostedAPIException("failed to create blacklists")
def createDataset(self, schema)
Expand source code
def createDataset(self, schema):
    request_url = "/api/datasets"
    headers = {"Authorization": "ApiKey " + self.api_key, "Content-Type": "application/json"}
    s = json.dumps(schema)
    logger.info("Creating dataset with schema " + s)
    res = requests.post(
        self.base_uri + request_url, data=s, headers=headers, **self._request_params
    )
    if res.ok:
        return res.json()["result"]
    else:
        raise BoostedAPIException("Dataset creation failed.")
def createPortfolioWithPortfolioSettings(self, model_id, portfolio_name, portfolio_description, portfolio_settings, timeout=600)
Expand source code
def createPortfolioWithPortfolioSettings(
    self, model_id, portfolio_name, portfolio_description, portfolio_settings, timeout=600
):
    url = self.base_uri + "/api/models/{0}/constraints/add".format(model_id)
    headers = {"Authorization": "ApiKey " + self.api_key, "Content-Type": "application/json"}
    setting_string = json.dumps(portfolio_settings.settings)
    logger.info("Creating new portfolio with specified setting: {}".format(setting_string))
    params = {
        "name": portfolio_name,
        "description": portfolio_description,
        "constraints": setting_string,
        "validate": "true",
    }
    res = requests.put(url, json=params, headers=headers, **self._request_params)
    response = res.json()
    if res.ok:
        return response
    else:
        error_msg = self._try_extract_error_code(res)
        logger.error(error_msg)
        raise BoostedAPIException(
            "Failed to create portfolio with the specified settings: {0}.".format(error_msg)
        )
def createSignalsModel(self, csv_data, model_name, timeout=600)
Expand source code
def createSignalsModel(self, csv_data, model_name, timeout=600):
    warnings = []
    url = self.base_uri + "/api/models/upload/signals/create"
    request_data = {"modelName": model_name, "uploadName": model_name}
    res, warnings = self.add_uploaded_model_data(url, csv_data, request_data, timeout)
    result = res.json()["result"]
    model_id = result["modelId"]
    self.sendModelRecalc(model_id)
    return model_id, warnings
def create_hedge_experiment(self, name: str, description: str, experiment_type: Literal['HEDGE', 'MIMIC'], target_securities: Union[Dict[GbiIdSecurity, float], str]) ‑> HedgeExperiment
Expand source code
def create_hedge_experiment(
    self,
    name: str,
    description: str,
    experiment_type: hedge_experiment_type,
    target_securities: Union[Dict[GbiIdSecurity, float], str],
) -> HedgeExperiment:
    # we don't pass target_securities here (as much as id like to) because the
    # graphql input doesn't support it at this point

    # note that this query returns a lot of null fields at this point, but
    # they are necessary for building a HE.
    create_qry = """
        mutation createDraftMutation($input: CreateHedgeExperimentDraftInput!) {
            createHedgeExperimentDraft(input: $input) {
                hedgeExperiment {
                    hedgeExperimentId
                    experimentName
                    userId
                    config
                    description
                    experimentType
                    lastCalculated
                    lastModified
                    status
                    portfolioCalcStatus
                    targetSecurities {
                        gbiId
                        security {
                            gbiId
                            name
                            symbol
                        }
                        weight
                    }
                    baselineModel {
                        id
                        name
                    }
                    baselineScenario {
                        hedgeExperimentScenarioId
                        scenarioName
                        description
                        portfolioSettingsJson
                        hedgeExperimentPortfolios {
                            portfolio {
                                id
                                name
                                modelId
                                performanceGridHeader
                                performanceGrid
                                status
                                tearSheet {
                                    groupName
                                    members {
                                        name
                                        value
                                    }
                                }
                            }
                        }
                        status
                    }
                    baselineStockUniverseId
                }
            }
        }
    """

    create_input = {"name": name, "experimentType": experiment_type, "description": description}
    if isinstance(target_securities, dict):
        create_input["setTargetSecurities"] = [
            {"gbiId": sec.gbi_id, "weight": weight}
            for (sec, weight) in target_securities.items()
        ]
    elif isinstance(target_securities, str):
        create_input["setTargetPortfolios"] = [{"portfolioId": target_securities}]
    elif target_securities is None:
        pass
    else:
        raise TypeError(
            "Expected value of type Union[Dict[GbiIdSecurity, str], str] for "
            f"argument 'target_securities'; got {type(target_securities)}"
        )
    resp = requests.post(
        f"{self.base_uri}/api/graphql",
        json={"query": create_qry, "variables": {"input": create_input}},
        headers={"Authorization": "ApiKey " + self.api_key},
        params=self._request_params,
    )

    json_resp = resp.json()
    if (resp.ok and "errors" in json_resp) or not resp.ok:
        error_msg = self._try_extract_error_code(resp)
        logger.error(error_msg)
        raise BoostedAPIException(
            (f"Failed to create hedge experiment: {resp.status_code=}; {error_msg=}")
        )

    exp_dict = json_resp["data"]["createHedgeExperimentDraft"]["hedgeExperiment"]
    experiment = HedgeExperiment.from_json_dict(exp_dict)
    return experiment
def create_portfolio_group(self, group_name: str, portfolios: Optional[List[Dict[~KT, ~VT]]] = None) ‑> Dict[~KT, ~VT]

Create a new portfolio group

Parameters

group_name : str
 

name of the new group

portfolios : List of Dict [:
 
portfolio_id : str
 
rank_in_group : Optional[int] = None
 

]

Returns:

Dict: { group_id: str UUID identifier for the portfolio group

created: int num groups created, 1 == success

added: int num portfolios added to the group, should match the length of 'portfolios' argument }

Expand source code
def create_portfolio_group(
    self,
    group_name: str,
    portfolios: Optional[List[Dict]] = None,
) -> Dict:
    """
    Create a new portfolio group

    Parameters
    ----------

    group_name: str
       name of the new group

    portfolios: List of Dict [:

    portfolio_id: str
    rank_in_group: Optional[int] = None
    ]

    Returns:
    ----------

    Dict: {
    group_id: str
       UUID identifier for the portfolio group

    created: int
       num groups created, 1 == success

    added: int
       num portfolios added to the group, should match the length of 'portfolios' argument
    }
    """
    url = f"{self.base_uri}{WATCHLIST_ROUTE_PREFIX}{PORTFOLIO_GROUP_ROUTE}/create"
    headers = {"Authorization": "ApiKey " + self.api_key, "Content-Type": "application/json"}
    req_json = {"group_name": group_name, "portfolios": portfolios}

    res = requests.post(url, json=req_json, headers=headers, **self._request_params)

    if not res.ok:
        error_msg = self._try_extract_error_code(res)
        logger.error(error_msg)
        raise BoostedAPIException(f"Failed to create portfolio group: {error_msg}")

    data = res.json()
    return data
def create_universe(self, universe: Union[pandas.core.frame.DataFrame, str], name: str, description: str) ‑> List[str]
Expand source code
def create_universe(
    self, universe: Union[pd.DataFrame, str], name: str, description: str
) -> List[str]:
    PRESENT = "PRESENT"
    ANY = "ANY"
    EARLIST_DATE = "1900-01-01"
    LATEST_DATE = "4000-01-01"

    if isinstance(universe, (str, bytes, os.PathLike)):
        universe = pd.read_csv(universe)

    universe.columns = universe.columns.str.lower()

    # Clients are free to leave out data. Fill in some defaults here.
    if "from" not in universe.columns:
        universe["from"] = EARLIST_DATE
    if "to" not in universe.columns:
        universe["to"] = LATEST_DATE
    if "currency" not in universe.columns:
        universe["currency"] = ANY
    if "country" not in universe.columns:
        universe["country"] = ANY
    if "isin" not in universe.columns:
        universe["isin"] = None
    if "symbol" not in universe.columns:
        universe["symbol"] = None

    # to prevent conflicts with python keywords
    universe.rename(columns={"from": "from_date", "to": "to_date"}, inplace=True)

    universe = universe.replace({np.nan: None})
    security_country_currency_date_list = []
    for i, r in enumerate(universe.itertuples()):
        id_type = ColumnSubRole.ISIN
        identifier = r.isin

        if identifier is None:
            id_type = ColumnSubRole.SYMBOL
            identifier = str(r.symbol)

        # if identifier is still None, it means that there is no ISIN or
        # SYMBOL for this row, in which case we throw an error
        if identifier is None:
            raise BoostedAPIException(
                (
                    f"Missing identifier column in universe row {i + 1}"
                    " should contain ISIN or Symbol"
                )
            )

        security_country_currency_date_list.append(
            DateIdentCountryCurrency(
                date=r.from_date or EARLIST_DATE,
                identifier=identifier,
                country=r.country or ANY,
                currency=r.currency or ANY,
                id_type=id_type,
            )
        )

    gbi_id_objs = self.getGbiIdFromIdentCountryCurrencyDate(security_country_currency_date_list)

    security_list = []
    for i, r in enumerate(universe.itertuples()):
        # if we have a None here, we failed to map to a gbi id
        if gbi_id_objs[i] is None:
            raise BoostedAPIException(f"Unable to map row: {tuple(r)}")

        security_list.append(
            {
                "stockId": gbi_id_objs[i].gbi_id,
                "fromZ": r.from_date or EARLIST_DATE,
                "toZ": LATEST_DATE if r.to_date in (PRESENT, None) else r.to_date,
                "removal": False,
                "source": "UPLOAD",
            }
        )

    url = self.base_uri + "/api/template-universe/save"
    headers = {"Authorization": "ApiKey " + self.api_key, "Content-Type": "application/json"}
    req = {"name": name, "description": description, "modificationDaos": security_list}

    res = requests.post(url, json=req, headers=headers, **self._request_params)
    self._check_ok_or_err_with_msg(res, "Failed to create universe")

    if "warnings" in res.json():
        logger.info("Warnings: {0}.".format(res.json()["warnings"]))
        return res.json()["warnings"].splitlines()
    else:
        return []
def create_watchlist(self, name: str) ‑> str
Expand source code
def create_watchlist(self, name: str) -> str:
    url = f"{self.base_uri}{WATCHLIST_ROUTE_PREFIX}{DAL_WATCHLIST_ROUTE}/create/"
    headers = {"Authorization": "ApiKey " + self.api_key, "Content-Type": "application/json"}
    req_json = {"name": name}
    res = requests.post(url, json=req_json, headers=headers, **self._request_params)

    if not res.ok:
        error_msg = self._try_extract_error_code(res)
        logger.error(error_msg)
        raise BoostedAPIException(f"Failed to get user models: {error_msg}")

    data = res.json()
    return data["watchlist_id"]
def create_watchlist_from_file(self, name: str, filepath: str) ‑> str
Expand source code
def create_watchlist_from_file(self, name: str, filepath: str) -> str:

    url = f"{self.base_uri}{ROUTE_PREFIX}{DAL_WATCHLIST_ROUTE}/create_watchlist_from_file/"
    headers = {"Authorization": "ApiKey " + self.api_key}

    with open(filepath, "rb") as fp:
        file_bytes = fp.read()

    file_bytes_base64 = base64.b64encode(file_bytes).decode("ascii")
    json_req = {
        "content_type": mimetypes.guess_type(filepath)[0],
        "file_bytes_base64": file_bytes_base64,
        "name": name,
    }

    res = requests.post(url, json=json_req, headers=headers)

    if not res.ok:
        error_msg = self._try_extract_error_code(res)
        logger.error(error_msg)
        raise BoostedAPIException(f"Failed to create watchlist from file: {error_msg}")

    data = res.json()
    return data["watchlist_id"]
def deleteBlacklist(self, blacklist_id)
Expand source code
def deleteBlacklist(self, blacklist_id):
    url = self.base_uri + f"/api/blacklist/{blacklist_id}"
    headers = {"Authorization": "ApiKey " + self.api_key}
    res = requests.delete(url, headers=headers, **self._request_params)
    if res.ok:
        result = res.json()
        return result
    else:
        error_msg = self._try_extract_error_code(res)
        logger.error(error_msg)
        raise BoostedAPIException(
            f"Failed to delete blacklist with id {blacklist_id}: {error_msg}"
        )
def delete_hedge_experiment(self, experiment_id: str) ‑> bool
Expand source code
def delete_hedge_experiment(self, experiment_id: str) -> bool:
    delete_qry = """
        mutation($input: DeleteHedgeExperimentsInput!) {
            deleteHedgeExperiments(input: $input) {
                success
            }
        }
    """
    delete_input = {"hedgeExperimentIds": [experiment_id]}
    resp = requests.post(
        f"{self.base_uri}/api/graphql",
        json={"query": delete_qry, "variables": {"input": delete_input}},
        headers={"Authorization": "ApiKey " + self.api_key},
        params=self._request_params,
    )

    json_resp = resp.json()
    if (resp.ok and "errors" in json_resp) or not resp.ok:
        error_msg = self._try_extract_error_code(resp)
        logger.error(error_msg)
        raise BoostedAPIException(
            (
                f"Failed to delete hedge experiment {experiment_id=}: "
                + f"status_code={resp.status_code}; error_msg={error_msg}"
            )
        )

    return json_resp["data"]["deleteHedgeExperiments"]["success"]
def delete_portfolio_group(self, group_id: str) ‑> Dict[~KT, ~VT]

Delete a portfolio group

Parameters

group_id : str,
 

UUID str identifying a portfolio group

Returns:

Dict { removed_groups: int number of successful changes

removed_portfolios: int
   number of successful changes

}

Expand source code
def delete_portfolio_group(
    self,
    group_id: str,
) -> Dict:
    """
    Delete a portfolio group

    Parameters
    ----------

    group_id: str,
       UUID str identifying a portfolio group


    Returns:
    -------
    Dict {
        removed_groups: int
           number of successful changes

        removed_portfolios: int
           number of successful changes
    }
    """
    url = f"{self.base_uri}{WATCHLIST_ROUTE_PREFIX}{PORTFOLIO_GROUP_ROUTE}/remove"
    headers = {"Authorization": "ApiKey " + self.api_key, "Content-Type": "application/json"}
    req_json = {"group_id": group_id}
    res = requests.post(url, json=req_json, headers=headers, **self._request_params)

    if not res.ok:
        error_msg = self._try_extract_error_code(res)
        logger.error(error_msg)
        raise BoostedAPIException(f"Failed to delete portfolio group: {error_msg}")

    data = res.json()
    return data
def diff_hedge_experiment_portfolio_data(self, hedge_experiment_id: str, comparison_portfolios: List[str], categories: List[str]) ‑> Dict[~KT, ~VT]
Expand source code
def diff_hedge_experiment_portfolio_data(
    self,
    hedge_experiment_id: str,
    comparison_portfolios: List[str],
    categories: List[str],
) -> Dict:
    qry = """
    query diffHedgeExperimentPortfolios(
        $input: DiffHedgeExperimentPortfoliosInput!
    ) {
        diffHedgeExperimentPortfolios(input: $input) {
        data {
            diffs {
                volatility {
                    date
                    vol5D
                    vol10D
                    vol21D
                    vol21D
                    vol63D
                    vol126D
                    vol189D
                    vol252D
                    vol315D
                    vol378D
                    vol441D
                    vol504D
                }
                performance {
                    date
                    value
                }
                performanceGrid {
                    headerRow
                    values
                }
                factors {
                    date
                    momentum
                    growth
                    size
                    value
                    dividendYield
                    volatility
                }
            }
        }
        errors
        }
    }
    """
    headers = {"Authorization": "ApiKey " + self.api_key}
    params = {
        "hedgeExperimentId": hedge_experiment_id,
        "portfolioIds": comparison_portfolios,
        "categories": categories,
    }
    resp = requests.post(
        f"{self.base_uri}/api/graphql",
        json={"query": qry, "variables": params},
        headers=headers,
        params=self._request_params,
    )

    json_resp = resp.json()

    # graphql endpoints typically return 200 or 400 status codes, so we must
    # check if we have any errors, even with a 200
    if (resp.ok and "errors" in json_resp) or not resp.ok:
        error_msg = self._try_extract_error_code(resp)
        logger.error(error_msg)
        raise BoostedAPIException(
            (
                f"Failed to get portfolio diffs for {hedge_experiment_id=}: "
                f"{resp.status_code=}; {error_msg=}"
            )
        )

    diffs = json_resp["data"]["diffHedgeExperimentPortfolios"]["data"]["diffs"]
    comparisons = {}
    for pf, cmp in zip(comparison_portfolios, diffs):
        res = {
            "performance": None,
            "performanceGrid": None,
            "factors": None,
            "volatility": None,
        }
        if "performanceGrid" in cmp:
            grid = cmp["performanceGrid"]
            grid_df = pd.DataFrame(grid["values"], columns=grid["headerRow"])
            res["performanceGrid"] = grid_df
        if "performance" in cmp:
            perf_df = pd.DataFrame(cmp["performance"]).set_index("date")
            perf_df.index = pd.to_datetime(perf_df.index)
            res["performance"] = perf_df
        if "volatility" in cmp:
            vol_df = pd.DataFrame(cmp["volatility"]).set_index("date")
            vol_df.index = pd.to_datetime(vol_df.index)
            res["volatility"] = vol_df
        if "factors" in cmp:
            factors_df = pd.DataFrame(cmp["factors"]).set_index("date")
            factors_df.index = pd.to_datetime(factors_df.index)
            res["factors"] = factors_df
        comparisons[pf] = res
    return comparisons
def download_complete_portfolio_data(self, model_id: str, portfolio_id: str, download_filepath: str)
Expand source code
def download_complete_portfolio_data(
    self, model_id: str, portfolio_id: str, download_filepath: str
):
    headers = {"Authorization": "ApiKey " + self.api_key, "Content-Type": "application/json"}
    url = self.base_uri + f"/api/models/{model_id}/{portfolio_id}/excel"

    res = requests.get(url, headers=headers, **self._request_params)
    self._check_ok_or_err_with_msg(
        res, f"Failed to get full data for {model_id=}, {portfolio_id=}"
    )

    with open(download_filepath, "wb") as f:
        f.write(res.content)
def export_data(self, dataset_id, start=datetime.date(1997, 9, 21), end=datetime.date(2022, 9, 15), timeout=600)
Expand source code
def export_data(
    self,
    dataset_id,
    start=(datetime.date.today() - timedelta(days=365 * 25)),
    end=datetime.date.today(),
    timeout=600,
):
    logger.info("Requesting start={0} end={1}.".format(start, end))
    request_url = "/api/datasets/" + dataset_id + "/export-data"
    headers = {"Authorization": "ApiKey " + self.api_key}
    start = self.__iso_format(start)
    end = self.__iso_format(end)
    params = {"start": start, "end": end}
    logger.debug("URL={0}, headers={1}, params={2}".format(request_url, headers, params))
    res = requests.get(
        self.base_uri + request_url,
        headers=headers,
        params=params,
        timeout=timeout,
        **self._request_params,
    )
    if res.ok or self._check_status_code(res):
        buf = io.StringIO(res.text)
        df = pd.read_csv(buf, index_col=0, parse_dates=True)
        if "price" in df.columns:
            df = df.drop("price", axis=1)
        return df
    else:
        error_msg = self._try_extract_error_code(res)
        logger.error(error_msg)
        raise BoostedAPIException("Failed to query dataset: {0}.".format(error_msg))
def export_dependent_data(self, dataset_id, start=datetime.date(1997, 9, 21), end=datetime.date(2022, 9, 15), timeout=600)
Expand source code
def export_dependent_data(
    self,
    dataset_id,
    start=(datetime.date.today() - timedelta(days=365 * 25)),
    end=datetime.date.today(),
    timeout=600,
):
    query_info = self.query_dataset(dataset_id)
    if DataSetType[query_info["type"]] != DataSetType.STOCK:
        raise BoostedAPIException(
            f"Incorrect dataset type: {query_info['type']}" f" - Expected {DataSetType.STOCK}"
        )
    return self.export_data(dataset_id, start, end, timeout)
def export_global_data(self, dataset_id, start=datetime.date(1997, 9, 21), end=datetime.date(2022, 9, 15), timeout=600)
Expand source code
def export_global_data(
    self,
    dataset_id,
    start=(datetime.date.today() - timedelta(days=365 * 25)),
    end=datetime.date.today(),
    timeout=600,
):
    query_info = self.query_dataset(dataset_id)
    if DataSetType[query_info["type"]] != DataSetType.GLOBAL:
        raise BoostedAPIException(
            f"Incorrect dataset type: {query_info['type']}" f" - Expected {DataSetType.GLOBAL}"
        )
    return self.export_data(dataset_id, start, end, timeout)
def export_independent_data(self, dataset_id, start=datetime.date(1997, 9, 21), end=datetime.date(2022, 9, 15), timeout=600)
Expand source code
def export_independent_data(
    self,
    dataset_id,
    start=(datetime.date.today() - timedelta(days=365 * 25)),
    end=datetime.date.today(),
    timeout=600,
):
    query_info = self.query_dataset(dataset_id)
    if DataSetType[query_info["type"]] != DataSetType.STRATEGY:
        raise BoostedAPIException(
            f"Incorrect dataset type: {query_info['type']}"
            f" - Expected {DataSetType.STRATEGY}"
        )
    return self.export_data(dataset_id, start, end, timeout)
def getAllModelDetails(self) ‑> Dict[str, Dict[str, Any]]
Expand source code
def getAllModelDetails(self) -> Dict[str, Dict[str, Any]]:
    url = f"{self.base_uri}/api/graphql"
    headers = {"Authorization": "ApiKey " + self.api_key, "Content-Type": "application/json"}
    req_json = {
        "query": "query listOfModels {\n models { id name lastUpdated portfolios { id name }}}",
        "variables": {},
    }
    res = requests.post(url, json=req_json, headers=headers, **self._request_params)
    if not res.ok:
        error_msg = self._try_extract_error_code(res)
        logger.error(error_msg)
        raise BoostedAPIException(f"Failed to get user models: {error_msg}")
    data = res.json()
    if data["data"]["models"] is None:
        return {}

    output_data = {}
    for rec in data["data"]["models"]:
        model_id = rec["id"]
        output_data[model_id] = {
            "name": rec["name"],
            "last_updated": parser.parse(rec["lastUpdated"]),
            "portfolios": rec["portfolios"],
        }

    return output_data
def getAllModelNames(self) ‑> Dict[str, str]
Expand source code
def getAllModelNames(self) -> Dict[str, str]:
    url = f"{self.base_uri}/api/graphql"
    headers = {"Authorization": "ApiKey " + self.api_key, "Content-Type": "application/json"}
    req_json = {"query": "query listOfModels {\n models { id name }}", "variables": {}}
    res = requests.post(url, json=req_json, headers=headers, **self._request_params)
    if not res.ok:
        error_msg = self._try_extract_error_code(res)
        logger.error(error_msg)
        raise BoostedAPIException(f"Failed to get user models: {error_msg}")
    data = res.json()
    if data["data"]["models"] is None:
        return {}
    return {rec["id"]: rec["name"] for rec in data["data"]["models"]}
def getAllocationsByDates(self, portfolio_id, dates=None)
Expand source code
def getAllocationsByDates(self, portfolio_id, dates=None):
    url = self.base_uri + "/api/portfolios/{0}/allocationsByDate".format(portfolio_id)
    headers = {"Authorization": "ApiKey " + self.api_key}
    if dates is not None:
        fmt_dates = []
        for d in dates:
            fmt_dates.append(self.__iso_format(d))
        fmt_dates = ",".join(fmt_dates)
        params = {"dates": fmt_dates}
        logger.info("Retrieving allocations information for dates {0}.".format(fmt_dates))
    else:
        params = {"dates": None}
        logger.info("Retrieving allocations information for all dates")
    res = requests.get(url, params=params, headers=headers, **self._request_params)
    if res.ok:
        logger.info("Allocations retrieval successful.")
        return res.json()
    else:
        error_msg = self._try_extract_error_code(res)
        raise BoostedAPIException("Failed to retrieve allocations: {0}.".format(error_msg))
def getAllocationsForDate(self, portfolio_id, date, rollback_to_last_available_date)
Expand source code
def getAllocationsForDate(self, portfolio_id, date, rollback_to_last_available_date):
    date = self.__iso_format(date)
    endpoint = "latest-allocations" if rollback_to_last_available_date else "allocations"
    url = self.base_uri + "/api/portfolios/{0}/{1}".format(portfolio_id, endpoint)
    headers = {"Authorization": "ApiKey " + self.api_key}
    params = {"date": date}
    logger.info("Retrieving allocations information for date {0}.".format(date))
    res = requests.get(url, params=params, headers=headers, **self._request_params)
    if res.ok:
        logger.info("Allocations retrieval successful.")
        return res.json()
    else:
        error_msg = self._try_extract_error_code(res)
        raise BoostedAPIException("Failed to retrieve allocations: {0}.".format(error_msg))
def getAllocationsForDateV2(self, portfolio_id, date, rollback_to_last_available_date)
Expand source code
def getAllocationsForDateV2(self, portfolio_id, date, rollback_to_last_available_date):
    date = self.__iso_format(date)
    endpoint = "latest-allocations-v2" if rollback_to_last_available_date else "allocations-v2"
    url = self.base_uri + "/api/portfolios/{0}/{1}".format(portfolio_id, endpoint)
    headers = {"Authorization": "ApiKey " + self.api_key}
    params = {"date": date}
    logger.info("Retrieving allocations information for date {0}.".format(date))
    res = requests.get(url, params=params, headers=headers, **self._request_params)
    if res.ok:
        logger.info("Allocations retrieval successful.")
        return res.json()
    else:
        error_msg = self._try_extract_error_code(res)
        raise BoostedAPIException("Failed to retrieve allocations: {0}.".format(error_msg))
def getBlacklist(self, blacklist_id)
Expand source code
def getBlacklist(self, blacklist_id):
    url = self.base_uri + f"/api/blacklist/{blacklist_id}"
    headers = {"Authorization": "ApiKey " + self.api_key}
    res = requests.get(url, headers=headers, **self._request_params)
    if res.ok:
        result = res.json()
        return result
    error_msg = self._try_extract_error_code(res)
    logger.error(error_msg)
    raise BoostedAPIException(f"Failed to get blacklist with id {blacklist_id}: {error_msg}")
def getBlacklists(self, model_id=None, company_id=None, last_N=None)
Expand source code
def getBlacklists(self, model_id=None, company_id=None, last_N=None):
    params = {}
    if last_N:
        params["lastN"] = last_N
    if model_id:
        params["modelId"] = model_id
    if company_id:
        params["companyId"] = company_id
    url = self.base_uri + f"/api/blacklist"
    headers = {"Authorization": "ApiKey " + self.api_key}
    res = requests.get(url, headers=headers, params=params, **self._request_params)
    if res.ok:
        result = res.json()
        return result
    error_msg = self._try_extract_error_code(res)
    logger.error(error_msg)
    raise BoostedAPIException(
        f"""Failed to get blacklists with \
        model_id {model_id} company_id {company_id} last_N {last_N}: {error_msg}"""
    )
def getDatasetDates(self, dataset_id)
Expand source code
def getDatasetDates(self, dataset_id):
    url = self.base_uri + f"/api/datasets/{dataset_id}"
    headers = {"Authorization": "ApiKey " + self.api_key}
    res = requests.get(url, headers=headers, **self._request_params)
    if res.ok:
        dataset = res.json()
        valid_to_array = dataset.get("validTo")
        valid_to_date = None
        valid_from_array = dataset.get("validFrom")
        valid_from_date = None
        if valid_to_array:
            valid_to_date = datetime.date(
                valid_to_array[0], valid_to_array[1], valid_to_array[2]
            )
        if valid_from_array:
            valid_from_date = datetime.date(
                valid_from_array[0], valid_from_array[1], valid_from_array[2]
            )
        return {"validTo": valid_to_date, "validFrom": valid_from_date}
    else:
        error_msg = self._try_extract_error_code(res)
        logger.error(error_msg)
        raise BoostedAPIException("Failed to query dataset: {0}.".format(error_msg))
def getDenseSignals(self, model_id, portfolio_id, file_name=None, location='./')
Expand source code
def getDenseSignals(self, model_id, portfolio_id, file_name=None, location="./"):
    url = self.base_uri + f"/api/models/{model_id}/{portfolio_id}/dense-signals"
    headers = {"Authorization": "ApiKey " + self.api_key}
    res = requests.get(url, headers=headers, **self._request_params)
    if file_name is None:
        file_name = f"{model_id}-{portfolio_id}_dense_signals.csv"
    download_location = os.path.join(location, file_name)
    if res.ok:
        with open(download_location, "wb") as file:
            file.write(res.content)
        print("Download Complete")
    elif res.status_code == 404:
        raise BoostedAPIException(
            f"""Dense Singals file does not exist for model:
             {model_id} - portfolio: {portfolio_id}"""
        )
    else:
        error_msg = self._try_extract_error_code(res)
        logger.error(error_msg)
        raise BoostedAPIException(
            f"""Failed to download dense singals file for model:
             {model_id} - portfolio: {portfolio_id}"""
        )
def getEquityAccuracy(self, model_id: str, portfolio_id: str, tickers: List[str], start_date: Optional[datetime.datetime] = None, end_date: Optional[datetime.datetime] = None) ‑> Dict[str, Dict[str, Any]]
Expand source code
def getEquityAccuracy(
    self,
    model_id: str,
    portfolio_id: str,
    tickers: List[str],
    start_date: Optional[datetime.datetime] = None,
    end_date: Optional[datetime.datetime] = None,
) -> Dict[str, Dict[str, Any]]:
    validate_start_and_end_dates(start_date, end_date)
    data = {}
    if start_date is not None:
        data["startDate"] = start_date
    if end_date is not None:
        data["endDate"] = end_date

    tickers_stream = ",".join(tickers)
    data["tickers"] = tickers_stream
    data["timestamp"] = time.strftime("%H:%M:%S")
    data["shouldRecalc"] = True
    url = self.base_uri + f"/api/analysis/equity-accuracy/{model_id}/{portfolio_id}"
    headers = {"Authorization": "ApiKey " + self.api_key, "Content-Type": "application/json"}

    logger.info(
        f"Retrieving equity accuracy data for date range {start_date} to {end_date} "
        f"for tickers: {tickers}."
    )

    # Now create dataframes from the JSON output.
    metrics = [
        "hit_rate_mean",
        "hit_rate_median",
        "excess_return_mean",
        "excess_return_median",
        "return",
        "excess_return",
    ]

    # send the request, retry if failed
    MAX_RETRIES = 10  # max of number of retries until timeout
    SLEEP_TIME = 3  # waiting time between requests

    num_retries = 0
    success = False
    while not success and num_retries < MAX_RETRIES:
        res = requests.post(url, data=json.dumps(data), headers=headers, **self._request_params)
        if res.ok:
            logger.info("Equity Accuracy Data retrieval successful.")
            info = res.json()
            success = True
        else:
            data["shouldRecalc"] = False
            num_retries += 1
            time.sleep(SLEEP_TIME)

    if not success:
        raise BoostedAPIException("Failed to retrieve equity accuracy: Request timeout.")

    for ticker, accuracy_data in info.items():
        for metric in metrics:
            metric_matrix = accuracy_data[metric]
            if not isinstance(metric_matrix, str):
                # Set the index to the quintile label, and remove it from the data
                index = []
                for row in metric_matrix[1:]:
                    index.append(row.pop(0))

                # columns are "1D", "5D", etc.
                df = pd.DataFrame(metric_matrix[1:], columns=metric_matrix[0][1:], index=index)
                accuracy_data[metric] = df
    return info
def getFeatureImportance(self, model_id, date, N=None)
Expand source code
def getFeatureImportance(self, model_id, date, N=None):
    url = self.base_uri + f"/api/analysis/explainability/{model_id}"
    headers = {"Authorization": "ApiKey " + self.api_key}
    logger.info("Retrieving rankings information for date {0}.".format(date))
    res = requests.get(url, headers=headers, **self._request_params)
    if not res.ok:
        error_msg = self._try_extract_error_code(res)
        logger.error(error_msg)
        raise BoostedAPIException(
            f"Failed to fetch feature importance for model/portfolio {model_id}: {error_msg}"
        )

    json_data = res.json()
    if "all" not in json_data.keys() or not json_data["all"]:
        raise BoostedAPIException(f"Unexpected formatting of feature importance response")

    feature_data = json_data["all"]
    # find the right period (assuming returned json has dates in descending order)
    date_obj = self.__to_date_obj(date)
    start_date_for_return_data = self.__to_date_obj(feature_data[0]["date"])
    features_for_requested_period = None

    if date_obj > start_date_for_return_data:
        features_for_requested_period = feature_data[0]["variable"]
    else:
        i = 0
        while i < len(feature_data) - 1:
            current_date = self.__to_date_obj(feature_data[i]["date"])
            next_date = self.__to_date_obj(feature_data[i + 1]["date"])
            if next_date <= date_obj <= current_date:
                features_for_requested_period = feature_data[i + 1]["variable"]
                start_date_for_return_data = next_date
                break
            i += 1

    if features_for_requested_period is None:
        raise BoostedAPIException(f"No feature data was found for requested date: {date_obj}")

    features_for_requested_period.sort(key=lambda x: x["value"], reverse=True)

    if type(N) is int and N > 0:
        df = pd.DataFrame.from_dict(features_for_requested_period[0:N])
    else:
        df = pd.DataFrame.from_dict(features_for_requested_period)
    result = df[["feature", "value"]]

    return result.rename(columns={"feature": f"feature ({start_date_for_return_data})"})
def getGbiIdFromIdentCountryCurrencyDate(self, ident_country_currency_dates: List[DateIdentCountryCurrency], timeout: int = 600) ‑> List[GbiIdSecurity]
Expand source code
def getGbiIdFromIdentCountryCurrencyDate(
    self, ident_country_currency_dates: List[DateIdentCountryCurrency], timeout: int = 600
) -> List[GbiIdSecurity]:
    url = self.base_uri + "/api/custom-stock-data/map-identifiers-simple"
    headers = {"Authorization": "ApiKey " + self.api_key, "Content-Type": "application/json"}
    identifiers = [
        {
            "row": idx,
            "date": identifier.date,
            "isin": identifier.identifier if identifier.id_type == ColumnSubRole.ISIN else None,
            "symbol": identifier.identifier
            if identifier.id_type == ColumnSubRole.SYMBOL
            else None,
            "countryPreference": identifier.country,
            "currencyPreference": identifier.currency,
        }
        for idx, identifier in enumerate(ident_country_currency_dates)
    ]
    params = json.dumps({"identifiers": identifiers})
    logger.info(
        "Retrieving GBI-ID mapping for {} identifier tuples...".format(
            len(ident_country_currency_dates)
        )
    )
    res = requests.post(url, data=params, headers=headers, **self._request_params)

    if res.ok:
        result = res.json()
        warnings = result["warnings"]
        if warnings:
            for warning in warnings:
                logger.warn(f"Mapping warning: {warning}")
        gbiSecurities = []
        for idx, ident in enumerate(result["mappedIdentifiers"]):
            if ident is None:
                security = None
            else:
                security = GbiIdSecurity(
                    ident["gbiId"],
                    ident_country_currency_dates[idx],
                    ident["symbol"],
                    ident["companyName"],
                )
            gbiSecurities.append(security)

        return gbiSecurities
    else:
        error_msg = self._try_extract_error_code(res)
        raise BoostedAPIException(
            "Failed to retrieve identifier mappings: {0}.".format(error_msg)
        )
def getGbiIdFromIsinCountryCurrencyDate(self, isin_country_currency_dates, timeout=600)
Expand source code
def getGbiIdFromIsinCountryCurrencyDate(self, isin_country_currency_dates, timeout=600):
    return self.getGbiIdFromIdentCountryCurrencyDate(
        ident_country_currency_dates=isin_country_currency_dates, timeout=timeout
    )
def getHistoricalTradeDates(self, portfolio_id, start_date=None, end_date=None)
Expand source code
def getHistoricalTradeDates(self, portfolio_id, start_date=None, end_date=None):
    end_date = self.__to_date_obj(end_date or datetime.date.today())
    start_date = self.__iso_format(start_date or (end_date - timedelta(days=365)))
    end_date = self.__iso_format(end_date)

    url = self.base_uri + "/api/portfolios/{0}/tradingDates".format(portfolio_id)
    headers = {"Authorization": "ApiKey " + self.api_key}
    params = {"startDate": start_date, "endDate": end_date}

    logger.info(
        "Retrieving historical trade dates data for date range {0} to {1}.".format(
            start_date, end_date
        )
    )
    res = requests.get(url, params=params, headers=headers, **self._request_params)
    if res.ok:
        logger.info("Trading dates retrieval successful.")
        return res.json()["dates"]
    else:
        error_msg = self._try_extract_error_code(res)
        raise BoostedAPIException("Failed to retrieve trading dates: {0}.".format(error_msg))
def getPortfolioSettings(self, portfolio_id, timeout=600)
Expand source code
def getPortfolioSettings(self, portfolio_id, timeout=600):
    url = self.base_uri + "/api/portfolio-settings/{0}".format(portfolio_id)
    headers = {"Authorization": "ApiKey " + self.api_key}
    res = requests.get(url, headers=headers, **self._request_params)
    if res.ok:
        return PortfolioSettings(res.json())
    else:
        error_msg = self._try_extract_error_code(res)
        logger.error(error_msg)
        raise BoostedAPIException(
            "Failed to retrieve portfolio settings: {0}.".format(error_msg)
        )
def getPortfolioStatus(self, model_id, portfolio_id, job_date)
Expand source code
def getPortfolioStatus(self, model_id, portfolio_id, job_date):
    url = (
        self.base_uri
        + f"/api/analysis/portfolioStatus/{model_id}/{portfolio_id}?jobDate={job_date}"
    )
    headers = {"Authorization": "ApiKey " + self.api_key, "Content-Type": "application/json"}
    res = requests.get(url, headers=headers, **self._request_params)
    if res.ok:
        result = res.json()
        return {
            "is_complete": result["status"],
            "last_update": None if result["lastUpdate"] is None else result["lastUpdate"][:10],
            "next_update": None if result["nextUpdate"] is None else result["nextUpdate"][:10],
        }
    else:
        error_msg = self._try_extract_error_code(res)
        logger.error(error_msg)
        raise BoostedAPIException("Failed to get portfolio status: {0}".format(error_msg))
def getRanking2DateAnalysisFile(self, model_id, portfolio_id, date, file_name=None, location='./')
Expand source code
def getRanking2DateAnalysisFile(
    self, model_id, portfolio_id, date, file_name=None, location="./"
):
    formatted_date = self.__iso_format(date)
    s3_file_name = f"{formatted_date}_analysis.xlsx"
    download_url = (
        self.base_uri + f"/api/models/{model_id}/{portfolio_id}/ranking-file/{s3_file_name}"
    )
    generate_url = (
        self.base_uri
        + f"/api/explain-trades/{model_id}/{portfolio_id}/generate/date-data/{formatted_date}"
    )
    headers = {"Authorization": "ApiKey " + self.api_key}
    if file_name is None:
        file_name = f"{model_id}-{portfolio_id}_statistical_analysis_{formatted_date}.xlsx"
    download_location = os.path.join(location, file_name)

    res = requests.get(download_url, headers=headers, **self._request_params)
    if res.ok:
        with open(download_location, "wb") as file:
            file.write(res.content)
        print("Download Complete")
    elif res.status_code == 404:
        generate_res = requests.get(generate_url, headers=headers, **self._request_params)
        if generate_res.ok:
            download_res = requests.get(download_url, headers=headers, **self._request_params)
            while download_res.status_code == 404 or (
                download_res.ok and len(download_res.content) == 0
            ):
                print("waiting for file to be generated")
                time.sleep(5)
                download_res = requests.get(
                    download_url, headers=headers, **self._request_params
                )
            if download_res.ok:
                with open(download_location, "wb") as file:
                    file.write(download_res.content)
                print("Download Complete")
        else:
            error_msg = self._try_extract_error_code(res)
            logger.error(error_msg)
            raise BoostedAPIException(
                f"""Failed to generate ranking analysis file for model:
                {model_id} - portfolio: {portfolio_id} on date: {formatted_date}"""
            )
    else:
        error_msg = self._try_extract_error_code(res)
        logger.error(error_msg)
        raise BoostedAPIException(
            f"""Failed to download ranking analysis file for model:
             {model_id} - portfolio: {portfolio_id} on date: {formatted_date}"""
        )
def getRanking2DateExplainFile(self, model_id, portfolio_id, date, file_name=None, location='./', overwrite: bool = False)
Expand source code
def getRanking2DateExplainFile(
    self, model_id, portfolio_id, date, file_name=None, location="./", overwrite: bool = False
):
    formatted_date = self.__iso_format(date)
    s3_file_name = f"{formatted_date}_explaindata.xlsx"
    download_url = (
        self.base_uri + f"/api/models/{model_id}/{portfolio_id}/ranking-file/{s3_file_name}"
    )
    generate_url = (
        self.base_uri
        + f"/api/explain-trades/{model_id}/{portfolio_id}/generate/date-data/{formatted_date}"
    )
    headers = {"Authorization": "ApiKey " + self.api_key}
    if file_name is None:
        file_name = f"{model_id}-{portfolio_id}_explain_data_{formatted_date}.xlsx"
    download_location = os.path.join(location, file_name)

    if not overwrite:
        res = requests.get(download_url, headers=headers, **self._request_params)
    if not overwrite and res.ok:
        with open(download_location, "wb") as file:
            file.write(res.content)
        print("Download Complete")
    elif overwrite or res.status_code == 404:
        generate_res = requests.get(generate_url, headers=headers, **self._request_params)
        if generate_res.ok:
            download_res = requests.get(download_url, headers=headers, **self._request_params)
            while download_res.status_code == 404 or (
                download_res.ok and len(download_res.content) == 0
            ):
                print("waiting for file to be generated")
                time.sleep(5)
                download_res = requests.get(
                    download_url, headers=headers, **self._request_params
                )
            if download_res.ok:
                with open(download_location, "wb") as file:
                    file.write(download_res.content)
                print("Download Complete")
        else:
            error_msg = self._try_extract_error_code(res)
            logger.error(error_msg)
            raise BoostedAPIException(
                f"""Failed to generate ranking explain file for model:
                {model_id} - portfolio: {portfolio_id} on date: {formatted_date}"""
            )
    else:
        error_msg = self._try_extract_error_code(res)
        logger.error(error_msg)
        raise BoostedAPIException(
            f"""Failed to download ranking explain file for model:
             {model_id} - portfolio: {portfolio_id} on date: {formatted_date}"""
        )
def getRankingAnalysis(self, model_id, date)
Expand source code
def getRankingAnalysis(self, model_id, date):
    url = (
        self.base_uri
        + f"/api/explain-trades/analysis/{model_id}/{self.__iso_format(date)}/json"
    )
    headers = {"Authorization": "ApiKey " + self.api_key}
    analysis_res = requests.get(url, headers=headers, **self._request_params)
    if analysis_res.ok:
        ranking_dict = analysis_res.json()
        feature_name_dict = self.__get_rankings_ref_translation(model_id)
        columns = [feature_name_dict[col] for col in ranking_dict["columns"]]

        df = protoCubeJsonDataToDataFrame(
            ranking_dict["data"],
            "Data Buckets",
            ranking_dict["rows"],
            "Feature Names",
            columns,
            ranking_dict["fields"],
        )
        return df
    else:
        error_msg = self._try_extract_error_code(analysis_res)
        logger.error(error_msg)
        raise BoostedAPIException("Failed to get ranking analysis: {0}.".format(error_msg))
def getRankingExplain(self, model_id, date)
Expand source code
def getRankingExplain(self, model_id, date):
    url = (
        self.base_uri + f"/api/explain-trades/explain/{model_id}/{self.__iso_format(date)}/json"
    )
    headers = {"Authorization": "ApiKey " + self.api_key, "Content-Type": "application/json"}
    explain_res = requests.get(url, headers=headers, **self._request_params)
    if explain_res.ok:
        ranking_dict = explain_res.json()
        rows = ranking_dict["rows"]
        stock_summary_url = f"/api/stock-summaries/{model_id}"
        stock_summary_body = {"gbiIds": ranking_dict["rows"]}
        summary_res = requests.post(
            self.base_uri + stock_summary_url,
            data=json.dumps(stock_summary_body),
            headers=headers,
            **self._request_params,
        )
        if summary_res.ok:
            stock_summary = summary_res.json()
            rows = [stock_summary[row]["isin"] for row in ranking_dict["rows"]]
        else:
            error_msg = self._try_extract_error_code(summary_res)
            logger.error(error_msg)
            raise BoostedAPIException(
                "Failed to get isin information ranking explain: {0}.".format(error_msg)
            )

        feature_name_dict = self.__get_rankings_ref_translation(model_id)
        columns = [feature_name_dict[col] for col in ranking_dict["columns"]]

        df = protoCubeJsonDataToDataFrame(
            ranking_dict["data"],
            "ISINs",
            rows,
            "Feature Names",
            columns,
            ranking_dict["fields"],
        )
        return df
    else:
        error_msg = self._try_extract_error_code(explain_res)
        logger.error(error_msg)
        raise BoostedAPIException("Failed to get ranking explain: {0}.".format(error_msg))
def getRankingsForAllDates(self, portfolio_id, dates=None)
Expand source code
def getRankingsForAllDates(self, portfolio_id, dates=None):
    url = self.base_uri + "/api/portfolios/{0}/rankingsByDate".format(portfolio_id)
    headers = {"Authorization": "ApiKey " + self.api_key}
    params = {}
    if dates is not None:
        fmt_dates = []
        for d in dates:
            fmt_dates.append(self.__iso_format(d))
        fmt_dates = ",".join(fmt_dates)
        params = {"dates": fmt_dates}
        logger.info("Retrieving rankings information for date {0}.".format(fmt_dates))
    else:
        params = {"dates": None}
        logger.info("Retrieving rankings information for all dates")
    res = requests.get(url, params=params, headers=headers, **self._request_params)
    if res.ok:
        logger.info("Rankings retrieval successful.")
        return res.json()
    else:
        error_msg = self._try_extract_error_code(res)
        raise BoostedAPIException("Failed to retrieve rankings: {0}.".format(error_msg))
def getRankingsForDate(self, portfolio_id, date, rollback_to_last_available_date)
Expand source code
def getRankingsForDate(self, portfolio_id, date, rollback_to_last_available_date):
    date = self.__iso_format(date)
    endpoint = "latest-rankings" if rollback_to_last_available_date else "rankings"
    url = self.base_uri + "/api/{0}/{1}/{2}".format(endpoint, portfolio_id, date)
    headers = {"Authorization": "ApiKey " + self.api_key}
    logger.info("Retrieving rankings information for date {0}.".format(date))
    res = requests.get(url, headers=headers, **self._request_params)
    if res.ok:
        logger.info("Rankings retrieval successful.")
        return res.json()
    else:
        error_msg = self._try_extract_error_code(res)
        raise BoostedAPIException("Failed to retrieve rankings: {0}.".format(error_msg))
def getSignalsForAllDates(self, portfolio_id, dates=None)
Expand source code
def getSignalsForAllDates(self, portfolio_id, dates=None):
    url = self.base_uri + "/api/portfolios/{0}/signalsByDate".format(portfolio_id)
    headers = {"Authorization": "ApiKey " + self.api_key}
    params = {}
    if dates is not None:
        fmt_dates = []
        for d in dates:
            fmt_dates.append(self.__iso_format(d))
        fmt_dates = ",".join(fmt_dates)
        params = {"dates": fmt_dates}
        logger.info("Retrieving signals information for dates {0}.".format(fmt_dates))
    else:
        params = {"dates": None}
        logger.info("Retrieving signals information for all dates")
    res = requests.get(url, params=params, headers=headers, **self._request_params)
    if res.ok:
        logger.info("Signals retrieval successful.")
        return res.json()
    else:
        error_msg = self._try_extract_error_code(res)
        raise BoostedAPIException("Failed to retrieve signals: {0}.".format(error_msg))
def getSignalsForDate(self, portfolio_id, date, rollback_to_last_available_date)
Expand source code
def getSignalsForDate(self, portfolio_id, date, rollback_to_last_available_date):
    date = self.__iso_format(date)
    endpoint = "latest-signals" if rollback_to_last_available_date else "signals"
    url = self.base_uri + "/api/portfolios/{0}/{1}".format(portfolio_id, endpoint)
    headers = {"Authorization": "ApiKey " + self.api_key}
    params = {"date": date}
    logger.info("Retrieving signals information for date {0}.".format(date))
    res = requests.get(url, params=params, headers=headers, **self._request_params)
    if res.ok:
        logger.info("Signals retrieval successful.")
        return res.json()
    else:
        error_msg = self._try_extract_error_code(res)
        raise BoostedAPIException("Failed to retrieve signals: {0}.".format(error_msg))
def getSignalsFromUploadedModel(self, model_id, date=None)
Expand source code
def getSignalsFromUploadedModel(self, model_id, date=None):
    date = self.__iso_format(date)
    url = self.base_uri + "/api/models/{0}/upload/signals".format(model_id)
    headers = {"Authorization": "ApiKey " + self.api_key}
    params = {"date": date}
    logger.info("Retrieving uploaded signals information")
    res = requests.get(url, params=params, headers=headers, **self._request_params)
    if res.ok:
        result = pd.DataFrame.from_dict(res.json()["result"])
        # ensure column order
        result = result[["date", "isin", "country", "currency", "weight"]]
        result["date"] = pd.to_datetime(result["date"], format="%Y-%m-%d")
        result = result.set_index("date")
        logger.info("Signals retrieval successful.")
        return result
    else:
        error_msg = self._try_extract_error_code(res)
        raise BoostedAPIException("Failed to retrieve signals: {0}.".format(error_msg))
def getStockDataTableForDate(self, model_id: str, portfolio_id: str, date: datetime.date) ‑> pandas.core.frame.DataFrame
Expand source code
def getStockDataTableForDate(
    self, model_id: str, portfolio_id: str, date: datetime.date
) -> pd.DataFrame:
    headers = {"Authorization": "ApiKey " + self.api_key, "Content-Type": "application/json"}

    url_base = f"{self.base_uri}/api/analysis"
    url_params = f"{model_id}/{portfolio_id}"
    formatted_date = date.strftime("%Y-%m-%d")

    stock_prices_url = f"{url_base}/stock-prices/{url_params}/{formatted_date}"
    stock_factors_url = f"{url_base}/stock-factors/{url_params}/date/{formatted_date}"

    prices_params = {"useTicker": "true"}

    prices_resp = requests.get(
        stock_prices_url, headers=headers, params=prices_params, **self._request_params
    )
    factors_resp = requests.get(stock_factors_url, headers=headers, **self._request_params)

    frames = []
    for res in (prices_resp, factors_resp):
        if not res.ok:
            error_msg = self._try_extract_error_code(res)
            logger.error(error_msg)
            raise BoostedAPIException(
                f"Failed to fetch stock data table for model {model_id}: {error_msg=}"
            )
        result = res.json()
        frames.append(pd.DataFrame(result))

    output_df = pd.concat(frames)
    return output_df
def getTearSheet(self, model_id, portfolio_id, start_date=None, end_date=None, block=False)
Expand source code
def getTearSheet(self, model_id, portfolio_id, start_date=None, end_date=None, block=False):
    if start_date is None or end_date is None:
        if start_date is not None or end_date is not None:
            raise ValueError("start_date and end_date must both be None or both be defined")
        return self._getCurrentTearSheet(model_id, portfolio_id)

    start_date_obj = self.__to_date_obj(start_date)
    end_date_obj = self.__to_date_obj(end_date)
    if start_date_obj >= end_date_obj:
        raise ValueError("end_date must be later than the start_date")

    # get for the given date
    url = self.base_uri + f"/api/analysis/keyfacts/{model_id}/{portfolio_id}"
    data = {
        "startDate": self.__iso_format(start_date),
        "endDate": self.__iso_format(end_date),
        "shouldRecalc": True,
    }
    headers = {"Authorization": "ApiKey " + self.api_key, "Content-Type": "application/json"}
    res = requests.post(url, data=json.dumps(data), headers=headers, **self._request_params)
    if res.status_code == 404 and block:
        retries = 0
        data["shouldRecalc"] = False
        while retries < 10:
            time.sleep(10)
            retries += 1
            res = requests.post(
                url, data=json.dumps(data), headers=headers, **self._request_params
            )
            if res.status_code != 404:
                break
    if res.ok:
        return res.json()
    else:
        error_msg = self._try_extract_error_code(res)
        logger.error(error_msg)
        raise BoostedAPIException(
            "Failed to get tear sheet data: {0} {1}.".format(error_msg, str(res.status_code))
        )
def getTradeExplain(self, portfolio_id, date=None)
Expand source code
def getTradeExplain(self, portfolio_id, date=None):
    url = self.base_uri + f"/api/explain/{portfolio_id}"
    explain_date = self.__iso_format(date)
    if explain_date:
        url = self.base_uri + f"/api/explain/{portfolio_id}/{explain_date}"
    headers = {"Authorization": "ApiKey " + self.api_key}
    res = requests.get(url, headers=headers, **self._request_params)
    if res.ok:
        buf = io.StringIO(res.text)
        df = pd.read_csv(buf, index_col=0, parse_dates=True)
        return df
    else:
        error_msg = self._try_extract_error_code(res)
        logger.error(error_msg)
        raise BoostedAPIException("Failed to get trade explain: {0}.".format(error_msg))
def getUniverse(self, modelId, date=None)
Expand source code
def getUniverse(self, modelId, date=None):
    if date is not None:
        url = "/api/models/{0}/universe/{1}".format(modelId, self.__iso_format(date))
        logger.info("Getting universe for date: {0}.".format(date))
    else:
        url = "/api/models/{0}/universe/".format(modelId)
    headers = {"Authorization": "ApiKey " + self.api_key}
    res = requests.get(self.base_uri + url, headers=headers, **self._request_params)
    if res.ok:
        buf = io.StringIO(res.text)
        df = pd.read_csv(buf, index_col=0, parse_dates=True)
        return df
    else:
        error = self._try_extract_error_code(res)
        logger.error(
            "There was a problem getting this universe or model ID: {0}.".format(error)
        )
        raise BoostedAPIException("Failed to get universe: {0}".format(error))
def get_coverage_csv(self, watchlist_id: str, portfolio_group_id: str, filepath: Optional[str] = None) ‑> Optional[str]

Converts the coverage contents to CSV format Parameters


watchlist_id : str
UUID str identifying the coverage watchlist
portfolio_group_id : str
UUID str identifying the group of portfolio to use for analysis
filepath : Optional[str]
UUID str identifying the group of portfolio to use for analysis

Returns:

None if filepath is provided, else a string with a csv's contents is returned

Expand source code
def get_coverage_csv(
    self, watchlist_id: str, portfolio_group_id: str, filepath: Optional[str] = None
) -> Optional[str]:
    """
    Converts the coverage contents to CSV format
    Parameters
    ----------
    watchlist_id: str
        UUID str identifying the coverage watchlist
    portfolio_group_id: str
        UUID str identifying the group of portfolio to use for analysis
    filepath: Optional[str]
        UUID str identifying the group of portfolio to use for analysis

    Returns:
    ----------
    None if filepath is provided, else a string with a csv's contents is returned
    """

    df = self.get_coverage_info(watchlist_id, portfolio_group_id)

    return df.to_csv(filepath, index=False, float_format="%.4f")
def get_coverage_info(self, watchlist_id: str, portfolio_group_id: str) ‑> pandas.core.frame.DataFrame
Expand source code
def get_coverage_info(self, watchlist_id: str, portfolio_group_id: str) -> pd.DataFrame:

    # get securities list in watchlist
    watchlist_details = self.get_watchlist_details(watchlist_id)
    security_list = watchlist_details["targets"]

    gbi_ids = [x["gbi_id"] for x in security_list]

    gbi_data = {x: {} for x in gbi_ids}

    # get security info ticker, name, industry etc
    sec_info = self._get_security_info(gbi_ids)

    for sec in sec_info["data"]["securities"]:
        gbi_id = sec["gbiId"]
        for k in ["symbol", "name", "isin", "country", "currency"]:
            gbi_data[gbi_id][self._coverage_column_name_format(k)] = sec[k]

        gbi_data[gbi_id][self._coverage_column_name_format("Sector")] = sec["sector"][
            "topParentName"
        ]

    # get portfolios list in portfolio_Group
    portfolio_group = self.get_portfolio_group(portfolio_group_id)
    portfolio_ids = [x["portfolio_id"] for x in portfolio_group["portfolios"]]
    portfolio_info = {x["portfolio_id"]: x for x in portfolio_group["portfolios"]}

    model_resp = self._get_models_for_portfolio(portfolio_ids=portfolio_ids)
    for portfolio in model_resp["data"]["portfolios"]:
        portfolio_info[portfolio["id"]].update(portfolio)

    model_info = {
        x["modelId"]: portfolio_info[x["id"]] for x in model_resp["data"]["portfolios"]
    }

    # model_ids and portfolio_ids are parallel arrays
    model_ids = [portfolio_info[x]["modelId"] for x in portfolio_ids]

    # graphql: get watchlist analysis
    wl_analysis = self._get_watchlist_analysis(
        gbi_ids=gbi_ids,
        model_ids=model_ids,
        portfolio_ids=portfolio_ids,
        asof_date=datetime.date.today(),
    )

    portfolio_gbi_data = {k: {} for k in portfolio_ids}
    for pi, v in portfolio_gbi_data.items():
        v.update({k: {} for k in gbi_data.keys()})

    equity_explorer_date = wl_analysis["data"]["watchlistAnalysis"][0]["analysisDates"][0][
        "date"
    ]
    for wla in wl_analysis["data"]["watchlistAnalysis"]:
        gbi_id = wla["gbiId"]
        gbi_data[gbi_id]["Composite Rating"] = wla["analysisDates"][0]["aggregateSignal"][
            "rating"
        ]
        gbi_data[gbi_id]["Composite Rating Delta"] = wla["analysisDates"][0]["aggregateSignal"][
            "ratingDelta"
        ]

        for p in wla["analysisDates"][0]["portfoliosSignals"]:
            model_name = portfolio_info[p["portfolioId"]]["modelName"]

            portfolio_gbi_data[p["portfolioId"]][gbi_id][
                model_name + self._coverage_column_name_format(": rank")
            ] = (p["rank"] + 1)
            portfolio_gbi_data[p["portfolioId"]][gbi_id][
                model_name + self._coverage_column_name_format(": rank delta")
            ] = (-1 * p["signalDelta"])
            portfolio_gbi_data[p["portfolioId"]][gbi_id][
                model_name + self._coverage_column_name_format(": rating")
            ] = p["rating"]
            portfolio_gbi_data[p["portfolioId"]][gbi_id][
                model_name + self._coverage_column_name_format(": rating delta")
            ] = p["ratingDelta"]

    neg_rec = {k: {} for k in gbi_data.keys()}
    pos_rec = {k: {} for k in gbi_data.keys()}
    for wla in wl_analysis["data"]["watchlistAnalysis"]:
        gbi_id = wla["gbiId"]

        for pid, signals in zip(portfolio_ids, wla["analysisDates"][0]["portfoliosSignals"]):
            model_name = portfolio_info[pid]["modelName"]
            neg_rec[gbi_id][
                model_name + self._coverage_column_name_format(": negative recommendation")
            ] = signals["explainWeightNeg"]
            pos_rec[gbi_id][
                model_name + self._coverage_column_name_format(": positive recommendation")
            ] = signals["explainWeightPos"]

    # graphql: GetExcessReturn - slugging pct
    er_sp = self._get_excess_return(
        model_ids=model_ids, gbi_ids=gbi_ids, asof_date=equity_explorer_date
    )

    for model in er_sp["data"]["models"]:
        model_name = model_info[model["id"]]["modelName"]
        for stat in model["equityExplorerData"]["equityExplorerSummaryStatistics"]:
            portfolioId = model_info[model["id"]]["id"]
            portfolio_gbi_data[portfolioId][int(stat["gbiId"])][
                model_name + self._coverage_column_name_format(": slugging %")
            ] = (stat["ER"]["SP"]["oneMonth"] * 100)

    # add rank, rating, slugging
    for pid, v in portfolio_gbi_data.items():
        for gbi_id, vv in v.items():
            gbi_data[gbi_id].update(vv)

    # add neg/pos rec scores
    for rec in [neg_rec, pos_rec]:
        for k, v in rec.items():
            gbi_data[k].update(v)

    df = pd.DataFrame.from_records([v for _, v in gbi_data.items()])

    return df
def get_csv_buffer(self)
Expand source code
def get_csv_buffer(self):
    return io.StringIO()
def get_dataset_schema(self, dataset_id)
Expand source code
def get_dataset_schema(self, dataset_id):
    url = self.base_uri + "/api/datasets/{0}/schema".format(dataset_id)
    headers = {"Authorization": "ApiKey " + self.api_key}
    res = requests.get(url, headers=headers, **self._request_params)
    if res.ok:
        json_schema = res.json()
    else:
        error_msg = self._try_extract_error_code(res)
        logger.error(error_msg)
        raise BoostedAPIException("Failed to query dataset: {0}.".format(error_msg))
    return DataSetConfig.fromDict(json_schema["result"])
def get_hedge_experiment_details(self, experiment_id: str)
Expand source code
def get_hedge_experiment_details(self, experiment_id: str):
    url = self.base_uri + "/api/graphql"
    qry = """
        query getHedgeExperimentDetails($hedgeExperimentId: ID!) {
            hedgeExperiment(hedgeExperimentId: $hedgeExperimentId) {
            ...HedgeExperimentDetailsSummaryListFragment
            }
        }

        fragment HedgeExperimentDetailsSummaryListFragment on HedgeExperiment {
            hedgeExperimentId
            experimentName
            userId
            config
            description
            experimentType
            lastCalculated
            lastModified
            status
            portfolioCalcStatus
            targetSecurities {
                gbiId
                security {
                    gbiId
                    symbol
                    name
                }
                weight
            }
            selectedModels {
                id
                name
                stockUniverse {
                    name
                }
            }
            hedgeExperimentScenarios {
                ...experimentScenarioFragment
            }
            selectedDummyHedgeExperimentModels {
                id
                name
                stockUniverse {
                    name
                }
            }
            targetPortfolios {
                portfolioId
            }
            baselineModel {
                id
                name

            }
            baselineScenario {
                hedgeExperimentScenarioId
                scenarioName
                description
                portfolioSettingsJson
                hedgeExperimentPortfolios {
                    portfolio {
                        id
                        name
                        modelId
                        performanceGridHeader
                        performanceGrid
                        status
                        tearSheet {
                            groupName
                            members {
                                name
                                value
                            }
                        }
                    }
                }
                status
            }
            baselineStockUniverseId
        }

        fragment experimentScenarioFragment on HedgeExperimentScenario {
            hedgeExperimentScenarioId
            scenarioName
            status
            description
            portfolioSettingsJson
            hedgeExperimentPortfolios {
                portfolio {
                    id
                    name
                    modelId
                    performanceGridHeader
                    performanceGrid
                    status
                    tearSheet {
                        groupName
                        members {
                            name
                            value
                        }
                    }
                }
            }
        }
    """
    headers = {"Authorization": "ApiKey " + self.api_key}
    resp = requests.post(
        url,
        json={"query": qry, "variables": {"hedgeExperimentId": experiment_id}},
        headers=headers,
        params=self._request_params,
    )

    json_resp = resp.json()
    # graphql endpoints typically return 200 or 400 status codes, so we must
    # check if we have any errors, even with a 200
    if (resp.ok and "errors" in json_resp) or not resp.ok:
        error_msg = self._try_extract_error_code(resp)
        logger.error(error_msg)
        raise BoostedAPIException(
            (
                f"Failed to get hedge experiment results for {experiment_id=}: "
                f"{resp.status_code=}; {error_msg=}"
            )
        )

    json_exp_results = json_resp["data"]["hedgeExperiment"]
    if json_exp_results is None:
        return None  # issued a request with a non-existent experiment_id
    exp_results = HedgeExperimentDetails.from_json_dict(json_exp_results)
    return exp_results
def get_hedge_experiments(self)
Expand source code
def get_hedge_experiments(self):
    url = self.base_uri + "/api/graphql"
    qry = """
        query getHedgeExperiments {
            hedgeExperiments {
                hedgeExperimentId
                experimentName
                userId
                config
                description
                experimentType
                lastCalculated
                lastModified
                status
                portfolioCalcStatus
                targetSecurities {
                    gbiId
                    security {
                        gbiId
                        symbol
                        name
                    }
                    weight
                }
                targetPortfolios {
                    portfolioId
                }
                baselineModel {
                    id
                    name

                }
                baselineScenario {
                    hedgeExperimentScenarioId
                    scenarioName
                    description
                    portfolioSettingsJson
                    hedgeExperimentPortfolios {
                        portfolio {
                            id
                            name
                            modelId
                            performanceGridHeader
                            performanceGrid
                            status
                            tearSheet {
                                groupName
                                members {
                                    name
                                    value
                                }
                            }
                        }
                    }
                    status
                }
                baselineStockUniverseId
            }
        }
    """

    headers = {"Authorization": "ApiKey " + self.api_key}
    resp = requests.post(url, json={"query": qry}, headers=headers, params=self._request_params)

    json_resp = resp.json()
    # graphql endpoints typically return 200 or 400 status codes, so we must
    # check if we have any errors, even with a 200
    if (resp.ok and "errors" in json_resp) or not resp.ok:
        error_msg = self._try_extract_error_code(resp)
        logger.error(error_msg)
        raise BoostedAPIException(
            (f"Failed to get hedge experiments: {resp.status_code=}; {error_msg=}")
        )

    json_experiments = resp.json()["data"]["hedgeExperiments"]
    experiments = [HedgeExperiment.from_json_dict(exp_json) for exp_json in json_experiments]
    return experiments
def get_inference(self, model_id, inference_date=datetime.date(2022, 9, 15), block=False, timeout_minutes=30)
Expand source code
def get_inference(
    self, model_id, inference_date=datetime.date.today(), block=False, timeout_minutes=30
):
    start_time = datetime.datetime.now()
    while True:
        for numRetries in range(3):
            res, status = self._get_inference(model_id, inference_date)
            if res is not None:
                continue
            else:
                if status == Status.FAIL:
                    return Status.FAIL
                logger.info("Retrying...")
        if res is None:
            logger.error("Max retries reached.  Request failed.")
            return None

        json_data = res.json()
        if "result" in json_data.keys():
            if json_data["result"]["status"] == "RUNNING":
                still_running = True
                if not block:
                    logger.warn("Inference job is still running.")
                    return None
                else:
                    logger.info(
                        "Inference job is still running.  Time elapsed={0}.".format(
                            datetime.datetime.now() - start_time
                        )
                    )
                    time.sleep(10)
            else:
                still_running = False

            if not still_running and json_data["result"]["status"] == "COMPLETE":
                csv = json_data["result"]["signals"]
                logger.info(json_data["result"])
                if self._check_status_code(res, isInference=True):
                    logger.info(
                        "Total run time = {0}.".format(datetime.datetime.now() - start_time)
                    )
                    return csv
        else:
            if "errors" in json_data.keys():
                logger.error(json_data["errors"])
            else:
                logger.error("Error getting inference for date {0}.".format(inference_date))
            return None
        if (datetime.datetime.now() - start_time).total_seconds() / 60.0 > timeout_minutes:
            logger.error("Timeout waiting for job completion.")
            return None
def get_portfolio_accuracy(self, model_id: str, portfolio_id: str) ‑> dict
Expand source code
def get_portfolio_accuracy(self, model_id: str, portfolio_id: str) -> dict:
    # TODO: Later change this URI to not use the watchlist prefix. It is misnamed.
    url = f"{self.base_uri}{WATCHLIST_ROUTE_PREFIX}{DAL_PA_ROUTE}/get-hit-rate/"
    headers = {"Authorization": "ApiKey " + self.api_key, "Content-Type": "application/json"}
    req_json = {"model_id": model_id, "portfolio_id": portfolio_id}
    res = requests.post(url, json=req_json, headers=headers, **self._request_params)

    if not res.ok:
        error_msg = self._try_extract_error_code(res)
        logger.error(error_msg)
        raise BoostedAPIException(f"Failed to get Hit Rate: {error_msg}")

    data = res.json()
    return data
def get_portfolio_factors(self, model_id: str, portfolio_id: str) ‑> pandas.core.frame.DataFrame
Expand source code
def get_portfolio_factors(self, model_id: str, portfolio_id: str) -> pd.DataFrame:
    url = f"{self.base_uri}/api/analysis/factors/{model_id}/{portfolio_id}"
    headers = {"Authorization": "ApiKey " + self.api_key}
    resp = requests.get(url, headers=headers, params=self._request_params)

    json_resp = resp.json()
    if (resp.ok and "errors" in json_resp) or not resp.ok:
        error_msg = json_resp["errors"][0]
        if self._is_portfolio_still_running(error_msg):
            return pd.DataFrame()
        logger.error(error_msg)
        raise BoostedAPIException(
            (
                f"Failed to get portfolio factors for {portfolio_id=}: "
                f"{resp.status_code=}; {error_msg=}"
            )
        )

    df = pd.DataFrame(json_resp["data"], columns=json_resp["header_row"])

    def to_lower_snake_case(s):  # why are we linting lambdas? :(
        return "_".join(w.lower() for w in s.split(" "))

    df = df.rename(columns={old: to_lower_snake_case(old) for old in df.columns}).set_index(
        "date"
    )
    df.index = pd.to_datetime(df.index)
    return df
def get_portfolio_group(self, portfolio_group_id: str) ‑> Dict[~KT, ~VT]

Parameters: portfolio_group_id: str UUID identifier for the portfolio group

Returns:

PortfolioGroup: Dict: { group_id: str group_name: str portfolios: List[PortfolioInGroup] } where PortfolioInGroup is defined as = Dict { portfolio_id: str portfolio_name: str rank_in_group: Optional[int] }

Expand source code
def get_portfolio_group(self, portfolio_group_id: str) -> Dict:
    """
    Parameters:
    portfolio_group_id: str
       UUID identifier for the portfolio group


    Returns:
    ----------

    PortfolioGroup: Dict:  {
    group_id: str
    group_name: str
    portfolios: List[PortfolioInGroup]
    }
    where PortfolioInGroup is defined as = Dict {
    portfolio_id: str
    portfolio_name: str
    rank_in_group: Optional[int]
    }
    """
    url = f"{self.base_uri}{WATCHLIST_ROUTE_PREFIX}{PORTFOLIO_GROUP_ROUTE}/get-one"
    headers = {"Authorization": "ApiKey " + self.api_key, "Content-Type": "application/json"}
    req_json = {"portfolio_group_id": portfolio_group_id}
    res = requests.post(url, json=req_json, headers=headers, **self._request_params)

    if not res.ok:
        error_msg = self._try_extract_error_code(res)
        logger.error(error_msg)
        raise BoostedAPIException(f"Failed to get user portfolio groups: {error_msg}")

    data = res.json()
    return data
def get_portfolio_groups(self) ‑> Dict[~KT, ~VT]

Parameters: None

Returns:

Dict: { user_id: str portfolio_groups: List[PortfolioGroup] } where PortfolioGroup is defined as = Dict { group_id: str group_name: str portfolios: List[PortfolioInGroup] } where PortfolioInGroup is defined as = Dict { portfolio_id: str rank_in_group: Optional[int] }

Expand source code
def get_portfolio_groups(
    self,
) -> Dict:
    """
    Parameters: None


    Returns:
    ----------

    Dict:  {
    user_id: str
    portfolio_groups: List[PortfolioGroup]
    }
    where PortfolioGroup is defined as = Dict {
    group_id: str
    group_name: str
    portfolios: List[PortfolioInGroup]
    }
    where PortfolioInGroup is defined as = Dict {
    portfolio_id: str
    rank_in_group: Optional[int]
    }
    """
    url = f"{self.base_uri}{WATCHLIST_ROUTE_PREFIX}{PORTFOLIO_GROUP_ROUTE}/get"
    headers = {"Authorization": "ApiKey " + self.api_key, "Content-Type": "application/json"}
    req_json = {}
    res = requests.post(url, json=req_json, headers=headers, **self._request_params)

    if not res.ok:
        error_msg = self._try_extract_error_code(res)
        logger.error(error_msg)
        raise BoostedAPIException(f"Failed to get user portfolio groups: {error_msg}")

    data = res.json()
    return data
def get_portfolio_holdings(self, model_id: str, portfolio_id: str) ‑> pandas.core.frame.DataFrame
Expand source code
def get_portfolio_holdings(self, model_id: str, portfolio_id: str) -> pd.DataFrame:
    url = f"{self.base_uri}/api/models/{model_id}/{portfolio_id}/basket-data"
    headers = {"Authorization": "ApiKey " + self.api_key}
    resp = requests.get(url, headers=headers, params=self._request_params)

    # this is a classic abuse of try/except as control flow: we try to get json body
    # from the response so that we can error-check. if this fails, we assume we have
    # a legit text response (corresponding to the csv data we care about)
    try:
        json_resp = resp.json()
    except json.decoder.JSONDecodeError:
        df = pd.read_csv(io.StringIO(resp.text), header=[0])
    else:
        error_msg = json_resp["errors"][0]
        if self._is_portfolio_still_running(error_msg):
            return pd.DataFrame()
        else:
            logger.error(error_msg)
            raise BoostedAPIException(
                (
                    f"Failed to get portfolio holdings for {portfolio_id=}: "
                    f"{resp.status_code=}; {error_msg=}"
                )
            )

    df = df.rename(columns={old: old.lower() for old in df.columns}).set_index("date")
    df.index = pd.to_datetime(df.index)
    return df
def get_portfolio_performance(self, portfolio_id: str) ‑> pandas.core.frame.DataFrame
Expand source code
def get_portfolio_performance(self, portfolio_id: str) -> pd.DataFrame:
    url = f"{self.base_uri}/api/graphql"
    qry = """
        query getPortfolioPerformance($portfolioId: ID!) {
            portfolio(id: $portfolioId) {
                id
                modelId
                name
                status
                performance {
                    benchmark
                    date
                    turnover
                    value
                }
            }
        }
    """

    headers = {"Authorization": "ApiKey " + self.api_key}
    resp = requests.post(
        url,
        json={"query": qry, "variables": {"portfolioId": portfolio_id}},
        headers=headers,
        params=self._request_params,
    )

    json_resp = resp.json()
    # the webserver returns an error for non-ready portfolios, so we have to check
    # for this prior to the error check below
    pf = json_resp["data"].get("portfolio")
    if pf is not None and pf["status"] != "READY":
        return pd.DataFrame()

    # graphql endpoints typically return 200 or 400 status codes, so we must
    # check if we have any errors, even with a 200
    if (resp.ok and "errors" in json_resp) or not resp.ok:
        error_msg = self._try_extract_error_code(resp)
        logger.error(error_msg)
        raise BoostedAPIException(
            (
                f"Failed to get portfolio performance for {portfolio_id=}: "
                f"{resp.status_code=}; {error_msg=}"
            )
        )

    perf = json_resp["data"]["portfolio"]["performance"]
    df = pd.DataFrame(perf).set_index("date").rename(columns={"value": "portfolio"})
    df.index = pd.to_datetime(df.index)
    return df.astype(float)
def get_portfolio_quantiles(self, model_id: str, portfolio_id: str, id_type: Literal['TICKER', 'ISIN'] = 'TICKER')
Expand source code
def get_portfolio_quantiles(
    self,
    model_id: str,
    portfolio_id: str,
    id_type: Literal["TICKER", "ISIN"] = "TICKER",
):
    headers = {"Authorization": "ApiKey " + self.api_key, "Content-Type": "application/json"}
    date = datetime.date.today().strftime("%Y-%m-%d")

    payload = {
        "model_id": model_id,
        "portfolio_id": portfolio_id,
        "fields": ["quantile"],
        "min_date": date,
        "max_date": date,
        "return_format": "json",
    }
    # TODO: Later change this URI to not use the watchlist prefix. It is misnamed.
    url = f"{self.base_uri}{WATCHLIST_ROUTE_PREFIX}{DAL_PA_ROUTE}/get-data/"

    resp = requests.post(url, json=payload, headers=headers, **self._request_params)
    self._check_ok_or_err_with_msg(resp, "Unable to get quantile data")

    resp = resp.json()
    quantile_index = resp["field_map"]["Quantile"]
    quantile_data = [[c[quantile_index] for c in r] for r in resp["data"]]
    date_cols = pd.to_datetime(resp["columns"])

    # Need to map gbi id's to isins or tickers
    gbi_ids = [int(i) for i in resp["rows"]]
    security_info = self._get_security_info(gbi_ids)

    # We now have security data, go through and create a map from internal
    # gbi id to client facing identifier
    id_key = "isin" if id_type == "ISIN" else "symbol"
    gbi_identifier_map = {
        sec["gbiId"]: sec[id_key] for sec in security_info["data"]["securities"]
    }

    df = pd.DataFrame(quantile_data, index=gbi_ids, columns=date_cols).transpose()
    df = df.rename(columns=gbi_identifier_map)
    return df
def get_portfolio_volatility(self, model_id: str, portfolio_id: str) ‑> pandas.core.frame.DataFrame
Expand source code
def get_portfolio_volatility(self, model_id: str, portfolio_id: str) -> pd.DataFrame:
    url = f"{self.base_uri}/api/analysis/volatility_rolling/{model_id}/{portfolio_id}"
    headers = {"Authorization": "ApiKey " + self.api_key}
    resp = requests.get(url, headers=headers, params=self._request_params)

    json_resp = resp.json()
    if (resp.ok and "errors" in json_resp) or not resp.ok:
        error_msg = json_resp["errors"][0]
        if self._is_portfolio_still_running(error_msg):
            return pd.DataFrame()
        logger.error(error_msg)
        raise BoostedAPIException(
            (
                f"Failed to get portfolio volatility for {portfolio_id=}: "
                f"{resp.status_code=}; {error_msg=}"
            )
        )

    df = pd.DataFrame(json_resp["data"], columns=json_resp["headerRow"])
    df = df.rename(
        columns={old: old.lower().replace("avg", "avg_") for old in df.columns}
    ).set_index("date")
    df.index = pd.to_datetime(df.index)
    return df
def get_prior_ranking_date(self, ranking_dates: List[datetime.date], starting_date: datetime.date) ‑> datetime.date

Given a starting date and a list of ranking dates, return the most recent previous ranking date.

Expand source code
def get_prior_ranking_date(
    self, ranking_dates: List[datetime.date], starting_date: datetime.date
) -> datetime.date:
    """
    Given a starting date and a list of ranking dates, return the most
    recent previous ranking date.
    """
    # order from most recent to least
    ranking_dates.sort(reverse=True)

    for d in ranking_dates:
        if d <= starting_date:
            return d

    # if we get here, the starting date is before the earliest ranking date
    raise BoostedAPIException(f"No rankins exist on or before {starting_date}")
def get_ranking_dates(self, model_id: str, portfolio_id: str) ‑> List[datetime.date]
Expand source code
def get_ranking_dates(self, model_id: str, portfolio_id: str) -> List[datetime.date]:
    headers = {"Authorization": "ApiKey " + self.api_key, "Content-Type": "application/json"}
    url = self.base_uri + f"/api/analysis/ranking-dates/{model_id}/{portfolio_id}"
    res = requests.get(url, headers=headers, **self._request_params)
    self._check_ok_or_err_with_msg(res, "Failed to get ranking dates")
    data = res.json().get("ranking_dates", [])

    return [parser.parse(d).date() for d in data]
def get_risk_factors_discovered_descriptors(self, model_id: str, portfolio_id: str, date: datetime.date, use_v2: bool = False) ‑> pandas.core.frame.DataFrame
Expand source code
def get_risk_factors_discovered_descriptors(
    self, model_id: str, portfolio_id: str, date: datetime.date, use_v2: bool = False
) -> pd.DataFrame:
    # first get the group descriptors
    descriptors = self._get_risk_factors_descriptors(model_id, portfolio_id)

    # calculate the most recent prior rankings date. This is the date
    # we need to use to query for risk group data.
    ranking_dates = self.get_ranking_dates(model_id, portfolio_id)
    ranking_date = self.get_prior_ranking_date(ranking_dates, date)
    date_str = ranking_date.strftime("%Y-%m-%d")

    risk_factor = RISK_FACTOR_V2 if use_v2 else RISK_FACTOR

    headers = {"Authorization": "ApiKey " + self.api_key, "Content-Type": "application/json"}
    url = (
        self.base_uri
        + f"/api/{risk_factor}/{model_id}/{portfolio_id}/risk-descriptors/json/{date_str}"
    )
    res = requests.get(url, headers=headers, **self._request_params)

    self._check_ok_or_err_with_msg(
        res, f"Failed to get risk factors for {model_id=}, {portfolio_id=}, {date=}"
    )

    # Endpoint returns a nested list of floats
    df = pd.DataFrame(res.json(), columns=RISK_FACTOR_COLUMNS)

    # This flat dataframe represents a potentially doubly nested structure
    # of Sector -> (high/low volatility) -> security. We don't care about
    # the high/low volatility rows, (which will have negative identifiers)
    # so we can filter these out.
    df = df[df["identifier"] >= 0]

    # now, any values that had a depth of 2 should be set to a depth of 1,
    # since we removed the double nesting.
    df.replace(to_replace=2, value=1, inplace=True)

    # This dataframe represents data that is nested on the UI, so the
    # "depth" field indicates which level of nesting each row is at. At this
    # point, a depth of 0 indicates a sector, and following depth 1 rows are
    # securities within the sector.

    # Identifiers in rows with depth 1 will be gbi ids, need to convert to
    # symbols.
    gbi_ids = df[df["depth"] == 1]["identifier"].tolist()
    sec_info = self._get_security_info(gbi_ids)["data"]["securities"]
    sec_map = {s["gbiId"]: s["symbol"] for s in sec_info}

    def convert_ids(row: pd.Series) -> pd.Series:
        # convert each row's "identifier" to the appropriate id type. If the
        # depth is 0, the identifier should be a sector, otherwise it should
        # be a ticker.
        ident = int(row["identifier"])
        row["identifier"] = (
            descriptors.get(ident).title() if row["depth"] == 0 else sec_map.get(ident)
        )
        return row

    df["depth"] = df["depth"].astype(int)
    df["stock_count"] = df["stock_count"].astype(int)
    df = df.apply(convert_ids, axis=1)
    df = df.reset_index(drop=True)
    return df
def get_risk_factors_sectors(self, model_id: str, portfolio_id: str, date: datetime.date, use_v2: bool = False) ‑> pandas.core.frame.DataFrame
Expand source code
def get_risk_factors_sectors(
    self, model_id: str, portfolio_id: str, date: datetime.date, use_v2: bool = False
) -> pd.DataFrame:
    # first get the group descriptors
    sectors = {s["id"]: s["name"] for s in self._get_sector_info()}

    # calculate the most recent prior rankings date. This is the date
    # we need to use to query for risk group data.
    ranking_dates = self.get_ranking_dates(model_id, portfolio_id)
    ranking_date = self.get_prior_ranking_date(ranking_dates, date)
    date_str = ranking_date.strftime("%Y-%m-%d")

    risk_factor = RISK_FACTOR_V2 if use_v2 else RISK_FACTOR

    headers = {"Authorization": "ApiKey " + self.api_key, "Content-Type": "application/json"}
    url = (
        self.base_uri
        + f"/api/{risk_factor}/{model_id}/{portfolio_id}/risk-sectors/json/{date_str}"
    )
    res = requests.get(url, headers=headers, **self._request_params)

    self._check_ok_or_err_with_msg(
        res, f"Failed to get risk factors for {model_id=}, {portfolio_id=}, {date=}"
    )

    # Endpoint returns a nested list of floats
    df = pd.DataFrame(res.json(), columns=RISK_FACTOR_COLUMNS)

    # identifier is a gics sector identifier
    df["identifier"] = df["identifier"].apply(lambda i: sectors.get(int(i)))

    # This dataframe represents data that is nested on the UI, so the
    # "depth" field indicates which level of nesting each row is at. For
    # risk factors sectors, each "depth" represents a level of specificity
    # for the sector. E.g. Energy -> Energy Equipment -> Oil & Gas Equipment
    df["depth"] = df["depth"].astype(int)
    df["stock_count"] = df["stock_count"].astype(int)
    df = df.reset_index(drop=True)
    return df
def get_risk_groups(self, model_id: str, portfolio_id: str, date: datetime.date, use_v2: bool = False) ‑> List[Dict[str, Any]]
Expand source code
def get_risk_groups(
    self, model_id: str, portfolio_id: str, date: datetime.date, use_v2: bool = False
) -> List[Dict[str, Any]]:
    # first get the group descriptors
    descriptors = self._get_risk_factors_descriptors(model_id, portfolio_id, use_v2)

    # calculate the most recent prior rankings date. This is the date
    # we need to use to query for risk group data.
    ranking_dates = self.get_ranking_dates(model_id, portfolio_id)
    ranking_date = self.get_prior_ranking_date(ranking_dates, date)
    date_str = ranking_date.strftime("%Y-%m-%d")

    risk_factor = RISK_FACTOR_V2 if use_v2 else RISK_FACTOR

    headers = {"Authorization": "ApiKey " + self.api_key, "Content-Type": "application/json"}
    url = self.base_uri + f"/api/{risk_factor}/{model_id}/{portfolio_id}/risk-groups/{date_str}"
    res = requests.get(url, headers=headers, **self._request_params)

    self._check_ok_or_err_with_msg(
        res, f"Failed to get risk factors for {model_id=}, {portfolio_id=}, {date=}"
    )

    # Response is a list of objects like:
    # [
    #   [
    #     0,
    #     14,
    #     1
    #   ],
    #   [
    #     25,
    #     12,
    #     13
    #   ],
    # 0.67013
    # ],
    #
    # Where each integer in the lists is a descriptor id.

    groups = []
    for row in res.json():
        row_map = {}
        # map descriptor id to name
        row_map["risk_group_a"] = [descriptors[i] for i in row[0]]
        row_map["risk_group_b"] = [descriptors[i] for i in row[1]]
        row_map["volatility_explained"] = row[2]
        groups.append(row_map)

    return groups
def get_rolling_signal_strength(self, model_id: str, portfolio_id: str) ‑> pandas.core.frame.DataFrame
Expand source code
def get_rolling_signal_strength(self, model_id: str, portfolio_id: str) -> pd.DataFrame:
    url = self.base_uri + f"/api/analysis/signal_strength_rolling/{model_id}/{portfolio_id}"
    headers = {"Authorization": "ApiKey " + self.api_key}

    logger.info(f"Retrieving rolling portfolio signals for {model_id=}, {portfolio_id=}")

    # Response format is a json object with a "header_row" key for column
    # names, and then a nested list of data.
    resp = requests.get(url, headers=headers, **self._request_params)
    self._check_ok_or_err_with_msg(
        resp, f"Failed to get rolling portfolio signals for {model_id=}, {portfolio_id=}"
    )

    data = resp.json()

    df = pd.DataFrame(data=data["data"], columns=data["header_row"])
    df["Date"] = pd.to_datetime(df["Date"])
    df = df.set_index("Date")
    return df.astype(float)
def get_signal_strength(self, model_id: str, portfolio_id: str) ‑> pandas.core.frame.DataFrame
Expand source code
def get_signal_strength(self, model_id: str, portfolio_id: str) -> pd.DataFrame:
    url = self.base_uri + f"/api/analysis/signal_strength/{model_id}/{portfolio_id}"
    headers = {"Authorization": "ApiKey " + self.api_key}

    logger.info(f"Retrieving portfolio signals for {model_id=}, {portfolio_id=}")

    # Response format is a json object with a "header_row" key for column
    # names, and then a nested list of data.
    resp = requests.get(url, headers=headers, **self._request_params)
    self._check_ok_or_err_with_msg(
        resp, f"Failed to get portfolio signals for {model_id=}, {portfolio_id=}"
    )

    data = resp.json()

    df = pd.DataFrame(data=data["data"], columns=data["header_row"])
    df["Date"] = pd.to_datetime(df["Date"])
    df = df.set_index("Date")
    return df.astype(float)
def get_sticky_portfolio_group(self) ‑> Dict[~KT, ~VT]

Get sticky portfolio group for the user

Parameters

Returns:

Dict { group_id: str group_name: str portfolios: List[PortfolioInGroup(Dict)] PortfolioInGroup(Dict): portfolio_id: str rank_in_group: Optional[int] = None portfolio_name: Optional[str] = None }

Expand source code
def get_sticky_portfolio_group(
    self,
) -> Dict:
    """
    Get sticky portfolio group for the user

    Parameters
    ----------

    Returns:
    -------
    Dict {
        group_id: str
        group_name: str
        portfolios: List[PortfolioInGroup(Dict)]
              PortfolioInGroup(Dict):
                       portfolio_id: str
                       rank_in_group: Optional[int] = None
                       portfolio_name: Optional[str] = None
    }
    """
    url = f"{self.base_uri}{WATCHLIST_ROUTE_PREFIX}{PORTFOLIO_GROUP_ROUTE}/get-sticky"
    headers = {"Authorization": "ApiKey " + self.api_key, "Content-Type": "application/json"}
    req_json = {}
    res = requests.post(url, json=req_json, headers=headers, **self._request_params)

    if not res.ok:
        error_msg = self._try_extract_error_code(res)
        logger.error(error_msg)
        raise BoostedAPIException(f"Failed to get sticky portfolio group: {error_msg}")

    data = res.json()
    return data
def get_watchlist_contents(self, watchlist_id) ‑> Dict[~KT, ~VT]
Expand source code
def get_watchlist_contents(self, watchlist_id) -> Dict:

    url = f"{self.base_uri}{ROUTE_PREFIX}{DAL_WATCHLIST_ROUTE}/contents/"
    headers = {"Authorization": "ApiKey " + self.api_key, "Content-Type": "application/json"}
    req_json = {"watchlist_id": watchlist_id}
    res = requests.post(url, json=req_json, headers=headers, **self._request_params)

    if not res.ok:
        error_msg = self._try_extract_error_code(res)
        logger.error(error_msg)
        raise BoostedAPIException(f"Failed to get watchlist contents: {error_msg}")

    data = res.json()
    return data
def get_watchlist_contents_as_csv(self, watchlist_id, filepath) ‑> None
Expand source code
def get_watchlist_contents_as_csv(self, watchlist_id, filepath) -> None:
    data = self.get_watchlist_contents(watchlist_id)
    df = pd.DataFrame(data["contents"])
    df.to_csv(filepath, index=False)
def get_watchlist_details(self, watchlist_id: str) ‑> Dict[~KT, ~VT]
Expand source code
def get_watchlist_details(self, watchlist_id: str) -> Dict:

    url = f"{self.base_uri}{ROUTE_PREFIX}{DAL_WATCHLIST_ROUTE}/details/"
    headers = {"Authorization": "ApiKey " + self.api_key, "Content-Type": "application/json"}
    req_json = {"watchlist_id": watchlist_id}
    res = requests.post(url, json=req_json, headers=headers, **self._request_params)

    if not res.ok:
        error_msg = self._try_extract_error_code(res)
        logger.error(error_msg)
        raise BoostedAPIException(f"Failed to get user models: {error_msg}")

    data = res.json()
    return data
def get_watchlists(self) ‑> List[Dict[~KT, ~VT]]
Expand source code
def get_watchlists(self) -> List[Dict]:

    url = f"{self.base_uri}{ROUTE_PREFIX}{DAL_WATCHLIST_ROUTE}/get_user_watchlists/"
    headers = {"Authorization": "ApiKey " + self.api_key, "Content-Type": "application/json"}
    req_json = {}
    res = requests.post(url, json=req_json, headers=headers, **self._request_params)

    if not res.ok:
        error_msg = self._try_extract_error_code(res)
        logger.error(error_msg)
        raise BoostedAPIException(f"Failed to get user watchlists: {error_msg}")

    data = res.json()
    return data["watchlists"]
def modify_hedge_experiment(self, experiment_id: str, name: Optional[str] = None, description: Optional[str] = None, experiment_type: Optional[Literal['HEDGE', 'MIMIC']] = None, target_securities: Union[Dict[GbiIdSecurity, float], str, None] = None, model_ids: Optional[List[str]] = None, stock_universe_ids: Optional[List[str]] = None, create_default_scenario: bool = True, baseline_model_id: Optional[str] = None, baseline_stock_universe_id: Optional[str] = None, baseline_portfolio_settings: Optional[str] = None) ‑> HedgeExperiment
Expand source code
def modify_hedge_experiment(
    self,
    experiment_id: str,
    name: Optional[str] = None,
    description: Optional[str] = None,
    experiment_type: Optional[hedge_experiment_type] = None,
    target_securities: Union[Dict[GbiIdSecurity, float], str, None] = None,
    model_ids: Optional[List[str]] = None,
    stock_universe_ids: Optional[List[str]] = None,
    create_default_scenario: bool = True,
    baseline_model_id: Optional[str] = None,
    baseline_stock_universe_id: Optional[str] = None,
    baseline_portfolio_settings: Optional[str] = None,
) -> HedgeExperiment:

    mod_qry = """
        mutation modifyHedgeExperimentDraft(
            $input: ModifyHedgeExperimentDraftInput!
        ) {
            modifyHedgeExperimentDraft(input: $input) {
                hedgeExperiment {
                ...HedgeExperimentSelectedSecuritiesPageFragment
                }
            }
        }

        fragment HedgeExperimentSelectedSecuritiesPageFragment on HedgeExperiment {
            hedgeExperimentId
            experimentName
            userId
            config
            description
            experimentType
            lastCalculated
            lastModified
            status
            portfolioCalcStatus
            targetSecurities {
                gbiId
                security {
                    gbiId
                    name
                    symbol
                }
                weight
            }
            targetPortfolios {
                portfolioId
            }
            baselineModel {
                id
                name
            }
            baselineScenario {
                hedgeExperimentScenarioId
                scenarioName
                description
                portfolioSettingsJson
                hedgeExperimentPortfolios {
                    portfolio {
                        id
                        name
                        modelId
                        performanceGridHeader
                        performanceGrid
                        status
                        tearSheet {
                            groupName
                            members {
                                name
                                value
                            }
                        }
                    }
                }
                status
            }
            baselineStockUniverseId
        }
    """
    mod_input = {
        "hedgeExperimentId": experiment_id,
        "createDefaultScenario": create_default_scenario,
    }
    if name is not None:
        mod_input["newExperimentName"] = name
    if description is not None:
        mod_input["newExperimentDescription"] = description
    if experiment_type is not None:
        mod_input["newExperimentType"] = experiment_type
    if model_ids is not None:
        mod_input["setSelectdModels"] = model_ids
    if stock_universe_ids is not None:
        mod_input["selectedStockUniverseIds"] = stock_universe_ids
    if baseline_model_id is not None:
        mod_input["setBaselineModel"] = baseline_model_id
    if baseline_stock_universe_id is not None:
        mod_input["setBaselineStockUniverse"] = baseline_stock_universe_id
    if baseline_portfolio_settings is not None:
        mod_input["setBaselinePortfolioSettings"] = baseline_portfolio_settings
    # note that the behaviors bound to these data are mutually exclusive,
    # and its possible the opposite was set earlier in the DRAFT phase
    # of experiment creation, so when setting one, we must unset the other
    if isinstance(target_securities, dict):
        mod_input["setTargetSecurities"] = [
            {"gbiId": sec.gbi_id, "weight": weight}
            for (sec, weight) in target_securities.items()
        ]
        mod_input["setTargetPortfolios"] = None
    elif isinstance(target_securities, str):
        mod_input["setTargetPortfolios"] = [{"portfolioId": target_securities}]
        mod_input["setTargetSecurities"] = None
    elif target_securities is None:
        pass
    else:
        raise TypeError(
            "Expected value of type Union[Dict[GbiIdSecurity, str], str] "
            f"for argument 'target_securities'; got {type(target_securities)}"
        )

    resp = requests.post(
        f"{self.base_uri}/api/graphql",
        json={"query": mod_qry, "variables": {"input": mod_input}},
        headers={"Authorization": "ApiKey " + self.api_key},
        params=self._request_params,
    )

    json_resp = resp.json()
    if (resp.ok and "errors" in json_resp) or not resp.ok:
        error_msg = self._try_extract_error_code(resp)
        logger.error(error_msg)
        raise BoostedAPIException(
            (
                f"Failed to modify hedge experiment in preparation for start {experiment_id=}: "
                f"{resp.status_code=}; {error_msg=}"
            )
        )

    exp_dict = json_resp["data"]["modifyHedgeExperimentDraft"]["hedgeExperiment"]
    experiment = HedgeExperiment.from_json_dict(exp_dict)
    return experiment
def query_dataset(self, dataset_id)
Expand source code
def query_dataset(self, dataset_id):
    url = self.base_uri + "/api/datasets/{0}".format(dataset_id)
    headers = {"Authorization": "ApiKey " + self.api_key}
    res = requests.get(url, headers=headers, **self._request_params)
    if res.ok:
        return res.json()
    else:
        error_msg = self._try_extract_error_code(res)
        logger.error(error_msg)
        raise BoostedAPIException("Failed to query dataset: {0}.".format(error_msg))
def remove_from_portfolio_group(self, group_id: str, portfolios: List[str]) ‑> Dict[~KT, ~VT]

Remove portfolios from a group

Parameters

group_id : str,
 

UUID str identifying a portfolio group

portfolios : List of str
 

Returns:

Dict { removed: int number of successful changes }

Expand source code
def remove_from_portfolio_group(
    self,
    group_id: str,
    portfolios: List[str],
) -> Dict:
    """
    Remove portfolios from a group

    Parameters
    ----------

    group_id: str,
       UUID str identifying a portfolio group

    portfolios: List of str


    Returns:
    -------
    Dict {
        removed: int
           number of successful changes
    }
    """
    url = f"{self.base_uri}{WATCHLIST_ROUTE_PREFIX}{PORTFOLIO_GROUP_ROUTE}/remove-from-group"
    headers = {"Authorization": "ApiKey " + self.api_key, "Content-Type": "application/json"}
    req_json = {"group_id": group_id, "portfolios": portfolios}
    res = requests.post(url, json=req_json, headers=headers, **self._request_params)

    if not res.ok:
        error_msg = self._try_extract_error_code(res)
        logger.error(error_msg)
        raise BoostedAPIException(
            f"Failed to remove portfolios from portfolio group: {error_msg}"
        )

    data = res.json()
    return data
def remove_securities_from_watchlist(self, watchlist_id: str, identifiers: List[str], identifier_type: Literal['TICKER', 'ISIN']) ‑> Dict[~KT, ~VT]
Expand source code
def remove_securities_from_watchlist(
    self, watchlist_id: str, identifiers: List[str], identifier_type: Literal["TICKER", "ISIN"]
) -> Dict:
    # should we just make the arg lower? all caps has a flag-like feel to it
    id_type = identifier_type.lower()
    url = f"{self.base_uri}{WATCHLIST_ROUTE_PREFIX}{DAL_WATCHLIST_ROUTE}/remove_{id_type}s/"
    headers = {"Authorization": "ApiKey " + self.api_key, "Content-Type": "application/json"}
    req_json = {"watchlist_id": watchlist_id, id_type: identifiers}
    res = requests.post(url, json=req_json, headers=headers, **self._request_params)

    if not res.ok:
        error_msg = self._try_extract_error_code(res)
        logger.error(error_msg)
        raise BoostedAPIException(f"Failed to get user models: {error_msg}")

    data = res.json()
    return data
def rename_portfolio_group(self, group_id: str, group_name: str) ‑> Dict[~KT, ~VT]

Rename a portfolio group

Parameters

group_id : str,
 

UUID str identifying a portfolio group

group_name : str,
 

The new name for the porfolio

Returns:

Dict { changed: int - 1 == success }

Expand source code
def rename_portfolio_group(
    self,
    group_id: str,
    group_name: str,
) -> Dict:
    """
    Rename a portfolio group

    Parameters
    ----------

    group_id: str,
       UUID str identifying a portfolio group

    group_name: str,
       The new name for the porfolio

    Returns:
    -------
    Dict {
        changed: int - 1 == success
    }
    """
    url = f"{self.base_uri}{WATCHLIST_ROUTE_PREFIX}{PORTFOLIO_GROUP_ROUTE}/rename"
    headers = {"Authorization": "ApiKey " + self.api_key, "Content-Type": "application/json"}
    req_json = {"group_id": group_id, "group_name": group_name}
    res = requests.post(url, json=req_json, headers=headers, **self._request_params)

    if not res.ok:
        error_msg = self._try_extract_error_code(res)
        logger.error(error_msg)
        raise BoostedAPIException(f"Failed to rename portfolio group: {error_msg}")

    data = res.json()
    return data
def sendModelRecalc(self, model_id)
Expand source code
def sendModelRecalc(self, model_id):
    url = self.base_uri + "/api/models/{0}/recalc".format(model_id)
    logger.info("Sending model recalc request for model {0}".format(model_id))
    headers = {"Authorization": "ApiKey " + self.api_key}
    res = requests.put(url, headers=headers, **self._request_params)
    if not res.ok:
        error_msg = self._try_extract_error_code(res)
        logger.error(error_msg)
        raise BoostedAPIException(
            "Failed to send model recalc request - "
            + "the model in UI may be out of date: {0}.".format(error_msg)
        )
def set_portfolio_group_for_watchlist(self, portfolio_group_id: str, watchlist_id: str) ‑> Dict[~KT, ~VT]

Set portfolio group for watchlist.

Parameters

portfolio_group_id : str,
 

UUID str identifying a portfolio group

watchlist_id : str,
 

UUID str identifying a watchlist

Returns:

Dict { success: bool errors: data: Dict changed: int }

Expand source code
def set_portfolio_group_for_watchlist(
    self,
    portfolio_group_id: str,
    watchlist_id: str,
) -> Dict:
    """
    Set portfolio group for watchlist.

    Parameters
    ----------

    portfolio_group_id: str,
       UUID str identifying a portfolio group

    watchlist_id: str,
       UUID str identifying a watchlist


    Returns:
    -------
    Dict {
        success: bool
        errors:
        data: Dict
            changed: int
    }
    """
    url = f"{self.base_uri}{WATCHLIST_ROUTE_PREFIX}{DAL_WATCHLIST_ROUTE}/set-portfolio-groups/"
    headers = {"Authorization": "ApiKey " + self.api_key, "Content-Type": "application/json"}
    req_json = {"portfolio_group_id": portfolio_group_id, "watchlist_id": watchlist_id}
    res = requests.post(url, json=req_json, headers=headers, **self._request_params)

    if not res.ok:
        error_msg = self._try_extract_error_code(res)
        logger.error(error_msg)
        raise BoostedAPIException(f"Failed to set portfolio group for watchlist: {error_msg}")

    return res.json()
def set_sticky_portfolio_group(self, portfolio_group_id: str) ‑> Dict[~KT, ~VT]

Set sticky portfolio group

Parameters

group_id : str,
 

UUID str identifying a portfolio group

Returns:

Dict { changed: int - 1 == success }

Expand source code
def set_sticky_portfolio_group(
    self,
    portfolio_group_id: str,
) -> Dict:
    """
    Set sticky portfolio group

    Parameters
    ----------

    group_id: str,
       UUID str identifying a portfolio group

    Returns:
    -------
    Dict {
        changed: int - 1 == success
    }
    """
    url = f"{self.base_uri}{WATCHLIST_ROUTE_PREFIX}{PORTFOLIO_GROUP_ROUTE}/set-sticky"
    headers = {"Authorization": "ApiKey " + self.api_key, "Content-Type": "application/json"}
    req_json = {"portfolio_group_id": portfolio_group_id}
    res = requests.post(url, json=req_json, headers=headers, **self._request_params)

    if not res.ok:
        error_msg = self._try_extract_error_code(res)
        logger.error(error_msg)
        raise BoostedAPIException(f"Failed to set sticky portfolio group: {error_msg}")

    data = res.json()
    return data
def setup_chunk_and_upload_data(self, dataset_id, csv_data, data_type, timeout=600, block=True, no_exception_on_chunk_error=False)
Expand source code
def setup_chunk_and_upload_data(
    self,
    dataset_id,
    csv_data,
    data_type,
    timeout=600,
    block=True,
    no_exception_on_chunk_error=False,
):
    chunk_id = self.start_chunked_upload(dataset_id)
    logger.info("Obtained lock on dataset for upload: " + chunk_id)
    try:
        warnings, errors = self.chunk_and_upload_data(
            dataset_id, chunk_id, csv_data, timeout, no_exception_on_chunk_error
        )
        commit_warnings, commit_errors = self._commit_chunked_upload(
            dataset_id, chunk_id, data_type, block, timeout
        )
        return warnings + commit_warnings, errors + commit_errors
    except Exception:
        self.abort_chunked_upload(dataset_id, chunk_id)
        raise
def start_chunked_upload(self, dataset_id)
Expand source code
def start_chunked_upload(self, dataset_id):
    url = self.base_uri + "/api/datasets/{0}/start-chunked-upload".format(dataset_id)
    headers = {"Authorization": "ApiKey " + self.api_key}
    res = requests.post(url, headers=headers, **self._request_params)
    if res.ok:
        return res.json()["result"]
    else:
        error_msg = self._try_extract_error_code(res)
        logger.error(error_msg)
        raise BoostedAPIException(
            "Failed to obtain dataset lock for upload: {0}.".format(error_msg)
        )
def start_hedge_experiment(self, experiment_id: str, *scenario_ids: str) ‑> HedgeExperiment
Expand source code
def start_hedge_experiment(self, experiment_id: str, *scenario_ids: str) -> HedgeExperiment:
    start_qry = """
        mutation startHedgeExperiment($input: StartHedgeExperimentInput!) {
            startHedgeExperiment(input: $input) {
                hedgeExperiment {
                    hedgeExperimentId
                    experimentName
                    userId
                    config
                    description
                    experimentType
                    lastCalculated
                    lastModified
                    status
                    portfolioCalcStatus
                    targetSecurities {
                        gbiId
                        security {
                            gbiId
                            name
                            symbol
                        }
                        weight
                    }
                    targetPortfolios {
                        portfolioId
                    }
                    baselineModel {
                        id
                        name
                    }
                    baselineScenario {
                        hedgeExperimentScenarioId
                        scenarioName
                        description
                        portfolioSettingsJson
                        hedgeExperimentPortfolios {
                            portfolio {
                                id
                                name
                                modelId
                                performanceGridHeader
                                performanceGrid
                                status
                                tearSheet {
                                    groupName
                                    members {
                                        name
                                        value
                                    }
                                }
                            }
                        }
                        status
                    }
                    baselineStockUniverseId
                }
            }
        }
    """
    start_input = {"hedgeExperimentId": experiment_id}
    if len(scenario_ids) > 0:
        start_input["hedgeExperimentScenarioIds"] = list(scenario_ids)

    resp = requests.post(
        f"{self.base_uri}/api/graphql",
        json={"query": start_qry, "variables": {"input": start_input}},
        headers={"Authorization": "ApiKey " + self.api_key},
        params=self._request_params,
    )

    json_resp = resp.json()
    if (resp.ok and "errors" in json_resp) or not resp.ok:
        error_msg = self._try_extract_error_code(resp)
        logger.error(error_msg)
        raise BoostedAPIException(
            (
                f"Failed to start hedge experiment {experiment_id=}: "
                f"{resp.status_code=}; {error_msg=}"
            )
        )

    exp_dict = json_resp["data"]["startHedgeExperiment"]["hedgeExperiment"]
    experiment = HedgeExperiment.from_json_dict(exp_dict)
    return experiment
def updateBlacklist(self, blacklist_id, long_short=None, start_date=None, end_date=None)
Expand source code
def updateBlacklist(self, blacklist_id, long_short=None, start_date=None, end_date=None):
    params = {}
    if long_short:
        params["longShort"] = long_short
    if start_date:
        params["startDate"] = start_date
    if end_date:
        params["endDate"] = end_date
    url = self.base_uri + f"/api/blacklist/{blacklist_id}"
    headers = {"Authorization": "ApiKey " + self.api_key, "Content-Type": "application/json"}
    res = requests.patch(url, json=params, headers=headers, **self._request_params)
    if res.ok:
        return res.json()
    else:
        error_msg = self._try_extract_error_code(res)
        logger.error(error_msg)
        raise BoostedAPIException(
            f"Failed to update blacklist with id {blacklist_id}: {error_msg}"
        )
def updateUniverse(self, modelId, universe_df, date=datetime.date(2022, 9, 16))
Expand source code
def updateUniverse(self, modelId, universe_df, date=datetime.date.today() + timedelta(1)):
    date = self.__iso_format(date)
    url = self.base_uri + "/api/models/{0}/universe/{1}".format(modelId, date)
    headers = {"Authorization": "ApiKey " + self.api_key}
    logger.info("Updating universe for date {0}.".format(date))
    if isinstance(universe_df, pd.core.frame.DataFrame):
        buf = io.StringIO()
        universe_df.to_csv(buf)
        target = ("uploaded_universe.csv", buf.getvalue(), "text/csv")
        files_req = {}
        files_req["universe"] = target
        res = requests.post(url, files=files_req, headers=headers, **self._request_params)
    elif isinstance(universe_df, str):
        target = ("uploaded_universe.csv", universe_df, "text/csv")
        files_req = {}
        files_req["universe"] = target
        res = requests.post(url, files=files_req, headers=headers, **self._request_params)
    else:
        raise BoostedAPIException("Expected CSV as str or Pandas DataFrame.")
    if res.ok:
        logger.info("Universe update successful.")
        if "warnings" in res.json():
            logger.info("Warnings: {0}.".format(res.json()["warnings"]))
            return res.json()["warnings"]
        else:
            return "No warnings."
    else:
        error_msg = self._try_extract_error_code(res)
        raise BoostedAPIException("Failed to get universe: {0}.".format(error_msg))
def upload_dataset_chunk(self, chunk_descriptor, dataset_id, chunk_id, csv_data, timeout=600, no_exception_on_chunk_error=False)
Expand source code
def upload_dataset_chunk(
    self,
    chunk_descriptor,
    dataset_id,
    chunk_id,
    csv_data,
    timeout=600,
    no_exception_on_chunk_error=False,
):
    logger.info("Starting upload: " + chunk_descriptor)
    url = self.base_uri + "/api/datasets/{0}/upload-dataset-chunk".format(dataset_id)
    headers = {"Authorization": "ApiKey " + self.api_key}
    files_req = {}
    warnings = []
    errors = []

    # make the network request
    target = ("uploaded_data.csv", csv_data, "text/csv")
    files_req["dataFile"] = target
    params = {"uploadGroupId": chunk_id}
    res = requests.post(
        url,
        params=params,
        files=files_req,
        headers=headers,
        timeout=timeout,
        **self._request_params,
    )

    if res.ok:
        logger.info(
            (
                "Chunk upload completed.  "
                "Ingestion started.  "
                "Please wait until the data is in AVAILABLE state."
            )
        )
        if "warnings" in res.json():
            warnings = res.json()["warnings"]
            if len(warnings) > 0:
                logger.warning("Uploaded chunk encountered data warnings: ")
            for w in warnings:
                logger.warning(w)
    else:
        reason = "Upload failed: {0}, {1}".format(res.text, res.reason)
        logger.error(reason)
        if no_exception_on_chunk_error:
            errors.append(
                "Chunk {0} failed: {1}. ".format(chunk_descriptor, reason)
                + "Your data was only PARTIALLY uploaded. "
                + "Please reattempt the upload of this chunk."
            )
        else:
            raise BoostedAPIException("Upload failed.")

    return res, warnings, errors
def validate_dataframe(self, df)
Expand source code
def validate_dataframe(self, df):
    if not isinstance(df, pd.core.frame.DataFrame):
        logger.error("Dataset must be of type Dataframe.")
        return False
    if type(df.index) != pd.core.indexes.datetimes.DatetimeIndex:
        logger.error("Index must be DatetimeIndex.")
        return False
    if len(df.columns) == 0:
        logger.error("No feature columns exist.")
        return False
    if len(df) == 0:
        logger.error("No rows exist.")
    return True