boosted.api.api_client

   1# Copyright (C) 2020 Gradient Boosted Investments, Inc. - All Rights Reserved
   2
   3import base64
   4import csv
   5import datetime
   6import functools
   7import io
   8import itertools
   9import json
  10import logging
  11import math
  12import mimetypes
  13import os
  14import sys
  15import tempfile
  16import time
  17from datetime import date, timedelta
  18from typing import Any, Dict, List, Literal, Optional, Tuple, Union
  19from urllib import parse
  20
  21import numpy as np
  22import pandas
  23import pandas as pd
  24import requests
  25from dateutil import parser
  26
  27import boosted.api.graphql_queries as graphql_queries
  28from boosted.api.api_type import (
  29    BoostedAPIException,
  30    BoostedDate,
  31    ChunkStatus,
  32    ColumnSubRole,
  33    DataAddType,
  34    DataSetConfig,
  35    DataSetType,
  36    DateIdentCountryCurrency,
  37    GbiIdSecurity,
  38    GbiIdTickerISIN,
  39    HedgeExperiment,
  40    HedgeExperimentDetails,
  41    HedgeExperimentScenario,
  42    Language,
  43    NewsHorizon,
  44    PortfolioSettings,
  45    Status,
  46    ThemeUniverse,
  47    hedge_experiment_type,
  48)
  49from boosted.api.api_util import (
  50    get_valid_iso_dates,
  51    infer_dataset_schema,
  52    protoCubeJsonDataToDataFrame,
  53    to_camel_case,
  54    validate_start_and_end_dates,
  55)
  56
  57logger = logging.getLogger("boosted.api.client")
  58logging.basicConfig()
  59
  60g_boosted_api_url = "https://insights.boosted.ai"
  61g_boosted_api_url_dev = "https://insights-dev.boosted.ai"
  62WATCHLIST_ROUTE_PREFIX = "/api/dal/watchlist"
  63ROUTE_PREFIX = WATCHLIST_ROUTE_PREFIX
  64DAL_WATCHLIST_ROUTE = "/api/v0/watchlist"
  65DAL_SECURITIES_ROUTE = "/api/v0/securities"
  66DAL_PA_ROUTE = "/api/v0/portfolio-analysis"
  67PORTFOLIO_GROUP_ROUTE = "/api/v0/portfolio-group"
  68
  69RISK_FACTOR = "risk-factor"
  70RISK_FACTOR_V2 = "risk-factor-v2"
  71RISK_FACTOR_COLUMNS = [
  72    "depth",
  73    "identifier",
  74    "stock_count",
  75    "volatility",
  76    "exposure",
  77    "rating",
  78    "rating_delta",
  79]
  80
  81
  82def convert_date(date: BoostedDate) -> datetime.date:
  83    if isinstance(date, str):
  84        try:
  85            return parser.parse(date).date()
  86        except Exception as e:
  87            raise BoostedAPIException(f"Unable to parse date: {str(e)}")
  88    return date
  89
  90
  91class BoostedClient:
  92    def __init__(
  93        self, api_key, override_uri=None, debug=False, proxy=None, disable_verify_ssl=False
  94    ):
  95        """
  96        Parameters
  97        ----------
  98        api_key: str
  99            Your API key provided by the Boosted application.  See your profile
 100            to generate a new key.
 101        proxy: str
 102            Your organization may require the use of a proxy for access.
 103            The address of a HTTPS proxy in the format of <address>:<port>.
 104            Examples are "123.456.789:123" or "my.proxy.com:123".
 105            Do not prepend with "https://".
 106        disable_verify_ssl: bool
 107            Your networking setup may be behind a firewall which performs SSL
 108            inspection. Either set the REQUESTS_CA_BUNDLE environment variable
 109            to point to the location of a custom certificate bundle, or set this
 110            parameter to True to disable SSL verification as a workaround.
 111        """
 112        if override_uri is None:
 113            self.base_uri = g_boosted_api_url
 114        else:
 115            self.base_uri = override_uri
 116        self.api_key = api_key
 117        self.debug = debug
 118        self._request_params: Dict = {}
 119        if debug:
 120            logger.setLevel(logging.DEBUG)
 121        else:
 122            logger.setLevel(logging.INFO)
 123        if proxy is not None:
 124            self._request_params["proxies"] = {"https": proxy}
 125        if disable_verify_ssl:
 126            self._request_params["verify"] = False
 127
 128    def __print_json_info(self, json_data, isInference=False):
 129        if "warnings" in json_data.keys():
 130            for warning in json_data["warnings"]:
 131                logger.warning("  {0}".format(warning))
 132        if "errors" in json_data.keys():
 133            for error in json_data["errors"]:
 134                logger.error("  {0}".format(error))
 135                return Status.FAIL
 136
 137        if "result" in json_data.keys():
 138            results_data = json_data["result"]
 139            if isInference:
 140                if "inferenceResultsUrl" in results_data.keys():
 141                    res_url = parse.urlparse(results_data["inferenceResultsUrl"])
 142                    logger.debug(res_url)
 143                    logger.info("Inference started.")
 144            if "updateCount" in results_data.keys():
 145                logger.info("Updated {0} rows.".format(results_data["updateCount"]))
 146            if "createCount" in results_data.keys():
 147                logger.info("Created {0} rows.".format(results_data["createCount"]))
 148            return Status.SUCCESS
 149
 150    def __to_date_obj(self, dt):
 151        if isinstance(dt, datetime.datetime):
 152            dt = dt.date()
 153        elif isinstance(dt, datetime.date):
 154            return dt
 155        elif isinstance(dt, str):
 156            try:
 157                dt = parser.parse(dt).date()
 158            except ValueError:
 159                raise ValueError('dt: "' + dt + '" is not a valid date.')
 160        return dt
 161
 162    def __iso_format(self, dt):
 163        date = self.__to_date_obj(dt)
 164        if date is not None:
 165            date = date.isoformat()
 166        return date
 167
 168    def _check_status_code(self, response, isInference=False):
 169        has_json = False
 170        try:
 171            logger.debug(response.headers)
 172            if "Content-Type" in response.headers:
 173                if response.headers["Content-Type"].startswith("application/json"):
 174                    json_data = response.json()
 175                    has_json = True
 176            else:
 177                has_json = False
 178        except json.JSONDecodeError:
 179            logger.error("ERROR: response has no JSON payload.")
 180        if response.status_code == 200 or response.status_code == 202:
 181            if has_json:
 182                self.__print_json_info(json_data, isInference)
 183            else:
 184                pass
 185            return Status.SUCCESS
 186        if response.status_code == 404:
 187            if has_json:
 188                self.__print_json_info(json_data, isInference)
 189            raise BoostedAPIException(
 190                'Server "{0}" not reachable.  Code {1}.'.format(
 191                    self.base_uri, response.status_code
 192                ),
 193                data=response,
 194            )
 195        if response.status_code == 400:
 196            if has_json:
 197                self.__print_json_info(json_data, isInference)
 198            if isInference:
 199                return Status.FAIL
 200            else:
 201                raise BoostedAPIException("Error, bad request.  Check the dataset ID.", response)
 202        if response.status_code == 401:
 203            if has_json:
 204                self.__print_json_info(json_data, isInference)
 205            raise BoostedAPIException("Authorization error.", response)
 206        else:
 207            if has_json:
 208                self.__print_json_info(json_data, isInference)
 209            raise BoostedAPIException(
 210                "Error in API response.  Status code={0} {1}\n{2}".format(
 211                    response.status_code, response.reason, response.headers
 212                ),
 213                response,
 214            )
 215
 216    def _try_extract_error_code(self, result):
 217        logger.info(result.headers)
 218        if "Content-Type" in result.headers:
 219            if result.headers["Content-Type"].startswith("application/json"):
 220                if "errors" in result.json():
 221                    return result.json()["errors"]
 222            if result.headers["Content-Type"].startswith("text/plain"):
 223                return result.text
 224        return str(result.reason)
 225
 226    def _check_ok_or_err_with_msg(self, res, potential_error_msg: str):
 227        if not res.ok:
 228            error = self._try_extract_error_code(res)
 229            logger.error(error)
 230            raise BoostedAPIException(f"{potential_error_msg}: {error}")
 231
 232    def _get_portfolio_rebalance_from_periods(
 233        self, portfolio_id: str, rel_periods: List[str]
 234    ) -> List[datetime.date]:
 235        """
 236        Returns a list of rebalance dates for a portfolio given a list of
 237        relative periods of format '1D', '1W', '3M', etc.
 238        """
 239        resp = self._get_graphql(
 240            query=graphql_queries.GET_PORTFOLIO_RELATIVE_DATES_QUERY,
 241            variables={"portfolioId": portfolio_id, "relativePeriods": rel_periods},
 242        )
 243        dates = resp["data"]["portfolio"]["relativeDates"]
 244        return [datetime.datetime.strptime(d["date"], "%Y-%m-%d").date() for d in dates]
 245
 246    def translate_text(self, language: Optional[Union[Language, str]], text: str) -> str:
 247        if not language or language == Language.ENGLISH:
 248            # By default, do not translate English
 249            return text
 250
 251        params = {"text": text, "langCode": language}
 252        url = self.base_uri + "/api/translate/translate-text"
 253        headers = {"Authorization": "ApiKey " + self.api_key}
 254        logger.info("Translating text...")
 255        res = requests.post(url, json=params, headers=headers, **self._request_params)
 256        try:
 257            result = res.json()["translatedText"]
 258        except Exception:
 259            raise BoostedAPIException("Error translating text")
 260        return result
 261
 262    def query_dataset(self, dataset_id):
 263        url = self.base_uri + "/api/datasets/{0}".format(dataset_id)
 264        headers = {"Authorization": "ApiKey " + self.api_key}
 265        res = requests.get(url, headers=headers, **self._request_params)
 266        if res.ok:
 267            return res.json()
 268        else:
 269            error_msg = self._try_extract_error_code(res)
 270            logger.error(error_msg)
 271            raise BoostedAPIException("Failed to query dataset: {0}.".format(error_msg))
 272
 273    def query_namespace_dataset_id(self, namespace, data_type):
 274        url = self.base_uri + f"/api/custom-security-dataset/{namespace}/{data_type}"
 275        headers = {"Authorization": "ApiKey " + self.api_key}
 276        res = requests.get(url, headers=headers, **self._request_params)
 277        if res.ok:
 278            return res.json()["result"]["id"]
 279        else:
 280            if res.status_code != 404:
 281                error_msg = self._try_extract_error_code(res)
 282                logger.error(error_msg)
 283                raise BoostedAPIException("Failed to query dataset: {0}.".format(error_msg))
 284            else:
 285                return None
 286
 287    def export_global_data(
 288        self,
 289        dataset_id,
 290        start=(datetime.date.today() - timedelta(days=365 * 25)),
 291        end=datetime.date.today(),
 292        timeout=600,
 293    ):
 294        query_info = self.query_dataset(dataset_id)
 295        if DataSetType[query_info["type"]] != DataSetType.GLOBAL:
 296            raise BoostedAPIException(
 297                f"Incorrect dataset type: {query_info['type']}" f" - Expected {DataSetType.GLOBAL}"
 298            )
 299        return self.export_data(dataset_id, start, end, timeout)
 300
 301    def export_independent_data(
 302        self,
 303        dataset_id,
 304        start=(datetime.date.today() - timedelta(days=365 * 25)),
 305        end=datetime.date.today(),
 306        timeout=600,
 307    ):
 308        query_info = self.query_dataset(dataset_id)
 309        if DataSetType[query_info["type"]] != DataSetType.STRATEGY:
 310            raise BoostedAPIException(
 311                f"Incorrect dataset type: {query_info['type']}"
 312                f" - Expected {DataSetType.STRATEGY}"
 313            )
 314        return self.export_data(dataset_id, start, end, timeout)
 315
 316    def export_dependent_data(
 317        self,
 318        dataset_id,
 319        start=None,
 320        end=None,
 321        timeout=600,
 322    ):
 323        query_info = self.query_dataset(dataset_id)
 324        if DataSetType[query_info["type"]] != DataSetType.STOCK:
 325            raise BoostedAPIException(
 326                f"Incorrect dataset type: {query_info['type']}" f" - Expected {DataSetType.STOCK}"
 327            )
 328
 329        valid_date_range = self.getDatasetDates(dataset_id)
 330        validStart = valid_date_range["validFrom"]
 331        validEnd = valid_date_range["validTo"]
 332
 333        if start is None:
 334            logger.info("Since no start date provided, starting from {0}.".format(validStart))
 335            start = validStart
 336        if end is None:
 337            logger.info("Since no end date provided, ending at {0}.".format(validEnd))
 338            end = validEnd
 339        start = self.__to_date_obj(start)
 340        end = self.__to_date_obj(end)
 341        if start < validStart:
 342            logger.info("Data does not exist before {0}.".format(validStart))
 343            logger.info("Starting from {0}.".format(validStart))
 344            start = validStart
 345        if end > validEnd:
 346            logger.info("Data does not exist after {0}.".format(validEnd))
 347            logger.info("Ending at {0}.".format(validEnd))
 348            end = validEnd
 349        validate_start_and_end_dates(start, end)
 350
 351        logger.info("Data exists from {0} to {1}.".format(start, end))
 352        request_url = "/api/datasets/" + dataset_id + "/export-data"
 353        headers = {"Authorization": "ApiKey " + self.api_key}
 354
 355        data_chunks = []
 356        chunk_size_days = 90
 357        while start <= end:
 358            chunk_end = start + timedelta(days=chunk_size_days)
 359            if chunk_end > end:
 360                chunk_end = end
 361
 362            logger.info("Requesting start={0} end={1}.".format(start, chunk_end))
 363            params = {"start": self.__iso_format(start), "end": self.__iso_format(chunk_end)}
 364            logger.debug("URL={0}, headers={1}, params={2}".format(request_url, headers, params))
 365
 366            res = requests.get(
 367                self.base_uri + request_url,
 368                headers=headers,
 369                params=params,
 370                timeout=timeout,
 371                **self._request_params,
 372            )
 373
 374            if res.ok:
 375                buf = io.StringIO(res.text)
 376                df = pd.read_csv(buf, index_col=0, parse_dates=True)
 377                if "price" in df.columns:
 378                    df = df.drop("price", axis=1)
 379                data_chunks.append(df)
 380            else:
 381                error_msg = self._try_extract_error_code(res)
 382                logger.error(error_msg)
 383                raise BoostedAPIException("Failed to query dataset: {0}.".format(error_msg))
 384
 385            start = start + timedelta(days=chunk_size_days + 1)
 386
 387        return pd.concat(data_chunks)
 388
 389    def export_custom_security_data(
 390        self,
 391        dataset_id,
 392        start=(date.today() - timedelta(days=365 * 25)),
 393        end=date.today(),
 394        timeout=600,
 395    ):
 396        query_info = self.query_dataset(dataset_id)
 397        if DataSetType[query_info["type"]] != DataSetType.SECURITIES_DAILY:
 398            raise BoostedAPIException(
 399                f"Incorrect dataset type: {query_info['type']}"
 400                f" - Expected {DataSetType.SECURITIES_DAILY}"
 401            )
 402        return self.export_data(dataset_id, start, end, timeout)
 403
 404    def export_data(
 405        self,
 406        dataset_id,
 407        start=(datetime.date.today() - timedelta(days=365 * 25)),
 408        end=datetime.date.today(),
 409        timeout=600,
 410    ):
 411        logger.info("Requesting start={0} end={1}.".format(start, end))
 412        request_url = "/api/datasets/" + dataset_id + "/export-data"
 413        headers = {"Authorization": "ApiKey " + self.api_key}
 414        start = self.__iso_format(start)
 415        end = self.__iso_format(end)
 416        params = {"start": start, "end": end}
 417        logger.debug("URL={0}, headers={1}, params={2}".format(request_url, headers, params))
 418        res = requests.get(
 419            self.base_uri + request_url,
 420            headers=headers,
 421            params=params,
 422            timeout=timeout,
 423            **self._request_params,
 424        )
 425        if res.ok or self._check_status_code(res):
 426            buf = io.StringIO(res.text)
 427            df = pd.read_csv(buf, index_col=0, parse_dates=True)
 428            if "price" in df.columns:
 429                df = df.drop("price", axis=1)
 430            return df
 431        else:
 432            error_msg = self._try_extract_error_code(res)
 433            logger.error(error_msg)
 434            raise BoostedAPIException("Failed to query dataset: {0}.".format(error_msg))
 435
 436    def _get_inference(self, model_id, inference_date=datetime.date.today()):
 437        request_url = "/api/models/" + model_id + "/inference-results"
 438        headers = {"Authorization": "ApiKey " + self.api_key}
 439        params = {}
 440        params["date"] = self.__iso_format(inference_date)
 441        logger.debug(request_url + ", " + str(headers) + ", " + str(params))
 442        res = requests.get(
 443            self.base_uri + request_url, headers=headers, params=params, **self._request_params
 444        )
 445        status = self._check_status_code(res, isInference=True)
 446        if status == Status.SUCCESS:
 447            return res, status
 448        else:
 449            return None, status
 450
 451    def get_inference(
 452        self, model_id, inference_date=datetime.date.today(), block=False, timeout_minutes=30
 453    ):
 454        start_time = datetime.datetime.now()
 455        while True:
 456            for numRetries in range(3):
 457                res, status = self._get_inference(model_id, inference_date)
 458                if res is not None:
 459                    continue
 460                else:
 461                    if status == Status.FAIL:
 462                        return Status.FAIL
 463                    logger.info("Retrying...")
 464            if res is None:
 465                logger.error("Max retries reached.  Request failed.")
 466                return None
 467
 468            json_data = res.json()
 469            if "result" in json_data.keys():
 470                if json_data["result"]["status"] == "RUNNING":
 471                    still_running = True
 472                    if not block:
 473                        logger.warn("Inference job is still running.")
 474                        return None
 475                    else:
 476                        logger.info(
 477                            "Inference job is still running.  Time elapsed={0}.".format(
 478                                datetime.datetime.now() - start_time
 479                            )
 480                        )
 481                        time.sleep(10)
 482                else:
 483                    still_running = False
 484
 485                if not still_running and json_data["result"]["status"] == "COMPLETE":
 486                    csv = json_data["result"]["signals"]
 487                    logger.info(json_data["result"])
 488                    if self._check_status_code(res, isInference=True):
 489                        logger.info(
 490                            "Total run time = {0}.".format(datetime.datetime.now() - start_time)
 491                        )
 492                        return csv
 493            else:
 494                if "errors" in json_data.keys():
 495                    logger.error(json_data["errors"])
 496                else:
 497                    logger.error("Error getting inference for date {0}.".format(inference_date))
 498                return None
 499            if (datetime.datetime.now() - start_time).total_seconds() / 60.0 > timeout_minutes:
 500                logger.error("Timeout waiting for job completion.")
 501                return None
 502
 503    def createDataset(self, schema):
 504        request_url = "/api/datasets"
 505        headers = {"Authorization": "ApiKey " + self.api_key, "Content-Type": "application/json"}
 506        s = json.dumps(schema)
 507        logger.info("Creating dataset with schema " + s)
 508        res = requests.post(
 509            self.base_uri + request_url, data=s, headers=headers, **self._request_params
 510        )
 511        if res.ok:
 512            return res.json()["result"]
 513        else:
 514            raise BoostedAPIException("Dataset creation failed.")
 515
 516    def create_custom_namespace_dataset(self, namespace, schema):
 517        request_url = f"/api/custom-security-dataset/{namespace}"
 518        headers = {"Authorization": "ApiKey " + self.api_key, "Content-Type": "application/json"}
 519        s = json.dumps(schema)
 520        logger.info("Creating dataset with schema " + s)
 521        res = requests.post(
 522            self.base_uri + request_url, data=s, headers=headers, **self._request_params
 523        )
 524        if res.ok:
 525            return res.json()["result"]
 526        else:
 527            raise BoostedAPIException("Dataset creation failed.")
 528
 529    def getUniverse(self, modelId, date=None):
 530        if date is not None:
 531            url = "/api/models/{0}/universe/{1}".format(modelId, self.__iso_format(date))
 532            logger.info("Getting universe for date: {0}.".format(date))
 533        else:
 534            url = "/api/models/{0}/universe/".format(modelId)
 535        headers = {"Authorization": "ApiKey " + self.api_key}
 536        res = requests.get(self.base_uri + url, headers=headers, **self._request_params)
 537        if res.ok:
 538            buf = io.StringIO(res.text)
 539            df = pd.read_csv(buf, index_col=0, parse_dates=True)
 540            return df
 541        else:
 542            error = self._try_extract_error_code(res)
 543            logger.error(
 544                "There was a problem getting this universe or model ID: {0}.".format(error)
 545            )
 546            raise BoostedAPIException("Failed to get universe: {0}".format(error))
 547
 548    def add_custom_security_namespace_members(
 549        self, namespace, members: Union[pandas.DataFrame, str]
 550    ) -> Tuple[pandas.DataFrame, str]:
 551        url = self.base_uri + "/api/synthetic-datasets/{0}/generate".format(namespace)
 552        headers = {"Authorization": "ApiKey " + self.api_key}
 553        headers["Content-Type"] = "application/json"
 554        logger.info("Adding custom security namespace for namespace: {0}".format(namespace))
 555        strbuf = None
 556        if isinstance(members, pandas.DataFrame):
 557            df = members
 558            df_canon = df.rename(columns={_: to_camel_case(_) for _ in df.columns})
 559            canon_cols = ["Currency", "Symbol", "Country", "Name"]
 560            if set(canon_cols).difference(df_canon.columns):
 561                raise BoostedAPIException(f"Expected columns: {canon_cols}")
 562            df_canon = df_canon.loc[:, canon_cols]
 563            buf = io.StringIO()
 564            df_canon.to_json(buf, orient="records")
 565            strbuf = buf.getvalue()
 566        elif isinstance(members, str):
 567            strbuf = members
 568        else:
 569            raise BoostedAPIException(f"Unsupported members argument type: {type(members)}")
 570        res = requests.post(url, data=strbuf, headers=headers, **self._request_params)
 571        if res.ok:
 572            res_obj = res.json()
 573            res_df = pandas.Series(res_obj["generatedISIN"]).to_frame()
 574            res_df.index.name = "Symbol"
 575            res_df.columns = ["ISIN"]
 576            logger.info("Add to custom security namespace successful.")
 577            if "warnings" in res_obj:
 578                logger.info("Warnings: {0}.".format(res.json()["warnings"]))
 579                return res_df, res.json()["warnings"]
 580            else:
 581                return res_df, "No warnings."
 582        else:
 583            error_msg = self._try_extract_error_code(res)
 584            raise BoostedAPIException("Failed to get universe: {0}.".format(error_msg))
 585
 586    def updateUniverse(self, modelId, universe_df, date=datetime.date.today() + timedelta(1)):
 587        date = self.__iso_format(date)
 588        url = self.base_uri + "/api/models/{0}/universe/{1}".format(modelId, date)
 589        headers = {"Authorization": "ApiKey " + self.api_key}
 590        logger.info("Updating universe for date {0}.".format(date))
 591        if isinstance(universe_df, pd.core.frame.DataFrame):
 592            buf = io.StringIO()
 593            universe_df.to_csv(buf)
 594            target = ("uploaded_universe.csv", buf.getvalue(), "text/csv")
 595            files_req = {}
 596            files_req["universe"] = target
 597            res = requests.post(url, files=files_req, headers=headers, **self._request_params)
 598        elif isinstance(universe_df, str):
 599            target = ("uploaded_universe.csv", universe_df, "text/csv")
 600            files_req = {}
 601            files_req["universe"] = target
 602            res = requests.post(url, files=files_req, headers=headers, **self._request_params)
 603        else:
 604            raise BoostedAPIException("Expected CSV as str or Pandas DataFrame.")
 605        if res.ok:
 606            logger.info("Universe update successful.")
 607            if "warnings" in res.json():
 608                logger.info("Warnings: {0}.".format(res.json()["warnings"]))
 609                return res.json()["warnings"]
 610            else:
 611                return "No warnings."
 612        else:
 613            error_msg = self._try_extract_error_code(res)
 614            raise BoostedAPIException("Failed to get universe: {0}.".format(error_msg))
 615
 616    def create_universe(
 617        self, universe: Union[pd.DataFrame, str], name: str, description: str
 618    ) -> List[str]:
 619        PRESENT = "PRESENT"
 620        ANY = "ANY"
 621        EARLIST_DATE = "1900-01-01"
 622        LATEST_DATE = "4000-01-01"
 623
 624        if isinstance(universe, (str, bytes, os.PathLike)):
 625            universe = pd.read_csv(universe)
 626
 627        universe.columns = universe.columns.str.lower()
 628
 629        # Clients are free to leave out data. Fill in some defaults here.
 630        if "from" not in universe.columns:
 631            universe["from"] = EARLIST_DATE
 632        if "to" not in universe.columns:
 633            universe["to"] = LATEST_DATE
 634        if "currency" not in universe.columns:
 635            universe["currency"] = ANY
 636        if "country" not in universe.columns:
 637            universe["country"] = ANY
 638        if "isin" not in universe.columns:
 639            universe["isin"] = None
 640        if "symbol" not in universe.columns:
 641            universe["symbol"] = None
 642
 643        # to prevent conflicts with python keywords
 644        universe.rename(columns={"from": "from_date", "to": "to_date"}, inplace=True)
 645
 646        universe = universe.replace({np.nan: None})
 647        security_country_currency_date_list = []
 648        for i, r in enumerate(universe.itertuples()):
 649            id_type = ColumnSubRole.ISIN
 650            identifier = r.isin
 651
 652            if identifier is None:
 653                id_type = ColumnSubRole.SYMBOL
 654                identifier = str(r.symbol)
 655
 656            # if identifier is still None, it means that there is no ISIN or
 657            # SYMBOL for this row, in which case we throw an error
 658            if identifier is None:
 659                raise BoostedAPIException(
 660                    (
 661                        f"Missing identifier column in universe row {i + 1}"
 662                        " should contain ISIN or Symbol"
 663                    )
 664                )
 665
 666            security_country_currency_date_list.append(
 667                DateIdentCountryCurrency(
 668                    date=r.from_date or EARLIST_DATE,
 669                    identifier=identifier,
 670                    country=r.country or ANY,
 671                    currency=r.currency or ANY,
 672                    id_type=id_type,
 673                )
 674            )
 675
 676        gbi_id_objs = self.getGbiIdFromIdentCountryCurrencyDate(security_country_currency_date_list)
 677
 678        security_list = []
 679        for i, r in enumerate(universe.itertuples()):
 680            # if we have a None here, we failed to map to a gbi id
 681            if gbi_id_objs[i] is None:
 682                raise BoostedAPIException(f"Unable to map row: {tuple(r)}")
 683
 684            security_list.append(
 685                {
 686                    "stockId": gbi_id_objs[i].gbi_id,
 687                    "fromZ": r.from_date or EARLIST_DATE,
 688                    "toZ": LATEST_DATE if r.to_date in (PRESENT, None) else r.to_date,
 689                    "removal": False,
 690                    "source": "UPLOAD",
 691                }
 692            )
 693
 694        url = self.base_uri + "/api/template-universe/save"
 695        headers = {"Authorization": "ApiKey " + self.api_key, "Content-Type": "application/json"}
 696        req = {"name": name, "description": description, "modificationDaos": security_list}
 697
 698        res = requests.post(url, json=req, headers=headers, **self._request_params)
 699        self._check_ok_or_err_with_msg(res, "Failed to create universe")
 700
 701        if "warnings" in res.json():
 702            logger.info("Warnings: {0}.".format(res.json()["warnings"]))
 703            return res.json()["warnings"].splitlines()
 704        else:
 705            return []
 706
 707    def validate_dataframe(self, df):
 708        if not isinstance(df, pd.core.frame.DataFrame):
 709            logger.error("Dataset must be of type Dataframe.")
 710            return False
 711        if type(df.index) != pd.core.indexes.datetimes.DatetimeIndex:
 712            logger.error("Index must be DatetimeIndex.")
 713            return False
 714        if len(df.columns) == 0:
 715            logger.error("No feature columns exist.")
 716            return False
 717        if len(df) == 0:
 718            logger.error("No rows exist.")
 719        return True
 720
 721    def get_dataset_schema(self, dataset_id):
 722        url = self.base_uri + "/api/datasets/{0}/schema".format(dataset_id)
 723        headers = {"Authorization": "ApiKey " + self.api_key}
 724        res = requests.get(url, headers=headers, **self._request_params)
 725        if res.ok:
 726            json_schema = res.json()
 727        else:
 728            error_msg = self._try_extract_error_code(res)
 729            logger.error(error_msg)
 730            raise BoostedAPIException("Failed to query dataset: {0}.".format(error_msg))
 731        return DataSetConfig.fromDict(json_schema["result"])
 732
 733    def add_custom_security_daily_dataset(
 734        self, namespace, dataset, schema=None, timeout=600, block=True
 735    ):
 736        result = self.add_custom_security_daily_dataset_with_warnings(
 737            namespace, dataset, schema, timeout, block
 738        )
 739        return result["dataset_id"]
 740
 741    def add_custom_security_daily_dataset_with_warnings(
 742        self,
 743        namespace,
 744        dataset,
 745        schema=None,
 746        timeout=600,
 747        block=True,
 748        no_exception_on_chunk_error=False,
 749    ):
 750        dataset_type = DataSetType.SECURITIES_DAILY
 751        dsid = self.query_namespace_dataset_id(namespace, dataset_type)
 752
 753        if not self.validate_dataframe(dataset):
 754            logger.error("dataset failed validation.")
 755            return None
 756
 757        if dsid is None:
 758            # create the dataset if not exist.
 759            schema = infer_dataset_schema(
 760                "custom_security_daily", dataset, dataset_type, infer_from_column_names=True
 761            )
 762            dsid = self.create_custom_namespace_dataset(namespace, schema.toDict())
 763            data_type = DataAddType.CREATION
 764        elif schema is not None:
 765            raise ValueError(
 766                f"Dataset schema already exists for namespace={namespace}, type={dataset_type}",
 767                ", cannot create another!",
 768            )
 769        else:
 770            data_type = DataAddType.HISTORICAL
 771
 772        logger.info("Created dataset with ID = {0}, uploading...".format(dsid))
 773        result = self.add_custom_security_daily_data(
 774            dsid,
 775            dataset,
 776            timeout,
 777            block,
 778            data_type=data_type,
 779            no_exception_on_chunk_error=no_exception_on_chunk_error,
 780        )
 781        return {
 782            "namespace": namespace,
 783            "dataset_id": dsid,
 784            "warnings": result["warnings"],
 785            "errors": result["errors"],
 786        }
 787
 788    def add_custom_security_daily_data(
 789        self,
 790        dataset_id,
 791        csv_data,
 792        timeout=600,
 793        block=True,
 794        data_type=DataAddType.HISTORICAL,
 795        no_exception_on_chunk_error=False,
 796    ):
 797        warnings = []
 798        query_info = self.query_dataset(dataset_id)
 799        if DataSetType[query_info["type"]] != DataSetType.SECURITIES_DAILY:
 800            raise BoostedAPIException(
 801                f"Incorrect dataset type: {query_info['type']}"
 802                f" - Expected {DataSetType.SECURITIES_DAILY}"
 803            )
 804        warnings, errors = self.setup_chunk_and_upload_data(
 805            dataset_id, csv_data, data_type, timeout, block, no_exception_on_chunk_error
 806        )
 807        if len(warnings) > 0:
 808            logger.warning(
 809                "Encountered {0} total warnings while uploading dataset.".format(len(warnings))
 810            )
 811        if len(errors) > 0:
 812            raise BoostedAPIException(
 813                "Encountered {0} total ERRORS while uploading dataset".format(len(errors))
 814                + "\n".join(errors)
 815            )
 816        return {"warnings": warnings, "errors": errors}
 817
 818    def add_dependent_dataset(
 819        self, dataset, datasetName="DependentDataset", schema=None, timeout=600, block=True
 820    ):
 821        result = self.add_dependent_dataset_with_warnings(
 822            dataset, datasetName, schema, timeout, block
 823        )
 824        return result["dataset_id"]
 825
 826    def add_dependent_dataset_with_warnings(
 827        self,
 828        dataset,
 829        datasetName="DependentDataset",
 830        schema=None,
 831        timeout=600,
 832        block=True,
 833        no_exception_on_chunk_error=False,
 834    ):
 835        if not self.validate_dataframe(dataset):
 836            logger.error("dataset failed validation.")
 837            return None
 838        if schema is None:
 839            schema = infer_dataset_schema(datasetName, dataset, DataSetType.STOCK)
 840        dsid = self.createDataset(schema.toDict())
 841        logger.info("Creating dataset with ID = {0}.".format(dsid))
 842        result = self.add_dependent_data(
 843            dsid,
 844            dataset,
 845            timeout,
 846            block,
 847            data_type=DataAddType.CREATION,
 848            no_exception_on_chunk_error=no_exception_on_chunk_error,
 849        )
 850        return {"dataset_id": dsid, "warnings": result["warnings"], "errors": result["errors"]}
 851
 852    def add_independent_dataset(
 853        self, dataset, datasetName="IndependentDataset", schema=None, timeout=600, block=True
 854    ):
 855        result = self.add_independent_dataset_with_warnings(
 856            dataset, datasetName, schema, timeout, block
 857        )
 858        return result["dataset_id"]
 859
 860    def add_independent_dataset_with_warnings(
 861        self,
 862        dataset,
 863        datasetName="IndependentDataset",
 864        schema=None,
 865        timeout=600,
 866        block=True,
 867        no_exception_on_chunk_error=False,
 868    ):
 869        if not self.validate_dataframe(dataset):
 870            logger.error("dataset failed validation.")
 871            return None
 872        if schema is None:
 873            schema = infer_dataset_schema(datasetName, dataset, DataSetType.STRATEGY)
 874        schemaDict = schema.toDict()
 875        if "configurationDataJson" not in schemaDict:
 876            schemaDict["configurationDataJson"] = "{}"
 877        dsid = self.createDataset(schemaDict)
 878        logger.info("Creating dataset with ID = {0}.".format(dsid))
 879        result = self.add_independent_data(
 880            dsid,
 881            dataset,
 882            timeout,
 883            block,
 884            data_type=DataAddType.CREATION,
 885            no_exception_on_chunk_error=no_exception_on_chunk_error,
 886        )
 887        return {"dataset_id": dsid, "warnings": result["warnings"], "errors": result["errors"]}
 888
 889    def add_global_dataset(
 890        self, dataset, datasetName="GlobalDataset", schema=None, timeout=600, block=True
 891    ):
 892        result = self.add_global_dataset_with_warnings(dataset, datasetName, schema, timeout, block)
 893        return result["dataset_id"]
 894
 895    def add_global_dataset_with_warnings(
 896        self,
 897        dataset,
 898        datasetName="GlobalDataset",
 899        schema=None,
 900        timeout=600,
 901        block=True,
 902        no_exception_on_chunk_error=False,
 903    ):
 904        if not self.validate_dataframe(dataset):
 905            logger.error("dataset failed validation.")
 906            return None
 907        if schema is None:
 908            schema = infer_dataset_schema(datasetName, dataset, DataSetType.GLOBAL)
 909        dsid = self.createDataset(schema.toDict())
 910        logger.info("Creating dataset with ID = {0}.".format(dsid))
 911        result = self.add_global_data(
 912            dsid,
 913            dataset,
 914            timeout,
 915            block,
 916            data_type=DataAddType.CREATION,
 917            no_exception_on_chunk_error=no_exception_on_chunk_error,
 918        )
 919        return {"dataset_id": dsid, "warnings": result["warnings"], "errors": result["errors"]}
 920
 921    def add_independent_data(
 922        self,
 923        dataset_id,
 924        csv_data,
 925        timeout=600,
 926        block=True,
 927        data_type=DataAddType.HISTORICAL,
 928        no_exception_on_chunk_error=False,
 929    ):
 930        query_info = self.query_dataset(dataset_id)
 931        if DataSetType[query_info["type"]] != DataSetType.STRATEGY:
 932            raise BoostedAPIException(
 933                f"Incorrect dataset type: {query_info['type']}"
 934                f" - Expected {DataSetType.STRATEGY}"
 935            )
 936        warnings, errors = self.setup_chunk_and_upload_data(
 937            dataset_id, csv_data, data_type, timeout, block, no_exception_on_chunk_error
 938        )
 939        if len(warnings) > 0:
 940            logger.warning(
 941                "Encountered {0} total warnings while uploading dataset.".format(len(warnings))
 942            )
 943        if len(errors) > 0:
 944            raise BoostedAPIException(
 945                "Encountered {0} total ERRORS while uploading dataset".format(len(errors))
 946                + "\n".join(errors)
 947            )
 948        return {"warnings": warnings, "errors": errors}
 949
 950    def add_dependent_data(
 951        self,
 952        dataset_id,
 953        csv_data,
 954        timeout=600,
 955        block=True,
 956        data_type=DataAddType.HISTORICAL,
 957        no_exception_on_chunk_error=False,
 958    ):
 959        warnings = []
 960        query_info = self.query_dataset(dataset_id)
 961        if DataSetType[query_info["type"]] != DataSetType.STOCK:
 962            raise BoostedAPIException(
 963                f"Incorrect dataset type: {query_info['type']}" f" - Expected {DataSetType.STOCK}"
 964            )
 965        warnings, errors = self.setup_chunk_and_upload_data(
 966            dataset_id, csv_data, data_type, timeout, block, no_exception_on_chunk_error
 967        )
 968        if len(warnings) > 0:
 969            logger.warning(
 970                "Encountered {0} total warnings while uploading dataset.".format(len(warnings))
 971            )
 972        if len(errors) > 0:
 973            raise BoostedAPIException(
 974                "Encountered {0} total ERRORS while uploading dataset".format(len(errors))
 975                + "\n".join(errors)
 976            )
 977        return {"warnings": warnings, "errors": errors}
 978
 979    def add_global_data(
 980        self,
 981        dataset_id,
 982        csv_data,
 983        timeout=600,
 984        block=True,
 985        data_type=DataAddType.HISTORICAL,
 986        no_exception_on_chunk_error=False,
 987    ):
 988        query_info = self.query_dataset(dataset_id)
 989        if DataSetType[query_info["type"]] != DataSetType.GLOBAL:
 990            raise BoostedAPIException(
 991                f"Incorrect dataset type: {query_info['type']}" f" - Expected {DataSetType.GLOBAL}"
 992            )
 993        warnings, errors = self.setup_chunk_and_upload_data(
 994            dataset_id, csv_data, data_type, timeout, block, no_exception_on_chunk_error
 995        )
 996        if len(warnings) > 0:
 997            logger.warning(
 998                "Encountered {0} total warnings while uploading dataset.".format(len(warnings))
 999            )
1000        if len(errors) > 0:
1001            raise BoostedAPIException(
1002                "Encountered {0} total ERRORS while uploading dataset".format(len(errors))
1003                + "\n".join(errors)
1004            )
1005        return {"warnings": warnings, "errors": errors}
1006
1007    def get_csv_buffer(self):
1008        return io.StringIO()
1009
1010    def start_chunked_upload(self, dataset_id):
1011        url = self.base_uri + "/api/datasets/{0}/start-chunked-upload".format(dataset_id)
1012        headers = {"Authorization": "ApiKey " + self.api_key}
1013        res = requests.post(url, headers=headers, **self._request_params)
1014        if res.ok:
1015            return res.json()["result"]
1016        else:
1017            error_msg = self._try_extract_error_code(res)
1018            logger.error(error_msg)
1019            raise BoostedAPIException(
1020                "Failed to obtain dataset lock for upload: {0}.".format(error_msg)
1021            )
1022
1023    def abort_chunked_upload(self, dataset_id, chunk_id):
1024        url = self.base_uri + "/api/datasets/{0}/abort-chunked-upload".format(dataset_id)
1025        headers = {"Authorization": "ApiKey " + self.api_key}
1026        params = {"uploadGroupId": chunk_id}
1027        res = requests.post(url, headers=headers, **self._request_params, params=params)
1028        if not res.ok:
1029            error_msg = self._try_extract_error_code(res)
1030            logger.error(error_msg)
1031            raise BoostedAPIException(
1032                "Failed to abort dataset lock during error: {0}.".format(error_msg)
1033            )
1034
1035    def check_dataset_ingestion_completion(self, dataset_id, chunk_id, start_time):
1036        url = self.base_uri + "/api/datasets/{0}/upload-chunk-status".format(dataset_id)
1037        headers = {"Authorization": "ApiKey " + self.api_key}
1038        params = {"uploadGroupId": chunk_id}
1039        res = requests.get(url, headers=headers, **self._request_params, params=params)
1040        res = res.json()
1041
1042        finished = False
1043        warnings = []
1044        errors = []
1045
1046        if type(res) == dict:
1047            dataset_status = res["datasetStatus"]
1048            chunk_status = res["chunkStatus"]
1049            if chunk_status != ChunkStatus.PROCESSING.value:
1050                finished = True
1051                errors = res["errors"]
1052                warnings = res["warnings"]
1053                successful_rows = res["successfulRows"]
1054                total_rows = res["totalRows"]
1055                logger.info(
1056                    f"Successfully ingested {successful_rows} out of {total_rows} uploaded rows."
1057                )
1058                if chunk_status in [
1059                    ChunkStatus.SUCCESS.value,
1060                    ChunkStatus.WARNING.value,
1061                    ChunkStatus.ERROR.value,
1062                ]:
1063                    if dataset_status != "AVAILABLE":
1064                        raise BoostedAPIException(
1065                            "Dataset was unexpectedly unavailable after chunk upload finished."
1066                        )
1067                    else:
1068                        logger.info("Ingestion complete.  Uploaded data is ready for use.")
1069                elif chunk_status == ChunkStatus.ABORTED.value:
1070                    errors.append(
1071                        "Dataset chunk upload was aborted by server! Upload did not succeed."
1072                    )
1073                else:
1074                    errors.append("Unexpected data ingestion status: {0}.".format(chunk_status))
1075            logger.info(
1076                "Data ingestion still running.  Time elapsed={0}.".format(
1077                    datetime.datetime.now() - start_time
1078                )
1079            )
1080        else:
1081            raise BoostedAPIException("Unable to get status of dataset ingestion.")
1082        return {"finished": finished, "warnings": warnings, "errors": errors}
1083
1084    def _commit_chunked_upload(self, dataset_id, chunk_id, data_type, block=True, timeout=600):
1085        url = self.base_uri + "/api/datasets/{0}/commit-chunked-upload".format(dataset_id)
1086        headers = {"Authorization": "ApiKey " + self.api_key}
1087        params = {
1088            "uploadGroupId": chunk_id,
1089            "dataAddType": data_type,
1090            "sendCompletionEmail": not block,
1091        }
1092        res = requests.post(url, headers=headers, **self._request_params, params=params)
1093        if not res.ok:
1094            error_msg = self._try_extract_error_code(res)
1095            logger.error(error_msg)
1096            raise BoostedAPIException("Failed to commit dataset files: {0}.".format(error_msg))
1097
1098        if block:
1099            start_time = datetime.datetime.now()
1100            # Keep waiting until upload is no longer in UPDATING state...
1101            while True:
1102                result = self.check_dataset_ingestion_completion(dataset_id, chunk_id, start_time)
1103                if result["finished"]:
1104                    break
1105
1106                if (datetime.datetime.now() - start_time).total_seconds() > timeout:
1107                    err_str = (
1108                        f"Timeout waiting for commit of dataset: {dataset_id} | chunk: {chunk_id}"
1109                    )
1110                    logger.error(err_str)
1111                    return [], [err_str]
1112
1113                time.sleep(10)
1114            return result["warnings"], result["errors"]
1115        else:
1116            return [], []
1117
1118    def setup_chunk_and_upload_data(
1119        self,
1120        dataset_id,
1121        csv_data,
1122        data_type,
1123        timeout=600,
1124        block=True,
1125        no_exception_on_chunk_error=False,
1126    ):
1127        chunk_id = self.start_chunked_upload(dataset_id)
1128        logger.info("Obtained lock on dataset for upload: " + chunk_id)
1129        try:
1130            warnings, errors = self.chunk_and_upload_data(
1131                dataset_id, chunk_id, csv_data, timeout, no_exception_on_chunk_error
1132            )
1133            commit_warnings, commit_errors = self._commit_chunked_upload(
1134                dataset_id, chunk_id, data_type, block, timeout
1135            )
1136            return warnings + commit_warnings, errors + commit_errors
1137        except Exception:
1138            self.abort_chunked_upload(dataset_id, chunk_id)
1139            raise
1140
1141    def chunk_and_upload_data(
1142        self, dataset_id, chunk_id, csv_data, timeout=600, no_exception_on_chunk_error=False
1143    ):
1144        if isinstance(csv_data, pd.core.frame.DataFrame):
1145            if not isinstance(csv_data.index, pd.core.indexes.datetimes.DatetimeIndex):
1146                raise BoostedAPIException("DataFrame must have DatetimeIndex as index type.")
1147
1148            warnings = []
1149            errors = []
1150            logger.info("Uploading yearly.")
1151            for t in csv_data.index.to_period("Y").unique():
1152                if t is pd.NaT:
1153                    continue
1154
1155                # serialize bit to string
1156                buf = self.get_csv_buffer()
1157                yearly_csv = csv_data.loc[str(t)]
1158                yearly_csv.to_csv(buf, header=True)
1159                raw_csv = buf.getvalue()
1160
1161                # we are already chunking yearly... but if the csv still exceeds a healthy
1162                # limit of 50mb the final line of defence is to ignore date boundaries and
1163                # just chunk the rows. This is mostly for the cloudflare upload limit.
1164                size_lim = 50 * 1000 * 1000
1165                est_csv_size = sys.getsizeof(raw_csv)
1166                if est_csv_size > size_lim:
1167                    del raw_csv, buf
1168                    logger.info("Yearly data too large for single upload, chunking further...")
1169                    chunks = []
1170                    nchunks = math.ceil(est_csv_size / size_lim)
1171                    rows_per_chunk = math.ceil(len(yearly_csv) / nchunks)
1172                    for i in range(0, len(yearly_csv), rows_per_chunk):
1173                        buf = self.get_csv_buffer()
1174                        split_csv = yearly_csv.iloc[i : i + rows_per_chunk]
1175                        split_csv.to_csv(buf, header=True)
1176                        split_csv = buf.getvalue()
1177                        chunks.append(
1178                            (
1179                                "{0}-{1}".format(i + 1, min(len(yearly_csv), i + rows_per_chunk)),
1180                                split_csv,
1181                            )
1182                        )
1183                else:
1184                    chunks = [("all", raw_csv)]
1185
1186                for i, (rows_descriptor, chunk_csv) in enumerate(chunks):
1187                    chunk_descriptor = "{0} in yearly chunk {1}".format(rows_descriptor, t)
1188                    logger.info(
1189                        "Uploading rows:"
1190                        + chunk_descriptor
1191                        + " (chunk {0} of {1}):".format(i + 1, len(chunks))
1192                    )
1193                    _, new_warnings, new_errors = self.upload_dataset_chunk(
1194                        chunk_descriptor,
1195                        dataset_id,
1196                        chunk_id,
1197                        chunk_csv,
1198                        timeout,
1199                        no_exception_on_chunk_error,
1200                    )
1201                    warnings.extend(new_warnings)
1202                    errors.extend(new_errors)
1203            return warnings, errors
1204
1205        elif isinstance(csv_data, str):
1206            _, warnings, errors = self.upload_dataset_chunk(
1207                "all data", dataset_id, chunk_id, csv_data, timeout, no_exception_on_chunk_error
1208            )
1209            return warnings, errors
1210        else:
1211            raise BoostedAPIException("Expected CSV as str or Pandas DataFrame.")
1212
1213    def upload_dataset_chunk(
1214        self,
1215        chunk_descriptor,
1216        dataset_id,
1217        chunk_id,
1218        csv_data,
1219        timeout=600,
1220        no_exception_on_chunk_error=False,
1221    ):
1222        logger.info("Starting upload: " + chunk_descriptor)
1223        url = self.base_uri + "/api/datasets/{0}/upload-dataset-chunk".format(dataset_id)
1224        headers = {"Authorization": "ApiKey " + self.api_key}
1225        files_req = {}
1226        warnings = []
1227        errors = []
1228
1229        # make the network request
1230        target = ("uploaded_data.csv", csv_data, "text/csv")
1231        files_req["dataFile"] = target
1232        params = {"uploadGroupId": chunk_id}
1233        res = requests.post(
1234            url,
1235            params=params,
1236            files=files_req,
1237            headers=headers,
1238            timeout=timeout,
1239            **self._request_params,
1240        )
1241
1242        if res.ok:
1243            logger.info(
1244                (
1245                    "Chunk upload completed.  "
1246                    "Ingestion started.  "
1247                    "Please wait until the data is in AVAILABLE state."
1248                )
1249            )
1250            if "warnings" in res.json():
1251                warnings = res.json()["warnings"]
1252                if len(warnings) > 0:
1253                    logger.warning("Uploaded chunk encountered data warnings: ")
1254                for w in warnings:
1255                    logger.warning(w)
1256        else:
1257            reason = "Upload failed: {0}, {1}".format(res.text, res.reason)
1258            logger.error(reason)
1259            if no_exception_on_chunk_error:
1260                errors.append(
1261                    "Chunk {0} failed: {1}. ".format(chunk_descriptor, reason)
1262                    + "Your data was only PARTIALLY uploaded. "
1263                    + "Please reattempt the upload of this chunk."
1264                )
1265            else:
1266                raise BoostedAPIException(reason)
1267
1268        return res, warnings, errors
1269
1270    def getAllocationsForDate(self, portfolio_id, date, rollback_to_last_available_date):
1271        date = self.__iso_format(date)
1272        endpoint = "latest-allocations" if rollback_to_last_available_date else "allocations"
1273        url = self.base_uri + "/api/portfolios/{0}/{1}".format(portfolio_id, endpoint)
1274        headers = {"Authorization": "ApiKey " + self.api_key}
1275        params = {"date": date}
1276        logger.info("Retrieving allocations information for date {0}.".format(date))
1277        res = requests.get(url, params=params, headers=headers, **self._request_params)
1278        if res.ok:
1279            logger.info("Allocations retrieval successful.")
1280            return res.json()
1281        else:
1282            error_msg = self._try_extract_error_code(res)
1283            raise BoostedAPIException("Failed to retrieve allocations: {0}.".format(error_msg))
1284
1285    # New API method for fetching data from portfolio_holdings.pb2 file.
1286    def getAllocationsForDateV2(self, portfolio_id, date, rollback_to_last_available_date):
1287        date = self.__iso_format(date)
1288        endpoint = "latest-allocations-v2" if rollback_to_last_available_date else "allocations-v2"
1289        url = self.base_uri + "/api/portfolios/{0}/{1}".format(portfolio_id, endpoint)
1290        headers = {"Authorization": "ApiKey " + self.api_key}
1291        params = {"date": date}
1292        logger.info("Retrieving allocations information for date {0}.".format(date))
1293        res = requests.get(url, params=params, headers=headers, **self._request_params)
1294        if res.ok:
1295            logger.info("Allocations retrieval successful.")
1296            return res.json()
1297        else:
1298            error_msg = self._try_extract_error_code(res)
1299            raise BoostedAPIException("Failed to retrieve allocations: {0}.".format(error_msg))
1300
1301    def getAllocationsByDates(self, portfolio_id, dates=None):
1302        url = self.base_uri + "/api/portfolios/{0}/allocationsByDate".format(portfolio_id)
1303        headers = {"Authorization": "ApiKey " + self.api_key}
1304        if dates is not None:
1305            fmt_dates = []
1306            for d in dates:
1307                fmt_dates.append(self.__iso_format(d))
1308            fmt_dates_str = ",".join(fmt_dates)
1309            params: Dict = {"dates": fmt_dates_str}
1310            logger.info("Retrieving allocations information for dates {0}.".format(fmt_dates))
1311        else:
1312            params = {"dates": None}
1313            logger.info("Retrieving allocations information for all dates")
1314        res = requests.get(url, params=params, headers=headers, **self._request_params)
1315        if res.ok:
1316            logger.info("Allocations retrieval successful.")
1317            return res.json()
1318        else:
1319            error_msg = self._try_extract_error_code(res)
1320            raise BoostedAPIException("Failed to retrieve allocations: {0}.".format(error_msg))
1321
1322    def getSignalsForDate(self, portfolio_id, date, rollback_to_last_available_date):
1323        date = self.__iso_format(date)
1324        endpoint = "latest-signals" if rollback_to_last_available_date else "signals"
1325        url = self.base_uri + "/api/portfolios/{0}/{1}".format(portfolio_id, endpoint)
1326        headers = {"Authorization": "ApiKey " + self.api_key}
1327        params = {"date": date}
1328        logger.info("Retrieving signals information for date {0}.".format(date))
1329        res = requests.get(url, params=params, headers=headers, **self._request_params)
1330        if res.ok:
1331            logger.info("Signals retrieval successful.")
1332            return res.json()
1333        else:
1334            error_msg = self._try_extract_error_code(res)
1335            raise BoostedAPIException("Failed to retrieve signals: {0}.".format(error_msg))
1336
1337    def getSignalsForAllDates(self, portfolio_id, dates=None):
1338        url = self.base_uri + "/api/portfolios/{0}/signalsByDate".format(portfolio_id)
1339        headers = {"Authorization": "ApiKey " + self.api_key}
1340        params = {}
1341        if dates is not None:
1342            fmt_dates = []
1343            for d in dates:
1344                fmt_dates.append(self.__iso_format(d))
1345            fmt_dates_str = ",".join(fmt_dates)
1346            params = {"dates": fmt_dates_str}
1347            logger.info("Retrieving signals information for dates {0}.".format(fmt_dates))
1348        else:
1349            params = {"dates": None}
1350            logger.info("Retrieving signals information for all dates")
1351        res = requests.get(url, params=params, headers=headers, **self._request_params)
1352        if res.ok:
1353            logger.info("Signals retrieval successful.")
1354            return res.json()
1355        else:
1356            error_msg = self._try_extract_error_code(res)
1357            raise BoostedAPIException("Failed to retrieve signals: {0}.".format(error_msg))
1358
1359    def getEquityAccuracy(
1360        self,
1361        model_id: str,
1362        portfolio_id: str,
1363        tickers: List[str],
1364        start_date: Optional[BoostedDate] = None,
1365        end_date: Optional[BoostedDate] = None,
1366    ) -> Dict[str, Dict[str, Any]]:
1367        data: Dict[str, Any] = {}
1368        if start_date is not None:
1369            start_date = convert_date(start_date)
1370            data["startDate"] = start_date.isoformat()
1371        if end_date is not None:
1372            end_date = convert_date(end_date)
1373            data["endDate"] = end_date.isoformat()
1374
1375        if start_date and end_date:
1376            validate_start_and_end_dates(start_date, end_date)
1377
1378        tickers_stream = ",".join(tickers)
1379        data["tickers"] = tickers_stream
1380        data["timestamp"] = time.strftime("%H:%M:%S")
1381        data["shouldRecalc"] = True
1382        url = self.base_uri + f"/api/analysis/equity-accuracy/{model_id}/{portfolio_id}"
1383        headers = {"Authorization": "ApiKey " + self.api_key, "Content-Type": "application/json"}
1384
1385        logger.info(
1386            f"Retrieving equity accuracy data for date range {start_date} to {end_date} "
1387            f"for tickers: {tickers}."
1388        )
1389
1390        # Now create dataframes from the JSON output.
1391        metrics = [
1392            "hit_rate_mean",
1393            "hit_rate_median",
1394            "excess_return_mean",
1395            "excess_return_median",
1396            "return",
1397            "excess_return",
1398        ]
1399
1400        # send the request, retry if failed
1401        MAX_RETRIES = 10  # max of number of retries until timeout
1402        SLEEP_TIME = 3  # waiting time between requests
1403
1404        num_retries = 0
1405        success = False
1406        while not success and num_retries < MAX_RETRIES:
1407            res = requests.post(url, data=json.dumps(data), headers=headers, **self._request_params)
1408            if res.ok:
1409                logger.info("Equity Accuracy Data retrieval successful.")
1410                info = res.json()
1411                success = True
1412            else:
1413                data["shouldRecalc"] = False
1414                num_retries += 1
1415                time.sleep(SLEEP_TIME)
1416
1417        if not success:
1418            raise BoostedAPIException("Failed to retrieve equity accuracy: Request timeout.")
1419
1420        for ticker, accuracy_data in info.items():
1421            for metric in metrics:
1422                metric_matrix = accuracy_data[metric]
1423                if not isinstance(metric_matrix, str):
1424                    # Set the index to the quintile label, and remove it from the data
1425                    index = []
1426                    for row in metric_matrix[1:]:
1427                        index.append(row.pop(0))
1428
1429                    # columns are "1D", "5D", etc.
1430                    df = pd.DataFrame(metric_matrix[1:], columns=metric_matrix[0][1:], index=index)
1431                    accuracy_data[metric] = df
1432        return info
1433
1434    def getHistoricalTradeDates(self, portfolio_id, start_date=None, end_date=None):
1435        end_date = self.__to_date_obj(end_date or datetime.date.today())
1436        start_date = self.__iso_format(start_date or (end_date - timedelta(days=365)))
1437        end_date = self.__iso_format(end_date)
1438
1439        url = self.base_uri + "/api/portfolios/{0}/tradingDates".format(portfolio_id)
1440        headers = {"Authorization": "ApiKey " + self.api_key}
1441        params = {"startDate": start_date, "endDate": end_date}
1442
1443        logger.info(
1444            "Retrieving historical trade dates data for date range {0} to {1}.".format(
1445                start_date, end_date
1446            )
1447        )
1448        res = requests.get(url, params=params, headers=headers, **self._request_params)
1449        if res.ok:
1450            logger.info("Trading dates retrieval successful.")
1451            return res.json()["dates"]
1452        else:
1453            error_msg = self._try_extract_error_code(res)
1454            raise BoostedAPIException("Failed to retrieve trading dates: {0}.".format(error_msg))
1455
1456    def getRankingsForAllDates(self, portfolio_id, dates=None):
1457        url = self.base_uri + "/api/portfolios/{0}/rankingsByDate".format(portfolio_id)
1458        headers = {"Authorization": "ApiKey " + self.api_key}
1459        params = {}
1460        if dates is not None:
1461            fmt_dates = []
1462            for d in dates:
1463                fmt_dates.append(self.__iso_format(d))
1464            fmt_dates_str = ",".join(fmt_dates)
1465            params = {"dates": fmt_dates_str}
1466            logger.info("Retrieving rankings information for date {0}.".format(fmt_dates_str))
1467        else:
1468            params = {"dates": None}
1469            logger.info("Retrieving rankings information for all dates")
1470        res = requests.get(url, params=params, headers=headers, **self._request_params)
1471        if res.ok:
1472            logger.info("Rankings retrieval successful.")
1473            return res.json()
1474        else:
1475            error_msg = self._try_extract_error_code(res)
1476            raise BoostedAPIException("Failed to retrieve rankings: {0}.".format(error_msg))
1477
1478    def getRankingsForDate(self, portfolio_id, date, rollback_to_last_available_date):
1479        date = self.__iso_format(date)
1480        endpoint = "latest-rankings" if rollback_to_last_available_date else "rankings"
1481        url = self.base_uri + "/api/{0}/{1}/{2}".format(endpoint, portfolio_id, date)
1482        headers = {"Authorization": "ApiKey " + self.api_key}
1483        logger.info("Retrieving rankings information for date {0}.".format(date))
1484        res = requests.get(url, headers=headers, **self._request_params)
1485        if res.ok:
1486            logger.info("Rankings retrieval successful.")
1487            return res.json()
1488        else:
1489            error_msg = self._try_extract_error_code(res)
1490            raise BoostedAPIException("Failed to retrieve rankings: {0}.".format(error_msg))
1491
1492    def sendModelRecalc(self, model_id):
1493        url = self.base_uri + "/api/models/{0}/recalc".format(model_id)
1494        logger.info("Sending model recalc request for model {0}".format(model_id))
1495        headers = {"Authorization": "ApiKey " + self.api_key}
1496        res = requests.put(url, headers=headers, **self._request_params)
1497        if not res.ok:
1498            error_msg = self._try_extract_error_code(res)
1499            logger.error(error_msg)
1500            raise BoostedAPIException(
1501                "Failed to send model recalc request - "
1502                + "the model in UI may be out of date: {0}.".format(error_msg)
1503            )
1504
1505    def sendRecalcAllModelPortfolios(self, model_id: str):
1506        """Recalculates all portfolios under a given model ID.
1507
1508        Args:
1509            model_id: the model ID
1510        Raises:
1511            BoostedAPIException: if the Boosted API request fails
1512        """
1513        url = self.base_uri + f"/api/models/{model_id}/recalc-all-portfolios"
1514        logger.info(f"Sending portfolio recalc requests for all portfolios under {model_id=}.")
1515        headers = {"Authorization": "ApiKey " + self.api_key}
1516        res = requests.put(url, headers=headers, **self._request_params)
1517        if not res.ok:
1518            error_msg = self._try_extract_error_code(res)
1519            logger.error(error_msg)
1520            raise BoostedAPIException(
1521                f"Failed to send recalc request for all portfolios under {model_id=} - {error_msg}."
1522            )
1523
1524    def sendPortfolioRecalc(self, portfolio_id: str):
1525        """Recalculates a single portfolio by its portfolio ID.
1526
1527        Args:
1528            portfolio_id: the portfolio ID to recalculate
1529        Raises:
1530            BoostedAPIException: if the Boosted API request fails
1531        """
1532        url = self.base_uri + "/api/graphql"
1533        logger.info(f"Sending portfolio recalc request for {portfolio_id=}.")
1534        headers = {"Authorization": "ApiKey " + self.api_key, "Content-Type": "application/json"}
1535        qry = """
1536            mutation recalcPortfolio($input: RecalculatePortfolioInput!) {
1537                recalculatePortfolio(input: $input) {
1538                    success
1539                    errors
1540                }
1541            }
1542            """
1543        req_json = {
1544            "query": qry,
1545            "variables": {"input": {"portfolioId": f"{portfolio_id}", "allowForceRecalc": "true"}},
1546        }
1547        res = requests.post(url, json=req_json, headers=headers, **self._request_params)
1548        if not res.ok or res.json().get("errors"):
1549            error_msg = self._try_extract_error_code(res)
1550            logger.error(error_msg)
1551            raise BoostedAPIException(
1552                f"Failed to send portfolio recalc request for {portfolio_id=} - {error_msg}."
1553            )
1554
1555    def add_uploaded_model_data(self, url, csv_data, request_data, timeout=600):
1556        logger.info("Starting upload.")
1557        headers = {"Authorization": "ApiKey " + self.api_key}
1558        files_req: Dict = {}
1559        target: Tuple[str, Any, str] = ("data.csv", None, "text/csv")
1560        warnings = []
1561        if isinstance(csv_data, pd.core.frame.DataFrame):
1562            buf = io.StringIO()
1563            csv_data.to_csv(buf, header=False)
1564            if not isinstance(csv_data.index, pd.core.indexes.datetimes.DatetimeIndex):
1565                raise BoostedAPIException("DataFrame must have DatetimeIndex as index type.")
1566            target = ("uploaded_data.csv", buf.getvalue(), "text/csv")
1567            files_req["dataFile"] = target
1568            res = requests.post(
1569                url,
1570                files=files_req,
1571                data=request_data,
1572                headers=headers,
1573                timeout=timeout,
1574                **self._request_params,
1575            )
1576        elif isinstance(csv_data, str):
1577            target = ("uploaded_data.csv", csv_data, "text/csv")
1578            files_req["dataFile"] = target
1579            res = requests.post(
1580                url,
1581                files=files_req,
1582                data=request_data,
1583                headers=headers,
1584                timeout=timeout,
1585                **self._request_params,
1586            )
1587        else:
1588            raise BoostedAPIException("Expected CSV as str or Pandas DataFrame.")
1589        if res.ok:
1590            logger.info("Signals upload completed.")
1591            result = res.json()["result"]
1592            if "warningMessages" in result:
1593                warnings = result["warningMessages"]
1594        else:
1595            error_str = "Signals upload failed: {0}, {1}".format(res.text, res.reason)
1596            logger.error(error_str)
1597            raise BoostedAPIException(error_str)
1598
1599        return res, warnings
1600
1601    def createSignalsModel(self, csv_data, model_name, timeout=600):
1602        warnings = []
1603        url = self.base_uri + "/api/models/upload/signals/create"
1604        request_data = {"modelName": model_name, "uploadName": model_name}
1605        res, warnings = self.add_uploaded_model_data(url, csv_data, request_data, timeout)
1606        result = res.json()["result"]
1607        model_id = result["modelId"]
1608        self.sendModelRecalc(model_id)
1609        return model_id, warnings
1610
1611    def addToUploadedModel(self, model_id, csv_data, timeout=600, recalc_model=True):
1612        warnings = []
1613        url = self.base_uri + "/api/models/{0}/upload/add-data".format(model_id)
1614        request_data: Dict = {}
1615        _, warnings = self.add_uploaded_model_data(url, csv_data, request_data, timeout)
1616        if recalc_model:
1617            self.sendModelRecalc(model_id)
1618        return warnings
1619
1620    def addSignalsToUploadedModel(
1621        self,
1622        model_id: str,
1623        csv_data: Union[pd.core.frame.DataFrame, str],
1624        timeout: int = 600,
1625        recalc_all: bool = False,
1626        recalc_portfolio_ids: Optional[List[str]] = None,
1627    ) -> List[str]:
1628        """
1629        Add signals to an uploaded model and then recalculate a random portfolio under that model.
1630
1631        Args:
1632            model_id: model ID
1633            csv_data: pandas DataFrame, or a string with signals to upload.
1634            timeout (optional): Timeout for initial upload request in seconds.
1635            recalc_all (optional): if True, recalculates all portfolios in the model.
1636            recalc_portfolio_ids (optional): List of portfolio IDs under the model to re-calculate.
1637        """
1638        warnings = self.addToUploadedModel(model_id, csv_data, timeout, recalc_model=False)
1639
1640        if recalc_all:
1641            self.sendRecalcAllModelPortfolios(model_id)
1642        elif recalc_portfolio_ids:
1643            for portfolio_id in recalc_portfolio_ids:
1644                self.sendPortfolioRecalc(portfolio_id)
1645        else:
1646            self.sendModelRecalc(model_id)
1647        return warnings
1648
1649    def getSignalsFromUploadedModel(self, model_id, date=None):
1650        date = self.__iso_format(date)
1651        url = self.base_uri + "/api/models/{0}/upload/signals".format(model_id)
1652        headers = {"Authorization": "ApiKey " + self.api_key}
1653        params = {"date": date}
1654        logger.info("Retrieving uploaded signals information")
1655        res = requests.get(url, params=params, headers=headers, **self._request_params)
1656        if res.ok:
1657            result = pd.DataFrame.from_dict(res.json()["result"])
1658            # ensure column order
1659            result = result[["date", "isin", "country", "currency", "weight"]]
1660            result["date"] = pd.to_datetime(result["date"], format="%Y-%m-%d")
1661            result = result.set_index("date")
1662            logger.info("Signals retrieval successful.")
1663            return result
1664        else:
1665            error_msg = self._try_extract_error_code(res)
1666            raise BoostedAPIException("Failed to retrieve signals: {0}.".format(error_msg))
1667
1668    def getPortfolioSettings(self, portfolio_id, timeout=600):
1669        url = self.base_uri + "/api/portfolio-settings/{0}".format(portfolio_id)
1670        headers = {"Authorization": "ApiKey " + self.api_key}
1671        res = requests.get(url, headers=headers, **self._request_params)
1672        if res.ok:
1673            return PortfolioSettings(res.json())
1674        else:
1675            error_msg = self._try_extract_error_code(res)
1676            logger.error(error_msg)
1677            raise BoostedAPIException(
1678                "Failed to retrieve portfolio settings: {0}.".format(error_msg)
1679            )
1680
1681    def createPortfolioWithPortfolioSettings(
1682        self, model_id, portfolio_name, portfolio_description, portfolio_settings, timeout=600
1683    ):
1684        url = self.base_uri + "/api/models/{0}/constraints/add".format(model_id)
1685        headers = {"Authorization": "ApiKey " + self.api_key, "Content-Type": "application/json"}
1686        setting_string = json.dumps(portfolio_settings.settings)
1687        logger.info("Creating new portfolio with specified setting: {}".format(setting_string))
1688        params = {
1689            "name": portfolio_name,
1690            "description": portfolio_description,
1691            "settings": setting_string,
1692            "validate": "true",
1693        }
1694        res = requests.put(url, json=params, headers=headers, **self._request_params)
1695        response = res.json()
1696        if res.ok:
1697            return response
1698        else:
1699            error_msg = self._try_extract_error_code(res)
1700            logger.error(error_msg)
1701            raise BoostedAPIException(
1702                "Failed to create portfolio with the specified settings: {0}.".format(error_msg)
1703            )
1704
1705    def getGbiIdFromIdentCountryCurrencyDate(
1706        self, ident_country_currency_dates: List[DateIdentCountryCurrency], timeout: int = 600
1707    ) -> List[Optional[GbiIdSecurity]]:
1708        url = self.base_uri + "/api/custom-stock-data/map-identifiers-simple"
1709        headers = {"Authorization": "ApiKey " + self.api_key, "Content-Type": "application/json"}
1710        identifiers = [
1711            {
1712                "row": idx,
1713                "date": identifier.date,
1714                "isin": identifier.identifier if identifier.id_type == ColumnSubRole.ISIN else None,
1715                "symbol": (
1716                    identifier.identifier if identifier.id_type == ColumnSubRole.SYMBOL else None
1717                ),
1718                "countryPreference": identifier.country,
1719                "currencyPreference": identifier.currency,
1720            }
1721            for idx, identifier in enumerate(ident_country_currency_dates)
1722        ]
1723        params = json.dumps({"identifiers": identifiers})
1724        logger.info(
1725            "Retrieving GBI-ID mapping for {} identifier tuples...".format(
1726                len(ident_country_currency_dates)
1727            )
1728        )
1729        res = requests.post(url, data=params, headers=headers, **self._request_params)
1730
1731        if res.ok:
1732            result = res.json()
1733            warnings = result["warnings"]
1734            if warnings:
1735                for warning in warnings:
1736                    logger.warn(f"Mapping warning: {warning}")
1737            gbiSecurities = []
1738            for idx, ident in enumerate(result["mappedIdentifiers"]):
1739                if ident is None:
1740                    security = None
1741                else:
1742                    security = GbiIdSecurity(
1743                        ident["gbiId"],
1744                        ident_country_currency_dates[idx],
1745                        ident["symbol"],
1746                        ident["companyName"],
1747                    )
1748                gbiSecurities.append(security)
1749
1750            return gbiSecurities
1751        else:
1752            error_msg = self._try_extract_error_code(res)
1753            raise BoostedAPIException(
1754                "Failed to retrieve identifier mappings: {0}.".format(error_msg)
1755            )
1756
1757    # exists for backwards compatibility purposes.
1758    def getGbiIdFromIsinCountryCurrencyDate(self, isin_country_currency_dates, timeout=600):
1759        return self.getGbiIdFromIdentCountryCurrencyDate(
1760            ident_country_currency_dates=isin_country_currency_dates, timeout=timeout
1761        )
1762
1763    # model_id: str
1764    # returns: Dict[str, str] representing the translation from the rankings ID (feature refs)
1765    # to human readable names
1766    def __get_rankings_ref_translation(self, model_id: str) -> Dict[str, str]:
1767        headers = {"Authorization": "ApiKey " + self.api_key, "Content-Type": "application/json"}
1768        feature_name_url = f"/api/models/{model_id}/advanced-explain/translate-feature-ref/"
1769        feature_name_res = requests.post(
1770            self.base_uri + feature_name_url,
1771            data=json.dumps({}),
1772            headers=headers,
1773            **self._request_params,
1774        )
1775
1776        if feature_name_res.ok:
1777            feature_name_dict = feature_name_res.json()
1778            return {
1779                id: "-".join(
1780                    [names["variable_name"], names["transform_name"], names["normalization_name"]]
1781                )
1782                for id, names in feature_name_dict.items()
1783            }
1784        else:
1785            raise Exception(
1786                """Failed to get feature names for model,
1787                    this model doesn't fully support rankings 2.0"""
1788            )
1789
1790    def getDatasetDates(self, dataset_id):
1791        url = self.base_uri + f"/api/datasets/{dataset_id}"
1792        headers = {"Authorization": "ApiKey " + self.api_key}
1793        res = requests.get(url, headers=headers, **self._request_params)
1794        if res.ok:
1795            dataset = res.json()
1796            valid_to_array = dataset.get("validTo")
1797            valid_to_date = None
1798            valid_from_array = dataset.get("validFrom")
1799            valid_from_date = None
1800            if valid_to_array:
1801                valid_to_date = datetime.date(
1802                    valid_to_array[0], valid_to_array[1], valid_to_array[2]
1803                )
1804            if valid_from_array:
1805                valid_from_date = datetime.date(
1806                    valid_from_array[0], valid_from_array[1], valid_from_array[2]
1807                )
1808            return {"validTo": valid_to_date, "validFrom": valid_from_date}
1809        else:
1810            error_msg = self._try_extract_error_code(res)
1811            logger.error(error_msg)
1812            raise BoostedAPIException("Failed to query dataset: {0}.".format(error_msg))
1813
1814    def getRankingAnalysis(self, model_id, date):
1815        url = (
1816            self.base_uri
1817            + f"/api/explain-trades/analysis/{model_id}/{self.__iso_format(date)}/json"
1818        )
1819        headers = {"Authorization": "ApiKey " + self.api_key}
1820        analysis_res = requests.get(url, headers=headers, **self._request_params)
1821        if analysis_res.ok:
1822            ranking_dict = analysis_res.json()
1823            feature_name_dict = self.__get_rankings_ref_translation(model_id)
1824            columns = [feature_name_dict[col] for col in ranking_dict["columns"]]
1825
1826            df = protoCubeJsonDataToDataFrame(
1827                ranking_dict["data"],
1828                "Data Buckets",
1829                ranking_dict["rows"],
1830                "Feature Names",
1831                columns,
1832                ranking_dict["fields"],
1833            )
1834            return df
1835        else:
1836            error_msg = self._try_extract_error_code(analysis_res)
1837            logger.error(error_msg)
1838            raise BoostedAPIException("Failed to get ranking analysis: {0}.".format(error_msg))
1839
1840    def getExplainForPortfolio(
1841        self,
1842        model_id,
1843        portfolio_id,
1844        date,
1845        index_by_symbol: bool = False,
1846        index_by_all_metadata: bool = False,
1847    ):
1848        """
1849        Gets the ranking 2.0 explain data for the given model on the given date
1850        filtered by portfolio.
1851
1852        Parameters
1853        ----------
1854        model_id: str
1855            Model ID.  Model IDs can be retrieved by clicking on the copy to clipboard
1856            button next to your model's name in the Model Summary Page in Boosted
1857            Insights.
1858        portfolio_id: str
1859            Portfolio ID.  Portfolio IDs can be retrieved from portfolio's configuration page.
1860        date: datetime.date or YYYY-MM-DD string
1861            Date of the data to retrieve.
1862        index_by_symbol: bool
1863            If true, index by stock symbol instead of ISIN.
1864        index_by_all_metadata: bool
1865            If true, index by all metadata: ISIN, stock symbol, currency, and country.
1866            Overrides index_by_symbol.
1867
1868        Returns
1869        -------
1870        pandas.DataFrame
1871            Pandas DataFrame containing your data indexed by ISINs/Symbol/all metadata
1872            and feature names, filtered by portfolio.
1873        ___
1874        """
1875        indices = ["Symbol", "ISINs", "Country", "Currency"]
1876        raw_explain_df = self.getRankingExplain(
1877            model_id, date, index_by_symbol=False, index_by_all_metadata=True
1878        )
1879        pa_ratings_dict = self.getRankingsForDate(portfolio_id, date, False)
1880
1881        ratings = pa_ratings_dict["rankings"]
1882        ratings_df = pd.DataFrame(ratings)
1883        ratings_df = ratings_df[["symbol", "isin", "country", "currency"]]
1884        ratings_df.columns = pd.Index(indices)
1885        ratings_df.set_index(indices, inplace=True)
1886
1887        # inner join to only get the securities in both data frames
1888        result_df = raw_explain_df.merge(ratings_df, left_index=True, right_index=True, how="inner")
1889
1890        # set index based on input parameters
1891        if index_by_symbol and not index_by_all_metadata:
1892            result_df = result_df.reset_index()
1893            result_df = result_df.drop(columns=["ISINs", "Currency", "Country"])
1894            result_df.set_index(["Symbol", "Feature Names"], inplace=True)
1895        elif not index_by_symbol and not index_by_all_metadata:
1896            result_df = result_df.reset_index()
1897            result_df = result_df.drop(columns=["Symbol", "Currency", "Country"])
1898            result_df.set_index(["ISINs", "Feature Names"], inplace=True)
1899
1900        return result_df
1901
1902    def getRankingExplain(
1903        self, model_id, date, index_by_symbol: bool = False, index_by_all_metadata: bool = False
1904    ):
1905        """
1906        Gets the ranking 2.0 explain data for the given model on the given date
1907
1908        Parameters
1909        ----------
1910        model_id: str
1911            Model ID.  Model IDs can be retrieved by clicking on the copy to clipboard
1912            button next to your model's name in the Model Summary Page in Boosted
1913            Insights.
1914        date: datetime.date or YYYY-MM-DD string
1915            Date of the data to retrieve.
1916        index_by_symbol: bool
1917            If true, index by stock symbol instead of ISIN.
1918        index_by_all_metadata: bool
1919            If true, index by all metadata: ISIN, stock symbol, currency, and country.
1920            Overrides index_by_symbol.
1921
1922        Returns
1923        -------
1924        pandas.DataFrame
1925            Pandas DataFrame containing your data indexed by ISINs/Symbol/all metadata
1926            and feature names.
1927        ___
1928        """
1929        url = (
1930            self.base_uri + f"/api/explain-trades/explain/{model_id}/{self.__iso_format(date)}/json"
1931        )
1932        headers = {"Authorization": "ApiKey " + self.api_key, "Content-Type": "application/json"}
1933        explain_res = requests.get(url, headers=headers, **self._request_params)
1934        if explain_res.ok:
1935            ranking_dict = explain_res.json()
1936            rows = ranking_dict["rows"]
1937            stock_summary_url = f"/api/stock-summaries/{model_id}"
1938            stock_summary_body = {"gbiIds": ranking_dict["rows"]}
1939            summary_res = requests.post(
1940                self.base_uri + stock_summary_url,
1941                data=json.dumps(stock_summary_body),
1942                headers=headers,
1943                **self._request_params,
1944            )
1945            if summary_res.ok:
1946                stock_summary = summary_res.json()
1947                if index_by_symbol:
1948                    rows = [stock_summary[row]["symbol"] for row in ranking_dict["rows"]]
1949                elif index_by_all_metadata:
1950                    rows = [
1951                        [
1952                            stock_summary[row]["isin"],
1953                            stock_summary[row]["symbol"],
1954                            stock_summary[row]["currency"],
1955                            stock_summary[row]["country"],
1956                        ]
1957                        for row in ranking_dict["rows"]
1958                    ]
1959                else:
1960                    rows = [stock_summary[row]["isin"] for row in ranking_dict["rows"]]
1961            else:
1962                error_msg = self._try_extract_error_code(summary_res)
1963                logger.error(error_msg)
1964                raise BoostedAPIException(
1965                    "Failed to get isin information ranking explain: {0}.".format(error_msg)
1966                )
1967
1968            feature_name_dict = self.__get_rankings_ref_translation(model_id)
1969            columns = [feature_name_dict[col] for col in ranking_dict["columns"]]
1970
1971            id_col_name = "Symbols" if index_by_symbol else "ISINs"
1972
1973            if index_by_all_metadata:
1974                pc_list = []
1975                pf = ranking_dict["data"]
1976                for row_idx, row in enumerate(rows):
1977                    for col_idx, col in enumerate(columns):
1978                        pc_list.append([row, col] + pf[row_idx]["columns"][col_idx]["fields"])
1979                df = pd.DataFrame(pc_list)
1980                df = df.set_axis(
1981                    ["Metadata", "Feature Names"] + ranking_dict["fields"], axis="columns"
1982                )
1983
1984                metadata_df = df["Metadata"].apply(pd.Series)
1985                metadata_df.columns = pd.Index(["ISINs", "Symbol", "Currency", "Country"])
1986                result_df = pd.concat([metadata_df, df], axis=1).drop("Metadata", axis=1)
1987                result_df.set_index(
1988                    ["ISINs", "Symbol", "Currency", "Country", "Feature Names"], inplace=True
1989                )
1990                return result_df
1991
1992            else:
1993                df = protoCubeJsonDataToDataFrame(
1994                    ranking_dict["data"],
1995                    id_col_name,
1996                    rows,
1997                    "Feature Names",
1998                    columns,
1999                    ranking_dict["fields"],
2000                )
2001
2002                return df
2003        else:
2004            error_msg = self._try_extract_error_code(explain_res)
2005            logger.error(error_msg)
2006            raise BoostedAPIException("Failed to get ranking explain: {0}.".format(error_msg))
2007
2008    def getDenseSignalsForDate(self, portfolio_id, date, rollback_to_last_available_date):
2009        date = self.__iso_format(date)
2010        url = self.base_uri + f"/api/portfolios/{portfolio_id}/denseSignalsByDate"
2011        headers = {"Authorization": "ApiKey " + self.api_key}
2012        params = {
2013            "startDate": date,
2014            "endDate": date,
2015            "rollbackToMostRecentDate": rollback_to_last_available_date,
2016        }
2017        logger.info("Retrieving dense signals information for date {0}.".format(date))
2018        res = requests.get(url, params=params, headers=headers, **self._request_params)
2019        if res.ok:
2020            logger.info("Signals retrieval successful.")
2021            d = res.json()
2022            # reshape date to output format
2023            date = list(d["signals"].keys())[0]
2024            model_id = d["model_id"]
2025            signals_list = list(d["signals"].values())[0]
2026            return {"date": date, "signals": [{"model_id": model_id, "signals_info": signals_list}]}
2027        else:
2028            error_msg = self._try_extract_error_code(res)
2029            raise BoostedAPIException("Failed to retrieve dense signals: {0}.".format(error_msg))
2030
2031    def getDenseSignals(self, model_id, portfolio_id, file_name=None, location="./"):
2032        url = self.base_uri + f"/api/models/{model_id}/{portfolio_id}/dense-signals"
2033        headers = {"Authorization": "ApiKey " + self.api_key}
2034        res = requests.get(url, headers=headers, **self._request_params)
2035        if file_name is None:
2036            file_name = f"{model_id}-{portfolio_id}_dense_signals.csv"
2037        download_location = os.path.join(location, file_name)
2038        if res.ok:
2039            with open(download_location, "wb") as file:
2040                file.write(res.content)
2041            print("Download Complete")
2042        elif res.status_code == 404:
2043            raise BoostedAPIException(
2044                f"""Dense Singals file does not exist for model:
2045                 {model_id} - portfolio: {portfolio_id}"""
2046            )
2047        else:
2048            error_msg = self._try_extract_error_code(res)
2049            logger.error(error_msg)
2050            raise BoostedAPIException(
2051                f"""Failed to download dense singals file for model:
2052                 {model_id} - portfolio: {portfolio_id}"""
2053            )
2054
2055    def _getIsPortfolioReadyForProcessing(self, model_id, portfolio_id, formatted_date):
2056        headers = {"Authorization": "ApiKey " + self.api_key}
2057        url = (
2058            self.base_uri
2059            + f"/api/explain-trades/{model_id}/{portfolio_id}"
2060            + f"/is-ready-for-processing/{formatted_date}"
2061        )
2062        res = requests.get(url, headers=headers, **self._request_params)
2063
2064        try:
2065            if res.ok:
2066                body = res.json()
2067                if "ready" in body:
2068                    if body["ready"]:
2069                        return True, ""
2070                    else:
2071                        reason_from_api = (
2072                            body["notReadyReason"] if "notReadyReason" in body else "Unavailable"
2073                        )
2074
2075                        returned_reason = reason_from_api
2076
2077                        if returned_reason == "SKIP":
2078                            returned_reason = "holiday- market closed"
2079
2080                        if returned_reason == "WAITING":
2081                            returned_reason = "calculations pending"
2082
2083                        return False, returned_reason
2084                else:
2085                    return False, "Unavailable"
2086            else:
2087                error_msg = self._try_extract_error_code(res)
2088                logger.error(error_msg)
2089                raise BoostedAPIException(
2090                    f"""Failed to generate file for model:
2091                    {model_id} - portfolio: {portfolio_id} on date: {formatted_date}"""
2092                )
2093        except Exception as e:
2094            raise BoostedAPIException(
2095                f"""Failed to generate file for model:
2096                {model_id} - portfolio: {portfolio_id} on date: {formatted_date} {e}"""
2097            )
2098
2099    def getRanking2DateAnalysisFile(
2100        self, model_id, portfolio_id, date, file_name=None, location="./"
2101    ):
2102        formatted_date = self.__iso_format(date)
2103        s3_file_name = f"{formatted_date}_analysis.xlsx"
2104        download_url = (
2105            self.base_uri + f"/api/models/{model_id}/{portfolio_id}/ranking-file/{s3_file_name}"
2106        )
2107        headers = {"Authorization": "ApiKey " + self.api_key}
2108        if file_name is None:
2109            file_name = f"{model_id}-{portfolio_id}_statistical_analysis_{formatted_date}.xlsx"
2110        download_location = os.path.join(location, file_name)
2111
2112        res = requests.get(download_url, headers=headers, **self._request_params)
2113        if res.ok:
2114            with open(download_location, "wb") as file:
2115                file.write(res.content)
2116            print("Download Complete")
2117        elif res.status_code == 404:
2118            (
2119                is_portfolio_ready_for_processing,
2120                portfolio_ready_status,
2121            ) = self._getIsPortfolioReadyForProcessing(model_id, portfolio_id, formatted_date)
2122
2123            if not is_portfolio_ready_for_processing:
2124                logger.info(
2125                    f"""\nPortfolio {portfolio_id} for model {model_id}
2126                    on date {date} unavailable for Ranking2Date Analysis file.
2127                    Status: {portfolio_ready_status}\n"""
2128                )
2129                return
2130
2131            generate_url = (
2132                self.base_uri
2133                + f"/api/explain-trades/{model_id}/{portfolio_id}"
2134                + f"/generate/date-data/{formatted_date}"
2135            )
2136
2137            generate_res = requests.get(generate_url, headers=headers, **self._request_params)
2138            if generate_res.ok:
2139                download_res = requests.get(download_url, headers=headers, **self._request_params)
2140                while download_res.status_code == 404 or (
2141                    download_res.ok and len(download_res.content) == 0
2142                ):
2143                    print("waiting for file to be generated")
2144                    time.sleep(5)
2145                    download_res = requests.get(
2146                        download_url, headers=headers, **self._request_params
2147                    )
2148                if download_res.ok:
2149                    with open(download_location, "wb") as file:
2150                        file.write(download_res.content)
2151                    print("Download Complete")
2152            else:
2153                error_msg = self._try_extract_error_code(res)
2154                logger.error(error_msg)
2155                raise BoostedAPIException(
2156                    f"""Failed to generate ranking analysis file for model:
2157                    {model_id} - portfolio: {portfolio_id} on date: {formatted_date}"""
2158                )
2159        else:
2160            error_msg = self._try_extract_error_code(res)
2161            logger.error(error_msg)
2162            raise BoostedAPIException(
2163                f"""Failed to download ranking analysis file for model:
2164                 {model_id} - portfolio: {portfolio_id} on date: {formatted_date}"""
2165            )
2166
2167    def getRanking2DateExplainFile(
2168        self,
2169        model_id,
2170        portfolio_id,
2171        date,
2172        file_name=None,
2173        location="./",
2174        overwrite: bool = False,
2175        index_by_all_metadata: bool = False,
2176    ):
2177        """
2178        Downloads the ranking explain file for the provided portfolio and model.
2179        If no file exists then it will send a request to generate the file and continuously
2180        poll the server every 5 seconds to try and download the file until the file is downloaded.
2181
2182        Parameters
2183        ----------
2184        model_id: str
2185            Model ID.  Model IDs can be retrieved by clicking on the copy to clipboard
2186            button next to your model's name in the Model Summary Page in Boosted
2187            Insights.
2188        portfolio_id: str
2189            Portfolio ID.  Portfolio IDs can be retrieved from portfolio's configuration page.
2190        date: datetime.date or YYYY-MM-DD string
2191            Date of the data to retrieve.
2192        file_name: str
2193            File name of the dense signals file to save as.
2194            If no file name is given the file name will be
2195            "<model_id>-<portfolio_id>_explain_data_<date>.xlsx"
2196        location: str
2197            The location to save the file to.
2198            If no location is given then it will be saved to the current directory.
2199        overwrite: bool
2200            Defaults to False, set to True to regenerate the file.
2201        index_by_all_metadata: bool
2202            If true, index by all metadata: ISIN, stock symbol, currency, and country.
2203
2204
2205        Returns
2206        -------
2207        None
2208        ___
2209        """
2210        formatted_date = self.__iso_format(date)
2211        if index_by_all_metadata:
2212            s3_file_name = f"{formatted_date}_explaindata_withmetadata.xlsx"
2213        else:
2214            s3_file_name = f"{formatted_date}_explaindata.xlsx"
2215        download_url = (
2216            self.base_uri + f"/api/models/{model_id}/{portfolio_id}/ranking-file/{s3_file_name}"
2217        )
2218        headers = {"Authorization": "ApiKey " + self.api_key}
2219        if file_name is None:
2220            file_name = f"{model_id}-{portfolio_id}_explain_data_{formatted_date}.xlsx"
2221        download_location = os.path.join(location, file_name)
2222
2223        if not overwrite:
2224            res = requests.get(download_url, headers=headers, **self._request_params)
2225        if not overwrite and res.ok:
2226            with open(download_location, "wb") as file:
2227                file.write(res.content)
2228            print("Download Complete")
2229        elif overwrite or res.status_code == 404:
2230            (
2231                is_portfolio_ready_for_processing,
2232                portfolio_ready_status,
2233            ) = self._getIsPortfolioReadyForProcessing(model_id, portfolio_id, formatted_date)
2234
2235            if not is_portfolio_ready_for_processing:
2236                logger.info(
2237                    f"""\nPortfolio {portfolio_id} for model {model_id}
2238                    on date {date} unavailable for Ranking2Date Explain file.
2239                    Status: {portfolio_ready_status}\n"""
2240                )
2241                return
2242
2243            generate_url = (
2244                self.base_uri
2245                + f"/api/explain-trades/{model_id}/{portfolio_id}"
2246                + f"/generate/date-data/{formatted_date}"
2247                + f"/{'true' if index_by_all_metadata else 'false'}"
2248            )
2249
2250            generate_res = requests.get(generate_url, headers=headers, **self._request_params)
2251            if generate_res.ok:
2252                download_res = requests.get(download_url, headers=headers, **self._request_params)
2253                while download_res.status_code == 404 or (
2254                    download_res.ok and len(download_res.content) == 0
2255                ):
2256                    print("waiting for file to be generated")
2257                    time.sleep(5)
2258                    download_res = requests.get(
2259                        download_url, headers=headers, **self._request_params
2260                    )
2261                if download_res.ok:
2262                    with open(download_location, "wb") as file:
2263                        file.write(download_res.content)
2264                    print("Download Complete")
2265            else:
2266                error_msg = self._try_extract_error_code(res)
2267                logger.error(error_msg)
2268                raise BoostedAPIException(
2269                    f"""Failed to generate ranking explain file for model:
2270                    {model_id} - portfolio: {portfolio_id} on date: {formatted_date}"""
2271                )
2272        else:
2273            error_msg = self._try_extract_error_code(res)
2274            logger.error(error_msg)
2275            raise BoostedAPIException(
2276                f"""Failed to download ranking explain file for model:
2277                 {model_id} - portfolio: {portfolio_id} on date: {formatted_date}"""
2278            )
2279
2280    def getRanking2DateExplain(
2281        self,
2282        model_id: str,
2283        portfolio_id: str,
2284        date: Optional[datetime.date],
2285        overwrite: bool = False,
2286    ) -> Dict[str, pd.DataFrame]:
2287        """
2288        Wrapper around getRanking2DateExplainFile, but returns a pandas
2289        dataframe instead of downloading to a path. Dataframe is indexed by
2290        symbol and should always have 'rating' and 'rating_delta' columns. Other
2291        columns will be determined by model's features.
2292        """
2293        file_name = "explaindata.xlsx"
2294        with tempfile.TemporaryDirectory() as tmpdirname:
2295            self.getRanking2DateExplainFile(
2296                model_id=model_id,
2297                portfolio_id=portfolio_id,
2298                date=date,
2299                file_name=file_name,
2300                location=tmpdirname,
2301                overwrite=overwrite,
2302            )
2303            full_path = os.path.join(tmpdirname, file_name)
2304            excel_file = pd.ExcelFile(full_path)
2305            df_map = pd.read_excel(excel_file, sheet_name=None)
2306            df_map_final = {str(sheet): df.set_index("Symbol") for (sheet, df) in df_map.items()}
2307
2308        return df_map_final
2309
2310    def getTearSheet(self, model_id, portfolio_id, start_date=None, end_date=None, block=False):
2311        if start_date is None or end_date is None:
2312            if start_date is not None or end_date is not None:
2313                raise ValueError("start_date and end_date must both be None or both be defined")
2314            return self._getCurrentTearSheet(model_id, portfolio_id)
2315
2316        start_date_obj = self.__to_date_obj(start_date)
2317        end_date_obj = self.__to_date_obj(end_date)
2318        if start_date_obj >= end_date_obj:
2319            raise ValueError("end_date must be later than the start_date")
2320
2321        # get for the given date
2322        url = self.base_uri + f"/api/analysis/keyfacts/{model_id}/{portfolio_id}"
2323        data = {
2324            "startDate": self.__iso_format(start_date),
2325            "endDate": self.__iso_format(end_date),
2326            "shouldRecalc": True,
2327        }
2328        headers = {"Authorization": "ApiKey " + self.api_key, "Content-Type": "application/json"}
2329        res = requests.post(url, data=json.dumps(data), headers=headers, **self._request_params)
2330        if res.status_code == 404 and block:
2331            retries = 0
2332            data["shouldRecalc"] = False
2333            while retries < 10:
2334                time.sleep(10)
2335                retries += 1
2336                res = requests.post(
2337                    url, data=json.dumps(data), headers=headers, **self._request_params
2338                )
2339                if res.status_code != 404:
2340                    break
2341        if res.ok:
2342            return res.json()
2343        else:
2344            error_msg = self._try_extract_error_code(res)
2345            logger.error(error_msg)
2346            raise BoostedAPIException(
2347                "Failed to get tear sheet data: {0} {1}.".format(error_msg, str(res.status_code))
2348            )
2349
2350    def _getCurrentTearSheet(self, model_id, portfolio_id):
2351        url = self.base_uri + f"/api/model-summaries/{model_id}/{portfolio_id}"
2352        headers = {"Authorization": "ApiKey " + self.api_key, "Content-Type": "application/json"}
2353        res = requests.get(url, headers=headers, **self._request_params)
2354        if res.ok:
2355            json = res.json()
2356            return json.get("tearSheet", {})
2357        else:
2358            error_msg = self._try_extract_error_code(res)
2359            logger.error(error_msg)
2360            raise BoostedAPIException("Failed to get tear sheet data: {0}.".format(error_msg))
2361
2362    def getPortfolioStatus(self, model_id, portfolio_id, job_date):
2363        url = (
2364            self.base_uri
2365            + f"/api/analysis/portfolioStatus/{model_id}/{portfolio_id}?jobDate={job_date}"
2366        )
2367        headers = {"Authorization": "ApiKey " + self.api_key, "Content-Type": "application/json"}
2368        res = requests.get(url, headers=headers, **self._request_params)
2369        if res.ok:
2370            result = res.json()
2371            return {
2372                "is_complete": result["status"],
2373                "last_update": None if result["lastUpdate"] is None else result["lastUpdate"][:10],
2374                "next_update": None if result["nextUpdate"] is None else result["nextUpdate"][:10],
2375            }
2376        else:
2377            error_msg = self._try_extract_error_code(res)
2378            logger.error(error_msg)
2379            raise BoostedAPIException("Failed to get portfolio status: {0}".format(error_msg))
2380
2381    def _query_portfolio_factor_attribution(
2382        self,
2383        portfolio_id: str,
2384        start_date: Optional[BoostedDate] = None,
2385        end_date: Optional[BoostedDate] = None,
2386    ):
2387        response = self._get_graphql(
2388            query=graphql_queries.GET_PORTFOLIO_FACTOR_ATTRIBUTION_QUERY,
2389            variables={
2390                "portfolioId": portfolio_id,
2391                "startDate": str(start_date) if start_date else None,
2392                "endDate": str(end_date) if start_date else None,
2393            },
2394            error_msg_prefix="Failed to get factor attribution: ",
2395        )
2396        return response
2397
2398    def get_portfolio_factor_attribution(
2399        self,
2400        portfolio_id: str,
2401        start_date: Optional[BoostedDate] = None,
2402        end_date: Optional[BoostedDate] = None,
2403    ):
2404        """Get portfolio factor attribution for a portfolio
2405
2406        Args:
2407            portfolio_id (str): a valid UUID string
2408            start_date (BoostedDate, optional): The start date. Defaults to None.
2409            end_date (BoostedDate, optional): The end date. Defaults to None.
2410        """
2411        response = self._query_portfolio_factor_attribution(portfolio_id, start_date, end_date)
2412        factor_attribution = response["data"]["portfolio"]["factorAttribution"]
2413        dates = pd.DatetimeIndex(data=factor_attribution["dates"])
2414        beta = factor_attribution["factorBetas"]
2415        beta_df = pd.DataFrame(index=dates, data={x["name"]: x["data"] for x in beta})
2416        beta_df = beta_df.add_suffix("_beta")
2417        returns = factor_attribution["portfolioFactorPerformance"]
2418        returns_df = pd.DataFrame(index=dates, data={x["name"]: x["data"] for x in returns})
2419        returns_df = returns_df.add_suffix("_return")
2420        returns_df = (returns_df - 1) * 100
2421
2422        final_df = pd.concat([returns_df, beta_df], axis=1)
2423        ordered_columns = list(itertools.chain(*zip(returns_df.columns, beta_df.columns)))
2424        ordered_final_df = final_df.reindex(columns=ordered_columns)
2425
2426        # Add the column `total_return` which is the sum of returns_data
2427        ordered_final_df["total_return"] = returns_df.sum(axis=1)
2428        return ordered_final_df
2429
2430    def getBlacklist(self, blacklist_id):
2431        url = self.base_uri + f"/api/blacklist/{blacklist_id}"
2432        headers = {"Authorization": "ApiKey " + self.api_key}
2433        res = requests.get(url, headers=headers, **self._request_params)
2434        if res.ok:
2435            result = res.json()
2436            return result
2437        error_msg = self._try_extract_error_code(res)
2438        logger.error(error_msg)
2439        raise BoostedAPIException(f"Failed to get blacklist with id {blacklist_id}: {error_msg}")
2440
2441    def getBlacklists(self, model_id=None, company_id=None, last_N=None):
2442        params = {}
2443        if last_N:
2444            params["lastN"] = last_N
2445        if model_id:
2446            params["modelId"] = model_id
2447        if company_id:
2448            params["companyId"] = company_id
2449        url = self.base_uri + f"/api/blacklist"
2450        headers = {"Authorization": "ApiKey " + self.api_key}
2451        res = requests.get(url, headers=headers, params=params, **self._request_params)
2452        if res.ok:
2453            result = res.json()
2454            return result
2455        error_msg = self._try_extract_error_code(res)
2456        logger.error(error_msg)
2457        raise BoostedAPIException(
2458            f"""Failed to get blacklists with \
2459            model_id {model_id} company_id {company_id} last_N {last_N}: {error_msg}"""
2460        )
2461
2462    def createBlacklist(
2463        self,
2464        isin,
2465        long_short=2,
2466        start_date=datetime.date.today(),
2467        end_date="4000-01-01",
2468        model_id=None,
2469    ):
2470        url = self.base_uri + f"/api/blacklist"
2471        data = {
2472            "modelId": model_id,
2473            "isin": isin,
2474            "longShort": long_short,
2475            "startDate": self.__iso_format(start_date),
2476            "endDate": self.__iso_format(end_date),
2477        }
2478        headers = {"Authorization": "ApiKey " + self.api_key, "Content-Type": "application/json"}
2479        res = requests.post(url, data=json.dumps(data), headers=headers, **self._request_params)
2480        if res.ok:
2481            return res.json()
2482        else:
2483            error_msg = self._try_extract_error_code(res)
2484            logger.error(error_msg)
2485            raise BoostedAPIException(
2486                f"""Failed to create the blacklist with \
2487                  isin {isin} long_short {long_short} start_date {start_date} end_date {end_date} \
2488                  model_id {model_id}: {error_msg}."""
2489            )
2490
2491    def createBlacklistsFromCSV(self, csv_name):
2492        url = self.base_uri + f"/api/blacklists"
2493        data = []
2494        with open(csv_name, mode="r") as f:
2495            csv_reader = csv.DictReader(f)
2496            for row in csv_reader:
2497                blacklist = {"modelId": row["ModelID"], "isin": row["ISIN"]}
2498                if not row.get("LongShort"):
2499                    blacklist["longShort"] = 2
2500                else:
2501                    blacklist["longShort"] = row["LongShort"]
2502
2503                if not row.get("StartDate"):
2504                    blacklist["startDate"] = self.__iso_format(datetime.date.today())
2505                else:
2506                    blacklist["startDate"] = self.__iso_format(row["StartDate"])
2507
2508                if not row.get("EndDate"):
2509                    blacklist["endDate"] = self.__iso_format("4000-01-01")
2510                else:
2511                    blacklist["endDate"] = self.__iso_format(row["EndDate"])
2512                data.append(blacklist)
2513        print(f"Processed {len(data)} blacklists.")
2514        headers = {"Authorization": "ApiKey " + self.api_key, "Content-Type": "application/json"}
2515        res = requests.post(url, data=json.dumps(data), headers=headers, **self._request_params)
2516        if res.ok:
2517            return res.json()
2518        else:
2519            error_msg = self._try_extract_error_code(res)
2520            logger.error(error_msg)
2521            raise BoostedAPIException("failed to create blacklists")
2522
2523    def updateBlacklist(self, blacklist_id, long_short=None, start_date=None, end_date=None):
2524        params = {}
2525        if long_short:
2526            params["longShort"] = long_short
2527        if start_date:
2528            params["startDate"] = start_date
2529        if end_date:
2530            params["endDate"] = end_date
2531        url = self.base_uri + f"/api/blacklist/{blacklist_id}"
2532        headers = {"Authorization": "ApiKey " + self.api_key, "Content-Type": "application/json"}
2533        res = requests.patch(url, json=params, headers=headers, **self._request_params)
2534        if res.ok:
2535            return res.json()
2536        else:
2537            error_msg = self._try_extract_error_code(res)
2538            logger.error(error_msg)
2539            raise BoostedAPIException(
2540                f"Failed to update blacklist with id {blacklist_id}: {error_msg}"
2541            )
2542
2543    def deleteBlacklist(self, blacklist_id):
2544        url = self.base_uri + f"/api/blacklist/{blacklist_id}"
2545        headers = {"Authorization": "ApiKey " + self.api_key}
2546        res = requests.delete(url, headers=headers, **self._request_params)
2547        if res.ok:
2548            result = res.json()
2549            return result
2550        else:
2551            error_msg = self._try_extract_error_code(res)
2552            logger.error(error_msg)
2553            raise BoostedAPIException(
2554                f"Failed to delete blacklist with id {blacklist_id}: {error_msg}"
2555            )
2556
2557    def getFeatureImportance(self, model_id, date, N=None):
2558        url = self.base_uri + f"/api/analysis/explainability/{model_id}"
2559        headers = {"Authorization": "ApiKey " + self.api_key}
2560        logger.info("Retrieving rankings information for date {0}.".format(date))
2561        res = requests.get(url, headers=headers, **self._request_params)
2562        if not res.ok:
2563            error_msg = self._try_extract_error_code(res)
2564            logger.error(error_msg)
2565            raise BoostedAPIException(
2566                f"Failed to fetch feature importance for model/portfolio {model_id}: {error_msg}"
2567            )
2568
2569        json_data = res.json()
2570        if "all" not in json_data.keys() or not json_data["all"]:
2571            raise BoostedAPIException(f"Unexpected formatting of feature importance response")
2572
2573        feature_data = json_data["all"]
2574        # find the right period (assuming returned json has dates in descending order)
2575        date_obj = self.__to_date_obj(date)
2576        start_date_for_return_data = self.__to_date_obj(feature_data[0]["date"])
2577        features_for_requested_period = None
2578
2579        if date_obj > start_date_for_return_data:
2580            features_for_requested_period = feature_data[0]["variable"]
2581        else:
2582            i = 0
2583            while i < len(feature_data) - 1:
2584                current_date = self.__to_date_obj(feature_data[i]["date"])
2585                next_date = self.__to_date_obj(feature_data[i + 1]["date"])
2586                if next_date <= date_obj <= current_date:
2587                    features_for_requested_period = feature_data[i + 1]["variable"]
2588                    start_date_for_return_data = next_date
2589                    break
2590                i += 1
2591
2592        if features_for_requested_period is None:
2593            raise BoostedAPIException(f"No feature data was found for requested date: {date_obj}")
2594
2595        features_for_requested_period.sort(key=lambda x: x["value"], reverse=True)
2596
2597        if type(N) is int and N > 0:
2598            df = pd.DataFrame.from_dict(features_for_requested_period[0:N])
2599        else:
2600            df = pd.DataFrame.from_dict(features_for_requested_period)
2601        result = df[["feature", "value"]]
2602
2603        return result.rename(columns={"feature": f"feature ({start_date_for_return_data})"})
2604
2605    def getAllModelNames(self) -> Dict[str, str]:
2606        url = f"{self.base_uri}/api/graphql"
2607        headers = {"Authorization": "ApiKey " + self.api_key, "Content-Type": "application/json"}
2608        req_json = {"query": "query listOfModels {\n models { id name }}", "variables": {}}
2609        res = requests.post(url, json=req_json, headers=headers, **self._request_params)
2610        if not res.ok:
2611            error_msg = self._try_extract_error_code(res)
2612            logger.error(error_msg)
2613            raise BoostedAPIException(f"Failed to get user models: {error_msg}")
2614        data = res.json()
2615        if data["data"]["models"] is None:
2616            return {}
2617        return {rec["id"]: rec["name"] for rec in data["data"]["models"]}
2618
2619    def getAllModelDetails(self) -> Dict[str, Dict[str, Any]]:
2620        url = f"{self.base_uri}/api/graphql"
2621        headers = {"Authorization": "ApiKey " + self.api_key, "Content-Type": "application/json"}
2622        req_json = {
2623            "query": "query listOfModels {\n models { id name lastUpdated portfolios { id name }}}",
2624            "variables": {},
2625        }
2626        res = requests.post(url, json=req_json, headers=headers, **self._request_params)
2627        if not res.ok:
2628            error_msg = self._try_extract_error_code(res)
2629            logger.error(error_msg)
2630            raise BoostedAPIException(f"Failed to get user models: {error_msg}")
2631        data = res.json()
2632        if data["data"]["models"] is None:
2633            return {}
2634
2635        output_data = {}
2636        for rec in data["data"]["models"]:
2637            model_id = rec["id"]
2638            output_data[model_id] = {
2639                "name": rec["name"],
2640                "last_updated": parser.parse(rec["lastUpdated"]),
2641                "portfolios": rec["portfolios"],
2642            }
2643
2644        return output_data
2645
2646    def get_hedge_experiments(self):
2647        url = self.base_uri + "/api/graphql"
2648        qry = """
2649            query getHedgeExperiments {
2650                hedgeExperiments {
2651                    hedgeExperimentId
2652                    experimentName
2653                    userId
2654                    config
2655                    description
2656                    experimentType
2657                    lastCalculated
2658                    lastModified
2659                    status
2660                    portfolioCalcStatus
2661                    targetSecurities {
2662                        gbiId
2663                        security {
2664                            gbiId
2665                            symbol
2666                            name
2667                        }
2668                        weight
2669                    }
2670                    targetPortfolios {
2671                        portfolioId
2672                    }
2673                    baselineModel {
2674                        id
2675                        name
2676
2677                    }
2678                    baselineScenario {
2679                        hedgeExperimentScenarioId
2680                        scenarioName
2681                        description
2682                        portfolioSettingsJson
2683                        hedgeExperimentPortfolios {
2684                            portfolio {
2685                                id
2686                                name
2687                                modelId
2688                                performanceGridHeader
2689                                performanceGrid
2690                                status
2691                                tearSheet {
2692                                    groupName
2693                                    members {
2694                                        name
2695                                        value
2696                                    }
2697                                }
2698                            }
2699                        }
2700                        status
2701                    }
2702                    baselineStockUniverseId
2703                }
2704            }
2705        """
2706
2707        headers = {"Authorization": "ApiKey " + self.api_key}
2708        resp = requests.post(url, json={"query": qry}, headers=headers, params=self._request_params)
2709
2710        json_resp = resp.json()
2711        # graphql endpoints typically return 200 or 400 status codes, so we must
2712        # check if we have any errors, even with a 200
2713        if (resp.ok and "errors" in json_resp) or not resp.ok:
2714            error_msg = self._try_extract_error_code(resp)
2715            logger.error(error_msg)
2716            raise BoostedAPIException(
2717                (f"Failed to get hedge experiments: {resp.status_code=}; {error_msg=}")
2718            )
2719
2720        json_experiments = resp.json()["data"]["hedgeExperiments"]
2721        experiments = [HedgeExperiment.from_json_dict(exp_json) for exp_json in json_experiments]
2722        return experiments
2723
2724    def get_hedge_experiment_details(self, experiment_id: str):
2725        url = self.base_uri + "/api/graphql"
2726        qry = """
2727            query getHedgeExperimentDetails($hedgeExperimentId: ID!) {
2728                hedgeExperiment(hedgeExperimentId: $hedgeExperimentId) {
2729                ...HedgeExperimentDetailsSummaryListFragment
2730                }
2731            }
2732
2733            fragment HedgeExperimentDetailsSummaryListFragment on HedgeExperiment {
2734                hedgeExperimentId
2735                experimentName
2736                userId
2737                config
2738                description
2739                experimentType
2740                lastCalculated
2741                lastModified
2742                status
2743                portfolioCalcStatus
2744                targetSecurities {
2745                    gbiId
2746                    security {
2747                        gbiId
2748                        symbol
2749                        name
2750                    }
2751                    weight
2752                }
2753                selectedModels {
2754                    id
2755                    name
2756                    stockUniverse {
2757                        name
2758                    }
2759                }
2760                hedgeExperimentScenarios {
2761                    ...experimentScenarioFragment
2762                }
2763                selectedDummyHedgeExperimentModels {
2764                    id
2765                    name
2766                    stockUniverse {
2767                        name
2768                    }
2769                }
2770                targetPortfolios {
2771                    portfolioId
2772                }
2773                baselineModel {
2774                    id
2775                    name
2776
2777                }
2778                baselineScenario {
2779                    hedgeExperimentScenarioId
2780                    scenarioName
2781                    description
2782                    portfolioSettingsJson
2783                    hedgeExperimentPortfolios {
2784                        portfolio {
2785                            id
2786                            name
2787                            modelId
2788                            performanceGridHeader
2789                            performanceGrid
2790                            status
2791                            tearSheet {
2792                                groupName
2793                                members {
2794                                    name
2795                                    value
2796                                }
2797                            }
2798                        }
2799                    }
2800                    status
2801                }
2802                baselineStockUniverseId
2803            }
2804
2805            fragment experimentScenarioFragment on HedgeExperimentScenario {
2806                hedgeExperimentScenarioId
2807                scenarioName
2808                status
2809                description
2810                portfolioSettingsJson
2811                hedgeExperimentPortfolios {
2812                    portfolio {
2813                        id
2814                        name
2815                        modelId
2816                        performanceGridHeader
2817                        performanceGrid
2818                        status
2819                        tearSheet {
2820                            groupName
2821                            members {
2822                                name
2823                                value
2824                            }
2825                        }
2826                    }
2827                }
2828            }
2829        """
2830        headers = {"Authorization": "ApiKey " + self.api_key}
2831        resp = requests.post(
2832            url,
2833            json={"query": qry, "variables": {"hedgeExperimentId": experiment_id}},
2834            headers=headers,
2835            params=self._request_params,
2836        )
2837
2838        json_resp = resp.json()
2839        # graphql endpoints typically return 200 or 400 status codes, so we must
2840        # check if we have any errors, even with a 200
2841        if (resp.ok and "errors" in json_resp) or not resp.ok:
2842            error_msg = self._try_extract_error_code(resp)
2843            logger.error(error_msg)
2844            raise BoostedAPIException(
2845                (
2846                    f"Failed to get hedge experiment results for {experiment_id=}: "
2847                    f"{resp.status_code=}; {error_msg=}"
2848                )
2849            )
2850
2851        json_exp_results = json_resp["data"]["hedgeExperiment"]
2852        if json_exp_results is None:
2853            return None  # issued a request with a non-existent experiment_id
2854        exp_results = HedgeExperimentDetails.from_json_dict(json_exp_results)
2855        return exp_results
2856
2857    def get_portfolio_performance(
2858        self,
2859        portfolio_id: str,
2860        start_date: Optional[datetime.date],
2861        end_date: Optional[datetime.date],
2862        daily_returns: bool,
2863    ) -> pd.DataFrame:
2864        """
2865        Get performance data for a portfolio.
2866
2867        Parameters
2868        ----------
2869        portfolio_id: str
2870            UUID corresponding to the portfolio in question.
2871        start_date: datetime.date
2872            Starting cutoff date to filter performance data
2873        end_date: datetime.date
2874            Ending cutoff date to filter performance data
2875        daily_returns: bool
2876            Flag indicating whether to add a new column with the daily return pct calculated
2877
2878        Returns
2879        -------
2880        pd.DataFrame object
2881            Portfolio and benchmark performance.
2882            -index:
2883                "date": pd.DatetimeIndex
2884            -columns:
2885                "benchmark": benchmark performance, % return
2886                "turnover": portfolio turnover, % of equity
2887                "portfolio": return since beginning of portfolio, % return
2888                "daily_returns": daily percent change in value of the portfolio, % return
2889                                (this column is optional and depends on the daily_returns flag)
2890        """
2891        url = f"{self.base_uri}/api/graphql"
2892        qry = """
2893            query getPortfolioPerformance($portfolioId: ID!) {
2894                portfolio(id: $portfolioId) {
2895                    id
2896                    modelId
2897                    name
2898                    status
2899                    performance {
2900                        benchmark
2901                        date
2902                        turnover
2903                        value
2904                    }
2905                }
2906            }
2907        """
2908
2909        headers = {"Authorization": "ApiKey " + self.api_key}
2910        resp = requests.post(
2911            url,
2912            json={"query": qry, "variables": {"portfolioId": portfolio_id}},
2913            headers=headers,
2914            params=self._request_params,
2915        )
2916
2917        json_resp = resp.json()
2918        # the webserver returns an error for non-ready portfolios, so we have to check
2919        # for this prior to the error check below
2920        pf = json_resp["data"].get("portfolio")
2921        if pf is not None and pf["status"] != "READY":
2922            return pd.DataFrame()
2923
2924        # graphql endpoints typically return 200 or 400 status codes, so we must
2925        # check if we have any errors, even with a 200
2926        if (resp.ok and "errors" in json_resp) or not resp.ok:
2927            error_msg = self._try_extract_error_code(resp)
2928            logger.error(error_msg)
2929            raise BoostedAPIException(
2930                (
2931                    f"Failed to get portfolio performance for {portfolio_id=}: "
2932                    f"{resp.status_code=}; {error_msg=}"
2933                )
2934            )
2935
2936        perf = json_resp["data"]["portfolio"]["performance"]
2937        df = pd.DataFrame(perf).set_index("date").rename(columns={"value": "portfolio"})
2938        df.index = pd.to_datetime(df.index)
2939        if daily_returns:
2940            df["daily_returns"] = pd.to_numeric(df["portfolio"]).pct_change()
2941            df = df.dropna(subset=["daily_returns"])
2942        if start_date:
2943            df = df[df.index >= pd.to_datetime(start_date)]
2944        if end_date:
2945            df = df[df.index <= pd.to_datetime(end_date)]
2946        return df.astype(float)
2947
2948    def _is_portfolio_still_running(self, error_msg: str) -> bool:
2949        # this is jank af. a proper fix of this is either at the webserver
2950        # returning a better response for a portfolio in draft HT2-226, OR
2951        # a bigger refactor of the API that moves to more OOP, which would allow us
2952        # to have this data all in one place
2953        return "Could not find a model with this ID" in error_msg
2954
2955    def get_portfolio_factors(self, model_id: str, portfolio_id: str) -> pd.DataFrame:
2956        url = f"{self.base_uri}/api/analysis/factors/{model_id}/{portfolio_id}"
2957        headers = {"Authorization": "ApiKey " + self.api_key}
2958        resp = requests.get(url, headers=headers, params=self._request_params)
2959
2960        json_resp = resp.json()
2961        if (resp.ok and "errors" in json_resp) or not resp.ok:
2962            error_msg = json_resp["errors"][0]
2963            if self._is_portfolio_still_running(error_msg):
2964                return pd.DataFrame()
2965            logger.error(error_msg)
2966            raise BoostedAPIException(
2967                (
2968                    f"Failed to get portfolio factors for {portfolio_id=}: "
2969                    f"{resp.status_code=}; {error_msg=}"
2970                )
2971            )
2972
2973        df = pd.DataFrame(json_resp["data"], columns=json_resp["header_row"])
2974
2975        def to_lower_snake_case(s):  # why are we linting lambdas? :(
2976            return "_".join(w.lower() for w in s.split(" "))
2977
2978        df = df.rename(columns={old: to_lower_snake_case(old) for old in df.columns}).set_index(
2979            "date"
2980        )
2981        df.index = pd.to_datetime(df.index)
2982        return df
2983
2984    def get_portfolio_volatility(self, model_id: str, portfolio_id: str) -> pd.DataFrame:
2985        url = f"{self.base_uri}/api/analysis/volatility_rolling/{model_id}/{portfolio_id}"
2986        headers = {"Authorization": "ApiKey " + self.api_key}
2987        resp = requests.get(url, headers=headers, params=self._request_params)
2988
2989        json_resp = resp.json()
2990        if (resp.ok and "errors" in json_resp) or not resp.ok:
2991            error_msg = json_resp["errors"][0]
2992            if self._is_portfolio_still_running(error_msg):
2993                return pd.DataFrame()
2994            logger.error(error_msg)
2995            raise BoostedAPIException(
2996                (
2997                    f"Failed to get portfolio volatility for {portfolio_id=}: "
2998                    f"{resp.status_code=}; {error_msg=}"
2999                )
3000            )
3001
3002        df = pd.DataFrame(json_resp["data"], columns=json_resp["headerRow"])
3003        df = df.rename(
3004            columns={old: old.lower().replace("avg", "avg_") for old in df.columns}  # type: ignore
3005        ).set_index("date")
3006        df.index = pd.to_datetime(df.index)
3007        return df
3008
3009    def get_portfolio_holdings(self, model_id: str, portfolio_id: str) -> pd.DataFrame:
3010        url = f"{self.base_uri}/api/models/{model_id}/{portfolio_id}/basket-data"
3011        headers = {"Authorization": "ApiKey " + self.api_key}
3012        resp = requests.get(url, headers=headers, params=self._request_params)
3013
3014        # this is a classic abuse of try/except as control flow: we try to get json body
3015        # from the response so that we can error-check. if this fails, we assume we have
3016        # a legit text response (corresponding to the csv data we care about)
3017        try:
3018            json_resp = resp.json()
3019        except json.decoder.JSONDecodeError:
3020            df = pd.read_csv(io.StringIO(resp.text), header=[0])
3021        else:
3022            error_msg = json_resp["errors"][0]
3023            if self._is_portfolio_still_running(error_msg):
3024                return pd.DataFrame()
3025            else:
3026                logger.error(error_msg)
3027                raise BoostedAPIException(
3028                    (
3029                        f"Failed to get portfolio holdings for {portfolio_id=}: "
3030                        f"{resp.status_code=}; {error_msg=}"
3031                    )
3032                )
3033
3034        df = df.rename(columns={old: old.lower() for old in df.columns}).set_index("date")
3035        df.index = pd.to_datetime(df.index)
3036        return df
3037
3038    def getStockDataTableForDate(
3039        self, model_id: str, portfolio_id: str, date: datetime.date
3040    ) -> pd.DataFrame:
3041        headers = {"Authorization": "ApiKey " + self.api_key, "Content-Type": "application/json"}
3042
3043        url_base = f"{self.base_uri}/api/analysis"
3044        url_params = f"{model_id}/{portfolio_id}"
3045        formatted_date = date.strftime("%Y-%m-%d")
3046
3047        stock_prices_url = f"{url_base}/stock-prices/{url_params}/{formatted_date}"
3048        stock_factors_url = f"{url_base}/stock-factors/{url_params}/date/{formatted_date}"
3049
3050        prices_params = {"useTicker": "false", "useCurrentSignals": "true"}
3051        factors_param = {"useTicker": "false", "useCurrentSignals": "true"}
3052
3053        prices_resp = requests.get(
3054            stock_prices_url, headers=headers, params=prices_params, **self._request_params
3055        )
3056        factors_resp = requests.get(
3057            stock_factors_url, headers=headers, params=factors_param, **self._request_params
3058        )
3059
3060        frames = []
3061        gbi_ids = set()
3062        for res in (prices_resp, factors_resp):
3063            if not res.ok:
3064                error_msg = self._try_extract_error_code(res)
3065                logger.error(error_msg)
3066                raise BoostedAPIException(
3067                    (
3068                        f"Failed to fetch stock data table for model {model_id}"
3069                        f" (it's possible no data is present for the given date: {date})."
3070                        f" Error message: {error_msg}"
3071                    )
3072                )
3073            result = res.json()
3074            df = pd.DataFrame(result)
3075            gbi_ids.update(df.columns.to_list())
3076            frames.append(pd.DataFrame(result))
3077
3078        all_gbiid_df = pd.concat(frames)
3079
3080        # Get the metadata of all GBI IDs
3081        gbiid_metadata_res = self._get_graphql(
3082            query=graphql_queries.GET_SEC_INFO_QRY, variables={"ids": [int(x) for x in gbi_ids]}
3083        )
3084        # Build a DF of metadata x GBI IDs
3085        gbiid_metadata_df = pd.DataFrame(
3086            {str(x["gbiId"]): x for x in gbiid_metadata_res["data"]["securities"]}
3087        )
3088        # Slice metadata we care. We'll drop "symbol" at the end.
3089        isin_country_currency_df = gbiid_metadata_df.loc[["isin", "country", "currency", "symbol"]]
3090        # Concatenate metadata to the existing stock data DF
3091        all_gbiid_with_metadata_df = pd.concat([all_gbiid_df, isin_country_currency_df])
3092        gbiid_with_symbol_df = all_gbiid_with_metadata_df.loc[
3093            :, all_gbiid_with_metadata_df.loc["symbol"].notna()
3094        ]
3095        renamed_df = gbiid_with_symbol_df.rename(
3096            index={"isin": "ISIN"}, columns=gbiid_with_symbol_df.loc["symbol"].to_dict()
3097        )
3098        output_df = renamed_df.drop(index=["symbol"])
3099        return output_df
3100
3101    def add_hedge_experiment_scenario(
3102        self,
3103        experiment_id: str,
3104        scenario_name: str,
3105        scenario_settings: PortfolioSettings,
3106        run_scenario_immediately: bool,
3107    ) -> HedgeExperimentScenario:
3108        add_scenario_input = {
3109            "hedgeExperimentId": experiment_id,
3110            "scenarioName": scenario_name,
3111            "portfolioSettingsJson": str(scenario_settings),
3112            "runExperimentOnScenario": run_scenario_immediately,
3113            "createDefaultPortfolio": "false",
3114        }
3115        qry = """
3116            mutation addHedgeExperimentScenario(
3117                $input: AddHedgeExperimentScenarioInput!
3118            ) {
3119                addHedgeExperimentScenario(input: $input) {
3120                    hedgeExperimentScenario {
3121                        hedgeExperimentScenarioId
3122                        scenarioName
3123                        description
3124                        portfolioSettingsJson
3125                    }
3126                }
3127            }
3128
3129        """
3130
3131        url = f"{self.base_uri}/api/graphql"
3132
3133        resp = requests.post(
3134            url,
3135            headers={"Authorization": "ApiKey " + self.api_key},
3136            json={"query": qry, "variables": {"input": add_scenario_input}},
3137        )
3138
3139        json_resp = resp.json()
3140        if (resp.ok and "errors" in json_resp) or not resp.ok:
3141            error_msg = self._try_extract_error_code(resp)
3142            logger.error(error_msg)
3143            raise BoostedAPIException(
3144                (f"Failed to add scenario: {resp.status_code=}; {error_msg=}")
3145            )
3146
3147        scenario_dict = json_resp["data"]["addHedgeExperimentScenario"]["hedgeExperimentScenario"]
3148        if scenario_dict is None:
3149            raise BoostedAPIException(
3150                "Failed to add scenario, likely due to bad experiment id or api key"
3151            )
3152        s = HedgeExperimentScenario.from_json_dict(scenario_dict)
3153        return s
3154
3155    # experiment life cycle has 4 steps:
3156    # 1. creation - essentially a very simple registration of a new instance, returning an id
3157    # 2. modify - populate with settings
3158    # 3. start - run the experiment
3159    # 4. delete - drop the experiment
3160    # while i would prefer to just have 2 funcs for (1,2,3) and (4) for a simpler api,
3161    # we need to expose finer-grained control becuase of how scenarios work.
3162    def create_hedge_experiment(
3163        self,
3164        name: str,
3165        description: str,
3166        experiment_type: hedge_experiment_type,
3167        target_securities: Union[Dict[GbiIdSecurity, float], str, None],
3168    ) -> HedgeExperiment:
3169        # we don't pass target_securities here (as much as id like to) because the
3170        # graphql input doesn't support it at this point
3171
3172        # note that this query returns a lot of null fields at this point, but
3173        # they are necessary for building a HE.
3174        create_qry = """
3175            mutation createDraftMutation($input: CreateHedgeExperimentDraftInput!) {
3176                createHedgeExperimentDraft(input: $input) {
3177                    hedgeExperiment {
3178                        hedgeExperimentId
3179                        experimentName
3180                        userId
3181                        config
3182                        description
3183                        experimentType
3184                        lastCalculated
3185                        lastModified
3186                        status
3187                        portfolioCalcStatus
3188                        targetSecurities {
3189                            gbiId
3190                            security {
3191                                gbiId
3192                                name
3193                                symbol
3194                            }
3195                            weight
3196                        }
3197                        baselineModel {
3198                            id
3199                            name
3200                        }
3201                        baselineScenario {
3202                            hedgeExperimentScenarioId
3203                            scenarioName
3204                            description
3205                            portfolioSettingsJson
3206                            hedgeExperimentPortfolios {
3207                                portfolio {
3208                                    id
3209                                    name
3210                                    modelId
3211                                    performanceGridHeader
3212                                    performanceGrid
3213                                    status
3214                                    tearSheet {
3215                                        groupName
3216                                        members {
3217                                            name
3218                                            value
3219                                        }
3220                                    }
3221                                }
3222                            }
3223                            status
3224                        }
3225                        baselineStockUniverseId
3226                    }
3227                }
3228            }
3229        """
3230
3231        create_input: Dict[str, Any] = {
3232            "name": name,
3233            "experimentType": experiment_type,
3234            "description": description,
3235        }
3236        if isinstance(target_securities, dict):
3237            create_input["setTargetSecurities"] = [
3238                {"gbiId": sec.gbi_id, "weight": weight}
3239                for (sec, weight) in target_securities.items()
3240            ]
3241        elif isinstance(target_securities, str):
3242            create_input["setTargetPortfolios"] = [{"portfolioId": target_securities}]
3243        elif target_securities is None:
3244            pass
3245        else:
3246            raise TypeError(
3247                "Expected value of type Union[Dict[GbiIdSecurity, str], str] for "
3248                f"argument 'target_securities'; got {type(target_securities)}"
3249            )
3250        resp = requests.post(
3251            f"{self.base_uri}/api/graphql",
3252            json={"query": create_qry, "variables": {"input": create_input}},
3253            headers={"Authorization": "ApiKey " + self.api_key},
3254            params=self._request_params,
3255        )
3256
3257        json_resp = resp.json()
3258        if (resp.ok and "errors" in json_resp) or not resp.ok:
3259            error_msg = self._try_extract_error_code(resp)
3260            logger.error(error_msg)
3261            raise BoostedAPIException(
3262                (f"Failed to create hedge experiment: {resp.status_code=}; {error_msg=}")
3263            )
3264
3265        exp_dict = json_resp["data"]["createHedgeExperimentDraft"]["hedgeExperiment"]
3266        experiment = HedgeExperiment.from_json_dict(exp_dict)
3267        return experiment
3268
3269    def modify_hedge_experiment(
3270        self,
3271        experiment_id: str,
3272        name: Optional[str] = None,
3273        description: Optional[str] = None,
3274        experiment_type: Optional[hedge_experiment_type] = None,
3275        target_securities: Union[Dict[GbiIdSecurity, float], str, None] = None,
3276        model_ids: Optional[List[str]] = None,
3277        stock_universe_ids: Optional[List[str]] = None,
3278        create_default_scenario: bool = True,
3279        baseline_model_id: Optional[str] = None,
3280        baseline_stock_universe_id: Optional[str] = None,
3281        baseline_portfolio_settings: Optional[str] = None,
3282    ) -> HedgeExperiment:
3283        mod_qry = """
3284            mutation modifyHedgeExperimentDraft(
3285                $input: ModifyHedgeExperimentDraftInput!
3286            ) {
3287                modifyHedgeExperimentDraft(input: $input) {
3288                    hedgeExperiment {
3289                    ...HedgeExperimentSelectedSecuritiesPageFragment
3290                    }
3291                }
3292            }
3293
3294            fragment HedgeExperimentSelectedSecuritiesPageFragment on HedgeExperiment {
3295                hedgeExperimentId
3296                experimentName
3297                userId
3298                config
3299                description
3300                experimentType
3301                lastCalculated
3302                lastModified
3303                status
3304                portfolioCalcStatus
3305                targetSecurities {
3306                    gbiId
3307                    security {
3308                        gbiId
3309                        name
3310                        symbol
3311                    }
3312                    weight
3313                }
3314                targetPortfolios {
3315                    portfolioId
3316                }
3317                baselineModel {
3318                    id
3319                    name
3320                }
3321                baselineScenario {
3322                    hedgeExperimentScenarioId
3323                    scenarioName
3324                    description
3325                    portfolioSettingsJson
3326                    hedgeExperimentPortfolios {
3327                        portfolio {
3328                            id
3329                            name
3330                            modelId
3331                            performanceGridHeader
3332                            performanceGrid
3333                            status
3334                            tearSheet {
3335                                groupName
3336                                members {
3337                                    name
3338                                    value
3339                                }
3340                            }
3341                        }
3342                    }
3343                    status
3344                }
3345                baselineStockUniverseId
3346            }
3347        """
3348        mod_input = {
3349            "hedgeExperimentId": experiment_id,
3350            "createDefaultScenario": create_default_scenario,
3351        }
3352        if name is not None:
3353            mod_input["newExperimentName"] = name
3354        if description is not None:
3355            mod_input["newExperimentDescription"] = description
3356        if experiment_type is not None:
3357            mod_input["newExperimentType"] = experiment_type
3358        if model_ids is not None:
3359            mod_input["setSelectdModels"] = model_ids
3360        if stock_universe_ids is not None:
3361            mod_input["selectedStockUniverseIds"] = stock_universe_ids
3362        if baseline_model_id is not None:
3363            mod_input["setBaselineModel"] = baseline_model_id
3364        if baseline_stock_universe_id is not None:
3365            mod_input["setBaselineStockUniverse"] = baseline_stock_universe_id
3366        if baseline_portfolio_settings is not None:
3367            mod_input["setBaselinePortfolioSettings"] = baseline_portfolio_settings
3368        # note that the behaviors bound to these data are mutually exclusive,
3369        # and its possible the opposite was set earlier in the DRAFT phase
3370        # of experiment creation, so when setting one, we must unset the other
3371        if isinstance(target_securities, dict):
3372            mod_input["setTargetSecurities"] = [
3373                {"gbiId": sec.gbi_id, "weight": weight}
3374                for (sec, weight) in target_securities.items()
3375            ]
3376            mod_input["setTargetPortfolios"] = None
3377        elif isinstance(target_securities, str):
3378            mod_input["setTargetPortfolios"] = [{"portfolioId": target_securities}]
3379            mod_input["setTargetSecurities"] = None
3380        elif target_securities is None:
3381            pass
3382        else:
3383            raise TypeError(
3384                "Expected value of type Union[Dict[GbiIdSecurity, str], str] "
3385                f"for argument 'target_securities'; got {type(target_securities)}"
3386            )
3387
3388        resp = requests.post(
3389            f"{self.base_uri}/api/graphql",
3390            json={"query": mod_qry, "variables": {"input": mod_input}},
3391            headers={"Authorization": "ApiKey " + self.api_key},
3392            params=self._request_params,
3393        )
3394
3395        json_resp = resp.json()
3396        if (resp.ok and "errors" in json_resp) or not resp.ok:
3397            error_msg = self._try_extract_error_code(resp)
3398            logger.error(error_msg)
3399            raise BoostedAPIException(
3400                (
3401                    f"Failed to modify hedge experiment in preparation for start {experiment_id=}: "
3402                    f"{resp.status_code=}; {error_msg=}"
3403                )
3404            )
3405
3406        exp_dict = json_resp["data"]["modifyHedgeExperimentDraft"]["hedgeExperiment"]
3407        experiment = HedgeExperiment.from_json_dict(exp_dict)
3408        return experiment
3409
3410    def start_hedge_experiment(self, experiment_id: str, *scenario_ids: str) -> HedgeExperiment:
3411        start_qry = """
3412            mutation startHedgeExperiment($input: StartHedgeExperimentInput!) {
3413                startHedgeExperiment(input: $input) {
3414                    hedgeExperiment {
3415                        hedgeExperimentId
3416                        experimentName
3417                        userId
3418                        config
3419                        description
3420                        experimentType
3421                        lastCalculated
3422                        lastModified
3423                        status
3424                        portfolioCalcStatus
3425                        targetSecurities {
3426                            gbiId
3427                            security {
3428                                gbiId
3429                                name
3430                                symbol
3431                            }
3432                            weight
3433                        }
3434                        targetPortfolios {
3435                            portfolioId
3436                        }
3437                        baselineModel {
3438                            id
3439                            name
3440                        }
3441                        baselineScenario {
3442                            hedgeExperimentScenarioId
3443                            scenarioName
3444                            description
3445                            portfolioSettingsJson
3446                            hedgeExperimentPortfolios {
3447                                portfolio {
3448                                    id
3449                                    name
3450                                    modelId
3451                                    performanceGridHeader
3452                                    performanceGrid
3453                                    status
3454                                    tearSheet {
3455                                        groupName
3456                                        members {
3457                                            name
3458                                            value
3459                                        }
3460                                    }
3461                                }
3462                            }
3463                            status
3464                        }
3465                        baselineStockUniverseId
3466                    }
3467                }
3468            }
3469        """
3470        start_input: Dict[str, Any] = {"hedgeExperimentId": experiment_id}
3471        if len(scenario_ids) > 0:
3472            start_input["hedgeExperimentScenarioIds"] = list(scenario_ids)
3473
3474        resp = requests.post(
3475            f"{self.base_uri}/api/graphql",
3476            json={"query": start_qry, "variables": {"input": start_input}},
3477            headers={"Authorization": "ApiKey " + self.api_key},
3478            params=self._request_params,
3479        )
3480
3481        json_resp = resp.json()
3482        if (resp.ok and "errors" in json_resp) or not resp.ok:
3483            error_msg = self._try_extract_error_code(resp)
3484            logger.error(error_msg)
3485            raise BoostedAPIException(
3486                (
3487                    f"Failed to start hedge experiment {experiment_id=}: "
3488                    f"{resp.status_code=}; {error_msg=}"
3489                )
3490            )
3491
3492        exp_dict = json_resp["data"]["startHedgeExperiment"]["hedgeExperiment"]
3493        experiment = HedgeExperiment.from_json_dict(exp_dict)
3494        return experiment
3495
3496    def delete_hedge_experiment(self, experiment_id: str) -> bool:
3497        delete_qry = """
3498            mutation($input: DeleteHedgeExperimentsInput!) {
3499                deleteHedgeExperiments(input: $input) {
3500                    success
3501                }
3502            }
3503        """
3504        delete_input = {"hedgeExperimentIds": [experiment_id]}
3505        resp = requests.post(
3506            f"{self.base_uri}/api/graphql",
3507            json={"query": delete_qry, "variables": {"input": delete_input}},
3508            headers={"Authorization": "ApiKey " + self.api_key},
3509            params=self._request_params,
3510        )
3511
3512        json_resp = resp.json()
3513        if (resp.ok and "errors" in json_resp) or not resp.ok:
3514            error_msg = self._try_extract_error_code(resp)
3515            logger.error(error_msg)
3516            raise BoostedAPIException(
3517                (
3518                    f"Failed to delete hedge experiment {experiment_id=}: "
3519                    + f"status_code={resp.status_code}; error_msg={error_msg}"
3520                )
3521            )
3522
3523        return json_resp["data"]["deleteHedgeExperiments"]["success"]
3524
3525    def create_hedge_basket_position_bounds_from_csv(
3526        self,
3527        filepath: str,
3528        name: str,
3529        description: Optional[str],
3530        mapping_result_filepath: Optional[str],
3531    ) -> str:
3532        DATE = "Date"
3533        ISIN = "ISIN"
3534        COUNTRY = "Country"
3535        CURRENCY = "Currency"
3536        LOWER_BOUND = "Lower Bound"
3537        UPPER_BOUND = "Upper Bound"
3538        supported_columns = {
3539            DATE,
3540            ISIN,
3541            COUNTRY,
3542            CURRENCY,
3543            LOWER_BOUND,
3544            UPPER_BOUND,
3545        }
3546        required_columns = {ISIN, LOWER_BOUND, UPPER_BOUND}
3547
3548        try:
3549            df: pd.DataFrame = pd.read_csv(filepath, parse_dates=True)
3550        except Exception as e:
3551            raise BoostedAPIException(f"Error reading {filepath=}: {e}")
3552
3553        columns = set(df.columns)
3554
3555        # First perform basic data validation
3556        missing_required_columns = required_columns - columns
3557        if missing_required_columns:
3558            raise BoostedAPIException(
3559                f"The following required columns are missing: {missing_required_columns}"
3560            )
3561        extra_columns = columns - supported_columns
3562        if extra_columns:
3563            logger.warning(
3564                f"The following columns are unsupported and will be ignored: {extra_columns}"
3565            )
3566        try:
3567            df[LOWER_BOUND] = df[LOWER_BOUND].astype(float)
3568            df[UPPER_BOUND] = df[UPPER_BOUND].astype(float)
3569            df[ISIN] = df[ISIN].astype(str)
3570        except Exception as e:
3571            raise BoostedAPIException(f"Column datatypes are incorrect: {e}")
3572        lb_gt_ub = df[df[LOWER_BOUND] > df[UPPER_BOUND]]
3573        if not lb_gt_ub.empty:
3574            raise BoostedAPIException(
3575                f"Lower Bound must be <= Upper Bound, but these are not: {lb_gt_ub[ISIN].tolist()}"
3576            )
3577        out_of_range = df[
3578            (
3579                (df[LOWER_BOUND] < 0)
3580                | (df[LOWER_BOUND] > 1)
3581                | (df[UPPER_BOUND] < 0)
3582                | (df[UPPER_BOUND] > 1)
3583            )
3584        ]
3585        if not out_of_range.empty:
3586            raise BoostedAPIException("Lower Bound and Upper Bound values must be in range [0, 1]")
3587
3588        # Now map the security info into GBI IDs
3589        rows = list(df.to_dict(orient="index").values())
3590        sec_data_list = self.getGbiIdFromIdentCountryCurrencyDate(
3591            ident_country_currency_dates=[
3592                DateIdentCountryCurrency(
3593                    date=row.get(DATE, datetime.date.today().isoformat()),
3594                    identifier=row.get(ISIN),
3595                    id_type=ColumnSubRole.ISIN,
3596                    country=row.get(COUNTRY),
3597                    currency=row.get(CURRENCY),
3598                )
3599                for row in rows
3600            ]
3601        )
3602
3603        # Now take each row and its gbi id mapping, and create the bounds list
3604        bounds = []
3605        for row, sec_data in zip(rows, sec_data_list):
3606            if sec_data is None:
3607                logger.warning(f"Failed to map {row[ISIN]}, skipping this security.")
3608            else:
3609                bounds.append(
3610                    {"gbi_id": str(sec_data.gbi_id), "lb": row[LOWER_BOUND], "ub": row[UPPER_BOUND]}
3611                )
3612
3613                # Add security metadata to see the mapping
3614                row["Mapped GBI ID"] = sec_data.gbi_id
3615                row[f"Mapped {ISIN}"] = sec_data.isin_info.identifier
3616                row[f"Mapped {COUNTRY}"] = sec_data.isin_info.country
3617                row[f"Mapped {CURRENCY}"] = sec_data.isin_info.currency
3618                row["Mapped Ticker"] = sec_data.ticker
3619                row["Mapped Company Name"] = sec_data.company_name
3620
3621        # Call endpoint to create the bounds settings template
3622        qry = """
3623              mutation CreatePartialStrategyTemplate(
3624                $portfolioSettingsKey: String!
3625                $partialSettings: String!
3626                $name: String!
3627                $description: String
3628              ) {
3629                createPartialStrategyTemplate(
3630                  portfolioSettingsKey: $portfolioSettingsKey
3631                  partialSettings: $partialSettings
3632                  name: $name
3633                  description: $description
3634                )
3635              }
3636            """
3637        variables = {
3638            "portfolioSettingsKey": "basketTrading.positionSizeBounds",
3639            "partialSettings": json.dumps(bounds),
3640            "name": name,
3641            "description": description,
3642        }
3643        resp = self._get_graphql(qry, variables=variables)
3644
3645        # Write mapped csv for reference
3646        if mapping_result_filepath is not None:
3647            pd.DataFrame(rows).to_csv(mapping_result_filepath)
3648
3649        return resp["data"]["createPartialStrategyTemplate"]
3650
3651    def get_portfolio_accuracy(
3652        self,
3653        model_id: str,
3654        portfolio_id: str,
3655        start_date: Optional[BoostedDate] = None,
3656        end_date: Optional[BoostedDate] = None,
3657    ) -> dict:
3658        if start_date and end_date:
3659            validate_start_and_end_dates(start_date=start_date, end_date=end_date)
3660            start_date = convert_date(start_date)
3661            end_date = convert_date(end_date)
3662
3663        # TODO: Later change this URI to not use the watchlist prefix. It is misnamed.
3664        url = f"{self.base_uri}{WATCHLIST_ROUTE_PREFIX}{DAL_PA_ROUTE}/get-hit-rate/"
3665        headers = {"Authorization": "ApiKey " + self.api_key, "Content-Type": "application/json"}
3666        req_json = {"model_id": model_id, "portfolio_id": portfolio_id}
3667        if start_date and end_date:
3668            req_json["start_date"] = start_date.isoformat()
3669            req_json["end_date"] = end_date.isoformat()
3670        res = requests.post(url, json=req_json, headers=headers, **self._request_params)
3671
3672        if not res.ok:
3673            error_msg = self._try_extract_error_code(res)
3674            logger.error(error_msg)
3675            raise BoostedAPIException(f"Failed to get Hit Rate: {error_msg}")
3676
3677        data = res.json()
3678        return data
3679
3680    def create_watchlist(self, name: str) -> str:
3681        url = f"{self.base_uri}{WATCHLIST_ROUTE_PREFIX}{DAL_WATCHLIST_ROUTE}/create/"
3682        headers = {"Authorization": "ApiKey " + self.api_key, "Content-Type": "application/json"}
3683        req_json = {"name": name}
3684        res = requests.post(url, json=req_json, headers=headers, **self._request_params)
3685
3686        if not res.ok:
3687            error_msg = self._try_extract_error_code(res)
3688            logger.error(error_msg)
3689            raise BoostedAPIException(f"Failed to get user models: {error_msg}")
3690
3691        data = res.json()
3692        return data["watchlist_id"]
3693
3694    def _get_graphql(
3695        self,
3696        query: str,
3697        variables: Dict,
3698        error_msg_prefix: str = "Failed to get graphql result: ",
3699        log_error: bool = True,
3700    ) -> Dict:
3701        headers = {"Authorization": "ApiKey " + self.api_key}
3702        json_req = {"query": query, "variables": variables}
3703
3704        url = self.base_uri + "/api/graphql"
3705        resp = requests.post(
3706            url,
3707            json=json_req,
3708            headers=headers,
3709            params=self._request_params,
3710        )
3711
3712        # graphql endpoints typically return 200 or 400 status codes, so we must
3713        # check if we have any errors, even with a 200
3714        if not resp.ok or (resp.ok and "errors" in resp.json()):
3715            error_msg = self._try_extract_error_code(resp)
3716            error_str = str(error_msg_prefix) + f" {resp.status_code=}; {error_msg=}"
3717            if log_error:
3718                logger.error(error_str)
3719            raise BoostedAPIException(error_str)
3720
3721        json_resp = resp.json()
3722        return json_resp
3723
3724    def _get_security_info(self, gbi_ids: List[int]) -> Dict:
3725        query = graphql_queries.GET_SEC_INFO_QRY
3726        variables = {
3727            "ids": [] if not gbi_ids else gbi_ids,
3728        }
3729
3730        error_msg_prefix = "Failed to get Security Details:"
3731        return self._get_graphql(
3732            query=query, variables=variables, error_msg_prefix=error_msg_prefix
3733        )
3734
3735    def _get_sector_info(self) -> Dict:
3736        """
3737        Returns a list of sector objects, e.g.
3738        {
3739            "id": 1010,
3740            "parentId": 10,
3741            "name": "Energy",
3742            "topParentName": null,
3743            "spiqSectorId": -1,
3744            "legacy": false
3745        }
3746        """
3747        url = f"{self.base_uri}/api/sectors"
3748        headers = {"Authorization": "ApiKey " + self.api_key}
3749        res = requests.get(url, headers=headers, **self._request_params)
3750        self._check_ok_or_err_with_msg(res, "Failed to get sectors data")
3751        return res.json()["sectors"]
3752
3753    def _get_watchlist_analysis(
3754        self,
3755        gbi_ids: List[int],
3756        model_ids: List[str],
3757        portfolio_ids: List[str],
3758        asof_date=datetime.date.today(),
3759    ) -> Dict:
3760        query = graphql_queries.WATCHLIST_ANALYSIS_QRY
3761        variables = {
3762            "gbiIds": gbi_ids,
3763            "modelIds": model_ids,
3764            "portfolioIds": portfolio_ids,
3765            "date": self.__iso_format(asof_date),
3766        }
3767        error_msg_prefix = "Failed to get Coverage Analysis:"
3768        return self._get_graphql(
3769            query=query, variables=variables, error_msg_prefix=error_msg_prefix
3770        )
3771
3772    def _get_models_for_portfolio(self, portfolio_ids: List[str]) -> Dict:
3773        query = graphql_queries.GET_MODELS_FOR_PORTFOLIOS_QRY
3774        variables = {"ids": portfolio_ids}
3775        error_msg_prefix = "Failed to get Models for Portfolios: "
3776        return self._get_graphql(
3777            query=query, variables=variables, error_msg_prefix=error_msg_prefix
3778        )
3779
3780    def _get_excess_return(
3781        self, model_ids: List[str], gbi_ids: List[int], asof_date=datetime.date.today()
3782    ) -> Dict:
3783        query = graphql_queries.GET_EXCESS_RETURN_QRY
3784
3785        variables = {
3786            "modelIds": model_ids,
3787            "gbiIds": gbi_ids,
3788            "date": self.__iso_format(asof_date),
3789        }
3790        error_msg_prefix = "Failed to get Excess Return Slugging Pct: "
3791        return self._get_graphql(
3792            query=query, variables=variables, error_msg_prefix=error_msg_prefix
3793        )
3794
3795    def _coverage_column_name_format(self, in_str) -> str:
3796        if in_str.upper() == "ISIN":
3797            return "ISIN"
3798
3799        return in_str.title()
3800
3801    def _get_model_stocks(self, model_id: str) -> List[GbiIdTickerISIN]:
3802        # first, get the universe id
3803        resp = self._get_graphql(
3804            graphql_queries.GET_MODEL_STOCK_UNIVERSE_ID_QUERY,
3805            variables={"modelId": model_id},
3806            error_msg_prefix="Failed to get model stock universe ID",
3807        )
3808        universe_id = resp["data"]["model"]["stockUniverseId"]
3809
3810        # now, query for universe stocks
3811        url = self.base_uri + f"/api/stocks/model-universe/{universe_id}"
3812        headers = {"Authorization": "ApiKey " + self.api_key}
3813        universe_resp = requests.get(url, headers=headers, **self._request_params)
3814        universe = universe_resp.json()["stockUniverse"]
3815        securities = [
3816            GbiIdTickerISIN(gbi_id=security["id"], ticker=security["symbol"], isin=security["isin"])
3817            for security in universe
3818        ]
3819        return securities
3820
3821    def get_coverage_info(self, watchlist_id: str, portfolio_group_id: str) -> pd.DataFrame:
3822        # get securities list in watchlist
3823        watchlist_details = self.get_watchlist_details(watchlist_id)
3824        security_list = watchlist_details["targets"]
3825
3826        gbi_ids = [x["gbi_id"] for x in security_list]
3827
3828        gbi_data: Dict[Any, Dict] = {x: {} for x in gbi_ids}
3829
3830        # get security info ticker, name, industry etc
3831        sec_info = self._get_security_info(gbi_ids)
3832
3833        for sec in sec_info["data"]["securities"]:
3834            gbi_id = sec["gbiId"]
3835            for k in ["symbol", "name", "isin", "country", "currency"]:
3836                gbi_data[gbi_id][self._coverage_column_name_format(k)] = sec[k]
3837
3838            gbi_data[gbi_id][self._coverage_column_name_format("Sector")] = sec["sector"][
3839                "topParentName"
3840            ]
3841
3842        # get portfolios list in portfolio_Group
3843        portfolio_group = self.get_portfolio_group(portfolio_group_id)
3844        portfolio_ids = [x["portfolio_id"] for x in portfolio_group["portfolios"]]
3845        portfolio_info = {x["portfolio_id"]: x for x in portfolio_group["portfolios"]}
3846
3847        model_resp = self._get_models_for_portfolio(portfolio_ids=portfolio_ids)
3848        for portfolio in model_resp["data"]["portfolios"]:
3849            portfolio_info[portfolio["id"]].update(portfolio)
3850
3851        model_info = {
3852            x["modelId"]: portfolio_info[x["id"]] for x in model_resp["data"]["portfolios"]
3853        }
3854
3855        # model_ids and portfolio_ids are parallel arrays
3856        model_ids = [portfolio_info[x]["modelId"] for x in portfolio_ids]
3857
3858        # graphql: get watchlist analysis
3859        wl_analysis = self._get_watchlist_analysis(
3860            gbi_ids=gbi_ids,
3861            model_ids=model_ids,
3862            portfolio_ids=portfolio_ids,
3863            asof_date=datetime.date.today(),
3864        )
3865
3866        portfolio_gbi_data: Dict[Any, Dict] = {k: {} for k in portfolio_ids}
3867        for pi, v in portfolio_gbi_data.items():
3868            v.update({k: {} for k in gbi_data.keys()})
3869
3870        equity_explorer_date = wl_analysis["data"]["watchlistAnalysis"][0]["analysisDates"][0][
3871            "date"
3872        ]
3873        for wla in wl_analysis["data"]["watchlistAnalysis"]:
3874            gbi_id = wla["gbiId"]
3875            gbi_data[gbi_id]["Composite Rating"] = wla["analysisDates"][0]["aggregateSignal"][
3876                "rating"
3877            ]
3878            gbi_data[gbi_id]["Composite Rating Delta"] = wla["analysisDates"][0]["aggregateSignal"][
3879                "ratingDelta"
3880            ]
3881
3882            for p in wla["analysisDates"][0]["portfoliosSignals"]:
3883                model_name = portfolio_info[p["portfolioId"]]["modelName"]
3884
3885                portfolio_gbi_data[p["portfolioId"]][gbi_id][
3886                    model_name + self._coverage_column_name_format(": rank")
3887                ] = (p["rank"] + 1)
3888                portfolio_gbi_data[p["portfolioId"]][gbi_id][
3889                    model_name + self._coverage_column_name_format(": rank delta")
3890                ] = (-1 * p["signalDelta"])
3891                portfolio_gbi_data[p["portfolioId"]][gbi_id][
3892                    model_name + self._coverage_column_name_format(": rating")
3893                ] = p["rating"]
3894                portfolio_gbi_data[p["portfolioId"]][gbi_id][
3895                    model_name + self._coverage_column_name_format(": rating delta")
3896                ] = p["ratingDelta"]
3897
3898        neg_rec: Dict[Any, Dict] = {k: {} for k in gbi_data.keys()}
3899        pos_rec: Dict[Any, Dict] = {k: {} for k in gbi_data.keys()}
3900        for wla in wl_analysis["data"]["watchlistAnalysis"]:
3901            gbi_id = wla["gbiId"]
3902
3903            for pid, signals in zip(portfolio_ids, wla["analysisDates"][0]["portfoliosSignals"]):
3904                model_name = portfolio_info[pid]["modelName"]
3905                neg_rec[gbi_id][
3906                    model_name + self._coverage_column_name_format(": negative recommendation")
3907                ] = signals["explainWeightNeg"]
3908                pos_rec[gbi_id][
3909                    model_name + self._coverage_column_name_format(": positive recommendation")
3910                ] = signals["explainWeightPos"]
3911
3912        # graphql: GetExcessReturn - slugging pct
3913        er_sp = self._get_excess_return(
3914            model_ids=model_ids, gbi_ids=gbi_ids, asof_date=equity_explorer_date
3915        )
3916
3917        for model in er_sp["data"]["models"]:
3918            model_name = model_info[model["id"]]["modelName"]
3919            for stat in model["equityExplorerData"]["equityExplorerSummaryStatistics"]:
3920                portfolioId = model_info[model["id"]]["id"]
3921                portfolio_gbi_data[portfolioId][int(stat["gbiId"])][
3922                    model_name + self._coverage_column_name_format(": slugging %")
3923                ] = (stat["ER"]["SP"]["sixMonthWindowOneMonthHorizon"] * 100)
3924
3925        # add rank, rating, slugging
3926        for pid, v in portfolio_gbi_data.items():
3927            for gbi_id, vv in v.items():
3928                gbi_data[gbi_id].update(vv)
3929
3930        # add neg/pos rec scores
3931        for rec in [neg_rec, pos_rec]:
3932            for k, v in rec.items():
3933                gbi_data[k].update(v)
3934
3935        df = pd.DataFrame.from_records([v for _, v in gbi_data.items()])
3936
3937        return df
3938
3939    def get_coverage_csv(
3940        self, watchlist_id: str, portfolio_group_id: str, filepath: Optional[str] = None
3941    ) -> Optional[str]:
3942        """
3943        Converts the coverage contents to CSV format
3944        Parameters
3945        ----------
3946        watchlist_id: str
3947            UUID str identifying the coverage watchlist
3948        portfolio_group_id: str
3949            UUID str identifying the group of portfolio to use for analysis
3950        filepath: Optional[str]
3951            UUID str identifying the group of portfolio to use for analysis
3952
3953        Returns:
3954        ----------
3955        None if filepath is provided, else a string with a csv's contents is returned
3956        """
3957
3958        df = self.get_coverage_info(watchlist_id, portfolio_group_id)
3959
3960        return df.to_csv(filepath, index=False, float_format="%.4f")
3961
3962    def get_watchlist_details(self, watchlist_id: str) -> Dict:
3963        url = f"{self.base_uri}{ROUTE_PREFIX}{DAL_WATCHLIST_ROUTE}/details/"
3964        headers = {"Authorization": "ApiKey " + self.api_key, "Content-Type": "application/json"}
3965        req_json = {"watchlist_id": watchlist_id}
3966        res = requests.post(url, json=req_json, headers=headers, **self._request_params)
3967
3968        if not res.ok:
3969            error_msg = self._try_extract_error_code(res)
3970            logger.error(error_msg)
3971            raise BoostedAPIException(f"Failed to get user models: {error_msg}")
3972
3973        data = res.json()
3974        return data
3975
3976    def create_watchlist_from_file(self, name: str, filepath: str) -> str:
3977        url = f"{self.base_uri}{ROUTE_PREFIX}{DAL_WATCHLIST_ROUTE}/create_watchlist_from_file/"
3978        headers = {"Authorization": "ApiKey " + self.api_key}
3979
3980        with open(filepath, "rb") as fp:
3981            file_bytes = fp.read()
3982
3983        file_bytes_base64 = base64.b64encode(file_bytes).decode("ascii")
3984        json_req = {
3985            "content_type": mimetypes.guess_type(filepath)[0],
3986            "file_bytes_base64": file_bytes_base64,
3987            "name": name,
3988        }
3989
3990        res = requests.post(url, json=json_req, headers=headers)
3991
3992        if not res.ok:
3993            error_msg = self._try_extract_error_code(res)
3994            logger.error(error_msg)
3995            raise BoostedAPIException(f"Failed to create watchlist from file: {error_msg}")
3996
3997        data = res.json()
3998        return data["watchlist_id"]
3999
4000    def get_watchlists(self) -> List[Dict]:
4001        url = f"{self.base_uri}{ROUTE_PREFIX}{DAL_WATCHLIST_ROUTE}/get_user_watchlists/"
4002        headers = {"Authorization": "ApiKey " + self.api_key, "Content-Type": "application/json"}
4003        req_json: Dict = {}
4004        res = requests.post(url, json=req_json, headers=headers, **self._request_params)
4005
4006        if not res.ok:
4007            error_msg = self._try_extract_error_code(res)
4008            logger.error(error_msg)
4009            raise BoostedAPIException(f"Failed to get user watchlists: {error_msg}")
4010
4011        data = res.json()
4012        return data["watchlists"]
4013
4014    def get_watchlist_contents(self, watchlist_id) -> Dict:
4015        url = f"{self.base_uri}{ROUTE_PREFIX}{DAL_WATCHLIST_ROUTE}/contents/"
4016        headers = {"Authorization": "ApiKey " + self.api_key, "Content-Type": "application/json"}
4017        req_json = {"watchlist_id": watchlist_id}
4018        res = requests.post(url, json=req_json, headers=headers, **self._request_params)
4019
4020        if not res.ok:
4021            error_msg = self._try_extract_error_code(res)
4022            logger.error(error_msg)
4023            raise BoostedAPIException(f"Failed to get watchlist contents: {error_msg}")
4024
4025        data = res.json()
4026        return data
4027
4028    def get_watchlist_contents_as_csv(self, watchlist_id, filepath) -> None:
4029        data = self.get_watchlist_contents(watchlist_id)
4030        df = pd.DataFrame(data["contents"])
4031        df.to_csv(filepath, index=False)
4032
4033    # TODO this will need to be enhanced to accept country/currency overrides
4034    def add_securities_to_watchlist(
4035        self, watchlist_id: str, identifiers: List[str], identifier_type: Literal["TICKER", "ISIN"]
4036    ) -> Dict:
4037        # should we just make the arg lower? all caps has a flag-like feel to it
4038        id_type = identifier_type.lower()
4039        url = f"{self.base_uri}{WATCHLIST_ROUTE_PREFIX}{DAL_WATCHLIST_ROUTE}/add_{id_type}s/"
4040        headers = {"Authorization": "ApiKey " + self.api_key, "Content-Type": "application/json"}
4041        req_json = {"watchlist_id": watchlist_id, id_type: identifiers}
4042        res = requests.post(url, json=req_json, headers=headers, **self._request_params)
4043
4044        if not res.ok:
4045            error_msg = self._try_extract_error_code(res)
4046            logger.error(error_msg)
4047            raise BoostedAPIException(f"Failed to get user models: {error_msg}")
4048
4049        data = res.json()
4050        return data
4051
4052    def remove_securities_from_watchlist(
4053        self, watchlist_id: str, identifiers: List[str], identifier_type: Literal["TICKER", "ISIN"]
4054    ) -> Dict:
4055        # should we just make the arg lower? all caps has a flag-like feel to it
4056        id_type = identifier_type.lower()
4057        url = f"{self.base_uri}{WATCHLIST_ROUTE_PREFIX}{DAL_WATCHLIST_ROUTE}/remove_{id_type}s/"
4058        headers = {"Authorization": "ApiKey " + self.api_key, "Content-Type": "application/json"}
4059        req_json = {"watchlist_id": watchlist_id, id_type: identifiers}
4060        res = requests.post(url, json=req_json, headers=headers, **self._request_params)
4061
4062        if not res.ok:
4063            error_msg = self._try_extract_error_code(res)
4064            logger.error(error_msg)
4065            raise BoostedAPIException(f"Failed to get user models: {error_msg}")
4066
4067        data = res.json()
4068        return data
4069
4070    def get_portfolio_groups(
4071        self,
4072    ) -> Dict:
4073        """
4074        Parameters: None
4075
4076
4077        Returns:
4078        ----------
4079
4080        Dict:  {
4081        user_id: str
4082        portfolio_groups: List[PortfolioGroup]
4083        }
4084        where PortfolioGroup is defined as = Dict {
4085        group_id: str
4086        group_name: str
4087        portfolios: List[PortfolioInGroup]
4088        }
4089        where PortfolioInGroup is defined as = Dict {
4090        portfolio_id: str
4091        rank_in_group: Optional[int]
4092        }
4093        """
4094        url = f"{self.base_uri}{WATCHLIST_ROUTE_PREFIX}{PORTFOLIO_GROUP_ROUTE}/get"
4095        headers = {"Authorization": "ApiKey " + self.api_key, "Content-Type": "application/json"}
4096        req_json: Dict = {}
4097        res = requests.post(url, json=req_json, headers=headers, **self._request_params)
4098
4099        if not res.ok:
4100            error_msg = self._try_extract_error_code(res)
4101            logger.error(error_msg)
4102            raise BoostedAPIException(f"Failed to get user portfolio groups: {error_msg}")
4103
4104        data = res.json()
4105        return data
4106
4107    def get_portfolio_group(self, portfolio_group_id: str) -> Dict:
4108        """
4109        Parameters:
4110        portfolio_group_id: str
4111           UUID identifier for the portfolio group
4112
4113
4114        Returns:
4115        ----------
4116
4117        PortfolioGroup: Dict:  {
4118        group_id: str
4119        group_name: str
4120        portfolios: List[PortfolioInGroup]
4121        }
4122        where PortfolioInGroup is defined as = Dict {
4123        portfolio_id: str
4124        portfolio_name: str
4125        rank_in_group: Optional[int]
4126        }
4127        """
4128        url = f"{self.base_uri}{WATCHLIST_ROUTE_PREFIX}{PORTFOLIO_GROUP_ROUTE}/get-one"
4129        headers = {"Authorization": "ApiKey " + self.api_key, "Content-Type": "application/json"}
4130        req_json = {"portfolio_group_id": portfolio_group_id}
4131        res = requests.post(url, json=req_json, headers=headers, **self._request_params)
4132
4133        if not res.ok:
4134            error_msg = self._try_extract_error_code(res)
4135            logger.error(error_msg)
4136            raise BoostedAPIException(f"Failed to get user portfolio groups: {error_msg}")
4137
4138        data = res.json()
4139        return data
4140
4141    def set_sticky_portfolio_group(
4142        self,
4143        portfolio_group_id: str,
4144    ) -> Dict:
4145        """
4146        Set sticky portfolio group
4147
4148        Parameters
4149        ----------
4150
4151        group_id: str,
4152           UUID str identifying a portfolio group
4153
4154        Returns:
4155        -------
4156        Dict {
4157            changed: int - 1 == success
4158        }
4159        """
4160        url = f"{self.base_uri}{WATCHLIST_ROUTE_PREFIX}{PORTFOLIO_GROUP_ROUTE}/set-sticky"
4161        headers = {"Authorization": "ApiKey " + self.api_key, "Content-Type": "application/json"}
4162        req_json = {"portfolio_group_id": portfolio_group_id}
4163        res = requests.post(url, json=req_json, headers=headers, **self._request_params)
4164
4165        if not res.ok:
4166            error_msg = self._try_extract_error_code(res)
4167            logger.error(error_msg)
4168            raise BoostedAPIException(f"Failed to set sticky portfolio group: {error_msg}")
4169
4170        data = res.json()
4171        return data
4172
4173    def get_sticky_portfolio_group(
4174        self,
4175    ) -> Dict:
4176        """
4177        Get sticky portfolio group for the user
4178
4179        Parameters
4180        ----------
4181
4182        Returns:
4183        -------
4184        Dict {
4185            group_id: str
4186            group_name: str
4187            portfolios: List[PortfolioInGroup(Dict)]
4188                  PortfolioInGroup(Dict):
4189                           portfolio_id: str
4190                           rank_in_group: Optional[int] = None
4191                           portfolio_name: Optional[str] = None
4192        }
4193        """
4194        url = f"{self.base_uri}{WATCHLIST_ROUTE_PREFIX}{PORTFOLIO_GROUP_ROUTE}/get-sticky"
4195        headers = {"Authorization": "ApiKey " + self.api_key, "Content-Type": "application/json"}
4196        req_json: Dict = {}
4197        res = requests.post(url, json=req_json, headers=headers, **self._request_params)
4198
4199        if not res.ok:
4200            error_msg = self._try_extract_error_code(res)
4201            logger.error(error_msg)
4202            raise BoostedAPIException(f"Failed to get sticky portfolio group: {error_msg}")
4203
4204        data = res.json()
4205        return data
4206
4207    def create_portfolio_group(
4208        self,
4209        group_name: str,
4210        portfolios: Optional[List[Dict]] = None,
4211    ) -> Dict:
4212        """
4213        Create a new portfolio group
4214
4215        Parameters
4216        ----------
4217
4218        group_name: str
4219           name of the new group
4220
4221        portfolios: List of Dict [:
4222
4223        portfolio_id: str
4224        rank_in_group: Optional[int] = None
4225        ]
4226
4227        Returns:
4228        ----------
4229
4230        Dict: {
4231        group_id: str
4232           UUID identifier for the portfolio group
4233
4234        created: int
4235           num groups created, 1 == success
4236
4237        added: int
4238           num portfolios added to the group, should match the length of 'portfolios' argument
4239        }
4240        """
4241        url = f"{self.base_uri}{WATCHLIST_ROUTE_PREFIX}{PORTFOLIO_GROUP_ROUTE}/create"
4242        headers = {"Authorization": "ApiKey " + self.api_key, "Content-Type": "application/json"}
4243        req_json = {"group_name": group_name, "portfolios": portfolios}
4244
4245        res = requests.post(url, json=req_json, headers=headers, **self._request_params)
4246
4247        if not res.ok:
4248            error_msg = self._try_extract_error_code(res)
4249            logger.error(error_msg)
4250            raise BoostedAPIException(f"Failed to create portfolio group: {error_msg}")
4251
4252        data = res.json()
4253        return data
4254
4255    def rename_portfolio_group(
4256        self,
4257        group_id: str,
4258        group_name: str,
4259    ) -> Dict:
4260        """
4261        Rename a portfolio group
4262
4263        Parameters
4264        ----------
4265
4266        group_id: str,
4267           UUID str identifying a portfolio group
4268
4269        group_name: str,
4270           The new name for the porfolio
4271
4272        Returns:
4273        -------
4274        Dict {
4275            changed: int - 1 == success
4276        }
4277        """
4278        url = f"{self.base_uri}{WATCHLIST_ROUTE_PREFIX}{PORTFOLIO_GROUP_ROUTE}/rename"
4279        headers = {"Authorization": "ApiKey " + self.api_key, "Content-Type": "application/json"}
4280        req_json = {"group_id": group_id, "group_name": group_name}
4281        res = requests.post(url, json=req_json, headers=headers, **self._request_params)
4282
4283        if not res.ok:
4284            error_msg = self._try_extract_error_code(res)
4285            logger.error(error_msg)
4286            raise BoostedAPIException(f"Failed to rename portfolio group: {error_msg}")
4287
4288        data = res.json()
4289        return data
4290
4291    def add_to_portfolio_group(
4292        self,
4293        group_id: str,
4294        portfolios: List[Dict],
4295    ) -> Dict:
4296        """
4297        Add portfolios to a group
4298
4299        Parameters
4300        ----------
4301
4302        group_id: str,
4303           UUID str identifying a portfolio group
4304
4305        portfolios: List of Dict [:
4306            portfolio_id: str
4307            rank_in_group: Optional[int] = None
4308        ]
4309
4310
4311        Returns:
4312        -------
4313        Dict {
4314            added: int
4315               number of successful changes
4316        }
4317        """
4318        url = f"{self.base_uri}{WATCHLIST_ROUTE_PREFIX}{PORTFOLIO_GROUP_ROUTE}/add-to-group"
4319        headers = {"Authorization": "ApiKey " + self.api_key, "Content-Type": "application/json"}
4320        req_json = {"group_id": group_id, "portfolios": portfolios}
4321
4322        res = requests.post(url, json=req_json, headers=headers, **self._request_params)
4323
4324        if not res.ok:
4325            error_msg = self._try_extract_error_code(res)
4326            logger.error(error_msg)
4327            raise BoostedAPIException(f"Failed to add portfolios to portfolio group: {error_msg}")
4328
4329        data = res.json()
4330        return data
4331
4332    def remove_from_portfolio_group(
4333        self,
4334        group_id: str,
4335        portfolios: List[str],
4336    ) -> Dict:
4337        """
4338        Remove portfolios from a group
4339
4340        Parameters
4341        ----------
4342
4343        group_id: str,
4344           UUID str identifying a portfolio group
4345
4346        portfolios: List of str
4347
4348
4349        Returns:
4350        -------
4351        Dict {
4352            removed: int
4353               number of successful changes
4354        }
4355        """
4356        url = f"{self.base_uri}{WATCHLIST_ROUTE_PREFIX}{PORTFOLIO_GROUP_ROUTE}/remove-from-group"
4357        headers = {"Authorization": "ApiKey " + self.api_key, "Content-Type": "application/json"}
4358        req_json = {"group_id": group_id, "portfolios": portfolios}
4359        res = requests.post(url, json=req_json, headers=headers, **self._request_params)
4360
4361        if not res.ok:
4362            error_msg = self._try_extract_error_code(res)
4363            logger.error(error_msg)
4364            raise BoostedAPIException(
4365                f"Failed to remove portfolios from portfolio group: {error_msg}"
4366            )
4367
4368        data = res.json()
4369        return data
4370
4371    def delete_portfolio_group(
4372        self,
4373        group_id: str,
4374    ) -> Dict:
4375        """
4376        Delete a portfolio group
4377
4378        Parameters
4379        ----------
4380
4381        group_id: str,
4382           UUID str identifying a portfolio group
4383
4384
4385        Returns:
4386        -------
4387        Dict {
4388            removed_groups: int
4389               number of successful changes
4390
4391            removed_portfolios: int
4392               number of successful changes
4393        }
4394        """
4395        url = f"{self.base_uri}{WATCHLIST_ROUTE_PREFIX}{PORTFOLIO_GROUP_ROUTE}/remove"
4396        headers = {"Authorization": "ApiKey " + self.api_key, "Content-Type": "application/json"}
4397        req_json = {"group_id": group_id}
4398        res = requests.post(url, json=req_json, headers=headers, **self._request_params)
4399
4400        if not res.ok:
4401            error_msg = self._try_extract_error_code(res)
4402            logger.error(error_msg)
4403            raise BoostedAPIException(f"Failed to delete portfolio group: {error_msg}")
4404
4405        data = res.json()
4406        return data
4407
4408    def set_portfolio_group_for_watchlist(
4409        self,
4410        portfolio_group_id: str,
4411        watchlist_id: str,
4412    ) -> Dict:
4413        """
4414        Set portfolio group for watchlist.
4415
4416        Parameters
4417        ----------
4418
4419        portfolio_group_id: str,
4420           UUID str identifying a portfolio group
4421
4422        watchlist_id: str,
4423           UUID str identifying a watchlist
4424
4425
4426        Returns:
4427        -------
4428        Dict {
4429            success: bool
4430            errors:
4431            data: Dict
4432                changed: int
4433        }
4434        """
4435        url = f"{self.base_uri}{WATCHLIST_ROUTE_PREFIX}{DAL_WATCHLIST_ROUTE}/set-portfolio-groups/"
4436        headers = {"Authorization": "ApiKey " + self.api_key, "Content-Type": "application/json"}
4437        req_json = {"portfolio_group_id": portfolio_group_id, "watchlist_id": watchlist_id}
4438        res = requests.post(url, json=req_json, headers=headers, **self._request_params)
4439
4440        if not res.ok:
4441            error_msg = self._try_extract_error_code(res)
4442            logger.error(error_msg)
4443            raise BoostedAPIException(f"Failed to set portfolio group for watchlist: {error_msg}")
4444
4445        return res.json()
4446
4447    def get_ranking_dates(self, model_id: str, portfolio_id: str) -> List[datetime.date]:
4448        headers = {"Authorization": "ApiKey " + self.api_key, "Content-Type": "application/json"}
4449        url = self.base_uri + f"/api/analysis/ranking-dates/{model_id}/{portfolio_id}"
4450        res = requests.get(url, headers=headers, **self._request_params)
4451        self._check_ok_or_err_with_msg(res, "Failed to get ranking dates")
4452        data = res.json().get("ranking_dates", [])
4453
4454        return [parser.parse(d).date() for d in data]
4455
4456    def get_prior_ranking_date(
4457        self, ranking_dates: List[datetime.date], starting_date: datetime.date
4458    ) -> datetime.date:
4459        """
4460        Given a starting date and a list of ranking dates, return the most
4461        recent previous ranking date.
4462        """
4463        # order from most recent to least
4464        ranking_dates.sort(reverse=True)
4465
4466        for d in ranking_dates:
4467            if d <= starting_date:
4468                return d
4469
4470        # if we get here, the starting date is before the earliest ranking date
4471        raise BoostedAPIException(f"No rankins exist on or before {starting_date}")
4472
4473    def _get_risk_factors_descriptors(
4474        self, model_id: str, portfolio_id: str, use_v2: bool = False
4475    ) -> Dict[int, str]:
4476        """Returns a map from descriptor id to descriptor name."""
4477        headers = {"Authorization": "ApiKey " + self.api_key, "Content-Type": "application/json"}
4478
4479        risk_factor = RISK_FACTOR_V2 if use_v2 else RISK_FACTOR
4480        url = self.base_uri + f"/api/{risk_factor}/{model_id}/{portfolio_id}/descriptors"
4481        res = requests.get(url, headers=headers, **self._request_params)
4482
4483        self._check_ok_or_err_with_msg(res, "Failed to get risk factor descriptors")
4484
4485        descriptors = {int(i): name for i, name in res.json().items() if i.isnumeric()}
4486        return descriptors
4487
4488    def get_risk_groups(
4489        self, model_id: str, portfolio_id: str, date: datetime.date, use_v2: bool = False
4490    ) -> List[Dict[str, Any]]:
4491        # first get the group descriptors
4492        descriptors = self._get_risk_factors_descriptors(model_id, portfolio_id, use_v2)
4493
4494        # calculate the most recent prior rankings date. This is the date
4495        # we need to use to query for risk group data.
4496        ranking_dates = self.get_ranking_dates(model_id, portfolio_id)
4497        ranking_date = self.get_prior_ranking_date(ranking_dates, date)
4498        date_str = ranking_date.strftime("%Y-%m-%d")
4499
4500        risk_factor = RISK_FACTOR_V2 if use_v2 else RISK_FACTOR
4501
4502        headers = {"Authorization": "ApiKey " + self.api_key, "Content-Type": "application/json"}
4503        url = self.base_uri + f"/api/{risk_factor}/{model_id}/{portfolio_id}/risk-groups/{date_str}"
4504        res = requests.get(url, headers=headers, **self._request_params)
4505
4506        self._check_ok_or_err_with_msg(
4507            res, f"Failed to get risk factors for {model_id=}, {portfolio_id=}, {date=}"
4508        )
4509
4510        # Response is a list of objects like:
4511        # [
4512        #   [
4513        #     0,
4514        #     14,
4515        #     1
4516        #   ],
4517        #   [
4518        #     25,
4519        #     12,
4520        #     13
4521        #   ],
4522        # 0.67013
4523        # ],
4524        #
4525        # Where each integer in the lists is a descriptor id.
4526
4527        groups = []
4528        for i, row in enumerate(res.json()):
4529            row_map: Dict[str, Any] = {}
4530            # map descriptor id to name
4531            row_map["machine"] = i + 1  # start at 1 not 0
4532            row_map["risk_group_a"] = [descriptors[i] for i in row[0]]
4533            row_map["risk_group_b"] = [descriptors[i] for i in row[1]]
4534            row_map["volatility_explained"] = row[2]
4535            groups.append(row_map)
4536
4537        return groups
4538
4539    def get_risk_factors_discovered_descriptors(
4540        self, model_id: str, portfolio_id: str, date: datetime.date, use_v2: bool = False
4541    ) -> pd.DataFrame:
4542        # first get the group descriptors
4543        descriptors = self._get_risk_factors_descriptors(model_id, portfolio_id)
4544
4545        # calculate the most recent prior rankings date. This is the date
4546        # we need to use to query for risk group data.
4547        ranking_dates = self.get_ranking_dates(model_id, portfolio_id)
4548        ranking_date = self.get_prior_ranking_date(ranking_dates, date)
4549        date_str = ranking_date.strftime("%Y-%m-%d")
4550
4551        risk_factor = RISK_FACTOR_V2 if use_v2 else RISK_FACTOR
4552
4553        headers = {"Authorization": "ApiKey " + self.api_key, "Content-Type": "application/json"}
4554        url = (
4555            self.base_uri
4556            + f"/api/{risk_factor}/{model_id}/{portfolio_id}/risk-descriptors/json/{date_str}"
4557        )
4558        res = requests.get(url, headers=headers, **self._request_params)
4559
4560        self._check_ok_or_err_with_msg(
4561            res, f"Failed to get risk factors for {model_id=}, {portfolio_id=}, {date=}"
4562        )
4563
4564        # Endpoint returns a nested list of floats
4565        df = pd.DataFrame(res.json(), columns=RISK_FACTOR_COLUMNS)
4566
4567        # This flat dataframe represents a potentially doubly nested structure
4568        # of Sector -> (high/low volatility) -> security. We don't care about
4569        # the high/low volatility rows, (which will have negative identifiers)
4570        # so we can filter these out.
4571        df = df[df["identifier"] >= 0]
4572
4573        # now, any values that had a depth of 2 should be set to a depth of 1,
4574        # since we removed the double nesting.
4575        df.replace(to_replace=2, value=1, inplace=True)
4576
4577        # This dataframe represents data that is nested on the UI, so the
4578        # "depth" field indicates which level of nesting each row is at. At this
4579        # point, a depth of 0 indicates a sector, and following depth 1 rows are
4580        # securities within the sector.
4581
4582        # Identifiers in rows with depth 1 will be gbi ids, need to convert to
4583        # symbols.
4584        gbi_ids = df[df["depth"] == 1]["identifier"].tolist()
4585        sec_info = self._get_security_info(gbi_ids)["data"]["securities"]
4586        sec_map = {s["gbiId"]: s["symbol"] for s in sec_info}
4587
4588        def convert_ids(row: pd.Series) -> pd.Series:
4589            # convert each row's "identifier" to the appropriate id type. If the
4590            # depth is 0, the identifier should be a sector, otherwise it should
4591            # be a ticker.
4592            ident = int(row["identifier"])
4593            row["identifier"] = (
4594                descriptors.get(ident).title() if row["depth"] == 0 else sec_map.get(ident)
4595            )
4596            return row
4597
4598        df["depth"] = df["depth"].astype(int)
4599        df["stock_count"] = df["stock_count"].astype(int)
4600        df = df.apply(convert_ids, axis=1)
4601        df = df.reset_index(drop=True)
4602        return df
4603
4604    def get_risk_factors_sectors(
4605        self, model_id: str, portfolio_id: str, date: datetime.date, use_v2: bool = False
4606    ) -> pd.DataFrame:
4607        # first get the group descriptors
4608        sectors = {s["id"]: s["name"] for s in self._get_sector_info()}
4609
4610        # calculate the most recent prior rankings date. This is the date
4611        # we need to use to query for risk group data.
4612        ranking_dates = self.get_ranking_dates(model_id, portfolio_id)
4613        ranking_date = self.get_prior_ranking_date(ranking_dates, date)
4614        date_str = ranking_date.strftime("%Y-%m-%d")
4615
4616        risk_factor = RISK_FACTOR_V2 if use_v2 else RISK_FACTOR
4617
4618        headers = {"Authorization": "ApiKey " + self.api_key, "Content-Type": "application/json"}
4619        url = (
4620            self.base_uri
4621            + f"/api/{risk_factor}/{model_id}/{portfolio_id}/risk-sectors/json/{date_str}"
4622        )
4623        res = requests.get(url, headers=headers, **self._request_params)
4624
4625        self._check_ok_or_err_with_msg(
4626            res, f"Failed to get risk factors for {model_id=}, {portfolio_id=}, {date=}"
4627        )
4628
4629        # Endpoint returns a nested list of floats
4630        df = pd.DataFrame(res.json(), columns=RISK_FACTOR_COLUMNS)
4631
4632        # identifier is a gics sector identifier
4633        df["identifier"] = df["identifier"].apply(lambda i: sectors.get(int(i), None))
4634
4635        # This dataframe represents data that is nested on the UI, so the
4636        # "depth" field indicates which level of nesting each row is at. For
4637        # risk factors sectors, each "depth" represents a level of specificity
4638        # for the sector. E.g. Energy -> Energy Equipment -> Oil & Gas Equipment
4639        df["depth"] = df["depth"].astype(int)
4640        df["stock_count"] = df["stock_count"].astype(int)
4641        df = df.reset_index(drop=True)
4642        return df
4643
4644    def download_complete_portfolio_data(
4645        self, model_id: str, portfolio_id: str, download_filepath: str
4646    ):
4647        headers = {"Authorization": "ApiKey " + self.api_key, "Content-Type": "application/json"}
4648        url = self.base_uri + f"/api/models/{model_id}/{portfolio_id}/excel"
4649
4650        res = requests.get(url, headers=headers, **self._request_params)
4651        self._check_ok_or_err_with_msg(
4652            res, f"Failed to get full data for {model_id=}, {portfolio_id=}"
4653        )
4654
4655        with open(download_filepath, "wb") as f:
4656            f.write(res.content)
4657
4658    def diff_hedge_experiment_portfolio_data(
4659        self,
4660        hedge_experiment_id: str,
4661        comparison_portfolios: List[str],
4662        categories: List[str],
4663    ) -> Dict:
4664        qry = """
4665        query diffHedgeExperimentPortfolios(
4666            $input: DiffHedgeExperimentPortfoliosInput!
4667        ) {
4668            diffHedgeExperimentPortfolios(input: $input) {
4669            data {
4670                diffs {
4671                    volatility {
4672                        date
4673                        vol5D
4674                        vol10D
4675                        vol21D
4676                        vol21D
4677                        vol63D
4678                        vol126D
4679                        vol189D
4680                        vol252D
4681                        vol315D
4682                        vol378D
4683                        vol441D
4684                        vol504D
4685                    }
4686                    performance {
4687                        date
4688                        value
4689                    }
4690                    performanceGrid {
4691                        headerRow
4692                        values
4693                    }
4694                    factors {
4695                        date
4696                        momentum
4697                        growth
4698                        size
4699                        value
4700                        dividendYield
4701                        volatility
4702                    }
4703                }
4704            }
4705            errors
4706            }
4707        }
4708        """
4709        headers = {"Authorization": "ApiKey " + self.api_key}
4710        params = {
4711            "hedgeExperimentId": hedge_experiment_id,
4712            "portfolioIds": comparison_portfolios,
4713            "categories": categories,
4714        }
4715        resp = requests.post(
4716            f"{self.base_uri}/api/graphql",
4717            json={"query": qry, "variables": params},
4718            headers=headers,
4719            params=self._request_params,
4720        )
4721
4722        json_resp = resp.json()
4723
4724        # graphql endpoints typically return 200 or 400 status codes, so we must
4725        # check if we have any errors, even with a 200
4726        if (resp.ok and "errors" in json_resp) or not resp.ok:
4727            error_msg = self._try_extract_error_code(resp)
4728            logger.error(error_msg)
4729            raise BoostedAPIException(
4730                (
4731                    f"Failed to get portfolio diffs for {hedge_experiment_id=}: "
4732                    f"{resp.status_code=}; {error_msg=}"
4733                )
4734            )
4735
4736        diffs = json_resp["data"]["diffHedgeExperimentPortfolios"]["data"]["diffs"]
4737        comparisons = {}
4738        for pf, cmp in zip(comparison_portfolios, diffs):
4739            res: Dict[str, Any] = {
4740                "performance": None,
4741                "performanceGrid": None,
4742                "factors": None,
4743                "volatility": None,
4744            }
4745            if "performanceGrid" in cmp:
4746                grid = cmp["performanceGrid"]
4747                grid_df = pd.DataFrame(grid["values"], columns=grid["headerRow"])
4748                res["performanceGrid"] = grid_df
4749            if "performance" in cmp:
4750                perf_df = pd.DataFrame(cmp["performance"]).set_index("date")
4751                perf_df.index = pd.to_datetime(perf_df.index)
4752                res["performance"] = perf_df
4753            if "volatility" in cmp:
4754                vol_df = pd.DataFrame(cmp["volatility"]).set_index("date")
4755                vol_df.index = pd.to_datetime(vol_df.index)
4756                res["volatility"] = vol_df
4757            if "factors" in cmp:
4758                factors_df = pd.DataFrame(cmp["factors"]).set_index("date")
4759                factors_df.index = pd.to_datetime(factors_df.index)
4760                res["factors"] = factors_df
4761            comparisons[pf] = res
4762        return comparisons
4763
4764    def get_signal_strength(self, model_id: str, portfolio_id: str) -> pd.DataFrame:
4765        url = self.base_uri + f"/api/analysis/signal_strength/{model_id}/{portfolio_id}"
4766        headers = {"Authorization": "ApiKey " + self.api_key}
4767
4768        logger.info(f"Retrieving portfolio signals for {model_id=}, {portfolio_id=}")
4769
4770        # Response format is a json object with a "header_row" key for column
4771        # names, and then a nested list of data.
4772        resp = requests.get(url, headers=headers, **self._request_params)
4773        self._check_ok_or_err_with_msg(
4774            resp, f"Failed to get portfolio signals for {model_id=}, {portfolio_id=}"
4775        )
4776
4777        data = resp.json()
4778
4779        df = pd.DataFrame(data=data["data"], columns=data["header_row"])
4780        df["Date"] = pd.to_datetime(df["Date"])
4781        df = df.set_index("Date")
4782        return df.astype(float)
4783
4784    def get_rolling_signal_strength(self, model_id: str, portfolio_id: str) -> pd.DataFrame:
4785        url = self.base_uri + f"/api/analysis/signal_strength_rolling/{model_id}/{portfolio_id}"
4786        headers = {"Authorization": "ApiKey " + self.api_key}
4787
4788        logger.info(f"Retrieving rolling portfolio signals for {model_id=}, {portfolio_id=}")
4789
4790        # Response format is a json object with a "header_row" key for column
4791        # names, and then a nested list of data.
4792        resp = requests.get(url, headers=headers, **self._request_params)
4793        self._check_ok_or_err_with_msg(
4794            resp, f"Failed to get rolling portfolio signals for {model_id=}, {portfolio_id=}"
4795        )
4796
4797        data = resp.json()
4798
4799        df = pd.DataFrame(data=data["data"], columns=data["header_row"])
4800        df["Date"] = pd.to_datetime(df["Date"])
4801        df = df.set_index("Date")
4802        return df.astype(float)
4803
4804    def get_portfolio_quantiles(
4805        self,
4806        model_id: str,
4807        portfolio_id: str,
4808        id_type: Literal["TICKER", "ISIN"] = "TICKER",
4809    ):
4810        headers = {"Authorization": "ApiKey " + self.api_key, "Content-Type": "application/json"}
4811        date = datetime.date.today().strftime("%Y-%m-%d")
4812
4813        payload = {
4814            "model_id": model_id,
4815            "portfolio_id": portfolio_id,
4816            "fields": ["quantile"],
4817            "min_date": date,
4818            "max_date": date,
4819            "return_format": "json",
4820        }
4821        # TODO: Later change this URI to not use the watchlist prefix. It is misnamed.
4822        url = f"{self.base_uri}{WATCHLIST_ROUTE_PREFIX}{DAL_PA_ROUTE}/get-data/"
4823
4824        res: requests.Response = requests.post(
4825            url, json=payload, headers=headers, **self._request_params
4826        )
4827        self._check_ok_or_err_with_msg(res, "Unable to get quantile data")
4828
4829        resp: Dict = res.json()
4830        quantile_index = resp["field_map"]["Quantile"]
4831        quantile_data = [[c[quantile_index] for c in r] for r in resp["data"]]
4832        date_cols = pd.to_datetime(resp["columns"])
4833
4834        # Need to map gbi id's to isins or tickers
4835        gbi_ids = [int(i) for i in resp["rows"]]
4836        security_info = self._get_security_info(gbi_ids)
4837
4838        # We now have security data, go through and create a map from internal
4839        # gbi id to client facing identifier
4840        id_key = "isin" if id_type == "ISIN" else "symbol"
4841        gbi_identifier_map = {
4842            sec["gbiId"]: sec[id_key] for sec in security_info["data"]["securities"]
4843        }
4844
4845        df = pd.DataFrame(quantile_data, index=gbi_ids, columns=date_cols).transpose()
4846        df = df.rename(columns=gbi_identifier_map)
4847        return df
4848
4849    def get_similar_stocks(
4850        self,
4851        model_id: str,
4852        portfolio_id: str,
4853        symbol_list: List[str],
4854        date: BoostedDate,
4855        identifier_type: Literal["TICKER", "ISIN"],
4856        preferred_country: Optional[str] = None,
4857        preferred_currency: Optional[str] = None,
4858    ) -> pd.DataFrame:
4859        date_str = convert_date(date).strftime("%Y-%m-%d")
4860
4861        sec_data = self.getGbiIdFromIdentCountryCurrencyDate(
4862            ident_country_currency_dates=[
4863                DateIdentCountryCurrency(
4864                    date=datetime.date.today().isoformat(),
4865                    identifier=s,
4866                    id_type=(
4867                        ColumnSubRole.SYMBOL if identifier_type == "TICKER" else ColumnSubRole.ISIN
4868                    ),
4869                    country=preferred_country,
4870                    currency=preferred_currency,
4871                )
4872                for s in symbol_list
4873            ]
4874        )
4875
4876        gbi_id_ident_map: Dict[int, str] = {}
4877        for sec in sec_data:
4878            ident = sec.ticker if identifier_type == "TICKER" else sec.isin_info.identifier
4879            gbi_id_ident_map[sec.gbi_id] = ident
4880        gbi_ids = list(gbi_id_ident_map.keys())
4881
4882        qry = """
4883          query GetSimilarStocks(
4884            $modelId: ID!
4885            $portfolioId: ID!
4886            $gbiIds: [Int]!
4887            $startDate: String!
4888            $endDate: String!
4889            $includeCorrelation: Boolean
4890          ) {
4891            similarStocks(
4892              modelId: $modelId,
4893              portfolioId: $portfolioId,
4894              gbiIds: $gbiIds,
4895              startDate: $startDate,
4896              endDate: $endDate,
4897              includeCorrelation: $includeCorrelation
4898            ) {
4899              gbiId
4900              overallSimilarityScore
4901              priceSimilarityScore
4902              factorSimilarityScore
4903              correlation
4904            }
4905          }
4906        """
4907        variables = {
4908            "startDate": date_str,
4909            "endDate": date_str,
4910            "modelId": model_id,
4911            "portfolioId": portfolio_id,
4912            "gbiIds": gbi_ids,
4913            "includeCorrelation": True,
4914        }
4915
4916        resp = self._get_graphql(
4917            qry, variables=variables, error_msg_prefix="Failed to get similar stocks result: "
4918        )
4919        df = pd.DataFrame(resp["data"]["similarStocks"])
4920
4921        # Now that we have the rest of the securities in the portfolio, we need
4922        # to map them back to the correct identifiers
4923        all_gbi_ids = df["gbiId"].tolist()
4924        sec_info = self._get_security_info(all_gbi_ids)
4925        for s in sec_info["data"]["securities"]:
4926            ident = s["symbol"] if identifier_type == "TICKER" else s["isin"]
4927            gbi_id_ident_map[s["gbiId"]] = ident
4928        df["identifier"] = df["gbiId"].map(gbi_id_ident_map)
4929        df = df.set_index("identifier")
4930        return df.drop("gbiId", axis=1)
4931
4932    def get_portfolio_trades(
4933        self,
4934        model_id: str,
4935        portfolio_id: str,
4936        start_date: Optional[BoostedDate] = None,
4937        end_date: Optional[BoostedDate] = None,
4938    ) -> pd.DataFrame:
4939        if not end_date:
4940            end_date = datetime.date.today()
4941        end_date = convert_date(end_date)
4942
4943        if not start_date:
4944            # default to a year of data
4945            start_date = end_date - datetime.timedelta(days=365)
4946        start_date = convert_date(start_date)
4947
4948        start_date_str = start_date.strftime("%Y-%m-%d")
4949        end_date_str = end_date.strftime("%Y-%m-%d")
4950
4951        if end_date - start_date > datetime.timedelta(days=365 * 7):
4952            raise BoostedAPIException(
4953                f"Date range ({start_date_str}, {end_date_str}) too large, max 7 years"
4954            )
4955
4956        url = f"{self.base_uri}{ROUTE_PREFIX}{DAL_PA_ROUTE}/get-data/"
4957        headers = {"Authorization": "ApiKey " + self.api_key, "Content-Type": "application/json"}
4958        payload = {
4959            "model_id": model_id,
4960            "portfolio_id": portfolio_id,
4961            "fields": ["price", "shares_traded", "shares_owned"],
4962            "min_date": start_date_str,
4963            "max_date": end_date_str,
4964            "return_format": "json",
4965        }
4966
4967        res: requests.Response = requests.post(
4968            url, json=payload, headers=headers, **self._request_params
4969        )
4970        self._check_ok_or_err_with_msg(res, "Unable to get portfolio trades data")
4971
4972        data = res.json()
4973        gbi_ids = [int(ident) for ident in data["rows"]]
4974
4975        # need both isin and ticker to distinguish between possible duplicates
4976        isin_map = {
4977            str(s["gbiId"]): s["isin"]
4978            for s in self._get_security_info(gbi_ids)["data"]["securities"]
4979        }
4980        ticker_map = {
4981            str(s["gbiId"]): s["symbol"]
4982            for s in self._get_security_info(gbi_ids)["data"]["securities"]
4983        }
4984
4985        # construct individual dataframes for each security, then join them together
4986        dfs: List[pd.DataFrame] = []
4987        full_data = data["data"]
4988        for i, gbi_id in enumerate(data["rows"]):
4989            df = pd.DataFrame(
4990                index=pd.to_datetime(data["columns"]), columns=data["fields"], data=full_data[i]
4991            )
4992            # drop rows where no shares are owned or traded
4993            df.drop(
4994                df.loc[((df["shares_owned"] == 0.0) & (df["shares_traded"] == 0.0))].index,
4995                inplace=True,
4996            )
4997            df["isin"] = isin_map[gbi_id]
4998            df["ticker"] = ticker_map[gbi_id]
4999            dfs.append(df)
5000
5001        full_df = pd.concat(dfs)
5002        full_df["date"] = full_df.index
5003        full_df.sort_index(inplace=True)
5004        full_df.reset_index(drop=True, inplace=True)
5005
5006        # reorder the columns to match the spreadsheet
5007        columns = ["isin", "ticker", "date", *data["fields"]]
5008        return full_df[columns]
5009
5010    def get_ideas(
5011        self,
5012        model_id: str,
5013        portfolio_id: str,
5014        investment_horizon: Literal["1M", "3M", "1Y"] = "1M",
5015        delta_horizon: str = "1M",
5016    ):
5017        if investment_horizon not in ("1M", "3M", "1Y"):
5018            raise BoostedAPIException(f"Invalid investment horizon: {investment_horizon}")
5019
5020        if delta_horizon not in ("1W", "1M", "3M", "6M", "9M", "1Y"):
5021            raise BoostedAPIException(f"Invalid delta horizon: {delta_horizon}")
5022
5023        # First compute dates based on the delta horizon. "0D" is the latest rebalance.
5024        try:
5025            dates = self._get_portfolio_rebalance_from_periods(
5026                portfolio_id=portfolio_id, rel_periods=["0D", delta_horizon]
5027            )
5028        except Exception:
5029            raise BoostedAPIException(
5030                f"Portfolio {portfolio_id} does not exist or you do not have permission to view it."
5031            )
5032        end_date = dates[0].strftime("%Y-%m-%d")
5033        start_date = dates[1].strftime("%Y-%m-%d")
5034
5035        resp = self._get_graphql(
5036            graphql_queries.GET_IDEAS_QUERY,
5037            variables={
5038                "modelId": model_id,
5039                "portfolioId": portfolio_id,
5040                "horizon": investment_horizon,
5041                "deltaHorizon": delta_horizon,
5042                "startDate": start_date,
5043                "endDate": end_date,
5044                # Note: market data date is needed to fetch market cap.
5045                # we don't fetch that data from this endpoint so we stub
5046                # out the mandatory parameter with the end date requested
5047                "marketDataDate": end_date,
5048            },
5049            error_msg_prefix="Failed to get ideas: ",
5050        )
5051        # rows is a list of dicts like:
5052        # {
5053        #   "category": "Strong Sell",
5054        #   "dividendYield": 0.0,
5055        #   "reason": "Boosted Insights has given this stock...",
5056        #   "rating": 0.458167,
5057        #   "ratingDelta": 0.438087,
5058        #   "risk": {
5059        #     "text": "high"
5060        #   },
5061        #   "security": {
5062        #     "symbol": "BA"
5063        #   }
5064        # }
5065        try:
5066            rows = resp["data"]["recommendations"]["recommendations"]
5067            data = [
5068                {
5069                    "symbol": r["security"]["symbol"],
5070                    "recommendation": r["category"],
5071                    "rating": r["rating"],
5072                    "rating_delta": r["ratingDelta"],
5073                    "dividend_yield": r["dividendYield"],
5074                    "predicted_excess_return_1m": r["ER"]["oneMonth"],
5075                    "predicted_excess_return_3m": r["ER"]["threeMonth"],
5076                    "predicted_excess_return_1y": r["ER"]["oneYear"],
5077                    "risk": r["risk"]["text"],
5078                    "reward": r["reward"]["text"],
5079                    "reason": r["reason"],
5080                }
5081                for r in rows
5082            ]
5083            df = pd.DataFrame(data)
5084            df.set_index("symbol", inplace=True)
5085        except Exception:
5086            # Don't show old exception info to client
5087            raise BoostedAPIException(
5088                "No recommendations found, try selecting another horizon."
5089            ) from None
5090
5091        return df
5092
5093    def get_stock_recommendations(
5094        self,
5095        model_id: str,
5096        portfolio_id: str,
5097        symbols: Optional[List[str]] = None,
5098        investment_horizon: Literal["1M", "3M", "1Y"] = "1M",
5099    ) -> pd.DataFrame:
5100        model_stocks = self._get_model_stocks(model_id)
5101
5102        symbols_to_gbiids = {s.ticker: s.gbi_id for s in model_stocks}
5103        gbi_ids_to_symbols = {s.gbi_id: s.ticker for s in model_stocks}
5104
5105        variables: Dict[str, Any] = {
5106            "strategyId": portfolio_id,
5107        }
5108        if symbols:
5109            variables["gbiIds"] = [
5110                symbols_to_gbiids.get(symbol) for symbol in symbols if symbols_to_gbiids.get(symbol)
5111            ]
5112        try:
5113            recs = self._get_graphql(
5114                graphql_queries.MULTI_STOCK_RECOMMENDATION_QUERY,
5115                variables=variables,
5116                log_error=False,
5117            )["data"]["currentRecommendationsFull"]
5118        except BoostedAPIException:
5119            raise BoostedAPIException(f"Error getting recommendations for strategy {portfolio_id}")
5120
5121        data = []
5122        recommendation_key = f"recommendation{investment_horizon}"
5123        for rec in recs:
5124            # Keys to rec are:
5125            # ['ER', 'rewardCategories', 'riskCategories', 'reasons',
5126            #  'recommendation', 'rewardCategory', 'riskCategory']
5127            # need to flatten these out and add to a DF
5128            rec_data = rec[recommendation_key]
5129            reasons_dict = {r["type"]: r["text"] for r in rec_data["reasons"]}
5130            row = {
5131                "symbol": gbi_ids_to_symbols[rec["gbiId"]],
5132                "recommendation": rec_data["currentCategory"],
5133                "predicted_excess_return_1m": rec_data["ER"]["oneMonth"],
5134                "predicted_excess_return_3m": rec_data["ER"]["threeMonth"],
5135                "predicted_excess_return_1y": rec_data["ER"]["oneYear"],
5136                "risk": rec_data["risk"]["text"],
5137                "reward": rec_data["reward"]["text"],
5138                "reasons": reasons_dict,
5139            }
5140
5141            data.append(row)
5142        df = pd.DataFrame(data)
5143        df.set_index("symbol", inplace=True)
5144        return df
5145
5146    # NOTE: this could be easily expanded to the entire stockRecommendation
5147    # entity, but that only includes all horizons' excess returns and risk/reward
5148    # which we already get from getIdeas
5149    def get_stock_recommendation_reasons(
5150        self,
5151        model_id: str,
5152        portfolio_id: str,
5153        investment_horizon: Literal["1M", "3M", "1Y"] = "1M",
5154        symbols: Optional[List[str]] = None,
5155    ) -> Dict[str, Optional[List[str]]]:
5156        if investment_horizon not in ("1M", "3M", "1Y"):
5157            raise BoostedAPIException(f"Invalid investment horizon: {investment_horizon}")
5158
5159        # "0D" is the latest rebalance - its all we have in terms of recs
5160        dates = self._get_portfolio_rebalance_from_periods(
5161            portfolio_id=portfolio_id, rel_periods=["0D"]
5162        )
5163        date = dates[0].strftime("%Y-%m-%d")
5164
5165        model_stocks = self._get_model_stocks(model_id)
5166
5167        symbols_to_gbiids = {s.ticker: s.gbi_id for s in model_stocks}
5168        if symbols is None:  # potentially iterate through all holdings
5169            symbols = symbols_to_gbiids.keys()  # type: ignore
5170
5171        reasons: Dict[str, Optional[List[str]]] = {}
5172        for sym in symbols:
5173            # it's possible that a passed symbol was not actually a portfolio holding
5174            try:
5175                gbi_id = symbols_to_gbiids[sym]
5176            except KeyError:
5177                logger.warning(f"Symbol={sym} not found for in universe on {date=}")
5178                reasons[sym] = None
5179                continue
5180
5181            try:
5182                recs = self._get_graphql(
5183                    graphql_queries.STOCK_RECOMMENDATION_QUERY,
5184                    variables={
5185                        "modelId": model_id,
5186                        "portfolioId": portfolio_id,
5187                        "horizon": investment_horizon,
5188                        "gbiId": gbi_id,
5189                        "date": date,
5190                    },
5191                    log_error=False,
5192                )
5193                reasons[sym] = [
5194                    reason["text"] for reason in recs["data"]["stockRecommendation"]["reasons"]
5195                ]
5196            except BoostedAPIException:
5197                logger.warning(f"No recommendation for: {sym}, skipping...")
5198        return reasons
5199
5200    def get_stock_mapping_alternatives(
5201        self,
5202        isin: Optional[str] = None,
5203        symbol: Optional[str] = None,
5204        country: Optional[str] = None,
5205        currency: Optional[str] = None,
5206        asof_date: Optional[BoostedDate] = None,
5207    ) -> Dict:
5208        """
5209        Return the stock mapping for the given criteria,
5210        also suggestions for alternate matches,
5211        if the mapping is not what is wanted
5212
5213
5214            Parameters [One of either ISIN or SYMBOL must be provided]
5215            ----------
5216            isin: Optional[str]
5217                search by ISIN
5218            symbol: Optional[str]
5219                search by Ticker Symbol
5220            country: Optional[str]
5221                Additionally filter by country code - ex: None, "ANY", "p_USA", "CAN"
5222            currency: Optional[str]
5223                Additionally filter by currency code - ex: None, "ANY", "p_USD", "CAD"
5224            asof_date: Optional[date]
5225                as of which date to perform the search, default is today()
5226
5227            Note: country/currency filter starting with "p_" indicates
5228                  only a soft preference but allows other matches
5229
5230        Returns
5231        -------
5232        Dictionary Representing this 'MapSecurityResponse' structure:
5233
5234        class MapSecurityResponse():
5235            stock_mapping: Optional[SecurityInfo]
5236               The mapping we would perform given your inputs
5237
5238            alternatives: Optional[List[SecurityInfo]]
5239               Alternative suggestions based on your input
5240
5241            error: Optional[str]
5242
5243        class SecurityInfo():
5244            gbi_id: int
5245            isin: str
5246            symbol: Optional[str]
5247            country: str
5248            currency: str
5249            name: str
5250            from_date: date
5251            to_date: date
5252            is_primary_trading_item: bool
5253
5254        """
5255
5256        url = f"{self.base_uri}/api/stock-mapping/alternatives"
5257        headers = {"Authorization": "ApiKey " + self.api_key, "Content-Type": "application/json"}
5258        req_json: Dict = {
5259            "isin": isin,
5260            "symbol": symbol,
5261            "countryPreference": country,
5262            "currencyPreference": currency,
5263        }
5264
5265        if asof_date:
5266            req_json["date"] = convert_date(asof_date).isoformat()
5267
5268        res = requests.post(url, json=req_json, headers=headers, **self._request_params)
5269
5270        if not res.ok:
5271            error_msg = self._try_extract_error_code(res)
5272            logger.error(error_msg)
5273            raise BoostedAPIException(f"Failed to get user watchlists: {error_msg}")
5274
5275        data = res.json()
5276        return data
5277
5278    def get_pros_cons_for_stocks(
5279        self,
5280        model_id: Optional[str] = None,
5281        symbols: Optional[List[str]] = None,
5282        preferred_country: Optional[str] = None,
5283        preferred_currency: Optional[str] = None,
5284    ) -> Dict[str, Dict[str, List]]:
5285        if symbols:
5286            ident_objs = [
5287                DateIdentCountryCurrency(
5288                    date=datetime.date.today().strftime("%Y-%m-%d"),
5289                    identifier=symbol,
5290                    country=preferred_country,
5291                    currency=preferred_currency,
5292                    id_type=ColumnSubRole.SYMBOL,
5293                )
5294                for symbol in symbols
5295            ]
5296            sec_objs = self.getGbiIdFromIdentCountryCurrencyDate(
5297                ident_country_currency_dates=ident_objs
5298            )
5299            gbi_id_ticker_map = {sec.gbi_id: sec.ticker for sec in sec_objs if sec}
5300        elif model_id:
5301            gbi_id_ticker_map = {
5302                sec.gbi_id: sec.ticker for sec in self._get_model_stocks(model_id=model_id)
5303            }
5304        gbi_id_pros_cons_map = {}
5305        gbi_ids = list(gbi_id_ticker_map.keys())
5306        data = self._get_graphql(
5307            query=graphql_queries.GET_PROS_CONS_QUERY,
5308            variables={"gbiIds": gbi_ids},
5309            error_msg_prefix="Failed to get pros/cons:",
5310        )
5311        gbi_id_pros_cons_map = {
5312            row["gbiId"]: {"pros": row["pros"], "cons": row["cons"]}
5313            for row in data["data"]["bulkSecurityProsCons"]
5314        }
5315
5316        return {
5317            gbi_id_ticker_map[gbi_id]: pros_cons
5318            for gbi_id, pros_cons in gbi_id_pros_cons_map.items()
5319        }
5320
5321    def generate_theme(self, theme_name: str, stock_universes: List[ThemeUniverse]) -> str:
5322        # First get universe name and id mappings
5323        try:
5324            resp = self._get_graphql(
5325                query=graphql_queries.GET_MARKET_TRENDS_UNIVERSES_QUERY, variables={}
5326            )
5327            data = resp["data"]["getMarketTrendsUniverses"]
5328        except Exception:
5329            raise BoostedAPIException(f"Failed to load market trends universes mapping")
5330
5331        universe_name_to_id = {u["name"]: u["id"] for u in data}
5332        universe_ids = [universe_name_to_id[u.value] for u in stock_universes]
5333        try:
5334            resp = self._get_graphql(
5335                query=graphql_queries.GENERATE_THEME_QUERY,
5336                variables={"input": {"themeName": theme_name, "universeIds": universe_ids}},
5337            )
5338            data = resp["data"]["generateTheme"]
5339        except Exception:
5340            raise BoostedAPIException(f"Failed to generate theme: {theme_name}")
5341
5342        if not data["success"]:
5343            raise BoostedAPIException(f"Failed to generate theme: {theme_name}")
5344
5345        logger.info(
5346            f"Successfully generated theme: {theme_name}. The theme ID is {data['themeId']}"
5347        )
5348        return data["themeId"]
5349
5350    def _get_stock_universe_id(self, universe: ThemeUniverse) -> str:
5351        try:
5352            resp = self._get_graphql(
5353                query=graphql_queries.GET_MARKET_TRENDS_UNIVERSES_QUERY, variables={}
5354            )
5355            data = resp["data"]["getMarketTrendsUniverses"]
5356        except Exception:
5357            raise BoostedAPIException(f"Failed to load market trends universes mapping")
5358
5359        for u in data:
5360            if u["name"] == universe.value:
5361                universe_id = u["id"]
5362                return universe_id
5363
5364        raise BoostedAPIException(f"Failed to find universe: {universe.value}")
5365
5366    def get_themes_for_stock_universe(
5367        self,
5368        stock_universe: ThemeUniverse,
5369        start_date: Optional[BoostedDate] = None,
5370        end_date: Optional[BoostedDate] = None,
5371        language: Optional[Union[str, Language]] = None,
5372    ) -> List[Dict]:
5373        """Get all themes data for a particular stock universe
5374        (start_date, end_date) are used to calculate the theme importance for ranking purpose. If
5375        None, default to past 30 days
5376        Returns: A list of below dictionaries
5377        {
5378            themeId: str
5379            themeName: str
5380            themeImportance: float
5381            volatility: float
5382            positiveStockPerformance: float
5383            negativeStockPerformance: float
5384        }
5385        """
5386        translate = functools.partial(self.translate_text, language)
5387        # First get universe name and id mappings
5388        universe_id = self._get_stock_universe_id(stock_universe)
5389
5390        start_date_iso, end_date_iso = get_valid_iso_dates(start_date, end_date)
5391
5392        try:
5393            resp = self._get_graphql(
5394                query=graphql_queries.GET_THEMES,
5395                variables={
5396                    "type": "UNIVERSE",
5397                    "id": universe_id,
5398                    "startDate": start_date_iso,
5399                    "endDate": end_date_iso,
5400                    "deltaHorizon": "",  # not needed here
5401                },
5402            )
5403            data = resp["data"]["themes"]
5404        except Exception:
5405            raise BoostedAPIException(
5406                f"Failed to get themes for stock universe: {stock_universe.name}"
5407            )
5408
5409        for theme_data in data:
5410            theme_data["themeName"] = translate(theme_data["themeName"])
5411        return data
5412
5413    def get_themes_for_stock(
5414        self,
5415        isin: str,
5416        currency: Optional[str] = None,
5417        country: Optional[str] = None,
5418        start_date: Optional[BoostedDate] = None,
5419        end_date: Optional[BoostedDate] = None,
5420        language: Optional[Union[str, Language]] = None,
5421    ) -> List[Dict]:
5422        """Get all themes data for a particular stock
5423        (ISIN, currency, country) compose a unique identifier for a stock for us to map to GBI ID
5424        (start_date, end_date) are used to calculate the theme importance for ranking purpose. If
5425        None, default to past 30 days
5426
5427        Returns
5428        A list of below dictionaries
5429        {
5430            themeId: str
5431            themeName: str
5432            importanceScore: float
5433            similarityScore: float
5434            positiveThemeRelation: bool
5435            reason: String
5436        }
5437        """
5438        translate = functools.partial(self.translate_text, language)
5439        security_info = self.get_stock_mapping_alternatives(
5440            isin, country=country, currency=currency
5441        )
5442        gbi_id = security_info["stock_mapping"]["gbi_id"]
5443
5444        if (start_date and not end_date) or (end_date and not start_date):
5445            raise BoostedAPIException("Must provide both start and end dates or neither")
5446        elif not end_date and not start_date:
5447            end_date = datetime.date.today()
5448            start_date = end_date - datetime.timedelta(days=30)
5449            end_date = end_date.isoformat()
5450            start_date = start_date.isoformat()
5451        else:
5452            if isinstance(start_date, datetime.date):
5453                start_date = start_date.isoformat()
5454            if isinstance(end_date, datetime.date):
5455                end_date = end_date.isoformat()
5456
5457        try:
5458            resp = self._get_graphql(
5459                query=graphql_queries.GET_THEMES_FOR_STOCK_WITH_REASONS,
5460                variables={"gbiId": gbi_id, "startDate": start_date, "endDate": end_date},
5461            )
5462            data = resp["data"]["themesForStockWithReasons"]
5463        except Exception:
5464            raise BoostedAPIException(f"Failed to get themes for stock: {isin}")
5465
5466        for item in data:
5467            item["themeName"] = translate(item["themeName"])
5468            item["reason"] = translate(item["reason"])
5469        return data
5470
5471    def get_stock_news(
5472        self,
5473        time_horizon: NewsHorizon,
5474        isin: str,
5475        currency: Optional[str] = None,
5476        country: Optional[str] = None,
5477        language: Optional[Union[str, Language]] = None,
5478    ) -> Dict:
5479        """
5480        The API to get a stock's news summary for a given time horizon, the topics summarized by
5481        these news and the corresponding news to these topics
5482        Returns
5483        -------
5484        A nested dictionary in the following format:
5485        {
5486            summary: str
5487            topics: [
5488                {
5489                    topicId: str
5490                    topicLabel: str
5491                    topicDescription: str
5492                    topicPolarity: str
5493                    newsItems: [
5494                        {
5495                            newsId: str
5496                            headline: str
5497                            url: str
5498                            summary: str
5499                            source: str
5500                            publishedAt: str
5501                        }
5502                    ]
5503                }
5504            ]
5505            other_news_count: int
5506        }
5507        """
5508        translate = functools.partial(self.translate_text, language)
5509        security_info = self.get_stock_mapping_alternatives(
5510            isin, country=country, currency=currency
5511        )
5512        gbi_id = security_info["stock_mapping"]["gbi_id"]
5513
5514        try:
5515            resp = self._get_graphql(
5516                query=graphql_queries.GET_STOCK_NEWS_QUERY,
5517                variables={"gbiId": gbi_id, "deltaHorizon": time_horizon.value},
5518            )
5519            data = resp["data"]
5520        except Exception:
5521            raise BoostedAPIException(f"Failed to get themes for stock: {isin}")
5522
5523        outputs: Dict[str, Any] = {}
5524        outputs["summary"] = translate(data["getStockNewsSummary"]["summary"])
5525        # Return the top 10 topics
5526        outputs["topics"] = data["getStockNewsTopics"][:10]
5527
5528        for topic in outputs["topics"]:
5529            topic["topicLabel"] = translate(topic["topicLabel"])
5530            topic["topicDescription"] = translate(topic["topicDescription"])
5531
5532        other_news_count = 0
5533        for source_count in data["getStockNewsSummary"]["sourceCounts"]:
5534            other_news_count += source_count["count"]
5535
5536        for topic in outputs["topics"]:
5537            other_news_count -= len(topic["newsItems"])
5538
5539        outputs["other_news_count"] = other_news_count
5540
5541        return outputs
5542
5543    def get_theme_details(
5544        self,
5545        theme_id: str,
5546        universe: ThemeUniverse,
5547        language: Optional[Union[str, Language]] = None,
5548    ) -> Dict[str, Any]:
5549        translate = functools.partial(self.translate_text, language)
5550        universe_id = self._get_stock_universe_id(universe)
5551        date = datetime.date.today()
5552        prev_date = date - datetime.timedelta(days=30)
5553        result = self._get_graphql(
5554            query=graphql_queries.GET_THEME_DEEPDIVE_DETAILS,
5555            variables={
5556                "deltaHorizon": "1W",
5557                "startDate": prev_date.strftime("%Y-%m-%d"),
5558                "endDate": date.strftime("%Y-%m-%d"),
5559                "id": universe_id,
5560                "themeId": theme_id,
5561                "type": "UNIVERSE",
5562            },
5563            error_msg_prefix="Failed to get theme details",
5564        )["data"]["marketThemes"]
5565
5566        gbi_id_stock_data_map: Dict[int, Dict] = {}
5567
5568        stocks = []
5569        for stock_info in result["stockInfos"]:
5570            gbi_id_stock_data_map[stock_info["gbiId"]] = stock_info["security"]
5571            stocks.append(
5572                {
5573                    "isin": stock_info["security"]["isin"],
5574                    "name": stock_info["security"]["name"],
5575                    "reason": translate(stock_info["polarityReasonScores"]["reason"]),
5576                    "positive_theme_relation": stock_info["polarityReasonScores"][
5577                        "positiveThemeRelation"
5578                    ],
5579                    "theme_stock_impact_score": stock_info["polarityReasonScores"][
5580                        "similarityScore"
5581                    ],
5582                }
5583            )
5584
5585        impacts = []
5586        for impact in result["impactInfos"]:
5587            articles = [
5588                {
5589                    "title": newsitem["headline"],
5590                    "url": newsitem["url"],
5591                    "source": newsitem["source"],
5592                    "publish_date": newsitem["publishedAt"],
5593                }
5594                for newsitem in impact["newsItems"]
5595            ]
5596
5597            impact_stocks = []
5598            for impact_stock_data in impact["stocks"]:
5599                stock_metadata = gbi_id_stock_data_map[impact_stock_data["gbiId"]]
5600                impact_stocks.append(
5601                    {
5602                        "isin": stock_metadata["isin"],
5603                        "name": stock_metadata["name"],
5604                        "positive_impact_relation": impact_stock_data["positiveThemeRelation"],
5605                    }
5606                )
5607
5608            impact_dict = {
5609                "impact_name": translate(impact["impactName"]),
5610                "impact_description": translate(impact["impactDescription"]),
5611                "impact_score": impact["impactScore"],
5612                "articles": articles,
5613                "impact_stocks": impact_stocks,
5614            }
5615            impacts.append(impact_dict)
5616
5617        developments = []
5618        for dev in result["themeDevelopments"]:
5619            developments.append(
5620                {
5621                    "name": dev["label"],
5622                    "article_count": dev["articleCount"],
5623                    "date": parser.parse(dev["date"]).date(),
5624                    "description": dev["description"],
5625                    "is_major_development": dev["isMajorDevelopment"],
5626                    "sentiment": dev["sentiment"],
5627                    "news": [
5628                        {
5629                            "headline": entry["headline"],
5630                            "published_at": parser.parse(entry["publishedAt"]),
5631                            "source": entry["source"],
5632                            "url": entry["url"],
5633                        }
5634                        for entry in dev["news"]
5635                    ],
5636                }
5637            )
5638
5639        developments = sorted(developments, key=lambda d: d["date"], reverse=True)
5640
5641        output = {
5642            "theme_name": translate(result["themeName"]),
5643            "theme_summary": translate(result["themeDescription"]),
5644            "impacts": impacts,
5645            "stocks": stocks,
5646            "developments": developments,
5647        }
5648        return output
5649
5650    def get_all_theme_metadata(
5651        self, language: Optional[Union[str, Language]] = None
5652    ) -> List[Dict[str, Any]]:
5653        translate = functools.partial(self.translate_text, language)
5654        result = self._get_graphql(
5655            graphql_queries.GET_ALL_THEMES,
5656            variables={"universeIds": None},
5657            error_msg_prefix="Failed to fetch all themes metadata",
5658        )
5659
5660        try:
5661            resp = self._get_graphql(
5662                query=graphql_queries.GET_MARKET_TRENDS_UNIVERSES_QUERY, variables={}
5663            )
5664            data = resp["data"]["getMarketTrendsUniverses"]
5665        except Exception:
5666            raise BoostedAPIException(f"Failed to load market trends universes mapping")
5667        universe_id_to_name = {u["id"]: u["name"] for u in data}
5668
5669        outputs = []
5670        for theme in result["data"]["getAllThemesForUser"]:
5671            # map universe ID to universe ticker
5672            universe_tickers = []
5673            for universe_id in theme["universeIds"]:
5674                if universe_id in universe_id_to_name:  # don't support unlisted universes - skip
5675                    universe_name = universe_id_to_name[universe_id]
5676                    ticker = ThemeUniverse.get_ticker_from_name(universe_name)
5677                    if ticker:
5678                        universe_tickers.append(ticker)
5679
5680            outputs.append(
5681                {
5682                    "theme_id": theme["themeId"],
5683                    "theme_name": translate(theme["themeName"]),
5684                    "universes": universe_tickers,
5685                }
5686            )
5687
5688        return outputs
5689
5690    def get_earnings_impacting_security(
5691        self,
5692        isin: str,
5693        currency: Optional[str] = None,
5694        country: Optional[str] = None,
5695        language: Optional[Union[str, Language]] = None,
5696    ) -> List[Dict[str, Any]]:
5697        translate = functools.partial(self.translate_text, language)
5698        date = datetime.date.today().strftime("%Y-%m-%d")
5699        company_data = self.getGbiIdFromIdentCountryCurrencyDate(
5700            ident_country_currency_dates=[
5701                DateIdentCountryCurrency(
5702                    date=date, identifier=isin, country=country, currency=currency
5703                )
5704            ]
5705        )
5706        try:
5707            gbi_id = company_data[0].gbi_id
5708        except Exception:
5709            raise BoostedAPIException(f"ISIN {isin} not found")
5710
5711        result = self._get_graphql(
5712            graphql_queries.EARNINGS_IMPACTS_CALENDAR_FOR_STOCK,
5713            variables={"date": date, "days": 180, "gbiId": gbi_id},
5714            error_msg_prefix="Failed to fetch earnings impacts data for stock",
5715        )
5716        earnings_events = result["data"]["earningsCalendarForStock"]
5717        output_events = []
5718        for event in earnings_events:
5719            if not event["impactedCompanies"]:
5720                continue
5721            fixed_event = {
5722                "event_date": event["eventDate"],
5723                "company_name": event["security"]["name"],
5724                "symbol": event["security"]["symbol"],
5725                "isin": event["security"]["isin"],
5726                "impact_reason": translate(event["impactedCompanies"][0]["reason"]),
5727            }
5728            output_events.append(fixed_event)
5729
5730        return output_events
5731
5732    def get_earnings_insights_for_stocks(
5733        self, isin: str, currency: Optional[str] = None, country: Optional[str] = None
5734    ) -> Dict[str, Any]:
5735        date = datetime.date.today().strftime("%Y-%m-%d")
5736        company_data = self.getGbiIdFromIdentCountryCurrencyDate(
5737            ident_country_currency_dates=[
5738                DateIdentCountryCurrency(
5739                    date=date, identifier=isin, country=country, currency=currency
5740                )
5741            ]
5742        )
5743        gbi_id_isin_map = {
5744            company.gbi_id: company.isin_info.identifier
5745            for company in company_data
5746            if company is not None
5747        }
5748        try:
5749            resp = self._get_graphql(
5750                query=graphql_queries.GET_EARNINGS_INSIGHTS_SUMMARIES,
5751                variables={"gbiIds": list(gbi_id_isin_map.keys())},
5752            )
5753            # list of objects with gbi id and data
5754            summaries = resp["data"]["getEarningsSummaries"]
5755            resp = self._get_graphql(
5756                query=graphql_queries.GET_EARNINGS_COMPARISONS,
5757                variables={"gbiIds": list(gbi_id_isin_map.keys())},
5758            )
5759            # list of objects with gbi id and data
5760            comparison = resp["data"]["getLatestEarningsChanges"]
5761        except Exception:
5762            raise BoostedAPIException(f"Failed to earnings insights data")
5763
5764        if not summaries:
5765            raise BoostedAPIException(
5766                (
5767                    f"Failed to find earnings insights data for {isin}"
5768                    ", please try with another security"
5769                )
5770            )
5771
5772        output: Dict[str, Any] = {}
5773        reports = sorted(summaries[0]["reports"], key=lambda r: r["date"], reverse=True)
5774        current_report = reports[0]
5775
5776        def is_aligned_formatter(acc: Tuple[List, List], cur: Dict[str, Any]):
5777            if cur["isAligned"]:
5778                acc[0].append({k: cur[k] for k in cur if k != "isAligned"})
5779            else:
5780                acc[1].append({k: cur[k] for k in cur if k != "isAligned"})
5781            return acc
5782
5783        current_report_common_remarks: Union[List[Dict[str, Any]], List]
5784        current_report_dropped_remarks: Union[List[Dict[str, Any]], List]
5785        current_report_common_remarks, current_report_dropped_remarks = functools.reduce(
5786            is_aligned_formatter, current_report["details"], ([], [])
5787        )
5788        prev_report_common_remarks: Union[List[Dict[str, Any]], List]
5789        prev_report_new_remarks: Union[List[Dict[str, Any]], List]
5790        prev_report_common_remarks, prev_report_new_remarks = functools.reduce(
5791            is_aligned_formatter, current_report["details"], ([], [])
5792        )
5793
5794        output["earnings_report"] = {
5795            "release_date": datetime.datetime.strptime(current_report["date"], "%Y-%m-%d").date(),
5796            "quarter": current_report["quarter"],
5797            "year": current_report["year"],
5798            "details": [
5799                {
5800                    "header": detail_obj["header"],
5801                    "detail": detail_obj["detail"],
5802                    "sentiment": detail_obj["sentiment"],
5803                }
5804                for detail_obj in current_report["details"]
5805            ],
5806            "call_summary": current_report["highlights"],
5807            "common_remarks": current_report_common_remarks,
5808            "dropped_remarks": current_report_dropped_remarks,
5809            "qa_summary": current_report["qaHighlights"],
5810            "qa_details": current_report["qaDetails"],
5811        }
5812        prev_report = summaries[0]["reports"][1]
5813        output["prior_earnings_report"] = {
5814            "release_date": datetime.datetime.strptime(prev_report["date"], "%Y-%m-%d").date(),
5815            "quarter": prev_report["quarter"],
5816            "year": prev_report["year"],
5817            "details": [
5818                {
5819                    "header": detail_obj["header"],
5820                    "detail": detail_obj["detail"],
5821                    "sentiment": detail_obj["sentiment"],
5822                }
5823                for detail_obj in prev_report["details"]
5824            ],
5825            "call_summary": prev_report["highlights"],
5826            "common_remarks": prev_report_common_remarks,
5827            "new_remarks": prev_report_new_remarks,
5828            "qa_summary": prev_report["qaHighlights"],
5829            "qa_details": prev_report["qaDetails"],
5830        }
5831
5832        if not comparison:
5833            output["report_comparison"] = []
5834        else:
5835            output["report_comparison"] = comparison[0]["changes"]
5836
5837        return output
5838
5839    def get_portfolio_inference_status(self, portfolio_id: str, inference_date: str) -> dict:
5840        url = f"{self.base_uri}/api/inference/status/{portfolio_id}/{inference_date}"
5841        headers = {"Authorization": "ApiKey " + self.api_key}
5842        res = requests.get(url, headers=headers)
5843
5844        if not res.ok:
5845            error_msg = self._try_extract_error_code(res)
5846            logger.error(error_msg)
5847            raise BoostedAPIException(
5848                f"Failed to get portfolio inference status, portfolio_id={portfolio_id}, "
5849                f"inference_date={inference_date}: {error_msg}"
5850            )
5851
5852        data = res.json()
5853        return data