boosted.api.api_client

   1# Copyright (C) 2020 Gradient Boosted Investments, Inc. - All Rights Reserved
   2
   3import base64
   4import csv
   5import datetime
   6import functools
   7import io
   8import itertools
   9import json
  10import logging
  11import math
  12import mimetypes
  13import os
  14import sys
  15import tempfile
  16import time
  17from datetime import timedelta
  18from typing import Any, Dict, List, Literal, Optional, Tuple, Union
  19from urllib import parse
  20
  21import boosted.api.graphql_queries as graphql_queries
  22import numpy as np
  23import pandas as pd
  24import requests
  25from boosted.api.api_type import (
  26    BoostedAPIException,
  27    BoostedDate,
  28    ChunkStatus,
  29    ColumnSubRole,
  30    DataAddType,
  31    DataSetConfig,
  32    DataSetType,
  33    DateIdentCountryCurrency,
  34    GbiIdSecurity,
  35    GbiIdTickerISIN,
  36    HedgeExperiment,
  37    HedgeExperimentDetails,
  38    HedgeExperimentScenario,
  39    Language,
  40    NewsHorizon,
  41    PortfolioSettings,
  42    Status,
  43    ThemeUniverse,
  44    hedge_experiment_type,
  45)
  46from boosted.api.api_util import (
  47    get_valid_iso_dates,
  48    infer_dataset_schema,
  49    protoCubeJsonDataToDataFrame,
  50    validate_start_and_end_dates,
  51)
  52from dateutil import parser
  53
  54logger = logging.getLogger("boosted.api.client")
  55logging.basicConfig()
  56
  57g_boosted_api_url = "https://insights.boosted.ai"
  58g_boosted_api_url_dev = "https://insights-dev.boosted.ai"
  59WATCHLIST_ROUTE_PREFIX = "/api/dal/watchlist"
  60ROUTE_PREFIX = WATCHLIST_ROUTE_PREFIX
  61DAL_WATCHLIST_ROUTE = "/api/v0/watchlist"
  62DAL_SECURITIES_ROUTE = "/api/v0/securities"
  63DAL_PA_ROUTE = "/api/v0/portfolio-analysis"
  64PORTFOLIO_GROUP_ROUTE = "/api/v0/portfolio-group"
  65
  66RISK_FACTOR = "risk-factor"
  67RISK_FACTOR_V2 = "risk-factor-v2"
  68RISK_FACTOR_COLUMNS = [
  69    "depth",
  70    "identifier",
  71    "stock_count",
  72    "volatility",
  73    "exposure",
  74    "rating",
  75    "rating_delta",
  76]
  77
  78
  79def convert_date(date: BoostedDate) -> datetime.date:
  80    if isinstance(date, str):
  81        try:
  82            return parser.parse(date).date()
  83        except Exception as e:
  84            raise BoostedAPIException(f"Unable to parse date: {str(e)}")
  85    return date
  86
  87
  88class BoostedClient:
  89    def __init__(
  90        self, api_key, override_uri=None, debug=False, proxy=None, disable_verify_ssl=False
  91    ):
  92        """
  93        Parameters
  94        ----------
  95        api_key: str
  96            Your API key provided by the Boosted application.  See your profile
  97            to generate a new key.
  98        proxy: str
  99            Your organization may require the use of a proxy for access.
 100            The address of a HTTPS proxy in the format of <address>:<port>.
 101            Examples are "123.456.789:123" or "my.proxy.com:123".
 102            Do not prepend with "https://".
 103        disable_verify_ssl: bool
 104            Your networking setup may be behind a firewall which performs SSL
 105            inspection. Either set the REQUESTS_CA_BUNDLE environment variable
 106            to point to the location of a custom certificate bundle, or set this
 107            parameter to True to disable SSL verification as a workaround.
 108        """
 109        if override_uri is None:
 110            self.base_uri = g_boosted_api_url
 111        else:
 112            self.base_uri = override_uri
 113        self.api_key = api_key
 114        self.debug = debug
 115        self._request_params: Dict = {}
 116        if debug:
 117            logger.setLevel(logging.DEBUG)
 118        else:
 119            logger.setLevel(logging.INFO)
 120        if proxy is not None:
 121            self._request_params["proxies"] = {"https": proxy}
 122        if disable_verify_ssl:
 123            self._request_params["verify"] = False
 124
 125    def __print_json_info(self, json_data, isInference=False):
 126        if "warnings" in json_data.keys():
 127            for warning in json_data["warnings"]:
 128                logger.warning("  {0}".format(warning))
 129        if "errors" in json_data.keys():
 130            for error in json_data["errors"]:
 131                logger.error("  {0}".format(error))
 132                return Status.FAIL
 133
 134        if "result" in json_data.keys():
 135            results_data = json_data["result"]
 136            if isInference:
 137                if "inferenceResultsUrl" in results_data.keys():
 138                    res_url = parse.urlparse(results_data["inferenceResultsUrl"])
 139                    logger.debug(res_url)
 140                    logger.info("Inference started.")
 141            if "updateCount" in results_data.keys():
 142                logger.info("Updated {0} rows.".format(results_data["updateCount"]))
 143            if "createCount" in results_data.keys():
 144                logger.info("Created {0} rows.".format(results_data["createCount"]))
 145            return Status.SUCCESS
 146
 147    def __to_date_obj(self, dt):
 148        if isinstance(dt, datetime.datetime):
 149            dt = dt.date()
 150        elif isinstance(dt, datetime.date):
 151            return dt
 152        elif isinstance(dt, str):
 153            try:
 154                dt = parser.parse(dt).date()
 155            except ValueError:
 156                raise ValueError('dt: "' + dt + '" is not a valid date.')
 157        return dt
 158
 159    def __iso_format(self, dt):
 160        date = self.__to_date_obj(dt)
 161        if date is not None:
 162            date = date.isoformat()
 163        return date
 164
 165    def _check_status_code(self, response, isInference=False):
 166        has_json = False
 167        try:
 168            logger.debug(response.headers)
 169            if "Content-Type" in response.headers:
 170                if response.headers["Content-Type"].startswith("application/json"):
 171                    json_data = response.json()
 172                    has_json = True
 173            else:
 174                has_json = False
 175        except json.JSONDecodeError:
 176            logger.error("ERROR: response has no JSON payload.")
 177        if response.status_code == 200 or response.status_code == 202:
 178            if has_json:
 179                self.__print_json_info(json_data, isInference)
 180            else:
 181                pass
 182            return Status.SUCCESS
 183        if response.status_code == 404:
 184            if has_json:
 185                self.__print_json_info(json_data, isInference)
 186            raise BoostedAPIException(
 187                'Server "{0}" not reachable.  Code {1}.'.format(
 188                    self.base_uri, response.status_code
 189                ),
 190                data=response,
 191            )
 192        if response.status_code == 400:
 193            if has_json:
 194                self.__print_json_info(json_data, isInference)
 195            if isInference:
 196                return Status.FAIL
 197            else:
 198                raise BoostedAPIException("Error, bad request.  Check the dataset ID.", response)
 199        if response.status_code == 401:
 200            if has_json:
 201                self.__print_json_info(json_data, isInference)
 202            raise BoostedAPIException("Authorization error.", response)
 203        else:
 204            if has_json:
 205                self.__print_json_info(json_data, isInference)
 206            raise BoostedAPIException(
 207                "Error in API response.  Status code={0} {1}\n{2}".format(
 208                    response.status_code, response.reason, response.headers
 209                ),
 210                response,
 211            )
 212
 213    def _try_extract_error_code(self, result):
 214        logger.info(result.headers)
 215        if "Content-Type" in result.headers:
 216            if result.headers["Content-Type"].startswith("application/json"):
 217                if "errors" in result.json():
 218                    return result.json()["errors"]
 219            if result.headers["Content-Type"].startswith("text/plain"):
 220                return result.text
 221        return str(result.reason)
 222
 223    def _check_ok_or_err_with_msg(self, res, potential_error_msg: str):
 224        if not res.ok:
 225            error = self._try_extract_error_code(res)
 226            logger.error(error)
 227            raise BoostedAPIException(f"{potential_error_msg}: {error}")
 228
 229    def _get_portfolio_rebalance_from_periods(
 230        self, portfolio_id: str, rel_periods: List[str]
 231    ) -> List[datetime.date]:
 232        """
 233        Returns a list of rebalance dates for a portfolio given a list of
 234        relative periods of format '1D', '1W', '3M', etc.
 235        """
 236        resp = self._get_graphql(
 237            query=graphql_queries.GET_PORTFOLIO_RELATIVE_DATES_QUERY,
 238            variables={"portfolioId": portfolio_id, "relativePeriods": rel_periods},
 239        )
 240        dates = resp["data"]["portfolio"]["relativeDates"]
 241        return [datetime.datetime.strptime(d["date"], "%Y-%m-%d").date() for d in dates]
 242
 243    def translate_text(self, language: Optional[Union[Language, str]], text: str) -> str:
 244        if not language or language == Language.ENGLISH:
 245            # By default, do not translate English
 246            return text
 247
 248        params = {"text": text, "langCode": language}
 249        url = self.base_uri + "/api/translate/translate-text"
 250        headers = {"Authorization": "ApiKey " + self.api_key}
 251        logger.info("Translating text...")
 252        res = requests.post(url, json=params, headers=headers, **self._request_params)
 253        try:
 254            result = res.json()["translatedText"]
 255        except Exception:
 256            raise BoostedAPIException("Error translating text")
 257        return result
 258
 259    def query_dataset(self, dataset_id):
 260        url = self.base_uri + "/api/datasets/{0}".format(dataset_id)
 261        headers = {"Authorization": "ApiKey " + self.api_key}
 262        res = requests.get(url, headers=headers, **self._request_params)
 263        if res.ok:
 264            return res.json()
 265        else:
 266            error_msg = self._try_extract_error_code(res)
 267            logger.error(error_msg)
 268            raise BoostedAPIException("Failed to query dataset: {0}.".format(error_msg))
 269
 270    def export_global_data(
 271        self,
 272        dataset_id,
 273        start=(datetime.date.today() - timedelta(days=365 * 25)),
 274        end=datetime.date.today(),
 275        timeout=600,
 276    ):
 277        query_info = self.query_dataset(dataset_id)
 278        if DataSetType[query_info["type"]] != DataSetType.GLOBAL:
 279            raise BoostedAPIException(
 280                f"Incorrect dataset type: {query_info['type']}" f" - Expected {DataSetType.GLOBAL}"
 281            )
 282        return self.export_data(dataset_id, start, end, timeout)
 283
 284    def export_independent_data(
 285        self,
 286        dataset_id,
 287        start=(datetime.date.today() - timedelta(days=365 * 25)),
 288        end=datetime.date.today(),
 289        timeout=600,
 290    ):
 291        query_info = self.query_dataset(dataset_id)
 292        if DataSetType[query_info["type"]] != DataSetType.STRATEGY:
 293            raise BoostedAPIException(
 294                f"Incorrect dataset type: {query_info['type']}"
 295                f" - Expected {DataSetType.STRATEGY}"
 296            )
 297        return self.export_data(dataset_id, start, end, timeout)
 298
 299    def export_dependent_data(
 300        self,
 301        dataset_id,
 302        start=None,
 303        end=None,
 304        timeout=600,
 305    ):
 306        query_info = self.query_dataset(dataset_id)
 307        if DataSetType[query_info["type"]] != DataSetType.STOCK:
 308            raise BoostedAPIException(
 309                f"Incorrect dataset type: {query_info['type']}" f" - Expected {DataSetType.STOCK}"
 310            )
 311
 312        valid_date_range = self.getDatasetDates(dataset_id)
 313        validStart = valid_date_range["validFrom"]
 314        validEnd = valid_date_range["validTo"]
 315
 316        if start is None:
 317            logger.info("Since no start date provided, starting from {0}.".format(validStart))
 318            start = validStart
 319        if end is None:
 320            logger.info("Since no end date provided, ending at {0}.".format(validEnd))
 321            end = validEnd
 322        start = self.__to_date_obj(start)
 323        end = self.__to_date_obj(end)
 324        if start < validStart:
 325            logger.info("Data does not exist before {0}.".format(validStart))
 326            logger.info("Starting from {0}.".format(validStart))
 327            start = validStart
 328        if end > validEnd:
 329            logger.info("Data does not exist after {0}.".format(validEnd))
 330            logger.info("Ending at {0}.".format(validEnd))
 331            end = validEnd
 332        validate_start_and_end_dates(start, end)
 333
 334        logger.info("Data exists from {0} to {1}.".format(start, end))
 335        request_url = "/api/datasets/" + dataset_id + "/export-data"
 336        headers = {"Authorization": "ApiKey " + self.api_key}
 337
 338        data_chunks = []
 339        chunk_size_days = 90
 340        while start <= end:
 341            chunk_end = start + timedelta(days=chunk_size_days)
 342            if chunk_end > end:
 343                chunk_end = end
 344
 345            logger.info("Requesting start={0} end={1}.".format(start, chunk_end))
 346            params = {"start": self.__iso_format(start), "end": self.__iso_format(chunk_end)}
 347            logger.debug("URL={0}, headers={1}, params={2}".format(request_url, headers, params))
 348
 349            res = requests.get(
 350                self.base_uri + request_url,
 351                headers=headers,
 352                params=params,
 353                timeout=timeout,
 354                **self._request_params,
 355            )
 356
 357            if res.ok:
 358                buf = io.StringIO(res.text)
 359                df = pd.read_csv(buf, index_col=0, parse_dates=True)
 360                if "price" in df.columns:
 361                    df = df.drop("price", axis=1)
 362                data_chunks.append(df)
 363            else:
 364                error_msg = self._try_extract_error_code(res)
 365                logger.error(error_msg)
 366                raise BoostedAPIException("Failed to query dataset: {0}.".format(error_msg))
 367
 368            start = start + timedelta(days=chunk_size_days + 1)
 369
 370        return pd.concat(data_chunks)
 371
 372    def export_data(
 373        self,
 374        dataset_id,
 375        start=(datetime.date.today() - timedelta(days=365 * 25)),
 376        end=datetime.date.today(),
 377        timeout=600,
 378    ):
 379        logger.info("Requesting start={0} end={1}.".format(start, end))
 380        request_url = "/api/datasets/" + dataset_id + "/export-data"
 381        headers = {"Authorization": "ApiKey " + self.api_key}
 382        start = self.__iso_format(start)
 383        end = self.__iso_format(end)
 384        params = {"start": start, "end": end}
 385        logger.debug("URL={0}, headers={1}, params={2}".format(request_url, headers, params))
 386        res = requests.get(
 387            self.base_uri + request_url,
 388            headers=headers,
 389            params=params,
 390            timeout=timeout,
 391            **self._request_params,
 392        )
 393        if res.ok or self._check_status_code(res):
 394            buf = io.StringIO(res.text)
 395            df = pd.read_csv(buf, index_col=0, parse_dates=True)
 396            if "price" in df.columns:
 397                df = df.drop("price", axis=1)
 398            return df
 399        else:
 400            error_msg = self._try_extract_error_code(res)
 401            logger.error(error_msg)
 402            raise BoostedAPIException("Failed to query dataset: {0}.".format(error_msg))
 403
 404    def _get_inference(self, model_id, inference_date=datetime.date.today()):
 405        request_url = "/api/models/" + model_id + "/inference-results"
 406        headers = {"Authorization": "ApiKey " + self.api_key}
 407        params = {}
 408        params["date"] = self.__iso_format(inference_date)
 409        logger.debug(request_url + ", " + str(headers) + ", " + str(params))
 410        res = requests.get(
 411            self.base_uri + request_url, headers=headers, params=params, **self._request_params
 412        )
 413        status = self._check_status_code(res, isInference=True)
 414        if status == Status.SUCCESS:
 415            return res, status
 416        else:
 417            return None, status
 418
 419    def get_inference(
 420        self, model_id, inference_date=datetime.date.today(), block=False, timeout_minutes=30
 421    ):
 422        start_time = datetime.datetime.now()
 423        while True:
 424            for numRetries in range(3):
 425                res, status = self._get_inference(model_id, inference_date)
 426                if res is not None:
 427                    continue
 428                else:
 429                    if status == Status.FAIL:
 430                        return Status.FAIL
 431                    logger.info("Retrying...")
 432            if res is None:
 433                logger.error("Max retries reached.  Request failed.")
 434                return None
 435
 436            json_data = res.json()
 437            if "result" in json_data.keys():
 438                if json_data["result"]["status"] == "RUNNING":
 439                    still_running = True
 440                    if not block:
 441                        logger.warn("Inference job is still running.")
 442                        return None
 443                    else:
 444                        logger.info(
 445                            "Inference job is still running.  Time elapsed={0}.".format(
 446                                datetime.datetime.now() - start_time
 447                            )
 448                        )
 449                        time.sleep(10)
 450                else:
 451                    still_running = False
 452
 453                if not still_running and json_data["result"]["status"] == "COMPLETE":
 454                    csv = json_data["result"]["signals"]
 455                    logger.info(json_data["result"])
 456                    if self._check_status_code(res, isInference=True):
 457                        logger.info(
 458                            "Total run time = {0}.".format(datetime.datetime.now() - start_time)
 459                        )
 460                        return csv
 461            else:
 462                if "errors" in json_data.keys():
 463                    logger.error(json_data["errors"])
 464                else:
 465                    logger.error("Error getting inference for date {0}.".format(inference_date))
 466                return None
 467            if (datetime.datetime.now() - start_time).total_seconds() / 60.0 > timeout_minutes:
 468                logger.error("Timeout waiting for job completion.")
 469                return None
 470
 471    def createDataset(self, schema):
 472        request_url = "/api/datasets"
 473        headers = {"Authorization": "ApiKey " + self.api_key, "Content-Type": "application/json"}
 474        s = json.dumps(schema)
 475        logger.info("Creating dataset with schema " + s)
 476        res = requests.post(
 477            self.base_uri + request_url, data=s, headers=headers, **self._request_params
 478        )
 479        if res.ok:
 480            return res.json()["result"]
 481        else:
 482            raise BoostedAPIException("Dataset creation failed.")
 483
 484    def getUniverse(self, modelId, date=None):
 485        if date is not None:
 486            url = "/api/models/{0}/universe/{1}".format(modelId, self.__iso_format(date))
 487            logger.info("Getting universe for date: {0}.".format(date))
 488        else:
 489            url = "/api/models/{0}/universe/".format(modelId)
 490        headers = {"Authorization": "ApiKey " + self.api_key}
 491        res = requests.get(self.base_uri + url, headers=headers, **self._request_params)
 492        if res.ok:
 493            buf = io.StringIO(res.text)
 494            df = pd.read_csv(buf, index_col=0, parse_dates=True)
 495            return df
 496        else:
 497            error = self._try_extract_error_code(res)
 498            logger.error(
 499                "There was a problem getting this universe or model ID: {0}.".format(error)
 500            )
 501            raise BoostedAPIException("Failed to get universe: {0}".format(error))
 502
 503    def updateUniverse(self, modelId, universe_df, date=datetime.date.today() + timedelta(1)):
 504        date = self.__iso_format(date)
 505        url = self.base_uri + "/api/models/{0}/universe/{1}".format(modelId, date)
 506        headers = {"Authorization": "ApiKey " + self.api_key}
 507        logger.info("Updating universe for date {0}.".format(date))
 508        if isinstance(universe_df, pd.core.frame.DataFrame):
 509            buf = io.StringIO()
 510            universe_df.to_csv(buf)
 511            target = ("uploaded_universe.csv", buf.getvalue(), "text/csv")
 512            files_req = {}
 513            files_req["universe"] = target
 514            res = requests.post(url, files=files_req, headers=headers, **self._request_params)
 515        elif isinstance(universe_df, str):
 516            target = ("uploaded_universe.csv", universe_df, "text/csv")
 517            files_req = {}
 518            files_req["universe"] = target
 519            res = requests.post(url, files=files_req, headers=headers, **self._request_params)
 520        else:
 521            raise BoostedAPIException("Expected CSV as str or Pandas DataFrame.")
 522        if res.ok:
 523            logger.info("Universe update successful.")
 524            if "warnings" in res.json():
 525                logger.info("Warnings: {0}.".format(res.json()["warnings"]))
 526                return res.json()["warnings"]
 527            else:
 528                return "No warnings."
 529        else:
 530            error_msg = self._try_extract_error_code(res)
 531            raise BoostedAPIException("Failed to get universe: {0}.".format(error_msg))
 532
 533    def create_universe(
 534        self, universe: Union[pd.DataFrame, str], name: str, description: str
 535    ) -> List[str]:
 536        PRESENT = "PRESENT"
 537        ANY = "ANY"
 538        EARLIST_DATE = "1900-01-01"
 539        LATEST_DATE = "4000-01-01"
 540
 541        if isinstance(universe, (str, bytes, os.PathLike)):
 542            universe = pd.read_csv(universe)
 543
 544        universe.columns = universe.columns.str.lower()
 545
 546        # Clients are free to leave out data. Fill in some defaults here.
 547        if "from" not in universe.columns:
 548            universe["from"] = EARLIST_DATE
 549        if "to" not in universe.columns:
 550            universe["to"] = LATEST_DATE
 551        if "currency" not in universe.columns:
 552            universe["currency"] = ANY
 553        if "country" not in universe.columns:
 554            universe["country"] = ANY
 555        if "isin" not in universe.columns:
 556            universe["isin"] = None
 557        if "symbol" not in universe.columns:
 558            universe["symbol"] = None
 559
 560        # to prevent conflicts with python keywords
 561        universe.rename(columns={"from": "from_date", "to": "to_date"}, inplace=True)
 562
 563        universe = universe.replace({np.nan: None})
 564        security_country_currency_date_list = []
 565        for i, r in enumerate(universe.itertuples()):
 566            id_type = ColumnSubRole.ISIN
 567            identifier = r.isin
 568
 569            if identifier is None:
 570                id_type = ColumnSubRole.SYMBOL
 571                identifier = str(r.symbol)
 572
 573            # if identifier is still None, it means that there is no ISIN or
 574            # SYMBOL for this row, in which case we throw an error
 575            if identifier is None:
 576                raise BoostedAPIException(
 577                    (
 578                        f"Missing identifier column in universe row {i + 1}"
 579                        " should contain ISIN or Symbol"
 580                    )
 581                )
 582
 583            security_country_currency_date_list.append(
 584                DateIdentCountryCurrency(
 585                    date=r.from_date or EARLIST_DATE,
 586                    identifier=identifier,
 587                    country=r.country or ANY,
 588                    currency=r.currency or ANY,
 589                    id_type=id_type,
 590                )
 591            )
 592
 593        gbi_id_objs = self.getGbiIdFromIdentCountryCurrencyDate(security_country_currency_date_list)
 594
 595        security_list = []
 596        for i, r in enumerate(universe.itertuples()):
 597            # if we have a None here, we failed to map to a gbi id
 598            if gbi_id_objs[i] is None:
 599                raise BoostedAPIException(f"Unable to map row: {tuple(r)}")
 600
 601            security_list.append(
 602                {
 603                    "stockId": gbi_id_objs[i].gbi_id,
 604                    "fromZ": r.from_date or EARLIST_DATE,
 605                    "toZ": LATEST_DATE if r.to_date in (PRESENT, None) else r.to_date,
 606                    "removal": False,
 607                    "source": "UPLOAD",
 608                }
 609            )
 610
 611        url = self.base_uri + "/api/template-universe/save"
 612        headers = {"Authorization": "ApiKey " + self.api_key, "Content-Type": "application/json"}
 613        req = {"name": name, "description": description, "modificationDaos": security_list}
 614
 615        res = requests.post(url, json=req, headers=headers, **self._request_params)
 616        self._check_ok_or_err_with_msg(res, "Failed to create universe")
 617
 618        if "warnings" in res.json():
 619            logger.info("Warnings: {0}.".format(res.json()["warnings"]))
 620            return res.json()["warnings"].splitlines()
 621        else:
 622            return []
 623
 624    def validate_dataframe(self, df):
 625        if not isinstance(df, pd.core.frame.DataFrame):
 626            logger.error("Dataset must be of type Dataframe.")
 627            return False
 628        if type(df.index) != pd.core.indexes.datetimes.DatetimeIndex:
 629            logger.error("Index must be DatetimeIndex.")
 630            return False
 631        if len(df.columns) == 0:
 632            logger.error("No feature columns exist.")
 633            return False
 634        if len(df) == 0:
 635            logger.error("No rows exist.")
 636        return True
 637
 638    def get_dataset_schema(self, dataset_id):
 639        url = self.base_uri + "/api/datasets/{0}/schema".format(dataset_id)
 640        headers = {"Authorization": "ApiKey " + self.api_key}
 641        res = requests.get(url, headers=headers, **self._request_params)
 642        if res.ok:
 643            json_schema = res.json()
 644        else:
 645            error_msg = self._try_extract_error_code(res)
 646            logger.error(error_msg)
 647            raise BoostedAPIException("Failed to query dataset: {0}.".format(error_msg))
 648        return DataSetConfig.fromDict(json_schema["result"])
 649
 650    def add_dependent_dataset(
 651        self, dataset, datasetName="DependentDataset", schema=None, timeout=600, block=True
 652    ):
 653        result = self.add_dependent_dataset_with_warnings(
 654            dataset, datasetName, schema, timeout, block
 655        )
 656        return result["dataset_id"]
 657
 658    def add_dependent_dataset_with_warnings(
 659        self,
 660        dataset,
 661        datasetName="DependentDataset",
 662        schema=None,
 663        timeout=600,
 664        block=True,
 665        no_exception_on_chunk_error=False,
 666    ):
 667        if not self.validate_dataframe(dataset):
 668            logger.error("dataset failed validation.")
 669            return None
 670        if schema is None:
 671            schema = infer_dataset_schema(datasetName, dataset, DataSetType.STOCK)
 672        dsid = self.createDataset(schema.toDict())
 673        logger.info("Creating dataset with ID = {0}.".format(dsid))
 674        result = self.add_dependent_data(
 675            dsid,
 676            dataset,
 677            timeout,
 678            block,
 679            data_type=DataAddType.CREATION,
 680            no_exception_on_chunk_error=no_exception_on_chunk_error,
 681        )
 682        return {"dataset_id": dsid, "warnings": result["warnings"], "errors": result["errors"]}
 683
 684    def add_independent_dataset(
 685        self, dataset, datasetName="IndependentDataset", schema=None, timeout=600, block=True
 686    ):
 687        result = self.add_independent_dataset_with_warnings(
 688            dataset, datasetName, schema, timeout, block
 689        )
 690        return result["dataset_id"]
 691
 692    def add_independent_dataset_with_warnings(
 693        self,
 694        dataset,
 695        datasetName="IndependentDataset",
 696        schema=None,
 697        timeout=600,
 698        block=True,
 699        no_exception_on_chunk_error=False,
 700    ):
 701        if not self.validate_dataframe(dataset):
 702            logger.error("dataset failed validation.")
 703            return None
 704        if schema is None:
 705            schema = infer_dataset_schema(datasetName, dataset, DataSetType.STRATEGY)
 706        schemaDict = schema.toDict()
 707        if "configurationDataJson" not in schemaDict:
 708            schemaDict["configurationDataJson"] = "{}"
 709        dsid = self.createDataset(schemaDict)
 710        logger.info("Creating dataset with ID = {0}.".format(dsid))
 711        result = self.add_independent_data(
 712            dsid,
 713            dataset,
 714            timeout,
 715            block,
 716            data_type=DataAddType.CREATION,
 717            no_exception_on_chunk_error=no_exception_on_chunk_error,
 718        )
 719        return {"dataset_id": dsid, "warnings": result["warnings"], "errors": result["errors"]}
 720
 721    def add_global_dataset(
 722        self, dataset, datasetName="GlobalDataset", schema=None, timeout=600, block=True
 723    ):
 724        result = self.add_global_dataset_with_warnings(dataset, datasetName, schema, timeout, block)
 725        return result["dataset_id"]
 726
 727    def add_global_dataset_with_warnings(
 728        self,
 729        dataset,
 730        datasetName="GlobalDataset",
 731        schema=None,
 732        timeout=600,
 733        block=True,
 734        no_exception_on_chunk_error=False,
 735    ):
 736        if not self.validate_dataframe(dataset):
 737            logger.error("dataset failed validation.")
 738            return None
 739        if schema is None:
 740            schema = infer_dataset_schema(datasetName, dataset, DataSetType.GLOBAL)
 741        dsid = self.createDataset(schema.toDict())
 742        logger.info("Creating dataset with ID = {0}.".format(dsid))
 743        result = self.add_global_data(
 744            dsid,
 745            dataset,
 746            timeout,
 747            block,
 748            data_type=DataAddType.CREATION,
 749            no_exception_on_chunk_error=no_exception_on_chunk_error,
 750        )
 751        return {"dataset_id": dsid, "warnings": result["warnings"], "errors": result["errors"]}
 752
 753    def add_independent_data(
 754        self,
 755        dataset_id,
 756        csv_data,
 757        timeout=600,
 758        block=True,
 759        data_type=DataAddType.HISTORICAL,
 760        no_exception_on_chunk_error=False,
 761    ):
 762        query_info = self.query_dataset(dataset_id)
 763        if DataSetType[query_info["type"]] != DataSetType.STRATEGY:
 764            raise BoostedAPIException(
 765                f"Incorrect dataset type: {query_info['type']}"
 766                f" - Expected {DataSetType.STRATEGY}"
 767            )
 768        warnings, errors = self.setup_chunk_and_upload_data(
 769            dataset_id, csv_data, data_type, timeout, block, no_exception_on_chunk_error
 770        )
 771        if len(warnings) > 0:
 772            logger.warning(
 773                "Encountered {0} total warnings while uploading dataset.".format(len(warnings))
 774            )
 775        if len(errors) > 0:
 776            raise BoostedAPIException(
 777                "Encountered {0} total ERRORS while uploading dataset".format(len(errors))
 778                + "\n".join(errors)
 779            )
 780        return {"warnings": warnings, "errors": errors}
 781
 782    def add_dependent_data(
 783        self,
 784        dataset_id,
 785        csv_data,
 786        timeout=600,
 787        block=True,
 788        data_type=DataAddType.HISTORICAL,
 789        no_exception_on_chunk_error=False,
 790    ):
 791        warnings = []
 792        query_info = self.query_dataset(dataset_id)
 793        if DataSetType[query_info["type"]] != DataSetType.STOCK:
 794            raise BoostedAPIException(
 795                f"Incorrect dataset type: {query_info['type']}" f" - Expected {DataSetType.STOCK}"
 796            )
 797        warnings, errors = self.setup_chunk_and_upload_data(
 798            dataset_id, csv_data, data_type, timeout, block, no_exception_on_chunk_error
 799        )
 800        if len(warnings) > 0:
 801            logger.warning(
 802                "Encountered {0} total warnings while uploading dataset.".format(len(warnings))
 803            )
 804        if len(errors) > 0:
 805            raise BoostedAPIException(
 806                "Encountered {0} total ERRORS while uploading dataset".format(len(errors))
 807                + "\n".join(errors)
 808            )
 809        return {"warnings": warnings, "errors": errors}
 810
 811    def add_global_data(
 812        self,
 813        dataset_id,
 814        csv_data,
 815        timeout=600,
 816        block=True,
 817        data_type=DataAddType.HISTORICAL,
 818        no_exception_on_chunk_error=False,
 819    ):
 820        query_info = self.query_dataset(dataset_id)
 821        if DataSetType[query_info["type"]] != DataSetType.GLOBAL:
 822            raise BoostedAPIException(
 823                f"Incorrect dataset type: {query_info['type']}" f" - Expected {DataSetType.GLOBAL}"
 824            )
 825        warnings, errors = self.setup_chunk_and_upload_data(
 826            dataset_id, csv_data, data_type, timeout, block, no_exception_on_chunk_error
 827        )
 828        if len(warnings) > 0:
 829            logger.warning(
 830                "Encountered {0} total warnings while uploading dataset.".format(len(warnings))
 831            )
 832        if len(errors) > 0:
 833            raise BoostedAPIException(
 834                "Encountered {0} total ERRORS while uploading dataset".format(len(errors))
 835                + "\n".join(errors)
 836            )
 837        return {"warnings": warnings, "errors": errors}
 838
 839    def get_csv_buffer(self):
 840        return io.StringIO()
 841
 842    def start_chunked_upload(self, dataset_id):
 843        url = self.base_uri + "/api/datasets/{0}/start-chunked-upload".format(dataset_id)
 844        headers = {"Authorization": "ApiKey " + self.api_key}
 845        res = requests.post(url, headers=headers, **self._request_params)
 846        if res.ok:
 847            return res.json()["result"]
 848        else:
 849            error_msg = self._try_extract_error_code(res)
 850            logger.error(error_msg)
 851            raise BoostedAPIException(
 852                "Failed to obtain dataset lock for upload: {0}.".format(error_msg)
 853            )
 854
 855    def abort_chunked_upload(self, dataset_id, chunk_id):
 856        url = self.base_uri + "/api/datasets/{0}/abort-chunked-upload".format(dataset_id)
 857        headers = {"Authorization": "ApiKey " + self.api_key}
 858        params = {"uploadGroupId": chunk_id}
 859        res = requests.post(url, headers=headers, **self._request_params, params=params)
 860        if not res.ok:
 861            error_msg = self._try_extract_error_code(res)
 862            logger.error(error_msg)
 863            raise BoostedAPIException(
 864                "Failed to abort dataset lock during error: {0}.".format(error_msg)
 865            )
 866
 867    def check_dataset_ingestion_completion(self, dataset_id, chunk_id, start_time):
 868        url = self.base_uri + "/api/datasets/{0}/upload-chunk-status".format(dataset_id)
 869        headers = {"Authorization": "ApiKey " + self.api_key}
 870        params = {"uploadGroupId": chunk_id}
 871        res = requests.get(url, headers=headers, **self._request_params, params=params)
 872        res = res.json()
 873
 874        finished = False
 875        warnings = []
 876        errors = []
 877
 878        if type(res) == dict:
 879            dataset_status = res["datasetStatus"]
 880            chunk_status = res["chunkStatus"]
 881            if chunk_status != ChunkStatus.PROCESSING.value:
 882                finished = True
 883                errors = res["errors"]
 884                warnings = res["warnings"]
 885                successful_rows = res["successfulRows"]
 886                total_rows = res["totalRows"]
 887                logger.info(
 888                    f"Successfully ingested {successful_rows} out of {total_rows} uploaded rows."
 889                )
 890                if chunk_status in [
 891                    ChunkStatus.SUCCESS.value,
 892                    ChunkStatus.WARNING.value,
 893                    ChunkStatus.ERROR.value,
 894                ]:
 895                    if dataset_status != "AVAILABLE":
 896                        raise BoostedAPIException(
 897                            "Dataset was unexpectedly unavailable after chunk upload finished."
 898                        )
 899                    else:
 900                        logger.info("Ingestion complete.  Uploaded data is ready for use.")
 901                elif chunk_status == ChunkStatus.ABORTED.value:
 902                    errors.append(
 903                        "Dataset chunk upload was aborted by server! Upload did not succeed."
 904                    )
 905                else:
 906                    errors.append("Unexpected data ingestion status: {0}.".format(chunk_status))
 907            logger.info(
 908                "Data ingestion still running.  Time elapsed={0}.".format(
 909                    datetime.datetime.now() - start_time
 910                )
 911            )
 912        else:
 913            raise BoostedAPIException("Unable to get status of dataset ingestion.")
 914        return {"finished": finished, "warnings": warnings, "errors": errors}
 915
 916    def _commit_chunked_upload(self, dataset_id, chunk_id, data_type, block=True, timeout=600):
 917        url = self.base_uri + "/api/datasets/{0}/commit-chunked-upload".format(dataset_id)
 918        headers = {"Authorization": "ApiKey " + self.api_key}
 919        params = {
 920            "uploadGroupId": chunk_id,
 921            "dataAddType": data_type,
 922            "sendCompletionEmail": not block,
 923        }
 924        res = requests.post(url, headers=headers, **self._request_params, params=params)
 925        if not res.ok:
 926            error_msg = self._try_extract_error_code(res)
 927            logger.error(error_msg)
 928            raise BoostedAPIException("Failed to commit dataset files: {0}.".format(error_msg))
 929
 930        if block:
 931            start_time = datetime.datetime.now()
 932            # Keep waiting until upload is no longer in UPDATING state...
 933            while True:
 934                result = self.check_dataset_ingestion_completion(dataset_id, chunk_id, start_time)
 935                if result["finished"]:
 936                    break
 937
 938                if (datetime.datetime.now() - start_time).total_seconds() > timeout:
 939                    err_str = (
 940                        f"Timeout waiting for commit of dataset: {dataset_id} | chunk: {chunk_id}"
 941                    )
 942                    logger.error(err_str)
 943                    return [], [err_str]
 944
 945                time.sleep(10)
 946            return result["warnings"], result["errors"]
 947        else:
 948            return [], []
 949
 950    def setup_chunk_and_upload_data(
 951        self,
 952        dataset_id,
 953        csv_data,
 954        data_type,
 955        timeout=600,
 956        block=True,
 957        no_exception_on_chunk_error=False,
 958    ):
 959        chunk_id = self.start_chunked_upload(dataset_id)
 960        logger.info("Obtained lock on dataset for upload: " + chunk_id)
 961        try:
 962            warnings, errors = self.chunk_and_upload_data(
 963                dataset_id, chunk_id, csv_data, timeout, no_exception_on_chunk_error
 964            )
 965            commit_warnings, commit_errors = self._commit_chunked_upload(
 966                dataset_id, chunk_id, data_type, block, timeout
 967            )
 968            return warnings + commit_warnings, errors + commit_errors
 969        except Exception:
 970            self.abort_chunked_upload(dataset_id, chunk_id)
 971            raise
 972
 973    def chunk_and_upload_data(
 974        self, dataset_id, chunk_id, csv_data, timeout=600, no_exception_on_chunk_error=False
 975    ):
 976        if isinstance(csv_data, pd.core.frame.DataFrame):
 977            if not isinstance(csv_data.index, pd.core.indexes.datetimes.DatetimeIndex):
 978                raise BoostedAPIException("DataFrame must have DatetimeIndex as index type.")
 979
 980            warnings = []
 981            errors = []
 982            logger.info("Uploading yearly.")
 983            for t in csv_data.index.to_period("Y").unique():
 984                if t is pd.NaT:
 985                    continue
 986
 987                # serialize bit to string
 988                buf = self.get_csv_buffer()
 989                yearly_csv = csv_data.loc[str(t)]
 990                yearly_csv.to_csv(buf, header=True)
 991                raw_csv = buf.getvalue()
 992
 993                # we are already chunking yearly... but if the csv still exceeds a healthy
 994                # limit of 50mb the final line of defence is to ignore date boundaries and
 995                # just chunk the rows. This is mostly for the cloudflare upload limit.
 996                size_lim = 50 * 1000 * 1000
 997                est_csv_size = sys.getsizeof(raw_csv)
 998                if est_csv_size > size_lim:
 999                    del raw_csv, buf
1000                    logger.info("Yearly data too large for single upload, chunking further...")
1001                    chunks = []
1002                    nchunks = math.ceil(est_csv_size / size_lim)
1003                    rows_per_chunk = math.ceil(len(yearly_csv) / nchunks)
1004                    for i in range(0, len(yearly_csv), rows_per_chunk):
1005                        buf = self.get_csv_buffer()
1006                        split_csv = yearly_csv.iloc[i : i + rows_per_chunk]
1007                        split_csv.to_csv(buf, header=True)
1008                        split_csv = buf.getvalue()
1009                        chunks.append(
1010                            (
1011                                "{0}-{1}".format(i + 1, min(len(yearly_csv), i + rows_per_chunk)),
1012                                split_csv,
1013                            )
1014                        )
1015                else:
1016                    chunks = [("all", raw_csv)]
1017
1018                for i, (rows_descriptor, chunk_csv) in enumerate(chunks):
1019                    chunk_descriptor = "{0} in yearly chunk {1}".format(rows_descriptor, t)
1020                    logger.info(
1021                        "Uploading rows:"
1022                        + chunk_descriptor
1023                        + " (chunk {0} of {1}):".format(i + 1, len(chunks))
1024                    )
1025                    _, new_warnings, new_errors = self.upload_dataset_chunk(
1026                        chunk_descriptor,
1027                        dataset_id,
1028                        chunk_id,
1029                        chunk_csv,
1030                        timeout,
1031                        no_exception_on_chunk_error,
1032                    )
1033                    warnings.extend(new_warnings)
1034                    errors.extend(new_errors)
1035            return warnings, errors
1036
1037        elif isinstance(csv_data, str):
1038            _, warnings, errors = self.upload_dataset_chunk(
1039                "all data", dataset_id, chunk_id, csv_data, timeout, no_exception_on_chunk_error
1040            )
1041            return warnings, errors
1042        else:
1043            raise BoostedAPIException("Expected CSV as str or Pandas DataFrame.")
1044
1045    def upload_dataset_chunk(
1046        self,
1047        chunk_descriptor,
1048        dataset_id,
1049        chunk_id,
1050        csv_data,
1051        timeout=600,
1052        no_exception_on_chunk_error=False,
1053    ):
1054        logger.info("Starting upload: " + chunk_descriptor)
1055        url = self.base_uri + "/api/datasets/{0}/upload-dataset-chunk".format(dataset_id)
1056        headers = {"Authorization": "ApiKey " + self.api_key}
1057        files_req = {}
1058        warnings = []
1059        errors = []
1060
1061        # make the network request
1062        target = ("uploaded_data.csv", csv_data, "text/csv")
1063        files_req["dataFile"] = target
1064        params = {"uploadGroupId": chunk_id}
1065        res = requests.post(
1066            url,
1067            params=params,
1068            files=files_req,
1069            headers=headers,
1070            timeout=timeout,
1071            **self._request_params,
1072        )
1073
1074        if res.ok:
1075            logger.info(
1076                (
1077                    "Chunk upload completed.  "
1078                    "Ingestion started.  "
1079                    "Please wait until the data is in AVAILABLE state."
1080                )
1081            )
1082            if "warnings" in res.json():
1083                warnings = res.json()["warnings"]
1084                if len(warnings) > 0:
1085                    logger.warning("Uploaded chunk encountered data warnings: ")
1086                for w in warnings:
1087                    logger.warning(w)
1088        else:
1089            reason = "Upload failed: {0}, {1}".format(res.text, res.reason)
1090            logger.error(reason)
1091            if no_exception_on_chunk_error:
1092                errors.append(
1093                    "Chunk {0} failed: {1}. ".format(chunk_descriptor, reason)
1094                    + "Your data was only PARTIALLY uploaded. "
1095                    + "Please reattempt the upload of this chunk."
1096                )
1097            else:
1098                raise BoostedAPIException(reason)
1099
1100        return res, warnings, errors
1101
1102    def getAllocationsForDate(self, portfolio_id, date, rollback_to_last_available_date):
1103        date = self.__iso_format(date)
1104        endpoint = "latest-allocations" if rollback_to_last_available_date else "allocations"
1105        url = self.base_uri + "/api/portfolios/{0}/{1}".format(portfolio_id, endpoint)
1106        headers = {"Authorization": "ApiKey " + self.api_key}
1107        params = {"date": date}
1108        logger.info("Retrieving allocations information for date {0}.".format(date))
1109        res = requests.get(url, params=params, headers=headers, **self._request_params)
1110        if res.ok:
1111            logger.info("Allocations retrieval successful.")
1112            return res.json()
1113        else:
1114            error_msg = self._try_extract_error_code(res)
1115            raise BoostedAPIException("Failed to retrieve allocations: {0}.".format(error_msg))
1116
1117    # New API method for fetching data from portfolio_holdings.pb2 file.
1118    def getAllocationsForDateV2(self, portfolio_id, date, rollback_to_last_available_date):
1119        date = self.__iso_format(date)
1120        endpoint = "latest-allocations-v2" if rollback_to_last_available_date else "allocations-v2"
1121        url = self.base_uri + "/api/portfolios/{0}/{1}".format(portfolio_id, endpoint)
1122        headers = {"Authorization": "ApiKey " + self.api_key}
1123        params = {"date": date}
1124        logger.info("Retrieving allocations information for date {0}.".format(date))
1125        res = requests.get(url, params=params, headers=headers, **self._request_params)
1126        if res.ok:
1127            logger.info("Allocations retrieval successful.")
1128            return res.json()
1129        else:
1130            error_msg = self._try_extract_error_code(res)
1131            raise BoostedAPIException("Failed to retrieve allocations: {0}.".format(error_msg))
1132
1133    def getAllocationsByDates(self, portfolio_id, dates=None):
1134        url = self.base_uri + "/api/portfolios/{0}/allocationsByDate".format(portfolio_id)
1135        headers = {"Authorization": "ApiKey " + self.api_key}
1136        if dates is not None:
1137            fmt_dates = []
1138            for d in dates:
1139                fmt_dates.append(self.__iso_format(d))
1140            fmt_dates_str = ",".join(fmt_dates)
1141            params: Dict = {"dates": fmt_dates_str}
1142            logger.info("Retrieving allocations information for dates {0}.".format(fmt_dates))
1143        else:
1144            params = {"dates": None}
1145            logger.info("Retrieving allocations information for all dates")
1146        res = requests.get(url, params=params, headers=headers, **self._request_params)
1147        if res.ok:
1148            logger.info("Allocations retrieval successful.")
1149            return res.json()
1150        else:
1151            error_msg = self._try_extract_error_code(res)
1152            raise BoostedAPIException("Failed to retrieve allocations: {0}.".format(error_msg))
1153
1154    def getSignalsForDate(self, portfolio_id, date, rollback_to_last_available_date):
1155        date = self.__iso_format(date)
1156        endpoint = "latest-signals" if rollback_to_last_available_date else "signals"
1157        url = self.base_uri + "/api/portfolios/{0}/{1}".format(portfolio_id, endpoint)
1158        headers = {"Authorization": "ApiKey " + self.api_key}
1159        params = {"date": date}
1160        logger.info("Retrieving signals information for date {0}.".format(date))
1161        res = requests.get(url, params=params, headers=headers, **self._request_params)
1162        if res.ok:
1163            logger.info("Signals retrieval successful.")
1164            return res.json()
1165        else:
1166            error_msg = self._try_extract_error_code(res)
1167            raise BoostedAPIException("Failed to retrieve signals: {0}.".format(error_msg))
1168
1169    def getSignalsForAllDates(self, portfolio_id, dates=None):
1170        url = self.base_uri + "/api/portfolios/{0}/signalsByDate".format(portfolio_id)
1171        headers = {"Authorization": "ApiKey " + self.api_key}
1172        params = {}
1173        if dates is not None:
1174            fmt_dates = []
1175            for d in dates:
1176                fmt_dates.append(self.__iso_format(d))
1177            fmt_dates_str = ",".join(fmt_dates)
1178            params = {"dates": fmt_dates_str}
1179            logger.info("Retrieving signals information for dates {0}.".format(fmt_dates))
1180        else:
1181            params = {"dates": None}
1182            logger.info("Retrieving signals information for all dates")
1183        res = requests.get(url, params=params, headers=headers, **self._request_params)
1184        if res.ok:
1185            logger.info("Signals retrieval successful.")
1186            return res.json()
1187        else:
1188            error_msg = self._try_extract_error_code(res)
1189            raise BoostedAPIException("Failed to retrieve signals: {0}.".format(error_msg))
1190
1191    def getEquityAccuracy(
1192        self,
1193        model_id: str,
1194        portfolio_id: str,
1195        tickers: List[str],
1196        start_date: Optional[BoostedDate] = None,
1197        end_date: Optional[BoostedDate] = None,
1198    ) -> Dict[str, Dict[str, Any]]:
1199        data: Dict[str, Any] = {}
1200        if start_date is not None:
1201            start_date = convert_date(start_date)
1202            data["startDate"] = start_date.isoformat()
1203        if end_date is not None:
1204            end_date = convert_date(end_date)
1205            data["endDate"] = end_date.isoformat()
1206
1207        if start_date and end_date:
1208            validate_start_and_end_dates(start_date, end_date)
1209
1210        tickers_stream = ",".join(tickers)
1211        data["tickers"] = tickers_stream
1212        data["timestamp"] = time.strftime("%H:%M:%S")
1213        data["shouldRecalc"] = True
1214        url = self.base_uri + f"/api/analysis/equity-accuracy/{model_id}/{portfolio_id}"
1215        headers = {"Authorization": "ApiKey " + self.api_key, "Content-Type": "application/json"}
1216
1217        logger.info(
1218            f"Retrieving equity accuracy data for date range {start_date} to {end_date} "
1219            f"for tickers: {tickers}."
1220        )
1221
1222        # Now create dataframes from the JSON output.
1223        metrics = [
1224            "hit_rate_mean",
1225            "hit_rate_median",
1226            "excess_return_mean",
1227            "excess_return_median",
1228            "return",
1229            "excess_return",
1230        ]
1231
1232        # send the request, retry if failed
1233        MAX_RETRIES = 10  # max of number of retries until timeout
1234        SLEEP_TIME = 3  # waiting time between requests
1235
1236        num_retries = 0
1237        success = False
1238        while not success and num_retries < MAX_RETRIES:
1239            res = requests.post(url, data=json.dumps(data), headers=headers, **self._request_params)
1240            if res.ok:
1241                logger.info("Equity Accuracy Data retrieval successful.")
1242                info = res.json()
1243                success = True
1244            else:
1245                data["shouldRecalc"] = False
1246                num_retries += 1
1247                time.sleep(SLEEP_TIME)
1248
1249        if not success:
1250            raise BoostedAPIException("Failed to retrieve equity accuracy: Request timeout.")
1251
1252        for ticker, accuracy_data in info.items():
1253            for metric in metrics:
1254                metric_matrix = accuracy_data[metric]
1255                if not isinstance(metric_matrix, str):
1256                    # Set the index to the quintile label, and remove it from the data
1257                    index = []
1258                    for row in metric_matrix[1:]:
1259                        index.append(row.pop(0))
1260
1261                    # columns are "1D", "5D", etc.
1262                    df = pd.DataFrame(metric_matrix[1:], columns=metric_matrix[0][1:], index=index)
1263                    accuracy_data[metric] = df
1264        return info
1265
1266    def getHistoricalTradeDates(self, portfolio_id, start_date=None, end_date=None):
1267        end_date = self.__to_date_obj(end_date or datetime.date.today())
1268        start_date = self.__iso_format(start_date or (end_date - timedelta(days=365)))
1269        end_date = self.__iso_format(end_date)
1270
1271        url = self.base_uri + "/api/portfolios/{0}/tradingDates".format(portfolio_id)
1272        headers = {"Authorization": "ApiKey " + self.api_key}
1273        params = {"startDate": start_date, "endDate": end_date}
1274
1275        logger.info(
1276            "Retrieving historical trade dates data for date range {0} to {1}.".format(
1277                start_date, end_date
1278            )
1279        )
1280        res = requests.get(url, params=params, headers=headers, **self._request_params)
1281        if res.ok:
1282            logger.info("Trading dates retrieval successful.")
1283            return res.json()["dates"]
1284        else:
1285            error_msg = self._try_extract_error_code(res)
1286            raise BoostedAPIException("Failed to retrieve trading dates: {0}.".format(error_msg))
1287
1288    def getRankingsForAllDates(self, portfolio_id, dates=None):
1289        url = self.base_uri + "/api/portfolios/{0}/rankingsByDate".format(portfolio_id)
1290        headers = {"Authorization": "ApiKey " + self.api_key}
1291        params = {}
1292        if dates is not None:
1293            fmt_dates = []
1294            for d in dates:
1295                fmt_dates.append(self.__iso_format(d))
1296            fmt_dates_str = ",".join(fmt_dates)
1297            params = {"dates": fmt_dates_str}
1298            logger.info("Retrieving rankings information for date {0}.".format(fmt_dates_str))
1299        else:
1300            params = {"dates": None}
1301            logger.info("Retrieving rankings information for all dates")
1302        res = requests.get(url, params=params, headers=headers, **self._request_params)
1303        if res.ok:
1304            logger.info("Rankings retrieval successful.")
1305            return res.json()
1306        else:
1307            error_msg = self._try_extract_error_code(res)
1308            raise BoostedAPIException("Failed to retrieve rankings: {0}.".format(error_msg))
1309
1310    def getRankingsForDate(self, portfolio_id, date, rollback_to_last_available_date):
1311        date = self.__iso_format(date)
1312        endpoint = "latest-rankings" if rollback_to_last_available_date else "rankings"
1313        url = self.base_uri + "/api/{0}/{1}/{2}".format(endpoint, portfolio_id, date)
1314        headers = {"Authorization": "ApiKey " + self.api_key}
1315        logger.info("Retrieving rankings information for date {0}.".format(date))
1316        res = requests.get(url, headers=headers, **self._request_params)
1317        if res.ok:
1318            logger.info("Rankings retrieval successful.")
1319            return res.json()
1320        else:
1321            error_msg = self._try_extract_error_code(res)
1322            raise BoostedAPIException("Failed to retrieve rankings: {0}.".format(error_msg))
1323
1324    def sendModelRecalc(self, model_id):
1325        url = self.base_uri + "/api/models/{0}/recalc".format(model_id)
1326        logger.info("Sending model recalc request for model {0}".format(model_id))
1327        headers = {"Authorization": "ApiKey " + self.api_key}
1328        res = requests.put(url, headers=headers, **self._request_params)
1329        if not res.ok:
1330            error_msg = self._try_extract_error_code(res)
1331            logger.error(error_msg)
1332            raise BoostedAPIException(
1333                "Failed to send model recalc request - "
1334                + "the model in UI may be out of date: {0}.".format(error_msg)
1335            )
1336
1337    def sendRecalcAllModelPortfolios(self, model_id: str):
1338        """Recalculates all portfolios under a given model ID.
1339
1340        Args:
1341            model_id: the model ID
1342        Raises:
1343            BoostedAPIException: if the Boosted API request fails
1344        """
1345        url = self.base_uri + f"/api/models/{model_id}/recalc-all-portfolios"
1346        logger.info(f"Sending portfolio recalc requests for all portfolios under {model_id=}.")
1347        headers = {"Authorization": "ApiKey " + self.api_key}
1348        res = requests.put(url, headers=headers, **self._request_params)
1349        if not res.ok:
1350            error_msg = self._try_extract_error_code(res)
1351            logger.error(error_msg)
1352            raise BoostedAPIException(
1353                f"Failed to send recalc request for all portfolios under {model_id=} - {error_msg}."
1354            )
1355
1356    def sendPortfolioRecalc(self, portfolio_id: str):
1357        """Recalculates a single portfolio by its portfolio ID.
1358
1359        Args:
1360            portfolio_id: the portfolio ID to recalculate
1361        Raises:
1362            BoostedAPIException: if the Boosted API request fails
1363        """
1364        url = self.base_uri + "/api/graphql"
1365        logger.info(f"Sending portfolio recalc request for {portfolio_id=}.")
1366        headers = {"Authorization": "ApiKey " + self.api_key, "Content-Type": "application/json"}
1367        qry = """
1368            mutation recalcPortfolio($input: RecalculatePortfolioInput!) {
1369                recalculatePortfolio(input: $input) {
1370                    success
1371                    errors
1372                }
1373            }
1374            """
1375        req_json = {
1376            "query": qry,
1377            "variables": {"input": {"portfolioId": f"{portfolio_id}", "allowForceRecalc": "true"}},
1378        }
1379        res = requests.post(url, json=req_json, headers=headers, **self._request_params)
1380        if not res.ok or res.json().get("errors"):
1381            error_msg = self._try_extract_error_code(res)
1382            logger.error(error_msg)
1383            raise BoostedAPIException(
1384                f"Failed to send portfolio recalc request for {portfolio_id=} - {error_msg}."
1385            )
1386
1387    def add_uploaded_model_data(self, url, csv_data, request_data, timeout=600):
1388        logger.info("Starting upload.")
1389        headers = {"Authorization": "ApiKey " + self.api_key}
1390        files_req: Dict = {}
1391        target: Tuple[str, Any, str] = ("data.csv", None, "text/csv")
1392        warnings = []
1393        if isinstance(csv_data, pd.core.frame.DataFrame):
1394            buf = io.StringIO()
1395            csv_data.to_csv(buf, header=False)
1396            if not isinstance(csv_data.index, pd.core.indexes.datetimes.DatetimeIndex):
1397                raise BoostedAPIException("DataFrame must have DatetimeIndex as index type.")
1398            target = ("uploaded_data.csv", buf.getvalue(), "text/csv")
1399            files_req["dataFile"] = target
1400            res = requests.post(
1401                url,
1402                files=files_req,
1403                data=request_data,
1404                headers=headers,
1405                timeout=timeout,
1406                **self._request_params,
1407            )
1408        elif isinstance(csv_data, str):
1409            target = ("uploaded_data.csv", csv_data, "text/csv")
1410            files_req["dataFile"] = target
1411            res = requests.post(
1412                url,
1413                files=files_req,
1414                data=request_data,
1415                headers=headers,
1416                timeout=timeout,
1417                **self._request_params,
1418            )
1419        else:
1420            raise BoostedAPIException("Expected CSV as str or Pandas DataFrame.")
1421        if res.ok:
1422            logger.info("Signals upload completed.")
1423            result = res.json()["result"]
1424            if "warningMessages" in result:
1425                warnings = result["warningMessages"]
1426        else:
1427            error_str = "Signals upload failed: {0}, {1}".format(res.text, res.reason)
1428            logger.error(error_str)
1429            raise BoostedAPIException(error_str)
1430
1431        return res, warnings
1432
1433    def createSignalsModel(self, csv_data, model_name, timeout=600):
1434        warnings = []
1435        url = self.base_uri + "/api/models/upload/signals/create"
1436        request_data = {"modelName": model_name, "uploadName": model_name}
1437        res, warnings = self.add_uploaded_model_data(url, csv_data, request_data, timeout)
1438        result = res.json()["result"]
1439        model_id = result["modelId"]
1440        self.sendModelRecalc(model_id)
1441        return model_id, warnings
1442
1443    def addToUploadedModel(self, model_id, csv_data, timeout=600, recalc_model=True):
1444        warnings = []
1445        url = self.base_uri + "/api/models/{0}/upload/add-data".format(model_id)
1446        request_data: Dict = {}
1447        _, warnings = self.add_uploaded_model_data(url, csv_data, request_data, timeout)
1448        if recalc_model:
1449            self.sendModelRecalc(model_id)
1450        return warnings
1451
1452    def addSignalsToUploadedModel(
1453        self,
1454        model_id: str,
1455        csv_data: Union[pd.core.frame.DataFrame, str],
1456        timeout: int = 600,
1457        recalc_all: bool = False,
1458        recalc_portfolio_ids: Optional[List[str]] = None,
1459    ) -> List[str]:
1460        """
1461        Add signals to an uploaded model and then recalculate a random portfolio under that model.
1462
1463        Args:
1464            model_id: model ID
1465            csv_data: pandas DataFrame, or a string with signals to upload.
1466            timeout (optional): Timeout for initial upload request in seconds.
1467            recalc_all (optional): if True, recalculates all portfolios in the model.
1468            recalc_portfolio_ids (optional): List of portfolio IDs under the model to re-calculate.
1469        """
1470        warnings = self.addToUploadedModel(model_id, csv_data, timeout, recalc_model=False)
1471
1472        if recalc_all:
1473            self.sendRecalcAllModelPortfolios(model_id)
1474        elif recalc_portfolio_ids:
1475            for portfolio_id in recalc_portfolio_ids:
1476                self.sendPortfolioRecalc(portfolio_id)
1477        else:
1478            self.sendModelRecalc(model_id)
1479        return warnings
1480
1481    def getSignalsFromUploadedModel(self, model_id, date=None):
1482        date = self.__iso_format(date)
1483        url = self.base_uri + "/api/models/{0}/upload/signals".format(model_id)
1484        headers = {"Authorization": "ApiKey " + self.api_key}
1485        params = {"date": date}
1486        logger.info("Retrieving uploaded signals information")
1487        res = requests.get(url, params=params, headers=headers, **self._request_params)
1488        if res.ok:
1489            result = pd.DataFrame.from_dict(res.json()["result"])
1490            # ensure column order
1491            result = result[["date", "isin", "country", "currency", "weight"]]
1492            result["date"] = pd.to_datetime(result["date"], format="%Y-%m-%d")
1493            result = result.set_index("date")
1494            logger.info("Signals retrieval successful.")
1495            return result
1496        else:
1497            error_msg = self._try_extract_error_code(res)
1498            raise BoostedAPIException("Failed to retrieve signals: {0}.".format(error_msg))
1499
1500    def getPortfolioSettings(self, portfolio_id, timeout=600):
1501        url = self.base_uri + "/api/portfolio-settings/{0}".format(portfolio_id)
1502        headers = {"Authorization": "ApiKey " + self.api_key}
1503        res = requests.get(url, headers=headers, **self._request_params)
1504        if res.ok:
1505            return PortfolioSettings(res.json())
1506        else:
1507            error_msg = self._try_extract_error_code(res)
1508            logger.error(error_msg)
1509            raise BoostedAPIException(
1510                "Failed to retrieve portfolio settings: {0}.".format(error_msg)
1511            )
1512
1513    def createPortfolioWithPortfolioSettings(
1514        self, model_id, portfolio_name, portfolio_description, portfolio_settings, timeout=600
1515    ):
1516        url = self.base_uri + "/api/models/{0}/constraints/add".format(model_id)
1517        headers = {"Authorization": "ApiKey " + self.api_key, "Content-Type": "application/json"}
1518        setting_string = json.dumps(portfolio_settings.settings)
1519        logger.info("Creating new portfolio with specified setting: {}".format(setting_string))
1520        params = {
1521            "name": portfolio_name,
1522            "description": portfolio_description,
1523            "constraints": setting_string,
1524            "validate": "true",
1525        }
1526        res = requests.put(url, json=params, headers=headers, **self._request_params)
1527        response = res.json()
1528        if res.ok:
1529            return response
1530        else:
1531            error_msg = self._try_extract_error_code(res)
1532            logger.error(error_msg)
1533            raise BoostedAPIException(
1534                "Failed to create portfolio with the specified settings: {0}.".format(error_msg)
1535            )
1536
1537    def getGbiIdFromIdentCountryCurrencyDate(
1538        self, ident_country_currency_dates: List[DateIdentCountryCurrency], timeout: int = 600
1539    ) -> List[Optional[GbiIdSecurity]]:
1540        url = self.base_uri + "/api/custom-stock-data/map-identifiers-simple"
1541        headers = {"Authorization": "ApiKey " + self.api_key, "Content-Type": "application/json"}
1542        identifiers = [
1543            {
1544                "row": idx,
1545                "date": identifier.date,
1546                "isin": identifier.identifier if identifier.id_type == ColumnSubRole.ISIN else None,
1547                "symbol": (
1548                    identifier.identifier if identifier.id_type == ColumnSubRole.SYMBOL else None
1549                ),
1550                "countryPreference": identifier.country,
1551                "currencyPreference": identifier.currency,
1552            }
1553            for idx, identifier in enumerate(ident_country_currency_dates)
1554        ]
1555        params = json.dumps({"identifiers": identifiers})
1556        logger.info(
1557            "Retrieving GBI-ID mapping for {} identifier tuples...".format(
1558                len(ident_country_currency_dates)
1559            )
1560        )
1561        res = requests.post(url, data=params, headers=headers, **self._request_params)
1562
1563        if res.ok:
1564            result = res.json()
1565            warnings = result["warnings"]
1566            if warnings:
1567                for warning in warnings:
1568                    logger.warn(f"Mapping warning: {warning}")
1569            gbiSecurities = []
1570            for idx, ident in enumerate(result["mappedIdentifiers"]):
1571                if ident is None:
1572                    security = None
1573                else:
1574                    security = GbiIdSecurity(
1575                        ident["gbiId"],
1576                        ident_country_currency_dates[idx],
1577                        ident["symbol"],
1578                        ident["companyName"],
1579                    )
1580                gbiSecurities.append(security)
1581
1582            return gbiSecurities
1583        else:
1584            error_msg = self._try_extract_error_code(res)
1585            raise BoostedAPIException(
1586                "Failed to retrieve identifier mappings: {0}.".format(error_msg)
1587            )
1588
1589    # exists for backwards compatibility purposes.
1590    def getGbiIdFromIsinCountryCurrencyDate(self, isin_country_currency_dates, timeout=600):
1591        return self.getGbiIdFromIdentCountryCurrencyDate(
1592            ident_country_currency_dates=isin_country_currency_dates, timeout=timeout
1593        )
1594
1595    # model_id: str
1596    # returns: Dict[str, str] representing the translation from the rankings ID (feature refs)
1597    # to human readable names
1598    def __get_rankings_ref_translation(self, model_id: str) -> Dict[str, str]:
1599        headers = {"Authorization": "ApiKey " + self.api_key, "Content-Type": "application/json"}
1600        feature_name_url = f"/api/models/{model_id}/advanced-explain/translate-feature-ref/"
1601        feature_name_res = requests.post(
1602            self.base_uri + feature_name_url,
1603            data=json.dumps({}),
1604            headers=headers,
1605            **self._request_params,
1606        )
1607
1608        if feature_name_res.ok:
1609            feature_name_dict = feature_name_res.json()
1610            return {
1611                id: "-".join(
1612                    [names["variable_name"], names["transform_name"], names["normalization_name"]]
1613                )
1614                for id, names in feature_name_dict.items()
1615            }
1616        else:
1617            raise Exception(
1618                """Failed to get feature names for model,
1619                    this model doesn't fully support rankings 2.0"""
1620            )
1621
1622    def getDatasetDates(self, dataset_id):
1623        url = self.base_uri + f"/api/datasets/{dataset_id}"
1624        headers = {"Authorization": "ApiKey " + self.api_key}
1625        res = requests.get(url, headers=headers, **self._request_params)
1626        if res.ok:
1627            dataset = res.json()
1628            valid_to_array = dataset.get("validTo")
1629            valid_to_date = None
1630            valid_from_array = dataset.get("validFrom")
1631            valid_from_date = None
1632            if valid_to_array:
1633                valid_to_date = datetime.date(
1634                    valid_to_array[0], valid_to_array[1], valid_to_array[2]
1635                )
1636            if valid_from_array:
1637                valid_from_date = datetime.date(
1638                    valid_from_array[0], valid_from_array[1], valid_from_array[2]
1639                )
1640            return {"validTo": valid_to_date, "validFrom": valid_from_date}
1641        else:
1642            error_msg = self._try_extract_error_code(res)
1643            logger.error(error_msg)
1644            raise BoostedAPIException("Failed to query dataset: {0}.".format(error_msg))
1645
1646    def getRankingAnalysis(self, model_id, date):
1647        url = (
1648            self.base_uri
1649            + f"/api/explain-trades/analysis/{model_id}/{self.__iso_format(date)}/json"
1650        )
1651        headers = {"Authorization": "ApiKey " + self.api_key}
1652        analysis_res = requests.get(url, headers=headers, **self._request_params)
1653        if analysis_res.ok:
1654            ranking_dict = analysis_res.json()
1655            feature_name_dict = self.__get_rankings_ref_translation(model_id)
1656            columns = [feature_name_dict[col] for col in ranking_dict["columns"]]
1657
1658            df = protoCubeJsonDataToDataFrame(
1659                ranking_dict["data"],
1660                "Data Buckets",
1661                ranking_dict["rows"],
1662                "Feature Names",
1663                columns,
1664                ranking_dict["fields"],
1665            )
1666            return df
1667        else:
1668            error_msg = self._try_extract_error_code(analysis_res)
1669            logger.error(error_msg)
1670            raise BoostedAPIException("Failed to get ranking analysis: {0}.".format(error_msg))
1671
1672    def getExplainForPortfolio(
1673        self,
1674        model_id,
1675        portfolio_id,
1676        date,
1677        index_by_symbol: bool = False,
1678        index_by_all_metadata: bool = False,
1679    ):
1680        """
1681        Gets the ranking 2.0 explain data for the given model on the given date
1682        filtered by portfolio.
1683
1684        Parameters
1685        ----------
1686        model_id: str
1687            Model ID.  Model IDs can be retrieved by clicking on the copy to clipboard
1688            button next to your model's name in the Model Summary Page in Boosted
1689            Insights.
1690        portfolio_id: str
1691            Portfolio ID.  Portfolio IDs can be retrieved from portfolio's configuration page.
1692        date: datetime.date or YYYY-MM-DD string
1693            Date of the data to retrieve.
1694        index_by_symbol: bool
1695            If true, index by stock symbol instead of ISIN.
1696        index_by_all_metadata: bool
1697            If true, index by all metadata: ISIN, stock symbol, currency, and country.
1698            Overrides index_by_symbol.
1699
1700        Returns
1701        -------
1702        pandas.DataFrame
1703            Pandas DataFrame containing your data indexed by ISINs/Symbol/all metadata
1704            and feature names, filtered by portfolio.
1705        ___
1706        """
1707        indices = ["Symbol", "ISINs", "Country", "Currency"]
1708        raw_explain_df = self.getRankingExplain(
1709            model_id, date, index_by_symbol=False, index_by_all_metadata=True
1710        )
1711        pa_ratings_dict = self.getRankingsForDate(portfolio_id, date, False)
1712
1713        ratings = pa_ratings_dict["rankings"]
1714        ratings_df = pd.DataFrame(ratings)
1715        ratings_df = ratings_df[["symbol", "isin", "country", "currency"]]
1716        ratings_df.columns = pd.Index(indices)
1717        ratings_df.set_index(indices, inplace=True)
1718
1719        # inner join to only get the securities in both data frames
1720        result_df = raw_explain_df.merge(ratings_df, left_index=True, right_index=True, how="inner")
1721
1722        # set index based on input parameters
1723        if index_by_symbol and not index_by_all_metadata:
1724            result_df = result_df.reset_index()
1725            result_df = result_df.drop(columns=["ISINs", "Currency", "Country"])
1726            result_df.set_index(["Symbol", "Feature Names"], inplace=True)
1727        elif not index_by_symbol and not index_by_all_metadata:
1728            result_df = result_df.reset_index()
1729            result_df = result_df.drop(columns=["Symbol", "Currency", "Country"])
1730            result_df.set_index(["ISINs", "Feature Names"], inplace=True)
1731
1732        return result_df
1733
1734    def getRankingExplain(
1735        self, model_id, date, index_by_symbol: bool = False, index_by_all_metadata: bool = False
1736    ):
1737        """
1738        Gets the ranking 2.0 explain data for the given model on the given date
1739
1740        Parameters
1741        ----------
1742        model_id: str
1743            Model ID.  Model IDs can be retrieved by clicking on the copy to clipboard
1744            button next to your model's name in the Model Summary Page in Boosted
1745            Insights.
1746        date: datetime.date or YYYY-MM-DD string
1747            Date of the data to retrieve.
1748        index_by_symbol: bool
1749            If true, index by stock symbol instead of ISIN.
1750        index_by_all_metadata: bool
1751            If true, index by all metadata: ISIN, stock symbol, currency, and country.
1752            Overrides index_by_symbol.
1753
1754        Returns
1755        -------
1756        pandas.DataFrame
1757            Pandas DataFrame containing your data indexed by ISINs/Symbol/all metadata
1758            and feature names.
1759        ___
1760        """
1761        url = (
1762            self.base_uri + f"/api/explain-trades/explain/{model_id}/{self.__iso_format(date)}/json"
1763        )
1764        headers = {"Authorization": "ApiKey " + self.api_key, "Content-Type": "application/json"}
1765        explain_res = requests.get(url, headers=headers, **self._request_params)
1766        if explain_res.ok:
1767            ranking_dict = explain_res.json()
1768            rows = ranking_dict["rows"]
1769            stock_summary_url = f"/api/stock-summaries/{model_id}"
1770            stock_summary_body = {"gbiIds": ranking_dict["rows"]}
1771            summary_res = requests.post(
1772                self.base_uri + stock_summary_url,
1773                data=json.dumps(stock_summary_body),
1774                headers=headers,
1775                **self._request_params,
1776            )
1777            if summary_res.ok:
1778                stock_summary = summary_res.json()
1779                if index_by_symbol:
1780                    rows = [stock_summary[row]["symbol"] for row in ranking_dict["rows"]]
1781                elif index_by_all_metadata:
1782                    rows = [
1783                        [
1784                            stock_summary[row]["isin"],
1785                            stock_summary[row]["symbol"],
1786                            stock_summary[row]["currency"],
1787                            stock_summary[row]["country"],
1788                        ]
1789                        for row in ranking_dict["rows"]
1790                    ]
1791                else:
1792                    rows = [stock_summary[row]["isin"] for row in ranking_dict["rows"]]
1793            else:
1794                error_msg = self._try_extract_error_code(summary_res)
1795                logger.error(error_msg)
1796                raise BoostedAPIException(
1797                    "Failed to get isin information ranking explain: {0}.".format(error_msg)
1798                )
1799
1800            feature_name_dict = self.__get_rankings_ref_translation(model_id)
1801            columns = [feature_name_dict[col] for col in ranking_dict["columns"]]
1802
1803            id_col_name = "Symbols" if index_by_symbol else "ISINs"
1804
1805            if index_by_all_metadata:
1806                pc_list = []
1807                pf = ranking_dict["data"]
1808                for row_idx, row in enumerate(rows):
1809                    for col_idx, col in enumerate(columns):
1810                        pc_list.append([row, col] + pf[row_idx]["columns"][col_idx]["fields"])
1811                df = pd.DataFrame(pc_list)
1812                df = df.set_axis(
1813                    ["Metadata", "Feature Names"] + ranking_dict["fields"], axis="columns"
1814                )
1815
1816                metadata_df = df["Metadata"].apply(pd.Series)
1817                metadata_df.columns = pd.Index(["ISINs", "Symbol", "Currency", "Country"])
1818                result_df = pd.concat([metadata_df, df], axis=1).drop("Metadata", axis=1)
1819                result_df.set_index(
1820                    ["ISINs", "Symbol", "Currency", "Country", "Feature Names"], inplace=True
1821                )
1822                return result_df
1823
1824            else:
1825                df = protoCubeJsonDataToDataFrame(
1826                    ranking_dict["data"],
1827                    id_col_name,
1828                    rows,
1829                    "Feature Names",
1830                    columns,
1831                    ranking_dict["fields"],
1832                )
1833
1834                return df
1835        else:
1836            error_msg = self._try_extract_error_code(explain_res)
1837            logger.error(error_msg)
1838            raise BoostedAPIException("Failed to get ranking explain: {0}.".format(error_msg))
1839
1840    def getDenseSignalsForDate(self, portfolio_id, date, rollback_to_last_available_date):
1841        date = self.__iso_format(date)
1842        url = self.base_uri + f"/api/portfolios/{portfolio_id}/denseSignalsByDate"
1843        headers = {"Authorization": "ApiKey " + self.api_key}
1844        params = {
1845            "startDate": date,
1846            "endDate": date,
1847            "rollbackToMostRecentDate": rollback_to_last_available_date,
1848        }
1849        logger.info("Retrieving dense signals information for date {0}.".format(date))
1850        res = requests.get(url, params=params, headers=headers, **self._request_params)
1851        if res.ok:
1852            logger.info("Signals retrieval successful.")
1853            d = res.json()
1854            # reshape date to output format
1855            date = list(d["signals"].keys())[0]
1856            model_id = d["model_id"]
1857            signals_list = list(d["signals"].values())[0]
1858            return {"date": date, "signals": [{"model_id": model_id, "signals_info": signals_list}]}
1859        else:
1860            error_msg = self._try_extract_error_code(res)
1861            raise BoostedAPIException("Failed to retrieve dense signals: {0}.".format(error_msg))
1862
1863    def getDenseSignals(self, model_id, portfolio_id, file_name=None, location="./"):
1864        url = self.base_uri + f"/api/models/{model_id}/{portfolio_id}/dense-signals"
1865        headers = {"Authorization": "ApiKey " + self.api_key}
1866        res = requests.get(url, headers=headers, **self._request_params)
1867        if file_name is None:
1868            file_name = f"{model_id}-{portfolio_id}_dense_signals.csv"
1869        download_location = os.path.join(location, file_name)
1870        if res.ok:
1871            with open(download_location, "wb") as file:
1872                file.write(res.content)
1873            print("Download Complete")
1874        elif res.status_code == 404:
1875            raise BoostedAPIException(
1876                f"""Dense Singals file does not exist for model:
1877                 {model_id} - portfolio: {portfolio_id}"""
1878            )
1879        else:
1880            error_msg = self._try_extract_error_code(res)
1881            logger.error(error_msg)
1882            raise BoostedAPIException(
1883                f"""Failed to download dense singals file for model:
1884                 {model_id} - portfolio: {portfolio_id}"""
1885            )
1886
1887    def _getIsPortfolioReadyForProcessing(self, model_id, portfolio_id, formatted_date):
1888        headers = {"Authorization": "ApiKey " + self.api_key}
1889        url = (
1890            self.base_uri
1891            + f"/api/explain-trades/{model_id}/{portfolio_id}"
1892            + f"/is-ready-for-processing/{formatted_date}"
1893        )
1894        res = requests.get(url, headers=headers, **self._request_params)
1895
1896        try:
1897            if res.ok:
1898                body = res.json()
1899                if "ready" in body:
1900                    if body["ready"]:
1901                        return True, ""
1902                    else:
1903                        reason_from_api = (
1904                            body["notReadyReason"] if "notReadyReason" in body else "Unavailable"
1905                        )
1906
1907                        returned_reason = reason_from_api
1908
1909                        if returned_reason == "SKIP":
1910                            returned_reason = "holiday- market closed"
1911
1912                        if returned_reason == "WAITING":
1913                            returned_reason = "calculations pending"
1914
1915                        return False, returned_reason
1916                else:
1917                    return False, "Unavailable"
1918            else:
1919                error_msg = self._try_extract_error_code(res)
1920                logger.error(error_msg)
1921                raise BoostedAPIException(
1922                    f"""Failed to generate file for model:
1923                    {model_id} - portfolio: {portfolio_id} on date: {formatted_date}"""
1924                )
1925        except Exception as e:
1926            raise BoostedAPIException(
1927                f"""Failed to generate file for model:
1928                {model_id} - portfolio: {portfolio_id} on date: {formatted_date} {e}"""
1929            )
1930
1931    def getRanking2DateAnalysisFile(
1932        self, model_id, portfolio_id, date, file_name=None, location="./"
1933    ):
1934        formatted_date = self.__iso_format(date)
1935        s3_file_name = f"{formatted_date}_analysis.xlsx"
1936        download_url = (
1937            self.base_uri + f"/api/models/{model_id}/{portfolio_id}/ranking-file/{s3_file_name}"
1938        )
1939        headers = {"Authorization": "ApiKey " + self.api_key}
1940        if file_name is None:
1941            file_name = f"{model_id}-{portfolio_id}_statistical_analysis_{formatted_date}.xlsx"
1942        download_location = os.path.join(location, file_name)
1943
1944        res = requests.get(download_url, headers=headers, **self._request_params)
1945        if res.ok:
1946            with open(download_location, "wb") as file:
1947                file.write(res.content)
1948            print("Download Complete")
1949        elif res.status_code == 404:
1950            (
1951                is_portfolio_ready_for_processing,
1952                portfolio_ready_status,
1953            ) = self._getIsPortfolioReadyForProcessing(model_id, portfolio_id, formatted_date)
1954
1955            if not is_portfolio_ready_for_processing:
1956                logger.info(
1957                    f"""\nPortfolio {portfolio_id} for model {model_id}
1958                    on date {date} unavailable for Ranking2Date Analysis file.
1959                    Status: {portfolio_ready_status}\n"""
1960                )
1961                return
1962
1963            generate_url = (
1964                self.base_uri
1965                + f"/api/explain-trades/{model_id}/{portfolio_id}"
1966                + f"/generate/date-data/{formatted_date}"
1967            )
1968
1969            generate_res = requests.get(generate_url, headers=headers, **self._request_params)
1970            if generate_res.ok:
1971                download_res = requests.get(download_url, headers=headers, **self._request_params)
1972                while download_res.status_code == 404 or (
1973                    download_res.ok and len(download_res.content) == 0
1974                ):
1975                    print("waiting for file to be generated")
1976                    time.sleep(5)
1977                    download_res = requests.get(
1978                        download_url, headers=headers, **self._request_params
1979                    )
1980                if download_res.ok:
1981                    with open(download_location, "wb") as file:
1982                        file.write(download_res.content)
1983                    print("Download Complete")
1984            else:
1985                error_msg = self._try_extract_error_code(res)
1986                logger.error(error_msg)
1987                raise BoostedAPIException(
1988                    f"""Failed to generate ranking analysis file for model:
1989                    {model_id} - portfolio: {portfolio_id} on date: {formatted_date}"""
1990                )
1991        else:
1992            error_msg = self._try_extract_error_code(res)
1993            logger.error(error_msg)
1994            raise BoostedAPIException(
1995                f"""Failed to download ranking analysis file for model:
1996                 {model_id} - portfolio: {portfolio_id} on date: {formatted_date}"""
1997            )
1998
1999    def getRanking2DateExplainFile(
2000        self,
2001        model_id,
2002        portfolio_id,
2003        date,
2004        file_name=None,
2005        location="./",
2006        overwrite: bool = False,
2007        index_by_all_metadata: bool = False,
2008    ):
2009        """
2010        Downloads the ranking explain file for the provided portfolio and model.
2011        If no file exists then it will send a request to generate the file and continuously
2012        poll the server every 5 seconds to try and download the file until the file is downloaded.
2013
2014        Parameters
2015        ----------
2016        model_id: str
2017            Model ID.  Model IDs can be retrieved by clicking on the copy to clipboard
2018            button next to your model's name in the Model Summary Page in Boosted
2019            Insights.
2020        portfolio_id: str
2021            Portfolio ID.  Portfolio IDs can be retrieved from portfolio's configuration page.
2022        date: datetime.date or YYYY-MM-DD string
2023            Date of the data to retrieve.
2024        file_name: str
2025            File name of the dense signals file to save as.
2026            If no file name is given the file name will be
2027            "<model_id>-<portfolio_id>_explain_data_<date>.xlsx"
2028        location: str
2029            The location to save the file to.
2030            If no location is given then it will be saved to the current directory.
2031        overwrite: bool
2032            Defaults to False, set to True to regenerate the file.
2033        index_by_all_metadata: bool
2034            If true, index by all metadata: ISIN, stock symbol, currency, and country.
2035
2036
2037        Returns
2038        -------
2039        None
2040        ___
2041        """
2042        formatted_date = self.__iso_format(date)
2043        if index_by_all_metadata:
2044            s3_file_name = f"{formatted_date}_explaindata_withmetadata.xlsx"
2045        else:
2046            s3_file_name = f"{formatted_date}_explaindata.xlsx"
2047        download_url = (
2048            self.base_uri + f"/api/models/{model_id}/{portfolio_id}/ranking-file/{s3_file_name}"
2049        )
2050        headers = {"Authorization": "ApiKey " + self.api_key}
2051        if file_name is None:
2052            file_name = f"{model_id}-{portfolio_id}_explain_data_{formatted_date}.xlsx"
2053        download_location = os.path.join(location, file_name)
2054
2055        if not overwrite:
2056            res = requests.get(download_url, headers=headers, **self._request_params)
2057        if not overwrite and res.ok:
2058            with open(download_location, "wb") as file:
2059                file.write(res.content)
2060            print("Download Complete")
2061        elif overwrite or res.status_code == 404:
2062            (
2063                is_portfolio_ready_for_processing,
2064                portfolio_ready_status,
2065            ) = self._getIsPortfolioReadyForProcessing(model_id, portfolio_id, formatted_date)
2066
2067            if not is_portfolio_ready_for_processing:
2068                logger.info(
2069                    f"""\nPortfolio {portfolio_id} for model {model_id}
2070                    on date {date} unavailable for Ranking2Date Explain file.
2071                    Status: {portfolio_ready_status}\n"""
2072                )
2073                return
2074
2075            generate_url = (
2076                self.base_uri
2077                + f"/api/explain-trades/{model_id}/{portfolio_id}"
2078                + f"/generate/date-data/{formatted_date}"
2079                + f"/{'true' if index_by_all_metadata else 'false'}"
2080            )
2081
2082            generate_res = requests.get(generate_url, headers=headers, **self._request_params)
2083            if generate_res.ok:
2084                download_res = requests.get(download_url, headers=headers, **self._request_params)
2085                while download_res.status_code == 404 or (
2086                    download_res.ok and len(download_res.content) == 0
2087                ):
2088                    print("waiting for file to be generated")
2089                    time.sleep(5)
2090                    download_res = requests.get(
2091                        download_url, headers=headers, **self._request_params
2092                    )
2093                if download_res.ok:
2094                    with open(download_location, "wb") as file:
2095                        file.write(download_res.content)
2096                    print("Download Complete")
2097            else:
2098                error_msg = self._try_extract_error_code(res)
2099                logger.error(error_msg)
2100                raise BoostedAPIException(
2101                    f"""Failed to generate ranking explain file for model:
2102                    {model_id} - portfolio: {portfolio_id} on date: {formatted_date}"""
2103                )
2104        else:
2105            error_msg = self._try_extract_error_code(res)
2106            logger.error(error_msg)
2107            raise BoostedAPIException(
2108                f"""Failed to download ranking explain file for model:
2109                 {model_id} - portfolio: {portfolio_id} on date: {formatted_date}"""
2110            )
2111
2112    def getRanking2DateExplain(
2113        self,
2114        model_id: str,
2115        portfolio_id: str,
2116        date: Optional[datetime.date],
2117        overwrite: bool = False,
2118    ) -> Dict[str, pd.DataFrame]:
2119        """
2120        Wrapper around getRanking2DateExplainFile, but returns a pandas
2121        dataframe instead of downloading to a path. Dataframe is indexed by
2122        symbol and should always have 'rating' and 'rating_delta' columns. Other
2123        columns will be determined by model's features.
2124        """
2125        file_name = "explaindata.xlsx"
2126        with tempfile.TemporaryDirectory() as tmpdirname:
2127            self.getRanking2DateExplainFile(
2128                model_id=model_id,
2129                portfolio_id=portfolio_id,
2130                date=date,
2131                file_name=file_name,
2132                location=tmpdirname,
2133                overwrite=overwrite,
2134            )
2135            full_path = os.path.join(tmpdirname, file_name)
2136            excel_file = pd.ExcelFile(full_path)
2137            df_map = pd.read_excel(excel_file, sheet_name=None)
2138            df_map_final = {str(sheet): df.set_index("Symbol") for (sheet, df) in df_map.items()}
2139
2140        return df_map_final
2141
2142    def getTearSheet(self, model_id, portfolio_id, start_date=None, end_date=None, block=False):
2143        if start_date is None or end_date is None:
2144            if start_date is not None or end_date is not None:
2145                raise ValueError("start_date and end_date must both be None or both be defined")
2146            return self._getCurrentTearSheet(model_id, portfolio_id)
2147
2148        start_date_obj = self.__to_date_obj(start_date)
2149        end_date_obj = self.__to_date_obj(end_date)
2150        if start_date_obj >= end_date_obj:
2151            raise ValueError("end_date must be later than the start_date")
2152
2153        # get for the given date
2154        url = self.base_uri + f"/api/analysis/keyfacts/{model_id}/{portfolio_id}"
2155        data = {
2156            "startDate": self.__iso_format(start_date),
2157            "endDate": self.__iso_format(end_date),
2158            "shouldRecalc": True,
2159        }
2160        headers = {"Authorization": "ApiKey " + self.api_key, "Content-Type": "application/json"}
2161        res = requests.post(url, data=json.dumps(data), headers=headers, **self._request_params)
2162        if res.status_code == 404 and block:
2163            retries = 0
2164            data["shouldRecalc"] = False
2165            while retries < 10:
2166                time.sleep(10)
2167                retries += 1
2168                res = requests.post(
2169                    url, data=json.dumps(data), headers=headers, **self._request_params
2170                )
2171                if res.status_code != 404:
2172                    break
2173        if res.ok:
2174            return res.json()
2175        else:
2176            error_msg = self._try_extract_error_code(res)
2177            logger.error(error_msg)
2178            raise BoostedAPIException(
2179                "Failed to get tear sheet data: {0} {1}.".format(error_msg, str(res.status_code))
2180            )
2181
2182    def _getCurrentTearSheet(self, model_id, portfolio_id):
2183        url = self.base_uri + f"/api/model-summaries/{model_id}/{portfolio_id}"
2184        headers = {"Authorization": "ApiKey " + self.api_key, "Content-Type": "application/json"}
2185        res = requests.get(url, headers=headers, **self._request_params)
2186        if res.ok:
2187            json = res.json()
2188            return json.get("tearSheet", {})
2189        else:
2190            error_msg = self._try_extract_error_code(res)
2191            logger.error(error_msg)
2192            raise BoostedAPIException("Failed to get tear sheet data: {0}.".format(error_msg))
2193
2194    def getPortfolioStatus(self, model_id, portfolio_id, job_date):
2195        url = (
2196            self.base_uri
2197            + f"/api/analysis/portfolioStatus/{model_id}/{portfolio_id}?jobDate={job_date}"
2198        )
2199        headers = {"Authorization": "ApiKey " + self.api_key, "Content-Type": "application/json"}
2200        res = requests.get(url, headers=headers, **self._request_params)
2201        if res.ok:
2202            result = res.json()
2203            return {
2204                "is_complete": result["status"],
2205                "last_update": None if result["lastUpdate"] is None else result["lastUpdate"][:10],
2206                "next_update": None if result["nextUpdate"] is None else result["nextUpdate"][:10],
2207            }
2208        else:
2209            error_msg = self._try_extract_error_code(res)
2210            logger.error(error_msg)
2211            raise BoostedAPIException("Failed to get portfolio status: {0}".format(error_msg))
2212
2213    def _query_portfolio_factor_attribution(
2214        self,
2215        portfolio_id: str,
2216        start_date: Optional[BoostedDate] = None,
2217        end_date: Optional[BoostedDate] = None,
2218    ):
2219        response = self._get_graphql(
2220            query=graphql_queries.GET_PORTFOLIO_FACTOR_ATTRIBUTION_QUERY,
2221            variables={
2222                "portfolioId": portfolio_id,
2223                "startDate": str(start_date) if start_date else None,
2224                "endDate": str(end_date) if start_date else None,
2225            },
2226            error_msg_prefix="Failed to get factor attribution: ",
2227        )
2228        return response
2229
2230    def get_portfolio_factor_attribution(
2231        self,
2232        portfolio_id: str,
2233        start_date: Optional[BoostedDate] = None,
2234        end_date: Optional[BoostedDate] = None,
2235    ):
2236        """Get portfolio factor attribution for a portfolio
2237
2238        Args:
2239            portfolio_id (str): a valid UUID string
2240            start_date (BoostedDate, optional): The start date. Defaults to None.
2241            end_date (BoostedDate, optional): The end date. Defaults to None.
2242        """
2243        response = self._query_portfolio_factor_attribution(portfolio_id, start_date, end_date)
2244        factor_attribution = response["data"]["portfolio"]["factorAttribution"]
2245        dates = pd.DatetimeIndex(data=factor_attribution["dates"])
2246        beta = factor_attribution["factorBetas"]
2247        beta_df = pd.DataFrame(index=dates, data={x["name"]: x["data"] for x in beta})
2248        beta_df = beta_df.add_suffix("_beta")
2249        returns = factor_attribution["portfolioFactorPerformance"]
2250        returns_df = pd.DataFrame(index=dates, data={x["name"]: x["data"] for x in returns})
2251        returns_df = returns_df.add_suffix("_return")
2252        returns_df = (returns_df - 1) * 100
2253
2254        final_df = pd.concat([returns_df, beta_df], axis=1)
2255        ordered_columns = list(itertools.chain(*zip(returns_df.columns, beta_df.columns)))
2256        ordered_final_df = final_df.reindex(columns=ordered_columns)
2257
2258        # Add the column `total_return` which is the sum of returns_data
2259        ordered_final_df["total_return"] = returns_df.sum(axis=1)
2260        return ordered_final_df
2261
2262    def getBlacklist(self, blacklist_id):
2263        url = self.base_uri + f"/api/blacklist/{blacklist_id}"
2264        headers = {"Authorization": "ApiKey " + self.api_key}
2265        res = requests.get(url, headers=headers, **self._request_params)
2266        if res.ok:
2267            result = res.json()
2268            return result
2269        error_msg = self._try_extract_error_code(res)
2270        logger.error(error_msg)
2271        raise BoostedAPIException(f"Failed to get blacklist with id {blacklist_id}: {error_msg}")
2272
2273    def getBlacklists(self, model_id=None, company_id=None, last_N=None):
2274        params = {}
2275        if last_N:
2276            params["lastN"] = last_N
2277        if model_id:
2278            params["modelId"] = model_id
2279        if company_id:
2280            params["companyId"] = company_id
2281        url = self.base_uri + f"/api/blacklist"
2282        headers = {"Authorization": "ApiKey " + self.api_key}
2283        res = requests.get(url, headers=headers, params=params, **self._request_params)
2284        if res.ok:
2285            result = res.json()
2286            return result
2287        error_msg = self._try_extract_error_code(res)
2288        logger.error(error_msg)
2289        raise BoostedAPIException(
2290            f"""Failed to get blacklists with \
2291            model_id {model_id} company_id {company_id} last_N {last_N}: {error_msg}"""
2292        )
2293
2294    def createBlacklist(
2295        self,
2296        isin,
2297        long_short=2,
2298        start_date=datetime.date.today(),
2299        end_date="4000-01-01",
2300        model_id=None,
2301    ):
2302        url = self.base_uri + f"/api/blacklist"
2303        data = {
2304            "modelId": model_id,
2305            "isin": isin,
2306            "longShort": long_short,
2307            "startDate": self.__iso_format(start_date),
2308            "endDate": self.__iso_format(end_date),
2309        }
2310        headers = {"Authorization": "ApiKey " + self.api_key, "Content-Type": "application/json"}
2311        res = requests.post(url, data=json.dumps(data), headers=headers, **self._request_params)
2312        if res.ok:
2313            return res.json()
2314        else:
2315            error_msg = self._try_extract_error_code(res)
2316            logger.error(error_msg)
2317            raise BoostedAPIException(
2318                f"""Failed to create the blacklist with \
2319                  isin {isin} long_short {long_short} start_date {start_date} end_date {end_date} \
2320                  model_id {model_id}: {error_msg}."""
2321            )
2322
2323    def createBlacklistsFromCSV(self, csv_name):
2324        url = self.base_uri + f"/api/blacklists"
2325        data = []
2326        with open(csv_name, mode="r") as f:
2327            csv_reader = csv.DictReader(f)
2328            for row in csv_reader:
2329                blacklist = {"modelId": row["ModelID"], "isin": row["ISIN"]}
2330                if not row.get("LongShort"):
2331                    blacklist["longShort"] = 2
2332                else:
2333                    blacklist["longShort"] = row["LongShort"]
2334
2335                if not row.get("StartDate"):
2336                    blacklist["startDate"] = self.__iso_format(datetime.date.today())
2337                else:
2338                    blacklist["startDate"] = self.__iso_format(row["StartDate"])
2339
2340                if not row.get("EndDate"):
2341                    blacklist["endDate"] = self.__iso_format("4000-01-01")
2342                else:
2343                    blacklist["endDate"] = self.__iso_format(row["EndDate"])
2344                data.append(blacklist)
2345        print(f"Processed {len(data)} blacklists.")
2346        headers = {"Authorization": "ApiKey " + self.api_key, "Content-Type": "application/json"}
2347        res = requests.post(url, data=json.dumps(data), headers=headers, **self._request_params)
2348        if res.ok:
2349            return res.json()
2350        else:
2351            error_msg = self._try_extract_error_code(res)
2352            logger.error(error_msg)
2353            raise BoostedAPIException("failed to create blacklists")
2354
2355    def updateBlacklist(self, blacklist_id, long_short=None, start_date=None, end_date=None):
2356        params = {}
2357        if long_short:
2358            params["longShort"] = long_short
2359        if start_date:
2360            params["startDate"] = start_date
2361        if end_date:
2362            params["endDate"] = end_date
2363        url = self.base_uri + f"/api/blacklist/{blacklist_id}"
2364        headers = {"Authorization": "ApiKey " + self.api_key, "Content-Type": "application/json"}
2365        res = requests.patch(url, json=params, headers=headers, **self._request_params)
2366        if res.ok:
2367            return res.json()
2368        else:
2369            error_msg = self._try_extract_error_code(res)
2370            logger.error(error_msg)
2371            raise BoostedAPIException(
2372                f"Failed to update blacklist with id {blacklist_id}: {error_msg}"
2373            )
2374
2375    def deleteBlacklist(self, blacklist_id):
2376        url = self.base_uri + f"/api/blacklist/{blacklist_id}"
2377        headers = {"Authorization": "ApiKey " + self.api_key}
2378        res = requests.delete(url, headers=headers, **self._request_params)
2379        if res.ok:
2380            result = res.json()
2381            return result
2382        else:
2383            error_msg = self._try_extract_error_code(res)
2384            logger.error(error_msg)
2385            raise BoostedAPIException(
2386                f"Failed to delete blacklist with id {blacklist_id}: {error_msg}"
2387            )
2388
2389    def getFeatureImportance(self, model_id, date, N=None):
2390        url = self.base_uri + f"/api/analysis/explainability/{model_id}"
2391        headers = {"Authorization": "ApiKey " + self.api_key}
2392        logger.info("Retrieving rankings information for date {0}.".format(date))
2393        res = requests.get(url, headers=headers, **self._request_params)
2394        if not res.ok:
2395            error_msg = self._try_extract_error_code(res)
2396            logger.error(error_msg)
2397            raise BoostedAPIException(
2398                f"Failed to fetch feature importance for model/portfolio {model_id}: {error_msg}"
2399            )
2400
2401        json_data = res.json()
2402        if "all" not in json_data.keys() or not json_data["all"]:
2403            raise BoostedAPIException(f"Unexpected formatting of feature importance response")
2404
2405        feature_data = json_data["all"]
2406        # find the right period (assuming returned json has dates in descending order)
2407        date_obj = self.__to_date_obj(date)
2408        start_date_for_return_data = self.__to_date_obj(feature_data[0]["date"])
2409        features_for_requested_period = None
2410
2411        if date_obj > start_date_for_return_data:
2412            features_for_requested_period = feature_data[0]["variable"]
2413        else:
2414            i = 0
2415            while i < len(feature_data) - 1:
2416                current_date = self.__to_date_obj(feature_data[i]["date"])
2417                next_date = self.__to_date_obj(feature_data[i + 1]["date"])
2418                if next_date <= date_obj <= current_date:
2419                    features_for_requested_period = feature_data[i + 1]["variable"]
2420                    start_date_for_return_data = next_date
2421                    break
2422                i += 1
2423
2424        if features_for_requested_period is None:
2425            raise BoostedAPIException(f"No feature data was found for requested date: {date_obj}")
2426
2427        features_for_requested_period.sort(key=lambda x: x["value"], reverse=True)
2428
2429        if type(N) is int and N > 0:
2430            df = pd.DataFrame.from_dict(features_for_requested_period[0:N])
2431        else:
2432            df = pd.DataFrame.from_dict(features_for_requested_period)
2433        result = df[["feature", "value"]]
2434
2435        return result.rename(columns={"feature": f"feature ({start_date_for_return_data})"})
2436
2437    def getAllModelNames(self) -> Dict[str, str]:
2438        url = f"{self.base_uri}/api/graphql"
2439        headers = {"Authorization": "ApiKey " + self.api_key, "Content-Type": "application/json"}
2440        req_json = {"query": "query listOfModels {\n models { id name }}", "variables": {}}
2441        res = requests.post(url, json=req_json, headers=headers, **self._request_params)
2442        if not res.ok:
2443            error_msg = self._try_extract_error_code(res)
2444            logger.error(error_msg)
2445            raise BoostedAPIException(f"Failed to get user models: {error_msg}")
2446        data = res.json()
2447        if data["data"]["models"] is None:
2448            return {}
2449        return {rec["id"]: rec["name"] for rec in data["data"]["models"]}
2450
2451    def getAllModelDetails(self) -> Dict[str, Dict[str, Any]]:
2452        url = f"{self.base_uri}/api/graphql"
2453        headers = {"Authorization": "ApiKey " + self.api_key, "Content-Type": "application/json"}
2454        req_json = {
2455            "query": "query listOfModels {\n models { id name lastUpdated portfolios { id name }}}",
2456            "variables": {},
2457        }
2458        res = requests.post(url, json=req_json, headers=headers, **self._request_params)
2459        if not res.ok:
2460            error_msg = self._try_extract_error_code(res)
2461            logger.error(error_msg)
2462            raise BoostedAPIException(f"Failed to get user models: {error_msg}")
2463        data = res.json()
2464        if data["data"]["models"] is None:
2465            return {}
2466
2467        output_data = {}
2468        for rec in data["data"]["models"]:
2469            model_id = rec["id"]
2470            output_data[model_id] = {
2471                "name": rec["name"],
2472                "last_updated": parser.parse(rec["lastUpdated"]),
2473                "portfolios": rec["portfolios"],
2474            }
2475
2476        return output_data
2477
2478    def get_hedge_experiments(self):
2479        url = self.base_uri + "/api/graphql"
2480        qry = """
2481            query getHedgeExperiments {
2482                hedgeExperiments {
2483                    hedgeExperimentId
2484                    experimentName
2485                    userId
2486                    config
2487                    description
2488                    experimentType
2489                    lastCalculated
2490                    lastModified
2491                    status
2492                    portfolioCalcStatus
2493                    targetSecurities {
2494                        gbiId
2495                        security {
2496                            gbiId
2497                            symbol
2498                            name
2499                        }
2500                        weight
2501                    }
2502                    targetPortfolios {
2503                        portfolioId
2504                    }
2505                    baselineModel {
2506                        id
2507                        name
2508
2509                    }
2510                    baselineScenario {
2511                        hedgeExperimentScenarioId
2512                        scenarioName
2513                        description
2514                        portfolioSettingsJson
2515                        hedgeExperimentPortfolios {
2516                            portfolio {
2517                                id
2518                                name
2519                                modelId
2520                                performanceGridHeader
2521                                performanceGrid
2522                                status
2523                                tearSheet {
2524                                    groupName
2525                                    members {
2526                                        name
2527                                        value
2528                                    }
2529                                }
2530                            }
2531                        }
2532                        status
2533                    }
2534                    baselineStockUniverseId
2535                }
2536            }
2537        """
2538
2539        headers = {"Authorization": "ApiKey " + self.api_key}
2540        resp = requests.post(url, json={"query": qry}, headers=headers, params=self._request_params)
2541
2542        json_resp = resp.json()
2543        # graphql endpoints typically return 200 or 400 status codes, so we must
2544        # check if we have any errors, even with a 200
2545        if (resp.ok and "errors" in json_resp) or not resp.ok:
2546            error_msg = self._try_extract_error_code(resp)
2547            logger.error(error_msg)
2548            raise BoostedAPIException(
2549                (f"Failed to get hedge experiments: {resp.status_code=}; {error_msg=}")
2550            )
2551
2552        json_experiments = resp.json()["data"]["hedgeExperiments"]
2553        experiments = [HedgeExperiment.from_json_dict(exp_json) for exp_json in json_experiments]
2554        return experiments
2555
2556    def get_hedge_experiment_details(self, experiment_id: str):
2557        url = self.base_uri + "/api/graphql"
2558        qry = """
2559            query getHedgeExperimentDetails($hedgeExperimentId: ID!) {
2560                hedgeExperiment(hedgeExperimentId: $hedgeExperimentId) {
2561                ...HedgeExperimentDetailsSummaryListFragment
2562                }
2563            }
2564
2565            fragment HedgeExperimentDetailsSummaryListFragment on HedgeExperiment {
2566                hedgeExperimentId
2567                experimentName
2568                userId
2569                config
2570                description
2571                experimentType
2572                lastCalculated
2573                lastModified
2574                status
2575                portfolioCalcStatus
2576                targetSecurities {
2577                    gbiId
2578                    security {
2579                        gbiId
2580                        symbol
2581                        name
2582                    }
2583                    weight
2584                }
2585                selectedModels {
2586                    id
2587                    name
2588                    stockUniverse {
2589                        name
2590                    }
2591                }
2592                hedgeExperimentScenarios {
2593                    ...experimentScenarioFragment
2594                }
2595                selectedDummyHedgeExperimentModels {
2596                    id
2597                    name
2598                    stockUniverse {
2599                        name
2600                    }
2601                }
2602                targetPortfolios {
2603                    portfolioId
2604                }
2605                baselineModel {
2606                    id
2607                    name
2608
2609                }
2610                baselineScenario {
2611                    hedgeExperimentScenarioId
2612                    scenarioName
2613                    description
2614                    portfolioSettingsJson
2615                    hedgeExperimentPortfolios {
2616                        portfolio {
2617                            id
2618                            name
2619                            modelId
2620                            performanceGridHeader
2621                            performanceGrid
2622                            status
2623                            tearSheet {
2624                                groupName
2625                                members {
2626                                    name
2627                                    value
2628                                }
2629                            }
2630                        }
2631                    }
2632                    status
2633                }
2634                baselineStockUniverseId
2635            }
2636
2637            fragment experimentScenarioFragment on HedgeExperimentScenario {
2638                hedgeExperimentScenarioId
2639                scenarioName
2640                status
2641                description
2642                portfolioSettingsJson
2643                hedgeExperimentPortfolios {
2644                    portfolio {
2645                        id
2646                        name
2647                        modelId
2648                        performanceGridHeader
2649                        performanceGrid
2650                        status
2651                        tearSheet {
2652                            groupName
2653                            members {
2654                                name
2655                                value
2656                            }
2657                        }
2658                    }
2659                }
2660            }
2661        """
2662        headers = {"Authorization": "ApiKey " + self.api_key}
2663        resp = requests.post(
2664            url,
2665            json={"query": qry, "variables": {"hedgeExperimentId": experiment_id}},
2666            headers=headers,
2667            params=self._request_params,
2668        )
2669
2670        json_resp = resp.json()
2671        # graphql endpoints typically return 200 or 400 status codes, so we must
2672        # check if we have any errors, even with a 200
2673        if (resp.ok and "errors" in json_resp) or not resp.ok:
2674            error_msg = self._try_extract_error_code(resp)
2675            logger.error(error_msg)
2676            raise BoostedAPIException(
2677                (
2678                    f"Failed to get hedge experiment results for {experiment_id=}: "
2679                    f"{resp.status_code=}; {error_msg=}"
2680                )
2681            )
2682
2683        json_exp_results = json_resp["data"]["hedgeExperiment"]
2684        if json_exp_results is None:
2685            return None  # issued a request with a non-existent experiment_id
2686        exp_results = HedgeExperimentDetails.from_json_dict(json_exp_results)
2687        return exp_results
2688
2689    def get_portfolio_performance(
2690        self,
2691        portfolio_id: str,
2692        start_date: Optional[datetime.date],
2693        end_date: Optional[datetime.date],
2694        daily_returns: bool,
2695    ) -> pd.DataFrame:
2696        """
2697        Get performance data for a portfolio.
2698
2699        Parameters
2700        ----------
2701        portfolio_id: str
2702            UUID corresponding to the portfolio in question.
2703        start_date: datetime.date
2704            Starting cutoff date to filter performance data
2705        end_date: datetime.date
2706            Ending cutoff date to filter performance data
2707        daily_returns: bool
2708            Flag indicating whether to add a new column with the daily return pct calculated
2709
2710        Returns
2711        -------
2712        pd.DataFrame object
2713            Portfolio and benchmark performance.
2714            -index:
2715                "date": pd.DatetimeIndex
2716            -columns:
2717                "benchmark": benchmark performance, % return
2718                "turnover": portfolio turnover, % of equity
2719                "portfolio": return since beginning of portfolio, % return
2720                "daily_returns": daily percent change in value of the portfolio, % return
2721                                (this column is optional and depends on the daily_returns flag)
2722        """
2723        url = f"{self.base_uri}/api/graphql"
2724        qry = """
2725            query getPortfolioPerformance($portfolioId: ID!) {
2726                portfolio(id: $portfolioId) {
2727                    id
2728                    modelId
2729                    name
2730                    status
2731                    performance {
2732                        benchmark
2733                        date
2734                        turnover
2735                        value
2736                    }
2737                }
2738            }
2739        """
2740
2741        headers = {"Authorization": "ApiKey " + self.api_key}
2742        resp = requests.post(
2743            url,
2744            json={"query": qry, "variables": {"portfolioId": portfolio_id}},
2745            headers=headers,
2746            params=self._request_params,
2747        )
2748
2749        json_resp = resp.json()
2750        # the webserver returns an error for non-ready portfolios, so we have to check
2751        # for this prior to the error check below
2752        pf = json_resp["data"].get("portfolio")
2753        if pf is not None and pf["status"] != "READY":
2754            return pd.DataFrame()
2755
2756        # graphql endpoints typically return 200 or 400 status codes, so we must
2757        # check if we have any errors, even with a 200
2758        if (resp.ok and "errors" in json_resp) or not resp.ok:
2759            error_msg = self._try_extract_error_code(resp)
2760            logger.error(error_msg)
2761            raise BoostedAPIException(
2762                (
2763                    f"Failed to get portfolio performance for {portfolio_id=}: "
2764                    f"{resp.status_code=}; {error_msg=}"
2765                )
2766            )
2767
2768        perf = json_resp["data"]["portfolio"]["performance"]
2769        df = pd.DataFrame(perf).set_index("date").rename(columns={"value": "portfolio"})
2770        df.index = pd.to_datetime(df.index)
2771        if daily_returns:
2772            df["daily_returns"] = pd.to_numeric(df["portfolio"]).pct_change()
2773            df = df.dropna(subset=["daily_returns"])
2774        if start_date:
2775            df = df[df.index >= pd.to_datetime(start_date)]
2776        if end_date:
2777            df = df[df.index <= pd.to_datetime(end_date)]
2778        return df.astype(float)
2779
2780    def _is_portfolio_still_running(self, error_msg: str) -> bool:
2781        # this is jank af. a proper fix of this is either at the webserver
2782        # returning a better response for a portfolio in draft HT2-226, OR
2783        # a bigger refactor of the API that moves to more OOP, which would allow us
2784        # to have this data all in one place
2785        return "Could not find a model with this ID" in error_msg
2786
2787    def get_portfolio_factors(self, model_id: str, portfolio_id: str) -> pd.DataFrame:
2788        url = f"{self.base_uri}/api/analysis/factors/{model_id}/{portfolio_id}"
2789        headers = {"Authorization": "ApiKey " + self.api_key}
2790        resp = requests.get(url, headers=headers, params=self._request_params)
2791
2792        json_resp = resp.json()
2793        if (resp.ok and "errors" in json_resp) or not resp.ok:
2794            error_msg = json_resp["errors"][0]
2795            if self._is_portfolio_still_running(error_msg):
2796                return pd.DataFrame()
2797            logger.error(error_msg)
2798            raise BoostedAPIException(
2799                (
2800                    f"Failed to get portfolio factors for {portfolio_id=}: "
2801                    f"{resp.status_code=}; {error_msg=}"
2802                )
2803            )
2804
2805        df = pd.DataFrame(json_resp["data"], columns=json_resp["header_row"])
2806
2807        def to_lower_snake_case(s):  # why are we linting lambdas? :(
2808            return "_".join(w.lower() for w in s.split(" "))
2809
2810        df = df.rename(columns={old: to_lower_snake_case(old) for old in df.columns}).set_index(
2811            "date"
2812        )
2813        df.index = pd.to_datetime(df.index)
2814        return df
2815
2816    def get_portfolio_volatility(self, model_id: str, portfolio_id: str) -> pd.DataFrame:
2817        url = f"{self.base_uri}/api/analysis/volatility_rolling/{model_id}/{portfolio_id}"
2818        headers = {"Authorization": "ApiKey " + self.api_key}
2819        resp = requests.get(url, headers=headers, params=self._request_params)
2820
2821        json_resp = resp.json()
2822        if (resp.ok and "errors" in json_resp) or not resp.ok:
2823            error_msg = json_resp["errors"][0]
2824            if self._is_portfolio_still_running(error_msg):
2825                return pd.DataFrame()
2826            logger.error(error_msg)
2827            raise BoostedAPIException(
2828                (
2829                    f"Failed to get portfolio volatility for {portfolio_id=}: "
2830                    f"{resp.status_code=}; {error_msg=}"
2831                )
2832            )
2833
2834        df = pd.DataFrame(json_resp["data"], columns=json_resp["headerRow"])
2835        df = df.rename(
2836            columns={old: old.lower().replace("avg", "avg_") for old in df.columns}  # type: ignore
2837        ).set_index("date")
2838        df.index = pd.to_datetime(df.index)
2839        return df
2840
2841    def get_portfolio_holdings(self, model_id: str, portfolio_id: str) -> pd.DataFrame:
2842        url = f"{self.base_uri}/api/models/{model_id}/{portfolio_id}/basket-data"
2843        headers = {"Authorization": "ApiKey " + self.api_key}
2844        resp = requests.get(url, headers=headers, params=self._request_params)
2845
2846        # this is a classic abuse of try/except as control flow: we try to get json body
2847        # from the response so that we can error-check. if this fails, we assume we have
2848        # a legit text response (corresponding to the csv data we care about)
2849        try:
2850            json_resp = resp.json()
2851        except json.decoder.JSONDecodeError:
2852            df = pd.read_csv(io.StringIO(resp.text), header=[0])
2853        else:
2854            error_msg = json_resp["errors"][0]
2855            if self._is_portfolio_still_running(error_msg):
2856                return pd.DataFrame()
2857            else:
2858                logger.error(error_msg)
2859                raise BoostedAPIException(
2860                    (
2861                        f"Failed to get portfolio holdings for {portfolio_id=}: "
2862                        f"{resp.status_code=}; {error_msg=}"
2863                    )
2864                )
2865
2866        df = df.rename(columns={old: old.lower() for old in df.columns}).set_index("date")
2867        df.index = pd.to_datetime(df.index)
2868        return df
2869
2870    def getStockDataTableForDate(
2871        self, model_id: str, portfolio_id: str, date: datetime.date
2872    ) -> pd.DataFrame:
2873        headers = {"Authorization": "ApiKey " + self.api_key, "Content-Type": "application/json"}
2874
2875        url_base = f"{self.base_uri}/api/analysis"
2876        url_params = f"{model_id}/{portfolio_id}"
2877        formatted_date = date.strftime("%Y-%m-%d")
2878
2879        stock_prices_url = f"{url_base}/stock-prices/{url_params}/{formatted_date}"
2880        stock_factors_url = f"{url_base}/stock-factors/{url_params}/date/{formatted_date}"
2881
2882        prices_params = {"useTicker": "false", "useCurrentSignals": "true"}
2883        factors_param = {"useTicker": "false", "useCurrentSignals": "true"}
2884
2885        prices_resp = requests.get(
2886            stock_prices_url, headers=headers, params=prices_params, **self._request_params
2887        )
2888        factors_resp = requests.get(
2889            stock_factors_url, headers=headers, params=factors_param, **self._request_params
2890        )
2891
2892        frames = []
2893        gbi_ids = set()
2894        for res in (prices_resp, factors_resp):
2895            if not res.ok:
2896                error_msg = self._try_extract_error_code(res)
2897                logger.error(error_msg)
2898                raise BoostedAPIException(
2899                    (
2900                        f"Failed to fetch stock data table for model {model_id}"
2901                        f" (it's possible no data is present for the given date: {date})."
2902                        f" Error message: {error_msg}"
2903                    )
2904                )
2905            result = res.json()
2906            df = pd.DataFrame(result)
2907            gbi_ids.update(df.columns.to_list())
2908            frames.append(pd.DataFrame(result))
2909
2910        all_gbiid_df = pd.concat(frames)
2911
2912        # Get the metadata of all GBI IDs
2913        gbiid_metadata_res = self._get_graphql(
2914            query=graphql_queries.GET_SEC_INFO_QRY, variables={"ids": [int(x) for x in gbi_ids]}
2915        )
2916        # Build a DF of metadata x GBI IDs
2917        gbiid_metadata_df = pd.DataFrame(
2918            {str(x["gbiId"]): x for x in gbiid_metadata_res["data"]["securities"]}
2919        )
2920        # Slice metadata we care. We'll drop "symbol" at the end.
2921        isin_country_currency_df = gbiid_metadata_df.loc[["isin", "country", "currency", "symbol"]]
2922        # Concatenate metadata to the existing stock data DF
2923        all_gbiid_with_metadata_df = pd.concat([all_gbiid_df, isin_country_currency_df])
2924        gbiid_with_symbol_df = all_gbiid_with_metadata_df.loc[
2925            :, all_gbiid_with_metadata_df.loc["symbol"].notna()
2926        ]
2927        renamed_df = gbiid_with_symbol_df.rename(
2928            index={"isin": "ISIN"}, columns=gbiid_with_symbol_df.loc["symbol"].to_dict()
2929        )
2930        output_df = renamed_df.drop(index=["symbol"])
2931        return output_df
2932
2933    def add_hedge_experiment_scenario(
2934        self,
2935        experiment_id: str,
2936        scenario_name: str,
2937        scenario_settings: PortfolioSettings,
2938        run_scenario_immediately: bool,
2939    ) -> HedgeExperimentScenario:
2940        add_scenario_input = {
2941            "hedgeExperimentId": experiment_id,
2942            "scenarioName": scenario_name,
2943            "portfolioSettingsJson": str(scenario_settings),
2944            "runExperimentOnScenario": run_scenario_immediately,
2945            "createDefaultPortfolio": "false",
2946        }
2947        qry = """
2948            mutation addHedgeExperimentScenario(
2949                $input: AddHedgeExperimentScenarioInput!
2950            ) {
2951                addHedgeExperimentScenario(input: $input) {
2952                    hedgeExperimentScenario {
2953                        hedgeExperimentScenarioId
2954                        scenarioName
2955                        description
2956                        portfolioSettingsJson
2957                    }
2958                }
2959            }
2960
2961        """
2962
2963        url = f"{self.base_uri}/api/graphql"
2964
2965        resp = requests.post(
2966            url,
2967            headers={"Authorization": "ApiKey " + self.api_key},
2968            json={"query": qry, "variables": {"input": add_scenario_input}},
2969        )
2970
2971        json_resp = resp.json()
2972        if (resp.ok and "errors" in json_resp) or not resp.ok:
2973            error_msg = self._try_extract_error_code(resp)
2974            logger.error(error_msg)
2975            raise BoostedAPIException(
2976                (f"Failed to add scenario: {resp.status_code=}; {error_msg=}")
2977            )
2978
2979        scenario_dict = json_resp["data"]["addHedgeExperimentScenario"]["hedgeExperimentScenario"]
2980        if scenario_dict is None:
2981            raise BoostedAPIException(
2982                "Failed to add scenario, likely due to bad experiment id or api key"
2983            )
2984        s = HedgeExperimentScenario.from_json_dict(scenario_dict)
2985        return s
2986
2987    # experiment life cycle has 4 steps:
2988    # 1. creation - essentially a very simple registration of a new instance, returning an id
2989    # 2. modify - populate with settings
2990    # 3. start - run the experiment
2991    # 4. delete - drop the experiment
2992    # while i would prefer to just have 2 funcs for (1,2,3) and (4) for a simpler api,
2993    # we need to expose finer-grained control becuase of how scenarios work.
2994    def create_hedge_experiment(
2995        self,
2996        name: str,
2997        description: str,
2998        experiment_type: hedge_experiment_type,
2999        target_securities: Union[Dict[GbiIdSecurity, float], str, None],
3000    ) -> HedgeExperiment:
3001        # we don't pass target_securities here (as much as id like to) because the
3002        # graphql input doesn't support it at this point
3003
3004        # note that this query returns a lot of null fields at this point, but
3005        # they are necessary for building a HE.
3006        create_qry = """
3007            mutation createDraftMutation($input: CreateHedgeExperimentDraftInput!) {
3008                createHedgeExperimentDraft(input: $input) {
3009                    hedgeExperiment {
3010                        hedgeExperimentId
3011                        experimentName
3012                        userId
3013                        config
3014                        description
3015                        experimentType
3016                        lastCalculated
3017                        lastModified
3018                        status
3019                        portfolioCalcStatus
3020                        targetSecurities {
3021                            gbiId
3022                            security {
3023                                gbiId
3024                                name
3025                                symbol
3026                            }
3027                            weight
3028                        }
3029                        baselineModel {
3030                            id
3031                            name
3032                        }
3033                        baselineScenario {
3034                            hedgeExperimentScenarioId
3035                            scenarioName
3036                            description
3037                            portfolioSettingsJson
3038                            hedgeExperimentPortfolios {
3039                                portfolio {
3040                                    id
3041                                    name
3042                                    modelId
3043                                    performanceGridHeader
3044                                    performanceGrid
3045                                    status
3046                                    tearSheet {
3047                                        groupName
3048                                        members {
3049                                            name
3050                                            value
3051                                        }
3052                                    }
3053                                }
3054                            }
3055                            status
3056                        }
3057                        baselineStockUniverseId
3058                    }
3059                }
3060            }
3061        """
3062
3063        create_input: Dict[str, Any] = {
3064            "name": name,
3065            "experimentType": experiment_type,
3066            "description": description,
3067        }
3068        if isinstance(target_securities, dict):
3069            create_input["setTargetSecurities"] = [
3070                {"gbiId": sec.gbi_id, "weight": weight}
3071                for (sec, weight) in target_securities.items()
3072            ]
3073        elif isinstance(target_securities, str):
3074            create_input["setTargetPortfolios"] = [{"portfolioId": target_securities}]
3075        elif target_securities is None:
3076            pass
3077        else:
3078            raise TypeError(
3079                "Expected value of type Union[Dict[GbiIdSecurity, str], str] for "
3080                f"argument 'target_securities'; got {type(target_securities)}"
3081            )
3082        resp = requests.post(
3083            f"{self.base_uri}/api/graphql",
3084            json={"query": create_qry, "variables": {"input": create_input}},
3085            headers={"Authorization": "ApiKey " + self.api_key},
3086            params=self._request_params,
3087        )
3088
3089        json_resp = resp.json()
3090        if (resp.ok and "errors" in json_resp) or not resp.ok:
3091            error_msg = self._try_extract_error_code(resp)
3092            logger.error(error_msg)
3093            raise BoostedAPIException(
3094                (f"Failed to create hedge experiment: {resp.status_code=}; {error_msg=}")
3095            )
3096
3097        exp_dict = json_resp["data"]["createHedgeExperimentDraft"]["hedgeExperiment"]
3098        experiment = HedgeExperiment.from_json_dict(exp_dict)
3099        return experiment
3100
3101    def modify_hedge_experiment(
3102        self,
3103        experiment_id: str,
3104        name: Optional[str] = None,
3105        description: Optional[str] = None,
3106        experiment_type: Optional[hedge_experiment_type] = None,
3107        target_securities: Union[Dict[GbiIdSecurity, float], str, None] = None,
3108        model_ids: Optional[List[str]] = None,
3109        stock_universe_ids: Optional[List[str]] = None,
3110        create_default_scenario: bool = True,
3111        baseline_model_id: Optional[str] = None,
3112        baseline_stock_universe_id: Optional[str] = None,
3113        baseline_portfolio_settings: Optional[str] = None,
3114    ) -> HedgeExperiment:
3115        mod_qry = """
3116            mutation modifyHedgeExperimentDraft(
3117                $input: ModifyHedgeExperimentDraftInput!
3118            ) {
3119                modifyHedgeExperimentDraft(input: $input) {
3120                    hedgeExperiment {
3121                    ...HedgeExperimentSelectedSecuritiesPageFragment
3122                    }
3123                }
3124            }
3125
3126            fragment HedgeExperimentSelectedSecuritiesPageFragment on HedgeExperiment {
3127                hedgeExperimentId
3128                experimentName
3129                userId
3130                config
3131                description
3132                experimentType
3133                lastCalculated
3134                lastModified
3135                status
3136                portfolioCalcStatus
3137                targetSecurities {
3138                    gbiId
3139                    security {
3140                        gbiId
3141                        name
3142                        symbol
3143                    }
3144                    weight
3145                }
3146                targetPortfolios {
3147                    portfolioId
3148                }
3149                baselineModel {
3150                    id
3151                    name
3152                }
3153                baselineScenario {
3154                    hedgeExperimentScenarioId
3155                    scenarioName
3156                    description
3157                    portfolioSettingsJson
3158                    hedgeExperimentPortfolios {
3159                        portfolio {
3160                            id
3161                            name
3162                            modelId
3163                            performanceGridHeader
3164                            performanceGrid
3165                            status
3166                            tearSheet {
3167                                groupName
3168                                members {
3169                                    name
3170                                    value
3171                                }
3172                            }
3173                        }
3174                    }
3175                    status
3176                }
3177                baselineStockUniverseId
3178            }
3179        """
3180        mod_input = {
3181            "hedgeExperimentId": experiment_id,
3182            "createDefaultScenario": create_default_scenario,
3183        }
3184        if name is not None:
3185            mod_input["newExperimentName"] = name
3186        if description is not None:
3187            mod_input["newExperimentDescription"] = description
3188        if experiment_type is not None:
3189            mod_input["newExperimentType"] = experiment_type
3190        if model_ids is not None:
3191            mod_input["setSelectdModels"] = model_ids
3192        if stock_universe_ids is not None:
3193            mod_input["selectedStockUniverseIds"] = stock_universe_ids
3194        if baseline_model_id is not None:
3195            mod_input["setBaselineModel"] = baseline_model_id
3196        if baseline_stock_universe_id is not None:
3197            mod_input["setBaselineStockUniverse"] = baseline_stock_universe_id
3198        if baseline_portfolio_settings is not None:
3199            mod_input["setBaselinePortfolioSettings"] = baseline_portfolio_settings
3200        # note that the behaviors bound to these data are mutually exclusive,
3201        # and its possible the opposite was set earlier in the DRAFT phase
3202        # of experiment creation, so when setting one, we must unset the other
3203        if isinstance(target_securities, dict):
3204            mod_input["setTargetSecurities"] = [
3205                {"gbiId": sec.gbi_id, "weight": weight}
3206                for (sec, weight) in target_securities.items()
3207            ]
3208            mod_input["setTargetPortfolios"] = None
3209        elif isinstance(target_securities, str):
3210            mod_input["setTargetPortfolios"] = [{"portfolioId": target_securities}]
3211            mod_input["setTargetSecurities"] = None
3212        elif target_securities is None:
3213            pass
3214        else:
3215            raise TypeError(
3216                "Expected value of type Union[Dict[GbiIdSecurity, str], str] "
3217                f"for argument 'target_securities'; got {type(target_securities)}"
3218            )
3219
3220        resp = requests.post(
3221            f"{self.base_uri}/api/graphql",
3222            json={"query": mod_qry, "variables": {"input": mod_input}},
3223            headers={"Authorization": "ApiKey " + self.api_key},
3224            params=self._request_params,
3225        )
3226
3227        json_resp = resp.json()
3228        if (resp.ok and "errors" in json_resp) or not resp.ok:
3229            error_msg = self._try_extract_error_code(resp)
3230            logger.error(error_msg)
3231            raise BoostedAPIException(
3232                (
3233                    f"Failed to modify hedge experiment in preparation for start {experiment_id=}: "
3234                    f"{resp.status_code=}; {error_msg=}"
3235                )
3236            )
3237
3238        exp_dict = json_resp["data"]["modifyHedgeExperimentDraft"]["hedgeExperiment"]
3239        experiment = HedgeExperiment.from_json_dict(exp_dict)
3240        return experiment
3241
3242    def start_hedge_experiment(self, experiment_id: str, *scenario_ids: str) -> HedgeExperiment:
3243        start_qry = """
3244            mutation startHedgeExperiment($input: StartHedgeExperimentInput!) {
3245                startHedgeExperiment(input: $input) {
3246                    hedgeExperiment {
3247                        hedgeExperimentId
3248                        experimentName
3249                        userId
3250                        config
3251                        description
3252                        experimentType
3253                        lastCalculated
3254                        lastModified
3255                        status
3256                        portfolioCalcStatus
3257                        targetSecurities {
3258                            gbiId
3259                            security {
3260                                gbiId
3261                                name
3262                                symbol
3263                            }
3264                            weight
3265                        }
3266                        targetPortfolios {
3267                            portfolioId
3268                        }
3269                        baselineModel {
3270                            id
3271                            name
3272                        }
3273                        baselineScenario {
3274                            hedgeExperimentScenarioId
3275                            scenarioName
3276                            description
3277                            portfolioSettingsJson
3278                            hedgeExperimentPortfolios {
3279                                portfolio {
3280                                    id
3281                                    name
3282                                    modelId
3283                                    performanceGridHeader
3284                                    performanceGrid
3285                                    status
3286                                    tearSheet {
3287                                        groupName
3288                                        members {
3289                                            name
3290                                            value
3291                                        }
3292                                    }
3293                                }
3294                            }
3295                            status
3296                        }
3297                        baselineStockUniverseId
3298                    }
3299                }
3300            }
3301        """
3302        start_input: Dict[str, Any] = {"hedgeExperimentId": experiment_id}
3303        if len(scenario_ids) > 0:
3304            start_input["hedgeExperimentScenarioIds"] = list(scenario_ids)
3305
3306        resp = requests.post(
3307            f"{self.base_uri}/api/graphql",
3308            json={"query": start_qry, "variables": {"input": start_input}},
3309            headers={"Authorization": "ApiKey " + self.api_key},
3310            params=self._request_params,
3311        )
3312
3313        json_resp = resp.json()
3314        if (resp.ok and "errors" in json_resp) or not resp.ok:
3315            error_msg = self._try_extract_error_code(resp)
3316            logger.error(error_msg)
3317            raise BoostedAPIException(
3318                (
3319                    f"Failed to start hedge experiment {experiment_id=}: "
3320                    f"{resp.status_code=}; {error_msg=}"
3321                )
3322            )
3323
3324        exp_dict = json_resp["data"]["startHedgeExperiment"]["hedgeExperiment"]
3325        experiment = HedgeExperiment.from_json_dict(exp_dict)
3326        return experiment
3327
3328    def delete_hedge_experiment(self, experiment_id: str) -> bool:
3329        delete_qry = """
3330            mutation($input: DeleteHedgeExperimentsInput!) {
3331                deleteHedgeExperiments(input: $input) {
3332                    success
3333                }
3334            }
3335        """
3336        delete_input = {"hedgeExperimentIds": [experiment_id]}
3337        resp = requests.post(
3338            f"{self.base_uri}/api/graphql",
3339            json={"query": delete_qry, "variables": {"input": delete_input}},
3340            headers={"Authorization": "ApiKey " + self.api_key},
3341            params=self._request_params,
3342        )
3343
3344        json_resp = resp.json()
3345        if (resp.ok and "errors" in json_resp) or not resp.ok:
3346            error_msg = self._try_extract_error_code(resp)
3347            logger.error(error_msg)
3348            raise BoostedAPIException(
3349                (
3350                    f"Failed to delete hedge experiment {experiment_id=}: "
3351                    + f"status_code={resp.status_code}; error_msg={error_msg}"
3352                )
3353            )
3354
3355        return json_resp["data"]["deleteHedgeExperiments"]["success"]
3356
3357    def create_hedge_basket_position_bounds_from_csv(
3358        self,
3359        filepath: str,
3360        name: str,
3361        description: Optional[str],
3362        mapping_result_filepath: Optional[str],
3363    ) -> str:
3364        DATE = "Date"
3365        ISIN = "ISIN"
3366        COUNTRY = "Country"
3367        CURRENCY = "Currency"
3368        LOWER_BOUND = "Lower Bound"
3369        UPPER_BOUND = "Upper Bound"
3370        supported_columns = {
3371            DATE,
3372            ISIN,
3373            COUNTRY,
3374            CURRENCY,
3375            LOWER_BOUND,
3376            UPPER_BOUND,
3377        }
3378        required_columns = {ISIN, LOWER_BOUND, UPPER_BOUND}
3379
3380        try:
3381            df: pd.DataFrame = pd.read_csv(filepath, parse_dates=True)
3382        except Exception as e:
3383            raise BoostedAPIException(f"Error reading {filepath=}: {e}")
3384
3385        columns = set(df.columns)
3386
3387        # First perform basic data validation
3388        missing_required_columns = required_columns - columns
3389        if missing_required_columns:
3390            raise BoostedAPIException(
3391                f"The following required columns are missing: {missing_required_columns}"
3392            )
3393        extra_columns = columns - supported_columns
3394        if extra_columns:
3395            logger.warning(
3396                f"The following columns are unsupported and will be ignored: {extra_columns}"
3397            )
3398        try:
3399            df[LOWER_BOUND] = df[LOWER_BOUND].astype(float)
3400            df[UPPER_BOUND] = df[UPPER_BOUND].astype(float)
3401            df[ISIN] = df[ISIN].astype(str)
3402        except Exception as e:
3403            raise BoostedAPIException(f"Column datatypes are incorrect: {e}")
3404        lb_gt_ub = df[df[LOWER_BOUND] > df[UPPER_BOUND]]
3405        if not lb_gt_ub.empty:
3406            raise BoostedAPIException(
3407                f"Lower Bound must be <= Upper Bound, but these are not: {lb_gt_ub[ISIN].tolist()}"
3408            )
3409        out_of_range = df[
3410            (
3411                (df[LOWER_BOUND] < 0)
3412                | (df[LOWER_BOUND] > 1)
3413                | (df[UPPER_BOUND] < 0)
3414                | (df[UPPER_BOUND] > 1)
3415            )
3416        ]
3417        if not out_of_range.empty:
3418            raise BoostedAPIException("Lower Bound and Upper Bound values must be in range [0, 1]")
3419
3420        # Now map the security info into GBI IDs
3421        rows = list(df.to_dict(orient="index").values())
3422        sec_data_list = self.getGbiIdFromIdentCountryCurrencyDate(
3423            ident_country_currency_dates=[
3424                DateIdentCountryCurrency(
3425                    date=row.get(DATE, datetime.date.today().isoformat()),
3426                    identifier=row.get(ISIN),
3427                    id_type=ColumnSubRole.ISIN,
3428                    country=row.get(COUNTRY),
3429                    currency=row.get(CURRENCY),
3430                )
3431                for row in rows
3432            ]
3433        )
3434
3435        # Now take each row and its gbi id mapping, and create the bounds list
3436        bounds = []
3437        for row, sec_data in zip(rows, sec_data_list):
3438            if sec_data is None:
3439                logger.warning(f"Failed to map {row[ISIN]}, skipping this security.")
3440            else:
3441                bounds.append(
3442                    {"gbi_id": str(sec_data.gbi_id), "lb": row[LOWER_BOUND], "ub": row[UPPER_BOUND]}
3443                )
3444
3445                # Add security metadata to see the mapping
3446                row["Mapped GBI ID"] = sec_data.gbi_id
3447                row[f"Mapped {ISIN}"] = sec_data.isin_info.identifier
3448                row[f"Mapped {COUNTRY}"] = sec_data.isin_info.country
3449                row[f"Mapped {CURRENCY}"] = sec_data.isin_info.currency
3450                row["Mapped Ticker"] = sec_data.ticker
3451                row["Mapped Company Name"] = sec_data.company_name
3452
3453        # Call endpoint to create the bounds settings template
3454        qry = """
3455              mutation CreatePartialStrategyTemplate(
3456                $portfolioSettingsKey: String!
3457                $partialSettings: String!
3458                $name: String!
3459                $description: String
3460              ) {
3461                createPartialStrategyTemplate(
3462                  portfolioSettingsKey: $portfolioSettingsKey
3463                  partialSettings: $partialSettings
3464                  name: $name
3465                  description: $description
3466                )
3467              }
3468            """
3469        variables = {
3470            "portfolioSettingsKey": "basketTrading.positionSizeBounds",
3471            "partialSettings": json.dumps(bounds),
3472            "name": name,
3473            "description": description,
3474        }
3475        resp = self._get_graphql(qry, variables=variables)
3476
3477        # Write mapped csv for reference
3478        if mapping_result_filepath is not None:
3479            pd.DataFrame(rows).to_csv(mapping_result_filepath)
3480
3481        return resp["data"]["createPartialStrategyTemplate"]
3482
3483    def get_portfolio_accuracy(
3484        self,
3485        model_id: str,
3486        portfolio_id: str,
3487        start_date: Optional[BoostedDate] = None,
3488        end_date: Optional[BoostedDate] = None,
3489    ) -> dict:
3490        if start_date and end_date:
3491            validate_start_and_end_dates(start_date=start_date, end_date=end_date)
3492            start_date = convert_date(start_date)
3493            end_date = convert_date(end_date)
3494
3495        # TODO: Later change this URI to not use the watchlist prefix. It is misnamed.
3496        url = f"{self.base_uri}{WATCHLIST_ROUTE_PREFIX}{DAL_PA_ROUTE}/get-hit-rate/"
3497        headers = {"Authorization": "ApiKey " + self.api_key, "Content-Type": "application/json"}
3498        req_json = {"model_id": model_id, "portfolio_id": portfolio_id}
3499        if start_date and end_date:
3500            req_json["start_date"] = start_date.isoformat()
3501            req_json["end_date"] = end_date.isoformat()
3502        res = requests.post(url, json=req_json, headers=headers, **self._request_params)
3503
3504        if not res.ok:
3505            error_msg = self._try_extract_error_code(res)
3506            logger.error(error_msg)
3507            raise BoostedAPIException(f"Failed to get Hit Rate: {error_msg}")
3508
3509        data = res.json()
3510        return data
3511
3512    def create_watchlist(self, name: str) -> str:
3513        url = f"{self.base_uri}{WATCHLIST_ROUTE_PREFIX}{DAL_WATCHLIST_ROUTE}/create/"
3514        headers = {"Authorization": "ApiKey " + self.api_key, "Content-Type": "application/json"}
3515        req_json = {"name": name}
3516        res = requests.post(url, json=req_json, headers=headers, **self._request_params)
3517
3518        if not res.ok:
3519            error_msg = self._try_extract_error_code(res)
3520            logger.error(error_msg)
3521            raise BoostedAPIException(f"Failed to get user models: {error_msg}")
3522
3523        data = res.json()
3524        return data["watchlist_id"]
3525
3526    def _get_graphql(
3527        self,
3528        query: str,
3529        variables: Dict,
3530        error_msg_prefix: str = "Failed to get graphql result: ",
3531        log_error: bool = True,
3532    ) -> Dict:
3533        headers = {"Authorization": "ApiKey " + self.api_key}
3534        json_req = {"query": query, "variables": variables}
3535
3536        url = self.base_uri + "/api/graphql"
3537        resp = requests.post(
3538            url,
3539            json=json_req,
3540            headers=headers,
3541            params=self._request_params,
3542        )
3543
3544        # graphql endpoints typically return 200 or 400 status codes, so we must
3545        # check if we have any errors, even with a 200
3546        if not resp.ok or (resp.ok and "errors" in resp.json()):
3547            error_msg = self._try_extract_error_code(resp)
3548            error_str = str(error_msg_prefix) + f" {resp.status_code=}; {error_msg=}"
3549            if log_error:
3550                logger.error(error_str)
3551            raise BoostedAPIException(error_str)
3552
3553        json_resp = resp.json()
3554        return json_resp
3555
3556    def _get_security_info(self, gbi_ids: List[int]) -> Dict:
3557        query = graphql_queries.GET_SEC_INFO_QRY
3558        variables = {
3559            "ids": [] if not gbi_ids else gbi_ids,
3560        }
3561
3562        error_msg_prefix = "Failed to get Security Details:"
3563        return self._get_graphql(
3564            query=query, variables=variables, error_msg_prefix=error_msg_prefix
3565        )
3566
3567    def _get_sector_info(self) -> Dict:
3568        """
3569        Returns a list of sector objects, e.g.
3570        {
3571            "id": 1010,
3572            "parentId": 10,
3573            "name": "Energy",
3574            "topParentName": null,
3575            "spiqSectorId": -1,
3576            "legacy": false
3577        }
3578        """
3579        url = f"{self.base_uri}/api/sectors"
3580        headers = {"Authorization": "ApiKey " + self.api_key}
3581        res = requests.get(url, headers=headers, **self._request_params)
3582        self._check_ok_or_err_with_msg(res, "Failed to get sectors data")
3583        return res.json()["sectors"]
3584
3585    def _get_watchlist_analysis(
3586        self,
3587        gbi_ids: List[int],
3588        model_ids: List[str],
3589        portfolio_ids: List[str],
3590        asof_date=datetime.date.today(),
3591    ) -> Dict:
3592        query = graphql_queries.WATCHLIST_ANALYSIS_QRY
3593        variables = {
3594            "gbiIds": gbi_ids,
3595            "modelIds": model_ids,
3596            "portfolioIds": portfolio_ids,
3597            "date": self.__iso_format(asof_date),
3598        }
3599        error_msg_prefix = "Failed to get Coverage Analysis:"
3600        return self._get_graphql(
3601            query=query, variables=variables, error_msg_prefix=error_msg_prefix
3602        )
3603
3604    def _get_models_for_portfolio(self, portfolio_ids: List[str]) -> Dict:
3605        query = graphql_queries.GET_MODELS_FOR_PORTFOLIOS_QRY
3606        variables = {"ids": portfolio_ids}
3607        error_msg_prefix = "Failed to get Models for Portfolios: "
3608        return self._get_graphql(
3609            query=query, variables=variables, error_msg_prefix=error_msg_prefix
3610        )
3611
3612    def _get_excess_return(
3613        self, model_ids: List[str], gbi_ids: List[int], asof_date=datetime.date.today()
3614    ) -> Dict:
3615        query = graphql_queries.GET_EXCESS_RETURN_QRY
3616
3617        variables = {
3618            "modelIds": model_ids,
3619            "gbiIds": gbi_ids,
3620            "date": self.__iso_format(asof_date),
3621        }
3622        error_msg_prefix = "Failed to get Excess Return Slugging Pct: "
3623        return self._get_graphql(
3624            query=query, variables=variables, error_msg_prefix=error_msg_prefix
3625        )
3626
3627    def _coverage_column_name_format(self, in_str) -> str:
3628        if in_str.upper() == "ISIN":
3629            return "ISIN"
3630
3631        return in_str.title()
3632
3633    def _get_model_stocks(self, model_id: str) -> List[GbiIdTickerISIN]:
3634        # first, get the universe id
3635        resp = self._get_graphql(
3636            graphql_queries.GET_MODEL_STOCK_UNIVERSE_ID_QUERY,
3637            variables={"modelId": model_id},
3638            error_msg_prefix="Failed to get model stock universe ID",
3639        )
3640        universe_id = resp["data"]["model"]["stockUniverseId"]
3641
3642        # now, query for universe stocks
3643        url = self.base_uri + f"/api/stocks/model-universe/{universe_id}"
3644        headers = {"Authorization": "ApiKey " + self.api_key}
3645        universe_resp = requests.get(url, headers=headers, **self._request_params)
3646        universe = universe_resp.json()["stockUniverse"]
3647        securities = [
3648            GbiIdTickerISIN(gbi_id=security["id"], ticker=security["symbol"], isin=security["isin"])
3649            for security in universe
3650        ]
3651        return securities
3652
3653    def get_coverage_info(self, watchlist_id: str, portfolio_group_id: str) -> pd.DataFrame:
3654        # get securities list in watchlist
3655        watchlist_details = self.get_watchlist_details(watchlist_id)
3656        security_list = watchlist_details["targets"]
3657
3658        gbi_ids = [x["gbi_id"] for x in security_list]
3659
3660        gbi_data: Dict[Any, Dict] = {x: {} for x in gbi_ids}
3661
3662        # get security info ticker, name, industry etc
3663        sec_info = self._get_security_info(gbi_ids)
3664
3665        for sec in sec_info["data"]["securities"]:
3666            gbi_id = sec["gbiId"]
3667            for k in ["symbol", "name", "isin", "country", "currency"]:
3668                gbi_data[gbi_id][self._coverage_column_name_format(k)] = sec[k]
3669
3670            gbi_data[gbi_id][self._coverage_column_name_format("Sector")] = sec["sector"][
3671                "topParentName"
3672            ]
3673
3674        # get portfolios list in portfolio_Group
3675        portfolio_group = self.get_portfolio_group(portfolio_group_id)
3676        portfolio_ids = [x["portfolio_id"] for x in portfolio_group["portfolios"]]
3677        portfolio_info = {x["portfolio_id"]: x for x in portfolio_group["portfolios"]}
3678
3679        model_resp = self._get_models_for_portfolio(portfolio_ids=portfolio_ids)
3680        for portfolio in model_resp["data"]["portfolios"]:
3681            portfolio_info[portfolio["id"]].update(portfolio)
3682
3683        model_info = {
3684            x["modelId"]: portfolio_info[x["id"]] for x in model_resp["data"]["portfolios"]
3685        }
3686
3687        # model_ids and portfolio_ids are parallel arrays
3688        model_ids = [portfolio_info[x]["modelId"] for x in portfolio_ids]
3689
3690        # graphql: get watchlist analysis
3691        wl_analysis = self._get_watchlist_analysis(
3692            gbi_ids=gbi_ids,
3693            model_ids=model_ids,
3694            portfolio_ids=portfolio_ids,
3695            asof_date=datetime.date.today(),
3696        )
3697
3698        portfolio_gbi_data: Dict[Any, Dict] = {k: {} for k in portfolio_ids}
3699        for pi, v in portfolio_gbi_data.items():
3700            v.update({k: {} for k in gbi_data.keys()})
3701
3702        equity_explorer_date = wl_analysis["data"]["watchlistAnalysis"][0]["analysisDates"][0][
3703            "date"
3704        ]
3705        for wla in wl_analysis["data"]["watchlistAnalysis"]:
3706            gbi_id = wla["gbiId"]
3707            gbi_data[gbi_id]["Composite Rating"] = wla["analysisDates"][0]["aggregateSignal"][
3708                "rating"
3709            ]
3710            gbi_data[gbi_id]["Composite Rating Delta"] = wla["analysisDates"][0]["aggregateSignal"][
3711                "ratingDelta"
3712            ]
3713
3714            for p in wla["analysisDates"][0]["portfoliosSignals"]:
3715                model_name = portfolio_info[p["portfolioId"]]["modelName"]
3716
3717                portfolio_gbi_data[p["portfolioId"]][gbi_id][
3718                    model_name + self._coverage_column_name_format(": rank")
3719                ] = (p["rank"] + 1)
3720                portfolio_gbi_data[p["portfolioId"]][gbi_id][
3721                    model_name + self._coverage_column_name_format(": rank delta")
3722                ] = (-1 * p["signalDelta"])
3723                portfolio_gbi_data[p["portfolioId"]][gbi_id][
3724                    model_name + self._coverage_column_name_format(": rating")
3725                ] = p["rating"]
3726                portfolio_gbi_data[p["portfolioId"]][gbi_id][
3727                    model_name + self._coverage_column_name_format(": rating delta")
3728                ] = p["ratingDelta"]
3729
3730        neg_rec: Dict[Any, Dict] = {k: {} for k in gbi_data.keys()}
3731        pos_rec: Dict[Any, Dict] = {k: {} for k in gbi_data.keys()}
3732        for wla in wl_analysis["data"]["watchlistAnalysis"]:
3733            gbi_id = wla["gbiId"]
3734
3735            for pid, signals in zip(portfolio_ids, wla["analysisDates"][0]["portfoliosSignals"]):
3736                model_name = portfolio_info[pid]["modelName"]
3737                neg_rec[gbi_id][
3738                    model_name + self._coverage_column_name_format(": negative recommendation")
3739                ] = signals["explainWeightNeg"]
3740                pos_rec[gbi_id][
3741                    model_name + self._coverage_column_name_format(": positive recommendation")
3742                ] = signals["explainWeightPos"]
3743
3744        # graphql: GetExcessReturn - slugging pct
3745        er_sp = self._get_excess_return(
3746            model_ids=model_ids, gbi_ids=gbi_ids, asof_date=equity_explorer_date
3747        )
3748
3749        for model in er_sp["data"]["models"]:
3750            model_name = model_info[model["id"]]["modelName"]
3751            for stat in model["equityExplorerData"]["equityExplorerSummaryStatistics"]:
3752                portfolioId = model_info[model["id"]]["id"]
3753                portfolio_gbi_data[portfolioId][int(stat["gbiId"])][
3754                    model_name + self._coverage_column_name_format(": slugging %")
3755                ] = (stat["ER"]["SP"]["sixMonthWindowOneMonthHorizon"] * 100)
3756
3757        # add rank, rating, slugging
3758        for pid, v in portfolio_gbi_data.items():
3759            for gbi_id, vv in v.items():
3760                gbi_data[gbi_id].update(vv)
3761
3762        # add neg/pos rec scores
3763        for rec in [neg_rec, pos_rec]:
3764            for k, v in rec.items():
3765                gbi_data[k].update(v)
3766
3767        df = pd.DataFrame.from_records([v for _, v in gbi_data.items()])
3768
3769        return df
3770
3771    def get_coverage_csv(
3772        self, watchlist_id: str, portfolio_group_id: str, filepath: Optional[str] = None
3773    ) -> Optional[str]:
3774        """
3775        Converts the coverage contents to CSV format
3776        Parameters
3777        ----------
3778        watchlist_id: str
3779            UUID str identifying the coverage watchlist
3780        portfolio_group_id: str
3781            UUID str identifying the group of portfolio to use for analysis
3782        filepath: Optional[str]
3783            UUID str identifying the group of portfolio to use for analysis
3784
3785        Returns:
3786        ----------
3787        None if filepath is provided, else a string with a csv's contents is returned
3788        """
3789
3790        df = self.get_coverage_info(watchlist_id, portfolio_group_id)
3791
3792        return df.to_csv(filepath, index=False, float_format="%.4f")
3793
3794    def get_watchlist_details(self, watchlist_id: str) -> Dict:
3795        url = f"{self.base_uri}{ROUTE_PREFIX}{DAL_WATCHLIST_ROUTE}/details/"
3796        headers = {"Authorization": "ApiKey " + self.api_key, "Content-Type": "application/json"}
3797        req_json = {"watchlist_id": watchlist_id}
3798        res = requests.post(url, json=req_json, headers=headers, **self._request_params)
3799
3800        if not res.ok:
3801            error_msg = self._try_extract_error_code(res)
3802            logger.error(error_msg)
3803            raise BoostedAPIException(f"Failed to get user models: {error_msg}")
3804
3805        data = res.json()
3806        return data
3807
3808    def create_watchlist_from_file(self, name: str, filepath: str) -> str:
3809        url = f"{self.base_uri}{ROUTE_PREFIX}{DAL_WATCHLIST_ROUTE}/create_watchlist_from_file/"
3810        headers = {"Authorization": "ApiKey " + self.api_key}
3811
3812        with open(filepath, "rb") as fp:
3813            file_bytes = fp.read()
3814
3815        file_bytes_base64 = base64.b64encode(file_bytes).decode("ascii")
3816        json_req = {
3817            "content_type": mimetypes.guess_type(filepath)[0],
3818            "file_bytes_base64": file_bytes_base64,
3819            "name": name,
3820        }
3821
3822        res = requests.post(url, json=json_req, headers=headers)
3823
3824        if not res.ok:
3825            error_msg = self._try_extract_error_code(res)
3826            logger.error(error_msg)
3827            raise BoostedAPIException(f"Failed to create watchlist from file: {error_msg}")
3828
3829        data = res.json()
3830        return data["watchlist_id"]
3831
3832    def get_watchlists(self) -> List[Dict]:
3833        url = f"{self.base_uri}{ROUTE_PREFIX}{DAL_WATCHLIST_ROUTE}/get_user_watchlists/"
3834        headers = {"Authorization": "ApiKey " + self.api_key, "Content-Type": "application/json"}
3835        req_json: Dict = {}
3836        res = requests.post(url, json=req_json, headers=headers, **self._request_params)
3837
3838        if not res.ok:
3839            error_msg = self._try_extract_error_code(res)
3840            logger.error(error_msg)
3841            raise BoostedAPIException(f"Failed to get user watchlists: {error_msg}")
3842
3843        data = res.json()
3844        return data["watchlists"]
3845
3846    def get_watchlist_contents(self, watchlist_id) -> Dict:
3847        url = f"{self.base_uri}{ROUTE_PREFIX}{DAL_WATCHLIST_ROUTE}/contents/"
3848        headers = {"Authorization": "ApiKey " + self.api_key, "Content-Type": "application/json"}
3849        req_json = {"watchlist_id": watchlist_id}
3850        res = requests.post(url, json=req_json, headers=headers, **self._request_params)
3851
3852        if not res.ok:
3853            error_msg = self._try_extract_error_code(res)
3854            logger.error(error_msg)
3855            raise BoostedAPIException(f"Failed to get watchlist contents: {error_msg}")
3856
3857        data = res.json()
3858        return data
3859
3860    def get_watchlist_contents_as_csv(self, watchlist_id, filepath) -> None:
3861        data = self.get_watchlist_contents(watchlist_id)
3862        df = pd.DataFrame(data["contents"])
3863        df.to_csv(filepath, index=False)
3864
3865    # TODO this will need to be enhanced to accept country/currency overrides
3866    def add_securities_to_watchlist(
3867        self, watchlist_id: str, identifiers: List[str], identifier_type: Literal["TICKER", "ISIN"]
3868    ) -> Dict:
3869        # should we just make the arg lower? all caps has a flag-like feel to it
3870        id_type = identifier_type.lower()
3871        url = f"{self.base_uri}{WATCHLIST_ROUTE_PREFIX}{DAL_WATCHLIST_ROUTE}/add_{id_type}s/"
3872        headers = {"Authorization": "ApiKey " + self.api_key, "Content-Type": "application/json"}
3873        req_json = {"watchlist_id": watchlist_id, id_type: identifiers}
3874        res = requests.post(url, json=req_json, headers=headers, **self._request_params)
3875
3876        if not res.ok:
3877            error_msg = self._try_extract_error_code(res)
3878            logger.error(error_msg)
3879            raise BoostedAPIException(f"Failed to get user models: {error_msg}")
3880
3881        data = res.json()
3882        return data
3883
3884    def remove_securities_from_watchlist(
3885        self, watchlist_id: str, identifiers: List[str], identifier_type: Literal["TICKER", "ISIN"]
3886    ) -> Dict:
3887        # should we just make the arg lower? all caps has a flag-like feel to it
3888        id_type = identifier_type.lower()
3889        url = f"{self.base_uri}{WATCHLIST_ROUTE_PREFIX}{DAL_WATCHLIST_ROUTE}/remove_{id_type}s/"
3890        headers = {"Authorization": "ApiKey " + self.api_key, "Content-Type": "application/json"}
3891        req_json = {"watchlist_id": watchlist_id, id_type: identifiers}
3892        res = requests.post(url, json=req_json, headers=headers, **self._request_params)
3893
3894        if not res.ok:
3895            error_msg = self._try_extract_error_code(res)
3896            logger.error(error_msg)
3897            raise BoostedAPIException(f"Failed to get user models: {error_msg}")
3898
3899        data = res.json()
3900        return data
3901
3902    def get_portfolio_groups(
3903        self,
3904    ) -> Dict:
3905        """
3906        Parameters: None
3907
3908
3909        Returns:
3910        ----------
3911
3912        Dict:  {
3913        user_id: str
3914        portfolio_groups: List[PortfolioGroup]
3915        }
3916        where PortfolioGroup is defined as = Dict {
3917        group_id: str
3918        group_name: str
3919        portfolios: List[PortfolioInGroup]
3920        }
3921        where PortfolioInGroup is defined as = Dict {
3922        portfolio_id: str
3923        rank_in_group: Optional[int]
3924        }
3925        """
3926        url = f"{self.base_uri}{WATCHLIST_ROUTE_PREFIX}{PORTFOLIO_GROUP_ROUTE}/get"
3927        headers = {"Authorization": "ApiKey " + self.api_key, "Content-Type": "application/json"}
3928        req_json: Dict = {}
3929        res = requests.post(url, json=req_json, headers=headers, **self._request_params)
3930
3931        if not res.ok:
3932            error_msg = self._try_extract_error_code(res)
3933            logger.error(error_msg)
3934            raise BoostedAPIException(f"Failed to get user portfolio groups: {error_msg}")
3935
3936        data = res.json()
3937        return data
3938
3939    def get_portfolio_group(self, portfolio_group_id: str) -> Dict:
3940        """
3941        Parameters:
3942        portfolio_group_id: str
3943           UUID identifier for the portfolio group
3944
3945
3946        Returns:
3947        ----------
3948
3949        PortfolioGroup: Dict:  {
3950        group_id: str
3951        group_name: str
3952        portfolios: List[PortfolioInGroup]
3953        }
3954        where PortfolioInGroup is defined as = Dict {
3955        portfolio_id: str
3956        portfolio_name: str
3957        rank_in_group: Optional[int]
3958        }
3959        """
3960        url = f"{self.base_uri}{WATCHLIST_ROUTE_PREFIX}{PORTFOLIO_GROUP_ROUTE}/get-one"
3961        headers = {"Authorization": "ApiKey " + self.api_key, "Content-Type": "application/json"}
3962        req_json = {"portfolio_group_id": portfolio_group_id}
3963        res = requests.post(url, json=req_json, headers=headers, **self._request_params)
3964
3965        if not res.ok:
3966            error_msg = self._try_extract_error_code(res)
3967            logger.error(error_msg)
3968            raise BoostedAPIException(f"Failed to get user portfolio groups: {error_msg}")
3969
3970        data = res.json()
3971        return data
3972
3973    def set_sticky_portfolio_group(
3974        self,
3975        portfolio_group_id: str,
3976    ) -> Dict:
3977        """
3978        Set sticky portfolio group
3979
3980        Parameters
3981        ----------
3982
3983        group_id: str,
3984           UUID str identifying a portfolio group
3985
3986        Returns:
3987        -------
3988        Dict {
3989            changed: int - 1 == success
3990        }
3991        """
3992        url = f"{self.base_uri}{WATCHLIST_ROUTE_PREFIX}{PORTFOLIO_GROUP_ROUTE}/set-sticky"
3993        headers = {"Authorization": "ApiKey " + self.api_key, "Content-Type": "application/json"}
3994        req_json = {"portfolio_group_id": portfolio_group_id}
3995        res = requests.post(url, json=req_json, headers=headers, **self._request_params)
3996
3997        if not res.ok:
3998            error_msg = self._try_extract_error_code(res)
3999            logger.error(error_msg)
4000            raise BoostedAPIException(f"Failed to set sticky portfolio group: {error_msg}")
4001
4002        data = res.json()
4003        return data
4004
4005    def get_sticky_portfolio_group(
4006        self,
4007    ) -> Dict:
4008        """
4009        Get sticky portfolio group for the user
4010
4011        Parameters
4012        ----------
4013
4014        Returns:
4015        -------
4016        Dict {
4017            group_id: str
4018            group_name: str
4019            portfolios: List[PortfolioInGroup(Dict)]
4020                  PortfolioInGroup(Dict):
4021                           portfolio_id: str
4022                           rank_in_group: Optional[int] = None
4023                           portfolio_name: Optional[str] = None
4024        }
4025        """
4026        url = f"{self.base_uri}{WATCHLIST_ROUTE_PREFIX}{PORTFOLIO_GROUP_ROUTE}/get-sticky"
4027        headers = {"Authorization": "ApiKey " + self.api_key, "Content-Type": "application/json"}
4028        req_json: Dict = {}
4029        res = requests.post(url, json=req_json, headers=headers, **self._request_params)
4030
4031        if not res.ok:
4032            error_msg = self._try_extract_error_code(res)
4033            logger.error(error_msg)
4034            raise BoostedAPIException(f"Failed to get sticky portfolio group: {error_msg}")
4035
4036        data = res.json()
4037        return data
4038
4039    def create_portfolio_group(
4040        self,
4041        group_name: str,