add sectors and countries for ETFs (#410)

* Update changelog

Co-Authored-By: Valentin Zickner <valentin@coderworks.de>

Co-authored-by: Thomas Kaul <4159106+dtslvr@users.noreply.github.com>

CHANGELOG.md

@@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## Unreleased
 
+### Added
+
+- Added a data enhancer for symbol profile data (countries and sectors) via _Trackinsight_
+
 ### Changed
 
 - Changed the values of the global heat map to fixed-point notation

apps/api/src/app/cache/cache.module.ts

@@ -2,11 +2,7 @@ import { CacheService } from '@ghostfolio/api/app/cache/cache.service';
 import { RedisCacheModule } from '@ghostfolio/api/app/redis-cache/redis-cache.module';
 import { ConfigurationService } from '@ghostfolio/api/services/configuration.service';
 import { DataGatheringService } from '@ghostfolio/api/services/data-gathering.service';
-import { AlphaVantageService } from '@ghostfolio/api/services/data-provider/alpha-vantage/alpha-vantage.service';
+import { DataProviderModule } from '@ghostfolio/api/services/data-provider/data-provider.module';
-import { DataProviderService } from '@ghostfolio/api/services/data-provider/data-provider.service';
-import { GhostfolioScraperApiService } from '@ghostfolio/api/services/data-provider/ghostfolio-scraper-api/ghostfolio-scraper-api.service';
-import { RakutenRapidApiService } from '@ghostfolio/api/services/data-provider/rakuten-rapid-api/rakuten-rapid-api.service';
-import { YahooFinanceService } from '@ghostfolio/api/services/data-provider/yahoo-finance/yahoo-finance.service';
 import { ExchangeRateDataModule } from '@ghostfolio/api/services/exchange-rate-data.module';
 import { PrismaService } from '@ghostfolio/api/services/prisma.service';
 import { Module } from '@nestjs/common';
@@ -14,18 +10,13 @@ import { Module } from '@nestjs/common';
 import { CacheController } from './cache.controller';
 
 @Module({
-  imports: [ExchangeRateDataModule, RedisCacheModule],
+  imports: [DataProviderModule, ExchangeRateDataModule, RedisCacheModule],
   controllers: [CacheController],
   providers: [
-    AlphaVantageService,
     CacheService,
     ConfigurationService,
     DataGatheringService,
-    DataProviderService,
+    PrismaService
-    GhostfolioScraperApiService,
-    PrismaService,
-    RakutenRapidApiService,
-    YahooFinanceService
   ]
 })
 export class CacheModule {}

apps/api/src/app/info/info.module.ts

@@ -1,10 +1,6 @@
 import { ConfigurationService } from '@ghostfolio/api/services/configuration.service';
 import { DataGatheringService } from '@ghostfolio/api/services/data-gathering.service';
-import { AlphaVantageService } from '@ghostfolio/api/services/data-provider/alpha-vantage/alpha-vantage.service';
+import { DataProviderModule } from '@ghostfolio/api/services/data-provider/data-provider.module';
-import { DataProviderService } from '@ghostfolio/api/services/data-provider/data-provider.service';
-import { GhostfolioScraperApiService } from '@ghostfolio/api/services/data-provider/ghostfolio-scraper-api/ghostfolio-scraper-api.service';
-import { RakutenRapidApiService } from '@ghostfolio/api/services/data-provider/rakuten-rapid-api/rakuten-rapid-api.service';
-import { YahooFinanceService } from '@ghostfolio/api/services/data-provider/yahoo-finance/yahoo-finance.service';
 import { ExchangeRateDataModule } from '@ghostfolio/api/services/exchange-rate-data.module';
 import { PrismaService } from '@ghostfolio/api/services/prisma.service';
 import { Module } from '@nestjs/common';
@@ -15,6 +11,7 @@ import { InfoService } from './info.service';
 
 @Module({
   imports: [
+    DataProviderModule,
     ExchangeRateDataModule,
     JwtModule.register({
       secret: process.env.JWT_SECRET_KEY,
@@ -23,15 +20,10 @@ import { InfoService } from './info.service';
   ],
   controllers: [InfoController],
   providers: [
-    AlphaVantageService,
     ConfigurationService,
     DataGatheringService,
-    DataProviderService,
-    GhostfolioScraperApiService,
     InfoService,
-    PrismaService,
+    PrismaService
-    RakutenRapidApiService,
-    YahooFinanceService
   ]
 })
 export class InfoModule {}

apps/api/src/app/portfolio/current-rate.service.spec.ts

@@ -75,6 +75,7 @@ describe('CurrentRateService', () => {
     dataProviderService = new DataProviderService(
       null,
       null,
+      [],
       null,
       null,
       null,

apps/api/src/services/data-gathering.service.ts

@@ -132,7 +132,15 @@ export class DataGatheringService {
 
     for (const [
       symbol,
-      { assetClass, assetSubClass, countries, currency, dataSource, name }
+      {
+        assetClass,
+        assetSubClass,
+        countries,
+        currency,
+        dataSource,
+        name,
+        sectors
+      }
     ] of Object.entries(currentData)) {
       try {
         await this.prismaService.symbolProfile.upsert({
@@ -143,6 +151,7 @@ export class DataGatheringService {
             currency,
             dataSource,
             name,
+            sectors,
             symbol
           },
           update: {
@@ -150,7 +159,8 @@ export class DataGatheringService {
             assetSubClass,
             countries,
             currency,
-            name
+            name,
+            sectors
           },
           where: {
             dataSource_symbol: {

apps/api/src/services/data-provider/alpha-vantage/alpha-vantage.service.ts

@@ -6,11 +6,11 @@ import { Injectable } from '@nestjs/common';
 import { DataSource } from '@prisma/client';
 import { isAfter, isBefore, parse } from 'date-fns';
 
-import { DataProviderInterface } from '../../interfaces/data-provider.interface';
 import {
   IDataProviderHistoricalResponse,
   IDataProviderResponse
 } from '../../interfaces/interfaces';
+import { DataProviderInterface } from '../interfaces/data-provider.interface';
 import { IAlphaVantageHistoricalResponse } from './interfaces/interfaces';
 
 @Injectable()

apps/api/src/services/data-provider/data-enhancer/trackinsight/trackinsight.service.ts

@@ -0,0 +1,73 @@
+import { DataEnhancerInterface } from '@ghostfolio/api/services/data-provider/interfaces/data-enhancer.interface';
+import { IDataProviderResponse } from '@ghostfolio/api/services/interfaces/interfaces';
+import bent from 'bent';
+
+const getJSON = bent('json');
+
+export class TrackinsightDataEnhancerService implements DataEnhancerInterface {
+  private static baseUrl = 'https://data.trackinsight.com/holdings';
+  private static countries = require('countries-list/dist/countries.json');
+  private static sectorsMapping = {
+    'Consumer Discretionary': 'Consumer Cyclical',
+    'Consumer Defensive': 'Consumer Staples',
+    'Health Care': 'Healthcare',
+    'Information Technology': 'Technology'
+  };
+
+  public async enhance({
+    response,
+    symbol
+  }: {
+    response: IDataProviderResponse;
+    symbol: string;
+  }): Promise<IDataProviderResponse> {
+    if (
+      !(response.assetClass === 'EQUITY' && response.assetSubClass === 'ETF')
+    ) {
+      return response;
+    }
+
+    const holdings = await getJSON(
+      `${TrackinsightDataEnhancerService.baseUrl}/${symbol}.json`
+    ).catch(() => {
+      return getJSON(
+        `${TrackinsightDataEnhancerService.baseUrl}/${
+          symbol.split('.')[0]
+        }.json`
+      );
+    });
+
+    if (!response.countries || response.countries.length === 0) {
+      response.countries = [];
+      for (const [name, value] of Object.entries<any>(holdings.countries)) {
+        let countryCode: string;
+
+        for (const [key, country] of Object.entries<any>(
+          TrackinsightDataEnhancerService.countries
+        )) {
+          if (country.name === name) {
+            countryCode = key;
+            break;
+          }
+        }
+
+        response.countries.push({
+          code: countryCode,
+          weight: value.weight
+        });
+      }
+    }
+
+    if (!response.sectors || response.sectors.length === 0) {
+      response.sectors = [];
+      for (const [name, value] of Object.entries<any>(holdings.sectors)) {
+        response.sectors.push({
+          name: TrackinsightDataEnhancerService.sectorsMapping[name] ?? name,
+          weight: value.weight
+        });
+      }
+    }
+
+    return Promise.resolve(response);
+  }
+}

apps/api/src/services/data-provider/data-provider.module.ts

@@ -1,4 +1,5 @@
 import { ConfigurationModule } from '@ghostfolio/api/services/configuration.module';
+import { TrackinsightDataEnhancerService } from '@ghostfolio/api/services/data-provider/data-enhancer/trackinsight/trackinsight.service';
 import { GhostfolioScraperApiService } from '@ghostfolio/api/services/data-provider/ghostfolio-scraper-api/ghostfolio-scraper-api.service';
 import { RakutenRapidApiService } from '@ghostfolio/api/services/data-provider/rakuten-rapid-api/rakuten-rapid-api.service';
 import { YahooFinanceService } from '@ghostfolio/api/services/data-provider/yahoo-finance/yahoo-finance.service';
@@ -15,7 +16,13 @@ import { DataProviderService } from './data-provider.service';
     DataProviderService,
     GhostfolioScraperApiService,
     RakutenRapidApiService,
-    YahooFinanceService
+    TrackinsightDataEnhancerService,
+    YahooFinanceService,
+    {
+      inject: [TrackinsightDataEnhancerService],
+      provide: 'DataEnhancers',
+      useFactory: (trackinsight) => [trackinsight]
+    }
   ],
   exports: [DataProviderService, GhostfolioScraperApiService]
 })

apps/api/src/services/data-provider/data-provider.service.ts

@@ -1,5 +1,6 @@
 import { LookupItem } from '@ghostfolio/api/app/symbol/interfaces/lookup-item.interface';
 import { ConfigurationService } from '@ghostfolio/api/services/configuration.service';
+import { DataEnhancerInterface } from '@ghostfolio/api/services/data-provider/interfaces/data-enhancer.interface';
 import {
   IDataGatheringItem,
   IDataProviderHistoricalResponse,
@@ -8,7 +9,7 @@ import {
 import { PrismaService } from '@ghostfolio/api/services/prisma.service';
 import { DATE_FORMAT } from '@ghostfolio/common/helper';
 import { Granularity } from '@ghostfolio/common/types';
-import { Injectable } from '@nestjs/common';
+import { Inject, Injectable } from '@nestjs/common';
 import { DataSource, MarketData } from '@prisma/client';
 import { format } from 'date-fns';
 import { isEmpty } from 'lodash';
@@ -16,16 +17,15 @@ import { isEmpty } from 'lodash';
 import { AlphaVantageService } from './alpha-vantage/alpha-vantage.service';
 import { GhostfolioScraperApiService } from './ghostfolio-scraper-api/ghostfolio-scraper-api.service';
 import { RakutenRapidApiService } from './rakuten-rapid-api/rakuten-rapid-api.service';
-import {
+import { YahooFinanceService } from './yahoo-finance/yahoo-finance.service';
-  YahooFinanceService,
-  convertToYahooFinanceSymbol
-} from './yahoo-finance/yahoo-finance.service';
 
 @Injectable()
 export class DataProviderService {
   public constructor(
     private readonly alphaVantageService: AlphaVantageService,
     private readonly configurationService: ConfigurationService,
+    @Inject('DataEnhancers')
+    private readonly dataEnhancers: DataEnhancerInterface[],
     private readonly ghostfolioScraperApiService: GhostfolioScraperApiService,
     private readonly prismaService: PrismaService,
     private readonly rakutenRapidApiService: RakutenRapidApiService,
@@ -42,27 +42,35 @@ export class DataProviderService {
     } = {};
 
     for (const item of items) {
-      if (item.dataSource === DataSource.ALPHA_VANTAGE) {
+      const dataProvider = this.getDataProvider(item.dataSource);
-        response[item.symbol] = (
+      response[item.symbol] = (await dataProvider.get([item.symbol]))[
-          await this.alphaVantageService.get([item.symbol])
+        item.symbol
-        )[item.symbol];
+      ];
-      } else if (item.dataSource === DataSource.GHOSTFOLIO) {
-        response[item.symbol] = (
-          await this.ghostfolioScraperApiService.get([item.symbol])
-        )[item.symbol];
-      } else if (item.dataSource === DataSource.RAKUTEN) {
-        response[item.symbol] = (
-          await this.rakutenRapidApiService.get([item.symbol])
-        )[item.symbol];
-      } else if (item.dataSource === DataSource.YAHOO) {
-        response[item.symbol] = (
-          await this.yahooFinanceService.get([
-            convertToYahooFinanceSymbol(item.symbol)
-          ])
-        )[item.symbol];
-      }
     }
 
+    const promises = [];
+    for (const symbol of Object.keys(response)) {
+      let promise = Promise.resolve(response[symbol]);
+      for (const dataEnhancer of this.dataEnhancers) {
+        promise = promise.then((currentResponse) =>
+          dataEnhancer
+            .enhance({ symbol, response: currentResponse })
+            .catch((error) => {
+              console.error(
+                `Failed to enhance data for symbol ${symbol}`,
+                error
+              );
+              return currentResponse;
+            })
+        );
+      }
+      promises.push(
+        promise.then((currentResponse) => (response[symbol] = currentResponse))
+      );
+    }
+
+    await Promise.all(promises);
+
     return response;
   }
 
@@ -103,11 +111,13 @@ export class DataProviderService {
     });
 
     try {
-      const queryRaw = `SELECT * FROM "MarketData" WHERE "dataSource" IN ('${dataSources.join(
+      const queryRaw = `SELECT *
-        `','`
+                        FROM "MarketData"
-      )}') AND "symbol" IN ('${symbols.join(
+                        WHERE "dataSource" IN ('${dataSources.join(`','`)}')
-        `','`
+                          AND "symbol" IN ('${symbols.join(
-      )}') ${granularityQuery} ${rangeQuery} ORDER BY date;`;
+                            `','`
+                          )}') ${granularityQuery} ${rangeQuery}
+                        ORDER BY date;`;
 
       const marketDataByGranularity: MarketData[] =
         await this.prismaService.$queryRaw(queryRaw);

apps/api/src/services/data-provider/ghostfolio-scraper-api/ghostfolio-scraper-api.service.ts

@@ -12,13 +12,13 @@ import * as bent from 'bent';
 import * as cheerio from 'cheerio';
 import { format } from 'date-fns';
 
-import { DataProviderInterface } from '../../interfaces/data-provider.interface';
 import {
   IDataGatheringItem,
   IDataProviderHistoricalResponse,
   IDataProviderResponse,
   MarketState
 } from '../../interfaces/interfaces';
+import { DataProviderInterface } from '../interfaces/data-provider.interface';
 import { ScraperConfig } from './interfaces/scraper-config.interface';
 
 @Injectable()

apps/api/src/services/data-provider/interfaces/data-enhancer.interface.ts

@@ -0,0 +1,11 @@
+import { IDataProviderResponse } from '@ghostfolio/api/services/interfaces/interfaces';
+
+export interface DataEnhancerInterface {
+  enhance({
+    response,
+    symbol
+  }: {
+    response: IDataProviderResponse;
+    symbol: string;
+  }): Promise<IDataProviderResponse>;
+}

apps/api/src/services/data-provider/interfaces/data-provider.interface.ts

@@ -4,7 +4,7 @@ import { Granularity } from '@ghostfolio/common/types';
 import {
   IDataProviderHistoricalResponse,
   IDataProviderResponse
-} from './interfaces';
+} from '../../interfaces/interfaces';
 
 export interface DataProviderInterface {
   canHandle(symbol: string): boolean;

apps/api/src/services/data-provider/rakuten-rapid-api/rakuten-rapid-api.service.ts

@@ -14,12 +14,12 @@ import { DataSource } from '@prisma/client';
 import * as bent from 'bent';
 import { format, subMonths, subWeeks, subYears } from 'date-fns';
 
-import { DataProviderInterface } from '../../interfaces/data-provider.interface';
 import {
   IDataProviderHistoricalResponse,
   IDataProviderResponse,
   MarketState
 } from '../../interfaces/interfaces';
+import { DataProviderInterface } from '../interfaces/data-provider.interface';
 
 @Injectable()
 export class RakutenRapidApiService implements DataProviderInterface {

apps/api/src/services/data-provider/yahoo-finance/yahoo-finance.service.ts

@@ -10,12 +10,12 @@ import { countries } from 'countries-list';
 import { format } from 'date-fns';
 import * as yahooFinance from 'yahoo-finance';
 
-import { DataProviderInterface } from '../../interfaces/data-provider.interface';
 import {
   IDataProviderHistoricalResponse,
   IDataProviderResponse,
   MarketState
 } from '../../interfaces/interfaces';
+import { DataProviderInterface } from '../interfaces/data-provider.interface';
 import {
   IYahooFinanceHistoricalResponse,
   IYahooFinancePrice,
@@ -33,11 +33,14 @@ export class YahooFinanceService implements DataProviderInterface {
   }
 
   public async get(
-    aYahooFinanceSymbols: string[]
+    aSymbols: string[]
   ): Promise<{ [symbol: string]: IDataProviderResponse }> {
-    if (aYahooFinanceSymbols.length <= 0) {
+    if (aSymbols.length <= 0) {
       return {};
     }
+    const yahooFinanceSymbols = aSymbols.map((symbol) =>
+      this.convertToYahooFinanceSymbol(symbol)
+    );
 
     try {
       const response: { [symbol: string]: IDataProviderResponse } = {};
@@ -46,12 +49,12 @@ export class YahooFinanceService implements DataProviderInterface {
         [symbol: string]: IYahooFinanceQuoteResponse;
       } = await yahooFinance.quote({
         modules: ['price', 'summaryProfile'],
-        symbols: aYahooFinanceSymbols
+        symbols: yahooFinanceSymbols
       });
 
       for (const [yahooFinanceSymbol, value] of Object.entries(data)) {
         // Convert symbols back
-        const symbol = convertFromYahooFinanceSymbol(yahooFinanceSymbol);
+        const symbol = this.convertFromYahooFinanceSymbol(yahooFinanceSymbol);
 
         const { assetClass, assetSubClass } = this.parseAssetClass(value.price);
 
@@ -93,6 +96,12 @@ export class YahooFinanceService implements DataProviderInterface {
               response[symbol].countries = [{ code, weight: 1 }];
             }
           } catch {}
+
+          if (value.summaryProfile?.sector) {
+            response[symbol].sectors = [
+              { name: value.summaryProfile?.sector, weight: 1 }
+            ];
+          }
         }
 
         // Add url if available
@@ -123,7 +132,7 @@ export class YahooFinanceService implements DataProviderInterface {
     }
 
     const yahooFinanceSymbols = aSymbols.map((symbol) => {
-      return convertToYahooFinanceSymbol(symbol);
+      return this.convertToYahooFinanceSymbol(symbol);
     });
 
     try {
@@ -143,7 +152,7 @@ export class YahooFinanceService implements DataProviderInterface {
         historicalData
       )) {
         // Convert symbols back
-        const symbol = convertFromYahooFinanceSymbol(yahooFinanceSymbol);
+        const symbol = this.convertFromYahooFinanceSymbol(yahooFinanceSymbol);
         response[symbol] = {};
 
         timeSeries.forEach((timeSerie) => {
@@ -214,6 +223,40 @@ export class YahooFinanceService implements DataProviderInterface {
     return { items };
   }
 
+  private convertFromYahooFinanceSymbol(aYahooFinanceSymbol: string) {
+    const symbol = aYahooFinanceSymbol.replace('-', '');
+    return symbol.replace('=X', '');
+  }
+
+  /**
+   * Converts a symbol to a Yahoo Finance symbol
+   *
+   * Currency:        USDCHF  -> USDCHF=X
+   * Cryptocurrency:  BTCUSD  -> BTC-USD
+   *                  DOGEUSD -> DOGE-USD
+   *                  SOL1USD -> SOL1-USD
+   */
+  private convertToYahooFinanceSymbol(aSymbol: string) {
+    if (
+      (aSymbol.includes('CHF') ||
+        aSymbol.includes('EUR') ||
+        aSymbol.includes('USD')) &&
+      aSymbol.length >= 6
+    ) {
+      if (isCurrency(aSymbol.substring(0, aSymbol.length - 3))) {
+        return `${aSymbol}=X`;
+      } else if (isCrypto(aSymbol) || isCrypto(aSymbol.replace('1', ''))) {
+        // Add a dash before the last three characters
+        // BTCUSD  -> BTC-USD
+        // DOGEUSD -> DOGE-USD
+        // SOL1USD -> SOL1-USD
+        return aSymbol.replace('USD', '-USD');
+      }
+    }
+
+    return aSymbol;
+  }
+
   private parseAssetClass(aPrice: IYahooFinancePrice): {
     assetClass: AssetClass;
     assetSubClass: AssetSubClass;
@@ -247,37 +290,3 @@ export class YahooFinanceService implements DataProviderInterface {
     return aString;
   }
 }
-
-export const convertFromYahooFinanceSymbol = (aYahooFinanceSymbol: string) => {
-  const symbol = aYahooFinanceSymbol.replace('-', '');
-  return symbol.replace('=X', '');
-};
-
-/**
- * Converts a symbol to a Yahoo Finance symbol
- *
- * Currency:        USDCHF  -> USDCHF=X
- * Cryptocurrency:  BTCUSD  -> BTC-USD
- *                  DOGEUSD -> DOGE-USD
- *                  SOL1USD -> SOL1-USD
- */
-export const convertToYahooFinanceSymbol = (aSymbol: string) => {
-  if (
-    (aSymbol.includes('CHF') ||
-      aSymbol.includes('EUR') ||
-      aSymbol.includes('USD')) &&
-    aSymbol.length >= 6
-  ) {
-    if (isCurrency(aSymbol.substring(0, aSymbol.length - 3))) {
-      return `${aSymbol}=X`;
-    } else if (isCrypto(aSymbol) || isCrypto(aSymbol.replace('1', ''))) {
-      // Add a dash before the last three characters
-      // BTCUSD  -> BTC-USD
-      // DOGEUSD -> DOGE-USD
-      // SOL1USD -> SOL1-USD
-      return aSymbol.replace('USD', '-USD');
-    }
-  }
-
-  return aSymbol;
-};

apps/api/src/services/interfaces/interfaces.ts

@@ -45,6 +45,7 @@ export interface IDataProviderResponse {
   marketPrice: number;
   marketState: MarketState;
   name?: string;
+  sectors?: { name: string; weight: number }[];
   url?: string;
 }