Feature/extend data source eod historical data by asset class and isin (#1791)

* Extend EodHistoricalDataService

* asset and asset sub class
* isin

* Update changelog

CHANGELOG.md

@@ -7,6 +7,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## Unreleased
 
+### Added
+
+- Added support for asset and asset sub class to the `EOD_HISTORICAL_DATA` data source type
+- Added `isin` to the asset profile model
+
 ### Changed
 
 - Improved the language localization for _Gather Data_

apps/api/src/app/import/import.service.ts

@@ -254,6 +254,7 @@ export class ImportService {
             countries: null,
             createdAt: undefined,
             id: undefined,
+            isin: null,
             name: null,
             scraperConfiguration: null,
             sectors: null,

apps/api/src/app/symbol/symbol.controller.ts

@@ -1,15 +1,18 @@
 import { TransformDataSourceInRequestInterceptor } from '@ghostfolio/api/interceptors/transform-data-source-in-request.interceptor';
 import { TransformDataSourceInResponseInterceptor } from '@ghostfolio/api/interceptors/transform-data-source-in-response.interceptor';
 import { IDataProviderHistoricalResponse } from '@ghostfolio/api/services/interfaces/interfaces';
+import type { RequestWithUser } from '@ghostfolio/common/types';
 import {
   Controller,
   Get,
   HttpException,
+  Inject,
   Param,
   Query,
   UseGuards,
   UseInterceptors
 } from '@nestjs/common';
+import { REQUEST } from '@nestjs/core';
 import { AuthGuard } from '@nestjs/passport';
 import { DataSource } from '@prisma/client';
 import { StatusCodes, getReasonPhrase } from 'http-status-codes';
@@ -21,7 +24,10 @@ import { SymbolService } from './symbol.service';
 
 @Controller('symbol')
 export class SymbolController {
-  public constructor(private readonly symbolService: SymbolService) {}
+  public constructor(
+    @Inject(REQUEST) private readonly request: RequestWithUser,
+    private readonly symbolService: SymbolService
+  ) {}
 
   /**
    * Must be before /:symbol
@@ -33,7 +39,10 @@ export class SymbolController {
     @Query() { query = '' }
   ): Promise<{ items: LookupItem[] }> {
     try {
-      return this.symbolService.lookup(query.toLowerCase());
+      return this.symbolService.lookup({
+        query: query.toLowerCase(),
+        user: this.request.user
+      });
     } catch {
       throw new HttpException(
         getReasonPhrase(StatusCodes.INTERNAL_SERVER_ERROR),

apps/api/src/app/symbol/symbol.service.ts

@@ -5,7 +5,10 @@ import {
 } from '@ghostfolio/api/services/interfaces/interfaces';
 import { MarketDataService } from '@ghostfolio/api/services/market-data.service';
 import { DATE_FORMAT } from '@ghostfolio/common/helper';
-import { HistoricalDataItem } from '@ghostfolio/common/interfaces';
+import {
+  HistoricalDataItem,
+  UserWithSettings
+} from '@ghostfolio/common/interfaces';
 import { Injectable, Logger } from '@nestjs/common';
 import { format, subDays } from 'date-fns';
 
@@ -79,15 +82,24 @@ export class SymbolService {
     };
   }
 
-  public async lookup(aQuery: string): Promise<{ items: LookupItem[] }> {
+  public async lookup({
+    query,
+    user
+  }: {
+    query: string;
+    user: UserWithSettings;
+  }): Promise<{ items: LookupItem[] }> {
     const results: { items: LookupItem[] } = { items: [] };
 
-    if (!aQuery) {
+    if (!query) {
       return results;
     }
 
     try {
-      const { items } = await this.dataProviderService.search(aQuery);
+      const { items } = await this.dataProviderService.search({
+        query,
+        user
+      });
       results.items = items;
       return results;
     } catch (error) {

apps/api/src/services/data-gathering.service.ts

@@ -152,10 +152,11 @@ export class DataGatheringService {
         countries,
         currency,
         dataSource,
+        isin,
         name,
         sectors,
         url
-      } = assetProfiles[symbol];
+      } = assetProfile;
 
       try {
         await this.prismaService.symbolProfile.upsert({
@@ -165,6 +166,7 @@ export class DataGatheringService {
             countries,
             currency,
             dataSource,
+            isin,
             name,
             sectors,
             symbol,
@@ -175,6 +177,7 @@ export class DataGatheringService {
             assetSubClass,
             countries,
             currency,
+            isin,
             name,
             sectors,
             url

apps/api/src/services/data-provider/data-provider.service.ts

@@ -8,6 +8,7 @@ import {
 } from '@ghostfolio/api/services/interfaces/interfaces';
 import { PrismaService } from '@ghostfolio/api/services/prisma.service';
 import { DATE_FORMAT } from '@ghostfolio/common/helper';
+import { UserWithSettings } from '@ghostfolio/common/interfaces';
 import { Granularity } from '@ghostfolio/common/types';
 import { Inject, Injectable, Logger } from '@nestjs/common';
 import { DataSource, MarketData, SymbolProfile } from '@prisma/client';
@@ -260,18 +261,33 @@ export class DataProviderService {
     return response;
   }
 
-  public async search(aQuery: string): Promise<{ items: LookupItem[] }> {
+  public async search({
+    query,
+    user
+  }: {
+    query: string;
+    user: UserWithSettings;
+  }): Promise<{ items: LookupItem[] }> {
     const promises: Promise<{ items: LookupItem[] }>[] = [];
     let lookupItems: LookupItem[] = [];
 
-    if (aQuery?.length < 2) {
+    if (query?.length < 2) {
       return { items: lookupItems };
     }
 
-    for (const dataSource of this.configurationService.get('DATA_SOURCES')) {
+    let dataSources = this.configurationService.get('DATA_SOURCES');
-      promises.push(
+
-        this.getDataProvider(DataSource[dataSource]).search(aQuery)
+    if (
-      );
+      this.configurationService.get('ENABLE_FEATURE_SUBSCRIPTION') &&
+      user.subscription.type === 'Basic'
+    ) {
+      dataSources = dataSources.filter((dataSource) => {
+        return !this.isPremiumDataSource(DataSource[dataSource]);
+      });
+    }
+
+    for (const dataSource of dataSources) {
+      promises.push(this.getDataProvider(DataSource[dataSource]).search(query));
     }
 
     const searchResults = await Promise.all(promises);
@@ -305,4 +321,9 @@ export class DataProviderService {
 
     throw new Error('No data provider has been found.');
   }
+
+  private isPremiumDataSource(aDataSource: DataSource) {
+    const premiumDataSources: DataSource[] = [DataSource.EOD_HISTORICAL_DATA];
+    return premiumDataSources.includes(aDataSource);
+  }
 }

apps/api/src/services/data-provider/eod-historical-data/eod-historical-data.service.ts

@@ -8,7 +8,12 @@ import {
 import { DATE_FORMAT } from '@ghostfolio/common/helper';
 import { Granularity } from '@ghostfolio/common/types';
 import { Injectable, Logger } from '@nestjs/common';
-import { DataSource, SymbolProfile } from '@prisma/client';
+import {
+  AssetClass,
+  AssetSubClass,
+  DataSource,
+  SymbolProfile
+} from '@prisma/client';
 import bent from 'bent';
 import { format, isToday } from 'date-fns';
 
@@ -30,12 +35,15 @@ export class EodHistoricalDataService implements DataProviderInterface {
   public async getAssetProfile(
     aSymbol: string
   ): Promise<Partial<SymbolProfile>> {
-    const { items } = await this.search(aSymbol);
+    const [searchResult] = await this.getSearchResult(aSymbol);
 
     return {
-      currency: items[0]?.currency,
+      assetClass: searchResult?.assetClass,
+      assetSubClass: searchResult?.assetSubClass,
+      currency: searchResult?.currency,
       dataSource: this.getName(),
-      name: items[0]?.name
+      isin: searchResult?.isin,
+      name: searchResult?.name
     };
   }
 
@@ -156,7 +164,27 @@ export class EodHistoricalDataService implements DataProviderInterface {
   }
 
   public async search(aQuery: string): Promise<{ items: LookupItem[] }> {
-    let items: LookupItem[] = [];
+    const searchResult = await this.getSearchResult(aQuery);
+
+    return {
+      items: searchResult
+        .filter(({ symbol }) => {
+          return !symbol.toLowerCase().endsWith('forex');
+        })
+        .map(({ currency, dataSource, name, symbol }) => {
+          return { currency, dataSource, name, symbol };
+        })
+    };
+  }
+
+  private async getSearchResult(aQuery: string): Promise<
+    (LookupItem & {
+      assetClass: AssetClass;
+      assetSubClass: AssetSubClass;
+      isin: string;
+    })[]
+  > {
+    let searchResult = [];
 
     try {
       const get = bent(
@@ -167,10 +195,25 @@ export class EodHistoricalDataService implements DataProviderInterface {
       );
       const response = await get();
 
-      items = response.map(
+      searchResult = response.map(
-        ({ Code, Currency: currency, Exchange, Name: name }) => {
+        ({
+          Code,
+          Currency: currency,
+          Exchange,
+          ISIN: isin,
+          Name: name,
+          Type
+        }) => {
+          const { assetClass, assetSubClass } = this.parseAssetClass({
+            Exchange,
+            Type
+          });
+
           return {
+            assetClass,
+            assetSubClass,
             currency,
+            isin,
             name,
             dataSource: this.getName(),
             symbol: `${Code}.${Exchange}`
@@ -181,6 +224,41 @@ export class EodHistoricalDataService implements DataProviderInterface {
       Logger.error(error, 'EodHistoricalDataService');
     }
 
-    return { items };
+    return searchResult;
+  }
+
+  private parseAssetClass({
+    Exchange,
+    Type
+  }: {
+    Exchange: string;
+    Type: string;
+  }): {
+    assetClass: AssetClass;
+    assetSubClass: AssetSubClass;
+  } {
+    let assetClass: AssetClass;
+    let assetSubClass: AssetSubClass;
+
+    switch (Type?.toLowerCase()) {
+      case 'common stock':
+        assetClass = AssetClass.EQUITY;
+        assetSubClass = AssetSubClass.STOCK;
+        break;
+      case 'currency':
+        assetClass = AssetClass.CASH;
+
+        if (Exchange?.toLowerCase() === 'cc') {
+          assetSubClass = AssetSubClass.CRYPTOCURRENCY;
+        }
+
+        break;
+      case 'etf':
+        assetClass = AssetClass.EQUITY;
+        assetSubClass = AssetSubClass.ETF;
+        break;
+    }
+
+    return { assetClass, assetSubClass };
   }
 }

apps/client/src/app/components/subscription-interstitial-dialog/subscription-interstitial-dialog.html

@@ -17,17 +17,21 @@
       <ion-icon class="mr-1" name="checkmark-circle-outline"></ion-icon>
       <span i18n>Portfolio Summary</span>
     </li>
+    <li class="align-items-center d-flex mb-1">
+      <ion-icon class="mr-1" name="checkmark-circle-outline"></ion-icon>
+      <span i18n>Portfolio Allocations</span>
+    </li>
     <li class="align-items-center d-flex mb-1">
       <ion-icon class="mr-1" name="checkmark-circle-outline"></ion-icon>
       <span i18n>Performance Benchmarks</span>
     </li>
     <li class="align-items-center d-flex mb-1">
       <ion-icon class="mr-1" name="checkmark-circle-outline"></ion-icon>
-      <span i18n>Allocations</span>
+      <span i18n>FIRE Calculator</span>
     </li>
     <li class="align-items-center d-flex mb-1">
       <ion-icon class="mr-1" name="checkmark-circle-outline"></ion-icon>
-      <span i18n>FIRE Calculator</span>
+      <span i18n>Professional Data Provider</span>
     </li>
     <li class="align-items-center d-flex mb-1">
       <ion-icon class="mr-1" name="checkmark-circle-outline"></ion-icon>

apps/client/src/app/pages/pricing/pricing-page.html

@@ -280,6 +280,13 @@
                     <ion-icon name="information-circle-outline"></ion-icon>
                   </span>
                 </li>
+                <li class="align-items-center d-flex mb-1">
+                  <ion-icon
+                    class="mr-1"
+                    name="checkmark-circle-outline"
+                  ></ion-icon>
+                  <span i18n>Professional Data Provider</span>
+                </li>
                 <li class="align-items-center d-flex mb-1">
                   <ion-icon
                     class="mr-1"

prisma/migrations/20230318081658_added_isin_to_symbol_profile/migration.sql

@@ -0,0 +1,2 @@
+-- AlterTable
+ALTER TABLE "SymbolProfile" ADD COLUMN "isin" TEXT;

prisma/schema.prisma

@@ -119,6 +119,7 @@ model SymbolProfile {
   currency               String
   dataSource             DataSource
   id                     String                  @id @default(uuid())
+  isin                   String?
   name                   String?
   updatedAt              DateTime                @updatedAt
   scraperConfiguration   Json?