Feature/extend data source eod historical data by asset class and isin (#1791)

* Extend EodHistoricalDataService * asset and asset sub class * isin * Update changelog
2023-03-18 10:09:11 +01:00 · 2023-03-18 10:09:11 +01:00 · c8ca82b803
commit c8ca82b803
parent 5db2faa17d
11 changed files with 166 additions and 23 deletions
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@ -7,6 +7,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0

 ## Unreleased

+### Added
+
+- Added support for asset and asset sub class to the `EOD_HISTORICAL_DATA` data source type
+- Added `isin` to the asset profile model
+
 ### Changed

 - Improved the language localization for _Gather Data_
--- a/apps/api/src/app/import/import.service.ts
+++ b/apps/api/src/app/import/import.service.ts
@ -254,6 +254,7 @@ export class ImportService {
            countries: null,
            createdAt: undefined,
            id: undefined,
+            isin: null,
            name: null,
            scraperConfiguration: null,
            sectors: null,
--- a/apps/api/src/app/symbol/symbol.controller.ts
+++ b/apps/api/src/app/symbol/symbol.controller.ts
@ -1,15 +1,18 @@
 import { TransformDataSourceInRequestInterceptor } from '@ghostfolio/api/interceptors/transform-data-source-in-request.interceptor';
 import { TransformDataSourceInResponseInterceptor } from '@ghostfolio/api/interceptors/transform-data-source-in-response.interceptor';
 import { IDataProviderHistoricalResponse } from '@ghostfolio/api/services/interfaces/interfaces';
+import type { RequestWithUser } from '@ghostfolio/common/types';
 import {
  Controller,
  Get,
  HttpException,
+  Inject,
  Param,
  Query,
  UseGuards,
  UseInterceptors
 } from '@nestjs/common';
+import { REQUEST } from '@nestjs/core';
 import { AuthGuard } from '@nestjs/passport';
 import { DataSource } from '@prisma/client';
 import { StatusCodes, getReasonPhrase } from 'http-status-codes';
@ -21,7 +24,10 @@ import { SymbolService } from './symbol.service';

@Controller('symbol')
 export class SymbolController {
-  public constructor(private readonly symbolService: SymbolService) {}
+  public constructor(
+    @Inject(REQUEST) private readonly request: RequestWithUser,
+    private readonly symbolService: SymbolService
+  ) {}

  /**
   * Must be before /:symbol
@ -33,7 +39,10 @@ export class SymbolController {
    @Query() { query = '' }
  ): Promise<{ items: LookupItem[] }> {
    try {
-      return this.symbolService.lookup(query.toLowerCase());
+      return this.symbolService.lookup({
+        query: query.toLowerCase(),
+        user: this.request.user
+      });
    } catch {
      throw new HttpException(
        getReasonPhrase(StatusCodes.INTERNAL_SERVER_ERROR),
--- a/apps/api/src/app/symbol/symbol.service.ts
+++ b/apps/api/src/app/symbol/symbol.service.ts
@ -5,7 +5,10 @@ import {
 } from '@ghostfolio/api/services/interfaces/interfaces';
 import { MarketDataService } from '@ghostfolio/api/services/market-data.service';
 import { DATE_FORMAT } from '@ghostfolio/common/helper';
-import { HistoricalDataItem } from '@ghostfolio/common/interfaces';
+import {
+  HistoricalDataItem,
+  UserWithSettings
+} from '@ghostfolio/common/interfaces';
 import { Injectable, Logger } from '@nestjs/common';
 import { format, subDays } from 'date-fns';

@ -79,15 +82,24 @@ export class SymbolService {
    };
  }

-  public async lookup(aQuery: string): Promise<{ items: LookupItem[] }> {
+  public async lookup({
+    query,
+    user
+  }: {
+    query: string;
+    user: UserWithSettings;
+  }): Promise<{ items: LookupItem[] }> {
    const results: { items: LookupItem[] } = { items: [] };

-    if (!aQuery) {
+    if (!query) {
      return results;
    }

    try {
-      const { items } = await this.dataProviderService.search(aQuery);
+      const { items } = await this.dataProviderService.search({
+        query,
+        user
+      });
      results.items = items;
      return results;
    } catch (error) {
--- a/apps/api/src/services/data-gathering.service.ts
+++ b/apps/api/src/services/data-gathering.service.ts
@ -152,10 +152,11 @@ export class DataGatheringService {
        countries,
        currency,
        dataSource,
+        isin,
        name,
        sectors,
        url
-      } = assetProfiles[symbol];
+      } = assetProfile;

      try {
        await this.prismaService.symbolProfile.upsert({
@ -165,6 +166,7 @@ export class DataGatheringService {
            countries,
            currency,
            dataSource,
+            isin,
            name,
            sectors,
            symbol,
@ -175,6 +177,7 @@ export class DataGatheringService {
            assetSubClass,
            countries,
            currency,
+            isin,
            name,
            sectors,
            url
--- a/apps/api/src/services/data-provider/data-provider.service.ts
+++ b/apps/api/src/services/data-provider/data-provider.service.ts
@ -8,6 +8,7 @@ import {
 } from '@ghostfolio/api/services/interfaces/interfaces';
 import { PrismaService } from '@ghostfolio/api/services/prisma.service';
 import { DATE_FORMAT } from '@ghostfolio/common/helper';
+import { UserWithSettings } from '@ghostfolio/common/interfaces';
 import { Granularity } from '@ghostfolio/common/types';
 import { Inject, Injectable, Logger } from '@nestjs/common';
 import { DataSource, MarketData, SymbolProfile } from '@prisma/client';
@ -260,18 +261,33 @@ export class DataProviderService {
    return response;
  }

-  public async search(aQuery: string): Promise<{ items: LookupItem[] }> {
+  public async search({
+    query,
+    user
+  }: {
+    query: string;
+    user: UserWithSettings;
+  }): Promise<{ items: LookupItem[] }> {
    const promises: Promise<{ items: LookupItem[] }>[] = [];
    let lookupItems: LookupItem[] = [];

-    if (aQuery?.length < 2) {
+    if (query?.length < 2) {
      return { items: lookupItems };
    }

-    for (const dataSource of this.configurationService.get('DATA_SOURCES')) {
-      promises.push(
-        this.getDataProvider(DataSource[dataSource]).search(aQuery)
-      );
+    let dataSources = this.configurationService.get('DATA_SOURCES');
+
+    if (
+      this.configurationService.get('ENABLE_FEATURE_SUBSCRIPTION') &&
+      user.subscription.type === 'Basic'
+    ) {
+      dataSources = dataSources.filter((dataSource) => {
+        return !this.isPremiumDataSource(DataSource[dataSource]);
+      });
+    }
+
+    for (const dataSource of dataSources) {
+      promises.push(this.getDataProvider(DataSource[dataSource]).search(query));
    }

    const searchResults = await Promise.all(promises);
@ -305,4 +321,9 @@ export class DataProviderService {

    throw new Error('No data provider has been found.');
  }
+
+  private isPremiumDataSource(aDataSource: DataSource) {
+    const premiumDataSources: DataSource[] = [DataSource.EOD_HISTORICAL_DATA];
+    return premiumDataSources.includes(aDataSource);
+  }
 }
--- a/apps/api/src/services/data-provider/eod-historical-data/eod-historical-data.service.ts
+++ b/apps/api/src/services/data-provider/eod-historical-data/eod-historical-data.service.ts
@ -8,7 +8,12 @@ import {
 import { DATE_FORMAT } from '@ghostfolio/common/helper';
 import { Granularity } from '@ghostfolio/common/types';
 import { Injectable, Logger } from '@nestjs/common';
-import { DataSource, SymbolProfile } from '@prisma/client';
+import {
+  AssetClass,
+  AssetSubClass,
+  DataSource,
+  SymbolProfile
+} from '@prisma/client';
 import bent from 'bent';
 import { format, isToday } from 'date-fns';

@ -30,12 +35,15 @@ export class EodHistoricalDataService implements DataProviderInterface {
  public async getAssetProfile(
    aSymbol: string
  ): Promise<Partial<SymbolProfile>> {
-    const { items } = await this.search(aSymbol);
+    const [searchResult] = await this.getSearchResult(aSymbol);

    return {
-      currency: items[0]?.currency,
+      assetClass: searchResult?.assetClass,
+      assetSubClass: searchResult?.assetSubClass,
+      currency: searchResult?.currency,
      dataSource: this.getName(),
-      name: items[0]?.name
+      isin: searchResult?.isin,
+      name: searchResult?.name
    };
  }

@ -156,7 +164,27 @@ export class EodHistoricalDataService implements DataProviderInterface {
  }

  public async search(aQuery: string): Promise<{ items: LookupItem[] }> {
-    let items: LookupItem[] = [];
+    const searchResult = await this.getSearchResult(aQuery);
+
+    return {
+      items: searchResult
+        .filter(({ symbol }) => {
+          return !symbol.toLowerCase().endsWith('forex');
+        })
+        .map(({ currency, dataSource, name, symbol }) => {
+          return { currency, dataSource, name, symbol };
+        })
+    };
+  }
+
+  private async getSearchResult(aQuery: string): Promise<
+    (LookupItem & {
+      assetClass: AssetClass;
+      assetSubClass: AssetSubClass;
+      isin: string;
+    })[]
+  > {
+    let searchResult = [];

    try {
      const get = bent(
@ -167,10 +195,25 @@ export class EodHistoricalDataService implements DataProviderInterface {
      );
      const response = await get();

-      items = response.map(
-        ({ Code, Currency: currency, Exchange, Name: name }) => {
+      searchResult = response.map(
+        ({
+          Code,
+          Currency: currency,
+          Exchange,
+          ISIN: isin,
+          Name: name,
+          Type
+        }) => {
+          const { assetClass, assetSubClass } = this.parseAssetClass({
+            Exchange,
+            Type
+          });
+
          return {
+            assetClass,
+            assetSubClass,
            currency,
+            isin,
            name,
            dataSource: this.getName(),
            symbol: `${Code}.${Exchange}`
@ -181,6 +224,41 @@ export class EodHistoricalDataService implements DataProviderInterface {
      Logger.error(error, 'EodHistoricalDataService');
    }

-    return { items };
+    return searchResult;
+  }
+
+  private parseAssetClass({
+    Exchange,
+    Type
+  }: {
+    Exchange: string;
+    Type: string;
+  }): {
+    assetClass: AssetClass;
+    assetSubClass: AssetSubClass;
+  } {
+    let assetClass: AssetClass;
+    let assetSubClass: AssetSubClass;
+
+    switch (Type?.toLowerCase()) {
+      case 'common stock':
+        assetClass = AssetClass.EQUITY;
+        assetSubClass = AssetSubClass.STOCK;
+        break;
+      case 'currency':
+        assetClass = AssetClass.CASH;
+
+        if (Exchange?.toLowerCase() === 'cc') {
+          assetSubClass = AssetSubClass.CRYPTOCURRENCY;
+        }
+
+        break;
+      case 'etf':
+        assetClass = AssetClass.EQUITY;
+        assetSubClass = AssetSubClass.ETF;
+        break;
+    }
+
+    return { assetClass, assetSubClass };
  }
 }
--- a/apps/client/src/app/components/subscription-interstitial-dialog/subscription-interstitial-dialog.html
+++ b/apps/client/src/app/components/subscription-interstitial-dialog/subscription-interstitial-dialog.html
@ -17,17 +17,21 @@
      <ion-icon class="mr-1" name="checkmark-circle-outline"></ion-icon>
      <span i18n>Portfolio Summary</span>
    </li>
+    <li class="align-items-center d-flex mb-1">
+      <ion-icon class="mr-1" name="checkmark-circle-outline"></ion-icon>
+      <span i18n>Portfolio Allocations</span>
+    </li>
    <li class="align-items-center d-flex mb-1">
      <ion-icon class="mr-1" name="checkmark-circle-outline"></ion-icon>
      <span i18n>Performance Benchmarks</span>
    </li>
    <li class="align-items-center d-flex mb-1">
      <ion-icon class="mr-1" name="checkmark-circle-outline"></ion-icon>
-      <span i18n>Allocations</span>
+      <span i18n>FIRE Calculator</span>
    </li>
    <li class="align-items-center d-flex mb-1">
      <ion-icon class="mr-1" name="checkmark-circle-outline"></ion-icon>
-      <span i18n>FIRE Calculator</span>
+      <span i18n>Professional Data Provider</span>
    </li>
    <li class="align-items-center d-flex mb-1">
      <ion-icon class="mr-1" name="checkmark-circle-outline"></ion-icon>
--- a/apps/client/src/app/pages/pricing/pricing-page.html
+++ b/apps/client/src/app/pages/pricing/pricing-page.html
@ -280,6 +280,13 @@
                    <ion-icon name="information-circle-outline"></ion-icon>
                  </span>
                </li>
+                <li class="align-items-center d-flex mb-1">
+                  <ion-icon
+                    class="mr-1"
+                    name="checkmark-circle-outline"
+                  ></ion-icon>
+                  <span i18n>Professional Data Provider</span>
+                </li>
                <li class="align-items-center d-flex mb-1">
                  <ion-icon
                    class="mr-1"
--- a/prisma/migrations/20230318081658_added_isin_to_symbol_profile/migration.sql
+++ b/prisma/migrations/20230318081658_added_isin_to_symbol_profile/migration.sql
@ -0,0 +1,2 @@
+-- AlterTable
+ALTER TABLE "SymbolProfile" ADD COLUMN "isin" TEXT;
--- a/prisma/schema.prisma
+++ b/prisma/schema.prisma
@ -119,6 +119,7 @@ model SymbolProfile {
  currency               String
  dataSource             DataSource
  id                     String                  @id @default(uuid())
+  isin                   String?
  name                   String?
  updatedAt              DateTime                @updatedAt
  scraperConfiguration   Json?