Feature/eliminate ghostfolio scraper api service (#1706)

* Eliminate GhostfolioScraperApiService

* Update changelog

CHANGELOG.md

@@ -5,6 +5,12 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## Unreleased
+
+### Changed
+
+- Eliminated the `GhostfolioScraperApiService`
+
 ## 1.234.0 - 2023-02-15
 
 ### Added

apps/api/src/services/configuration.service.ts

@@ -19,7 +19,7 @@ export class ConfigurationService {
       CACHE_TTL: num({ default: 1 }),
       DATA_SOURCE_PRIMARY: str({ default: DataSource.YAHOO }),
       DATA_SOURCES: json({
-        default: [DataSource.GHOSTFOLIO, DataSource.MANUAL, DataSource.YAHOO]
+        default: [DataSource.MANUAL, DataSource.YAHOO]
       }),
       ENABLE_FEATURE_BLOG: bool({ default: false }),
       ENABLE_FEATURE_CUSTOM_SYMBOLS: bool({ default: false }),

apps/api/src/services/data-gathering.service.ts

@@ -278,7 +278,6 @@ export class DataGatheringService {
     return symbolProfiles
       .filter(({ dataSource }) => {
         return (
-          dataSource !== DataSource.GHOSTFOLIO &&
           dataSource !== DataSource.MANUAL &&
           dataSource !== DataSource.RAPID_API
         );

apps/api/src/services/data-provider/data-provider.module.ts

@@ -2,7 +2,6 @@ import { ConfigurationModule } from '@ghostfolio/api/services/configuration.modu
 import { CryptocurrencyModule } from '@ghostfolio/api/services/cryptocurrency/cryptocurrency.module';
 import { AlphaVantageService } from '@ghostfolio/api/services/data-provider/alpha-vantage/alpha-vantage.service';
 import { EodHistoricalDataService } from '@ghostfolio/api/services/data-provider/eod-historical-data/eod-historical-data.service';
-import { GhostfolioScraperApiService } from '@ghostfolio/api/services/data-provider/ghostfolio-scraper-api/ghostfolio-scraper-api.service';
 import { GoogleSheetsService } from '@ghostfolio/api/services/data-provider/google-sheets/google-sheets.service';
 import { ManualService } from '@ghostfolio/api/services/data-provider/manual/manual.service';
 import { RapidApiService } from '@ghostfolio/api/services/data-provider/rapid-api/rapid-api.service';
@@ -24,7 +23,6 @@ import { DataProviderService } from './data-provider.service';
     AlphaVantageService,
     DataProviderService,
     EodHistoricalDataService,
-    GhostfolioScraperApiService,
     GoogleSheetsService,
     ManualService,
     RapidApiService,
@@ -33,7 +31,6 @@ import { DataProviderService } from './data-provider.service';
       inject: [
         AlphaVantageService,
         EodHistoricalDataService,
-        GhostfolioScraperApiService,
         GoogleSheetsService,
         ManualService,
         RapidApiService,
@@ -43,7 +40,6 @@ import { DataProviderService } from './data-provider.service';
       useFactory: (
         alphaVantageService,
         eodHistoricalDataService,
-        ghostfolioScraperApiService,
         googleSheetsService,
         manualService,
         rapidApiService,
@@ -51,7 +47,6 @@ import { DataProviderService } from './data-provider.service';
       ) => [
         alphaVantageService,
         eodHistoricalDataService,
-        ghostfolioScraperApiService,
         googleSheetsService,
         manualService,
         rapidApiService,
@@ -59,10 +54,6 @@ import { DataProviderService } from './data-provider.service';
       ]
     }
   ],
-  exports: [
+  exports: [DataProviderService, YahooFinanceService]
-    DataProviderService,
-    GhostfolioScraperApiService,
-    YahooFinanceService
-  ]
 })
 export class DataProviderModule {}

apps/api/src/services/data-provider/ghostfolio-scraper-api/ghostfolio-scraper-api.service.ts

@@ -1,194 +0,0 @@
-import { LookupItem } from '@ghostfolio/api/app/symbol/interfaces/lookup-item.interface';
-import { DataProviderInterface } from '@ghostfolio/api/services/data-provider/interfaces/data-provider.interface';
-import {
-  IDataProviderHistoricalResponse,
-  IDataProviderResponse
-} from '@ghostfolio/api/services/interfaces/interfaces';
-import { PrismaService } from '@ghostfolio/api/services/prisma.service';
-import { SymbolProfileService } from '@ghostfolio/api/services/symbol-profile.service';
-import {
-  DATE_FORMAT,
-  extractNumberFromString,
-  getYesterday
-} from '@ghostfolio/common/helper';
-import { Granularity } from '@ghostfolio/common/types';
-import { Injectable, Logger } from '@nestjs/common';
-import { DataSource, SymbolProfile } from '@prisma/client';
-import bent from 'bent';
-import * as cheerio from 'cheerio';
-import { addDays, format, isBefore } from 'date-fns';
-
-@Injectable()
-export class GhostfolioScraperApiService implements DataProviderInterface {
-  public constructor(
-    private readonly prismaService: PrismaService,
-    private readonly symbolProfileService: SymbolProfileService
-  ) {}
-
-  public canHandle(symbol: string) {
-    return true;
-  }
-
-  public async getAssetProfile(
-    aSymbol: string
-  ): Promise<Partial<SymbolProfile>> {
-    return {
-      dataSource: this.getName()
-    };
-  }
-
-  public async getDividends({
-    from,
-    granularity = 'day',
-    symbol,
-    to
-  }: {
-    from: Date;
-    granularity: Granularity;
-    symbol: string;
-    to: Date;
-  }) {
-    return {};
-  }
-
-  public async getHistorical(
-    aSymbol: string,
-    aGranularity: Granularity = 'day',
-    from: Date,
-    to: Date
-  ): Promise<{
-    [symbol: string]: { [date: string]: IDataProviderHistoricalResponse };
-  }> {
-    try {
-      const symbol = aSymbol;
-
-      const [symbolProfile] =
-        await this.symbolProfileService.getSymbolProfilesBySymbols([symbol]);
-      const { defaultMarketPrice, selector, url } =
-        symbolProfile.scraperConfiguration ?? {};
-
-      if (defaultMarketPrice) {
-        const historical: {
-          [symbol: string]: { [date: string]: IDataProviderHistoricalResponse };
-        } = {
-          [symbol]: {}
-        };
-        let date = from;
-
-        while (isBefore(date, to)) {
-          historical[symbol][format(date, DATE_FORMAT)] = {
-            marketPrice: defaultMarketPrice
-          };
-
-          date = addDays(date, 1);
-        }
-
-        return historical;
-      } else if (selector === undefined || url === undefined) {
-        return {};
-      }
-
-      const get = bent(url, 'GET', 'string', 200, {});
-
-      const html = await get();
-      const $ = cheerio.load(html);
-
-      const value = extractNumberFromString($(selector).text());
-
-      return {
-        [symbol]: {
-          [format(getYesterday(), DATE_FORMAT)]: {
-            marketPrice: value
-          }
-        }
-      };
-    } catch (error) {
-      throw new Error(
-        `Could not get historical market data for ${aSymbol} (${this.getName()}) from ${format(
-          from,
-          DATE_FORMAT
-        )} to ${format(to, DATE_FORMAT)}: [${error.name}] ${error.message}`
-      );
-    }
-  }
-
-  public getName(): DataSource {
-    return DataSource.GHOSTFOLIO;
-  }
-
-  public async getQuotes(
-    aSymbols: string[]
-  ): Promise<{ [symbol: string]: IDataProviderResponse }> {
-    const response: { [symbol: string]: IDataProviderResponse } = {};
-
-    if (aSymbols.length <= 0) {
-      return response;
-    }
-
-    try {
-      const symbolProfiles =
-        await this.symbolProfileService.getSymbolProfilesBySymbols(aSymbols);
-
-      const marketData = await this.prismaService.marketData.findMany({
-        distinct: ['symbol'],
-        orderBy: {
-          date: 'desc'
-        },
-        take: aSymbols.length,
-        where: {
-          symbol: {
-            in: aSymbols
-          }
-        }
-      });
-
-      for (const symbolProfile of symbolProfiles) {
-        response[symbolProfile.symbol] = {
-          currency: symbolProfile.currency,
-          dataSource: this.getName(),
-          marketPrice: marketData.find((marketDataItem) => {
-            return marketDataItem.symbol === symbolProfile.symbol;
-          })?.marketPrice,
-          marketState: 'delayed'
-        };
-      }
-
-      return response;
-    } catch (error) {
-      Logger.error(error, 'GhostfolioScraperApiService');
-    }
-
-    return {};
-  }
-
-  public async search(aQuery: string): Promise<{ items: LookupItem[] }> {
-    const items = await this.prismaService.symbolProfile.findMany({
-      select: {
-        currency: true,
-        dataSource: true,
-        name: true,
-        symbol: true
-      },
-      where: {
-        OR: [
-          {
-            dataSource: this.getName(),
-            name: {
-              mode: 'insensitive',
-              startsWith: aQuery
-            }
-          },
-          {
-            dataSource: this.getName(),
-            symbol: {
-              mode: 'insensitive',
-              startsWith: aQuery
-            }
-          }
-        ]
-      }
-    });
-
-    return { items };
-  }
-}

prisma/migrations/20230215174320_changed_data_source_from_ghostfolio_to_manual/migration.sql

@@ -0,0 +1,2 @@
+UPDATE "MarketData" SET "dataSource" = 'MANUAL' WHERE "dataSource" = 'GHOSTFOLIO';
+UPDATE "SymbolProfile" SET "dataSource" = 'MANUAL' WHERE "dataSource" = 'GHOSTFOLIO';