Feature/implement scraper (#25)

* Clean up imports

* Implement scraper

* Sort imports

apps/api/src/app/admin/admin.module.ts

@@ -4,6 +4,7 @@ import { ConfigurationService } from '../../services/configuration.service';
 import { DataGatheringService } from '../../services/data-gathering.service';
 import { DataProviderService } from '../../services/data-provider.service';
 import { AlphaVantageService } from '../../services/data-provider/alpha-vantage/alpha-vantage.service';
+import { GhostfolioScraperApiService } from '../../services/data-provider/ghostfolio-scraper-api/ghostfolio-scraper-api.service';
 import { RakutenRapidApiService } from '../../services/data-provider/rakuten-rapid-api/rakuten-rapid-api.service';
 import { YahooFinanceService } from '../../services/data-provider/yahoo-finance/yahoo-finance.service';
 import { ExchangeRateDataService } from '../../services/exchange-rate-data.service';
@@ -21,6 +22,7 @@ import { AdminService } from './admin.service';
     DataGatheringService,
     DataProviderService,
     ExchangeRateDataService,
+    GhostfolioScraperApiService,
     PrismaService,
     RakutenRapidApiService,
     YahooFinanceService

apps/api/src/app/app.module.ts

@@ -10,6 +10,7 @@ import { CronService } from '../services/cron.service';
 import { DataGatheringService } from '../services/data-gathering.service';
 import { DataProviderService } from '../services/data-provider.service';
 import { AlphaVantageService } from '../services/data-provider/alpha-vantage/alpha-vantage.service';
+import { GhostfolioScraperApiService } from '../services/data-provider/ghostfolio-scraper-api/ghostfolio-scraper-api.service';
 import { RakutenRapidApiService } from '../services/data-provider/rakuten-rapid-api/rakuten-rapid-api.service';
 import { YahooFinanceService } from '../services/data-provider/yahoo-finance/yahoo-finance.service';
 import { ExchangeRateDataService } from '../services/exchange-rate-data.service';
@@ -65,6 +66,7 @@ import { UserModule } from './user/user.module';
     DataGatheringService,
     DataProviderService,
     ExchangeRateDataService,
+    GhostfolioScraperApiService,
     PrismaService,
     RakutenRapidApiService,
     YahooFinanceService

apps/api/src/app/experimental/experimental.module.ts

@@ -3,6 +3,7 @@ import { Module } from '@nestjs/common';
 import { ConfigurationService } from '../../services/configuration.service';
 import { DataProviderService } from '../../services/data-provider.service';
 import { AlphaVantageService } from '../../services/data-provider/alpha-vantage/alpha-vantage.service';
+import { GhostfolioScraperApiService } from '../../services/data-provider/ghostfolio-scraper-api/ghostfolio-scraper-api.service';
 import { RakutenRapidApiService } from '../../services/data-provider/rakuten-rapid-api/rakuten-rapid-api.service';
 import { YahooFinanceService } from '../../services/data-provider/yahoo-finance/yahoo-finance.service';
 import { ExchangeRateDataService } from '../../services/exchange-rate-data.service';
@@ -20,6 +21,7 @@ import { ExperimentalService } from './experimental.service';
     DataProviderService,
     ExchangeRateDataService,
     ExperimentalService,
+    GhostfolioScraperApiService,
     PrismaService,
     RakutenRapidApiService,
     RulesService,

apps/api/src/app/order/order.module.ts

@@ -4,6 +4,7 @@ import { ConfigurationService } from '../../services/configuration.service';
 import { DataGatheringService } from '../../services/data-gathering.service';
 import { DataProviderService } from '../../services/data-provider.service';
 import { AlphaVantageService } from '../../services/data-provider/alpha-vantage/alpha-vantage.service';
+import { GhostfolioScraperApiService } from '../../services/data-provider/ghostfolio-scraper-api/ghostfolio-scraper-api.service';
 import { RakutenRapidApiService } from '../../services/data-provider/rakuten-rapid-api/rakuten-rapid-api.service';
 import { YahooFinanceService } from '../../services/data-provider/yahoo-finance/yahoo-finance.service';
 import { ImpersonationService } from '../../services/impersonation.service';
@@ -22,6 +23,7 @@ import { OrderService } from './order.service';
     ConfigurationService,
     DataGatheringService,
     DataProviderService,
+    GhostfolioScraperApiService,
     ImpersonationService,
     OrderService,
     PrismaService,

apps/api/src/app/portfolio/portfolio.module.ts

@@ -4,6 +4,7 @@ import { ConfigurationService } from '../../services/configuration.service';
 import { DataGatheringService } from '../../services/data-gathering.service';
 import { DataProviderService } from '../../services/data-provider.service';
 import { AlphaVantageService } from '../../services/data-provider/alpha-vantage/alpha-vantage.service';
+import { GhostfolioScraperApiService } from '../../services/data-provider/ghostfolio-scraper-api/ghostfolio-scraper-api.service';
 import { RakutenRapidApiService } from '../../services/data-provider/rakuten-rapid-api/rakuten-rapid-api.service';
 import { YahooFinanceService } from '../../services/data-provider/yahoo-finance/yahoo-finance.service';
 import { ExchangeRateDataService } from '../../services/exchange-rate-data.service';
@@ -27,6 +28,7 @@ import { PortfolioService } from './portfolio.service';
     DataGatheringService,
     DataProviderService,
     ExchangeRateDataService,
+    GhostfolioScraperApiService,
     ImpersonationService,
     OrderService,
     PortfolioService,

apps/api/src/app/symbol/symbol.module.ts

@@ -3,6 +3,7 @@ import { Module } from '@nestjs/common';
 import { ConfigurationService } from '../../services/configuration.service';
 import { DataProviderService } from '../../services/data-provider.service';
 import { AlphaVantageService } from '../../services/data-provider/alpha-vantage/alpha-vantage.service';
+import { GhostfolioScraperApiService } from '../../services/data-provider/ghostfolio-scraper-api/ghostfolio-scraper-api.service';
 import { RakutenRapidApiService } from '../../services/data-provider/rakuten-rapid-api/rakuten-rapid-api.service';
 import { YahooFinanceService } from '../../services/data-provider/yahoo-finance/yahoo-finance.service';
 import { PrismaService } from '../../services/prisma.service';
@@ -16,6 +17,7 @@ import { SymbolService } from './symbol.service';
     AlphaVantageService,
     ConfigurationService,
     DataProviderService,
+    GhostfolioScraperApiService,
     PrismaService,
     RakutenRapidApiService,
     SymbolService,

apps/api/src/services/configuration.service.ts

@@ -12,6 +12,7 @@ export class ConfigurationService {
       ACCESS_TOKEN_SALT: str(),
       ALPHA_VANTAGE_API_KEY: str({ default: '' }),
       CACHE_TTL: num({ default: 1 }),
+      ENABLE_FEATURE_CUSTOM_SYMBOLS: bool({ default: false }),
       ENABLE_FEATURE_FEAR_AND_GREED_INDEX: bool({ default: false }),
       ENABLE_FEATURE_SOCIAL_LOGIN: bool({ default: false }),
       GOOGLE_CLIENT_ID: str({ default: 'dummyClientId' }),

apps/api/src/services/data-gathering.service.ts

@@ -200,10 +200,36 @@ export class DataGatheringService {
     return benchmarksToGather;
   }
 
+  private async getCustomSymbolsToGather(startDate: Date) {
+    const customSymbolsToGather = [];
+
+    if (this.configurationService.get('ENABLE_FEATURE_CUSTOM_SYMBOLS')) {
+      try {
+        const {
+          value: scraperConfigString
+        } = await this.prisma.property.findFirst({
+          select: {
+            value: true
+          },
+          where: { key: 'SCRAPER_CONFIG' }
+        });
+
+        JSON.parse(scraperConfigString).forEach((item) => {
+          customSymbolsToGather.push({
+            date: startDate,
+            symbol: item.symbol
+          });
+        });
+      } catch {}
+    }
+
+    return customSymbolsToGather;
+  }
+
   private async getSymbols7D(): Promise<{ date: Date; symbol: string }[]> {
     const startDate = subDays(resetHours(new Date()), 7);
 
-    let distinctOrders = await this.prisma.order.findMany({
+    const distinctOrders = await this.prisma.order.findMany({
       distinct: ['symbol'],
       orderBy: [{ symbol: 'asc' }],
       select: { symbol: true }
@@ -223,8 +249,13 @@ export class DataGatheringService {
       };
     });
 
+    const customSymbolsToGather = await this.getCustomSymbolsToGather(
+      startDate
+    );
+
     return [
       ...this.getBenchmarksToGather(startDate),
+      ...customSymbolsToGather,
       ...currencyPairsToGather,
       ...distinctOrdersWithDate
     ];
@@ -233,11 +264,9 @@ export class DataGatheringService {
   private async getSymbolsMax() {
     const startDate = new Date(getUtc('2000-01-01'));
 
-    let distinctOrders = await this.prisma.order.findMany({
-      distinct: ['symbol'],
-      orderBy: [{ date: 'asc' }],
-      select: { date: true, symbol: true }
-    });
+    const customSymbolsToGather = await this.getCustomSymbolsToGather(
+      startDate
+    );
 
     const currencyPairsToGather = currencyPairs.map((symbol) => {
       return {
@@ -246,8 +275,15 @@ export class DataGatheringService {
       };
     });
 
+    const distinctOrders = await this.prisma.order.findMany({
+      distinct: ['symbol'],
+      orderBy: [{ date: 'asc' }],
+      select: { date: true, symbol: true }
+    });
+
     return [
       ...this.getBenchmarksToGather(startDate),
+      ...customSymbolsToGather,
       ...currencyPairsToGather,
       ...distinctOrders
     ];

apps/api/src/services/data-provider.service.ts

@@ -1,10 +1,15 @@
-import { isCrypto, isRakutenRapidApi } from '@ghostfolio/helper';
+import {
+  isCrypto,
+  isGhostfolioScraperApi,
+  isRakutenRapidApi
+} from '@ghostfolio/helper';
 import { Injectable } from '@nestjs/common';
 import { MarketData } from '@prisma/client';
 import { format } from 'date-fns';
 
 import { ConfigurationService } from './configuration.service';
 import { AlphaVantageService } from './data-provider/alpha-vantage/alpha-vantage.service';
+import { GhostfolioScraperApiService } from './data-provider/ghostfolio-scraper-api/ghostfolio-scraper-api.service';
 import { RakutenRapidApiService } from './data-provider/rakuten-rapid-api/rakuten-rapid-api.service';
 import { YahooFinanceService } from './data-provider/yahoo-finance/yahoo-finance.service';
 import { DataProviderInterface } from './interfaces/data-provider.interface';
@@ -20,6 +25,7 @@ export class DataProviderService implements DataProviderInterface {
   public constructor(
     private readonly alphaVantageService: AlphaVantageService,
     private readonly configurationService: ConfigurationService,
+    private readonly ghostfolioScraperApiService: GhostfolioScraperApiService,
     private prisma: PrismaService,
     private readonly rakutenRapidApiService: RakutenRapidApiService,
     private readonly yahooFinanceService: YahooFinanceService
@@ -33,7 +39,9 @@ export class DataProviderService implements DataProviderInterface {
     if (aSymbols.length === 1) {
       const symbol = aSymbols[0];
 
-      if (isRakutenRapidApi(symbol)) {
+      if (isGhostfolioScraperApi(symbol)) {
+        return this.ghostfolioScraperApiService.get(aSymbols);
+      } else if (isRakutenRapidApi(symbol)) {
         return this.rakutenRapidApiService.get(aSymbols);
       }
     }
@@ -53,12 +61,12 @@ export class DataProviderService implements DataProviderInterface {
       [symbol: string]: { [date: string]: IDataProviderHistoricalResponse };
     } = {};
 
-    let granularityQuery =
+    const granularityQuery =
       aGranularity === 'month'
         ? `AND (date_part('day', date) = 1 OR date >= TIMESTAMP 'yesterday')`
         : '';
 
-    let rangeQuery =
+    const rangeQuery =
       from && to
         ? `AND date >= '${format(from, 'yyyy-MM-dd')}' AND date <= '${format(
             to,
@@ -127,6 +135,15 @@ export class DataProviderService implements DataProviderInterface {
             ...dataOfAlphaVantage[symbol]
           }
         };
+      } else if (isGhostfolioScraperApi(symbol)) {
+        const dataOfGhostfolioScraperApi = await this.ghostfolioScraperApiService.getHistorical(
+          [symbol],
+          undefined,
+          from,
+          to
+        );
+
+        return dataOfGhostfolioScraperApi;
       } else if (
         isRakutenRapidApi(symbol) &&
         this.configurationService.get('RAKUTEN_RAPID_API_KEY')

apps/api/src/services/data-provider/ghostfolio-scraper-api/ghostfolio-scraper-api.service.ts

@@ -0,0 +1,103 @@
+import { getYesterday } from '@ghostfolio/helper';
+import { Injectable } from '@nestjs/common';
+import * as bent from 'bent';
+import * as cheerio from 'cheerio';
+import { format } from 'date-fns';
+
+import { DataProviderInterface } from '../../interfaces/data-provider.interface';
+import { Granularity } from '../../interfaces/granularity.type';
+import {
+  IDataProviderHistoricalResponse,
+  IDataProviderResponse
+} from '../../interfaces/interfaces';
+import { PrismaService } from '../../prisma.service';
+import { Currency } from '.prisma/client';
+
+@Injectable()
+export class GhostfolioScraperApiService implements DataProviderInterface {
+  public constructor(private prisma: PrismaService) {}
+
+  public async get(
+    aSymbols: string[]
+  ): Promise<{ [symbol: string]: IDataProviderResponse }> {
+    if (aSymbols.length <= 0) {
+      return {};
+    }
+
+    try {
+      const symbol = aSymbols[0];
+      const { marketPrice } = await this.prisma.marketData.findFirst({
+        orderBy: {
+          date: 'desc'
+        },
+        where: {
+          symbol
+        }
+      });
+
+      return {
+        [symbol]: {
+          marketPrice,
+          currency: Currency.CHF,
+          isMarketOpen: true,
+          name: symbol
+        }
+      };
+    } catch (error) {
+      console.error(error);
+    }
+
+    return {};
+  }
+
+  public async getHistorical(
+    aSymbols: string[],
+    aGranularity: Granularity = 'day',
+    from: Date,
+    to: Date
+  ): Promise<{
+    [symbol: string]: { [date: string]: IDataProviderHistoricalResponse };
+  }> {
+    if (aSymbols.length <= 0) {
+      return {};
+    }
+
+    try {
+      const symbol = aSymbols[0];
+
+      const {
+        value: scraperConfigString
+      } = await this.prisma.property.findFirst({
+        select: {
+          value: true
+        },
+        where: { key: 'SCRAPER_CONFIG' }
+      });
+
+      const scraperConfig = JSON.parse(scraperConfigString).find((item) => {
+        return item.symbol === symbol;
+      });
+
+      const get = bent(scraperConfig.url, 'GET', 'string', 200, {});
+
+      const html = await get();
+      const $ = cheerio.load(html);
+
+      const string = $(scraperConfig.selector).text().replace('CHF', '').trim();
+
+      const value = parseFloat(string);
+
+      return {
+        [symbol]: {
+          [format(getYesterday(), 'yyyy-MM-dd')]: {
+            marketPrice: value
+          }
+        }
+      };
+    } catch (error) {
+      console.error(error);
+    }
+
+    return {};
+  }
+}

apps/api/src/services/interfaces/environment.interface.ts

@@ -4,6 +4,7 @@ export interface Environment extends CleanedEnvAccessors {
   ACCESS_TOKEN_SALT: string;
   ALPHA_VANTAGE_API_KEY: string;
   CACHE_TTL: number;
+  ENABLE_FEATURE_CUSTOM_SYMBOLS: boolean;
   ENABLE_FEATURE_FEAR_AND_GREED_INDEX: boolean;
   ENABLE_FEATURE_SOCIAL_LOGIN: boolean;
   GOOGLE_CLIENT_ID: string;