Feature/move scraper configuration to symbol profile (#456)

* Move scraper configuration

* Update changelog

CHANGELOG.md

@@ -10,6 +10,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ### Changed
 
 - Improved the validation of `json` files in the import functionality for transactions
+- Moved the scraper configuration to the symbol profile model
+
+### Todo
+
+- Apply data migration (`yarn database:migrate`)
 
 ## 1.69.0 - 07.11.2021
 

apps/api/src/services/data-gathering.service.ts

@@ -272,21 +272,6 @@ export class DataGatheringService {
     }
   }
 
-  public async getCustomSymbolsToGather(
-    startDate?: Date
-  ): Promise<IDataGatheringItem[]> {
-    const scraperConfigurations =
-      await this.ghostfolioScraperApi.getScraperConfigurations();
-
-    return scraperConfigurations.map((scraperConfiguration) => {
-      return {
-        dataSource: DataSource.GHOSTFOLIO,
-        date: startDate,
-        symbol: scraperConfiguration.symbol
-      };
-    });
-  }
-
   public async getIsInProgress() {
     return await this.prismaService.property.findUnique({
       where: { key: 'LOCKED_DATA_GATHERING' }
@@ -343,6 +328,7 @@ export class DataGatheringService {
         orderBy: [{ symbol: 'asc' }],
         select: {
           dataSource: true,
+          scraperConfiguration: true,
           symbol: true
         }
       })
@@ -363,12 +349,8 @@ export class DataGatheringService {
         };
       });
 
-    const customSymbolsToGather =
-      await this.ghostfolioScraperApi.getCustomSymbolsToGather(startDate);
-
     return [
       ...this.getBenchmarksToGather(startDate),
-      ...customSymbolsToGather,
       ...currencyPairsToGather,
       ...symbolProfilesToGather
     ];
@@ -382,9 +364,6 @@ export class DataGatheringService {
         })
       )?.date ?? new Date();
 
-    const customSymbolsToGather =
-      await this.ghostfolioScraperApi.getCustomSymbolsToGather(startDate);
-
     const currencyPairsToGather = this.exchangeRateDataService
       .getCurrencyPairs()
       .map(({ dataSource, symbol }) => {
@@ -405,20 +384,19 @@ export class DataGatheringService {
             select: { date: true },
             take: 1
           },
+          scraperConfiguration: true,
           symbol: true
         }
       })
-    ).map((item) => {
+    ).map((symbolProfile) => {
       return {
-        dataSource: item.dataSource,
+        ...symbolProfile,
-        date: item.Order?.[0]?.date ?? startDate,
+        date: symbolProfile.Order?.[0]?.date ?? startDate
-        symbol: item.symbol
       };
     });
 
     return [
       ...this.getBenchmarksToGather(startDate),
-      ...customSymbolsToGather,
       ...currencyPairsToGather,
       ...symbolProfilesToGather
     ];

apps/api/src/services/data-provider/data-provider.module.ts

@@ -4,13 +4,19 @@ import { GhostfolioScraperApiService } from '@ghostfolio/api/services/data-provi
 import { RakutenRapidApiService } from '@ghostfolio/api/services/data-provider/rakuten-rapid-api/rakuten-rapid-api.service';
 import { YahooFinanceService } from '@ghostfolio/api/services/data-provider/yahoo-finance/yahoo-finance.service';
 import { PrismaModule } from '@ghostfolio/api/services/prisma.module';
+import { SymbolProfileModule } from '@ghostfolio/api/services/symbol-profile.module';
 import { Module } from '@nestjs/common';
 
 import { AlphaVantageService } from './alpha-vantage/alpha-vantage.service';
 import { DataProviderService } from './data-provider.service';
 
 @Module({
-  imports: [ConfigurationModule, CryptocurrencyModule, PrismaModule],
+  imports: [
+    ConfigurationModule,
+    CryptocurrencyModule,
+    PrismaModule,
+    SymbolProfileModule
+  ],
   providers: [
     AlphaVantageService,
     DataProviderService,

apps/api/src/services/data-provider/ghostfolio-scraper-api/ghostfolio-scraper-api.service.ts

@@ -1,5 +1,6 @@
 import { LookupItem } from '@ghostfolio/api/app/symbol/interfaces/lookup-item.interface';
 import { PrismaService } from '@ghostfolio/api/services/prisma.service';
+import { SymbolProfileService } from '@ghostfolio/api/services/symbol-profile.service';
 import {
   DATE_FORMAT,
   getYesterday,
@@ -13,19 +14,20 @@ import * as cheerio from 'cheerio';
 import { format } from 'date-fns';
 
 import {
-  IDataGatheringItem,
   IDataProviderHistoricalResponse,
   IDataProviderResponse,
   MarketState
 } from '../../interfaces/interfaces';
 import { DataProviderInterface } from '../interfaces/data-provider.interface';
-import { ScraperConfig } from './interfaces/scraper-config.interface';
 
 @Injectable()
 export class GhostfolioScraperApiService implements DataProviderInterface {
   private static NUMERIC_REGEXP = /[-]{0,1}[\d]*[.,]{0,1}[\d]+/g;
 
-  public constructor(private readonly prismaService: PrismaService) {}
+  public constructor(
+    private readonly prismaService: PrismaService,
+    private readonly symbolProfileService: SymbolProfileService
+  ) {}
 
   public canHandle(symbol: string) {
     return isGhostfolioScraperApiSymbol(symbol);
@@ -39,9 +41,10 @@ export class GhostfolioScraperApiService implements DataProviderInterface {
     }
 
     try {
-      const symbol = aSymbols[0];
+      const [symbol] = aSymbols;
-
+      const [symbolProfile] = await this.symbolProfileService.getSymbolProfiles(
-      const scraperConfig = await this.getScraperConfigurationBySymbol(symbol);
+        [symbol]
+      );
 
       const { marketPrice } = await this.prismaService.marketData.findFirst({
         orderBy: {
@@ -55,7 +58,7 @@ export class GhostfolioScraperApiService implements DataProviderInterface {
       return {
         [symbol]: {
           marketPrice,
-          currency: scraperConfig?.currency,
+          currency: symbolProfile?.currency,
           dataSource: DataSource.GHOSTFOLIO,
           marketState: MarketState.delayed
         }
@@ -67,25 +70,6 @@ export class GhostfolioScraperApiService implements DataProviderInterface {
     return {};
   }
 
-  public async getCustomSymbolsToGather(
-    startDate?: Date
-  ): Promise<IDataGatheringItem[]> {
-    const ghostfolioSymbolProfiles =
-      await this.prismaService.symbolProfile.findMany({
-        where: {
-          dataSource: DataSource.GHOSTFOLIO
-        }
-      });
-
-    return ghostfolioSymbolProfiles.map(({ dataSource, symbol }) => {
-      return {
-        dataSource,
-        symbol,
-        date: startDate
-      };
-    });
-  }
-
   public async getHistorical(
     aSymbols: string[],
     aGranularity: Granularity = 'day',
@@ -99,11 +83,11 @@ export class GhostfolioScraperApiService implements DataProviderInterface {
     }
 
     try {
-      const symbol = aSymbols[0];
+      const [symbol] = aSymbols;
-
+      const [symbolProfile] = await this.symbolProfileService.getSymbolProfiles(
-      const scraperConfiguration = await this.getScraperConfigurationBySymbol(
+        [symbol]
-        symbol
       );
+      const scraperConfiguration = symbolProfile?.scraperConfiguration;
 
       const get = bent(scraperConfiguration?.url, 'GET', 'string', 200, {});
 
@@ -128,22 +112,6 @@ export class GhostfolioScraperApiService implements DataProviderInterface {
     return {};
   }
 
-  public async getScraperConfigurations(): Promise<ScraperConfig[]> {
-    try {
-      const { value: scraperConfigString } =
-        await this.prismaService.property.findFirst({
-          select: {
-            value: true
-          },
-          where: { key: 'SCRAPER_CONFIG' }
-        });
-
-      return JSON.parse(scraperConfigString);
-    } catch {}
-
-    return [];
-  }
-
   public getName(): DataSource {
     return DataSource.GHOSTFOLIO;
   }
@@ -162,11 +130,4 @@ export class GhostfolioScraperApiService implements DataProviderInterface {
       return undefined;
     }
   }
-
-  private async getScraperConfigurationBySymbol(aSymbol: string) {
-    const scraperConfigurations = await this.getScraperConfigurations();
-    return scraperConfigurations.find((scraperConfiguration) => {
-      return scraperConfiguration.symbol === aSymbol;
-    });
-  }
 }

apps/api/src/services/data-provider/ghostfolio-scraper-api/interfaces/scraper-config.interface.ts

@@ -1,6 +0,0 @@
-export interface ScraperConfig {
-  currency: string;
-  selector: string;
-  symbol: string;
-  url: string;
-}

apps/api/src/services/data-provider/ghostfolio-scraper-api/interfaces/scraper-configuration.interface.ts

@@ -0,0 +1,4 @@
+export interface ScraperConfiguration {
+  selector: string;
+  url: string;
+}

apps/api/src/services/interfaces/symbol-profile.interface.ts

@@ -1,3 +1,4 @@
+import { ScraperConfiguration } from '@ghostfolio/api/services/data-provider/ghostfolio-scraper-api/interfaces/scraper-configuration.interface';
 import { Country } from '@ghostfolio/common/interfaces/country.interface';
 import { Sector } from '@ghostfolio/common/interfaces/sector.interface';
 import { AssetClass, AssetSubClass, DataSource } from '@prisma/client';
@@ -11,6 +12,7 @@ export interface EnhancedSymbolProfile {
   dataSource: DataSource;
   id: string;
   name: string | null;
+  scraperConfiguration?: ScraperConfiguration;
   sectors: Sector[];
   symbol: string;
   symbolMapping?: { [key: string]: string };

apps/api/src/services/symbol-profile.service.ts

@@ -7,6 +7,8 @@ import { Injectable } from '@nestjs/common';
 import { Prisma, SymbolProfile } from '@prisma/client';
 import { continents, countries } from 'countries-list';
 
+import { ScraperConfiguration } from './data-provider/ghostfolio-scraper-api/interfaces/scraper-configuration.interface';
+
 @Injectable()
 export class SymbolProfileService {
   constructor(private readonly prismaService: PrismaService) {}
@@ -29,6 +31,7 @@ export class SymbolProfileService {
     return symbolProfiles.map((symbolProfile) => ({
       ...symbolProfile,
       countries: this.getCountries(symbolProfile),
+      scraperConfiguration: this.getScraperConfiguration(symbolProfile),
       sectors: this.getSectors(symbolProfile),
       symbolMapping: this.getSymbolMapping(symbolProfile)
     }));
@@ -50,6 +53,18 @@ export class SymbolProfileService {
     );
   }
 
+  private getScraperConfiguration(
+    symbolProfile: SymbolProfile
+  ): ScraperConfiguration {
+    const scraperConfiguration =
+      symbolProfile.scraperConfiguration as Prisma.JsonObject;
+
+    return {
+      selector: scraperConfiguration.selector as string,
+      url: scraperConfiguration.url as string
+    };
+  }
+
   private getSectors(symbolProfile: SymbolProfile): Sector[] {
     return ((symbolProfile?.sectors as Prisma.JsonArray) ?? []).map(
       (sector) => {

prisma/migrations/20211107171624_added_scraper_configuration_to_symbol_profile/migration.sql

@@ -0,0 +1,2 @@
+-- AlterTable
+ALTER TABLE "SymbolProfile" ADD COLUMN     "scraperConfiguration" JSONB;

prisma/schema.prisma

@@ -118,19 +118,20 @@ model Settings {
 }
 
 model SymbolProfile {
-  assetClass    AssetClass?
+  assetClass           AssetClass?
-  assetSubClass AssetSubClass?
+  assetSubClass        AssetSubClass?
-  countries     Json?
+  countries            Json?
-  createdAt     DateTime       @default(now())
+  createdAt            DateTime       @default(now())
-  currency      String?
+  currency             String?
-  dataSource    DataSource
+  dataSource           DataSource
-  id            String         @id @default(uuid())
+  id                   String         @id @default(uuid())
-  name          String?
+  name                 String?
-  Order         Order[]
+  Order                Order[]
-  updatedAt     DateTime       @updatedAt
+  updatedAt            DateTime       @updatedAt
-  sectors       Json?
+  scraperConfiguration Json?
-  symbol        String
+  sectors              Json?
-  symbolMapping Json?
+  symbol               String
+  symbolMapping        Json?
 
   @@unique([dataSource, symbol])
 }