Improve scraper (#28)

CHANGELOG.md

@@ -5,6 +5,16 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## Unreleased
+
+### Changed
+
+- Reverted the restoring of the scroll position when opening a new page
+
+### Fixed
+
+- Fixed some issues in the generic scraper
+
 ## 0.87.0 - 19.04.2021
 
 ### Added

apps/api/src/app/portfolio/portfolio.service.ts

@@ -48,53 +48,6 @@ export class PortfolioService {
     private readonly userService: UserService
   ) {}
 
-  private convertDateRangeToDate(aDateRange: DateRange, aMinDate: Date) {
-    let currentDate = new Date();
-
-    const normalizedMinDate =
-      getDate(aMinDate) === 1
-        ? aMinDate
-        : add(setDate(aMinDate, 1), { months: 1 });
-
-    const year = getYear(currentDate);
-    const month = getMonth(currentDate);
-    const day = getDate(currentDate);
-
-    currentDate = new Date(Date.UTC(year, month, day, 0));
-
-    switch (aDateRange) {
-      case '1d':
-        return sub(currentDate, {
-          days: 1
-        });
-      case 'ytd':
-        currentDate = setDate(currentDate, 1);
-        currentDate = setMonth(currentDate, 0);
-        return isAfter(currentDate, normalizedMinDate)
-          ? currentDate
-          : undefined;
-      case '1y':
-        currentDate = setDate(currentDate, 1);
-        currentDate = sub(currentDate, {
-          years: 1
-        });
-        return isAfter(currentDate, normalizedMinDate)
-          ? currentDate
-          : undefined;
-      case '5y':
-        currentDate = setDate(currentDate, 1);
-        currentDate = sub(currentDate, {
-          years: 5
-        });
-        return isAfter(currentDate, normalizedMinDate)
-          ? currentDate
-          : undefined;
-      default:
-        // Gets handled as all data
-        return undefined;
-    }
-  }
-
   public async createPortfolio(aUserId: string): Promise<Portfolio> {
     let portfolio: Portfolio;
     let stringifiedPortfolio = await this.redisCacheService.get(
@@ -382,4 +335,51 @@ export class PortfolioService {
       symbol: aSymbol
     };
   }
+
+  private convertDateRangeToDate(aDateRange: DateRange, aMinDate: Date) {
+    let currentDate = new Date();
+
+    const normalizedMinDate =
+      getDate(aMinDate) === 1
+        ? aMinDate
+        : add(setDate(aMinDate, 1), { months: 1 });
+
+    const year = getYear(currentDate);
+    const month = getMonth(currentDate);
+    const day = getDate(currentDate);
+
+    currentDate = new Date(Date.UTC(year, month, day, 0));
+
+    switch (aDateRange) {
+      case '1d':
+        return sub(currentDate, {
+          days: 1
+        });
+      case 'ytd':
+        currentDate = setDate(currentDate, 1);
+        currentDate = setMonth(currentDate, 0);
+        return isAfter(currentDate, normalizedMinDate)
+          ? currentDate
+          : undefined;
+      case '1y':
+        currentDate = setDate(currentDate, 1);
+        currentDate = sub(currentDate, {
+          years: 1
+        });
+        return isAfter(currentDate, normalizedMinDate)
+          ? currentDate
+          : undefined;
+      case '5y':
+        currentDate = setDate(currentDate, 1);
+        currentDate = sub(currentDate, {
+          years: 5
+        });
+        return isAfter(currentDate, normalizedMinDate)
+          ? currentDate
+          : undefined;
+      default:
+        // Gets handled as all data
+        return undefined;
+    }
+  }
 }

apps/api/src/app/user/user.service.ts

@@ -44,10 +44,6 @@ export class UserService {
       currentPermissions.push(permissions.accessFearAndGreedIndex);
     }
 
-    if (this.configurationService.get('ENABLE_FEATURE_SOCIAL_LOGIN')) {
-      currentPermissions.push(permissions.useSocialLogin);
-    }
-
     return {
       alias,
       id,
@@ -162,18 +158,6 @@ export class UserService {
     });
   }
 
-  private getRandomString(length: number) {
-    const characters = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789';
-    const result = [];
-
-    for (let i = 0; i < length; i++) {
-      result.push(
-        characters.charAt(Math.floor(Math.random() * characters.length))
-      );
-    }
-    return result.join('');
-  }
-
   public async updateUserSettings({
     currency,
     userId
@@ -200,4 +184,16 @@ export class UserService {
 
     return;
   }
+
+  private getRandomString(length: number) {
+    const characters = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789';
+    const result = [];
+
+    for (let i = 0; i < length; i++) {
+      result.push(
+        characters.charAt(Math.floor(Math.random() * characters.length))
+      );
+    }
+    return result.join('');
+  }
 }

apps/api/src/models/portfolio.ts

@@ -79,7 +79,9 @@ export class Portfolio implements PortfolioInterface {
         investmentInOriginalCurrency:
           portfolioItemsYesterday?.positions[symbol]
             ?.investmentInOriginalCurrency,
-        marketPrice: currentData[symbol]?.marketPrice,
+        marketPrice:
+          currentData[symbol]?.marketPrice ??
+          portfolioItemsYesterday.positions[symbol]?.marketPrice,
         quantity: portfolioItemsYesterday?.positions[symbol]?.quantity
       };
     });
@@ -158,53 +160,6 @@ export class Portfolio implements PortfolioInterface {
     return this;
   }
 
-  private convertDateRangeToDate(aDateRange: DateRange, aMinDate: Date) {
-    let currentDate = new Date();
-
-    const normalizedMinDate =
-      getDate(aMinDate) === 1
-        ? aMinDate
-        : add(setDate(aMinDate, 1), { months: 1 });
-
-    const year = getYear(currentDate);
-    const month = getMonth(currentDate);
-    const day = getDate(currentDate);
-
-    currentDate = new Date(Date.UTC(year, month, day, 0));
-
-    switch (aDateRange) {
-      case '1d':
-        return sub(currentDate, {
-          days: 1
-        });
-      case 'ytd':
-        currentDate = setDate(currentDate, 1);
-        currentDate = setMonth(currentDate, 0);
-        return isAfter(currentDate, normalizedMinDate)
-          ? currentDate
-          : undefined;
-      case '1y':
-        currentDate = setDate(currentDate, 1);
-        currentDate = sub(currentDate, {
-          years: 1
-        });
-        return isAfter(currentDate, normalizedMinDate)
-          ? currentDate
-          : undefined;
-      case '5y':
-        currentDate = setDate(currentDate, 1);
-        currentDate = sub(currentDate, {
-          years: 5
-        });
-        return isAfter(currentDate, normalizedMinDate)
-          ? currentDate
-          : undefined;
-      default:
-        // Gets handled as all data
-        return undefined;
-    }
-  }
-
   public get(aDate?: Date): PortfolioItem[] {
     if (aDate) {
       const filteredPortfolio = this.portfolioItems.find((item) => {
@@ -528,12 +483,6 @@ export class Portfolio implements PortfolioInterface {
     return this.orders;
   }
 
-  private getOrdersByType(aFilter: string[]) {
-    return this.orders.filter((order) => {
-      return aFilter.includes(order.getType());
-    });
-  }
-
   public getValue(aDate = getToday()) {
     const positions = this.getPositions(aDate);
     let value = 0;
@@ -692,6 +641,53 @@ export class Portfolio implements PortfolioInterface {
     this.updatePortfolioItems();
   }
 
+  private convertDateRangeToDate(aDateRange: DateRange, aMinDate: Date) {
+    let currentDate = new Date();
+
+    const normalizedMinDate =
+      getDate(aMinDate) === 1
+        ? aMinDate
+        : add(setDate(aMinDate, 1), { months: 1 });
+
+    const year = getYear(currentDate);
+    const month = getMonth(currentDate);
+    const day = getDate(currentDate);
+
+    currentDate = new Date(Date.UTC(year, month, day, 0));
+
+    switch (aDateRange) {
+      case '1d':
+        return sub(currentDate, {
+          days: 1
+        });
+      case 'ytd':
+        currentDate = setDate(currentDate, 1);
+        currentDate = setMonth(currentDate, 0);
+        return isAfter(currentDate, normalizedMinDate)
+          ? currentDate
+          : undefined;
+      case '1y':
+        currentDate = setDate(currentDate, 1);
+        currentDate = sub(currentDate, {
+          years: 1
+        });
+        return isAfter(currentDate, normalizedMinDate)
+          ? currentDate
+          : undefined;
+      case '5y':
+        currentDate = setDate(currentDate, 1);
+        currentDate = sub(currentDate, {
+          years: 5
+        });
+        return isAfter(currentDate, normalizedMinDate)
+          ? currentDate
+          : undefined;
+      default:
+        // Gets handled as all data
+        return undefined;
+    }
+  }
+
   private updatePortfolioItems() {
     // console.time('update-portfolio-items');
 

apps/api/src/services/data-gathering.service.ts

@@ -2,6 +2,7 @@ import {
   benchmarks,
   currencyPairs,
   getUtc,
+  isGhostfolioScraperApiSymbol,
   resetHours
 } from '@ghostfolio/helper';
 import { Injectable } from '@nestjs/common';
@@ -235,12 +236,16 @@ export class DataGatheringService {
       select: { symbol: true }
     });
 
-    const distinctOrdersWithDate = distinctOrders.map((distinctOrder) => {
+    const distinctOrdersWithDate = distinctOrders
-      return {
+      .filter((distinctOrder) => {
-        ...distinctOrder,
+        return !isGhostfolioScraperApiSymbol(distinctOrder.symbol);
-        date: startDate
+      })
-      };
+      .map((distinctOrder) => {
-    });
+        return {
+          ...distinctOrder,
+          date: startDate
+        };
+      });
 
     const currencyPairsToGather = currencyPairs.map((symbol) => {
       return {

apps/api/src/services/data-provider.service.ts

@@ -1,7 +1,7 @@
 import {
   isCrypto,
-  isGhostfolioScraperApi,
+  isGhostfolioScraperApiSymbol,
-  isRakutenRapidApi
+  isRakutenRapidApiSymbol
 } from '@ghostfolio/helper';
 import { Injectable } from '@nestjs/common';
 import { MarketData } from '@prisma/client';
@@ -39,14 +39,33 @@ export class DataProviderService implements DataProviderInterface {
     if (aSymbols.length === 1) {
       const symbol = aSymbols[0];
 
-      if (isGhostfolioScraperApi(symbol)) {
+      if (isGhostfolioScraperApiSymbol(symbol)) {
         return this.ghostfolioScraperApiService.get(aSymbols);
-      } else if (isRakutenRapidApi(symbol)) {
+      } else if (isRakutenRapidApiSymbol(symbol)) {
         return this.rakutenRapidApiService.get(aSymbols);
       }
     }
 
-    return this.yahooFinanceService.get(aSymbols);
+    const yahooFinanceSymbols = aSymbols.filter((symbol) => {
+      return !isGhostfolioScraperApiSymbol(symbol);
+    });
+
+    const response = await this.yahooFinanceService.get(yahooFinanceSymbols);
+
+    const ghostfolioScraperApiSymbols = aSymbols.filter((symbol) => {
+      return isGhostfolioScraperApiSymbol(symbol);
+    });
+
+    for (const symbol of ghostfolioScraperApiSymbols) {
+      if (symbol) {
+        const ghostfolioScraperApiResult = await this.ghostfolioScraperApiService.get(
+          [symbol]
+        );
+        response[symbol] = ghostfolioScraperApiResult[symbol];
+      }
+    }
+
+    return response;
   }
 
   public async getHistorical(
@@ -107,8 +126,12 @@ export class DataProviderService implements DataProviderInterface {
   ): Promise<{
     [symbol: string]: { [date: string]: IDataProviderHistoricalResponse };
   }> {
+    const filteredSymbols = aSymbols.filter((symbol) => {
+      return !isGhostfolioScraperApiSymbol(symbol);
+    });
+
     const dataOfYahoo = await this.yahooFinanceService.getHistorical(
-      aSymbols,
+      filteredSymbols,
       undefined,
       from,
       to
@@ -135,7 +158,7 @@ export class DataProviderService implements DataProviderInterface {
             ...dataOfAlphaVantage[symbol]
           }
         };
-      } else if (isGhostfolioScraperApi(symbol)) {
+      } else if (isGhostfolioScraperApiSymbol(symbol)) {
         const dataOfGhostfolioScraperApi = await this.ghostfolioScraperApiService.getHistorical(
           [symbol],
           undefined,
@@ -145,7 +168,7 @@ export class DataProviderService implements DataProviderInterface {
 
         return dataOfGhostfolioScraperApi;
       } else if (
-        isRakutenRapidApi(symbol) &&
+        isRakutenRapidApiSymbol(symbol) &&
         this.configurationService.get('RAKUTEN_RAPID_API_KEY')
       ) {
         const dataOfRakutenRapidApi = await this.rakutenRapidApiService.getHistorical(

apps/api/src/services/data-provider/ghostfolio-scraper-api/ghostfolio-scraper-api.service.ts

@@ -11,7 +11,6 @@ import {
   IDataProviderResponse
 } from '../../interfaces/interfaces';
 import { PrismaService } from '../../prisma.service';
-import { Currency } from '.prisma/client';
 
 @Injectable()
 export class GhostfolioScraperApiService implements DataProviderInterface {
@@ -26,6 +25,9 @@ export class GhostfolioScraperApiService implements DataProviderInterface {
 
     try {
       const symbol = aSymbols[0];
+
+      const scraperConfig = await this.getScraperConfig(symbol);
+
       const { marketPrice } = await this.prisma.marketData.findFirst({
         orderBy: {
           date: 'desc'
@@ -38,9 +40,9 @@ export class GhostfolioScraperApiService implements DataProviderInterface {
       return {
         [symbol]: {
           marketPrice,
-          currency: Currency.CHF,
+          currency: scraperConfig?.currency,
-          isMarketOpen: true,
+          isMarketOpen: false,
-          name: symbol
+          name: scraperConfig?.name
         }
       };
     } catch (error) {
@@ -65,25 +67,17 @@ export class GhostfolioScraperApiService implements DataProviderInterface {
     try {
       const symbol = aSymbols[0];
 
-      const {
+      const scraperConfig = await this.getScraperConfig(symbol);
-        value: scraperConfigString
-      } = await this.prisma.property.findFirst({
-        select: {
-          value: true
-        },
-        where: { key: 'SCRAPER_CONFIG' }
-      });
 
-      const scraperConfig = JSON.parse(scraperConfigString).find((item) => {
+      const get = bent(scraperConfig?.url, 'GET', 'string', 200, {});
-        return item.symbol === symbol;
-      });
-
-      const get = bent(scraperConfig.url, 'GET', 'string', 200, {});
 
       const html = await get();
       const $ = cheerio.load(html);
 
-      const string = $(scraperConfig.selector).text().replace('CHF', '').trim();
+      const string = $(scraperConfig?.selector)
+        .text()
+        .replace('CHF', '')
+        .trim();
 
       const value = parseFloat(string);
 
@@ -100,4 +94,23 @@ export class GhostfolioScraperApiService implements DataProviderInterface {
 
     return {};
   }
+
+  private async getScraperConfig(aSymbol: string) {
+    try {
+      const {
+        value: scraperConfigString
+      } = await this.prisma.property.findFirst({
+        select: {
+          value: true
+        },
+        where: { key: 'SCRAPER_CONFIG' }
+      });
+
+      return JSON.parse(scraperConfigString).find((item) => {
+        return item.symbol === aSymbol;
+      });
+    } catch {}
+
+    return {};
+  }
 }

apps/client/src/app/app-routing.module.ts

@@ -81,8 +81,7 @@ const routes: Routes = [
       {
         preloadingStrategy: ModulePreloadService,
         // enableTracing: true // <-- debugging purposes only
-        relativeLinkResolution: 'legacy',
+        relativeLinkResolution: 'legacy'
-        scrollPositionRestoration: 'enabled'
       }
     )
   ],

apps/client/src/app/components/position/position.component.html

@@ -40,7 +40,9 @@
             class="ml-1"
             [url]="position?.url"
           ></gf-symbol-icon>
-          <span class="ml-2 text-muted">({{ position?.exchange }})</span>
+          <span *ngIf="position?.exchange" class="ml-2 text-muted"
+            >({{ position.exchange }})</span
+          >
         </div>
         <div class="d-flex mt-1">
           <gf-value

apps/client/src/app/pages/home/home-page.component.ts

@@ -85,6 +85,18 @@ export class HomePageComponent implements OnDestroy, OnInit {
             user.permissions,
             permissions.accessFearAndGreedIndex
           );
+
+          if (this.hasPermissionToAccessFearAndGreedIndex) {
+            this.dataService
+              .fetchSymbolItem('GF.FEAR_AND_GREED_INDEX')
+              .pipe(takeUntil(this.unsubscribeSubject))
+              .subscribe(({ marketPrice }) => {
+                this.fearAndGreedIndex = marketPrice;
+
+                this.cd.markForCheck();
+              });
+          }
+
           this.hasPermissionToReadForeignPortfolio = hasPermission(
             user.permissions,
             permissions.readForeignPortfolio
@@ -180,17 +192,6 @@ export class HomePageComponent implements OnDestroy, OnInit {
         this.cd.markForCheck();
       });
 
-    if (this.hasPermissionToAccessFearAndGreedIndex) {
-      this.dataService
-        .fetchSymbolItem('GF.FEAR_AND_GREED_INDEX')
-        .pipe(takeUntil(this.unsubscribeSubject))
-        .subscribe(({ marketPrice }) => {
-          this.fearAndGreedIndex = marketPrice;
-
-          this.cd.markForCheck();
-        });
-    }
-
     this.cd.markForCheck();
   }
 

libs/helper/src/lib/helper.ts

@@ -66,11 +66,11 @@ export function isCurrency(aSymbol = '') {
   );
 }
 
-export function isGhostfolioScraperApi(aSymbol = '') {
+export function isGhostfolioScraperApiSymbol(aSymbol = '') {
   return aSymbol.startsWith('[GF]');
 }
 
-export function isRakutenRapidApi(aSymbol = '') {
+export function isRakutenRapidApiSymbol(aSymbol = '') {
   return aSymbol === 'GF.FEAR_AND_GREED_INDEX';
 }