Feature/improve data provider service (#637)

* Improve data provider service

* Update changelog

CHANGELOG.md

@@ -7,9 +7,18 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## Unreleased
 
+### Added
+
+- Added support for fetching multiple symbols in the `GOOGLE_SHEETS` data provider
+
+### Changed
+
+- Improved the data provider with grouping by data source and thereby reducing the number of requests
+
 ### Fixed
 
 - Fixed the unresolved account names in the _X-ray_ section
+- Fixed the date conversion in the `GOOGLE_SHEETS` data provider
 
 ## 1.104.0 - 16.01.2022
 

apps/api/src/services/data-provider/data-provider.service.ts

@@ -12,7 +12,7 @@ import { Granularity } from '@ghostfolio/common/types';
 import { Inject, Injectable, Logger } from '@nestjs/common';
 import { DataSource, MarketData } from '@prisma/client';
 import { format, isValid } from 'date-fns';
-import { isEmpty } from 'lodash';
+import { groupBy, isEmpty } from 'lodash';
 
 @Injectable()
 export class DataProviderService {
@@ -30,18 +30,27 @@ export class DataProviderService {
       [symbol: string]: IDataProviderResponse;
     } = {};
 
-    for (const item of items) {
+    const itemsGroupedByDataSource = groupBy(items, (item) => item.dataSource);
-      const dataProvider = this.getDataProvider(item.dataSource);
-      response[item.symbol] = (await dataProvider.get([item.symbol]))[
-        item.symbol
-      ];
-    }
 
     const promises = [];
-    for (const symbol of Object.keys(response)) {
+
-      const promise = Promise.resolve(response[symbol]);
+    for (const [dataSource, dataGatheringItems] of Object.entries(
+      itemsGroupedByDataSource
+    )) {
+      const symbols = dataGatheringItems.map((dataGatheringItem) => {
+        return dataGatheringItem.symbol;
+      });
+
+      const promise = Promise.resolve(
+        this.getDataProvider(DataSource[dataSource]).get(symbols)
+      );
+
       promises.push(
-        promise.then((currentResponse) => (response[symbol] = currentResponse))
+        promise.then((result) => {
+          for (const [symbol, dataProviderResponse] of Object.entries(result)) {
+            response[symbol] = dataProviderResponse;
+          }
+        })
       );
     }
 

apps/api/src/services/data-provider/google-sheets/google-sheets.service.ts

@@ -8,7 +8,7 @@ import {
 } from '@ghostfolio/api/services/interfaces/interfaces';
 import { PrismaService } from '@ghostfolio/api/services/prisma.service';
 import { SymbolProfileService } from '@ghostfolio/api/services/symbol-profile.service';
-import { DATE_FORMAT } from '@ghostfolio/common/helper';
+import { DATE_FORMAT, parseDate } from '@ghostfolio/common/helper';
 import { Granularity } from '@ghostfolio/common/types';
 import { Injectable, Logger } from '@nestjs/common';
 import { DataSource } from '@prisma/client';
@@ -35,27 +35,36 @@ export class GoogleSheetsService implements DataProviderInterface {
     }
 
     try {
-      const [symbol] = aSymbols;
+      const response: { [symbol: string]: IDataProviderResponse } = {};
-      const [symbolProfile] = await this.symbolProfileService.getSymbolProfiles(
+
-        [symbol]
+      const symbolProfiles = await this.symbolProfileService.getSymbolProfiles(
+        aSymbols
       );
 
       const sheet = await this.getSheet({
         sheetId: this.configurationService.get('GOOGLE_SHEETS_ID'),
-        symbol
+        symbol: 'Overview'
       });
-      const marketPrice = parseFloat(
-        (await sheet.getCellByA1('B1').value) as string
-      );
 
-      return {
+      const rows = await sheet.getRows();
-        [symbol]: {
+
-          marketPrice,
+      for (const row of rows) {
-          currency: symbolProfile?.currency,
+        const marketPrice = parseFloat(row['marketPrice']);
-          dataSource: this.getName(),
+        const symbol = row['symbol'];
-          marketState: MarketState.delayed
+
+        if (aSymbols.includes(symbol)) {
+          response[symbol] = {
+            marketPrice,
+            currency: symbolProfiles.find((symbolProfile) => {
+              return symbolProfile.symbol === symbol;
+            })?.currency,
+            dataSource: this.getName(),
+            marketState: MarketState.delayed
+          };
         }
-      };
+      }
+
+      return response;
     } catch (error) {
       Logger.error(error);
     }
@@ -94,7 +103,7 @@ export class GoogleSheetsService implements DataProviderInterface {
           return index >= 1;
         })
         .forEach((row) => {
-          const date = new Date(row._rawData[0]);
+          const date = parseDate(row._rawData[0]);
           const close = parseFloat(row._rawData[1]);
 
           historicalData[format(date, DATE_FORMAT)] = { marketPrice: close };

apps/client/src/app/components/admin-market-data/admin-market-data.html

@@ -16,8 +16,8 @@
               class="cursor-pointer mat-row"
               (click)="setCurrentSymbol(item.symbol)"
             >
-              <td class="mat-cell px-1 py-2">{{ item.symbol }}</td>
+              <td class="mat-cell px-1 py-2">{{ item.symbol }}</td>
-              <td class="mat-cell px-1 py-2">{{ item.dataSource}}</td>
+              <td class="mat-cell px-1 py-2">{{ item.dataSource }}</td>
               <td class="mat-cell px-1 py-2">
                 {{ (item.date | date: defaultDateFormat) ?? '' }}
               </td>