Feature/introduce max number of symbols per data provider request (#1111)

* Introduce maximum number of symbols per request

* Set log level settings

* Update changelog

CHANGELOG.md

@@ -5,6 +5,13 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## Unreleased
+
+### Changed
+
+- Improved the performance of data provider requests by introducing a maximum number of symbols per request (chunk size)
+- Changed the log level settings
+
 ## 1.175.0 - 29.07.2022
 
 ### Added

apps/api/src/main.ts

@@ -5,7 +5,12 @@ import { AppModule } from './app/app.module';
 import { environment } from './environments/environment';
 
 async function bootstrap() {
-  const app = await NestFactory.create(AppModule);
+  const app = await NestFactory.create(AppModule, {
+    logger:
+      process.env.NODE_ENV === 'production'
+        ? ['error', 'log', 'warn']
+        : ['debug', 'error', 'log', 'verbose', 'warn']
+  });
   app.enableCors();
   app.enableVersioning({
     defaultVersion: '1',

apps/api/src/services/data-provider/data-provider.service.ts

@@ -168,6 +168,7 @@ export class DataProviderService {
     const response: {
       [symbol: string]: IDataProviderResponse;
     } = {};
+    const startTimeTotal = performance.now();
 
     const itemsGroupedByDataSource = groupBy(items, (item) => item.dataSource);
 
@@ -176,25 +177,59 @@ export class DataProviderService {
     for (const [dataSource, dataGatheringItems] of Object.entries(
       itemsGroupedByDataSource
     )) {
+      const dataProvider = this.getDataProvider(DataSource[dataSource]);
+
       const symbols = dataGatheringItems.map((dataGatheringItem) => {
         return dataGatheringItem.symbol;
       });
 
-      const promise = Promise.resolve(
+      const maximumNumberOfSymbolsPerRequest =
-        this.getDataProvider(DataSource[dataSource]).getQuotes(symbols)
+        dataProvider.getMaxNumberOfSymbolsPerRequest?.() ??
-      );
+        Number.MAX_SAFE_INTEGER;
+      for (
+        let i = 0;
+        i < symbols.length;
+        i += maximumNumberOfSymbolsPerRequest
+      ) {
+        const startTimeDataSource = performance.now();
 
-      promises.push(
+        const symbolsChunk = symbols.slice(
-        promise.then((result) => {
+          i,
-          for (const [symbol, dataProviderResponse] of Object.entries(result)) {
+          i + maximumNumberOfSymbolsPerRequest
-            response[symbol] = dataProviderResponse;
+        );
-          }
+
-        })
+        const promise = Promise.resolve(dataProvider.getQuotes(symbolsChunk));
-      );
+
+        promises.push(
+          promise.then((result) => {
+            for (const [symbol, dataProviderResponse] of Object.entries(
+              result
+            )) {
+              response[symbol] = dataProviderResponse;
+            }
+
+            Logger.debug(
+              `Fetched ${symbolsChunk.length} quotes from ${dataSource} in ${(
+                (performance.now() - startTimeDataSource) /
+                1000
+              ).toFixed(3)} seconds`
+            );
+          })
+        );
+      }
     }
 
     await Promise.all(promises);
 
+    Logger.debug('------------------------------------------------');
+    Logger.debug(
+      `Fetched ${items.length} quotes in ${(
+        (performance.now() - startTimeTotal) /
+        1000
+      ).toFixed(3)} seconds`
+    );
+    Logger.debug('================================================');
+
     return response;
   }
 

apps/api/src/services/data-provider/eod-historical-data/eod-historical-data.service.ts

@@ -81,6 +81,12 @@ export class EodHistoricalDataService implements DataProviderInterface {
     }
   }
 
+  public getMaxNumberOfSymbolsPerRequest() {
+    // It is not recommended using more than 15-20 tickers per request
+    // https://eodhistoricaldata.com/financial-apis/live-realtime-stocks-api
+    return 20;
+  }
+
   public getName(): DataSource {
     return DataSource.EOD_HISTORICAL_DATA;
   }

apps/api/src/services/data-provider/interfaces/data-provider.interface.ts

@@ -20,6 +20,8 @@ export interface DataProviderInterface {
     [symbol: string]: { [date: string]: IDataProviderHistoricalResponse };
   }>; // TODO: Return only one symbol
 
+  getMaxNumberOfSymbolsPerRequest?(): number;
+
   getName(): DataSource;
 
   getQuotes(

apps/api/src/services/data-provider/yahoo-finance/yahoo-finance.service.ts

@@ -208,6 +208,10 @@ export class YahooFinanceService implements DataProviderInterface {
     }
   }
 
+  public getMaxNumberOfSymbolsPerRequest() {
+    return 50;
+  }
+
   public getName(): DataSource {
     return DataSource.YAHOO;
   }