diff --git a/CHANGELOG.md b/CHANGELOG.md index 100e43b5..673233c5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed - Deactivated asset profiles automatically on delisting in the _Yahoo Finance_ service +- Optimized the query of the data range functionality (`getRange()`) in the market data service - Upgraded `Nx` from version `20.7.1` to `20.8.0` - Upgraded `prisma` from version `6.5.0` to `6.6.0` - Upgraded `storybook` from version `8.4.7` to `8.6.12` diff --git a/apps/api/src/app/portfolio/current-rate.service.spec.ts b/apps/api/src/app/portfolio/current-rate.service.spec.ts index d0e61c8c..d8b7482e 100644 --- a/apps/api/src/app/portfolio/current-rate.service.spec.ts +++ b/apps/api/src/app/portfolio/current-rate.service.spec.ts @@ -6,6 +6,7 @@ import { AssetProfileIdentifier } from '@ghostfolio/common/interfaces'; import { DataSource, MarketData } from '@prisma/client'; import { CurrentRateService } from './current-rate.service'; +import { DateQuery } from './interfaces/date-query.interface'; import { GetValuesObject } from './interfaces/get-values-object.interface'; jest.mock('@ghostfolio/api/services/market-data/market-data.service', () => { @@ -25,33 +26,40 @@ jest.mock('@ghostfolio/api/services/market-data/market-data.service', () => { }, getRange: ({ assetProfileIdentifiers, - dateRangeEnd, - dateRangeStart + dateQuery }: { assetProfileIdentifiers: AssetProfileIdentifier[]; - dateRangeEnd: Date; - dateRangeStart: Date; + dateQuery: DateQuery; + skip?: number; + take?: number; }) => { return Promise.resolve([ { - createdAt: dateRangeStart, + createdAt: dateQuery.gte, dataSource: assetProfileIdentifiers[0].dataSource, - date: dateRangeStart, + date: dateQuery.gte, id: '8fa48fde-f397-4b0d-adbc-fb940e830e6d', marketPrice: 1841.823902, state: 'CLOSE', symbol: assetProfileIdentifiers[0].symbol }, { - createdAt: dateRangeEnd, + createdAt: dateQuery.lt, dataSource: assetProfileIdentifiers[0].dataSource, - date: dateRangeEnd, + date: dateQuery.lt, id: '082d6893-df27-4c91-8a5d-092e84315b56', marketPrice: 1847.839966, state: 'CLOSE', symbol: assetProfileIdentifiers[0].symbol } ]); + }, + getRangeCount: ({}: { + assetProfileIdentifiers: AssetProfileIdentifier[]; + dateRangeEnd: Date; + dateRangeStart: Date; + }) => { + return Promise.resolve(2); } }; }) @@ -128,9 +136,15 @@ describe('CurrentRateService', () => { values: [ { dataSource: 'YAHOO', - date: undefined, + date: new Date('2020-01-01T00:00:00.000Z'), marketPrice: 1841.823902, symbol: 'AMZN' + }, + { + dataSource: 'YAHOO', + date: new Date('2020-01-02T00:00:00.000Z'), + marketPrice: 1847.839966, + symbol: 'AMZN' } ] }); diff --git a/apps/api/src/app/portfolio/current-rate.service.ts b/apps/api/src/app/portfolio/current-rate.service.ts index 058bf1dd..5d39a54b 100644 --- a/apps/api/src/app/portfolio/current-rate.service.ts +++ b/apps/api/src/app/portfolio/current-rate.service.ts @@ -21,6 +21,8 @@ import { GetValuesParams } from './interfaces/get-values-params.interface'; @Injectable() export class CurrentRateService { + private static readonly MARKET_DATA_PAGE_SIZE = 50000; + public constructor( private readonly dataProviderService: DataProviderService, private readonly marketDataService: MarketDataService, @@ -41,42 +43,37 @@ export class CurrentRateService { (!dateQuery.gte || isBefore(dateQuery.gte, new Date())) && (!dateQuery.in || this.containsToday(dateQuery.in)); - const promises: Promise[] = []; const quoteErrors: ResponseError['errors'] = []; const today = resetHours(new Date()); + const values: GetValueObject[] = []; if (includesToday) { - promises.push( - this.dataProviderService - .getQuotes({ items: dataGatheringItems, user: this.request?.user }) - .then((dataResultProvider) => { - const result: GetValueObject[] = []; + const quotesBySymbol = await this.dataProviderService.getQuotes({ + items: dataGatheringItems, + user: this.request?.user + }); - for (const { dataSource, symbol } of dataGatheringItems) { - if (dataResultProvider?.[symbol]?.dataProviderInfo) { - dataProviderInfos.push( - dataResultProvider[symbol].dataProviderInfo - ); - } + for (const { dataSource, symbol } of dataGatheringItems) { + const quote = quotesBySymbol[symbol]; - if (dataResultProvider?.[symbol]?.marketPrice) { - result.push({ - dataSource, - symbol, - date: today, - marketPrice: dataResultProvider?.[symbol]?.marketPrice - }); - } else { - quoteErrors.push({ - dataSource, - symbol - }); - } - } + if (quote?.dataProviderInfo) { + dataProviderInfos.push(quote.dataProviderInfo); + } - return result; - }) - ); + if (quote?.marketPrice) { + values.push({ + dataSource, + symbol, + date: today, + marketPrice: quote.marketPrice + }); + } else { + quoteErrors.push({ + dataSource, + symbol + }); + } + } } const assetProfileIdentifiers: AssetProfileIdentifier[] = @@ -84,34 +81,42 @@ export class CurrentRateService { return { dataSource, symbol }; }); - promises.push( - this.marketDataService - .getRange({ - assetProfileIdentifiers, - dateQuery - }) - .then((data) => { - return data.map(({ dataSource, date, marketPrice, symbol }) => { - return { - dataSource, - date, - marketPrice, - symbol - }; - }); - }) - ); - - const values = await Promise.all(promises).then((array) => { - return array.flat(); + const marketDataCount = await this.marketDataService.getRangeCount({ + assetProfileIdentifiers, + dateQuery }); + for ( + let i = 0; + i < marketDataCount; + i += CurrentRateService.MARKET_DATA_PAGE_SIZE + ) { + // Use page size to limit the number of records fetched at once + const data = await this.marketDataService.getRange({ + assetProfileIdentifiers, + dateQuery, + skip: i, + take: CurrentRateService.MARKET_DATA_PAGE_SIZE + }); + + values.push( + ...data.map(({ dataSource, date, marketPrice, symbol }) => ({ + dataSource, + date, + marketPrice, + symbol + })) + ); + } + const response: GetValuesObject = { dataProviderInfos, errors: quoteErrors.map(({ dataSource, symbol }) => { return { dataSource, symbol }; }), - values: uniqBy(values, ({ date, symbol }) => `${date}-${symbol}`) + values: uniqBy(values, ({ date, symbol }) => { + return `${date}-${symbol}`; + }) }; if (!isEmpty(quoteErrors)) { diff --git a/apps/api/src/services/market-data/market-data.service.ts b/apps/api/src/services/market-data/market-data.service.ts index 390586b3..58b9b09e 100644 --- a/apps/api/src/services/market-data/market-data.service.ts +++ b/apps/api/src/services/market-data/market-data.service.ts @@ -60,12 +60,18 @@ export class MarketDataService { public async getRange({ assetProfileIdentifiers, - dateQuery + dateQuery, + skip, + take }: { assetProfileIdentifiers: AssetProfileIdentifier[]; dateQuery: DateQuery; + skip?: number; + take?: number; }): Promise { return this.prismaService.marketData.findMany({ + skip, + take, orderBy: [ { date: 'asc' @@ -75,17 +81,33 @@ export class MarketDataService { } ], where: { - dataSource: { - in: assetProfileIdentifiers.map(({ dataSource }) => { - return dataSource; - }) - }, date: dateQuery, - symbol: { - in: assetProfileIdentifiers.map(({ symbol }) => { - return symbol; - }) - } + OR: assetProfileIdentifiers.map(({ dataSource, symbol }) => { + return { + dataSource, + symbol + }; + }) + } + }); + } + + public async getRangeCount({ + assetProfileIdentifiers, + dateQuery + }: { + assetProfileIdentifiers: AssetProfileIdentifier[]; + dateQuery: DateQuery; + }): Promise { + return this.prismaService.marketData.count({ + where: { + date: dateQuery, + OR: assetProfileIdentifiers.map(({ dataSource, symbol }) => { + return { + dataSource, + symbol + }; + }) } }); }