sudacode/ghostfolio
1
0
Fork 0
Code Issues Pull Requests Actions Packages Releases Activity

Feature/improve handling of derived currencies (#2891)

Browse Source 
* Improve handling of derived currencies

* Update changelog
This commit is contained in:
Thomas Kaul 2024-01-21 11:12:48 +01:00 committed by GitHub
parent 6167f105fe
commit 5ba5b86d5f
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
6 changed files with 217 additions and 196 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
CHANGELOG.md
View File

@ -14,6 +14,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
### Changed ### Changed
- Improved the handling of derived currencies
- Improved the labels in the portfolio evolution chart and investment timeline on the analysis page - Improved the labels in the portfolio evolution chart and investment timeline on the analysis page
- Improved the language localization for German (`de`) - Improved the language localization for German (`de`)
- Upgraded `prisma` from version `5.7.1` to `5.8.1` - Upgraded `prisma` from version `5.7.1` to `5.8.1`

198
apps/api/src/services/data-provider/data-provider.service.ts
View File

@ -9,14 +9,19 @@ import {
import { MarketDataService } from '@ghostfolio/api/services/market-data/market-data.service'; import { MarketDataService } from '@ghostfolio/api/services/market-data/market-data.service';
import { PrismaService } from '@ghostfolio/api/services/prisma/prisma.service'; import { PrismaService } from '@ghostfolio/api/services/prisma/prisma.service';
import { PropertyService } from '@ghostfolio/api/services/property/property.service'; import { PropertyService } from '@ghostfolio/api/services/property/property.service';
import { PROPERTY_DATA_SOURCE_MAPPING } from '@ghostfolio/common/config'; import {
DEFAULT_CURRENCY,
DERIVED_CURRENCIES,
PROPERTY_DATA_SOURCE_MAPPING
} from '@ghostfolio/common/config';
import { DATE_FORMAT, getStartOfUtcDate } from '@ghostfolio/common/helper'; import { DATE_FORMAT, getStartOfUtcDate } from '@ghostfolio/common/helper';
import { UniqueAsset } from '@ghostfolio/common/interfaces'; import { UniqueAsset } from '@ghostfolio/common/interfaces';
import type { Granularity, UserWithSettings } from '@ghostfolio/common/types'; import type { Granularity, UserWithSettings } from '@ghostfolio/common/types';
import { Inject, Injectable, Logger } from '@nestjs/common'; import { Inject, Injectable, Logger } from '@nestjs/common';
import { DataSource, MarketData, SymbolProfile } from '@prisma/client'; import { DataSource, MarketData, SymbolProfile } from '@prisma/client';
import { format, isValid } from 'date-fns'; import Big from 'big.js';
import { groupBy, isEmpty, isNumber } from 'lodash'; import { eachDayOfInterval, format, isValid } from 'date-fns';
import { groupBy, isEmpty, isNumber, uniqWith } from 'lodash';
import ms from 'ms'; import ms from 'ms';
@Injectable() @Injectable()
@ -205,6 +210,31 @@ export class DataProviderService {
): Promise<{ ): Promise<{
[symbol: string]: { [date: string]: IDataProviderHistoricalResponse }; [symbol: string]: { [date: string]: IDataProviderHistoricalResponse };
}> { }> {
let dataGatheringItems = aDataGatheringItems;
for (const { currency, rootCurrency } of DERIVED_CURRENCIES) {
if (
this.hasCurrency({
dataGatheringItems,
currency: `${DEFAULT_CURRENCY}${currency}`
})
) {
// Skip derived currency
dataGatheringItems = dataGatheringItems.filter(({ symbol }) => {
return symbol !== `${DEFAULT_CURRENCY}${currency}`;
});
// Add root currency
dataGatheringItems.push({
dataSource: this.getDataSourceForExchangeRates(),
symbol: `${DEFAULT_CURRENCY}${rootCurrency}`
});
}
}
dataGatheringItems = uniqWith(dataGatheringItems, (obj1, obj2) => {
return obj1.dataSource === obj2.dataSource && obj1.symbol === obj2.symbol;
});
const result: { const result: {
[symbol: string]: { [date: string]: IDataProviderHistoricalResponse }; [symbol: string]: { [date: string]: IDataProviderHistoricalResponse };
} = {}; } = {};
@ -213,25 +243,59 @@ export class DataProviderService {
data: { [date: string]: IDataProviderHistoricalResponse }; data: { [date: string]: IDataProviderHistoricalResponse };
symbol: string; symbol: string;
}>[] = []; }>[] = [];
for (const { dataSource, symbol } of aDataGatheringItems) { for (const { dataSource, symbol } of dataGatheringItems) {
const dataProvider = this.getDataProvider(dataSource); const dataProvider = this.getDataProvider(dataSource);
if (dataProvider.canHandle(symbol)) { if (dataProvider.canHandle(symbol)) {
promises.push( if (symbol === `${DEFAULT_CURRENCY}USX`) {
dataProvider const data: {
.getHistorical({ [date: string]: IDataProviderHistoricalResponse;
from, } = {};
symbol,
to, for (const date of eachDayOfInterval({ end: to, start: from })) {
requestTimeout: ms('30 seconds') data[format(date, DATE_FORMAT)] = { marketPrice: 100 };
}
promises.push(
Promise.resolve({
data,
symbol
}) })
.then((data) => ({ data: data?.[symbol], symbol })) );
); } else {
promises.push(
dataProvider
.getHistorical({
from,
symbol,
to,
requestTimeout: ms('30 seconds')
})
.then((data) => {
return { symbol, data: data?.[symbol] };
})
);
}
} }
} }
try { try {
const allData = await Promise.all(promises); const allData = await Promise.all(promises);
for (const { data, symbol } of allData) { for (const { data, symbol } of allData) {
const currency = DERIVED_CURRENCIES.find(({ rootCurrency }) => {
return `${DEFAULT_CURRENCY}${rootCurrency}` === symbol;
});
if (currency) {
// Add derived currency
result[`${DEFAULT_CURRENCY}${currency.currency}`] =
this.transformHistoricalData({
allData,
currency: `${DEFAULT_CURRENCY}${currency.rootCurrency}`,
factor: currency.factor
});
}
result[symbol] = data; result[symbol] = data;
} }
} catch (error) { } catch (error) {
@ -257,6 +321,19 @@ export class DataProviderService {
} = {}; } = {};
const startTimeTotal = performance.now(); const startTimeTotal = performance.now();
if (
items.some(({ symbol }) => {
return symbol === `${DEFAULT_CURRENCY}USX`;
})
) {
response[`${DEFAULT_CURRENCY}USX`] = {
currency: 'USX',
dataSource: this.getDataSourceForExchangeRates(),
marketPrice: 100,
marketState: 'open'
};
}
// Get items from cache // Get items from cache
const itemsToFetch: UniqueAsset[] = []; const itemsToFetch: UniqueAsset[] = [];
@ -326,19 +403,56 @@ export class DataProviderService {
promises.push( promises.push(
promise.then(async (result) => { promise.then(async (result) => {
for (const [symbol, dataProviderResponse] of Object.entries( for (let [symbol, dataProviderResponse] of Object.entries(result)) {
result if (
)) { [
...DERIVED_CURRENCIES.map(({ currency }) => {
return `${DEFAULT_CURRENCY}${currency}`;
}),
`${DEFAULT_CURRENCY}USX`
].includes(symbol)
) {
continue;
}
response[symbol] = dataProviderResponse; response[symbol] = dataProviderResponse;
this.redisCacheService.set( this.redisCacheService.set(
this.redisCacheService.getQuoteKey({ this.redisCacheService.getQuoteKey({
dataSource: DataSource[dataSource], symbol,
symbol dataSource: DataSource[dataSource]
}), }),
JSON.stringify(dataProviderResponse), JSON.stringify(response[symbol]),
this.configurationService.get('CACHE_QUOTES_TTL') this.configurationService.get('CACHE_QUOTES_TTL')
); );
for (const {
currency,
factor,
rootCurrency
} of DERIVED_CURRENCIES) {
if (symbol === `${DEFAULT_CURRENCY}${rootCurrency}`) {
response[`${DEFAULT_CURRENCY}${currency}`] = {
...dataProviderResponse,
currency,
marketPrice: new Big(
result[`${DEFAULT_CURRENCY}${rootCurrency}`].marketPrice
)
.mul(factor)
.toNumber(),
marketState: 'open'
};
this.redisCacheService.set(
this.redisCacheService.getQuoteKey({
dataSource: DataSource[dataSource],
symbol: `${DEFAULT_CURRENCY}${currency}`
}),
JSON.stringify(response[`${DEFAULT_CURRENCY}${currency}`]),
this.configurationService.get('CACHE_QUOTES_TTL')
);
}
}
} }
Logger.debug( Logger.debug(
@ -472,6 +586,21 @@ export class DataProviderService {
throw new Error('No data provider has been found.'); throw new Error('No data provider has been found.');
} }
private hasCurrency({
currency,
dataGatheringItems
}: {
currency: string;
dataGatheringItems: UniqueAsset[];
}) {
return dataGatheringItems.some(({ dataSource, symbol }) => {
return (
dataSource === this.getDataSourceForExchangeRates() &&
symbol === currency
);
});
}
private isPremiumDataSource(aDataSource: DataSource) { private isPremiumDataSource(aDataSource: DataSource) {
const premiumDataSources: DataSource[] = [ const premiumDataSources: DataSource[] = [
DataSource.EOD_HISTORICAL_DATA, DataSource.EOD_HISTORICAL_DATA,
@ -479,4 +608,35 @@ export class DataProviderService {
]; ];
return premiumDataSources.includes(aDataSource); return premiumDataSources.includes(aDataSource);
} }
private transformHistoricalData({
allData,
currency,
factor
}: {
allData: {
data: {
[date: string]: IDataProviderHistoricalResponse;
};
symbol: string;
}[];
currency: string;
factor: number;
}) {
const rootData = allData.find(({ symbol }) => {
return symbol === currency;
})?.data;
const data: {
[date: string]: IDataProviderHistoricalResponse;
} = {};
for (const date in rootData) {
data[date] = {
marketPrice: new Big(factor).mul(rootData[date].marketPrice).toNumber()
};
}
return data;
}
} }

69
apps/api/src/services/data-provider/eod-historical-data/eod-historical-data.service.ts
View File

@ -20,7 +20,6 @@ import {
DataSource, DataSource,
SymbolProfile SymbolProfile
} from '@prisma/client'; } from '@prisma/client';
import Big from 'big.js';
import { format, isToday } from 'date-fns'; import { format, isToday } from 'date-fns';
import got from 'got'; import got from 'got';
@ -93,10 +92,7 @@ export class EodHistoricalDataService implements DataProviderInterface {
return response.reduce( return response.reduce(
(result, historicalItem, index, array) => { (result, historicalItem, index, array) => {
result[this.convertFromEodSymbol(symbol)][historicalItem.date] = { result[this.convertFromEodSymbol(symbol)][historicalItem.date] = {
marketPrice: this.getConvertedValue({ marketPrice: historicalItem.close
symbol: symbol,
value: historicalItem.close
})
}; };
return result; return result;
@ -196,48 +192,6 @@ export class EodHistoricalDataService implements DataProviderInterface {
{} {}
); );
if (response[`${DEFAULT_CURRENCY}GBP`]) {
response[`${DEFAULT_CURRENCY}GBp`] = {
...response[`${DEFAULT_CURRENCY}GBP`],
currency: 'GBp',
marketPrice: this.getConvertedValue({
symbol: `${DEFAULT_CURRENCY}GBp`,
value: response[`${DEFAULT_CURRENCY}GBP`].marketPrice
})
};
}
if (response[`${DEFAULT_CURRENCY}ILS`]) {
response[`${DEFAULT_CURRENCY}ILA`] = {
...response[`${DEFAULT_CURRENCY}ILS`],
currency: 'ILA',
marketPrice: this.getConvertedValue({
symbol: `${DEFAULT_CURRENCY}ILA`,
value: response[`${DEFAULT_CURRENCY}ILS`].marketPrice
})
};
}
if (response[`${DEFAULT_CURRENCY}USX`]) {
response[`${DEFAULT_CURRENCY}USX`] = {
currency: 'USX',
dataSource: this.getName(),
marketPrice: new Big(1).mul(100).toNumber(),
marketState: 'open'
};
}
if (response[`${DEFAULT_CURRENCY}ZAR`]) {
response[`${DEFAULT_CURRENCY}ZAc`] = {
...response[`${DEFAULT_CURRENCY}ZAR`],
currency: 'ZAc',
marketPrice: this.getConvertedValue({
symbol: `${DEFAULT_CURRENCY}ZAc`,
value: response[`${DEFAULT_CURRENCY}ZAR`].marketPrice
})
};
}
return response; return response;
} catch (error) { } catch (error) {
let message = error; let message = error;
@ -337,27 +291,6 @@ export class EodHistoricalDataService implements DataProviderInterface {
return aSymbol; return aSymbol;
} }
private getConvertedValue({
symbol,
value
}: {
symbol: string;
value: number;
}) {
if (symbol === `${DEFAULT_CURRENCY}GBp`) {
// Convert GPB to GBp (pence)
return new Big(value).mul(100).toNumber();
} else if (symbol === `${DEFAULT_CURRENCY}ILA`) {
// Convert ILS to ILA
return new Big(value).mul(100).toNumber();
} else if (symbol === `${DEFAULT_CURRENCY}ZAc`) {
// Convert ZAR to ZAc
return new Big(value).mul(100).toNumber();
}
return value;
}
private async getSearchResult(aQuery: string): Promise< private async getSearchResult(aQuery: string): Promise<
(LookupItem & { (LookupItem & {
assetClass: AssetClass; assetClass: AssetClass;

83
apps/api/src/services/data-provider/yahoo-finance/yahoo-finance.service.ts
View File

@ -16,7 +16,6 @@ import { DEFAULT_CURRENCY } from '@ghostfolio/common/config';
import { DATE_FORMAT } from '@ghostfolio/common/helper'; import { DATE_FORMAT } from '@ghostfolio/common/helper';
import { Injectable, Logger } from '@nestjs/common'; import { Injectable, Logger } from '@nestjs/common';
import { DataSource, SymbolProfile } from '@prisma/client'; import { DataSource, SymbolProfile } from '@prisma/client';
import Big from 'big.js';
import { addDays, format, isSameDay } from 'date-fns'; import { addDays, format, isSameDay } from 'date-fns';
import yahooFinance from 'yahoo-finance2'; import yahooFinance from 'yahoo-finance2';
import { Quote } from 'yahoo-finance2/dist/esm/src/modules/quote'; import { Quote } from 'yahoo-finance2/dist/esm/src/modules/quote';
@ -77,10 +76,7 @@ export class YahooFinanceService implements DataProviderInterface {
for (const historicalItem of historicalResult) { for (const historicalItem of historicalResult) {
response[format(historicalItem.date, DATE_FORMAT)] = { response[format(historicalItem.date, DATE_FORMAT)] = {
marketPrice: this.getConvertedValue({ marketPrice: historicalItem.dividends
symbol,
value: historicalItem.dividends
})
}; };
} }
@ -129,10 +125,7 @@ export class YahooFinanceService implements DataProviderInterface {
for (const historicalItem of historicalResult) { for (const historicalItem of historicalResult) {
response[symbol][format(historicalItem.date, DATE_FORMAT)] = { response[symbol][format(historicalItem.date, DATE_FORMAT)] = {
marketPrice: this.getConvertedValue({ marketPrice: historicalItem.close
symbol: symbol,
value: historicalItem.close
})
}; };
} }
@ -204,57 +197,6 @@ export class YahooFinanceService implements DataProviderInterface {
: 'closed', : 'closed',
marketPrice: quote.regularMarketPrice || 0 marketPrice: quote.regularMarketPrice || 0
}; };
if (
symbol === `${DEFAULT_CURRENCY}GBP` &&
yahooFinanceSymbols.includes(`${DEFAULT_CURRENCY}GBp=X`)
) {
// Convert GPB to GBp (pence)
response[`${DEFAULT_CURRENCY}GBp`] = {
...response[symbol],
currency: 'GBp',
marketPrice: this.getConvertedValue({
symbol: `${DEFAULT_CURRENCY}GBp`,
value: response[symbol].marketPrice
})
};
} else if (
symbol === `${DEFAULT_CURRENCY}ILS` &&
yahooFinanceSymbols.includes(`${DEFAULT_CURRENCY}ILA=X`)
) {
// Convert ILS to ILA
response[`${DEFAULT_CURRENCY}ILA`] = {
...response[symbol],
currency: 'ILA',
marketPrice: this.getConvertedValue({
symbol: `${DEFAULT_CURRENCY}ILA`,
value: response[symbol].marketPrice
})
};
} else if (
symbol === `${DEFAULT_CURRENCY}ZAR` &&
yahooFinanceSymbols.includes(`${DEFAULT_CURRENCY}ZAc=X`)
) {
// Convert ZAR to ZAc (cents)
response[`${DEFAULT_CURRENCY}ZAc`] = {
...response[symbol],
currency: 'ZAc',
marketPrice: this.getConvertedValue({
symbol: `${DEFAULT_CURRENCY}ZAc`,
value: response[symbol].marketPrice
})
};
}
}
if (yahooFinanceSymbols.includes(`${DEFAULT_CURRENCY}USX=X`)) {
// Convert USD to USX (cent)
response[`${DEFAULT_CURRENCY}USX`] = {
currency: 'USX',
dataSource: this.getName(),
marketPrice: new Big(1).mul(100).toNumber(),
marketState: 'open'
};
} }
return response; return response;
@ -357,27 +299,6 @@ export class YahooFinanceService implements DataProviderInterface {
return { items }; return { items };
} }
private getConvertedValue({
symbol,
value
}: {
symbol: string;
value: number;
}) {
if (symbol === `${DEFAULT_CURRENCY}GBp`) {
// Convert GPB to GBp (pence)
return new Big(value).mul(100).toNumber();
} else if (symbol === `${DEFAULT_CURRENCY}ILA`) {
// Convert ILS to ILA
return new Big(value).mul(100).toNumber();
} else if (symbol === `${DEFAULT_CURRENCY}ZAc`) {
// Convert ZAR to ZAc (cents)
return new Big(value).mul(100).toNumber();
}
return value;
}
private async getQuotesWithQuoteSummary(aYahooFinanceSymbols: string[]) { private async getQuotesWithQuoteSummary(aYahooFinanceSymbols: string[]) {
const quoteSummaryPromises = aYahooFinanceSymbols.map((symbol) => { const quoteSummaryPromises = aYahooFinanceSymbols.map((symbol) => {
return yahooFinance.quoteSummary(symbol).catch(() => { return yahooFinance.quoteSummary(symbol).catch(() => {

43
apps/api/src/services/exchange-rate-data/exchange-rate-data.service.ts
View File

@ -5,6 +5,7 @@ import { PrismaService } from '@ghostfolio/api/services/prisma/prisma.service';
import { PropertyService } from '@ghostfolio/api/services/property/property.service'; import { PropertyService } from '@ghostfolio/api/services/property/property.service';
import { import {
DEFAULT_CURRENCY, DEFAULT_CURRENCY,
DERIVED_CURRENCIES,
PROPERTY_CURRENCIES PROPERTY_CURRENCIES
} from '@ghostfolio/common/config'; } from '@ghostfolio/common/config';
import { import {
@ -168,30 +169,6 @@ export class ExchangeRateDataService {
const [currency1, currency2] = symbol.match(/.{1,3}/g); const [currency1, currency2] = symbol.match(/.{1,3}/g);
const [date] = Object.keys(result[symbol]); const [date] = Object.keys(result[symbol]);
// Add derived currencies
if (currency2 === 'GBP') {
resultExtended[`${currency1}GBp`] = {
[date]: {
marketPrice:
result[`${currency1}${currency2}`][date].marketPrice * 100
}
};
} else if (currency2 === 'ILS') {
resultExtended[`${currency1}ILA`] = {
[date]: {
marketPrice:
result[`${currency1}${currency2}`][date].marketPrice * 100
}
};
} else if (currency2 === 'ZAR') {
resultExtended[`${currency1}ZAc`] = {
[date]: {
marketPrice:
result[`${currency1}${currency2}`][date].marketPrice * 100
}
};
}
// Calculate the opposite direction // Calculate the opposite direction
resultExtended[`${currency2}${currency1}`] = { resultExtended[`${currency2}${currency1}`] = {
[date]: { [date]: {
@ -486,8 +463,8 @@ export class ExchangeRateDataService {
} }
} }
}) })
).forEach((account) => { ).forEach(({ currency }) => {
currencies.push(account.currency); currencies.push(currency);
}); });
( (
@ -496,8 +473,8 @@ export class ExchangeRateDataService {
orderBy: [{ currency: 'asc' }], orderBy: [{ currency: 'asc' }],
select: { currency: true } select: { currency: true }
}) })
).forEach((symbolProfile) => { ).forEach(({ currency }) => {
currencies.push(symbolProfile.currency); currencies.push(currency);
}); });
const customCurrencies = (await this.propertyService.getByKey( const customCurrencies = (await this.propertyService.getByKey(
@ -508,6 +485,16 @@ export class ExchangeRateDataService {
currencies = currencies.concat(customCurrencies); currencies = currencies.concat(customCurrencies);
} }
// Add derived currencies
currencies.push('USX');
for (const { currency, rootCurrency } of DERIVED_CURRENCIES) {
if (currencies.includes(currency) || currencies.includes(rootCurrency)) {
currencies.push(currency);
currencies.push(rootCurrency);
}
}
return uniq(currencies).filter(Boolean).sort(); return uniq(currencies).filter(Boolean).sort();
} }

19
libs/common/src/lib/config.ts
View File

@ -41,6 +41,25 @@ export const DEFAULT_LANGUAGE_CODE = 'en';
export const DEFAULT_PAGE_SIZE = 50; export const DEFAULT_PAGE_SIZE = 50;
export const DEFAULT_ROOT_URL = 'http://localhost:4200'; export const DEFAULT_ROOT_URL = 'http://localhost:4200';
// USX is handled separately
export const DERIVED_CURRENCIES = [
{
currency: 'GBp',
factor: 100,
rootCurrency: 'GBP'
},
{
currency: 'ILA',
factor: 100,
rootCurrency: 'ILS'
},
{
currency: 'ZAc',
factor: 100,
rootCurrency: 'ZAR'
}
];
export const EMERGENCY_FUND_TAG_ID = '4452656d-9fa4-4bd0-ba38-70492e31d180'; export const EMERGENCY_FUND_TAG_ID = '4452656d-9fa4-4bd0-ba38-70492e31d180';
export const GATHER_ASSET_PROFILE_PROCESS = 'GATHER_ASSET_PROFILE'; export const GATHER_ASSET_PROFILE_PROCESS = 'GATHER_ASSET_PROFILE';