sudacode/ghostfolio
1
0
Fork 0
Code Issues Pull Requests Actions Packages Releases Activity

Feature/introduce env variable data source exchange rates and data source import (#1910)

Browse Source 
* Introduce env variables DATA_SOURCE_EXCHANGE_RATES and DATA_SOURCE_IMPORT

* Update changelog
This commit is contained in:
Thomas Kaul 2023-04-30 18:26:34 +02:00 committed by GitHub
parent 4090b03406
commit e500ccb61b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 215 additions and 100 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
CHANGELOG.md
View File

@ -5,6 +5,12 @@ All notable changes to this project will be documented in this file.
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
## Unreleased
### Changed
- Split the environment variable `DATA_SOURCE_PRIMARY` in `DATA_SOURCE_EXCHANGE_RATES` and `DATA_SOURCE_IMPORT`
## 1.262.0 - 2023-04-29
### Added

7
apps/api/src/app/import/import.service.ts
View File

@ -15,7 +15,7 @@ import {
OrderWithAccount
} from '@ghostfolio/common/types';
import { Injectable } from '@nestjs/common';
import { Prisma, SymbolProfile } from '@prisma/client';
import { DataSource, Prisma, SymbolProfile } from '@prisma/client';
import Big from 'big.js';
import { endOfToday, isAfter, isSameDay, parseISO } from 'date-fns';
import { v4 as uuidv4 } from 'uuid';
@ -183,9 +183,10 @@ export class ImportService {
for (const activity of activitiesDto) {
if (!activity.dataSource) {
if (activity.type === 'ITEM') {
activity.dataSource = 'MANUAL';
activity.dataSource = DataSource.MANUAL;
} else {
activity.dataSource = this.dataProviderService.getPrimaryDataSource();
activity.dataSource =
this.dataProviderService.getDataSourceForImport();
}
}

3
apps/api/src/services/configuration/configuration.service.ts
View File

@ -16,7 +16,8 @@ export class ConfigurationService {
default: 'USD'
}),
CACHE_TTL: num({ default: 1 }),
DATA_SOURCE_PRIMARY: str({ default: DataSource.YAHOO }),
DATA_SOURCE_EXCHANGE_RATES: str({ default: DataSource.YAHOO }),
DATA_SOURCE_IMPORT: str({ default: DataSource.YAHOO }),
DATA_SOURCES: json({
default: [DataSource.COINGECKO, DataSource.MANUAL, DataSource.YAHOO]
}),

86
apps/api/src/services/data-provider/data-provider.service.ts
View File

@ -58,6 +58,52 @@ export class DataProviderService {
return false;
}
public async getAssetProfiles(items: IDataGatheringItem[]): Promise<{
[symbol: string]: Partial<SymbolProfile>;
}> {
const response: {
[symbol: string]: Partial<SymbolProfile>;
} = {};
const itemsGroupedByDataSource = groupBy(items, (item) => item.dataSource);
const promises = [];
for (const [dataSource, dataGatheringItems] of Object.entries(
itemsGroupedByDataSource
)) {
const symbols = dataGatheringItems.map((dataGatheringItem) => {
return dataGatheringItem.symbol;
});
for (const symbol of symbols) {
const promise = Promise.resolve(
this.getDataProvider(DataSource[dataSource]).getAssetProfile(symbol)
);
promises.push(
promise.then((symbolProfile) => {
response[symbol] = symbolProfile;
})
);
}
}
await Promise.all(promises);
return response;
}
public getDataSourceForExchangeRates(): DataSource {
return DataSource[
this.configurationService.get('DATA_SOURCE_EXCHANGE_RATES')
];
}
public getDataSourceForImport(): DataSource {
return DataSource[this.configurationService.get('DATA_SOURCE_IMPORT')];
}
public async getDividends({
dataSource,
from,
@ -182,46 +228,6 @@ export class DataProviderService {
return result;
}
public getPrimaryDataSource(): DataSource {
return DataSource[this.configurationService.get('DATA_SOURCE_PRIMARY')];
}
public async getAssetProfiles(items: IDataGatheringItem[]): Promise<{
[symbol: string]: Partial<SymbolProfile>;
}> {
const response: {
[symbol: string]: Partial<SymbolProfile>;
} = {};
const itemsGroupedByDataSource = groupBy(items, (item) => item.dataSource);
const promises = [];
for (const [dataSource, dataGatheringItems] of Object.entries(
itemsGroupedByDataSource
)) {
const symbols = dataGatheringItems.map((dataGatheringItem) => {
return dataGatheringItem.symbol;
});
for (const symbol of symbols) {
const promise = Promise.resolve(
this.getDataProvider(DataSource[dataSource]).getAssetProfile(symbol)
);
promises.push(
promise.then((symbolProfile) => {
response[symbol] = symbolProfile;
})
);
}
}
await Promise.all(promises);
return response;
}
public async getQuotes(items: IDataGatheringItem[]): Promise<{
[symbol: string]: IDataProviderResponse;
}> {

147
apps/api/src/services/data-provider/eod-historical-data/eod-historical-data.service.ts
View File

@ -5,7 +5,7 @@ import {
IDataProviderHistoricalResponse,
IDataProviderResponse
} from '@ghostfolio/api/services/interfaces/interfaces';
import { DATE_FORMAT } from '@ghostfolio/common/helper';
import { DATE_FORMAT, isCurrency } from '@ghostfolio/common/helper';
import { Granularity } from '@ghostfolio/common/types';
import { Injectable, Logger } from '@nestjs/common';
import {
@ -15,17 +15,20 @@ import {
SymbolProfile
} from '@prisma/client';
import bent from 'bent';
import Big from 'big.js';
import { format, isToday } from 'date-fns';
@Injectable()
export class EodHistoricalDataService implements DataProviderInterface {
private apiKey: string;
private baseCurrency: string;
private readonly URL = 'https://eodhistoricaldata.com/api';
public constructor(
private readonly configurationService: ConfigurationService
) {
this.apiKey = this.configurationService.get('EOD_HISTORICAL_DATA_API_KEY');
this.baseCurrency = this.configurationService.get('BASE_CURRENCY');
}
public canHandle(symbol: string) {
@ -70,9 +73,11 @@ export class EodHistoricalDataService implements DataProviderInterface {
): Promise<{
[symbol: string]: { [date: string]: IDataProviderHistoricalResponse };
}> {
const symbol = this.convertToEodSymbol(aSymbol);
try {
const get = bent(
`${this.URL}/eod/${aSymbol}?api_token=${
`${this.URL}/eod/${symbol}?api_token=${
this.apiKey
}&fmt=json&from=${format(from, DATE_FORMAT)}&to=${format(
to,
@ -87,14 +92,17 @@ export class EodHistoricalDataService implements DataProviderInterface {
return response.reduce(
(result, historicalItem, index, array) => {
result[aSymbol][historicalItem.date] = {
marketPrice: historicalItem.close,
result[this.convertFromEodSymbol(symbol)][historicalItem.date] = {
marketPrice: this.getConvertedValue({
symbol: aSymbol,
value: historicalItem.close
}),
performance: historicalItem.open - historicalItem.close
};
return result;
},
{ [aSymbol]: {} }
{ [this.convertFromEodSymbol(symbol)]: {} }
);
} catch (error) {
throw new Error(
@ -119,52 +127,87 @@ export class EodHistoricalDataService implements DataProviderInterface {
public async getQuotes(
aSymbols: string[]
): Promise<{ [symbol: string]: IDataProviderResponse }> {
if (aSymbols.length <= 0) {
const symbols = aSymbols.map((symbol) => {
return this.convertToEodSymbol(symbol);
});
if (symbols.length <= 0) {
return {};
}
try {
const get = bent(
`${this.URL}/real-time/${aSymbols[0]}?api_token=${
`${this.URL}/real-time/${symbols[0]}?api_token=${
this.apiKey
}&fmt=json&s=${aSymbols.join(',')}`,
}&fmt=json&s=${symbols.join(',')}`,
'GET',
'json',
200
);
const [realTimeResponse, searchResponse] = await Promise.all([
get(),
this.search(aSymbols[0])
]);
const realTimeResponse = await get();
const quotes =
aSymbols.length === 1 ? [realTimeResponse] : realTimeResponse;
symbols.length === 1 ? [realTimeResponse] : realTimeResponse;
return quotes.reduce(
const searchResponse = await Promise.all(
symbols
.filter((symbol) => {
return !symbol.endsWith('.FOREX');
})
.map((symbol) => {
return this.search(symbol);
})
);
const lookupItems = searchResponse.flat().map(({ items }) => {
return items[0];
});
const response = quotes.reduce(
(
result: { [symbol: string]: IDataProviderResponse },
{ close, code, timestamp }
) => {
const currency = this.convertCurrency(
searchResponse?.items[0]?.currency
);
const currency = lookupItems.find((lookupItem) => {
return lookupItem.symbol === code;
})?.currency;
if (currency) {
result[code] = {
currency,
result[this.convertFromEodSymbol(code)] = {
currency: currency ?? this.baseCurrency,
dataSource: DataSource.EOD_HISTORICAL_DATA,
marketPrice: close,
marketState: isToday(new Date(timestamp * 1000))
? 'open'
: 'closed'
marketState: isToday(new Date(timestamp * 1000)) ? 'open' : 'closed'
};
}
return result;
},
{}
);
if (response[`${this.baseCurrency}GBP`]) {
response[`${this.baseCurrency}GBp`] = {
...response[`${this.baseCurrency}GBP`],
currency: `${this.baseCurrency}GBp`,
marketPrice: this.getConvertedValue({
symbol: `${this.baseCurrency}GBp`,
value: response[`${this.baseCurrency}GBP`].marketPrice
})
};
}
if (response[`${this.baseCurrency}ILS`]) {
response[`${this.baseCurrency}ILA`] = {
...response[`${this.baseCurrency}ILS`],
currency: `${this.baseCurrency}ILA`,
marketPrice: this.getConvertedValue({
symbol: `${this.baseCurrency}ILA`,
value: response[`${this.baseCurrency}ILS`].marketPrice
})
};
}
return response;
} catch (error) {
Logger.error(error, 'EodHistoricalDataService');
}
@ -182,7 +225,7 @@ export class EodHistoricalDataService implements DataProviderInterface {
return {
items: searchResult
.filter(({ symbol }) => {
return !symbol.toLowerCase().endsWith('forex');
return !symbol.endsWith('.FOREX');
})
.map(
({
@ -216,6 +259,60 @@ export class EodHistoricalDataService implements DataProviderInterface {
return currency;
}
private convertFromEodSymbol(aEodSymbol: string) {
let symbol = aEodSymbol;
if (symbol.endsWith('.FOREX')) {
symbol = symbol.replace('GBX', 'GBp');
symbol = symbol.replace('.FOREX', '');
symbol = `${this.baseCurrency}${symbol}`;
}
return symbol;
}
/**
* Converts a symbol to a EOD symbol
*
* Currency: USDCHF -> CHF.FOREX
*/
private convertToEodSymbol(aSymbol: string) {
if (
aSymbol.startsWith(this.baseCurrency) &&
aSymbol.length > this.baseCurrency.length
) {
if (
isCurrency(
aSymbol.substring(0, aSymbol.length - this.baseCurrency.length)
)
) {
return `${aSymbol
.replace('GBp', 'GBX')
.replace(this.baseCurrency, '')}.FOREX`;
}
}
return aSymbol;
}
private getConvertedValue({
symbol,
value
}: {
symbol: string;
value: number;
}) {
if (symbol === `${this.baseCurrency}GBp`) {
// Convert GPB to GBp (pence)
return new Big(value).mul(100).toNumber();
} else if (symbol === `${this.baseCurrency}ILA`) {
// Convert ILS to ILA
return new Big(value).mul(100).toNumber();
}
return value;
}
private async getSearchResult(aQuery: string): Promise<
(LookupItem & {
assetClass: AssetClass;

21
apps/api/src/services/data-provider/yahoo-finance/yahoo-finance.service.ts
View File

@ -203,9 +203,10 @@ export class YahooFinanceService implements DataProviderInterface {
response[`${this.baseCurrency}GBp`] = {
...response[symbol],
currency: 'GBp',
marketPrice: new Big(response[symbol].marketPrice)
.mul(100)
.toNumber()
marketPrice: this.getConvertedValue({
symbol: `${this.baseCurrency}GBp`,
value: response[symbol].marketPrice
})
};
} else if (
symbol === `${this.baseCurrency}ILS` &&
@ -215,9 +216,10 @@ export class YahooFinanceService implements DataProviderInterface {
response[`${this.baseCurrency}ILA`] = {
...response[symbol],
currency: 'ILA',
marketPrice: new Big(response[symbol].marketPrice)
.mul(100)
.toNumber()
marketPrice: this.getConvertedValue({
symbol: `${this.baseCurrency}ILA`,
value: response[symbol].marketPrice
})
};
} else if (
symbol === `${this.baseCurrency}ZAR` &&
@ -227,9 +229,10 @@ export class YahooFinanceService implements DataProviderInterface {
response[`${this.baseCurrency}ZAc`] = {
...response[symbol],
currency: 'ZAc',
marketPrice: new Big(response[symbol].marketPrice)
.mul(100)
.toNumber()
marketPrice: this.getConvertedValue({
symbol: `${this.baseCurrency}ZAc`,
value: response[symbol].marketPrice
})
};
}
}

36
apps/api/src/services/exchange-rate-data/exchange-rate-data.service.ts
View File

@ -61,42 +61,41 @@ export class ExchangeRateDataService {
getYesterday()
);
// TODO: add fallback
/*if (Object.keys(result).length !== this.currencyPairs.length) {
if (Object.keys(result).length !== this.currencyPairs.length) {
// Load currencies directly from data provider as a fallback
// if historical data is not fully available
const historicalData = await this.dataProviderService.getQuotes(
const quotes = await this.dataProviderService.getQuotes(
this.currencyPairs.map(({ dataSource, symbol }) => {
return { dataSource, symbol };
})
);
Object.keys(historicalData).forEach((key) => {
if (isNumber(historicalData[key].marketPrice)) {
result[key] = {
for (const symbol of Object.keys(quotes)) {
if (isNumber(quotes[symbol].marketPrice)) {
result[symbol] = {
[format(getYesterday(), DATE_FORMAT)]: {
marketPrice: historicalData[key].marketPrice
marketPrice: quotes[symbol].marketPrice
}
};
}
});
}*/
}
}
const resultExtended = result;
Object.keys(result).forEach((pair) => {
const [currency1, currency2] = pair.match(/.{1,3}/g);
const [date] = Object.keys(result[pair]);
for (const symbol of Object.keys(result)) {
const [currency1, currency2] = symbol.match(/.{1,3}/g);
const [date] = Object.keys(result[symbol]);
// Calculate the opposite direction
resultExtended[`${currency2}${currency1}`] = {
[date]: {
marketPrice: 1 / result[pair][date].marketPrice
marketPrice: 1 / result[symbol][date].marketPrice
}
};
});
}
Object.keys(resultExtended).forEach((symbol) => {
for (const symbol of Object.keys(resultExtended)) {
const [currency1, currency2] = symbol.match(/.{1,3}/g);
const date = format(getYesterday(), DATE_FORMAT);
@ -114,7 +113,7 @@ export class ExchangeRateDataService {
this.exchangeRates[`${currency2}${currency1}`] =
1 / this.exchangeRates[symbol];
}
});
}
}
public toCurrency(
@ -173,7 +172,8 @@ export class ExchangeRateDataService {
let factor: number;
if (aFromCurrency !== aToCurrency) {
const dataSource = this.dataProviderService.getPrimaryDataSource();
const dataSource =
this.dataProviderService.getDataSourceForExchangeRates();
const symbol = `${aFromCurrency}${aToCurrency}`;
const marketData = await this.marketDataService.get({
@ -274,7 +274,7 @@ export class ExchangeRateDataService {
return {
currency1: this.baseCurrency,
currency2: currency,
dataSource: this.dataProviderService.getPrimaryDataSource(),
dataSource: this.dataProviderService.getDataSourceForExchangeRates(),
symbol: `${this.baseCurrency}${currency}`
};
});

3
apps/api/src/services/interfaces/environment.interface.ts
View File

@ -5,7 +5,8 @@ export interface Environment extends CleanedEnvAccessors {
ALPHA_VANTAGE_API_KEY: string;
BASE_CURRENCY: string;
CACHE_TTL: number;
DATA_SOURCE_PRIMARY: string;
DATA_SOURCE_EXCHANGE_RATES: string;
DATA_SOURCE_IMPORT: string;
DATA_SOURCES: string[];
ENABLE_FEATURE_BLOG: boolean;
ENABLE_FEATURE_FEAR_AND_GREED_INDEX: boolean;