sudacode/ghostfolio
1
0
Fork 0
Code Issues Pull Requests Actions Packages Releases Activity

Feature/refactor search functionality (#105)

Browse Source 
* Refactor search functionality

* Update changelog

* Improvements after code review
This commit is contained in:
Thomas 2021-05-20 20:36:08 +02:00 committed by GitHub
parent 79edc09710
commit 200a7d2d65
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
15 changed files with 132 additions and 87 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
CHANGELOG.md
View File

@ -17,6 +17,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
- Hid unknown exchange in the position overview
- Disable the base currency selector for the demo user
- Refactored the portfolio unit tests to work without database
- Refactored the search functionality of the data management (aligned with data source)
- Renamed shared helper to `@ghostfolio/common/helper`
- Moved shared interfaces to `@ghostfolio/common/interfaces`
- Moved shared types to `@ghostfolio/common/types`

3
apps/api/src/app/symbol/interfaces/lookup-item.interface.ts
View File

@ -1,4 +1,7 @@
import { DataSource } from '@prisma/client';
export interface LookupItem {
dataSource: DataSource;
name: string;
symbol: string;
}

7
apps/api/src/app/symbol/symbol.controller.ts
View File

@ -28,9 +28,12 @@ export class SymbolController {
*/
@Get('lookup')
@UseGuards(AuthGuard('jwt'))
public async lookupSymbol(@Query() { query }): Promise<LookupItem[]> {
public async lookupSymbol(
@Query() { query = '' }
): Promise<{ items: LookupItem[] }> {
try {
return this.symbolService.lookup(query);
const encodedQuery = encodeURIComponent(query.toLowerCase());
return this.symbolService.lookup(encodedQuery);
} catch {
throw new HttpException(
getReasonPhrase(StatusCodes.INTERNAL_SERVER_ERROR),

56
apps/api/src/app/symbol/symbol.service.ts
View File

@ -1,10 +1,8 @@
import { DataGatheringService } from '@ghostfolio/api/services/data-gathering.service';
import { DataProviderService } from '@ghostfolio/api/services/data-provider.service';
import { GhostfolioScraperApiService } from '@ghostfolio/api/services/data-provider/ghostfolio-scraper-api/ghostfolio-scraper-api.service';
import { convertFromYahooSymbol } from '@ghostfolio/api/services/data-provider/yahoo-finance/yahoo-finance.service';
import { Injectable } from '@nestjs/common';
import { Currency } from '@prisma/client';
import * as bent from 'bent';
import { Currency, DataSource } from '@prisma/client';
import { LookupItem } from './interfaces/lookup-item.interface';
import { SymbolItem } from './interfaces/symbol-item.interface';
@ -27,62 +25,30 @@ export class SymbolService {
};
}
public async lookup(aQuery = ''): Promise<LookupItem[]> {
const query = aQuery.toLowerCase();
const results: LookupItem[] = [];
public async lookup(aQuery: string): Promise<{ items: LookupItem[] }> {
const results: { items: LookupItem[] } = { items: [] };
if (!query) {
if (!aQuery) {
return results;
}
const get = bent(
`https://query1.finance.yahoo.com/v1/finance/search?q=${query}&lang=en-US&region=US&quotesCount=8&newsCount=0&enableFuzzyQuery=false&quotesQueryId=tss_match_phrase_query&multiQuoteQueryId=multi_quote_single_token_query&newsQueryId=news_cie_vespa&enableCb=true&enableNavLinks=false&enableEnhancedTrivialQuery=true`,
'GET',
'json',
200
);
try {
const { items } = await this.dataProviderService.search(aQuery);
results.items = items;
// Add custom symbols
const scraperConfigurations = await this.ghostfolioScraperApiService.getScraperConfigurations();
scraperConfigurations.forEach((scraperConfiguration) => {
if (scraperConfiguration.name.toLowerCase().startsWith(query)) {
results.push({
if (scraperConfiguration.name.toLowerCase().startsWith(aQuery)) {
results.items.push({
dataSource: DataSource.GHOSTFOLIO,
name: scraperConfiguration.name,
symbol: scraperConfiguration.symbol
});
}
});
try {
const { quotes } = await get();
const searchResult = quotes
.filter(({ isYahooFinance }) => {
return isYahooFinance;
})
.filter(({ quoteType }) => {
return (
quoteType === 'CRYPTOCURRENCY' ||
quoteType === 'EQUITY' ||
quoteType === 'ETF'
);
})
.filter(({ quoteType, symbol }) => {
if (quoteType === 'CRYPTOCURRENCY') {
// Only allow cryptocurrencies in USD
return symbol.includes('USD');
}
return true;
})
.map(({ longname, shortname, symbol }) => {
return {
name: longname || shortname,
symbol: convertFromYahooSymbol(symbol)
};
});
return results.concat(searchResult);
return results;
} catch (error) {
console.error(error);

4
apps/api/src/services/configuration.service.ts
View File

@ -1,7 +1,8 @@
import { Injectable } from '@nestjs/common';
import { bool, cleanEnv, num, port, str } from 'envalid';
import { bool, cleanEnv, json, num, port, str } from 'envalid';
import { Environment } from './interfaces/environment.interface';
import { DataSource } from '.prisma/client';
@Injectable()
export class ConfigurationService {
@ -12,6 +13,7 @@ export class ConfigurationService {
ACCESS_TOKEN_SALT: str(),
ALPHA_VANTAGE_API_KEY: str({ default: '' }),
CACHE_TTL: num({ default: 1 }),
DATA_SOURCES: json({ default: JSON.stringify([DataSource.YAHOO]) }),
ENABLE_FEATURE_CUSTOM_SYMBOLS: bool({ default: false }),
ENABLE_FEATURE_FEAR_AND_GREED_INDEX: bool({ default: false }),
ENABLE_FEATURE_SOCIAL_LOGIN: bool({ default: false }),

18
apps/api/src/services/data-provider.service.ts
View File

@ -1,3 +1,4 @@
import { LookupItem } from '@ghostfolio/api/app/symbol/interfaces/lookup-item.interface';
import {
isCrypto,
isGhostfolioScraperApiSymbol,
@ -5,7 +6,7 @@ import {
} from '@ghostfolio/common/helper';
import { Granularity } from '@ghostfolio/common/types';
import { Injectable } from '@nestjs/common';
import { MarketData } from '@prisma/client';
import { DataSource, MarketData } from '@prisma/client';
import { format } from 'date-fns';
import { ConfigurationService } from './configuration.service';
@ -184,4 +185,19 @@ export class DataProviderService implements DataProviderInterface {
return dataOfYahoo;
}
public async search(aSymbol: string) {
return this.getDataProvider().search(aSymbol);
}
private getDataProvider() {
switch (this.configurationService.get('DATA_SOURCES')[0]) {
case DataSource.ALPHA_VANTAGE:
return this.alphaVantageService;
case DataSource.YAHOO:
return this.yahooFinanceService;
default:
throw new Error('No data provider has been found.');
}
}
}

16
apps/api/src/services/data-provider/alpha-vantage/alpha-vantage.service.ts
View File

@ -1,5 +1,7 @@
import { LookupItem } from '@ghostfolio/api/app/symbol/interfaces/lookup-item.interface';
import { Granularity } from '@ghostfolio/common/types';
import { Injectable } from '@nestjs/common';
import { DataSource } from '@prisma/client';
import { isAfter, isBefore, parse } from 'date-fns';
import { ConfigurationService } from '../../configuration.service';
@ -77,7 +79,17 @@ export class AlphaVantageService implements DataProviderInterface {
}
}
public search(aSymbol: string) {
return this.alphaVantage.data.search(aSymbol);
public async search(aSymbol: string): Promise<{ items: LookupItem[] }> {
const result = await this.alphaVantage.data.search(aSymbol);
return {
items: result?.bestMatches?.map((bestMatch) => {
return {
dataSource: DataSource.ALPHA_VANTAGE,
name: bestMatch['2. name'],
symbol: bestMatch['1. symbol']
};
})
};
}
}

4
apps/api/src/services/data-provider/ghostfolio-scraper-api/ghostfolio-scraper-api.service.ts
View File

@ -117,6 +117,10 @@ export class GhostfolioScraperApiService implements DataProviderInterface {
return [];
}
public async search(aSymbol: string) {
return { items: [] };
}
private extractNumberFromString(aString: string): number {
try {
const [numberString] = aString.match(

12
apps/api/src/services/data-provider/rakuten-rapid-api/rakuten-rapid-api.service.ts
View File

@ -117,6 +117,14 @@ export class RakutenRapidApiService implements DataProviderInterface {
return {};
}
public async search(aSymbol: string) {
return { items: [] };
}
public setPrisma(aPrismaService: PrismaService) {
this.prisma = aPrismaService;
}
private async getFearAndGreedIndex(): Promise<{
now: { value: number; valueText: string };
previousClose: { value: number; valueText: string };
@ -147,8 +155,4 @@ export class RakutenRapidApiService implements DataProviderInterface {
return undefined;
}
}
public setPrisma(aPrismaService: PrismaService) {
this.prisma = aPrismaService;
}
}

24
apps/api/src/services/data-provider/yahoo-finance/yahoo-finance.service.spec.ts___
View File

@ -1,24 +0,0 @@
/*
import { Test } from '@nestjs/testing';
import { YahooFinanceService } from './yahoo-finance.service';
describe('AppService', () => {
let service: YahooFinanceService;
beforeAll(async () => {
const app = await Test.createTestingModule({
imports: [],
providers: [YahooFinanceService]
}).compile();
service = app.get<YahooFinanceService>(YahooFinanceService);
});
describe('get', () => {
it('should return data for USDCHF', () => {
expect(service.get(['USDCHF'])).toEqual('{}');
});
});
});
*/

47
apps/api/src/services/data-provider/yahoo-finance/yahoo-finance.service.ts
View File

@ -1,8 +1,10 @@
import { LookupItem } from '@ghostfolio/api/app/symbol/interfaces/lookup-item.interface';
import { UNKNOWN_KEY } from '@ghostfolio/common/config';
import { isCrypto, isCurrency, parseCurrency } from '@ghostfolio/common/helper';
import { Granularity } from '@ghostfolio/common/types';
import { Injectable } from '@nestjs/common';
import { DataSource } from '@prisma/client';
import * as bent from 'bent';
import { format } from 'date-fns';
import * as yahooFinance from 'yahoo-finance';
@ -22,6 +24,8 @@ import {
@Injectable()
export class YahooFinanceService implements DataProviderInterface {
private yahooFinanceHostname = 'https://query1.finance.yahoo.com';
public constructor() {}
public async get(
@ -136,6 +140,49 @@ export class YahooFinanceService implements DataProviderInterface {
}
}
public async search(aSymbol: string): Promise<{ items: LookupItem[] }> {
let items = [];
try {
const get = bent(
`${this.yahooFinanceHostname}/v1/finance/search?q=${aSymbol}&lang=en-US&region=US&quotesCount=8&newsCount=0&enableFuzzyQuery=false&quotesQueryId=tss_match_phrase_query&multiQuoteQueryId=multi_quote_single_token_query&newsQueryId=news_cie_vespa&enableCb=true&enableNavLinks=false&enableEnhancedTrivialQuery=true`,
'GET',
'json',
200
);
const result = await get();
items = result.quotes
.filter((quote) => {
return quote.isYahooFinance;
})
.filter(({ quoteType }) => {
return (
quoteType === 'CRYPTOCURRENCY' ||
quoteType === 'EQUITY' ||
quoteType === 'ETF'
);
})
.filter(({ quoteType, symbol }) => {
if (quoteType === 'CRYPTOCURRENCY') {
// Only allow cryptocurrencies in USD
return symbol.includes('USD');
}
return true;
})
.map(({ longname, shortname, symbol }) => {
return {
dataSource: DataSource.YAHOO,
name: longname || shortname,
symbol: convertFromYahooSymbol(symbol)
};
});
} catch {}
return { items };
}
/**
* Converts a symbol to a Yahoo symbol
*

3
apps/api/src/services/interfaces/data-provider.interface.ts
View File

@ -1,3 +1,4 @@
import { LookupItem } from '@ghostfolio/api/app/symbol/interfaces/lookup-item.interface';
import { Granularity } from '@ghostfolio/common/types';
import {
@ -16,4 +17,6 @@ export interface DataProviderInterface {
): Promise<{
[symbol: string]: { [date: string]: IDataProviderHistoricalResponse };
}>;
search(aSymbol: string): Promise<{ items: LookupItem[] }>;
}

1
apps/api/src/services/interfaces/environment.interface.ts
View File

@ -4,6 +4,7 @@ export interface Environment extends CleanedEnvAccessors {
ACCESS_TOKEN_SALT: string;
ALPHA_VANTAGE_API_KEY: string;
CACHE_TTL: number;
DATA_SOURCES: string | string[]; // string is not correct, error in envalid?
ENABLE_FEATURE_CUSTOM_SYMBOLS: boolean;
ENABLE_FEATURE_FEAR_AND_GREED_INDEX: boolean;
ENABLE_FEATURE_SOCIAL_LOGIN: boolean;

8
apps/client/src/app/services/data.service.ts
View File

@ -102,7 +102,13 @@ export class DataService {
}
public fetchSymbols(aQuery: string) {
return this.http.get<LookupItem[]>(`/api/symbol/lookup?query=${aQuery}`);
return this.http
.get<{ items: LookupItem[] }>(`/api/symbol/lookup?query=${aQuery}`)
.pipe(
map((respose) => {
return respose.items;
})
);
}
public fetchOrders(): Observable<OrderModel[]> {

1
prisma/schema.prisma
View File

@ -127,6 +127,7 @@ enum Currency {
}
enum DataSource {
ALPHA_VANTAGE
GHOSTFOLIO
RAKUTEN
YAHOO