sudacode/ghostfolio
1
0
Fork 0
Code Issues Pull Requests Actions Packages Releases Activity

Add scraper symbols to symbol lookup results (#51)

Browse Source
This commit is contained in:
Thomas 2021-04-27 21:03:22 +02:00 committed by GitHub
parent cd9e974c40
commit 94e0feac68
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 89 additions and 53 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
CHANGELOG.md
View File

@ -5,6 +5,12 @@ All notable changes to this project will be documented in this file.
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
## Unreleased
### Added
- Added the generic scraper symbols to the symbol lookup results
## 0.93.0 - 26.04.2021
### Changed

31
apps/api/src/app/symbol/symbol.service.ts
View File

@ -1,4 +1,6 @@
import { DataGatheringService } from '@ghostfolio/api/services/data-gathering.service';
import { DataProviderService } from '@ghostfolio/api/services/data-provider.service';
import { GhostfolioScraperApiService } from '@ghostfolio/api/services/data-provider/ghostfolio-scraper-api/ghostfolio-scraper-api.service';
import { convertFromYahooSymbol } from '@ghostfolio/api/services/data-provider/yahoo-finance/yahoo-finance.service';
import { Injectable } from '@nestjs/common';
import { Currency } from '@prisma/client';
@ -10,7 +12,8 @@ import { SymbolItem } from './interfaces/symbol-item.interface';
@Injectable()
export class SymbolService {
public constructor(
private readonly dataProviderService: DataProviderService
private readonly dataProviderService: DataProviderService,
private readonly ghostfolioScraperApiService: GhostfolioScraperApiService
) {}
public async get(aSymbol: string): Promise<SymbolItem> {
@ -23,18 +26,36 @@ export class SymbolService {
};
}
public async lookup(aQuery: string): Promise<LookupItem[]> {
public async lookup(aQuery = ''): Promise<LookupItem[]> {
const query = aQuery.toLowerCase();
const results: LookupItem[] = [];
if (!query) {
return results;
}
const get = bent(
`https://query1.finance.yahoo.com/v1/finance/search?q=${aQuery}&lang=en-US&region=US&quotesCount=8&newsCount=0&enableFuzzyQuery=false&quotesQueryId=tss_match_phrase_query&multiQuoteQueryId=multi_quote_single_token_query&newsQueryId=news_cie_vespa&enableCb=true&enableNavLinks=false&enableEnhancedTrivialQuery=true`,
`https://query1.finance.yahoo.com/v1/finance/search?q=${query}&lang=en-US&region=US&quotesCount=8&newsCount=0&enableFuzzyQuery=false&quotesQueryId=tss_match_phrase_query&multiQuoteQueryId=multi_quote_single_token_query&newsQueryId=news_cie_vespa&enableCb=true&enableNavLinks=false&enableEnhancedTrivialQuery=true`,
'GET',
'json',
200
);
// Add custom symbols
const scraperConfigurations = await this.ghostfolioScraperApiService.getScraperConfigurations();
scraperConfigurations.forEach((scraperConfiguration) => {
if (scraperConfiguration.name.toLowerCase().startsWith(query)) {
results.push({
name: scraperConfiguration.name,
symbol: scraperConfiguration.symbol
});
}
});
try {
const { quotes } = await get();
return quotes
const searchResult = quotes
.filter(({ isYahooFinance }) => {
return isYahooFinance;
})
@ -59,6 +80,8 @@ export class SymbolService {
symbol: convertFromYahooSymbol(symbol)
};
});
return results.concat(searchResult);
} catch (error) {
console.error(error);

39
apps/api/src/services/data-gathering.service.ts
View File

@ -18,6 +18,7 @@ import {
import { ConfigurationService } from './configuration.service';
import { DataProviderService } from './data-provider.service';
import { GhostfolioScraperApiService } from './data-provider/ghostfolio-scraper-api/ghostfolio-scraper-api.service';
import { PrismaService } from './prisma.service';
@Injectable()
@ -25,6 +26,7 @@ export class DataGatheringService {
public constructor(
private readonly configurationService: ConfigurationService,
private readonly dataProviderService: DataProviderService,
private readonly ghostfolioScraperApi: GhostfolioScraperApiService,
private prisma: PrismaService
) {}
@ -183,6 +185,17 @@ export class DataGatheringService {
}
}
public async getCustomSymbolsToGather(startDate?: Date) {
const scraperConfigurations = await this.ghostfolioScraperApi.getScraperConfigurations();
return scraperConfigurations.map((scraperConfiguration) => {
return {
date: startDate,
symbol: scraperConfiguration.symbol
};
});
}
private getBenchmarksToGather(startDate: Date) {
const benchmarksToGather = benchmarks.map((symbol) => {
return {
@ -201,32 +214,6 @@ export class DataGatheringService {
return benchmarksToGather;
}
private async getCustomSymbolsToGather(startDate: Date) {
const customSymbolsToGather = [];
if (this.configurationService.get('ENABLE_FEATURE_CUSTOM_SYMBOLS')) {
try {
const {
value: scraperConfigString
} = await this.prisma.property.findFirst({
select: {
value: true
},
where: { key: 'SCRAPER_CONFIG' }
});
JSON.parse(scraperConfigString).forEach((item) => {
customSymbolsToGather.push({
date: startDate,
symbol: item.symbol
});
});
} catch {}
}
return customSymbolsToGather;
}
private async getSymbols7D(): Promise<{ date: Date; symbol: string }[]> {
const startDate = subDays(resetHours(new Date()), 7);

50
apps/api/src/services/data-provider/ghostfolio-scraper-api/ghostfolio-scraper-api.service.ts
View File

@ -12,6 +12,7 @@ import {
MarketState
} from '../../interfaces/interfaces';
import { PrismaService } from '../../prisma.service';
import { ScraperConfig } from './interfaces/scraper-config.interface';
@Injectable()
export class GhostfolioScraperApiService implements DataProviderInterface {
@ -29,7 +30,7 @@ export class GhostfolioScraperApiService implements DataProviderInterface {
try {
const symbol = aSymbols[0];
const scraperConfig = await this.getScraperConfig(symbol);
const scraperConfig = await this.getScraperConfigurationBySymbol(symbol);
const { marketPrice } = await this.prisma.marketData.findFirst({
orderBy: {
@ -70,15 +71,17 @@ export class GhostfolioScraperApiService implements DataProviderInterface {
try {
const symbol = aSymbols[0];
const scraperConfig = await this.getScraperConfig(symbol);
const scraperConfiguration = await this.getScraperConfigurationBySymbol(
symbol
);
const get = bent(scraperConfig?.url, 'GET', 'string', 200, {});
const get = bent(scraperConfiguration?.url, 'GET', 'string', 200, {});
const html = await get();
const $ = cheerio.load(html);
const value = this.extractNumberFromString(
$(scraperConfig?.selector).text()
$(scraperConfiguration?.selector).text()
);
return {
@ -95,6 +98,23 @@ export class GhostfolioScraperApiService implements DataProviderInterface {
return {};
}
public async getScraperConfigurations(): Promise<ScraperConfig[]> {
try {
const {
value: scraperConfigString
} = await this.prisma.property.findFirst({
select: {
value: true
},
where: { key: 'SCRAPER_CONFIG' }
});
return JSON.parse(scraperConfigString);
} catch {}
return [];
}
private extractNumberFromString(aString: string): number {
try {
const [numberString] = aString.match(
@ -106,22 +126,10 @@ export class GhostfolioScraperApiService implements DataProviderInterface {
}
}
private async getScraperConfig(aSymbol: string) {
try {
const {
value: scraperConfigString
} = await this.prisma.property.findFirst({
select: {
value: true
},
where: { key: 'SCRAPER_CONFIG' }
});
return JSON.parse(scraperConfigString).find((item) => {
return item.symbol === aSymbol;
});
} catch {}
return {};
private async getScraperConfigurationBySymbol(aSymbol: string) {
const scraperConfigurations = await this.getScraperConfigurations();
return scraperConfigurations.find((scraperConfiguration) => {
return scraperConfiguration.symbol === aSymbol;
});
}
}

9
apps/api/src/services/data-provider/ghostfolio-scraper-api/interfaces/scraper-config.interface.ts Normal file
View File

@ -0,0 +1,9 @@
import { Currency } from '.prisma/client';
export interface ScraperConfig {
currency: Currency;
name: string;
selector: string;
symbol: string;
url: string;
}

2
apps/client/src/app/pages/transactions/create-or-update-transaction-dialog/create-or-update-transaction-dialog.html
View File

@ -25,7 +25,7 @@
class="autocomplete"
[value]="lookupItem.symbol"
>
<span class="mr-2 symbol">{{ lookupItem.symbol }}</span
<span class="mr-2 symbol">{{ lookupItem.symbol | gfSymbol }}</span
><span><b>{{ lookupItem.name }}</b></span>
</mat-option>
</ng-container>

2
apps/client/src/app/pages/transactions/create-or-update-transaction-dialog/create-or-update-transaction-dialog.module.ts
View File

@ -9,6 +9,7 @@ import { MatFormFieldModule } from '@angular/material/form-field';
import { MatInputModule } from '@angular/material/input';
import { MatProgressSpinnerModule } from '@angular/material/progress-spinner';
import { MatSelectModule } from '@angular/material/select';
import { GfSymbolModule } from '@ghostfolio/client/pipes/symbol/symbol.module';
import { CreateOrUpdateTransactionDialog } from './create-or-update-transaction-dialog.component';
@ -17,6 +18,7 @@ import { CreateOrUpdateTransactionDialog } from './create-or-update-transaction-
exports: [],
imports: [
CommonModule,
GfSymbolModule,
FormsModule,
MatAutocompleteModule,
MatButtonModule,

3
apps/client/src/app/pages/transactions/create-or-update-transaction-dialog/interfaces/interfaces.ts
View File

@ -1,6 +1,7 @@
import { Order } from '../../interfaces/order.interface';
import { Account } from '@prisma/client';
import { Order } from '../../interfaces/order.interface';
export interface CreateOrUpdateTransactionDialogParams {
accountId: string;
accounts: Account[];