sudacode/ghostfolio
1
0
Fork 0
Code Issues Pull Requests Actions Packages Releases Activity

Improve scraper (#28)

Browse Source
This commit is contained in:
Thomas 2021-04-20 21:52:01 +02:00 committed by GitHub
parent 3e3395aff9
commit 069006145a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
11 changed files with 210 additions and 165 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
CHANGELOG.md
View File

@ -5,6 +5,16 @@ All notable changes to this project will be documented in this file.
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
## Unreleased
### Changed
- Reverted the restoring of the scroll position when opening a new page
### Fixed
- Fixed some issues in the generic scraper
## 0.87.0 - 19.04.2021
### Added

94
apps/api/src/app/portfolio/portfolio.service.ts
View File

@ -48,53 +48,6 @@ export class PortfolioService {
private readonly userService: UserService
) {}
private convertDateRangeToDate(aDateRange: DateRange, aMinDate: Date) {
let currentDate = new Date();
const normalizedMinDate =
getDate(aMinDate) === 1
? aMinDate
: add(setDate(aMinDate, 1), { months: 1 });
const year = getYear(currentDate);
const month = getMonth(currentDate);
const day = getDate(currentDate);
currentDate = new Date(Date.UTC(year, month, day, 0));
switch (aDateRange) {
case '1d':
return sub(currentDate, {
days: 1
});
case 'ytd':
currentDate = setDate(currentDate, 1);
currentDate = setMonth(currentDate, 0);
return isAfter(currentDate, normalizedMinDate)
? currentDate
: undefined;
case '1y':
currentDate = setDate(currentDate, 1);
currentDate = sub(currentDate, {
years: 1
});
return isAfter(currentDate, normalizedMinDate)
? currentDate
: undefined;
case '5y':
currentDate = setDate(currentDate, 1);
currentDate = sub(currentDate, {
years: 5
});
return isAfter(currentDate, normalizedMinDate)
? currentDate
: undefined;
default:
// Gets handled as all data
return undefined;
}
}
public async createPortfolio(aUserId: string): Promise<Portfolio> {
let portfolio: Portfolio;
let stringifiedPortfolio = await this.redisCacheService.get(
@ -382,4 +335,51 @@ export class PortfolioService {
symbol: aSymbol
};
}
private convertDateRangeToDate(aDateRange: DateRange, aMinDate: Date) {
let currentDate = new Date();
const normalizedMinDate =
getDate(aMinDate) === 1
? aMinDate
: add(setDate(aMinDate, 1), { months: 1 });
const year = getYear(currentDate);
const month = getMonth(currentDate);
const day = getDate(currentDate);
currentDate = new Date(Date.UTC(year, month, day, 0));
switch (aDateRange) {
case '1d':
return sub(currentDate, {
days: 1
});
case 'ytd':
currentDate = setDate(currentDate, 1);
currentDate = setMonth(currentDate, 0);
return isAfter(currentDate, normalizedMinDate)
? currentDate
: undefined;
case '1y':
currentDate = setDate(currentDate, 1);
currentDate = sub(currentDate, {
years: 1
});
return isAfter(currentDate, normalizedMinDate)
? currentDate
: undefined;
case '5y':
currentDate = setDate(currentDate, 1);
currentDate = sub(currentDate, {
years: 5
});
return isAfter(currentDate, normalizedMinDate)
? currentDate
: undefined;
default:
// Gets handled as all data
return undefined;
}
}
}

28
apps/api/src/app/user/user.service.ts
View File

@ -44,10 +44,6 @@ export class UserService {
currentPermissions.push(permissions.accessFearAndGreedIndex);
}
if (this.configurationService.get('ENABLE_FEATURE_SOCIAL_LOGIN')) {
currentPermissions.push(permissions.useSocialLogin);
}
return {
alias,
id,
@ -162,18 +158,6 @@ export class UserService {
});
}
private getRandomString(length: number) {
const characters = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789';
const result = [];
for (let i = 0; i < length; i++) {
result.push(
characters.charAt(Math.floor(Math.random() * characters.length))
);
}
return result.join('');
}
public async updateUserSettings({
currency,
userId
@ -200,4 +184,16 @@ export class UserService {
return;
}
private getRandomString(length: number) {
const characters = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789';
const result = [];
for (let i = 0; i < length; i++) {
result.push(
characters.charAt(Math.floor(Math.random() * characters.length))
);
}
return result.join('');
}
}

104
apps/api/src/models/portfolio.ts
View File

@ -79,7 +79,9 @@ export class Portfolio implements PortfolioInterface {
investmentInOriginalCurrency:
portfolioItemsYesterday?.positions[symbol]
?.investmentInOriginalCurrency,
marketPrice: currentData[symbol]?.marketPrice,
marketPrice:
currentData[symbol]?.marketPrice ??
portfolioItemsYesterday.positions[symbol]?.marketPrice,
quantity: portfolioItemsYesterday?.positions[symbol]?.quantity
};
});
@ -158,53 +160,6 @@ export class Portfolio implements PortfolioInterface {
return this;
}
private convertDateRangeToDate(aDateRange: DateRange, aMinDate: Date) {
let currentDate = new Date();
const normalizedMinDate =
getDate(aMinDate) === 1
? aMinDate
: add(setDate(aMinDate, 1), { months: 1 });
const year = getYear(currentDate);
const month = getMonth(currentDate);
const day = getDate(currentDate);
currentDate = new Date(Date.UTC(year, month, day, 0));
switch (aDateRange) {
case '1d':
return sub(currentDate, {
days: 1
});
case 'ytd':
currentDate = setDate(currentDate, 1);
currentDate = setMonth(currentDate, 0);
return isAfter(currentDate, normalizedMinDate)
? currentDate
: undefined;
case '1y':
currentDate = setDate(currentDate, 1);
currentDate = sub(currentDate, {
years: 1
});
return isAfter(currentDate, normalizedMinDate)
? currentDate
: undefined;
case '5y':
currentDate = setDate(currentDate, 1);
currentDate = sub(currentDate, {
years: 5
});
return isAfter(currentDate, normalizedMinDate)
? currentDate
: undefined;
default:
// Gets handled as all data
return undefined;
}
}
public get(aDate?: Date): PortfolioItem[] {
if (aDate) {
const filteredPortfolio = this.portfolioItems.find((item) => {
@ -528,12 +483,6 @@ export class Portfolio implements PortfolioInterface {
return this.orders;
}
private getOrdersByType(aFilter: string[]) {
return this.orders.filter((order) => {
return aFilter.includes(order.getType());
});
}
public getValue(aDate = getToday()) {
const positions = this.getPositions(aDate);
let value = 0;
@ -692,6 +641,53 @@ export class Portfolio implements PortfolioInterface {
this.updatePortfolioItems();
}
private convertDateRangeToDate(aDateRange: DateRange, aMinDate: Date) {
let currentDate = new Date();
const normalizedMinDate =
getDate(aMinDate) === 1
? aMinDate
: add(setDate(aMinDate, 1), { months: 1 });
const year = getYear(currentDate);
const month = getMonth(currentDate);
const day = getDate(currentDate);
currentDate = new Date(Date.UTC(year, month, day, 0));
switch (aDateRange) {
case '1d':
return sub(currentDate, {
days: 1
});
case 'ytd':
currentDate = setDate(currentDate, 1);
currentDate = setMonth(currentDate, 0);
return isAfter(currentDate, normalizedMinDate)
? currentDate
: undefined;
case '1y':
currentDate = setDate(currentDate, 1);
currentDate = sub(currentDate, {
years: 1
});
return isAfter(currentDate, normalizedMinDate)
? currentDate
: undefined;
case '5y':
currentDate = setDate(currentDate, 1);
currentDate = sub(currentDate, {
years: 5
});
return isAfter(currentDate, normalizedMinDate)
? currentDate
: undefined;
default:
// Gets handled as all data
return undefined;
}
}
private updatePortfolioItems() {
// console.time('update-portfolio-items');

7
apps/api/src/services/data-gathering.service.ts
View File

@ -2,6 +2,7 @@ import {
benchmarks,
currencyPairs,
getUtc,
isGhostfolioScraperApiSymbol,
resetHours
} from '@ghostfolio/helper';
import { Injectable } from '@nestjs/common';
@ -235,7 +236,11 @@ export class DataGatheringService {
select: { symbol: true }
});
const distinctOrdersWithDate = distinctOrders.map((distinctOrder) => {
const distinctOrdersWithDate = distinctOrders
.filter((distinctOrder) => {
return !isGhostfolioScraperApiSymbol(distinctOrder.symbol);
})
.map((distinctOrder) => {
return {
...distinctOrder,
date: startDate

39
apps/api/src/services/data-provider.service.ts
View File

@ -1,7 +1,7 @@
import {
isCrypto,
isGhostfolioScraperApi,
isRakutenRapidApi
isGhostfolioScraperApiSymbol,
isRakutenRapidApiSymbol
} from '@ghostfolio/helper';
import { Injectable } from '@nestjs/common';
import { MarketData } from '@prisma/client';
@ -39,14 +39,33 @@ export class DataProviderService implements DataProviderInterface {
if (aSymbols.length === 1) {
const symbol = aSymbols[0];
if (isGhostfolioScraperApi(symbol)) {
if (isGhostfolioScraperApiSymbol(symbol)) {
return this.ghostfolioScraperApiService.get(aSymbols);
} else if (isRakutenRapidApi(symbol)) {
} else if (isRakutenRapidApiSymbol(symbol)) {
return this.rakutenRapidApiService.get(aSymbols);
}
}
return this.yahooFinanceService.get(aSymbols);
const yahooFinanceSymbols = aSymbols.filter((symbol) => {
return !isGhostfolioScraperApiSymbol(symbol);
});
const response = await this.yahooFinanceService.get(yahooFinanceSymbols);
const ghostfolioScraperApiSymbols = aSymbols.filter((symbol) => {
return isGhostfolioScraperApiSymbol(symbol);
});
for (const symbol of ghostfolioScraperApiSymbols) {
if (symbol) {
const ghostfolioScraperApiResult = await this.ghostfolioScraperApiService.get(
[symbol]
);
response[symbol] = ghostfolioScraperApiResult[symbol];
}
}
return response;
}
public async getHistorical(
@ -107,8 +126,12 @@ export class DataProviderService implements DataProviderInterface {
): Promise<{
[symbol: string]: { [date: string]: IDataProviderHistoricalResponse };
}> {
const filteredSymbols = aSymbols.filter((symbol) => {
return !isGhostfolioScraperApiSymbol(symbol);
});
const dataOfYahoo = await this.yahooFinanceService.getHistorical(
aSymbols,
filteredSymbols,
undefined,
from,
to
@ -135,7 +158,7 @@ export class DataProviderService implements DataProviderInterface {
...dataOfAlphaVantage[symbol]
}
};
} else if (isGhostfolioScraperApi(symbol)) {
} else if (isGhostfolioScraperApiSymbol(symbol)) {
const dataOfGhostfolioScraperApi = await this.ghostfolioScraperApiService.getHistorical(
[symbol],
undefined,
@ -145,7 +168,7 @@ export class DataProviderService implements DataProviderInterface {
return dataOfGhostfolioScraperApi;
} else if (
isRakutenRapidApi(symbol) &&
isRakutenRapidApiSymbol(symbol) &&
this.configurationService.get('RAKUTEN_RAPID_API_KEY')
) {
const dataOfRakutenRapidApi = await this.rakutenRapidApiService.getHistorical(

49
apps/api/src/services/data-provider/ghostfolio-scraper-api/ghostfolio-scraper-api.service.ts
View File

@ -11,7 +11,6 @@ import {
IDataProviderResponse
} from '../../interfaces/interfaces';
import { PrismaService } from '../../prisma.service';
import { Currency } from '.prisma/client';
@Injectable()
export class GhostfolioScraperApiService implements DataProviderInterface {
@ -26,6 +25,9 @@ export class GhostfolioScraperApiService implements DataProviderInterface {
try {
const symbol = aSymbols[0];
const scraperConfig = await this.getScraperConfig(symbol);
const { marketPrice } = await this.prisma.marketData.findFirst({
orderBy: {
date: 'desc'
@ -38,9 +40,9 @@ export class GhostfolioScraperApiService implements DataProviderInterface {
return {
[symbol]: {
marketPrice,
currency: Currency.CHF,
isMarketOpen: true,
name: symbol
currency: scraperConfig?.currency,
isMarketOpen: false,
name: scraperConfig?.name
}
};
} catch (error) {
@ -65,25 +67,17 @@ export class GhostfolioScraperApiService implements DataProviderInterface {
try {
const symbol = aSymbols[0];
const {
value: scraperConfigString
} = await this.prisma.property.findFirst({
select: {
value: true
},
where: { key: 'SCRAPER_CONFIG' }
});
const scraperConfig = await this.getScraperConfig(symbol);
const scraperConfig = JSON.parse(scraperConfigString).find((item) => {
return item.symbol === symbol;
});
const get = bent(scraperConfig.url, 'GET', 'string', 200, {});
const get = bent(scraperConfig?.url, 'GET', 'string', 200, {});
const html = await get();
const $ = cheerio.load(html);
const string = $(scraperConfig.selector).text().replace('CHF', '').trim();
const string = $(scraperConfig?.selector)
.text()
.replace('CHF', '')
.trim();
const value = parseFloat(string);
@ -100,4 +94,23 @@ export class GhostfolioScraperApiService implements DataProviderInterface {
return {};
}
private async getScraperConfig(aSymbol: string) {
try {
const {
value: scraperConfigString
} = await this.prisma.property.findFirst({
select: {
value: true
},
where: { key: 'SCRAPER_CONFIG' }
});
return JSON.parse(scraperConfigString).find((item) => {
return item.symbol === aSymbol;
});
} catch {}
return {};
}
}

3
apps/client/src/app/app-routing.module.ts
View File

@ -81,8 +81,7 @@ const routes: Routes = [
{
preloadingStrategy: ModulePreloadService,
// enableTracing: true // <-- debugging purposes only
relativeLinkResolution: 'legacy',
scrollPositionRestoration: 'enabled'
relativeLinkResolution: 'legacy'
}
)
],

4
apps/client/src/app/components/position/position.component.html
View File

@ -40,7 +40,9 @@
class="ml-1"
[url]="position?.url"
></gf-symbol-icon>
<span class="ml-2 text-muted">({{ position?.exchange }})</span>
<span *ngIf="position?.exchange" class="ml-2 text-muted"
>({{ position.exchange }})</span
>
</div>
<div class="d-flex mt-1">
<gf-value

23
apps/client/src/app/pages/home/home-page.component.ts
View File

@ -85,6 +85,18 @@ export class HomePageComponent implements OnDestroy, OnInit {
user.permissions,
permissions.accessFearAndGreedIndex
);
if (this.hasPermissionToAccessFearAndGreedIndex) {
this.dataService
.fetchSymbolItem('GF.FEAR_AND_GREED_INDEX')
.pipe(takeUntil(this.unsubscribeSubject))
.subscribe(({ marketPrice }) => {
this.fearAndGreedIndex = marketPrice;
this.cd.markForCheck();
});
}
this.hasPermissionToReadForeignPortfolio = hasPermission(
user.permissions,
permissions.readForeignPortfolio
@ -180,17 +192,6 @@ export class HomePageComponent implements OnDestroy, OnInit {
this.cd.markForCheck();
});
if (this.hasPermissionToAccessFearAndGreedIndex) {
this.dataService
.fetchSymbolItem('GF.FEAR_AND_GREED_INDEX')
.pipe(takeUntil(this.unsubscribeSubject))
.subscribe(({ marketPrice }) => {
this.fearAndGreedIndex = marketPrice;
this.cd.markForCheck();
});
}
this.cd.markForCheck();
}

4
libs/helper/src/lib/helper.ts
View File

@ -66,11 +66,11 @@ export function isCurrency(aSymbol = '') {
);
}
export function isGhostfolioScraperApi(aSymbol = '') {
export function isGhostfolioScraperApiSymbol(aSymbol = '') {
return aSymbol.startsWith('[GF]');
}
export function isRakutenRapidApi(aSymbol = '') {
export function isRakutenRapidApiSymbol(aSymbol = '') {
return aSymbol === 'GF.FEAR_AND_GREED_INDEX';
}