sudacode/ghostfolio
1
0
Fork 0
Code Issues Pull Requests Actions Packages Releases Activity

Feature/add data gathering for symbol profile data (#228)

Browse Source 
* Implement profile data gathering

* Update changelog
This commit is contained in:
Thomas 2021-07-24 21:13:48 +02:00 committed by GitHub
parent be8d60968d
commit 6996e5a140
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
11 changed files with 171 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
CHANGELOG.md
View File

@ -7,6 +7,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
## Unreleased
### Added
- Extended the data management by symbol profile data
- Added a currency attribute to the symbol profile model
### Changed
- Improved the style of the active page in the navigation on desktop

21
apps/api/src/app/admin/admin.controller.ts
View File

@ -61,8 +61,29 @@ export class AdminController {
);
}
await this.dataGatheringService.gatherProfileData();
this.dataGatheringService.gatherMax();
return;
}
@Post('gather/profile-data')
@UseGuards(AuthGuard('jwt'))
public async gatherProfileData(): Promise<void> {
if (
!hasPermission(
getPermissions(this.request.user.role),
permissions.accessAdminControl
)
) {
throw new HttpException(
getReasonPhrase(StatusCodes.FORBIDDEN),
StatusCodes.FORBIDDEN
);
}
this.dataGatheringService.gatherProfileData();
return;
}
}

2
apps/api/src/app/order/order.service.ts
View File

@ -62,6 +62,8 @@ export class OrderService {
]);
}
this.dataGatheringService.gatherProfileData([data.symbol]);
await this.cacheService.flush(aUserId);
return this.prisma.order.create({

1
apps/api/src/services/cron.service.ts
View File

@ -18,6 +18,7 @@ export class CronService {
@Cron(CronExpression.EVERY_12_HOURS)
public async runEveryTwelveHours() {
await this.dataGatheringService.gatherProfileData();
await this.exchangeRateDataService.loadCurrencies();
}
}

74
apps/api/src/services/data-gathering.service.ts
View File

@ -2,6 +2,7 @@ import { benchmarks, currencyPairs } from '@ghostfolio/common/config';
import {
getUtc,
isGhostfolioScraperApiSymbol,
isRakutenRapidApiSymbol,
resetHours
} from '@ghostfolio/common/helper';
import { Injectable } from '@nestjs/common';
@ -37,7 +38,7 @@ export class DataGatheringService {
if (isDataGatheringNeeded) {
console.log('7d data gathering has been started.');
console.time('data-gathering');
console.time('7d-data-gathering');
await this.prisma.property.create({
data: {
@ -70,7 +71,7 @@ export class DataGatheringService {
});
console.log('7d data gathering has been completed.');
console.timeEnd('data-gathering');
console.timeEnd('7d-data-gathering');
}
}
@ -81,7 +82,7 @@ export class DataGatheringService {
if (!isDataGatheringLocked) {
console.log('Max data gathering has been started.');
console.time('data-gathering');
console.time('max-data-gathering');
await this.prisma.property.create({
data: {
@ -114,10 +115,56 @@ export class DataGatheringService {
});
console.log('Max data gathering has been completed.');
console.timeEnd('data-gathering');
console.timeEnd('max-data-gathering');
}
}
public async gatherProfileData(aSymbols?: string[]) {
console.log('Profile data gathering has been started.');
console.time('profile-data-gathering');
let symbols = aSymbols;
if (!symbols) {
const dataGatheringItems = await this.getSymbolsProfileData();
symbols = dataGatheringItems.map((dataGatheringItem) => {
return dataGatheringItem.symbol;
});
}
const currentData = await this.dataProviderService.get(symbols);
for (const [symbol, { currency, dataSource, name }] of Object.entries(
currentData
)) {
try {
await this.prisma.symbolProfile.upsert({
create: {
currency,
dataSource,
name,
symbol
},
update: {
currency,
name
},
where: {
dataSource_symbol: {
dataSource,
symbol
}
}
});
} catch (error) {
console.error(`${symbol}: ${error?.meta?.cause}`);
}
}
console.log('Profile data gathering has been completed.');
console.timeEnd('profile-data-gathering');
}
public async gatherSymbols(aSymbolsWithStartDate: IDataGatheringItem[]) {
let hasError = false;
@ -303,6 +350,25 @@ export class DataGatheringService {
];
}
private async getSymbolsProfileData(): Promise<IDataGatheringItem[]> {
const startDate = subDays(resetHours(new Date()), 7);
const distinctOrders = await this.prisma.order.findMany({
distinct: ['symbol'],
orderBy: [{ symbol: 'asc' }],
select: { dataSource: true, symbol: true }
});
return [...this.getBenchmarksToGather(startDate), ...distinctOrders].filter(
(distinctOrder) => {
return (
distinctOrder.dataSource !== DataSource.GHOSTFOLIO &&
distinctOrder.dataSource !== DataSource.RAKUTEN
);
}
);
}
private async isDataGatheringNeeded() {
const lastDataGathering = await this.prisma.property.findUnique({
where: { key: 'LAST_DATA_GATHERING' }

22
apps/api/src/services/data-provider.service.ts
View File

@ -46,7 +46,10 @@ export class DataProviderService {
}
const yahooFinanceSymbols = aSymbols.filter((symbol) => {
return !isGhostfolioScraperApiSymbol(symbol);
return (
!isGhostfolioScraperApiSymbol(symbol) &&
!isRakutenRapidApiSymbol(symbol)
);
});
const response = await this.yahooFinanceService.get(yahooFinanceSymbols);
@ -57,13 +60,24 @@ export class DataProviderService {
for (const symbol of ghostfolioScraperApiSymbols) {
if (symbol) {
const ghostfolioScraperApiResult = await this.ghostfolioScraperApiService.get(
[symbol]
);
const ghostfolioScraperApiResult =
await this.ghostfolioScraperApiService.get([symbol]);
response[symbol] = ghostfolioScraperApiResult[symbol];
}
}
const rakutenRapidApiSymbols = aSymbols.filter((symbol) => {
return isRakutenRapidApiSymbol(symbol);
});
for (const symbol of rakutenRapidApiSymbols) {
if (symbol) {
const rakutenRapidApiResult =
await this.ghostfolioScraperApiService.get([symbol]);
response[symbol] = rakutenRapidApiResult[symbol];
}
}
return response;
}

7
apps/client/src/app/pages/admin/admin-page.component.ts
View File

@ -85,6 +85,13 @@ export class AdminPageComponent implements OnDestroy, OnInit {
}
}
public onGatherProfileData() {
this.adminService
.gatherProfileData()
.pipe(takeUntil(this.unsubscribeSubject))
.subscribe(() => {});
}
public formatDistanceToNow(aDateString: string) {
if (aDateString) {
const distanceString = formatDistanceToNowStrict(parseISO(aDateString), {

25
apps/client/src/app/pages/admin/admin-page.html
View File

@ -27,15 +27,21 @@
>
</div>
<div class="mt-2 overflow-hidden">
<div class="mb-2">
<button
class="mb-2 mr-2 mw-100"
class="mw-100"
color="accent"
mat-flat-button
(click)="onFlushCache()"
>
<ion-icon class="mr-1" name="close-circle-outline"></ion-icon>
<ion-icon
class="mr-1"
name="close-circle-outline"
></ion-icon>
<span i18n>Reset Data Gathering</span>
</button>
</div>
<div class="mb-2">
<button
class="mw-100"
color="warn"
@ -47,6 +53,21 @@
<span i18n>Gather All Data</span>
</button>
</div>
<div>
<button
class="mb-2 mr-2 mw-100"
color="accent"
mat-flat-button
(click)="onGatherProfileData()"
>
<ion-icon
class="mr-1"
name="cloud-download-outline"
></ion-icon>
<span i18n>Gather Profile Data</span>
</button>
</div>
</div>
</div>
</div>
<div class="d-flex my-3">

4
apps/client/src/app/services/admin.service.ts
View File

@ -10,4 +10,8 @@ export class AdminService {
public gatherMax() {
return this.http.post<void>(`/api/admin/gather/max`, {});
}
public gatherProfileData() {
return this.http.post<void>(`/api/admin/gather/profile-data`, {});
}
}

2
prisma/migrations/20210724160404_added_currency_to_symbol_profile/migration.sql Normal file
View File

@ -0,0 +1,2 @@
-- AlterTable
ALTER TABLE "SymbolProfile" ADD COLUMN "currency" "Currency";

1
prisma/schema.prisma
View File

@ -117,6 +117,7 @@ model Settings {
model SymbolProfile {
countries Json?
createdAt DateTime @default(now())
currency Currency?
dataSource DataSource
id String @id @default(uuid())
name String?