sudacode/ghostfolio
1
0
Fork 0
Code Issues Pull Requests Actions Packages Releases Activity

Feature/improve check for duplicates in import #1932 (#1940)

Browse Source 
* Improve check for duplicates in import

* Update changelog
This commit is contained in:
Visrut 2023-05-09 01:15:59 +05:30 committed by GitHub
parent b923cf7752
commit 7d6a74a67d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 129 additions and 59 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
CHANGELOG.md
View File

@ -5,6 +5,12 @@ All notable changes to this project will be documented in this file.
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
## Unreleased
### Changed
- Improved the preview step of the activities import by unchecking duplicates
## 1.267.0 - 2023-05-07
### Added

169
apps/api/src/app/import/import.service.ts
View File

@ -84,6 +84,7 @@ export class ImportService {
feeInBaseCurrency: 0,
id: assetProfile.id,
isDraft: false,
isDuplicate: false, // TODO: Use evaluated state
SymbolProfile: <SymbolProfile>(<unknown>assetProfile),
symbolProfileId: assetProfile.id,
type: 'DIVIDEND',
@ -204,9 +205,14 @@ export class ImportService {
userId
});
const activitiesMarkedAsDuplicates = await this.markActivitiesAsDuplicates({
activitiesDto,
userId
});
const accounts = (await this.accountService.getAccounts(userId)).map(
(account) => {
return { id: account.id, name: account.name };
({ id, name }) => {
return { id, name };
}
);
@ -221,16 +227,14 @@ export class ImportService {
for (const {
accountId,
comment,
currency,
dataSource,
date: dateString,
date,
fee,
isDuplicate,
quantity,
symbol,
SymbolProfile: assetProfile,
type,
unitPrice
} of activitiesDto) {
const date = parseISO(<string>(<unknown>dateString));
} of activitiesMarkedAsDuplicates) {
const validatedAccount = accounts.find(({ id }) => {
return id === accountId;
});
@ -256,29 +260,33 @@ export class ImportService {
id: uuidv4(),
isDraft: isAfter(date, endOfToday()),
SymbolProfile: {
currency,
dataSource,
symbol,
assetClass: null,
assetSubClass: null,
comment: null,
countries: null,
createdAt: undefined,
id: undefined,
isin: null,
name: null,
scraperConfiguration: null,
sectors: null,
symbolMapping: null,
updatedAt: undefined,
url: null,
...assetProfiles[symbol]
assetClass: assetProfile.assetClass,
assetSubClass: assetProfile.assetSubClass,
comment: assetProfile.comment,
countries: assetProfile.countries,
createdAt: assetProfile.createdAt,
currency: assetProfile.currency,
dataSource: assetProfile.dataSource,
id: assetProfile.id,
isin: assetProfile.isin,
name: assetProfile.name,
scraperConfiguration: assetProfile.scraperConfiguration,
sectors: assetProfile.sectors,
symbol: assetProfile.currency,
symbolMapping: assetProfile.symbolMapping,
updatedAt: assetProfile.updatedAt,
url: assetProfile.url,
...assetProfiles[assetProfile.symbol]
},
Account: validatedAccount,
symbolProfileId: undefined,
updatedAt: new Date()
};
} else {
if (isDuplicate) {
continue;
}
order = await this.orderService.createOrder({
comment,
date,
@ -291,14 +299,14 @@ export class ImportService {
SymbolProfile: {
connectOrCreate: {
create: {
currency,
dataSource,
symbol
currency: assetProfile.currency,
dataSource: assetProfile.dataSource,
symbol: assetProfile.symbol
},
where: {
dataSource_symbol: {
dataSource,
symbol
dataSource: assetProfile.dataSource,
symbol: assetProfile.symbol
}
}
}
@ -313,15 +321,16 @@ export class ImportService {
//@ts-ignore
activities.push({
...order,
isDuplicate,
value,
feeInBaseCurrency: this.exchangeRateDataService.toCurrency(
fee,
currency,
assetProfile.currency,
userCurrency
),
valueInBaseCurrency: this.exchangeRateDataService.toCurrency(
value,
currency,
assetProfile.currency,
userCurrency
)
});
@ -340,6 +349,78 @@ export class ImportService {
return uniqueAccountIds.size === 1;
}
private async markActivitiesAsDuplicates({
activitiesDto,
userId
}: {
activitiesDto: Partial<CreateOrderDto>[];
userId: string;
}): Promise<Partial<Activity>[]> {
const existingActivities = await this.orderService.orders({
include: { SymbolProfile: true },
orderBy: { date: 'desc' },
where: { userId }
});
return activitiesDto.map(
({
accountId,
comment,
currency,
dataSource,
date: dateString,
fee,
quantity,
symbol,
type,
unitPrice
}) => {
const date = parseISO(<string>(<unknown>dateString));
const isDuplicate = existingActivities.some((activity) => {
return (
activity.SymbolProfile.currency === currency &&
activity.SymbolProfile.dataSource === dataSource &&
isSameDay(activity.date, date) &&
activity.fee === fee &&
activity.quantity === quantity &&
activity.SymbolProfile.symbol === symbol &&
activity.type === type &&
activity.unitPrice === unitPrice
);
});
return {
accountId,
comment,
date,
fee,
isDuplicate,
quantity,
type,
unitPrice,
SymbolProfile: {
currency,
dataSource,
symbol,
assetClass: null,
assetSubClass: null,
comment: null,
countries: null,
createdAt: undefined,
id: undefined,
isin: null,
name: null,
scraperConfiguration: null,
sectors: null,
symbolMapping: null,
updatedAt: undefined,
url: null
}
};
}
);
}
private async validateActivities({
activitiesDto,
maxActivitiesToImport,
@ -356,33 +437,11 @@ export class ImportService {
const assetProfiles: {
[symbol: string]: Partial<SymbolProfile>;
} = {};
const existingActivities = await this.orderService.orders({
include: { SymbolProfile: true },
orderBy: { date: 'desc' },
where: { userId }
});
for (const [
index,
{ currency, dataSource, date, fee, quantity, symbol, type, unitPrice }
{ currency, dataSource, symbol }
] of activitiesDto.entries()) {
const duplicateActivity = existingActivities.find((activity) => {
return (
activity.SymbolProfile.currency === currency &&
activity.SymbolProfile.dataSource === dataSource &&
isSameDay(activity.date, parseISO(<string>(<unknown>date))) &&
activity.fee === fee &&
activity.quantity === quantity &&
activity.SymbolProfile.symbol === symbol &&
activity.type === type &&
activity.unitPrice === unitPrice
);
});
if (duplicateActivity) {
throw new Error(`activities.${index} is a duplicate activity`);
}
if (dataSource !== 'MANUAL') {
const assetProfile = (
await this.dataProviderService.getAssetProfiles([

1
apps/api/src/app/order/interfaces/activities.interface.ts
View File

@ -6,6 +6,7 @@ export interface Activities {
export interface Activity extends OrderWithAccount {
feeInBaseCurrency: number;
isDuplicate: boolean;
updateAccountBalance?: boolean;
value: number;
valueInBaseCurrency: number;

1
apps/api/src/app/order/order.service.ts
View File

@ -333,6 +333,7 @@ export class OrderService {
order.SymbolProfile.currency,
userCurrency
),
isDuplicate: false,
valueInBaseCurrency: this.exchangeRateDataService.toCurrency(
value,
order.SymbolProfile.currency,

7
libs/ui/src/lib/activities-table/activities-table.component.html
View File

@ -76,7 +76,6 @@
<ng-container matColumnDef="select">
<th *matHeaderCellDef class="px-1" mat-header-cell>
<mat-checkbox
class="mt-2"
color="primary"
[checked]="selectedRows.hasValue() && areAllRowsSelected()"
[indeterminate]="selectedRows.hasValue() && !areAllRowsSelected()"
@ -85,9 +84,11 @@
</th>
<td *matCellDef="let element" class="px-1" mat-cell>
<mat-checkbox
class="mt-2"
color="primary"
[checked]="selectedRows.isSelected(element)"
[checked]="
element.isDuplicate ? false : selectedRows.isSelected(element)
"
[disabled]="element.isDuplicate"
(change)="$event ? selectedRows.toggle(element) : null"
(click)="$event.stopPropagation()"
></mat-checkbox>

2
libs/ui/src/lib/activities-table/activities-table.component.ts
View File

@ -177,7 +177,9 @@ export class ActivitiesTableComponent implements OnChanges, OnDestroy, OnInit {
public onClickActivity(activity: Activity) {
if (this.showCheckbox) {
if (!activity.isDuplicate) {
this.selectedRows.toggle(activity);
}
} else if (
this.hasPermissionToOpenDetails &&
!activity.isDraft &&