Feature/extend scraper configuration support (#2110)

* Extend scraper configuration support

* Update changelog

CHANGELOG.md

@@ -10,6 +10,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ### Added
 
 - Added pagination to the historical market data table of the admin control panel
+- Added the attribute `headers` to the scraper configuration
+
+### Changed
+
+- Extended the asset profile details dialog in the admin control panel by the scraper configuration
 
 ## 1.284.0 - 2023-06-27
 

apps/api/src/app/admin/admin.service.ts

@@ -249,12 +249,14 @@ export class AdminService {
   public async patchAssetProfileData({
     comment,
     dataSource,
+    scraperConfiguration,
     symbol,
     symbolMapping
   }: Prisma.SymbolProfileUpdateInput & UniqueAsset) {
     await this.symbolProfileService.updateSymbolProfile({
       comment,
       dataSource,
+      scraperConfiguration,
       symbol,
       symbolMapping
     });

apps/api/src/app/admin/update-asset-profile.dto.ts

@@ -1,3 +1,4 @@
+import { Prisma } from '@prisma/client';
 import { IsObject, IsOptional, IsString } from 'class-validator';
 
 export class UpdateAssetProfileDto {
@@ -5,6 +6,10 @@ export class UpdateAssetProfileDto {
   @IsOptional()
   comment?: string;
 
+  @IsObject()
+  @IsOptional()
+  scraperConfiguration?: Prisma.InputJsonObject;
+
   @IsObject()
   @IsOptional()
   symbolMapping?: {

apps/api/src/services/data-provider/manual/manual.service.ts

@@ -67,8 +67,12 @@ export class ManualService implements DataProviderInterface {
       const [symbolProfile] = await this.symbolProfileService.getSymbolProfiles(
         [{ symbol, dataSource: this.getName() }]
       );
-      const { defaultMarketPrice, selector, url } =
+      const {
-        symbolProfile.scraperConfiguration ?? {};
+        defaultMarketPrice,
+        headers = {},
+        selector,
+        url
+      } = symbolProfile.scraperConfiguration ?? {};
 
       if (defaultMarketPrice) {
         const historical: {
@@ -91,7 +95,7 @@ export class ManualService implements DataProviderInterface {
         return {};
       }
 
-      const get = bent(url, 'GET', 'string', 200, {});
+      const get = bent(url, 'GET', 'string', 200, headers);
 
       const html = await get();
       const $ = cheerio.load(html);

apps/api/src/services/symbol-profile/symbol-profile.service.ts

@@ -96,11 +96,12 @@ export class SymbolProfileService {
   public updateSymbolProfile({
     comment,
     dataSource,
+    scraperConfiguration,
     symbol,
     symbolMapping
   }: Prisma.SymbolProfileUpdateInput & UniqueAsset) {
     return this.prismaService.symbolProfile.update({
-      data: { comment, symbolMapping },
+      data: { comment, scraperConfiguration, symbolMapping },
       where: { dataSource_symbol: { dataSource, symbol } }
     });
   }
@@ -195,6 +196,8 @@ export class SymbolProfileService {
     if (scraperConfiguration) {
       return {
         defaultMarketPrice: scraperConfiguration.defaultMarketPrice as number,
+        headers:
+          scraperConfiguration.headers as ScraperConfiguration['headers'],
         selector: scraperConfiguration.selector as string,
         url: scraperConfiguration.url as string
       };

apps/client/src/app/components/admin-market-data/asset-profile-dialog/asset-profile-dialog.component.ts

@@ -13,6 +13,7 @@ import { AdminService } from '@ghostfolio/client/services/admin.service';
 import { DataService } from '@ghostfolio/client/services/data.service';
 import {
   AdminMarketDataDetails,
+  ScraperConfiguration,
   UniqueAsset
 } from '@ghostfolio/common/interfaces';
 import { translate } from '@ghostfolio/ui/i18n';
@@ -34,6 +35,7 @@ export class AssetProfileDialog implements OnDestroy, OnInit {
   public assetProfile: AdminMarketDataDetails['assetProfile'];
   public assetProfileForm = this.formBuilder.group({
     comment: '',
+    scraperConfiguration: '',
     symbolMapping: ''
   });
   public assetSubClass: string;
@@ -103,6 +105,9 @@ export class AssetProfileDialog implements OnDestroy, OnInit {
 
         this.assetProfileForm.setValue({
           comment: this.assetProfile?.comment ?? '',
+          scraperConfiguration: JSON.stringify(
+            this.assetProfile?.scraperConfiguration ?? {}
+          ),
           symbolMapping: JSON.stringify(this.assetProfile?.symbolMapping ?? {})
         });
 
@@ -148,8 +153,15 @@ export class AssetProfileDialog implements OnDestroy, OnInit {
   }
 
   public onSubmit() {
+    let scraperConfiguration = {};
     let symbolMapping = {};
 
+    try {
+      scraperConfiguration = JSON.parse(
+        this.assetProfileForm.controls['scraperConfiguration'].value
+      );
+    } catch {}
+
     try {
       symbolMapping = JSON.parse(
         this.assetProfileForm.controls['symbolMapping'].value
@@ -157,6 +169,7 @@ export class AssetProfileDialog implements OnDestroy, OnInit {
     } catch {}
 
     const assetProfileData: UpdateAssetProfileDto = {
+      scraperConfiguration,
       symbolMapping,
       comment: this.assetProfileForm.controls['comment'].value ?? null
     };

apps/client/src/app/components/admin-market-data/asset-profile-dialog/asset-profile-dialog.html

@@ -162,6 +162,17 @@
         ></textarea>
       </mat-form-field>
     </div>
+    <div *ngIf="assetProfile?.dataSource === 'MANUAL'">
+      <mat-form-field appearance="outline" class="w-100">
+        <mat-label i18n>Scraper Configuration</mat-label>
+        <textarea
+          cdkTextareaAutosize
+          formControlName="scraperConfiguration"
+          matInput
+          type="text"
+        ></textarea>
+      </mat-form-field>
+    </div>
     <div>
       <mat-form-field appearance="outline" class="w-100">
         <mat-label i18n>Note</mat-label>

apps/client/src/app/services/admin.service.ts

@@ -191,12 +191,13 @@ export class AdminService {
   public patchAssetProfile({
     comment,
     dataSource,
+    scraperConfiguration,
     symbol,
     symbolMapping
   }: UniqueAsset & UpdateAssetProfileDto) {
     return this.http.patch<EnhancedSymbolProfile>(
       `/api/v1/admin/profile-data/${dataSource}/${symbol}`,
-      { comment, symbolMapping }
+      { comment, scraperConfiguration, symbolMapping }
     );
   }
 

libs/common/src/lib/interfaces/scraper-configuration.interface.ts

@@ -1,5 +1,6 @@
 export interface ScraperConfiguration {
   defaultMarketPrice?: number;
+  headers?: { [key: string]: string };
   selector: string;
   url: string;
 }