Improve transformers filtering again

This commit is contained in:
AuroraWright
2024-06-26 16:31:00 +02:00
parent cf3df01ee8
commit 743c64aa16

View File

@@ -292,11 +292,12 @@ class TextFiltering:
final_blocks = [] final_blocks = []
if self.accurate_filtering: if self.accurate_filtering:
detection_results = self.pipe(new_blocks, top_k=2, truncation=True) detection_results = self.pipe(new_blocks, top_k=3, truncation=True)
for idx, block in enumerate(new_blocks): for idx, block in enumerate(new_blocks):
if((detection_results[idx][0]['label'] == 'ja' and detection_results[idx][0]['score'] >= 0.20) or for result in detection_results[idx]:
(detection_results[idx][1]['label'] == 'ja' and detection_results[idx][1]['score'] >= 0.20)): if result['label'] == 'ja':
final_blocks.append(block) final_blocks.append(block)
break
else: else:
for block in new_blocks: for block in new_blocks:
if self.classify(block)[0] == 'ja': if self.classify(block)[0] == 'ja':