Skip to content
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
25 changes: 17 additions & 8 deletions jablib/src/main/java/org/jabref/logic/crawler/StudyFetcher.java
Original file line number Diff line number Diff line change
Expand Up @@ -59,17 +59,26 @@ private List<FetchResult> performSearchOnQuery(StudyQuery searchQuery) {
private FetchResult performSearchOnQueryForFetcher(StudyQuery searchQuery, SearchBasedFetcher fetcher) {
try {
List<BibEntry> fetchResult = new ArrayList<>();
String catalogOverride = searchQuery.getCatalogSpecific().get(fetcher.getName());
Comment thread
calixtus marked this conversation as resolved.
Outdated
if (fetcher instanceof PagedSearchBasedFetcher basedFetcher) {
int limit = resultLimits.getOrDefault(fetcher.getName(), StudyRepository.DEFAULT_RESULT_LIMIT);
int pages = (int) Math.ceil((double) limit / basedFetcher.getPageSize());
for (int page = 0; page < pages; page++) {
fetchResult.addAll(basedFetcher.performSearchPaged(searchQuery.getQuery(), page).getContent());
}
if (fetchResult.size() > limit) {
fetchResult = new ArrayList<>(fetchResult.subList(0, limit));
if (catalogOverride != null) {
fetchResult.addAll(basedFetcher.performRawSearchQueryPaged(catalogOverride, 0).getContent());
} else {
Comment thread
qodo-free-for-open-source-projects[bot] marked this conversation as resolved.
Outdated
int limit = resultLimits.getOrDefault(fetcher.getName(), StudyRepository.DEFAULT_RESULT_LIMIT);
int pages = (int) Math.ceil((double) limit / basedFetcher.getPageSize());
for (int page = 0; page < pages; page++) {
fetchResult.addAll(basedFetcher.performSearchPaged(searchQuery.getQuery(), page).getContent());
}
if (fetchResult.size() > limit) {
fetchResult = new ArrayList<>(fetchResult.subList(0, limit));
}
}
} else {
fetchResult = fetcher.performSearch(searchQuery.getQuery());
if (catalogOverride != null) {
fetchResult = fetcher.performRawSearchQuery(catalogOverride);
} else {
fetchResult = fetcher.performSearch(searchQuery.getQuery());
}
}
return new FetchResult(fetcher.getName(), new BibDatabase(fetchResult));
} catch (FetcherException e) {
Comment thread
calixtus marked this conversation as resolved.
Outdated
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,79 @@
package org.jabref.logic.crawler;

import java.util.List;
import java.util.Map;

import org.jabref.logic.importer.FetcherException;
import org.jabref.logic.importer.PagedSearchBasedFetcher;
import org.jabref.logic.importer.SearchBasedFetcher;
import org.jabref.model.entry.BibEntry;
import org.jabref.model.paging.Page;
import org.jabref.model.study.QueryResult;
import org.jabref.model.study.StudyQuery;

import org.jspecify.annotations.NullMarked;
import org.junit.jupiter.api.Test;

import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;

@NullMarked
class StudyFetcherTest {

@Test
void catalogSpecificOverrideUsesRawPathForPagedFetcher() throws FetcherException {
StudyQuery studyQuery = new StudyQuery("machine learning");
studyQuery.setCatalogSpecific(Map.of("TestPagedFetcher", "native:query"));

PagedSearchBasedFetcher pagedFetcher = mock(PagedSearchBasedFetcher.class);
when(pagedFetcher.getName()).thenReturn("TestPagedFetcher");
when(pagedFetcher.performRawSearchQueryPaged("native:query", 0))
.thenReturn(new Page<>("native:query", 0, List.of(new BibEntry())));

Comment on lines +39 to +43

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The paged tests only page 0, so the multi-page loop (the thing that was fixed) isn't actually verified

StudyFetcher studyFetcher = new StudyFetcher(List.of(pagedFetcher), List.of(studyQuery), Map.of());
List<QueryResult> results = studyFetcher.crawl();

assertFalse(results.isEmpty());
verify(pagedFetcher).performRawSearchQueryPaged("native:query", 0);
verify(pagedFetcher, never()).performSearchPaged(org.mockito.ArgumentMatchers.anyString(), org.mockito.ArgumentMatchers.anyInt());
}

@Test
void catalogSpecificOverrideUsesRawPathForPlainFetcher() throws FetcherException {
StudyQuery studyQuery = new StudyQuery("machine learning");
studyQuery.setCatalogSpecific(Map.of("TestPlainFetcher", "native:query"));

SearchBasedFetcher plainFetcher = mock(SearchBasedFetcher.class);
when(plainFetcher.getName()).thenReturn("TestPlainFetcher");
when(plainFetcher.performRawSearchQuery("native:query")).thenReturn(List.of(new BibEntry()));

StudyFetcher studyFetcher = new StudyFetcher(List.of(plainFetcher), List.of(studyQuery), Map.of());
List<QueryResult> results = studyFetcher.crawl();

assertFalse(results.isEmpty());
verify(plainFetcher).performRawSearchQuery("native:query");
verify(plainFetcher, never()).performSearch(org.mockito.ArgumentMatchers.anyString());
}

@Test
void noOverrideFallsBackToStandardPath() throws FetcherException {
StudyQuery studyQuery = new StudyQuery("machine learning");

PagedSearchBasedFetcher pagedFetcher = mock(PagedSearchBasedFetcher.class);
when(pagedFetcher.getName()).thenReturn("TestPagedFetcher");
when(pagedFetcher.getPageSize()).thenReturn(10);
when(pagedFetcher.performSearchPaged(org.mockito.ArgumentMatchers.anyString(), org.mockito.ArgumentMatchers.anyInt()))
.thenReturn(new Page<>("machine learning", 0, List.of()));
when(pagedFetcher.performSearchPaged("machine learning", 0))
.thenReturn(new Page<>("machine learning", 0, List.of(new BibEntry())));

StudyFetcher studyFetcher = new StudyFetcher(List.of(pagedFetcher), List.of(studyQuery), Map.of());
studyFetcher.crawl();

verify(pagedFetcher).performSearchPaged("machine learning", 0);
verify(pagedFetcher, never()).performRawSearchQueryPaged(org.mockito.ArgumentMatchers.anyString(), org.mockito.ArgumentMatchers.anyInt());
}
}
Loading