Automate

Search-query can be used to automate different steps of the search process, such as searching for records in an API, filtering retrieved records, storing the results, and creating a search file.

Installation requirements

The examples below requires the colrev package to be installed. You can install it via pip:

pip install colrev==0.16.1

Important: colrev version 0.16.1 requires Python 3.10 or higher. If you are on a different Python version, create a Python 3.10 environment (e.g., via uv, venv, or conda) before installing.

API retrieval

Search queries can be used as part of automated API-based literature searches. The following example uses a search-query term to construct a Crossref request, retrieve the corresponding records, store them as a BibTeX file, and document the search in a search file.

 1import datetime
 2from pathlib import Path
 3from urllib.parse import quote_plus
 4
 5from colrev.packages.crossref.src import crossref_api
 6from colrev.writer.write_utils import write_file
 7
 8from search_query.constants import Fields
 9from search_query.query import Query
10from search_query.query_term import Term
11from search_query.search_file import SearchFile
12
13
14def to_crossref_url(query: Query) -> str:
15    """Create a Crossref URL for an individual search term."""
16    if not query.is_term():
17        raise ValueError(
18            "Crossref retrieval expects an individual search term."
19        )
20
21    if query.field is None or query.field.value != Fields.TITLE:
22        raise ValueError(
23            f"Only the title field is supported in this example "
24            f"({query.field})."
25        )
26
27    query_value = query.value.strip().strip('"')
28
29    return (
30        "https://api.crossref.org/works"
31        f"?query.title={quote_plus(query_value)}"
32    )
33
34
35if __name__ == "__main__":
36
37    query = Term(
38        "microsourcing",
39        field="title",
40    )
41
42    url = to_crossref_url(query)
43
44    api_crossref = crossref_api.CrossrefAPI(url=url)
45    records = api_crossref.get_records()
46
47    sf = SearchFile(
48        search_string=query.to_string(),
49        platform="crossref",
50        authors=[{"name": "Gerit Wagner"}],
51        record_info={
52            "source": "manual",
53            "url": url,
54        },
55        date={
56            "data_entry": datetime.datetime.now().strftime(
57                "%Y-%m-%d %H:%M"
58            )
59        },
60        field="title",
61        description="Crossref search for research on microsourcing",
62    )
63
64    sf.save("test/microsourcing_search.json")
65
66    records_dict = {
67        record.get_value("doi"): record.get_data()
68        for record in records
69    }
70
71    write_file(
72        records_dict=records_dict,
73        filename=Path("test/crossref_records.bib"),
74    )

Query emulation

Some academic search APIs only support simple keyword searches and cannot execute nested Boolean queries directly. In such cases, the query tree provided by search-query can be used to implement an independent query-processing layer.

The following example illustrates the retrieval approach of QuEALS (Query Emulation for Academic Literature Searches) using the Crossref API. QuEALS recursively processes a search-query tree and combines API-based retrieval with local query processing.

The approach is described in:

Geßler, A., Schnickmann, K., and Wagner, G. (2026). “Advancing literature review automation through API-based searches: Design of an independent emulator”. Conditionally accepted: Proceedings of the International Conference on Information Systems.

Note

Crossref-specific retrieval functionality is intentionally not part of the search-query package. The example demonstrates how search-query can provide the query representation and local processing required for query emulation, while API-specific retrieval remains with the corresponding source implementation.

  1from urllib.parse import quote_plus
  2
  3from colrev.packages.crossref.src import crossref_api
  4
  5from search_query.constants import Fields, Operators
  6from search_query.query import Query
  7from search_query.query_and import AndQuery
  8from search_query.query_or import OrQuery
  9
 10
 11def to_crossref_url(query: Query) -> str:
 12    """Create a Crossref URL for an individual search term."""
 13    if not query.is_term():
 14        raise ValueError(
 15            "Crossref retrieval expects an individual search term."
 16        )
 17
 18    if query.field is None or query.field.value != Fields.TITLE:
 19        raise ValueError(
 20            f"Only the title field is supported in this example "
 21            f"({query.field})."
 22        )
 23
 24    query_value = query.value.strip().strip('"')
 25
 26    return (
 27        "https://api.crossref.org/works"
 28        f"?query.title={quote_plus(query_value)}"
 29    )
 30
 31
 32def get_crossref_yield(query: Query) -> int:
 33    """Get the estimated number of records for an individual term."""
 34    url = to_crossref_url(query)
 35
 36    api_crossref = crossref_api.CrossrefAPI(url=url)
 37
 38    return api_crossref.get_len_total()
 39
 40
 41def estimate_yield(query: Query) -> int:
 42    """Estimate the yield of a query recursively."""
 43    if query.is_term():
 44        return get_crossref_yield(query)
 45
 46    estimates = [
 47        estimate_yield(child)
 48        for child in query.children
 49    ]
 50
 51    if query.value == Operators.AND:
 52        return min(estimates)
 53
 54    if query.value == Operators.OR:
 55        return sum(estimates)
 56
 57    raise ValueError(f"Unsupported operator: {query.value}")
 58
 59
 60def retrieve_term(query: Query) -> list[dict]:
 61    """Retrieve records for an individual search term from Crossref."""
 62    url = to_crossref_url(query)
 63
 64    api_crossref = crossref_api.CrossrefAPI(url=url)
 65    records = api_crossref.get_records()
 66
 67    return [
 68        record.get_data()
 69        for record in records
 70    ]
 71
 72
 73def deduplicate(records: list[dict]) -> list[dict]:
 74    """Remove records retrieved through multiple query branches."""
 75    records_by_doi = {
 76        record["doi"]: record
 77        for record in records
 78    }
 79
 80    return list(records_by_doi.values())
 81
 82
 83def retrieve(query: Query) -> list[dict]:
 84    """Retrieve records using the QuEALS approach."""
 85    if query.is_term():
 86        return retrieve_term(query)
 87
 88    if query.value == Operators.OR:
 89        records = []
 90
 91        for child in query.children:
 92            records.extend(retrieve(child))
 93
 94        return deduplicate(records)
 95
 96    if query.value == Operators.AND:
 97        child = min(
 98            query.children,
 99            key=estimate_yield,
100        )
101
102        records = retrieve(child)
103
104        return [
105            record
106            for record in records
107            if query.selects(record_dict=record)
108        ]
109
110    raise ValueError(f"Unsupported operator: {query.value}")
111
112
113if __name__ == "__main__":
114
115    query = AndQuery(
116        [
117            OrQuery(
118                ["strategy", "strategic"],
119                field="title",
120            ),
121            OrQuery(
122                ["technology", "digital"],
123                field="title",
124            ),
125        ]
126    )
127
128    records = retrieve(query)
129
130    # See "Automated API retrieval" above for an example of creating
131    # a SearchFile and writing the retrieved records to a BibTeX file.