{ "@context": "https://schema.org", "@type": "DefinedTermSet", "name": "unlob Web Search & Retrieval Glossary", "description": "63 definitions covering web search, retrieval, indexing, knowledge graphs, crawling and agent context.", "url": "https://www.unlob.com/glossary", "dateModified": "2026-08-05", "categories": [ { "id": "retrieval", "label": "Retrieval", "blurb": "How a query becomes a set of results." }, { "id": "index", "label": "Indexing", "blurb": "What goes into a search index, and what it costs to keep there." }, { "id": "graph", "label": "Graphs & trust", "blurb": "Relationships between documents, and judging what to believe." }, { "id": "agents", "label": "Agents & context", "blurb": "How autonomous systems consume retrieval." }, { "id": "crawl", "label": "Crawling", "blurb": "Getting the web in the first place." }, { "id": "ops", "label": "Operations", "blurb": "Running and paying for all of it." } ], "hasDefinedTerm": [ { "@type": "DefinedTerm", "name": "Hybrid search", "slug": "hybrid-search", "category": "retrieval", "description": "Hybrid search runs a lexical query and a vector query over the same corpus and fuses the two rankings into one result set.", "detail": [ "Lexical retrieval matches words and fails when the user and the document use different vocabulary. Vector retrieval matches meaning and fails on exact identifiers, which have no useful neighbourhood in embedding space. Hybrid search runs both and combines them, so a result strong in either space surfaces.", "The fusion step matters more than it appears. BM25 scores and cosine similarities are not comparable numbers, so weighting them against each other requires tuning that does not transfer between corpora. Reciprocal rank fusion sidesteps this by combining positions rather than scores — each result is scored by the reciprocal of its rank in each list, which needs no tuning at all.", "unlob runs hybrid by default, fusing by reciprocal rank. It costs about 2.5 ms more than keyword-only at the median, which for most workloads is not a consideration." ], "relatedTerms": [ "bm25", "vector-search", "reciprocal-rank-fusion", "semantic-search" ], "url": "https://www.unlob.com/glossary/hybrid-search" }, { "@type": "DefinedTerm", "name": "BM25", "slug": "bm25", "category": "retrieval", "description": "BM25 is a ranking function that scores documents by term frequency and inverse document frequency, with saturation so repeated terms have diminishing returns.", "detail": [ "The standard lexical ranking function in information retrieval, and still extremely hard to beat on queries where the words matter. It scores a document by how often the query terms appear, discounted by how common those terms are across the corpus, with a saturation curve so the tenth occurrence of a word counts for much less than the second.", "Its enduring advantage is that it is exact. A document either contains the term or it does not, which makes BM25 the correct tool for identifiers, error codes and quoted phrases — precisely the queries where semantic retrieval degrades.", "BM25 is the lexical half of hybrid search almost everywhere, including here." ], "relatedTerms": [ "hybrid-search", "inverted-index", "salient-term", "filtering" ], "url": "https://www.unlob.com/glossary/bm25" }, { "@type": "DefinedTerm", "name": "Semantic search", "slug": "semantic-search", "category": "retrieval", "description": "Semantic search retrieves documents by meaning rather than by word overlap, using vector embeddings to place queries and documents in a shared space.", "detail": [ "Both the query and every document are converted to vectors by the same model, and retrieval becomes a nearest-neighbour problem. Because the embedding captures meaning rather than spelling, a query for \"car\" retrieves a document about \"automobiles\", and — with a multilingual model — a document in German.", "The failure mode is exact matching. An error code or a part number has no meaningful neighbourhood in embedding space, so semantic search returns things that are merely about errors. This is why production systems almost always run hybrid.", "unlob puts all 101 supported languages in a single shared space, so an English query retrieves a relevant German passage with no translation step and no per-language index." ], "relatedTerms": [ "vector-search", "embedding", "hybrid-search", "multilingual-search" ], "url": "https://www.unlob.com/glossary/semantic-search" }, { "@type": "DefinedTerm", "name": "Vector search", "slug": "vector-search", "category": "retrieval", "description": "Vector search finds the nearest neighbours of a query vector among document vectors, usually using an approximate index because exact search over millions of vectors is too slow.", "detail": [ "Exact nearest-neighbour search means comparing the query against every vector, which does not scale. Approximate methods trade a small amount of recall for large speed gains — clustering the space and searching only the nearest cells, or building a navigable graph over the vectors.", "The binding constraint at scale is memory. Holding raw float vectors in RAM costs about 3 KB each at 768 dimensions, so a billion vectors would need several terabytes. Quantisation reduces this dramatically: one-bit quantisation stores that same vector in about 96 bytes.", "How a given engine resolves that trade-off is one of the main things separating vector search products, and most treat it as proprietary." ], "relatedTerms": [ "embedding", "quantisation", "ann", "semantic-search" ], "url": "https://www.unlob.com/glossary/vector-search" }, { "@type": "DefinedTerm", "name": "Approximate nearest neighbour (ANN)", "slug": "ann", "category": "retrieval", "description": "ANN search finds vectors close to a query vector without guaranteeing the exact closest, trading a small recall loss for orders-of-magnitude speed improvement.", "detail": [ "Exact nearest-neighbour search is linear in the corpus size. ANN algorithms make it sublinear by restricting the search: inverted-file methods cluster vectors and search only the nearest clusters, while graph methods build a navigable structure and walk it greedily.", "The recall-speed trade is tunable. Searching more clusters or exploring more of the graph improves recall at the cost of latency, which means the same index can serve a fast path and a thorough path." ], "relatedTerms": [ "vector-search", "quantisation", "embedding" ], "url": "https://www.unlob.com/glossary/ann" }, { "@type": "DefinedTerm", "name": "Reciprocal rank fusion", "slug": "reciprocal-rank-fusion", "category": "retrieval", "description": "Reciprocal rank fusion merges several ranked lists by scoring each document by the reciprocal of its position in each list, requiring no score normalisation.", "detail": [ "The problem with combining a BM25 ranking and a vector ranking is that their scores are not comparable — one is an unbounded relevance score, the other a bounded similarity. Any weighted sum requires normalisation that has to be tuned per corpus and quietly stops working when the corpus changes.", "Reciprocal rank fusion ignores the scores entirely and uses positions. A document ranked third in one list and eighth in the other scores 1/(k+3) + 1/(k+8), where k is a smoothing constant conventionally set to 60. No tuning, no normalisation, and it works across arbitrary numbers of lists." ], "relatedTerms": [ "hybrid-search", "bm25", "vector-search" ], "url": "https://www.unlob.com/glossary/reciprocal-rank-fusion" }, { "@type": "DefinedTerm", "name": "Embedding", "slug": "embedding", "category": "retrieval", "description": "An embedding is a fixed-length vector representing the meaning of a piece of text, produced by a model so that similar texts land near each other.", "detail": [ "Embeddings turn semantic similarity into geometric proximity, which is what makes vector search possible. Dimensionality is a trade: more dimensions capture more nuance and cost more memory and compute per comparison.", "Generation cost varies enormously by model class, and at web scale that cost is often the largest single line in building an index. Which model class a search provider uses is therefore a commercial decision as much as a technical one, and few publish it." ], "relatedTerms": [ "vector-search", "semantic-search", "quantisation", "multilingual-search" ], "url": "https://www.unlob.com/glossary/embedding" }, { "@type": "DefinedTerm", "name": "Quantisation", "slug": "quantisation", "category": "retrieval", "description": "Quantisation compresses vectors by storing each dimension in fewer bits, cutting memory and speeding comparison at a small cost in precision.", "detail": [ "A 768-dimensional float32 vector occupies about 3 KB. Quantising to int8 cuts that to 768 bytes; binary quantisation, which stores only the sign of each dimension, cuts it to 96 bytes — a 32× reduction either way.", "The precision loss is real but recoverable. The standard pattern is a two-stage search: retrieve a generous candidate set using the cheap binary codes, then rerank that much smaller set with higher-precision vectors. Recall approaches the uncompressed baseline at a fraction of the memory." ], "relatedTerms": [ "vector-search", "embedding", "ann", "resident-memory" ], "url": "https://www.unlob.com/glossary/quantisation" }, { "@type": "DefinedTerm", "name": "Reranking", "slug": "reranking", "category": "retrieval", "description": "Reranking reorders an initial candidate set with a more expensive, more accurate model than the one used to retrieve it.", "detail": [ "Retrieval is optimised for recall over millions of documents; reranking is optimised for precision over a few dozen. Running the expensive scorer only on candidates that survived the first stage makes accuracy affordable.", "The same principle appears inside a vector index — retrieve on quantised codes, rerank on higher-precision vectors — and at the application layer, where a cross-encoder reorders the top fifty results.", "unlob does the first internally. The second is worth considering in your application if precision at rank one matters more than latency." ], "relatedTerms": [ "vector-search", "quantisation", "hybrid-search", "rag" ], "url": "https://www.unlob.com/glossary/reranking" }, { "@type": "DefinedTerm", "name": "Query expansion", "slug": "query-expansion", "category": "retrieval", "description": "Query expansion adds related terms to a query to improve recall when the user’s vocabulary differs from the documents’.", "detail": [ "The classic fix for vocabulary mismatch in lexical search: add synonyms, expand abbreviations, include morphological variants. It reliably improves recall and reliably damages precision, because expansion terms are not always relevant.", "Semantic retrieval makes most query expansion unnecessary — the embedding already places synonyms near each other. It remains useful for abbreviations and domain jargon that the embedder was not trained on." ], "relatedTerms": [ "semantic-search", "bm25", "hybrid-search" ], "url": "https://www.unlob.com/glossary/query-expansion" }, { "@type": "DefinedTerm", "name": "Recall and precision", "slug": "recall-and-precision", "category": "retrieval", "description": "Recall is the share of relevant documents a search returns; precision is the share of returned documents that are relevant.", "detail": [ "The two trade against each other. Returning more results raises recall and lowers precision; returning fewer does the opposite. Which one to optimise depends entirely on who is reading the results.", "Human search engines optimise recall and rank: a person skims and discards, so a hundred results with the good one first is a fine outcome. Agents cannot do that — every irrelevant result costs context and reasoning tokens. For agent retrieval, precision dominates.", "That inversion is the argument for admission control: move quality from query-time ranking to index-time admission, and return fewer, better things." ], "relatedTerms": [ "admission-control", "agentic-search", "ranking" ], "url": "https://www.unlob.com/glossary/recall-and-precision" }, { "@type": "DefinedTerm", "name": "Ranking", "slug": "ranking", "category": "retrieval", "description": "Ranking orders retrieved documents by predicted relevance, traditionally combining a text-matching score with query-independent quality signals.", "detail": [ "A ranking function typically blends a relevance score with signals like link authority, freshness and page quality. Web search engines invest enormously here because the top three results receive nearly all attention.", "Ranking assumes a reader who will look at the first result and stop if it is good. An agent consuming ten results pays for all ten, which is why for agent retrieval what gets *into* the result set matters more than the order within it." ], "relatedTerms": [ "recall-and-precision", "admission-control", "host-rank", "centrality" ], "url": "https://www.unlob.com/glossary/ranking" }, { "@type": "DefinedTerm", "name": "Faceted search", "slug": "faceted-search", "category": "retrieval", "description": "Faceted search returns counts across categorical dimensions alongside results, letting a user or agent see the shape of a result set before reading it.", "detail": [ "Facets answer questions the results themselves do not: is this coverage concentrated in one publisher, spread evenly across years, dominated by one content type. For a human interface they drive filter menus.", "For an agent they are a cheap control loop. One request with facets tells the agent whether to broaden, narrow or disambiguate — decisions it would otherwise make by reading documents and paying for the tokens." ], "relatedTerms": [ "filtering", "collapsing", "agentic-search" ], "url": "https://www.unlob.com/glossary/faceted-search" }, { "@type": "DefinedTerm", "name": "Filtering", "slug": "filtering", "category": "retrieval", "description": "Filtering restricts a search to documents matching structured criteria — date, language, domain, quality — before or during scoring.", "detail": [ "Where the filter is applied determines whether it costs or saves time. Post-filtering scores everything and discards afterwards, which is wasteful and can return fewer results than requested. Pre-filtering narrows the candidate set first, which makes heavily filtered queries faster than unfiltered ones.", "Pre-filtering requires the filterable attributes to be stored as fast fields in the index. unlob writes quality, host rank, centrality, corroboration counts, content type, topic, language and both timestamps at build time, so filtering narrows before scoring." ], "relatedTerms": [ "faceted-search", "fast-field", "collapsing" ], "url": "https://www.unlob.com/glossary/filtering" }, { "@type": "DefinedTerm", "name": "Result collapsing", "slug": "collapsing", "category": "retrieval", "description": "Collapsing groups near-identical results and returns one representative each, so a single syndicated item cannot occupy an entire result set.", "detail": [ "A wire story republished by forty outlets is forty near-identical documents. Ranked by relevance they take every slot, and an agent reading them concludes — reasonably but wrongly — that the claim is exceptionally well supported because it saw it ten times.", "Collapsing by story cluster returns one representative per cluster with a group size attached, converting ten wasted slots into ten distinct facts.", "The important subtlety is that collapsing must not destroy the corroboration signal. unlob records the host of every rejected duplicate against the surviving passage, so independent source counts still reflect every outlet that carried the story." ], "relatedTerms": [ "deduplication", "corroboration", "story-cluster", "near-duplicate-detection" ], "url": "https://www.unlob.com/glossary/collapsing" }, { "@type": "DefinedTerm", "name": "Admission control", "slug": "admission-control", "category": "index", "description": "Admission control decides at index time which documents are worth indexing at all, rather than indexing everything and sorting it out at query time.", "detail": [ "Conventional search indexes almost everything and relies on ranking to surface the good material. That works when a human reads the results, because a person discards bad results at a glance and at no cost.", "An agent cannot. It cannot disambiguate an ambiguous query and cannot triage results without paying for them in context and reasoning tokens. So quality has to move earlier: admit only what is worth returning.", "unlob is built on this principle: we admit selectively rather than indexing everything, so a search returns fewer and better hits. The trade is a smaller index, which we treat as something to be honest about — why_not(url) reports whether any URL is present, was removed with a typed reason, or was never admitted. How the admission decision itself is made is not published." ], "relatedTerms": [ "bounded-index", "recall-and-precision", "pruning", "coverage-transparency" ], "url": "https://www.unlob.com/glossary/admission-control" }, { "@type": "DefinedTerm", "name": "Salient term", "slug": "salient-term", "category": "index", "description": "A salient term is a distinctive, low-frequency token that identifies specific content — an error code, an identifier, a proper noun — as opposed to common vocabulary.", "detail": [ "Salience is essentially inverse document frequency: a term appearing in a handful of documents carries far more identifying information than one appearing everywhere. These are the terms users search when they want something specific rather than something similar.", "They are also the terms semantic retrieval handles worst, since an identifier has no meaningful embedding neighbourhood.", "This is the failure mode a selective index has to get right, and unlob commits to it as a guarantee rather than a best effort: an exact identifier that is in the index stays findable, and does not quietly disappear as the corpus is compacted. Use the term filter to require one." ], "relatedTerms": [ "bm25", "pruning", "bounded-index", "filtering" ], "url": "https://www.unlob.com/glossary/salient-term" }, { "@type": "DefinedTerm", "name": "Bounded index", "slug": "bounded-index", "category": "index", "description": "A bounded index caps its own size and prunes or replaces existing entries rather than growing without limit.", "detail": [ "Unbounded indexes have unbounded costs. A bounded index sets a ceiling, competitively replaces weaker entries as better candidates arrive, and fills genuine gaps on demand rather than pre-emptively.", "For agent retrieval the trade is favourable: a smaller curated index returns better context per token than a larger one full of near-duplicates. The obligation it creates is transparency — if you are deliberately excluding things, you should be able to say what and why." ], "relatedTerms": [ "admission-control", "pruning", "coverage-transparency", "crawl-on-miss" ], "url": "https://www.unlob.com/glossary/bounded-index" }, { "@type": "DefinedTerm", "name": "Index pruning", "slug": "pruning", "category": "index", "description": "Pruning removes documents from an index to reclaim space, based on redundancy, staleness, access patterns or competitive replacement.", "detail": [ "A bounded index must remove things to admit new ones. The question is what, and on what evidence: near-duplicates, superseded versions, content past its useful life, passages nothing has retrieved in a long time, or weaker members of a saturated semantic cell.", "Pruning is where indexes quietly get worse. Removing the last carrier of a rare term destroys exact-match recall for that term permanently, and nothing in the semantic space substitutes for it — which is why that specific removal is prohibited here.", "unlob records every removal in an append-only ledger with a typed reason, queryable through why_not." ], "relatedTerms": [ "bounded-index", "salient-term", "coverage-transparency", "removal-ledger" ], "url": "https://www.unlob.com/glossary/pruning" }, { "@type": "DefinedTerm", "name": "Removal ledger", "slug": "removal-ledger", "category": "index", "description": "A removal ledger is an append-only record of every document removed from an index, with the reason and any replacement.", "detail": [ "Most indexes remove documents silently. That makes absence ambiguous: a missing URL could have been rejected, dropped, superseded or never crawled, and nothing distinguishes them.", "An append-only ledger with typed reasons — redundant, superseded, stale, access-starved, lost-replacement, tombstoned-source — makes absence explainable. For regulated buyers it is the difference between \"we searched\" and \"here is what was considered and what was excluded\"." ], "relatedTerms": [ "pruning", "coverage-transparency", "bounded-index" ], "url": "https://www.unlob.com/glossary/removal-ledger" }, { "@type": "DefinedTerm", "name": "Coverage transparency", "slug": "coverage-transparency", "category": "index", "description": "Coverage transparency is the ability to ask a search index why a specific URL is or is not present, and get a substantive answer.", "detail": [ "Search APIs are precise about what they found and silent about what they did not. An empty result set could mean the page does not exist, was never crawled, or was indexed and later removed — three very different facts that an agent will resolve badly if it has to guess.", "Coverage transparency resolves it: present, removed with a typed reason and any replacement, or never admitted. As far as we know, unlob is the only web search API that offers it." ], "relatedTerms": [ "removal-ledger", "pruning", "bounded-index", "provenance" ], "url": "https://www.unlob.com/glossary/coverage-transparency" }, { "@type": "DefinedTerm", "name": "Inverted index", "slug": "inverted-index", "category": "index", "description": "An inverted index maps each term to the list of documents containing it, which is what makes lexical search fast.", "detail": [ "Rather than scanning documents for terms, an inverted index stores, for every term, a posting list of the documents containing it and where. A query intersects or unions the relevant posting lists, touching only documents that contain the query terms.", "Posting lists compress well and page from disk efficiently, which means the lexical tier of a search engine need not be resident in memory to be fast." ], "relatedTerms": [ "bm25", "fast-field", "resident-memory" ], "url": "https://www.unlob.com/glossary/inverted-index" }, { "@type": "DefinedTerm", "name": "Fast field", "slug": "fast-field", "category": "index", "description": "A fast field is a document attribute stored column-wise in the index for direct lookup, enabling filtering and sorting without reading the document.", "detail": [ "Attributes you filter or sort on — dates, scores, categories, numeric signals — are stored in a column layout rather than inside documents, so a filter reads a contiguous array instead of decoding records.", "This is why filtering can make a query faster rather than slower: the candidate set narrows before scoring, and the narrowing itself is nearly free.", "unlob stores quality, host rank, centrality, community, independent source counts, in-degree, both timestamps, content type, topic and language as fast fields." ], "relatedTerms": [ "filtering", "inverted-index", "centrality", "host-rank" ], "url": "https://www.unlob.com/glossary/fast-field" }, { "@type": "DefinedTerm", "name": "Resident memory", "slug": "resident-memory", "category": "index", "description": "Resident memory is the portion of an index that must stay in RAM to serve queries, and it is usually what caps how large an index can economically grow.", "detail": [ "Vector search systems conventionally hold their codes in memory, making resident memory linear in corpus size and putting a hard economic ceiling on index growth — a billion float vectors would need a terabyte of RAM.", "Designs that page most of the index from disk and keep only a compact navigational structure in memory break that linear relationship, which is why resident-memory strategy is one of the more closely held parts of any vector search engine.", "For a buyer the practical question is not which strategy a provider uses but whether the pricing reflects one that works — an engine paying for linear RAM growth has to charge for it eventually." ], "relatedTerms": [ "quantisation", "vector-search", "object-storage-first" ], "url": "https://www.unlob.com/glossary/resident-memory" }, { "@type": "DefinedTerm", "name": "Sharding", "slug": "sharding", "category": "index", "description": "Sharding splits an index across several nodes, each holding a portion, with a gateway scattering queries and gathering results.", "detail": [ "Horizontal scaling for search: each shard searches its own portion in parallel and the gateway merges the ranked results. Query latency becomes the slowest shard rather than the sum.", "A well-designed gateway holds no index itself, so its resident memory is approximately zero and it can be scaled independently. A dead shard should degrade results rather than fail the request — losing a tenth of the corpus is preferable to losing the query." ], "relatedTerms": [ "resident-memory", "object-storage-first", "stateless-serving" ], "url": "https://www.unlob.com/glossary/sharding" }, { "@type": "DefinedTerm", "name": "Near-duplicate detection", "slug": "near-duplicate-detection", "category": "index", "description": "Near-duplicate detection identifies documents with substantially the same content despite differing in wording, formatting or boilerplate.", "detail": [ "The web is enormously redundant: syndicated articles, mirrored documentation, scraped copies. Exact hashing misses all of it because a single differing character changes the hash, so near-duplicate detection uses locality-sensitive hashing or embedding similarity.", "Detection is only half the job. Discarding duplicates naively destroys the evidence that many outlets carried the same claim — which is exactly the signal you need to judge it. Recording the rejected copy's host against the survivor keeps the index compact and the corroboration signal intact." ], "relatedTerms": [ "deduplication", "story-cluster", "corroboration", "collapsing" ], "url": "https://www.unlob.com/glossary/near-duplicate-detection" }, { "@type": "DefinedTerm", "name": "Deduplication", "slug": "deduplication", "category": "index", "description": "Deduplication removes redundant copies of the same content from an index or a result set.", "detail": [ "At index time it prevents the same content occupying storage many times. At query time it prevents one item occupying every slot in a result set.", "The trap is that deduplication discards information about how widely something was published — which for judging a claim is the most valuable information available. Deduplication and corroboration counting have to be designed together, or you get a compact index that cannot tell you whether anyone corroborated anything." ], "relatedTerms": [ "near-duplicate-detection", "collapsing", "corroboration", "story-cluster" ], "url": "https://www.unlob.com/glossary/deduplication" }, { "@type": "DefinedTerm", "name": "Coverage graph", "slug": "coverage-graph", "category": "graph", "description": "A coverage graph is a typed graph over a search index recording how documents relate — who published them, which report the same story, what they are about and who links to whom.", "detail": [ "Search engines compute these relationships in order to build an index: the link graph for authority, near-duplicate clusters for deduplication, entity and topic extraction for classification. Almost all of them discard the structure afterwards and return a ranked list.", "Publishing it instead lets an agent traverse relationships rather than reconstructing them from ranked lists in its context window. The cost is close to zero, because every input already existed.", "unlob exposes six traversal operations over passages, hosts, stories, topics and entities." ], "relatedTerms": [ "graphrag", "centrality", "corroboration", "entity-extraction" ], "url": "https://www.unlob.com/glossary/coverage-graph" }, { "@type": "DefinedTerm", "name": "GraphRAG", "slug": "graphrag", "category": "graph", "description": "GraphRAG is retrieval-augmented generation over a knowledge graph, retrieving connected subgraphs of entities and relationships rather than isolated text chunks.", "detail": [ "Conventional RAG retrieves chunks independently, which fails on questions requiring several documents to be connected — \"who influenced whom\", \"what evidence supports this\". GraphRAG extracts entities and relationships into a graph and retrieves neighbourhoods, communities or paths.", "The standard implementations — Microsoft GraphRAG, Neo4j — build the graph over *your* documents: you supply the corpus, run extraction, and operate the infrastructure. That is substantial work before the first query.", "A web-scale search API that ships a graph over the open web removes the ingestion step entirely. That is unusual: as of 2026, graph traversal over arbitrary web documents is largely unavailable as a product." ], "relatedTerms": [ "coverage-graph", "rag", "knowledge-graph", "context-assembly" ], "url": "https://www.unlob.com/glossary/graphrag" }, { "@type": "DefinedTerm", "name": "Knowledge graph", "slug": "knowledge-graph", "category": "graph", "description": "A knowledge graph stores entities and the relationships between them as a queryable structure of typed nodes and edges.", "detail": [ "Entities are nodes, relationships are typed edges, and both can carry attributes. The value is in the traversal: multi-hop questions that no amount of text retrieval answers become single queries.", "It is worth distinguishing a knowledge graph of *things* — a company with a headcount and a founder — from a graph of *documents and their coverage*. Diffbot builds the former; a coverage graph is the latter. Different structures for different questions." ], "relatedTerms": [ "coverage-graph", "graphrag", "entity-extraction" ], "url": "https://www.unlob.com/glossary/knowledge-graph" }, { "@type": "DefinedTerm", "name": "Centrality", "slug": "centrality", "category": "graph", "description": "Centrality measures how well-connected a node is within a graph, and is used to identify the documents a field treats as foundational.", "detail": [ "PageRank-style centrality scores a node by the weighted importance of what points at it, computed iteratively. In a coverage graph it identifies passages the rest of the corpus treats as load-bearing.", "It answers a different question from relevance. Relevance asks what matches the query; centrality asks what the field considers canonical. For an agent entering an unfamiliar domain the second is usually more useful — it surfaces what an expert would have read first.", "Computed at build time and stored as a fast field, so filtering and sorting on it costs nothing at query time." ], "relatedTerms": [ "coverage-graph", "host-rank", "ranking", "authority" ], "url": "https://www.unlob.com/glossary/centrality" }, { "@type": "DefinedTerm", "name": "Host rank", "slug": "host-rank", "category": "graph", "description": "Host rank scores the authority of a publishing domain from the link graph, independent of any particular query.", "detail": [ "Where centrality scores individual documents, host rank scores publishers. It is stable, query-independent, and the single bluntest effective filter against low-quality sources.", "It is about the publisher, not the page: a high-authority host can still publish a thin page, which is what a separate per-passage quality score is for." ], "relatedTerms": [ "centrality", "authority", "ranking", "quality-score" ], "url": "https://www.unlob.com/glossary/host-rank" }, { "@type": "DefinedTerm", "name": "Corroboration", "slug": "corroboration", "category": "graph", "description": "Corroboration counts how many independent sources assert the same claim, distinguishing widely reported facts from one story copied many times.", "detail": [ "Twelve copies of one wire story and six independent reports look identical in a ranked list — and the twelve copies look more convincing. That is exactly backwards, and it is a reliable way for an agent to become confidently wrong.", "Counting distinct asserting hosts separates them. The engineering subtlety is that deduplication must not destroy the count: when a near-duplicate is rejected at admission, its host has to be recorded against the surviving passage, or compaction erases the evidence.", "unlob exposes this as independent_sources on every hit, as a min_independent_sources filter, and through the corroborate endpoint." ], "relatedTerms": [ "coverage-graph", "deduplication", "story-cluster", "hallucination" ], "url": "https://www.unlob.com/glossary/corroboration" }, { "@type": "DefinedTerm", "name": "Story cluster", "slug": "story-cluster", "category": "graph", "description": "A story cluster groups documents reporting the same underlying event, across publishers and wordings.", "detail": [ "Clustering by story is what makes news retrieval usable. It converts forty syndicated copies into one item with a group size, and provides the unit over which corroboration is counted.", "Clusters are built from near-duplicate detection plus embedding similarity, so they catch both verbatim syndication and independent reporting of the same event." ], "relatedTerms": [ "near-duplicate-detection", "corroboration", "collapsing", "coverage-graph" ], "url": "https://www.unlob.com/glossary/story-cluster" }, { "@type": "DefinedTerm", "name": "Entity extraction", "slug": "entity-extraction", "category": "graph", "description": "Entity extraction identifies people, organisations, places and products mentioned in text and links them to canonical identifiers.", "detail": [ "The step that turns unstructured text into graph nodes. The hard part is not detection but resolution — deciding that two mentions refer to the same entity, and that a third refers to a different one with the same name.", "Once resolved, entities support queries that text retrieval cannot express: everything mentioning this organisation, everything co-mentioned with it, the shortest chain connecting it to something else." ], "relatedTerms": [ "knowledge-graph", "coverage-graph", "dossier" ], "url": "https://www.unlob.com/glossary/entity-extraction" }, { "@type": "DefinedTerm", "name": "Entity dossier", "slug": "dossier", "category": "graph", "description": "A dossier is a one-hop summary of an entity: where it is mentioned, which sources cover it, and which entities appear alongside it.", "detail": [ "Assembling this by hand means roughly ten searches and a manual merge, all in context and all billed. As a graph query it is a single traversal of an entity node.", "The co-mention list is frequently the most useful part — it surfaces the entities you did not know to ask about, which is usually where the next question comes from." ], "relatedTerms": [ "entity-extraction", "coverage-graph", "knowledge-graph" ], "url": "https://www.unlob.com/glossary/dossier" }, { "@type": "DefinedTerm", "name": "Community detection", "slug": "community-detection", "category": "graph", "description": "Community detection partitions a graph into clusters of nodes that reference each other far more than the rest of the graph.", "detail": [ "In a coverage graph, communities correspond to discourses — groups of publishers and documents that cite each other and share vocabulary. Label propagation is the usual algorithm at scale, because it is near-linear.", "The retrieval use is disambiguation. A term that means different things in different fields produces results spanning several communities; filtering to one keeps the agent inside a single discourse." ], "relatedTerms": [ "coverage-graph", "centrality", "filtering" ], "url": "https://www.unlob.com/glossary/community-detection" }, { "@type": "DefinedTerm", "name": "Provenance", "slug": "provenance", "category": "graph", "description": "Provenance records where a document came from and how it entered the index — which crawl, when fetched, and when published.", "detail": [ "Two timestamps matter and they are frequently confused. Fetch time is when the crawler retrieved the page; publication time is when the content says it was published. A page crawled yesterday may date from 2019.", "For auditable work, provenance is what lets you reconstruct the evidence base as it stood at a point in time — and separating the two clocks is what makes that possible." ], "relatedTerms": [ "coverage-transparency", "crawl-frontier", "freshness" ], "url": "https://www.unlob.com/glossary/provenance" }, { "@type": "DefinedTerm", "name": "Agentic search", "slug": "agentic-search", "category": "agents", "description": "Agentic search is web search consumed by an autonomous system rather than a person, which inverts most of the design assumptions of consumer search.", "detail": [ "A person skims ten results and discards nine at no cost. An agent pays for all ten in context and reasoning tokens, cannot ask a clarifying question, and cannot tell a good source from a plausible one without help.", "That changes what a search API should return: fewer, better results; metadata rather than page bodies; explicit trust signals rather than an implicit ranking; and a way to distinguish \"not found\" from \"does not exist\"." ], "relatedTerms": [ "recall-and-precision", "admission-control", "context-window", "mcp" ], "url": "https://www.unlob.com/glossary/agentic-search" }, { "@type": "DefinedTerm", "name": "Retrieval-augmented generation (RAG)", "slug": "rag", "category": "agents", "description": "RAG retrieves relevant documents and places them in a model’s context so it can answer from current, specific sources rather than from training data alone.", "detail": [ "The standard pattern for grounding a model: retrieve, assemble context, generate. It addresses staleness and specificity, and it makes answers checkable because the sources are known.", "In practice the retrieval half is where quality is won or lost, and most of the work is not retrieval at all — it is deduplication, trust assessment, reranking and truncation to fit a budget. That loop is expensive in exactly the way that is easy to overlook." ], "relatedTerms": [ "context-assembly", "graphrag", "context-window", "chunking" ], "url": "https://www.unlob.com/glossary/rag" }, { "@type": "DefinedTerm", "name": "Context assembly", "slug": "context-assembly", "category": "agents", "description": "Context assembly selects, orders and truncates retrieved passages to fit a token budget before they enter a model’s context.", "detail": [ "The step between retrieval and generation, and the one that most determines answer quality. Deduplicate, assess trust, rank, and cut to budget — usually implemented as bespoke application code that nobody revisits.", "Doing it server-side lets the engine use signals the application does not have, such as corroboration counts across rejected duplicates. Attaching a reason to each included passage makes the selection explainable, which matters when an answer has to be defended." ], "relatedTerms": [ "rag", "context-window", "corroboration", "graphrag" ], "url": "https://www.unlob.com/glossary/context-assembly" }, { "@type": "DefinedTerm", "name": "Context window", "slug": "context-window", "category": "agents", "description": "The context window is the maximum amount of text a model can consider at once, and it is the budget every retrieval decision spends against.", "detail": [ "Context is finite and priced per token, which makes retrieval a budgeting problem rather than a recall problem. Ten full page bodies can consume a window before the model has decided which page it wanted.", "Returning metadata first and full text on request is the direct response: the agent reads titles, snippets and trust signals, decides, and fetches only what it selected." ], "relatedTerms": [ "context-assembly", "rag", "agentic-search", "chunking" ], "url": "https://www.unlob.com/glossary/context-window" }, { "@type": "DefinedTerm", "name": "Model Context Protocol (MCP)", "slug": "mcp", "category": "agents", "description": "MCP is an open protocol for exposing tools and data sources to AI models through a uniform interface, so any compatible client can use any compatible server.", "detail": [ "Before MCP, every tool integration was bespoke per model and per framework. MCP standardises tool discovery and invocation over JSON-RPC, so one server works with any client that speaks the protocol.", "For a search API this means one implementation reaches Claude Code, Claude Desktop, Cursor, ChatGPT and anything else that adopts it — and the user configures it once.", "unlob exposes eleven tools over MCP, including all six coverage-graph operations." ], "relatedTerms": [ "agentic-search", "tool-use", "function-calling" ], "url": "https://www.unlob.com/glossary/mcp" }, { "@type": "DefinedTerm", "name": "Tool use", "slug": "tool-use", "category": "agents", "description": "Tool use lets a model invoke external functions — search, calculation, database queries — and incorporate the results into its reasoning.", "detail": [ "The mechanism that connects a model to anything outside its weights. The model receives tool descriptions, decides which to call and with what arguments, and continues with the result.", "Tool design matters more than it looks. A tool that exposes twenty parameters invites the model to guess badly at nineteen of them; a tool with two parameters and sensible pinned defaults produces far better results. Most retrieval quality problems in agents are filter problems in disguise." ], "relatedTerms": [ "mcp", "function-calling", "agentic-search" ], "url": "https://www.unlob.com/glossary/tool-use" }, { "@type": "DefinedTerm", "name": "Function calling", "slug": "function-calling", "category": "agents", "description": "Function calling is a model’s ability to emit structured arguments for a named function rather than free text, enabling reliable tool invocation.", "detail": [ "The model is given function schemas and returns a structured call the application executes. Schema-constrained output is what makes tool use reliable enough to build on.", "A tight schema is a feature rather than a limitation — the fewer degrees of freedom the model has, the fewer ways the call can be wrong." ], "relatedTerms": [ "tool-use", "mcp" ], "url": "https://www.unlob.com/glossary/function-calling" }, { "@type": "DefinedTerm", "name": "Hallucination", "slug": "hallucination", "category": "agents", "description": "A hallucination is a confident, plausible model output that is not supported by any source.", "detail": [ "Retrieval reduces hallucination by grounding answers in documents, but it does not eliminate it — an agent given twelve copies of one unverified claim will treat it as well established, because repetition looks like corroboration.", "Two structural defences help more than prompt engineering. Counting distinct asserting hosts distinguishes independently reported facts from echoes. Answering \"why is this URL absent\" prevents the agent inferring non-existence from an empty result set." ], "relatedTerms": [ "corroboration", "rag", "coverage-transparency", "agentic-search" ], "url": "https://www.unlob.com/glossary/hallucination" }, { "@type": "DefinedTerm", "name": "Chunking", "slug": "chunking", "category": "agents", "description": "Chunking splits documents into passages small enough to embed and retrieve independently.", "detail": [ "Embedding models have input limits, and retrieving a whole document when one paragraph is relevant wastes context. Chunking splits documents into retrievable units — the trade being that too-small chunks lose context and too-large ones dilute the embedding.", "Chunking on structural boundaries — sections, paragraphs — generally beats fixed token counts, because it respects the document's own units of meaning." ], "relatedTerms": [ "embedding", "rag", "passage", "context-window" ], "url": "https://www.unlob.com/glossary/chunking" }, { "@type": "DefinedTerm", "name": "Passage", "slug": "passage", "category": "agents", "description": "A passage is a retrievable unit of text — typically a section or several paragraphs — rather than a whole document.", "detail": [ "Retrieving passages rather than documents is more precise: the agent gets the relevant part rather than a whole page to search through itself.", "It also changes what corroboration means. Two passages from the same page are not two sources, which is why counting distinct hosts rather than distinct passages is the meaningful measure." ], "relatedTerms": [ "chunking", "corroboration", "rag" ], "url": "https://www.unlob.com/glossary/passage" }, { "@type": "DefinedTerm", "name": "Web crawler", "slug": "web-crawler", "category": "crawl", "description": "A web crawler discovers and fetches pages by following links, respecting robots.txt and rate limits.", "detail": [ "The component that acquires the corpus. It maintains a frontier of URLs to fetch, applies politeness constraints per host, and feeds an extraction pipeline.", "Crawling politely is a constraint on throughput and an ethical obligation: honour robots.txt including crawl-delay, identify yourself with a real user agent, and hold to roughly one request per second per host. Where a site blocks you, the correct response is to accept the gap rather than escalate." ], "relatedTerms": [ "robots-txt", "crawl-frontier", "crawl-on-miss", "freshness" ], "url": "https://www.unlob.com/glossary/web-crawler" }, { "@type": "DefinedTerm", "name": "robots.txt", "slug": "robots-txt", "category": "crawl", "description": "robots.txt is a file at a site’s root telling crawlers which paths they may fetch and how fast.", "detail": [ "The convention since 1994. Allow and Disallow rules are matched by longest prefix, with wildcards and end-anchors; Crawl-delay sets a minimum interval between requests.", "It is voluntary, which makes honouring it a statement about how a crawler operates. A missing or unreachable robots.txt is conventionally treated as permission, and a crawler should fail open rather than hammering a site that cannot serve the file." ], "relatedTerms": [ "web-crawler", "crawl-frontier" ], "url": "https://www.unlob.com/glossary/robots-txt" }, { "@type": "DefinedTerm", "name": "Crawl frontier", "slug": "crawl-frontier", "category": "crawl", "description": "The crawl frontier is the prioritised queue of URLs a crawler intends to fetch, subject to per-host politeness constraints.", "detail": [ "Managing the frontier is most of what a crawler does: deciding what is worth fetching next, when a host may next be contacted, and what to discard.", "Prioritisation is where crawl budget is won or lost. Crawling breadth-first spends heavily on regions nobody queries; prioritising by demand spends where there is demand." ], "relatedTerms": [ "web-crawler", "crawl-on-miss", "freshness" ], "url": "https://www.unlob.com/glossary/crawl-frontier" }, { "@type": "DefinedTerm", "name": "Crawl-on-miss", "slug": "crawl-on-miss", "category": "crawl", "description": "Crawl-on-miss triggers a fetch when a query finds nothing adequate, so the index fills gaps where there is demonstrated demand.", "detail": [ "Rather than pre-emptively crawling everything, record the gap and go and get it. The index improves precisely where people are looking rather than where a crawl schedule happened to point.", "The economic property is that cost tracks demand rather than corpus size, which is a materially different curve from crawling the web speculatively and hoping the coverage is useful." ], "relatedTerms": [ "crawl-frontier", "bounded-index", "coverage-transparency" ], "url": "https://www.unlob.com/glossary/crawl-on-miss" }, { "@type": "DefinedTerm", "name": "Freshness", "slug": "freshness", "category": "crawl", "description": "Freshness is how current an index is relative to the live web, maintained by recrawling changed pages rather than everything.", "detail": [ "The web changes at roughly one to five percent a day, so keeping an index current is a matter of detecting change rather than recrawling volume. Conditional requests make unchanged pages nearly free — a 304 response costs a round trip and no content.", "Sitemaps, feeds and last-modified headers narrow it further, which is why a well-behaved crawler asks for those before it asks for pages." ], "relatedTerms": [ "web-crawler", "crawl-frontier", "provenance" ], "url": "https://www.unlob.com/glossary/freshness" }, { "@type": "DefinedTerm", "name": "Boilerplate removal", "slug": "boilerplate-removal", "category": "crawl", "description": "Boilerplate removal strips navigation, headers, footers, cookie banners and adverts from a page to leave the actual content.", "detail": [ "A typical web page is mostly not content. Feeding raw HTML to a model wastes most of the tokens on navigation and legal notices, and dilutes retrieval quality with text that appears identically on every page of the site.", "Extraction identifies the main content block structurally. Doing it at index time rather than at query time means the cost is paid once per page rather than once per retrieval." ], "relatedTerms": [ "content-extraction", "passage", "chunking" ], "url": "https://www.unlob.com/glossary/boilerplate-removal" }, { "@type": "DefinedTerm", "name": "Content extraction", "slug": "content-extraction", "category": "crawl", "description": "Content extraction converts a fetched page into clean structured text, handling HTML, JavaScript-rendered pages and documents.", "detail": [ "A tiered pipeline is the practical approach: a fast structural reader for ordinary HTML, a model-based fallback for pages it cannot parse, and a headless browser for pages that only render under JavaScript. Each tier costs more than the last, so ordering them by cost matters.", "Doing extraction in-house avoids a per-page API bill that, at web scale, dominates every other cost in the pipeline." ], "relatedTerms": [ "boilerplate-removal", "web-crawler", "passage" ], "url": "https://www.unlob.com/glossary/content-extraction" }, { "@type": "DefinedTerm", "name": "Object-storage-first architecture", "slug": "object-storage-first", "category": "ops", "description": "An object-storage-first architecture keeps the authoritative copy of the data in object storage and treats compute nodes as replaceable caches rather than as the system of record.", "detail": [ "The conventional design keeps data on disks attached to always-running nodes, with a coordinator and metadata database tracking it. That standing infrastructure is most of the cost, and it scales with the size of the data rather than with usage.", "The pattern is now common in analytical databases and data lakes, where separating storage from compute is what allows the query tier to scale independently of the volume it queries. It is increasingly applied to search and vector indexes for the same reason." ], "relatedTerms": [ "stateless-serving", "resident-memory", "sharding" ], "url": "https://www.unlob.com/glossary/object-storage-first" }, { "@type": "DefinedTerm", "name": "Stateless serving", "slug": "stateless-serving", "category": "ops", "description": "A stateless serving tier holds no authoritative data, so any node can be replaced without coordination or data loss.", "detail": [ "If the authoritative copy of the index lives elsewhere, a serving node is a cache. It can be killed, replaced, autoscaled or moved with no migration and no rebalancing — which is what makes horizontal scaling an operational non-event rather than a project.", "The property a caller notices is failure behaviour: a stateless tier degrades rather than errors, because any node can answer any request." ], "relatedTerms": [ "object-storage-first", "sharding", "resident-memory" ], "url": "https://www.unlob.com/glossary/stateless-serving" }, { "@type": "DefinedTerm", "name": "Quality score", "slug": "quality-score", "category": "ops", "description": "A quality score rates an individual passage on text density, structure and extraction confidence, independently of its publisher’s authority.", "detail": [ "Host rank measures the publisher; quality measures the page. A well-regarded domain can still publish a thin, boilerplate-heavy page, and a quality floor is what catches it.", "Assigned at admission and stored as a fast field, so filtering on it costs nothing at query time." ], "relatedTerms": [ "host-rank", "admission-control", "fast-field" ], "url": "https://www.unlob.com/glossary/quality-score" }, { "@type": "DefinedTerm", "name": "Authority", "slug": "authority", "category": "ops", "description": "Authority is a coarse classification of a domain’s institutional class — educational, governmental, organisational, commercial — derived from the domain itself.", "detail": [ "Cruder than host rank and useful for a different reason: for regulatory, medical or academic questions, restricting to governmental and educational sources removes an entire category of noise in one parameter.", "It says nothing about quality within a class. A government page can be outdated and a commercial one authoritative; it is a filter on institutional type, not on correctness." ], "relatedTerms": [ "host-rank", "centrality", "filtering" ], "url": "https://www.unlob.com/glossary/authority" }, { "@type": "DefinedTerm", "name": "Multilingual search", "slug": "multilingual-search", "category": "ops", "description": "Multilingual search retrieves documents across languages, ideally by placing all languages in one embedding space so no translation step is needed.", "detail": [ "The naive approach runs a separate index per language and translates queries, which multiplies infrastructure and loses meaning at the translation boundary. A multilingual embedder places every language in one space, so an English query retrieves a relevant German passage directly.", "Languages whose scripts do not delimit words — Chinese, Japanese, Thai — need character-level tokenisation at both index and query time. unlob uses character bigrams, so 量子力学 indexes as 量子, 子力 and 力学." ], "relatedTerms": [ "embedding", "semantic-search", "tokenisation" ], "url": "https://www.unlob.com/glossary/multilingual-search" }, { "@type": "DefinedTerm", "name": "Tokenisation", "slug": "tokenisation", "category": "ops", "description": "Tokenisation splits text into indexable units — usually words, but character n-grams for scripts without word delimiters.", "detail": [ "Straightforward for space-delimited languages and genuinely hard otherwise. Chinese, Japanese and Thai write without spaces, so a tokeniser must either segment statistically or fall back to character n-grams.", "Character bigrams are robust: they need no language-specific model, they degrade gracefully on mixed-script text, and the same rule applies at index and query time so the two always agree." ], "relatedTerms": [ "multilingual-search", "inverted-index", "bm25" ], "url": "https://www.unlob.com/glossary/tokenisation" }, { "@type": "DefinedTerm", "name": "Rate limiting", "slug": "rate-limiting", "category": "ops", "description": "Rate limiting caps how many requests a client may make per interval, protecting a service from overload and enforcing plan tiers.", "detail": [ "Usually implemented per replica rather than globally, because a global counter needs shared state on the hot path. The consequence is that published limits are floors: a fleet of replicas gives an effective ceiling above the stated number.", "Distinguish it from a quota. A rate limit bounds requests per minute; a quota bounds them per billing period. Exceeding the first returns 429 and should be retried with backoff; exceeding the second is a billing event." ], "relatedTerms": [ "quota" ], "url": "https://www.unlob.com/glossary/rate-limiting" }, { "@type": "DefinedTerm", "name": "Quota", "slug": "quota", "category": "ops", "description": "A quota is the total number of requests allowed in a billing period, as distinct from a rate limit on requests per minute.", "detail": [ "Hitting a rate limit is temporary and resolves with backoff. Exhausting a quota is a billing decision: either requests stop, or overage applies.", "Hard-capping free tiers is the responsible default — with no card on file, a clear stop is better than an unexpected bill." ], "relatedTerms": [ "rate-limiting" ], "url": "https://www.unlob.com/glossary/quota" } ] }