Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
37 changes: 37 additions & 0 deletions postgres/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -266,6 +266,43 @@ mod tests {
assert_eq!(inspected, Some(vec!["rare".to_owned()]));
}

#[pg_test]
fn max_score_excludes_nonmatching_documents() {
for (name, matching, nonmatching, query) in [
(
"boolean",
"beer wine",
"beer beer beer beer beer",
"beer^1 AND wine^0",
),
(
"phrase",
"beer beer wine",
"beer noise beer noise beer noise beer noise wine",
"\"beer beer\"^1 AND wine^0",
),
(
"positional",
"beer wine",
"wine beer beer beer beer beer",
"beer BEFORE wine^0",
),
] {
Spi::run(&format!(
"CREATE TABLE lite_max_score_{name} (body text);
INSERT INTO lite_max_score_{name} VALUES ('{matching}'), ('{nonmatching}');
CREATE INDEX ON lite_max_score_{name} USING tin (body);"
))
.unwrap();
let (score, max) = Spi::get_two::<f32, f32>(&format!(
"SELECT tin.full_score(ctid), tin.max_score(ctid)
FROM lite_max_score_{name} WHERE body ==> '{query}'"
))
.unwrap();
assert_eq!(max, score, "{name}");
}
}

#[pg_test]
fn full_score_normalization_matches_tin() {
Spi::run(
Expand Down
9 changes: 7 additions & 2 deletions postgres/src/score.rs
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ use pgrx::{
use rustc_hash::FxHashMap;
use std::cell::RefCell;
use std::ffi::{CStr, CString, c_void};
use tinql::runtime::{Query, SpanTermSlot, parse_tinql_to_query};
use tinql::runtime::{Query, SpanTermSlot, evaluate, parse_tinql_to_query, tokenize_doc};
use tokenizer::{CompiledTokenizerPipeline, Tokenizer};

#[derive(Clone, Debug, Eq, PartialEq)]
Expand Down Expand Up @@ -220,7 +220,12 @@ fn build_corpus(
scorer.score_count(tf, tokens.len() as u32)
}
}));
max = max.max(score);
let matched = evaluate(&query, &tokenize_doc(&document, &tokenizer))
.unwrap_or_else(|error| pgrx::error!("tin score query evaluation failed: {error}"))
.matched;
if matched {
max = max.max(score);
}
by_document.insert(document, score);
}
ScoreCorpus {
Expand Down