From 24846eb9de199e2424ff9c2f83faa3b46dc4ec0c Mon Sep 17 00:00:00 2001 From: Roger Johansson Date: Thu, 22 Jan 2026 21:31:28 +0100 Subject: [PATCH] Align README scoring docs with Score --- README.md | 8 ++-- cmd/quickdup/strategy_score_test.go | 68 +++++++++++++++++++++++++++++ 2 files changed, 72 insertions(+), 4 deletions(-) create mode 100644 cmd/quickdup/strategy_score_test.go diff --git a/README.md b/README.md index 77f7aa7..d25fc8c 100644 --- a/README.md +++ b/README.md @@ -65,10 +65,10 @@ Patterns with similar structure but different actual code are filtered: 1. Tokenize source lines of each occurrence 2. Compute Jaccard similarity (intersection/union of token sets) -3. Filter patterns below threshold (default: 50%) -4. Score patterns: `uniqueWords + (similarity × 5)` +3. Filter patterns below threshold (default: 75%) +4. Score patterns: `max(uniqueWords - indentImbalance, 0) * adjustedSimilarity^3 + (patternLength / 20)` -This eliminates most false positives like "all error handlers look similar structurally but have different messages." High similarity (especially 100% verbatim matches) boosts the score, surfacing the most actionable duplications first. +Where `adjustedSimilarity = max(similarity * 2 - 1, 0)`. This eliminates most false positives like "all error handlers look similar structurally but have different messages." High similarity (especially 100% verbatim matches) is heavily rewarded, surfacing the most actionable duplications first. ### Phase 4: Output @@ -133,7 +133,7 @@ quickdup -path . -ext .go -debug | `-min` | `2` | Minimum occurrences to report | | `-min-size` | `3` | Base pattern size (lines) to start growing from | | `-max-size` | `0` | Maximum pattern size to grow to (0 = no limit) | -| `-min-score` | `5` | Minimum score (unique words + similarity bonus) | +| `-min-score` | `5` | Minimum score (effective words * adjusted similarity^3 + size bonus) | | `-min-similarity` | `0.75` | Minimum token similarity between occurrences (0.0-1.0) | | `-top` | `10` | Show top N patterns by score | | `-select` | | Show detailed output for patterns (format: `skip..limit`) | diff --git a/cmd/quickdup/strategy_score_test.go b/cmd/quickdup/strategy_score_test.go new file mode 100644 index 0000000..417f20c --- /dev/null +++ b/cmd/quickdup/strategy_score_test.go @@ -0,0 +1,68 @@ +package main + +import "testing" + +func TestNormalizedIndentScoreBalancedFullSimilarity(t *testing.T) { + strategy := &NormalizedIndentStrategy{} + entries := []Entry{ + NewNormalizedIndentEntry(1, "if"), + NewNormalizedIndentEntry(0, "return"), + NewNormalizedIndentEntry(-1, "}"), + } + + score := strategy.Score(entries, 1.0) + if score != 3 { + t.Fatalf("expected score 3, got %d", score) + } +} + +func TestNormalizedIndentScoreImbalancePenalty(t *testing.T) { + strategy := &NormalizedIndentStrategy{} + entries := []Entry{ + NewNormalizedIndentEntry(-1, "}"), + NewNormalizedIndentEntry(0, "return"), + } + + score := strategy.Score(entries, 1.0) + if score != 1 { + t.Fatalf("expected score 1, got %d", score) + } +} + +func TestNormalizedIndentScoreLowSimilarityDropsWordBonus(t *testing.T) { + strategy := &NormalizedIndentStrategy{} + entries := []Entry{ + NewNormalizedIndentEntry(1, "if"), + NewNormalizedIndentEntry(-1, "}"), + } + + score := strategy.Score(entries, 0.4) + if score != 0 { + t.Fatalf("expected score 0, got %d", score) + } +} + +func TestWordIndentScoreSizeBonusAndSimilarityCurve(t *testing.T) { + strategy := &WordIndentStrategy{} + words := []string{ + "if", + "for", + "switch", + "return", + "case", + "else", + "break", + "continue", + "goto", + "defer", + } + entries := make([]Entry, 0, 20) + for i := 0; i < 20; i++ { + entries = append(entries, NewWordIndentEntry(0, words[i%len(words)])) + } + + score := strategy.Score(entries, 0.85) + if score != 4 { + t.Fatalf("expected score 4, got %d", score) + } +}