Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/).
stanza. `brew` commands that load the cask printed "Calling `postflight` is
deprecated! Use `postflight_steps` instead." on every invocation. The
quarantine strip now runs through the declarative `postflight_steps` DSL.
- Entity extraction no longer treats numbers, versions, and addresses with no
letter (`0.52`, `127.0.0.1`) or dotted abbreviations (`e.g`, `i.e`) as file
paths, and no longer admits ordinary words written in capitals (`NEVER`,
`ONLY`, `NOTE`, `FIXED`) as acronyms. Such tokens linked unrelated memories
through entity edges.

## [0.2.7] - 2026-09-01

Expand Down
50 changes: 47 additions & 3 deletions internal/memory/graph/entity.go
Original file line number Diff line number Diff line change
Expand Up @@ -37,13 +37,16 @@ func ValidEntityMode(mode EntityMode) bool {
}
}

// filePathPattern matches file paths and dotted names (e.g., ./cmd/root.go,
// /etc/config.yml, example.org). Its matches must also pass looksLikePath.
var filePathPattern = regexp.MustCompile(`(?:^|[\s"'(])([.\w/-]+\.\w{1,10})(?:[\s"'),.]|$)`)

var entityPatterns = []*regexp.Regexp{
// CamelCase identifiers (e.g., MyClass, HttpServer)
regexp.MustCompile(`\b([A-Z][a-z]+(?:[A-Z][a-z]+)+)\b`),
// ALLCAPS acronyms 2-6 chars (e.g., API, HTTP, gRPC, SQL)
regexp.MustCompile(`\b([A-Z]{2,6})\b`),
// File paths (e.g., ./cmd/root.go, /etc/config.yml)
regexp.MustCompile(`(?:^|[\s"'(])([.\w/-]+\.\w{1,10})(?:[\s"'),.]|$)`),
filePathPattern,
// URLs
regexp.MustCompile(`https?://[^\s"'<>)]+`),
// @mentions
Expand Down Expand Up @@ -102,7 +105,12 @@ var techDictionary = map[string]bool{
"MAGMA": true, "MCP": true, "RLM": true,
}

// acronymStopwords filters out common English acronyms that aren't tech entities.
// acronymStopwords filters out all-caps tokens that aren't tech entities:
// short common English words, and ordinary words written in capitals for
// emphasis or as labels (NEVER, ONLY, NOTE, FIXED). In a real store the latter
// were among the most frequent "entities" and linked unrelated memories.
// Words that double as technical names (HEAD, POST, PATH, HOME, REST, DELETE,
// LINE) are deliberately absent.
var acronymStopwords = map[string]bool{
"IN": true, "ON": true, "AT": true, "TO": true, "BY": true,
"OR": true, "AN": true, "IF": true, "IS": true, "IT": true,
Expand All @@ -117,6 +125,23 @@ var acronymStopwords = map[string]bool{
"SHE": true, "TWO": true, "USE": true, "BOY": true, "DID": true,
"GET": true, "HIM": true, "HIS": true, "LET": true, "PUT": true,
"TOP": true, "TOO": true, "ANY": true,
// Ordinary words written in capitals for emphasis or as labels.
"OK": true, "YES": true, "TRUE": true, "FALSE": true,
"ONLY": true, "BOTH": true, "NEVER": true, "ALWAYS": true, "MUST": true,
"STILL": true, "SAME": true, "ALSO": true, "EVERY": true, "OTHER": true,
"OWN": true, "REAL": true, "WORTH": true, "HARD": true, "FULL": true,
"EMPTY": true, "ZERO": true, "KEEP": true, "SKIP": true, "LEFT": true,
"WITH": true, "FROM": true, "THAT": true, "THIS": true, "THAN": true,
"WHAT": true, "WHY": true, "DOES": true, "CANNOT": true, "NEEDS": true,
"BEFORE": true, "AFTER": true, "FIRST": true, "SECOND": true, "THIRD": true,
"FINAL": true, "THREE": true, "FOUR": true, "FIVE": true, "HIGH": true,
"PASS": true, "FAIL": true, "FAILED": true, "FIX": true, "FIXED": true,
"DONE": true, "WRONG": true, "STALE": true, "ARMED": true, "OPEN": true,
"CLOSED": true, "FOUND": true, "SENT": true, "SAFE": true, "CLEAN": true,
"VERIFY": true, "CHECK": true, "REVIEW": true, "DRAFT": true, "PROOF": true,
"NOTE": true, "NOTES": true, "LESSON": true, "RULE": true, "FACT": true,
"FACTS": true, "DEFECT": true, "CAUSE": true, "GOTCHA": true, "TRAP": true,
"BUG": true, "ISSUE": true, "ERROR": true,
}

// ExtractEntities extracts named entities from text using regex patterns
Expand All @@ -137,6 +162,9 @@ func ExtractEntities(text string) []string {
if acronymStopwords[entity] {
continue
}
if pat == filePathPattern && !looksLikePath(entity) {
continue
}
seen[entity] = true
entities = append(entities, entity)
}
Expand All @@ -154,6 +182,22 @@ func ExtractEntities(text string) []string {
return entities
}

// looksLikePath reports whether a file-path match could name a file or
// domain. The path pattern also matches numbers, versions, and addresses with
// no letter (0.52, 127.0.0.1) and dotted single letters (e.g, i.e, J.Y, p.4);
// in a real store those were thousands of entities that named nothing.
func looksLikePath(s string) bool {
if !strings.ContainsFunc(s, func(r rune) bool { return (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') }) {
return false
}
for _, segment := range strings.Split(s, ".") {
if len(segment) > 1 {
return true
}
}
return false
}

// ResolveEntities returns the final entity list for a content/provided pair.
func ResolveEntities(content string, provided []string, mode EntityMode) []string {
return ResolveEntitiesIndexed(content, provided, mode, nil)
Expand Down
28 changes: 28 additions & 0 deletions internal/memory/graph/entity_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -399,6 +399,34 @@ func TestResolveEntitiesIndexed_NilIndexEqualsResolveEntities(t *testing.T) {
}
}

func TestExtractEntities_FilePathsSkipNumbersAndAbbreviations(t *testing.T) {
entities := ExtractEntities("Scores 0.52 and 127.0.0.1, e.g. the J.Y. case on p.4, live in cmd/root.go, llama.cpp and example.org")
has := toSet(entities)
for _, want := range []string{"cmd/root.go", "llama.cpp", "example.org"} {
if !has[want] {
t.Errorf("want path %q, got %v", want, entities)
}
}
for _, noise := range []string{"0.52", "127.0.0.1", "e.g", "J.Y", "p.4"} {
if has[noise] {
t.Errorf("number or abbreviation %q must not be an entity, got %v", noise, entities)
}
}
}

func TestExtractEntities_EmphasisWordsAreNotAcronyms(t *testing.T) {
entities := ExtractEntities("NEVER skip the API check; ONLY use HTTP. Status: OK, FIXED")
has := toSet(entities)
if !has["API"] || !has["HTTP"] {
t.Errorf("want acronyms API and HTTP, got %v", entities)
}
for _, word := range []string{"NEVER", "ONLY", "OK", "FIXED"} {
if has[word] {
t.Errorf("emphasis word %q must not be an entity, got %v", word, entities)
}
}
}

func equalAsSets(a, b []string) bool {
if len(a) != len(b) {
return false
Expand Down
Loading