From ad306600b8fa48717495953a6f5077cc1c0b5349 Mon Sep 17 00:00:00 2001 From: John Lambert Date: Thu, 8 Oct 2026 21:47:19 -0400 Subject: [PATCH 01/20] Restore analysis of literal rewrites without phonological features --- .../FeatureAnalysisRewriteRuleSpec.cs | 18 ++++++ .../FeatureModel/StringFeatureValue.cs | 5 ++ .../StrRepRewriteRuleTests.cs | 59 +++++++++++++++++++ 3 files changed, 82 insertions(+) create mode 100644 tests/SIL.Machine.Morphology.HermitCrab.Tests/PhonologicalRules/StrRepRewriteRuleTests.cs diff --git a/src/SIL.Machine.Morphology.HermitCrab/PhonologicalRules/FeatureAnalysisRewriteRuleSpec.cs b/src/SIL.Machine.Morphology.HermitCrab/PhonologicalRules/FeatureAnalysisRewriteRuleSpec.cs index 588f12fdf..575b1e185 100644 --- a/src/SIL.Machine.Morphology.HermitCrab/PhonologicalRules/FeatureAnalysisRewriteRuleSpec.cs +++ b/src/SIL.Machine.Morphology.HermitCrab/PhonologicalRules/FeatureAnalysisRewriteRuleSpec.cs @@ -55,6 +55,13 @@ Tuple, PatternNode> tuple in lhs.C FeatureStruct fs = rhsConstraint.FeatureStruct.AntiFeatureStruct(); fs.Subtract(lhsConstraint.FeatureStruct.AntiFeatureStruct()); + StringFeatureValue lhsStrRep; + if (lhsConstraint.FeatureStruct.TryGetValue(HCFeatureSystem.StrRep, out lhsStrRep)) + { + // StrRep is a built-in feature: inversion adds the input spelling, not its complement. + fs.AddValue(HCFeatureSystem.StrRep, lhsStrRep); + rhsAntiFSs[i].AddValue(HCFeatureSystem.StrRep, lhsStrRep); + } fs.AddValue(HCFeatureSystem.Type, HCFeatureSystem.Segment); _analysisRhs.Children.Add(new Constraint(fs)); @@ -95,6 +102,17 @@ private bool IsUnapplicationNonvacuous(Match match, IEnumerable } } } + foreach (StringFeature sf in fs.Features.OfType()) + { + StringFeatureValue nodeSfv; + if ( + node.Annotation.FeatureStruct.TryGetValue(sf, out nodeSfv) + && !nodeSfv.IsSupersetOf(fs.GetValue(sf)) + ) + { + return true; + } + } i++; } diff --git a/src/SIL.Machine/FeatureModel/StringFeatureValue.cs b/src/SIL.Machine/FeatureModel/StringFeatureValue.cs index ae05be2a9..6df5eaf05 100644 --- a/src/SIL.Machine/FeatureModel/StringFeatureValue.cs +++ b/src/SIL.Machine/FeatureModel/StringFeatureValue.cs @@ -80,6 +80,11 @@ public bool Overlaps(IEnumerable strings) return Not ? !_values.Overlaps(strings) : _values.Overlaps(strings); } + public bool IsSupersetOf(StringFeatureValue other, bool notOther = false) + { + return IsSupersetOf(false, other, notOther); + } + protected override bool IsSupersetOf(bool not, SimpleFeatureValue other, bool notOther) { if (!(other is StringFeatureValue otherSfv)) diff --git a/tests/SIL.Machine.Morphology.HermitCrab.Tests/PhonologicalRules/StrRepRewriteRuleTests.cs b/tests/SIL.Machine.Morphology.HermitCrab.Tests/PhonologicalRules/StrRepRewriteRuleTests.cs new file mode 100644 index 000000000..e669cda71 --- /dev/null +++ b/tests/SIL.Machine.Morphology.HermitCrab.Tests/PhonologicalRules/StrRepRewriteRuleTests.cs @@ -0,0 +1,59 @@ +using NUnit.Framework; +using SIL.Machine.Annotations; +using SIL.Machine.FeatureModel; +using SIL.Machine.Matching; + +namespace SIL.Machine.Morphology.HermitCrab.PhonologicalRules; + +public class StrRepRewriteRuleTests +{ + [Test] + public void LiteralRewriteWithoutFeaturesAnalyzesItsSynthesisAndRejectsVacuousUnapplication() + { + var table = new CharacterDefinitionTable(); + foreach (string segment in new[] { "m", "p", "x", "u", "a" }) + table.AddSegment(segment); + var language = new Language(); + language.SyntacticFeatureSystem.AddPartsOfSpeech(new FeatureSymbol("N")); + var stratum = new Stratum(table); + language.Strata.Add(stratum); + var entries = new Dictionary(); + foreach (string form in new[] { "xmuma", "muma", "xuma" }) + { + var entry = new LexEntry + { + Id = form, + SyntacticFeatureStruct = FeatureStruct.New(language.SyntacticFeatureSystem).Symbol("N").Value, + }; + entry.Allomorphs.Add(new RootAllomorph(new Segments(table, form))); + stratum.Entries.Add(entry); + entries.Add(form, entry); + } + var rule = new RewriteRule { Lhs = Pattern.New().Annotation(table["m"].FeatureStruct).Value }; + rule.Subrules.Add( + new RewriteSubrule + { + Rhs = Pattern.New().Annotation(table["p"].FeatureStruct).Value, + LeftEnvironment = Pattern.New().Annotation(table["x"].FeatureStruct).Value, + } + ); + stratum.PhonologicalRules.Add(rule); + var morpher = new Morpher(new TraceManager(), language); + + Assert.That( + morpher.GenerateWords(entries["xmuma"], Array.Empty(), new FeatureStruct()), + Is.EquivalentTo(new[] { "xpuma" }) + ); + Assert.That(morpher.ParseWord("xpuma").Select(w => w.RootAllomorph.Morpheme.Id), Is.EqualTo(new[] { "xmuma" })); + Assert.That(morpher.ParseWord("muma").Select(w => w.RootAllomorph.Morpheme.Id), Is.EqualTo(new[] { "muma" })); + Assert.That(morpher.ParseWord("xuma").Select(w => w.RootAllomorph.Morpheme.Id), Is.EqualTo(new[] { "xuma" })); + Assert.That(morpher.ParseWord("xmuma"), Is.Empty); + Assert.That(morpher.ParseWord("puma"), Is.Empty); + + var analysisRule = new AnalysisRewriteRule(morpher, rule); + var word = new Word(stratum, new Segments(table, "xpuma").Shape.Clone()); + Assert.That(analysisRule.Apply(word), Is.Not.Empty); + word.ResetDirty(); + Assert.That(analysisRule.Apply(word), Is.Empty); + } +} From a02432ea50e80c7c0f708542ac051e40be1fbaf2 Mon Sep 17 00:00:00 2001 From: John Lambert Date: Thu, 8 Oct 2026 21:47:19 -0400 Subject: [PATCH 02/20] Shorten a pre-existing comment rejected by the hygiene gate --- src/SIL.Machine/Corpora/ScrVersExtensions.cs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/SIL.Machine/Corpora/ScrVersExtensions.cs b/src/SIL.Machine/Corpora/ScrVersExtensions.cs index 620074a68..f8cab53db 100644 --- a/src/SIL.Machine/Corpora/ScrVersExtensions.cs +++ b/src/SIL.Machine/Corpora/ScrVersExtensions.cs @@ -67,8 +67,7 @@ public static bool IsEquivalentTo(this ScrVers scrVers, ScrVers other) if (scrVers.Equals(other)) return true; - // If all verses in the versifications are 1) equal (accounts for mapping) - // and 2) graphically identical in regard to book, chapter, and verse, then the versifications are equivalent + // Equivalence requires both mapped equality and identical book, chapter, and verse numbers. foreach ( (VerseRef thisVerse, VerseRef otherVerse) in scrVers .AllIncludedVerses() From 51cb654f519a9a59d19e90c7a80efaf0b71e7a3c Mon Sep 17 00:00:00 2001 From: John Lambert Date: Thu, 8 Oct 2026 22:36:26 -0400 Subject: [PATCH 03/20] Pin analysis of kad and rejection of kat after a literal rewrite --- .../StrRepRewriteRuleTests.cs | 38 +++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/tests/SIL.Machine.Morphology.HermitCrab.Tests/PhonologicalRules/StrRepRewriteRuleTests.cs b/tests/SIL.Machine.Morphology.HermitCrab.Tests/PhonologicalRules/StrRepRewriteRuleTests.cs index e669cda71..52f5bc7e1 100644 --- a/tests/SIL.Machine.Morphology.HermitCrab.Tests/PhonologicalRules/StrRepRewriteRuleTests.cs +++ b/tests/SIL.Machine.Morphology.HermitCrab.Tests/PhonologicalRules/StrRepRewriteRuleTests.cs @@ -7,6 +7,44 @@ namespace SIL.Machine.Morphology.HermitCrab.PhonologicalRules; public class StrRepRewriteRuleTests { + [Test] + public void LiteralRewriteAfterSegmentClassAnalyzesKadAndRejectsKat() + { + var table = new CharacterDefinitionTable(); + foreach (string segment in new[] { "a", "t", "d", "k" }) + table.AddSegment(segment); + var language = new Language(); + language.SyntacticFeatureSystem.AddPartsOfSpeech(new FeatureSymbol("N")); + var stratum = new Stratum(table); + language.Strata.Add(stratum); + var entry = new LexEntry + { + Id = "kat", + SyntacticFeatureStruct = FeatureStruct.New(language.SyntacticFeatureSystem).Symbol("N").Value, + }; + entry.Allomorphs.Add(new RootAllomorph(new Segments(table, "kat"))); + stratum.Entries.Add(entry); + var vowels = new SegmentNaturalClass(new[] { table["a"] }); + var rule = new RewriteRule { Lhs = Pattern.New().Annotation(table["t"].FeatureStruct).Value }; + rule.Subrules.Add( + new RewriteSubrule + { + Rhs = Pattern.New().Annotation(table["d"].FeatureStruct).Value, + LeftEnvironment = Pattern.New().Annotation(vowels.FeatureStruct).Value, + } + ); + stratum.PhonologicalRules.Add(rule); + var morpher = new Morpher(new TraceManager(), language); + + Assert.That( + morpher.GenerateWords(entry, Array.Empty(), new FeatureStruct()), + Is.EquivalentTo(new[] { "kad" }) + ); + Assert.That(morpher.ParseWord("kad").Select(w => w.RootAllomorph.Morpheme.Id), Is.EqualTo(new[] { "kat" })); + foreach (string neighbor in new[] { "kat", "kak", "dat", "tat", "kta", "ka" }) + Assert.That(morpher.ParseWord(neighbor), Is.Empty, neighbor); + } + [Test] public void LiteralRewriteWithoutFeaturesAnalyzesItsSynthesisAndRejectsVacuousUnapplication() { From fbcf36a5199f5d73a980f5a7fbc5a9d60d0633c8 Mon Sep 17 00:00:00 2001 From: John Lambert Date: Tue, 20 Oct 2026 14:12:05 -0400 Subject: [PATCH 04/20] Give each analysis rewrite its own copy of the literal StrRep value instead of sharing one instance Co-Authored-By: Claude Opus 5.5 --- .../PhonologicalRules/FeatureAnalysisRewriteRuleSpec.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/SIL.Machine.Morphology.HermitCrab/PhonologicalRules/FeatureAnalysisRewriteRuleSpec.cs b/src/SIL.Machine.Morphology.HermitCrab/PhonologicalRules/FeatureAnalysisRewriteRuleSpec.cs index 575b1e185..13ccb09a1 100644 --- a/src/SIL.Machine.Morphology.HermitCrab/PhonologicalRules/FeatureAnalysisRewriteRuleSpec.cs +++ b/src/SIL.Machine.Morphology.HermitCrab/PhonologicalRules/FeatureAnalysisRewriteRuleSpec.cs @@ -59,8 +59,8 @@ Tuple, PatternNode> tuple in lhs.C if (lhsConstraint.FeatureStruct.TryGetValue(HCFeatureSystem.StrRep, out lhsStrRep)) { // StrRep is a built-in feature: inversion adds the input spelling, not its complement. - fs.AddValue(HCFeatureSystem.StrRep, lhsStrRep); - rhsAntiFSs[i].AddValue(HCFeatureSystem.StrRep, lhsStrRep); + fs.AddValue(HCFeatureSystem.StrRep, lhsStrRep.Clone()); + rhsAntiFSs[i].AddValue(HCFeatureSystem.StrRep, lhsStrRep.Clone()); } fs.AddValue(HCFeatureSystem.Type, HCFeatureSystem.Segment); _analysisRhs.Children.Add(new Constraint(fs)); From f915dd285235818764b2fe1e0d5a07da2261f67b Mon Sep 17 00:00:00 2001 From: John Lambert Date: Tue, 20 Oct 2026 14:13:12 -0400 Subject: [PATCH 05/20] Run the literal-rewrite analysis test in both Iterative and Simultaneous application modes Co-Authored-By: Claude Opus 5.5 --- .../PhonologicalRules/StrRepRewriteRuleTests.cs | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/tests/SIL.Machine.Morphology.HermitCrab.Tests/PhonologicalRules/StrRepRewriteRuleTests.cs b/tests/SIL.Machine.Morphology.HermitCrab.Tests/PhonologicalRules/StrRepRewriteRuleTests.cs index 52f5bc7e1..7249dfac8 100644 --- a/tests/SIL.Machine.Morphology.HermitCrab.Tests/PhonologicalRules/StrRepRewriteRuleTests.cs +++ b/tests/SIL.Machine.Morphology.HermitCrab.Tests/PhonologicalRules/StrRepRewriteRuleTests.cs @@ -45,8 +45,11 @@ public void LiteralRewriteAfterSegmentClassAnalyzesKadAndRejectsKat() Assert.That(morpher.ParseWord(neighbor), Is.Empty, neighbor); } - [Test] - public void LiteralRewriteWithoutFeaturesAnalyzesItsSynthesisAndRejectsVacuousUnapplication() + [TestCase(RewriteApplicationMode.Iterative)] + [TestCase(RewriteApplicationMode.Simultaneous)] + public void LiteralRewriteWithoutFeaturesAnalyzesItsSynthesisAndRejectsVacuousUnapplication( + RewriteApplicationMode applicationMode + ) { var table = new CharacterDefinitionTable(); foreach (string segment in new[] { "m", "p", "x", "u", "a" }) @@ -67,7 +70,11 @@ public void LiteralRewriteWithoutFeaturesAnalyzesItsSynthesisAndRejectsVacuousUn stratum.Entries.Add(entry); entries.Add(form, entry); } - var rule = new RewriteRule { Lhs = Pattern.New().Annotation(table["m"].FeatureStruct).Value }; + var rule = new RewriteRule + { + Lhs = Pattern.New().Annotation(table["m"].FeatureStruct).Value, + ApplicationMode = applicationMode, + }; rule.Subrules.Add( new RewriteSubrule { From ebd3a3837bfd363157e461386fb1d91e204d57ab Mon Sep 17 00:00:00 2001 From: John Lambert Date: Wed, 21 Oct 2026 10:00:31 -0400 Subject: [PATCH 06/20] Add MultiSpellingLiteralRewriteAnalyzesEachSourceSpellingAndRejectsUnrewrittenForms covering a {t,k} -> d StrRep lhs in both application modes Co-Authored-By: Claude Opus 5.5 --- .../StrRepRewriteRuleTests.cs | 62 +++++++++++++++++++ 1 file changed, 62 insertions(+) diff --git a/tests/SIL.Machine.Morphology.HermitCrab.Tests/PhonologicalRules/StrRepRewriteRuleTests.cs b/tests/SIL.Machine.Morphology.HermitCrab.Tests/PhonologicalRules/StrRepRewriteRuleTests.cs index 7249dfac8..f93c70ca5 100644 --- a/tests/SIL.Machine.Morphology.HermitCrab.Tests/PhonologicalRules/StrRepRewriteRuleTests.cs +++ b/tests/SIL.Machine.Morphology.HermitCrab.Tests/PhonologicalRules/StrRepRewriteRuleTests.cs @@ -101,4 +101,66 @@ RewriteApplicationMode applicationMode word.ResetDirty(); Assert.That(analysisRule.Apply(word), Is.Empty); } + + [TestCase(RewriteApplicationMode.Iterative)] + [TestCase(RewriteApplicationMode.Simultaneous)] + public void MultiSpellingLiteralRewriteAnalyzesEachSourceSpellingAndRejectsUnrewrittenForms( + RewriteApplicationMode applicationMode + ) + { + var table = new CharacterDefinitionTable(); + foreach (string segment in new[] { "a", "t", "k", "d", "p" }) + table.AddSegment(segment); + var language = new Language(); + language.SyntacticFeatureSystem.AddPartsOfSpeech(new FeatureSymbol("N")); + var stratum = new Stratum(table); + language.Strata.Add(stratum); + var entries = new Dictionary(); + foreach (string form in new[] { "ata", "aka", "apa" }) + { + var entry = new LexEntry + { + Id = form, + SyntacticFeatureStruct = FeatureStruct.New(language.SyntacticFeatureSystem).Symbol("N").Value, + }; + entry.Allomorphs.Add(new RootAllomorph(new Segments(table, form))); + stratum.Entries.Add(entry); + entries.Add(form, entry); + } + var stops = new SegmentNaturalClass(new[] { table["t"], table["k"] }); + var rule = new RewriteRule + { + Lhs = Pattern.New().Annotation(stops.FeatureStruct).Value, + ApplicationMode = applicationMode, + }; + rule.Subrules.Add( + new RewriteSubrule + { + Rhs = Pattern.New().Annotation(table["d"].FeatureStruct).Value, + LeftEnvironment = Pattern.New().Annotation(table["a"].FeatureStruct).Value, + } + ); + stratum.PhonologicalRules.Add(rule); + var morpher = new Morpher(new TraceManager(), language); + + foreach (string form in new[] { "ata", "aka" }) + { + Assert.That( + morpher.GenerateWords(entries[form], Array.Empty(), new FeatureStruct()), + Is.EquivalentTo(new[] { "ada" }), + form + ); + } + Assert.That( + morpher.ParseWord("ada").Select(w => w.RootAllomorph.Morpheme.Id), + Is.EquivalentTo(new[] { "ata", "aka" }) + ); + Assert.That( + morpher.GenerateWords(entries["apa"], Array.Empty(), new FeatureStruct()), + Is.EquivalentTo(new[] { "apa" }) + ); + Assert.That(morpher.ParseWord("apa").Select(w => w.RootAllomorph.Morpheme.Id), Is.EqualTo(new[] { "apa" })); + foreach (string unrewritten in new[] { "ata", "aka", "adp" }) + Assert.That(morpher.ParseWord(unrewritten), Is.Empty, unrewritten); + } } From 28ef6160bb9fb911cffe0b43d90035aad6d16260 Mon Sep 17 00:00:00 2001 From: John Lambert Date: Thu, 24 Sep 2026 11:02:34 -0400 Subject: [PATCH 07/20] Prune reduplication matches whose copies cannot agree When unapplying an affix process rule that copies a part more than once, Morpher.PruneDisagreeingCopies (off by default) skips matches whose copies cannot unify segment by segment. Synthesis writes every copy from the same input, so such a match never survives synthesis. Copies containing an optional node or modified by the rule are always kept. Co-Authored-By: Claude Opus 5.5 --- .../Morpher.cs | 8 ++ .../AnalysisAffixProcessRule.cs | 3 +- .../AnalysisMorphologicalTransform.cs | 76 ++++++++++++++++++ .../AnalysisRealizationalAffixProcessRule.cs | 3 +- .../CopyAgreementPatternRule.cs | 38 +++++++++ .../MorphologicalOutputAction.cs | 5 ++ .../AffixProcessRuleTests.cs | 28 ++++--- .../CopyAgreementPruneTests.cs | 79 +++++++++++++++++++ 8 files changed, 225 insertions(+), 15 deletions(-) create mode 100644 src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/CopyAgreementPatternRule.cs create mode 100644 tests/SIL.Machine.Morphology.HermitCrab.Tests/MorphologicalRules/CopyAgreementPruneTests.cs diff --git a/src/SIL.Machine.Morphology.HermitCrab/Morpher.cs b/src/SIL.Machine.Morphology.HermitCrab/Morpher.cs index 26660edc1..9712dac42 100644 --- a/src/SIL.Machine.Morphology.HermitCrab/Morpher.cs +++ b/src/SIL.Machine.Morphology.HermitCrab/Morpher.cs @@ -110,6 +110,14 @@ public ITraceManager TraceManager /// public bool AlwaysEnforceFinalTemplates { get; set; } + /// + /// When unapplying a rule that copies a part more than once (reduplication), skip matches whose + /// copies cannot unify segment by segment. Such a match can never survive synthesis, so this removes + /// only doomed analyses; on reduplication-heavy grammars it is the difference between a parse that + /// exhausts memory and one that finishes in seconds. + /// + public bool PruneDisagreeingCopies { get; set; } + /// /// Caps the concurrency used within a single parse or generation -- analysis cascade, /// affix-template unapplication and synthesis alike. A value of 1 runs the work fully diff --git a/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisAffixProcessRule.cs b/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisAffixProcessRule.cs index 7ccba3372..bdcab7f26 100644 --- a/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisAffixProcessRule.cs +++ b/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisAffixProcessRule.cs @@ -29,7 +29,8 @@ public AnalysisAffixProcessRule(Morpher morpher, AffixProcessRule rule) foreach (AffixProcessAllomorph allo in rule.Allomorphs) { _rules.Add( - new MultiplePatternRule( + new CopyAgreementPatternRule( + morpher, new AnalysisAffixProcessAllomorphRuleSpec(allo), new MatcherSettings { diff --git a/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisMorphologicalTransform.cs b/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisMorphologicalTransform.cs index 93e7a0ef6..90948926d 100644 --- a/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisMorphologicalTransform.cs +++ b/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisMorphologicalTransform.cs @@ -46,6 +46,82 @@ protected IDictionary CapturedParts get { return _capturedParts; } } + internal bool HasRepeatedParts + { + get { return _capturedParts.Values.Any(count => count >= 2); } + } + + /// + /// Synthesis writes every copy of a part from the same input, and anything that later changes one + /// copy is unapplied before this rule, so copies proven to disagree segment by segment cannot lead + /// to a valid analysis. A copy containing an optional node (an unapplied deletion), or a part the + /// rule modifies, cannot be judged and never counts as disagreeing. + /// + internal bool HasDisagreeingCopies(Match match) + { + foreach (KeyValuePair capturedPart in _capturedParts) + { + string partName = capturedPart.Key; + int copyCount = capturedPart.Value; + if (copyCount < 2) + continue; + + if (_modifyFromInfos.ContainsKey(partName)) + continue; + + var copies = new List>(copyCount); + bool partUndecidable = false; + for (int i = 0; i < copyCount; i++) + { + GroupCapture capture = match.GroupCaptures[GetGroupName(partName, i)]; + if (!capture.Success) + { + partUndecidable = true; + break; + } + + List nodes = GetCapturedNodes(match.Input.Shape, capture.Range); + if (nodes.Any(node => node.Annotation.Optional)) + { + partUndecidable = true; + break; + } + + copies.Add(nodes.Where(node => node.Annotation.Type() == HCFeatureSystem.Segment).ToList()); + } + + if (partUndecidable) + continue; + + List firstCopy = copies[0]; + for (int i = 1; i < copies.Count; i++) + { + List otherCopy = copies[i]; + if (firstCopy.Count != otherCopy.Count) + return true; + + for (int j = 0; j < firstCopy.Count; j++) + { + if (!firstCopy[j].Annotation.FeatureStruct.IsUnifiable(otherCopy[j].Annotation.FeatureStruct)) + return true; + } + } + } + + return false; + } + + /// + /// Includes optional nodes the matcher skipped before the capture when they reach the start of the + /// shape, so an unapplied word-initial deletion is still seen. + /// + private static List GetCapturedNodes(Shape shape, Range range) + { + var nodes = new List(MorphologicalOutputAction.SkippedOptionalNodes(shape, range)); + nodes.AddRange(shape.GetNodes(range)); + return nodes; + } + public Pattern Pattern { get { return _pattern; } diff --git a/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisRealizationalAffixProcessRule.cs b/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisRealizationalAffixProcessRule.cs index 89e30fbe2..a3f3fc8cb 100644 --- a/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisRealizationalAffixProcessRule.cs +++ b/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisRealizationalAffixProcessRule.cs @@ -22,7 +22,8 @@ public AnalysisRealizationalAffixProcessRule(Morpher morpher, RealizationalAffix foreach (AffixProcessAllomorph allo in rule.Allomorphs) { _rules.Add( - new MultiplePatternRule( + new CopyAgreementPatternRule( + morpher, new AnalysisAffixProcessAllomorphRuleSpec(allo), new MatcherSettings { diff --git a/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/CopyAgreementPatternRule.cs b/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/CopyAgreementPatternRule.cs new file mode 100644 index 000000000..58d5cf58e --- /dev/null +++ b/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/CopyAgreementPatternRule.cs @@ -0,0 +1,38 @@ +using System.Collections.Generic; +using SIL.Machine.Annotations; +using SIL.Machine.Matching; +using SIL.Machine.Rules; + +namespace SIL.Machine.Morphology.HermitCrab.MorphologicalRules +{ + internal sealed class CopyAgreementPatternRule : MultiplePatternRule + { + private readonly Morpher _morpher; + private readonly AnalysisAffixProcessAllomorphRuleSpec _spec; + + public CopyAgreementPatternRule( + Morpher morpher, + AnalysisAffixProcessAllomorphRuleSpec ruleSpec, + MatcherSettings matcherSettings + ) + : base(ruleSpec, matcherSettings) + { + _morpher = morpher; + _spec = ruleSpec; + } + + protected override IEnumerable ApplyImpl(Word input, ShapeNode start) + { + bool prune = _morpher.PruneDisagreeingCopies && _spec.HasRepeatedParts; + var results = new List(); + foreach (Match match in Matcher.AllMatches(input, start)) + { + if (prune && _spec.HasDisagreeingCopies(match)) + continue; + + results.Add(RuleSpec.ApplyRhs(this, match)); + } + return results; + } + } +} diff --git a/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/MorphologicalOutputAction.cs b/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/MorphologicalOutputAction.cs index 5595a232f..508a458c0 100644 --- a/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/MorphologicalOutputAction.cs +++ b/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/MorphologicalOutputAction.cs @@ -38,6 +38,11 @@ IDictionary capturedParts public abstract IEnumerable> Apply(Match match, Word output); protected IEnumerable GetSkippedOptionalNodes(Shape shape, Range range) + { + return SkippedOptionalNodes(shape, range); + } + + internal static IEnumerable SkippedOptionalNodes(Shape shape, Range range) { ShapeNode node = range.Start.Prev; var skippedNodes = new List(); diff --git a/tests/SIL.Machine.Morphology.HermitCrab.Tests/MorphologicalRules/AffixProcessRuleTests.cs b/tests/SIL.Machine.Morphology.HermitCrab.Tests/MorphologicalRules/AffixProcessRuleTests.cs index 329074617..579d997db 100644 --- a/tests/SIL.Machine.Morphology.HermitCrab.Tests/MorphologicalRules/AffixProcessRuleTests.cs +++ b/tests/SIL.Machine.Morphology.HermitCrab.Tests/MorphologicalRules/AffixProcessRuleTests.cs @@ -965,8 +965,9 @@ public void SimulfixRules() AssertMorphsEqual(morpher.ParseWord("sɯɯpu"), "50 SIMUL"); } - [Test] - public void ReduplicationRules() + [TestCase(false)] + [TestCase(true)] + public void ReduplicationRules(bool pruneCopies) { var any = FeatureStruct.New().Symbol(HCFeatureSystem.Segment).Value; var cons = FeatureStruct @@ -1015,7 +1016,7 @@ public void ReduplicationRules() ); Morphophonemic.MorphologicalRules.Add(redup); - var morpher = new Morpher(TraceManager, Language); + var morpher = new Morpher(TraceManager, Language) { PruneDisagreeingCopies = pruneCopies }; AssertMorphsEqual(morpher.ParseWord("sasag"), "RED 32"); var voicing = new RewriteRule @@ -1033,7 +1034,7 @@ public void ReduplicationRules() ); Allophonic.PhonologicalRules.Add(voicing); - morpher = new Morpher(TraceManager, Language); + morpher = new Morpher(TraceManager, Language) { PruneDisagreeingCopies = pruneCopies }; AssertMorphsEqual(morpher.ParseWord("sazag"), "RED 32"); var affrication = new RewriteRule @@ -1050,7 +1051,7 @@ public void ReduplicationRules() ); Allophonic.PhonologicalRules.Add(affrication); - morpher = new Morpher(TraceManager, Language); + morpher = new Morpher(TraceManager, Language) { PruneDisagreeingCopies = pruneCopies }; AssertMorphsEqual(morpher.ParseWord("tsazag"), "RED 32"); redup.Allomorphs.Clear(); @@ -1066,7 +1067,7 @@ public void ReduplicationRules() } ); - morpher = new Morpher(TraceManager, Language); + morpher = new Morpher(TraceManager, Language) { PruneDisagreeingCopies = pruneCopies }; AssertMorphsEqual(morpher.ParseWord("tsagag"), "32 RED"); Allophonic.PhonologicalRules.Clear(); @@ -1084,7 +1085,7 @@ public void ReduplicationRules() } ); - morpher = new Morpher(TraceManager, Language); + morpher = new Morpher(TraceManager, Language) { PruneDisagreeingCopies = pruneCopies }; AssertMorphsEqual(morpher.ParseWord("sagsag"), "32 RED"); AssertMorphsEqual(morpher.ParseWord("sasibudbud"), "38 RED"); @@ -1115,7 +1116,7 @@ public void ReduplicationRules() ); Allophonic.PhonologicalRules.Add(gDelete); - morpher = new Morpher(TraceManager, Language); + morpher = new Morpher(TraceManager, Language) { PruneDisagreeingCopies = pruneCopies }; AssertMorphsEqual(morpher.ParseWord("saag"), "32 RED"); gDelete.Subrules.Clear(); @@ -1126,7 +1127,7 @@ public void ReduplicationRules() } ); - morpher = new Morpher(TraceManager, Language); + morpher = new Morpher(TraceManager, Language) { PruneDisagreeingCopies = pruneCopies }; AssertMorphsEqual(morpher.ParseWord("saga"), "32 RED"); Allophonic.PhonologicalRules.Clear(); @@ -1152,7 +1153,7 @@ public void ReduplicationRules() } ); - morpher = new Morpher(TraceManager, Language); + morpher = new Morpher(TraceManager, Language) { PruneDisagreeingCopies = pruneCopies }; AssertMorphsEqual(morpher.ParseWord("buiuib"), "30 RED", "31 RED"); } @@ -1900,8 +1901,9 @@ public void SubsumedAffix() AssertMorphsEqual(morpher.ParseWord("bubib"), "42 PRES", "43 PRES"); } - [Test] - public void ModifyFromInputRules() + [TestCase(false)] + [TestCase(true)] + public void ModifyFromInputRules(bool pruneCopies) { var any = FeatureStruct.New().Symbol(HCFeatureSystem.Segment).Value; var vowel = FeatureStruct @@ -1941,7 +1943,7 @@ public void ModifyFromInputRules() ); Morphophonemic.MorphologicalRules.Add(sSuffix); - var morpher = new Morpher(TraceManager, Language); + var morpher = new Morpher(TraceManager, Language) { PruneDisagreeingCopies = pruneCopies }; AssertMorphsEqual(morpher.ParseWord("puso"), "52 PL"); } diff --git a/tests/SIL.Machine.Morphology.HermitCrab.Tests/MorphologicalRules/CopyAgreementPruneTests.cs b/tests/SIL.Machine.Morphology.HermitCrab.Tests/MorphologicalRules/CopyAgreementPruneTests.cs new file mode 100644 index 000000000..62206d6ca --- /dev/null +++ b/tests/SIL.Machine.Morphology.HermitCrab.Tests/MorphologicalRules/CopyAgreementPruneTests.cs @@ -0,0 +1,79 @@ +using NUnit.Framework; +using SIL.Machine.Annotations; +using SIL.Machine.FeatureModel; +using SIL.Machine.Matching; + +namespace SIL.Machine.Morphology.HermitCrab.MorphologicalRules; + +[TestFixture] +public class CopyAgreementPruneTests : HermitCrabTestBase +{ + [TestCase(false, 3)] + [TestCase(true, 1)] + public void FullCopyKeepsOnlyTheSplitWhoseCopiesAgree(bool pruneCopies, int expectedOutputs) + { + (string first, string second) = FindIncompatibleSegments(); + var anySegment = FeatureStruct.New().Symbol(HCFeatureSystem.Segment).Value; + AffixProcessRule rule = CreateCopyRule( + Pattern.New("copy").Annotation(anySegment).OneOrMore.Value + ); + var morpher = new Morpher(TraceManager, Language) { PruneDisagreeingCopies = pruneCopies }; + + List outputs = new AnalysisAffixProcessRule(morpher, rule) + .Apply(CreateInput(first + second + first + second)) + .ToList(); + + Assert.That(outputs.Count, Is.EqualTo(expectedOutputs)); + if (pruneCopies) + Assert.That(outputs.Single().Shape.Count, Is.EqualTo(2)); + } + + [TestCase(false, 1)] + [TestCase(true, 0)] + public void DisagreeingCopiesAreRemovedOnlyWhenPruning(bool pruneCopies, int expectedOutputs) + { + (string first, string second) = FindIncompatibleSegments(); + var anySegment = FeatureStruct.New().Symbol(HCFeatureSystem.Segment).Value; + AffixProcessRule rule = CreateCopyRule(Pattern.New("copy").Annotation(anySegment).Value); + var morpher = new Morpher(TraceManager, Language) { PruneDisagreeingCopies = pruneCopies }; + + List outputs = new AnalysisAffixProcessRule(morpher, rule).Apply(CreateInput(first + second)).ToList(); + + Assert.That(outputs.Count, Is.EqualTo(expectedOutputs)); + } + + private static AffixProcessRule CreateCopyRule(Pattern part) + { + var rule = new AffixProcessRule + { + Name = "full_copy", + Gloss = "RED", + RequiredSyntacticFeatureStruct = FeatureStruct.New().Value, + OutSyntacticFeatureStruct = FeatureStruct.New().Value, + }; + rule.Allomorphs.Add( + new AffixProcessAllomorph { Lhs = { part }, Rhs = { new CopyFromInput("copy"), new CopyFromInput("copy") } } + ); + return rule; + } + + private (string First, string Second) FindIncompatibleSegments() + { + CharacterDefinition[] segments = Table3 + .Where(definition => definition.Type == HCFeatureSystem.Segment) + .ToArray(); + var pair = segments + .SelectMany( + (first, index) => segments.Skip(index + 1).Select(second => new { First = first, Second = second }) + ) + .First(candidate => !candidate.First.FeatureStruct.IsUnifiable(candidate.Second.FeatureStruct)); + return (pair.First.Representations.First(), pair.Second.Representations.First()); + } + + private Word CreateInput(string representation) + { + var input = new Word(Surface, Table3.Segment(representation)) { AnalysisScope = new AnalysisScope() }; + input.Freeze(); + return input; + } +} From dfac60c6d1b05b986fd0547d89e5ab179f740293 Mon Sep 17 00:00:00 2001 From: John Lambert Date: Thu, 24 Sep 2026 13:25:40 -0400 Subject: [PATCH 08/20] Pin copy pruning against a cap-limited deletion in one copy A self-feeding deletion strips two segments from the second copy only; with DeletionReapplications 0 analysis restores one. Forward generation defines the expected parse, and pruning must return the unpruned result. Co-Authored-By: Claude Opus 5.5 --- .../CopyAgreementPruneTests.cs | 68 +++++++++++++++++++ 1 file changed, 68 insertions(+) diff --git a/tests/SIL.Machine.Morphology.HermitCrab.Tests/MorphologicalRules/CopyAgreementPruneTests.cs b/tests/SIL.Machine.Morphology.HermitCrab.Tests/MorphologicalRules/CopyAgreementPruneTests.cs index 62206d6ca..bfc421488 100644 --- a/tests/SIL.Machine.Morphology.HermitCrab.Tests/MorphologicalRules/CopyAgreementPruneTests.cs +++ b/tests/SIL.Machine.Morphology.HermitCrab.Tests/MorphologicalRules/CopyAgreementPruneTests.cs @@ -2,6 +2,7 @@ using SIL.Machine.Annotations; using SIL.Machine.FeatureModel; using SIL.Machine.Matching; +using SIL.Machine.Morphology.HermitCrab.PhonologicalRules; namespace SIL.Machine.Morphology.HermitCrab.MorphologicalRules; @@ -42,6 +43,73 @@ public void DisagreeingCopiesAreRemovedOnlyWhenPruning(bool pruneCopies, int exp Assert.That(outputs.Count, Is.EqualTo(expectedOutputs)); } + /// + /// The prune's weakest point: a self-feeding deletion strips two segments from the second copy only, + /// and = 0 lets analysis restore just one of them. + /// + [TestCase(0)] + [TestCase(1)] + public void CapLimitedDeletionInOneCopyParsesTheSameWithAndWithoutPruning(int deletionReapplications) + { + var any = FeatureStruct.New().Symbol(HCFeatureSystem.Segment).Value; + var vowel = FeatureStruct + .New(Language.PhonologicalFeatureSystem) + .Symbol(HCFeatureSystem.Segment) + .Symbol("voc+") + .Value; + AffixProcessRule redup = CreateCopyRule(Pattern.New("copy").Annotation(any).OneOrMore.Value); + redup.RequiredSyntacticFeatureStruct = FeatureStruct.New(Language.SyntacticFeatureSystem).Symbol("V").Value; + Morphophonemic.MorphologicalRules.Add(redup); + + var gDelete = new RewriteRule + { + Name = "g_delete", + Lhs = Pattern.New().Annotation(Character(Table1, "g")).Value, + }; + gDelete.Subrules.Add( + new RewriteSubrule { LeftEnvironment = Pattern.New().Annotation(vowel).Value } + ); + Allophonic.PhonologicalRules.Add(gDelete); + + LexEntry root = AddEntry( + "GG", + FeatureStruct.New(Language.SyntacticFeatureSystem).Symbol("V").Value, + Morphophonemic, + "ggasa" + ); + try + { + var generator = new Morpher(TraceManager, Language); + Assert.That( + generator.GenerateWords(root, new Morpheme[] { redup }, FeatureStruct.New().Value), + Is.EquivalentTo(new[] { "ggasaasa" }), + "forward derivation defines the expected parse" + ); + + string[] Parse(bool pruneCopies) + { + var morpher = new Morpher(TraceManager, Language) + { + DeletionReapplications = deletionReapplications, + PruneDisagreeingCopies = pruneCopies, + }; + return morpher + .ParseWord("ggasaasa") + .Select(w => string.Join(" ", w.AllomorphsInMorphOrder.Select(a => a.Morpheme.Gloss))) + .OrderBy(s => s, StringComparer.Ordinal) + .ToArray(); + } + + string[] unpruned = Parse(false); + Assert.That(unpruned, Does.Contain("GG RED"), "the unpruned engine finds the generated analysis"); + Assert.That(Parse(true), Is.EqualTo(unpruned)); + } + finally + { + Morphophonemic.Entries.Remove(root); + } + } + private static AffixProcessRule CreateCopyRule(Pattern part) { var rule = new AffixProcessRule From c403288b73942ce040f946c816c7fefdf9d0b269 Mon Sep 17 00:00:00 2001 From: John Lambert Date: Thu, 24 Sep 2026 21:46:20 -0400 Subject: [PATCH 09/20] Turn copy-agreement pruning on by default Co-Authored-By: Claude Opus 5.5 --- src/SIL.Machine.Morphology.HermitCrab/Morpher.cs | 4 ++-- .../MorphologicalRules/CopyAgreementPruneTests.cs | 6 ++++++ 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/src/SIL.Machine.Morphology.HermitCrab/Morpher.cs b/src/SIL.Machine.Morphology.HermitCrab/Morpher.cs index 9712dac42..452e0526d 100644 --- a/src/SIL.Machine.Morphology.HermitCrab/Morpher.cs +++ b/src/SIL.Machine.Morphology.HermitCrab/Morpher.cs @@ -61,6 +61,7 @@ public Morpher(ITraceManager traceManager, Language lang, int maxDegreeOfParalle MaxStemCount = 2; MaxAlternatives = 0; MergeEquivalentAnalyses = true; + PruneDisagreeingCopies = true; LexEntrySelector = entry => true; RuleSelector = rule => true; @@ -113,8 +114,7 @@ public ITraceManager TraceManager /// /// When unapplying a rule that copies a part more than once (reduplication), skip matches whose /// copies cannot unify segment by segment. Such a match can never survive synthesis, so this removes - /// only doomed analyses; on reduplication-heavy grammars it is the difference between a parse that - /// exhausts memory and one that finishes in seconds. + /// only doomed analyses. On by default. /// public bool PruneDisagreeingCopies { get; set; } diff --git a/tests/SIL.Machine.Morphology.HermitCrab.Tests/MorphologicalRules/CopyAgreementPruneTests.cs b/tests/SIL.Machine.Morphology.HermitCrab.Tests/MorphologicalRules/CopyAgreementPruneTests.cs index bfc421488..5f9a5d74c 100644 --- a/tests/SIL.Machine.Morphology.HermitCrab.Tests/MorphologicalRules/CopyAgreementPruneTests.cs +++ b/tests/SIL.Machine.Morphology.HermitCrab.Tests/MorphologicalRules/CopyAgreementPruneTests.cs @@ -9,6 +9,12 @@ namespace SIL.Machine.Morphology.HermitCrab.MorphologicalRules; [TestFixture] public class CopyAgreementPruneTests : HermitCrabTestBase { + [Test] + public void PruningIsOnByDefault() + { + Assert.That(new Morpher(TraceManager, Language).PruneDisagreeingCopies, Is.True); + } + [TestCase(false, 3)] [TestCase(true, 1)] public void FullCopyKeepsOnlyTheSplitWhoseCopiesAgree(bool pruneCopies, int expectedOutputs) From 4bdf2d067996164f9304be20eafaef15aee00cc3 Mon Sep 17 00:00:00 2001 From: John Lambert Date: Fri, 25 Sep 2026 18:41:03 -0400 Subject: [PATCH 10/20] Compute HasRepeatedParts once per copy-agreement rule Rules that copy no part twice no longer rescan their captured parts on every unapplication. Mirrors PanGloss 77aca87f, which measured 0.62-0.64x of the prior time on Aweti with identical analyses. Co-Authored-By: Claude Opus 5.5 --- .../MorphologicalRules/CopyAgreementPatternRule.cs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/CopyAgreementPatternRule.cs b/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/CopyAgreementPatternRule.cs index 58d5cf58e..78abce1ff 100644 --- a/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/CopyAgreementPatternRule.cs +++ b/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/CopyAgreementPatternRule.cs @@ -9,6 +9,7 @@ internal sealed class CopyAgreementPatternRule : MultiplePatternRule matcherSettings { _morpher = morpher; _spec = ruleSpec; + _hasRepeatedParts = ruleSpec.HasRepeatedParts; } protected override IEnumerable ApplyImpl(Word input, ShapeNode start) { - bool prune = _morpher.PruneDisagreeingCopies && _spec.HasRepeatedParts; + bool prune = _morpher.PruneDisagreeingCopies && _hasRepeatedParts; var results = new List(); foreach (Match match in Matcher.AllMatches(input, start)) { From 90f5cda005904342eb2010c4f0860fdbe0a6f464 Mon Sep 17 00:00:00 2001 From: John Lambert Date: Wed, 30 Sep 2026 15:20:24 -0400 Subject: [PATCH 11/20] Address review of copy pruning - Prune compounding rules as well as affix-process rules. - Compare every pair of copies, since unifiability is not transitive. - Skip pruning while tracing, as MergeEquivalentAnalyses does, so traces still show each doomed split failing in synthesis. - Collapse the SkippedOptionalNodes forwarder into a protected internal static GetSkippedOptionalNodes; split HasDisagreeingCopies into helpers. - Rename to DisagreeingCopiesPruningRule / PruneDisagreeingCopiesTests. - Add tests: realizational and compounding paths, three copies, parallelism and memo settings, partial morphemes with and without AlwaysEnforceFinalTemplates, DeletionReapplications = 2, tracing, and a counterbleeding opacity case pinned against forward generation. Co-Authored-By: Claude Opus 5.5 --- .../Morpher.cs | 7 +- .../AnalysisAffixProcessRule.cs | 2 +- .../AnalysisCompoundingRule.cs | 3 +- .../AnalysisMorphologicalTransform.cs | 93 ++--- .../AnalysisRealizationalAffixProcessRule.cs | 2 +- ...ule.cs => DisagreeingCopiesPruningRule.cs} | 10 +- .../MorphologicalOutputAction.cs | 7 +- .../CopyAgreementPruneTests.cs | 153 -------- .../PruneDisagreeingCopiesTests.cs | 367 ++++++++++++++++++ 9 files changed, 428 insertions(+), 216 deletions(-) rename src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/{CopyAgreementPatternRule.cs => DisagreeingCopiesPruningRule.cs} (77%) delete mode 100644 tests/SIL.Machine.Morphology.HermitCrab.Tests/MorphologicalRules/CopyAgreementPruneTests.cs create mode 100644 tests/SIL.Machine.Morphology.HermitCrab.Tests/MorphologicalRules/PruneDisagreeingCopiesTests.cs diff --git a/src/SIL.Machine.Morphology.HermitCrab/Morpher.cs b/src/SIL.Machine.Morphology.HermitCrab/Morpher.cs index 452e0526d..66df6ab1d 100644 --- a/src/SIL.Machine.Morphology.HermitCrab/Morpher.cs +++ b/src/SIL.Machine.Morphology.HermitCrab/Morpher.cs @@ -112,9 +112,10 @@ public ITraceManager TraceManager public bool AlwaysEnforceFinalTemplates { get; set; } /// - /// When unapplying a rule that copies a part more than once (reduplication), skip matches whose - /// copies cannot unify segment by segment. Such a match can never survive synthesis, so this removes - /// only doomed analyses. On by default. + /// When unapplying an affix-process or compounding rule that copies a part more than once + /// (reduplication), skip matches whose copies cannot unify segment by segment. Such a match can never + /// survive synthesis, so this removes only doomed analyses. On by default; ignored while tracing, so + /// a trace still shows each such match failing in synthesis. /// public bool PruneDisagreeingCopies { get; set; } diff --git a/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisAffixProcessRule.cs b/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisAffixProcessRule.cs index bdcab7f26..b7c204dd0 100644 --- a/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisAffixProcessRule.cs +++ b/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisAffixProcessRule.cs @@ -29,7 +29,7 @@ public AnalysisAffixProcessRule(Morpher morpher, AffixProcessRule rule) foreach (AffixProcessAllomorph allo in rule.Allomorphs) { _rules.Add( - new CopyAgreementPatternRule( + new DisagreeingCopiesPruningRule( morpher, new AnalysisAffixProcessAllomorphRuleSpec(allo), new MatcherSettings diff --git a/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisCompoundingRule.cs b/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisCompoundingRule.cs index f77bf96e4..6dbf11b9f 100644 --- a/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisCompoundingRule.cs +++ b/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisCompoundingRule.cs @@ -28,7 +28,8 @@ public AnalysisCompoundingRule(Morpher morpher, CompoundingRule rule) foreach (CompoundingSubrule sr in rule.Subrules) { _rules.Add( - new MultiplePatternRule( + new DisagreeingCopiesPruningRule( + morpher, new AnalysisCompoundingSubruleRuleSpec(sr), new MatcherSettings { diff --git a/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisMorphologicalTransform.cs b/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisMorphologicalTransform.cs index 90948926d..c3244d021 100644 --- a/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisMorphologicalTransform.cs +++ b/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisMorphologicalTransform.cs @@ -54,74 +54,75 @@ internal bool HasRepeatedParts /// /// Synthesis writes every copy of a part from the same input, and anything that later changes one /// copy is unapplied before this rule, so copies proven to disagree segment by segment cannot lead - /// to a valid analysis. A copy containing an optional node (an unapplied deletion), or a part the - /// rule modifies, cannot be judged and never counts as disagreeing. + /// to a valid analysis. A part the rule modifies, or a copy containing an unapplied deletion, is + /// never reported as disagreeing. /// internal bool HasDisagreeingCopies(Match match) { foreach (KeyValuePair capturedPart in _capturedParts) { - string partName = capturedPart.Key; - int copyCount = capturedPart.Value; - if (copyCount < 2) + if (capturedPart.Value < 2 || _modifyFromInfos.ContainsKey(capturedPart.Key)) continue; - if (_modifyFromInfos.ContainsKey(partName)) - continue; - - var copies = new List>(copyCount); - bool partUndecidable = false; - for (int i = 0; i < copyCount; i++) + if ( + TryGetSegmentCopies(match, capturedPart.Key, capturedPart.Value, out List> copies) + && AnyCopiesDisagree(copies) + ) { - GroupCapture capture = match.GroupCaptures[GetGroupName(partName, i)]; - if (!capture.Success) - { - partUndecidable = true; - break; - } - - List nodes = GetCapturedNodes(match.Input.Shape, capture.Range); - if (nodes.Any(node => node.Annotation.Optional)) - { - partUndecidable = true; - break; - } - - copies.Add(nodes.Where(node => node.Annotation.Type() == HCFeatureSystem.Segment).ToList()); + return true; } + } - if (partUndecidable) - continue; + return false; + } + + private static bool TryGetSegmentCopies( + Match match, + string partName, + int copyCount, + out List> copies + ) + { + copies = new List>(copyCount); + for (int i = 0; i < copyCount; i++) + { + GroupCapture capture = match.GroupCaptures[GetGroupName(partName, i)]; + if (!capture.Success) + return false; + + // an unapplied word-initial deletion sits before the capture, not inside it + List nodes = MorphologicalOutputAction + .GetSkippedOptionalNodes(match.Input.Shape, capture.Range) + .Concat(match.Input.Shape.GetNodes(capture.Range)) + .ToList(); + if (nodes.Any(node => node.Annotation.Optional)) + return false; + + copies.Add(nodes.Where(node => node.Annotation.Type() == HCFeatureSystem.Segment).ToList()); + } + return true; + } - List firstCopy = copies[0]; - for (int i = 1; i < copies.Count; i++) + // unifiability is not transitive, so every pair is compared, not just each copy against the first + private static bool AnyCopiesDisagree(List> copies) + { + for (int i = 0; i < copies.Count; i++) + { + for (int j = i + 1; j < copies.Count; j++) { - List otherCopy = copies[i]; - if (firstCopy.Count != otherCopy.Count) + if (copies[i].Count != copies[j].Count) return true; - for (int j = 0; j < firstCopy.Count; j++) + for (int k = 0; k < copies[i].Count; k++) { - if (!firstCopy[j].Annotation.FeatureStruct.IsUnifiable(otherCopy[j].Annotation.FeatureStruct)) + if (!copies[i][k].Annotation.FeatureStruct.IsUnifiable(copies[j][k].Annotation.FeatureStruct)) return true; } } } - return false; } - /// - /// Includes optional nodes the matcher skipped before the capture when they reach the start of the - /// shape, so an unapplied word-initial deletion is still seen. - /// - private static List GetCapturedNodes(Shape shape, Range range) - { - var nodes = new List(MorphologicalOutputAction.SkippedOptionalNodes(shape, range)); - nodes.AddRange(shape.GetNodes(range)); - return nodes; - } - public Pattern Pattern { get { return _pattern; } diff --git a/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisRealizationalAffixProcessRule.cs b/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisRealizationalAffixProcessRule.cs index a3f3fc8cb..9862db6bf 100644 --- a/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisRealizationalAffixProcessRule.cs +++ b/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisRealizationalAffixProcessRule.cs @@ -22,7 +22,7 @@ public AnalysisRealizationalAffixProcessRule(Morpher morpher, RealizationalAffix foreach (AffixProcessAllomorph allo in rule.Allomorphs) { _rules.Add( - new CopyAgreementPatternRule( + new DisagreeingCopiesPruningRule( morpher, new AnalysisAffixProcessAllomorphRuleSpec(allo), new MatcherSettings diff --git a/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/CopyAgreementPatternRule.cs b/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/DisagreeingCopiesPruningRule.cs similarity index 77% rename from src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/CopyAgreementPatternRule.cs rename to src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/DisagreeingCopiesPruningRule.cs index 78abce1ff..e4431ab44 100644 --- a/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/CopyAgreementPatternRule.cs +++ b/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/DisagreeingCopiesPruningRule.cs @@ -5,15 +5,15 @@ namespace SIL.Machine.Morphology.HermitCrab.MorphologicalRules { - internal sealed class CopyAgreementPatternRule : MultiplePatternRule + internal sealed class DisagreeingCopiesPruningRule : MultiplePatternRule { private readonly Morpher _morpher; - private readonly AnalysisAffixProcessAllomorphRuleSpec _spec; + private readonly AnalysisMorphologicalTransformRuleSpec _spec; private readonly bool _hasRepeatedParts; - public CopyAgreementPatternRule( + public DisagreeingCopiesPruningRule( Morpher morpher, - AnalysisAffixProcessAllomorphRuleSpec ruleSpec, + AnalysisMorphologicalTransformRuleSpec ruleSpec, MatcherSettings matcherSettings ) : base(ruleSpec, matcherSettings) @@ -25,7 +25,7 @@ MatcherSettings matcherSettings protected override IEnumerable ApplyImpl(Word input, ShapeNode start) { - bool prune = _morpher.PruneDisagreeingCopies && _hasRepeatedParts; + bool prune = _morpher.PruneDisagreeingCopies && _hasRepeatedParts && !_morpher.TraceManager.IsTracing; var results = new List(); foreach (Match match in Matcher.AllMatches(input, start)) { diff --git a/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/MorphologicalOutputAction.cs b/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/MorphologicalOutputAction.cs index 508a458c0..53ed1ddd0 100644 --- a/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/MorphologicalOutputAction.cs +++ b/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/MorphologicalOutputAction.cs @@ -37,12 +37,7 @@ IDictionary capturedParts /// The output word synthesis. public abstract IEnumerable> Apply(Match match, Word output); - protected IEnumerable GetSkippedOptionalNodes(Shape shape, Range range) - { - return SkippedOptionalNodes(shape, range); - } - - internal static IEnumerable SkippedOptionalNodes(Shape shape, Range range) + protected internal static IEnumerable GetSkippedOptionalNodes(Shape shape, Range range) { ShapeNode node = range.Start.Prev; var skippedNodes = new List(); diff --git a/tests/SIL.Machine.Morphology.HermitCrab.Tests/MorphologicalRules/CopyAgreementPruneTests.cs b/tests/SIL.Machine.Morphology.HermitCrab.Tests/MorphologicalRules/CopyAgreementPruneTests.cs deleted file mode 100644 index 5f9a5d74c..000000000 --- a/tests/SIL.Machine.Morphology.HermitCrab.Tests/MorphologicalRules/CopyAgreementPruneTests.cs +++ /dev/null @@ -1,153 +0,0 @@ -using NUnit.Framework; -using SIL.Machine.Annotations; -using SIL.Machine.FeatureModel; -using SIL.Machine.Matching; -using SIL.Machine.Morphology.HermitCrab.PhonologicalRules; - -namespace SIL.Machine.Morphology.HermitCrab.MorphologicalRules; - -[TestFixture] -public class CopyAgreementPruneTests : HermitCrabTestBase -{ - [Test] - public void PruningIsOnByDefault() - { - Assert.That(new Morpher(TraceManager, Language).PruneDisagreeingCopies, Is.True); - } - - [TestCase(false, 3)] - [TestCase(true, 1)] - public void FullCopyKeepsOnlyTheSplitWhoseCopiesAgree(bool pruneCopies, int expectedOutputs) - { - (string first, string second) = FindIncompatibleSegments(); - var anySegment = FeatureStruct.New().Symbol(HCFeatureSystem.Segment).Value; - AffixProcessRule rule = CreateCopyRule( - Pattern.New("copy").Annotation(anySegment).OneOrMore.Value - ); - var morpher = new Morpher(TraceManager, Language) { PruneDisagreeingCopies = pruneCopies }; - - List outputs = new AnalysisAffixProcessRule(morpher, rule) - .Apply(CreateInput(first + second + first + second)) - .ToList(); - - Assert.That(outputs.Count, Is.EqualTo(expectedOutputs)); - if (pruneCopies) - Assert.That(outputs.Single().Shape.Count, Is.EqualTo(2)); - } - - [TestCase(false, 1)] - [TestCase(true, 0)] - public void DisagreeingCopiesAreRemovedOnlyWhenPruning(bool pruneCopies, int expectedOutputs) - { - (string first, string second) = FindIncompatibleSegments(); - var anySegment = FeatureStruct.New().Symbol(HCFeatureSystem.Segment).Value; - AffixProcessRule rule = CreateCopyRule(Pattern.New("copy").Annotation(anySegment).Value); - var morpher = new Morpher(TraceManager, Language) { PruneDisagreeingCopies = pruneCopies }; - - List outputs = new AnalysisAffixProcessRule(morpher, rule).Apply(CreateInput(first + second)).ToList(); - - Assert.That(outputs.Count, Is.EqualTo(expectedOutputs)); - } - - /// - /// The prune's weakest point: a self-feeding deletion strips two segments from the second copy only, - /// and = 0 lets analysis restore just one of them. - /// - [TestCase(0)] - [TestCase(1)] - public void CapLimitedDeletionInOneCopyParsesTheSameWithAndWithoutPruning(int deletionReapplications) - { - var any = FeatureStruct.New().Symbol(HCFeatureSystem.Segment).Value; - var vowel = FeatureStruct - .New(Language.PhonologicalFeatureSystem) - .Symbol(HCFeatureSystem.Segment) - .Symbol("voc+") - .Value; - AffixProcessRule redup = CreateCopyRule(Pattern.New("copy").Annotation(any).OneOrMore.Value); - redup.RequiredSyntacticFeatureStruct = FeatureStruct.New(Language.SyntacticFeatureSystem).Symbol("V").Value; - Morphophonemic.MorphologicalRules.Add(redup); - - var gDelete = new RewriteRule - { - Name = "g_delete", - Lhs = Pattern.New().Annotation(Character(Table1, "g")).Value, - }; - gDelete.Subrules.Add( - new RewriteSubrule { LeftEnvironment = Pattern.New().Annotation(vowel).Value } - ); - Allophonic.PhonologicalRules.Add(gDelete); - - LexEntry root = AddEntry( - "GG", - FeatureStruct.New(Language.SyntacticFeatureSystem).Symbol("V").Value, - Morphophonemic, - "ggasa" - ); - try - { - var generator = new Morpher(TraceManager, Language); - Assert.That( - generator.GenerateWords(root, new Morpheme[] { redup }, FeatureStruct.New().Value), - Is.EquivalentTo(new[] { "ggasaasa" }), - "forward derivation defines the expected parse" - ); - - string[] Parse(bool pruneCopies) - { - var morpher = new Morpher(TraceManager, Language) - { - DeletionReapplications = deletionReapplications, - PruneDisagreeingCopies = pruneCopies, - }; - return morpher - .ParseWord("ggasaasa") - .Select(w => string.Join(" ", w.AllomorphsInMorphOrder.Select(a => a.Morpheme.Gloss))) - .OrderBy(s => s, StringComparer.Ordinal) - .ToArray(); - } - - string[] unpruned = Parse(false); - Assert.That(unpruned, Does.Contain("GG RED"), "the unpruned engine finds the generated analysis"); - Assert.That(Parse(true), Is.EqualTo(unpruned)); - } - finally - { - Morphophonemic.Entries.Remove(root); - } - } - - private static AffixProcessRule CreateCopyRule(Pattern part) - { - var rule = new AffixProcessRule - { - Name = "full_copy", - Gloss = "RED", - RequiredSyntacticFeatureStruct = FeatureStruct.New().Value, - OutSyntacticFeatureStruct = FeatureStruct.New().Value, - }; - rule.Allomorphs.Add( - new AffixProcessAllomorph { Lhs = { part }, Rhs = { new CopyFromInput("copy"), new CopyFromInput("copy") } } - ); - return rule; - } - - private (string First, string Second) FindIncompatibleSegments() - { - CharacterDefinition[] segments = Table3 - .Where(definition => definition.Type == HCFeatureSystem.Segment) - .ToArray(); - var pair = segments - .SelectMany( - (first, index) => segments.Skip(index + 1).Select(second => new { First = first, Second = second }) - ) - .First(candidate => !candidate.First.FeatureStruct.IsUnifiable(candidate.Second.FeatureStruct)); - return (pair.First.Representations.First(), pair.Second.Representations.First()); - } - - private Word CreateInput(string representation) - { - var input = new Word(Surface, Table3.Segment(representation)) { AnalysisScope = new AnalysisScope() }; - input.Freeze(); - return input; - } -} diff --git a/tests/SIL.Machine.Morphology.HermitCrab.Tests/MorphologicalRules/PruneDisagreeingCopiesTests.cs b/tests/SIL.Machine.Morphology.HermitCrab.Tests/MorphologicalRules/PruneDisagreeingCopiesTests.cs new file mode 100644 index 000000000..2d2b3aaa0 --- /dev/null +++ b/tests/SIL.Machine.Morphology.HermitCrab.Tests/MorphologicalRules/PruneDisagreeingCopiesTests.cs @@ -0,0 +1,367 @@ +using NUnit.Framework; +using SIL.Machine.Annotations; +using SIL.Machine.FeatureModel; +using SIL.Machine.Matching; +using SIL.Machine.Morphology.HermitCrab.PhonologicalRules; + +namespace SIL.Machine.Morphology.HermitCrab.MorphologicalRules; + +public class PruneDisagreeingCopiesTests : HermitCrabTestBase +{ + private static readonly FeatureStruct AnySegment = FeatureStruct.New().Symbol(HCFeatureSystem.Segment).Value; + + [Test] + public void PruningIsOnByDefault() + { + Assert.That(new Morpher(TraceManager, Language).PruneDisagreeingCopies, Is.True); + } + + [TestCase(false, 3)] + [TestCase(true, 1)] + public void FullCopyKeepsOnlyTheSplitWhoseCopiesAgree(bool pruneCopies, int expectedOutputs) + { + (string first, string second) = FindIncompatibleSegments(); + AffixProcessRule rule = CreateCopyRule(FullPart()); + + List outputs = new AnalysisAffixProcessRule(CreateMorpher(pruneCopies), rule) + .Apply(CreateInput(first + second + first + second)) + .ToList(); + + Assert.That(outputs.Count, Is.EqualTo(expectedOutputs)); + if (pruneCopies) + Assert.That(outputs.Single().Shape.Count, Is.EqualTo(2)); + } + + [TestCase(false, 1)] + [TestCase(true, 0)] + public void DisagreeingCopiesAreRemovedOnlyWhenPruning(bool pruneCopies, int expectedOutputs) + { + (string first, string second) = FindIncompatibleSegments(); + AffixProcessRule rule = CreateCopyRule(SingleSegmentPart()); + + List outputs = new AnalysisAffixProcessRule(CreateMorpher(pruneCopies), rule) + .Apply(CreateInput(first + second)) + .ToList(); + + Assert.That(outputs.Count, Is.EqualTo(expectedOutputs)); + } + + [Test] + public void TracingKeepsDisagreeingCopies() + { + (string first, string second) = FindIncompatibleSegments(); + AffixProcessRule rule = CreateCopyRule(SingleSegmentPart()); + var input = new Word(Surface, Table3.Segment(first + second)) { AnalysisScope = new AnalysisScope() }; + TraceManager.AnalyzeWord(Language, input); + input.Freeze(); + TraceManager.IsTracing = true; + try + { + List outputs = new AnalysisAffixProcessRule(CreateMorpher(true), rule).Apply(input).ToList(); + + Assert.That(outputs.Count, Is.EqualTo(1)); + } + finally + { + TraceManager.IsTracing = false; + } + } + + [TestCase(false, 1)] + [TestCase(true, 0)] + public void EveryPairOfCopiesIsCompared(bool pruneCopies, int expectedOutputs) + { + // the first copy is underspecified, as unapplied phonology can leave it, so it unifies with both others + (string first, string second) = FindIncompatibleSegments(); + AffixProcessRule rule = CreateCopyRule(SingleSegmentPart(), copyCount: 3); + var input = new Word(Surface, Table3.Segment(first + first + second)) { AnalysisScope = new AnalysisScope() }; + input.Shape.First.Annotation.FeatureStruct = AnySegment.Clone(); + input.Freeze(); + + List outputs = new AnalysisAffixProcessRule(CreateMorpher(pruneCopies), rule).Apply(input).ToList(); + + Assert.That(outputs.Count, Is.EqualTo(expectedOutputs)); + } + + [TestCase(false, 3)] + [TestCase(true, 1)] + public void RealizationalFullCopyKeepsOnlyTheSplitWhoseCopiesAgree(bool pruneCopies, int expectedOutputs) + { + (string first, string second) = FindIncompatibleSegments(); + var rule = new RealizationalAffixProcessRule { Name = "real_copy", Gloss = "RED" }; + rule.Allomorphs.Add(CreateCopyAllomorph(FullPart(), copyCount: 2)); + + List outputs = new AnalysisRealizationalAffixProcessRule(CreateMorpher(pruneCopies), rule) + .Apply(CreateInput(first + second + first + second)) + .ToList(); + + Assert.That(outputs.Count, Is.EqualTo(expectedOutputs)); + } + + [TestCase(false, 3)] + [TestCase(true, 1)] + public void CompoundHeadCopiedTwiceKeepsOnlyTheSplitWhoseCopiesAgree(bool pruneCopies, int expectedOutputs) + { + var crule = new CompoundingRule { Name = "head_copy_compound" }; + crule.Subrules.Add( + new CompoundingSubrule + { + HeadLhs = { Pattern.New("head").Annotation(AnySegment).OneOrMore.Value }, + NonHeadLhs = { Pattern.New("nonHead").Annotation(AnySegment).OneOrMore.Value }, + Rhs = { new CopyFromInput("head"), new CopyFromInput("head"), new CopyFromInput("nonHead") }, + } + ); + Morphophonemic.MorphologicalRules.Add(crule); + + // "pu" is the only root the input can end in, so every analysis differs only in how "tata" splits + List outputs = new AnalysisCompoundingRule(CreateMorpher(pruneCopies), crule) + .Apply(CreateInput("tatapu")) + .ToList(); + + Assert.That(outputs.Count, Is.EqualTo(expectedOutputs)); + Assert.That(outputs.Select(w => w.CurrentNonHead.RootAllomorph.Morpheme.Gloss), Is.All.EqualTo("52")); + } + + [TestCase(0)] + [TestCase(1)] + [TestCase(2)] + public void ParsesTheSameUnderEveryParallelismSetting(int maxDegreeOfParallelism) + { + AffixProcessRule redup = CreateCopyRule(FullPart()); + redup.RequiredSyntacticFeatureStruct = FeatureStruct.New(Language.SyntacticFeatureSystem).Symbol("V").Value; + Morphophonemic.MorphologicalRules.Add(redup); + + string[] unpruned = ParseGlosses(CreateMorpher(false, maxDegreeOfParallelism: 1), "sagsag"); + Assert.That(unpruned, Is.EqualTo(new[] { "32 RED" })); + Assert.That(ParseGlosses(CreateMorpher(true, maxDegreeOfParallelism), "sagsag"), Is.EqualTo(unpruned)); + } + + [TestCase(false)] + [TestCase(true)] + public void PartialMorphemesParseTheSameWithAndWithoutPruning(bool alwaysEnforceFinalTemplates) + { + Morphophonemic.MorphologicalRules.Add(CreateCopyRule(FullPart())); + + string[] Parse(bool pruneCopies, string word) + { + Morpher morpher = CreateMorpher(pruneCopies); + Assert.That(morpher.IsPartial, Is.True, "entry 54 has an empty syntactic feature structure"); + morpher.AlwaysEnforceFinalTemplates = alwaysEnforceFinalTemplates; + return ParseGlosses(morpher, word); + } + + Assert.That(Parse(false, "pipi"), Does.Contain("54 RED")); + foreach (string word in new[] { "pipi", "sagsag", "sagsa" }) + Assert.That(Parse(true, word), Is.EqualTo(Parse(false, word)), word); + } + + /// + /// The prune's weakest point: a self-feeding deletion strips two segments from the second copy only, + /// and = 0 lets analysis restore just one of them. + /// + [TestCase(0)] + [TestCase(1)] + [TestCase(2)] + public void CapLimitedDeletionInOneCopyParsesTheSameWithAndWithoutPruning(int deletionReapplications) + { + var vowel = FeatureStruct + .New(Language.PhonologicalFeatureSystem) + .Symbol(HCFeatureSystem.Segment) + .Symbol("voc+") + .Value; + AffixProcessRule redup = CreateCopyRule(FullPart()); + redup.RequiredSyntacticFeatureStruct = FeatureStruct.New(Language.SyntacticFeatureSystem).Symbol("V").Value; + Morphophonemic.MorphologicalRules.Add(redup); + + var gDelete = new RewriteRule + { + Name = "g_delete", + Lhs = Pattern.New().Annotation(Character(Table1, "g")).Value, + }; + gDelete.Subrules.Add( + new RewriteSubrule { LeftEnvironment = Pattern.New().Annotation(vowel).Value } + ); + Allophonic.PhonologicalRules.Add(gDelete); + + LexEntry root = AddEntry( + "GG", + FeatureStruct.New(Language.SyntacticFeatureSystem).Symbol("V").Value, + Morphophonemic, + "ggasa" + ); + try + { + var generator = new Morpher(TraceManager, Language); + Assert.That( + generator.GenerateWords(root, new Morpheme[] { redup }, FeatureStruct.New().Value), + Is.EquivalentTo(new[] { "ggasaasa" }), + "forward derivation defines the expected parse" + ); + + string[] Parse(bool pruneCopies) + { + Morpher morpher = CreateMorpher(pruneCopies); + morpher.DeletionReapplications = deletionReapplications; + return ParseGlosses(morpher, "ggasaasa"); + } + + string[] unpruned = Parse(false); + Assert.That(unpruned, Does.Contain("GG RED"), "the unpruned engine finds the generated analysis"); + Assert.That(Parse(true), Is.EqualTo(unpruned)); + } + finally + { + Morphophonemic.Entries.Remove(root); + } + } + + /// + /// Counterbleeding opacity: voicing changes only the second copy, then raising destroys the vowel that + /// conditioned it, so analysis must restore the copy through an environment that is no longer on the surface. + /// + [Test] + public void OpaqueChangeToOneCopyParsesTheSameWithAndWithoutPruning() + { + AffixProcessRule redup = CreateCopyRule(FullPart()); + redup.RequiredSyntacticFeatureStruct = FeatureStruct.New(Language.SyntacticFeatureSystem).Symbol("V").Value; + Morphophonemic.MorphologicalRules.Add(redup); + + var voicing = new RewriteRule + { + Name = "voicing", + Lhs = Pattern.New().Annotation(Character(Table1, "s")).Value, + }; + voicing.Subrules.Add( + new RewriteSubrule + { + Rhs = Pattern + .New() + .Annotation( + FeatureStruct + .New(Language.PhonologicalFeatureSystem) + .Symbol(HCFeatureSystem.Segment) + .Symbol("vd+") + .Value + ) + .Value, + LeftEnvironment = Pattern.New().Annotation(Character(Table1, "a")).Value, + RightEnvironment = Pattern.New().Annotation(Character(Table1, "a")).Value, + } + ); + Allophonic.PhonologicalRules.Add(voicing); + + var raising = new RewriteRule + { + Name = "raising", + Lhs = Pattern.New().Annotation(Character(Table1, "a")).Value, + }; + raising.Subrules.Add( + new RewriteSubrule + { + Rhs = Pattern + .New() + .Annotation( + FeatureStruct + .New(Language.PhonologicalFeatureSystem) + .Symbol(HCFeatureSystem.Segment) + .Symbol("high+") + .Symbol("low-") + .Symbol("back-") + .Value + ) + .Value, + LeftEnvironment = Pattern.New().Annotation(Character(Table1, "z")).Value, + } + ); + Allophonic.PhonologicalRules.Add(raising); + + LexEntry root = AddEntry( + "SA", + FeatureStruct.New(Language.SyntacticFeatureSystem).Symbol("V").Value, + Morphophonemic, + "sa" + ); + try + { + var generator = new Morpher(TraceManager, Language); + Assert.That( + generator.GenerateWords(root, new Morpheme[] { redup }, FeatureStruct.New().Value), + Is.EquivalentTo(new[] { "sazi" }), + "forward derivation defines the expected parse" + ); + + string[] unpruned = ParseGlosses(CreateMorpher(false), "sazi"); + Assert.That(unpruned, Does.Contain("SA RED"), "the unpruned engine finds the generated analysis"); + Assert.That(ParseGlosses(CreateMorpher(true), "sazi"), Is.EqualTo(unpruned)); + } + finally + { + Morphophonemic.Entries.Remove(root); + } + } + + private Morpher CreateMorpher(bool pruneCopies, int maxDegreeOfParallelism = 0) + { + return new Morpher(TraceManager, Language, maxDegreeOfParallelism) { PruneDisagreeingCopies = pruneCopies }; + } + + private static string[] ParseGlosses(Morpher morpher, string word) + { + return morpher + .ParseWord(word) + .Select(w => string.Join(" ", w.AllomorphsInMorphOrder.Select(a => a.Morpheme.Gloss))) + .OrderBy(s => s, StringComparer.Ordinal) + .ToArray(); + } + + private static Pattern FullPart() + { + return Pattern.New("copy").Annotation(AnySegment).OneOrMore.Value; + } + + private static Pattern SingleSegmentPart() + { + return Pattern.New("copy").Annotation(AnySegment).Value; + } + + private static AffixProcessRule CreateCopyRule(Pattern part, int copyCount = 2) + { + var rule = new AffixProcessRule + { + Name = "full_copy", + Gloss = "RED", + RequiredSyntacticFeatureStruct = FeatureStruct.New().Value, + OutSyntacticFeatureStruct = FeatureStruct.New().Value, + }; + rule.Allomorphs.Add(CreateCopyAllomorph(part, copyCount)); + return rule; + } + + private static AffixProcessAllomorph CreateCopyAllomorph(Pattern part, int copyCount) + { + var allomorph = new AffixProcessAllomorph { Lhs = { part } }; + for (int i = 0; i < copyCount; i++) + allomorph.Rhs.Add(new CopyFromInput("copy")); + return allomorph; + } + + private (string First, string Second) FindIncompatibleSegments() + { + CharacterDefinition[] segments = Table3 + .Where(definition => definition.Type == HCFeatureSystem.Segment) + .ToArray(); + var pair = segments + .SelectMany( + (first, index) => segments.Skip(index + 1).Select(second => new { First = first, Second = second }) + ) + .First(candidate => !candidate.First.FeatureStruct.IsUnifiable(candidate.Second.FeatureStruct)); + return (pair.First.Representations.First(), pair.Second.Representations.First()); + } + + private Word CreateInput(string representation) + { + var input = new Word(Surface, Table3.Segment(representation)) { AnalysisScope = new AnalysisScope() }; + input.Freeze(); + return input; + } +} From 92cf570160c620ac8ddbd15fcb96901f9b328b01 Mon Sep 17 00:00:00 2001 From: John Lambert Date: Fri, 9 Oct 2026 21:53:22 -0400 Subject: [PATCH 12/20] Prune disagreeing copies through Pattern.Acceptable, also while tracing The copy check is now the analysis rule spec's Pattern.Acceptable, so the three analysis rules use MultiplePatternRule unchanged and DisagreeingCopiesPruningRule is deleted. Affix-process specs set the predicate only when a part is copied, so other rules pay nothing; the compounding spec combines it with its existing head-capture check. Tracing no longer turns pruning off: a pruned match is a pattern that did not match, which a trace already leaves out. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../Morpher.cs | 3 +- .../AnalysisAffixProcessAllomorphRuleSpec.cs | 7 +++- .../AnalysisAffixProcessRule.cs | 5 +-- .../AnalysisCompoundingRule.cs | 5 +-- .../AnalysisCompoundingSubruleRuleSpec.cs | 7 ++-- .../AnalysisMorphologicalTransformRuleSpec.cs | 20 +++++++++- .../AnalysisRealizationalAffixProcessRule.cs | 5 +-- .../DisagreeingCopiesPruningRule.cs | 40 ------------------- .../PruneDisagreeingCopiesTests.cs | 4 +- 9 files changed, 36 insertions(+), 60 deletions(-) delete mode 100644 src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/DisagreeingCopiesPruningRule.cs diff --git a/src/SIL.Machine.Morphology.HermitCrab/Morpher.cs b/src/SIL.Machine.Morphology.HermitCrab/Morpher.cs index 66df6ab1d..c48d327fe 100644 --- a/src/SIL.Machine.Morphology.HermitCrab/Morpher.cs +++ b/src/SIL.Machine.Morphology.HermitCrab/Morpher.cs @@ -114,8 +114,7 @@ public ITraceManager TraceManager /// /// When unapplying an affix-process or compounding rule that copies a part more than once /// (reduplication), skip matches whose copies cannot unify segment by segment. Such a match can never - /// survive synthesis, so this removes only doomed analyses. On by default; ignored while tracing, so - /// a trace still shows each such match failing in synthesis. + /// survive synthesis, so this removes only doomed analyses. On by default, including while tracing. /// public bool PruneDisagreeingCopies { get; set; } diff --git a/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisAffixProcessAllomorphRuleSpec.cs b/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisAffixProcessAllomorphRuleSpec.cs index 9ce94c148..895f9eb18 100644 --- a/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisAffixProcessAllomorphRuleSpec.cs +++ b/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisAffixProcessAllomorphRuleSpec.cs @@ -8,10 +8,13 @@ public class AnalysisAffixProcessAllomorphRuleSpec : AnalysisMorphologicalTransf { private readonly AffixProcessAllomorph _allomorph; - public AnalysisAffixProcessAllomorphRuleSpec(AffixProcessAllomorph allomorph) - : base(allomorph.Lhs, allomorph.Rhs) + public AnalysisAffixProcessAllomorphRuleSpec(AffixProcessAllomorph allomorph, Morpher morpher = null) + : base(allomorph.Lhs, allomorph.Rhs, morpher) { _allomorph = allomorph; + // the matcher builds a Match for every candidate it tests, so rules that copy nothing get no check + if (HasRepeatedParts) + Pattern.Acceptable = CopiesMayAgree; Pattern.Freeze(); } diff --git a/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisAffixProcessRule.cs b/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisAffixProcessRule.cs index b7c204dd0..fcf93da2f 100644 --- a/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisAffixProcessRule.cs +++ b/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisAffixProcessRule.cs @@ -29,9 +29,8 @@ public AnalysisAffixProcessRule(Morpher morpher, AffixProcessRule rule) foreach (AffixProcessAllomorph allo in rule.Allomorphs) { _rules.Add( - new DisagreeingCopiesPruningRule( - morpher, - new AnalysisAffixProcessAllomorphRuleSpec(allo), + new MultiplePatternRule( + new AnalysisAffixProcessAllomorphRuleSpec(allo, morpher), new MatcherSettings { Filter = ann => ann.Type() == HCFeatureSystem.Segment, diff --git a/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisCompoundingRule.cs b/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisCompoundingRule.cs index 6dbf11b9f..8df7e3900 100644 --- a/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisCompoundingRule.cs +++ b/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisCompoundingRule.cs @@ -28,9 +28,8 @@ public AnalysisCompoundingRule(Morpher morpher, CompoundingRule rule) foreach (CompoundingSubrule sr in rule.Subrules) { _rules.Add( - new DisagreeingCopiesPruningRule( - morpher, - new AnalysisCompoundingSubruleRuleSpec(sr), + new MultiplePatternRule( + new AnalysisCompoundingSubruleRuleSpec(sr, morpher), new MatcherSettings { Filter = ann => ann.Type() == HCFeatureSystem.Segment, diff --git a/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisCompoundingSubruleRuleSpec.cs b/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisCompoundingSubruleRuleSpec.cs index 50f6177bb..db11fa690 100644 --- a/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisCompoundingSubruleRuleSpec.cs +++ b/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisCompoundingSubruleRuleSpec.cs @@ -9,11 +9,12 @@ public class AnalysisCompoundingSubruleRuleSpec : AnalysisMorphologicalTransform { private readonly CompoundingSubrule _subrule; - public AnalysisCompoundingSubruleRuleSpec(CompoundingSubrule subrule) - : base(subrule.HeadLhs.Concat(subrule.NonHeadLhs), subrule.Rhs) + public AnalysisCompoundingSubruleRuleSpec(CompoundingSubrule subrule, Morpher morpher = null) + : base(subrule.HeadLhs.Concat(subrule.NonHeadLhs), subrule.Rhs, morpher) { _subrule = subrule; - Pattern.Acceptable = match => _subrule.HeadLhs.Any(part => IsPartCaptured(match, part.Name)); + Pattern.Acceptable = match => + _subrule.HeadLhs.Any(part => IsPartCaptured(match, part.Name)) && CopiesMayAgree(match); Pattern.Freeze(); } diff --git a/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisMorphologicalTransformRuleSpec.cs b/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisMorphologicalTransformRuleSpec.cs index a23ae84b1..826259988 100644 --- a/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisMorphologicalTransformRuleSpec.cs +++ b/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisMorphologicalTransformRuleSpec.cs @@ -9,11 +9,27 @@ public abstract class AnalysisMorphologicalTransformRuleSpec : AnalysisMorphologicalTransform, IPatternRuleSpec { + private readonly Morpher _morpher; + private readonly bool _hasRepeatedParts; + protected AnalysisMorphologicalTransformRuleSpec( IEnumerable> lhs, - IList rhs + IList rhs, + Morpher morpher ) - : base(lhs, rhs) { } + : base(lhs, rhs) + { + _morpher = morpher; + _hasRepeatedParts = HasRepeatedParts; + } + + protected bool CopiesMayAgree(Match match) + { + return _morpher == null + || !_morpher.PruneDisagreeingCopies + || !_hasRepeatedParts + || !HasDisagreeingCopies(match); + } public bool IsApplicable(Word input) { diff --git a/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisRealizationalAffixProcessRule.cs b/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisRealizationalAffixProcessRule.cs index 9862db6bf..544e531c6 100644 --- a/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisRealizationalAffixProcessRule.cs +++ b/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisRealizationalAffixProcessRule.cs @@ -22,9 +22,8 @@ public AnalysisRealizationalAffixProcessRule(Morpher morpher, RealizationalAffix foreach (AffixProcessAllomorph allo in rule.Allomorphs) { _rules.Add( - new DisagreeingCopiesPruningRule( - morpher, - new AnalysisAffixProcessAllomorphRuleSpec(allo), + new MultiplePatternRule( + new AnalysisAffixProcessAllomorphRuleSpec(allo, morpher), new MatcherSettings { Filter = ann => ann.Type() == HCFeatureSystem.Segment, diff --git a/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/DisagreeingCopiesPruningRule.cs b/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/DisagreeingCopiesPruningRule.cs deleted file mode 100644 index e4431ab44..000000000 --- a/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/DisagreeingCopiesPruningRule.cs +++ /dev/null @@ -1,40 +0,0 @@ -using System.Collections.Generic; -using SIL.Machine.Annotations; -using SIL.Machine.Matching; -using SIL.Machine.Rules; - -namespace SIL.Machine.Morphology.HermitCrab.MorphologicalRules -{ - internal sealed class DisagreeingCopiesPruningRule : MultiplePatternRule - { - private readonly Morpher _morpher; - private readonly AnalysisMorphologicalTransformRuleSpec _spec; - private readonly bool _hasRepeatedParts; - - public DisagreeingCopiesPruningRule( - Morpher morpher, - AnalysisMorphologicalTransformRuleSpec ruleSpec, - MatcherSettings matcherSettings - ) - : base(ruleSpec, matcherSettings) - { - _morpher = morpher; - _spec = ruleSpec; - _hasRepeatedParts = ruleSpec.HasRepeatedParts; - } - - protected override IEnumerable ApplyImpl(Word input, ShapeNode start) - { - bool prune = _morpher.PruneDisagreeingCopies && _hasRepeatedParts && !_morpher.TraceManager.IsTracing; - var results = new List(); - foreach (Match match in Matcher.AllMatches(input, start)) - { - if (prune && _spec.HasDisagreeingCopies(match)) - continue; - - results.Add(RuleSpec.ApplyRhs(this, match)); - } - return results; - } - } -} diff --git a/tests/SIL.Machine.Morphology.HermitCrab.Tests/MorphologicalRules/PruneDisagreeingCopiesTests.cs b/tests/SIL.Machine.Morphology.HermitCrab.Tests/MorphologicalRules/PruneDisagreeingCopiesTests.cs index 2d2b3aaa0..b3af6bc06 100644 --- a/tests/SIL.Machine.Morphology.HermitCrab.Tests/MorphologicalRules/PruneDisagreeingCopiesTests.cs +++ b/tests/SIL.Machine.Morphology.HermitCrab.Tests/MorphologicalRules/PruneDisagreeingCopiesTests.cs @@ -47,7 +47,7 @@ public void DisagreeingCopiesAreRemovedOnlyWhenPruning(bool pruneCopies, int exp } [Test] - public void TracingKeepsDisagreeingCopies() + public void TracingStillPrunesDisagreeingCopies() { (string first, string second) = FindIncompatibleSegments(); AffixProcessRule rule = CreateCopyRule(SingleSegmentPart()); @@ -59,7 +59,7 @@ public void TracingKeepsDisagreeingCopies() { List outputs = new AnalysisAffixProcessRule(CreateMorpher(true), rule).Apply(input).ToList(); - Assert.That(outputs.Count, Is.EqualTo(1)); + Assert.That(outputs, Is.Empty); } finally { From db8b3d108fc01db907a0001f24e6a31488ace810 Mon Sep 17 00:00:00 2001 From: John Lambert Date: Sat, 10 Oct 2026 07:44:47 -0400 Subject: [PATCH 13/20] Line up copies across unapplied deletions instead of keeping them A copy containing an optional segment was always kept. Now copies are compared by alignment: an optional segment may stand for any number of segments of the other copy it unifies with, including none, and the copies disagree only if no alignment exists. A single optional node can stand for several segments because DeletionReapplications (0 by default) lets analysis restore fewer segments than a deletion removed; matching it to at most one segment pruned the valid parse in CapLimitedDeletionInOneCopyParsesTheSameWithAndWithoutPruning. Copies without optional nodes keep the linear comparison. Optional nodes that are not segments still make a copy undecidable. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../AnalysisMorphologicalTransform.cs | 59 ++++++++++++++++--- .../PruneDisagreeingCopiesTests.cs | 33 +++++++++++ 2 files changed, 84 insertions(+), 8 deletions(-) diff --git a/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisMorphologicalTransform.cs b/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisMorphologicalTransform.cs index c3244d021..5871dbb7c 100644 --- a/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisMorphologicalTransform.cs +++ b/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisMorphologicalTransform.cs @@ -54,8 +54,10 @@ internal bool HasRepeatedParts /// /// Synthesis writes every copy of a part from the same input, and anything that later changes one /// copy is unapplied before this rule, so copies proven to disagree segment by segment cannot lead - /// to a valid analysis. A part the rule modifies, or a copy containing an unapplied deletion, is - /// never reported as disagreeing. + /// to a valid analysis. An optional segment (an unapplied deletion) stands for any number of segments + /// it unifies with, because can restore fewer segments + /// than were deleted, so copies disagree only if no such choice lines them up. A part the rule + /// modifies, or a copy with an optional node that is not a segment, is never reported as disagreeing. /// internal bool HasDisagreeingCopies(Match match) { @@ -95,7 +97,7 @@ out List> copies .GetSkippedOptionalNodes(match.Input.Shape, capture.Range) .Concat(match.Input.Shape.GetNodes(capture.Range)) .ToList(); - if (nodes.Any(node => node.Annotation.Optional)) + if (nodes.Any(node => node.Annotation.Optional && node.Annotation.Type() != HCFeatureSystem.Segment)) return false; copies.Add(nodes.Where(node => node.Annotation.Type() == HCFeatureSystem.Segment).ToList()); @@ -110,17 +112,58 @@ private static bool AnyCopiesDisagree(List> copies) { for (int j = i + 1; j < copies.Count; j++) { - if (copies[i].Count != copies[j].Count) + if (!CanAlign(copies[i], copies[j])) return true; + } + } + return false; + } - for (int k = 0; k < copies[i].Count; k++) + private static bool CanAlign(List first, List second) + { + if (!first.Any(node => node.Annotation.Optional) && !second.Any(node => node.Annotation.Optional)) + { + if (first.Count != second.Count) + return false; + for (int k = 0; k < first.Count; k++) + { + if (!first[k].Annotation.FeatureStruct.IsUnifiable(second[k].Annotation.FeatureStruct)) + return false; + } + return true; + } + + // reachable[i, j]: the first i nodes of one copy can be lined up with the first j of the other; + // an optional node is passed over, or absorbs a node of the other copy and stays available + var reachable = new bool[first.Count + 1, second.Count + 1]; + reachable[0, 0] = true; + for (int i = 0; i <= first.Count; i++) + { + for (int j = 0; j <= second.Count; j++) + { + if (!reachable[i, j]) + continue; + bool firstOptional = i < first.Count && first[i].Annotation.Optional; + bool secondOptional = j < second.Count && second[j].Annotation.Optional; + if (firstOptional) + reachable[i + 1, j] = true; + if (secondOptional) + reachable[i, j + 1] = true; + if ( + i < first.Count + && j < second.Count + && first[i].Annotation.FeatureStruct.IsUnifiable(second[j].Annotation.FeatureStruct) + ) { - if (!copies[i][k].Annotation.FeatureStruct.IsUnifiable(copies[j][k].Annotation.FeatureStruct)) - return true; + reachable[i + 1, j + 1] = true; + if (firstOptional) + reachable[i, j + 1] = true; + if (secondOptional) + reachable[i + 1, j] = true; } } } - return false; + return reachable[first.Count, second.Count]; } public Pattern Pattern diff --git a/tests/SIL.Machine.Morphology.HermitCrab.Tests/MorphologicalRules/PruneDisagreeingCopiesTests.cs b/tests/SIL.Machine.Morphology.HermitCrab.Tests/MorphologicalRules/PruneDisagreeingCopiesTests.cs index b3af6bc06..fa573bf21 100644 --- a/tests/SIL.Machine.Morphology.HermitCrab.Tests/MorphologicalRules/PruneDisagreeingCopiesTests.cs +++ b/tests/SIL.Machine.Morphology.HermitCrab.Tests/MorphologicalRules/PruneDisagreeingCopiesTests.cs @@ -83,6 +83,31 @@ public void EveryPairOfCopiesIsCompared(bool pruneCopies, int expectedOutputs) Assert.That(outputs.Count, Is.EqualTo(expectedOutputs)); } + [Test] + public void OptionalSegmentThatCannotLineUpIsPruned() + { + // skipping or keeping the restored segment, every split still pairs the two incompatible segments + (string first, string second) = FindIncompatibleSegments(); + AffixProcessRule rule = CreateCopyRule(FullPart()); + Word input = CreateInputWithOptionalSecondSegment(first + second + second); + + Assert.That(new AnalysisAffixProcessRule(CreateMorpher(false), rule).Apply(input), Is.Not.Empty); + Assert.That(new AnalysisAffixProcessRule(CreateMorpher(true), rule).Apply(input), Is.Empty); + } + + [Test] + public void OptionalSegmentThatLinesUpWhenSkippedIsKept() + { + (string first, string second) = FindIncompatibleSegments(); + AffixProcessRule rule = CreateCopyRule(FullPart()); + Word input = CreateInputWithOptionalSecondSegment(first + second + first); + + Assert.That( + new AnalysisAffixProcessRule(CreateMorpher(true), rule).Apply(input).Count(), + Is.EqualTo(new AnalysisAffixProcessRule(CreateMorpher(false), rule).Apply(input).Count()) + ); + } + [TestCase(false, 3)] [TestCase(true, 1)] public void RealizationalFullCopyKeepsOnlyTheSplitWhoseCopiesAgree(bool pruneCopies, int expectedOutputs) @@ -358,6 +383,14 @@ private static AffixProcessAllomorph CreateCopyAllomorph(Pattern Date: Tue, 20 Oct 2026 13:43:28 -0400 Subject: [PATCH 14/20] Restore the public and protected signatures of the analysis rule specs and wire the morpher through an internal property The pruning change had altered the protected constructor of AnalysisMorphologicalTransformRuleSpec, added an optional morpher parameter to the public AnalysisAffixProcessAllomorphRuleSpec and AnalysisCompoundingSubruleRuleSpec constructors, and turned GetSkippedOptionalNodes into a protected internal static. Those are back to their master form; the rules now set an internal PruningMorpher property and the pruning code calls an internal static CollectSkippedOptionalNodes. Co-Authored-By: Claude Sonnet 5.5 --- .../AnalysisAffixProcessAllomorphRuleSpec.cs | 4 +-- .../AnalysisAffixProcessRule.cs | 2 +- .../AnalysisCompoundingRule.cs | 2 +- .../AnalysisCompoundingSubruleRuleSpec.cs | 6 ++-- .../AnalysisMorphologicalTransform.cs | 12 ++++---- .../AnalysisMorphologicalTransformRuleSpec.cs | 28 +++++++++---------- .../AnalysisRealizationalAffixProcessRule.cs | 2 +- .../MorphologicalOutputAction.cs | 7 ++++- 8 files changed, 33 insertions(+), 30 deletions(-) diff --git a/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisAffixProcessAllomorphRuleSpec.cs b/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisAffixProcessAllomorphRuleSpec.cs index 895f9eb18..8103be96f 100644 --- a/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisAffixProcessAllomorphRuleSpec.cs +++ b/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisAffixProcessAllomorphRuleSpec.cs @@ -8,8 +8,8 @@ public class AnalysisAffixProcessAllomorphRuleSpec : AnalysisMorphologicalTransf { private readonly AffixProcessAllomorph _allomorph; - public AnalysisAffixProcessAllomorphRuleSpec(AffixProcessAllomorph allomorph, Morpher morpher = null) - : base(allomorph.Lhs, allomorph.Rhs, morpher) + public AnalysisAffixProcessAllomorphRuleSpec(AffixProcessAllomorph allomorph) + : base(allomorph.Lhs, allomorph.Rhs) { _allomorph = allomorph; // the matcher builds a Match for every candidate it tests, so rules that copy nothing get no check diff --git a/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisAffixProcessRule.cs b/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisAffixProcessRule.cs index fcf93da2f..59651247c 100644 --- a/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisAffixProcessRule.cs +++ b/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisAffixProcessRule.cs @@ -30,7 +30,7 @@ public AnalysisAffixProcessRule(Morpher morpher, AffixProcessRule rule) { _rules.Add( new MultiplePatternRule( - new AnalysisAffixProcessAllomorphRuleSpec(allo, morpher), + new AnalysisAffixProcessAllomorphRuleSpec(allo) { PruningMorpher = morpher }, new MatcherSettings { Filter = ann => ann.Type() == HCFeatureSystem.Segment, diff --git a/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisCompoundingRule.cs b/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisCompoundingRule.cs index 8df7e3900..c5cdfb11d 100644 --- a/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisCompoundingRule.cs +++ b/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisCompoundingRule.cs @@ -29,7 +29,7 @@ public AnalysisCompoundingRule(Morpher morpher, CompoundingRule rule) { _rules.Add( new MultiplePatternRule( - new AnalysisCompoundingSubruleRuleSpec(sr, morpher), + new AnalysisCompoundingSubruleRuleSpec(sr) { PruningMorpher = morpher }, new MatcherSettings { Filter = ann => ann.Type() == HCFeatureSystem.Segment, diff --git a/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisCompoundingSubruleRuleSpec.cs b/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisCompoundingSubruleRuleSpec.cs index db11fa690..0de9ea95e 100644 --- a/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisCompoundingSubruleRuleSpec.cs +++ b/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisCompoundingSubruleRuleSpec.cs @@ -1,4 +1,4 @@ -using System.Linq; +using System.Linq; using SIL.Machine.Annotations; using SIL.Machine.Matching; using SIL.Machine.Rules; @@ -9,8 +9,8 @@ public class AnalysisCompoundingSubruleRuleSpec : AnalysisMorphologicalTransform { private readonly CompoundingSubrule _subrule; - public AnalysisCompoundingSubruleRuleSpec(CompoundingSubrule subrule, Morpher morpher = null) - : base(subrule.HeadLhs.Concat(subrule.NonHeadLhs), subrule.Rhs, morpher) + public AnalysisCompoundingSubruleRuleSpec(CompoundingSubrule subrule) + : base(subrule.HeadLhs.Concat(subrule.NonHeadLhs), subrule.Rhs) { _subrule = subrule; Pattern.Acceptable = match => diff --git a/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisMorphologicalTransform.cs b/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisMorphologicalTransform.cs index 5871dbb7c..e7b923faf 100644 --- a/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisMorphologicalTransform.cs +++ b/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisMorphologicalTransform.cs @@ -36,6 +36,11 @@ IList rhs } } + internal bool HasRepeatedParts + { + get { return _capturedParts.Values.Any(count => count >= 2); } + } + internal static string GetGroupName(string partName, int index) { return string.Format("{0}_{1}", partName, index); @@ -46,11 +51,6 @@ protected IDictionary CapturedParts get { return _capturedParts; } } - internal bool HasRepeatedParts - { - get { return _capturedParts.Values.Any(count => count >= 2); } - } - /// /// Synthesis writes every copy of a part from the same input, and anything that later changes one /// copy is unapplied before this rule, so copies proven to disagree segment by segment cannot lead @@ -94,7 +94,7 @@ out List> copies // an unapplied word-initial deletion sits before the capture, not inside it List nodes = MorphologicalOutputAction - .GetSkippedOptionalNodes(match.Input.Shape, capture.Range) + .CollectSkippedOptionalNodes(match.Input.Shape, capture.Range) .Concat(match.Input.Shape.GetNodes(capture.Range)) .ToList(); if (nodes.Any(node => node.Annotation.Optional && node.Annotation.Type() != HCFeatureSystem.Segment)) diff --git a/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisMorphologicalTransformRuleSpec.cs b/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisMorphologicalTransformRuleSpec.cs index 826259988..254fcd3be 100644 --- a/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisMorphologicalTransformRuleSpec.cs +++ b/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisMorphologicalTransformRuleSpec.cs @@ -1,4 +1,5 @@ -using System.Collections.Generic; +using System; +using System.Collections.Generic; using SIL.Machine.Annotations; using SIL.Machine.Matching; using SIL.Machine.Rules; @@ -9,25 +10,22 @@ public abstract class AnalysisMorphologicalTransformRuleSpec : AnalysisMorphologicalTransform, IPatternRuleSpec { - private readonly Morpher _morpher; - private readonly bool _hasRepeatedParts; - protected AnalysisMorphologicalTransformRuleSpec( IEnumerable> lhs, - IList rhs, - Morpher morpher + IList rhs ) - : base(lhs, rhs) - { - _morpher = morpher; - _hasRepeatedParts = HasRepeatedParts; - } + : base(lhs, rhs) { } + + /// + /// The morpher whose setting governs pruning; unset means no pruning. + /// + internal Morpher PruningMorpher { get; set; } - protected bool CopiesMayAgree(Match match) + internal bool CopiesMayAgree(Match match) { - return _morpher == null - || !_morpher.PruneDisagreeingCopies - || !_hasRepeatedParts + return PruningMorpher == null + || !PruningMorpher.PruneDisagreeingCopies + || !HasRepeatedParts || !HasDisagreeingCopies(match); } diff --git a/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisRealizationalAffixProcessRule.cs b/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisRealizationalAffixProcessRule.cs index 544e531c6..db1c99cb8 100644 --- a/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisRealizationalAffixProcessRule.cs +++ b/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisRealizationalAffixProcessRule.cs @@ -23,7 +23,7 @@ public AnalysisRealizationalAffixProcessRule(Morpher morpher, RealizationalAffix { _rules.Add( new MultiplePatternRule( - new AnalysisAffixProcessAllomorphRuleSpec(allo, morpher), + new AnalysisAffixProcessAllomorphRuleSpec(allo) { PruningMorpher = morpher }, new MatcherSettings { Filter = ann => ann.Type() == HCFeatureSystem.Segment, diff --git a/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/MorphologicalOutputAction.cs b/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/MorphologicalOutputAction.cs index 53ed1ddd0..2bb7eaa57 100644 --- a/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/MorphologicalOutputAction.cs +++ b/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/MorphologicalOutputAction.cs @@ -37,7 +37,12 @@ IDictionary capturedParts /// The output word synthesis. public abstract IEnumerable> Apply(Match match, Word output); - protected internal static IEnumerable GetSkippedOptionalNodes(Shape shape, Range range) + protected IEnumerable GetSkippedOptionalNodes(Shape shape, Range range) + { + return CollectSkippedOptionalNodes(shape, range); + } + + internal static IEnumerable CollectSkippedOptionalNodes(Shape shape, Range range) { ShapeNode node = range.Start.Prev; var skippedNodes = new List(); From 7dbf4d98871907b40e8e5742ceedd93d725a5054 Mon Sep 17 00:00:00 2001 From: John Lambert Date: Tue, 20 Oct 2026 13:51:08 -0400 Subject: [PATCH 15/20] Pin reduplication pruning for optional non-segments, word-initial optional segments and empty copies with exact expected analyses Each new test asserts the exact analysis forms produced with pruning off and on. WordInitialOptionalNonSegmentIsNeverPruned fails if the skipped word-initial nodes are dropped, OptionalNonSegmentInACopyIsNeverPruned fails if the optional non-segment bailout is removed, and EmptyCopyIsNeverPruned covers the unsuccessful-capture bailout; with the existing tests, TryGetSegmentCopies is now fully line and branch covered. Co-Authored-By: Claude Sonnet 5.5 --- .../PruneDisagreeingCopiesTests.cs | 78 +++++++++++++++++++ 1 file changed, 78 insertions(+) diff --git a/tests/SIL.Machine.Morphology.HermitCrab.Tests/MorphologicalRules/PruneDisagreeingCopiesTests.cs b/tests/SIL.Machine.Morphology.HermitCrab.Tests/MorphologicalRules/PruneDisagreeingCopiesTests.cs index fa573bf21..43ab16b87 100644 --- a/tests/SIL.Machine.Morphology.HermitCrab.Tests/MorphologicalRules/PruneDisagreeingCopiesTests.cs +++ b/tests/SIL.Machine.Morphology.HermitCrab.Tests/MorphologicalRules/PruneDisagreeingCopiesTests.cs @@ -108,6 +108,67 @@ public void OptionalSegmentThatLinesUpWhenSkippedIsKept() ); } + [Test] + public void WordInitialOptionalSegmentThatLinesUpWhenRestoredIsKept() + { + // "(a)bab" for stem "ab": the optional first segment can be restored into the first copy so [a,b] meets [a,b] + (string first, string second) = FindIncompatibleSegments(); + AffixProcessRule rule = CreateCopyRule(FullPart()); + Word input = CreateInputWithOptionalFirstSegment(first + second + first + second); + + string[] unpruned = AnalysisForms(false, rule, input); + Assert.That(unpruned, Is.EquivalentTo(new[] { first, first + second, first + second + first })); + // only the split that pairs "(a)b" with "ab" lines up; the other splits pair incompatible segments + Assert.That(AnalysisForms(true, rule, input), Is.EqualTo(new[] { first + second })); + } + + [Test] + public void OptionalNonSegmentInACopyIsNeverPruned() + { + (string first, string second) = FindIncompatibleSegments(); + AffixProcessRule rule = CreateCopyRule(FullPart()); + var input = new Word(Surface, Table3.Segment(first + second + second)) { AnalysisScope = new AnalysisScope() }; + input.Shape.AddAfter(input.Shape.First.Next, FeatureStruct.New().Symbol(HCFeatureSystem.Boundary).Value, true); + input.Freeze(); + + // a copy holding an optional boundary is never reported as disagreeing, so its split survives pruning; + // the split of two plain, incompatible copies is the only one removed + Assert.That(AnalysisForms(false, rule, input), Is.EquivalentTo(new[] { first, first + second })); + Assert.That(AnalysisForms(true, rule, input), Is.EqualTo(new[] { first })); + } + + [Test] + public void WordInitialOptionalNonSegmentIsNeverPruned() + { + // the matcher skips the leading boundary, so it sits before the first copy rather than inside it + (string first, string second) = FindIncompatibleSegments(); + AffixProcessRule rule = CreateCopyRule(FullPart()); + var input = new Word(Surface, Table3.Segment(first + second + first + second)) + { + AnalysisScope = new AnalysisScope(), + }; + input.Shape.AddAfter(input.Shape.Begin, FeatureStruct.New().Symbol(HCFeatureSystem.Boundary).Value, true); + input.Freeze(); + + // nothing is pruned: with the boundary ignored, pruning keeps exactly what the unpruned rule produces + string[] expected = { first, first + second, first + second + first }; + Assert.That(AnalysisForms(false, rule, input), Is.EquivalentTo(expected)); + Assert.That(AnalysisForms(true, rule, input), Is.EquivalentTo(expected)); + } + + [Test] + public void EmptyCopyIsNeverPruned() + { + (string first, string second) = FindIncompatibleSegments(); + Pattern part = Pattern.New("copy").Annotation(AnySegment).ZeroOrMore.Value; + AffixProcessRule rule = CreateCopyRule(part); + Word input = CreateInput(first + second); + + // the split with an empty copy has nothing to compare, so only the split of two incompatible copies is removed + Assert.That(AnalysisForms(false, rule, input), Is.EquivalentTo(new[] { first, first + second })); + Assert.That(AnalysisForms(true, rule, input), Is.EqualTo(new[] { first + second })); + } + [TestCase(false, 3)] [TestCase(true, 1)] public void RealizationalFullCopyKeepsOnlyTheSplitWhoseCopiesAgree(bool pruneCopies, int expectedOutputs) @@ -383,6 +444,15 @@ private static AffixProcessAllomorph CreateCopyAllomorph(Pattern output.Shape.ToString(Table3, false)) + .OrderBy(form => form, StringComparer.Ordinal) + .ToArray(); + } + private Word CreateInputWithOptionalSecondSegment(string representation) { var input = new Word(Surface, Table3.Segment(representation)) { AnalysisScope = new AnalysisScope() }; @@ -391,6 +461,14 @@ private Word CreateInputWithOptionalSecondSegment(string representation) return input; } + private Word CreateInputWithOptionalFirstSegment(string representation) + { + var input = new Word(Surface, Table3.Segment(representation)) { AnalysisScope = new AnalysisScope() }; + input.Shape.First.Annotation.Optional = true; + input.Freeze(); + return input; + } + private Word CreateInput(string representation) { var input = new Word(Surface, Table3.Segment(representation)) { AnalysisScope = new AnalysisScope() }; From 1cbfd5403db5c2d5cd2ce6be74a80ee33ba97648 Mon Sep 17 00:00:00 2001 From: John Lambert Date: Tue, 20 Oct 2026 13:53:10 -0400 Subject: [PATCH 16/20] Decide once in the transform base whether a rule repeats a part and install the copy-agreement predicate only then The base now records HasRepeatedParts once at construction and exposes CopyAgreementCheck, which is null when nothing repeats. The affix spec and the compounding spec both consume it, so the per-spec LINQ re-evaluation and the third check inside CopiesMayAgree are gone; behaviour is unchanged. This also drops a stray byte-order mark the earlier commit had added to AnalysisCompoundingSubruleRuleSpec.cs. Co-Authored-By: Claude Sonnet 5.5 --- .../AnalysisAffixProcessAllomorphRuleSpec.cs | 4 +--- .../AnalysisCompoundingSubruleRuleSpec.cs | 7 +++++-- .../AnalysisMorphologicalTransform.cs | 6 ++---- .../AnalysisMorphologicalTransformRuleSpec.cs | 18 ++++++++++++------ 4 files changed, 20 insertions(+), 15 deletions(-) diff --git a/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisAffixProcessAllomorphRuleSpec.cs b/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisAffixProcessAllomorphRuleSpec.cs index 8103be96f..580b663fb 100644 --- a/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisAffixProcessAllomorphRuleSpec.cs +++ b/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisAffixProcessAllomorphRuleSpec.cs @@ -12,9 +12,7 @@ public AnalysisAffixProcessAllomorphRuleSpec(AffixProcessAllomorph allomorph) : base(allomorph.Lhs, allomorph.Rhs) { _allomorph = allomorph; - // the matcher builds a Match for every candidate it tests, so rules that copy nothing get no check - if (HasRepeatedParts) - Pattern.Acceptable = CopiesMayAgree; + Pattern.Acceptable = CopyAgreementCheck; Pattern.Freeze(); } diff --git a/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisCompoundingSubruleRuleSpec.cs b/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisCompoundingSubruleRuleSpec.cs index 0de9ea95e..b8ff171e9 100644 --- a/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisCompoundingSubruleRuleSpec.cs +++ b/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisCompoundingSubruleRuleSpec.cs @@ -1,4 +1,5 @@ -using System.Linq; +using System; +using System.Linq; using SIL.Machine.Annotations; using SIL.Machine.Matching; using SIL.Machine.Rules; @@ -13,8 +14,10 @@ public AnalysisCompoundingSubruleRuleSpec(CompoundingSubrule subrule) : base(subrule.HeadLhs.Concat(subrule.NonHeadLhs), subrule.Rhs) { _subrule = subrule; + Func, bool> copyAgreement = CopyAgreementCheck; Pattern.Acceptable = match => - _subrule.HeadLhs.Any(part => IsPartCaptured(match, part.Name)) && CopiesMayAgree(match); + _subrule.HeadLhs.Any(part => IsPartCaptured(match, part.Name)) + && (copyAgreement == null || copyAgreement(match)); Pattern.Freeze(); } diff --git a/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisMorphologicalTransform.cs b/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisMorphologicalTransform.cs index e7b923faf..d2dfdb32d 100644 --- a/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisMorphologicalTransform.cs +++ b/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisMorphologicalTransform.cs @@ -34,12 +34,10 @@ IList rhs ); } } + HasRepeatedParts = _capturedParts.Values.Any(count => count >= 2); } - internal bool HasRepeatedParts - { - get { return _capturedParts.Values.Any(count => count >= 2); } - } + internal bool HasRepeatedParts { get; } internal static string GetGroupName(string partName, int index) { diff --git a/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisMorphologicalTransformRuleSpec.cs b/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisMorphologicalTransformRuleSpec.cs index 254fcd3be..d36d640b1 100644 --- a/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisMorphologicalTransformRuleSpec.cs +++ b/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisMorphologicalTransformRuleSpec.cs @@ -17,16 +17,22 @@ IList rhs : base(lhs, rhs) { } /// - /// The morpher whose setting governs pruning; unset means no pruning. + /// The morpher whose setting governs pruning; unset means none. /// internal Morpher PruningMorpher { get; set; } - internal bool CopiesMayAgree(Match match) + /// + /// The acceptance predicate that prunes matches whose copies cannot agree, or null when no part repeats and + /// there is nothing to check (the matcher builds a Match for every candidate it tests). + /// + internal Func, bool> CopyAgreementCheck + { + get { return HasRepeatedParts ? CopiesMayAgree : (Func, bool>)null; } + } + + private bool CopiesMayAgree(Match match) { - return PruningMorpher == null - || !PruningMorpher.PruneDisagreeingCopies - || !HasRepeatedParts - || !HasDisagreeingCopies(match); + return PruningMorpher == null || !PruningMorpher.PruneDisagreeingCopies || !HasDisagreeingCopies(match); } public bool IsApplicable(Word input) From 9778d9937b83f754e564b9927e8bebffb7159f3c Mon Sep 17 00:00:00 2001 From: John Lambert Date: Tue, 20 Oct 2026 14:06:02 -0400 Subject: [PATCH 17/20] Pin that a literal rewrite of one copy parses the same with and without pruning Depends on #538's literal-rewrite unapplication (fix/hc-strrep-unapplication). Co-Authored-By: Claude Opus 5.5 --- .../PruneDisagreeingCopiesTests.cs | 99 +++++++++++++++++++ 1 file changed, 99 insertions(+) diff --git a/tests/SIL.Machine.Morphology.HermitCrab.Tests/MorphologicalRules/PruneDisagreeingCopiesTests.cs b/tests/SIL.Machine.Morphology.HermitCrab.Tests/MorphologicalRules/PruneDisagreeingCopiesTests.cs index 43ab16b87..3748834db 100644 --- a/tests/SIL.Machine.Morphology.HermitCrab.Tests/MorphologicalRules/PruneDisagreeingCopiesTests.cs +++ b/tests/SIL.Machine.Morphology.HermitCrab.Tests/MorphologicalRules/PruneDisagreeingCopiesTests.cs @@ -386,6 +386,105 @@ public void OpaqueChangeToOneCopyParsesTheSameWithAndWithoutPruning() } } + /// + /// Feature-bearing rewrite (s -> z between vowels, voicing): analysis unapplies it, so pruning keeps the parse. + /// + [TestCase(false)] + [TestCase(true)] + public void FeatureRewriteOfOneCopyParsesTheSameWithAndWithoutPruning(bool pruneCopies) + { + AffixProcessRule redup = CreateCopyRule(FullPart()); + redup.RequiredSyntacticFeatureStruct = FeatureStruct.New(Language.SyntacticFeatureSystem).Symbol("V").Value; + Morphophonemic.MorphologicalRules.Add(redup); + Allophonic.PhonologicalRules.Add(CreateBetweenVowelsRewrite(Table1, "s", Character(Table1, "z"))); + LexEntry root = AddEntry( + "ROOT", + FeatureStruct.New(Language.SyntacticFeatureSystem).Symbol("V").Value, + Morphophonemic, + "sa" + ); + try + { + Assert.That( + new Morpher(TraceManager, Language).GenerateWords( + root, + new Morpheme[] { redup }, + FeatureStruct.New().Value + ), + Is.EquivalentTo(new[] { "saza" }) + ); + Assert.That(ParseGlosses(CreateMorpher(pruneCopies), "saza"), Does.Contain("ROOT RED")); + } + finally + { + Morphophonemic.Entries.Remove(root); + } + } + + /// + /// A rewrite between segments that differ only by their string representation (no symbolic features) is + /// never unapplied, so analysis cannot make the changed copy agree with the other; pruning must not drop it. + /// + [TestCase(false)] + [TestCase(true)] + public void StrRepOnlyRewriteOfOneCopyParsesTheSameWithAndWithoutPruning(bool pruneCopies) + { + var table = new CharacterDefinitionTable { Name = "strrep" }; + table.AddSegment("a"); + table.AddSegment("s"); + table.AddSegment("z"); + var syntactic = new SyntacticFeatureSystem(); + syntactic.AddPartsOfSpeech(new FeatureSymbol("V", "Verb")); + syntactic.Freeze(); + var phonological = new FeatureSystem(); + phonological.Freeze(); + var stratum = new Stratum(table) { Name = "S", MorphologicalRuleOrder = MorphologicalRuleOrder.Unordered }; + var language = new Language + { + Name = "StrRepOnly", + PhonologicalFeatureSystem = phonological, + SyntacticFeatureSystem = syntactic, + Strata = { stratum }, + }; + AffixProcessRule redup = CreateCopyRule(FullPart()); + redup.RequiredSyntacticFeatureStruct = FeatureStruct.New(syntactic).Symbol("V").Value; + stratum.MorphologicalRules.Add(redup); + stratum.PhonologicalRules.Add(CreateBetweenVowelsRewrite(table, "s", table["z"].FeatureStruct)); + var root = new LexEntry + { + Id = "ROOT", + Gloss = "ROOT", + SyntacticFeatureStruct = FeatureStruct.New(syntactic).Symbol("V").Value, + }; + root.Allomorphs.Add(new RootAllomorph(new Segments(table, "sa", true))); + stratum.Entries.Add(root); + + var morpher = new Morpher(TraceManager, language) { PruneDisagreeingCopies = pruneCopies }; + Assert.That( + morpher.GenerateWords(root, new Morpheme[] { redup }, FeatureStruct.New().Value), + Is.EquivalentTo(new[] { "saza" }) + ); + Assert.That(ParseGlosses(morpher, "saza"), Does.Contain("ROOT RED")); + } + + private RewriteRule CreateBetweenVowelsRewrite(CharacterDefinitionTable table, string from, FeatureStruct rhs) + { + var rule = new RewriteRule + { + Name = "rewrite", + Lhs = Pattern.New().Annotation(Character(table, from)).Value, + }; + rule.Subrules.Add( + new RewriteSubrule + { + Rhs = Pattern.New().Annotation(rhs).Value, + LeftEnvironment = Pattern.New().Annotation(Character(table, "a")).Value, + RightEnvironment = Pattern.New().Annotation(Character(table, "a")).Value, + } + ); + return rule; + } + private Morpher CreateMorpher(bool pruneCopies, int maxDegreeOfParallelism = 0) { return new Morpher(TraceManager, Language, maxDegreeOfParallelism) { PruneDisagreeingCopies = pruneCopies }; From d55b65a2638ea36cb6bea24c9d97b7f2e93c2520 Mon Sep 17 00:00:00 2001 From: John Lambert Date: Wed, 21 Oct 2026 09:57:51 -0400 Subject: [PATCH 18/20] Reword the StrRep-only rewrite test comment to say analysis must unapply the rewrite for pruning to keep the parse Co-Authored-By: Claude Opus 5.5 --- .../MorphologicalRules/PruneDisagreeingCopiesTests.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/SIL.Machine.Morphology.HermitCrab.Tests/MorphologicalRules/PruneDisagreeingCopiesTests.cs b/tests/SIL.Machine.Morphology.HermitCrab.Tests/MorphologicalRules/PruneDisagreeingCopiesTests.cs index 3748834db..fc7814b69 100644 --- a/tests/SIL.Machine.Morphology.HermitCrab.Tests/MorphologicalRules/PruneDisagreeingCopiesTests.cs +++ b/tests/SIL.Machine.Morphology.HermitCrab.Tests/MorphologicalRules/PruneDisagreeingCopiesTests.cs @@ -422,8 +422,8 @@ public void FeatureRewriteOfOneCopyParsesTheSameWithAndWithoutPruning(bool prune } /// - /// A rewrite between segments that differ only by their string representation (no symbolic features) is - /// never unapplied, so analysis cannot make the changed copy agree with the other; pruning must not drop it. + /// A rewrite between segments that differ only by their string representation (no symbolic features) must + /// be unapplied by analysis so the changed copy can agree with the other; otherwise pruning would drop the parse. /// [TestCase(false)] [TestCase(true)] From 394e7303d2fcc3f4f49ff02b80590d4d0849c266 Mon Sep 17 00:00:00 2001 From: John Lambert Date: Wed, 21 Oct 2026 10:00:31 -0400 Subject: [PATCH 19/20] Skip the copy-agreement predicate for a part that is only repeated because the rule also modifies it Co-Authored-By: Claude Opus 5.5 --- .../AnalysisMorphologicalTransform.cs | 3 +- .../PruneDisagreeingCopiesTests.cs | 64 +++++++++++++++++++ 2 files changed, 66 insertions(+), 1 deletion(-) diff --git a/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisMorphologicalTransform.cs b/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisMorphologicalTransform.cs index d2dfdb32d..f0d1fa738 100644 --- a/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisMorphologicalTransform.cs +++ b/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisMorphologicalTransform.cs @@ -34,7 +34,8 @@ IList rhs ); } } - HasRepeatedParts = _capturedParts.Values.Any(count => count >= 2); + // a modified part is never checked by HasDisagreeingCopies, so repeating it needs no predicate + HasRepeatedParts = _capturedParts.Any(part => part.Value >= 2 && !_modifyFromInfos.ContainsKey(part.Key)); } internal bool HasRepeatedParts { get; } diff --git a/tests/SIL.Machine.Morphology.HermitCrab.Tests/MorphologicalRules/PruneDisagreeingCopiesTests.cs b/tests/SIL.Machine.Morphology.HermitCrab.Tests/MorphologicalRules/PruneDisagreeingCopiesTests.cs index fc7814b69..d532e6457 100644 --- a/tests/SIL.Machine.Morphology.HermitCrab.Tests/MorphologicalRules/PruneDisagreeingCopiesTests.cs +++ b/tests/SIL.Machine.Morphology.HermitCrab.Tests/MorphologicalRules/PruneDisagreeingCopiesTests.cs @@ -467,6 +467,70 @@ public void StrRepOnlyRewriteOfOneCopyParsesTheSameWithAndWithoutPruning(bool pr Assert.That(ParseGlosses(morpher, "saza"), Does.Contain("ROOT RED")); } + [Test] + public void CopyAndModifyOfTheSamePartDoesNotInstallTheCopyAgreementPredicate() + { + AffixProcessAllomorph allomorph = CreateCopyAndModifyAllomorph(); + + Assert.That(new AnalysisAffixProcessAllomorphRuleSpec(allomorph).CopyAgreementCheck, Is.Null); + } + + [Test] + public void TwoCopiesOfAPartStillInstallTheCopyAgreementPredicate() + { + AffixProcessAllomorph allomorph = CreateCopyAllomorph(FullPart(), 2); + + Assert.That(new AnalysisAffixProcessAllomorphRuleSpec(allomorph).CopyAgreementCheck, Is.Not.Null); + } + + [TestCase(false)] + [TestCase(true)] + public void CopyAndModifyOfTheSamePartParsesTheSameWithAndWithoutPruning(bool pruneCopies) + { + var rule = new AffixProcessRule + { + Name = "copy_modify", + Gloss = "RED", + RequiredSyntacticFeatureStruct = FeatureStruct.New().Value, + OutSyntacticFeatureStruct = FeatureStruct.New().Value, + }; + rule.Allomorphs.Add(CreateCopyAndModifyAllomorph()); + Morphophonemic.MorphologicalRules.Add(rule); + LexEntry root = AddEntry( + "ROOT", + FeatureStruct.New(Language.SyntacticFeatureSystem).Symbol("V").Value, + Morphophonemic, + "pi" + ); + try + { + string[] generated = new Morpher(TraceManager, Language) + .GenerateWords(root, new Morpheme[] { rule }, FeatureStruct.New().Value) + .ToArray(); + Assert.That(generated, Is.Not.Empty); + Assert.That(ParseGlosses(CreateMorpher(pruneCopies), generated[0]), Does.Contain("ROOT RED")); + } + finally + { + Morphophonemic.Entries.Remove(root); + Morphophonemic.MorphologicalRules.Remove(rule); + } + } + + private AffixProcessAllomorph CreateCopyAndModifyAllomorph() + { + FeatureStruct voiced = FeatureStruct + .New(Language.PhonologicalFeatureSystem) + .Symbol(HCFeatureSystem.Segment) + .Symbol("vd+") + .Value; + return new AffixProcessAllomorph + { + Lhs = { Pattern.New("1").Annotation(AnySegment).OneOrMore.Value }, + Rhs = { new CopyFromInput("1"), new ModifyFromInput("1", voiced) }, + }; + } + private RewriteRule CreateBetweenVowelsRewrite(CharacterDefinitionTable table, string from, FeatureStruct rhs) { var rule = new RewriteRule From c8860f0cf1018bcf584b21ee3eaedfc3048f230d Mon Sep 17 00:00:00 2001 From: John Lambert Date: Wed, 21 Oct 2026 15:09:15 -0400 Subject: [PATCH 20/20] Build the list of copy-checked parts once in the transform constructor HasRepeatedParts and HasDisagreeingCopies each restated which parts are compared (at least two copies, not modified). Both now read one list, so the predicate's installation and its loop cannot disagree again. Co-Authored-By: Claude Opus 5.5 --- .../AnalysisMorphologicalTransform.cs | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisMorphologicalTransform.cs b/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisMorphologicalTransform.cs index f0d1fa738..ed77b3cb3 100644 --- a/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisMorphologicalTransform.cs +++ b/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisMorphologicalTransform.cs @@ -12,6 +12,7 @@ public class AnalysisMorphologicalTransform private readonly Pattern _pattern; private readonly Dictionary> _modifyFromInfos; private readonly Dictionary _capturedParts; + private readonly List> _copyCheckedParts; public AnalysisMorphologicalTransform( IEnumerable> lhs, @@ -34,8 +35,11 @@ IList rhs ); } } - // a modified part is never checked by HasDisagreeingCopies, so repeating it needs no predicate - HasRepeatedParts = _capturedParts.Any(part => part.Value >= 2 && !_modifyFromInfos.ContainsKey(part.Key)); + // a modified part is never checked for disagreeing copies, so repeating it needs no predicate + _copyCheckedParts = _capturedParts + .Where(part => part.Value >= 2 && !_modifyFromInfos.ContainsKey(part.Key)) + .ToList(); + HasRepeatedParts = _copyCheckedParts.Count > 0; } internal bool HasRepeatedParts { get; } @@ -60,11 +64,8 @@ protected IDictionary CapturedParts /// internal bool HasDisagreeingCopies(Match match) { - foreach (KeyValuePair capturedPart in _capturedParts) + foreach (KeyValuePair capturedPart in _copyCheckedParts) { - if (capturedPart.Value < 2 || _modifyFromInfos.ContainsKey(capturedPart.Key)) - continue; - if ( TryGetSegmentCopies(match, capturedPart.Key, capturedPart.Value, out List> copies) && AnyCopiesDisagree(copies)