Prune reduplication matches whose copies cannot agree

src/SIL.Machine.Morphology.HermitCrab/Morpher.cs

@@ -61,6 +61,7 @@ public Morpher(ITraceManager traceManager, Language lang, int maxDegreeOfParalle
             MaxStemCount = 2;
             MaxAlternatives = 0;
             MergeEquivalentAnalyses = true;
+            PruneDisagreeingCopies = true;
             LexEntrySelector = entry => true;
             RuleSelector = rule => true;
 
@@ -110,6 +111,13 @@ public ITraceManager TraceManager
         /// </summary>
         public bool AlwaysEnforceFinalTemplates { get; set; }
 
+        /// <summary>
+        /// When unapplying an affix-process or compounding rule that copies a part more than once
+        /// (reduplication), skip matches whose copies cannot unify segment by segment. Such a match can never
+        /// survive synthesis, so this removes only doomed analyses. On by default, including while tracing.
+        /// </summary>
+        public bool PruneDisagreeingCopies { get; set; }
+
         /// <summary>
         /// Caps the concurrency used within a single parse or generation -- analysis cascade,
         /// affix-template unapplication and synthesis alike. A value of 1 runs the work fully

src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisAffixProcessAllomorphRuleSpec.cs

@@ -12,6 +12,7 @@ public AnalysisAffixProcessAllomorphRuleSpec(AffixProcessAllomorph allomorph)
             : base(allomorph.Lhs, allomorph.Rhs)
         {
             _allomorph = allomorph;
+            Pattern.Acceptable = CopyAgreementCheck;
             Pattern.Freeze();
         }
 

src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisAffixProcessRule.cs

@@ -30,7 +30,7 @@ public AnalysisAffixProcessRule(Morpher morpher, AffixProcessRule rule)
             {
                 _rules.Add(
                     new MultiplePatternRule<Word, ShapeNode>(
-                        new AnalysisAffixProcessAllomorphRuleSpec(allo),
+                        new AnalysisAffixProcessAllomorphRuleSpec(allo) { PruningMorpher = morpher },
                         new MatcherSettings<ShapeNode>
                         {
                             Filter = ann => ann.Type() == HCFeatureSystem.Segment,

src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisCompoundingRule.cs

@@ -29,7 +29,7 @@ public AnalysisCompoundingRule(Morpher morpher, CompoundingRule rule)
             {
                 _rules.Add(
                     new MultiplePatternRule<Word, ShapeNode>(
-                        new AnalysisCompoundingSubruleRuleSpec(sr),
+                        new AnalysisCompoundingSubruleRuleSpec(sr) { PruningMorpher = morpher },
                         new MatcherSettings<ShapeNode>
                         {
                             Filter = ann => ann.Type() == HCFeatureSystem.Segment,

src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisCompoundingSubruleRuleSpec.cs

@@ -1,3 +1,4 @@
+using System;
 using System.Linq;
 using SIL.Machine.Annotations;
 using SIL.Machine.Matching;
@@ -13,7 +14,10 @@ public AnalysisCompoundingSubruleRuleSpec(CompoundingSubrule subrule)
             : base(subrule.HeadLhs.Concat(subrule.NonHeadLhs), subrule.Rhs)
         {
             _subrule = subrule;
-            Pattern.Acceptable = match => _subrule.HeadLhs.Any(part => IsPartCaptured(match, part.Name));
+            Func<Match<Word, ShapeNode>, bool> copyAgreement = CopyAgreementCheck;
+            Pattern.Acceptable = match =>
+                _subrule.HeadLhs.Any(part => IsPartCaptured(match, part.Name))
+                && (copyAgreement == null || copyAgreement(match));
             Pattern.Freeze();
         }
 

src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisMorphologicalTransform.cs

@@ -12,6 +12,7 @@ public class AnalysisMorphologicalTransform
         private readonly Pattern<Word, ShapeNode> _pattern;
         private readonly Dictionary<string, Tuple<int, FeatureStruct>> _modifyFromInfos;
         private readonly Dictionary<string, int> _capturedParts;
+        private readonly List<KeyValuePair<string, int>> _copyCheckedParts;
 
         public AnalysisMorphologicalTransform(
             IEnumerable<Pattern<Word, ShapeNode>> lhs,
@@ -34,8 +35,15 @@ IList<MorphologicalOutputAction> rhs
                     );
                 }
             }
+            // a modified part is never checked for disagreeing copies, so repeating it needs no predicate
+            _copyCheckedParts = _capturedParts
+                .Where(part => part.Value >= 2 && !_modifyFromInfos.ContainsKey(part.Key))
+                .ToList();
+            HasRepeatedParts = _copyCheckedParts.Count > 0;
         }
 
+        internal bool HasRepeatedParts { get; }
+
         internal static string GetGroupName(string partName, int index)
         {
             return string.Format("{0}_{1}", partName, index);
@@ -46,6 +54,118 @@ protected IDictionary<string, int> CapturedParts
             get { return _capturedParts; }
         }
 
+        /// <summary>
+        /// Synthesis writes every copy of a part from the same input, and anything that later changes one
+        /// copy is unapplied before this rule, so copies proven to disagree segment by segment cannot lead
+        /// to a valid analysis. An optional segment (an unapplied deletion) stands for any number of segments
+        /// it unifies with, because <see cref="Morpher.DeletionReapplications"/> can restore fewer segments
+        /// than were deleted, so copies disagree only if no such choice lines them up. A part the rule
+        /// modifies, or a copy with an optional node that is not a segment, is never reported as disagreeing.
+        /// </summary>
+        internal bool HasDisagreeingCopies(Match<Word, ShapeNode> match)
+        {
+            foreach (KeyValuePair<string, int> capturedPart in _copyCheckedParts)
+            {
+                if (
+                    TryGetSegmentCopies(match, capturedPart.Key, capturedPart.Value, out List<List<ShapeNode>> copies)
+                    && AnyCopiesDisagree(copies)
+                )
+                {
+                    return true;
+                }
+            }
+
+            return false;
+        }
+
+        private static bool TryGetSegmentCopies(
+            Match<Word, ShapeNode> match,
+            string partName,
+            int copyCount,
+            out List<List<ShapeNode>> copies
+        )
+        {
+            copies = new List<List<ShapeNode>>(copyCount);
+            for (int i = 0; i < copyCount; i++)
+            {
+                GroupCapture<ShapeNode> capture = match.GroupCaptures[GetGroupName(partName, i)];
+                if (!capture.Success)
+                    return false;
+
+                // an unapplied word-initial deletion sits before the capture, not inside it
+                List<ShapeNode> nodes = MorphologicalOutputAction
+                    .CollectSkippedOptionalNodes(match.Input.Shape, capture.Range)
+                    .Concat(match.Input.Shape.GetNodes(capture.Range))
+                    .ToList();
+                if (nodes.Any(node => node.Annotation.Optional && node.Annotation.Type() != HCFeatureSystem.Segment))
+                    return false;
+
+                copies.Add(nodes.Where(node => node.Annotation.Type() == HCFeatureSystem.Segment).ToList());
+            }
+            return true;
+        }
+
+        // unifiability is not transitive, so every pair is compared, not just each copy against the first
+        private static bool AnyCopiesDisagree(List<List<ShapeNode>> copies)
+        {
+            for (int i = 0; i < copies.Count; i++)
+            {
+                for (int j = i + 1; j < copies.Count; j++)
+                {
+                    if (!CanAlign(copies[i], copies[j]))
+                        return true;
+                }
+            }
+            return false;
+        }
+
+        private static bool CanAlign(List<ShapeNode> first, List<ShapeNode> second)
+        {
+            if (!first.Any(node => node.Annotation.Optional) && !second.Any(node => node.Annotation.Optional))
+            {
+                if (first.Count != second.Count)
+                    return false;
+                for (int k = 0; k < first.Count; k++)
+                {
+                    if (!first[k].Annotation.FeatureStruct.IsUnifiable(second[k].Annotation.FeatureStruct))
+                        return false;
+                }
+                return true;
+            }
+
+            // reachable[i, j]: the first i nodes of one copy can be lined up with the first j of the other;
+            // an optional node is passed over, or absorbs a node of the other copy and stays available
+            var reachable = new bool[first.Count + 1, second.Count + 1];
+            reachable[0, 0] = true;
+            for (int i = 0; i <= first.Count; i++)
+            {
+                for (int j = 0; j <= second.Count; j++)
+                {
+                    if (!reachable[i, j])
+                        continue;
+                    bool firstOptional = i < first.Count && first[i].Annotation.Optional;
+                    bool secondOptional = j < second.Count && second[j].Annotation.Optional;
+                    if (firstOptional)
+                        reachable[i + 1, j] = true;
+                    if (secondOptional)
+                        reachable[i, j + 1] = true;
+                    if (
+                        i < first.Count
+                        && j < second.Count
+                        && first[i].Annotation.FeatureStruct.IsUnifiable(second[j].Annotation.FeatureStruct)
+                    )
+                    {
+                        reachable[i + 1, j + 1] = true;
+                        if (firstOptional)
+                            reachable[i, j + 1] = true;
+                        if (secondOptional)
+                            reachable[i + 1, j] = true;
+                    }
+                }
+            }
+            return reachable[first.Count, second.Count];
+        }
+
         public Pattern<Word, ShapeNode> Pattern
         {
             get { return _pattern; }

src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisMorphologicalTransformRuleSpec.cs

@@ -1,4 +1,5 @@
-using System.Collections.Generic;
+using System;
+using System.Collections.Generic;
 using SIL.Machine.Annotations;
 using SIL.Machine.Matching;
 using SIL.Machine.Rules;
@@ -15,6 +16,25 @@ IList<MorphologicalOutputAction> rhs
         )
             : base(lhs, rhs) { }
 
+        /// <summary>
+        /// The morpher whose <see cref="Morpher.PruneDisagreeingCopies"/> setting governs pruning; unset means none.
+        /// </summary>
+        internal Morpher PruningMorpher { get; set; }
+
+        /// <summary>
+        /// The acceptance predicate that prunes matches whose copies cannot agree, or null when no part repeats and
+        /// there is nothing to check (the matcher builds a Match for every candidate it tests).
+        /// </summary>
+        internal Func<Match<Word, ShapeNode>, bool> CopyAgreementCheck
+        {
+            get { return HasRepeatedParts ? CopiesMayAgree : (Func<Match<Word, ShapeNode>, bool>)null; }
+        }
+
+        private bool CopiesMayAgree(Match<Word, ShapeNode> match)
+        {
+            return PruningMorpher == null || !PruningMorpher.PruneDisagreeingCopies || !HasDisagreeingCopies(match);
+        }
+
         public bool IsApplicable(Word input)
         {
             return true;

src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisRealizationalAffixProcessRule.cs

@@ -23,7 +23,7 @@ public AnalysisRealizationalAffixProcessRule(Morpher morpher, RealizationalAffix
             {
                 _rules.Add(
                     new MultiplePatternRule<Word, ShapeNode>(
-                        new AnalysisAffixProcessAllomorphRuleSpec(allo),
+                        new AnalysisAffixProcessAllomorphRuleSpec(allo) { PruningMorpher = morpher },
                         new MatcherSettings<ShapeNode>
                         {
                             Filter = ann => ann.Type() == HCFeatureSystem.Segment,

src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/MorphologicalOutputAction.cs

@@ -38,6 +38,11 @@ IDictionary<string, int> capturedParts
         public abstract IEnumerable<Tuple<ShapeNode, ShapeNode>> Apply(Match<Word, ShapeNode> match, Word output);
 
         protected IEnumerable<ShapeNode> GetSkippedOptionalNodes(Shape shape, Range<ShapeNode> range)
+        {
+            return CollectSkippedOptionalNodes(shape, range);
+        }
+
+        internal static IEnumerable<ShapeNode> CollectSkippedOptionalNodes(Shape shape, Range<ShapeNode> range)
         {
             ShapeNode node = range.Start.Prev;
             var skippedNodes = new List<ShapeNode>();

src/SIL.Machine.Morphology.HermitCrab/PhonologicalRules/FeatureAnalysisRewriteRuleSpec.cs

@@ -55,6 +55,13 @@ Tuple<PatternNode<Word, ShapeNode>, PatternNode<Word, ShapeNode>> tuple in lhs.C
 
                     FeatureStruct fs = rhsConstraint.FeatureStruct.AntiFeatureStruct();
                     fs.Subtract(lhsConstraint.FeatureStruct.AntiFeatureStruct());
+                    StringFeatureValue lhsStrRep;
+                    if (lhsConstraint.FeatureStruct.TryGetValue(HCFeatureSystem.StrRep, out lhsStrRep))
+                    {
+                        // StrRep is a built-in feature: inversion adds the input spelling, not its complement.
+                        fs.AddValue(HCFeatureSystem.StrRep, lhsStrRep.Clone());
+                        rhsAntiFSs[i].AddValue(HCFeatureSystem.StrRep, lhsStrRep.Clone());
+                    }
                     fs.AddValue(HCFeatureSystem.Type, HCFeatureSystem.Segment);
                     _analysisRhs.Children.Add(new Constraint<Word, ShapeNode>(fs));
 
@@ -95,6 +102,17 @@ private bool IsUnapplicationNonvacuous(Match<Word, ShapeNode> match, IEnumerable
                         }
                     }
                 }
+                foreach (StringFeature sf in fs.Features.OfType<StringFeature>())
+                {
+                    StringFeatureValue nodeSfv;
+                    if (
+                        node.Annotation.FeatureStruct.TryGetValue(sf, out nodeSfv)
+                        && !nodeSfv.IsSupersetOf(fs.GetValue(sf))
+                    )
+                    {
+                        return true;
+                    }
+                }
                 i++;
             }
 

src/SIL.Machine/Corpora/ScrVersExtensions.cs

@@ -67,8 +67,7 @@ public static bool IsEquivalentTo(this ScrVers scrVers, ScrVers other)
             if (scrVers.Equals(other))
                 return true;
 
-            // If all verses in the versifications are 1) equal (accounts for mapping)
-            // and 2) graphically identical in regard to book, chapter, and verse, then the versifications are equivalent
+            // Equivalence requires both mapped equality and identical book, chapter, and verse numbers.
             foreach (
                 (VerseRef thisVerse, VerseRef otherVerse) in scrVers
                     .AllIncludedVerses()

src/SIL.Machine/FeatureModel/StringFeatureValue.cs

@@ -80,6 +80,11 @@ public bool Overlaps(IEnumerable<string> strings)
             return Not ? !_values.Overlaps(strings) : _values.Overlaps(strings);
         }
 
+        public bool IsSupersetOf(StringFeatureValue other, bool notOther = false)
+        {
+            return IsSupersetOf(false, other, notOther);
+        }
+
         protected override bool IsSupersetOf(bool not, SimpleFeatureValue other, bool notOther)
         {
             if (!(other is StringFeatureValue otherSfv))