fix: read a number grouped with an apostrophe, such as 1'239.09, under de-CH

docs/operations.md

@@ -80,6 +80,11 @@ culture that read a column — the empty string for the invariant one, the same
 a caller whose files come from elsewhere should list its own (`["", "fr-FR"]`). An unknown name is
 refused when the analyzer is created.
 
+A culture reads the thousands separator it groups with, and a few look-alikes, because the
+separator a runtime reports depends on its ICU data and files are typed by hand: a culture that groups
+with a no-break space (`fr-FR`, `pl-PL`) reads `1 234,56` with an ordinary space too, and one that
+groups with an apostrophe (`de-CH`) reads `1'234.56` and `1’234.56` alike.
+
 Under invariant globalization (`InvariantGlobalization` / `DOTNET_SYSTEM_GLOBALIZATION_INVARIANT`,
 common in slim container images) only the invariant culture exists. Analysis then leaves the named
 cultures out instead of failing — the profile's `ParseCounts` show which cultures were used — and a

src/TriasDev.Tabular/Analysis/ColumnProfiler.cs

@@ -510,6 +510,11 @@ private sealed class CultureAccumulator(string name, int outlierLimit)
             ? CultureInfo.InvariantCulture.NumberFormat
             : CultureInfo.GetCultureInfo(name).NumberFormat);
 
+        /// <summary>The apostrophe this culture does not group with, when it groups with one, looked up once rather than per value.</summary>
+        private char OtherApostrophe { get; } = NumberReading.OtherApostropheOf(name.Length == 0
+            ? CultureInfo.InvariantCulture.NumberFormat
+            : CultureInfo.GetCultureInfo(name).NumberFormat);
+
         /// <summary>This culture's date separator, when it is one character: what its own dates are written with.</summary>
         private char DateSeparator { get; } = DateSeparatorOf(name);
 
@@ -631,15 +636,40 @@ public bool ReadsNumbersLike(CultureAccumulator other)
         /// </param>
         public NumberRead ReadNumber(string text, bool couldBeNumeric)
         {
-            bool grouped = couldBeNumeric && NumberReading.HasWellFormedGroups(text, _culture.NumberFormat);
+            if (!couldBeNumeric)
+            {
+                return default;
+            }
+
+            // NumberReading.TryParse, inlined: the apostrophe looked up once per culture, not per value.
+            return OtherApostrophe != '\0' && text.Contains(OtherApostrophe)
+                ? ReadNumberWithOwnApostrophe(text)
+                : ReadNumberAsWritten(text);
+        }
+
+        // Apart from ReadNumber so the copy's stack space is taken only by the rare value that needs it.
+        private NumberRead ReadNumberWithOwnApostrophe(string text)
+        {
+            Span<char> own = text.Length <= NumberReading.StackCopyLimit ? stackalloc char[text.Length] : new char[text.Length];
+            NumberReading.ToOwnApostrophe(text, OtherApostrophe, own);
+
+            return ReadNumberAsWritten(own);
+        }
+
+        // The groups checked once for both parses: a decimal fails the integer parse first.
+        private NumberRead ReadNumberAsWritten(ReadOnlySpan<char> text)
+        {
+            if (!NumberReading.HasWellFormedGroups(text, _culture.NumberFormat))
+            {
+                return default;
+            }
 
-            if (grouped
-                && long.TryParse(text, NumberStyles.Integer | NumberStyles.AllowThousands, _culture, out long whole))
+            if (long.TryParse(text, NumberReading.IntegerStyles, _culture, out long whole))
             {
                 return new NumberRead(NumberKind.Integer, whole);
             }
 
-            if (grouped && decimal.TryParse(text, NumberReading.DecimalStyles, _culture, out decimal fraction))
+            if (decimal.TryParse(text, NumberReading.DecimalStyles, _culture, out decimal fraction))
             {
                 return new NumberRead(NumberKind.Decimal, fraction);
             }
@@ -730,9 +760,9 @@ public static bool CouldBeNumeric(string text)
                     continue;
                 }
 
-                // The separators, sign and exponent the tried cultures use, and the two spaces that
-                // some of them group with.
-                if (c is '.' or ',' or '-' or '+' or 'e' or 'E' or ' ' or '\u00A0' or '\u202F')
+                // The separators, sign and exponent the tried cultures use, and the two spaces and
+                // two apostrophes that some of them group with (de-CH: 1'234.5, #126).
+                if (c is '.' or ',' or '-' or '+' or 'e' or 'E' or ' ' or '\u00A0' or '\u202F' or '\'' or '\u2019')
                 {
                     continue;
                 }

src/TriasDev.Tabular/Analysis/NumberReading.cs

@@ -1,4 +1,5 @@
 using System.Globalization;
+using System.Numerics;
 
 namespace TriasDev.Tabular;
 
@@ -19,6 +20,67 @@ internal static class NumberReading
     /// </remarks>
     public const NumberStyles DecimalStyles = NumberStyles.Number | NumberStyles.AllowExponent;
 
+    /// <summary>How a whole number written as text is parsed: grouped and signed, no exponent.</summary>
+    public const NumberStyles IntegerStyles = NumberStyles.Integer | NumberStyles.AllowThousands;
+
+    /// <summary>How long a value may be for its copy with the culture's own apostrophe to sit on the stack.</summary>
+    public const int StackCopyLimit = 128;
+
+    /// <summary>
+    /// Parses a number under a culture as the profile and the import both do: its groups held to three
+    /// digits, and either apostrophe taken for the culture's own.
+    /// </summary>
+    /// <remarks>
+    /// Which apostrophe <c>de-CH</c> groups with depends on the runtime's ICU data — <c>'</c> on one
+    /// system, <c>’</c> on another — and .NET matches the culture's separator only, so a file typed with
+    /// the ASCII apostrophe read on one machine and not on the next (#126). The other apostrophe is
+    /// replaced in a copy, made only for a culture that groups with one and a value that holds the other.
+    /// The profiler does the same inline, with the apostrophe looked up once per culture and the groups
+    /// checked once for both parses: it asks this of every numeric-looking value under every culture.
+    /// </remarks>
+    public static bool TryParse<T>(ReadOnlySpan<char> text, NumberStyles styles, CultureInfo culture, out T value)
+        where T : INumberBase<T>
+    {
+        char other = OtherApostropheOf(culture.NumberFormat);
+
+        if (other != '\0' && text.Contains(other))
+        {
+            Span<char> own = text.Length <= StackCopyLimit ? stackalloc char[text.Length] : new char[text.Length];
+            ToOwnApostrophe(text, other, own);
+
+            return TryParseAsWritten(own, styles, culture, out value);
+        }
+
+        return TryParseAsWritten(text, styles, culture, out value);
+    }
+
+    private static bool TryParseAsWritten<T>(ReadOnlySpan<char> text, NumberStyles styles, CultureInfo culture, out T value)
+        where T : INumberBase<T>
+    {
+        if (HasWellFormedGroups(text, culture.NumberFormat))
+        {
+            return T.TryParse(text, styles, culture, out value!);
+        }
+
+        value = T.Zero;
+        return false;
+    }
+
+    /// <summary>The apostrophe a culture that groups with one does not use — <c>’</c> for <c>'</c> and back — or <c>\0</c>.</summary>
+    public static char OtherApostropheOf(NumberFormatInfo format) => format.NumberGroupSeparator switch
+    {
+        "'" => '’',
+        "’" => '\'',
+        _ => '\0',
+    };
+
+    /// <summary>Copies a value into <paramref name="own"/>, as long as it, with the other apostrophe made the culture's own.</summary>
+    public static void ToOwnApostrophe(ReadOnlySpan<char> text, char other, Span<char> own)
+    {
+        text.CopyTo(own);
+        own.Replace(other, other == '\'' ? '’' : '\'');
+    }
+
     /// <summary>
     /// Reads a value that writes the decimal separator the other way from the culture, where that is
     /// the only way it can be read: digits, one <c>.</c> or <c>,</c> — whichever the culture does not

src/TriasDev.Tabular/Mapping/ValueReading.cs

@@ -113,9 +113,8 @@ private static bool TryReadInteger(in RawCell cell, string text, CultureInfo cul
             return true;
         }
 
-        // The profiler's grouping rule, applied here too: under de-DE "1.5" parses as 15 otherwise.
-        if (NumberReading.HasWellFormedGroups(text, culture.NumberFormat)
-            && long.TryParse(text, NumberStyles.Integer | NumberStyles.AllowThousands, culture, out long whole))
+        // The profiler's reading, applied here too: under de-DE "1.5" parses as 15 otherwise.
+        if (NumberReading.TryParse(text, NumberReading.IntegerStyles, culture, out long whole))
         {
             value = MappedValue.FromInteger(whole);
             return true;
@@ -138,8 +137,7 @@ private static bool TryReadDecimal(in RawCell cell, string text, CultureInfo cul
             return fits;
         }
 
-        if (NumberReading.HasWellFormedGroups(text, culture.NumberFormat)
-            && decimal.TryParse(text, NumberReading.DecimalStyles, culture, out decimal fraction))
+        if (NumberReading.TryParse(text, NumberReading.DecimalStyles, culture, out decimal fraction))
         {
             value = MappedValue.FromDecimal(fraction);
             return true;

tests/TriasDev.Tabular.Tests/Analysis/ApostropheGroupTests.cs

@@ -0,0 +1,119 @@
+using System.Text;
+
+using TriasDev.Tabular.Csv;
+
+using Xunit;
+
+namespace TriasDev.Tabular.Tests.Analysis;
+
+/// <summary>
+/// A number grouped with an apostrophe — <c>1'239.09</c>, as Swiss formats write it — is a number under a
+/// culture that groups that way, in the profile and the import alike, whichever apostrophe it is typed
+/// with (#126).
+/// </summary>
+/// <remarks>
+/// The profile's filter let no apostrophe through, so <c>de-CH</c> in the tried cultures changed nothing
+/// for exactly the numbers it writes, while the import read them. And which apostrophe <c>de-CH</c> groups
+/// with depends on the runtime's ICU data — <c>'</c> on one system, <c>’</c> on another — so each value
+/// here is tried with both: one of the two is always the other one.
+/// </remarks>
+public sealed class ApostropheGroupTests
+{
+    private static CancellationToken Token => TestContext.Current.CancellationToken;
+
+    private static readonly DecimalImportField Amount = ImportField.Decimal("amount");
+
+    private static readonly IntegerImportField Count = ImportField.Integer("count");
+
+    public static TheoryData<char> Apostrophes => ['\'', '’'];
+
+    private static CsvCursor Csv(IEnumerable<string> values) =>
+        new(new MemoryStream(Encoding.UTF8.GetBytes("v\n" + string.Concat(values.Select(v => v + "\n")))), "t.csv");
+
+    private static string[] Grouped(char apostrophe, params string[] values) => [.. values.Select(v => v.Replace('\'', apostrophe))];
+
+    private static ColumnProfile Profile(IEnumerable<string> values, params string[] cultures)
+    {
+        using CsvCursor cursor = Csv(values);
+        return TabularAnalyzer.Analyze(cursor, new AnalysisOptions { Cultures = cultures }, cancellationToken: Token).Sheets[0].Columns[0];
+    }
+
+    private static ImportResult<T> Import<T>(IEnumerable<string> values, ImportField field, TabularRowMapper<T> map)
+    {
+        using CsvCursor cursor = Csv(values);
+        using ImportRun<T> run = TabularImporter.Import(
+            cursor,
+            new MappingPlan { Culture = "de-CH", Bindings = [new ColumnBinding { ColumnIndex = 0, FieldName = field.Name, Header = "v" }] },
+            new ImportSchema { Fields = [field] },
+            map,
+            cancellationToken: Token);
+
+        return run.ReadAll(cancellationToken: Token);
+    }
+
+    [Theory]
+    [MemberData(nameof(Apostrophes))]
+    public void ProfilesApostropheGroupedDecimalsAsDecimalUnderASwissCulture(char apostrophe)
+    {
+        ColumnProfile column = Profile(Grouped(apostrophe, "1'239.09", "1'234'567.89", "-2'345.5"), "", "de-DE", "en-US", "de-CH");
+
+        TypeHypothesis best = column.Hypotheses[0];
+        Assert.Equal(ColumnType.Decimal, best.Type);
+        Assert.Equal("de-CH", best.Culture);
+        Assert.Equal(1.0, best.Confidence);
+        Assert.Equal(-2345.5m, column.Facts.MinNumeric);
+        Assert.Equal(1234567.89m, column.Facts.MaxNumeric);
+    }
+
+    [Theory]
+    [MemberData(nameof(Apostrophes))]
+    public void ProfilesApostropheGroupedWholeNumbersAsIntegers(char apostrophe)
+    {
+        ColumnProfile column = Profile(Grouped(apostrophe, "1'234", "12'345'678"), "de-CH");
+
+        Assert.Equal(ColumnType.Integer, column.Hypotheses[0].Type);
+        Assert.Equal(12345678m, column.Facts.MaxNumeric);
+    }
+
+    [Theory]
+    [MemberData(nameof(Apostrophes))]
+    public void ImportsApostropheGroupedDecimalsAsTheProfileProposed(char apostrophe)
+    {
+        ImportResult<decimal?> result = Import(Grouped(apostrophe, "1'239.09", "1'234'567.89", "-2'345.5"), Amount, row => row[Amount]);
+
+        Assert.Empty(result.Errors);
+        Assert.Equal([1239.09m, 1234567.89m, -2345.5m], result.Items);
+    }
+
+    [Theory]
+    [MemberData(nameof(Apostrophes))]
+    public void ImportsApostropheGroupedIntegers(char apostrophe)
+    {
+        ImportResult<long?> result = Import(Grouped(apostrophe, "1'234", "12'345'678"), Count, row => row[Count]);
+
+        Assert.Empty(result.Errors);
+        Assert.Equal([1234L, 12345678L], result.Items);
+    }
+
+    [Theory]
+    [MemberData(nameof(Apostrophes))]
+    public void HoldsAnApostropheGroupToThreeDigits(char apostrophe)
+    {
+        // The grouping rule every separator answers to: 12'34 is no number, under either apostrophe.
+        ColumnProfile column = Profile(Grouped(apostrophe, "1'234.5", "12'34.5"), "de-CH");
+
+        CultureParseCounts swiss = Assert.Single(column.Facts.ParseCounts);
+        Assert.Equal(1, swiss.Decimal);
+        Assert.Equal(Grouped(apostrophe, "12'34.5")[0], Assert.Single(swiss.NumericOutliers).RawValue);
+    }
+
+    [Theory]
+    [MemberData(nameof(Apostrophes))]
+    public void LeavesApostropheGroupsTextUnderCulturesThatDoNotGroupSo(char apostrophe)
+    {
+        ColumnProfile column = Profile(Grouped(apostrophe, "1'239.09", "1'234"), "", "de-DE", "en-US");
+
+        Assert.Equal(ColumnType.Text, column.Hypotheses[0].Type);
+        Assert.All(column.Facts.ParseCounts, counts => Assert.Equal(0, counts.Integer + counts.Decimal));
+    }
+}

tests/TriasDev.Tabular.Tests/GuidePromisesTests.cs

@@ -81,4 +81,15 @@ public void LeavesUniquenessUndeterminedWhenTheDistinctBudgetRanOut()
         PrecheckFinding finding = Assert.Single(result.Findings);
         Assert.Equal((ErrorCodes.Value.NotUnique, PrecheckSeverity.Undetermined), (finding.Code, finding.Severity));
     }
+
+    [Theory]
+    [InlineData("fr-FR")]
+    [InlineData("pl-PL")]
+    public void ReadsAnOrdinarySpaceAsTheGroupOfACultureThatGroupsWithANoBreakSpace(string culture)
+    {
+        ColumnProfile column = Profile("v\n\"1 234,56\"\n\"-12 345 678,9\"\n", new AnalysisOptions { Cultures = [culture] }).Sheets[0].Columns[0];
+
+        Assert.Equal(ColumnType.Decimal, column.Hypotheses[0].Type);
+        Assert.Equal((-12345678.9m, 1234.56m), (column.Facts.MinNumeric, column.Facts.MaxNumeric));
+    }
 }