diff --git a/docs/operations.md b/docs/operations.md index 07c9c0a..074105a 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -80,6 +80,11 @@ culture that read a column — the empty string for the invariant one, the same a caller whose files come from elsewhere should list its own (`["", "fr-FR"]`). An unknown name is refused when the analyzer is created. +A culture reads the thousands separator it groups with, and a few look-alikes, because the +separator a runtime reports depends on its ICU data and files are typed by hand: a culture that groups +with a no-break space (`fr-FR`, `pl-PL`) reads `1 234,56` with an ordinary space too, and one that +groups with an apostrophe (`de-CH`) reads `1'234.56` and `1’234.56` alike. + Under invariant globalization (`InvariantGlobalization` / `DOTNET_SYSTEM_GLOBALIZATION_INVARIANT`, common in slim container images) only the invariant culture exists. Analysis then leaves the named cultures out instead of failing — the profile's `ParseCounts` show which cultures were used — and a diff --git a/src/TriasDev.Tabular/Analysis/ColumnProfiler.cs b/src/TriasDev.Tabular/Analysis/ColumnProfiler.cs index 684faa5..a9f4485 100644 --- a/src/TriasDev.Tabular/Analysis/ColumnProfiler.cs +++ b/src/TriasDev.Tabular/Analysis/ColumnProfiler.cs @@ -510,6 +510,11 @@ private sealed class CultureAccumulator(string name, int outlierLimit) ? CultureInfo.InvariantCulture.NumberFormat : CultureInfo.GetCultureInfo(name).NumberFormat); + /// The apostrophe this culture does not group with, when it groups with one, looked up once rather than per value. + private char OtherApostrophe { get; } = NumberReading.OtherApostropheOf(name.Length == 0 + ? CultureInfo.InvariantCulture.NumberFormat + : CultureInfo.GetCultureInfo(name).NumberFormat); + /// This culture's date separator, when it is one character: what its own dates are written with. private char DateSeparator { get; } = DateSeparatorOf(name); @@ -631,15 +636,40 @@ public bool ReadsNumbersLike(CultureAccumulator other) /// public NumberRead ReadNumber(string text, bool couldBeNumeric) { - bool grouped = couldBeNumeric && NumberReading.HasWellFormedGroups(text, _culture.NumberFormat); + if (!couldBeNumeric) + { + return default; + } + + // NumberReading.TryParse, inlined: the apostrophe looked up once per culture, not per value. + return OtherApostrophe != '\0' && text.Contains(OtherApostrophe) + ? ReadNumberWithOwnApostrophe(text) + : ReadNumberAsWritten(text); + } + + // Apart from ReadNumber so the copy's stack space is taken only by the rare value that needs it. + private NumberRead ReadNumberWithOwnApostrophe(string text) + { + Span own = text.Length <= NumberReading.StackCopyLimit ? stackalloc char[text.Length] : new char[text.Length]; + NumberReading.ToOwnApostrophe(text, OtherApostrophe, own); + + return ReadNumberAsWritten(own); + } + + // The groups checked once for both parses: a decimal fails the integer parse first. + private NumberRead ReadNumberAsWritten(ReadOnlySpan text) + { + if (!NumberReading.HasWellFormedGroups(text, _culture.NumberFormat)) + { + return default; + } - if (grouped - && long.TryParse(text, NumberStyles.Integer | NumberStyles.AllowThousands, _culture, out long whole)) + if (long.TryParse(text, NumberReading.IntegerStyles, _culture, out long whole)) { return new NumberRead(NumberKind.Integer, whole); } - if (grouped && decimal.TryParse(text, NumberReading.DecimalStyles, _culture, out decimal fraction)) + if (decimal.TryParse(text, NumberReading.DecimalStyles, _culture, out decimal fraction)) { return new NumberRead(NumberKind.Decimal, fraction); } @@ -730,9 +760,9 @@ public static bool CouldBeNumeric(string text) continue; } - // The separators, sign and exponent the tried cultures use, and the two spaces that - // some of them group with. - if (c is '.' or ',' or '-' or '+' or 'e' or 'E' or ' ' or '\u00A0' or '\u202F') + // The separators, sign and exponent the tried cultures use, and the two spaces and + // two apostrophes that some of them group with (de-CH: 1'234.5, #126). + if (c is '.' or ',' or '-' or '+' or 'e' or 'E' or ' ' or '\u00A0' or '\u202F' or '\'' or '\u2019') { continue; } diff --git a/src/TriasDev.Tabular/Analysis/NumberReading.cs b/src/TriasDev.Tabular/Analysis/NumberReading.cs index f35f222..08477bd 100644 --- a/src/TriasDev.Tabular/Analysis/NumberReading.cs +++ b/src/TriasDev.Tabular/Analysis/NumberReading.cs @@ -1,4 +1,5 @@ using System.Globalization; +using System.Numerics; namespace TriasDev.Tabular; @@ -19,6 +20,67 @@ internal static class NumberReading /// public const NumberStyles DecimalStyles = NumberStyles.Number | NumberStyles.AllowExponent; + /// How a whole number written as text is parsed: grouped and signed, no exponent. + public const NumberStyles IntegerStyles = NumberStyles.Integer | NumberStyles.AllowThousands; + + /// How long a value may be for its copy with the culture's own apostrophe to sit on the stack. + public const int StackCopyLimit = 128; + + /// + /// Parses a number under a culture as the profile and the import both do: its groups held to three + /// digits, and either apostrophe taken for the culture's own. + /// + /// + /// Which apostrophe de-CH groups with depends on the runtime's ICU data — ' on one + /// system, ’ on another — and .NET matches the culture's separator only, so a file typed with + /// the ASCII apostrophe read on one machine and not on the next (#126). The other apostrophe is + /// replaced in a copy, made only for a culture that groups with one and a value that holds the other. + /// The profiler does the same inline, with the apostrophe looked up once per culture and the groups + /// checked once for both parses: it asks this of every numeric-looking value under every culture. + /// + public static bool TryParse(ReadOnlySpan text, NumberStyles styles, CultureInfo culture, out T value) + where T : INumberBase + { + char other = OtherApostropheOf(culture.NumberFormat); + + if (other != '\0' && text.Contains(other)) + { + Span own = text.Length <= StackCopyLimit ? stackalloc char[text.Length] : new char[text.Length]; + ToOwnApostrophe(text, other, own); + + return TryParseAsWritten(own, styles, culture, out value); + } + + return TryParseAsWritten(text, styles, culture, out value); + } + + private static bool TryParseAsWritten(ReadOnlySpan text, NumberStyles styles, CultureInfo culture, out T value) + where T : INumberBase + { + if (HasWellFormedGroups(text, culture.NumberFormat)) + { + return T.TryParse(text, styles, culture, out value!); + } + + value = T.Zero; + return false; + } + + /// The apostrophe a culture that groups with one does not use — ’ for ' and back — or \0. + public static char OtherApostropheOf(NumberFormatInfo format) => format.NumberGroupSeparator switch + { + "'" => '’', + "’" => '\'', + _ => '\0', + }; + + /// Copies a value into , as long as it, with the other apostrophe made the culture's own. + public static void ToOwnApostrophe(ReadOnlySpan text, char other, Span own) + { + text.CopyTo(own); + own.Replace(other, other == '\'' ? '’' : '\''); + } + /// /// Reads a value that writes the decimal separator the other way from the culture, where that is /// the only way it can be read: digits, one . or , — whichever the culture does not diff --git a/src/TriasDev.Tabular/Mapping/ValueReading.cs b/src/TriasDev.Tabular/Mapping/ValueReading.cs index 143b786..08ee5f0 100644 --- a/src/TriasDev.Tabular/Mapping/ValueReading.cs +++ b/src/TriasDev.Tabular/Mapping/ValueReading.cs @@ -113,9 +113,8 @@ private static bool TryReadInteger(in RawCell cell, string text, CultureInfo cul return true; } - // The profiler's grouping rule, applied here too: under de-DE "1.5" parses as 15 otherwise. - if (NumberReading.HasWellFormedGroups(text, culture.NumberFormat) - && long.TryParse(text, NumberStyles.Integer | NumberStyles.AllowThousands, culture, out long whole)) + // The profiler's reading, applied here too: under de-DE "1.5" parses as 15 otherwise. + if (NumberReading.TryParse(text, NumberReading.IntegerStyles, culture, out long whole)) { value = MappedValue.FromInteger(whole); return true; @@ -138,8 +137,7 @@ private static bool TryReadDecimal(in RawCell cell, string text, CultureInfo cul return fits; } - if (NumberReading.HasWellFormedGroups(text, culture.NumberFormat) - && decimal.TryParse(text, NumberReading.DecimalStyles, culture, out decimal fraction)) + if (NumberReading.TryParse(text, NumberReading.DecimalStyles, culture, out decimal fraction)) { value = MappedValue.FromDecimal(fraction); return true; diff --git a/tests/TriasDev.Tabular.Tests/Analysis/ApostropheGroupTests.cs b/tests/TriasDev.Tabular.Tests/Analysis/ApostropheGroupTests.cs new file mode 100644 index 0000000..355cd13 --- /dev/null +++ b/tests/TriasDev.Tabular.Tests/Analysis/ApostropheGroupTests.cs @@ -0,0 +1,119 @@ +using System.Text; + +using TriasDev.Tabular.Csv; + +using Xunit; + +namespace TriasDev.Tabular.Tests.Analysis; + +/// +/// A number grouped with an apostrophe — 1'239.09, as Swiss formats write it — is a number under a +/// culture that groups that way, in the profile and the import alike, whichever apostrophe it is typed +/// with (#126). +/// +/// +/// The profile's filter let no apostrophe through, so de-CH in the tried cultures changed nothing +/// for exactly the numbers it writes, while the import read them. And which apostrophe de-CH groups +/// with depends on the runtime's ICU data — ' on one system, ’ on another — so each value +/// here is tried with both: one of the two is always the other one. +/// +public sealed class ApostropheGroupTests +{ + private static CancellationToken Token => TestContext.Current.CancellationToken; + + private static readonly DecimalImportField Amount = ImportField.Decimal("amount"); + + private static readonly IntegerImportField Count = ImportField.Integer("count"); + + public static TheoryData Apostrophes => ['\'', '’']; + + private static CsvCursor Csv(IEnumerable values) => + new(new MemoryStream(Encoding.UTF8.GetBytes("v\n" + string.Concat(values.Select(v => v + "\n")))), "t.csv"); + + private static string[] Grouped(char apostrophe, params string[] values) => [.. values.Select(v => v.Replace('\'', apostrophe))]; + + private static ColumnProfile Profile(IEnumerable values, params string[] cultures) + { + using CsvCursor cursor = Csv(values); + return TabularAnalyzer.Analyze(cursor, new AnalysisOptions { Cultures = cultures }, cancellationToken: Token).Sheets[0].Columns[0]; + } + + private static ImportResult Import(IEnumerable values, ImportField field, TabularRowMapper map) + { + using CsvCursor cursor = Csv(values); + using ImportRun run = TabularImporter.Import( + cursor, + new MappingPlan { Culture = "de-CH", Bindings = [new ColumnBinding { ColumnIndex = 0, FieldName = field.Name, Header = "v" }] }, + new ImportSchema { Fields = [field] }, + map, + cancellationToken: Token); + + return run.ReadAll(cancellationToken: Token); + } + + [Theory] + [MemberData(nameof(Apostrophes))] + public void ProfilesApostropheGroupedDecimalsAsDecimalUnderASwissCulture(char apostrophe) + { + ColumnProfile column = Profile(Grouped(apostrophe, "1'239.09", "1'234'567.89", "-2'345.5"), "", "de-DE", "en-US", "de-CH"); + + TypeHypothesis best = column.Hypotheses[0]; + Assert.Equal(ColumnType.Decimal, best.Type); + Assert.Equal("de-CH", best.Culture); + Assert.Equal(1.0, best.Confidence); + Assert.Equal(-2345.5m, column.Facts.MinNumeric); + Assert.Equal(1234567.89m, column.Facts.MaxNumeric); + } + + [Theory] + [MemberData(nameof(Apostrophes))] + public void ProfilesApostropheGroupedWholeNumbersAsIntegers(char apostrophe) + { + ColumnProfile column = Profile(Grouped(apostrophe, "1'234", "12'345'678"), "de-CH"); + + Assert.Equal(ColumnType.Integer, column.Hypotheses[0].Type); + Assert.Equal(12345678m, column.Facts.MaxNumeric); + } + + [Theory] + [MemberData(nameof(Apostrophes))] + public void ImportsApostropheGroupedDecimalsAsTheProfileProposed(char apostrophe) + { + ImportResult result = Import(Grouped(apostrophe, "1'239.09", "1'234'567.89", "-2'345.5"), Amount, row => row[Amount]); + + Assert.Empty(result.Errors); + Assert.Equal([1239.09m, 1234567.89m, -2345.5m], result.Items); + } + + [Theory] + [MemberData(nameof(Apostrophes))] + public void ImportsApostropheGroupedIntegers(char apostrophe) + { + ImportResult result = Import(Grouped(apostrophe, "1'234", "12'345'678"), Count, row => row[Count]); + + Assert.Empty(result.Errors); + Assert.Equal([1234L, 12345678L], result.Items); + } + + [Theory] + [MemberData(nameof(Apostrophes))] + public void HoldsAnApostropheGroupToThreeDigits(char apostrophe) + { + // The grouping rule every separator answers to: 12'34 is no number, under either apostrophe. + ColumnProfile column = Profile(Grouped(apostrophe, "1'234.5", "12'34.5"), "de-CH"); + + CultureParseCounts swiss = Assert.Single(column.Facts.ParseCounts); + Assert.Equal(1, swiss.Decimal); + Assert.Equal(Grouped(apostrophe, "12'34.5")[0], Assert.Single(swiss.NumericOutliers).RawValue); + } + + [Theory] + [MemberData(nameof(Apostrophes))] + public void LeavesApostropheGroupsTextUnderCulturesThatDoNotGroupSo(char apostrophe) + { + ColumnProfile column = Profile(Grouped(apostrophe, "1'239.09", "1'234"), "", "de-DE", "en-US"); + + Assert.Equal(ColumnType.Text, column.Hypotheses[0].Type); + Assert.All(column.Facts.ParseCounts, counts => Assert.Equal(0, counts.Integer + counts.Decimal)); + } +} diff --git a/tests/TriasDev.Tabular.Tests/GuidePromisesTests.cs b/tests/TriasDev.Tabular.Tests/GuidePromisesTests.cs index 351ad31..00b4d53 100644 --- a/tests/TriasDev.Tabular.Tests/GuidePromisesTests.cs +++ b/tests/TriasDev.Tabular.Tests/GuidePromisesTests.cs @@ -81,4 +81,15 @@ public void LeavesUniquenessUndeterminedWhenTheDistinctBudgetRanOut() PrecheckFinding finding = Assert.Single(result.Findings); Assert.Equal((ErrorCodes.Value.NotUnique, PrecheckSeverity.Undetermined), (finding.Code, finding.Severity)); } + + [Theory] + [InlineData("fr-FR")] + [InlineData("pl-PL")] + public void ReadsAnOrdinarySpaceAsTheGroupOfACultureThatGroupsWithANoBreakSpace(string culture) + { + ColumnProfile column = Profile("v\n\"1 234,56\"\n\"-12 345 678,9\"\n", new AnalysisOptions { Cultures = [culture] }).Sheets[0].Columns[0]; + + Assert.Equal(ColumnType.Decimal, column.Hypotheses[0].Type); + Assert.Equal((-12345678.9m, 1234.56m), (column.Facts.MinNumeric, column.Facts.MaxNumeric)); + } }