[2/2] Add a script to check whether week, first_weekday, first_workday in glibc agrees with CLDR.

Message ID 20260923093817.2901603-2-mfabian@redhat.com (mailing list archive)
State New
Headers
Series [1/2] localedata: Fix `week`, `first_weekday`, `first_workday` in all locales to agree with CLDR |

Checks

Context Check Description
redhat-pt-bot/TryBot-apply_patch success Patch applied to master at the time it was sent
linaro-tcwg-bot/tcwg_glibc_build--master-aarch64 success Build passed
linaro-tcwg-bot/tcwg_glibc_check--master-aarch64 success Test passed
linaro-tcwg-bot/tcwg_glibc_build--master-arm success Build passed
redhat-pt-bot/TryBot-32bit success Build for i686
linaro-tcwg-bot/tcwg_glibc_check--master-arm success Test passed

Commit Message

Mike FABIAN Sept. 23, 2026, 9:38 a.m. UTC
  See: https://github.com/unicode-org/cldr/blob/main/common/supplemental/supplementalData.xml

Run of check script:

$ ./cldr_week_data_check.py ../../../cldr/common/supplemental/supplementalData.xml '../locales/*_[A-Z][A-Z]'
$

If the the data in CLDR and glibc agrees, this script prints nothing.

If there are differences it prints warnings like for example:

$ ./cldr_week_data_check.py /local/mfabian/src/cldr/common/supplemental/supplementalData.xml '../locales/*_[A-Z][A-Z]'
WARNING ru_UA: LC_TIME copied from ru_RU but expected CLDR minDays differs.
$ ./cldr_week_data_check.py /local/mfabian/src/cldr/common/supplemental/supplementalData.xml '../locales/*_[A-Z][A-Z]'
WARNING: ru_UA: week MINWEEKLEN = 4, CLDR minDays = 1
$
---
 .../unicode-gen/cldr_week_data_check.py       | 258 ++++++++++++++++++
 1 file changed, 258 insertions(+)
 create mode 100755 localedata/unicode-gen/cldr_week_data_check.py
  

Patch

diff --git a/localedata/unicode-gen/cldr_week_data_check.py b/localedata/unicode-gen/cldr_week_data_check.py
new file mode 100755
index 0000000000..5dada02603
--- /dev/null
+++ b/localedata/unicode-gen/cldr_week_data_check.py
@@ -0,0 +1,258 @@ 
+#!/usr/bin/python3
+# Copyright (C) 2026 Free Software Foundation, Inc.
+# Copyright The GNU Toolchain Authors.
+# This file is part of the GNU C Library.
+#
+# The GNU C Library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# The GNU C Library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with the GNU C Library; if not, see
+# <https://www.gnu.org/licenses/>.
+'''
+This script compares the week data in cldr/common/supplemental/supplementalData.xml
+with the week data in the glibc locales.
+'''
+import glob
+import re
+import sys
+import xml.etree.ElementTree as ET
+from pathlib import Path
+
+# CLDR names -> glibc LC_TIME day numbers.
+# glibc's day list is:
+#   1 = Sunday
+#   2 = Monday
+#   3 = Tuesday
+#   4 = Wednesday
+#   5 = Thursday
+#   6 = Friday
+#   7 = Saturday
+DAY_NUMBERS = {
+    'sun': 1,
+    'mon': 2,
+    'tue': 3,
+    'wed': 4,
+    'thu': 5,
+    'fri': 6,
+    'sat': 7,
+}
+
+def parse_territories(value: str) -> list[str]:
+    '''Split a CLDR territories attribute into territory codes.'''
+    return value.split()
+
+def parse_week_data(xml_file: Path) -> tuple[
+    dict[str, int],
+    dict[str, int],
+    dict[str, int],
+    dict[str, int],
+]:
+    '''Parse CLDR supplementalData.xml weekData.'''
+    tree = ET.parse(xml_file)
+    root = tree.getroot()
+    week_data = root.find('.//weekData')
+    if week_data is None:
+        raise ValueError('Could not find <weekData>')
+    min_days: dict[str, int] = {}
+    first_day: dict[str, int] = {}
+    weekend_start: dict[str, int] = {}
+    weekend_end: dict[str, int] = {}
+    for element in week_data:
+        # Ignore entries such as:
+        #
+        #   <firstDay ... alt="variant" .../>
+        #
+        # We want the normal/default CLDR data.
+        if 'alt' in element.attrib:
+            continue
+        territories = parse_territories(element.get('territories', ''))
+        if element.tag == 'minDays':
+            count = int(element.attrib['count'])
+            for territory in territories:
+                min_days[territory] = count
+        elif element.tag in {'firstDay', 'weekendStart', 'weekendEnd'}:
+            day = element.attrib['day']
+            try:
+                day_number = DAY_NUMBERS[day]
+            except KeyError as exc:
+                raise ValueError(f'Unknown CLDR day: {day!r}') from exc
+            target = {
+                'firstDay': first_day,
+                'weekendStart': weekend_start,
+                'weekendEnd': weekend_end,
+            }[element.tag]
+
+            for territory in territories:
+                target[territory] = day_number
+        else:
+            # Ignore other weekData elements if CLDR adds any in future.
+            continue
+    return min_days, first_day, weekend_start, weekend_end
+
+def extract_lc_time(text: str) -> str | None:
+    '''Return the contents of the LC_TIME section.'''
+    match = re.search(r'(?ms)^LC_TIME\b(.*?)(?=^END LC_TIME\b)', text)
+    if match is None:
+        return None
+    return match.group(1)
+
+def parse_lc_time(lc_time: str) -> tuple[str | None, int | None, int | None, int | None]:
+    '''
+    Parse copy, week, first_weekday, and first_workday
+    from an LC_TIME section.
+
+    Returns:
+        (copy_locale, min_days, first_weekday, first_workday)
+    '''
+    copy_match = re.search(r'(?m)^\s*copy\s+"([^"]+)"', lc_time)
+    copy_locale = copy_match.group(1) if copy_match else None
+    week_match = re.search(r"(?m)^\s*week\s+([^#\n]+)", lc_time)
+    min_days = None
+    if week_match:
+        fields = week_match.group(1).strip().split(";")
+        if len(fields) != 3:
+            print('WARNING: malformed week line:', week_match.group(0).strip())
+        else:
+            try:
+                min_days = int(fields[2])
+            except ValueError:
+                print('WARNING: invalid MINWEEKLEN in:', week_match.group(0).strip())
+    first_weekday_match = re.search(r'(?m)^\s*first_weekday\s+(\d+)', lc_time)
+    first_weekday = (
+        int(first_weekday_match.group(1))
+        if first_weekday_match
+        else None
+    )
+    first_workday_match = re.search(r'(?m)^\s*first_workday\s+(\d+)', lc_time)
+    first_workday = (
+        int(first_workday_match.group(1))
+        if first_workday_match
+        else None
+    )
+    return (copy_locale, min_days, first_weekday, first_workday)
+
+def expected_value(
+    values: dict[str, int],
+    territory: str,
+) -> int:
+    '''Get a CLDR value, using territory 001 as the default.'''
+    return values.get(territory, values['001'])
+
+def expected_first_workday(
+    weekend_end: dict[str, int],
+    territory: str,
+) -> int:
+    '''
+    The first workday is the day after CLDR weekendEnd.
+
+    With glibc day numbers:
+        Sunday    = 1
+        ...
+        Saturday  = 7
+    '''
+    end_day = expected_value(weekend_end, territory)
+    return end_day % 7 + 1
+
+def check_locale(
+    filename: str,
+    *,
+    min_days: dict[str, int],
+    first_day: dict[str, int],
+    weekend_end: dict[str, int],
+) -> None:
+    '''Check one glibc locale file.'''
+    path = Path(filename)
+    # For *_CC locale names, the territory is the final two characters.
+    # Examples:
+    #   pt_PT -> PT
+    #   chr_US -> US
+    #   ber_MA -> MA
+    territory = path.name[-2:]
+    text = path.read_text(encoding='utf-8')
+    lc_time = extract_lc_time(text)
+    if lc_time is None:
+        print(f'WARNING: {path.name}: no LC_TIME section')
+        return
+    (
+        copy_locale,
+        locale_min_days,
+        locale_first_weekday,
+        locale_first_workday,
+    ) = parse_lc_time(lc_time)
+    expected_min_days = expected_value(min_days, territory)
+    expected_first_weekday = expected_value(first_day, territory)
+    expected_workday = expected_first_workday(weekend_end, territory)
+    if copy_locale is not None:
+        copy_territory = copy_locale[-2:]
+        if expected_min_days != expected_value(min_days, copy_territory):
+            print(f'WARNING {path.name}: LC_TIME copied from {copy_locale} but expected CLDR minDays differs.')
+        if expected_first_weekday != expected_value(first_day, copy_territory):
+            print(f'WARNING {path.name}: LC_TIME copied from {copy_locale} but expected CLDR firstDay differs.')
+        if expected_workday != expected_first_workday(weekend_end, copy_territory):
+            print(f'WARNING {path.name}: LC_TIME copied from {copy_locale} but expected CLDR weekendEnd differs.')
+        return
+    # Check that all three values exist.
+    missing = []
+    if locale_min_days is None:
+        missing.append('week')
+    if locale_first_weekday is None:
+        missing.append('first_weekday')
+    if locale_first_workday is None:
+        missing.append('first_workday')
+    if missing:
+        print(f'WARNING: {path.name}: missing ' + ', '.join(missing))
+        return
+    if locale_min_days != expected_min_days:
+        print(f'WARNING: {path.name}: '
+              f'week MINWEEKLEN = {locale_min_days}, '
+              f'CLDR minDays = {expected_min_days}')
+    if locale_first_weekday != expected_first_weekday:
+        print(f'WARNING: {path.name}: '
+              f'first_weekday = {locale_first_weekday}, '
+              f'CLDR firstDay = {expected_first_weekday}')
+    if locale_first_workday != expected_workday:
+        print(f'WARNING: {path.name}: '
+              f'first_workday = {locale_first_workday}, '
+              f'CLDR weekendEnd implies {expected_workday}')
+
+def main() -> None:
+    if len(sys.argv) != 3:
+        print(f'usage: {sys.argv[0]} '
+              "supplementalData.xml 'locales/*_[A-Z][A-Z]'",
+              file=sys.stderr)
+        sys.exit(2)
+    xml_file = Path(sys.argv[1])
+    locale_pattern = sys.argv[2]
+    (
+        min_days,
+        first_day,
+        weekend_start,
+        weekend_end,
+    ) = parse_week_data(xml_file)
+    # Make sure CLDR has its 001 defaults.
+    for name, values in (
+        ('min_days', min_days),
+        ('first_day', first_day),
+        ('weekend_start', weekend_start),
+        ('weekend_end', weekend_end),
+    ):
+        if '001' not in values:
+            raise ValueError(f'{name} has no CLDR 001 default')
+    for filename in sorted(glob.glob(locale_pattern)):
+        check_locale(
+            filename,
+            min_days=min_days,
+            first_day=first_day,
+            weekend_end=weekend_end,
+        )
+
+if __name__ == '__main__':
+    main()