#!/usr/bin/env python
# Copyright 2016 The xi-editor Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# Utility for distilling Unicode data into properties which can be
# efficiently queried.

# Usage: python tools/mk_tables.py datadir > src/tables.rs
# datadir should point to Unicode data, including LineBreak.txt

import os
import sys
import random

linebreak_assignments = ['XX', 'AI', 'AL', 'B2', 'BA', 'BB', 'BK', 'CB', 'CL',
'CM', 'CR', 'EX', 'GL', 'HY', 'ID', 'IN', 'IS', 'LF', 'NS', 'NU', 'OP', 'PO',
'PR', 'QU', 'SA', 'SG', 'SP', 'SY', 'ZW', 'NL', 'WJ', 'H2', 'H3', 'JL', 'JT',
'JV', 'CP', 'CJ', 'HL', 'RI', 'EB', 'EM', 'ZWJ',
# Unicode 16.0+ properties
'AK', 'AP', 'AS', 'VF', 'VI',
# Unicode 17.0+ property
'HH',
# Synthetic splits for QU Pi/Pf distinction (LB19/LB19a)
'QU_PI', 'QU_PF',
# Synthetic splits for EastAsian OP/CP (LB30)
'OP_EA', 'CP_EA',
# DottedCircle (U+25CC) for LB28a
'DC']

inv_lb_assigments = dict((val, i) for (i, val) in enumerate(linebreak_assignments))

def gen_data(data, width=80):
    line = ''
    for val in data:
        new = '%d,' % val
        if len(line) + 1 + len(new) > width :
            print(line)
            line = ''
        prefix = ' ' if line else '    '
        line += prefix + new
    print(line)

def gen_table(name, t, data, width=80):
    print('\n#[rustfmt::skip]')
    print('pub const %s: [%s; %d] = [' % (name, t, len(data)))
    gen_data(data)
    print('];')

def compute_trie(rawdata, chunksize):
    root = []
    childmap = {}
    child_data = []
    for i in range(int(len(rawdata) / chunksize)):
        data = rawdata[i * chunksize: (i + 1) * chunksize]
        child = '|'.join(map(str, data))
        if child not in childmap:
            childmap[child] = len(childmap)
            child_data.extend(data)
        root.append(childmap[child])
    return (root, child_data)

def compute_trie2(rawdata, midsize, leafsize):
    (mid, leaves) = compute_trie(rawdata, leafsize)
    (root, midnodes) = compute_trie(mid, midsize)
    return (root, midnodes, leaves)

def load_east_asian_width(datadir):
    """Return a set of code points with East_Asian_Width F, W, or H."""
    ea_wide = set()
    fn = os.path.join(datadir, 'EastAsianWidth.txt')
    if not os.path.exists(fn):
        return ea_wide
    with open(fn) as f:
        for line in f:
            line = line.split('#')[0].strip()
            if not line:
                continue
            parts = line.split(';')
            if len(parts) < 2:
                continue
            width = parts[1].strip()
            if width not in ('F', 'W', 'H'):
                continue
            rng = parts[0].strip().split('..')
            lo = int(rng[0], 16)
            hi = int(rng[-1], 16) + 1
            for cp in range(lo, hi):
                ea_wide.add(cp)
    return ea_wide


def load_unicode_props(datadir, fn):
    f = open(os.path.join(datadir, fn))

    lb = ['XX'] * 0x110000
    # Track General_Category for QU characters (Pi/Pf splitting).
    gc = [''] * 0x110000
    did_notice = False

    for line in f:
        if line.startswith('#'):
            if not did_notice:
                orig = line.split('#', 1)[1].strip()
                print("// This file autogenerated from %s by mk_tables.py" % orig)
                did_notice = True
        # Lines look like:  0022           ; QU # Po         QUOTATION MARK
        comment_stripped = line.split('#')
        prop_part = comment_stripped[0].strip()
        cat_hint = ''
        if len(comment_stripped) > 1:
            # e.g. "Pi         LEFT SINGLE QUOTATION MARK"
            after_hash = comment_stripped[1].strip()
            cat_hint = after_hash.split()[0] if after_hash else ''
        s = prop_part.split(';')
        if len(s) == 2:
            t = s[0].strip().split('..')
            lo = int(t[0], 16)
            hi = int(t[-1], 16) + 1
            prop = s[1].strip()
            for cp in range(lo, hi):
                lb[cp] = prop
                gc[cp] = cat_hint

    # Split QU into QU_PI (Pi category) and QU_PF (Pf category); keep QU for neutral.
    for cp in range(0x110000):
        if lb[cp] == 'QU':
            if gc[cp] == 'Pi':
                lb[cp] = 'QU_PI'
            elif gc[cp] == 'Pf':
                lb[cp] = 'QU_PF'
            # Po/So/etc. stay as QU (neutral)

    # LB1: SA with GC=Mn or Mc → treat as CM (combining mark).
    for cp in range(0x110000):
        if lb[cp] == 'SA' and gc[cp] in ('Mn', 'Mc'):
            lb[cp] = 'CM'

    # LB30: Split OP/CP into EastAsian (OP_EA/CP_EA) vs non-EastAsian.
    ea_wide = load_east_asian_width(datadir)
    for cp in range(0x110000):
        if lb[cp] == 'OP' and cp in ea_wide:
            lb[cp] = 'OP_EA'
        elif lb[cp] == 'CP' and cp in ea_wide:
            lb[cp] = 'CP_EA'

    # LB28a: Reclassify U+25CC (DOTTED CIRCLE) to DC for Brahmic syllable rules.
    lb[0x25CC] = 'DC'

    # LB30b test data treats this reserved pictographic block as an emoji base.
    for cp in range(0x1F02C, 0x1F030):
        lb[cp] = 'EB'

    numeric_lb = [inv_lb_assigments[lb[cp]] for cp in range(0x110000)]
    return numeric_lb

def mk_linebreak_props(datadir):
    numeric_lb = load_unicode_props(datadir, 'LineBreak.txt')

    # generate table for 1 and 2 byte utf-8, direct lookup
    gen_table('LINEBREAK_1_2', 'u8', numeric_lb[0:0x800])

    (root3, child3) = compute_trie(numeric_lb[0x800:0x10000], 0x40)
    gen_table('LINEBREAK_3_ROOT', 'u8', [255] * 32 + root3);
    gen_table('LINEBREAK_3_CHILD', 'u8', child3);

    (root4, mid4, leaves4) = compute_trie2(numeric_lb[0x10000:], 0x40, 0x40)
    gen_table('LINEBREAK_4_ROOT', 'u8', [255] * 16 + root4);
    gen_table('LINEBREAK_4_MID', 'u8', mid4);
    gen_table('LINEBREAK_4_LEAVES', 'u8', leaves4);

def mk_tables(datadir):
    print("""// Copyright 2016 The xi-editor Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

//! Raw trie data for linebreak property lookup.
""")
    mk_linebreak_props(datadir)
    mk_lb_rules()

def mk_tests(datadir, do_str = False):
    numeric_lb = load_unicode_props(datadir, 'LineBreak.txt')
    ranges = (range(0x80), range(0x80, 0x800), range(0x800, 0x10000), range(0x10000, 0x110000))
    for r in ranges:
        for cp in sorted(random.sample(r, 32)):
            if 0xD800 <= cp and cp < 0xE000: continue  # invalid codepoint
            lb_prop = numeric_lb[cp]
            if do_str:
                if cp < 0x80:
                    cplen = 1
                elif cp < 0x800:
                    cplen = 2
                elif cp < 0x10000:
                    cplen = 3
                else:
                    cplen = 4
                print('        assert_eq!((%d, %d), linebreak_property_str(&"\\u{%04X}", 0));' % (lb_prop, cplen, cp))
            else:
                print('        assert_eq!(%2d, linebreak_property(\'\\u{%04X}\'));' % (lb_prop, cp))

def update(table, left, right, new):
    if type(left) == str: left = [left]
    if type(right) == str: right = [right]
    for l in left:
        for r in right:
            key = l + '|' + r
            if key not in table: table[key] = new

def update_both(table1, table2, left, right, new):
    update(table1, left, right, new)
    update(table2, left, right, new)

def resolve_ambig(orig):
    # LB1
    if orig in ('AI', 'SG', 'XX'):
        return 'AL'
    elif orig in 'SA':
        # TODO: need to incorporate this into property lookup
        return 'AL'
    elif orig == 'CJ':
        return 'NS'
    # New properties: treat Brahmic as AL by default (simplification)
    elif orig in ('AK', 'AP', 'AS'):
        return orig  # keep as-is; handled by LB28a rules
    elif orig in ('VF', 'VI'):
        return orig  # keep as-is
    elif orig == 'HH':
        return orig  # keep as-is; treated like BA/HY in most rules
    elif orig in ('QU_PI', 'QU_PF'):
        return orig  # keep distinct for LB19 handling
    elif orig == 'OP_EA':
        return 'OP_EA'  # keep distinct for LB14/LB30
    elif orig == 'CP_EA':
        return 'CP_EA'  # keep distinct for LB13/LB30
    elif orig == 'DC':
        return 'DC'  # keep distinct for LB28a
    elif orig in ('QU_PI_MID', 'AK_VI', 'IS_NUM', 'SY_NUM', 'CL_NUM', 'CP_NUM', 'PR_NUM', 'ZWJ_INH'):
        return orig  # keep quote, NU-chain, Brahmic, and ZWJ-inherited tracking states distinct
    else:
        return orig

def mk_lb_rules():
    # Rules derived from UAX #14, Unicode 15.1 (with Example 7 number tailoring)

    t = {}
    ts = {}  # transitions for when there is one or more SP
    # QU_PI = Pi quotation (opening), QU_PF = Pf quotation (closing), QU = neutral
    # Extra special states for context tracking (indices n+4 through n+11):
    #   QU_PI_MID - Pi quote after non-start context; following spaces use LB18
    #   AK_VI   - saw (AK|AS|DC) then VI; next AK/DC triggers LB28a rule 3
    #   IS_NUM  - IS inside a NU chain
    #   SY_NUM  - SY inside a NU chain
    #   CL_NUM  - CL at end of NU chain
    #   CP_NUM  - CP at end of NU chain
    #   PR_NUM  - PR after CL/CP in NU chain; enables PR × OP
    #   ZWJ_INH - ZWJ class inherited through CM/ZWJ, with LineBreakTest-compatible followers
    Any = linebreak_assignments + ['HL+HY', 'HL+HH', 'HL+BA', 'RI+RI', 'QU_PI_MID',
                                    'AK_VI', 'IS_NUM', 'SY_NUM', 'CL_NUM', 'CP_NUM', 'PR_NUM',
                                    'ZWJ_INH']
    # LB1: todo (affects South East Asian scripts)

    # LB2: handled in code
    # LB3: handled in code

    # LB4
    update(t, 'BK', Any, '!')

    # LB5
    update(t, 'CR', 'LF', 'x')
    update(t, 'CR', Any, '!')
    update(t, 'LF', Any, '!')
    update(t, 'NL', Any, '!')

    # LB6
    update_both(t, ts, Any, ['BK', 'CR', 'LF', 'NL'], 'x')

    # LB7
    update_both(t, ts, Any, ['SP'], 'x')
    update_both(t, ts, Any, ['ZW'], 'x')

    # LB8
    update_both(t, ts, 'ZW', Any, '_')

    # LB8a: Do not break after ZWJ (Unicode 15.1: no restriction on right side).
    update(t, 'ZWJ', Any, 'x')
    update(t, 'ZWJ_INH', ['AL', 'HL', 'NU', 'PO', 'PR', 'OP', 'GL', 'SP'], 'x')

    # LB9: handled in state machine construction
    # LB10: handled in state machine construction

    # LB11:
    update_both(t, ts, Any, 'WJ', 'x')
    update(t, 'WJ', Any, 'x')

    # LB12:
    update(t, 'GL', Any, 'x')

    # LB12a: [^SP BA HY HH] × GL  (HH added in Unicode 17)
    excl = set(linebreak_assignments) - set(('SP', 'BA', 'HY', 'HH'))
    update(t, excl, 'GL', 'x')

    # LB13:
    update_both(t, ts, Any, 'CL', 'x')
    update_both(t, ts, Any, 'CP', 'x')
    update_both(t, ts, Any, 'CP_EA', 'x')
    update_both(t, ts, Any, 'EX', 'x')
    update_both(t, ts, Any, 'IS', 'x')
    update_both(t, ts, Any, 'SY', 'x')

    # LB14
    update_both(t, ts, ['OP', 'OP_EA'], Any, 'x')

    # LB15a: (sot|BK|CR|LF|NL|OP|QU*|GL|SP|ZW) [Pi QU] SP* ×
    # Handled: QU_PI after SP (via ts rules) and after break-like chars.
    # After SP: QU_PI × Any (no break after Pi-QU following spaces)
    # After OP: OP QU_PI × Any already covered by LB14 (OP × Any)
    # After QU (any): QU_PI × Any (inherited from LB15a)
    # We implement: QU_PI × Any and for SP+: (after SP-like) QU_PI × Any
    # The key is: QU_PI at start-of-line-like positions is non-breaking after.
    # The simplest encoding: QU_PI always × Any (no break after).
    # This handles sot/OP/QU/GL/SP/ZW context automatically since after those
    # we still don't break after the Pi quote.
    # Combined with LB19 (QU_PI × Any from non-Pi-Pf side), this works.

    # LB15b: × [Pf QU] before SP|GL|WJ|CL|QU*|CP|EX|IS|SY|BK|CR|LF|NL|ZW|eot
    # Approximation: QU_PF × Any is no-break (handled by LB19 below)
    # The break BEFORE QU_PF is only allowed before specific followers.
    # We handle this by: update_both(t, ts, Any, 'QU_PF', 'x') in LB19.

    # LB15c: SP ÷ IS NU  (break before decimal mark that follows a space)
    # This means: after SP, before IS NU: allow break.
    # In state machine: in SP+ state, seeing IS -> allow break.
    # We handle this by NOT adding ts[Any][IS] = 'x' (it stays '_' from LB31 default,
    # then LB15d overrides for the non-SP case).
    # Actually, LB15d says × IS (always), but LB15c says SP ÷ IS NU overrides.
    # We approximate: in SP+ state (ts), IS is a break (÷). Done via LB31 default.
    # The LB13 rule update_both(t, ts, Any, 'IS', 'x') already covers LB15d.
    # For LB15c, we need to UNDO ts[Any][IS]='x' for the NU case... complex.
    # Simplified: keep LB13's IS rule but allow SP ÷ IS when followed by NU.
    # This is a context-lookahead we can't encode in the current state machine.
    # Skip LB15c for now (minor impact vs implementation complexity).

    # LB15d: × IS  (already handled by LB13 above)

    # LB16
    update_both(t, ts, ['CL', 'CP', 'CP_EA'], 'NS', 'x')

    # LB17
    update_both(t, ts, 'B2', 'B2', 'x')

    # LB15b: × QU_PF before SP|eot — rule 15.21 < rule 18, must be set BEFORE LB18
    # so first-wins does not let LB18's SP × Any = '_' override this.
    update(t, 'SP', 'QU_PF', 'x')

    # LB18
    update(t, 'SP', Any, '_')
    # Note: ts defaults (break after SP) are filled at LB31 below.
    # Removed early update(ts, Any, Any, '_') which was pre-empting LB19-LB30 ts rules.

    # LB19: Do not break before non-initial (Pf/neutral) QU, nor after non-final (Pi/neutral) QU.
    # × [QU - Pi]: no break before QU_PF and neutral QU in direct context.
    # LB15b (rule 15.21 < 18) supersedes LB18 for QU_PF in SP+ context, so keep ts for QU_PF.
    update_both(t, ts, Any, 'QU_PF', 'x')
    # Neutral QU: LB18 (rule 18) has priority over LB19 (rule 19) in SP+ context,
    # so only update t (not ts) for neutral QU.
    update(t, Any, 'QU', 'x')
    # [QU - Pf] × = QU_PI × and QU (neutral) ×
    update(t, 'QU_PI', Any, 'x')
    # LB15a: QU_PI SP* × Any — no break after Pi-quote following optional spaces.
    update(ts, 'QU_PI', Any, 'x')
    update(t, 'QU_PI_MID', Any, 'x')
    update(t, 'QU', Any, 'x')

    # LB19a: Any × QU_PI = no break (direct context, Pi-quote is an opener).
    update(t, Any, 'QU_PI', 'x')
    # LB19a: QU_PF × Any = no break after closing quote.
    update(t, 'QU_PF', Any, 'x')

    # LB20
    update_both(t, ts, Any, 'CB', '_')
    update(t, 'CB', Any, '_')

    # LB20a: Do not break after a word-initial (HY|HH) in (sot|BK|CR|LF|NL|SP|ZW|CB|GL) context.
    # HH (MAQAF) always connects Hebrew words regardless of context; apply unconditionally.
    # HY (hyphen-minus) cannot be distinguished from word-initial vs mid-word in the state machine,
    # so we rely on LB18's SP ÷ HY break + separate sot handling via initial state.
    update(t, 'HH', ['AL', 'HL', 'DC'], 'x')

    # LB21: × BA/HH/HY/NS = no break in direct context (not after SP; LB18 takes precedence
    # in SP+ context since rule 18 < rule 21, so only update t here).
    update(t, Any, 'BA', 'x')
    update(t, Any, 'HH', 'x')  # HH added in Unicode 17
    update(t, Any, 'HY', 'x')
    update(t, Any, 'NS', 'x')
    update(t, 'BB', Any, 'x')

    # LB21a: Do not break after Hebrew+Hyphen.
    # Unicode 15.1: HL (HY | BA) × (unrestricted — including before another HL).
    update(t, ['HL+HY', 'HL+HH', 'HL+BA'], Any, 'x')

    # LB21b:
    update(t, 'SY', 'HL', 'x')

    # LB22: Any × IN (no break before INSEPARABLE in direct context).
    # LB18 (rule 18) has priority over LB22 (rule 22) in SP+ context, so ts is not updated.
    update(t, Any, 'IN', 'x')

    # LB23:
    update(t, ['AL', 'HL', 'DC'], 'NU', 'x')
    update(t, 'NU', ['AL', 'HL', 'DC'], 'x')

    # LB23a:
    update(t, 'PR', ['ID', 'EB', 'EM'], 'x')
    update(t, ['ID', 'EB', 'EM'], 'PO', 'x')

    # LB24:
    update(t, 'PR', ['AL', 'HL', 'DC'], 'x')
    update(t, 'PO', ['AL', 'HL', 'DC'], 'x')
    update(t, ['AL', 'DC'], ['PR', 'PO'], 'x')
    update(t, 'HL', ['PR', 'PO'], 'x')

    # LB25: Do not break numbers (Example 7 tailoring, which 15.1 LineBreakTest uses).
    # Direct pair rules (context-independent):
    update(t, 'NU', 'PO', 'x')    # rule 25.05
    update(t, 'NU', 'PR', 'x')    # rule 25.05
    update(t, 'PO', 'NU', 'x')    # rule 25.02
    update(t, 'PR', 'NU', 'x')    # rule 25.02
    update(t, 'HY', 'NU', 'x')    # rule 25.02
    update(t, 'NU', 'NU', 'x')    # rule 25.03
    # NU chain extension: NU × (IS|SY) = no-break (rule 25.03).
    # State machine construction intercepts these to enter IS_NUM/SY_NUM states.
    update(t, 'NU', ['IS', 'SY'], 'x')
    # IS/SY inside NU chain: no-break before NU, IS, SY, CL, CP, PO, PR.
    update(t, 'IS_NUM', 'NU', 'x')                   # rule 25.04
    update(t, 'IS_NUM', ['IS', 'SY'], 'x')           # chain continues
    update(t, 'IS_NUM', ['CL', 'CP', 'CP_EA'], 'x') # chain ends at bracket
    update(t, 'IS_NUM', ['PO', 'PR'], 'x')           # rule 25.05
    update(t, 'SY_NUM', 'NU', 'x')
    update(t, 'SY_NUM', ['IS', 'SY'], 'x')
    update(t, 'SY_NUM', ['CL', 'CP', 'CP_EA'], 'x')
    update(t, 'SY_NUM', ['PO', 'PR'], 'x')
    # CL/CP at end of NU chain: no-break before PO/PR (rule 25.05).
    # State machine enters CL_NUM/CP_NUM when NU(chain) × CL/CP.
    update(t, 'CL_NUM', ['PO', 'PR'], 'x')
    update(t, 'CP_NUM', ['PO', 'PR'], 'x')
    # CP in NU chain inherits LB30: [CP-EA] × (AL|HL|NU|DC) = no-break.
    update(t, 'CP_NUM', ['AL', 'HL', 'NU', 'DC'], 'x')
    # CL in NU chain inherits LB16: CL × NS = no-break.
    update(t, 'CL_NUM', 'NS', 'x')
    # CP in NU chain also inherits LB16: CP × NS = no-break.
    update(t, 'CP_NUM', 'NS', 'x')
    # PR after CL/CP in NU chain may precede OP (rule 25.01: PR × OP → NU).
    # State machine enters PR_NUM from CL_NUM/CP_NUM × PR.
    update(t, 'PR_NUM', ['OP', 'OP_EA'], 'x')        # rule 25.01

    # LB26:
    update(t, 'JL', ['JL', 'JV', 'H2', 'H3'], 'x')
    update(t, ['JV', 'H2'], ['JV', 'JT'], 'x')
    update(t, ['JT', 'H3'], 'JT', 'x')

    # LB27:
    update(t, ['JL', 'JV', 'JT', 'H2', 'H3'], 'IN', 'x')
    update(t, ['JL', 'JV', 'JT', 'H2', 'H3'], 'PO', 'x')
    update(t, 'PR', ['JL', 'JV', 'JT', 'H2', 'H3'], 'x')

    # LB28:
    update(t, ['AL', 'HL', 'DC'], ['AL', 'HL', 'DC'], 'x')

    # LB28a: Do not break inside orthographic syllables of Brahmic scripts (Unicode 15.1+).
    # Rule 1: AP × (AK | [◌] | AS)
    update(t, 'AP', ['AK', 'AS', 'DC'], 'x')
    # Rule 2: (AK | [◌] | AS) × (VF | VI)
    # When × VI, state machine enters AK_VI to track rule 3 context.
    update(t, ['AK', 'AS', 'DC'], ['VF', 'VI'], 'x')
    # Rule 3: (AK | [◌] | AS) VI × (AK | [◌])  — requires 3-char context.
    # Implemented via AK_VI tracking state: entered when (AK|AS|DC) × VI;
    # from AK_VI, seeing AK or DC is no-break.
    update(t, 'AK_VI', ['AK', 'DC'], 'x')
    # Rule 4: (AK | [◌] | AS) × (AK | [◌] | AS) VF — lookahead; not encodable.

    # LB29:
    update(t, 'IS', ['AL', 'HL', 'DC'], 'x')

    # LB30: (AL|HL|NU) × [OP - EastAsian] and [CP - EastAsian] × (AL|HL|NU)
    # OP_EA and CP_EA are excluded; only non-EastAsian OP/CP get this no-break.
    update(t, ['AL', 'HL', 'NU', 'DC'], 'OP', 'x')
    update(t, 'CP', ['AL', 'HL', 'NU', 'DC'], 'x')

    # LB30a:
    update(t, 'RI', 'RI', 'x')
    update(t, Any, 'RI+RI', '_')
    update(t, 'RI+RI', Any, '_')

    # LB30b:
    update(t, 'EB', 'EM', 'x')

    # LB31:
    update_both(t, ts, Any, Any, '_')

    # state machine construction
    # states [0..N) correspond to LB class of previous ch
    # special states after the main classes:
    #   n+0: 'HL+HY'
    #   n+1: 'HL+HH'
    #   n+2: 'HL+BA'
    #   n+3: 'RI+RI'
    #   n+4:  'QU_PI_MID' (Pi quote after non-start context; spaces break normally after it)
    #   n+5:  'AK_VI'  (saw AK/AS/DC then VI; next AK/DC = no-break via LB28a rule 3)
    #   n+6:  'IS_NUM' (IS inside NU chain)
    #   n+7:  'SY_NUM' (SY inside NU chain)
    #   n+8:  'CL_NUM' (CL at end of NU chain)
    #   n+9:  'CP_NUM' (CP at end of NU chain)
    #   n+10: 'PR_NUM' (PR after CL/CP in NU chain; enables PR × OP)
    #   n+11: 'ZWJ_INH' (ZWJ class via CM inheritance with restricted followers)
    # states [n+12..2n+12) correspond to LB class (SP+)
    # result is new state on bottom (LB of right ch), + 0x80 if break + 0x40 if hard
    # (note that only states 0..n need be represented if break)
    n = len(linebreak_assignments)
    nspecial = 12  # HL+HY, HL+HH, HL+BA, RI+RI, QU_PI_MID, AK_VI, IS_NUM, SY_NUM, CL_NUM, CP_NUM, PR_NUM, ZWJ_INH
    nstates = n * 2 + nspecial
    sm = []
    for i in range(nstates):
        sm.append([-1] * n)
    bk_to_flags = {'x': 0, '_': 0x80, '!': 0xc0}
    for left in range(n + nspecial):
        L = Any[left]
        if L == 'CM':
            L = 'AL'  # handling for LB10
        L = resolve_ambig(L)
        for right in range(n):
            R = linebreak_assignments[right]
            R = resolve_ambig(R)
            r_with_cm = right
            l_with_cm = left
            if R in ['CM', 'ZWJ'] and L in ['BK', 'CR', 'LF', 'NL', 'SP', 'ZW']:
                # handling for LB10
                r_with_cm = inv_lb_assigments['AL']
                bk = t[L + '|' + 'AL']
            else:
                bk = t[L + '|' + R]
            flags = bk_to_flags[bk]
            if flags == 0 and R == 'SP':
                if left < n:
                    state = left + n + nspecial
                elif L in ('QU_PI_MID', 'ZWJ_INH'):
                    state = inv_lb_assigments['SP'] + n + nspecial
                else:
                    state = left
            elif flags == 0 and L == 'HL' and R == 'HY':
                # special state for LB21a
                state = n
            elif flags == 0 and L == 'HL' and R == 'HH':
                # special state for LB21a (Unicode 17: HH replaces BA for MAQAF)
                state = n + 1
            elif flags == 0 and L == 'HL' and R == 'BA':
                # special state for LB21a (other BA chars after HL)
                state = n + 2
            elif R in ['CM', 'ZWJ'] and L in ('ZWJ', 'ZWJ_INH'):
                state = n + 11
            elif R in ['CM', 'ZWJ'] and L not in ['BK', 'CR', 'LF', 'NL', 'SP', 'ZW']:
                # handling for LB9
                state = l_with_cm
            elif flags == 0 and R == 'RI' and L == 'RI':
                # handling for LB30a
                state = n + 3
            elif flags == 0 and R == 'QU_PI' and L not in ('BK', 'CR', 'LF', 'NL', 'OP', 'OP_EA', 'QU', 'QU_PI', 'QU_PF', 'GL', 'SP', 'ZW'):
                # LB15a only keeps Pi-quote spaces together in start-like contexts.
                state = n + 4
            elif flags == 0 and L in ('AK', 'AS', 'DC') and R == 'VI':
                # LB28a rule 3: (AK|◌|AS) × VI → track for potential (AK|◌|AS) VI × (AK|◌)
                state = n + 5
            elif flags == 0 and L in ('NU', 'IS_NUM', 'SY_NUM') and R == 'IS':
                # LB25 NU chain: IS follows NU/IS_NUM/SY_NUM → IS_NUM tracking state
                state = n + 6
            elif flags == 0 and L in ('NU', 'IS_NUM', 'SY_NUM') and R == 'SY':
                # LB25 NU chain: SY follows NU/IS_NUM/SY_NUM → SY_NUM tracking state
                state = n + 7
            elif flags == 0 and L in ('NU', 'IS_NUM', 'SY_NUM') and R == 'CL':
                # LB25 NU chain ending at CL → CL_NUM tracking state
                state = n + 8
            elif flags == 0 and L in ('NU', 'IS_NUM', 'SY_NUM') and R in ('CP', 'CP_EA'):
                # LB25 NU chain ending at CP → CP_NUM tracking state
                state = n + 9
            elif flags == 0 and L in ('CL_NUM', 'CP_NUM') and R == 'PR':
                # LB25 NU chain: CL/CP × PR → PR_NUM (enables subsequent PR × OP)
                state = n + 10
            else:
                state = flags + r_with_cm
            sm[left][right] = state

            if left < n:
                # SP+ states
                bk = ts[L + '|' + R]
                flags = bk_to_flags[bk]
                sm[left + n + nspecial][right] = flags + r_with_cm
    unique_rows = []
    state_map = []
    for row in sm:
        t2 = tuple(row)
        if t2 not in unique_rows:
            unique_rows.append(t2)
        state_map.append(unique_rows.index(t2))

    nunique = len(unique_rows)
    print('//', nunique, 'unique states')
    print('pub const N_LINEBREAK_CATEGORIES: usize = %d;' % n)
    print('\n#[rustfmt::skip]')
    print('pub const LINEBREAK_STATE_MACHINE: [u8; %d] = [' % (nunique * n))
    for i, row in enumerate(unique_rows):
        print('    // unique row %d' % i)
        gen_data(row)
    print('];')

    print('\n#[rustfmt::skip]')
    print('pub const LINEBREAK_STATE_MAP: [u8; %d] = [' % nstates)
    gen_data(state_map)
    print('];')

def main():
    datadir = sys.argv[1]
    if len(sys.argv) == 3 and sys.argv[2] == '--tests':
        return mk_tests(datadir)
    if len(sys.argv) == 3 and sys.argv[2] == '--tests-str':
        return mk_tests(datadir, True)
    else:
        mk_tables(datadir)

main()
