source/data/brkitr/rules/line_loose_cj.txt - Issue 2440913002: Update ICU to 58.1

Unified Diff: source/data/brkitr/rules/line_loose_cj.txt

Issue 2440913002: Update ICU to 58.1

Patch Set: Created 4 years, 2 months ago

Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.

Jump to:

View side-by-side diff with in-line comments

Index: source/data/brkitr/rules/line_loose_cj.txt

diff --git a/source/data/brkitr/line_loose_cj.txt b/source/data/brkitr/rules/line_loose_cj.txt

similarity index 51%

rename from source/data/brkitr/line_loose_cj.txt

rename to source/data/brkitr/rules/line_loose_cj.txt

index 9934fe3aa1d077407f538b8d3ba6a649fdf7622d..9b781f875a6dedad5e3e3e8f9b0c282b6f68372f 100644

--- a/source/data/brkitr/line_loose_cj.txt

+++ b/source/data/brkitr/rules/line_loose_cj.txt

@@ -1,13 +1,19 @@

+# License & terms of use: http://www.unicode.org/copyright.html

# file: line_loose_cj.txt

# Line Breaking Rules

-# Implement default line breaking as defined by

-# Unicode Standard Annex #14 Revision 34 for Unicode 8.0

+# Implement default line breaking as defined by

+# Unicode Standard Annex #14 Revision 37 for Unicode 9.0

# http://www.unicode.org/reports/tr14/

-# tailored as noted in 2nd paragraph below..

+# Includes extensions to the handling of emoji ZWJ sequences from

+# https://goo.gl/cluFCn

+# tailored as noted in 2nd paragraph below.

# TODO: Rule LB 8 remains as it was in Unicode 5.2

# This is only because of a limitation of ICU break engine implementation,

@@ -33,44 +39,6 @@

!!chain;

-!!LBCMNoChain;

-!!lookAheadHardBreak;

-# !!lookAheadHardBreak Described here because it is (as yet) undocumented elsewhere

-# and only used for the line break rules.

-# It is used in the implementation of rule LB 10

-# which says to treat any combining mark that is not attached to a base

-# character as if it were of class AL (alphabetic).

-# The problem occurs in the reverse rules.

-# Consider a sequence like, with correct breaks as shown

-# LF ID CM AL AL

-# ^ ^ ^

-# Then consider the sequence without the initial ID (ideographic)

-# LF CM AL AL

-# ^ ^

-# Our CM, which in the first example was attached to the ideograph,

-# is now unattached, becomes an alpha, and joins in with the other

-# alphas.

-# When iterating forwards, these sequences do not present any problems

-# When iterating backwards, we need to look ahead when encountering

-# a CM to see whether it attaches to something further on or not.

-# (Look-ahead in a reverse rule is looking towards the start)

-# If the CM is unattached, we need to force a break.

-# !!lookAheadHardBreak forces the run time state machine to

-# stop immediately when a look ahead rule ( '/' operator) matches,

-# and set the match position to that of the look-ahead operator,

-# no matter what other rules may be in play at the time.

-# See rule LB 19 for an example.

$AI = [:LineBreak = Ambiguous:];

$AL = [:LineBreak = Alphabetic:];

@@ -82,9 +50,11 @@ $B2 = [:LineBreak = Break_Both:];

$CB = [:LineBreak = Contingent_Break:];

$CJ = [:LineBreak = Conditional_Japanese_Starter:];

$CL = [:LineBreak = Close_Punctuation:];

-$CM = [:LineBreak = Combining_Mark:];

+# $CM = [:LineBreak = Combining_Mark:];

$CP = [:LineBreak = Close_Parenthesis:];

$CR = [:LineBreak = Carriage_Return:];

+$EB = [[:LineBreak = EB:]\U0001F3C2\U0001F3C7\U0001F3CC\U0001F46A-\U0001F46D\U0001F46F\U0001F574\U0001F6CC];

+$EM = [:LineBreak = EM:];

$EXX = [\uFF01 \uFF1F];

$EX = [[:LineBreak = Exclamation:] - $EXX];

$GL = [:LineBreak = Glue:];

@@ -92,6 +62,7 @@ $HL = [:LineBreak = Hebrew_Letter:];

$HY = [:LineBreak = Hyphen:];

$H2 = [:LineBreak = H2:];

$H3 = [:LineBreak = H3:];

+# CSS Loose tailoring: CJ resolves to ID

$ID = [[:LineBreak = Ideographic:] $CJ];

$IN = [:LineBreak = Inseperable:];

$IS = [:LineBreak = Infix_Numeric:];

@@ -117,100 +88,38 @@ $SY = [:LineBreak = Break_Symbols:];

$WJ = [:LineBreak = Word_Joiner:];

$XX = [:LineBreak = Unknown:];

$ZW = [:LineBreak = ZWSpace:];

+$ZWJ = [:LineBreak = ZWJ:];

+$EmojiNRK = [[\p{Emoji}] - [$RI \u002a\u00230-9©®™〰〽]];

+# Data for Extended Pictographic scraped from CLDR common/properties/ExtendedPictographic.txt, r12773

+$Extended_Pict = [\U0001F774-\U0001F77F\u2700-\u2701\u2703-\u2704\u270E\u2710-\u2711\u2765-\u2767\U0001F030-\U0001F093\U0001F094-\U0001F09F\U0001F10D-\U0001F10F\U0001F12F\U0001F16C-\U0001F16F\U0001F1AD-\U0001F1E5\U0001F203-\U0001F20F\U0001F23C-\U0001F23F\U0001F249-\U0001F24F\U0001F252-\U0001F2FF\U0001F7D5-\U0001F7FF\U0001F000-\U0001F003\U0001F005-\U0001F02B\U0001F02C-\U0001F02F\U0001F322-\U0001F323\U0001F394-\U0001F395\U0001F398\U0001F39C-\U0001F39D\U0001F3F1-\U0001F3F2\U0001F3F6\U0001F4FE\U0001F53E-\U0001F548\U0001F54F\U0001F568-\U0001F56E\U0001F571-\U0001F572\U0001F57B-\U0001F586\U0001F588-\U0001F589\U0001F58E-\U0001F58F\U0001F591-\U0001F594\U0001F597-\U0001F5A3\U0001F5A6-\U0001F5A7\U0001F5A9-\U0001F5B0\U0001F5B3-\U0001F5BB\U0001F5BD-\U0001F5C1\U0001F5C5-\U0001F5D0\U0001F5D4-\U0001F5DB\U0001F5DF-\U0001F5E0\U0001F5E2\U0001F5E4-\U0001F5E7\U0001F5E9-\U0001F5EE\U0001F5F0-\U0001F5F2\U0001F5F4-\U0001F5F9\u2605\u2607-\u260D\u260F-\u2610\u2612\u2616-\u2617\u2619-\u261C\u261E-\u261F\u2621\u2624-\u2625\u2627-\u2629\u262B-\u262D\u2630-\u2637\u263B-\u2647\u2654-\u265F\u2661-\u2662\u2664\u2667\u2669-\u267A\u267C-\u267E\u2680-\u2691\u2695\u2698\u269A\u269D-\u269F\u26A2-\u26A9\u26AC-\u26AF\u26B2-\u26BC\u26BF-\u26C3\u26C6-\u26C7\u26C9-\u26CD\u26D0\u26D2\u26D5-\u26E8\u26EB-\u26EF\u26F6\u26FB-\u26FC\u26FE-\u26FF\u2388\U0001FA00-\U0001FFFD\U0001F0A0-\U0001F0AE\U0001F0B1-\U0001F0BF\U0001F0C1-\U0001F0CF\U0001F0D1-\U0001F0F5\U0001F0AF-\U0001F0B0\U0001F0C0\U0001F0D0\U0001F0F6-\U0001F0FF\U0001F80C-\U0001F80F\U0001F848-\U0001F84F\U0001F85A-\U0001F85F\U0001F888-\U0001F88F\U0001F8AE-\U0001F8FF\U0001F900-\U0001F90F\U0001F91F\U0001F928-\U0001F92F\U0001F931-\U0001F932\U0001F93F\U0001F94C-\U0001F94F\U0001F95F-\U0001F97F\U0001F992-\U0001F9BF\U0001F9C1-\U0001F9FF\U0001F6C6-\U0001F6CA\U0001F6E6-\U0001F6E8\U0001F6EA\U0001F6F1-\U0001F6F2\U0001F6D3-\U0001F6DF\U0001F6ED-\U0001F6EF\U0001F6F7-\U0001F6FF];

+# By LB9, a ZWJ also behaves as a CM. Including it in the definition of CM avoids having to explicitly

+# list it in the numerous rules that use CM.

+# By LB1, SA characters with general categor of Mn or Mc also resolve to CM.

+$CM = [[:LineBreak = Combining_Mark:] $ZWJ [$SA & [[:Mn:][:Mc:]]]];

# Dictionary character set, for triggering language-based break engines. Currently

-# limited to LineBreak=Complex_Context. Note that this set only works in Unicode

-# 5.0 or later as the definition of Complex_Context was corrected to include all

-# characters requiring dictionary break.

+# limited to LineBreak=Complex_Context (SA).

-$dictionary = [:LineBreak = Complex_Context:];

+$dictionary = [$SA];

# Rule LB1. By default, treat AI (characters with ambiguous east Asian width),

-# SA (South East Asian: Thai, Lao, Khmer)

+# SA (Dictionary chars, excluding Mn and Mc)

# SG (Unpaired Surrogates)

# XX (Unknown, unassigned)

# as $AL (Alphabetic)

-$ALPlus = [$AL $AI $SA $SG $XX];

-# Combining Marks. X $CM* behaves as if it were X. Rule LB6.

-$ALcm = $ALPlus $CM*;

-$BAcm = $BA $CM*;

-$BAXcm = $BAX $CM*;

-$BBcm = $BB $CM*;

-$B2cm = $B2 $CM*;

-$CLcm = $CL $CM*;

-$CPcm = $CP $CM*;

-$EXcm = $EX $CM*;

-$EXXcm = $EXX $CM*;

-$GLcm = $GL $CM*;

-$HLcm = $HL $CM*;

-$HYcm = $HY $CM*;

-$H2cm = $H2 $CM*;

-$H3cm = $H3 $CM*;

-$IDcm = $ID $CM*;

-$INcm = $IN $CM*;

-$IScm = $IS $CM*;

-$JLcm = $JL $CM*;

-$JVcm = $JV $CM*;

-$JTcm = $JT $CM*;

-$NScm = $NS $CM*;

-$NSXcm = $NSX $CM*;

-$NUcm = $NU $CM*;

-$OPcm = $OP $CM*;

-$POcm = $PO $CM*;

-$POXcm = $POX $CM*;

-$PRcm = $PR $CM*;

-$PRXcm = $PRX $CM*;

-$QUcm = $QU $CM*;

-$RIcm = $RI $CM*;

-$SYcm = $SY $CM*;

-$WJcm = $WJ $CM*;

+$ALPlus = [$AL $AI $SG $XX [$SA-[[:Mn:][:Mc:]]]];

## -------------------------------------------------

!!forward;

-# Each class of character can stand by itself as an unbroken token, with trailing combining stuff

-$ALPlus $CM+;

-$BA $CM+;

-$BAX $CM+;

-$BB $CM+;

-$B2 $CM+;

-$CL $CM+;

-$CP $CM+;

-$EX $CM+;

-$EXX $CM+;

-$GL $CM+;

-$HL $CM+;

-$HY $CM+;

-$H2 $CM+;

-$H3 $CM+;

-$ID $CM+;

-$IN $CM+;

-$IS $CM+;

-$JL $CM+;

-$JV $CM+;

-$JT $CM+;

-$NS $CM+;

-$NSX $CM+;

-$NU $CM+;

-$OP $CM+;

-$PO $CM+;

-$POX $CM+;

-$PR $CM+;

-$PRX $CM+;

-$QU $CM+;

-$RI $CM+;

-$SY $CM+;

-$WJ $CM+;

# CAN_CM is the set of characters that may combine with CM combining chars.

# Note that Linebreak UAX 14's concept of a combining char and the rules

# for what they can combine with are _very_ different from the rest of Unicode.

@@ -224,19 +133,15 @@ $CANT_CM = [ $SP $BK $CR $LF $NL $ZW $CM]; # Bases that can't take CMs

# AL_FOLLOW set of chars that can unconditionally follow an AL

# Needed in rules where stand-alone $CM s are treated as AL.

-# Chaining is disabled with CM because it causes other failures,

-# so for this one case we need to manually list out longer sequences.

-$AL_FOLLOW_NOCM = [$BK $CR $LF $NL $ZW $SP];

-$AL_FOLLOW_CM = [$CL $CP $EX $HL $IS $SY $WJ $GL $OP $QU $BA $HY $NS $IN $NU $ALPlus];

-$AL_FOLLOW = [$AL_FOLLOW_NOCM $AL_FOLLOW_CM];

+$AL_FOLLOW = [$BK $CR $LF $NL $ZW $SP $CL $CP $EX $HL $IS $SY $WJ $GL $OP $QU $BA $HY $NS $IN $NU $PR $PO $POX $ALPlus];

# Rule LB 4, 5 Mandatory (Hard) breaks.

$LB4Breaks = [$BK $CR $LF $NL];

-$LB4NonBreaks = [^$BK $CR $LF $NL];

+$LB4NonBreaks = [^$BK $CR $LF $NL $CM];

$CR $LF {100};

@@ -244,13 +149,13 @@ $CR $LF {100};

$LB4NonBreaks? $LB4Breaks {100}; # LB 5 do not break before hard breaks.

$CAN_CM $CM* $LB4Breaks {100};

-$CM+ $LB4Breaks {100};

+^$CM+ $LB4Breaks {100};

# LB 7 x SP

# x ZW

$LB4NonBreaks [$SP $ZW];

$CAN_CM $CM* [$SP $ZW];

-$CM+ [$SP $ZW];

+^$CM+ [$SP $ZW];

# LB 8 Break after zero width space

@@ -261,38 +166,38 @@ $CM+ [$SP $ZW];

$LB8Breaks = [$LB4Breaks $ZW];

$LB8NonBreaks = [[$LB4NonBreaks] - [$ZW]];

+# LB 8a ZWJ x (ID | Extended_Pict | EmojiNRK) Emoji ZWJ sequences.

+$ZWJ ($ID | $Extended_Pict | $EmojiNRK);

-# LB 9 Combining marks. X $CM needs to behave like X, where X is not $SP, $BK $CR $LF $NL

-# $CM not covered by the above needs to behave like $AL

+# LB 9 Combining marks. X $CM needs to behave like X, where X is not $SP, $BK $CR $LF $NL

+# $CM not covered by the above needs to behave like $AL

# See definition of $CAN_CM.

$CAN_CM $CM+; # Stick together any combining sequences that don't match other rules.

-$CM+;

+^$CM+;

# LB 11 Do not break before or after WORD JOINER & related characters.

-$CAN_CM $CM* $WJcm;

-$LB8NonBreaks $WJcm;

-$CM+ $WJcm;

+$CAN_CM $CM* $WJ;

+$LB8NonBreaks $WJ;

+^$CM+ $WJ;

-$WJcm $CANT_CM;

-$WJcm $CAN_CM $CM*;

+$WJ $CM* .;

# LB 12 Do not break after NBSP and related characters.

# GL x

-$GLcm $CAN_CM $CM*;

-$GLcm $CANT_CM;

+$GL $CM* .;

# LB 12a Do not break before NBSP and related characters ...

# [^SP BA HY] x GL

-[[$LB8NonBreaks] - [$SP $BA $BAX $HY]] $CM* $GLcm;

-$CM+ GLcm;

+[[$LB8NonBreaks] - [$SP $BA $BAX $HY]] $CM* $GL;

+^$CM+ $GL;

@@ -301,43 +206,43 @@ $CM+ GLcm;

# Do not include $EXX here

$LB8NonBreaks $CL;

$CAN_CM $CM* $CL;

-$CM+ $CL; # by rule 10, stand-alone CM behaves as AL

+^$CM+ $CL; # by rule 10, stand-alone CM behaves as AL

$LB8NonBreaks $CP;

$CAN_CM $CM* $CP;

-$CM+ $CP; # by rule 10, stand-alone CM behaves as AL

+^$CM+ $CP; # by rule 10, stand-alone CM behaves as AL

$LB8NonBreaks $EX;

$CAN_CM $CM* $EX;

-$CM+ $EX; # by rule 10, stand-alone CM behaves as AL

+^$CM+ $EX; # by rule 10, stand-alone CM behaves as AL

$LB8NonBreaks $IS;

$CAN_CM $CM* $IS;

-$CM+ $IS; # by rule 10, stand-alone CM behaves as AL

+^$CM+ $IS; # by rule 10, stand-alone CM behaves as AL

$LB8NonBreaks $SY;

$CAN_CM $CM* $SY;

-$CM+ $SY; # by rule 10, stand-alone CM behaves as AL

+^$CM+ $SY; # by rule 10, stand-alone CM behaves as AL

# LB 14 Do not break after OP, even after spaces

-$OPcm $SP* $CAN_CM $CM*;

-$OPcm $SP* $CANT_CM;

+$OP $CM* $SP* .;

-$OPcm $SP+ $CM+ $AL_FOLLOW?; # by rule 10, stand-alone CM behaves as AL

+$OP $CM* $SP+ $CM+ $AL_FOLLOW?; # by rule 10, stand-alone CM behaves as AL

+ # by rule 8, CM following a SP is stand-alone.

# LB 15

-$QUcm $SP* $OPcm;

+$QU $CM* $SP* $OP;

# LB 16

# Do not break between closing punctuation and $NS, even with intervening spaces

# But DO allow a break between closing punctuation and $NSX, don't include it here

-($CLcm | $CPcm) $SP* $NScm;

+($CL | $CP) $CM* $SP* $NS;

# LB 17

-$B2cm $SP* $B2cm;

+$B2 $CM* $SP* $B2;

# LB 18 Break after spaces.

@@ -348,13 +253,11 @@ $LB18Breaks = [$LB8Breaks $SP];

# LB 19

# x QU

-$LB18NonBreaks $CM* $QUcm;

-$CM+ $QUcm;

+$LB18NonBreaks $CM* $QU;

+^$CM+ $QU;

# QU x

-$QUcm .?;

-$QUcm $LB18NonBreaks $CM*; # Don't let a combining mark go onto $CR, $BK, etc.

- # TODO: I don't think this rule is needed.

+$QU $CM* .;

# LB 20

@@ -366,81 +269,96 @@ $LB20NonBreaks = [$LB18NonBreaks - $CB];

# LB 21 x (BA | HY | NS)

# BB x

-# DO allow breaks here before $BAXcm and $NSXcm, so don't include them

-$LB20NonBreaks $CM* ($BAcm | $HYcm | $NScm);

+# DO allow breaks here before $BAX and $NSX, so don't include them

+$LB20NonBreaks $CM* ($BA | $HY | $NS);

+^$CM+ ($BA | $HY | $NS);

-$BBcm [^$CB]; # $BB x

-$BBcm $LB20NonBreaks $CM*;

+$BB $CM* [^$CB]; # $BB x

+$BB $CM* $LB20NonBreaks;

# LB 21a Don't break after Hebrew + Hyphen

# HL (HY | BA) x

-$HLcm ($HYcm | $BAcm | $BAXcm) [^$CB]?;

+$HL $CM* ($HY | $BA | $BAX) $CM* [^$CB]?;

# LB 21b (forward) Don't break between SY and HL

# (break between HL and SY already disallowed by LB 13 above)

-$SYcm $HLcm;

+$SY $CM* $HL;

# LB 22

-($ALcm | $HLcm) $INcm;

-$CM+ $INcm; # by rule 10, any otherwise unattached CM behaves as AL

-$EXcm $INcm;

-$IDcm $INcm;

-# $INcm $INcm; # delete this rule for CSS loose

-$NUcm $INcm;

+($ALPlus | $HL) $CM* $IN;

+^$CM+ $IN; # by rule 10, any otherwise unattached CM behaves as AL

+$EX $CM* $IN;

+($ID | $EB | $EM) $CM* $IN;

+# $IN $CM* $IN; # delete this rule for CSS loose

+$NU $CM* $IN;

-# LB 23

+# $LB 23

+($ALPlus | $HL) $CM* $NU;

+^$CM+ $NU; # Rule 10, any otherwise unattached CM behaves as AL

+$NU $CM* ($ALPlus | $HL);

+# LB 23a

# Do not include $POX here

-$IDcm $POcm;

-$ALcm $NUcm; # includes $LB19

-$HLcm $NUcm;

-$CM+ $NUcm; # Rule 10, any otherwise unattached CM behaves as AL

-$NUcm $ALcm;

-$NUcm $HLcm;

+$PR $CM* ($ID | $EB | $EM);

+($ID | $EB | $EM) $CM* $PO;

# LB 24

# Do not include $PRX here

-$PRcm $IDcm;

-$PRcm ($ALcm | $HLcm);

-($POcm | $POXcm) ($ALcm | $HLcm);

+($PR | $PO | $POX) $CM* ($ALPlus | $HL);

+($ALPlus | $HL) $CM* ($PR | $PO | $POX); # TODO: should this be ($PR | $PRX | $PO)

+^$CM+ ($PR | $PO | $POX); # Rule 10, any otherwise unattached CM behaves as AL

# LB 25 Numbers.

# Here do not include $PRX at the beginning or $POX at the end

+(($PR | $PO | $POX) $CM*)? (($OP | $HY) $CM*)? $NU ($CM* ($NU | $SY | $IS))*

+ ($CM* ($CL | $CP))? ($CM* ($PR | $PRX | $PO))?;

# LB 26 Do not break a Korean syllable

-$JLcm ($JLcm | $JVcm | $H2cm | $H3cm);

-($JVcm | $H2cm) ($JVcm | $JTcm);

-($JTcm | $H3cm) $JTcm;

+$JL $CM* ($JL | $JV | $H2 | $H3);

+($JV | $H2) $CM* ($JV | $JT);

+($JT | $H3) $CM* $JT;

# LB 27 Treat korean Syllable Block the same as ID (don't break it)

# Do not include $POX or $PRX here

-($JLcm | $JVcm | $JTcm | $H2cm | $H3cm) $INcm;

-($JLcm | $JVcm | $JTcm | $H2cm | $H3cm) $POcm;

-$PRcm ($JLcm | $JVcm | $JTcm | $H2cm | $H3cm);

+($JL | $JV | $JT | $H2 | $H3) $CM* $IN;

+($JL | $JV | $JT | $H2 | $H3) $CM* $PO;

+$PR $CM* ($JL | $JV | $JT | $H2 | $H3);

# LB 28 Do not break between alphabetics

-($ALcm | $HLcm) ($ALcm | $HLcm);

-$CM+ ($ALcm | $HLcm); # The $CM+ is from rule 10, an unattached CM is treated as AL

+($ALPlus | $HL) $CM* ($ALPlus | $HL);

+^$CM+ ($ALPlus | $HL); # The $CM+ is from rule 10, an unattached CM is treated as AL

# LB 29

-$IScm ($ALcm | $HLcm);

+$IS $CM* ($ALPlus | $HL);

# LB 30

-($ALcm | $HLcm | $NUcm) $OPcm;

-$CM+ $OPcm; # The $CM+ is from rule 10, an unattached CM is treated as AL.

-$CPcm ($ALcm | $HLcm | $NUcm);

+($ALPlus | $HL | $NU) $CM* $OP;

+^$CM+ $OP; # The $CM+ is from rule 10, an unattached CM is treated as AL.

+$CP $CM* ($ALPlus | $HL | $NU);

+# LB 30a Do not break between regional indicators. Break after pairs of them.

+# Tricky interaction with LB8a: ZWJ x ID

+$RI $CM* $RI / [[^$BK $CR $LF $NL $SP $ZW $WJ $CL $CP $EX $IS $SY $GL $QU $BA $HY $NS $CM] {eof}];

+$RI $CM* $RI $CM* $ZWJ / [[^$BK $CR $LF $NL $SP $ZW $WJ $CL $CP $EX $IS $SY $GL $QU $BA $HY $NS $CM $ID $Extended_Pict $EmojiNRK] {eof}];

+$RI $CM* $RI $CM* [$CM-$ZWJ] / [[^$BK $CR $LF $NL $SP $ZW $WJ $CL $CP $EX $IS $SY $GL $QU $BA $HY $NS $CM] {eof}];

+$RI $CM* $RI $CM* [$BK $CR $LF $NL $SP $ZW $WJ $CL $CP $EX $IS $SY $GL $QU $BA $HY $NS {eof}];

+$RI $CM* $RI $CM* $ZWJ ($ID | $Extended_Pict | $EmojiNRK);

-# LB 30a Do not break between regional indicators.

-$RIcm $RIcm;

+# LB 30b Do not break between an Emoji Base and an Emoji Modifier

+$EB $CM* $EM;

# Reverse Rules.

@@ -449,40 +367,10 @@ $RIcm $RIcm;

!!reverse;

-$CM+ $ALPlus;

-$CM+ $BA;

-$CM+ $BAX;

-$CM+ $BB;

-$CM+ $B2;

-$CM+ $CL;

-$CM+ $CP;

-$CM+ $EX;

-$CM+ $EXX;

-$CM+ $GL;

-$CM+ $HL;

-$CM+ $HY;

-$CM+ $H2;

-$CM+ $H3;

-$CM+ $ID;

-$CM+ $IN;

-$CM+ $IS;

-$CM+ $JL;

-$CM+ $JV;

-$CM+ $JT;

-$CM+ $NS;

-$CM+ $NSX;

-$CM+ $NU;

-$CM+ $OP;

-$CM+ $PO;

-$CM+ $POX;

-$CM+ $PR;

-$CM+ $PRX;

-$CM+ $QU;

-$CM+ $RI;

-$CM+ $SY;

-$CM+ $WJ;

-$CM+;

+# LB 9 Combining Marks.

+# Stick together any combining sequences that don't match other rules.

+^$CM+ $CAN_CM?;

# Sequences of the form (shown forwards)

@@ -493,26 +381,16 @@ $AL_FOLLOW $CM+ / (

[$BK $CR $LF $NL $ZW {eof}] |

$SP+ $CM+ $SP |

$SP+ $CM* ([^$OP $CM $SP] | [$AL {eof}])); # if LB 14 will match, need to surpress this break.

- # LB14 says OP SP* x .

+ # LB14 says OP SP* x .

# becomes OP SP* x AL

# becomes OP SP* x CM+ AL_FOLLOW

# Further note: the $AL in [$AL {eof}] is only to work around

# a rule compiler bug which complains about

# empty sets otherwise.

-# Sequences of the form (shown forwards)

-# [CANT_CM] <break> [CM] <break> [PR]

-# The CM needs to behave as an AL

-# This rule is concerned about getting the second of the two <breaks> in place.

-[$PR $PRX ] / $CM+ [$BK $CR $LF $NL $ZW $SP {eof}];

-# LB 4, 5, 5

+# LB 4, 5, 6

$LB4Breaks [$LB4NonBreaks-$CM];

$LB4Breaks $CM+ $CAN_CM;

@@ -529,30 +407,36 @@ $LF $CR;

# Requires an engine enhancement.

# / $SP* $ZW

+# LB 8a ZWJ x (ID | Extended_Pict | EmojiNRK)

+($ID | $Extended_Pict | $EmojiNRK) $ZWJ $CM* $CAN_CM?;

# LB 9,10 Combining marks.

# X $CM needs to behave like X, where X is not $SP or controls.

# $CM not covered by the above needs to behave like $AL

# Stick together any combining sequences that don't match other rules.

-$CM+ $CAN_CM;

+^$CM+ $CAN_CM;

# LB 11

-$CM* $WJ $CM* $CAN_CM;

-$CM* $WJ [$LB8NonBreaks-$CM];

+$WJ $CM* $CAN_CM;

+$WJ [$LB8NonBreaks-$CM];

$CANT_CM $CM* $WJ;

-$CM* $CAN_CM $CM* $WJ;

+$CAN_CM $CM* $WJ;

# LB 12a

# [^SP BA HY] x GL

-$CM* $GL $CM* [$LB8NonBreaks-[$CM $SP $BA $BAX $HY]];

+$GL $CM* [$LB8NonBreaks-[$CM $SP $BA $BAX $HY]];

# LB 12

# GL x

$CANT_CM $CM* $GL;

-$CM* $CAN_CM $CM* $GL;

+$CAN_CM $CM* $GL;

# LB 13

@@ -569,34 +453,22 @@ $EX [$LB8NonBreaks-$CM];

$IS [$LB8NonBreaks-$CM];

$SY [$LB8NonBreaks-$CM];

-# Rule 13 & 14 taken together for an edge case.

-# Match this, shown forward

-# OP SP+ ($CM+ behaving as $AL) (CL | CP | EX | IS | IY)

-# This really wants to chain at the $CM+ (which is acting as an $AL)

-# except for $CM chaining being disabled.

-[$CL $CP $EX $IS $SY] $CM+ $SP+ $CM* $OP;

# LB 14 OP SP* x

-$CM* $CAN_CM $SP* $CM* $OP;

- $CANT_CM $SP* $CM* $OP;

-$AL_FOLLOW? $CM+ $SP $SP* $CM* $OP; # by LB 10, behaves like $AL_FOLLOW? $AL $SP* $CM* $OP

- $AL_FOLLOW_NOCM $CM+ $SP+ $CM* $OP;

-$CM* $AL_FOLLOW_CM $CM+ $SP+ $CM* $OP;

-$SY $CM $SP+ $OP; # TODO: Experiment. Remove.

+. $SP* $CM* $OP;

+$AL_FOLLOW? $CM+ $SP+ $CM* $OP; # by LB 10, behaves like $AL_FOLLOW? $AL $SP+ $CM* $OP

# LB 15

-$CM* $OP $SP* $CM* $QU;

+$OP $SP* $CM* $QU;

# LB 16

# Don't include $NSX here

-$CM* $NS $SP* $CM* ($CL | $CP);

+$NS $SP* $CM* ($CL | $CP);

# LB 17

-$CM* $B2 $SP* $CM* $B2;

+$B2 $SP* $CM* $B2;

# LB 18 break after spaces

# Nothing explicit needed here.

@@ -605,13 +477,13 @@ $CM* $B2 $SP* $CM* $B2;

# LB 19

-$CM* $QU $CM* $CAN_CM; # . x QU

-$CM* $QU $LB18NonBreaks;

+$QU $CM* $CAN_CM; # . x QU

+$QU $LB18NonBreaks;

-$CM* $CAN_CM $CM* $QU; # QU x .

+$CAN_CM $CM* $QU; # QU x .

$CANT_CM $CM* $QU;

# LB 20 Break before and after CB.

# nothing needed here.

@@ -619,35 +491,38 @@ $CM* $CAN_CM $CM* $QU; # QU x .

# LB 21

# Don't include $BAX or $NSX here

-$CM* ($BA | $HY | $NS) $CM* [$LB20NonBreaks-$CM]; # . x (BA | HY | NS)

+($BA | $HY | $NS) $CM* [$LB20NonBreaks-$CM]; # . x (BA | HY | NS)

-$CM* [$LB20NonBreaks-$CM] $CM* $BB; # BB x .

-[^$CB] $CM* $BB; #

+[$LB20NonBreaks-$CM] $CM* $BB; # BB x .

+[^$CB] $CM* $BB; #

# LB21a

-[^$CB]? $CM* ($HY | $BA | $BAX) $CM* $HL;

+[^$CM $CB]? $CM* ($HY | $BA | $BAX) $CM* $HL;

# LB21b (reverse)

-$CM* $HL $CM* $SY;

+$HL $CM* $SY;

# LB 22

-$CM* $IN $CM* ($ALPlus | $HL);

-$CM* $IN $CM* $EX;

-$CM* $IN $CM* $ID;

-# $CM* $IN $CM* $IN; # delete this rule for CSS loose

-$CM* $IN $CM* $NU;

+$IN $CM* ($ALPlus | $HL);

+$IN $CM* $EX;

+$IN $CM* ($ID | $EB | $EM);

+# $IN $CM* $IN; # delete this rule for CSS loose

+$IN $CM* $NU;

# LB 23

# Do not include $POX here

-$CM* $PO $CM* $ID;

-$CM* $NU $CM* ($ALPlus | $HL);

-$CM* ($ALPlus | $HL) $CM* $NU;

+$NU $CM* ($ALPlus | $HL);

+($ALPlus | $HL) $CM* $NU;

+# LB23a

+# Do not include $PRX here

+($ID | $EB | $EM) $CM* $PR;

+$PO $CM* ($ID | $EB | $EM);

# LB 24

# Do not include $PRX here

-$CM* $ID $CM* $PR;

-$CM* ($ALPlus | $HL) $CM* $PR;

-$CM* ($ALPlus | $HL) $CM* ($PO | $POX);

+($ALPlus | $HL) $CM* ($PR | $PO | $POX);

+($PR | $PO | $POX) $CM* ($ALPlus | $HL);

# LB 25

@@ -655,37 +530,56 @@ $CM* ($ALPlus | $HL) $CM* ($PO | $POX);

($CM* ($PR | $PRX | $PO))? ($CM* ($CL | $CP))? ($CM* ($NU | $IS | $SY))* $CM* $NU ($CM* ($OP | $HY))? ($CM* ($PR | $PO | $POX))?;

# LB 26

-$CM* ($H3 | $H2 | $JV | $JL) $CM* $JL;

-$CM* ($JT | $JV) $CM* ($H2 | $JV);

-$CM* $JT $CM* ($H3 | $JT);

+($H3 | $H2 | $JV | $JL) $CM* $JL;

+($JT | $JV) $CM* ($H2 | $JV);

+$JT $CM* ($H3 | $JT);

# LB 27

# Do not include $POX or $PRX here

-$CM* $IN $CM* ($H3 | $H2 | $JT | $JV | $JL);

-$CM* $PO $CM* ($H3 | $H2 | $JT | $JV | $JL);

-$CM* ($H3 | $H2 | $JT | $JV | $JL) $CM* $PR;

+$IN $CM* ($H3 | $H2 | $JT | $JV | $JL);

+$PO $CM* ($H3 | $H2 | $JT | $JV | $JL);

+ ($H3 | $H2 | $JT | $JV | $JL) $CM* $PR;

# LB 28

-$CM* ($ALPlus | $HL) $CM* ($ALPlus | $HL);

+($ALPlus | $HL) $CM* ($ALPlus | $HL);

# LB 29

-$CM* ($ALPlus | $HL) $CM* $IS;

+($ALPlus | $HL) $CM* $IS;

# LB 30

-$CM* $OP $CM* ($ALPlus | $HL | $NU);

-$CM* ($ALPlus | $HL | $NU) $CM* $CP;

+$OP $CM* ($ALPlus | $HL | $NU);

+($ALPlus | $HL | $NU) $CM* $CP;

# LB 30a

-$CM* $RI $CM* $RI;

+# Pairs of Regional Indicators.

+# The following two rules are nearly identical. The first matches only sequences with an odd number of adjacent RIs,

+# the second with an even number. Stripping away the cruft they look like

+# [^RI] RI / (RI RI)+ ^RI;

+# [^RI] RI RI / (RI RI)+ ^RI;

+# Line Loose tailoring: Don't include NSX here.

+[{bof} $NS $HY $BA $QU $CL $CP $EX $IS $SY $WJ $GL $ZW $SP $BK $CR $LF $NL $ZWJ] $CM* $RI / ($CM* $RI $CM* $RI)+ $CM* [{eof}[^$RI $CM]];

+[{bof} $NS $HY $BA $QU $CL $CP $EX $IS $SY $WJ $GL $ZW $SP $BK $CR $LF $NL $ZWJ] $CM* $RI $CM* $RI / ($CM* $RI $CM* $RI)+ $CM* [{eof}[^$RI $CM]];

+# In general, adjacent RIs stay together. The hard-break rules, above, overide this, forcing in the boundaries between pairs.

+$RI $CM* $RI;

+# WJ, GL, QU, etc. are classes with rules like "WJ x " which includes "WJ x RI".

+$RI $CM* ([$WJ $GL $QU $BB] | (($HY | $BA)$CM* $HL));

+# LB 30b Do not break between an Emoji Base and an Emoji Modifier

+$EM $CM* $EB;

## -------------------------------------------------

!!safe_reverse;

# LB 9

-$CM+ [^$CM $BK $CR $LF $NL $ZW $SP];

-$CM+ $SP / .;

+^$CM+ [^$CM $BK $CR $LF $NL $ZW $SP];

+^$CM+ $SP / .;

# LB 14

$SP+ $CM* $OP;

@@ -706,6 +600,9 @@ $CM* ($HY | $BA | $BAX) $CM* $HL;

($CM* ($IS | $SY))+ $CM* $NU;

($CL | $CP) $CM* ($NU | $IS | $SY);

+# LB 30

+($CM* $RI)+;

# For dictionary-based break

$dictionary $dictionary;

@@ -722,6 +619,6 @@ $dictionary $dictionary;

# turn off rule chaining. We don't want to move more

# than necessary.

-[$CM $OP $QU $CL $CP $B2 $PR $PRX $HY $BA $BAX $SP $dictionary]+ [^$CM $OP $QU $CL $CP $B2 $PR $PRX $HY $BA $BAX $dictionary];

+^[$CM $OP $QU $CL $CP $B2 $PR $PRX $HY $BA $BAX $SP $RI $ZWJ $dictionary]+ [^$CM $OP $QU $CL $CP $B2 $PR $PRX $HY $BA $BAX $RI $ZWJ $dictionary];

$dictionary $dictionary;

« no previous file with comments | « source/data/brkitr/rules/line_loose.txt ('k') | source/data/brkitr/rules/line_loose_fi.txt » ('j') | no next file with comments »