diff lib/unigbrk/u-grapheme-breaks.h @ 19273:638b6d1fdf36 ueno/unicode-9.0.0

libunistring: update to Unicode 9.0.0 * lib/gen-uni-tables.c (fill_properties): Recognize Sentence_Terminal and Prepended_Concatenation_Mark. (is_property_default_ignorable_code_point): Exclude U+08E2. (fill_arabicshaping): Allow missing whitespace when parsing; recognize "AFRICAN FEH", "AFRICAN QAF", and "AFRICAN MOON". (output_blocks): Increase the element size of the level1 table to accommodate more blocks. (get_lbp): Recognize ZWJ, E_Base, and E_Modifier characters; Update each class according to the standard. (get_wbp): Recognize ZWJ, E_Base, E_Modifier, Glue_After_Zwj, and E_Base_GAZ characters. (output_gbp_table): Recognize ZWJ, E_Base, E_Modifier, Glue_After_Zwj, and E_Base_GAZ characters. * lib/unictype.in.h (UC_JOINING_GROUP_AFRICAN_FEH, UC_JOINING_GROUP_AFRICAN_QAF, UC_JOINING_GROUP_AFRICAN_MOON): New enum value. * lib/unilbrk/lbrktables.h (LBP_ZWJ, LBP_EB, LBP_EM): New enum value. * lib/unilbrk/lbrktables.c (unilbrk_table): Extend the table with LBP_ZWJ, LBP_EB, and LBP_EM. * lib/uniwbrk.in.h (WBP_ZWJ, WBP_EB, WBP_EM, WBP_GAZ, WBP_EBG): New enum value. * lib/uniwbrk/u-wordbreaks.h: Implement WB3c, WB15, and WB16. * lib/uniwbrk/wbrktable.h (uniwbrk_prop_index): New variable declaration. * lib/uniwbrk/wbrktable.c (uniwbrk_prop_index): New variable. (uniwbrk_table): Implement WB14. * tests/uniwbrk/test-uc-wordbreaks.c (wordbreakproperty_to_string): Check WBP_ZWJ, WBP_EB, WBP_EM, WBP_GAZ, and WBP_EBG. * modules/unigbrk/u{32,16,8}-grapheme-breaks: No longer depend on uc-is-grapheme-break. * modules/unigbrk/uc-grapheme-breaks: New module. * modules/unigbrk/uc-grapheme-breaks-tests: New module. * lib/unigbrk.in.h (GBP_ZWJ, GBP_EB, GBP_EM, GBP_GAZ, GBP_EBG): New enum value. (uc_grapheme_breaks): New function, replacing uc_is_grapheme_break. * lib/unigbrk/u-grapheme-breaks.h: New file. * lib/unigbrk/u{32,16,8}-grapheme-breaks.c: Rewrite using u-grapheme-breaks.h instead of uc_is_grapheme_break. * lib/unigbrk/uc-grapheme-breaks.c: New file. * lib/unigbrk/uc-is-grapheme-break.c: Partially update to TR29 rev 29. * tests/unigbrk/test-uc-gbrk-prop.c (graphemebreakproperty_to_string): Check GBP_ZWJ, GBP_EB, GBP_EM, GBP_GAZ, and GBP_EBG. * tests/unigbrk/test-uc-grapheme-breaks.c: New test. * tests/unigbrk/test-uc-is-grapheme-break.c (graphemebreakproperty_to_string): Check GBP_ZWJ, GBP_EB, GBP_EM, GBP_GAZ, and GBP_EBG. (main): Skip unsupported rules involving 3 or more characters, namely GB10, GB12, and GB13. * lib/uniwidth/width.c (nonspacing_table_data): Update.
author Daiki Ueno <ueno@gnu.org>
date Wed, 12 Oct 2016 17:40:37 +0200
parents
children c1cbd8206d4b
line wrap: on
line diff
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/lib/unigbrk/u-grapheme-breaks.h	Wed Oct 12 17:40:37 2016 +0200
@@ -0,0 +1,122 @@
+/* Grapheme cluster break function.
+   Copyright (C) 2010-2017 Free Software Foundation, Inc.
+   Written by Ben Pfaff <blp@cs.stanford.edu>, 2010.
+
+   This program is free software: you can redistribute it and/or modify it
+   under the terms of the GNU Lesser General Public License as published
+   by the Free Software Foundation; either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.  */
+
+void
+FUNC (const UNIT *s, size_t n, char *p)
+{
+  if (n > 0)
+    {
+      const UNIT *s_end = s + n;
+
+      /* Grapheme Cluster break property of the last character.
+         -1 at the very beginning of the string.  */
+      int last_char_prop = -1;
+
+      /* Grapheme Cluster break property of the last complex character.
+         -1 at the very beginning of the string.  */
+      int last_compchar_prop = -1;
+
+      size_t ri_count = 0;
+
+      /* Don't break inside multibyte characters.  */
+      memset (p, 0, n);
+
+      while (s < s_end)
+        {
+          ucs4_t uc;
+          int count = U_MBTOUC (&uc, s, s_end - s);
+          int prop = uc_graphemeclusterbreak_property (uc);
+
+          /* Break at the start of the string (GB1).  */
+          if (last_char_prop < 0)
+            *p = 1;
+          else
+            {
+              /* No break between CR and LF (GB3).  */
+              if (last_char_prop == GBP_CR && prop == GBP_LF)
+                /* *p = 0 */;
+              /* Break before and after newlines (GB4, GB5).  */
+              else if ((last_char_prop == GBP_CR
+                        || last_char_prop == GBP_LF
+                        || last_char_prop == GBP_CONTROL)
+                       || (prop == GBP_CR
+                           || prop == GBP_LF
+                           || prop == GBP_CONTROL))
+                *p = 1;
+              /* No break between Hangul syllable sequences (GB6, GB7, GB8).  */
+              else if ((last_char_prop == GBP_L
+                        && (prop == GBP_L
+                            || prop == GBP_V
+                            || prop == GBP_LV
+                            || prop == GBP_LVT))
+                       || ((last_char_prop == GBP_LV
+                            || last_char_prop == GBP_V)
+                           && (prop == GBP_V
+                               || prop == GBP_T))
+                       || ((last_char_prop == GBP_LVT
+                            || last_char_prop == GBP_T)
+                           && prop == GBP_T))
+                /* *p = 0 */;
+              /* No break before extending characters or ZWJ (GB9).  */
+              else if (prop == GBP_EXTEND || prop == GBP_ZWJ)
+                /* *p = 0 */;
+              /* No break before SpacingMarks (GB9a).  */
+              else if (prop == GBP_SPACINGMARK)
+                /* *p = 0 */;
+              /* No break after Prepend characters (GB9b).  */
+              else if (last_char_prop == GBP_PREPEND)
+                /* *p = 0 */;
+              /* No break within emoji modifier sequences (GB10).  */
+              else if ((last_compchar_prop == GBP_EB
+                        || last_compchar_prop == GBP_EBG)
+                       && prop == GBP_EM)
+                /* *p = 0 */;
+              /* No break within emoji zwj sequences (GB11).  */
+              else if (last_char_prop == GBP_ZWJ
+                       && (prop == GBP_GAZ
+                           || prop == GBP_EBG))
+                /* *p = 0 */;
+              /* No break between RI if there is an odd number of RI
+                 characters before (GB12, GB13).  */
+              else if (prop == GBP_RI)
+                {
+                  if (ri_count % 2 == 0)
+                    *p = 1;
+                  /* else *p = 0; */
+                }
+              /* Break everywhere (GBP999).  */
+              else
+                *p = 1;
+            }
+
+          last_char_prop = prop;
+
+          if (!(prop == GBP_EXTEND
+                && (last_compchar_prop == GBP_EB
+                    || last_compchar_prop == GBP_EBG)))
+            last_compchar_prop = prop;
+
+          if (prop == GBP_RI)
+            ri_count++;
+          else
+            ri_count = 0;
+
+          s += count;
+          p += count;
+        }
+    }
+}