view lib/unilbrk/lbrktables.h @ 19273:638b6d1fdf36 ueno/unicode-9.0.0

libunistring: update to Unicode 9.0.0 * lib/gen-uni-tables.c (fill_properties): Recognize Sentence_Terminal and Prepended_Concatenation_Mark. (is_property_default_ignorable_code_point): Exclude U+08E2. (fill_arabicshaping): Allow missing whitespace when parsing; recognize "AFRICAN FEH", "AFRICAN QAF", and "AFRICAN MOON". (output_blocks): Increase the element size of the level1 table to accommodate more blocks. (get_lbp): Recognize ZWJ, E_Base, and E_Modifier characters; Update each class according to the standard. (get_wbp): Recognize ZWJ, E_Base, E_Modifier, Glue_After_Zwj, and E_Base_GAZ characters. (output_gbp_table): Recognize ZWJ, E_Base, E_Modifier, Glue_After_Zwj, and E_Base_GAZ characters. * lib/unictype.in.h (UC_JOINING_GROUP_AFRICAN_FEH, UC_JOINING_GROUP_AFRICAN_QAF, UC_JOINING_GROUP_AFRICAN_MOON): New enum value. * lib/unilbrk/lbrktables.h (LBP_ZWJ, LBP_EB, LBP_EM): New enum value. * lib/unilbrk/lbrktables.c (unilbrk_table): Extend the table with LBP_ZWJ, LBP_EB, and LBP_EM. * lib/uniwbrk.in.h (WBP_ZWJ, WBP_EB, WBP_EM, WBP_GAZ, WBP_EBG): New enum value. * lib/uniwbrk/u-wordbreaks.h: Implement WB3c, WB15, and WB16. * lib/uniwbrk/wbrktable.h (uniwbrk_prop_index): New variable declaration. * lib/uniwbrk/wbrktable.c (uniwbrk_prop_index): New variable. (uniwbrk_table): Implement WB14. * tests/uniwbrk/test-uc-wordbreaks.c (wordbreakproperty_to_string): Check WBP_ZWJ, WBP_EB, WBP_EM, WBP_GAZ, and WBP_EBG. * modules/unigbrk/u{32,16,8}-grapheme-breaks: No longer depend on uc-is-grapheme-break. * modules/unigbrk/uc-grapheme-breaks: New module. * modules/unigbrk/uc-grapheme-breaks-tests: New module. * lib/unigbrk.in.h (GBP_ZWJ, GBP_EB, GBP_EM, GBP_GAZ, GBP_EBG): New enum value. (uc_grapheme_breaks): New function, replacing uc_is_grapheme_break. * lib/unigbrk/u-grapheme-breaks.h: New file. * lib/unigbrk/u{32,16,8}-grapheme-breaks.c: Rewrite using u-grapheme-breaks.h instead of uc_is_grapheme_break. * lib/unigbrk/uc-grapheme-breaks.c: New file. * lib/unigbrk/uc-is-grapheme-break.c: Partially update to TR29 rev 29. * tests/unigbrk/test-uc-gbrk-prop.c (graphemebreakproperty_to_string): Check GBP_ZWJ, GBP_EB, GBP_EM, GBP_GAZ, and GBP_EBG. * tests/unigbrk/test-uc-grapheme-breaks.c: New test. * tests/unigbrk/test-uc-is-grapheme-break.c (graphemebreakproperty_to_string): Check GBP_ZWJ, GBP_EB, GBP_EM, GBP_GAZ, and GBP_EBG. (main): Skip unsupported rules involving 3 or more characters, namely GB10, GB12, and GB13. * lib/uniwidth/width.c (nonspacing_table_data): Update.
author Daiki Ueno <ueno@gnu.org>
date Wed, 12 Oct 2016 17:40:37 +0200
parents 9759915b2aca
children 10eb9086bea0
line wrap: on
line source

/* Line breaking auxiliary tables.
   Copyright (C) 2001-2003, 2006-2017 Free Software Foundation, Inc.
   Written by Bruno Haible <bruno@clisp.org>, 2001.

   This program is free software: you can redistribute it and/or modify it
   under the terms of the GNU Lesser General Public License as published
   by the Free Software Foundation; either version 3 of the License, or
   (at your option) any later version.

   This program is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
   Lesser General Public License for more details.

   You should have received a copy of the GNU Lesser General Public License
   along with this program.  If not, see <https://www.gnu.org/licenses/>.  */

#include "unitypes.h"

/* Line breaking classification.  */

enum
{
  /* Values >= 30 are resolved at run time. */
  LBP_BK = 30, /* mandatory break */
/*LBP_CR,         carriage return - not used here because it's a DOSism */
/*LBP_LF,         line feed - not used here because it's a DOSism */
  LBP_CM = 31, /* attached characters and combining marks */
/*LBP_NL,         next line - not used here because it's equivalent to LBP_BK */
/*LBP_SG,         surrogates - not used here because they are not characters */
  LBP_WJ =  0, /* word joiner */
  LBP_ZW = 32, /* zero width space */
  LBP_GL =  1, /* non-breaking (glue) */
  LBP_SP = 33, /* space */
  LBP_B2 =  2, /* break opportunity before and after */
  LBP_BA =  3, /* break opportunity after */
  LBP_BB =  4, /* break opportunity before */
  LBP_HY =  5, /* hyphen */
  LBP_CB = 34, /* contingent break opportunity */
  LBP_CL =  6, /* closing punctuation */
  LBP_CP =  7, /* closing parenthesis */
  LBP_EX =  8, /* exclamation/interrogation */
  LBP_IN =  9, /* inseparable */
  LBP_NS = 10, /* non starter */
  LBP_OP = 11, /* opening punctuation */
  LBP_QU = 12, /* ambiguous quotation */
  LBP_IS = 13, /* infix separator (numeric) */
  LBP_NU = 14, /* numeric */
  LBP_PO = 15, /* postfix (numeric) */
  LBP_PR = 16, /* prefix (numeric) */
  LBP_SY = 17, /* symbols allowing breaks */
  LBP_AI = 35, /* ambiguous (alphabetic or ideograph) */
  LBP_AL = 18, /* ordinary alphabetic and symbol characters */
/*LBP_CJ,         conditional Japanese starters, resolved to NS */
  LBP_H2 = 19, /* Hangul LV syllable */
  LBP_H3 = 20, /* Hangul LVT syllable */
  LBP_HL = 25, /* Hebrew letter */
  LBP_ID = 21, /* ideographic */
  LBP_JL = 22, /* Hangul L Jamo */
  LBP_JV = 23, /* Hangul V Jamo */
  LBP_JT = 24, /* Hangul T Jamo */
  LBP_RI = 26, /* regional indicator */
  LBP_SA = 36, /* complex context (South East Asian) */
  LBP_ZWJ = 27, /* zero width joiner */
  LBP_EB = 28, /* emoji base */
  LBP_EM = 29, /* emoji modifier */
  LBP_XX = 37  /* unknown */
};

#include "lbrkprop1.h"

static inline unsigned char
unilbrkprop_lookup (ucs4_t uc)
{
  unsigned int index1 = uc >> lbrkprop_header_0;
  if (index1 < lbrkprop_header_1)
    {
      int lookup1 = unilbrkprop.level1[index1];
      if (lookup1 >= 0)
        {
          unsigned int index2 = (uc >> lbrkprop_header_2) & lbrkprop_header_3;
          int lookup2 = unilbrkprop.level2[lookup1 + index2];
          if (lookup2 >= 0)
            {
              unsigned int index3 = uc & lbrkprop_header_4;
              return unilbrkprop.level3[lookup2 + index3];
            }
        }
    }
  return LBP_XX;
}

/* Table indexed by two line breaking classifications.  */
#define D 1  /* direct break opportunity, empty in table 7.3 of UTR #14 */
#define I 2  /* indirect break opportunity, '%' in table 7.3 of UTR #14 */
#define P 3  /* prohibited break,           '^' in table 7.3 of UTR #14 */

extern const unsigned char unilbrk_table[30][30];

/* We don't support line breaking of complex-context dependent characters
   (Thai, Lao, Myanmar, Khmer) yet, because it requires dictionary lookup. */