Mercurial > gnulib
view tests/uniwbrk/test-uc-wordbreaks.c @ 19273:638b6d1fdf36 ueno/unicode-9.0.0
libunistring: update to Unicode 9.0.0
* lib/gen-uni-tables.c (fill_properties): Recognize Sentence_Terminal
and Prepended_Concatenation_Mark.
(is_property_default_ignorable_code_point): Exclude U+08E2.
(fill_arabicshaping): Allow missing whitespace when parsing;
recognize "AFRICAN FEH", "AFRICAN QAF", and "AFRICAN MOON".
(output_blocks): Increase the element size of the level1 table to
accommodate more blocks.
(get_lbp): Recognize ZWJ, E_Base, and E_Modifier characters;
Update each class according to the standard.
(get_wbp): Recognize ZWJ, E_Base, E_Modifier, Glue_After_Zwj, and
E_Base_GAZ characters.
(output_gbp_table): Recognize ZWJ, E_Base, E_Modifier, Glue_After_Zwj,
and E_Base_GAZ characters.
* lib/unictype.in.h (UC_JOINING_GROUP_AFRICAN_FEH,
UC_JOINING_GROUP_AFRICAN_QAF, UC_JOINING_GROUP_AFRICAN_MOON): New enum value.
* lib/unilbrk/lbrktables.h (LBP_ZWJ, LBP_EB, LBP_EM): New enum value.
* lib/unilbrk/lbrktables.c (unilbrk_table): Extend the table with
LBP_ZWJ, LBP_EB, and LBP_EM.
* lib/uniwbrk.in.h (WBP_ZWJ, WBP_EB, WBP_EM, WBP_GAZ, WBP_EBG): New
enum value.
* lib/uniwbrk/u-wordbreaks.h: Implement WB3c, WB15, and WB16.
* lib/uniwbrk/wbrktable.h (uniwbrk_prop_index): New variable declaration.
* lib/uniwbrk/wbrktable.c (uniwbrk_prop_index): New variable.
(uniwbrk_table): Implement WB14.
* tests/uniwbrk/test-uc-wordbreaks.c (wordbreakproperty_to_string):
Check WBP_ZWJ, WBP_EB, WBP_EM, WBP_GAZ, and WBP_EBG.
* modules/unigbrk/u{32,16,8}-grapheme-breaks: No longer depend on
uc-is-grapheme-break.
* modules/unigbrk/uc-grapheme-breaks: New module.
* modules/unigbrk/uc-grapheme-breaks-tests: New module.
* lib/unigbrk.in.h (GBP_ZWJ, GBP_EB, GBP_EM, GBP_GAZ, GBP_EBG): New
enum value.
(uc_grapheme_breaks): New function, replacing uc_is_grapheme_break.
* lib/unigbrk/u-grapheme-breaks.h: New file.
* lib/unigbrk/u{32,16,8}-grapheme-breaks.c: Rewrite using
u-grapheme-breaks.h instead of uc_is_grapheme_break.
* lib/unigbrk/uc-grapheme-breaks.c: New file.
* lib/unigbrk/uc-is-grapheme-break.c: Partially update to TR29 rev 29.
* tests/unigbrk/test-uc-gbrk-prop.c (graphemebreakproperty_to_string):
Check GBP_ZWJ, GBP_EB, GBP_EM, GBP_GAZ, and GBP_EBG.
* tests/unigbrk/test-uc-grapheme-breaks.c: New test.
* tests/unigbrk/test-uc-is-grapheme-break.c (graphemebreakproperty_to_string):
Check GBP_ZWJ, GBP_EB, GBP_EM, GBP_GAZ, and GBP_EBG.
(main): Skip unsupported rules involving 3 or more characters, namely
GB10, GB12, and GB13.
* lib/uniwidth/width.c (nonspacing_table_data): Update.
author | Daiki Ueno <ueno@gnu.org> |
---|---|
date | Wed, 12 Oct 2016 17:40:37 +0200 |
parents | 9759915b2aca |
children | 10eb9086bea0 |
line wrap: on
line source
/* Word break function test, using test data from UCD. Copyright (C) 2010-2017 Free Software Foundation, Inc. This program is free software: you can redistribute it and/or modify it under the terms of the GNU Lesser General Public License as published by the Free Software Foundation; either version 3 of the License, or (at your option) any later version. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more details. You should have received a copy of the GNU Lesser General Public License along with this program. If not, see <https://www.gnu.org/licenses/>. */ /* Written by Daiki Ueno <ueno@gnu.org>, 2014. Largely based on unigbrk/test-uc-is-grapheme-break.c, written by Ben Pfaff <blp@cs.stanford.edu>, 2010. */ #include <config.h> /* Specification. */ #include <uniwbrk.h> #include <stdio.h> #include <stdlib.h> #include <string.h> const char * wordbreakproperty_to_string (int wbp) { switch (wbp) { #define CASE(VALUE) case WBP_##VALUE: return #VALUE; CASE(OTHER) CASE(CR) CASE(LF) CASE(NEWLINE) CASE(EXTEND) CASE(FORMAT) CASE(KATAKANA) CASE(ALETTER) CASE(MIDNUMLET) CASE(MIDLETTER) CASE(MIDNUM) CASE(NUMERIC) CASE(EXTENDNUMLET) CASE(RI) CASE(DQ) CASE(SQ) CASE(HL) CASE(ZWJ) CASE(EB) CASE(EM) CASE(GAZ) CASE(EBG) } abort (); } int main (int argc, char *argv[]) { const char *filename; char line[4096]; int exit_code; FILE *stream; int lineno; if (argc != 2) { fprintf (stderr, "usage: %s FILENAME\n" "where FILENAME is the location of the WordBreakTest.txt\n" "test file.\n", argv[0]); exit (1); } filename = argv[1]; stream = fopen (filename, "r"); if (stream == NULL) { fprintf (stderr, "error during fopen of '%s'\n", filename); exit (1); } exit_code = 0; lineno = 0; while (fgets (line, sizeof line, stream)) { char *comment; const char *p; uint32_t input[100]; char breaks[101]; char breaks_expected[101]; int i; lineno++; memset (breaks, 0, sizeof (breaks)); memset (breaks_expected, 0, sizeof (breaks_expected)); comment = strchr (line, '#'); if (comment != NULL) *comment = '\0'; if (line[strspn (line, " \t\r\n")] == '\0') continue; i = 0; p = line; do { p += strspn (p, " \t\r\n"); if (!strncmp (p, "\303\267" /* ÷ */, 2)) { breaks_expected[i] = 1; p += 2; } else if (!strncmp (p, "\303\227" /* × */, 2)) { breaks_expected[i] = 0; p += 2; } else { fprintf (stderr, "%s:%d.%d: syntax error expecting '÷' or '×'\n", filename, lineno, (int) (p - line + 1)); exit (1); } p += strspn (p, " \t\r\n"); if (*p != '\0') { unsigned int next_int; int n; if (sscanf (p, "%x%n", &next_int, &n) != 1) { fprintf (stderr, "%s:%d.%d: syntax error at '%s' " "expecting hexadecimal Unicode code point number\n", filename, lineno, (int) (p - line + 1), p); exit (1); } p += n; input[i] = next_int; } p += strspn (p, " \t\r\n"); i++; } while (*p != '\0'); u32_wordbreaks (input, i - 1, breaks); /* u32_wordbreaks always set BREAKS[0] to 0. */ breaks[0] = breaks_expected[0] = 1; if (memcmp (breaks, breaks_expected, i - 1) != 0) { int j; fprintf (stderr, "%s:%d: expected: ", filename, lineno); for (j = 0; j < i - 1; j++) { int input_wbp = uc_wordbreak_property (input[j]); fprintf (stderr, "%s U+%04X (%s) ", breaks_expected[j] == 1 ? "\303\267" : "\303\227", input[j], wordbreakproperty_to_string (input_wbp)); } fprintf (stderr, "\n"); fprintf (stderr, "%s:%d: actual: ", filename, lineno); for (j = 0; j < i - 1; j++) { int input_wbp = uc_wordbreak_property (input[j]); fprintf (stderr, "%s U+%04X (%s) ", breaks[j] == 1 ? "\303\267" : "\303\227", input[j], wordbreakproperty_to_string (input_wbp)); } fprintf (stderr, "\n"); exit_code = 1; } } return exit_code; }