Google GitGoogle Git
Sign in
third-party-mirror/SchedMD/slurm/40d92e7cd3d0fd1fbf7dc9a0de70c50cbb12f4ce/./testsuite/python/check/common/test_xutf.c
blob: 0e063e451a1ca8581bf0101283c8e965f627f218 [file]
/******************************************************************************
* Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
*
* Test the xutf UTF-8 handler (src/common/xutf.c) directly.
*
* The cases pin the handler against The Unicode Standard, Version 15.0:
* - Table 3-7 "Well-Formed UTF-8 Byte Sequences" (the canonical accept/reject
* specification for UTF-8 byte sequences),
* - D14 noncharacters, D49 private-use, D71-D73 surrogates, D92 UTF-8 (the
* shortest-form / no-overlong requirement),
* - the U+0000..U+10FFFF codespace bounds,
* - the BOM signatures of Table 2-4,
* - the newline (5.8), space (Table 6-2) and C0/C1 control sets.
*****************************************************************************/
#include <check.h>
#include <inttypes.h>
#include <stdint.h>
#include <stdlib.h>
#include <string.h>
#include "slurm/slurm_errno.h"
#include "src/common/log.h"
#include "src/common/read_config.h"
#include "src/common/xmalloc.h"
#include "src/common/xstring.h"
#include "src/common/xutf.h"
/*
* Build a const utf8_t[] compound literal inline so a byte sequence can be
* written as RB(0xED, 0xA0, 0x80). The literal has automatic storage tied to
* the enclosing block, so it stays valid for the helper call it is passed to.
*/
#define RB(...) ((const utf8_t[]) { __VA_ARGS__ })
/* ------------------------------------------------------------------ helpers */
/* assert that src[0..len) decodes to exactly exp_code in exp_bytes bytes */
static void assert_read(const char *desc, const utf8_t *src, size_t len,
bool check_valid, utf_code_t exp_code, int exp_bytes)
{
utf_code_t utf = 0;
int bytes = 0;
int rc = utf8_read_character(src, src + len, &utf, &bytes, check_valid);
ck_assert_msg(rc == SLURM_SUCCESS,
"%s: expected success, got rc=%d (%s)", desc, rc,
slurm_strerror(rc));
ck_assert_msg(utf == exp_code,
"%s: expected U+%06" PRIX32 ", got U+%06" PRIX32, desc,
exp_code, utf);
ck_assert_msg(bytes == exp_bytes, "%s: expected %d bytes, got %d", desc,
exp_bytes, bytes);
}
/*
* assert that src[0..len) is rejected. If exp_rc is non-zero, the exact error
* code must match. On rejection the handler sets *utf to UTF_REPLACEMENT_CODE
* and *bytes to the resync count, which must equal exp_bytes exactly.
*
* exp_bytes is how far a scanning caller advances past an ill-formed sequence.
* An off-by-one there silently swallows the character that follows it.
*/
static void assert_not_read(const char *desc, const utf8_t *src, size_t len,
bool check_valid, int exp_rc, int exp_bytes)
{
utf_code_t utf = 0;
int bytes = 0;
int rc = utf8_read_character(src, src + len, &utf, &bytes, check_valid);
ck_assert_msg(rc != SLURM_SUCCESS,
"%s: expected rejection, got success (U+%06" PRIX32 ")",
desc, utf);
ck_assert_msg(rc == exp_rc, "%s: expected rc=%d (%s), got rc=%d (%s)",
desc, exp_rc, slurm_strerror(exp_rc), rc,
slurm_strerror(rc));
ck_assert_msg(utf == UTF_REPLACEMENT_CODE,
"%s: utf not U+FFFD on error (got U+%06" PRIX32 ")", desc,
utf);
ck_assert_msg(bytes == exp_bytes,
"%s: expected resync of %d bytes, got %d", desc,
exp_bytes, bytes);
/* a resync must always make progress and stay inside the buffer */
ck_assert_msg((bytes >= 1) && (bytes <= (int) len),
"%s: resync count out of range (got %d, len %zu)", desc,
bytes, len);
}
/* one code unit produced by the read-and-advance loop */
typedef struct {
utf_code_t utf; /* decoded scalar, or UTF_REPLACEMENT_CODE */
int bytes; /* bytes consumed producing it */
} utf_step_t;
/*
* Feed src[0..len) through the read-and-advance loop that callers scanning a
* whole string use, and assert the exact sequence of (code, advance) pairs it
* produces against exp[0..exp_cnt).
*
* The advance decides how many U+FFFD a scanner emits and which byte it resumes
* on. A loop that over-advances drops the valid character following an
* ill-formed sequence without raising any error.
*
* NOTE: several of these expectations deliberately differ from Unicode 15.0 3.9
* "U+FFFD Substitution of Maximal Subparts". The reader accepts any 0x80..0xBF
* continuation byte rather than applying Table 3-7's per-lead second-byte
* ranges, so it rejects by code value afterwards and emits one U+FFFD for the
* whole sequence where 3.9 calls for one per byte (E0 80 80, ED A0 80,
* F0 80 80 80, F4 90 80 80), and conversely splits E0 A0 into two where 3.9
* gives one. These pin the handler as it behaves today; if the reader is ever
* made Table 3-7 conformant, these are the expectations to update.
*/
static void assert_read_sequence(const char *desc, const utf8_t *src,
size_t len, const utf_step_t *exp,
size_t exp_cnt)
{
const utf8_t *ptr = src;
const utf8_t *end = src + len;
size_t i = 0;
while (ptr < end) {
utf_code_t utf = 0;
int bytes = 0;
int rc = utf8_read_character(ptr, end, &utf, &bytes, true);
ck_assert_msg(i < exp_cnt,
"%s: produced more than the %zu expected units",
desc, exp_cnt);
ck_assert_msg((bytes >= 1) && (bytes <= (end - ptr)),
"%s: advance %d out of range (%ld remaining)",
desc, bytes, (long) (end - ptr));
ck_assert_msg(utf == exp[i].utf,
"%s: unit %zu expected U+%06" PRIX32
", got U+%06" PRIX32,
desc, i, exp[i].utf, utf);
ck_assert_msg(bytes == exp[i].bytes,
"%s: unit %zu expected advance %d, got %d", desc,
i, exp[i].bytes, bytes);
if (rc != SLURM_SUCCESS)
ck_assert_msg(utf == UTF_REPLACEMENT_CODE,
"%s: gave U+%06" PRIX32 ", not U+FFFD",
desc, utf);
ptr += bytes;
i++;
}
ck_assert_msg(i == exp_cnt, "%s: expected %zu units, got %zu", desc,
exp_cnt, i);
ck_assert_msg(ptr == end, "%s: buffer not fully consumed", desc);
}
/* call assert_read_sequence() with an inline expectation array */
#define ASSERT_SEQ(desc, src, len, ...) \
do { \
const utf_step_t _exp[] = { __VA_ARGS__ }; \
assert_read_sequence((desc), (src), (len), _exp, \
ARRAY_SIZE(_exp)); \
} while (false)
/* assert that utf encodes to exactly the exp[0..exp_len) byte sequence */
static void assert_write(const char *desc, utf_code_t utf, const utf8_t *exp,
int exp_len)
{
utf8_t dst[UTF8_CHAR_MAX_BYTES] = { 0 };
int bytes = 0;
/*
* The public macro is what callers use; it handles ASCII inline without
* entering the function. test_write_ascii_direct() covers the
* function's own ASCII branch.
*/
int rc = utf8_write_character(utf, dst, &bytes);
ck_assert_msg(rc == SLURM_SUCCESS, "%s: write rc=%d (%s)", desc, rc,
slurm_strerror(rc));
ck_assert_msg(bytes == exp_len, "%s: expected %d bytes, got %d", desc,
exp_len, bytes);
ck_assert_msg(!memcmp(dst, exp, exp_len),
"%s: encoded bytes do not match expected", desc);
}
static void setup(void)
{
log_options_t log_opts = LOG_OPTS_INITIALIZER;
const char *debug_env = getenv("SLURM_DEBUG");
const char *debug_flags_env = getenv("SLURM_DEBUG_FLAGS");
/*
* xutf's LOG() is gated on DEBUG_FLAG_DATA plus debug5, and that trace
* is the practical way to find which of the 1114111 code points
* test_codespace_roundtrip failed on.
*/
if (debug_env)
log_opts.stderr_level = log_string2num(debug_env);
if (debug_flags_env)
debug_str2flags(debug_flags_env, &slurm_conf.debug_flags);
log_init("xutf-test", log_opts, 0, NULL);
}
static void teardown(void)
{
log_fini();
}
/* ------------------------------ slurm_read_utf8_character: Table 3-7 */
/* Table 3-7 row 1: U+0000..U+007F (single ASCII byte) */
START_TEST(test_read_ascii)
{
assert_read("U+0001", RB(0x01), 1, true, 0x01, 1);
assert_read("U+0041 'A'", RB(0x41), 1, true, 0x41, 1);
assert_read("U+007F boundary", RB(0x7F), 1, true, 0x7F, 1);
/* U+0000 is rejected by policy (check_valid only) */
assert_not_read("U+0000 NUL", RB(0x00), 1, true, ESLURM_UTF_NULL_CODE,
1);
}
END_TEST
/* Table 3-7 row 2: U+0080..U+07FF, lead C2..DF */
START_TEST(test_read_two_byte)
{
assert_read("U+0080 low boundary", RB(0xC2, 0x80), 2, true, 0x0080, 2);
assert_read("U+00F1 'n-tilde'", RB(0xC3, 0xB1), 2, true, 0x00F1, 2);
assert_read("U+07FF high boundary", RB(0xDF, 0xBF), 2, true, 0x07FF, 2);
}
END_TEST
/* Table 3-7 rows 3-6: U+0800..U+FFFF, leads E0,E1..EC,ED,EE..EF */
START_TEST(test_read_three_byte)
{
assert_read("U+0800 row-3 low", RB(0xE0, 0xA0, 0x80), 3, true, 0x0800,
3);
assert_read("U+0FFF row-3 high", RB(0xE0, 0xBF, 0xBF), 3, true, 0x0FFF,
3);
assert_read("U+1000 row-4 low", RB(0xE1, 0x80, 0x80), 3, true, 0x1000,
3);
assert_read("U+20AC euro", RB(0xE2, 0x82, 0xAC), 3, true, 0x20AC, 3);
assert_read("U+CFFF row-4 high", RB(0xEC, 0xBF, 0xBF), 3, true, 0xCFFF,
3);
assert_read("U+D000 row-5 low", RB(0xED, 0x80, 0x80), 3, true, 0xD000,
3);
assert_read("U+D7FF row-5 high", RB(0xED, 0x9F, 0xBF), 3, true, 0xD7FF,
3);
assert_read("U+FFFD replacement", RB(0xEF, 0xBF, 0xBD), 3, true, 0xFFFD,
3);
}
END_TEST
/* Table 3-7 rows 7-9: U+10000..U+10FFFF, leads F0,F1..F3,F4 */
START_TEST(test_read_four_byte)
{
assert_read("U+10000 row-7 low", RB(0xF0, 0x90, 0x80, 0x80), 4, true,
0x10000, 4);
assert_read("U+1F600 emoji", RB(0xF0, 0x9F, 0x98, 0x80), 4, true,
0x1F600, 4);
assert_read("U+40000 row-8 low", RB(0xF1, 0x80, 0x80, 0x80), 4, true,
0x40000, 4);
assert_read("U+80000 row-8 mid", RB(0xF2, 0x80, 0x80, 0x80), 4, true,
0x80000, 4);
assert_read("U+E0100 plane-14", RB(0xF3, 0xA0, 0x84, 0x80), 4, true,
0xE0100, 4);
/* U+100000..U+10FFFF (row 9, lead F4) is all private-use or
* noncharacters, so there is no valid plane-16 scalar to accept here;
* its boundaries are exercised by the private/noncharacter cases. */
}
END_TEST
/*
* D92 / Table 3-7: a code point encoded in more bytes than the minimum is
* ill-formed (overlong). The C0 and C1 lead bytes can only ever introduce an
* overlong two-byte form, so the handler rejects them through the overlong
* check (ESLURM_UTF8_INVALID_READ), not as structurally illegal leads.
*/
START_TEST(test_read_overlong)
{
assert_not_read("overlong U+0000 (C0 80)", RB(0xC0, 0x80), 2, true,
ESLURM_UTF8_INVALID_READ, 2);
assert_not_read("overlong U+007F (C1 BF)", RB(0xC1, 0xBF), 2, true,
ESLURM_UTF8_INVALID_READ, 2);
assert_not_read("overlong U+0000 (E0 80 80)", RB(0xE0, 0x80, 0x80), 3,
true, ESLURM_UTF8_INVALID_READ, 3);
assert_not_read("overlong U+07FF (E0 9F BF)", RB(0xE0, 0x9F, 0xBF), 3,
true, ESLURM_UTF8_INVALID_READ, 3);
assert_not_read("overlong U+0000 (F0 80 80 80)",
RB(0xF0, 0x80, 0x80, 0x80), 4, true,
ESLURM_UTF8_INVALID_READ, 4);
assert_not_read("overlong U+FFFF (F0 8F BF BF)",
RB(0xF0, 0x8F, 0xBF, 0xBF), 4, true,
ESLURM_UTF8_INVALID_READ, 4);
}
END_TEST
/*
* D71-D73: surrogate code points U+D800..U+DFFF are ill-formed in UTF-8 and are
* rejected by utf_is_valid() when check_valid is set.
*/
START_TEST(test_read_surrogate_bytes)
{
assert_not_read("U+D800 surrogate", RB(0xED, 0xA0, 0x80), 3, true,
ESLURM_UTF16_SURROGATE_CODE, 3);
assert_not_read("U+DFFF surrogate", RB(0xED, 0xBF, 0xBF), 3, true,
ESLURM_UTF16_SURROGATE_CODE, 3);
}
END_TEST
/* Codespace bound: U+10FFFF is the maximum; anything above is ill-formed. */
START_TEST(test_read_above_max)
{
assert_not_read("U+110000 (F4 90 80 80)", RB(0xF4, 0x90, 0x80, 0x80), 4,
true, ESLURM_UTF_INVALID_CODE, 4);
}
END_TEST
/*
* Table 3-7 lists no row whose lead byte is F5..FF: those bytes can never begin
* a sequence. This handler splits that set. F8..FF are refused structurally as
* bad leads (test_read_illegal_lead), but the 4-byte branch masks with
* (*src & 0xf8) == 0xf0, which matches F0..F7 -- so F5, F6 and F7 decode as
* 4-byte leads, yielding U+140000..U+1FFFFF, and are caught only afterwards by
* the codespace bound inside utf_is_valid().
*
* That makes F5..F7 the only Table 3-7 structural violation gated behind
* check_valid; overlongs, bad continuations, truncation and F8..FF are all
* unconditional. With check_valid=false these decode successfully to
* out-of-codespace scalars (test_read_no_check_valid).
*
* If the reader is made Table 3-7 conformant, the expectation becomes
* ESLURM_UTF8_INVALID_READ with a 1-byte resync.
*/
START_TEST(test_read_out_of_range_lead)
{
assert_not_read("U+140000 (F5 80 80 80)", RB(0xF5, 0x80, 0x80, 0x80), 4,
true, ESLURM_UTF_INVALID_CODE, 4);
assert_not_read("U+180000 (F6 80 80 80)", RB(0xF6, 0x80, 0x80, 0x80), 4,
true, ESLURM_UTF_INVALID_CODE, 4);
assert_not_read("U+1FFFFF (F7 BF BF BF)", RB(0xF7, 0xBF, 0xBF, 0xBF), 4,
true, ESLURM_UTF_INVALID_CODE, 4);
/* a lone F5..F7 has no room for the 4-byte form it claims */
assert_not_read("truncated lead (F5)", RB(0xF5), 1, true,
ESLURM_UTF8_READ_ILLEGAL_TERMINATION, 1);
assert_not_read("truncated lead (F7)", RB(0xF7), 1, true,
ESLURM_UTF8_READ_ILLEGAL_TERMINATION, 1);
}
END_TEST
/* Table 3-7: bytes that are never valid leads, and stray continuation bytes. */
START_TEST(test_read_illegal_lead)
{
/* every continuation byte 0x80..0xBF is ill-formed as a lead */
for (int b = 0x80; b <= 0xBF; b++) {
utf8_t seq[1] = { b };
char desc[32];
snprintf(desc, sizeof(desc), "stray continuation 0x%02X", b);
assert_not_read(desc, seq, 1, true, ESLURM_UTF8_INVALID_READ,
1);
}
/*
* F8..FF can never begin a UTF-8 sequence. F5..F7 are equally illegal
* leads per Table 3-7 but are not refused here; see
* test_read_out_of_range_lead().
*/
for (int b = 0xF8; b <= 0xFF; b++) {
utf8_t seq[1] = { b };
char desc[32];
snprintf(desc, sizeof(desc), "illegal lead 0x%02X", b);
assert_not_read(desc, seq, 1, true, ESLURM_UTF8_INVALID_READ,
1);
}
}
END_TEST
/*
* Table 3-7 / D92: a multibyte sequence that runs past the end of the buffer is
* an illegal termination; a non-continuation in byte 2/3/4 is an invalid byte.
*/
START_TEST(test_read_truncated)
{
assert_not_read("2-byte truncated (C3)", RB(0xC3), 1, true,
ESLURM_UTF8_READ_ILLEGAL_TERMINATION, 1);
assert_not_read("3-byte truncated (E2 82)", RB(0xE2, 0x82), 2, true,
ESLURM_UTF8_READ_ILLEGAL_TERMINATION, 1);
assert_not_read("4-byte truncated (F0 9F 98)", RB(0xF0, 0x9F, 0x98), 3,
true, ESLURM_UTF8_READ_ILLEGAL_TERMINATION, 1);
assert_not_read("bad byte 2 (C2 41)", RB(0xC2, 0x41), 2, true,
ESLURM_UTF8_INVALID_BYTE_2, 1);
assert_not_read("bad byte 2 (E2 41 82)", RB(0xE2, 0x41, 0x82), 3, true,
ESLURM_UTF8_INVALID_BYTE_2, 1);
assert_not_read("bad byte 2 (F0 41 80 80)", RB(0xF0, 0x41, 0x80, 0x80),
4, true, ESLURM_UTF8_INVALID_BYTE_2, 1);
assert_not_read("bad byte 3 (E2 82 41)", RB(0xE2, 0x82, 0x41), 3, true,
ESLURM_UTF8_INVALID_BYTE_3, 2);
assert_not_read("bad byte 3 (F0 9F 41 80)", RB(0xF0, 0x9F, 0x41, 0x80),
4, true, ESLURM_UTF8_INVALID_BYTE_3, 2);
assert_not_read("bad byte 4 (F0 9F 98 41)", RB(0xF0, 0x9F, 0x98, 0x41),
4, true, ESLURM_UTF8_INVALID_BYTE_4, 3);
}
END_TEST
/*
* Ill-formed sequences must resync safely: every code unit produced and every
* byte advance is pinned exactly. See assert_read_sequence() for where these
* expectations knowingly differ from Unicode 15.0 3.9.
*/
START_TEST(test_read_maximal_subpart)
{
/* out-of-range 2nd byte / overlong / surrogate / above-max / bad lead */
ASSERT_SEQ("overlong E0 80 80", RB(0xE0, 0x80, 0x80), 3,
{ UTF_REPLACEMENT_CODE, 3 });
ASSERT_SEQ("overlong E0 9F BF", RB(0xE0, 0x9F, 0xBF), 3,
{ UTF_REPLACEMENT_CODE, 3 });
ASSERT_SEQ("surrogate ED A0 80", RB(0xED, 0xA0, 0x80), 3,
{ UTF_REPLACEMENT_CODE, 3 });
ASSERT_SEQ("overlong F0 80 80 80", RB(0xF0, 0x80, 0x80, 0x80), 4,
{ UTF_REPLACEMENT_CODE, 4 });
ASSERT_SEQ("above max F4 90 80 80", RB(0xF4, 0x90, 0x80, 0x80), 4,
{ UTF_REPLACEMENT_CODE, 4 });
ASSERT_SEQ("overlong C0 AF", RB(0xC0, 0xAF), 2,
{ UTF_REPLACEMENT_CODE, 2 });
ASSERT_SEQ("bad lead F5 80 80 80", RB(0xF5, 0x80, 0x80, 0x80), 4,
{ UTF_REPLACEMENT_CODE, 4 });
ASSERT_SEQ("lone continuation 80", RB(0x80), 1,
{ UTF_REPLACEMENT_CODE, 1 });
/*
* Bad continuation byte followed by decodable ASCII. Each sequence is
* long enough to reach the continuation-byte check it is named for; a
* shorter buffer trips the length guard first and lands in the
* truncation cases below instead.
*/
ASSERT_SEQ("bad 2nd C2 41", RB(0xC2, 0x41), 2,
{ UTF_REPLACEMENT_CODE, 1 }, { 0x41, 1 });
ASSERT_SEQ("bad 2nd E0 41 42", RB(0xE0, 0x41, 0x42), 3,
{ UTF_REPLACEMENT_CODE, 1 }, { 0x41, 1 }, { 0x42, 1 });
ASSERT_SEQ("bad 2nd F0 41 42 43", RB(0xF0, 0x41, 0x42, 0x43), 4,
{ UTF_REPLACEMENT_CODE, 1 }, { 0x41, 1 }, { 0x42, 1 },
{ 0x43, 1 });
ASSERT_SEQ("bad 3rd E2 82 41", RB(0xE2, 0x82, 0x41), 3,
{ UTF_REPLACEMENT_CODE, 2 }, { 0x41, 1 });
ASSERT_SEQ("bad 3rd F0 90 41 42", RB(0xF0, 0x90, 0x41, 0x42), 4,
{ UTF_REPLACEMENT_CODE, 2 }, { 0x41, 1 }, { 0x42, 1 });
ASSERT_SEQ("bad 4th F0 90 80 41", RB(0xF0, 0x90, 0x80, 0x41), 4,
{ UTF_REPLACEMENT_CODE, 3 }, { 0x41, 1 });
/* sequences truncated at the end of the buffer */
ASSERT_SEQ("truncated E0", RB(0xE0), 1, { UTF_REPLACEMENT_CODE, 1 });
ASSERT_SEQ("truncated E0 A0", RB(0xE0, 0xA0), 2,
{ UTF_REPLACEMENT_CODE, 1 }, { UTF_REPLACEMENT_CODE, 1 });
ASSERT_SEQ("truncated F0 90", RB(0xF0, 0x90), 2,
{ UTF_REPLACEMENT_CODE, 1 }, { UTF_REPLACEMENT_CODE, 1 });
ASSERT_SEQ("truncated F0 90 80", RB(0xF0, 0x90, 0x80), 3,
{ UTF_REPLACEMENT_CODE, 1 }, { UTF_REPLACEMENT_CODE, 1 },
{ UTF_REPLACEMENT_CODE, 1 });
/* mixed valid and ill-formed, exercising resync across boundaries */
ASSERT_SEQ("mixed valid/ill-formed",
RB(0x61, 0xF1, 0x80, 0x80, 0xE1, 0x80, 0xC2, 0x62, 0x80,
0x63, 0x80, 0xBF, 0x64),
13, { 0x61, 1 }, { UTF_REPLACEMENT_CODE, 3 },
{ UTF_REPLACEMENT_CODE, 2 }, { UTF_REPLACEMENT_CODE, 1 },
{ 0x62, 1 }, { UTF_REPLACEMENT_CODE, 1 }, { 0x63, 1 },
{ UTF_REPLACEMENT_CODE, 1 }, { UTF_REPLACEMENT_CODE, 1 },
{ 0x64, 1 });
}
END_TEST
/*
* With check_valid=true the decoder also applies utf_is_valid() policy: it
* rejects noncharacters, private-use and reserved scalars even though their
* byte sequences are structurally well-formed.
*/
START_TEST(test_read_policy_rejects)
{
assert_not_read("U+E000 private (EE 80 80)", RB(0xEE, 0x80, 0x80), 3,
true, ESLURM_UTF_PRIVATE_CODE, 3);
assert_not_read("U+FFFE noncharacter (EF BF BE)", RB(0xEF, 0xBF, 0xBE),
3, true, ESLURM_UTF_NONCHARACTER_CODE, 3);
assert_not_read("U+FFFF noncharacter (EF BF BF)", RB(0xEF, 0xBF, 0xBF),
3, true, ESLURM_UTF_NONCHARACTER_CODE, 3);
assert_not_read("U+FFF0 reserved (EF BF B0)", RB(0xEF, 0xBF, 0xB0), 3,
true, ESLURM_UTF_RESERVED_CODE, 3);
assert_not_read("U+10FFFF noncharacter (F4 8F BF BF)",
RB(0xF4, 0x8F, 0xBF, 0xBF), 4, true,
ESLURM_UTF_NONCHARACTER_CODE, 4);
assert_not_read("U+100000 private (F4 80 80 80)",
RB(0xF4, 0x80, 0x80, 0x80), 4, true,
ESLURM_UTF_PRIVATE_CODE, 4);
}
END_TEST
/*
* check_valid=false skips the utf_is_valid() policy and nothing else, and
* nothing asserts on that path, so the mode returns whatever the byte pattern
* decodes to -- including scalars the standard forbids.
*
* The reader has no structural surrogate or above-max check at all; it rejects
* those solely via the policy call. So the mode is only observable in what it
* ACCEPTS, which is what the second block below pins.
*/
START_TEST(test_read_no_check_valid)
{
assert_read("U+0041 (no check)", RB(0x41), 1, false, 0x41, 1);
assert_read("U+00F1 (no check)", RB(0xC3, 0xB1), 2, false, 0x00F1, 2);
assert_read("U+20AC (no check)", RB(0xE2, 0x82, 0xAC), 3, false, 0x20AC,
3);
assert_read("U+1F600 (no check)", RB(0xF0, 0x9F, 0x98, 0x80), 4, false,
0x1F600, 4);
/*
* Policy-invalid but structurally parsable input passes straight
* through. Each of these is rejected when check_valid is set.
*/
assert_read("U+0000 NUL (no check)", RB(0x00), 1, false, 0x0, 1);
assert_read("U+D800 surrogate (no check)", RB(0xED, 0xA0, 0x80), 3,
false, 0xD800, 3);
assert_read("U+E000 private (no check)", RB(0xEE, 0x80, 0x80), 3, false,
0xE000, 3);
assert_read("U+FFFF noncharacter (no check)", RB(0xEF, 0xBF, 0xBF), 3,
false, 0xFFFF, 3);
assert_read("U+110000 above max (no check)", RB(0xF4, 0x90, 0x80, 0x80),
4, false, 0x110000, 4);
assert_read("U+1FFFFF lead F7 (no check)", RB(0xF7, 0xBF, 0xBF, 0xBF),
4, false, 0x1FFFFF, 4);
/* structural rejects are unconditional and still fire */
assert_not_read("overlong C0 80 (no check)", RB(0xC0, 0x80), 2, false,
ESLURM_UTF8_INVALID_READ, 2);
assert_not_read("stray continuation 80 (no check)", RB(0x80), 1, false,
ESLURM_UTF8_INVALID_READ, 1);
assert_not_read("illegal lead F8 (no check)", RB(0xF8), 1, false,
ESLURM_UTF8_INVALID_READ, 1);
assert_not_read("truncated C3 (no check)", RB(0xC3), 1, false,
ESLURM_UTF8_READ_ILLEGAL_TERMINATION, 1);
assert_not_read("bad byte 2 (C2 41) (no check)", RB(0xC2, 0x41), 2,
false, ESLURM_UTF8_INVALID_BYTE_2, 1);
}
END_TEST
/*
* An empty range satisfies every xassert (src <= end, remaining >= 0) and is
* the state a scanning loop reaches at string termination. It is also the only
* path returning bytes = 0, so a caller advancing on bytes without checking rc
* would spin.
*/
START_TEST(test_read_empty_range)
{
const utf8_t *src = RB(0x41);
utf_code_t utf = 0xDEAD;
int bytes = -1;
ck_assert_int_eq(utf8_read_character(src, src, &utf, &bytes, true),
EINVAL);
ck_assert_int_eq(bytes, 0);
ck_assert_int_eq(utf, 0);
}
END_TEST
/* The utf8_read_character() macro short-circuits the ASCII fast path. */
START_TEST(test_read_macro)
{
const utf8_t *src;
const utf8_t *end;
utf_code_t utf = 0;
int bytes = 0;
src = RB(0x41);
end = src + 1;
ck_assert_int_eq(utf8_read_character(src, end, &utf, &bytes, true),
SLURM_SUCCESS);
ck_assert_int_eq(utf, 0x41);
ck_assert_int_eq(bytes, 1);
/* non-ASCII delegates to slurm_read_utf8_character() */
src = RB(0xE2, 0x82, 0xAC);
end = src + 3;
utf = 0;
bytes = 0;
ck_assert_int_eq(utf8_read_character(src, end, &utf, &bytes, true),
SLURM_SUCCESS);
ck_assert_int_eq(utf, 0x20AC);
ck_assert_int_eq(bytes, 3);
}
END_TEST
/* ----------------------------------------------- slurm_write_utf8_character */
START_TEST(test_write_valid)
{
assert_write("U+0041 'A'", 0x41, RB(0x41), 1);
assert_write("U+0080", 0x80, RB(0xC2, 0x80), 2);
assert_write("U+07FF", 0x7FF, RB(0xDF, 0xBF), 2);
assert_write("U+0800", 0x800, RB(0xE0, 0xA0, 0x80), 3);
assert_write("U+20AC euro", 0x20AC, RB(0xE2, 0x82, 0xAC), 3);
assert_write("U+FFFD replacement", 0xFFFD, RB(0xEF, 0xBF, 0xBD), 3);
assert_write("U+10000", 0x10000, RB(0xF0, 0x90, 0x80, 0x80), 4);
assert_write("U+1F600 emoji", 0x1F600, RB(0xF0, 0x9F, 0x98, 0x80), 4);
assert_write("U+E0100 plane-14", 0xE0100, RB(0xF3, 0xA0, 0x84, 0x80),
4);
}
END_TEST
/* invalid input is replaced with U+FFFD (3 bytes), never emitted raw */
START_TEST(test_write_invalid_to_replacement)
{
const utf8_t rep[] = { 0xEF, 0xBF, 0xBD }; /* U+FFFD */
assert_write("U+0000 -> replacement", 0x0, rep, 3);
assert_write("U+D800 surrogate -> replacement", 0xD800, rep, 3);
assert_write("U+FFFE noncharacter -> replacement", 0xFFFE, rep, 3);
assert_write("U+E000 private -> replacement", 0xE000, rep, 3);
assert_write("U+10FFFF noncharacter -> replacement", 0x10FFFF, rep, 3);
assert_write("U+110000 above max -> replacement", 0x110000, rep, 3);
/*
* utf_code_t is unsigned, so -1 converts to 0xFFFFFFFF at the parameter
* boundary and is rejected for being above U+10FFFF, not for being
* negative.
*/
assert_write("0xFFFFFFFF -> replacement", -1, rep, 3);
}
END_TEST
/*
* The utf8_write_character() macro handles ASCII without entering the function,
* so reach the function's own ASCII branch by calling it directly. Safe despite
* slurm_is_utf_valid() asserting non-ASCII: the function tests validity via the
* utf_is_valid() macro, which short-circuits ASCII and never calls it.
*/
START_TEST(test_write_ascii_direct)
{
utf8_t dst[UTF8_CHAR_MAX_BYTES] = { 0 };
int bytes = 0;
ck_assert_int_eq(slurm_write_utf8_character(0x41, dst, &bytes, true),
SLURM_SUCCESS);
ck_assert_int_eq(bytes, 1);
ck_assert_int_eq(dst[0], 0x41);
ck_assert_int_eq(slurm_write_utf8_character(0x7F, dst, &bytes, true),
SLURM_SUCCESS);
ck_assert_int_eq(bytes, 1);
ck_assert_int_eq(dst[0], 0x7F);
/* log=false must behave identically */
ck_assert_int_eq(slurm_write_utf8_character(0x41, dst, &bytes, false),
SLURM_SUCCESS);
ck_assert_int_eq(bytes, 1);
ck_assert_int_eq(dst[0], 0x41);
}
END_TEST
/* ----------------------------------------------------- utf_is_valid() */
START_TEST(test_is_utf_valid)
{
/* valid scalars across the byte-length ranges */
ck_assert_int_eq(utf_is_valid(0x0001), SLURM_SUCCESS);
ck_assert_int_eq(utf_is_valid(0x007F), SLURM_SUCCESS);
ck_assert_int_eq(utf_is_valid(0x0080), SLURM_SUCCESS);
ck_assert_int_eq(utf_is_valid(0x07FF), SLURM_SUCCESS);
ck_assert_int_eq(utf_is_valid(0x0800), SLURM_SUCCESS);
ck_assert_int_eq(utf_is_valid(0xD7FF), SLURM_SUCCESS);
ck_assert_int_eq(utf_is_valid(0xFDCF), SLURM_SUCCESS); /* below FDD0 */
ck_assert_int_eq(utf_is_valid(0xFDF0), SLURM_SUCCESS); /* above FDEF */
ck_assert_int_eq(utf_is_valid(0xFEFF),
SLURM_SUCCESS); /* BOM is valid */
ck_assert_int_eq(utf_is_valid(0xFFFD), SLURM_SUCCESS);
ck_assert_int_eq(utf_is_valid(0x10000), SLURM_SUCCESS);
ck_assert_int_eq(utf_is_valid(0x1FFFD), SLURM_SUCCESS);
ck_assert_int_eq(utf_is_valid(0xE0100), SLURM_SUCCESS);
/* U+0000 and out-of-range */
ck_assert_int_eq(utf_is_valid(0x0), ESLURM_UTF_NULL_CODE);
/* unsigned utf_code_t: -1 converts to 0xFFFFFFFF, i.e. above U+10FFFF */
ck_assert_int_eq(utf_is_valid(-1), ESLURM_UTF_INVALID_CODE);
ck_assert_int_eq(utf_is_valid(0x110000), ESLURM_UTF_INVALID_CODE);
/* surrogates */
ck_assert_int_eq(utf_is_valid(0xD800), ESLURM_UTF16_SURROGATE_CODE);
ck_assert_int_eq(utf_is_valid(0xDFFF), ESLURM_UTF16_SURROGATE_CODE);
/* noncharacters: U+FDD0..U+FDEF and U+nFFFE/U+nFFFF for every plane */
ck_assert_int_eq(utf_is_valid(0xFDD0), ESLURM_UTF_NONCHARACTER_CODE);
ck_assert_int_eq(utf_is_valid(0xFDEF), ESLURM_UTF_NONCHARACTER_CODE);
ck_assert_int_eq(utf_is_valid(0xFFFE), ESLURM_UTF_NONCHARACTER_CODE);
ck_assert_int_eq(utf_is_valid(0xFFFF), ESLURM_UTF_NONCHARACTER_CODE);
ck_assert_int_eq(utf_is_valid(0x1FFFE), ESLURM_UTF_NONCHARACTER_CODE);
ck_assert_int_eq(utf_is_valid(0x2FFFE), ESLURM_UTF_NONCHARACTER_CODE);
ck_assert_int_eq(utf_is_valid(0xFFFFE), ESLURM_UTF_NONCHARACTER_CODE);
ck_assert_int_eq(utf_is_valid(0x10FFFE), ESLURM_UTF_NONCHARACTER_CODE);
ck_assert_int_eq(utf_is_valid(0x10FFFF), ESLURM_UTF_NONCHARACTER_CODE);
/* private-use: all three D49 ranges */
ck_assert_int_eq(utf_is_valid(0xE000), ESLURM_UTF_PRIVATE_CODE);
ck_assert_int_eq(utf_is_valid(0xF8FF), ESLURM_UTF_PRIVATE_CODE);
ck_assert_int_eq(utf_is_valid(0xF0000), ESLURM_UTF_PRIVATE_CODE);
ck_assert_int_eq(utf_is_valid(0xFFFFD), ESLURM_UTF_PRIVATE_CODE);
ck_assert_int_eq(utf_is_valid(0x100000), ESLURM_UTF_PRIVATE_CODE);
ck_assert_int_eq(utf_is_valid(0x10FFFD), ESLURM_UTF_PRIVATE_CODE);
/* reserved U+FFF0..U+FFF8 */
ck_assert_int_eq(utf_is_valid(0xFFF0), ESLURM_UTF_RESERVED_CODE);
ck_assert_int_eq(utf_is_valid(0xFFF8), ESLURM_UTF_RESERVED_CODE);
}
END_TEST
/* ----------------------------------------------------- utf16_to_coding() */
START_TEST(test_utf16_to_coding)
{
utf_code_t utf = 0;
/* BMP scalar (no low surrogate) */
ck_assert_int_eq(utf16_to_coding(0x0041, 0, &utf), SLURM_SUCCESS);
ck_assert_int_eq(utf, 0x0041);
ck_assert_int_eq(utf16_to_coding(0xD7FF, 0, &utf), SLURM_SUCCESS);
ck_assert_int_eq(utf, 0xD7FF);
/* valid surrogate pairs */
ck_assert_int_eq(utf16_to_coding(0xD834, 0xDD1E, &utf), SLURM_SUCCESS);
ck_assert_int_eq(utf, 0x1D11E); /* G clef */
ck_assert_int_eq(utf16_to_coding(0xD800, 0xDC00, &utf), SLURM_SUCCESS);
ck_assert_int_eq(utf, 0x10000); /* lowest supplementary */
/* lone / mismatched surrogates */
ck_assert_int_eq(utf16_to_coding(0xD800, 0, &utf),
ESLURM_UTF16_SURROGATE_CODE);
ck_assert_int_eq(utf16_to_coding(0xDC00, 0, &utf),
ESLURM_UTF16_SURROGATE_CODE);
ck_assert_int_eq(utf16_to_coding(0x0041, 0xDC00, &utf),
ESLURM_UTF16_SURROGATE_CODE);
ck_assert_int_eq(utf16_to_coding(0xD800, 0x0041, &utf),
ESLURM_UTF16_SURROGATE_CODE);
/*
* Pairs whose low-surrogate offset has its top bit set (v % 0x400 >=
* 0x200); the pairs above all have an offset below 0x200.
*/
ck_assert_int_eq(utf16_to_coding(0xD800, 0xDF00, &utf), SLURM_SUCCESS);
ck_assert_int_eq(utf, 0x10300);
ck_assert_int_eq(utf16_to_coding(0xD83D, 0xDE00, &utf), SLURM_SUCCESS);
ck_assert_int_eq(utf, 0x1F600);
/* a pair that resolves to a policy-rejected scalar */
ck_assert_int_eq(utf16_to_coding(0xDBFF, 0xDFFF, &utf),
ESLURM_UTF_NONCHARACTER_CODE); /* U+10FFFF */
ck_assert_int_eq(utf16_to_coding(0xDB80, 0xDC00, &utf),
ESLURM_UTF_PRIVATE_CODE); /* U+F0000 */
}
END_TEST
/* --------------------------------------------------- utf16_from_coding() */
START_TEST(test_utf16_from_coding)
{
utf16_t high = 0, low = 0;
utf_code_t utf = 0;
/* BMP scalar (no low surrogate) */
ck_assert_int_eq(utf16_from_coding(0x0041, &high, &low), SLURM_SUCCESS);
ck_assert_int_eq(high, 0x0041);
ck_assert_int_eq(low, 0);
ck_assert_int_eq(utf16_from_coding(0xD7FF, &high, &low), SLURM_SUCCESS);
ck_assert_int_eq(high, 0xD7FF);
ck_assert_int_eq(low, 0);
/* supplementary scalars split into surrogate pairs */
ck_assert_int_eq(utf16_from_coding(0x1D11E, &high, &low),
SLURM_SUCCESS);
ck_assert_int_eq(high, 0xD834); /* G clef */
ck_assert_int_eq(low, 0xDD1E);
ck_assert_int_eq(utf16_from_coding(0x10000, &high, &low),
SLURM_SUCCESS);
ck_assert_int_eq(high, 0xD800); /* lowest supplementary */
ck_assert_int_eq(low, 0xDC00);
/*
* Scalars whose offset into the supplementary plane has bit 0x200 set;
* U+1D11E (0x11E) and U+10000 (0x000) both leave it clear.
*/
ck_assert_int_eq(utf16_from_coding(0x10300, &high, &low),
SLURM_SUCCESS);
ck_assert_int_eq(high, 0xD800);
ck_assert_int_eq(low, 0xDF00);
ck_assert_int_eq(utf16_from_coding(0x1F600, &high, &low),
SLURM_SUCCESS);
ck_assert_int_eq(high, 0xD83D);
ck_assert_int_eq(low, 0xDE00);
/* round-trip: from_coding() then to_coding() returns the scalar */
ck_assert_int_eq(utf16_from_coding(0x1D11E, &high, &low),
SLURM_SUCCESS);
ck_assert_int_eq(utf16_to_coding(high, low, &utf), SLURM_SUCCESS);
ck_assert_int_eq(utf, 0x1D11E);
ck_assert_int_eq(utf16_from_coding(0x00E9, &high, &low), SLURM_SUCCESS);
ck_assert_int_eq(utf16_to_coding(high, low, &utf), SLURM_SUCCESS);
ck_assert_int_eq(utf, 0x00E9);
/* invalid scalars are rejected with the utf_is_valid() error */
ck_assert_int_eq(utf16_from_coding(0x0, &high, &low),
ESLURM_UTF_NULL_CODE);
ck_assert_int_eq(utf16_from_coding(0xD800, &high, &low),
ESLURM_UTF16_SURROGATE_CODE);
ck_assert_int_eq(utf16_from_coding(0xE000, &high, &low),
ESLURM_UTF_PRIVATE_CODE);
ck_assert_int_eq(utf16_from_coding(0x10FFFF, &high, &low),
ESLURM_UTF_NONCHARACTER_CODE);
ck_assert_int_eq(utf16_from_coding(0x110000, &high, &low),
ESLURM_UTF_INVALID_CODE);
}
END_TEST
/* --------------------------------------------- is_utf16 surrogate macros */
START_TEST(test_utf16_surrogate_macros)
{
ck_assert(utf16_is_high_surrogate(0xD800));
ck_assert(utf16_is_high_surrogate(0xDBFF));
ck_assert(!utf16_is_high_surrogate(0xD7FF));
ck_assert(!utf16_is_high_surrogate(0xDC00));
ck_assert(utf16_is_low_surrogate(0xDC00));
ck_assert(utf16_is_low_surrogate(0xDFFF));
ck_assert(!utf16_is_low_surrogate(0xDBFF));
ck_assert(!utf16_is_low_surrogate(0xE000));
}
END_TEST
/* ----------------------------------------------- utf_read_encoding_schema() */
START_TEST(test_read_encoding_schema)
{
const utf8_t utf8[] = UTF8_BYTE_ORDER_MARK_SEQ;
const utf8_t utf16be[] = UTF16BE_BYTE_ORDER_MARK_SEQ;
const utf8_t utf16le[] = UTF16LE_BYTE_ORDER_MARK_SEQ;
const utf8_t utf32be[] = UTF32BE_BYTE_ORDER_MARK_SEQ;
const utf8_t utf32le[] = UTF32LE_BYTE_ORDER_MARK_SEQ;
const utf8_t none[] = { 0x41, 0x42, 0x43, 0x44 };
int bytes = -1;
ck_assert_int_eq(utf_read_encoding_schema(utf8, utf8 + sizeof(utf8),
&bytes),
UTF_8_ENCODING);
ck_assert_int_eq(bytes, sizeof(utf8));
ck_assert_int_eq(utf_read_encoding_schema(utf16be,
utf16be + sizeof(utf16be),
&bytes),
UTF_16BE_ENCODING);
ck_assert_int_eq(bytes, sizeof(utf16be));
ck_assert_int_eq(utf_read_encoding_schema(utf16le,
utf16le + sizeof(utf16le),
&bytes),
UTF_16LE_ENCODING);
ck_assert_int_eq(bytes, sizeof(utf16le));
ck_assert_int_eq(utf_read_encoding_schema(utf32be,
utf32be + sizeof(utf32be),
&bytes),
UTF_32BE_ENCODING);
ck_assert_int_eq(bytes, sizeof(utf32be));
/*
* The UTF-32LE BOM (FF FE 00 00) begins with the UTF-16LE BOM (FF FE);
* it must still be detected as UTF-32LE.
*/
ck_assert_int_eq(utf_read_encoding_schema(utf32le,
utf32le + sizeof(utf32le),
&bytes),
UTF_32LE_ENCODING);
ck_assert_int_eq(bytes, sizeof(utf32le));
ck_assert_int_eq(utf_read_encoding_schema(none, none + sizeof(none),
&bytes),
UTF_UNKNOWN_ENCODING);
ck_assert_int_eq(bytes, 0);
/* too short to hold any BOM */
ck_assert_int_eq(utf_read_encoding_schema(utf8, utf8, &bytes),
UTF_UNKNOWN_ENCODING);
ck_assert_int_eq(bytes, 0);
/*
* Buffers holding a proper prefix of a BOM, which exercise the
* sizeof(bom) <= bytes guards that keep each memcmp() inside the buffer.
*/
ck_assert_int_eq(utf_read_encoding_schema(utf8, utf8 + 1, &bytes),
UTF_UNKNOWN_ENCODING); /* EF */
ck_assert_int_eq(bytes, 0);
ck_assert_int_eq(utf_read_encoding_schema(utf8, utf8 + 2, &bytes),
UTF_UNKNOWN_ENCODING); /* EF BB */
ck_assert_int_eq(bytes, 0);
ck_assert_int_eq(utf_read_encoding_schema(utf32be, utf32be + 3, &bytes),
UTF_UNKNOWN_ENCODING); /* 00 00 FE */
ck_assert_int_eq(bytes, 0);
/*
* A 3-byte prefix of the UTF-32LE BOM is too short for UTF-32LE but is
* still a complete UTF-16LE BOM.
*/
ck_assert_int_eq(utf_read_encoding_schema(utf32le, utf32le + 3, &bytes),
UTF_16LE_ENCODING); /* FF FE 00 */
ck_assert_int_eq(bytes, sizeof(utf16le));
}
END_TEST
/* ----------------------------------------------- classification macros */
START_TEST(test_is_utf8_newline)
{
/* Unicode 5.8: CR, LF, VT, FF, NEL, LS, PS */
ck_assert(utf8_is_newline(0x0A)); /* LF */
ck_assert(utf8_is_newline(0x0B)); /* VT */
ck_assert(utf8_is_newline(0x0C)); /* FF */
ck_assert(utf8_is_newline(0x0D)); /* CR */
ck_assert(utf8_is_newline(0x85)); /* NEL */
ck_assert(utf8_is_newline(0x2028)); /* LS */
ck_assert(utf8_is_newline(0x2029)); /* PS */
/*
* Deliberate superset of 5.8: the macro also treats the C0 separators
* FS/GS/RS/US as newlines, so U+001C..U+001F are simultaneously newline
* and control (test_is_utf8_control).
*/
ck_assert(utf8_is_newline(0x1C)); /* FS */
ck_assert(utf8_is_newline(0x1D)); /* GS */
ck_assert(utf8_is_newline(0x1E)); /* RS */
ck_assert(utf8_is_newline(0x1F)); /* US */
ck_assert(!utf8_is_newline(0x41)); /* 'A' */
ck_assert(!utf8_is_newline(0x20)); /* space */
ck_assert(!utf8_is_newline(0x09)); /* tab */
ck_assert(!utf8_is_newline(0x1B)); /* ESC, below the FS..US run */
ck_assert(!utf8_is_newline(0x2027)); /* just below LS */
ck_assert(!utf8_is_newline(0x202A)); /* just above PS */
}
END_TEST
START_TEST(test_is_utf8_space)
{
/* Table 6-2 spaces (a representative span) */
ck_assert(utf8_is_space(0x20)); /* space */
ck_assert(utf8_is_space(0x09)); /* tab (treated as horizontal space) */
ck_assert(utf8_is_space(0xA0)); /* no-break space */
ck_assert(utf8_is_space(0x1680)); /* ogham space mark */
ck_assert(utf8_is_space(0x2000)); /* en quad */
ck_assert(utf8_is_space(0x200A)); /* hair space */
ck_assert(utf8_is_space(0x202F)); /* narrow no-break space */
ck_assert(utf8_is_space(0x205F)); /* medium mathematical space */
ck_assert(utf8_is_space(0x3000)); /* ideographic space */
/* deliberate superset of Table 6-2: these four are not spaces there */
ck_assert(utf8_is_space(0x180E)); /* mongolian vowel separator */
ck_assert(utf8_is_space(0x200B)); /* zero width space */
ck_assert(utf8_is_space(0x2800)); /* braille pattern blank */
ck_assert(utf8_is_space(0xFFA0)); /* halfwidth hangul filler */
ck_assert(!utf8_is_space(0x41));
ck_assert(!utf8_is_space(0x0A)); /* newline is not horizontal space */
ck_assert(!utf8_is_space(0x200C)); /* ZWNJ: control, not space */
ck_assert(!utf8_is_space(0x1FFF)); /* just below the en quad run */
}
END_TEST
START_TEST(test_is_utf8_control)
{
/* C0 controls (minus tab/newlines), DEL, C1 controls */
ck_assert(utf8_is_control(0x00));
ck_assert(utf8_is_control(0x08));
ck_assert(utf8_is_control(0x1F));
ck_assert(utf8_is_control(0x7F)); /* DEL */
ck_assert(utf8_is_control(0x80)); /* C1 */
ck_assert(utf8_is_control(0x9F)); /* C1 */
ck_assert(utf8_is_control(0x200C)); /* zero width non-joiner */
ck_assert(utf8_is_control(0x0E)); /* SO, low end of 0x0E..0x1F */
ck_assert(utf8_is_control(0x1C)); /* FS: control *and* newline */
ck_assert(utf8_is_control(0x200D)); /* zero width joiner */
ck_assert(utf8_is_control(0x2060)); /* word joiner */
ck_assert(utf8_is_control(0x206F)); /* nominal digit shapes */
ck_assert(utf8_is_control(0xFFF9)); /* interlinear annotation anchor */
ck_assert(utf8_is_control(0xFFFB)); /* interlinear annotation term. */
ck_assert(utf8_is_control(0x34F)); /* combining grapheme joiner */
ck_assert(utf8_is_control(0x61C)); /* arabic letter mark */
ck_assert(utf8_is_control(0x115F)); /* hangul jamo block */
ck_assert(utf8_is_control(0x1160)); /* hangul jungseong filler */
ck_assert(utf8_is_control(0x3164)); /* hangul filler */
ck_assert(utf8_is_control(0xE0001)); /* language tag (deprecated) */
ck_assert(!utf8_is_control(0x09)); /* tab */
ck_assert(!utf8_is_control(0x0A)); /* newline */
ck_assert(!utf8_is_control(0x41)); /* 'A' */
ck_assert(!utf8_is_control(0x20)); /* space */
ck_assert(!utf8_is_control(0xA0)); /* NBSP is a space */
ck_assert(!utf8_is_control(0x0D)); /* CR is a newline */
}
END_TEST
START_TEST(test_is_utf8_whitespace)
{
ck_assert(utf8_is_whitespace(0x20)); /* space */
ck_assert(utf8_is_whitespace(0x09)); /* tab */
ck_assert(utf8_is_whitespace(0x0A)); /* newline */
ck_assert(utf8_is_whitespace(0x3000)); /* ideographic space */
ck_assert(utf8_is_whitespace(0x2028)); /* line separator */
ck_assert(utf8_is_whitespace(0x1C)); /* newline half of the union */
ck_assert(utf8_is_whitespace(0x180E)); /* space half of the union */
ck_assert(!utf8_is_whitespace(0x41));
ck_assert(!utf8_is_whitespace(0x00));
ck_assert(!utf8_is_whitespace(0x200C)); /* control, neither half */
}
END_TEST
/* --------------------------------------------------- utf8_get_loggable() */
START_TEST(test_get_utf8_loggable)
{
/* Control Pictures substitutions */
ck_assert_int_eq(utf8_get_loggable(0x00), 0x2400); /* NUL */
ck_assert_int_eq(utf8_get_loggable(0x0A), 0x240A); /* LF */
ck_assert_int_eq(utf8_get_loggable(0x1F), 0x241F); /* US */
ck_assert_int_eq(utf8_get_loggable(0x20), 0x2420); /* SPACE */
ck_assert_int_eq(utf8_get_loggable(0x7F), 0x2421); /* DEL */
ck_assert_int_eq(utf8_get_loggable(0x85), UTF_RETURN_SYMBOL_CODE);
/*
* Bidi controls. An unsubstituted LRO/RLO in a log line reorders
* everything after it, which is the Trojan-Source log-spoofing vector.
* They are also the only table entries past U+0085, so they guard the
* table staying sorted: the lookup breaks as soon as an entry exceeds
* the code, so a reordering drops them to the control fallback below.
*/
ck_assert_int_eq(utf8_get_loggable(0x200E), 0x2AAA); /* LRM */
ck_assert_int_eq(utf8_get_loggable(0x200F), 0x2AAB); /* RLM */
ck_assert_int_eq(utf8_get_loggable(0x202A), 0x2AAA); /* LRE */
ck_assert_int_eq(utf8_get_loggable(0x202B), 0x2AAB); /* RLE */
ck_assert_int_eq(utf8_get_loggable(0x202C), 0x2AA4); /* PDF */
ck_assert_int_eq(utf8_get_loggable(0x202D), 0x2AAA); /* LRO */
ck_assert_int_eq(utf8_get_loggable(0x202E), 0x2AAB); /* RLO */
/*
* The fallbacks for codes with no table entry. These are the only
* producers of their respective constants.
*/
ck_assert_int_eq(utf8_get_loggable(0x2028), UTF_RETURN_SYMBOL_CODE);
ck_assert_int_eq(utf8_get_loggable(0x2029), UTF_RETURN_SYMBOL_CODE);
ck_assert_int_eq(utf8_get_loggable(0x00A0), UTF_SPACE_REPLACEMENT_CODE);
ck_assert_int_eq(utf8_get_loggable(0x3000), UTF_SPACE_REPLACEMENT_CODE);
ck_assert_int_eq(utf8_get_loggable(0x0080),
UTF_CONTROL_REPLACEMENT_CODE);
ck_assert_int_eq(utf8_get_loggable(0x009F),
UTF_CONTROL_REPLACEMENT_CODE);
ck_assert_int_eq(utf8_get_loggable(0x200C),
UTF_CONTROL_REPLACEMENT_CODE);
/* a normal printable scalar is returned unchanged */
ck_assert_int_eq(utf8_get_loggable(0x41), 0x41);
ck_assert_int_eq(utf8_get_loggable(0x20AC), 0x20AC);
/* an invalid scalar that is not in the substitution table -> U+FFFD */
ck_assert_int_eq(utf8_get_loggable(0xD800), UTF_REPLACEMENT_CODE);
ck_assert_int_eq(utf8_get_loggable(0x110000), UTF_REPLACEMENT_CODE);
}
END_TEST
/* -------------------------------------- utf_encoding_scheme_to_string */
START_TEST(test_encoding_scheme_to_string)
{
ck_assert(!xstrcmp(utf_encoding_scheme_to_string(UTF_8_ENCODING),
"UTF-8"));
ck_assert(!xstrcmp(utf_encoding_scheme_to_string(UTF_16BE_ENCODING),
"UTF-16BE"));
ck_assert(!xstrcmp(utf_encoding_scheme_to_string(UTF_16LE_ENCODING),
"UTF-16LE"));
ck_assert(!xstrcmp(utf_encoding_scheme_to_string(UTF_32BE_ENCODING),
"UTF-32BE"));
ck_assert(!xstrcmp(utf_encoding_scheme_to_string(UTF_32LE_ENCODING),
"UTF-32LE"));
ck_assert(!xstrcmp(utf_encoding_scheme_to_string(UTF_UNKNOWN_ENCODING),
"UNKNOWN"));
/* the lookup falls through to xassert(false) if a row goes missing */
ck_assert(!xstrcmp(utf_encoding_scheme_to_string(UTF_INVALID),
"INVALID"));
ck_assert(!xstrcmp(utf_encoding_scheme_to_string(UTF_INVALID_MAX),
"INVALID"));
}
END_TEST
/* --------------------------------------------------- string wrappers */
START_TEST(test_utf8_string_wrappers)
{
/* U+00E9 (2 bytes) followed by 'a': 3 bytes, 2 characters */
const utf8_t src[] = { 0xC3, 0xA9, 'a', 0 };
const utf8_t ascii[] = { 'a', 'b', 'c', 0 };
utf8_t *dup = utf8_dup(src);
utf8_t *ndup = utf8_ndup(src, 2);
utf8_t *split = utf8_ndup(src, 1);
utf8_t *adup = utf8_dup(ascii);
/* all three wrappers count BYTES, not characters */
ck_assert_int_eq(utf8_strlen(src), 3);
ck_assert_int_eq(utf8_strlen(ascii), 3);
ck_assert_ptr_nonnull(dup);
ck_assert_int_eq(utf8_strlen(dup), 3);
ck_assert(!memcmp(dup, src, 4));
/* n counts bytes, so this keeps the whole 2-byte character */
ck_assert_ptr_nonnull(ndup);
ck_assert_int_eq(utf8_strlen(ndup), 2);
ck_assert_int_eq(ndup[0], 0xC3);
ck_assert_int_eq(ndup[1], 0xA9);
/* n does not character-align: this truncates mid-character */
ck_assert_ptr_nonnull(split);
ck_assert_int_eq(utf8_strlen(split), 1);
ck_assert_int_eq(split[0], 0xC3);
ck_assert(adup && !xstrcmp((const char *) adup, "abc"));
xfree(dup);
xfree(ndup);
xfree(split);
xfree(adup);
}
END_TEST
/* --------------------------------------------------- codespace capstone */
/*
* Exhaustive completeness check across the entire U+0001..U+10FFFF codespace:
* every code point is encoded and decoded back. Valid scalars must round-trip
* exactly; invalid scalars must be encoded as the U+FFFD replacement, which
* itself decodes back to U+FFFD.
*/
START_TEST(test_codespace_roundtrip)
{
long valid = 0, invalid = 0;
for (utf_code_t cp = 1; cp <= 0x10FFFF; cp++) {
utf8_t buf[UTF8_CHAR_MAX_BYTES] = { 0 };
int wbytes = 0, rbytes = 0;
utf_code_t rcode = 0;
int vrc = utf_is_valid(cp);
ck_assert_int_eq(utf8_write_character(cp, buf, &wbytes),
SLURM_SUCCESS);
ck_assert_msg(utf8_read_character(buf, buf + wbytes, &rcode,
&rbytes,
true) == SLURM_SUCCESS,
"readback failed for U+%06" PRIX32, cp);
ck_assert_int_eq(rbytes, wbytes);
if (!vrc) {
ck_assert_msg(rcode == cp,
"roundtrip U+%06" PRIX32
" -> U+%06" PRIX32,
cp, rcode);
valid++;
} else {
ck_assert_msg(rcode == UTF_REPLACEMENT_CODE,
"invalid U+%06" PRIX32
" not replaced (got U+%06" PRIX32 ")",
cp, rcode);
invalid++;
}
}
printf("codespace roundtrip: %ld valid, %ld invalid scalars\n", valid,
invalid);
/*
* The round-trip above classifies each code point with the same
* utf_is_valid() the reader uses, so on its own it holds under any
* policy, right or wrong. These totals are what make the sweep an
* assertion about the policy itself, and they are exact:
*
* 2048 surrogates U+D800..U+DFFF
* 66 noncharacters U+FDD0..U+FDEF (32) + U+nFFFE/U+nFFFF
* for planes 0..16 (34)
* 137468 private use U+E000..U+F8FF (6400)
* + U+F0000..U+FFFFD (65534)
* + U+100000..U+10FFFD (65534)
* 9 reserved U+FFF0..U+FFF8
* ------
* 139591 invalid, and 1114111 - 139591 = 974520 valid.
*
* The ranges do not overlap: U+nFFFE/U+nFFFF sit above the end of every
* private-use range, and U+FFF0..U+FFF8 is disjoint from both.
*/
ck_assert_int_eq(invalid, 139591);
ck_assert_int_eq(valid, 974520);
ck_assert_int_eq(valid + invalid, 0x10FFFF);
}
END_TEST
extern int main(int argc, char **argv)
{
int failures;
TCase *tcase = tcase_create("xutf");
Suite *suite = suite_create("xutf");
SRunner *sr = NULL;
tcase_add_unchecked_fixture(tcase, setup, teardown);
tcase_set_timeout(tcase, 120);
tcase_add_test(tcase, test_read_ascii);
tcase_add_test(tcase, test_read_two_byte);
tcase_add_test(tcase, test_read_three_byte);
tcase_add_test(tcase, test_read_four_byte);
tcase_add_test(tcase, test_read_overlong);
tcase_add_test(tcase, test_read_surrogate_bytes);
tcase_add_test(tcase, test_read_above_max);
tcase_add_test(tcase, test_read_out_of_range_lead);
tcase_add_test(tcase, test_read_illegal_lead);
tcase_add_test(tcase, test_read_truncated);
tcase_add_test(tcase, test_read_maximal_subpart);
tcase_add_test(tcase, test_read_policy_rejects);
tcase_add_test(tcase, test_read_no_check_valid);
tcase_add_test(tcase, test_read_empty_range);
tcase_add_test(tcase, test_read_macro);
tcase_add_test(tcase, test_write_valid);
tcase_add_test(tcase, test_write_invalid_to_replacement);
tcase_add_test(tcase, test_write_ascii_direct);
tcase_add_test(tcase, test_is_utf_valid);
tcase_add_test(tcase, test_utf16_to_coding);
tcase_add_test(tcase, test_utf16_from_coding);
tcase_add_test(tcase, test_utf16_surrogate_macros);
tcase_add_test(tcase, test_read_encoding_schema);
tcase_add_test(tcase, test_is_utf8_newline);
tcase_add_test(tcase, test_is_utf8_space);
tcase_add_test(tcase, test_is_utf8_control);
tcase_add_test(tcase, test_is_utf8_whitespace);
tcase_add_test(tcase, test_get_utf8_loggable);
tcase_add_test(tcase, test_encoding_scheme_to_string);
tcase_add_test(tcase, test_utf8_string_wrappers);
tcase_add_test(tcase, test_codespace_roundtrip);
suite_add_tcase(suite, tcase);
sr = srunner_create(suite);
srunner_run_all(sr, CK_VERBOSE);
failures = srunner_ntests_failed(sr);
srunner_free(sr);
return failures;
}