diff options
| -rw-r--r-- | CHANGELOG.md | 6 | ||||
| -rw-r--r-- | CMakeLists.txt | 6 | ||||
| -rw-r--r-- | Dockerfile | 4 | ||||
| -rw-r--r-- | doc/justify.1 | 4 | ||||
| -rw-r--r-- | include/linebuffer.h | 8 | ||||
| -rw-r--r-- | include/utf8.h | 14 | ||||
| -rw-r--r-- | src/linebuffer.c | 27 | ||||
| -rw-r--r-- | src/utf8.c | 74 | ||||
| -rw-r--r-- | tests/CMakeLists.txt | 9 | ||||
| -rw-r--r-- | tests/justify.bats | 39 | ||||
| -rw-r--r-- | tests/test_linebuffer.c | 10 | ||||
| -rw-r--r-- | tests/test_utf8.c | 118 |
12 files changed, 298 insertions, 21 deletions
diff --git a/CHANGELOG.md b/CHANGELOG.md index 1f67106..845932f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,12 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [Unreleased] + +### Added + +- UTF-8 Unicode support for proper text justification with multi-byte characters + ## 0.1.3 - 2025-06-12 ### Changed diff --git a/CMakeLists.txt b/CMakeLists.txt index b03d8e2..ef829da 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -11,14 +11,20 @@ if (CMAKE_C_COMPILER_ID STREQUAL "Clang") add_compile_options(-Wall -Wextra -ansi) endif() +# Find ICU library using pkg-config +find_package(PkgConfig REQUIRED) +pkg_check_modules(ICU REQUIRED IMPORTED_TARGET icu-uc icu-i18n) + set(SRC_FILES src/linebuffer.c src/word.c + src/utf8.c src/main.c ) include_directories(src) add_executable(justify ${SRC_FILES}) +target_link_libraries(justify PkgConfig::ICU) install(TARGETS justify RUNTIME DESTINATION bin) install(FILES doc/justify.1 DESTINATION share/man/man1) @@ -12,7 +12,9 @@ RUN apk update && \ cmake \ valgrind \ bash \ - vim + vim \ + icu-dev \ + icu-libs WORKDIR /app COPY . /app diff --git a/doc/justify.1 b/doc/justify.1 index 1b7a4f6..cba6024 100644 --- a/doc/justify.1 +++ b/doc/justify.1 @@ -44,8 +44,8 @@ An error occurred (e.g., read error). .SH NOTES Before appending a word to the current line, the program ensures there is enough space for the word (and a space if not the first word). If not enough space remains, the current line is flushed (justified) and a new line is started. -.SH LIMITATIONS -The program assumes ASCII text. This means if multi-byte characters are provided, the output would not be justified correctly. +.SH UNICODE SUPPORT +The program supports UTF-8 encoded text and correctly handles multi-byte characters when calculating line widths. Display width is determined using Unicode East Asian Width properties, so full-width and wide characters (such as CJK characters) are counted as taking two columns, while most other characters take one column. .SH AUTHOR Eugene Triguba <hello@eugenetriguba.com> diff --git a/include/linebuffer.h b/include/linebuffer.h index a3d9e4c..b11cb8f 100644 --- a/include/linebuffer.h +++ b/include/linebuffer.h @@ -6,9 +6,11 @@ #include <stdio.h> typedef struct { - size_t _num_words; - size_t _buffer_length; - size_t _buffer_capacity; + size_t _num_words; // Number of words in the buffer + size_t _line_width; // Max display width (e.g., 72 columns) + size_t _buffer_length; // Current bytes used + size_t _display_width; // Current display width + size_t _buffer_capacity; // Byte capacity (line_width * 4 for UTF-8) char *_buffer; } LineBuffer; diff --git a/include/utf8.h b/include/utf8.h new file mode 100644 index 0000000..aa2cac5 --- /dev/null +++ b/include/utf8.h @@ -0,0 +1,14 @@ +#ifndef UTF8_H +#define UTF8_H + +#include <stddef.h> + +/** + * @brief Calculate the display width of a UTF-8 string + * + * @param str The UTF-8 encoded string + * @return The number of characters (display width) + */ +size_t utf8_display_width(const char *str); + +#endif diff --git a/src/linebuffer.c b/src/linebuffer.c index 6773ca1..3fa08b0 100644 --- a/src/linebuffer.c +++ b/src/linebuffer.c @@ -1,4 +1,5 @@ #include "linebuffer.h" +#include "utf8.h" #include <ctype.h> #include <stdbool.h> #include <stdio.h> @@ -6,19 +7,23 @@ #include <string.h> #define NULL_TERMINATOR_LEN 1 +#define UTF8_MAX_BYTES_PER_CHAR 4 -LineBuffer *linebuffer_create(size_t buffer_capacity) { +LineBuffer *linebuffer_create(size_t line_width) { LineBuffer *lb = malloc(sizeof(LineBuffer)); if (lb == NULL) { return NULL; } + size_t buffer_capacity = line_width * UTF8_MAX_BYTES_PER_CHAR; lb->_buffer = malloc(buffer_capacity + NULL_TERMINATOR_LEN); if (lb->_buffer == NULL) { free(lb); return NULL; } lb->_num_words = 0; + lb->_line_width = line_width; lb->_buffer_length = 0; + lb->_display_width = 0; lb->_buffer_capacity = buffer_capacity; return lb; } @@ -31,6 +36,7 @@ void linebuffer_destroy(LineBuffer *lb) { void linebuffer_clear(LineBuffer *lb) { lb->_num_words = 0; lb->_buffer_length = 0; + lb->_display_width = 0; } bool linebuffer_append_word(LineBuffer *lb, const char *word) { @@ -40,25 +46,28 @@ bool linebuffer_append_word(LineBuffer *lb, const char *word) { if (lb->_num_words > 0) { lb->_buffer[lb->_buffer_length] = ' '; lb->_buffer_length++; + lb->_display_width++; } - size_t word_length = strlen(word); - memcpy(lb->_buffer + lb->_buffer_length, word, word_length); - lb->_buffer_length += word_length; + size_t word_byte_length = strlen(word); + size_t word_display_width = utf8_display_width(word); + memcpy(lb->_buffer + lb->_buffer_length, word, word_byte_length); + lb->_buffer_length += word_byte_length; + lb->_display_width += word_display_width; lb->_num_words++; return true; } bool linebuffer_can_fit(LineBuffer *lb, const char *word) { - size_t append_len = strlen(word); + size_t word_display_width = utf8_display_width(word); + size_t required_width = word_display_width; if (lb->_num_words > 0) { - // Account for space before the word - append_len++; + required_width++; } - return append_len <= linebuffer_space_remaining(lb); + return required_width <= linebuffer_space_remaining(lb); } size_t linebuffer_space_remaining(LineBuffer *lb) { - return lb->_buffer_capacity - lb->_buffer_length; + return lb->_line_width - lb->_display_width; } int linebuffer_write(LineBuffer *lb, FILE *fp) { diff --git a/src/utf8.c b/src/utf8.c new file mode 100644 index 0000000..07b751e --- /dev/null +++ b/src/utf8.c @@ -0,0 +1,74 @@ +#include "utf8.h" +#include <unicode/ustring.h> +#include <unicode/uchar.h> +#include <unicode/ubrk.h> +#include <string.h> +#include <stdlib.h> + +size_t utf8_display_width(const char *str) { + if (str == NULL) { + return 0; + } + + UErrorCode status = U_ZERO_ERROR; + int32_t utf8_len = (int32_t)strlen(str); + + // Convert UTF-8 to UTF-16 for ICU processing + UChar *utf16 = NULL; + int32_t utf16_len = 0; + + // First call to get required length + u_strFromUTF8(NULL, 0, &utf16_len, str, utf8_len, &status); + if (status != U_BUFFER_OVERFLOW_ERROR && U_FAILURE(status)) { + return 0; + } + + status = U_ZERO_ERROR; + utf16 = (UChar *)malloc((utf16_len + 1) * sizeof(UChar)); + if (utf16 == NULL) { + return 0; + } + + u_strFromUTF8(utf16, utf16_len + 1, NULL, str, utf8_len, &status); + if (U_FAILURE(status)) { + free(utf16); + return 0; + } + + // Use break iterator to count grapheme clusters + UBreakIterator *bi = ubrk_open(UBRK_CHARACTER, NULL, utf16, utf16_len, &status); + if (U_FAILURE(status)) { + free(utf16); + return 0; + } + + size_t width = 0; + int32_t start = ubrk_first(bi); + int32_t end = ubrk_next(bi); + + while (end != UBRK_DONE) { + // Get the first code point of this grapheme cluster + UChar32 ch; + int32_t idx = start; + U16_NEXT(utf16, idx, utf16_len, ch); + + // Determine display width based on East Asian Width property + int32_t ea_width = u_getIntPropertyValue(ch, UCHAR_EAST_ASIAN_WIDTH); + + // Full-width and Wide characters take 2 columns + if (ea_width == U_EA_FULLWIDTH || ea_width == U_EA_WIDE) { + width += 2; + } else { + // All others (Narrow, Halfwidth, Neutral, Ambiguous) take 1 column + width += 1; + } + + start = end; + end = ubrk_next(bi); + } + + ubrk_close(bi); + free(utf16); + + return width; +} diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index a0a8afa..fb75bc6 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -1,14 +1,19 @@ -set(TEST_LINEBUFFER_SOURCES test_linebuffer.c ../src/linebuffer.c) +set(TEST_LINEBUFFER_SOURCES test_linebuffer.c ../src/linebuffer.c ../src/utf8.c) set(TEST_WORD_SOURCES test_word.c ../src/word.c) +set(TEST_UTF8_SOURCES test_utf8.c ../src/utf8.c) add_executable(test_linebuffer ${TEST_LINEBUFFER_SOURCES}) add_executable(test_word ${TEST_WORD_SOURCES}) +add_executable(test_utf8 ${TEST_UTF8_SOURCES}) add_test(NAME test_linebuffer COMMAND test_linebuffer) add_test(NAME test_word COMMAND test_word) +add_test(NAME test_utf8 COMMAND test_utf8) find_package(PkgConfig REQUIRED) pkg_check_modules(CMOCKA REQUIRED cmocka) +pkg_check_modules(ICU REQUIRED IMPORTED_TARGET icu-uc icu-i18n) include_directories(${CMOCKA_INCLUDE_DIRS}) -target_link_libraries(test_linebuffer ${CMOCKA_LIBRARIES}) +target_link_libraries(test_linebuffer ${CMOCKA_LIBRARIES} PkgConfig::ICU) target_link_libraries(test_word ${CMOCKA_LIBRARIES}) +target_link_libraries(test_utf8 ${CMOCKA_LIBRARIES} PkgConfig::ICU) diff --git a/tests/justify.bats b/tests/justify.bats index 806ea81..525c5d8 100644 --- a/tests/justify.bats +++ b/tests/justify.bats @@ -94,3 +94,42 @@ EOF assert_success assert_output "" } + +@test "unicode text with accented characters is justified correctly" { + run "${JUSTIFY_BIN}" -w 30 <<EOF +café résumé naïve Zürich another word +EOF + assert_success + expected_output="café résumé naïve Zürich +another word" + assert_output "$expected_output" +} + +@test "unicode text with CJK characters is justified correctly" { + run "${JUSTIFY_BIN}" -w 20 <<EOF +Hello 世界 test this is a longer line +EOF + assert_success + expected_output="Hello 世界 test this +is a longer line" + assert_output "$expected_output" +} + +@test "mixed ASCII and unicode text wraps correctly" { + run "${JUSTIFY_BIN}" -w 25 <<EOF +This is café and résumé text that should wrap properly +EOF + assert_success + expected_output="This is café and résumé +text that should wrap +properly" + assert_output "$expected_output" +} + +@test "unicode text on last line is left-aligned" { + run "${JUSTIFY_BIN}" <<EOF +café résumé +EOF + assert_success + assert_output "café résumé" +} diff --git a/tests/test_linebuffer.c b/tests/test_linebuffer.c index ae233cc..6d84780 100644 --- a/tests/test_linebuffer.c +++ b/tests/test_linebuffer.c @@ -10,17 +10,19 @@ #include <string.h> /** - * GIVEN a specific buffer capacity + * GIVEN a specific line width * WHEN we attempt to create a LineBuffer * THEN we expect it to be created successfully * with the expected internal state */ void test_linebuffer_create(void **state) { - const int buffer_capacity = 50; - LineBuffer *lb = linebuffer_create(buffer_capacity); + const int line_width = 50; + LineBuffer *lb = linebuffer_create(line_width); assert_non_null(lb); - assert_int_equal(lb->_buffer_capacity, buffer_capacity); + assert_int_equal(lb->_line_width, line_width); + assert_int_equal(lb->_buffer_capacity, line_width * 4); assert_int_equal(lb->_buffer_length, 0); + assert_int_equal(lb->_display_width, 0); assert_int_equal(lb->_num_words, 0); linebuffer_destroy(lb); } diff --git a/tests/test_utf8.c b/tests/test_utf8.c new file mode 100644 index 0000000..623b607 --- /dev/null +++ b/tests/test_utf8.c @@ -0,0 +1,118 @@ +#include "utf8.h" +// clang-format off +#include <stdarg.h> +#include <stddef.h> +#include <setjmp.h> +#include <cmocka.h> +// clang-format on + +/** + * GIVEN an ASCII string + * WHEN we calculate the display width + * THEN we expect the width to equal the byte length + */ +void test_utf8_display_width_ascii(void **state) { + const char *input = "hello"; + size_t expected = 5; + + size_t result = utf8_display_width(input); + + assert_int_equal(result, expected); +} + +/** + * GIVEN a string with accented characters (combining marks) + * WHEN we calculate the display width + * THEN we expect the width to be less than the byte length + */ +void test_utf8_display_width_accented(void **state) { + const char *input = "café"; // é is 2 bytes in UTF-8 + size_t expected = 4; // 4 visible characters + + size_t result = utf8_display_width(input); + + assert_int_equal(result, expected); +} + +/** + * GIVEN a string with East Asian wide characters + * WHEN we calculate the display width + * THEN we expect the width to be double the character count + */ +void test_utf8_display_width_cjk(void **state) { + const char *input = "日本語"; // 3 Japanese characters + size_t expected = 6; // Each CJK character is 2 columns wide + + size_t result = utf8_display_width(input); + + assert_int_equal(result, expected); +} + +/** + * GIVEN a string with mixed ASCII and UTF-8 characters + * WHEN we calculate the display width + * THEN we expect the correct combined width + */ +void test_utf8_display_width_mixed(void **state) { + const char *input = "Hello 世界"; // "Hello " (6) + "世界" (4) + size_t expected = 10; // 6 ASCII + 2 CJK (2 columns each) + + size_t result = utf8_display_width(input); + + assert_int_equal(result, expected); +} + +/** + * GIVEN an empty string + * WHEN we calculate the display width + * THEN we expect width of 0 + */ +void test_utf8_display_width_empty(void **state) { + const char *input = ""; + size_t expected = 0; + + size_t result = utf8_display_width(input); + + assert_int_equal(result, expected); +} + +/** + * GIVEN a NULL pointer + * WHEN we calculate the display width + * THEN we expect width of 0 + */ +void test_utf8_display_width_null(void **state) { + size_t expected = 0; + + size_t result = utf8_display_width(NULL); + + assert_int_equal(result, expected); +} + +/** + * GIVEN a string with various Unicode characters + * WHEN we calculate the display width + * THEN we expect the correct width + */ +void test_utf8_display_width_various_unicode(void **state) { + const char *input = "Zürich"; // ü is 2 bytes in UTF-8 + size_t expected = 6; // 6 visible characters + + size_t result = utf8_display_width(input); + + assert_int_equal(result, expected); +} + +int main(void) { + const struct CMUnitTest tests[] = { + cmocka_unit_test(test_utf8_display_width_ascii), + cmocka_unit_test(test_utf8_display_width_accented), + cmocka_unit_test(test_utf8_display_width_cjk), + cmocka_unit_test(test_utf8_display_width_mixed), + cmocka_unit_test(test_utf8_display_width_empty), + cmocka_unit_test(test_utf8_display_width_null), + cmocka_unit_test(test_utf8_display_width_various_unicode), + }; + + return cmocka_run_group_tests(tests, NULL, NULL); +} |